From 6621730afbae84e6b6075655adc29a516cd4715a Mon Sep 17 00:00:00 2001 From: Joseph Milazzo Date: Tue, 26 Jan 2021 10:03:06 -0600 Subject: [PATCH] Refactored more archive code into the service and updated documentation now that methods are public. --- API.Tests/Services/ArchiveServiceTests.cs | 18 ++- API.Tests/Services/ScannerServiceTests.cs | 26 ++--- .../thumbnail.expected.jpg | Bin .../thumbnail.jpg | Bin .../v10 - nested folder.cbz | Bin .../v10 - nested folder.expected.jpg | Bin .../v10 - with folder.cbz | Bin .../v10 - with folder.expected.jpg | Bin .../v10.cbz | Bin .../v10.expected.jpg | Bin API/Interfaces/IArchiveService.cs | 4 +- API/Services/ArchiveService.cs | 106 ++++++++++++++++- API/Services/ScannerService.cs | 108 ++---------------- 13 files changed, 137 insertions(+), 125 deletions(-) rename API.Tests/Services/Test Data/{ScannerService => CoverImageTests}/thumbnail.expected.jpg (100%) rename API.Tests/Services/Test Data/{ScannerService => CoverImageTests}/thumbnail.jpg (100%) rename API.Tests/Services/Test Data/{ScannerService => CoverImageTests}/v10 - nested folder.cbz (100%) rename API.Tests/Services/Test Data/{ScannerService => CoverImageTests}/v10 - nested folder.expected.jpg (100%) rename API.Tests/Services/Test Data/{ScannerService => CoverImageTests}/v10 - with folder.cbz (100%) rename API.Tests/Services/Test Data/{ScannerService => CoverImageTests}/v10 - with folder.expected.jpg (100%) rename API.Tests/Services/Test Data/{ScannerService => CoverImageTests}/v10.cbz (100%) rename API.Tests/Services/Test Data/{ScannerService => CoverImageTests}/v10.expected.jpg (100%) diff --git a/API.Tests/Services/ArchiveServiceTests.cs b/API.Tests/Services/ArchiveServiceTests.cs index 25980fe1a..44453a7f5 100644 --- a/API.Tests/Services/ArchiveServiceTests.cs +++ b/API.Tests/Services/ArchiveServiceTests.cs @@ -1,6 +1,5 @@ using System.IO; using System.IO.Compression; -using API.Extensions; using API.Interfaces; using API.Services; using Microsoft.Extensions.Logging; @@ -14,9 +13,6 @@ namespace API.Tests.Services private readonly IArchiveService _archiveService; private readonly ILogger _logger = Substitute.For>(); - private readonly string _testDirectory = - Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/ArchiveService"); - public ArchiveServiceTests() { _archiveService = new ArchiveService(_logger); @@ -29,9 +25,21 @@ namespace API.Tests.Services [InlineData("file in folder_alt.zip", true)] public void ArchiveNeedsFlatteningTest(string archivePath, bool expected) { - var file = Path.Join(_testDirectory, archivePath); + var testDirectory = Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/ArchiveService"); + var file = Path.Join(testDirectory, archivePath); using ZipArchive archive = ZipFile.OpenRead(file); Assert.Equal(expected, _archiveService.ArchiveNeedsFlattening(archive)); } + + [Theory] + [InlineData("v10.cbz", "v10.expected.jpg")] + [InlineData("v10 - with folder.cbz", "v10 - with folder.expected.jpg")] + [InlineData("v10 - nested folder.cbz", "v10 - nested folder.expected.jpg")] + public void GetCoverImageTest(string inputFile, string expectedOutputFile) + { + var testDirectory = Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/CoverImageTests"); + var expectedBytes = File.ReadAllBytes(Path.Join(testDirectory, expectedOutputFile)); + Assert.Equal(expectedBytes, _archiveService.GetCoverImage(Path.Join(testDirectory, inputFile))); + } } } \ No newline at end of file diff --git a/API.Tests/Services/ScannerServiceTests.cs b/API.Tests/Services/ScannerServiceTests.cs index 609caf94d..efde36e52 100644 --- a/API.Tests/Services/ScannerServiceTests.cs +++ b/API.Tests/Services/ScannerServiceTests.cs @@ -9,23 +9,15 @@ namespace API.Tests.Services { public class ScannerServiceTests { - private readonly ScannerService _scannerService; - private readonly ILogger _logger = Substitute.For>(); - private readonly IUnitOfWork _unitOfWork = Substitute.For(); - private readonly string _testDirectory = Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/ScannerService"); - public ScannerServiceTests() - { - _scannerService = new ScannerService(_unitOfWork, _logger); - } + // private readonly ScannerService _scannerService; + // private readonly ILogger _logger = Substitute.For>(); + // private readonly IUnitOfWork _unitOfWork = Substitute.For(); + // private readonly string _testDirectory = Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/ScannerService"); + // public ScannerServiceTests() + // { + // _scannerService = new ScannerService(_unitOfWork, _logger); + // } - [Theory] - [InlineData("v10.cbz", "v10.expected.jpg")] - [InlineData("v10 - with folder.cbz", "v10 - with folder.expected.jpg")] - [InlineData("v10 - nested folder.cbz", "v10 - nested folder.expected.jpg")] - public void GetCoverImageTest(string inputFile, string expectedOutputFile) - { - var expectedBytes = File.ReadAllBytes(Path.Join(_testDirectory, expectedOutputFile)); - Assert.Equal(expectedBytes, _scannerService.GetCoverImage(Path.Join(_testDirectory, inputFile))); - } + } } \ No newline at end of file diff --git a/API.Tests/Services/Test Data/ScannerService/thumbnail.expected.jpg b/API.Tests/Services/Test Data/CoverImageTests/thumbnail.expected.jpg similarity index 100% rename from API.Tests/Services/Test Data/ScannerService/thumbnail.expected.jpg rename to API.Tests/Services/Test Data/CoverImageTests/thumbnail.expected.jpg diff --git a/API.Tests/Services/Test Data/ScannerService/thumbnail.jpg b/API.Tests/Services/Test Data/CoverImageTests/thumbnail.jpg similarity index 100% rename from API.Tests/Services/Test Data/ScannerService/thumbnail.jpg rename to API.Tests/Services/Test Data/CoverImageTests/thumbnail.jpg diff --git a/API.Tests/Services/Test Data/ScannerService/v10 - nested folder.cbz b/API.Tests/Services/Test Data/CoverImageTests/v10 - nested folder.cbz similarity index 100% rename from API.Tests/Services/Test Data/ScannerService/v10 - nested folder.cbz rename to API.Tests/Services/Test Data/CoverImageTests/v10 - nested folder.cbz diff --git a/API.Tests/Services/Test Data/ScannerService/v10 - nested folder.expected.jpg b/API.Tests/Services/Test Data/CoverImageTests/v10 - nested folder.expected.jpg similarity index 100% rename from API.Tests/Services/Test Data/ScannerService/v10 - nested folder.expected.jpg rename to API.Tests/Services/Test Data/CoverImageTests/v10 - nested folder.expected.jpg diff --git a/API.Tests/Services/Test Data/ScannerService/v10 - with folder.cbz b/API.Tests/Services/Test Data/CoverImageTests/v10 - with folder.cbz similarity index 100% rename from API.Tests/Services/Test Data/ScannerService/v10 - with folder.cbz rename to API.Tests/Services/Test Data/CoverImageTests/v10 - with folder.cbz diff --git a/API.Tests/Services/Test Data/ScannerService/v10 - with folder.expected.jpg b/API.Tests/Services/Test Data/CoverImageTests/v10 - with folder.expected.jpg similarity index 100% rename from API.Tests/Services/Test Data/ScannerService/v10 - with folder.expected.jpg rename to API.Tests/Services/Test Data/CoverImageTests/v10 - with folder.expected.jpg diff --git a/API.Tests/Services/Test Data/ScannerService/v10.cbz b/API.Tests/Services/Test Data/CoverImageTests/v10.cbz similarity index 100% rename from API.Tests/Services/Test Data/ScannerService/v10.cbz rename to API.Tests/Services/Test Data/CoverImageTests/v10.cbz diff --git a/API.Tests/Services/Test Data/ScannerService/v10.expected.jpg b/API.Tests/Services/Test Data/CoverImageTests/v10.expected.jpg similarity index 100% rename from API.Tests/Services/Test Data/ScannerService/v10.expected.jpg rename to API.Tests/Services/Test Data/CoverImageTests/v10.expected.jpg diff --git a/API/Interfaces/IArchiveService.cs b/API/Interfaces/IArchiveService.cs index 5420c3a74..07d6b6206 100644 --- a/API/Interfaces/IArchiveService.cs +++ b/API/Interfaces/IArchiveService.cs @@ -5,6 +5,8 @@ namespace API.Interfaces public interface IArchiveService { bool ArchiveNeedsFlattening(ZipArchive archive); - public void ExtractArchive(string archivePath, string extractPath); + void ExtractArchive(string archivePath, string extractPath); + int GetNumberOfPagesFromArchive(string archivePath); + byte[] GetCoverImage(string filepath, bool createThumbnail = false); } } \ No newline at end of file diff --git a/API/Services/ArchiveService.cs b/API/Services/ArchiveService.cs index 22d470e29..6ff593f59 100644 --- a/API/Services/ArchiveService.cs +++ b/API/Services/ArchiveService.cs @@ -1,10 +1,12 @@ -using System.Diagnostics; +using System; +using System.Diagnostics; using System.IO; using System.IO.Compression; using System.Linq; using API.Extensions; using API.Interfaces; using Microsoft.Extensions.Logging; +using NetVips; namespace API.Services { @@ -19,17 +21,111 @@ namespace API.Services { _logger = logger; } + + public int GetNumberOfPagesFromArchive(string archivePath) + { + if (!File.Exists(archivePath) || !Parser.Parser.IsArchive(archivePath)) + { + _logger.LogError($"Archive {archivePath} could not be found."); + return 0; + } + + _logger.LogDebug($"Getting Page numbers from {archivePath}"); + try + { + using ZipArchive archive = ZipFile.OpenRead(archivePath); // ZIPFILE + return archive.Entries.Count(e => Parser.Parser.IsImage(e.FullName)); + } + catch (Exception ex) + { + _logger.LogError(ex, "There was an exception when reading archive stream."); + return 0; + } + + + } + + /// + /// Generates byte array of cover image. + /// Given a path to a compressed file (zip, rar, cbz, cbr, etc), will ensure the first image is returned unless + /// a folder.extension exists in the root directory of the compressed file. + /// + /// + /// Create a smaller variant of file extracted from archive. Archive images are usually 1MB each. + /// + public byte[] GetCoverImage(string filepath, bool createThumbnail = false) + { + try + { + if (string.IsNullOrEmpty(filepath) || !File.Exists(filepath) || !Parser.Parser.IsArchive(filepath)) return Array.Empty(); + + _logger.LogDebug($"Extracting Cover image from {filepath}"); + using ZipArchive archive = ZipFile.OpenRead(filepath); + if (!archive.HasFiles()) return Array.Empty(); + + var folder = archive.Entries.SingleOrDefault(x => Path.GetFileNameWithoutExtension(x.Name).ToLower() == "folder"); + var entries = archive.Entries.Where(x => Path.HasExtension(x.FullName) && Parser.Parser.IsImage(x.FullName)).OrderBy(x => x.FullName).ToList(); + ZipArchiveEntry entry; + + if (folder != null) + { + entry = folder; + } else if (!entries.Any()) + { + return Array.Empty(); + } + else + { + entry = entries[0]; + } + + + if (createThumbnail) + { + try + { + using var stream = entry.Open(); + var thumbnail = Image.ThumbnailStream(stream, 320); + return thumbnail.WriteToBuffer(".jpg"); + } + catch (Exception ex) + { + _logger.LogError(ex, "There was a critical error and prevented thumbnail generation."); + } + } + + return ExtractEntryToImage(entry); + } + catch (Exception ex) + { + _logger.LogError(ex, "There was an exception when reading archive stream."); + return Array.Empty(); + } + } + + private static byte[] ExtractEntryToImage(ZipArchiveEntry entry) + { + using var stream = entry.Open(); + using var ms = new MemoryStream(); + stream.CopyTo(ms); + var data = ms.ToArray(); + + return data; + } + + /// + /// Given an archive stream, will assess whether directory needs to be flattened so that the extracted archive files are directly + /// under extract path and not nested in subfolders. See Flatten method. + /// + /// An opened archive stream + /// public bool ArchiveNeedsFlattening(ZipArchive archive) { // Sometimes ZipArchive will list the directory and others it will just keep it in the FullName return archive.Entries.Count > 0 && !Path.HasExtension(archive.Entries.ElementAt(0).FullName) || archive.Entries.Any(e => e.FullName.Contains(Path.AltDirectorySeparatorChar)); - - // return archive.Entries.Count > 0 && - // archive.Entries.Any(e => e.FullName.Contains(Path.AltDirectorySeparatorChar)); - //return archive.Entries.Count > 0 && !Path.HasExtension(archive.Entries.ElementAt(0).FullName); } /// diff --git a/API/Services/ScannerService.cs b/API/Services/ScannerService.cs index b697d5880..1b4516c94 100644 --- a/API/Services/ScannerService.cs +++ b/API/Services/ScannerService.cs @@ -20,12 +20,14 @@ namespace API.Services { private readonly IUnitOfWork _unitOfWork; private readonly ILogger _logger; + private readonly IArchiveService _archiveService; private ConcurrentDictionary> _scannedSeries; - public ScannerService(IUnitOfWork unitOfWork, ILogger logger) + public ScannerService(IUnitOfWork unitOfWork, ILogger logger, IArchiveService archiveService) { _unitOfWork = unitOfWork; _logger = logger; + _archiveService = archiveService; } public void ScanLibraries() @@ -218,7 +220,7 @@ namespace API.Services FilePath = info.FullFilePath, Chapter = chapter, Format = info.Format, - NumberOfPages = info.Format == MangaFormat.Archive ? GetNumberOfPagesFromArchive(info.FullFilePath): 1 + NumberOfPages = info.Format == MangaFormat.Archive ? _archiveService.GetNumberOfPagesFromArchive(info.FullFilePath): 1 }; } @@ -250,7 +252,7 @@ namespace API.Services { existingFile.Chapter = MinimumNumberFromRange(info.Chapters); existingFile.Format = info.Format; - existingFile.NumberOfPages = GetNumberOfPagesFromArchive(info.FullFilePath); + existingFile.NumberOfPages = _archiveService.GetNumberOfPagesFromArchive(info.FullFilePath); } else { @@ -298,7 +300,7 @@ namespace API.Services if (forceUpdate || volume.CoverImage == null || !volume.Files.Any()) { var firstFile = volume.Files.OrderBy(x => x.Chapter).FirstOrDefault(); - if (firstFile != null) volume.CoverImage = GetCoverImage(firstFile.FilePath, true); // ZIPFILE + if (firstFile != null) volume.CoverImage = _archiveService.GetCoverImage(firstFile.FilePath, true); } volume.Pages = volume.Files.Sum(x => x.NumberOfPages); @@ -307,105 +309,17 @@ namespace API.Services return volumes; } - + public void ScanSeries(int libraryId, int seriesId) - { - throw new NotImplementedException(); - } - - private int GetNumberOfPagesFromArchive(string archivePath) - { - if (!File.Exists(archivePath) || !Parser.Parser.IsArchive(archivePath)) - { - _logger.LogError($"Archive {archivePath} could not be found."); - return 0; - } - - _logger.LogDebug($"Getting Page numbers from {archivePath}"); - - try - { - using ZipArchive archive = ZipFile.OpenRead(archivePath); // ZIPFILE - return archive.Entries.Count(e => Parser.Parser.IsImage(e.FullName)); - } - catch (Exception ex) - { - _logger.LogError(ex, "There was an exception when reading archive stream."); - return 0; - } - - - } - - /// - /// Generates byte array of cover image. - /// Given a path to a compressed file (zip, rar, cbz, cbr, etc), will ensure the first image is returned unless - /// a folder.extension exists in the root directory of the compressed file. - /// - /// - /// Create a smaller variant of file extracted from archive. Archive images are usually 1MB each. - /// - public byte[] GetCoverImage(string filepath, bool createThumbnail = false) - { - try - { - if (string.IsNullOrEmpty(filepath) || !File.Exists(filepath) || !Parser.Parser.IsArchive(filepath)) return Array.Empty(); - - _logger.LogDebug($"Extracting Cover image from {filepath}"); - using ZipArchive archive = ZipFile.OpenRead(filepath); - if (!archive.HasFiles()) return Array.Empty(); - - var folder = archive.Entries.SingleOrDefault(x => Path.GetFileNameWithoutExtension(x.Name).ToLower() == "folder"); - var entries = archive.Entries.Where(x => Path.HasExtension(x.FullName) && Parser.Parser.IsImage(x.FullName)).OrderBy(x => x.FullName).ToList(); - ZipArchiveEntry entry; - - if (folder != null) - { - entry = folder; - } else if (!entries.Any()) - { - return Array.Empty(); - } - else - { - entry = entries[0]; - } + { + throw new NotImplementedException(); + } - if (createThumbnail) - { - try - { - using var stream = entry.Open(); - var thumbnail = Image.ThumbnailStream(stream, 320); - return thumbnail.WriteToBuffer(".jpg"); - } - catch (Exception ex) - { - _logger.LogError(ex, "There was a critical error and prevented thumbnail generation."); - } - } - - return ExtractEntryToImage(entry); - } - catch (Exception ex) - { - _logger.LogError(ex, "There was an exception when reading archive stream."); - return Array.Empty(); - } - } - - private static byte[] ExtractEntryToImage(ZipArchiveEntry entry) - { - using var stream = entry.Open(); - using var ms = new MemoryStream(); - stream.CopyTo(ms); - var data = ms.ToArray(); - return data; - } + } } \ No newline at end of file