diff --git a/API.Tests/API.Tests.csproj b/API.Tests/API.Tests.csproj index b70443941..95af52570 100644 --- a/API.Tests/API.Tests.csproj +++ b/API.Tests/API.Tests.csproj @@ -25,7 +25,7 @@ - + diff --git a/API.Tests/ParserTest.cs b/API.Tests/ParserTest.cs index 38d512077..de1f77936 100644 --- a/API.Tests/ParserTest.cs +++ b/API.Tests/ParserTest.cs @@ -109,6 +109,7 @@ namespace API.Tests [InlineData("Yumekui-Merry_DKThias_Chapter21.zip", "21")] [InlineData("Yumekui_Merry_v01_c01[Bakayarou-Kuu].rar", "1")] [InlineData("Yumekui-Merry_DKThias_Chapter11v2.zip", "11")] + [InlineData("Beelzebub_53[KSH].zip", "53")] //[InlineData("[Tempus Edax Rerum] Epigraph of the Closed Curve - Chapter 6.zip", "6")] public void ParseChaptersTest(string filename, string expected) { diff --git a/API.Tests/Services/ArchiveServiceTests.cs b/API.Tests/Services/ArchiveServiceTests.cs new file mode 100644 index 000000000..25980fe1a --- /dev/null +++ b/API.Tests/Services/ArchiveServiceTests.cs @@ -0,0 +1,37 @@ +using System.IO; +using System.IO.Compression; +using API.Extensions; +using API.Interfaces; +using API.Services; +using Microsoft.Extensions.Logging; +using NSubstitute; +using Xunit; + +namespace API.Tests.Services +{ + public class ArchiveServiceTests + { + private readonly IArchiveService _archiveService; + private readonly ILogger _logger = Substitute.For>(); + + private readonly string _testDirectory = + Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/ArchiveService"); + + public ArchiveServiceTests() + { + _archiveService = new ArchiveService(_logger); + } + + [Theory] + [InlineData("flat file.zip", false)] + [InlineData("file in folder in folder.zip", true)] + [InlineData("file in folder.zip", true)] + [InlineData("file in folder_alt.zip", true)] + public void ArchiveNeedsFlatteningTest(string archivePath, bool expected) + { + var file = Path.Join(_testDirectory, archivePath); + using ZipArchive archive = ZipFile.OpenRead(file); + Assert.Equal(expected, _archiveService.ArchiveNeedsFlattening(archive)); + } + } +} \ No newline at end of file diff --git a/API.Tests/Services/ScannerServiceTests.cs b/API.Tests/Services/ScannerServiceTests.cs index 78591b212..609caf94d 100644 --- a/API.Tests/Services/ScannerServiceTests.cs +++ b/API.Tests/Services/ScannerServiceTests.cs @@ -12,6 +12,7 @@ namespace API.Tests.Services private readonly ScannerService _scannerService; private readonly ILogger _logger = Substitute.For>(); private readonly IUnitOfWork _unitOfWork = Substitute.For(); + private readonly string _testDirectory = Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/ScannerService"); public ScannerServiceTests() { _scannerService = new ScannerService(_unitOfWork, _logger); @@ -23,9 +24,8 @@ namespace API.Tests.Services [InlineData("v10 - nested folder.cbz", "v10 - nested folder.expected.jpg")] public void GetCoverImageTest(string inputFile, string expectedOutputFile) { - var testDirectory = Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/ImageProvider"); - var expectedBytes = File.ReadAllBytes(Path.Join(testDirectory, expectedOutputFile)); - Assert.Equal(expectedBytes, _scannerService.GetCoverImage(Path.Join(testDirectory, inputFile))); + var expectedBytes = File.ReadAllBytes(Path.Join(_testDirectory, expectedOutputFile)); + Assert.Equal(expectedBytes, _scannerService.GetCoverImage(Path.Join(_testDirectory, inputFile))); } } } \ No newline at end of file diff --git a/API.Tests/Services/Test Data/ArchiveService/LICENSE.md b/API.Tests/Services/Test Data/ArchiveService/LICENSE.md new file mode 100644 index 000000000..580f5f351 --- /dev/null +++ b/API.Tests/Services/Test Data/ArchiveService/LICENSE.md @@ -0,0 +1,2 @@ +Files in this test are all royalty free and can be found here: +https://www.pexels.com/photo/snow-wood-light-art-6551949/ \ No newline at end of file diff --git a/API.Tests/Services/Test Data/ArchiveService/file in folder in folder.zip b/API.Tests/Services/Test Data/ArchiveService/file in folder in folder.zip new file mode 100644 index 000000000..7598e0fa3 Binary files /dev/null and b/API.Tests/Services/Test Data/ArchiveService/file in folder in folder.zip differ diff --git a/API.Tests/Services/Test Data/ArchiveService/file in folder.zip b/API.Tests/Services/Test Data/ArchiveService/file in folder.zip new file mode 100644 index 000000000..b13a312b8 Binary files /dev/null and b/API.Tests/Services/Test Data/ArchiveService/file in folder.zip differ diff --git a/API.Tests/Services/Test Data/ArchiveService/file in folder_alt.zip b/API.Tests/Services/Test Data/ArchiveService/file in folder_alt.zip new file mode 100644 index 000000000..6607659b8 Binary files /dev/null and b/API.Tests/Services/Test Data/ArchiveService/file in folder_alt.zip differ diff --git a/API.Tests/Services/Test Data/ArchiveService/flat file.zip b/API.Tests/Services/Test Data/ArchiveService/flat file.zip new file mode 100644 index 000000000..344d49d64 Binary files /dev/null and b/API.Tests/Services/Test Data/ArchiveService/flat file.zip differ diff --git a/API.Tests/Services/Test Data/ImageProvider/thumbnail.expected.jpg b/API.Tests/Services/Test Data/ScannerService/thumbnail.expected.jpg similarity index 100% rename from API.Tests/Services/Test Data/ImageProvider/thumbnail.expected.jpg rename to API.Tests/Services/Test Data/ScannerService/thumbnail.expected.jpg diff --git a/API.Tests/Services/Test Data/ImageProvider/thumbnail.jpg b/API.Tests/Services/Test Data/ScannerService/thumbnail.jpg similarity index 100% rename from API.Tests/Services/Test Data/ImageProvider/thumbnail.jpg rename to API.Tests/Services/Test Data/ScannerService/thumbnail.jpg diff --git a/API.Tests/Services/Test Data/ImageProvider/v10 - nested folder.cbz b/API.Tests/Services/Test Data/ScannerService/v10 - nested folder.cbz similarity index 100% rename from API.Tests/Services/Test Data/ImageProvider/v10 - nested folder.cbz rename to API.Tests/Services/Test Data/ScannerService/v10 - nested folder.cbz diff --git a/API.Tests/Services/Test Data/ImageProvider/v10 - nested folder.expected.jpg b/API.Tests/Services/Test Data/ScannerService/v10 - nested folder.expected.jpg similarity index 100% rename from API.Tests/Services/Test Data/ImageProvider/v10 - nested folder.expected.jpg rename to API.Tests/Services/Test Data/ScannerService/v10 - nested folder.expected.jpg diff --git a/API.Tests/Services/Test Data/ImageProvider/v10 - with folder.cbz b/API.Tests/Services/Test Data/ScannerService/v10 - with folder.cbz similarity index 100% rename from API.Tests/Services/Test Data/ImageProvider/v10 - with folder.cbz rename to API.Tests/Services/Test Data/ScannerService/v10 - with folder.cbz diff --git a/API.Tests/Services/Test Data/ImageProvider/v10 - with folder.expected.jpg b/API.Tests/Services/Test Data/ScannerService/v10 - with folder.expected.jpg similarity index 100% rename from API.Tests/Services/Test Data/ImageProvider/v10 - with folder.expected.jpg rename to API.Tests/Services/Test Data/ScannerService/v10 - with folder.expected.jpg diff --git a/API.Tests/Services/Test Data/ImageProvider/v10.cbz b/API.Tests/Services/Test Data/ScannerService/v10.cbz similarity index 100% rename from API.Tests/Services/Test Data/ImageProvider/v10.cbz rename to API.Tests/Services/Test Data/ScannerService/v10.cbz diff --git a/API.Tests/Services/Test Data/ImageProvider/v10.expected.jpg b/API.Tests/Services/Test Data/ScannerService/v10.expected.jpg similarity index 100% rename from API.Tests/Services/Test Data/ImageProvider/v10.expected.jpg rename to API.Tests/Services/Test Data/ScannerService/v10.expected.jpg diff --git a/API/Data/SeriesRepository.cs b/API/Data/SeriesRepository.cs index 77e9b579f..27dc38e43 100644 --- a/API/Data/SeriesRepository.cs +++ b/API/Data/SeriesRepository.cs @@ -129,10 +129,12 @@ namespace API.Data .Include(vol => vol.Files) .ProjectTo(_mapper.ConfigurationProvider) .SingleAsync(vol => vol.Id == volumeId); - + var volumeList = new List() {volume}; await AddVolumeModifiers(userId, volumeList); + volumeList[0].Files = volumeList[0].Files.OrderBy(f => f.Chapter).ToList(); + return volumeList[0]; } diff --git a/API/Interfaces/IArchiveService.cs b/API/Interfaces/IArchiveService.cs new file mode 100644 index 000000000..5420c3a74 --- /dev/null +++ b/API/Interfaces/IArchiveService.cs @@ -0,0 +1,10 @@ +using System.IO.Compression; + +namespace API.Interfaces +{ + public interface IArchiveService + { + bool ArchiveNeedsFlattening(ZipArchive archive); + public void ExtractArchive(string archivePath, string extractPath); + } +} \ No newline at end of file diff --git a/API/Parser/Parser.cs b/API/Parser/Parser.cs index 111ae0fbd..431d56330 100644 --- a/API/Parser/Parser.cs +++ b/API/Parser/Parser.cs @@ -125,7 +125,7 @@ namespace API.Parser RegexOptions.IgnoreCase | RegexOptions.Compiled), // Beelzebub_01_[Noodles].zip new Regex( - @"^((?!v|vo|vol|Volume).)*( |_)(?\.?\d+)( |_)", + @"^((?!v|vo|vol|Volume).)*( |_)(?\.?\d+)( |_|\[|\()", RegexOptions.IgnoreCase | RegexOptions.Compiled), // Yumekui-Merry_DKThias_Chapter21.zip new Regex( diff --git a/API/Services/ArchiveService.cs b/API/Services/ArchiveService.cs new file mode 100644 index 000000000..22d470e29 --- /dev/null +++ b/API/Services/ArchiveService.cs @@ -0,0 +1,74 @@ +using System.Diagnostics; +using System.IO; +using System.IO.Compression; +using System.Linq; +using API.Extensions; +using API.Interfaces; +using Microsoft.Extensions.Logging; + +namespace API.Services +{ + /// + /// Responsible for manipulating Archive files. Used by almost exclusively. + /// + public class ArchiveService : IArchiveService + { + private readonly ILogger _logger; + + public ArchiveService(ILogger logger) + { + _logger = logger; + } + + public bool ArchiveNeedsFlattening(ZipArchive archive) + { + // Sometimes ZipArchive will list the directory and others it will just keep it in the FullName + return archive.Entries.Count > 0 && + !Path.HasExtension(archive.Entries.ElementAt(0).FullName) || + archive.Entries.Any(e => e.FullName.Contains(Path.AltDirectorySeparatorChar)); + + // return archive.Entries.Count > 0 && + // archive.Entries.Any(e => e.FullName.Contains(Path.AltDirectorySeparatorChar)); + //return archive.Entries.Count > 0 && !Path.HasExtension(archive.Entries.ElementAt(0).FullName); + } + + /// + /// Extracts an archive to a temp cache directory. Returns path to new directory. If temp cache directory already exists, + /// will return that without performing an extraction. Returns empty string if there are any invalidations which would + /// prevent operations to perform correctly (missing archivePath file, empty archive, etc). + /// + /// A valid file to an archive file. + /// Path to extract to + /// + public void ExtractArchive(string archivePath, string extractPath) + { + if (!File.Exists(archivePath) || !Parser.Parser.IsArchive(archivePath)) + { + _logger.LogError($"Archive {archivePath} could not be found."); + return; + } + + if (Directory.Exists(extractPath)) + { + _logger.LogDebug($"Archive {archivePath} has already been extracted. Returning existing folder."); + return; + } + + Stopwatch sw = Stopwatch.StartNew(); + using ZipArchive archive = ZipFile.OpenRead(archivePath); + var needsFlattening = ArchiveNeedsFlattening(archive); + if (!archive.HasFiles() && !needsFlattening) return; + + archive.ExtractToDirectory(extractPath); + _logger.LogDebug($"Extracted archive to {extractPath} in {sw.ElapsedMilliseconds} milliseconds."); + + if (needsFlattening) + { + sw = Stopwatch.StartNew(); + _logger.LogInformation("Extracted archive is nested in root folder, flattening..."); + new DirectoryInfo(extractPath).Flatten(); + _logger.LogInformation($"Flattened in {sw.ElapsedMilliseconds} milliseconds"); + } + } + } +} \ No newline at end of file diff --git a/API/Services/CacheService.cs b/API/Services/CacheService.cs index 778310124..66e8c9e2c 100644 --- a/API/Services/CacheService.cs +++ b/API/Services/CacheService.cs @@ -1,7 +1,6 @@ using System; -using System.Diagnostics; +using System.Collections.Generic; using System.IO; -using System.IO.Compression; using System.Linq; using System.Threading.Tasks; using API.Comparators; @@ -17,14 +16,16 @@ namespace API.Services private readonly IDirectoryService _directoryService; private readonly ILogger _logger; private readonly IUnitOfWork _unitOfWork; + private readonly IArchiveService _archiveService; private readonly NumericComparer _numericComparer; public static readonly string CacheDirectory = Path.GetFullPath(Path.Join(Directory.GetCurrentDirectory(), "../cache/")); - public CacheService(IDirectoryService directoryService, ILogger logger, IUnitOfWork unitOfWork) + public CacheService(IDirectoryService directoryService, ILogger logger, IUnitOfWork unitOfWork, IArchiveService archiveService) { _directoryService = directoryService; _logger = logger; _unitOfWork = unitOfWork; + _archiveService = archiveService; _numericComparer = new NumericComparer(); } @@ -46,7 +47,7 @@ namespace API.Services foreach (var file in volume.Files) { var extractPath = GetVolumeCachePath(volumeId, file); - ExtractArchive(file.FilePath, extractPath); + _archiveService.ExtractArchive(file.FilePath, extractPath); } return volume; @@ -92,44 +93,7 @@ namespace API.Services _logger.LogInformation("Cache directory purged"); } - /// - /// Extracts an archive to a temp cache directory. Returns path to new directory. If temp cache directory already exists, - /// will return that without performing an extraction. Returns empty string if there are any invalidations which would - /// prevent operations to perform correctly (missing archivePath file, empty archive, etc). - /// - /// A valid file to an archive file. - /// Path to extract to - /// - private void ExtractArchive(string archivePath, string extractPath) - { - if (!File.Exists(archivePath) || !Parser.Parser.IsArchive(archivePath)) - { - _logger.LogError($"Archive {archivePath} could not be found."); - return; - } - - if (Directory.Exists(extractPath)) - { - _logger.LogDebug($"Archive {archivePath} has already been extracted. Returning existing folder."); - return; - } - - Stopwatch sw = Stopwatch.StartNew(); - using ZipArchive archive = ZipFile.OpenRead(archivePath); - var needsFlattening = archive.Entries.Count > 0 && !Path.HasExtension(archive.Entries.ElementAt(0).FullName); - if (!archive.HasFiles() && !needsFlattening) return; - - archive.ExtractToDirectory(extractPath); - _logger.LogDebug($"Extracted archive to {extractPath} in {sw.ElapsedMilliseconds} milliseconds."); - - if (needsFlattening) - { - sw = Stopwatch.StartNew(); - _logger.LogInformation("Extracted archive is nested in root folder, flattening..."); - new DirectoryInfo(extractPath).Flatten(); - _logger.LogInformation($"Flattened in {sw.ElapsedMilliseconds} milliseconds"); - } - } + private string GetVolumeCachePath(int volumeId, MangaFile file) @@ -142,11 +106,18 @@ namespace API.Services return extractPath; } + private IEnumerable GetOrderedChapters(ICollection files) + { + return files.OrderBy(f => f.Chapter).Where(f => f.Chapter != 0); + } + public string GetCachedPagePath(Volume volume, int page) { // Calculate what chapter the page belongs to var pagesSoFar = 0; - foreach (var mangaFile in volume.Files.OrderBy(f => f.Chapter)) + // Do not allow chapters with 0, as those are specials and break ordering for reading. + var orderedChapters = GetOrderedChapters(volume.Files); + foreach (var mangaFile in orderedChapters) { if (page + 1 < (mangaFile.NumberOfPages + pagesSoFar)) {