Cover Image - First and tests (#170)

* Changed how natural sort works to cover more cases

* Changed the name of CoverImage regex for Parser and added more cases.

* Changed how we get result from Task.Run()

* Defer execution of a loop till we really need it and added another TODO for later this iteration.

* Big refactor to cover image code to unify between IOCompression and SharpCompress. Both use methods to find the correct file. This results in one extra loop through entries, but simplifies code signficantly.

In addition, new unit tests for the methods that actually do the logic on choosing cover file and first file.

* Removed dead code

* Added missing doc
This commit is contained in:
Joseph Milazzo 2021-04-11 18:15:12 -05:00 committed by GitHub
parent 9e5bcb8501
commit 6ba00477e7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 169 additions and 99 deletions

View File

@ -7,6 +7,8 @@ namespace API.Tests.Comparers
{
public class NaturalSortComparerTest
{
private readonly NaturalSortComparer _nc = new NaturalSortComparer();
[Theory]
[InlineData(
new[] {"x1.jpg", "x10.jpg", "x3.jpg", "x4.jpg", "x11.jpg"},
@ -20,10 +22,25 @@ namespace API.Tests.Comparers
new[] {"[SCX-Scans]_Vandread_v02_Act02.zip", "[SCX-Scans]_Vandread_v02_Act01.zip",},
new[] {"[SCX-Scans]_Vandread_v02_Act01.zip", "[SCX-Scans]_Vandread_v02_Act02.zip",}
)]
[InlineData(
new[] {"Frogman v01 001.jpg", "Frogman v01 ch01 p00 Credits.jpg",},
new[] {"Frogman v01 001.jpg", "Frogman v01 ch01 p00 Credits.jpg",}
)]
[InlineData(
new[] {"001.jpg", "10.jpg",},
new[] {"001.jpg", "10.jpg",}
)]
[InlineData(
new[] {"10/001.jpg", "10.jpg",},
new[] {"10.jpg", "10/001.jpg",}
)]
[InlineData(
new[] {"Batman - Black white vol 1 #04.cbr", "Batman - Black white vol 1 #03.cbr", "Batman - Black white vol 1 #01.cbr", "Batman - Black white vol 1 #02.cbr"},
new[] {"Batman - Black white vol 1 #01.cbr", "Batman - Black white vol 1 #02.cbr", "Batman - Black white vol 1 #03.cbr", "Batman - Black white vol 1 #04.cbr"}
)]
public void TestNaturalSortComparer(string[] input, string[] expected)
{
NaturalSortComparer nc = new NaturalSortComparer();
Array.Sort(input, nc);
Array.Sort(input, _nc);
var i = 0;
foreach (var s in input)
@ -51,10 +68,25 @@ namespace API.Tests.Comparers
new[] {"[SCX-Scans]_Vandread_v02_Act02.zip", "[SCX-Scans]_Vandread_v02_Act01.zip","[SCX-Scans]_Vandread_v02_Act07.zip",},
new[] {"[SCX-Scans]_Vandread_v02_Act01.zip", "[SCX-Scans]_Vandread_v02_Act02.zip","[SCX-Scans]_Vandread_v02_Act07.zip",}
)]
[InlineData(
new[] {"Frogman v01 001.jpg", "Frogman v01 ch01 p00 Credits.jpg",},
new[] {"Frogman v01 001.jpg", "Frogman v01 ch01 p00 Credits.jpg",}
)]
[InlineData(
new[] {"001.jpg", "10.jpg",},
new[] {"001.jpg", "10.jpg",}
)]
[InlineData(
new[] {"10/001.jpg", "10.jpg",},
new[] {"10.jpg", "10/001.jpg",}
)]
[InlineData(
new[] {"Batman - Black white vol 1 #04.cbr", "Batman - Black white vol 1 #03.cbr", "Batman - Black white vol 1 #01.cbr", "Batman - Black white vol 1 #02.cbr"},
new[] {"Batman - Black white vol 1 #01.cbr", "Batman - Black white vol 1 #02.cbr", "Batman - Black white vol 1 #03.cbr", "Batman - Black white vol 1 #04.cbr"}
)]
public void TestNaturalSortComparerLinq(string[] input, string[] expected)
{
NaturalSortComparer nc = new NaturalSortComparer();
var output = input.OrderBy(c => c, nc);
var output = input.OrderBy(c => c, _nc);
var i = 0;
foreach (var s in output)

View File

@ -366,6 +366,9 @@ namespace API.Tests
[InlineData("DearS_v01_cover.jpg", true)]
[InlineData("DearS_v01_covers.jpg", false)]
[InlineData("!cover.jpg", true)]
[InlineData("cover.jpg", true)]
[InlineData("cover.png", true)]
[InlineData("ch1/cover.png", true)]
public void IsCoverImageTest(string inputPath, bool expected)
{
Assert.Equal(expected, IsCoverImage(inputPath));

View File

@ -1,4 +1,5 @@
using System.Diagnostics;
using System.Collections.ObjectModel;
using System.Diagnostics;
using System.IO;
using System.IO.Compression;
using API.Archive;
@ -6,6 +7,7 @@ using API.Interfaces.Services;
using API.Services;
using Microsoft.Extensions.Logging;
using NSubstitute;
using NSubstitute.Extensions;
using Xunit;
using Xunit.Abstractions;
@ -14,7 +16,7 @@ namespace API.Tests.Services
public class ArchiveServiceTests
{
private readonly ITestOutputHelper _testOutputHelper;
private readonly IArchiveService _archiveService;
private readonly ArchiveService _archiveService;
private readonly ILogger<ArchiveService> _logger = Substitute.For<ILogger<ArchiveService>>();
public ArchiveServiceTests(ITestOutputHelper testOutputHelper)
@ -113,6 +115,34 @@ namespace API.Tests.Services
DirectoryService.ClearAndDeleteDirectory(extractDirectory);
}
[Theory]
[InlineData(new [] {"folder.jpg"}, "folder.jpg")]
[InlineData(new [] {"vol1/"}, "")]
[InlineData(new [] {"folder.jpg", "vol1/folder.jpg"}, "folder.jpg")]
[InlineData(new [] {"cover.jpg", "vol1/folder.jpg"}, "cover.jpg")]
[InlineData(new [] {"__MACOSX/cover.jpg", "vol1/page 01.jpg"}, "")]
[InlineData(new [] {"Akame ga KILL! ZERO - c055 (v10) - p000 [Digital] [LuCaZ].jpg", "Akame ga KILL! ZERO - c055 (v10) - p000 [Digital] [LuCaZ].jpg", "Akame ga KILL! ZERO - c060 (v10) - p200 [Digital] [LuCaZ].jpg", "folder.jpg"}, "folder.jpg")]
public void FindFolderEntry(string[] files, string expected)
{
var foundFile = _archiveService.FindFolderEntry(files);
Assert.Equal(expected, string.IsNullOrEmpty(foundFile) ? "" : foundFile);
}
[Theory]
[InlineData(new [] {"folder.jpg"}, "folder.jpg")]
[InlineData(new [] {"vol1/"}, "")]
[InlineData(new [] {"folder.jpg", "vol1/folder.jpg"}, "folder.jpg")]
[InlineData(new [] {"cover.jpg", "vol1/folder.jpg"}, "cover.jpg")]
[InlineData(new [] {"page 2.jpg", "page 10.jpg"}, "page 2.jpg")]
[InlineData(new [] {"__MACOSX/cover.jpg", "vol1/page 01.jpg"}, "vol1/page 01.jpg")]
[InlineData(new [] {"Akame ga KILL! ZERO - c055 (v10) - p000 [Digital] [LuCaZ].jpg", "Akame ga KILL! ZERO - c055 (v10) - p000 [Digital] [LuCaZ].jpg", "Akame ga KILL! ZERO - c060 (v10) - p200 [Digital] [LuCaZ].jpg", "folder.jpg"}, "Akame ga KILL! ZERO - c055 (v10) - p000 [Digital] [LuCaZ].jpg")]
public void FindFirstEntry(string[] files, string expected)
{
var foundFile = _archiveService.FirstFileEntry(files);
Assert.Equal(expected, string.IsNullOrEmpty(foundFile) ? "" : foundFile);
}
@ -122,12 +152,37 @@ namespace API.Tests.Services
[InlineData("v10 - nested folder.cbz", "v10 - nested folder.expected.jpg")]
//[InlineData("png.zip", "png.PNG")]
[InlineData("macos_native.zip", "macos_native.jpg")]
public void GetCoverImageTest(string inputFile, string expectedOutputFile)
[InlineData("v10 - duplicate covers.cbz", "v10 - duplicate covers.expected.jpg")]
[InlineData("sorting.zip", "sorting.expected.jpg")]
public void GetCoverImage_Default_Test(string inputFile, string expectedOutputFile)
{
var archiveService = Substitute.For<ArchiveService>(_logger);
var testDirectory = Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/ArchiveService/CoverImages");
var expectedBytes = File.ReadAllBytes(Path.Join(testDirectory, expectedOutputFile));
archiveService.Configure().CanOpen(Path.Join(testDirectory, inputFile)).Returns(ArchiveLibrary.Default);
Stopwatch sw = Stopwatch.StartNew();
Assert.Equal(expectedBytes, _archiveService.GetCoverImage(Path.Join(testDirectory, inputFile)));
Assert.Equal(expectedBytes, archiveService.GetCoverImage(Path.Join(testDirectory, inputFile)));
_testOutputHelper.WriteLine($"Processed in {sw.ElapsedMilliseconds} ms");
}
[Theory]
[InlineData("v10.cbz", "v10.expected.jpg")]
[InlineData("v10 - with folder.cbz", "v10 - with folder.expected.jpg")]
[InlineData("v10 - nested folder.cbz", "v10 - nested folder.expected.jpg")]
//[InlineData("png.zip", "png.PNG")]
[InlineData("macos_native.zip", "macos_native.jpg")]
[InlineData("v10 - duplicate covers.cbz", "v10 - duplicate covers.expected.jpg")]
[InlineData("sorting.zip", "sorting.expected.jpg")]
public void GetCoverImage_SharpCompress_Test(string inputFile, string expectedOutputFile)
{
var archiveService = Substitute.For<ArchiveService>(_logger);
var testDirectory = Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/ArchiveService/CoverImages");
var expectedBytes = File.ReadAllBytes(Path.Join(testDirectory, expectedOutputFile));
archiveService.Configure().CanOpen(Path.Join(testDirectory, inputFile)).Returns(ArchiveLibrary.SharpCompress);
Stopwatch sw = Stopwatch.StartNew();
Assert.Equal(expectedBytes, archiveService.GetCoverImage(Path.Join(testDirectory, inputFile)));
_testOutputHelper.WriteLine($"Processed in {sw.ElapsedMilliseconds} ms");
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 122 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 385 KiB

View File

@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.Text.RegularExpressions;
using static System.GC;
using static System.String;
namespace API.Comparators
{
@ -17,20 +18,18 @@ namespace API.Comparators
int IComparer<string>.Compare(string x, string y)
{
if (x == y)
return 0;
if (x == y) return 0;
string[] x1, y1;
if (!_table.TryGetValue(x, out x1))
if (!_table.TryGetValue(x, out var x1))
{
x1 = Regex.Split(x.Replace(" ", ""), "([0-9]+)");
// .Replace(" ", Empty)
x1 = Regex.Split(x, "([0-9]+)");
_table.Add(x, x1);
}
if (!_table.TryGetValue(y ?? string.Empty, out y1))
if (!_table.TryGetValue(y, out var y1))
{
y1 = Regex.Split(y?.Replace(" ", ""), "([0-9]+)");
y1 = Regex.Split(y, "([0-9]+)");
_table.Add(y, y1);
}
@ -61,12 +60,11 @@ namespace API.Comparators
private static int PartCompare(string left, string right)
{
int x, y;
if (!int.TryParse(left, out x))
return left.CompareTo(right);
if (!int.TryParse(left, out var x))
return Compare(left, right, StringComparison.Ordinal);
if (!int.TryParse(right, out y))
return left.CompareTo(right);
if (!int.TryParse(right, out var y))
return Compare(left, right, StringComparison.Ordinal);
return x.CompareTo(y);
}

View File

@ -92,6 +92,7 @@ namespace API.Data
/// <returns></returns>
public async Task<Library> GetFullLibraryForIdAsync(int libraryId)
{
return await _context.Library
.Where(x => x.Id == libraryId)
.Include(f => f.Folders)

View File

@ -15,7 +15,7 @@ namespace API.Parser
private static readonly Regex ImageRegex = new Regex(ImageFileExtensions, RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex ArchiveFileRegex = new Regex(ArchiveFileExtensions, RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex XmlRegex = new Regex(XmlRegexExtensions, RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex FolderRegex = new Regex(@"(?<![[a-z]\d])(?:!?)(cover|folder)(?![\w\d])", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex CoverImageRegex = new Regex(@"(?<![[a-z]\d])(?:!?)(cover|folder)(?![\w\d])", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex[] MangaVolumeRegex = new[]
{
@ -752,7 +752,7 @@ namespace API.Parser
/// <returns></returns>
public static bool IsCoverImage(string name)
{
return IsImage(name, true) && (FolderRegex.IsMatch(name));
return IsImage(name, true) && (CoverImageRegex.IsMatch(name));
}
public static bool HasBlacklistedFolderInPath(string path)

View File

@ -39,7 +39,7 @@ namespace API.Services
/// </summary>
/// <param name="archivePath"></param>
/// <returns></returns>
public ArchiveLibrary CanOpen(string archivePath)
public virtual ArchiveLibrary CanOpen(string archivePath)
{
if (!File.Exists(archivePath) || !Parser.Parser.IsArchive(archivePath)) return ArchiveLibrary.NotSupported;
@ -103,11 +103,42 @@ namespace API.Services
return 0;
}
}
/// <summary>
/// Finds the first instance of a folder entry and returns it
/// </summary>
/// <param name="entryFullNames"></param>
/// <returns>Entry name of match, null if no match</returns>
public string FindFolderEntry(IEnumerable<string> entryFullNames)
{
var result = entryFullNames
.FirstOrDefault(x => !Path.EndsInDirectorySeparator(x) && !Parser.Parser.HasBlacklistedFolderInPath(x)
&& Parser.Parser.IsCoverImage(x));
return string.IsNullOrEmpty(result) ? null : result;
}
/// <summary>
/// Returns first entry that is an image and is not in a blacklisted folder path. Uses <see cref="NaturalSortComparer"/> for ordering files
/// </summary>
/// <param name="entryFullNames"></param>
/// <returns>Entry name of match, null if no match</returns>
public string FirstFileEntry(IEnumerable<string> entryFullNames)
{
var result = entryFullNames.OrderBy(Path.GetFileName, _comparer)
.FirstOrDefault(x => !Parser.Parser.HasBlacklistedFolderInPath(x)
&& Parser.Parser.IsImage(x));
return string.IsNullOrEmpty(result) ? null : result;
}
/// <summary>
/// Generates byte array of cover image.
/// Given a path to a compressed file (zip, rar, cbz, cbr, etc), will ensure the first image is returned unless
/// a folder.extension exists in the root directory of the compressed file.
/// Given a path to a compressed file <see cref="Parser.Parser.ArchiveFileExtensions"/>, will ensure the first image (respects directory structure) is returned unless
/// a folder/cover.(image extension) exists in the the compressed file (if duplicate, the first is chosen)
///
/// This skips over any __MACOSX folder/file iteration.
/// </summary>
/// <param name="archivePath"></param>
/// <param name="createThumbnail">Create a smaller variant of file extracted from archive. Archive images are usually 1MB each.</param>
@ -124,28 +155,28 @@ namespace API.Services
{
_logger.LogDebug("Using default compression handling");
using var archive = ZipFile.OpenRead(archivePath);
// NOTE: We can probably reduce our iteration by performing 1 filter on MACOSX then do our folder check and image chack.
var folder = archive.Entries.SingleOrDefault(x => !Parser.Parser.HasBlacklistedFolderInPath(x.FullName)
&& Parser.Parser.IsImage(x.FullName)
&& Parser.Parser.IsCoverImage(x.FullName));
var entries = archive.Entries.Where(x => Path.HasExtension(x.FullName)
&& !Parser.Parser.HasBlacklistedFolderInPath(x.FullName)
&& Parser.Parser.IsImage(x.FullName))
.OrderBy(x => x.FullName, _comparer).ToList();
var entry = folder ?? entries[0];
var entryNames = archive.Entries.Select(e => e.FullName).ToArray();
return createThumbnail ? CreateThumbnail(entry) : ConvertEntryToByteArray(entry);
var entryName = FindFolderEntry(entryNames) ?? FirstFileEntry(entryNames);
var entry = archive.Entries.Single(e => e.FullName == entryName);
using var stream = entry.Open();
return createThumbnail ? CreateThumbnail(entry.FullName, stream) : ConvertEntryToByteArray(entry);
}
case ArchiveLibrary.SharpCompress:
{
_logger.LogDebug("Using SharpCompress compression handling");
using var archive = ArchiveFactory.Open(archivePath);
var entries = archive.Entries
.Where(entry => !entry.IsDirectory
&& !Parser.Parser.HasBlacklistedFolderInPath(Path.GetDirectoryName(entry.Key) ?? string.Empty)
&& Parser.Parser.IsImage(entry.Key))
.OrderBy(x => x.Key, _comparer);
return FindCoverImage(entries, createThumbnail);
var entryNames = archive.Entries.Where(entry => !entry.IsDirectory).Select(e => e.Key).ToList();
var entryName = FindFolderEntry(entryNames) ?? FirstFileEntry(entryNames);
var entry = archive.Entries.Single(e => e.Key == entryName);
using var ms = _streamManager.GetStream();
entry.WriteTo(ms);
ms.Position = 0;
return createThumbnail ? CreateThumbnail(entry.Key, ms, Path.GetExtension(entry.Key)) : ms.ToArray();
}
case ArchiveLibrary.NotSupported:
_logger.LogError("[GetCoverImage] This archive cannot be read: {ArchivePath}. Defaulting to no cover image", archivePath);
@ -163,35 +194,6 @@ namespace API.Services
return Array.Empty<byte>();
}
private byte[] FindCoverImage(IEnumerable<IArchiveEntry> entries, bool createThumbnail)
{
var images = entries.ToList();
foreach (var entry in images)
{
if (Path.GetFileNameWithoutExtension(entry.Key).ToLower() == "folder")
{
using var ms = _streamManager.GetStream();
entry.WriteTo(ms);
ms.Position = 0;
var data = ms.ToArray();
return createThumbnail ? CreateThumbnail(data, Path.GetExtension(entry.Key)) : data;
}
}
if (images.Any())
{
var entry = images.OrderBy(e => e.Key).FirstOrDefault();
if (entry == null) return Array.Empty<byte>();
using var ms = _streamManager.GetStream();
entry.WriteTo(ms);
ms.Position = 0;
var data = ms.ToArray();
return createThumbnail ? CreateThumbnail(data, Path.GetExtension(entry.Key)) : data;
}
return Array.Empty<byte>();
}
private static byte[] ConvertEntryToByteArray(ZipArchiveEntry entry)
{
using var stream = entry.Open();
@ -213,28 +215,8 @@ namespace API.Services
!Path.HasExtension(archive.Entries.ElementAt(0).FullName) ||
archive.Entries.Any(e => e.FullName.Contains(Path.AltDirectorySeparatorChar) && !Parser.Parser.HasBlacklistedFolderInPath(e.FullName));
}
private byte[] CreateThumbnail(byte[] entry, string formatExtension = ".jpg")
{
if (!formatExtension.StartsWith("."))
{
formatExtension = "." + formatExtension;
}
try
{
using var thumbnail = Image.ThumbnailBuffer(entry, ThumbnailWidth);
return thumbnail.WriteToBuffer(formatExtension);
}
catch (Exception ex)
{
_logger.LogError(ex, "[CreateThumbnail] There was a critical error and prevented thumbnail generation. Defaulting to no cover image. Format Extension {Extension}", formatExtension);
}
return Array.Empty<byte>();
}
private byte[] CreateThumbnail(ZipArchiveEntry entry, string formatExtension = ".jpg")
private byte[] CreateThumbnail(string entryName, Stream stream, string formatExtension = ".jpg")
{
if (!formatExtension.StartsWith("."))
{
@ -242,13 +224,12 @@ namespace API.Services
}
try
{
using var stream = entry.Open();
using var thumbnail = Image.ThumbnailStream(stream, ThumbnailWidth);
return thumbnail.WriteToBuffer(formatExtension);
}
catch (Exception ex)
{
_logger.LogError(ex, "There was a critical error and prevented thumbnail generation on {EntryName}. Defaulting to no cover image", entry.FullName);
_logger.LogError(ex, "There was a critical error and prevented thumbnail generation on {EntryName}. Defaulting to no cover image", entryName);
}
return Array.Empty<byte>();

View File

@ -4,7 +4,6 @@ using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using API.Comparators;
using API.Entities;
using API.Extensions;
using API.Interfaces;
@ -41,19 +40,19 @@ namespace API.Services
}
}
public void UpdateMetadata(Volume volume, bool forceUpdate)
{
if (volume != null && ShouldFindCoverImage(volume.CoverImage, forceUpdate))
{
// TODO: Replace this with ChapterSortComparator
volume.Chapters ??= new List<Chapter>();
var firstChapter = volume.Chapters.OrderBy(x => double.Parse(x.Number)).FirstOrDefault();
var firstChapter = volume.Chapters.OrderBy(x => double.Parse(x.Number)).FirstOrDefault();
var firstFile = firstChapter?.Files.OrderBy(x => x.Chapter).FirstOrDefault();
// Skip calculating Cover Image (I/O) if the chapter already has it set
if (firstChapter == null || ShouldFindCoverImage(firstChapter.CoverImage))
{
var firstFile = firstChapter?.Files.OrderBy(x => x.Chapter).FirstOrDefault();
if (firstFile != null && !new FileInfo(firstFile.FilePath).IsLastWriteLessThan(firstFile.LastModified))
{
volume.CoverImage = _archiveService.GetCoverImage(firstFile.FilePath, true);
@ -112,6 +111,7 @@ namespace API.Services
var sw = Stopwatch.StartNew();
var library = Task.Run(() => _unitOfWork.LibraryRepository.GetFullLibraryForIdAsync(libraryId)).Result;
// TODO: See if we can break this up into multiple threads that process 20 series at a time then save so we can reduce amount of memory used
_logger.LogInformation("Beginning metadata refresh of {LibraryName}", library.Name);
foreach (var series in library.Series)
{

View File

@ -40,7 +40,7 @@ namespace API.Services
{
_logger.LogInformation("Scheduling reoccurring tasks");
string setting = Task.Run(() => _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.TaskScan)).Result.Value;
string setting = Task.Run(() => _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.TaskScan)).GetAwaiter().GetResult().Value;
if (setting != null)
{
_logger.LogDebug("Scheduling Scan Library Task for {Setting}", setting);