diff --git a/API.Tests/Services/ImageProviderTest.cs b/API.Tests/Services/ImageProviderTest.cs index eae737cb7..19a4a317b 100644 --- a/API.Tests/Services/ImageProviderTest.cs +++ b/API.Tests/Services/ImageProviderTest.cs @@ -1,7 +1,9 @@ using System; using System.IO; using API.IO; +using NetVips; using Xunit; +using Xunit.Abstractions; namespace API.Tests.Services { @@ -10,6 +12,8 @@ namespace API.Tests.Services [Theory] [InlineData("v10.cbz", "v10.expected.jpg")] [InlineData("v10 - with folder.cbz", "v10 - with folder.expected.jpg")] + //[InlineData("v10 - nested folder.cbz", "v10 - nested folder.expected.jpg")] + [InlineData("Akame ga KILL! ZERO v06 (2017) (Digital) (LuCaZ).cbz", "Akame ga KILL! ZERO v06 (2017) (Digital) (LuCaZ).expected.jpg")] public void GetCoverImageTest(string inputFile, string expectedOutputFile) { var testDirectory = Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/ImageProvider"); diff --git a/API.Tests/Services/Test Data/ImageProvider/Akame ga KILL! ZERO v06 (2017) (Digital) (LuCaZ).cbz b/API.Tests/Services/Test Data/ImageProvider/Akame ga KILL! ZERO v06 (2017) (Digital) (LuCaZ).cbz new file mode 100644 index 000000000..b1ac4413e Binary files /dev/null and b/API.Tests/Services/Test Data/ImageProvider/Akame ga KILL! ZERO v06 (2017) (Digital) (LuCaZ).cbz differ diff --git a/API.Tests/Services/Test Data/ImageProvider/Akame ga KILL! ZERO v06 (2017) (Digital) (LuCaZ).expected.jpg b/API.Tests/Services/Test Data/ImageProvider/Akame ga KILL! ZERO v06 (2017) (Digital) (LuCaZ).expected.jpg new file mode 100644 index 000000000..f7370c1ce Binary files /dev/null and b/API.Tests/Services/Test Data/ImageProvider/Akame ga KILL! ZERO v06 (2017) (Digital) (LuCaZ).expected.jpg differ diff --git a/API.Tests/Services/Test Data/ImageProvider/thumbnail.expected.jpg b/API.Tests/Services/Test Data/ImageProvider/thumbnail.expected.jpg new file mode 100644 index 000000000..7cbc36328 Binary files /dev/null and b/API.Tests/Services/Test Data/ImageProvider/thumbnail.expected.jpg differ diff --git a/API.Tests/Services/Test Data/ImageProvider/thumbnail.jpg b/API.Tests/Services/Test Data/ImageProvider/thumbnail.jpg new file mode 100644 index 000000000..b192a9c62 Binary files /dev/null and b/API.Tests/Services/Test Data/ImageProvider/thumbnail.jpg differ diff --git a/API/API.csproj b/API/API.csproj index 1b99ef72e..73575c864 100644 --- a/API/API.csproj +++ b/API/API.csproj @@ -20,6 +20,8 @@ + + all runtime; build; native; contentfiles; analyzers; buildtransitive @@ -30,6 +32,20 @@ + + + + + + + + + + + + + + diff --git a/API/Controllers/LibraryController.cs b/API/Controllers/LibraryController.cs index b72094f17..7bdc4eec6 100644 --- a/API/Controllers/LibraryController.cs +++ b/API/Controllers/LibraryController.cs @@ -72,7 +72,7 @@ namespace API.Controllers if (await _userRepository.SaveAllAsync()) { var createdLibrary = await _libraryRepository.GetLibraryForNameAsync(library.Name); - BackgroundJob.Enqueue(() => _directoryService.ScanLibrary(createdLibrary.Id)); + BackgroundJob.Enqueue(() => _directoryService.ScanLibrary(createdLibrary.Id, false)); return Ok(); } @@ -131,7 +131,7 @@ namespace API.Controllers [HttpPost("scan")] public ActionResult ScanLibrary(int libraryId) { - BackgroundJob.Enqueue(() => _directoryService.ScanLibrary(libraryId)); + BackgroundJob.Enqueue(() => _directoryService.ScanLibrary(libraryId, true)); return Ok(); } diff --git a/API/Data/SeriesRepository.cs b/API/Data/SeriesRepository.cs index 011550348..bed3bb093 100644 --- a/API/Data/SeriesRepository.cs +++ b/API/Data/SeriesRepository.cs @@ -66,6 +66,7 @@ namespace API.Data { return _context.Volume .Where(vol => vol.SeriesId == seriesId) + .Include(vol => vol.Files) .OrderBy(vol => vol.Number) .ToList(); } diff --git a/API/IO/ImageProvider.cs b/API/IO/ImageProvider.cs index 7a71c8813..9374ce9bf 100644 --- a/API/IO/ImageProvider.cs +++ b/API/IO/ImageProvider.cs @@ -2,6 +2,7 @@ using System.IO; using System.IO.Compression; using System.Linq; +using NetVips; namespace API.IO { @@ -13,28 +14,51 @@ namespace API.IO /// a folder.extension exists in the root directory of the compressed file. /// /// + /// Create a smaller variant of file extracted from archive. Archive images are usually 1MB each. /// - public static byte[] GetCoverImage(string filepath) + public static byte[] GetCoverImage(string filepath, bool createThumbnail = false) { if (!File.Exists(filepath) || !Parser.Parser.IsArchive(filepath)) return Array.Empty(); using ZipArchive archive = ZipFile.OpenRead(filepath); if (archive.Entries.Count <= 0) return Array.Empty(); + + var folder = archive.Entries.SingleOrDefault(x => Path.GetFileNameWithoutExtension(x.Name).ToLower() == "folder"); - var entry = archive.Entries[0]; - + var entry = archive.Entries.OrderBy(x => x.FullName).ToList()[0]; + if (folder != null) { entry = folder; } - - return ExtractEntryToImage(entry); - } + if (entry.FullName.EndsWith(Path.PathSeparator)) + { + // TODO: Implement nested directory support + } + + if (createThumbnail) + { + try + { + using var stream = entry.Open(); + var thumbnail = Image.ThumbnailStream(stream, 320); + Console.WriteLine(thumbnail.ToString()); + return thumbnail.WriteToBuffer(".jpg"); + } + catch (Exception ex) + { + Console.WriteLine("There was a critical error and prevented thumbnail generation."); + } + } + + return ExtractEntryToImage(entry); + } + private static byte[] ExtractEntryToImage(ZipArchiveEntry entry) { - var stream = entry.Open(); + using var stream = entry.Open(); using var ms = new MemoryStream(); stream.CopyTo(ms); var data = ms.ToArray(); diff --git a/API/Interfaces/IDirectoryService.cs b/API/Interfaces/IDirectoryService.cs index 351e67a68..8d1240172 100644 --- a/API/Interfaces/IDirectoryService.cs +++ b/API/Interfaces/IDirectoryService.cs @@ -6,6 +6,6 @@ namespace API.Interfaces { IEnumerable ListDirectory(string rootPath); - void ScanLibrary(int libraryId); + void ScanLibrary(int libraryId, bool forceUpdate = false); } } \ No newline at end of file diff --git a/API/Services/DirectoryService.cs b/API/Services/DirectoryService.cs index 85344f40e..c599be2f4 100644 --- a/API/Services/DirectoryService.cs +++ b/API/Services/DirectoryService.cs @@ -12,6 +12,7 @@ using API.Entities; using API.Interfaces; using API.IO; using API.Parser; +using Hangfire; using Microsoft.Extensions.Logging; namespace API.Services @@ -21,7 +22,7 @@ namespace API.Services private readonly ILogger _logger; private readonly ISeriesRepository _seriesRepository; private readonly ILibraryRepository _libraryRepository; - + private ConcurrentDictionary> _scannedSeries; public DirectoryService(ILogger logger, @@ -70,67 +71,45 @@ namespace API.Services /// - /// Processes files found during a library scan. + /// Processes files found during a library scan. Generates a collection of series->volume->files for DB processing later. /// - /// + /// Path of a file private void Process(string path) { - // NOTE: In current implementation, this never runs. We can probably remove. - if (Directory.Exists(path)) + var fileName = Path.GetFileName(path); + _logger.LogDebug($"Parsing file {fileName}"); + + var info = Parser.Parser.Parse(fileName); + info.FullFilePath = path; + if (info.Volumes == string.Empty) { - DirectoryInfo di = new DirectoryInfo(path); - _logger.LogDebug($"Parsing directory {di.Name}"); + return; + } - var seriesName = Parser.Parser.ParseSeries(di.Name); - if (string.IsNullOrEmpty(seriesName)) + ConcurrentBag tempBag; + ConcurrentBag newBag = new ConcurrentBag(); + if (_scannedSeries.TryGetValue(info.Series, out tempBag)) + { + var existingInfos = tempBag.ToArray(); + foreach (var existingInfo in existingInfos) { - return; - } - - // We don't need ContainsKey, this is a race condition. We can replace with TryAdd instead - if (!_scannedSeries.ContainsKey(seriesName)) - { - _scannedSeries.TryAdd(seriesName, new ConcurrentBag()); + newBag.Add(existingInfo); } } else { - var fileName = Path.GetFileName(path); - _logger.LogDebug($"Parsing file {fileName}"); + tempBag = new ConcurrentBag(); + } - var info = Parser.Parser.Parse(fileName); - info.FullFilePath = path; - if (info.Volumes == string.Empty) - { - return; - } - - ConcurrentBag tempBag; - ConcurrentBag newBag = new ConcurrentBag(); - if (_scannedSeries.TryGetValue(info.Series, out tempBag)) - { - var existingInfos = tempBag.ToArray(); - foreach (var existingInfo in existingInfos) - { - newBag.Add(existingInfo); - } - } - else - { - tempBag = new ConcurrentBag(); - } - - newBag.Add(info); - - if (!_scannedSeries.TryUpdate(info.Series, newBag, tempBag)) - { - _scannedSeries.TryAdd(info.Series, newBag); - } + newBag.Add(info); + if (!_scannedSeries.TryUpdate(info.Series, newBag, tempBag)) + { + _scannedSeries.TryAdd(info.Series, newBag); } } - private Series UpdateSeries(string seriesName, ParserInfo[] infos) + private Series UpdateSeries(string seriesName, ParserInfo[] infos, bool forceUpdate) { var series = _seriesRepository.GetSeriesByName(seriesName); @@ -145,8 +124,9 @@ namespace API.Services }; } - var volumes = UpdateVolumes(series, infos); + var volumes = UpdateVolumes(series, infos, forceUpdate); series.Volumes = volumes; + // TODO: Instead of taking first entry, re-calculate without compression series.CoverImage = volumes.OrderBy(x => x.Number).FirstOrDefault()?.CoverImage; return series; } @@ -156,12 +136,13 @@ namespace API.Services /// /// Series wanting to be updated /// Parser info + /// Forces metadata update (cover image) even if it's already been set. /// Updated Volumes for given series - private ICollection UpdateVolumes(Series series, ParserInfo[] infos) + private ICollection UpdateVolumes(Series series, ParserInfo[] infos, bool forceUpdate) { ICollection volumes = new List(); IList existingVolumes = _seriesRepository.GetVolumes(series.Id).ToList(); - //IList existingVolumes = Task.Run(() => _seriesRepository.GetVolumesAsync(series.Id)).Result.ToList(); + foreach (var info in infos) { var existingVolume = existingVolumes.SingleOrDefault(v => v.Name == info.Volumes); @@ -175,6 +156,11 @@ namespace API.Services FilePath = info.File } }; + + if (forceUpdate || existingVolume.CoverImage == null || existingVolumes.Count == 0) + { + existingVolume.CoverImage = ImageProvider.GetCoverImage(info.FullFilePath, true); + } volumes.Add(existingVolume); } else @@ -183,7 +169,7 @@ namespace API.Services { Name = info.Volumes, Number = Int32.Parse(info.Volumes), - CoverImage = ImageProvider.GetCoverImage(info.FullFilePath), + CoverImage = ImageProvider.GetCoverImage(info.FullFilePath, true), Files = new List() { new MangaFile() @@ -201,7 +187,7 @@ namespace API.Services return volumes; } - public void ScanLibrary(int libraryId) + public void ScanLibrary(int libraryId, bool forceUpdate = false) { var library = Task.Run(() => _libraryRepository.GetLibraryForIdAsync(libraryId)).Result; _scannedSeries = new ConcurrentDictionary>(); @@ -234,7 +220,7 @@ namespace API.Services library.Series = new List(); // Temp delete everything until we can mark items Unavailable foreach (var seriesKey in series.Keys) { - var s = UpdateSeries(seriesKey, series[seriesKey].ToArray()); + var s = UpdateSeries(seriesKey, series[seriesKey].ToArray(), forceUpdate); _logger.LogInformation($"Created/Updated series {s.Name}"); library.Series.Add(s); } @@ -251,7 +237,6 @@ namespace API.Services { _logger.LogError("There was a critical error that resulted in a failed scan. Please rescan."); } - _scannedSeries = null; } @@ -351,9 +336,6 @@ namespace API.Services // For diagnostic purposes. Console.WriteLine("Processed {0} files in {1} milliseconds", fileCount, sw.ElapsedMilliseconds); } - - - } } \ No newline at end of file diff --git a/API/Startup.cs b/API/Startup.cs index 4ad56e85a..199327e43 100644 --- a/API/Startup.cs +++ b/API/Startup.cs @@ -46,7 +46,7 @@ namespace API } app.UseHangfireDashboard(); - backgroundJobs.Enqueue(() => Console.WriteLine("Hello world from Hangfire!")); + //backgroundJobs.Enqueue(() => Console.WriteLine("Hello world from Hangfire!")); app.UseHttpsRedirection();