From fe2b9b86bcd00cce65fb69878ecbbdbc344c8ec9 Mon Sep 17 00:00:00 2001 From: Joe Milazzo Date: Sat, 11 Nov 2023 13:50:11 -0600 Subject: [PATCH] Image-only Libraries + Library Fixes (#2427) --- API.Tests/Parser/DefaultParserTests.cs | 109 ++++++++++++++---- API/Entities/Enums/LibraryType.cs | 5 + API/Entities/Library.cs | 3 + API/Helpers/Builders/ChapterBuilder.cs | 4 +- .../Tasks/Scanner/Parser/DefaultParser.cs | 80 +++++++------ API/Services/Tasks/Scanner/ProcessSeries.cs | 38 +++--- API/Services/Tasks/ScannerService.cs | 31 +++-- openapi.json | 32 +++-- 8 files changed, 195 insertions(+), 107 deletions(-) diff --git a/API.Tests/Parser/DefaultParserTests.cs b/API.Tests/Parser/DefaultParserTests.cs index 61ed57aca..a658080de 100644 --- a/API.Tests/Parser/DefaultParserTests.cs +++ b/API.Tests/Parser/DefaultParserTests.cs @@ -239,15 +239,6 @@ public class DefaultParserTests FullFilePath = filepath, IsSpecial = false }); - // Note: Fallback to folder will parse Monster #8 and get Monster - filepath = @"E:\Manga\Monster #8\Ch. 001-016 [MangaPlus] [Digital] [amit34521]\Monster #8 Ch. 001 [MangaPlus] [Digital] [amit34521]\13.jpg"; - expected.Add(filepath, new ParserInfo - { - Series = "Monster", Volumes = "0", Edition = "", - Chapters = "1", Filename = "13.jpg", Format = MangaFormat.Image, - FullFilePath = filepath, IsSpecial = false - }); - filepath = @"E:\Manga\Air Gear\Air Gear Omnibus v01 (2016) (Digital) (Shadowcat-Empire).cbz"; expected.Add(filepath, new ParserInfo { @@ -256,22 +247,6 @@ public class DefaultParserTests FullFilePath = filepath, IsSpecial = false }); - filepath = @"E:\Manga\Extra layer for no reason\Just Images the second\Vol19\ch186\Vol. 19 p106.gif"; - expected.Add(filepath, new ParserInfo - { - Series = "Just Images the second", Volumes = "19", Edition = "", - Chapters = "186", Filename = "Vol. 19 p106.gif", Format = MangaFormat.Image, - FullFilePath = filepath, IsSpecial = false - }); - - filepath = @"E:\Manga\Extra layer for no reason\Just Images the second\Blank Folder\Vol19\ch186\Vol. 19 p106.gif"; - expected.Add(filepath, new ParserInfo - { - Series = "Just Images the second", Volumes = "19", Edition = "", - Chapters = "186", Filename = "Vol. 19 p106.gif", Format = MangaFormat.Image, - FullFilePath = filepath, IsSpecial = false - }); - filepath = @"E:\Manga\Harrison, Kim - The Good, The Bad, and the Undead - Hollows Vol 2.5.epub"; expected.Add(filepath, new ParserInfo { @@ -308,6 +283,90 @@ public class DefaultParserTests } } + [Fact] + public void Parse_ParseInfo_Manga_ImageOnly() + { + // Images don't have root path as E:\Manga, but rather as the path of the folder + + // Note: Fallback to folder will parse Monster #8 and get Monster + var filepath = @"E:\Manga\Monster #8\Ch. 001-016 [MangaPlus] [Digital] [amit34521]\Monster #8 Ch. 001 [MangaPlus] [Digital] [amit34521]\13.jpg"; + var expectedInfo2 = new ParserInfo + { + Series = "Monster #8", Volumes = "0", Edition = "", + Chapters = "1", Filename = "13.jpg", Format = MangaFormat.Image, + FullFilePath = filepath, IsSpecial = false + }; + var actual2 = _defaultParser.Parse(filepath, @"E:\Manga\Monster #8"); + Assert.NotNull(actual2); + _testOutputHelper.WriteLine($"Validating {filepath}"); + Assert.Equal(expectedInfo2.Format, actual2.Format); + _testOutputHelper.WriteLine("Format ✓"); + Assert.Equal(expectedInfo2.Series, actual2.Series); + _testOutputHelper.WriteLine("Series ✓"); + Assert.Equal(expectedInfo2.Chapters, actual2.Chapters); + _testOutputHelper.WriteLine("Chapters ✓"); + Assert.Equal(expectedInfo2.Volumes, actual2.Volumes); + _testOutputHelper.WriteLine("Volumes ✓"); + Assert.Equal(expectedInfo2.Edition, actual2.Edition); + _testOutputHelper.WriteLine("Edition ✓"); + Assert.Equal(expectedInfo2.Filename, actual2.Filename); + _testOutputHelper.WriteLine("Filename ✓"); + Assert.Equal(expectedInfo2.FullFilePath, actual2.FullFilePath); + _testOutputHelper.WriteLine("FullFilePath ✓"); + + filepath = @"E:\Manga\Extra layer for no reason\Just Images the second\Vol19\ch186\Vol. 19 p106.gif"; + expectedInfo2 = new ParserInfo + { + Series = "Just Images the second", Volumes = "19", Edition = "", + Chapters = "186", Filename = "Vol. 19 p106.gif", Format = MangaFormat.Image, + FullFilePath = filepath, IsSpecial = false + }; + + actual2 = _defaultParser.Parse(filepath, @"E:\Manga\Extra layer for no reason\"); + Assert.NotNull(actual2); + _testOutputHelper.WriteLine($"Validating {filepath}"); + Assert.Equal(expectedInfo2.Format, actual2.Format); + _testOutputHelper.WriteLine("Format ✓"); + Assert.Equal(expectedInfo2.Series, actual2.Series); + _testOutputHelper.WriteLine("Series ✓"); + Assert.Equal(expectedInfo2.Chapters, actual2.Chapters); + _testOutputHelper.WriteLine("Chapters ✓"); + Assert.Equal(expectedInfo2.Volumes, actual2.Volumes); + _testOutputHelper.WriteLine("Volumes ✓"); + Assert.Equal(expectedInfo2.Edition, actual2.Edition); + _testOutputHelper.WriteLine("Edition ✓"); + Assert.Equal(expectedInfo2.Filename, actual2.Filename); + _testOutputHelper.WriteLine("Filename ✓"); + Assert.Equal(expectedInfo2.FullFilePath, actual2.FullFilePath); + _testOutputHelper.WriteLine("FullFilePath ✓"); + + filepath = @"E:\Manga\Extra layer for no reason\Just Images the second\Blank Folder\Vol19\ch186\Vol. 19 p106.gif"; + expectedInfo2 = new ParserInfo + { + Series = "Just Images the second", Volumes = "19", Edition = "", + Chapters = "186", Filename = "Vol. 19 p106.gif", Format = MangaFormat.Image, + FullFilePath = filepath, IsSpecial = false + }; + + actual2 = _defaultParser.Parse(filepath, @"E:\Manga\Extra layer for no reason\"); + Assert.NotNull(actual2); + _testOutputHelper.WriteLine($"Validating {filepath}"); + Assert.Equal(expectedInfo2.Format, actual2.Format); + _testOutputHelper.WriteLine("Format ✓"); + Assert.Equal(expectedInfo2.Series, actual2.Series); + _testOutputHelper.WriteLine("Series ✓"); + Assert.Equal(expectedInfo2.Chapters, actual2.Chapters); + _testOutputHelper.WriteLine("Chapters ✓"); + Assert.Equal(expectedInfo2.Volumes, actual2.Volumes); + _testOutputHelper.WriteLine("Volumes ✓"); + Assert.Equal(expectedInfo2.Edition, actual2.Edition); + _testOutputHelper.WriteLine("Edition ✓"); + Assert.Equal(expectedInfo2.Filename, actual2.Filename); + _testOutputHelper.WriteLine("Filename ✓"); + Assert.Equal(expectedInfo2.FullFilePath, actual2.FullFilePath); + _testOutputHelper.WriteLine("FullFilePath ✓"); + } + [Fact] public void Parse_ParseInfo_Manga_WithSpecialsFolder() { diff --git a/API/Entities/Enums/LibraryType.cs b/API/Entities/Enums/LibraryType.cs index 5f4ab1cc7..038ce7172 100644 --- a/API/Entities/Enums/LibraryType.cs +++ b/API/Entities/Enums/LibraryType.cs @@ -19,4 +19,9 @@ public enum LibraryType /// [Description("Book")] Book = 2, + /// + /// Uses a different type of grouping and parsing mechanism + /// + [Description("Image")] + Image = 3, } diff --git a/API/Entities/Library.cs b/API/Entities/Library.cs index af15d06f6..f0674b8a9 100644 --- a/API/Entities/Library.cs +++ b/API/Entities/Library.cs @@ -41,6 +41,9 @@ public class Library : IEntityDate /// /// Scrobbling requires a valid LicenseKey public bool AllowScrobbling { get; set; } = true; + + + public DateTime Created { get; set; } public DateTime LastModified { get; set; } public DateTime CreatedUtc { get; set; } diff --git a/API/Helpers/Builders/ChapterBuilder.cs b/API/Helpers/Builders/ChapterBuilder.cs index c4d6c5785..80de027c3 100644 --- a/API/Helpers/Builders/ChapterBuilder.cs +++ b/API/Helpers/Builders/ChapterBuilder.cs @@ -28,8 +28,8 @@ public class ChapterBuilder : IEntityBuilder { var specialTreatment = info.IsSpecialInfo(); var specialTitle = specialTreatment ? info.Filename : info.Chapters; - var builder = new ChapterBuilder(Services.Tasks.Scanner.Parser.Parser.DefaultChapter); - return builder.WithNumber(specialTreatment ? Services.Tasks.Scanner.Parser.Parser.DefaultChapter : Services.Tasks.Scanner.Parser.Parser.MinNumberFromRange(info.Chapters) + string.Empty) + var builder = new ChapterBuilder(Parser.DefaultChapter); + return builder.WithNumber(specialTreatment ? Parser.DefaultChapter : Parser.MinNumberFromRange(info.Chapters) + string.Empty) .WithRange(specialTreatment ? info.Filename : info.Chapters) .WithTitle((specialTreatment && info.Format == MangaFormat.Epub) ? info.Title diff --git a/API/Services/Tasks/Scanner/Parser/DefaultParser.cs b/API/Services/Tasks/Scanner/Parser/DefaultParser.cs index 5ea5a1a0a..dd6328272 100644 --- a/API/Services/Tasks/Scanner/Parser/DefaultParser.cs +++ b/API/Services/Tasks/Scanner/Parser/DefaultParser.cs @@ -35,43 +35,38 @@ public class DefaultParser : IDefaultParser { var fileName = _directoryService.FileSystem.Path.GetFileNameWithoutExtension(filePath); // TODO: Potential Bug: This will return null, but on Image libraries, if all images, we would want to include this. (we can probably remove this and have users use kavitaignore) - if (Parser.IsCoverImage(_directoryService.FileSystem.Path.GetFileName(filePath))) return null; + if (type != LibraryType.Image && Parser.IsCoverImage(_directoryService.FileSystem.Path.GetFileName(filePath))) return null; - ParserInfo ret; - - if (Parser.IsEpub(filePath)) // NOTE: Will this ever be called? Because we use ReadingService to handle parse + var ret = new ParserInfo() { - ret = new ParserInfo - { - Chapters = Parser.ParseChapter(fileName) ?? Parser.ParseComicChapter(fileName), - Series = Parser.ParseSeries(fileName) ?? Parser.ParseComicSeries(fileName), - Volumes = Parser.ParseVolume(fileName) ?? Parser.ParseComicVolume(fileName), - Filename = Path.GetFileName(filePath), - Format = Parser.ParseFormat(filePath), - FullFilePath = filePath - }; + Filename = Path.GetFileName(filePath), + Format = Parser.ParseFormat(filePath), + Title = Path.GetFileNameWithoutExtension(fileName), + FullFilePath = filePath, + Series = string.Empty + }; + + // If library type is Image or this is not a cover image in a non-image library, then use dedicated parsing mechanism + if (type == LibraryType.Image || Parser.IsImage(filePath)) + { + return ParseImage(filePath, rootPath, ret); + } + + + // This will be called if the epub is already parsed once then we call and merge the information, if the + if (Parser.IsEpub(filePath)) + { + ret.Chapters = Parser.ParseChapter(fileName); + ret.Series = Parser.ParseSeries(fileName); + ret.Volumes = Parser.ParseVolume(fileName); } else { - ret = new ParserInfo - { - Chapters = type == LibraryType.Comic ? Parser.ParseComicChapter(fileName) : Parser.ParseChapter(fileName), - Series = type == LibraryType.Comic ? Parser.ParseComicSeries(fileName) : Parser.ParseSeries(fileName), - Volumes = type == LibraryType.Comic ? Parser.ParseComicVolume(fileName) : Parser.ParseVolume(fileName), - Filename = Path.GetFileName(filePath), - Format = Parser.ParseFormat(filePath), - Title = Path.GetFileNameWithoutExtension(fileName), - FullFilePath = filePath - }; - } - - - if (Parser.IsImage(filePath)) - { - // Reset Chapters, Volumes, and Series as images are not good to parse information out of. Better to use folders. - ret.Volumes = Parser.DefaultVolume; - ret.Chapters = Parser.DefaultChapter; - ret.Series = string.Empty; + ret.Chapters = type == LibraryType.Comic + ? Parser.ParseComicChapter(fileName) + : Parser.ParseChapter(fileName); + ret.Series = type == LibraryType.Comic ? Parser.ParseComicSeries(fileName) : Parser.ParseSeries(fileName); + ret.Volumes = type == LibraryType.Comic ? Parser.ParseComicVolume(fileName) : Parser.ParseVolume(fileName); } if (ret.Series == string.Empty || Parser.IsImage(filePath)) @@ -120,6 +115,23 @@ public class DefaultParser : IDefaultParser return ret.Series == string.Empty ? null : ret; } + private ParserInfo ParseImage(string filePath, string rootPath, ParserInfo ret) + { + ret.Volumes = Parser.DefaultVolume; + ret.Chapters = Parser.DefaultChapter; + // Next we need to see if the image has a folder between rootPath and filePath. + // if so, take that folder as a volume 0 chapter 0 special and group everything under there (if we can't parse a volume/chapter) + ParseFromFallbackFolders(filePath, rootPath, LibraryType.Image, ref ret); + if ((string.IsNullOrEmpty(ret.Chapters) || ret.Chapters == Parser.DefaultChapter) && + (string.IsNullOrEmpty(ret.Volumes) || ret.Volumes == Parser.DefaultVolume)) + { + ret.IsSpecial = true; + } + + ret.Series = _directoryService.FileSystem.DirectoryInfo.New(rootPath).Name; + return ret; + } + /// /// Fills out by trying to parse volume, chapters, and series from folders /// @@ -160,11 +172,11 @@ public class DefaultParser : IDefaultParser if (!parsedVolume.Equals(Parser.DefaultVolume) || !parsedChapter.Equals(Parser.DefaultChapter)) { - if ((string.IsNullOrEmpty(ret.Volumes) || ret.Volumes.Equals(Parser.DefaultVolume)) && !parsedVolume.Equals(Parser.DefaultVolume)) + if ((string.IsNullOrEmpty(ret.Volumes) || ret.Volumes.Equals(Parser.DefaultVolume)) && !string.IsNullOrEmpty(parsedVolume) && !parsedVolume.Equals(Parser.DefaultVolume)) { ret.Volumes = parsedVolume; } - if ((string.IsNullOrEmpty(ret.Chapters) || ret.Chapters.Equals(Parser.DefaultChapter)) && !parsedChapter.Equals(Parser.DefaultChapter)) + if ((string.IsNullOrEmpty(ret.Chapters) || ret.Chapters.Equals(Parser.DefaultChapter)) && !string.IsNullOrEmpty(parsedChapter) && !parsedChapter.Equals(Parser.DefaultChapter)) { ret.Chapters = parsedChapter; } diff --git a/API/Services/Tasks/Scanner/ProcessSeries.cs b/API/Services/Tasks/Scanner/ProcessSeries.cs index 32492f22d..3a13e8d25 100644 --- a/API/Services/Tasks/Scanner/ProcessSeries.cs +++ b/API/Services/Tasks/Scanner/ProcessSeries.cs @@ -62,9 +62,6 @@ public class ProcessSeries : IProcessSeries private IList _people; private Dictionary _tags; private Dictionary _collectionTags; - private readonly object _peopleLock = new object(); - private readonly object _genreLock = new object(); - private readonly object _tagLock = new object(); public ProcessSeries(IUnitOfWork unitOfWork, ILogger logger, IEventHub eventHub, IDirectoryService directoryService, ICacheHelper cacheHelper, IReadingItemService readingItemService, @@ -845,23 +842,20 @@ public class ProcessSeries : IProcessSeries /// private void UpdatePeople(IEnumerable names, PersonRole role, Action action) { - lock (_peopleLock) + var allPeopleTypeRole = _people.Where(p => p.Role == role).ToList(); + + foreach (var name in names) { - var allPeopleTypeRole = _people.Where(p => p.Role == role).ToList(); + var normalizedName = name.ToNormalized(); + var person = allPeopleTypeRole.Find(p => + p.NormalizedName != null && p.NormalizedName.Equals(normalizedName)); - foreach (var name in names) + if (person == null) { - var normalizedName = name.ToNormalized(); - var person = allPeopleTypeRole.Find(p => - p.NormalizedName != null && p.NormalizedName.Equals(normalizedName)); - - if (person == null) - { - person = new PersonBuilder(name, role).Build(); - _people.Add(person); - } - action(person); + person = new PersonBuilder(name, role).Build(); + _people.Add(person); } + action(person); } } @@ -882,11 +876,8 @@ public class ProcessSeries : IProcessSeries if (newTag) { genre = new GenreBuilder(name).Build(); - lock (_genreLock) - { - _genres.Add(normalizedName, genre); - _unitOfWork.GenreRepository.Attach(genre); - } + _genres.Add(normalizedName, genre); + _unitOfWork.GenreRepository.Attach(genre); } action(genre!, newTag); @@ -911,10 +902,7 @@ public class ProcessSeries : IProcessSeries if (tag == null) { tag = new TagBuilder(name).Build(); - lock (_tagLock) - { - _tags.Add(normalizedName, tag); - } + _tags.Add(normalizedName, tag); } action(tag, added); diff --git a/API/Services/Tasks/ScannerService.cs b/API/Services/Tasks/ScannerService.cs index 6a08306df..08d1f86f4 100644 --- a/API/Services/Tasks/ScannerService.cs +++ b/API/Services/Tasks/ScannerService.cs @@ -4,6 +4,7 @@ using System.Diagnostics; using System.Globalization; using System.IO; using System.Linq; +using System.Threading; using System.Threading.Tasks; using API.Data; using API.Data.Repositories; @@ -85,6 +86,8 @@ public class ScannerService : IScannerService private readonly IProcessSeries _processSeries; private readonly IWordCountAnalyzerService _wordCountAnalyzerService; + private readonly SemaphoreSlim _seriesProcessingSemaphore = new SemaphoreSlim(1, 1); + public ScannerService(IUnitOfWork unitOfWork, ILogger logger, IMetadataService metadataService, ICacheService cacheService, IEventHub eventHub, IDirectoryService directoryService, IReadingItemService readingItemService, @@ -495,10 +498,10 @@ public class ScannerService : IScannerService var scanElapsedTime = await ScanFiles(library, libraryFolderPaths, shouldUseLibraryScan, TrackFiles, forceUpdate); // NOTE: This runs sync after every file is scanned - foreach (var task in processTasks) - { - await task(); - } + // foreach (var task in processTasks) + // { + // await task(); + // } // TODO: We might be able to do Task.WhenAll await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, @@ -566,17 +569,18 @@ public class ScannerService : IScannerService BackgroundJob.Enqueue(() => _directoryService.ClearDirectory(_directoryService.TempDirectory)); return; - Task TrackFiles(Tuple> parsedInfo) + // Responsible for transforming parsedInfo into an actual ParsedSeries then calling the actual processing of the series + async Task TrackFiles(Tuple> parsedInfo) { var skippedScan = parsedInfo.Item1; var parsedFiles = parsedInfo.Item2; - if (parsedFiles.Count == 0) return Task.CompletedTask; + if (parsedFiles.Count == 0) return; var foundParsedSeries = new ParsedSeries() { Name = parsedFiles[0].Series, NormalizedName = Scanner.Parser.Parser.Normalize(parsedFiles[0].Series), - Format = parsedFiles[0].Format + Format = parsedFiles[0].Format, }; if (skippedScan) @@ -587,15 +591,22 @@ public class ScannerService : IScannerService NormalizedName = Scanner.Parser.Parser.Normalize(pf.Series), Format = pf.Format })); - return Task.CompletedTask; + return; } totalFiles += parsedFiles.Count; seenSeries.Add(foundParsedSeries); - processTasks.Add(async () => await _processSeries.ProcessSeriesAsync(parsedFiles, library, forceUpdate)); - return Task.CompletedTask; + await _seriesProcessingSemaphore.WaitAsync(); + try + { + await _processSeries.ProcessSeriesAsync(parsedFiles, library, forceUpdate); + } + finally + { + _seriesProcessingSemaphore.Release(); + } } } diff --git a/openapi.json b/openapi.json index 72c09edd8..3e8139442 100644 --- a/openapi.json +++ b/openapi.json @@ -7,7 +7,7 @@ "name": "GPL-3.0", "url": "https://github.com/Kareadita/Kavita/blob/develop/LICENSE" }, - "version": "0.7.10.9" + "version": "0.7.10.11" }, "servers": [ { @@ -2925,7 +2925,8 @@ "enum": [ 0, 1, - 2 + 2, + 3 ], "type": "integer", "format": "int32" @@ -2936,7 +2937,8 @@ "enum": [ 0, 1, - 2 + 2, + 3 ], "type": "integer", "format": "int32" @@ -2947,7 +2949,8 @@ "enum": [ 0, 1, - 2 + 2, + 3 ], "type": "integer", "format": "int32" @@ -13404,7 +13407,8 @@ "enum": [ 0, 1, - 2 + 2, + 3 ], "type": "integer", "format": "int32" @@ -13965,7 +13969,8 @@ "enum": [ 0, 1, - 2 + 2, + 3 ], "type": "integer", "description": "Library type", @@ -15457,7 +15462,8 @@ "enum": [ 0, 1, - 2 + 2, + 3 ], "type": "integer", "format": "int32" @@ -15555,7 +15561,8 @@ "enum": [ 0, 1, - 2 + 2, + 3 ], "type": "integer", "format": "int32" @@ -16488,7 +16495,8 @@ "enum": [ 0, 1, - 2 + 2, + 3 ], "type": "integer", "format": "int32" @@ -16539,7 +16547,8 @@ "enum": [ 0, 1, - 2 + 2, + 3 ], "type": "integer", "format": "int32" @@ -18795,7 +18804,8 @@ "enum": [ 0, 1, - 2 + 2, + 3 ], "type": "integer", "format": "int32"