diff --git a/API.Tests/Parser/ComicParserTests.cs b/API.Tests/Parser/ComicParserTests.cs index fa0448ff9..689327d98 100644 --- a/API.Tests/Parser/ComicParserTests.cs +++ b/API.Tests/Parser/ComicParserTests.cs @@ -194,7 +194,7 @@ public class ComicParserTests [InlineData("Asterix - HS - Les 12 travaux d'Astérix", true)] [InlineData("Sillage Hors Série - Le Collectionneur - Concordance-DKFR", true)] [InlineData("laughs", false)] - [InlineData("Annual Days of Summer", false)] + [InlineData("Annual Days of Summer", true)] [InlineData("Adventure Time 2013 Annual #001 (2013)", true)] [InlineData("Adventure Time 2013_Annual_#001 (2013)", true)] [InlineData("Adventure Time 2013_-_Annual #001 (2013)", true)] @@ -202,6 +202,13 @@ public class ComicParserTests [InlineData("Mazebook 001", false)] [InlineData("X-23 One Shot (2010)", true)] [InlineData("Casus Belli v1 Hors-Série 21 - Mousquetaires et Sorcellerie", true)] + [InlineData("Batman Beyond Annual", true)] + [InlineData("Batman Beyond Bonus", true)] + [InlineData("Batman Beyond OneShot", true)] + [InlineData("Batman Beyond Specials", true)] + [InlineData("Batman Beyond Omnibus (1999)", true)] + [InlineData("Batman Beyond Omnibus", true)] + [InlineData("01 Annual Batman Beyond", true)] public void IsComicSpecialTest(string input, bool expected) { Assert.Equal(expected, API.Services.Tasks.Scanner.Parser.Parser.IsComicSpecial(input)); diff --git a/API.Tests/Parser/DefaultParserTests.cs b/API.Tests/Parser/DefaultParserTests.cs index 2640aa6c2..7f843b552 100644 --- a/API.Tests/Parser/DefaultParserTests.cs +++ b/API.Tests/Parser/DefaultParserTests.cs @@ -46,6 +46,7 @@ public class DefaultParserTests [InlineData("/manga/Btooom!/Vol.1/Chapter 1/1.cbz", "Btooom!~1~1")] [InlineData("/manga/Btooom!/Vol.1 Chapter 2/1.cbz", "Btooom!~1~2")] [InlineData("/manga/Monster/Ch. 001-016 [MangaPlus] [Digital] [amit34521]/Monster Ch. 001 [MangaPlus] [Digital] [amit34521]/13.jpg", "Monster~0~1")] + [InlineData("/manga/Hajime no Ippo/Artbook/Hajime no Ippo - Artbook.cbz", "Hajime no Ippo~0~0")] public void ParseFromFallbackFolders_ShouldParseSeriesVolumeAndChapter(string inputFile, string expectedParseInfo) { const string rootDirectory = "/manga/"; @@ -80,6 +81,7 @@ public class DefaultParserTests [Theory] [InlineData("/manga/Btooom!/Specials/Art Book.cbz", "Btooom!")] + [InlineData("/manga/Hajime no Ippo/Artbook/Hajime no Ippo - Artbook.cbz", "Hajime no Ippo")] public void ParseFromFallbackFolders_ShouldUseExistingSeriesName_NewScanLoop(string inputFile, string expectedParseInfo) { const string rootDirectory = "/manga/"; diff --git a/API.Tests/Parser/MangaParserTests.cs b/API.Tests/Parser/MangaParserTests.cs index 89b1112f5..20c1a27ae 100644 --- a/API.Tests/Parser/MangaParserTests.cs +++ b/API.Tests/Parser/MangaParserTests.cs @@ -195,6 +195,7 @@ public class MangaParserTests [InlineData("Манга Глава 2-2", "Манга")] [InlineData("Манга Том 1 3-4 Глава", "Манга")] [InlineData("Esquire 6권 2021년 10월호", "Esquire")] + [InlineData("Accel World: Vol 1", "Accel World")] public void ParseSeriesTest(string filename, string expected) { Assert.Equal(expected, API.Services.Tasks.Scanner.Parser.Parser.ParseSeries(filename)); @@ -314,8 +315,8 @@ public class MangaParserTests [InlineData("Beastars SP01", false)] [InlineData("The League of Extraordinary Gentlemen", false)] [InlineData("The League of Extra-ordinary Gentlemen", false)] - [InlineData("Gifting The Wonderful World With Blessings! - 3 Side Stories [yuNS][Unknown].epub", true)] - [InlineData("Dr. Ramune - Mysterious Disease Specialist v01 (2020) (Digital) (danke-Empire).cbz", false)] + [InlineData("Dr. Ramune - Mysterious Disease Specialist v01 (2020) (Digital) (danke-Empire)", false)] + [InlineData("Hajime no Ippo - Artbook", false)] public void IsMangaSpecialTest(string input, bool expected) { Assert.Equal(expected, API.Services.Tasks.Scanner.Parser.Parser.IsMangaSpecial(input)); diff --git a/API.Tests/Services/ArchiveServiceTests.cs b/API.Tests/Services/ArchiveServiceTests.cs index f399cb790..b59ee097e 100644 --- a/API.Tests/Services/ArchiveServiceTests.cs +++ b/API.Tests/Services/ArchiveServiceTests.cs @@ -281,6 +281,17 @@ public class ArchiveServiceTests Assert.Equal("BTOOOM! - Duplicate", comicInfo.Series); } + [Fact] + public void ShouldHaveComicInfo_OutsideRoot_SharpCompress() + { + var testDirectory = Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/ArchiveService/ComicInfos"); + var archive = Path.Join(testDirectory, "ComicInfo_outside_root_SharpCompress.cb7"); + + var comicInfo = _archiveService.GetComicInfo(archive); + Assert.NotNull(comicInfo); + Assert.Equal("Fire Punch", comicInfo.Series); + } + #endregion #region CanParseComicInfo diff --git a/API.Tests/Services/BookServiceTests.cs b/API.Tests/Services/BookServiceTests.cs index 38a5da896..4665ab691 100644 --- a/API.Tests/Services/BookServiceTests.cs +++ b/API.Tests/Services/BookServiceTests.cs @@ -54,4 +54,28 @@ public class BookServiceTests Assert.Equal("Roger Starbuck,Junya Inoue", comicInfo.Writer); } + [Fact] + public void ShouldParseAsVolumeGroup_WithoutSeriesIndex() + { + var testDirectory = Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/BookService"); + var archive = Path.Join(testDirectory, "TitleWithVolume_NoSeriesOrSeriesIndex.epub"); + + var comicInfo = _bookService.GetComicInfo(archive); + Assert.NotNull(comicInfo); + Assert.Equal("1", comicInfo.Volume); + Assert.Equal("Accel World", comicInfo.Series); + } + + [Fact] + public void ShouldParseAsVolumeGroup_WithSeriesIndex() + { + var testDirectory = Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/BookService"); + var archive = Path.Join(testDirectory, "TitleWithVolume.epub"); + + var comicInfo = _bookService.GetComicInfo(archive); + Assert.NotNull(comicInfo); + Assert.Equal("1.0", comicInfo.Volume); + Assert.Equal("Accel World", comicInfo.Series); + } + } diff --git a/API.Tests/Services/DirectoryServiceTests.cs b/API.Tests/Services/DirectoryServiceTests.cs index 193114865..134dc2361 100644 --- a/API.Tests/Services/DirectoryServiceTests.cs +++ b/API.Tests/Services/DirectoryServiceTests.cs @@ -603,7 +603,7 @@ public class DirectoryServiceTests var ds = new DirectoryService(Substitute.For>(), fileSystem); ds.CopyFilesToDirectory(new []{MockUnixSupport.Path($"{testDirectory}file.zip")}, "/manga/output/", new [] {"01"}); var outputFiles = ds.GetFiles("/manga/output/").Select(API.Services.Tasks.Scanner.Parser.Parser.NormalizePath).ToList(); - Assert.Equal(1, outputFiles.Count()); // we have 2 already there and 2 copies + Assert.Single(outputFiles); // For some reason, this has C:/ on directory even though everything is emulated (System.IO.Abstractions issue, not changing) // https://github.com/TestableIO/System.IO.Abstractions/issues/831 Assert.True(outputFiles.Contains(API.Services.Tasks.Scanner.Parser.Parser.NormalizePath("/manga/output/01.zip")) diff --git a/API.Tests/Services/Test Data/ArchiveService/ComicInfos/ComicInfo_outside_root_SharpCompress.cb7 b/API.Tests/Services/Test Data/ArchiveService/ComicInfos/ComicInfo_outside_root_SharpCompress.cb7 new file mode 100644 index 000000000..14944cbfe Binary files /dev/null and b/API.Tests/Services/Test Data/ArchiveService/ComicInfos/ComicInfo_outside_root_SharpCompress.cb7 differ diff --git a/API.Tests/Services/Test Data/BookService/TitleWithVolume.epub b/API.Tests/Services/Test Data/BookService/TitleWithVolume.epub new file mode 100644 index 000000000..2d8c25b26 Binary files /dev/null and b/API.Tests/Services/Test Data/BookService/TitleWithVolume.epub differ diff --git a/API.Tests/Services/Test Data/BookService/TitleWithVolume_NoSeriesOrSeriesIndex.epub b/API.Tests/Services/Test Data/BookService/TitleWithVolume_NoSeriesOrSeriesIndex.epub new file mode 100644 index 000000000..56dc6f5a8 Binary files /dev/null and b/API.Tests/Services/Test Data/BookService/TitleWithVolume_NoSeriesOrSeriesIndex.epub differ diff --git a/API/API.csproj b/API/API.csproj index 4504e7804..ba8759d03 100644 --- a/API/API.csproj +++ b/API/API.csproj @@ -54,6 +54,7 @@ + diff --git a/API/Logging/LogLevelOptions.cs b/API/Logging/LogLevelOptions.cs index 34d7d353f..51ed86632 100644 --- a/API/Logging/LogLevelOptions.cs +++ b/API/Logging/LogLevelOptions.cs @@ -71,7 +71,7 @@ public static class LogLevelOptions AspNetCoreLogLevelSwitch.MinimumLevel = LogEventLevel.Warning; break; case "Information": - LogLevelSwitch.MinimumLevel = LogEventLevel.Error; + LogLevelSwitch.MinimumLevel = LogEventLevel.Information; MicrosoftLogLevelSwitch.MinimumLevel = LogEventLevel.Error; MicrosoftHostingLifetimeLogLevelSwitch.MinimumLevel = LogEventLevel.Error; AspNetCoreLogLevelSwitch.MinimumLevel = LogEventLevel.Error; diff --git a/API/Services/ArchiveService.cs b/API/Services/ArchiveService.cs index b370f178d..211d85df7 100644 --- a/API/Services/ArchiveService.cs +++ b/API/Services/ArchiveService.cs @@ -331,7 +331,7 @@ public class ArchiveService : IArchiveService private static bool IsComicInfoArchiveEntry(string fullName, string name) { return !Tasks.Scanner.Parser.Parser.HasBlacklistedFolderInPath(fullName) - && name.Equals(ComicInfoFilename, StringComparison.OrdinalIgnoreCase) + && name.EndsWith(ComicInfoFilename, StringComparison.OrdinalIgnoreCase) && !name.StartsWith(Tasks.Scanner.Parser.Parser.MacOsMetadataFileStartsWith); } diff --git a/API/Services/BookService.cs b/API/Services/BookService.cs index 728b6f8ff..8156a56ff 100644 --- a/API/Services/BookService.cs +++ b/API/Services/BookService.cs @@ -451,9 +451,22 @@ public class BookService : IBookService info.Series = metadataItem.Content; info.SeriesSort = metadataItem.Content; break; + case "calibre:series_index": + info.Volume = metadataItem.Content; + break; } } + var hasVolumeInSeries = !Tasks.Scanner.Parser.Parser.ParseVolume(info.Title) + .Equals(Tasks.Scanner.Parser.Parser.DefaultVolume); + + if (string.IsNullOrEmpty(info.Volume) && hasVolumeInSeries && (!info.Series.Equals(info.Title) || string.IsNullOrEmpty(info.Series))) + { + // This is likely a light novel for which we can set series from parsed title + info.Series = Tasks.Scanner.Parser.Parser.ParseSeries(info.Title); + info.Volume = Tasks.Scanner.Parser.Parser.ParseVolume(info.Title); + } + return info; } catch (Exception ex) diff --git a/API/Services/ReadingItemService.cs b/API/Services/ReadingItemService.cs index 3f2122a08..551d1b668 100644 --- a/API/Services/ReadingItemService.cs +++ b/API/Services/ReadingItemService.cs @@ -72,8 +72,23 @@ public class ReadingItemService : IReadingItemService // This catches when original library type is Manga/Comic and when parsing with non if (Tasks.Scanner.Parser.Parser.IsEpub(path) && Tasks.Scanner.Parser.Parser.ParseVolume(info.Series) != Tasks.Scanner.Parser.Parser.DefaultVolume) // Shouldn't this be info.Volume != DefaultVolume? { - var info2 = _defaultParser.Parse(path, rootPath, LibraryType.Book); - info.Merge(info2); + var hasVolumeInTitle = !Tasks.Scanner.Parser.Parser.ParseVolume(info.Title) + .Equals(Tasks.Scanner.Parser.Parser.DefaultVolume); + var hasVolumeInSeries = !Tasks.Scanner.Parser.Parser.ParseVolume(info.Series) + .Equals(Tasks.Scanner.Parser.Parser.DefaultVolume); + + if (string.IsNullOrEmpty(info.ComicInfo?.Volume) && hasVolumeInTitle && (hasVolumeInSeries || string.IsNullOrEmpty(info.Series))) + { + // This is likely a light novel for which we can set series from parsed title + info.Series = Tasks.Scanner.Parser.Parser.ParseSeries(info.Title); + info.Volumes = Tasks.Scanner.Parser.Parser.ParseVolume(info.Title); + } + else + { + var info2 = _defaultParser.Parse(path, rootPath, LibraryType.Book); + info.Merge(info2); + } + } info.ComicInfo = GetComicInfo(path); diff --git a/API/Services/Tasks/Scanner/LibraryWatcher.cs b/API/Services/Tasks/Scanner/LibraryWatcher.cs index fea30b7fe..fee51f562 100644 --- a/API/Services/Tasks/Scanner/LibraryWatcher.cs +++ b/API/Services/Tasks/Scanner/LibraryWatcher.cs @@ -192,6 +192,7 @@ public class LibraryWatcher : ILibraryWatcher /// This is public only because Hangfire will invoke it. Do not call external to this class. /// File or folder that changed /// If the change is on a directory and not a file + [DisableConcurrentExecution(60)] // ReSharper disable once MemberCanBePrivate.Global public async Task ProcessChange(string filePath, bool isDirectoryChange = false) { diff --git a/API/Services/Tasks/Scanner/ParseScannedFiles.cs b/API/Services/Tasks/Scanner/ParseScannedFiles.cs index dbd23d970..1a23af727 100644 --- a/API/Services/Tasks/Scanner/ParseScannedFiles.cs +++ b/API/Services/Tasks/Scanner/ParseScannedFiles.cs @@ -38,7 +38,10 @@ public class SeriesModified public IEnumerable LibraryRoots { get; set; } } - +/// +/// Responsible for taking parsed info from ReadingItemService and DirectoryService and combining them to emit DB work +/// on a series by series. +/// public class ParseScannedFiles { private readonly ILogger _logger; diff --git a/API/Services/Tasks/Scanner/Parser/DefaultParser.cs b/API/Services/Tasks/Scanner/Parser/DefaultParser.cs index 4bab428a3..072b1e44e 100644 --- a/API/Services/Tasks/Scanner/Parser/DefaultParser.cs +++ b/API/Services/Tasks/Scanner/Parser/DefaultParser.cs @@ -34,6 +34,8 @@ public class DefaultParser : IDefaultParser public ParserInfo Parse(string filePath, string rootPath, LibraryType type = LibraryType.Manga) { var fileName = _directoryService.FileSystem.Path.GetFileNameWithoutExtension(filePath); + // TODO: Potential Bug: This will return null, but on Image libraries, if all images, we would want to include this. + if (Parser.IsCoverImage(_directoryService.FileSystem.Path.GetFileName(filePath))) return null; ParserInfo ret; if (Parser.IsEpub(filePath)) @@ -62,7 +64,6 @@ public class DefaultParser : IDefaultParser }; } - if (Parser.IsCoverImage(_directoryService.FileSystem.Path.GetFileName(filePath))) return null; if (Parser.IsImage(filePath)) { diff --git a/API/Services/Tasks/Scanner/Parser/Parser.cs b/API/Services/Tasks/Scanner/Parser/Parser.cs index 8a7e16933..13cce0feb 100644 --- a/API/Services/Tasks/Scanner/Parser/Parser.cs +++ b/API/Services/Tasks/Scanner/Parser/Parser.cs @@ -200,11 +200,11 @@ public static class Parser MatchOptions, RegexTimeout), // [dmntsf.net] One Piece - Digital Colored Comics Vol. 20 Ch. 177 - 30 Million vs 81 Million.cbz new Regex( - @"(?.*) (\b|_|-)(vol)\.?(\s|-|_)?\d+", + @"(?.+?):? (\b|_|-)(vol)\.?(\s|-|_)?\d+", MatchOptions, RegexTimeout), // [xPearse] Kyochuu Rettou Volume 1 [English] [Manga] [Volume Scans] new Regex( - @"(?.*) (\b|_|-)(vol)(ume)", + @"(?.+?):? (\b|_|-)(vol)(ume)", MatchOptions, RegexTimeout), //Knights of Sidonia c000 (S2 LE BD Omake - BLAME!) [Habanero Scans] @@ -596,7 +596,7 @@ public static class Parser private static readonly Regex ComicSpecialRegex = new Regex( // All Keywords, does not account for checking if contains volume/chapter identification. Parser.Parse() will handle. - $@"\b(?:{CommonSpecial}|\d.+?\WAnnual|Annual\W\d.+?|Book \d.+?|Compendium \d.+?|Omnibus \d.+?|FCBD \d.+?|Absolute \d.+?|Preview \d.+?|Hors[ -]S[ée]rie|TPB|HS|THS)\b", + $@"\b(?:{CommonSpecial}|\d.+?(\W|-|^)Annual|Annual(\W|-|$)|Book \d.+?|Compendium(\W|-|$|\s.+?)|Omnibus(\W|-|$|\s.+?)|FCBD \d.+?|Absolute(\W|-|$|\s.+?)|Preview(\W|-|$|\s.+?)|Hors[ -]S[ée]rie|TPB|HS|THS)\b", MatchOptions, RegexTimeout ); diff --git a/API/Services/Tasks/ScannerService.cs b/API/Services/Tasks/ScannerService.cs index 18cb219e0..f934e6ba6 100644 --- a/API/Services/Tasks/ScannerService.cs +++ b/API/Services/Tasks/ScannerService.cs @@ -160,6 +160,7 @@ public class ScannerService : IScannerService var sw = Stopwatch.StartNew(); var files = await _unitOfWork.SeriesRepository.GetFilesForSeries(seriesId); var series = await _unitOfWork.SeriesRepository.GetFullSeriesForSeriesIdAsync(seriesId); + if (series == null) return; // This can occur when UI deletes a series but doesn't update and user re-requests update var chapterIds = await _unitOfWork.SeriesRepository.GetChapterIdsForSeriesAsync(new[] {seriesId}); var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(series.LibraryId, LibraryIncludes.Folders); var libraryPaths = library.Folders.Select(f => f.Path).ToList(); diff --git a/API/Startup.cs b/API/Startup.cs index 00351a3fa..a9fef97b8 100644 --- a/API/Startup.cs +++ b/API/Startup.cs @@ -177,7 +177,8 @@ public class Startup services.AddHangfire(configuration => configuration .UseSimpleAssemblyNameTypeSerializer() .UseRecommendedSerializerSettings() - .UseSQLiteStorage("config/Hangfire.db")); // UseSQLiteStorage - SQLite has some issues around resuming jobs when aborted + .UseInMemoryStorage()); + //.UseSQLiteStorage("config/Hangfire.db")); // UseSQLiteStorage - SQLite has some issues around resuming jobs when aborted (and locking can cause high utilization) // Add the processing server as IHostedService services.AddHangfireServer(options => diff --git a/UI/Web/src/app/series-detail/series-detail.component.ts b/UI/Web/src/app/series-detail/series-detail.component.ts index b342b519c..69230e135 100644 --- a/UI/Web/src/app/series-detail/series-detail.component.ts +++ b/UI/Web/src/app/series-detail/series-detail.component.ts @@ -440,7 +440,7 @@ export class SeriesDetailComponent implements OnInit, OnDestroy, AfterContentChe async deleteSeries(series: Series) { - this.actionService.deleteSeries(series, (result: boolean) => { + await this.actionService.deleteSeries(series, (result: boolean) => { this.changeDetectionRef.markForCheck(); if (result) { this.router.navigate(['library', this.libraryId]);