v0.6.1 Hotfix RC (#1635)

* Swapped out SQLite for Memory, but the one from hangfire. Added DisableConcurrentExecution on ProcessChange to avoid duplication when multiple threads execute at once.

* Fixed the Hangfire SQL issues with CPU/ram utilization some users are facing

* Fixed a case in SharpCompress fallback where an invalid ComicInfo wasn't picked up.

* When parsing epubs, if there is a volume in the epub title, try to parse and group. This is beneficial for Light Novels which are generally tagged this way.

* Fixed delete series in series detail not triggering

* Fixed some parsing logic for how we treat specials, like Annual and Omnibus.

* When scanning files, if the file is the cover image (loose leaf image), we reject it more quickly than previously.

* Added a potential bug marker

* Fixed a bug where Info was only showing Error level loggers

* Code smells
This commit is contained in:
Joe Milazzo 2022-11-05 09:56:48 -04:00 committed by GitHub
parent 6dd79d8c6a
commit 3f51cb2a02
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 96 additions and 15 deletions

View File

@ -194,7 +194,7 @@ public class ComicParserTests
[InlineData("Asterix - HS - Les 12 travaux d'Astérix", true)]
[InlineData("Sillage Hors Série - Le Collectionneur - Concordance-DKFR", true)]
[InlineData("laughs", false)]
[InlineData("Annual Days of Summer", false)]
[InlineData("Annual Days of Summer", true)]
[InlineData("Adventure Time 2013 Annual #001 (2013)", true)]
[InlineData("Adventure Time 2013_Annual_#001 (2013)", true)]
[InlineData("Adventure Time 2013_-_Annual #001 (2013)", true)]
@ -202,6 +202,13 @@ public class ComicParserTests
[InlineData("Mazebook 001", false)]
[InlineData("X-23 One Shot (2010)", true)]
[InlineData("Casus Belli v1 Hors-Série 21 - Mousquetaires et Sorcellerie", true)]
[InlineData("Batman Beyond Annual", true)]
[InlineData("Batman Beyond Bonus", true)]
[InlineData("Batman Beyond OneShot", true)]
[InlineData("Batman Beyond Specials", true)]
[InlineData("Batman Beyond Omnibus (1999)", true)]
[InlineData("Batman Beyond Omnibus", true)]
[InlineData("01 Annual Batman Beyond", true)]
public void IsComicSpecialTest(string input, bool expected)
{
Assert.Equal(expected, API.Services.Tasks.Scanner.Parser.Parser.IsComicSpecial(input));

View File

@ -46,6 +46,7 @@ public class DefaultParserTests
[InlineData("/manga/Btooom!/Vol.1/Chapter 1/1.cbz", "Btooom!~1~1")]
[InlineData("/manga/Btooom!/Vol.1 Chapter 2/1.cbz", "Btooom!~1~2")]
[InlineData("/manga/Monster/Ch. 001-016 [MangaPlus] [Digital] [amit34521]/Monster Ch. 001 [MangaPlus] [Digital] [amit34521]/13.jpg", "Monster~0~1")]
[InlineData("/manga/Hajime no Ippo/Artbook/Hajime no Ippo - Artbook.cbz", "Hajime no Ippo~0~0")]
public void ParseFromFallbackFolders_ShouldParseSeriesVolumeAndChapter(string inputFile, string expectedParseInfo)
{
const string rootDirectory = "/manga/";
@ -80,6 +81,7 @@ public class DefaultParserTests
[Theory]
[InlineData("/manga/Btooom!/Specials/Art Book.cbz", "Btooom!")]
[InlineData("/manga/Hajime no Ippo/Artbook/Hajime no Ippo - Artbook.cbz", "Hajime no Ippo")]
public void ParseFromFallbackFolders_ShouldUseExistingSeriesName_NewScanLoop(string inputFile, string expectedParseInfo)
{
const string rootDirectory = "/manga/";

View File

@ -195,6 +195,7 @@ public class MangaParserTests
[InlineData("Манга Глава 2-2", "Манга")]
[InlineData("Манга Том 1 3-4 Глава", "Манга")]
[InlineData("Esquire 6권 2021년 10월호", "Esquire")]
[InlineData("Accel World: Vol 1", "Accel World")]
public void ParseSeriesTest(string filename, string expected)
{
Assert.Equal(expected, API.Services.Tasks.Scanner.Parser.Parser.ParseSeries(filename));
@ -314,8 +315,8 @@ public class MangaParserTests
[InlineData("Beastars SP01", false)]
[InlineData("The League of Extraordinary Gentlemen", false)]
[InlineData("The League of Extra-ordinary Gentlemen", false)]
[InlineData("Gifting The Wonderful World With Blessings! - 3 Side Stories [yuNS][Unknown].epub", true)]
[InlineData("Dr. Ramune - Mysterious Disease Specialist v01 (2020) (Digital) (danke-Empire).cbz", false)]
[InlineData("Dr. Ramune - Mysterious Disease Specialist v01 (2020) (Digital) (danke-Empire)", false)]
[InlineData("Hajime no Ippo - Artbook", false)]
public void IsMangaSpecialTest(string input, bool expected)
{
Assert.Equal(expected, API.Services.Tasks.Scanner.Parser.Parser.IsMangaSpecial(input));

View File

@ -281,6 +281,17 @@ public class ArchiveServiceTests
Assert.Equal("BTOOOM! - Duplicate", comicInfo.Series);
}
[Fact]
public void ShouldHaveComicInfo_OutsideRoot_SharpCompress()
{
var testDirectory = Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/ArchiveService/ComicInfos");
var archive = Path.Join(testDirectory, "ComicInfo_outside_root_SharpCompress.cb7");
var comicInfo = _archiveService.GetComicInfo(archive);
Assert.NotNull(comicInfo);
Assert.Equal("Fire Punch", comicInfo.Series);
}
#endregion
#region CanParseComicInfo

View File

@ -54,4 +54,28 @@ public class BookServiceTests
Assert.Equal("Roger Starbuck,Junya Inoue", comicInfo.Writer);
}
[Fact]
public void ShouldParseAsVolumeGroup_WithoutSeriesIndex()
{
var testDirectory = Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/BookService");
var archive = Path.Join(testDirectory, "TitleWithVolume_NoSeriesOrSeriesIndex.epub");
var comicInfo = _bookService.GetComicInfo(archive);
Assert.NotNull(comicInfo);
Assert.Equal("1", comicInfo.Volume);
Assert.Equal("Accel World", comicInfo.Series);
}
[Fact]
public void ShouldParseAsVolumeGroup_WithSeriesIndex()
{
var testDirectory = Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/BookService");
var archive = Path.Join(testDirectory, "TitleWithVolume.epub");
var comicInfo = _bookService.GetComicInfo(archive);
Assert.NotNull(comicInfo);
Assert.Equal("1.0", comicInfo.Volume);
Assert.Equal("Accel World", comicInfo.Series);
}
}

View File

@ -603,7 +603,7 @@ public class DirectoryServiceTests
var ds = new DirectoryService(Substitute.For<ILogger<DirectoryService>>(), fileSystem);
ds.CopyFilesToDirectory(new []{MockUnixSupport.Path($"{testDirectory}file.zip")}, "/manga/output/", new [] {"01"});
var outputFiles = ds.GetFiles("/manga/output/").Select(API.Services.Tasks.Scanner.Parser.Parser.NormalizePath).ToList();
Assert.Equal(1, outputFiles.Count()); // we have 2 already there and 2 copies
Assert.Single(outputFiles);
// For some reason, this has C:/ on directory even though everything is emulated (System.IO.Abstractions issue, not changing)
// https://github.com/TestableIO/System.IO.Abstractions/issues/831
Assert.True(outputFiles.Contains(API.Services.Tasks.Scanner.Parser.Parser.NormalizePath("/manga/output/01.zip"))

View File

@ -54,6 +54,7 @@
<PackageReference Include="Flurl.Http" Version="3.2.4" />
<PackageReference Include="Hangfire" Version="1.7.31" />
<PackageReference Include="Hangfire.AspNetCore" Version="1.7.31" />
<PackageReference Include="Hangfire.InMemory" Version="0.3.4" />
<PackageReference Include="Hangfire.MaximumConcurrentExecutions" Version="1.1.0" />
<PackageReference Include="Hangfire.MemoryStorage.Core" Version="1.4.0" />
<PackageReference Include="Hangfire.Storage.SQLite" Version="0.3.2" />

View File

@ -71,7 +71,7 @@ public static class LogLevelOptions
AspNetCoreLogLevelSwitch.MinimumLevel = LogEventLevel.Warning;
break;
case "Information":
LogLevelSwitch.MinimumLevel = LogEventLevel.Error;
LogLevelSwitch.MinimumLevel = LogEventLevel.Information;
MicrosoftLogLevelSwitch.MinimumLevel = LogEventLevel.Error;
MicrosoftHostingLifetimeLogLevelSwitch.MinimumLevel = LogEventLevel.Error;
AspNetCoreLogLevelSwitch.MinimumLevel = LogEventLevel.Error;

View File

@ -331,7 +331,7 @@ public class ArchiveService : IArchiveService
private static bool IsComicInfoArchiveEntry(string fullName, string name)
{
return !Tasks.Scanner.Parser.Parser.HasBlacklistedFolderInPath(fullName)
&& name.Equals(ComicInfoFilename, StringComparison.OrdinalIgnoreCase)
&& name.EndsWith(ComicInfoFilename, StringComparison.OrdinalIgnoreCase)
&& !name.StartsWith(Tasks.Scanner.Parser.Parser.MacOsMetadataFileStartsWith);
}

View File

@ -451,9 +451,22 @@ public class BookService : IBookService
info.Series = metadataItem.Content;
info.SeriesSort = metadataItem.Content;
break;
case "calibre:series_index":
info.Volume = metadataItem.Content;
break;
}
}
var hasVolumeInSeries = !Tasks.Scanner.Parser.Parser.ParseVolume(info.Title)
.Equals(Tasks.Scanner.Parser.Parser.DefaultVolume);
if (string.IsNullOrEmpty(info.Volume) && hasVolumeInSeries && (!info.Series.Equals(info.Title) || string.IsNullOrEmpty(info.Series)))
{
// This is likely a light novel for which we can set series from parsed title
info.Series = Tasks.Scanner.Parser.Parser.ParseSeries(info.Title);
info.Volume = Tasks.Scanner.Parser.Parser.ParseVolume(info.Title);
}
return info;
}
catch (Exception ex)

View File

@ -72,8 +72,23 @@ public class ReadingItemService : IReadingItemService
// This catches when original library type is Manga/Comic and when parsing with non
if (Tasks.Scanner.Parser.Parser.IsEpub(path) && Tasks.Scanner.Parser.Parser.ParseVolume(info.Series) != Tasks.Scanner.Parser.Parser.DefaultVolume) // Shouldn't this be info.Volume != DefaultVolume?
{
var info2 = _defaultParser.Parse(path, rootPath, LibraryType.Book);
info.Merge(info2);
var hasVolumeInTitle = !Tasks.Scanner.Parser.Parser.ParseVolume(info.Title)
.Equals(Tasks.Scanner.Parser.Parser.DefaultVolume);
var hasVolumeInSeries = !Tasks.Scanner.Parser.Parser.ParseVolume(info.Series)
.Equals(Tasks.Scanner.Parser.Parser.DefaultVolume);
if (string.IsNullOrEmpty(info.ComicInfo?.Volume) && hasVolumeInTitle && (hasVolumeInSeries || string.IsNullOrEmpty(info.Series)))
{
// This is likely a light novel for which we can set series from parsed title
info.Series = Tasks.Scanner.Parser.Parser.ParseSeries(info.Title);
info.Volumes = Tasks.Scanner.Parser.Parser.ParseVolume(info.Title);
}
else
{
var info2 = _defaultParser.Parse(path, rootPath, LibraryType.Book);
info.Merge(info2);
}
}
info.ComicInfo = GetComicInfo(path);

View File

@ -192,6 +192,7 @@ public class LibraryWatcher : ILibraryWatcher
/// <remarks>This is public only because Hangfire will invoke it. Do not call external to this class.</remarks>
/// <param name="filePath">File or folder that changed</param>
/// <param name="isDirectoryChange">If the change is on a directory and not a file</param>
[DisableConcurrentExecution(60)]
// ReSharper disable once MemberCanBePrivate.Global
public async Task ProcessChange(string filePath, bool isDirectoryChange = false)
{

View File

@ -38,7 +38,10 @@ public class SeriesModified
public IEnumerable<string> LibraryRoots { get; set; }
}
/// <summary>
/// Responsible for taking parsed info from ReadingItemService and DirectoryService and combining them to emit DB work
/// on a series by series.
/// </summary>
public class ParseScannedFiles
{
private readonly ILogger _logger;

View File

@ -34,6 +34,8 @@ public class DefaultParser : IDefaultParser
public ParserInfo Parse(string filePath, string rootPath, LibraryType type = LibraryType.Manga)
{
var fileName = _directoryService.FileSystem.Path.GetFileNameWithoutExtension(filePath);
// TODO: Potential Bug: This will return null, but on Image libraries, if all images, we would want to include this.
if (Parser.IsCoverImage(_directoryService.FileSystem.Path.GetFileName(filePath))) return null;
ParserInfo ret;
if (Parser.IsEpub(filePath))
@ -62,7 +64,6 @@ public class DefaultParser : IDefaultParser
};
}
if (Parser.IsCoverImage(_directoryService.FileSystem.Path.GetFileName(filePath))) return null;
if (Parser.IsImage(filePath))
{

View File

@ -200,11 +200,11 @@ public static class Parser
MatchOptions, RegexTimeout),
// [dmntsf.net] One Piece - Digital Colored Comics Vol. 20 Ch. 177 - 30 Million vs 81 Million.cbz
new Regex(
@"(?<Series>.*) (\b|_|-)(vol)\.?(\s|-|_)?\d+",
@"(?<Series>.+?):? (\b|_|-)(vol)\.?(\s|-|_)?\d+",
MatchOptions, RegexTimeout),
// [xPearse] Kyochuu Rettou Volume 1 [English] [Manga] [Volume Scans]
new Regex(
@"(?<Series>.*) (\b|_|-)(vol)(ume)",
@"(?<Series>.+?):? (\b|_|-)(vol)(ume)",
MatchOptions,
RegexTimeout),
//Knights of Sidonia c000 (S2 LE BD Omake - BLAME!) [Habanero Scans]
@ -596,7 +596,7 @@ public static class Parser
private static readonly Regex ComicSpecialRegex = new Regex(
// All Keywords, does not account for checking if contains volume/chapter identification. Parser.Parse() will handle.
$@"\b(?:{CommonSpecial}|\d.+?\WAnnual|Annual\W\d.+?|Book \d.+?|Compendium \d.+?|Omnibus \d.+?|FCBD \d.+?|Absolute \d.+?|Preview \d.+?|Hors[ -]S[ée]rie|TPB|HS|THS)\b",
$@"\b(?:{CommonSpecial}|\d.+?(\W|-|^)Annual|Annual(\W|-|$)|Book \d.+?|Compendium(\W|-|$|\s.+?)|Omnibus(\W|-|$|\s.+?)|FCBD \d.+?|Absolute(\W|-|$|\s.+?)|Preview(\W|-|$|\s.+?)|Hors[ -]S[ée]rie|TPB|HS|THS)\b",
MatchOptions, RegexTimeout
);

View File

@ -160,6 +160,7 @@ public class ScannerService : IScannerService
var sw = Stopwatch.StartNew();
var files = await _unitOfWork.SeriesRepository.GetFilesForSeries(seriesId);
var series = await _unitOfWork.SeriesRepository.GetFullSeriesForSeriesIdAsync(seriesId);
if (series == null) return; // This can occur when UI deletes a series but doesn't update and user re-requests update
var chapterIds = await _unitOfWork.SeriesRepository.GetChapterIdsForSeriesAsync(new[] {seriesId});
var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(series.LibraryId, LibraryIncludes.Folders);
var libraryPaths = library.Folders.Select(f => f.Path).ToList();

View File

@ -177,7 +177,8 @@ public class Startup
services.AddHangfire(configuration => configuration
.UseSimpleAssemblyNameTypeSerializer()
.UseRecommendedSerializerSettings()
.UseSQLiteStorage("config/Hangfire.db")); // UseSQLiteStorage - SQLite has some issues around resuming jobs when aborted
.UseInMemoryStorage());
//.UseSQLiteStorage("config/Hangfire.db")); // UseSQLiteStorage - SQLite has some issues around resuming jobs when aborted (and locking can cause high utilization)
// Add the processing server as IHostedService
services.AddHangfireServer(options =>

View File

@ -440,7 +440,7 @@ export class SeriesDetailComponent implements OnInit, OnDestroy, AfterContentChe
async deleteSeries(series: Series) {
this.actionService.deleteSeries(series, (result: boolean) => {
await this.actionService.deleteSeries(series, (result: boolean) => {
this.changeDetectionRef.markForCheck();
if (result) {
this.router.navigate(['library', this.libraryId]);