mirror of
https://github.com/Kareadita/Kavita.git
synced 2025-07-09 03:04:19 -04:00
v0.6.1 Hotfix RC (#1635)
* Swapped out SQLite for Memory, but the one from hangfire. Added DisableConcurrentExecution on ProcessChange to avoid duplication when multiple threads execute at once. * Fixed the Hangfire SQL issues with CPU/ram utilization some users are facing * Fixed a case in SharpCompress fallback where an invalid ComicInfo wasn't picked up. * When parsing epubs, if there is a volume in the epub title, try to parse and group. This is beneficial for Light Novels which are generally tagged this way. * Fixed delete series in series detail not triggering * Fixed some parsing logic for how we treat specials, like Annual and Omnibus. * When scanning files, if the file is the cover image (loose leaf image), we reject it more quickly than previously. * Added a potential bug marker * Fixed a bug where Info was only showing Error level loggers * Code smells
This commit is contained in:
parent
6dd79d8c6a
commit
3f51cb2a02
@ -194,7 +194,7 @@ public class ComicParserTests
|
||||
[InlineData("Asterix - HS - Les 12 travaux d'Astérix", true)]
|
||||
[InlineData("Sillage Hors Série - Le Collectionneur - Concordance-DKFR", true)]
|
||||
[InlineData("laughs", false)]
|
||||
[InlineData("Annual Days of Summer", false)]
|
||||
[InlineData("Annual Days of Summer", true)]
|
||||
[InlineData("Adventure Time 2013 Annual #001 (2013)", true)]
|
||||
[InlineData("Adventure Time 2013_Annual_#001 (2013)", true)]
|
||||
[InlineData("Adventure Time 2013_-_Annual #001 (2013)", true)]
|
||||
@ -202,6 +202,13 @@ public class ComicParserTests
|
||||
[InlineData("Mazebook 001", false)]
|
||||
[InlineData("X-23 One Shot (2010)", true)]
|
||||
[InlineData("Casus Belli v1 Hors-Série 21 - Mousquetaires et Sorcellerie", true)]
|
||||
[InlineData("Batman Beyond Annual", true)]
|
||||
[InlineData("Batman Beyond Bonus", true)]
|
||||
[InlineData("Batman Beyond OneShot", true)]
|
||||
[InlineData("Batman Beyond Specials", true)]
|
||||
[InlineData("Batman Beyond Omnibus (1999)", true)]
|
||||
[InlineData("Batman Beyond Omnibus", true)]
|
||||
[InlineData("01 Annual Batman Beyond", true)]
|
||||
public void IsComicSpecialTest(string input, bool expected)
|
||||
{
|
||||
Assert.Equal(expected, API.Services.Tasks.Scanner.Parser.Parser.IsComicSpecial(input));
|
||||
|
@ -46,6 +46,7 @@ public class DefaultParserTests
|
||||
[InlineData("/manga/Btooom!/Vol.1/Chapter 1/1.cbz", "Btooom!~1~1")]
|
||||
[InlineData("/manga/Btooom!/Vol.1 Chapter 2/1.cbz", "Btooom!~1~2")]
|
||||
[InlineData("/manga/Monster/Ch. 001-016 [MangaPlus] [Digital] [amit34521]/Monster Ch. 001 [MangaPlus] [Digital] [amit34521]/13.jpg", "Monster~0~1")]
|
||||
[InlineData("/manga/Hajime no Ippo/Artbook/Hajime no Ippo - Artbook.cbz", "Hajime no Ippo~0~0")]
|
||||
public void ParseFromFallbackFolders_ShouldParseSeriesVolumeAndChapter(string inputFile, string expectedParseInfo)
|
||||
{
|
||||
const string rootDirectory = "/manga/";
|
||||
@ -80,6 +81,7 @@ public class DefaultParserTests
|
||||
|
||||
[Theory]
|
||||
[InlineData("/manga/Btooom!/Specials/Art Book.cbz", "Btooom!")]
|
||||
[InlineData("/manga/Hajime no Ippo/Artbook/Hajime no Ippo - Artbook.cbz", "Hajime no Ippo")]
|
||||
public void ParseFromFallbackFolders_ShouldUseExistingSeriesName_NewScanLoop(string inputFile, string expectedParseInfo)
|
||||
{
|
||||
const string rootDirectory = "/manga/";
|
||||
|
@ -195,6 +195,7 @@ public class MangaParserTests
|
||||
[InlineData("Манга Глава 2-2", "Манга")]
|
||||
[InlineData("Манга Том 1 3-4 Глава", "Манга")]
|
||||
[InlineData("Esquire 6권 2021년 10월호", "Esquire")]
|
||||
[InlineData("Accel World: Vol 1", "Accel World")]
|
||||
public void ParseSeriesTest(string filename, string expected)
|
||||
{
|
||||
Assert.Equal(expected, API.Services.Tasks.Scanner.Parser.Parser.ParseSeries(filename));
|
||||
@ -314,8 +315,8 @@ public class MangaParserTests
|
||||
[InlineData("Beastars SP01", false)]
|
||||
[InlineData("The League of Extraordinary Gentlemen", false)]
|
||||
[InlineData("The League of Extra-ordinary Gentlemen", false)]
|
||||
[InlineData("Gifting The Wonderful World With Blessings! - 3 Side Stories [yuNS][Unknown].epub", true)]
|
||||
[InlineData("Dr. Ramune - Mysterious Disease Specialist v01 (2020) (Digital) (danke-Empire).cbz", false)]
|
||||
[InlineData("Dr. Ramune - Mysterious Disease Specialist v01 (2020) (Digital) (danke-Empire)", false)]
|
||||
[InlineData("Hajime no Ippo - Artbook", false)]
|
||||
public void IsMangaSpecialTest(string input, bool expected)
|
||||
{
|
||||
Assert.Equal(expected, API.Services.Tasks.Scanner.Parser.Parser.IsMangaSpecial(input));
|
||||
|
@ -281,6 +281,17 @@ public class ArchiveServiceTests
|
||||
Assert.Equal("BTOOOM! - Duplicate", comicInfo.Series);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ShouldHaveComicInfo_OutsideRoot_SharpCompress()
|
||||
{
|
||||
var testDirectory = Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/ArchiveService/ComicInfos");
|
||||
var archive = Path.Join(testDirectory, "ComicInfo_outside_root_SharpCompress.cb7");
|
||||
|
||||
var comicInfo = _archiveService.GetComicInfo(archive);
|
||||
Assert.NotNull(comicInfo);
|
||||
Assert.Equal("Fire Punch", comicInfo.Series);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region CanParseComicInfo
|
||||
|
@ -54,4 +54,28 @@ public class BookServiceTests
|
||||
Assert.Equal("Roger Starbuck,Junya Inoue", comicInfo.Writer);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ShouldParseAsVolumeGroup_WithoutSeriesIndex()
|
||||
{
|
||||
var testDirectory = Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/BookService");
|
||||
var archive = Path.Join(testDirectory, "TitleWithVolume_NoSeriesOrSeriesIndex.epub");
|
||||
|
||||
var comicInfo = _bookService.GetComicInfo(archive);
|
||||
Assert.NotNull(comicInfo);
|
||||
Assert.Equal("1", comicInfo.Volume);
|
||||
Assert.Equal("Accel World", comicInfo.Series);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ShouldParseAsVolumeGroup_WithSeriesIndex()
|
||||
{
|
||||
var testDirectory = Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/BookService");
|
||||
var archive = Path.Join(testDirectory, "TitleWithVolume.epub");
|
||||
|
||||
var comicInfo = _bookService.GetComicInfo(archive);
|
||||
Assert.NotNull(comicInfo);
|
||||
Assert.Equal("1.0", comicInfo.Volume);
|
||||
Assert.Equal("Accel World", comicInfo.Series);
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -603,7 +603,7 @@ public class DirectoryServiceTests
|
||||
var ds = new DirectoryService(Substitute.For<ILogger<DirectoryService>>(), fileSystem);
|
||||
ds.CopyFilesToDirectory(new []{MockUnixSupport.Path($"{testDirectory}file.zip")}, "/manga/output/", new [] {"01"});
|
||||
var outputFiles = ds.GetFiles("/manga/output/").Select(API.Services.Tasks.Scanner.Parser.Parser.NormalizePath).ToList();
|
||||
Assert.Equal(1, outputFiles.Count()); // we have 2 already there and 2 copies
|
||||
Assert.Single(outputFiles);
|
||||
// For some reason, this has C:/ on directory even though everything is emulated (System.IO.Abstractions issue, not changing)
|
||||
// https://github.com/TestableIO/System.IO.Abstractions/issues/831
|
||||
Assert.True(outputFiles.Contains(API.Services.Tasks.Scanner.Parser.Parser.NormalizePath("/manga/output/01.zip"))
|
||||
|
Binary file not shown.
BIN
API.Tests/Services/Test Data/BookService/TitleWithVolume.epub
Normal file
BIN
API.Tests/Services/Test Data/BookService/TitleWithVolume.epub
Normal file
Binary file not shown.
Binary file not shown.
@ -54,6 +54,7 @@
|
||||
<PackageReference Include="Flurl.Http" Version="3.2.4" />
|
||||
<PackageReference Include="Hangfire" Version="1.7.31" />
|
||||
<PackageReference Include="Hangfire.AspNetCore" Version="1.7.31" />
|
||||
<PackageReference Include="Hangfire.InMemory" Version="0.3.4" />
|
||||
<PackageReference Include="Hangfire.MaximumConcurrentExecutions" Version="1.1.0" />
|
||||
<PackageReference Include="Hangfire.MemoryStorage.Core" Version="1.4.0" />
|
||||
<PackageReference Include="Hangfire.Storage.SQLite" Version="0.3.2" />
|
||||
|
@ -71,7 +71,7 @@ public static class LogLevelOptions
|
||||
AspNetCoreLogLevelSwitch.MinimumLevel = LogEventLevel.Warning;
|
||||
break;
|
||||
case "Information":
|
||||
LogLevelSwitch.MinimumLevel = LogEventLevel.Error;
|
||||
LogLevelSwitch.MinimumLevel = LogEventLevel.Information;
|
||||
MicrosoftLogLevelSwitch.MinimumLevel = LogEventLevel.Error;
|
||||
MicrosoftHostingLifetimeLogLevelSwitch.MinimumLevel = LogEventLevel.Error;
|
||||
AspNetCoreLogLevelSwitch.MinimumLevel = LogEventLevel.Error;
|
||||
|
@ -331,7 +331,7 @@ public class ArchiveService : IArchiveService
|
||||
private static bool IsComicInfoArchiveEntry(string fullName, string name)
|
||||
{
|
||||
return !Tasks.Scanner.Parser.Parser.HasBlacklistedFolderInPath(fullName)
|
||||
&& name.Equals(ComicInfoFilename, StringComparison.OrdinalIgnoreCase)
|
||||
&& name.EndsWith(ComicInfoFilename, StringComparison.OrdinalIgnoreCase)
|
||||
&& !name.StartsWith(Tasks.Scanner.Parser.Parser.MacOsMetadataFileStartsWith);
|
||||
}
|
||||
|
||||
|
@ -451,9 +451,22 @@ public class BookService : IBookService
|
||||
info.Series = metadataItem.Content;
|
||||
info.SeriesSort = metadataItem.Content;
|
||||
break;
|
||||
case "calibre:series_index":
|
||||
info.Volume = metadataItem.Content;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
var hasVolumeInSeries = !Tasks.Scanner.Parser.Parser.ParseVolume(info.Title)
|
||||
.Equals(Tasks.Scanner.Parser.Parser.DefaultVolume);
|
||||
|
||||
if (string.IsNullOrEmpty(info.Volume) && hasVolumeInSeries && (!info.Series.Equals(info.Title) || string.IsNullOrEmpty(info.Series)))
|
||||
{
|
||||
// This is likely a light novel for which we can set series from parsed title
|
||||
info.Series = Tasks.Scanner.Parser.Parser.ParseSeries(info.Title);
|
||||
info.Volume = Tasks.Scanner.Parser.Parser.ParseVolume(info.Title);
|
||||
}
|
||||
|
||||
return info;
|
||||
}
|
||||
catch (Exception ex)
|
||||
|
@ -72,8 +72,23 @@ public class ReadingItemService : IReadingItemService
|
||||
// This catches when original library type is Manga/Comic and when parsing with non
|
||||
if (Tasks.Scanner.Parser.Parser.IsEpub(path) && Tasks.Scanner.Parser.Parser.ParseVolume(info.Series) != Tasks.Scanner.Parser.Parser.DefaultVolume) // Shouldn't this be info.Volume != DefaultVolume?
|
||||
{
|
||||
var info2 = _defaultParser.Parse(path, rootPath, LibraryType.Book);
|
||||
info.Merge(info2);
|
||||
var hasVolumeInTitle = !Tasks.Scanner.Parser.Parser.ParseVolume(info.Title)
|
||||
.Equals(Tasks.Scanner.Parser.Parser.DefaultVolume);
|
||||
var hasVolumeInSeries = !Tasks.Scanner.Parser.Parser.ParseVolume(info.Series)
|
||||
.Equals(Tasks.Scanner.Parser.Parser.DefaultVolume);
|
||||
|
||||
if (string.IsNullOrEmpty(info.ComicInfo?.Volume) && hasVolumeInTitle && (hasVolumeInSeries || string.IsNullOrEmpty(info.Series)))
|
||||
{
|
||||
// This is likely a light novel for which we can set series from parsed title
|
||||
info.Series = Tasks.Scanner.Parser.Parser.ParseSeries(info.Title);
|
||||
info.Volumes = Tasks.Scanner.Parser.Parser.ParseVolume(info.Title);
|
||||
}
|
||||
else
|
||||
{
|
||||
var info2 = _defaultParser.Parse(path, rootPath, LibraryType.Book);
|
||||
info.Merge(info2);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
info.ComicInfo = GetComicInfo(path);
|
||||
|
@ -192,6 +192,7 @@ public class LibraryWatcher : ILibraryWatcher
|
||||
/// <remarks>This is public only because Hangfire will invoke it. Do not call external to this class.</remarks>
|
||||
/// <param name="filePath">File or folder that changed</param>
|
||||
/// <param name="isDirectoryChange">If the change is on a directory and not a file</param>
|
||||
[DisableConcurrentExecution(60)]
|
||||
// ReSharper disable once MemberCanBePrivate.Global
|
||||
public async Task ProcessChange(string filePath, bool isDirectoryChange = false)
|
||||
{
|
||||
|
@ -38,7 +38,10 @@ public class SeriesModified
|
||||
public IEnumerable<string> LibraryRoots { get; set; }
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Responsible for taking parsed info from ReadingItemService and DirectoryService and combining them to emit DB work
|
||||
/// on a series by series.
|
||||
/// </summary>
|
||||
public class ParseScannedFiles
|
||||
{
|
||||
private readonly ILogger _logger;
|
||||
|
@ -34,6 +34,8 @@ public class DefaultParser : IDefaultParser
|
||||
public ParserInfo Parse(string filePath, string rootPath, LibraryType type = LibraryType.Manga)
|
||||
{
|
||||
var fileName = _directoryService.FileSystem.Path.GetFileNameWithoutExtension(filePath);
|
||||
// TODO: Potential Bug: This will return null, but on Image libraries, if all images, we would want to include this.
|
||||
if (Parser.IsCoverImage(_directoryService.FileSystem.Path.GetFileName(filePath))) return null;
|
||||
ParserInfo ret;
|
||||
|
||||
if (Parser.IsEpub(filePath))
|
||||
@ -62,7 +64,6 @@ public class DefaultParser : IDefaultParser
|
||||
};
|
||||
}
|
||||
|
||||
if (Parser.IsCoverImage(_directoryService.FileSystem.Path.GetFileName(filePath))) return null;
|
||||
|
||||
if (Parser.IsImage(filePath))
|
||||
{
|
||||
|
@ -200,11 +200,11 @@ public static class Parser
|
||||
MatchOptions, RegexTimeout),
|
||||
// [dmntsf.net] One Piece - Digital Colored Comics Vol. 20 Ch. 177 - 30 Million vs 81 Million.cbz
|
||||
new Regex(
|
||||
@"(?<Series>.*) (\b|_|-)(vol)\.?(\s|-|_)?\d+",
|
||||
@"(?<Series>.+?):? (\b|_|-)(vol)\.?(\s|-|_)?\d+",
|
||||
MatchOptions, RegexTimeout),
|
||||
// [xPearse] Kyochuu Rettou Volume 1 [English] [Manga] [Volume Scans]
|
||||
new Regex(
|
||||
@"(?<Series>.*) (\b|_|-)(vol)(ume)",
|
||||
@"(?<Series>.+?):? (\b|_|-)(vol)(ume)",
|
||||
MatchOptions,
|
||||
RegexTimeout),
|
||||
//Knights of Sidonia c000 (S2 LE BD Omake - BLAME!) [Habanero Scans]
|
||||
@ -596,7 +596,7 @@ public static class Parser
|
||||
|
||||
private static readonly Regex ComicSpecialRegex = new Regex(
|
||||
// All Keywords, does not account for checking if contains volume/chapter identification. Parser.Parse() will handle.
|
||||
$@"\b(?:{CommonSpecial}|\d.+?\WAnnual|Annual\W\d.+?|Book \d.+?|Compendium \d.+?|Omnibus \d.+?|FCBD \d.+?|Absolute \d.+?|Preview \d.+?|Hors[ -]S[ée]rie|TPB|HS|THS)\b",
|
||||
$@"\b(?:{CommonSpecial}|\d.+?(\W|-|^)Annual|Annual(\W|-|$)|Book \d.+?|Compendium(\W|-|$|\s.+?)|Omnibus(\W|-|$|\s.+?)|FCBD \d.+?|Absolute(\W|-|$|\s.+?)|Preview(\W|-|$|\s.+?)|Hors[ -]S[ée]rie|TPB|HS|THS)\b",
|
||||
MatchOptions, RegexTimeout
|
||||
);
|
||||
|
||||
|
@ -160,6 +160,7 @@ public class ScannerService : IScannerService
|
||||
var sw = Stopwatch.StartNew();
|
||||
var files = await _unitOfWork.SeriesRepository.GetFilesForSeries(seriesId);
|
||||
var series = await _unitOfWork.SeriesRepository.GetFullSeriesForSeriesIdAsync(seriesId);
|
||||
if (series == null) return; // This can occur when UI deletes a series but doesn't update and user re-requests update
|
||||
var chapterIds = await _unitOfWork.SeriesRepository.GetChapterIdsForSeriesAsync(new[] {seriesId});
|
||||
var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(series.LibraryId, LibraryIncludes.Folders);
|
||||
var libraryPaths = library.Folders.Select(f => f.Path).ToList();
|
||||
|
@ -177,7 +177,8 @@ public class Startup
|
||||
services.AddHangfire(configuration => configuration
|
||||
.UseSimpleAssemblyNameTypeSerializer()
|
||||
.UseRecommendedSerializerSettings()
|
||||
.UseSQLiteStorage("config/Hangfire.db")); // UseSQLiteStorage - SQLite has some issues around resuming jobs when aborted
|
||||
.UseInMemoryStorage());
|
||||
//.UseSQLiteStorage("config/Hangfire.db")); // UseSQLiteStorage - SQLite has some issues around resuming jobs when aborted (and locking can cause high utilization)
|
||||
|
||||
// Add the processing server as IHostedService
|
||||
services.AddHangfireServer(options =>
|
||||
|
@ -440,7 +440,7 @@ export class SeriesDetailComponent implements OnInit, OnDestroy, AfterContentChe
|
||||
|
||||
|
||||
async deleteSeries(series: Series) {
|
||||
this.actionService.deleteSeries(series, (result: boolean) => {
|
||||
await this.actionService.deleteSeries(series, (result: boolean) => {
|
||||
this.changeDetectionRef.markForCheck();
|
||||
if (result) {
|
||||
this.router.navigate(['library', this.libraryId]);
|
||||
|
Loading…
x
Reference in New Issue
Block a user