Bugfix/scanner issue (#576)

* Refactored the scanner to hopefully fix a hard to reproduce KeyNotFoundException from  GetInfosByName.

* Added parsing support for "A Compendium of Ghosts - 031 - The Third Story_ Part 12 (Digital) (Cobalt001)"
This commit is contained in:
Joseph Milazzo 2021-09-12 09:48:21 -07:00 committed by GitHub
parent dd6dec46c1
commit a9ea03469f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 40 additions and 13 deletions

View File

@ -1,7 +1,9 @@
using System; using System;
using System.IO; using System.IO;
using API.Data;
using API.Entities.Enums; using API.Entities.Enums;
using API.Interfaces.Services; using API.Interfaces.Services;
using API.Parser;
using API.Services; using API.Services;
using API.Services.Tasks.Scanner; using API.Services.Tasks.Scanner;
using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Attributes;
@ -14,7 +16,7 @@ namespace API.Benchmark
[MemoryDiagnoser] [MemoryDiagnoser]
[Orderer(SummaryOrderPolicy.FastestToSlowest)] [Orderer(SummaryOrderPolicy.FastestToSlowest)]
[RankColumn] [RankColumn]
[SimpleJob(launchCount: 1, warmupCount: 3, targetCount: 5, invocationCount: 100, id: "Test"), ShortRunJob] //[SimpleJob(launchCount: 1, warmupCount: 3, targetCount: 5, invocationCount: 100, id: "Test"), ShortRunJob]
public class ParseScannedFilesBenchmarks public class ParseScannedFilesBenchmarks
{ {
private readonly ParseScannedFiles _parseScannedFiles; private readonly ParseScannedFiles _parseScannedFiles;
@ -27,13 +29,37 @@ namespace API.Benchmark
_parseScannedFiles = new ParseScannedFiles(bookService, _logger); _parseScannedFiles = new ParseScannedFiles(bookService, _logger);
} }
// [Benchmark]
// public void Test()
// {
// var libraryPath = Path.Join(Directory.GetCurrentDirectory(),
// "../../../Services/Test Data/ScannerService/Manga");
// var parsedSeries = _parseScannedFiles.ScanLibrariesForSeries(LibraryType.Manga, new string[] {libraryPath},
// out var totalFiles, out var scanElapsedTime);
// }
/// <summary>
/// Generate a list of Series and another list with
/// </summary>
[Benchmark] [Benchmark]
public void Test() public void MergeName()
{ {
var libraryPath = Path.Join(Directory.GetCurrentDirectory(), var libraryPath = Path.Join(Directory.GetCurrentDirectory(),
"../../../Services/Test Data/ScannerService/Manga"); "../../../Services/Test Data/ScannerService/Manga");
var p1 = new ParserInfo()
{
Chapters = "0",
Edition = "",
Format = MangaFormat.Archive,
FullFilePath = Path.Join(libraryPath, "A Town Where You Live", "A_Town_Where_You_Live_v01.zip"),
IsSpecial = false,
Series = "A Town Where You Live",
Title = "A Town Where You Live",
Volumes = "1"
};
var parsedSeries = _parseScannedFiles.ScanLibrariesForSeries(LibraryType.Manga, new string[] {libraryPath}, var parsedSeries = _parseScannedFiles.ScanLibrariesForSeries(LibraryType.Manga, new string[] {libraryPath},
out var totalFiles, out var scanElapsedTime); out var totalFiles, out var scanElapsedTime);
_parseScannedFiles.MergeName(p1);
} }
} }
} }

View File

@ -12,8 +12,8 @@ namespace API.Benchmark
{ {
static void Main(string[] args) static void Main(string[] args)
{ {
//BenchmarkRunner.Run<ParseScannedFilesBenchmarks>(); BenchmarkRunner.Run<ParseScannedFilesBenchmarks>();
BenchmarkRunner.Run<TestBenchmark>(); //BenchmarkRunner.Run<TestBenchmark>();
} }
} }
} }

View File

@ -159,6 +159,7 @@ namespace API.Tests.Parser
[InlineData("Hentai Ouji to Warawanai Neko. - Vol. 06 Ch. 034.5", "Hentai Ouji to Warawanai Neko.")] [InlineData("Hentai Ouji to Warawanai Neko. - Vol. 06 Ch. 034.5", "Hentai Ouji to Warawanai Neko.")]
[InlineData("The 100 Girlfriends Who Really, Really, Really, Really, Really Love You - Vol. 03 Ch. 023.5 - Volume 3 Extras.cbz", "The 100 Girlfriends Who Really, Really, Really, Really, Really Love You")] [InlineData("The 100 Girlfriends Who Really, Really, Really, Really, Really Love You - Vol. 03 Ch. 023.5 - Volume 3 Extras.cbz", "The 100 Girlfriends Who Really, Really, Really, Really, Really Love You")]
[InlineData("Kimi no Koto ga Daidaidaidaidaisuki na 100-nin no Kanojo Chapter 1-10", "Kimi no Koto ga Daidaidaidaidaisuki na 100-nin no Kanojo")] [InlineData("Kimi no Koto ga Daidaidaidaidaisuki na 100-nin no Kanojo Chapter 1-10", "Kimi no Koto ga Daidaidaidaidaisuki na 100-nin no Kanojo")]
[InlineData("A Compendium of Ghosts - 031 - The Third Story_ Part 12 (Digital) (Cobalt001)", "A Compendium of Ghosts")]
public void ParseSeriesTest(string filename, string expected) public void ParseSeriesTest(string filename, string expected)
{ {
Assert.Equal(expected, API.Parser.Parser.ParseSeries(filename)); Assert.Equal(expected, API.Parser.Parser.ParseSeries(filename));

View File

@ -245,9 +245,9 @@ namespace API.Parser
@"(?<Series>.*)(\s|_|-)#", @"(?<Series>.*)(\s|_|-)#",
RegexOptions.IgnoreCase | RegexOptions.Compiled, RegexOptions.IgnoreCase | RegexOptions.Compiled,
RegexTimeout), RegexTimeout),
// Baketeriya ch01-05.zip, Akiiro Bousou Biyori - 01.jpg, Beelzebub_172_RHS.zip, Cynthia the Mission 29.rar // Baketeriya ch01-05.zip, Akiiro Bousou Biyori - 01.jpg, Beelzebub_172_RHS.zip, Cynthia the Mission 29.rar, A Compendium of Ghosts - 031 - The Third Story_ Part 12 (Digital) (Cobalt001)
new Regex( new Regex(
@"^(?!Vol\.?)(?<Series>.*)( |_|-)(?<!-)(ch)?\d+-?\d*", @"^(?!Vol\.?)(?<Series>.+?)( |_|-)(?<!-)(ch)?\d+-?\d*",
RegexOptions.IgnoreCase | RegexOptions.Compiled, RegexOptions.IgnoreCase | RegexOptions.Compiled,
RegexTimeout), RegexTimeout),
// [BAA]_Darker_than_Black_c1 (This is very greedy, make sure it's close to last) // [BAA]_Darker_than_Black_c1 (This is very greedy, make sure it's close to last)

View File

@ -39,18 +39,18 @@ namespace API.Services.Tasks.Scanner
_scannedSeries = new ConcurrentDictionary<ParsedSeries, List<ParserInfo>>(); _scannedSeries = new ConcurrentDictionary<ParsedSeries, List<ParserInfo>>();
} }
/// <summary>
/// Gets the list of parserInfos given a Series. If the series does not exist within, return empty list.
/// </summary>
/// <param name="parsedSeries"></param>
/// <param name="series"></param>
/// <returns></returns>
public static IList<ParserInfo> GetInfosByName(Dictionary<ParsedSeries, List<ParserInfo>> parsedSeries, Series series) public static IList<ParserInfo> GetInfosByName(Dictionary<ParsedSeries, List<ParserInfo>> parsedSeries, Series series)
{ {
var existingKey = parsedSeries.Keys.FirstOrDefault(ps => var existingKey = parsedSeries.Keys.FirstOrDefault(ps =>
ps.Format == series.Format && ps.NormalizedName == Parser.Parser.Normalize(series.OriginalName)); ps.Format == series.Format && ps.NormalizedName == Parser.Parser.Normalize(series.OriginalName));
existingKey ??= new ParsedSeries()
{
Format = series.Format,
Name = series.OriginalName,
NormalizedName = Parser.Parser.Normalize(series.OriginalName)
};
return parsedSeries[existingKey]; return existingKey != null ? parsedSeries[existingKey] : new List<ParserInfo>();
} }
/// <summary> /// <summary>