From c5e5aa19d5f811fb8df2551c16ab7dcdfa8cbac7 Mon Sep 17 00:00:00 2001 From: Joseph Milazzo Date: Sat, 8 Jan 2022 11:36:47 -0800 Subject: [PATCH] Misc Fixes (#914) * Fixed the book reader off by one issue with loading last page * Fixed a case where scanner would not delete a series if another series with same name but different format was added in that same scan. * Added some missing tag generation (chapter language and summary) --- API.Tests/Helpers/CacheHelperTests.cs | 3 - API.Tests/Helpers/ParserInfoFactory.cs | 52 +++++++++++++- API.Tests/Helpers/ParserInfoHelperTests.cs | 75 +++++++++++++++++++ API.Tests/Parser/ComicParserTests.cs | 4 ++ API.Tests/Parser/ParserInfoTests.cs | 18 ++--- API.Tests/Services/DirectoryServiceTests.cs | 12 ---- API.Tests/Services/ScannerServiceTests.cs | 79 +++++++++------------ API/Entities/Chapter.cs | 1 - API/Helpers/ParserInfoHelpers.cs | 50 +++++++++++++ API/Services/MetadataService.cs | 12 +++- API/Services/Tasks/ScannerService.cs | 36 +--------- 11 files changed, 232 insertions(+), 110 deletions(-) create mode 100644 API.Tests/Helpers/ParserInfoHelperTests.cs create mode 100644 API/Helpers/ParserInfoHelpers.cs diff --git a/API.Tests/Helpers/CacheHelperTests.cs b/API.Tests/Helpers/CacheHelperTests.cs index ba9a8ad45..55580eb49 100644 --- a/API.Tests/Helpers/CacheHelperTests.cs +++ b/API.Tests/Helpers/CacheHelperTests.cs @@ -1,13 +1,10 @@ using System; using System.Collections.Generic; using System.IO; -using System.IO.Abstractions; using System.IO.Abstractions.TestingHelpers; using API.Entities; using API.Helpers; using API.Services; -using Microsoft.Extensions.Logging; -using NSubstitute; using Xunit; namespace API.Tests.Helpers; diff --git a/API.Tests/Helpers/ParserInfoFactory.cs b/API.Tests/Helpers/ParserInfoFactory.cs index 7dcf564e1..2dc2f2869 100644 --- a/API.Tests/Helpers/ParserInfoFactory.cs +++ b/API.Tests/Helpers/ParserInfoFactory.cs @@ -1,6 +1,10 @@ -using System.IO; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Linq; using API.Entities.Enums; using API.Parser; +using API.Services.Tasks.Scanner; namespace API.Tests.Helpers { @@ -21,5 +25,49 @@ namespace API.Tests.Helpers Volumes = volumes }; } + + public static void AddToParsedInfo(IDictionary> collectedSeries, ParserInfo info) + { + var existingKey = collectedSeries.Keys.FirstOrDefault(ps => + ps.Format == info.Format && ps.NormalizedName == API.Parser.Parser.Normalize(info.Series)); + existingKey ??= new ParsedSeries() + { + Format = info.Format, + Name = info.Series, + NormalizedName = API.Parser.Parser.Normalize(info.Series) + }; + if (collectedSeries.GetType() == typeof(ConcurrentDictionary<,>)) + { + ((ConcurrentDictionary>) collectedSeries).AddOrUpdate(existingKey, new List() {info}, (_, oldValue) => + { + oldValue ??= new List(); + if (!oldValue.Contains(info)) + { + oldValue.Add(info); + } + + return oldValue; + }); + } + else + { + if (!collectedSeries.ContainsKey(existingKey)) + { + collectedSeries.Add(existingKey, new List() {info}); + } + else + { + var list = collectedSeries[existingKey]; + if (!list.Contains(info)) + { + list.Add(info); + } + + collectedSeries[existingKey] = list; + } + + } + + } } -} \ No newline at end of file +} diff --git a/API.Tests/Helpers/ParserInfoHelperTests.cs b/API.Tests/Helpers/ParserInfoHelperTests.cs new file mode 100644 index 000000000..d3b58d96b --- /dev/null +++ b/API.Tests/Helpers/ParserInfoHelperTests.cs @@ -0,0 +1,75 @@ +using System.Collections.Generic; +using API.Entities; +using API.Entities.Enums; +using API.Entities.Metadata; +using API.Helpers; +using API.Parser; +using API.Services.Tasks.Scanner; +using Xunit; + +namespace API.Tests.Helpers; + +public class ParserInfoHelperTests +{ + #region SeriesHasMatchingParserInfoFormat + + [Fact] + public void SeriesHasMatchingParserInfoFormat_ShouldBeFalse() + { + var infos = new Dictionary>(); + + ParserInfoFactory.AddToParsedInfo(infos, new ParserInfo() {Series = "Darker than Black", Volumes = "1", Format = MangaFormat.Archive}); + //AddToParsedInfo(infos, new ParserInfo() {Series = "Darker than Black", Volumes = "1", Format = MangaFormat.Epub}); + + var series = new Series() + { + Name = "Darker Than Black", + LocalizedName = "Darker Than Black", + OriginalName = "Darker Than Black", + Volumes = new List() + { + new Volume() + { + Number = 1, + Name = "1" + } + }, + NormalizedName = API.Parser.Parser.Normalize("Darker Than Black"), + Metadata = new SeriesMetadata(), + Format = MangaFormat.Epub + }; + + Assert.False(ParserInfoHelpers.SeriesHasMatchingParserInfoFormat(series, infos)); + } + + [Fact] + public void SeriesHasMatchingParserInfoFormat_ShouldBeTrue() + { + var infos = new Dictionary>(); + + ParserInfoFactory.AddToParsedInfo(infos, new ParserInfo() {Series = "Darker than Black", Volumes = "1", Format = MangaFormat.Archive}); + ParserInfoFactory.AddToParsedInfo(infos, new ParserInfo() {Series = "Darker than Black", Volumes = "1", Format = MangaFormat.Epub}); + + var series = new Series() + { + Name = "Darker Than Black", + LocalizedName = "Darker Than Black", + OriginalName = "Darker Than Black", + Volumes = new List() + { + new Volume() + { + Number = 1, + Name = "1" + } + }, + NormalizedName = API.Parser.Parser.Normalize("Darker Than Black"), + Metadata = new SeriesMetadata(), + Format = MangaFormat.Epub + }; + + Assert.True(ParserInfoHelpers.SeriesHasMatchingParserInfoFormat(series, infos)); + } + + #endregion +} diff --git a/API.Tests/Parser/ComicParserTests.cs b/API.Tests/Parser/ComicParserTests.cs index aa7b01294..afff938c3 100644 --- a/API.Tests/Parser/ComicParserTests.cs +++ b/API.Tests/Parser/ComicParserTests.cs @@ -124,6 +124,8 @@ namespace API.Tests.Parser [InlineData("Conquistador_Tome_2", "2")] [InlineData("Max_l_explorateur-_Tome_0", "0")] [InlineData("Chevaliers d'Héliopolis T3 - Rubedo, l'oeuvre au rouge (Jodorowsky & Jérémy)", "3")] + [InlineData("Adventure Time (2012)/Adventure Time #1 (2012)", "0")] + [InlineData("Adventure Time TPB (2012)/Adventure Time v01 (2012).cbz", "1")] public void ParseComicVolumeTest(string filename, string expected) { Assert.Equal(expected, API.Parser.Parser.ParseComicVolume(filename)); @@ -167,6 +169,8 @@ namespace API.Tests.Parser [InlineData("2000 AD 0366 [1984-04-28] (flopbie)", "366")] [InlineData("Daredevil - v6 - 10 - (2019)", "10")] [InlineData("Batman Beyond 2016 - Chapter 001.cbz", "1")] + [InlineData("Adventure Time (2012)/Adventure Time #1 (2012)", "1")] + [InlineData("Adventure Time TPB (2012)/Adventure Time v01 (2012).cbz", "0")] public void ParseComicChapterTest(string filename, string expected) { Assert.Equal(expected, API.Parser.Parser.ParseComicChapter(filename)); diff --git a/API.Tests/Parser/ParserInfoTests.cs b/API.Tests/Parser/ParserInfoTests.cs index 78b879de7..16906cf55 100644 --- a/API.Tests/Parser/ParserInfoTests.cs +++ b/API.Tests/Parser/ParserInfoTests.cs @@ -20,7 +20,7 @@ namespace API.Tests.Parser Title = "darker than black", Volumes = "0" }; - + var p2 = new ParserInfo() { Chapters = "1", @@ -32,7 +32,7 @@ namespace API.Tests.Parser Title = "Darker Than Black", Volumes = "0" }; - + var expected = new ParserInfo() { Chapters = "1", @@ -45,11 +45,11 @@ namespace API.Tests.Parser Volumes = "0" }; p1.Merge(p2); - + AssertSame(expected, p1); } - + [Fact] public void MergeFromTest2() { @@ -64,7 +64,7 @@ namespace API.Tests.Parser Title = "darker than black", Volumes = "0" }; - + var p2 = new ParserInfo() { Chapters = "0", @@ -76,7 +76,7 @@ namespace API.Tests.Parser Title = "Darker Than Black", Volumes = "1" }; - + var expected = new ParserInfo() { Chapters = "1", @@ -93,9 +93,9 @@ namespace API.Tests.Parser AssertSame(expected, p1); } - - private void AssertSame(ParserInfo expected, ParserInfo actual) + + private static void AssertSame(ParserInfo expected, ParserInfo actual) { Assert.Equal(expected.Chapters, actual.Chapters); Assert.Equal(expected.Volumes, actual.Volumes); @@ -107,4 +107,4 @@ namespace API.Tests.Parser Assert.Equal(expected.FullFilePath, actual.FullFilePath); } } -} \ No newline at end of file +} diff --git a/API.Tests/Services/DirectoryServiceTests.cs b/API.Tests/Services/DirectoryServiceTests.cs index ba6002141..4900425de 100644 --- a/API.Tests/Services/DirectoryServiceTests.cs +++ b/API.Tests/Services/DirectoryServiceTests.cs @@ -16,20 +16,8 @@ namespace API.Tests.Services public class DirectoryServiceTests { - private readonly DirectoryService _directoryService; private readonly ILogger _logger = Substitute.For>(); - public DirectoryServiceTests() - { - var filesystem = new MockFileSystem() - { - - }; - - _directoryService = new DirectoryService(_logger, filesystem); - } - - #region TraverseTreeParallelForEach [Fact] public void TraverseTreeParallelForEach_JustArchives_ShouldBe28() diff --git a/API.Tests/Services/ScannerServiceTests.cs b/API.Tests/Services/ScannerServiceTests.cs index 9e3c5d81c..b78c6be35 100644 --- a/API.Tests/Services/ScannerServiceTests.cs +++ b/API.Tests/Services/ScannerServiceTests.cs @@ -16,6 +16,7 @@ using API.Services; using API.Services.Tasks; using API.Services.Tasks.Scanner; using API.SignalR; +using API.Tests.Helpers; using AutoMapper; using Microsoft.AspNetCore.SignalR; using Microsoft.Data.Sqlite; @@ -30,9 +31,35 @@ namespace API.Tests.Services public class ScannerServiceTests { [Fact] - public void AddOrUpdateFileForChapter() + public void FindSeriesNotOnDisk_Should_Remove1() { - // TODO: This can be tested, it has _filesystem mocked + var infos = new Dictionary>(); + + ParserInfoFactory.AddToParsedInfo(infos, new ParserInfo() {Series = "Darker than Black", Volumes = "1", Format = MangaFormat.Archive}); + //AddToParsedInfo(infos, new ParserInfo() {Series = "Darker than Black", Volumes = "1", Format = MangaFormat.Epub}); + + var existingSeries = new List + { + new Series() + { + Name = "Darker Than Black", + LocalizedName = "Darker Than Black", + OriginalName = "Darker Than Black", + Volumes = new List() + { + new Volume() + { + Number = 1, + Name = "1" + } + }, + NormalizedName = API.Parser.Parser.Normalize("Darker Than Black"), + Metadata = new SeriesMetadata(), + Format = MangaFormat.Epub + } + }; + + Assert.Equal(1, ScannerService.FindSeriesNotOnDisk(existingSeries, infos).Count()); } [Fact] @@ -40,9 +67,9 @@ namespace API.Tests.Services { var infos = new Dictionary>(); - AddToParsedInfo(infos, new ParserInfo() {Series = "Darker than Black", Format = MangaFormat.Archive}); - AddToParsedInfo(infos, new ParserInfo() {Series = "Cage of Eden", Volumes = "1", Format = MangaFormat.Archive}); - AddToParsedInfo(infos, new ParserInfo() {Series = "Cage of Eden", Volumes = "10", Format = MangaFormat.Archive}); + ParserInfoFactory.AddToParsedInfo(infos, new ParserInfo() {Series = "Darker than Black", Format = MangaFormat.Archive}); + ParserInfoFactory.AddToParsedInfo(infos, new ParserInfo() {Series = "Cage of Eden", Volumes = "1", Format = MangaFormat.Archive}); + ParserInfoFactory.AddToParsedInfo(infos, new ParserInfo() {Series = "Cage of Eden", Volumes = "10", Format = MangaFormat.Archive}); var existingSeries = new List { @@ -114,48 +141,6 @@ namespace API.Tests.Services // Assert.Equal(missingSeries.Count, removeCount); // } - private void AddToParsedInfo(IDictionary> collectedSeries, ParserInfo info) - { - var existingKey = collectedSeries.Keys.FirstOrDefault(ps => - ps.Format == info.Format && ps.NormalizedName == API.Parser.Parser.Normalize(info.Series)); - existingKey ??= new ParsedSeries() - { - Format = info.Format, - Name = info.Series, - NormalizedName = API.Parser.Parser.Normalize(info.Series) - }; - if (collectedSeries.GetType() == typeof(ConcurrentDictionary<,>)) - { - ((ConcurrentDictionary>) collectedSeries).AddOrUpdate(existingKey, new List() {info}, (_, oldValue) => - { - oldValue ??= new List(); - if (!oldValue.Contains(info)) - { - oldValue.Add(info); - } - return oldValue; - }); - } - else - { - if (!collectedSeries.ContainsKey(existingKey)) - { - collectedSeries.Add(existingKey, new List() {info}); - } - else - { - var list = collectedSeries[existingKey]; - if (!list.Contains(info)) - { - list.Add(info); - } - - collectedSeries[existingKey] = list; - } - - } - - } } } diff --git a/API/Entities/Chapter.cs b/API/Entities/Chapter.cs index 56c3a259e..1aeeed3d1 100644 --- a/API/Entities/Chapter.cs +++ b/API/Entities/Chapter.cs @@ -46,7 +46,6 @@ namespace API.Entities /// public AgeRating AgeRating { get; set; } - /// /// Chapter title /// diff --git a/API/Helpers/ParserInfoHelpers.cs b/API/Helpers/ParserInfoHelpers.cs new file mode 100644 index 000000000..48421cd70 --- /dev/null +++ b/API/Helpers/ParserInfoHelpers.cs @@ -0,0 +1,50 @@ +using System.Collections.Generic; +using API.Entities; +using API.Entities.Enums; +using API.Extensions; +using API.Parser; +using API.Services.Tasks.Scanner; + +namespace API.Helpers; + +public static class ParserInfoHelpers +{ + /// + /// Checks each parser info to see if there is a name match and if so, checks if the format matches the Series object. + /// This accounts for if the Series has an Unknown type and if so, considers it matching. + /// + /// + /// + /// + public static bool SeriesHasMatchingParserInfoFormat(Series series, + Dictionary> parsedSeries) + { + var format = MangaFormat.Unknown; + foreach (var pSeries in parsedSeries.Keys) + { + var name = pSeries.Name; + var normalizedName = Parser.Parser.Normalize(name); + + //if (series.NameInParserInfo(pSeries.)) + if (normalizedName == series.NormalizedName || + normalizedName == Parser.Parser.Normalize(series.Name) || + name == series.Name || name == series.LocalizedName || + name == series.OriginalName || + normalizedName == Parser.Parser.Normalize(series.OriginalName)) + { + format = pSeries.Format; + if (format == series.Format) + { + return true; + } + } + } + + if (series.Format == MangaFormat.Unknown && format != MangaFormat.Unknown) + { + return true; + } + + return format == series.Format; + } +} diff --git a/API/Services/MetadataService.cs b/API/Services/MetadataService.cs index 975da1261..eb7d171d0 100644 --- a/API/Services/MetadataService.cs +++ b/API/Services/MetadataService.cs @@ -97,6 +97,16 @@ public class MetadataService : IMetadataService chapter.TitleName = comicInfo.Title.Trim(); } + if (!string.IsNullOrEmpty(comicInfo.Summary)) + { + chapter.Summary = comicInfo.Summary; + } + + if (!string.IsNullOrEmpty(comicInfo.LanguageISO)) + { + chapter.Language = comicInfo.LanguageISO; + } + if (comicInfo.Year > 0) { var day = Math.Max(comicInfo.Day, 1); @@ -227,7 +237,7 @@ public class MetadataService : IMetadataService } /// - /// Updates metadata for Series + /// Updates cover image for Series /// /// /// Force updating cover image even if underlying file has not been modified or chapter already has a cover image diff --git a/API/Services/Tasks/ScannerService.cs b/API/Services/Tasks/ScannerService.cs index 51dbdff85..0841ac7bb 100644 --- a/API/Services/Tasks/ScannerService.cs +++ b/API/Services/Tasks/ScannerService.cs @@ -467,44 +467,10 @@ public class ScannerService : IScannerService public static IEnumerable FindSeriesNotOnDisk(IEnumerable existingSeries, Dictionary> parsedSeries) { - var foundSeries = parsedSeries.Select(s => s.Key.Name).ToList(); - return existingSeries.Where(es => !es.NameInList(foundSeries) && !SeriesHasMatchingParserInfoFormat(es, parsedSeries)); + return existingSeries.Where(es => !ParserInfoHelpers.SeriesHasMatchingParserInfoFormat(es, parsedSeries)); } - /// - /// Checks each parser info to see if there is a name match and if so, checks if the format matches the Series object. - /// This accounts for if the Series has an Unknown type and if so, considers it matching. - /// - /// - /// - /// - private static bool SeriesHasMatchingParserInfoFormat(Series series, - Dictionary> parsedSeries) - { - var format = MangaFormat.Unknown; - foreach (var pSeries in parsedSeries.Keys) - { - var name = pSeries.Name; - var normalizedName = Parser.Parser.Normalize(name); - if (normalizedName == series.NormalizedName || - normalizedName == Parser.Parser.Normalize(series.Name) || - name == series.Name || name == series.LocalizedName || - name == series.OriginalName || - normalizedName == Parser.Parser.Normalize(series.OriginalName)) - { - format = pSeries.Format; - break; - } - } - - if (series.Format == MangaFormat.Unknown && format != MangaFormat.Unknown) - { - return true; - } - - return format == series.Format; - } private void UpdateVolumes(Series series, IList parsedInfos) {