diff --git a/API.Tests/ParserTest.cs b/API.Tests/ParserTest.cs index 76a457177..1380d5ab5 100644 --- a/API.Tests/ParserTest.cs +++ b/API.Tests/ParserTest.cs @@ -53,6 +53,8 @@ namespace API.Tests [InlineData("Kodomo no Jikan vol. 1.cbz", "1")] [InlineData("Kodomo no Jikan vol. 10.cbz", "10")] [InlineData("Kedouin Makoto - Corpse Party Musume, Chapter 12 [Dametrans][v2]", "0")] + [InlineData("Vagabond_v03", "3")] + [InlineData("Mujaki No Rakune Volume 10.cbz", "10")] public void ParseVolumeTest(string filename, string expected) { Assert.Equal(expected, ParseVolume(filename)); @@ -105,6 +107,11 @@ namespace API.Tests [InlineData("Goblin Slayer Side Story - Year One 025.5", "Goblin Slayer Side Story - Year One")] [InlineData("Goblin Slayer - Brand New Day 006.5 (2019) (Digital) (danke-Empire)", "Goblin Slayer - Brand New Day")] [InlineData("Kedouin Makoto - Corpse Party Musume, Chapter 01 [Dametrans][v2]", "Kedouin Makoto - Corpse Party Musume")] + [InlineData("Vagabond_v03", "Vagabond")] + [InlineData("[AN] Mahoutsukai to Deshi no Futekisetsu na Kankei Chp. 1", "Mahoutsukai to Deshi no Futekisetsu na Kankei")] + [InlineData("Beelzebub_Side_Story_02_RHS.zip", "Beelzebub Side Story")] + [InlineData("Corpse Party -The Anthology- Sachikos game of love Hysteric Birthday 2U Chapter 01", "Corpse Party -The Anthology- Sachikos game of love Hysteric Birthday 2U")] + [InlineData("[BAA]_Darker_than_Black_Omake-1.zip", "Darker than Black")] public void ParseSeriesTest(string filename, string expected) { Assert.Equal(expected, ParseSeries(filename)); @@ -146,6 +153,10 @@ namespace API.Tests [InlineData("VanDread-v01-c001[MD].zip", "1")] [InlineData("Goblin Slayer Side Story - Year One 025.5", "25.5")] [InlineData("Kedouin Makoto - Corpse Party Musume, Chapter 01", "1")] + [InlineData("To Love Ru v11 Uncensored (Ch.089-097+Omake)", "89-97")] + [InlineData("To Love Ru v18 Uncensored (Ch.153-162.5)", "153-162.5")] + [InlineData("[AN] Mahoutsukai to Deshi no Futekisetsu na Kankei Chp. 1", "1")] + [InlineData("Beelzebub_Side_Story_02_RHS.zip", "2")] public void ParseChaptersTest(string filename, string expected) { Assert.Equal(expected, ParseChapter(filename)); @@ -197,10 +208,23 @@ namespace API.Tests [InlineData("Tenjou Tenge Omnibus", "Omnibus")] [InlineData("Tenjou Tenge {Full Contact Edition}", "Full Contact Edition")] [InlineData("Tenjo Tenge {Full Contact Edition} v01 (2011) (Digital) (ASTC).cbz", "Full Contact Edition")] + [InlineData("Wotakoi - Love is Hard for Otaku Omnibus v01 (2018) (Digital) (danke-Empire)", "Omnibus")] + [InlineData("To Love Ru v01 Uncensored (Ch.001-007)", "Uncensored")] + [InlineData("Chobits Omnibus Edition v01 [Dark Horse]", "Omnibus Edition")] public void ParseEditionTest(string input, string expected) { Assert.Equal(expected, ParseEdition(input)); } + [Theory] + [InlineData("Beelzebub Special OneShot - Minna no Kochikame x Beelzebub (2016) [Mangastream].cbz", true)] + [InlineData("Beelzebub_Omake_June_2012_RHS", true)] + [InlineData("Beelzebub_Side_Story_02_RHS.zip", false)] + [InlineData("Darker than Black Shikkoku no Hana Special [Simple Scans].zip", true)] + [InlineData("Darker than Black Shikkoku no Hana Fanbook Extra [Simple Scans].zip", true)] + public void ParseMangaSpecialTest(string input, bool expected) + { + Assert.Equal(expected, ParseMangaSpecial(input) != ""); + } [Theory] [InlineData("12-14", 12)] @@ -236,6 +260,7 @@ namespace API.Tests [InlineData("Scott Pilgrim 01 - Scott Pilgrim's Precious Little Life (2004)", "Scott Pilgrim")] [InlineData("Teen Titans v1 001 (1966-02) (digital) (OkC.O.M.P.U.T.O.-Novus)", "Teen Titans")] [InlineData("Scott Pilgrim 02 - Scott Pilgrim vs. The World (2005)", "Scott Pilgrim")] + [InlineData("Wolverine - Origins 003 (2006) (digital) (Minutemen-PhD)", "Wolverine - Origins")] public void ParseComicSeriesTest(string filename, string expected) { Assert.Equal(expected, ParseComicSeries(filename)); diff --git a/API/Data/kavita.db b/API/Data/kavita.db new file mode 100644 index 000000000..c946be3fb Binary files /dev/null and b/API/Data/kavita.db differ diff --git a/API/Parser/Parser.cs b/API/Parser/Parser.cs index c089a688e..569ba93d5 100644 --- a/API/Parser/Parser.cs +++ b/API/Parser/Parser.cs @@ -14,8 +14,7 @@ namespace API.Parser private static readonly Regex ImageRegex = new Regex(ImageFileExtensions, RegexOptions.IgnoreCase | RegexOptions.Compiled); private static readonly Regex MangaFileRegex = new Regex(MangaFileExtensions, RegexOptions.IgnoreCase | RegexOptions.Compiled); private static readonly Regex XmlRegex = new Regex(XmlRegexExtensions, RegexOptions.IgnoreCase | RegexOptions.Compiled); - - //?: is a non-capturing group in C#, else anything in () will be a group + private static readonly Regex[] MangaVolumeRegex = new[] { // Dance in the Vampire Bund v16-17 @@ -32,11 +31,11 @@ namespace API.Parser RegexOptions.IgnoreCase | RegexOptions.Compiled), // Killing Bites Vol. 0001 Ch. 0001 - Galactica Scanlations (gb) new Regex( - @"(vol\.? ?)(?0*[1-9]+)", + @"(vol\.? ?)(?\d+)", RegexOptions.IgnoreCase | RegexOptions.Compiled), // Tonikaku Cawaii [Volume 11].cbz new Regex( - @"(volume )(?0?[1-9]+)", + @"(volume )(?\d+)", RegexOptions.IgnoreCase | RegexOptions.Compiled), // Tower Of God S01 014 (CBT) (digital).cbz @@ -101,13 +100,21 @@ namespace API.Parser new Regex( @"(?.*)(_)(v|vo|c|volume)( |_)\d+", RegexOptions.IgnoreCase | RegexOptions.Compiled), + // Mahoutsukai to Deshi no Futekisetsu na Kankei Chp. 1 + new Regex( + @"(?.*)( |_)(?:Chp.? ?\d+)", + RegexOptions.IgnoreCase | RegexOptions.Compiled), + // Corpse Party -The Anthology- Sachikos game of love Hysteric Birthday 2U Chapter 01 + new Regex( + @"^(?!Vol)(?.*)( |_)Chapter( |_)(\d+)", // TODO: This is breaking a ton of cases + RegexOptions.IgnoreCase | RegexOptions.Compiled), // Akiiro Bousou Biyori - 01.jpg, Beelzebub_172_RHS.zip, Cynthia the Mission 29.rar new Regex( - @"^(?!Vol)(?.*)( |_)(\d+)", + @"^(?!Vol)(?.*)( |_|-)(\d+)", RegexOptions.IgnoreCase | RegexOptions.Compiled), // [BAA]_Darker_than_Black_c1 (This is very greedy, make sure it's close to last) new Regex( - @"(?.*)( |_)(c)\d+", + @"(?.*)( |_|-)(c)\d+", RegexOptions.IgnoreCase | RegexOptions.Compiled), }; @@ -223,8 +230,9 @@ namespace API.Parser private static readonly Regex[] MangaChapterRegex = new[] { + // Historys Strongest Disciple Kenichi_v11_c90-98.zip, ...c90.5-100.5 new Regex( - @"(c|ch)(\.? ?)(?\d+(?:.\d+|-\d+)?)", + @"(c|ch)(\.? ?)(?(\d+(\.\d)?)-?(\d+(\.\d)?)?)", RegexOptions.IgnoreCase | RegexOptions.Compiled), // [Suihei Kiki]_Kasumi_Otoko_no_Ko_[Taruby]_v1.1.zip new Regex( @@ -251,13 +259,17 @@ namespace API.Parser }; private static readonly Regex[] MangaEditionRegex = { - //Tenjo Tenge {Full Contact Edition} v01 (2011) (Digital) (ASTC).cbz + // Tenjo Tenge {Full Contact Edition} v01 (2011) (Digital) (ASTC).cbz new Regex( @"(?({|\(|\[).* Edition(}|\)|\]))", RegexOptions.IgnoreCase | RegexOptions.Compiled), - //Tenjo Tenge {Full Contact Edition} v01 (2011) (Digital) (ASTC).cbz + // Tenjo Tenge {Full Contact Edition} v01 (2011) (Digital) (ASTC).cbz new Regex( - @"(\b|_)(?Omnibus)(\b|_)", + @"(\b|_)(?Omnibus(( |_)?Edition)?)(\b|_)?", + RegexOptions.IgnoreCase | RegexOptions.Compiled), + // To Love Ru v01 Uncensored (Ch.001-007) + new Regex( + @"(\b|_)(?Uncensored)(\b|_)", RegexOptions.IgnoreCase | RegexOptions.Compiled), }; @@ -277,6 +289,14 @@ namespace API.Parser RegexOptions.IgnoreCase | RegexOptions.Compiled), }; + private static readonly Regex[] MangaSpecialRegex = + { + // All Keywords, does not account for checking if contains volume/chapter identification. Parser.Parse() will handle. + new Regex( + @"(?Special|OneShot|One\-Shot|Omake|Extra)", + RegexOptions.IgnoreCase | RegexOptions.Compiled), + }; + /// /// Parses information out of a file path. Will fallback to using directory name if Series couldn't be parsed @@ -314,6 +334,13 @@ namespace API.Parser ret.Series = CleanTitle(ret.Series.Replace(edition, "")); ret.Edition = edition; } + + var isSpecial = ParseMangaSpecial(fileName); + if (ret.Chapters == "0" && ret.Volumes == "0" && !string.IsNullOrEmpty(isSpecial)) + { + ret.IsSpecial = true; + } + return ret.Series == string.Empty ? null : ret; @@ -346,6 +373,23 @@ namespace API.Parser return string.Empty; } + public static string ParseMangaSpecial(string filePath) + { + foreach (var regex in MangaSpecialRegex) + { + var matches = regex.Matches(filePath); + foreach (Match match in matches) + { + if (match.Groups["Special"].Success && match.Groups["Special"].Value != string.Empty) + { + return match.Groups["Special"].Value; + } + } + } + + return string.Empty; + } + public static string ParseSeries(string filename) { foreach (var regex in MangaSeriesRegex) @@ -496,6 +540,25 @@ namespace API.Parser return title; } + private static string RemoveSpecialTags(string title) + { + foreach (var regex in MangaSpecialRegex) + { + var matches = regex.Matches(title); + foreach (Match match in matches) + { + if (match.Success) + { + title = title.Replace(match.Value, ""); + } + } + } + + return title; + } + + + /// /// Translates _ -> spaces, trims front and back of string, removes release groups /// @@ -507,6 +570,8 @@ namespace API.Parser title = RemoveEditionTagHolders(title); + title = RemoveSpecialTags(title); + title = title.Replace("_", " ").Trim(); if (title.EndsWith("-")) { diff --git a/API/Parser/ParserInfo.cs b/API/Parser/ParserInfo.cs index ee92ddd9f..4b7d5985e 100644 --- a/API/Parser/ParserInfo.cs +++ b/API/Parser/ParserInfo.cs @@ -24,5 +24,10 @@ namespace API.Parser /// This can potentially story things like "Omnibus, Color, Full Contact Edition, Extra, Final, etc" /// public string Edition { get; set; } = ""; + + /// + /// If the file contains no volume/chapter information and contains Special Keywords + /// + public bool IsSpecial { get; set; } = false; } } \ No newline at end of file diff --git a/API/Services/Tasks/ScannerService.cs b/API/Services/Tasks/ScannerService.cs index 4be3d749d..e8101cc75 100644 --- a/API/Services/Tasks/ScannerService.cs +++ b/API/Services/Tasks/ScannerService.cs @@ -49,15 +49,15 @@ namespace API.Services.Tasks { // NOTE: This solution isn't the best, but it has potential. We need to handle a few other cases so it works great. return false; - - // if (/*_environment.IsProduction() && */!_forceUpdate && Directory.GetLastWriteTime(folder.Path) < folder.LastScanned) + + // if (!_forceUpdate && Directory.GetLastWriteTime(folder.Path) < folder.LastScanned) // { - // _logger.LogDebug($"{folder.Path} hasn't been updated since last scan. Skipping."); + // _logger.LogDebug("{FolderPath} hasn't been modified since last scan. Skipping", folder.Path); // skippedFolders += 1; // return true; // } - // - // return false; + + //return false; } private void Cleanup() @@ -134,7 +134,6 @@ namespace API.Services.Tasks if (Task.Run(() => _unitOfWork.Complete()).Result) { - _logger.LogInformation("Scan completed on {LibraryName}. Parsed {ParsedSeriesCount} series in {ElapsedScanTime} ms", library.Name, series.Keys.Count, sw.ElapsedMilliseconds); } else @@ -149,6 +148,13 @@ namespace API.Services.Tasks { if (parsedSeries == null) throw new ArgumentNullException(nameof(parsedSeries)); + // For all parsedSeries, any infos that contain same series name and IsSpecial is true are combined + // foreach (var series in parsedSeries) + // { + // var seriesName = series.Key; + // if (parsedSeries.ContainsKey(seriesName)) + // } + // First, remove any series that are not in parsedSeries list var foundSeries = parsedSeries.Select(s => Parser.Parser.Normalize(s.Key)).ToList(); var missingSeries = library.Series.Where(existingSeries => @@ -222,7 +228,7 @@ namespace API.Services.Tasks series.Volumes.Add(volume); } - volume.IsSpecial = volume.Number == 0 && infos.All(p => p.Chapters == "0"); + volume.IsSpecial = volume.Number == 0 && infos.All(p => p.Chapters == "0" || p.IsSpecial); _logger.LogDebug("Parsing {SeriesName} - Volume {VolumeNumber}", series.Name, volume.Name); UpdateChapters(volume, infos); volume.Pages = volume.Chapters.Sum(c => c.Pages); @@ -314,7 +320,7 @@ namespace API.Services.Tasks private void TrackSeries(ParserInfo info) { if (info.Series == string.Empty) return; - + _scannedSeries.AddOrUpdate(info.Series, new List() {info}, (_, oldValue) => { oldValue ??= new List(); diff --git a/API/Startup.cs b/API/Startup.cs index 081249137..736d712f6 100644 --- a/API/Startup.cs +++ b/API/Startup.cs @@ -136,7 +136,7 @@ namespace API applicationLifetime.ApplicationStopping.Register(OnShutdown); applicationLifetime.ApplicationStarted.Register(() => { - Console.WriteLine("Kavita - v0.3.5"); + Console.WriteLine("Kavita - v0.3.6"); }); } diff --git a/build.sh b/build.sh index a8d59b9d1..de97ff72f 100644 --- a/build.sh +++ b/build.sh @@ -65,8 +65,8 @@ Package() # TODO: Use no-restore? Because Build should have already done it for us echo "Building" cd API - echo dotnet publish -c release --self-contained --runtime $runtime -o "$lOutputFolder" --framework $framework - dotnet publish -c release --self-contained --runtime $runtime -o "$lOutputFolder" --framework $framework + echo dotnet publish -c Release --self-contained --runtime $runtime -o "$lOutputFolder" --framework $framework + dotnet publish -c Release --self-contained --runtime $runtime -o "$lOutputFolder" --framework $framework echo "Copying Install information" cp ../INSTALL.txt "$lOutputFolder"/README.txt