diff --git a/.gitignore b/.gitignore index 75589ba35..343c37b63 100644 --- a/.gitignore +++ b/.gitignore @@ -445,4 +445,6 @@ $RECYCLE.BIN/ appsettings.json /API/kavita.db /API/kavita.db-shm -/API/kavita.db-wal \ No newline at end of file +/API/kavita.db-wal +/API/Hangfire.db +/API/Hangfire-log.db \ No newline at end of file diff --git a/API.Tests/API.Tests.csproj b/API.Tests/API.Tests.csproj new file mode 100644 index 000000000..e19d7abc9 --- /dev/null +++ b/API.Tests/API.Tests.csproj @@ -0,0 +1,26 @@ + + + + net5.0 + + false + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + diff --git a/API.Tests/ParserTest.cs b/API.Tests/ParserTest.cs new file mode 100644 index 000000000..6205eaa62 --- /dev/null +++ b/API.Tests/ParserTest.cs @@ -0,0 +1,82 @@ +using Xunit; +using static API.Parser.Parser; + +namespace API.Tests +{ + public class ParserTests + { + [Theory] + [InlineData("Killing Bites Vol. 0001 Ch. 0001 - Galactica Scanlations (gb)", "1")] + [InlineData("My Girlfriend Is Shobitch v01 - ch. 09 - pg. 008.png", "1")] + [InlineData("Historys Strongest Disciple Kenichi_v11_c90-98.zip", "11")] + [InlineData("B_Gata_H_Kei_v01[SlowManga&OverloadScans]", "1")] + [InlineData("BTOOOM! v01 (2013) (Digital) (Shadowcat-Empire)", "1")] + [InlineData("Gokukoku no Brynhildr - c001-008 (v01) [TrinityBAKumA]", "1")] + [InlineData("Dance in the Vampire Bund v16-17 (Digital) (NiceDragon)", "16-17")] + [InlineData("v001", "1")] + public void ParseVolumeTest(string filename, string expected) + { + var result = ParseVolume(filename); + Assert.Equal(expected, result); + } + + [Theory] + [InlineData("Killing Bites Vol. 0001 Ch. 0001 - Galactica Scanlations (gb)", "Killing Bites")] + [InlineData("My Girlfriend Is Shobitch v01 - ch. 09 - pg. 008.png", "My Girlfriend Is Shobitch")] + [InlineData("Historys Strongest Disciple Kenichi_v11_c90-98.zip", "Historys Strongest Disciple Kenichi")] + [InlineData("B_Gata_H_Kei_v01[SlowManga&OverloadScans]", "B Gata H Kei")] + [InlineData("BTOOOM! v01 (2013) (Digital) (Shadowcat-Empire)", "BTOOOM!")] + [InlineData("Gokukoku no Brynhildr - c001-008 (v01) [TrinityBAKumA]", "Gokukoku no Brynhildr")] + [InlineData("Dance in the Vampire Bund v16-17 (Digital) (NiceDragon)", "Dance in the Vampire Bund")] + [InlineData("v001", "")] + public void ParseSeriesTest(string filename, string expected) + { + var result = ParseSeries(filename); + Assert.Equal(expected, result); + } + + [Theory] + [InlineData("Killing Bites Vol. 0001 Ch. 0001 - Galactica Scanlations (gb)", "1")] + [InlineData("My Girlfriend Is Shobitch v01 - ch. 09 - pg. 008.png", "9")] + [InlineData("Historys Strongest Disciple Kenichi_v11_c90-98.zip", "90-98")] + [InlineData("B_Gata_H_Kei_v01[SlowManga&OverloadScans]", "")] + [InlineData("BTOOOM! v01 (2013) (Digital) (Shadowcat-Empire)", "")] + [InlineData("Gokukoku no Brynhildr - c001-008 (v01) [TrinityBAKumA]", "1-8")] + [InlineData("Dance in the Vampire Bund v16-17 (Digital) (NiceDragon)", "")] + [InlineData("c001", "1")] + public void ParseChaptersTest(string filename, string expected) + { + var result = ParseChapter(filename); + Assert.Equal(expected, result); + } + + + [Theory] + [InlineData("0001", "1")] + [InlineData("1", "1")] + [InlineData("0013", "13")] + public void RemoveLeadingZeroesTest(string input, string expected) + { + Assert.Equal(expected, RemoveLeadingZeroes(input)); + } + + [Theory] + [InlineData("1", "001")] + [InlineData("10", "010")] + [InlineData("100", "100")] + public void PadZerosTest(string input, string expected) + { + Assert.Equal(expected, PadZeros(input)); + } + + [Theory] + [InlineData("Hello_I_am_here", "Hello I am here")] + [InlineData("Hello_I_am_here ", "Hello I am here")] + [InlineData("[ReleaseGroup] The Title", "The Title")] + [InlineData("[ReleaseGroup]_The_Title", "The Title")] + public void CleanTitleTest(string input, string expected) + { + Assert.Equal(expected, CleanTitle(input)); + } + } +} \ No newline at end of file diff --git a/API/API.csproj b/API/API.csproj index cdd2f9198..c6a733fcd 100644 --- a/API/API.csproj +++ b/API/API.csproj @@ -32,4 +32,8 @@ + + + + diff --git a/API/Controllers/AdminController.cs b/API/Controllers/AdminController.cs index fa495b62e..2c2e64bd7 100644 --- a/API/Controllers/AdminController.cs +++ b/API/Controllers/AdminController.cs @@ -8,12 +8,10 @@ namespace API.Controllers { public class AdminController : BaseApiController { - private readonly IUserRepository _userRepository; private readonly UserManager _userManager; - public AdminController(IUserRepository userRepository, UserManager userManager) + public AdminController(UserManager userManager) { - _userRepository = userRepository; _userManager = userManager; } diff --git a/API/Controllers/UsersController.cs b/API/Controllers/UsersController.cs index 6102cc5e5..337d54959 100644 --- a/API/Controllers/UsersController.cs +++ b/API/Controllers/UsersController.cs @@ -38,7 +38,7 @@ namespace API.Controllers return BadRequest("Library name already exists. Please choose a unique name to the server."); } - // TODO: We probably need to clean the folders before we insert + // TODO: We probably need to normalize the folders before we insert var library = new Library { Name = createLibraryDto.Name.ToLower(), diff --git a/API/Hangfire-log.db b/API/Hangfire-log.db deleted file mode 100644 index d8fc774c2..000000000 Binary files a/API/Hangfire-log.db and /dev/null differ diff --git a/API/Hangfire.db b/API/Hangfire.db deleted file mode 100644 index db5987848..000000000 Binary files a/API/Hangfire.db and /dev/null differ diff --git a/API/Parser/Parser.cs b/API/Parser/Parser.cs new file mode 100644 index 000000000..ee22771fb --- /dev/null +++ b/API/Parser/Parser.cs @@ -0,0 +1,200 @@ +using System; +using System.Text.RegularExpressions; + +namespace API.Parser +{ + public static class Parser + { + //?: is a non-capturing group in C#, else anything in () will be a group + private static readonly Regex[] MangaVolumeRegex = new[] + { + // Historys Strongest Disciple Kenichi_v11_c90-98.zip + new Regex( + + @"(?.*)(\b|_)v(?\d+)", + RegexOptions.IgnoreCase | RegexOptions.Compiled), + // Killing Bites Vol. 0001 Ch. 0001 - Galactica Scanlations (gb) + new Regex( + @"(vol. ?)(?0*[1-9]+)", + RegexOptions.IgnoreCase | RegexOptions.Compiled), + // Dance in the Vampire Bund v16-17 + new Regex( + + @"(?.*)(\b|_)v(?\d+-?\d+)", + RegexOptions.IgnoreCase | RegexOptions.Compiled), + new Regex( + @"(?:v)(?0*[1-9]+)", + RegexOptions.IgnoreCase | RegexOptions.Compiled), + + }; + + private static readonly Regex[] MangaSeriesRegex = new[] + { + // Gokukoku no Brynhildr - c001-008 (v01) [TrinityBAKumA], Black Bullet - v4 c17 [batoto] + new Regex( + + @"(?.*)( - )(?:v|vo|c)\d", + RegexOptions.IgnoreCase | RegexOptions.Compiled), + // Historys Strongest Disciple Kenichi_v11_c90-98.zip, Killing Bites Vol. 0001 Ch. 0001 - Galactica Scanlations (gb) + new Regex( + + @"(?.*)(\b|_)v", + RegexOptions.IgnoreCase | RegexOptions.Compiled), + + // Black Bullet + new Regex( + + @"(?.*)(\b|_)(v|vo|c)", + RegexOptions.IgnoreCase | RegexOptions.Compiled), + + // [BAA]_Darker_than_Black_c1 (This is very greedy, make sure it's always last) + new Regex( + @"(?.*)(\b|_)(c)", + RegexOptions.IgnoreCase | RegexOptions.Compiled), + + + }; + + private static readonly Regex[] ReleaseGroupRegex = new[] + { + // [TrinityBAKumA Finella&anon], [BAA]_, [SlowManga&OverloadScans], [batoto] + new Regex(@"(?:\[(?(?!\s).+?(?(?!\s).+?(?\d+-?\d*)", + RegexOptions.IgnoreCase | RegexOptions.Compiled), + }; + + public static string ParseSeries(string filename) + { + foreach (var regex in MangaSeriesRegex) + { + var matches = regex.Matches(filename); + foreach (Match match in matches) + { + if (match.Groups["Volume"] != Match.Empty) + { + return CleanTitle(match.Groups["Series"].Value); + } + + } + } + + Console.WriteLine("Unable to parse {0}", filename); + return ""; + } + + public static string ParseVolume(string filename) + { + foreach (var regex in MangaVolumeRegex) + { + var matches = regex.Matches(filename); + foreach (Match match in matches) + { + if (match.Groups["Volume"] != Match.Empty) + { + return RemoveLeadingZeroes(match.Groups["Volume"].Value); + } + + } + } + + Console.WriteLine("Unable to parse {0}", filename); + return ""; + } + + public static string ParseChapter(string filename) + { + foreach (var regex in MangaChapterRegex) + { + var matches = regex.Matches(filename); + foreach (Match match in matches) + { + if (match.Groups["Chapter"] != Match.Empty) + { + var value = match.Groups["Chapter"].Value; + + + if (value.Contains("-")) + { + var tokens = value.Split("-"); + var from = RemoveLeadingZeroes(tokens[0]); + var to = RemoveLeadingZeroes(tokens[1]); + return $"{from}-{to}"; + } + + return RemoveLeadingZeroes(match.Groups["Chapter"].Value); + } + + } + } + + return ""; + } + + /// + /// Translates _ -> spaces, trims front and back of string, removes release groups + /// + /// + /// + public static string CleanTitle(string title) + { + foreach (var regex in ReleaseGroupRegex) + { + var matches = regex.Matches(title); + foreach (Match match in matches) + { + if (match.Success) + { + title = title.Replace(match.Value, ""); + } + } + } + + title = title.Replace("_", " "); + return title.Trim(); + } + + + /// + /// Pads the start of a number string with 0's so ordering works fine if there are over 100 items. + /// Handles ranges (ie 4-8) -> (004-008). + /// + /// + /// A zero padded number + public static string PadZeros(string number) + { + if (number.Contains("-")) + { + var tokens = number.Split("-"); + return $"{PerformPadding(tokens[0])}-{PerformPadding(tokens[1])}"; + } + + return PerformPadding(number); + } + + private static string PerformPadding(string number) + { + var num = Int32.Parse(number); + return num switch + { + < 10 => "00" + num, + < 100 => "0" + num, + _ => number + }; + } + + public static string RemoveLeadingZeroes(string title) + { + return title.TrimStart(new Char[] { '0' }); + } + } +} \ No newline at end of file diff --git a/API/Parser/ParserInfo.cs b/API/Parser/ParserInfo.cs new file mode 100644 index 000000000..f2d8bcef4 --- /dev/null +++ b/API/Parser/ParserInfo.cs @@ -0,0 +1,14 @@ +using System.Collections.Generic; + +namespace API.Parser +{ + public class ParserInfo + { + // This can be multiple + public string Chapters { get; set; } + public string Series { get; set; } + // This can be multiple + public string Volume { get; set; } + public IEnumerable Files { get; init; } + } +} \ No newline at end of file diff --git a/API/Services/DirectoryService.cs b/API/Services/DirectoryService.cs index 8f09750b2..909d14faa 100644 --- a/API/Services/DirectoryService.cs +++ b/API/Services/DirectoryService.cs @@ -62,7 +62,7 @@ namespace API.Services }); } catch (ArgumentException) { - Console.WriteLine(@"The directory 'C:\Program Files' does not exist."); + _logger.LogError($"The directory '{folderPath}' does not exist"); } } } @@ -79,7 +79,7 @@ namespace API.Services var sw = Stopwatch.StartNew(); // Determine whether to parallelize file processing on each folder based on processor count. - int procCount = System.Environment.ProcessorCount; + int procCount = Environment.ProcessorCount; // Data structure to hold names of subfolders to be examined for files. Stack dirs = new Stack(); diff --git a/API/appsettings.Development.json b/API/appsettings.Development.json index 5d8c460c5..740eb4c1e 100644 --- a/API/appsettings.Development.json +++ b/API/appsettings.Development.json @@ -1,7 +1,6 @@ { "ConnectionStrings": { "DefaultConnection": "Data source=kavita.db", - "HangfireConnection": "Data source=hangfire.db" }, "TokenKey": "super secret unguessable key", "Logging": { diff --git a/Kavita.sln b/Kavita.sln index c1f023634..74927a34f 100644 --- a/Kavita.sln +++ b/Kavita.sln @@ -5,6 +5,8 @@ VisualStudioVersion = 15.0.26124.0 MinimumVisualStudioVersion = 15.0.26124.0 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "API", "API\API.csproj", "{1BC0273F-FEBE-4DA1-BC04-3A3167E4C86C}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "API.Tests", "API.Tests\API.Tests.csproj", "{6F7910F2-1B95-4570-A490-519C8935B9D1}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -30,5 +32,17 @@ Global {1BC0273F-FEBE-4DA1-BC04-3A3167E4C86C}.Release|x64.Build.0 = Release|Any CPU {1BC0273F-FEBE-4DA1-BC04-3A3167E4C86C}.Release|x86.ActiveCfg = Release|Any CPU {1BC0273F-FEBE-4DA1-BC04-3A3167E4C86C}.Release|x86.Build.0 = Release|Any CPU + {6F7910F2-1B95-4570-A490-519C8935B9D1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6F7910F2-1B95-4570-A490-519C8935B9D1}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6F7910F2-1B95-4570-A490-519C8935B9D1}.Debug|x64.ActiveCfg = Debug|Any CPU + {6F7910F2-1B95-4570-A490-519C8935B9D1}.Debug|x64.Build.0 = Debug|Any CPU + {6F7910F2-1B95-4570-A490-519C8935B9D1}.Debug|x86.ActiveCfg = Debug|Any CPU + {6F7910F2-1B95-4570-A490-519C8935B9D1}.Debug|x86.Build.0 = Debug|Any CPU + {6F7910F2-1B95-4570-A490-519C8935B9D1}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6F7910F2-1B95-4570-A490-519C8935B9D1}.Release|Any CPU.Build.0 = Release|Any CPU + {6F7910F2-1B95-4570-A490-519C8935B9D1}.Release|x64.ActiveCfg = Release|Any CPU + {6F7910F2-1B95-4570-A490-519C8935B9D1}.Release|x64.Build.0 = Release|Any CPU + {6F7910F2-1B95-4570-A490-519C8935B9D1}.Release|x86.ActiveCfg = Release|Any CPU + {6F7910F2-1B95-4570-A490-519C8935B9D1}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection EndGlobal