diff --git a/API.Tests/ParserTest.cs b/API.Tests/ParserTest.cs index 6205eaa62..3a5330280 100644 --- a/API.Tests/ParserTest.cs +++ b/API.Tests/ParserTest.cs @@ -13,7 +13,9 @@ namespace API.Tests [InlineData("BTOOOM! v01 (2013) (Digital) (Shadowcat-Empire)", "1")] [InlineData("Gokukoku no Brynhildr - c001-008 (v01) [TrinityBAKumA]", "1")] [InlineData("Dance in the Vampire Bund v16-17 (Digital) (NiceDragon)", "16-17")] + [InlineData("Akame ga KILL! ZERO v01 (2016) (Digital) (LuCaZ).cbz", "1")] [InlineData("v001", "1")] + [InlineData("[Suihei Kiki]_Kasumi_Otoko_no_Ko_[Taruby]_v1.1.zip", "1")] public void ParseVolumeTest(string filename, string expected) { var result = ParseVolume(filename); @@ -29,6 +31,7 @@ namespace API.Tests [InlineData("Gokukoku no Brynhildr - c001-008 (v01) [TrinityBAKumA]", "Gokukoku no Brynhildr")] [InlineData("Dance in the Vampire Bund v16-17 (Digital) (NiceDragon)", "Dance in the Vampire Bund")] [InlineData("v001", "")] + [InlineData("Akame ga KILL! ZERO (2016-2019) (Digital) (LuCaZ)", "Akame ga KILL! ZERO")] public void ParseSeriesTest(string filename, string expected) { var result = ParseSeries(filename); @@ -44,6 +47,7 @@ namespace API.Tests [InlineData("Gokukoku no Brynhildr - c001-008 (v01) [TrinityBAKumA]", "1-8")] [InlineData("Dance in the Vampire Bund v16-17 (Digital) (NiceDragon)", "")] [InlineData("c001", "1")] + [InlineData("[Suihei Kiki]_Kasumi_Otoko_no_Ko_[Taruby]_v1.12.zip", "12")] public void ParseChaptersTest(string filename, string expected) { var result = ParseChapter(filename); diff --git a/API/API.csproj b/API/API.csproj index c6a733fcd..1b99ef72e 100644 --- a/API/API.csproj +++ b/API/API.csproj @@ -28,10 +28,6 @@ - - - - diff --git a/API/Entities/Series.cs b/API/Entities/Series.cs new file mode 100644 index 000000000..7545fccf7 --- /dev/null +++ b/API/Entities/Series.cs @@ -0,0 +1,28 @@ +using System.Collections.Generic; + +namespace API.Entities +{ + public class Series + { + /// + /// The UI visible Name of the Series. This may or may not be the same as the OriginalName + /// + public string Name { get; set; } + /// + /// Original Japanese Name + /// + public string OriginalName { get; set; } + /// + /// The name used to sort the Series. By default, will be the same as Name. + /// + public string SortName { get; set; } + /// + /// Summary information related to the Series + /// + public string Summary { get; set; } + + public ICollection Volumes { get; set; } + + + } +} \ No newline at end of file diff --git a/API/Entities/Volume.cs b/API/Entities/Volume.cs new file mode 100644 index 000000000..54d56804c --- /dev/null +++ b/API/Entities/Volume.cs @@ -0,0 +1,14 @@ +using System.Collections.Generic; + +namespace API.Entities +{ + public class Volume + { + public string Number { get; set; } + public ICollection Files { get; set; } + + // Many-to-Many relationships + public Series Series { get; set; } + public int SeriesId { get; set; } + } +} \ No newline at end of file diff --git a/API/Parser/Parser.cs b/API/Parser/Parser.cs index ee22771fb..be6b26bae 100644 --- a/API/Parser/Parser.cs +++ b/API/Parser/Parser.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Text.RegularExpressions; namespace API.Parser @@ -47,10 +48,20 @@ namespace API.Parser @"(?.*)(\b|_)(v|vo|c)", RegexOptions.IgnoreCase | RegexOptions.Compiled), + // Akame ga KILL! ZERO (2016-2019) (Digital) (LuCaZ) + new Regex( + + @"(?.*)\(\d", + RegexOptions.IgnoreCase | RegexOptions.Compiled), + // [BAA]_Darker_than_Black_c1 (This is very greedy, make sure it's always last) new Regex( @"(?.*)(\b|_)(c)", RegexOptions.IgnoreCase | RegexOptions.Compiled), + // Darker Than Black (This takes anything, we have to account for perfectly named folders) + new Regex( + @"(?.*)", + RegexOptions.IgnoreCase | RegexOptions.Compiled), }; @@ -72,6 +83,18 @@ namespace API.Parser @"(c|ch)(\.? ?)(?\d+-?\d*)", RegexOptions.IgnoreCase | RegexOptions.Compiled), }; + + + public static ParserInfo Parse(string filePath) + { + return new ParserInfo() + { + Chapters = ParseChapter(filePath), + Series = ParseSeries(filePath), + Volumes = ParseVolume(filePath), + File = filePath + }; + } public static string ParseSeries(string filename) { diff --git a/API/Parser/ParserInfo.cs b/API/Parser/ParserInfo.cs index f2d8bcef4..c6d2fd9a6 100644 --- a/API/Parser/ParserInfo.cs +++ b/API/Parser/ParserInfo.cs @@ -2,13 +2,17 @@ namespace API.Parser { + /// + /// This represents a single file + /// public class ParserInfo { // This can be multiple public string Chapters { get; set; } public string Series { get; set; } // This can be multiple - public string Volume { get; set; } - public IEnumerable Files { get; init; } + public string Volumes { get; set; } + public string File { get; init; } + //public IEnumerable Files { get; init; } } } \ No newline at end of file diff --git a/API/Services/DirectoryService.cs b/API/Services/DirectoryService.cs index 7a5c92d19..26710873d 100644 --- a/API/Services/DirectoryService.cs +++ b/API/Services/DirectoryService.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Collections.Immutable; using System.Diagnostics; @@ -10,6 +11,8 @@ using System.Threading; using System.Threading.Tasks; using API.DTOs; using API.Interfaces; +using API.Parser; +using Microsoft.EntityFrameworkCore.Metadata.Internal; using Microsoft.Extensions.Logging; namespace API.Services @@ -18,6 +21,7 @@ namespace API.Services { private readonly ILogger _logger; private static readonly string MangaFileExtensions = @"\.cbz|\.cbr|\.png|\.jpeg|\.jpg|\.zip|\.rar"; + private ConcurrentDictionary> _scannedSeries; public DirectoryService(ILogger logger) { @@ -62,17 +66,131 @@ namespace API.Services return dirs; } + // TODO: Refactor API layer to use this + public IEnumerable ListDirectories(string rootPath) + { + if (!Directory.Exists(rootPath)) return ImmutableList.Empty; + + var di = new DirectoryInfo(rootPath); + var dirs = di.GetDirectories() + .Where(dir => !(dir.Attributes.HasFlag(FileAttributes.Hidden) || dir.Attributes.HasFlag(FileAttributes.System))) + .ToImmutableList(); + + + return dirs; + } + + private void Process(string path) + { + if (Directory.Exists(path)) + { + DirectoryInfo di = new DirectoryInfo(path); + Console.WriteLine($"Parsing directory {di.Name}"); + + var seriesName = Parser.Parser.ParseSeries(di.Name); + if (string.IsNullOrEmpty(seriesName)) + { + return; + } + + // We don't need ContainsKey, this is a race condition. We can replace with TryAdd instead + if (!_scannedSeries.ContainsKey(seriesName)) + { + _scannedSeries.TryAdd(seriesName, new ConcurrentBag()); + } + } + else + { + var fileName = Path.GetFileName(path); + Console.WriteLine($"Parsing file {fileName}"); + + var info = Parser.Parser.Parse(fileName); + if (info.Volumes != string.Empty) + { + ConcurrentBag tempBag; + ConcurrentBag newBag = new ConcurrentBag(); + if (_scannedSeries.TryGetValue(info.Series, out tempBag)) + { + var existingInfos = tempBag.ToArray(); + foreach (var existingInfo in existingInfos) + { + newBag.Add(existingInfo); + } + } + else + { + tempBag = new ConcurrentBag(); + } + + newBag.Add(info); + + if (!_scannedSeries.TryUpdate(info.Series, newBag, tempBag)) + { + _scannedSeries.TryAdd(info.Series, newBag); + } + + } + + + } + } + public void ScanLibrary(LibraryDto library) { + _scannedSeries = new ConcurrentDictionary>(); + //Dictionary> series = new Dictionary>(); + _logger.LogInformation($"Beginning scan on {library.Name}"); foreach (var folderPath in library.Folders) { try { + // // Temporarily, let's build a simple scanner then optimize to parallelization + // + // // First, let's see if there are any files in rootPath + // var files = GetFiles(folderPath, MangaFileExtensions); + // + // foreach (var file in files) + // { + // // These do not have a folder, so we need to parse them directly + // var parserInfo = Parser.Parser.Parse(file); + // Console.WriteLine(parserInfo); + // } + // + // // Get Directories + // var directories = ListDirectories(folderPath); + // foreach (var directory in directories) + // { + // _logger.LogDebug($"Scanning {directory.Name}"); + // var parsedSeries = Parser.Parser.ParseSeries(directory.Name); + // + // // For now, let's skip directories we can't parse information out of. (we are assuming one level deep root) + // if (string.IsNullOrEmpty(parsedSeries)) continue; + // + // _logger.LogDebug($"Parsed Series: {parsedSeries}"); + // + // if (!series.ContainsKey(parsedSeries)) + // { + // series[parsedSeries] = new List(); + // } + // + // var foundFiles = GetFiles(directory.FullName, MangaFileExtensions); + // foreach (var foundFile in foundFiles) + // { + // var info = Parser.Parser.Parse(foundFile); + // if (info.Volumes != string.Empty) + // { + // series[parsedSeries].Add(info); + // } + // } + // } + + TraverseTreeParallelForEach(folderPath, (f) => { // Exceptions are no-ops. try { - ProcessManga(folderPath, f); + Process(f); + //ProcessManga(folderPath, f); } catch (FileNotFoundException) {} catch (IOException) {} @@ -87,12 +205,38 @@ namespace API.Services _logger.LogError($"The directory '{folderPath}' does not exist"); } } + + // var filtered = series.Where(kvp => kvp.Value.Count > 0); + // series = filtered.ToDictionary(v => v.Key, v => v.Value); + // Console.WriteLine(series); + + // var filtered = _scannedSeries.Where(kvp => kvp.Value.Count > 0); + // series = filtered.ToDictionary(v => v.Key, v => v.Value); + // Console.WriteLine(series); + var filtered = _scannedSeries.Where(kvp => !kvp.Value.IsEmpty); + var series = filtered.ToImmutableDictionary(v => v.Key, v => v.Value); + Console.WriteLine(series); + + // TODO: Perform DB activities on ImmutableDictionary + + + //_logger.LogInformation($"Scan completed on {library.Name}. Parsed {series.Keys.Count} series."); + _logger.LogInformation($"Scan completed on {library.Name}. Parsed {series.Keys.Count()} series."); + _scannedSeries = null; + + } private static void ProcessManga(string folderPath, string filename) { + Console.WriteLine($"[ProcessManga] Folder: {folderPath}"); + Console.WriteLine($"Found {filename}"); var series = Parser.Parser.ParseSeries(filename); + if (series == string.Empty) + { + series = Parser.Parser.ParseSeries(folderPath); + } Console.WriteLine($"Series: {series}"); }