Cleaned up the code from previous enhancements.

This commit is contained in:
Joseph Milazzo 2020-12-29 11:01:52 -06:00
parent 0a49b07570
commit 104c63b2b9
5 changed files with 24 additions and 108 deletions

View File

@ -1,8 +1,6 @@
using System; using System.Collections.Generic;
using System.Collections.Generic;
using System.IO; using System.IO;
using System.Threading.Tasks; using System.Threading.Tasks;
using API.Data;
using API.DTOs; using API.DTOs;
using API.Entities; using API.Entities;
using API.Interfaces; using API.Interfaces;

View File

@ -1,11 +1,12 @@
using System; using System;
using System.Collections.Generic;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
namespace API.Parser namespace API.Parser
{ {
public static class Parser public static class Parser
{ {
public static readonly string MangaFileExtensions = @"\.cbz|\.cbr|\.png|\.jpeg|\.jpg|\.zip|\.rar";
//?: is a non-capturing group in C#, else anything in () will be a group //?: is a non-capturing group in C#, else anything in () will be a group
private static readonly Regex[] MangaVolumeRegex = new[] private static readonly Regex[] MangaVolumeRegex = new[]
{ {
@ -217,7 +218,7 @@ namespace API.Parser
public static string RemoveLeadingZeroes(string title) public static string RemoveLeadingZeroes(string title)
{ {
return title.TrimStart(new Char[] { '0' }); return title.TrimStart(new[] { '0' });
} }
} }
} }

View File

@ -1,5 +1,4 @@
using System.Collections.Generic; 
namespace API.Parser namespace API.Parser
{ {
/// <summary> /// <summary>
@ -13,6 +12,5 @@ namespace API.Parser
// This can be multiple // This can be multiple
public string Volumes { get; set; } public string Volumes { get; set; }
public string File { get; init; } public string File { get; init; }
//public IEnumerable<string> Files { get; init; }
} }
} }

View File

@ -12,7 +12,6 @@ using System.Threading.Tasks;
using API.DTOs; using API.DTOs;
using API.Interfaces; using API.Interfaces;
using API.Parser; using API.Parser;
using Microsoft.EntityFrameworkCore.Metadata.Internal;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
namespace API.Services namespace API.Services
@ -20,7 +19,6 @@ namespace API.Services
public class DirectoryService : IDirectoryService public class DirectoryService : IDirectoryService
{ {
private readonly ILogger<DirectoryService> _logger; private readonly ILogger<DirectoryService> _logger;
private static readonly string MangaFileExtensions = @"\.cbz|\.cbr|\.png|\.jpeg|\.jpg|\.zip|\.rar";
private ConcurrentDictionary<string, ConcurrentBag<ParserInfo>> _scannedSeries; private ConcurrentDictionary<string, ConcurrentBag<ParserInfo>> _scannedSeries;
public DirectoryService(ILogger<DirectoryService> logger) public DirectoryService(ILogger<DirectoryService> logger)
@ -66,26 +64,18 @@ namespace API.Services
return dirs; return dirs;
} }
// TODO: Refactor API layer to use this
public IEnumerable<DirectoryInfo> ListDirectories(string rootPath)
{
if (!Directory.Exists(rootPath)) return ImmutableList<DirectoryInfo>.Empty;
var di = new DirectoryInfo(rootPath);
var dirs = di.GetDirectories()
.Where(dir => !(dir.Attributes.HasFlag(FileAttributes.Hidden) || dir.Attributes.HasFlag(FileAttributes.System)))
.ToImmutableList();
return dirs;
}
/// <summary>
/// Processes files found during a library scan.
/// </summary>
/// <param name="path"></param>
private void Process(string path) private void Process(string path)
{ {
// NOTE: In current implementation, this never runs. We can probably remove.
if (Directory.Exists(path)) if (Directory.Exists(path))
{ {
DirectoryInfo di = new DirectoryInfo(path); DirectoryInfo di = new DirectoryInfo(path);
Console.WriteLine($"Parsing directory {di.Name}"); _logger.LogDebug($"Parsing directory {di.Name}");
var seriesName = Parser.Parser.ParseSeries(di.Name); var seriesName = Parser.Parser.ParseSeries(di.Name);
if (string.IsNullOrEmpty(seriesName)) if (string.IsNullOrEmpty(seriesName))
@ -102,7 +92,7 @@ namespace API.Services
else else
{ {
var fileName = Path.GetFileName(path); var fileName = Path.GetFileName(path);
Console.WriteLine($"Parsing file {fileName}"); _logger.LogDebug($"Parsing file {fileName}");
var info = Parser.Parser.Parse(fileName); var info = Parser.Parser.Parse(fileName);
if (info.Volumes != string.Empty) if (info.Volumes != string.Empty)
@ -138,109 +128,40 @@ namespace API.Services
public void ScanLibrary(LibraryDto library) public void ScanLibrary(LibraryDto library)
{ {
_scannedSeries = new ConcurrentDictionary<string, ConcurrentBag<ParserInfo>>(); _scannedSeries = new ConcurrentDictionary<string, ConcurrentBag<ParserInfo>>();
//Dictionary<string, IList<ParserInfo>> series = new Dictionary<string, IList<ParserInfo>>();
_logger.LogInformation($"Beginning scan on {library.Name}"); _logger.LogInformation($"Beginning scan on {library.Name}");
foreach (var folderPath in library.Folders) foreach (var folderPath in library.Folders)
{ {
try { try {
// // Temporarily, let's build a simple scanner then optimize to parallelization
//
// // First, let's see if there are any files in rootPath
// var files = GetFiles(folderPath, MangaFileExtensions);
//
// foreach (var file in files)
// {
// // These do not have a folder, so we need to parse them directly
// var parserInfo = Parser.Parser.Parse(file);
// Console.WriteLine(parserInfo);
// }
//
// // Get Directories
// var directories = ListDirectories(folderPath);
// foreach (var directory in directories)
// {
// _logger.LogDebug($"Scanning {directory.Name}");
// var parsedSeries = Parser.Parser.ParseSeries(directory.Name);
//
// // For now, let's skip directories we can't parse information out of. (we are assuming one level deep root)
// if (string.IsNullOrEmpty(parsedSeries)) continue;
//
// _logger.LogDebug($"Parsed Series: {parsedSeries}");
//
// if (!series.ContainsKey(parsedSeries))
// {
// series[parsedSeries] = new List<ParserInfo>();
// }
//
// var foundFiles = GetFiles(directory.FullName, MangaFileExtensions);
// foreach (var foundFile in foundFiles)
// {
// var info = Parser.Parser.Parse(foundFile);
// if (info.Volumes != string.Empty)
// {
// series[parsedSeries].Add(info);
// }
// }
// }
TraverseTreeParallelForEach(folderPath, (f) => TraverseTreeParallelForEach(folderPath, (f) =>
{ {
// Exceptions are no-ops. // Exceptions are no-ops.
try try
{ {
Process(f); Process(f);
//ProcessManga(folderPath, f);
} }
catch (FileNotFoundException) {} catch (FileNotFoundException) {}
catch (IOException) {} catch (IOException) {}
catch (UnauthorizedAccessException) {} catch (UnauthorizedAccessException) {}
catch (SecurityException) {} catch (SecurityException) {}
// Display the filename.
Console.WriteLine(f);
}); });
} }
catch (ArgumentException ex) { catch (ArgumentException ex) {
_logger.LogError(ex, "There was an issue scanning the directory"); _logger.LogError(ex, "The directory '{folderPath}' does not exist");
_logger.LogError($"The directory '{folderPath}' does not exist");
} }
} }
// var filtered = series.Where(kvp => kvp.Value.Count > 0);
// series = filtered.ToDictionary(v => v.Key, v => v.Value);
// Console.WriteLine(series);
// var filtered = _scannedSeries.Where(kvp => kvp.Value.Count > 0);
// series = filtered.ToDictionary(v => v.Key, v => v.Value);
// Console.WriteLine(series);
var filtered = _scannedSeries.Where(kvp => !kvp.Value.IsEmpty); var filtered = _scannedSeries.Where(kvp => !kvp.Value.IsEmpty);
var series = filtered.ToImmutableDictionary(v => v.Key, v => v.Value); var series = filtered.ToImmutableDictionary(v => v.Key, v => v.Value);
Console.WriteLine(series);
// TODO: Perform DB activities on ImmutableDictionary // TODO: Perform DB activities on ImmutableDictionary
//_logger.LogInformation($"Scan completed on {library.Name}. Parsed {series.Keys.Count} series.");
_logger.LogInformation($"Scan completed on {library.Name}. Parsed {series.Keys.Count()} series."); _logger.LogInformation($"Scan completed on {library.Name}. Parsed {series.Keys.Count()} series.");
_scannedSeries = null; _scannedSeries = null;
} }
private static void ProcessManga(string folderPath, string filename) private static void TraverseTreeParallelForEach(string root, Action<string> action)
{
Console.WriteLine($"[ProcessManga] Folder: {folderPath}");
Console.WriteLine($"Found {filename}");
var series = Parser.Parser.ParseSeries(filename);
if (series == string.Empty)
{
series = Parser.Parser.ParseSeries(folderPath);
}
Console.WriteLine($"Series: {series}");
}
public static void TraverseTreeParallelForEach(string root, Action<string> action)
{ {
//Count of files traversed and timer for diagnostic output //Count of files traversed and timer for diagnostic output
int fileCount = 0; int fileCount = 0;
@ -253,14 +174,14 @@ namespace API.Services
Stack<string> dirs = new Stack<string>(); Stack<string> dirs = new Stack<string>();
if (!Directory.Exists(root)) { if (!Directory.Exists(root)) {
throw new ArgumentException(); throw new ArgumentException("The directory doesn't exist");
} }
dirs.Push(root); dirs.Push(root);
while (dirs.Count > 0) { while (dirs.Count > 0) {
string currentDir = dirs.Pop(); string currentDir = dirs.Pop();
string[] subDirs = {}; string[] subDirs;
string[] files = {}; string[] files;
try { try {
subDirs = Directory.GetDirectories(currentDir); subDirs = Directory.GetDirectories(currentDir);
@ -277,10 +198,8 @@ namespace API.Services
} }
try { try {
//files = Directory.GetFiles(currentDir, "*.") files = DirectoryService.GetFiles(currentDir, Parser.Parser.MangaFileExtensions)
files = DirectoryService.GetFiles(currentDir, MangaFileExtensions)
.ToArray(); .ToArray();
//files = Directory.GetFiles(currentDir);
} }
catch (UnauthorizedAccessException e) { catch (UnauthorizedAccessException e) {
Console.WriteLine(e.Message); Console.WriteLine(e.Message);
@ -306,7 +225,7 @@ namespace API.Services
} }
} }
else { else {
Parallel.ForEach(files, () => 0, (file, loopState, localCount) => Parallel.ForEach(files, () => 0, (file, _, localCount) =>
{ action(file); { action(file);
return ++localCount; return ++localCount;
}, },

View File

@ -5,11 +5,11 @@ namespace API.Services
{ {
public class TaskScheduler : ITaskScheduler public class TaskScheduler : ITaskScheduler
{ {
private readonly BackgroundJobServer _client; private BackgroundJobServer Client { get; }
public TaskScheduler() public TaskScheduler()
{ {
_client = new BackgroundJobServer(); Client = new BackgroundJobServer();
} }