using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Collections.Immutable; using System.Diagnostics; using System.IO; using System.Linq; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using API.Entities; using API.Interfaces; using API.IO; using API.Parser; using Hangfire; using Microsoft.Extensions.Logging; namespace API.Services { public class DirectoryService : IDirectoryService { private readonly ILogger _logger; private readonly ISeriesRepository _seriesRepository; private readonly ILibraryRepository _libraryRepository; private ConcurrentDictionary> _scannedSeries; public DirectoryService(ILogger logger, ISeriesRepository seriesRepository, ILibraryRepository libraryRepository) { _logger = logger; _seriesRepository = seriesRepository; _libraryRepository = libraryRepository; } /// /// Given a set of regex search criteria, get files in the given path. /// /// Directory to search /// Regex version of search pattern (ie \.mp3|\.mp4) /// SearchOption to use, defaults to TopDirectoryOnly /// List of file paths public static IEnumerable GetFiles(string path, string searchPatternExpression = "", SearchOption searchOption = SearchOption.TopDirectoryOnly) { Regex reSearchPattern = new Regex(searchPatternExpression, RegexOptions.IgnoreCase); return Directory.EnumerateFiles(path, "*", searchOption) .Where(file => reSearchPattern.IsMatch(Path.GetExtension(file))); } /// /// Lists out top-level folders for a given directory. Filters out System and Hidden folders. /// /// Absolute path /// List of folder names public IEnumerable ListDirectory(string rootPath) { if (!Directory.Exists(rootPath)) return ImmutableList.Empty; var di = new DirectoryInfo(rootPath); var dirs = di.GetDirectories() .Where(dir => !(dir.Attributes.HasFlag(FileAttributes.Hidden) || dir.Attributes.HasFlag(FileAttributes.System))) .Select(d => d.Name).ToImmutableList(); return dirs; } /// /// Processes files found during a library scan. Generates a collection of series->volume->files for DB processing later. /// /// Path of a file private void Process(string path) { var fileName = Path.GetFileName(path); _logger.LogDebug($"Parsing file {fileName}"); var info = Parser.Parser.Parse(fileName); info.FullFilePath = path; if (info.Volumes == string.Empty) { return; } ConcurrentBag tempBag; ConcurrentBag newBag = new ConcurrentBag(); if (_scannedSeries.TryGetValue(info.Series, out tempBag)) { var existingInfos = tempBag.ToArray(); foreach (var existingInfo in existingInfos) { newBag.Add(existingInfo); } } else { tempBag = new ConcurrentBag(); } newBag.Add(info); if (!_scannedSeries.TryUpdate(info.Series, newBag, tempBag)) { _scannedSeries.TryAdd(info.Series, newBag); } } private Series UpdateSeries(string seriesName, ParserInfo[] infos, bool forceUpdate) { var series = _seriesRepository.GetSeriesByName(seriesName); if (series == null) { series = new Series() { Name = seriesName, OriginalName = seriesName, SortName = seriesName, Summary = "" }; } var volumes = UpdateVolumes(series, infos, forceUpdate); series.Volumes = volumes; // TODO: Instead of taking first entry, re-calculate without compression series.CoverImage = volumes.OrderBy(x => x.Number).FirstOrDefault()?.CoverImage; return series; } /// /// Creates or Updates volumes for a given series /// /// Series wanting to be updated /// Parser info /// Forces metadata update (cover image) even if it's already been set. /// Updated Volumes for given series private ICollection UpdateVolumes(Series series, ParserInfo[] infos, bool forceUpdate) { ICollection volumes = new List(); IList existingVolumes = _seriesRepository.GetVolumes(series.Id).ToList(); foreach (var info in infos) { var existingVolume = existingVolumes.SingleOrDefault(v => v.Name == info.Volumes); if (existingVolume != null) { // Temp let's overwrite all files (we need to enhance to update files) existingVolume.Files = new List() { new MangaFile() { FilePath = info.File } }; if (forceUpdate || existingVolume.CoverImage == null || existingVolumes.Count == 0) { existingVolume.CoverImage = ImageProvider.GetCoverImage(info.FullFilePath, true); } volumes.Add(existingVolume); } else { var vol = new Volume() { Name = info.Volumes, Number = Int32.Parse(info.Volumes), CoverImage = ImageProvider.GetCoverImage(info.FullFilePath, true), Files = new List() { new MangaFile() { FilePath = info.File } } }; volumes.Add(vol); } Console.WriteLine($"Adding volume {volumes.Last().Number} with File: {info.File}"); } return volumes; } public void ScanLibrary(int libraryId, bool forceUpdate = false) { var library = Task.Run(() => _libraryRepository.GetLibraryForIdAsync(libraryId)).Result; _scannedSeries = new ConcurrentDictionary>(); _logger.LogInformation($"Beginning scan on {library.Name}"); foreach (var folderPath in library.Folders) { try { TraverseTreeParallelForEach(folderPath.Path, (f) => { try { Process(f); } catch (FileNotFoundException exception) { _logger.LogError(exception, "The file could not be found"); } }); } catch (ArgumentException ex) { _logger.LogError(ex, $"The directory '{folderPath}' does not exist"); } } var filtered = _scannedSeries.Where(kvp => !kvp.Value.IsEmpty); var series = filtered.ToImmutableDictionary(v => v.Key, v => v.Value); // Perform DB activities library.Series = new List(); // Temp delete everything until we can mark items Unavailable foreach (var seriesKey in series.Keys) { var s = UpdateSeries(seriesKey, series[seriesKey].ToArray(), forceUpdate); _logger.LogInformation($"Created/Updated series {s.Name}"); library.Series.Add(s); } _libraryRepository.Update(library); if (_libraryRepository.SaveAll()) { _logger.LogInformation($"Scan completed on {library.Name}. Parsed {series.Keys.Count()} series."); } else { _logger.LogError("There was a critical error that resulted in a failed scan. Please rescan."); } _scannedSeries = null; } private static void TraverseTreeParallelForEach(string root, Action action) { //Count of files traversed and timer for diagnostic output int fileCount = 0; var sw = Stopwatch.StartNew(); // Determine whether to parallelize file processing on each folder based on processor count. int procCount = Environment.ProcessorCount; // Data structure to hold names of subfolders to be examined for files. Stack dirs = new Stack(); if (!Directory.Exists(root)) { throw new ArgumentException("The directory doesn't exist"); } dirs.Push(root); while (dirs.Count > 0) { string currentDir = dirs.Pop(); string[] subDirs; string[] files; try { subDirs = Directory.GetDirectories(currentDir); } // Thrown if we do not have discovery permission on the directory. catch (UnauthorizedAccessException e) { Console.WriteLine(e.Message); continue; } // Thrown if another process has deleted the directory after we retrieved its name. catch (DirectoryNotFoundException e) { Console.WriteLine(e.Message); continue; } try { files = DirectoryService.GetFiles(currentDir, Parser.Parser.MangaFileExtensions) .ToArray(); } catch (UnauthorizedAccessException e) { Console.WriteLine(e.Message); continue; } catch (DirectoryNotFoundException e) { Console.WriteLine(e.Message); continue; } catch (IOException e) { Console.WriteLine(e.Message); continue; } // Execute in parallel if there are enough files in the directory. // Otherwise, execute sequentially.Files are opened and processed // synchronously but this could be modified to perform async I/O. try { if (files.Length < procCount) { foreach (var file in files) { action(file); fileCount++; } } else { Parallel.ForEach(files, () => 0, (file, _, localCount) => { action(file); return ++localCount; }, (c) => { Interlocked.Add(ref fileCount, c); }); } } catch (AggregateException ae) { ae.Handle((ex) => { if (ex is UnauthorizedAccessException) { // Here we just output a message and go on. Console.WriteLine(ex.Message); return true; } // Handle other exceptions here if necessary... return false; }); } // Push the subdirectories onto the stack for traversal. // This could also be done before handing the files. foreach (string str in subDirs) dirs.Push(str); } // For diagnostic purposes. Console.WriteLine("Processed {0} files in {1} milliseconds", fileCount, sw.ElapsedMilliseconds); } } }