using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Collections.Immutable; using System.Diagnostics; using System.IO; using System.IO.Compression; using System.Linq; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using API.DTOs; using API.Entities; using API.Extensions; using API.Interfaces; using API.IO; using API.Parser; using Microsoft.Extensions.Logging; using NetVips; namespace API.Services { public class DirectoryService : IDirectoryService { private readonly ILogger _logger; private readonly ISeriesRepository _seriesRepository; private readonly ILibraryRepository _libraryRepository; private ConcurrentDictionary> _scannedSeries; public DirectoryService(ILogger logger, ISeriesRepository seriesRepository, ILibraryRepository libraryRepository) { _logger = logger; _seriesRepository = seriesRepository; _libraryRepository = libraryRepository; } /// /// Given a set of regex search criteria, get files in the given path. /// /// Directory to search /// Regex version of search pattern (ie \.mp3|\.mp4) /// SearchOption to use, defaults to TopDirectoryOnly /// List of file paths private static IEnumerable GetFiles(string path, string searchPatternExpression = "", SearchOption searchOption = SearchOption.TopDirectoryOnly) { var reSearchPattern = new Regex(searchPatternExpression, RegexOptions.IgnoreCase); return Directory.EnumerateFiles(path, "*", searchOption) .Where(file => reSearchPattern.IsMatch(Path.GetExtension(file))); } public IEnumerable ListDirectory(string rootPath) { if (!Directory.Exists(rootPath)) return ImmutableList.Empty; var di = new DirectoryInfo(rootPath); var dirs = di.GetDirectories() .Where(dir => !(dir.Attributes.HasFlag(FileAttributes.Hidden) || dir.Attributes.HasFlag(FileAttributes.System))) .Select(d => d.Name).ToImmutableList(); return dirs; } public IList ListFiles(string rootPath) { if (!Directory.Exists(rootPath)) return ImmutableList.Empty; return Directory.GetFiles(rootPath); } /// /// Processes files found during a library scan. Generates a collection of series->volume->files for DB processing later. /// /// Path of a file private void Process(string path) { var fileName = Path.GetFileName(path); _logger.LogDebug($"Parsing file {fileName}"); var info = Parser.Parser.Parse(fileName); info.FullFilePath = path; if (info.Volumes == string.Empty) { return; } ConcurrentBag newBag = new ConcurrentBag(); if (_scannedSeries.TryGetValue(info.Series, out var tempBag)) { var existingInfos = tempBag.ToArray(); foreach (var existingInfo in existingInfos) { newBag.Add(existingInfo); } } else { tempBag = new ConcurrentBag(); } newBag.Add(info); if (!_scannedSeries.TryUpdate(info.Series, newBag, tempBag)) { _scannedSeries.TryAdd(info.Series, newBag); } } private Series UpdateSeries(string seriesName, ParserInfo[] infos, bool forceUpdate) { var series = _seriesRepository.GetSeriesByName(seriesName) ?? new Series { Name = seriesName, OriginalName = seriesName, SortName = seriesName, Summary = "" // TODO: Check if comicInfo.xml in file and parse metadata out. }; var volumes = UpdateVolumes(series, infos, forceUpdate); series.Volumes = volumes; series.CoverImage = volumes.OrderBy(x => x.Number).FirstOrDefault()?.CoverImage; return series; } private MangaFile CreateMangaFile(ParserInfo info) { _logger.LogDebug($"Creating File Entry for {info.FullFilePath}"); int.TryParse(info.Chapters, out var chapter); _logger.LogDebug($"Found Chapter: {chapter}"); return new MangaFile() { FilePath = info.FullFilePath, Chapter = chapter, NumberOfPages = GetNumberOfPagesFromArchive(info.FullFilePath) }; } /// /// Creates or Updates volumes for a given series /// /// Series wanting to be updated /// Parser info /// Forces metadata update (cover image) even if it's already been set. /// Updated Volumes for given series private ICollection UpdateVolumes(Series series, ParserInfo[] infos, bool forceUpdate) { ICollection volumes = new List(); IList existingVolumes = _seriesRepository.GetVolumes(series.Id).ToList(); foreach (var info in infos) { var existingVolume = existingVolumes.SingleOrDefault(v => v.Name == info.Volumes); if (existingVolume != null) { var existingFile = existingVolume.Files.SingleOrDefault(f => f.FilePath == info.FullFilePath); if (existingFile != null) { existingFile.Chapter = Int32.Parse(info.Chapters); existingFile.Format = info.Format; existingFile.NumberOfPages = GetNumberOfPagesFromArchive(info.FullFilePath); } else { existingVolume.Files.Add(CreateMangaFile(info)); } volumes.Add(existingVolume); } else { existingVolume = volumes.SingleOrDefault(v => v.Name == info.Volumes); if (existingVolume != null) { existingVolume.Files.Add(CreateMangaFile(info)); } else { var vol = new Volume() { Name = info.Volumes, Number = Int32.Parse(info.Volumes), Files = new List() { CreateMangaFile(info) } }; volumes.Add(vol); } } Console.WriteLine($"Adding volume {volumes.Last().Number} with File: {info.Filename}"); } foreach (var volume in volumes) { if (forceUpdate || volume.CoverImage == null || !volume.Files.Any()) { var firstFile = volume.Files.OrderBy(x => x.Chapter).FirstOrDefault()?.FilePath; volume.CoverImage = ImageProvider.GetCoverImage(firstFile, true); } volume.Pages = volume.Files.Sum(x => x.NumberOfPages); } return volumes; } public void ScanLibrary(int libraryId, bool forceUpdate) { var sw = Stopwatch.StartNew(); Library library; try { library = Task.Run(() => _libraryRepository.GetLibraryForIdAsync(libraryId)).Result; } catch (Exception ex) { // This usually only fails if user is not authenticated. _logger.LogError($"There was an issue fetching Library {libraryId}.", ex); return; } _scannedSeries = new ConcurrentDictionary>(); _logger.LogInformation($"Beginning scan on {library.Name}"); var totalFiles = 0; foreach (var folderPath in library.Folders) { try { totalFiles = TraverseTreeParallelForEach(folderPath.Path, (f) => { try { Process(f); } catch (FileNotFoundException exception) { _logger.LogError(exception, "The file could not be found"); } }); } catch (ArgumentException ex) { _logger.LogError(ex, $"The directory '{folderPath}' does not exist"); } } var filtered = _scannedSeries.Where(kvp => !kvp.Value.IsEmpty); var series = filtered.ToImmutableDictionary(v => v.Key, v => v.Value); // Perform DB activities library.Series = new List(); // Temp delete everything until we can mark items Unavailable foreach (var seriesKey in series.Keys) { var mangaSeries = UpdateSeries(seriesKey, series[seriesKey].ToArray(), forceUpdate); _logger.LogInformation($"Created/Updated series {mangaSeries.Name}"); library.Series.Add(mangaSeries); } _libraryRepository.Update(library); if (_libraryRepository.SaveAll()) { _logger.LogInformation($"Scan completed on {library.Name}. Parsed {series.Keys.Count()} series."); } else { _logger.LogError("There was a critical error that resulted in a failed scan. Please rescan."); } _scannedSeries = null; _logger.LogInformation("Processed {0} files in {1} milliseconds for {2}", totalFiles, sw.ElapsedMilliseconds, library.Name); } public string GetExtractPath(int volumeId) { return Path.Join(Directory.GetCurrentDirectory(), $"../cache/{volumeId}/"); } public string ExtractArchive(string archivePath, int volumeId) { if (!File.Exists(archivePath) || !Parser.Parser.IsArchive(archivePath)) { _logger.LogError($"Archive {archivePath} could not be found."); return ""; } var extractPath = GetExtractPath(volumeId); if (Directory.Exists(extractPath)) { _logger.LogInformation($"Archive {archivePath} has already been extracted. Returning existing folder."); return extractPath; } using ZipArchive archive = ZipFile.OpenRead(archivePath); if (!archive.HasFiles()) return ""; archive.ExtractToDirectory(extractPath); _logger.LogInformation($"Extracting archive to {extractPath}"); return extractPath; } public string ExtractArchive(string archivePath, string extractPath) { if (!File.Exists(archivePath) || !Parser.Parser.IsArchive(archivePath)) { _logger.LogError($"Archive {archivePath} could not be found."); return ""; } if (Directory.Exists(extractPath)) { _logger.LogDebug($"Archive {archivePath} has already been extracted. Returning existing folder."); return extractPath; } using ZipArchive archive = ZipFile.OpenRead(archivePath); if (!archive.HasFiles()) return ""; archive.ExtractToDirectory(extractPath); _logger.LogDebug($"Extracting archive to {extractPath}"); return extractPath; } private int GetNumberOfPagesFromArchive(string archivePath) { if (!File.Exists(archivePath) || !Parser.Parser.IsArchive(archivePath)) { _logger.LogError($"Archive {archivePath} could not be found."); return 0; } using ZipArchive archive = ZipFile.OpenRead(archivePath); Console.WriteLine(); return archive.Entries.Count(e => Parser.Parser.IsImage(e.FullName)); } public async Task ReadImageAsync(string imagePath) { using var image = Image.NewFromFile(imagePath); return new ImageDto { Content = await File.ReadAllBytesAsync(imagePath), Filename = Path.GetFileNameWithoutExtension(imagePath), FullPath = Path.GetFullPath(imagePath), Width = image.Width, Height = image.Height, Format = image.Format }; } /// /// Recursively scans files and applies an action on them. This uses as many cores the underlying PC has to speed /// up processing. /// /// Directory to scan /// Action to apply on file path /// private static int TraverseTreeParallelForEach(string root, Action action) { //Count of files traversed and timer for diagnostic output int fileCount = 0; // Determine whether to parallelize file processing on each folder based on processor count. int procCount = Environment.ProcessorCount; // Data structure to hold names of subfolders to be examined for files. Stack dirs = new Stack(); if (!Directory.Exists(root)) { throw new ArgumentException("The directory doesn't exist"); } dirs.Push(root); while (dirs.Count > 0) { string currentDir = dirs.Pop(); string[] subDirs; string[] files; try { subDirs = Directory.GetDirectories(currentDir); } // Thrown if we do not have discovery permission on the directory. catch (UnauthorizedAccessException e) { Console.WriteLine(e.Message); continue; } // Thrown if another process has deleted the directory after we retrieved its name. catch (DirectoryNotFoundException e) { Console.WriteLine(e.Message); continue; } try { files = DirectoryService.GetFiles(currentDir, Parser.Parser.MangaFileExtensions) .ToArray(); } catch (UnauthorizedAccessException e) { Console.WriteLine(e.Message); continue; } catch (DirectoryNotFoundException e) { Console.WriteLine(e.Message); continue; } catch (IOException e) { Console.WriteLine(e.Message); continue; } // Execute in parallel if there are enough files in the directory. // Otherwise, execute sequentially.Files are opened and processed // synchronously but this could be modified to perform async I/O. try { if (files.Length < procCount) { foreach (var file in files) { action(file); fileCount++; } } else { Parallel.ForEach(files, () => 0, (file, _, localCount) => { action(file); return ++localCount; }, (c) => { Interlocked.Add(ref fileCount, c); }); } } catch (AggregateException ae) { ae.Handle((ex) => { if (ex is UnauthorizedAccessException) { // Here we just output a message and go on. Console.WriteLine(ex.Message); return true; } // Handle other exceptions here if necessary... return false; }); } // Push the subdirectories onto the stack for traversal. // This could also be done before handing the files. foreach (string str in subDirs) dirs.Push(str); } return fileCount; } } }