using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Collections.Immutable; using System.Diagnostics; using System.IO; using System.Linq; using System.Threading.Tasks; using API.Entities; using API.Interfaces; using API.Parser; using Microsoft.Extensions.Logging; namespace API.Services { public class ScannerService : IScannerService { private readonly IUnitOfWork _unitOfWork; private readonly ILogger _logger; private readonly IArchiveService _archiveService; private ConcurrentDictionary> _scannedSeries; public ScannerService(IUnitOfWork unitOfWork, ILogger logger, IArchiveService archiveService) { _unitOfWork = unitOfWork; _logger = logger; _archiveService = archiveService; } public void ScanLibraries() { var libraries = Task.Run(() => _unitOfWork.LibraryRepository.GetLibrariesAsync()).Result.ToList(); foreach (var lib in libraries) { ScanLibrary(lib.Id, false); } } public void ScanLibrary(int libraryId, bool forceUpdate) { var sw = Stopwatch.StartNew(); Library library; try { library = Task.Run(() => _unitOfWork.LibraryRepository.GetLibraryForIdAsync(libraryId)).Result; } catch (Exception ex) { // This usually only fails if user is not authenticated. _logger.LogError($"There was an issue fetching Library {libraryId}.", ex); return; } _scannedSeries = new ConcurrentDictionary>(); _logger.LogInformation($"Beginning scan on {library.Name}. Forcing metadata update: {forceUpdate}"); var totalFiles = 0; foreach (var folderPath in library.Folders) { if (!forceUpdate && Directory.GetLastWriteTime(folderPath.Path) <= folderPath.LastScanned) { _logger.LogDebug($"{folderPath.Path} hasn't been updated since last scan. Skipping."); continue; } try { totalFiles += DirectoryService.TraverseTreeParallelForEach(folderPath.Path, (f) => { try { ProcessFile(f, folderPath.Path); } catch (FileNotFoundException exception) { _logger.LogError(exception, "The file could not be found"); } }); } catch (ArgumentException ex) { _logger.LogError(ex, $"The directory '{folderPath}' does not exist"); } } var filtered = _scannedSeries.Where(kvp => kvp.Value.Count != 0); var series = filtered.ToImmutableDictionary(v => v.Key, v => v.Value); // Perform DB activities var allSeries = UpsertSeries(libraryId, forceUpdate, series, library); // Remove series that are no longer on disk RemoveSeriesNotOnDisk(allSeries, series, library); foreach (var folder in library.Folders) folder.LastScanned = DateTime.Now; _unitOfWork.LibraryRepository.Update(library); if (Task.Run(() => _unitOfWork.Complete()).Result) { _logger.LogInformation($"Scan completed on {library.Name}. Parsed {series.Keys.Count()} series."); } else { _logger.LogError("There was a critical error that resulted in a failed scan. Please rescan."); } _scannedSeries = null; _logger.LogInformation("Processed {0} files in {1} milliseconds for {2}", totalFiles, sw.ElapsedMilliseconds, library.Name); } private List UpsertSeries(int libraryId, bool forceUpdate, ImmutableDictionary> series, Library library) { var allSeries = Task.Run(() => _unitOfWork.SeriesRepository.GetSeriesForLibraryIdAsync(libraryId)).Result.ToList(); foreach (var seriesKey in series.Keys) { var mangaSeries = allSeries.SingleOrDefault(s => s.Name == seriesKey) ?? new Series { Name = seriesKey, OriginalName = seriesKey, SortName = seriesKey, Summary = "" }; try { mangaSeries = UpdateSeries(mangaSeries, series[seriesKey].ToArray(), forceUpdate); _logger.LogInformation($"Created/Updated series {mangaSeries.Name} for {library.Name} library"); library.Series ??= new List(); library.Series.Add(mangaSeries); } catch (Exception ex) { _logger.LogError(ex, $"There was an error during scanning of library. {seriesKey} will be skipped."); } } return allSeries; } private void RemoveSeriesNotOnDisk(List allSeries, ImmutableDictionary> series, Library library) { var count = 0; foreach (var existingSeries in allSeries) { if (!series.ContainsKey(existingSeries.Name) || !series.ContainsKey(existingSeries.OriginalName)) { // Delete series, there is no file to backup any longer. library.Series?.Remove(existingSeries); count++; } } _logger.LogInformation($"Removed {count} series that are no longer on disk"); } /// /// Attempts to either add a new instance of a show mapping to the scannedSeries bag or adds to an existing. /// /// private void TrackSeries(ParserInfo info) { if (info.Series == string.Empty) return; _scannedSeries.AddOrUpdate(info.Series, new List() {info}, (_, oldValue) => { oldValue ??= new List(); if (!oldValue.Contains(info)) { oldValue.Add(info); } return oldValue; }); } /// /// Processes files found during a library scan. /// Populates a collection of for DB updates later. /// /// Path of a file /// private void ProcessFile(string path, string rootPath) { var info = Parser.Parser.Parse(path, rootPath); if (info == null) { _logger.LogInformation($"Could not parse series from {path}"); return; } TrackSeries(info); } private Series UpdateSeries(Series series, ParserInfo[] infos, bool forceUpdate) { var volumes = UpdateVolumesWithChapters(series, infos, forceUpdate); series.Volumes = volumes; series.Pages = volumes.Sum(v => v.Pages); if (ShouldFindCoverImage(forceUpdate, series.CoverImage)) { var firstCover = volumes.OrderBy(x => x.Number).FirstOrDefault(x => x.Number != 0); if (firstCover == null && volumes.Any()) { firstCover = volumes.FirstOrDefault(x => x.Number == 0); } series.CoverImage = firstCover?.CoverImage; } if (string.IsNullOrEmpty(series.Summary) || forceUpdate) { series.Summary = ""; } return series; } private MangaFile CreateMangaFile(ParserInfo info) { _logger.LogDebug($"Creating File Entry for {info.FullFilePath}"); return new MangaFile() { FilePath = info.FullFilePath, Format = info.Format, NumberOfPages = info.Format == MangaFormat.Archive ? _archiveService.GetNumberOfPagesFromArchive(info.FullFilePath): 1 }; } private bool ShouldFindCoverImage(bool forceUpdate, byte[] coverImage) { return forceUpdate || coverImage == null || !coverImage.Any(); } /// /// /// /// /// /// /// private ICollection UpdateChapters(Volume volume, IEnumerable infos, bool forceUpdate) { var chapters = new List(); foreach (var info in infos) { volume.Chapters ??= new List(); var chapter = volume.Chapters.SingleOrDefault(c => c.Range == info.Chapters) ?? chapters.SingleOrDefault(v => v.Range == info.Chapters) ?? new Chapter() { Number = Parser.Parser.MinimumNumberFromRange(info.Chapters) + "", Range = info.Chapters, }; chapter.Files ??= new List(); var existingFile = chapter.Files.SingleOrDefault(f => f.FilePath == info.FullFilePath); if (existingFile != null) { existingFile.Format = info.Format; existingFile.NumberOfPages = _archiveService.GetNumberOfPagesFromArchive(info.FullFilePath); } else { if (info.Format == MangaFormat.Archive) { chapter.Files.Add(CreateMangaFile(info)); } else { _logger.LogDebug($"Ignoring {info.Filename} as it is not an archive."); } } chapter.Number = Parser.Parser.MinimumNumberFromRange(info.Chapters) + ""; chapter.Range = info.Chapters; chapters.Add(chapter); } foreach (var chapter in chapters) { chapter.Pages = chapter.Files.Sum(f => f.NumberOfPages); if (ShouldFindCoverImage(forceUpdate, chapter.CoverImage)) { chapter.Files ??= new List(); var firstFile = chapter.Files.OrderBy(x => x.Chapter).FirstOrDefault(); if (firstFile != null) chapter.CoverImage = _archiveService.GetCoverImage(firstFile.FilePath, true); } } return chapters; } private ICollection UpdateVolumesWithChapters(Series series, ParserInfo[] infos, bool forceUpdate) { ICollection volumes = new List(); IList existingVolumes = _unitOfWork.SeriesRepository.GetVolumes(series.Id).ToList(); foreach (var info in infos) { var volume = (existingVolumes.SingleOrDefault(v => v.Name == info.Volumes) ?? volumes.SingleOrDefault(v => v.Name == info.Volumes)) ?? new Volume { Name = info.Volumes, Number = Parser.Parser.MinimumNumberFromRange(info.Volumes), }; var chapters = UpdateChapters(volume, infos.Where(pi => pi.Volumes == volume.Name).ToArray(), forceUpdate); volume.Chapters = chapters; volume.Pages = chapters.Sum(c => c.Pages); volumes.Add(volume); } foreach (var volume in volumes) { if (ShouldFindCoverImage(forceUpdate, volume.CoverImage)) { // TODO: Create a custom sorter for Chapters so it's consistent across the application var firstChapter = volume.Chapters.OrderBy(x => Double.Parse(x.Number)).FirstOrDefault(); var firstFile = firstChapter?.Files.OrderBy(x => x.Chapter).FirstOrDefault(); if (firstFile != null) volume.CoverImage = _archiveService.GetCoverImage(firstFile.FilePath, true); } } return volumes; } public void ScanSeries(int libraryId, int seriesId) { throw new NotImplementedException(); } } }