using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Linq; using System.Threading.Tasks; using API.Comparators; using API.Data; using API.Data.Metadata; using API.Data.Repositories; using API.Data.Scanner; using API.Entities; using API.Entities.Enums; using API.Extensions; using API.Helpers; using API.Interfaces; using API.Interfaces.Services; using API.SignalR; using Microsoft.AspNetCore.SignalR; using Microsoft.Extensions.Logging; namespace API.Services; public class MetadataService : IMetadataService { private readonly IUnitOfWork _unitOfWork; private readonly ILogger _logger; private readonly IArchiveService _archiveService; private readonly IBookService _bookService; private readonly IImageService _imageService; private readonly IHubContext _messageHub; private readonly ICacheHelper _cacheHelper; private readonly ChapterSortComparerZeroFirst _chapterSortComparerForInChapterSorting = new ChapterSortComparerZeroFirst(); public MetadataService(IUnitOfWork unitOfWork, ILogger logger, IArchiveService archiveService, IBookService bookService, IImageService imageService, IHubContext messageHub, ICacheHelper cacheHelper) { _unitOfWork = unitOfWork; _logger = logger; _archiveService = archiveService; _bookService = bookService; _imageService = imageService; _messageHub = messageHub; _cacheHelper = cacheHelper; } /// /// Gets the cover image for the file /// /// Has side effect of marking the file as updated /// /// /// /// private string GetCoverImage(MangaFile file, int volumeId, int chapterId) { //file.UpdateLastModified(); switch (file.Format) { case MangaFormat.Pdf: case MangaFormat.Epub: return _bookService.GetCoverImage(file.FilePath, ImageService.GetChapterFormat(chapterId, volumeId)); case MangaFormat.Image: var coverImage = _imageService.GetCoverFile(file); return _imageService.GetCoverImage(coverImage, ImageService.GetChapterFormat(chapterId, volumeId)); case MangaFormat.Archive: return _archiveService.GetCoverImage(file.FilePath, ImageService.GetChapterFormat(chapterId, volumeId)); case MangaFormat.Unknown: default: return string.Empty; } } /// /// Updates the metadata for a Chapter /// /// /// Force updating cover image even if underlying file has not been modified or chapter already has a cover image private bool UpdateChapterCoverImage(Chapter chapter, bool forceUpdate) { var firstFile = chapter.Files.OrderBy(x => x.Chapter).FirstOrDefault(); if (!_cacheHelper.ShouldUpdateCoverImage(Path.Join(DirectoryService.CoverImageDirectory, chapter.CoverImage), firstFile, chapter.Created, forceUpdate, chapter.CoverImageLocked)) return false; _logger.LogDebug("[MetadataService] Generating cover image for {File}", firstFile?.FilePath); chapter.CoverImage = GetCoverImage(firstFile, chapter.VolumeId, chapter.Id); return true; } private void UpdateChapterMetadata(Chapter chapter, ICollection allPeople, bool forceUpdate) { var firstFile = chapter.Files.OrderBy(x => x.Chapter).FirstOrDefault(); if (firstFile == null || _cacheHelper.HasFileNotChangedSinceCreationOrLastScan(chapter, forceUpdate, firstFile)) return; UpdateChapterFromComicInfo(chapter, allPeople, firstFile); firstFile.UpdateLastModified(); } private void UpdateChapterFromComicInfo(Chapter chapter, ICollection allPeople, MangaFile firstFile) { var comicInfo = GetComicInfo(firstFile); // TODO: Think about letting the higher level loop have access for series to avoid duplicate IO operations if (comicInfo == null) return; if (!string.IsNullOrEmpty(comicInfo.Title)) { chapter.TitleName = comicInfo.Title.Trim(); } if (!string.IsNullOrEmpty(comicInfo.Colorist)) { var people = comicInfo.Colorist.Split(","); PersonHelper.RemovePeople(chapter.People, people, PersonRole.Colorist); PersonHelper.UpdatePeople(allPeople, people, PersonRole.Colorist, person => PersonHelper.AddPersonIfNotExists(chapter.People, person)); } if (!string.IsNullOrEmpty(comicInfo.Writer)) { var people = comicInfo.Writer.Split(","); PersonHelper.RemovePeople(chapter.People, people, PersonRole.Writer); PersonHelper.UpdatePeople(allPeople, people, PersonRole.Writer, person => PersonHelper.AddPersonIfNotExists(chapter.People, person)); } if (!string.IsNullOrEmpty(comicInfo.Editor)) { var people = comicInfo.Editor.Split(","); PersonHelper.RemovePeople(chapter.People, people, PersonRole.Editor); PersonHelper.UpdatePeople(allPeople, people, PersonRole.Editor, person => PersonHelper.AddPersonIfNotExists(chapter.People, person)); } if (!string.IsNullOrEmpty(comicInfo.Inker)) { var people = comicInfo.Inker.Split(","); PersonHelper.RemovePeople(chapter.People, people, PersonRole.Inker); PersonHelper.UpdatePeople(allPeople, people, PersonRole.Inker, person => PersonHelper.AddPersonIfNotExists(chapter.People, person)); } if (!string.IsNullOrEmpty(comicInfo.Letterer)) { var people = comicInfo.Letterer.Split(","); PersonHelper.RemovePeople(chapter.People, people, PersonRole.Letterer); PersonHelper.UpdatePeople(allPeople, people, PersonRole.Letterer, person => PersonHelper.AddPersonIfNotExists(chapter.People, person)); } if (!string.IsNullOrEmpty(comicInfo.Penciller)) { var people = comicInfo.Penciller.Split(","); PersonHelper.RemovePeople(chapter.People, people, PersonRole.Penciller); PersonHelper.UpdatePeople(allPeople, people, PersonRole.Penciller, person => PersonHelper.AddPersonIfNotExists(chapter.People, person)); } if (!string.IsNullOrEmpty(comicInfo.CoverArtist)) { var people = comicInfo.CoverArtist.Split(","); PersonHelper.RemovePeople(chapter.People, people, PersonRole.CoverArtist); PersonHelper.UpdatePeople(allPeople, people, PersonRole.CoverArtist, person => PersonHelper.AddPersonIfNotExists(chapter.People, person)); } if (!string.IsNullOrEmpty(comicInfo.Publisher)) { var people = comicInfo.Publisher.Split(","); PersonHelper.RemovePeople(chapter.People, people, PersonRole.Publisher); PersonHelper.UpdatePeople(allPeople, people, PersonRole.Publisher, person => PersonHelper.AddPersonIfNotExists(chapter.People, person)); } } /// /// Updates the cover image for a Volume /// /// /// Force updating cover image even if underlying file has not been modified or chapter already has a cover image private bool UpdateVolumeCoverImage(Volume volume, bool forceUpdate) { // We need to check if Volume coverImage matches first chapters if forceUpdate is false if (volume == null || !_cacheHelper.ShouldUpdateCoverImage(Path.Join(DirectoryService.CoverImageDirectory, volume.CoverImage), null, volume.Created, forceUpdate)) return false; volume.Chapters ??= new List(); var firstChapter = volume.Chapters.OrderBy(x => double.Parse(x.Number), _chapterSortComparerForInChapterSorting).FirstOrDefault(); if (firstChapter == null) return false; volume.CoverImage = firstChapter.CoverImage; return true; } /// /// Updates metadata for Series /// /// /// Force updating cover image even if underlying file has not been modified or chapter already has a cover image private void UpdateSeriesCoverImage(Series series, bool forceUpdate) { if (series == null) return; // NOTE: This will fail if we replace the cover of the first volume on a first scan. Because the series will already have a cover image if (!_cacheHelper.ShouldUpdateCoverImage(Path.Join(DirectoryService.CoverImageDirectory, series.CoverImage), null, series.Created, forceUpdate, series.CoverImageLocked)) return; series.Volumes ??= new List(); var firstCover = series.Volumes.GetCoverImage(series.Format); string coverImage = null; if (firstCover == null && series.Volumes.Any()) { // If firstCover is null and one volume, the whole series is Chapters under Vol 0. if (series.Volumes.Count == 1) { coverImage = series.Volumes[0].Chapters.OrderBy(c => double.Parse(c.Number), _chapterSortComparerForInChapterSorting) .FirstOrDefault(c => !c.IsSpecial)?.CoverImage; } if (!_cacheHelper.CoverImageExists(coverImage)) { coverImage = series.Volumes[0].Chapters.OrderBy(c => double.Parse(c.Number), _chapterSortComparerForInChapterSorting) .FirstOrDefault()?.CoverImage; } } series.CoverImage = firstCover?.CoverImage ?? coverImage; } private void UpdateSeriesMetadata(Series series, ICollection allPeople, ICollection allGenres, bool forceUpdate) { var isBook = series.Library.Type == LibraryType.Book; var firstVolume = series.Volumes.OrderBy(c => c.Number, new ChapterSortComparer()).FirstWithChapters(isBook); var firstChapter = firstVolume?.Chapters.GetFirstChapterWithFiles(); var firstFile = firstChapter?.Files.FirstOrDefault(); if (firstFile == null || _cacheHelper.HasFileNotChangedSinceCreationOrLastScan(firstChapter, forceUpdate, firstFile)) return; if (Parser.Parser.IsPdf(firstFile.FilePath)) return; var comicInfo = GetComicInfo(firstFile); if (comicInfo == null) return; // Summary Info if (!string.IsNullOrEmpty(comicInfo.Summary)) { series.Metadata.Summary = comicInfo.Summary; // NOTE: I can move this to the bottom as I have a comicInfo selection, save me an extra read } foreach (var chapter in series.Volumes.SelectMany(volume => volume.Chapters)) { PersonHelper.UpdatePeople(allPeople, chapter.People.Where(p => p.Role == PersonRole.Writer).Select(p => p.Name), PersonRole.Writer, person => PersonHelper.AddPersonIfNotExists(series.Metadata.People, person)); PersonHelper.UpdatePeople(allPeople, chapter.People.Where(p => p.Role == PersonRole.CoverArtist).Select(p => p.Name), PersonRole.CoverArtist, person => PersonHelper.AddPersonIfNotExists(series.Metadata.People, person)); PersonHelper.UpdatePeople(allPeople, chapter.People.Where(p => p.Role == PersonRole.Publisher).Select(p => p.Name), PersonRole.Publisher, person => PersonHelper.AddPersonIfNotExists(series.Metadata.People, person)); PersonHelper.UpdatePeople(allPeople, chapter.People.Where(p => p.Role == PersonRole.Character).Select(p => p.Name), PersonRole.Character, person => PersonHelper.AddPersonIfNotExists(series.Metadata.People, person)); PersonHelper.UpdatePeople(allPeople, chapter.People.Where(p => p.Role == PersonRole.Colorist).Select(p => p.Name), PersonRole.Colorist, person => PersonHelper.AddPersonIfNotExists(series.Metadata.People, person)); PersonHelper.UpdatePeople(allPeople, chapter.People.Where(p => p.Role == PersonRole.Editor).Select(p => p.Name), PersonRole.Editor, person => PersonHelper.AddPersonIfNotExists(series.Metadata.People, person)); PersonHelper.UpdatePeople(allPeople, chapter.People.Where(p => p.Role == PersonRole.Inker).Select(p => p.Name), PersonRole.Inker, person => PersonHelper.AddPersonIfNotExists(series.Metadata.People, person)); PersonHelper.UpdatePeople(allPeople, chapter.People.Where(p => p.Role == PersonRole.Letterer).Select(p => p.Name), PersonRole.Letterer, person => PersonHelper.AddPersonIfNotExists(series.Metadata.People, person)); PersonHelper.UpdatePeople(allPeople, chapter.People.Where(p => p.Role == PersonRole.Penciller).Select(p => p.Name), PersonRole.Penciller, person => PersonHelper.AddPersonIfNotExists(series.Metadata.People, person)); } var comicInfos = series.Volumes .SelectMany(volume => volume.Chapters) .SelectMany(c => c.Files) .Select(GetComicInfo) .Where(ci => ci != null) .ToList(); var genres = comicInfos.SelectMany(i => i.Genre.Split(",")).Distinct().ToList(); var people = series.Volumes.SelectMany(volume => volume.Chapters).SelectMany(c => c.People).ToList(); PersonHelper.KeepOnlySamePeopleBetweenLists(series.Metadata.People, people, person => series.Metadata.People.Remove(person)); GenreHelper.UpdateGenre(allGenres, genres, false, genre => GenreHelper.AddGenreIfNotExists(series.Metadata.Genres, genre)); GenreHelper.KeepOnlySameGenreBetweenLists(series.Metadata.Genres, genres.Select(g => DbFactory.Genre(g, false)).ToList(), genre => series.Metadata.Genres.Remove(genre)); } private ComicInfo GetComicInfo(MangaFile firstFile) { if (firstFile?.Format is MangaFormat.Archive or MangaFormat.Epub) { return Parser.Parser.IsEpub(firstFile.FilePath) ? _bookService.GetComicInfo(firstFile.FilePath) : _archiveService.GetComicInfo(firstFile.FilePath); } return null; } /// /// /// /// This cannot have any Async code within. It is used within Parallel.ForEach /// /// private void ProcessSeriesMetadataUpdate(Series series, IDictionary> chapterIds, ICollection allPeople, ICollection allGenres, bool forceUpdate) { _logger.LogDebug("[MetadataService] Processing series {SeriesName}", series.OriginalName); try { var volumeUpdated = false; foreach (var volume in series.Volumes) { var chapterUpdated = false; foreach (var chapter in volume.Chapters) { chapterUpdated = UpdateChapterCoverImage(chapter, forceUpdate); UpdateChapterMetadata(chapter, allPeople, forceUpdate || chapterUpdated); } volumeUpdated = UpdateVolumeCoverImage(volume, chapterUpdated || forceUpdate); } UpdateSeriesCoverImage(series, volumeUpdated || forceUpdate); UpdateSeriesMetadata(series, allPeople, allGenres, forceUpdate); } catch (Exception ex) { _logger.LogError(ex, "[MetadataService] There was an exception during updating metadata for {SeriesName} ", series.Name); } } /// /// Refreshes Metadata for a whole library /// /// This can be heavy on memory first run /// /// Force updating cover image even if underlying file has not been modified or chapter already has a cover image public async Task RefreshMetadata(int libraryId, bool forceUpdate = false) { var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(libraryId, LibraryIncludes.None); _logger.LogInformation("[MetadataService] Beginning metadata refresh of {LibraryName}", library.Name); var chunkInfo = await _unitOfWork.SeriesRepository.GetChunkInfo(library.Id); var stopwatch = Stopwatch.StartNew(); var totalTime = 0L; _logger.LogInformation("[MetadataService] Refreshing Library {LibraryName}. Total Items: {TotalSize}. Total Chunks: {TotalChunks} with {ChunkSize} size", library.Name, chunkInfo.TotalSize, chunkInfo.TotalChunks, chunkInfo.ChunkSize); await _messageHub.Clients.All.SendAsync(SignalREvents.RefreshMetadataProgress, MessageFactory.RefreshMetadataProgressEvent(library.Id, 0F)); for (var chunk = 1; chunk <= chunkInfo.TotalChunks; chunk++) { if (chunkInfo.TotalChunks == 0) continue; totalTime += stopwatch.ElapsedMilliseconds; stopwatch.Restart(); _logger.LogInformation("[MetadataService] Processing chunk {ChunkNumber} / {TotalChunks} with size {ChunkSize}. Series ({SeriesStart} - {SeriesEnd}", chunk, chunkInfo.TotalChunks, chunkInfo.ChunkSize, chunk * chunkInfo.ChunkSize, (chunk + 1) * chunkInfo.ChunkSize); var nonLibrarySeries = await _unitOfWork.SeriesRepository.GetFullSeriesForLibraryIdAsync(library.Id, new UserParams() { PageNumber = chunk, PageSize = chunkInfo.ChunkSize }); _logger.LogDebug("[MetadataService] Fetched {SeriesCount} series for refresh", nonLibrarySeries.Count); var chapterIds = await _unitOfWork.SeriesRepository.GetChapterIdWithSeriesIdForSeriesAsync(nonLibrarySeries.Select(s => s.Id).ToArray()); var allPeople = await _unitOfWork.PersonRepository.GetAllPeople(); var allGenres = await _unitOfWork.GenreRepository.GetAllGenres(); var seriesIndex = 0; foreach (var series in nonLibrarySeries) { try { ProcessSeriesMetadataUpdate(series, chapterIds, allPeople, allGenres, forceUpdate); } catch (Exception ex) { _logger.LogError(ex, "[MetadataService] There was an exception during metadata refresh for {SeriesName}", series.Name); } var index = chunk * seriesIndex; var progress = Math.Max(0F, Math.Min(1F, index * 1F / chunkInfo.TotalSize)); await _messageHub.Clients.All.SendAsync(SignalREvents.RefreshMetadataProgress, MessageFactory.RefreshMetadataProgressEvent(library.Id, progress)); seriesIndex++; } await _unitOfWork.CommitAsync(); foreach (var series in nonLibrarySeries) { await _messageHub.Clients.All.SendAsync(SignalREvents.RefreshMetadata, MessageFactory.RefreshMetadataEvent(library.Id, series.Id)); } _logger.LogInformation( "[MetadataService] Processed {SeriesStart} - {SeriesEnd} out of {TotalSeries} series in {ElapsedScanTime} milliseconds for {LibraryName}", chunk * chunkInfo.ChunkSize, (chunk * chunkInfo.ChunkSize) + nonLibrarySeries.Count, chunkInfo.TotalSize, stopwatch.ElapsedMilliseconds, library.Name); } await _messageHub.Clients.All.SendAsync(SignalREvents.RefreshMetadataProgress, MessageFactory.RefreshMetadataProgressEvent(library.Id, 1F)); // TODO: Remove any leftover People from DB await _unitOfWork.PersonRepository.RemoveAllPeopleNoLongerAssociated(); await _unitOfWork.GenreRepository.RemoveAllGenreNoLongerAssociated(); _logger.LogInformation("[MetadataService] Updated metadata for {SeriesNumber} series in library {LibraryName} in {ElapsedMilliseconds} milliseconds total", chunkInfo.TotalSize, library.Name, totalTime); } // TODO: I can probably refactor RefreshMetadata and RefreshMetadataForSeries to be the same by utilizing chunk size of 1, so most of the code can be the same. private async Task PerformScan(Library library, bool forceUpdate, Action action) { var chunkInfo = await _unitOfWork.SeriesRepository.GetChunkInfo(library.Id); var stopwatch = Stopwatch.StartNew(); var totalTime = 0L; _logger.LogInformation("[MetadataService] Refreshing Library {LibraryName}. Total Items: {TotalSize}. Total Chunks: {TotalChunks} with {ChunkSize} size", library.Name, chunkInfo.TotalSize, chunkInfo.TotalChunks, chunkInfo.ChunkSize); await _messageHub.Clients.All.SendAsync(SignalREvents.RefreshMetadataProgress, MessageFactory.RefreshMetadataProgressEvent(library.Id, 0F)); for (var chunk = 1; chunk <= chunkInfo.TotalChunks; chunk++) { if (chunkInfo.TotalChunks == 0) continue; totalTime += stopwatch.ElapsedMilliseconds; stopwatch.Restart(); action(chunk, chunkInfo); // _logger.LogInformation("[MetadataService] Processing chunk {ChunkNumber} / {TotalChunks} with size {ChunkSize}. Series ({SeriesStart} - {SeriesEnd}", // chunk, chunkInfo.TotalChunks, chunkInfo.ChunkSize, chunk * chunkInfo.ChunkSize, (chunk + 1) * chunkInfo.ChunkSize); // var nonLibrarySeries = await _unitOfWork.SeriesRepository.GetFullSeriesForLibraryIdAsync(library.Id, // new UserParams() // { // PageNumber = chunk, // PageSize = chunkInfo.ChunkSize // }); // _logger.LogDebug("[MetadataService] Fetched {SeriesCount} series for refresh", nonLibrarySeries.Count); // // var chapterIds = await _unitOfWork.SeriesRepository.GetChapterIdWithSeriesIdForSeriesAsync(nonLibrarySeries.Select(s => s.Id).ToArray()); // var allPeople = await _unitOfWork.PersonRepository.GetAllPeople(); // var allGenres = await _unitOfWork.GenreRepository.GetAllGenres(); // // // var seriesIndex = 0; // foreach (var series in nonLibrarySeries) // { // try // { // ProcessSeriesMetadataUpdate(series, chapterIds, allPeople, allGenres, forceUpdate); // } // catch (Exception ex) // { // _logger.LogError(ex, "[MetadataService] There was an exception during metadata refresh for {SeriesName}", series.Name); // } // var index = chunk * seriesIndex; // var progress = Math.Max(0F, Math.Min(1F, index * 1F / chunkInfo.TotalSize)); // // await _messageHub.Clients.All.SendAsync(SignalREvents.RefreshMetadataProgress, // MessageFactory.RefreshMetadataProgressEvent(library.Id, progress)); // seriesIndex++; // } await _unitOfWork.CommitAsync(); } } /// /// Refreshes Metadata for a Series. Will always force updates. /// /// /// public async Task RefreshMetadataForSeries(int libraryId, int seriesId, bool forceUpdate = true) { var sw = Stopwatch.StartNew(); var series = await _unitOfWork.SeriesRepository.GetFullSeriesForSeriesIdAsync(seriesId); if (series == null) { _logger.LogError("[MetadataService] Series {SeriesId} was not found on Library {LibraryId}", seriesId, libraryId); return; } await _messageHub.Clients.All.SendAsync(SignalREvents.RefreshMetadataProgress, MessageFactory.RefreshMetadataProgressEvent(libraryId, 0F)); var chapterIds = await _unitOfWork.SeriesRepository.GetChapterIdWithSeriesIdForSeriesAsync(new [] { seriesId }); var allPeople = await _unitOfWork.PersonRepository.GetAllPeople(); var allGenres = await _unitOfWork.GenreRepository.GetAllGenres(); ProcessSeriesMetadataUpdate(series, chapterIds, allPeople, allGenres, forceUpdate); await _messageHub.Clients.All.SendAsync(SignalREvents.RefreshMetadataProgress, MessageFactory.RefreshMetadataProgressEvent(libraryId, 1F)); if (_unitOfWork.HasChanges() && await _unitOfWork.CommitAsync()) { await _messageHub.Clients.All.SendAsync(SignalREvents.RefreshMetadata, MessageFactory.RefreshMetadataEvent(series.LibraryId, series.Id)); } _logger.LogInformation("[MetadataService] Updated metadata for {SeriesName} in {ElapsedMilliseconds} milliseconds", series.Name, sw.ElapsedMilliseconds); } }