using System; using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Threading.Tasks; using API.Comparators; using API.Data; using API.Data.Repositories; using API.Data.Scanner; using API.Entities; using API.Extensions; using API.Helpers; using API.SignalR; using Microsoft.AspNetCore.SignalR; using Microsoft.Extensions.Logging; namespace API.Services; public interface IMetadataService { /// /// Recalculates metadata for all entities in a library. /// /// /// Task RefreshMetadata(int libraryId, bool forceUpdate = false); /// /// Performs a forced refresh of metadata just for a series and it's nested entities /// /// /// Task RefreshMetadataForSeries(int libraryId, int seriesId, bool forceUpdate = false); } public class MetadataService : IMetadataService { private readonly IUnitOfWork _unitOfWork; private readonly ILogger _logger; private readonly IHubContext _messageHub; private readonly ICacheHelper _cacheHelper; private readonly IReadingItemService _readingItemService; private readonly IDirectoryService _directoryService; private readonly ChapterSortComparerZeroFirst _chapterSortComparerForInChapterSorting = new ChapterSortComparerZeroFirst(); public MetadataService(IUnitOfWork unitOfWork, ILogger logger, IHubContext messageHub, ICacheHelper cacheHelper, IReadingItemService readingItemService, IDirectoryService directoryService) { _unitOfWork = unitOfWork; _logger = logger; _messageHub = messageHub; _cacheHelper = cacheHelper; _readingItemService = readingItemService; _directoryService = directoryService; } /// /// Updates the metadata for a Chapter /// /// /// Force updating cover image even if underlying file has not been modified or chapter already has a cover image private bool UpdateChapterCoverImage(Chapter chapter, bool forceUpdate) { var firstFile = chapter.Files.OrderBy(x => x.Chapter).FirstOrDefault(); if (!_cacheHelper.ShouldUpdateCoverImage(_directoryService.FileSystem.Path.Join(_directoryService.CoverImageDirectory, chapter.CoverImage), firstFile, chapter.Created, forceUpdate, chapter.CoverImageLocked)) return false; if (firstFile == null) return false; _logger.LogDebug("[MetadataService] Generating cover image for {File}", firstFile.FilePath); chapter.CoverImage = _readingItemService.GetCoverImage(firstFile.FilePath, ImageService.GetChapterFormat(chapter.Id, chapter.VolumeId), firstFile.Format); return true; } private void UpdateChapterLastModified(Chapter chapter, bool forceUpdate) { var firstFile = chapter.Files.OrderBy(x => x.Chapter).FirstOrDefault(); if (firstFile == null || _cacheHelper.HasFileNotChangedSinceCreationOrLastScan(chapter, forceUpdate, firstFile)) return; firstFile.UpdateLastModified(); } /// /// Updates the cover image for a Volume /// /// /// Force updating cover image even if underlying file has not been modified or chapter already has a cover image private bool UpdateVolumeCoverImage(Volume volume, bool forceUpdate) { // We need to check if Volume coverImage matches first chapters if forceUpdate is false if (volume == null || !_cacheHelper.ShouldUpdateCoverImage( _directoryService.FileSystem.Path.Join(_directoryService.CoverImageDirectory, volume.CoverImage), null, volume.Created, forceUpdate)) return false; volume.Chapters ??= new List(); var firstChapter = volume.Chapters.OrderBy(x => double.Parse(x.Number), _chapterSortComparerForInChapterSorting).FirstOrDefault(); if (firstChapter == null) return false; volume.CoverImage = firstChapter.CoverImage; return true; } /// /// Updates cover image for Series /// /// /// Force updating cover image even if underlying file has not been modified or chapter already has a cover image private void UpdateSeriesCoverImage(Series series, bool forceUpdate) { if (series == null) return; if (!_cacheHelper.ShouldUpdateCoverImage(_directoryService.FileSystem.Path.Join(_directoryService.CoverImageDirectory, series.CoverImage), null, series.Created, forceUpdate, series.CoverImageLocked)) return; series.Volumes ??= new List(); var firstCover = series.Volumes.GetCoverImage(series.Format); string coverImage = null; if (firstCover == null && series.Volumes.Any()) { // If firstCover is null and one volume, the whole series is Chapters under Vol 0. if (series.Volumes.Count == 1) { coverImage = series.Volumes[0].Chapters.OrderBy(c => double.Parse(c.Number), _chapterSortComparerForInChapterSorting) .FirstOrDefault(c => !c.IsSpecial)?.CoverImage; } if (!_cacheHelper.CoverImageExists(coverImage)) { coverImage = series.Volumes[0].Chapters.OrderBy(c => double.Parse(c.Number), _chapterSortComparerForInChapterSorting) .FirstOrDefault()?.CoverImage; } } series.CoverImage = firstCover?.CoverImage ?? coverImage; } /// /// /// /// /// private void ProcessSeriesMetadataUpdate(Series series, bool forceUpdate) { _logger.LogDebug("[MetadataService] Processing series {SeriesName}", series.OriginalName); try { var volumeIndex = 0; var firstVolumeUpdated = false; foreach (var volume in series.Volumes) { var firstChapterUpdated = false; // This only needs to be FirstChapter updated var index = 0; foreach (var chapter in volume.Chapters) { var chapterUpdated = UpdateChapterCoverImage(chapter, forceUpdate); // If cover was update, either the file has changed or first scan and we should force a metadata update UpdateChapterLastModified(chapter, forceUpdate || chapterUpdated); if (index == 0 && chapterUpdated) { firstChapterUpdated = true; } index++; } var volumeUpdated = UpdateVolumeCoverImage(volume, firstChapterUpdated || forceUpdate); if (volumeIndex == 0 && volumeUpdated) { firstVolumeUpdated = true; } volumeIndex++; } UpdateSeriesCoverImage(series, firstVolumeUpdated || forceUpdate); } catch (Exception ex) { _logger.LogError(ex, "[MetadataService] There was an exception during updating metadata for {SeriesName} ", series.Name); } } /// /// Refreshes Metadata for a whole library /// /// This can be heavy on memory first run /// /// Force updating cover image even if underlying file has not been modified or chapter already has a cover image public async Task RefreshMetadata(int libraryId, bool forceUpdate = false) { var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(libraryId, LibraryIncludes.None); _logger.LogInformation("[MetadataService] Beginning metadata refresh of {LibraryName}", library.Name); var chunkInfo = await _unitOfWork.SeriesRepository.GetChunkInfo(library.Id); var stopwatch = Stopwatch.StartNew(); var totalTime = 0L; _logger.LogInformation("[MetadataService] Refreshing Library {LibraryName}. Total Items: {TotalSize}. Total Chunks: {TotalChunks} with {ChunkSize} size", library.Name, chunkInfo.TotalSize, chunkInfo.TotalChunks, chunkInfo.ChunkSize); await _messageHub.Clients.All.SendAsync(SignalREvents.RefreshMetadataProgress, MessageFactory.RefreshMetadataProgressEvent(library.Id, 0F)); for (var chunk = 1; chunk <= chunkInfo.TotalChunks; chunk++) { if (chunkInfo.TotalChunks == 0) continue; totalTime += stopwatch.ElapsedMilliseconds; stopwatch.Restart(); _logger.LogInformation("[MetadataService] Processing chunk {ChunkNumber} / {TotalChunks} with size {ChunkSize}. Series ({SeriesStart} - {SeriesEnd}", chunk, chunkInfo.TotalChunks, chunkInfo.ChunkSize, chunk * chunkInfo.ChunkSize, (chunk + 1) * chunkInfo.ChunkSize); var nonLibrarySeries = await _unitOfWork.SeriesRepository.GetFullSeriesForLibraryIdAsync(library.Id, new UserParams() { PageNumber = chunk, PageSize = chunkInfo.ChunkSize }); _logger.LogDebug("[MetadataService] Fetched {SeriesCount} series for refresh", nonLibrarySeries.Count); var seriesIndex = 0; foreach (var series in nonLibrarySeries) { try { ProcessSeriesMetadataUpdate(series, forceUpdate); } catch (Exception ex) { _logger.LogError(ex, "[MetadataService] There was an exception during metadata refresh for {SeriesName}", series.Name); } var index = chunk * seriesIndex; var progress = Math.Max(0F, Math.Min(1F, index * 1F / chunkInfo.TotalSize)); await _messageHub.Clients.All.SendAsync(SignalREvents.RefreshMetadataProgress, MessageFactory.RefreshMetadataProgressEvent(library.Id, progress)); seriesIndex++; } await _unitOfWork.CommitAsync(); foreach (var series in nonLibrarySeries) { await _messageHub.Clients.All.SendAsync(SignalREvents.RefreshMetadata, MessageFactory.RefreshMetadataEvent(library.Id, series.Id)); } _logger.LogInformation( "[MetadataService] Processed {SeriesStart} - {SeriesEnd} out of {TotalSeries} series in {ElapsedScanTime} milliseconds for {LibraryName}", chunk * chunkInfo.ChunkSize, (chunk * chunkInfo.ChunkSize) + nonLibrarySeries.Count, chunkInfo.TotalSize, stopwatch.ElapsedMilliseconds, library.Name); } await _messageHub.Clients.All.SendAsync(SignalREvents.RefreshMetadataProgress, MessageFactory.RefreshMetadataProgressEvent(library.Id, 1F)); await RemoveAbandonedMetadataKeys(); _logger.LogInformation("[MetadataService] Updated metadata for {SeriesNumber} series in library {LibraryName} in {ElapsedMilliseconds} milliseconds total", chunkInfo.TotalSize, library.Name, totalTime); } private async Task RemoveAbandonedMetadataKeys() { await _unitOfWork.TagRepository.RemoveAllTagNoLongerAssociated(); await _unitOfWork.PersonRepository.RemoveAllPeopleNoLongerAssociated(); await _unitOfWork.GenreRepository.RemoveAllGenreNoLongerAssociated(); } // TODO: Write out a single piece of code that can iterate over a collection/chunk and perform custom actions private async Task PerformScan(Library library, bool forceUpdate, Action action) { var chunkInfo = await _unitOfWork.SeriesRepository.GetChunkInfo(library.Id); var stopwatch = Stopwatch.StartNew(); _logger.LogInformation("[MetadataService] Refreshing Library {LibraryName}. Total Items: {TotalSize}. Total Chunks: {TotalChunks} with {ChunkSize} size", library.Name, chunkInfo.TotalSize, chunkInfo.TotalChunks, chunkInfo.ChunkSize); await _messageHub.Clients.All.SendAsync(SignalREvents.RefreshMetadataProgress, MessageFactory.RefreshMetadataProgressEvent(library.Id, 0F)); for (var chunk = 1; chunk <= chunkInfo.TotalChunks; chunk++) { if (chunkInfo.TotalChunks == 0) continue; stopwatch.Restart(); action(chunk, chunkInfo); // _logger.LogInformation("[MetadataService] Processing chunk {ChunkNumber} / {TotalChunks} with size {ChunkSize}. Series ({SeriesStart} - {SeriesEnd}", // chunk, chunkInfo.TotalChunks, chunkInfo.ChunkSize, chunk * chunkInfo.ChunkSize, (chunk + 1) * chunkInfo.ChunkSize); // var nonLibrarySeries = await _unitOfWork.SeriesRepository.GetFullSeriesForLibraryIdAsync(library.Id, // new UserParams() // { // PageNumber = chunk, // PageSize = chunkInfo.ChunkSize // }); // _logger.LogDebug("[MetadataService] Fetched {SeriesCount} series for refresh", nonLibrarySeries.Count); // // var chapterIds = await _unitOfWork.SeriesRepository.GetChapterIdWithSeriesIdForSeriesAsync(nonLibrarySeries.Select(s => s.Id).ToArray()); // var allPeople = await _unitOfWork.PersonRepository.GetAllPeople(); // var allGenres = await _unitOfWork.GenreRepository.GetAllGenres(); // // // var seriesIndex = 0; // foreach (var series in nonLibrarySeries) // { // try // { // ProcessSeriesMetadataUpdate(series, chapterIds, allPeople, allGenres, forceUpdate); // } // catch (Exception ex) // { // _logger.LogError(ex, "[MetadataService] There was an exception during metadata refresh for {SeriesName}", series.Name); // } // var index = chunk * seriesIndex; // var progress = Math.Max(0F, Math.Min(1F, index * 1F / chunkInfo.TotalSize)); // // await _messageHub.Clients.All.SendAsync(SignalREvents.RefreshMetadataProgress, // MessageFactory.RefreshMetadataProgressEvent(library.Id, progress)); // seriesIndex++; // } await _unitOfWork.CommitAsync(); } } /// /// Refreshes Metadata for a Series. Will always force updates. /// /// /// public async Task RefreshMetadataForSeries(int libraryId, int seriesId, bool forceUpdate = true) { var sw = Stopwatch.StartNew(); var series = await _unitOfWork.SeriesRepository.GetFullSeriesForSeriesIdAsync(seriesId); if (series == null) { _logger.LogError("[MetadataService] Series {SeriesId} was not found on Library {LibraryId}", seriesId, libraryId); return; } await _messageHub.Clients.All.SendAsync(SignalREvents.RefreshMetadataProgress, MessageFactory.RefreshMetadataProgressEvent(libraryId, 0F)); ProcessSeriesMetadataUpdate(series, forceUpdate); await _messageHub.Clients.All.SendAsync(SignalREvents.RefreshMetadataProgress, MessageFactory.RefreshMetadataProgressEvent(libraryId, 1F)); if (_unitOfWork.HasChanges() && await _unitOfWork.CommitAsync()) { await _messageHub.Clients.All.SendAsync(SignalREvents.RefreshMetadata, MessageFactory.RefreshMetadataEvent(series.LibraryId, series.Id)); } await RemoveAbandonedMetadataKeys(); _logger.LogInformation("[MetadataService] Updated metadata for {SeriesName} in {ElapsedMilliseconds} milliseconds", series.Name, sw.ElapsedMilliseconds); } }