From 6d742152621231d499d6e2240e6c1d4589b2f8b4 Mon Sep 17 00:00:00 2001 From: Joseph Milazzo Date: Fri, 30 Apr 2021 17:23:31 -0500 Subject: [PATCH] Feature/feedback (#185) * Remove automatic retry for scanLibraries as if something fails, it wont pass magically. Catch exceptions when opening books for parsing and swallow to ignore the file. * Delete extra attempts * Switched to using FirstOrDefault for finding existing series. This will help avoid pointless crashes. * Updated message when duplicate series are found (not sure how this happens) * Fixed a negation for deleting volumes where files still exist. * Implemented the ability to automatically scale the manga reader based on screen size. * Default to automatic scaling * Fix an issue where malformed epubs wouldn't be readable due to incorrect keys in the OPF. We now check if key is valid and if not, try to correct it. This makes a page load about a second on malformed books. * Fixed #176. Refactored the recently added query to be restricted to user's access to libraries. * Fixed a one off bug with In Progress series * Implemented the ability to refresh metadata of just a single series directly --- API/Controllers/BookController.cs | 29 +++++++++++++++++ API/Controllers/SeriesController.cs | 11 ++++++- API/DTOs/RefreshSeriesDto.cs | 8 +++++ API/DTOs/SeriesDto.cs | 6 +++- API/Data/SeriesRepository.cs | 33 ++++++++++++++----- API/Entities/AppUserPreferences.cs | 2 +- API/Helpers/PagedList.cs | 1 + API/Interfaces/ISeriesRepository.cs | 2 +- API/Interfaces/ITaskScheduler.cs | 1 + API/Interfaces/Services/IMetadataService.cs | 6 ++++ API/Services/BookService.cs | 2 -- API/Services/MetadataService.cs | 35 ++++++++++++++++++++- API/Services/TaskScheduler.cs | 6 ++++ API/Services/Tasks/ScannerService.cs | 1 - 14 files changed, 127 insertions(+), 16 deletions(-) create mode 100644 API/DTOs/RefreshSeriesDto.cs diff --git a/API/Controllers/BookController.cs b/API/Controllers/BookController.cs index b7589e6dd..9cc7bd65e 100644 --- a/API/Controllers/BookController.cs +++ b/API/Controllers/BookController.cs @@ -158,6 +158,19 @@ namespace API.Controllers foreach (var styleLinks in styleNodes) { var key = BookService.CleanContentKeys(styleLinks.Attributes["href"].Value); + // Some epubs are malformed the key in content.opf might be: content/resources/filelist_0_0.xml but the actual html links to resources/filelist_0_0.xml + // In this case, we will do a search for the key that ends with + if (!book.Content.Css.ContainsKey(key)) + { + var correctedKey = book.Content.Css.Keys.SingleOrDefault(s => s.EndsWith(key)); + if (correctedKey == null) + { + _logger.LogError("Epub is Malformed, key: {Key} is not matching OPF file", key); + continue; + } + + key = correctedKey; + } var styleContent = await _bookService.ScopeStyles(await book.Content.Css[key].ReadContentAsync(), apiBase); body.PrependChild(HtmlNode.CreateNode($"")); } @@ -183,6 +196,14 @@ namespace API.Controllers if (image.Attributes["src"] != null) { var imageFile = image.Attributes["src"].Value; + if (!book.Content.Images.ContainsKey(imageFile)) + { + var correctedKey = book.Content.Images.Keys.SingleOrDefault(s => s.EndsWith(imageFile)); + if (correctedKey != null) + { + imageFile = correctedKey; + } + } image.Attributes.Remove("src"); image.Attributes.Add("src", $"{apiBase}" + imageFile); } @@ -199,6 +220,14 @@ namespace API.Controllers if (image.Attributes["xlink:href"] != null) { var imageFile = image.Attributes["xlink:href"].Value; + if (!book.Content.Images.ContainsKey(imageFile)) + { + var correctedKey = book.Content.Images.Keys.SingleOrDefault(s => s.EndsWith(imageFile)); + if (correctedKey != null) + { + imageFile = correctedKey; + } + } image.Attributes.Remove("xlink:href"); image.Attributes.Add("xlink:href", $"{apiBase}" + imageFile); } diff --git a/API/Controllers/SeriesController.cs b/API/Controllers/SeriesController.cs index 94a5b024a..d0d47cb5d 100644 --- a/API/Controllers/SeriesController.cs +++ b/API/Controllers/SeriesController.cs @@ -145,7 +145,8 @@ namespace API.Controllers [HttpGet("recently-added")] public async Task>> GetRecentlyAdded(int libraryId = 0, int limit = 20) { - return Ok(await _unitOfWork.SeriesRepository.GetRecentlyAdded(libraryId, limit)); + var user = await _unitOfWork.UserRepository.GetUserByUsernameAsync(User.GetUsername()); + return Ok(await _unitOfWork.SeriesRepository.GetRecentlyAdded(user.Id, libraryId, limit)); } [HttpGet("in-progress")] @@ -154,5 +155,13 @@ namespace API.Controllers var user = await _unitOfWork.UserRepository.GetUserByUsernameAsync(User.GetUsername()); return Ok(await _unitOfWork.SeriesRepository.GetInProgress(user.Id, libraryId, limit)); } + + [Authorize(Policy = "RequireAdminRole")] + [HttpPost("refresh-metadata")] + public ActionResult RefreshSeriesMetadata(RefreshSeriesDto refreshSeriesDto) + { + _taskScheduler.RefreshSeriesMetadata(refreshSeriesDto.LibraryId, refreshSeriesDto.SeriesId); + return Ok(); + } } } \ No newline at end of file diff --git a/API/DTOs/RefreshSeriesDto.cs b/API/DTOs/RefreshSeriesDto.cs new file mode 100644 index 000000000..bc6344ea2 --- /dev/null +++ b/API/DTOs/RefreshSeriesDto.cs @@ -0,0 +1,8 @@ +namespace API.DTOs +{ + public class RefreshSeriesDto + { + public int LibraryId { get; set; } + public int SeriesId { get; set; } + } +} \ No newline at end of file diff --git a/API/DTOs/SeriesDto.cs b/API/DTOs/SeriesDto.cs index 3cb7f120b..0f8f4263c 100644 --- a/API/DTOs/SeriesDto.cs +++ b/API/DTOs/SeriesDto.cs @@ -1,4 +1,6 @@ -namespace API.DTOs +using System; + +namespace API.DTOs { public class SeriesDto { @@ -21,6 +23,8 @@ /// Review from logged in user. Calculated at API-time. /// public string UserReview { get; set; } + + public DateTime Created { get; set; } public int LibraryId { get; set; } public string LibraryName { get; set; } diff --git a/API/Data/SeriesRepository.cs b/API/Data/SeriesRepository.cs index de35a8321..9250d592a 100644 --- a/API/Data/SeriesRepository.cs +++ b/API/Data/SeriesRepository.cs @@ -80,15 +80,12 @@ namespace API.Data public async Task> GetSeriesDtoForLibraryIdAsync(int libraryId, int userId, UserParams userParams) { - var sw = Stopwatch.StartNew(); var query = _context.Series .Where(s => s.LibraryId == libraryId) .OrderBy(s => s.SortName) .ProjectTo(_mapper.ConfigurationProvider) .AsNoTracking(); - - _logger.LogDebug("Processed GetSeriesDtoForLibraryIdAsync in {ElapsedMilliseconds} milliseconds", sw.ElapsedMilliseconds); return await PagedList.CreateAsync(query, userParams.PageNumber, userParams.PageSize); } @@ -295,15 +292,35 @@ namespace API.Data /// Library to restrict to, if 0, will apply to all libraries /// How many series to pick. /// - public async Task> GetRecentlyAdded(int libraryId, int limit) + public async Task> GetRecentlyAdded(int userId, int libraryId, int limit) { + if (libraryId == 0) + { + var userLibraries = _context.Library + .Include(l => l.AppUsers) + .Where(library => library.AppUsers.Any(user => user.Id == userId)) + .AsNoTracking() + .Select(library => library.Id) + .ToList(); + + return await _context.Series + .Where(s => userLibraries.Contains(s.LibraryId)) + .AsNoTracking() + .OrderByDescending(s => s.Created) + .Take(limit) + .ProjectTo(_mapper.ConfigurationProvider) + .ToListAsync(); + } + return await _context.Series - .Where(s => (libraryId <= 0 || s.LibraryId == libraryId)) - .Take(limit) - .OrderByDescending(s => s.Created) + .Where(s => s.LibraryId == libraryId) .AsNoTracking() + .OrderByDescending(s => s.Created) + .Take(limit) .ProjectTo(_mapper.ConfigurationProvider) .ToListAsync(); + + } /// @@ -325,7 +342,7 @@ namespace API.Data }) .Where(s => s.AppUserId == userId && s.PagesRead > 0 - && s.PagesRead < s.Series.Pages + && s.PagesRead < (s.Series.Pages - 1) // - 1 because when reading, we start at 0 then go to pages - 1. But when summing, pages assumes starting at 1 && (libraryId <= 0 || s.Series.LibraryId == libraryId)) .Take(limit) .OrderByDescending(s => s.LastModified) diff --git a/API/Entities/AppUserPreferences.cs b/API/Entities/AppUserPreferences.cs index a4a773a38..4671c2692 100644 --- a/API/Entities/AppUserPreferences.cs +++ b/API/Entities/AppUserPreferences.cs @@ -12,7 +12,7 @@ namespace API.Entities /// /// Manga Reader Option: How should the image be scaled to screen /// - public ScalingOption ScalingOption { get; set; } = ScalingOption.FitToHeight; + public ScalingOption ScalingOption { get; set; } = ScalingOption.Automatic; /// /// Manga Reader Option: Which side of a split image should we show first /// diff --git a/API/Helpers/PagedList.cs b/API/Helpers/PagedList.cs index 0900f02a5..b87687a6e 100644 --- a/API/Helpers/PagedList.cs +++ b/API/Helpers/PagedList.cs @@ -24,6 +24,7 @@ namespace API.Helpers public static async Task> CreateAsync(IQueryable source, int pageNumber, int pageSize) { + // NOTE: OrderBy warning being thrown here even if query has the orderby statement var count = await source.CountAsync(); var items = await source.Skip((pageNumber - 1) * pageSize).Take(pageSize).ToListAsync(); return new PagedList(items, count, pageNumber, pageSize); diff --git a/API/Interfaces/ISeriesRepository.cs b/API/Interfaces/ISeriesRepository.cs index be39e10d4..eff8e7c08 100644 --- a/API/Interfaces/ISeriesRepository.cs +++ b/API/Interfaces/ISeriesRepository.cs @@ -58,6 +58,6 @@ namespace API.Interfaces Task GetVolumeCoverImageAsync(int volumeId); Task GetSeriesCoverImageAsync(int seriesId); Task> GetInProgress(int userId, int libraryId, int limit); - Task> GetRecentlyAdded(int libraryId, int limit); + Task> GetRecentlyAdded(int userId, int libraryId, int limit); } } \ No newline at end of file diff --git a/API/Interfaces/ITaskScheduler.cs b/API/Interfaces/ITaskScheduler.cs index 5de2f6941..75f70c1fa 100644 --- a/API/Interfaces/ITaskScheduler.cs +++ b/API/Interfaces/ITaskScheduler.cs @@ -10,5 +10,6 @@ void CleanupChapters(int[] chapterIds); void RefreshMetadata(int libraryId, bool forceUpdate = true); void CleanupTemp(); + void RefreshSeriesMetadata(int libraryId, int seriesId); } } \ No newline at end of file diff --git a/API/Interfaces/Services/IMetadataService.cs b/API/Interfaces/Services/IMetadataService.cs index 830cab1eb..70b10b861 100644 --- a/API/Interfaces/Services/IMetadataService.cs +++ b/API/Interfaces/Services/IMetadataService.cs @@ -14,5 +14,11 @@ namespace API.Interfaces.Services public void UpdateMetadata(Chapter chapter, bool forceUpdate); public void UpdateMetadata(Volume volume, bool forceUpdate); public void UpdateMetadata(Series series, bool forceUpdate); + /// + /// Performs a forced refresh of metatdata just for a series and it's nested entities + /// + /// + /// + void RefreshMetadataForSeries(int libraryId, int seriesId); } } \ No newline at end of file diff --git a/API/Services/BookService.cs b/API/Services/BookService.cs index e09485645..2dfbd4798 100644 --- a/API/Services/BookService.cs +++ b/API/Services/BookService.cs @@ -5,7 +5,6 @@ using System.Linq; using System.Text.RegularExpressions; using System.Threading.Tasks; using API.Entities.Enums; -using API.Entities.Interfaces; using API.Interfaces; using API.Parser; using ExCSS; @@ -13,7 +12,6 @@ using HtmlAgilityPack; using Microsoft.Extensions.Logging; using NetVips; using VersOne.Epub; -using VersOne.Epub.Schema; namespace API.Services { diff --git a/API/Services/MetadataService.cs b/API/Services/MetadataService.cs index 5e1f125bb..8d479ed90 100644 --- a/API/Services/MetadataService.cs +++ b/API/Services/MetadataService.cs @@ -122,7 +122,7 @@ namespace API.Services // NOTE: This suffers from code changes not taking effect due to stale data var firstFile = firstChapter?.Files.FirstOrDefault(); if (firstFile != null && - (forceUpdate || !firstFile.HasFileBeenModified())) + (forceUpdate || firstFile.HasFileBeenModified())) // !new FileInfo(firstFile.FilePath).IsLastWriteLessThan(firstFile.LastModified) { series.Summary = isBook ? _bookService.GetSummaryInfo(firstFile.FilePath) : _archiveService.GetSummaryInfo(firstFile.FilePath); @@ -160,5 +160,38 @@ namespace API.Services _logger.LogInformation("Updated metadata for {LibraryName} in {ElapsedMilliseconds} milliseconds", library.Name, sw.ElapsedMilliseconds); } } + + + public void RefreshMetadataForSeries(int libraryId, int seriesId) + { + var sw = Stopwatch.StartNew(); + var library = Task.Run(() => _unitOfWork.LibraryRepository.GetFullLibraryForIdAsync(libraryId)).GetAwaiter().GetResult(); + + var series = library.Series.SingleOrDefault(s => s.Id == seriesId); + if (series == null) + { + _logger.LogError("Series {SeriesId} was not found on Library {LibraryName}", seriesId, libraryId); + return; + } + _logger.LogInformation("Beginning metadata refresh of {SeriesName}", series.Name); + foreach (var volume in series.Volumes) + { + foreach (var chapter in volume.Chapters) + { + UpdateMetadata(chapter, true); + } + + UpdateMetadata(volume, true); + } + + UpdateMetadata(series, true); + _unitOfWork.SeriesRepository.Update(series); + + + if (_unitOfWork.HasChanges() && Task.Run(() => _unitOfWork.Complete()).Result) + { + _logger.LogInformation("Updated metadata for {SeriesName} in {ElapsedMilliseconds} milliseconds", series.Name, sw.ElapsedMilliseconds); + } + } } } \ No newline at end of file diff --git a/API/Services/TaskScheduler.cs b/API/Services/TaskScheduler.cs index 8857865c0..b284fd9f7 100644 --- a/API/Services/TaskScheduler.cs +++ b/API/Services/TaskScheduler.cs @@ -90,6 +90,12 @@ namespace API.Services BackgroundJob.Enqueue((() => DirectoryService.ClearDirectory(tempDirectory))); } + public void RefreshSeriesMetadata(int libraryId, int seriesId) + { + _logger.LogInformation("Enqueuing series metadata refresh for: {SeriesId}", seriesId); + BackgroundJob.Enqueue((() => _metadataService.RefreshMetadataForSeries(libraryId, seriesId))); + } + public void BackupDatabase() { BackgroundJob.Enqueue(() => _backupService.BackupDatabase()); diff --git a/API/Services/Tasks/ScannerService.cs b/API/Services/Tasks/ScannerService.cs index f558c3a3a..726e31dc3 100644 --- a/API/Services/Tasks/ScannerService.cs +++ b/API/Services/Tasks/ScannerService.cs @@ -9,7 +9,6 @@ using API.Comparators; using API.Data; using API.Entities; using API.Entities.Enums; -using API.Entities.Interfaces; using API.Extensions; using API.Interfaces; using API.Interfaces.Services;