From 85f3b620afa4d3586d8f61b6dfc38df8b1faaca2 Mon Sep 17 00:00:00 2001 From: Joseph Milazzo Date: Sun, 8 May 2022 11:15:43 -0500 Subject: [PATCH] Bugfix polishing (#1245) * Fixed a bug where volumes that are a range fail to generate series detail * Moved tags closer to genre instead of between different people * Optimized the query for On Deck * Adjusted mime types to map to cbX types instead of their generic compression methods. * Added wiki documentation into invite user flow and register admin user to help users understand email isn't required and they can host their own service. * Refactored the document height to be set and removed on nav service, so the book reader and manga reader aren't broken. * Refactored On Deck to first be completely streamed to UI, without having to do any processing in memory. Rewrote the query so that we sort by progress then chapter added. Progress is 30 days inclusive, chapter added is 7 days. * Fixed an issue where epub date parsing would sometimes fail when it's only a year or not a year at all * Fixed a bug where incognito mode would report progress * Fixed a bug where bulk selection in storyline tab wouldn't properly run the action on the correct chapters (if selecting from volume -> chapter). * Removed a - 1 from total page from card progress bar as the original bug was fixed some time ago * Fixed a bug where the logic for filtering out a progress event for current logged in user didn't check properly when user is logged out. * When a file doesn't exist and we are trying to read, throw a kavita exception to the UI layer and log. * Removed unneeded variable and added some jsdoc --- API/Controllers/BookController.cs | 1 - API/Controllers/OPDSController.cs | 6 +- API/Controllers/SeriesController.cs | 6 +- API/Data/Repositories/SeriesRepository.cs | 88 ++++++++++--------- API/Services/ArchiveService.cs | 6 ++ API/Services/BookService.cs | 25 +++++- API/Services/CacheService.cs | 7 ++ API/Services/DownloadService.cs | 10 ++- API/Services/SeriesService.cs | 2 +- UI/Web/src/app/_services/nav.service.ts | 4 + .../invite-user/invite-user.component.html | 2 +- UI/Web/src/app/app.component.html | 4 +- .../cards/card-item/card-item.component.html | 2 +- .../cards/card-item/card-item.component.scss | 6 +- .../cards/card-item/card-item.component.ts | 17 ++-- .../manga-reader/manga-reader.component.ts | 2 +- .../register/register.component.html | 8 +- .../series-detail/series-detail.component.ts | 6 +- .../series-metadata-detail.component.html | 24 ++--- UI/Web/src/styles.scss | 4 - 20 files changed, 135 insertions(+), 95 deletions(-) diff --git a/API/Controllers/BookController.cs b/API/Controllers/BookController.cs index e1207f919..7b4b49a9f 100644 --- a/API/Controllers/BookController.cs +++ b/API/Controllers/BookController.cs @@ -211,7 +211,6 @@ namespace API.Controllers var chapter = await _cacheService.Ensure(chapterId); var path = _cacheService.GetCachedEpubFile(chapter.Id, chapter); - using var book = await EpubReader.OpenBookAsync(path, BookService.BookReaderOptions); var mappings = await _bookService.CreateKeyToPageMappingAsync(book); diff --git a/API/Controllers/OPDSController.cs b/API/Controllers/OPDSController.cs index eaa778121..a221f06c1 100644 --- a/API/Controllers/OPDSController.cs +++ b/API/Controllers/OPDSController.cs @@ -389,12 +389,8 @@ public class OpdsController : BaseApiController var userParams = new UserParams() { PageNumber = pageNumber, - PageSize = 20 }; - var results = await _unitOfWork.SeriesRepository.GetOnDeck(userId, 0, userParams, _filterDto); - var listResults = results.DistinctBy(s => s.Name).Skip((userParams.PageNumber - 1) * userParams.PageSize) - .Take(userParams.PageSize).ToList(); - var pagedList = new PagedList(listResults, listResults.Count, userParams.PageNumber, userParams.PageSize); + var pagedList = await _unitOfWork.SeriesRepository.GetOnDeck(userId, 0, userParams, _filterDto); Response.AddPaginationHeader(pagedList.CurrentPage, pagedList.PageSize, pagedList.TotalCount, pagedList.TotalPages); diff --git a/API/Controllers/SeriesController.cs b/API/Controllers/SeriesController.cs index 39fd10938..34e90d818 100644 --- a/API/Controllers/SeriesController.cs +++ b/API/Controllers/SeriesController.cs @@ -243,12 +243,8 @@ namespace API.Controllers [HttpPost("on-deck")] public async Task>> GetOnDeck(FilterDto filterDto, [FromQuery] UserParams userParams, [FromQuery] int libraryId = 0) { - // NOTE: This has to be done manually like this due to the DistinctBy requirement var userId = await _unitOfWork.UserRepository.GetUserIdByUsernameAsync(User.GetUsername()); - var results = await _unitOfWork.SeriesRepository.GetOnDeck(userId, libraryId, userParams, filterDto); - - var listResults = results.DistinctBy(s => s.Name).Skip((userParams.PageNumber - 1) * userParams.PageSize).Take(userParams.PageSize).ToList(); - var pagedList = new PagedList(listResults, listResults.Count, userParams.PageNumber, userParams.PageSize); + var pagedList = await _unitOfWork.SeriesRepository.GetOnDeck(userId, libraryId, userParams, filterDto); await _unitOfWork.SeriesRepository.AddSeriesModifiers(userId, pagedList); diff --git a/API/Data/Repositories/SeriesRepository.cs b/API/Data/Repositories/SeriesRepository.cs index 15384b8ed..c36bcc4cb 100644 --- a/API/Data/Repositories/SeriesRepository.cs +++ b/API/Data/Repositories/SeriesRepository.cs @@ -94,7 +94,7 @@ public interface ISeriesRepository /// Task AddSeriesModifiers(int userId, List series); Task GetSeriesCoverImageAsync(int seriesId); - Task> GetOnDeck(int userId, int libraryId, UserParams userParams, FilterDto filter, bool cutoffOnDate = true); + Task> GetOnDeck(int userId, int libraryId, UserParams userParams, FilterDto filter); Task> GetRecentlyAdded(int libraryId, int userId, UserParams userParams, FilterDto filter); Task GetSeriesMetadata(int seriesId); Task> GetSeriesDtoForCollectionAsync(int collectionId, int userId, UserParams userParams); @@ -111,7 +111,6 @@ public interface ISeriesRepository IEnumerable GetAllPublicationStatusesDtosForLibrariesAsync(List libraryIds); // TODO: Move to LibraryRepository Task> GetRecentlyUpdatedSeries(int userId, int pageSize = 30); Task GetRelatedSeries(int userId, int seriesId); - Task> GetSeriesForRelationKind(int userId, int seriesId, RelationKind kind); Task> GetQuickReads(int userId, int libraryId, UserParams userParams); Task> GetHighlyRated(int userId, int libraryId, UserParams userParams); @@ -669,50 +668,48 @@ public class SeriesRepository : ISeriesRepository } /// - /// Returns Series that the user has some partial progress on. Sorts based on activity. Sort first by User progress, but if a series - /// has been updated recently, bump it to the front. + /// Returns Series that the user has some partial progress on. Sorts based on activity. Sort first by User progress, then + /// by when chapters have been added to series. Restricts progress in the past 30 days and chapters being added to last 7. /// /// /// Library to restrict to, if 0, will apply to all libraries /// Pagination information /// Optional (default null) filter on query /// - public async Task> GetOnDeck(int userId, int libraryId, UserParams userParams, FilterDto filter, bool cutoffOnDate = true) + public async Task> GetOnDeck(int userId, int libraryId, UserParams userParams, FilterDto filter) { - var query = (await CreateFilteredSearchQueryable(userId, libraryId, filter)) - .Join(_context.AppUserProgresses, s => s.Id, progress => progress.SeriesId, (s, progress) => - new - { - Series = s, - PagesRead = _context.AppUserProgresses.Where(s1 => s1.SeriesId == s.Id && s1.AppUserId == userId) - .Sum(s1 => s1.PagesRead), - progress.AppUserId, - LastReadingProgress = _context.AppUserProgresses - .Where(p => p.Id == progress.Id && p.AppUserId == userId) - .Max(p => p.LastModified), - LastModified = _context.AppUserProgresses.Where(p => p.Id == progress.Id && p.AppUserId == userId).Max(p => p.LastModified), - s.LastChapterAdded - }); - if (cutoffOnDate) - { - var cutoffProgressPoint = DateTime.Now - TimeSpan.FromDays(30); - query = query.Where(d => d.LastReadingProgress >= cutoffProgressPoint || d.LastChapterAdded >= cutoffProgressPoint); - } + var cutoffProgressPoint = DateTime.Now - TimeSpan.FromDays(30); + var cutoffLastAddedPoint = DateTime.Now - TimeSpan.FromDays(7); - var retSeries = query.Where(s => s.AppUserId == userId - && s.PagesRead > 0 - && s.PagesRead < s.Series.Pages) - .OrderByDescending(s => s.LastChapterAdded) - .ThenByDescending(s => s.LastReadingProgress) + var libraryIds = GetLibraryIdsForUser(userId, libraryId); + var usersSeriesIds = GetSeriesIdsForLibraryIds(libraryIds); + + + var query = _context.Series + .Where(s => usersSeriesIds.Contains(s.Id)) + .Select(s => new + { + Series = s, + PagesRead = _context.AppUserProgresses.Where(p => p.SeriesId == s.Id && p.AppUserId == userId) + .Sum(s1 => s1.PagesRead), + LatestReadDate = _context.AppUserProgresses + .Where(p => p.SeriesId == s.Id && p.AppUserId == userId) + .Max(p => p.LastModified), + s.LastChapterAdded, + }) + .Where(s => s.PagesRead > 0 + && s.PagesRead < s.Series.Pages) + .Where(d => d.LatestReadDate >= cutoffProgressPoint || d.LastChapterAdded >= cutoffLastAddedPoint).OrderByDescending(s => s.LatestReadDate) + .ThenByDescending(s => s.LastChapterAdded) .Select(s => s.Series) .ProjectTo(_mapper.ConfigurationProvider) .AsSplitQuery() .AsNoTracking(); - // Pagination does not work for this query as when we pull the data back, we get multiple rows of the same series. See controller for pagination code - return await retSeries.ToListAsync(); + return await PagedList.CreateAsync(query, userParams.PageNumber, userParams.PageSize); } + private async Task> CreateFilteredSearchQueryable(int userId, int libraryId, FilterDto filter) { var userLibraries = await GetUserLibraries(libraryId, userId); @@ -1044,9 +1041,7 @@ public class SeriesRepository : ISeriesRepository public async Task> GetMoreIn(int userId, int libraryId, int genreId, UserParams userParams) { - var libraryIds = _context.AppUser - .Where(u => u.Id == userId) - .SelectMany(l => l.Libraries.Where(l => l.Id == libraryId).Select(lib => lib.Id)); + var libraryIds = GetLibraryIdsForUser(userId, libraryId); var usersSeriesIds = GetSeriesIdsForLibraryIds(libraryIds); var query = _context.Series @@ -1061,9 +1056,7 @@ public class SeriesRepository : ISeriesRepository public async Task> GetRediscover(int userId, int libraryId, UserParams userParams) { - var libraryIds = _context.AppUser - .Where(u => u.Id == userId) - .SelectMany(l => l.Libraries.Where(l => l.Id == libraryId).Select(lib => lib.Id)); + var libraryIds = GetLibraryIdsForUser(userId, libraryId); var usersSeriesIds = GetSeriesIdsForLibraryIds(libraryIds); var distinctSeriesIdsWithProgress = _context.AppUserProgresses .Where(s => usersSeriesIds.Contains(s.SeriesId)) @@ -1110,9 +1103,7 @@ public class SeriesRepository : ISeriesRepository public async Task> GetHighlyRated(int userId, int libraryId, UserParams userParams) { - var libraryIds = _context.AppUser - .Where(u => u.Id == userId) - .SelectMany(l => l.Libraries.Where(l => l.Id == libraryId).Select(lib => lib.Id)); + var libraryIds = GetLibraryIdsForUser(userId, libraryId); var usersSeriesIds = GetSeriesIdsForLibraryIds(libraryIds); var distinctSeriesIdsWithHighRating = _context.AppUserRating .Where(s => usersSeriesIds.Contains(s.SeriesId) && s.Rating > 4) @@ -1131,9 +1122,7 @@ public class SeriesRepository : ISeriesRepository public async Task> GetQuickReads(int userId, int libraryId, UserParams userParams) { - var libraryIds = _context.AppUser - .Where(u => u.Id == userId) - .SelectMany(l => l.Libraries.Where(l => l.Id == libraryId).Select(lib => lib.Id)); + var libraryIds = GetLibraryIdsForUser(userId, libraryId); var usersSeriesIds = GetSeriesIdsForLibraryIds(libraryIds); var distinctSeriesIdsWithProgress = _context.AppUserProgresses .Where(s => usersSeriesIds.Contains(s.SeriesId)) @@ -1152,6 +1141,19 @@ public class SeriesRepository : ISeriesRepository return await PagedList.CreateAsync(query, userParams.PageNumber, userParams.PageSize); } + /// + /// Returns all library ids for a user + /// + /// + /// 0 for no library filter + /// + private IQueryable GetLibraryIdsForUser(int userId, int libraryId) + { + return _context.AppUser + .Where(u => u.Id == userId) + .SelectMany(l => l.Libraries.Where(l => l.Id == libraryId || libraryId == 0).Select(lib => lib.Id)); + } + public async Task GetRelatedSeries(int userId, int seriesId) { var libraryIds = GetLibraryIdsForUser(userId); diff --git a/API/Services/ArchiveService.cs b/API/Services/ArchiveService.cs index 4320c9d0c..97202b71d 100644 --- a/API/Services/ArchiveService.cs +++ b/API/Services/ArchiveService.cs @@ -437,6 +437,12 @@ namespace API.Services if (Directory.Exists(extractPath)) return; + if (!_directoryService.FileSystem.File.Exists(archivePath)) + { + _logger.LogError("{Archive} does not exist on disk", archivePath); + throw new KavitaException($"{archivePath} does not exist on disk"); + } + var sw = Stopwatch.StartNew(); try diff --git a/API/Services/BookService.cs b/API/Services/BookService.cs index a8627a200..42ec38331 100644 --- a/API/Services/BookService.cs +++ b/API/Services/BookService.cs @@ -399,19 +399,36 @@ namespace API.Services { publicationDate = epubBook.Schema.Package.Metadata.Dates.FirstOrDefault()?.Date; } + var dateParsed = DateTime.TryParse(publicationDate, out var date); + var year = 0; + var month = 0; + var day = 0; + switch (dateParsed) + { + case true: + year = date.Year; + month = date.Month; + day = date.Day; + break; + case false when !string.IsNullOrEmpty(publicationDate) && publicationDate.Length == 4: + int.TryParse(publicationDate, out year); + break; + } + var info = new ComicInfo() { Summary = epubBook.Schema.Package.Metadata.Description, Writer = string.Join(",", epubBook.Schema.Package.Metadata.Creators.Select(c => Parser.Parser.CleanAuthor(c.Creator))), Publisher = string.Join(",", epubBook.Schema.Package.Metadata.Publishers), - Month = !string.IsNullOrEmpty(publicationDate) ? DateTime.Parse(publicationDate).Month : 0, - Day = !string.IsNullOrEmpty(publicationDate) ? DateTime.Parse(publicationDate).Day : 0, - Year = !string.IsNullOrEmpty(publicationDate) ? DateTime.Parse(publicationDate).Year : 0, + Month = month, + Day = day, + Year = year, Title = epubBook.Title, Genre = string.Join(",", epubBook.Schema.Package.Metadata.Subjects.Select(s => s.ToLower().Trim())), LanguageISO = epubBook.Schema.Package.Metadata.Languages.FirstOrDefault() ?? string.Empty - }; + ComicInfo.CleanComicInfo(info); + // Parse tags not exposed via Library foreach (var metadataItem in epubBook.Schema.Package.Metadata.MetaItems) { diff --git a/API/Services/CacheService.cs b/API/Services/CacheService.cs index 5c37d431e..4dba47fc9 100644 --- a/API/Services/CacheService.cs +++ b/API/Services/CacheService.cs @@ -7,6 +7,7 @@ using API.Data; using API.Entities; using API.Entities.Enums; using API.Extensions; +using Kavita.Common; using Microsoft.Extensions.Logging; namespace API.Services @@ -145,6 +146,12 @@ namespace API.Services else if (file.Format == MangaFormat.Epub) { removeNonImages = false; + if (!_directoryService.FileSystem.File.Exists(files[0].FilePath)) + { + _logger.LogError("{Archive} does not exist on disk", files[0].FilePath); + throw new KavitaException($"{files[0].FilePath} does not exist on disk"); + } + _directoryService.ExistOrCreate(extractPath); _directoryService.CopyFileToDirectory(files[0].FilePath, extractPath); } diff --git a/API/Services/DownloadService.cs b/API/Services/DownloadService.cs index 7fc1418e6..76365d3d3 100644 --- a/API/Services/DownloadService.cs +++ b/API/Services/DownloadService.cs @@ -45,12 +45,16 @@ public class DownloadService : IDownloadService { contentType = Path.GetExtension(filepath).ToLowerInvariant() switch { - ".cbz" => "application/zip", - ".cbr" => "application/vnd.rar", - ".cb7" => "application/x-compressed", + ".cbz" => "application/x-cbz", + ".cbr" => "application/x-cbr", + ".cb7" => "application/x-cb7", + ".cbt" => "application/x-cbt", ".epub" => "application/epub+zip", ".7z" => "application/x-7z-compressed", ".7zip" => "application/x-7z-compressed", + ".rar" => "application/vnd.rar", + ".zip" => "application/zip", + ".tar.gz" => "application/gzip", ".pdf" => "application/pdf", _ => contentType }; diff --git a/API/Services/SeriesService.cs b/API/Services/SeriesService.cs index f796586a4..4dec796c2 100644 --- a/API/Services/SeriesService.cs +++ b/API/Services/SeriesService.cs @@ -456,7 +456,7 @@ public class SeriesService : ISeriesService var libraryType = await _unitOfWork.LibraryRepository.GetLibraryTypeAsync(series.LibraryId); var volumes = (await _unitOfWork.VolumeRepository.GetVolumesDtoAsync(seriesId, userId)) - .OrderBy(v => float.Parse(v.Name)) + .OrderBy(v => Parser.Parser.MinimumNumberFromRange(v.Name)) .ToList(); var chapters = volumes.SelectMany(v => v.Chapters).ToList(); diff --git a/UI/Web/src/app/_services/nav.service.ts b/UI/Web/src/app/_services/nav.service.ts index a30e963e6..db1c6b048 100644 --- a/UI/Web/src/app/_services/nav.service.ts +++ b/UI/Web/src/app/_services/nav.service.ts @@ -41,6 +41,8 @@ export class NavService { */ showNavBar() { this.renderer.setStyle(this.document.querySelector('body'), 'margin-top', '56px'); + this.renderer.setStyle(this.document.querySelector('body'), 'height', 'calc(var(--vh)*100 - 56px)'); + this.renderer.setStyle(this.document.querySelector('html'), 'height', 'calc(var(--vh)*100 - 56px)'); this.navbarVisibleSource.next(true); } @@ -49,6 +51,8 @@ export class NavService { */ hideNavBar() { this.renderer.setStyle(this.document.querySelector('body'), 'margin-top', '0px'); + this.renderer.removeStyle(this.document.querySelector('body'), 'height'); + this.renderer.removeStyle(this.document.querySelector('html'), 'height'); this.navbarVisibleSource.next(false); } diff --git a/UI/Web/src/app/admin/invite-user/invite-user.component.html b/UI/Web/src/app/admin/invite-user/invite-user.component.html index 2010ee353..e8c56e484 100644 --- a/UI/Web/src/app/admin/invite-user/invite-user.component.html +++ b/UI/Web/src/app/admin/invite-user/invite-user.component.html @@ -6,7 +6,7 @@