From 70690b747e310c23142018f75fff5fdbcbb299df Mon Sep 17 00:00:00 2001 From: Joe Milazzo Date: Fri, 12 May 2023 15:31:23 -0500 Subject: [PATCH] AVIF Support & Much More! (#1992) * Expand the list of potential favicon icons to grab. * Added a url mapping functionality to use alternative urls for fetching icons * Initial commit to streamline media encoding. No DB migration yet, No UI changes, no Task changes. * Started refactoring code so that webp queries use encoding format instead. * More refactoring to remove hardcoded webp references. * Moved manual migrations to their own folder to keep things organized. Manually drop the obsolete webp keys. * Removed old apis for converting media and now have one. Reworked where the conversion code was located and streamlined events and whatnot. * Make favicon encode setting aware * Cleaned up favicon conversion * Updated format counter to now just use Extension from MangaFile now that it's been out a while. * Tweaked jumpbar code to reduce a lookup to hashmap. * Added AVIF (8-bit only) support. * In UpdatePeopleList, use FirstOrDefault as Single adds extra checks that may not be needed. * You can now remove weblinks from edit series page and you can leave empty cells, they will just be removed on backend. * Forgot a file * Don't prompt to write a review, just show the pencil. It's the same amount of clicks if you do, less if you dont. * Fixed Refresh token using wrong Claim to look up the user. * Refactored how we refresh authentication to perform it every 10 m ins to ensure we always stay authenticated. * Changed Version update code to run more throughout the day. Updated some hangfire to newer method signatures. --- API.Tests/Services/ArchiveServiceTests.cs | 10 +- API.Tests/Services/BookmarkServiceTests.cs | 2 +- API.Tests/Services/CacheServiceTests.cs | 2 +- API.Tests/Services/ParseScannedFilesTests.cs | 2 +- API/Controllers/ImageController.cs | 9 +- API/Controllers/ServerController.cs | 30 +- API/Controllers/SettingsController.cs | 10 +- API/Controllers/UploadController.cs | 6 +- API/DTOs/Settings/ServerSettingDTO.cs | 13 +- API/DTOs/Stats/ServerInfoDto.cs | 11 +- .../MigrateBrokenGMT1Dates.cs | 2 +- .../MigrateChangePasswordRoles.cs | 2 +- .../MigrateChangeRestrictionRoles.cs | 2 +- .../MigrateLoginRole.cs | 2 +- .../MigrateNormalizedEverything.cs | 2 +- .../MigrateNormalizedLocalizedName.cs | 2 +- .../MigrateReadingListAgeRating.cs | 2 +- .../MigrateRemoveExtraThemes.cs | 2 +- .../MigrateRemoveWebPSettingRows.cs | 31 ++ .../MigrateSeriesRelationsExport.cs | 2 +- .../MigrateSeriesRelationsImport.cs | 2 +- .../MigrateToUtcDates.cs | 2 +- .../MigrateUserProgressLibraryId.cs | 2 +- .../Migrations/DataContextModelSnapshot.cs | 62 ++-- .../Repositories/AppUserProgressRepository.cs | 1 + API/Data/Repositories/ChapterRepository.cs | 8 +- .../Repositories/CollectionTagRepository.cs | 8 +- API/Data/Repositories/LibraryRepository.cs | 12 +- .../Repositories/ReadingListRepository.cs | 7 +- API/Data/Repositories/SeriesRepository.cs | 9 +- API/Data/Repositories/SettingsRepository.cs | 6 + API/Data/Repositories/VolumeRepository.cs | 8 +- API/Data/Seed.cs | 3 +- API/Entities/Enums/EncodeFormat.cs | 13 + API/Entities/Enums/ServerSettingKey.cs | 12 +- .../ApplicationServiceExtensions.cs | 1 + API/Extensions/EncodeFormatExtensions.cs | 18 + .../Converters/ServerSettingConverter.cs | 10 +- API/Helpers/PersonHelper.cs | 14 +- API/Program.cs | 1 + API/Services/ArchiveService.cs | 13 +- API/Services/BookService.cs | 14 +- API/Services/BookmarkService.cs | 223 +------------ API/Services/ImageService.cs | 102 ++++-- API/Services/MediaConversionService.cs | 312 ++++++++++++++++++ API/Services/MetadataService.cs | 29 +- API/Services/ReaderService.cs | 7 +- API/Services/ReadingItemService.cs | 12 +- API/Services/ReadingListService.cs | 2 +- API/Services/TaskScheduler.cs | 55 +-- API/Services/Tasks/CleanupService.cs | 8 +- API/Services/Tasks/Scanner/Parser/Parser.cs | 2 +- API/Services/Tasks/Scanner/ProcessSeries.cs | 2 +- API/Services/Tasks/StatsService.cs | 9 +- API/Services/Tasks/VersionUpdaterService.cs | 4 +- API/Services/TokenService.cs | 4 +- API/SignalR/MessageFactory.cs | 4 +- API/Startup.cs | 4 + Kavita.sln.DotSettings | 1 + UI/Web/src/app/_services/account.service.ts | 23 +- UI/Web/src/app/_services/server.service.ts | 8 +- UI/Web/src/app/admin/_models/encode-format.ts | 7 + .../src/app/admin/_models/server-settings.ts | 5 +- .../manage-media-settings.component.html | 31 +- .../manage-media-settings.component.ts | 12 +- .../manage-settings.component.ts | 4 +- .../manage-tasks-settings.component.ts | 22 +- .../edit-series-modal.component.html | 6 +- .../edit-series-modal.component.ts | 13 +- .../series-detail.component.html | 2 +- .../series-detail/series-detail.component.ts | 13 - UI/Web/src/index.html | 2 +- openapi.json | 46 +-- 73 files changed, 778 insertions(+), 566 deletions(-) rename API/Data/{ => ManualMigrations}/MigrateBrokenGMT1Dates.cs (99%) rename API/Data/{ => ManualMigrations}/MigrateChangePasswordRoles.cs (96%) rename API/Data/{ => ManualMigrations}/MigrateChangeRestrictionRoles.cs (97%) rename API/Data/{ => ManualMigrations}/MigrateLoginRole.cs (97%) rename API/Data/{ => ManualMigrations}/MigrateNormalizedEverything.cs (99%) rename API/Data/{ => ManualMigrations}/MigrateNormalizedLocalizedName.cs (97%) rename API/Data/{ => ManualMigrations}/MigrateReadingListAgeRating.cs (97%) rename API/Data/{ => ManualMigrations}/MigrateRemoveExtraThemes.cs (97%) create mode 100644 API/Data/ManualMigrations/MigrateRemoveWebPSettingRows.cs rename API/Data/{ => ManualMigrations}/MigrateSeriesRelationsExport.cs (99%) rename API/Data/{ => ManualMigrations}/MigrateSeriesRelationsImport.cs (98%) rename API/Data/{ => ManualMigrations}/MigrateToUtcDates.cs (99%) rename API/Data/{ => ManualMigrations}/MigrateUserProgressLibraryId.cs (97%) create mode 100644 API/Entities/Enums/EncodeFormat.cs create mode 100644 API/Extensions/EncodeFormatExtensions.cs create mode 100644 API/Services/MediaConversionService.cs create mode 100644 UI/Web/src/app/admin/_models/encode-format.ts diff --git a/API.Tests/Services/ArchiveServiceTests.cs b/API.Tests/Services/ArchiveServiceTests.cs index 7961bc5dc..24041198e 100644 --- a/API.Tests/Services/ArchiveServiceTests.cs +++ b/API.Tests/Services/ArchiveServiceTests.cs @@ -5,6 +5,7 @@ using System.IO.Abstractions.TestingHelpers; using System.IO.Compression; using System.Linq; using API.Archive; +using API.Entities.Enums; using API.Services; using Microsoft.Extensions.Logging; using NetVips; @@ -178,7 +179,7 @@ public class ArchiveServiceTests _directoryService.ExistOrCreate(outputDir); var coverImagePath = archiveService.GetCoverImage(Path.Join(testDirectory, inputFile), - Path.GetFileNameWithoutExtension(inputFile) + "_output", outputDir); + Path.GetFileNameWithoutExtension(inputFile) + "_output", outputDir, EncodeFormat.PNG); var actual = File.ReadAllBytes(Path.Join(outputDir, coverImagePath)); @@ -208,7 +209,7 @@ public class ArchiveServiceTests archiveService.Configure().CanOpen(Path.Join(testDirectory, inputFile)).Returns(ArchiveLibrary.SharpCompress); var coverOutputFile = archiveService.GetCoverImage(Path.Join(testDirectory, inputFile), - Path.GetFileNameWithoutExtension(inputFile), outputDir); + Path.GetFileNameWithoutExtension(inputFile), outputDir, EncodeFormat.PNG); var actualBytes = File.ReadAllBytes(Path.Join(outputDir, coverOutputFile)); var expectedBytes = File.ReadAllBytes(Path.Join(testDirectory, expectedOutputFile)); Assert.Equal(expectedBytes, actualBytes); @@ -222,13 +223,14 @@ public class ArchiveServiceTests public void CanParseCoverImage(string inputFile) { var imageService = Substitute.For(); - imageService.WriteCoverThumbnail(Arg.Any(), Arg.Any(), Arg.Any()).Returns(x => "cover.jpg"); + imageService.WriteCoverThumbnail(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(x => "cover.jpg"); var archiveService = new ArchiveService(_logger, _directoryService, imageService, Substitute.For()); var testDirectory = Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/ArchiveService/"); var inputPath = Path.GetFullPath(Path.Join(testDirectory, inputFile)); var outputPath = Path.Join(testDirectory, Path.GetFileNameWithoutExtension(inputFile) + "_output"); new DirectoryInfo(outputPath).Create(); - var expectedImage = archiveService.GetCoverImage(inputPath, inputFile, outputPath); + var expectedImage = archiveService.GetCoverImage(inputPath, inputFile, outputPath, EncodeFormat.PNG); Assert.Equal("cover.jpg", expectedImage); new DirectoryInfo(outputPath).Delete(); } diff --git a/API.Tests/Services/BookmarkServiceTests.cs b/API.Tests/Services/BookmarkServiceTests.cs index a96d8be46..e3fafd2e1 100644 --- a/API.Tests/Services/BookmarkServiceTests.cs +++ b/API.Tests/Services/BookmarkServiceTests.cs @@ -55,7 +55,7 @@ public class BookmarkServiceTests private BookmarkService Create(IDirectoryService ds) { return new BookmarkService(Substitute.For>(), _unitOfWork, ds, - Substitute.For(), Substitute.For()); +Substitute.For()); } #region Setup diff --git a/API.Tests/Services/CacheServiceTests.cs b/API.Tests/Services/CacheServiceTests.cs index 4bf31f386..f59afcf74 100644 --- a/API.Tests/Services/CacheServiceTests.cs +++ b/API.Tests/Services/CacheServiceTests.cs @@ -42,7 +42,7 @@ internal class MockReadingItemServiceForCacheService : IReadingItemService return 1; } - public string GetCoverImage(string fileFilePath, string fileName, MangaFormat format, bool saveAsWebP) + public string GetCoverImage(string fileFilePath, string fileName, MangaFormat format, EncodeFormat encodeFormat) { return string.Empty; } diff --git a/API.Tests/Services/ParseScannedFilesTests.cs b/API.Tests/Services/ParseScannedFilesTests.cs index ff9ca3ae4..915e584ca 100644 --- a/API.Tests/Services/ParseScannedFilesTests.cs +++ b/API.Tests/Services/ParseScannedFilesTests.cs @@ -43,7 +43,7 @@ internal class MockReadingItemService : IReadingItemService return 1; } - public string GetCoverImage(string fileFilePath, string fileName, MangaFormat format, bool saveAsWebP) + public string GetCoverImage(string fileFilePath, string fileName, MangaFormat format, EncodeFormat encodeFormat) { return string.Empty; } diff --git a/API/Controllers/ImageController.cs b/API/Controllers/ImageController.cs index 268a41b5e..76002946a 100644 --- a/API/Controllers/ImageController.cs +++ b/API/Controllers/ImageController.cs @@ -163,27 +163,26 @@ public class ImageController : BaseApiController /// /// Returns the image associated with a web-link /// - /// - /// /// /// [HttpGet("web-link")] [ResponseCache(CacheProfileName = ResponseCacheProfiles.Month, VaryByQueryKeys = new []{"url", "apiKey"})] - public async Task GetBookmarkImage(string url, string apiKey) + public async Task GetWebLinkImage(string url, string apiKey) { var userId = await _unitOfWork.UserRepository.GetUserIdByApiKeyAsync(apiKey); if (userId == 0) return BadRequest(); if (string.IsNullOrEmpty(url)) return BadRequest("Url cannot be null"); + var encodeFormat = (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).EncodeMediaAs; // Check if the domain exists - var domainFilePath = _directoryService.FileSystem.Path.Join(_directoryService.FaviconDirectory, ImageService.GetWebLinkFormat(url)); + var domainFilePath = _directoryService.FileSystem.Path.Join(_directoryService.FaviconDirectory, ImageService.GetWebLinkFormat(url, encodeFormat)); if (!_directoryService.FileSystem.File.Exists(domainFilePath)) { // We need to request the favicon and save it try { domainFilePath = _directoryService.FileSystem.Path.Join(_directoryService.FaviconDirectory, - await _imageService.DownloadFaviconAsync(url)); + await _imageService.DownloadFaviconAsync(url, encodeFormat)); } catch (Exception) { diff --git a/API/Controllers/ServerController.cs b/API/Controllers/ServerController.cs index 82ac5d507..c799944b3 100644 --- a/API/Controllers/ServerController.cs +++ b/API/Controllers/ServerController.cs @@ -8,6 +8,7 @@ using API.DTOs.Jobs; using API.DTOs.MediaErrors; using API.DTOs.Stats; using API.DTOs.Update; +using API.Entities.Enums; using API.Extensions; using API.Helpers; using API.Services; @@ -119,29 +120,22 @@ public class ServerController : BaseApiController return Ok(await _statsService.GetServerInfo()); } - /// - /// Triggers the scheduling of the convert bookmarks job. Only one job will run at a time. - /// - /// - [HttpPost("convert-bookmarks")] - public ActionResult ScheduleConvertBookmarks() - { - if (TaskScheduler.HasAlreadyEnqueuedTask(BookmarkService.Name, "ConvertAllBookmarkToWebP", Array.Empty(), - TaskScheduler.DefaultQueue, true)) return Ok(); - BackgroundJob.Enqueue(() => _bookmarkService.ConvertAllBookmarkToWebP()); - return Ok(); - } /// - /// Triggers the scheduling of the convert covers job. Only one job will run at a time. + /// Triggers the scheduling of the convert media job. This will convert all media to the target encoding (except for PNG). Only one job will run at a time. /// /// - [HttpPost("convert-covers")] - public ActionResult ScheduleConvertCovers() + [HttpPost("convert-media")] + public async Task ScheduleConvertCovers() { - if (TaskScheduler.HasAlreadyEnqueuedTask(BookmarkService.Name, "ConvertAllCoverToWebP", Array.Empty(), - TaskScheduler.DefaultQueue, true)) return Ok(); - BackgroundJob.Enqueue(() => _taskScheduler.CovertAllCoversToWebP()); + var encoding = (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).EncodeMediaAs; + if (encoding == EncodeFormat.PNG) + { + return BadRequest( + "You cannot convert to PNG. For covers, use Refresh Covers. Bookmarks and favicons cannot be encoded back."); + } + BackgroundJob.Enqueue(() => _taskScheduler.CovertAllCoversToEncoding()); + return Ok(); } diff --git a/API/Controllers/SettingsController.cs b/API/Controllers/SettingsController.cs index 11ec8de31..29393f6ce 100644 --- a/API/Controllers/SettingsController.cs +++ b/API/Controllers/SettingsController.cs @@ -231,15 +231,9 @@ public class SettingsController : BaseApiController _unitOfWork.SettingsRepository.Update(setting); } - if (setting.Key == ServerSettingKey.ConvertBookmarkToWebP && updateSettingsDto.ConvertBookmarkToWebP + string.Empty != setting.Value) + if (setting.Key == ServerSettingKey.EncodeMediaAs && updateSettingsDto.EncodeMediaAs + string.Empty != setting.Value) { - setting.Value = updateSettingsDto.ConvertBookmarkToWebP + string.Empty; - _unitOfWork.SettingsRepository.Update(setting); - } - - if (setting.Key == ServerSettingKey.ConvertCoverToWebP && updateSettingsDto.ConvertCoverToWebP + string.Empty != setting.Value) - { - setting.Value = updateSettingsDto.ConvertCoverToWebP + string.Empty; + setting.Value = updateSettingsDto.EncodeMediaAs + string.Empty; _unitOfWork.SettingsRepository.Update(setting); } diff --git a/API/Controllers/UploadController.cs b/API/Controllers/UploadController.cs index 82ff9237b..b3a832a74 100644 --- a/API/Controllers/UploadController.cs +++ b/API/Controllers/UploadController.cs @@ -222,15 +222,15 @@ public class UploadController : BaseApiController private async Task CreateThumbnail(UploadFileDto uploadFileDto, string filename, int thumbnailSize = 0) { - var convertToWebP = (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).ConvertCoverToWebP; + var encodeFormat = (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).EncodeMediaAs; if (thumbnailSize > 0) { return _imageService.CreateThumbnailFromBase64(uploadFileDto.Url, - filename, convertToWebP, thumbnailSize); + filename, encodeFormat, thumbnailSize); } return _imageService.CreateThumbnailFromBase64(uploadFileDto.Url, - filename, convertToWebP); + filename, encodeFormat); } /// diff --git a/API/DTOs/Settings/ServerSettingDTO.cs b/API/DTOs/Settings/ServerSettingDTO.cs index 90a0901b3..c5bd0470b 100644 --- a/API/DTOs/Settings/ServerSettingDTO.cs +++ b/API/DTOs/Settings/ServerSettingDTO.cs @@ -1,4 +1,5 @@ -using API.Services; +using API.Entities.Enums; +using API.Services; namespace API.DTOs.Settings; @@ -47,9 +48,11 @@ public class ServerSettingDto /// public string InstallId { get; set; } = default!; /// - /// If the server should save bookmarks as WebP encoding + /// The format that should be used when saving media for Kavita /// - public bool ConvertBookmarkToWebP { get; set; } + /// This includes things like: Covers, Bookmarks, Favicons + public EncodeFormat EncodeMediaAs { get; set; } + /// /// The amount of Backups before cleanup /// @@ -65,10 +68,6 @@ public class ServerSettingDto /// Value should be between 1 and 30 public int TotalLogs { get; set; } /// - /// If the server should save covers as WebP encoding - /// - public bool ConvertCoverToWebP { get; set; } - /// /// The Host name (ie Reverse proxy domain name) for the server /// public string HostName { get; set; } diff --git a/API/DTOs/Stats/ServerInfoDto.cs b/API/DTOs/Stats/ServerInfoDto.cs index d8c60920e..f0ed72e7e 100644 --- a/API/DTOs/Stats/ServerInfoDto.cs +++ b/API/DTOs/Stats/ServerInfoDto.cs @@ -85,11 +85,6 @@ public class ServerInfoDto /// Introduced in v0.5.4 public int TotalPeople { get; set; } /// - /// Is this instance storing bookmarks as WebP - /// - /// Introduced in v0.5.4 - public bool StoreBookmarksAsWebP { get; set; } - /// /// Number of users on this instance using Card Layout /// /// Introduced in v0.5.4 @@ -175,8 +170,8 @@ public class ServerInfoDto /// Introduced in v0.7.0 public long TotalReadingHours { get; set; } /// - /// Is the Server saving covers as WebP + /// The encoding the server is using to save media /// - /// Added in v0.7.0 - public bool StoreCoversAsWebP { get; set; } + /// Added in v0.7.3 + public EncodeFormat EncodeMediaAs { get; set; } } diff --git a/API/Data/MigrateBrokenGMT1Dates.cs b/API/Data/ManualMigrations/MigrateBrokenGMT1Dates.cs similarity index 99% rename from API/Data/MigrateBrokenGMT1Dates.cs rename to API/Data/ManualMigrations/MigrateBrokenGMT1Dates.cs index 20939b1ef..8b4576696 100644 --- a/API/Data/MigrateBrokenGMT1Dates.cs +++ b/API/Data/ManualMigrations/MigrateBrokenGMT1Dates.cs @@ -3,7 +3,7 @@ using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; -namespace API.Data; +namespace API.Data.ManualMigrations; /// /// v0.7 introduced UTC dates and GMT+1 users would sometimes have dates stored as '0000-12-31 23:00:00'. diff --git a/API/Data/MigrateChangePasswordRoles.cs b/API/Data/ManualMigrations/MigrateChangePasswordRoles.cs similarity index 96% rename from API/Data/MigrateChangePasswordRoles.cs rename to API/Data/ManualMigrations/MigrateChangePasswordRoles.cs index 722f92a7d..74344775f 100644 --- a/API/Data/MigrateChangePasswordRoles.cs +++ b/API/Data/ManualMigrations/MigrateChangePasswordRoles.cs @@ -3,7 +3,7 @@ using API.Constants; using API.Entities; using Microsoft.AspNetCore.Identity; -namespace API.Data; +namespace API.Data.ManualMigrations; /// /// New role introduced in v0.5.1. Adds the role to all users. diff --git a/API/Data/MigrateChangeRestrictionRoles.cs b/API/Data/ManualMigrations/MigrateChangeRestrictionRoles.cs similarity index 97% rename from API/Data/MigrateChangeRestrictionRoles.cs rename to API/Data/ManualMigrations/MigrateChangeRestrictionRoles.cs index 7e64a7098..0b22b3f23 100644 --- a/API/Data/MigrateChangeRestrictionRoles.cs +++ b/API/Data/ManualMigrations/MigrateChangeRestrictionRoles.cs @@ -4,7 +4,7 @@ using API.Entities; using Microsoft.AspNetCore.Identity; using Microsoft.Extensions.Logging; -namespace API.Data; +namespace API.Data.ManualMigrations; /// /// New role introduced in v0.6. Adds the role to all users. diff --git a/API/Data/MigrateLoginRole.cs b/API/Data/ManualMigrations/MigrateLoginRole.cs similarity index 97% rename from API/Data/MigrateLoginRole.cs rename to API/Data/ManualMigrations/MigrateLoginRole.cs index 93f839589..0a582b761 100644 --- a/API/Data/MigrateLoginRole.cs +++ b/API/Data/ManualMigrations/MigrateLoginRole.cs @@ -4,7 +4,7 @@ using API.Entities; using Microsoft.AspNetCore.Identity; using Microsoft.Extensions.Logging; -namespace API.Data; +namespace API.Data.ManualMigrations; /// /// Added in v0.7.1.18 diff --git a/API/Data/MigrateNormalizedEverything.cs b/API/Data/ManualMigrations/MigrateNormalizedEverything.cs similarity index 99% rename from API/Data/MigrateNormalizedEverything.cs rename to API/Data/ManualMigrations/MigrateNormalizedEverything.cs index 69a3e2728..d5ba39ab6 100644 --- a/API/Data/MigrateNormalizedEverything.cs +++ b/API/Data/ManualMigrations/MigrateNormalizedEverything.cs @@ -3,7 +3,7 @@ using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; -namespace API.Data; +namespace API.Data.ManualMigrations; /// /// v0.6.0 introduced a change in how Normalization works and hence every normalized field needs to be re-calculated diff --git a/API/Data/MigrateNormalizedLocalizedName.cs b/API/Data/ManualMigrations/MigrateNormalizedLocalizedName.cs similarity index 97% rename from API/Data/MigrateNormalizedLocalizedName.cs rename to API/Data/ManualMigrations/MigrateNormalizedLocalizedName.cs index 37ea705e3..dcb5e9370 100644 --- a/API/Data/MigrateNormalizedLocalizedName.cs +++ b/API/Data/ManualMigrations/MigrateNormalizedLocalizedName.cs @@ -3,7 +3,7 @@ using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; -namespace API.Data; +namespace API.Data.ManualMigrations; /// /// v0.5.6 introduced Normalized Localized Name, which allows for faster lookups and less memory usage. This migration will calculate them once diff --git a/API/Data/MigrateReadingListAgeRating.cs b/API/Data/ManualMigrations/MigrateReadingListAgeRating.cs similarity index 97% rename from API/Data/MigrateReadingListAgeRating.cs rename to API/Data/ManualMigrations/MigrateReadingListAgeRating.cs index b057d702b..4541801c7 100644 --- a/API/Data/MigrateReadingListAgeRating.cs +++ b/API/Data/ManualMigrations/MigrateReadingListAgeRating.cs @@ -4,7 +4,7 @@ using API.Services; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; -namespace API.Data; +namespace API.Data.ManualMigrations; /// /// New role introduced in v0.6. Calculates the Age Rating on all Reading Lists diff --git a/API/Data/MigrateRemoveExtraThemes.cs b/API/Data/ManualMigrations/MigrateRemoveExtraThemes.cs similarity index 97% rename from API/Data/MigrateRemoveExtraThemes.cs rename to API/Data/ManualMigrations/MigrateRemoveExtraThemes.cs index 747c910c0..3d78db37d 100644 --- a/API/Data/MigrateRemoveExtraThemes.cs +++ b/API/Data/ManualMigrations/MigrateRemoveExtraThemes.cs @@ -3,7 +3,7 @@ using System.Linq; using System.Threading.Tasks; using API.Services.Tasks; -namespace API.Data; +namespace API.Data.ManualMigrations; /// /// In v0.5.3, we removed Light and E-Ink themes. This migration will remove the themes from the DB and default anyone on diff --git a/API/Data/ManualMigrations/MigrateRemoveWebPSettingRows.cs b/API/Data/ManualMigrations/MigrateRemoveWebPSettingRows.cs new file mode 100644 index 000000000..bbabf1905 --- /dev/null +++ b/API/Data/ManualMigrations/MigrateRemoveWebPSettingRows.cs @@ -0,0 +1,31 @@ +using System.Threading.Tasks; +using API.Entities.Enums; +using Microsoft.Extensions.Logging; + +namespace API.Data.ManualMigrations; + +/// +/// Added in v0.7.2.7/v0.7.3 in which the ConvertXToWebP Setting keys were removed. This migration will remove them. +/// +public static class MigrateRemoveWebPSettingRows +{ + public static async Task Migrate(IUnitOfWork unitOfWork, ILogger logger) + { + logger.LogCritical("Running MigrateRemoveWebPSettingRows migration - Please be patient, this may take some time. This is not an error"); + + var key = await unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.ConvertBookmarkToWebP); + var key2 = await unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.ConvertCoverToWebP); + if (key == null && key2 == null) + { + logger.LogCritical("Running MigrateRemoveWebPSettingRows migration - complete. Nothing to do"); + return; + } + + unitOfWork.SettingsRepository.Remove(key); + unitOfWork.SettingsRepository.Remove(key2); + + await unitOfWork.CommitAsync(); + + logger.LogCritical("Running MigrateRemoveWebPSettingRows migration - Completed. This is not an error"); + } +} diff --git a/API/Data/MigrateSeriesRelationsExport.cs b/API/Data/ManualMigrations/MigrateSeriesRelationsExport.cs similarity index 99% rename from API/Data/MigrateSeriesRelationsExport.cs rename to API/Data/ManualMigrations/MigrateSeriesRelationsExport.cs index f31688641..4427a6687 100644 --- a/API/Data/MigrateSeriesRelationsExport.cs +++ b/API/Data/ManualMigrations/MigrateSeriesRelationsExport.cs @@ -10,7 +10,7 @@ using Kavita.Common.EnvironmentInfo; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; -namespace API.Data; +namespace API.Data.ManualMigrations; internal sealed class SeriesRelationMigrationOutput { diff --git a/API/Data/MigrateSeriesRelationsImport.cs b/API/Data/ManualMigrations/MigrateSeriesRelationsImport.cs similarity index 98% rename from API/Data/MigrateSeriesRelationsImport.cs rename to API/Data/ManualMigrations/MigrateSeriesRelationsImport.cs index 8035e8c4b..9faefde6a 100644 --- a/API/Data/MigrateSeriesRelationsImport.cs +++ b/API/Data/ManualMigrations/MigrateSeriesRelationsImport.cs @@ -8,7 +8,7 @@ using CsvHelper; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; -namespace API.Data; +namespace API.Data.ManualMigrations; /// /// Introduced in v0.6.1.2 and v0.7, this imports to a temp file the existing series relationships. It is a 3 part migration. diff --git a/API/Data/MigrateToUtcDates.cs b/API/Data/ManualMigrations/MigrateToUtcDates.cs similarity index 99% rename from API/Data/MigrateToUtcDates.cs rename to API/Data/ManualMigrations/MigrateToUtcDates.cs index 1cdf01445..a1e758bdb 100644 --- a/API/Data/MigrateToUtcDates.cs +++ b/API/Data/ManualMigrations/MigrateToUtcDates.cs @@ -3,7 +3,7 @@ using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; -namespace API.Data; +namespace API.Data.ManualMigrations; /// /// Introduced in v0.6.1.38 or v0.7.0, diff --git a/API/Data/MigrateUserProgressLibraryId.cs b/API/Data/ManualMigrations/MigrateUserProgressLibraryId.cs similarity index 97% rename from API/Data/MigrateUserProgressLibraryId.cs rename to API/Data/ManualMigrations/MigrateUserProgressLibraryId.cs index 78e9933da..575be95ae 100644 --- a/API/Data/MigrateUserProgressLibraryId.cs +++ b/API/Data/ManualMigrations/MigrateUserProgressLibraryId.cs @@ -1,7 +1,7 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; -namespace API.Data; +namespace API.Data.ManualMigrations; /// /// Introduced in v0.6.1.8 and v0.7, this adds library ids to all User Progress to allow for easier queries against progress diff --git a/API/Data/Migrations/DataContextModelSnapshot.cs b/API/Data/Migrations/DataContextModelSnapshot.cs index e8d7f2aad..ac4fa03e0 100644 --- a/API/Data/Migrations/DataContextModelSnapshot.cs +++ b/API/Data/Migrations/DataContextModelSnapshot.cs @@ -177,7 +177,7 @@ namespace API.Data.Migrations b.HasIndex("AppUserId"); - b.ToTable("AppUserBookmark"); + b.ToTable("AppUserBookmark", (string)null); }); modelBuilder.Entity("API.Entities.AppUserPreferences", b => @@ -282,7 +282,7 @@ namespace API.Data.Migrations b.HasIndex("ThemeId"); - b.ToTable("AppUserPreferences"); + b.ToTable("AppUserPreferences", (string)null); }); modelBuilder.Entity("API.Entities.AppUserProgress", b => @@ -332,7 +332,7 @@ namespace API.Data.Migrations b.HasIndex("SeriesId"); - b.ToTable("AppUserProgresses"); + b.ToTable("AppUserProgresses", (string)null); }); modelBuilder.Entity("API.Entities.AppUserRating", b => @@ -359,7 +359,7 @@ namespace API.Data.Migrations b.HasIndex("SeriesId"); - b.ToTable("AppUserRating"); + b.ToTable("AppUserRating", (string)null); }); modelBuilder.Entity("API.Entities.AppUserRole", b => @@ -484,7 +484,7 @@ namespace API.Data.Migrations b.HasIndex("VolumeId"); - b.ToTable("Chapter"); + b.ToTable("Chapter", (string)null); }); modelBuilder.Entity("API.Entities.CollectionTag", b => @@ -519,7 +519,7 @@ namespace API.Data.Migrations b.HasIndex("Id", "Promoted") .IsUnique(); - b.ToTable("CollectionTag"); + b.ToTable("CollectionTag", (string)null); }); modelBuilder.Entity("API.Entities.Device", b => @@ -565,7 +565,7 @@ namespace API.Data.Migrations b.HasIndex("AppUserId"); - b.ToTable("Device"); + b.ToTable("Device", (string)null); }); modelBuilder.Entity("API.Entities.FolderPath", b => @@ -587,7 +587,7 @@ namespace API.Data.Migrations b.HasIndex("LibraryId"); - b.ToTable("FolderPath"); + b.ToTable("FolderPath", (string)null); }); modelBuilder.Entity("API.Entities.Genre", b => @@ -607,7 +607,7 @@ namespace API.Data.Migrations b.HasIndex("NormalizedTitle") .IsUnique(); - b.ToTable("Genre"); + b.ToTable("Genre", (string)null); }); modelBuilder.Entity("API.Entities.Library", b => @@ -672,7 +672,7 @@ namespace API.Data.Migrations b.HasKey("Id"); - b.ToTable("Library"); + b.ToTable("Library", (string)null); }); modelBuilder.Entity("API.Entities.MangaFile", b => @@ -721,7 +721,7 @@ namespace API.Data.Migrations b.HasIndex("ChapterId"); - b.ToTable("MangaFile"); + b.ToTable("MangaFile", (string)null); }); modelBuilder.Entity("API.Entities.MediaError", b => @@ -756,7 +756,7 @@ namespace API.Data.Migrations b.HasKey("Id"); - b.ToTable("MediaError"); + b.ToTable("MediaError", (string)null); }); modelBuilder.Entity("API.Entities.Metadata.SeriesMetadata", b => @@ -857,7 +857,7 @@ namespace API.Data.Migrations b.HasIndex("Id", "SeriesId") .IsUnique(); - b.ToTable("SeriesMetadata"); + b.ToTable("SeriesMetadata", (string)null); }); modelBuilder.Entity("API.Entities.Metadata.SeriesRelation", b => @@ -881,7 +881,7 @@ namespace API.Data.Migrations b.HasIndex("TargetSeriesId"); - b.ToTable("SeriesRelation"); + b.ToTable("SeriesRelation", (string)null); }); modelBuilder.Entity("API.Entities.Person", b => @@ -901,7 +901,7 @@ namespace API.Data.Migrations b.HasKey("Id"); - b.ToTable("Person"); + b.ToTable("Person", (string)null); }); modelBuilder.Entity("API.Entities.ReadingList", b => @@ -962,7 +962,7 @@ namespace API.Data.Migrations b.HasIndex("AppUserId"); - b.ToTable("ReadingList"); + b.ToTable("ReadingList", (string)null); }); modelBuilder.Entity("API.Entities.ReadingListItem", b => @@ -996,7 +996,7 @@ namespace API.Data.Migrations b.HasIndex("VolumeId"); - b.ToTable("ReadingListItem"); + b.ToTable("ReadingListItem", (string)null); }); modelBuilder.Entity("API.Entities.Series", b => @@ -1095,7 +1095,7 @@ namespace API.Data.Migrations b.HasIndex("LibraryId"); - b.ToTable("Series"); + b.ToTable("Series", (string)null); }); modelBuilder.Entity("API.Entities.ServerSetting", b => @@ -1112,7 +1112,7 @@ namespace API.Data.Migrations b.HasKey("Key"); - b.ToTable("ServerSetting"); + b.ToTable("ServerSetting", (string)null); }); modelBuilder.Entity("API.Entities.ServerStatistics", b => @@ -1150,7 +1150,7 @@ namespace API.Data.Migrations b.HasKey("Id"); - b.ToTable("ServerStatistics"); + b.ToTable("ServerStatistics", (string)null); }); modelBuilder.Entity("API.Entities.SiteTheme", b => @@ -1188,7 +1188,7 @@ namespace API.Data.Migrations b.HasKey("Id"); - b.ToTable("SiteTheme"); + b.ToTable("SiteTheme", (string)null); }); modelBuilder.Entity("API.Entities.Tag", b => @@ -1208,7 +1208,7 @@ namespace API.Data.Migrations b.HasIndex("NormalizedTitle") .IsUnique(); - b.ToTable("Tag"); + b.ToTable("Tag", (string)null); }); modelBuilder.Entity("API.Entities.Volume", b => @@ -1260,7 +1260,7 @@ namespace API.Data.Migrations b.HasIndex("SeriesId"); - b.ToTable("Volume"); + b.ToTable("Volume", (string)null); }); modelBuilder.Entity("AppUserLibrary", b => @@ -1275,7 +1275,7 @@ namespace API.Data.Migrations b.HasIndex("LibrariesId"); - b.ToTable("AppUserLibrary"); + b.ToTable("AppUserLibrary", (string)null); }); modelBuilder.Entity("ChapterGenre", b => @@ -1290,7 +1290,7 @@ namespace API.Data.Migrations b.HasIndex("GenresId"); - b.ToTable("ChapterGenre"); + b.ToTable("ChapterGenre", (string)null); }); modelBuilder.Entity("ChapterPerson", b => @@ -1305,7 +1305,7 @@ namespace API.Data.Migrations b.HasIndex("PeopleId"); - b.ToTable("ChapterPerson"); + b.ToTable("ChapterPerson", (string)null); }); modelBuilder.Entity("ChapterTag", b => @@ -1320,7 +1320,7 @@ namespace API.Data.Migrations b.HasIndex("TagsId"); - b.ToTable("ChapterTag"); + b.ToTable("ChapterTag", (string)null); }); modelBuilder.Entity("CollectionTagSeriesMetadata", b => @@ -1335,7 +1335,7 @@ namespace API.Data.Migrations b.HasIndex("SeriesMetadatasId"); - b.ToTable("CollectionTagSeriesMetadata"); + b.ToTable("CollectionTagSeriesMetadata", (string)null); }); modelBuilder.Entity("GenreSeriesMetadata", b => @@ -1350,7 +1350,7 @@ namespace API.Data.Migrations b.HasIndex("SeriesMetadatasId"); - b.ToTable("GenreSeriesMetadata"); + b.ToTable("GenreSeriesMetadata", (string)null); }); modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim", b => @@ -1449,7 +1449,7 @@ namespace API.Data.Migrations b.HasIndex("SeriesMetadatasId"); - b.ToTable("PersonSeriesMetadata"); + b.ToTable("PersonSeriesMetadata", (string)null); }); modelBuilder.Entity("SeriesMetadataTag", b => @@ -1464,7 +1464,7 @@ namespace API.Data.Migrations b.HasIndex("TagsId"); - b.ToTable("SeriesMetadataTag"); + b.ToTable("SeriesMetadataTag", (string)null); }); modelBuilder.Entity("API.Entities.AppUserBookmark", b => diff --git a/API/Data/Repositories/AppUserProgressRepository.cs b/API/Data/Repositories/AppUserProgressRepository.cs index fdd48b649..a25b22fb9 100644 --- a/API/Data/Repositories/AppUserProgressRepository.cs +++ b/API/Data/Repositories/AppUserProgressRepository.cs @@ -1,6 +1,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using API.Data.ManualMigrations; using API.DTOs; using API.Entities; using API.Entities.Enums; diff --git a/API/Data/Repositories/ChapterRepository.cs b/API/Data/Repositories/ChapterRepository.cs index f10f036fc..c31e059b2 100644 --- a/API/Data/Repositories/ChapterRepository.cs +++ b/API/Data/Repositories/ChapterRepository.cs @@ -6,6 +6,7 @@ using API.DTOs; using API.DTOs.Metadata; using API.DTOs.Reader; using API.Entities; +using API.Entities.Enums; using API.Extensions; using API.Extensions.QueryExtensions; using AutoMapper; @@ -36,7 +37,7 @@ public interface IChapterRepository Task> GetFilesForChaptersAsync(IReadOnlyList chapterIds); Task GetChapterCoverImageAsync(int chapterId); Task> GetAllCoverImagesAsync(); - Task> GetAllChaptersWithNonWebPCovers(); + Task> GetAllChaptersWithCoversInDifferentEncoding(EncodeFormat format); Task> GetCoverImagesForLockedChaptersAsync(); Task AddChapterModifiers(int userId, ChapterDto chapter); } @@ -208,10 +209,11 @@ public class ChapterRepository : IChapterRepository .ToListAsync())!; } - public async Task> GetAllChaptersWithNonWebPCovers() + public async Task> GetAllChaptersWithCoversInDifferentEncoding(EncodeFormat format) { + var extension = format.GetExtension(); return await _context.Chapter - .Where(c => !string.IsNullOrEmpty(c.CoverImage) && !c.CoverImage.EndsWith(".webp")) + .Where(c => !string.IsNullOrEmpty(c.CoverImage) && !c.CoverImage.EndsWith(extension)) .ToListAsync(); } diff --git a/API/Data/Repositories/CollectionTagRepository.cs b/API/Data/Repositories/CollectionTagRepository.cs index dd9d375b4..8d6dffcc9 100644 --- a/API/Data/Repositories/CollectionTagRepository.cs +++ b/API/Data/Repositories/CollectionTagRepository.cs @@ -5,6 +5,7 @@ using System.Threading.Tasks; using API.Data.Misc; using API.DTOs.CollectionTags; using API.Entities; +using API.Entities.Enums; using API.Extensions; using API.Extensions.QueryExtensions; using AutoMapper; @@ -34,7 +35,7 @@ public interface ICollectionTagRepository Task> GetAllTagsAsync(CollectionTagIncludes includes = CollectionTagIncludes.None); Task> GetAllCoverImagesAsync(); Task TagExists(string title); - Task> GetAllWithNonWebPCovers(); + Task> GetAllWithCoversInDifferentEncoding(EncodeFormat encodeFormat); } public class CollectionTagRepository : ICollectionTagRepository { @@ -108,10 +109,11 @@ public class CollectionTagRepository : ICollectionTagRepository .AnyAsync(x => x.NormalizedTitle != null && x.NormalizedTitle.Equals(normalized)); } - public async Task> GetAllWithNonWebPCovers() + public async Task> GetAllWithCoversInDifferentEncoding(EncodeFormat encodeFormat) { + var extension = encodeFormat.GetExtension(); return await _context.CollectionTag - .Where(c => !string.IsNullOrEmpty(c.CoverImage) && !c.CoverImage.EndsWith(".webp")) + .Where(c => !string.IsNullOrEmpty(c.CoverImage) && !c.CoverImage.EndsWith(extension)) .ToListAsync(); } diff --git a/API/Data/Repositories/LibraryRepository.cs b/API/Data/Repositories/LibraryRepository.cs index c8be8929d..a35ab411b 100644 --- a/API/Data/Repositories/LibraryRepository.cs +++ b/API/Data/Repositories/LibraryRepository.cs @@ -52,7 +52,7 @@ public interface ILibraryRepository Task GetLibraryCoverImageAsync(int libraryId); Task> GetAllCoverImagesAsync(); Task> GetLibraryTypesForIdsAsync(IEnumerable libraryIds); - Task> GetAllWithNonWebPCovers(); + Task> GetAllWithCoversInDifferentEncoding(EncodeFormat encodeFormat); } public class LibraryRepository : ILibraryRepository @@ -170,10 +170,7 @@ public class LibraryRepository : ILibraryRepository var c = sortChar; var isAlpha = char.IsLetter(sortChar); if (!isAlpha) c = '#'; - if (!firstCharacterMap.ContainsKey(c)) - { - firstCharacterMap[c] = 0; - } + firstCharacterMap.TryAdd(c, 0); firstCharacterMap[c] += 1; } @@ -371,10 +368,11 @@ public class LibraryRepository : ILibraryRepository return dict; } - public async Task> GetAllWithNonWebPCovers() + public async Task> GetAllWithCoversInDifferentEncoding(EncodeFormat encodeFormat) { + var extension = encodeFormat.GetExtension(); return await _context.Library - .Where(c => !string.IsNullOrEmpty(c.CoverImage) && !c.CoverImage.EndsWith(".webp")) + .Where(c => !string.IsNullOrEmpty(c.CoverImage) && !c.CoverImage.EndsWith(extension)) .ToListAsync(); } } diff --git a/API/Data/Repositories/ReadingListRepository.cs b/API/Data/Repositories/ReadingListRepository.cs index 649ee4a9b..207d12bc6 100644 --- a/API/Data/Repositories/ReadingListRepository.cs +++ b/API/Data/Repositories/ReadingListRepository.cs @@ -45,7 +45,7 @@ public interface IReadingListRepository Task> GetAllCoverImagesAsync(); Task ReadingListExists(string name); IEnumerable GetReadingListCharactersAsync(int readingListId); - Task> GetAllWithNonWebPCovers(); + Task> GetAllWithCoversInDifferentEncoding(EncodeFormat encodeFormat); Task> GetFirstFourCoverImagesByReadingListId(int readingListId); Task RemoveReadingListsWithoutSeries(); Task GetReadingListByTitleAsync(string name, int userId, ReadingListIncludes includes = ReadingListIncludes.Items); @@ -110,10 +110,11 @@ public class ReadingListRepository : IReadingListRepository .AsEnumerable(); } - public async Task> GetAllWithNonWebPCovers() + public async Task> GetAllWithCoversInDifferentEncoding(EncodeFormat encodeFormat) { + var extension = encodeFormat.GetExtension(); return await _context.ReadingList - .Where(c => !string.IsNullOrEmpty(c.CoverImage) && !c.CoverImage.EndsWith(".webp")) + .Where(c => !string.IsNullOrEmpty(c.CoverImage) && !c.CoverImage.EndsWith(extension)) .ToListAsync(); } diff --git a/API/Data/Repositories/SeriesRepository.cs b/API/Data/Repositories/SeriesRepository.cs index e611e841c..46b591ee2 100644 --- a/API/Data/Repositories/SeriesRepository.cs +++ b/API/Data/Repositories/SeriesRepository.cs @@ -4,6 +4,7 @@ using System.Drawing; using System.Linq; using System.Text.RegularExpressions; using System.Threading.Tasks; +using API.Data.ManualMigrations; using API.Data.Misc; using API.Data.Scanner; using API.DTOs; @@ -132,7 +133,7 @@ public interface ISeriesRepository Task> GetLibraryIdsForSeriesAsync(); Task> GetSeriesMetadataForIds(IEnumerable seriesIds); - Task> GetAllWithNonWebPCovers(bool customOnly = true); + Task> GetAllWithCoversInDifferentEncoding(EncodeFormat encodeFormat, bool customOnly = true); } public class SeriesRepository : ISeriesRepository @@ -565,12 +566,14 @@ public class SeriesRepository : ISeriesRepository /// Returns custom images only /// /// - public async Task> GetAllWithNonWebPCovers(bool customOnly = true) + public async Task> GetAllWithCoversInDifferentEncoding(EncodeFormat encodeFormat, + bool customOnly = true) { + var extension = encodeFormat.GetExtension(); var prefix = ImageService.GetSeriesFormat(0).Replace("0", string.Empty); return await _context.Series .Where(c => !string.IsNullOrEmpty(c.CoverImage) - && !c.CoverImage.EndsWith(".webp") + && !c.CoverImage.EndsWith(extension) && (!customOnly || c.CoverImage.StartsWith(prefix))) .ToListAsync(); } diff --git a/API/Data/Repositories/SettingsRepository.cs b/API/Data/Repositories/SettingsRepository.cs index c6a682391..f142947b1 100644 --- a/API/Data/Repositories/SettingsRepository.cs +++ b/API/Data/Repositories/SettingsRepository.cs @@ -15,6 +15,7 @@ public interface ISettingsRepository Task GetSettingsDtoAsync(); Task GetSettingAsync(ServerSettingKey key); Task> GetSettingsAsync(); + void Remove(ServerSetting setting); } public class SettingsRepository : ISettingsRepository { @@ -32,6 +33,11 @@ public class SettingsRepository : ISettingsRepository _context.Entry(settings).State = EntityState.Modified; } + public void Remove(ServerSetting setting) + { + _context.Remove(setting); + } + public async Task GetSettingsDtoAsync() { var settings = await _context.ServerSetting diff --git a/API/Data/Repositories/VolumeRepository.cs b/API/Data/Repositories/VolumeRepository.cs index 833ea9055..a849ab7f0 100644 --- a/API/Data/Repositories/VolumeRepository.cs +++ b/API/Data/Repositories/VolumeRepository.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Threading.Tasks; using API.DTOs; using API.Entities; +using API.Entities.Enums; using API.Extensions; using API.Services; using AutoMapper; @@ -26,7 +27,7 @@ public interface IVolumeRepository Task> GetVolumesForSeriesAsync(IList seriesIds, bool includeChapters = false); Task> GetVolumes(int seriesId); Task GetVolumeByIdAsync(int volumeId); - Task> GetAllWithNonWebPCovers(); + Task> GetAllWithCoversInDifferentEncoding(EncodeFormat encodeFormat); } public class VolumeRepository : IVolumeRepository { @@ -200,10 +201,11 @@ public class VolumeRepository : IVolumeRepository return await _context.Volume.SingleOrDefaultAsync(x => x.Id == volumeId); } - public async Task> GetAllWithNonWebPCovers() + public async Task> GetAllWithCoversInDifferentEncoding(EncodeFormat encodeFormat) { + var extension = encodeFormat.GetExtension(); return await _context.Volume - .Where(c => !string.IsNullOrEmpty(c.CoverImage) && !c.CoverImage.EndsWith(".webp")) + .Where(c => !string.IsNullOrEmpty(c.CoverImage) && !c.CoverImage.EndsWith(extension)) .ToListAsync(); } diff --git a/API/Data/Seed.cs b/API/Data/Seed.cs index 0cde90b6d..24f0236f5 100644 --- a/API/Data/Seed.cs +++ b/API/Data/Seed.cs @@ -101,12 +101,11 @@ public static class Seed new() {Key = ServerSettingKey.InstallVersion, Value = BuildInfo.Version.ToString()}, new() {Key = ServerSettingKey.BookmarkDirectory, Value = directoryService.BookmarkDirectory}, new() {Key = ServerSettingKey.EmailServiceUrl, Value = EmailService.DefaultApiUrl}, - new() {Key = ServerSettingKey.ConvertBookmarkToWebP, Value = "false"}, new() {Key = ServerSettingKey.TotalBackups, Value = "30"}, new() {Key = ServerSettingKey.TotalLogs, Value = "30"}, new() {Key = ServerSettingKey.EnableFolderWatching, Value = "false"}, - new() {Key = ServerSettingKey.ConvertCoverToWebP, Value = "false"}, new() {Key = ServerSettingKey.HostName, Value = string.Empty}, + new() {Key = ServerSettingKey.EncodeMediaAs, Value = EncodeFormat.PNG.ToString()}, }.ToArray()); foreach (var defaultSetting in DefaultSettings) diff --git a/API/Entities/Enums/EncodeFormat.cs b/API/Entities/Enums/EncodeFormat.cs new file mode 100644 index 000000000..70345f1db --- /dev/null +++ b/API/Entities/Enums/EncodeFormat.cs @@ -0,0 +1,13 @@ +using System.ComponentModel; + +namespace API.Entities.Enums; + +public enum EncodeFormat +{ + [Description("PNG")] + PNG = 0, + [Description("WebP")] + WEBP = 1, + [Description("AVIF")] + AVIF = 2 +} diff --git a/API/Entities/Enums/ServerSettingKey.cs b/API/Entities/Enums/ServerSettingKey.cs index 877429177..02e9b1e90 100644 --- a/API/Entities/Enums/ServerSettingKey.cs +++ b/API/Entities/Enums/ServerSettingKey.cs @@ -1,4 +1,5 @@ -using System.ComponentModel; +using System; +using System.ComponentModel; namespace API.Entities.Enums; @@ -82,6 +83,7 @@ public enum ServerSettingKey /// /// If Kavita should save bookmarks as WebP images /// + [Obsolete("Use EncodeMediaAs instead")] [Description("ConvertBookmarkToWebP")] ConvertBookmarkToWebP = 14, /// @@ -102,6 +104,7 @@ public enum ServerSettingKey /// /// If Kavita should save covers as WebP images /// + [Obsolete("Use EncodeMediaAs instead")] [Description("ConvertCoverToWebP")] ConvertCoverToWebP = 19, /// @@ -114,4 +117,11 @@ public enum ServerSettingKey /// [Description("IpAddresses")] IpAddresses = 21, + /// + /// Encode all media as PNG/WebP/AVIF/etc. + /// + /// As of v0.7.3 this replaced ConvertCoverToWebP and ConvertBookmarkToWebP + [Description("EncodeMediaAs")] + EncodeMediaAs = 22, + } diff --git a/API/Extensions/ApplicationServiceExtensions.cs b/API/Extensions/ApplicationServiceExtensions.cs index d9c5c3d06..ba2b201ae 100644 --- a/API/Extensions/ApplicationServiceExtensions.cs +++ b/API/Extensions/ApplicationServiceExtensions.cs @@ -50,6 +50,7 @@ public static class ApplicationServiceExtensions services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/API/Extensions/EncodeFormatExtensions.cs b/API/Extensions/EncodeFormatExtensions.cs new file mode 100644 index 000000000..bede8e721 --- /dev/null +++ b/API/Extensions/EncodeFormatExtensions.cs @@ -0,0 +1,18 @@ +using System; +using API.Entities.Enums; + +namespace API.Extensions; + +public static class EncodeFormatExtensions +{ + public static string GetExtension(this EncodeFormat encodeFormat) + { + return encodeFormat switch + { + EncodeFormat.PNG => ".png", + EncodeFormat.WEBP => ".webp", + EncodeFormat.AVIF => ".avif", + _ => throw new ArgumentOutOfRangeException(nameof(encodeFormat), encodeFormat, null) + }; + } +} diff --git a/API/Helpers/Converters/ServerSettingConverter.cs b/API/Helpers/Converters/ServerSettingConverter.cs index 86493a9e2..3afc3ec4e 100644 --- a/API/Helpers/Converters/ServerSettingConverter.cs +++ b/API/Helpers/Converters/ServerSettingConverter.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using API.DTOs.Settings; using API.Entities; using API.Entities.Enums; @@ -51,11 +52,8 @@ public class ServerSettingConverter : ITypeConverter, case ServerSettingKey.InstallVersion: destination.InstallVersion = row.Value; break; - case ServerSettingKey.ConvertBookmarkToWebP: - destination.ConvertBookmarkToWebP = bool.Parse(row.Value); - break; - case ServerSettingKey.ConvertCoverToWebP: - destination.ConvertCoverToWebP = bool.Parse(row.Value); + case ServerSettingKey.EncodeMediaAs: + destination.EncodeMediaAs = Enum.Parse(row.Value); break; case ServerSettingKey.TotalBackups: destination.TotalBackups = int.Parse(row.Value); diff --git a/API/Helpers/PersonHelper.cs b/API/Helpers/PersonHelper.cs index ff73c6a74..98d6f7acc 100644 --- a/API/Helpers/PersonHelper.cs +++ b/API/Helpers/PersonHelper.cs @@ -115,21 +115,21 @@ public static class PersonHelper /// For a given role and people dtos, update a series /// /// - /// + /// /// - /// + /// /// This will call with an existing or new tag, but the method does not update the series Metadata /// - public static void UpdatePeopleList(PersonRole role, ICollection? tags, Series series, IReadOnlyCollection allTags, + public static void UpdatePeopleList(PersonRole role, ICollection? people, Series series, IReadOnlyCollection allPeople, Action handleAdd, Action onModified) { - if (tags == null) return; + if (people == null) return; var isModified = false; // I want a union of these 2 lists. Return only elements that are in both lists, but the list types are different var existingTags = series.Metadata.People.Where(p => p.Role == role).ToList(); foreach (var existing in existingTags) { - if (tags.SingleOrDefault(t => t.Id == existing.Id) == null) // This needs to check against role + if (people.SingleOrDefault(t => t.Id == existing.Id) == null) // This needs to check against role { // Remove tag series.Metadata.People.Remove(existing); @@ -138,9 +138,9 @@ public static class PersonHelper } // At this point, all tags that aren't in dto have been removed. - foreach (var tag in tags) + foreach (var tag in people) { - var existingTag = allTags.SingleOrDefault(t => t.Name == tag.Name && t.Role == tag.Role); + var existingTag = allPeople.FirstOrDefault(t => t.Name == tag.Name && t.Role == tag.Role); if (existingTag != null) { if (series.Metadata.People.Where(t => t.Role == tag.Role).All(t => t.Name != null && !t.Name.Equals(tag.Name))) diff --git a/API/Program.cs b/API/Program.cs index 35835be92..f0ebbf23e 100644 --- a/API/Program.cs +++ b/API/Program.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Security.Cryptography; using System.Threading.Tasks; using API.Data; +using API.Data.ManualMigrations; using API.Entities; using API.Entities.Enums; using API.Logging; diff --git a/API/Services/ArchiveService.cs b/API/Services/ArchiveService.cs index ad9d42139..e05102ad3 100644 --- a/API/Services/ArchiveService.cs +++ b/API/Services/ArchiveService.cs @@ -7,6 +7,7 @@ using System.Linq; using System.Xml.Serialization; using API.Archive; using API.Data.Metadata; +using API.Entities.Enums; using API.Extensions; using API.Services.Tasks; using Kavita.Common; @@ -20,7 +21,7 @@ public interface IArchiveService { void ExtractArchive(string archivePath, string extractPath); int GetNumberOfPagesFromArchive(string archivePath); - string GetCoverImage(string archivePath, string fileName, string outputDirectory, bool saveAsWebP = false); + string GetCoverImage(string archivePath, string fileName, string outputDirectory, EncodeFormat format); bool IsValidArchive(string archivePath); ComicInfo? GetComicInfo(string archivePath); ArchiveLibrary CanOpen(string archivePath); @@ -201,9 +202,9 @@ public class ArchiveService : IArchiveService /// /// File name to use based on context of entity. /// Where to output the file, defaults to covers directory - /// When saving the file, use WebP encoding instead of PNG + /// When saving the file, use encoding /// - public string GetCoverImage(string archivePath, string fileName, string outputDirectory, bool saveAsWebP = false) + public string GetCoverImage(string archivePath, string fileName, string outputDirectory, EncodeFormat encodeFormat) { if (archivePath == null || !IsValidArchive(archivePath)) return string.Empty; try @@ -219,7 +220,7 @@ public class ArchiveService : IArchiveService var entry = archive.Entries.Single(e => e.FullName == entryName); using var stream = entry.Open(); - return _imageService.WriteCoverThumbnail(stream, fileName, outputDirectory, saveAsWebP); + return _imageService.WriteCoverThumbnail(stream, fileName, outputDirectory, encodeFormat); } case ArchiveLibrary.SharpCompress: { @@ -230,7 +231,7 @@ public class ArchiveService : IArchiveService var entry = archive.Entries.Single(e => e.Key == entryName); using var stream = entry.OpenEntryStream(); - return _imageService.WriteCoverThumbnail(stream, fileName, outputDirectory, saveAsWebP); + return _imageService.WriteCoverThumbnail(stream, fileName, outputDirectory, encodeFormat); } case ArchiveLibrary.NotSupported: _logger.LogWarning("[GetCoverImage] This archive cannot be read: {ArchivePath}. Defaulting to no cover image", archivePath); @@ -426,7 +427,7 @@ public class ArchiveService : IArchiveService { entry.WriteToDirectory(extractPath, new ExtractionOptions() { - ExtractFullPath = true, // Don't flatten, let the flatterner ensure correct order of nested folders + ExtractFullPath = true, // Don't flatten, let the flattener ensure correct order of nested folders Overwrite = false }); } diff --git a/API/Services/BookService.cs b/API/Services/BookService.cs index b36829674..0ba1f5dd3 100644 --- a/API/Services/BookService.cs +++ b/API/Services/BookService.cs @@ -34,7 +34,7 @@ namespace API.Services; public interface IBookService { int GetNumberOfPages(string filePath); - string GetCoverImage(string fileFilePath, string fileName, string outputDirectory, bool saveAsWebP = false); + string GetCoverImage(string fileFilePath, string fileName, string outputDirectory, EncodeFormat encodeFormat); ComicInfo? GetComicInfo(string filePath); ParserInfo? ParseInfo(string filePath); /// @@ -1062,15 +1062,15 @@ public class BookService : IBookService /// /// Name of the new file. /// Where to output the file, defaults to covers directory - /// When saving the file, use WebP encoding instead of PNG + /// When saving the file, use encoding /// - public string GetCoverImage(string fileFilePath, string fileName, string outputDirectory, bool saveAsWebP = false) + public string GetCoverImage(string fileFilePath, string fileName, string outputDirectory, EncodeFormat encodeFormat) { if (!IsValidFile(fileFilePath)) return string.Empty; if (Parser.IsPdf(fileFilePath)) { - return GetPdfCoverImage(fileFilePath, fileName, outputDirectory, saveAsWebP); + return GetPdfCoverImage(fileFilePath, fileName, outputDirectory, encodeFormat); } using var epubBook = EpubReader.OpenBook(fileFilePath, BookReaderOptions); @@ -1085,7 +1085,7 @@ public class BookService : IBookService if (coverImageContent == null) return string.Empty; using var stream = coverImageContent.GetContentStream(); - return _imageService.WriteCoverThumbnail(stream, fileName, outputDirectory, saveAsWebP); + return _imageService.WriteCoverThumbnail(stream, fileName, outputDirectory, encodeFormat); } catch (Exception ex) { @@ -1098,7 +1098,7 @@ public class BookService : IBookService } - private string GetPdfCoverImage(string fileFilePath, string fileName, string outputDirectory, bool saveAsWebP) + private string GetPdfCoverImage(string fileFilePath, string fileName, string outputDirectory, EncodeFormat encodeFormat) { try { @@ -1108,7 +1108,7 @@ public class BookService : IBookService using var stream = StreamManager.GetStream("BookService.GetPdfPage"); GetPdfPage(docReader, 0, stream); - return _imageService.WriteCoverThumbnail(stream, fileName, outputDirectory, saveAsWebP); + return _imageService.WriteCoverThumbnail(stream, fileName, outputDirectory, encodeFormat); } catch (Exception ex) diff --git a/API/Services/BookmarkService.cs b/API/Services/BookmarkService.cs index c3b450214..7ff7cd0ad 100644 --- a/API/Services/BookmarkService.cs +++ b/API/Services/BookmarkService.cs @@ -7,7 +7,6 @@ using API.Data; using API.DTOs.Reader; using API.Entities; using API.Entities.Enums; -using API.SignalR; using Hangfire; using Microsoft.Extensions.Logging; @@ -19,9 +18,6 @@ public interface IBookmarkService Task BookmarkPage(AppUser userWithBookmarks, BookmarkDto bookmarkDto, string imageToBookmark); Task RemoveBookmarkPage(AppUser userWithBookmarks, BookmarkDto bookmarkDto); Task> GetBookmarkFilesById(IEnumerable bookmarkIds); - [DisableConcurrentExecution(timeoutInSeconds: 2 * 60 * 60), AutomaticRetry(Attempts = 0)] - Task ConvertAllBookmarkToWebP(); - Task ConvertAllCoverToWebP(); } public class BookmarkService : IBookmarkService @@ -30,17 +26,15 @@ public class BookmarkService : IBookmarkService private readonly ILogger _logger; private readonly IUnitOfWork _unitOfWork; private readonly IDirectoryService _directoryService; - private readonly IImageService _imageService; - private readonly IEventHub _eventHub; + private readonly IMediaConversionService _mediaConversionService; public BookmarkService(ILogger logger, IUnitOfWork unitOfWork, - IDirectoryService directoryService, IImageService imageService, IEventHub eventHub) + IDirectoryService directoryService, IMediaConversionService mediaConversionService) { _logger = logger; _unitOfWork = unitOfWork; _directoryService = directoryService; - _imageService = imageService; - _eventHub = eventHub; + _mediaConversionService = mediaConversionService; } /// @@ -77,21 +71,25 @@ public class BookmarkService : IBookmarkService /// This is a job that runs after a bookmark is saved /// /// This must be public - public async Task ConvertBookmarkToWebP(int bookmarkId) + public async Task ConvertBookmarkToEncoding(int bookmarkId) { var bookmarkDirectory = (await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.BookmarkDirectory)).Value; - var convertBookmarkToWebP = - (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).ConvertBookmarkToWebP; + var encodeFormat = + (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).EncodeMediaAs; - if (!convertBookmarkToWebP) return; + if (encodeFormat == EncodeFormat.PNG) + { + _logger.LogError("Cannot convert media to PNG"); + return; + } // Validate the bookmark still exists var bookmark = await _unitOfWork.UserRepository.GetBookmarkAsync(bookmarkId); if (bookmark == null) return; - bookmark.FileName = await SaveAsWebP(bookmarkDirectory, bookmark.FileName, - BookmarkStem(bookmark.AppUserId, bookmark.SeriesId, bookmark.ChapterId)); + bookmark.FileName = await _mediaConversionService.SaveAsEncodingFormat(bookmarkDirectory, bookmark.FileName, + BookmarkStem(bookmark.AppUserId, bookmark.SeriesId, bookmark.ChapterId), encodeFormat); _unitOfWork.UserRepository.Update(bookmark); await _unitOfWork.CommitAsync(); @@ -137,10 +135,10 @@ public class BookmarkService : IBookmarkService _unitOfWork.UserRepository.Add(bookmark); await _unitOfWork.CommitAsync(); - if (settings.ConvertBookmarkToWebP) + if (settings.EncodeMediaAs == EncodeFormat.WEBP) { // Enqueue a task to convert the bookmark to webP - BackgroundJob.Enqueue(() => ConvertBookmarkToWebP(bookmark.Id)); + BackgroundJob.Enqueue(() => ConvertBookmarkToEncoding(bookmark.Id)); } } catch (Exception ex) @@ -192,198 +190,9 @@ public class BookmarkService : IBookmarkService b.FileName))); } - /// - /// This is a long-running job that will convert all bookmarks into WebP. Do not invoke anyway except via Hangfire. - /// - [DisableConcurrentExecution(timeoutInSeconds: 2 * 60 * 60), AutomaticRetry(Attempts = 0)] - public async Task ConvertAllBookmarkToWebP() - { - var bookmarkDirectory = - (await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.BookmarkDirectory)).Value; - - await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, - MessageFactory.ConvertBookmarksProgressEvent(0F, ProgressEventType.Started)); - var bookmarks = (await _unitOfWork.UserRepository.GetAllBookmarksAsync()) - .Where(b => !b.FileName.EndsWith(".webp")).ToList(); - - var count = 1F; - foreach (var bookmark in bookmarks) - { - bookmark.FileName = await SaveAsWebP(bookmarkDirectory, bookmark.FileName, - BookmarkStem(bookmark.AppUserId, bookmark.SeriesId, bookmark.ChapterId)); - _unitOfWork.UserRepository.Update(bookmark); - await _unitOfWork.CommitAsync(); - await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, - MessageFactory.ConvertBookmarksProgressEvent(count / bookmarks.Count, ProgressEventType.Started)); - count++; - } - - await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, - MessageFactory.ConvertBookmarksProgressEvent(1F, ProgressEventType.Ended)); - - _logger.LogInformation("[BookmarkService] Converted bookmarks to WebP"); - } - - /// - /// This is a long-running job that will convert all covers into WebP. Do not invoke anyway except via Hangfire. - /// - [DisableConcurrentExecution(timeoutInSeconds: 2 * 60 * 60), AutomaticRetry(Attempts = 0)] - public async Task ConvertAllCoverToWebP() - { - _logger.LogInformation("[BookmarkService] Starting conversion of all covers to webp"); - var coverDirectory = _directoryService.CoverImageDirectory; - - await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, - MessageFactory.ConvertCoverProgressEvent(0F, ProgressEventType.Started)); - var chapterCovers = await _unitOfWork.ChapterRepository.GetAllChaptersWithNonWebPCovers(); - var seriesCovers = await _unitOfWork.SeriesRepository.GetAllWithNonWebPCovers(); - - var readingListCovers = await _unitOfWork.ReadingListRepository.GetAllWithNonWebPCovers(); - var libraryCovers = await _unitOfWork.LibraryRepository.GetAllWithNonWebPCovers(); - var collectionCovers = await _unitOfWork.CollectionTagRepository.GetAllWithNonWebPCovers(); - - var totalCount = chapterCovers.Count + seriesCovers.Count + readingListCovers.Count + - libraryCovers.Count + collectionCovers.Count; - - var count = 1F; - _logger.LogInformation("[BookmarkService] Starting conversion of chapters"); - foreach (var chapter in chapterCovers) - { - if (string.IsNullOrEmpty(chapter.CoverImage)) continue; - - var newFile = await SaveAsWebP(coverDirectory, chapter.CoverImage, coverDirectory); - chapter.CoverImage = Path.GetFileName(newFile); - _unitOfWork.ChapterRepository.Update(chapter); - await _unitOfWork.CommitAsync(); - await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, - MessageFactory.ConvertCoverProgressEvent(count / totalCount, ProgressEventType.Started)); - count++; - } - - _logger.LogInformation("[BookmarkService] Starting conversion of series"); - foreach (var series in seriesCovers) - { - if (string.IsNullOrEmpty(series.CoverImage)) continue; - - var newFile = await SaveAsWebP(coverDirectory, series.CoverImage, coverDirectory); - series.CoverImage = Path.GetFileName(newFile); - _unitOfWork.SeriesRepository.Update(series); - await _unitOfWork.CommitAsync(); - await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, - MessageFactory.ConvertCoverProgressEvent(count / totalCount, ProgressEventType.Started)); - count++; - } - - _logger.LogInformation("[BookmarkService] Starting conversion of libraries"); - foreach (var library in libraryCovers) - { - if (string.IsNullOrEmpty(library.CoverImage)) continue; - - var newFile = await SaveAsWebP(coverDirectory, library.CoverImage, coverDirectory); - library.CoverImage = Path.GetFileName(newFile); - _unitOfWork.LibraryRepository.Update(library); - await _unitOfWork.CommitAsync(); - await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, - MessageFactory.ConvertCoverProgressEvent(count / totalCount, ProgressEventType.Started)); - count++; - } - - _logger.LogInformation("[BookmarkService] Starting conversion of reading lists"); - foreach (var readingList in readingListCovers) - { - if (string.IsNullOrEmpty(readingList.CoverImage)) continue; - - var newFile = await SaveAsWebP(coverDirectory, readingList.CoverImage, coverDirectory); - readingList.CoverImage = Path.GetFileName(newFile); - _unitOfWork.ReadingListRepository.Update(readingList); - await _unitOfWork.CommitAsync(); - await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, - MessageFactory.ConvertCoverProgressEvent(count / totalCount, ProgressEventType.Started)); - count++; - } - - _logger.LogInformation("[BookmarkService] Starting conversion of collections"); - foreach (var collection in collectionCovers) - { - if (string.IsNullOrEmpty(collection.CoverImage)) continue; - - var newFile = await SaveAsWebP(coverDirectory, collection.CoverImage, coverDirectory); - collection.CoverImage = Path.GetFileName(newFile); - _unitOfWork.CollectionTagRepository.Update(collection); - await _unitOfWork.CommitAsync(); - await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, - MessageFactory.ConvertCoverProgressEvent(count / totalCount, ProgressEventType.Started)); - count++; - } - - // Now null out all series and volumes that aren't webp or custom - var nonCustomOrConvertedVolumeCovers = await _unitOfWork.VolumeRepository.GetAllWithNonWebPCovers(); - foreach (var volume in nonCustomOrConvertedVolumeCovers) - { - if (string.IsNullOrEmpty(volume.CoverImage)) continue; - volume.CoverImage = null; // We null it out so when we call Refresh Metadata it will auto update from first chapter - _unitOfWork.VolumeRepository.Update(volume); - await _unitOfWork.CommitAsync(); - } - - var nonCustomOrConvertedSeriesCovers = await _unitOfWork.SeriesRepository.GetAllWithNonWebPCovers(false); - foreach (var series in nonCustomOrConvertedSeriesCovers) - { - if (string.IsNullOrEmpty(series.CoverImage)) continue; - series.CoverImage = null; // We null it out so when we call Refresh Metadata it will auto update from first chapter - _unitOfWork.SeriesRepository.Update(series); - await _unitOfWork.CommitAsync(); - } - - await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, - MessageFactory.ConvertCoverProgressEvent(1F, ProgressEventType.Ended)); - - _logger.LogInformation("[BookmarkService] Converted covers to WebP"); - } - /// - /// Converts an image file, deletes original and returns the new path back - /// - /// Full Path to where files are stored - /// The file to convert - /// Full path to where files should be stored or any stem - /// - public async Task SaveAsWebP(string imageDirectory, string filename, string targetFolder) - { - // This must be Public as it's used in via Hangfire as a background task - var fullSourcePath = _directoryService.FileSystem.Path.Join(imageDirectory, filename); - var fullTargetDirectory = fullSourcePath.Replace(new FileInfo(filename).Name, string.Empty); - - var newFilename = string.Empty; - _logger.LogDebug("Converting {Source} image into WebP at {Target}", fullSourcePath, fullTargetDirectory); - - try - { - // Convert target file to webp then delete original target file and update bookmark - - try - { - var targetFile = await _imageService.ConvertToWebP(fullSourcePath, fullTargetDirectory); - var targetName = new FileInfo(targetFile).Name; - newFilename = Path.Join(targetFolder, targetName); - _directoryService.DeleteFiles(new[] {fullSourcePath}); - } - catch (Exception ex) - { - _logger.LogError(ex, "Could not convert image {FilePath}", filename); - newFilename = filename; - } - } - catch (Exception ex) - { - _logger.LogError(ex, "Could not convert image to WebP"); - } - - return newFilename; - } - - private static string BookmarkStem(int userId, int seriesId, int chapterId) + public static string BookmarkStem(int userId, int seriesId, int chapterId) { return Path.Join($"{userId}", $"{seriesId}", $"{chapterId}"); } diff --git a/API/Services/ImageService.cs b/API/Services/ImageService.cs index 9cfdc0813..b153cd6c7 100644 --- a/API/Services/ImageService.cs +++ b/API/Services/ImageService.cs @@ -3,6 +3,8 @@ using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading.Tasks; +using API.Entities.Enums; +using API.Extensions; using Flurl; using Flurl.Http; using HtmlAgilityPack; @@ -16,49 +18,49 @@ namespace API.Services; public interface IImageService { void ExtractImages(string fileFilePath, string targetDirectory, int fileCount = 1); - string GetCoverImage(string path, string fileName, string outputDirectory, bool saveAsWebP = false); + string GetCoverImage(string path, string fileName, string outputDirectory, EncodeFormat encodeFormat); /// /// Creates a Thumbnail version of a base64 image /// /// base64 encoded image /// - /// Convert and save as webp + /// Convert and save as encoding format /// Width of thumbnail /// File name with extension of the file. This will always write to - string CreateThumbnailFromBase64(string encodedImage, string fileName, bool saveAsWebP = false, int thumbnailWidth = 320); + string CreateThumbnailFromBase64(string encodedImage, string fileName, EncodeFormat encodeFormat, int thumbnailWidth = 320); /// /// Writes out a thumbnail by stream input /// /// /// /// - /// + /// /// - string WriteCoverThumbnail(Stream stream, string fileName, string outputDirectory, bool saveAsWebP = false); + string WriteCoverThumbnail(Stream stream, string fileName, string outputDirectory, EncodeFormat encodeFormat); /// /// Writes out a thumbnail by file path input /// /// /// /// - /// + /// /// - string WriteCoverThumbnail(string sourceFile, string fileName, string outputDirectory, bool saveAsWebP = false); + string WriteCoverThumbnail(string sourceFile, string fileName, string outputDirectory, EncodeFormat encodeFormat); /// - /// Converts the passed image to webP and outputs it in the same directory + /// Converts the passed image to encoding and outputs it in the same directory /// /// Full path to the image to convert /// Where to output the file - /// File of written webp image - Task ConvertToWebP(string filePath, string outputPath); - + /// File of written encoded image + Task ConvertToEncodingFormat(string filePath, string outputPath, EncodeFormat encodeFormat); Task IsImage(string filePath); - Task DownloadFaviconAsync(string url); + Task DownloadFaviconAsync(string url, EncodeFormat encodeFormat); } public class ImageService : IImageService { + public const string Name = "BookmarkService"; private readonly ILogger _logger; private readonly IDirectoryService _directoryService; public const string ChapterCoverImageRegex = @"v\d+_c\d+"; @@ -75,6 +77,20 @@ public class ImageService : IImageService /// public const int LibraryThumbnailWidth = 32; + private static readonly string[] ValidIconRelations = { + "icon", + "apple-touch-icon", + "apple-touch-icon-precomposed" + }; + + /// + /// A mapping of urls that need to get the icon from another url, due to strangeness (like app.plex.tv loading a black icon) + /// + private static readonly IDictionary FaviconUrlMapper = new Dictionary + { + ["https://app.plex.tv"] = "https://plex.tv" + }; + public ImageService(ILogger logger, IDirectoryService directoryService) { _logger = logger; @@ -96,14 +112,14 @@ public class ImageService : IImageService } } - public string GetCoverImage(string path, string fileName, string outputDirectory, bool saveAsWebP = false) + public string GetCoverImage(string path, string fileName, string outputDirectory, EncodeFormat encodeFormat) { if (string.IsNullOrEmpty(path)) return string.Empty; try { using var thumbnail = Image.Thumbnail(path, ThumbnailWidth); - var filename = fileName + (saveAsWebP ? ".webp" : ".png"); + var filename = fileName + encodeFormat.GetExtension(); thumbnail.WriteToFile(_directoryService.FileSystem.Path.Join(outputDirectory, filename)); return filename; } @@ -122,12 +138,12 @@ public class ImageService : IImageService /// Stream to write to disk. Ensure this is rewinded. /// filename to save as without extension /// Where to output the file, defaults to covers directory - /// Export the file as webP otherwise will default to png + /// Export the file as the passed encoding /// File name with extension of the file. This will always write to - public string WriteCoverThumbnail(Stream stream, string fileName, string outputDirectory, bool saveAsWebP = false) + public string WriteCoverThumbnail(Stream stream, string fileName, string outputDirectory, EncodeFormat encodeFormat) { using var thumbnail = Image.ThumbnailStream(stream, ThumbnailWidth); - var filename = fileName + (saveAsWebP ? ".webp" : ".png"); + var filename = fileName + encodeFormat.GetExtension(); _directoryService.ExistOrCreate(outputDirectory); try { @@ -137,10 +153,10 @@ public class ImageService : IImageService return filename; } - public string WriteCoverThumbnail(string sourceFile, string fileName, string outputDirectory, bool saveAsWebP = false) + public string WriteCoverThumbnail(string sourceFile, string fileName, string outputDirectory, EncodeFormat encodeFormat) { using var thumbnail = Image.Thumbnail(sourceFile, ThumbnailWidth); - var filename = fileName + (saveAsWebP ? ".webp" : ".png"); + var filename = fileName + encodeFormat.GetExtension(); _directoryService.ExistOrCreate(outputDirectory); try { @@ -150,11 +166,11 @@ public class ImageService : IImageService return filename; } - public Task ConvertToWebP(string filePath, string outputPath) + public Task ConvertToEncodingFormat(string filePath, string outputPath, EncodeFormat encodeFormat) { var file = _directoryService.FileSystem.FileInfo.New(filePath); var fileName = file.Name.Replace(file.Extension, string.Empty); - var outputFile = Path.Join(outputPath, fileName + ".webp"); + var outputFile = Path.Join(outputPath, fileName + encodeFormat.GetExtension()); using var sourceImage = Image.NewFromFile(filePath, false, Enums.Access.SequentialUnbuffered); sourceImage.WriteToFile(outputFile); @@ -183,24 +199,26 @@ public class ImageService : IImageService return false; } - public async Task DownloadFaviconAsync(string url) + public async Task DownloadFaviconAsync(string url, EncodeFormat encodeFormat) { // Parse the URL to get the domain (including subdomain) var uri = new Uri(url); var domain = uri.Host; var baseUrl = uri.Scheme + "://" + uri.Host; + + + if (FaviconUrlMapper.TryGetValue(baseUrl, out var value)) + { + url = value; + } + try { - var validIconRelations = new[] - { - "icon", - "apple-touch-icon", - }; var htmlContent = url.GetStringAsync().Result; var htmlDocument = new HtmlDocument(); htmlDocument.LoadHtml(htmlContent); var pngLinks = htmlDocument.DocumentNode.Descendants("link") - .Where(link => validIconRelations.Contains(link.GetAttributeValue("rel", string.Empty))) + .Where(link => ValidIconRelations.Contains(link.GetAttributeValue("rel", string.Empty))) .Select(link => link.GetAttributeValue("href", string.Empty)) .Where(href => href.EndsWith(".png") || href.EndsWith(".PNG")) .ToList(); @@ -228,9 +246,23 @@ public class ImageService : IImageService .GetStreamAsync(); // Create the destination file path - var filename = $"{domain}.png"; using var image = Image.PngloadStream(faviconStream); - image.Pngsave(Path.Combine(_directoryService.FaviconDirectory, filename)); + var filename = $"{domain}{encodeFormat.GetExtension()}"; + switch (encodeFormat) + { + case EncodeFormat.PNG: + image.Pngsave(Path.Combine(_directoryService.FaviconDirectory, filename)); + break; + case EncodeFormat.WEBP: + image.Webpsave(Path.Combine(_directoryService.FaviconDirectory, filename)); + break; + case EncodeFormat.AVIF: + image.Heifsave(Path.Combine(_directoryService.FaviconDirectory, filename)); + break; + default: + throw new ArgumentOutOfRangeException(nameof(encodeFormat), encodeFormat, null); + } + _logger.LogDebug("Favicon.png for {Domain} downloaded and saved successfully", domain); return filename; @@ -242,14 +274,13 @@ public class ImageService : IImageService } } - /// - public string CreateThumbnailFromBase64(string encodedImage, string fileName, bool saveAsWebP = false, int thumbnailWidth = ThumbnailWidth) + public string CreateThumbnailFromBase64(string encodedImage, string fileName, EncodeFormat encodeFormat, int thumbnailWidth = ThumbnailWidth) { try { using var thumbnail = Image.ThumbnailBuffer(Convert.FromBase64String(encodedImage), thumbnailWidth); - fileName += (saveAsWebP ? ".webp" : ".png"); + fileName += encodeFormat.GetExtension(); thumbnail.WriteToFile(_directoryService.FileSystem.Path.Join(_directoryService.CoverImageDirectory, fileName)); return fileName; } @@ -309,6 +340,7 @@ public class ImageService : IImageService /// public static string GetReadingListFormat(int readingListId) { + // ReSharper disable once StringLiteralTypo return $"readinglist{readingListId}"; } @@ -322,9 +354,9 @@ public class ImageService : IImageService return $"thumbnail{chapterId}"; } - public static string GetWebLinkFormat(string url) + public static string GetWebLinkFormat(string url, EncodeFormat encodeFormat) { - return $"{new Uri(url).Host}.png"; + return $"{new Uri(url).Host}{encodeFormat.GetExtension()}"; } diff --git a/API/Services/MediaConversionService.cs b/API/Services/MediaConversionService.cs new file mode 100644 index 000000000..26d88765b --- /dev/null +++ b/API/Services/MediaConversionService.cs @@ -0,0 +1,312 @@ +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using API.Data; +using API.Entities.Enums; +using API.Extensions; +using API.SignalR; +using Hangfire; +using Microsoft.Extensions.Logging; + +namespace API.Services; + +public interface IMediaConversionService +{ + [DisableConcurrentExecution(timeoutInSeconds: 2 * 60 * 60), AutomaticRetry(Attempts = 0)] + Task ConvertAllBookmarkToEncoding(); + [DisableConcurrentExecution(timeoutInSeconds: 2 * 60 * 60), AutomaticRetry(Attempts = 0)] + Task ConvertAllCoversToEncoding(); + [DisableConcurrentExecution(timeoutInSeconds: 2 * 60 * 60), AutomaticRetry(Attempts = 0)] + Task ConvertAllManagedMediaToEncodingFormat(); + + Task SaveAsEncodingFormat(string imageDirectory, string filename, string targetFolder, + EncodeFormat encodeFormat); +} + +public class MediaConversionService : IMediaConversionService +{ + public const string Name = "MediaConversionService"; + public static readonly string[] ConversionMethods = {"ConvertAllBookmarkToEncoding", "ConvertAllCoversToEncoding", "ConvertAllManagedMediaToEncodingFormat"}; + private readonly IUnitOfWork _unitOfWork; + private readonly IImageService _imageService; + private readonly IEventHub _eventHub; + private readonly IDirectoryService _directoryService; + private readonly ILogger _logger; + + public MediaConversionService(IUnitOfWork unitOfWork, IImageService imageService, IEventHub eventHub, + IDirectoryService directoryService, ILogger logger) + { + _unitOfWork = unitOfWork; + _imageService = imageService; + _eventHub = eventHub; + _directoryService = directoryService; + _logger = logger; + } + + /// + /// Converts all Kavita managed media (bookmarks, covers, favicons, etc) to the saved target encoding. + /// Do not invoke anyway except via Hangfire. + /// + /// This is a long-running job + /// + [DisableConcurrentExecution(timeoutInSeconds: 2 * 60 * 60), AutomaticRetry(Attempts = 0)] + public async Task ConvertAllManagedMediaToEncodingFormat() + { + await ConvertAllBookmarkToEncoding(); + await ConvertAllCoversToEncoding(); + await CoverAllFaviconsToEncoding(); + + } + + /// + /// This is a long-running job that will convert all bookmarks into a format that is not PNG. Do not invoke anyway except via Hangfire. + /// + [DisableConcurrentExecution(timeoutInSeconds: 2 * 60 * 60), AutomaticRetry(Attempts = 0)] + public async Task ConvertAllBookmarkToEncoding() + { + var bookmarkDirectory = + (await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.BookmarkDirectory)).Value; + var encodeFormat = + (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).EncodeMediaAs; + + if (encodeFormat == EncodeFormat.PNG) + { + _logger.LogError("Cannot convert media to PNG"); + return; + } + + await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, + MessageFactory.ConvertBookmarksProgressEvent(0F, ProgressEventType.Started)); + var bookmarks = (await _unitOfWork.UserRepository.GetAllBookmarksAsync()) + .Where(b => !b.FileName.EndsWith(encodeFormat.GetExtension())).ToList(); + + var count = 1F; + foreach (var bookmark in bookmarks) + { + bookmark.FileName = await SaveAsEncodingFormat(bookmarkDirectory, bookmark.FileName, + BookmarkService.BookmarkStem(bookmark.AppUserId, bookmark.SeriesId, bookmark.ChapterId), encodeFormat); + _unitOfWork.UserRepository.Update(bookmark); + await _unitOfWork.CommitAsync(); + await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, + MessageFactory.ConvertBookmarksProgressEvent(count / bookmarks.Count, ProgressEventType.Updated)); + count++; + } + + await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, + MessageFactory.ConvertBookmarksProgressEvent(1F, ProgressEventType.Ended)); + + _logger.LogInformation("[MediaConversionService] Converted bookmarks to {Format}", encodeFormat); + } + + /// + /// This is a long-running job that will convert all covers into WebP. Do not invoke anyway except via Hangfire. + /// + [DisableConcurrentExecution(timeoutInSeconds: 2 * 60 * 60), AutomaticRetry(Attempts = 0)] + public async Task ConvertAllCoversToEncoding() + { + var coverDirectory = _directoryService.CoverImageDirectory; + var encodeFormat = + (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).EncodeMediaAs; + + if (encodeFormat == EncodeFormat.PNG) + { + _logger.LogError("Cannot convert media to PNG"); + return; + } + + _logger.LogInformation("[MediaConversionService] Starting conversion of all covers to {Format}", encodeFormat); + await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, + MessageFactory.ConvertCoverProgressEvent(0F, ProgressEventType.Started)); + + var chapterCovers = await _unitOfWork.ChapterRepository.GetAllChaptersWithCoversInDifferentEncoding(encodeFormat); + var seriesCovers = await _unitOfWork.SeriesRepository.GetAllWithCoversInDifferentEncoding(encodeFormat); + + var readingListCovers = await _unitOfWork.ReadingListRepository.GetAllWithCoversInDifferentEncoding(encodeFormat); + var libraryCovers = await _unitOfWork.LibraryRepository.GetAllWithCoversInDifferentEncoding(encodeFormat); + var collectionCovers = await _unitOfWork.CollectionTagRepository.GetAllWithCoversInDifferentEncoding(encodeFormat); + + var totalCount = chapterCovers.Count + seriesCovers.Count + readingListCovers.Count + + libraryCovers.Count + collectionCovers.Count; + + var count = 1F; + _logger.LogInformation("[MediaConversionService] Starting conversion of chapters"); + await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, + MessageFactory.ConvertCoverProgressEvent(0, ProgressEventType.Started)); + foreach (var chapter in chapterCovers) + { + if (string.IsNullOrEmpty(chapter.CoverImage)) continue; + + var newFile = await SaveAsEncodingFormat(coverDirectory, chapter.CoverImage, coverDirectory, encodeFormat); + chapter.CoverImage = Path.GetFileName(newFile); + _unitOfWork.ChapterRepository.Update(chapter); + await _unitOfWork.CommitAsync(); + await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, + MessageFactory.ConvertCoverProgressEvent(count / totalCount, ProgressEventType.Updated)); + count++; + } + + _logger.LogInformation("[MediaConversionService] Starting conversion of series"); + foreach (var series in seriesCovers) + { + if (string.IsNullOrEmpty(series.CoverImage)) continue; + + var newFile = await SaveAsEncodingFormat(coverDirectory, series.CoverImage, coverDirectory, encodeFormat); + series.CoverImage = Path.GetFileName(newFile); + _unitOfWork.SeriesRepository.Update(series); + await _unitOfWork.CommitAsync(); + await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, + MessageFactory.ConvertCoverProgressEvent(count / totalCount, ProgressEventType.Updated)); + count++; + } + + _logger.LogInformation("[MediaConversionService] Starting conversion of libraries"); + foreach (var library in libraryCovers) + { + if (string.IsNullOrEmpty(library.CoverImage)) continue; + + var newFile = await SaveAsEncodingFormat(coverDirectory, library.CoverImage, coverDirectory, encodeFormat); + library.CoverImage = Path.GetFileName(newFile); + _unitOfWork.LibraryRepository.Update(library); + await _unitOfWork.CommitAsync(); + await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, + MessageFactory.ConvertCoverProgressEvent(count / totalCount, ProgressEventType.Updated)); + count++; + } + + _logger.LogInformation("[MediaConversionService] Starting conversion of reading lists"); + foreach (var readingList in readingListCovers) + { + if (string.IsNullOrEmpty(readingList.CoverImage)) continue; + + var newFile = await SaveAsEncodingFormat(coverDirectory, readingList.CoverImage, coverDirectory, encodeFormat); + readingList.CoverImage = Path.GetFileName(newFile); + _unitOfWork.ReadingListRepository.Update(readingList); + await _unitOfWork.CommitAsync(); + await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, + MessageFactory.ConvertCoverProgressEvent(count / totalCount, ProgressEventType.Updated)); + count++; + } + + _logger.LogInformation("[MediaConversionService] Starting conversion of collections"); + foreach (var collection in collectionCovers) + { + if (string.IsNullOrEmpty(collection.CoverImage)) continue; + + var newFile = await SaveAsEncodingFormat(coverDirectory, collection.CoverImage, coverDirectory, encodeFormat); + collection.CoverImage = Path.GetFileName(newFile); + _unitOfWork.CollectionTagRepository.Update(collection); + await _unitOfWork.CommitAsync(); + await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, + MessageFactory.ConvertCoverProgressEvent(count / totalCount, ProgressEventType.Updated)); + count++; + } + + // Now null out all series and volumes that aren't webp or custom + var nonCustomOrConvertedVolumeCovers = await _unitOfWork.VolumeRepository.GetAllWithCoversInDifferentEncoding(encodeFormat); + foreach (var volume in nonCustomOrConvertedVolumeCovers) + { + if (string.IsNullOrEmpty(volume.CoverImage)) continue; + volume.CoverImage = null; // We null it out so when we call Refresh Metadata it will auto update from first chapter + _unitOfWork.VolumeRepository.Update(volume); + await _unitOfWork.CommitAsync(); + } + + var nonCustomOrConvertedSeriesCovers = await _unitOfWork.SeriesRepository.GetAllWithCoversInDifferentEncoding(encodeFormat, false); + foreach (var series in nonCustomOrConvertedSeriesCovers) + { + if (string.IsNullOrEmpty(series.CoverImage)) continue; + series.CoverImage = null; // We null it out so when we call Refresh Metadata it will auto update from first chapter + _unitOfWork.SeriesRepository.Update(series); + await _unitOfWork.CommitAsync(); + } + + await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, + MessageFactory.ConvertCoverProgressEvent(1F, ProgressEventType.Ended)); + + _logger.LogInformation("[MediaConversionService] Converted covers to {Format}", encodeFormat); + } + + private async Task CoverAllFaviconsToEncoding() + { + var encodeFormat = + (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).EncodeMediaAs; + + if (encodeFormat == EncodeFormat.PNG) + { + _logger.LogError("Cannot convert media to PNG"); + return; + } + + _logger.LogInformation("[MediaConversionService] Starting conversion of favicons to {Format}", encodeFormat); + await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, + MessageFactory.ConvertBookmarksProgressEvent(0F, ProgressEventType.Started)); + var pngFavicons = _directoryService.GetFiles(_directoryService.FaviconDirectory) + .Where(b => !b.EndsWith(encodeFormat.GetExtension())). + ToList(); + + var count = 1F; + foreach (var file in pngFavicons) + { + await SaveAsEncodingFormat(_directoryService.FaviconDirectory, _directoryService.FileSystem.FileInfo.New(file).Name, _directoryService.FaviconDirectory, + encodeFormat); + await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, + MessageFactory.ConvertBookmarksProgressEvent(count / pngFavicons.Count, ProgressEventType.Updated)); + count++; + } + + + await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, + MessageFactory.ConvertBookmarksProgressEvent(1F, ProgressEventType.Ended)); + + _logger.LogInformation("[MediaConversionService] Converted favicons to {Format}", encodeFormat); + } + + + /// + /// Converts an image file, deletes original and returns the new path back + /// + /// Full Path to where files are stored + /// The file to convert + /// Full path to where files should be stored or any stem + /// + public async Task SaveAsEncodingFormat(string imageDirectory, string filename, string targetFolder, EncodeFormat encodeFormat) + { + // This must be Public as it's used in via Hangfire as a background task + var fullSourcePath = _directoryService.FileSystem.Path.Join(imageDirectory, filename); + var fullTargetDirectory = fullSourcePath.Replace(new FileInfo(filename).Name, string.Empty); + + var newFilename = string.Empty; + _logger.LogDebug("Converting {Source} image into {Encoding} at {Target}", fullSourcePath, encodeFormat, fullTargetDirectory); + + if (!File.Exists(fullSourcePath)) + { + _logger.LogError("Requested to convert {File} but it doesn't exist", fullSourcePath); + return newFilename; + } + + try + { + // Convert target file to format then delete original target file + try + { + var targetFile = await _imageService.ConvertToEncodingFormat(fullSourcePath, fullTargetDirectory, encodeFormat); + var targetName = new FileInfo(targetFile).Name; + newFilename = Path.Join(targetFolder, targetName); + _directoryService.DeleteFiles(new[] {fullSourcePath}); + } + catch (Exception ex) + { + _logger.LogError(ex, "Could not convert image {FilePath} to {Format}", filename, encodeFormat); + newFilename = filename; + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Could not convert image to {Format}", encodeFormat); + } + + return newFilename; + } + +} diff --git a/API/Services/MetadataService.cs b/API/Services/MetadataService.cs index 369d555f3..ff5a18df2 100644 --- a/API/Services/MetadataService.cs +++ b/API/Services/MetadataService.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using API.Comparators; using API.Data; using API.Entities; +using API.Entities.Enums; using API.Extensions; using API.Helpers; using API.SignalR; @@ -32,7 +33,7 @@ public interface IMetadataService /// Overrides any cache logic and forces execution Task GenerateCoversForSeries(int libraryId, int seriesId, bool forceUpdate = true); - Task GenerateCoversForSeries(Series series, bool convertToWebP, bool forceUpdate = false); + Task GenerateCoversForSeries(Series series, EncodeFormat encodeFormat, bool forceUpdate = false); Task RemoveAbandonedMetadataKeys(); } @@ -63,8 +64,8 @@ public class MetadataService : IMetadataService /// /// /// Force updating cover image even if underlying file has not been modified or chapter already has a cover image - /// Convert image to WebP when extracting the cover - private Task UpdateChapterCoverImage(Chapter chapter, bool forceUpdate, bool convertToWebPOnWrite) + /// Convert image to Encoding Format when extracting the cover + private Task UpdateChapterCoverImage(Chapter chapter, bool forceUpdate, EncodeFormat encodeFormat) { var firstFile = chapter.Files.MinBy(x => x.Chapter); if (firstFile == null) return Task.FromResult(false); @@ -78,7 +79,7 @@ public class MetadataService : IMetadataService _logger.LogDebug("[MetadataService] Generating cover image for {File}", firstFile.FilePath); chapter.CoverImage = _readingItemService.GetCoverImage(firstFile.FilePath, - ImageService.GetChapterFormat(chapter.Id, chapter.VolumeId), firstFile.Format, convertToWebPOnWrite); + ImageService.GetChapterFormat(chapter.Id, chapter.VolumeId), firstFile.Format, encodeFormat); _unitOfWork.ChapterRepository.Update(chapter); _updateEvents.Add(MessageFactory.CoverUpdateEvent(chapter.Id, MessageFactoryEntityTypes.Chapter)); return Task.FromResult(true); @@ -141,8 +142,8 @@ public class MetadataService : IMetadataService /// /// /// - /// - private async Task ProcessSeriesCoverGen(Series series, bool forceUpdate, bool convertToWebP) + /// + private async Task ProcessSeriesCoverGen(Series series, bool forceUpdate, EncodeFormat encodeFormat) { _logger.LogDebug("[MetadataService] Processing cover image generation for series: {SeriesName}", series.OriginalName); try @@ -155,7 +156,7 @@ public class MetadataService : IMetadataService var index = 0; foreach (var chapter in volume.Chapters) { - var chapterUpdated = await UpdateChapterCoverImage(chapter, forceUpdate, convertToWebP); + var chapterUpdated = await UpdateChapterCoverImage(chapter, forceUpdate, encodeFormat); // If cover was update, either the file has changed or first scan and we should force a metadata update UpdateChapterLastModified(chapter, forceUpdate || chapterUpdated); if (index == 0 && chapterUpdated) @@ -207,7 +208,7 @@ public class MetadataService : IMetadataService await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.CoverUpdateProgressEvent(library.Id, 0F, ProgressEventType.Started, $"Starting {library.Name}")); - var convertToWebP = (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).ConvertCoverToWebP; + var encodeFormat = (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).EncodeMediaAs; for (var chunk = 1; chunk <= chunkInfo.TotalChunks; chunk++) { @@ -237,7 +238,7 @@ public class MetadataService : IMetadataService try { - await ProcessSeriesCoverGen(series, forceUpdate, convertToWebP); + await ProcessSeriesCoverGen(series, forceUpdate, encodeFormat); } catch (Exception ex) { @@ -287,23 +288,23 @@ public class MetadataService : IMetadataService return; } - var convertToWebP = (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).ConvertCoverToWebP; - await GenerateCoversForSeries(series, convertToWebP, forceUpdate); + var encodeFormat = (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).EncodeMediaAs; + await GenerateCoversForSeries(series, encodeFormat, forceUpdate); } /// /// Generate Cover for a Series. This is used by Scan Loop and should not be invoked directly via User Interaction. /// /// A full Series, with metadata, chapters, etc - /// When saving the file, use WebP encoding instead of PNG + /// When saving the file, what encoding should be used /// - public async Task GenerateCoversForSeries(Series series, bool convertToWebP, bool forceUpdate = false) + public async Task GenerateCoversForSeries(Series series, EncodeFormat encodeFormat, bool forceUpdate = false) { var sw = Stopwatch.StartNew(); await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.CoverUpdateProgressEvent(series.LibraryId, 0F, ProgressEventType.Started, series.Name)); - await ProcessSeriesCoverGen(series, forceUpdate, convertToWebP); + await ProcessSeriesCoverGen(series, forceUpdate, encodeFormat); if (_unitOfWork.HasChanges()) diff --git a/API/Services/ReaderService.cs b/API/Services/ReaderService.cs index d044575f8..657431bf4 100644 --- a/API/Services/ReaderService.cs +++ b/API/Services/ReaderService.cs @@ -236,7 +236,6 @@ public class ReaderService : IReaderService try { - // TODO: Rewrite this code to just pull user object with progress for that particular appuserprogress, else create it var userProgress = await _unitOfWork.AppUserProgressRepository.GetUserProgressAsync(progressDto.ChapterId, userId); @@ -667,15 +666,15 @@ public class ReaderService : IReaderService _directoryService.FileSystem.Path.Join(_directoryService.TempDirectory, ImageService.GetThumbnailFormat(chapter.Id)); try { - var saveAsWebp = - (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).ConvertBookmarkToWebP; + var encodeFormat = + (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).EncodeMediaAs; if (!Directory.Exists(outputDirectory)) { var outputtedThumbnails = cachedImages .Select((img, idx) => _directoryService.FileSystem.Path.Join(outputDirectory, - _imageService.WriteCoverThumbnail(img, $"{idx}", outputDirectory, saveAsWebp))) + _imageService.WriteCoverThumbnail(img, $"{idx}", outputDirectory, encodeFormat))) .ToArray(); return CacheService.GetPageFromFiles(outputtedThumbnails, pageNum); } diff --git a/API/Services/ReadingItemService.cs b/API/Services/ReadingItemService.cs index 7ef4e63ae..8f3ab67bc 100644 --- a/API/Services/ReadingItemService.cs +++ b/API/Services/ReadingItemService.cs @@ -9,7 +9,7 @@ public interface IReadingItemService { ComicInfo? GetComicInfo(string filePath); int GetNumberOfPages(string filePath, MangaFormat format); - string GetCoverImage(string filePath, string fileName, MangaFormat format, bool saveAsWebP); + string GetCoverImage(string filePath, string fileName, MangaFormat format, EncodeFormat encodeFormat); void Extract(string fileFilePath, string targetDirectory, MangaFormat format, int imageCount = 1); ParserInfo? ParseFile(string path, string rootPath, LibraryType type); } @@ -161,7 +161,7 @@ public class ReadingItemService : IReadingItemService } } - public string GetCoverImage(string filePath, string fileName, MangaFormat format, bool saveAsWebP) + public string GetCoverImage(string filePath, string fileName, MangaFormat format, EncodeFormat encodeFormat) { if (string.IsNullOrEmpty(filePath) || string.IsNullOrEmpty(fileName)) { @@ -171,10 +171,10 @@ public class ReadingItemService : IReadingItemService return format switch { - MangaFormat.Epub => _bookService.GetCoverImage(filePath, fileName, _directoryService.CoverImageDirectory, saveAsWebP), - MangaFormat.Archive => _archiveService.GetCoverImage(filePath, fileName, _directoryService.CoverImageDirectory, saveAsWebP), - MangaFormat.Image => _imageService.GetCoverImage(filePath, fileName, _directoryService.CoverImageDirectory, saveAsWebP), - MangaFormat.Pdf => _bookService.GetCoverImage(filePath, fileName, _directoryService.CoverImageDirectory, saveAsWebP), + MangaFormat.Epub => _bookService.GetCoverImage(filePath, fileName, _directoryService.CoverImageDirectory, encodeFormat), + MangaFormat.Archive => _archiveService.GetCoverImage(filePath, fileName, _directoryService.CoverImageDirectory, encodeFormat), + MangaFormat.Image => _imageService.GetCoverImage(filePath, fileName, _directoryService.CoverImageDirectory, encodeFormat), + MangaFormat.Pdf => _bookService.GetCoverImage(filePath, fileName, _directoryService.CoverImageDirectory, encodeFormat), _ => string.Empty }; } diff --git a/API/Services/ReadingListService.cs b/API/Services/ReadingListService.cs index 76202d63b..17a726767 100644 --- a/API/Services/ReadingListService.cs +++ b/API/Services/ReadingListService.cs @@ -336,7 +336,7 @@ public class ReadingListService : IReadingListService // .Select(c => _directoryService.FileSystem.Path.Join(_directoryService.CoverImageDirectory, c)).ToList(); // // var combinedFile = ImageService.CreateMergedImage(fullImages, _directoryService.FileSystem.Path.Join(_directoryService.TempDirectory, $"{readingListId}.png")); - // // webp needs to be handled + // // webp/avif needs to be handled // return combinedFile; } diff --git a/API/Services/TaskScheduler.cs b/API/Services/TaskScheduler.cs index dfe9c4cac..2f6905ef4 100644 --- a/API/Services/TaskScheduler.cs +++ b/API/Services/TaskScheduler.cs @@ -31,7 +31,7 @@ public interface ITaskScheduler void CancelStatsTasks(); Task RunStatCollection(); void ScanSiteThemes(); - Task CovertAllCoversToWebP(); + Task CovertAllCoversToEncoding(); Task CleanupDbEntries(); } @@ -50,9 +50,9 @@ public class TaskScheduler : ITaskScheduler private readonly IThemeService _themeService; private readonly IWordCountAnalyzerService _wordCountAnalyzerService; private readonly IStatisticService _statisticService; - private readonly IBookmarkService _bookmarkService; + private readonly IMediaConversionService _mediaConversionService; - public static BackgroundJobServer Client => new BackgroundJobServer(); + public static BackgroundJobServer Client => new (); public const string ScanQueue = "scan"; public const string DefaultQueue = "default"; public const string RemoveFromWantToReadTaskId = "remove-from-want-to-read"; @@ -68,12 +68,17 @@ public class TaskScheduler : ITaskScheduler private static readonly Random Rnd = new Random(); + private static readonly RecurringJobOptions RecurringJobOptions = new RecurringJobOptions() + { + TimeZone = TimeZoneInfo.Local + }; + public TaskScheduler(ICacheService cacheService, ILogger logger, IScannerService scannerService, IUnitOfWork unitOfWork, IMetadataService metadataService, IBackupService backupService, ICleanupService cleanupService, IStatsService statsService, IVersionUpdaterService versionUpdaterService, IThemeService themeService, IWordCountAnalyzerService wordCountAnalyzerService, IStatisticService statisticService, - IBookmarkService bookmarkService) + IMediaConversionService mediaConversionService) { _cacheService = cacheService; _logger = logger; @@ -87,7 +92,7 @@ public class TaskScheduler : ITaskScheduler _themeService = themeService; _wordCountAnalyzerService = wordCountAnalyzerService; _statisticService = statisticService; - _bookmarkService = bookmarkService; + _mediaConversionService = mediaConversionService; } public async Task ScheduleTasks() @@ -100,28 +105,28 @@ public class TaskScheduler : ITaskScheduler var scanLibrarySetting = setting; _logger.LogDebug("Scheduling Scan Library Task for {Setting}", scanLibrarySetting); RecurringJob.AddOrUpdate(ScanLibrariesTaskId, () => _scannerService.ScanLibraries(false), - () => CronConverter.ConvertToCronNotation(scanLibrarySetting), TimeZoneInfo.Local); + () => CronConverter.ConvertToCronNotation(scanLibrarySetting), RecurringJobOptions); } else { - RecurringJob.AddOrUpdate(ScanLibrariesTaskId, () => ScanLibraries(false), Cron.Daily, TimeZoneInfo.Local); + RecurringJob.AddOrUpdate(ScanLibrariesTaskId, () => ScanLibraries(false), Cron.Daily, RecurringJobOptions); } setting = (await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.TaskBackup)).Value; if (setting != null) { _logger.LogDebug("Scheduling Backup Task for {Setting}", setting); - RecurringJob.AddOrUpdate(BackupTaskId, () => _backupService.BackupDatabase(), () => CronConverter.ConvertToCronNotation(setting), TimeZoneInfo.Local); + RecurringJob.AddOrUpdate(BackupTaskId, () => _backupService.BackupDatabase(), () => CronConverter.ConvertToCronNotation(setting), RecurringJobOptions); } else { - RecurringJob.AddOrUpdate(BackupTaskId, () => _backupService.BackupDatabase(), Cron.Weekly, TimeZoneInfo.Local); + RecurringJob.AddOrUpdate(BackupTaskId, () => _backupService.BackupDatabase(), Cron.Weekly, RecurringJobOptions); } - RecurringJob.AddOrUpdate(CleanupTaskId, () => _cleanupService.Cleanup(), Cron.Daily, TimeZoneInfo.Local); - RecurringJob.AddOrUpdate(CleanupDbTaskId, () => _cleanupService.CleanupDbEntries(), Cron.Daily, TimeZoneInfo.Local); - RecurringJob.AddOrUpdate(RemoveFromWantToReadTaskId, () => _cleanupService.CleanupWantToRead(), Cron.Daily, TimeZoneInfo.Local); - RecurringJob.AddOrUpdate(UpdateYearlyStatsTaskId, () => _statisticService.UpdateServerStatistics(), Cron.Monthly, TimeZoneInfo.Local); + RecurringJob.AddOrUpdate(CleanupTaskId, () => _cleanupService.Cleanup(), Cron.Daily, RecurringJobOptions); + RecurringJob.AddOrUpdate(CleanupDbTaskId, () => _cleanupService.CleanupDbEntries(), Cron.Daily, RecurringJobOptions); + RecurringJob.AddOrUpdate(RemoveFromWantToReadTaskId, () => _cleanupService.CleanupWantToRead(), Cron.Daily, RecurringJobOptions); + RecurringJob.AddOrUpdate(UpdateYearlyStatsTaskId, () => _statisticService.UpdateServerStatistics(), Cron.Monthly, RecurringJobOptions); } #region StatsTasks @@ -137,7 +142,7 @@ public class TaskScheduler : ITaskScheduler } _logger.LogDebug("Scheduling stat collection daily"); - RecurringJob.AddOrUpdate(ReportStatsTaskId, () => _statsService.Send(), Cron.Daily(Rnd.Next(0, 22)), TimeZoneInfo.Local); + RecurringJob.AddOrUpdate(ReportStatsTaskId, () => _statsService.Send(), Cron.Daily(Rnd.Next(0, 22)), RecurringJobOptions); } public void AnalyzeFilesForLibrary(int libraryId, bool forceUpdate = false) @@ -182,10 +187,20 @@ public class TaskScheduler : ITaskScheduler BackgroundJob.Enqueue(() => _themeService.Scan()); } - public async Task CovertAllCoversToWebP() + /// + /// Do not invoke this manually, always enqueue on a background thread + /// + public async Task CovertAllCoversToEncoding() { - await _bookmarkService.ConvertAllCoverToWebP(); - _logger.LogInformation("[BookmarkService] Queuing tasks to update Series and Volume references via Cover Refresh"); + var defaultParams = Array.Empty(); + if (MediaConversionService.ConversionMethods.Any(method => + HasAlreadyEnqueuedTask(MediaConversionService.Name, method, defaultParams, DefaultQueue, true))) + { + return; + } + + await _mediaConversionService.ConvertAllManagedMediaToEncodingFormat(); + _logger.LogInformation("Queuing tasks to update Series and Volume references via Cover Refresh"); var libraryIds = await _unitOfWork.LibraryRepository.GetLibrariesAsync(); foreach (var lib in libraryIds) { @@ -200,8 +215,10 @@ public class TaskScheduler : ITaskScheduler public void ScheduleUpdaterTasks() { _logger.LogInformation("Scheduling Auto-Update tasks"); - // Schedule update check between noon and 6pm local time - RecurringJob.AddOrUpdate("check-updates", () => CheckForUpdate(), Cron.Daily(Rnd.Next(12, 18)), TimeZoneInfo.Local); + RecurringJob.AddOrUpdate("check-updates", () => CheckForUpdate(), Cron.Daily(Rnd.Next(5, 23)), new RecurringJobOptions() + { + TimeZone = TimeZoneInfo.Local + }); } public void ScanFolder(string folderPath, TimeSpan delay) diff --git a/API/Services/Tasks/CleanupService.cs b/API/Services/Tasks/CleanupService.cs index b51df5b44..257103708 100644 --- a/API/Services/Tasks/CleanupService.cs +++ b/API/Services/Tasks/CleanupService.cs @@ -58,14 +58,14 @@ public class CleanupService : ICleanupService [AutomaticRetry(Attempts = 3, LogEvents = false, OnAttemptsExceeded = AttemptsExceededAction.Fail)] public async Task Cleanup() { - if (TaskScheduler.HasAlreadyEnqueuedTask(BookmarkService.Name, "ConvertAllCoverToWebP", Array.Empty(), + if (TaskScheduler.HasAlreadyEnqueuedTask(BookmarkService.Name, "ConvertAllCoverToEncoding", Array.Empty(), TaskScheduler.DefaultQueue, true) || - TaskScheduler.HasAlreadyEnqueuedTask(BookmarkService.Name, "ConvertAllBookmarkToWebP", Array.Empty(), + TaskScheduler.HasAlreadyEnqueuedTask(BookmarkService.Name, "ConvertAllBookmarkToEncoding", Array.Empty(), TaskScheduler.DefaultQueue, true)) { - _logger.LogInformation("Cleanup put on hold as a conversion to WebP in progress"); + _logger.LogInformation("Cleanup put on hold as a media conversion in progress"); await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, - MessageFactory.ErrorEvent("Cleanup", "Cleanup put on hold as a conversion to WebP in progress")); + MessageFactory.ErrorEvent("Cleanup", "Cleanup put on hold as a media conversion in progress")); return; } diff --git a/API/Services/Tasks/Scanner/Parser/Parser.cs b/API/Services/Tasks/Scanner/Parser/Parser.cs index 6777e052b..19bff0fe9 100644 --- a/API/Services/Tasks/Scanner/Parser/Parser.cs +++ b/API/Services/Tasks/Scanner/Parser/Parser.cs @@ -14,7 +14,7 @@ public static class Parser private const int RegexTimeoutMs = 5000000; // 500 ms public static readonly TimeSpan RegexTimeout = TimeSpan.FromMilliseconds(500); - public const string ImageFileExtensions = @"^(\.png|\.jpeg|\.jpg|\.webp|\.gif)"; + public const string ImageFileExtensions = @"^(\.png|\.jpeg|\.jpg|\.webp|\.gif|\.avif)"; public const string ArchiveFileExtensions = @"\.cbz|\.zip|\.rar|\.cbr|\.tar.gz|\.7zip|\.7z|\.cb7|\.cbt"; private const string BookFileExtensions = @"\.epub|\.pdf"; private const string XmlRegexExtensions = @"\.xml"; diff --git a/API/Services/Tasks/Scanner/ProcessSeries.cs b/API/Services/Tasks/Scanner/ProcessSeries.cs index 812f48335..711ab8a64 100644 --- a/API/Services/Tasks/Scanner/ProcessSeries.cs +++ b/API/Services/Tasks/Scanner/ProcessSeries.cs @@ -230,7 +230,7 @@ public class ProcessSeries : IProcessSeries _logger.LogError(ex, "[ScannerService] There was an exception updating series for {SeriesName}", series.Name); } - await _metadataService.GenerateCoversForSeries(series, (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).ConvertCoverToWebP); + await _metadataService.GenerateCoversForSeries(series, (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).EncodeMediaAs); EnqueuePostSeriesProcessTasks(series.LibraryId, series.Id); } diff --git a/API/Services/Tasks/StatsService.cs b/API/Services/Tasks/StatsService.cs index e6035d23a..9cbb9fa74 100644 --- a/API/Services/Tasks/StatsService.cs +++ b/API/Services/Tasks/StatsService.cs @@ -34,7 +34,7 @@ public class StatsService : IStatsService private readonly IUnitOfWork _unitOfWork; private readonly DataContext _context; private readonly IStatisticService _statisticService; - private const string ApiUrl = "https://stats.kavitareader.com"; + private const string ApiUrl = "http://localhost:5003"; public StatsService(ILogger logger, IUnitOfWork unitOfWork, DataContext context, IStatisticService statisticService) { @@ -139,8 +139,7 @@ public class StatsService : IStatsService TotalGenres = await _unitOfWork.GenreRepository.GetCountAsync(), TotalPeople = await _unitOfWork.PersonRepository.GetCountAsync(), UsingSeriesRelationships = await GetIfUsingSeriesRelationship(), - StoreBookmarksAsWebP = serverSettings.ConvertBookmarkToWebP, - StoreCoversAsWebP = serverSettings.ConvertCoverToWebP, + EncodeMediaAs = serverSettings.EncodeMediaAs, MaxSeriesInALibrary = await MaxSeriesInAnyLibrary(), MaxVolumesInASeries = await MaxVolumesInASeries(), MaxChaptersInASeries = await MaxChaptersInASeries(), @@ -292,14 +291,14 @@ public class StatsService : IStatsService private IEnumerable AllFormats() { - // TODO: Rewrite this with new migration code in feature/basic-stats + var results = _context.MangaFile .AsNoTracking() .AsEnumerable() .Select(m => new FileFormatDto() { Format = m.Format, - Extension = Path.GetExtension(m.FilePath)?.ToLowerInvariant()! + Extension = m.Extension }) .DistinctBy(f => f.Extension) .ToList(); diff --git a/API/Services/Tasks/VersionUpdaterService.cs b/API/Services/Tasks/VersionUpdaterService.cs index 3f5d81b22..3abbb0868 100644 --- a/API/Services/Tasks/VersionUpdaterService.cs +++ b/API/Services/Tasks/VersionUpdaterService.cs @@ -113,13 +113,13 @@ public class VersionUpdaterService : IVersionUpdaterService if (BuildInfo.Version < updateVersion) { - _logger.LogInformation("Server is out of date. Current: {CurrentVersion}. Available: {AvailableUpdate}", BuildInfo.Version, updateVersion); + _logger.LogWarning("Server is out of date. Current: {CurrentVersion}. Available: {AvailableUpdate}", BuildInfo.Version, updateVersion); await _eventHub.SendMessageAsync(MessageFactory.UpdateAvailable, MessageFactory.UpdateVersionEvent(update), true); } else if (Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT") == Environments.Development) { - _logger.LogInformation("Server is up to date. Current: {CurrentVersion}", BuildInfo.Version); + _logger.LogWarning("Server is up to date. Current: {CurrentVersion}", BuildInfo.Version); await _eventHub.SendMessageAsync(MessageFactory.UpdateAvailable, MessageFactory.UpdateVersionEvent(update), true); } diff --git a/API/Services/TokenService.cs b/API/Services/TokenService.cs index 666ba83a9..85808a2bb 100644 --- a/API/Services/TokenService.cs +++ b/API/Services/TokenService.cs @@ -77,9 +77,9 @@ public class TokenService : ITokenService { var tokenHandler = new JwtSecurityTokenHandler(); var tokenContent = tokenHandler.ReadJwtToken(request.Token); - var username = tokenContent.Claims.FirstOrDefault(q => q.Type == JwtRegisteredClaimNames.NameId)?.Value; + var username = tokenContent.Claims.FirstOrDefault(q => q.Type == JwtRegisteredClaimNames.Name)?.Value; if (string.IsNullOrEmpty(username)) return null; - var user = await _userManager.FindByIdAsync(username); + var user = await _userManager.FindByNameAsync(username); if (user == null) return null; // This forces a logout var validated = await _userManager.VerifyUserTokenAsync(user, TokenOptions.DefaultProvider, RefreshTokenName, request.RefreshToken); if (!validated) return null; diff --git a/API/SignalR/MessageFactory.cs b/API/SignalR/MessageFactory.cs index e71d8fda8..358af8040 100644 --- a/API/SignalR/MessageFactory.cs +++ b/API/SignalR/MessageFactory.cs @@ -484,7 +484,7 @@ public static class MessageFactory return new SignalRMessage() { Name = ConvertBookmarksProgress, - Title = "Converting Bookmarks to WebP", + Title = "Converting Bookmarks", SubTitle = string.Empty, EventType = eventType, Progress = ProgressType.Determinate, @@ -501,7 +501,7 @@ public static class MessageFactory return new SignalRMessage() { Name = ConvertCoversProgress, - Title = "Converting Covers to WebP", + Title = "Converting Covers", SubTitle = string.Empty, EventType = eventType, Progress = ProgressType.Determinate, diff --git a/API/Startup.cs b/API/Startup.cs index 8e8a2e1ce..109e018eb 100644 --- a/API/Startup.cs +++ b/API/Startup.cs @@ -10,6 +10,7 @@ using System.Threading.RateLimiting; using System.Threading.Tasks; using API.Constants; using API.Data; +using API.Data.ManualMigrations; using API.Entities; using API.Entities.Enums; using API.Extensions; @@ -249,6 +250,9 @@ public class Startup // v0.7.2 await MigrateLoginRoles.Migrate(unitOfWork, userManager, logger); + // v0.7.3 + await MigrateRemoveWebPSettingRows.Migrate(unitOfWork, logger); + // Update the version in the DB after all migrations are run var installVersion = await unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.InstallVersion); installVersion.Value = BuildInfo.Version.ToString(); diff --git a/Kavita.sln.DotSettings b/Kavita.sln.DotSettings index 2ebdeb970..80187ccfa 100644 --- a/Kavita.sln.DotSettings +++ b/Kavita.sln.DotSettings @@ -6,6 +6,7 @@ True True True + True True True True diff --git a/UI/Web/src/app/_services/account.service.ts b/UI/Web/src/app/_services/account.service.ts index f0df81974..ec65fb376 100644 --- a/UI/Web/src/app/_services/account.service.ts +++ b/UI/Web/src/app/_services/account.service.ts @@ -115,10 +115,10 @@ export class AccountService implements OnDestroy { this.currentUser = user; this.currentUserSource.next(user); + this.stopRefreshTokenTimer(); + if (this.currentUser !== undefined) { this.startRefreshTokenTimer(); - } else { - this.stopRefreshTokenTimer(); } } @@ -264,7 +264,6 @@ export class AccountService implements OnDestroy { private refreshToken() { if (this.currentUser === null || this.currentUser === undefined) return of(); - return this.httpClient.post<{token: string, refreshToken: string}>(this.baseUrl + 'account/refresh-token', {token: this.currentUser.token, refreshToken: this.currentUser.refreshToken}).pipe(map(user => { if (this.currentUser) { @@ -277,23 +276,23 @@ export class AccountService implements OnDestroy { })); } + /** + * Every 10 mins refresh the token + */ private startRefreshTokenTimer() { - if (this.currentUser === null || this.currentUser === undefined) return; - - if (this.refreshTokenTimeout !== undefined) { + if (this.currentUser === null || this.currentUser === undefined) { this.stopRefreshTokenTimer(); + return; } - const jwtToken = JSON.parse(atob(this.currentUser.token.split('.')[1])); - // set a timeout to refresh the token 10 mins before it expires - const expires = new Date(jwtToken.exp * 1000); - const timeout = expires.getTime() - Date.now() - (60 * 10000); - this.refreshTokenTimeout = setTimeout(() => this.refreshToken().subscribe(() => {}), timeout); + this.stopRefreshTokenTimer(); + + this.refreshTokenTimeout = setInterval(() => this.refreshToken().subscribe(() => {}), (60 * 10_000)); } private stopRefreshTokenTimer() { if (this.refreshTokenTimeout !== undefined) { - clearTimeout(this.refreshTokenTimeout); + clearInterval(this.refreshTokenTimeout); } } diff --git a/UI/Web/src/app/_services/server.service.ts b/UI/Web/src/app/_services/server.service.ts index c8baf8548..fdb88ae8d 100644 --- a/UI/Web/src/app/_services/server.service.ts +++ b/UI/Web/src/app/_services/server.service.ts @@ -55,12 +55,8 @@ export class ServerService { return this.httpClient.get(this.baseUrl + 'server/jobs'); } - convertBookmarks() { - return this.httpClient.post(this.baseUrl + 'server/convert-bookmarks', {}); - } - - convertCovers() { - return this.httpClient.post(this.baseUrl + 'server/convert-covers', {}); + convertMedia() { + return this.httpClient.post(this.baseUrl + 'server/convert-media', {}); } getMediaErrors() { diff --git a/UI/Web/src/app/admin/_models/encode-format.ts b/UI/Web/src/app/admin/_models/encode-format.ts new file mode 100644 index 000000000..9a386c4f7 --- /dev/null +++ b/UI/Web/src/app/admin/_models/encode-format.ts @@ -0,0 +1,7 @@ +export enum EncodeFormat { + PNG = 0, + WebP = 1, + AVIF = 2 +} + +export const EncodeFormats = [{value: EncodeFormat.PNG, title: 'PNG'}, {value: EncodeFormat.WebP, title: 'WebP'}, {value: EncodeFormat.AVIF, title: 'AVIF'}]; \ No newline at end of file diff --git a/UI/Web/src/app/admin/_models/server-settings.ts b/UI/Web/src/app/admin/_models/server-settings.ts index 1cb0ace79..a0b6667d7 100644 --- a/UI/Web/src/app/admin/_models/server-settings.ts +++ b/UI/Web/src/app/admin/_models/server-settings.ts @@ -1,3 +1,5 @@ +import { EncodeFormat } from "./encode-format"; + export interface ServerSettings { cacheDirectory: string; taskScan: string; @@ -10,8 +12,7 @@ export interface ServerSettings { baseUrl: string; bookmarksDirectory: string; emailServiceUrl: string; - convertBookmarkToWebP: boolean; - convertCoverToWebP: boolean; + encodeMediaAs: EncodeFormat; totalBackups: number; totalLogs: number; enableFolderWatching: boolean; diff --git a/UI/Web/src/app/admin/manage-media-settings/manage-media-settings.component.html b/UI/Web/src/app/admin/manage-media-settings/manage-media-settings.component.html index 6f6d529a8..0b4485cea 100644 --- a/UI/Web/src/app/admin/manage-media-settings/manage-media-settings.component.html +++ b/UI/Web/src/app/admin/manage-media-settings/manage-media-settings.component.html @@ -2,28 +2,17 @@
-

WebP can drastically reduce space requirements for files. WebP is not supported on all browsers or versions. To learn if these settings are appropriate for your setup, visit Can I Use.

- +

WebP/AVIF can drastically reduce space requirements for files. WebP/AVIF is not supported on all browsers or versions. To learn if these settings are appropriate for your setup, visit Can I Use WebP or Can I Use AVIF. + You cannot convert back to PNG once you've gone to WebP/AVIF. You would need to refresh covers on your libraries to regenerate all covers. Bookmarks and favicons cannot be converted.

+
- - - When saving bookmarks, convert them to WebP. - -
- - -
-
- -
- - - When generating covers, convert them to WebP. - -
- - -
+ + + All media Kavita manages (covers, bookmarks, favicons) will be encoded as this type. + +
diff --git a/UI/Web/src/app/admin/manage-media-settings/manage-media-settings.component.ts b/UI/Web/src/app/admin/manage-media-settings/manage-media-settings.component.ts index e7153b6f1..dbbd0a8b8 100644 --- a/UI/Web/src/app/admin/manage-media-settings/manage-media-settings.component.ts +++ b/UI/Web/src/app/admin/manage-media-settings/manage-media-settings.component.ts @@ -6,6 +6,7 @@ import { SettingsService } from '../settings.service'; import { ServerSettings } from '../_models/server-settings'; import { DirectoryPickerComponent, DirectoryPickerResult } from '../_modals/directory-picker/directory-picker.component'; import { NgbModal } from '@ng-bootstrap/ng-bootstrap'; +import { EncodeFormats } from '../_models/encode-format'; @Component({ selector: 'app-manage-media-settings', @@ -16,29 +17,28 @@ export class ManageMediaSettingsComponent implements OnInit { serverSettings!: ServerSettings; settingsForm: FormGroup = new FormGroup({}); + + get EncodeFormats() { return EncodeFormats; } constructor(private settingsService: SettingsService, private toastr: ToastrService, private modalService: NgbModal, ) { } ngOnInit(): void { this.settingsService.getServerSettings().pipe(take(1)).subscribe((settings: ServerSettings) => { this.serverSettings = settings; - this.settingsForm.addControl('convertBookmarkToWebP', new FormControl(this.serverSettings.convertBookmarkToWebP, [Validators.required])); - this.settingsForm.addControl('convertCoverToWebP', new FormControl(this.serverSettings.convertCoverToWebP, [Validators.required])); + this.settingsForm.addControl('encodeMediaAs', new FormControl(this.serverSettings.encodeMediaAs, [Validators.required])); this.settingsForm.addControl('bookmarksDirectory', new FormControl(this.serverSettings.bookmarksDirectory, [Validators.required])); }); } resetForm() { - this.settingsForm.get('convertBookmarkToWebP')?.setValue(this.serverSettings.convertBookmarkToWebP); - this.settingsForm.get('convertCoverToWebP')?.setValue(this.serverSettings.convertCoverToWebP); + this.settingsForm.get('encodeMediaAs')?.setValue(this.serverSettings.encodeMediaAs); this.settingsForm.get('bookmarksDirectory')?.setValue(this.serverSettings.bookmarksDirectory); this.settingsForm.markAsPristine(); } saveSettings() { const modelSettings = Object.assign({}, this.serverSettings); - modelSettings.convertBookmarkToWebP = this.settingsForm.get('convertBookmarkToWebP')?.value; - modelSettings.convertCoverToWebP = this.settingsForm.get('convertCoverToWebP')?.value; + modelSettings.encodeMediaAs = parseInt(this.settingsForm.get('encodeMediaAs')?.value, 10); this.settingsService.updateServerSettings(modelSettings).pipe(take(1)).subscribe(async (settings: ServerSettings) => { this.serverSettings = settings; diff --git a/UI/Web/src/app/admin/manage-settings/manage-settings.component.ts b/UI/Web/src/app/admin/manage-settings/manage-settings.component.ts index 1122fd34a..19e121bf3 100644 --- a/UI/Web/src/app/admin/manage-settings/manage-settings.component.ts +++ b/UI/Web/src/app/admin/manage-settings/manage-settings.component.ts @@ -50,7 +50,7 @@ export class ManageSettingsComponent implements OnInit { this.settingsForm.addControl('totalBackups', new FormControl(this.serverSettings.totalBackups, [Validators.required, Validators.min(1), Validators.max(30)])); this.settingsForm.addControl('totalLogs', new FormControl(this.serverSettings.totalLogs, [Validators.required, Validators.min(1), Validators.max(30)])); this.settingsForm.addControl('enableFolderWatching', new FormControl(this.serverSettings.enableFolderWatching, [Validators.required])); - this.settingsForm.addControl('convertBookmarkToWebP', new FormControl(this.serverSettings.convertBookmarkToWebP, [])); + this.settingsForm.addControl('encodeMediaAs', new FormControl(this.serverSettings.encodeMediaAs, [])); this.settingsForm.addControl('hostName', new FormControl(this.serverSettings.hostName, [Validators.pattern(/^(http:|https:)+[^\s]+[\w]$/)])); this.serverService.getServerInfo().subscribe(info => { @@ -76,7 +76,7 @@ export class ManageSettingsComponent implements OnInit { this.settingsForm.get('totalBackups')?.setValue(this.serverSettings.totalBackups); this.settingsForm.get('totalLogs')?.setValue(this.serverSettings.totalLogs); this.settingsForm.get('enableFolderWatching')?.setValue(this.serverSettings.enableFolderWatching); - this.settingsForm.get('convertBookmarkToWebP')?.setValue(this.serverSettings.convertBookmarkToWebP); + this.settingsForm.get('encodeMediaAs')?.setValue(this.serverSettings.encodeMediaAs); this.settingsForm.get('hostName')?.setValue(this.serverSettings.hostName); this.settingsForm.markAsPristine(); } diff --git a/UI/Web/src/app/admin/manage-tasks-settings/manage-tasks-settings.component.ts b/UI/Web/src/app/admin/manage-tasks-settings/manage-tasks-settings.component.ts index d0576af8d..e5a89d504 100644 --- a/UI/Web/src/app/admin/manage-tasks-settings/manage-tasks-settings.component.ts +++ b/UI/Web/src/app/admin/manage-tasks-settings/manage-tasks-settings.component.ts @@ -34,16 +34,10 @@ export class ManageTasksSettingsComponent implements OnInit { recurringTasks$: Observable> = of([]); adhocTasks: Array = [ { - name: 'Convert Bookmarks to WebP', - description: 'Runs a long-running task which will convert all bookmarks to WebP. This is slow (especially on ARM devices).', - api: this.serverService.convertBookmarks(), - successMessage: 'Conversion of Bookmarks has been queued' - }, - { - name: 'Convert Covers to WebP', - description: 'Runs a long-running task which will convert all existing covers to WebP. This is slow (especially on ARM devices).', - api: this.serverService.convertCovers(), - successMessage: 'Conversion of Covers has been queued' + name: 'Convert Media to Target Encoding', + description: 'Runs a long-running task which will convert all kavita-managed media to the target encoding. This is slow (especially on ARM devices).', + api: this.serverService.convertMedia(), + successMessage: 'Conversion of Media to Target Encoding has been queued' }, { name: 'Clear Cache', @@ -144,12 +138,6 @@ export class ManageTasksSettingsComponent implements OnInit { }); } - runAdhocConvert() { - this.serverService.convertBookmarks().subscribe(() => { - this.toastr.success('Conversion of Bookmarks has been queued.'); - }); - } - runAdhoc(task: AdhocTask) { task.api.subscribe((data: any) => { if (task.successMessage.length > 0) { @@ -159,6 +147,8 @@ export class ManageTasksSettingsComponent implements OnInit { if (task.successFunction) { task.successFunction(data); } + }, (err: any) => { + console.error('error: ', err); }); } diff --git a/UI/Web/src/app/cards/_modals/edit-series-modal/edit-series-modal.component.html b/UI/Web/src/app/cards/_modals/edit-series-modal/edit-series-modal.component.html index 3349fad89..40f510491 100644 --- a/UI/Web/src/app/cards/_modals/edit-series-modal/edit-series-modal.component.html +++ b/UI/Web/src/app/cards/_modals/edit-series-modal/edit-series-modal.component.html @@ -357,10 +357,14 @@
- +
diff --git a/UI/Web/src/app/cards/_modals/edit-series-modal/edit-series-modal.component.ts b/UI/Web/src/app/cards/_modals/edit-series-modal/edit-series-modal.component.ts index dea791800..5eaf0b05b 100644 --- a/UI/Web/src/app/cards/_modals/edit-series-modal/edit-series-modal.component.ts +++ b/UI/Web/src/app/cards/_modals/edit-series-modal/edit-series-modal.component.ts @@ -175,7 +175,7 @@ export class EditSeriesModalComponent implements OnInit, OnDestroy { this.editSeriesForm.get('releaseYear')?.patchValue(this.metadata.releaseYear); this.WebLinks.forEach((link, index) => { - this.editSeriesForm.addControl('link' + index, new FormControl(link, [Validators.required])); + this.editSeriesForm.addControl('link' + index, new FormControl(link, [])); }); this.cdRef.markForCheck(); @@ -521,7 +521,16 @@ export class EditSeriesModalComponent implements OnInit, OnDestroy { addWebLink() { this.metadata.webLinks += ','; - this.editSeriesForm.addControl('link' + (this.WebLinks.length - 1), new FormControl('', [Validators.required])); + this.editSeriesForm.addControl('link' + (this.WebLinks.length - 1), new FormControl('', [])); + this.cdRef.markForCheck(); + } + + removeWebLink(index: number) { + const tokens = this.metadata.webLinks.split(','); + const tokenToRemove = tokens[index]; + + this.metadata.webLinks = tokens.filter(t => t != tokenToRemove).join(','); + this.editSeriesForm.removeControl('link' + index, {emitEvent: true}); this.cdRef.markForCheck(); } diff --git a/UI/Web/src/app/series-detail/_components/series-detail/series-detail.component.html b/UI/Web/src/app/series-detail/_components/series-detail/series-detail.component.html index e919489ac..6dbd498d5 100644 --- a/UI/Web/src/app/series-detail/_components/series-detail/series-detail.component.html +++ b/UI/Web/src/app/series-detail/_components/series-detail/series-detail.component.html @@ -128,7 +128,7 @@
- + ★ diff --git a/UI/Web/src/app/series-detail/_components/series-detail/series-detail.component.ts b/UI/Web/src/app/series-detail/_components/series-detail/series-detail.component.ts index aad510c26..71da06877 100644 --- a/UI/Web/src/app/series-detail/_components/series-detail/series-detail.component.ts +++ b/UI/Web/src/app/series-detail/_components/series-detail/series-detail.component.ts @@ -740,19 +740,6 @@ export class SeriesDetailComponent implements OnInit, OnDestroy, AfterContentChe }); } - async promptToReview() { - // TODO: After a review has been set, we might just want to show an edit icon next to star rating which opens the review, instead of prompting each time. - const shouldPrompt = this.isNullOrEmpty(this.series.userReview); - const config = new ConfirmConfig(); - config.header = 'Confirm'; - config.content = 'Do you want to write a review?'; - config.buttons.push({text: 'No', type: 'secondary'}); - config.buttons.push({text: 'Yes', type: 'primary'}); - if (shouldPrompt && await this.confirmService.confirm('Do you want to write a review?', config)) { - this.openReviewModal(); - } - } - openReviewModal(force = false) { const modalRef = this.modalService.open(ReviewSeriesModalComponent, { scrollable: true, size: 'lg' }); modalRef.componentInstance.series = this.series; diff --git a/UI/Web/src/index.html b/UI/Web/src/index.html index 4ce7fd914..a8ce4b48c 100644 --- a/UI/Web/src/index.html +++ b/UI/Web/src/index.html @@ -15,7 +15,7 @@ - + diff --git a/openapi.json b/openapi.json index c8c958960..9e7058045 100644 --- a/openapi.json +++ b/openapi.json @@ -7,7 +7,7 @@ "name": "GPL-3.0", "url": "https://github.com/Kareadita/Kavita/blob/develop/LICENSE" }, - "version": "0.7.2.3" + "version": "0.7.2.6" }, "servers": [ { @@ -7490,25 +7490,12 @@ } } }, - "/api/Server/convert-bookmarks": { + "/api/Server/convert-media": { "post": { "tags": [ "Server" ], - "summary": "Triggers the scheduling of the convert bookmarks job. Only one job will run at a time.", - "responses": { - "200": { - "description": "Success" - } - } - } - }, - "/api/Server/convert-covers": { - "post": { - "tags": [ - "Server" - ], - "summary": "Triggers the scheduling of the convert covers job. Only one job will run at a time.", + "summary": "Triggers the scheduling of the convert media job. This will convert all media to the target encoding (except for PNG). Only one job will run at a time.", "responses": { "200": { "description": "Success" @@ -11451,6 +11438,15 @@ "additionalProperties": false, "description": "Represents if Test Email Service URL was successful or not and if any error occured" }, + "EncodeFormat": { + "enum": [ + 0, + 1, + 2 + ], + "type": "integer", + "format": "int32" + }, "FileDimensionDto": { "type": "object", "properties": { @@ -14137,10 +14133,6 @@ "description": "Total number of People in the instance", "format": "int32" }, - "storeBookmarksAsWebP": { - "type": "boolean", - "description": "Is this instance storing bookmarks as WebP" - }, "usersOnCardLayout": { "type": "integer", "description": "Number of users on this instance using Card Layout", @@ -14236,9 +14228,8 @@ "description": "Total reading hours of all users", "format": "int64" }, - "storeCoversAsWebP": { - "type": "boolean", - "description": "Is the Server saving covers as WebP" + "encodeMediaAs": { + "$ref": "#/components/schemas/EncodeFormat" } }, "additionalProperties": false, @@ -14306,9 +14297,8 @@ "description": "Represents a unique Id to this Kavita installation. Only used in Stats to identify unique installs.", "nullable": true }, - "convertBookmarkToWebP": { - "type": "boolean", - "description": "If the server should save bookmarks as WebP encoding" + "encodeMediaAs": { + "$ref": "#/components/schemas/EncodeFormat" }, "totalBackups": { "type": "integer", @@ -14324,10 +14314,6 @@ "description": "Total number of days worth of logs to keep at a given time.", "format": "int32" }, - "convertCoverToWebP": { - "type": "boolean", - "description": "If the server should save covers as WebP encoding" - }, "hostName": { "type": "string", "description": "The Host name (ie Reverse proxy domain name) for the server",