From 3c887f53773686b03dcdd6026680b3f0f3633c94 Mon Sep 17 00:00:00 2001 From: Joe Milazzo Date: Sat, 20 May 2023 09:21:27 -0500 Subject: [PATCH] More Polish (#2005) * Implemented sort title extraction from epub 3 files. * Added link to wiki for media errors * Fixed the hack to reduce JWT refresh token expiration * Fixed up a case where favicon downloading wasn't correcting links that started with // correctly. Added a fallback for sites that just don't pngs available. * Implemented a mechanism to fallback to Kavita's website for favicons which can be dynamically added/updated by the community. * Reworked the logic for bookwalker which will fail to get the base html, so we have to rely on the fallback handler. --- API/Data/Seed.cs | 10 +++ API/Services/BookService.cs | 78 ++++++++++++++----- API/Services/ImageService.cs | 47 ++++++++--- API/Services/TokenService.cs | 2 +- .../manage-alerts.component.html | 6 +- openapi.json | 2 +- 6 files changed, 112 insertions(+), 33 deletions(-) diff --git a/API/Data/Seed.cs b/API/Data/Seed.cs index 24f0236f5..fb7e97f40 100644 --- a/API/Data/Seed.cs +++ b/API/Data/Seed.cs @@ -144,4 +144,14 @@ public static class Seed } await context.SaveChangesAsync(); } + + // /// + // /// Responsible to copy (not overwrite) a set of favicons that Kavita can't parse from websites. + // /// + // /// + // /// + // public static Task SeedFavicons(IDirectoryService directoryService) + // { + // + // } } diff --git a/API/Services/BookService.cs b/API/Services/BookService.cs index aa8660418..f952ad0a8 100644 --- a/API/Services/BookService.cs +++ b/API/Services/BookService.cs @@ -472,7 +472,10 @@ public class BookService : IBookService break; case "calibre:series": info.Series = metadataItem.Content; - info.SeriesSort = metadataItem.Content; + if (string.IsNullOrEmpty(info.SeriesSort)) + { + info.SeriesSort = metadataItem.Content; + } break; case "calibre:series_index": info.Volume = metadataItem.Content; @@ -488,7 +491,10 @@ public class BookService : IBookService break; case "belongs-to-collection": info.Series = metadataItem.Content; - info.SeriesSort = metadataItem.Content; + if (string.IsNullOrEmpty(info.SeriesSort)) + { + info.SeriesSort = metadataItem.Content; + } break; case "collection-type": // These look to be genres from https://manual.calibre-ebook.com/sub_groups.html or can be "series" @@ -504,32 +510,24 @@ public class BookService : IBookService PopulatePerson(metadataItem, info, person); break; case "title-type": - if (!metadataItem.Content.Equals("collection")) break; - var titleId = metadataItem.Refines?.Replace("#", string.Empty); - var readingListElem = epubBook.Schema.Package.Metadata.Titles - .FirstOrDefault(item => item.Id == titleId); - if (readingListElem == null) break; + if (metadataItem.Content.Equals("collection")) + { + ExtractCollectionOrReadingList(metadataItem, epubBook, info); + } - var count = epubBook.Schema.Package.Metadata.MetaItems - .FirstOrDefault(item => - item.Property == "display-seq" && item.Refines == metadataItem.Refines); - if (count == null || count.Content == "0") + if (metadataItem.Content.Equals("main")) { - // TODO: Rewrite this to use a StringBuilder - // Treat this as a Collection - info.SeriesGroup += (string.IsNullOrEmpty(info.StoryArc) ? string.Empty : ",") + readingListElem.Title.Replace(",", "_"); - } - else - { - // Treat as a reading list - info.AlternateSeries += (string.IsNullOrEmpty(info.AlternateSeries) ? string.Empty : ",") + readingListElem.Title.Replace(",", "_"); - info.AlternateNumber += (string.IsNullOrEmpty(info.AlternateNumber) ? string.Empty : ",") + count.Content; + ExtractSortTitle(metadataItem, epubBook, info); } + break; } } + // Check if there is a SortTitle + + // Include regular Writer as well, for cases where there is no special tag info.Writer = string.Join(",", epubBook.Schema.Package.Metadata.Creators.Select(c => Parser.CleanAuthor(c.Creator))); @@ -556,6 +554,46 @@ public class BookService : IBookService return null; } + private static void ExtractSortTitle(EpubMetadataMeta metadataItem, EpubBookRef epubBook, ComicInfo info) + { + var titleId = metadataItem.Refines?.Replace("#", string.Empty); + var titleElem = epubBook.Schema.Package.Metadata.Titles + .FirstOrDefault(item => item.Id == titleId); + if (titleElem == null) return; + + var sortTitleElem = epubBook.Schema.Package.Metadata.MetaItems + .FirstOrDefault(item => + item.Property == "file-as" && item.Refines == metadataItem.Refines); + if (sortTitleElem == null || string.IsNullOrWhiteSpace(sortTitleElem.Content)) return; + info.SeriesSort = sortTitleElem.Content; + } + + private static void ExtractCollectionOrReadingList(EpubMetadataMeta metadataItem, EpubBookRef epubBook, ComicInfo info) + { + var titleId = metadataItem.Refines?.Replace("#", string.Empty); + var readingListElem = epubBook.Schema.Package.Metadata.Titles + .FirstOrDefault(item => item.Id == titleId); + if (readingListElem == null) return; + + var count = epubBook.Schema.Package.Metadata.MetaItems + .FirstOrDefault(item => + item.Property == "display-seq" && item.Refines == metadataItem.Refines); + if (count == null || count.Content == "0") + { + // TODO: Rewrite this to use a StringBuilder + // Treat this as a Collection + info.SeriesGroup += (string.IsNullOrEmpty(info.StoryArc) ? string.Empty : ",") + + readingListElem.Title.Replace(",", "_"); + } + else + { + // Treat as a reading list + info.AlternateSeries += (string.IsNullOrEmpty(info.AlternateSeries) ? string.Empty : ",") + + readingListElem.Title.Replace(",", "_"); + info.AlternateNumber += (string.IsNullOrEmpty(info.AlternateNumber) ? string.Empty : ",") + count.Content; + } + } + private static void PopulatePerson(EpubMetadataMeta metadataItem, ComicInfo info, EpubMetadataCreator person) { switch (metadataItem.Content) diff --git a/API/Services/ImageService.cs b/API/Services/ImageService.cs index 2230850c3..656b427a6 100644 --- a/API/Services/ImageService.cs +++ b/API/Services/ImageService.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; @@ -227,6 +226,8 @@ public class ImageService : IImageService url = value; } + var correctSizeLink = string.Empty; + try { var htmlContent = url.GetStringAsync().Result; @@ -238,11 +239,16 @@ public class ImageService : IImageService .Where(href => href.Split("?")[0].EndsWith(".png", StringComparison.InvariantCultureIgnoreCase)) .ToList(); - if (pngLinks == null) - { - throw new KavitaException($"Could not grab favicon from {baseUrl}"); - } - var correctSizeLink = pngLinks.FirstOrDefault(pngLink => pngLink.Contains("32")) ?? pngLinks.FirstOrDefault(); + correctSizeLink = (pngLinks?.FirstOrDefault(pngLink => pngLink.Contains("32")) ?? pngLinks?.FirstOrDefault()); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error downloading favicon.png for {Domain}, will try fallback methods", domain); + } + + try + { + correctSizeLink = FallbackToKavitaReaderFavicon(baseUrl); if (string.IsNullOrEmpty(correctSizeLink)) { throw new KavitaException($"Could not grab favicon from {baseUrl}"); @@ -253,7 +259,7 @@ public class ImageService : IImageService // If starts with //, it's coming usually from an offsite cdn if (correctSizeLink.StartsWith("//")) { - finalUrl = Url.Combine("https:", correctSizeLink); + finalUrl = "https:" + correctSizeLink; } else if (!correctSizeLink.StartsWith(uri.Scheme)) { @@ -287,14 +293,37 @@ public class ImageService : IImageService _logger.LogDebug("Favicon.png for {Domain} downloaded and saved successfully", domain); return filename; - } - catch (Exception ex) + }catch (Exception ex) { _logger.LogError(ex, "Error downloading favicon.png for {Domain}", domain); throw; } } + private static string FallbackToKavitaReaderFavicon(string baseUrl) + { + var correctSizeLink = string.Empty; + var allOverrides = "https://kavitareader.com/assets/favicons/urls.txt".GetStringAsync().Result; + if (!string.IsNullOrEmpty(allOverrides)) + { + var cleanedBaseUrl = baseUrl.Replace("https://", string.Empty); + var externalFile = allOverrides + .Split("\n") + .FirstOrDefault(url => + cleanedBaseUrl.Equals(url.Replace(".png", string.Empty)) || + cleanedBaseUrl.Replace("www.", string.Empty).Equals(url.Replace(".png", string.Empty) + )); + if (string.IsNullOrEmpty(externalFile)) + { + throw new KavitaException($"Could not grab favicon from {baseUrl}"); + } + + correctSizeLink = "https://kavitareader.com/assets/favicons/" + externalFile; + } + + return correctSizeLink; + } + /// public string CreateThumbnailFromBase64(string encodedImage, string fileName, EncodeFormat encodeFormat, int thumbnailWidth = ThumbnailWidth) { diff --git a/API/Services/TokenService.cs b/API/Services/TokenService.cs index 13faee4e8..443cea5b9 100644 --- a/API/Services/TokenService.cs +++ b/API/Services/TokenService.cs @@ -97,7 +97,7 @@ public class TokenService : ITokenService } var validated = await _userManager.VerifyUserTokenAsync(user, TokenOptions.DefaultProvider, RefreshTokenName, request.RefreshToken); - if (!validated && tokenContent.ValidTo > DateTime.UtcNow.Add(TimeSpan.FromHours(1))) + if (!validated && tokenContent.ValidTo <= DateTime.UtcNow.Add(TimeSpan.FromHours(1))) { _logger.LogDebug("[RefreshToken] failed to validate due to invalid refresh token"); return null; diff --git a/UI/Web/src/app/admin/manage-alerts/manage-alerts.component.html b/UI/Web/src/app/admin/manage-alerts/manage-alerts.component.html index 2f1848515..c98c4ca53 100644 --- a/UI/Web/src/app/admin/manage-alerts/manage-alerts.component.html +++ b/UI/Web/src/app/admin/manage-alerts/manage-alerts.component.html @@ -1,4 +1,6 @@ -

This table contains issues found during scan or reading of your media. This list is non-managed. You can clear it at any time and use Library (Force) Scan to perform analysis.

+

This table contains issues found during scan or reading of your media. This list is non-managed. + You can clear it at any time and use Library (Force) Scan to perform analysis. A list of some common errors and what + they mean can be found on the wiki.

@@ -48,4 +50,4 @@ - \ No newline at end of file + diff --git a/openapi.json b/openapi.json index 3f35d590d..7112a2e25 100644 --- a/openapi.json +++ b/openapi.json @@ -7,7 +7,7 @@ "name": "GPL-3.0", "url": "https://github.com/Kareadita/Kavita/blob/develop/LICENSE" }, - "version": "0.7.2.12" + "version": "0.7.2.13" }, "servers": [ {