diff --git a/API.Benchmark/ArchiveServiceBenchmark.cs b/API.Benchmark/ArchiveServiceBenchmark.cs index d8418ee26..e24589205 100644 --- a/API.Benchmark/ArchiveServiceBenchmark.cs +++ b/API.Benchmark/ArchiveServiceBenchmark.cs @@ -1,9 +1,14 @@ using System; +using System.IO; using System.IO.Abstractions; using Microsoft.Extensions.Logging.Abstractions; using API.Services; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Order; +using SixLabors.ImageSharp; +using SixLabors.ImageSharp.Formats.Png; +using SixLabors.ImageSharp.Formats.Webp; +using SixLabors.ImageSharp.Processing; namespace API.Benchmark; @@ -17,6 +22,10 @@ public class ArchiveServiceBenchmark private readonly ArchiveService _archiveService; private readonly IDirectoryService _directoryService; private readonly IImageService _imageService; + private readonly PngEncoder _pngEncoder = new PngEncoder(); + private readonly WebpEncoder _webPEncoder = new WebpEncoder(); + private const string SourceImage = "C:/Users/josep/Pictures/obey_by_grrsa-d6llkaa_colored_by_me.png"; + public ArchiveServiceBenchmark() { @@ -49,6 +58,52 @@ public class ArchiveServiceBenchmark } } + [Benchmark] + public void ImageSharp_ExtractImage_PNG() + { + var outputDirectory = "C:/Users/josep/Pictures/imagesharp/"; + _directoryService.ExistOrCreate(outputDirectory); + + using var stream = new FileStream(SourceImage, FileMode.Open); + using var thumbnail2 = SixLabors.ImageSharp.Image.Load(stream); + thumbnail2.Mutate(x => x.Resize(320, 0)); + thumbnail2.Save(_directoryService.FileSystem.Path.Join(outputDirectory, "imagesharp.png"), _pngEncoder); + } + + [Benchmark] + public void ImageSharp_ExtractImage_WebP() + { + var outputDirectory = "C:/Users/josep/Pictures/imagesharp/"; + _directoryService.ExistOrCreate(outputDirectory); + + using var stream = new FileStream(SourceImage, FileMode.Open); + using var thumbnail2 = SixLabors.ImageSharp.Image.Load(stream); + thumbnail2.Mutate(x => x.Resize(320, 0)); + thumbnail2.Save(_directoryService.FileSystem.Path.Join(outputDirectory, "imagesharp.webp"), _webPEncoder); + } + + [Benchmark] + public void NetVips_ExtractImage_PNG() + { + var outputDirectory = "C:/Users/josep/Pictures/netvips/"; + _directoryService.ExistOrCreate(outputDirectory); + + using var stream = new FileStream(SourceImage, FileMode.Open); + using var thumbnail = NetVips.Image.ThumbnailStream(stream, 320); + thumbnail.WriteToFile(_directoryService.FileSystem.Path.Join(outputDirectory, "netvips.png")); + } + + [Benchmark] + public void NetVips_ExtractImage_WebP() + { + var outputDirectory = "C:/Users/josep/Pictures/netvips/"; + _directoryService.ExistOrCreate(outputDirectory); + + using var stream = new FileStream(SourceImage, FileMode.Open); + using var thumbnail = NetVips.Image.ThumbnailStream(stream, 320); + thumbnail.WriteToFile(_directoryService.FileSystem.Path.Join(outputDirectory, "netvips.webp")); + } + // Benchmark to test default GetNumberOfPages from archive // vs a new method where I try to open the archive and return said stream } diff --git a/API.Benchmark/EpubBenchmark.cs b/API.Benchmark/EpubBenchmark.cs index fd4fe4da4..1df4f176e 100644 --- a/API.Benchmark/EpubBenchmark.cs +++ b/API.Benchmark/EpubBenchmark.cs @@ -1,5 +1,6 @@ using System; using System.Linq; +using System.Text.RegularExpressions; using System.Threading.Tasks; using API.Services; using BenchmarkDotNet.Attributes; @@ -9,34 +10,58 @@ using VersOne.Epub; namespace API.Benchmark; +[StopOnFirstError] [MemoryDiagnoser] -[Orderer(SummaryOrderPolicy.FastestToSlowest)] [RankColumn] -[SimpleJob(launchCount: 1, warmupCount: 3, targetCount: 5, invocationCount: 100, id: "Epub"), ShortRunJob] +[Orderer(SummaryOrderPolicy.FastestToSlowest)] +[SimpleJob(launchCount: 1, warmupCount: 5, targetCount: 20)] public class EpubBenchmark { + private const string FilePath = @"E:\Books\Invaders of the Rokujouma\Invaders of the Rokujouma - Volume 01.epub"; + private readonly Regex WordRegex = new Regex(@"\b\w+\b", RegexOptions.Compiled | RegexOptions.IgnoreCase); + + // [Benchmark] + // public async Task GetWordCount_PassByString() + // { + // using var book = await EpubReader.OpenBookAsync(FilePath, BookService.BookReaderOptions); + // foreach (var bookFile in book.Content.Html.Values) + // { + // GetBookWordCount_PassByString(await bookFile.ReadContentAsTextAsync()); + // ; + // } + // } + [Benchmark] - public static async Task GetWordCount_PassByString() + public async Task GetWordCount_PassByRef() { - using var book = await EpubReader.OpenBookAsync("Data/book-test.epub", BookService.BookReaderOptions); + using var book = await EpubReader.OpenBookAsync(FilePath, BookService.BookReaderOptions); foreach (var bookFile in book.Content.Html.Values) { - Console.WriteLine(GetBookWordCount_PassByString(await bookFile.ReadContentAsTextAsync())); - ; + await GetBookWordCount_PassByRef(bookFile); } } [Benchmark] - public static async Task GetWordCount_PassByRef() + public async Task GetBookWordCount_SumEarlier() { - using var book = await EpubReader.OpenBookAsync("Data/book-test.epub", BookService.BookReaderOptions); + using var book = await EpubReader.OpenBookAsync(FilePath, BookService.BookReaderOptions); foreach (var bookFile in book.Content.Html.Values) { - Console.WriteLine(await GetBookWordCount_PassByRef(bookFile)); + await GetBookWordCount_SumEarlier(bookFile); } } - private static int GetBookWordCount_PassByString(string fileContents) + [Benchmark] + public async Task GetBookWordCount_Regex() + { + using var book = await EpubReader.OpenBookAsync(FilePath, BookService.BookReaderOptions); + foreach (var bookFile in book.Content.Html.Values) + { + await GetBookWordCount_Regex(bookFile); + } + } + + private int GetBookWordCount_PassByString(string fileContents) { var doc = new HtmlDocument(); doc.LoadHtml(fileContents); @@ -51,18 +76,41 @@ public class EpubBenchmark .Sum(); } - private static async Task GetBookWordCount_PassByRef(EpubContentFileRef bookFile) + private async Task GetBookWordCount_PassByRef(EpubContentFileRef bookFile) { var doc = new HtmlDocument(); doc.LoadHtml(await bookFile.ReadContentAsTextAsync()); var delimiter = new char[] {' '}; - return doc.DocumentNode.SelectNodes("//body//text()[not(parent::script)]") - .Select(node => node.InnerText) + var textNodes = doc.DocumentNode.SelectNodes("//body//text()[not(parent::script)]"); + if (textNodes == null) return 0; + return textNodes.Select(node => node.InnerText) .Select(text => text.Split(delimiter, StringSplitOptions.RemoveEmptyEntries) .Where(s => char.IsLetter(s[0]))) .Select(words => words.Count()) .Where(wordCount => wordCount > 0) .Sum(); } + + private async Task GetBookWordCount_SumEarlier(EpubContentFileRef bookFile) + { + var doc = new HtmlDocument(); + doc.LoadHtml(await bookFile.ReadContentAsTextAsync()); + + return doc.DocumentNode.SelectNodes("//body//text()[not(parent::script)]") + .DefaultIfEmpty() + .Select(node => node.InnerText.Split(' ', StringSplitOptions.RemoveEmptyEntries) + .Where(s => char.IsLetter(s[0]))) + .Sum(words => words.Count()); + } + + private async Task GetBookWordCount_Regex(EpubContentFileRef bookFile) + { + var doc = new HtmlDocument(); + doc.LoadHtml(await bookFile.ReadContentAsTextAsync()); + + + return doc.DocumentNode.SelectNodes("//body//text()[not(parent::script)]") + .Sum(node => WordRegex.Matches(node.InnerText).Count); + } } diff --git a/API/DTOs/ProgressDto.cs b/API/DTOs/ProgressDto.cs index 1f5142078..b3ffd2914 100644 --- a/API/DTOs/ProgressDto.cs +++ b/API/DTOs/ProgressDto.cs @@ -1,4 +1,5 @@ -using System.ComponentModel.DataAnnotations; +using System; +using System.ComponentModel.DataAnnotations; namespace API.DTOs; @@ -19,4 +20,8 @@ public class ProgressDto /// on pages that combine multiple "chapters". /// public string BookScrollId { get; set; } + /// + /// Last time the progress was synced from UI or external app + /// + public DateTime LastModified { get; set; } } diff --git a/API/Data/Repositories/ChapterRepository.cs b/API/Data/Repositories/ChapterRepository.cs index 9bc127280..695de9366 100644 --- a/API/Data/Repositories/ChapterRepository.cs +++ b/API/Data/Repositories/ChapterRepository.cs @@ -52,7 +52,7 @@ public class ChapterRepository : IChapterRepository _context.Entry(chapter).State = EntityState.Modified; } - public async Task> GetChaptersByIdsAsync(IList chapterIds, ChapterIncludes includes) + public async Task> GetChaptersByIdsAsync(IList chapterIds, ChapterIncludes includes = ChapterIncludes.None) { return await _context.Chapter .Where(c => chapterIds.Contains(c.Id)) diff --git a/API/Services/BookService.cs b/API/Services/BookService.cs index 369184925..32761105b 100644 --- a/API/Services/BookService.cs +++ b/API/Services/BookService.cs @@ -49,7 +49,7 @@ public interface IBookService /// /// Extracts a PDF file's pages as images to an target directory /// - /// This method relies on Docnet which has explict patches from Kavita for ARM support. This should only be used with Tachiyomi + /// This method relies on Docnet which has explicit patches from Kavita for ARM support. This should only be used with Tachiyomi /// /// Where the files will be extracted to. If doesn't exist, will be created. void ExtractPdfImages(string fileFilePath, string targetDirectory); @@ -401,7 +401,7 @@ public class BookService : IBookService { using var epubBook = EpubReader.OpenBook(filePath, BookReaderOptions); var publicationDate = - epubBook.Schema.Package.Metadata.Dates.FirstOrDefault(date => date.Event == "publication")?.Date; + epubBook.Schema.Package.Metadata.Dates.FirstOrDefault(pDate => pDate.Event == "publication")?.Date; if (string.IsNullOrEmpty(publicationDate)) { @@ -533,7 +533,7 @@ public class BookService : IBookService return 0; } - public static string EscapeTags(string content) + private static string EscapeTags(string content) { content = Regex.Replace(content, @")", ""); content = Regex.Replace(content, @")", ""); @@ -830,43 +830,50 @@ public class BookService : IBookService var bookPages = await book.GetReadingOrderAsync(); - foreach (var contentFileRef in bookPages) + try { - if (page != counter) + foreach (var contentFileRef in bookPages) { - counter++; - continue; - } - - var content = await contentFileRef.ReadContentAsync(); - if (contentFileRef.ContentType != EpubContentType.XHTML_1_1) return content; - - // In more cases than not, due to this being XML not HTML, we need to escape the script tags. - content = BookService.EscapeTags(content); - - doc.LoadHtml(content); - var body = doc.DocumentNode.SelectSingleNode("//body"); - - if (body == null) - { - if (doc.ParseErrors.Any()) + if (page != counter) { - LogBookErrors(book, contentFileRef, doc); - throw new KavitaException("The file is malformed! Cannot read."); + counter++; + continue; } - _logger.LogError("{FilePath} has no body tag! Generating one for support. Book may be skewed", book.FilePath); - doc.DocumentNode.SelectSingleNode("/html").AppendChild(HtmlNode.CreateNode("")); - body = doc.DocumentNode.SelectSingleNode("/html/body"); - } - return await ScopePage(doc, book, apiBase, body, mappings, page); + var content = await contentFileRef.ReadContentAsync(); + if (contentFileRef.ContentType != EpubContentType.XHTML_1_1) return content; + + // In more cases than not, due to this being XML not HTML, we need to escape the script tags. + content = BookService.EscapeTags(content); + + doc.LoadHtml(content); + var body = doc.DocumentNode.SelectSingleNode("//body"); + + if (body == null) + { + if (doc.ParseErrors.Any()) + { + LogBookErrors(book, contentFileRef, doc); + throw new KavitaException("The file is malformed! Cannot read."); + } + _logger.LogError("{FilePath} has no body tag! Generating one for support. Book may be skewed", book.FilePath); + doc.DocumentNode.SelectSingleNode("/html").AppendChild(HtmlNode.CreateNode("")); + body = doc.DocumentNode.SelectSingleNode("/html/body"); + } + + return await ScopePage(doc, book, apiBase, body, mappings, page); + } + } catch (Exception ex) + { + // NOTE: We can log this to media analysis service + _logger.LogError(ex, "There was an issue reading one of the pages for {Book}", book.FilePath); } throw new KavitaException("Could not find the appropriate html for that page"); } - private static void CreateToCChapter(EpubNavigationItemRef navigationItem, IList nestedChapters, IList chaptersList, - IReadOnlyDictionary mappings) + private static void CreateToCChapter(EpubNavigationItemRef navigationItem, IList nestedChapters, + ICollection chaptersList, IReadOnlyDictionary mappings) { if (navigationItem.Link == null) { diff --git a/API/Services/ImageService.cs b/API/Services/ImageService.cs index b9fc252f7..ac2933b75 100644 --- a/API/Services/ImageService.cs +++ b/API/Services/ImageService.cs @@ -2,6 +2,7 @@ using System.IO; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using NetVips; using SixLabors.ImageSharp; using Image = NetVips.Image; @@ -113,15 +114,15 @@ public class ImageService : IImageService return filename; } - public async Task ConvertToWebP(string filePath, string outputPath) + public Task ConvertToWebP(string filePath, string outputPath) { var file = _directoryService.FileSystem.FileInfo.FromFileName(filePath); var fileName = file.Name.Replace(file.Extension, string.Empty); var outputFile = Path.Join(outputPath, fileName + ".webp"); - using var sourceImage = await SixLabors.ImageSharp.Image.LoadAsync(filePath); - await sourceImage.SaveAsWebpAsync(outputFile); - return outputFile; + using var sourceImage = Image.NewFromFile(filePath, false, Enums.Access.SequentialUnbuffered); + sourceImage.WriteToFile(outputFile); + return Task.FromResult(outputFile); } public async Task IsImage(string filePath) diff --git a/API/Services/StatisticService.cs b/API/Services/StatisticService.cs index e1b9f89be..1ab88dd37 100644 --- a/API/Services/StatisticService.cs +++ b/API/Services/StatisticService.cs @@ -26,7 +26,6 @@ public interface IStatisticService Task GetFileBreakdown(); Task> GetTopUsers(int days); Task> GetReadingHistory(int userId); - Task> GetHistory(); Task>> ReadCountByDay(int userId = 0); } @@ -71,20 +70,6 @@ public class StatisticService : IStatisticService .Where(c => chapterIds.Contains(c.Id)) .SumAsync(c => c.AvgHoursToRead); - // Maybe make this top 5 genres? But usually there are 3-5 genres that are always common... - // Maybe use rating to calculate top genres? - // var genres = await _context.Series - // .Where(s => seriesIds.Contains(s.Id)) - // .Select(s => s.Metadata) - // .SelectMany(sm => sm.Genres) - // //.DistinctBy(g => g.NormalizedTitle) - // .ToListAsync(); - - // How many series of each format have you read? (Epub, Archive, etc) - - // Percentage of libraries read. For each library, get the total pages vs read - //var allLibraryIds = await _context.Library.GetUserLibraries(userId).ToListAsync(); - var chaptersRead = await _context.AppUserProgresses .Where(p => p.AppUserId == userId) .Where(p => libraryIds.Contains(p.LibraryId)) @@ -344,43 +329,6 @@ public class StatisticService : IStatisticService .ToListAsync(); } - public Task> GetHistory() - { - // _context.AppUserProgresses - // .AsSplitQuery() - // .AsEnumerable() - // .GroupBy(sm => sm.LastModified) - // .Select(sm => new - // { - // User = _context.AppUser.Single(u => u.Id == sm.Key), - // Chapters = _context.Chapter.Where(c => _context.AppUserProgresses - // .Where(u => u.AppUserId == sm.Key) - // .Where(p => p.PagesRead > 0) - // .Select(p => p.ChapterId) - // .Distinct() - // .Contains(c.Id)) - // }) - // .OrderByDescending(d => d.Chapters.Sum(c => c.AvgHoursToRead)) - // .Take(5) - // .ToList(); - - var firstOfWeek = DateTime.Now.StartOfWeek(DayOfWeek.Monday); - var groupedReadingDays = _context.AppUserProgresses - .Where(x => x.LastModified >= firstOfWeek) - .GroupBy(x => x.LastModified.Day) - .Select(g => new StatCount() - { - Value = g.Key, - Count = _context.AppUserProgresses.Where(p => p.LastModified.Day == g.Key).Select(p => p.ChapterId).Distinct().Count() - }) - .AsEnumerable(); - - // var records = firstOfWeek.Range(7) - // .GroupJoin(groupedReadingDays, wd => wd.Day, lg => lg.Key, (_, lg) => lg.Any() ? lg.First().Count() : 0).ToArray(); - return Task.FromResult>(null); - } - - public async Task> GetTopUsers(int days) { var libraries = (await _unitOfWork.LibraryRepository.GetLibrariesAsync()).ToList(); diff --git a/API/Services/Tasks/Metadata/WordCountAnalyzerService.cs b/API/Services/Tasks/Metadata/WordCountAnalyzerService.cs index 1bc20a359..763287a34 100644 --- a/API/Services/Tasks/Metadata/WordCountAnalyzerService.cs +++ b/API/Services/Tasks/Metadata/WordCountAnalyzerService.cs @@ -196,8 +196,7 @@ public class WordCountAnalyzerService : IWordCountAnalyzerService return; } - file.LastFileAnalysis = DateTime.Now; - _unitOfWork.MangaFileRepository.Update(file); + UpdateFileAnalysis(file); } chapter.WordCount = sum; @@ -211,8 +210,7 @@ public class WordCountAnalyzerService : IWordCountAnalyzerService chapter.AvgHoursToRead = est.AvgHours; foreach (var file in chapter.Files) { - file.LastFileAnalysis = DateTime.Now; - _unitOfWork.MangaFileRepository.Update(file); + UpdateFileAnalysis(file); } _unitOfWork.ChapterRepository.Update(chapter); } @@ -233,22 +231,22 @@ public class WordCountAnalyzerService : IWordCountAnalyzerService _unitOfWork.SeriesRepository.Update(series); } + private void UpdateFileAnalysis(MangaFile file) + { + file.LastFileAnalysis = DateTime.Now; + _unitOfWork.MangaFileRepository.Update(file); + } + private static async Task GetWordCountFromHtml(EpubContentFileRef bookFile) { var doc = new HtmlDocument(); doc.LoadHtml(await bookFile.ReadContentAsTextAsync()); - var textNodes = doc.DocumentNode.SelectNodes("//body//text()[not(parent::script)]"); - if (textNodes == null) return 0; - - return textNodes + return doc.DocumentNode.SelectNodes("//body//text()[not(parent::script)]") + .DefaultIfEmpty() .Select(node => node.InnerText.Split(' ', StringSplitOptions.RemoveEmptyEntries) .Where(s => char.IsLetter(s[0]))) - .Select(words => words.Count()) - .Where(wordCount => wordCount > 0) - .Sum(); + .Sum(words => words.Count()); } - - } diff --git a/Kavita.sln.DotSettings b/Kavita.sln.DotSettings index 55f8e0090..277751040 100644 --- a/Kavita.sln.DotSettings +++ b/Kavita.sln.DotSettings @@ -2,8 +2,14 @@ ExplicitlyExcluded True True + True + True True True + True + True + True True True - True \ No newline at end of file + True + True \ No newline at end of file diff --git a/UI/Web/src/app/admin/manage-media-settings/manage-media-settings.component.html b/UI/Web/src/app/admin/manage-media-settings/manage-media-settings.component.html index 22178a052..2e93b6498 100644 --- a/UI/Web/src/app/admin/manage-media-settings/manage-media-settings.component.html +++ b/UI/Web/src/app/admin/manage-media-settings/manage-media-settings.component.html @@ -32,4 +32,7 @@ + + + \ No newline at end of file diff --git a/UI/Web/src/app/manga-reader/_components/manga-reader/manga-reader.component.ts b/UI/Web/src/app/manga-reader/_components/manga-reader/manga-reader.component.ts index cdc15ad7b..9a43f5c7b 100644 --- a/UI/Web/src/app/manga-reader/_components/manga-reader/manga-reader.component.ts +++ b/UI/Web/src/app/manga-reader/_components/manga-reader/manga-reader.component.ts @@ -349,7 +349,7 @@ export class MangaReaderComponent implements OnInit, AfterViewInit, OnDestroy { } get SplitIconClass() { - // NOTE: This could be rewritten to valueChanges.pipe(map()) and | async in the UI instead of the getter + // TODO: make this a pipe if (this.mangaReaderService.isSplitLeftToRight(this.pageSplitOption)) { return 'left-side'; } else if (this.mangaReaderService.isNoSplit(this.pageSplitOption)) { @@ -593,7 +593,7 @@ export class MangaReaderComponent implements OnInit, AfterViewInit, OnDestroy { pageSplit: parseInt(this.generalSettingsForm.get('pageSplitOption')?.value, 10), fitting: (this.generalSettingsForm.get('fittingOption')?.value as FITTING_OPTION), layoutMode: this.layoutMode, - darkness: 100, + darkness: parseInt(this.generalSettingsForm.get('darkness')?.value + '', 10) || 100, pagingDirection: this.pagingDirection, readerMode: this.readerMode, emulateBook: this.generalSettingsForm.get('emulateBook')?.value, diff --git a/UI/Web/src/app/statistics/_components/publication-status-stats/publication-status-stats.component.html b/UI/Web/src/app/statistics/_components/publication-status-stats/publication-status-stats.component.html index 054507001..accf312f4 100644 --- a/UI/Web/src/app/statistics/_components/publication-status-stats/publication-status-stats.component.html +++ b/UI/Web/src/app/statistics/_components/publication-status-stats/publication-status-stats.component.html @@ -1,7 +1,6 @@

Publication Status -

@@ -14,8 +13,6 @@
- - diff --git a/UI/Web/src/app/statistics/_components/read-by-day-and/read-by-day-and.component.html b/UI/Web/src/app/statistics/_components/read-by-day-and/read-by-day-and.component.html index 229864efa..6e5dad93f 100644 --- a/UI/Web/src/app/statistics/_components/read-by-day-and/read-by-day-and.component.html +++ b/UI/Web/src/app/statistics/_components/read-by-day-and/read-by-day-and.component.html @@ -30,8 +30,9 @@ [showGridLines]="false" [showRefLines]="true" [roundDomains]="true" + [autoScale]="true" xAxisLabel="Time" - yAxisLabel="Reading Events" + yAxisLabel="Reading Activity" [timeline]="false" [results]="data" > diff --git a/UI/Web/src/app/statistics/_components/stat-list/stat-list.component.html b/UI/Web/src/app/statistics/_components/stat-list/stat-list.component.html index 16e7190f5..5367b2b23 100644 --- a/UI/Web/src/app/statistics/_components/stat-list/stat-list.component.html +++ b/UI/Web/src/app/statistics/_components/stat-list/stat-list.component.html @@ -9,7 +9,7 @@ - {{item.name}} {{item.value}} {{label}} + {{item.name}} {{item.value | compactNumber}} {{label}} diff --git a/openapi.json b/openapi.json index dc2bb178a..9439ff99f 100644 --- a/openapi.json +++ b/openapi.json @@ -7,7 +7,7 @@ "name": "GPL-3.0", "url": "https://github.com/Kareadita/Kavita/blob/develop/LICENSE" }, - "version": "0.6.1.16" + "version": "0.6.1.17" }, "servers": [ {