From c809597cf058e99cfb842b31adcaac4640d7f5a5 Mon Sep 17 00:00:00 2001 From: Joseph Milazzo Date: Mon, 27 Jun 2022 10:23:32 -0500 Subject: [PATCH] More Stat collection (#1337) * Ensure that Scan Series triggers a file analysis task. * Tweaked concurrency for Analyze Files * Implemented new stats tracking for upcoming performance release. --- API/DTOs/Stats/ServerInfoDto.cs | 81 ++++++++++++++++++- API/Data/Repositories/GenreRepository.cs | 6 ++ API/Data/Repositories/PersonRepository.cs | 6 ++ API/Data/Repositories/UserRepository.cs | 7 ++ .../Metadata/WordCountAnalyzerService.cs | 4 +- API/Services/Tasks/ScannerService.cs | 1 + API/Services/Tasks/StatsService.cs | 61 +++++++++++++- 7 files changed, 157 insertions(+), 9 deletions(-) diff --git a/API/DTOs/Stats/ServerInfoDto.cs b/API/DTOs/Stats/ServerInfoDto.cs index 45a73236b..4b037a108 100644 --- a/API/DTOs/Stats/ServerInfoDto.cs +++ b/API/DTOs/Stats/ServerInfoDto.cs @@ -2,49 +2,122 @@ namespace API.DTOs.Stats { + /// + /// Represents information about a Kavita Installation + /// public class ServerInfoDto { + /// + /// Unique Id that represents a unique install + /// public string InstallId { get; set; } public string Os { get; set; } + /// + /// If the Kavita install is using Docker + /// public bool IsDocker { get; set; } + /// + /// Version of .NET instance is running + /// public string DotnetVersion { get; set; } + /// + /// Version of Kavita + /// public string KavitaVersion { get; set; } + /// + /// Number of Cores on the instance + /// public int NumOfCores { get; set; } + /// + /// The number of libraries on the instance + /// public int NumberOfLibraries { get; set; } + /// + /// Does any user have bookmarks + /// public bool HasBookmarks { get; set; } /// /// The site theme the install is using /// + /// Introduced in v0.5.2 public string ActiveSiteTheme { get; set; } - /// /// The reading mode the main user has as a preference /// + /// Introduced in v0.5.2 public ReaderMode MangaReaderMode { get; set; } /// /// Number of users on the install /// + /// Introduced in v0.5.2 public int NumberOfUsers { get; set; } /// /// Number of collections on the install /// + /// Introduced in v0.5.2 public int NumberOfCollections { get; set; } - /// /// Number of reading lists on the install (Sum of all users) /// + /// Introduced in v0.5.2 public int NumberOfReadingLists { get; set; } - /// /// Is OPDS enabled /// + /// Introduced in v0.5.2 public bool OPDSEnabled { get; set; } - /// /// Total number of files in the instance /// + /// Introduced in v0.5.2 public int TotalFiles { get; set; } + /// + /// Total number of Genres in the instance + /// + /// Introduced in v0.5.4 + public int TotalGenres { get; set; } + /// + /// Total number of People in the instance + /// + /// Introduced in v0.5.4 + public int TotalPeople { get; set; } + /// + /// Is this instance storing bookmarks as WebP + /// + /// Introduced in v0.5.4 + public bool StoreBookmarksAsWebP { get; set; } + /// + /// Number of users on this instance using Card Layout + /// + /// Introduced in v0.5.4 + public int UsersOnCardLayout { get; set; } + /// + /// Number of users on this instance using List Layout + /// + /// Introduced in v0.5.4 + public int UsersOnListLayout { get; set; } + /// + /// Max number of Series for any library on the instance + /// + /// Introduced in v0.5.4 + public int MaxSeriesInALibrary { get; set; } + /// + /// Max number of Volumes for any library on the instance + /// + /// Introduced in v0.5.4 + public int MaxVolumesInASeries { get; set; } + /// + /// Max number of Chapters for any library on the instance + /// + /// Introduced in v0.5.4 + public int MaxChaptersInASeries { get; set; } + /// + /// Does this instance have relationships setup between series + /// + /// Introduced in v0.5.4 + public bool UsingSeriesRelationships { get; set; } + } } diff --git a/API/Data/Repositories/GenreRepository.cs b/API/Data/Repositories/GenreRepository.cs index c2d5db2af..f1c9b84eb 100644 --- a/API/Data/Repositories/GenreRepository.cs +++ b/API/Data/Repositories/GenreRepository.cs @@ -18,6 +18,7 @@ public interface IGenreRepository Task> GetAllGenreDtosAsync(); Task RemoveAllGenreNoLongerAssociated(bool removeExternal = false); Task> GetAllGenreDtosForLibrariesAsync(IList libraryIds); + Task GetCountAsync(); } public class GenreRepository : IGenreRepository @@ -72,6 +73,11 @@ public class GenreRepository : IGenreRepository .ToListAsync(); } + public async Task GetCountAsync() + { + return await _context.Genre.CountAsync(); + } + public async Task> GetAllGenresAsync() { return await _context.Genre.ToListAsync(); diff --git a/API/Data/Repositories/PersonRepository.cs b/API/Data/Repositories/PersonRepository.cs index 371558459..98794670e 100644 --- a/API/Data/Repositories/PersonRepository.cs +++ b/API/Data/Repositories/PersonRepository.cs @@ -16,6 +16,7 @@ public interface IPersonRepository Task> GetAllPeople(); Task RemoveAllPeopleNoLongerAssociated(bool removeExternal = false); Task> GetAllPeopleDtosForLibrariesAsync(List libraryIds); + Task GetCountAsync(); } public class PersonRepository : IPersonRepository @@ -72,6 +73,11 @@ public class PersonRepository : IPersonRepository .ToListAsync(); } + public async Task GetCountAsync() + { + return await _context.Person.CountAsync(); + } + public async Task> GetAllPeople() { diff --git a/API/Data/Repositories/UserRepository.cs b/API/Data/Repositories/UserRepository.cs index dd9c279cd..c63603dbc 100644 --- a/API/Data/Repositories/UserRepository.cs +++ b/API/Data/Repositories/UserRepository.cs @@ -57,6 +57,7 @@ public interface IUserRepository Task> GetAllPreferencesByThemeAsync(int themeId); Task HasAccessToLibrary(int libraryId, int userId); + Task> GetAllUsersAsync(AppUserIncludes includeFlags); } public class UserRepository : IUserRepository @@ -246,6 +247,12 @@ public class UserRepository : IUserRepository .AnyAsync(library => library.AppUsers.Any(user => user.Id == userId)); } + public async Task> GetAllUsersAsync(AppUserIncludes includeFlags) + { + var query = AddIncludesToQuery(_context.Users.AsQueryable(), includeFlags); + return await query.ToListAsync(); + } + public async Task> GetAdminUsersAsync() { return await _userManager.GetUsersInRoleAsync(PolicyConstants.AdminRole); diff --git a/API/Services/Tasks/Metadata/WordCountAnalyzerService.cs b/API/Services/Tasks/Metadata/WordCountAnalyzerService.cs index 2b5327244..be6c6d0b4 100644 --- a/API/Services/Tasks/Metadata/WordCountAnalyzerService.cs +++ b/API/Services/Tasks/Metadata/WordCountAnalyzerService.cs @@ -18,7 +18,7 @@ namespace API.Services.Tasks.Metadata; public interface IWordCountAnalyzerService { [DisableConcurrentExecution(timeoutInSeconds: 60 * 60 * 60)] - [AutomaticRetry(Attempts = 0, OnAttemptsExceeded = AttemptsExceededAction.Delete)] + [AutomaticRetry(Attempts = 2, OnAttemptsExceeded = AttemptsExceededAction.Delete)] Task ScanLibrary(int libraryId, bool forceUpdate = false); Task ScanSeries(int libraryId, int seriesId, bool forceUpdate = true); } @@ -46,7 +46,7 @@ public class WordCountAnalyzerService : IWordCountAnalyzerService [DisableConcurrentExecution(timeoutInSeconds: 60 * 60 * 60)] - [AutomaticRetry(Attempts = 0, OnAttemptsExceeded = AttemptsExceededAction.Delete)] + [AutomaticRetry(Attempts = 2, OnAttemptsExceeded = AttemptsExceededAction.Delete)] public async Task ScanLibrary(int libraryId, bool forceUpdate = false) { var sw = Stopwatch.StartNew(); diff --git a/API/Services/Tasks/ScannerService.cs b/API/Services/Tasks/ScannerService.cs index 00caa39f2..33341f8e5 100644 --- a/API/Services/Tasks/ScannerService.cs +++ b/API/Services/Tasks/ScannerService.cs @@ -191,6 +191,7 @@ public class ScannerService : IScannerService await CleanupDbEntities(); BackgroundJob.Enqueue(() => _cacheService.CleanupChapters(chapterIds)); BackgroundJob.Enqueue(() => _metadataService.RefreshMetadataForSeries(libraryId, series.Id, false)); + BackgroundJob.Enqueue(() => _wordCountAnalyzerService.ScanSeries(libraryId, series.Id, false)); } private static void RemoveParsedInfosNotForSeries(Dictionary> parsedSeries, Series series) diff --git a/API/Services/Tasks/StatsService.cs b/API/Services/Tasks/StatsService.cs index 831e8f3a0..41bbb4a92 100644 --- a/API/Services/Tasks/StatsService.cs +++ b/API/Services/Tasks/StatsService.cs @@ -4,13 +4,15 @@ using System.Net.Http; using System.Runtime.InteropServices; using System.Threading.Tasks; using API.Data; +using API.Data.Repositories; using API.DTOs.Stats; -using API.DTOs.Theme; using API.Entities.Enums; +using API.Entities.Enums.UserPreferences; using Flurl.Http; using Kavita.Common.EnvironmentInfo; using Kavita.Common.Helpers; using Microsoft.AspNetCore.Http; +using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; namespace API.Services.Tasks; @@ -24,12 +26,14 @@ public class StatsService : IStatsService { private readonly ILogger _logger; private readonly IUnitOfWork _unitOfWork; + private readonly DataContext _context; private const string ApiUrl = "https://stats.kavitareader.com"; - public StatsService(ILogger logger, IUnitOfWork unitOfWork) + public StatsService(ILogger logger, IUnitOfWork unitOfWork, DataContext context) { _logger = logger; _unitOfWork = unitOfWork; + _context = context; FlurlHttp.ConfigureClient(ApiUrl, cli => cli.Settings.HttpClientFactory = new UntrustedCertClientFactory()); @@ -102,6 +106,8 @@ public class StatsService : IStatsService var installId = await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.InstallId); var installVersion = await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.InstallVersion); + var serverSettings = await _unitOfWork.SettingsRepository.GetSettingsDtoAsync(); + var serverInfo = new ServerInfoDto { InstallId = installId.Value, @@ -114,11 +120,24 @@ public class StatsService : IStatsService NumberOfLibraries = (await _unitOfWork.LibraryRepository.GetLibrariesAsync()).Count(), NumberOfCollections = (await _unitOfWork.CollectionTagRepository.GetAllTagsAsync()).Count(), NumberOfReadingLists = await _unitOfWork.ReadingListRepository.Count(), - OPDSEnabled = (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).EnableOpds, + OPDSEnabled = serverSettings.EnableOpds, NumberOfUsers = (await _unitOfWork.UserRepository.GetAllUsers()).Count(), TotalFiles = await _unitOfWork.LibraryRepository.GetTotalFiles(), + TotalGenres = await _unitOfWork.GenreRepository.GetCountAsync(), + TotalPeople = await _unitOfWork.PersonRepository.GetCountAsync(), + UsingSeriesRelationships = await GetIfUsingSeriesRelationship(), + StoreBookmarksAsWebP = serverSettings.ConvertBookmarkToWebP, + MaxSeriesInALibrary = await MaxSeriesInAnyLibrary(), + MaxVolumesInASeries = await MaxVolumesInASeries(), + MaxChaptersInASeries = await MaxChaptersInASeries(), }; + var usersWithPref = (await _unitOfWork.UserRepository.GetAllUsersAsync(AppUserIncludes.UserPreferences)).ToList(); + serverInfo.UsersOnCardLayout = + usersWithPref.Count(u => u.UserPreferences.GlobalPageLayoutMode == PageLayoutMode.Cards); + serverInfo.UsersOnListLayout = + usersWithPref.Count(u => u.UserPreferences.GlobalPageLayoutMode == PageLayoutMode.List); + var firstAdminUser = (await _unitOfWork.UserRepository.GetAdminUsersAsync()).FirstOrDefault(); if (firstAdminUser != null) @@ -132,4 +151,40 @@ public class StatsService : IStatsService return serverInfo; } + + private Task GetIfUsingSeriesRelationship() + { + return _context.SeriesRelation.AnyAsync(); + } + + private Task MaxSeriesInAnyLibrary() + { + return _context.Series + .Select(s => new + { + LibraryId = s.LibraryId, + Count = _context.Library.Where(l => l.Id == s.LibraryId).SelectMany(l => l.Series).Count() + }) + .MaxAsync(d => d.Count); + } + + private Task MaxVolumesInASeries() + { + return _context.Volume + .Select(v => new + { + v.SeriesId, + Count = _context.Series.Where(s => s.Id == v.SeriesId).SelectMany(s => s.Volumes).Count() + }) + .MaxAsync(d => d.Count); + } + + private Task MaxChaptersInASeries() + { + return _context.Series + .MaxAsync(s => s.Volumes + .Where(v => v.Number == 0) + .SelectMany(v => v.Chapters) + .Count()); + } }