using System; using System.Collections.Generic; using System.Diagnostics; using System.Globalization; using System.Linq; using System.Net.Http; using System.Runtime.InteropServices; using System.Threading.Tasks; using API.Data; using API.Data.Misc; using API.Data.Repositories; using API.DTOs.Stats; using API.DTOs.Stats.V3; using API.Entities; using API.Entities.Enums; using API.Services.Plus; using Flurl.Http; using Kavita.Common.EnvironmentInfo; using Kavita.Common.Helpers; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Identity; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; namespace API.Services.Tasks; #nullable enable public interface IStatsService { Task Send(); Task GetServerInfoSlim(); Task SendCancellation(); } /// /// This is for reporting to the stat server /// public class StatsService : IStatsService { private readonly ILogger _logger; private readonly IUnitOfWork _unitOfWork; private readonly DataContext _context; private readonly ILicenseService _licenseService; private readonly UserManager _userManager; private readonly IEmailService _emailService; private readonly ICacheService _cacheService; private const string ApiUrl = "https://stats.kavitareader.com"; private const string ApiKey = "MsnvA2DfQqxSK5jh"; // It's not important this is public, just a way to keep bots from hitting the API willy-nilly public StatsService(ILogger logger, IUnitOfWork unitOfWork, DataContext context, ILicenseService licenseService, UserManager userManager, IEmailService emailService, ICacheService cacheService) { _logger = logger; _unitOfWork = unitOfWork; _context = context; _licenseService = licenseService; _userManager = userManager; _emailService = emailService; _cacheService = cacheService; FlurlHttp.ConfigureClient(ApiUrl, cli => cli.Settings.HttpClientFactory = new UntrustedCertClientFactory()); } /// /// Due to all instances firing this at the same time, we can DDOS our server. This task when fired will schedule the task to be run /// randomly over a six-hour spread /// public async Task Send() { var allowStatCollection = (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).AllowStatCollection; if (!allowStatCollection) { return; } await SendData(); } /// /// This must be public for Hangfire. Do not call this directly. /// // ReSharper disable once MemberCanBePrivate.Global public async Task SendData() { var sw = Stopwatch.StartNew(); var data = await GetStatV3Payload(); _logger.LogDebug("Collecting stats took {Time} ms", sw.ElapsedMilliseconds); sw.Stop(); await SendDataToStatsServer(data); } private async Task SendDataToStatsServer(ServerInfoV3Dto data) { var responseContent = string.Empty; try { var response = await (ApiUrl + "/api/v3/stats") .WithHeader("Accept", "application/json") .WithHeader("User-Agent", "Kavita") .WithHeader("x-api-key", ApiKey) .WithHeader("x-kavita-version", BuildInfo.Version) .WithHeader("Content-Type", "application/json") .WithTimeout(TimeSpan.FromSeconds(30)) .PostJsonAsync(data); if (response.StatusCode != StatusCodes.Status200OK) { _logger.LogError("KavitaStats did not respond successfully. {Content}", response); } } catch (HttpRequestException e) { var info = new { dataSent = data, response = responseContent }; _logger.LogError(e, "KavitaStats did not respond successfully. {Content}", info); } catch (Exception e) { _logger.LogError(e, "An error happened during the request to KavitaStats"); } } public async Task GetServerInfoSlim() { var serverSettings = await _unitOfWork.SettingsRepository.GetSettingsDtoAsync(); return new ServerInfoSlimDto() { InstallId = serverSettings.InstallId, KavitaVersion = serverSettings.InstallVersion, IsDocker = OsInfo.IsDocker, FirstInstallDate = serverSettings.FirstInstallDate, FirstInstallVersion = serverSettings.FirstInstallVersion }; } public async Task SendCancellation() { _logger.LogInformation("Informing KavitaStats that this instance is no longer sending stats"); var installId = (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).InstallId; var responseContent = string.Empty; try { var response = await (ApiUrl + "/api/v2/stats/opt-out?installId=" + installId) .WithHeader("Accept", "application/json") .WithHeader("User-Agent", "Kavita") .WithHeader("x-api-key", ApiKey) .WithHeader("x-kavita-version", BuildInfo.Version) .WithHeader("Content-Type", "application/json") .WithTimeout(TimeSpan.FromSeconds(30)) .PostAsync(); if (response.StatusCode != StatusCodes.Status200OK) { _logger.LogError("KavitaStats did not respond successfully. {Content}", response); } } catch (HttpRequestException e) { _logger.LogError(e, "KavitaStats did not respond successfully. {Response}", responseContent); } catch (Exception e) { _logger.LogError(e, "An error happened during the request to KavitaStats"); } } private static async Task PingStatsApi() { try { var sw = Stopwatch.StartNew(); var response = await (ApiUrl + "/api/health/") .WithHeader("Accept", "application/json") .WithHeader("User-Agent", "Kavita") .WithHeader("x-api-key", ApiKey) .WithHeader("x-kavita-version", BuildInfo.Version) .WithHeader("Content-Type", "application/json") .WithTimeout(TimeSpan.FromSeconds(30)) .GetAsync(); if (response.StatusCode == StatusCodes.Status200OK) { sw.Stop(); return sw.ElapsedMilliseconds; } } catch (Exception) { /* Swallow */ } return 0; } private async Task MaxSeriesInAnyLibrary() { // If first time flow, just return 0 if (!await _context.Series.AnyAsync()) return 0; return await _context.Series .Select(s => _context.Library.Where(l => l.Id == s.LibraryId).SelectMany(l => l.Series!).Count()) .MaxAsync(); } private async Task MaxVolumesInASeries() { // If first time flow, just return 0 if (!await _context.Volume.AnyAsync()) return 0; return await _context.Volume .Select(v => new { v.SeriesId, Count = _context.Series.Where(s => s.Id == v.SeriesId).SelectMany(s => s.Volumes!).Count() }) .AsNoTracking() .AsSplitQuery() .MaxAsync(d => d.Count); } private async Task MaxChaptersInASeries() { // If first time flow, just return 0 if (!await _context.Chapter.AnyAsync()) return 0; return await _context.Series .AsNoTracking() .AsSplitQuery() .MaxAsync(s => s.Volumes! .Where(v => v.MinNumber == 0) .SelectMany(v => v.Chapters!) .Count()); } private async Task GetStatV3Payload() { var serverSettings = await _unitOfWork.SettingsRepository.GetSettingsDtoAsync(); var dto = new ServerInfoV3Dto() { InstallId = serverSettings.InstallId, KavitaVersion = serverSettings.InstallVersion, InitialKavitaVersion = serverSettings.FirstInstallVersion, InitialInstallDate = (DateTime)serverSettings.FirstInstallDate!, IsDocker = OsInfo.IsDocker, Os = RuntimeInformation.OSDescription, NumOfCores = Math.Max(Environment.ProcessorCount, 1), DotnetVersion = Environment.Version.ToString(), OpdsEnabled = serverSettings.EnableOpds, EncodeMediaAs = serverSettings.EncodeMediaAs, }; dto.OsLocale = CultureInfo.CurrentCulture.EnglishName; dto.LastReadTime = await _unitOfWork.AppUserProgressRepository.GetLatestProgress(); dto.MaxSeriesInALibrary = await MaxSeriesInAnyLibrary(); dto.MaxVolumesInASeries = await MaxVolumesInASeries(); dto.MaxChaptersInASeries = await MaxChaptersInASeries(); dto.TotalFiles = await _unitOfWork.LibraryRepository.GetTotalFiles(); dto.TotalGenres = await _unitOfWork.GenreRepository.GetCountAsync(); dto.TotalPeople = await _unitOfWork.PersonRepository.GetCountAsync(); dto.TotalSeries = await _unitOfWork.SeriesRepository.GetCountAsync(); dto.TotalLibraries = (await _unitOfWork.LibraryRepository.GetLibrariesAsync()).Count(); dto.NumberOfCollections = (await _unitOfWork.CollectionTagRepository.GetAllCollectionsAsync()).Count(); dto.NumberOfReadingLists = await _unitOfWork.ReadingListRepository.Count(); try { var license = (await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.LicenseKey)).Value; dto.ActiveKavitaPlusSubscription = await _licenseService.HasActiveSubscription(license); } catch (Exception) { dto.ActiveKavitaPlusSubscription = false; } // Find a random cbz/zip file and open it for reading await OpenRandomFile(dto); dto.TimeToPingKavitaStatsApi = await PingStatsApi(); #region Relationships dto.Relationships = await _context.SeriesRelation .GroupBy(sr => sr.RelationKind) .Select(g => new RelationshipStatV3 { Relationship = g.Key, Count = g.Count() }) .ToListAsync(); #endregion #region Libraries var allLibraries = (await _unitOfWork.LibraryRepository.GetLibrariesAsync(LibraryIncludes.Folders | LibraryIncludes.FileTypes | LibraryIncludes.ExcludePatterns | LibraryIncludes.AppUser)).ToList(); dto.Libraries ??= []; foreach (var library in allLibraries) { var libDto = new LibraryStatV3(); libDto.IncludeInDashboard = library.IncludeInDashboard; libDto.IncludeInSearch = library.IncludeInSearch; libDto.LastScanned = library.LastScanned; libDto.NumberOfFolders = library.Folders.Count; libDto.FileTypes = library.LibraryFileTypes.Select(s => s.FileTypeGroup).Distinct().ToList(); libDto.UsingExcludePatterns = library.LibraryExcludePatterns.Any(p => !string.IsNullOrEmpty(p.Pattern)); libDto.UsingFolderWatching = library.FolderWatching; libDto.CreateCollectionsFromMetadata = library.ManageCollections; libDto.CreateReadingListsFromMetadata = library.ManageReadingLists; dto.Libraries.Add(libDto); } #endregion #region Users // Create a dictionary mapping user IDs to the libraries they have access to var userLibraryAccess = allLibraries .SelectMany(l => l.AppUsers.Select(appUser => new { l, appUser.Id })) .GroupBy(x => x.Id) .ToDictionary(g => g.Key, g => g.Select(x => x.l).ToList()); dto.Users ??= []; var allUsers = await _unitOfWork.UserRepository.GetAllUsersAsync(AppUserIncludes.UserPreferences | AppUserIncludes.ReadingLists | AppUserIncludes.Bookmarks | AppUserIncludes.Collections | AppUserIncludes.Devices | AppUserIncludes.Progress | AppUserIncludes.Ratings | AppUserIncludes.SmartFilters | AppUserIncludes.WantToRead, false); foreach (var user in allUsers) { var userDto = new UserStatV3(); userDto.HasMALToken = !string.IsNullOrEmpty(user.MalAccessToken); userDto.HasAniListToken = !string.IsNullOrEmpty(user.AniListAccessToken); userDto.AgeRestriction = new AgeRestriction() { AgeRating = user.AgeRestriction, IncludeUnknowns = user.AgeRestrictionIncludeUnknowns }; userDto.Locale = user.UserPreferences.Locale; userDto.Roles = [.. _userManager.GetRolesAsync(user).Result]; userDto.LastLogin = user.LastActiveUtc; userDto.HasValidEmail = user.Email != null && _emailService.IsValidEmail(user.Email); userDto.IsEmailConfirmed = user.EmailConfirmed; userDto.ActiveTheme = user.UserPreferences.Theme.Name; userDto.CollectionsCreatedCount = user.Collections.Count; userDto.ReadingListsCreatedCount = user.ReadingLists.Count; userDto.LastReadTime = user.Progresses .Select(p => p.LastModifiedUtc) .DefaultIfEmpty() .Max(); userDto.DevicePlatforms = user.Devices.Select(d => d.Platform).ToList(); userDto.SeriesBookmarksCreatedCount = user.Bookmarks.Count; userDto.SmartFilterCreatedCount = user.SmartFilters.Count; userDto.WantToReadSeriesCount = user.WantToRead.Count; if (allLibraries.Count > 0 && userLibraryAccess.TryGetValue(user.Id, out var accessibleLibraries)) { userDto.PercentageOfLibrariesHasAccess = (1f * accessibleLibraries.Count) / allLibraries.Count; } else { userDto.PercentageOfLibrariesHasAccess = 0; } dto.Users.Add(userDto); } #endregion return dto; } private async Task OpenRandomFile(ServerInfoV3Dto dto) { var random = new Random(); List extensions = [".cbz", ".zip"]; // Count the total number of files that match the criteria var count = await _context.MangaFile.AsNoTracking() .Where(r => r.Extension != null && extensions.Contains(r.Extension)) .CountAsync(); if (count == 0) { dto.TimeToOpeCbzMs = 0; dto.TimeToOpenCbzPages = 0; return; } // Generate a random skip value var skip = random.Next(count); // Fetch the random file var randomFile = await _context.MangaFile.AsNoTracking() .Where(r => r.Extension != null && extensions.Contains(r.Extension)) .Skip(skip) .Take(1) .FirstAsync(); var sw = Stopwatch.StartNew(); await _cacheService.Ensure(randomFile.ChapterId); var time = sw.ElapsedMilliseconds; sw.Stop(); dto.TimeToOpeCbzMs = time; dto.TimeToOpenCbzPages = randomFile.Pages; } }