Cutlery/Kavita
1
0
Fork 0
mirror of https://github.com/Kareadita/Kavita.git synced 2025-05-24 00:52:23 -04:00
Code Issues Packages Projects Releases Wiki Activity

Stats Rework (#765)

Browse Source 
* Fixed a duplicate check for updates. Changed checking from weekly to daily.

* Refactored how dark variables were accessed to reduce size of component css. Refactored Stats code to use lesser information for reporting.

* Use the installId from the database which is most unlikely to change.

* Fixed a missing interface with stat service

* Added DotnetVersion back into collection

* Updated url to new host.
This commit is contained in:
Joseph Milazzo 2021-11-16 15:11:17 -06:00 committed by GitHub
parent b2831c7606
commit 1ada34984f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 44 additions and 349 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
API/Controllers/ServerController.cs
View File

@ -26,10 +26,11 @@ namespace API.Controllers
private readonly IArchiveService _archiveService;
private readonly ICacheService _cacheService;
private readonly IVersionUpdaterService _versionUpdaterService;
private readonly IStatsService _statsService;
public ServerController(IHostApplicationLifetime applicationLifetime, ILogger<ServerController> logger, IConfiguration config,
IBackupService backupService, IArchiveService archiveService, ICacheService cacheService,
IVersionUpdaterService versionUpdaterService)
IVersionUpdaterService versionUpdaterService, IStatsService statsService)
{
_applicationLifetime = applicationLifetime;
_logger = logger;
@ -38,6 +39,7 @@ namespace API.Controllers
_archiveService = archiveService;
_cacheService = cacheService;
_versionUpdaterService = versionUpdaterService;
_statsService = statsService;
}
/// <summary>
@ -84,9 +86,9 @@ namespace API.Controllers
/// </summary>
/// <returns></returns>
[HttpGet("server-info")]
public ActionResult<ServerInfoDto> GetVersion()
public async Task<ActionResult<ServerInfoDto>> GetVersion()
{
return Ok(StatsService.GetServerInfo());
return Ok(await _statsService.GetServerInfo());
}
[HttpGet("logs")]

39
API/Controllers/StatsController.cs
View File

@ -1,39 +0,0 @@
using System;
using System.Threading.Tasks;
using API.DTOs.Stats;
using API.Interfaces.Services;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
namespace API.Controllers
{
public class StatsController : BaseApiController
{
private readonly ILogger<StatsController> _logger;
private readonly IStatsService _statsService;
public StatsController(ILogger<StatsController> logger, IStatsService statsService)
{
_logger = logger;
_statsService = statsService;
}
[AllowAnonymous]
[HttpPost("client-info")]
public async Task<IActionResult> AddClientInfo([FromBody] ClientInfoDto clientInfoDto)
{
try
{
await _statsService.RecordClientInfo(clientInfoDto);
return Ok();
}
catch (Exception ex)
{
_logger.LogError(ex, "Error updating the usage statistics");
throw;
}
}
}
}

37
API/DTOs/Stats/ClientInfoDto.cs
View File

@ -1,37 +0,0 @@
using System;
namespace API.DTOs.Stats
{
public class ClientInfoDto
{
public ClientInfoDto()
{
CollectedAt = DateTime.UtcNow;
}
public string KavitaUiVersion { get; set; }
public string ScreenResolution { get; set; }
public string PlatformType { get; set; }
public DetailsVersion Browser { get; set; }
public DetailsVersion Os { get; set; }
public DateTime? CollectedAt { get; set; }
public bool UsingDarkTheme { get; set; }
public bool IsTheSameDevice(ClientInfoDto clientInfoDto)
{
return (clientInfoDto.ScreenResolution ?? string.Empty).Equals(ScreenResolution) &&
(clientInfoDto.PlatformType ?? string.Empty).Equals(PlatformType) &&
(clientInfoDto.Browser?.Name ?? string.Empty).Equals(Browser?.Name) &&
(clientInfoDto.Os?.Name ?? string.Empty).Equals(Os?.Name) &&
clientInfoDto.CollectedAt.GetValueOrDefault().ToString("yyyy-MM-dd")
.Equals(CollectedAt.GetValueOrDefault().ToString("yyyy-MM-dd"));
}
}
public class DetailsVersion
{
public string Name { get; set; }
public string Version { get; set; }
}
}

8
API/DTOs/Stats/ServerInfoDto.cs
View File

@ -2,13 +2,11 @@
{
public class ServerInfoDto
{
public string InstallId { get; set; }
public string Os { get; set; }
public string DotNetVersion { get; set; }
public string RunTimeVersion { get; set; }
public string KavitaVersion { get; set; }
public string BuildBranch { get; set; }
public string Culture { get; set; }
public bool IsDocker { get; set; }
public string DotnetVersion { get; set; }
public string KavitaVersion { get; set; }
public int NumOfCores { get; set; }
}
}

24
API/DTOs/Stats/UsageInfoDto.cs
View File

@ -1,24 +0,0 @@
using System.Collections.Generic;
using API.Entities.Enums;
namespace API.DTOs.Stats
{
public class UsageInfoDto
{
public UsageInfoDto()
{
FileTypes = new HashSet<string>();
LibraryTypesCreated = new HashSet<LibInfo>();
}
public int UsersCount { get; set; }
public IEnumerable<string> FileTypes { get; set; }
public IEnumerable<LibInfo> LibraryTypesCreated { get; set; }
}
public class LibInfo
{
public LibraryType Type { get; set; }
public int Count { get; set; }
}
}

33
API/DTOs/Stats/UsageStatisticsDto.cs
View File

@ -1,33 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
namespace API.DTOs.Stats
{
public class UsageStatisticsDto
{
public UsageStatisticsDto()
{
MarkAsUpdatedNow();
ClientsInfo = new List<ClientInfoDto>();
}
public string InstallId { get; set; }
public DateTime LastUpdate { get; set; }
public UsageInfoDto UsageInfo { get; set; }
public ServerInfoDto ServerInfo { get; set; }
public List<ClientInfoDto> ClientsInfo { get; set; }
public void MarkAsUpdatedNow()
{
LastUpdate = DateTime.UtcNow;
}
public void AddClientInfo(ClientInfoDto clientInfoDto)
{
if (ClientsInfo.Any(x => x.IsTheSameDevice(clientInfoDto))) return;
ClientsInfo.Add(clientInfoDto);
}
}
}

2
API/Interfaces/Services/IStatsService.cs
View File

@ -5,7 +5,7 @@ namespace API.Interfaces.Services
{
public interface IStatsService
{
Task RecordClientInfo(ClientInfoDto clientInfoDto);
Task Send();
Task<ServerInfoDto> GetServerInfo();
}
}

1
API/Services/DirectoryService.cs
View File

@ -21,7 +21,6 @@ namespace API.Services
public static readonly string CacheDirectory = Path.Join(Directory.GetCurrentDirectory(), "config", "cache");
public static readonly string CoverImageDirectory = Path.Join(Directory.GetCurrentDirectory(), "config", "covers");
public static readonly string BackupDirectory = Path.Join(Directory.GetCurrentDirectory(), "config", "backups");
public static readonly string StatsDirectory = Path.Join(Directory.GetCurrentDirectory(), "config", "stats");
public static readonly string ConfigDirectory = Path.Join(Directory.GetCurrentDirectory(), "config");
public DirectoryService(ILogger<DirectoryService> logger)

8
API/Services/TaskScheduler.cs
View File

@ -74,8 +74,6 @@ namespace API.Services
RecurringJob.AddOrUpdate("cleanup", () => _cleanupService.Cleanup(), Cron.Daily, TimeZoneInfo.Local);
// Schedule update check between noon and 6pm local time
RecurringJob.AddOrUpdate("check-for-updates", () => _scannerService.ScanLibraries(), Cron.Daily(Rnd.Next(12, 18)), TimeZoneInfo.Local);
}
#region StatsTasks
@ -91,7 +89,7 @@ namespace API.Services
}
_logger.LogDebug("Scheduling stat collection daily");
RecurringJob.AddOrUpdate(SendDataTask, () => _statsService.Send(), Cron.Daily, TimeZoneInfo.Local);
RecurringJob.AddOrUpdate(SendDataTask, () => _statsService.Send(), Cron.Daily(Rnd.Next(0, 22)), TimeZoneInfo.Local);
}
public void CancelStatsTasks()
@ -114,8 +112,8 @@ namespace API.Services
public void ScheduleUpdaterTasks()
{
_logger.LogInformation("Scheduling Auto-Update tasks");
RecurringJob.AddOrUpdate("check-updates", () => CheckForUpdate(), Cron.Weekly, TimeZoneInfo.Local);
// Schedule update check between noon and 6pm local time
RecurringJob.AddOrUpdate("check-updates", () => _versionUpdaterService.CheckForUpdate(), Cron.Daily(Rnd.Next(12, 18)), TimeZoneInfo.Local);
}
#endregion

150
API/Services/Tasks/StatsService.cs
View File

@ -1,44 +1,29 @@
using System;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Runtime.InteropServices;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using API.Data;
using API.DTOs.Stats;
using API.Entities.Enums;
using API.Interfaces;
using API.Interfaces.Services;
using Flurl.Http;
using Hangfire;
using Kavita.Common;
using Kavita.Common.EnvironmentInfo;
using Microsoft.AspNetCore.Http;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace API.Services.Tasks
{
public class StatsService : IStatsService
{
private const string StatFileName = "app_stats.json";
private readonly DataContext _dbContext;
private readonly ILogger<StatsService> _logger;
private readonly IUnitOfWork _unitOfWork;
#pragma warning disable S1075
private const string ApiUrl = "http://stats.kavitareader.com";
private const string ApiUrl = "http://stats2.kavitareader.com";
#pragma warning restore S1075
private static readonly string StatsFilePath = Path.Combine(DirectoryService.StatsDirectory, StatFileName);
private static bool FileExists => File.Exists(StatsFilePath);
public StatsService(DataContext dbContext, ILogger<StatsService> logger,
IUnitOfWork unitOfWork)
public StatsService(ILogger<StatsService> logger, IUnitOfWork unitOfWork)
{
_dbContext = dbContext;
_logger = logger;
_unitOfWork = unitOfWork;
}
@ -55,17 +40,7 @@ namespace API.Services.Tasks
return;
}
var rnd = new Random();
var offset = rnd.Next(0, 6);
if (offset == 0)
{
await SendData();
}
else
{
_logger.LogInformation("KavitaStats upload has been schedule to run in {Offset} hours", offset);
BackgroundJob.Schedule(() => SendData(), DateTimeOffset.Now.AddHours(offset));
}
await SendData();
}
/// <summary>
@ -74,55 +49,21 @@ namespace API.Services.Tasks
// ReSharper disable once MemberCanBePrivate.Global
public async Task SendData()
{
await CollectRelevantData();
await FinalizeStats();
var data = await GetServerInfo();
await SendDataToStatsServer(data);
}
public async Task RecordClientInfo(ClientInfoDto clientInfoDto)
{
var statisticsDto = await GetData();
statisticsDto.AddClientInfo(clientInfoDto);
await SaveFile(statisticsDto);
}
private async Task CollectRelevantData()
{
var usageInfo = await GetUsageInfo();
var serverInfo = GetServerInfo();
await PathData(serverInfo, usageInfo);
}
private async Task FinalizeStats()
{
try
{
var data = await GetExistingData<UsageStatisticsDto>();
var successful = await SendDataToStatsServer(data);
if (successful)
{
ResetStats();
}
}
catch (Exception ex)
{
_logger.LogError(ex, "There was an exception while sending data to KavitaStats");
}
}
private async Task<bool> SendDataToStatsServer(UsageStatisticsDto data)
private async Task SendDataToStatsServer(ServerInfoDto data)
{
var responseContent = string.Empty;
try
{
var response = await (ApiUrl + "/api/InstallationStats")
var response = await (ApiUrl + "/api/v2/stats")
.WithHeader("Accept", "application/json")
.WithHeader("User-Agent", "Kavita")
.WithHeader("x-api-key", "MsnvA2DfQqxSK5jh")
.WithHeader("api-key", "MsnvA2DfQqxSK5jh")
.WithHeader("x-kavita-version", BuildInfo.Version)
.WithTimeout(TimeSpan.FromSeconds(30))
.PostJsonAsync(data);
@ -130,10 +71,7 @@ namespace API.Services.Tasks
if (response.StatusCode != StatusCodes.Status200OK)
{
_logger.LogError("KavitaStats did not respond successfully. {Content}", response);
return false;
}
return true;
}
catch (HttpRequestException e)
{
@ -149,84 +87,22 @@ namespace API.Services.Tasks
{
_logger.LogError(e, "An error happened during the request to KavitaStats");
}
return false;
}
private static void ResetStats()
{
if (FileExists) File.Delete(StatsFilePath);
}
private async Task PathData(ServerInfoDto serverInfoDto, UsageInfoDto usageInfoDto)
{
var data = await GetData();
data.ServerInfo = serverInfoDto;
data.UsageInfo = usageInfoDto;
data.MarkAsUpdatedNow();
await SaveFile(data);
}
private static async ValueTask<UsageStatisticsDto> GetData()
{
if (!FileExists) return new UsageStatisticsDto {InstallId = HashUtil.AnonymousToken()};
return await GetExistingData<UsageStatisticsDto>();
}
private async Task<UsageInfoDto> GetUsageInfo()
{
var usersCount = await _dbContext.Users.CountAsync();
var libsCountByType = await _dbContext.Library
.AsNoTracking()
.GroupBy(x => x.Type)
.Select(x => new LibInfo {Type = x.Key, Count = x.Count()})
.ToArrayAsync();
var uniqueFileTypes = await _unitOfWork.FileRepository.GetFileExtensions();
var usageInfo = new UsageInfoDto
{
UsersCount = usersCount,
LibraryTypesCreated = libsCountByType,
FileTypes = uniqueFileTypes
};
return usageInfo;
}
public static ServerInfoDto GetServerInfo()
public async Task<ServerInfoDto> GetServerInfo()
{
var installId = await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.InstallId);
var serverInfo = new ServerInfoDto
{
InstallId = installId.Value,
Os = RuntimeInformation.OSDescription,
DotNetVersion = Environment.Version.ToString(),
RunTimeVersion = RuntimeInformation.FrameworkDescription,
KavitaVersion = BuildInfo.Version.ToString(),
Culture = Thread.CurrentThread.CurrentCulture.Name,
BuildBranch = BuildInfo.Branch,
DotnetVersion = Environment.Version.ToString(),
IsDocker = new OsInfo(Array.Empty<IOsVersionAdapter>()).IsDocker,
NumOfCores = Environment.ProcessorCount
NumOfCores = Math.Max(Environment.ProcessorCount, 1)
};
return serverInfo;
}
private static async Task<T> GetExistingData<T>()
{
var json = await File.ReadAllTextAsync(StatsFilePath);
return JsonSerializer.Deserialize<T>(json);
}
private static async Task SaveFile(UsageStatisticsDto statisticsDto)
{
DirectoryService.ExistOrCreate(DirectoryService.StatsDirectory);
await File.WriteAllTextAsync(StatsFilePath, JsonSerializer.Serialize(statisticsDto));
}
}
}

41
UI/Web/src/app/_services/stats.service.ts
View File

@ -1,41 +0,0 @@
import { HttpClient } from "@angular/common/http";
import { Injectable } from "@angular/core";
import * as Bowser from "bowser";
import { take } from "rxjs/operators";
import { environment } from "src/environments/environment";
import { ClientInfo } from "../_models/stats/client-info";
import { DetailsVersion } from "../_models/stats/details-version";
import { NavService } from "./nav.service";
import { version } from '../../../package.json';
@Injectable({
providedIn: 'root'
})
export class StatsService {
baseUrl = environment.apiUrl;
constructor(private httpClient: HttpClient, private navService: NavService) { }
public sendClientInfo(data: ClientInfo) {
return this.httpClient.post(this.baseUrl + 'stats/client-info', data);
}
public async getInfo(): Promise<ClientInfo> {
const screenResolution = `${window.screen.width} x ${window.screen.height}`;
const browser = Bowser.getParser(window.navigator.userAgent);
const usingDarkTheme = await this.navService.darkMode$.pipe(take(1)).toPromise();
return {
os: browser.getOS() as DetailsVersion,
browser: browser.getBrowser() as DetailsVersion,
platformType: browser.getPlatformType(),
kavitaUiVersion: version,
screenResolution,
usingDarkTheme
};
}
}

9
UI/Web/src/app/app.component.ts
View File

@ -5,7 +5,6 @@ import { AccountService } from './_services/account.service';
import { LibraryService } from './_services/library.service';
import { MessageHubService } from './_services/message-hub.service';
import { NavService } from './_services/nav.service';
import { StatsService } from './_services/stats.service';
import { filter } from 'rxjs/operators';
import { NgbModal } from '@ng-bootstrap/ng-bootstrap';
@ -17,8 +16,8 @@ import { NgbModal } from '@ng-bootstrap/ng-bootstrap';
export class AppComponent implements OnInit {
constructor(private accountService: AccountService, public navService: NavService,
private statsService: StatsService, private messageHub: MessageHubService,
private libraryService: LibraryService, private router: Router, private ngbModal: NgbModal) {
private messageHub: MessageHubService, private libraryService: LibraryService,
private router: Router, private ngbModal: NgbModal) {
// Close any open modals when a route change occurs
router.events
@ -32,10 +31,6 @@ export class AppComponent implements OnInit {
ngOnInit(): void {
this.setCurrentUser();
this.statsService.getInfo().then(data => {
this.statsService.sendClientInfo(data).subscribe(() => {/* No Operation */});
});
}

1
UI/Web/src/app/cards/bulk-operations/bulk-operations.component.scss
View File

@ -1,5 +1,4 @@
@import "../../../theme/colors";
@import "../../../assets/themes/dark.scss";
.bulk-select {
background-color: $dark-form-background-no-opacity;

2
UI/Web/src/app/typeahead/typeahead.component.scss
View File

@ -1,4 +1,4 @@
@import '../../assets/themes/dark.scss';
@import "../../theme/colors";
input {
width: 15px;

15
UI/Web/src/assets/themes/dark.scss
View File

@ -1,17 +1,6 @@
// All dark style overrides should live here
$dark-bg-color: #343a40;
$dark-primary-color: rgba(74, 198, 148, 0.9);
$dark-text-color: #efefef;
$dark-hover-color: #4ac694;
$dark-form-background: rgba(1, 4, 9, 0.5);
$dark-form-background-no-opacity: rgb(1, 4, 9);
$dark-form-placeholder: #efefef;
$dark-link-color: rgb(88, 166, 255);
$dark-icon-color: white;
$dark-form-border: rgba(239, 239, 239, 0.125);
$dark-form-readonly: #434648;
$dark-item-accent-bg: #292d32;
@import "../../theme/colors";
.bg-dark {
color: $dark-text-color;
@ -177,7 +166,7 @@ $dark-item-accent-bg: #292d32;
.card {
background-color: $dark-bg-color;
color: $dark-text-color;
border-color: $dark-form-border; //#343a40
border-color: $dark-form-border;
}
.section-title {

2
UI/Web/src/styles.scss
View File

@ -32,7 +32,7 @@ label, select, .clickable {
@font-face {
font-family: "EBGarmond";
src: url(assets/fonts/EBGarmond/EBGaramond-VariableFont_wght.ttf) format("truetype");
src: url("assets/fonts/EBGarmond/EBGaramond-VariableFont_wght.ttf") format("truetype");
}
@font-face {

13
UI/Web/src/theme/_colors.scss
View File

@ -1,6 +1,19 @@
$primary-color: #4ac694; //(74,198,148)
$error-color: #ff4136; // #bb2929 good color for contrast rating
$dark-bg-color: #343a40;
$dark-primary-color: rgba(74, 198, 148, 0.9);
$dark-text-color: #efefef;
$dark-hover-color: #4ac694;
$dark-form-background: rgba(1, 4, 9, 0.5);
$dark-form-background-no-opacity: rgb(1, 4, 9);
$dark-form-placeholder: #efefef;
$dark-link-color: rgb(88, 166, 255);
$dark-icon-color: white;
$dark-form-border: rgba(239, 239, 239, 0.125);
$dark-form-readonly: #434648;
$dark-item-accent-bg: #292d32;
$theme-colors: (
"primary": $primary-color,
"danger": $error-color