Cutlery/Kavita
1
0
Fork 0
mirror of https://github.com/Kareadita/Kavita.git synced 2025-05-24 00:52:23 -04:00
Code Issues Packages Projects Releases Wiki Activity

OPDS Cleanup (#534)

Browse Source 
* Fixed opds url display

* Rewrote how stat collection works, now we check in multiple places and always run stat collection in a background thread, to not block main thread.

* Cleaned up the ParseInfoTest to be more verbose

* Added benchmarking
This commit is contained in:
Joseph Milazzo 2021-08-28 15:32:24 -07:00 committed by GitHub
parent d36c3d62ce
commit 51b9d1a45a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 133 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
API.Benchmark/API.Benchmark.csproj Normal file
View File

@ -0,0 +1,18 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net5.0</TargetFramework>
<OutputType>Exe</OutputType>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\API\API.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="BenchmarkDotNet" Version="0.13.1" />
<PackageReference Include="BenchmarkDotNet.Annotations" Version="0.13.1" />
<PackageReference Include="NSubstitute" Version="4.2.2" />
</ItemGroup>
</Project>

40
API.Benchmark/ParseScannedFilesBenchmarks.cs Normal file
View File

@ -0,0 +1,40 @@
using System;
using System.IO;
using API.Entities.Enums;
using API.Interfaces.Services;
using API.Services;
using API.Services.Tasks.Scanner;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Order;
using Microsoft.Extensions.Logging;
using NSubstitute;
namespace API.Benchmark
{
[MemoryDiagnoser]
[Orderer(SummaryOrderPolicy.FastestToSlowest)]
[RankColumn]
[SimpleJob(launchCount: 1, warmupCount: 3, targetCount: 5, invocationCount: 100, id: "Test"), ShortRunJob]
public class ParseScannedFilesBenchmarks
{
private readonly ParseScannedFiles _parseScannedFiles;
private readonly ILogger<ParseScannedFiles> _logger = Substitute.For<ILogger<ParseScannedFiles>>();
private readonly ILogger<BookService> _bookLogger = Substitute.For<ILogger<BookService>>();
public ParseScannedFilesBenchmarks()
{
IBookService bookService = new BookService(_bookLogger);
_parseScannedFiles = new ParseScannedFiles(bookService, _logger);
}
[Benchmark]
public void Test()
{
var libraryPath = Path.Join(Directory.GetCurrentDirectory(),
"../../../Services/Test Data/ScannerService/Manga");
var parsedSeries = _parseScannedFiles.ScanLibrariesForSeries(LibraryType.Manga, new string[] {libraryPath},
out var totalFiles, out var scanElapsedTime);
}
}
}

18
API.Benchmark/Program.cs Normal file
View File

@ -0,0 +1,18 @@
using BenchmarkDotNet.Running;
namespace API.Benchmark
{
/// <summary>
/// To build this, cd into API.Benchmark directory and run
/// dotnet build -c Release
/// then copy the outputted dll
/// dotnet copied_string\API.Benchmark.dll
/// </summary>
public class Program
{
static void Main(string[] args)
{
BenchmarkRunner.Run<ParseScannedFilesBenchmarks>();
}
}
}

28
API.Tests/Parser/MangaParserTests.cs
View File

@ -312,14 +312,14 @@ namespace API.Tests.Parser
const string rootPath = @"E:/Manga/";
var expected = new Dictionary<string, ParserInfo>();
var filepath = @"E:/Manga/Mujaki no Rakuen/Mujaki no Rakuen Vol12 ch76.cbz";
expected.Add(filepath, new ParserInfo
{
Series = "Mujaki no Rakuen", Volumes = "12",
Chapters = "76", Filename = "Mujaki no Rakuen Vol12 ch76.cbz", Format = MangaFormat.Archive,
FullFilePath = filepath
});
expected.Add(filepath, new ParserInfo
{
Series = "Mujaki no Rakuen", Volumes = "12",
Chapters = "76", Filename = "Mujaki no Rakuen Vol12 ch76.cbz", Format = MangaFormat.Archive,
FullFilePath = filepath
});
filepath = @"E:/Manga/Shimoneta to Iu Gainen ga Sonzai Shinai Taikutsu na Sekai Man-hen/Vol 1.cbz";
filepath = @"E:/Manga/Shimoneta to Iu Gainen ga Sonzai Shinai Taikutsu na Sekai Man-hen/Vol 1.cbz";
expected.Add(filepath, new ParserInfo
{
Series = "Shimoneta to Iu Gainen ga Sonzai Shinai Taikutsu na Sekai Man-hen", Volumes = "1",
@ -423,20 +423,20 @@ namespace API.Tests.Parser
}
Assert.NotNull(actual);
_testOutputHelper.WriteLine($"Validating {file}");
_testOutputHelper.WriteLine("Format");
Assert.Equal(expectedInfo.Format, actual.Format);
_testOutputHelper.WriteLine("Series");
_testOutputHelper.WriteLine("Format ✓");
Assert.Equal(expectedInfo.Series, actual.Series);
_testOutputHelper.WriteLine("Chapters");
_testOutputHelper.WriteLine("Series ✓");
Assert.Equal(expectedInfo.Chapters, actual.Chapters);
_testOutputHelper.WriteLine("Volumes");
_testOutputHelper.WriteLine("Chapters ✓");
Assert.Equal(expectedInfo.Volumes, actual.Volumes);
_testOutputHelper.WriteLine("Edition");
_testOutputHelper.WriteLine("Volumes ✓");
Assert.Equal(expectedInfo.Edition, actual.Edition);
_testOutputHelper.WriteLine("Filename");
_testOutputHelper.WriteLine("Edition ✓");
Assert.Equal(expectedInfo.Filename, actual.Filename);
_testOutputHelper.WriteLine("FullFilePath");
_testOutputHelper.WriteLine("Filename ✓");
Assert.Equal(expectedInfo.FullFilePath, actual.FullFilePath);
_testOutputHelper.WriteLine("FullFilePath ✓");
}
}
}

7
API/Interfaces/ITaskScheduler.cs
View File

@ -1,4 +1,6 @@
namespace API.Interfaces
using System.Threading.Tasks;
namespace API.Interfaces
{
public interface ITaskScheduler
{
@ -6,7 +8,7 @@
/// For use on Server startup
/// </summary>
void ScheduleTasks();
void ScheduleStatsTasks();
Task ScheduleStatsTasks();
void ScheduleUpdaterTasks();
void ScanLibrary(int libraryId, bool forceUpdate = false);
void CleanupChapters(int[] chapterIds);
@ -15,5 +17,6 @@
void RefreshSeriesMetadata(int libraryId, int seriesId);
void ScanSeries(int libraryId, int seriesId, bool forceUpdate = false);
void CancelStatsTasks();
void RunStatCollection();
}
}

2
API/Interfaces/Services/IStatsService.cs
View File

@ -6,8 +6,6 @@ namespace API.Interfaces.Services
public interface IStatsService
{
Task PathData(ClientInfoDto clientInfoDto);
Task FinalizeStats();
Task CollectRelevantData();
Task CollectAndSendStatsData();
}
}

21
API/Services/HostedServices/StartupTasksHostedService.cs
View File

@ -2,7 +2,6 @@
using System.Threading;
using System.Threading.Tasks;
using API.Interfaces;
using API.Interfaces.Services;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
@ -27,7 +26,10 @@ namespace API.Services.HostedServices
try
{
await ManageStartupStatsTasks(scope, taskScheduler);
// These methods will automatically check if stat collection is disabled to prevent sending any data regardless
// of when setting was changed
await taskScheduler.ScheduleStatsTasks();
taskScheduler.RunStatCollection();
}
catch (Exception)
{
@ -35,21 +37,6 @@ namespace API.Services.HostedServices
}
}
private async Task ManageStartupStatsTasks(IServiceScope serviceScope, ITaskScheduler taskScheduler)
{
var unitOfWork = serviceScope.ServiceProvider.GetRequiredService<IUnitOfWork>();
var settingsDto = await unitOfWork.SettingsRepository.GetSettingsDtoAsync();
if (!settingsDto.AllowStatCollection) return;
taskScheduler.ScheduleStatsTasks();
var statsService = serviceScope.ServiceProvider.GetRequiredService<IStatsService>();
await statsService.CollectAndSendStatsData();
}
public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
}
}

17
API/Services/TaskScheduler.cs
View File

@ -22,6 +22,7 @@ namespace API.Services
private readonly IStatsService _statsService;
private readonly IVersionUpdaterService _versionUpdaterService;
private const string SendDataTask = "finalize-stats";
public static BackgroundJobServer Client => new BackgroundJobServer();
@ -76,19 +77,17 @@ namespace API.Services
#region StatsTasks
private const string SendDataTask = "finalize-stats";
public void ScheduleStatsTasks()
public async Task ScheduleStatsTasks()
{
var allowStatCollection = bool.Parse(Task.Run(() => _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.AllowStatCollection)).GetAwaiter().GetResult().Value);
var allowStatCollection = (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).AllowStatCollection;
if (!allowStatCollection)
{
_logger.LogDebug("User has opted out of stat collection, not registering tasks");
return;
}
_logger.LogDebug("Adding StatsTasks");
_logger.LogDebug("Scheduling Send data to the Stats server {Setting}", nameof(Cron.Daily));
_logger.LogDebug("Scheduling stat collection daily");
RecurringJob.AddOrUpdate(SendDataTask, () => _statsService.CollectAndSendStatsData(), Cron.Daily);
}
@ -99,6 +98,12 @@ namespace API.Services
RecurringJob.RemoveIfExists(SendDataTask);
}
public void RunStatCollection()
{
_logger.LogInformation("Enqueuing stat collection");
BackgroundJob.Enqueue(() => _statsService.CollectAndSendStatsData());
}
#endregion
#region UpdateTasks

10
API/Services/Tasks/StatsService.cs
View File

@ -50,7 +50,7 @@ namespace API.Services.Tasks
await SaveFile(statisticsDto);
}
public async Task CollectRelevantData()
private async Task CollectRelevantData()
{
_logger.LogDebug("Collecting data from the server and database");
@ -63,7 +63,7 @@ namespace API.Services.Tasks
await PathData(serverInfo, usageInfo);
}
public async Task FinalizeStats()
private async Task FinalizeStats()
{
try
{
@ -86,6 +86,12 @@ namespace API.Services.Tasks
public async Task CollectAndSendStatsData()
{
var allowStatCollection = (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).AllowStatCollection;
if (!allowStatCollection)
{
_logger.LogDebug("User has opted out of stat collection, not registering tasks");
return;
}
await CollectRelevantData();
await FinalizeStats();
}

1
Kavita.Common/HashUtil.cs
View File

@ -1,5 +1,4 @@
using System;
using System.Security.Cryptography;
using System.Text;
namespace Kavita.Common

14
Kavita.sln
View File

@ -9,6 +9,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "API.Tests", "API.Tests\API.
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Kavita.Common", "Kavita.Common\Kavita.Common.csproj", "{165A86F5-9E74-4C05-9305-A6F0BA32C9EE}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "API.Benchmark", "API.Benchmark\API.Benchmark.csproj", "{3D781D18-2452-421F-A81A-59254FEE1FEC}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -58,5 +60,17 @@ Global
{165A86F5-9E74-4C05-9305-A6F0BA32C9EE}.Release|x64.Build.0 = Release|Any CPU
{165A86F5-9E74-4C05-9305-A6F0BA32C9EE}.Release|x86.ActiveCfg = Release|Any CPU
{165A86F5-9E74-4C05-9305-A6F0BA32C9EE}.Release|x86.Build.0 = Release|Any CPU
{3D781D18-2452-421F-A81A-59254FEE1FEC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{3D781D18-2452-421F-A81A-59254FEE1FEC}.Debug|Any CPU.Build.0 = Debug|Any CPU
{3D781D18-2452-421F-A81A-59254FEE1FEC}.Debug|x64.ActiveCfg = Debug|Any CPU
{3D781D18-2452-421F-A81A-59254FEE1FEC}.Debug|x64.Build.0 = Debug|Any CPU
{3D781D18-2452-421F-A81A-59254FEE1FEC}.Debug|x86.ActiveCfg = Debug|Any CPU
{3D781D18-2452-421F-A81A-59254FEE1FEC}.Debug|x86.Build.0 = Debug|Any CPU
{3D781D18-2452-421F-A81A-59254FEE1FEC}.Release|Any CPU.ActiveCfg = Release|Any CPU
{3D781D18-2452-421F-A81A-59254FEE1FEC}.Release|Any CPU.Build.0 = Release|Any CPU
{3D781D18-2452-421F-A81A-59254FEE1FEC}.Release|x64.ActiveCfg = Release|Any CPU
{3D781D18-2452-421F-A81A-59254FEE1FEC}.Release|x64.Build.0 = Release|Any CPU
{3D781D18-2452-421F-A81A-59254FEE1FEC}.Release|x86.ActiveCfg = Release|Any CPU
{3D781D18-2452-421F-A81A-59254FEE1FEC}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection
EndGlobal

9
UI/Web/src/app/user-settings/user-preferences/user-preferences.component.ts
View File

@ -187,13 +187,6 @@ export class UserPreferencesComponent implements OnInit, OnDestroy {
}
transformKeyToOpdsUrl(key: string) {
let apiUrl = environment.apiUrl;
if (environment.production) {
apiUrl = `${location.protocol}//${location.origin}`;
if (location.port != '80') {
apiUrl += ':' + location.port;
}
}
return `${apiUrl}opds/${key}`;
return `${location.origin}/api/opds/${key}`;
}
}