OPDS Cleanup (#534)

* Fixed opds url display

* Rewrote how stat collection works, now we check in multiple places and always run stat collection in a background thread, to not block main thread.

* Cleaned up the ParseInfoTest to be more verbose

* Added benchmarking

API.Benchmark/API.Benchmark.csproj

@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net5.0</TargetFramework>
+        <OutputType>Exe</OutputType>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\API\API.csproj" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <PackageReference Include="BenchmarkDotNet" Version="0.13.1" />
+      <PackageReference Include="BenchmarkDotNet.Annotations" Version="0.13.1" />
+      <PackageReference Include="NSubstitute" Version="4.2.2" />
+    </ItemGroup>
+
+</Project>

API.Benchmark/ParseScannedFilesBenchmarks.cs

@@ -0,0 +1,40 @@
+using System;
+using System.IO;
+using API.Entities.Enums;
+using API.Interfaces.Services;
+using API.Services;
+using API.Services.Tasks.Scanner;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Order;
+using Microsoft.Extensions.Logging;
+using NSubstitute;
+
+namespace API.Benchmark
+{
+    [MemoryDiagnoser]
+    [Orderer(SummaryOrderPolicy.FastestToSlowest)]
+    [RankColumn]
+    [SimpleJob(launchCount: 1, warmupCount: 3, targetCount: 5, invocationCount: 100, id: "Test"), ShortRunJob]
+    public class ParseScannedFilesBenchmarks
+    {
+        private readonly ParseScannedFiles _parseScannedFiles;
+        private readonly ILogger<ParseScannedFiles> _logger = Substitute.For<ILogger<ParseScannedFiles>>();
+        private readonly ILogger<BookService> _bookLogger = Substitute.For<ILogger<BookService>>();
+
+        public ParseScannedFilesBenchmarks()
+        {
+            IBookService bookService = new BookService(_bookLogger);
+            _parseScannedFiles = new ParseScannedFiles(bookService, _logger);
+        }
+
+        [Benchmark]
+        public void Test()
+        {
+            var libraryPath = Path.Join(Directory.GetCurrentDirectory(),
+                "../../../Services/Test Data/ScannerService/Manga");
+            var parsedSeries = _parseScannedFiles.ScanLibrariesForSeries(LibraryType.Manga, new string[] {libraryPath},
+                out var totalFiles, out var scanElapsedTime);
+        }
+
+    }
+}

API.Benchmark/Program.cs

@@ -0,0 +1,18 @@
+using BenchmarkDotNet.Running;
+
+namespace API.Benchmark
+{
+    /// <summary>
+    /// To build this, cd into API.Benchmark directory and run
+    /// dotnet build -c Release
+    /// then copy the outputted dll
+    /// dotnet copied_string\API.Benchmark.dll
+    /// </summary>
+    public class Program
+    {
+        static void Main(string[] args)
+        {
+            BenchmarkRunner.Run<ParseScannedFilesBenchmarks>();
+        }
+    }
+}

API.Tests/Parser/MangaParserTests.cs

@@ -312,14 +312,14 @@ namespace API.Tests.Parser
             const string rootPath = @"E:/Manga/";
             var expected = new Dictionary<string, ParserInfo>();
             var filepath = @"E:/Manga/Mujaki no Rakuen/Mujaki no Rakuen Vol12 ch76.cbz";
-            expected.Add(filepath, new ParserInfo
+             expected.Add(filepath, new ParserInfo
-            {
+             {
-                Series = "Mujaki no Rakuen", Volumes = "12",
+                 Series = "Mujaki no Rakuen", Volumes = "12",
-                Chapters = "76", Filename = "Mujaki no Rakuen Vol12 ch76.cbz", Format = MangaFormat.Archive,
+                 Chapters = "76", Filename = "Mujaki no Rakuen Vol12 ch76.cbz", Format = MangaFormat.Archive,
-                FullFilePath = filepath
+                 FullFilePath = filepath
-            });
+             });
 
-            filepath = @"E:/Manga/Shimoneta to Iu Gainen ga Sonzai Shinai Taikutsu na Sekai Man-hen/Vol 1.cbz";
+             filepath = @"E:/Manga/Shimoneta to Iu Gainen ga Sonzai Shinai Taikutsu na Sekai Man-hen/Vol 1.cbz";
             expected.Add(filepath, new ParserInfo
             {
                 Series = "Shimoneta to Iu Gainen ga Sonzai Shinai Taikutsu na Sekai Man-hen", Volumes = "1",
@@ -423,20 +423,20 @@ namespace API.Tests.Parser
                 }
                 Assert.NotNull(actual);
                 _testOutputHelper.WriteLine($"Validating {file}");
-                _testOutputHelper.WriteLine("Format");
                 Assert.Equal(expectedInfo.Format, actual.Format);
-                _testOutputHelper.WriteLine("Series");
+                _testOutputHelper.WriteLine("Format ✓");
                 Assert.Equal(expectedInfo.Series, actual.Series);
-                _testOutputHelper.WriteLine("Chapters");
+                _testOutputHelper.WriteLine("Series ✓");
                 Assert.Equal(expectedInfo.Chapters, actual.Chapters);
-                _testOutputHelper.WriteLine("Volumes");
+                _testOutputHelper.WriteLine("Chapters ✓");
                 Assert.Equal(expectedInfo.Volumes, actual.Volumes);
-                _testOutputHelper.WriteLine("Edition");
+                _testOutputHelper.WriteLine("Volumes ✓");
                 Assert.Equal(expectedInfo.Edition, actual.Edition);
-                _testOutputHelper.WriteLine("Filename");
+                _testOutputHelper.WriteLine("Edition ✓");
                 Assert.Equal(expectedInfo.Filename, actual.Filename);
-                _testOutputHelper.WriteLine("FullFilePath");
+                _testOutputHelper.WriteLine("Filename ✓");
                 Assert.Equal(expectedInfo.FullFilePath, actual.FullFilePath);
+                _testOutputHelper.WriteLine("FullFilePath ✓");
             }
         }
     }

API/Interfaces/ITaskScheduler.cs

@@ -1,4 +1,6 @@
-namespace API.Interfaces
+using System.Threading.Tasks;
+
+namespace API.Interfaces
 {
     public interface ITaskScheduler
     {
@@ -6,7 +8,7 @@
         /// For use on Server startup
         /// </summary>
         void ScheduleTasks();
-        void ScheduleStatsTasks();
+        Task ScheduleStatsTasks();
         void ScheduleUpdaterTasks();
         void ScanLibrary(int libraryId, bool forceUpdate = false);
         void CleanupChapters(int[] chapterIds);
@@ -15,5 +17,6 @@
         void RefreshSeriesMetadata(int libraryId, int seriesId);
         void ScanSeries(int libraryId, int seriesId, bool forceUpdate = false);
         void CancelStatsTasks();
+        void RunStatCollection();
     }
 }

API/Interfaces/Services/IStatsService.cs

@@ -6,8 +6,6 @@ namespace API.Interfaces.Services
     public interface IStatsService
     {
         Task PathData(ClientInfoDto clientInfoDto);
-        Task FinalizeStats();
-        Task CollectRelevantData();
         Task CollectAndSendStatsData();
     }
 }

API/Services/HostedServices/StartupTasksHostedService.cs

@@ -2,7 +2,6 @@
 using System.Threading;
 using System.Threading.Tasks;
 using API.Interfaces;
-using API.Interfaces.Services;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 
@@ -27,7 +26,10 @@ namespace API.Services.HostedServices
 
             try
             {
-                await ManageStartupStatsTasks(scope, taskScheduler);
+                // These methods will automatically check if stat collection is disabled to prevent sending any data regardless
+                // of when setting was changed
+                await taskScheduler.ScheduleStatsTasks();
+                taskScheduler.RunStatCollection();
             }
             catch (Exception)
             {
@@ -35,21 +37,6 @@ namespace API.Services.HostedServices
             }
         }
 
-        private async Task ManageStartupStatsTasks(IServiceScope serviceScope, ITaskScheduler taskScheduler)
-        {
-            var unitOfWork = serviceScope.ServiceProvider.GetRequiredService<IUnitOfWork>();
-
-            var settingsDto = await unitOfWork.SettingsRepository.GetSettingsDtoAsync();
-
-            if (!settingsDto.AllowStatCollection) return;
-
-            taskScheduler.ScheduleStatsTasks();
-
-            var statsService = serviceScope.ServiceProvider.GetRequiredService<IStatsService>();
-
-            await statsService.CollectAndSendStatsData();
-        }
-
         public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
     }
 }

API/Services/TaskScheduler.cs

@@ -22,6 +22,7 @@ namespace API.Services
 
         private readonly IStatsService _statsService;
         private readonly IVersionUpdaterService _versionUpdaterService;
+        private const string SendDataTask = "finalize-stats";
 
         public static BackgroundJobServer Client => new BackgroundJobServer();
 
@@ -76,19 +77,17 @@ namespace API.Services
 
         #region StatsTasks
 
-        private const string SendDataTask = "finalize-stats";
+
-        public void ScheduleStatsTasks()
+        public async Task ScheduleStatsTasks()
         {
-            var allowStatCollection = bool.Parse(Task.Run(() => _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.AllowStatCollection)).GetAwaiter().GetResult().Value);
+            var allowStatCollection  = (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).AllowStatCollection;
             if (!allowStatCollection)
             {
                 _logger.LogDebug("User has opted out of stat collection, not registering tasks");
                 return;
             }
 
-            _logger.LogDebug("Adding StatsTasks");
+            _logger.LogDebug("Scheduling stat collection daily");
-
-            _logger.LogDebug("Scheduling Send data to the Stats server {Setting}", nameof(Cron.Daily));
             RecurringJob.AddOrUpdate(SendDataTask, () => _statsService.CollectAndSendStatsData(), Cron.Daily);
         }
 
@@ -99,6 +98,12 @@ namespace API.Services
             RecurringJob.RemoveIfExists(SendDataTask);
         }
 
+        public void RunStatCollection()
+        {
+            _logger.LogInformation("Enqueuing stat collection");
+            BackgroundJob.Enqueue(() => _statsService.CollectAndSendStatsData());
+        }
+
         #endregion
 
         #region UpdateTasks

API/Services/Tasks/StatsService.cs

@@ -50,7 +50,7 @@ namespace API.Services.Tasks
             await SaveFile(statisticsDto);
         }
 
-        public async Task CollectRelevantData()
+        private async Task CollectRelevantData()
         {
             _logger.LogDebug("Collecting data from the server and database");
 
@@ -63,7 +63,7 @@ namespace API.Services.Tasks
             await PathData(serverInfo, usageInfo);
         }
 
-        public async Task FinalizeStats()
+        private async Task FinalizeStats()
         {
             try
             {
@@ -86,6 +86,12 @@ namespace API.Services.Tasks
 
         public async Task CollectAndSendStatsData()
         {
+            var allowStatCollection = (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).AllowStatCollection;
+            if (!allowStatCollection)
+            {
+                _logger.LogDebug("User has opted out of stat collection, not registering tasks");
+                return;
+            }
             await CollectRelevantData();
             await FinalizeStats();
         }

Kavita.Common/HashUtil.cs

@@ -1,5 +1,4 @@
 using System;
-using System.Security.Cryptography;
 using System.Text;
 
 namespace Kavita.Common

Kavita.sln

@@ -9,6 +9,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "API.Tests", "API.Tests\API.
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Kavita.Common", "Kavita.Common\Kavita.Common.csproj", "{165A86F5-9E74-4C05-9305-A6F0BA32C9EE}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "API.Benchmark", "API.Benchmark\API.Benchmark.csproj", "{3D781D18-2452-421F-A81A-59254FEE1FEC}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -58,5 +60,17 @@ Global
 		{165A86F5-9E74-4C05-9305-A6F0BA32C9EE}.Release|x64.Build.0 = Release|Any CPU
 		{165A86F5-9E74-4C05-9305-A6F0BA32C9EE}.Release|x86.ActiveCfg = Release|Any CPU
 		{165A86F5-9E74-4C05-9305-A6F0BA32C9EE}.Release|x86.Build.0 = Release|Any CPU
+		{3D781D18-2452-421F-A81A-59254FEE1FEC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{3D781D18-2452-421F-A81A-59254FEE1FEC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{3D781D18-2452-421F-A81A-59254FEE1FEC}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{3D781D18-2452-421F-A81A-59254FEE1FEC}.Debug|x64.Build.0 = Debug|Any CPU
+		{3D781D18-2452-421F-A81A-59254FEE1FEC}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{3D781D18-2452-421F-A81A-59254FEE1FEC}.Debug|x86.Build.0 = Debug|Any CPU
+		{3D781D18-2452-421F-A81A-59254FEE1FEC}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{3D781D18-2452-421F-A81A-59254FEE1FEC}.Release|Any CPU.Build.0 = Release|Any CPU
+		{3D781D18-2452-421F-A81A-59254FEE1FEC}.Release|x64.ActiveCfg = Release|Any CPU
+		{3D781D18-2452-421F-A81A-59254FEE1FEC}.Release|x64.Build.0 = Release|Any CPU
+		{3D781D18-2452-421F-A81A-59254FEE1FEC}.Release|x86.ActiveCfg = Release|Any CPU
+		{3D781D18-2452-421F-A81A-59254FEE1FEC}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal

UI/Web/src/app/user-settings/user-preferences/user-preferences.component.ts

@@ -187,13 +187,6 @@ export class UserPreferencesComponent implements OnInit, OnDestroy {
   }
 
   transformKeyToOpdsUrl(key: string) {
-    let apiUrl = environment.apiUrl;
+    return `${location.origin}/api/opds/${key}`;
-    if (environment.production) {
-      apiUrl = `${location.protocol}//${location.origin}`;
-      if (location.port != '80') {
-        apiUrl += ':' + location.port;
-      }
-    }
-    return `${apiUrl}opds/${key}`;
   }
 }