PDF Rendering on Pi (64bit) & Backup Fix (#1204)

* Updated dependencies. SharpCompress has been updated to v2.1.0 which should fix pdf rendering on pi/arm64 devices.

* Removed some dependencies not needed and updated the Backup code to account for themes and ensure everything gets copied every time.

API.Tests/API.Tests.csproj

@@ -7,10 +7,10 @@
     </PropertyGroup>
 
     <ItemGroup>
-        <PackageReference Include="Microsoft.EntityFrameworkCore.InMemory" Version="6.0.2" />
+        <PackageReference Include="Microsoft.EntityFrameworkCore.InMemory" Version="6.0.3" />
         <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
         <PackageReference Include="NSubstitute" Version="4.3.0" />
-        <PackageReference Include="System.IO.Abstractions.TestingHelpers" Version="16.1.15" />
+        <PackageReference Include="System.IO.Abstractions.TestingHelpers" Version="16.1.25" />
         <PackageReference Include="xunit" Version="2.4.1" />
         <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
             <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

API.Tests/Services/BackupServiceTests.cs

@@ -1,6 +1,7 @@
 using System.Collections.Generic;
 using System.Data.Common;
 using System.IO.Abstractions.TestingHelpers;
+using System.IO.Compression;
 using System.Linq;
 using System.Threading.Tasks;
 using API.Data;
@@ -36,6 +37,9 @@ public class BackupServiceTests
     private const string CoverImageDirectory = "C:/kavita/config/covers/";
     private const string BackupDirectory = "C:/kavita/config/backups/";
     private const string LogDirectory = "C:/kavita/config/logs/";
+    private const string ConfigDirectory = "C:/kavita/config/";
+    private const string BookmarkDirectory = "C:/kavita/config/bookmarks";
+    private const string ThemesDirectory = "C:/kavita/config/theme";
 
     public BackupServiceTests()
     {
@@ -110,6 +114,8 @@ public class BackupServiceTests
         fileSystem.AddDirectory(CoverImageDirectory);
         fileSystem.AddDirectory(BackupDirectory);
         fileSystem.AddDirectory(LogDirectory);
+        fileSystem.AddDirectory(ThemesDirectory);
+        fileSystem.AddDirectory(BookmarkDirectory);
         fileSystem.AddDirectory("C:/data/");
 
         return fileSystem;
@@ -121,6 +127,7 @@ public class BackupServiceTests
 
     #region GetLogFiles
 
+    [Fact]
     public void GetLogFiles_ExpectAllFiles_NoRollingFiles()
     {
         var filesystem = CreateFileSystem();
@@ -128,16 +135,82 @@ public class BackupServiceTests
         filesystem.AddFile($"{LogDirectory}kavita1.log", new MockFileData(""));
 
         var ds = new DirectoryService(Substitute.For<ILogger<DirectoryService>>(), filesystem);
-        // You can't mock _config extensions because they are static
+        var inMemorySettings = new Dictionary<string, string> {
-        _config.GetMaxRollingFiles().Returns(1);
+            {"Logging:File:Path", "config/logs/kavita.log"},
-        _config.GetLoggingFileName().Returns(ds.FileSystem.Path.Join(LogDirectory, "kavita.log"));
+            {"Logging:File:MaxRollingFiles", "0"},
+        };
+        IConfiguration configuration = new ConfigurationBuilder()
+            .AddInMemoryCollection(inMemorySettings)
+            .Build();
 
-        var backupService = new BackupService(_logger, _unitOfWork, ds, _config, _messageHub);
+        var backupService = new BackupService(_logger, _unitOfWork, ds, configuration, _messageHub);
 
-        Assert.Single(backupService.GetLogFiles(1, LogDirectory));
+        var backupLogFiles = backupService.GetLogFiles(0, LogDirectory).ToList();
+        Assert.Single(backupLogFiles);
+        Assert.Equal(API.Parser.Parser.NormalizePath($"{LogDirectory}kavita.log"), API.Parser.Parser.NormalizePath(backupLogFiles.First()));
+    }
+
+    [Fact]
+    public void GetLogFiles_ExpectAllFiles_WithRollingFiles()
+    {
+        var filesystem = CreateFileSystem();
+        filesystem.AddFile($"{LogDirectory}kavita.log", new MockFileData(""));
+        filesystem.AddFile($"{LogDirectory}kavita1.log", new MockFileData(""));
+
+        var ds = new DirectoryService(Substitute.For<ILogger<DirectoryService>>(), filesystem);
+        var inMemorySettings = new Dictionary<string, string> {
+            {"Logging:File:Path", "config/logs/kavita.log"},
+            {"Logging:File:MaxRollingFiles", "1"},
+        };
+        IConfiguration configuration = new ConfigurationBuilder()
+            .AddInMemoryCollection(inMemorySettings)
+            .Build();
+
+        var backupService = new BackupService(_logger, _unitOfWork, ds, configuration, _messageHub);
+
+        var backupLogFiles = backupService.GetLogFiles(1, LogDirectory).Select(API.Parser.Parser.NormalizePath).ToList();
+        Assert.NotEmpty(backupLogFiles.Where(file => file.Equals(API.Parser.Parser.NormalizePath($"{LogDirectory}kavita.log")) || file.Equals(API.Parser.Parser.NormalizePath($"{LogDirectory}kavita1.log"))));
     }
 
 
     #endregion
 
+    #region BackupFiles
+
+    // I don't think I can unit test this due to ZipFile.Create
+    // [Fact]
+    // public async Task BackupDatabase_ExpectAllFiles()
+    // {
+    //     var filesystem = CreateFileSystem();
+    //     filesystem.AddFile($"{LogDirectory}kavita.log", new MockFileData(""));
+    //     filesystem.AddFile($"{ConfigDirectory}kavita.db", new MockFileData(""));
+    //     filesystem.AddFile($"{CoverImageDirectory}1.png", new MockFileData(""));
+    //     filesystem.AddFile($"{BookmarkDirectory}1.png", new MockFileData(""));
+    //     filesystem.AddFile($"{ConfigDirectory}appsettings.json", new MockFileData(""));
+    //     filesystem.AddFile($"{ThemesDirectory}joe.css", new MockFileData(""));
+    //
+    //
+    //     var ds = new DirectoryService(Substitute.For<ILogger<DirectoryService>>(), filesystem);
+    //     var inMemorySettings = new Dictionary<string, string> {
+    //         {"Logging:File:Path", $"{LogDirectory}kavita.log"},
+    //         {"Logging:File:MaxRollingFiles", "0"},
+    //     };
+    //     IConfiguration configuration = new ConfigurationBuilder()
+    //         .AddInMemoryCollection(inMemorySettings)
+    //         .Build();
+    //
+    //     var backupService = new BackupService(_logger, _unitOfWork, ds, configuration, _messageHub);
+    //
+    //     await backupService.BackupDatabase();
+    //
+    //
+    //     var files = ds.GetFiles(BackupDirectory).ToList();
+    //     Assert.NotEmpty(files);
+    //     var zipFile = files.FirstOrDefault();
+    //     Assert.NotNull(zipFile);
+    //     using var zipArchive = ZipFile.OpenRead(zipFile);
+    //
+    // }
+
+    #endregion
 }

API.Tests/Services/DirectoryServiceTests.cs

@@ -508,6 +508,21 @@ namespace API.Tests.Services
             Assert.Equal(2, ds.GetFiles("/manga/output/").Count());
         }
 
+        [Fact]
+        public void CopyFilesToDirectory_ShouldMoveAllFilesAndNotFailOnNonExistentFiles()
+        {
+            const string testDirectory = "/manga/";
+            var fileSystem = new MockFileSystem();
+            for (var i = 0; i < 10; i++)
+            {
+                fileSystem.AddFile($"{testDirectory}file_{i}.zip", new MockFileData(""));
+            }
+
+            var ds = new DirectoryService(Substitute.For<ILogger<DirectoryService>>(), fileSystem);
+            ds.CopyFilesToDirectory(new []{$"{testDirectory}file_{0}.zip", $"{testDirectory}file_{200}.zip", $"{testDirectory}file_{1}.zip"}, "/manga/output/");
+            Assert.Equal(2, ds.GetFiles("/manga/output/").Count());
+        }
+
         [Fact]
         public void CopyFilesToDirectory_ShouldMoveAllFiles_InclFilesInNestedFolders()
         {

API/API.csproj

@@ -48,32 +48,30 @@
     <PackageReference Include="Hangfire.MemoryStorage.Core" Version="1.4.0" />
     <PackageReference Include="HtmlAgilityPack" Version="1.11.42" />
     <PackageReference Include="MarkdownDeep.NET.Core" Version="1.5.0.4" />
-    <PackageReference Include="MediatR" Version="10.0.1" />
+    <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="6.0.3" />
-    <PackageReference Include="MediatR.Extensions.Microsoft.DependencyInjection" Version="10.0.1" />
+    <PackageReference Include="Microsoft.AspNetCore.Authentication.OpenIdConnect" Version="6.0.3" />
-    <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="6.0.2" />
+    <PackageReference Include="Microsoft.AspNetCore.Identity.EntityFrameworkCore" Version="6.0.3" />
-    <PackageReference Include="Microsoft.AspNetCore.Authentication.OpenIdConnect" Version="6.0.2" />
-    <PackageReference Include="Microsoft.AspNetCore.Identity.EntityFrameworkCore" Version="6.0.2" />
     <PackageReference Include="Microsoft.AspNetCore.SignalR" Version="1.1.0" />
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="6.0.2">
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="6.0.3">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="6.0.2" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="6.0.3" />
     <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="6.0.0" />
     <PackageReference Include="Microsoft.IO.RecyclableMemoryStream" Version="2.2.0" />
     <PackageReference Include="NetVips" Version="2.1.0" />
     <PackageReference Include="NetVips.Native" Version="8.12.2" />
     <PackageReference Include="NReco.Logging.File" Version="1.1.4" />
-    <PackageReference Include="SharpCompress" Version="0.30.1" />
+    <PackageReference Include="SharpCompress" Version="0.31.0" />
-    <PackageReference Include="SixLabors.ImageSharp" Version="2.0.0" />
+    <PackageReference Include="SixLabors.ImageSharp" Version="2.1.0" />
-    <PackageReference Include="SonarAnalyzer.CSharp" Version="8.36.0.43782">
+    <PackageReference Include="SonarAnalyzer.CSharp" Version="8.37.0.45539">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.2.3" />
+    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.3.0" />
     <PackageReference Include="System.Drawing.Common" Version="6.0.0" />
-    <PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="6.16.0" />
+    <PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="6.17.0" />
-    <PackageReference Include="System.IO.Abstractions" Version="16.1.15" />
+    <PackageReference Include="System.IO.Abstractions" Version="16.1.25" />
     <PackageReference Include="VersOne.Epub" Version="3.0.3.1" />
   </ItemGroup>
 

API/Controllers/UsersController.cs

@@ -86,6 +86,7 @@ namespace API.Controllers
             existingPreferences.BookReaderFontSize = preferencesDto.BookReaderFontSize;
             existingPreferences.BookReaderTapToPaginate = preferencesDto.BookReaderTapToPaginate;
             existingPreferences.BookReaderReadingDirection = preferencesDto.BookReaderReadingDirection;
+            preferencesDto.Theme ??= await _unitOfWork.SiteThemeRepository.GetDefaultTheme();
             existingPreferences.Theme = await _unitOfWork.SiteThemeRepository.GetThemeById(preferencesDto.Theme.Id);
 
             // TODO: Remove this code - this overrides layout mode to be single until the mode is released

API/DTOs/UpdateUserRole.cs

@@ -1,10 +0,0 @@
-using System.Collections.Generic;
-using MediatR;
-
-namespace API.DTOs;
-
-public class UpdateUserRole : IRequest<bool>
-{
-    public string Username { get; init; }
-    public IList<string> Roles { get; init; }
-}

API/Services/DirectoryService.cs

@@ -373,6 +373,11 @@ namespace API.Services
                {
                    currentFile = file;
 
+                   if (!FileSystem.File.Exists(file))
+                   {
+                       _logger.LogError("Unable to copy {File} to {DirectoryPath} as it doesn't exist", file, directoryPath);
+                       continue;
+                   }
                    var fileInfo = FileSystem.FileInfo.FromFileName(file);
                    var targetFile = FileSystem.FileInfo.FromFileName(RenameFileForCopy(file, directoryPath, prepend));
 

API/Services/Tasks/BackupService.cs

@@ -31,6 +31,7 @@ public class BackupService : IBackupService
     private readonly IUnitOfWork _unitOfWork;
     private readonly ILogger<BackupService> _logger;
     private readonly IDirectoryService _directoryService;
+    private readonly IConfiguration _config;
     private readonly IEventHub _eventHub;
 
     private readonly IList<string> _backupFiles;
@@ -41,11 +42,12 @@ public class BackupService : IBackupService
         _unitOfWork = unitOfWork;
         _logger = logger;
         _directoryService = directoryService;
+        _config = config;
         _eventHub = eventHub;
 
-        var maxRollingFiles = config.GetMaxRollingFiles();
+        // var maxRollingFiles = config.GetMaxRollingFiles();
-        var loggingSection = config.GetLoggingFileName();
+        // var loggingSection = config.GetLoggingFileName();
-        var files = GetLogFiles(maxRollingFiles, loggingSection);
+        // var files = GetLogFiles(maxRollingFiles, loggingSection);
 
 
         _backupFiles = new List<string>()
@@ -58,12 +60,10 @@ public class BackupService : IBackupService
             "kavita.db-wal" // This wont always be there
         };
 
-        foreach (var file in files.Select(f => (_directoryService.FileSystem.FileInfo.FromFileName(f)).Name).ToList())
+        // foreach (var file in files.Select(f => (_directoryService.FileSystem.FileInfo.FromFileName(f)).Name))
-        {
+        // {
-            _backupFiles.Add(file);
+        //     _backupFiles.Add(file);
-        }
+        // }
-
-
     }
 
     public IEnumerable<string> GetLogFiles(int maxRollingFiles, string logFileName)
@@ -74,7 +74,7 @@ public class BackupService : IBackupService
         var files = maxRollingFiles > 0
             ? _directoryService.GetFiles(_directoryService.LogDirectory,
                 $@"{_directoryService.FileSystem.Path.GetFileNameWithoutExtension(fi.Name)}{multipleFileRegex}\.log")
-            : new[] {"kavita.log"};
+            : new[] {_directoryService.FileSystem.Path.Join(_directoryService.LogDirectory, "kavita.log")};
         return files;
     }
 
@@ -97,6 +97,7 @@ public class BackupService : IBackupService
         }
 
         await SendProgress(0F, "Started backup");
+        await SendProgress(0.1F, "Copying core files");
 
         var dateString = $"{DateTime.Now.ToShortDateString()}_{DateTime.Now.ToLongTimeString()}".Replace("/", "_").Replace(":", "_");
         var zipPath = _directoryService.FileSystem.Path.Join(backupDirectory, $"kavita_backup_{dateString}.zip");
@@ -116,15 +117,19 @@ public class BackupService : IBackupService
         _directoryService.CopyFilesToDirectory(
             _backupFiles.Select(file => _directoryService.FileSystem.Path.Join(_directoryService.ConfigDirectory, file)).ToList(), tempDirectory);
 
-        await SendProgress(0.25F, "Copying core files");
+        CopyLogsToBackupDirectory(tempDirectory);
+
+        await SendProgress(0.25F, "Copying cover images");
 
         await CopyCoverImagesToBackupDirectory(tempDirectory);
 
-        await SendProgress(0.5F, "Copying cover images");
+        await SendProgress(0.5F, "Copying bookmarks");
 
         await CopyBookmarksToBackupDirectory(tempDirectory);
 
-        await SendProgress(0.75F, "Copying bookmarks");
+        await SendProgress(0.75F, "Copying themes");
+
+        CopyThemesToBackupDirectory(tempDirectory);
 
         try
         {
@@ -140,6 +145,14 @@ public class BackupService : IBackupService
         await SendProgress(1F, "Completed backup");
     }
 
+    private void CopyLogsToBackupDirectory(string tempDirectory)
+    {
+        var maxRollingFiles = _config.GetMaxRollingFiles();
+        var loggingSection = _config.GetLoggingFileName();
+        var files = GetLogFiles(maxRollingFiles, loggingSection);
+        _directoryService.CopyFilesToDirectory(files, _directoryService.FileSystem.Path.Join(tempDirectory, "logs"));
+    }
+
     private async Task CopyCoverImagesToBackupDirectory(string tempDirectory)
     {
         var outputTempDir = Path.Join(tempDirectory, "covers");
@@ -193,6 +206,26 @@ public class BackupService : IBackupService
         }
     }
 
+    private void CopyThemesToBackupDirectory(string tempDirectory)
+    {
+        var outputTempDir = Path.Join(tempDirectory, "themes");
+        _directoryService.ExistOrCreate(outputTempDir);
+
+        try
+        {
+            _directoryService.CopyDirectoryToDirectory(_directoryService.SiteThemeDirectory, outputTempDir);
+        }
+        catch (IOException)
+        {
+            // Swallow exception.
+        }
+
+        if (!_directoryService.GetFiles(outputTempDir, searchOption: SearchOption.AllDirectories).Any())
+        {
+            _directoryService.ClearAndDeleteDirectory(outputTempDir);
+        }
+    }
+
     private async Task SendProgress(float progress, string subtitle)
     {
         await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,

API/Startup.cs

@@ -19,7 +19,6 @@ using Hangfire;
 using Hangfire.MemoryStorage;
 using Kavita.Common;
 using Kavita.Common.EnvironmentInfo;
-using MediatR;
 using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.AspNetCore.Http;
@@ -132,8 +131,6 @@ namespace API
             // Add IHostedService for startup tasks
             // Any services that should be bootstrapped go here
             services.AddHostedService<StartupTasksHostedService>();
-
-            services.AddMediatR(typeof(Startup));
         }
 
         // This method gets called by the runtime. Use this method to configure the HTTP request pipeline.

Kavita.Common/Kavita.Common.csproj

@@ -12,7 +12,7 @@
       <PackageReference Include="Flurl.Http" Version="3.2.2" />
       <PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="6.0.0" />
       <PackageReference Include="Microsoft.Extensions.Hosting" Version="6.0.1" />
-      <PackageReference Include="SonarAnalyzer.CSharp" Version="8.36.0.43782">
+      <PackageReference Include="SonarAnalyzer.CSharp" Version="8.37.0.45539">
         <PrivateAssets>all</PrivateAssets>
         <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       </PackageReference>

UI/Web/src/app/dashboard/dashboard.component.ts

@@ -1,8 +1,6 @@
 import { Component, OnInit } from '@angular/core';
 import { Title } from '@angular/platform-browser';
 import { ActivatedRoute } from '@angular/router';
-import { ToastrService } from 'ngx-toastr';
-import { ServerService } from '../_services/server.service';
 
 @Component({
   selector: 'app-dashboard',
@@ -11,23 +9,8 @@ import { ServerService } from '../_services/server.service';
 })
 export class DashboardComponent implements OnInit {
 
-  tabs: Array<{title: string, fragment: string}> = [
-    {title: 'Libraries', fragment: ''},
-    {title: 'Lists', fragment: 'lists'},
-    {title: 'Collections', fragment: 'collections'},
-  ];
-  active = this.tabs[0];
 
-  constructor(public route: ActivatedRoute, private serverService: ServerService, 
+  constructor(public route: ActivatedRoute, private titleService: Title) {
-    private toastr: ToastrService, private titleService: Title) {
-    this.route.fragment.subscribe(frag => {
-      const tab = this.tabs.filter(item => item.fragment === frag);
-      if (tab.length > 0) {
-        this.active = tab[0];
-      } else {
-        this.active = this.tabs[0]; // Default to first tab
-      }
-    });
     this.titleService.setTitle('Kavita - Dashboard');
   }