From 06508fd90985e8bf0321d8e3d470fc25b4c0e3d0 Mon Sep 17 00:00:00 2001 From: Joseph Milazzo Date: Mon, 11 Oct 2021 06:57:50 -0700 Subject: [PATCH] Remove Base Url Support (#652) * Fixed some issues with base url. All Scheduled jobs are now in user's timezone * Hide Base Url support from UI and removed some code around it on the backend to prevent it from interfering. Patched back in changes from base-href branch like Timezone on scheduled jobs and enhanced logging. * Added parser support for "2000 AD 0366 [1984-04-28] (flopbie)" and removed for "01 Spiderman 01". --- API.Tests/Parser/ComicParserTests.cs | 7 ++- API/Controllers/SettingsController.cs | 3 + API/Parser/Parser.cs | 4 +- API/Services/MetadataService.cs | 16 ++++-- API/Services/TaskScheduler.cs | 19 ++++--- API/Services/Tasks/ScannerService.cs | 8 ++- API/Startup.cs | 24 ++++---- API/appsettings.Development.json | 4 +- Kavita.Common/Configuration.cs | 57 +------------------ .../manage-settings.component.html | 4 +- UI/Web/src/index.html | 37 +++--------- 11 files changed, 63 insertions(+), 120 deletions(-) diff --git a/API.Tests/Parser/ComicParserTests.cs b/API.Tests/Parser/ComicParserTests.cs index ce9f14065..c35d8f05c 100644 --- a/API.Tests/Parser/ComicParserTests.cs +++ b/API.Tests/Parser/ComicParserTests.cs @@ -17,7 +17,6 @@ namespace API.Tests.Parser } [Theory] - [InlineData("01 Spider-Man & Wolverine 01.cbr", "Spider-Man & Wolverine")] [InlineData("04 - Asterix the Gladiator (1964) (Digital-Empire) (WebP by Doc MaKS)", "Asterix the Gladiator")] [InlineData("The First Asterix Frieze (WebP by Doc MaKS)", "The First Asterix Frieze")] [InlineData("Batman & Catwoman - Trail of the Gun 01", "Batman & Catwoman - Trail of the Gun")] @@ -54,8 +53,8 @@ namespace API.Tests.Parser [InlineData("Cyberpunk 2077 - Trauma Team #04.cbz", "Cyberpunk 2077 - Trauma Team")] [InlineData("Batgirl Vol.2000 #57 (December, 2004)", "Batgirl")] [InlineData("Batgirl V2000 #57", "Batgirl")] - [InlineData("Fables 021 (2004) (Digital) (Nahga-Empire).cbr", "Fables")] - + [InlineData("Fables 021 (2004) (Digital) (Nahga-Empire)", "Fables")] + [InlineData("2000 AD 0366 [1984-04-28] (flopbie)", "2000 AD")] public void ParseComicSeriesTest(string filename, string expected) { Assert.Equal(expected, API.Parser.Parser.ParseComicSeries(filename)); @@ -91,6 +90,7 @@ namespace API.Tests.Parser [InlineData("Batgirl V2000 #57", "2000")] [InlineData("Fables 021 (2004) (Digital) (Nahga-Empire).cbr", "0")] [InlineData("Cyberpunk 2077 - Trauma Team 04.cbz", "0")] + [InlineData("2000 AD 0366 [1984-04-28] (flopbie)", "0")] public void ParseComicVolumeTest(string filename, string expected) { Assert.Equal(expected, API.Parser.Parser.ParseComicVolume(filename)); @@ -130,6 +130,7 @@ namespace API.Tests.Parser [InlineData("Batgirl V2000 #57", "57")] [InlineData("Fables 021 (2004) (Digital) (Nahga-Empire).cbr", "21")] [InlineData("Cyberpunk 2077 - Trauma Team #04.cbz", "4")] + [InlineData("2000 AD 0366 [1984-04-28] (flopbie)", "366")] public void ParseComicChapterTest(string filename, string expected) { Assert.Equal(expected, API.Parser.Parser.ParseComicChapter(filename)); diff --git a/API/Controllers/SettingsController.cs b/API/Controllers/SettingsController.cs index abde87eec..ef3fe8997 100644 --- a/API/Controllers/SettingsController.cs +++ b/API/Controllers/SettingsController.cs @@ -98,6 +98,9 @@ namespace API.Controllers var path = !updateSettingsDto.BaseUrl.StartsWith("/") ? $"/{updateSettingsDto.BaseUrl}" : updateSettingsDto.BaseUrl; + path = !path.EndsWith("/") + ? $"{path}/" + : path; setting.Value = path; _unitOfWork.SettingsRepository.Update(setting); } diff --git a/API/Parser/Parser.cs b/API/Parser/Parser.cs index bd462da28..100e831bd 100644 --- a/API/Parser/Parser.cs +++ b/API/Parser/Parser.cs @@ -232,11 +232,11 @@ namespace API.Parser MatchOptions, RegexTimeout), // 04 - Asterix the Gladiator (1964) (Digital-Empire) (WebP by Doc MaKS) new Regex( - @"^(?\d+) (- |_)?(?.*(\d{4})?)( |_)(\(|\d+)", + @"^(?\d+)\s(-\s|_)(?.*(\d{4})?)( |_)(\(|\d+)", MatchOptions, RegexTimeout), // 01 Spider-Man & Wolverine 01.cbr new Regex( - @"^(?\d+) (?:- )?(?.*) (\d+)?", + @"^(?\d+)\s(?:-\s)(?.*) (\d+)?", MatchOptions, RegexTimeout), // Batman & Wildcat (1 of 3) new Regex( diff --git a/API/Services/MetadataService.cs b/API/Services/MetadataService.cs index 820b9246c..acae153c4 100644 --- a/API/Services/MetadataService.cs +++ b/API/Services/MetadataService.cs @@ -216,8 +216,10 @@ namespace API.Services var chunkInfo = await _unitOfWork.SeriesRepository.GetChunkInfo(library.Id); var stopwatch = Stopwatch.StartNew(); var totalTime = 0L; + _logger.LogDebug($"[MetadataService] Refreshing Library {library.Name}. Total Items: {chunkInfo.TotalSize}. Total Chunks: {chunkInfo.TotalChunks} with {chunkInfo.ChunkSize} size."); - for (var chunk = 0; chunk <= chunkInfo.TotalChunks; chunk++) + // This technically does + for (var chunk = 1; chunk <= chunkInfo.TotalChunks; chunk++) { totalTime += stopwatch.ElapsedMilliseconds; stopwatch.Restart(); @@ -228,7 +230,7 @@ namespace API.Services PageNumber = chunk, PageSize = chunkInfo.ChunkSize }); - + _logger.LogDebug($"[MetadataService] Fetched {nonLibrarySeries.Count} series for refresh"); Parallel.ForEach(nonLibrarySeries, series => { _logger.LogDebug("[MetadataService] Processing series {SeriesName}", series.OriginalName); @@ -250,14 +252,20 @@ namespace API.Services if (_unitOfWork.HasChanges() && await _unitOfWork.CommitAsync()) { _logger.LogInformation( - "[MetadataService] Processed {SeriesStart} - {SeriesEnd} series in {ElapsedScanTime} milliseconds for {LibraryName}", - chunk * chunkInfo.ChunkSize, (chunk + 1) * chunkInfo.ChunkSize, stopwatch.ElapsedMilliseconds, library.Name); + "[MetadataService] Processed {SeriesStart} - {SeriesEnd} out of {TotalSeries} series in {ElapsedScanTime} milliseconds for {LibraryName}", + chunk * chunkInfo.ChunkSize, (chunk * chunkInfo.ChunkSize) + nonLibrarySeries.Count, chunkInfo.TotalSize, stopwatch.ElapsedMilliseconds, library.Name); foreach (var series in nonLibrarySeries) { await _messageHub.Clients.All.SendAsync(SignalREvents.RefreshMetadata, MessageFactory.RefreshMetadataEvent(library.Id, series.Id)); } } + else + { + _logger.LogInformation( + "[MetadataService] Processed {SeriesStart} - {SeriesEnd} out of {TotalSeries} series in {ElapsedScanTime} milliseconds for {LibraryName}", + chunk * chunkInfo.ChunkSize, (chunk * chunkInfo.ChunkSize) + nonLibrarySeries.Count, chunkInfo.TotalSize, stopwatch.ElapsedMilliseconds, library.Name); + } } _logger.LogInformation("[MetadataService] Updated metadata for {SeriesNumber} series in library {LibraryName} in {ElapsedMilliseconds} milliseconds total", chunkInfo.TotalSize, library.Name, totalTime); diff --git a/API/Services/TaskScheduler.cs b/API/Services/TaskScheduler.cs index 6400dd79e..e60161b61 100644 --- a/API/Services/TaskScheduler.cs +++ b/API/Services/TaskScheduler.cs @@ -1,4 +1,5 @@ -using System.IO; +using System; +using System.IO; using System.Threading; using System.Threading.Tasks; using API.Entities.Enums; @@ -52,27 +53,27 @@ namespace API.Services var scanLibrarySetting = setting; _logger.LogDebug("Scheduling Scan Library Task for {Setting}", scanLibrarySetting); RecurringJob.AddOrUpdate("scan-libraries", () => _scannerService.ScanLibraries(), - () => CronConverter.ConvertToCronNotation(scanLibrarySetting)); + () => CronConverter.ConvertToCronNotation(scanLibrarySetting), TimeZoneInfo.Local); } else { - RecurringJob.AddOrUpdate("scan-libraries", () => _scannerService.ScanLibraries(), Cron.Daily); + RecurringJob.AddOrUpdate("scan-libraries", () => _scannerService.ScanLibraries(), Cron.Daily, TimeZoneInfo.Local); } setting = Task.Run(() => _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.TaskBackup)).Result.Value; if (setting != null) { _logger.LogDebug("Scheduling Backup Task for {Setting}", setting); - RecurringJob.AddOrUpdate("backup", () => _backupService.BackupDatabase(), () => CronConverter.ConvertToCronNotation(setting)); + RecurringJob.AddOrUpdate("backup", () => _backupService.BackupDatabase(), () => CronConverter.ConvertToCronNotation(setting), TimeZoneInfo.Local); } else { - RecurringJob.AddOrUpdate("backup", () => _backupService.BackupDatabase(), Cron.Weekly); + RecurringJob.AddOrUpdate("backup", () => _backupService.BackupDatabase(), Cron.Weekly, TimeZoneInfo.Local); } - RecurringJob.AddOrUpdate("cleanup", () => _cleanupService.Cleanup(), Cron.Daily); + RecurringJob.AddOrUpdate("cleanup", () => _cleanupService.Cleanup(), Cron.Daily, TimeZoneInfo.Local); - RecurringJob.AddOrUpdate("check-for-updates", () => _scannerService.ScanLibraries(), Cron.Daily); + RecurringJob.AddOrUpdate("check-for-updates", () => _scannerService.ScanLibraries(), Cron.Daily, TimeZoneInfo.Local); } #region StatsTasks @@ -88,7 +89,7 @@ namespace API.Services } _logger.LogDebug("Scheduling stat collection daily"); - RecurringJob.AddOrUpdate(SendDataTask, () => _statsService.CollectAndSendStatsData(), Cron.Daily); + RecurringJob.AddOrUpdate(SendDataTask, () => _statsService.CollectAndSendStatsData(), Cron.Daily, TimeZoneInfo.Local); } public void CancelStatsTasks() @@ -111,7 +112,7 @@ namespace API.Services public void ScheduleUpdaterTasks() { _logger.LogInformation("Scheduling Auto-Update tasks"); - RecurringJob.AddOrUpdate("check-updates", () => CheckForUpdate(), Cron.Weekly); + RecurringJob.AddOrUpdate("check-updates", () => CheckForUpdate(), Cron.Weekly, TimeZoneInfo.Local); } #endregion diff --git a/API/Services/Tasks/ScannerService.cs b/API/Services/Tasks/ScannerService.cs index 2f16bf524..7f7986eb0 100644 --- a/API/Services/Tasks/ScannerService.cs +++ b/API/Services/Tasks/ScannerService.cs @@ -262,7 +262,7 @@ namespace API.Services.Tasks // Update existing series _logger.LogDebug("[ScannerService] Updating existing series"); - for (var chunk = 0; chunk <= chunkInfo.TotalChunks; chunk++) + for (var chunk = 1; chunk <= chunkInfo.TotalChunks; chunk++) { if (chunkInfo.TotalChunks == 0) continue; totalTime += stopwatch.ElapsedMilliseconds; @@ -302,7 +302,7 @@ namespace API.Services.Tasks await _unitOfWork.CommitAsync(); _logger.LogInformation( "[ScannerService] Processed {SeriesStart} - {SeriesEnd} series in {ElapsedScanTime} milliseconds for {LibraryName}", - chunk * chunkInfo.ChunkSize, (chunk + 1) * chunkInfo.ChunkSize, totalTime, library.Name); + chunk * chunkInfo.ChunkSize, (chunk * chunkInfo.ChunkSize) + nonLibrarySeries.Count, totalTime, library.Name); // Emit any series removed foreach (var missing in missingSeries) @@ -385,6 +385,10 @@ namespace API.Services.Tasks _logger.LogError(ex, "[ScannerService] There was an exception updating volumes for {SeriesName}", series.Name); } } + + _logger.LogDebug( + "[ScannerService] Added {NewSeries} series in {ElapsedScanTime} milliseconds for {LibraryName}", + newSeries.Count, stopwatch.ElapsedMilliseconds, library.Name); } private void UpdateSeries(Series series, Dictionary> parsedSeries) diff --git a/API/Startup.cs b/API/Startup.cs index d3c63cc3b..e04d61279 100644 --- a/API/Startup.cs +++ b/API/Startup.cs @@ -162,16 +162,20 @@ namespace API app.UseDefaultFiles(); - var service = serviceProvider.GetRequiredService(); - var settings = service.SettingsRepository.GetSettingsDto(); - if (!string.IsNullOrEmpty(settings.BaseUrl) && !settings.BaseUrl.Equals("/")) - { - var path = !Configuration.BaseUrl.StartsWith("/") - ? $"/{Configuration.BaseUrl}" - : Configuration.BaseUrl; - app.UsePathBase(path); - Console.WriteLine("Starting with base url as " + path); - } + // This is not implemented completely. Commenting out until implemented + // var service = serviceProvider.GetRequiredService(); + // var settings = service.SettingsRepository.GetSettingsDto(); + // if (!string.IsNullOrEmpty(settings.BaseUrl) && !settings.BaseUrl.Equals("/")) + // { + // var path = !settings.BaseUrl.StartsWith("/") + // ? $"/{settings.BaseUrl}" + // : settings.BaseUrl; + // path = !path.EndsWith("/") + // ? $"{path}/" + // : path; + // app.UsePathBase(path); + // Console.WriteLine("Starting with base url as " + path); + // } app.UseStaticFiles(new StaticFileOptions { diff --git a/API/appsettings.Development.json b/API/appsettings.Development.json index 14d557be3..b5dc22df1 100644 --- a/API/appsettings.Development.json +++ b/API/appsettings.Development.json @@ -18,7 +18,5 @@ "MaxRollingFiles": 5 } }, - "Port": 5000, - "BaseUrl": "/" - + "Port": 5000 } diff --git a/Kavita.Common/Configuration.cs b/Kavita.Common/Configuration.cs index 0c5fdfc95..c2967c883 100644 --- a/Kavita.Common/Configuration.cs +++ b/Kavita.Common/Configuration.cs @@ -8,7 +8,7 @@ namespace Kavita.Common { public static class Configuration { - private static string AppSettingsFilename = GetAppSettingFilename(); + private static readonly string AppSettingsFilename = GetAppSettingFilename(); public static string Branch { get => GetBranch(GetAppSettingFilename()); @@ -33,12 +33,6 @@ namespace Kavita.Common set => SetLogLevel(GetAppSettingFilename(), value); } - public static string BaseUrl - { - get => GetBaseUrl(GetAppSettingFilename()); - set => SetBaseUrl(GetAppSettingFilename(), value); - } - private static string GetAppSettingFilename() { if (!string.IsNullOrEmpty(AppSettingsFilename)) @@ -157,55 +151,6 @@ namespace Kavita.Common #endregion - #region BaseUrl - private static string GetBaseUrl(string filePath) - { - if (new OsInfo(Array.Empty()).IsDocker) - { - return "/"; - } - - try - { - var json = File.ReadAllText(filePath); - var jsonObj = JsonSerializer.Deserialize(json); - const string key = "BaseUrl"; - - if (jsonObj.TryGetProperty(key, out JsonElement tokenElement)) - { - return tokenElement.GetString(); - } - } - catch (Exception ex) - { - Console.WriteLine("Error reading app settings: " + ex.Message); - } - - return "/"; - } - - private static void SetBaseUrl(string filePath, string value) - { - if (new OsInfo(Array.Empty()).IsDocker) - { - return; - } - - var currentBaseUrl = GetBaseUrl(filePath); - var json = File.ReadAllText(filePath); - if (!json.Contains("BaseUrl")) - { - var lastBracket = json.LastIndexOf("}", StringComparison.Ordinal) - 1; - json = (json.Substring(0, lastBracket) + (",\n \"BaseUrl\": " + currentBaseUrl) + "}"); - } - else - { - json = json.Replace("\"BaseUrl\": " + currentBaseUrl, "\"BaseUrl\": " + value); - } - File.WriteAllText(filePath, json); - } - #endregion - #region LogLevel private static void SetLogLevel(string filePath, string logLevel) diff --git a/UI/Web/src/app/admin/manage-settings/manage-settings.component.html b/UI/Web/src/app/admin/manage-settings/manage-settings.component.html index 4467bcb6d..728876284 100644 --- a/UI/Web/src/app/admin/manage-settings/manage-settings.component.html +++ b/UI/Web/src/app/admin/manage-settings/manage-settings.component.html @@ -8,12 +8,12 @@ -
+
  diff --git a/UI/Web/src/index.html b/UI/Web/src/index.html index 0b9f60c62..ced9095a7 100644 --- a/UI/Web/src/index.html +++ b/UI/Web/src/index.html @@ -5,35 +5,14 @@ Kavita - - - - - - - - - - - - - - - + + + + + + + +