From e89a06865c4531c28359c833c3cfef107625c864 Mon Sep 17 00:00:00 2001 From: Joseph Milazzo Date: Sun, 18 Sep 2022 12:24:30 -0500 Subject: [PATCH] Misc Polish and Fixes (#1542) * Moved LibraryWatcher to utilize a queue for calculating the change event to ensure the Watcher doesn't get overwhelmed on large moves. * Fixed a security vulnerability (https://huntr.dev/bounties/8a3e652f-d6bf-436e-877e-0eaf5c69ef95/). This will be disclosed in Stable release changelog. * Tweaked the log message template * Removed some dead code from Configuration json patcher * Fixed a bug with the ComicInfo finding to properly handle root level. Fixed a bug where sometimes scanner wouldn't choose the first file with ComicInfo for filling out information. * Added new setting for managing how many logs files are allowed, just like how backups work. * Added unit tests for new CleanupLogs code * Fixed a bug where manga reader background color wasn't actually sending from the UI * Added new stats for tracking to help understand usage in the app and what features are used or not. * Fixed Stats url * Fixed a bug where volumes that had larger than 1 difference wouldn't properly return next/prev chapter (for continuous reader) * Remove a redundant test step in build pipeline, since it's already done at PR stage. * Updated dockerfile to use the new Heath check endpoint * Allow force to pass through to scan loop * Removed some old config stuff from a safety check on config in entrypoint.sh * Fixed broken unit tests due to new RBS check and how we setup mock data. --- .github/workflows/sonar-scan.yml | 4 +- API.Tests/Services/CleanupServiceTests.cs | 57 +++ API.Tests/Services/ReaderServiceTests.cs | 54 ++ API.Tests/Services/SeriesServiceTests.cs | 482 +++++++++++------- API/Controllers/HealthController.cs | 2 +- API/Controllers/SettingsController.cs | 10 + API/DTOs/Settings/ServerSettingDTO.cs | 9 +- API/DTOs/Stats/FileFormatDto.cs | 15 + API/DTOs/Stats/ServerInfoDto.cs | 24 +- API/Data/Repositories/LibraryRepository.cs | 14 + API/Data/Seed.cs | 1 + API/Entities/Enums/ServerSettingKey.cs | 5 + .../Converters/ServerSettingConverter.cs | 3 + API/Logging/LogLevelOptions.cs | 7 +- API/Services/ArchiveService.cs | 2 +- API/Services/ReaderService.cs | 17 +- API/Services/SeriesService.cs | 4 + API/Services/TaskScheduler.cs | 12 +- API/Services/Tasks/CleanupService.cs | 26 + API/Services/Tasks/Scanner/LibraryWatcher.cs | 80 ++- API/Services/Tasks/Scanner/ProcessSeries.cs | 7 +- API/Services/Tasks/ScannerService.cs | 2 +- API/Services/Tasks/StatsService.cs | 71 +++ Dockerfile | 2 +- Kavita.Common/Configuration.cs | 44 -- .../src/app/admin/_models/server-settings.ts | 1 + .../manage-settings.component.html | 24 +- .../manage-settings.component.ts | 2 + .../series-detail/series-detail.component.ts | 8 +- .../user-preferences.component.ts | 2 +- entrypoint.sh | 19 +- 31 files changed, 702 insertions(+), 308 deletions(-) create mode 100644 API/DTOs/Stats/FileFormatDto.cs diff --git a/.github/workflows/sonar-scan.yml b/.github/workflows/sonar-scan.yml index 21a4b50d5..e0f98f393 100644 --- a/.github/workflows/sonar-scan.yml +++ b/.github/workflows/sonar-scan.yml @@ -123,7 +123,7 @@ jobs: develop: name: Build Nightly Docker if Develop push - needs: [ build, test, version ] + needs: [ build, version ] runs-on: ubuntu-latest if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/develop' }} steps: @@ -232,7 +232,7 @@ jobs: stable: name: Build Stable Docker if Main push - needs: [ build, test ] + needs: [ build ] runs-on: ubuntu-latest if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} steps: diff --git a/API.Tests/Services/CleanupServiceTests.cs b/API.Tests/Services/CleanupServiceTests.cs index a0934a5ca..26216cd0a 100644 --- a/API.Tests/Services/CleanupServiceTests.cs +++ b/API.Tests/Services/CleanupServiceTests.cs @@ -38,6 +38,7 @@ public class CleanupServiceTests private const string CacheDirectory = "C:/kavita/config/cache/"; private const string CoverImageDirectory = "C:/kavita/config/covers/"; private const string BackupDirectory = "C:/kavita/config/backups/"; + private const string LogDirectory = "C:/kavita/config/logs/"; private const string BookmarkDirectory = "C:/kavita/config/bookmarks/"; @@ -84,6 +85,9 @@ public class CleanupServiceTests setting = await _context.ServerSetting.Where(s => s.Key == ServerSettingKey.BookmarkDirectory).SingleAsync(); setting.Value = BookmarkDirectory; + setting = await _context.ServerSetting.Where(s => s.Key == ServerSettingKey.TotalLogs).SingleAsync(); + setting.Value = "10"; + _context.ServerSetting.Update(setting); _context.Library.Add(new Library() @@ -412,6 +416,59 @@ public class CleanupServiceTests #endregion + #region CleanupLogs + + [Fact] + public async Task CleanupLogs_LeaveOneFile_SinceAllAreExpired() + { + var filesystem = CreateFileSystem(); + foreach (var i in Enumerable.Range(1, 10)) + { + var day = API.Services.Tasks.Scanner.Parser.Parser.PadZeros($"{i}"); + filesystem.AddFile($"{LogDirectory}kavita202009{day}.log", new MockFileData("") + { + CreationTime = DateTimeOffset.Now.Subtract(TimeSpan.FromDays(31)) + }); + } + + var ds = new DirectoryService(Substitute.For>(), filesystem); + var cleanupService = new CleanupService(_logger, _unitOfWork, _messageHub, + ds); + await cleanupService.CleanupLogs(); + Assert.Single(ds.GetFiles(LogDirectory, searchOption: SearchOption.AllDirectories)); + } + + [Fact] + public async Task CleanupLogs_LeaveLestExpired() + { + var filesystem = CreateFileSystem(); + foreach (var i in Enumerable.Range(1, 9)) + { + var day = API.Services.Tasks.Scanner.Parser.Parser.PadZeros($"{i}"); + filesystem.AddFile($"{LogDirectory}kavita202009{day}.log", new MockFileData("") + { + CreationTime = DateTimeOffset.Now.Subtract(TimeSpan.FromDays(31 - i)) + }); + } + filesystem.AddFile($"{LogDirectory}kavita20200910.log", new MockFileData("") + { + CreationTime = DateTimeOffset.Now.Subtract(TimeSpan.FromDays(31 - 10)) + }); + filesystem.AddFile($"{LogDirectory}kavita20200911.log", new MockFileData("") + { + CreationTime = DateTimeOffset.Now.Subtract(TimeSpan.FromDays(31 - 11)) + }); + + + var ds = new DirectoryService(Substitute.For>(), filesystem); + var cleanupService = new CleanupService(_logger, _unitOfWork, _messageHub, + ds); + await cleanupService.CleanupLogs(); + Assert.True(filesystem.File.Exists($"{LogDirectory}kavita20200911.log")); + } + + #endregion + // #region CleanupBookmarks // // [Fact] diff --git a/API.Tests/Services/ReaderServiceTests.cs b/API.Tests/Services/ReaderServiceTests.cs index f50cdd196..5bee50e32 100644 --- a/API.Tests/Services/ReaderServiceTests.cs +++ b/API.Tests/Services/ReaderServiceTests.cs @@ -895,6 +895,60 @@ public class ReaderServiceTests Assert.Equal("1", actualChapter.Range); } + [Fact] + public async Task GetPrevChapterIdAsync_ShouldGetPrevVolume_2() + { + await ResetDb(); + + _context.Series.Add(new Series() + { + Name = "Test", + Library = new Library() { + Name = "Test LIb", + Type = LibraryType.Manga, + }, + Volumes = new List() + { + EntityFactory.CreateVolume("0", new List() + { + EntityFactory.CreateChapter("40", false, new List(), 1), + EntityFactory.CreateChapter("50", false, new List(), 1), + EntityFactory.CreateChapter("60", false, new List(), 1), + EntityFactory.CreateChapter("Some Special Title", true, new List(), 1), + }), + EntityFactory.CreateVolume("1997", new List() + { + EntityFactory.CreateChapter("1", false, new List(), 1), + }), + EntityFactory.CreateVolume("2001", new List() + { + EntityFactory.CreateChapter("21", false, new List(), 1), + }), + EntityFactory.CreateVolume("2005", new List() + { + EntityFactory.CreateChapter("31", false, new List(), 1), + }), + } + }); + + + _context.AppUser.Add(new AppUser() + { + UserName = "majora2007" + }); + + await _context.SaveChangesAsync(); + + var readerService = new ReaderService(_unitOfWork, Substitute.For>(), Substitute.For()); + + // prevChapter should be id from ch.21 from volume 2001 + var prevChapter = await readerService.GetPrevChapterIdAsync(1, 4, 7, 1); + + var actualChapter = await _unitOfWork.ChapterRepository.GetChapterAsync(prevChapter); + Assert.NotNull(actualChapter); + Assert.Equal("21", actualChapter.Range); + } + [Fact] public async Task GetPrevChapterIdAsync_ShouldRollIntoPrevVolume() { diff --git a/API.Tests/Services/SeriesServiceTests.cs b/API.Tests/Services/SeriesServiceTests.cs index 82ebd2197..b7d134fad 100644 --- a/API.Tests/Services/SeriesServiceTests.cs +++ b/API.Tests/Services/SeriesServiceTests.cs @@ -85,19 +85,19 @@ public class SeriesServiceTests _context.ServerSetting.Update(setting); - var lib = new Library() - { - Name = "Manga", Folders = new List() {new FolderPath() {Path = "C:/data/"}} - }; - - _context.AppUser.Add(new AppUser() - { - UserName = "majora2007", - Libraries = new List() - { - lib - } - }); + // var lib = new Library() + // { + // Name = "Manga", Folders = new List() {new FolderPath() {Path = "C:/data/"}} + // }; + // + // _context.AppUser.Add(new AppUser() + // { + // UserName = "majora2007", + // Libraries = new List() + // { + // lib + // } + // }); return await _context.SaveChangesAsync() > 0; } @@ -109,6 +109,7 @@ public class SeriesServiceTests _context.Genre.RemoveRange(_context.Genre.ToList()); _context.CollectionTag.RemoveRange(_context.CollectionTag.ToList()); _context.Person.RemoveRange(_context.Person.ToList()); + _context.Library.RemoveRange(_context.Library.ToList()); await _context.SaveChangesAsync(); } @@ -135,33 +136,45 @@ public class SeriesServiceTests { await ResetDb(); - _context.Series.Add(new Series() + _context.Library.Add(new Library() { - Name = "Test", - Library = new Library() { - Name = "Test LIb", - Type = LibraryType.Manga, - }, - Volumes = new List() + AppUsers = new List() { - EntityFactory.CreateVolume("0", new List() + new AppUser() { - EntityFactory.CreateChapter("Omake", true, new List()), - EntityFactory.CreateChapter("Something SP02", true, new List()), - }), - EntityFactory.CreateVolume("2", new List() + UserName = "majora2007" + } + }, + Name = "Test LIb", + Type = LibraryType.Book, + Series = new List() + { + new Series() { - EntityFactory.CreateChapter("21", false, new List()), - EntityFactory.CreateChapter("22", false, new List()), - }), - EntityFactory.CreateVolume("3", new List() - { - EntityFactory.CreateChapter("31", false, new List()), - EntityFactory.CreateChapter("32", false, new List()), - }), + Name = "Test", + Volumes = new List() + { + EntityFactory.CreateVolume("0", new List() + { + EntityFactory.CreateChapter("Omake", true, new List()), + EntityFactory.CreateChapter("Something SP02", true, new List()), + }), + EntityFactory.CreateVolume("2", new List() + { + EntityFactory.CreateChapter("21", false, new List()), + EntityFactory.CreateChapter("22", false, new List()), + }), + EntityFactory.CreateVolume("3", new List() + { + EntityFactory.CreateChapter("31", false, new List()), + EntityFactory.CreateChapter("32", false, new List()), + }), + } + } } }); + await _context.SaveChangesAsync(); var expectedRanges = new[] {"Omake", "Something SP02"}; @@ -177,30 +190,41 @@ public class SeriesServiceTests { await ResetDb(); - _context.Series.Add(new Series() + _context.Library.Add(new Library() { - Name = "Test", - Library = new Library() { - Name = "Test LIb", - Type = LibraryType.Manga, - }, - Volumes = new List() + AppUsers = new List() { - EntityFactory.CreateVolume("0", new List() + new AppUser() { - EntityFactory.CreateChapter("1", false, new List()), - EntityFactory.CreateChapter("2", false, new List()), - }), - EntityFactory.CreateVolume("2", new List() + UserName = "majora2007" + } + }, + Name = "Test LIb", + Type = LibraryType.Manga, + Series = new List() + { + new Series() { - EntityFactory.CreateChapter("21", false, new List()), - EntityFactory.CreateChapter("22", false, new List()), - }), - EntityFactory.CreateVolume("3", new List() - { - EntityFactory.CreateChapter("31", false, new List()), - EntityFactory.CreateChapter("32", false, new List()), - }), + Name = "Test", + Volumes = new List() + { + EntityFactory.CreateVolume("0", new List() + { + EntityFactory.CreateChapter("1", false, new List()), + EntityFactory.CreateChapter("2", false, new List()), + }), + EntityFactory.CreateVolume("2", new List() + { + EntityFactory.CreateChapter("21", false, new List()), + EntityFactory.CreateChapter("22", false, new List()), + }), + EntityFactory.CreateVolume("3", new List() + { + EntityFactory.CreateChapter("31", false, new List()), + EntityFactory.CreateChapter("32", false, new List()), + }), + } + } } }); @@ -220,28 +244,39 @@ public class SeriesServiceTests { await ResetDb(); - _context.Series.Add(new Series() + _context.Library.Add(new Library() { - Name = "Test", - Library = new Library() { - Name = "Test LIb", - Type = LibraryType.Manga, - }, - Volumes = new List() + AppUsers = new List() { - EntityFactory.CreateVolume("0", new List() + new AppUser() { - EntityFactory.CreateChapter("1", false, new List()), - EntityFactory.CreateChapter("2", false, new List()), - }), - EntityFactory.CreateVolume("2", new List() + UserName = "majora2007" + } + }, + Name = "Test LIb", + Type = LibraryType.Manga, + Series = new List() + { + new Series() { - EntityFactory.CreateChapter("0", false, new List()), - }), - EntityFactory.CreateVolume("3", new List() - { - EntityFactory.CreateChapter("31", false, new List()), - }), + Name = "Test", + Volumes = new List() + { + EntityFactory.CreateVolume("0", new List() + { + EntityFactory.CreateChapter("1", false, new List()), + EntityFactory.CreateChapter("2", false, new List()), + }), + EntityFactory.CreateVolume("2", new List() + { + EntityFactory.CreateChapter("0", false, new List()), + }), + EntityFactory.CreateVolume("3", new List() + { + EntityFactory.CreateChapter("31", false, new List()), + }), + } + } } }); @@ -261,28 +296,39 @@ public class SeriesServiceTests { await ResetDb(); - _context.Series.Add(new Series() + _context.Library.Add(new Library() { - Name = "Test", - Library = new Library() { - Name = "Test LIb", - Type = LibraryType.Manga, - }, - Volumes = new List() + AppUsers = new List() { - EntityFactory.CreateVolume("0", new List() + new AppUser() { - EntityFactory.CreateChapter("1", false, new List()), - EntityFactory.CreateChapter("2", false, new List()), - }), - EntityFactory.CreateVolume("2", new List() + UserName = "majora2007" + } + }, + Name = "Test LIb", + Type = LibraryType.Manga, + Series = new List() + { + new Series() { - EntityFactory.CreateChapter("0", false, new List()), - }), - EntityFactory.CreateVolume("3", new List() - { - EntityFactory.CreateChapter("31", false, new List()), - }), + Name = "Test", + Volumes = new List() + { + EntityFactory.CreateVolume("0", new List() + { + EntityFactory.CreateChapter("1", false, new List()), + EntityFactory.CreateChapter("2", false, new List()), + }), + EntityFactory.CreateVolume("2", new List() + { + EntityFactory.CreateChapter("0", false, new List()), + }), + EntityFactory.CreateVolume("3", new List() + { + EntityFactory.CreateChapter("31", false, new List()), + }), + } + } } }); @@ -305,26 +351,38 @@ public class SeriesServiceTests { await ResetDb(); - _context.Series.Add(new Series() + _context.Library.Add(new Library() { - Name = "Test", - Library = new Library() { - Name = "Test LIb", - Type = LibraryType.Book, - }, - Volumes = new List() + AppUsers = new List() { - EntityFactory.CreateVolume("2", new List() + new AppUser() { - EntityFactory.CreateChapter("0", false, new List()), - }), - EntityFactory.CreateVolume("3", new List() + UserName = "majora2007" + } + }, + Name = "Test LIb", + Type = LibraryType.Book, + Series = new List() + { + new Series() { - EntityFactory.CreateChapter("0", false, new List()), - }), + Name = "Test", + Volumes = new List() + { + EntityFactory.CreateVolume("2", new List() + { + EntityFactory.CreateChapter("0", false, new List()), + }), + EntityFactory.CreateVolume("3", new List() + { + EntityFactory.CreateChapter("0", false, new List()), + }), + } + } } }); + await _context.SaveChangesAsync(); var detail = await _seriesService.GetSeriesDetail(1, 1); @@ -339,26 +397,39 @@ public class SeriesServiceTests { await ResetDb(); - _context.Series.Add(new Series() + _context.Library.Add(new Library() { - Name = "Test", - Library = new Library() { - Name = "Test LIb", - Type = LibraryType.Book, - }, - Volumes = new List() + AppUsers = new List() { - EntityFactory.CreateVolume("0", new List() + new AppUser() { - EntityFactory.CreateChapter("Ano Orokamono ni mo Kyakkou wo! - Volume 1.epub", true, new List()), - }), - EntityFactory.CreateVolume("2", new List() + UserName = "majora2007" + } + }, + Name = "Test LIb", + Type = LibraryType.Book, + Series = new List() + { + new Series() { - EntityFactory.CreateChapter("Ano Orokamono ni mo Kyakkou wo! - Volume 2.epub", false, new List()), - }), + Name = "Test", + Volumes = new List() + { + EntityFactory.CreateVolume("0", new List() + { + EntityFactory.CreateChapter("Ano Orokamono ni mo Kyakkou wo! - Volume 1.epub", true, new List()), + }), + EntityFactory.CreateVolume("2", new List() + { + EntityFactory.CreateChapter("Ano Orokamono ni mo Kyakkou wo! - Volume 2.epub", false, new List()), + }), + } + } } }); + + await _context.SaveChangesAsync(); var detail = await _seriesService.GetSeriesDetail(1, 1); @@ -379,36 +450,48 @@ public class SeriesServiceTests { await ResetDb(); - _context.Series.Add(new Series() + _context.Library.Add(new Library() { - Name = "Test", - Library = new Library() { - Name = "Test LIb", - Type = LibraryType.Book, - }, - Volumes = new List() + AppUsers = new List() { - EntityFactory.CreateVolume("2", new List() + new AppUser() { - EntityFactory.CreateChapter("0", false, new List()), - }), - EntityFactory.CreateVolume("1.2", new List() + UserName = "majora2007" + } + }, + Name = "Test LIb", + Type = LibraryType.Manga, + Series = new List() + { + new Series() { - EntityFactory.CreateChapter("0", false, new List()), - }), - EntityFactory.CreateVolume("1", new List() - { - EntityFactory.CreateChapter("0", false, new List()), - }), + Name = "Test", + Volumes = new List() + { + EntityFactory.CreateVolume("2", new List() + { + EntityFactory.CreateChapter("0", false, new List()), + }), + EntityFactory.CreateVolume("1.2", new List() + { + EntityFactory.CreateChapter("0", false, new List()), + }), + EntityFactory.CreateVolume("1", new List() + { + EntityFactory.CreateChapter("0", false, new List()), + }), + } + } } }); + await _context.SaveChangesAsync(); var detail = await _seriesService.GetSeriesDetail(1, 1); - Assert.Equal("1", detail.Volumes.ElementAt(0).Name); - Assert.Equal("1.2", detail.Volumes.ElementAt(1).Name); - Assert.Equal("2", detail.Volumes.ElementAt(2).Name); + Assert.Equal("Volume 1", detail.Volumes.ElementAt(0).Name); + Assert.Equal("Volume 1.2", detail.Volumes.ElementAt(1).Name); + Assert.Equal("Volume 2", detail.Volumes.ElementAt(2).Name); } @@ -422,28 +505,34 @@ public class SeriesServiceTests { await ResetDb(); - _context.Series.Add(new Series() + _context.Library.Add(new Library() { - Name = "Test", - Library = new Library() { - Name = "Test LIb", - Type = LibraryType.Manga, - }, - Volumes = new List() + AppUsers = new List() { - new Volume() + new AppUser() { - Chapters = new List() + UserName = "majora2007" + } + }, + Name = "Test LIb", + Type = LibraryType.Manga, + Series = new List() + { + new Series() + { + Name = "Test", + Volumes = new List() { - new Chapter() + EntityFactory.CreateVolume("1", new List() { - Pages = 1 - } + EntityFactory.CreateChapter("1", false, new List(), 1), + }), } } } }); + await _context.SaveChangesAsync(); @@ -470,23 +559,28 @@ public class SeriesServiceTests { await ResetDb(); - _context.Series.Add(new Series() + _context.Library.Add(new Library() { - Name = "Test", - Library = new Library() { - Name = "Test LIb", - Type = LibraryType.Manga, - }, - Volumes = new List() + AppUsers = new List() { - new Volume() + new AppUser() { - Chapters = new List() + UserName = "majora2007" + } + }, + Name = "Test LIb", + Type = LibraryType.Manga, + Series = new List() + { + new Series() + { + Name = "Test", + Volumes = new List() { - new Chapter() + EntityFactory.CreateVolume("1", new List() { - Pages = 1 - } + EntityFactory.CreateChapter("1", false, new List(), 1), + }), } } } @@ -536,23 +630,28 @@ public class SeriesServiceTests { await ResetDb(); - _context.Series.Add(new Series() + _context.Library.Add(new Library() { - Name = "Test", - Library = new Library() { - Name = "Test LIb", - Type = LibraryType.Manga, - }, - Volumes = new List() + AppUsers = new List() { - new Volume() + new AppUser() { - Chapters = new List() + UserName = "majora2007" + } + }, + Name = "Test LIb", + Type = LibraryType.Manga, + Series = new List() + { + new Series() + { + Name = "Test", + Volumes = new List() { - new Chapter() + EntityFactory.CreateVolume("1", new List() { - Pages = 1 - } + EntityFactory.CreateChapter("1", false, new List(), 1), + }), } } } @@ -583,23 +682,28 @@ public class SeriesServiceTests { await ResetDb(); - _context.Series.Add(new Series() + _context.Library.Add(new Library() { - Name = "Test", - Library = new Library() { - Name = "Test LIb", - Type = LibraryType.Manga, - }, - Volumes = new List() + AppUsers = new List() { - new Volume() + new AppUser() { - Chapters = new List() + UserName = "majora2007" + } + }, + Name = "Test LIb", + Type = LibraryType.Manga, + Series = new List() + { + new Series() + { + Name = "Test", + Volumes = new List() { - new Chapter() + EntityFactory.CreateVolume("1", new List() { - Pages = 1 - } + EntityFactory.CreateChapter("1", false, new List(), 1), + }), } } } @@ -626,18 +730,6 @@ public class SeriesServiceTests #region UpdateSeriesMetadata - private void SetupUpdateSeriesMetadataDb() - { - _context.Series.Add(new Series() - { - Name = "Test", - Library = new Library() { - Name = "Test LIb", - Type = LibraryType.Book, - } - }); - } - [Fact] public async Task UpdateSeriesMetadata_ShouldCreateEmptyMetadata_IfDoesntExist() { diff --git a/API/Controllers/HealthController.cs b/API/Controllers/HealthController.cs index 8d588fb44..c0d44582f 100644 --- a/API/Controllers/HealthController.cs +++ b/API/Controllers/HealthController.cs @@ -9,7 +9,7 @@ namespace API.Controllers; public class HealthController : BaseApiController { - [HttpGet()] + [HttpGet] public ActionResult GetHealth() { return Ok("Ok"); diff --git a/API/Controllers/SettingsController.cs b/API/Controllers/SettingsController.cs index fa3b3321e..739cb6e18 100644 --- a/API/Controllers/SettingsController.cs +++ b/API/Controllers/SettingsController.cs @@ -223,6 +223,16 @@ public class SettingsController : BaseApiController _unitOfWork.SettingsRepository.Update(setting); } + if (setting.Key == ServerSettingKey.TotalLogs && updateSettingsDto.TotalLogs + string.Empty != setting.Value) + { + if (updateSettingsDto.TotalLogs > 30 || updateSettingsDto.TotalLogs < 1) + { + return BadRequest("Total Logs must be between 1 and 30"); + } + setting.Value = updateSettingsDto.TotalLogs + string.Empty; + _unitOfWork.SettingsRepository.Update(setting); + } + if (setting.Key == ServerSettingKey.EmailServiceUrl && updateSettingsDto.EmailServiceUrl + string.Empty != setting.Value) { setting.Value = string.IsNullOrEmpty(updateSettingsDto.EmailServiceUrl) ? EmailService.DefaultApiUrl : updateSettingsDto.EmailServiceUrl; diff --git a/API/DTOs/Settings/ServerSettingDTO.cs b/API/DTOs/Settings/ServerSettingDTO.cs index 481ee2721..041c9300d 100644 --- a/API/DTOs/Settings/ServerSettingDTO.cs +++ b/API/DTOs/Settings/ServerSettingDTO.cs @@ -1,4 +1,5 @@ -using API.Services; +using System.ComponentModel.DataAnnotations; +using API.Services; namespace API.DTOs.Settings; @@ -50,7 +51,6 @@ public class ServerSettingDto /// If the Swagger UI Should be exposed. Does not require authentication, but does require a JWT. /// public bool EnableSwaggerUi { get; set; } - /// /// The amount of Backups before cleanup /// @@ -60,4 +60,9 @@ public class ServerSettingDto /// If Kavita should watch the library folders and process changes /// public bool EnableFolderWatching { get; set; } = true; + /// + /// Total number of days worth of logs to keep at a given time. + /// + /// Value should be between 1 and 30 + public int TotalLogs { get; set; } } diff --git a/API/DTOs/Stats/FileFormatDto.cs b/API/DTOs/Stats/FileFormatDto.cs new file mode 100644 index 000000000..67385e746 --- /dev/null +++ b/API/DTOs/Stats/FileFormatDto.cs @@ -0,0 +1,15 @@ +using API.Entities.Enums; + +namespace API.DTOs.Stats; + +public class FileFormatDto +{ + /// + /// The extension with the ., in lowercase + /// + public string Extension { get; set; } + /// + /// Format of extension + /// + public MangaFormat Format { get; set; } +} diff --git a/API/DTOs/Stats/ServerInfoDto.cs b/API/DTOs/Stats/ServerInfoDto.cs index 1d56c02a1..955a73f4b 100644 --- a/API/DTOs/Stats/ServerInfoDto.cs +++ b/API/DTOs/Stats/ServerInfoDto.cs @@ -1,4 +1,6 @@ -using API.Entities.Enums; +using System.Collections.Generic; +using API.Entities.Enums; +using Microsoft.AspNetCore.Mvc.RazorPages; namespace API.DTOs.Stats; @@ -118,4 +120,24 @@ public class ServerInfoDto /// /// Introduced in v0.5.4 public bool UsingSeriesRelationships { get; set; } + /// + /// A list of background colors set on the instance + /// + /// Introduced in v0.6.0 + public IEnumerable MangaReaderBackgroundColors { get; set; } + /// + /// A list of Page Split defaults being used on the instance + /// + /// Introduced in v0.6.0 + public IEnumerable MangaReaderPageSplittingModes { get; set; } + /// + /// A list of Layout Mode defaults being used on the instance + /// + /// Introduced in v0.6.0 + public IEnumerable MangaReaderLayoutModes { get; set; } + /// + /// A list of file formats existing in the instance + /// + /// Introduced in v0.6.0 + public IEnumerable FileFormats { get; set; } } diff --git a/API/Data/Repositories/LibraryRepository.cs b/API/Data/Repositories/LibraryRepository.cs index 410c3b81b..ce31660c3 100644 --- a/API/Data/Repositories/LibraryRepository.cs +++ b/API/Data/Repositories/LibraryRepository.cs @@ -38,6 +38,7 @@ public interface ILibraryRepository Task> GetLibrariesAsync(LibraryIncludes includes = LibraryIncludes.None); Task DeleteLibrary(int libraryId); Task> GetLibrariesForUserIdAsync(int userId); + Task> GetLibraryIdsForUserIdAsync(int userId); Task GetLibraryTypeAsync(int libraryId); Task> GetLibraryForIdsAsync(IEnumerable libraryIds, LibraryIncludes includes = LibraryIncludes.None); Task GetTotalFiles(); @@ -111,6 +112,11 @@ public class LibraryRepository : ILibraryRepository return await _context.SaveChangesAsync() > 0; } + /// + /// This does not track + /// + /// + /// public async Task> GetLibrariesForUserIdAsync(int userId) { return await _context.Library @@ -120,6 +126,14 @@ public class LibraryRepository : ILibraryRepository .ToListAsync(); } + public async Task> GetLibraryIdsForUserIdAsync(int userId) + { + return await _context.Library + .Where(l => l.AppUsers.Select(ap => ap.Id).Contains(userId)) + .Select(l => l.Id) + .ToListAsync(); + } + public async Task GetLibraryTypeAsync(int libraryId) { return await _context.Library diff --git a/API/Data/Seed.cs b/API/Data/Seed.cs index 3f55f31f2..15e68abeb 100644 --- a/API/Data/Seed.cs +++ b/API/Data/Seed.cs @@ -103,6 +103,7 @@ public static class Seed new() {Key = ServerSettingKey.ConvertBookmarkToWebP, Value = "false"}, new() {Key = ServerSettingKey.EnableSwaggerUi, Value = "false"}, new() {Key = ServerSettingKey.TotalBackups, Value = "30"}, + new() {Key = ServerSettingKey.TotalLogs, Value = "30"}, new() {Key = ServerSettingKey.EnableFolderWatching, Value = "false"}, }.ToArray()); diff --git a/API/Entities/Enums/ServerSettingKey.cs b/API/Entities/Enums/ServerSettingKey.cs index 16eec2cec..5c4ac7bf8 100644 --- a/API/Entities/Enums/ServerSettingKey.cs +++ b/API/Entities/Enums/ServerSettingKey.cs @@ -96,4 +96,9 @@ public enum ServerSettingKey /// [Description("EnableFolderWatching")] EnableFolderWatching = 17, + /// + /// Total number of days worth of logs to keep + /// + [Description("TotalLogs")] + TotalLogs = 18, } diff --git a/API/Helpers/Converters/ServerSettingConverter.cs b/API/Helpers/Converters/ServerSettingConverter.cs index 56c8e618f..f23fddca7 100644 --- a/API/Helpers/Converters/ServerSettingConverter.cs +++ b/API/Helpers/Converters/ServerSettingConverter.cs @@ -63,6 +63,9 @@ public class ServerSettingConverter : ITypeConverter, case ServerSettingKey.EnableFolderWatching: destination.EnableFolderWatching = bool.Parse(row.Value); break; + case ServerSettingKey.TotalLogs: + destination.TotalLogs = int.Parse(row.Value); + break; } } diff --git a/API/Logging/LogLevelOptions.cs b/API/Logging/LogLevelOptions.cs index 55a8bb9d5..66bdbe423 100644 --- a/API/Logging/LogLevelOptions.cs +++ b/API/Logging/LogLevelOptions.cs @@ -51,7 +51,7 @@ public static class LogLevelOptions .WriteTo.File(LogFile, shared: true, rollingInterval: RollingInterval.Day, - outputTemplate: "[{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} {CorrelationId} {Level}] {Message:lj}{NewLine}{Exception}"); + outputTemplate: "[{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} {CorrelationId}] [{Level}] {Message:lj}{NewLine}{Exception}"); } public static void SwitchLogLevel(string level) @@ -60,26 +60,31 @@ public static class LogLevelOptions { case "Debug": LogLevelSwitch.MinimumLevel = LogEventLevel.Debug; + MicrosoftLogLevelSwitch.MinimumLevel = LogEventLevel.Information; MicrosoftHostingLifetimeLogLevelSwitch.MinimumLevel = LogEventLevel.Debug; AspNetCoreLogLevelSwitch.MinimumLevel = LogEventLevel.Debug; break; case "Information": LogLevelSwitch.MinimumLevel = LogEventLevel.Error; + MicrosoftLogLevelSwitch.MinimumLevel = LogEventLevel.Error; MicrosoftHostingLifetimeLogLevelSwitch.MinimumLevel = LogEventLevel.Error; AspNetCoreLogLevelSwitch.MinimumLevel = LogEventLevel.Error; break; case "Trace": LogLevelSwitch.MinimumLevel = LogEventLevel.Verbose; + MicrosoftLogLevelSwitch.MinimumLevel = LogEventLevel.Information; MicrosoftHostingLifetimeLogLevelSwitch.MinimumLevel = LogEventLevel.Debug; AspNetCoreLogLevelSwitch.MinimumLevel = LogEventLevel.Debug; break; case "Warning": LogLevelSwitch.MinimumLevel = LogEventLevel.Warning; + MicrosoftLogLevelSwitch.MinimumLevel = LogEventLevel.Error; MicrosoftHostingLifetimeLogLevelSwitch.MinimumLevel = LogEventLevel.Error; AspNetCoreLogLevelSwitch.MinimumLevel = LogEventLevel.Error; break; case "Critical": LogLevelSwitch.MinimumLevel = LogEventLevel.Fatal; + MicrosoftLogLevelSwitch.MinimumLevel = LogEventLevel.Error; MicrosoftHostingLifetimeLogLevelSwitch.MinimumLevel = LogEventLevel.Error; AspNetCoreLogLevelSwitch.MinimumLevel = LogEventLevel.Error; break; diff --git a/API/Services/ArchiveService.cs b/API/Services/ArchiveService.cs index 3ec81ea7e..12c0a4029 100644 --- a/API/Services/ArchiveService.cs +++ b/API/Services/ArchiveService.cs @@ -332,7 +332,7 @@ public class ArchiveService : IArchiveService { var filenameWithoutExtension = Path.GetFileNameWithoutExtension(name).ToLower(); return !Tasks.Scanner.Parser.Parser.HasBlacklistedFolderInPath(fullName) - && fullName.Equals(ComicInfoFilename) + && (fullName.Equals(ComicInfoFilename) || (string.IsNullOrEmpty(fullName) && name.Equals(ComicInfoFilename))) && !filenameWithoutExtension.StartsWith(Tasks.Scanner.Parser.Parser.MacOsMetadataFileStartsWith); } diff --git a/API/Services/ReaderService.cs b/API/Services/ReaderService.cs index d9002c689..25cc4d35e 100644 --- a/API/Services/ReaderService.cs +++ b/API/Services/ReaderService.cs @@ -313,6 +313,7 @@ public class ReaderService : IReaderService if (chapterId > 0) return chapterId; } + var next = false; foreach (var volume in volumes) { if (volume.Number == currentVolume.Number && volume.Chapters.Count > 1) @@ -322,10 +323,17 @@ public class ReaderService : IReaderService var chapterId = GetNextChapterId(currentVolume.Chapters.OrderBy(x => double.Parse(x.Number), _chapterSortComparer), currentChapter.Range, dto => dto.Range); if (chapterId > 0) return chapterId; - + next = true; + continue; } - if (volume.Number != currentVolume.Number + 1) continue; + if (volume.Number == currentVolume.Number) + { + next = true; + continue; + } + + if (!next) continue; // Handle Chapters within next Volume // ! When selecting the chapter for the next volume, we need to make sure a c0 comes before a c1+ @@ -389,6 +397,7 @@ public class ReaderService : IReaderService if (chapterId > 0) return chapterId; } + var next = false; foreach (var volume in volumes) { if (volume.Number == currentVolume.Number) @@ -396,8 +405,10 @@ public class ReaderService : IReaderService var chapterId = GetNextChapterId(currentVolume.Chapters.OrderBy(x => double.Parse(x.Number), _chapterSortComparerForInChapterSorting).Reverse(), currentChapter.Range, dto => dto.Range); if (chapterId > 0) return chapterId; + next = true; // When the diff between volumes is more than 1, we need to explicitly tell that next volume is our use case + continue; } - if (volume.Number == currentVolume.Number - 1) + if (next) { if (currentVolume.Number - 1 == 0) break; // If we have walked all the way to chapter volume, then we should break so logic outside can work var lastChapter = volume.Chapters.MaxBy(x => double.Parse(x.Number), _chapterSortComparerForInChapterSorting); diff --git a/API/Services/SeriesService.cs b/API/Services/SeriesService.cs index 471cb2b16..c072266bb 100644 --- a/API/Services/SeriesService.cs +++ b/API/Services/SeriesService.cs @@ -13,6 +13,7 @@ using API.Entities; using API.Entities.Enums; using API.Helpers; using API.SignalR; +using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; namespace API.Services; @@ -462,6 +463,9 @@ public class SeriesService : ISeriesService public async Task GetSeriesDetail(int seriesId, int userId) { var series = await _unitOfWork.SeriesRepository.GetSeriesDtoByIdAsync(seriesId, userId); + var libraryIds = (await _unitOfWork.LibraryRepository.GetLibraryIdsForUserIdAsync(userId)); + if (!libraryIds.Contains(series.LibraryId)) + throw new UnauthorizedAccessException("User does not have access to the library this series belongs to"); var libraryType = await _unitOfWork.LibraryRepository.GetLibraryTypeAsync(series.LibraryId); var volumes = (await _unitOfWork.VolumeRepository.GetVolumesDtoAsync(seriesId, userId)) diff --git a/API/Services/TaskScheduler.cs b/API/Services/TaskScheduler.cs index b14e7c8bf..055a73e08 100644 --- a/API/Services/TaskScheduler.cs +++ b/API/Services/TaskScheduler.cs @@ -53,6 +53,7 @@ public class TaskScheduler : ITaskScheduler public const string CleanupTaskId = "cleanup"; public const string BackupTaskId = "backup"; public const string ScanLibrariesTaskId = "scan-libraries"; + public const string ReportStatsTaskId = "report-stats"; private static readonly ImmutableArray ScanTasks = ImmutableArray.Create("ScannerService", "ScanLibrary", "ScanLibraries", "ScanFolder", "ScanSeries"); @@ -123,7 +124,7 @@ public class TaskScheduler : ITaskScheduler } _logger.LogDebug("Scheduling stat collection daily"); - RecurringJob.AddOrUpdate("report-stats", () => _statsService.Send(), Cron.Daily(Rnd.Next(0, 22)), TimeZoneInfo.Local); + RecurringJob.AddOrUpdate(ReportStatsTaskId, () => _statsService.Send(), Cron.Daily(Rnd.Next(0, 22)), TimeZoneInfo.Local); } public void AnalyzeFilesForLibrary(int libraryId, bool forceUpdate = false) @@ -131,11 +132,14 @@ public class TaskScheduler : ITaskScheduler BackgroundJob.Enqueue(() => _wordCountAnalyzerService.ScanLibrary(libraryId, forceUpdate)); } + /// + /// Upon cancelling stat, we do report to the Stat service that we are no longer going to be reporting + /// public void CancelStatsTasks() { - _logger.LogDebug("Cancelling/Removing StatsTasks"); - - RecurringJob.RemoveIfExists("report-stats"); + _logger.LogDebug("Stopping Stat collection as user has opted out"); + RecurringJob.RemoveIfExists(ReportStatsTaskId); + _statsService.SendCancellation(); } /// diff --git a/API/Services/Tasks/CleanupService.cs b/API/Services/Tasks/CleanupService.cs index 1044c7ef1..bf39d8ad8 100644 --- a/API/Services/Tasks/CleanupService.cs +++ b/API/Services/Tasks/CleanupService.cs @@ -25,6 +25,7 @@ public interface ICleanupService Task DeleteChapterCoverImages(); Task DeleteTagCoverImages(); Task CleanupBackups(); + Task CleanupLogs(); void CleanupTemp(); /// /// Responsible to remove Series from Want To Read when user's have fully read the series and the series has Publication Status of Completed or Cancelled. @@ -76,6 +77,8 @@ public class CleanupService : ICleanupService await SendProgress(0.7F, "Cleaning deleted cover images"); await DeleteTagCoverImages(); await DeleteReadingListCoverImages(); + await SendProgress(0.8F, "Cleaning old logs"); + await CleanupLogs(); await SendProgress(1F, "Cleanup finished"); _logger.LogInformation("Cleanup finished"); } @@ -189,6 +192,29 @@ public class CleanupService : ICleanupService _logger.LogInformation("Finished cleanup of Database backups at {Time}", DateTime.Now); } + public async Task CleanupLogs() + { + _logger.LogInformation("Performing cleanup of logs directory"); + var dayThreshold = (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).TotalLogs; + var deltaTime = DateTime.Today.Subtract(TimeSpan.FromDays(dayThreshold)); + var allLogs = _directoryService.GetFiles(_directoryService.LogDirectory).ToList(); + var expiredLogs = allLogs.Select(filename => _directoryService.FileSystem.FileInfo.FromFileName(filename)) + .Where(f => f.CreationTime < deltaTime) + .ToList(); + + if (expiredLogs.Count == allLogs.Count) + { + _logger.LogInformation("All expired backups are older than {Threshold} days. Removing all but last backup", dayThreshold); + var toDelete = expiredLogs.OrderBy(f => f.CreationTime).ToList(); + _directoryService.DeleteFiles(toDelete.Take(toDelete.Count - 1).Select(f => f.FullName)); + } + else + { + _directoryService.DeleteFiles(expiredLogs.Select(f => f.FullName)); + } + _logger.LogInformation("Finished cleanup of logs at {Time}", DateTime.Now); + } + public void CleanupTemp() { _logger.LogInformation("Performing cleanup of Temp directory"); diff --git a/API/Services/Tasks/Scanner/LibraryWatcher.cs b/API/Services/Tasks/Scanner/LibraryWatcher.cs index 17ea744c9..bf3abc72d 100644 --- a/API/Services/Tasks/Scanner/LibraryWatcher.cs +++ b/API/Services/Tasks/Scanner/LibraryWatcher.cs @@ -91,8 +91,11 @@ public class LibraryWatcher : ILibraryWatcher /// This is just here to prevent GC from Disposing our watchers /// private readonly IList _fileWatchers = new List(); - private IList _libraryFolders = new List(); - + private static IList _libraryFolders = new List(); + /// + /// The amount of time until the Schedule ScanFolder task should be executed + /// + /// The Job will be enqueued instantly private readonly TimeSpan _queueWaitTime; @@ -109,7 +112,7 @@ public class LibraryWatcher : ILibraryWatcher public async Task StartWatching() { - _logger.LogInformation("Starting file watchers"); + _logger.LogInformation("[LibraryWatcher] Starting file watchers"); _libraryFolders = (await _unitOfWork.LibraryRepository.GetLibraryDtosAsync()) .SelectMany(l => l.Folders) @@ -119,7 +122,7 @@ public class LibraryWatcher : ILibraryWatcher .ToList(); foreach (var libraryFolder in _libraryFolders) { - _logger.LogDebug("Watching {FolderPath}", libraryFolder); + _logger.LogDebug("[LibraryWatcher] Watching {FolderPath}", libraryFolder); var watcher = new FileSystemWatcher(libraryFolder); watcher.Changed += OnChanged; @@ -138,17 +141,19 @@ public class LibraryWatcher : ILibraryWatcher _watcherDictionary[libraryFolder].Add(watcher); } + _logger.LogInformation("[LibraryWatcher] Watching {Count} folders", _fileWatchers.Count); } public void StopWatching() { - _logger.LogInformation("Stopping watching folders"); + _logger.LogInformation("[LibraryWatcher] Stopping watching folders"); foreach (var fileSystemWatcher in _watcherDictionary.Values.SelectMany(watcher => watcher)) { fileSystemWatcher.EnableRaisingEvents = false; fileSystemWatcher.Changed -= OnChanged; fileSystemWatcher.Created -= OnCreated; fileSystemWatcher.Deleted -= OnDeleted; + fileSystemWatcher.Error -= OnError; fileSystemWatcher.Dispose(); } _fileWatchers.Clear(); @@ -165,13 +170,13 @@ public class LibraryWatcher : ILibraryWatcher { if (e.ChangeType != WatcherChangeTypes.Changed) return; _logger.LogDebug("[LibraryWatcher] Changed: {FullPath}, {Name}", e.FullPath, e.Name); - ProcessChange(e.FullPath, string.IsNullOrEmpty(_directoryService.FileSystem.Path.GetExtension(e.Name))); + BackgroundJob.Enqueue(() => ProcessChange(e.FullPath, string.IsNullOrEmpty(_directoryService.FileSystem.Path.GetExtension(e.Name)))); } private void OnCreated(object sender, FileSystemEventArgs e) { _logger.LogDebug("[LibraryWatcher] Created: {FullPath}, {Name}", e.FullPath, e.Name); - ProcessChange(e.FullPath, !_directoryService.FileSystem.File.Exists(e.Name)); + BackgroundJob.Enqueue(() => ProcessChange(e.FullPath, !_directoryService.FileSystem.File.Exists(e.Name))); } /// @@ -183,7 +188,7 @@ public class LibraryWatcher : ILibraryWatcher var isDirectory = string.IsNullOrEmpty(_directoryService.FileSystem.Path.GetExtension(e.Name)); if (!isDirectory) return; _logger.LogDebug("[LibraryWatcher] Deleted: {FullPath}, {Name}", e.FullPath, e.Name); - ProcessChange(e.FullPath, true); + BackgroundJob.Enqueue(() => ProcessChange(e.FullPath, true)); } @@ -198,35 +203,42 @@ public class LibraryWatcher : ILibraryWatcher /// Processes the file or folder change. If the change is a file change and not from a supported extension, it will be ignored. /// /// This will ignore image files that are added to the system. However, they may still trigger scans due to folder changes. + /// This is public only because Hangfire will invoke it. Do not call external to this class. /// File or folder that changed /// If the change is on a directory and not a file - private void ProcessChange(string filePath, bool isDirectoryChange = false) + public void ProcessChange(string filePath, bool isDirectoryChange = false) { var sw = Stopwatch.StartNew(); + _logger.LogDebug("[LibraryWatcher] Processing change of {FilePath}", filePath); try { - // We need to check if directory or not + // If not a directory change AND file is not an archive or book, ignore if (!isDirectoryChange && - !(Parser.Parser.IsArchive(filePath) || Parser.Parser.IsBook(filePath))) return; + !(Parser.Parser.IsArchive(filePath) || Parser.Parser.IsBook(filePath))) + { + _logger.LogDebug("[LibraryWatcher] Change from {FilePath} is not an archive or book, ignoring change", filePath); + return; + } - var parentDirectory = _directoryService.GetParentDirectoryName(filePath); - if (string.IsNullOrEmpty(parentDirectory)) return; + // var libraryFolders = (await _unitOfWork.LibraryRepository.GetLibraryDtosAsync()) + // .SelectMany(l => l.Folders) + // .Distinct() + // .Select(Parser.Parser.NormalizePath) + // .Where(_directoryService.Exists) + // .ToList(); - // We need to find the library this creation belongs to - // Multiple libraries can point to the same base folder. In this case, we need use FirstOrDefault - var libraryFolder = _libraryFolders.FirstOrDefault(f => parentDirectory.Contains(f)); - if (string.IsNullOrEmpty(libraryFolder)) return; + var fullPath = GetFolder(filePath, _libraryFolders); + if (string.IsNullOrEmpty(fullPath)) + { + _logger.LogDebug("[LibraryWatcher] Change from {FilePath} could not find root level folder, ignoring change", filePath); + return; + } - var rootFolder = _directoryService.GetFoldersTillRoot(libraryFolder, filePath).ToList(); - if (!rootFolder.Any()) return; - - // Select the first folder and join with library folder, this should give us the folder to scan. - var fullPath = - Parser.Parser.NormalizePath(_directoryService.FileSystem.Path.Join(libraryFolder, rootFolder.First())); + // Check if this task has already enqueued or is being processed, before enquing var alreadyScheduled = TaskScheduler.HasAlreadyEnqueuedTask(ScannerService.Name, "ScanFolder", new object[] {fullPath}); - _logger.LogDebug("{FullPath} already enqueued: {Value}", fullPath, alreadyScheduled); + _logger.LogDebug("[LibraryWatcher] {FullPath} already enqueued: {Value}", fullPath, alreadyScheduled); if (!alreadyScheduled) { _logger.LogDebug("[LibraryWatcher] Scheduling ScanFolder for {Folder}", fullPath); @@ -242,9 +254,27 @@ public class LibraryWatcher : ILibraryWatcher { _logger.LogError(ex, "[LibraryWatcher] An error occured when processing a watch event"); } - _logger.LogDebug("ProcessChange occured in {ElapsedMilliseconds}ms", sw.ElapsedMilliseconds); + _logger.LogDebug("[LibraryWatcher] ProcessChange ran in {ElapsedMilliseconds}ms", sw.ElapsedMilliseconds); } + private string GetFolder(string filePath, IList libraryFolders) + { + var parentDirectory = _directoryService.GetParentDirectoryName(filePath); + if (string.IsNullOrEmpty(parentDirectory)) + { + return string.Empty; + } + if (string.IsNullOrEmpty(parentDirectory)) return string.Empty; + // We need to find the library this creation belongs to + // Multiple libraries can point to the same base folder. In this case, we need use FirstOrDefault + var libraryFolder = libraryFolders.FirstOrDefault(f => parentDirectory.Contains(f)); + if (string.IsNullOrEmpty(libraryFolder)) return string.Empty; + var rootFolder = _directoryService.GetFoldersTillRoot(libraryFolder, filePath).ToList(); + if (!rootFolder.Any()) return string.Empty; + + // Select the first folder and join with library folder, this should give us the folder to scan. + return Parser.Parser.NormalizePath(_directoryService.FileSystem.Path.Join(libraryFolder, rootFolder.First())); + } } diff --git a/API/Services/Tasks/Scanner/ProcessSeries.cs b/API/Services/Tasks/Scanner/ProcessSeries.cs index 465d7c624..545031abc 100644 --- a/API/Services/Tasks/Scanner/ProcessSeries.cs +++ b/API/Services/Tasks/Scanner/ProcessSeries.cs @@ -116,7 +116,8 @@ public class ProcessSeries : IProcessSeries { _logger.LogInformation("[ScannerService] Processing series {SeriesName}", series.OriginalName); - var firstParsedInfo = parsedInfos[0]; + // parsedInfos[0] is not the first volume or chapter. We need to find it using a ComicInfo check (as it uses firstParsedInfo for series sort) + var firstParsedInfo = parsedInfos.FirstOrDefault(p => p.ComicInfo != null, parsedInfos[0]); UpdateVolumes(series, parsedInfos); series.Pages = series.Volumes.Sum(v => v.Pages); @@ -479,10 +480,10 @@ public class ProcessSeries : IProcessSeries var deletedVolumes = series.Volumes.Except(nonDeletedVolumes); foreach (var volume in deletedVolumes) { - var file = volume.Chapters.FirstOrDefault()?.Files?.FirstOrDefault()?.FilePath ?? ""; + var file = volume.Chapters.FirstOrDefault()?.Files?.FirstOrDefault()?.FilePath ?? string.Empty; if (!string.IsNullOrEmpty(file) && _directoryService.FileSystem.File.Exists(file)) { - _logger.LogError( + _logger.LogInformation( "[ScannerService] Volume cleanup code was trying to remove a volume with a file still existing on disk. File: {File}", file); } diff --git a/API/Services/Tasks/ScannerService.cs b/API/Services/Tasks/ScannerService.cs index 02684c792..5b26768fc 100644 --- a/API/Services/Tasks/ScannerService.cs +++ b/API/Services/Tasks/ScannerService.cs @@ -458,7 +458,7 @@ public class ScannerService : IScannerService } - var scanElapsedTime = await ScanFiles(library, libraryFolderPaths, shouldUseLibraryScan, TrackFiles); + var scanElapsedTime = await ScanFiles(library, libraryFolderPaths, shouldUseLibraryScan, TrackFiles, forceUpdate); await Task.WhenAll(processTasks); diff --git a/API/Services/Tasks/StatsService.cs b/API/Services/Tasks/StatsService.cs index a190a4113..21af4a46b 100644 --- a/API/Services/Tasks/StatsService.cs +++ b/API/Services/Tasks/StatsService.cs @@ -1,4 +1,6 @@ using System; +using System.Collections.Generic; +using System.IO; using System.Linq; using System.Net.Http; using System.Runtime.InteropServices; @@ -21,6 +23,7 @@ public interface IStatsService { Task Send(); Task GetServerInfo(); + Task SendCancellation(); } public class StatsService : IStatsService { @@ -127,6 +130,10 @@ public class StatsService : IStatsService MaxSeriesInALibrary = await MaxSeriesInAnyLibrary(), MaxVolumesInASeries = await MaxVolumesInASeries(), MaxChaptersInASeries = await MaxChaptersInASeries(), + MangaReaderBackgroundColors = await AllMangaReaderBackgroundColors(), + MangaReaderPageSplittingModes = await AllMangaReaderPageSplitting(), + MangaReaderLayoutModes = await AllMangaReaderLayoutModes(), + FileFormats = AllFormats(), }; var usersWithPref = (await _unitOfWork.UserRepository.GetAllUsersAsync(AppUserIncludes.UserPreferences)).ToList(); @@ -149,6 +156,39 @@ public class StatsService : IStatsService return serverInfo; } + public async Task SendCancellation() + { + _logger.LogInformation("Informing KavitaStats that this instance is no longer sending stats"); + var installId = (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).InstallId; + + var responseContent = string.Empty; + + try + { + var response = await (ApiUrl + "/api/v2/stats/opt-out?installId=" + installId) + .WithHeader("Accept", "application/json") + .WithHeader("User-Agent", "Kavita") + .WithHeader("x-api-key", "MsnvA2DfQqxSK5jh") + .WithHeader("x-kavita-version", BuildInfo.Version) + .WithHeader("Content-Type", "application/json") + .WithTimeout(TimeSpan.FromSeconds(30)) + .PostAsync(); + + if (response.StatusCode != StatusCodes.Status200OK) + { + _logger.LogError("KavitaStats did not respond successfully. {Content}", response); + } + } + catch (HttpRequestException e) + { + _logger.LogError(e, "KavitaStats did not respond successfully. {Response}", responseContent); + } + catch (Exception e) + { + _logger.LogError(e, "An error happened during the request to KavitaStats"); + } + } + private Task GetIfUsingSeriesRelationship() { return _context.SeriesRelation.AnyAsync(); @@ -190,4 +230,35 @@ public class StatsService : IStatsService .SelectMany(v => v.Chapters) .Count()); } + + private async Task> AllMangaReaderBackgroundColors() + { + return await _context.AppUserPreferences.Select(p => p.BackgroundColor).Distinct().ToListAsync(); + } + + private async Task> AllMangaReaderPageSplitting() + { + return await _context.AppUserPreferences.Select(p => p.PageSplitOption).Distinct().ToListAsync(); + } + + private async Task> AllMangaReaderLayoutModes() + { + return await _context.AppUserPreferences.Select(p => p.LayoutMode).Distinct().ToListAsync(); + } + + private IEnumerable AllFormats() + { + var results = _context.MangaFile + .AsNoTracking() + .AsEnumerable() + .Select(m => new FileFormatDto() + { + Format = m.Format, + Extension = Path.GetExtension(m.FilePath)?.ToLowerInvariant() + }) + .DistinctBy(f => f.Extension) + .ToList(); + + return results; + } } diff --git a/Dockerfile b/Dockerfile index c8e090534..c7757581c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,7 +29,7 @@ EXPOSE 5000 WORKDIR /kavita -HEALTHCHECK --interval=30s --timeout=15s --start-period=30s --retries=3 CMD curl --fail http://localhost:5000 || exit 1 +HEALTHCHECK --interval=30s --timeout=15s --start-period=30s --retries=3 CMD curl --fail http://localhost:5000/api/health || exit 1 ENTRYPOINT [ "/bin/bash" ] CMD ["/entrypoint.sh"] diff --git a/Kavita.Common/Configuration.cs b/Kavita.Common/Configuration.cs index 5457c4e7b..0302372d6 100644 --- a/Kavita.Common/Configuration.cs +++ b/Kavita.Common/Configuration.cs @@ -11,12 +11,6 @@ public static class Configuration { public static readonly string AppSettingsFilename = Path.Join("config", GetAppSettingFilename()); - public static string Branch - { - get => GetBranch(GetAppSettingFilename()); - set => SetBranch(GetAppSettingFilename(), value); - } - public static int Port { get => GetPort(GetAppSettingFilename()); @@ -146,42 +140,4 @@ public static class Configuration } #endregion - - private static string GetBranch(string filePath) - { - const string defaultBranch = "main"; - - try - { - var json = File.ReadAllText(filePath); - var jsonObj = JsonSerializer.Deserialize(json); - const string key = "Branch"; - - if (jsonObj.TryGetProperty(key, out JsonElement tokenElement)) - { - return tokenElement.GetString(); - } - } - catch (Exception ex) - { - Console.WriteLine("Error reading app settings: " + ex.Message); - } - - return defaultBranch; - } - - private static void SetBranch(string filePath, string updatedBranch) - { - try - { - var currentBranch = GetBranch(filePath); - var json = File.ReadAllText(filePath) - .Replace("\"Branch\": " + currentBranch, "\"Branch\": " + updatedBranch); - File.WriteAllText(filePath, json); - } - catch (Exception) - { - /* Swallow Exception */ - } - } } diff --git a/UI/Web/src/app/admin/_models/server-settings.ts b/UI/Web/src/app/admin/_models/server-settings.ts index 72438a431..f7e05f895 100644 --- a/UI/Web/src/app/admin/_models/server-settings.ts +++ b/UI/Web/src/app/admin/_models/server-settings.ts @@ -12,5 +12,6 @@ export interface ServerSettings { convertBookmarkToWebP: boolean; enableSwaggerUi: boolean; totalBackups: number; + totalLogs: number; enableFolderWatching: boolean; } diff --git a/UI/Web/src/app/admin/manage-settings/manage-settings.component.html b/UI/Web/src/app/admin/manage-settings/manage-settings.component.html index 793076ef5..ae603eda3 100644 --- a/UI/Web/src/app/admin/manage-settings/manage-settings.component.html +++ b/UI/Web/src/app/admin/manage-settings/manage-settings.component.html @@ -21,14 +21,14 @@
-
+
  Port the server listens on. This is fixed if you are running on Docker. Requires restart to take effect.
-
+
  The number of backups to maintain. Default is 30, minumum is 1, maximum is 30. @@ -45,8 +45,26 @@

+ +
+   + The number of logs to maintain. Default is 30, minumum is 1, maximum is 30. + + + +

+ You must have at least 1 log +

+

+ You cannot have more than {{errors.max.max}} logs +

+

+ This field is required +

+ +
-
+
  Use debug to help identify issues. Debug can eat up a lot of disk space. diff --git a/UI/Web/src/app/admin/manage-settings/manage-settings.component.ts b/UI/Web/src/app/admin/manage-settings/manage-settings.component.ts index 913fe6f27..9aef206e9 100644 --- a/UI/Web/src/app/admin/manage-settings/manage-settings.component.ts +++ b/UI/Web/src/app/admin/manage-settings/manage-settings.component.ts @@ -49,6 +49,7 @@ export class ManageSettingsComponent implements OnInit { this.settingsForm.addControl('emailServiceUrl', new FormControl(this.serverSettings.emailServiceUrl, [Validators.required])); this.settingsForm.addControl('enableSwaggerUi', new FormControl(this.serverSettings.enableSwaggerUi, [Validators.required])); this.settingsForm.addControl('totalBackups', new FormControl(this.serverSettings.totalBackups, [Validators.required, Validators.min(1), Validators.max(30)])); + this.settingsForm.addControl('totalLogs', new FormControl(this.serverSettings.totalLogs, [Validators.required, Validators.min(1), Validators.max(30)])); this.settingsForm.addControl('enableFolderWatching', new FormControl(this.serverSettings.enableFolderWatching, [Validators.required])); this.settingsForm.addControl('convertBookmarkToWebP', new FormControl(this.serverSettings.convertBookmarkToWebP, [])); }); @@ -67,6 +68,7 @@ export class ManageSettingsComponent implements OnInit { this.settingsForm.get('emailServiceUrl')?.setValue(this.serverSettings.emailServiceUrl); this.settingsForm.get('enableSwaggerUi')?.setValue(this.serverSettings.enableSwaggerUi); this.settingsForm.get('totalBackups')?.setValue(this.serverSettings.totalBackups); + this.settingsForm.get('totalLogs')?.setValue(this.serverSettings.totalLogs); this.settingsForm.get('enableFolderWatching')?.setValue(this.serverSettings.enableFolderWatching); this.settingsForm.get('convertBookmarkToWebP')?.setValue(this.serverSettings.convertBookmarkToWebP); this.settingsForm.markAsPristine(); diff --git a/UI/Web/src/app/series-detail/series-detail.component.ts b/UI/Web/src/app/series-detail/series-detail.component.ts index 41c6bdc4d..9b42022bc 100644 --- a/UI/Web/src/app/series-detail/series-detail.component.ts +++ b/UI/Web/src/app/series-detail/series-detail.component.ts @@ -3,7 +3,7 @@ import { Title } from '@angular/platform-browser'; import { ActivatedRoute, Router } from '@angular/router'; import { NgbModal, NgbNavChangeEvent, NgbOffcanvas } from '@ng-bootstrap/ng-bootstrap'; import { ToastrService } from 'ngx-toastr'; -import { forkJoin, Subject } from 'rxjs'; +import { catchError, forkJoin, of, Subject } from 'rxjs'; import { take, takeUntil } from 'rxjs/operators'; import { BulkSelectionService } from '../cards/bulk-selection.service'; import { EditSeriesModalComponent } from '../cards/_modals/edit-series-modal/edit-series-modal.component'; @@ -511,7 +511,11 @@ export class SeriesDetailComponent implements OnInit, OnDestroy, AfterContentChe } }); - this.seriesService.getSeriesDetail(this.seriesId).subscribe(detail => { + this.seriesService.getSeriesDetail(this.seriesId).pipe(catchError(err => { + this.router.navigateByUrl('/libraries'); + return of(null); + })).subscribe(detail => { + if (detail == null) return; this.hasSpecials = detail.specials.length > 0; this.specials = detail.specials; diff --git a/UI/Web/src/app/user-settings/user-preferences/user-preferences.component.ts b/UI/Web/src/app/user-settings/user-preferences/user-preferences.component.ts index bf1210156..efbcb9599 100644 --- a/UI/Web/src/app/user-settings/user-preferences/user-preferences.component.ts +++ b/UI/Web/src/app/user-settings/user-preferences/user-preferences.component.ts @@ -210,7 +210,7 @@ export class UserPreferencesComponent implements OnInit, OnDestroy { readerMode: parseInt(modelSettings.readerMode, 10), layoutMode: parseInt(modelSettings.layoutMode, 10), showScreenHints: modelSettings.showScreenHints, - backgroundColor: modelSettings.backgroundColor, + backgroundColor: this.user.preferences.backgroundColor, bookReaderFontFamily: modelSettings.bookReaderFontFamily, bookReaderLineSpacing: modelSettings.bookReaderLineSpacing, bookReaderFontSize: modelSettings.bookReaderFontSize, diff --git a/entrypoint.sh b/entrypoint.sh index 53bed162f..155cf62e7 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -3,24 +3,7 @@ if [ ! -f "/kavita/config/appsettings.json" ]; then echo "Kavita configuration file does not exist, creating..." echo '{ - "ConnectionStrings": { - "DefaultConnection": "Data source=config//kavita.db" - }, "TokenKey": "super secret unguessable key", - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft": "Information", - "Microsoft.Hosting.Lifetime": "Error", - "Hangfire": "Information", - "Microsoft.AspNetCore.Hosting.Internal.WebHost": "Information" - }, - "File": { - "Path": "config//logs/kavita.log", - "Append": "True", - "FileSizeLimitBytes": 26214400, - "MaxRollingFiles": 2 - } }, "Port": 5000 }' >> /kavita/config/appsettings.json @@ -28,4 +11,4 @@ fi chmod +x Kavita -./Kavita \ No newline at end of file +./Kavita