From 6ea9f2c73e46419bb2e7a86842d842fd5c43977f Mon Sep 17 00:00:00 2001 From: Joe Milazzo Date: Sun, 9 Oct 2022 11:23:41 -0500 Subject: [PATCH] Scan Loop Fortification (#1573) * Cleanup some messaging in the scan loop to be more context bearing * Added Response Caching to Series Detail for 1 min, due to the heavy nature of the call. * Refactored code to make it so that processing of series runs sync correctly. Added a log to inform the user of corrupted volume from buggy code in v0.5.6. * Moved folder watching out of experimental * Fixed an issue where empty folders could break the scan loop * Another fix for when dates aren't valid, the scanner wouldn't get the proper min and would throw an exception (develop) * Implemented the ability to edit release year from the UI for a series. * Added a unit test for some new logic * Code smells * Rewrote the handler for suspending watching to be more resilient and ensure no two threads have a race condition. * More error handling for when a ScanFolder is invoked but multiple series belong to that folder, log it to the user and default to a library scan. * ScanSeries now will check for kavitaignores higher than it's own folder and respect library level. * Fixed an issue where image series with a folder name containing the word "folder" could get ignored as it thought the image was a cover image. When a series folder is moved or deleted, skip parent ignore finding. * Removed some old files, added in scanFolder a check if the series found for a folder is in a book library and if so to always do a library scan (as books are often nested into one folder with multiple series). Added some unit tests * Refactored some scan loop logic into ComicInfo, wrote tests and updated some documentation to make the fields more clear. * Added a test for GetLastWriteTime based on recent bug * Cleaned up some redundant code * Fixed a bad merge * Code smells * Removed a package that's no longer used. * Ensure we check against ScanQueue on ScanFolder enqueuing * Documentation and more bullet proofing to ensure Hangfire checks work more as expected --- .gitignore | 1 + API.Tests/Entities/ComicInfoTests.cs | 58 ++++++++++ .../Extensions/ChapterListExtensionsTests.cs | 44 +++++++- API.Tests/Parser/DefaultParserTests.cs | 41 +++++-- API.Tests/Services/DirectoryServiceTests.cs | 16 +++ API/Data/Metadata/ComicInfo.cs | 18 +++ API/Data/Repositories/SeriesRepository.cs | 82 ++++++-------- API/Entities/Chapter.cs | 5 +- API/Extensions/ChapterListExtensions.cs | 10 ++ API/Services/DirectoryService.cs | 2 +- API/Services/TaskScheduler.cs | 71 +++++++++++- API/Services/Tasks/Scanner/LibraryWatcher.cs | 80 +++++++------- .../Tasks/Scanner/ParseScannedFiles.cs | 39 ++++++- .../Tasks/Scanner/Parser/DefaultParser.cs | 2 +- API/Services/Tasks/Scanner/ProcessSeries.cs | 18 +-- API/Services/Tasks/ScannerService.cs | 40 +++++-- Kavita.sln.DotSettings | 2 + UI/Web/package-lock.json | 8 -- UI/Web/package.json | 1 - action-build.sh.OLD | 103 ------------------ 20 files changed, 391 insertions(+), 250 deletions(-) delete mode 100644 action-build.sh.OLD diff --git a/.gitignore b/.gitignore index 9e470748b..078b6108c 100644 --- a/.gitignore +++ b/.gitignore @@ -530,3 +530,4 @@ API.Tests/TestResults/ UI/Web/.vscode/settings.json /API.Tests/Services/Test Data/ArchiveService/CoverImages/output/* UI/Web/.angular/ +BenchmarkDotNet.Artifacts \ No newline at end of file diff --git a/API.Tests/Entities/ComicInfoTests.cs b/API.Tests/Entities/ComicInfoTests.cs index 325299cf8..ea8b0187d 100644 --- a/API.Tests/Entities/ComicInfoTests.cs +++ b/API.Tests/Entities/ComicInfoTests.cs @@ -35,4 +35,62 @@ public class ComicInfoTests Assert.Equal(AgeRating.RatingPending, ComicInfo.ConvertAgeRatingToEnum("rating pending")); } #endregion + + + #region CalculatedCount + + [Fact] + public void CalculatedCount_ReturnsVolumeCount() + { + var ci = new ComicInfo() + { + Number = "5", + Volume = "10", + Count = 10 + }; + + Assert.Equal(5, ci.CalculatedCount()); + } + + [Fact] + public void CalculatedCount_ReturnsNoCountWhenCountNotSet() + { + var ci = new ComicInfo() + { + Number = "5", + Volume = "10", + Count = 0 + }; + + Assert.Equal(5, ci.CalculatedCount()); + } + + [Fact] + public void CalculatedCount_ReturnsNumberCount() + { + var ci = new ComicInfo() + { + Number = "5", + Volume = "", + Count = 10 + }; + + Assert.Equal(5, ci.CalculatedCount()); + } + + [Fact] + public void CalculatedCount_ReturnsNumberCount_OnlyWholeNumber() + { + var ci = new ComicInfo() + { + Number = "5.7", + Volume = "", + Count = 10 + }; + + Assert.Equal(5, ci.CalculatedCount()); + } + + + #endregion } diff --git a/API.Tests/Extensions/ChapterListExtensionsTests.cs b/API.Tests/Extensions/ChapterListExtensionsTests.cs index 698ab185b..f6ea62408 100644 --- a/API.Tests/Extensions/ChapterListExtensionsTests.cs +++ b/API.Tests/Extensions/ChapterListExtensionsTests.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.Linq; using API.Entities; @@ -108,7 +109,6 @@ public class ChapterListExtensionsTests var actualChapter = chapterList.GetChapterByRange(info); Assert.Equal(chapterList[0], actualChapter); - } #region GetFirstChapterWithFiles @@ -140,5 +140,47 @@ public class ChapterListExtensionsTests } + #endregion + + #region MinimumReleaseYear + + [Fact] + public void MinimumReleaseYear_ZeroIfNoChapters() + { + var chapterList = new List(); + + Assert.Equal(0, chapterList.MinimumReleaseYear()); + } + + [Fact] + public void MinimumReleaseYear_ZeroIfNoValidDates() + { + var chapterList = new List() + { + CreateChapter("detective comics", "0", CreateFile("/manga/detective comics #001.cbz", MangaFormat.Archive), true), + CreateChapter("detective comics", "0", CreateFile("/manga/detective comics #001.cbz", MangaFormat.Archive), true) + }; + + chapterList[0].ReleaseDate = new DateTime(10, 1, 1); + chapterList[1].ReleaseDate = DateTime.MinValue; + + Assert.Equal(0, chapterList.MinimumReleaseYear()); + } + + [Fact] + public void MinimumReleaseYear_MinValidReleaseYear() + { + var chapterList = new List() + { + CreateChapter("detective comics", "0", CreateFile("/manga/detective comics #001.cbz", MangaFormat.Archive), true), + CreateChapter("detective comics", "0", CreateFile("/manga/detective comics #001.cbz", MangaFormat.Archive), true) + }; + + chapterList[0].ReleaseDate = new DateTime(2002, 1, 1); + chapterList[1].ReleaseDate = new DateTime(2012, 2, 1); + + Assert.Equal(2002, chapterList.MinimumReleaseYear()); + } + #endregion } diff --git a/API.Tests/Parser/DefaultParserTests.cs b/API.Tests/Parser/DefaultParserTests.cs index 32768d11a..2640aa6c2 100644 --- a/API.Tests/Parser/DefaultParserTests.cs +++ b/API.Tests/Parser/DefaultParserTests.cs @@ -103,6 +103,7 @@ public class DefaultParserTests { const string rootPath = @"E:/Manga/"; var expected = new Dictionary(); + var filepath = @"E:/Manga/Mujaki no Rakuen/Mujaki no Rakuen Vol12 ch76.cbz"; expected.Add(filepath, new ParserInfo { @@ -215,14 +216,6 @@ public class DefaultParserTests FullFilePath = filepath, IsSpecial = false }); - filepath = @"E:\Manga\Harrison, Kim - The Good, The Bad, and the Undead - Hollows Vol 2.5.epub"; - expected.Add(filepath, new ParserInfo - { - Series = "Harrison, Kim - The Good, The Bad, and the Undead - Hollows", Volumes = "2.5", Edition = "", - Chapters = "0", Filename = "Harrison, Kim - The Good, The Bad, and the Undead - Hollows Vol 2.5.epub", Format = MangaFormat.Epub, - FullFilePath = filepath, IsSpecial = false - }); - // If an image is cover exclusively, ignore it filepath = @"E:\Manga\Seraph of the End\cover.png"; expected.Add(filepath, null); @@ -235,11 +228,12 @@ public class DefaultParserTests FullFilePath = filepath, IsSpecial = false }); + // Note: Fallback to folder will parse Monster #8 and get Monster filepath = @"E:\Manga\Monster #8\Ch. 001-016 [MangaPlus] [Digital] [amit34521]\Monster #8 Ch. 001 [MangaPlus] [Digital] [amit34521]\13.jpg"; expected.Add(filepath, new ParserInfo { - Series = "Monster #8", Volumes = "0", Edition = "", - Chapters = "1", Filename = "13.jpg", Format = MangaFormat.Archive, + Series = "Monster", Volumes = "0", Edition = "", + Chapters = "1", Filename = "13.jpg", Format = MangaFormat.Image, FullFilePath = filepath, IsSpecial = false }); @@ -251,6 +245,29 @@ public class DefaultParserTests FullFilePath = filepath, IsSpecial = false }); + filepath = @"E:\Manga\Extra layer for no reason\Just Images the second\Vol19\ch186\Vol. 19 p106.gif"; + expected.Add(filepath, new ParserInfo + { + Series = "Just Images the second", Volumes = "19", Edition = "", + Chapters = "186", Filename = "Vol. 19 p106.gif", Format = MangaFormat.Image, + FullFilePath = filepath, IsSpecial = false + }); + + filepath = @"E:\Manga\Extra layer for no reason\Just Images the second\Blank Folder\Vol19\ch186\Vol. 19 p106.gif"; + expected.Add(filepath, new ParserInfo + { + Series = "Just Images the second", Volumes = "19", Edition = "", + Chapters = "186", Filename = "Vol. 19 p106.gif", Format = MangaFormat.Image, + FullFilePath = filepath, IsSpecial = false + }); + + filepath = @"E:\Manga\Harrison, Kim - The Good, The Bad, and the Undead - Hollows Vol 2.5.epub"; + expected.Add(filepath, new ParserInfo + { + Series = "Harrison, Kim - The Good, The Bad, and the Undead - Hollows", Volumes = "2.5", Edition = "", + Chapters = "0", Filename = "Harrison, Kim - The Good, The Bad, and the Undead - Hollows Vol 2.5.epub", Format = MangaFormat.Epub, + FullFilePath = filepath, IsSpecial = false + }); foreach (var file in expected.Keys) { @@ -259,7 +276,7 @@ public class DefaultParserTests if (expectedInfo == null) { Assert.Null(actual); - return; + continue; } Assert.NotNull(actual); _testOutputHelper.WriteLine($"Validating {file}"); @@ -399,7 +416,7 @@ public class DefaultParserTests if (expectedInfo == null) { Assert.Null(actual); - return; + continue; } Assert.NotNull(actual); _testOutputHelper.WriteLine($"Validating {file}"); diff --git a/API.Tests/Services/DirectoryServiceTests.cs b/API.Tests/Services/DirectoryServiceTests.cs index 0f6340c5b..254d851fa 100644 --- a/API.Tests/Services/DirectoryServiceTests.cs +++ b/API.Tests/Services/DirectoryServiceTests.cs @@ -995,4 +995,20 @@ public class DirectoryServiceTests } #endregion + + #region GetLastWriteTime + + [Fact] + public void GetLastWriteTime_ShouldReturnMaxTime_IfNoFiles() + { + const string dir = "C:/manga/"; + var filesystem = new MockFileSystem(); + filesystem.AddDirectory("C:/"); + filesystem.AddDirectory(dir); + var ds = new DirectoryService(Substitute.For>(), filesystem); + + Assert.Equal(DateTime.MaxValue, ds.GetLastWriteTime(dir)); + } + + #endregion } diff --git a/API/Data/Metadata/ComicInfo.cs b/API/Data/Metadata/ComicInfo.cs index 20061f063..e1b4ee994 100644 --- a/API/Data/Metadata/ComicInfo.cs +++ b/API/Data/Metadata/ComicInfo.cs @@ -124,5 +124,23 @@ public class ComicInfo info.CoverArtist = Services.Tasks.Scanner.Parser.Parser.CleanAuthor(info.CoverArtist); } + /// + /// Uses both Volume and Number to make an educated guess as to what count refers to and it's highest number. + /// + /// + public int CalculatedCount() + { + if (!string.IsNullOrEmpty(Number) && float.Parse(Number) > 0) + { + return (int) Math.Floor(float.Parse(Number)); + } + if (!string.IsNullOrEmpty(Volume) && float.Parse(Volume) > 0) + { + return Math.Max(Count, (int) Math.Floor(float.Parse(Volume))); + } + + return Count; + } + } diff --git a/API/Data/Repositories/SeriesRepository.cs b/API/Data/Repositories/SeriesRepository.cs index d9ac6779e..dbb78641a 100644 --- a/API/Data/Repositories/SeriesRepository.cs +++ b/API/Data/Repositories/SeriesRepository.cs @@ -33,8 +33,7 @@ public enum SeriesIncludes Volumes = 2, Metadata = 4, Related = 8, - //Related = 16, - //UserPreferences = 32 + Library = 16, } internal class RecentlyAddedSeries @@ -120,8 +119,7 @@ public interface ISeriesRepository Task GetSeriesForChapter(int chapterId, int userId); Task> GetWantToReadForUserAsync(int userId, UserParams userParams, FilterDto filter); Task GetSeriesIdByFolder(string folder); - Task GetSeriesByFolderPath(string folder); - Task GetFullSeriesByName(string series, int libraryId); + Task GetSeriesByFolderPath(string folder, SeriesIncludes includes = SeriesIncludes.None); Task GetFullSeriesByAnyName(string seriesName, string localizedName, int libraryId, MangaFormat format, bool withFullIncludes = true); Task> RemoveSeriesNotInList(IList seenSeries, int libraryId); Task>> GetFolderPathMap(int libraryId); @@ -1173,52 +1171,16 @@ public class SeriesRepository : ISeriesRepository /// Return a Series by Folder path. Null if not found. /// /// This will be normalized in the query + /// Additional relationships to include with the base query /// - public async Task GetSeriesByFolderPath(string folder) + public async Task GetSeriesByFolderPath(string folder, SeriesIncludes includes = SeriesIncludes.None) { var normalized = Services.Tasks.Scanner.Parser.Parser.NormalizePath(folder); - return await _context.Series.SingleOrDefaultAsync(s => s.FolderPath.Equals(normalized)); - } + var query = _context.Series.Where(s => s.FolderPath.Equals(normalized)); - /// - /// Finds a series by series name for a given library. - /// - /// This pulls everything with the Series, so should be used only when needing tracking on all related tables - /// - /// - /// - public Task GetFullSeriesByName(string series, int libraryId) - { - var localizedSeries = Services.Tasks.Scanner.Parser.Parser.Normalize(series); - return _context.Series - .Where(s => (s.NormalizedName.Equals(localizedSeries) - || s.LocalizedName.Equals(series)) && s.LibraryId == libraryId) - .Include(s => s.Metadata) - .ThenInclude(m => m.People) - .Include(s => s.Metadata) - .ThenInclude(m => m.Genres) - .Include(s => s.Library) - .Include(s => s.Volumes) - .ThenInclude(v => v.Chapters) - .ThenInclude(cm => cm.People) + query = AddIncludesToQuery(query, includes); - .Include(s => s.Volumes) - .ThenInclude(v => v.Chapters) - .ThenInclude(c => c.Tags) - - .Include(s => s.Volumes) - .ThenInclude(v => v.Chapters) - .ThenInclude(c => c.Genres) - - - .Include(s => s.Metadata) - .ThenInclude(m => m.Tags) - - .Include(s => s.Volumes) - .ThenInclude(v => v.Chapters) - .ThenInclude(c => c.Files) - .AsSplitQuery() - .SingleOrDefaultAsync(); + return await query.SingleOrDefaultAsync(); } /// @@ -1240,6 +1202,7 @@ public class SeriesRepository : ISeriesRepository .Where(s => s.Format == format && format != MangaFormat.Unknown) .Where(s => s.NormalizedName.Equals(normalizedSeries) || (s.NormalizedLocalizedName.Equals(normalizedSeries) && s.NormalizedLocalizedName != string.Empty)); + if (!string.IsNullOrEmpty(normalizedLocalized)) { query = query.Where(s => @@ -1516,7 +1479,8 @@ public class SeriesRepository : ISeriesRepository LastScanned = s.LastFolderScanned, SeriesName = s.Name, FolderPath = s.FolderPath, - Format = s.Format + Format = s.Format, + LibraryRoots = s.Library.Folders.Select(f => f.Path) }).ToListAsync(); var map = new Dictionary>(); @@ -1538,4 +1502,30 @@ public class SeriesRepository : ISeriesRepository return map; } + + private static IQueryable AddIncludesToQuery(IQueryable query, SeriesIncludes includeFlags) + { + if (includeFlags.HasFlag(SeriesIncludes.Library)) + { + query = query.Include(u => u.Library); + } + + if (includeFlags.HasFlag(SeriesIncludes.Related)) + { + query = query.Include(u => u.Relations); + } + + if (includeFlags.HasFlag(SeriesIncludes.Metadata)) + { + query = query.Include(u => u.Metadata); + } + + if (includeFlags.HasFlag(SeriesIncludes.Volumes)) + { + query = query.Include(u => u.Volumes); + } + + + return query; + } } diff --git a/API/Entities/Chapter.cs b/API/Entities/Chapter.cs index eb0cecc17..cc0db195c 100644 --- a/API/Entities/Chapter.cs +++ b/API/Entities/Chapter.cs @@ -65,11 +65,12 @@ public class Chapter : IEntityDate, IHasReadTimeEstimate /// public string Language { get; set; } /// - /// Total number of issues in the series + /// Total number of issues or volumes in the series /// + /// Users may use Volume count or issue count. Kavita performs some light logic to help Count match up with TotalCount public int TotalCount { get; set; } = 0; /// - /// Number in the Total Count + /// Number of the Total Count (progress the Series is complete) /// public int Count { get; set; } = 0; diff --git a/API/Extensions/ChapterListExtensions.cs b/API/Extensions/ChapterListExtensions.cs index 94a9675b8..c00fa1873 100644 --- a/API/Extensions/ChapterListExtensions.cs +++ b/API/Extensions/ChapterListExtensions.cs @@ -31,4 +31,14 @@ public static class ChapterListExtensions ? chapters.FirstOrDefault(c => c.Range == info.Filename || (c.Files.Select(f => f.FilePath).Contains(info.FullFilePath))) : chapters.FirstOrDefault(c => c.Range == info.Chapters); } + + /// + /// Returns the minimum Release Year from all Chapters that meets the year requirement (>= 1000) + /// + /// + /// + public static int MinimumReleaseYear(this IList chapters) + { + return chapters.Select(v => v.ReleaseDate.Year).Where(y => y >= 1000).DefaultIfEmpty().Min(); + } } diff --git a/API/Services/DirectoryService.cs b/API/Services/DirectoryService.cs index 039e0ef8d..ef99aaacc 100644 --- a/API/Services/DirectoryService.cs +++ b/API/Services/DirectoryService.cs @@ -651,7 +651,7 @@ public class DirectoryService : IDirectoryService public DateTime GetLastWriteTime(string folderPath) { if (!FileSystem.Directory.Exists(folderPath)) throw new IOException($"{folderPath} does not exist"); - var fileEntries = Directory.GetFileSystemEntries(folderPath, "*.*", SearchOption.AllDirectories); + var fileEntries = FileSystem.Directory.GetFileSystemEntries(folderPath, "*.*", SearchOption.AllDirectories); if (fileEntries.Length == 0) return DateTime.MaxValue; return fileEntries.Max(path => FileSystem.File.GetLastWriteTime(path)); } diff --git a/API/Services/TaskScheduler.cs b/API/Services/TaskScheduler.cs index 8a7a62471..070bdb5e7 100644 --- a/API/Services/TaskScheduler.cs +++ b/API/Services/TaskScheduler.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; +using System.Threading; using System.Threading.Tasks; using API.Data; using API.Entities.Enums; @@ -18,6 +19,7 @@ public interface ITaskScheduler Task ScheduleTasks(); Task ScheduleStatsTasks(); void ScheduleUpdaterTasks(); + void ScanFolder(string folderPath, TimeSpan delay); void ScanFolder(string folderPath); void ScanLibrary(int libraryId, bool force = false); void CleanupChapters(int[] chapterIds); @@ -179,9 +181,32 @@ public class TaskScheduler : ITaskScheduler RecurringJob.AddOrUpdate("check-updates", () => CheckForUpdate(), Cron.Daily(Rnd.Next(12, 18)), TimeZoneInfo.Local); } + public void ScanFolder(string folderPath, TimeSpan delay) + { + var normalizedFolder = Tasks.Scanner.Parser.Parser.NormalizePath(folderPath); + if (HasAlreadyEnqueuedTask(ScannerService.Name, "ScanFolder", new object[] { normalizedFolder })) + { + _logger.LogInformation("Skipped scheduling ScanFolder for {Folder} as a job already queued", + normalizedFolder); + return; + } + + _logger.LogInformation("Scheduling ScanFolder for {Folder}", normalizedFolder); + BackgroundJob.Schedule(() => _scannerService.ScanFolder(normalizedFolder), delay); + } + public void ScanFolder(string folderPath) { - _scannerService.ScanFolder(Tasks.Scanner.Parser.Parser.NormalizePath(folderPath)); + var normalizedFolder = Tasks.Scanner.Parser.Parser.NormalizePath(folderPath); + if (HasAlreadyEnqueuedTask(ScannerService.Name, "ScanFolder", new object[] {normalizedFolder})) + { + _logger.LogInformation("Skipped scheduling ScanFolder for {Folder} as a job already queued", + normalizedFolder); + return; + } + + _logger.LogInformation("Scheduling ScanFolder for {Folder}", normalizedFolder); + _scannerService.ScanFolder(normalizedFolder); } #endregion @@ -298,15 +323,32 @@ public class TaskScheduler : ITaskScheduler await _versionUpdaterService.PushUpdate(update); } + /// + /// If there is an enqueued or scheduled tak for method + /// + /// + /// public static bool HasScanTaskRunningForLibrary(int libraryId) { return - HasAlreadyEnqueuedTask("ScannerService", "ScanLibrary", new object[] {libraryId, true}, ScanQueue) || - HasAlreadyEnqueuedTask("ScannerService", "ScanLibrary", new object[] {libraryId, false}, ScanQueue); + HasAlreadyEnqueuedTask(ScannerService.Name, "ScanLibrary", new object[] {libraryId, true}, ScanQueue) || + HasAlreadyEnqueuedTask(ScannerService.Name, "ScanLibrary", new object[] {libraryId, false}, ScanQueue); } /// - /// Checks if this same invocation is already enqueued + /// If there is an enqueued or scheduled tak for method + /// + /// + /// + public static bool HasScanTaskRunningForSeries(int seriesId) + { + return + HasAlreadyEnqueuedTask(ScannerService.Name, "ScanSeries", new object[] {seriesId, true}, ScanQueue) || + HasAlreadyEnqueuedTask(ScannerService.Name, "ScanSeries", new object[] {seriesId, false}, ScanQueue); + } + + /// + /// Checks if this same invocation is already enqueued or scheduled /// /// Method name that was enqueued /// Class name the method resides on @@ -316,16 +358,33 @@ public class TaskScheduler : ITaskScheduler public static bool HasAlreadyEnqueuedTask(string className, string methodName, object[] args, string queue = DefaultQueue) { var enqueuedJobs = JobStorage.Current.GetMonitoringApi().EnqueuedJobs(queue, 0, int.MaxValue); - return enqueuedJobs.Any(j => j.Value.InEnqueuedState && + var ret = enqueuedJobs.Any(j => j.Value.InEnqueuedState && j.Value.Job.Method.DeclaringType != null && j.Value.Job.Args.SequenceEqual(args) && j.Value.Job.Method.Name.Equals(methodName) && j.Value.Job.Method.DeclaringType.Name.Equals(className)); + if (ret) return true; + + var scheduledJobs = JobStorage.Current.GetMonitoringApi().ScheduledJobs(0, int.MaxValue); + return scheduledJobs.Any(j => + j.Value.Job.Method.DeclaringType != null && j.Value.Job.Args.SequenceEqual(args) && + j.Value.Job.Method.Name.Equals(methodName) && + j.Value.Job.Method.DeclaringType.Name.Equals(className)); } + /// + /// Checks against any jobs that are running or about to run + /// + /// + /// + /// public static bool RunningAnyTasksByMethod(IEnumerable classNames, string queue = DefaultQueue) { var enqueuedJobs = JobStorage.Current.GetMonitoringApi().EnqueuedJobs(queue, 0, int.MaxValue); - return enqueuedJobs.Any(j => !j.Value.InEnqueuedState && + var ret = enqueuedJobs.Any(j => !j.Value.InEnqueuedState && classNames.Contains(j.Value.Job.Method.DeclaringType?.Name)); + if (ret) return true; + + var runningJobs = JobStorage.Current.GetMonitoringApi().ProcessingJobs(0, int.MaxValue); + return runningJobs.Any(j => classNames.Contains(j.Value.Job.Method.DeclaringType?.Name)); } } diff --git a/API/Services/Tasks/Scanner/LibraryWatcher.cs b/API/Services/Tasks/Scanner/LibraryWatcher.cs index 02c6fa8f1..1143b0622 100644 --- a/API/Services/Tasks/Scanner/LibraryWatcher.cs +++ b/API/Services/Tasks/Scanner/LibraryWatcher.cs @@ -38,7 +38,7 @@ public class LibraryWatcher : ILibraryWatcher private readonly IDirectoryService _directoryService; private readonly IUnitOfWork _unitOfWork; private readonly ILogger _logger; - private readonly IScannerService _scannerService; + private readonly ITaskScheduler _taskScheduler; private static readonly Dictionary> WatcherDictionary = new (); /// @@ -54,18 +54,19 @@ public class LibraryWatcher : ILibraryWatcher /// /// Counts within a time frame how many times the buffer became full. Is used to reschedule LibraryWatcher to start monitoring much later rather than instantly /// - private int _bufferFullCounter = 0; + private int _bufferFullCounter; + /// + /// Used to lock buffer Full Counter + /// + private static readonly object Lock = new (); - private DateTime _lastBufferOverflow = DateTime.MinValue; - - - - public LibraryWatcher(IDirectoryService directoryService, IUnitOfWork unitOfWork, ILogger logger, IScannerService scannerService, IHostEnvironment environment) + public LibraryWatcher(IDirectoryService directoryService, IUnitOfWork unitOfWork, + ILogger logger, IHostEnvironment environment, ITaskScheduler taskScheduler) { _directoryService = directoryService; _unitOfWork = unitOfWork; _logger = logger; - _scannerService = scannerService; + _taskScheduler = taskScheduler; _queueWaitTime = environment.IsDevelopment() ? TimeSpan.FromSeconds(30) : TimeSpan.FromMinutes(5); @@ -91,8 +92,8 @@ public class LibraryWatcher : ILibraryWatcher watcher.Created += OnCreated; watcher.Deleted += OnDeleted; watcher.Error += OnError; - watcher.Disposed += (sender, args) => - _logger.LogError("[LibraryWatcher] watcher was disposed when it shouldn't have been"); + watcher.Disposed += (_, _) => + _logger.LogError("[LibraryWatcher] watcher was disposed when it shouldn't have been. Please report this to Kavita dev"); watcher.Filter = "*.*"; watcher.IncludeSubdirectories = true; @@ -127,16 +128,14 @@ public class LibraryWatcher : ILibraryWatcher { _logger.LogDebug("[LibraryWatcher] Restarting watcher"); - UpdateBufferOverflow(); - StopWatching(); await StartWatching(); } private void OnChanged(object sender, FileSystemEventArgs e) { + _logger.LogDebug("[LibraryWatcher] Changed: {FullPath}, {Name}, {ChangeType}", e.FullPath, e.Name, e.ChangeType); if (e.ChangeType != WatcherChangeTypes.Changed) return; - _logger.LogDebug("[LibraryWatcher] Changed: {FullPath}, {Name}", e.FullPath, e.Name); BackgroundJob.Enqueue(() => ProcessChange(e.FullPath, string.IsNullOrEmpty(_directoryService.FileSystem.Path.GetExtension(e.Name)))); } @@ -158,20 +157,31 @@ public class LibraryWatcher : ILibraryWatcher BackgroundJob.Enqueue(() => ProcessChange(e.FullPath, true)); } - + /// + /// On error, we count the number of errors that have occured. If the number of errors has been more than 2 in last 10 minutes, then we suspend listening for an hour + /// + /// This will schedule jobs to decrement the buffer full counter + /// + /// private void OnError(object sender, ErrorEventArgs e) { _logger.LogError(e.GetException(), "[LibraryWatcher] An error occured, likely too many changes occured at once or the folder being watched was deleted. Restarting Watchers"); - _bufferFullCounter += 1; - _lastBufferOverflow = DateTime.Now; + bool condition; + lock (Lock) + { + _bufferFullCounter += 1; + condition = _bufferFullCounter >= 3; + } - if (_bufferFullCounter >= 3) + if (condition) { _logger.LogInformation("[LibraryWatcher] Internal buffer has been overflown multiple times in past 10 minutes. Suspending file watching for an hour"); + StopWatching(); BackgroundJob.Schedule(() => RestartWatching(), TimeSpan.FromHours(1)); return; } Task.Run(RestartWatching); + BackgroundJob.Schedule(() => UpdateLastBufferOverflow(), TimeSpan.FromMinutes(10)); } @@ -185,8 +195,6 @@ public class LibraryWatcher : ILibraryWatcher // ReSharper disable once MemberCanBePrivate.Global public async Task ProcessChange(string filePath, bool isDirectoryChange = false) { - UpdateBufferOverflow(); - var sw = Stopwatch.StartNew(); _logger.LogDebug("[LibraryWatcher] Processing change of {FilePath}", filePath); try @@ -214,29 +222,16 @@ public class LibraryWatcher : ILibraryWatcher return; } - // Check if this task has already enqueued or is being processed, before enqueing - - var alreadyScheduled = - TaskScheduler.HasAlreadyEnqueuedTask(ScannerService.Name, "ScanFolder", new object[] {fullPath}); - if (!alreadyScheduled) - { - _logger.LogInformation("[LibraryWatcher] Scheduling ScanFolder for {Folder}", fullPath); - BackgroundJob.Schedule(() => _scannerService.ScanFolder(fullPath), _queueWaitTime); - } - else - { - _logger.LogInformation("[LibraryWatcher] Skipped scheduling ScanFolder for {Folder} as a job already queued", - fullPath); - } + _taskScheduler.ScanFolder(fullPath, _queueWaitTime); } catch (Exception ex) { _logger.LogError(ex, "[LibraryWatcher] An error occured when processing a watch event"); } - _logger.LogDebug("[LibraryWatcher] ProcessChange ran in {ElapsedMilliseconds}ms", sw.ElapsedMilliseconds); + _logger.LogDebug("[LibraryWatcher] ProcessChange completed in {ElapsedMilliseconds}ms", sw.ElapsedMilliseconds); } - private string GetFolder(string filePath, IList libraryFolders) + private string GetFolder(string filePath, IEnumerable libraryFolders) { var parentDirectory = _directoryService.GetParentDirectoryName(filePath); _logger.LogDebug("[LibraryWatcher] Parent Directory: {ParentDirectory}", parentDirectory); @@ -256,14 +251,17 @@ public class LibraryWatcher : ILibraryWatcher return Parser.Parser.NormalizePath(_directoryService.FileSystem.Path.Join(libraryFolder, rootFolder.First())); } - private void UpdateBufferOverflow() + + /// + /// This is called via Hangfire to decrement the counter. Must work around a lock + /// + // ReSharper disable once MemberCanBePrivate.Global + public void UpdateLastBufferOverflow() { - if (_bufferFullCounter == 0) return; - // If the last buffer overflow is over 5 mins back, we can remove a buffer count - if (_lastBufferOverflow < DateTime.Now.Subtract(TimeSpan.FromMinutes(5))) + lock (Lock) { - _bufferFullCounter = Math.Min(0, _bufferFullCounter - 1); - _lastBufferOverflow = DateTime.Now; + if (_bufferFullCounter == 0) return; + _bufferFullCounter -= 1; } } } diff --git a/API/Services/Tasks/Scanner/ParseScannedFiles.cs b/API/Services/Tasks/Scanner/ParseScannedFiles.cs index 64eae4973..45e598957 100644 --- a/API/Services/Tasks/Scanner/ParseScannedFiles.cs +++ b/API/Services/Tasks/Scanner/ParseScannedFiles.cs @@ -1,12 +1,14 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Threading.Tasks; using API.Entities.Enums; using API.Extensions; using API.Parser; using API.SignalR; +using Kavita.Common.Helpers; using Microsoft.Extensions.Logging; namespace API.Services.Tasks.Scanner; @@ -39,6 +41,7 @@ public class SeriesModified public string SeriesName { get; set; } public DateTime LastScanned { get; set; } public MangaFormat Format { get; set; } + public IEnumerable LibraryRoots { get; set; } } @@ -109,7 +112,41 @@ public class ParseScannedFiles await folderAction(new List(), folderPath); return; } - await folderAction(_directoryService.ScanFiles(folderPath), folderPath); + // We need to calculate all folders till library root and see if any kavitaignores + var seriesMatcher = new GlobMatcher(); + try + { + var roots = seriesPaths[folderPath][0].LibraryRoots.Select(Scanner.Parser.Parser.NormalizePath).ToList(); + var libraryFolder = roots.SingleOrDefault(folderPath.Contains); + + if (string.IsNullOrEmpty(libraryFolder) || !Directory.Exists(folderPath)) + { + await folderAction(_directoryService.ScanFiles(folderPath, seriesMatcher), folderPath); + return; + } + + var allParents = _directoryService.GetFoldersTillRoot(libraryFolder, folderPath); + var path = libraryFolder; + + // Apply the library root level kavitaignore + var potentialIgnoreFile = _directoryService.FileSystem.Path.Join(path, DirectoryService.KavitaIgnoreFile); + seriesMatcher.Merge(_directoryService.CreateMatcherFromFile(potentialIgnoreFile)); + + // Then apply kavitaignores for each folder down to where the series folder is + foreach (var folderPart in allParents.Reverse()) + { + path = Parser.Parser.NormalizePath(Path.Join(libraryFolder, folderPart)); + potentialIgnoreFile = _directoryService.FileSystem.Path.Join(path, DirectoryService.KavitaIgnoreFile); + seriesMatcher.Merge(_directoryService.CreateMatcherFromFile(potentialIgnoreFile)); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "There was an error trying to find and apply .kavitaignores above the Series Folder. Scanning without them present"); + } + + + await folderAction(_directoryService.ScanFiles(folderPath, seriesMatcher), folderPath); } diff --git a/API/Services/Tasks/Scanner/Parser/DefaultParser.cs b/API/Services/Tasks/Scanner/Parser/DefaultParser.cs index f9889d74f..4bab428a3 100644 --- a/API/Services/Tasks/Scanner/Parser/DefaultParser.cs +++ b/API/Services/Tasks/Scanner/Parser/DefaultParser.cs @@ -62,7 +62,7 @@ public class DefaultParser : IDefaultParser }; } - if (Parser.IsCoverImage(filePath)) return null; + if (Parser.IsCoverImage(_directoryService.FileSystem.Path.GetFileName(filePath))) return null; if (Parser.IsImage(filePath)) { diff --git a/API/Services/Tasks/Scanner/ProcessSeries.cs b/API/Services/Tasks/Scanner/ProcessSeries.cs index 3611949bd..c61b72bdb 100644 --- a/API/Services/Tasks/Scanner/ProcessSeries.cs +++ b/API/Services/Tasks/Scanner/ProcessSeries.cs @@ -238,13 +238,7 @@ public class ProcessSeries : IProcessSeries // Update Metadata based on Chapter metadata if (!series.Metadata.ReleaseYearLocked) { - series.Metadata.ReleaseYear = chapters.Select(v => v.ReleaseDate.Year).Where(y => y >= 1000).DefaultIfEmpty().Min(); - - if (series.Metadata.ReleaseYear < 1000) - { - // Not a valid year, default to 0 - series.Metadata.ReleaseYear = 0; - } + series.Metadata.ReleaseYear = chapters.MinimumReleaseYear(); } // Set the AgeRating as highest in all the comicInfos @@ -637,14 +631,7 @@ public class ProcessSeries : IProcessSeries } // This needs to check against both Number and Volume to calculate Count - if (!string.IsNullOrEmpty(comicInfo.Number) && float.Parse(comicInfo.Number) > 0) - { - chapter.Count = (int) Math.Floor(float.Parse(comicInfo.Number)); - } - if (!string.IsNullOrEmpty(comicInfo.Volume) && float.Parse(comicInfo.Volume) > 0) - { - chapter.Count = Math.Max(chapter.Count, (int) Math.Floor(float.Parse(comicInfo.Volume))); - } + chapter.Count = comicInfo.CalculatedCount(); void AddPerson(Person person) { @@ -755,7 +742,6 @@ public class ProcessSeries : IProcessSeries /// private void UpdatePeople(IEnumerable names, PersonRole role, Action action) { - var allPeopleTypeRole = _people.Where(p => p.Role == role).ToList(); foreach (var name in names) diff --git a/API/Services/Tasks/ScannerService.cs b/API/Services/Tasks/ScannerService.cs index 737a4ed91..bc28ff578 100644 --- a/API/Services/Tasks/ScannerService.cs +++ b/API/Services/Tasks/ScannerService.cs @@ -8,6 +8,7 @@ using System.Threading.Tasks; using API.Data; using API.Data.Repositories; using API.Entities; +using API.Entities.Enums; using API.Extensions; using API.Helpers; using API.Parser; @@ -97,24 +98,39 @@ public class ScannerService : IScannerService _wordCountAnalyzerService = wordCountAnalyzerService; } + /// + /// Given a generic folder path, will invoke a Series scan or Library scan. + /// + /// This will Schedule the job to run 1 minute in the future to allow for any close-by duplicate requests to be dropped + /// public async Task ScanFolder(string folder) { - var seriesId = await _unitOfWork.SeriesRepository.GetSeriesIdByFolder(folder); - if (seriesId > 0) + Series series = null; + try { - if (TaskScheduler.HasAlreadyEnqueuedTask(Name, "ScanSeries", - new object[] {seriesId, true})) + series = await _unitOfWork.SeriesRepository.GetSeriesByFolderPath(folder, SeriesIncludes.Library); + } + catch (InvalidOperationException ex) + { + if (ex.Message.Equals("Sequence contains more than one element.")) + { + _logger.LogCritical("[ScannerService] Multiple series map to this folder. Library scan will be used for ScanFolder"); + } + } + if (series != null && series.Library.Type != LibraryType.Book) + { + if (TaskScheduler.HasScanTaskRunningForSeries(series.Id)) { _logger.LogInformation("[ScannerService] Scan folder invoked for {Folder} but a task is already queued for this series. Dropping request", folder); return; } - BackgroundJob.Enqueue(() => ScanSeries(seriesId, true)); + BackgroundJob.Schedule(() => ScanSeries(series.Id, true), TimeSpan.FromMinutes(1)); return; } // This is basically rework of what's already done in Library Watcher but is needed if invoked via API var parentDirectory = _directoryService.GetParentDirectoryName(folder); - if (string.IsNullOrEmpty(parentDirectory)) return; // This should never happen as it's calculated before enqueing + if (string.IsNullOrEmpty(parentDirectory)) return; var libraries = (await _unitOfWork.LibraryRepository.GetLibraryDtosAsync()).ToList(); var libraryFolders = libraries.SelectMany(l => l.Folders); @@ -125,18 +141,17 @@ public class ScannerService : IScannerService var library = libraries.FirstOrDefault(l => l.Folders.Select(Scanner.Parser.Parser.NormalizePath).Contains(libraryFolder)); if (library != null) { - if (TaskScheduler.HasAlreadyEnqueuedTask(Name, "ScanLibrary", - new object[] {library.Id, false})) + if (TaskScheduler.HasScanTaskRunningForLibrary(library.Id)) { _logger.LogInformation("[ScannerService] Scan folder invoked for {Folder} but a task is already queued for this library. Dropping request", folder); return; } - BackgroundJob.Enqueue(() => ScanLibrary(library.Id, false)); + BackgroundJob.Schedule(() => ScanLibrary(library.Id, false), TimeSpan.FromMinutes(1)); } } /// - /// + /// Scans just an existing Series for changes. If the series doesn't exist, will delete it. /// /// /// Not Used. Scan series will always force @@ -186,6 +201,7 @@ public class ScannerService : IScannerService return; } + // If the series path doesn't exist anymore, it was either moved or renamed. We need to essentially delete it var parsedSeries = new Dictionary>(); await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Started, series.Name)); @@ -213,11 +229,13 @@ public class ScannerService : IScannerService } _logger.LogInformation("Beginning file scan on {SeriesName}", series.Name); - var scanElapsedTime = await ScanFiles(library, new []{folderPath}, false, TrackFiles, true); + var scanElapsedTime = await ScanFiles(library, new []{ folderPath }, false, TrackFiles, true); _logger.LogInformation("ScanFiles for {Series} took {Time}", series.Name, scanElapsedTime); await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Ended, series.Name)); + + // Remove any parsedSeries keys that don't belong to our series. This can occur when users store 2 series in the same folder RemoveParsedInfosNotForSeries(parsedSeries, series); diff --git a/Kavita.sln.DotSettings b/Kavita.sln.DotSettings index fb5e739fb..55f8e0090 100644 --- a/Kavita.sln.DotSettings +++ b/Kavita.sln.DotSettings @@ -2,6 +2,8 @@ ExplicitlyExcluded True True + True + True True True True \ No newline at end of file diff --git a/UI/Web/package-lock.json b/UI/Web/package-lock.json index 87bfaaf59..1e6b9b82b 100644 --- a/UI/Web/package-lock.json +++ b/UI/Web/package-lock.json @@ -12666,14 +12666,6 @@ "tslib": "^2.3.0" } }, - "ngx-infinite-scroll": { - "version": "13.0.2", - "resolved": "https://registry.npmjs.org/ngx-infinite-scroll/-/ngx-infinite-scroll-13.0.2.tgz", - "integrity": "sha512-RSezL0DUxo1B57SyRMOSt3a/5lLXJs6P8lavtxOh10uhX+hn662cMYHUO7LiU2a/vJxef2R020s4jkUqhnXTcg==", - "requires": { - "tslib": "^2.3.0" - } - }, "ngx-toastr": { "version": "14.2.1", "resolved": "https://registry.npmjs.org/ngx-toastr/-/ngx-toastr-14.2.1.tgz", diff --git a/UI/Web/package.json b/UI/Web/package.json index 12405c3bc..c78cdc29e 100644 --- a/UI/Web/package.json +++ b/UI/Web/package.json @@ -41,7 +41,6 @@ "ngx-color-picker": "^12.0.0", "ngx-extended-pdf-viewer": "^15.0.0", "ngx-file-drop": "^14.0.1", - "ngx-infinite-scroll": "^13.0.2", "ngx-toastr": "^14.2.1", "requires": "^1.0.2", "rxjs": "~7.5.4", diff --git a/action-build.sh.OLD b/action-build.sh.OLD deleted file mode 100644 index b4c82b6d0..000000000 --- a/action-build.sh.OLD +++ /dev/null @@ -1,103 +0,0 @@ -#! /bin/bash -set -e - -outputFolder='_output' - -ProgressStart() -{ - echo "Start '$1'" -} - -ProgressEnd() -{ - echo "Finish '$1'" -} - -Build() -{ - local RID="$1" - - ProgressStart "Build for $RID" - - slnFile=Kavita.sln - - dotnet clean $slnFile -c Release - - dotnet msbuild -restore $slnFile -p:Configuration=Release -p:Platform="Any CPU" -p:RuntimeIdentifiers=$RID - - ProgressEnd "Build for $RID" -} - -Package() -{ - local framework="$1" - local runtime="$2" - local lOutputFolder=../_output/"$runtime"/Kavita - - ProgressStart "Creating $runtime Package for $framework" - - # TODO: Use no-restore? Because Build should have already done it for us - echo "Building" - cd API - echo dotnet publish -c Release --no-restore --self-contained --runtime $runtime -o "$lOutputFolder" --framework $framework - dotnet publish -c Release --no-restore --self-contained --runtime $runtime -o "$lOutputFolder" --framework $framework - - echo "Renaming API -> Kavita" - mv "$lOutputFolder"/API "$lOutputFolder"/Kavita - - echo "Copying webui wwwroot to build" - cp -r wwwroot/* "$lOutputFolder"/wwwroot/ - - echo "Copying Install information" - cp ../INSTALL.txt "$lOutputFolder"/README.txt - - echo "Copying LICENSE" - cp ../LICENSE "$lOutputFolder"/LICENSE.txt - - echo "Creating tar" - cd ../$outputFolder/"$runtime"/ - tar -czvf ../kavita-$runtime.tar.gz Kavita - - ProgressEnd "Creating $runtime Package for $framework" - -} - -BuildUI() -{ - ProgressStart 'Building UI' - echo 'Removing old wwwroot' - rm -rf API/wwwroot/* - cd ../Kavita-webui/ || exit - echo 'Installing web dependencies' - npm install - echo 'Building UI' - npm run prod - ls -l dist - echo 'Copying back to Kavita wwwroot' - cp -r dist/* ../Kavita/API/wwwroot - ls -l ../Kavita/API/wwwroot - cd ../Kavita/ || exit - ProgressEnd 'Building UI' -} - -dir=$PWD - -if [ -d _output ] -then - rm -r _output/ -fi - -#Build for x64 -Build "linux-x64" -Package "net5.0" "linux-x64" -cd "$dir" - -#Build for arm -Build "linux-arm" -Package "net5.0" "linux-arm" -cd "$dir" - -#Build for arm64 -Build "linux-arm64" -Package "net5.0" "linux-arm64" -cd "$dir" \ No newline at end of file