From b07aaf1eb547a65c2edde2a9b77fc829d65e6ea9 Mon Sep 17 00:00:00 2001 From: Joseph Milazzo Date: Sun, 28 Aug 2022 15:20:46 -0500 Subject: [PATCH] Simplify Folder Watcher (#1484) * Refactored Library Watcher to use Hangfire under the hood. * Support .kavitaignore at root level. * Refactored a lot of the library watching code to process faster and handle when FileSystemWatcher runs out of internal buffer space. It's still not perfect, but good enough for basic use. * Make folder watching as experimental and default it to off by default. * Revert #1479 * Tweaked the messaging for OPDS to remove a note about download role. Moved some code closer to where it's used. * Cleaned up how the events widget reports * Fixed a null issue when deleting series in the UI * Cleaned up some debug code * Added more information for when we skip a scan * Cleaned up some logging messages in CoverGen tasks * More log message tweaks * Added some debug to help identify a rare issue * Fixed a bug where save bookmarks as webp could get reset to false when saving other server settings * Updated some documentation on library watcher. * Make LibraryWatcher fire every 5 mins --- API/Data/Seed.cs | 2 +- API/Services/DirectoryService.cs | 41 ++-- API/Services/MetadataService.cs | 22 +- API/Services/TaskScheduler.cs | 6 +- API/Services/Tasks/Scanner/LibraryWatcher.cs | 195 +++++++++--------- .../Tasks/Scanner/ParseScannedFiles.cs | 8 +- .../Tasks/Scanner}/Parser/DefaultParser.cs | 0 .../Tasks/Scanner}/Parser/Parser.cs | 2 - .../Tasks/Scanner}/Parser/ParserInfo.cs | 0 API/Services/Tasks/Scanner/ScanLibrary.cs | 111 ---------- API/Services/Tasks/ScannerService.cs | 14 +- API/SignalR/MessageFactory.cs | 19 +- Dockerfile | 4 - .../manage-settings.component.html | 4 +- .../manage-settings.component.ts | 7 + .../manage-users/manage-users.component.ts | 2 +- .../events-widget/events-widget.component.ts | 1 - .../app/shared/_services/utility.service.ts | 6 + .../shared/tag-badge/tag-badge.component.ts | 2 +- 19 files changed, 187 insertions(+), 259 deletions(-) rename API/{ => Services/Tasks/Scanner}/Parser/DefaultParser.cs (100%) rename API/{ => Services/Tasks/Scanner}/Parser/Parser.cs (99%) rename API/{ => Services/Tasks/Scanner}/Parser/ParserInfo.cs (100%) delete mode 100644 API/Services/Tasks/Scanner/ScanLibrary.cs diff --git a/API/Data/Seed.cs b/API/Data/Seed.cs index 88acf41cd..fb1256362 100644 --- a/API/Data/Seed.cs +++ b/API/Data/Seed.cs @@ -103,7 +103,7 @@ namespace API.Data new() {Key = ServerSettingKey.ConvertBookmarkToWebP, Value = "false"}, new() {Key = ServerSettingKey.EnableSwaggerUi, Value = "false"}, new() {Key = ServerSettingKey.TotalBackups, Value = "30"}, - new() {Key = ServerSettingKey.EnableFolderWatching, Value = "true"}, + new() {Key = ServerSettingKey.EnableFolderWatching, Value = "false"}, }.ToArray()); foreach (var defaultSetting in DefaultSettings) diff --git a/API/Services/DirectoryService.cs b/API/Services/DirectoryService.cs index aefcbee56..145106a33 100644 --- a/API/Services/DirectoryService.cs +++ b/API/Services/DirectoryService.cs @@ -10,6 +10,7 @@ using API.DTOs.System; using API.Entities.Enums; using API.Extensions; using Kavita.Common.Helpers; +using Microsoft.Extensions.FileSystemGlobbing; using Microsoft.Extensions.Logging; namespace API.Services @@ -64,14 +65,17 @@ namespace API.Services SearchOption searchOption = SearchOption.TopDirectoryOnly); IEnumerable GetDirectories(string folderPath); + IEnumerable GetDirectories(string folderPath, GlobMatcher matcher); string GetParentDirectoryName(string fileOrFolder); #nullable enable IList ScanFiles(string folderPath, GlobMatcher? matcher = null); DateTime GetLastWriteTime(string folderPath); + GlobMatcher CreateMatcherFromFile(string filePath); #nullable disable } public class DirectoryService : IDirectoryService { + public const string KavitaIgnoreFile = ".kavitaignore"; public IFileSystem FileSystem { get; } public string CacheDirectory { get; } public string CoverImageDirectory { get; } @@ -531,6 +535,21 @@ namespace API.Services .Where(path => ExcludeDirectories.Matches(path).Count == 0); } + /// + /// Gets a set of directories from the folder path. Automatically excludes directories that shouldn't be in scope. + /// + /// + /// A set of glob rules that will filter directories out + /// List of directory paths, empty if path doesn't exist + public IEnumerable GetDirectories(string folderPath, GlobMatcher matcher) + { + if (matcher == null) return GetDirectories(folderPath); + + return GetDirectories(folderPath) + .Where(folder => !matcher.ExcludeMatches( + $"{FileSystem.DirectoryInfo.FromDirectoryName(folder).Name}{FileSystem.Path.AltDirectorySeparatorChar}")); + } + /// /// Returns all directories, including subdirectories. Automatically excludes directories that shouldn't be in scope. /// @@ -580,7 +599,7 @@ namespace API.Services var files = new List(); if (!Exists(folderPath)) return files; - var potentialIgnoreFile = FileSystem.Path.Join(folderPath, ".kavitaignore"); + var potentialIgnoreFile = FileSystem.Path.Join(folderPath, KavitaIgnoreFile); if (matcher == null) { matcher = CreateMatcherFromFile(potentialIgnoreFile); @@ -591,17 +610,7 @@ namespace API.Services } - IEnumerable directories; - if (matcher == null) - { - directories = GetDirectories(folderPath); - } - else - { - directories = GetDirectories(folderPath) - .Where(folder => matcher != null && - !matcher.ExcludeMatches($"{FileSystem.DirectoryInfo.FromDirectoryName(folder).Name}{FileSystem.Path.AltDirectorySeparatorChar}")); - } + var directories = GetDirectories(folderPath, matcher); foreach (var directory in directories) { @@ -640,8 +649,12 @@ namespace API.Services return directories.Max(d => FileSystem.Directory.GetLastWriteTime(d)); } - - private GlobMatcher CreateMatcherFromFile(string filePath) + /// + /// Generates a GlobMatcher from a .kavitaignore file found at path. Returns null otherwise. + /// + /// + /// + public GlobMatcher CreateMatcherFromFile(string filePath) { if (!FileSystem.File.Exists(filePath)) { diff --git a/API/Services/MetadataService.cs b/API/Services/MetadataService.cs index bde0d89a6..361d77737 100644 --- a/API/Services/MetadataService.cs +++ b/API/Services/MetadataService.cs @@ -80,8 +80,8 @@ public class MetadataService : IMetadataService _logger.LogDebug("[MetadataService] Generating cover image for {File}", firstFile.FilePath); chapter.CoverImage = _readingItemService.GetCoverImage(firstFile.FilePath, ImageService.GetChapterFormat(chapter.Id, chapter.VolumeId), firstFile.Format); - _unitOfWork.ChapterRepository.Update(chapter); // BUG: CoverImage isn't saving for Monter Masume with new scan loop - _updateEvents.Add(MessageFactory.CoverUpdateEvent(chapter.Id, MessageFactoryEntityTypes.Chapter)); // TODO: IDEA: Instead of firing here where it's not yet saved, maybe collect the ids and fire after save + _unitOfWork.ChapterRepository.Update(chapter); + _updateEvents.Add(MessageFactory.CoverUpdateEvent(chapter.Id, MessageFactoryEntityTypes.Chapter)); return Task.FromResult(true); } @@ -111,7 +111,6 @@ public class MetadataService : IMetadataService if (firstChapter == null) return Task.FromResult(false); volume.CoverImage = firstChapter.CoverImage; - //await _eventHub.SendMessageAsync(MessageFactory.CoverUpdate, MessageFactory.CoverUpdateEvent(volume.Id, MessageFactoryEntityTypes.Volume), false); _updateEvents.Add(MessageFactory.CoverUpdateEvent(volume.Id, MessageFactoryEntityTypes.Volume)); return Task.FromResult(true); @@ -148,7 +147,6 @@ public class MetadataService : IMetadataService } } series.CoverImage = firstCover?.CoverImage ?? coverImage; - //await _eventHub.SendMessageAsync(MessageFactory.CoverUpdate, MessageFactory.CoverUpdateEvent(series.Id, MessageFactoryEntityTypes.Series), false); _updateEvents.Add(MessageFactory.CoverUpdateEvent(series.Id, MessageFactoryEntityTypes.Series)); return Task.CompletedTask; } @@ -161,7 +159,7 @@ public class MetadataService : IMetadataService /// private async Task ProcessSeriesCoverGen(Series series, bool forceUpdate) { - _logger.LogDebug("[MetadataService] Generating cover images for series: {SeriesName}", series.OriginalName); + _logger.LogDebug("[MetadataService] Processing cover image generation for series: {SeriesName}", series.OriginalName); try { var volumeIndex = 0; @@ -195,7 +193,7 @@ public class MetadataService : IMetadataService } catch (Exception ex) { - _logger.LogError(ex, "[MetadataService] There was an exception during updating metadata for {SeriesName} ", series.Name); + _logger.LogError(ex, "[MetadataService] There was an exception during cover generation for {SeriesName} ", series.Name); } } @@ -211,14 +209,14 @@ public class MetadataService : IMetadataService public async Task GenerateCoversForLibrary(int libraryId, bool forceUpdate = false) { var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(libraryId, LibraryIncludes.None); - _logger.LogInformation("[MetadataService] Beginning metadata refresh of {LibraryName}", library.Name); + _logger.LogInformation("[MetadataService] Beginning cover generation refresh of {LibraryName}", library.Name); _updateEvents.Clear(); var chunkInfo = await _unitOfWork.SeriesRepository.GetChunkInfo(library.Id); var stopwatch = Stopwatch.StartNew(); var totalTime = 0L; - _logger.LogInformation("[MetadataService] Refreshing Library {LibraryName}. Total Items: {TotalSize}. Total Chunks: {TotalChunks} with {ChunkSize} size", library.Name, chunkInfo.TotalSize, chunkInfo.TotalChunks, chunkInfo.ChunkSize); + _logger.LogInformation("[MetadataService] Refreshing Library {LibraryName} for cover generation. Total Items: {TotalSize}. Total Chunks: {TotalChunks} with {ChunkSize} size", library.Name, chunkInfo.TotalSize, chunkInfo.TotalChunks, chunkInfo.ChunkSize); await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.CoverUpdateProgressEvent(library.Id, 0F, ProgressEventType.Started, $"Starting {library.Name}")); @@ -229,7 +227,7 @@ public class MetadataService : IMetadataService totalTime += stopwatch.ElapsedMilliseconds; stopwatch.Restart(); - _logger.LogInformation("[MetadataService] Processing chunk {ChunkNumber} / {TotalChunks} with size {ChunkSize}. Series ({SeriesStart} - {SeriesEnd}", + _logger.LogDebug("[MetadataService] Processing chunk {ChunkNumber} / {TotalChunks} with size {ChunkSize}. Series ({SeriesStart} - {SeriesEnd})", chunk, chunkInfo.TotalChunks, chunkInfo.ChunkSize, chunk * chunkInfo.ChunkSize, (chunk + 1) * chunkInfo.ChunkSize); var nonLibrarySeries = await _unitOfWork.SeriesRepository.GetFullSeriesForLibraryIdAsync(library.Id, @@ -255,7 +253,7 @@ public class MetadataService : IMetadataService } catch (Exception ex) { - _logger.LogError(ex, "[MetadataService] There was an exception during metadata refresh for {SeriesName}", series.Name); + _logger.LogError(ex, "[MetadataService] There was an exception during cover generation refresh for {SeriesName}", series.Name); } seriesIndex++; } @@ -272,7 +270,7 @@ public class MetadataService : IMetadataService await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.CoverUpdateProgressEvent(library.Id, 1F, ProgressEventType.Ended, $"Complete")); - _logger.LogInformation("[MetadataService] Updated metadata for {SeriesNumber} series in library {LibraryName} in {ElapsedMilliseconds} milliseconds total", chunkInfo.TotalSize, library.Name, totalTime); + _logger.LogInformation("[MetadataService] Updated covers for {SeriesNumber} series in library {LibraryName} in {ElapsedMilliseconds} milliseconds total", chunkInfo.TotalSize, library.Name, totalTime); } @@ -321,7 +319,7 @@ public class MetadataService : IMetadataService if (_unitOfWork.HasChanges()) { await _unitOfWork.CommitAsync(); - _logger.LogInformation("[MetadataService] Updated cover images for {SeriesName} in {ElapsedMilliseconds} milliseconds", series.Name, sw.ElapsedMilliseconds); + _logger.LogInformation("[MetadataService] Updated covers for {SeriesName} in {ElapsedMilliseconds} milliseconds", series.Name, sw.ElapsedMilliseconds); } await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, diff --git a/API/Services/TaskScheduler.cs b/API/Services/TaskScheduler.cs index a9f34245d..f61a77a9e 100644 --- a/API/Services/TaskScheduler.cs +++ b/API/Services/TaskScheduler.cs @@ -239,12 +239,12 @@ public class TaskScheduler : ITaskScheduler public void ScanSeries(int libraryId, int seriesId, bool forceUpdate = false) { - if (HasAlreadyEnqueuedTask("ScannerService", "ScanSeries", new object[] {seriesId, forceUpdate}, ScanQueue)) + if (HasAlreadyEnqueuedTask(ScannerService.Name, "ScanSeries", new object[] {seriesId, forceUpdate}, ScanQueue)) { _logger.LogInformation("A duplicate request to scan series occured. Skipping"); return; } - if (RunningAnyTasksByMethod(new List() {"ScannerService", "ScanLibrary", "ScanLibraries", "ScanFolder", "ScanSeries"}, ScanQueue)) + if (RunningAnyTasksByMethod(new List() {ScannerService.Name, "ScanLibrary", "ScanLibraries", "ScanFolder", "ScanSeries"}, ScanQueue)) { _logger.LogInformation("A Scan is already running, rescheduling ScanSeries in 10 minutes"); BackgroundJob.Schedule(() => ScanSeries(libraryId, seriesId, forceUpdate), TimeSpan.FromMinutes(10)); @@ -290,7 +290,7 @@ public class TaskScheduler : ITaskScheduler /// object[] of arguments in the order they are passed to enqueued job /// Queue to check against. Defaults to "default" /// - private static bool HasAlreadyEnqueuedTask(string className, string methodName, object[] args, string queue = DefaultQueue) + public static bool HasAlreadyEnqueuedTask(string className, string methodName, object[] args, string queue = DefaultQueue) { var enqueuedJobs = JobStorage.Current.GetMonitoringApi().EnqueuedJobs(queue, 0, int.MaxValue); return enqueuedJobs.Any(j => j.Value.InEnqueuedState && diff --git a/API/Services/Tasks/Scanner/LibraryWatcher.cs b/API/Services/Tasks/Scanner/LibraryWatcher.cs index 16e810710..f27f4119b 100644 --- a/API/Services/Tasks/Scanner/LibraryWatcher.cs +++ b/API/Services/Tasks/Scanner/LibraryWatcher.cs @@ -1,8 +1,8 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Linq; -using System.Text.RegularExpressions; using System.Threading.Tasks; using API.Data; using Hangfire; @@ -11,6 +11,52 @@ using Microsoft.Extensions.Logging; namespace API.Services.Tasks.Scanner; +/// +/// Change information +/// +public class Change +{ + /// + /// Gets or sets the type of the change. + /// + /// + /// The type of the change. + /// + public WatcherChangeTypes ChangeType { get; set; } + + /// + /// Gets or sets the full path. + /// + /// + /// The full path. + /// + public string FullPath { get; set; } + + /// + /// Gets or sets the name. + /// + /// + /// The name. + /// + public string Name { get; set; } + + /// + /// Gets or sets the old full path. + /// + /// + /// The old full path. + /// + public string OldFullPath { get; set; } + + /// + /// Gets or sets the old name. + /// + /// + /// The old name. + /// + public string OldName { get; set; } +} + public interface ILibraryWatcher { /// @@ -29,29 +75,6 @@ public interface ILibraryWatcher Task RestartWatching(); } -internal class FolderScanQueueable -{ - public DateTime QueueTime { get; set; } - public string FolderPath { get; set; } -} - -internal class FolderScanQueueableComparer : IEqualityComparer -{ - public bool Equals(FolderScanQueueable x, FolderScanQueueable y) - { - if (ReferenceEquals(x, y)) return true; - if (ReferenceEquals(x, null)) return false; - if (ReferenceEquals(y, null)) return false; - if (x.GetType() != y.GetType()) return false; - return x.FolderPath == y.FolderPath; - } - - public int GetHashCode(FolderScanQueueable obj) - { - return HashCode.Combine(obj.FolderPath); - } -} - /// /// Responsible for watching the file system and processing change events. This is mainly responsible for invoking /// Scanner to quickly pickup on changes. @@ -64,11 +87,13 @@ public class LibraryWatcher : ILibraryWatcher private readonly IScannerService _scannerService; private readonly Dictionary> _watcherDictionary = new (); + /// + /// This is just here to prevent GC from Disposing our watchers + /// + private readonly IList _fileWatchers = new List(); private IList _libraryFolders = new List(); - private readonly Queue _scanQueue = new Queue(); private readonly TimeSpan _queueWaitTime; - private readonly FolderScanQueueableComparer _folderScanQueueableComparer = new FolderScanQueueableComparer(); public LibraryWatcher(IDirectoryService directoryService, IUnitOfWork unitOfWork, ILogger logger, IScannerService scannerService, IHostEnvironment environment) @@ -78,7 +103,7 @@ public class LibraryWatcher : ILibraryWatcher _logger = logger; _scannerService = scannerService; - _queueWaitTime = environment.IsDevelopment() ? TimeSpan.FromSeconds(10) : TimeSpan.FromMinutes(1); + _queueWaitTime = environment.IsDevelopment() ? TimeSpan.FromSeconds(30) : TimeSpan.FromMinutes(5); } @@ -95,20 +120,16 @@ public class LibraryWatcher : ILibraryWatcher { _logger.LogDebug("Watching {FolderPath}", libraryFolder); var watcher = new FileSystemWatcher(libraryFolder); - watcher.NotifyFilter = NotifyFilters.CreationTime - | NotifyFilters.DirectoryName - | NotifyFilters.FileName - | NotifyFilters.LastWrite - | NotifyFilters.Size; watcher.Changed += OnChanged; watcher.Created += OnCreated; watcher.Deleted += OnDeleted; - watcher.Renamed += OnRenamed; + watcher.Error += OnError; watcher.Filter = "*.*"; watcher.IncludeSubdirectories = true; watcher.EnableRaisingEvents = true; + _fileWatchers.Add(watcher); if (!_watcherDictionary.ContainsKey(libraryFolder)) { _watcherDictionary.Add(libraryFolder, new List()); @@ -127,9 +148,9 @@ public class LibraryWatcher : ILibraryWatcher fileSystemWatcher.Changed -= OnChanged; fileSystemWatcher.Created -= OnCreated; fileSystemWatcher.Deleted -= OnDeleted; - fileSystemWatcher.Renamed -= OnRenamed; fileSystemWatcher.Dispose(); } + _fileWatchers.Clear(); _watcherDictionary.Clear(); } @@ -143,7 +164,7 @@ public class LibraryWatcher : ILibraryWatcher { if (e.ChangeType != WatcherChangeTypes.Changed) return; _logger.LogDebug("[LibraryWatcher] Changed: {FullPath}, {Name}", e.FullPath, e.Name); - ProcessChange(e.FullPath); + ProcessChange(e.FullPath, string.IsNullOrEmpty(_directoryService.FileSystem.Path.GetExtension(e.Name))); } private void OnCreated(object sender, FileSystemEventArgs e) @@ -152,87 +173,77 @@ public class LibraryWatcher : ILibraryWatcher ProcessChange(e.FullPath, !_directoryService.FileSystem.File.Exists(e.Name)); } + /// + /// From testing, on Deleted only needs to pass through the event when a folder is deleted. If a file is deleted, Changed will handle automatically. + /// + /// + /// private void OnDeleted(object sender, FileSystemEventArgs e) { + var isDirectory = string.IsNullOrEmpty(_directoryService.FileSystem.Path.GetExtension(e.Name)); + if (!isDirectory) return; _logger.LogDebug("[LibraryWatcher] Deleted: {FullPath}, {Name}", e.FullPath, e.Name); - - // On deletion, we need another type of check. We need to check if e.Name has an extension or not - // NOTE: File deletion will trigger a folder change event, so this might not be needed - ProcessChange(e.FullPath, string.IsNullOrEmpty(_directoryService.FileSystem.Path.GetExtension(e.Name))); + ProcessChange(e.FullPath, true); } - - private void OnRenamed(object sender, RenamedEventArgs e) + private void OnError(object sender, ErrorEventArgs e) { - _logger.LogDebug($"[LibraryWatcher] Renamed:"); - _logger.LogDebug(" Old: {OldFullPath}", e.OldFullPath); - _logger.LogDebug(" New: {FullPath}", e.FullPath); - ProcessChange(e.FullPath, _directoryService.FileSystem.Directory.Exists(e.FullPath)); + _logger.LogError(e.GetException(), "[LibraryWatcher] An error occured, likely too many watches occured at once. Restarting Watchers"); + Task.Run(RestartWatching); } + /// - /// Processes the file or folder change. + /// Processes the file or folder change. If the change is a file change and not from a supported extension, it will be ignored. /// + /// This will ignore image files that are added to the system. However, they may still trigger scans due to folder changes. /// File or folder that changed /// If the change is on a directory and not a file private void ProcessChange(string filePath, bool isDirectoryChange = false) { - // We need to check if directory or not - if (!isDirectoryChange && !new Regex(Parser.Parser.SupportedExtensions).IsMatch(new FileInfo(filePath).Extension)) return; - - var parentDirectory = _directoryService.GetParentDirectoryName(filePath); - if (string.IsNullOrEmpty(parentDirectory)) return; - - // We need to find the library this creation belongs to - // Multiple libraries can point to the same base folder. In this case, we need use FirstOrDefault - var libraryFolder = _libraryFolders.FirstOrDefault(f => parentDirectory.Contains(f)); - if (string.IsNullOrEmpty(libraryFolder)) return; - - var rootFolder = _directoryService.GetFoldersTillRoot(libraryFolder, filePath).ToList(); - if (!rootFolder.Any()) return; - - // Select the first folder and join with library folder, this should give us the folder to scan. - var fullPath = Parser.Parser.NormalizePath(_directoryService.FileSystem.Path.Join(libraryFolder, rootFolder.First())); - var queueItem = new FolderScanQueueable() + var sw = Stopwatch.StartNew(); + try { - FolderPath = fullPath, - QueueTime = DateTime.Now - }; - if (!_scanQueue.Contains(queueItem, _folderScanQueueableComparer)) - { - _logger.LogDebug("[LibraryWatcher] Queuing job for {Folder} at {TimeStamp}", fullPath, DateTime.Now); - _scanQueue.Enqueue(queueItem); - } + // We need to check if directory or not + if (!isDirectoryChange && + !(Parser.Parser.IsArchive(filePath) || Parser.Parser.IsBook(filePath))) return; - ProcessQueue(); - } + var parentDirectory = _directoryService.GetParentDirectoryName(filePath); + if (string.IsNullOrEmpty(parentDirectory)) return; - /// - /// Instead of making things complicated with a separate thread, this service will process the queue whenever a change occurs - /// - private void ProcessQueue() - { - var i = 0; - while (i < _scanQueue.Count) - { - var item = _scanQueue.Peek(); - if (item.QueueTime < DateTime.Now.Subtract(_queueWaitTime)) + // We need to find the library this creation belongs to + // Multiple libraries can point to the same base folder. In this case, we need use FirstOrDefault + var libraryFolder = _libraryFolders.FirstOrDefault(f => parentDirectory.Contains(f)); + if (string.IsNullOrEmpty(libraryFolder)) return; + + var rootFolder = _directoryService.GetFoldersTillRoot(libraryFolder, filePath).ToList(); + if (!rootFolder.Any()) return; + + // Select the first folder and join with library folder, this should give us the folder to scan. + var fullPath = + Parser.Parser.NormalizePath(_directoryService.FileSystem.Path.Join(libraryFolder, rootFolder.First())); + + var alreadyScheduled = + TaskScheduler.HasAlreadyEnqueuedTask(ScannerService.Name, "ScanFolder", new object[] {fullPath}); + _logger.LogDebug("{FullPath} already enqueued: {Value}", fullPath, alreadyScheduled); + if (!alreadyScheduled) { - _logger.LogDebug("[LibraryWatcher] Scheduling ScanSeriesFolder for {Folder}", item.FolderPath); - BackgroundJob.Enqueue(() => _scannerService.ScanFolder(item.FolderPath)); - _scanQueue.Dequeue(); + _logger.LogDebug("[LibraryWatcher] Scheduling ScanFolder for {Folder}", fullPath); + BackgroundJob.Schedule(() => _scannerService.ScanFolder(fullPath), _queueWaitTime); } else { - i++; + _logger.LogDebug("[LibraryWatcher] Skipped scheduling ScanFolder for {Folder} as a job already queued", + fullPath); } - } - - if (_scanQueue.Count > 0) + catch (Exception ex) { - Task.Delay(TimeSpan.FromSeconds(30)).ContinueWith(t=> ProcessQueue()); + _logger.LogError(ex, "[LibraryWatcher] An error occured when processing a watch event"); } - + _logger.LogDebug("ProcessChange occured in {ElapsedMilliseconds}ms", sw.ElapsedMilliseconds); } + + + } diff --git a/API/Services/Tasks/Scanner/ParseScannedFiles.cs b/API/Services/Tasks/Scanner/ParseScannedFiles.cs index bd3a21012..dab44e1ba 100644 --- a/API/Services/Tasks/Scanner/ParseScannedFiles.cs +++ b/API/Services/Tasks/Scanner/ParseScannedFiles.cs @@ -80,7 +80,9 @@ namespace API.Services.Tasks.Scanner string normalizedPath; if (scanDirectoryByDirectory) { - var directories = _directoryService.GetDirectories(folderPath).ToList(); + // This is used in library scan, so we should check first for a ignore file and use that here as well + var potentialIgnoreFile = _directoryService.FileSystem.Path.Join(folderPath, DirectoryService.KavitaIgnoreFile); + var directories = _directoryService.GetDirectories(folderPath, _directoryService.CreateMatcherFromFile(potentialIgnoreFile)).ToList(); foreach (var directory in directories) { @@ -219,7 +221,7 @@ namespace API.Services.Tasks.Scanner IDictionary> seriesPaths, Action>> processSeriesInfos, bool forceCheck = false) { - await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.FileScanProgressEvent("Starting file scan", libraryName, ProgressEventType.Started)); + await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.FileScanProgressEvent("File Scan Starting", libraryName, ProgressEventType.Started)); foreach (var folderPath in folders) { @@ -284,7 +286,7 @@ namespace API.Services.Tasks.Scanner } } - await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.FileScanProgressEvent(string.Empty, libraryName, ProgressEventType.Ended)); + await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.FileScanProgressEvent("File Scan Done", libraryName, ProgressEventType.Ended)); } private bool HasSeriesFolderNotChangedSinceLastScan(IDictionary> seriesPaths, string normalizedFolder, bool forceCheck = false) diff --git a/API/Parser/DefaultParser.cs b/API/Services/Tasks/Scanner/Parser/DefaultParser.cs similarity index 100% rename from API/Parser/DefaultParser.cs rename to API/Services/Tasks/Scanner/Parser/DefaultParser.cs diff --git a/API/Parser/Parser.cs b/API/Services/Tasks/Scanner/Parser/Parser.cs similarity index 99% rename from API/Parser/Parser.cs rename to API/Services/Tasks/Scanner/Parser/Parser.cs index f30d686ad..8ddac4126 100644 --- a/API/Parser/Parser.cs +++ b/API/Services/Tasks/Scanner/Parser/Parser.cs @@ -21,8 +21,6 @@ namespace API.Parser public const string SupportedExtensions = ArchiveFileExtensions + "|" + ImageFileExtensions + "|" + BookFileExtensions; - public static readonly string[] SupportedGlobExtensions = new [] {@"**/*.png", @"**/*.cbz", @"**/*.pdf"}; - private const RegexOptions MatchOptions = RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.CultureInvariant; diff --git a/API/Parser/ParserInfo.cs b/API/Services/Tasks/Scanner/Parser/ParserInfo.cs similarity index 100% rename from API/Parser/ParserInfo.cs rename to API/Services/Tasks/Scanner/Parser/ParserInfo.cs diff --git a/API/Services/Tasks/Scanner/ScanLibrary.cs b/API/Services/Tasks/Scanner/ScanLibrary.cs deleted file mode 100644 index 2aea6f34e..000000000 --- a/API/Services/Tasks/Scanner/ScanLibrary.cs +++ /dev/null @@ -1,111 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading.Tasks; -using API.Data; -using API.Entities; -using API.Helpers; -using API.Parser; -using Kavita.Common.Helpers; -using Microsoft.Extensions.Logging; - -namespace API.Services.Tasks.Scanner; - -/// -/// This is responsible for scanning and updating a Library -/// -public class ScanLibrary -{ - private readonly IDirectoryService _directoryService; - private readonly IUnitOfWork _unitOfWork; - private readonly ILogger _logger; - - public ScanLibrary(IDirectoryService directoryService, IUnitOfWork unitOfWork, ILogger logger) - { - _directoryService = directoryService; - _unitOfWork = unitOfWork; - _logger = logger; - } - - - // public Task UpdateLibrary(Library library) - // { - // - // - // } - - - - - /// - /// Gets the list of all parserInfos given a Series (Will match on Name, LocalizedName, OriginalName). If the series does not exist within, return empty list. - /// - /// - /// - /// - public static IList GetInfosByName(Dictionary> parsedSeries, Series series) - { - var allKeys = parsedSeries.Keys.Where(ps => - SeriesHelper.FindSeries(series, ps)); - - var infos = new List(); - foreach (var key in allKeys) - { - infos.AddRange(parsedSeries[key]); - } - - return infos; - } - - - /// - /// This will Scan all files in a folder path. For each folder within the folderPath, FolderAction will be invoked for all files contained - /// - /// A library folder or series folder - /// A callback async Task to be called once all files for each folder path are found - public async Task ProcessFiles(string folderPath, bool isLibraryFolder, Func, string,Task> folderAction) - { - if (isLibraryFolder) - { - var directories = _directoryService.GetDirectories(folderPath).ToList(); - - foreach (var directory in directories) - { - // For a scan, this is doing everything in the directory loop before the folder Action is called...which leads to no progress indication - await folderAction(_directoryService.ScanFiles(directory), directory); - } - } - else - { - //folderAction(ScanFiles(folderPath)); - await folderAction(_directoryService.ScanFiles(folderPath), folderPath); - } - } - - - - private GlobMatcher CreateIgnoreMatcher(string ignoreFile) - { - if (!_directoryService.FileSystem.File.Exists(ignoreFile)) - { - return null; - } - - // Read file in and add each line to Matcher - var lines = _directoryService.FileSystem.File.ReadAllLines(ignoreFile); - if (lines.Length == 0) - { - _logger.LogError("Kavita Ignore file found but empty, ignoring: {IgnoreFile}", ignoreFile); - return null; - } - - GlobMatcher matcher = new(); - foreach (var line in lines) - { - matcher.AddExclude(line); - } - - return matcher; - } -} diff --git a/API/Services/Tasks/ScannerService.cs b/API/Services/Tasks/ScannerService.cs index 09b1884ac..7f3bf7a86 100644 --- a/API/Services/Tasks/ScannerService.cs +++ b/API/Services/Tasks/ScannerService.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.Globalization; using System.IO; using System.Linq; using System.Threading.Tasks; @@ -68,6 +69,7 @@ public enum ScanCancelReason */ public class ScannerService : IScannerService { + public const string Name = "ScannerService"; private readonly IUnitOfWork _unitOfWork; private readonly ILogger _logger; private readonly IMetadataService _metadataService; @@ -277,7 +279,7 @@ public class ScannerService : IScannerService return ScanCancelReason.FolderMount; } - // If all series Folder paths haven't been modified since last scan, abort + // If all series Folder paths haven't been modified since last scan, abort (NOTE: This flow never happens as ScanSeries will always bypass) if (!bypassFolderChecks) { @@ -293,7 +295,7 @@ public class ScannerService : IScannerService series.Name); await _eventHub.SendMessageAsync(MessageFactory.Info, MessageFactory.InfoEvent($"{series.Name} scan has no work to do", - "All folders have not been changed since last scan. Scan will be aborted.")); + $"All folders have not been changed since last scan ({series.LastFolderScanned.ToString(CultureInfo.CurrentCulture)}). Scan will be aborted.")); return ScanCancelReason.NoChange; } } @@ -304,7 +306,7 @@ public class ScannerService : IScannerService series.Name); await _eventHub.SendMessageAsync(MessageFactory.Info, MessageFactory.ErrorEvent($"{series.Name} scan has no work to do", - "The folder the series is in is missing. Delete series manually or perform a library scan.")); + "The folder the series was in is missing. Delete series manually or perform a library scan.")); return ScanCancelReason.NoCancel; } } @@ -316,7 +318,7 @@ public class ScannerService : IScannerService private static void RemoveParsedInfosNotForSeries(Dictionary> parsedSeries, Series series) { var keys = parsedSeries.Keys; - foreach (var key in keys.Where(key => !SeriesHelper.FindSeries(series, key))) // series.Format != key.Format || + foreach (var key in keys.Where(key => !SeriesHelper.FindSeries(series, key))) { parsedSeries.Remove(key); } @@ -420,7 +422,7 @@ public class ScannerService : IScannerService _logger.LogInformation("[ScannerService] {LibraryName} scan has no work to do. All folders have not been changed since last scan", library.Name); await _eventHub.SendMessageAsync(MessageFactory.Info, MessageFactory.InfoEvent($"{library.Name} scan has no work to do", - "All folders have not been changed since last scan. Scan will be aborted.")); + $"All folders have not been changed since last scan ({library.Folders.Max(f => f.LastScanned).ToString(CultureInfo.CurrentCulture)}). Scan will be aborted.")); BackgroundJob.Enqueue(() => _metadataService.GenerateCoversForLibrary(library.Id, false)); BackgroundJob.Enqueue(() => _wordCountAnalyzerService.ScanLibrary(library.Id, false)); @@ -485,7 +487,7 @@ public class ScannerService : IScannerService await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.FileScanProgressEvent(string.Empty, library.Name, ProgressEventType.Ended)); - _logger.LogInformation("[ScannerService] Finished file scan in {ScanAndUpdateTime}. Updating database", scanElapsedTime); + _logger.LogInformation("[ScannerService] Finished file scan in {ScanAndUpdateTime} milliseconds. Updating database", scanElapsedTime); var time = DateTime.Now; foreach (var folderPath in library.Folders) diff --git a/API/SignalR/MessageFactory.cs b/API/SignalR/MessageFactory.cs index f8a8de873..74ee4cc0f 100644 --- a/API/SignalR/MessageFactory.cs +++ b/API/SignalR/MessageFactory.cs @@ -337,35 +337,42 @@ namespace API.SignalR /// Represents a file being scanned by Kavita for processing and grouping /// /// Does not have a progress as it's unknown how many files there are. Instead sends -1 to represent indeterminate - /// + /// /// /// /// - public static SignalRMessage FileScanProgressEvent(string filename, string libraryName, string eventType) + public static SignalRMessage FileScanProgressEvent(string folderPath, string libraryName, string eventType) { return new SignalRMessage() { Name = FileScanProgress, Title = $"Scanning {libraryName}", - SubTitle = Path.GetFileName(filename), + SubTitle = folderPath, EventType = eventType, Progress = ProgressType.Indeterminate, Body = new { Title = $"Scanning {libraryName}", - Subtitle = filename, - Filename = filename, + Subtitle = folderPath, + Filename = folderPath, EventTime = DateTime.Now, } }; } + /// + /// This informs the UI with details about what is being processed by the Scanner + /// + /// + /// + /// + /// public static SignalRMessage LibraryScanProgressEvent(string libraryName, string eventType, string seriesName = "") { return new SignalRMessage() { Name = ScanProgress, - Title = $"Scanning {libraryName}", + Title = $"Processing {seriesName}", SubTitle = seriesName, EventType = eventType, Progress = ProgressType.Indeterminate, diff --git a/Dockerfile b/Dockerfile index 7512c0d1e..c8e090534 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,12 +21,8 @@ COPY --from=copytask /files/wwwroot /kavita/wwwroot #Installs program dependencies RUN apt-get update \ && apt-get install -y libicu-dev libssl1.1 libgdiplus curl \ - && apt-get install -y libvips --no-install-recommends \ && rm -rf /var/lib/apt/lists/* -#Removes the libvips.so.42 file to fix the AVX CPU requirement issue -RUN rm /kavita/libvips.so.42 - COPY entrypoint.sh /entrypoint.sh EXPOSE 5000 diff --git a/UI/Web/src/app/admin/manage-settings/manage-settings.component.html b/UI/Web/src/app/admin/manage-settings/manage-settings.component.html index fc990d48c..f8927f89a 100644 --- a/UI/Web/src/app/admin/manage-settings/manage-settings.component.html +++ b/UI/Web/src/app/admin/manage-settings/manage-settings.component.html @@ -77,7 +77,7 @@
-

OPDS support will allow all users to use OPDS to read and download content from the server. If OPDS is enabled, a user will not need download permissions to download media while using it.

+

OPDS support will allow all users to use OPDS to read and download content from the server.

@@ -85,7 +85,7 @@
- + Expiremental

Allows Kavita to monitor Library Folders to detect changes and invoke scanning on those changes. This allows content to be updated without manually invoking scans or waiting for nightly scans.

diff --git a/UI/Web/src/app/admin/manage-settings/manage-settings.component.ts b/UI/Web/src/app/admin/manage-settings/manage-settings.component.ts index 31d4c3a0f..913fe6f27 100644 --- a/UI/Web/src/app/admin/manage-settings/manage-settings.component.ts +++ b/UI/Web/src/app/admin/manage-settings/manage-settings.component.ts @@ -3,6 +3,7 @@ import { FormGroup, Validators, FormControl } from '@angular/forms'; import { NgbModal } from '@ng-bootstrap/ng-bootstrap'; import { ToastrService } from 'ngx-toastr'; import { take } from 'rxjs/operators'; +import { TagBadgeCursor } from 'src/app/shared/tag-badge/tag-badge.component'; import { SettingsService } from '../settings.service'; import { DirectoryPickerComponent, DirectoryPickerResult } from '../_modals/directory-picker/directory-picker.component'; import { ServerSettings } from '../_models/server-settings'; @@ -20,6 +21,10 @@ export class ManageSettingsComponent implements OnInit { taskFrequencies: Array = []; logLevels: Array = []; + get TagBadgeCursor() { + return TagBadgeCursor; + } + constructor(private settingsService: SettingsService, private toastr: ToastrService, private modalService: NgbModal) { } @@ -45,6 +50,7 @@ export class ManageSettingsComponent implements OnInit { this.settingsForm.addControl('enableSwaggerUi', new FormControl(this.serverSettings.enableSwaggerUi, [Validators.required])); this.settingsForm.addControl('totalBackups', new FormControl(this.serverSettings.totalBackups, [Validators.required, Validators.min(1), Validators.max(30)])); this.settingsForm.addControl('enableFolderWatching', new FormControl(this.serverSettings.enableFolderWatching, [Validators.required])); + this.settingsForm.addControl('convertBookmarkToWebP', new FormControl(this.serverSettings.convertBookmarkToWebP, [])); }); } @@ -62,6 +68,7 @@ export class ManageSettingsComponent implements OnInit { this.settingsForm.get('enableSwaggerUi')?.setValue(this.serverSettings.enableSwaggerUi); this.settingsForm.get('totalBackups')?.setValue(this.serverSettings.totalBackups); this.settingsForm.get('enableFolderWatching')?.setValue(this.serverSettings.enableFolderWatching); + this.settingsForm.get('convertBookmarkToWebP')?.setValue(this.serverSettings.convertBookmarkToWebP); this.settingsForm.markAsPristine(); } diff --git a/UI/Web/src/app/admin/manage-users/manage-users.component.ts b/UI/Web/src/app/admin/manage-users/manage-users.component.ts index fc127a886..2d4857d4c 100644 --- a/UI/Web/src/app/admin/manage-users/manage-users.component.ts +++ b/UI/Web/src/app/admin/manage-users/manage-users.component.ts @@ -141,7 +141,7 @@ export class ManageUsersComponent implements OnInit, OnDestroy { setup(member: Member) { this.accountService.getInviteUrl(member.id, false).subscribe(url => { - console.log('Url: ', url); + console.log('Invite Url: ', url); if (url) { this.router.navigateByUrl(url); } diff --git a/UI/Web/src/app/nav/events-widget/events-widget.component.ts b/UI/Web/src/app/nav/events-widget/events-widget.component.ts index 2f7bfef62..9d6672279 100644 --- a/UI/Web/src/app/nav/events-widget/events-widget.component.ts +++ b/UI/Web/src/app/nav/events-widget/events-widget.component.ts @@ -68,7 +68,6 @@ export class EventsWidgetComponent implements OnInit, OnDestroy { ngOnInit(): void { this.messageHub.messages$.pipe(takeUntil(this.onDestroy)).subscribe(event => { if (event.event === EVENTS.NotificationProgress) { - console.log('[Event Widget]: Event came in ', event.payload); this.processNotificationProgressEvent(event); } else if (event.event === EVENTS.Error) { const values = this.errorSource.getValue(); diff --git a/UI/Web/src/app/shared/_services/utility.service.ts b/UI/Web/src/app/shared/_services/utility.service.ts index 6107d6197..ffe1143ad 100644 --- a/UI/Web/src/app/shared/_services/utility.service.ts +++ b/UI/Web/src/app/shared/_services/utility.service.ts @@ -140,6 +140,12 @@ export class UtilityService { } deepEqual(object1: any, object2: any) { + if ((object1 === null || object1 === undefined) && (object2 !== null || object2 !== undefined)) return false; + if ((object2 === null || object2 === undefined) && (object1 !== null || object1 !== undefined)) return false; + if (object1 === null && object2 === null) return true; + if (object1 === undefined && object2 === undefined) return true; + + const keys1 = Object.keys(object1); const keys2 = Object.keys(object2); if (keys1.length !== keys2.length) { diff --git a/UI/Web/src/app/shared/tag-badge/tag-badge.component.ts b/UI/Web/src/app/shared/tag-badge/tag-badge.component.ts index 6936cc9a2..546ca755a 100644 --- a/UI/Web/src/app/shared/tag-badge/tag-badge.component.ts +++ b/UI/Web/src/app/shared/tag-badge/tag-badge.component.ts @@ -1,4 +1,4 @@ -import { ChangeDetectionStrategy, ChangeDetectorRef, Component, Input, OnInit } from '@angular/core'; +import { ChangeDetectionStrategy, Component, Input } from '@angular/core'; /** * What type of cursor to apply to the tag badge