From 037a1a5a8c4d0cbb9c2cf6a2392b70c476e23782 Mon Sep 17 00:00:00 2001 From: Joseph Milazzo Date: Tue, 23 Aug 2022 11:45:11 -0500 Subject: [PATCH] Folder Watching (#1467) * Hooked in a server setting to enable/disable folder watching * Validated the file rename change event * Validated delete file works * Tweaked some logic to determine if a change occurs on a folder or a file. * Added a note for an upcoming branch * Some minor changes in the loop that just shift where code runs. * Implemented ScanFolder api * Ensure we restart watchers when we modify a library folder. * Fixed a unit test --- API.Tests/Parser/ParserTest.cs | 2 +- API.Tests/Services/DirectoryServiceTests.cs | 1 + API/Controllers/LibraryController.cs | 40 ++++- API/Controllers/SettingsController.cs | 20 ++- API/DTOs/ScanFolderDto.cs | 17 +++ API/DTOs/Settings/ServerSettingDTO.cs | 11 +- API/Data/Seed.cs | 1 + API/Entities/Enums/ServerSettingKey.cs | 5 + .../Converters/ServerSettingConverter.cs | 3 + API/Parser/Parser.cs | 3 +- API/Services/DirectoryService.cs | 6 +- .../StartupTasksHostedService.cs | 17 ++- API/Services/TaskScheduler.cs | 7 + API/Services/Tasks/Scanner/LibraryWatcher.cs | 138 +++++++++++------- API/Services/Tasks/Scanner/ProcessSeries.cs | 55 ++++--- API/Services/Tasks/ScannerService.cs | 5 +- .../src/app/admin/_models/server-settings.ts | 1 + .../manage-settings.component.html | 9 ++ .../manage-settings.component.ts | 30 ++-- 19 files changed, 269 insertions(+), 102 deletions(-) create mode 100644 API/DTOs/ScanFolderDto.cs diff --git a/API.Tests/Parser/ParserTest.cs b/API.Tests/Parser/ParserTest.cs index 4ae75d91b..3d3c95e5c 100644 --- a/API.Tests/Parser/ParserTest.cs +++ b/API.Tests/Parser/ParserTest.cs @@ -223,7 +223,7 @@ namespace API.Tests.Parser [InlineData("/manga/1/1/1", "/manga/1/1/1")] [InlineData("/manga/1/1/1.jpg", "/manga/1/1/1.jpg")] [InlineData(@"/manga/1/1\1.jpg", @"/manga/1/1/1.jpg")] - [InlineData("/manga/1/1//1", "/manga/1/1//1")] + [InlineData("/manga/1/1//1", "/manga/1/1/1")] [InlineData("/manga/1\\1\\1", "/manga/1/1/1")] [InlineData("C:/manga/1\\1\\1.jpg", "C:/manga/1/1/1.jpg")] public void NormalizePathTest(string inputPath, string expected) diff --git a/API.Tests/Services/DirectoryServiceTests.cs b/API.Tests/Services/DirectoryServiceTests.cs index 2e02641b9..602579ced 100644 --- a/API.Tests/Services/DirectoryServiceTests.cs +++ b/API.Tests/Services/DirectoryServiceTests.cs @@ -677,6 +677,7 @@ namespace API.Tests.Services [InlineData(new [] {"C:/Manga/"}, new [] {"C:/Manga/Love Hina/Vol. 01.cbz"}, "C:/Manga/Love Hina")] [InlineData(new [] {"C:/Manga/Dir 1/", "c://Manga/Dir 2/"}, new [] {"C:/Manga/Dir 1/Love Hina/Vol. 01.cbz"}, "C:/Manga/Dir 1/Love Hina")] [InlineData(new [] {"C:/Manga/Dir 1/", "c://Manga/"}, new [] {"D:/Manga/Love Hina/Vol. 01.cbz", "D:/Manga/Vol. 01.cbz"}, "")] + [InlineData(new [] {"C:/Manga/"}, new [] {"C:/Manga//Love Hina/Vol. 01.cbz"}, "C:/Manga/Love Hina")] public void FindHighestDirectoriesFromFilesTest(string[] rootDirectories, string[] files, string expectedDirectory) { var fileSystem = new MockFileSystem(); diff --git a/API/Controllers/LibraryController.cs b/API/Controllers/LibraryController.cs index 321d0d06f..beadd2a95 100644 --- a/API/Controllers/LibraryController.cs +++ b/API/Controllers/LibraryController.cs @@ -13,6 +13,7 @@ using API.Entities; using API.Entities.Enums; using API.Extensions; using API.Services; +using API.Services.Tasks.Scanner; using API.SignalR; using AutoMapper; using Microsoft.AspNetCore.Authorization; @@ -30,10 +31,11 @@ namespace API.Controllers private readonly ITaskScheduler _taskScheduler; private readonly IUnitOfWork _unitOfWork; private readonly IEventHub _eventHub; + private readonly ILibraryWatcher _libraryWatcher; public LibraryController(IDirectoryService directoryService, ILogger logger, IMapper mapper, ITaskScheduler taskScheduler, - IUnitOfWork unitOfWork, IEventHub eventHub) + IUnitOfWork unitOfWork, IEventHub eventHub, ILibraryWatcher libraryWatcher) { _directoryService = directoryService; _logger = logger; @@ -41,6 +43,7 @@ namespace API.Controllers _taskScheduler = taskScheduler; _unitOfWork = unitOfWork; _eventHub = eventHub; + _libraryWatcher = libraryWatcher; } /// @@ -77,6 +80,7 @@ namespace API.Controllers if (!await _unitOfWork.CommitAsync()) return BadRequest("There was a critical issue. Please try again."); _logger.LogInformation("Created a new library: {LibraryName}", library.Name); + await _libraryWatcher.RestartWatching(); _taskScheduler.ScanLibrary(library.Id); await _eventHub.SendMessageAsync(MessageFactory.LibraryModified, MessageFactory.LibraryModifiedEvent(library.Id, "create"), false); @@ -196,6 +200,37 @@ namespace API.Controllers return Ok(await _unitOfWork.LibraryRepository.GetLibraryDtosForUsernameAsync(User.GetUsername())); } + /// + /// Given a valid path, will invoke either a Scan Series or Scan Library. If the folder does not exist within Kavita, the request will be ignored + /// + /// + /// + [AllowAnonymous] + [HttpPost("scan-folder")] + public async Task ScanFolder(ScanFolderDto dto) + { + var userId = await _unitOfWork.UserRepository.GetUserIdByApiKeyAsync(dto.ApiKey); + var user = await _unitOfWork.UserRepository.GetUserByIdAsync(userId); + // Validate user has Admin privileges + var isAdmin = await _unitOfWork.UserRepository.IsUserAdminAsync(user); + if (!isAdmin) return BadRequest("API key must belong to an admin"); + if (dto.FolderPath.Contains("..")) return BadRequest("Invalid Path"); + + dto.FolderPath = Parser.Parser.NormalizePath(dto.FolderPath); + + var libraryFolder = (await _unitOfWork.LibraryRepository.GetLibraryDtosAsync()) + .SelectMany(l => l.Folders) + .Distinct() + .Select(Parser.Parser.NormalizePath); + + var seriesFolder = _directoryService.FindHighestDirectoriesFromFiles(libraryFolder, + new List() {dto.FolderPath}); + + _taskScheduler.ScanFolder(seriesFolder.Keys.Count == 1 ? seriesFolder.Keys.First() : dto.FolderPath); + + return Ok(); + } + [Authorize(Policy = "RequireAdminRole")] [HttpDelete("delete")] public async Task> DeleteLibrary(int libraryId) @@ -221,6 +256,8 @@ namespace API.Controllers _taskScheduler.CleanupChapters(chapterIds); } + await _libraryWatcher.RestartWatching(); + foreach (var seriesId in seriesIds) { await _eventHub.SendMessageAsync(MessageFactory.SeriesRemoved, @@ -264,6 +301,7 @@ namespace API.Controllers if (!await _unitOfWork.CommitAsync()) return BadRequest("There was a critical issue updating the library."); if (originalFolders.Count != libraryForUserDto.Folders.Count() || typeUpdate) { + await _libraryWatcher.RestartWatching(); _taskScheduler.ScanLibrary(library.Id); } diff --git a/API/Controllers/SettingsController.cs b/API/Controllers/SettingsController.cs index a712a39cc..e1a758775 100644 --- a/API/Controllers/SettingsController.cs +++ b/API/Controllers/SettingsController.cs @@ -10,6 +10,7 @@ using API.Entities.Enums; using API.Extensions; using API.Helpers.Converters; using API.Services; +using API.Services.Tasks.Scanner; using AutoMapper; using Flurl.Http; using Kavita.Common; @@ -29,9 +30,10 @@ namespace API.Controllers private readonly IDirectoryService _directoryService; private readonly IMapper _mapper; private readonly IEmailService _emailService; + private readonly ILibraryWatcher _libraryWatcher; public SettingsController(ILogger logger, IUnitOfWork unitOfWork, ITaskScheduler taskScheduler, - IDirectoryService directoryService, IMapper mapper, IEmailService emailService) + IDirectoryService directoryService, IMapper mapper, IEmailService emailService, ILibraryWatcher libraryWatcher) { _logger = logger; _unitOfWork = unitOfWork; @@ -39,6 +41,7 @@ namespace API.Controllers _directoryService = directoryService; _mapper = mapper; _emailService = emailService; + _libraryWatcher = libraryWatcher; } [AllowAnonymous] @@ -227,6 +230,21 @@ namespace API.Controllers _unitOfWork.SettingsRepository.Update(setting); } + + if (setting.Key == ServerSettingKey.EnableFolderWatching && updateSettingsDto.EnableFolderWatching + string.Empty != setting.Value) + { + setting.Value = updateSettingsDto.EnableFolderWatching + string.Empty; + _unitOfWork.SettingsRepository.Update(setting); + + if (updateSettingsDto.EnableFolderWatching) + { + await _libraryWatcher.StartWatching(); + } + else + { + _libraryWatcher.StopWatching(); + } + } } if (!_unitOfWork.HasChanges()) return Ok(updateSettingsDto); diff --git a/API/DTOs/ScanFolderDto.cs b/API/DTOs/ScanFolderDto.cs new file mode 100644 index 000000000..59ce4d0b5 --- /dev/null +++ b/API/DTOs/ScanFolderDto.cs @@ -0,0 +1,17 @@ +namespace API.DTOs; + +/// +/// DTO for requesting a folder to be scanned +/// +public class ScanFolderDto +{ + /// + /// Api key for a user with Admin permissions + /// + public string ApiKey { get; set; } + /// + /// Folder Path to Scan + /// + /// JSON cannot accept /, so you may need to use // escaping on paths + public string FolderPath { get; set; } +} diff --git a/API/DTOs/Settings/ServerSettingDTO.cs b/API/DTOs/Settings/ServerSettingDTO.cs index 9f33b6908..f979684af 100644 --- a/API/DTOs/Settings/ServerSettingDTO.cs +++ b/API/DTOs/Settings/ServerSettingDTO.cs @@ -1,5 +1,4 @@ -using System.Collections.Generic; -using API.Services; +using API.Services; namespace API.DTOs.Settings { @@ -43,7 +42,9 @@ namespace API.DTOs.Settings /// Represents a unique Id to this Kavita installation. Only used in Stats to identify unique installs. /// public string InstallId { get; set; } - + /// + /// If the server should save bookmarks as WebP encoding + /// public bool ConvertBookmarkToWebP { get; set; } /// /// If the Swagger UI Should be exposed. Does not require authentication, but does require a JWT. @@ -55,5 +56,9 @@ namespace API.DTOs.Settings /// /// Value should be between 1 and 30 public int TotalBackups { get; set; } = 30; + /// + /// If Kavita should watch the library folders and process changes + /// + public bool EnableFolderWatching { get; set; } = true; } } diff --git a/API/Data/Seed.cs b/API/Data/Seed.cs index 893256357..88acf41cd 100644 --- a/API/Data/Seed.cs +++ b/API/Data/Seed.cs @@ -103,6 +103,7 @@ namespace API.Data new() {Key = ServerSettingKey.ConvertBookmarkToWebP, Value = "false"}, new() {Key = ServerSettingKey.EnableSwaggerUi, Value = "false"}, new() {Key = ServerSettingKey.TotalBackups, Value = "30"}, + new() {Key = ServerSettingKey.EnableFolderWatching, Value = "true"}, }.ToArray()); foreach (var defaultSetting in DefaultSettings) diff --git a/API/Entities/Enums/ServerSettingKey.cs b/API/Entities/Enums/ServerSettingKey.cs index b387f1d85..3fcf938b2 100644 --- a/API/Entities/Enums/ServerSettingKey.cs +++ b/API/Entities/Enums/ServerSettingKey.cs @@ -91,5 +91,10 @@ namespace API.Entities.Enums /// [Description("TotalBackups")] TotalBackups = 16, + /// + /// If Kavita should watch the library folders and process changes + /// + [Description("EnableFolderWatching")] + EnableFolderWatching = 17, } } diff --git a/API/Helpers/Converters/ServerSettingConverter.cs b/API/Helpers/Converters/ServerSettingConverter.cs index 35c2428f3..6cc48e9eb 100644 --- a/API/Helpers/Converters/ServerSettingConverter.cs +++ b/API/Helpers/Converters/ServerSettingConverter.cs @@ -60,6 +60,9 @@ namespace API.Helpers.Converters case ServerSettingKey.InstallId: destination.InstallId = row.Value; break; + case ServerSettingKey.EnableFolderWatching: + destination.EnableFolderWatching = bool.Parse(row.Value); + break; } } diff --git a/API/Parser/Parser.cs b/API/Parser/Parser.cs index 35594caa3..f30d686ad 100644 --- a/API/Parser/Parser.cs +++ b/API/Parser/Parser.cs @@ -1074,7 +1074,8 @@ namespace API.Parser /// public static string NormalizePath(string path) { - return path.Replace(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + return path.Replace(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar) + .Replace(@"//", Path.AltDirectorySeparatorChar + string.Empty); } /// diff --git a/API/Services/DirectoryService.cs b/API/Services/DirectoryService.cs index 3c064dc11..6473f7033 100644 --- a/API/Services/DirectoryService.cs +++ b/API/Services/DirectoryService.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Collections.Immutable; using System.IO; @@ -511,7 +511,7 @@ namespace API.Services var fullPath = Path.Join(folder, parts.Last()); if (!dirs.ContainsKey(fullPath)) { - dirs.Add(fullPath, string.Empty); + dirs.Add(Parser.Parser.NormalizePath(fullPath), string.Empty); } } } @@ -560,7 +560,7 @@ namespace API.Services { try { - return Parser.Parser.NormalizePath(Directory.GetParent(fileOrFolder).FullName); + return Parser.Parser.NormalizePath(Directory.GetParent(fileOrFolder)?.FullName); } catch (Exception) { diff --git a/API/Services/HostedServices/StartupTasksHostedService.cs b/API/Services/HostedServices/StartupTasksHostedService.cs index 7be79f7f8..df7692c7c 100644 --- a/API/Services/HostedServices/StartupTasksHostedService.cs +++ b/API/Services/HostedServices/StartupTasksHostedService.cs @@ -1,6 +1,7 @@ using System; using System.Threading; using System.Threading.Tasks; +using API.Data; using API.Services.Tasks.Scanner; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; @@ -38,8 +39,20 @@ namespace API.Services.HostedServices //If stats startup fail the user can keep using the app } - var libraryWatcher = scope.ServiceProvider.GetRequiredService(); - //await libraryWatcher.StartWatchingLibraries(); // TODO: Enable this in the next PR + try + { + var unitOfWork = scope.ServiceProvider.GetRequiredService(); + if ((await unitOfWork.SettingsRepository.GetSettingsDtoAsync()).EnableFolderWatching) + { + var libraryWatcher = scope.ServiceProvider.GetRequiredService(); + await libraryWatcher.StartWatching(); + } + } + catch (Exception) + { + // Fail silently + } + } public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; diff --git a/API/Services/TaskScheduler.cs b/API/Services/TaskScheduler.cs index df7f20152..a9f34245d 100644 --- a/API/Services/TaskScheduler.cs +++ b/API/Services/TaskScheduler.cs @@ -19,6 +19,7 @@ public interface ITaskScheduler Task ScheduleTasks(); Task ScheduleStatsTasks(); void ScheduleUpdaterTasks(); + void ScanFolder(string folderPath); void ScanLibrary(int libraryId, bool force = false); void CleanupChapters(int[] chapterIds); void RefreshMetadata(int libraryId, bool forceUpdate = true); @@ -161,6 +162,12 @@ public class TaskScheduler : ITaskScheduler // Schedule update check between noon and 6pm local time RecurringJob.AddOrUpdate("check-updates", () => CheckForUpdate(), Cron.Daily(Rnd.Next(12, 18)), TimeZoneInfo.Local); } + + public void ScanFolder(string folderPath) + { + _scannerService.ScanFolder(Parser.Parser.NormalizePath(folderPath)); + } + #endregion public void ScanLibraries() diff --git a/API/Services/Tasks/Scanner/LibraryWatcher.cs b/API/Services/Tasks/Scanner/LibraryWatcher.cs index f4c2224ea..8a1063924 100644 --- a/API/Services/Tasks/Scanner/LibraryWatcher.cs +++ b/API/Services/Tasks/Scanner/LibraryWatcher.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; @@ -14,7 +13,20 @@ namespace API.Services.Tasks.Scanner; public interface ILibraryWatcher { - Task StartWatchingLibraries(); + /// + /// Start watching all library folders + /// + /// + Task StartWatching(); + /// + /// Stop watching all folders + /// + void StopWatching(); + /// + /// Essentially stops then starts watching. Useful if there is a change in folders or libraries + /// + /// + Task RestartWatching(); } internal class FolderScanQueueable @@ -51,17 +63,12 @@ public class LibraryWatcher : ILibraryWatcher private readonly ILogger _logger; private readonly IScannerService _scannerService; - private readonly IList _watchers = new List(); - private readonly Dictionary> _watcherDictionary = new (); - private IList _libraryFolders = new List(); - // TODO: This needs to be blocking so we can consume from another thread private readonly Queue _scanQueue = new Queue(); - //public readonly BlockingCollection ScanQueue = new BlockingCollection(); private readonly TimeSpan _queueWaitTime; - + private readonly FolderScanQueueableComparer _folderScanQueueableComparer = new FolderScanQueueableComparer(); public LibraryWatcher(IDirectoryService directoryService, IUnitOfWork unitOfWork, ILogger logger, IScannerService scannerService, IHostEnvironment environment) @@ -75,43 +82,63 @@ public class LibraryWatcher : ILibraryWatcher } - public async Task StartWatchingLibraries() + public async Task StartWatching() { _logger.LogInformation("Starting file watchers"); - _libraryFolders = (await _unitOfWork.LibraryRepository.GetLibraryDtosAsync()).SelectMany(l => l.Folders).ToList(); - foreach (var library in await _unitOfWork.LibraryRepository.GetLibraryDtosAsync()) + _libraryFolders = (await _unitOfWork.LibraryRepository.GetLibraryDtosAsync()) + .SelectMany(l => l.Folders) + .Distinct() + .Select(Parser.Parser.NormalizePath) + .ToList(); + foreach (var libraryFolder in _libraryFolders) { - foreach (var libraryFolder in library.Folders) + _logger.LogInformation("Watching {FolderPath}", libraryFolder); + var watcher = new FileSystemWatcher(libraryFolder); + watcher.NotifyFilter = NotifyFilters.CreationTime + | NotifyFilters.DirectoryName + | NotifyFilters.FileName + | NotifyFilters.LastWrite + | NotifyFilters.Size; + + watcher.Changed += OnChanged; + watcher.Created += OnCreated; + watcher.Deleted += OnDeleted; + watcher.Renamed += OnRenamed; + + watcher.Filter = "*.*"; + watcher.IncludeSubdirectories = true; + watcher.EnableRaisingEvents = true; + if (!_watcherDictionary.ContainsKey(libraryFolder)) { - _logger.LogInformation("Watching {FolderPath}", libraryFolder); - var watcher = new FileSystemWatcher(libraryFolder); - watcher.NotifyFilter = NotifyFilters.CreationTime - | NotifyFilters.DirectoryName - | NotifyFilters.FileName - | NotifyFilters.LastWrite - | NotifyFilters.Size; - - watcher.Changed += OnChanged; - watcher.Created += OnCreated; - watcher.Deleted += OnDeleted; - watcher.Renamed += OnRenamed; - - watcher.Filter = "*.*"; // TODO: Configure with Parser files - watcher.IncludeSubdirectories = true; - watcher.EnableRaisingEvents = true; - _logger.LogInformation("Watching {Folder}", libraryFolder); - _watchers.Add(watcher); - if (!_watcherDictionary.ContainsKey(libraryFolder)) - { - _watcherDictionary.Add(libraryFolder, new List()); - } - - _watcherDictionary[libraryFolder].Add(watcher); + _watcherDictionary.Add(libraryFolder, new List()); } + + _watcherDictionary[libraryFolder].Add(watcher); } } + public void StopWatching() + { + _logger.LogInformation("Stopping watching folders"); + foreach (var fileSystemWatcher in _watcherDictionary.Values.SelectMany(watcher => watcher)) + { + fileSystemWatcher.EnableRaisingEvents = false; + fileSystemWatcher.Changed -= OnChanged; + fileSystemWatcher.Created -= OnCreated; + fileSystemWatcher.Deleted -= OnDeleted; + fileSystemWatcher.Renamed -= OnRenamed; + fileSystemWatcher.Dispose(); + } + _watcherDictionary.Clear(); + } + + public async Task RestartWatching() + { + StopWatching(); + await StartWatching(); + } + private void OnChanged(object sender, FileSystemEventArgs e) { if (e.ChangeType != WatcherChangeTypes.Changed) return; @@ -122,12 +149,15 @@ public class LibraryWatcher : ILibraryWatcher private void OnCreated(object sender, FileSystemEventArgs e) { Console.WriteLine($"Created: {e.FullPath}, {e.Name}"); - ProcessChange(e.FullPath); + ProcessChange(e.FullPath, !_directoryService.FileSystem.File.Exists(e.Name)); } private void OnDeleted(object sender, FileSystemEventArgs e) { Console.WriteLine($"Deleted: {e.FullPath}, {e.Name}"); - ProcessChange(e.FullPath); + + // On deletion, we need another type of check. We need to check if e.Name has an extension or not + // NOTE: File deletion will trigger a folder change event, so this might not be needed + ProcessChange(e.FullPath, string.IsNullOrEmpty(_directoryService.FileSystem.Path.GetExtension(e.Name))); } @@ -137,18 +167,25 @@ public class LibraryWatcher : ILibraryWatcher Console.WriteLine($"Renamed:"); Console.WriteLine($" Old: {e.OldFullPath}"); Console.WriteLine($" New: {e.FullPath}"); - ProcessChange(e.FullPath); + ProcessChange(e.FullPath, _directoryService.FileSystem.Directory.Exists(e.FullPath)); } - private void ProcessChange(string filePath) + /// + /// Processes the file or folder change. + /// + /// File or folder that changed + /// If the change is on a directory and not a file + private void ProcessChange(string filePath, bool isDirectoryChange = false) { - if (!new Regex(Parser.Parser.SupportedExtensions).IsMatch(new FileInfo(filePath).Extension)) return; + // We need to check if directory or not + if (!isDirectoryChange && !new Regex(Parser.Parser.SupportedExtensions).IsMatch(new FileInfo(filePath).Extension)) return; // Don't do anything if a Library or ScanSeries in progress - if (TaskScheduler.RunningAnyTasksByMethod(new[] {"MetadataService", "ScannerService"})) - { - _logger.LogDebug("Suppressing Change due to scan being inprogress"); - return; - } + // if (TaskScheduler.RunningAnyTasksByMethod(new[] {"MetadataService", "ScannerService"})) + // { + // // NOTE: I'm not sure we need this to be honest. Now with the speed of the new loop and the queue, we should just put in queue for processing + // _logger.LogDebug("Suppressing Change due to scan being inprogress"); + // return; + // } var parentDirectory = _directoryService.GetParentDirectoryName(filePath); @@ -156,21 +193,20 @@ public class LibraryWatcher : ILibraryWatcher // We need to find the library this creation belongs to // Multiple libraries can point to the same base folder. In this case, we need use FirstOrDefault - var libraryFolder = _libraryFolders.Select(Parser.Parser.NormalizePath).FirstOrDefault(f => f.Contains(parentDirectory)); - + var libraryFolder = _libraryFolders.FirstOrDefault(f => parentDirectory.Contains(f)); if (string.IsNullOrEmpty(libraryFolder)) return; var rootFolder = _directoryService.GetFoldersTillRoot(libraryFolder, filePath).ToList(); if (!rootFolder.Any()) return; // Select the first folder and join with library folder, this should give us the folder to scan. - var fullPath = _directoryService.FileSystem.Path.Join(libraryFolder, rootFolder.First()); + var fullPath = Parser.Parser.NormalizePath(_directoryService.FileSystem.Path.Join(libraryFolder, rootFolder.First())); var queueItem = new FolderScanQueueable() { FolderPath = fullPath, QueueTime = DateTime.Now }; - if (_scanQueue.Contains(queueItem, new FolderScanQueueableComparer())) + if (_scanQueue.Contains(queueItem, _folderScanQueueableComparer)) { ProcessQueue(); return; @@ -205,7 +241,7 @@ public class LibraryWatcher : ILibraryWatcher if (_scanQueue.Count > 0) { - Task.Delay(TimeSpan.FromSeconds(10)).ContinueWith(t=> ProcessQueue()); + Task.Delay(_queueWaitTime).ContinueWith(t=> ProcessQueue()); } } diff --git a/API/Services/Tasks/Scanner/ProcessSeries.cs b/API/Services/Tasks/Scanner/ProcessSeries.cs index bebfae4ea..c20c7fa9b 100644 --- a/API/Services/Tasks/Scanner/ProcessSeries.cs +++ b/API/Services/Tasks/Scanner/ProcessSeries.cs @@ -116,14 +116,16 @@ public class ProcessSeries : IProcessSeries { _logger.LogInformation("[ScannerService] Processing series {SeriesName}", series.OriginalName); + var firstParsedInfo = parsedInfos[0]; + UpdateVolumes(series, parsedInfos); series.Pages = series.Volumes.Sum(v => v.Pages); series.NormalizedName = Parser.Parser.Normalize(series.Name); - series.OriginalName ??= parsedInfos[0].Series; + series.OriginalName ??= firstParsedInfo.Series; if (series.Format == MangaFormat.Unknown) { - series.Format = parsedInfos[0].Format; + series.Format = firstParsedInfo.Format; } if (string.IsNullOrEmpty(series.SortName)) @@ -133,9 +135,9 @@ public class ProcessSeries : IProcessSeries if (!series.SortNameLocked) { series.SortName = series.Name; - if (!string.IsNullOrEmpty(parsedInfos[0].SeriesSort)) + if (!string.IsNullOrEmpty(firstParsedInfo.SeriesSort)) { - series.SortName = parsedInfos[0].SeriesSort; + series.SortName = firstParsedInfo.SeriesSort; } } @@ -147,27 +149,11 @@ public class ProcessSeries : IProcessSeries series.NormalizedLocalizedName = Parser.Parser.Normalize(series.LocalizedName); } - // Update series FolderPath here (TODO: Move this into it's own private method) - var seriesDirs = _directoryService.FindHighestDirectoriesFromFiles(library.Folders.Select(l => l.Path), parsedInfos.Select(f => f.FullFilePath).ToList()); - if (seriesDirs.Keys.Count == 0) - { - _logger.LogCritical("Scan Series has files spread outside a main series folder. This has negative performance effects. Please ensure all series are under a single folder from library"); - await _eventHub.SendMessageAsync(MessageFactory.Info, - MessageFactory.InfoEvent($"{series.Name} has files spread outside a single series folder", - "This has negative performance effects. Please ensure all series are under a single folder from library")); - } - else - { - // Don't save FolderPath if it's a library Folder - if (!library.Folders.Select(f => f.Path).Contains(seriesDirs.Keys.First())) - { - series.FolderPath = Parser.Parser.NormalizePath(seriesDirs.Keys.First()); - } - } - - series.Metadata ??= DbFactory.SeriesMetadata(new List()); UpdateSeriesMetadata(series, library.Type); + // Update series FolderPath here + await UpdateSeriesFolderPath(parsedInfos, library, series); + series.LastFolderScanned = DateTime.Now; _unitOfWork.SeriesRepository.Attach(series); @@ -200,6 +186,28 @@ public class ProcessSeries : IProcessSeries EnqueuePostSeriesProcessTasks(series.LibraryId, series.Id, false); } + private async Task UpdateSeriesFolderPath(IEnumerable parsedInfos, Library library, Series series) + { + var seriesDirs = _directoryService.FindHighestDirectoriesFromFiles(library.Folders.Select(l => l.Path), + parsedInfos.Select(f => f.FullFilePath).ToList()); + if (seriesDirs.Keys.Count == 0) + { + _logger.LogCritical( + "Scan Series has files spread outside a main series folder. This has negative performance effects. Please ensure all series are under a single folder from library"); + await _eventHub.SendMessageAsync(MessageFactory.Info, + MessageFactory.InfoEvent($"{series.Name} has files spread outside a single series folder", + "This has negative performance effects. Please ensure all series are under a single folder from library")); + } + else + { + // Don't save FolderPath if it's a library Folder + if (!library.Folders.Select(f => f.Path).Contains(seriesDirs.Keys.First())) + { + series.FolderPath = Parser.Parser.NormalizePath(seriesDirs.Keys.First()); + } + } + } + public void EnqueuePostSeriesProcessTasks(int libraryId, int seriesId, bool forceUpdate = false) { BackgroundJob.Enqueue(() => _metadataService.GenerateCoversForSeries(libraryId, seriesId, forceUpdate)); @@ -208,6 +216,7 @@ public class ProcessSeries : IProcessSeries private static void UpdateSeriesMetadata(Series series, LibraryType libraryType) { + series.Metadata ??= DbFactory.SeriesMetadata(new List()); var isBook = libraryType == LibraryType.Book; var firstChapter = SeriesService.GetFirstChapterForMetadata(series, isBook); diff --git a/API/Services/Tasks/ScannerService.cs b/API/Services/Tasks/ScannerService.cs index 54953cbdf..45c001bd8 100644 --- a/API/Services/Tasks/ScannerService.cs +++ b/API/Services/Tasks/ScannerService.cs @@ -100,8 +100,6 @@ public class ScannerService : IScannerService [Queue(TaskScheduler.ScanQueue)] public async Task ScanFolder(string folder) { - // NOTE: I might want to move a lot of this code to the LibraryWatcher or something and just pack libraryId and seriesId - // Validate if we are scanning a new series (that belongs to a library) or an existing series var seriesId = await _unitOfWork.SeriesRepository.GetSeriesIdByFolder(folder); if (seriesId > 0) { @@ -109,6 +107,7 @@ public class ScannerService : IScannerService return; } + // This is basically rework of what's already done in Library Watcher but is needed if invoked via API var parentDirectory = _directoryService.GetParentDirectoryName(folder); if (string.IsNullOrEmpty(parentDirectory)) return; // This should never happen as it's calculated before enqueing @@ -456,6 +455,8 @@ public class ScannerService : IScannerService Format = parsedFiles.First().Format }; + // NOTE: Could we check if there are multiple found series (different series) and process each one? + if (skippedScan) { seenSeries.AddRange(parsedFiles.Select(pf => new ParsedSeries() diff --git a/UI/Web/src/app/admin/_models/server-settings.ts b/UI/Web/src/app/admin/_models/server-settings.ts index 736cd39f2..72438a431 100644 --- a/UI/Web/src/app/admin/_models/server-settings.ts +++ b/UI/Web/src/app/admin/_models/server-settings.ts @@ -12,4 +12,5 @@ export interface ServerSettings { convertBookmarkToWebP: boolean; enableSwaggerUi: boolean; totalBackups: number; + enableFolderWatching: boolean; } diff --git a/UI/Web/src/app/admin/manage-settings/manage-settings.component.html b/UI/Web/src/app/admin/manage-settings/manage-settings.component.html index d104e9bd6..fc990d48c 100644 --- a/UI/Web/src/app/admin/manage-settings/manage-settings.component.html +++ b/UI/Web/src/app/admin/manage-settings/manage-settings.component.html @@ -83,6 +83,15 @@ + +
+ +

Allows Kavita to monitor Library Folders to detect changes and invoke scanning on those changes. This allows content to be updated without manually invoking scans or waiting for nightly scans.

+
+ + +
+
diff --git a/UI/Web/src/app/admin/manage-settings/manage-settings.component.ts b/UI/Web/src/app/admin/manage-settings/manage-settings.component.ts index d20d4e2fb..31d4c3a0f 100644 --- a/UI/Web/src/app/admin/manage-settings/manage-settings.component.ts +++ b/UI/Web/src/app/admin/manage-settings/manage-settings.component.ts @@ -1,5 +1,5 @@ import { Component, OnInit } from '@angular/core'; -import { UntypedFormGroup, UntypedFormControl, Validators } from '@angular/forms'; +import { FormGroup, Validators, FormControl } from '@angular/forms'; import { NgbModal } from '@ng-bootstrap/ng-bootstrap'; import { ToastrService } from 'ngx-toastr'; import { take } from 'rxjs/operators'; @@ -16,7 +16,7 @@ import { ServerSettings } from '../_models/server-settings'; export class ManageSettingsComponent implements OnInit { serverSettings!: ServerSettings; - settingsForm: UntypedFormGroup = new UntypedFormGroup({}); + settingsForm: FormGroup = new FormGroup({}); taskFrequencies: Array = []; logLevels: Array = []; @@ -32,18 +32,19 @@ export class ManageSettingsComponent implements OnInit { }); this.settingsService.getServerSettings().pipe(take(1)).subscribe((settings: ServerSettings) => { this.serverSettings = settings; - this.settingsForm.addControl('cacheDirectory', new UntypedFormControl(this.serverSettings.cacheDirectory, [Validators.required])); - this.settingsForm.addControl('bookmarksDirectory', new UntypedFormControl(this.serverSettings.bookmarksDirectory, [Validators.required])); - this.settingsForm.addControl('taskScan', new UntypedFormControl(this.serverSettings.taskScan, [Validators.required])); - this.settingsForm.addControl('taskBackup', new UntypedFormControl(this.serverSettings.taskBackup, [Validators.required])); - this.settingsForm.addControl('port', new UntypedFormControl(this.serverSettings.port, [Validators.required])); - this.settingsForm.addControl('loggingLevel', new UntypedFormControl(this.serverSettings.loggingLevel, [Validators.required])); - this.settingsForm.addControl('allowStatCollection', new UntypedFormControl(this.serverSettings.allowStatCollection, [Validators.required])); - this.settingsForm.addControl('enableOpds', new UntypedFormControl(this.serverSettings.enableOpds, [Validators.required])); - this.settingsForm.addControl('baseUrl', new UntypedFormControl(this.serverSettings.baseUrl, [Validators.required])); - this.settingsForm.addControl('emailServiceUrl', new UntypedFormControl(this.serverSettings.emailServiceUrl, [Validators.required])); - this.settingsForm.addControl('enableSwaggerUi', new UntypedFormControl(this.serverSettings.enableSwaggerUi, [Validators.required])); - this.settingsForm.addControl('totalBackups', new UntypedFormControl(this.serverSettings.totalBackups, [Validators.required, Validators.min(1), Validators.max(30)])); + this.settingsForm.addControl('cacheDirectory', new FormControl(this.serverSettings.cacheDirectory, [Validators.required])); + this.settingsForm.addControl('bookmarksDirectory', new FormControl(this.serverSettings.bookmarksDirectory, [Validators.required])); + this.settingsForm.addControl('taskScan', new FormControl(this.serverSettings.taskScan, [Validators.required])); + this.settingsForm.addControl('taskBackup', new FormControl(this.serverSettings.taskBackup, [Validators.required])); + this.settingsForm.addControl('port', new FormControl(this.serverSettings.port, [Validators.required])); + this.settingsForm.addControl('loggingLevel', new FormControl(this.serverSettings.loggingLevel, [Validators.required])); + this.settingsForm.addControl('allowStatCollection', new FormControl(this.serverSettings.allowStatCollection, [Validators.required])); + this.settingsForm.addControl('enableOpds', new FormControl(this.serverSettings.enableOpds, [Validators.required])); + this.settingsForm.addControl('baseUrl', new FormControl(this.serverSettings.baseUrl, [Validators.required])); + this.settingsForm.addControl('emailServiceUrl', new FormControl(this.serverSettings.emailServiceUrl, [Validators.required])); + this.settingsForm.addControl('enableSwaggerUi', new FormControl(this.serverSettings.enableSwaggerUi, [Validators.required])); + this.settingsForm.addControl('totalBackups', new FormControl(this.serverSettings.totalBackups, [Validators.required, Validators.min(1), Validators.max(30)])); + this.settingsForm.addControl('enableFolderWatching', new FormControl(this.serverSettings.enableFolderWatching, [Validators.required])); }); } @@ -60,6 +61,7 @@ export class ManageSettingsComponent implements OnInit { this.settingsForm.get('emailServiceUrl')?.setValue(this.serverSettings.emailServiceUrl); this.settingsForm.get('enableSwaggerUi')?.setValue(this.serverSettings.enableSwaggerUi); this.settingsForm.get('totalBackups')?.setValue(this.serverSettings.totalBackups); + this.settingsForm.get('enableFolderWatching')?.setValue(this.serverSettings.enableFolderWatching); this.settingsForm.markAsPristine(); }