diff --git a/.gitignore b/.gitignore index eec036cbe..9e470748b 100644 --- a/.gitignore +++ b/.gitignore @@ -485,7 +485,6 @@ Thumbs.db ssl/ # App specific -appsettings.json /API/kavita.db /API/kavita.db-shm /API/kavita.db-wal diff --git a/API.Tests/Services/DirectoryServiceTests.cs b/API.Tests/Services/DirectoryServiceTests.cs index 602579ced..0bbd5728b 100644 --- a/API.Tests/Services/DirectoryServiceTests.cs +++ b/API.Tests/Services/DirectoryServiceTests.cs @@ -678,6 +678,7 @@ namespace API.Tests.Services [InlineData(new [] {"C:/Manga/Dir 1/", "c://Manga/Dir 2/"}, new [] {"C:/Manga/Dir 1/Love Hina/Vol. 01.cbz"}, "C:/Manga/Dir 1/Love Hina")] [InlineData(new [] {"C:/Manga/Dir 1/", "c://Manga/"}, new [] {"D:/Manga/Love Hina/Vol. 01.cbz", "D:/Manga/Vol. 01.cbz"}, "")] [InlineData(new [] {"C:/Manga/"}, new [] {"C:/Manga//Love Hina/Vol. 01.cbz"}, "C:/Manga/Love Hina")] + [InlineData(new [] {@"C:\mount\drive\Library\Test Library\Comics\"}, new [] {@"C:\mount\drive\Library\Test Library\Comics\Bruce Lee (1994)\Bruce Lee #001 (1994).cbz"}, @"C:/mount/drive/Library/Test Library/Comics/Bruce Lee (1994)")] public void FindHighestDirectoriesFromFilesTest(string[] rootDirectories, string[] files, string expectedDirectory) { var fileSystem = new MockFileSystem(); diff --git a/API/Data/UnitOfWork.cs b/API/Data/UnitOfWork.cs index 2e99ac32d..2d2adac42 100644 --- a/API/Data/UnitOfWork.cs +++ b/API/Data/UnitOfWork.cs @@ -1,4 +1,5 @@ -using System.Threading.Tasks; +using System; +using System.Threading.Tasks; using API.Data.Repositories; using API.Entities; using AutoMapper; @@ -26,7 +27,6 @@ public interface IUnitOfWork bool Commit(); Task CommitAsync(); bool HasChanges(); - bool Rollback(); Task RollbackAsync(); } public class UnitOfWork : IUnitOfWork @@ -93,16 +93,15 @@ public class UnitOfWork : IUnitOfWork /// public async Task RollbackAsync() { - await _context.DisposeAsync(); - return true; - } - /// - /// Rollback transaction - /// - /// - public bool Rollback() - { - _context.Dispose(); + try + { + await _context.Database.RollbackTransactionAsync(); + } + catch (Exception) + { + // Swallow exception (this might be used in places where a transaction isn't setup) + } + return true; } } diff --git a/API/Services/DirectoryService.cs b/API/Services/DirectoryService.cs index 6473f7033..aefcbee56 100644 --- a/API/Services/DirectoryService.cs +++ b/API/Services/DirectoryService.cs @@ -508,10 +508,10 @@ namespace API.Services break; } - var fullPath = Path.Join(folder, parts.Last()); + var fullPath = Parser.Parser.NormalizePath(Path.Join(folder, parts.Last())); if (!dirs.ContainsKey(fullPath)) { - dirs.Add(Parser.Parser.NormalizePath(fullPath), string.Empty); + dirs.Add(fullPath, string.Empty); } } } diff --git a/API/Services/MetadataService.cs b/API/Services/MetadataService.cs index 9a36eb639..bde0d89a6 100644 --- a/API/Services/MetadataService.cs +++ b/API/Services/MetadataService.cs @@ -161,7 +161,7 @@ public class MetadataService : IMetadataService /// private async Task ProcessSeriesCoverGen(Series series, bool forceUpdate) { - _logger.LogDebug("[MetadataService] Processing series {SeriesName}", series.OriginalName); + _logger.LogDebug("[MetadataService] Generating cover images for series: {SeriesName}", series.OriginalName); try { var volumeIndex = 0; diff --git a/API/Services/Tasks/Scanner/LibraryWatcher.cs b/API/Services/Tasks/Scanner/LibraryWatcher.cs index 8a1063924..4105e431b 100644 --- a/API/Services/Tasks/Scanner/LibraryWatcher.cs +++ b/API/Services/Tasks/Scanner/LibraryWatcher.cs @@ -78,7 +78,7 @@ public class LibraryWatcher : ILibraryWatcher _logger = logger; _scannerService = scannerService; - _queueWaitTime = environment.IsDevelopment() ? TimeSpan.FromSeconds(10) : TimeSpan.FromMinutes(5); + _queueWaitTime = environment.IsDevelopment() ? TimeSpan.FromSeconds(10) : TimeSpan.FromSeconds(30); } @@ -142,18 +142,18 @@ public class LibraryWatcher : ILibraryWatcher private void OnChanged(object sender, FileSystemEventArgs e) { if (e.ChangeType != WatcherChangeTypes.Changed) return; - Console.WriteLine($"Changed: {e.FullPath}, {e.Name}"); + _logger.LogDebug("[LibraryWatcher] Changed: {FullPath}, {Name}", e.FullPath, e.Name); ProcessChange(e.FullPath); } private void OnCreated(object sender, FileSystemEventArgs e) { - Console.WriteLine($"Created: {e.FullPath}, {e.Name}"); + _logger.LogDebug("[LibraryWatcher] Created: {FullPath}, {Name}", e.FullPath, e.Name); ProcessChange(e.FullPath, !_directoryService.FileSystem.File.Exists(e.Name)); } private void OnDeleted(object sender, FileSystemEventArgs e) { - Console.WriteLine($"Deleted: {e.FullPath}, {e.Name}"); + _logger.LogDebug("[LibraryWatcher] Deleted: {FullPath}, {Name}", e.FullPath, e.Name); // On deletion, we need another type of check. We need to check if e.Name has an extension or not // NOTE: File deletion will trigger a folder change event, so this might not be needed @@ -164,9 +164,9 @@ public class LibraryWatcher : ILibraryWatcher private void OnRenamed(object sender, RenamedEventArgs e) { - Console.WriteLine($"Renamed:"); - Console.WriteLine($" Old: {e.OldFullPath}"); - Console.WriteLine($" New: {e.FullPath}"); + _logger.LogDebug($"[LibraryWatcher] Renamed:"); + _logger.LogDebug(" Old: {OldFullPath}", e.OldFullPath); + _logger.LogDebug(" New: {FullPath}", e.FullPath); ProcessChange(e.FullPath, _directoryService.FileSystem.Directory.Exists(e.FullPath)); } @@ -179,14 +179,6 @@ public class LibraryWatcher : ILibraryWatcher { // We need to check if directory or not if (!isDirectoryChange && !new Regex(Parser.Parser.SupportedExtensions).IsMatch(new FileInfo(filePath).Extension)) return; - // Don't do anything if a Library or ScanSeries in progress - // if (TaskScheduler.RunningAnyTasksByMethod(new[] {"MetadataService", "ScannerService"})) - // { - // // NOTE: I'm not sure we need this to be honest. Now with the speed of the new loop and the queue, we should just put in queue for processing - // _logger.LogDebug("Suppressing Change due to scan being inprogress"); - // return; - // } - var parentDirectory = _directoryService.GetParentDirectoryName(filePath); if (string.IsNullOrEmpty(parentDirectory)) return; @@ -206,14 +198,12 @@ public class LibraryWatcher : ILibraryWatcher FolderPath = fullPath, QueueTime = DateTime.Now }; - if (_scanQueue.Contains(queueItem, _folderScanQueueableComparer)) + if (!_scanQueue.Contains(queueItem, _folderScanQueueableComparer)) { - ProcessQueue(); - return; + _logger.LogDebug("[LibraryWatcher] Queuing job for {Folder}", fullPath); + _scanQueue.Enqueue(queueItem); } - _scanQueue.Enqueue(queueItem); - ProcessQueue(); } @@ -228,7 +218,7 @@ public class LibraryWatcher : ILibraryWatcher var item = _scanQueue.Peek(); if (item.QueueTime < DateTime.Now.Subtract(_queueWaitTime)) { - _logger.LogDebug("Scheduling ScanSeriesFolder for {Folder}", item.FolderPath); + _logger.LogDebug("[LibraryWatcher] Scheduling ScanSeriesFolder for {Folder}", item.FolderPath); BackgroundJob.Enqueue(() => _scannerService.ScanFolder(item.FolderPath)); _scanQueue.Dequeue(); i++; diff --git a/API/Services/Tasks/Scanner/ParseScannedFiles.cs b/API/Services/Tasks/Scanner/ParseScannedFiles.cs index d993c2c9e..bd3a21012 100644 --- a/API/Services/Tasks/Scanner/ParseScannedFiles.cs +++ b/API/Services/Tasks/Scanner/ParseScannedFiles.cs @@ -320,7 +320,10 @@ namespace API.Services.Tasks.Scanner // NOTE: If we have multiple series in a folder with a localized title, then this will fail. It will group into one series. User needs to fix this themselves. string nonLocalizedSeries; - var nonLocalizedSeriesFound = infos.Where(i => !i.IsSpecial).Select(i => i.Series).Distinct().ToList(); + // Normalize this as many of the cases is a capitalization difference + var nonLocalizedSeriesFound = infos + .Where(i => !i.IsSpecial) + .Select(i => i.Series).DistinctBy(Parser.Parser.Normalize).ToList(); if (nonLocalizedSeriesFound.Count == 1) { nonLocalizedSeries = nonLocalizedSeriesFound.First(); @@ -330,7 +333,7 @@ namespace API.Services.Tasks.Scanner // There can be a case where there are multiple series in a folder that causes merging. if (nonLocalizedSeriesFound.Count > 2) { - _logger.LogError("[ScannerService] There are multiple series within one folder that contain localized series. This will cause them to group incorrectly. Please separate series into their own dedicated folder: {LocalizedSeries}", string.Join(", ", nonLocalizedSeriesFound)); + _logger.LogError("[ScannerService] There are multiple series within one folder that contain localized series. This will cause them to group incorrectly. Please separate series into their own dedicated folder or ensure there is only 2 potential series (localized and series): {LocalizedSeries}", string.Join(", ", nonLocalizedSeriesFound)); } nonLocalizedSeries = nonLocalizedSeriesFound.FirstOrDefault(s => !s.Equals(localizedSeries)); } diff --git a/API/Services/Tasks/Scanner/ProcessSeries.cs b/API/Services/Tasks/Scanner/ProcessSeries.cs index c20c7fa9b..30a166e44 100644 --- a/API/Services/Tasks/Scanner/ProcessSeries.cs +++ b/API/Services/Tasks/Scanner/ProcessSeries.cs @@ -183,7 +183,7 @@ public class ProcessSeries : IProcessSeries } _logger.LogInformation("[ScannerService] Finished series update on {SeriesName} in {Milliseconds} ms", seriesName, scanWatch.ElapsedMilliseconds); - EnqueuePostSeriesProcessTasks(series.LibraryId, series.Id, false); + EnqueuePostSeriesProcessTasks(series.LibraryId, series.Id); } private async Task UpdateSeriesFolderPath(IEnumerable parsedInfos, Library library, Series series) @@ -431,7 +431,6 @@ public class ProcessSeries : IProcessSeries volume = DbFactory.Volume(volumeNumber); volume.SeriesId = series.Id; series.Volumes.Add(volume); - _unitOfWork.VolumeRepository.Add(volume); } volume.Name = volumeNumber; diff --git a/API/Services/Tasks/ScannerService.cs b/API/Services/Tasks/ScannerService.cs index 45c001bd8..febc3c4ce 100644 --- a/API/Services/Tasks/ScannerService.cs +++ b/API/Services/Tasks/ScannerService.cs @@ -1,10 +1,8 @@ using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Linq; -using System.Threading; using System.Threading.Tasks; using API.Data; using API.Data.Repositories; @@ -41,9 +39,6 @@ public interface IScannerService [AutomaticRetry(Attempts = 3, OnAttemptsExceeded = AttemptsExceededAction.Delete)] Task ScanSeries(int seriesId, bool bypassFolderOptimizationChecks = true); - [Queue(TaskScheduler.ScanQueue)] - [DisableConcurrentExecution(60 * 60 * 60)] - [AutomaticRetry(Attempts = 3, OnAttemptsExceeded = AttemptsExceededAction.Delete)] Task ScanFolder(string folder); } @@ -81,11 +76,12 @@ public class ScannerService : IScannerService private readonly IDirectoryService _directoryService; private readonly IReadingItemService _readingItemService; private readonly IProcessSeries _processSeries; + private readonly IWordCountAnalyzerService _wordCountAnalyzerService; public ScannerService(IUnitOfWork unitOfWork, ILogger logger, IMetadataService metadataService, ICacheService cacheService, IEventHub eventHub, IDirectoryService directoryService, IReadingItemService readingItemService, - IProcessSeries processSeries) + IProcessSeries processSeries, IWordCountAnalyzerService wordCountAnalyzerService) { _unitOfWork = unitOfWork; _logger = logger; @@ -95,9 +91,9 @@ public class ScannerService : IScannerService _directoryService = directoryService; _readingItemService = readingItemService; _processSeries = processSeries; + _wordCountAnalyzerService = wordCountAnalyzerService; } - [Queue(TaskScheduler.ScanQueue)] public async Task ScanFolder(string folder) { var seriesId = await _unitOfWork.SeriesRepository.GetSeriesIdByFolder(folder); @@ -138,7 +134,12 @@ public class ScannerService : IScannerService var chapterIds = await _unitOfWork.SeriesRepository.GetChapterIdsForSeriesAsync(new[] {seriesId}); var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(series.LibraryId, LibraryIncludes.Folders); var libraryPaths = library.Folders.Select(f => f.Path).ToList(); - if (await ShouldScanSeries(seriesId, library, libraryPaths, series, true) != ScanCancelReason.NoCancel) return; + if (await ShouldScanSeries(seriesId, library, libraryPaths, series, true) != ScanCancelReason.NoCancel) + { + BackgroundJob.Enqueue(() => _metadataService.GenerateCoversForSeries(series.LibraryId, seriesId, false)); + BackgroundJob.Enqueue(() => _wordCountAnalyzerService.ScanSeries(library.Id, seriesId, false)); + return; + } var folderPath = series.FolderPath; if (string.IsNullOrEmpty(folderPath) || !_directoryService.Exists(folderPath)) @@ -420,6 +421,9 @@ public class ScannerService : IScannerService await _eventHub.SendMessageAsync(MessageFactory.Info, MessageFactory.InfoEvent($"{library.Name} scan has no work to do", "All folders have not been changed since last scan. Scan will be aborted.")); + + BackgroundJob.Enqueue(() => _metadataService.GenerateCoversForLibrary(library.Id, false)); + BackgroundJob.Enqueue(() => _wordCountAnalyzerService.ScanLibrary(library.Id, false)); return; } } @@ -455,7 +459,7 @@ public class ScannerService : IScannerService Format = parsedFiles.First().Format }; - // NOTE: Could we check if there are multiple found series (different series) and process each one? + // NOTE: Could we check if there are multiple found series (different series) and process each one? if (skippedScan) { diff --git a/API/config/appsettings.Development.json b/API/config/appsettings.Development.json index 0d7c12bda..bd19064c4 100644 --- a/API/config/appsettings.Development.json +++ b/API/config/appsettings.Development.json @@ -6,10 +6,10 @@ "Logging": { "LogLevel": { "Default": "Debug", - "Microsoft": "Information", + "Microsoft": "Error", "Microsoft.Hosting.Lifetime": "Error", - "Hangfire": "Information", - "Microsoft.AspNetCore.Hosting.Internal.WebHost": "Information" + "Hangfire": "Error", + "Microsoft.AspNetCore.Hosting.Internal.WebHost": "Error" }, "File": { "Path": "config//logs/kavita.log", diff --git a/API/config/appsettings.json b/API/config/appsettings.json new file mode 100644 index 000000000..19637b881 --- /dev/null +++ b/API/config/appsettings.json @@ -0,0 +1,22 @@ +{ + "ConnectionStrings": { + "DefaultConnection": "Data source=config/kavita.db" + }, + "TokenKey": "super secret unguessable key", + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft": "Error", + "Microsoft.Hosting.Lifetime": "Error", + "Hangfire": "Error", + "Microsoft.AspNetCore.Hosting.Internal.WebHost": "Error" + }, + "File": { + "Path": "config/logs/kavita.log", + "Append": "True", + "FileSizeLimitBytes": 10485760, + "MaxRollingFiles": 1 + } + }, + "Port": 5000 +} diff --git a/UI/Web/src/app/_services/jumpbar.service.ts b/UI/Web/src/app/_services/jumpbar.service.ts index e5c22597c..43dc5ed3d 100644 --- a/UI/Web/src/app/_services/jumpbar.service.ts +++ b/UI/Web/src/app/_services/jumpbar.service.ts @@ -31,7 +31,7 @@ export class JumpbarService { const jumpBarKeysToRender: Array = []; const targetNumberOfKeys = parseInt(Math.floor(currentSize / keySize) + '', 10); const removeCount = jumpBarKeys.length - targetNumberOfKeys - 3; - if (removeCount <= 0) return jumpBarKeysToRender; + if (removeCount <= 0) return [...jumpBarKeys]; const removalTimes = Math.ceil(removeCount / 2); const midPoint = Math.floor(jumpBarKeys.length / 2); diff --git a/UI/Web/src/app/cards/card-detail-layout/card-detail-layout.component.html b/UI/Web/src/app/cards/card-detail-layout/card-detail-layout.component.html index 4839f00e8..636d71a6d 100644 --- a/UI/Web/src/app/cards/card-detail-layout/card-detail-layout.component.html +++ b/UI/Web/src/app/cards/card-detail-layout/card-detail-layout.component.html @@ -13,13 +13,13 @@ -
+

- +
@@ -29,7 +29,7 @@
- +
diff --git a/UI/Web/src/app/cards/card-detail-layout/card-detail-layout.component.ts b/UI/Web/src/app/cards/card-detail-layout/card-detail-layout.component.ts index f931879de..2326b4521 100644 --- a/UI/Web/src/app/cards/card-detail-layout/card-detail-layout.component.ts +++ b/UI/Web/src/app/cards/card-detail-layout/card-detail-layout.component.ts @@ -46,8 +46,8 @@ export class CardDetailLayoutComponent implements OnInit, OnDestroy, OnChanges { @Input() refresh!: EventEmitter; - @Input() jumpBarKeys: Array = []; // This is aprox 784 pixels wide - jumpBarKeysToRender: Array = []; // Original + @Input() jumpBarKeys: Array = []; // This is aprox 784 pixels tall, original keys + jumpBarKeysToRender: Array = []; // What is rendered on screen @Output() itemClicked: EventEmitter = new EventEmitter(); @Output() applyFilter: EventEmitter = new EventEmitter(); @@ -114,8 +114,7 @@ export class CardDetailLayoutComponent implements OnInit, OnDestroy, OnChanges { ngOnChanges(): void { this.jumpBarKeysToRender = [...this.jumpBarKeys]; this.resizeJumpBar(); - - + if (!this.hasResumedJumpKey && this.jumpBarKeysToRender.length > 0) { const resumeKey = this.jumpbarService.getResumeKey(this.router.url); if (resumeKey === '') return; @@ -156,6 +155,5 @@ export class CardDetailLayoutComponent implements OnInit, OnDestroy, OnChanges { this.virtualScroller.scrollToIndex(targetIndex, true, 0, 1000); this.jumpbarService.saveResumeKey(this.router.url, jumpKey.key); this.changeDetectionRef.markForCheck(); - return; } }