From ab540c0ea62d5149db5a455f8b6732d1a017c9fb Mon Sep 17 00:00:00 2001 From: Joe Milazzo Date: Wed, 12 Mar 2025 17:25:15 -0500 Subject: [PATCH] Scanner Fixes (#3619) Co-authored-by: Fesaa <77553571+Fesaa@users.noreply.github.com> --- API.Tests/Helpers/ScannerHelper.cs | 8 +- API.Tests/Services/ParseScannedFilesTests.cs | 106 +++++++++++- API.Tests/Services/ScannerServiceTests.cs | 153 +++++++++++++++++- ...scanning fix publisher layout - Comic.json | 8 + ...s scanning all series changes - Manga.json | 11 ++ .../Tasks/Scanner/ParseScannedFiles.cs | 50 +++++- API/Services/Tasks/Scanner/ProcessSeries.cs | 3 + API/Services/Tasks/ScannerService.cs | 20 ++- 8 files changed, 340 insertions(+), 19 deletions(-) create mode 100644 API.Tests/Services/Test Data/ScannerService/TestCases/Subfolder always scanning fix publisher layout - Comic.json create mode 100644 API.Tests/Services/Test Data/ScannerService/TestCases/Subfolders always scanning all series changes - Manga.json diff --git a/API.Tests/Helpers/ScannerHelper.cs b/API.Tests/Helpers/ScannerHelper.cs index e164d015e..6abe5b01b 100644 --- a/API.Tests/Helpers/ScannerHelper.cs +++ b/API.Tests/Helpers/ScannerHelper.cs @@ -63,10 +63,10 @@ public class ScannerHelper return library; } - public ScannerService CreateServices() + public ScannerService CreateServices(DirectoryService ds = null, IFileSystem fs = null) { - var fs = new FileSystem(); - var ds = new DirectoryService(Substitute.For>(), fs); + fs ??= new FileSystem(); + ds ??= new DirectoryService(Substitute.For>(), fs); var archiveService = new ArchiveService(Substitute.For>(), ds, Substitute.For(), Substitute.For()); var readingItemService = new ReadingItemService(archiveService, Substitute.For(), @@ -133,7 +133,7 @@ public class ScannerHelper _testOutputHelper.WriteLine($"Test Directory Path: {testDirectory}"); - return testDirectory; + return Path.GetFullPath(testDirectory); } diff --git a/API.Tests/Services/ParseScannedFilesTests.cs b/API.Tests/Services/ParseScannedFilesTests.cs index ff4868a8c..b7bdaf57b 100644 --- a/API.Tests/Services/ParseScannedFilesTests.cs +++ b/API.Tests/Services/ParseScannedFilesTests.cs @@ -1,8 +1,11 @@ using System; using System.Collections.Generic; using System.Data.Common; +using System.IO; +using System.IO.Abstractions; using System.IO.Abstractions.TestingHelpers; using System.Linq; +using System.Threading; using System.Threading.Tasks; using API.Data; using API.Data.Metadata; @@ -15,13 +18,16 @@ using API.Services; using API.Services.Tasks.Scanner; using API.Services.Tasks.Scanner.Parser; using API.SignalR; +using API.Tests.Helpers; using AutoMapper; +using Hangfire; using Microsoft.Data.Sqlite; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.Extensions.Logging; using NSubstitute; using Xunit; +using Xunit.Abstractions; namespace API.Tests.Services; @@ -97,11 +103,13 @@ public class MockReadingItemService : IReadingItemService public class ParseScannedFilesTests : AbstractDbTest { private readonly ILogger _logger = Substitute.For>(); + private readonly ScannerHelper _scannerHelper; - public ParseScannedFilesTests() + public ParseScannedFilesTests(ITestOutputHelper testOutputHelper) { // Since ProcessFile relies on _readingItemService, we can implement our own versions of _readingItemService so we have control over how the calls work - + GlobalConfiguration.Configuration.UseInMemoryStorage(); + _scannerHelper = new ScannerHelper(_unitOfWork, testOutputHelper); } protected override async Task ResetDb() @@ -349,4 +357,98 @@ public class ParseScannedFilesTests : AbstractDbTest #endregion + + [Fact] + public async Task HasSeriesFolderNotChangedSinceLastScan_AllSeriesFoldersHaveChanges() + { + const string testcase = "Subfolders always scanning all series changes - Manga.json"; + var infos = new Dictionary(); + var library = await _scannerHelper.GenerateScannerData(testcase, infos); + var testDirectoryPath = library.Folders.First().Path; + + _unitOfWork.LibraryRepository.Update(library); + await _unitOfWork.CommitAsync(); + + var fs = new FileSystem(); + var ds = new DirectoryService(Substitute.For>(), fs); + var psf = new ParseScannedFiles(Substitute.For>(), ds, + new MockReadingItemService(ds, Substitute.For()), Substitute.For()); + + var scanner = _scannerHelper.CreateServices(ds, fs); + await scanner.ScanLibrary(library.Id); + + var postLib = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(library.Id, LibraryIncludes.Series); + Assert.NotNull(postLib); + Assert.Equal(4, postLib.Series.Count); + + var spiceAndWolf = postLib.Series.First(x => x.Name == "Spice and Wolf"); + Assert.Equal(2, spiceAndWolf.Volumes.Count); + + var frieren = postLib.Series.First(x => x.Name == "Frieren - Beyond Journey's End"); + Assert.Single(frieren.Volumes); + + var executionerAndHerWayOfLife = postLib.Series.First(x => x.Name == "The Executioner and Her Way of Life"); + Assert.Equal(2, executionerAndHerWayOfLife.Volumes.Count); + + Thread.Sleep(1100); // Ensure at least one second has passed since library scan + + // Add a new chapter to a volume of the series, and scan. Validate that only, and all directories of this + // series are marked as HasChanged + var executionerCopyDir = Path.Join(Path.Join(testDirectoryPath, "The Executioner and Her Way of Life"), + "The Executioner and Her Way of Life Vol. 1"); + File.Copy(Path.Join(executionerCopyDir, "The Executioner and Her Way of Life Vol. 1 Ch. 0001.cbz"), + Path.Join(executionerCopyDir, "The Executioner and Her Way of Life Vol. 1 Ch. 0002.cbz")); + + // 4 series, of which 2 have volumes as directories + var folderMap = await _unitOfWork.SeriesRepository.GetFolderPathMap(postLib.Id); + Assert.Equal(6, folderMap.Count); + + var res = await psf.ScanFiles(testDirectoryPath, true, folderMap, postLib); + var changes = res.Where(sc => sc.HasChanged).ToList(); + Assert.Equal(2, changes.Count); + // Only volumes of The Executioner and Her Way of Life should be marked as HasChanged (Spice and Wolf also has 2 volumes dirs) + Assert.Equal(2, changes.Count(sc => sc.Folder.Contains("The Executioner and Her Way of Life"))); + } + + [Fact] + public async Task HasSeriesFolderNotChangedSinceLastScan_PublisherLayout() + { + const string testcase = "Subfolder always scanning fix publisher layout - Comic.json"; + var infos = new Dictionary(); + var library = await _scannerHelper.GenerateScannerData(testcase, infos); + var testDirectoryPath = library.Folders.First().Path; + + _unitOfWork.LibraryRepository.Update(library); + await _unitOfWork.CommitAsync(); + + var fs = new FileSystem(); + var ds = new DirectoryService(Substitute.For>(), fs); + var psf = new ParseScannedFiles(Substitute.For>(), ds, + new MockReadingItemService(ds, Substitute.For()), Substitute.For()); + + var scanner = _scannerHelper.CreateServices(ds, fs); + await scanner.ScanLibrary(library.Id); + + var postLib = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(library.Id, LibraryIncludes.Series); + Assert.NotNull(postLib); + Assert.Equal(4, postLib.Series.Count); + + var spiceAndWolf = postLib.Series.First(x => x.Name == "Spice and Wolf"); + Assert.Equal(2, spiceAndWolf.Volumes.Count); + + var frieren = postLib.Series.First(x => x.Name == "Frieren - Beyond Journey's End"); + Assert.Equal(2, frieren.Volumes.Count); + + Thread.Sleep(1100); // Ensure at least one second has passed since library scan + + // Add a volume to a series, and scan. Ensure only this series is marked as HasChanged + var executionerCopyDir = Path.Join(Path.Join(testDirectoryPath, "YenPress"), "The Executioner and Her Way of Life"); + File.Copy(Path.Join(executionerCopyDir, "The Executioner and Her Way of Life Vol. 1.cbz"), + Path.Join(executionerCopyDir, "The Executioner and Her Way of Life Vol. 2.cbz")); + + var res = await psf.ScanFiles(testDirectoryPath, true, + await _unitOfWork.SeriesRepository.GetFolderPathMap(postLib.Id), postLib); + var changes = res.Count(sc => sc.HasChanged); + Assert.Equal(1, changes); + } } diff --git a/API.Tests/Services/ScannerServiceTests.cs b/API.Tests/Services/ScannerServiceTests.cs index 71e8785fa..38b32ae8d 100644 --- a/API.Tests/Services/ScannerServiceTests.cs +++ b/API.Tests/Services/ScannerServiceTests.cs @@ -6,6 +6,7 @@ using System.IO.Compression; using System.Linq; using System.Text; using System.Text.Json; +using System.Threading; using System.Threading.Tasks; using System.Xml; using System.Xml.Serialization; @@ -519,7 +520,7 @@ public class ScannerServiceTests : AbstractDbTest } [Fact] - public async Task ScanLibrary_MultipleRoots_MultipleScans_DataPersists() + public async Task ScanLibrary_MultipleRoots_MultipleScans_DataPersists_Forced() { const string testcase = "Multiple Roots - Manga.json"; @@ -552,22 +553,83 @@ public class ScannerServiceTests : AbstractDbTest var s2 = postLib.Series.First(s => s.Name == "Accel"); Assert.Single(s2.Volumes); + // Make a change (copy a file into only 1 root) + var root1PlushFolder = Path.Join(testDirectoryPath, "Root 1/Antarctic Press/Plush"); + File.Copy(Path.Join(root1PlushFolder, "Plush v02.cbz"), Path.Join(root1PlushFolder, "Plush v03.cbz")); + // Rescan to ensure nothing changes yet again await scanner.ScanLibrary(library.Id, true); postLib = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(library.Id, LibraryIncludes.Series); Assert.Equal(2, postLib.Series.Count); s = postLib.Series.First(s => s.Name == "Plush"); - Assert.Equal(2, s.Volumes.Count); + Assert.Equal(3, s.Volumes.Count); s2 = postLib.Series.First(s => s.Name == "Accel"); Assert.Single(s2.Volumes); } - //[Fact] + /// + /// Regression bug appeared where multi-root and one root gets a new file, on next scan of library, + /// the series in the other root are deleted. (This is actually failing because the file in Root 1 isn't being detected) + /// + [Fact] + public async Task ScanLibrary_MultipleRoots_MultipleScans_DataPersists_NonForced() + { + const string testcase = "Multiple Roots - Manga.json"; + + // Get the first file and generate a ComicInfo + var infos = new Dictionary(); + var library = await _scannerHelper.GenerateScannerData(testcase, infos); + + var testDirectoryPath = + Path.Join( + Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/ScannerService/ScanTests"), + testcase.Replace(".json", string.Empty)); + library.Folders = + [ + new FolderPath() {Path = Path.Join(testDirectoryPath, "Root 1")}, + new FolderPath() {Path = Path.Join(testDirectoryPath, "Root 2")} + ]; + + _unitOfWork.LibraryRepository.Update(library); + await _unitOfWork.CommitAsync(); + + + var scanner = _scannerHelper.CreateServices(); + await scanner.ScanLibrary(library.Id); + var postLib = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(library.Id, LibraryIncludes.Series); + + Assert.NotNull(postLib); + Assert.Equal(2, postLib.Series.Count); + var s = postLib.Series.First(s => s.Name == "Plush"); + Assert.Equal(2, s.Volumes.Count); + var s2 = postLib.Series.First(s => s.Name == "Accel"); + Assert.Single(s2.Volumes); + + // Make a change (copy a file into only 1 root) + var root1PlushFolder = Path.Join(testDirectoryPath, "Root 1/Antarctic Press/Plush"); + File.Copy(Path.Join(root1PlushFolder, "Plush v02.cbz"), Path.Join(root1PlushFolder, "Plush v03.cbz")); + + // Emulate time passage by updating lastFolderScan to be a min in the past + s.LastFolderScanned = DateTime.Now.Subtract(TimeSpan.FromMinutes(1)); + _context.Series.Update(s); + await _context.SaveChangesAsync(); + + // Rescan to ensure nothing changes yet again + await scanner.ScanLibrary(library.Id, false); + + postLib = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(library.Id, LibraryIncludes.Series); + Assert.Equal(2, postLib.Series.Count); + s = postLib.Series.First(s => s.Name == "Plush"); + Assert.Equal(3, s.Volumes.Count); + s2 = postLib.Series.First(s => s.Name == "Accel"); + Assert.Single(s2.Volumes); + } + + [Fact] public async Task ScanLibrary_AlternatingRemoval_IssueReplication() { // https://github.com/Kareadita/Kavita/issues/3476#issuecomment-2661635558 - // TODO: Come back to this, it's complicated const string testcase = "Alternating Removal - Manga.json"; // Setup: Generate test library @@ -602,6 +664,14 @@ public class ScannerServiceTests : AbstractDbTest _unitOfWork.LibraryRepository.Update(library); await _unitOfWork.CommitAsync(); + // Emulate time passage by updating lastFolderScan to be a min in the past + foreach (var s in postLib.Series) + { + s.LastFolderScanned = DateTime.Now.Subtract(TimeSpan.FromMinutes(1)); + _context.Series.Update(s); + } + await _context.SaveChangesAsync(); + await scanner.ScanLibrary(library.Id); postLib = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(library.Id, LibraryIncludes.Series); @@ -617,12 +687,28 @@ public class ScannerServiceTests : AbstractDbTest _unitOfWork.LibraryRepository.Update(library); await _unitOfWork.CommitAsync(); + // Emulate time passage by updating lastFolderScan to be a min in the past + foreach (var s in postLib.Series) + { + s.LastFolderScanned = DateTime.Now.Subtract(TimeSpan.FromMinutes(1)); + _context.Series.Update(s); + } + await _context.SaveChangesAsync(); + await scanner.ScanLibrary(library.Id); postLib = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(library.Id, LibraryIncludes.Series); Assert.Contains(postLib.Series, s => s.Name == "Accel"); // Accel should be back Assert.Contains(postLib.Series, s => s.Name == "Plush"); + // Emulate time passage by updating lastFolderScan to be a min in the past + foreach (var s in postLib.Series) + { + s.LastFolderScanned = DateTime.Now.Subtract(TimeSpan.FromMinutes(1)); + _context.Series.Update(s); + } + await _context.SaveChangesAsync(); + // Fourth Scan: Run again to check stability (should not remove Accel) await scanner.ScanLibrary(library.Id); postLib = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(library.Id, LibraryIncludes.Series); @@ -677,4 +763,63 @@ public class ScannerServiceTests : AbstractDbTest Assert.Contains(postLib.Series, s => s.Name == "Accel"); // Ensure Accel is gone Assert.Contains(postLib.Series, s => s.Name == "Plush"); } + + [Fact] + public async Task SubFolders_NoRemovals_ChangesFound() + { + const string testcase = "Subfolders always scanning all series changes - Manga.json"; + var infos = new Dictionary(); + var library = await _scannerHelper.GenerateScannerData(testcase, infos); + var testDirectoryPath = library.Folders.First().Path; + + _unitOfWork.LibraryRepository.Update(library); + await _unitOfWork.CommitAsync(); + + var scanner = _scannerHelper.CreateServices(); + await scanner.ScanLibrary(library.Id); + + var postLib = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(library.Id, LibraryIncludes.Series); + Assert.NotNull(postLib); + Assert.Equal(4, postLib.Series.Count); + + var spiceAndWolf = postLib.Series.First(x => x.Name == "Spice and Wolf"); + Assert.Equal(2, spiceAndWolf.Volumes.Count); + Assert.Equal(3, spiceAndWolf.Volumes.Sum(v => v.Chapters.Count)); + + var frieren = postLib.Series.First(x => x.Name == "Frieren - Beyond Journey's End"); + Assert.Single(frieren.Volumes); + Assert.Equal(2, frieren.Volumes.Sum(v => v.Chapters.Count)); + + var executionerAndHerWayOfLife = postLib.Series.First(x => x.Name == "The Executioner and Her Way of Life"); + Assert.Equal(2, executionerAndHerWayOfLife.Volumes.Count); + Assert.Equal(2, executionerAndHerWayOfLife.Volumes.Sum(v => v.Chapters.Count)); + + Thread.Sleep(1100); // Ensure at least one second has passed since library scan + + // Add a new chapter to a volume of the series, and scan. Validate that no chapters were lost, and the new + // chapter was added + var executionerCopyDir = Path.Join(Path.Join(testDirectoryPath, "The Executioner and Her Way of Life"), + "The Executioner and Her Way of Life Vol. 1"); + File.Copy(Path.Join(executionerCopyDir, "The Executioner and Her Way of Life Vol. 1 Ch. 0001.cbz"), + Path.Join(executionerCopyDir, "The Executioner and Her Way of Life Vol. 1 Ch. 0002.cbz")); + + await scanner.ScanLibrary(library.Id); + await _unitOfWork.CommitAsync(); + + postLib = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(library.Id, LibraryIncludes.Series); + Assert.NotNull(postLib); + Assert.Equal(4, postLib.Series.Count); + + spiceAndWolf = postLib.Series.First(x => x.Name == "Spice and Wolf"); + Assert.Equal(2, spiceAndWolf.Volumes.Count); + Assert.Equal(3, spiceAndWolf.Volumes.Sum(v => v.Chapters.Count)); + + frieren = postLib.Series.First(x => x.Name == "Frieren - Beyond Journey's End"); + Assert.Single(frieren.Volumes); + Assert.Equal(2, frieren.Volumes.Sum(v => v.Chapters.Count)); + + executionerAndHerWayOfLife = postLib.Series.First(x => x.Name == "The Executioner and Her Way of Life"); + Assert.Equal(2, executionerAndHerWayOfLife.Volumes.Count); + Assert.Equal(3, executionerAndHerWayOfLife.Volumes.Sum(v => v.Chapters.Count)); // Incremented by 1 + } } diff --git a/API.Tests/Services/Test Data/ScannerService/TestCases/Subfolder always scanning fix publisher layout - Comic.json b/API.Tests/Services/Test Data/ScannerService/TestCases/Subfolder always scanning fix publisher layout - Comic.json new file mode 100644 index 000000000..4574ddb4e --- /dev/null +++ b/API.Tests/Services/Test Data/ScannerService/TestCases/Subfolder always scanning fix publisher layout - Comic.json @@ -0,0 +1,8 @@ +[ + "VizMedia/Frieren - Beyond Journey's End/Frieren - Beyond Journey's End Vol. 1.cbz", + "VizMedia/Frieren - Beyond Journey's End/Frieren - Beyond Journey's End Vol. 2.cbz", + "VizMedia/Seraph of the End/Seraph of the End Vol. 1.cbz", + "YenPress/Spice and Wolf/Spice and Wolf Vol. 1.cbz", + "YenPress/Spice and Wolf/Spice and Wolf Vol. 2.cbz", + "YenPress/The Executioner and Her Way of Life/The Executioner and Her Way of Life Vol. 1.cbz" +] diff --git a/API.Tests/Services/Test Data/ScannerService/TestCases/Subfolders always scanning all series changes - Manga.json b/API.Tests/Services/Test Data/ScannerService/TestCases/Subfolders always scanning all series changes - Manga.json new file mode 100644 index 000000000..3d7c74d5c --- /dev/null +++ b/API.Tests/Services/Test Data/ScannerService/TestCases/Subfolders always scanning all series changes - Manga.json @@ -0,0 +1,11 @@ +[ + "Frieren - Beyond Journey's End/Frieren - Beyond Journey's End Vol. 1/Frieren - Beyond Journey's End Ch. 0001.cbz", + "Frieren - Beyond Journey's End/Frieren - Beyond Journey's End Vol. 1/Frieren - Beyond Journey's End Ch. 0002.cbz", + "Seraph of the End/Seraph of the End Vol. 1/Seraph of the End Ch. 0001.cbz", + "Spice and Wolf/Spice and Wolf Vol. 1/Spice and Wolf Vol. 1 Ch. 0001.cbz", + "Spice and Wolf/Spice and Wolf Vol. 1/Spice and Wolf Vol. 1 Ch. 0002.cbz", + "Spice and Wolf/Spice and Wolf Vol. 2/Spice and Wolf Vol. 2 Ch. 0003.cbz", + "The Executioner and Her Way of Life/The Executioner and Her Way of Life Vol. 1/The Executioner and Her Way of Life Vol. 1 Ch. 0001.cbz", + "The Executioner and Her Way of Life/The Executioner and Her Way of Life Vol. 2/The Executioner and Her Way of Life Vol. 2 Ch. 0003.cbz" +] + diff --git a/API/Services/Tasks/Scanner/ParseScannedFiles.cs b/API/Services/Tasks/Scanner/ParseScannedFiles.cs index ba8ab7457..d18d4a2f2 100644 --- a/API/Services/Tasks/Scanner/ParseScannedFiles.cs +++ b/API/Services/Tasks/Scanner/ParseScannedFiles.cs @@ -32,6 +32,10 @@ public class ParsedSeries /// Format of the Series /// public required MangaFormat Format { get; init; } + /// + /// Has this Series changed or not aka do we need to process it or not. + /// + public bool HasChanged { get; set; } } public class ScanResult @@ -178,7 +182,7 @@ public class ParseScannedFiles await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.FileScanProgressEvent(directory, library.Name, ProgressEventType.Updated)); - if (HasSeriesFolderNotChangedSinceLastScan(seriesPaths, directory, forceCheck)) + if (HasSeriesFolderNotChangedSinceLastScan(library, seriesPaths, directory, forceCheck)) { HandleUnchangedFolder(result, folderPath, directory); } @@ -196,15 +200,21 @@ public class ParseScannedFiles /// /// Checks against all folder paths on file if the last scanned is >= the directory's last write time, down to the second /// + /// /// /// This should be normalized /// /// - private bool HasSeriesFolderNotChangedSinceLastScan(IDictionary> seriesPaths, string directory, bool forceCheck) + private bool HasSeriesFolderNotChangedSinceLastScan(Library library, IDictionary> seriesPaths, string directory, bool forceCheck) { - // With the bottom-up approach, this can report a false positive where a nested folder will get scanned even though a parent is the series - // This can't really be avoided. This is more likely to happen on Image chapter folder library layouts. - if (forceCheck || !seriesPaths.TryGetValue(directory, out var seriesList)) + if (forceCheck) + { + return false; + } + + // TryGetSeriesList falls back to parent folders to match to seriesList + var seriesList = TryGetSeriesList(library, seriesPaths, directory); + if (seriesList == null) { return false; } @@ -222,6 +232,31 @@ public class ParseScannedFiles return true; } + private IList? TryGetSeriesList(Library library, IDictionary> seriesPaths, string directory) + { + if (seriesPaths.Count == 0) + { + return null; + } + + if (string.IsNullOrEmpty(directory)) + { + return null; + } + + if (library.Folders.Any(fp => fp.Path.Equals(directory))) + { + return null; + } + + if (seriesPaths.TryGetValue(directory, out var seriesList)) + { + return seriesList; + } + + return TryGetSeriesList(library, seriesPaths, _directoryService.GetParentDirectoryName(directory)); + } + /// /// Handles directories that haven't changed since the last scan. /// @@ -280,7 +315,7 @@ public class ParseScannedFiles await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.FileScanProgressEvent(normalizedPath, library.Name, ProgressEventType.Updated)); - if (HasSeriesFolderNotChangedSinceLastScan(seriesPaths, normalizedPath, forceCheck)) + if (HasSeriesFolderNotChangedSinceLastScan(library, seriesPaths, normalizedPath, forceCheck)) { result.Add(CreateScanResult(folderPath, libraryRoot, false, ArraySegment.Empty)); } @@ -721,7 +756,8 @@ public class ParseScannedFiles // If folder hasn't changed, generate fake ParserInfos if (!result.HasChanged) { - result.ParserInfos = seriesPaths[normalizedFolder] + // We are certain TryGetSeriesList will return a valid result here, if the series wasn't present yet. It will have been changed. + result.ParserInfos = TryGetSeriesList(library, seriesPaths, normalizedFolder)! .Select(fp => new ParserInfo { Series = fp.SeriesName, Format = fp.Format }) .ToList(); diff --git a/API/Services/Tasks/Scanner/ProcessSeries.cs b/API/Services/Tasks/Scanner/ProcessSeries.cs index 8f21c8a04..ec65e8dbd 100644 --- a/API/Services/Tasks/Scanner/ProcessSeries.cs +++ b/API/Services/Tasks/Scanner/ProcessSeries.cs @@ -865,8 +865,11 @@ public class ProcessSeries : IProcessSeries var fileInfo = _directoryService.FileSystem.FileInfo.New(info.FullFilePath); if (existingFile != null) { + // TODO: I wonder if we can simplify this force check. existingFile.Format = info.Format; + if (!forceUpdate && !_fileService.HasFileBeenModifiedSince(existingFile.FilePath, existingFile.LastModified) && existingFile.Pages != 0) return; + existingFile.Pages = _readingItemService.GetNumberOfPages(info.FullFilePath, info.Format); existingFile.Extension = fileInfo.Extension.ToLowerInvariant(); existingFile.FileName = Parser.Parser.RemoveExtensionIfSupported(existingFile.FilePath); diff --git a/API/Services/Tasks/ScannerService.cs b/API/Services/Tasks/ScannerService.cs index d22fe4e68..eb1c5dd0d 100644 --- a/API/Services/Tasks/ScannerService.cs +++ b/API/Services/Tasks/ScannerService.cs @@ -335,11 +335,21 @@ public class ScannerService : IScannerService private static Dictionary> TrackFoundSeriesAndFiles(IList seenSeries) { + // Why does this only grab things that have changed? var parsedSeries = new Dictionary>(); - foreach (var series in seenSeries.Where(s => s.ParsedInfos.Count > 0 && s.HasChanged)) + foreach (var series in seenSeries.Where(s => s.ParsedInfos.Count > 0)) // && s.HasChanged { var parsedFiles = series.ParsedInfos; - parsedSeries.Add(series.ParsedSeries, parsedFiles); + series.ParsedSeries.HasChanged = series.HasChanged; + + if (series.HasChanged) + { + parsedSeries.Add(series.ParsedSeries, parsedFiles); + } + else + { + parsedSeries.Add(series.ParsedSeries, []); + } } return parsedSeries; @@ -601,6 +611,12 @@ public class ScannerService : IScannerService foreach (var series in parsedSeries) { + if (!series.Key.HasChanged) + { + _logger.LogDebug("{Series} hasn't changed", series.Key.Name); + continue; + } + // Filter out ParserInfos where FullFilePath is empty (i.e., folder not modified) var validInfos = series.Value.Where(info => !string.IsNullOrEmpty(info.Filename)).ToList();