Cutlery/Kavita
1
0
Fork 0
mirror of https://github.com/Kareadita/Kavita.git synced 2025-07-31 14:33:50 -04:00
Code Issues Packages Projects Releases Wiki Activity

Further changes around ScanLibrary. Refactored DirectoryService search pattern to allow for greater re-usability. Fixed a bug where leftover chapters and volumes wouldn't get cleaned up when removed from disk.

Browse Source
This commit is contained in:
Joseph Milazzo 2021-02-09 09:22:26 -06:00
parent d8d01ffaf6
commit 5c913ba615
7 changed files with 1967 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1864
API.Tests/Helpers/PrivateObjectPrivateType.cs Normal file
View File

File diff suppressed because it is too large Load Diff

17
API.Tests/ParserTest.cs
View File

@ -47,7 +47,8 @@ namespace API.Tests
[InlineData("Yumekui_Merry_v01_c01[Bakayarou-Kuu].rar", "1")]
[InlineData("Yumekui-Merry_DKThias_Chapter11v2.zip", "0")]
[InlineData("Itoshi no Karin - c001-006x1 (v01) [Renzokusei Scans]", "1")]
[InlineData("Kedouin Makoto - Corpse Party Musume, Chapter 12", "0")]
[InlineData("VanDread-v01-c001[MD].zip", "1")]
public void ParseVolumeTest(string filename, string expected)
{
Assert.Equal(expected, ParseVolume(filename));
@ -95,6 +96,9 @@ namespace API.Tests
[InlineData("Tonikaku Kawaii Vol-1 (Ch 01-08)", "Tonikaku Kawaii")]
[InlineData("Tonikaku Kawaii (Ch 59-67) (Ongoing)", "Tonikaku Kawaii")]
[InlineData("7thGARDEN v01 (2016) (Digital) (danke).cbz", "7thGARDEN")]
[InlineData("Kedouin Makoto - Corpse Party Musume, Chapter 12", "Kedouin Makoto - Corpse Party Musume")]
[InlineData("Kedouin Makoto - Corpse Party Musume, Chapter 09", "Kedouin Makoto - Corpse Party Musume")]
[InlineData("Goblin Slayer Side Story - Year One 025.5", "Goblin Slayer Side Story - Year One")]
public void ParseSeriesTest(string filename, string expected)
{
Assert.Equal(expected, ParseSeries(filename));
@ -130,7 +134,10 @@ namespace API.Tests
[InlineData("Black Bullet - v4 c20.5 [batoto]", "20.5")]
[InlineData("Itoshi no Karin - c001-006x1 (v01) [Renzokusei Scans]", "1-6")]
[InlineData("APOSIMZ 040 (2020) (Digital) (danke-Empire).cbz", "40")]
[InlineData("Kedouin Makoto - Corpse Party Musume, Chapter 12", "12")]
[InlineData("Vol 1", "0")]
[InlineData("VanDread-v01-c001[MD].zip", "1")]
[InlineData("Goblin Slayer Side Story - Year One 025.5", "25.5")]
//[InlineData("[Tempus Edax Rerum] Epigraph of the Closed Curve - Chapter 6.zip", "6")]
public void ParseChaptersTest(string filename, string expected)
{
@ -278,6 +285,14 @@ namespace API.Tests
FullFilePath = filepath
});
filepath = @"E:\Manga\Corpse Party Musume\Kedouin Makoto - Corpse Party Musume, Chapter 09.cbz";
expected.Add(filepath, new ParserInfo
{
Series = "Corpse Party Musume - Coprse Party", Volumes = "0", Edition = "",
Chapters = "9", Filename = "Kedouin Makoto - Corpse Party Musume, Chapter 09.cbz", Format = MangaFormat.Archive,
FullFilePath = filepath
});

29
API.Tests/Services/DirectoryServiceTests.cs
View File

@ -1,7 +1,32 @@
namespace API.Tests.Services
using API.Interfaces;
using API.Services;
using Microsoft.Extensions.Logging;
using NSubstitute;
using Xunit;
namespace API.Tests.Services
{
public class DirectoryServiceTests
{
private readonly DirectoryService _directoryService;
private readonly ILogger<DirectoryService> _logger = Substitute.For<ILogger<DirectoryService>>();
public DirectoryServiceTests()
{
_directoryService = new DirectoryService(_logger);
}
[Fact]
public void GetFiles_Test()
{
//_directoryService.GetFiles()
}
[Fact]
public void ListDirectory_Test()
{
}
}
}

57
API.Tests/Services/ScannerServiceTests.cs
View File

@ -1,27 +1,37 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using API.Entities;
using API.Entities.Enums;
using API.Interfaces;
using API.Interfaces.Services;
using API.Parser;
using API.Services;
using Microsoft.Extensions.Logging;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using NSubstitute;
using Xunit;
using Xunit.Abstractions;
namespace API.Tests.Services
{
public class ScannerServiceTests
{
private readonly ITestOutputHelper _testOutputHelper;
private readonly ScannerService _scannerService;
private readonly ILogger<ScannerService> _logger = Substitute.For<ILogger<ScannerService>>();
private readonly IUnitOfWork _unitOfWork = Substitute.For<IUnitOfWork>();
private readonly IArchiveService _archiveService = Substitute.For<IArchiveService>();
//private readonly IDirectoryService _directoryService = Substitute.For<DirectoryService>();
private readonly IMetadataService _metadataService;
private readonly ILogger<MetadataService> _metadataLogger = Substitute.For<ILogger<MetadataService>>();
private Library _libraryMock;
public ScannerServiceTests()
public ScannerServiceTests(ITestOutputHelper testOutputHelper)
{
_scannerService = new ScannerService(_unitOfWork, _logger, _archiveService);
_testOutputHelper = testOutputHelper;
_scannerService = new ScannerService(_unitOfWork, _logger, _archiveService, _metadataService);
_metadataService= Substitute.For<MetadataService>(_unitOfWork, _metadataLogger, _archiveService);
_libraryMock = new Library()
{
Id = 1,
@ -59,6 +69,7 @@ namespace API.Tests.Services
new Series() {Id = 4, Name = "Akame Ga Kill"},
};
Assert.Equal(_libraryMock.Series.ElementAt(0).Id, ScannerService.ExistingOrDefault(_libraryMock, allSeries, "Darker Than Black").Id);
Assert.Equal(_libraryMock.Series.ElementAt(0).Id, ScannerService.ExistingOrDefault(_libraryMock, allSeries, "Darker than Black").Id);
}
[Fact]
@ -85,31 +96,23 @@ namespace API.Tests.Services
Assert.Null(ScannerService.ExistingOrDefault(_libraryMock, allSeries, "Non existing series"));
}
// [Fact]
// public void ScanLibrary_Should_Skip()
// {
//
Library lib = new Library()
[Fact]
public void Should_CreateSeries_Test()
{
Id = 1,
Name = "Darker Than Black",
Folders = new List<FolderPath>()
var allSeries = new List<Series>();
var parsedSeries = new Dictionary<string, List<ParserInfo>>();
parsedSeries.Add("Darker Than Black", new List<ParserInfo>()
{
new FolderPath()
{
Id = 1,
LastScanned = DateTime.Now,
LibraryId = 1,
Path = "E:/Manga"
}
},
LastModified = DateTime.Now
};
//
// _unitOfWork.LibraryRepository.GetLibraryForIdAsync(1).Returns(lib);
//
// _scannerService.ScanLibrary(1, false);
// }
new ParserInfo() {Chapters = "0", Filename = "Something.cbz", Format = MangaFormat.Archive, FullFilePath = "E:/Manga/Something.cbz", Series = "Darker Than Black", Volumes = "1"},
new ParserInfo() {Chapters = "0", Filename = "Something.cbz", Format = MangaFormat.Archive, FullFilePath = "E:/Manga/Something.cbz", Series = "Darker than Black", Volumes = "2"}
});
_scannerService.UpsertSeries(_libraryMock, parsedSeries, allSeries);
Assert.Equal(1, _libraryMock.Series.Count);
Assert.Equal(2, _libraryMock.Series.ElementAt(0).Volumes.Count);
_testOutputHelper.WriteLine(_libraryMock.ToString());
}
}
}

2
API/Entities/Series.cs
View File

@ -36,7 +36,7 @@ namespace API.Entities
public int Pages { get; set; }
// Relationships
public ICollection<Volume> Volumes { get; set; }
public List<Volume> Volumes { get; set; }
public Library Library { get; set; }
public int LibraryId { get; set; }
}

4
API/Services/DirectoryService.cs
View File

@ -91,7 +91,7 @@ namespace API.Services
/// <param name="root">Directory to scan</param>
/// <param name="action">Action to apply on file path</param>
/// <exception cref="ArgumentException"></exception>
public static int TraverseTreeParallelForEach(string root, Action<string> action)
public static int TraverseTreeParallelForEach(string root, Action<string> action, string searchPattern)
{
//Count of files traversed and timer for diagnostic output
var fileCount = 0;
@ -130,7 +130,7 @@ namespace API.Services
// TODO: In future, we need to take LibraryType into consideration for what extensions to allow (RAW should allow images)
// or we need to move this filtering to another area (Process)
// or we can get all files and put a check in place during Process to abandon files
files = GetFilesWithCertainExtensions(currentDir, Parser.Parser.MangaFileExtensions)
files = GetFilesWithCertainExtensions(currentDir, searchPattern)
.ToArray();
}
catch (UnauthorizedAccessException e) {

36
API/Services/ScannerService.cs
View File

@ -67,7 +67,7 @@ namespace API.Services
_scannedSeries = null;
}
[DisableConcurrentExecution(timeoutInSeconds: 120)]
[DisableConcurrentExecution(timeoutInSeconds: 360)]
public void ScanLibrary(int libraryId, bool forceUpdate)
{
_forceUpdate = forceUpdate;
@ -105,7 +105,7 @@ namespace API.Services
{
_logger.LogError(exception, $"The file {f} could not be found");
}
});
}, Parser.Parser.MangaFileExtensions);
}
catch (ArgumentException ex) {
_logger.LogError(ex, $"The directory '{folderPath.Path}' does not exist");
@ -170,17 +170,18 @@ namespace API.Services
{
try
{
// TODO: I don't need library here. It will always pull from allSeries
var mangaSeries = ExistingOrDefault(library, allSeries, seriesKey) ?? new Series
{
Name = seriesKey, // NOTE: Should I apply Title casing here
Name = seriesKey,
OriginalName = seriesKey,
NormalizedName = Parser.Parser.Normalize(seriesKey),
SortName = seriesKey,
Summary = ""
};
mangaSeries.NormalizedName = Parser.Parser.Normalize(mangaSeries.Name);
UpdateSeries(ref mangaSeries, parsedSeries[seriesKey].ToArray());
if (library.Series.Any(s => Parser.Parser.Normalize(s.Name) == mangaSeries.NormalizedName)) continue;
_logger.LogInformation($"Added series {mangaSeries.Name}");
@ -215,6 +216,20 @@ namespace API.Services
}
_logger.LogInformation($"Removed {count} series that are no longer on disk");
}
private void RemoveVolumesNotOnDisk(Series series)
{
var volumes = series.Volumes.ToList();
foreach (var volume in volumes)
{
var chapters = volume.Chapters;
if (!chapters.Any())
{
series.Volumes.Remove(volume);
//chapters.Select(c => c.Files).Any()
}
}
}
/// <summary>
@ -260,9 +275,11 @@ namespace API.Services
{
_logger.LogInformation($"Updating entries for {series.Name}. {infos.Length} related files.");
UpdateVolumes(series, infos);
series.Pages = series.Volumes.Sum(v => v.Pages);
UpdateVolumes(series, infos);
RemoveVolumesNotOnDisk(series);
series.Pages = series.Volumes.Sum(v => v.Pages);
_metadataService.UpdateMetadata(series, _forceUpdate);
_logger.LogDebug($"Created {series.Volumes.Count} volumes on {series.Name}");
}
@ -352,10 +369,11 @@ namespace API.Services
}
private void UpdateVolumes(Series series, ParserInfo[] infos)
private void UpdateVolumes(Series series, IReadOnlyCollection<ParserInfo> infos)
{
// BUG: If a volume no longer exists, it is not getting deleted.
series.Volumes ??= new List<Volume>();
_logger.LogDebug($"Updating Volumes for {series.Name}. {infos.Length} related files.");
_logger.LogDebug($"Updating Volumes for {series.Name}. {infos.Count} related files.");
var existingVolumes = _unitOfWork.SeriesRepository.GetVolumes(series.Id).ToList();
foreach (var info in infos)