Cutlery/Kavita
1
0
Fork 0
mirror of https://github.com/Kareadita/Kavita.git synced 2025-07-09 03:04:19 -04:00
Code Issues Packages Projects Releases Wiki Activity

Feature/parse scanned files tests (#934)

Browse Source 
* Fixed a bug in CleanupBookmarks where the Except was deleting all files because the path separators didn't match.

* Added unit tests for ParseScannedFiles.cs.

* Fixed some unit tests. Parser will now clear out multiple spaces in a row and replace with a single.
This commit is contained in:
Joseph Milazzo 2022-01-13 11:40:45 -08:00 committed by GitHub
parent f2bbdaaefc
commit 996c68d753
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 167 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
API.Tests/Parser/ComicParserTests.cs
View File

@ -68,8 +68,8 @@ namespace API.Tests.Parser
[InlineData("Demon 012 (Sep 1973) c2c", "Demon")]
[InlineData("Dragon Age - Until We Sleep 01 (of 03)", "Dragon Age - Until We Sleep")]
[InlineData("Green Lantern v2 017 - The Spy-Eye that doomed Green Lantern v2", "Green Lantern")]
[InlineData("Green Lantern - Circle of Fire Special - Adam Strange (2000)", "Green Lantern - Circle of Fire - Adam Strange")]
[InlineData("Identity Crisis Extra - Rags Morales Sketches (2005)", "Identity Crisis - Rags Morales Sketches")]
[InlineData("Green Lantern - Circle of Fire Special - Adam Strange (2000)", "Green Lantern - Circle of Fire - Adam Strange")]
[InlineData("Identity Crisis Extra - Rags Morales Sketches (2005)", "Identity Crisis - Rags Morales Sketches")]
[InlineData("Daredevil - t6 - 10 - (2019)", "Daredevil")]
[InlineData("Batgirl T2000 #57", "Batgirl")]
[InlineData("Teen Titans t1 001 (1966-02) (digital) (OkC.O.M.P.U.T.O.-Novus)", "Teen Titans")]

162
API.Tests/Services/ParseScannedFilesTests.cs
View File

@ -11,6 +11,7 @@ using API.Parser;
using API.Services;
using API.Services.Tasks.Scanner;
using API.SignalR;
using API.Tests.Helpers;
using AutoMapper;
using Microsoft.AspNetCore.SignalR;
using Microsoft.Data.Sqlite;
@ -24,19 +25,26 @@ namespace API.Tests.Services;
internal class MockReadingItemService : IReadingItemService
{
private readonly DefaultParser _defaultParser;
public MockReadingItemService(DefaultParser defaultParser)
{
_defaultParser = defaultParser;
}
public ComicInfo GetComicInfo(string filePath)
{
throw new System.NotImplementedException();
return null;
}
public int GetNumberOfPages(string filePath, MangaFormat format)
{
throw new System.NotImplementedException();
return 1;
}
public string GetCoverImage(string fileFilePath, string fileName, MangaFormat format)
{
throw new System.NotImplementedException();
return string.Empty;
}
public void Extract(string fileFilePath, string targetDirectory, MangaFormat format, int imageCount = 1)
@ -46,7 +54,7 @@ internal class MockReadingItemService : IReadingItemService
public ParserInfo Parse(string path, string rootPath, LibraryType type)
{
throw new System.NotImplementedException();
return _defaultParser.Parse(path, rootPath, type);
}
}
@ -145,8 +153,88 @@ public class ParseScannedFilesTests
#region GetInfosByName
[Fact]
public void GetInfosByName()
public void GetInfosByName_ShouldReturnGivenMatchingSeriesName()
{
var fileSystem = new MockFileSystem();
var ds = new DirectoryService(Substitute.For<ILogger<DirectoryService>>(), fileSystem);
var psf = new ParseScannedFiles(Substitute.For<ILogger<ParseScannedFiles>>(), ds,
new MockReadingItemService(new DefaultParser(ds)));
var infos = new List<ParserInfo>()
{
ParserInfoFactory.CreateParsedInfo("Accel World", "1", "0", "Accel World v1.cbz", false),
ParserInfoFactory.CreateParsedInfo("Accel World", "2", "0", "Accel World v2.cbz", false)
};
var parsedSeries = new Dictionary<ParsedSeries, List<ParserInfo>>
{
{
new ParsedSeries()
{
Format = MangaFormat.Archive,
Name = "Accel World",
NormalizedName = API.Parser.Parser.Normalize("Accel World")
},
infos
},
{
new ParsedSeries()
{
Format = MangaFormat.Pdf,
Name = "Accel World",
NormalizedName = API.Parser.Parser.Normalize("Accel World")
},
new List<ParserInfo>()
}
};
var series = DbFactory.Series("Accel World");
series.Format = MangaFormat.Pdf;
Assert.Empty(ParseScannedFiles.GetInfosByName(parsedSeries, series));
series.Format = MangaFormat.Archive;
Assert.Equal(2, ParseScannedFiles.GetInfosByName(parsedSeries, series).Count());
}
[Fact]
public void GetInfosByName_ShouldReturnGivenMatchingNormalizedSeriesName()
{
var fileSystem = new MockFileSystem();
var ds = new DirectoryService(Substitute.For<ILogger<DirectoryService>>(), fileSystem);
var psf = new ParseScannedFiles(Substitute.For<ILogger<ParseScannedFiles>>(), ds,
new MockReadingItemService(new DefaultParser(ds)));
var infos = new List<ParserInfo>()
{
ParserInfoFactory.CreateParsedInfo("Accel World", "1", "0", "Accel World v1.cbz", false),
ParserInfoFactory.CreateParsedInfo("Accel World", "2", "0", "Accel World v2.cbz", false)
};
var parsedSeries = new Dictionary<ParsedSeries, List<ParserInfo>>
{
{
new ParsedSeries()
{
Format = MangaFormat.Archive,
Name = "Accel World",
NormalizedName = API.Parser.Parser.Normalize("Accel World")
},
infos
},
{
new ParsedSeries()
{
Format = MangaFormat.Pdf,
Name = "Accel World",
NormalizedName = API.Parser.Parser.Normalize("Accel World")
},
new List<ParserInfo>()
}
};
var series = DbFactory.Series("accel world");
series.Format = MangaFormat.Archive;
Assert.Equal(2, ParseScannedFiles.GetInfosByName(parsedSeries, series).Count());
}
@ -155,10 +243,72 @@ public class ParseScannedFilesTests
#region MergeName
[Fact]
public void MergeName_()
public void MergeName_ShouldMergeMatchingFormatAndName()
{
var fileSystem = new MockFileSystem();
fileSystem.AddDirectory("C:/Data/");
fileSystem.AddFile("C:/Data/Accel World v1.cbz", new MockFileData(string.Empty));
fileSystem.AddFile("C:/Data/Accel World v2.cbz", new MockFileData(string.Empty));
fileSystem.AddFile("C:/Data/Accel World v2.pdf", new MockFileData(string.Empty));
var ds = new DirectoryService(Substitute.For<ILogger<DirectoryService>>(), fileSystem);
var psf = new ParseScannedFiles(Substitute.For<ILogger<ParseScannedFiles>>(), ds,
new MockReadingItemService(new DefaultParser(ds)));
psf.ScanLibrariesForSeries(LibraryType.Manga, new List<string>() {"C:/Data/"}, out _, out _);
Assert.Equal("Accel World", psf.MergeName(ParserInfoFactory.CreateParsedInfo("Accel World", "1", "0", "Accel World v1.cbz", false)));
Assert.Equal("Accel World", psf.MergeName(ParserInfoFactory.CreateParsedInfo("accel_world", "1", "0", "Accel World v1.cbz", false)));
Assert.Equal("Accel World", psf.MergeName(ParserInfoFactory.CreateParsedInfo("accelworld", "1", "0", "Accel World v1.cbz", false)));
}
[Fact]
public void MergeName_ShouldMerge_MismatchedFormatSameName()
{
var fileSystem = new MockFileSystem();
fileSystem.AddDirectory("C:/Data/");
fileSystem.AddFile("C:/Data/Accel World v1.cbz", new MockFileData(string.Empty));
fileSystem.AddFile("C:/Data/Accel World v2.cbz", new MockFileData(string.Empty));
fileSystem.AddFile("C:/Data/Accel World v2.pdf", new MockFileData(string.Empty));
var ds = new DirectoryService(Substitute.For<ILogger<DirectoryService>>(), fileSystem);
var psf = new ParseScannedFiles(Substitute.For<ILogger<ParseScannedFiles>>(), ds,
new MockReadingItemService(new DefaultParser(ds)));
psf.ScanLibrariesForSeries(LibraryType.Manga, new List<string>() {"C:/Data/"}, out _, out _);
Assert.Equal("Accel World", psf.MergeName(ParserInfoFactory.CreateParsedInfo("Accel World", "1", "0", "Accel World v1.epub", false)));
Assert.Equal("Accel World", psf.MergeName(ParserInfoFactory.CreateParsedInfo("accel_world", "1", "0", "Accel World v1.epub", false)));
}
#endregion
#region ScanLibrariesForSeries
[Fact]
public void ScanLibrariesForSeries_ShouldFindFiles()
{
var fileSystem = new MockFileSystem();
fileSystem.AddDirectory("C:/Data/");
fileSystem.AddFile("C:/Data/Accel World v1.cbz", new MockFileData(string.Empty));
fileSystem.AddFile("C:/Data/Accel World v2.cbz", new MockFileData(string.Empty));
fileSystem.AddFile("C:/Data/Accel World v2.pdf", new MockFileData(string.Empty));
fileSystem.AddFile("C:/Data/Nothing.pdf", new MockFileData(string.Empty));
var ds = new DirectoryService(Substitute.For<ILogger<DirectoryService>>(), fileSystem);
var psf = new ParseScannedFiles(Substitute.For<ILogger<ParseScannedFiles>>(), ds,
new MockReadingItemService(new DefaultParser(ds)));
var parsedSeries = psf.ScanLibrariesForSeries(LibraryType.Manga, new List<string>() {"C:/Data/"}, out _, out _);
Assert.Equal(3, parsedSeries.Values.Count);
Assert.NotEmpty(parsedSeries.Keys.Where(p => p.Format == MangaFormat.Archive && p.Name.Equals("Accel World")));
}
#endregion
}

7
API/Parser/Parser.cs
View File

@ -502,6 +502,11 @@ namespace API.Parser
MatchOptions, RegexTimeout
);
private static readonly Regex EmptySpaceRegex = new Regex(
@"(?!=.+)(\s{2,})(?!=.+)",
MatchOptions, RegexTimeout
);
public static MangaFormat ParseFormat(string filePath)
{
if (IsArchive(filePath)) return MangaFormat.Archive;
@ -841,6 +846,8 @@ namespace API.Parser
title = title.Substring(1);
}
title = EmptySpaceRegex.Replace(title, " ");
return title.Trim();
}

6
API/Services/Tasks/Scanner/ParseScannedFiles.cs
View File

@ -4,7 +4,6 @@ using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using API.Data.Metadata;
using API.Entities;
using API.Entities.Enums;
using API.Parser;
@ -151,7 +150,7 @@ namespace API.Services.Tasks.Scanner
/// same normalized name, it merges into the existing one. This is important as some manga may have a slight difference with punctuation or capitalization.
/// </summary>
/// <param name="info"></param>
/// <returns></returns>
/// <returns>Series Name to group this info into</returns>
public string MergeName(ParserInfo info)
{
var normalizedSeries = Parser.Parser.Normalize(info.Series);
@ -179,7 +178,6 @@ namespace API.Services.Tasks.Scanner
{
var sw = Stopwatch.StartNew();
totalFiles = 0;
var searchPattern = Parser.Parser.SupportedExtensions;
foreach (var folderPath in folders)
{
try
@ -194,7 +192,7 @@ namespace API.Services.Tasks.Scanner
{
_logger.LogError(exception, "The file {Filename} could not be found", f);
}
}, searchPattern, _logger);
}, Parser.Parser.SupportedExtensions, _logger);
}
catch (ArgumentException ex)
{