Cutlery/Kavita
1
0
Fork 0
mirror of https://github.com/Kareadita/Kavita.git synced 2025-07-09 03:04:19 -04:00
Code Issues Packages Projects Releases Wiki Activity

Image-only Libraries + Library Fixes (#2427)

Browse Source
This commit is contained in:
Joe Milazzo 2023-11-11 13:50:11 -06:00 committed by GitHub
parent 5963ea5f63
commit fe2b9b86bc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 195 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

109
API.Tests/Parser/DefaultParserTests.cs
View File

@ -239,15 +239,6 @@ public class DefaultParserTests
FullFilePath = filepath, IsSpecial = false
});
// Note: Fallback to folder will parse Monster #8 and get Monster
filepath = @"E:\Manga\Monster #8\Ch. 001-016 [MangaPlus] [Digital] [amit34521]\Monster #8 Ch. 001 [MangaPlus] [Digital] [amit34521]\13.jpg";
expected.Add(filepath, new ParserInfo
{
Series = "Monster", Volumes = "0", Edition = "",
Chapters = "1", Filename = "13.jpg", Format = MangaFormat.Image,
FullFilePath = filepath, IsSpecial = false
});
filepath = @"E:\Manga\Air Gear\Air Gear Omnibus v01 (2016) (Digital) (Shadowcat-Empire).cbz";
expected.Add(filepath, new ParserInfo
{
@ -256,22 +247,6 @@ public class DefaultParserTests
FullFilePath = filepath, IsSpecial = false
});
filepath = @"E:\Manga\Extra layer for no reason\Just Images the second\Vol19\ch186\Vol. 19 p106.gif";
expected.Add(filepath, new ParserInfo
{
Series = "Just Images the second", Volumes = "19", Edition = "",
Chapters = "186", Filename = "Vol. 19 p106.gif", Format = MangaFormat.Image,
FullFilePath = filepath, IsSpecial = false
});
filepath = @"E:\Manga\Extra layer for no reason\Just Images the second\Blank Folder\Vol19\ch186\Vol. 19 p106.gif";
expected.Add(filepath, new ParserInfo
{
Series = "Just Images the second", Volumes = "19", Edition = "",
Chapters = "186", Filename = "Vol. 19 p106.gif", Format = MangaFormat.Image,
FullFilePath = filepath, IsSpecial = false
});
filepath = @"E:\Manga\Harrison, Kim - The Good, The Bad, and the Undead - Hollows Vol 2.5.epub";
expected.Add(filepath, new ParserInfo
{
@ -308,6 +283,90 @@ public class DefaultParserTests
}
}
[Fact]
public void Parse_ParseInfo_Manga_ImageOnly()
{
// Images don't have root path as E:\Manga, but rather as the path of the folder
// Note: Fallback to folder will parse Monster #8 and get Monster
var filepath = @"E:\Manga\Monster #8\Ch. 001-016 [MangaPlus] [Digital] [amit34521]\Monster #8 Ch. 001 [MangaPlus] [Digital] [amit34521]\13.jpg";
var expectedInfo2 = new ParserInfo
{
Series = "Monster #8", Volumes = "0", Edition = "",
Chapters = "1", Filename = "13.jpg", Format = MangaFormat.Image,
FullFilePath = filepath, IsSpecial = false
};
var actual2 = _defaultParser.Parse(filepath, @"E:\Manga\Monster #8");
Assert.NotNull(actual2);
_testOutputHelper.WriteLine($"Validating {filepath}");
Assert.Equal(expectedInfo2.Format, actual2.Format);
_testOutputHelper.WriteLine("Format ✓");
Assert.Equal(expectedInfo2.Series, actual2.Series);
_testOutputHelper.WriteLine("Series ✓");
Assert.Equal(expectedInfo2.Chapters, actual2.Chapters);
_testOutputHelper.WriteLine("Chapters ✓");
Assert.Equal(expectedInfo2.Volumes, actual2.Volumes);
_testOutputHelper.WriteLine("Volumes ✓");
Assert.Equal(expectedInfo2.Edition, actual2.Edition);
_testOutputHelper.WriteLine("Edition ✓");
Assert.Equal(expectedInfo2.Filename, actual2.Filename);
_testOutputHelper.WriteLine("Filename ✓");
Assert.Equal(expectedInfo2.FullFilePath, actual2.FullFilePath);
_testOutputHelper.WriteLine("FullFilePath ✓");
filepath = @"E:\Manga\Extra layer for no reason\Just Images the second\Vol19\ch186\Vol. 19 p106.gif";
expectedInfo2 = new ParserInfo
{
Series = "Just Images the second", Volumes = "19", Edition = "",
Chapters = "186", Filename = "Vol. 19 p106.gif", Format = MangaFormat.Image,
FullFilePath = filepath, IsSpecial = false
};
actual2 = _defaultParser.Parse(filepath, @"E:\Manga\Extra layer for no reason\");
Assert.NotNull(actual2);
_testOutputHelper.WriteLine($"Validating {filepath}");
Assert.Equal(expectedInfo2.Format, actual2.Format);
_testOutputHelper.WriteLine("Format ✓");
Assert.Equal(expectedInfo2.Series, actual2.Series);
_testOutputHelper.WriteLine("Series ✓");
Assert.Equal(expectedInfo2.Chapters, actual2.Chapters);
_testOutputHelper.WriteLine("Chapters ✓");
Assert.Equal(expectedInfo2.Volumes, actual2.Volumes);
_testOutputHelper.WriteLine("Volumes ✓");
Assert.Equal(expectedInfo2.Edition, actual2.Edition);
_testOutputHelper.WriteLine("Edition ✓");
Assert.Equal(expectedInfo2.Filename, actual2.Filename);
_testOutputHelper.WriteLine("Filename ✓");
Assert.Equal(expectedInfo2.FullFilePath, actual2.FullFilePath);
_testOutputHelper.WriteLine("FullFilePath ✓");
filepath = @"E:\Manga\Extra layer for no reason\Just Images the second\Blank Folder\Vol19\ch186\Vol. 19 p106.gif";
expectedInfo2 = new ParserInfo
{
Series = "Just Images the second", Volumes = "19", Edition = "",
Chapters = "186", Filename = "Vol. 19 p106.gif", Format = MangaFormat.Image,
FullFilePath = filepath, IsSpecial = false
};
actual2 = _defaultParser.Parse(filepath, @"E:\Manga\Extra layer for no reason\");
Assert.NotNull(actual2);
_testOutputHelper.WriteLine($"Validating {filepath}");
Assert.Equal(expectedInfo2.Format, actual2.Format);
_testOutputHelper.WriteLine("Format ✓");
Assert.Equal(expectedInfo2.Series, actual2.Series);
_testOutputHelper.WriteLine("Series ✓");
Assert.Equal(expectedInfo2.Chapters, actual2.Chapters);
_testOutputHelper.WriteLine("Chapters ✓");
Assert.Equal(expectedInfo2.Volumes, actual2.Volumes);
_testOutputHelper.WriteLine("Volumes ✓");
Assert.Equal(expectedInfo2.Edition, actual2.Edition);
_testOutputHelper.WriteLine("Edition ✓");
Assert.Equal(expectedInfo2.Filename, actual2.Filename);
_testOutputHelper.WriteLine("Filename ✓");
Assert.Equal(expectedInfo2.FullFilePath, actual2.FullFilePath);
_testOutputHelper.WriteLine("FullFilePath ✓");
}
[Fact]
public void Parse_ParseInfo_Manga_WithSpecialsFolder()
{

5
API/Entities/Enums/LibraryType.cs
View File

@ -19,4 +19,9 @@ public enum LibraryType
/// </summary>
[Description("Book")]
Book = 2,
/// <summary>
/// Uses a different type of grouping and parsing mechanism
/// </summary>
[Description("Image")]
Image = 3,
}

3
API/Entities/Library.cs
View File

@ -41,6 +41,9 @@ public class Library : IEntityDate
/// </summary>
/// <remarks>Scrobbling requires a valid LicenseKey</remarks>
public bool AllowScrobbling { get; set; } = true;
public DateTime Created { get; set; }
public DateTime LastModified { get; set; }
public DateTime CreatedUtc { get; set; }

4
API/Helpers/Builders/ChapterBuilder.cs
View File

@ -28,8 +28,8 @@ public class ChapterBuilder : IEntityBuilder<Chapter>
{
var specialTreatment = info.IsSpecialInfo();
var specialTitle = specialTreatment ? info.Filename : info.Chapters;
var builder = new ChapterBuilder(Services.Tasks.Scanner.Parser.Parser.DefaultChapter);
return builder.WithNumber(specialTreatment ? Services.Tasks.Scanner.Parser.Parser.DefaultChapter : Services.Tasks.Scanner.Parser.Parser.MinNumberFromRange(info.Chapters) + string.Empty)
var builder = new ChapterBuilder(Parser.DefaultChapter);
return builder.WithNumber(specialTreatment ? Parser.DefaultChapter : Parser.MinNumberFromRange(info.Chapters) + string.Empty)
.WithRange(specialTreatment ? info.Filename : info.Chapters)
.WithTitle((specialTreatment && info.Format == MangaFormat.Epub)
? info.Title

80
API/Services/Tasks/Scanner/Parser/DefaultParser.cs
View File

@ -35,43 +35,38 @@ public class DefaultParser : IDefaultParser
{
var fileName = _directoryService.FileSystem.Path.GetFileNameWithoutExtension(filePath);
// TODO: Potential Bug: This will return null, but on Image libraries, if all images, we would want to include this. (we can probably remove this and have users use kavitaignore)
if (Parser.IsCoverImage(_directoryService.FileSystem.Path.GetFileName(filePath))) return null;
if (type != LibraryType.Image && Parser.IsCoverImage(_directoryService.FileSystem.Path.GetFileName(filePath))) return null;
ParserInfo ret;
if (Parser.IsEpub(filePath)) // NOTE: Will this ever be called? Because we use ReadingService to handle parse
var ret = new ParserInfo()
{
ret = new ParserInfo
{
Chapters = Parser.ParseChapter(fileName) ?? Parser.ParseComicChapter(fileName),
Series = Parser.ParseSeries(fileName) ?? Parser.ParseComicSeries(fileName),
Volumes = Parser.ParseVolume(fileName) ?? Parser.ParseComicVolume(fileName),
Filename = Path.GetFileName(filePath),
Format = Parser.ParseFormat(filePath),
FullFilePath = filePath
};
Filename = Path.GetFileName(filePath),
Format = Parser.ParseFormat(filePath),
Title = Path.GetFileNameWithoutExtension(fileName),
FullFilePath = filePath,
Series = string.Empty
};
// If library type is Image or this is not a cover image in a non-image library, then use dedicated parsing mechanism
if (type == LibraryType.Image || Parser.IsImage(filePath))
{
return ParseImage(filePath, rootPath, ret);
}
// This will be called if the epub is already parsed once then we call and merge the information, if the
if (Parser.IsEpub(filePath))
{
ret.Chapters = Parser.ParseChapter(fileName);
ret.Series = Parser.ParseSeries(fileName);
ret.Volumes = Parser.ParseVolume(fileName);
}
else
{
ret = new ParserInfo
{
Chapters = type == LibraryType.Comic ? Parser.ParseComicChapter(fileName) : Parser.ParseChapter(fileName),
Series = type == LibraryType.Comic ? Parser.ParseComicSeries(fileName) : Parser.ParseSeries(fileName),
Volumes = type == LibraryType.Comic ? Parser.ParseComicVolume(fileName) : Parser.ParseVolume(fileName),
Filename = Path.GetFileName(filePath),
Format = Parser.ParseFormat(filePath),
Title = Path.GetFileNameWithoutExtension(fileName),
FullFilePath = filePath
};
}
if (Parser.IsImage(filePath))
{
// Reset Chapters, Volumes, and Series as images are not good to parse information out of. Better to use folders.
ret.Volumes = Parser.DefaultVolume;
ret.Chapters = Parser.DefaultChapter;
ret.Series = string.Empty;
ret.Chapters = type == LibraryType.Comic
? Parser.ParseComicChapter(fileName)
: Parser.ParseChapter(fileName);
ret.Series = type == LibraryType.Comic ? Parser.ParseComicSeries(fileName) : Parser.ParseSeries(fileName);
ret.Volumes = type == LibraryType.Comic ? Parser.ParseComicVolume(fileName) : Parser.ParseVolume(fileName);
}
if (ret.Series == string.Empty || Parser.IsImage(filePath))
@ -120,6 +115,23 @@ public class DefaultParser : IDefaultParser
return ret.Series == string.Empty ? null : ret;
}
private ParserInfo ParseImage(string filePath, string rootPath, ParserInfo ret)
{
ret.Volumes = Parser.DefaultVolume;
ret.Chapters = Parser.DefaultChapter;
// Next we need to see if the image has a folder between rootPath and filePath.
// if so, take that folder as a volume 0 chapter 0 special and group everything under there (if we can't parse a volume/chapter)
ParseFromFallbackFolders(filePath, rootPath, LibraryType.Image, ref ret);
if ((string.IsNullOrEmpty(ret.Chapters) || ret.Chapters == Parser.DefaultChapter) &&
(string.IsNullOrEmpty(ret.Volumes) || ret.Volumes == Parser.DefaultVolume))
{
ret.IsSpecial = true;
}
ret.Series = _directoryService.FileSystem.DirectoryInfo.New(rootPath).Name;
return ret;
}
/// <summary>
/// Fills out <see cref="ParserInfo"/> by trying to parse volume, chapters, and series from folders
/// </summary>
@ -160,11 +172,11 @@ public class DefaultParser : IDefaultParser
if (!parsedVolume.Equals(Parser.DefaultVolume) || !parsedChapter.Equals(Parser.DefaultChapter))
{
if ((string.IsNullOrEmpty(ret.Volumes) || ret.Volumes.Equals(Parser.DefaultVolume)) && !parsedVolume.Equals(Parser.DefaultVolume))
if ((string.IsNullOrEmpty(ret.Volumes) || ret.Volumes.Equals(Parser.DefaultVolume)) && !string.IsNullOrEmpty(parsedVolume) && !parsedVolume.Equals(Parser.DefaultVolume))
{
ret.Volumes = parsedVolume;
}
if ((string.IsNullOrEmpty(ret.Chapters) || ret.Chapters.Equals(Parser.DefaultChapter)) && !parsedChapter.Equals(Parser.DefaultChapter))
if ((string.IsNullOrEmpty(ret.Chapters) || ret.Chapters.Equals(Parser.DefaultChapter)) && !string.IsNullOrEmpty(parsedChapter) && !parsedChapter.Equals(Parser.DefaultChapter))
{
ret.Chapters = parsedChapter;
}

38
API/Services/Tasks/Scanner/ProcessSeries.cs
View File

@ -62,9 +62,6 @@ public class ProcessSeries : IProcessSeries
private IList<Person> _people;
private Dictionary<string, Tag> _tags;
private Dictionary<string, CollectionTag> _collectionTags;
private readonly object _peopleLock = new object();
private readonly object _genreLock = new object();
private readonly object _tagLock = new object();
public ProcessSeries(IUnitOfWork unitOfWork, ILogger<ProcessSeries> logger, IEventHub eventHub,
IDirectoryService directoryService, ICacheHelper cacheHelper, IReadingItemService readingItemService,
@ -845,23 +842,20 @@ public class ProcessSeries : IProcessSeries
/// <param name="action"></param>
private void UpdatePeople(IEnumerable<string> names, PersonRole role, Action<Person> action)
{
lock (_peopleLock)
var allPeopleTypeRole = _people.Where(p => p.Role == role).ToList();
foreach (var name in names)
{
var allPeopleTypeRole = _people.Where(p => p.Role == role).ToList();
var normalizedName = name.ToNormalized();
var person = allPeopleTypeRole.Find(p =>
p.NormalizedName != null && p.NormalizedName.Equals(normalizedName));
foreach (var name in names)
if (person == null)
{
var normalizedName = name.ToNormalized();
var person = allPeopleTypeRole.Find(p =>
p.NormalizedName != null && p.NormalizedName.Equals(normalizedName));
if (person == null)
{
person = new PersonBuilder(name, role).Build();
_people.Add(person);
}
action(person);
person = new PersonBuilder(name, role).Build();
_people.Add(person);
}
action(person);
}
}
@ -882,11 +876,8 @@ public class ProcessSeries : IProcessSeries
if (newTag)
{
genre = new GenreBuilder(name).Build();
lock (_genreLock)
{
_genres.Add(normalizedName, genre);
_unitOfWork.GenreRepository.Attach(genre);
}
_genres.Add(normalizedName, genre);
_unitOfWork.GenreRepository.Attach(genre);
}
action(genre!, newTag);
@ -911,10 +902,7 @@ public class ProcessSeries : IProcessSeries
if (tag == null)
{
tag = new TagBuilder(name).Build();
lock (_tagLock)
{
_tags.Add(normalizedName, tag);
}
_tags.Add(normalizedName, tag);
}
action(tag, added);

31
API/Services/Tasks/ScannerService.cs
View File

@ -4,6 +4,7 @@ using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using API.Data;
using API.Data.Repositories;
@ -85,6 +86,8 @@ public class ScannerService : IScannerService
private readonly IProcessSeries _processSeries;
private readonly IWordCountAnalyzerService _wordCountAnalyzerService;
private readonly SemaphoreSlim _seriesProcessingSemaphore = new SemaphoreSlim(1, 1);
public ScannerService(IUnitOfWork unitOfWork, ILogger<ScannerService> logger,
IMetadataService metadataService, ICacheService cacheService, IEventHub eventHub,
IDirectoryService directoryService, IReadingItemService readingItemService,
@ -495,10 +498,10 @@ public class ScannerService : IScannerService
var scanElapsedTime = await ScanFiles(library, libraryFolderPaths, shouldUseLibraryScan, TrackFiles, forceUpdate);
// NOTE: This runs sync after every file is scanned
foreach (var task in processTasks)
{
await task();
}
// foreach (var task in processTasks)
// {
// await task();
// }
// TODO: We might be able to do Task.WhenAll
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
@ -566,17 +569,18 @@ public class ScannerService : IScannerService
BackgroundJob.Enqueue(() => _directoryService.ClearDirectory(_directoryService.TempDirectory));
return;
Task TrackFiles(Tuple<bool, IList<ParserInfo>> parsedInfo)
// Responsible for transforming parsedInfo into an actual ParsedSeries then calling the actual processing of the series
async Task TrackFiles(Tuple<bool, IList<ParserInfo>> parsedInfo)
{
var skippedScan = parsedInfo.Item1;
var parsedFiles = parsedInfo.Item2;
if (parsedFiles.Count == 0) return Task.CompletedTask;
if (parsedFiles.Count == 0) return;
var foundParsedSeries = new ParsedSeries()
{
Name = parsedFiles[0].Series,
NormalizedName = Scanner.Parser.Parser.Normalize(parsedFiles[0].Series),
Format = parsedFiles[0].Format
Format = parsedFiles[0].Format,
};
if (skippedScan)
@ -587,15 +591,22 @@ public class ScannerService : IScannerService
NormalizedName = Scanner.Parser.Parser.Normalize(pf.Series),
Format = pf.Format
}));
return Task.CompletedTask;
return;
}
totalFiles += parsedFiles.Count;
seenSeries.Add(foundParsedSeries);
processTasks.Add(async () => await _processSeries.ProcessSeriesAsync(parsedFiles, library, forceUpdate));
return Task.CompletedTask;
await _seriesProcessingSemaphore.WaitAsync();
try
{
await _processSeries.ProcessSeriesAsync(parsedFiles, library, forceUpdate);
}
finally
{
_seriesProcessingSemaphore.Release();
}
}
}

32
openapi.json
View File

@ -7,7 +7,7 @@
"name": "GPL-3.0",
"url": "https://github.com/Kareadita/Kavita/blob/develop/LICENSE"
},
"version": "0.7.10.9"
"version": "0.7.10.11"
},
"servers": [
{
@ -2925,7 +2925,8 @@
"enum": [
0,
1,
2
2,
3
],
"type": "integer",
"format": "int32"
@ -2936,7 +2937,8 @@
"enum": [
0,
1,
2
2,
3
],
"type": "integer",
"format": "int32"
@ -2947,7 +2949,8 @@
"enum": [
0,
1,
2
2,
3
],
"type": "integer",
"format": "int32"
@ -13404,7 +13407,8 @@
"enum": [
0,
1,
2
2,
3
],
"type": "integer",
"format": "int32"
@ -13965,7 +13969,8 @@
"enum": [
0,
1,
2
2,
3
],
"type": "integer",
"description": "Library type",
@ -15457,7 +15462,8 @@
"enum": [
0,
1,
2
2,
3
],
"type": "integer",
"format": "int32"
@ -15555,7 +15561,8 @@
"enum": [
0,
1,
2
2,
3
],
"type": "integer",
"format": "int32"
@ -16488,7 +16495,8 @@
"enum": [
0,
1,
2
2,
3
],
"type": "integer",
"format": "int32"
@ -16539,7 +16547,8 @@
"enum": [
0,
1,
2
2,
3
],
"type": "integer",
"format": "int32"
@ -18795,7 +18804,8 @@
"enum": [
0,
1,
2
2,
3
],
"type": "integer",
"format": "int32"