diff --git a/API.Tests/Parser/DefaultParserTests.cs b/API.Tests/Parser/DefaultParserTests.cs
index 61ed57aca..a658080de 100644
--- a/API.Tests/Parser/DefaultParserTests.cs
+++ b/API.Tests/Parser/DefaultParserTests.cs
@@ -239,15 +239,6 @@ public class DefaultParserTests
FullFilePath = filepath, IsSpecial = false
});
- // Note: Fallback to folder will parse Monster #8 and get Monster
- filepath = @"E:\Manga\Monster #8\Ch. 001-016 [MangaPlus] [Digital] [amit34521]\Monster #8 Ch. 001 [MangaPlus] [Digital] [amit34521]\13.jpg";
- expected.Add(filepath, new ParserInfo
- {
- Series = "Monster", Volumes = "0", Edition = "",
- Chapters = "1", Filename = "13.jpg", Format = MangaFormat.Image,
- FullFilePath = filepath, IsSpecial = false
- });
-
filepath = @"E:\Manga\Air Gear\Air Gear Omnibus v01 (2016) (Digital) (Shadowcat-Empire).cbz";
expected.Add(filepath, new ParserInfo
{
@@ -256,22 +247,6 @@ public class DefaultParserTests
FullFilePath = filepath, IsSpecial = false
});
- filepath = @"E:\Manga\Extra layer for no reason\Just Images the second\Vol19\ch186\Vol. 19 p106.gif";
- expected.Add(filepath, new ParserInfo
- {
- Series = "Just Images the second", Volumes = "19", Edition = "",
- Chapters = "186", Filename = "Vol. 19 p106.gif", Format = MangaFormat.Image,
- FullFilePath = filepath, IsSpecial = false
- });
-
- filepath = @"E:\Manga\Extra layer for no reason\Just Images the second\Blank Folder\Vol19\ch186\Vol. 19 p106.gif";
- expected.Add(filepath, new ParserInfo
- {
- Series = "Just Images the second", Volumes = "19", Edition = "",
- Chapters = "186", Filename = "Vol. 19 p106.gif", Format = MangaFormat.Image,
- FullFilePath = filepath, IsSpecial = false
- });
-
filepath = @"E:\Manga\Harrison, Kim - The Good, The Bad, and the Undead - Hollows Vol 2.5.epub";
expected.Add(filepath, new ParserInfo
{
@@ -308,6 +283,90 @@ public class DefaultParserTests
}
}
+ [Fact]
+ public void Parse_ParseInfo_Manga_ImageOnly()
+ {
+ // Images don't have root path as E:\Manga, but rather as the path of the folder
+
+ // Note: Fallback to folder will parse Monster #8 and get Monster
+ var filepath = @"E:\Manga\Monster #8\Ch. 001-016 [MangaPlus] [Digital] [amit34521]\Monster #8 Ch. 001 [MangaPlus] [Digital] [amit34521]\13.jpg";
+ var expectedInfo2 = new ParserInfo
+ {
+ Series = "Monster #8", Volumes = "0", Edition = "",
+ Chapters = "1", Filename = "13.jpg", Format = MangaFormat.Image,
+ FullFilePath = filepath, IsSpecial = false
+ };
+ var actual2 = _defaultParser.Parse(filepath, @"E:\Manga\Monster #8");
+ Assert.NotNull(actual2);
+ _testOutputHelper.WriteLine($"Validating {filepath}");
+ Assert.Equal(expectedInfo2.Format, actual2.Format);
+ _testOutputHelper.WriteLine("Format ✓");
+ Assert.Equal(expectedInfo2.Series, actual2.Series);
+ _testOutputHelper.WriteLine("Series ✓");
+ Assert.Equal(expectedInfo2.Chapters, actual2.Chapters);
+ _testOutputHelper.WriteLine("Chapters ✓");
+ Assert.Equal(expectedInfo2.Volumes, actual2.Volumes);
+ _testOutputHelper.WriteLine("Volumes ✓");
+ Assert.Equal(expectedInfo2.Edition, actual2.Edition);
+ _testOutputHelper.WriteLine("Edition ✓");
+ Assert.Equal(expectedInfo2.Filename, actual2.Filename);
+ _testOutputHelper.WriteLine("Filename ✓");
+ Assert.Equal(expectedInfo2.FullFilePath, actual2.FullFilePath);
+ _testOutputHelper.WriteLine("FullFilePath ✓");
+
+ filepath = @"E:\Manga\Extra layer for no reason\Just Images the second\Vol19\ch186\Vol. 19 p106.gif";
+ expectedInfo2 = new ParserInfo
+ {
+ Series = "Just Images the second", Volumes = "19", Edition = "",
+ Chapters = "186", Filename = "Vol. 19 p106.gif", Format = MangaFormat.Image,
+ FullFilePath = filepath, IsSpecial = false
+ };
+
+ actual2 = _defaultParser.Parse(filepath, @"E:\Manga\Extra layer for no reason\");
+ Assert.NotNull(actual2);
+ _testOutputHelper.WriteLine($"Validating {filepath}");
+ Assert.Equal(expectedInfo2.Format, actual2.Format);
+ _testOutputHelper.WriteLine("Format ✓");
+ Assert.Equal(expectedInfo2.Series, actual2.Series);
+ _testOutputHelper.WriteLine("Series ✓");
+ Assert.Equal(expectedInfo2.Chapters, actual2.Chapters);
+ _testOutputHelper.WriteLine("Chapters ✓");
+ Assert.Equal(expectedInfo2.Volumes, actual2.Volumes);
+ _testOutputHelper.WriteLine("Volumes ✓");
+ Assert.Equal(expectedInfo2.Edition, actual2.Edition);
+ _testOutputHelper.WriteLine("Edition ✓");
+ Assert.Equal(expectedInfo2.Filename, actual2.Filename);
+ _testOutputHelper.WriteLine("Filename ✓");
+ Assert.Equal(expectedInfo2.FullFilePath, actual2.FullFilePath);
+ _testOutputHelper.WriteLine("FullFilePath ✓");
+
+ filepath = @"E:\Manga\Extra layer for no reason\Just Images the second\Blank Folder\Vol19\ch186\Vol. 19 p106.gif";
+ expectedInfo2 = new ParserInfo
+ {
+ Series = "Just Images the second", Volumes = "19", Edition = "",
+ Chapters = "186", Filename = "Vol. 19 p106.gif", Format = MangaFormat.Image,
+ FullFilePath = filepath, IsSpecial = false
+ };
+
+ actual2 = _defaultParser.Parse(filepath, @"E:\Manga\Extra layer for no reason\");
+ Assert.NotNull(actual2);
+ _testOutputHelper.WriteLine($"Validating {filepath}");
+ Assert.Equal(expectedInfo2.Format, actual2.Format);
+ _testOutputHelper.WriteLine("Format ✓");
+ Assert.Equal(expectedInfo2.Series, actual2.Series);
+ _testOutputHelper.WriteLine("Series ✓");
+ Assert.Equal(expectedInfo2.Chapters, actual2.Chapters);
+ _testOutputHelper.WriteLine("Chapters ✓");
+ Assert.Equal(expectedInfo2.Volumes, actual2.Volumes);
+ _testOutputHelper.WriteLine("Volumes ✓");
+ Assert.Equal(expectedInfo2.Edition, actual2.Edition);
+ _testOutputHelper.WriteLine("Edition ✓");
+ Assert.Equal(expectedInfo2.Filename, actual2.Filename);
+ _testOutputHelper.WriteLine("Filename ✓");
+ Assert.Equal(expectedInfo2.FullFilePath, actual2.FullFilePath);
+ _testOutputHelper.WriteLine("FullFilePath ✓");
+ }
+
[Fact]
public void Parse_ParseInfo_Manga_WithSpecialsFolder()
{
diff --git a/API/Entities/Enums/LibraryType.cs b/API/Entities/Enums/LibraryType.cs
index 5f4ab1cc7..038ce7172 100644
--- a/API/Entities/Enums/LibraryType.cs
+++ b/API/Entities/Enums/LibraryType.cs
@@ -19,4 +19,9 @@ public enum LibraryType
///
[Description("Book")]
Book = 2,
+ ///
+ /// Uses a different type of grouping and parsing mechanism
+ ///
+ [Description("Image")]
+ Image = 3,
}
diff --git a/API/Entities/Library.cs b/API/Entities/Library.cs
index af15d06f6..f0674b8a9 100644
--- a/API/Entities/Library.cs
+++ b/API/Entities/Library.cs
@@ -41,6 +41,9 @@ public class Library : IEntityDate
///
/// Scrobbling requires a valid LicenseKey
public bool AllowScrobbling { get; set; } = true;
+
+
+
public DateTime Created { get; set; }
public DateTime LastModified { get; set; }
public DateTime CreatedUtc { get; set; }
diff --git a/API/Helpers/Builders/ChapterBuilder.cs b/API/Helpers/Builders/ChapterBuilder.cs
index c4d6c5785..80de027c3 100644
--- a/API/Helpers/Builders/ChapterBuilder.cs
+++ b/API/Helpers/Builders/ChapterBuilder.cs
@@ -28,8 +28,8 @@ public class ChapterBuilder : IEntityBuilder
{
var specialTreatment = info.IsSpecialInfo();
var specialTitle = specialTreatment ? info.Filename : info.Chapters;
- var builder = new ChapterBuilder(Services.Tasks.Scanner.Parser.Parser.DefaultChapter);
- return builder.WithNumber(specialTreatment ? Services.Tasks.Scanner.Parser.Parser.DefaultChapter : Services.Tasks.Scanner.Parser.Parser.MinNumberFromRange(info.Chapters) + string.Empty)
+ var builder = new ChapterBuilder(Parser.DefaultChapter);
+ return builder.WithNumber(specialTreatment ? Parser.DefaultChapter : Parser.MinNumberFromRange(info.Chapters) + string.Empty)
.WithRange(specialTreatment ? info.Filename : info.Chapters)
.WithTitle((specialTreatment && info.Format == MangaFormat.Epub)
? info.Title
diff --git a/API/Services/Tasks/Scanner/Parser/DefaultParser.cs b/API/Services/Tasks/Scanner/Parser/DefaultParser.cs
index 5ea5a1a0a..dd6328272 100644
--- a/API/Services/Tasks/Scanner/Parser/DefaultParser.cs
+++ b/API/Services/Tasks/Scanner/Parser/DefaultParser.cs
@@ -35,43 +35,38 @@ public class DefaultParser : IDefaultParser
{
var fileName = _directoryService.FileSystem.Path.GetFileNameWithoutExtension(filePath);
// TODO: Potential Bug: This will return null, but on Image libraries, if all images, we would want to include this. (we can probably remove this and have users use kavitaignore)
- if (Parser.IsCoverImage(_directoryService.FileSystem.Path.GetFileName(filePath))) return null;
+ if (type != LibraryType.Image && Parser.IsCoverImage(_directoryService.FileSystem.Path.GetFileName(filePath))) return null;
- ParserInfo ret;
-
- if (Parser.IsEpub(filePath)) // NOTE: Will this ever be called? Because we use ReadingService to handle parse
+ var ret = new ParserInfo()
{
- ret = new ParserInfo
- {
- Chapters = Parser.ParseChapter(fileName) ?? Parser.ParseComicChapter(fileName),
- Series = Parser.ParseSeries(fileName) ?? Parser.ParseComicSeries(fileName),
- Volumes = Parser.ParseVolume(fileName) ?? Parser.ParseComicVolume(fileName),
- Filename = Path.GetFileName(filePath),
- Format = Parser.ParseFormat(filePath),
- FullFilePath = filePath
- };
+ Filename = Path.GetFileName(filePath),
+ Format = Parser.ParseFormat(filePath),
+ Title = Path.GetFileNameWithoutExtension(fileName),
+ FullFilePath = filePath,
+ Series = string.Empty
+ };
+
+ // If library type is Image or this is not a cover image in a non-image library, then use dedicated parsing mechanism
+ if (type == LibraryType.Image || Parser.IsImage(filePath))
+ {
+ return ParseImage(filePath, rootPath, ret);
+ }
+
+
+ // This will be called if the epub is already parsed once then we call and merge the information, if the
+ if (Parser.IsEpub(filePath))
+ {
+ ret.Chapters = Parser.ParseChapter(fileName);
+ ret.Series = Parser.ParseSeries(fileName);
+ ret.Volumes = Parser.ParseVolume(fileName);
}
else
{
- ret = new ParserInfo
- {
- Chapters = type == LibraryType.Comic ? Parser.ParseComicChapter(fileName) : Parser.ParseChapter(fileName),
- Series = type == LibraryType.Comic ? Parser.ParseComicSeries(fileName) : Parser.ParseSeries(fileName),
- Volumes = type == LibraryType.Comic ? Parser.ParseComicVolume(fileName) : Parser.ParseVolume(fileName),
- Filename = Path.GetFileName(filePath),
- Format = Parser.ParseFormat(filePath),
- Title = Path.GetFileNameWithoutExtension(fileName),
- FullFilePath = filePath
- };
- }
-
-
- if (Parser.IsImage(filePath))
- {
- // Reset Chapters, Volumes, and Series as images are not good to parse information out of. Better to use folders.
- ret.Volumes = Parser.DefaultVolume;
- ret.Chapters = Parser.DefaultChapter;
- ret.Series = string.Empty;
+ ret.Chapters = type == LibraryType.Comic
+ ? Parser.ParseComicChapter(fileName)
+ : Parser.ParseChapter(fileName);
+ ret.Series = type == LibraryType.Comic ? Parser.ParseComicSeries(fileName) : Parser.ParseSeries(fileName);
+ ret.Volumes = type == LibraryType.Comic ? Parser.ParseComicVolume(fileName) : Parser.ParseVolume(fileName);
}
if (ret.Series == string.Empty || Parser.IsImage(filePath))
@@ -120,6 +115,23 @@ public class DefaultParser : IDefaultParser
return ret.Series == string.Empty ? null : ret;
}
+ private ParserInfo ParseImage(string filePath, string rootPath, ParserInfo ret)
+ {
+ ret.Volumes = Parser.DefaultVolume;
+ ret.Chapters = Parser.DefaultChapter;
+ // Next we need to see if the image has a folder between rootPath and filePath.
+ // if so, take that folder as a volume 0 chapter 0 special and group everything under there (if we can't parse a volume/chapter)
+ ParseFromFallbackFolders(filePath, rootPath, LibraryType.Image, ref ret);
+ if ((string.IsNullOrEmpty(ret.Chapters) || ret.Chapters == Parser.DefaultChapter) &&
+ (string.IsNullOrEmpty(ret.Volumes) || ret.Volumes == Parser.DefaultVolume))
+ {
+ ret.IsSpecial = true;
+ }
+
+ ret.Series = _directoryService.FileSystem.DirectoryInfo.New(rootPath).Name;
+ return ret;
+ }
+
///
/// Fills out by trying to parse volume, chapters, and series from folders
///
@@ -160,11 +172,11 @@ public class DefaultParser : IDefaultParser
if (!parsedVolume.Equals(Parser.DefaultVolume) || !parsedChapter.Equals(Parser.DefaultChapter))
{
- if ((string.IsNullOrEmpty(ret.Volumes) || ret.Volumes.Equals(Parser.DefaultVolume)) && !parsedVolume.Equals(Parser.DefaultVolume))
+ if ((string.IsNullOrEmpty(ret.Volumes) || ret.Volumes.Equals(Parser.DefaultVolume)) && !string.IsNullOrEmpty(parsedVolume) && !parsedVolume.Equals(Parser.DefaultVolume))
{
ret.Volumes = parsedVolume;
}
- if ((string.IsNullOrEmpty(ret.Chapters) || ret.Chapters.Equals(Parser.DefaultChapter)) && !parsedChapter.Equals(Parser.DefaultChapter))
+ if ((string.IsNullOrEmpty(ret.Chapters) || ret.Chapters.Equals(Parser.DefaultChapter)) && !string.IsNullOrEmpty(parsedChapter) && !parsedChapter.Equals(Parser.DefaultChapter))
{
ret.Chapters = parsedChapter;
}
diff --git a/API/Services/Tasks/Scanner/ProcessSeries.cs b/API/Services/Tasks/Scanner/ProcessSeries.cs
index 32492f22d..3a13e8d25 100644
--- a/API/Services/Tasks/Scanner/ProcessSeries.cs
+++ b/API/Services/Tasks/Scanner/ProcessSeries.cs
@@ -62,9 +62,6 @@ public class ProcessSeries : IProcessSeries
private IList _people;
private Dictionary _tags;
private Dictionary _collectionTags;
- private readonly object _peopleLock = new object();
- private readonly object _genreLock = new object();
- private readonly object _tagLock = new object();
public ProcessSeries(IUnitOfWork unitOfWork, ILogger logger, IEventHub eventHub,
IDirectoryService directoryService, ICacheHelper cacheHelper, IReadingItemService readingItemService,
@@ -845,23 +842,20 @@ public class ProcessSeries : IProcessSeries
///
private void UpdatePeople(IEnumerable names, PersonRole role, Action action)
{
- lock (_peopleLock)
+ var allPeopleTypeRole = _people.Where(p => p.Role == role).ToList();
+
+ foreach (var name in names)
{
- var allPeopleTypeRole = _people.Where(p => p.Role == role).ToList();
+ var normalizedName = name.ToNormalized();
+ var person = allPeopleTypeRole.Find(p =>
+ p.NormalizedName != null && p.NormalizedName.Equals(normalizedName));
- foreach (var name in names)
+ if (person == null)
{
- var normalizedName = name.ToNormalized();
- var person = allPeopleTypeRole.Find(p =>
- p.NormalizedName != null && p.NormalizedName.Equals(normalizedName));
-
- if (person == null)
- {
- person = new PersonBuilder(name, role).Build();
- _people.Add(person);
- }
- action(person);
+ person = new PersonBuilder(name, role).Build();
+ _people.Add(person);
}
+ action(person);
}
}
@@ -882,11 +876,8 @@ public class ProcessSeries : IProcessSeries
if (newTag)
{
genre = new GenreBuilder(name).Build();
- lock (_genreLock)
- {
- _genres.Add(normalizedName, genre);
- _unitOfWork.GenreRepository.Attach(genre);
- }
+ _genres.Add(normalizedName, genre);
+ _unitOfWork.GenreRepository.Attach(genre);
}
action(genre!, newTag);
@@ -911,10 +902,7 @@ public class ProcessSeries : IProcessSeries
if (tag == null)
{
tag = new TagBuilder(name).Build();
- lock (_tagLock)
- {
- _tags.Add(normalizedName, tag);
- }
+ _tags.Add(normalizedName, tag);
}
action(tag, added);
diff --git a/API/Services/Tasks/ScannerService.cs b/API/Services/Tasks/ScannerService.cs
index 6a08306df..08d1f86f4 100644
--- a/API/Services/Tasks/ScannerService.cs
+++ b/API/Services/Tasks/ScannerService.cs
@@ -4,6 +4,7 @@ using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Linq;
+using System.Threading;
using System.Threading.Tasks;
using API.Data;
using API.Data.Repositories;
@@ -85,6 +86,8 @@ public class ScannerService : IScannerService
private readonly IProcessSeries _processSeries;
private readonly IWordCountAnalyzerService _wordCountAnalyzerService;
+ private readonly SemaphoreSlim _seriesProcessingSemaphore = new SemaphoreSlim(1, 1);
+
public ScannerService(IUnitOfWork unitOfWork, ILogger logger,
IMetadataService metadataService, ICacheService cacheService, IEventHub eventHub,
IDirectoryService directoryService, IReadingItemService readingItemService,
@@ -495,10 +498,10 @@ public class ScannerService : IScannerService
var scanElapsedTime = await ScanFiles(library, libraryFolderPaths, shouldUseLibraryScan, TrackFiles, forceUpdate);
// NOTE: This runs sync after every file is scanned
- foreach (var task in processTasks)
- {
- await task();
- }
+ // foreach (var task in processTasks)
+ // {
+ // await task();
+ // }
// TODO: We might be able to do Task.WhenAll
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
@@ -566,17 +569,18 @@ public class ScannerService : IScannerService
BackgroundJob.Enqueue(() => _directoryService.ClearDirectory(_directoryService.TempDirectory));
return;
- Task TrackFiles(Tuple> parsedInfo)
+ // Responsible for transforming parsedInfo into an actual ParsedSeries then calling the actual processing of the series
+ async Task TrackFiles(Tuple> parsedInfo)
{
var skippedScan = parsedInfo.Item1;
var parsedFiles = parsedInfo.Item2;
- if (parsedFiles.Count == 0) return Task.CompletedTask;
+ if (parsedFiles.Count == 0) return;
var foundParsedSeries = new ParsedSeries()
{
Name = parsedFiles[0].Series,
NormalizedName = Scanner.Parser.Parser.Normalize(parsedFiles[0].Series),
- Format = parsedFiles[0].Format
+ Format = parsedFiles[0].Format,
};
if (skippedScan)
@@ -587,15 +591,22 @@ public class ScannerService : IScannerService
NormalizedName = Scanner.Parser.Parser.Normalize(pf.Series),
Format = pf.Format
}));
- return Task.CompletedTask;
+ return;
}
totalFiles += parsedFiles.Count;
seenSeries.Add(foundParsedSeries);
- processTasks.Add(async () => await _processSeries.ProcessSeriesAsync(parsedFiles, library, forceUpdate));
- return Task.CompletedTask;
+ await _seriesProcessingSemaphore.WaitAsync();
+ try
+ {
+ await _processSeries.ProcessSeriesAsync(parsedFiles, library, forceUpdate);
+ }
+ finally
+ {
+ _seriesProcessingSemaphore.Release();
+ }
}
}
diff --git a/openapi.json b/openapi.json
index 72c09edd8..3e8139442 100644
--- a/openapi.json
+++ b/openapi.json
@@ -7,7 +7,7 @@
"name": "GPL-3.0",
"url": "https://github.com/Kareadita/Kavita/blob/develop/LICENSE"
},
- "version": "0.7.10.9"
+ "version": "0.7.10.11"
},
"servers": [
{
@@ -2925,7 +2925,8 @@
"enum": [
0,
1,
- 2
+ 2,
+ 3
],
"type": "integer",
"format": "int32"
@@ -2936,7 +2937,8 @@
"enum": [
0,
1,
- 2
+ 2,
+ 3
],
"type": "integer",
"format": "int32"
@@ -2947,7 +2949,8 @@
"enum": [
0,
1,
- 2
+ 2,
+ 3
],
"type": "integer",
"format": "int32"
@@ -13404,7 +13407,8 @@
"enum": [
0,
1,
- 2
+ 2,
+ 3
],
"type": "integer",
"format": "int32"
@@ -13965,7 +13969,8 @@
"enum": [
0,
1,
- 2
+ 2,
+ 3
],
"type": "integer",
"description": "Library type",
@@ -15457,7 +15462,8 @@
"enum": [
0,
1,
- 2
+ 2,
+ 3
],
"type": "integer",
"format": "int32"
@@ -15555,7 +15561,8 @@
"enum": [
0,
1,
- 2
+ 2,
+ 3
],
"type": "integer",
"format": "int32"
@@ -16488,7 +16495,8 @@
"enum": [
0,
1,
- 2
+ 2,
+ 3
],
"type": "integer",
"format": "int32"
@@ -16539,7 +16547,8 @@
"enum": [
0,
1,
- 2
+ 2,
+ 3
],
"type": "integer",
"format": "int32"
@@ -18795,7 +18804,8 @@
"enum": [
0,
1,
- 2
+ 2,
+ 3
],
"type": "integer",
"format": "int32"