Cutlery/Kavita
1
0
Fork 0
mirror of https://github.com/Kareadita/Kavita.git synced 2025-07-09 03:04:19 -04:00
Code Issues Packages Projects Releases Wiki Activity

Scanner Performance Improvements (#1774)

Browse Source 
* Refactored the Genre code to be faster and used a dictonary to avoid some lookups. May fix the rare foreign constraint issue.

* Refactored tag to the same implementation as Genre. Ensure when grabbing tags from ComicInfo, we normalize and throw out duplicates.

* Removed an internal "external" field that was planned for Genres and Tags, but now with new plugin architecture, not needed.
This commit is contained in:
Joe Milazzo 2023-02-03 04:52:51 -08:00 committed by GitHub
parent 48aebfc3c2
commit 8a0a2f0961
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 1925 additions and 152 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
API.Tests/Helpers/GenreHelperTests.cs
View File

@ -13,13 +13,13 @@ public class GenreHelperTests
{ {
var allGenres = new List<Genre> var allGenres = new List<Genre>
{ {
DbFactory.Genre("Action", false), DbFactory.Genre("Action"),
DbFactory.Genre("action", false), DbFactory.Genre("action"),
DbFactory.Genre("Sci-fi", false), DbFactory.Genre("Sci-fi"),
}; };
var genreAdded = new List<Genre>(); var genreAdded = new List<Genre>();
GenreHelper.UpdateGenre(allGenres, new[] {"Action", "Adventure"}, false, genre => GenreHelper.UpdateGenre(allGenres, new[] {"Action", "Adventure"}, genre =>
{ {
genreAdded.Add(genre); genreAdded.Add(genre);
}); });
@ -33,19 +33,20 @@ public class GenreHelperTests
{ {
var allGenres = new List<Genre> var allGenres = new List<Genre>
{ {
DbFactory.Genre("Action", false), DbFactory.Genre("Action"),
DbFactory.Genre("action", false), DbFactory.Genre("action"),
DbFactory.Genre("Sci-fi", false), DbFactory.Genre("Sci-fi"),
}; };
var genreAdded = new List<Genre>(); var genreAdded = new List<Genre>();
GenreHelper.UpdateGenre(allGenres, new[] {"Action", "Scifi"}, false, genre => GenreHelper.UpdateGenre(allGenres, new[] {"Action", "Scifi"}, genre =>
{ {
genreAdded.Add(genre); genreAdded.Add(genre);
}); });
Assert.Equal(3, allGenres.Count); Assert.Equal(3, allGenres.Count);
Assert.Equal(2, genreAdded.Count);
} }
[Fact] [Fact]
@ -53,49 +54,34 @@ public class GenreHelperTests
{ {
var existingGenres = new List<Genre> var existingGenres = new List<Genre>
{ {
DbFactory.Genre("Action", false), DbFactory.Genre("Action"),
DbFactory.Genre("action", false), DbFactory.Genre("action"),
DbFactory.Genre("Sci-fi", false), DbFactory.Genre("Sci-fi"),
}; };
GenreHelper.AddGenreIfNotExists(existingGenres, DbFactory.Genre("Action", false)); GenreHelper.AddGenreIfNotExists(existingGenres, DbFactory.Genre("Action"));
Assert.Equal(3, existingGenres.Count); Assert.Equal(3, existingGenres.Count);
GenreHelper.AddGenreIfNotExists(existingGenres, DbFactory.Genre("action", false)); GenreHelper.AddGenreIfNotExists(existingGenres, DbFactory.Genre("action"));
Assert.Equal(3, existingGenres.Count); Assert.Equal(3, existingGenres.Count);
GenreHelper.AddGenreIfNotExists(existingGenres, DbFactory.Genre("Shonen", false)); GenreHelper.AddGenreIfNotExists(existingGenres, DbFactory.Genre("Shonen"));
Assert.Equal(4, existingGenres.Count); Assert.Equal(4, existingGenres.Count);
} }
[Fact]
public void AddGenre_ShouldNotAddSameNameAndExternal()
{
var existingGenres = new List<Genre>
{
DbFactory.Genre("Action", false),
DbFactory.Genre("action", false),
DbFactory.Genre("Sci-fi", false),
};
GenreHelper.AddGenreIfNotExists(existingGenres, DbFactory.Genre("Action", true));
Assert.Equal(3, existingGenres.Count);
}
[Fact] [Fact]
public void KeepOnlySamePeopleBetweenLists() public void KeepOnlySamePeopleBetweenLists()
{ {
var existingGenres = new List<Genre> var existingGenres = new List<Genre>
{ {
DbFactory.Genre("Action", false), DbFactory.Genre("Action"),
DbFactory.Genre("Sci-fi", false), DbFactory.Genre("Sci-fi"),
}; };
var peopleFromChapters = new List<Genre> var peopleFromChapters = new List<Genre>
{ {
DbFactory.Genre("Action", false), DbFactory.Genre("Action"),
}; };
var genreRemoved = new List<Genre>(); var genreRemoved = new List<Genre>();
@ -113,8 +99,8 @@ public class GenreHelperTests
{ {
var existingGenres = new List<Genre> var existingGenres = new List<Genre>
{ {
DbFactory.Genre("Action", false), DbFactory.Genre("Action"),
DbFactory.Genre("Sci-fi", false), DbFactory.Genre("Sci-fi"),
}; };
var peopleFromChapters = new List<Genre>(); var peopleFromChapters = new List<Genre>();

53
API.Tests/Helpers/TagHelperTests.cs
View File

@ -13,13 +13,13 @@ public class TagHelperTests
{ {
var allTags = new List<Tag> var allTags = new List<Tag>
{ {
DbFactory.Tag("Action", false), DbFactory.Tag("Action"),
DbFactory.Tag("action", false), DbFactory.Tag("action"),
DbFactory.Tag("Sci-fi", false), DbFactory.Tag("Sci-fi"),
}; };
var tagAdded = new List<Tag>(); var tagAdded = new List<Tag>();
TagHelper.UpdateTag(allTags, new[] {"Action", "Adventure"}, false, (tag, added) => TagHelper.UpdateTag(allTags, new[] {"Action", "Adventure"}, (tag, added) =>
{ {
if (added) if (added)
{ {
@ -37,14 +37,14 @@ public class TagHelperTests
{ {
var allTags = new List<Tag> var allTags = new List<Tag>
{ {
DbFactory.Tag("Action", false), DbFactory.Tag("Action"),
DbFactory.Tag("action", false), DbFactory.Tag("action"),
DbFactory.Tag("Sci-fi", false), DbFactory.Tag("Sci-fi"),
}; };
var tagAdded = new List<Tag>(); var tagAdded = new List<Tag>();
TagHelper.UpdateTag(allTags, new[] {"Action", "Scifi"}, false, (tag, added) => TagHelper.UpdateTag(allTags, new[] {"Action", "Scifi"}, (tag, added) =>
{ {
if (added) if (added)
{ {
@ -62,49 +62,34 @@ public class TagHelperTests
{ {
var existingTags = new List<Tag> var existingTags = new List<Tag>
{ {
DbFactory.Tag("Action", false), DbFactory.Tag("Action"),
DbFactory.Tag("action", false), DbFactory.Tag("action"),
DbFactory.Tag("Sci-fi", false), DbFactory.Tag("Sci-fi"),
}; };
TagHelper.AddTagIfNotExists(existingTags, DbFactory.Tag("Action", false)); TagHelper.AddTagIfNotExists(existingTags, DbFactory.Tag("Action"));
Assert.Equal(3, existingTags.Count); Assert.Equal(3, existingTags.Count);
TagHelper.AddTagIfNotExists(existingTags, DbFactory.Tag("action", false)); TagHelper.AddTagIfNotExists(existingTags, DbFactory.Tag("action"));
Assert.Equal(3, existingTags.Count); Assert.Equal(3, existingTags.Count);
TagHelper.AddTagIfNotExists(existingTags, DbFactory.Tag("Shonen", false)); TagHelper.AddTagIfNotExists(existingTags, DbFactory.Tag("Shonen"));
Assert.Equal(4, existingTags.Count); Assert.Equal(4, existingTags.Count);
} }
[Fact]
public void AddTag_ShouldNotAddSameNameAndExternal()
{
var existingTags = new List<Tag>
{
DbFactory.Tag("Action", false),
DbFactory.Tag("action", false),
DbFactory.Tag("Sci-fi", false),
};
TagHelper.AddTagIfNotExists(existingTags, DbFactory.Tag("Action", true));
Assert.Equal(3, existingTags.Count);
}
[Fact] [Fact]
public void KeepOnlySamePeopleBetweenLists() public void KeepOnlySamePeopleBetweenLists()
{ {
var existingTags = new List<Tag> var existingTags = new List<Tag>
{ {
DbFactory.Tag("Action", false), DbFactory.Tag("Action"),
DbFactory.Tag("Sci-fi", false), DbFactory.Tag("Sci-fi"),
}; };
var peopleFromChapters = new List<Tag> var peopleFromChapters = new List<Tag>
{ {
DbFactory.Tag("Action", false), DbFactory.Tag("Action"),
}; };
var tagRemoved = new List<Tag>(); var tagRemoved = new List<Tag>();
@ -122,8 +107,8 @@ public class TagHelperTests
{ {
var existingTags = new List<Tag> var existingTags = new List<Tag>
{ {
DbFactory.Tag("Action", false), DbFactory.Tag("Action"),
DbFactory.Tag("Sci-fi", false), DbFactory.Tag("Sci-fi"),
}; };
var peopleFromChapters = new List<Tag>(); var peopleFromChapters = new List<Tag>();

10
API.Tests/Services/SeriesServiceTests.cs
View File

@ -762,7 +762,7 @@ public class SeriesServiceTests : AbstractDbTest
}, },
Metadata = DbFactory.SeriesMetadata(new List<CollectionTag>()) Metadata = DbFactory.SeriesMetadata(new List<CollectionTag>())
}; };
var g = DbFactory.Genre("Existing Genre", false); var g = DbFactory.Genre("Existing Genre");
s.Metadata.Genres = new List<Genre>() {g}; s.Metadata.Genres = new List<Genre>() {g};
_context.Series.Add(s); _context.Series.Add(s);
@ -918,7 +918,7 @@ public class SeriesServiceTests : AbstractDbTest
}, },
Metadata = DbFactory.SeriesMetadata(new List<CollectionTag>()) Metadata = DbFactory.SeriesMetadata(new List<CollectionTag>())
}; };
var g = DbFactory.Genre("Existing Genre", false); var g = DbFactory.Genre("Existing Genre");
s.Metadata.Genres = new List<Genre>() {g}; s.Metadata.Genres = new List<Genre>() {g};
s.Metadata.GenresLocked = true; s.Metadata.GenresLocked = true;
_context.Series.Add(s); _context.Series.Add(s);
@ -1555,5 +1555,11 @@ public class SeriesServiceTests : AbstractDbTest
Assert.Null(await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(1)); Assert.Null(await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(1));
} }
#endregion
#region UpdateRelatedList
#endregion #endregion
} }

6
API/Data/DbFactory.cs
View File

@ -121,23 +121,21 @@ public static class DbFactory
}; };
} }
public static Genre Genre(string name, bool external) public static Genre Genre(string name)
{ {
return new Genre() return new Genre()
{ {
Title = name.Trim().SentenceCase(), Title = name.Trim().SentenceCase(),
NormalizedTitle = Services.Tasks.Scanner.Parser.Parser.Normalize(name), NormalizedTitle = Services.Tasks.Scanner.Parser.Parser.Normalize(name),
ExternalTag = external
}; };
} }
public static Tag Tag(string name, bool external) public static Tag Tag(string name)
{ {
return new Tag() return new Tag()
{ {
Title = name.Trim().SentenceCase(), Title = name.Trim().SentenceCase(),
NormalizedTitle = Services.Tasks.Scanner.Parser.Parser.Normalize(name), NormalizedTitle = Services.Tasks.Scanner.Parser.Parser.Normalize(name),
ExternalTag = external
}; };
} }

1748
API/Data/Migrations/20230203112022_RemoveExternalFromTagAndGenre.Designer.cs generated Normal file
View File

File diff suppressed because it is too large Load Diff

77
API/Data/Migrations/20230203112022_RemoveExternalFromTagAndGenre.cs Normal file
View File

@ -0,0 +1,77 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace API.Data.Migrations
{
public partial class RemoveExternalFromTagAndGenre : Migration
{
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropIndex(
name: "IX_Tag_NormalizedTitle_ExternalTag",
table: "Tag");
migrationBuilder.DropIndex(
name: "IX_Genre_NormalizedTitle_ExternalTag",
table: "Genre");
migrationBuilder.DropColumn(
name: "ExternalTag",
table: "Tag");
migrationBuilder.DropColumn(
name: "ExternalTag",
table: "Genre");
migrationBuilder.CreateIndex(
name: "IX_Tag_NormalizedTitle",
table: "Tag",
column: "NormalizedTitle",
unique: true);
migrationBuilder.CreateIndex(
name: "IX_Genre_NormalizedTitle",
table: "Genre",
column: "NormalizedTitle",
unique: true);
}
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropIndex(
name: "IX_Tag_NormalizedTitle",
table: "Tag");
migrationBuilder.DropIndex(
name: "IX_Genre_NormalizedTitle",
table: "Genre");
migrationBuilder.AddColumn<bool>(
name: "ExternalTag",
table: "Tag",
type: "INTEGER",
nullable: false,
defaultValue: false);
migrationBuilder.AddColumn<bool>(
name: "ExternalTag",
table: "Genre",
type: "INTEGER",
nullable: false,
defaultValue: false);
migrationBuilder.CreateIndex(
name: "IX_Tag_NormalizedTitle_ExternalTag",
table: "Tag",
columns: new[] { "NormalizedTitle", "ExternalTag" },
unique: true);
migrationBuilder.CreateIndex(
name: "IX_Genre_NormalizedTitle_ExternalTag",
table: "Genre",
columns: new[] { "NormalizedTitle", "ExternalTag" },
unique: true);
}
}
}

10
API/Data/Migrations/DataContextModelSnapshot.cs
View File

@ -528,9 +528,6 @@ namespace API.Data.Migrations
.ValueGeneratedOnAdd() .ValueGeneratedOnAdd()
.HasColumnType("INTEGER"); .HasColumnType("INTEGER");
b.Property<bool>("ExternalTag")
.HasColumnType("INTEGER");
b.Property<string>("NormalizedTitle") b.Property<string>("NormalizedTitle")
.HasColumnType("TEXT"); .HasColumnType("TEXT");
@ -539,7 +536,7 @@ namespace API.Data.Migrations
b.HasKey("Id"); b.HasKey("Id");
b.HasIndex("NormalizedTitle", "ExternalTag") b.HasIndex("NormalizedTitle")
.IsUnique(); .IsUnique();
b.ToTable("Genre"); b.ToTable("Genre");
@ -1036,9 +1033,6 @@ namespace API.Data.Migrations
.ValueGeneratedOnAdd() .ValueGeneratedOnAdd()
.HasColumnType("INTEGER"); .HasColumnType("INTEGER");
b.Property<bool>("ExternalTag")
.HasColumnType("INTEGER");
b.Property<string>("NormalizedTitle") b.Property<string>("NormalizedTitle")
.HasColumnType("TEXT"); .HasColumnType("TEXT");
@ -1047,7 +1041,7 @@ namespace API.Data.Migrations
b.HasKey("Id"); b.HasKey("Id");
b.HasIndex("NormalizedTitle", "ExternalTag") b.HasIndex("NormalizedTitle")
.IsUnique(); .IsUnique();
b.ToTable("Tag"); b.ToTable("Tag");

2
API/Data/Repositories/GenreRepository.cs
View File

@ -56,7 +56,7 @@ public class GenreRepository : IGenreRepository
var genresWithNoConnections = await _context.Genre var genresWithNoConnections = await _context.Genre
.Include(p => p.SeriesMetadatas) .Include(p => p.SeriesMetadatas)
.Include(p => p.Chapters) .Include(p => p.Chapters)
.Where(p => p.SeriesMetadatas.Count == 0 && p.Chapters.Count == 0 && p.ExternalTag == removeExternal) .Where(p => p.SeriesMetadatas.Count == 0 && p.Chapters.Count == 0)
.AsSplitQuery() .AsSplitQuery()
.ToListAsync(); .ToListAsync();

2
API/Data/Repositories/TagRepository.cs
View File

@ -46,7 +46,7 @@ public class TagRepository : ITagRepository
var tagsWithNoConnections = await _context.Tag var tagsWithNoConnections = await _context.Tag
.Include(p => p.SeriesMetadatas) .Include(p => p.SeriesMetadatas)
.Include(p => p.Chapters) .Include(p => p.Chapters)
.Where(p => p.SeriesMetadatas.Count == 0 && p.Chapters.Count == 0 && p.ExternalTag == removeExternal) .Where(p => p.SeriesMetadatas.Count == 0 && p.Chapters.Count == 0)
.AsSplitQuery() .AsSplitQuery()
.ToListAsync(); .ToListAsync();

3
API/Entities/Genre.cs
View File

@ -4,13 +4,12 @@ using Microsoft.EntityFrameworkCore;
namespace API.Entities; namespace API.Entities;
[Index(nameof(NormalizedTitle), nameof(ExternalTag), IsUnique = true)] [Index(nameof(NormalizedTitle), IsUnique = true)]
public class Genre public class Genre
{ {
public int Id { get; set; } public int Id { get; set; }
public string Title { get; set; } public string Title { get; set; }
public string NormalizedTitle { get; set; } public string NormalizedTitle { get; set; }
public bool ExternalTag { get; set; }
public ICollection<SeriesMetadata> SeriesMetadatas { get; set; } public ICollection<SeriesMetadata> SeriesMetadatas { get; set; }
public ICollection<Chapter> Chapters { get; set; } public ICollection<Chapter> Chapters { get; set; }

6
API/Entities/Person.cs
View File

@ -4,18 +4,12 @@ using API.Entities.Metadata;
namespace API.Entities; namespace API.Entities;
public enum ProviderSource
{
Local = 1,
External = 2
}
public class Person public class Person
{ {
public int Id { get; set; } public int Id { get; set; }
public string Name { get; set; } public string Name { get; set; }
public string NormalizedName { get; set; } public string NormalizedName { get; set; }
public PersonRole Role { get; set; } public PersonRole Role { get; set; }
//public ProviderSource Source { get; set; }
// Relationships // Relationships
public ICollection<SeriesMetadata> SeriesMetadatas { get; set; } public ICollection<SeriesMetadata> SeriesMetadatas { get; set; }

3
API/Entities/Tag.cs
View File

@ -4,13 +4,12 @@ using Microsoft.EntityFrameworkCore;
namespace API.Entities; namespace API.Entities;
[Index(nameof(NormalizedTitle), nameof(ExternalTag), IsUnique = true)] [Index(nameof(NormalizedTitle), IsUnique = true)]
public class Tag public class Tag
{ {
public int Id { get; set; } public int Id { get; set; }
public string Title { get; set; } public string Title { get; set; }
public string NormalizedTitle { get; set; } public string NormalizedTitle { get; set; }
public bool ExternalTag { get; set; }
public ICollection<SeriesMetadata> SeriesMetadatas { get; set; } public ICollection<SeriesMetadata> SeriesMetadatas { get; set; }
public ICollection<Chapter> Chapters { get; set; } public ICollection<Chapter> Chapters { get; set; }

10
API/Helpers/GenreHelper.cs
View File

@ -14,20 +14,18 @@ public static class GenreHelper
/// </summary> /// </summary>
/// <param name="allGenres"></param> /// <param name="allGenres"></param>
/// <param name="names"></param> /// <param name="names"></param>
/// <param name="isExternal"></param>
/// <param name="action"></param> /// <param name="action"></param>
public static void UpdateGenre(ICollection<Genre> allGenres, IEnumerable<string> names, bool isExternal, Action<Genre> action) public static void UpdateGenre(ICollection<Genre> allGenres, IEnumerable<string> names, Action<Genre> action)
{ {
foreach (var name in names) foreach (var name in names)
{ {
if (string.IsNullOrEmpty(name.Trim())) continue; if (string.IsNullOrEmpty(name.Trim())) continue;
var normalizedName = Services.Tasks.Scanner.Parser.Parser.Normalize(name); var normalizedName = Services.Tasks.Scanner.Parser.Parser.Normalize(name);
var genre = allGenres.FirstOrDefault(p => var genre = allGenres.FirstOrDefault(p => p.NormalizedTitle.Equals(normalizedName));
p.NormalizedTitle.Equals(normalizedName) && p.ExternalTag == isExternal);
if (genre == null) if (genre == null)
{ {
genre = DbFactory.Genre(name, false); genre = DbFactory.Genre(name);
allGenres.Add(genre); allGenres.Add(genre);
} }
@ -41,7 +39,7 @@ public static class GenreHelper
var existing = existingGenres.ToList(); var existing = existingGenres.ToList();
foreach (var genre in existing) foreach (var genre in existing)
{ {
var existingPerson = removeAllExcept.FirstOrDefault(g => g.ExternalTag == genre.ExternalTag && genre.NormalizedTitle.Equals(g.NormalizedTitle)); var existingPerson = removeAllExcept.FirstOrDefault(g => genre.NormalizedTitle.Equals(g.NormalizedTitle));
if (existingPerson != null) continue; if (existingPerson != null) continue;
existingGenres.Remove(genre); existingGenres.Remove(genre);
action?.Invoke(genre); action?.Invoke(genre);

13
API/Helpers/TagHelper.cs
View File

@ -14,9 +14,8 @@ public static class TagHelper
/// </summary> /// </summary>
/// <param name="allTags"></param> /// <param name="allTags"></param>
/// <param name="names"></param> /// <param name="names"></param>
/// <param name="isExternal"></param>
/// <param name="action">Callback for every item. Will give said item back and a bool if item was added</param> /// <param name="action">Callback for every item. Will give said item back and a bool if item was added</param>
public static void UpdateTag(ICollection<Tag> allTags, IEnumerable<string> names, bool isExternal, Action<Tag, bool> action) public static void UpdateTag(ICollection<Tag> allTags, IEnumerable<string> names, Action<Tag, bool> action)
{ {
foreach (var name in names) foreach (var name in names)
{ {
@ -26,11 +25,11 @@ public static class TagHelper
var normalizedName = Services.Tasks.Scanner.Parser.Parser.Normalize(name); var normalizedName = Services.Tasks.Scanner.Parser.Parser.Normalize(name);
var genre = allTags.FirstOrDefault(p => var genre = allTags.FirstOrDefault(p =>
p.NormalizedTitle.Equals(normalizedName) && p.ExternalTag == isExternal); p.NormalizedTitle.Equals(normalizedName));
if (genre == null) if (genre == null)
{ {
added = true; added = true;
genre = DbFactory.Tag(name, false); genre = DbFactory.Tag(name);
allTags.Add(genre); allTags.Add(genre);
} }
@ -43,7 +42,7 @@ public static class TagHelper
var existing = existingTags.ToList(); var existing = existingTags.ToList();
foreach (var genre in existing) foreach (var genre in existing)
{ {
var existingPerson = removeAllExcept.FirstOrDefault(g => g.ExternalTag == genre.ExternalTag && genre.NormalizedTitle.Equals(g.NormalizedTitle)); var existingPerson = removeAllExcept.FirstOrDefault(g => genre.NormalizedTitle.Equals(g.NormalizedTitle));
if (existingPerson != null) continue; if (existingPerson != null) continue;
existingTags.Remove(genre); existingTags.Remove(genre);
action?.Invoke(genre); action?.Invoke(genre);
@ -84,12 +83,12 @@ public static class TagHelper
/// <param name="tags">Tags from metadata</param> /// <param name="tags">Tags from metadata</param>
/// <param name="isExternal">Remove external tags?</param> /// <param name="isExternal">Remove external tags?</param>
/// <param name="action">Callback which will be executed for each tag removed</param> /// <param name="action">Callback which will be executed for each tag removed</param>
public static void RemoveTags(ICollection<Tag> existingTags, IEnumerable<string> tags, bool isExternal, Action<Tag> action = null) public static void RemoveTags(ICollection<Tag> existingTags, IEnumerable<string> tags, Action<Tag> action = null)
{ {
var normalizedTags = tags.Select(Services.Tasks.Scanner.Parser.Parser.Normalize).ToList(); var normalizedTags = tags.Select(Services.Tasks.Scanner.Parser.Parser.Normalize).ToList();
foreach (var person in normalizedTags) foreach (var person in normalizedTags)
{ {
var existingTag = existingTags.FirstOrDefault(p => p.ExternalTag == isExternal && person.Equals(p.NormalizedTitle)); var existingTag = existingTags.FirstOrDefault(p => person.Equals(p.NormalizedTitle));
if (existingTag == null) continue; if (existingTag == null) continue;
existingTags.Remove(existingTag); existingTags.Remove(existingTag);

8
API/Services/SeriesService.cs
View File

@ -115,7 +115,7 @@ public class SeriesService : ISeriesService
} }
series.Metadata.CollectionTags ??= new List<CollectionTag>(); series.Metadata.CollectionTags ??= new List<CollectionTag>();
UpdateRelatedList(updateSeriesMetadataDto.CollectionTags, series, allCollectionTags, (tag) => UpdateCollectionsList(updateSeriesMetadataDto.CollectionTags, series, allCollectionTags, (tag) =>
{ {
series.Metadata.CollectionTags.Add(tag); series.Metadata.CollectionTags.Add(tag);
}); });
@ -210,7 +210,7 @@ public class SeriesService : ISeriesService
} }
private static void UpdateRelatedList(ICollection<CollectionTagDto> tags, Series series, IReadOnlyCollection<CollectionTag> allTags, public static void UpdateCollectionsList(ICollection<CollectionTagDto> tags, Series series, IReadOnlyCollection<CollectionTag> allTags,
Action<CollectionTag> handleAdd) Action<CollectionTag> handleAdd)
{ {
// TODO: Move UpdateRelatedList to a helper so we can easily test // TODO: Move UpdateRelatedList to a helper so we can easily test
@ -278,7 +278,7 @@ public class SeriesService : ISeriesService
else else
{ {
// Add new tag // Add new tag
handleAdd(DbFactory.Genre(tagTitle, false)); handleAdd(DbFactory.Genre(tagTitle));
isModified = true; isModified = true;
} }
} }
@ -320,7 +320,7 @@ public class SeriesService : ISeriesService
else else
{ {
// Add new tag // Add new tag
handleAdd(DbFactory.Tag(tagTitle, false)); handleAdd(DbFactory.Tag(tagTitle));
isModified = true; isModified = true;
} }
} }

2
API/Services/Tasks/Scanner/Parser/Parser.cs
View File

@ -945,7 +945,7 @@ public static class Parser
public static string Normalize(string name) public static string Normalize(string name)
{ {
return NormalizeRegex.Replace(name, string.Empty).ToLower(); return NormalizeRegex.Replace(name, string.Empty).Trim().ToLower();
} }
/// <summary> /// <summary>

62
API/Services/Tasks/Scanner/ProcessSeries.cs
View File

@ -48,9 +48,9 @@ public class ProcessSeries : IProcessSeries
private readonly IWordCountAnalyzerService _wordCountAnalyzerService; private readonly IWordCountAnalyzerService _wordCountAnalyzerService;
private readonly ICollectionTagService _collectionTagService; private readonly ICollectionTagService _collectionTagService;
private IList<Genre> _genres; private Dictionary<string, Genre> _genres;
private IList<Person> _people; private IList<Person> _people;
private IList<Tag> _tags; private Dictionary<string, Tag> _tags;
private Dictionary<string, CollectionTag> _collectionTags; private Dictionary<string, CollectionTag> _collectionTags;
public ProcessSeries(IUnitOfWork unitOfWork, ILogger<ProcessSeries> logger, IEventHub eventHub, public ProcessSeries(IUnitOfWork unitOfWork, ILogger<ProcessSeries> logger, IEventHub eventHub,
@ -75,9 +75,9 @@ public class ProcessSeries : IProcessSeries
/// </summary> /// </summary>
public async Task Prime() public async Task Prime()
{ {
_genres = await _unitOfWork.GenreRepository.GetAllGenresAsync(); _genres = (await _unitOfWork.GenreRepository.GetAllGenresAsync()).ToDictionary(t => t.NormalizedTitle);
_people = await _unitOfWork.PersonRepository.GetAllPeople(); _people = await _unitOfWork.PersonRepository.GetAllPeople();
_tags = await _unitOfWork.TagRepository.GetAllTagsAsync(); _tags = (await _unitOfWork.TagRepository.GetAllTagsAsync()).ToDictionary(t => t.NormalizedTitle);
_collectionTags = (await _unitOfWork.CollectionTagRepository.GetAllTagsAsync(CollectionTagIncludes.SeriesMetadata)) _collectionTags = (await _unitOfWork.CollectionTagRepository.GetAllTagsAsync(CollectionTagIncludes.SeriesMetadata))
.ToDictionary(t => t.NormalizedTitle); .ToDictionary(t => t.NormalizedTitle);
@ -673,14 +673,14 @@ public class ProcessSeries : IProcessSeries
PersonHelper.AddPersonIfNotExists(chapter.People, person); PersonHelper.AddPersonIfNotExists(chapter.People, person);
} }
void AddGenre(Genre genre) void AddGenre(Genre genre, bool newTag)
{ {
GenreHelper.AddGenreIfNotExists(chapter.Genres, genre); chapter.Genres.Add(genre);
} }
void AddTag(Tag tag, bool added) void AddTag(Tag tag, bool added)
{ {
TagHelper.AddTagIfNotExists(chapter.Tags, tag); chapter.Tags.Add(tag);
} }
@ -745,14 +745,13 @@ public class ProcessSeries : IProcessSeries
AddPerson); AddPerson);
var genres = GetTagValues(comicInfo.Genre); var genres = GetTagValues(comicInfo.Genre);
GenreHelper.KeepOnlySameGenreBetweenLists(chapter.Genres, genres.Select(g => DbFactory.Genre(g, false)).ToList()); GenreHelper.KeepOnlySameGenreBetweenLists(chapter.Genres,
UpdateGenre(genres, false, genres.Select(DbFactory.Genre).ToList());
AddGenre); UpdateGenre(genres, AddGenre);
var tags = GetTagValues(comicInfo.Tags); var tags = GetTagValues(comicInfo.Tags);
TagHelper.KeepOnlySameTagBetweenLists(chapter.Tags, tags.Select(t => DbFactory.Tag(t, false)).ToList()); TagHelper.KeepOnlySameTagBetweenLists(chapter.Tags, tags.Select(DbFactory.Tag).ToList());
UpdateTag(tags, false, UpdateTag(tags, AddTag);
AddTag);
} }
private static IList<string> GetTagValues(string comicInfoTagSeparatedByComma) private static IList<string> GetTagValues(string comicInfoTagSeparatedByComma)
@ -760,7 +759,7 @@ public class ProcessSeries : IProcessSeries
if (!string.IsNullOrEmpty(comicInfoTagSeparatedByComma)) if (!string.IsNullOrEmpty(comicInfoTagSeparatedByComma))
{ {
return comicInfoTagSeparatedByComma.Split(",").Select(s => s.Trim()).ToList(); return comicInfoTagSeparatedByComma.Split(",").Select(s => s.Trim()).DistinctBy(s => s.Normalize()).ToList();
} }
return ImmutableList<string>.Empty; return ImmutableList<string>.Empty;
} }
@ -801,27 +800,27 @@ public class ProcessSeries : IProcessSeries
/// ///
/// </summary> /// </summary>
/// <param name="names"></param> /// <param name="names"></param>
/// <param name="isExternal"></param> /// <param name="action">Executes for each tag</param>
/// <param name="action"></param> private void UpdateGenre(IEnumerable<string> names, Action<Genre, bool> action)
private void UpdateGenre(IEnumerable<string> names, bool isExternal, Action<Genre> action)
{ {
foreach (var name in names) foreach (var name in names)
{ {
if (string.IsNullOrEmpty(name.Trim())) continue;
var normalizedName = Parser.Parser.Normalize(name); var normalizedName = Parser.Parser.Normalize(name);
var genre = _genres.FirstOrDefault(p => if (string.IsNullOrEmpty(normalizedName)) continue;
p.NormalizedTitle.Equals(normalizedName) && p.ExternalTag == isExternal);
if (genre == null) _genres.TryGetValue(normalizedName, out var genre);
var newTag = genre == null;
if (newTag)
{ {
genre = DbFactory.Genre(name, false); genre = DbFactory.Genre(name);
lock (_genres) lock (_genres)
{ {
_genres.Add(genre); _genres.Add(normalizedName, genre);
_unitOfWork.GenreRepository.Attach(genre);
} }
} }
action(genre); action(genre, newTag);
} }
} }
@ -829,26 +828,23 @@ public class ProcessSeries : IProcessSeries
/// ///
/// </summary> /// </summary>
/// <param name="names"></param> /// <param name="names"></param>
/// <param name="isExternal"></param>
/// <param name="action">Callback for every item. Will give said item back and a bool if item was added</param> /// <param name="action">Callback for every item. Will give said item back and a bool if item was added</param>
private void UpdateTag(IEnumerable<string> names, bool isExternal, Action<Tag, bool> action) private void UpdateTag(IEnumerable<string> names, Action<Tag, bool> action)
{ {
foreach (var name in names) foreach (var name in names)
{ {
if (string.IsNullOrEmpty(name.Trim())) continue; if (string.IsNullOrEmpty(name.Trim())) continue;
var added = false;
var normalizedName = Parser.Parser.Normalize(name); var normalizedName = Parser.Parser.Normalize(name);
_tags.TryGetValue(normalizedName, out var tag);
var tag = _tags.FirstOrDefault(p => var added = tag == null;
p.NormalizedTitle.Equals(normalizedName) && p.ExternalTag == isExternal);
if (tag == null) if (tag == null)
{ {
added = true; tag = DbFactory.Tag(name);
tag = DbFactory.Tag(name, false);
lock (_tags) lock (_tags)
{ {
_tags.Add(tag); _tags.Add(normalizedName, tag);
} }
} }

8
openapi.json
View File

@ -7,7 +7,7 @@
"name": "GPL-3.0", "name": "GPL-3.0",
"url": "https://github.com/Kareadita/Kavita/blob/develop/LICENSE" "url": "https://github.com/Kareadita/Kavita/blob/develop/LICENSE"
}, },
"version": "0.6.1.33" "version": "0.6.1.34"
}, },
"servers": [ "servers": [
{ {
@ -10935,9 +10935,6 @@
"type": "string", "type": "string",
"nullable": true "nullable": true
}, },
"externalTag": {
"type": "boolean"
},
"seriesMetadatas": { "seriesMetadatas": {
"type": "array", "type": "array",
"items": { "items": {
@ -13562,9 +13559,6 @@
"type": "string", "type": "string",
"nullable": true "nullable": true
}, },
"externalTag": {
"type": "boolean"
},
"seriesMetadatas": { "seriesMetadatas": {
"type": "array", "type": "array",
"items": { "items": {