Cutlery/Kavita
1
0
Fork 0
mirror of https://github.com/Kareadita/Kavita.git synced 2025-07-09 03:04:19 -04:00
Code Issues Packages Projects Releases Wiki Activity

Scanner Performance Improvements (#1774)

Browse Source 
* Refactored the Genre code to be faster and used a dictonary to avoid some lookups. May fix the rare foreign constraint issue.

* Refactored tag to the same implementation as Genre. Ensure when grabbing tags from ComicInfo, we normalize and throw out duplicates.

* Removed an internal "external" field that was planned for Genres and Tags, but now with new plugin architecture, not needed.
This commit is contained in:
Joe Milazzo 2023-02-03 04:52:51 -08:00 committed by GitHub
parent 48aebfc3c2
commit 8a0a2f0961
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 1925 additions and 152 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
API.Tests/Helpers/GenreHelperTests.cs
View File

@ -13,13 +13,13 @@ public class GenreHelperTests
{
var allGenres = new List<Genre>
{
DbFactory.Genre("Action", false),
DbFactory.Genre("action", false),
DbFactory.Genre("Sci-fi", false),
DbFactory.Genre("Action"),
DbFactory.Genre("action"),
DbFactory.Genre("Sci-fi"),
};
var genreAdded = new List<Genre>();
GenreHelper.UpdateGenre(allGenres, new[] {"Action", "Adventure"}, false, genre =>
GenreHelper.UpdateGenre(allGenres, new[] {"Action", "Adventure"}, genre =>
{
genreAdded.Add(genre);
});
@ -33,19 +33,20 @@ public class GenreHelperTests
{
var allGenres = new List<Genre>
{
DbFactory.Genre("Action", false),
DbFactory.Genre("action", false),
DbFactory.Genre("Sci-fi", false),
DbFactory.Genre("Action"),
DbFactory.Genre("action"),
DbFactory.Genre("Sci-fi"),
};
var genreAdded = new List<Genre>();
GenreHelper.UpdateGenre(allGenres, new[] {"Action", "Scifi"}, false, genre =>
GenreHelper.UpdateGenre(allGenres, new[] {"Action", "Scifi"}, genre =>
{
genreAdded.Add(genre);
});
Assert.Equal(3, allGenres.Count);
Assert.Equal(2, genreAdded.Count);
}
[Fact]
@ -53,49 +54,34 @@ public class GenreHelperTests
{
var existingGenres = new List<Genre>
{
DbFactory.Genre("Action", false),
DbFactory.Genre("action", false),
DbFactory.Genre("Sci-fi", false),
DbFactory.Genre("Action"),
DbFactory.Genre("action"),
DbFactory.Genre("Sci-fi"),
};
GenreHelper.AddGenreIfNotExists(existingGenres, DbFactory.Genre("Action", false));
GenreHelper.AddGenreIfNotExists(existingGenres, DbFactory.Genre("Action"));
Assert.Equal(3, existingGenres.Count);
GenreHelper.AddGenreIfNotExists(existingGenres, DbFactory.Genre("action", false));
GenreHelper.AddGenreIfNotExists(existingGenres, DbFactory.Genre("action"));
Assert.Equal(3, existingGenres.Count);
GenreHelper.AddGenreIfNotExists(existingGenres, DbFactory.Genre("Shonen", false));
GenreHelper.AddGenreIfNotExists(existingGenres, DbFactory.Genre("Shonen"));
Assert.Equal(4, existingGenres.Count);
}
[Fact]
public void AddGenre_ShouldNotAddSameNameAndExternal()
{
var existingGenres = new List<Genre>
{
DbFactory.Genre("Action", false),
DbFactory.Genre("action", false),
DbFactory.Genre("Sci-fi", false),
};
GenreHelper.AddGenreIfNotExists(existingGenres, DbFactory.Genre("Action", true));
Assert.Equal(3, existingGenres.Count);
}
[Fact]
public void KeepOnlySamePeopleBetweenLists()
{
var existingGenres = new List<Genre>
{
DbFactory.Genre("Action", false),
DbFactory.Genre("Sci-fi", false),
DbFactory.Genre("Action"),
DbFactory.Genre("Sci-fi"),
};
var peopleFromChapters = new List<Genre>
{
DbFactory.Genre("Action", false),
DbFactory.Genre("Action"),
};
var genreRemoved = new List<Genre>();
@ -113,8 +99,8 @@ public class GenreHelperTests
{
var existingGenres = new List<Genre>
{
DbFactory.Genre("Action", false),
DbFactory.Genre("Sci-fi", false),
DbFactory.Genre("Action"),
DbFactory.Genre("Sci-fi"),
};
var peopleFromChapters = new List<Genre>();

53
API.Tests/Helpers/TagHelperTests.cs
View File

@ -13,13 +13,13 @@ public class TagHelperTests
{
var allTags = new List<Tag>
{
DbFactory.Tag("Action", false),
DbFactory.Tag("action", false),
DbFactory.Tag("Sci-fi", false),
DbFactory.Tag("Action"),
DbFactory.Tag("action"),
DbFactory.Tag("Sci-fi"),
};
var tagAdded = new List<Tag>();
TagHelper.UpdateTag(allTags, new[] {"Action", "Adventure"}, false, (tag, added) =>
TagHelper.UpdateTag(allTags, new[] {"Action", "Adventure"}, (tag, added) =>
{
if (added)
{
@ -37,14 +37,14 @@ public class TagHelperTests
{
var allTags = new List<Tag>
{
DbFactory.Tag("Action", false),
DbFactory.Tag("action", false),
DbFactory.Tag("Sci-fi", false),
DbFactory.Tag("Action"),
DbFactory.Tag("action"),
DbFactory.Tag("Sci-fi"),
};
var tagAdded = new List<Tag>();
TagHelper.UpdateTag(allTags, new[] {"Action", "Scifi"}, false, (tag, added) =>
TagHelper.UpdateTag(allTags, new[] {"Action", "Scifi"}, (tag, added) =>
{
if (added)
{
@ -62,49 +62,34 @@ public class TagHelperTests
{
var existingTags = new List<Tag>
{
DbFactory.Tag("Action", false),
DbFactory.Tag("action", false),
DbFactory.Tag("Sci-fi", false),
DbFactory.Tag("Action"),
DbFactory.Tag("action"),
DbFactory.Tag("Sci-fi"),
};
TagHelper.AddTagIfNotExists(existingTags, DbFactory.Tag("Action", false));
TagHelper.AddTagIfNotExists(existingTags, DbFactory.Tag("Action"));
Assert.Equal(3, existingTags.Count);
TagHelper.AddTagIfNotExists(existingTags, DbFactory.Tag("action", false));
TagHelper.AddTagIfNotExists(existingTags, DbFactory.Tag("action"));
Assert.Equal(3, existingTags.Count);
TagHelper.AddTagIfNotExists(existingTags, DbFactory.Tag("Shonen", false));
TagHelper.AddTagIfNotExists(existingTags, DbFactory.Tag("Shonen"));
Assert.Equal(4, existingTags.Count);
}
[Fact]
public void AddTag_ShouldNotAddSameNameAndExternal()
{
var existingTags = new List<Tag>
{
DbFactory.Tag("Action", false),
DbFactory.Tag("action", false),
DbFactory.Tag("Sci-fi", false),
};
TagHelper.AddTagIfNotExists(existingTags, DbFactory.Tag("Action", true));
Assert.Equal(3, existingTags.Count);
}
[Fact]
public void KeepOnlySamePeopleBetweenLists()
{
var existingTags = new List<Tag>
{
DbFactory.Tag("Action", false),
DbFactory.Tag("Sci-fi", false),
DbFactory.Tag("Action"),
DbFactory.Tag("Sci-fi"),
};
var peopleFromChapters = new List<Tag>
{
DbFactory.Tag("Action", false),
DbFactory.Tag("Action"),
};
var tagRemoved = new List<Tag>();
@ -122,8 +107,8 @@ public class TagHelperTests
{
var existingTags = new List<Tag>
{
DbFactory.Tag("Action", false),
DbFactory.Tag("Sci-fi", false),
DbFactory.Tag("Action"),
DbFactory.Tag("Sci-fi"),
};
var peopleFromChapters = new List<Tag>();

10
API.Tests/Services/SeriesServiceTests.cs
View File

@ -762,7 +762,7 @@ public class SeriesServiceTests : AbstractDbTest
},
Metadata = DbFactory.SeriesMetadata(new List<CollectionTag>())
};
var g = DbFactory.Genre("Existing Genre", false);
var g = DbFactory.Genre("Existing Genre");
s.Metadata.Genres = new List<Genre>() {g};
_context.Series.Add(s);
@ -918,7 +918,7 @@ public class SeriesServiceTests : AbstractDbTest
},
Metadata = DbFactory.SeriesMetadata(new List<CollectionTag>())
};
var g = DbFactory.Genre("Existing Genre", false);
var g = DbFactory.Genre("Existing Genre");
s.Metadata.Genres = new List<Genre>() {g};
s.Metadata.GenresLocked = true;
_context.Series.Add(s);
@ -1555,5 +1555,11 @@ public class SeriesServiceTests : AbstractDbTest
Assert.Null(await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(1));
}
#endregion
#region UpdateRelatedList
#endregion
}

6
API/Data/DbFactory.cs
View File

@ -121,23 +121,21 @@ public static class DbFactory
};
}
public static Genre Genre(string name, bool external)
public static Genre Genre(string name)
{
return new Genre()
{
Title = name.Trim().SentenceCase(),
NormalizedTitle = Services.Tasks.Scanner.Parser.Parser.Normalize(name),
ExternalTag = external
};
}
public static Tag Tag(string name, bool external)
public static Tag Tag(string name)
{
return new Tag()
{
Title = name.Trim().SentenceCase(),
NormalizedTitle = Services.Tasks.Scanner.Parser.Parser.Normalize(name),
ExternalTag = external
};
}

1748
API/Data/Migrations/20230203112022_RemoveExternalFromTagAndGenre.Designer.cs generated Normal file
View File

File diff suppressed because it is too large Load Diff

77
API/Data/Migrations/20230203112022_RemoveExternalFromTagAndGenre.cs Normal file
View File

@ -0,0 +1,77 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace API.Data.Migrations
{
public partial class RemoveExternalFromTagAndGenre : Migration
{
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropIndex(
name: "IX_Tag_NormalizedTitle_ExternalTag",
table: "Tag");
migrationBuilder.DropIndex(
name: "IX_Genre_NormalizedTitle_ExternalTag",
table: "Genre");
migrationBuilder.DropColumn(
name: "ExternalTag",
table: "Tag");
migrationBuilder.DropColumn(
name: "ExternalTag",
table: "Genre");
migrationBuilder.CreateIndex(
name: "IX_Tag_NormalizedTitle",
table: "Tag",
column: "NormalizedTitle",
unique: true);
migrationBuilder.CreateIndex(
name: "IX_Genre_NormalizedTitle",
table: "Genre",
column: "NormalizedTitle",
unique: true);
}
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropIndex(
name: "IX_Tag_NormalizedTitle",
table: "Tag");
migrationBuilder.DropIndex(
name: "IX_Genre_NormalizedTitle",
table: "Genre");
migrationBuilder.AddColumn<bool>(
name: "ExternalTag",
table: "Tag",
type: "INTEGER",
nullable: false,
defaultValue: false);
migrationBuilder.AddColumn<bool>(
name: "ExternalTag",
table: "Genre",
type: "INTEGER",
nullable: false,
defaultValue: false);
migrationBuilder.CreateIndex(
name: "IX_Tag_NormalizedTitle_ExternalTag",
table: "Tag",
columns: new[] { "NormalizedTitle", "ExternalTag" },
unique: true);
migrationBuilder.CreateIndex(
name: "IX_Genre_NormalizedTitle_ExternalTag",
table: "Genre",
columns: new[] { "NormalizedTitle", "ExternalTag" },
unique: true);
}
}
}

10
API/Data/Migrations/DataContextModelSnapshot.cs
View File

@ -528,9 +528,6 @@ namespace API.Data.Migrations
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<bool>("ExternalTag")
.HasColumnType("INTEGER");
b.Property<string>("NormalizedTitle")
.HasColumnType("TEXT");
@ -539,7 +536,7 @@ namespace API.Data.Migrations
b.HasKey("Id");
b.HasIndex("NormalizedTitle", "ExternalTag")
b.HasIndex("NormalizedTitle")
.IsUnique();
b.ToTable("Genre");
@ -1036,9 +1033,6 @@ namespace API.Data.Migrations
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<bool>("ExternalTag")
.HasColumnType("INTEGER");
b.Property<string>("NormalizedTitle")
.HasColumnType("TEXT");
@ -1047,7 +1041,7 @@ namespace API.Data.Migrations
b.HasKey("Id");
b.HasIndex("NormalizedTitle", "ExternalTag")
b.HasIndex("NormalizedTitle")
.IsUnique();
b.ToTable("Tag");

2
API/Data/Repositories/GenreRepository.cs
View File

@ -56,7 +56,7 @@ public class GenreRepository : IGenreRepository
var genresWithNoConnections = await _context.Genre
.Include(p => p.SeriesMetadatas)
.Include(p => p.Chapters)
.Where(p => p.SeriesMetadatas.Count == 0 && p.Chapters.Count == 0 && p.ExternalTag == removeExternal)
.Where(p => p.SeriesMetadatas.Count == 0 && p.Chapters.Count == 0)
.AsSplitQuery()
.ToListAsync();

2
API/Data/Repositories/TagRepository.cs
View File

@ -46,7 +46,7 @@ public class TagRepository : ITagRepository
var tagsWithNoConnections = await _context.Tag
.Include(p => p.SeriesMetadatas)
.Include(p => p.Chapters)
.Where(p => p.SeriesMetadatas.Count == 0 && p.Chapters.Count == 0 && p.ExternalTag == removeExternal)
.Where(p => p.SeriesMetadatas.Count == 0 && p.Chapters.Count == 0)
.AsSplitQuery()
.ToListAsync();

3
API/Entities/Genre.cs
View File

@ -4,13 +4,12 @@ using Microsoft.EntityFrameworkCore;
namespace API.Entities;
[Index(nameof(NormalizedTitle), nameof(ExternalTag), IsUnique = true)]
[Index(nameof(NormalizedTitle), IsUnique = true)]
public class Genre
{
public int Id { get; set; }
public string Title { get; set; }
public string NormalizedTitle { get; set; }
public bool ExternalTag { get; set; }
public ICollection<SeriesMetadata> SeriesMetadatas { get; set; }
public ICollection<Chapter> Chapters { get; set; }

6
API/Entities/Person.cs
View File

@ -4,18 +4,12 @@ using API.Entities.Metadata;
namespace API.Entities;
public enum ProviderSource
{
Local = 1,
External = 2
}
public class Person
{
public int Id { get; set; }
public string Name { get; set; }
public string NormalizedName { get; set; }
public PersonRole Role { get; set; }
//public ProviderSource Source { get; set; }
// Relationships
public ICollection<SeriesMetadata> SeriesMetadatas { get; set; }

3
API/Entities/Tag.cs
View File

@ -4,13 +4,12 @@ using Microsoft.EntityFrameworkCore;
namespace API.Entities;
[Index(nameof(NormalizedTitle), nameof(ExternalTag), IsUnique = true)]
[Index(nameof(NormalizedTitle), IsUnique = true)]
public class Tag
{
public int Id { get; set; }
public string Title { get; set; }
public string NormalizedTitle { get; set; }
public bool ExternalTag { get; set; }
public ICollection<SeriesMetadata> SeriesMetadatas { get; set; }
public ICollection<Chapter> Chapters { get; set; }

10
API/Helpers/GenreHelper.cs
View File

@ -14,20 +14,18 @@ public static class GenreHelper
/// </summary>
/// <param name="allGenres"></param>
/// <param name="names"></param>
/// <param name="isExternal"></param>
/// <param name="action"></param>
public static void UpdateGenre(ICollection<Genre> allGenres, IEnumerable<string> names, bool isExternal, Action<Genre> action)
public static void UpdateGenre(ICollection<Genre> allGenres, IEnumerable<string> names, Action<Genre> action)
{
foreach (var name in names)
{
if (string.IsNullOrEmpty(name.Trim())) continue;
var normalizedName = Services.Tasks.Scanner.Parser.Parser.Normalize(name);
var genre = allGenres.FirstOrDefault(p =>
p.NormalizedTitle.Equals(normalizedName) && p.ExternalTag == isExternal);
var genre = allGenres.FirstOrDefault(p => p.NormalizedTitle.Equals(normalizedName));
if (genre == null)
{
genre = DbFactory.Genre(name, false);
genre = DbFactory.Genre(name);
allGenres.Add(genre);
}
@ -41,7 +39,7 @@ public static class GenreHelper
var existing = existingGenres.ToList();
foreach (var genre in existing)
{
var existingPerson = removeAllExcept.FirstOrDefault(g => g.ExternalTag == genre.ExternalTag && genre.NormalizedTitle.Equals(g.NormalizedTitle));
var existingPerson = removeAllExcept.FirstOrDefault(g => genre.NormalizedTitle.Equals(g.NormalizedTitle));
if (existingPerson != null) continue;
existingGenres.Remove(genre);
action?.Invoke(genre);

13
API/Helpers/TagHelper.cs
View File

@ -14,9 +14,8 @@ public static class TagHelper
/// </summary>
/// <param name="allTags"></param>
/// <param name="names"></param>
/// <param name="isExternal"></param>
/// <param name="action">Callback for every item. Will give said item back and a bool if item was added</param>
public static void UpdateTag(ICollection<Tag> allTags, IEnumerable<string> names, bool isExternal, Action<Tag, bool> action)
public static void UpdateTag(ICollection<Tag> allTags, IEnumerable<string> names, Action<Tag, bool> action)
{
foreach (var name in names)
{
@ -26,11 +25,11 @@ public static class TagHelper
var normalizedName = Services.Tasks.Scanner.Parser.Parser.Normalize(name);
var genre = allTags.FirstOrDefault(p =>
p.NormalizedTitle.Equals(normalizedName) && p.ExternalTag == isExternal);
p.NormalizedTitle.Equals(normalizedName));
if (genre == null)
{
added = true;
genre = DbFactory.Tag(name, false);
genre = DbFactory.Tag(name);
allTags.Add(genre);
}
@ -43,7 +42,7 @@ public static class TagHelper
var existing = existingTags.ToList();
foreach (var genre in existing)
{
var existingPerson = removeAllExcept.FirstOrDefault(g => g.ExternalTag == genre.ExternalTag && genre.NormalizedTitle.Equals(g.NormalizedTitle));
var existingPerson = removeAllExcept.FirstOrDefault(g => genre.NormalizedTitle.Equals(g.NormalizedTitle));
if (existingPerson != null) continue;
existingTags.Remove(genre);
action?.Invoke(genre);
@ -84,12 +83,12 @@ public static class TagHelper
/// <param name="tags">Tags from metadata</param>
/// <param name="isExternal">Remove external tags?</param>
/// <param name="action">Callback which will be executed for each tag removed</param>
public static void RemoveTags(ICollection<Tag> existingTags, IEnumerable<string> tags, bool isExternal, Action<Tag> action = null)
public static void RemoveTags(ICollection<Tag> existingTags, IEnumerable<string> tags, Action<Tag> action = null)
{
var normalizedTags = tags.Select(Services.Tasks.Scanner.Parser.Parser.Normalize).ToList();
foreach (var person in normalizedTags)
{
var existingTag = existingTags.FirstOrDefault(p => p.ExternalTag == isExternal && person.Equals(p.NormalizedTitle));
var existingTag = existingTags.FirstOrDefault(p => person.Equals(p.NormalizedTitle));
if (existingTag == null) continue;
existingTags.Remove(existingTag);

8
API/Services/SeriesService.cs
View File

@ -115,7 +115,7 @@ public class SeriesService : ISeriesService
}
series.Metadata.CollectionTags ??= new List<CollectionTag>();
UpdateRelatedList(updateSeriesMetadataDto.CollectionTags, series, allCollectionTags, (tag) =>
UpdateCollectionsList(updateSeriesMetadataDto.CollectionTags, series, allCollectionTags, (tag) =>
{
series.Metadata.CollectionTags.Add(tag);
});
@ -210,7 +210,7 @@ public class SeriesService : ISeriesService
}
private static void UpdateRelatedList(ICollection<CollectionTagDto> tags, Series series, IReadOnlyCollection<CollectionTag> allTags,
public static void UpdateCollectionsList(ICollection<CollectionTagDto> tags, Series series, IReadOnlyCollection<CollectionTag> allTags,
Action<CollectionTag> handleAdd)
{
// TODO: Move UpdateRelatedList to a helper so we can easily test
@ -278,7 +278,7 @@ public class SeriesService : ISeriesService
else
{
// Add new tag
handleAdd(DbFactory.Genre(tagTitle, false));
handleAdd(DbFactory.Genre(tagTitle));
isModified = true;
}
}
@ -320,7 +320,7 @@ public class SeriesService : ISeriesService
else
{
// Add new tag
handleAdd(DbFactory.Tag(tagTitle, false));
handleAdd(DbFactory.Tag(tagTitle));
isModified = true;
}
}

2
API/Services/Tasks/Scanner/Parser/Parser.cs
View File

@ -945,7 +945,7 @@ public static class Parser
public static string Normalize(string name)
{
return NormalizeRegex.Replace(name, string.Empty).ToLower();
return NormalizeRegex.Replace(name, string.Empty).Trim().ToLower();
}
/// <summary>

62
API/Services/Tasks/Scanner/ProcessSeries.cs
View File

@ -48,9 +48,9 @@ public class ProcessSeries : IProcessSeries
private readonly IWordCountAnalyzerService _wordCountAnalyzerService;
private readonly ICollectionTagService _collectionTagService;
private IList<Genre> _genres;
private Dictionary<string, Genre> _genres;
private IList<Person> _people;
private IList<Tag> _tags;
private Dictionary<string, Tag> _tags;
private Dictionary<string, CollectionTag> _collectionTags;
public ProcessSeries(IUnitOfWork unitOfWork, ILogger<ProcessSeries> logger, IEventHub eventHub,
@ -75,9 +75,9 @@ public class ProcessSeries : IProcessSeries
/// </summary>
public async Task Prime()
{
_genres = await _unitOfWork.GenreRepository.GetAllGenresAsync();
_genres = (await _unitOfWork.GenreRepository.GetAllGenresAsync()).ToDictionary(t => t.NormalizedTitle);
_people = await _unitOfWork.PersonRepository.GetAllPeople();
_tags = await _unitOfWork.TagRepository.GetAllTagsAsync();
_tags = (await _unitOfWork.TagRepository.GetAllTagsAsync()).ToDictionary(t => t.NormalizedTitle);
_collectionTags = (await _unitOfWork.CollectionTagRepository.GetAllTagsAsync(CollectionTagIncludes.SeriesMetadata))
.ToDictionary(t => t.NormalizedTitle);
@ -673,14 +673,14 @@ public class ProcessSeries : IProcessSeries
PersonHelper.AddPersonIfNotExists(chapter.People, person);
}
void AddGenre(Genre genre)
void AddGenre(Genre genre, bool newTag)
{
GenreHelper.AddGenreIfNotExists(chapter.Genres, genre);
chapter.Genres.Add(genre);
}
void AddTag(Tag tag, bool added)
{
TagHelper.AddTagIfNotExists(chapter.Tags, tag);
chapter.Tags.Add(tag);
}
@ -745,14 +745,13 @@ public class ProcessSeries : IProcessSeries
AddPerson);
var genres = GetTagValues(comicInfo.Genre);
GenreHelper.KeepOnlySameGenreBetweenLists(chapter.Genres, genres.Select(g => DbFactory.Genre(g, false)).ToList());
UpdateGenre(genres, false,
AddGenre);
GenreHelper.KeepOnlySameGenreBetweenLists(chapter.Genres,
genres.Select(DbFactory.Genre).ToList());
UpdateGenre(genres, AddGenre);
var tags = GetTagValues(comicInfo.Tags);
TagHelper.KeepOnlySameTagBetweenLists(chapter.Tags, tags.Select(t => DbFactory.Tag(t, false)).ToList());
UpdateTag(tags, false,
AddTag);
TagHelper.KeepOnlySameTagBetweenLists(chapter.Tags, tags.Select(DbFactory.Tag).ToList());
UpdateTag(tags, AddTag);
}
private static IList<string> GetTagValues(string comicInfoTagSeparatedByComma)
@ -760,7 +759,7 @@ public class ProcessSeries : IProcessSeries
if (!string.IsNullOrEmpty(comicInfoTagSeparatedByComma))
{
return comicInfoTagSeparatedByComma.Split(",").Select(s => s.Trim()).ToList();
return comicInfoTagSeparatedByComma.Split(",").Select(s => s.Trim()).DistinctBy(s => s.Normalize()).ToList();
}
return ImmutableList<string>.Empty;
}
@ -801,27 +800,27 @@ public class ProcessSeries : IProcessSeries
///
/// </summary>
/// <param name="names"></param>
/// <param name="isExternal"></param>
/// <param name="action"></param>
private void UpdateGenre(IEnumerable<string> names, bool isExternal, Action<Genre> action)
/// <param name="action">Executes for each tag</param>
private void UpdateGenre(IEnumerable<string> names, Action<Genre, bool> action)
{
foreach (var name in names)
{
if (string.IsNullOrEmpty(name.Trim())) continue;
var normalizedName = Parser.Parser.Normalize(name);
var genre = _genres.FirstOrDefault(p =>
p.NormalizedTitle.Equals(normalizedName) && p.ExternalTag == isExternal);
if (genre == null)
if (string.IsNullOrEmpty(normalizedName)) continue;
_genres.TryGetValue(normalizedName, out var genre);
var newTag = genre == null;
if (newTag)
{
genre = DbFactory.Genre(name, false);
genre = DbFactory.Genre(name);
lock (_genres)
{
_genres.Add(genre);
_genres.Add(normalizedName, genre);
_unitOfWork.GenreRepository.Attach(genre);
}
}
action(genre);
action(genre, newTag);
}
}
@ -829,26 +828,23 @@ public class ProcessSeries : IProcessSeries
///
/// </summary>
/// <param name="names"></param>
/// <param name="isExternal"></param>
/// <param name="action">Callback for every item. Will give said item back and a bool if item was added</param>
private void UpdateTag(IEnumerable<string> names, bool isExternal, Action<Tag, bool> action)
private void UpdateTag(IEnumerable<string> names, Action<Tag, bool> action)
{
foreach (var name in names)
{
if (string.IsNullOrEmpty(name.Trim())) continue;
var added = false;
var normalizedName = Parser.Parser.Normalize(name);
_tags.TryGetValue(normalizedName, out var tag);
var tag = _tags.FirstOrDefault(p =>
p.NormalizedTitle.Equals(normalizedName) && p.ExternalTag == isExternal);
var added = tag == null;
if (tag == null)
{
added = true;
tag = DbFactory.Tag(name, false);
tag = DbFactory.Tag(name);
lock (_tags)
{
_tags.Add(tag);
_tags.Add(normalizedName, tag);
}
}

8
openapi.json
View File

@ -7,7 +7,7 @@
"name": "GPL-3.0",
"url": "https://github.com/Kareadita/Kavita/blob/develop/LICENSE"
},
"version": "0.6.1.33"
"version": "0.6.1.34"
},
"servers": [
{
@ -10935,9 +10935,6 @@
"type": "string",
"nullable": true
},
"externalTag": {
"type": "boolean"
},
"seriesMetadatas": {
"type": "array",
"items": {
@ -13562,9 +13559,6 @@
"type": "string",
"nullable": true
},
"externalTag": {
"type": "boolean"
},
"seriesMetadatas": {
"type": "array",
"items": {