Cutlery/jellyfin
1
0
Fork 0
mirror of https://github.com/jellyfin/jellyfin.git synced 2025-10-31 10:37:22 -04:00
Code Issues Packages Projects Releases Wiki Activity

Fix Deduplication and Save of Items

Browse Source
This commit is contained in:
JPVenson 2024-11-13 14:25:26 +00:00
parent fcb1dfc010
commit 7b81a39ee1
6 changed files with 1694 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
Jellyfin.Server.Implementations/Item/BaseItemRepository.cs
View File

@ -1270,6 +1270,7 @@ public sealed class BaseItemRepository(
}
else
{
context.BaseItemProviders.Where(e => e.ItemId == entity.Id).ExecuteDelete();
context.BaseItems.Attach(entity).State = EntityState.Modified;
}
@ -1289,22 +1290,23 @@ public sealed class BaseItemRepository(
}
var itemValuesToSave = GetItemValuesToSave(item.Item, item.InheritedTags);
var itemValues = itemValuesToSave.Select(e => e.Value).ToArray();
context.ItemValuesMap.Where(e => e.ItemId == entity.Id).ExecuteDelete();
entity.ItemValues = new List<ItemValueMap>();
var referenceValues = context.ItemValues.Where(e => itemValues.Any(f => f == e.CleanValue)).ToArray();
foreach (var itemValue in itemValuesToSave)
{
var refValue = referenceValues.FirstOrDefault(f => f.CleanValue == itemValue.Value && (int)f.Type == itemValue.MagicNumber);
if (refValue is not null)
var refValue = context.ItemValues
.Where(f => f.CleanValue == GetCleanValue(itemValue.Value) && (int)f.Type == itemValue.MagicNumber)
.Select(e => e.ItemValueId)
.FirstOrDefault();
if (!refValue.IsEmpty())
{
entity.ItemValues.Add(new ItemValueMap()
{
Item = entity,
ItemId = entity.Id,
ItemValue = null!,
ItemValueId = refValue.ItemValueId
ItemValueId = refValue
});
}
else

1595
Jellyfin.Server.Implementations/Migrations/20241113133548_EnforceUniqueItemValue.Designer.cs generated Normal file
View File

File diff suppressed because it is too large Load Diff

37
Jellyfin.Server.Implementations/Migrations/20241113133548_EnforceUniqueItemValue.cs Normal file
View File

@ -0,0 +1,37 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Jellyfin.Server.Implementations.Migrations
{
/// <inheritdoc />
public partial class EnforceUniqueItemValue : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropIndex(
name: "IX_ItemValues_Type_CleanValue",
table: "ItemValues");
migrationBuilder.CreateIndex(
name: "IX_ItemValues_Type_CleanValue",
table: "ItemValues",
columns: new[] { "Type", "CleanValue" },
unique: true);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropIndex(
name: "IX_ItemValues_Type_CleanValue",
table: "ItemValues");
migrationBuilder.CreateIndex(
name: "IX_ItemValues_Type_CleanValue",
table: "ItemValues",
columns: new[] { "Type", "CleanValue" });
}
}
}

3
Jellyfin.Server.Implementations/Migrations/JellyfinDbModelSnapshot.cs
View File

@ -690,7 +690,8 @@ namespace Jellyfin.Server.Implementations.Migrations
b.HasKey("ItemValueId");
b.HasIndex("Type", "CleanValue");
b.HasIndex("Type", "CleanValue")
.IsUnique();
b.ToTable("ItemValues");
});

2
Jellyfin.Server.Implementations/ModelConfiguration/ItemValuesConfiguration.cs
View File

@ -14,6 +14,6 @@ public class ItemValuesConfiguration : IEntityTypeConfiguration<ItemValue>
public void Configure(EntityTypeBuilder<ItemValue> builder)
{
builder.HasKey(e => e.ItemValueId);
builder.HasIndex(e => new { e.Type, e.CleanValue });
builder.HasIndex(e => new { e.Type, e.CleanValue }).IsUnique();
}
}

93
Jellyfin.Server/Migrations/Routines/MigrateLibraryDb.cs
View File

@ -107,6 +107,45 @@ public class MigrateLibraryDb : IMigrationRoutine
_logger.LogInformation("Saving BaseItems entries took {0}.", stopwatch.Elapsed);
stopwatch.Restart();
_logger.LogInformation("Start moving ItemValues.");
// do not migrate inherited types as they are now properly mapped in search and lookup.
var itemValueQuery = "select ItemId, Type, Value, CleanValue FROM ItemValues WHERE Type <> 6";
dbContext.ItemValues.ExecuteDelete();
// EFCores local lookup sucks.
var localItems = new Dictionary<(int Type, string CleanValue), (ItemValue ItemValue, List<Guid> ItemIds)>();
foreach (SqliteDataReader dto in connection.Query(itemValueQuery))
{
var itemId = dto.GetGuid(0);
var entity = GetItemValue(dto);
var key = ((int)entity.Type, entity.CleanValue);
if (!localItems.TryGetValue(key, out var existing))
{
localItems[key] = existing = (entity, []);
}
existing.ItemIds.Add(itemId);
}
foreach (var item in localItems)
{
dbContext.ItemValues.Add(item.Value.ItemValue);
dbContext.ItemValuesMap.AddRange(item.Value.ItemIds.Distinct().Select(f => new ItemValueMap()
{
Item = null!,
ItemValue = null!,
ItemId = f,
ItemValueId = item.Value.ItemValue.ItemValueId
}));
}
_logger.LogInformation("Try saving {0} ItemValues entries.", dbContext.ItemValues.Local.Count);
dbContext.SaveChanges();
migrationTotalTime += stopwatch.Elapsed;
_logger.LogInformation("Saving People ItemValues took {0}.", stopwatch.Elapsed);
stopwatch.Restart();
_logger.LogInformation("Start moving UserData.");
var queryResult = connection.Query("SELECT key, userId, rating, played, playCount, isFavorite, playbackPositionTicks, lastPlayedDate, AudioStreamIndex, SubtitleStreamIndex FROM UserDatas");
@ -158,6 +197,8 @@ public class MigrateLibraryDb : IMigrationRoutine
dbContext.Peoples.ExecuteDelete();
dbContext.PeopleBaseItemMap.ExecuteDelete();
var peopleCache = new Dictionary<string, (People Person, List<PeopleBaseItemMap> Items)>();
foreach (SqliteDataReader reader in connection.Query(personsQuery))
{
var itemId = reader.GetGuid(0);
@ -168,11 +209,9 @@ public class MigrateLibraryDb : IMigrationRoutine
}
var entity = GetPerson(reader);
var existingPerson = dbContext.Peoples.FirstOrDefault(e => e.Name == entity.Name);
if (existingPerson is null)
if (!peopleCache.TryGetValue(entity.Name, out var personCache))
{
dbContext.Peoples.Add(entity);
existingPerson = entity;
peopleCache[entity.Name] = personCache = (entity, []);
}
if (reader.TryGetString(2, out var role))
@ -183,58 +222,30 @@ public class MigrateLibraryDb : IMigrationRoutine
{
}
dbContext.PeopleBaseItemMap.Add(new PeopleBaseItemMap()
personCache.Items.Add(new PeopleBaseItemMap()
{
Item = null!,
ItemId = itemId,
People = existingPerson,
PeopleId = existingPerson.Id,
People = null!,
PeopleId = personCache.Person.Id,
ListOrder = sortOrder,
SortOrder = sortOrder,
Role = role
});
}
foreach (var item in peopleCache)
{
dbContext.Peoples.Add(item.Value.Person);
dbContext.PeopleBaseItemMap.AddRange(item.Value.Items.DistinctBy(e => (e.ItemId, e.PeopleId)));
}
_logger.LogInformation("Try saving {0} People entries.", dbContext.MediaStreamInfos.Local.Count);
dbContext.SaveChanges();
migrationTotalTime += stopwatch.Elapsed;
_logger.LogInformation("Saving People entries took {0}.", stopwatch.Elapsed);
stopwatch.Restart();
_logger.LogInformation("Start moving ItemValues.");
// do not migrate inherited types as they are now properly mapped in search and lookup.
var itemValueQuery = "select ItemId, Type, Value, CleanValue FROM ItemValues WHERE Type <> 6";
dbContext.ItemValues.ExecuteDelete();
foreach (SqliteDataReader dto in connection.Query(itemValueQuery))
{
var itemId = dto.GetGuid(0);
var entity = GetItemValue(dto);
var existingItemValue = dbContext.ItemValues.FirstOrDefault(f => f.Type == entity.Type && f.Value == entity.Value);
if (existingItemValue is null)
{
dbContext.ItemValues.Add(entity);
}
else
{
entity = existingItemValue;
}
dbContext.ItemValuesMap.Add(new ItemValueMap()
{
Item = null!,
ItemValue = null!,
ItemId = itemId,
ItemValueId = entity.ItemValueId
});
}
_logger.LogInformation("Try saving {0} ItemValues entries.", dbContext.ItemValues.Local.Count);
dbContext.SaveChanges();
migrationTotalTime += stopwatch.Elapsed;
_logger.LogInformation("Saving People ItemValues took {0}.", stopwatch.Elapsed);
stopwatch.Restart();
_logger.LogInformation("Start moving Chapters.");
var chapterQuery = "select ItemId,StartPositionTicks,Name,ImagePath,ImageDateModified,ChapterIndex from Chapters2";
dbContext.Chapters.ExecuteDelete();