Fix Deduplication and Save of Items

Jellyfin.Server.Implementations/Item/BaseItemRepository.cs

@@ -1270,6 +1270,7 @@ public sealed class BaseItemRepository(
             }
             else
             {
+                context.BaseItemProviders.Where(e => e.ItemId == entity.Id).ExecuteDelete();
                 context.BaseItems.Attach(entity).State = EntityState.Modified;
             }
 
@@ -1289,22 +1290,23 @@ public sealed class BaseItemRepository(
             }
 
             var itemValuesToSave = GetItemValuesToSave(item.Item, item.InheritedTags);
-            var itemValues = itemValuesToSave.Select(e => e.Value).ToArray();
             context.ItemValuesMap.Where(e => e.ItemId == entity.Id).ExecuteDelete();
             entity.ItemValues = new List<ItemValueMap>();
-            var referenceValues = context.ItemValues.Where(e => itemValues.Any(f => f == e.CleanValue)).ToArray();
 
             foreach (var itemValue in itemValuesToSave)
             {
-                var refValue = referenceValues.FirstOrDefault(f => f.CleanValue == itemValue.Value && (int)f.Type == itemValue.MagicNumber);
+                var refValue = context.ItemValues
-                if (refValue is not null)
+                    .Where(f => f.CleanValue == GetCleanValue(itemValue.Value) && (int)f.Type == itemValue.MagicNumber)
+                    .Select(e => e.ItemValueId)
+                    .FirstOrDefault();
+                if (!refValue.IsEmpty())
                 {
                     entity.ItemValues.Add(new ItemValueMap()
                     {
                         Item = entity,
                         ItemId = entity.Id,
                         ItemValue = null!,
-                        ItemValueId = refValue.ItemValueId
+                        ItemValueId = refValue
                     });
                 }
                 else

Jellyfin.Server.Implementations/Migrations/20241113133548_EnforceUniqueItemValue.cs

@@ -0,0 +1,37 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace Jellyfin.Server.Implementations.Migrations
+{
+    /// <inheritdoc />
+    public partial class EnforceUniqueItemValue : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropIndex(
+                name: "IX_ItemValues_Type_CleanValue",
+                table: "ItemValues");
+
+            migrationBuilder.CreateIndex(
+                name: "IX_ItemValues_Type_CleanValue",
+                table: "ItemValues",
+                columns: new[] { "Type", "CleanValue" },
+                unique: true);
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropIndex(
+                name: "IX_ItemValues_Type_CleanValue",
+                table: "ItemValues");
+
+            migrationBuilder.CreateIndex(
+                name: "IX_ItemValues_Type_CleanValue",
+                table: "ItemValues",
+                columns: new[] { "Type", "CleanValue" });
+        }
+    }
+}

Jellyfin.Server.Implementations/Migrations/JellyfinDbModelSnapshot.cs

@@ -690,7 +690,8 @@ namespace Jellyfin.Server.Implementations.Migrations
 
                     b.HasKey("ItemValueId");
 
-                    b.HasIndex("Type", "CleanValue");
+                    b.HasIndex("Type", "CleanValue")
+                        .IsUnique();
 
                     b.ToTable("ItemValues");
                 });

Jellyfin.Server.Implementations/ModelConfiguration/ItemValuesConfiguration.cs

@@ -14,6 +14,6 @@ public class ItemValuesConfiguration : IEntityTypeConfiguration<ItemValue>
     public void Configure(EntityTypeBuilder<ItemValue> builder)
     {
         builder.HasKey(e => e.ItemValueId);
-        builder.HasIndex(e => new { e.Type, e.CleanValue });
+        builder.HasIndex(e => new { e.Type, e.CleanValue }).IsUnique();
     }
 }

Jellyfin.Server/Migrations/Routines/MigrateLibraryDb.cs

@@ -107,6 +107,45 @@ public class MigrateLibraryDb : IMigrationRoutine
         _logger.LogInformation("Saving BaseItems entries took {0}.", stopwatch.Elapsed);
         stopwatch.Restart();
 
+        _logger.LogInformation("Start moving ItemValues.");
+        // do not migrate inherited types as they are now properly mapped in search and lookup.
+        var itemValueQuery = "select ItemId, Type, Value, CleanValue FROM ItemValues WHERE Type <> 6";
+        dbContext.ItemValues.ExecuteDelete();
+
+        // EFCores local lookup sucks.
+        var localItems = new Dictionary<(int Type, string CleanValue), (ItemValue ItemValue, List<Guid> ItemIds)>();
+
+        foreach (SqliteDataReader dto in connection.Query(itemValueQuery))
+        {
+            var itemId = dto.GetGuid(0);
+            var entity = GetItemValue(dto);
+            var key = ((int)entity.Type, entity.CleanValue);
+            if (!localItems.TryGetValue(key, out var existing))
+            {
+                localItems[key] = existing = (entity, []);
+            }
+
+            existing.ItemIds.Add(itemId);
+        }
+
+        foreach (var item in localItems)
+        {
+            dbContext.ItemValues.Add(item.Value.ItemValue);
+            dbContext.ItemValuesMap.AddRange(item.Value.ItemIds.Distinct().Select(f => new ItemValueMap()
+            {
+                Item = null!,
+                ItemValue = null!,
+                ItemId = f,
+                ItemValueId = item.Value.ItemValue.ItemValueId
+            }));
+        }
+
+        _logger.LogInformation("Try saving {0} ItemValues entries.", dbContext.ItemValues.Local.Count);
+        dbContext.SaveChanges();
+        migrationTotalTime += stopwatch.Elapsed;
+        _logger.LogInformation("Saving People ItemValues took {0}.", stopwatch.Elapsed);
+        stopwatch.Restart();
+
         _logger.LogInformation("Start moving UserData.");
         var queryResult = connection.Query("SELECT key, userId, rating, played, playCount, isFavorite, playbackPositionTicks, lastPlayedDate, AudioStreamIndex, SubtitleStreamIndex FROM UserDatas");
 
@@ -158,6 +197,8 @@ public class MigrateLibraryDb : IMigrationRoutine
         dbContext.Peoples.ExecuteDelete();
         dbContext.PeopleBaseItemMap.ExecuteDelete();
 
+        var peopleCache = new Dictionary<string, (People Person, List<PeopleBaseItemMap> Items)>();
+
         foreach (SqliteDataReader reader in connection.Query(personsQuery))
         {
             var itemId = reader.GetGuid(0);
@@ -168,11 +209,9 @@ public class MigrateLibraryDb : IMigrationRoutine
             }
 
             var entity = GetPerson(reader);
-            var existingPerson = dbContext.Peoples.FirstOrDefault(e => e.Name == entity.Name);
+            if (!peopleCache.TryGetValue(entity.Name, out var personCache))
-            if (existingPerson is null)
             {
-                dbContext.Peoples.Add(entity);
+                peopleCache[entity.Name] = personCache = (entity, []);
-                existingPerson = entity;
             }
 
             if (reader.TryGetString(2, out var role))
@@ -183,58 +222,30 @@ public class MigrateLibraryDb : IMigrationRoutine
             {
             }
 
-            dbContext.PeopleBaseItemMap.Add(new PeopleBaseItemMap()
+            personCache.Items.Add(new PeopleBaseItemMap()
             {
                 Item = null!,
                 ItemId = itemId,
-                People = existingPerson,
+                People = null!,
-                PeopleId = existingPerson.Id,
+                PeopleId = personCache.Person.Id,
                 ListOrder = sortOrder,
                 SortOrder = sortOrder,
                 Role = role
             });
         }
 
+        foreach (var item in peopleCache)
+        {
+            dbContext.Peoples.Add(item.Value.Person);
+            dbContext.PeopleBaseItemMap.AddRange(item.Value.Items.DistinctBy(e => (e.ItemId, e.PeopleId)));
+        }
+
         _logger.LogInformation("Try saving {0} People entries.", dbContext.MediaStreamInfos.Local.Count);
         dbContext.SaveChanges();
         migrationTotalTime += stopwatch.Elapsed;
         _logger.LogInformation("Saving People entries took {0}.", stopwatch.Elapsed);
         stopwatch.Restart();
 
-        _logger.LogInformation("Start moving ItemValues.");
-        // do not migrate inherited types as they are now properly mapped in search and lookup.
-        var itemValueQuery = "select ItemId, Type, Value, CleanValue FROM ItemValues WHERE Type <> 6";
-        dbContext.ItemValues.ExecuteDelete();
-
-        foreach (SqliteDataReader dto in connection.Query(itemValueQuery))
-        {
-            var itemId = dto.GetGuid(0);
-            var entity = GetItemValue(dto);
-            var existingItemValue = dbContext.ItemValues.FirstOrDefault(f => f.Type == entity.Type && f.Value == entity.Value);
-            if (existingItemValue is null)
-            {
-                dbContext.ItemValues.Add(entity);
-            }
-            else
-            {
-                entity = existingItemValue;
-            }
-
-            dbContext.ItemValuesMap.Add(new ItemValueMap()
-            {
-                Item = null!,
-                ItemValue = null!,
-                ItemId = itemId,
-                ItemValueId = entity.ItemValueId
-            });
-        }
-
-        _logger.LogInformation("Try saving {0} ItemValues entries.", dbContext.ItemValues.Local.Count);
-        dbContext.SaveChanges();
-        migrationTotalTime += stopwatch.Elapsed;
-        _logger.LogInformation("Saving People ItemValues took {0}.", stopwatch.Elapsed);
-        stopwatch.Restart();
-
         _logger.LogInformation("Start moving Chapters.");
         var chapterQuery = "select ItemId,StartPositionTicks,Name,ImagePath,ImageDateModified,ChapterIndex from Chapters2";
         dbContext.Chapters.ExecuteDelete();