mirror of
https://github.com/jellyfin/jellyfin.git
synced 2025-05-24 02:02:29 -04:00
Library.db migration impovements (#13809)
* Fixes cleanup of wrong table in migration * use dedicated context for each step * Use prepared Context * Fix measurement of UserData migration time * Update logging and combine cleanup to its own stage * fix people map not logging migrate only readonly database * Add id blacklisting in migration to avoid duplicated log entires
This commit is contained in:
JPVenson
GitHub
parent
476a0d6932
commit
90a6cca92b
@ -73,273 +73,328 @@ internal class MigrateLibraryDb : IDatabaseMigrationRoutine
|
||||
|
||||
var dataPath = _paths.DataPath;
|
||||
var libraryDbPath = Path.Combine(dataPath, DbFilename);
|
||||
using var connection = new SqliteConnection($"Filename={libraryDbPath}");
|
||||
var migrationTotalTime = TimeSpan.Zero;
|
||||
using var connection = new SqliteConnection($"Filename={libraryDbPath};Mode=ReadOnly");
|
||||
|
||||
var stopwatch = new Stopwatch();
|
||||
stopwatch.Start();
|
||||
var fullOperationTimer = new Stopwatch();
|
||||
fullOperationTimer.Start();
|
||||
|
||||
connection.Open();
|
||||
using var dbContext = _provider.CreateDbContext();
|
||||
|
||||
migrationTotalTime += stopwatch.Elapsed;
|
||||
_logger.LogInformation("Saving UserData entries took {0}.", stopwatch.Elapsed);
|
||||
stopwatch.Restart();
|
||||
|
||||
_logger.LogInformation("Start moving TypedBaseItem.");
|
||||
const string typedBaseItemsQuery = """
|
||||
SELECT guid, type, data, StartDate, EndDate, ChannelId, IsMovie,
|
||||
IsSeries, EpisodeTitle, IsRepeat, CommunityRating, CustomRating, IndexNumber, IsLocked, PreferredMetadataLanguage,
|
||||
PreferredMetadataCountryCode, Width, Height, DateLastRefreshed, Name, Path, PremiereDate, Overview, ParentIndexNumber,
|
||||
ProductionYear, OfficialRating, ForcedSortName, RunTimeTicks, Size, DateCreated, DateModified, Genres, ParentId, TopParentId,
|
||||
Audio, ExternalServiceId, IsInMixedFolder, DateLastSaved, LockedFields, Studios, Tags, TrailerTypes, OriginalTitle, PrimaryVersionId,
|
||||
DateLastMediaAdded, Album, LUFS, NormalizationGain, CriticRating, IsVirtualItem, SeriesName, UserDataKey, SeasonName, SeasonId, SeriesId,
|
||||
PresentationUniqueKey, InheritedParentalRatingValue, ExternalSeriesId, Tagline, ProviderIds, Images, ProductionLocations, ExtraIds, TotalBitrate,
|
||||
ExtraType, Artists, AlbumArtists, ExternalId, SeriesPresentationUniqueKey, ShowId, OwnerId, MediaType, SortName, CleanName, UnratedType FROM TypedBaseItems
|
||||
""";
|
||||
dbContext.BaseItems.ExecuteDelete();
|
||||
using (var operation = GetPreparedDbContext("Cleanup database"))
|
||||
{
|
||||
operation.JellyfinDbContext.BaseItems.ExecuteDelete();
|
||||
operation.JellyfinDbContext.ItemValues.ExecuteDelete();
|
||||
operation.JellyfinDbContext.UserData.ExecuteDelete();
|
||||
operation.JellyfinDbContext.MediaStreamInfos.ExecuteDelete();
|
||||
operation.JellyfinDbContext.Peoples.ExecuteDelete();
|
||||
operation.JellyfinDbContext.PeopleBaseItemMap.ExecuteDelete();
|
||||
operation.JellyfinDbContext.Chapters.ExecuteDelete();
|
||||
operation.JellyfinDbContext.AncestorIds.ExecuteDelete();
|
||||
}
|
||||
|
||||
var legacyBaseItemWithUserKeys = new Dictionary<string, BaseItemEntity>();
|
||||
foreach (SqliteDataReader dto in connection.Query(typedBaseItemsQuery))
|
||||
connection.Open();
|
||||
|
||||
var baseItemIds = new HashSet<Guid>();
|
||||
using (var operation = GetPreparedDbContext("moving TypedBaseItem"))
|
||||
{
|
||||
var baseItem = GetItem(dto);
|
||||
dbContext.BaseItems.Add(baseItem.BaseItem);
|
||||
foreach (var dataKey in baseItem.LegacyUserDataKey)
|
||||
const string typedBaseItemsQuery =
|
||||
"""
|
||||
SELECT guid, type, data, StartDate, EndDate, ChannelId, IsMovie,
|
||||
IsSeries, EpisodeTitle, IsRepeat, CommunityRating, CustomRating, IndexNumber, IsLocked, PreferredMetadataLanguage,
|
||||
PreferredMetadataCountryCode, Width, Height, DateLastRefreshed, Name, Path, PremiereDate, Overview, ParentIndexNumber,
|
||||
ProductionYear, OfficialRating, ForcedSortName, RunTimeTicks, Size, DateCreated, DateModified, Genres, ParentId, TopParentId,
|
||||
Audio, ExternalServiceId, IsInMixedFolder, DateLastSaved, LockedFields, Studios, Tags, TrailerTypes, OriginalTitle, PrimaryVersionId,
|
||||
DateLastMediaAdded, Album, LUFS, NormalizationGain, CriticRating, IsVirtualItem, SeriesName, UserDataKey, SeasonName, SeasonId, SeriesId,
|
||||
PresentationUniqueKey, InheritedParentalRatingValue, ExternalSeriesId, Tagline, ProviderIds, Images, ProductionLocations, ExtraIds, TotalBitrate,
|
||||
ExtraType, Artists, AlbumArtists, ExternalId, SeriesPresentationUniqueKey, ShowId, OwnerId, MediaType, SortName, CleanName, UnratedType FROM TypedBaseItems
|
||||
""";
|
||||
using (new TrackedMigrationStep("Loading TypedBaseItems", _logger))
|
||||
{
|
||||
legacyBaseItemWithUserKeys[dataKey] = baseItem.BaseItem;
|
||||
foreach (SqliteDataReader dto in connection.Query(typedBaseItemsQuery))
|
||||
{
|
||||
var baseItem = GetItem(dto);
|
||||
operation.JellyfinDbContext.BaseItems.Add(baseItem.BaseItem);
|
||||
baseItemIds.Add(baseItem.BaseItem.Id);
|
||||
foreach (var dataKey in baseItem.LegacyUserDataKey)
|
||||
{
|
||||
legacyBaseItemWithUserKeys[dataKey] = baseItem.BaseItem;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.BaseItems.Local.Count} BaseItem entries", _logger))
|
||||
{
|
||||
operation.JellyfinDbContext.SaveChanges();
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogInformation("Try saving {0} BaseItem entries.", dbContext.BaseItems.Local.Count);
|
||||
dbContext.SaveChanges();
|
||||
migrationTotalTime += stopwatch.Elapsed;
|
||||
_logger.LogInformation("Saving BaseItems entries took {0}.", stopwatch.Elapsed);
|
||||
stopwatch.Restart();
|
||||
|
||||
_logger.LogInformation("Start moving ItemValues.");
|
||||
// do not migrate inherited types as they are now properly mapped in search and lookup.
|
||||
const string itemValueQuery =
|
||||
"""
|
||||
SELECT ItemId, Type, Value, CleanValue FROM ItemValues
|
||||
WHERE Type <> 6 AND EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = ItemValues.ItemId)
|
||||
""";
|
||||
dbContext.ItemValues.ExecuteDelete();
|
||||
|
||||
// EFCores local lookup sucks. We cannot use context.ItemValues.Local here because its just super slow.
|
||||
var localItems = new Dictionary<(int Type, string CleanValue), (Database.Implementations.Entities.ItemValue ItemValue, List<Guid> ItemIds)>();
|
||||
|
||||
foreach (SqliteDataReader dto in connection.Query(itemValueQuery))
|
||||
using (var operation = GetPreparedDbContext("moving ItemValues"))
|
||||
{
|
||||
var itemId = dto.GetGuid(0);
|
||||
var entity = GetItemValue(dto);
|
||||
var key = ((int)entity.Type, entity.CleanValue);
|
||||
if (!localItems.TryGetValue(key, out var existing))
|
||||
// do not migrate inherited types as they are now properly mapped in search and lookup.
|
||||
const string itemValueQuery =
|
||||
"""
|
||||
SELECT ItemId, Type, Value, CleanValue FROM ItemValues
|
||||
WHERE Type <> 6 AND EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = ItemValues.ItemId)
|
||||
""";
|
||||
|
||||
// EFCores local lookup sucks. We cannot use context.ItemValues.Local here because its just super slow.
|
||||
var localItems = new Dictionary<(int Type, string CleanValue), (Database.Implementations.Entities.ItemValue ItemValue, List<Guid> ItemIds)>();
|
||||
using (new TrackedMigrationStep("loading ItemValues", _logger))
|
||||
{
|
||||
localItems[key] = existing = (entity, []);
|
||||
foreach (SqliteDataReader dto in connection.Query(itemValueQuery))
|
||||
{
|
||||
var itemId = dto.GetGuid(0);
|
||||
var entity = GetItemValue(dto);
|
||||
var key = ((int)entity.Type, entity.CleanValue);
|
||||
if (!localItems.TryGetValue(key, out var existing))
|
||||
{
|
||||
localItems[key] = existing = (entity, []);
|
||||
}
|
||||
|
||||
existing.ItemIds.Add(itemId);
|
||||
}
|
||||
|
||||
foreach (var item in localItems)
|
||||
{
|
||||
operation.JellyfinDbContext.ItemValues.Add(item.Value.ItemValue);
|
||||
operation.JellyfinDbContext.ItemValuesMap.AddRange(item.Value.ItemIds.Distinct().Select(f => new ItemValueMap()
|
||||
{
|
||||
Item = null!,
|
||||
ItemValue = null!,
|
||||
ItemId = f,
|
||||
ItemValueId = item.Value.ItemValue.ItemValueId
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
existing.ItemIds.Add(itemId);
|
||||
using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.ItemValues.Local.Count} ItemValues entries", _logger))
|
||||
{
|
||||
operation.JellyfinDbContext.SaveChanges();
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var item in localItems)
|
||||
using (var operation = GetPreparedDbContext("moving UserData"))
|
||||
{
|
||||
dbContext.ItemValues.Add(item.Value.ItemValue);
|
||||
dbContext.ItemValuesMap.AddRange(item.Value.ItemIds.Distinct().Select(f => new ItemValueMap()
|
||||
var queryResult = connection.Query(
|
||||
"""
|
||||
SELECT key, userId, rating, played, playCount, isFavorite, playbackPositionTicks, lastPlayedDate, AudioStreamIndex, SubtitleStreamIndex FROM UserDatas
|
||||
|
||||
WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.UserDataKey = UserDatas.key)
|
||||
""");
|
||||
|
||||
using (new TrackedMigrationStep("loading UserData", _logger))
|
||||
{
|
||||
Item = null!,
|
||||
ItemValue = null!,
|
||||
ItemId = f,
|
||||
ItemValueId = item.Value.ItemValue.ItemValueId
|
||||
}));
|
||||
}
|
||||
var users = operation.JellyfinDbContext.Users.AsNoTracking().ToImmutableArray();
|
||||
var userIdBlacklist = new HashSet<int>();
|
||||
|
||||
_logger.LogInformation("Try saving {0} ItemValues entries.", dbContext.ItemValues.Local.Count);
|
||||
dbContext.SaveChanges();
|
||||
migrationTotalTime += stopwatch.Elapsed;
|
||||
_logger.LogInformation("Saving People ItemValues took {0}.", stopwatch.Elapsed);
|
||||
stopwatch.Restart();
|
||||
foreach (var entity in queryResult)
|
||||
{
|
||||
var userData = GetUserData(users, entity, userIdBlacklist);
|
||||
if (userData is null)
|
||||
{
|
||||
var userDataId = entity.GetString(0);
|
||||
var internalUserId = entity.GetInt32(1);
|
||||
|
||||
_logger.LogInformation("Start moving UserData.");
|
||||
var queryResult = connection.Query("""
|
||||
SELECT key, userId, rating, played, playCount, isFavorite, playbackPositionTicks, lastPlayedDate, AudioStreamIndex, SubtitleStreamIndex FROM UserDatas
|
||||
if (!userIdBlacklist.Contains(internalUserId))
|
||||
{
|
||||
_logger.LogError("Was not able to migrate user data with key {0} because its id {InternalId} does not match any existing user.", userDataId, internalUserId);
|
||||
userIdBlacklist.Add(internalUserId);
|
||||
}
|
||||
|
||||
WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.UserDataKey = UserDatas.key)
|
||||
""");
|
||||
continue;
|
||||
}
|
||||
|
||||
dbContext.UserData.ExecuteDelete();
|
||||
if (!legacyBaseItemWithUserKeys.TryGetValue(userData.CustomDataKey!, out var refItem))
|
||||
{
|
||||
_logger.LogError("Was not able to migrate user data with key {0} because it does not reference a valid BaseItem.", entity.GetString(0));
|
||||
continue;
|
||||
}
|
||||
|
||||
var users = dbContext.Users.AsNoTracking().ToImmutableArray();
|
||||
userData.ItemId = refItem.Id;
|
||||
operation.JellyfinDbContext.UserData.Add(userData);
|
||||
}
|
||||
|
||||
foreach (var entity in queryResult)
|
||||
{
|
||||
var userData = GetUserData(users, entity);
|
||||
if (userData is null)
|
||||
{
|
||||
_logger.LogError("Was not able to migrate user data with key {0}", entity.GetString(0));
|
||||
continue;
|
||||
users.Clear();
|
||||
}
|
||||
|
||||
if (!legacyBaseItemWithUserKeys.TryGetValue(userData.CustomDataKey!, out var refItem))
|
||||
legacyBaseItemWithUserKeys.Clear();
|
||||
|
||||
using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.UserData.Local.Count} UserData entries", _logger))
|
||||
{
|
||||
_logger.LogError("Was not able to migrate user data with key {0} because it does not reference a valid BaseItem.", entity.GetString(0));
|
||||
continue;
|
||||
operation.JellyfinDbContext.SaveChanges();
|
||||
}
|
||||
}
|
||||
|
||||
using (var operation = GetPreparedDbContext("moving MediaStreamInfos"))
|
||||
{
|
||||
const string mediaStreamQuery =
|
||||
"""
|
||||
SELECT ItemId, StreamIndex, StreamType, Codec, Language, ChannelLayout, Profile, AspectRatio, Path,
|
||||
IsInterlaced, BitRate, Channels, SampleRate, IsDefault, IsForced, IsExternal, Height, Width,
|
||||
AverageFrameRate, RealFrameRate, Level, PixelFormat, BitDepth, IsAnamorphic, RefFrames, CodecTag,
|
||||
Comment, NalLengthSize, IsAvc, Title, TimeBase, CodecTimeBase, ColorPrimaries, ColorSpace, ColorTransfer,
|
||||
DvVersionMajor, DvVersionMinor, DvProfile, DvLevel, RpuPresentFlag, ElPresentFlag, BlPresentFlag, DvBlSignalCompatibilityId, IsHearingImpaired
|
||||
FROM MediaStreams
|
||||
WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = MediaStreams.ItemId)
|
||||
""";
|
||||
|
||||
using (new TrackedMigrationStep("loading MediaStreamInfos", _logger))
|
||||
{
|
||||
foreach (SqliteDataReader dto in connection.Query(mediaStreamQuery))
|
||||
{
|
||||
operation.JellyfinDbContext.MediaStreamInfos.Add(GetMediaStream(dto));
|
||||
}
|
||||
}
|
||||
|
||||
userData.ItemId = refItem.Id;
|
||||
dbContext.UserData.Add(userData);
|
||||
}
|
||||
|
||||
users.Clear();
|
||||
legacyBaseItemWithUserKeys.Clear();
|
||||
_logger.LogInformation("Try saving {0} UserData entries.", dbContext.UserData.Local.Count);
|
||||
dbContext.SaveChanges();
|
||||
|
||||
_logger.LogInformation("Start moving MediaStreamInfos.");
|
||||
const string mediaStreamQuery = """
|
||||
SELECT ItemId, StreamIndex, StreamType, Codec, Language, ChannelLayout, Profile, AspectRatio, Path,
|
||||
IsInterlaced, BitRate, Channels, SampleRate, IsDefault, IsForced, IsExternal, Height, Width,
|
||||
AverageFrameRate, RealFrameRate, Level, PixelFormat, BitDepth, IsAnamorphic, RefFrames, CodecTag,
|
||||
Comment, NalLengthSize, IsAvc, Title, TimeBase, CodecTimeBase, ColorPrimaries, ColorSpace, ColorTransfer,
|
||||
DvVersionMajor, DvVersionMinor, DvProfile, DvLevel, RpuPresentFlag, ElPresentFlag, BlPresentFlag, DvBlSignalCompatibilityId, IsHearingImpaired
|
||||
FROM MediaStreams
|
||||
WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = MediaStreams.ItemId)
|
||||
""";
|
||||
dbContext.MediaStreamInfos.ExecuteDelete();
|
||||
|
||||
foreach (SqliteDataReader dto in connection.Query(mediaStreamQuery))
|
||||
{
|
||||
dbContext.MediaStreamInfos.Add(GetMediaStream(dto));
|
||||
}
|
||||
|
||||
_logger.LogInformation("Try saving {0} MediaStreamInfos entries.", dbContext.MediaStreamInfos.Local.Count);
|
||||
dbContext.SaveChanges();
|
||||
|
||||
migrationTotalTime += stopwatch.Elapsed;
|
||||
_logger.LogInformation("Saving MediaStreamInfos entries took {0}.", stopwatch.Elapsed);
|
||||
stopwatch.Restart();
|
||||
|
||||
_logger.LogInformation("Start moving People.");
|
||||
const string personsQuery = """
|
||||
SELECT ItemId, Name, Role, PersonType, SortOrder FROM People
|
||||
WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = People.ItemId)
|
||||
""";
|
||||
dbContext.Peoples.ExecuteDelete();
|
||||
dbContext.PeopleBaseItemMap.ExecuteDelete();
|
||||
|
||||
var peopleCache = new Dictionary<string, (People Person, List<PeopleBaseItemMap> Items)>();
|
||||
var baseItemIds = dbContext.BaseItems.Select(b => b.Id).ToHashSet();
|
||||
|
||||
foreach (SqliteDataReader reader in connection.Query(personsQuery))
|
||||
{
|
||||
var itemId = reader.GetGuid(0);
|
||||
if (!baseItemIds.Contains(itemId))
|
||||
using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.MediaStreamInfos.Local.Count} MediaStreamInfos entries", _logger))
|
||||
{
|
||||
_logger.LogError("Dont save person {0} because its not in use by any BaseItem", reader.GetString(1));
|
||||
continue;
|
||||
operation.JellyfinDbContext.SaveChanges();
|
||||
}
|
||||
}
|
||||
|
||||
using (var operation = GetPreparedDbContext("moving People"))
|
||||
{
|
||||
const string personsQuery =
|
||||
"""
|
||||
SELECT ItemId, Name, Role, PersonType, SortOrder FROM People
|
||||
WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = People.ItemId)
|
||||
""";
|
||||
|
||||
var peopleCache = new Dictionary<string, (People Person, List<PeopleBaseItemMap> Items)>();
|
||||
|
||||
using (new TrackedMigrationStep("loading People", _logger))
|
||||
{
|
||||
foreach (SqliteDataReader reader in connection.Query(personsQuery))
|
||||
{
|
||||
var itemId = reader.GetGuid(0);
|
||||
if (!baseItemIds.Contains(itemId))
|
||||
{
|
||||
_logger.LogError("Dont save person {0} because its not in use by any BaseItem", reader.GetString(1));
|
||||
continue;
|
||||
}
|
||||
|
||||
var entity = GetPerson(reader);
|
||||
if (!peopleCache.TryGetValue(entity.Name, out var personCache))
|
||||
{
|
||||
peopleCache[entity.Name] = personCache = (entity, []);
|
||||
}
|
||||
|
||||
if (reader.TryGetString(2, out var role))
|
||||
{
|
||||
}
|
||||
|
||||
int? sortOrder = reader.IsDBNull(4) ? null : reader.GetInt32(4);
|
||||
|
||||
personCache.Items.Add(new PeopleBaseItemMap()
|
||||
{
|
||||
Item = null!,
|
||||
ItemId = itemId,
|
||||
People = null!,
|
||||
PeopleId = personCache.Person.Id,
|
||||
ListOrder = sortOrder,
|
||||
SortOrder = sortOrder,
|
||||
Role = role
|
||||
});
|
||||
}
|
||||
|
||||
baseItemIds.Clear();
|
||||
|
||||
foreach (var item in peopleCache)
|
||||
{
|
||||
operation.JellyfinDbContext.Peoples.Add(item.Value.Person);
|
||||
operation.JellyfinDbContext.PeopleBaseItemMap.AddRange(item.Value.Items.DistinctBy(e => (e.ItemId, e.PeopleId)));
|
||||
}
|
||||
|
||||
peopleCache.Clear();
|
||||
}
|
||||
|
||||
var entity = GetPerson(reader);
|
||||
if (!peopleCache.TryGetValue(entity.Name, out var personCache))
|
||||
using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.Peoples.Local.Count} People entries and {operation.JellyfinDbContext.PeopleBaseItemMap.Local.Count} maps", _logger))
|
||||
{
|
||||
peopleCache[entity.Name] = personCache = (entity, []);
|
||||
operation.JellyfinDbContext.SaveChanges();
|
||||
}
|
||||
}
|
||||
|
||||
using (var operation = GetPreparedDbContext("moving Chapters"))
|
||||
{
|
||||
const string chapterQuery =
|
||||
"""
|
||||
SELECT ItemId,StartPositionTicks,Name,ImagePath,ImageDateModified,ChapterIndex from Chapters2
|
||||
WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = Chapters2.ItemId)
|
||||
""";
|
||||
|
||||
using (new TrackedMigrationStep("loading Chapters", _logger))
|
||||
{
|
||||
foreach (SqliteDataReader dto in connection.Query(chapterQuery))
|
||||
{
|
||||
var chapter = GetChapter(dto);
|
||||
operation.JellyfinDbContext.Chapters.Add(chapter);
|
||||
}
|
||||
}
|
||||
|
||||
if (reader.TryGetString(2, out var role))
|
||||
using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.Chapters.Local.Count} Chapters entries", _logger))
|
||||
{
|
||||
operation.JellyfinDbContext.SaveChanges();
|
||||
}
|
||||
}
|
||||
|
||||
using (var operation = GetPreparedDbContext("moving AncestorIds"))
|
||||
{
|
||||
const string ancestorIdsQuery =
|
||||
"""
|
||||
SELECT ItemId, AncestorId, AncestorIdText FROM AncestorIds
|
||||
WHERE
|
||||
EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = AncestorIds.ItemId)
|
||||
AND
|
||||
EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = AncestorIds.AncestorId)
|
||||
""";
|
||||
|
||||
using (new TrackedMigrationStep("loading AncestorIds", _logger))
|
||||
{
|
||||
foreach (SqliteDataReader dto in connection.Query(ancestorIdsQuery))
|
||||
{
|
||||
var ancestorId = GetAncestorId(dto);
|
||||
operation.JellyfinDbContext.AncestorIds.Add(ancestorId);
|
||||
}
|
||||
}
|
||||
|
||||
int? sortOrder = reader.IsDBNull(4) ? null : reader.GetInt32(4);
|
||||
|
||||
personCache.Items.Add(new PeopleBaseItemMap()
|
||||
using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.AncestorIds.Local.Count} AncestorId entries", _logger))
|
||||
{
|
||||
Item = null!,
|
||||
ItemId = itemId,
|
||||
People = null!,
|
||||
PeopleId = personCache.Person.Id,
|
||||
ListOrder = sortOrder,
|
||||
SortOrder = sortOrder,
|
||||
Role = role
|
||||
});
|
||||
operation.JellyfinDbContext.SaveChanges();
|
||||
}
|
||||
}
|
||||
|
||||
baseItemIds.Clear();
|
||||
|
||||
foreach (var item in peopleCache)
|
||||
{
|
||||
dbContext.Peoples.Add(item.Value.Person);
|
||||
dbContext.PeopleBaseItemMap.AddRange(item.Value.Items.DistinctBy(e => (e.ItemId, e.PeopleId)));
|
||||
}
|
||||
|
||||
peopleCache.Clear();
|
||||
|
||||
_logger.LogInformation("Try saving {0} People entries.", dbContext.Peoples.Local.Count);
|
||||
dbContext.SaveChanges();
|
||||
migrationTotalTime += stopwatch.Elapsed;
|
||||
_logger.LogInformation("Saving People entries took {0}.", stopwatch.Elapsed);
|
||||
stopwatch.Restart();
|
||||
|
||||
_logger.LogInformation("Start moving Chapters.");
|
||||
const string chapterQuery = """
|
||||
SELECT ItemId,StartPositionTicks,Name,ImagePath,ImageDateModified,ChapterIndex from Chapters2
|
||||
WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = Chapters2.ItemId)
|
||||
""";
|
||||
dbContext.Chapters.ExecuteDelete();
|
||||
|
||||
foreach (SqliteDataReader dto in connection.Query(chapterQuery))
|
||||
{
|
||||
var chapter = GetChapter(dto);
|
||||
dbContext.Chapters.Add(chapter);
|
||||
}
|
||||
|
||||
_logger.LogInformation("Try saving {0} Chapters entries.", dbContext.Chapters.Local.Count);
|
||||
dbContext.SaveChanges();
|
||||
migrationTotalTime += stopwatch.Elapsed;
|
||||
_logger.LogInformation("Saving Chapters took {0}.", stopwatch.Elapsed);
|
||||
stopwatch.Restart();
|
||||
|
||||
_logger.LogInformation("Start moving AncestorIds.");
|
||||
const string ancestorIdsQuery = """
|
||||
SELECT ItemId, AncestorId, AncestorIdText FROM AncestorIds
|
||||
WHERE
|
||||
EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = AncestorIds.ItemId)
|
||||
AND
|
||||
EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = AncestorIds.AncestorId)
|
||||
""";
|
||||
dbContext.AncestorIds.ExecuteDelete();
|
||||
|
||||
foreach (SqliteDataReader dto in connection.Query(ancestorIdsQuery))
|
||||
{
|
||||
var ancestorId = GetAncestorId(dto);
|
||||
dbContext.AncestorIds.Add(ancestorId);
|
||||
}
|
||||
|
||||
_logger.LogInformation("Try saving {0} AncestorIds entries.", dbContext.AncestorIds.Local.Count);
|
||||
|
||||
dbContext.SaveChanges();
|
||||
migrationTotalTime += stopwatch.Elapsed;
|
||||
_logger.LogInformation("Saving AncestorIds took {0}.", stopwatch.Elapsed);
|
||||
stopwatch.Restart();
|
||||
|
||||
connection.Close();
|
||||
|
||||
_logger.LogInformation("Migration of the Library.db done.");
|
||||
_logger.LogInformation("Move {0} to {1}.", libraryDbPath, libraryDbPath + ".old");
|
||||
_logger.LogInformation("Migrating Library db took {0}.", fullOperationTimer.Elapsed);
|
||||
|
||||
SqliteConnection.ClearAllPools();
|
||||
|
||||
_logger.LogInformation("Move {0} to {1}.", libraryDbPath, libraryDbPath + ".old");
|
||||
File.Move(libraryDbPath, libraryDbPath + ".old", true);
|
||||
|
||||
_logger.LogInformation("Migrating Library db took {0}.", migrationTotalTime);
|
||||
|
||||
_jellyfinDatabaseProvider.RunScheduledOptimisation(CancellationToken.None).ConfigureAwait(false).GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
private UserData? GetUserData(ImmutableArray<User> users, SqliteDataReader dto)
|
||||
private DatabaseMigrationStep GetPreparedDbContext(string operationName)
|
||||
{
|
||||
var dbContext = _provider.CreateDbContext();
|
||||
dbContext.ChangeTracker.AutoDetectChangesEnabled = false;
|
||||
dbContext.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking;
|
||||
return new DatabaseMigrationStep(dbContext, operationName, _logger);
|
||||
}
|
||||
|
||||
private UserData? GetUserData(ImmutableArray<User> users, SqliteDataReader dto, HashSet<int> userIdBlacklist)
|
||||
{
|
||||
var internalUserId = dto.GetInt32(1);
|
||||
var user = users.FirstOrDefault(e => e.InternalId == internalUserId);
|
||||
|
||||
if (user is null)
|
||||
{
|
||||
if (userIdBlacklist.Contains(internalUserId))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
_logger.LogError("Tried to find user with index '{Idx}' but there are only '{MaxIdx}' users.", internalUserId, users.Length);
|
||||
return null;
|
||||
}
|
||||
@ -1214,4 +1269,58 @@ internal class MigrateLibraryDb : IDatabaseMigrationRoutine
|
||||
|
||||
return image;
|
||||
}
|
||||
|
||||
private class TrackedMigrationStep : IDisposable
|
||||
{
|
||||
private readonly string _operationName;
|
||||
private readonly ILogger _logger;
|
||||
private readonly Stopwatch _operationTimer;
|
||||
private bool _disposed;
|
||||
|
||||
public TrackedMigrationStep(string operationName, ILogger logger)
|
||||
{
|
||||
_operationName = operationName;
|
||||
_logger = logger;
|
||||
_operationTimer = Stopwatch.StartNew();
|
||||
logger.LogInformation("Start {OperationName}", operationName);
|
||||
}
|
||||
|
||||
public bool Disposed
|
||||
{
|
||||
get => _disposed;
|
||||
set => _disposed = value;
|
||||
}
|
||||
|
||||
public virtual void Dispose()
|
||||
{
|
||||
if (Disposed)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
Disposed = true;
|
||||
_logger.LogInformation("{OperationName} took '{Time}'", _operationName, _operationTimer.Elapsed);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class DatabaseMigrationStep : TrackedMigrationStep
|
||||
{
|
||||
public DatabaseMigrationStep(JellyfinDbContext jellyfinDbContext, string operationName, ILogger logger) : base(operationName, logger)
|
||||
{
|
||||
JellyfinDbContext = jellyfinDbContext;
|
||||
}
|
||||
|
||||
public JellyfinDbContext JellyfinDbContext { get; }
|
||||
|
||||
public override void Dispose()
|
||||
{
|
||||
if (Disposed)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
JellyfinDbContext.Dispose();
|
||||
base.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user