From ce16651dbd43908770180054c4525e2188d95ed0 Mon Sep 17 00:00:00 2001 From: Vasily Date: Fri, 15 May 2020 01:55:00 +0300 Subject: [PATCH 1/5] Fix a check broken by https://github.com/jellyfin/jellyfin/pull/2105 --- Emby.Naming/Video/VideoListResolver.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Emby.Naming/Video/VideoListResolver.cs b/Emby.Naming/Video/VideoListResolver.cs index 7f755fd25e..948fe037b5 100644 --- a/Emby.Naming/Video/VideoListResolver.cs +++ b/Emby.Naming/Video/VideoListResolver.cs @@ -227,7 +227,7 @@ namespace Emby.Naming.Video } return remainingFiles - .Where(i => i.ExtraType == null) + .Where(i => i.ExtraType != null) .Where(i => baseNames.Any(b => i.FileNameWithoutExtension.StartsWith(b, StringComparison.OrdinalIgnoreCase))) .ToList(); From 7c571345353417db6990700b350fd22a2778ee4f Mon Sep 17 00:00:00 2001 From: Vasily Date: Fri, 15 May 2020 02:30:28 +0300 Subject: [PATCH 2/5] Implement a cleanup migration --- Jellyfin.Server/Migrations/MigrationRunner.cs | 3 +- .../Migrations/Routines/RemoveBuggedExtras.cs | 50 +++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 Jellyfin.Server/Migrations/Routines/RemoveBuggedExtras.cs diff --git a/Jellyfin.Server/Migrations/MigrationRunner.cs b/Jellyfin.Server/Migrations/MigrationRunner.cs index ca17482820..7a881be0f6 100644 --- a/Jellyfin.Server/Migrations/MigrationRunner.cs +++ b/Jellyfin.Server/Migrations/MigrationRunner.cs @@ -17,7 +17,8 @@ namespace Jellyfin.Server.Migrations private static readonly Type[] _migrationTypes = { typeof(Routines.DisableTranscodingThrottling), - typeof(Routines.CreateUserLoggingConfigFile) + typeof(Routines.CreateUserLoggingConfigFile), + typeof(Routines.RemoveBuggedExtras) }; /// diff --git a/Jellyfin.Server/Migrations/Routines/RemoveBuggedExtras.cs b/Jellyfin.Server/Migrations/Routines/RemoveBuggedExtras.cs new file mode 100644 index 0000000000..7e45f99f94 --- /dev/null +++ b/Jellyfin.Server/Migrations/Routines/RemoveBuggedExtras.cs @@ -0,0 +1,50 @@ +using System; +using System.IO; + +using MediaBrowser.Controller; +using Microsoft.Extensions.Logging; +using SQLitePCL.pretty; + +namespace Jellyfin.Server.Migrations.Routines +{ + /// + /// Remove duplicate entries which were caused by a bug where a file was considered to be an "Extra" to itself. + /// + internal class RemoveBuggedExtras : IMigrationRoutine + { + private const string DbFilename = "library.db"; + private readonly ILogger _logger; + private readonly IServerApplicationPaths _paths; + + public RemoveBuggedExtras(ILogger logger, IServerApplicationPaths paths) + { + _logger = logger; + _paths = paths; + } + + /// + public Guid Id => Guid.Parse("{ACBE17B7-8435-4A83-8B64-6FCF162CB9BD}"); + + /// + public string Name => "RemoveBuggedExtras"; + + /// + public void Perform() + { + var dataPath = _paths.DataPath; + using (var connection = SQLite3.Open( + Path.Combine(dataPath, DbFilename), + ConnectionFlags.ReadWrite, + null)) + { + var queryResult = connection.Query("SELECT t1.Path FROM TypedBaseItems AS t1, TypedBaseItems AS t2 WHERE t1.Path=t2.Path AND t1.Type!=t2.Type AND t1.Type='MediaBrowser.Controller.Entities.Video'"); + var bads = string.Join(", ", queryResult.SelectScalarString()); + if (bads.Length != 0) + { + _logger.LogInformation("Removing found duplicated extras for the following items: {0}", bads); + connection.Execute("DELETE FROM TypedBaseItems WHERE rowid IN (SELECT t1.rowid FROM TypedBaseItems AS t1, TypedBaseItems AS t2 WHERE t1.Path=t2.Path AND t1.Type!=t2.Type AND t1.Type='MediaBrowser.Controller.Entities.Video')"); + } + } + } + } +} From 034fe97eebb709d87d7642151d6e0e5ec5f2a391 Mon Sep 17 00:00:00 2001 From: Vasily Date: Fri, 15 May 2020 21:32:56 +0300 Subject: [PATCH 3/5] Apply suggestions from code review Co-authored-by: Mark Monteiro --- Jellyfin.Server/Migrations/Routines/RemoveBuggedExtras.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jellyfin.Server/Migrations/Routines/RemoveBuggedExtras.cs b/Jellyfin.Server/Migrations/Routines/RemoveBuggedExtras.cs index 7e45f99f94..512bec0bf7 100644 --- a/Jellyfin.Server/Migrations/Routines/RemoveBuggedExtras.cs +++ b/Jellyfin.Server/Migrations/Routines/RemoveBuggedExtras.cs @@ -10,7 +10,7 @@ namespace Jellyfin.Server.Migrations.Routines /// /// Remove duplicate entries which were caused by a bug where a file was considered to be an "Extra" to itself. /// - internal class RemoveBuggedExtras : IMigrationRoutine + internal class RemoveDuplicateExtras : IMigrationRoutine { private const string DbFilename = "library.db"; private readonly ILogger _logger; @@ -41,7 +41,7 @@ namespace Jellyfin.Server.Migrations.Routines var bads = string.Join(", ", queryResult.SelectScalarString()); if (bads.Length != 0) { - _logger.LogInformation("Removing found duplicated extras for the following items: {0}", bads); + _logger.LogInformation("Removing found duplicated extras for the following items: {DuplicateExtras}", bads); connection.Execute("DELETE FROM TypedBaseItems WHERE rowid IN (SELECT t1.rowid FROM TypedBaseItems AS t1, TypedBaseItems AS t2 WHERE t1.Path=t2.Path AND t1.Type!=t2.Type AND t1.Type='MediaBrowser.Controller.Entities.Video')"); } } From 43dc604e8739614752a0c4e8a4ff00af5d74085d Mon Sep 17 00:00:00 2001 From: Vasily Date: Fri, 15 May 2020 21:49:45 +0300 Subject: [PATCH 4/5] Fixed compilation, added backing db before removing extras --- Jellyfin.Server/Migrations/MigrationRunner.cs | 2 +- ...ggedExtras.cs => RemoveDuplicateExtras.cs} | 27 ++++++++++++++++--- 2 files changed, 25 insertions(+), 4 deletions(-) rename Jellyfin.Server/Migrations/Routines/{RemoveBuggedExtras.cs => RemoveDuplicateExtras.cs} (61%) diff --git a/Jellyfin.Server/Migrations/MigrationRunner.cs b/Jellyfin.Server/Migrations/MigrationRunner.cs index 7a881be0f6..3941700655 100644 --- a/Jellyfin.Server/Migrations/MigrationRunner.cs +++ b/Jellyfin.Server/Migrations/MigrationRunner.cs @@ -18,7 +18,7 @@ namespace Jellyfin.Server.Migrations { typeof(Routines.DisableTranscodingThrottling), typeof(Routines.CreateUserLoggingConfigFile), - typeof(Routines.RemoveBuggedExtras) + typeof(Routines.RemoveDuplicateExtras) }; /// diff --git a/Jellyfin.Server/Migrations/Routines/RemoveBuggedExtras.cs b/Jellyfin.Server/Migrations/Routines/RemoveDuplicateExtras.cs similarity index 61% rename from Jellyfin.Server/Migrations/Routines/RemoveBuggedExtras.cs rename to Jellyfin.Server/Migrations/Routines/RemoveDuplicateExtras.cs index 512bec0bf7..46de2d5c39 100644 --- a/Jellyfin.Server/Migrations/Routines/RemoveBuggedExtras.cs +++ b/Jellyfin.Server/Migrations/Routines/RemoveDuplicateExtras.cs @@ -1,4 +1,5 @@ using System; +using System.Globalization; using System.IO; using MediaBrowser.Controller; @@ -16,7 +17,7 @@ namespace Jellyfin.Server.Migrations.Routines private readonly ILogger _logger; private readonly IServerApplicationPaths _paths; - public RemoveBuggedExtras(ILogger logger, IServerApplicationPaths paths) + public RemoveDuplicateExtras(ILogger logger, IServerApplicationPaths paths) { _logger = logger; _paths = paths; @@ -26,14 +27,15 @@ namespace Jellyfin.Server.Migrations.Routines public Guid Id => Guid.Parse("{ACBE17B7-8435-4A83-8B64-6FCF162CB9BD}"); /// - public string Name => "RemoveBuggedExtras"; + public string Name => "RemoveDuplicateExtras"; /// public void Perform() { var dataPath = _paths.DataPath; + var dbPath = Path.Combine(dataPath, DbFilename); using (var connection = SQLite3.Open( - Path.Combine(dataPath, DbFilename), + dbPath, ConnectionFlags.ReadWrite, null)) { @@ -41,6 +43,25 @@ namespace Jellyfin.Server.Migrations.Routines var bads = string.Join(", ", queryResult.SelectScalarString()); if (bads.Length != 0) { + _logger.LogInformation("Found duplicate extras, making {Library} backup", DbFilename); + for (int i = 1; ; i++) + { + var bakPath = string.Format(CultureInfo.InvariantCulture, "{0}.bak{1}", dbPath, i); + if (!File.Exists(bakPath)) + { + try + { + File.Copy(dbPath, bakPath); + break; + } + catch (Exception ex) + { + _logger.LogError(ex, "Cannot make a backup of {Library}", DbFilename); + throw; + } + } + } + _logger.LogInformation("Removing found duplicated extras for the following items: {DuplicateExtras}", bads); connection.Execute("DELETE FROM TypedBaseItems WHERE rowid IN (SELECT t1.rowid FROM TypedBaseItems AS t1, TypedBaseItems AS t2 WHERE t1.Path=t2.Path AND t1.Type!=t2.Type AND t1.Type='MediaBrowser.Controller.Entities.Video')"); } From 6e68702799b2b3de9660babad6a66493d16fec72 Mon Sep 17 00:00:00 2001 From: Mark Monteiro Date: Fri, 15 May 2020 15:10:41 -0400 Subject: [PATCH 5/5] Do not run DELETE command if no extras are detected Also log a message if no extras were detected Also log the path used for the database backup Also add some comments to explain the migration --- .../Routines/RemoveDuplicateExtras.cs | 44 +++++++++++-------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/Jellyfin.Server/Migrations/Routines/RemoveDuplicateExtras.cs b/Jellyfin.Server/Migrations/Routines/RemoveDuplicateExtras.cs index 46de2d5c39..e955363881 100644 --- a/Jellyfin.Server/Migrations/Routines/RemoveDuplicateExtras.cs +++ b/Jellyfin.Server/Migrations/Routines/RemoveDuplicateExtras.cs @@ -39,32 +39,40 @@ namespace Jellyfin.Server.Migrations.Routines ConnectionFlags.ReadWrite, null)) { + // Query the database for the ids of duplicate extras var queryResult = connection.Query("SELECT t1.Path FROM TypedBaseItems AS t1, TypedBaseItems AS t2 WHERE t1.Path=t2.Path AND t1.Type!=t2.Type AND t1.Type='MediaBrowser.Controller.Entities.Video'"); var bads = string.Join(", ", queryResult.SelectScalarString()); - if (bads.Length != 0) + + // Do nothing if no duplicate extras were detected + if (bads.Length == 0) { - _logger.LogInformation("Found duplicate extras, making {Library} backup", DbFilename); - for (int i = 1; ; i++) + _logger.LogInformation("No duplicate extras detected, skipping migration."); + return; + } + + // Back up the database before deleting any entries + for (int i = 1; ; i++) + { + var bakPath = string.Format(CultureInfo.InvariantCulture, "{0}.bak{1}", dbPath, i); + if (!File.Exists(bakPath)) { - var bakPath = string.Format(CultureInfo.InvariantCulture, "{0}.bak{1}", dbPath, i); - if (!File.Exists(bakPath)) + try { - try - { - File.Copy(dbPath, bakPath); - break; - } - catch (Exception ex) - { - _logger.LogError(ex, "Cannot make a backup of {Library}", DbFilename); - throw; - } + File.Copy(dbPath, bakPath); + _logger.LogInformation("Library database backed up to {BackupPath}", bakPath); + break; + } + catch (Exception ex) + { + _logger.LogError(ex, "Cannot make a backup of {Library} at path {BackupPath}", DbFilename, bakPath); + throw; } } - - _logger.LogInformation("Removing found duplicated extras for the following items: {DuplicateExtras}", bads); - connection.Execute("DELETE FROM TypedBaseItems WHERE rowid IN (SELECT t1.rowid FROM TypedBaseItems AS t1, TypedBaseItems AS t2 WHERE t1.Path=t2.Path AND t1.Type!=t2.Type AND t1.Type='MediaBrowser.Controller.Entities.Video')"); } + + // Delete all duplicate extras + _logger.LogInformation("Removing found duplicated extras for the following items: {DuplicateExtras}", bads); + connection.Execute("DELETE FROM TypedBaseItems WHERE rowid IN (SELECT t1.rowid FROM TypedBaseItems AS t1, TypedBaseItems AS t2 WHERE t1.Path=t2.Path AND t1.Type!=t2.Type AND t1.Type='MediaBrowser.Controller.Entities.Video')"); } } }