From c2276b17cb196de44f1bbb42b91b000cde8fdc7f Mon Sep 17 00:00:00 2001 From: Gary Wilber Date: Wed, 30 Sep 2020 19:33:34 -0700 Subject: [PATCH 01/11] Increase library scan and metadata refresh speed --- .../ApplicationHost.cs | 1 + MediaBrowser.Controller/Entities/Folder.cs | 220 +++++++++++------- .../Library/TaskMethods.cs | 133 +++++++++++ .../Configuration/ServerConfiguration.cs | 12 + 4 files changed, 283 insertions(+), 83 deletions(-) create mode 100644 MediaBrowser.Controller/Library/TaskMethods.cs diff --git a/Emby.Server.Implementations/ApplicationHost.cs b/Emby.Server.Implementations/ApplicationHost.cs index 7a46fdf2e7..6d7239f724 100644 --- a/Emby.Server.Implementations/ApplicationHost.cs +++ b/Emby.Server.Implementations/ApplicationHost.cs @@ -758,6 +758,7 @@ namespace Emby.Server.Implementations BaseItem.FileSystem = _fileSystemManager; BaseItem.UserDataManager = Resolve(); BaseItem.ChannelManager = Resolve(); + TaskMethods.ConfigurationManager = ServerConfigurationManager; Video.LiveTvManager = Resolve(); Folder.UserViewManager = Resolve(); UserView.TVSeriesManager = Resolve(); diff --git a/MediaBrowser.Controller/Entities/Folder.cs b/MediaBrowser.Controller/Entities/Folder.cs index 901ea875bc..666455cff4 100644 --- a/MediaBrowser.Controller/Entities/Folder.cs +++ b/MediaBrowser.Controller/Entities/Folder.cs @@ -35,6 +35,46 @@ namespace MediaBrowser.Controller.Entities /// public class Folder : BaseItem { + /// + /// Contains constants used when reporting scan progress. + /// + private static class ProgressHelpers + { + /// + /// Reported after the folders immediate children are retrieved. + /// + public const int RetrievedChildren = 5; + + /// + /// Reported after add, updating, or deleting child items from the LibraryManager. + /// + public const int UpdatedChildItems = 10; + + /// + /// Reported once subfolders are scanned. + /// When scanning subfolders, the progress will be between [UpdatedItems, ScannedSubfolders]. + /// + public const int ScannedSubfolders = 50; + + /// + /// Reported once metadata is refreshed. + /// When refreshing metadata, the progress will be between [ScannedSubfolders, MetadataRefreshed]. + /// + public const int RefreshedMetadata = 100; + + /// + /// Gets the current progress given the previous step, next step, and progress in between. + /// + /// The previous progress step. + /// The next progress step. + /// The current progress step. + /// The progress. + public static double GetProgress(int previousProgressStep, int nextProgressStep, double currentProgress) + { + return previousProgressStep + ((nextProgressStep - previousProgressStep) * (currentProgress / 100)); + } + } + public static IUserViewManager UserViewManager { get; set; } /// @@ -327,11 +367,11 @@ namespace MediaBrowser.Controller.Entities return; } - progress.Report(5); + progress.Report(ProgressHelpers.RetrievedChildren); if (recursive) { - ProviderManager.OnRefreshProgress(this, 5); + ProviderManager.OnRefreshProgress(this, ProgressHelpers.RetrievedChildren); } // Build a dictionary of the current children we have now by Id so we can compare quickly and easily @@ -392,11 +432,11 @@ namespace MediaBrowser.Controller.Entities validChildrenNeedGeneration = true; } - progress.Report(10); + progress.Report(ProgressHelpers.UpdatedChildItems); if (recursive) { - ProviderManager.OnRefreshProgress(this, 10); + ProviderManager.OnRefreshProgress(this, ProgressHelpers.UpdatedChildItems); } cancellationToken.ThrowIfCancellationRequested(); @@ -406,11 +446,13 @@ namespace MediaBrowser.Controller.Entities var innerProgress = new ActionableProgress(); var folder = this; - innerProgress.RegisterAction(p => + innerProgress.RegisterAction(innerPercent => { - double newPct = 0.80 * p + 10; - progress.Report(newPct); - ProviderManager.OnRefreshProgress(folder, newPct); + var percent = ProgressHelpers.GetProgress(ProgressHelpers.UpdatedChildItems, ProgressHelpers.ScannedSubfolders, innerPercent); + + progress.Report(percent); + + ProviderManager.OnRefreshProgress(folder, percent); }); if (validChildrenNeedGeneration) @@ -424,11 +466,11 @@ namespace MediaBrowser.Controller.Entities if (refreshChildMetadata) { - progress.Report(90); + progress.Report(ProgressHelpers.ScannedSubfolders); if (recursive) { - ProviderManager.OnRefreshProgress(this, 90); + ProviderManager.OnRefreshProgress(this, ProgressHelpers.ScannedSubfolders); } var container = this as IMetadataContainer; @@ -436,13 +478,15 @@ namespace MediaBrowser.Controller.Entities var innerProgress = new ActionableProgress(); var folder = this; - innerProgress.RegisterAction(p => + innerProgress.RegisterAction(innerPercent => { - double newPct = 0.10 * p + 90; - progress.Report(newPct); + var percent = ProgressHelpers.GetProgress(ProgressHelpers.ScannedSubfolders, ProgressHelpers.RefreshedMetadata, innerPercent); + + progress.Report(percent); + if (recursive) { - ProviderManager.OnRefreshProgress(folder, newPct); + ProviderManager.OnRefreshProgress(folder, percent); } }); @@ -457,55 +501,37 @@ namespace MediaBrowser.Controller.Entities validChildren = Children.ToList(); } - await RefreshMetadataRecursive(validChildren, refreshOptions, recursive, innerProgress, cancellationToken); + await RefreshMetadataRecursive(validChildren, refreshOptions, recursive, innerProgress, cancellationToken).ConfigureAwait(false); } } } - private async Task RefreshMetadataRecursive(List children, MetadataRefreshOptions refreshOptions, bool recursive, IProgress progress, CancellationToken cancellationToken) + private Task RefreshMetadataRecursive(IList children, MetadataRefreshOptions refreshOptions, bool recursive, IProgress progress, CancellationToken cancellationToken) { - var numComplete = 0; - var count = children.Count; - double currentPercent = 0; + var progressableTasks = children + .Select, Task>>(child => + innerProgress => RefreshChildMetadata(child, refreshOptions, recursive && child.IsFolder, innerProgress, cancellationToken)) + .ToList(); - foreach (var child in children) - { - cancellationToken.ThrowIfCancellationRequested(); - - var innerProgress = new ActionableProgress(); - - // Avoid implicitly captured closure - var currentInnerPercent = currentPercent; - - innerProgress.RegisterAction(p => - { - double innerPercent = currentInnerPercent; - innerPercent += p / count; - progress.Report(innerPercent); - }); - - await RefreshChildMetadata(child, refreshOptions, recursive && child.IsFolder, innerProgress, cancellationToken) - .ConfigureAwait(false); - - numComplete++; - double percent = numComplete; - percent /= count; - percent *= 100; - currentPercent = percent; - - progress.Report(percent); - } + return RunTasks(progressableTasks, progress, cancellationToken); } private async Task RefreshAllMetadataForContainer(IMetadataContainer container, MetadataRefreshOptions refreshOptions, IProgress progress, CancellationToken cancellationToken) { - var series = container as Series; - if (series != null) - { - await series.RefreshMetadata(refreshOptions, cancellationToken).ConfigureAwait(false); - } + // limit the amount of concurrent metadata refreshes + await TaskMethods.RunThrottled( + TaskMethods.SharedThrottleId.RefreshMetadata, + async () => + { + var series = container as Series; + if (series != null) + { + await series.RefreshMetadata(refreshOptions, cancellationToken).ConfigureAwait(false); + } - await container.RefreshAllMetadata(refreshOptions, progress, cancellationToken).ConfigureAwait(false); + await container.RefreshAllMetadata(refreshOptions, progress, cancellationToken).ConfigureAwait(false); + }, + cancellationToken).ConfigureAwait(false); } private async Task RefreshChildMetadata(BaseItem child, MetadataRefreshOptions refreshOptions, bool recursive, IProgress progress, CancellationToken cancellationToken) @@ -520,12 +546,16 @@ namespace MediaBrowser.Controller.Entities { if (refreshOptions.RefreshItem(child)) { - await child.RefreshMetadata(refreshOptions, cancellationToken).ConfigureAwait(false); + // limit the amount of concurrent metadata refreshes + await TaskMethods.RunThrottled( + TaskMethods.SharedThrottleId.RefreshMetadata, + async () => await child.RefreshMetadata(refreshOptions, cancellationToken).ConfigureAwait(false), + cancellationToken).ConfigureAwait(false); } if (recursive && child is Folder folder) { - await folder.RefreshMetadataRecursive(folder.Children.ToList(), refreshOptions, true, progress, cancellationToken); + await folder.RefreshMetadataRecursive(folder.Children.ToList(), refreshOptions, true, progress, cancellationToken).ConfigureAwait(false); } } } @@ -538,39 +568,63 @@ namespace MediaBrowser.Controller.Entities /// The progress. /// The cancellation token. /// Task. - private async Task ValidateSubFolders(IList children, IDirectoryService directoryService, IProgress progress, CancellationToken cancellationToken) + private Task ValidateSubFolders(IList children, IDirectoryService directoryService, IProgress progress, CancellationToken cancellationToken) { - var numComplete = 0; - var count = children.Count; - double currentPercent = 0; + var progressableTasks = children + .Select, Task>>(child => + innerProgress => child.ValidateChildrenInternal(innerProgress, cancellationToken, true, false, null, directoryService)) + .ToList(); - foreach (var child in children) + return RunTasks(progressableTasks, progress, cancellationToken); + } + + /// + /// Runs a set of tasks concurrently with progress. + /// + /// A list of tasks. + /// The progress. + /// The cancellation token. + /// Task. + private async Task RunTasks(IList, Task>> tasks, IProgress progress, CancellationToken cancellationToken) + { + var childrenCount = tasks.Count; + var childrenProgress = new double[childrenCount]; + var actions = new Func[childrenCount]; + + void UpdateProgress() { - cancellationToken.ThrowIfCancellationRequested(); - - var innerProgress = new ActionableProgress(); - - // Avoid implicitly captured closure - var currentInnerPercent = currentPercent; - - innerProgress.RegisterAction(p => - { - double innerPercent = currentInnerPercent; - innerPercent += p / count; - progress.Report(innerPercent); - }); - - await child.ValidateChildrenInternal(innerProgress, cancellationToken, true, false, null, directoryService) - .ConfigureAwait(false); - - numComplete++; - double percent = numComplete; - percent /= count; - percent *= 100; - currentPercent = percent; - - progress.Report(percent); + progress.Report(childrenProgress.Average()); } + + for (var i = 0; i < childrenCount; i++) + { + var childIndex = i; + var child = tasks[childIndex]; + + actions[childIndex] = async () => + { + var innerProgress = new ActionableProgress(); + + innerProgress.RegisterAction(innerPercent => + { + // round the percent and only update progress if it changed to prevent excessive UpdateProgress calls + var innerPercentRounded = Math.Round(innerPercent); + if (childrenProgress[childIndex] != innerPercentRounded) + { + childrenProgress[childIndex] = innerPercentRounded; + UpdateProgress(); + } + }); + + await tasks[childIndex](innerProgress).ConfigureAwait(false); + + childrenProgress[childIndex] = 100; + + UpdateProgress(); + }; + } + + await TaskMethods.WhenAllThrottled(TaskMethods.SharedThrottleId.ScanFanout, actions, cancellationToken).ConfigureAwait(false); } /// diff --git a/MediaBrowser.Controller/Library/TaskMethods.cs b/MediaBrowser.Controller/Library/TaskMethods.cs new file mode 100644 index 0000000000..66bfbe0d9e --- /dev/null +++ b/MediaBrowser.Controller/Library/TaskMethods.cs @@ -0,0 +1,133 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using MediaBrowser.Controller.Configuration; + +namespace MediaBrowser.Controller.Library +{ + /// + /// Helper methods for running tasks concurrently. + /// + public static class TaskMethods + { + private static readonly int _processorCount = Environment.ProcessorCount; + + private static readonly ConcurrentDictionary _sharedThrottlers = new ConcurrentDictionary(); + + /// + /// Throttle id for sharing a concurrency limit. + /// + public enum SharedThrottleId + { + /// + /// Library scan fan out + /// + ScanFanout, + + /// + /// Refresh metadata + /// + RefreshMetadata, + } + + /// + /// Gets or sets the configuration manager. + /// + public static IServerConfigurationManager ConfigurationManager { get; set; } + + /// + /// Similiar to Task.WhenAll but only allows running a certain amount of tasks at the same time. + /// + /// The throttle id. Multiple calls to this method with the same throttle id will share a concurrency limit. + /// List of actions to run. + /// The cancellation token. + /// A representing the result of the asynchronous operation. + public static async Task WhenAllThrottled(SharedThrottleId throttleId, IEnumerable> actions, CancellationToken cancellationToken) + { + var taskThrottler = throttleId == SharedThrottleId.ScanFanout ? + new SemaphoreSlim(GetConcurrencyLimit(throttleId)) : + _sharedThrottlers.GetOrAdd(throttleId, id => new SemaphoreSlim(GetConcurrencyLimit(id))); + + try + { + var tasks = new List(); + + foreach (var action in actions) + { + await taskThrottler.WaitAsync(cancellationToken).ConfigureAwait(false); + + tasks.Add(Task.Run(async () => + { + try + { + await action().ConfigureAwait(false); + } + finally + { + taskThrottler.Release(); + } + })); + } + + await Task.WhenAll(tasks).ConfigureAwait(false); + } + finally + { + if (throttleId == SharedThrottleId.ScanFanout) + { + taskThrottler.Dispose(); + } + } + } + + /// + /// Runs a task within a given throttler. + /// + /// The throttle id. Multiple calls to this method with the same throttle id will share a concurrency limit. + /// The action to run. + /// The cancellation token. + /// A representing the result of the asynchronous operation. + public static async Task RunThrottled(SharedThrottleId throttleId, Func action, CancellationToken cancellationToken) + { + if (throttleId == SharedThrottleId.ScanFanout) + { + // just await the task instead + throw new InvalidOperationException("Invalid throttle id"); + } + + var taskThrottler = _sharedThrottlers.GetOrAdd(throttleId, id => new SemaphoreSlim(GetConcurrencyLimit(id))); + + await taskThrottler.WaitAsync(cancellationToken).ConfigureAwait(false); + + try + { + await action().ConfigureAwait(false); + } + finally + { + taskThrottler.Release(); + } + } + + /// + /// Get the concurrency limit for the given throttle id. + /// + /// The throttle id. + /// The concurrency limit. + private static int GetConcurrencyLimit(SharedThrottleId throttleId) + { + var concurrency = throttleId == SharedThrottleId.RefreshMetadata ? + ConfigurationManager.Configuration.LibraryMetadataRefreshConcurrency : + ConfigurationManager.Configuration.LibraryScanFanoutConcurrency; + + if (concurrency <= 0) + { + concurrency = _processorCount; + } + + return concurrency; + } + } +} diff --git a/MediaBrowser.Model/Configuration/ServerConfiguration.cs b/MediaBrowser.Model/Configuration/ServerConfiguration.cs index 8b78ad842e..14bfcbf9e2 100644 --- a/MediaBrowser.Model/Configuration/ServerConfiguration.cs +++ b/MediaBrowser.Model/Configuration/ServerConfiguration.cs @@ -271,6 +271,16 @@ namespace MediaBrowser.Model.Configuration /// public string[] KnownProxies { get; set; } + /// + /// Gets or sets the how the library scan fans out. + /// + public int LibraryScanFanoutConcurrency { get; set; } + + /// + /// Gets or sets the how many metadata refreshes can run concurrently. + /// + public int LibraryMetadataRefreshConcurrency { get; set; } + /// /// Initializes a new instance of the class. /// @@ -381,6 +391,8 @@ namespace MediaBrowser.Model.Configuration SlowResponseThresholdMs = 500; CorsHosts = new[] { "*" }; KnownProxies = Array.Empty(); + LibraryMetadataRefreshConcurrency = 0; + LibraryScanFanoutConcurrency = 0; } } From 8f2fbf7a99c5f67daf2a1589fa12b1287a076bed Mon Sep 17 00:00:00 2001 From: Gary Wilber Date: Thu, 1 Oct 2020 16:24:35 -0700 Subject: [PATCH 02/11] Switch to TPL dataflow for subfolder scan --- .../ApplicationHost.cs | 1 - MediaBrowser.Controller/Entities/Folder.cs | 173 +++++++++++------- .../Library/TaskMethods.cs | 133 -------------- .../MediaBrowser.Controller.csproj | 3 +- 4 files changed, 106 insertions(+), 204 deletions(-) delete mode 100644 MediaBrowser.Controller/Library/TaskMethods.cs diff --git a/Emby.Server.Implementations/ApplicationHost.cs b/Emby.Server.Implementations/ApplicationHost.cs index 6d7239f724..7a46fdf2e7 100644 --- a/Emby.Server.Implementations/ApplicationHost.cs +++ b/Emby.Server.Implementations/ApplicationHost.cs @@ -758,7 +758,6 @@ namespace Emby.Server.Implementations BaseItem.FileSystem = _fileSystemManager; BaseItem.UserDataManager = Resolve(); BaseItem.ChannelManager = Resolve(); - TaskMethods.ConfigurationManager = ServerConfigurationManager; Video.LiveTvManager = Resolve(); Folder.UserViewManager = Resolve(); UserView.TVSeriesManager = Resolve(); diff --git a/MediaBrowser.Controller/Entities/Folder.cs b/MediaBrowser.Controller/Entities/Folder.cs index 666455cff4..8cea8755cb 100644 --- a/MediaBrowser.Controller/Entities/Folder.cs +++ b/MediaBrowser.Controller/Entities/Folder.cs @@ -8,6 +8,7 @@ using System.Linq; using System.Text.Json.Serialization; using System.Threading; using System.Threading.Tasks; +using System.Threading.Tasks.Dataflow; using Jellyfin.Data.Entities; using Jellyfin.Data.Enums; using MediaBrowser.Common.Progress; @@ -35,45 +36,16 @@ namespace MediaBrowser.Controller.Entities /// public class Folder : BaseItem { - /// - /// Contains constants used when reporting scan progress. - /// - private static class ProgressHelpers - { - /// - /// Reported after the folders immediate children are retrieved. - /// - public const int RetrievedChildren = 5; + private static Lazy _metadataRefreshThrottler = new Lazy(() => { + var concurrency = ConfigurationManager.Configuration.LibraryMetadataRefreshConcurrency; - /// - /// Reported after add, updating, or deleting child items from the LibraryManager. - /// - public const int UpdatedChildItems = 10; - - /// - /// Reported once subfolders are scanned. - /// When scanning subfolders, the progress will be between [UpdatedItems, ScannedSubfolders]. - /// - public const int ScannedSubfolders = 50; - - /// - /// Reported once metadata is refreshed. - /// When refreshing metadata, the progress will be between [ScannedSubfolders, MetadataRefreshed]. - /// - public const int RefreshedMetadata = 100; - - /// - /// Gets the current progress given the previous step, next step, and progress in between. - /// - /// The previous progress step. - /// The next progress step. - /// The current progress step. - /// The progress. - public static double GetProgress(int previousProgressStep, int nextProgressStep, double currentProgress) + if (concurrency <= 0) { - return previousProgressStep + ((nextProgressStep - previousProgressStep) * (currentProgress / 100)); + concurrency = Environment.ProcessorCount; } - } + + return new SemaphoreSlim(concurrency); + }); public static IUserViewManager UserViewManager { get; set; } @@ -508,19 +480,17 @@ namespace MediaBrowser.Controller.Entities private Task RefreshMetadataRecursive(IList children, MetadataRefreshOptions refreshOptions, bool recursive, IProgress progress, CancellationToken cancellationToken) { - var progressableTasks = children - .Select, Task>>(child => - innerProgress => RefreshChildMetadata(child, refreshOptions, recursive && child.IsFolder, innerProgress, cancellationToken)) - .ToList(); - - return RunTasks(progressableTasks, progress, cancellationToken); + return RunTasks( + (baseItem, innerProgress) => RefreshChildMetadata(baseItem, refreshOptions, recursive && baseItem.IsFolder, innerProgress, cancellationToken), + children, + progress, + cancellationToken); } private async Task RefreshAllMetadataForContainer(IMetadataContainer container, MetadataRefreshOptions refreshOptions, IProgress progress, CancellationToken cancellationToken) { // limit the amount of concurrent metadata refreshes - await TaskMethods.RunThrottled( - TaskMethods.SharedThrottleId.RefreshMetadata, + await RunMetadataRefresh( async () => { var series = container as Series; @@ -547,8 +517,7 @@ namespace MediaBrowser.Controller.Entities if (refreshOptions.RefreshItem(child)) { // limit the amount of concurrent metadata refreshes - await TaskMethods.RunThrottled( - TaskMethods.SharedThrottleId.RefreshMetadata, + await RunMetadataRefresh( async () => await child.RefreshMetadata(refreshOptions, cancellationToken).ConfigureAwait(false), cancellationToken).ConfigureAwait(false); } @@ -570,38 +539,33 @@ namespace MediaBrowser.Controller.Entities /// Task. private Task ValidateSubFolders(IList children, IDirectoryService directoryService, IProgress progress, CancellationToken cancellationToken) { - var progressableTasks = children - .Select, Task>>(child => - innerProgress => child.ValidateChildrenInternal(innerProgress, cancellationToken, true, false, null, directoryService)) - .ToList(); - - return RunTasks(progressableTasks, progress, cancellationToken); + return RunTasks( + (folder, innerProgress) => folder.ValidateChildrenInternal(innerProgress, cancellationToken, true, false, null, directoryService), + children, + progress, + cancellationToken); } /// - /// Runs a set of tasks concurrently with progress. + /// Runs an action block on a list of children. /// - /// A list of tasks. + /// The task to run for each child. + /// The list of children. /// The progress. /// The cancellation token. /// Task. - private async Task RunTasks(IList, Task>> tasks, IProgress progress, CancellationToken cancellationToken) + private async Task RunTasks(Func, Task> task, IList children, IProgress progress, CancellationToken cancellationToken) { - var childrenCount = tasks.Count; + var childrenCount = children.Count; var childrenProgress = new double[childrenCount]; - var actions = new Func[childrenCount]; void UpdateProgress() { progress.Report(childrenProgress.Average()); } - for (var i = 0; i < childrenCount; i++) - { - var childIndex = i; - var child = tasks[childIndex]; - - actions[childIndex] = async () => + var actionBlock = new ActionBlock( + async i => { var innerProgress = new ActionableProgress(); @@ -609,22 +573,33 @@ namespace MediaBrowser.Controller.Entities { // round the percent and only update progress if it changed to prevent excessive UpdateProgress calls var innerPercentRounded = Math.Round(innerPercent); - if (childrenProgress[childIndex] != innerPercentRounded) + if (childrenProgress[i] != innerPercentRounded) { - childrenProgress[childIndex] = innerPercentRounded; + childrenProgress[i] = innerPercentRounded; UpdateProgress(); } }); - await tasks[childIndex](innerProgress).ConfigureAwait(false); + await task(children[i], innerProgress).ConfigureAwait(false); - childrenProgress[childIndex] = 100; + childrenProgress[i] = 100; UpdateProgress(); - }; + }, + new ExecutionDataflowBlockOptions + { + MaxDegreeOfParallelism = ConfigurationManager.Configuration.LibraryScanFanoutConcurrency, + CancellationToken = cancellationToken, + }); + + for (var i = 0; i < childrenCount; i++) + { + actionBlock.Post(i); } - await TaskMethods.WhenAllThrottled(TaskMethods.SharedThrottleId.ScanFanout, actions, cancellationToken).ConfigureAwait(false); + actionBlock.Complete(); + + await actionBlock.Completion.ConfigureAwait(false); } /// @@ -1272,6 +1247,26 @@ namespace MediaBrowser.Controller.Entities return true; } + /// + /// Runs multiple metadata refreshes concurrently. + /// + /// The action to run. + /// The cancellation token. + /// A representing the result of the asynchronous operation. + private static async Task RunMetadataRefresh(Func action, CancellationToken cancellationToken) + { + await _metadataRefreshThrottler.Value.WaitAsync(cancellationToken).ConfigureAwait(false); + + try + { + await action().ConfigureAwait(false); + } + finally + { + _metadataRefreshThrottler.Value.Release(); + } + } + public List GetChildren(User user, bool includeLinkedChildren) { if (user == null) @@ -1819,5 +1814,45 @@ namespace MediaBrowser.Controller.Entities } } } + + /// + /// Contains constants used when reporting scan progress. + /// + private static class ProgressHelpers + { + /// + /// Reported after the folders immediate children are retrieved. + /// + public const int RetrievedChildren = 5; + + /// + /// Reported after add, updating, or deleting child items from the LibraryManager. + /// + public const int UpdatedChildItems = 10; + + /// + /// Reported once subfolders are scanned. + /// When scanning subfolders, the progress will be between [UpdatedItems, ScannedSubfolders]. + /// + public const int ScannedSubfolders = 50; + + /// + /// Reported once metadata is refreshed. + /// When refreshing metadata, the progress will be between [ScannedSubfolders, MetadataRefreshed]. + /// + public const int RefreshedMetadata = 100; + + /// + /// Gets the current progress given the previous step, next step, and progress in between. + /// + /// The previous progress step. + /// The next progress step. + /// The current progress step. + /// The progress. + public static double GetProgress(int previousProgressStep, int nextProgressStep, double currentProgress) + { + return previousProgressStep + ((nextProgressStep - previousProgressStep) * (currentProgress / 100)); + } + } } } diff --git a/MediaBrowser.Controller/Library/TaskMethods.cs b/MediaBrowser.Controller/Library/TaskMethods.cs deleted file mode 100644 index 66bfbe0d9e..0000000000 --- a/MediaBrowser.Controller/Library/TaskMethods.cs +++ /dev/null @@ -1,133 +0,0 @@ -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; -using MediaBrowser.Controller.Configuration; - -namespace MediaBrowser.Controller.Library -{ - /// - /// Helper methods for running tasks concurrently. - /// - public static class TaskMethods - { - private static readonly int _processorCount = Environment.ProcessorCount; - - private static readonly ConcurrentDictionary _sharedThrottlers = new ConcurrentDictionary(); - - /// - /// Throttle id for sharing a concurrency limit. - /// - public enum SharedThrottleId - { - /// - /// Library scan fan out - /// - ScanFanout, - - /// - /// Refresh metadata - /// - RefreshMetadata, - } - - /// - /// Gets or sets the configuration manager. - /// - public static IServerConfigurationManager ConfigurationManager { get; set; } - - /// - /// Similiar to Task.WhenAll but only allows running a certain amount of tasks at the same time. - /// - /// The throttle id. Multiple calls to this method with the same throttle id will share a concurrency limit. - /// List of actions to run. - /// The cancellation token. - /// A representing the result of the asynchronous operation. - public static async Task WhenAllThrottled(SharedThrottleId throttleId, IEnumerable> actions, CancellationToken cancellationToken) - { - var taskThrottler = throttleId == SharedThrottleId.ScanFanout ? - new SemaphoreSlim(GetConcurrencyLimit(throttleId)) : - _sharedThrottlers.GetOrAdd(throttleId, id => new SemaphoreSlim(GetConcurrencyLimit(id))); - - try - { - var tasks = new List(); - - foreach (var action in actions) - { - await taskThrottler.WaitAsync(cancellationToken).ConfigureAwait(false); - - tasks.Add(Task.Run(async () => - { - try - { - await action().ConfigureAwait(false); - } - finally - { - taskThrottler.Release(); - } - })); - } - - await Task.WhenAll(tasks).ConfigureAwait(false); - } - finally - { - if (throttleId == SharedThrottleId.ScanFanout) - { - taskThrottler.Dispose(); - } - } - } - - /// - /// Runs a task within a given throttler. - /// - /// The throttle id. Multiple calls to this method with the same throttle id will share a concurrency limit. - /// The action to run. - /// The cancellation token. - /// A representing the result of the asynchronous operation. - public static async Task RunThrottled(SharedThrottleId throttleId, Func action, CancellationToken cancellationToken) - { - if (throttleId == SharedThrottleId.ScanFanout) - { - // just await the task instead - throw new InvalidOperationException("Invalid throttle id"); - } - - var taskThrottler = _sharedThrottlers.GetOrAdd(throttleId, id => new SemaphoreSlim(GetConcurrencyLimit(id))); - - await taskThrottler.WaitAsync(cancellationToken).ConfigureAwait(false); - - try - { - await action().ConfigureAwait(false); - } - finally - { - taskThrottler.Release(); - } - } - - /// - /// Get the concurrency limit for the given throttle id. - /// - /// The throttle id. - /// The concurrency limit. - private static int GetConcurrencyLimit(SharedThrottleId throttleId) - { - var concurrency = throttleId == SharedThrottleId.RefreshMetadata ? - ConfigurationManager.Configuration.LibraryMetadataRefreshConcurrency : - ConfigurationManager.Configuration.LibraryScanFanoutConcurrency; - - if (concurrency <= 0) - { - concurrency = _processorCount; - } - - return concurrency; - } - } -} diff --git a/MediaBrowser.Controller/MediaBrowser.Controller.csproj b/MediaBrowser.Controller/MediaBrowser.Controller.csproj index 6544704065..243b8cd02b 100644 --- a/MediaBrowser.Controller/MediaBrowser.Controller.csproj +++ b/MediaBrowser.Controller/MediaBrowser.Controller.csproj @@ -16,7 +16,8 @@ - + + From ac6d51554c7b358980e6ba181a21b54424d64b69 Mon Sep 17 00:00:00 2001 From: Gary Wilber Date: Thu, 1 Oct 2020 16:25:01 -0700 Subject: [PATCH 03/11] Fix lastProgressMessageTimes dictionary errors --- .../EntryPoints/LibraryChangedNotifier.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Emby.Server.Implementations/EntryPoints/LibraryChangedNotifier.cs b/Emby.Server.Implementations/EntryPoints/LibraryChangedNotifier.cs index c9d21d9638..2e9d638ec7 100644 --- a/Emby.Server.Implementations/EntryPoints/LibraryChangedNotifier.cs +++ b/Emby.Server.Implementations/EntryPoints/LibraryChangedNotifier.cs @@ -1,6 +1,7 @@ #pragma warning disable CS1591 using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Globalization; using System.Linq; @@ -43,7 +44,7 @@ namespace Emby.Server.Implementations.EntryPoints private readonly List _itemsAdded = new List(); private readonly List _itemsRemoved = new List(); private readonly List _itemsUpdated = new List(); - private readonly Dictionary _lastProgressMessageTimes = new Dictionary(); + private readonly ConcurrentDictionary _lastProgressMessageTimes = new ConcurrentDictionary(); public LibraryChangedNotifier( ILibraryManager libraryManager, @@ -97,7 +98,7 @@ namespace Emby.Server.Implementations.EntryPoints } } - _lastProgressMessageTimes[item.Id] = DateTime.UtcNow; + _lastProgressMessageTimes.AddOrUpdate(item.Id, key => DateTime.UtcNow, (key, existing) => DateTime.UtcNow); var dict = new Dictionary(); dict["ItemId"] = item.Id.ToString("N", CultureInfo.InvariantCulture); @@ -139,6 +140,8 @@ namespace Emby.Server.Implementations.EntryPoints private void OnProviderRefreshCompleted(object sender, GenericEventArgs e) { OnProviderRefreshProgress(sender, new GenericEventArgs>(new Tuple(e.Argument, 100))); + + _lastProgressMessageTimes.TryRemove(e.Argument.Id, out DateTime removed); } private static bool EnableRefreshMessage(BaseItem item) From a51adec836ae967a827cdddffc5572ef621b69d3 Mon Sep 17 00:00:00 2001 From: Gary Wilber Date: Fri, 2 Oct 2020 18:26:10 -0700 Subject: [PATCH 04/11] fix parallelism when set to 0 --- MediaBrowser.Controller/Entities/Folder.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/MediaBrowser.Controller/Entities/Folder.cs b/MediaBrowser.Controller/Entities/Folder.cs index 8cea8755cb..db59cdc314 100644 --- a/MediaBrowser.Controller/Entities/Folder.cs +++ b/MediaBrowser.Controller/Entities/Folder.cs @@ -564,6 +564,9 @@ namespace MediaBrowser.Controller.Entities progress.Report(childrenProgress.Average()); } + var fanoutConcurrency = ConfigurationManager.Configuration.LibraryScanFanoutConcurrency; + var parallelism = fanoutConcurrency == 0 ? Environment.ProcessorCount : fanoutConcurrency; + var actionBlock = new ActionBlock( async i => { @@ -588,7 +591,7 @@ namespace MediaBrowser.Controller.Entities }, new ExecutionDataflowBlockOptions { - MaxDegreeOfParallelism = ConfigurationManager.Configuration.LibraryScanFanoutConcurrency, + MaxDegreeOfParallelism = parallelism, CancellationToken = cancellationToken, }); From f680a0fbbed346a5a5e98b2066604c463791190c Mon Sep 17 00:00:00 2001 From: Gary Wilber Date: Sun, 8 Nov 2020 11:55:38 -0800 Subject: [PATCH 05/11] fix merge --- MediaBrowser.Model/Configuration/ServerConfiguration.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/MediaBrowser.Model/Configuration/ServerConfiguration.cs b/MediaBrowser.Model/Configuration/ServerConfiguration.cs index 4d24e45b0c..be313bde1b 100644 --- a/MediaBrowser.Model/Configuration/ServerConfiguration.cs +++ b/MediaBrowser.Model/Configuration/ServerConfiguration.cs @@ -271,6 +271,7 @@ namespace MediaBrowser.Model.Configuration /// public string[] KnownProxies { get; set; } + /// /// Gets or sets the number of days we should retain activity logs. /// public int? ActivityLogRetentionDays { get; set; } From d6585e7ff2e14234201e3fe6a75a5af2170c7804 Mon Sep 17 00:00:00 2001 From: Gary Wilber Date: Thu, 19 Nov 2020 18:42:41 -0800 Subject: [PATCH 06/11] Upgrade System.Threading.Tasks.Dataflow --- MediaBrowser.Controller/MediaBrowser.Controller.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/MediaBrowser.Controller/MediaBrowser.Controller.csproj b/MediaBrowser.Controller/MediaBrowser.Controller.csproj index b2bce02a29..e99483dcc9 100644 --- a/MediaBrowser.Controller/MediaBrowser.Controller.csproj +++ b/MediaBrowser.Controller/MediaBrowser.Controller.csproj @@ -16,8 +16,8 @@ - - + + From f4edca7c2790207b5be9abb3b49029c9fbe04828 Mon Sep 17 00:00:00 2001 From: Gary Wilber Date: Wed, 2 Dec 2020 18:51:53 -0800 Subject: [PATCH 07/11] Move MetadataRefreshThrottler to BaseItemManager --- .../BaseItemManager/BaseItemManager.cs | 17 ++++++++- .../BaseItemManager/IBaseItemManager.cs | 9 ++++- MediaBrowser.Controller/Entities/Folder.cs | 35 ++----------------- .../Providers/IProviderManager.cs | 8 +++++ .../Manager/ProviderManager.cs | 20 +++++++++++ 5 files changed, 54 insertions(+), 35 deletions(-) diff --git a/MediaBrowser.Controller/BaseItemManager/BaseItemManager.cs b/MediaBrowser.Controller/BaseItemManager/BaseItemManager.cs index 67aa7f3383..0c8fa82447 100644 --- a/MediaBrowser.Controller/BaseItemManager/BaseItemManager.cs +++ b/MediaBrowser.Controller/BaseItemManager/BaseItemManager.cs @@ -1,5 +1,6 @@ -using System; +using System; using System.Linq; +using System.Threading; using MediaBrowser.Controller.Channels; using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Entities; @@ -19,8 +20,22 @@ namespace MediaBrowser.Controller.BaseItemManager public BaseItemManager(IServerConfigurationManager serverConfigurationManager) { _serverConfigurationManager = serverConfigurationManager; + + MetadataRefreshThrottler = new Lazy(() => { + var concurrency = _serverConfigurationManager.Configuration.LibraryMetadataRefreshConcurrency; + + if (concurrency <= 0) + { + concurrency = Environment.ProcessorCount; + } + + return new SemaphoreSlim(concurrency); + }); } + /// + public Lazy MetadataRefreshThrottler { get; private set; } + /// public bool IsMetadataFetcherEnabled(BaseItem baseItem, LibraryOptions libraryOptions, string name) { diff --git a/MediaBrowser.Controller/BaseItemManager/IBaseItemManager.cs b/MediaBrowser.Controller/BaseItemManager/IBaseItemManager.cs index ee4d3dcdcc..d5f36dc2e2 100644 --- a/MediaBrowser.Controller/BaseItemManager/IBaseItemManager.cs +++ b/MediaBrowser.Controller/BaseItemManager/IBaseItemManager.cs @@ -1,4 +1,6 @@ -using MediaBrowser.Controller.Entities; +using System; +using System.Threading; +using MediaBrowser.Controller.Entities; using MediaBrowser.Model.Configuration; namespace MediaBrowser.Controller.BaseItemManager @@ -8,6 +10,11 @@ namespace MediaBrowser.Controller.BaseItemManager /// public interface IBaseItemManager { + /// + /// Gets the semaphore used to limit the amount of concurrent metadata refreshes. + /// + Lazy MetadataRefreshThrottler { get; } + /// /// Is metadata fetcher enabled. /// diff --git a/MediaBrowser.Controller/Entities/Folder.cs b/MediaBrowser.Controller/Entities/Folder.cs index e41407d7f4..57d04ddfac 100644 --- a/MediaBrowser.Controller/Entities/Folder.cs +++ b/MediaBrowser.Controller/Entities/Folder.cs @@ -36,17 +36,6 @@ namespace MediaBrowser.Controller.Entities /// public class Folder : BaseItem { - private static Lazy _metadataRefreshThrottler = new Lazy(() => { - var concurrency = ConfigurationManager.Configuration.LibraryMetadataRefreshConcurrency; - - if (concurrency <= 0) - { - concurrency = Environment.ProcessorCount; - } - - return new SemaphoreSlim(concurrency); - }); - public static IUserViewManager UserViewManager { get; set; } /// @@ -491,7 +480,7 @@ namespace MediaBrowser.Controller.Entities private async Task RefreshAllMetadataForContainer(IMetadataContainer container, MetadataRefreshOptions refreshOptions, IProgress progress, CancellationToken cancellationToken) { // limit the amount of concurrent metadata refreshes - await RunMetadataRefresh( + await ProviderManager.RunMetadataRefresh( async () => { var series = container as Series; @@ -518,7 +507,7 @@ namespace MediaBrowser.Controller.Entities if (refreshOptions.RefreshItem(child)) { // limit the amount of concurrent metadata refreshes - await RunMetadataRefresh( + await ProviderManager.RunMetadataRefresh( async () => await child.RefreshMetadata(refreshOptions, cancellationToken).ConfigureAwait(false), cancellationToken).ConfigureAwait(false); } @@ -1252,26 +1241,6 @@ namespace MediaBrowser.Controller.Entities return true; } - /// - /// Runs multiple metadata refreshes concurrently. - /// - /// The action to run. - /// The cancellation token. - /// A representing the result of the asynchronous operation. - private static async Task RunMetadataRefresh(Func action, CancellationToken cancellationToken) - { - await _metadataRefreshThrottler.Value.WaitAsync(cancellationToken).ConfigureAwait(false); - - try - { - await action().ConfigureAwait(false); - } - finally - { - _metadataRefreshThrottler.Value.Release(); - } - } - public List GetChildren(User user, bool includeLinkedChildren) { if (user == null) diff --git a/MediaBrowser.Controller/Providers/IProviderManager.cs b/MediaBrowser.Controller/Providers/IProviderManager.cs index 996ec27c09..0a4967223f 100644 --- a/MediaBrowser.Controller/Providers/IProviderManager.cs +++ b/MediaBrowser.Controller/Providers/IProviderManager.cs @@ -45,6 +45,14 @@ namespace MediaBrowser.Controller.Providers /// Task. Task RefreshSingleItem(BaseItem item, MetadataRefreshOptions options, CancellationToken cancellationToken); + /// + /// Runs multiple metadata refreshes concurrently. + /// + /// The action to run. + /// The cancellation token. + /// A representing the result of the asynchronous operation. + Task RunMetadataRefresh(Func action, CancellationToken cancellationToken); + /// /// Saves the image. /// diff --git a/MediaBrowser.Providers/Manager/ProviderManager.cs b/MediaBrowser.Providers/Manager/ProviderManager.cs index e7e44876db..fbf4bc68b4 100644 --- a/MediaBrowser.Providers/Manager/ProviderManager.cs +++ b/MediaBrowser.Providers/Manager/ProviderManager.cs @@ -1167,6 +1167,26 @@ namespace MediaBrowser.Providers.Manager return RefreshItem(item, options, cancellationToken); } + /// + /// Runs multiple metadata refreshes concurrently. + /// + /// The action to run. + /// The cancellation token. + /// A representing the result of the asynchronous operation. + public async Task RunMetadataRefresh(Func action, CancellationToken cancellationToken) + { + await _baseItemManager.MetadataRefreshThrottler.Value.WaitAsync(cancellationToken).ConfigureAwait(false); + + try + { + await action().ConfigureAwait(false); + } + finally + { + _baseItemManager.MetadataRefreshThrottler.Value.Release(); + } + } + /// public void Dispose() { From 26a05e69748285e2cb2a7647496e44e6c70a0d97 Mon Sep 17 00:00:00 2001 From: Gary Wilber Date: Wed, 2 Dec 2020 23:07:51 -0800 Subject: [PATCH 08/11] Handle config updates --- .../BaseItemManager/BaseItemManager.cs | 42 +++++++++++++++---- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/MediaBrowser.Controller/BaseItemManager/BaseItemManager.cs b/MediaBrowser.Controller/BaseItemManager/BaseItemManager.cs index 0c8fa82447..4981b09273 100644 --- a/MediaBrowser.Controller/BaseItemManager/BaseItemManager.cs +++ b/MediaBrowser.Controller/BaseItemManager/BaseItemManager.cs @@ -1,6 +1,7 @@ using System; using System.Linq; using System.Threading; +using MediaBrowser.Common.Configuration; using MediaBrowser.Controller.Channels; using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Entities; @@ -13,6 +14,8 @@ namespace MediaBrowser.Controller.BaseItemManager { private readonly IServerConfigurationManager _serverConfigurationManager; + private int _metadataRefreshConcurrency = 0; + /// /// Initializes a new instance of the class. /// @@ -21,16 +24,18 @@ namespace MediaBrowser.Controller.BaseItemManager { _serverConfigurationManager = serverConfigurationManager; - MetadataRefreshThrottler = new Lazy(() => { - var concurrency = _serverConfigurationManager.Configuration.LibraryMetadataRefreshConcurrency; + _metadataRefreshConcurrency = GetMetadataRefreshConcurrency(); + SetupMetadataThrottler(); - if (concurrency <= 0) + _serverConfigurationManager.ConfigurationUpdated += (object sender, EventArgs e) => + { + int newMetadataRefreshConcurrency = GetMetadataRefreshConcurrency(); + if (_metadataRefreshConcurrency != newMetadataRefreshConcurrency) { - concurrency = Environment.ProcessorCount; + _metadataRefreshConcurrency = newMetadataRefreshConcurrency; + SetupMetadataThrottler(); } - - return new SemaphoreSlim(concurrency); - }); + }; } /// @@ -97,5 +102,28 @@ namespace MediaBrowser.Controller.BaseItemManager return itemConfig == null || !itemConfig.DisabledImageFetchers.Contains(name, StringComparer.OrdinalIgnoreCase); } + + /// + /// Creates the metadata refresh throttler. + /// + private void SetupMetadataThrottler() + { + MetadataRefreshThrottler = new Lazy(() => new SemaphoreSlim(_metadataRefreshConcurrency)); + } + + /// + /// Returns the metadata refresh concurrency. + /// + private int GetMetadataRefreshConcurrency() + { + var concurrency = _serverConfigurationManager.Configuration.LibraryMetadataRefreshConcurrency; + + if (concurrency <= 0) + { + concurrency = Environment.ProcessorCount; + } + + return concurrency; + } } } From 74f211960a5df659d51b4e836ee784497829a3ed Mon Sep 17 00:00:00 2001 From: Gary Wilber Date: Wed, 2 Dec 2020 23:14:18 -0800 Subject: [PATCH 09/11] only recreate the lazy when needed --- MediaBrowser.Controller/BaseItemManager/BaseItemManager.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/MediaBrowser.Controller/BaseItemManager/BaseItemManager.cs b/MediaBrowser.Controller/BaseItemManager/BaseItemManager.cs index 4981b09273..7aec3b3228 100644 --- a/MediaBrowser.Controller/BaseItemManager/BaseItemManager.cs +++ b/MediaBrowser.Controller/BaseItemManager/BaseItemManager.cs @@ -108,7 +108,10 @@ namespace MediaBrowser.Controller.BaseItemManager /// private void SetupMetadataThrottler() { - MetadataRefreshThrottler = new Lazy(() => new SemaphoreSlim(_metadataRefreshConcurrency)); + if (MetadataRefreshThrottler == null || MetadataRefreshThrottler.IsValueCreated) + { + MetadataRefreshThrottler = new Lazy(() => new SemaphoreSlim(_metadataRefreshConcurrency)); + } } /// From 2b6b2cbf31d44c03e1bf68eff155fc0918df9dfb Mon Sep 17 00:00:00 2001 From: Gary Wilber Date: Wed, 2 Dec 2020 23:23:13 -0800 Subject: [PATCH 10/11] Remove Lazy --- MediaBrowser.Controller/BaseItemManager/BaseItemManager.cs | 7 ++----- .../BaseItemManager/IBaseItemManager.cs | 2 +- MediaBrowser.Providers/Manager/ProviderManager.cs | 7 +++++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/MediaBrowser.Controller/BaseItemManager/BaseItemManager.cs b/MediaBrowser.Controller/BaseItemManager/BaseItemManager.cs index 7aec3b3228..c018627b45 100644 --- a/MediaBrowser.Controller/BaseItemManager/BaseItemManager.cs +++ b/MediaBrowser.Controller/BaseItemManager/BaseItemManager.cs @@ -39,7 +39,7 @@ namespace MediaBrowser.Controller.BaseItemManager } /// - public Lazy MetadataRefreshThrottler { get; private set; } + public SemaphoreSlim MetadataRefreshThrottler { get; private set; } /// public bool IsMetadataFetcherEnabled(BaseItem baseItem, LibraryOptions libraryOptions, string name) @@ -108,10 +108,7 @@ namespace MediaBrowser.Controller.BaseItemManager /// private void SetupMetadataThrottler() { - if (MetadataRefreshThrottler == null || MetadataRefreshThrottler.IsValueCreated) - { - MetadataRefreshThrottler = new Lazy(() => new SemaphoreSlim(_metadataRefreshConcurrency)); - } + MetadataRefreshThrottler = new SemaphoreSlim(_metadataRefreshConcurrency); } /// diff --git a/MediaBrowser.Controller/BaseItemManager/IBaseItemManager.cs b/MediaBrowser.Controller/BaseItemManager/IBaseItemManager.cs index d5f36dc2e2..e1f5d05a60 100644 --- a/MediaBrowser.Controller/BaseItemManager/IBaseItemManager.cs +++ b/MediaBrowser.Controller/BaseItemManager/IBaseItemManager.cs @@ -13,7 +13,7 @@ namespace MediaBrowser.Controller.BaseItemManager /// /// Gets the semaphore used to limit the amount of concurrent metadata refreshes. /// - Lazy MetadataRefreshThrottler { get; } + SemaphoreSlim MetadataRefreshThrottler { get; } /// /// Is metadata fetcher enabled. diff --git a/MediaBrowser.Providers/Manager/ProviderManager.cs b/MediaBrowser.Providers/Manager/ProviderManager.cs index fbf4bc68b4..a20c47cf2e 100644 --- a/MediaBrowser.Providers/Manager/ProviderManager.cs +++ b/MediaBrowser.Providers/Manager/ProviderManager.cs @@ -1175,7 +1175,10 @@ namespace MediaBrowser.Providers.Manager /// A representing the result of the asynchronous operation. public async Task RunMetadataRefresh(Func action, CancellationToken cancellationToken) { - await _baseItemManager.MetadataRefreshThrottler.Value.WaitAsync(cancellationToken).ConfigureAwait(false); + // create a variable for this since it is possible MetadataRefreshThrottler could change due to a config update during a scan + var metadataRefreshThrottler = _baseItemManager.MetadataRefreshThrottler; + + await metadataRefreshThrottler.WaitAsync(cancellationToken).ConfigureAwait(false); try { @@ -1183,7 +1186,7 @@ namespace MediaBrowser.Providers.Manager } finally { - _baseItemManager.MetadataRefreshThrottler.Value.Release(); + metadataRefreshThrottler.Release(); } } From a9367b616927c9ec0b100bed49be14e3a4c67531 Mon Sep 17 00:00:00 2001 From: Gary Wilber Date: Thu, 3 Dec 2020 17:11:53 -0800 Subject: [PATCH 11/11] Add OnConfigurationUpdated method --- .../BaseItemManager/BaseItemManager.cs | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/MediaBrowser.Controller/BaseItemManager/BaseItemManager.cs b/MediaBrowser.Controller/BaseItemManager/BaseItemManager.cs index c018627b45..085f769d0c 100644 --- a/MediaBrowser.Controller/BaseItemManager/BaseItemManager.cs +++ b/MediaBrowser.Controller/BaseItemManager/BaseItemManager.cs @@ -27,15 +27,7 @@ namespace MediaBrowser.Controller.BaseItemManager _metadataRefreshConcurrency = GetMetadataRefreshConcurrency(); SetupMetadataThrottler(); - _serverConfigurationManager.ConfigurationUpdated += (object sender, EventArgs e) => - { - int newMetadataRefreshConcurrency = GetMetadataRefreshConcurrency(); - if (_metadataRefreshConcurrency != newMetadataRefreshConcurrency) - { - _metadataRefreshConcurrency = newMetadataRefreshConcurrency; - SetupMetadataThrottler(); - } - }; + _serverConfigurationManager.ConfigurationUpdated += OnConfigurationUpdated; } /// @@ -103,6 +95,20 @@ namespace MediaBrowser.Controller.BaseItemManager return itemConfig == null || !itemConfig.DisabledImageFetchers.Contains(name, StringComparer.OrdinalIgnoreCase); } + /// + /// Called when the configuration is updated. + /// It will refresh the metadata throttler if the relevant config changed. + /// + private void OnConfigurationUpdated(object sender, EventArgs e) + { + int newMetadataRefreshConcurrency = GetMetadataRefreshConcurrency(); + if (_metadataRefreshConcurrency != newMetadataRefreshConcurrency) + { + _metadataRefreshConcurrency = newMetadataRefreshConcurrency; + SetupMetadataThrottler(); + } + } + /// /// Creates the metadata refresh throttler. ///