diff --git a/Emby.Server.Implementations/EntryPoints/LibraryChangedNotifier.cs b/Emby.Server.Implementations/EntryPoints/LibraryChangedNotifier.cs index ff64e217a0..ae1b51b4c3 100644 --- a/Emby.Server.Implementations/EntryPoints/LibraryChangedNotifier.cs +++ b/Emby.Server.Implementations/EntryPoints/LibraryChangedNotifier.cs @@ -1,6 +1,7 @@ #pragma warning disable CS1591 using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Globalization; using System.Linq; @@ -44,7 +45,7 @@ namespace Emby.Server.Implementations.EntryPoints private readonly List _itemsAdded = new List(); private readonly List _itemsRemoved = new List(); private readonly List _itemsUpdated = new List(); - private readonly Dictionary _lastProgressMessageTimes = new Dictionary(); + private readonly ConcurrentDictionary _lastProgressMessageTimes = new ConcurrentDictionary(); public LibraryChangedNotifier( ILibraryManager libraryManager, @@ -98,7 +99,7 @@ namespace Emby.Server.Implementations.EntryPoints } } - _lastProgressMessageTimes[item.Id] = DateTime.UtcNow; + _lastProgressMessageTimes.AddOrUpdate(item.Id, key => DateTime.UtcNow, (key, existing) => DateTime.UtcNow); var dict = new Dictionary(); dict["ItemId"] = item.Id.ToString("N", CultureInfo.InvariantCulture); @@ -140,6 +141,8 @@ namespace Emby.Server.Implementations.EntryPoints private void OnProviderRefreshCompleted(object sender, GenericEventArgs e) { OnProviderRefreshProgress(sender, new GenericEventArgs>(new Tuple(e.Argument, 100))); + + _lastProgressMessageTimes.TryRemove(e.Argument.Id, out DateTime removed); } private static bool EnableRefreshMessage(BaseItem item) diff --git a/MediaBrowser.Controller/BaseItemManager/BaseItemManager.cs b/MediaBrowser.Controller/BaseItemManager/BaseItemManager.cs index 67aa7f3383..085f769d0c 100644 --- a/MediaBrowser.Controller/BaseItemManager/BaseItemManager.cs +++ b/MediaBrowser.Controller/BaseItemManager/BaseItemManager.cs @@ -1,5 +1,7 @@ -using System; +using System; using System.Linq; +using System.Threading; +using MediaBrowser.Common.Configuration; using MediaBrowser.Controller.Channels; using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Entities; @@ -12,6 +14,8 @@ namespace MediaBrowser.Controller.BaseItemManager { private readonly IServerConfigurationManager _serverConfigurationManager; + private int _metadataRefreshConcurrency = 0; + /// /// Initializes a new instance of the class. /// @@ -19,8 +23,16 @@ namespace MediaBrowser.Controller.BaseItemManager public BaseItemManager(IServerConfigurationManager serverConfigurationManager) { _serverConfigurationManager = serverConfigurationManager; + + _metadataRefreshConcurrency = GetMetadataRefreshConcurrency(); + SetupMetadataThrottler(); + + _serverConfigurationManager.ConfigurationUpdated += OnConfigurationUpdated; } + /// + public SemaphoreSlim MetadataRefreshThrottler { get; private set; } + /// public bool IsMetadataFetcherEnabled(BaseItem baseItem, LibraryOptions libraryOptions, string name) { @@ -82,5 +94,42 @@ namespace MediaBrowser.Controller.BaseItemManager return itemConfig == null || !itemConfig.DisabledImageFetchers.Contains(name, StringComparer.OrdinalIgnoreCase); } + + /// + /// Called when the configuration is updated. + /// It will refresh the metadata throttler if the relevant config changed. + /// + private void OnConfigurationUpdated(object sender, EventArgs e) + { + int newMetadataRefreshConcurrency = GetMetadataRefreshConcurrency(); + if (_metadataRefreshConcurrency != newMetadataRefreshConcurrency) + { + _metadataRefreshConcurrency = newMetadataRefreshConcurrency; + SetupMetadataThrottler(); + } + } + + /// + /// Creates the metadata refresh throttler. + /// + private void SetupMetadataThrottler() + { + MetadataRefreshThrottler = new SemaphoreSlim(_metadataRefreshConcurrency); + } + + /// + /// Returns the metadata refresh concurrency. + /// + private int GetMetadataRefreshConcurrency() + { + var concurrency = _serverConfigurationManager.Configuration.LibraryMetadataRefreshConcurrency; + + if (concurrency <= 0) + { + concurrency = Environment.ProcessorCount; + } + + return concurrency; + } } } diff --git a/MediaBrowser.Controller/BaseItemManager/IBaseItemManager.cs b/MediaBrowser.Controller/BaseItemManager/IBaseItemManager.cs index ee4d3dcdcc..e1f5d05a60 100644 --- a/MediaBrowser.Controller/BaseItemManager/IBaseItemManager.cs +++ b/MediaBrowser.Controller/BaseItemManager/IBaseItemManager.cs @@ -1,4 +1,6 @@ -using MediaBrowser.Controller.Entities; +using System; +using System.Threading; +using MediaBrowser.Controller.Entities; using MediaBrowser.Model.Configuration; namespace MediaBrowser.Controller.BaseItemManager @@ -8,6 +10,11 @@ namespace MediaBrowser.Controller.BaseItemManager /// public interface IBaseItemManager { + /// + /// Gets the semaphore used to limit the amount of concurrent metadata refreshes. + /// + SemaphoreSlim MetadataRefreshThrottler { get; } + /// /// Is metadata fetcher enabled. /// diff --git a/MediaBrowser.Controller/Entities/Folder.cs b/MediaBrowser.Controller/Entities/Folder.cs index 675cdbd963..23f4c00c10 100644 --- a/MediaBrowser.Controller/Entities/Folder.cs +++ b/MediaBrowser.Controller/Entities/Folder.cs @@ -8,6 +8,7 @@ using System.Linq; using System.Text.Json.Serialization; using System.Threading; using System.Threading.Tasks; +using System.Threading.Tasks.Dataflow; using Jellyfin.Data.Entities; using Jellyfin.Data.Enums; using MediaBrowser.Common.Progress; @@ -328,11 +329,11 @@ namespace MediaBrowser.Controller.Entities return; } - progress.Report(5); + progress.Report(ProgressHelpers.RetrievedChildren); if (recursive) { - ProviderManager.OnRefreshProgress(this, 5); + ProviderManager.OnRefreshProgress(this, ProgressHelpers.RetrievedChildren); } // Build a dictionary of the current children we have now by Id so we can compare quickly and easily @@ -388,11 +389,11 @@ namespace MediaBrowser.Controller.Entities validChildrenNeedGeneration = true; } - progress.Report(10); + progress.Report(ProgressHelpers.UpdatedChildItems); if (recursive) { - ProviderManager.OnRefreshProgress(this, 10); + ProviderManager.OnRefreshProgress(this, ProgressHelpers.UpdatedChildItems); } cancellationToken.ThrowIfCancellationRequested(); @@ -402,11 +403,13 @@ namespace MediaBrowser.Controller.Entities var innerProgress = new ActionableProgress(); var folder = this; - innerProgress.RegisterAction(p => + innerProgress.RegisterAction(innerPercent => { - double newPct = 0.80 * p + 10; - progress.Report(newPct); - ProviderManager.OnRefreshProgress(folder, newPct); + var percent = ProgressHelpers.GetProgress(ProgressHelpers.UpdatedChildItems, ProgressHelpers.ScannedSubfolders, innerPercent); + + progress.Report(percent); + + ProviderManager.OnRefreshProgress(folder, percent); }); if (validChildrenNeedGeneration) @@ -420,11 +423,11 @@ namespace MediaBrowser.Controller.Entities if (refreshChildMetadata) { - progress.Report(90); + progress.Report(ProgressHelpers.ScannedSubfolders); if (recursive) { - ProviderManager.OnRefreshProgress(this, 90); + ProviderManager.OnRefreshProgress(this, ProgressHelpers.ScannedSubfolders); } var container = this as IMetadataContainer; @@ -432,13 +435,15 @@ namespace MediaBrowser.Controller.Entities var innerProgress = new ActionableProgress(); var folder = this; - innerProgress.RegisterAction(p => + innerProgress.RegisterAction(innerPercent => { - double newPct = 0.10 * p + 90; - progress.Report(newPct); + var percent = ProgressHelpers.GetProgress(ProgressHelpers.ScannedSubfolders, ProgressHelpers.RefreshedMetadata, innerPercent); + + progress.Report(percent); + if (recursive) { - ProviderManager.OnRefreshProgress(folder, newPct); + ProviderManager.OnRefreshProgress(folder, percent); } }); @@ -453,55 +458,35 @@ namespace MediaBrowser.Controller.Entities validChildren = Children.ToList(); } - await RefreshMetadataRecursive(validChildren, refreshOptions, recursive, innerProgress, cancellationToken); + await RefreshMetadataRecursive(validChildren, refreshOptions, recursive, innerProgress, cancellationToken).ConfigureAwait(false); } } } - private async Task RefreshMetadataRecursive(List children, MetadataRefreshOptions refreshOptions, bool recursive, IProgress progress, CancellationToken cancellationToken) + private Task RefreshMetadataRecursive(IList children, MetadataRefreshOptions refreshOptions, bool recursive, IProgress progress, CancellationToken cancellationToken) { - var numComplete = 0; - var count = children.Count; - double currentPercent = 0; - - foreach (var child in children) - { - cancellationToken.ThrowIfCancellationRequested(); - - var innerProgress = new ActionableProgress(); - - // Avoid implicitly captured closure - var currentInnerPercent = currentPercent; - - innerProgress.RegisterAction(p => - { - double innerPercent = currentInnerPercent; - innerPercent += p / count; - progress.Report(innerPercent); - }); - - await RefreshChildMetadata(child, refreshOptions, recursive && child.IsFolder, innerProgress, cancellationToken) - .ConfigureAwait(false); - - numComplete++; - double percent = numComplete; - percent /= count; - percent *= 100; - currentPercent = percent; - - progress.Report(percent); - } + return RunTasks( + (baseItem, innerProgress) => RefreshChildMetadata(baseItem, refreshOptions, recursive && baseItem.IsFolder, innerProgress, cancellationToken), + children, + progress, + cancellationToken); } private async Task RefreshAllMetadataForContainer(IMetadataContainer container, MetadataRefreshOptions refreshOptions, IProgress progress, CancellationToken cancellationToken) { - var series = container as Series; - if (series != null) - { - await series.RefreshMetadata(refreshOptions, cancellationToken).ConfigureAwait(false); - } + // limit the amount of concurrent metadata refreshes + await ProviderManager.RunMetadataRefresh( + async () => + { + var series = container as Series; + if (series != null) + { + await series.RefreshMetadata(refreshOptions, cancellationToken).ConfigureAwait(false); + } - await container.RefreshAllMetadata(refreshOptions, progress, cancellationToken).ConfigureAwait(false); + await container.RefreshAllMetadata(refreshOptions, progress, cancellationToken).ConfigureAwait(false); + }, + cancellationToken).ConfigureAwait(false); } private async Task RefreshChildMetadata(BaseItem child, MetadataRefreshOptions refreshOptions, bool recursive, IProgress progress, CancellationToken cancellationToken) @@ -516,12 +501,15 @@ namespace MediaBrowser.Controller.Entities { if (refreshOptions.RefreshItem(child)) { - await child.RefreshMetadata(refreshOptions, cancellationToken).ConfigureAwait(false); + // limit the amount of concurrent metadata refreshes + await ProviderManager.RunMetadataRefresh( + async () => await child.RefreshMetadata(refreshOptions, cancellationToken).ConfigureAwait(false), + cancellationToken).ConfigureAwait(false); } if (recursive && child is Folder folder) { - await folder.RefreshMetadataRecursive(folder.Children.ToList(), refreshOptions, true, progress, cancellationToken); + await folder.RefreshMetadataRecursive(folder.Children.ToList(), refreshOptions, true, progress, cancellationToken).ConfigureAwait(false); } } } @@ -534,39 +522,72 @@ namespace MediaBrowser.Controller.Entities /// The progress. /// The cancellation token. /// Task. - private async Task ValidateSubFolders(IList children, IDirectoryService directoryService, IProgress progress, CancellationToken cancellationToken) + private Task ValidateSubFolders(IList children, IDirectoryService directoryService, IProgress progress, CancellationToken cancellationToken) { - var numComplete = 0; - var count = children.Count; - double currentPercent = 0; + return RunTasks( + (folder, innerProgress) => folder.ValidateChildrenInternal(innerProgress, cancellationToken, true, false, null, directoryService), + children, + progress, + cancellationToken); + } - foreach (var child in children) + /// + /// Runs an action block on a list of children. + /// + /// The task to run for each child. + /// The list of children. + /// The progress. + /// The cancellation token. + /// Task. + private async Task RunTasks(Func, Task> task, IList children, IProgress progress, CancellationToken cancellationToken) + { + var childrenCount = children.Count; + var childrenProgress = new double[childrenCount]; + + void UpdateProgress() { - cancellationToken.ThrowIfCancellationRequested(); + progress.Report(childrenProgress.Average()); + } - var innerProgress = new ActionableProgress(); + var fanoutConcurrency = ConfigurationManager.Configuration.LibraryScanFanoutConcurrency; + var parallelism = fanoutConcurrency == 0 ? Environment.ProcessorCount : fanoutConcurrency; - // Avoid implicitly captured closure - var currentInnerPercent = currentPercent; - - innerProgress.RegisterAction(p => + var actionBlock = new ActionBlock( + async i => { - double innerPercent = currentInnerPercent; - innerPercent += p / count; - progress.Report(innerPercent); + var innerProgress = new ActionableProgress(); + + innerProgress.RegisterAction(innerPercent => + { + // round the percent and only update progress if it changed to prevent excessive UpdateProgress calls + var innerPercentRounded = Math.Round(innerPercent); + if (childrenProgress[i] != innerPercentRounded) + { + childrenProgress[i] = innerPercentRounded; + UpdateProgress(); + } + }); + + await task(children[i], innerProgress).ConfigureAwait(false); + + childrenProgress[i] = 100; + + UpdateProgress(); + }, + new ExecutionDataflowBlockOptions + { + MaxDegreeOfParallelism = parallelism, + CancellationToken = cancellationToken, }); - await child.ValidateChildrenInternal(innerProgress, cancellationToken, true, false, null, directoryService) - .ConfigureAwait(false); - - numComplete++; - double percent = numComplete; - percent /= count; - percent *= 100; - currentPercent = percent; - - progress.Report(percent); + for (var i = 0; i < childrenCount; i++) + { + actionBlock.Post(i); } + + actionBlock.Complete(); + + await actionBlock.Completion.ConfigureAwait(false); } /// @@ -1763,5 +1784,45 @@ namespace MediaBrowser.Controller.Entities } } } + + /// + /// Contains constants used when reporting scan progress. + /// + private static class ProgressHelpers + { + /// + /// Reported after the folders immediate children are retrieved. + /// + public const int RetrievedChildren = 5; + + /// + /// Reported after add, updating, or deleting child items from the LibraryManager. + /// + public const int UpdatedChildItems = 10; + + /// + /// Reported once subfolders are scanned. + /// When scanning subfolders, the progress will be between [UpdatedItems, ScannedSubfolders]. + /// + public const int ScannedSubfolders = 50; + + /// + /// Reported once metadata is refreshed. + /// When refreshing metadata, the progress will be between [ScannedSubfolders, MetadataRefreshed]. + /// + public const int RefreshedMetadata = 100; + + /// + /// Gets the current progress given the previous step, next step, and progress in between. + /// + /// The previous progress step. + /// The next progress step. + /// The current progress step. + /// The progress. + public static double GetProgress(int previousProgressStep, int nextProgressStep, double currentProgress) + { + return previousProgressStep + ((nextProgressStep - previousProgressStep) * (currentProgress / 100)); + } + } } } diff --git a/MediaBrowser.Controller/MediaBrowser.Controller.csproj b/MediaBrowser.Controller/MediaBrowser.Controller.csproj index 9acc98dcec..5f75df54eb 100644 --- a/MediaBrowser.Controller/MediaBrowser.Controller.csproj +++ b/MediaBrowser.Controller/MediaBrowser.Controller.csproj @@ -17,6 +17,7 @@ + diff --git a/MediaBrowser.Controller/Providers/IProviderManager.cs b/MediaBrowser.Controller/Providers/IProviderManager.cs index 996ec27c09..0a4967223f 100644 --- a/MediaBrowser.Controller/Providers/IProviderManager.cs +++ b/MediaBrowser.Controller/Providers/IProviderManager.cs @@ -45,6 +45,14 @@ namespace MediaBrowser.Controller.Providers /// Task. Task RefreshSingleItem(BaseItem item, MetadataRefreshOptions options, CancellationToken cancellationToken); + /// + /// Runs multiple metadata refreshes concurrently. + /// + /// The action to run. + /// The cancellation token. + /// A representing the result of the asynchronous operation. + Task RunMetadataRefresh(Func action, CancellationToken cancellationToken); + /// /// Saves the image. /// diff --git a/MediaBrowser.Model/Configuration/ServerConfiguration.cs b/MediaBrowser.Model/Configuration/ServerConfiguration.cs index 830c8bd102..0dbd51bdc1 100644 --- a/MediaBrowser.Model/Configuration/ServerConfiguration.cs +++ b/MediaBrowser.Model/Configuration/ServerConfiguration.cs @@ -439,5 +439,15 @@ namespace MediaBrowser.Model.Configuration /// Gets or sets the number of days we should retain activity logs. /// public int? ActivityLogRetentionDays { get; set; } = 30; + + /// + /// Gets or sets the how the library scan fans out. + /// + public int LibraryScanFanoutConcurrency { get; set; } + + /// + /// Gets or sets the how many metadata refreshes can run concurrently. + /// + public int LibraryMetadataRefreshConcurrency { get; set; } } } diff --git a/MediaBrowser.Providers/Manager/ProviderManager.cs b/MediaBrowser.Providers/Manager/ProviderManager.cs index e7e44876db..a20c47cf2e 100644 --- a/MediaBrowser.Providers/Manager/ProviderManager.cs +++ b/MediaBrowser.Providers/Manager/ProviderManager.cs @@ -1167,6 +1167,29 @@ namespace MediaBrowser.Providers.Manager return RefreshItem(item, options, cancellationToken); } + /// + /// Runs multiple metadata refreshes concurrently. + /// + /// The action to run. + /// The cancellation token. + /// A representing the result of the asynchronous operation. + public async Task RunMetadataRefresh(Func action, CancellationToken cancellationToken) + { + // create a variable for this since it is possible MetadataRefreshThrottler could change due to a config update during a scan + var metadataRefreshThrottler = _baseItemManager.MetadataRefreshThrottler; + + await metadataRefreshThrottler.WaitAsync(cancellationToken).ConfigureAwait(false); + + try + { + await action().ConfigureAwait(false); + } + finally + { + metadataRefreshThrottler.Release(); + } + } + /// public void Dispose() {