Increase library scan and metadata refresh speed

Emby.Server.Implementations/ApplicationHost.cs

@@ -758,6 +758,7 @@ namespace Emby.Server.Implementations
             BaseItem.FileSystem = _fileSystemManager;
             BaseItem.UserDataManager = Resolve<IUserDataManager>();
             BaseItem.ChannelManager = Resolve<IChannelManager>();
+            TaskMethods.ConfigurationManager = ServerConfigurationManager;
             Video.LiveTvManager = Resolve<ILiveTvManager>();
             Folder.UserViewManager = Resolve<IUserViewManager>();
             UserView.TVSeriesManager = Resolve<ITVSeriesManager>();

MediaBrowser.Controller/Entities/Folder.cs

@@ -35,6 +35,46 @@ namespace MediaBrowser.Controller.Entities
     /// </summary>
     public class Folder : BaseItem
     {
+        /// <summary>
+        /// Contains constants used when reporting scan progress.
+        /// </summary>
+        private static class ProgressHelpers
+        {
+            /// <summary>
+            /// Reported after the folders immediate children are retrieved.
+            /// </summary>
+            public const int RetrievedChildren = 5;
+
+            /// <summary>
+            /// Reported after add, updating, or deleting child items from the LibraryManager.
+            /// </summary>
+            public const int UpdatedChildItems = 10;
+
+            /// <summary>
+            /// Reported once subfolders are scanned.
+            /// When scanning subfolders, the progress will be between [UpdatedItems, ScannedSubfolders].
+            /// </summary>
+            public const int ScannedSubfolders = 50;
+
+            /// <summary>
+            /// Reported once metadata is refreshed.
+            /// When refreshing metadata, the progress will be between [ScannedSubfolders, MetadataRefreshed].
+            /// </summary>
+            public const int RefreshedMetadata = 100;
+
+            /// <summary>
+            /// Gets the current progress given the previous step, next step, and progress in between.
+            /// </summary>
+            /// <param name="previousProgressStep">The previous progress step.</param>
+            /// <param name="nextProgressStep">The next progress step.</param>
+            /// <param name="currentProgress">The current progress step.</param>
+            /// <returns>The progress.</returns>
+            public static double GetProgress(int previousProgressStep, int nextProgressStep, double currentProgress)
+            {
+                return previousProgressStep + ((nextProgressStep - previousProgressStep) * (currentProgress / 100));
+            }
+        }
+
         public static IUserViewManager UserViewManager { get; set; }
 
         /// <summary>
@@ -327,11 +367,11 @@ namespace MediaBrowser.Controller.Entities
                     return;
                 }
 
-                progress.Report(5);
+                progress.Report(ProgressHelpers.RetrievedChildren);
 
                 if (recursive)
                 {
-                    ProviderManager.OnRefreshProgress(this, 5);
+                    ProviderManager.OnRefreshProgress(this, ProgressHelpers.RetrievedChildren);
                 }
 
                 // Build a dictionary of the current children we have now by Id so we can compare quickly and easily
@@ -392,11 +432,11 @@ namespace MediaBrowser.Controller.Entities
                 validChildrenNeedGeneration = true;
             }
 
-            progress.Report(10);
+            progress.Report(ProgressHelpers.UpdatedChildItems);
 
             if (recursive)
             {
-                ProviderManager.OnRefreshProgress(this, 10);
+                ProviderManager.OnRefreshProgress(this, ProgressHelpers.UpdatedChildItems);
             }
 
             cancellationToken.ThrowIfCancellationRequested();
@@ -406,11 +446,13 @@ namespace MediaBrowser.Controller.Entities
                 var innerProgress = new ActionableProgress<double>();
 
                 var folder = this;
-                innerProgress.RegisterAction(p =>
+                innerProgress.RegisterAction(innerPercent =>
                 {
-                    double newPct = 0.80 * p + 10;
+                    var percent = ProgressHelpers.GetProgress(ProgressHelpers.UpdatedChildItems, ProgressHelpers.ScannedSubfolders, innerPercent);
-                    progress.Report(newPct);
+
-                    ProviderManager.OnRefreshProgress(folder, newPct);
+                    progress.Report(percent);
+
+                    ProviderManager.OnRefreshProgress(folder, percent);
                 });
 
                 if (validChildrenNeedGeneration)
@@ -424,11 +466,11 @@ namespace MediaBrowser.Controller.Entities
 
             if (refreshChildMetadata)
             {
-                progress.Report(90);
+                progress.Report(ProgressHelpers.ScannedSubfolders);
 
                 if (recursive)
                 {
-                    ProviderManager.OnRefreshProgress(this, 90);
+                    ProviderManager.OnRefreshProgress(this, ProgressHelpers.ScannedSubfolders);
                 }
 
                 var container = this as IMetadataContainer;
@@ -436,13 +478,15 @@ namespace MediaBrowser.Controller.Entities
                 var innerProgress = new ActionableProgress<double>();
 
                 var folder = this;
-                innerProgress.RegisterAction(p =>
+                innerProgress.RegisterAction(innerPercent =>
                 {
-                    double newPct = 0.10 * p + 90;
+                    var percent = ProgressHelpers.GetProgress(ProgressHelpers.ScannedSubfolders, ProgressHelpers.RefreshedMetadata, innerPercent);
-                    progress.Report(newPct);
+
+                    progress.Report(percent);
+
                     if (recursive)
                     {
-                        ProviderManager.OnRefreshProgress(folder, newPct);
+                        ProviderManager.OnRefreshProgress(folder, percent);
                     }
                 });
 
@@ -457,55 +501,37 @@ namespace MediaBrowser.Controller.Entities
                         validChildren = Children.ToList();
                     }
 
-                    await RefreshMetadataRecursive(validChildren, refreshOptions, recursive, innerProgress, cancellationToken);
+                    await RefreshMetadataRecursive(validChildren, refreshOptions, recursive, innerProgress, cancellationToken).ConfigureAwait(false);
                 }
             }
         }
 
-        private async Task RefreshMetadataRecursive(List<BaseItem> children, MetadataRefreshOptions refreshOptions, bool recursive, IProgress<double> progress, CancellationToken cancellationToken)
+        private Task RefreshMetadataRecursive(IList<BaseItem> children, MetadataRefreshOptions refreshOptions, bool recursive, IProgress<double> progress, CancellationToken cancellationToken)
         {
-            var numComplete = 0;
+            var progressableTasks = children
-            var count = children.Count;
+                .Select<BaseItem, Func<IProgress<double>, Task>>(child =>
-            double currentPercent = 0;
+                    innerProgress => RefreshChildMetadata(child, refreshOptions, recursive && child.IsFolder, innerProgress, cancellationToken))
+                .ToList();
 
-            foreach (var child in children)
+            return RunTasks(progressableTasks, progress, cancellationToken);
-            {
-                cancellationToken.ThrowIfCancellationRequested();
-
-                var innerProgress = new ActionableProgress<double>();
-
-                // Avoid implicitly captured closure
-                var currentInnerPercent = currentPercent;
-
-                innerProgress.RegisterAction(p =>
-                {
-                    double innerPercent = currentInnerPercent;
-                    innerPercent += p / count;
-                    progress.Report(innerPercent);
-                });
-
-                await RefreshChildMetadata(child, refreshOptions, recursive && child.IsFolder, innerProgress, cancellationToken)
-                    .ConfigureAwait(false);
-
-                numComplete++;
-                double percent = numComplete;
-                percent /= count;
-                percent *= 100;
-                currentPercent = percent;
-
-                progress.Report(percent);
-            }
         }
 
         private async Task RefreshAllMetadataForContainer(IMetadataContainer container, MetadataRefreshOptions refreshOptions, IProgress<double> progress, CancellationToken cancellationToken)
         {
-            var series = container as Series;
+            // limit the amount of concurrent metadata refreshes
-            if (series != null)
+            await TaskMethods.RunThrottled(
-            {
+                TaskMethods.SharedThrottleId.RefreshMetadata,
-                await series.RefreshMetadata(refreshOptions, cancellationToken).ConfigureAwait(false);
+                async () =>
-            }
+                {
+                    var series = container as Series;
+                    if (series != null)
+                    {
+                        await series.RefreshMetadata(refreshOptions, cancellationToken).ConfigureAwait(false);
+                    }
 
-            await container.RefreshAllMetadata(refreshOptions, progress, cancellationToken).ConfigureAwait(false);
+                    await container.RefreshAllMetadata(refreshOptions, progress, cancellationToken).ConfigureAwait(false);
+                },
+                cancellationToken).ConfigureAwait(false);
         }
 
         private async Task RefreshChildMetadata(BaseItem child, MetadataRefreshOptions refreshOptions, bool recursive, IProgress<double> progress, CancellationToken cancellationToken)
@@ -520,12 +546,16 @@ namespace MediaBrowser.Controller.Entities
             {
                 if (refreshOptions.RefreshItem(child))
                 {
-                    await child.RefreshMetadata(refreshOptions, cancellationToken).ConfigureAwait(false);
+                    // limit the amount of concurrent metadata refreshes
+                    await TaskMethods.RunThrottled(
+                        TaskMethods.SharedThrottleId.RefreshMetadata,
+                        async () => await child.RefreshMetadata(refreshOptions, cancellationToken).ConfigureAwait(false),
+                        cancellationToken).ConfigureAwait(false);
                 }
 
                 if (recursive && child is Folder folder)
                 {
-                    await folder.RefreshMetadataRecursive(folder.Children.ToList(), refreshOptions, true, progress, cancellationToken);
+                    await folder.RefreshMetadataRecursive(folder.Children.ToList(), refreshOptions, true, progress, cancellationToken).ConfigureAwait(false);
                 }
             }
         }
@@ -538,39 +568,63 @@ namespace MediaBrowser.Controller.Entities
         /// <param name="progress">The progress.</param>
         /// <param name="cancellationToken">The cancellation token.</param>
         /// <returns>Task.</returns>
-        private async Task ValidateSubFolders(IList<Folder> children, IDirectoryService directoryService, IProgress<double> progress, CancellationToken cancellationToken)
+        private Task ValidateSubFolders(IList<Folder> children, IDirectoryService directoryService, IProgress<double> progress, CancellationToken cancellationToken)
         {
-            var numComplete = 0;
+            var progressableTasks = children
-            var count = children.Count;
+                .Select<Folder, Func<IProgress<double>, Task>>(child =>
-            double currentPercent = 0;
+                    innerProgress => child.ValidateChildrenInternal(innerProgress, cancellationToken, true, false, null, directoryService))
+                .ToList();
 
-            foreach (var child in children)
+            return RunTasks(progressableTasks, progress, cancellationToken);
+        }
+
+        /// <summary>
+        /// Runs a set of tasks concurrently with progress.
+        /// </summary>
+        /// <param name="tasks">A list of tasks.</param>
+        /// <param name="progress">The progress.</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <returns>Task.</returns>
+        private async Task RunTasks(IList<Func<IProgress<double>, Task>> tasks, IProgress<double> progress, CancellationToken cancellationToken)
+        {
+            var childrenCount = tasks.Count;
+            var childrenProgress = new double[childrenCount];
+            var actions = new Func<Task>[childrenCount];
+
+            void UpdateProgress()
             {
-                cancellationToken.ThrowIfCancellationRequested();
+                progress.Report(childrenProgress.Average());
-
-                var innerProgress = new ActionableProgress<double>();
-
-                // Avoid implicitly captured closure
-                var currentInnerPercent = currentPercent;
-
-                innerProgress.RegisterAction(p =>
-                {
-                    double innerPercent = currentInnerPercent;
-                    innerPercent += p / count;
-                    progress.Report(innerPercent);
-                });
-
-                await child.ValidateChildrenInternal(innerProgress, cancellationToken, true, false, null, directoryService)
-                        .ConfigureAwait(false);
-
-                numComplete++;
-                double percent = numComplete;
-                percent /= count;
-                percent *= 100;
-                currentPercent = percent;
-
-                progress.Report(percent);
             }
+
+            for (var i = 0; i < childrenCount; i++)
+            {
+                var childIndex = i;
+                var child = tasks[childIndex];
+
+                actions[childIndex] = async () =>
+                {
+                    var innerProgress = new ActionableProgress<double>();
+
+                    innerProgress.RegisterAction(innerPercent =>
+                    {
+                        // round the percent and only update progress if it changed to prevent excessive UpdateProgress calls
+                        var innerPercentRounded = Math.Round(innerPercent);
+                        if (childrenProgress[childIndex] != innerPercentRounded)
+                        {
+                            childrenProgress[childIndex] = innerPercentRounded;
+                            UpdateProgress();
+                        }
+                    });
+
+                    await tasks[childIndex](innerProgress).ConfigureAwait(false);
+
+                    childrenProgress[childIndex] = 100;
+
+                    UpdateProgress();
+                };
+            }
+
+            await TaskMethods.WhenAllThrottled(TaskMethods.SharedThrottleId.ScanFanout, actions, cancellationToken).ConfigureAwait(false);
         }
 
         /// <summary>

MediaBrowser.Controller/Library/TaskMethods.cs

@@ -0,0 +1,133 @@
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using MediaBrowser.Controller.Configuration;
+
+namespace MediaBrowser.Controller.Library
+{
+    /// <summary>
+    /// Helper methods for running tasks concurrently.
+    /// </summary>
+    public static class TaskMethods
+    {
+        private static readonly int _processorCount = Environment.ProcessorCount;
+
+        private static readonly ConcurrentDictionary<SharedThrottleId, SemaphoreSlim> _sharedThrottlers = new ConcurrentDictionary<SharedThrottleId, SemaphoreSlim>();
+
+        /// <summary>
+        /// Throttle id for sharing a concurrency limit.
+        /// </summary>
+        public enum SharedThrottleId
+        {
+            /// <summary>
+            /// Library scan fan out
+            /// </summary>
+            ScanFanout,
+
+            /// <summary>
+            /// Refresh metadata
+            /// </summary>
+            RefreshMetadata,
+        }
+
+        /// <summary>
+        /// Gets or sets the configuration manager.
+        /// </summary>
+        public static IServerConfigurationManager ConfigurationManager { get; set; }
+
+        /// <summary>
+        /// Similiar to Task.WhenAll but only allows running a certain amount of tasks at the same time.
+        /// </summary>
+        /// <param name="throttleId">The throttle id. Multiple calls to this method with the same throttle id will share a concurrency limit.</param>
+        /// <param name="actions">List of actions to run.</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <returns>A <see cref="Task"/> representing the result of the asynchronous operation.</returns>
+        public static async Task WhenAllThrottled(SharedThrottleId throttleId, IEnumerable<Func<Task>> actions, CancellationToken cancellationToken)
+        {
+            var taskThrottler = throttleId == SharedThrottleId.ScanFanout ?
+                new SemaphoreSlim(GetConcurrencyLimit(throttleId)) :
+                _sharedThrottlers.GetOrAdd(throttleId, id => new SemaphoreSlim(GetConcurrencyLimit(id)));
+
+            try
+            {
+                var tasks = new List<Task>();
+
+                foreach (var action in actions)
+                {
+                    await taskThrottler.WaitAsync(cancellationToken).ConfigureAwait(false);
+
+                    tasks.Add(Task.Run(async () =>
+                    {
+                        try
+                        {
+                            await action().ConfigureAwait(false);
+                        }
+                        finally
+                        {
+                            taskThrottler.Release();
+                        }
+                    }));
+                }
+
+                await Task.WhenAll(tasks).ConfigureAwait(false);
+            }
+            finally
+            {
+                if (throttleId == SharedThrottleId.ScanFanout)
+                {
+                    taskThrottler.Dispose();
+                }
+            }
+        }
+
+        /// <summary>
+        /// Runs a task within a given throttler.
+        /// </summary>
+        /// <param name="throttleId">The throttle id. Multiple calls to this method with the same throttle id will share a concurrency limit.</param>
+        /// <param name="action">The action to run.</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <returns>A <see cref="Task"/> representing the result of the asynchronous operation.</returns>
+        public static async Task RunThrottled(SharedThrottleId throttleId, Func<Task> action, CancellationToken cancellationToken)
+        {
+            if (throttleId == SharedThrottleId.ScanFanout)
+            {
+                // just await the task instead
+                throw new InvalidOperationException("Invalid throttle id");
+            }
+
+            var taskThrottler = _sharedThrottlers.GetOrAdd(throttleId, id => new SemaphoreSlim(GetConcurrencyLimit(id)));
+
+            await taskThrottler.WaitAsync(cancellationToken).ConfigureAwait(false);
+
+            try
+            {
+                await action().ConfigureAwait(false);
+            }
+            finally
+            {
+                taskThrottler.Release();
+            }
+        }
+
+        /// <summary>
+        /// Get the concurrency limit for the given throttle id.
+        /// </summary>
+        /// <param name="throttleId">The throttle id.</param>
+        /// <returns>The concurrency limit.</returns>
+        private static int GetConcurrencyLimit(SharedThrottleId throttleId)
+        {
+            var concurrency = throttleId == SharedThrottleId.RefreshMetadata ?
+                ConfigurationManager.Configuration.LibraryMetadataRefreshConcurrency :
+                ConfigurationManager.Configuration.LibraryScanFanoutConcurrency;
+
+            if (concurrency <= 0)
+            {
+                concurrency = _processorCount;
+            }
+
+            return concurrency;
+        }
+    }
+}

MediaBrowser.Model/Configuration/ServerConfiguration.cs

@@ -271,6 +271,16 @@ namespace MediaBrowser.Model.Configuration
         /// </summary>
         public string[] KnownProxies { get; set; }
 
+        /// <summary>
+        /// Gets or sets the how the library scan fans out.
+        /// </summary>
+        public int LibraryScanFanoutConcurrency { get; set; }
+
+        /// <summary>
+        /// Gets or sets the how many metadata refreshes can run concurrently.
+        /// </summary>
+        public int LibraryMetadataRefreshConcurrency { get; set; }
+
         /// <summary>
         /// Initializes a new instance of the <see cref="ServerConfiguration" /> class.
         /// </summary>
@@ -381,6 +391,8 @@ namespace MediaBrowser.Model.Configuration
             SlowResponseThresholdMs = 500;
             CorsHosts = new[] { "*" };
             KnownProxies = Array.Empty<string>();
+            LibraryMetadataRefreshConcurrency = 0;
+            LibraryScanFanoutConcurrency = 0;
         }
     }