limit number of tmdb requests per second

MediaBrowser.Controller/Providers/IProviderManager.cs

@@ -21,6 +21,8 @@ namespace MediaBrowser.Controller.Providers
         /// <exception cref="System.ArgumentNullException">item</exception>
         Task<string> DownloadAndSaveImage(BaseItem item, string source, string targetName, bool saveLocally, SemaphoreSlim resourcePool, CancellationToken cancellationToken);
 
+        Task<string> SaveImage(BaseItem item, Stream source, string targetName, bool saveLocally, CancellationToken cancellationToken);
+        
         /// <summary>
         /// Saves to library filesystem.
         /// </summary>

MediaBrowser.Controller/Providers/Movies/MovieDbImagesProvider.cs

@@ -207,11 +207,10 @@ namespace MediaBrowser.Controller.Providers.Movies
         /// <returns>Task{MovieImages}.</returns>
         private async Task<MovieImages> FetchImages(BaseItem item, string id, CancellationToken cancellationToken)
         {
-            using (var json = await _httpClient.Get(new HttpRequestOptions
+            using (var json = await MovieDbProvider.Current.GetMovieDbResponse(new HttpRequestOptions
             {
                 Url = string.Format(GetImages, id, MovieDbProvider.ApiKey, item is BoxSet ? "collection" : "movie"),
                 CancellationToken = cancellationToken,
-                ResourcePool = MovieDbProvider.Current.MovieDbResourcePool,
                 AcceptHeader = MovieDbProvider.AcceptHeader,
                 EnableResponseCache = true
 
@@ -264,14 +263,14 @@ namespace MediaBrowser.Controller.Providers.Movies
                 }
                 if (poster != null)
                 {
-                    try
+                    var img = await MovieDbProvider.Current.GetMovieDbResponse(new HttpRequestOptions
                     {
-                        item.PrimaryImagePath = await _providerManager.DownloadAndSaveImage(item, tmdbImageUrl + poster.file_path, "folder" + Path.GetExtension(poster.file_path), ConfigurationManager.Configuration.SaveLocalMeta && item.LocationType == LocationType.FileSystem, MovieDbProvider.Current.MovieDbResourcePool, cancellationToken).ConfigureAwait(false);
+                        Url = tmdbImageUrl + poster.file_path,
-                    }
+                        CancellationToken = cancellationToken
-                    catch (HttpException)
+
-                    {
+                    }).ConfigureAwait(false);
-                        status = ProviderRefreshStatus.CompletedWithErrors;
+
-                    }
+                    item.PrimaryImagePath = await _providerManager.SaveImage(item, img, "folder" + Path.GetExtension(poster.file_path), ConfigurationManager.Configuration.SaveLocalMeta && item.LocationType == LocationType.FileSystem, cancellationToken).ConfigureAwait(false);
                 }
             }
 
@@ -295,14 +294,14 @@ namespace MediaBrowser.Controller.Providers.Movies
 
                     if (ConfigurationManager.Configuration.RefreshItemImages || !hasLocalBackdrop)
                     {
-                        try
+                        var img = await MovieDbProvider.Current.GetMovieDbResponse(new HttpRequestOptions
                         {
-                            item.BackdropImagePaths.Add(await _providerManager.DownloadAndSaveImage(item, tmdbImageUrl + images.backdrops[i].file_path, bdName + Path.GetExtension(images.backdrops[i].file_path), ConfigurationManager.Configuration.SaveLocalMeta && item.LocationType == LocationType.FileSystem, MovieDbProvider.Current.MovieDbResourcePool, cancellationToken).ConfigureAwait(false));
+                            Url = tmdbImageUrl + images.backdrops[i].file_path,
-                        }
+                            CancellationToken = cancellationToken
-                        catch (HttpException)
+
-                        {
+                        }).ConfigureAwait(false);
-                            status = ProviderRefreshStatus.CompletedWithErrors;
+
-                        }
+                        item.BackdropImagePaths.Add(await _providerManager.SaveImage(item, img, bdName + Path.GetExtension(images.backdrops[i].file_path), ConfigurationManager.Configuration.SaveLocalMeta && item.LocationType == LocationType.FileSystem, cancellationToken).ConfigureAwait(false));
                     }
                 }
             }

MediaBrowser.Controller/Providers/Movies/MovieDbProvider.cs

@@ -40,7 +40,7 @@ namespace MediaBrowser.Controller.Providers.Movies
         /// <summary>
         /// The movie db
         /// </summary>
-        internal readonly SemaphoreSlim MovieDbResourcePool = new SemaphoreSlim(1,1);
+        private readonly SemaphoreSlim _movieDbResourcePool = new SemaphoreSlim(1,1);
 
         internal static MovieDbProvider Current { get; private set; }
 
@@ -81,7 +81,7 @@ namespace MediaBrowser.Controller.Providers.Movies
         {
             if (dispose)
             {
-                MovieDbResourcePool.Dispose();
+                _movieDbResourcePool.Dispose();
             }
         }
 
@@ -172,7 +172,7 @@ namespace MediaBrowser.Controller.Providers.Movies
         {
             get
             {
-                LazyInitializer.EnsureInitialized(ref _tmdbSettingsTask, ref _tmdbSettingsTaskInitialized, ref _tmdbSettingsTaskSyncLock, () => GetTmdbSettings(JsonSerializer, HttpClient));
+                LazyInitializer.EnsureInitialized(ref _tmdbSettingsTask, ref _tmdbSettingsTaskInitialized, ref _tmdbSettingsTaskSyncLock, () => GetTmdbSettings(JsonSerializer));
                 return _tmdbSettingsTask;
             }
         }
@@ -181,15 +181,14 @@ namespace MediaBrowser.Controller.Providers.Movies
         /// Gets the TMDB settings.
         /// </summary>
         /// <returns>Task{TmdbSettingsResult}.</returns>
-        private static async Task<TmdbSettingsResult> GetTmdbSettings(IJsonSerializer jsonSerializer, IHttpClient httpClient)
+        private async Task<TmdbSettingsResult> GetTmdbSettings(IJsonSerializer jsonSerializer)
         {
             try
             {
-                using (var json = await httpClient.Get(new HttpRequestOptions
+                using (var json = await GetMovieDbResponse(new HttpRequestOptions
                 {
                     Url = string.Format(TmdbConfigUrl, ApiKey),
                     CancellationToken = CancellationToken.None,
-                    ResourcePool = Current.MovieDbResourcePool,
                     AcceptHeader = AcceptHeader,
                     EnableResponseCache = true
 
@@ -560,11 +559,10 @@ namespace MediaBrowser.Controller.Providers.Movies
 
             try
             {
-                using (Stream json = await HttpClient.Get(new HttpRequestOptions
+                using (Stream json = await GetMovieDbResponse(new HttpRequestOptions
                 {
                     Url = url3,
                     CancellationToken = cancellationToken,
-                    ResourcePool = Current.MovieDbResourcePool,
                     AcceptHeader = AcceptHeader,
                     EnableResponseCache = true
 
@@ -600,11 +598,10 @@ namespace MediaBrowser.Controller.Providers.Movies
 
                 try
                 {
-                    using (var json = await HttpClient.Get(new HttpRequestOptions
+                    using (var json = await GetMovieDbResponse(new HttpRequestOptions
                     {
                         Url = url3,
                         CancellationToken = cancellationToken,
-                        ResourcePool = Current.MovieDbResourcePool,
                         AcceptHeader = AcceptHeader,
                         EnableResponseCache = true
 
@@ -647,11 +644,10 @@ namespace MediaBrowser.Controller.Providers.Movies
 
                         try
                         {
-                            using (var json = await HttpClient.Get(new HttpRequestOptions
+                            using (var json = await GetMovieDbResponse(new HttpRequestOptions
                             {
                                 Url = url3,
                                 CancellationToken = cancellationToken,
-                                ResourcePool = Current.MovieDbResourcePool,
                                 AcceptHeader = AcceptHeader,
                                 EnableResponseCache = true
 
@@ -737,11 +733,10 @@ namespace MediaBrowser.Controller.Providers.Movies
 
                 try
                 {
-                    using (Stream json = await HttpClient.Get(new HttpRequestOptions
+                    using (Stream json = await GetMovieDbResponse(new HttpRequestOptions
                     {
                         Url = url,
                         CancellationToken = cancellationToken,
-                        ResourcePool = Current.MovieDbResourcePool,
                         AcceptHeader = AcceptHeader,
                         EnableResponseCache = true
 
@@ -822,11 +817,10 @@ namespace MediaBrowser.Controller.Providers.Movies
 
             try
             {
-                using (var json = await HttpClient.Get(new HttpRequestOptions
+                using (var json = await GetMovieDbResponse(new HttpRequestOptions
                 {
                     Url = url,
                     CancellationToken = cancellationToken,
-                    ResourcePool = Current.MovieDbResourcePool,
                     AcceptHeader = AcceptHeader,
                     EnableResponseCache = true
 
@@ -863,11 +857,10 @@ namespace MediaBrowser.Controller.Providers.Movies
 
                     try
                     {
-                        using (Stream json = await HttpClient.Get(new HttpRequestOptions
+                        using (Stream json = await GetMovieDbResponse(new HttpRequestOptions
                         {
                             Url = url,
                             CancellationToken = cancellationToken,
-                            ResourcePool = Current.MovieDbResourcePool,
                             AcceptHeader = AcceptHeader,
                             EnableResponseCache = true
 
@@ -1022,6 +1015,38 @@ namespace MediaBrowser.Controller.Providers.Movies
 
         }
 
+        private DateTime _lastRequestDate = DateTime.MinValue;
+
+        /// <summary>
+        /// Gets the movie db response.
+        /// </summary>
+        internal async Task<Stream> GetMovieDbResponse(HttpRequestOptions options)
+        {
+            var cancellationToken = options.CancellationToken;
+
+            await _movieDbResourcePool.WaitAsync(cancellationToken).ConfigureAwait(false);
+
+            try
+            {
+                // Limit to three requests per second
+                var diff = 330 - (DateTime.Now - _lastRequestDate).TotalMilliseconds;
+
+                if (diff > 0)
+                {
+                    await Task.Delay(Convert.ToInt32(diff), cancellationToken).ConfigureAwait(false);
+                }
+
+                _lastRequestDate = DateTime.Now;
+
+                return await HttpClient.Get(options).ConfigureAwait(false);
+            }
+            finally
+            {
+                _lastRequestDate = DateTime.Now;
+
+                _movieDbResourcePool.Release();
+            }
+        }
 
         /// <summary>
         /// The remove

MediaBrowser.Controller/Providers/Movies/PersonProviderFromJson.cs

@@ -1,5 +1,4 @@
-using MediaBrowser.Common.Net;
+using MediaBrowser.Controller.Configuration;
-using MediaBrowser.Controller.Configuration;
 using MediaBrowser.Controller.Entities;
 using MediaBrowser.Model.Logging;
 using MediaBrowser.Model.Serialization;
@@ -15,8 +14,8 @@ namespace MediaBrowser.Controller.Providers.Movies
     /// </summary>
     class PersonProviderFromJson : TmdbPersonProvider
     {
-        public PersonProviderFromJson(IHttpClient httpClient, IJsonSerializer jsonSerializer, ILogManager logManager, IServerConfigurationManager configurationManager, IProviderManager providerManager) 
+        public PersonProviderFromJson(IJsonSerializer jsonSerializer, ILogManager logManager, IServerConfigurationManager configurationManager, IProviderManager providerManager) 
-            : base(httpClient, jsonSerializer, logManager, configurationManager, providerManager)
+            : base(jsonSerializer, logManager, configurationManager, providerManager)
         {
         }
 

MediaBrowser.Controller/Providers/Movies/TmdbPersonProvider.cs

@@ -28,18 +28,13 @@ namespace MediaBrowser.Controller.Providers.Movies
 
         protected readonly IProviderManager ProviderManager;
         
-        public TmdbPersonProvider(IHttpClient httpClient, IJsonSerializer jsonSerializer, ILogManager logManager, IServerConfigurationManager configurationManager, IProviderManager providerManager)
+        public TmdbPersonProvider(IJsonSerializer jsonSerializer, ILogManager logManager, IServerConfigurationManager configurationManager, IProviderManager providerManager)
             : base(logManager, configurationManager)
         {
             if (jsonSerializer == null)
             {
                 throw new ArgumentNullException("jsonSerializer");
             }
-            if (httpClient == null)
-            {
-                throw new ArgumentNullException("httpClient");
-            }
-            HttpClient = httpClient;
             JsonSerializer = jsonSerializer;
             ProviderManager = providerManager;
         }
@@ -50,12 +45,6 @@ namespace MediaBrowser.Controller.Providers.Movies
         /// <value>The json serializer.</value>
         protected IJsonSerializer JsonSerializer { get; private set; }
 
-        /// <summary>
-        /// Gets the HTTP client.
-        /// </summary>
-        /// <value>The HTTP client.</value>
-        protected IHttpClient HttpClient { get; private set; }
-
         /// <summary>
         /// Supportses the specified item.
         /// </summary>
@@ -172,11 +161,10 @@ namespace MediaBrowser.Controller.Providers.Movies
 
             try
             {
-                using (Stream json = await HttpClient.Get(new HttpRequestOptions
+                using (Stream json = await MovieDbProvider.Current.GetMovieDbResponse(new HttpRequestOptions
                 {
                     Url = url,
                     CancellationToken = cancellationToken,
-                    ResourcePool = MovieDbProvider.Current.MovieDbResourcePool,
                     AcceptHeader = MovieDbProvider.AcceptHeader,
                     EnableResponseCache = true
 
@@ -204,11 +192,10 @@ namespace MediaBrowser.Controller.Providers.Movies
             string url = string.Format(@"http://api.themoviedb.org/3/person/{1}?api_key={0}&append_to_response=credits,images", MovieDbProvider.ApiKey, id);
             PersonResult searchResult = null;
 
-            using (var json = await HttpClient.Get(new HttpRequestOptions
+            using (var json = await MovieDbProvider.Current.GetMovieDbResponse(new HttpRequestOptions
             {
                 Url = url,
                 CancellationToken = cancellationToken,
-                ResourcePool = MovieDbProvider.Current.MovieDbResourcePool,
                 AcceptHeader = MovieDbProvider.AcceptHeader,
                 EnableResponseCache = true
 
@@ -340,7 +327,12 @@ namespace MediaBrowser.Controller.Providers.Movies
             var localPath = Path.Combine(item.MetaLocation, targetName);
             if (!item.ResolveArgs.ContainsMetaFileByName(targetName))
             {
-                using (var sourceStream = await HttpClient.Get(source, MovieDbProvider.Current.MovieDbResourcePool, cancellationToken).ConfigureAwait(false))
+                using (var sourceStream = await MovieDbProvider.Current.GetMovieDbResponse(new HttpRequestOptions
+                {
+                    Url = source,
+                    CancellationToken = cancellationToken
+
+                }).ConfigureAwait(false))
                 {
                     await ProviderManager.SaveToLibraryFilesystem(item, localPath, sourceStream, cancellationToken).ConfigureAwait(false);
 

MediaBrowser.Controller/Providers/Music/FanArtAlbumProvider.cs

@@ -206,9 +206,7 @@ namespace MediaBrowser.Controller.Providers.Music
         /// <summary>
         /// The _last music brainz request
         /// </summary>
-        private DateTime _lastMusicBrainzRequest = DateTime.MinValue;
+        private DateTime _lastRequestDate = DateTime.MinValue;
-
-        private readonly SemaphoreSlim _musicBrainzSemaphore = new SemaphoreSlim(1, 1);
 
         /// <summary>
         /// Gets the music brainz response.
@@ -222,7 +220,7 @@ namespace MediaBrowser.Controller.Providers.Music
 
             try
             {
-                var diff = 1500 - (DateTime.Now - _lastMusicBrainzRequest).TotalMilliseconds;
+                var diff = 1500 - (DateTime.Now - _lastRequestDate).TotalMilliseconds;
 
                 // MusicBrainz is extremely adamant about limiting to one request per second
 
@@ -231,7 +229,7 @@ namespace MediaBrowser.Controller.Providers.Music
                     await Task.Delay(Convert.ToInt32(diff), cancellationToken).ConfigureAwait(false);
                 }
 
-                _lastMusicBrainzRequest = DateTime.Now;
+                _lastRequestDate = DateTime.Now;
 
                 var doc = new XmlDocument();
 
@@ -239,7 +237,6 @@ namespace MediaBrowser.Controller.Providers.Music
                 {
                     Url = url,
                     CancellationToken = cancellationToken,
-                    ResourcePool = _musicBrainzSemaphore,
                     UserAgent = "MediaBrowserServer/www.mediabrowser3.com",
                     EnableResponseCache = true
 
@@ -255,7 +252,7 @@ namespace MediaBrowser.Controller.Providers.Music
             }
             finally
             {
-                _lastMusicBrainzRequest = DateTime.Now;
+                _lastRequestDate = DateTime.Now;
 
                 _musicBrainzResourcePool.Release();
             }

MediaBrowser.Server.Implementations/Providers/ProviderManager.cs

@@ -357,16 +357,22 @@ namespace MediaBrowser.Server.Implementations.Providers
                 throw new ArgumentNullException("resourcePool");
             }
 
+            var img = await _httpClient.Get(source, resourcePool, cancellationToken).ConfigureAwait(false);
+
+            //download and save locally
+            return await SaveImage(item, img, targetName, saveLocally, cancellationToken).ConfigureAwait(false);
+        }
+
+        public async Task<string> SaveImage(BaseItem item, Stream source, string targetName, bool saveLocally, CancellationToken cancellationToken)
+        {
             //download and save locally
             var localPath = (saveLocally && item.MetaLocation != null) ?
                 Path.Combine(item.MetaLocation, targetName) :
                 _remoteImageCache.GetResourcePath(item.GetType().FullName + item.Path.ToLower(), targetName);
 
-            var img = await _httpClient.Get(source, resourcePool, cancellationToken).ConfigureAwait(false);
-
             if (saveLocally) // queue to media directories
             {
-                await SaveToLibraryFilesystem(item, localPath, img, cancellationToken).ConfigureAwait(false);
+                await SaveToLibraryFilesystem(item, localPath, source, cancellationToken).ConfigureAwait(false);
             }
             else
             {
@@ -376,7 +382,7 @@ namespace MediaBrowser.Server.Implementations.Providers
                 {
                     using (var fs = new FileStream(localPath, FileMode.Create, FileAccess.Write, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, FileOptions.Asynchronous))
                     {
-                        await img.CopyToAsync(fs, StreamDefaults.DefaultCopyToBufferSize, cancellationToken).ConfigureAwait(false);
+                        await source.CopyToAsync(fs, StreamDefaults.DefaultCopyToBufferSize, cancellationToken).ConfigureAwait(false);
                     }
                 }
                 catch (OperationCanceledException)
@@ -390,7 +396,7 @@ namespace MediaBrowser.Server.Implementations.Providers
                 }
                 finally
                 {
-                    img.Dispose();
+                    source.Dispose();
                 }
 
             }