Merge pull request #1105 from ploughpuff/ratelimit

Only delay making MusicBrainz request if necessary

Emby.Server.Implementations/ApplicationHost.cs

@@ -428,6 +428,12 @@ namespace Emby.Server.Implementations
         /// <value>The application user agent.</value>
         public string ApplicationUserAgent => Name.Replace(' ','-') + "/" + ApplicationVersion;
 
+        /// <summary>
+        /// Gets the email address for use within a comment section of a user agent field.
+        /// Presently used to provide contact information to MusicBrainz service.
+        /// </summary>
+        public string ApplicationUserAgentAddress { get; } = "team@jellyfin.org";
+
         private string _productName;
 
         /// <summary>

MediaBrowser.Common/IApplicationHost.cs

@@ -71,6 +71,12 @@ namespace MediaBrowser.Common
         /// <value>The application user agent.</value>
         string ApplicationUserAgent { get; }
 
+        /// <summary>
+        /// Gets the email address for use within a comment section of a user agent field.
+        /// Presently used to provide contact information to MusicBrainz service.
+        /// </summary>
+        string ApplicationUserAgentAddress { get; }
+
         /// <summary>
         /// Gets the exports.
         /// </summary>

MediaBrowser.Providers/Music/MusicBrainzAlbumProvider.cs

@@ -1,6 +1,6 @@
 using System;
 using System.Collections.Generic;
-using System.Globalization;
+using System.Diagnostics;
 using System.IO;
 using System.Linq;
 using System.Net;
@@ -28,9 +28,16 @@ namespace MediaBrowser.Providers.Music
         private readonly IApplicationHost _appHost;
         private readonly ILogger _logger;
         private readonly IJsonSerializer _json;
+        private Stopwatch _stopWatchMusicBrainz = new Stopwatch();
 
         public readonly string MusicBrainzBaseUrl;
 
+        // The Jellyfin user-agent is unrestricted but source IP must not exceed
+        // one request per second, therefore we rate limit to avoid throttling.
+        // Be prudent, use a value slightly above the minimun required.
+        // https://musicbrainz.org/doc/XML_Web_Service/Rate_Limiting
+        private const long MusicBrainzQueryIntervalMs = 1050u;
+
         public MusicBrainzAlbumProvider(
             IHttpClient httpClient,
             IApplicationHost appHost,
@@ -45,6 +52,9 @@ namespace MediaBrowser.Providers.Music
 
             MusicBrainzBaseUrl = configuration["MusicBrainz:BaseUrl"];
 
+            // Use a stopwatch to ensure we don't exceed the MusicBrainz rate limit
+            _stopWatchMusicBrainz.Start();
+
             Current = this;
         }
 
@@ -54,8 +64,6 @@ namespace MediaBrowser.Providers.Music
             var releaseGroupId = searchInfo.GetReleaseGroupId();
 
             string url;
-            var isNameSearch = false;
-            bool forceMusicBrainzProper = false;
 
             if (!string.IsNullOrEmpty(releaseId))
             {
@@ -64,7 +72,6 @@ namespace MediaBrowser.Providers.Music
             else if (!string.IsNullOrEmpty(releaseGroupId))
             {
                 url = string.Format("/ws/2/release?release-group={0}", releaseGroupId);
-                forceMusicBrainzProper = true;
             }
             else
             {
@@ -78,8 +85,6 @@ namespace MediaBrowser.Providers.Music
                 }
                 else
                 {
-                    isNameSearch = true;
-
                     // I'm sure there is a better way but for now it resolves search for 12" Mixes
                     var queryName = searchInfo.Name.Replace("\"", string.Empty);
 
@@ -91,7 +96,7 @@ namespace MediaBrowser.Providers.Music
 
             if (!string.IsNullOrWhiteSpace(url))
             {
-                using (var response = await GetMusicBrainzResponse(url, isNameSearch, forceMusicBrainzProper, cancellationToken).ConfigureAwait(false))
+                using (var response = await GetMusicBrainzResponse(url, cancellationToken).ConfigureAwait(false))
                 {
                     using (var stream = response.Content)
                     {
@@ -247,7 +252,7 @@ namespace MediaBrowser.Providers.Music
                 WebUtility.UrlEncode(albumName),
                 artistId);
 
-            using (var response = await GetMusicBrainzResponse(url, true, cancellationToken).ConfigureAwait(false))
+            using (var response = await GetMusicBrainzResponse(url, cancellationToken).ConfigureAwait(false))
             using (var stream = response.Content)
             using (var oReader = new StreamReader(stream, Encoding.UTF8))
             {
@@ -272,7 +277,7 @@ namespace MediaBrowser.Providers.Music
                 WebUtility.UrlEncode(albumName),
                 WebUtility.UrlEncode(artistName));
 
-            using (var response = await GetMusicBrainzResponse(url, true, cancellationToken).ConfigureAwait(false))
+            using (var response = await GetMusicBrainzResponse(url, cancellationToken).ConfigureAwait(false))
             using (var stream = response.Content)
             using (var oReader = new StreamReader(stream, Encoding.UTF8))
             {
@@ -589,7 +594,7 @@ namespace MediaBrowser.Providers.Music
         {
             var url = string.Format("/ws/2/release?release-group={0}", releaseGroupId);
 
-            using (var response = await GetMusicBrainzResponse(url, true, true, cancellationToken).ConfigureAwait(false))
+            using (var response = await GetMusicBrainzResponse(url, cancellationToken).ConfigureAwait(false))
             using (var stream = response.Content)
             using (var oReader = new StreamReader(stream, Encoding.UTF8))
             {
@@ -625,7 +630,7 @@ namespace MediaBrowser.Providers.Music
         {
             var url = string.Format("/ws/2/release-group/?query=reid:{0}", releaseEntryId);
 
-            using (var response = await GetMusicBrainzResponse(url, false, cancellationToken).ConfigureAwait(false))
+            using (var response = await GetMusicBrainzResponse(url, cancellationToken).ConfigureAwait(false))
             using (var stream = response.Content)
             using (var oReader = new StreamReader(stream, Encoding.UTF8))
             {
@@ -710,34 +715,29 @@ namespace MediaBrowser.Providers.Music
             return null;
         }
 
-        internal Task<HttpResponseInfo> GetMusicBrainzResponse(string url, bool isSearch, CancellationToken cancellationToken)
-        {
-            return GetMusicBrainzResponse(url, isSearch, false, cancellationToken);
-        }
-
         /// <summary>
-        /// Gets the music brainz response.
+        /// Makes request to MusicBrainz server and awaits a response.
         /// </summary>
-        internal async Task<HttpResponseInfo> GetMusicBrainzResponse(string url, bool isSearch, bool forceMusicBrainzProper, CancellationToken cancellationToken)
+        internal async Task<HttpResponseInfo> GetMusicBrainzResponse(string url, CancellationToken cancellationToken)
         {
-            var urlInfo = new MbzUrl(MusicBrainzBaseUrl, 1000);
+            if (_stopWatchMusicBrainz.ElapsedMilliseconds < MusicBrainzQueryIntervalMs)
-            var throttleMs = urlInfo.throttleMs;
-
-            if (throttleMs > 0)
             {
                 // MusicBrainz is extremely adamant about limiting to one request per second
-                _logger.LogDebug("Throttling MusicBrainz by {0}ms", throttleMs.ToString(CultureInfo.InvariantCulture));
+                var delayMs = MusicBrainzQueryIntervalMs - _stopWatchMusicBrainz.ElapsedMilliseconds;
-                await Task.Delay(throttleMs, cancellationToken).ConfigureAwait(false);
+                await Task.Delay((int)delayMs, cancellationToken).ConfigureAwait(false);
             }
 
-            url = urlInfo.url.TrimEnd('/') + url;
+            _logger.LogDebug("MusicBrainz time since previous request: {0}ms", _stopWatchMusicBrainz.ElapsedMilliseconds);
+            _stopWatchMusicBrainz.Restart();
 
             var options = new HttpRequestOptions
             {
-                Url = url,
+                Url = MusicBrainzBaseUrl.TrimEnd('/') + url,
                 CancellationToken = cancellationToken,
-                UserAgent = _appHost.ApplicationUserAgent,
+                // MusicBrainz request a contact email address is supplied, as comment, in user agent field:
-                BufferContent = throttleMs > 0
+                // https://musicbrainz.org/doc/XML_Web_Service/Rate_Limiting#User-Agent
+                UserAgent = string.Format("{0} ( {1} )", _appHost.ApplicationUserAgent, _appHost.ApplicationUserAgentAddress),
+                BufferContent = false
             };
 
             return await _httpClient.SendAsync(options, "GET").ConfigureAwait(false);
@@ -749,17 +749,5 @@ namespace MediaBrowser.Providers.Music
         {
             throw new NotImplementedException();
         }
-
-        internal class MbzUrl
-        {
-            internal MbzUrl(string url, int throttleMs)
-            {
-                this.url = url;
-                this.throttleMs = throttleMs;
-            }
-
-            public string url { get; set; }
-            public int throttleMs { get; set; }
-        }
     }
 }

MediaBrowser.Providers/Music/MusicBrainzArtistProvider.cs

@@ -31,7 +31,7 @@ namespace MediaBrowser.Providers.Music
             {
                 var url = string.Format("/ws/2/artist/?query=arid:{0}", musicBrainzId);
 
-                using (var response = await MusicBrainzAlbumProvider.Current.GetMusicBrainzResponse(url, false, cancellationToken).ConfigureAwait(false))
+                using (var response = await MusicBrainzAlbumProvider.Current.GetMusicBrainzResponse(url, cancellationToken).ConfigureAwait(false))
                 {
                     using (var stream = response.Content)
                     {
@@ -46,7 +46,7 @@ namespace MediaBrowser.Providers.Music
 
                 var url = string.Format("/ws/2/artist/?query=\"{0}\"&dismax=true", UrlEncode(nameToSearch));
 
-                using (var response = await MusicBrainzAlbumProvider.Current.GetMusicBrainzResponse(url, true, cancellationToken).ConfigureAwait(false))
+                using (var response = await MusicBrainzAlbumProvider.Current.GetMusicBrainzResponse(url, cancellationToken).ConfigureAwait(false))
                 {
                     using (var stream = response.Content)
                     {
@@ -64,7 +64,7 @@ namespace MediaBrowser.Providers.Music
                     // Try again using the search with accent characters url
                     url = string.Format("/ws/2/artist/?query=artistaccent:\"{0}\"", UrlEncode(nameToSearch));
 
-                    using (var response = await MusicBrainzAlbumProvider.Current.GetMusicBrainzResponse(url, true, cancellationToken).ConfigureAwait(false))
+                    using (var response = await MusicBrainzAlbumProvider.Current.GetMusicBrainzResponse(url, cancellationToken).ConfigureAwait(false))
                     {
                         using (var stream = response.Content)
                         {