From e621327042e64131639716538a4b054a68c85772 Mon Sep 17 00:00:00 2001
From: Zoe Roux <zoe.roux@sdg.moe>
Date: Tue, 16 Jun 2020 18:05:07 +0200
Subject: [PATCH] Spliting tasks into pools

---
 Kyoo.Common/Utility.cs |  6 ++++++
 Kyoo/Tasks/Crawler.cs  | 20 +++++++++++++++-----
 Kyoo/appsettings.json  |  4 +++-
 3 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/Kyoo.Common/Utility.cs b/Kyoo.Common/Utility.cs
index 1183e5ea..8b068fff 100644
--- a/Kyoo.Common/Utility.cs
+++ b/Kyoo.Common/Utility.cs
@@ -201,5 +201,11 @@ namespace Kyoo
 			                                                && t.GetGenericArguments().Any()) ?? listType;
 			return type.GetGenericArguments().First();
 		}
+
+		public static IEnumerable<List<T>> BatchBy<T>(this List<T> list, int countPerList)
+		{
+			for (int i = 0; i < list.Count; i += countPerList)
+				yield return list.GetRange(i, Math.Min(list.Count - i, i + countPerList));
+		}
 	}
 }
\ No newline at end of file
diff --git a/Kyoo/Tasks/Crawler.cs b/Kyoo/Tasks/Crawler.cs
index 8615002d..8947e083 100644
--- a/Kyoo/Tasks/Crawler.cs
+++ b/Kyoo/Tasks/Crawler.cs
@@ -27,6 +27,8 @@ namespace Kyoo.Controllers
 		private IProviderManager _metadataProvider;
 		private ITranscoder _transcoder;
 		private IConfiguration _config;
+
+		private int _parallelRegisters;
 		
 		public async Task<IEnumerable<string>> GetPossibleParameters()
 		{
@@ -50,6 +52,10 @@ namespace Kyoo.Controllers
 			_metadataProvider = serviceProvider.GetService<IProviderManager>();
 			_transcoder = serviceProvider.GetService<ITranscoder>();
 			_config = serviceProvider.GetService<IConfiguration>();
+			_parallelRegisters = _config.GetValue<int>("parallelRegisters");
+
+			if (_parallelRegisters <= 0)
+				_parallelRegisters = 10;
 
 			using IServiceScope serviceScope = _serviceProvider.CreateScope();
 			await using ILibraryManager libraryManager = serviceScope.ServiceProvider.GetService<ILibraryManager>();
@@ -111,15 +117,19 @@ namespace Kyoo.Controllers
 					.GroupBy(Path.GetDirectoryName)
 					.ToList();
 				
-				await Task.WhenAll(shows
+				List<Task> tasks = shows
 					.Select(x => x.First())
 					.Select(x => RegisterFile(x, x.Substring(path.Length), library, cancellationToken))
-					.ToArray());
+					.ToList();
+				foreach (List<Task> showTasks in tasks.BatchBy(_parallelRegisters))
+					await Task.WhenAll(showTasks);
 				
-				await Task.WhenAll(shows
+				tasks = shows
 					.SelectMany(x => x.Skip(1))
 					.Select(x => RegisterFile(x, x.Substring(path.Length), library, cancellationToken))
-					.ToArray());
+					.ToList();
+				foreach (List<Task> episodeTasks in tasks.BatchBy(_parallelRegisters * 3))
+					await Task.WhenAll(episodeTasks);
 			}).ToArray());
 		}
 
@@ -169,7 +179,7 @@ namespace Kyoo.Controllers
 			{
 				await Console.Error.WriteLineAsync($"Unknown exception thrown while registering episode at {path}." +
 				                                   $"\nException: {ex.Message}" +
-				                                   $"\nAt {ex.StackTrace}");
+				                                   $"\n{ex.StackTrace}");
 			}
 		}
 
diff --git a/Kyoo/appsettings.json b/Kyoo/appsettings.json
index 67593767..70b961b1 100644
--- a/Kyoo/appsettings.json
+++ b/Kyoo/appsettings.json
@@ -13,9 +13,11 @@
     }
   },
   "AllowedHosts": "*",
+  
   "ConnectionStrings": {
-    "Database": "Server=127.0.0.1; Port=5432; Database=kyooDB; User Id=kyoo; Password=kyooPassword; Pooling=true; MaxPoolSize=80; Timeout=0;"
+    "Database": "Server=127.0.0.1; Port=5432; Database=kyooDB; User Id=kyoo; Password=kyooPassword; Pooling=true; MaxPoolSize=80; Timeout=30;"
   },
+  "parallelRegisters": "30",
   
   "scheduledTasks": {
     "scan": "24:00:00"