Cutlery/Kyoo
1
0
Fork 0
mirror of https://github.com/zoriya/Kyoo.git synced 2025-07-09 03:04:20 -04:00
Code Issues Packages Projects Releases Wiki Activity

Spliting tasks into pools

Browse Source
This commit is contained in:
Zoe Roux 2020-06-16 18:05:07 +02:00
parent aef5214bef
commit e621327042
3 changed files with 24 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Kyoo.Common/Utility.cs
View File

@ -201,5 +201,11 @@ namespace Kyoo
&& t.GetGenericArguments().Any()) ?? listType;
return type.GetGenericArguments().First();
}
public static IEnumerable<List<T>> BatchBy<T>(this List<T> list, int countPerList)
{
for (int i = 0; i < list.Count; i += countPerList)
yield return list.GetRange(i, Math.Min(list.Count - i, i + countPerList));
}
}
}

20
Kyoo/Tasks/Crawler.cs
View File

@ -28,6 +28,8 @@ namespace Kyoo.Controllers
private ITranscoder _transcoder;
private IConfiguration _config;
private int _parallelRegisters;
public async Task<IEnumerable<string>> GetPossibleParameters()
{
using IServiceScope serviceScope = _serviceProvider.CreateScope();
@ -50,6 +52,10 @@ namespace Kyoo.Controllers
_metadataProvider = serviceProvider.GetService<IProviderManager>();
_transcoder = serviceProvider.GetService<ITranscoder>();
_config = serviceProvider.GetService<IConfiguration>();
_parallelRegisters = _config.GetValue<int>("parallelRegisters");
if (_parallelRegisters <= 0)
_parallelRegisters = 10;
using IServiceScope serviceScope = _serviceProvider.CreateScope();
await using ILibraryManager libraryManager = serviceScope.ServiceProvider.GetService<ILibraryManager>();
@ -111,15 +117,19 @@ namespace Kyoo.Controllers
.GroupBy(Path.GetDirectoryName)
.ToList();
await Task.WhenAll(shows
List<Task> tasks = shows
.Select(x => x.First())
.Select(x => RegisterFile(x, x.Substring(path.Length), library, cancellationToken))
.ToArray());
.ToList();
foreach (List<Task> showTasks in tasks.BatchBy(_parallelRegisters))
await Task.WhenAll(showTasks);
await Task.WhenAll(shows
tasks = shows
.SelectMany(x => x.Skip(1))
.Select(x => RegisterFile(x, x.Substring(path.Length), library, cancellationToken))
.ToArray());
.ToList();
foreach (List<Task> episodeTasks in tasks.BatchBy(_parallelRegisters * 3))
await Task.WhenAll(episodeTasks);
}).ToArray());
}
@ -169,7 +179,7 @@ namespace Kyoo.Controllers
{
await Console.Error.WriteLineAsync($"Unknown exception thrown while registering episode at {path}." +
$"\nException: {ex.Message}" +
$"\nAt {ex.StackTrace}");
$"\n{ex.StackTrace}");
}
}

4
Kyoo/appsettings.json
View File

@ -13,9 +13,11 @@
}
},
"AllowedHosts": "*",
"ConnectionStrings": {
"Database": "Server=127.0.0.1; Port=5432; Database=kyooDB; User Id=kyoo; Password=kyooPassword; Pooling=true; MaxPoolSize=80; Timeout=0;"
"Database": "Server=127.0.0.1; Port=5432; Database=kyooDB; User Id=kyoo; Password=kyooPassword; Pooling=true; MaxPoolSize=80; Timeout=30;"
},
"parallelRegisters": "30",
"scheduledTasks": {
"scan": "24:00:00"