Cutlery/Kyoo
1
0
Fork 0
mirror of https://github.com/zoriya/Kyoo.git synced 2025-07-09 03:04:20 -04:00
Code Issues Packages Projects Releases Wiki Activity

Spliting tasks into pools

Browse Source
This commit is contained in:
Zoe Roux 2020-06-16 18:05:07 +02:00
parent aef5214bef
commit e621327042
3 changed files with 24 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Kyoo.Common/Utility.cs
View File

@ -201,5 +201,11 @@ namespace Kyoo
&& t.GetGenericArguments().Any()) ?? listType; && t.GetGenericArguments().Any()) ?? listType;
return type.GetGenericArguments().First(); return type.GetGenericArguments().First();
} }
public static IEnumerable<List<T>> BatchBy<T>(this List<T> list, int countPerList)
{
for (int i = 0; i < list.Count; i += countPerList)
yield return list.GetRange(i, Math.Min(list.Count - i, i + countPerList));
}
} }
} }

20
Kyoo/Tasks/Crawler.cs
View File

@ -28,6 +28,8 @@ namespace Kyoo.Controllers
private ITranscoder _transcoder; private ITranscoder _transcoder;
private IConfiguration _config; private IConfiguration _config;
private int _parallelRegisters;
public async Task<IEnumerable<string>> GetPossibleParameters() public async Task<IEnumerable<string>> GetPossibleParameters()
{ {
using IServiceScope serviceScope = _serviceProvider.CreateScope(); using IServiceScope serviceScope = _serviceProvider.CreateScope();
@ -50,6 +52,10 @@ namespace Kyoo.Controllers
_metadataProvider = serviceProvider.GetService<IProviderManager>(); _metadataProvider = serviceProvider.GetService<IProviderManager>();
_transcoder = serviceProvider.GetService<ITranscoder>(); _transcoder = serviceProvider.GetService<ITranscoder>();
_config = serviceProvider.GetService<IConfiguration>(); _config = serviceProvider.GetService<IConfiguration>();
_parallelRegisters = _config.GetValue<int>("parallelRegisters");
if (_parallelRegisters <= 0)
_parallelRegisters = 10;
using IServiceScope serviceScope = _serviceProvider.CreateScope(); using IServiceScope serviceScope = _serviceProvider.CreateScope();
await using ILibraryManager libraryManager = serviceScope.ServiceProvider.GetService<ILibraryManager>(); await using ILibraryManager libraryManager = serviceScope.ServiceProvider.GetService<ILibraryManager>();
@ -111,15 +117,19 @@ namespace Kyoo.Controllers
.GroupBy(Path.GetDirectoryName) .GroupBy(Path.GetDirectoryName)
.ToList(); .ToList();
await Task.WhenAll(shows List<Task> tasks = shows
.Select(x => x.First()) .Select(x => x.First())
.Select(x => RegisterFile(x, x.Substring(path.Length), library, cancellationToken)) .Select(x => RegisterFile(x, x.Substring(path.Length), library, cancellationToken))
.ToArray()); .ToList();
foreach (List<Task> showTasks in tasks.BatchBy(_parallelRegisters))
await Task.WhenAll(showTasks);
await Task.WhenAll(shows tasks = shows
.SelectMany(x => x.Skip(1)) .SelectMany(x => x.Skip(1))
.Select(x => RegisterFile(x, x.Substring(path.Length), library, cancellationToken)) .Select(x => RegisterFile(x, x.Substring(path.Length), library, cancellationToken))
.ToArray()); .ToList();
foreach (List<Task> episodeTasks in tasks.BatchBy(_parallelRegisters * 3))
await Task.WhenAll(episodeTasks);
}).ToArray()); }).ToArray());
} }
@ -169,7 +179,7 @@ namespace Kyoo.Controllers
{ {
await Console.Error.WriteLineAsync($"Unknown exception thrown while registering episode at {path}." + await Console.Error.WriteLineAsync($"Unknown exception thrown while registering episode at {path}." +
$"\nException: {ex.Message}" + $"\nException: {ex.Message}" +
$"\nAt {ex.StackTrace}"); $"\n{ex.StackTrace}");
} }
} }

4
Kyoo/appsettings.json
View File

@ -13,9 +13,11 @@
} }
}, },
"AllowedHosts": "*", "AllowedHosts": "*",
"ConnectionStrings": { "ConnectionStrings": {
"Database": "Server=127.0.0.1; Port=5432; Database=kyooDB; User Id=kyoo; Password=kyooPassword; Pooling=true; MaxPoolSize=80; Timeout=0;" "Database": "Server=127.0.0.1; Port=5432; Database=kyooDB; User Id=kyoo; Password=kyooPassword; Pooling=true; MaxPoolSize=80; Timeout=30;"
}, },
"parallelRegisters": "30",
"scheduledTasks": { "scheduledTasks": {
"scan": "24:00:00" "scan": "24:00:00"