Cutlery/Kyoo
1
0
Fork 0
mirror of https://github.com/zoriya/Kyoo.git synced 2025-06-10 00:54:20 -04:00
Code Issues Packages Projects Releases Wiki Activity

The crawler is now fully multi-threaded

Browse Source
This commit is contained in:
Zoe Roux 2020-06-18 14:54:54 +02:00
parent 8e37505b8f
commit ef84a969b7
2 changed files with 24 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
Kyoo.Common/Utility.cs
View File

@ -207,5 +207,22 @@ namespace Kyoo
for (int i = 0; i < list.Count; i += countPerList) for (int i = 0; i < list.Count; i += countPerList)
yield return list.GetRange(i, Math.Min(list.Count - i, countPerList)); yield return list.GetRange(i, Math.Min(list.Count - i, countPerList));
} }
public static IEnumerable<T[]> BatchBy<T>(this IEnumerable<T> list, int countPerList)
{
T[] ret = new T[countPerList];
int i = 0;
using IEnumerator<T> enumerator = list.GetEnumerator();
while (enumerator.MoveNext())
{
ret[i] = enumerator.Current;
i++;
if (i < countPerList)
continue;
i = 0;
yield return ret;
}
}
} }
} }

13
Kyoo/Tasks/Crawler.cs
View File

@ -114,19 +114,16 @@ namespace Kyoo.Controllers
// Todo batch wth higher numbers per list once multi-services has been implemented. // Todo batch wth higher numbers per list once multi-services has been implemented.
List<Task> tasks = shows IEnumerable<Task> tasks = shows
.Select(x => x.First()) .Select(x => x.First())
.Select(x => RegisterFile(x, x.Substring(path.Length), library, cancellationToken)) .Select(x => RegisterFile(x, x.Substring(path.Length), library, cancellationToken));
.ToList(); foreach (Task[] showTasks in tasks.BatchBy(30))
// TODO EXECUTION OF THE TASKS ARE CALCULATED DURING THE .ToList() CALL, NO BACHING/ASYNC IS DONE.
foreach (List<Task> showTasks in tasks.BatchBy(1))
await Task.WhenAll(showTasks); await Task.WhenAll(showTasks);
tasks = shows tasks = shows
.SelectMany(x => x.Skip(1)) .SelectMany(x => x.Skip(1))
.Select(x => RegisterFile(x, x.Substring(path.Length), library, cancellationToken)) .Select(x => RegisterFile(x, x.Substring(path.Length), library, cancellationToken));
.ToList(); foreach (Task[] episodeTasks in tasks.BatchBy(50))
foreach (List<Task> episodeTasks in tasks.BatchBy(1))
await Task.WhenAll(episodeTasks); await Task.WhenAll(episodeTasks);
} }
} }