From 9e38c3f3339aa427a6a465f2d6348acac2dacffa Mon Sep 17 00:00:00 2001
From: Zoe Roux <zoe.roux@zoriya.dev>
Date: Sun, 21 Apr 2024 21:23:24 +0200
Subject: [PATCH] Chunk images redownload to prevent http timeouts

---
 .../Kyoo.Core/Controllers/MiscRepository.cs    | 18 +++++++++++++-----
 back/src/Kyoo.Core/Program.cs                  |  5 +++--
 2 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/back/src/Kyoo.Core/Controllers/MiscRepository.cs b/back/src/Kyoo.Core/Controllers/MiscRepository.cs
index 820b3571..f3013a5e 100644
--- a/back/src/Kyoo.Core/Controllers/MiscRepository.cs
+++ b/back/src/Kyoo.Core/Controllers/MiscRepository.cs
@@ -27,6 +27,7 @@ using Kyoo.Abstractions.Controllers;
 using Kyoo.Abstractions.Models;
 using Kyoo.Postgresql;
 using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.DependencyInjection;
 
 namespace Kyoo.Core.Controllers;
 
@@ -36,6 +37,12 @@ public class MiscRepository(
 	IThumbnailsManager thumbnails
 )
 {
+	public static async Task DownloadMissingImages(IServiceProvider services)
+	{
+		await using AsyncServiceScope scope = services.CreateAsyncScope();
+		await scope.ServiceProvider.GetRequiredService<MiscRepository>().DownloadMissingImages();
+	}
+
 	private async Task<ICollection<Image>> _GetAllImages()
 	{
 		string GetSql(string type) =>
@@ -60,11 +67,12 @@ public class MiscRepository(
 	public async Task DownloadMissingImages()
 	{
 		ICollection<Image> images = await _GetAllImages();
-		await Task.WhenAll(
-			images
-				.Where(x => !File.Exists(thumbnails.GetImagePath(x.Id, ImageQuality.Low)))
-				.Select(x => thumbnails.DownloadImage(x, x.Id.ToString()))
-		);
+		IEnumerable<Task> tasks = images
+			.Where(x => !File.Exists(thumbnails.GetImagePath(x.Id, ImageQuality.Low)))
+			.Select(x => thumbnails.DownloadImage(x, x.Id.ToString()));
+		// Chunk tasks to prevent http timouts
+		foreach (IEnumerable<Task> batch in tasks.Chunk(30))
+			await Task.WhenAll(batch);
 	}
 
 	public async Task<ICollection<string>> GetRegisteredPaths()
diff --git a/back/src/Kyoo.Core/Program.cs b/back/src/Kyoo.Core/Program.cs
index dd4455a7..b750bb60 100644
--- a/back/src/Kyoo.Core/Program.cs
+++ b/back/src/Kyoo.Core/Program.cs
@@ -95,7 +95,8 @@ app.Services.GetRequiredService<RabbitProducer>();
 await using (AsyncServiceScope scope = app.Services.CreateAsyncScope())
 {
 	await MeilisearchModule.Initialize(scope.ServiceProvider);
-	await scope.ServiceProvider.GetRequiredService<MiscRepository>().DownloadMissingImages();
 }
+// The methods takes care of creating a scope and will download images on the background.
+_ = MiscRepository.DownloadMissingImages(app.Services);
 
-app.Run(Environment.GetEnvironmentVariable("KYOO_BIND_URL") ?? "http://*:5000");
+await app.RunAsync(Environment.GetEnvironmentVariable("KYOO_BIND_URL") ?? "http://*:5000");