[v4] Added support for storing images in S3-compatible object storage

Signed-off-by: Fred Heinecke <fred.heinecke@yahoo.com>

back/src/Kyoo.Abstractions/Controllers/IThumbnailsManager.cs

@@ -30,7 +30,9 @@ public interface IThumbnailsManager
 
 	Task DownloadImage(Image? image, string what);
 
-	string GetImagePath(Guid imageId, ImageQuality quality);
+	Task<bool> IsImageSaved(Guid imageId, ImageQuality quality);
+
+	Task<Stream> GetImage(Guid imageId, ImageQuality quality);
 
 	Task DeleteImages<T>(T item)
 		where T : IThumbnails;

back/src/Kyoo.Core/Controllers/MiscRepository.cs

@@ -69,9 +69,12 @@ public class MiscRepository(
 	public async Task DownloadMissingImages()
 	{
 		ICollection<Image> images = await _GetAllImages();
-		IEnumerable<Task> tasks = images
+		var tasks = images
-			.Where(x => !File.Exists(thumbnails.GetImagePath(x.Id, ImageQuality.Low)))
+			.ToAsyncEnumerable()
-			.Select(x => thumbnails.DownloadImage(x, x.Id.ToString()));
+			.WhereAwait(async x => !await thumbnails.IsImageSaved(x.Id, ImageQuality.Low))
+			.Select(x => thumbnails.DownloadImage(x, x.Id.ToString()))
+			.ToEnumerable();
+
 		// Chunk tasks to prevent http timouts
 		foreach (IEnumerable<Task> batch in tasks.Chunk(30))
 			await Task.WhenAll(batch);

back/src/Kyoo.Core/Controllers/ThumbnailsManager.cs

@@ -17,7 +17,6 @@
 // along with Kyoo. If not, see <https://www.gnu.org/licenses/>.
 
 using System;
-using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Net.Http;
@@ -28,6 +27,7 @@ using Blurhash.SkiaSharp;
 using Kyoo.Abstractions.Controllers;
 using Kyoo.Abstractions.Models;
 using Kyoo.Abstractions.Models.Exceptions;
+using Kyoo.Core.Storage;
 using Microsoft.Extensions.Logging;
 using SkiaSharp;
 using SKSvg = SkiaSharp.Extended.Svg.SKSvg;
@@ -40,15 +40,15 @@ namespace Kyoo.Core.Controllers;
 public class ThumbnailsManager(
 	IHttpClientFactory clientFactory,
 	ILogger<ThumbnailsManager> logger,
+	IStorage storage,
 	Lazy<IRepository<User>> users
 ) : IThumbnailsManager
 {
-	private static async Task _WriteTo(SKBitmap bitmap, string path, int quality)
+	private async Task _SaveImage(SKBitmap bitmap, string path, int quality)
 	{
 		SKData data = bitmap.Encode(SKEncodedImageFormat.Webp, quality);
 		await using Stream reader = data.AsStream();
-		await using Stream file = File.Create(path);
+		await storage.Write(reader, path);
-		await reader.CopyToAsync(file);
 	}
 
 	private SKBitmap _SKBitmapFrom(Stream reader, bool isSvg)
@@ -100,19 +100,19 @@ public class ThumbnailsManager(
 				new SKSizeI(original.Width, original.Height),
 				SKFilterQuality.High
 			);
-			await _WriteTo(original, GetImagePath(image.Id, ImageQuality.High), 90);
+			await _SaveImage(original, _GetImagePath(image.Id, ImageQuality.High), 90);
 
 			using SKBitmap medium = high.Resize(
 				new SKSizeI((int)(high.Width / 1.5), (int)(high.Height / 1.5)),
 				SKFilterQuality.Medium
 			);
-			await _WriteTo(medium, GetImagePath(image.Id, ImageQuality.Medium), 75);
+			await _SaveImage(medium, _GetImagePath(image.Id, ImageQuality.Medium), 75);
 
 			using SKBitmap low = medium.Resize(
 				new SKSizeI(original.Width / 2, original.Height / 2),
 				SKFilterQuality.Low
 			);
-			await _WriteTo(low, GetImagePath(image.Id, ImageQuality.Low), 50);
+			await _SaveImage(low, _GetImagePath(image.Id, ImageQuality.Low), 50);
 
 			image.Blurhash = Blurhasher.Encode(low, 4, 3);
 		}
@@ -133,8 +133,20 @@ public class ThumbnailsManager(
 		await DownloadImage(item.Logo, $"The logo of {name}");
 	}
 
+	public async Task<bool> IsImageSaved(Guid imageId, ImageQuality quality) =>
+		await storage.DoesExist(_GetImagePath(imageId, quality));
+
+	public async Task<Stream> GetImage(Guid imageId, ImageQuality quality)
+	{
+		string path = _GetImagePath(imageId, quality);
+		if (await storage.DoesExist(path))
+			return await storage.Read(path);
+
+		throw new ItemNotFoundException();
+	}
+
 	/// <inheritdoc />
-	public string GetImagePath(Guid imageId, ImageQuality quality)
+	private string _GetImagePath(Guid imageId, ImageQuality quality)
 	{
 		return $"/metadata/{imageId}.{quality.ToString().ToLowerInvariant()}.webp";
 	}
@@ -143,7 +155,7 @@ public class ThumbnailsManager(
 	public Task DeleteImages<T>(T item)
 		where T : IThumbnails
 	{
-		IEnumerable<string> images = new[] { item.Poster?.Id, item.Thumbnail?.Id, item.Logo?.Id }
+		var imageDeletionTasks = new[] { item.Poster?.Id, item.Thumbnail?.Id, item.Logo?.Id }
 			.Where(x => x is not null)
 			.SelectMany(x => $"/metadata/{x}")
 			.SelectMany(x =>
@@ -153,21 +165,17 @@ public class ThumbnailsManager(
 					ImageQuality.Medium.ToString().ToLowerInvariant(),
 					ImageQuality.Low.ToString().ToLowerInvariant(),
 				}.Select(quality => $"{x}.{quality}.webp")
-			);
+			)
+			.Select(storage.Delete);
 
-		foreach (string image in images)
+		return Task.WhenAll(imageDeletionTasks);
-			File.Delete(image);
-		return Task.CompletedTask;
 	}
 
 	public async Task<Stream> GetUserImage(Guid userId)
 	{
-		try
+		var filePath = $"/metadata/user/{userId}.webp";
-		{
+		if (await storage.DoesExist(filePath))
-			return File.Open($"/metadata/user/{userId}.webp", FileMode.Open);
+			return await storage.Read(filePath);
-		}
-		catch (FileNotFoundException) { }
-		catch (DirectoryNotFoundException) { }
 
 		User user = await users.Value.Get(userId);
 		if (user.Email == null)
@@ -193,21 +201,18 @@ public class ThumbnailsManager(
 
 	public async Task SetUserImage(Guid userId, Stream? image)
 	{
-		if (image == null)
+		var filePath = $"/metadata/user/{userId}.webp";
+		if (image == null && await storage.DoesExist(filePath))
 		{
-			try
+			await storage.Delete(filePath);
-			{
-				File.Delete($"/metadata/user/{userId}.webp");
-			}
-			catch { }
 			return;
 		}
+
 		using SKCodec codec = SKCodec.Create(image);
 		SKImageInfo info = codec.Info;
 		info.ColorType = SKColorType.Rgba8888;
 		using SKBitmap original = SKBitmap.Decode(codec, info);
 		using SKBitmap ret = original.Resize(new SKSizeI(250, 250), SKFilterQuality.High);
-		Directory.CreateDirectory("/metadata/user");
+		await _SaveImage(ret, filePath, 75);
-		await _WriteTo(ret, $"/metadata/user/{userId}.webp", 75);
 	}
 }

back/src/Kyoo.Core/CoreModule.cs

@@ -18,11 +18,14 @@
 
 using System;
 using System.Linq;
+using Amazon.S3;
 using Kyoo.Abstractions.Controllers;
 using Kyoo.Abstractions.Models;
 using Kyoo.Core.Controllers;
+using Kyoo.Core.Storage;
 using Kyoo.Postgresql;
 using Microsoft.AspNetCore.Builder;
+using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 
 namespace Kyoo.Core;
@@ -46,6 +49,8 @@ public static class CoreModule
 
 	public static void ConfigureKyoo(this WebApplicationBuilder builder)
 	{
+		builder._AddStorage();
+
 		builder.Services.AddScoped<IThumbnailsManager, ThumbnailsManager>();
 		builder.Services.AddScoped<ILibraryManager, LibraryManager>();
 
@@ -67,4 +72,21 @@ public static class CoreModule
 		builder.Services.AddScoped<SqlVariableContext>();
 		builder.Services.AddScoped<MiscRepository>();
 	}
+
+	private static void _AddStorage(this WebApplicationBuilder builder)
+	{
+		var shouldUseS3 = !string.IsNullOrEmpty(
+			builder.Configuration.GetValue<string>(S3Storage.S3BucketEnvironmentVariable)
+		);
+
+		if (!shouldUseS3)
+		{
+			builder.Services.AddScoped<IStorage, FileStorage>();
+			return;
+		}
+
+		// Configuration (credentials, endpoint, etc.) are done via standard AWS env vars
+		builder.Services.AddScoped<IAmazonS3, AmazonS3Client>();
+		builder.Services.AddScoped<IStorage, S3Storage>();
+	}
 }

back/src/Kyoo.Core/Kyoo.Core.csproj

@@ -11,6 +11,7 @@
 
 	<ItemGroup>
 		<PackageReference Include="AspNetCore.Proxy" Version="4.5.0" />
+		<PackageReference Include="AWSSDK.S3" Version="3.7.416.15" />
 		<PackageReference Include="Blurhash.SkiaSharp" Version="2.0.0" />
 		<PackageReference Include="Dapper" Version="2.1.44" />
 		<PackageReference Include="InterpolatedSql.Dapper" Version="2.3.0" />
@@ -28,6 +29,7 @@
 			<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
 		</PackageReference>
 		<PackageReference Include="SkiaSharp.Svg" Version="1.60.0" />
+		<PackageReference Include="System.Linq.Async" Version="6.0.1" />
 	</ItemGroup>
 
 	<ItemGroup>

back/src/Kyoo.Core/Storage/FileStorage.cs

@@ -0,0 +1,51 @@
+// Kyoo - A portable and vast media library solution.
+// Copyright (c) Kyoo.
+//
+// See AUTHORS.md and LICENSE file in the project root for full license information.
+//
+// Kyoo is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// any later version.
+//
+// Kyoo is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Kyoo. If not, see <https://www.gnu.org/licenses/>.
+
+using System;
+using System.IO;
+using System.Threading.Tasks;
+
+namespace Kyoo.Core.Storage;
+
+/// <summary>
+/// File-backed storage.
+/// </summary>
+public class FileStorage : IStorage
+{
+	public Task<bool> DoesExist(string path) => Task.FromResult(File.Exists(path));
+
+	public async Task<Stream> Read(string path) =>
+		await Task.FromResult(File.Open(path, FileMode.Open, FileAccess.Read));
+
+	public async Task Write(Stream reader, string path)
+	{
+		Directory.CreateDirectory(
+			Path.GetDirectoryName(path) ?? throw new InvalidOperationException()
+		);
+		await using Stream file = File.Create(path);
+		await reader.CopyToAsync(file);
+	}
+
+	public Task Delete(string path)
+	{
+		if (File.Exists(path))
+			File.Delete(path);
+
+		return Task.CompletedTask;
+	}
+}

back/src/Kyoo.Core/Storage/IStorage.cs

@@ -0,0 +1,33 @@
+// Kyoo - A portable and vast media library solution.
+// Copyright (c) Kyoo.
+//
+// See AUTHORS.md and LICENSE file in the project root for full license information.
+//
+// Kyoo is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// any later version.
+//
+// Kyoo is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Kyoo. If not, see <https://www.gnu.org/licenses/>.
+
+using System.IO;
+using System.Threading.Tasks;
+
+namespace Kyoo.Core.Storage;
+
+/// <summary>
+/// Interface for storage operations.
+/// </summary>
+public interface IStorage
+{
+	Task<bool> DoesExist(string path);
+	Task<Stream> Read(string path);
+	Task Write(Stream stream, string path);
+	Task Delete(string path);
+}

back/src/Kyoo.Core/Storage/S3Storage.cs

@@ -0,0 +1,78 @@
+// Kyoo - A portable and vast media library solution.
+// Copyright (c) Kyoo.
+//
+// See AUTHORS.md and LICENSE file in the project root for full license information.
+//
+// Kyoo is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// any later version.
+//
+// Kyoo is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Kyoo. If not, see <https://www.gnu.org/licenses/>.
+
+using System;
+using System.IO;
+using System.Threading.Tasks;
+using Amazon.S3;
+using Microsoft.Extensions.Configuration;
+
+namespace Kyoo.Core.Storage;
+
+/// <summary>
+/// S3-backed storage.
+/// </summary>
+public class S3Storage(IAmazonS3 s3Client, IConfiguration configuration) : IStorage
+{
+	public const string S3BucketEnvironmentVariable = "S3_BUCKET_NAME";
+
+	public Task<bool> DoesExist(string path)
+	{
+		return s3Client
+			.GetObjectMetadataAsync(_GetBucketName(), path)
+			.ContinueWith(t =>
+			{
+				if (t.IsFaulted)
+					return false;
+
+				return t.Result.HttpStatusCode == System.Net.HttpStatusCode.OK;
+			});
+	}
+
+	public async Task<Stream> Read(string path)
+	{
+		var response = await s3Client.GetObjectAsync(_GetBucketName(), path);
+		return response.ResponseStream;
+	}
+
+	public Task Write(Stream reader, string path)
+	{
+		return s3Client.PutObjectAsync(
+			new Amazon.S3.Model.PutObjectRequest
+			{
+				BucketName = _GetBucketName(),
+				Key = path,
+				InputStream = reader
+			}
+		);
+	}
+
+	public Task Delete(string path)
+	{
+		return s3Client.DeleteObjectAsync(_GetBucketName(), path);
+	}
+
+	private string _GetBucketName()
+	{
+		var bucketName = configuration.GetValue<string>(S3BucketEnvironmentVariable);
+		if (string.IsNullOrEmpty(bucketName))
+			throw new InvalidOperationException("S3 bucket name is not configured.");
+
+		return bucketName;
+	}
+}

back/src/Kyoo.Core/Views/Content/ThumbnailsApi.cs

@@ -18,6 +18,7 @@
 
 using System;
 using System.IO;
+using System.Threading.Tasks;
 using Kyoo.Abstractions.Controllers;
 using Kyoo.Abstractions.Models;
 using Kyoo.Abstractions.Models.Attributes;
@@ -54,14 +55,14 @@ public class ThumbnailsApi(IThumbnailsManager thumbs) : BaseApi
 	[HttpGet("{id:guid}")]
 	[ProducesResponseType(StatusCodes.Status200OK)]
 	[ProducesResponseType(StatusCodes.Status404NotFound)]
-	public IActionResult GetPoster(Guid id, [FromQuery] ImageQuality? quality)
+	public async Task<IActionResult> GetPoster(Guid id, [FromQuery] ImageQuality? quality)
 	{
-		string path = thumbs.GetImagePath(id, quality ?? ImageQuality.High);
+		quality ??= ImageQuality.High;
-		if (!System.IO.File.Exists(path))
+		if (await thumbs.IsImageSaved(id, quality.Value))
 			return NotFound();
 
 		// Allow clients to cache the image for 6 month.
 		Response.Headers.CacheControl = $"public, max-age={60 * 60 * 24 * 31 * 6}";
-		return PhysicalFile(Path.GetFullPath(path), "image/webp", true);
+		return File(await thumbs.GetImage(id, quality.Value), "image/webp", true);
 	}
 }