diff --git a/back/src/Kyoo.Abstractions/Controllers/IThumbnailsManager.cs b/back/src/Kyoo.Abstractions/Controllers/IThumbnailsManager.cs index 21e68f70..a1a37aa2 100644 --- a/back/src/Kyoo.Abstractions/Controllers/IThumbnailsManager.cs +++ b/back/src/Kyoo.Abstractions/Controllers/IThumbnailsManager.cs @@ -30,7 +30,9 @@ public interface IThumbnailsManager Task DownloadImage(Image? image, string what); - string GetImagePath(Guid imageId, ImageQuality quality); + Task IsImageSaved(Guid imageId, ImageQuality quality); + + Task GetImage(Guid imageId, ImageQuality quality); Task DeleteImages(T item) where T : IThumbnails; diff --git a/back/src/Kyoo.Core/Controllers/MiscRepository.cs b/back/src/Kyoo.Core/Controllers/MiscRepository.cs index 48916ad8..28295623 100644 --- a/back/src/Kyoo.Core/Controllers/MiscRepository.cs +++ b/back/src/Kyoo.Core/Controllers/MiscRepository.cs @@ -69,9 +69,12 @@ public class MiscRepository( public async Task DownloadMissingImages() { ICollection images = await _GetAllImages(); - IEnumerable tasks = images - .Where(x => !File.Exists(thumbnails.GetImagePath(x.Id, ImageQuality.Low))) - .Select(x => thumbnails.DownloadImage(x, x.Id.ToString())); + var tasks = images + .ToAsyncEnumerable() + .WhereAwait(async x => !await thumbnails.IsImageSaved(x.Id, ImageQuality.Low)) + .Select(x => thumbnails.DownloadImage(x, x.Id.ToString())) + .ToEnumerable(); + // Chunk tasks to prevent http timouts foreach (IEnumerable batch in tasks.Chunk(30)) await Task.WhenAll(batch); diff --git a/back/src/Kyoo.Core/Controllers/ThumbnailsManager.cs b/back/src/Kyoo.Core/Controllers/ThumbnailsManager.cs index 8dfc7a84..5c9a784f 100644 --- a/back/src/Kyoo.Core/Controllers/ThumbnailsManager.cs +++ b/back/src/Kyoo.Core/Controllers/ThumbnailsManager.cs @@ -17,7 +17,6 @@ // along with Kyoo. If not, see . using System; -using System.Collections.Generic; using System.IO; using System.Linq; using System.Net.Http; @@ -28,6 +27,7 @@ using Blurhash.SkiaSharp; using Kyoo.Abstractions.Controllers; using Kyoo.Abstractions.Models; using Kyoo.Abstractions.Models.Exceptions; +using Kyoo.Core.Storage; using Microsoft.Extensions.Logging; using SkiaSharp; using SKSvg = SkiaSharp.Extended.Svg.SKSvg; @@ -40,15 +40,15 @@ namespace Kyoo.Core.Controllers; public class ThumbnailsManager( IHttpClientFactory clientFactory, ILogger logger, + IStorage storage, Lazy> users ) : IThumbnailsManager { - private static async Task _WriteTo(SKBitmap bitmap, string path, int quality) + private async Task _SaveImage(SKBitmap bitmap, string path, int quality) { SKData data = bitmap.Encode(SKEncodedImageFormat.Webp, quality); await using Stream reader = data.AsStream(); - await using Stream file = File.Create(path); - await reader.CopyToAsync(file); + await storage.Write(reader, path); } private SKBitmap _SKBitmapFrom(Stream reader, bool isSvg) @@ -100,19 +100,19 @@ public class ThumbnailsManager( new SKSizeI(original.Width, original.Height), SKFilterQuality.High ); - await _WriteTo(original, GetImagePath(image.Id, ImageQuality.High), 90); + await _SaveImage(original, _GetImagePath(image.Id, ImageQuality.High), 90); using SKBitmap medium = high.Resize( new SKSizeI((int)(high.Width / 1.5), (int)(high.Height / 1.5)), SKFilterQuality.Medium ); - await _WriteTo(medium, GetImagePath(image.Id, ImageQuality.Medium), 75); + await _SaveImage(medium, _GetImagePath(image.Id, ImageQuality.Medium), 75); using SKBitmap low = medium.Resize( new SKSizeI(original.Width / 2, original.Height / 2), SKFilterQuality.Low ); - await _WriteTo(low, GetImagePath(image.Id, ImageQuality.Low), 50); + await _SaveImage(low, _GetImagePath(image.Id, ImageQuality.Low), 50); image.Blurhash = Blurhasher.Encode(low, 4, 3); } @@ -133,8 +133,20 @@ public class ThumbnailsManager( await DownloadImage(item.Logo, $"The logo of {name}"); } + public async Task IsImageSaved(Guid imageId, ImageQuality quality) => + await storage.DoesExist(_GetImagePath(imageId, quality)); + + public async Task GetImage(Guid imageId, ImageQuality quality) + { + string path = _GetImagePath(imageId, quality); + if (await storage.DoesExist(path)) + return await storage.Read(path); + + throw new ItemNotFoundException(); + } + /// - public string GetImagePath(Guid imageId, ImageQuality quality) + private string _GetImagePath(Guid imageId, ImageQuality quality) { return $"/metadata/{imageId}.{quality.ToString().ToLowerInvariant()}.webp"; } @@ -143,7 +155,7 @@ public class ThumbnailsManager( public Task DeleteImages(T item) where T : IThumbnails { - IEnumerable images = new[] { item.Poster?.Id, item.Thumbnail?.Id, item.Logo?.Id } + var imageDeletionTasks = new[] { item.Poster?.Id, item.Thumbnail?.Id, item.Logo?.Id } .Where(x => x is not null) .SelectMany(x => $"/metadata/{x}") .SelectMany(x => @@ -153,21 +165,17 @@ public class ThumbnailsManager( ImageQuality.Medium.ToString().ToLowerInvariant(), ImageQuality.Low.ToString().ToLowerInvariant(), }.Select(quality => $"{x}.{quality}.webp") - ); + ) + .Select(storage.Delete); - foreach (string image in images) - File.Delete(image); - return Task.CompletedTask; + return Task.WhenAll(imageDeletionTasks); } public async Task GetUserImage(Guid userId) { - try - { - return File.Open($"/metadata/user/{userId}.webp", FileMode.Open); - } - catch (FileNotFoundException) { } - catch (DirectoryNotFoundException) { } + var filePath = $"/metadata/user/{userId}.webp"; + if (await storage.DoesExist(filePath)) + return await storage.Read(filePath); User user = await users.Value.Get(userId); if (user.Email == null) @@ -193,21 +201,18 @@ public class ThumbnailsManager( public async Task SetUserImage(Guid userId, Stream? image) { - if (image == null) + var filePath = $"/metadata/user/{userId}.webp"; + if (image == null && await storage.DoesExist(filePath)) { - try - { - File.Delete($"/metadata/user/{userId}.webp"); - } - catch { } + await storage.Delete(filePath); return; } + using SKCodec codec = SKCodec.Create(image); SKImageInfo info = codec.Info; info.ColorType = SKColorType.Rgba8888; using SKBitmap original = SKBitmap.Decode(codec, info); using SKBitmap ret = original.Resize(new SKSizeI(250, 250), SKFilterQuality.High); - Directory.CreateDirectory("/metadata/user"); - await _WriteTo(ret, $"/metadata/user/{userId}.webp", 75); + await _SaveImage(ret, filePath, 75); } } diff --git a/back/src/Kyoo.Core/CoreModule.cs b/back/src/Kyoo.Core/CoreModule.cs index c895fc25..abf7e5df 100644 --- a/back/src/Kyoo.Core/CoreModule.cs +++ b/back/src/Kyoo.Core/CoreModule.cs @@ -18,11 +18,14 @@ using System; using System.Linq; +using Amazon.S3; using Kyoo.Abstractions.Controllers; using Kyoo.Abstractions.Models; using Kyoo.Core.Controllers; +using Kyoo.Core.Storage; using Kyoo.Postgresql; using Microsoft.AspNetCore.Builder; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; namespace Kyoo.Core; @@ -46,6 +49,8 @@ public static class CoreModule public static void ConfigureKyoo(this WebApplicationBuilder builder) { + builder._AddStorage(); + builder.Services.AddScoped(); builder.Services.AddScoped(); @@ -67,4 +72,21 @@ public static class CoreModule builder.Services.AddScoped(); builder.Services.AddScoped(); } + + private static void _AddStorage(this WebApplicationBuilder builder) + { + var shouldUseS3 = !string.IsNullOrEmpty( + builder.Configuration.GetValue(S3Storage.S3BucketEnvironmentVariable) + ); + + if (!shouldUseS3) + { + builder.Services.AddScoped(); + return; + } + + // Configuration (credentials, endpoint, etc.) are done via standard AWS env vars + builder.Services.AddScoped(); + builder.Services.AddScoped(); + } } diff --git a/back/src/Kyoo.Core/Kyoo.Core.csproj b/back/src/Kyoo.Core/Kyoo.Core.csproj index 4d47b7c2..74531deb 100644 --- a/back/src/Kyoo.Core/Kyoo.Core.csproj +++ b/back/src/Kyoo.Core/Kyoo.Core.csproj @@ -11,6 +11,7 @@ + @@ -28,6 +29,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/back/src/Kyoo.Core/Storage/FileStorage.cs b/back/src/Kyoo.Core/Storage/FileStorage.cs new file mode 100644 index 00000000..45687bd6 --- /dev/null +++ b/back/src/Kyoo.Core/Storage/FileStorage.cs @@ -0,0 +1,51 @@ +// Kyoo - A portable and vast media library solution. +// Copyright (c) Kyoo. +// +// See AUTHORS.md and LICENSE file in the project root for full license information. +// +// Kyoo is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// any later version. +// +// Kyoo is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with Kyoo. If not, see . + +using System; +using System.IO; +using System.Threading.Tasks; + +namespace Kyoo.Core.Storage; + +/// +/// File-backed storage. +/// +public class FileStorage : IStorage +{ + public Task DoesExist(string path) => Task.FromResult(File.Exists(path)); + + public async Task Read(string path) => + await Task.FromResult(File.Open(path, FileMode.Open, FileAccess.Read)); + + public async Task Write(Stream reader, string path) + { + Directory.CreateDirectory( + Path.GetDirectoryName(path) ?? throw new InvalidOperationException() + ); + await using Stream file = File.Create(path); + await reader.CopyToAsync(file); + } + + public Task Delete(string path) + { + if (File.Exists(path)) + File.Delete(path); + + return Task.CompletedTask; + } +} diff --git a/back/src/Kyoo.Core/Storage/IStorage.cs b/back/src/Kyoo.Core/Storage/IStorage.cs new file mode 100644 index 00000000..2253661c --- /dev/null +++ b/back/src/Kyoo.Core/Storage/IStorage.cs @@ -0,0 +1,33 @@ +// Kyoo - A portable and vast media library solution. +// Copyright (c) Kyoo. +// +// See AUTHORS.md and LICENSE file in the project root for full license information. +// +// Kyoo is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// any later version. +// +// Kyoo is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with Kyoo. If not, see . + +using System.IO; +using System.Threading.Tasks; + +namespace Kyoo.Core.Storage; + +/// +/// Interface for storage operations. +/// +public interface IStorage +{ + Task DoesExist(string path); + Task Read(string path); + Task Write(Stream stream, string path); + Task Delete(string path); +} diff --git a/back/src/Kyoo.Core/Storage/S3Storage.cs b/back/src/Kyoo.Core/Storage/S3Storage.cs new file mode 100644 index 00000000..5c78c69d --- /dev/null +++ b/back/src/Kyoo.Core/Storage/S3Storage.cs @@ -0,0 +1,78 @@ +// Kyoo - A portable and vast media library solution. +// Copyright (c) Kyoo. +// +// See AUTHORS.md and LICENSE file in the project root for full license information. +// +// Kyoo is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// any later version. +// +// Kyoo is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with Kyoo. If not, see . + +using System; +using System.IO; +using System.Threading.Tasks; +using Amazon.S3; +using Microsoft.Extensions.Configuration; + +namespace Kyoo.Core.Storage; + +/// +/// S3-backed storage. +/// +public class S3Storage(IAmazonS3 s3Client, IConfiguration configuration) : IStorage +{ + public const string S3BucketEnvironmentVariable = "S3_BUCKET_NAME"; + + public Task DoesExist(string path) + { + return s3Client + .GetObjectMetadataAsync(_GetBucketName(), path) + .ContinueWith(t => + { + if (t.IsFaulted) + return false; + + return t.Result.HttpStatusCode == System.Net.HttpStatusCode.OK; + }); + } + + public async Task Read(string path) + { + var response = await s3Client.GetObjectAsync(_GetBucketName(), path); + return response.ResponseStream; + } + + public Task Write(Stream reader, string path) + { + return s3Client.PutObjectAsync( + new Amazon.S3.Model.PutObjectRequest + { + BucketName = _GetBucketName(), + Key = path, + InputStream = reader + } + ); + } + + public Task Delete(string path) + { + return s3Client.DeleteObjectAsync(_GetBucketName(), path); + } + + private string _GetBucketName() + { + var bucketName = configuration.GetValue(S3BucketEnvironmentVariable); + if (string.IsNullOrEmpty(bucketName)) + throw new InvalidOperationException("S3 bucket name is not configured."); + + return bucketName; + } +} diff --git a/back/src/Kyoo.Core/Views/Content/ThumbnailsApi.cs b/back/src/Kyoo.Core/Views/Content/ThumbnailsApi.cs index 908a88d7..529a9736 100644 --- a/back/src/Kyoo.Core/Views/Content/ThumbnailsApi.cs +++ b/back/src/Kyoo.Core/Views/Content/ThumbnailsApi.cs @@ -18,6 +18,7 @@ using System; using System.IO; +using System.Threading.Tasks; using Kyoo.Abstractions.Controllers; using Kyoo.Abstractions.Models; using Kyoo.Abstractions.Models.Attributes; @@ -54,14 +55,14 @@ public class ThumbnailsApi(IThumbnailsManager thumbs) : BaseApi [HttpGet("{id:guid}")] [ProducesResponseType(StatusCodes.Status200OK)] [ProducesResponseType(StatusCodes.Status404NotFound)] - public IActionResult GetPoster(Guid id, [FromQuery] ImageQuality? quality) + public async Task GetPoster(Guid id, [FromQuery] ImageQuality? quality) { - string path = thumbs.GetImagePath(id, quality ?? ImageQuality.High); - if (!System.IO.File.Exists(path)) + quality ??= ImageQuality.High; + if (await thumbs.IsImageSaved(id, quality.Value)) return NotFound(); // Allow clients to cache the image for 6 month. Response.Headers.CacheControl = $"public, max-age={60 * 60 * 24 * 31 * 6}"; - return PhysicalFile(Path.GetFullPath(path), "image/webp", true); + return File(await thumbs.GetImage(id, quality.Value), "image/webp", true); } }