From 0d91001376e3ee84d750eba21cbe44b73d1315cb Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Wed, 13 Mar 2024 19:27:10 +0100 Subject: [PATCH 01/26] Add rabbitmq --- .env.example | 3 +++ docker-compose.dev.yml | 12 +++++++++++- docker-compose.prod.yml | 11 ++++++++++- docker-compose.yml | 11 ++++++++++- 4 files changed, 34 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index 8611f813..c5849686 100644 --- a/.env.example +++ b/.env.example @@ -73,3 +73,6 @@ POSTGRES_PORT=5432 MEILI_HOST="http://meilisearch:7700" MEILI_MASTER_KEY="ghvjkgisbgkbgskegblfqbgjkebbhgwkjfb" + +RABBITMQ_DEFAULT_USER=kyoo +RABBITMQ_DEFAULT_PASS=aohohunuhouhuhhoahothonseuhaoensuthoaentsuhha diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 2956226f..9750308f 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -147,10 +147,20 @@ services: - .env healthcheck: test: ["CMD", "wget", "--no-verbose", "--spider", "http://localhost:7700/health"] - interval: 10s + interval: 30s timeout: 5s retries: 5 + rabbitmq: + image: rabbitmq:3-management-alpine + restart: on-failure + environment: + - RABBITMQ_DEFAULT_USER=${RABBITMQ_DEFAULT_USER} + - RABBITMQ_DEFAULT_PASS=${RABBITMQ_DEFAULT_PASS} + ports: + - 5672:5672 + - 15672:15672 + volumes: kyoo: db: diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 6701b3de..562c8243 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -120,10 +120,19 @@ services: - .env healthcheck: test: ["CMD", "wget", "--no-verbose", "--spider", "http://localhost:7700/health"] - interval: 10s + interval: 30s timeout: 5s retries: 5 + rabbitmq: + image: rabbitmq:3-alpine + restart: on-failure + environment: + - RABBITMQ_DEFAULT_USER=${RABBITMQ_DEFAULT_USER} + - RABBITMQ_DEFAULT_PASS=${RABBITMQ_DEFAULT_PASS} + ports: + - 5672:5672 + volumes: kyoo: db: diff --git a/docker-compose.yml b/docker-compose.yml index c30ed0f8..bef2c2b5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -119,10 +119,19 @@ services: - .env healthcheck: test: ["CMD", "wget", "--no-verbose", "--spider", "http://localhost:7700/health"] - interval: 10s + interval: 30s timeout: 5s retries: 5 + rabbitmq: + image: rabbitmq:3-alpine + restart: on-failure + environment: + - RABBITMQ_DEFAULT_USER=${RABBITMQ_DEFAULT_USER} + - RABBITMQ_DEFAULT_PASS=${RABBITMQ_DEFAULT_PASS} + ports: + - 5672:5672 + volumes: kyoo: db: From c15dcb02ec12dd0c1ef59e1dc14cf028571abe88 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 17 Mar 2024 12:23:11 +0100 Subject: [PATCH 02/26] Add watch status changed events --- .../Controllers/IWatchStatusRepository.cs | 22 ++++-- .../Repositories/WatchStatusRepository.cs | 67 ++++++++----------- 2 files changed, 43 insertions(+), 46 deletions(-) diff --git a/back/src/Kyoo.Abstractions/Controllers/IWatchStatusRepository.cs b/back/src/Kyoo.Abstractions/Controllers/IWatchStatusRepository.cs index e21806cf..8b666bd2 100644 --- a/back/src/Kyoo.Abstractions/Controllers/IWatchStatusRepository.cs +++ b/back/src/Kyoo.Abstractions/Controllers/IWatchStatusRepository.cs @@ -29,12 +29,7 @@ namespace Kyoo.Abstractions.Controllers; /// public interface IWatchStatusRepository { - // /// - // /// The event handler type for all events of this repository. - // /// - // /// The resource created/modified/deleted - // /// A representing the asynchronous operation. - // public delegate Task ResourceEventHandler(T resource); + public delegate Task ResourceEventHandler(T resource); Task> GetAll( Filter? filter = default, @@ -52,12 +47,22 @@ public interface IWatchStatusRepository int? percent ); + static event ResourceEventHandler OnMovieStatusChangedHandler; + + protected static Task OnMovieStatusChanged(Movie obj) => + OnMovieStatusChangedHandler?.Invoke(obj) ?? Task.CompletedTask; + Task DeleteMovieStatus(Guid movieId, Guid userId); Task GetShowStatus(Guid showId, Guid userId); Task SetShowStatus(Guid showId, Guid userId, WatchStatus status); + static event ResourceEventHandler OnShowStatusChangedHandler; + + protected static Task OnShowStatusChanged(Show obj) => + OnShowStatusChangedHandler?.Invoke(obj) ?? Task.CompletedTask; + Task DeleteShowStatus(Guid showId, Guid userId); Task GetEpisodeStatus(Guid episodeId, Guid userId); @@ -72,5 +77,10 @@ public interface IWatchStatusRepository int? percent ); + static event ResourceEventHandler OnEpisodeStatusChangedHandler; + + protected static Task OnEpisodeStatusChanged(Episode obj) => + OnEpisodeStatusChangedHandler?.Invoke(obj) ?? Task.CompletedTask; + Task DeleteEpisodeStatus(Guid episodeId, Guid userId); } diff --git a/back/src/Kyoo.Core/Controllers/Repositories/WatchStatusRepository.cs b/back/src/Kyoo.Core/Controllers/Repositories/WatchStatusRepository.cs index e5a12cfc..25c69bc6 100644 --- a/back/src/Kyoo.Core/Controllers/Repositories/WatchStatusRepository.cs +++ b/back/src/Kyoo.Core/Controllers/Repositories/WatchStatusRepository.cs @@ -33,7 +33,12 @@ using Microsoft.Extensions.DependencyInjection; namespace Kyoo.Core.Controllers; -public class WatchStatusRepository : IWatchStatusRepository +public class WatchStatusRepository( + DatabaseContext database, + IRepository movies, + DbConnection db, + SqlVariableContext context + ) : IWatchStatusRepository { /// /// If the watch percent is below this value, don't consider the item started. @@ -55,11 +60,6 @@ public class WatchStatusRepository : IWatchStatusRepository private WatchStatus Completed = WatchStatus.Completed; private WatchStatus Planned = WatchStatus.Planned; - private readonly DatabaseContext _database; - private readonly IRepository _movies; - private readonly DbConnection _db; - private readonly SqlVariableContext _context; - static WatchStatusRepository() { IRepository.OnCreated += async (ep) => @@ -78,19 +78,6 @@ public class WatchStatusRepository : IWatchStatusRepository }; } - public WatchStatusRepository( - DatabaseContext database, - IRepository movies, - DbConnection db, - SqlVariableContext context - ) - { - _database = database; - _movies = movies; - _db = db; - _context = context; - } - // language=PostgreSQL protected FormattableString Sql => $""" @@ -169,11 +156,11 @@ public class WatchStatusRepository : IWatchStatusRepository /// public Task GetOrDefault(Guid id, Include? include = null) { - return _db.QuerySingle( + return db.QuerySingle( Sql, Config, Mapper, - _context, + context, include, new Filter.Eq(nameof(IResource.Id), id) ); @@ -208,12 +195,12 @@ public class WatchStatusRepository : IWatchStatusRepository limit.AfterID = null; } - return await _db.Query( + return await db.Query( Sql, Config, Mapper, (id) => Get(id), - _context, + context, include, filter, null, @@ -224,7 +211,7 @@ public class WatchStatusRepository : IWatchStatusRepository /// public Task GetMovieStatus(Guid movieId, Guid userId) { - return _database.MovieWatchStatus.FirstOrDefaultAsync(x => + return database.MovieWatchStatus.FirstOrDefaultAsync(x => x.MovieId == movieId && x.UserId == userId ); } @@ -238,7 +225,7 @@ public class WatchStatusRepository : IWatchStatusRepository int? percent ) { - Movie movie = await _movies.Get(movieId); + Movie movie = await movies.Get(movieId); if (percent == null && watchedTime != null && movie.Runtime > 0) percent = (int)Math.Round(watchedTime.Value / (movie.Runtime.Value * 60f) * 100f); @@ -274,7 +261,7 @@ public class WatchStatusRepository : IWatchStatusRepository AddedDate = DateTime.UtcNow, PlayedDate = status == WatchStatus.Completed ? DateTime.UtcNow : null, }; - await _database + await database .MovieWatchStatus.Upsert(ret) .UpdateIf(x => status != Watching || x.Status != Completed) .RunAsync(); @@ -284,7 +271,7 @@ public class WatchStatusRepository : IWatchStatusRepository /// public async Task DeleteMovieStatus(Guid movieId, Guid userId) { - await _database + await database .MovieWatchStatus.Where(x => x.MovieId == movieId && x.UserId == userId) .ExecuteDeleteAsync(); } @@ -292,7 +279,7 @@ public class WatchStatusRepository : IWatchStatusRepository /// public Task GetShowStatus(Guid showId, Guid userId) { - return _database.ShowWatchStatus.FirstOrDefaultAsync(x => + return database.ShowWatchStatus.FirstOrDefaultAsync(x => x.ShowId == showId && x.UserId == userId ); } @@ -310,7 +297,7 @@ public class WatchStatusRepository : IWatchStatusRepository { int unseenEpisodeCount = status != WatchStatus.Completed - ? await _database + ? await database .Episodes.Where(x => x.ShowId == showId) .Where(x => x.Watched!.First(x => x.UserId == userId)!.Status != WatchStatus.Completed @@ -324,7 +311,7 @@ public class WatchStatusRepository : IWatchStatusRepository Guid? nextEpisodeId = null; if (status == WatchStatus.Watching) { - var cursor = await _database + var cursor = await database .Episodes.IgnoreQueryFilters() .Where(x => x.ShowId == showId) .OrderByDescending(x => x.AbsoluteNumber) @@ -346,7 +333,7 @@ public class WatchStatusRepository : IWatchStatusRepository nextEpisodeId = cursor?.Status.Status == WatchStatus.Watching ? cursor.Id - : await _database + : await database .Episodes.IgnoreQueryFilters() .Where(x => x.ShowId == showId) .OrderBy(x => x.AbsoluteNumber) @@ -374,11 +361,11 @@ public class WatchStatusRepository : IWatchStatusRepository } else if (status == WatchStatus.Completed) { - List episodes = await _database + List episodes = await database .Episodes.Where(x => x.ShowId == showId) .Select(x => x.Id) .ToListAsync(); - await _database + await database .EpisodeWatchStatus.UpsertRange( episodes.Select(episodeId => new EpisodeWatchStatus { @@ -412,7 +399,7 @@ public class WatchStatusRepository : IWatchStatusRepository UnseenEpisodesCount = unseenEpisodeCount, PlayedDate = status == WatchStatus.Completed ? DateTime.UtcNow : null, }; - await _database + await database .ShowWatchStatus.Upsert(ret) .UpdateIf(x => status != Watching || x.Status != Completed || newEpisode) .RunAsync(); @@ -422,11 +409,11 @@ public class WatchStatusRepository : IWatchStatusRepository /// public async Task DeleteShowStatus(Guid showId, Guid userId) { - await _database + await database .ShowWatchStatus.IgnoreAutoIncludes() .Where(x => x.ShowId == showId && x.UserId == userId) .ExecuteDeleteAsync(); - await _database + await database .EpisodeWatchStatus.Where(x => x.Episode.ShowId == showId && x.UserId == userId) .ExecuteDeleteAsync(); } @@ -434,7 +421,7 @@ public class WatchStatusRepository : IWatchStatusRepository /// public Task GetEpisodeStatus(Guid episodeId, Guid userId) { - return _database.EpisodeWatchStatus.FirstOrDefaultAsync(x => + return database.EpisodeWatchStatus.FirstOrDefaultAsync(x => x.EpisodeId == episodeId && x.UserId == userId ); } @@ -448,7 +435,7 @@ public class WatchStatusRepository : IWatchStatusRepository int? percent ) { - Episode episode = await _database.Episodes.FirstAsync(x => x.Id == episodeId); + Episode episode = await database.Episodes.FirstAsync(x => x.Id == episodeId); if (percent == null && watchedTime != null && episode.Runtime > 0) percent = (int)Math.Round(watchedTime.Value / (episode.Runtime.Value * 60f) * 100f); @@ -484,7 +471,7 @@ public class WatchStatusRepository : IWatchStatusRepository AddedDate = DateTime.UtcNow, PlayedDate = status == WatchStatus.Completed ? DateTime.UtcNow : null, }; - await _database + await database .EpisodeWatchStatus.Upsert(ret) .UpdateIf(x => status != Watching || x.Status != Completed) .RunAsync(); @@ -495,7 +482,7 @@ public class WatchStatusRepository : IWatchStatusRepository /// public async Task DeleteEpisodeStatus(Guid episodeId, Guid userId) { - await _database + await database .EpisodeWatchStatus.Where(x => x.EpisodeId == episodeId && x.UserId == userId) .ExecuteDeleteAsync(); } From 3a5d6ed2cdaa57de8ea7335e05b548317118c3ee Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 17 Mar 2024 12:35:10 +0100 Subject: [PATCH 03/26] Add deleted events --- .../Controllers/IWatchStatusRepository.cs | 29 +++++++++++++------ .../Repositories/WatchStatusRepository.cs | 6 ++++ 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/back/src/Kyoo.Abstractions/Controllers/IWatchStatusRepository.cs b/back/src/Kyoo.Abstractions/Controllers/IWatchStatusRepository.cs index 8b666bd2..23d6e6c6 100644 --- a/back/src/Kyoo.Abstractions/Controllers/IWatchStatusRepository.cs +++ b/back/src/Kyoo.Abstractions/Controllers/IWatchStatusRepository.cs @@ -30,6 +30,7 @@ namespace Kyoo.Abstractions.Controllers; public interface IWatchStatusRepository { public delegate Task ResourceEventHandler(T resource); + public delegate Task WatchStatusDeletedEventHandler(Guid resourceId, Guid userId); Task> GetAll( Filter? filter = default, @@ -47,24 +48,30 @@ public interface IWatchStatusRepository int? percent ); - static event ResourceEventHandler OnMovieStatusChangedHandler; - - protected static Task OnMovieStatusChanged(Movie obj) => + static event ResourceEventHandler OnMovieStatusChangedHandler; + protected static Task OnMovieStatusChanged(MovieWatchStatus obj) => OnMovieStatusChangedHandler?.Invoke(obj) ?? Task.CompletedTask; Task DeleteMovieStatus(Guid movieId, Guid userId); + static event WatchStatusDeletedEventHandler OnMovieStatusDeletedHandler; + protected static Task OnMovieStatusDeleted(Guid movieId, Guid userId) => + OnMovieStatusDeletedHandler?.Invoke(movieId, userId) ?? Task.CompletedTask; + Task GetShowStatus(Guid showId, Guid userId); Task SetShowStatus(Guid showId, Guid userId, WatchStatus status); - static event ResourceEventHandler OnShowStatusChangedHandler; - - protected static Task OnShowStatusChanged(Show obj) => + static event ResourceEventHandler OnShowStatusChangedHandler; + protected static Task OnShowStatusChanged(ShowWatchStatus obj) => OnShowStatusChangedHandler?.Invoke(obj) ?? Task.CompletedTask; Task DeleteShowStatus(Guid showId, Guid userId); + static event WatchStatusDeletedEventHandler OnShowStatusDeletedHandler; + protected static Task OnShowStatusDeleted(Guid showId, Guid userId) => + OnShowStatusDeletedHandler?.Invoke(showId, userId) ?? Task.CompletedTask; + Task GetEpisodeStatus(Guid episodeId, Guid userId); /// Where the user has stopped watching. Only usable if Status @@ -77,10 +84,14 @@ public interface IWatchStatusRepository int? percent ); - static event ResourceEventHandler OnEpisodeStatusChangedHandler; - - protected static Task OnEpisodeStatusChanged(Episode obj) => + static event ResourceEventHandler OnEpisodeStatusChangedHandler; + protected static Task OnEpisodeStatusChanged(EpisodeWatchStatus obj) => OnEpisodeStatusChangedHandler?.Invoke(obj) ?? Task.CompletedTask; Task DeleteEpisodeStatus(Guid episodeId, Guid userId); + + static event WatchStatusDeletedEventHandler OnEpisodeStatusDeletedHandler; + protected static Task OnEpisodeStatusDeleted(Guid episodeId, Guid userId) => + OnEpisodeStatusDeletedHandler?.Invoke(episodeId, userId) ?? Task.CompletedTask; + } diff --git a/back/src/Kyoo.Core/Controllers/Repositories/WatchStatusRepository.cs b/back/src/Kyoo.Core/Controllers/Repositories/WatchStatusRepository.cs index 25c69bc6..1465d938 100644 --- a/back/src/Kyoo.Core/Controllers/Repositories/WatchStatusRepository.cs +++ b/back/src/Kyoo.Core/Controllers/Repositories/WatchStatusRepository.cs @@ -265,6 +265,7 @@ public class WatchStatusRepository( .MovieWatchStatus.Upsert(ret) .UpdateIf(x => status != Watching || x.Status != Completed) .RunAsync(); + await IWatchStatusRepository.OnMovieStatusChanged(ret); return ret; } @@ -274,6 +275,7 @@ public class WatchStatusRepository( await database .MovieWatchStatus.Where(x => x.MovieId == movieId && x.UserId == userId) .ExecuteDeleteAsync(); + await IWatchStatusRepository.OnMovieStatusDeleted(movieId, userId); } /// @@ -403,6 +405,7 @@ public class WatchStatusRepository( .ShowWatchStatus.Upsert(ret) .UpdateIf(x => status != Watching || x.Status != Completed || newEpisode) .RunAsync(); + await IWatchStatusRepository.OnShowStatusChanged(ret); return ret; } @@ -416,6 +419,7 @@ public class WatchStatusRepository( await database .EpisodeWatchStatus.Where(x => x.Episode.ShowId == showId && x.UserId == userId) .ExecuteDeleteAsync(); + await IWatchStatusRepository.OnShowStatusDeleted(showId, userId); } /// @@ -475,6 +479,7 @@ public class WatchStatusRepository( .EpisodeWatchStatus.Upsert(ret) .UpdateIf(x => status != Watching || x.Status != Completed) .RunAsync(); + await IWatchStatusRepository.OnEpisodeStatusChanged(ret); await SetShowStatus(episode.ShowId, userId, WatchStatus.Watching); return ret; } @@ -485,5 +490,6 @@ public class WatchStatusRepository( await database .EpisodeWatchStatus.Where(x => x.EpisodeId == episodeId && x.UserId == userId) .ExecuteDeleteAsync(); + await IWatchStatusRepository.OnEpisodeStatusDeleted(episodeId, userId); } } From f1d72cb4806869e738e7bb7ac85450e8df82472a Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 17 Mar 2024 18:32:36 +0100 Subject: [PATCH 04/26] Publish Create/Update/Delete resource events to rabbit --- .env.example | 1 + back/Kyoo.sln | 20 ++++- back/src/Kyoo.Host/Kyoo.Host.csproj | 1 + back/src/Kyoo.Host/PluginsStartup.cs | 2 + back/src/Kyoo.RabbitMq/Kyoo.RabbitMq.csproj | 16 ++++ back/src/Kyoo.RabbitMq/RabbitMqModule.cs | 53 ++++++++++++ back/src/Kyoo.RabbitMq/RabbitProducer.cs | 90 +++++++++++++++++++++ 7 files changed, 182 insertions(+), 1 deletion(-) create mode 100644 back/src/Kyoo.RabbitMq/Kyoo.RabbitMq.csproj create mode 100644 back/src/Kyoo.RabbitMq/RabbitMqModule.cs create mode 100644 back/src/Kyoo.RabbitMq/RabbitProducer.cs diff --git a/.env.example b/.env.example index c5849686..d2c7391b 100644 --- a/.env.example +++ b/.env.example @@ -74,5 +74,6 @@ POSTGRES_PORT=5432 MEILI_HOST="http://meilisearch:7700" MEILI_MASTER_KEY="ghvjkgisbgkbgskegblfqbgjkebbhgwkjfb" +RABBITMQ_HOST=rabbitmq:5672 RABBITMQ_DEFAULT_USER=kyoo RABBITMQ_DEFAULT_PASS=aohohunuhouhuhhoahothonseuhaoensuthoaentsuhha diff --git a/back/Kyoo.sln b/back/Kyoo.sln index db805de4..10bf4f8b 100644 --- a/back/Kyoo.sln +++ b/back/Kyoo.sln @@ -1,4 +1,5 @@ -Microsoft Visual Studio Solution File, Format Version 12.00 +Microsoft Visual Studio Solution File, Format Version 12.00 +# Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Kyoo.Core", "src\Kyoo.Core\Kyoo.Core.csproj", "{0F8275B6-C7DD-42DF-A168-755C81B1C329}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Kyoo.Abstractions", "src\Kyoo.Abstractions\Kyoo.Abstractions.csproj", "{BAB2CAE1-AC28-4509-AA3E-8DC75BD59220}" @@ -13,6 +14,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Kyoo.Host", "src\Kyoo.Host\ EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Kyoo.Meilisearch", "src\Kyoo.Meilisearch\Kyoo.Meilisearch.csproj", "{F8E6018A-FD51-40EB-99FF-A26BA59F2762}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{787FB205-9D7E-4946-AFE0-BD68E286F569}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Kyoo.RabbitMq", "src\Kyoo.RabbitMq\Kyoo.RabbitMq.csproj", "{B97AD4A8-E6E6-41CD-87DF-5F1326FD7198}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -59,5 +64,18 @@ Global {F8E6018A-FD51-40EB-99FF-A26BA59F2762}.Debug|Any CPU.Build.0 = Debug|Any CPU {F8E6018A-FD51-40EB-99FF-A26BA59F2762}.Release|Any CPU.ActiveCfg = Release|Any CPU {F8E6018A-FD51-40EB-99FF-A26BA59F2762}.Release|Any CPU.Build.0 = Release|Any CPU + {44F2208F-C015-4A01-8D6A-20F82437AFDB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {44F2208F-C015-4A01-8D6A-20F82437AFDB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {44F2208F-C015-4A01-8D6A-20F82437AFDB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {44F2208F-C015-4A01-8D6A-20F82437AFDB}.Release|Any CPU.Build.0 = Release|Any CPU + {B97AD4A8-E6E6-41CD-87DF-5F1326FD7198}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B97AD4A8-E6E6-41CD-87DF-5F1326FD7198}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B97AD4A8-E6E6-41CD-87DF-5F1326FD7198}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B97AD4A8-E6E6-41CD-87DF-5F1326FD7198}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {0C8AA7EA-E723-4532-852F-35AA4E8AFED5} = {FEAE1B0E-D797-470F-9030-0EF743575ECC} + {44F2208F-C015-4A01-8D6A-20F82437AFDB} = {BB39454F-53E4-4238-9659-A39638496FB3} + {B97AD4A8-E6E6-41CD-87DF-5F1326FD7198} = {787FB205-9D7E-4946-AFE0-BD68E286F569} EndGlobalSection EndGlobal diff --git a/back/src/Kyoo.Host/Kyoo.Host.csproj b/back/src/Kyoo.Host/Kyoo.Host.csproj index dfed95e7..418d06f9 100644 --- a/back/src/Kyoo.Host/Kyoo.Host.csproj +++ b/back/src/Kyoo.Host/Kyoo.Host.csproj @@ -26,6 +26,7 @@ + diff --git a/back/src/Kyoo.Host/PluginsStartup.cs b/back/src/Kyoo.Host/PluginsStartup.cs index ddd6a57b..40f4c3d1 100644 --- a/back/src/Kyoo.Host/PluginsStartup.cs +++ b/back/src/Kyoo.Host/PluginsStartup.cs @@ -27,6 +27,7 @@ using Kyoo.Core; using Kyoo.Host.Controllers; using Kyoo.Meiliseach; using Kyoo.Postgresql; +using Kyoo.RabbitMq; using Kyoo.Swagger; using Kyoo.Utils; using Microsoft.AspNetCore.Builder; @@ -66,6 +67,7 @@ public class PluginsStartup typeof(AuthenticationModule), typeof(PostgresModule), typeof(MeilisearchModule), + typeof(RabbitMqModule), typeof(SwaggerModule) ); } diff --git a/back/src/Kyoo.RabbitMq/Kyoo.RabbitMq.csproj b/back/src/Kyoo.RabbitMq/Kyoo.RabbitMq.csproj new file mode 100644 index 00000000..382b5b7c --- /dev/null +++ b/back/src/Kyoo.RabbitMq/Kyoo.RabbitMq.csproj @@ -0,0 +1,16 @@ + + + enable + enable + Kyoo.RabbitMq + + + + + + + + + + + diff --git a/back/src/Kyoo.RabbitMq/RabbitMqModule.cs b/back/src/Kyoo.RabbitMq/RabbitMqModule.cs new file mode 100644 index 00000000..2e7031db --- /dev/null +++ b/back/src/Kyoo.RabbitMq/RabbitMqModule.cs @@ -0,0 +1,53 @@ +// Kyoo - A portable and vast media library solution. +// Copyright (c) Kyoo. +// +// See AUTHORS.md and LICENSE file in the project root for full license information. +// +// Kyoo is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// any later version. +// +// Kyoo is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with Kyoo. If not, see . + +using Autofac; +using Kyoo.Abstractions.Controllers; +using Microsoft.Extensions.Configuration; +using RabbitMQ.Client; + +namespace Kyoo.RabbitMq; + +public class RabbitMqModule(IConfiguration configuration) : IPlugin +{ + /// + public string Name => "RabbitMq"; + + /// + public void Configure(ContainerBuilder builder) + { + builder + .Register( + (_) => + { + ConnectionFactory factory = + new() + { + UserName = configuration.GetValue("RABBITMQ_DEFAULT_USER", "guest"), + Password = configuration.GetValue("RABBITMQ_DEFAULT_USER", "guest"), + HostName = configuration.GetValue("RABBITMQ_HOST", "rabbitmq:5672"), + }; + + return factory.CreateConnection(); + } + ) + .AsSelf() + .SingleInstance(); + builder.RegisterType().AsSelf().SingleInstance().AutoActivate(); + } +} diff --git a/back/src/Kyoo.RabbitMq/RabbitProducer.cs b/back/src/Kyoo.RabbitMq/RabbitProducer.cs new file mode 100644 index 00000000..e434989a --- /dev/null +++ b/back/src/Kyoo.RabbitMq/RabbitProducer.cs @@ -0,0 +1,90 @@ +// Kyoo - A portable and vast media library solution. +// Copyright (c) Kyoo. +// +// See AUTHORS.md and LICENSE file in the project root for full license information. +// +// Kyoo is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// any later version. +// +// Kyoo is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with Kyoo. If not, see . + +using System.Text; +using System.Text.Json; +using Kyoo.Abstractions.Controllers; +using Kyoo.Abstractions.Models; +using RabbitMQ.Client; + +namespace Kyoo.RabbitMq; + +public class RabbitProducer +{ + private readonly IModel _channel; + + public RabbitProducer(IConnection rabbitConnection) + { + _channel = rabbitConnection.CreateModel(); + + _channel.ExchangeDeclare(exchange: "events.resource.collection", type: ExchangeType.Fanout); + IRepository.OnCreated += _Publish("events.resource.collection", "created"); + IRepository.OnEdited += _Publish("events.resource.collection", "edited"); + IRepository.OnDeleted += _Publish("events.resource.collection", "deleted"); + + _channel.ExchangeDeclare(exchange: "events.resource.movie", type: ExchangeType.Fanout); + IRepository.OnCreated += _Publish("events.resource.movie", "created"); + IRepository.OnEdited += _Publish("events.resource.movie", "edited"); + IRepository.OnDeleted += _Publish("events.resource.movie", "deleted"); + + _channel.ExchangeDeclare(exchange: "events.resource.show", type: ExchangeType.Fanout); + IRepository.OnCreated += _Publish("events.resource.show", "created"); + IRepository.OnEdited += _Publish("events.resource.show", "edited"); + IRepository.OnDeleted += _Publish("events.resource.show", "deleted"); + + _channel.ExchangeDeclare(exchange: "events.resource.season", type: ExchangeType.Fanout); + IRepository.OnCreated += _Publish("events.resource.season", "created"); + IRepository.OnEdited += _Publish("events.resource.season", "edited"); + IRepository.OnDeleted += _Publish("events.resource.season", "deleted"); + + _channel.ExchangeDeclare(exchange: "events.resource.episode", type: ExchangeType.Fanout); + IRepository.OnCreated += _Publish("events.resource.episode", "created"); + IRepository.OnEdited += _Publish("events.resource.episode", "edited"); + IRepository.OnDeleted += _Publish("events.resource.episode", "deleted"); + + _channel.ExchangeDeclare(exchange: "events.resource.studio", type: ExchangeType.Fanout); + IRepository.OnCreated += _Publish("events.resource.studio", "created"); + IRepository.OnEdited += _Publish("events.resource.studio", "edited"); + IRepository.OnDeleted += _Publish("events.resource.studio", "deleted"); + + _channel.ExchangeDeclare(exchange: "events.resource.user", type: ExchangeType.Fanout); + IRepository.OnCreated += _Publish("events.resource.user", "created"); + IRepository.OnEdited += _Publish("events.resource.user", "edited"); + IRepository.OnDeleted += _Publish("events.resource.user", "deleted"); + } + + private IRepository.ResourceEventHandler _Publish(string exchange, string action) + where T : IResource, IQuery + { + return (T resource) => + { + var message = new + { + Action = action, + Type = typeof(T).Name.ToLowerInvariant(), + Value = resource, + }; + _channel.BasicPublish( + exchange, + routingKey: string.Empty, + body: Encoding.UTF8.GetBytes(JsonSerializer.Serialize(message)) + ); + return Task.CompletedTask; + }; + } +} From cbb05ac977f3ee9c18ad6f1c1c8850546f3c72f2 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 17 Mar 2024 18:45:16 +0100 Subject: [PATCH 05/26] Change rabbit channel from fanout to topic based --- back/src/Kyoo.RabbitMq/RabbitProducer.cs | 64 ++++++++++-------------- 1 file changed, 26 insertions(+), 38 deletions(-) diff --git a/back/src/Kyoo.RabbitMq/RabbitProducer.cs b/back/src/Kyoo.RabbitMq/RabbitProducer.cs index e434989a..51959a0a 100644 --- a/back/src/Kyoo.RabbitMq/RabbitProducer.cs +++ b/back/src/Kyoo.RabbitMq/RabbitProducer.cs @@ -32,43 +32,31 @@ public class RabbitProducer { _channel = rabbitConnection.CreateModel(); - _channel.ExchangeDeclare(exchange: "events.resource.collection", type: ExchangeType.Fanout); - IRepository.OnCreated += _Publish("events.resource.collection", "created"); - IRepository.OnEdited += _Publish("events.resource.collection", "edited"); - IRepository.OnDeleted += _Publish("events.resource.collection", "deleted"); - - _channel.ExchangeDeclare(exchange: "events.resource.movie", type: ExchangeType.Fanout); - IRepository.OnCreated += _Publish("events.resource.movie", "created"); - IRepository.OnEdited += _Publish("events.resource.movie", "edited"); - IRepository.OnDeleted += _Publish("events.resource.movie", "deleted"); - - _channel.ExchangeDeclare(exchange: "events.resource.show", type: ExchangeType.Fanout); - IRepository.OnCreated += _Publish("events.resource.show", "created"); - IRepository.OnEdited += _Publish("events.resource.show", "edited"); - IRepository.OnDeleted += _Publish("events.resource.show", "deleted"); - - _channel.ExchangeDeclare(exchange: "events.resource.season", type: ExchangeType.Fanout); - IRepository.OnCreated += _Publish("events.resource.season", "created"); - IRepository.OnEdited += _Publish("events.resource.season", "edited"); - IRepository.OnDeleted += _Publish("events.resource.season", "deleted"); - - _channel.ExchangeDeclare(exchange: "events.resource.episode", type: ExchangeType.Fanout); - IRepository.OnCreated += _Publish("events.resource.episode", "created"); - IRepository.OnEdited += _Publish("events.resource.episode", "edited"); - IRepository.OnDeleted += _Publish("events.resource.episode", "deleted"); - - _channel.ExchangeDeclare(exchange: "events.resource.studio", type: ExchangeType.Fanout); - IRepository.OnCreated += _Publish("events.resource.studio", "created"); - IRepository.OnEdited += _Publish("events.resource.studio", "edited"); - IRepository.OnDeleted += _Publish("events.resource.studio", "deleted"); - - _channel.ExchangeDeclare(exchange: "events.resource.user", type: ExchangeType.Fanout); - IRepository.OnCreated += _Publish("events.resource.user", "created"); - IRepository.OnEdited += _Publish("events.resource.user", "edited"); - IRepository.OnDeleted += _Publish("events.resource.user", "deleted"); + _channel.ExchangeDeclare("events.resource", ExchangeType.Topic); + _ListenResourceEvents("events.resource"); + _ListenResourceEvents("events.resource"); + _ListenResourceEvents("events.resource"); + _ListenResourceEvents("events.resource"); + _ListenResourceEvents("events.resource"); + _ListenResourceEvents("events.resource"); + _ListenResourceEvents("events.resource"); } - private IRepository.ResourceEventHandler _Publish(string exchange, string action) + private void _ListenResourceEvents(string exchange) + where T : IResource, IQuery + { + string type = typeof(T).Name.ToLowerInvariant(); + + IRepository.OnCreated += _Publish(exchange, type, "created"); + IRepository.OnEdited += _Publish(exchange, type, "edited"); + IRepository.OnDeleted += _Publish(exchange, type, "deleted"); + } + + private IRepository.ResourceEventHandler _Publish( + string exchange, + string type, + string action + ) where T : IResource, IQuery { return (T resource) => @@ -76,12 +64,12 @@ public class RabbitProducer var message = new { Action = action, - Type = typeof(T).Name.ToLowerInvariant(), - Value = resource, + Type = type, + Resource = resource, }; _channel.BasicPublish( exchange, - routingKey: string.Empty, + routingKey: $"{type}.{action}", body: Encoding.UTF8.GetBytes(JsonSerializer.Serialize(message)) ); return Task.CompletedTask; From b6f9c050e122ca01c7c4affc1cdc863f238c40ed Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 17 Mar 2024 19:25:28 +0100 Subject: [PATCH 06/26] Publish WatchStatus changes to rabbitmq --- .../Controllers/IWatchStatusRepository.cs | 14 ------ .../Models/Resources/WatchStatus.cs | 20 ++++++-- .../Repositories/WatchStatusRepository.cs | 32 ++++++++++-- back/src/Kyoo.RabbitMq/Message.cs | 39 +++++++++++++++ back/src/Kyoo.RabbitMq/RabbitProducer.cs | 50 +++++++++++++++---- 5 files changed, 124 insertions(+), 31 deletions(-) create mode 100644 back/src/Kyoo.RabbitMq/Message.cs diff --git a/back/src/Kyoo.Abstractions/Controllers/IWatchStatusRepository.cs b/back/src/Kyoo.Abstractions/Controllers/IWatchStatusRepository.cs index 23d6e6c6..a4372d23 100644 --- a/back/src/Kyoo.Abstractions/Controllers/IWatchStatusRepository.cs +++ b/back/src/Kyoo.Abstractions/Controllers/IWatchStatusRepository.cs @@ -30,7 +30,6 @@ namespace Kyoo.Abstractions.Controllers; public interface IWatchStatusRepository { public delegate Task ResourceEventHandler(T resource); - public delegate Task WatchStatusDeletedEventHandler(Guid resourceId, Guid userId); Task> GetAll( Filter? filter = default, @@ -54,10 +53,6 @@ public interface IWatchStatusRepository Task DeleteMovieStatus(Guid movieId, Guid userId); - static event WatchStatusDeletedEventHandler OnMovieStatusDeletedHandler; - protected static Task OnMovieStatusDeleted(Guid movieId, Guid userId) => - OnMovieStatusDeletedHandler?.Invoke(movieId, userId) ?? Task.CompletedTask; - Task GetShowStatus(Guid showId, Guid userId); Task SetShowStatus(Guid showId, Guid userId, WatchStatus status); @@ -68,10 +63,6 @@ public interface IWatchStatusRepository Task DeleteShowStatus(Guid showId, Guid userId); - static event WatchStatusDeletedEventHandler OnShowStatusDeletedHandler; - protected static Task OnShowStatusDeleted(Guid showId, Guid userId) => - OnShowStatusDeletedHandler?.Invoke(showId, userId) ?? Task.CompletedTask; - Task GetEpisodeStatus(Guid episodeId, Guid userId); /// Where the user has stopped watching. Only usable if Status @@ -89,9 +80,4 @@ public interface IWatchStatusRepository OnEpisodeStatusChangedHandler?.Invoke(obj) ?? Task.CompletedTask; Task DeleteEpisodeStatus(Guid episodeId, Guid userId); - - static event WatchStatusDeletedEventHandler OnEpisodeStatusDeletedHandler; - protected static Task OnEpisodeStatusDeleted(Guid episodeId, Guid userId) => - OnEpisodeStatusDeletedHandler?.Invoke(episodeId, userId) ?? Task.CompletedTask; - } diff --git a/back/src/Kyoo.Abstractions/Models/Resources/WatchStatus.cs b/back/src/Kyoo.Abstractions/Models/Resources/WatchStatus.cs index 928cd640..1caf380c 100644 --- a/back/src/Kyoo.Abstractions/Models/Resources/WatchStatus.cs +++ b/back/src/Kyoo.Abstractions/Models/Resources/WatchStatus.cs @@ -46,13 +46,27 @@ public enum WatchStatus /// The user has not started watching this but plans to. /// Planned, + + /// + /// The watch status was deleted and can not be retrived again. + /// + Deleted, } +public interface IWatchStatus +{ + /// + /// Has the user started watching, is it planned? + /// + public WatchStatus Status { get; set; } +} + + /// /// Metadata of what an user as started/planned to watch. /// [SqlFirstColumn(nameof(UserId))] -public class MovieWatchStatus : IAddedDate +public class MovieWatchStatus : IAddedDate, IWatchStatus { /// /// The ID of the user that started watching this episode. @@ -107,7 +121,7 @@ public class MovieWatchStatus : IAddedDate } [SqlFirstColumn(nameof(UserId))] -public class EpisodeWatchStatus : IAddedDate +public class EpisodeWatchStatus : IAddedDate, IWatchStatus { /// /// The ID of the user that started watching this episode. @@ -162,7 +176,7 @@ public class EpisodeWatchStatus : IAddedDate } [SqlFirstColumn(nameof(UserId))] -public class ShowWatchStatus : IAddedDate +public class ShowWatchStatus : IAddedDate, IWatchStatus { /// /// The ID of the user that started watching this episode. diff --git a/back/src/Kyoo.Core/Controllers/Repositories/WatchStatusRepository.cs b/back/src/Kyoo.Core/Controllers/Repositories/WatchStatusRepository.cs index 1465d938..d8a72070 100644 --- a/back/src/Kyoo.Core/Controllers/Repositories/WatchStatusRepository.cs +++ b/back/src/Kyoo.Core/Controllers/Repositories/WatchStatusRepository.cs @@ -38,7 +38,7 @@ public class WatchStatusRepository( IRepository movies, DbConnection db, SqlVariableContext context - ) : IWatchStatusRepository +) : IWatchStatusRepository { /// /// If the watch percent is below this value, don't consider the item started. @@ -275,7 +275,15 @@ public class WatchStatusRepository( await database .MovieWatchStatus.Where(x => x.MovieId == movieId && x.UserId == userId) .ExecuteDeleteAsync(); - await IWatchStatusRepository.OnMovieStatusDeleted(movieId, userId); + await IWatchStatusRepository.OnMovieStatusChanged( + new() + { + UserId = userId, + MovieId = movieId, + AddedDate = DateTime.UtcNow, + Status = WatchStatus.Deleted, + } + ); } /// @@ -419,7 +427,15 @@ public class WatchStatusRepository( await database .EpisodeWatchStatus.Where(x => x.Episode.ShowId == showId && x.UserId == userId) .ExecuteDeleteAsync(); - await IWatchStatusRepository.OnShowStatusDeleted(showId, userId); + await IWatchStatusRepository.OnShowStatusChanged( + new() + { + UserId = userId, + ShowId = showId, + AddedDate = DateTime.UtcNow, + Status = WatchStatus.Deleted, + } + ); } /// @@ -490,6 +506,14 @@ public class WatchStatusRepository( await database .EpisodeWatchStatus.Where(x => x.EpisodeId == episodeId && x.UserId == userId) .ExecuteDeleteAsync(); - await IWatchStatusRepository.OnEpisodeStatusDeleted(episodeId, userId); + await IWatchStatusRepository.OnEpisodeStatusChanged( + new() + { + UserId = userId, + EpisodeId = episodeId, + AddedDate = DateTime.UtcNow, + Status = WatchStatus.Deleted, + } + ); } } diff --git a/back/src/Kyoo.RabbitMq/Message.cs b/back/src/Kyoo.RabbitMq/Message.cs new file mode 100644 index 00000000..a1bbba9b --- /dev/null +++ b/back/src/Kyoo.RabbitMq/Message.cs @@ -0,0 +1,39 @@ +// Kyoo - A portable and vast media library solution. +// Copyright (c) Kyoo. +// +// See AUTHORS.md and LICENSE file in the project root for full license information. +// +// Kyoo is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// any later version. +// +// Kyoo is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with Kyoo. If not, see . + +using System.Text; +using System.Text.Json; + +namespace Kyoo.RabbitMq; + +public class Message +{ + public string Action { get; set; } + public string Type { get; set; } + public object Value { get; set; } + + public string AsRoutingKey() + { + return $"{Type}.{Action}"; + } + + public byte[] AsBytes() + { + return Encoding.UTF8.GetBytes(JsonSerializer.Serialize(this)); + } +} diff --git a/back/src/Kyoo.RabbitMq/RabbitProducer.cs b/back/src/Kyoo.RabbitMq/RabbitProducer.cs index 51959a0a..d04ffff7 100644 --- a/back/src/Kyoo.RabbitMq/RabbitProducer.cs +++ b/back/src/Kyoo.RabbitMq/RabbitProducer.cs @@ -16,8 +16,6 @@ // You should have received a copy of the GNU General Public License // along with Kyoo. If not, see . -using System.Text; -using System.Text.Json; using Kyoo.Abstractions.Controllers; using Kyoo.Abstractions.Models; using RabbitMQ.Client; @@ -40,6 +38,16 @@ public class RabbitProducer _ListenResourceEvents("events.resource"); _ListenResourceEvents("events.resource"); _ListenResourceEvents("events.resource"); + + _channel.ExchangeDeclare("events.watched", ExchangeType.Topic); + IWatchStatusRepository.OnMovieStatusChangedHandler += _PublishWatchStatus( + "movie" + ); + IWatchStatusRepository.OnShowStatusChangedHandler += _PublishWatchStatus( + "show" + ); + IWatchStatusRepository.OnEpisodeStatusChangedHandler += + _PublishWatchStatus("episode"); } private void _ListenResourceEvents(string exchange) @@ -61,16 +69,38 @@ public class RabbitProducer { return (T resource) => { - var message = new - { - Action = action, - Type = type, - Resource = resource, - }; + Message message = + new() + { + Action = action, + Type = type, + Value = resource, + }; _channel.BasicPublish( exchange, - routingKey: $"{type}.{action}", - body: Encoding.UTF8.GetBytes(JsonSerializer.Serialize(message)) + routingKey: message.AsRoutingKey(), + body: message.AsBytes() + ); + return Task.CompletedTask; + }; + } + + private IWatchStatusRepository.ResourceEventHandler _PublishWatchStatus(string resource) + where T : IWatchStatus + { + return (status) => + { + Message message = + new() + { + Type = resource, + Action = status.Status.ToString().ToLowerInvariant(), + Value = status, + }; + _channel.BasicPublish( + exchange: "events.watched", + routingKey: message.AsRoutingKey(), + body: message.AsBytes() ); return Task.CompletedTask; }; From 115b9fa4b3d516b2faf3c339a5e69a3bf0d50acb Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 17 Mar 2024 21:14:44 +0100 Subject: [PATCH 07/26] Fix rabbitmq config --- .env.example | 2 +- back/Dockerfile | 1 + back/Dockerfile.dev | 1 + back/src/Kyoo.RabbitMq/RabbitMqModule.cs | 5 +++-- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index d2c7391b..53e60d07 100644 --- a/.env.example +++ b/.env.example @@ -74,6 +74,6 @@ POSTGRES_PORT=5432 MEILI_HOST="http://meilisearch:7700" MEILI_MASTER_KEY="ghvjkgisbgkbgskegblfqbgjkebbhgwkjfb" -RABBITMQ_HOST=rabbitmq:5672 +RABBITMQ_HOST=rabbitmq RABBITMQ_DEFAULT_USER=kyoo RABBITMQ_DEFAULT_PASS=aohohunuhouhuhhoahothonseuhaoensuthoaentsuhha diff --git a/back/Dockerfile b/back/Dockerfile index 862dbbfe..aee9f87e 100644 --- a/back/Dockerfile +++ b/back/Dockerfile @@ -11,6 +11,7 @@ COPY src/Kyoo.Core/Kyoo.Core.csproj src/Kyoo.Core/Kyoo.Core.csproj COPY src/Kyoo.Host/Kyoo.Host.csproj src/Kyoo.Host/Kyoo.Host.csproj COPY src/Kyoo.Postgresql/Kyoo.Postgresql.csproj src/Kyoo.Postgresql/Kyoo.Postgresql.csproj COPY src/Kyoo.Meilisearch/Kyoo.Meilisearch.csproj src/Kyoo.Meilisearch/Kyoo.Meilisearch.csproj +COPY src/Kyoo.RabbitMq/Kyoo.RabbitMq.csproj src/Kyoo.RabbitMq/Kyoo.RabbitMq.csproj COPY src/Kyoo.Swagger/Kyoo.Swagger.csproj src/Kyoo.Swagger/Kyoo.Swagger.csproj RUN dotnet restore -a $TARGETARCH diff --git a/back/Dockerfile.dev b/back/Dockerfile.dev index 53213cbb..ac2a62e9 100644 --- a/back/Dockerfile.dev +++ b/back/Dockerfile.dev @@ -11,6 +11,7 @@ COPY src/Kyoo.Core/Kyoo.Core.csproj src/Kyoo.Core/Kyoo.Core.csproj COPY src/Kyoo.Host/Kyoo.Host.csproj src/Kyoo.Host/Kyoo.Host.csproj COPY src/Kyoo.Postgresql/Kyoo.Postgresql.csproj src/Kyoo.Postgresql/Kyoo.Postgresql.csproj COPY src/Kyoo.Meilisearch/Kyoo.Meilisearch.csproj src/Kyoo.Meilisearch/Kyoo.Meilisearch.csproj +COPY src/Kyoo.RabbitMq/Kyoo.RabbitMq.csproj src/Kyoo.RabbitMq/Kyoo.RabbitMq.csproj COPY src/Kyoo.Swagger/Kyoo.Swagger.csproj src/Kyoo.Swagger/Kyoo.Swagger.csproj RUN dotnet restore diff --git a/back/src/Kyoo.RabbitMq/RabbitMqModule.cs b/back/src/Kyoo.RabbitMq/RabbitMqModule.cs index 2e7031db..6aaacb55 100644 --- a/back/src/Kyoo.RabbitMq/RabbitMqModule.cs +++ b/back/src/Kyoo.RabbitMq/RabbitMqModule.cs @@ -39,8 +39,9 @@ public class RabbitMqModule(IConfiguration configuration) : IPlugin new() { UserName = configuration.GetValue("RABBITMQ_DEFAULT_USER", "guest"), - Password = configuration.GetValue("RABBITMQ_DEFAULT_USER", "guest"), - HostName = configuration.GetValue("RABBITMQ_HOST", "rabbitmq:5672"), + Password = configuration.GetValue("RABBITMQ_DEFAULT_PASS", "guest"), + HostName = configuration.GetValue("RABBITMQ_HOST", "rabbitmq"), + Port = 5672, }; return factory.CreateConnection(); From 44bb88910fa04402a1d46cc5f8150f5f86f1f4b2 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Mon, 18 Mar 2024 21:55:08 +0100 Subject: [PATCH 08/26] Add simkl oidc --- .env.example | 2 +- .../Controllers/OidcController.cs | 54 +++++++++++-------- .../Models/DTO/JwtProfile.cs | 43 ++++++++++++--- .../Models/Options/PermissionOption.cs | 29 +++++++++- front/packages/ui/src/settings/oidc.tsx | 1 - 5 files changed, 97 insertions(+), 32 deletions(-) diff --git a/.env.example b/.env.example index 53e60d07..a1089c44 100644 --- a/.env.example +++ b/.env.example @@ -41,7 +41,7 @@ THEMOVIEDB_APIKEY= # The url you can use to reach your kyoo instance. This is used during oidc to redirect users to your instance. PUBLIC_URL=http://localhost:5000 -# Use a builtin oidc service (google or discord): +# Use a builtin oidc service (google, discord, or simkl): # When you create a client_id, secret combo you may be asked for a redirect url. You need to specify https://YOUR-PUBLIC-URL/api/auth/logged/YOUR-SERVICE-NAME OIDC_DISCORD_CLIENTID= OIDC_DISCORD_SECRET= diff --git a/back/src/Kyoo.Authentication/Controllers/OidcController.cs b/back/src/Kyoo.Authentication/Controllers/OidcController.cs index 8e9b9eec..d5909256 100644 --- a/back/src/Kyoo.Authentication/Controllers/OidcController.cs +++ b/back/src/Kyoo.Authentication/Controllers/OidcController.cs @@ -22,6 +22,7 @@ using System.ComponentModel.DataAnnotations; using System.Net.Http; using System.Net.Http.Json; using System.Text; +using System.Text.Json; using System.Threading.Tasks; using Kyoo.Abstractions.Controllers; using Kyoo.Abstractions.Models; @@ -46,21 +47,18 @@ public class OidcController( Encoding.UTF8.GetBytes($"{prov.ClientId}:{prov.Secret}") ); client.DefaultRequestHeaders.Add("Authorization", $"Basic {auth}"); - - HttpResponseMessage resp = await client.PostAsync( - prov.TokenUrl, - new FormUrlEncodedContent( - new Dictionary() - { - ["code"] = code, - ["client_id"] = prov.ClientId, - ["client_secret"] = prov.Secret, - ["redirect_uri"] = - $"{options.PublicUrl.TrimEnd('/')}/api/auth/logged/{provider}", - ["grant_type"] = "authorization_code", - } - ) - ); + Dictionary data = + new() + { + ["code"] = code, + ["client_id"] = prov.ClientId, + ["client_secret"] = prov.Secret, + ["redirect_uri"] = $"{options.PublicUrl.TrimEnd('/')}/api/auth/logged/{provider}", + ["grant_type"] = "authorization_code", + }; + HttpResponseMessage resp = prov.TokenUseJsonBody + ? await client.PostAsJsonAsync(prov.TokenUrl, data) + : await client.PostAsync(prov.TokenUrl, new FormUrlEncodedContent(data)); if (!resp.IsSuccessStatusCode) throw new ValidationException( $"Invalid code or configuration. {resp.StatusCode}: {await resp.Content.ReadAsStringAsync()}" @@ -71,22 +69,36 @@ public class OidcController( client.DefaultRequestHeaders.Remove("Authorization"); client.DefaultRequestHeaders.Add("Authorization", $"{token.TokenType} {token.AccessToken}"); + Dictionary? extraHeaders = prov.GetExtraHeaders?.Invoke(prov); + if (extraHeaders is not null) + { + foreach ((string key, string value) in extraHeaders) + client.DefaultRequestHeaders.Add(key, value); + } + JwtProfile? profile = await client.GetFromJsonAsync(prov.ProfileUrl); if (profile is null || profile.Sub is null) - throw new ValidationException("Missing sub on user object"); - ExternalToken extToken = new() { Id = profile.Sub, Token = token, }; + throw new ValidationException( + $"Missing sub on user object. Got: {JsonSerializer.Serialize(profile)}" + ); + ExternalToken extToken = + new() + { + Id = profile.Sub, + Token = token, + ProfileUrl = prov.GetProfileUrl?.Invoke(profile), + }; User newUser = new(); if (profile.Email is not null) newUser.Email = profile.Email; - string? username = profile.Username ?? profile.Name; - if (username is null) + if (profile.Username is null) { throw new ValidationException( $"Could not find a username for the user. You may need to add more scopes. Fields: {string.Join(',', profile.Extra)}" ); } - extToken.Username = username; - newUser.Username = username; + extToken.Username = profile.Username; + newUser.Username = profile.Username; newUser.Slug = Utils.Utility.ToSlug(newUser.Username); newUser.ExternalId.Add(provider, extToken); return (newUser, extToken); diff --git a/back/src/Kyoo.Authentication/Models/DTO/JwtProfile.cs b/back/src/Kyoo.Authentication/Models/DTO/JwtProfile.cs index cba47a5e..85272f8b 100644 --- a/back/src/Kyoo.Authentication/Models/DTO/JwtProfile.cs +++ b/back/src/Kyoo.Authentication/Models/DTO/JwtProfile.cs @@ -17,30 +17,57 @@ // along with Kyoo. If not, see . using System.Collections.Generic; +using System.Text.Json.Nodes; using System.Text.Json.Serialization; namespace Kyoo.Authentication.Models.DTO; public class JwtProfile { - public string Sub { get; set; } - public string Uid + public string? Sub { get; set; } + public string? Uid { - set => Sub = value; + set => Sub ??= value; } - public string Id + public string? Id { - set => Sub = value; + set => Sub ??= value; } - public string Guid + public string? Guid { - set => Sub = value; + set => Sub ??= value; } - public string? Name { get; set; } public string? Username { get; set; } + public string? Name + { + set => Username ??= value; + } + public string? Email { get; set; } + public JsonObject? Account + { + set + { + if (value is null) + return; + // simkl store their ids there. + Sub ??= value["id"]?.ToString(); + } + } + + public JsonObject? User + { + set + { + if (value is null) + return; + // simkl store their name there. + Username ??= value["name"]?.ToString(); + } + } + [JsonExtensionData] public Dictionary Extra { get; set; } } diff --git a/back/src/Kyoo.Authentication/Models/Options/PermissionOption.cs b/back/src/Kyoo.Authentication/Models/Options/PermissionOption.cs index 5dd06ecd..cd41c0bb 100644 --- a/back/src/Kyoo.Authentication/Models/Options/PermissionOption.cs +++ b/back/src/Kyoo.Authentication/Models/Options/PermissionOption.cs @@ -20,6 +20,7 @@ using System; using System.Collections.Generic; using System.Linq; using Kyoo.Abstractions.Models.Permissions; +using Kyoo.Authentication.Models.DTO; namespace Kyoo.Authentication.Models; @@ -72,11 +73,20 @@ public class OidcProvider public string? LogoUrl { get; set; } public string AuthorizationUrl { get; set; } public string TokenUrl { get; set; } + + /// + /// Some token endpoints do net respect the spec and require a json body instead of a form url encoded. + /// + public bool TokenUseJsonBody { get; set; } + public string ProfileUrl { get; set; } public string? Scope { get; set; } public string ClientId { get; set; } public string Secret { get; set; } + public Func? GetProfileUrl { get; init; } + public Func>? GetExtraHeaders { get; init; } + public bool Enabled => AuthorizationUrl != null && TokenUrl != null @@ -97,6 +107,9 @@ public class OidcProvider Scope = KnownProviders[provider].Scope; ClientId = KnownProviders[provider].ClientId; Secret = KnownProviders[provider].Secret; + TokenUseJsonBody = KnownProviders[provider].TokenUseJsonBody; + GetProfileUrl = KnownProviders[provider].GetProfileUrl; + GetExtraHeaders = KnownProviders[provider].GetExtraHeaders; } } @@ -120,6 +133,20 @@ public class OidcProvider TokenUrl = "https://discord.com/api/oauth2/token", ProfileUrl = "https://discord.com/api/users/@me", Scope = "email+identify", - } + }, + ["simkl"] = new("simkl") + { + DisplayName = "Simkl", + LogoUrl = "https://logo.clearbit.com/simkl.com", + AuthorizationUrl = "https://simkl.com/oauth/authorize", + TokenUrl = "https://api.simkl.com/oauth/token", + ProfileUrl = "https://api.simkl.com/users/settings", + // does not seems to have scopes + Scope = null, + TokenUseJsonBody = true, + GetProfileUrl = (profile) => $"https://simkl.com/{profile.Sub}/dashboard/", + GetExtraHeaders = (OidcProvider self) => + new() { ["simkl-api-key"] = self.ClientId }, + }, }; } diff --git a/front/packages/ui/src/settings/oidc.tsx b/front/packages/ui/src/settings/oidc.tsx index 8ee5bbd0..7d75180a 100644 --- a/front/packages/ui/src/settings/oidc.tsx +++ b/front/packages/ui/src/settings/oidc.tsx @@ -101,7 +101,6 @@ export const OidcSettings = () => { text={t("settings.oidc.link")} as={Link} href={x.link} - target="_blank" {...css({ minWidth: rem(6) })} /> )} From 6937a982d48ca8ce24948fe3b88cd144a1958bf2 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Wed, 20 Mar 2024 00:29:13 +0100 Subject: [PATCH 09/26] Add autosync rabbitmq consumer --- autosync/.gitignore | 1 + autosync/Dockerfile | 8 ++++++++ autosync/autosync/__init__.py | 35 +++++++++++++++++++++++++++++++++++ autosync/autosync/__main__.py | 4 ++++ autosync/pyproject.toml | 2 ++ autosync/requirements.txt | 1 + shell.nix | 1 + 7 files changed, 52 insertions(+) create mode 100644 autosync/.gitignore create mode 100644 autosync/Dockerfile create mode 100644 autosync/autosync/__init__.py create mode 100644 autosync/autosync/__main__.py create mode 100644 autosync/pyproject.toml create mode 100644 autosync/requirements.txt diff --git a/autosync/.gitignore b/autosync/.gitignore new file mode 100644 index 00000000..bee8a64b --- /dev/null +++ b/autosync/.gitignore @@ -0,0 +1 @@ +__pycache__ diff --git a/autosync/Dockerfile b/autosync/Dockerfile new file mode 100644 index 00000000..5cde2bfe --- /dev/null +++ b/autosync/Dockerfile @@ -0,0 +1,8 @@ +FROM python:3.12 +WORKDIR /app + +COPY ./requirements.txt . +RUN pip3 install -r ./requirements.txt + +COPY . . +ENTRYPOINT ["python3", "-m", "autosync"] diff --git a/autosync/autosync/__init__.py b/autosync/autosync/__init__.py new file mode 100644 index 00000000..c0017d04 --- /dev/null +++ b/autosync/autosync/__init__.py @@ -0,0 +1,35 @@ +import os +import pika +from pika import spec +from pika.adapters.blocking_connection import BlockingChannel +import pika.credentials + + +def callback( + ch: BlockingChannel, + method: spec.Basic.Deliver, + properties: spec.BasicProperties, + body: bytes, +): + print(f" [x] {method.routing_key}:{body}") + + +def main(): + connection = pika.BlockingConnection( + pika.ConnectionParameters( + host=os.environ.get("RABBITMQ_HOST", "rabbitmq"), + credentials=pika.credentials.PlainCredentials( + os.environ.get("RABBITMQ_DEFAULT_USER", "guest"), + os.environ.get("RABBITMQ_DEFAULT_PASS", "guest"), + ), + ) + ) + channel = connection.channel() + + channel.exchange_declare(exchange="events.watched", exchange_type="topic") + result = channel.queue_declare("", exclusive=True) + queue_name = result.method.queue + channel.queue_bind(exchange="events.watched", queue=queue_name, routing_key="#") + + channel.basic_consume(queue=queue_name, on_message_callback=callback, auto_ack=True) + channel.start_consuming() diff --git a/autosync/autosync/__main__.py b/autosync/autosync/__main__.py new file mode 100644 index 00000000..85d7bc52 --- /dev/null +++ b/autosync/autosync/__main__.py @@ -0,0 +1,4 @@ +#!/usr/bin/env python +import autosync + +autosync.main() diff --git a/autosync/pyproject.toml b/autosync/pyproject.toml new file mode 100644 index 00000000..84e5d38b --- /dev/null +++ b/autosync/pyproject.toml @@ -0,0 +1,2 @@ +[tool.ruff.format] +indent-style = "tab" diff --git a/autosync/requirements.txt b/autosync/requirements.txt new file mode 100644 index 00000000..df7f4230 --- /dev/null +++ b/autosync/requirements.txt @@ -0,0 +1 @@ +pika diff --git a/shell.nix b/shell.nix index 09055872..8f4f17a6 100644 --- a/shell.nix +++ b/shell.nix @@ -5,6 +5,7 @@ aiohttp jsons watchfiles + pika ]); dotnet = with pkgs.dotnetCorePackages; combinePackages [ From 1f3a985d3ab0bae8c580f6824eae256f4832806c Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Wed, 20 Mar 2024 19:34:55 +0100 Subject: [PATCH 10/26] Fix watch status message payload --- .../Controllers/IWatchStatusRepository.cs | 12 ++-- .../Models/Resources/WatchStatus.cs | 57 +++++++++++++++---- .../Repositories/WatchStatusRepository.cs | 54 +++++++++++++++--- back/src/Kyoo.RabbitMq/Message.cs | 4 +- back/src/Kyoo.RabbitMq/RabbitProducer.cs | 20 +++---- 5 files changed, 107 insertions(+), 40 deletions(-) diff --git a/back/src/Kyoo.Abstractions/Controllers/IWatchStatusRepository.cs b/back/src/Kyoo.Abstractions/Controllers/IWatchStatusRepository.cs index a4372d23..17ebe40c 100644 --- a/back/src/Kyoo.Abstractions/Controllers/IWatchStatusRepository.cs +++ b/back/src/Kyoo.Abstractions/Controllers/IWatchStatusRepository.cs @@ -47,8 +47,8 @@ public interface IWatchStatusRepository int? percent ); - static event ResourceEventHandler OnMovieStatusChangedHandler; - protected static Task OnMovieStatusChanged(MovieWatchStatus obj) => + static event ResourceEventHandler> OnMovieStatusChangedHandler; + protected static Task OnMovieStatusChanged(WatchStatus obj) => OnMovieStatusChangedHandler?.Invoke(obj) ?? Task.CompletedTask; Task DeleteMovieStatus(Guid movieId, Guid userId); @@ -57,8 +57,8 @@ public interface IWatchStatusRepository Task SetShowStatus(Guid showId, Guid userId, WatchStatus status); - static event ResourceEventHandler OnShowStatusChangedHandler; - protected static Task OnShowStatusChanged(ShowWatchStatus obj) => + static event ResourceEventHandler> OnShowStatusChangedHandler; + protected static Task OnShowStatusChanged(WatchStatus obj) => OnShowStatusChangedHandler?.Invoke(obj) ?? Task.CompletedTask; Task DeleteShowStatus(Guid showId, Guid userId); @@ -75,8 +75,8 @@ public interface IWatchStatusRepository int? percent ); - static event ResourceEventHandler OnEpisodeStatusChangedHandler; - protected static Task OnEpisodeStatusChanged(EpisodeWatchStatus obj) => + static event ResourceEventHandler> OnEpisodeStatusChangedHandler; + protected static Task OnEpisodeStatusChanged(WatchStatus obj) => OnEpisodeStatusChangedHandler?.Invoke(obj) ?? Task.CompletedTask; Task DeleteEpisodeStatus(Guid episodeId, Guid userId); diff --git a/back/src/Kyoo.Abstractions/Models/Resources/WatchStatus.cs b/back/src/Kyoo.Abstractions/Models/Resources/WatchStatus.cs index 1caf380c..d6576f5e 100644 --- a/back/src/Kyoo.Abstractions/Models/Resources/WatchStatus.cs +++ b/back/src/Kyoo.Abstractions/Models/Resources/WatchStatus.cs @@ -53,20 +53,11 @@ public enum WatchStatus Deleted, } -public interface IWatchStatus -{ - /// - /// Has the user started watching, is it planned? - /// - public WatchStatus Status { get; set; } -} - - /// /// Metadata of what an user as started/planned to watch. /// [SqlFirstColumn(nameof(UserId))] -public class MovieWatchStatus : IAddedDate, IWatchStatus +public class MovieWatchStatus : IAddedDate { /// /// The ID of the user that started watching this episode. @@ -121,7 +112,7 @@ public class MovieWatchStatus : IAddedDate, IWatchStatus } [SqlFirstColumn(nameof(UserId))] -public class EpisodeWatchStatus : IAddedDate, IWatchStatus +public class EpisodeWatchStatus : IAddedDate { /// /// The ID of the user that started watching this episode. @@ -176,7 +167,7 @@ public class EpisodeWatchStatus : IAddedDate, IWatchStatus } [SqlFirstColumn(nameof(UserId))] -public class ShowWatchStatus : IAddedDate, IWatchStatus +public class ShowWatchStatus : IAddedDate { /// /// The ID of the user that started watching this episode. @@ -244,3 +235,45 @@ public class ShowWatchStatus : IAddedDate, IWatchStatus /// public int? WatchedPercent { get; set; } } + +public class WatchStatus : IAddedDate +{ + /// + /// Has the user started watching, is it planned? + /// + public required WatchStatus Status { get; set; } + + /// + public DateTime AddedDate { get; set; } + + /// + /// The date at which this item was played. + /// + public DateTime? PlayedDate { get; set; } + + /// + /// Where the player has stopped watching the episode (in seconds). + /// + /// + /// Null if the status is not Watching or if the next episode is not started. + /// + public int? WatchedTime { get; set; } + + /// + /// Where the player has stopped watching the episode (in percentage between 0 and 100). + /// + /// + /// Null if the status is not Watching or if the next episode is not started. + /// + public int? WatchedPercent { get; set; } + + /// + /// The user that started watching this episode. + /// + public required User User { get; set; } + + /// + /// The episode/show/movie whose status changed + /// + public required T Resource { get; set; } +} diff --git a/back/src/Kyoo.Core/Controllers/Repositories/WatchStatusRepository.cs b/back/src/Kyoo.Core/Controllers/Repositories/WatchStatusRepository.cs index d8a72070..17a3db8e 100644 --- a/back/src/Kyoo.Core/Controllers/Repositories/WatchStatusRepository.cs +++ b/back/src/Kyoo.Core/Controllers/Repositories/WatchStatusRepository.cs @@ -36,6 +36,9 @@ namespace Kyoo.Core.Controllers; public class WatchStatusRepository( DatabaseContext database, IRepository movies, + IRepository shows, + IRepository episodes, + IRepository users, DbConnection db, SqlVariableContext context ) : IWatchStatusRepository @@ -265,7 +268,18 @@ public class WatchStatusRepository( .MovieWatchStatus.Upsert(ret) .UpdateIf(x => status != Watching || x.Status != Completed) .RunAsync(); - await IWatchStatusRepository.OnMovieStatusChanged(ret); + await IWatchStatusRepository.OnMovieStatusChanged( + new() + { + User = await users.Get(ret.UserId), + Resource = await movies.Get(ret.MovieId), + Status = ret.Status, + WatchedTime = ret.WatchedTime, + WatchedPercent = ret.WatchedPercent, + AddedDate = ret.AddedDate, + PlayedDate = ret.PlayedDate, + } + ); return ret; } @@ -278,8 +292,8 @@ public class WatchStatusRepository( await IWatchStatusRepository.OnMovieStatusChanged( new() { - UserId = userId, - MovieId = movieId, + User = await users.Get(userId), + Resource = await movies.Get(movieId), AddedDate = DateTime.UtcNow, Status = WatchStatus.Deleted, } @@ -413,7 +427,18 @@ public class WatchStatusRepository( .ShowWatchStatus.Upsert(ret) .UpdateIf(x => status != Watching || x.Status != Completed || newEpisode) .RunAsync(); - await IWatchStatusRepository.OnShowStatusChanged(ret); + await IWatchStatusRepository.OnShowStatusChanged( + new() + { + User = await users.Get(ret.UserId), + Resource = await shows.Get(ret.ShowId), + Status = ret.Status, + WatchedTime = ret.WatchedTime, + WatchedPercent = ret.WatchedPercent, + AddedDate = ret.AddedDate, + PlayedDate = ret.PlayedDate, + } + ); return ret; } @@ -430,8 +455,8 @@ public class WatchStatusRepository( await IWatchStatusRepository.OnShowStatusChanged( new() { - UserId = userId, - ShowId = showId, + User = await users.Get(userId), + Resource = await shows.Get(showId), AddedDate = DateTime.UtcNow, Status = WatchStatus.Deleted, } @@ -495,7 +520,18 @@ public class WatchStatusRepository( .EpisodeWatchStatus.Upsert(ret) .UpdateIf(x => status != Watching || x.Status != Completed) .RunAsync(); - await IWatchStatusRepository.OnEpisodeStatusChanged(ret); + await IWatchStatusRepository.OnEpisodeStatusChanged( + new() + { + User = await users.Get(ret.UserId), + Resource = await episodes.Get(ret.EpisodeId), + Status = ret.Status, + WatchedTime = ret.WatchedTime, + WatchedPercent = ret.WatchedPercent, + AddedDate = ret.AddedDate, + PlayedDate = ret.PlayedDate, + } + ); await SetShowStatus(episode.ShowId, userId, WatchStatus.Watching); return ret; } @@ -509,8 +545,8 @@ public class WatchStatusRepository( await IWatchStatusRepository.OnEpisodeStatusChanged( new() { - UserId = userId, - EpisodeId = episodeId, + User = await users.Get(userId), + Resource = await episodes.Get(episodeId), AddedDate = DateTime.UtcNow, Status = WatchStatus.Deleted, } diff --git a/back/src/Kyoo.RabbitMq/Message.cs b/back/src/Kyoo.RabbitMq/Message.cs index a1bbba9b..67e12760 100644 --- a/back/src/Kyoo.RabbitMq/Message.cs +++ b/back/src/Kyoo.RabbitMq/Message.cs @@ -21,11 +21,11 @@ using System.Text.Json; namespace Kyoo.RabbitMq; -public class Message +public class Message { public string Action { get; set; } public string Type { get; set; } - public object Value { get; set; } + public T Value { get; set; } public string AsRoutingKey() { diff --git a/back/src/Kyoo.RabbitMq/RabbitProducer.cs b/back/src/Kyoo.RabbitMq/RabbitProducer.cs index d04ffff7..f3809dfd 100644 --- a/back/src/Kyoo.RabbitMq/RabbitProducer.cs +++ b/back/src/Kyoo.RabbitMq/RabbitProducer.cs @@ -40,14 +40,11 @@ public class RabbitProducer _ListenResourceEvents("events.resource"); _channel.ExchangeDeclare("events.watched", ExchangeType.Topic); - IWatchStatusRepository.OnMovieStatusChangedHandler += _PublishWatchStatus( - "movie" + IWatchStatusRepository.OnMovieStatusChangedHandler += _PublishWatchStatus("movie"); + IWatchStatusRepository.OnShowStatusChangedHandler += _PublishWatchStatus("show"); + IWatchStatusRepository.OnEpisodeStatusChangedHandler += _PublishWatchStatus( + "episode" ); - IWatchStatusRepository.OnShowStatusChangedHandler += _PublishWatchStatus( - "show" - ); - IWatchStatusRepository.OnEpisodeStatusChangedHandler += - _PublishWatchStatus("episode"); } private void _ListenResourceEvents(string exchange) @@ -69,7 +66,7 @@ public class RabbitProducer { return (T resource) => { - Message message = + Message message = new() { Action = action, @@ -85,12 +82,13 @@ public class RabbitProducer }; } - private IWatchStatusRepository.ResourceEventHandler _PublishWatchStatus(string resource) - where T : IWatchStatus + private IWatchStatusRepository.ResourceEventHandler> _PublishWatchStatus( + string resource + ) { return (status) => { - Message message = + Message> message = new() { Type = resource, From 31d8dcd6a8203ac0347c2713d6dc8d8ac7400926 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Wed, 20 Mar 2024 19:35:18 +0100 Subject: [PATCH 11/26] Add models in the autosync module --- autosync/autosync/models/episode.py | 10 ++++++++ autosync/autosync/models/metadataid.py | 8 +++++++ autosync/autosync/models/movie.py | 17 ++++++++++++++ autosync/autosync/models/show.py | 17 ++++++++++++++ autosync/autosync/models/user.py | 30 ++++++++++++++++++++++++ autosync/autosync/models/watch_status.py | 21 +++++++++++++++++ 6 files changed, 103 insertions(+) create mode 100644 autosync/autosync/models/episode.py create mode 100644 autosync/autosync/models/metadataid.py create mode 100644 autosync/autosync/models/movie.py create mode 100644 autosync/autosync/models/show.py create mode 100644 autosync/autosync/models/user.py create mode 100644 autosync/autosync/models/watch_status.py diff --git a/autosync/autosync/models/episode.py b/autosync/autosync/models/episode.py new file mode 100644 index 00000000..047344a6 --- /dev/null +++ b/autosync/autosync/models/episode.py @@ -0,0 +1,10 @@ +from typing import Literal +from dataclasses import dataclass + +from .metadataid import MetadataID + + +@dataclass +class Episode: + external_id: dict[str, MetadataID] + kind: Literal["episode"] diff --git a/autosync/autosync/models/metadataid.py b/autosync/autosync/models/metadataid.py new file mode 100644 index 00000000..87bfc915 --- /dev/null +++ b/autosync/autosync/models/metadataid.py @@ -0,0 +1,8 @@ +from dataclasses import dataclass +from typing import Optional + + +@dataclass +class MetadataID: + data_id: str + link: Optional[str] diff --git a/autosync/autosync/models/movie.py b/autosync/autosync/models/movie.py new file mode 100644 index 00000000..6d886868 --- /dev/null +++ b/autosync/autosync/models/movie.py @@ -0,0 +1,17 @@ +from typing import Literal, Optional +from datetime import date +from dataclasses import dataclass + +from .metadataid import MetadataID + + +@dataclass +class Movie: + name: str + air_date: Optional[date] + external_id: dict[str, MetadataID] + kind: Literal["movie"] + + @property + def year(self): + return self.air_date.year if self.air_date is not None else None diff --git a/autosync/autosync/models/show.py b/autosync/autosync/models/show.py new file mode 100644 index 00000000..364dd5b3 --- /dev/null +++ b/autosync/autosync/models/show.py @@ -0,0 +1,17 @@ +from typing import Literal, Optional +from datetime import date +from dataclasses import dataclass + +from .metadataid import MetadataID + + +@dataclass +class Show: + name: str + start_air: Optional[date] + external_id: dict[str, MetadataID] + kind: Literal["show"] + + @property + def year(self): + return self.start_air.year if self.start_air is not None else None diff --git a/autosync/autosync/models/user.py b/autosync/autosync/models/user.py new file mode 100644 index 00000000..653af8eb --- /dev/null +++ b/autosync/autosync/models/user.py @@ -0,0 +1,30 @@ +from datetime import datetime, time +from dataclasses import dataclass +from typing import Optional + + +@dataclass +class JwtToken: + token_type: str + access_token: str + refresh_token: str + expire_in: time + expire_at: datetime + + +@dataclass +class ExternalToken: + id: str + username: str + profileUrl: Optional[str] + token: JwtToken + + +@dataclass +class User: + id: str + username: str + email: str + permissions: list[str] + settings: dict[str, str] + external_id: dict[str, ExternalToken] diff --git a/autosync/autosync/models/watch_status.py b/autosync/autosync/models/watch_status.py new file mode 100644 index 00000000..5f42ab9d --- /dev/null +++ b/autosync/autosync/models/watch_status.py @@ -0,0 +1,21 @@ +from datetime import date +from dataclasses import dataclass +from typing import Optional +from enum import Enum + + +class Status(str, Enum): + COMPLETED = "completed" + WATCHING = "watching" + DROPED = "droped" + PLANNED = "planned" + DELETED = "deleted" + + +@dataclass +class WatchStatus: + added_date: date + played_date: date + status: Status + watched_time: Optional[int] + watched_percent: Optional[int] From c6f12ab2a885e89b938b1e3f3ecd78bd919297dd Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Wed, 20 Mar 2024 19:35:30 +0100 Subject: [PATCH 12/26] Add a simkl sync implementation --- autosync/autosync/services/simkl.py | 85 +++++++++++++++++++++++++++++ autosync/requirements.txt | 1 + scanner/pyproject.toml | 3 + shell.nix | 1 + 4 files changed, 90 insertions(+) create mode 100644 autosync/autosync/services/simkl.py diff --git a/autosync/autosync/services/simkl.py b/autosync/autosync/services/simkl.py new file mode 100644 index 00000000..12c18a21 --- /dev/null +++ b/autosync/autosync/services/simkl.py @@ -0,0 +1,85 @@ +import requests +import logging +from ..models.user import User +from ..models.show import Show +from ..models.movie import Movie +from ..models.episode import Episode +from ..models.watch_status import WatchStatus, Status + + +class Simkl: + def __init__(self) -> None: + self._api_key = "" + + def update(self, user: User, resource: Movie | Show | Episode, status: WatchStatus): + if "simkl" not in user.external_id: + return + + watch_date = status.played_date or status.added_date + + if resource.kind == "episode": + if status.status != Status.COMPLETED: + return + resp = requests.post( + "https://api.simkl.com/sync/history", + json={ + "episodes": { + "watched_at": watch_date, + "ids": { + service: id.data_id + for service, id in resource.external_id.items() + }, + } + }, + headers={ + "Authorization": f"Bearer {user.external_id["simkl"].token.access_token}", + "simkl_api_key": self._api_key, + }, + ) + logging.debug("Simkl response: %s", resp.json()) + return + + category = "movies" if resource.kind == "movie" else "shows" + + simkl_status = self._to_simkl_status(status.status) + if simkl_status is None: + return + + resp = requests.post( + "https://api.simkl.com/sync/add-to-list", + json={ + category: { + "to": simkl_status, + "watched_at": watch_date + if status.status == Status.COMPLETED + else None, + "title": resource.name, + "year": resource.year, + "ids": { + service: id.data_id + for service, id in resource.external_id.items() + }, + } + }, + headers={ + "Authorization": f"Bearer {user.external_id["simkl"].token.access_token}", + "simkl_api_key": self._api_key, + }, + ) + logging.debug("Simkl response: %s", resp.json()) + + def _to_simkl_status(self, status: Status): + match status: + case Status.COMPLETED: + return "completed" + case Status.WATCHING: + return "watching" + case Status.COMPLETED: + return "completed" + case Status.PLANNED: + return "plantowatch" + case Status.DELETED: + # do not delete items on simkl, most of deleted status are for a rewatch. + return None + case _: + return None diff --git a/autosync/requirements.txt b/autosync/requirements.txt index df7f4230..9f75faf9 100644 --- a/autosync/requirements.txt +++ b/autosync/requirements.txt @@ -1 +1,2 @@ pika +requets diff --git a/scanner/pyproject.toml b/scanner/pyproject.toml index 84e5d38b..ce8becbf 100644 --- a/scanner/pyproject.toml +++ b/scanner/pyproject.toml @@ -1,2 +1,5 @@ [tool.ruff.format] indent-style = "tab" + +[tool.pyright] +reportAbstractUsage = false diff --git a/shell.nix b/shell.nix index 8f4f17a6..5c7729e3 100644 --- a/shell.nix +++ b/shell.nix @@ -6,6 +6,7 @@ jsons watchfiles pika + requests ]); dotnet = with pkgs.dotnetCorePackages; combinePackages [ From e1f889f862e645afe0f65e1b34f7530d7e04f7ff Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Wed, 20 Mar 2024 20:41:15 +0100 Subject: [PATCH 13/26] Listen to watch status events to call simkl --- autosync/autosync/__init__.py | 15 ++++++++++++--- autosync/autosync/models/message.py | 20 ++++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 autosync/autosync/models/message.py diff --git a/autosync/autosync/__init__.py b/autosync/autosync/__init__.py index c0017d04..6c8cfe64 100644 --- a/autosync/autosync/__init__.py +++ b/autosync/autosync/__init__.py @@ -1,17 +1,24 @@ +import json import os import pika from pika import spec from pika.adapters.blocking_connection import BlockingChannel import pika.credentials +from autosync.services.simkl import Simkl -def callback( +# TODO: declare multiples services +service = Simkl() + + +def on_message( ch: BlockingChannel, method: spec.Basic.Deliver, properties: spec.BasicProperties, body: bytes, ): - print(f" [x] {method.routing_key}:{body}") + status = json.loads(body) + service.update(status.user, status.resource, status) def main(): @@ -31,5 +38,7 @@ def main(): queue_name = result.method.queue channel.queue_bind(exchange="events.watched", queue=queue_name, routing_key="#") - channel.basic_consume(queue=queue_name, on_message_callback=callback, auto_ack=True) + channel.basic_consume( + queue=queue_name, on_message_callback=on_message, auto_ack=True + ) channel.start_consuming() diff --git a/autosync/autosync/models/message.py b/autosync/autosync/models/message.py new file mode 100644 index 00000000..31d10779 --- /dev/null +++ b/autosync/autosync/models/message.py @@ -0,0 +1,20 @@ +from dataclasses import dataclass + +from autosync.models.episode import Episode +from autosync.models.movie import Movie +from autosync.models.show import Show +from autosync.models.user import User +from autosync.models.watch_status import WatchStatus + + +@dataclass +class WatchStatusMessage(WatchStatus): + user: User + resource: Movie | Show | Episode + + +@dataclass +class Message: + action: str + type: str + value: WatchStatusMessage From a5c7aef3b8d4b88f94db53a79cd8e265ae3db909 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Wed, 20 Mar 2024 21:04:41 +0100 Subject: [PATCH 14/26] Add service aggregates for autosync --- autosync/autosync/__init__.py | 4 ++-- autosync/autosync/services/aggregate.py | 21 +++++++++++++++++++++ autosync/autosync/services/service.py | 21 +++++++++++++++++++++ autosync/autosync/services/simkl.py | 18 +++++++++++++++--- 4 files changed, 59 insertions(+), 5 deletions(-) create mode 100644 autosync/autosync/services/aggregate.py create mode 100644 autosync/autosync/services/service.py diff --git a/autosync/autosync/__init__.py b/autosync/autosync/__init__.py index 6c8cfe64..d7e74d1f 100644 --- a/autosync/autosync/__init__.py +++ b/autosync/autosync/__init__.py @@ -4,11 +4,11 @@ import pika from pika import spec from pika.adapters.blocking_connection import BlockingChannel import pika.credentials +from autosync.services.aggregate import Aggregate from autosync.services.simkl import Simkl -# TODO: declare multiples services -service = Simkl() +service = Aggregate([Simkl()]) def on_message( diff --git a/autosync/autosync/services/aggregate.py b/autosync/autosync/services/aggregate.py new file mode 100644 index 00000000..e8771507 --- /dev/null +++ b/autosync/autosync/services/aggregate.py @@ -0,0 +1,21 @@ +import logging +from autosync.services.service import Service +from ..models.user import User +from ..models.show import Show +from ..models.movie import Movie +from ..models.episode import Episode +from ..models.watch_status import WatchStatus + + +class Aggregate(Service): + def __init__(self, services: list[Service]): + self._services = [x for x in services if x.enabled] + logging.info("Autosync enabled with %s", [x.name for x in self._services]) + + @property + def name(self) -> str: + return "aggragate" + + def update(self, user: User, resource: Movie | Show | Episode, status: WatchStatus): + for service in self._services: + service.update(user, resource, status) diff --git a/autosync/autosync/services/service.py b/autosync/autosync/services/service.py new file mode 100644 index 00000000..e6371135 --- /dev/null +++ b/autosync/autosync/services/service.py @@ -0,0 +1,21 @@ +from abc import abstractmethod, abstractproperty + +from ..models.user import User +from ..models.show import Show +from ..models.movie import Movie +from ..models.episode import Episode +from ..models.watch_status import WatchStatus + + +class Service: + @abstractproperty + def name(self) -> str: + raise NotImplementedError + + @abstractproperty + def enabled(self) -> bool: + return True + + @abstractmethod + def update(self, user: User, resource: Movie | Show | Episode, status: WatchStatus): + raise NotImplementedError diff --git a/autosync/autosync/services/simkl.py b/autosync/autosync/services/simkl.py index 12c18a21..8912e202 100644 --- a/autosync/autosync/services/simkl.py +++ b/autosync/autosync/services/simkl.py @@ -1,5 +1,9 @@ +import os +from typing_extensions import assert_type import requests import logging + +from autosync.services.service import Service from ..models.user import User from ..models.show import Show from ..models.movie import Movie @@ -7,12 +11,20 @@ from ..models.episode import Episode from ..models.watch_status import WatchStatus, Status -class Simkl: +class Simkl(Service): def __init__(self) -> None: - self._api_key = "" + self._api_key = os.environ.get("OIDC_SIMKL_CLIENTID") + + @property + def name(self) -> str: + return "simkl" + + @property + def enabled(self) -> bool: + return self._api_key is not None def update(self, user: User, resource: Movie | Show | Episode, status: WatchStatus): - if "simkl" not in user.external_id: + if "simkl" not in user.external_id or self._api_key is None: return watch_date = status.played_date or status.added_date From 22d0d064f7d3213aa294a438aaa61b3421c5e220 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Wed, 20 Mar 2024 23:12:15 +0100 Subject: [PATCH 15/26] Add rabbitmq healthchecks --- docker-compose.dev.yml | 17 +++++++++++++++++ docker-compose.prod.yml | 17 +++++++++++++++++ docker-compose.yml | 17 +++++++++++++++++ 3 files changed, 51 insertions(+) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 9750308f..13db941a 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -35,6 +35,8 @@ services: condition: service_healthy meilisearch: condition: service_healthy + rabbitmq: + condition: service_healthy volumes: - ./back:/app - /app/out/ @@ -71,6 +73,15 @@ services: volumes: - ${LIBRARY_ROOT}:/video:ro + autosync: + build: ./autosync + restart: on-failure + depends_on: + rabbitmq: + condition: service_healthy + env_file: + - ./.env + transcoder: <<: *transcoder-base profiles: [''] @@ -160,6 +171,12 @@ services: ports: - 5672:5672 - 15672:15672 + healthcheck: + test: rabbitmq-diagnostics -q ping + interval: 30s + timeout: 10s + retries: 5 + start_period: 10s volumes: kyoo: diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 562c8243..192ad16b 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -26,6 +26,8 @@ services: condition: service_healthy meilisearch: condition: service_healthy + rabbitmq: + condition: service_healthy volumes: - kyoo:/kyoo @@ -48,6 +50,15 @@ services: volumes: - ${LIBRARY_ROOT}:/video:ro + autosync: + build: ./autosync + restart: on-failure + depends_on: + rabbitmq: + condition: service_healthy + env_file: + - ./.env + transcoder: <<: *transcoder-base profiles: [''] @@ -132,6 +143,12 @@ services: - RABBITMQ_DEFAULT_PASS=${RABBITMQ_DEFAULT_PASS} ports: - 5672:5672 + healthcheck: + test: rabbitmq-diagnostics -q ping + interval: 30s + timeout: 10s + retries: 5 + start_period: 10s volumes: kyoo: diff --git a/docker-compose.yml b/docker-compose.yml index bef2c2b5..a3b62289 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,6 +25,8 @@ services: condition: service_healthy meilisearch: condition: service_healthy + rabbitmq: + condition: service_healthy volumes: - kyoo:/kyoo @@ -47,6 +49,15 @@ services: volumes: - ${LIBRARY_ROOT}:/video:ro + autosync: + build: ./autosync + restart: on-failure + depends_on: + rabbitmq: + condition: service_healthy + env_file: + - ./.env + transcoder: <<: *transcoder-base profiles: [''] @@ -131,6 +142,12 @@ services: - RABBITMQ_DEFAULT_PASS=${RABBITMQ_DEFAULT_PASS} ports: - 5672:5672 + healthcheck: + test: rabbitmq-diagnostics -q ping + interval: 30s + timeout: 10s + retries: 5 + start_period: 10s volumes: kyoo: From a8fe8e2e13e5e2c30c36c922c142886222658976 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Thu, 21 Mar 2024 00:34:11 +0100 Subject: [PATCH 16/26] Parse json messages on autosync --- autosync/autosync/__init__.py | 20 +++++++++++++++++--- autosync/autosync/models/episode.py | 3 ++- autosync/autosync/models/message.py | 3 +++ autosync/autosync/models/metadataid.py | 2 ++ autosync/autosync/models/movie.py | 2 ++ autosync/autosync/models/show.py | 2 ++ autosync/autosync/models/user.py | 4 ++++ autosync/autosync/models/watch_status.py | 14 ++++++++------ autosync/autosync/services/aggregate.py | 7 ++++++- autosync/autosync/services/simkl.py | 1 - autosync/requirements.txt | 3 ++- shell.nix | 1 + 12 files changed, 49 insertions(+), 13 deletions(-) diff --git a/autosync/autosync/__init__.py b/autosync/autosync/__init__.py index d7e74d1f..3fcca024 100644 --- a/autosync/autosync/__init__.py +++ b/autosync/autosync/__init__.py @@ -1,13 +1,22 @@ -import json +import logging import os +import dataclasses_json import pika from pika import spec from pika.adapters.blocking_connection import BlockingChannel import pika.credentials +from datetime import date, datetime +from autosync.models.message import Message from autosync.services.aggregate import Aggregate from autosync.services.simkl import Simkl +dataclasses_json.cfg.global_config.encoders[date] = date.isoformat +dataclasses_json.cfg.global_config.decoders[date] = date.fromisoformat +dataclasses_json.cfg.global_config.encoders[datetime] = datetime.isoformat +dataclasses_json.cfg.global_config.decoders[datetime] = datetime.fromisoformat + +logging.basicConfig(level=logging.INFO) service = Aggregate([Simkl()]) @@ -17,8 +26,12 @@ def on_message( properties: spec.BasicProperties, body: bytes, ): - status = json.loads(body) - service.update(status.user, status.resource, status) + try: + status = Message.from_json(body) + service.update(status.user, status.resource, status) + except Exception as e: + logging.exception("Error processing message.", exc_info=e) + logging.exception("Body: %s", body) def main(): @@ -41,4 +54,5 @@ def main(): channel.basic_consume( queue=queue_name, on_message_callback=on_message, auto_ack=True ) + logging.info("Listening for autosync.") channel.start_consuming() diff --git a/autosync/autosync/models/episode.py b/autosync/autosync/models/episode.py index 047344a6..8cbf8c71 100644 --- a/autosync/autosync/models/episode.py +++ b/autosync/autosync/models/episode.py @@ -1,9 +1,10 @@ from typing import Literal from dataclasses import dataclass +from dataclasses_json import dataclass_json, LetterCase from .metadataid import MetadataID - +@dataclass_json(letter_case=LetterCase.CAMEL) @dataclass class Episode: external_id: dict[str, MetadataID] diff --git a/autosync/autosync/models/message.py b/autosync/autosync/models/message.py index 31d10779..94b3312e 100644 --- a/autosync/autosync/models/message.py +++ b/autosync/autosync/models/message.py @@ -1,4 +1,5 @@ from dataclasses import dataclass +from dataclasses_json import dataclass_json, LetterCase from autosync.models.episode import Episode from autosync.models.movie import Movie @@ -7,12 +8,14 @@ from autosync.models.user import User from autosync.models.watch_status import WatchStatus +@dataclass_json(letter_case=LetterCase.CAMEL) @dataclass class WatchStatusMessage(WatchStatus): user: User resource: Movie | Show | Episode +@dataclass_json(letter_case=LetterCase.CAMEL) @dataclass class Message: action: str diff --git a/autosync/autosync/models/metadataid.py b/autosync/autosync/models/metadataid.py index 87bfc915..a9ec2267 100644 --- a/autosync/autosync/models/metadataid.py +++ b/autosync/autosync/models/metadataid.py @@ -1,7 +1,9 @@ from dataclasses import dataclass +from dataclasses_json import dataclass_json, LetterCase from typing import Optional +@dataclass_json(letter_case=LetterCase.CAMEL) @dataclass class MetadataID: data_id: str diff --git a/autosync/autosync/models/movie.py b/autosync/autosync/models/movie.py index 6d886868..03b2273d 100644 --- a/autosync/autosync/models/movie.py +++ b/autosync/autosync/models/movie.py @@ -1,10 +1,12 @@ from typing import Literal, Optional from datetime import date from dataclasses import dataclass +from dataclasses_json import dataclass_json, LetterCase from .metadataid import MetadataID +@dataclass_json(letter_case=LetterCase.CAMEL) @dataclass class Movie: name: str diff --git a/autosync/autosync/models/show.py b/autosync/autosync/models/show.py index 364dd5b3..5400af70 100644 --- a/autosync/autosync/models/show.py +++ b/autosync/autosync/models/show.py @@ -1,10 +1,12 @@ from typing import Literal, Optional from datetime import date from dataclasses import dataclass +from dataclasses_json import dataclass_json, LetterCase from .metadataid import MetadataID +@dataclass_json(letter_case=LetterCase.CAMEL) @dataclass class Show: name: str diff --git a/autosync/autosync/models/user.py b/autosync/autosync/models/user.py index 653af8eb..30f18b7a 100644 --- a/autosync/autosync/models/user.py +++ b/autosync/autosync/models/user.py @@ -1,8 +1,10 @@ from datetime import datetime, time from dataclasses import dataclass +from dataclasses_json import dataclass_json, LetterCase from typing import Optional +@dataclass_json(letter_case=LetterCase.CAMEL) @dataclass class JwtToken: token_type: str @@ -12,6 +14,7 @@ class JwtToken: expire_at: datetime +@dataclass_json(letter_case=LetterCase.CAMEL) @dataclass class ExternalToken: id: str @@ -20,6 +23,7 @@ class ExternalToken: token: JwtToken +@dataclass_json(letter_case=LetterCase.CAMEL) @dataclass class User: id: str diff --git a/autosync/autosync/models/watch_status.py b/autosync/autosync/models/watch_status.py index 5f42ab9d..6bd8bc7b 100644 --- a/autosync/autosync/models/watch_status.py +++ b/autosync/autosync/models/watch_status.py @@ -1,21 +1,23 @@ from datetime import date from dataclasses import dataclass +from dataclasses_json import dataclass_json, LetterCase from typing import Optional from enum import Enum class Status(str, Enum): - COMPLETED = "completed" - WATCHING = "watching" - DROPED = "droped" - PLANNED = "planned" - DELETED = "deleted" + COMPLETED = "Completed" + WATCHING = "Watching" + DROPED = "Droped" + PLANNED = "Planned" + DELETED = "Deleted" +@dataclass_json(letter_case=LetterCase.CAMEL) @dataclass class WatchStatus: added_date: date - played_date: date + played_date: Optional[date] status: Status watched_time: Optional[int] watched_percent: Optional[int] diff --git a/autosync/autosync/services/aggregate.py b/autosync/autosync/services/aggregate.py index e8771507..6044ed46 100644 --- a/autosync/autosync/services/aggregate.py +++ b/autosync/autosync/services/aggregate.py @@ -18,4 +18,9 @@ class Aggregate(Service): def update(self, user: User, resource: Movie | Show | Episode, status: WatchStatus): for service in self._services: - service.update(user, resource, status) + try: + service.update(user, resource, status) + except Exception as e: + logging.exception( + "Unhandled error on autosync %s:", service.name, exc_info=e + ) diff --git a/autosync/autosync/services/simkl.py b/autosync/autosync/services/simkl.py index 8912e202..7cb7ad9f 100644 --- a/autosync/autosync/services/simkl.py +++ b/autosync/autosync/services/simkl.py @@ -1,5 +1,4 @@ import os -from typing_extensions import assert_type import requests import logging diff --git a/autosync/requirements.txt b/autosync/requirements.txt index 9f75faf9..0976c85c 100644 --- a/autosync/requirements.txt +++ b/autosync/requirements.txt @@ -1,2 +1,3 @@ pika -requets +requests +dataclasses-json diff --git a/shell.nix b/shell.nix index 5c7729e3..7c1b81ee 100644 --- a/shell.nix +++ b/shell.nix @@ -7,6 +7,7 @@ watchfiles pika requests + dataclasses-json ]); dotnet = with pkgs.dotnetCorePackages; combinePackages [ From 380c80bbaf2f43a82b10837506a6044cf33378f1 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Thu, 21 Mar 2024 01:14:59 +0100 Subject: [PATCH 17/26] Fix datetime parsing on autosync side --- autosync/autosync/__init__.py | 22 +++++++++++++++------- autosync/autosync/models/movie.py | 4 ++-- autosync/autosync/models/show.py | 4 ++-- autosync/autosync/models/user.py | 2 +- autosync/autosync/models/watch_status.py | 6 +++--- 5 files changed, 23 insertions(+), 15 deletions(-) diff --git a/autosync/autosync/__init__.py b/autosync/autosync/__init__.py index 3fcca024..e6e44ec2 100644 --- a/autosync/autosync/__init__.py +++ b/autosync/autosync/__init__.py @@ -1,20 +1,28 @@ import logging import os + import dataclasses_json +from datetime import datetime +from marshmallow import fields + +dataclasses_json.cfg.global_config.encoders[datetime] = datetime.isoformat +dataclasses_json.cfg.global_config.decoders[datetime] = datetime.fromisoformat +dataclasses_json.cfg.global_config.mm_fields[datetime] = fields.DateTime(format="iso") +dataclasses_json.cfg.global_config.encoders[datetime | None] = datetime.isoformat +dataclasses_json.cfg.global_config.decoders[datetime | None] = datetime.fromisoformat +dataclasses_json.cfg.global_config.mm_fields[datetime | None] = fields.DateTime( + format="iso" +) + import pika from pika import spec from pika.adapters.blocking_connection import BlockingChannel import pika.credentials -from datetime import date, datetime from autosync.models.message import Message from autosync.services.aggregate import Aggregate from autosync.services.simkl import Simkl -dataclasses_json.cfg.global_config.encoders[date] = date.isoformat -dataclasses_json.cfg.global_config.decoders[date] = date.fromisoformat -dataclasses_json.cfg.global_config.encoders[datetime] = datetime.isoformat -dataclasses_json.cfg.global_config.decoders[datetime] = datetime.fromisoformat logging.basicConfig(level=logging.INFO) service = Aggregate([Simkl()]) @@ -27,8 +35,8 @@ def on_message( body: bytes, ): try: - status = Message.from_json(body) - service.update(status.user, status.resource, status) + message = Message.from_json(body) # type: Message + service.update(message.value.user, message.value.resource, message.value) except Exception as e: logging.exception("Error processing message.", exc_info=e) logging.exception("Body: %s", body) diff --git a/autosync/autosync/models/movie.py b/autosync/autosync/models/movie.py index 03b2273d..dd109ec6 100644 --- a/autosync/autosync/models/movie.py +++ b/autosync/autosync/models/movie.py @@ -1,5 +1,5 @@ from typing import Literal, Optional -from datetime import date +from datetime import datetime from dataclasses import dataclass from dataclasses_json import dataclass_json, LetterCase @@ -10,7 +10,7 @@ from .metadataid import MetadataID @dataclass class Movie: name: str - air_date: Optional[date] + air_date: Optional[datetime] external_id: dict[str, MetadataID] kind: Literal["movie"] diff --git a/autosync/autosync/models/show.py b/autosync/autosync/models/show.py index 5400af70..261d215c 100644 --- a/autosync/autosync/models/show.py +++ b/autosync/autosync/models/show.py @@ -1,5 +1,5 @@ from typing import Literal, Optional -from datetime import date +from datetime import datetime from dataclasses import dataclass from dataclasses_json import dataclass_json, LetterCase @@ -10,7 +10,7 @@ from .metadataid import MetadataID @dataclass class Show: name: str - start_air: Optional[date] + start_air: Optional[datetime] external_id: dict[str, MetadataID] kind: Literal["show"] diff --git a/autosync/autosync/models/user.py b/autosync/autosync/models/user.py index 30f18b7a..fe393499 100644 --- a/autosync/autosync/models/user.py +++ b/autosync/autosync/models/user.py @@ -9,7 +9,7 @@ from typing import Optional class JwtToken: token_type: str access_token: str - refresh_token: str + refresh_token: Optional[str] expire_in: time expire_at: datetime diff --git a/autosync/autosync/models/watch_status.py b/autosync/autosync/models/watch_status.py index 6bd8bc7b..2e85b542 100644 --- a/autosync/autosync/models/watch_status.py +++ b/autosync/autosync/models/watch_status.py @@ -1,4 +1,4 @@ -from datetime import date +from datetime import datetime from dataclasses import dataclass from dataclasses_json import dataclass_json, LetterCase from typing import Optional @@ -16,8 +16,8 @@ class Status(str, Enum): @dataclass_json(letter_case=LetterCase.CAMEL) @dataclass class WatchStatus: - added_date: date - played_date: Optional[date] + added_date: datetime + played_date: Optional[datetime] status: Status watched_time: Optional[int] watched_percent: Optional[int] From 1e8316e16daf0d9f332a75d5b0a02a179cf3cc67 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Thu, 21 Mar 2024 01:30:38 +0100 Subject: [PATCH 18/26] Fix simkl requests --- autosync/autosync/services/simkl.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/autosync/autosync/services/simkl.py b/autosync/autosync/services/simkl.py index 7cb7ad9f..2a830110 100644 --- a/autosync/autosync/services/simkl.py +++ b/autosync/autosync/services/simkl.py @@ -35,7 +35,7 @@ class Simkl(Service): "https://api.simkl.com/sync/history", json={ "episodes": { - "watched_at": watch_date, + "watched_at": watch_date.isoformat(), "ids": { service: id.data_id for service, id in resource.external_id.items() @@ -44,10 +44,10 @@ class Simkl(Service): }, headers={ "Authorization": f"Bearer {user.external_id["simkl"].token.access_token}", - "simkl_api_key": self._api_key, + "simkl-api-key": self._api_key, }, ) - logging.debug("Simkl response: %s", resp.json()) + logging.info("Simkl response: %s %s", resp.status_code, resp.text) return category = "movies" if resource.kind == "movie" else "shows" @@ -74,10 +74,10 @@ class Simkl(Service): }, headers={ "Authorization": f"Bearer {user.external_id["simkl"].token.access_token}", - "simkl_api_key": self._api_key, + "simkl-api-key": self._api_key, }, ) - logging.debug("Simkl response: %s", resp.json()) + logging.info("Simkl response: %s %s", resp.status_code, resp.text) def _to_simkl_status(self, status: Status): match status: From fe9aa865f939bbf196412782a7ef001ad8612c0d Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Thu, 21 Mar 2024 02:38:37 +0100 Subject: [PATCH 19/26] Add shows in episode watch status change events --- .../Controllers/Repositories/WatchStatusRepository.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/back/src/Kyoo.Core/Controllers/Repositories/WatchStatusRepository.cs b/back/src/Kyoo.Core/Controllers/Repositories/WatchStatusRepository.cs index 17a3db8e..a153352a 100644 --- a/back/src/Kyoo.Core/Controllers/Repositories/WatchStatusRepository.cs +++ b/back/src/Kyoo.Core/Controllers/Repositories/WatchStatusRepository.cs @@ -524,7 +524,7 @@ public class WatchStatusRepository( new() { User = await users.Get(ret.UserId), - Resource = await episodes.Get(ret.EpisodeId), + Resource = await episodes.Get(episodeId, new(nameof(Episode.Show))), Status = ret.Status, WatchedTime = ret.WatchedTime, WatchedPercent = ret.WatchedPercent, @@ -546,7 +546,7 @@ public class WatchStatusRepository( new() { User = await users.Get(userId), - Resource = await episodes.Get(episodeId), + Resource = await episodes.Get(episodeId, new(nameof(Episode.Show))), AddedDate = DateTime.UtcNow, Status = WatchStatus.Deleted, } From 2df874e786d2054780f0e9280147d498d36ce00e Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Thu, 21 Mar 2024 02:39:04 +0100 Subject: [PATCH 20/26] Fix simkl requests body and episode identification --- autosync/autosync/models/episode.py | 6 +++ autosync/autosync/services/simkl.py | 62 +++++++++++++++++++---------- 2 files changed, 47 insertions(+), 21 deletions(-) diff --git a/autosync/autosync/models/episode.py b/autosync/autosync/models/episode.py index 8cbf8c71..f428eade 100644 --- a/autosync/autosync/models/episode.py +++ b/autosync/autosync/models/episode.py @@ -2,10 +2,16 @@ from typing import Literal from dataclasses import dataclass from dataclasses_json import dataclass_json, LetterCase +from autosync.models.show import Show + from .metadataid import MetadataID @dataclass_json(letter_case=LetterCase.CAMEL) @dataclass class Episode: external_id: dict[str, MetadataID] + show: Show + season_number: int + episode_number: int + absolute_number: int kind: Literal["episode"] diff --git a/autosync/autosync/services/simkl.py b/autosync/autosync/services/simkl.py index 2a830110..a11fed99 100644 --- a/autosync/autosync/services/simkl.py +++ b/autosync/autosync/services/simkl.py @@ -1,6 +1,7 @@ import os import requests import logging +from autosync.models.metadataid import MetadataID from autosync.services.service import Service from ..models.user import User @@ -31,16 +32,24 @@ class Simkl(Service): if resource.kind == "episode": if status.status != Status.COMPLETED: return + resp = requests.post( "https://api.simkl.com/sync/history", json={ - "episodes": { - "watched_at": watch_date.isoformat(), - "ids": { - service: id.data_id - for service, id in resource.external_id.items() - }, - } + "shows": [ + { + "watched_at": watch_date.isoformat(), + "title": resource.show.name, + "year": resource.show.year, + "ids": self._map_external_ids(resource.show.external_id), + "seasons": [ + { + "number": resource.season_number, + "episodes": [{"number": resource.episode_number}], + }, + ], + } + ] }, headers={ "Authorization": f"Bearer {user.external_id["simkl"].token.access_token}", @@ -52,25 +61,24 @@ class Simkl(Service): category = "movies" if resource.kind == "movie" else "shows" - simkl_status = self._to_simkl_status(status.status) + simkl_status = self._map_status(status.status) if simkl_status is None: return resp = requests.post( "https://api.simkl.com/sync/add-to-list", json={ - category: { - "to": simkl_status, - "watched_at": watch_date - if status.status == Status.COMPLETED - else None, - "title": resource.name, - "year": resource.year, - "ids": { - service: id.data_id - for service, id in resource.external_id.items() - }, - } + category: [ + { + "to": simkl_status, + "watched_at": watch_date.isoformat() + if status.status == Status.COMPLETED + else None, + "title": resource.name, + "year": resource.year, + "ids": self._map_external_ids(resource.external_id), + } + ] }, headers={ "Authorization": f"Bearer {user.external_id["simkl"].token.access_token}", @@ -79,7 +87,7 @@ class Simkl(Service): ) logging.info("Simkl response: %s %s", resp.status_code, resp.text) - def _to_simkl_status(self, status: Status): + def _map_status(self, status: Status): match status: case Status.COMPLETED: return "completed" @@ -94,3 +102,15 @@ class Simkl(Service): return None case _: return None + + def _map_external_ids(self, ids: dict[str, MetadataID]): + return { + # "simkl": int(ids["simkl"].data_id) if "simkl" in ids else None, + # "mal": int(ids["mal"].data_id) if "mal" in ids else None, + # "tvdb": int(ids["tvdb"].data_id) if "tvdb" in ids else None, + "imdb": ids["imdb"].data_id if "imdb" in ids else None, + # "anidb": int(ids["anidb"].data_id) if "anidb" in ids else None, + "tmdb": int(ids["themoviedatabase"].data_id) + if "themoviedatabase" in ids + else None, + } From 71bf334ac4a19e2e8a68582b2b1e894be3c145c7 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Fri, 22 Mar 2024 21:21:11 +0100 Subject: [PATCH 21/26] Handle absolute hander for simkl sync --- autosync/autosync/services/simkl.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/autosync/autosync/services/simkl.py b/autosync/autosync/services/simkl.py index a11fed99..c78319d9 100644 --- a/autosync/autosync/services/simkl.py +++ b/autosync/autosync/services/simkl.py @@ -47,6 +47,10 @@ class Simkl(Service): "number": resource.season_number, "episodes": [{"number": resource.episode_number}], }, + { + "number": 1, + "episodes": [{"number": resource.absolute_number}], + }, ], } ] @@ -104,13 +108,8 @@ class Simkl(Service): return None def _map_external_ids(self, ids: dict[str, MetadataID]): - return { - # "simkl": int(ids["simkl"].data_id) if "simkl" in ids else None, - # "mal": int(ids["mal"].data_id) if "mal" in ids else None, - # "tvdb": int(ids["tvdb"].data_id) if "tvdb" in ids else None, - "imdb": ids["imdb"].data_id if "imdb" in ids else None, - # "anidb": int(ids["anidb"].data_id) if "anidb" in ids else None, + return {service: id.data_id for service, id in ids.items()} | { "tmdb": int(ids["themoviedatabase"].data_id) if "themoviedatabase" in ids - else None, + else None } From 567d2ac686254c00a3f3ed3ee1bea5910c86587e Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sat, 23 Mar 2024 14:04:56 +0100 Subject: [PATCH 22/26] Fix sln file --- autosync/autosync/models/episode.py | 1 + back/Kyoo.sln | 15 +--- .../Serializers/JsonSerializerContract.cs | 69 ------------------- 3 files changed, 3 insertions(+), 82 deletions(-) delete mode 100644 back/src/Kyoo.Core/Views/Helper/Serializers/JsonSerializerContract.cs diff --git a/autosync/autosync/models/episode.py b/autosync/autosync/models/episode.py index f428eade..6378e148 100644 --- a/autosync/autosync/models/episode.py +++ b/autosync/autosync/models/episode.py @@ -6,6 +6,7 @@ from autosync.models.show import Show from .metadataid import MetadataID + @dataclass_json(letter_case=LetterCase.CAMEL) @dataclass class Episode: diff --git a/back/Kyoo.sln b/back/Kyoo.sln index 10bf4f8b..d8aac686 100644 --- a/back/Kyoo.sln +++ b/back/Kyoo.sln @@ -1,5 +1,5 @@ -Microsoft Visual Studio Solution File, Format Version 12.00 -# +Microsoft Visual Studio Solution File, Format Version 12.00 +# Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Kyoo.Core", "src\Kyoo.Core\Kyoo.Core.csproj", "{0F8275B6-C7DD-42DF-A168-755C81B1C329}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Kyoo.Abstractions", "src\Kyoo.Abstractions\Kyoo.Abstractions.csproj", "{BAB2CAE1-AC28-4509-AA3E-8DC75BD59220}" @@ -14,8 +14,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Kyoo.Host", "src\Kyoo.Host\ EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Kyoo.Meilisearch", "src\Kyoo.Meilisearch\Kyoo.Meilisearch.csproj", "{F8E6018A-FD51-40EB-99FF-A26BA59F2762}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{787FB205-9D7E-4946-AFE0-BD68E286F569}" -EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Kyoo.RabbitMq", "src\Kyoo.RabbitMq\Kyoo.RabbitMq.csproj", "{B97AD4A8-E6E6-41CD-87DF-5F1326FD7198}" EndProject Global @@ -64,18 +62,9 @@ Global {F8E6018A-FD51-40EB-99FF-A26BA59F2762}.Debug|Any CPU.Build.0 = Debug|Any CPU {F8E6018A-FD51-40EB-99FF-A26BA59F2762}.Release|Any CPU.ActiveCfg = Release|Any CPU {F8E6018A-FD51-40EB-99FF-A26BA59F2762}.Release|Any CPU.Build.0 = Release|Any CPU - {44F2208F-C015-4A01-8D6A-20F82437AFDB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {44F2208F-C015-4A01-8D6A-20F82437AFDB}.Debug|Any CPU.Build.0 = Debug|Any CPU - {44F2208F-C015-4A01-8D6A-20F82437AFDB}.Release|Any CPU.ActiveCfg = Release|Any CPU - {44F2208F-C015-4A01-8D6A-20F82437AFDB}.Release|Any CPU.Build.0 = Release|Any CPU {B97AD4A8-E6E6-41CD-87DF-5F1326FD7198}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {B97AD4A8-E6E6-41CD-87DF-5F1326FD7198}.Debug|Any CPU.Build.0 = Debug|Any CPU {B97AD4A8-E6E6-41CD-87DF-5F1326FD7198}.Release|Any CPU.ActiveCfg = Release|Any CPU {B97AD4A8-E6E6-41CD-87DF-5F1326FD7198}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection - GlobalSection(NestedProjects) = preSolution - {0C8AA7EA-E723-4532-852F-35AA4E8AFED5} = {FEAE1B0E-D797-470F-9030-0EF743575ECC} - {44F2208F-C015-4A01-8D6A-20F82437AFDB} = {BB39454F-53E4-4238-9659-A39638496FB3} - {B97AD4A8-E6E6-41CD-87DF-5F1326FD7198} = {787FB205-9D7E-4946-AFE0-BD68E286F569} - EndGlobalSection EndGlobal diff --git a/back/src/Kyoo.Core/Views/Helper/Serializers/JsonSerializerContract.cs b/back/src/Kyoo.Core/Views/Helper/Serializers/JsonSerializerContract.cs deleted file mode 100644 index b3f5b0c7..00000000 --- a/back/src/Kyoo.Core/Views/Helper/Serializers/JsonSerializerContract.cs +++ /dev/null @@ -1,69 +0,0 @@ -// // Kyoo - A portable and vast media library solution. -// // Copyright (c) Kyoo. -// // -// // See AUTHORS.md and LICENSE file in the project root for full license information. -// // -// // Kyoo is free software: you can redistribute it and/or modify -// // it under the terms of the GNU General Public License as published by -// // the Free Software Foundation, either version 3 of the License, or -// // any later version. -// // -// // Kyoo is distributed in the hope that it will be useful, -// // but WITHOUT ANY WARRANTY; without even the implied warranty of -// // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// // GNU General Public License for more details. -// // -// // You should have received a copy of the GNU General Public License -// // along with Kyoo. If not, see . -// -// using System; -// using System.Collections.Generic; -// using System.Linq; -// using System.Net.Http.Formatting; -// using System.Reflection; -// using Kyoo.Abstractions.Models; -// using Kyoo.Abstractions.Models.Attributes; -// using Microsoft.AspNetCore.Http; -// using static System.Text.Json.JsonNamingPolicy; -// -// namespace Kyoo.Core.Api -// { -// /// -// /// A custom json serializer that respects and -// /// . It also handle via the -// /// fields query parameter and items. -// /// -// public class JsonSerializerContract(IHttpContextAccessor? httpContextAccessor, MediaTypeFormatter formatter) -// : JsonContractResolver(formatter) -// { -// /// -// protected override JsonProperty CreateProperty( -// MemberInfo member, -// MemberSerialization memberSerialization -// ) -// { -// JsonProperty property = base.CreateProperty(member, memberSerialization); -// -// LoadableRelationAttribute? relation = -// member.GetCustomAttribute(); -// if (relation != null) -// { -// if (httpContextAccessor != null) -// { -// property.ShouldSerialize = _ => -// { -// if ( -// httpContextAccessor.HttpContext!.Items["fields"] -// is not ICollection fields -// ) -// return false; -// return fields.Contains(member.Name); -// }; -// } -// else -// property.ShouldSerialize = _ => true; -// } -// return property; -// } -// } -// } From 851baa030c70dd8fa61056cf0e069d423165b831 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sat, 23 Mar 2024 14:10:07 +0100 Subject: [PATCH 23/26] Remove deprecated version value on docker-composes --- .github/workflows/coding-style.yml | 5 +---- docker-compose.dev.yml | 2 -- docker-compose.prod.yml | 2 -- docker-compose.yml | 2 -- 4 files changed, 1 insertion(+), 10 deletions(-) diff --git a/.github/workflows/coding-style.yml b/.github/workflows/coding-style.yml index 7c7ddb55..25e15b0c 100644 --- a/.github/workflows/coding-style.yml +++ b/.github/workflows/coding-style.yml @@ -39,11 +39,8 @@ jobs: run: yarn lint && yarn format scanner: - name: "Lint scanner" + name: "Lint scanner/autosync" runs-on: ubuntu-latest - defaults: - run: - working-directory: ./scanner steps: - uses: actions/checkout@v4 diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 13db941a..2fd6fc39 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,5 +1,3 @@ -version: "3.8" - x-transcoder: &transcoder-base build: context: ./transcoder diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 192ad16b..f01dcabe 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -1,5 +1,3 @@ -version: "3.8" - x-transcoder: &transcoder-base image: zoriya/kyoo_transcoder:edge networks: diff --git a/docker-compose.yml b/docker-compose.yml index a3b62289..6d9abfda 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: "3.8" - x-transcoder: &transcoder-base build: ./transcoder networks: From 7905edaf24aa9b2e3d992dee39770a3afe3aa090 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sat, 23 Mar 2024 15:09:08 +0100 Subject: [PATCH 24/26] Use new json serializer in rabbitmq contetxs --- back/Dockerfile.dev | 2 +- .../Utility/JsonKindResolver.cs} | 24 ++---------------- back/src/Kyoo.Abstractions/Utility/Utility.cs | 11 ++++++++ back/src/Kyoo.Core/CoreModule.cs | 5 ++-- .../Kyoo.Core/Views/Helper/IncludeBinder.cs | 25 +++++++++++++++++++ back/src/Kyoo.RabbitMq/Message.cs | 3 ++- 6 files changed, 44 insertions(+), 26 deletions(-) rename back/src/{Kyoo.Core/Views/Helper/Serializers/WithKindResolver.cs => Kyoo.Abstractions/Utility/JsonKindResolver.cs} (75%) diff --git a/back/Dockerfile.dev b/back/Dockerfile.dev index ac2a62e9..19be75df 100644 --- a/back/Dockerfile.dev +++ b/back/Dockerfile.dev @@ -20,4 +20,4 @@ EXPOSE 5000 ENV DOTNET_USE_POLLING_FILE_WATCHER 1 # HEALTHCHECK --interval=5s CMD curl --fail http://localhost:5000/health || exit HEALTHCHECK CMD true -ENTRYPOINT ["dotnet", "watch", "run", "--no-restore", "--project", "/app/src/Kyoo.Host"] +ENTRYPOINT ["dotnet", "watch", "--non-interactive", "run", "--no-restore", "--project", "/app/src/Kyoo.Host"] diff --git a/back/src/Kyoo.Core/Views/Helper/Serializers/WithKindResolver.cs b/back/src/Kyoo.Abstractions/Utility/JsonKindResolver.cs similarity index 75% rename from back/src/Kyoo.Core/Views/Helper/Serializers/WithKindResolver.cs rename to back/src/Kyoo.Abstractions/Utility/JsonKindResolver.cs index 0cf61131..45cdfdaa 100644 --- a/back/src/Kyoo.Core/Views/Helper/Serializers/WithKindResolver.cs +++ b/back/src/Kyoo.Abstractions/Utility/JsonKindResolver.cs @@ -28,9 +28,9 @@ using Kyoo.Abstractions.Models.Attributes; using Microsoft.AspNetCore.Http; using static System.Text.Json.JsonNamingPolicy; -namespace Kyoo.Core.Api; +namespace Kyoo.Utils; -public class WithKindResolver : DefaultJsonTypeInfoResolver +public class JsonKindResolver : DefaultJsonTypeInfoResolver { public override JsonTypeInfo GetTypeInfo(Type type, JsonSerializerOptions options) { @@ -76,24 +76,4 @@ public class WithKindResolver : DefaultJsonTypeInfoResolver return jsonTypeInfo; } - - private static readonly IHttpContextAccessor _accessor = new HttpContextAccessor(); - - public static void HandleLoadableFields(JsonTypeInfo info) - { - foreach (JsonPropertyInfo prop in info.Properties) - { - object[] attributes = - prop.AttributeProvider?.GetCustomAttributes(typeof(LoadableRelationAttribute), true) - ?? Array.Empty(); - if (attributes.FirstOrDefault() is not LoadableRelationAttribute relation) - continue; - prop.ShouldSerialize = (_, _) => - { - if (_accessor?.HttpContext?.Items["fields"] is not ICollection fields) - return false; - return fields.Contains(prop.Name, StringComparer.InvariantCultureIgnoreCase); - }; - } - } } diff --git a/back/src/Kyoo.Abstractions/Utility/Utility.cs b/back/src/Kyoo.Abstractions/Utility/Utility.cs index a964d1e4..dcd3b35c 100644 --- a/back/src/Kyoo.Abstractions/Utility/Utility.cs +++ b/back/src/Kyoo.Abstractions/Utility/Utility.cs @@ -23,6 +23,8 @@ using System.Linq; using System.Linq.Expressions; using System.Reflection; using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; using System.Text.RegularExpressions; namespace Kyoo.Utils; @@ -32,6 +34,15 @@ namespace Kyoo.Utils; /// public static class Utility { + public static readonly JsonSerializerOptions JsonOptions = + new() + { + TypeInfoResolver = new JsonKindResolver(), + Converters = { new JsonStringEnumConverter() }, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + }; + + /// /// Convert a string to snake case. Stollen from /// https://github.com/efcore/EFCore.NamingConventions/blob/main/EFCore.NamingConventions/Internal/SnakeCaseNameRewriter.cs diff --git a/back/src/Kyoo.Core/CoreModule.cs b/back/src/Kyoo.Core/CoreModule.cs index e7b27081..8d683b80 100644 --- a/back/src/Kyoo.Core/CoreModule.cs +++ b/back/src/Kyoo.Core/CoreModule.cs @@ -28,6 +28,7 @@ using Kyoo.Abstractions.Controllers; using Kyoo.Abstractions.Models.Utils; using Kyoo.Core.Api; using Kyoo.Core.Controllers; +using Kyoo.Utils; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Routing; @@ -95,9 +96,9 @@ public class CoreModule : IPlugin }) .AddJsonOptions(x => { - x.JsonSerializerOptions.TypeInfoResolver = new WithKindResolver() + x.JsonSerializerOptions.TypeInfoResolver = new JsonKindResolver() { - Modifiers = { WithKindResolver.HandleLoadableFields } + Modifiers = { IncludeBinder.HandleLoadableFields } }; x.JsonSerializerOptions.Converters.Add(new JsonStringEnumConverter()); x.JsonSerializerOptions.PropertyNamingPolicy = JsonNamingPolicy.CamelCase; diff --git a/back/src/Kyoo.Core/Views/Helper/IncludeBinder.cs b/back/src/Kyoo.Core/Views/Helper/IncludeBinder.cs index 3ddab91b..fd783946 100644 --- a/back/src/Kyoo.Core/Views/Helper/IncludeBinder.cs +++ b/back/src/Kyoo.Core/Views/Helper/IncludeBinder.cs @@ -17,9 +17,14 @@ // along with Kyoo. If not, see . using System; +using System.Collections.Generic; +using System.Linq; using System.Reflection; +using System.Text.Json.Serialization.Metadata; using System.Threading.Tasks; +using Kyoo.Abstractions.Models.Attributes; using Kyoo.Abstractions.Models.Utils; +using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc.ModelBinding; using Microsoft.AspNetCore.Mvc.ModelBinding.Binders; @@ -49,6 +54,26 @@ public class IncludeBinder : IModelBinder } } + private static readonly IHttpContextAccessor _accessor = new HttpContextAccessor(); + + public static void HandleLoadableFields(JsonTypeInfo info) + { + foreach (JsonPropertyInfo prop in info.Properties) + { + object[] attributes = + prop.AttributeProvider?.GetCustomAttributes(typeof(LoadableRelationAttribute), true) + ?? []; + if (attributes.FirstOrDefault() is not LoadableRelationAttribute relation) + continue; + prop.ShouldSerialize = (_, _) => + { + if (_accessor?.HttpContext?.Items["fields"] is not ICollection fields) + return false; + return fields.Contains(prop.Name, StringComparer.InvariantCultureIgnoreCase); + }; + } + } + public class Provider : IModelBinderProvider { public IModelBinder GetBinder(ModelBinderProviderContext context) diff --git a/back/src/Kyoo.RabbitMq/Message.cs b/back/src/Kyoo.RabbitMq/Message.cs index 67e12760..72fa2f37 100644 --- a/back/src/Kyoo.RabbitMq/Message.cs +++ b/back/src/Kyoo.RabbitMq/Message.cs @@ -18,6 +18,7 @@ using System.Text; using System.Text.Json; +using Kyoo.Utils; namespace Kyoo.RabbitMq; @@ -34,6 +35,6 @@ public class Message public byte[] AsBytes() { - return Encoding.UTF8.GetBytes(JsonSerializer.Serialize(this)); + return Encoding.UTF8.GetBytes(JsonSerializer.Serialize(this, Utility.JsonOptions)); } } From f89ce7a96512f3e679101f6071afd2d323e26348 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sat, 23 Mar 2024 15:33:39 +0100 Subject: [PATCH 25/26] Disable show update rabbit message when updating an episode --- .../Repositories/WatchStatusRepository.cs | 39 ++++++++++++------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/back/src/Kyoo.Core/Controllers/Repositories/WatchStatusRepository.cs b/back/src/Kyoo.Core/Controllers/Repositories/WatchStatusRepository.cs index a153352a..40162a38 100644 --- a/back/src/Kyoo.Core/Controllers/Repositories/WatchStatusRepository.cs +++ b/back/src/Kyoo.Core/Controllers/Repositories/WatchStatusRepository.cs @@ -77,7 +77,12 @@ public class WatchStatusRepository( .Select(x => x.UserId) .ToListAsync(); foreach (Guid userId in users) - await repo._SetShowStatus(ep.ShowId, userId, WatchStatus.Watching, true); + await repo._SetShowStatus( + ep.ShowId, + userId, + WatchStatus.Watching, + newEpisode: true + ); }; } @@ -316,7 +321,8 @@ public class WatchStatusRepository( Guid showId, Guid userId, WatchStatus status, - bool newEpisode = false + bool newEpisode = false, + bool skipStatusUpdate = false ) { int unseenEpisodeCount = @@ -427,18 +433,21 @@ public class WatchStatusRepository( .ShowWatchStatus.Upsert(ret) .UpdateIf(x => status != Watching || x.Status != Completed || newEpisode) .RunAsync(); - await IWatchStatusRepository.OnShowStatusChanged( - new() - { - User = await users.Get(ret.UserId), - Resource = await shows.Get(ret.ShowId), - Status = ret.Status, - WatchedTime = ret.WatchedTime, - WatchedPercent = ret.WatchedPercent, - AddedDate = ret.AddedDate, - PlayedDate = ret.PlayedDate, - } - ); + if (!skipStatusUpdate) + { + await IWatchStatusRepository.OnShowStatusChanged( + new() + { + User = await users.Get(ret.UserId), + Resource = await shows.Get(ret.ShowId), + Status = ret.Status, + WatchedTime = ret.WatchedTime, + WatchedPercent = ret.WatchedPercent, + AddedDate = ret.AddedDate, + PlayedDate = ret.PlayedDate, + } + ); + } return ret; } @@ -532,7 +541,7 @@ public class WatchStatusRepository( PlayedDate = ret.PlayedDate, } ); - await SetShowStatus(episode.ShowId, userId, WatchStatus.Watching); + await _SetShowStatus(episode.ShowId, userId, WatchStatus.Watching, skipStatusUpdate: true); return ret; } From 8bbccd42d53ccd81ca5e9ce3aaef535f85c6ed17 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sat, 23 Mar 2024 15:34:54 +0100 Subject: [PATCH 26/26] Format code --- back/src/Kyoo.Abstractions/Utility/Utility.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/back/src/Kyoo.Abstractions/Utility/Utility.cs b/back/src/Kyoo.Abstractions/Utility/Utility.cs index dcd3b35c..422d673d 100644 --- a/back/src/Kyoo.Abstractions/Utility/Utility.cs +++ b/back/src/Kyoo.Abstractions/Utility/Utility.cs @@ -42,7 +42,6 @@ public static class Utility PropertyNamingPolicy = JsonNamingPolicy.CamelCase, }; - /// /// Convert a string to snake case. Stollen from /// https://github.com/efcore/EFCore.NamingConventions/blob/main/EFCore.NamingConventions/Internal/SnakeCaseNameRewriter.cs