From db6670f6995052cc0250a5c4993334dd3d98a570 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Tue, 26 Mar 2024 00:26:53 +0100 Subject: [PATCH 1/6] Improve caching for migrations docker file --- back/Dockerfile.migrations | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/back/Dockerfile.migrations b/back/Dockerfile.migrations index fad77418..6252f907 100644 --- a/back/Dockerfile.migrations +++ b/back/Dockerfile.migrations @@ -4,8 +4,23 @@ WORKDIR /kyoo COPY .config/dotnet-tools.json .config/dotnet-tools.json RUN dotnet tool restore + +COPY Kyoo.sln ./Kyoo.sln +COPY nuget.config ./nuget.config +COPY src/Directory.Build.props src/Directory.Build.props +COPY src/Kyoo.Authentication/Kyoo.Authentication.csproj src/Kyoo.Authentication/Kyoo.Authentication.csproj +COPY src/Kyoo.Abstractions/Kyoo.Abstractions.csproj src/Kyoo.Abstractions/Kyoo.Abstractions.csproj +COPY src/Kyoo.Core/Kyoo.Core.csproj src/Kyoo.Core/Kyoo.Core.csproj +COPY src/Kyoo.Host/Kyoo.Host.csproj src/Kyoo.Host/Kyoo.Host.csproj +COPY src/Kyoo.Postgresql/Kyoo.Postgresql.csproj src/Kyoo.Postgresql/Kyoo.Postgresql.csproj +COPY src/Kyoo.Meilisearch/Kyoo.Meilisearch.csproj src/Kyoo.Meilisearch/Kyoo.Meilisearch.csproj +COPY src/Kyoo.RabbitMq/Kyoo.RabbitMq.csproj src/Kyoo.RabbitMq/Kyoo.RabbitMq.csproj +COPY src/Kyoo.Swagger/Kyoo.Swagger.csproj src/Kyoo.Swagger/Kyoo.Swagger.csproj +RUN dotnet restore -a $TARGETARCH + COPY . . -RUN dotnet ef migrations bundle --self-contained -f -o /app/migrate -p src/Kyoo.Postgresql --verbose +RUN dotnet build +RUN dotnet ef migrations bundle --no-build --self-contained -f -o /app/migrate -p src/Kyoo.Postgresql --verbose FROM mcr.microsoft.com/dotnet/runtime-deps:8.0 COPY --from=builder /app/migrate /app/migrate From 39ce6013441a37644c74b6a146c14eeaa5f4abd8 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Tue, 26 Mar 2024 00:27:17 +0100 Subject: [PATCH 2/6] Fix postgres enum mapping --- back/src/Kyoo.Postgresql/PostgresContext.cs | 52 ++++++++++++++++---- back/src/Kyoo.Postgresql/PostgresModule.cs | 54 +++------------------ 2 files changed, 50 insertions(+), 56 deletions(-) diff --git a/back/src/Kyoo.Postgresql/PostgresContext.cs b/back/src/Kyoo.Postgresql/PostgresContext.cs index fbc5ccf1..dac43056 100644 --- a/back/src/Kyoo.Postgresql/PostgresContext.cs +++ b/back/src/Kyoo.Postgresql/PostgresContext.cs @@ -24,6 +24,12 @@ using Microsoft.AspNetCore.Http; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Query.SqlExpressions; using Npgsql; +using System.Collections.Generic; +using System.Text.RegularExpressions; +using Dapper; +using InterpolatedSql.SqlBuilders; +using Kyoo.Postgresql.Utils; +using Kyoo.Utils; namespace Kyoo.Postgresql; @@ -42,15 +48,6 @@ public class PostgresContext : DatabaseContext /// private readonly bool _skipConfigure; - // TODO: This needs ot be updated but ef-core still does not offer a way to use this. - [Obsolete] - static PostgresContext() - { - NpgsqlConnection.GlobalTypeMapper.MapEnum(); - NpgsqlConnection.GlobalTypeMapper.MapEnum(); - NpgsqlConnection.GlobalTypeMapper.MapEnum(); - } - /// /// Design time constructor (dotnet ef migrations add). Do not use /// @@ -102,11 +99,46 @@ public class PostgresContext : DatabaseContext "md5", args, nullable: true, - argumentsPropagateNullability: new[] { false }, + argumentsPropagateNullability: [false], type: args[0].Type, typeMapping: args[0].TypeMapping )); + + SqlMapper.TypeMapProvider = (type) => + { + return new CustomPropertyTypeMap( + type, + (type, name) => + { + string newName = Regex.Replace( + name, + "(^|_)([a-z])", + (match) => match.Groups[2].Value.ToUpperInvariant() + ); + // TODO: Add images handling here (name: poster_source, newName: PosterSource) should set Poster.Source + return type.GetProperty(newName)!; + } + ); + }; + SqlMapper.AddTypeHandler( + typeof(Dictionary), + new JsonTypeHandler>() + ); + SqlMapper.AddTypeHandler( + typeof(Dictionary), + new JsonTypeHandler>() + ); + SqlMapper.AddTypeHandler( + typeof(Dictionary), + new JsonTypeHandler>() + ); + SqlMapper.AddTypeHandler(typeof(List), new ListTypeHandler()); + SqlMapper.AddTypeHandler(typeof(List), new ListTypeHandler()); + SqlMapper.AddTypeHandler(typeof(Wrapper), new Wrapper.Handler()); + InterpolatedSqlBuilderOptions.DefaultOptions.ReuseIdenticalParameters = true; + InterpolatedSqlBuilderOptions.DefaultOptions.AutoFixSingleQuotes = false; + base.OnModelCreating(modelBuilder); } diff --git a/back/src/Kyoo.Postgresql/PostgresModule.cs b/back/src/Kyoo.Postgresql/PostgresModule.cs index 56c65264..59b8bbdb 100644 --- a/back/src/Kyoo.Postgresql/PostgresModule.cs +++ b/back/src/Kyoo.Postgresql/PostgresModule.cs @@ -16,16 +16,9 @@ // You should have received a copy of the GNU General Public License // along with Kyoo. If not, see . -using System; -using System.Collections.Generic; using System.Data.Common; -using System.Text.RegularExpressions; -using Dapper; -using InterpolatedSql.SqlBuilders; -using Kyoo.Abstractions.Controllers; using Kyoo.Abstractions.Models; -using Kyoo.Postgresql.Utils; -using Kyoo.Utils; +using Kyoo.Abstractions.Controllers; using Microsoft.AspNetCore.Hosting; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; @@ -43,43 +36,6 @@ public class PostgresModule(IConfiguration configuration, IWebHostEnvironment en /// public string Name => "Postgresql"; - static PostgresModule() - { - SqlMapper.TypeMapProvider = (type) => - { - return new CustomPropertyTypeMap( - type, - (type, name) => - { - string newName = Regex.Replace( - name, - "(^|_)([a-z])", - (match) => match.Groups[2].Value.ToUpperInvariant() - ); - // TODO: Add images handling here (name: poster_source, newName: PosterSource) should set Poster.Source - return type.GetProperty(newName)!; - } - ); - }; - SqlMapper.AddTypeHandler( - typeof(Dictionary), - new JsonTypeHandler>() - ); - SqlMapper.AddTypeHandler( - typeof(Dictionary), - new JsonTypeHandler>() - ); - SqlMapper.AddTypeHandler( - typeof(Dictionary), - new JsonTypeHandler>() - ); - SqlMapper.AddTypeHandler(typeof(List), new ListTypeHandler()); - SqlMapper.AddTypeHandler(typeof(List), new ListTypeHandler()); - SqlMapper.AddTypeHandler(typeof(Wrapper), new Wrapper.Handler()); - InterpolatedSqlBuilderOptions.DefaultOptions.ReuseIdenticalParameters = true; - InterpolatedSqlBuilderOptions.DefaultOptions.AutoFixSingleQuotes = false; - } - /// public void Configure(IServiceCollection services) { @@ -96,10 +52,16 @@ public class PostgresModule(IConfiguration configuration, IWebHostEnvironment en ["TIMEOUT"] = "30" }; + NpgsqlDataSourceBuilder dsBuilder = new(builder.ConnectionString); + dsBuilder.MapEnum(); + dsBuilder.MapEnum(); + dsBuilder.MapEnum(); + var dataSource = dsBuilder.Build(); + services.AddDbContext( x => { - x.UseNpgsql(builder.ConnectionString).UseProjectables(); + x.UseNpgsql(dataSource).UseProjectables(); if (environment.IsDevelopment()) x.EnableDetailedErrors().EnableSensitiveDataLogging(); }, From 13df17544c35efb4d92ffd8b928eeb4278642fd6 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Tue, 26 Mar 2024 00:27:33 +0100 Subject: [PATCH 3/6] Display the invalid account page when the refresh token is invalid --- front/packages/ui/src/errors/connection.tsx | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/front/packages/ui/src/errors/connection.tsx b/front/packages/ui/src/errors/connection.tsx index b1b17c9c..ac085672 100644 --- a/front/packages/ui/src/errors/connection.tsx +++ b/front/packages/ui/src/errors/connection.tsx @@ -52,14 +52,15 @@ export const ConnectionError = () => { ); } - if (account.isVerified) return ; - return ( - -

{t("errors.needVerification")}

- - ); + if (!account.isVerified) { + return ( + +

{t("errors.needVerification")}

+ + ); + } } return ( From 8ddf22d661118dd876ee296b423f36015c06a238 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Tue, 26 Mar 2024 00:29:04 +0100 Subject: [PATCH 4/6] Use latest tagged version instead of edge as a default tag for the docker compose --- docker-compose.prod.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index da1f18db..3fc6d61c 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -1,5 +1,5 @@ x-transcoder: &transcoder-base - image: zoriya/kyoo_transcoder:edge + image: zoriya/kyoo_transcoder:latest networks: default: aliases: @@ -14,7 +14,7 @@ x-transcoder: &transcoder-base services: back: - image: zoriya/kyoo_back:edge + image: zoriya/kyoo_back:latest restart: unless-stopped cpus: 1.5 env_file: @@ -32,7 +32,7 @@ services: - kyoo:/kyoo migrations: - image: zoriya/kyoo_migrations:edge + image: zoriya/kyoo_migrations:latest restart: "no" depends_on: postgres: @@ -41,13 +41,13 @@ services: - ./.env front: - image: zoriya/kyoo_front:edge + image: zoriya/kyoo_front:latest restart: unless-stopped environment: - KYOO_URL=${KYOO_URL:-http://back:5000} scanner: - image: zoriya/kyoo_scanner:edge + image: zoriya/kyoo_scanner:latest restart: unless-stopped depends_on: back: From ecbf1f5db58ea689352ecd81bc5ff0d2554d6b88 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Tue, 26 Mar 2024 00:53:16 +0100 Subject: [PATCH 5/6] Fix enums handling with dapper --- back/src/Kyoo.Abstractions/Models/Resources/Movie.cs | 4 ++-- back/src/Kyoo.Postgresql/PostgresModule.cs | 8 +++++--- back/src/Kyoo.Postgresql/Utils/ListTypeHandler.cs | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/back/src/Kyoo.Abstractions/Models/Resources/Movie.cs b/back/src/Kyoo.Abstractions/Models/Resources/Movie.cs index e70371b1..4e1d0282 100644 --- a/back/src/Kyoo.Abstractions/Models/Resources/Movie.cs +++ b/back/src/Kyoo.Abstractions/Models/Resources/Movie.cs @@ -79,12 +79,12 @@ public class Movie /// /// A list of tags that match this movie. /// - public string[] Tags { get; set; } = Array.Empty(); + public string[] Tags { get; set; } = []; /// /// The list of genres (themes) this show has. /// - public Genre[] Genres { get; set; } = Array.Empty(); + public List Genres { get; set; } = []; /// /// Is this show airing, not aired yet or finished? diff --git a/back/src/Kyoo.Postgresql/PostgresModule.cs b/back/src/Kyoo.Postgresql/PostgresModule.cs index 59b8bbdb..478b00d1 100644 --- a/back/src/Kyoo.Postgresql/PostgresModule.cs +++ b/back/src/Kyoo.Postgresql/PostgresModule.cs @@ -17,8 +17,8 @@ // along with Kyoo. If not, see . using System.Data.Common; -using Kyoo.Abstractions.Models; using Kyoo.Abstractions.Controllers; +using Kyoo.Abstractions.Models; using Microsoft.AspNetCore.Hosting; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; @@ -56,7 +56,7 @@ public class PostgresModule(IConfiguration configuration, IWebHostEnvironment en dsBuilder.MapEnum(); dsBuilder.MapEnum(); dsBuilder.MapEnum(); - var dataSource = dsBuilder.Build(); + NpgsqlDataSource dataSource = dsBuilder.Build(); services.AddDbContext( x => @@ -67,7 +67,9 @@ public class PostgresModule(IConfiguration configuration, IWebHostEnvironment en }, ServiceLifetime.Transient ); - services.AddTransient((_) => new NpgsqlConnection(builder.ConnectionString)); + services.AddTransient( + (services) => services.GetRequiredService().Database.GetDbConnection() + ); services.AddHealthChecks().AddDbContextCheck(); } diff --git a/back/src/Kyoo.Postgresql/Utils/ListTypeHandler.cs b/back/src/Kyoo.Postgresql/Utils/ListTypeHandler.cs index 77b88845..837528b7 100644 --- a/back/src/Kyoo.Postgresql/Utils/ListTypeHandler.cs +++ b/back/src/Kyoo.Postgresql/Utils/ListTypeHandler.cs @@ -29,7 +29,7 @@ public class ListTypeHandler : SqlMapper.TypeHandler> public override List Parse(object value) { T[] typedValue = (T[])value; // looks like Dapper did not indicate the property type to Npgsql, so it defaults to string[] (default CLR type for text[] PostgreSQL type) - return typedValue?.ToList() ?? new(); + return typedValue?.ToList() ?? []; } public override void SetValue(IDbDataParameter parameter, List? value) From 87754ae928521548207c98db730b082688007c21 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Tue, 26 Mar 2024 01:11:55 +0100 Subject: [PATCH 6/6] Organize imports --- back/src/Kyoo.Postgresql/PostgresContext.cs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/back/src/Kyoo.Postgresql/PostgresContext.cs b/back/src/Kyoo.Postgresql/PostgresContext.cs index dac43056..7b643809 100644 --- a/back/src/Kyoo.Postgresql/PostgresContext.cs +++ b/back/src/Kyoo.Postgresql/PostgresContext.cs @@ -17,19 +17,19 @@ // along with Kyoo. If not, see . using System; +using System.Collections.Generic; using System.Globalization; +using System.Text.RegularExpressions; +using Dapper; using EFCore.NamingConventions.Internal; +using InterpolatedSql.SqlBuilders; using Kyoo.Abstractions.Models; +using Kyoo.Postgresql.Utils; +using Kyoo.Utils; using Microsoft.AspNetCore.Http; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Query.SqlExpressions; using Npgsql; -using System.Collections.Generic; -using System.Text.RegularExpressions; -using Dapper; -using InterpolatedSql.SqlBuilders; -using Kyoo.Postgresql.Utils; -using Kyoo.Utils; namespace Kyoo.Postgresql; @@ -104,7 +104,6 @@ public class PostgresContext : DatabaseContext typeMapping: args[0].TypeMapping )); - SqlMapper.TypeMapProvider = (type) => { return new CustomPropertyTypeMap(