From 928bfe787619565d0bc3eb1e65543dd285402be6 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Mon, 25 Mar 2024 22:22:56 +0100 Subject: [PATCH 1/9] Remove legacy meilisearch populate --- .../src/Kyoo.Meilisearch/MeilisearchModule.cs | 34 ++----------------- 1 file changed, 3 insertions(+), 31 deletions(-) diff --git a/back/src/Kyoo.Meilisearch/MeilisearchModule.cs b/back/src/Kyoo.Meilisearch/MeilisearchModule.cs index 03438ca8..3cbd903b 100644 --- a/back/src/Kyoo.Meilisearch/MeilisearchModule.cs +++ b/back/src/Kyoo.Meilisearch/MeilisearchModule.cs @@ -26,13 +26,11 @@ using static System.Text.Json.JsonNamingPolicy; namespace Kyoo.Meiliseach; -public class MeilisearchModule : IPlugin +public class MeilisearchModule(IConfiguration configuration) : IPlugin { /// public string Name => "Meilisearch"; - private readonly IConfiguration _configuration; - public static Dictionary IndexSettings => new() { @@ -122,11 +120,6 @@ public class MeilisearchModule : IPlugin }, }; - public MeilisearchModule(IConfiguration configuration) - { - _configuration = configuration; - } - /// /// Init meilisearch indexes. /// @@ -139,27 +132,6 @@ public class MeilisearchModule : IPlugin await _CreateIndex(client, "items", true); await _CreateIndex(client, nameof(Episode), false); await _CreateIndex(client, nameof(Studio), false); - - IndexStats info = await client.Index("items").GetStatsAsync(); - // If there is no documents in meilisearch, if a db exist and is not empty, add items to meilisearch. - if (info.NumberOfDocuments == 0) - { - ILibraryManager database = provider.GetRequiredService(); - MeiliSync search = provider.GetRequiredService(); - - // This is a naive implementation that absolutly does not care about performances. - // This will run only once on users that already had a database when they upgrade. - foreach (Movie movie in await database.Movies.GetAll(limit: 0)) - await search.CreateOrUpdate("items", movie, nameof(Movie)); - foreach (Show show in await database.Shows.GetAll(limit: 0)) - await search.CreateOrUpdate("items", show, nameof(Show)); - foreach (Collection collection in await database.Collections.GetAll(limit: 0)) - await search.CreateOrUpdate("items", collection, nameof(Collection)); - foreach (Episode episode in await database.Episodes.GetAll(limit: 0)) - await search.CreateOrUpdate(nameof(Episode), episode); - foreach (Studio studio in await database.Studios.GetAll(limit: 0)) - await search.CreateOrUpdate(nameof(Studio), studio); - } } private static async Task _CreateIndex(MeilisearchClient client, string index, bool hasKind) @@ -178,8 +150,8 @@ public class MeilisearchModule : IPlugin builder .RegisterInstance( new MeilisearchClient( - _configuration.GetValue("MEILI_HOST", "http://meilisearch:7700"), - _configuration.GetValue("MEILI_MASTER_KEY") + configuration.GetValue("MEILI_HOST", "http://meilisearch:7700"), + configuration.GetValue("MEILI_MASTER_KEY") ) ) .SingleInstance(); From fb1c006cd98d57fb216e12bbdd26056c69a7f322 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Mon, 25 Mar 2024 22:24:00 +0100 Subject: [PATCH 2/9] Remove postgres migrations handled by the api --- back/src/Kyoo.Host/Application.cs | 1 - back/src/Kyoo.Postgresql/PostgresModule.cs | 48 ++++------------------ 2 files changed, 8 insertions(+), 41 deletions(-) diff --git a/back/src/Kyoo.Host/Application.cs b/back/src/Kyoo.Host/Application.cs index 2647e758..0d090919 100644 --- a/back/src/Kyoo.Host/Application.cs +++ b/back/src/Kyoo.Host/Application.cs @@ -103,7 +103,6 @@ public class Application await using (AsyncServiceScope scope = host.Services.CreateAsyncScope()) { - PostgresModule.Initialize(scope.ServiceProvider); await MeilisearchModule.Initialize(scope.ServiceProvider); } diff --git a/back/src/Kyoo.Postgresql/PostgresModule.cs b/back/src/Kyoo.Postgresql/PostgresModule.cs index e3286c9b..56c65264 100644 --- a/back/src/Kyoo.Postgresql/PostgresModule.cs +++ b/back/src/Kyoo.Postgresql/PostgresModule.cs @@ -38,45 +38,13 @@ namespace Kyoo.Postgresql; /// /// A module to add postgresql capacity to the app. /// -public class PostgresModule : IPlugin +public class PostgresModule(IConfiguration configuration, IWebHostEnvironment environment) : IPlugin { /// public string Name => "Postgresql"; - /// - /// The configuration to use. The database connection string is pulled from it. - /// - private readonly IConfiguration _configuration; - - /// - /// The host environment to check if the app is in debug mode. - /// - private readonly IWebHostEnvironment _environment; - - /// - /// Create a new postgres module instance and use the given configuration and environment. - /// - /// The configuration to use - /// The environment that will be used (if the env is in development mode, more information will be displayed on errors. - public PostgresModule(IConfiguration configuration, IWebHostEnvironment env) + static PostgresModule() { - _configuration = configuration; - _environment = env; - } - - /// - /// Migrate the database. - /// - /// The service list to retrieve the database context - public static void Initialize(IServiceProvider provider) - { - DatabaseContext context = provider.GetRequiredService(); - context.Database.Migrate(); - - using NpgsqlConnection conn = (NpgsqlConnection)context.Database.GetDbConnection(); - conn.Open(); - conn.ReloadTypes(); - SqlMapper.TypeMapProvider = (type) => { return new CustomPropertyTypeMap( @@ -118,11 +86,11 @@ public class PostgresModule : IPlugin DbConnectionStringBuilder builder = new() { - ["USER ID"] = _configuration.GetValue("POSTGRES_USER", "KyooUser"), - ["PASSWORD"] = _configuration.GetValue("POSTGRES_PASSWORD", "KyooPassword"), - ["SERVER"] = _configuration.GetValue("POSTGRES_SERVER", "db"), - ["PORT"] = _configuration.GetValue("POSTGRES_PORT", "5432"), - ["DATABASE"] = _configuration.GetValue("POSTGRES_DB", "kyooDB"), + ["USER ID"] = configuration.GetValue("POSTGRES_USER", "KyooUser"), + ["PASSWORD"] = configuration.GetValue("POSTGRES_PASSWORD", "KyooPassword"), + ["SERVER"] = configuration.GetValue("POSTGRES_SERVER", "db"), + ["PORT"] = configuration.GetValue("POSTGRES_PORT", "5432"), + ["DATABASE"] = configuration.GetValue("POSTGRES_DB", "kyooDB"), ["POOLING"] = "true", ["MAXPOOLSIZE"] = "95", ["TIMEOUT"] = "30" @@ -132,7 +100,7 @@ public class PostgresModule : IPlugin x => { x.UseNpgsql(builder.ConnectionString).UseProjectables(); - if (_environment.IsDevelopment()) + if (environment.IsDevelopment()) x.EnableDetailedErrors().EnableSensitiveDataLogging(); }, ServiceLifetime.Transient From 034a554bf4a5a6b6cb09406b0cacaaf62e9a0be7 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Mon, 25 Mar 2024 22:24:22 +0100 Subject: [PATCH 3/9] Update dotnet tools --- back/.config/dotnet-tools.json | 2 +- back/.dockerignore | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/back/.config/dotnet-tools.json b/back/.config/dotnet-tools.json index 4a8f05ad..8fe65408 100644 --- a/back/.config/dotnet-tools.json +++ b/back/.config/dotnet-tools.json @@ -3,7 +3,7 @@ "isRoot": true, "tools": { "dotnet-ef": { - "version": "7.0.9", + "version": "8.0.3", "commands": [ "dotnet-ef" ] diff --git a/back/.dockerignore b/back/.dockerignore index 23880f5d..9f8aece3 100644 --- a/back/.dockerignore +++ b/back/.dockerignore @@ -1,18 +1,17 @@ Dockerfile Dockerfile.dev +Dockerfile.* .dockerignore .gitignore docker-compose.yml README.md **/build **/dist -src/Kyoo.WebApp/Front/nodes_modules **/bin **/obj out docs tests -!tests/Kyoo.Tests/Kyoo.Tests.csproj front video nginx.conf.template From 9740a5d0d49823a476c4b4dfac351e2de33310a4 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Mon, 25 Mar 2024 22:25:18 +0100 Subject: [PATCH 4/9] Remove custom dotnet output path --- back/src/Directory.Build.props | 5 ----- 1 file changed, 5 deletions(-) diff --git a/back/src/Directory.Build.props b/back/src/Directory.Build.props index 34530a3a..081a7d9b 100644 --- a/back/src/Directory.Build.props +++ b/back/src/Directory.Build.props @@ -23,11 +23,6 @@ true - - $(MsBuildThisFileDirectory)/../out/obj/$(MSBuildProjectName) - $(MsBuildThisFileDirectory)/../out/bin/$(MSBuildProjectName) - - true From d24d18ea8e04bd431aecc46dda31c89bececabbf Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Mon, 25 Mar 2024 22:25:31 +0100 Subject: [PATCH 5/9] Create a migration dockerfile --- back/Dockerfile.migrations | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 back/Dockerfile.migrations diff --git a/back/Dockerfile.migrations b/back/Dockerfile.migrations new file mode 100644 index 00000000..63dfe3ca --- /dev/null +++ b/back/Dockerfile.migrations @@ -0,0 +1,12 @@ +FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:8.0 as builder +ARG TARGETARCH +WORKDIR /kyoo + +COPY .config/dotnet-tools.json .config/dotnet-tools.json +RUN dotnet tool restore +COPY . . +RUN dotnet ef migrations bundle --self-contained -f -o /app/migrate -p src/Kyoo.Postgresql --verbose + +FROM alpine +COPY --from=builder /app /app +ENTRYPOINT ["/app/migrate"] From 646df0f393cb5357d1a2d5086e0c720a50114c45 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Mon, 25 Mar 2024 22:25:41 +0100 Subject: [PATCH 6/9] Add a migration service on the docker compose --- docker-compose.dev.yml | 10 ++++++++++ docker-compose.prod.yml | 8 ++++++++ docker-compose.yml | 10 ++++++++++ 3 files changed, 28 insertions(+) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 2fd6fc39..795cbe53 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -35,11 +35,21 @@ services: condition: service_healthy rabbitmq: condition: service_healthy + migrations: + condition: service_completed_successfully volumes: - ./back:/app - /app/out/ - kyoo:/kyoo + migrations: + build: + context: ./back + dockerfile: Dockerfile.migrations + restart: "no" + env_file: + - ./.env + front: build: context: ./front diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index f01dcabe..a117720c 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -26,9 +26,17 @@ services: condition: service_healthy rabbitmq: condition: service_healthy + migrations: + condition: service_completed_successfully volumes: - kyoo:/kyoo + migrations: + image: zoriya/kyoo_migrations:edge + restart: "no" + env_file: + - ./.env + front: image: zoriya/kyoo_front:edge restart: unless-stopped diff --git a/docker-compose.yml b/docker-compose.yml index 6d9abfda..99f94cf2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,9 +25,19 @@ services: condition: service_healthy rabbitmq: condition: service_healthy + migrations: + condition: service_completed_successfully volumes: - kyoo:/kyoo + migrations: + build: + context: ./back + dockerfile: Dockerfile.migrations + restart: "no" + env_file: + - ./.env + front: build: ./front restart: on-failure From ea979d9663b1cff1270abf43bde08328fe260b87 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Mon, 25 Mar 2024 23:41:00 +0100 Subject: [PATCH 7/9] Handle connection string for migrations --- back/Dockerfile | 2 +- back/Dockerfile.migrations | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/back/Dockerfile b/back/Dockerfile index aee9f87e..29cccc4d 100644 --- a/back/Dockerfile +++ b/back/Dockerfile @@ -19,7 +19,7 @@ COPY . . ARG VERSION RUN dotnet publish -a $TARGETARCH --no-restore -c Release -o /app "-p:Version=${VERSION:-"0.0.0-dev"}" src/Kyoo.Host -FROM mcr.microsoft.com/dotnet/aspnet:7.0 +FROM mcr.microsoft.com/dotnet/aspnet:8.0 RUN apt-get update && apt-get install -y curl COPY --from=builder /app /app diff --git a/back/Dockerfile.migrations b/back/Dockerfile.migrations index 63dfe3ca..fad77418 100644 --- a/back/Dockerfile.migrations +++ b/back/Dockerfile.migrations @@ -7,6 +7,7 @@ RUN dotnet tool restore COPY . . RUN dotnet ef migrations bundle --self-contained -f -o /app/migrate -p src/Kyoo.Postgresql --verbose -FROM alpine -COPY --from=builder /app /app -ENTRYPOINT ["/app/migrate"] +FROM mcr.microsoft.com/dotnet/runtime-deps:8.0 +COPY --from=builder /app/migrate /app/migrate + +ENTRYPOINT /app/migrate --connection "USER ID=${POSTGRES_USER};PASSWORD=${POSTGRES_PASSWORD};SERVER=postgres;PORT=5432;DATABASE=${POSTGRES_DB};" From f6bb77a6a57bdb5862449fdfd084590baf50fc8b Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Mon, 25 Mar 2024 23:47:58 +0100 Subject: [PATCH 8/9] Add postgres dependency on migration service --- docker-compose.dev.yml | 3 +++ docker-compose.prod.yml | 3 +++ docker-compose.yml | 3 +++ 3 files changed, 9 insertions(+) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 795cbe53..1c344dde 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -47,6 +47,9 @@ services: context: ./back dockerfile: Dockerfile.migrations restart: "no" + depends_on: + postgres: + condition: service_healthy env_file: - ./.env diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index a117720c..da1f18db 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -34,6 +34,9 @@ services: migrations: image: zoriya/kyoo_migrations:edge restart: "no" + depends_on: + postgres: + condition: service_healthy env_file: - ./.env diff --git a/docker-compose.yml b/docker-compose.yml index 99f94cf2..33f5e343 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,6 +35,9 @@ services: context: ./back dockerfile: Dockerfile.migrations restart: "no" + depends_on: + postgres: + condition: service_healthy env_file: - ./.env From da594d6df1dbadd600a17c1c6898e3edda96fc51 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Tue, 26 Mar 2024 00:02:30 +0100 Subject: [PATCH 9/9] Add migrations on the docker build ci --- .github/workflows/docker.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4bf970ec..d10d51a4 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -17,18 +17,32 @@ jobs: matrix: include: - context: ./back + dockerfile: Dockerfile label: back image: zoriya/kyoo_back + + - context: ./back + dockerfile: Dockerfile.migrations + label: migrations + image: zoriya/kyoo_migrations + - context: ./front + dockerfile: Dockerfile label: front image: zoriya/kyoo_front + - context: ./scanner + dockerfile: Dockerfile label: scanner image: zoriya/kyoo_scanner + - context: ./autosync + dockerfile: Dockerfile label: autosync image: zoriya/kyoo_autosync + - context: ./transcoder + dockerfile: Dockerfile label: transcoder image: zoriya/kyoo_transcoder name: Build ${{matrix.label}} @@ -77,6 +91,7 @@ jobs: if: steps.filter.outputs.should_run == 'true' || github.event_name == 'workflow_dispatch' with: context: ${{matrix.context}} + file: ${{matrix.context}}/${{matrix.dockerfile}} platforms: linux/amd64,linux/arm64 build-args: | VERSION=0.0.0