From 9a095727229d8b766a6756f922349aa4b6ee49ba Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Fri, 30 Sep 2022 12:21:10 +0900 Subject: [PATCH] Update the ci --- .env.example | 5 ++- .github/workflows/analysis.yml | 42 +++++++++++++++---- .github/workflows/coding-style.yml | 2 +- .github/workflows/docker.yml | 42 +++++++++++++++---- .github/workflows/documentation.yml | 2 +- .github/workflows/robot.yml | 29 ++++++++++--- .github/workflows/tests.yml | 27 ++++++++---- .gitignore | 4 +- back/.editorconfig | 9 ++++ back/Dockerfile | 21 +++++++++- .../Properties/launchSettings.json | 20 --------- back/src/Kyoo.Host.Generic/Application.cs | 17 ++------ back/src/Kyoo.Host.Generic/settings.json | 3 +- back/tests/robot/rest.resource | 2 +- docker-compose.dev.yml | 8 ++-- docker-compose.yml | 12 +++--- 16 files changed, 163 insertions(+), 82 deletions(-) delete mode 100644 back/src/Kyoo.Host.Console/Properties/launchSettings.json diff --git a/.env.example b/.env.example index f0ca0969..86bc1719 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,6 @@ TVDB__APIKEY= THEMOVIEDB__APIKEY= AUTHENTICATION_SECRET= -POSTGRES_USER= -POSTGRES_PASSWORD= +POSTGRES_USER=kyoousername +POSTGRES_PASSWORD=kyoopassword +POSTGRES_DB=kyooDB diff --git a/.github/workflows/analysis.yml b/.github/workflows/analysis.yml index 31716673..39e4f16a 100644 --- a/.github/workflows/analysis.yml +++ b/.github/workflows/analysis.yml @@ -1,58 +1,82 @@ name: Analysis -on: [push, pull_request, workflow_dispatch] +on: + push: + branches: + - master + - next + pull_request: + jobs: analysis: name: Static Analysis runs-on: ubuntu-latest - if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository steps: - uses: actions/checkout@v2 with: fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis + - name: Cache SonarCloud packages uses: actions/cache@v1 with: path: ~/sonar/cache key: ${{ runner.os }}-sonar restore-keys: ${{ runner.os }}-sonar + - name: Cache SonarCloud scanner id: cache-sonar-scanner uses: actions/cache@v1 with: - path: ./.sonar/scanner + path: ~/.sonar/scanner key: ${{ runner.os }}-sonar-scanner restore-keys: ${{ runner.os }}-sonar-scanner + - name: Install SonarCloud scanner if: steps.cache-sonar-scanner.outputs.cache-hit != 'true' shell: bash run: | - mkdir -p ./.sonar/scanner - dotnet tool update dotnet-sonarscanner --tool-path ./.sonar/scanner - - name: Wait for tests to run + cd back + mkdir -p ~/.sonar/scanner + dotnet tool update dotnet-sonarscanner --tool-path ~/.sonar/scanner + + - name: Wait for tests to run (Push) uses: lewagon/wait-on-check-action@master + if: github.event_name != 'pull_request' with: ref: ${{github.ref}} check-name: tests repo-token: ${{secrets.GITHUB_TOKEN}} running-workflow-name: analysis allowed-conclusions: success,skipped,cancelled,neutral,failure + - name: Wait for tests to run (PR) + uses: lewagon/wait-on-check-action@master + if: github.event_name == 'pull_request' + with: + ref: ${{github.event.pull_request.head.sha}} + check-name: tests + repo-token: ${{secrets.GITHUB_TOKEN}} + running-workflow-name: analysis + allowed-conclusions: success,skipped,cancelled,neutral,failure + - name: Download coverage report uses: dawidd6/action-download-artifact@v2 with: commit: ${{env.COMMIT_SHA}} workflow: tests.yml github_token: ${{secrets.GITHUB_TOKEN}} + - name: Build and analyze env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} shell: bash run: | + cp -r results.xml/ coverage.xml/ back/ + cd back find . -name 'coverage.opencover.xml' dotnet build-server shutdown - ./.sonar/scanner/dotnet-sonarscanner begin \ + ~/.sonar/scanner/dotnet-sonarscanner begin \ -k:"AnonymusRaccoon_Kyoo" \ -o:"anonymus-raccoon" \ -d:sonar.login="${{ secrets.SONAR_TOKEN }}" \ @@ -60,6 +84,6 @@ jobs: -d:sonar.cs.opencover.reportsPaths="**/coverage.opencover.xml" \ -d:sonar.cs.vstest.reportsPaths="**/TestOutputResults.xml" - dotnet build --no-incremental '-p:SkipTranscoder=true;SkipWebApp=true' + dotnet build --no-incremental '-p:SkipTranscoder=true' - ./.sonar/scanner/dotnet-sonarscanner end -d:sonar.login="${{ secrets.SONAR_TOKEN }}" + ~/.sonar/scanner/dotnet-sonarscanner end -d:sonar.login="${{ secrets.SONAR_TOKEN }}" diff --git a/.github/workflows/coding-style.yml b/.github/workflows/coding-style.yml index 97499167..8f3f2c9b 100644 --- a/.github/workflows/coding-style.yml +++ b/.github/workflows/coding-style.yml @@ -12,4 +12,4 @@ jobs: with: dotnet-version: 6.0.x - name: Build the app - run: dotnet build -p:CheckCodingStyle=true -p:TreatWarningsAsErrors=true '-p:SkipTranscoder=true;SkipWebApp=true' + run: cd back && dotnet build -p:CheckCodingStyle=true -p:TreatWarningsAsErrors=true '-p:SkipTranscoder=true' diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 8807730b..a868a7e3 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,37 +1,65 @@ name: Docker -on: [push, pull_request, workflow_dispatch] +on: + push: + branches: + - master + - next + pull_request: jobs: build: runs-on: ubuntu-latest - if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository + strategy: + fail-fast: false + matrix: + include: + - context: ./back + label: back + image: ghcr.io/${{github.repository_owner}}/kyoo_back + - context: ./front + label: front + image: ghcr.io/${{github.repository_owner}}/kyoo_front + name: Docker build ${{matrix.label}} steps: - uses: actions/checkout@v2 with: submodules: recursive fetch-depth: 0 + - name: Docker meta id: meta - uses: crazy-max/ghaction-docker-meta@v2 + uses: docker/metadata-action@v4 with: - images: ghcr.io/${{github.repository_owner}}/Kyoo + images: ${{matrix.image}} tags: | - type=ref,event=tag + type=edge type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + + - name: Check if a package should be pushed + run: | + echo "SHOULD_PUSH=$([ "${GITHUB_REF##*/}" == "master" ] || [ "${GITHUB_REF##*/}" == "next" ] && echo "true" || echo "false")" >> $GITHUB_ENV + - name: Set up QEMU uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 + - name: Login to DockerHub uses: docker/login-action@v1 with: registry: ghcr.io username: ${{github.repository_owner}} password: ${{secrets.GITHUB_TOKEN}} + - name: Build and push id: docker_build uses: docker/build-push-action@v2 with: - context: . - push: true + context: ${{matrix.context}} + build-args: | + VERSION=0.0.0 + push: ${{env.SHOULD_PUSH}} tags: ${{steps.meta.outputs.tags}} diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index ee26c63b..13f0149b 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -29,4 +29,4 @@ jobs: git push --force origin gh-pages env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_REPO: "github.com/AnonymusRaccoon/Kyoo" + GITHUB_REPO: "github.com/${{github.repository_owner}}/Kyoo" diff --git a/.github/workflows/robot.yml b/.github/workflows/robot.yml index 14d48199..3d5982e7 100644 --- a/.github/workflows/robot.yml +++ b/.github/workflows/robot.yml @@ -1,31 +1,48 @@ name: RobotTests -on: [push, pull_request, workflow_dispatch] +on: + push: + branches: + - master + - next + pull_request: + jobs: build: runs-on: ubuntu-latest - if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository steps: - uses: actions/checkout@v2 with: submodules: recursive fetch-depth: 0 + - name: Pull images + run: docker-compose pull + + - name: Docker cache + uses: satackey/action-docker-layer-caching@v0.0.11 + continue-on-error: true - name: Build the app run: docker-compose build - name: Start the service - run: docker-compose up -d + run: | + cp .env.example .env + docker-compose up -d - name: Perform healthchecks run: | docker-compose ps -a - wget --retry-connrefused http://localhost:8901 # /healthcheck + wget --retry-connrefused --retry-on-http-error=502 http://localhost:8901 #/api/healthcheck - name: Run robot tests run: | - pip install -r tests/robot/requirements.txt - robot -d out tests/robot/ + pip install -r back/tests/robot/requirements.txt + robot -d out back/tests/robot/ + + - name: Show logs + if: failure() + run: docker-compose logs - uses: actions/upload-artifact@v2 with: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1cbe0283..56bc72a6 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,11 +1,14 @@ name: Testing - -on: [push, pull_request] +on: + push: + branches: + - master + - next + pull_request: jobs: tests: runs-on: ubuntu-latest - if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository container: mcr.microsoft.com/dotnet/sdk:6.0 services: postgres: @@ -21,27 +24,33 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Restore dependencies - run: dotnet restore + - name: Build run: | - dotnet build --no-restore '-p:SkipWebApp=true;SkipTranscoder=true' -p:CopyLocalLockFileAssemblies=true - cp ./src/Kyoo.Abstractions/bin/Debug/net6.0/Microsoft.Extensions.DependencyInjection.Abstractions.dll ./tests/Kyoo.Tests/bin/Debug/net6.0/ + cd back + dotnet build '-p:SkipTranscoder=true' -p:CopyLocalLockFileAssemblies=true + cp ./out/bin/Kyoo.Abstractions/Debug/net6.0/Microsoft.Extensions.DependencyInjection.Abstractions.dll ./tests/Kyoo.Tests/bin/Debug/net6.0/ + - name: Test - run: dotnet test --no-build '-p:CollectCoverage=true;CoverletOutputFormat=opencover' --logger "trx;LogFileName=TestOutputResults.xml" + run: | + cd back + dotnet test --no-build '-p:CollectCoverage=true;CoverletOutputFormat=opencover' --logger "trx;LogFileName=TestOutputResults.xml" env: POSTGRES_HOST: postgres POSTGRES_USERNAME: postgres POSTGRES_PASSWORD: postgres + - name: Sanitize coverage output if: ${{ always() }} - run: sed -i "s'$(pwd)'.'" tests/Kyoo.Tests/coverage.opencover.xml + run: sed -i "s'$(pwd)/back'.'" back/tests/Kyoo.Tests/coverage.opencover.xml + - name: Upload tests results if: ${{ always() }} uses: actions/upload-artifact@v2 with: name: results.xml path: "**/TestOutputResults.xml" + - name: Upload coverage report if: ${{ always() }} uses: actions/upload-artifact@v2 diff --git a/.gitignore b/.gitignore index 6ffa6ab4..58ff38f1 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,6 @@ video .env .idea .vscode - +log.html +output.xml +report.html diff --git a/back/.editorconfig b/back/.editorconfig index 20d71183..985604f6 100644 --- a/back/.editorconfig +++ b/back/.editorconfig @@ -1,5 +1,14 @@ root = false +[*] +charset = utf-8 +end_of_line = lf +trim_trailing_whitespace = true +insert_final_newline = true +indent_style = tab +indent_size = tab +smart_tab = true + [*.cs] csharp_prefer_braces = false dotnet_diagnostic.IDE0130.severity = none diff --git a/back/Dockerfile b/back/Dockerfile index b428417f..b56244fe 100644 --- a/back/Dockerfile +++ b/back/Dockerfile @@ -6,8 +6,27 @@ RUN cmake . && make -j FROM mcr.microsoft.com/dotnet/sdk:6.0 as builder WORKDIR /kyoo + +COPY Kyoo.sln ./Kyoo.sln +COPY nuget.config ./nuget.config +COPY src/Directory.Build.props src/Directory.Build.props +COPY src/Kyoo.Authentication/Kyoo.Authentication.csproj src/Kyoo.Authentication/Kyoo.Authentication.csproj +COPY src/Kyoo.Database/Kyoo.Database.csproj src/Kyoo.Database/Kyoo.Database.csproj +COPY src/Kyoo.Host.Generic/Kyoo.Host.Generic.csproj src/Kyoo.Host.Generic/Kyoo.Host.Generic.csproj +COPY src/Kyoo.SqLite/Kyoo.SqLite.csproj src/Kyoo.SqLite/Kyoo.SqLite.csproj +COPY src/Kyoo.TheMovieDb/Kyoo.TheMovieDb.csproj src/Kyoo.TheMovieDb/Kyoo.TheMovieDb.csproj +COPY src/Kyoo.Abstractions/Kyoo.Abstractions.csproj src/Kyoo.Abstractions/Kyoo.Abstractions.csproj +COPY src/Kyoo.Core/Kyoo.Core.csproj src/Kyoo.Core/Kyoo.Core.csproj +COPY src/Kyoo.Host.Console/Kyoo.Host.Console.csproj src/Kyoo.Host.Console/Kyoo.Host.Console.csproj +COPY src/Kyoo.Postgresql/Kyoo.Postgresql.csproj src/Kyoo.Postgresql/Kyoo.Postgresql.csproj +COPY src/Kyoo.Swagger/Kyoo.Swagger.csproj src/Kyoo.Swagger/Kyoo.Swagger.csproj +COPY src/Kyoo.TheTvdb/Kyoo.TheTvdb.csproj src/Kyoo.TheTvdb/Kyoo.TheTvdb.csproj +COPY tests/Kyoo.Tests/Kyoo.Tests.csproj tests/Kyoo.Tests/Kyoo.Tests.csproj +RUN dotnet restore + COPY . . -RUN dotnet publish -c Release -o /opt/kyoo '-p:SkipTranscoder=true;CheckCodingStyle=false' src/Kyoo.Host.Console +ARG VERSION +RUN dotnet publish --no-restore -c Release -o /opt/kyoo "-p:Version=${VERSION:-"0.0.0-dev"};SkipTranscoder=true" src/Kyoo.Host.Console FROM mcr.microsoft.com/dotnet/aspnet:6.0 RUN apt-get update && apt-get install -y libavutil-dev libavcodec-dev libavformat-dev diff --git a/back/src/Kyoo.Host.Console/Properties/launchSettings.json b/back/src/Kyoo.Host.Console/Properties/launchSettings.json deleted file mode 100644 index 915c144b..00000000 --- a/back/src/Kyoo.Host.Console/Properties/launchSettings.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "profiles": { - "Console": { - "commandName": "Project", - "launchBrowser": false, - "environmentVariables": { - "KYOO_DATADIR": "./bin/KyooData" - }, - "applicationUrl": "http://localhost:5000" - }, - "Console-Browser": { - "commandName": "Project", - "launchBrowser": true, - "environmentVariables": { - "KYOO_DATADIR": "./bin/KyooData" - }, - "applicationUrl": "http://localhost:5000" - } - } -} diff --git a/back/src/Kyoo.Host.Generic/Application.cs b/back/src/Kyoo.Host.Generic/Application.cs index e9dc7d6f..62cca55e 100644 --- a/back/src/Kyoo.Host.Generic/Application.cs +++ b/back/src/Kyoo.Host.Generic/Application.cs @@ -159,18 +159,7 @@ namespace Kyoo.Host.Generic /// The current data directory. private string _SetupDataDir(string[] args) { - Dictionary registry = new(); - - if (OperatingSystem.IsWindows()) - { - object dataDir = Registry.GetValue(@"HKEY_LOCAL_MACHINE\Software\SDG\Kyoo\Settings", "DataDir", null) - ?? Registry.GetValue(@"HKEY_CURRENT_USER\Software\SDG\Kyoo\Settings", "DataDir", null); - if (dataDir is string data) - registry.Add("DataDir", data); - } - IConfiguration parsed = new ConfigurationBuilder() - .AddInMemoryCollection(registry) .AddEnvironmentVariables() .AddEnvironmentVariables("KYOO_") .AddCommandLine(args) @@ -186,8 +175,10 @@ namespace Kyoo.Host.Generic if (!File.Exists(GetConfigFile())) { - File.Copy(Path.Join(AppDomain.CurrentDomain.BaseDirectory, GetConfigFile()), - GetConfigFile()); + File.Copy( + Path.Join(AppDomain.CurrentDomain.BaseDirectory, GetConfigFile()), + GetConfigFile() + ); } return path; diff --git a/back/src/Kyoo.Host.Generic/settings.json b/back/src/Kyoo.Host.Generic/settings.json index 83f40858..842060f6 100644 --- a/back/src/Kyoo.Host.Generic/settings.json +++ b/back/src/Kyoo.Host.Generic/settings.json @@ -4,7 +4,6 @@ "pluginsPath": "plugins/", "transmuxPath": "cached/transmux", "transcodePath": "cached/transcode", - "metadataInShow": true, "metadataPath": "metadata/" }, @@ -19,7 +18,7 @@ "server": "127.0.0.1", "port": "5432", "database": "kyooDB", - "user ID": "kyoo", + "user": "kyoo", "password": "kyooPassword", "pooling": "true", "maxPoolSize": "95", diff --git a/back/tests/robot/rest.resource b/back/tests/robot/rest.resource index c4820ca3..348bbc08 100644 --- a/back/tests/robot/rest.resource +++ b/back/tests/robot/rest.resource @@ -1,4 +1,4 @@ *** Settings *** Documentation Common things to handle rest requests -Library REST http://localhost:5000 +Library REST http://localhost:8901/api diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 3894eb5f..89190494 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,7 +1,7 @@ version: "3.8" services: - kyoo: + back: build: context: ./back dockerfile: Dockerfile.dev @@ -34,18 +34,18 @@ services: - "3000:3000" restart: on-failure environment: - - KYOO_URL=http://kyoo:5000 + - KYOO_URL=http://back:5000 ingress: image: nginx restart: on-failure environment: - PORT=8901 - FRONT_URL=http://front:3000 - - BACK_URL=http://kyoo:5000 + - BACK_URL=http://back:5000 volumes: - ./nginx.conf.template:/etc/nginx/templates/kyoo.conf.template:ro depends_on: - - kyoo + - back - front ports: - "8901:8901" diff --git a/docker-compose.yml b/docker-compose.yml index 2bbbc72d..ab15b80a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: "3.8" services: - kyoo: + back: build: ./back restart: on-failure environment: @@ -10,6 +10,7 @@ services: - DATABASE__CONFIGURATIONS__POSTGRES__SERVER=postgres - DATABASE__CONFIGURATIONS__POSTGRES__USER=${POSTGRES_USER} - DATABASE__CONFIGURATIONS__POSTGRES__PASSWORD=${POSTGRES_PASSWORD} + - DATABASE__CONFIGURATIONS__POSTGRES__DATABASE=${POSTGRES_DB} - TVDB__APIKEY=${TVDB__APIKEY} - THEMOVIEDB__APIKEY=${THEMOVIEDB__APIKEY} depends_on: @@ -21,27 +22,28 @@ services: build: ./front restart: on-failure environment: - - KYOO_URL=http://kyoo:5000 + - KYOO_URL=http://back:5000 ingress: image: nginx restart: on-failure environment: - PORT=8901 - FRONT_URL=http://front:8901 - - BACK_URL=http://kyoo:5000 + - BACK_URL=http://back:5000 volumes: - ./nginx.conf.template:/etc/nginx/templates/kyoo.conf.template:ro depends_on: - - kyoo + - back - front ports: - "8901:8901" postgres: - image: "postgres" + image: postgres restart: on-failure environment: - POSTGRES_USER=${POSTGRES_USER} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - POSTGRES_DB=${POSTGRES_DB} volumes: - db:/var/lib/postgresql/data