diff --git a/.github/workflows/build-mobile.yml b/.github/workflows/build-mobile.yml index 406e8f89e1..7217b5267e 100644 --- a/.github/workflows/build-mobile.yml +++ b/.github/workflows/build-mobile.yml @@ -7,6 +7,15 @@ on: ref: required: false type: string + secrets: + KEY_JKS: + required: true + ALIAS: + required: true + ANDROID_KEY_PASSWORD: + required: true + ANDROID_STORE_PASSWORD: + required: true pull_request: push: branches: [main] @@ -15,14 +24,21 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: {} + jobs: pre-job: runs-on: ubuntu-latest + permissions: + contents: read outputs: should_run: ${{ steps.found_paths.outputs.mobile == 'true' || steps.should_force.outputs.should_force == 'true' }} steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false + - id: found_paths uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 with: @@ -38,22 +54,17 @@ jobs: build-sign-android: name: Build and sign Android needs: pre-job + permissions: + contents: read # Skip when PR from a fork if: ${{ !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]' && needs.pre-job.outputs.should_run == 'true' }} runs-on: macos-14 steps: - - name: Determine ref - id: get-ref - run: | - input_ref="${{ inputs.ref }}" - github_ref="${{ github.sha }}" - ref="${input_ref:-$github_ref}" - echo "ref=$ref" >> $GITHUB_OUTPUT - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: - ref: ${{ steps.get-ref.outputs.ref }} + ref: ${{ inputs.ref || github.sha }} + persist-credentials: false - uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4 with: diff --git a/.github/workflows/cache-cleanup.yml b/.github/workflows/cache-cleanup.yml index 0cc73c46c3..84adde08cf 100644 --- a/.github/workflows/cache-cleanup.yml +++ b/.github/workflows/cache-cleanup.yml @@ -8,31 +8,38 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: {} + jobs: cleanup: name: Cleanup runs-on: ubuntu-latest + permissions: + contents: read + actions: write steps: - name: Check out code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Cleanup + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REF: ${{ github.ref }} run: | gh extension install actions/gh-actions-cache REPO=${{ github.repository }} - BRANCH=${{ github.ref }} echo "Fetching list of cache keys" - cacheKeysForPR=$(gh actions-cache list -R $REPO -B $BRANCH -L 100 | cut -f 1 ) + cacheKeysForPR=$(gh actions-cache list -R $REPO -B ${REF} -L 100 | cut -f 1 ) ## Setting this to not fail the workflow while deleting cache keys. set +e echo "Deleting caches..." for cacheKey in $cacheKeysForPR do - gh actions-cache delete $cacheKey -R $REPO -B $BRANCH --confirm + gh actions-cache delete $cacheKey -R "$REPO" -B "${REF}" --confirm done echo "Done" - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.yml index 05326d063e..231ad141e4 100644 --- a/.github/workflows/cli.yml +++ b/.github/workflows/cli.yml @@ -16,19 +16,23 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true -permissions: - packages: write +permissions: {} jobs: publish: name: CLI Publish runs-on: ubuntu-latest + permissions: + contents: read defaults: run: working-directory: ./cli steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false + # Setup .npmrc file to publish to npm - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: @@ -48,11 +52,16 @@ jobs: docker: name: Docker runs-on: ubuntu-latest + permissions: + contents: read + packages: write needs: publish steps: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Set up QEMU uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 99ffee2e88..7d07785de7 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -24,6 +24,8 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: {} + jobs: analyze: name: Analyze @@ -43,6 +45,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d1bdb5e8e7..a78d3c25dc 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -12,18 +12,21 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true -permissions: - packages: write +permissions: {} jobs: pre-job: runs-on: ubuntu-latest + permissions: + contents: read outputs: should_run_server: ${{ steps.found_paths.outputs.server == 'true' || steps.should_force.outputs.should_force == 'true' }} should_run_ml: ${{ steps.found_paths.outputs.machine-learning == 'true' || steps.should_force.outputs.should_force == 'true' }} steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - id: found_paths uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 with: @@ -45,6 +48,9 @@ jobs: retag_ml: name: Re-Tag ML needs: pre-job + permissions: + contents: read + packages: write if: ${{ needs.pre-job.outputs.should_run_ml == 'false' && !github.event.pull_request.head.repo.fork }} runs-on: ubuntu-latest strategy: @@ -58,18 +64,22 @@ jobs: username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Re-tag image + env: + REGISTRY_NAME: 'ghcr.io' + REPOSITORY: ${{ github.repository_owner }}/immich-machine-learning + TAG_OLD: main${{ matrix.suffix }} + TAG_PR: ${{ github.event.number == 0 && github.ref_name || format('pr-{0}', github.event.number) }}${{ matrix.suffix }} + TAG_COMMIT: commit-${{ github.event_name != 'pull_request' && github.sha || github.event.pull_request.head.sha }}${{ matrix.suffix }} run: | - REGISTRY_NAME="ghcr.io" - REPOSITORY=${{ github.repository_owner }}/immich-machine-learning - TAG_OLD=main${{ matrix.suffix }} - TAG_PR=${{ github.event.number == 0 && github.ref_name || format('pr-{0}', github.event.number) }}${{ matrix.suffix }} - TAG_COMMIT=commit-${{ github.event_name != 'pull_request' && github.sha || github.event.pull_request.head.sha }}${{ matrix.suffix }} - docker buildx imagetools create -t $REGISTRY_NAME/$REPOSITORY:$TAG_PR $REGISTRY_NAME/$REPOSITORY:$TAG_OLD - docker buildx imagetools create -t $REGISTRY_NAME/$REPOSITORY:$TAG_COMMIT $REGISTRY_NAME/$REPOSITORY:$TAG_OLD + docker buildx imagetools create -t "${REGISTRY_NAME}/${REPOSITORY}:${TAG_PR}" "${REGISTRY_NAME}/${REPOSITORY}:${TAG_OLD}" + docker buildx imagetools create -t "${REGISTRY_NAME}/${REPOSITORY}:${TAG_COMMIT}" "${REGISTRY_NAME}/${REPOSITORY}:${TAG_OLD}" retag_server: name: Re-Tag Server needs: pre-job + permissions: + contents: read + packages: write if: ${{ needs.pre-job.outputs.should_run_server == 'false' && !github.event.pull_request.head.repo.fork }} runs-on: ubuntu-latest strategy: @@ -83,18 +93,22 @@ jobs: username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Re-tag image + env: + REGISTRY_NAME: 'ghcr.io' + REPOSITORY: ${{ github.repository_owner }}/immich-server + TAG_OLD: main${{ matrix.suffix }} + TAG_PR: ${{ github.event.number == 0 && github.ref_name || format('pr-{0}', github.event.number) }}${{ matrix.suffix }} + TAG_COMMIT: commit-${{ github.event_name != 'pull_request' && github.sha || github.event.pull_request.head.sha }}${{ matrix.suffix }} run: | - REGISTRY_NAME="ghcr.io" - REPOSITORY=${{ github.repository_owner }}/immich-server - TAG_OLD=main${{ matrix.suffix }} - TAG_PR=${{ github.event.number == 0 && github.ref_name || format('pr-{0}', github.event.number) }}${{ matrix.suffix }} - TAG_COMMIT=commit-${{ github.event_name != 'pull_request' && github.sha || github.event.pull_request.head.sha }}${{ matrix.suffix }} - docker buildx imagetools create -t $REGISTRY_NAME/$REPOSITORY:$TAG_PR $REGISTRY_NAME/$REPOSITORY:$TAG_OLD - docker buildx imagetools create -t $REGISTRY_NAME/$REPOSITORY:$TAG_COMMIT $REGISTRY_NAME/$REPOSITORY:$TAG_OLD + docker buildx imagetools create -t "${REGISTRY_NAME}/${REPOSITORY}:${TAG_PR}" "${REGISTRY_NAME}/${REPOSITORY}:${TAG_OLD}" + docker buildx imagetools create -t "${REGISTRY_NAME}/${REPOSITORY}:${TAG_COMMIT}" "${REGISTRY_NAME}/${REPOSITORY}:${TAG_OLD}" build_and_push_ml: name: Build and Push ML needs: pre-job + permissions: + contents: read + packages: write if: ${{ needs.pre-job.outputs.should_run_ml == 'true' }} runs-on: ${{ matrix.runner }} env: @@ -148,6 +162,8 @@ jobs: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Set up Docker Buildx uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 @@ -161,11 +177,14 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Generate cache key suffix + env: + REF: ${{ github.ref_name }} run: | if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo "CACHE_KEY_SUFFIX=pr-${{ github.event.number }}" >> $GITHUB_ENV else - echo "CACHE_KEY_SUFFIX=$(echo ${{ github.ref_name }} | sed 's/[^a-zA-Z0-9]/-/g')" >> $GITHUB_ENV + SUFFIX=$(echo "${REF}" | sed 's/[^a-zA-Z0-9]/-/g') + echo "CACHE_KEY_SUFFIX=${SUFFIX}" >> $GITHUB_ENV fi - name: Generate cache target @@ -175,7 +194,7 @@ jobs: # Essentially just ignore the cache output (forks can't write to registry cache) echo "cache-to=type=local,dest=/tmp/discard,ignore-error=true" >> $GITHUB_OUTPUT else - echo "cache-to=type=registry,ref=${{ env.GHCR_REPO }}-build-cache:${{ env.PLATFORM_PAIR }}-${{ matrix.device }}-${{ env.CACHE_KEY_SUFFIX }},mode=max,compression=zstd" >> $GITHUB_OUTPUT + echo "cache-to=type=registry,ref=${GHCR_REPO}-build-cache:${PLATFORM_PAIR}-${{ matrix.device }}-${CACHE_KEY_SUFFIX},mode=max,compression=zstd" >> $GITHUB_OUTPUT fi - name: Generate docker image tags @@ -221,6 +240,10 @@ jobs: merge_ml: name: Merge & Push ML runs-on: ubuntu-latest + permissions: + contents: read + actions: read + packages: write if: ${{ needs.pre-job.outputs.should_run_ml == 'true' && !github.event.pull_request.head.repo.fork }} env: GHCR_REPO: ghcr.io/${{ github.repository_owner }}/immich-machine-learning @@ -308,15 +331,16 @@ jobs: fi TAGS=$(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - SOURCE_ARGS=$(printf '${{ env.GHCR_REPO }}@sha256:%s ' *) - - echo "docker buildx imagetools create $TAGS "${ANNOTATIONS[@]}" $SOURCE_ARGS" + SOURCE_ARGS=$(printf "${GHCR_REPO}@sha256:%s " *) docker buildx imagetools create $TAGS "${ANNOTATIONS[@]}" $SOURCE_ARGS build_and_push_server: name: Build and Push Server runs-on: ${{ matrix.runner }} + permissions: + contents: read + packages: write needs: pre-job if: ${{ needs.pre-job.outputs.should_run_server == 'true' }} env: @@ -340,6 +364,8 @@ jobs: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Set up Docker Buildx uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3 @@ -353,11 +379,14 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Generate cache key suffix + env: + REF: ${{ github.ref_name }} run: | if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo "CACHE_KEY_SUFFIX=pr-${{ github.event.number }}" >> $GITHUB_ENV else - echo "CACHE_KEY_SUFFIX=$(echo ${{ github.ref_name }} | sed 's/[^a-zA-Z0-9]/-/g')" >> $GITHUB_ENV + SUFFIX=$(echo "${REF}" | sed 's/[^a-zA-Z0-9]/-/g') + echo "CACHE_KEY_SUFFIX=${SUFFIX}" >> $GITHUB_ENV fi - name: Generate cache target @@ -367,7 +396,7 @@ jobs: # Essentially just ignore the cache output (forks can't write to registry cache) echo "cache-to=type=local,dest=/tmp/discard,ignore-error=true" >> $GITHUB_OUTPUT else - echo "cache-to=type=registry,ref=${{ env.GHCR_REPO }}-build-cache:${{ env.PLATFORM_PAIR }}-${{ env.CACHE_KEY_SUFFIX }},mode=max,compression=zstd" >> $GITHUB_OUTPUT + echo "cache-to=type=registry,ref=${GHCR_REPO}-build-cache:${PLATFORM_PAIR}-${CACHE_KEY_SUFFIX},mode=max,compression=zstd" >> $GITHUB_OUTPUT fi - name: Generate docker image tags @@ -413,6 +442,10 @@ jobs: merge_server: name: Merge & Push Server runs-on: ubuntu-latest + permissions: + contents: read + actions: read + packages: write if: ${{ needs.pre-job.outputs.should_run_server == 'true' && !github.event.pull_request.head.repo.fork }} env: GHCR_REPO: ghcr.io/${{ github.repository_owner }}/immich-server @@ -486,15 +519,14 @@ jobs: fi TAGS=$(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - SOURCE_ARGS=$(printf '${{ env.GHCR_REPO }}@sha256:%s ' *) - - echo "docker buildx imagetools create $TAGS "${ANNOTATIONS[@]}" $SOURCE_ARGS" + SOURCE_ARGS=$(printf "${GHCR_REPO}@sha256:%s " *) docker buildx imagetools create $TAGS "${ANNOTATIONS[@]}" $SOURCE_ARGS success-check-server: name: Docker Build & Push Server Success needs: [merge_server, retag_server] + permissions: {} runs-on: ubuntu-latest if: always() steps: @@ -508,6 +540,7 @@ jobs: success-check-ml: name: Docker Build & Push ML Success needs: [merge_ml, retag_ml] + permissions: {} runs-on: ubuntu-latest if: always() steps: diff --git a/.github/workflows/docs-build.yml b/.github/workflows/docs-build.yml index fdd30034ee..ece3bbd248 100644 --- a/.github/workflows/docs-build.yml +++ b/.github/workflows/docs-build.yml @@ -10,14 +10,20 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: {} + jobs: pre-job: runs-on: ubuntu-latest + permissions: + contents: read outputs: should_run: ${{ steps.found_paths.outputs.docs == 'true' || steps.should_force.outputs.should_force == 'true' }} steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - id: found_paths uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 with: @@ -33,6 +39,8 @@ jobs: build: name: Docs Build needs: pre-job + permissions: + contents: read if: ${{ needs.pre-job.outputs.should_run == 'true' }} runs-on: ubuntu-latest defaults: @@ -42,6 +50,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Setup Node uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 diff --git a/.github/workflows/docs-deploy.yml b/.github/workflows/docs-deploy.yml index f33c0c4c03..10277a0c5e 100644 --- a/.github/workflows/docs-deploy.yml +++ b/.github/workflows/docs-deploy.yml @@ -9,6 +9,9 @@ jobs: checks: name: Docs Deploy Checks runs-on: ubuntu-latest + permissions: + actions: read + pull-requests: read outputs: parameters: ${{ steps.parameters.outputs.result }} artifact: ${{ steps.get-artifact.outputs.result }} @@ -36,6 +39,8 @@ jobs: - name: Determine deploy parameters id: parameters uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 + env: + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} with: script: | const eventType = context.payload.workflow_run.event; @@ -57,7 +62,8 @@ jobs: } else if (eventType == "pull_request") { let pull_number = context.payload.workflow_run.pull_requests[0]?.number; if(!pull_number) { - const response = await github.rest.search.issuesAndPullRequests({q: 'repo:${{ github.repository }} is:pr sha:${{ github.event.workflow_run.head_sha }}',per_page: 1,}) + const {HEAD_SHA} = process.env; + const response = await github.rest.search.issuesAndPullRequests({q: `repo:${{ github.repository }} is:pr sha:${HEAD_SHA}`,per_page: 1,}) const items = response.data.items if (items.length < 1) { throw new Error("No pull request found for the commit") @@ -95,10 +101,16 @@ jobs: name: Docs Deploy runs-on: ubuntu-latest needs: checks + permissions: + contents: read + actions: read + pull-requests: write if: ${{ fromJson(needs.checks.outputs.artifact).found && fromJson(needs.checks.outputs.parameters).shouldDeploy }} steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Load parameters id: parameters @@ -162,9 +174,11 @@ jobs: - name: Output Cleaning id: clean + env: + TG_OUTPUT: ${{ steps.docs-output.outputs.tg_action_output }} run: | - TG_OUT=$(echo '${{ steps.docs-output.outputs.tg_action_output }}' | sed 's|%0A|\n|g ; s|%3C|<|g' | jq -c .) - echo "output=$TG_OUT" >> $GITHUB_OUTPUT + CLEANED=$(echo "$TG_OUTPUT" | sed 's|%0A|\n|g ; s|%3C|<|g' | jq -c .) + echo "output=$CLEANED" >> $GITHUB_OUTPUT - name: Publish to Cloudflare Pages uses: cloudflare/pages-action@f0a1cd58cd66095dee69bfa18fa5efd1dde93bca # v1 diff --git a/.github/workflows/docs-destroy.yml b/.github/workflows/docs-destroy.yml index 99499528b6..9d1e4b6612 100644 --- a/.github/workflows/docs-destroy.yml +++ b/.github/workflows/docs-destroy.yml @@ -3,13 +3,20 @@ on: pull_request_target: types: [closed] +permissions: {} + jobs: deploy: name: Docs Destroy runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Destroy Docs Subdomain env: diff --git a/.github/workflows/fix-format.yml b/.github/workflows/fix-format.yml index 9c52691a52..77b86cb0b8 100644 --- a/.github/workflows/fix-format.yml +++ b/.github/workflows/fix-format.yml @@ -4,11 +4,14 @@ on: pull_request: types: [labeled] +permissions: {} + jobs: fix-formatting: runs-on: ubuntu-latest if: ${{ github.event.label.name == 'fix:formatting' }} permissions: + contents: write pull-requests: write steps: - name: Generate a token @@ -23,6 +26,7 @@ jobs: with: ref: ${{ github.event.pull_request.head.ref }} token: ${{ steps.generate-token.outputs.token }} + persist-credentials: true - name: Setup Node uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 diff --git a/.github/workflows/pr-label-validation.yml b/.github/workflows/pr-label-validation.yml index 247c625a96..8d34597a08 100644 --- a/.github/workflows/pr-label-validation.yml +++ b/.github/workflows/pr-label-validation.yml @@ -4,6 +4,8 @@ on: pull_request_target: types: [opened, labeled, unlabeled, synchronize] +permissions: {} + jobs: validate-release-label: runs-on: ubuntu-latest diff --git a/.github/workflows/pr-labeler.yml b/.github/workflows/pr-labeler.yml index 1b43c89889..5704f4275f 100644 --- a/.github/workflows/pr-labeler.yml +++ b/.github/workflows/pr-labeler.yml @@ -2,6 +2,8 @@ name: 'Pull Request Labeler' on: - pull_request_target +permissions: {} + jobs: labeler: permissions: diff --git a/.github/workflows/pr-require-conventional-commit.yml b/.github/workflows/pr-require-conventional-commit.yml index 20dd0492f4..78ba77495c 100644 --- a/.github/workflows/pr-require-conventional-commit.yml +++ b/.github/workflows/pr-require-conventional-commit.yml @@ -4,9 +4,13 @@ on: pull_request: types: [opened, synchronize, reopened, edited] +permissions: {} + jobs: validate-pr-title: runs-on: ubuntu-latest + permissions: + pull-requests: write steps: - name: PR Conventional Commit Validation uses: ytanikin/PRConventionalCommits@b628c5a234cc32513014b7bfdd1e47b532124d98 # 1.3.0 diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml index ffb24d8952..7971f7574a 100644 --- a/.github/workflows/prepare-release.yml +++ b/.github/workflows/prepare-release.yml @@ -21,13 +21,14 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }}-root cancel-in-progress: true +permissions: {} + jobs: bump_version: runs-on: ubuntu-latest - outputs: ref: ${{ steps.push-tag.outputs.commit_long_sha }} - + permissions: {} # No job-level permissions are needed because it uses the app-token steps: - name: Generate a token id: generate-token @@ -40,6 +41,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: token: ${{ steps.generate-token.outputs.token }} + persist-credentials: true - name: Install uv uses: astral-sh/setup-uv@0c5e2b8115b80b4c7c5ddf6ffdd634974642d182 # v5 @@ -59,14 +61,20 @@ jobs: build_mobile: uses: ./.github/workflows/build-mobile.yml needs: bump_version - secrets: inherit + secrets: + KEY_JKS: ${{ secrets.KEY_JKS }} + ALIAS: ${{ secrets.ALIAS }} + ANDROID_KEY_PASSWORD: ${{ secrets.ANDROID_KEY_PASSWORD }} + ANDROID_STORE_PASSWORD: ${{ secrets.ANDROID_STORE_PASSWORD }} with: ref: ${{ needs.bump_version.outputs.ref }} prepare_release: runs-on: ubuntu-latest needs: build_mobile - + permissions: + actions: read # To download the app artifact + # No content permissions are needed because it uses the app-token steps: - name: Generate a token id: generate-token @@ -79,6 +87,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: token: ${{ steps.generate-token.outputs.token }} + persist-credentials: false - name: Download APK uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4 @@ -90,6 +99,7 @@ jobs: with: draft: true tag_name: ${{ env.IMMICH_VERSION }} + token: ${{ steps.generate-token.outputs.token }} generate_release_notes: true body_path: misc/release/notes.tmpl files: | diff --git a/.github/workflows/preview-label.yaml b/.github/workflows/preview-label.yaml index 447a309a2e..4c445f13d0 100644 --- a/.github/workflows/preview-label.yaml +++ b/.github/workflows/preview-label.yaml @@ -4,6 +4,8 @@ on: pull_request: types: [labeled, closed] +permissions: {} + jobs: comment-status: runs-on: ubuntu-latest diff --git a/.github/workflows/sdk.yml b/.github/workflows/sdk.yml index cde2075423..482faf29f4 100644 --- a/.github/workflows/sdk.yml +++ b/.github/workflows/sdk.yml @@ -4,18 +4,22 @@ on: release: types: [published] -permissions: - packages: write +permissions: {} jobs: publish: name: Publish `@immich/sdk` runs-on: ubuntu-latest + permissions: + contents: read defaults: run: working-directory: ./open-api/typescript-sdk steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false + # Setup .npmrc file to publish to npm - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: diff --git a/.github/workflows/static_analysis.yml b/.github/workflows/static_analysis.yml index 615082f86a..1a3c11d3d5 100644 --- a/.github/workflows/static_analysis.yml +++ b/.github/workflows/static_analysis.yml @@ -9,14 +9,20 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: {} + jobs: pre-job: runs-on: ubuntu-latest + permissions: + contents: read outputs: should_run: ${{ steps.found_paths.outputs.mobile == 'true' || steps.should_force.outputs.should_force == 'true' }} steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - id: found_paths uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 with: @@ -33,12 +39,14 @@ jobs: name: Run Dart Code Analysis needs: pre-job if: ${{ needs.pre-job.outputs.should_run == 'true' }} - runs-on: ubuntu-latest - + permissions: + contents: read steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Setup Flutter SDK uses: subosito/flutter-action@e938fdf56512cc96ef2f93601a5a40bde3801046 # v2 @@ -69,9 +77,11 @@ jobs: - name: Verify files have not changed if: steps.verify-changed-files.outputs.files_changed == 'true' + env: + CHANGED_FILES: ${{ steps.verify-changed-files.outputs.changed_files }} run: | echo "ERROR: Generated files not up to date! Run make_build inside the mobile directory" - echo "Changed files: ${{ steps.verify-changed-files.outputs.changed_files }}" + echo "Changed files: ${CHANGED_FILES}" exit 1 - name: Run dart analyze diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5273698e4e..91389c25ff 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,9 +9,13 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: {} + jobs: pre-job: runs-on: ubuntu-latest + permissions: + contents: read outputs: should_run_web: ${{ steps.found_paths.outputs.web == 'true' || steps.should_force.outputs.should_force == 'true' }} should_run_server: ${{ steps.found_paths.outputs.server == 'true' || steps.should_force.outputs.should_force == 'true' }} @@ -25,6 +29,9 @@ jobs: steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false + - id: found_paths uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 with: @@ -58,6 +65,8 @@ jobs: needs: pre-job if: ${{ needs.pre-job.outputs.should_run_server == 'true' }} runs-on: ubuntu-latest + permissions: + contents: read defaults: run: working-directory: ./server @@ -65,6 +74,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Setup Node uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 @@ -95,6 +106,8 @@ jobs: needs: pre-job if: ${{ needs.pre-job.outputs.should_run_cli == 'true' }} runs-on: ubuntu-latest + permissions: + contents: read defaults: run: working-directory: ./cli @@ -102,6 +115,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Setup Node uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 @@ -136,6 +151,8 @@ jobs: needs: pre-job if: ${{ needs.pre-job.outputs.should_run_cli == 'true' }} runs-on: windows-latest + permissions: + contents: read defaults: run: working-directory: ./cli @@ -143,6 +160,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Setup Node uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 @@ -170,6 +189,8 @@ jobs: needs: pre-job if: ${{ needs.pre-job.outputs.should_run_web == 'true' }} runs-on: ubuntu-latest + permissions: + contents: read defaults: run: working-directory: ./web @@ -177,6 +198,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Setup Node uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 @@ -215,6 +238,8 @@ jobs: needs: pre-job if: ${{ needs.pre-job.outputs.should_run_e2e == 'true' }} runs-on: ubuntu-latest + permissions: + contents: read defaults: run: working-directory: ./e2e @@ -222,6 +247,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Setup Node uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 @@ -254,6 +281,8 @@ jobs: needs: pre-job if: ${{ needs.pre-job.outputs.should_run_server == 'true' }} runs-on: ubuntu-latest + permissions: + contents: read defaults: run: working-directory: ./server @@ -261,6 +290,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Setup Node uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 @@ -279,6 +310,8 @@ jobs: needs: pre-job if: ${{ needs.pre-job.outputs.should_run_e2e_server_cli == 'true' }} runs-on: mich + permissions: + contents: read defaults: run: working-directory: ./e2e @@ -287,6 +320,7 @@ jobs: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: + persist-credentials: false submodules: 'recursive' - name: Setup Node @@ -321,6 +355,8 @@ jobs: needs: pre-job if: ${{ needs.pre-job.outputs.should_run_e2e_web == 'true' }} runs-on: mich + permissions: + contents: read defaults: run: working-directory: ./e2e @@ -329,6 +365,7 @@ jobs: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: + persist-credentials: false submodules: 'recursive' - name: Setup Node @@ -362,8 +399,13 @@ jobs: needs: pre-job if: ${{ needs.pre-job.outputs.should_run_mobile == 'true' }} runs-on: ubuntu-latest + permissions: + contents: read steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false + - name: Setup Flutter SDK uses: subosito/flutter-action@e938fdf56512cc96ef2f93601a5a40bde3801046 # v2 with: @@ -378,11 +420,16 @@ jobs: needs: pre-job if: ${{ needs.pre-job.outputs.should_run_ml == 'true' }} runs-on: ubuntu-latest + permissions: + contents: read defaults: run: working-directory: ./machine-learning steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false + - name: Install uv uses: astral-sh/setup-uv@0c5e2b8115b80b4c7c5ddf6ffdd634974642d182 # v5 - uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5 @@ -411,6 +458,8 @@ jobs: needs: pre-job if: ${{ needs.pre-job.outputs['should_run_.github'] == 'true' }} runs-on: ubuntu-latest + permissions: + contents: read defaults: run: working-directory: ./.github @@ -418,6 +467,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Setup Node uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 @@ -434,22 +485,31 @@ jobs: shellcheck: name: ShellCheck runs-on: ubuntu-latest + permissions: + contents: read steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false + - name: Run ShellCheck uses: ludeeus/action-shellcheck@master with: ignore_paths: >- **/open-api/** - **/openapi/** + **/openapi** **/node_modules/** generated-api-up-to-date: name: OpenAPI Clients runs-on: ubuntu-latest + permissions: + contents: read steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Setup Node uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 @@ -476,14 +536,18 @@ jobs: - name: Verify files have not changed if: steps.verify-changed-files.outputs.files_changed == 'true' + env: + CHANGED_FILES: ${{ steps.verify-changed-files.outputs.changed_files }} run: | echo "ERROR: Generated files not up to date!" - echo "Changed files: ${{ steps.verify-changed-files.outputs.changed_files }}" + echo "Changed files: ${CHANGED_FILES}" exit 1 generated-typeorm-migrations-up-to-date: name: TypeORM Checks runs-on: ubuntu-latest + permissions: + contents: read services: postgres: image: tensorchord/pgvecto-rs:pg14-v0.2.0@sha256:739cdd626151ff1f796dc95a6591b55a714f341c737e27f045019ceabf8e8c52 @@ -505,6 +569,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Setup Node uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 @@ -521,7 +587,7 @@ jobs: run: npm run migrations:run - name: Test npm run schema:reset command works - run: npm run typeorm:schema:reset + run: npm run schema:reset - name: Generate new migrations continue-on-error: true @@ -535,9 +601,11 @@ jobs: server/src - name: Verify migration files have not changed if: steps.verify-changed-files.outputs.files_changed == 'true' + env: + CHANGED_FILES: ${{ steps.verify-changed-files.outputs.changed_files }} run: | echo "ERROR: Generated migration files not up to date!" - echo "Changed files: ${{ steps.verify-changed-files.outputs.changed_files }}" + echo "Changed files: ${CHANGED_FILES}" cat ./src/*-TestMigration.ts exit 1 @@ -555,9 +623,11 @@ jobs: - name: Verify SQL files have not changed if: steps.verify-changed-sql-files.outputs.files_changed == 'true' + env: + CHANGED_FILES: ${{ steps.verify-changed-sql-files.outputs.changed_files }} run: | echo "ERROR: Generated SQL files not up to date!" - echo "Changed files: ${{ steps.verify-changed-sql-files.outputs.changed_files }}" + echo "Changed files: ${CHANGED_FILES}" exit 1 # mobile-integration-tests: diff --git a/.github/workflows/weblate-lock.yml b/.github/workflows/weblate-lock.yml index 69dce3ac41..2aef5c472a 100644 --- a/.github/workflows/weblate-lock.yml +++ b/.github/workflows/weblate-lock.yml @@ -4,30 +4,32 @@ on: pull_request: branches: [main] +permissions: {} + jobs: pre-job: runs-on: ubuntu-latest + permissions: + contents: read outputs: should_run: ${{ steps.found_paths.outputs.i18n == 'true' && github.head_ref != 'chore/translations'}} steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - id: found_paths uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 with: filters: | i18n: - 'i18n/!(en)**\.json' - - name: Debug - run: | - echo "Should run: ${{ steps.found_paths.outputs.i18n == 'true' && github.head_ref != 'chore/translations'}}" - echo "Found i18n paths: ${{ steps.found_paths.outputs.i18n }}" - echo "Head ref: ${{ github.head_ref }}" enforce-lock: name: Check Weblate Lock needs: [pre-job] runs-on: ubuntu-latest + permissions: {} if: ${{ needs.pre-job.outputs.should_run == 'true' }} steps: - name: Check weblate lock @@ -47,6 +49,7 @@ jobs: name: Weblate Lock Check Success needs: [enforce-lock] runs-on: ubuntu-latest + permissions: {} if: always() steps: - name: Any jobs failed? diff --git a/docs/docs/developer/database-migrations.md b/docs/docs/developer/database-migrations.md index 2cddf5f386..c0c61340cb 100644 --- a/docs/docs/developer/database-migrations.md +++ b/docs/docs/developer/database-migrations.md @@ -1,14 +1,14 @@ # Database Migrations -After making any changes in the `server/src/entities`, a database migration need to run in order to register the changes in the database. Follow the steps below to create a new migration. +After making any changes in the `server/src/schema`, a database migration need to run in order to register the changes in the database. Follow the steps below to create a new migration. 1. Run the command ```bash -npm run typeorm:migrations:generate +npm run migrations:generate ``` 2. Check if the migration file makes sense. -3. Move the migration file to folder `./server/src/migrations` in your code editor. +3. Move the migration file to folder `./server/src/schema/migrations` in your code editor. The server will automatically detect `*.ts` file changes and restart. Part of the server start-up process includes running any new migrations, so it will be applied immediately. diff --git a/docs/src/pages/errors.md b/docs/src/pages/errors.md new file mode 100644 index 0000000000..e9bf09770c --- /dev/null +++ b/docs/src/pages/errors.md @@ -0,0 +1,5 @@ +# Errors + +## TypeORM Upgrade + +The upgrade to Immich `v2.x.x` has a required upgrade path to `v1.132.0+`. This means it is required to start up the application at least once on version `1.132.0` (or later). Doing so will complete database schema upgrades that are required for `v2.0.0`. After Immich has successfully booted on this version, shut the system down and try the `v2.x.x` upgrade again. diff --git a/i18n/en.json b/i18n/en.json index 3b52c2019e..c4b4746871 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -996,6 +996,7 @@ "filetype": "Filetype", "filter": "Filter", "filter_people": "Filter people", + "filter_places": "Filter places", "find_them_fast": "Find them fast by name with search", "fix_incorrect_match": "Fix incorrect match", "folder": "Folder", diff --git a/mobile/analysis_options.yaml b/mobile/analysis_options.yaml index 04f3145908..854f852e3c 100644 --- a/mobile/analysis_options.yaml +++ b/mobile/analysis_options.yaml @@ -35,6 +35,7 @@ linter: analyzer: exclude: - openapi/** + - build/** - lib/generated_plugin_registrant.dart - lib/**/*.g.dart - lib/**/*.drift.dart @@ -92,6 +93,9 @@ custom_lint: allowed: # required / wanted - lib/repositories/*_api.repository.dart + - lib/domain/models/sync_event.model.dart + - lib/{domain,infrastructure}/**/sync_stream.* + - lib/{domain,infrastructure}/**/sync_api.* - lib/infrastructure/repositories/*_api.repository.dart - lib/infrastructure/utils/*.converter.dart # acceptable exceptions for the time being @@ -144,7 +148,9 @@ dart_code_metrics: - avoid-global-state - avoid-inverted-boolean-checks - avoid-late-final-reassignment - - avoid-local-functions + - avoid-local-functions: + exclude: + - test/**.dart - avoid-negated-conditions - avoid-nested-streams-and-futures - avoid-referencing-subclasses diff --git a/mobile/devtools_options.yaml b/mobile/devtools_options.yaml index fa0b357c4f..f592d85a9b 100644 --- a/mobile/devtools_options.yaml +++ b/mobile/devtools_options.yaml @@ -1,3 +1,4 @@ description: This file stores settings for Dart & Flutter DevTools. documentation: https://docs.flutter.dev/tools/devtools/extensions#configure-extension-enablement-states extensions: + - drift: true \ No newline at end of file diff --git a/mobile/lib/constants/constants.dart b/mobile/lib/constants/constants.dart index 83d540d54c..a91e0a715d 100644 --- a/mobile/lib/constants/constants.dart +++ b/mobile/lib/constants/constants.dart @@ -5,5 +5,9 @@ const double downloadFailed = -2; // Number of log entries to retain on app start const int kLogTruncateLimit = 250; +// Sync +const int kSyncEventBatchSize = 5000; + +// Hash batch limits const int kBatchHashFileLimit = 128; const int kBatchHashSizeLimit = 1024 * 1024 * 1024; // 1GB diff --git a/mobile/lib/domain/interfaces/sync_api.interface.dart b/mobile/lib/domain/interfaces/sync_api.interface.dart index fb8f1aa46e..57abed2e7f 100644 --- a/mobile/lib/domain/interfaces/sync_api.interface.dart +++ b/mobile/lib/domain/interfaces/sync_api.interface.dart @@ -1,7 +1,12 @@ -import 'package:immich_mobile/domain/models/sync/sync_event.model.dart'; +import 'package:http/http.dart' as http; +import 'package:immich_mobile/domain/models/sync_event.model.dart'; abstract interface class ISyncApiRepository { - Future ack(String data); + Future ack(List data); - Stream> watchUserSyncEvent(); + Future streamChanges( + Function(List, Function() abort) onData, { + int batchSize, + http.Client? httpClient, + }); } diff --git a/mobile/lib/domain/interfaces/sync_stream.interface.dart b/mobile/lib/domain/interfaces/sync_stream.interface.dart new file mode 100644 index 0000000000..5f61d6b52f --- /dev/null +++ b/mobile/lib/domain/interfaces/sync_stream.interface.dart @@ -0,0 +1,18 @@ +import 'package:immich_mobile/domain/interfaces/db.interface.dart'; +import 'package:openapi/api.dart'; + +abstract interface class ISyncStreamRepository implements IDatabaseRepository { + Future updateUsersV1(Iterable data); + Future deleteUsersV1(Iterable data); + + Future updatePartnerV1(Iterable data); + Future deletePartnerV1(Iterable data); + + Future updateAssetsV1(Iterable data); + Future deleteAssetsV1(Iterable data); + Future updateAssetsExifV1(Iterable data); + + Future updatePartnerAssetsV1(Iterable data); + Future deletePartnerAssetsV1(Iterable data); + Future updatePartnerAssetsExifV1(Iterable data); +} diff --git a/mobile/lib/domain/models/sync/sync_event.model.dart b/mobile/lib/domain/models/sync/sync_event.model.dart deleted file mode 100644 index f4642d59cf..0000000000 --- a/mobile/lib/domain/models/sync/sync_event.model.dart +++ /dev/null @@ -1,14 +0,0 @@ -class SyncEvent { - // dynamic - final dynamic data; - - final String ack; - - SyncEvent({ - required this.data, - required this.ack, - }); - - @override - String toString() => 'SyncEvent(data: $data, ack: $ack)'; -} diff --git a/mobile/lib/domain/models/sync_event.model.dart b/mobile/lib/domain/models/sync_event.model.dart new file mode 100644 index 0000000000..2ad8a75fe1 --- /dev/null +++ b/mobile/lib/domain/models/sync_event.model.dart @@ -0,0 +1,13 @@ +import 'package:openapi/api.dart'; + +class SyncEvent { + final SyncEntityType type; + // ignore: avoid-dynamic + final dynamic data; + final String ack; + + const SyncEvent({required this.type, required this.data, required this.ack}); + + @override + String toString() => 'SyncEvent(type: $type, data: $data, ack: $ack)'; +} diff --git a/mobile/lib/domain/services/sync_stream.service.dart b/mobile/lib/domain/services/sync_stream.service.dart index 72e29b3677..ac63734b07 100644 --- a/mobile/lib/domain/services/sync_stream.service.dart +++ b/mobile/lib/domain/services/sync_stream.service.dart @@ -1,49 +1,92 @@ +// ignore_for_file: avoid-passing-async-when-sync-expected + import 'dart:async'; -import 'package:flutter/foundation.dart'; import 'package:immich_mobile/domain/interfaces/sync_api.interface.dart'; +import 'package:immich_mobile/domain/interfaces/sync_stream.interface.dart'; +import 'package:immich_mobile/domain/models/sync_event.model.dart'; +import 'package:logging/logging.dart'; import 'package:openapi/api.dart'; class SyncStreamService { + final Logger _logger = Logger('SyncStreamService'); + final ISyncApiRepository _syncApiRepository; + final ISyncStreamRepository _syncStreamRepository; + final bool Function()? _cancelChecker; - SyncStreamService(this._syncApiRepository); + SyncStreamService({ + required ISyncApiRepository syncApiRepository, + required ISyncStreamRepository syncStreamRepository, + bool Function()? cancelChecker, + }) : _syncApiRepository = syncApiRepository, + _syncStreamRepository = syncStreamRepository, + _cancelChecker = cancelChecker; - StreamSubscription? _userSyncSubscription; + bool get isCancelled => _cancelChecker?.call() ?? false; - void syncUsers() { - _userSyncSubscription = - _syncApiRepository.watchUserSyncEvent().listen((events) async { - for (final event in events) { - if (event.data is SyncUserV1) { - final data = event.data as SyncUserV1; - debugPrint("User Update: $data"); + Future sync() => _syncApiRepository.streamChanges(_handleEvents); - // final user = await _userRepository.get(data.id); - - // if (user == null) { - // continue; - // } - - // user.name = data.name; - // user.email = data.email; - // user.updatedAt = DateTime.now(); - - // await _userRepository.update(user); - // await _syncApiRepository.ack(event.ack); - } - - if (event.data is SyncUserDeleteV1) { - final data = event.data as SyncUserDeleteV1; - - debugPrint("User delete: $data"); - // await _syncApiRepository.ack(event.ack); - } + Future _handleEvents(List events, Function() abort) async { + List items = []; + for (final event in events) { + if (isCancelled) { + _logger.warning("Sync stream cancelled"); + abort(); + return; } - }); + + if (event.type != items.firstOrNull?.type) { + await _processBatch(items); + } + + items.add(event); + } + + await _processBatch(items); } - Future dispose() async { - await _userSyncSubscription?.cancel(); + Future _processBatch(List batch) async { + if (batch.isEmpty) { + return; + } + + final type = batch.first.type; + await _handleSyncData(type, batch.map((e) => e.data)); + await _syncApiRepository.ack([batch.last.ack]); + batch.clear(); + } + + Future _handleSyncData( + SyncEntityType type, + // ignore: avoid-dynamic + Iterable data, + ) async { + _logger.fine("Processing sync data for $type of length ${data.length}"); + // ignore: prefer-switch-expression + switch (type) { + case SyncEntityType.userV1: + return _syncStreamRepository.updateUsersV1(data.cast()); + case SyncEntityType.userDeleteV1: + return _syncStreamRepository.deleteUsersV1(data.cast()); + case SyncEntityType.partnerV1: + return _syncStreamRepository.updatePartnerV1(data.cast()); + case SyncEntityType.partnerDeleteV1: + return _syncStreamRepository.deletePartnerV1(data.cast()); + case SyncEntityType.assetV1: + return _syncStreamRepository.updateAssetsV1(data.cast()); + case SyncEntityType.assetDeleteV1: + return _syncStreamRepository.deleteAssetsV1(data.cast()); + case SyncEntityType.assetExifV1: + return _syncStreamRepository.updateAssetsExifV1(data.cast()); + case SyncEntityType.partnerAssetV1: + return _syncStreamRepository.updatePartnerAssetsV1(data.cast()); + case SyncEntityType.partnerAssetDeleteV1: + return _syncStreamRepository.deletePartnerAssetsV1(data.cast()); + case SyncEntityType.partnerAssetExifV1: + return _syncStreamRepository.updatePartnerAssetsExifV1(data.cast()); + default: + _logger.warning("Unknown sync data type: $type"); + } } } diff --git a/mobile/lib/domain/utils/background_sync.dart b/mobile/lib/domain/utils/background_sync.dart new file mode 100644 index 0000000000..f63dc81ba9 --- /dev/null +++ b/mobile/lib/domain/utils/background_sync.dart @@ -0,0 +1,39 @@ +// ignore_for_file: avoid-passing-async-when-sync-expected + +import 'dart:async'; + +import 'package:immich_mobile/providers/infrastructure/sync_stream.provider.dart'; +import 'package:immich_mobile/utils/isolate.dart'; +import 'package:worker_manager/worker_manager.dart'; + +class BackgroundSyncManager { + Cancelable? _syncTask; + + BackgroundSyncManager(); + + Future cancel() { + final futures = []; + + if (_syncTask != null) { + futures.add(_syncTask!.future); + } + _syncTask?.cancel(); + _syncTask = null; + + return Future.wait(futures); + } + + Future sync() { + if (_syncTask != null) { + return _syncTask!.future; + } + + _syncTask = runInIsolateGentle( + computation: (ref) => ref.read(syncStreamServiceProvider).sync(), + ); + _syncTask!.whenComplete(() { + _syncTask = null; + }); + return _syncTask!.future; + } +} diff --git a/mobile/lib/extensions/string_extensions.dart b/mobile/lib/extensions/string_extensions.dart index 67411013ee..73c8c2d34c 100644 --- a/mobile/lib/extensions/string_extensions.dart +++ b/mobile/lib/extensions/string_extensions.dart @@ -1,3 +1,7 @@ +import 'dart:typed_data'; + +import 'package:uuid/parsing.dart'; + extension StringExtension on String { String capitalize() { return split(" ") @@ -29,3 +33,8 @@ extension DurationExtension on String { return int.parse(this); } } + +extension UUIDExtension on String { + Uint8List toUuidByte({bool shouldValidate = false}) => + UuidParsing.parseAsByteList(this, validate: shouldValidate); +} diff --git a/mobile/lib/infrastructure/repositories/sync_api.repository.dart b/mobile/lib/infrastructure/repositories/sync_api.repository.dart index 88a6838c44..dd1ea208ba 100644 --- a/mobile/lib/infrastructure/repositories/sync_api.repository.dart +++ b/mobile/lib/infrastructure/repositories/sync_api.repository.dart @@ -1,55 +1,72 @@ import 'dart:async'; import 'dart:convert'; -import 'package:flutter/foundation.dart'; -import 'package:immich_mobile/domain/interfaces/sync_api.interface.dart'; -import 'package:immich_mobile/domain/models/sync/sync_event.model.dart'; -import 'package:immich_mobile/services/api.service.dart'; -import 'package:openapi/api.dart'; import 'package:http/http.dart' as http; +import 'package:immich_mobile/constants/constants.dart'; +import 'package:immich_mobile/domain/interfaces/sync_api.interface.dart'; +import 'package:immich_mobile/domain/models/sync_event.model.dart'; +import 'package:immich_mobile/services/api.service.dart'; +import 'package:logging/logging.dart'; +import 'package:openapi/api.dart'; class SyncApiRepository implements ISyncApiRepository { + final Logger _logger = Logger('SyncApiRepository'); final ApiService _api; - const SyncApiRepository(this._api); + SyncApiRepository(this._api); @override - Stream> watchUserSyncEvent() { - return _getSyncStream( - SyncStreamDto(types: [SyncRequestType.usersV1]), - ); + Future ack(List data) { + return _api.syncApi.sendSyncAck(SyncAckSetDto(acks: data)); } @override - Future ack(String data) { - return _api.syncApi.sendSyncAck(SyncAckSetDto(acks: [data])); - } - - Stream> _getSyncStream( - SyncStreamDto dto, { - int batchSize = 5000, - }) async* { - final client = http.Client(); + Future streamChanges( + Function(List, Function() abort) onData, { + int batchSize = kSyncEventBatchSize, + http.Client? httpClient, + }) async { + // ignore: avoid-unused-assignment + final stopwatch = Stopwatch()..start(); + final client = httpClient ?? http.Client(); final endpoint = "${_api.apiClient.basePath}/sync/stream"; - final headers = { + final headers = { 'Content-Type': 'application/json', 'Accept': 'application/jsonlines+json', }; - final queryParams = []; final headerParams = {}; - await _api.applyToParams(queryParams, headerParams); + await _api.applyToParams([], headerParams); headers.addAll(headerParams); final request = http.Request('POST', Uri.parse(endpoint)); request.headers.addAll(headers); - request.body = jsonEncode(dto.toJson()); + request.body = jsonEncode( + SyncStreamDto( + types: [ + SyncRequestType.usersV1, + SyncRequestType.partnersV1, + SyncRequestType.assetsV1, + SyncRequestType.partnerAssetsV1, + SyncRequestType.assetExifsV1, + SyncRequestType.partnerAssetExifsV1, + ], + ).toJson(), + ); String previousChunk = ''; List lines = []; + bool shouldAbort = false; + + void abort() { + _logger.warning("Abort requested, stopping sync stream"); + shouldAbort = true; + } + try { - final response = await client.send(request); + final response = + await client.send(request).timeout(const Duration(seconds: 20)); if (response.statusCode != 200) { final errorBody = await response.stream.bytesToString(); @@ -60,8 +77,12 @@ class SyncApiRepository implements ISyncApiRepository { } await for (final chunk in response.stream.transform(utf8.decoder)) { + if (shouldAbort) { + break; + } + previousChunk += chunk; - final parts = previousChunk.split('\n'); + final parts = previousChunk.toString().split('\n'); previousChunk = parts.removeLast(); lines.addAll(parts); @@ -69,44 +90,59 @@ class SyncApiRepository implements ISyncApiRepository { continue; } - yield await compute(_parseSyncResponse, lines); + await onData(_parseLines(lines), abort); lines.clear(); } - } finally { - if (lines.isNotEmpty) { - yield await compute(_parseSyncResponse, lines); + + if (lines.isNotEmpty && !shouldAbort) { + await onData(_parseLines(lines), abort); } + } catch (error, stack) { + _logger.severe("error processing stream", error, stack); + return Future.error(error, stack); + } finally { client.close(); } + stopwatch.stop(); + _logger + .info("Remote Sync completed in ${stopwatch.elapsed.inMilliseconds}ms"); + } + + List _parseLines(List lines) { + final List data = []; + + for (final line in lines) { + try { + final jsonData = jsonDecode(line); + final type = SyncEntityType.fromJson(jsonData['type'])!; + final dataJson = jsonData['data']; + final ack = jsonData['ack']; + final converter = _kResponseMap[type]; + if (converter == null) { + _logger.warning("[_parseSyncResponse] Unknown type $type"); + continue; + } + + data.add(SyncEvent(type: type, data: converter(dataJson), ack: ack)); + } catch (error, stack) { + _logger.severe("[_parseSyncResponse] Error parsing json", error, stack); + } + } + + return data; } } +// ignore: avoid-dynamic const _kResponseMap = { SyncEntityType.userV1: SyncUserV1.fromJson, SyncEntityType.userDeleteV1: SyncUserDeleteV1.fromJson, + SyncEntityType.partnerV1: SyncPartnerV1.fromJson, + SyncEntityType.partnerDeleteV1: SyncPartnerDeleteV1.fromJson, + SyncEntityType.assetV1: SyncAssetV1.fromJson, + SyncEntityType.assetDeleteV1: SyncAssetDeleteV1.fromJson, + SyncEntityType.assetExifV1: SyncAssetExifV1.fromJson, + SyncEntityType.partnerAssetV1: SyncAssetV1.fromJson, + SyncEntityType.partnerAssetDeleteV1: SyncAssetDeleteV1.fromJson, + SyncEntityType.partnerAssetExifV1: SyncAssetExifV1.fromJson, }; - -// Need to be outside of the class to be able to use compute -List _parseSyncResponse(List lines) { - final List data = []; - - for (var line in lines) { - try { - final jsonData = jsonDecode(line); - final type = SyncEntityType.fromJson(jsonData['type'])!; - final dataJson = jsonData['data']; - final ack = jsonData['ack']; - final converter = _kResponseMap[type]; - if (converter == null) { - debugPrint("[_parseSyncReponse] Unknown type $type"); - continue; - } - - data.add(SyncEvent(data: converter(dataJson), ack: ack)); - } catch (error, stack) { - debugPrint("[_parseSyncReponse] Error parsing json $error $stack"); - } - } - - return data; -} diff --git a/mobile/lib/infrastructure/repositories/sync_stream.repository.dart b/mobile/lib/infrastructure/repositories/sync_stream.repository.dart new file mode 100644 index 0000000000..5ad9a369df --- /dev/null +++ b/mobile/lib/infrastructure/repositories/sync_stream.repository.dart @@ -0,0 +1,134 @@ +import 'package:drift/drift.dart'; +import 'package:flutter/foundation.dart'; +import 'package:immich_mobile/domain/interfaces/sync_stream.interface.dart'; +import 'package:immich_mobile/extensions/string_extensions.dart'; +import 'package:immich_mobile/infrastructure/entities/partner.entity.drift.dart'; +import 'package:immich_mobile/infrastructure/entities/user.entity.drift.dart'; +import 'package:immich_mobile/infrastructure/repositories/db.repository.dart'; +import 'package:logging/logging.dart'; +import 'package:openapi/api.dart'; + +class DriftSyncStreamRepository extends DriftDatabaseRepository + implements ISyncStreamRepository { + final Logger _logger = Logger('DriftSyncStreamRepository'); + final Drift _db; + + DriftSyncStreamRepository(super.db) : _db = db; + + @override + Future deleteUsersV1(Iterable data) async { + try { + await _db.batch((batch) { + for (final user in data) { + batch.delete( + _db.userEntity, + UserEntityCompanion(id: Value(user.userId.toUuidByte())), + ); + } + }); + } catch (error, stack) { + _logger.severe('Error while processing SyncUserDeleteV1', error, stack); + rethrow; + } + } + + @override + Future updateUsersV1(Iterable data) async { + try { + await _db.batch((batch) { + for (final user in data) { + final companion = UserEntityCompanion( + name: Value(user.name), + email: Value(user.email), + ); + + batch.insert( + _db.userEntity, + companion.copyWith(id: Value(user.id.toUuidByte())), + onConflict: DoUpdate((_) => companion), + ); + } + }); + } catch (error, stack) { + _logger.severe('Error while processing SyncUserV1', error, stack); + rethrow; + } + } + + @override + Future deletePartnerV1(Iterable data) async { + try { + await _db.batch((batch) { + for (final partner in data) { + batch.delete( + _db.partnerEntity, + PartnerEntityCompanion( + sharedById: Value(partner.sharedById.toUuidByte()), + sharedWithId: Value(partner.sharedWithId.toUuidByte()), + ), + ); + } + }); + } catch (e, s) { + _logger.severe('Error while processing SyncPartnerDeleteV1', e, s); + rethrow; + } + } + + @override + Future updatePartnerV1(Iterable data) async { + try { + await _db.batch((batch) { + for (final partner in data) { + final companion = + PartnerEntityCompanion(inTimeline: Value(partner.inTimeline)); + + batch.insert( + _db.partnerEntity, + companion.copyWith( + sharedById: Value(partner.sharedById.toUuidByte()), + sharedWithId: Value(partner.sharedWithId.toUuidByte()), + ), + onConflict: DoUpdate((_) => companion), + ); + } + }); + } catch (e, s) { + _logger.severe('Error while processing SyncPartnerV1', e, s); + rethrow; + } + } + + // Assets + @override + Future updateAssetsV1(Iterable data) async { + debugPrint("updateAssetsV1 - ${data.length}"); + } + + @override + Future deleteAssetsV1(Iterable data) async { + debugPrint("deleteAssetsV1 - ${data.length}"); + } + + // Partner Assets + @override + Future updatePartnerAssetsV1(Iterable data) async { + debugPrint("updatePartnerAssetsV1 - ${data.length}"); + } + + @override + Future deletePartnerAssetsV1(Iterable data) async { + debugPrint("deletePartnerAssetsV1 - ${data.length}"); + } + + // EXIF + @override + Future updateAssetsExifV1(Iterable data) async { + debugPrint("updateAssetsExifV1 - ${data.length}"); + } + + @override + Future updatePartnerAssetsExifV1(Iterable data) async { + debugPrint("updatePartnerAssetsExifV1 - ${data.length}"); + } +} diff --git a/mobile/lib/main.dart b/mobile/lib/main.dart index 1a434aa359..73af81d69d 100644 --- a/mobile/lib/main.dart +++ b/mobile/lib/main.dart @@ -11,6 +11,7 @@ import 'package:flutter_displaymode/flutter_displaymode.dart'; import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/constants/locales.dart'; import 'package:immich_mobile/extensions/build_context_extensions.dart'; +import 'package:immich_mobile/generated/codegen_loader.g.dart'; import 'package:immich_mobile/providers/app_life_cycle.provider.dart'; import 'package:immich_mobile/providers/asset_viewer/share_intent_upload.provider.dart'; import 'package:immich_mobile/providers/db.provider.dart'; @@ -31,13 +32,15 @@ import 'package:immich_mobile/utils/migration.dart'; import 'package:intl/date_symbol_data_local.dart'; import 'package:logging/logging.dart'; import 'package:timezone/data/latest.dart'; -import 'package:immich_mobile/generated/codegen_loader.g.dart'; +import 'package:worker_manager/worker_manager.dart'; void main() async { ImmichWidgetsBinding(); final db = await Bootstrap.initIsar(); await Bootstrap.initDomain(db); await initApp(); + // Warm-up isolate pool for worker manager + await workerManager.init(dynamicSpawning: true); await migrateDatabaseIfNeeded(db); HttpOverrides.global = HttpSSLCertOverride(); diff --git a/mobile/lib/pages/library/people/people_collection.page.dart b/mobile/lib/pages/library/people/people_collection.page.dart index 5f587c0c76..27daf0a887 100644 --- a/mobile/lib/pages/library/people/people_collection.page.dart +++ b/mobile/lib/pages/library/people/people_collection.page.dart @@ -4,11 +4,11 @@ import 'package:flutter/material.dart'; import 'package:flutter_hooks/flutter_hooks.dart'; import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/extensions/build_context_extensions.dart'; -import 'package:immich_mobile/extensions/theme_extensions.dart'; import 'package:immich_mobile/providers/search/people.provider.dart'; import 'package:immich_mobile/routing/router.dart'; import 'package:immich_mobile/services/api.service.dart'; import 'package:immich_mobile/utils/image_url_builder.dart'; +import 'package:immich_mobile/widgets/common/search_field.dart'; import 'package:immich_mobile/widgets/search/person_name_edit_form.dart'; @RoutePage() @@ -42,47 +42,12 @@ class PeopleCollectionPage extends HookConsumerWidget { appBar: AppBar( automaticallyImplyLeading: search.value == null, title: search.value != null - ? TextField( + ? SearchField( focusNode: formFocus, onTapOutside: (_) => formFocus.unfocus(), onChanged: (value) => search.value = value, - decoration: InputDecoration( - contentPadding: const EdgeInsets.only(left: 24), - filled: true, - fillColor: context.primaryColor.withValues(alpha: 0.1), - hintStyle: context.textTheme.bodyLarge?.copyWith( - color: context.themeData.colorScheme.onSurfaceSecondary, - ), - border: OutlineInputBorder( - borderRadius: BorderRadius.circular(25), - borderSide: BorderSide( - color: context.colorScheme.surfaceContainerHighest, - ), - ), - enabledBorder: OutlineInputBorder( - borderRadius: BorderRadius.circular(25), - borderSide: BorderSide( - color: context.colorScheme.surfaceContainerHighest, - ), - ), - disabledBorder: OutlineInputBorder( - borderRadius: BorderRadius.circular(25), - borderSide: BorderSide( - color: context.colorScheme.surfaceContainerHighest, - ), - ), - focusedBorder: OutlineInputBorder( - borderRadius: BorderRadius.circular(25), - borderSide: BorderSide( - color: context.colorScheme.primary.withAlpha(150), - ), - ), - prefixIcon: Icon( - Icons.search_rounded, - color: context.colorScheme.primary, - ), - hintText: 'filter_people'.tr(), - ), + filled: true, + hintText: 'filter_people'.tr(), autofocus: true, ) : Text('people'.tr()), diff --git a/mobile/lib/pages/library/places/places_collection.page.dart b/mobile/lib/pages/library/places/places_collection.page.dart index d4da3ff37e..f9a2d4292c 100644 --- a/mobile/lib/pages/library/places/places_collection.page.dart +++ b/mobile/lib/pages/library/places/places_collection.page.dart @@ -2,6 +2,7 @@ import 'package:auto_route/auto_route.dart'; import 'package:cached_network_image/cached_network_image.dart'; import 'package:easy_localization/easy_localization.dart'; import 'package:flutter/material.dart'; +import 'package:flutter_hooks/flutter_hooks.dart' hide Store; import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/domain/models/store.model.dart'; import 'package:immich_mobile/entities/asset.entity.dart'; @@ -12,6 +13,7 @@ import 'package:immich_mobile/pages/common/large_leading_tile.dart'; import 'package:immich_mobile/providers/search/search_page_state.provider.dart'; import 'package:immich_mobile/routing/router.dart'; import 'package:immich_mobile/services/api.service.dart'; +import 'package:immich_mobile/widgets/common/search_field.dart'; import 'package:immich_mobile/widgets/map/map_thumbnail.dart'; import 'package:maplibre_gl/maplibre_gl.dart'; @@ -21,34 +23,62 @@ class PlacesCollectionPage extends HookConsumerWidget { @override Widget build(BuildContext context, WidgetRef ref) { final places = ref.watch(getAllPlacesProvider); + final formFocus = useFocusNode(); + final ValueNotifier search = useState(null); return Scaffold( appBar: AppBar( - title: Text('places'.tr()), + automaticallyImplyLeading: search.value == null, + title: search.value != null + ? SearchField( + autofocus: true, + filled: true, + focusNode: formFocus, + onChanged: (value) => search.value = value, + onTapOutside: (_) => formFocus.unfocus(), + hintText: 'filter_places'.tr(), + ) + : Text('places'.tr()), + actions: [ + IconButton( + icon: Icon(search.value != null ? Icons.close : Icons.search), + onPressed: () { + search.value = search.value == null ? '' : null; + }, + ), + ], ), body: ListView( shrinkWrap: true, children: [ - Padding( - padding: const EdgeInsets.all(16.0), - child: SizedBox( - height: 200, - width: context.width, - child: MapThumbnail( - onTap: (_, __) => context.pushRoute(const MapRoute()), - zoom: 8, - centre: const LatLng( - 21.44950, - -157.91959, + if (search.value == null) + Padding( + padding: const EdgeInsets.all(16.0), + child: SizedBox( + height: 200, + width: context.width, + child: MapThumbnail( + onTap: (_, __) => context.pushRoute(const MapRoute()), + zoom: 8, + centre: const LatLng( + 21.44950, + -157.91959, + ), + showAttribution: false, + themeMode: + context.isDarkTheme ? ThemeMode.dark : ThemeMode.light, ), - showAttribution: false, - themeMode: - context.isDarkTheme ? ThemeMode.dark : ThemeMode.light, ), ), - ), places.when( data: (places) { + if (search.value != null) { + places = places.where((place) { + return place.label + .toLowerCase() + .contains(search.value!.toLowerCase()); + }).toList(); + } return ListView.builder( shrinkWrap: true, physics: const NeverScrollableScrollPhysics(), diff --git a/mobile/lib/providers/background_sync.provider.dart b/mobile/lib/providers/background_sync.provider.dart new file mode 100644 index 0000000000..83d103bb3b --- /dev/null +++ b/mobile/lib/providers/background_sync.provider.dart @@ -0,0 +1,8 @@ +import 'package:hooks_riverpod/hooks_riverpod.dart'; +import 'package:immich_mobile/domain/utils/background_sync.dart'; + +final backgroundSyncProvider = Provider((ref) { + final manager = BackgroundSyncManager(); + ref.onDispose(manager.cancel); + return manager; +}); diff --git a/mobile/lib/providers/infrastructure/cancel.provider.dart b/mobile/lib/providers/infrastructure/cancel.provider.dart new file mode 100644 index 0000000000..6851861e1a --- /dev/null +++ b/mobile/lib/providers/infrastructure/cancel.provider.dart @@ -0,0 +1,12 @@ +import 'package:hooks_riverpod/hooks_riverpod.dart'; + +/// Provider holding a boolean function that returns true when cancellation is requested. +/// A computation running in the isolate uses the function to implement cooperative cancellation. +final cancellationProvider = Provider( + // This will be overridden in the isolate's container. + // Throwing ensures it's not used without an override. + (ref) => throw UnimplementedError( + "cancellationProvider must be overridden in the isolate's ProviderContainer and not to be used in the root isolate", + ), + name: 'cancellationProvider', +); diff --git a/mobile/lib/providers/infrastructure/db.provider.dart b/mobile/lib/providers/infrastructure/db.provider.dart index 84010b3b96..4eefbc556c 100644 --- a/mobile/lib/providers/infrastructure/db.provider.dart +++ b/mobile/lib/providers/infrastructure/db.provider.dart @@ -1,4 +1,7 @@ +import 'dart:async'; + import 'package:hooks_riverpod/hooks_riverpod.dart'; +import 'package:immich_mobile/infrastructure/repositories/db.repository.dart'; import 'package:isar/isar.dart'; import 'package:riverpod_annotation/riverpod_annotation.dart'; @@ -6,3 +9,9 @@ part 'db.provider.g.dart'; @Riverpod(keepAlive: true) Isar isar(Ref ref) => throw UnimplementedError('isar'); + +final driftProvider = Provider((ref) { + final drift = Drift(); + ref.onDispose(() => unawaited(drift.close())); + return drift; +}); diff --git a/mobile/lib/providers/infrastructure/sync_stream.provider.dart b/mobile/lib/providers/infrastructure/sync_stream.provider.dart index 64f1a6cb05..e313982a30 100644 --- a/mobile/lib/providers/infrastructure/sync_stream.provider.dart +++ b/mobile/lib/providers/infrastructure/sync_stream.provider.dart @@ -1,24 +1,23 @@ -import 'dart:async'; - import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/domain/services/sync_stream.service.dart'; import 'package:immich_mobile/infrastructure/repositories/sync_api.repository.dart'; +import 'package:immich_mobile/infrastructure/repositories/sync_stream.repository.dart'; import 'package:immich_mobile/providers/api.provider.dart'; +import 'package:immich_mobile/providers/infrastructure/cancel.provider.dart'; +import 'package:immich_mobile/providers/infrastructure/db.provider.dart'; final syncStreamServiceProvider = Provider( - (ref) { - final instance = SyncStreamService( - ref.watch(syncApiRepositoryProvider), - ); - - ref.onDispose(() => unawaited(instance.dispose())); - - return instance; - }, + (ref) => SyncStreamService( + syncApiRepository: ref.watch(syncApiRepositoryProvider), + syncStreamRepository: ref.watch(syncStreamRepositoryProvider), + cancelChecker: ref.watch(cancellationProvider), + ), ); final syncApiRepositoryProvider = Provider( - (ref) => SyncApiRepository( - ref.watch(apiServiceProvider), - ), + (ref) => SyncApiRepository(ref.watch(apiServiceProvider)), +); + +final syncStreamRepositoryProvider = Provider( + (ref) => DriftSyncStreamRepository(ref.watch(driftProvider)), ); diff --git a/mobile/lib/services/auth.service.dart b/mobile/lib/services/auth.service.dart index 20fa62dc4b..ec053c078b 100644 --- a/mobile/lib/services/auth.service.dart +++ b/mobile/lib/services/auth.service.dart @@ -3,12 +3,14 @@ import 'dart:io'; import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/domain/models/store.model.dart'; +import 'package:immich_mobile/domain/utils/background_sync.dart'; import 'package:immich_mobile/entities/store.entity.dart'; import 'package:immich_mobile/interfaces/auth.interface.dart'; import 'package:immich_mobile/interfaces/auth_api.interface.dart'; import 'package:immich_mobile/models/auth/auxilary_endpoint.model.dart'; import 'package:immich_mobile/models/auth/login_response.model.dart'; import 'package:immich_mobile/providers/api.provider.dart'; +import 'package:immich_mobile/providers/background_sync.provider.dart'; import 'package:immich_mobile/repositories/auth.repository.dart'; import 'package:immich_mobile/repositories/auth_api.repository.dart'; import 'package:immich_mobile/services/api.service.dart'; @@ -22,6 +24,7 @@ final authServiceProvider = Provider( ref.watch(authRepositoryProvider), ref.watch(apiServiceProvider), ref.watch(networkServiceProvider), + ref.watch(backgroundSyncProvider), ), ); @@ -30,6 +33,7 @@ class AuthService { final IAuthRepository _authRepository; final ApiService _apiService; final NetworkService _networkService; + final BackgroundSyncManager _backgroundSyncManager; final _log = Logger("AuthService"); @@ -38,6 +42,7 @@ class AuthService { this._authRepository, this._apiService, this._networkService, + this._backgroundSyncManager, ); /// Validates the provided server URL by resolving and setting the endpoint. @@ -115,8 +120,10 @@ class AuthService { /// - Asset ETag /// /// All deletions are executed in parallel using [Future.wait]. - Future clearLocalData() { - return Future.wait([ + Future clearLocalData() async { + // Cancel any ongoing background sync operations before clearing data + await _backgroundSyncManager.cancel(); + await Future.wait([ _authRepository.clearLocalData(), Store.delete(StoreKey.currentUser), Store.delete(StoreKey.accessToken), diff --git a/mobile/lib/services/backup.service.dart b/mobile/lib/services/backup.service.dart index a6468f249b..596ad8dc2e 100644 --- a/mobile/lib/services/backup.service.dart +++ b/mobile/lib/services/backup.service.dart @@ -351,7 +351,6 @@ class BackupService { ); baseRequest.headers.addAll(ApiService.getRequestHeaders()); - baseRequest.headers["Transfer-Encoding"] = "chunked"; baseRequest.fields['deviceAssetId'] = asset.localId!; baseRequest.fields['deviceId'] = deviceId; baseRequest.fields['fileCreatedAt'] = diff --git a/mobile/lib/utils/bootstrap.dart b/mobile/lib/utils/bootstrap.dart index 570752c6d9..26f3b49242 100644 --- a/mobile/lib/utils/bootstrap.dart +++ b/mobile/lib/utils/bootstrap.dart @@ -48,11 +48,15 @@ abstract final class Bootstrap { ); } - static Future initDomain(Isar db) async { + static Future initDomain( + Isar db, { + bool shouldBufferLogs = true, + }) async { await StoreService.init(storeRepository: IsarStoreRepository(db)); await LogService.init( logRepository: IsarLogRepository(db), storeRepository: IsarStoreRepository(db), + shouldBuffer: shouldBufferLogs, ); } } diff --git a/mobile/lib/utils/isolate.dart b/mobile/lib/utils/isolate.dart new file mode 100644 index 0000000000..cfbb1b544f --- /dev/null +++ b/mobile/lib/utils/isolate.dart @@ -0,0 +1,69 @@ +import 'dart:async'; +import 'dart:ui'; + +import 'package:flutter/services.dart'; +import 'package:hooks_riverpod/hooks_riverpod.dart'; +import 'package:immich_mobile/providers/db.provider.dart'; +import 'package:immich_mobile/providers/infrastructure/cancel.provider.dart'; +import 'package:immich_mobile/providers/infrastructure/db.provider.dart'; +import 'package:immich_mobile/utils/bootstrap.dart'; +import 'package:logging/logging.dart'; +import 'package:worker_manager/worker_manager.dart'; + +class InvalidIsolateUsageException implements Exception { + const InvalidIsolateUsageException(); + + @override + String toString() => + "IsolateHelper should only be used from the root isolate"; +} + +// !! Should be used only from the root isolate +Cancelable runInIsolateGentle({ + required Future Function(ProviderContainer ref) computation, + String? debugLabel, +}) { + final token = RootIsolateToken.instance; + if (token == null) { + throw const InvalidIsolateUsageException(); + } + + return workerManager.executeGentle((cancelledChecker) async { + BackgroundIsolateBinaryMessenger.ensureInitialized(token); + DartPluginRegistrant.ensureInitialized(); + + final db = await Bootstrap.initIsar(); + await Bootstrap.initDomain(db, shouldBufferLogs: false); + final ref = ProviderContainer( + overrides: [ + // TODO: Remove once isar is removed + dbProvider.overrideWithValue(db), + isarProvider.overrideWithValue(db), + cancellationProvider.overrideWithValue(cancelledChecker), + ], + ); + + Logger log = Logger("IsolateLogger"); + + try { + return await computation(ref); + } on CanceledError { + log.warning( + "Computation cancelled ${debugLabel == null ? '' : ' for $debugLabel'}", + ); + } catch (error, stack) { + log.severe( + "Error in runInIsolateGentle ${debugLabel == null ? '' : ' for $debugLabel'}", + error, + stack, + ); + } finally { + // Wait for the logs to flush + await Future.delayed(const Duration(seconds: 2)); + // Always close the new db connection on Isolate end + ref.read(driftProvider).close(); + ref.read(isarProvider).close(); + } + return null; + }); +} diff --git a/mobile/lib/widgets/common/immich_app_bar.dart b/mobile/lib/widgets/common/immich_app_bar.dart index 51b4faa014..4f95e657d9 100644 --- a/mobile/lib/widgets/common/immich_app_bar.dart +++ b/mobile/lib/widgets/common/immich_app_bar.dart @@ -1,11 +1,13 @@ import 'package:auto_route/auto_route.dart'; import 'package:easy_localization/easy_localization.dart'; +import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; import 'package:flutter_svg/svg.dart'; import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/extensions/build_context_extensions.dart'; import 'package:immich_mobile/models/backup/backup_state.model.dart'; import 'package:immich_mobile/models/server_info/server_info.model.dart'; +import 'package:immich_mobile/providers/background_sync.provider.dart'; import 'package:immich_mobile/providers/backup/backup.provider.dart'; import 'package:immich_mobile/providers/server_info.provider.dart'; import 'package:immich_mobile/providers/user.provider.dart'; @@ -178,6 +180,11 @@ class ImmichAppBar extends ConsumerWidget implements PreferredSizeWidget { child: action, ), ), + if (kDebugMode) + IconButton( + onPressed: () => ref.read(backgroundSyncProvider).sync(), + icon: const Icon(Icons.sync), + ), if (showUploadButton) Padding( padding: const EdgeInsets.only(right: 20), diff --git a/mobile/pubspec.lock b/mobile/pubspec.lock index 3731832296..235b3f71c3 100644 --- a/mobile/pubspec.lock +++ b/mobile/pubspec.lock @@ -1806,7 +1806,7 @@ packages: source: hosted version: "3.1.4" uuid: - dependency: transitive + dependency: "direct main" description: name: uuid sha256: a5be9ef6618a7ac1e964353ef476418026db906c4facdedaa299b7a2e71690ff @@ -1933,6 +1933,14 @@ packages: url: "https://pub.dev" source: hosted version: "0.0.3" + worker_manager: + dependency: "direct main" + description: + name: worker_manager + sha256: "086ed63e9b36266e851404ca90fd44e37c0f4c9bbf819e5f8d7c87f9741c0591" + url: "https://pub.dev" + source: hosted + version: "7.2.3" xdg_directories: dependency: transitive description: diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index 03c39810f6..fdd91e1f87 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -60,7 +60,9 @@ dependencies: thumbhash: 0.1.0+1 timezone: ^0.9.4 url_launcher: ^6.3.1 + uuid: ^4.5.1 wakelock_plus: ^1.2.10 + worker_manager: ^7.2.3 native_video_player: git: diff --git a/mobile/test/api.mocks.dart b/mobile/test/api.mocks.dart index d502ea0675..b0a4e9b8fd 100644 --- a/mobile/test/api.mocks.dart +++ b/mobile/test/api.mocks.dart @@ -2,3 +2,5 @@ import 'package:mocktail/mocktail.dart'; import 'package:openapi/api.dart'; class MockAssetsApi extends Mock implements AssetsApi {} + +class MockSyncApi extends Mock implements SyncApi {} diff --git a/mobile/test/domain/service.mock.dart b/mobile/test/domain/service.mock.dart index 53a173fc28..97a3f30294 100644 --- a/mobile/test/domain/service.mock.dart +++ b/mobile/test/domain/service.mock.dart @@ -1,7 +1,10 @@ import 'package:immich_mobile/domain/services/store.service.dart'; import 'package:immich_mobile/domain/services/user.service.dart'; +import 'package:immich_mobile/domain/utils/background_sync.dart'; import 'package:mocktail/mocktail.dart'; class MockStoreService extends Mock implements StoreService {} class MockUserService extends Mock implements UserService {} + +class MockBackgroundSyncManager extends Mock implements BackgroundSyncManager {} diff --git a/mobile/test/domain/services/sync_stream_service_test.dart b/mobile/test/domain/services/sync_stream_service_test.dart new file mode 100644 index 0000000000..b78a44342b --- /dev/null +++ b/mobile/test/domain/services/sync_stream_service_test.dart @@ -0,0 +1,222 @@ +// ignore_for_file: avoid-declaring-call-method, avoid-unnecessary-futures + +import 'dart:async'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:immich_mobile/domain/interfaces/sync_api.interface.dart'; +import 'package:immich_mobile/domain/interfaces/sync_stream.interface.dart'; +import 'package:immich_mobile/domain/models/sync_event.model.dart'; +import 'package:immich_mobile/domain/services/sync_stream.service.dart'; +import 'package:mocktail/mocktail.dart'; + +import '../../fixtures/sync_stream.stub.dart'; +import '../../infrastructure/repository.mock.dart'; + +class _AbortCallbackWrapper { + const _AbortCallbackWrapper(); + + bool call() => false; +} + +class _MockAbortCallbackWrapper extends Mock implements _AbortCallbackWrapper {} + +class _CancellationWrapper { + const _CancellationWrapper(); + + bool call() => false; +} + +class _MockCancellationWrapper extends Mock implements _CancellationWrapper {} + +void main() { + late SyncStreamService sut; + late ISyncStreamRepository mockSyncStreamRepo; + late ISyncApiRepository mockSyncApiRepo; + late Function(List, Function()) handleEventsCallback; + late _MockAbortCallbackWrapper mockAbortCallbackWrapper; + + successHandler(Invocation _) async => true; + + setUp(() { + mockSyncStreamRepo = MockSyncStreamRepository(); + mockSyncApiRepo = MockSyncApiRepository(); + mockAbortCallbackWrapper = _MockAbortCallbackWrapper(); + + when(() => mockAbortCallbackWrapper()).thenReturn(false); + + when(() => mockSyncApiRepo.streamChanges(any())) + .thenAnswer((invocation) async { + // ignore: avoid-unsafe-collection-methods + handleEventsCallback = invocation.positionalArguments.first; + }); + + when(() => mockSyncApiRepo.ack(any())).thenAnswer((_) async => {}); + + when(() => mockSyncStreamRepo.updateUsersV1(any())) + .thenAnswer(successHandler); + when(() => mockSyncStreamRepo.deleteUsersV1(any())) + .thenAnswer(successHandler); + when(() => mockSyncStreamRepo.updatePartnerV1(any())) + .thenAnswer(successHandler); + when(() => mockSyncStreamRepo.deletePartnerV1(any())) + .thenAnswer(successHandler); + when(() => mockSyncStreamRepo.updateAssetsV1(any())) + .thenAnswer(successHandler); + when(() => mockSyncStreamRepo.deleteAssetsV1(any())) + .thenAnswer(successHandler); + when(() => mockSyncStreamRepo.updateAssetsExifV1(any())) + .thenAnswer(successHandler); + when(() => mockSyncStreamRepo.updatePartnerAssetsV1(any())) + .thenAnswer(successHandler); + when(() => mockSyncStreamRepo.deletePartnerAssetsV1(any())) + .thenAnswer(successHandler); + when(() => mockSyncStreamRepo.updatePartnerAssetsExifV1(any())) + .thenAnswer(successHandler); + + sut = SyncStreamService( + syncApiRepository: mockSyncApiRepo, + syncStreamRepository: mockSyncStreamRepo, + ); + }); + + Future simulateEvents(List events) async { + await sut.sync(); + await handleEventsCallback(events, mockAbortCallbackWrapper.call); + } + + group("SyncStreamService - _handleEvents", () { + test( + "processes events and acks successfully when handlers succeed", + () async { + final events = [ + SyncStreamStub.userDeleteV1, + SyncStreamStub.userV1Admin, + SyncStreamStub.userV1User, + SyncStreamStub.partnerDeleteV1, + SyncStreamStub.partnerV1, + ]; + + await simulateEvents(events); + + verifyInOrder([ + () => mockSyncStreamRepo.deleteUsersV1(any()), + () => mockSyncApiRepo.ack(["2"]), + () => mockSyncStreamRepo.updateUsersV1(any()), + () => mockSyncApiRepo.ack(["5"]), + () => mockSyncStreamRepo.deletePartnerV1(any()), + () => mockSyncApiRepo.ack(["4"]), + () => mockSyncStreamRepo.updatePartnerV1(any()), + () => mockSyncApiRepo.ack(["3"]), + ]); + verifyNever(() => mockAbortCallbackWrapper()); + }, + ); + + test("processes final batch correctly", () async { + final events = [ + SyncStreamStub.userDeleteV1, + SyncStreamStub.userV1Admin, + ]; + + await simulateEvents(events); + + verifyInOrder([ + () => mockSyncStreamRepo.deleteUsersV1(any()), + () => mockSyncApiRepo.ack(["2"]), + () => mockSyncStreamRepo.updateUsersV1(any()), + () => mockSyncApiRepo.ack(["1"]), + ]); + verifyNever(() => mockAbortCallbackWrapper()); + }); + + test("does not process or ack when event list is empty", () async { + await simulateEvents([]); + + verifyNever(() => mockSyncStreamRepo.updateUsersV1(any())); + verifyNever(() => mockSyncStreamRepo.deleteUsersV1(any())); + verifyNever(() => mockSyncStreamRepo.updatePartnerV1(any())); + verifyNever(() => mockSyncStreamRepo.deletePartnerV1(any())); + verifyNever(() => mockAbortCallbackWrapper()); + verifyNever(() => mockSyncApiRepo.ack(any())); + }); + + test("aborts and stops processing if cancelled during iteration", () async { + final cancellationChecker = _MockCancellationWrapper(); + when(() => cancellationChecker()).thenReturn(false); + + sut = SyncStreamService( + syncApiRepository: mockSyncApiRepo, + syncStreamRepository: mockSyncStreamRepo, + cancelChecker: cancellationChecker.call, + ); + await sut.sync(); + + final events = [ + SyncStreamStub.userDeleteV1, + SyncStreamStub.userV1Admin, + SyncStreamStub.partnerDeleteV1, + ]; + + when(() => mockSyncStreamRepo.deleteUsersV1(any())).thenAnswer((_) async { + when(() => cancellationChecker()).thenReturn(true); + }); + + await handleEventsCallback(events, mockAbortCallbackWrapper.call); + + verify(() => mockSyncStreamRepo.deleteUsersV1(any())).called(1); + verifyNever(() => mockSyncStreamRepo.updateUsersV1(any())); + verifyNever(() => mockSyncStreamRepo.deletePartnerV1(any())); + + verify(() => mockAbortCallbackWrapper()).called(1); + + verify(() => mockSyncApiRepo.ack(["2"])).called(1); + }); + + test( + "aborts and stops processing if cancelled before processing batch", + () async { + final cancellationChecker = _MockCancellationWrapper(); + when(() => cancellationChecker()).thenReturn(false); + + final processingCompleter = Completer(); + bool handler1Started = false; + when(() => mockSyncStreamRepo.deleteUsersV1(any())) + .thenAnswer((_) async { + handler1Started = true; + return processingCompleter.future; + }); + + sut = SyncStreamService( + syncApiRepository: mockSyncApiRepo, + syncStreamRepository: mockSyncStreamRepo, + cancelChecker: cancellationChecker.call, + ); + + await sut.sync(); + + final events = [ + SyncStreamStub.userDeleteV1, + SyncStreamStub.userV1Admin, + SyncStreamStub.partnerDeleteV1, + ]; + + final processingFuture = + handleEventsCallback(events, mockAbortCallbackWrapper.call); + await pumpEventQueue(); + + expect(handler1Started, isTrue); + + // Signal cancellation while handler 1 is waiting + when(() => cancellationChecker()).thenReturn(true); + await pumpEventQueue(); + + processingCompleter.complete(); + await processingFuture; + + verifyNever(() => mockSyncStreamRepo.updateUsersV1(any())); + + verify(() => mockSyncApiRepo.ack(["2"])).called(1); + }, + ); + }); +} diff --git a/mobile/test/fixtures/sync_stream.stub.dart b/mobile/test/fixtures/sync_stream.stub.dart new file mode 100644 index 0000000000..ba97f1434a --- /dev/null +++ b/mobile/test/fixtures/sync_stream.stub.dart @@ -0,0 +1,45 @@ +import 'package:immich_mobile/domain/models/sync_event.model.dart'; +import 'package:openapi/api.dart'; + +abstract final class SyncStreamStub { + static final userV1Admin = SyncEvent( + type: SyncEntityType.userV1, + data: SyncUserV1( + deletedAt: DateTime(2020), + email: "admin@admin", + id: "1", + name: "Admin", + ), + ack: "1", + ); + static final userV1User = SyncEvent( + type: SyncEntityType.userV1, + data: SyncUserV1( + deletedAt: DateTime(2021), + email: "user@user", + id: "5", + name: "User", + ), + ack: "5", + ); + static final userDeleteV1 = SyncEvent( + type: SyncEntityType.userDeleteV1, + data: SyncUserDeleteV1(userId: "2"), + ack: "2", + ); + + static final partnerV1 = SyncEvent( + type: SyncEntityType.partnerV1, + data: SyncPartnerV1( + inTimeline: true, + sharedById: "1", + sharedWithId: "2", + ), + ack: "3", + ); + static final partnerDeleteV1 = SyncEvent( + type: SyncEntityType.partnerDeleteV1, + data: SyncPartnerDeleteV1(sharedById: "3", sharedWithId: "4"), + ack: "4", + ); +} diff --git a/mobile/test/infrastructure/repositories/sync_api_repository_test.dart b/mobile/test/infrastructure/repositories/sync_api_repository_test.dart new file mode 100644 index 0000000000..55b03a8116 --- /dev/null +++ b/mobile/test/infrastructure/repositories/sync_api_repository_test.dart @@ -0,0 +1,299 @@ +import 'dart:async'; +import 'dart:convert'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:http/http.dart' as http; +import 'package:immich_mobile/domain/models/sync_event.model.dart'; +import 'package:immich_mobile/infrastructure/repositories/sync_api.repository.dart'; +import 'package:mocktail/mocktail.dart'; +import 'package:openapi/api.dart'; + +import '../../api.mocks.dart'; +import '../../service.mocks.dart'; + +class MockHttpClient extends Mock implements http.Client {} + +class MockApiClient extends Mock implements ApiClient {} + +class MockStreamedResponse extends Mock implements http.StreamedResponse {} + +class FakeBaseRequest extends Fake implements http.BaseRequest {} + +String _createJsonLine(String type, Map data, String ack) { + return '${jsonEncode({'type': type, 'data': data, 'ack': ack})}\n'; +} + +void main() { + late SyncApiRepository sut; + late MockApiService mockApiService; + late MockApiClient mockApiClient; + late MockSyncApi mockSyncApi; + late MockHttpClient mockHttpClient; + late MockStreamedResponse mockStreamedResponse; + late StreamController> responseStreamController; + late int testBatchSize = 3; + + setUp(() { + mockApiService = MockApiService(); + mockApiClient = MockApiClient(); + mockSyncApi = MockSyncApi(); + mockHttpClient = MockHttpClient(); + mockStreamedResponse = MockStreamedResponse(); + responseStreamController = + StreamController>.broadcast(sync: true); + + registerFallbackValue(FakeBaseRequest()); + + when(() => mockApiService.apiClient).thenReturn(mockApiClient); + when(() => mockApiService.syncApi).thenReturn(mockSyncApi); + when(() => mockApiClient.basePath).thenReturn('http://demo.immich.app/api'); + when(() => mockApiService.applyToParams(any(), any())) + .thenAnswer((_) async => {}); + + // Mock HTTP client behavior + when(() => mockHttpClient.send(any())) + .thenAnswer((_) async => mockStreamedResponse); + when(() => mockStreamedResponse.statusCode).thenReturn(200); + when(() => mockStreamedResponse.stream) + .thenAnswer((_) => http.ByteStream(responseStreamController.stream)); + when(() => mockHttpClient.close()).thenAnswer((_) => {}); + + sut = SyncApiRepository(mockApiService); + }); + + tearDown(() async { + if (!responseStreamController.isClosed) { + await responseStreamController.close(); + } + }); + + Future streamChanges( + Function(List, Function() abort) onDataCallback, + ) { + return sut.streamChanges( + onDataCallback, + batchSize: testBatchSize, + httpClient: mockHttpClient, + ); + } + + test('streamChanges stops processing stream when abort is called', () async { + int onDataCallCount = 0; + bool abortWasCalledInCallback = false; + List receivedEventsBatch1 = []; + + onDataCallback(List events, Function() abort) { + onDataCallCount++; + if (onDataCallCount == 1) { + receivedEventsBatch1 = events; + abort(); + abortWasCalledInCallback = true; + } else { + fail("onData called more than once after abort was invoked"); + } + } + + final streamChangesFuture = streamChanges(onDataCallback); + + await pumpEventQueue(); + + for (int i = 0; i < testBatchSize; i++) { + responseStreamController.add( + utf8.encode( + _createJsonLine( + SyncEntityType.userDeleteV1.toString(), + SyncUserDeleteV1(userId: "user$i").toJson(), + 'ack$i', + ), + ), + ); + } + + for (int i = testBatchSize; i < testBatchSize * 2; i++) { + responseStreamController.add( + utf8.encode( + _createJsonLine( + SyncEntityType.userDeleteV1.toString(), + SyncUserDeleteV1(userId: "user$i").toJson(), + 'ack$i', + ), + ), + ); + } + + await responseStreamController.close(); + await expectLater(streamChangesFuture, completes); + + expect(onDataCallCount, 1); + expect(abortWasCalledInCallback, isTrue); + expect(receivedEventsBatch1.length, testBatchSize); + verify(() => mockHttpClient.close()).called(1); + }); + + test( + 'streamChanges does not process remaining lines in finally block if aborted', + () async { + int onDataCallCount = 0; + bool abortWasCalledInCallback = false; + + onDataCallback(List events, Function() abort) { + onDataCallCount++; + if (onDataCallCount == 1) { + abort(); + abortWasCalledInCallback = true; + } else { + fail("onData called more than once after abort was invoked"); + } + } + + final streamChangesFuture = streamChanges(onDataCallback); + + await pumpEventQueue(); + + for (int i = 0; i < testBatchSize; i++) { + responseStreamController.add( + utf8.encode( + _createJsonLine( + SyncEntityType.userDeleteV1.toString(), + SyncUserDeleteV1(userId: "user$i").toJson(), + 'ack$i', + ), + ), + ); + } + + // emit a single event to skip batching and trigger finally + responseStreamController.add( + utf8.encode( + _createJsonLine( + SyncEntityType.userDeleteV1.toString(), + SyncUserDeleteV1(userId: "user100").toJson(), + 'ack100', + ), + ), + ); + + await responseStreamController.close(); + await expectLater(streamChangesFuture, completes); + + expect(onDataCallCount, 1); + expect(abortWasCalledInCallback, isTrue); + verify(() => mockHttpClient.close()).called(1); + }, + ); + + test( + 'streamChanges processes remaining lines in finally block if not aborted', + () async { + int onDataCallCount = 0; + List receivedEventsBatch1 = []; + List receivedEventsBatch2 = []; + + onDataCallback(List events, Function() _) { + onDataCallCount++; + if (onDataCallCount == 1) { + receivedEventsBatch1 = events; + } else if (onDataCallCount == 2) { + receivedEventsBatch2 = events; + } else { + fail("onData called more than expected"); + } + } + + final streamChangesFuture = streamChanges(onDataCallback); + + await pumpEventQueue(); + + // Batch 1 + for (int i = 0; i < testBatchSize; i++) { + responseStreamController.add( + utf8.encode( + _createJsonLine( + SyncEntityType.userDeleteV1.toString(), + SyncUserDeleteV1(userId: "user$i").toJson(), + 'ack$i', + ), + ), + ); + } + + // Partial Batch 2 + responseStreamController.add( + utf8.encode( + _createJsonLine( + SyncEntityType.userDeleteV1.toString(), + SyncUserDeleteV1(userId: "user100").toJson(), + 'ack100', + ), + ), + ); + + await responseStreamController.close(); + await expectLater(streamChangesFuture, completes); + + expect(onDataCallCount, 2); + expect(receivedEventsBatch1.length, testBatchSize); + expect(receivedEventsBatch2.length, 1); + verify(() => mockHttpClient.close()).called(1); + }, + ); + + test('streamChanges handles stream error gracefully', () async { + final streamError = Exception("Network Error"); + int onDataCallCount = 0; + + onDataCallback(List events, Function() _) { + onDataCallCount++; + } + + final streamChangesFuture = streamChanges(onDataCallback); + + await pumpEventQueue(); + + responseStreamController.add( + utf8.encode( + _createJsonLine( + SyncEntityType.userDeleteV1.toString(), + SyncUserDeleteV1(userId: "user1").toJson(), + 'ack1', + ), + ), + ); + + responseStreamController.addError(streamError); + await expectLater(streamChangesFuture, throwsA(streamError)); + + expect(onDataCallCount, 0); + verify(() => mockHttpClient.close()).called(1); + }); + + test('streamChanges throws ApiException on non-200 status code', () async { + when(() => mockStreamedResponse.statusCode).thenReturn(401); + final errorBodyController = StreamController>(sync: true); + when(() => mockStreamedResponse.stream) + .thenAnswer((_) => http.ByteStream(errorBodyController.stream)); + + int onDataCallCount = 0; + + onDataCallback(List events, Function() _) { + onDataCallCount++; + } + + final future = streamChanges(onDataCallback); + + errorBodyController.add(utf8.encode('{"error":"Unauthorized"}')); + await errorBodyController.close(); + + await expectLater( + future, + throwsA( + isA() + .having((e) => e.code, 'code', 401) + .having((e) => e.message, 'message', contains('Unauthorized')), + ), + ); + + expect(onDataCallCount, 0); + verify(() => mockHttpClient.close()).called(1); + }); +} diff --git a/mobile/test/infrastructure/repository.mock.dart b/mobile/test/infrastructure/repository.mock.dart index 192858adff..c4a5680f71 100644 --- a/mobile/test/infrastructure/repository.mock.dart +++ b/mobile/test/infrastructure/repository.mock.dart @@ -1,6 +1,8 @@ import 'package:immich_mobile/domain/interfaces/device_asset.interface.dart'; import 'package:immich_mobile/domain/interfaces/log.interface.dart'; import 'package:immich_mobile/domain/interfaces/store.interface.dart'; +import 'package:immich_mobile/domain/interfaces/sync_api.interface.dart'; +import 'package:immich_mobile/domain/interfaces/sync_stream.interface.dart'; import 'package:immich_mobile/domain/interfaces/user.interface.dart'; import 'package:immich_mobile/domain/interfaces/user_api.interface.dart'; import 'package:mocktail/mocktail.dart'; @@ -14,5 +16,9 @@ class MockUserRepository extends Mock implements IUserRepository {} class MockDeviceAssetRepository extends Mock implements IDeviceAssetRepository {} +class MockSyncStreamRepository extends Mock implements ISyncStreamRepository {} + // API Repos class MockUserApiRepository extends Mock implements IUserApiRepository {} + +class MockSyncApiRepository extends Mock implements ISyncApiRepository {} diff --git a/mobile/test/service.mocks.dart b/mobile/test/service.mocks.dart index e1b8df40a3..87a8c01cf0 100644 --- a/mobile/test/service.mocks.dart +++ b/mobile/test/service.mocks.dart @@ -29,4 +29,3 @@ class MockSearchApi extends Mock implements SearchApi {} class MockAppSettingService extends Mock implements AppSettingsService {} class MockBackgroundService extends Mock implements BackgroundService {} - diff --git a/mobile/test/services/auth.service_test.dart b/mobile/test/services/auth.service_test.dart index e4f011d940..4ada98a6c9 100644 --- a/mobile/test/services/auth.service_test.dart +++ b/mobile/test/services/auth.service_test.dart @@ -8,6 +8,7 @@ import 'package:isar/isar.dart'; import 'package:mocktail/mocktail.dart'; import 'package:openapi/api.dart'; +import '../domain/service.mock.dart'; import '../repository.mocks.dart'; import '../service.mocks.dart'; import '../test_utils.dart'; @@ -18,6 +19,7 @@ void main() { late MockAuthRepository authRepository; late MockApiService apiService; late MockNetworkService networkService; + late MockBackgroundSyncManager backgroundSyncManager; late Isar db; setUp(() async { @@ -25,12 +27,14 @@ void main() { authRepository = MockAuthRepository(); apiService = MockApiService(); networkService = MockNetworkService(); + backgroundSyncManager = MockBackgroundSyncManager(); sut = AuthService( authApiRepository, authRepository, apiService, networkService, + backgroundSyncManager, ); registerFallbackValue(Uri()); @@ -116,24 +120,28 @@ void main() { group('logout', () { test('Should logout user', () async { when(() => authApiRepository.logout()).thenAnswer((_) async => {}); + when(() => backgroundSyncManager.cancel()).thenAnswer((_) async => {}); when(() => authRepository.clearLocalData()) .thenAnswer((_) => Future.value(null)); await sut.logout(); verify(() => authApiRepository.logout()).called(1); + verify(() => backgroundSyncManager.cancel()).called(1); verify(() => authRepository.clearLocalData()).called(1); }); test('Should clear local data even on server error', () async { when(() => authApiRepository.logout()) .thenThrow(Exception('Server error')); + when(() => backgroundSyncManager.cancel()).thenAnswer((_) async => {}); when(() => authRepository.clearLocalData()) .thenAnswer((_) => Future.value(null)); await sut.logout(); verify(() => authApiRepository.logout()).called(1); + verify(() => backgroundSyncManager.cancel()).called(1); verify(() => authRepository.clearLocalData()).called(1); }); }); diff --git a/server/package.json b/server/package.json index 76415da7c8..8e149d961e 100644 --- a/server/package.json +++ b/server/package.json @@ -26,9 +26,8 @@ "migrations:generate": "node ./dist/bin/migrations.js generate", "migrations:create": "node ./dist/bin/migrations.js create", "migrations:run": "node ./dist/bin/migrations.js run", - "typeorm:migrations:revert": "typeorm migration:revert -d ./dist/bin/database.js", - "typeorm:schema:drop": "typeorm query -d ./dist/bin/database.js 'DROP schema public cascade; CREATE schema public;'", - "typeorm:schema:reset": "npm run typeorm:schema:drop && npm run migrations:run", + "schema:drop": "node ./dist/bin/migrations.js query 'DROP schema public cascade; CREATE schema public;'", + "schema:reset": "npm run schema:drop && npm run migrations:run", "kysely:codegen": "npx kysely-codegen --include-pattern=\"(public|vectors).*\" --dialect postgres --url postgres://postgres:postgres@localhost/immich --log-level debug --out-file=./src/db.d.ts", "sync:open-api": "node ./dist/bin/sync-open-api.js", "sync:sql": "node ./dist/bin/sync-sql.js", diff --git a/server/src/bin/database.ts b/server/src/bin/database.ts deleted file mode 100644 index 7ea56e0fc0..0000000000 --- a/server/src/bin/database.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { ConfigRepository } from 'src/repositories/config.repository'; -import { DataSource } from 'typeorm'; - -const { database } = new ConfigRepository().getEnv(); - -/** - * @deprecated - DO NOT USE THIS - * - * this export is ONLY to be used for TypeORM commands in package.json#scripts - */ -export const dataSource = new DataSource({ ...database.config.typeorm, host: 'localhost' }); diff --git a/server/src/bin/migrations.ts b/server/src/bin/migrations.ts index fb50323dff..2ddc6776fb 100644 --- a/server/src/bin/migrations.ts +++ b/server/src/bin/migrations.ts @@ -1,8 +1,8 @@ #!/usr/bin/env node process.env.DB_URL = process.env.DB_URL || 'postgres://postgres:postgres@localhost:5432/immich'; -import { Kysely } from 'kysely'; -import { writeFileSync } from 'node:fs'; +import { Kysely, sql } from 'kysely'; +import { mkdirSync, writeFileSync } from 'node:fs'; import { basename, dirname, extname, join } from 'node:path'; import postgres from 'postgres'; import { ConfigRepository } from 'src/repositories/config.repository'; @@ -23,8 +23,13 @@ const main = async () => { } case 'run': { - const only = process.argv[3] as 'kysely' | 'typeorm' | undefined; - await run(only); + await runMigrations(); + return; + } + + case 'query': { + const query = process.argv[3]; + await runQuery(query); return; } @@ -48,14 +53,25 @@ const main = async () => { } }; -const run = async (only?: 'kysely' | 'typeorm') => { +const getDatabaseClient = () => { const configRepository = new ConfigRepository(); const { database } = configRepository.getEnv(); - const logger = new LoggingRepository(undefined, configRepository); - const db = new Kysely(getKyselyConfig(database.config.kysely)); - const databaseRepository = new DatabaseRepository(db, logger, configRepository); + return new Kysely(getKyselyConfig(database.config.kysely)); +}; - await databaseRepository.runMigrations({ only }); +const runQuery = async (query: string) => { + const db = getDatabaseClient(); + await sql.raw(query).execute(db); + await db.destroy(); +}; + +const runMigrations = async () => { + const configRepository = new ConfigRepository(); + const logger = new LoggingRepository(undefined, configRepository); + const db = getDatabaseClient(); + const databaseRepository = new DatabaseRepository(db, logger, configRepository); + await databaseRepository.runMigrations(); + await db.destroy(); }; const debug = async () => { @@ -81,7 +97,8 @@ const create = (path: string, up: string[], down: string[]) => { const filename = `${timestamp}-${name}.ts`; const folder = dirname(path); const fullPath = join(folder, filename); - writeFileSync(fullPath, asMigration('typeorm', { name, timestamp, up, down })); + mkdirSync(folder, { recursive: true }); + writeFileSync(fullPath, asMigration('kysely', { name, timestamp, up, down })); console.log(`Wrote ${fullPath}`); }; diff --git a/server/src/database.ts b/server/src/database.ts index b504d39579..27094958ed 100644 --- a/server/src/database.ts +++ b/server/src/database.ts @@ -1,13 +1,13 @@ import { Selectable } from 'kysely'; -import { AssetJobStatus as DatabaseAssetJobStatus, Exif as DatabaseExif } from 'src/db'; -import { AssetEntity } from 'src/entities/asset.entity'; +import { Albums, Exif as DatabaseExif } from 'src/db'; +import { MapAsset } from 'src/dtos/asset-response.dto'; import { AlbumUserRole, AssetFileType, - AssetStatus, AssetType, MemoryType, Permission, + SharedLinkType, SourceType, UserStatus, } from 'src/enum'; @@ -44,7 +44,7 @@ export type Library = { exclusionPatterns: string[]; deletedAt: Date | null; refreshedAt: Date | null; - assets?: Asset[]; + assets?: MapAsset[]; }; export type AuthApiKey = { @@ -96,7 +96,26 @@ export type Memory = { data: OnThisDayData; ownerId: string; isSaved: boolean; - assets: Asset[]; + assets: MapAsset[]; +}; + +export type Asset = { + id: string; + checksum: Buffer; + deviceAssetId: string; + deviceId: string; + fileCreatedAt: Date; + fileModifiedAt: Date; + isExternal: boolean; + isVisible: boolean; + libraryId: string | null; + livePhotoVideoId: string | null; + localDateTime: Date; + originalFileName: string; + originalPath: string; + ownerId: string; + sidecarPath: string | null; + type: AssetType; }; export type User = { @@ -128,39 +147,6 @@ export type StorageAsset = { encodedVideoPath: string | null; }; -export type Asset = { - createdAt: Date; - updatedAt: Date; - deletedAt: Date | null; - id: string; - updateId: string; - status: AssetStatus; - checksum: Buffer; - deviceAssetId: string; - deviceId: string; - duplicateId: string | null; - duration: string | null; - encodedVideoPath: string | null; - fileCreatedAt: Date | null; - fileModifiedAt: Date | null; - isArchived: boolean; - isExternal: boolean; - isFavorite: boolean; - isOffline: boolean; - isVisible: boolean; - libraryId: string | null; - livePhotoVideoId: string | null; - localDateTime: Date | null; - originalFileName: string; - originalPath: string; - ownerId: string; - sidecarPath: string | null; - stack?: Stack | null; - stackId: string | null; - thumbhash: Buffer | null; - type: AssetType; -}; - export type SidecarWriteAsset = { id: string; sidecarPath: string | null; @@ -173,7 +159,7 @@ export type Stack = { primaryAssetId: string; owner?: User; ownerId: string; - assets: AssetEntity[]; + assets: MapAsset[]; assetCount?: number; }; @@ -187,6 +173,28 @@ export type AuthSharedLink = { password: string | null; }; +export type SharedLink = { + id: string; + album?: Album | null; + albumId: string | null; + allowDownload: boolean; + allowUpload: boolean; + assets: MapAsset[]; + createdAt: Date; + description: string | null; + expiresAt: Date | null; + key: Buffer; + password: string | null; + showExif: boolean; + type: SharedLinkType; + userId: string; +}; + +export type Album = Selectable & { + owner: User; + assets: MapAsset[]; +}; + export type AuthSession = { id: string; }; @@ -256,10 +264,6 @@ export type AssetFace = { person?: Person | null; }; -export type AssetJobStatus = Selectable & { - asset: AssetEntity; -}; - const userColumns = ['id', 'name', 'email', 'profileImagePath', 'profileChangedAt'] as const; export const columns = { diff --git a/server/src/db.d.ts b/server/src/db.d.ts index a95beaa82f..4e9738ecec 100644 --- a/server/src/db.d.ts +++ b/server/src/db.d.ts @@ -143,8 +143,8 @@ export interface Assets { duplicateId: string | null; duration: string | null; encodedVideoPath: Generated; - fileCreatedAt: Timestamp | null; - fileModifiedAt: Timestamp | null; + fileCreatedAt: Timestamp; + fileModifiedAt: Timestamp; id: Generated; isArchived: Generated; isExternal: Generated; diff --git a/server/src/decorators.ts b/server/src/decorators.ts index 7085899af7..1af9342e0b 100644 --- a/server/src/decorators.ts +++ b/server/src/decorators.ts @@ -11,7 +11,8 @@ import { setUnion } from 'src/utils/set'; const GeneratedUuidV7Column = (options: Omit = {}) => Column({ ...options, type: 'uuid', nullable: false, default: () => `${immich_uuid_v7.name}()` }); -export const UpdateIdColumn = () => GeneratedUuidV7Column(); +export const UpdateIdColumn = (options: Omit = {}) => + GeneratedUuidV7Column(options); export const PrimaryGeneratedUuidV7Column = () => GeneratedUuidV7Column({ primary: true }); diff --git a/server/src/dtos/album.dto.ts b/server/src/dtos/album.dto.ts index c9934ec909..40e51ef729 100644 --- a/server/src/dtos/album.dto.ts +++ b/server/src/dtos/album.dto.ts @@ -2,10 +2,10 @@ import { ApiProperty } from '@nestjs/swagger'; import { Type } from 'class-transformer'; import { ArrayNotEmpty, IsArray, IsEnum, IsString, ValidateNested } from 'class-validator'; import _ from 'lodash'; -import { AssetResponseDto, mapAsset } from 'src/dtos/asset-response.dto'; +import { AlbumUser, AuthSharedLink, User } from 'src/database'; +import { AssetResponseDto, MapAsset, mapAsset } from 'src/dtos/asset-response.dto'; import { AuthDto } from 'src/dtos/auth.dto'; import { UserResponseDto, mapUser } from 'src/dtos/user.dto'; -import { AlbumEntity } from 'src/entities/album.entity'; import { AlbumUserRole, AssetOrder } from 'src/enum'; import { Optional, ValidateBoolean, ValidateUUID } from 'src/validation'; @@ -142,7 +142,23 @@ export class AlbumResponseDto { order?: AssetOrder; } -export const mapAlbum = (entity: AlbumEntity, withAssets: boolean, auth?: AuthDto): AlbumResponseDto => { +export type MapAlbumDto = { + albumUsers?: AlbumUser[]; + assets?: MapAsset[]; + sharedLinks?: AuthSharedLink[]; + albumName: string; + description: string; + albumThumbnailAssetId: string | null; + createdAt: Date; + updatedAt: Date; + id: string; + ownerId: string; + owner: User; + isActivityEnabled: boolean; + order: AssetOrder; +}; + +export const mapAlbum = (entity: MapAlbumDto, withAssets: boolean, auth?: AuthDto): AlbumResponseDto => { const albumUsers: AlbumUserResponseDto[] = []; if (entity.albumUsers) { @@ -159,7 +175,7 @@ export const mapAlbum = (entity: AlbumEntity, withAssets: boolean, auth?: AuthDt const assets = entity.assets || []; - const hasSharedLink = entity.sharedLinks?.length > 0; + const hasSharedLink = !!entity.sharedLinks && entity.sharedLinks.length > 0; const hasSharedUser = albumUsers.length > 0; let startDate = assets.at(0)?.localDateTime; @@ -190,5 +206,5 @@ export const mapAlbum = (entity: AlbumEntity, withAssets: boolean, auth?: AuthDt }; }; -export const mapAlbumWithAssets = (entity: AlbumEntity) => mapAlbum(entity, true); -export const mapAlbumWithoutAssets = (entity: AlbumEntity) => mapAlbum(entity, false); +export const mapAlbumWithAssets = (entity: MapAlbumDto) => mapAlbum(entity, true); +export const mapAlbumWithoutAssets = (entity: MapAlbumDto) => mapAlbum(entity, false); diff --git a/server/src/dtos/asset-response.dto.ts b/server/src/dtos/asset-response.dto.ts index 5640dbb26f..086ee0f668 100644 --- a/server/src/dtos/asset-response.dto.ts +++ b/server/src/dtos/asset-response.dto.ts @@ -1,5 +1,6 @@ import { ApiProperty } from '@nestjs/swagger'; -import { AssetFace } from 'src/database'; +import { Selectable } from 'kysely'; +import { AssetFace, AssetFile, Exif, Stack, Tag, User } from 'src/database'; import { PropertyLifecycle } from 'src/decorators'; import { AuthDto } from 'src/dtos/auth.dto'; import { ExifResponseDto, mapExif } from 'src/dtos/exif.dto'; @@ -11,8 +12,7 @@ import { } from 'src/dtos/person.dto'; import { TagResponseDto, mapTag } from 'src/dtos/tag.dto'; import { UserResponseDto, mapUser } from 'src/dtos/user.dto'; -import { AssetEntity } from 'src/entities/asset.entity'; -import { AssetType } from 'src/enum'; +import { AssetStatus, AssetType } from 'src/enum'; import { hexOrBufferToBase64 } from 'src/utils/bytes'; import { mimeTypes } from 'src/utils/mime-types'; @@ -57,6 +57,44 @@ export class AssetResponseDto extends SanitizedAssetResponseDto { resized?: boolean; } +export type MapAsset = { + createdAt: Date; + updatedAt: Date; + deletedAt: Date | null; + id: string; + updateId: string; + status: AssetStatus; + checksum: Buffer; + deviceAssetId: string; + deviceId: string; + duplicateId: string | null; + duration: string | null; + encodedVideoPath: string | null; + exifInfo?: Selectable | null; + faces?: AssetFace[]; + fileCreatedAt: Date; + fileModifiedAt: Date; + files?: AssetFile[]; + isArchived: boolean; + isExternal: boolean; + isFavorite: boolean; + isOffline: boolean; + isVisible: boolean; + libraryId: string | null; + livePhotoVideoId: string | null; + localDateTime: Date; + originalFileName: string; + originalPath: string; + owner?: User | null; + ownerId: string; + sidecarPath: string | null; + stack?: Stack | null; + stackId: string | null; + tags?: Tag[]; + thumbhash: Buffer | null; + type: AssetType; +}; + export class AssetStackResponseDto { id!: string; @@ -73,7 +111,7 @@ export type AssetMapOptions = { }; // TODO: this is inefficient -const peopleWithFaces = (faces: AssetFace[]): PersonWithFacesResponseDto[] => { +const peopleWithFaces = (faces?: AssetFace[]): PersonWithFacesResponseDto[] => { const result: PersonWithFacesResponseDto[] = []; if (faces) { for (const face of faces) { @@ -91,7 +129,7 @@ const peopleWithFaces = (faces: AssetFace[]): PersonWithFacesResponseDto[] => { return result; }; -const mapStack = (entity: AssetEntity) => { +const mapStack = (entity: { stack?: Stack | null }) => { if (!entity.stack) { return null; } @@ -103,7 +141,7 @@ const mapStack = (entity: AssetEntity) => { }; }; -export function mapAsset(entity: AssetEntity, options: AssetMapOptions = {}): AssetResponseDto { +export function mapAsset(entity: MapAsset, options: AssetMapOptions = {}): AssetResponseDto { const { stripMetadata = false, withStack = false } = options; if (stripMetadata) { diff --git a/server/src/dtos/memory.dto.ts b/server/src/dtos/memory.dto.ts index b3054d7a4c..98231a9035 100644 --- a/server/src/dtos/memory.dto.ts +++ b/server/src/dtos/memory.dto.ts @@ -4,7 +4,6 @@ import { IsEnum, IsInt, IsObject, IsPositive, ValidateNested } from 'class-valid import { Memory } from 'src/database'; import { AssetResponseDto, mapAsset } from 'src/dtos/asset-response.dto'; import { AuthDto } from 'src/dtos/auth.dto'; -import { AssetEntity } from 'src/entities/asset.entity'; import { MemoryType } from 'src/enum'; import { Optional, ValidateBoolean, ValidateDate, ValidateUUID } from 'src/validation'; @@ -103,6 +102,6 @@ export const mapMemory = (entity: Memory, auth: AuthDto): MemoryResponseDto => { type: entity.type as MemoryType, data: entity.data as unknown as MemoryData, isSaved: entity.isSaved, - assets: ('assets' in entity ? entity.assets : []).map((asset) => mapAsset(asset as AssetEntity, { auth })), + assets: ('assets' in entity ? entity.assets : []).map((asset) => mapAsset(asset, { auth })), }; }; diff --git a/server/src/dtos/shared-link.dto.ts b/server/src/dtos/shared-link.dto.ts index 6bb8ab1f0d..8d373b40b6 100644 --- a/server/src/dtos/shared-link.dto.ts +++ b/server/src/dtos/shared-link.dto.ts @@ -1,9 +1,9 @@ import { ApiProperty } from '@nestjs/swagger'; import { IsEnum, IsString } from 'class-validator'; import _ from 'lodash'; +import { SharedLink } from 'src/database'; import { AlbumResponseDto, mapAlbumWithoutAssets } from 'src/dtos/album.dto'; import { AssetResponseDto, mapAsset } from 'src/dtos/asset-response.dto'; -import { SharedLinkEntity } from 'src/entities/shared-link.entity'; import { SharedLinkType } from 'src/enum'; import { Optional, ValidateBoolean, ValidateDate, ValidateUUID } from 'src/validation'; @@ -102,7 +102,7 @@ export class SharedLinkResponseDto { showMetadata!: boolean; } -export function mapSharedLink(sharedLink: SharedLinkEntity): SharedLinkResponseDto { +export function mapSharedLink(sharedLink: SharedLink): SharedLinkResponseDto { const linkAssets = sharedLink.assets || []; return { @@ -122,7 +122,7 @@ export function mapSharedLink(sharedLink: SharedLinkEntity): SharedLinkResponseD }; } -export function mapSharedLinkWithoutMetadata(sharedLink: SharedLinkEntity): SharedLinkResponseDto { +export function mapSharedLinkWithoutMetadata(sharedLink: SharedLink): SharedLinkResponseDto { const linkAssets = sharedLink.assets || []; const albumAssets = (sharedLink?.album?.assets || []).map((asset) => asset); @@ -137,7 +137,7 @@ export function mapSharedLinkWithoutMetadata(sharedLink: SharedLinkEntity): Shar type: sharedLink.type, createdAt: sharedLink.createdAt, expiresAt: sharedLink.expiresAt, - assets: assets.map((asset) => mapAsset(asset, { stripMetadata: true })) as AssetResponseDto[], + assets: assets.map((asset) => mapAsset(asset, { stripMetadata: true })), album: sharedLink.album ? mapAlbumWithoutAssets(sharedLink.album) : undefined, allowUpload: sharedLink.allowUpload, allowDownload: sharedLink.allowDownload, diff --git a/server/src/entities/album.entity.ts b/server/src/entities/album.entity.ts deleted file mode 100644 index eb20c1afdd..0000000000 --- a/server/src/entities/album.entity.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { AlbumUser, User } from 'src/database'; -import { AssetEntity } from 'src/entities/asset.entity'; -import { SharedLinkEntity } from 'src/entities/shared-link.entity'; -import { AssetOrder } from 'src/enum'; - -export class AlbumEntity { - id!: string; - owner!: User; - ownerId!: string; - albumName!: string; - description!: string; - createdAt!: Date; - updatedAt!: Date; - updateId?: string; - deletedAt!: Date | null; - albumThumbnailAsset!: AssetEntity | null; - albumThumbnailAssetId!: string | null; - albumUsers!: AlbumUser[]; - assets!: AssetEntity[]; - sharedLinks!: SharedLinkEntity[]; - isActivityEnabled!: boolean; - order!: AssetOrder; -} diff --git a/server/src/entities/asset.entity.ts b/server/src/entities/asset.entity.ts deleted file mode 100644 index 4d5cdb1529..0000000000 --- a/server/src/entities/asset.entity.ts +++ /dev/null @@ -1,311 +0,0 @@ -import { - DeduplicateJoinsPlugin, - Expression, - expressionBuilder, - ExpressionBuilder, - Kysely, - SelectQueryBuilder, - sql, -} from 'kysely'; -import { jsonArrayFrom, jsonObjectFrom } from 'kysely/helpers/postgres'; -import { AssetFace, AssetFile, AssetJobStatus, columns, Exif, Stack, Tag, User } from 'src/database'; -import { DB } from 'src/db'; -import { SharedLinkEntity } from 'src/entities/shared-link.entity'; -import { AssetFileType, AssetStatus, AssetType } from 'src/enum'; -import { TimeBucketSize } from 'src/repositories/asset.repository'; -import { AssetSearchBuilderOptions } from 'src/repositories/search.repository'; -import { anyUuid, asUuid } from 'src/utils/database'; - -export const ASSET_CHECKSUM_CONSTRAINT = 'UQ_assets_owner_checksum'; - -export class AssetEntity { - id!: string; - deviceAssetId!: string; - owner!: User; - ownerId!: string; - libraryId?: string | null; - deviceId!: string; - type!: AssetType; - status!: AssetStatus; - originalPath!: string; - files!: AssetFile[]; - thumbhash!: Buffer | null; - encodedVideoPath!: string | null; - createdAt!: Date; - updatedAt!: Date; - updateId?: string; - deletedAt!: Date | null; - fileCreatedAt!: Date; - localDateTime!: Date; - fileModifiedAt!: Date; - isFavorite!: boolean; - isArchived!: boolean; - isExternal!: boolean; - isOffline!: boolean; - checksum!: Buffer; // sha1 checksum - duration!: string | null; - isVisible!: boolean; - livePhotoVideo!: AssetEntity | null; - livePhotoVideoId!: string | null; - originalFileName!: string; - sidecarPath!: string | null; - exifInfo?: Exif; - tags?: Tag[]; - sharedLinks!: SharedLinkEntity[]; - faces!: AssetFace[]; - stackId?: string | null; - stack?: Stack | null; - jobStatus?: AssetJobStatus; - duplicateId!: string | null; -} - -export function withExif(qb: SelectQueryBuilder) { - return qb - .leftJoin('exif', 'assets.id', 'exif.assetId') - .select((eb) => eb.fn.toJson(eb.table('exif')).$castTo().as('exifInfo')); -} - -export function withExifInner(qb: SelectQueryBuilder) { - return qb - .innerJoin('exif', 'assets.id', 'exif.assetId') - .select((eb) => eb.fn.toJson(eb.table('exif')).$castTo().as('exifInfo')); -} - -export function withSmartSearch(qb: SelectQueryBuilder) { - return qb - .leftJoin('smart_search', 'assets.id', 'smart_search.assetId') - .select((eb) => eb.fn.toJson(eb.table('smart_search')).as('smartSearch')); -} - -export function withFaces(eb: ExpressionBuilder, withDeletedFace?: boolean) { - return jsonArrayFrom( - eb - .selectFrom('asset_faces') - .selectAll('asset_faces') - .whereRef('asset_faces.assetId', '=', 'assets.id') - .$if(!withDeletedFace, (qb) => qb.where('asset_faces.deletedAt', 'is', null)), - ).as('faces'); -} - -export function withFiles(eb: ExpressionBuilder, type?: AssetFileType) { - return jsonArrayFrom( - eb - .selectFrom('asset_files') - .select(columns.assetFiles) - .whereRef('asset_files.assetId', '=', 'assets.id') - .$if(!!type, (qb) => qb.where('asset_files.type', '=', type!)), - ).as('files'); -} - -export function withFacesAndPeople(eb: ExpressionBuilder, withDeletedFace?: boolean) { - return jsonArrayFrom( - eb - .selectFrom('asset_faces') - .leftJoinLateral( - (eb) => - eb.selectFrom('person').selectAll('person').whereRef('asset_faces.personId', '=', 'person.id').as('person'), - (join) => join.onTrue(), - ) - .selectAll('asset_faces') - .select((eb) => eb.table('person').as('person')) - .whereRef('asset_faces.assetId', '=', 'assets.id') - .$if(!withDeletedFace, (qb) => qb.where('asset_faces.deletedAt', 'is', null)), - ).as('faces'); -} - -// export function hasPeople(qb: SelectQueryBuilder, personIds: string[]) { -// return qb.innerJoin( -// (eb) => -// eb -// .selectFrom('asset_faces') -// .select('assetId') -// .where('personId', '=', anyUuid(personIds!)) -// .where('deletedAt', 'is', null) -// .groupBy('assetId') -// .having((eb) => eb.fn.count('personId').distinct(), '=', personIds.length) -// .as('has_people'), -// (join) => join.onRef('has_people.assetId', '=', 'assets.id'), -// ); -// } -export function hasPeople(personIds: string[]) { - const eb = expressionBuilder(); - return eb - .selectFrom('asset_faces') - .select('assetId') - .where('personId', '=', anyUuid(personIds!)) - .where('deletedAt', 'is', null) - .groupBy('assetId') - .having((eb) => eb.fn.count('personId').distinct(), '=', personIds.length) - .as('has_people'); -} - -export function hasTags(qb: SelectQueryBuilder, tagIds: string[]) { - return qb.innerJoin( - (eb) => - eb - .selectFrom('tag_asset') - .select('assetsId') - .innerJoin('tags_closure', 'tag_asset.tagsId', 'tags_closure.id_descendant') - .where('tags_closure.id_ancestor', '=', anyUuid(tagIds)) - .groupBy('assetsId') - .having((eb) => eb.fn.count('tags_closure.id_ancestor').distinct(), '>=', tagIds.length) - .as('has_tags'), - (join) => join.onRef('has_tags.assetsId', '=', 'assets.id'), - ); -} - -export function withOwner(eb: ExpressionBuilder) { - return jsonObjectFrom(eb.selectFrom('users').selectAll().whereRef('users.id', '=', 'assets.ownerId')).as('owner'); -} - -export function withLibrary(eb: ExpressionBuilder) { - return jsonObjectFrom(eb.selectFrom('libraries').selectAll().whereRef('libraries.id', '=', 'assets.libraryId')).as( - 'library', - ); -} - -export function withTags(eb: ExpressionBuilder) { - return jsonArrayFrom( - eb - .selectFrom('tags') - .select(columns.tag) - .innerJoin('tag_asset', 'tags.id', 'tag_asset.tagsId') - .whereRef('assets.id', '=', 'tag_asset.assetsId'), - ).as('tags'); -} - -export function truncatedDate(size: TimeBucketSize) { - return sql`date_trunc(${size}, "localDateTime" at time zone 'UTC') at time zone 'UTC'`; -} - -// export function withTagId(qb: SelectQueryBuilder, tagId: string) { -// return qb.where((eb) => -// eb.exists( -// eb -// .selectFrom('tags_closure') -// .innerJoin('tag_asset', 'tag_asset.tagsId', 'tags_closure.id_descendant') -// .whereRef('tag_asset.assetsId', '=', 'assets.id') -// .where('tags_closure.id_ancestor', '=', tagId), -// ), -// ); -// } -export function withTagId(tagId: string, assetId: Expression) { - const eb = expressionBuilder(); - return eb.exists( - eb - .selectFrom('tags_closure') - .innerJoin('tag_asset', 'tag_asset.tagsId', 'tags_closure.id_descendant') - .whereRef('tag_asset.assetsId', '=', assetId) - .where('tags_closure.id_ancestor', '=', tagId), - ); -} - -const joinDeduplicationPlugin = new DeduplicateJoinsPlugin(); - -/** TODO: This should only be used for search-related queries, not as a general purpose query builder */ -export function searchAssetBuilder(kysely: Kysely, options: AssetSearchBuilderOptions) { - options.isArchived ??= options.withArchived ? undefined : false; - options.withDeleted ||= !!(options.trashedAfter || options.trashedBefore || options.isOffline); - return ( - kysely - .withPlugin(joinDeduplicationPlugin) - .selectFrom('assets') - .selectAll('assets') - .$if(!!options.tagIds && options.tagIds.length > 0, (qb) => hasTags(qb, options.tagIds!)) - // .$if(!!options.personIds && options.personIds.length > 0, (qb) => hasPeople(qb, options.personIds!)) - .$if(!!options.personIds && options.personIds.length > 0, (qb) => - qb.innerJoin( - () => hasPeople(options.personIds!), - (join) => join.onRef('has_people.assetId', '=', 'assets.id'), - ), - ) - .$if(!!options.createdBefore, (qb) => qb.where('assets.createdAt', '<=', options.createdBefore!)) - .$if(!!options.createdAfter, (qb) => qb.where('assets.createdAt', '>=', options.createdAfter!)) - .$if(!!options.updatedBefore, (qb) => qb.where('assets.updatedAt', '<=', options.updatedBefore!)) - .$if(!!options.updatedAfter, (qb) => qb.where('assets.updatedAt', '>=', options.updatedAfter!)) - .$if(!!options.trashedBefore, (qb) => qb.where('assets.deletedAt', '<=', options.trashedBefore!)) - .$if(!!options.trashedAfter, (qb) => qb.where('assets.deletedAt', '>=', options.trashedAfter!)) - .$if(!!options.takenBefore, (qb) => qb.where('assets.fileCreatedAt', '<=', options.takenBefore!)) - .$if(!!options.takenAfter, (qb) => qb.where('assets.fileCreatedAt', '>=', options.takenAfter!)) - .$if(options.city !== undefined, (qb) => - qb - .innerJoin('exif', 'assets.id', 'exif.assetId') - .where('exif.city', options.city === null ? 'is' : '=', options.city!), - ) - .$if(options.state !== undefined, (qb) => - qb - .innerJoin('exif', 'assets.id', 'exif.assetId') - .where('exif.state', options.state === null ? 'is' : '=', options.state!), - ) - .$if(options.country !== undefined, (qb) => - qb - .innerJoin('exif', 'assets.id', 'exif.assetId') - .where('exif.country', options.country === null ? 'is' : '=', options.country!), - ) - .$if(options.make !== undefined, (qb) => - qb - .innerJoin('exif', 'assets.id', 'exif.assetId') - .where('exif.make', options.make === null ? 'is' : '=', options.make!), - ) - .$if(options.model !== undefined, (qb) => - qb - .innerJoin('exif', 'assets.id', 'exif.assetId') - .where('exif.model', options.model === null ? 'is' : '=', options.model!), - ) - .$if(options.lensModel !== undefined, (qb) => - qb - .innerJoin('exif', 'assets.id', 'exif.assetId') - .where('exif.lensModel', options.lensModel === null ? 'is' : '=', options.lensModel!), - ) - .$if(options.rating !== undefined, (qb) => - qb - .innerJoin('exif', 'assets.id', 'exif.assetId') - .where('exif.rating', options.rating === null ? 'is' : '=', options.rating!), - ) - .$if(!!options.checksum, (qb) => qb.where('assets.checksum', '=', options.checksum!)) - .$if(!!options.deviceAssetId, (qb) => qb.where('assets.deviceAssetId', '=', options.deviceAssetId!)) - .$if(!!options.deviceId, (qb) => qb.where('assets.deviceId', '=', options.deviceId!)) - .$if(!!options.id, (qb) => qb.where('assets.id', '=', asUuid(options.id!))) - .$if(!!options.libraryId, (qb) => qb.where('assets.libraryId', '=', asUuid(options.libraryId!))) - .$if(!!options.userIds, (qb) => qb.where('assets.ownerId', '=', anyUuid(options.userIds!))) - .$if(!!options.encodedVideoPath, (qb) => qb.where('assets.encodedVideoPath', '=', options.encodedVideoPath!)) - .$if(!!options.originalPath, (qb) => - qb.where( - sql`f_unaccent(assets."originalPath")`, - 'ilike', - sql`'%' || f_unaccent(${options.originalPath}) || '%'`, - ), - ) - .$if(!!options.originalFileName, (qb) => - qb.where( - sql`f_unaccent(assets."originalFileName")`, - 'ilike', - sql`'%' || f_unaccent(${options.originalFileName}) || '%'`, - ), - ) - .$if(!!options.description, (qb) => - qb - .innerJoin('exif', 'assets.id', 'exif.assetId') - .where(sql`f_unaccent(exif.description)`, 'ilike', sql`'%' || f_unaccent(${options.description}) || '%'`), - ) - .$if(!!options.type, (qb) => qb.where('assets.type', '=', options.type!)) - .$if(options.isFavorite !== undefined, (qb) => qb.where('assets.isFavorite', '=', options.isFavorite!)) - .$if(options.isOffline !== undefined, (qb) => qb.where('assets.isOffline', '=', options.isOffline!)) - .$if(options.isVisible !== undefined, (qb) => qb.where('assets.isVisible', '=', options.isVisible!)) - .$if(options.isArchived !== undefined, (qb) => qb.where('assets.isArchived', '=', options.isArchived!)) - .$if(options.isEncoded !== undefined, (qb) => - qb.where('assets.encodedVideoPath', options.isEncoded ? 'is not' : 'is', null), - ) - .$if(options.isMotion !== undefined, (qb) => - qb.where('assets.livePhotoVideoId', options.isMotion ? 'is not' : 'is', null), - ) - .$if(!!options.isNotInAlbum, (qb) => - qb.where((eb) => - eb.not(eb.exists((eb) => eb.selectFrom('albums_assets_assets').whereRef('assetsId', '=', 'assets.id'))), - ), - ) - .$if(!!options.withExif, withExifInner) - .$if(!!(options.withFaces || options.withPeople || options.personIds), (qb) => qb.select(withFacesAndPeople)) - .$if(!options.withDeleted, (qb) => qb.where('assets.deletedAt', 'is', null)) - ); -} diff --git a/server/src/entities/shared-link.entity.ts b/server/src/entities/shared-link.entity.ts deleted file mode 100644 index 720ba424d1..0000000000 --- a/server/src/entities/shared-link.entity.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { AlbumEntity } from 'src/entities/album.entity'; -import { AssetEntity } from 'src/entities/asset.entity'; -import { SharedLinkType } from 'src/enum'; - -export class SharedLinkEntity { - id!: string; - description!: string | null; - password!: string | null; - userId!: string; - key!: Buffer; // use to access the inidividual asset - type!: SharedLinkType; - createdAt!: Date; - expiresAt!: Date | null; - allowUpload!: boolean; - allowDownload!: boolean; - showExif!: boolean; - assets!: AssetEntity[]; - album?: AlbumEntity; - albumId!: string | null; -} diff --git a/server/src/migrations/1744900200559-AddForeignKeyIndexes.ts b/server/src/migrations/1744900200559-AddForeignKeyIndexes.ts new file mode 100644 index 0000000000..db351d5bab --- /dev/null +++ b/server/src/migrations/1744900200559-AddForeignKeyIndexes.ts @@ -0,0 +1,51 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AddForeignKeyIndexes1744900200559 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`CREATE INDEX "IDX_0f6fc2fb195f24d19b0fb0d57c" ON "libraries" ("ownerId")`); + await queryRunner.query(`CREATE INDEX "IDX_91704e101438fd0653f582426d" ON "asset_stack" ("primaryAssetId")`); + await queryRunner.query(`CREATE INDEX "IDX_c05079e542fd74de3b5ecb5c1c" ON "asset_stack" ("ownerId")`); + await queryRunner.query(`CREATE INDEX "IDX_2c5ac0d6fb58b238fd2068de67" ON "assets" ("ownerId")`); + await queryRunner.query(`CREATE INDEX "IDX_16294b83fa8c0149719a1f631e" ON "assets" ("livePhotoVideoId")`); + await queryRunner.query(`CREATE INDEX "IDX_9977c3c1de01c3d848039a6b90" ON "assets" ("libraryId")`); + await queryRunner.query(`CREATE INDEX "IDX_f15d48fa3ea5e4bda05ca8ab20" ON "assets" ("stackId")`); + await queryRunner.query(`CREATE INDEX "IDX_b22c53f35ef20c28c21637c85f" ON "albums" ("ownerId")`); + await queryRunner.query(`CREATE INDEX "IDX_05895aa505a670300d4816debc" ON "albums" ("albumThumbnailAssetId")`); + await queryRunner.query(`CREATE INDEX "IDX_1af8519996fbfb3684b58df280" ON "activity" ("albumId")`); + await queryRunner.query(`CREATE INDEX "IDX_3571467bcbe021f66e2bdce96e" ON "activity" ("userId")`); + await queryRunner.query(`CREATE INDEX "IDX_8091ea76b12338cb4428d33d78" ON "activity" ("assetId")`); + await queryRunner.query(`CREATE INDEX "IDX_6c2e267ae764a9413b863a2934" ON "api_keys" ("userId")`); + await queryRunner.query(`CREATE INDEX "IDX_5527cc99f530a547093f9e577b" ON "person" ("ownerId")`); + await queryRunner.query(`CREATE INDEX "IDX_2bbabe31656b6778c6b87b6102" ON "person" ("faceAssetId")`); + await queryRunner.query(`CREATE INDEX "IDX_575842846f0c28fa5da46c99b1" ON "memories" ("ownerId")`); + await queryRunner.query(`CREATE INDEX "IDX_d7e875c6c60e661723dbf372fd" ON "partners" ("sharedWithId")`); + await queryRunner.query(`CREATE INDEX "IDX_57de40bc620f456c7311aa3a1e" ON "sessions" ("userId")`); + await queryRunner.query(`CREATE INDEX "IDX_66fe3837414c5a9f1c33ca4934" ON "shared_links" ("userId")`); + await queryRunner.query(`CREATE INDEX "IDX_d8ddd9d687816cc490432b3d4b" ON "session_sync_checkpoints" ("sessionId")`); + await queryRunner.query(`CREATE INDEX "IDX_9f9590cc11561f1f48ff034ef9" ON "tags" ("parentId")`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX "IDX_66fe3837414c5a9f1c33ca4934";`); + await queryRunner.query(`DROP INDEX "IDX_91704e101438fd0653f582426d";`); + await queryRunner.query(`DROP INDEX "IDX_c05079e542fd74de3b5ecb5c1c";`); + await queryRunner.query(`DROP INDEX "IDX_5527cc99f530a547093f9e577b";`); + await queryRunner.query(`DROP INDEX "IDX_2bbabe31656b6778c6b87b6102";`); + await queryRunner.query(`DROP INDEX "IDX_0f6fc2fb195f24d19b0fb0d57c";`); + await queryRunner.query(`DROP INDEX "IDX_9f9590cc11561f1f48ff034ef9";`); + await queryRunner.query(`DROP INDEX "IDX_2c5ac0d6fb58b238fd2068de67";`); + await queryRunner.query(`DROP INDEX "IDX_16294b83fa8c0149719a1f631e";`); + await queryRunner.query(`DROP INDEX "IDX_9977c3c1de01c3d848039a6b90";`); + await queryRunner.query(`DROP INDEX "IDX_f15d48fa3ea5e4bda05ca8ab20";`); + await queryRunner.query(`DROP INDEX "IDX_b22c53f35ef20c28c21637c85f";`); + await queryRunner.query(`DROP INDEX "IDX_05895aa505a670300d4816debc";`); + await queryRunner.query(`DROP INDEX "IDX_57de40bc620f456c7311aa3a1e";`); + await queryRunner.query(`DROP INDEX "IDX_d8ddd9d687816cc490432b3d4b";`); + await queryRunner.query(`DROP INDEX "IDX_d7e875c6c60e661723dbf372fd";`); + await queryRunner.query(`DROP INDEX "IDX_575842846f0c28fa5da46c99b1";`); + await queryRunner.query(`DROP INDEX "IDX_6c2e267ae764a9413b863a2934";`); + await queryRunner.query(`DROP INDEX "IDX_1af8519996fbfb3684b58df280";`); + await queryRunner.query(`DROP INDEX "IDX_3571467bcbe021f66e2bdce96e";`); + await queryRunner.query(`DROP INDEX "IDX_8091ea76b12338cb4428d33d78";`); + } +} diff --git a/server/src/migrations/1744910873956-AddMissingIndex.ts b/server/src/migrations/1744910873956-AddMissingIndex.ts new file mode 100644 index 0000000000..38dd6f4958 --- /dev/null +++ b/server/src/migrations/1744910873956-AddMissingIndex.ts @@ -0,0 +1,13 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AddMissingIndex1744910873956 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE INDEX IF NOT EXISTS "IDX_geodata_gist_earthcoord" ON "geodata_places" (ll_to_earth_public(latitude, longitude))`, + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX "IDX_geodata_gist_earthcoord";`); + } +} diff --git a/server/src/queries/asset.repository.sql b/server/src/queries/asset.repository.sql index cf17bb0276..a3dcb08c1e 100644 --- a/server/src/queries/asset.repository.sql +++ b/server/src/queries/asset.repository.sql @@ -82,7 +82,7 @@ from where "assets"."id" = any ($1::uuid[]) --- AssetRepository.getByIdsWithAllRelations +-- AssetRepository.getByIdsWithAllRelationsButStacks select "assets".*, ( @@ -127,28 +127,13 @@ select "assets"."id" = "tag_asset"."assetsId" ) as agg ) as "tags", - to_json("exif") as "exifInfo", - to_json("stacked_assets") as "stack" + to_json("exif") as "exifInfo" from "assets" left join "exif" on "assets"."id" = "exif"."assetId" left join "asset_stack" on "asset_stack"."id" = "assets"."stackId" - left join lateral ( - select - "asset_stack".*, - array_agg("stacked") as "assets" - from - "assets" as "stacked" - where - "stacked"."stackId" = "asset_stack"."id" - and "stacked"."id" != "asset_stack"."primaryAssetId" - and "stacked"."deletedAt" is null - and "stacked"."isArchived" = $1 - group by - "asset_stack"."id" - ) as "stacked_assets" on "asset_stack"."id" is not null where - "assets"."id" = any ($2::uuid[]) + "assets"."id" = any ($1::uuid[]) -- AssetRepository.deleteAll delete from "assets" diff --git a/server/src/repositories/album.repository.ts b/server/src/repositories/album.repository.ts index e21d5d73cd..1768135210 100644 --- a/server/src/repositories/album.repository.ts +++ b/server/src/repositories/album.repository.ts @@ -1,12 +1,11 @@ import { Injectable } from '@nestjs/common'; -import { ExpressionBuilder, Insertable, Kysely, sql, Updateable } from 'kysely'; +import { ExpressionBuilder, Insertable, Kysely, NotNull, sql, Updateable } from 'kysely'; import { jsonArrayFrom, jsonObjectFrom } from 'kysely/helpers/postgres'; import { InjectKysely } from 'nestjs-kysely'; -import { columns } from 'src/database'; +import { columns, Exif } from 'src/database'; import { Albums, DB } from 'src/db'; import { Chunked, ChunkedArray, ChunkedSet, DummyValue, GenerateSql } from 'src/decorators'; import { AlbumUserCreateDto } from 'src/dtos/album.dto'; -import { AlbumEntity } from 'src/entities/album.entity'; export interface AlbumAssetCount { albumId: string; @@ -21,9 +20,9 @@ export interface AlbumInfoOptions { } const withOwner = (eb: ExpressionBuilder) => { - return jsonObjectFrom(eb.selectFrom('users').select(columns.user).whereRef('users.id', '=', 'albums.ownerId')).as( - 'owner', - ); + return jsonObjectFrom(eb.selectFrom('users').select(columns.user).whereRef('users.id', '=', 'albums.ownerId')) + .$notNull() + .as('owner'); }; const withAlbumUsers = (eb: ExpressionBuilder) => { @@ -32,12 +31,14 @@ const withAlbumUsers = (eb: ExpressionBuilder) => { .selectFrom('albums_shared_users_users as album_users') .select('album_users.role') .select((eb) => - jsonObjectFrom(eb.selectFrom('users').select(columns.user).whereRef('users.id', '=', 'album_users.usersId')).as( - 'user', - ), + jsonObjectFrom(eb.selectFrom('users').select(columns.user).whereRef('users.id', '=', 'album_users.usersId')) + .$notNull() + .as('user'), ) .whereRef('album_users.albumsId', '=', 'albums.id'), - ).as('albumUsers'); + ) + .$notNull() + .as('albumUsers'); }; const withSharedLink = (eb: ExpressionBuilder) => { @@ -53,7 +54,7 @@ const withAssets = (eb: ExpressionBuilder) => { .selectFrom('assets') .selectAll('assets') .leftJoin('exif', 'assets.id', 'exif.assetId') - .select((eb) => eb.table('exif').as('exifInfo')) + .select((eb) => eb.table('exif').$castTo().as('exifInfo')) .innerJoin('albums_assets_assets', 'albums_assets_assets.assetsId', 'assets.id') .whereRef('albums_assets_assets.albumsId', '=', 'albums.id') .where('assets.deletedAt', 'is', null) @@ -69,7 +70,7 @@ export class AlbumRepository { constructor(@InjectKysely() private db: Kysely) {} @GenerateSql({ params: [DummyValue.UUID, { withAssets: true }] }) - async getById(id: string, options: AlbumInfoOptions): Promise { + async getById(id: string, options: AlbumInfoOptions) { return this.db .selectFrom('albums') .selectAll('albums') @@ -79,11 +80,12 @@ export class AlbumRepository { .select(withAlbumUsers) .select(withSharedLink) .$if(options.withAssets, (eb) => eb.select(withAssets)) - .executeTakeFirst() as Promise; + .$narrowType<{ assets: NotNull }>() + .executeTakeFirst(); } @GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID] }) - async getByAssetId(ownerId: string, assetId: string): Promise { + async getByAssetId(ownerId: string, assetId: string) { return this.db .selectFrom('albums') .selectAll('albums') @@ -105,7 +107,7 @@ export class AlbumRepository { .select(withOwner) .select(withAlbumUsers) .orderBy('albums.createdAt', 'desc') - .execute() as unknown as Promise; + .execute(); } @GenerateSql({ params: [[DummyValue.UUID]] }) @@ -134,7 +136,7 @@ export class AlbumRepository { } @GenerateSql({ params: [DummyValue.UUID] }) - async getOwned(ownerId: string): Promise { + async getOwned(ownerId: string) { return this.db .selectFrom('albums') .selectAll('albums') @@ -144,14 +146,14 @@ export class AlbumRepository { .where('albums.ownerId', '=', ownerId) .where('albums.deletedAt', 'is', null) .orderBy('albums.createdAt', 'desc') - .execute() as unknown as Promise; + .execute(); } /** * Get albums shared with and shared by owner. */ @GenerateSql({ params: [DummyValue.UUID] }) - async getShared(ownerId: string): Promise { + async getShared(ownerId: string) { return this.db .selectFrom('albums') .selectAll('albums') @@ -176,14 +178,14 @@ export class AlbumRepository { .select(withOwner) .select(withSharedLink) .orderBy('albums.createdAt', 'desc') - .execute() as unknown as Promise; + .execute(); } /** * Get albums of owner that are _not_ shared */ @GenerateSql({ params: [DummyValue.UUID] }) - async getNotShared(ownerId: string): Promise { + async getNotShared(ownerId: string) { return this.db .selectFrom('albums') .selectAll('albums') @@ -203,7 +205,7 @@ export class AlbumRepository { ) .select(withOwner) .orderBy('albums.createdAt', 'desc') - .execute() as unknown as Promise; + .execute(); } async restoreAll(userId: string): Promise { @@ -262,7 +264,7 @@ export class AlbumRepository { await this.addAssets(this.db, albumId, assetIds); } - create(album: Insertable, assetIds: string[], albumUsers: AlbumUserCreateDto[]): Promise { + create(album: Insertable, assetIds: string[], albumUsers: AlbumUserCreateDto[]) { return this.db.transaction().execute(async (tx) => { const newAlbum = await tx.insertInto('albums').values(album).returning('albums.id').executeTakeFirst(); @@ -290,11 +292,12 @@ export class AlbumRepository { .select(withOwner) .select(withAssets) .select(withAlbumUsers) - .executeTakeFirst() as unknown as Promise; + .$narrowType<{ assets: NotNull }>() + .executeTakeFirstOrThrow(); }); } - update(id: string, album: Updateable): Promise { + update(id: string, album: Updateable) { return this.db .updateTable('albums') .set(album) @@ -303,7 +306,7 @@ export class AlbumRepository { .returning(withOwner) .returning(withSharedLink) .returning(withAlbumUsers) - .executeTakeFirst() as unknown as Promise; + .executeTakeFirstOrThrow(); } async delete(id: string): Promise { diff --git a/server/src/repositories/asset-job.repository.ts b/server/src/repositories/asset-job.repository.ts index b507fa5445..4a2d52566f 100644 --- a/server/src/repositories/asset-job.repository.ts +++ b/server/src/repositories/asset-job.repository.ts @@ -5,10 +5,9 @@ import { InjectKysely } from 'nestjs-kysely'; import { columns } from 'src/database'; import { DB } from 'src/db'; import { DummyValue, GenerateSql } from 'src/decorators'; -import { withExifInner, withFaces, withFiles } from 'src/entities/asset.entity'; import { AssetFileType } from 'src/enum'; import { StorageAsset } from 'src/types'; -import { asUuid } from 'src/utils/database'; +import { anyUuid, asUuid, withExifInner, withFaces, withFiles } from 'src/utils/database'; @Injectable() export class AssetJobRepository { @@ -149,6 +148,21 @@ export class AssetJobRepository { .executeTakeFirst(); } + getForSyncAssets(ids: string[]) { + return this.db + .selectFrom('assets') + .select([ + 'assets.id', + 'assets.isOffline', + 'assets.libraryId', + 'assets.originalPath', + 'assets.status', + 'assets.fileModifiedAt', + ]) + .where('assets.id', '=', anyUuid(ids)) + .execute(); + } + private storageTemplateAssetQuery() { return this.db .selectFrom('assets') diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts index f32962611d..6153b31c21 100644 --- a/server/src/repositories/asset.repository.ts +++ b/server/src/repositories/asset.repository.ts @@ -1,14 +1,22 @@ import { Injectable } from '@nestjs/common'; -import { Insertable, Kysely, Selectable, UpdateResult, Updateable, sql } from 'kysely'; +import { Insertable, Kysely, NotNull, Selectable, UpdateResult, Updateable, sql } from 'kysely'; import { isEmpty, isUndefined, omitBy, round } from 'lodash'; import { InjectKysely } from 'nestjs-kysely'; +import { Stack } from 'src/database'; import { AssetFiles, AssetJobStatus, Assets, DB, Exif } from 'src/db'; import { Chunked, ChunkedArray, DummyValue, GenerateSql } from 'src/decorators'; +import { MapAsset } from 'src/dtos/asset-response.dto'; +import { AssetFileType, AssetOrder, AssetStatus, AssetType } from 'src/enum'; +import { AssetSearchOptions, SearchExploreItem, SearchExploreItemSet } from 'src/repositories/search.repository'; import { - AssetEntity, + anyUuid, + asUuid, hasPeople, + hasPeopleNoJoin, + removeUndefinedKeys, searchAssetBuilder, truncatedDate, + unnest, withExif, withFaces, withFacesAndPeople, @@ -17,13 +25,11 @@ import { withOwner, withSmartSearch, withTagId, + withTagIdNoWhere, withTags, -} from 'src/entities/asset.entity'; -import { AssetFileType, AssetOrder, AssetStatus, AssetType } from 'src/enum'; -import { AssetSearchOptions, SearchExploreItem, SearchExploreItemSet } from 'src/repositories/search.repository'; -import { anyUuid, asUuid, removeUndefinedKeys, unnest } from 'src/utils/database'; +} from 'src/utils/database'; import { globToSqlPattern } from 'src/utils/misc'; -import { Paginated, PaginationOptions, paginationHelper } from 'src/utils/pagination'; +import { PaginationOptions, paginationHelper } from 'src/utils/pagination'; import { TimeBucketAssets } from 'src/services/timeline.service.types'; import { isFlipped } from 'src/utils/asset.util'; @@ -130,8 +136,6 @@ export interface AssetGetByChecksumOptions { libraryId?: string; } -export type AssetPathEntity = Pick; - export interface GetByIdsRelations { exifInfo?: boolean; faces?: { person?: boolean; withDeleted?: boolean }; @@ -145,12 +149,12 @@ export interface GetByIdsRelations { export interface DuplicateGroup { duplicateId: string; - assets: AssetEntity[]; + assets: MapAsset[]; } export interface DayOfYearAssets { yearsAgo: number; - assets: AssetEntity[]; + assets: MapAsset[]; } @Injectable() @@ -238,12 +242,12 @@ export class AssetRepository { .execute(); } - create(asset: Insertable): Promise { - return this.db.insertInto('assets').values(asset).returningAll().executeTakeFirst() as any as Promise; + create(asset: Insertable) { + return this.db.insertInto('assets').values(asset).returningAll().executeTakeFirstOrThrow(); } - createAll(assets: Insertable[]): Promise { - return this.db.insertInto('assets').values(assets).returningAll().execute() as any as Promise; + createAll(assets: Insertable[]) { + return this.db.insertInto('assets').values(assets).returningAll().execute(); } @GenerateSql({ params: [DummyValue.UUID, { day: 1, month: 1 }] }) @@ -303,56 +307,13 @@ export class AssetRepository { @GenerateSql({ params: [[DummyValue.UUID]] }) @ChunkedArray() - async getByIds( - ids: string[], - { exifInfo, faces, files, library, owner, smartSearch, stack, tags }: GetByIdsRelations = {}, - ): Promise { - const res = await this.db - .selectFrom('assets') - .selectAll('assets') - .where('assets.id', '=', anyUuid(ids)) - .$if(!!exifInfo, withExif) - .$if(!!faces, (qb) => - qb.select((eb) => - faces?.person ? withFacesAndPeople(eb, faces.withDeleted) : withFaces(eb, faces?.withDeleted), - ), - ) - .$if(!!files, (qb) => qb.select(withFiles)) - .$if(!!library, (qb) => qb.select(withLibrary)) - .$if(!!owner, (qb) => qb.select(withOwner)) - .$if(!!smartSearch, withSmartSearch) - .$if(!!stack, (qb) => - qb - .leftJoin('asset_stack', 'asset_stack.id', 'assets.stackId') - .$if(!stack!.assets, (qb) => qb.select((eb) => eb.fn.toJson(eb.table('asset_stack')).as('stack'))) - .$if(!!stack!.assets, (qb) => - qb - .leftJoinLateral( - (eb) => - eb - .selectFrom('assets as stacked') - .selectAll('asset_stack') - .select((eb) => eb.fn('array_agg', [eb.table('stacked')]).as('assets')) - .whereRef('stacked.stackId', '=', 'asset_stack.id') - .whereRef('stacked.id', '!=', 'asset_stack.primaryAssetId') - .where('stacked.deletedAt', 'is', null) - .where('stacked.isArchived', '=', false) - .groupBy('asset_stack.id') - .as('stacked_assets'), - (join) => join.on('asset_stack.id', 'is not', null), - ) - .select((eb) => eb.fn.toJson(eb.table('stacked_assets')).as('stack')), - ), - ) - .$if(!!tags, (qb) => qb.select(withTags)) - .execute(); - - return res as any as AssetEntity[]; + getByIds(ids: string[]) { + return this.db.selectFrom('assets').selectAll('assets').where('assets.id', '=', anyUuid(ids)).execute(); } @GenerateSql({ params: [[DummyValue.UUID]] }) @ChunkedArray() - getByIdsWithAllRelations(ids: string[]): Promise { + getByIdsWithAllRelationsButStacks(ids: string[]) { return this.db .selectFrom('assets') .selectAll('assets') @@ -360,23 +321,8 @@ export class AssetRepository { .select(withTags) .$call(withExif) .leftJoin('asset_stack', 'asset_stack.id', 'assets.stackId') - .leftJoinLateral( - (eb) => - eb - .selectFrom('assets as stacked') - .selectAll('asset_stack') - .select((eb) => eb.fn('array_agg', [eb.table('stacked')]).as('assets')) - .whereRef('stacked.stackId', '=', 'asset_stack.id') - .whereRef('stacked.id', '!=', 'asset_stack.primaryAssetId') - .where('stacked.deletedAt', 'is', null) - .where('stacked.isArchived', '=', false) - .groupBy('asset_stack.id') - .as('stacked_assets'), - (join) => join.on('asset_stack.id', 'is not', null), - ) - .select((eb) => eb.fn.toJson(eb.table('stacked_assets')).as('stack')) .where('assets.id', '=', anyUuid(ids)) - .execute() as any as Promise; + .execute(); } @GenerateSql({ params: [DummyValue.UUID] }) @@ -396,36 +342,29 @@ export class AssetRepository { return assets.map((asset) => asset.deviceAssetId); } - getByUserId( - pagination: PaginationOptions, - userId: string, - options: Omit = {}, - ): Paginated { + getByUserId(pagination: PaginationOptions, userId: string, options: Omit = {}) { return this.getAll(pagination, { ...options, userIds: [userId] }); } @GenerateSql({ params: [DummyValue.UUID, DummyValue.STRING] }) - getByLibraryIdAndOriginalPath(libraryId: string, originalPath: string): Promise { + getByLibraryIdAndOriginalPath(libraryId: string, originalPath: string) { return this.db .selectFrom('assets') .selectAll('assets') .where('libraryId', '=', asUuid(libraryId)) .where('originalPath', '=', originalPath) .limit(1) - .executeTakeFirst() as any as Promise; + .executeTakeFirst(); } - async getAll( - pagination: PaginationOptions, - { orderDirection, ...options }: AssetSearchOptions = {}, - ): Paginated { + async getAll(pagination: PaginationOptions, { orderDirection, ...options }: AssetSearchOptions = {}) { const builder = searchAssetBuilder(this.db, options) .select(withFiles) .orderBy('assets.createdAt', orderDirection ?? 'asc') .limit(pagination.take + 1) .offset(pagination.skip ?? 0); const items = await builder.execute(); - return paginationHelper(items as any as AssetEntity[], pagination.take); + return paginationHelper(items, pagination.take); } /** @@ -460,23 +399,22 @@ export class AssetRepository { } @GenerateSql({ params: [DummyValue.UUID] }) - getById( - id: string, - { exifInfo, faces, files, library, owner, smartSearch, stack, tags }: GetByIdsRelations = {}, - ): Promise { + getById(id: string, { exifInfo, faces, files, library, owner, smartSearch, stack, tags }: GetByIdsRelations = {}) { return this.db .selectFrom('assets') .selectAll('assets') .where('assets.id', '=', asUuid(id)) .$if(!!exifInfo, withExif) - .$if(!!faces, (qb) => qb.select(faces?.person ? withFacesAndPeople : withFaces)) + .$if(!!faces, (qb) => qb.select(faces?.person ? withFacesAndPeople : withFaces).$narrowType<{ faces: NotNull }>()) .$if(!!library, (qb) => qb.select(withLibrary)) .$if(!!owner, (qb) => qb.select(withOwner)) .$if(!!smartSearch, withSmartSearch) .$if(!!stack, (qb) => qb .leftJoin('asset_stack', 'asset_stack.id', 'assets.stackId') - .$if(!stack!.assets, (qb) => qb.select((eb) => eb.fn.toJson(eb.table('asset_stack')).as('stack'))) + .$if(!stack!.assets, (qb) => + qb.select((eb) => eb.fn.toJson(eb.table('asset_stack')).$castTo().as('stack')), + ) .$if(!!stack!.assets, (qb) => qb .leftJoinLateral( @@ -493,13 +431,13 @@ export class AssetRepository { .as('stacked_assets'), (join) => join.on('asset_stack.id', 'is not', null), ) - .select((eb) => eb.fn.toJson(eb.table('stacked_assets')).as('stack')), + .select((eb) => eb.fn.toJson(eb.table('stacked_assets')).$castTo().as('stack')), ), ) .$if(!!files, (qb) => qb.select(withFiles)) .$if(!!tags, (qb) => qb.select(withTags)) .limit(1) - .executeTakeFirst() as any as Promise; + .executeTakeFirst(); } @GenerateSql({ params: [[DummyValue.UUID], { deviceId: DummyValue.STRING }] }) @@ -528,7 +466,7 @@ export class AssetRepository { .execute(); } - async update(asset: Updateable & { id: string }): Promise { + async update(asset: Updateable & { id: string }) { const value = omitBy(asset, isUndefined); delete value.id; if (!isEmpty(value)) { @@ -538,10 +476,10 @@ export class AssetRepository { .selectAll('assets') .$call(withExif) .$call((qb) => qb.select(withFacesAndPeople)) - .executeTakeFirst() as Promise; + .executeTakeFirst(); } - return this.getById(asset.id, { exifInfo: true, faces: { person: true } }) as Promise; + return this.getById(asset.id, { exifInfo: true, faces: { person: true } }); } async remove(asset: { id: string }): Promise { @@ -549,7 +487,7 @@ export class AssetRepository { } @GenerateSql({ params: [{ ownerId: DummyValue.UUID, libraryId: DummyValue.UUID, checksum: DummyValue.BUFFER }] }) - getByChecksum({ ownerId, libraryId, checksum }: AssetGetByChecksumOptions): Promise { + getByChecksum({ ownerId, libraryId, checksum }: AssetGetByChecksumOptions) { return this.db .selectFrom('assets') .selectAll('assets') @@ -557,17 +495,17 @@ export class AssetRepository { .where('checksum', '=', checksum) .$call((qb) => (libraryId ? qb.where('libraryId', '=', asUuid(libraryId)) : qb.where('libraryId', 'is', null))) .limit(1) - .executeTakeFirst() as Promise; + .executeTakeFirst(); } @GenerateSql({ params: [DummyValue.UUID, [DummyValue.BUFFER]] }) - getByChecksums(userId: string, checksums: Buffer[]): Promise { + getByChecksums(userId: string, checksums: Buffer[]) { return this.db .selectFrom('assets') .select(['id', 'checksum', 'deletedAt']) .where('ownerId', '=', asUuid(userId)) .where('checksum', 'in', checksums) - .execute() as any as Promise; + .execute(); } @GenerateSql({ params: [DummyValue.UUID, DummyValue.BUFFER] }) @@ -584,7 +522,7 @@ export class AssetRepository { return asset?.id; } - findLivePhotoMatch(options: LivePhotoSearchOptions): Promise { + findLivePhotoMatch(options: LivePhotoSearchOptions) { const { ownerId, otherAssetId, livePhotoCID, type } = options; return this.db .selectFrom('assets') @@ -595,7 +533,7 @@ export class AssetRepository { .where('type', '=', type) .where('exif.livePhotoCID', '=', livePhotoCID) .limit(1) - .executeTakeFirst() as Promise; + .executeTakeFirst(); } @GenerateSql( @@ -604,7 +542,7 @@ export class AssetRepository { params: [DummyValue.PAGINATION, property], })), ) - async getWithout(pagination: PaginationOptions, property: WithoutProperty): Paginated { + async getWithout(pagination: PaginationOptions, property: WithoutProperty) { const items = await this.db .selectFrom('assets') .selectAll('assets') @@ -666,7 +604,7 @@ export class AssetRepository { .orderBy('createdAt') .execute(); - return paginationHelper(items as any as AssetEntity[], pagination.take); + return paginationHelper(items, pagination.take); } getStatistics(ownerId: string, { isArchived, isFavorite, isTrashed }: AssetStatsOptions): Promise { @@ -685,7 +623,7 @@ export class AssetRepository { .executeTakeFirstOrThrow(); } - getRandom(userIds: string[], take: number): Promise { + getRandom(userIds: string[], take: number) { return this.db .selectFrom('assets') .selectAll('assets') @@ -695,50 +633,54 @@ export class AssetRepository { .where('deletedAt', 'is', null) .orderBy((eb) => eb.fn('random')) .limit(take) - .execute() as any as Promise; + .execute(); } @GenerateSql({ params: [{ size: TimeBucketSize.MONTH }] }) async getTimeBuckets(options: TimeBucketOptions): Promise { return ( this.db - .with('assets', (qb) => - qb - .selectFrom('assets') - .select(truncatedDate(options.size).as('timeBucket')) - .$if(!!options.isTrashed, (qb) => qb.where('assets.status', '!=', AssetStatus.DELETED)) - .where('assets.deletedAt', options.isTrashed ? 'is not' : 'is', null) - .where('assets.isVisible', '=', true) - .$if(!!options.albumId, (qb) => - qb - .innerJoin('albums_assets_assets', 'assets.id', 'albums_assets_assets.assetsId') - .where('albums_assets_assets.albumsId', '=', asUuid(options.albumId!)), - ) - // .$if(!!options.personId, (qb) => hasPeople(qb, [options.personId!])) - .$if(!!options.personId, (qb) => - qb.innerJoin( - () => hasPeople([options.personId!]), - (join) => join.onRef('has_people.assetId', '=', 'assets.id'), - ), - ) - .$if(!!options.withStacked, (qb) => - qb - .leftJoin('asset_stack', (join) => - join - .onRef('asset_stack.id', '=', 'assets.stackId') - .onRef('asset_stack.primaryAssetId', '=', 'assets.id'), - ) - .where((eb) => eb.or([eb('assets.stackId', 'is', null), eb(eb.table('asset_stack'), 'is not', null)])), - ) - .$if(!!options.userIds, (qb) => qb.where('assets.ownerId', '=', anyUuid(options.userIds!))) - .$if(options.isArchived !== undefined, (qb) => qb.where('assets.isArchived', '=', options.isArchived!)) - .$if(options.isFavorite !== undefined, (qb) => qb.where('assets.isFavorite', '=', options.isFavorite!)) - .$if(!!options.assetType, (qb) => qb.where('assets.type', '=', options.assetType!)) - .$if(options.isDuplicate !== undefined, (qb) => - qb.where('assets.duplicateId', options.isDuplicate ? 'is not' : 'is', null), - ) - // .$if(!!options.tagId, (qb) => withTagId(qb, options.tagId!)), - .$if(!!options.tagId, (qb) => qb.where((eb) => withTagId(options.tagId!, eb.ref('assets.id')))), + .with( + 'assets', + (qb) => + qb + .selectFrom('assets') + .select(truncatedDate(options.size).as('timeBucket')) + .$if(!!options.isTrashed, (qb) => qb.where('assets.status', '!=', AssetStatus.DELETED)) + .where('assets.deletedAt', options.isTrashed ? 'is not' : 'is', null) + .where('assets.isVisible', '=', true) + .$if(!!options.albumId, (qb) => + qb + .innerJoin('albums_assets_assets', 'assets.id', 'albums_assets_assets.assetsId') + .where('albums_assets_assets.albumsId', '=', asUuid(options.albumId!)), + ) + .$if(!!options.personId, (qb) => hasPeople(qb, [options.personId!])) + // .$if(!!options.personId, (qb) => + // qb.innerJoin( + // () => hasPeople([options.personId!]), + // (join) => join.onRef('has_people.assetId', '=', 'assets.id'), + // ), + // ) + .$if(!!options.withStacked, (qb) => + qb + .leftJoin('asset_stack', (join) => + join + .onRef('asset_stack.id', '=', 'assets.stackId') + .onRef('asset_stack.primaryAssetId', '=', 'assets.id'), + ) + .where((eb) => + eb.or([eb('assets.stackId', 'is', null), eb(eb.table('asset_stack'), 'is not', null)]), + ), + ) + .$if(!!options.userIds, (qb) => qb.where('assets.ownerId', '=', anyUuid(options.userIds!))) + .$if(options.isArchived !== undefined, (qb) => qb.where('assets.isArchived', '=', options.isArchived!)) + .$if(options.isFavorite !== undefined, (qb) => qb.where('assets.isFavorite', '=', options.isFavorite!)) + .$if(!!options.assetType, (qb) => qb.where('assets.type', '=', options.assetType!)) + .$if(options.isDuplicate !== undefined, (qb) => + qb.where('assets.duplicateId', options.isDuplicate ? 'is not' : 'is', null), + ) + .$if(!!options.tagId, (qb) => withTagId(qb, options.tagId!)), + // .$if(!!options.tagId, (qb) => qb.where((eb) => withTagId(options.tagId!, eb.ref('assets.id')))), ) .selectFrom('assets') .select('timeBucket') @@ -782,7 +724,7 @@ export class AssetRepository { // .$if(!!options.personId, (qb) => hasPeople(qb, [options.personId!])) .$if(!!options.personId, (qb) => qb.innerJoin( - () => hasPeople([options.personId!]), + () => hasPeopleNoJoin([options.personId!]), (join) => join.onRef('has_people.assetId', '=', 'assets.id'), ), ) @@ -808,14 +750,14 @@ export class AssetRepository { .as('stacked_assets'), (join) => join.on('asset_stack.id', 'is not', null), ) - .select((eb) => eb.fn.toJson(eb.table('stacked_assets')).as('stack')), + .select((eb) => eb.fn.toJson(eb.table('stacked_assets').$castTo()).as('stack')), ) .$if(!!options.assetType, (qb) => qb.where('assets.type', '=', options.assetType!)) .$if(options.isDuplicate !== undefined, (qb) => qb.where('assets.duplicateId', options.isDuplicate ? 'is not' : 'is', null), ) .$if(!!options.isTrashed, (qb) => qb.where('assets.status', '!=', AssetStatus.DELETED)) - .$if(!!options.tagId, (qb) => qb.where((eb) => withTagId(options.tagId!, eb.ref('assets.id')))) + .$if(!!options.tagId, (qb) => qb.where((eb) => withTagIdNoWhere(options.tagId!, eb.ref('assets.id')))) .where('assets.deletedAt', options.isTrashed ? 'is not' : 'is', null) .where('assets.isVisible', '=', true) .where(truncatedDate(options.size), '=', timeBucket.replace(/^[+-]/, '')) @@ -877,7 +819,7 @@ export class AssetRepository { }; } - mapStack(entity?: { id: string | null; primaryAssetId: string | null; assetCount: string | number | bigint | null }) { + mapStack(entity?: Stack | null) { if (!entity) { return; } @@ -890,7 +832,7 @@ export class AssetRepository { } @GenerateSql({ params: [DummyValue.UUID] }) - getDuplicates(userId: string): Promise { + getDuplicates(userId: string) { return ( this.db .with('duplicates', (qb) => @@ -907,9 +849,15 @@ export class AssetRepository { (join) => join.onTrue(), ) .select('assets.duplicateId') - .select((eb) => eb.fn('jsonb_agg', [eb.table('asset')]).as('assets')) + .select((eb) => + eb + .fn('jsonb_agg', [eb.table('asset')]) + .$castTo() + .as('assets'), + ) .where('assets.ownerId', '=', asUuid(userId)) .where('assets.duplicateId', 'is not', null) + .$narrowType<{ duplicateId: NotNull }>() .where('assets.deletedAt', 'is', null) .where('assets.isVisible', '=', true) .where('assets.stackId', 'is', null) @@ -934,7 +882,7 @@ export class AssetRepository { .where(({ not, exists }) => not(exists((eb) => eb.selectFrom('unique').whereRef('unique.duplicateId', '=', 'duplicates.duplicateId'))), ) - .execute() as any as Promise + .execute() ); } @@ -978,7 +926,7 @@ export class AssetRepository { }, ], }) - getAllForUserFullSync(options: AssetFullSyncOptions): Promise { + getAllForUserFullSync(options: AssetFullSyncOptions) { const { ownerId, lastId, updatedUntil, limit } = options; return this.db .selectFrom('assets') @@ -996,18 +944,18 @@ export class AssetRepository { .as('stacked_assets'), (join) => join.on('asset_stack.id', 'is not', null), ) - .select((eb) => eb.fn.toJson(eb.table('stacked_assets')).as('stack')) + .select((eb) => eb.fn.toJson(eb.table('stacked_assets')).$castTo().as('stack')) .where('assets.ownerId', '=', asUuid(ownerId)) .where('assets.isVisible', '=', true) .where('assets.updatedAt', '<=', updatedUntil) .$if(!!lastId, (qb) => qb.where('assets.id', '>', lastId!)) .orderBy('assets.id') .limit(limit) - .execute() as any as Promise; + .execute(); } @GenerateSql({ params: [{ userIds: [DummyValue.UUID], updatedAfter: DummyValue.DATE, limit: 100 }] }) - async getChangedDeltaSync(options: AssetDeltaSyncOptions): Promise { + async getChangedDeltaSync(options: AssetDeltaSyncOptions) { return this.db .selectFrom('assets') .selectAll('assets') @@ -1024,12 +972,12 @@ export class AssetRepository { .as('stacked_assets'), (join) => join.on('asset_stack.id', 'is not', null), ) - .select((eb) => eb.fn.toJson(eb.table('stacked_assets')).as('stack')) + .select((eb) => eb.fn.toJson(eb.table('stacked_assets').$castTo()).as('stack')) .where('assets.ownerId', '=', anyUuid(options.userIds)) .where('assets.isVisible', '=', true) .where('assets.updatedAt', '>', options.updatedAfter) .limit(options.limit) - .execute() as any as Promise; + .execute(); } async upsertFile(file: Pick, 'assetId' | 'path' | 'type'>): Promise { diff --git a/server/src/repositories/database.repository.ts b/server/src/repositories/database.repository.ts index ec0b263408..c70c2cbdd4 100644 --- a/server/src/repositories/database.repository.ts +++ b/server/src/repositories/database.repository.ts @@ -2,7 +2,6 @@ import { Injectable } from '@nestjs/common'; import AsyncLock from 'async-lock'; import { FileMigrationProvider, Kysely, Migrator, sql, Transaction } from 'kysely'; import { InjectKysely } from 'nestjs-kysely'; -import { existsSync } from 'node:fs'; import { readdir } from 'node:fs/promises'; import { join } from 'node:path'; import semver from 'semver'; @@ -197,62 +196,54 @@ export class DatabaseRepository { return dimSize; } - async runMigrations(options?: { transaction?: 'all' | 'none' | 'each'; only?: 'kysely' | 'typeorm' }): Promise { + async runMigrations(options?: { transaction?: 'all' | 'none' | 'each' }): Promise { const { database } = this.configRepository.getEnv(); - if (options?.only !== 'kysely') { - const dataSource = new DataSource(database.config.typeorm); - this.logger.log('Running migrations, this may take a while'); + this.logger.log('Running migrations, this may take a while'); + const tableExists = sql<{ result: string | null }>`select to_regclass('migrations') as "result"`; + const { rows } = await tableExists.execute(this.db); + const hasTypeOrmMigrations = !!rows[0]?.result; + if (hasTypeOrmMigrations) { this.logger.debug('Running typeorm migrations'); - + const dataSource = new DataSource(database.config.typeorm); await dataSource.initialize(); await dataSource.runMigrations(options); await dataSource.destroy(); - this.logger.debug('Finished running typeorm migrations'); } - if (options?.only !== 'typeorm') { - // eslint-disable-next-line unicorn/prefer-module - const migrationFolder = join(__dirname, '..', 'schema/migrations'); + this.logger.debug('Running kysely migrations'); + const migrator = new Migrator({ + db: this.db, + migrationLockTableName: 'kysely_migrations_lock', + migrationTableName: 'kysely_migrations', + provider: new FileMigrationProvider({ + fs: { readdir }, + path: { join }, + // eslint-disable-next-line unicorn/prefer-module + migrationFolder: join(__dirname, '..', 'schema/migrations'), + }), + }); - // TODO remove after we have at least one kysely migration - if (!existsSync(migrationFolder)) { - return; + const { error, results } = await migrator.migrateToLatest(); + + for (const result of results ?? []) { + if (result.status === 'Success') { + this.logger.log(`Migration "${result.migrationName}" succeeded`); } - this.logger.debug('Running kysely migrations'); - const migrator = new Migrator({ - db: this.db, - migrationLockTableName: 'kysely_migrations_lock', - migrationTableName: 'kysely_migrations', - provider: new FileMigrationProvider({ - fs: { readdir }, - path: { join }, - migrationFolder, - }), - }); - - const { error, results } = await migrator.migrateToLatest(); - - for (const result of results ?? []) { - if (result.status === 'Success') { - this.logger.log(`Migration "${result.migrationName}" succeeded`); - } - - if (result.status === 'Error') { - this.logger.warn(`Migration "${result.migrationName}" failed`); - } + if (result.status === 'Error') { + this.logger.warn(`Migration "${result.migrationName}" failed`); } - - if (error) { - this.logger.error(`Kysely migrations failed: ${error}`); - throw error; - } - - this.logger.debug('Finished running kysely migrations'); } + + if (error) { + this.logger.error(`Kysely migrations failed: ${error}`); + throw error; + } + + this.logger.debug('Finished running kysely migrations'); } async withLock(lock: DatabaseLock, callback: () => Promise): Promise { diff --git a/server/src/repositories/logging.repository.ts b/server/src/repositories/logging.repository.ts index 3f809db41e..05d2d45f4d 100644 --- a/server/src/repositories/logging.repository.ts +++ b/server/src/repositories/logging.repository.ts @@ -74,11 +74,21 @@ export class MyConsoleLogger extends ConsoleLogger { export class LoggingRepository { private logger: MyConsoleLogger; - constructor(@Inject(ClsService) cls: ClsService | undefined, configRepository: ConfigRepository) { - const { noColor } = configRepository.getEnv(); + constructor( + @Inject(ClsService) cls: ClsService | undefined, + @Inject(ConfigRepository) configRepository: ConfigRepository | undefined, + ) { + let noColor = false; + if (configRepository) { + noColor = configRepository.getEnv().noColor; + } this.logger = new MyConsoleLogger(cls, { context: LoggingRepository.name, color: !noColor }); } + static create() { + return new LoggingRepository(undefined, undefined); + } + setAppName(name: string): void { appName = name.charAt(0).toUpperCase() + name.slice(1); } diff --git a/server/src/repositories/search.repository.ts b/server/src/repositories/search.repository.ts index c86ae8f60e..5c1ebae69d 100644 --- a/server/src/repositories/search.repository.ts +++ b/server/src/repositories/search.repository.ts @@ -1,13 +1,12 @@ import { Injectable } from '@nestjs/common'; -import { Kysely, OrderByDirection, sql } from 'kysely'; +import { Kysely, OrderByDirection, Selectable, sql } from 'kysely'; import { InjectKysely } from 'nestjs-kysely'; import { randomUUID } from 'node:crypto'; -import { DB } from 'src/db'; +import { DB, Exif } from 'src/db'; import { DummyValue, GenerateSql } from 'src/decorators'; -import { AssetEntity, searchAssetBuilder } from 'src/entities/asset.entity'; +import { MapAsset } from 'src/dtos/asset-response.dto'; import { AssetStatus, AssetType } from 'src/enum'; -import { anyUuid, asUuid } from 'src/utils/database'; -import { Paginated } from 'src/utils/pagination'; +import { anyUuid, asUuid, searchAssetBuilder } from 'src/utils/database'; import { isValidInteger } from 'src/validation'; export interface SearchResult { @@ -216,7 +215,7 @@ export class SearchRepository { }, ], }) - async searchMetadata(pagination: SearchPaginationOptions, options: AssetSearchOptions): Paginated { + async searchMetadata(pagination: SearchPaginationOptions, options: AssetSearchOptions) { const orderDirection = (options.orderDirection?.toLowerCase() || 'desc') as OrderByDirection; const items = await searchAssetBuilder(this.db, options) .orderBy('assets.fileCreatedAt', orderDirection) @@ -225,7 +224,7 @@ export class SearchRepository { .execute(); const hasNextPage = items.length > pagination.size; items.splice(pagination.size); - return { items: items as any as AssetEntity[], hasNextPage }; + return { items, hasNextPage }; } @GenerateSql({ @@ -240,7 +239,7 @@ export class SearchRepository { }, ], }) - async searchRandom(size: number, options: AssetSearchOptions): Promise { + async searchRandom(size: number, options: AssetSearchOptions) { const uuid = randomUUID(); const builder = searchAssetBuilder(this.db, options); const lessThan = builder @@ -251,8 +250,8 @@ export class SearchRepository { .where('assets.id', '>', uuid) .orderBy(sql`random()`) .limit(size); - const { rows } = await sql`${lessThan} union all ${greaterThan} limit ${size}`.execute(this.db); - return rows as any as AssetEntity[]; + const { rows } = await sql`${lessThan} union all ${greaterThan} limit ${size}`.execute(this.db); + return rows; } @GenerateSql({ @@ -268,17 +267,17 @@ export class SearchRepository { }, ], }) - async searchSmart(pagination: SearchPaginationOptions, options: SmartSearchOptions): Paginated { + async searchSmart(pagination: SearchPaginationOptions, options: SmartSearchOptions) { if (!isValidInteger(pagination.size, { min: 1, max: 1000 })) { throw new Error(`Invalid value for 'size': ${pagination.size}`); } - const items = (await searchAssetBuilder(this.db, options) + const items = await searchAssetBuilder(this.db, options) .innerJoin('smart_search', 'assets.id', 'smart_search.assetId') .orderBy(sql`smart_search.embedding <=> ${options.embedding}`) .limit(pagination.size + 1) .offset((pagination.page - 1) * pagination.size) - .execute()) as any as AssetEntity[]; + .execute(); const hasNextPage = items.length > pagination.size; items.splice(pagination.size); @@ -392,7 +391,7 @@ export class SearchRepository { } @GenerateSql({ params: [[DummyValue.UUID]] }) - getAssetsByCity(userIds: string[]): Promise { + getAssetsByCity(userIds: string[]) { return this.db .withRecursive('cte', (qb) => { const base = qb @@ -434,9 +433,14 @@ export class SearchRepository { .innerJoin('exif', 'assets.id', 'exif.assetId') .innerJoin('cte', 'assets.id', 'cte.assetId') .selectAll('assets') - .select((eb) => eb.fn('to_jsonb', [eb.table('exif')]).as('exifInfo')) + .select((eb) => + eb + .fn('to_jsonb', [eb.table('exif')]) + .$castTo>() + .as('exifInfo'), + ) .orderBy('exif.city') - .execute() as any as Promise; + .execute(); } async upsert(assetId: string, embedding: string): Promise { diff --git a/server/src/repositories/shared-link.repository.ts b/server/src/repositories/shared-link.repository.ts index 272d7f3794..67a97dc2d5 100644 --- a/server/src/repositories/shared-link.repository.ts +++ b/server/src/repositories/shared-link.repository.ts @@ -1,12 +1,12 @@ import { Injectable } from '@nestjs/common'; -import { Insertable, Kysely, sql, Updateable } from 'kysely'; +import { Insertable, Kysely, NotNull, sql, Updateable } from 'kysely'; import { jsonObjectFrom } from 'kysely/helpers/postgres'; import _ from 'lodash'; import { InjectKysely } from 'nestjs-kysely'; -import { columns } from 'src/database'; +import { Album, columns } from 'src/database'; import { DB, SharedLinks } from 'src/db'; import { DummyValue, GenerateSql } from 'src/decorators'; -import { SharedLinkEntity } from 'src/entities/shared-link.entity'; +import { MapAsset } from 'src/dtos/asset-response.dto'; import { SharedLinkType } from 'src/enum'; export type SharedLinkSearchOptions = { @@ -19,7 +19,7 @@ export class SharedLinkRepository { constructor(@InjectKysely() private db: Kysely) {} @GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID] }) - get(userId: string, id: string): Promise { + get(userId: string, id: string) { return this.db .selectFrom('shared_links') .selectAll('shared_links') @@ -87,18 +87,23 @@ export class SharedLinkRepository { .as('album'), (join) => join.onTrue(), ) - .select((eb) => eb.fn.coalesce(eb.fn.jsonAgg('a').filterWhere('a.id', 'is not', null), sql`'[]'`).as('assets')) + .select((eb) => + eb.fn + .coalesce(eb.fn.jsonAgg('a').filterWhere('a.id', 'is not', null), sql`'[]'`) + .$castTo() + .as('assets'), + ) .groupBy(['shared_links.id', sql`"album".*`]) - .select((eb) => eb.fn.toJson('album').as('album')) + .select((eb) => eb.fn.toJson('album').$castTo().as('album')) .where('shared_links.id', '=', id) .where('shared_links.userId', '=', userId) .where((eb) => eb.or([eb('shared_links.type', '=', SharedLinkType.INDIVIDUAL), eb('album.id', 'is not', null)])) .orderBy('shared_links.createdAt', 'desc') - .executeTakeFirst() as Promise; + .executeTakeFirst(); } @GenerateSql({ params: [{ userId: DummyValue.UUID, albumId: DummyValue.UUID }] }) - getAll({ userId, albumId }: SharedLinkSearchOptions): Promise { + getAll({ userId, albumId }: SharedLinkSearchOptions) { return this.db .selectFrom('shared_links') .selectAll('shared_links') @@ -115,6 +120,7 @@ export class SharedLinkRepository { (join) => join.onTrue(), ) .select('assets.assets') + .$narrowType<{ assets: NotNull }>() .leftJoinLateral( (eb) => eb @@ -152,12 +158,12 @@ export class SharedLinkRepository { .as('album'), (join) => join.onTrue(), ) - .select((eb) => eb.fn.toJson('album').as('album')) + .select((eb) => eb.fn.toJson('album').$castTo().as('album')) .where((eb) => eb.or([eb('shared_links.type', '=', SharedLinkType.INDIVIDUAL), eb('album.id', 'is not', null)])) .$if(!!albumId, (eb) => eb.where('shared_links.albumId', '=', albumId!)) .orderBy('shared_links.createdAt', 'desc') .distinctOn(['shared_links.createdAt']) - .execute() as unknown as Promise; + .execute(); } @GenerateSql({ params: [DummyValue.BUFFER] }) @@ -177,7 +183,7 @@ export class SharedLinkRepository { .executeTakeFirst(); } - async create(entity: Insertable & { assetIds?: string[] }): Promise { + async create(entity: Insertable & { assetIds?: string[] }) { const { id } = await this.db .insertInto('shared_links') .values(_.omit(entity, 'assetIds')) @@ -194,7 +200,7 @@ export class SharedLinkRepository { return this.getSharedLinks(id); } - async update(entity: Updateable & { id: string; assetIds?: string[] }): Promise { + async update(entity: Updateable & { id: string; assetIds?: string[] }) { const { id } = await this.db .updateTable('shared_links') .set(_.omit(entity, 'assets', 'album', 'assetIds')) @@ -212,8 +218,8 @@ export class SharedLinkRepository { return this.getSharedLinks(id); } - async remove(entity: SharedLinkEntity): Promise { - await this.db.deleteFrom('shared_links').where('shared_links.id', '=', entity.id).execute(); + async remove(id: string): Promise { + await this.db.deleteFrom('shared_links').where('shared_links.id', '=', id).execute(); } private getSharedLinks(id: string) { @@ -236,9 +242,12 @@ export class SharedLinkRepository { (join) => join.onTrue(), ) .select((eb) => - eb.fn.coalesce(eb.fn.jsonAgg('assets').filterWhere('assets.id', 'is not', null), sql`'[]'`).as('assets'), + eb.fn + .coalesce(eb.fn.jsonAgg('assets').filterWhere('assets.id', 'is not', null), sql`'[]'`) + .$castTo() + .as('assets'), ) .groupBy('shared_links.id') - .executeTakeFirstOrThrow() as Promise; + .executeTakeFirstOrThrow(); } } diff --git a/server/src/repositories/stack.repository.ts b/server/src/repositories/stack.repository.ts index 75dd9b497f..c9d69fb37f 100644 --- a/server/src/repositories/stack.repository.ts +++ b/server/src/repositories/stack.repository.ts @@ -5,7 +5,6 @@ import { InjectKysely } from 'nestjs-kysely'; import { columns } from 'src/database'; import { AssetStack, DB } from 'src/db'; import { DummyValue, GenerateSql } from 'src/decorators'; -import { AssetEntity } from 'src/entities/asset.entity'; import { asUuid } from 'src/utils/database'; export interface StackSearch { @@ -36,9 +35,7 @@ const withAssets = (eb: ExpressionBuilder, withTags = false) .select((eb) => eb.fn.toJson('exifInfo').as('exifInfo')) .where('assets.deletedAt', 'is', null) .whereRef('assets.stackId', '=', 'asset_stack.id'), - ) - .$castTo() - .as('assets'); + ).as('assets'); }; @Injectable() diff --git a/server/src/repositories/view-repository.ts b/server/src/repositories/view-repository.ts index ae2303e9e2..e32933065c 100644 --- a/server/src/repositories/view-repository.ts +++ b/server/src/repositories/view-repository.ts @@ -2,8 +2,7 @@ import { Kysely } from 'kysely'; import { InjectKysely } from 'nestjs-kysely'; import { DB } from 'src/db'; import { DummyValue, GenerateSql } from 'src/decorators'; -import { withExif } from 'src/entities/asset.entity'; -import { asUuid } from 'src/utils/database'; +import { asUuid, withExif } from 'src/utils/database'; export class ViewRepository { constructor(@InjectKysely() private db: Kysely) {} diff --git a/server/src/schema/migrations/1744910873969-InitialMigration.ts b/server/src/schema/migrations/1744910873969-InitialMigration.ts new file mode 100644 index 0000000000..459534a26a --- /dev/null +++ b/server/src/schema/migrations/1744910873969-InitialMigration.ts @@ -0,0 +1,410 @@ +import { Kysely, sql } from 'kysely'; +import { DatabaseExtension } from 'src/enum'; +import { ConfigRepository } from 'src/repositories/config.repository'; +import { LoggingRepository } from 'src/repositories/logging.repository'; + +const vectorExtension = new ConfigRepository().getEnv().database.vectorExtension; +const lastMigrationSql = sql<{ name: string }>`SELECT "name" FROM "migrations" ORDER BY "timestamp" DESC LIMIT 1;`; +const tableExists = sql<{ result: string | null }>`select to_regclass('migrations') as "result"`; +const logger = LoggingRepository.create(); + +export async function up(db: Kysely): Promise { + const { rows } = await tableExists.execute(db); + const hasTypeOrmMigrations = !!rows[0]?.result; + if (hasTypeOrmMigrations) { + const { + rows: [lastMigration], + } = await lastMigrationSql.execute(db); + if (lastMigration?.name !== 'AddMissingIndex1744910873956') { + throw new Error( + 'Invalid upgrade path. For more information, see https://immich.app/errors#typeorm-upgrade', + ); + } + logger.log('Database has up to date TypeORM migrations, skipping initial Kysely migration'); + return; + } + + await sql`CREATE EXTENSION IF NOT EXISTS "uuid-ossp";`.execute(db); + await sql`CREATE EXTENSION IF NOT EXISTS "unaccent";`.execute(db); + await sql`CREATE EXTENSION IF NOT EXISTS "cube";`.execute(db); + await sql`CREATE EXTENSION IF NOT EXISTS "earthdistance";`.execute(db); + await sql`CREATE EXTENSION IF NOT EXISTS "pg_trgm";`.execute(db); + await sql`CREATE EXTENSION IF NOT EXISTS "vectors";`.execute(db); + await sql`CREATE OR REPLACE FUNCTION immich_uuid_v7(p_timestamp timestamp with time zone default clock_timestamp()) + RETURNS uuid + VOLATILE LANGUAGE SQL + AS $$ + select encode( + set_bit( + set_bit( + overlay(uuid_send(gen_random_uuid()) + placing substring(int8send(floor(extract(epoch from p_timestamp) * 1000)::bigint) from 3) + from 1 for 6 + ), + 52, 1 + ), + 53, 1 + ), + 'hex')::uuid; + $$;`.execute(db); + await sql`CREATE OR REPLACE FUNCTION updated_at() + RETURNS TRIGGER + LANGUAGE PLPGSQL + AS $$ + DECLARE + clock_timestamp TIMESTAMP := clock_timestamp(); + BEGIN + new."updatedAt" = clock_timestamp; + new."updateId" = immich_uuid_v7(clock_timestamp); + return new; + END; + $$;`.execute(db); + await sql`CREATE OR REPLACE FUNCTION f_concat_ws(text, text[]) + RETURNS text + PARALLEL SAFE IMMUTABLE LANGUAGE SQL + AS $$SELECT array_to_string($2, $1)$$;`.execute(db); + await sql`CREATE OR REPLACE FUNCTION f_unaccent(text) + RETURNS text + PARALLEL SAFE STRICT IMMUTABLE LANGUAGE SQL + RETURN unaccent('unaccent', $1)`.execute(db); + await sql`CREATE OR REPLACE FUNCTION ll_to_earth_public(latitude double precision, longitude double precision) + RETURNS public.earth + PARALLEL SAFE STRICT IMMUTABLE LANGUAGE SQL + AS $$ + SELECT public.cube(public.cube(public.cube(public.earth()*cos(radians(latitude))*cos(radians(longitude))),public.earth()*cos(radians(latitude))*sin(radians(longitude))),public.earth()*sin(radians(latitude)))::public.earth + $$;`.execute(db); + await sql`CREATE OR REPLACE FUNCTION users_delete_audit() + RETURNS TRIGGER + LANGUAGE PLPGSQL + AS $$ + BEGIN + INSERT INTO users_audit ("userId") + SELECT "id" + FROM OLD; + RETURN NULL; + END; + $$;`.execute(db); + await sql`CREATE OR REPLACE FUNCTION partners_delete_audit() + RETURNS TRIGGER + LANGUAGE PLPGSQL + AS $$ + BEGIN + INSERT INTO partners_audit ("sharedById", "sharedWithId") + SELECT "sharedById", "sharedWithId" + FROM OLD; + RETURN NULL; + END; + $$;`.execute(db); + await sql`CREATE OR REPLACE FUNCTION assets_delete_audit() + RETURNS TRIGGER + LANGUAGE PLPGSQL + AS $$ + BEGIN + INSERT INTO assets_audit ("assetId", "ownerId") + SELECT "id", "ownerId" + FROM OLD; + RETURN NULL; + END; + $$;`.execute(db); + if (vectorExtension === DatabaseExtension.VECTORS) { + await sql`SET search_path TO "$user", public, vectors`.execute(db); + await sql`SET vectors.pgvector_compatibility=on`.execute(db); + } + await sql`CREATE TYPE "assets_status_enum" AS ENUM ('active','trashed','deleted');`.execute(db); + await sql`CREATE TYPE "sourcetype" AS ENUM ('machine-learning','exif','manual');`.execute(db); + await sql`CREATE TABLE "users" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "email" character varying NOT NULL, "password" character varying NOT NULL DEFAULT '', "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "profileImagePath" character varying NOT NULL DEFAULT '', "isAdmin" boolean NOT NULL DEFAULT false, "shouldChangePassword" boolean NOT NULL DEFAULT true, "deletedAt" timestamp with time zone, "oauthId" character varying NOT NULL DEFAULT '', "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "storageLabel" character varying, "name" character varying NOT NULL DEFAULT '', "quotaSizeInBytes" bigint, "quotaUsageInBytes" bigint NOT NULL DEFAULT 0, "status" character varying NOT NULL DEFAULT 'active', "profileChangedAt" timestamp with time zone NOT NULL DEFAULT now(), "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "libraries" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "name" character varying NOT NULL, "ownerId" uuid NOT NULL, "importPaths" text[] NOT NULL, "exclusionPatterns" text[] NOT NULL, "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "deletedAt" timestamp with time zone, "refreshedAt" timestamp with time zone, "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "asset_stack" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "primaryAssetId" uuid NOT NULL, "ownerId" uuid NOT NULL);`.execute(db); + await sql`CREATE TABLE "assets" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "deviceAssetId" character varying NOT NULL, "ownerId" uuid NOT NULL, "deviceId" character varying NOT NULL, "type" character varying NOT NULL, "originalPath" character varying NOT NULL, "fileCreatedAt" timestamp with time zone NOT NULL, "fileModifiedAt" timestamp with time zone NOT NULL, "isFavorite" boolean NOT NULL DEFAULT false, "duration" character varying, "encodedVideoPath" character varying DEFAULT '', "checksum" bytea NOT NULL, "isVisible" boolean NOT NULL DEFAULT true, "livePhotoVideoId" uuid, "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "isArchived" boolean NOT NULL DEFAULT false, "originalFileName" character varying NOT NULL, "sidecarPath" character varying, "thumbhash" bytea, "isOffline" boolean NOT NULL DEFAULT false, "libraryId" uuid, "isExternal" boolean NOT NULL DEFAULT false, "deletedAt" timestamp with time zone, "localDateTime" timestamp with time zone NOT NULL, "stackId" uuid, "duplicateId" uuid, "status" assets_status_enum NOT NULL DEFAULT 'active', "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "albums" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "ownerId" uuid NOT NULL, "albumName" character varying NOT NULL DEFAULT 'Untitled Album', "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "albumThumbnailAssetId" uuid, "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "description" text NOT NULL DEFAULT '', "deletedAt" timestamp with time zone, "isActivityEnabled" boolean NOT NULL DEFAULT true, "order" character varying NOT NULL DEFAULT 'desc', "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`COMMENT ON COLUMN "albums"."albumThumbnailAssetId" IS 'Asset ID to be used as thumbnail';`.execute(db); + await sql`CREATE TABLE "activity" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "albumId" uuid NOT NULL, "userId" uuid NOT NULL, "assetId" uuid, "comment" text, "isLiked" boolean NOT NULL DEFAULT false, "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "albums_assets_assets" ("albumsId" uuid NOT NULL, "assetsId" uuid NOT NULL, "createdAt" timestamp with time zone NOT NULL DEFAULT now());`.execute(db); + await sql`CREATE TABLE "albums_shared_users_users" ("albumsId" uuid NOT NULL, "usersId" uuid NOT NULL, "role" character varying NOT NULL DEFAULT 'editor');`.execute(db); + await sql`CREATE TABLE "api_keys" ("name" character varying NOT NULL, "key" character varying NOT NULL, "userId" uuid NOT NULL, "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "id" uuid NOT NULL DEFAULT uuid_generate_v4(), "permissions" character varying[] NOT NULL, "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "assets_audit" ("id" uuid NOT NULL DEFAULT immich_uuid_v7(), "assetId" uuid NOT NULL, "ownerId" uuid NOT NULL, "deletedAt" timestamp with time zone NOT NULL DEFAULT clock_timestamp());`.execute(db); + await sql`CREATE TABLE "person" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "ownerId" uuid NOT NULL, "name" character varying NOT NULL DEFAULT '', "thumbnailPath" character varying NOT NULL DEFAULT '', "isHidden" boolean NOT NULL DEFAULT false, "birthDate" date, "faceAssetId" uuid, "isFavorite" boolean NOT NULL DEFAULT false, "color" character varying, "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "asset_faces" ("assetId" uuid NOT NULL, "personId" uuid, "imageWidth" integer NOT NULL DEFAULT 0, "imageHeight" integer NOT NULL DEFAULT 0, "boundingBoxX1" integer NOT NULL DEFAULT 0, "boundingBoxY1" integer NOT NULL DEFAULT 0, "boundingBoxX2" integer NOT NULL DEFAULT 0, "boundingBoxY2" integer NOT NULL DEFAULT 0, "id" uuid NOT NULL DEFAULT uuid_generate_v4(), "sourceType" sourcetype NOT NULL DEFAULT 'machine-learning', "deletedAt" timestamp with time zone);`.execute(db); + await sql`CREATE TABLE "asset_files" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "assetId" uuid NOT NULL, "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "type" character varying NOT NULL, "path" character varying NOT NULL, "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "asset_job_status" ("assetId" uuid NOT NULL, "facesRecognizedAt" timestamp with time zone, "metadataExtractedAt" timestamp with time zone, "duplicatesDetectedAt" timestamp with time zone, "previewAt" timestamp with time zone, "thumbnailAt" timestamp with time zone);`.execute(db); + await sql`CREATE TABLE "audit" ("id" serial NOT NULL, "entityType" character varying NOT NULL, "entityId" uuid NOT NULL, "action" character varying NOT NULL, "ownerId" uuid NOT NULL, "createdAt" timestamp with time zone NOT NULL DEFAULT now());`.execute(db); + await sql`CREATE TABLE "exif" ("assetId" uuid NOT NULL, "make" character varying, "model" character varying, "exifImageWidth" integer, "exifImageHeight" integer, "fileSizeInByte" bigint, "orientation" character varying, "dateTimeOriginal" timestamp with time zone, "modifyDate" timestamp with time zone, "lensModel" character varying, "fNumber" double precision, "focalLength" double precision, "iso" integer, "latitude" double precision, "longitude" double precision, "city" character varying, "state" character varying, "country" character varying, "description" text NOT NULL DEFAULT '', "fps" double precision, "exposureTime" character varying, "livePhotoCID" character varying, "timeZone" character varying, "projectionType" character varying, "profileDescription" character varying, "colorspace" character varying, "bitsPerSample" integer, "autoStackId" character varying, "rating" integer, "updatedAt" timestamp with time zone NOT NULL DEFAULT clock_timestamp(), "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "face_search" ("faceId" uuid NOT NULL, "embedding" vector(512) NOT NULL);`.execute(db); + await sql`CREATE TABLE "geodata_places" ("id" integer NOT NULL, "name" character varying(200) NOT NULL, "longitude" double precision NOT NULL, "latitude" double precision NOT NULL, "countryCode" character(2) NOT NULL, "admin1Code" character varying(20), "admin2Code" character varying(80), "modificationDate" date NOT NULL, "admin1Name" character varying, "admin2Name" character varying, "alternateNames" character varying);`.execute(db); + await sql`CREATE TABLE "memories" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "deletedAt" timestamp with time zone, "ownerId" uuid NOT NULL, "type" character varying NOT NULL, "data" jsonb NOT NULL, "isSaved" boolean NOT NULL DEFAULT false, "memoryAt" timestamp with time zone NOT NULL, "seenAt" timestamp with time zone, "showAt" timestamp with time zone, "hideAt" timestamp with time zone, "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "memories_assets_assets" ("memoriesId" uuid NOT NULL, "assetsId" uuid NOT NULL);`.execute(db); + await sql`CREATE TABLE "move_history" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "entityId" uuid NOT NULL, "pathType" character varying NOT NULL, "oldPath" character varying NOT NULL, "newPath" character varying NOT NULL);`.execute(db); + await sql`CREATE TABLE "naturalearth_countries" ("id" integer NOT NULL GENERATED ALWAYS AS IDENTITY, "admin" character varying(50) NOT NULL, "admin_a3" character varying(3) NOT NULL, "type" character varying(50) NOT NULL, "coordinates" polygon NOT NULL);`.execute(db); + await sql`CREATE TABLE "partners_audit" ("id" uuid NOT NULL DEFAULT immich_uuid_v7(), "sharedById" uuid NOT NULL, "sharedWithId" uuid NOT NULL, "deletedAt" timestamp with time zone NOT NULL DEFAULT clock_timestamp());`.execute(db); + await sql`CREATE TABLE "partners" ("sharedById" uuid NOT NULL, "sharedWithId" uuid NOT NULL, "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "inTimeline" boolean NOT NULL DEFAULT false, "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "sessions" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "token" character varying NOT NULL, "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "userId" uuid NOT NULL, "deviceType" character varying NOT NULL DEFAULT '', "deviceOS" character varying NOT NULL DEFAULT '', "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "shared_links" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "description" character varying, "userId" uuid NOT NULL, "key" bytea NOT NULL, "type" character varying NOT NULL, "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "expiresAt" timestamp with time zone, "allowUpload" boolean NOT NULL DEFAULT false, "albumId" uuid, "allowDownload" boolean NOT NULL DEFAULT true, "showExif" boolean NOT NULL DEFAULT true, "password" character varying);`.execute(db); + await sql`CREATE TABLE "shared_link__asset" ("assetsId" uuid NOT NULL, "sharedLinksId" uuid NOT NULL);`.execute(db); + await sql`CREATE TABLE "smart_search" ("assetId" uuid NOT NULL, "embedding" vector(512) NOT NULL);`.execute(db); + await sql`ALTER TABLE "smart_search" ALTER COLUMN "embedding" SET STORAGE EXTERNAL;`.execute(db); + await sql`CREATE TABLE "session_sync_checkpoints" ("sessionId" uuid NOT NULL, "type" character varying NOT NULL, "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "ack" character varying NOT NULL, "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "system_metadata" ("key" character varying NOT NULL, "value" jsonb NOT NULL);`.execute(db); + await sql`CREATE TABLE "tags" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "userId" uuid NOT NULL, "value" character varying NOT NULL, "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "color" character varying, "parentId" uuid, "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "tag_asset" ("assetsId" uuid NOT NULL, "tagsId" uuid NOT NULL);`.execute(db); + await sql`CREATE TABLE "tags_closure" ("id_ancestor" uuid NOT NULL, "id_descendant" uuid NOT NULL);`.execute(db); + await sql`CREATE TABLE "users_audit" ("userId" uuid NOT NULL, "deletedAt" timestamp with time zone NOT NULL DEFAULT clock_timestamp(), "id" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "user_metadata" ("userId" uuid NOT NULL, "key" character varying NOT NULL, "value" jsonb NOT NULL);`.execute(db); + await sql`CREATE TABLE "version_history" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "version" character varying NOT NULL);`.execute(db); + await sql`ALTER TABLE "users" ADD CONSTRAINT "PK_a3ffb1c0c8416b9fc6f907b7433" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "libraries" ADD CONSTRAINT "PK_505fedfcad00a09b3734b4223de" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "asset_stack" ADD CONSTRAINT "PK_74a27e7fcbd5852463d0af3034b" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "assets" ADD CONSTRAINT "PK_da96729a8b113377cfb6a62439c" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "albums" ADD CONSTRAINT "PK_7f71c7b5bc7c87b8f94c9a93a00" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "activity" ADD CONSTRAINT "PK_24625a1d6b1b089c8ae206fe467" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "albums_assets_assets" ADD CONSTRAINT "PK_c67bc36fa845fb7b18e0e398180" PRIMARY KEY ("albumsId", "assetsId");`.execute(db); + await sql`ALTER TABLE "albums_shared_users_users" ADD CONSTRAINT "PK_7df55657e0b2e8b626330a0ebc8" PRIMARY KEY ("albumsId", "usersId");`.execute(db); + await sql`ALTER TABLE "api_keys" ADD CONSTRAINT "PK_5c8a79801b44bd27b79228e1dad" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "assets_audit" ADD CONSTRAINT "PK_99bd5c015f81a641927a32b4212" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "person" ADD CONSTRAINT "PK_5fdaf670315c4b7e70cce85daa3" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "asset_faces" ADD CONSTRAINT "PK_6df76ab2eb6f5b57b7c2f1fc684" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "asset_files" ADD CONSTRAINT "PK_c41dc3e9ef5e1c57ca5a08a0004" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "asset_job_status" ADD CONSTRAINT "PK_420bec36fc02813bddf5c8b73d4" PRIMARY KEY ("assetId");`.execute(db); + await sql`ALTER TABLE "audit" ADD CONSTRAINT "PK_1d3d120ddaf7bc9b1ed68ed463a" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "exif" ADD CONSTRAINT "PK_c0117fdbc50b917ef9067740c44" PRIMARY KEY ("assetId");`.execute(db); + await sql`ALTER TABLE "face_search" ADD CONSTRAINT "face_search_pkey" PRIMARY KEY ("faceId");`.execute(db); + await sql`ALTER TABLE "geodata_places" ADD CONSTRAINT "PK_c29918988912ef4036f3d7fbff4" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "memories" ADD CONSTRAINT "PK_aaa0692d9496fe827b0568612f8" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "memories_assets_assets" ADD CONSTRAINT "PK_fcaf7112a013d1703c011c6793d" PRIMARY KEY ("memoriesId", "assetsId");`.execute(db); + await sql`ALTER TABLE "move_history" ADD CONSTRAINT "PK_af608f132233acf123f2949678d" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "naturalearth_countries" ADD CONSTRAINT "PK_21a6d86d1ab5d841648212e5353" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "partners_audit" ADD CONSTRAINT "PK_952b50217ff78198a7e380f0359" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "partners" ADD CONSTRAINT "PK_f1cc8f73d16b367f426261a8736" PRIMARY KEY ("sharedById", "sharedWithId");`.execute(db); + await sql`ALTER TABLE "sessions" ADD CONSTRAINT "PK_48cb6b5c20faa63157b3c1baf7f" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "shared_links" ADD CONSTRAINT "PK_642e2b0f619e4876e5f90a43465" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "shared_link__asset" ADD CONSTRAINT "PK_9b4f3687f9b31d1e311336b05e3" PRIMARY KEY ("assetsId", "sharedLinksId");`.execute(db); + await sql`ALTER TABLE "smart_search" ADD CONSTRAINT "smart_search_pkey" PRIMARY KEY ("assetId");`.execute(db); + await sql`ALTER TABLE "session_sync_checkpoints" ADD CONSTRAINT "PK_b846ab547a702863ef7cd9412fb" PRIMARY KEY ("sessionId", "type");`.execute(db); + await sql`ALTER TABLE "system_metadata" ADD CONSTRAINT "PK_fa94f6857470fb5b81ec6084465" PRIMARY KEY ("key");`.execute(db); + await sql`ALTER TABLE "tags" ADD CONSTRAINT "PK_e7dc17249a1148a1970748eda99" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "tag_asset" ADD CONSTRAINT "PK_ef5346fe522b5fb3bc96454747e" PRIMARY KEY ("assetsId", "tagsId");`.execute(db); + await sql`ALTER TABLE "tags_closure" ADD CONSTRAINT "PK_eab38eb12a3ec6df8376c95477c" PRIMARY KEY ("id_ancestor", "id_descendant");`.execute(db); + await sql`ALTER TABLE "users_audit" ADD CONSTRAINT "PK_e9b2bdfd90e7eb5961091175180" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "user_metadata" ADD CONSTRAINT "PK_5931462150b3438cbc83277fe5a" PRIMARY KEY ("userId", "key");`.execute(db); + await sql`ALTER TABLE "version_history" ADD CONSTRAINT "PK_5db259cbb09ce82c0d13cfd1b23" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "libraries" ADD CONSTRAINT "FK_0f6fc2fb195f24d19b0fb0d57c1" FOREIGN KEY ("ownerId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "asset_stack" ADD CONSTRAINT "FK_91704e101438fd0653f582426dc" FOREIGN KEY ("primaryAssetId") REFERENCES "assets" ("id") ON UPDATE NO ACTION ON DELETE NO ACTION;`.execute(db); + await sql`ALTER TABLE "asset_stack" ADD CONSTRAINT "FK_c05079e542fd74de3b5ecb5c1c8" FOREIGN KEY ("ownerId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "assets" ADD CONSTRAINT "FK_2c5ac0d6fb58b238fd2068de67d" FOREIGN KEY ("ownerId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "assets" ADD CONSTRAINT "FK_16294b83fa8c0149719a1f631ef" FOREIGN KEY ("livePhotoVideoId") REFERENCES "assets" ("id") ON UPDATE CASCADE ON DELETE SET NULL;`.execute(db); + await sql`ALTER TABLE "assets" ADD CONSTRAINT "FK_9977c3c1de01c3d848039a6b90c" FOREIGN KEY ("libraryId") REFERENCES "libraries" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "assets" ADD CONSTRAINT "FK_f15d48fa3ea5e4bda05ca8ab207" FOREIGN KEY ("stackId") REFERENCES "asset_stack" ("id") ON UPDATE CASCADE ON DELETE SET NULL;`.execute(db); + await sql`ALTER TABLE "albums" ADD CONSTRAINT "FK_b22c53f35ef20c28c21637c85f4" FOREIGN KEY ("ownerId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "albums" ADD CONSTRAINT "FK_05895aa505a670300d4816debce" FOREIGN KEY ("albumThumbnailAssetId") REFERENCES "assets" ("id") ON UPDATE CASCADE ON DELETE SET NULL;`.execute(db); + await sql`ALTER TABLE "activity" ADD CONSTRAINT "FK_1af8519996fbfb3684b58df280b" FOREIGN KEY ("albumId") REFERENCES "albums" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "activity" ADD CONSTRAINT "FK_3571467bcbe021f66e2bdce96ea" FOREIGN KEY ("userId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "activity" ADD CONSTRAINT "FK_8091ea76b12338cb4428d33d782" FOREIGN KEY ("assetId") REFERENCES "assets" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "albums_assets_assets" ADD CONSTRAINT "FK_e590fa396c6898fcd4a50e40927" FOREIGN KEY ("albumsId") REFERENCES "albums" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "albums_assets_assets" ADD CONSTRAINT "FK_4bd1303d199f4e72ccdf998c621" FOREIGN KEY ("assetsId") REFERENCES "assets" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "albums_shared_users_users" ADD CONSTRAINT "FK_427c350ad49bd3935a50baab737" FOREIGN KEY ("albumsId") REFERENCES "albums" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "albums_shared_users_users" ADD CONSTRAINT "FK_f48513bf9bccefd6ff3ad30bd06" FOREIGN KEY ("usersId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "api_keys" ADD CONSTRAINT "FK_6c2e267ae764a9413b863a29342" FOREIGN KEY ("userId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "person" ADD CONSTRAINT "FK_5527cc99f530a547093f9e577b6" FOREIGN KEY ("ownerId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "person" ADD CONSTRAINT "FK_2bbabe31656b6778c6b87b61023" FOREIGN KEY ("faceAssetId") REFERENCES "asset_faces" ("id") ON UPDATE NO ACTION ON DELETE SET NULL;`.execute(db); + await sql`ALTER TABLE "asset_faces" ADD CONSTRAINT "FK_02a43fd0b3c50fb6d7f0cb7282c" FOREIGN KEY ("assetId") REFERENCES "assets" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "asset_faces" ADD CONSTRAINT "FK_95ad7106dd7b484275443f580f9" FOREIGN KEY ("personId") REFERENCES "person" ("id") ON UPDATE CASCADE ON DELETE SET NULL;`.execute(db); + await sql`ALTER TABLE "asset_files" ADD CONSTRAINT "FK_e3e103a5f1d8bc8402999286040" FOREIGN KEY ("assetId") REFERENCES "assets" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "asset_job_status" ADD CONSTRAINT "FK_420bec36fc02813bddf5c8b73d4" FOREIGN KEY ("assetId") REFERENCES "assets" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "exif" ADD CONSTRAINT "FK_c0117fdbc50b917ef9067740c44" FOREIGN KEY ("assetId") REFERENCES "assets" ("id") ON UPDATE NO ACTION ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "face_search" ADD CONSTRAINT "face_search_faceId_fkey" FOREIGN KEY ("faceId") REFERENCES "asset_faces" ("id") ON UPDATE NO ACTION ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "memories" ADD CONSTRAINT "FK_575842846f0c28fa5da46c99b19" FOREIGN KEY ("ownerId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "memories_assets_assets" ADD CONSTRAINT "FK_984e5c9ab1f04d34538cd32334e" FOREIGN KEY ("memoriesId") REFERENCES "memories" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "memories_assets_assets" ADD CONSTRAINT "FK_6942ecf52d75d4273de19d2c16f" FOREIGN KEY ("assetsId") REFERENCES "assets" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "partners" ADD CONSTRAINT "FK_7e077a8b70b3530138610ff5e04" FOREIGN KEY ("sharedById") REFERENCES "users" ("id") ON UPDATE NO ACTION ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "partners" ADD CONSTRAINT "FK_d7e875c6c60e661723dbf372fd3" FOREIGN KEY ("sharedWithId") REFERENCES "users" ("id") ON UPDATE NO ACTION ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "sessions" ADD CONSTRAINT "FK_57de40bc620f456c7311aa3a1e6" FOREIGN KEY ("userId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "shared_links" ADD CONSTRAINT "FK_66fe3837414c5a9f1c33ca49340" FOREIGN KEY ("userId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "shared_links" ADD CONSTRAINT "FK_0c6ce9058c29f07cdf7014eac66" FOREIGN KEY ("albumId") REFERENCES "albums" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "shared_link__asset" ADD CONSTRAINT "FK_5b7decce6c8d3db9593d6111a66" FOREIGN KEY ("assetsId") REFERENCES "assets" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "shared_link__asset" ADD CONSTRAINT "FK_c9fab4aa97ffd1b034f3d6581ab" FOREIGN KEY ("sharedLinksId") REFERENCES "shared_links" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "smart_search" ADD CONSTRAINT "smart_search_assetId_fkey" FOREIGN KEY ("assetId") REFERENCES "assets" ("id") ON UPDATE NO ACTION ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "session_sync_checkpoints" ADD CONSTRAINT "FK_d8ddd9d687816cc490432b3d4bc" FOREIGN KEY ("sessionId") REFERENCES "sessions" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "tags" ADD CONSTRAINT "FK_92e67dc508c705dd66c94615576" FOREIGN KEY ("userId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "tags" ADD CONSTRAINT "FK_9f9590cc11561f1f48ff034ef99" FOREIGN KEY ("parentId") REFERENCES "tags" ("id") ON UPDATE NO ACTION ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "tag_asset" ADD CONSTRAINT "FK_f8e8a9e893cb5c54907f1b798e9" FOREIGN KEY ("assetsId") REFERENCES "assets" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "tag_asset" ADD CONSTRAINT "FK_e99f31ea4cdf3a2c35c7287eb42" FOREIGN KEY ("tagsId") REFERENCES "tags" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "tags_closure" ADD CONSTRAINT "FK_15fbcbc67663c6bfc07b354c22c" FOREIGN KEY ("id_ancestor") REFERENCES "tags" ("id") ON UPDATE NO ACTION ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "tags_closure" ADD CONSTRAINT "FK_b1a2a7ed45c29179b5ad51548a1" FOREIGN KEY ("id_descendant") REFERENCES "tags" ("id") ON UPDATE NO ACTION ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "user_metadata" ADD CONSTRAINT "FK_6afb43681a21cf7815932bc38ac" FOREIGN KEY ("userId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "users" ADD CONSTRAINT "UQ_97672ac88f789774dd47f7c8be3" UNIQUE ("email");`.execute(db); + await sql`ALTER TABLE "users" ADD CONSTRAINT "UQ_b309cf34fa58137c416b32cea3a" UNIQUE ("storageLabel");`.execute(db); + await sql`ALTER TABLE "asset_stack" ADD CONSTRAINT "REL_91704e101438fd0653f582426d" UNIQUE ("primaryAssetId");`.execute(db); + await sql`ALTER TABLE "asset_files" ADD CONSTRAINT "UQ_assetId_type" UNIQUE ("assetId", "type");`.execute(db); + await sql`ALTER TABLE "move_history" ADD CONSTRAINT "UQ_newPath" UNIQUE ("newPath");`.execute(db); + await sql`ALTER TABLE "move_history" ADD CONSTRAINT "UQ_entityId_pathType" UNIQUE ("entityId", "pathType");`.execute(db); + await sql`ALTER TABLE "shared_links" ADD CONSTRAINT "UQ_sharedlink_key" UNIQUE ("key");`.execute(db); + await sql`ALTER TABLE "tags" ADD CONSTRAINT "UQ_79d6f16e52bb2c7130375246793" UNIQUE ("userId", "value");`.execute(db); + await sql`ALTER TABLE "activity" ADD CONSTRAINT "CHK_2ab1e70f113f450eb40c1e3ec8" CHECK (("comment" IS NULL AND "isLiked" = true) OR ("comment" IS NOT NULL AND "isLiked" = false));`.execute(db); + await sql`ALTER TABLE "person" ADD CONSTRAINT "CHK_b0f82b0ed662bfc24fbb58bb45" CHECK ("birthDate" <= CURRENT_DATE);`.execute(db); + await sql`CREATE INDEX "IDX_users_updated_at_asc_id_asc" ON "users" ("updatedAt", "id")`.execute(db); + await sql`CREATE INDEX "IDX_users_update_id" ON "users" ("updateId")`.execute(db); + await sql`CREATE INDEX "IDX_0f6fc2fb195f24d19b0fb0d57c" ON "libraries" ("ownerId")`.execute(db); + await sql`CREATE INDEX "IDX_libraries_update_id" ON "libraries" ("updateId")`.execute(db); + await sql`CREATE INDEX "IDX_91704e101438fd0653f582426d" ON "asset_stack" ("primaryAssetId")`.execute(db); + await sql`CREATE INDEX "IDX_c05079e542fd74de3b5ecb5c1c" ON "asset_stack" ("ownerId")`.execute(db); + await sql`CREATE INDEX "idx_originalfilename_trigram" ON "assets" USING gin (f_unaccent("originalFileName") gin_trgm_ops)`.execute(db); + await sql`CREATE INDEX "IDX_asset_id_stackId" ON "assets" ("id", "stackId")`.execute(db); + await sql`CREATE INDEX "IDX_originalPath_libraryId" ON "assets" ("originalPath", "libraryId")`.execute(db); + await sql`CREATE INDEX "idx_local_date_time_month" ON "assets" ((date_trunc('MONTH'::text, ("localDateTime" AT TIME ZONE 'UTC'::text)) AT TIME ZONE 'UTC'::text))`.execute(db); + await sql`CREATE INDEX "idx_local_date_time" ON "assets" ((("localDateTime" at time zone 'UTC')::date))`.execute(db); + await sql`CREATE UNIQUE INDEX "UQ_assets_owner_library_checksum" ON "assets" ("ownerId", "libraryId", "checksum") WHERE ("libraryId" IS NOT NULL)`.execute(db); + await sql`CREATE UNIQUE INDEX "UQ_assets_owner_checksum" ON "assets" ("ownerId", "checksum") WHERE ("libraryId" IS NULL)`.execute(db); + await sql`CREATE INDEX "IDX_2c5ac0d6fb58b238fd2068de67" ON "assets" ("ownerId")`.execute(db); + await sql`CREATE INDEX "idx_asset_file_created_at" ON "assets" ("fileCreatedAt")`.execute(db); + await sql`CREATE INDEX "IDX_8d3efe36c0755849395e6ea866" ON "assets" ("checksum")`.execute(db); + await sql`CREATE INDEX "IDX_16294b83fa8c0149719a1f631e" ON "assets" ("livePhotoVideoId")`.execute(db); + await sql`CREATE INDEX "IDX_4d66e76dada1ca180f67a205dc" ON "assets" ("originalFileName")`.execute(db); + await sql`CREATE INDEX "IDX_9977c3c1de01c3d848039a6b90" ON "assets" ("libraryId")`.execute(db); + await sql`CREATE INDEX "IDX_f15d48fa3ea5e4bda05ca8ab20" ON "assets" ("stackId")`.execute(db); + await sql`CREATE INDEX "IDX_assets_duplicateId" ON "assets" ("duplicateId")`.execute(db); + await sql`CREATE INDEX "IDX_assets_update_id" ON "assets" ("updateId")`.execute(db); + await sql`CREATE INDEX "IDX_b22c53f35ef20c28c21637c85f" ON "albums" ("ownerId")`.execute(db); + await sql`CREATE INDEX "IDX_05895aa505a670300d4816debc" ON "albums" ("albumThumbnailAssetId")`.execute(db); + await sql`CREATE INDEX "IDX_albums_update_id" ON "albums" ("updateId")`.execute(db); + await sql`CREATE UNIQUE INDEX "IDX_activity_like" ON "activity" ("assetId", "userId", "albumId") WHERE ("isLiked" = true)`.execute(db); + await sql`CREATE INDEX "IDX_1af8519996fbfb3684b58df280" ON "activity" ("albumId")`.execute(db); + await sql`CREATE INDEX "IDX_3571467bcbe021f66e2bdce96e" ON "activity" ("userId")`.execute(db); + await sql`CREATE INDEX "IDX_8091ea76b12338cb4428d33d78" ON "activity" ("assetId")`.execute(db); + await sql`CREATE INDEX "IDX_activity_update_id" ON "activity" ("updateId")`.execute(db); + await sql`CREATE INDEX "IDX_e590fa396c6898fcd4a50e4092" ON "albums_assets_assets" ("albumsId")`.execute(db); + await sql`CREATE INDEX "IDX_4bd1303d199f4e72ccdf998c62" ON "albums_assets_assets" ("assetsId")`.execute(db); + await sql`CREATE INDEX "IDX_f48513bf9bccefd6ff3ad30bd0" ON "albums_shared_users_users" ("usersId")`.execute(db); + await sql`CREATE INDEX "IDX_427c350ad49bd3935a50baab73" ON "albums_shared_users_users" ("albumsId")`.execute(db); + await sql`CREATE INDEX "IDX_6c2e267ae764a9413b863a2934" ON "api_keys" ("userId")`.execute(db); + await sql`CREATE INDEX "IDX_api_keys_update_id" ON "api_keys" ("updateId")`.execute(db); + await sql`CREATE INDEX "IDX_assets_audit_asset_id" ON "assets_audit" ("assetId")`.execute(db); + await sql`CREATE INDEX "IDX_assets_audit_owner_id" ON "assets_audit" ("ownerId")`.execute(db); + await sql`CREATE INDEX "IDX_assets_audit_deleted_at" ON "assets_audit" ("deletedAt")`.execute(db); + await sql`CREATE INDEX "IDX_5527cc99f530a547093f9e577b" ON "person" ("ownerId")`.execute(db); + await sql`CREATE INDEX "IDX_2bbabe31656b6778c6b87b6102" ON "person" ("faceAssetId")`.execute(db); + await sql`CREATE INDEX "IDX_person_update_id" ON "person" ("updateId")`.execute(db); + await sql`CREATE INDEX "IDX_bf339a24070dac7e71304ec530" ON "asset_faces" ("personId", "assetId")`.execute(db); + await sql`CREATE INDEX "IDX_asset_faces_assetId_personId" ON "asset_faces" ("assetId", "personId")`.execute(db); + await sql`CREATE INDEX "IDX_02a43fd0b3c50fb6d7f0cb7282" ON "asset_faces" ("assetId")`.execute(db); + await sql`CREATE INDEX "IDX_95ad7106dd7b484275443f580f" ON "asset_faces" ("personId")`.execute(db); + await sql`CREATE INDEX "IDX_asset_files_assetId" ON "asset_files" ("assetId")`.execute(db); + await sql`CREATE INDEX "IDX_asset_files_update_id" ON "asset_files" ("updateId")`.execute(db); + await sql`CREATE INDEX "IDX_ownerId_createdAt" ON "audit" ("ownerId", "createdAt")`.execute(db); + await sql`CREATE INDEX "exif_city" ON "exif" ("city")`.execute(db); + await sql`CREATE INDEX "IDX_live_photo_cid" ON "exif" ("livePhotoCID")`.execute(db); + await sql`CREATE INDEX "IDX_auto_stack_id" ON "exif" ("autoStackId")`.execute(db); + await sql`CREATE INDEX "IDX_asset_exif_update_id" ON "exif" ("updateId")`.execute(db); + await sql`CREATE INDEX "face_index" ON "face_search" USING hnsw (embedding vector_cosine_ops) WITH (ef_construction = 300, m = 16)`.execute(db); + await sql`CREATE INDEX "IDX_geodata_gist_earthcoord" ON "geodata_places" (ll_to_earth_public(latitude, longitude))`.execute(db); + await sql`CREATE INDEX "idx_geodata_places_name" ON "geodata_places" USING gin (f_unaccent("name") gin_trgm_ops)`.execute(db); + await sql`CREATE INDEX "idx_geodata_places_admin2_name" ON "geodata_places" USING gin (f_unaccent("admin2Name") gin_trgm_ops)`.execute(db); + await sql`CREATE INDEX "idx_geodata_places_admin1_name" ON "geodata_places" USING gin (f_unaccent("admin1Name") gin_trgm_ops)`.execute(db); + await sql`CREATE INDEX "idx_geodata_places_alternate_names" ON "geodata_places" USING gin (f_unaccent("alternateNames") gin_trgm_ops)`.execute(db); + await sql`CREATE INDEX "IDX_575842846f0c28fa5da46c99b1" ON "memories" ("ownerId")`.execute(db); + await sql`CREATE INDEX "IDX_memories_update_id" ON "memories" ("updateId")`.execute(db); + await sql`CREATE INDEX "IDX_984e5c9ab1f04d34538cd32334" ON "memories_assets_assets" ("memoriesId")`.execute(db); + await sql`CREATE INDEX "IDX_6942ecf52d75d4273de19d2c16" ON "memories_assets_assets" ("assetsId")`.execute(db); + await sql`CREATE INDEX "IDX_partners_audit_shared_by_id" ON "partners_audit" ("sharedById")`.execute(db); + await sql`CREATE INDEX "IDX_partners_audit_shared_with_id" ON "partners_audit" ("sharedWithId")`.execute(db); + await sql`CREATE INDEX "IDX_partners_audit_deleted_at" ON "partners_audit" ("deletedAt")`.execute(db); + await sql`CREATE INDEX "IDX_7e077a8b70b3530138610ff5e0" ON "partners" ("sharedById")`.execute(db); + await sql`CREATE INDEX "IDX_d7e875c6c60e661723dbf372fd" ON "partners" ("sharedWithId")`.execute(db); + await sql`CREATE INDEX "IDX_partners_update_id" ON "partners" ("updateId")`.execute(db); + await sql`CREATE INDEX "IDX_57de40bc620f456c7311aa3a1e" ON "sessions" ("userId")`.execute(db); + await sql`CREATE INDEX "IDX_sessions_update_id" ON "sessions" ("updateId")`.execute(db); + await sql`CREATE INDEX "IDX_66fe3837414c5a9f1c33ca4934" ON "shared_links" ("userId")`.execute(db); + await sql`CREATE INDEX "IDX_sharedlink_key" ON "shared_links" ("key")`.execute(db); + await sql`CREATE INDEX "IDX_sharedlink_albumId" ON "shared_links" ("albumId")`.execute(db); + await sql`CREATE INDEX "IDX_5b7decce6c8d3db9593d6111a6" ON "shared_link__asset" ("assetsId")`.execute(db); + await sql`CREATE INDEX "IDX_c9fab4aa97ffd1b034f3d6581a" ON "shared_link__asset" ("sharedLinksId")`.execute(db); + await sql`CREATE INDEX "clip_index" ON "smart_search" USING hnsw (embedding vector_cosine_ops) WITH (ef_construction = 300, m = 16)`.execute(db); + await sql`CREATE INDEX "IDX_d8ddd9d687816cc490432b3d4b" ON "session_sync_checkpoints" ("sessionId")`.execute(db); + await sql`CREATE INDEX "IDX_session_sync_checkpoints_update_id" ON "session_sync_checkpoints" ("updateId")`.execute(db); + await sql`CREATE INDEX "IDX_92e67dc508c705dd66c9461557" ON "tags" ("userId")`.execute(db); + await sql`CREATE INDEX "IDX_9f9590cc11561f1f48ff034ef9" ON "tags" ("parentId")`.execute(db); + await sql`CREATE INDEX "IDX_tags_update_id" ON "tags" ("updateId")`.execute(db); + await sql`CREATE INDEX "IDX_tag_asset_assetsId_tagsId" ON "tag_asset" ("assetsId", "tagsId")`.execute(db); + await sql`CREATE INDEX "IDX_f8e8a9e893cb5c54907f1b798e" ON "tag_asset" ("assetsId")`.execute(db); + await sql`CREATE INDEX "IDX_e99f31ea4cdf3a2c35c7287eb4" ON "tag_asset" ("tagsId")`.execute(db); + await sql`CREATE INDEX "IDX_15fbcbc67663c6bfc07b354c22" ON "tags_closure" ("id_ancestor")`.execute(db); + await sql`CREATE INDEX "IDX_b1a2a7ed45c29179b5ad51548a" ON "tags_closure" ("id_descendant")`.execute(db); + await sql`CREATE INDEX "IDX_users_audit_deleted_at" ON "users_audit" ("deletedAt")`.execute(db); + await sql`CREATE INDEX "IDX_6afb43681a21cf7815932bc38a" ON "user_metadata" ("userId")`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "users_delete_audit" + AFTER DELETE ON "users" + REFERENCING OLD TABLE AS "old" + FOR EACH STATEMENT + WHEN (pg_trigger_depth() = 0) + EXECUTE FUNCTION users_delete_audit();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "users_updated_at" + BEFORE UPDATE ON "users" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "libraries_updated_at" + BEFORE UPDATE ON "libraries" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "assets_delete_audit" + AFTER DELETE ON "assets" + REFERENCING OLD TABLE AS "old" + FOR EACH STATEMENT + WHEN (pg_trigger_depth() = 0) + EXECUTE FUNCTION assets_delete_audit();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "assets_updated_at" + BEFORE UPDATE ON "assets" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "albums_updated_at" + BEFORE UPDATE ON "albums" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "activity_updated_at" + BEFORE UPDATE ON "activity" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "api_keys_updated_at" + BEFORE UPDATE ON "api_keys" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "person_updated_at" + BEFORE UPDATE ON "person" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "asset_files_updated_at" + BEFORE UPDATE ON "asset_files" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "asset_exif_updated_at" + BEFORE UPDATE ON "exif" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "memories_updated_at" + BEFORE UPDATE ON "memories" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "partners_delete_audit" + AFTER DELETE ON "partners" + REFERENCING OLD TABLE AS "old" + FOR EACH STATEMENT + WHEN (pg_trigger_depth() = 0) + EXECUTE FUNCTION partners_delete_audit();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "partners_updated_at" + BEFORE UPDATE ON "partners" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "sessions_updated_at" + BEFORE UPDATE ON "sessions" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "session_sync_checkpoints_updated_at" + BEFORE UPDATE ON "session_sync_checkpoints" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "tags_updated_at" + BEFORE UPDATE ON "tags" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); +} + +export async function down(): Promise { +// not implemented +} diff --git a/server/src/schema/tables/activity.table.ts b/server/src/schema/tables/activity.table.ts index e7a144722c..802a86a303 100644 --- a/server/src/schema/tables/activity.table.ts +++ b/server/src/schema/tables/activity.table.ts @@ -5,7 +5,6 @@ import { UserTable } from 'src/schema/tables/user.table'; import { Check, Column, - ColumnIndex, CreateDateColumn, ForeignKeyColumn, Index, @@ -51,7 +50,6 @@ export class ActivityTable { @Column({ type: 'boolean', default: false }) isLiked!: boolean; - @ColumnIndex('IDX_activity_update_id') - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_activity_update_id' }) updateId!: string; } diff --git a/server/src/schema/tables/album-asset.table.ts b/server/src/schema/tables/album-asset.table.ts index 1b931e3116..8054009c39 100644 --- a/server/src/schema/tables/album-asset.table.ts +++ b/server/src/schema/tables/album-asset.table.ts @@ -1,25 +1,13 @@ import { AlbumTable } from 'src/schema/tables/album.table'; import { AssetTable } from 'src/schema/tables/asset.table'; -import { ColumnIndex, CreateDateColumn, ForeignKeyColumn, Table } from 'src/sql-tools'; +import { CreateDateColumn, ForeignKeyColumn, Table } from 'src/sql-tools'; @Table({ name: 'albums_assets_assets', primaryConstraintName: 'PK_c67bc36fa845fb7b18e0e398180' }) export class AlbumAssetTable { - @ForeignKeyColumn(() => AlbumTable, { - onDelete: 'CASCADE', - onUpdate: 'CASCADE', - nullable: false, - primary: true, - }) - @ColumnIndex() + @ForeignKeyColumn(() => AlbumTable, { onDelete: 'CASCADE', onUpdate: 'CASCADE', nullable: false, primary: true }) albumsId!: string; - @ForeignKeyColumn(() => AssetTable, { - onDelete: 'CASCADE', - onUpdate: 'CASCADE', - nullable: false, - primary: true, - }) - @ColumnIndex() + @ForeignKeyColumn(() => AssetTable, { onDelete: 'CASCADE', onUpdate: 'CASCADE', nullable: false, primary: true }) assetsId!: string; @CreateDateColumn() diff --git a/server/src/schema/tables/album.table.ts b/server/src/schema/tables/album.table.ts index cdfd092b1b..428947fa51 100644 --- a/server/src/schema/tables/album.table.ts +++ b/server/src/schema/tables/album.table.ts @@ -4,7 +4,6 @@ import { AssetTable } from 'src/schema/tables/asset.table'; import { UserTable } from 'src/schema/tables/user.table'; import { Column, - ColumnIndex, CreateDateColumn, DeleteDateColumn, ForeignKeyColumn, @@ -51,7 +50,6 @@ export class AlbumTable { @Column({ default: AssetOrder.DESC }) order!: AssetOrder; - @ColumnIndex('IDX_albums_update_id') - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_albums_update_id' }) updateId?: string; } diff --git a/server/src/schema/tables/api-key.table.ts b/server/src/schema/tables/api-key.table.ts index 29c4ad2b0f..1d4cc83172 100644 --- a/server/src/schema/tables/api-key.table.ts +++ b/server/src/schema/tables/api-key.table.ts @@ -3,7 +3,6 @@ import { Permission } from 'src/enum'; import { UserTable } from 'src/schema/tables/user.table'; import { Column, - ColumnIndex, CreateDateColumn, ForeignKeyColumn, PrimaryGeneratedColumn, @@ -35,7 +34,6 @@ export class APIKeyTable { @Column({ array: true, type: 'character varying' }) permissions!: Permission[]; - @ColumnIndex({ name: 'IDX_api_keys_update_id' }) - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_api_keys_update_id' }) updateId?: string; } diff --git a/server/src/schema/tables/asset-audit.table.ts b/server/src/schema/tables/asset-audit.table.ts index 55d6f5c911..030256480c 100644 --- a/server/src/schema/tables/asset-audit.table.ts +++ b/server/src/schema/tables/asset-audit.table.ts @@ -1,20 +1,17 @@ import { PrimaryGeneratedUuidV7Column } from 'src/decorators'; -import { Column, ColumnIndex, CreateDateColumn, Table } from 'src/sql-tools'; +import { Column, CreateDateColumn, Table } from 'src/sql-tools'; @Table('assets_audit') export class AssetAuditTable { @PrimaryGeneratedUuidV7Column() id!: string; - @ColumnIndex('IDX_assets_audit_asset_id') - @Column({ type: 'uuid' }) + @Column({ type: 'uuid', indexName: 'IDX_assets_audit_asset_id' }) assetId!: string; - @ColumnIndex('IDX_assets_audit_owner_id') - @Column({ type: 'uuid' }) + @Column({ type: 'uuid', indexName: 'IDX_assets_audit_owner_id' }) ownerId!: string; - @ColumnIndex('IDX_assets_audit_deleted_at') - @CreateDateColumn({ default: () => 'clock_timestamp()' }) + @CreateDateColumn({ default: () => 'clock_timestamp()', indexName: 'IDX_assets_audit_deleted_at' }) deletedAt!: Date; } diff --git a/server/src/schema/tables/asset-face.table.ts b/server/src/schema/tables/asset-face.table.ts index 0ae99f44bf..52f4364a93 100644 --- a/server/src/schema/tables/asset-face.table.ts +++ b/server/src/schema/tables/asset-face.table.ts @@ -8,10 +8,21 @@ import { Column, DeleteDateColumn, ForeignKeyColumn, Index, PrimaryGeneratedColu @Index({ name: 'IDX_asset_faces_assetId_personId', columns: ['assetId', 'personId'] }) @Index({ columns: ['personId', 'assetId'] }) export class AssetFaceTable { - @ForeignKeyColumn(() => AssetTable, { onDelete: 'CASCADE', onUpdate: 'CASCADE' }) + @ForeignKeyColumn(() => AssetTable, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE', + // [assetId, personId] is the PK constraint + index: false, + }) assetId!: string; - @ForeignKeyColumn(() => PersonTable, { onDelete: 'SET NULL', onUpdate: 'CASCADE', nullable: true }) + @ForeignKeyColumn(() => PersonTable, { + onDelete: 'SET NULL', + onUpdate: 'CASCADE', + nullable: true, + // [personId, assetId] makes this redundant + index: false, + }) personId!: string | null; @Column({ default: 0, type: 'integer' }) diff --git a/server/src/schema/tables/asset-files.table.ts b/server/src/schema/tables/asset-files.table.ts index fb8750a8ef..0859bd5cf0 100644 --- a/server/src/schema/tables/asset-files.table.ts +++ b/server/src/schema/tables/asset-files.table.ts @@ -3,7 +3,6 @@ import { AssetFileType } from 'src/enum'; import { AssetTable } from 'src/schema/tables/asset.table'; import { Column, - ColumnIndex, CreateDateColumn, ForeignKeyColumn, PrimaryGeneratedColumn, @@ -19,8 +18,11 @@ export class AssetFileTable { @PrimaryGeneratedColumn() id!: string; - @ColumnIndex('IDX_asset_files_assetId') - @ForeignKeyColumn(() => AssetTable, { onDelete: 'CASCADE', onUpdate: 'CASCADE' }) + @ForeignKeyColumn(() => AssetTable, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE', + indexName: 'IDX_asset_files_assetId', + }) assetId?: string; @CreateDateColumn() @@ -35,7 +37,6 @@ export class AssetFileTable { @Column() path!: string; - @ColumnIndex('IDX_asset_files_update_id') - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_asset_files_update_id' }) updateId?: string; } diff --git a/server/src/schema/tables/asset.table.ts b/server/src/schema/tables/asset.table.ts index 250c3546a2..19ec8d2ef4 100644 --- a/server/src/schema/tables/asset.table.ts +++ b/server/src/schema/tables/asset.table.ts @@ -1,5 +1,4 @@ import { UpdatedAtTrigger, UpdateIdColumn } from 'src/decorators'; -import { ASSET_CHECKSUM_CONSTRAINT } from 'src/entities/asset.entity'; import { AssetStatus, AssetType } from 'src/enum'; import { assets_status_enum } from 'src/schema/enums'; import { assets_delete_audit } from 'src/schema/functions'; @@ -9,7 +8,6 @@ import { UserTable } from 'src/schema/tables/user.table'; import { AfterDeleteTrigger, Column, - ColumnIndex, CreateDateColumn, DeleteDateColumn, ForeignKeyColumn, @@ -18,6 +16,7 @@ import { Table, UpdateDateColumn, } from 'src/sql-tools'; +import { ASSET_CHECKSUM_CONSTRAINT } from 'src/utils/database'; @Table('assets') @UpdatedAtTrigger('assets_updated_at') @@ -78,8 +77,7 @@ export class AssetTable { @Column() originalPath!: string; - @ColumnIndex('idx_asset_file_created_at') - @Column({ type: 'timestamp with time zone' }) + @Column({ type: 'timestamp with time zone', indexName: 'idx_asset_file_created_at' }) fileCreatedAt!: Date; @Column({ type: 'timestamp with time zone' }) @@ -94,8 +92,7 @@ export class AssetTable { @Column({ type: 'character varying', nullable: true, default: '' }) encodedVideoPath!: string | null; - @Column({ type: 'bytea' }) - @ColumnIndex() + @Column({ type: 'bytea', index: true }) checksum!: Buffer; // sha1 checksum @Column({ type: 'boolean', default: true }) @@ -113,8 +110,7 @@ export class AssetTable { @Column({ type: 'boolean', default: false }) isArchived!: boolean; - @Column() - @ColumnIndex() + @Column({ index: true }) originalFileName!: string; @Column({ nullable: true }) @@ -141,14 +137,12 @@ export class AssetTable { @ForeignKeyColumn(() => StackTable, { nullable: true, onDelete: 'SET NULL', onUpdate: 'CASCADE' }) stackId?: string | null; - @ColumnIndex('IDX_assets_duplicateId') - @Column({ type: 'uuid', nullable: true }) + @Column({ type: 'uuid', nullable: true, indexName: 'IDX_assets_duplicateId' }) duplicateId!: string | null; @Column({ enum: assets_status_enum, default: AssetStatus.ACTIVE }) status!: AssetStatus; - @ColumnIndex('IDX_assets_update_id') - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_assets_update_id' }) updateId?: string; } diff --git a/server/src/schema/tables/exif.table.ts b/server/src/schema/tables/exif.table.ts index e40ce94b4f..ca300945c3 100644 --- a/server/src/schema/tables/exif.table.ts +++ b/server/src/schema/tables/exif.table.ts @@ -1,6 +1,6 @@ import { UpdatedAtTrigger, UpdateIdColumn } from 'src/decorators'; import { AssetTable } from 'src/schema/tables/asset.table'; -import { Column, ColumnIndex, ForeignKeyColumn, Table, UpdateDateColumn } from 'src/sql-tools'; +import { Column, ForeignKeyColumn, Table, UpdateDateColumn } from 'src/sql-tools'; @Table('exif') @UpdatedAtTrigger('asset_exif_updated_at') @@ -50,8 +50,7 @@ export class ExifTable { @Column({ type: 'double precision', nullable: true }) longitude!: number | null; - @ColumnIndex('exif_city') - @Column({ type: 'character varying', nullable: true }) + @Column({ type: 'character varying', nullable: true, indexName: 'exif_city' }) city!: string | null; @Column({ type: 'character varying', nullable: true }) @@ -69,8 +68,7 @@ export class ExifTable { @Column({ type: 'character varying', nullable: true }) exposureTime!: string | null; - @ColumnIndex('IDX_live_photo_cid') - @Column({ type: 'character varying', nullable: true }) + @Column({ type: 'character varying', nullable: true, indexName: 'IDX_live_photo_cid' }) livePhotoCID!: string | null; @Column({ type: 'character varying', nullable: true }) @@ -88,8 +86,7 @@ export class ExifTable { @Column({ type: 'integer', nullable: true }) bitsPerSample!: number | null; - @ColumnIndex('IDX_auto_stack_id') - @Column({ type: 'character varying', nullable: true }) + @Column({ type: 'character varying', nullable: true, indexName: 'IDX_auto_stack_id' }) autoStackId!: string | null; @Column({ type: 'integer', nullable: true }) @@ -98,7 +95,6 @@ export class ExifTable { @UpdateDateColumn({ default: () => 'clock_timestamp()' }) updatedAt?: Date; - @ColumnIndex('IDX_asset_exif_update_id') - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_asset_exif_update_id' }) updateId?: string; } diff --git a/server/src/schema/tables/geodata-places.table.ts b/server/src/schema/tables/geodata-places.table.ts index 631cfdff08..3e78b4cfcf 100644 --- a/server/src/schema/tables/geodata-places.table.ts +++ b/server/src/schema/tables/geodata-places.table.ts @@ -1,6 +1,6 @@ import { Column, Index, PrimaryColumn, Table } from 'src/sql-tools'; -@Table({ name: 'geodata_places' }) +@Table({ name: 'geodata_places', synchronize: false }) @Index({ name: 'idx_geodata_places_alternate_names', using: 'gin', @@ -26,11 +26,10 @@ import { Column, Index, PrimaryColumn, Table } from 'src/sql-tools'; synchronize: false, }) @Index({ - name: 'idx_geodata_places_gist_earthcoord', + name: 'IDX_geodata_gist_earthcoord', expression: 'll_to_earth_public(latitude, longitude)', synchronize: false, }) -@Table({ name: 'idx_geodata_places', synchronize: false }) export class GeodataPlacesTable { @PrimaryColumn({ type: 'integer' }) id!: number; diff --git a/server/src/schema/tables/library.table.ts b/server/src/schema/tables/library.table.ts index 54b3752f41..8b21d5feb0 100644 --- a/server/src/schema/tables/library.table.ts +++ b/server/src/schema/tables/library.table.ts @@ -2,7 +2,6 @@ import { UpdatedAtTrigger, UpdateIdColumn } from 'src/decorators'; import { UserTable } from 'src/schema/tables/user.table'; import { Column, - ColumnIndex, CreateDateColumn, DeleteDateColumn, ForeignKeyColumn, @@ -41,7 +40,6 @@ export class LibraryTable { @Column({ type: 'timestamp with time zone', nullable: true }) refreshedAt!: Date | null; - @ColumnIndex('IDX_libraries_update_id') - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_libraries_update_id' }) updateId?: string; } diff --git a/server/src/schema/tables/memory.table.ts b/server/src/schema/tables/memory.table.ts index 1926405565..32dafe3384 100644 --- a/server/src/schema/tables/memory.table.ts +++ b/server/src/schema/tables/memory.table.ts @@ -3,7 +3,6 @@ import { MemoryType } from 'src/enum'; import { UserTable } from 'src/schema/tables/user.table'; import { Column, - ColumnIndex, CreateDateColumn, DeleteDateColumn, ForeignKeyColumn, @@ -55,7 +54,6 @@ export class MemoryTable { @Column({ type: 'timestamp with time zone', nullable: true }) hideAt?: Date; - @ColumnIndex('IDX_memories_update_id') - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_memories_update_id' }) updateId?: string; } diff --git a/server/src/schema/tables/memory_asset.table.ts b/server/src/schema/tables/memory_asset.table.ts index 864e6291c7..0e5ca29a08 100644 --- a/server/src/schema/tables/memory_asset.table.ts +++ b/server/src/schema/tables/memory_asset.table.ts @@ -1,14 +1,12 @@ import { AssetTable } from 'src/schema/tables/asset.table'; import { MemoryTable } from 'src/schema/tables/memory.table'; -import { ColumnIndex, ForeignKeyColumn, Table } from 'src/sql-tools'; +import { ForeignKeyColumn, Table } from 'src/sql-tools'; @Table('memories_assets_assets') export class MemoryAssetTable { - @ColumnIndex() @ForeignKeyColumn(() => MemoryTable, { onUpdate: 'CASCADE', onDelete: 'CASCADE', primary: true }) memoriesId!: string; - @ColumnIndex() @ForeignKeyColumn(() => AssetTable, { onUpdate: 'CASCADE', onDelete: 'CASCADE', primary: true }) assetsId!: string; } diff --git a/server/src/schema/tables/partner-audit.table.ts b/server/src/schema/tables/partner-audit.table.ts index 08b6e94626..da5243dc75 100644 --- a/server/src/schema/tables/partner-audit.table.ts +++ b/server/src/schema/tables/partner-audit.table.ts @@ -1,20 +1,17 @@ import { PrimaryGeneratedUuidV7Column } from 'src/decorators'; -import { Column, ColumnIndex, CreateDateColumn, Table } from 'src/sql-tools'; +import { Column, CreateDateColumn, Table } from 'src/sql-tools'; @Table('partners_audit') export class PartnerAuditTable { @PrimaryGeneratedUuidV7Column() id!: string; - @ColumnIndex('IDX_partners_audit_shared_by_id') - @Column({ type: 'uuid' }) + @Column({ type: 'uuid', indexName: 'IDX_partners_audit_shared_by_id' }) sharedById!: string; - @ColumnIndex('IDX_partners_audit_shared_with_id') - @Column({ type: 'uuid' }) + @Column({ type: 'uuid', indexName: 'IDX_partners_audit_shared_with_id' }) sharedWithId!: string; - @ColumnIndex('IDX_partners_audit_deleted_at') - @CreateDateColumn({ default: () => 'clock_timestamp()' }) + @CreateDateColumn({ default: () => 'clock_timestamp()', indexName: 'IDX_partners_audit_deleted_at' }) deletedAt!: Date; } diff --git a/server/src/schema/tables/partner.table.ts b/server/src/schema/tables/partner.table.ts index 770107fe7a..0da60cfc0c 100644 --- a/server/src/schema/tables/partner.table.ts +++ b/server/src/schema/tables/partner.table.ts @@ -1,15 +1,7 @@ import { UpdatedAtTrigger, UpdateIdColumn } from 'src/decorators'; import { partners_delete_audit } from 'src/schema/functions'; import { UserTable } from 'src/schema/tables/user.table'; -import { - AfterDeleteTrigger, - Column, - ColumnIndex, - CreateDateColumn, - ForeignKeyColumn, - Table, - UpdateDateColumn, -} from 'src/sql-tools'; +import { AfterDeleteTrigger, Column, CreateDateColumn, ForeignKeyColumn, Table, UpdateDateColumn } from 'src/sql-tools'; @Table('partners') @UpdatedAtTrigger('partners_updated_at') @@ -21,7 +13,12 @@ import { when: 'pg_trigger_depth() = 0', }) export class PartnerTable { - @ForeignKeyColumn(() => UserTable, { onDelete: 'CASCADE', primary: true }) + @ForeignKeyColumn(() => UserTable, { + onDelete: 'CASCADE', + primary: true, + // [sharedById, sharedWithId] is the PK constraint + index: false, + }) sharedById!: string; @ForeignKeyColumn(() => UserTable, { onDelete: 'CASCADE', primary: true }) @@ -36,7 +33,6 @@ export class PartnerTable { @Column({ type: 'boolean', default: false }) inTimeline!: boolean; - @ColumnIndex('IDX_partners_update_id') - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_partners_update_id' }) updateId!: string; } diff --git a/server/src/schema/tables/person.table.ts b/server/src/schema/tables/person.table.ts index b96fc5b709..1320b91f18 100644 --- a/server/src/schema/tables/person.table.ts +++ b/server/src/schema/tables/person.table.ts @@ -4,7 +4,6 @@ import { UserTable } from 'src/schema/tables/user.table'; import { Check, Column, - ColumnIndex, CreateDateColumn, ForeignKeyColumn, PrimaryGeneratedColumn, @@ -49,7 +48,6 @@ export class PersonTable { @Column({ type: 'character varying', nullable: true, default: null }) color?: string | null; - @ColumnIndex('IDX_person_update_id') - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_person_update_id' }) updateId!: string; } diff --git a/server/src/schema/tables/session.table.ts b/server/src/schema/tables/session.table.ts index a66732a7d9..ad43d0d6e4 100644 --- a/server/src/schema/tables/session.table.ts +++ b/server/src/schema/tables/session.table.ts @@ -2,7 +2,6 @@ import { UpdatedAtTrigger, UpdateIdColumn } from 'src/decorators'; import { UserTable } from 'src/schema/tables/user.table'; import { Column, - ColumnIndex, CreateDateColumn, ForeignKeyColumn, PrimaryGeneratedColumn, @@ -35,7 +34,6 @@ export class SessionTable { @Column({ default: '' }) deviceOS!: string; - @ColumnIndex('IDX_sessions_update_id') - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_sessions_update_id' }) updateId!: string; } diff --git a/server/src/schema/tables/shared-link-asset.table.ts b/server/src/schema/tables/shared-link-asset.table.ts index 1eb294c1e8..66c9068441 100644 --- a/server/src/schema/tables/shared-link-asset.table.ts +++ b/server/src/schema/tables/shared-link-asset.table.ts @@ -1,14 +1,12 @@ import { AssetTable } from 'src/schema/tables/asset.table'; import { SharedLinkTable } from 'src/schema/tables/shared-link.table'; -import { ColumnIndex, ForeignKeyColumn, Table } from 'src/sql-tools'; +import { ForeignKeyColumn, Table } from 'src/sql-tools'; @Table('shared_link__asset') export class SharedLinkAssetTable { - @ColumnIndex() @ForeignKeyColumn(() => AssetTable, { onUpdate: 'CASCADE', onDelete: 'CASCADE', primary: true }) assetsId!: string; - @ColumnIndex() @ForeignKeyColumn(() => SharedLinkTable, { onUpdate: 'CASCADE', onDelete: 'CASCADE', primary: true }) sharedLinksId!: string; } diff --git a/server/src/schema/tables/shared-link.table.ts b/server/src/schema/tables/shared-link.table.ts index 36237c58ef..3bb36b36ed 100644 --- a/server/src/schema/tables/shared-link.table.ts +++ b/server/src/schema/tables/shared-link.table.ts @@ -1,15 +1,7 @@ import { SharedLinkType } from 'src/enum'; import { AlbumTable } from 'src/schema/tables/album.table'; import { UserTable } from 'src/schema/tables/user.table'; -import { - Column, - ColumnIndex, - CreateDateColumn, - ForeignKeyColumn, - PrimaryGeneratedColumn, - Table, - Unique, -} from 'src/sql-tools'; +import { Column, CreateDateColumn, ForeignKeyColumn, PrimaryGeneratedColumn, Table, Unique } from 'src/sql-tools'; @Table('shared_links') @Unique({ name: 'UQ_sharedlink_key', columns: ['key'] }) @@ -23,8 +15,7 @@ export class SharedLinkTable { @ForeignKeyColumn(() => UserTable, { onDelete: 'CASCADE', onUpdate: 'CASCADE' }) userId!: string; - @ColumnIndex('IDX_sharedlink_key') - @Column({ type: 'bytea' }) + @Column({ type: 'bytea', indexName: 'IDX_sharedlink_key' }) key!: Buffer; // use to access the inidividual asset @Column() @@ -39,8 +30,12 @@ export class SharedLinkTable { @Column({ type: 'boolean', default: false }) allowUpload!: boolean; - @ColumnIndex('IDX_sharedlink_albumId') - @ForeignKeyColumn(() => AlbumTable, { nullable: true, onDelete: 'CASCADE', onUpdate: 'CASCADE' }) + @ForeignKeyColumn(() => AlbumTable, { + nullable: true, + onDelete: 'CASCADE', + onUpdate: 'CASCADE', + indexName: 'IDX_sharedlink_albumId', + }) albumId!: string; @Column({ type: 'boolean', default: true }) diff --git a/server/src/schema/tables/sync-checkpoint.table.ts b/server/src/schema/tables/sync-checkpoint.table.ts index 831205ce7a..21fd7983ac 100644 --- a/server/src/schema/tables/sync-checkpoint.table.ts +++ b/server/src/schema/tables/sync-checkpoint.table.ts @@ -1,15 +1,7 @@ import { UpdatedAtTrigger, UpdateIdColumn } from 'src/decorators'; import { SyncEntityType } from 'src/enum'; import { SessionTable } from 'src/schema/tables/session.table'; -import { - Column, - ColumnIndex, - CreateDateColumn, - ForeignKeyColumn, - PrimaryColumn, - Table, - UpdateDateColumn, -} from 'src/sql-tools'; +import { Column, CreateDateColumn, ForeignKeyColumn, PrimaryColumn, Table, UpdateDateColumn } from 'src/sql-tools'; @Table('session_sync_checkpoints') @UpdatedAtTrigger('session_sync_checkpoints_updated_at') @@ -29,7 +21,6 @@ export class SessionSyncCheckpointTable { @Column() ack!: string; - @ColumnIndex('IDX_session_sync_checkpoints_update_id') - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_session_sync_checkpoints_update_id' }) updateId!: string; } diff --git a/server/src/schema/tables/tag-asset.table.ts b/server/src/schema/tables/tag-asset.table.ts index 5f24799cec..8793af0a8a 100644 --- a/server/src/schema/tables/tag-asset.table.ts +++ b/server/src/schema/tables/tag-asset.table.ts @@ -1,15 +1,13 @@ import { AssetTable } from 'src/schema/tables/asset.table'; import { TagTable } from 'src/schema/tables/tag.table'; -import { ColumnIndex, ForeignKeyColumn, Index, Table } from 'src/sql-tools'; +import { ForeignKeyColumn, Index, Table } from 'src/sql-tools'; @Index({ name: 'IDX_tag_asset_assetsId_tagsId', columns: ['assetsId', 'tagsId'] }) @Table('tag_asset') export class TagAssetTable { - @ColumnIndex() - @ForeignKeyColumn(() => AssetTable, { onUpdate: 'CASCADE', onDelete: 'CASCADE', primary: true }) + @ForeignKeyColumn(() => AssetTable, { onUpdate: 'CASCADE', onDelete: 'CASCADE', primary: true, index: true }) assetsId!: string; - @ColumnIndex() - @ForeignKeyColumn(() => TagTable, { onUpdate: 'CASCADE', onDelete: 'CASCADE', primary: true }) + @ForeignKeyColumn(() => TagTable, { onUpdate: 'CASCADE', onDelete: 'CASCADE', primary: true, index: true }) tagsId!: string; } diff --git a/server/src/schema/tables/tag-closure.table.ts b/server/src/schema/tables/tag-closure.table.ts index acde84b91d..8829e802e1 100644 --- a/server/src/schema/tables/tag-closure.table.ts +++ b/server/src/schema/tables/tag-closure.table.ts @@ -1,13 +1,11 @@ import { TagTable } from 'src/schema/tables/tag.table'; -import { ColumnIndex, ForeignKeyColumn, Table } from 'src/sql-tools'; +import { ForeignKeyColumn, Table } from 'src/sql-tools'; @Table('tags_closure') export class TagClosureTable { - @ColumnIndex() - @ForeignKeyColumn(() => TagTable, { primary: true, onDelete: 'CASCADE', onUpdate: 'NO ACTION' }) + @ForeignKeyColumn(() => TagTable, { primary: true, onDelete: 'CASCADE', onUpdate: 'NO ACTION', index: true }) id_ancestor!: string; - @ColumnIndex() - @ForeignKeyColumn(() => TagTable, { primary: true, onDelete: 'CASCADE', onUpdate: 'NO ACTION' }) + @ForeignKeyColumn(() => TagTable, { primary: true, onDelete: 'CASCADE', onUpdate: 'NO ACTION', index: true }) id_descendant!: string; } diff --git a/server/src/schema/tables/tag.table.ts b/server/src/schema/tables/tag.table.ts index 5042e2eb0e..a9f2a57f27 100644 --- a/server/src/schema/tables/tag.table.ts +++ b/server/src/schema/tables/tag.table.ts @@ -2,7 +2,6 @@ import { UpdatedAtTrigger, UpdateIdColumn } from 'src/decorators'; import { UserTable } from 'src/schema/tables/user.table'; import { Column, - ColumnIndex, CreateDateColumn, ForeignKeyColumn, PrimaryGeneratedColumn, @@ -18,7 +17,12 @@ export class TagTable { @PrimaryGeneratedColumn() id!: string; - @ForeignKeyColumn(() => UserTable, { onUpdate: 'CASCADE', onDelete: 'CASCADE' }) + @ForeignKeyColumn(() => UserTable, { + onUpdate: 'CASCADE', + onDelete: 'CASCADE', + // [userId, value] makes this redundant + index: false, + }) userId!: string; @Column() @@ -36,7 +40,6 @@ export class TagTable { @ForeignKeyColumn(() => TagTable, { nullable: true, onDelete: 'CASCADE' }) parentId?: string; - @ColumnIndex('IDX_tags_update_id') - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_tags_update_id' }) updateId!: string; } diff --git a/server/src/schema/tables/user-audit.table.ts b/server/src/schema/tables/user-audit.table.ts index 0f881ccc9a..e0c9afcdc3 100644 --- a/server/src/schema/tables/user-audit.table.ts +++ b/server/src/schema/tables/user-audit.table.ts @@ -1,13 +1,12 @@ import { PrimaryGeneratedUuidV7Column } from 'src/decorators'; -import { Column, ColumnIndex, CreateDateColumn, Table } from 'src/sql-tools'; +import { Column, CreateDateColumn, Table } from 'src/sql-tools'; @Table('users_audit') export class UserAuditTable { @Column({ type: 'uuid' }) userId!: string; - @ColumnIndex('IDX_users_audit_deleted_at') - @CreateDateColumn({ default: () => 'clock_timestamp()' }) + @CreateDateColumn({ default: () => 'clock_timestamp()', indexName: 'IDX_users_audit_deleted_at' }) deletedAt!: Date; @PrimaryGeneratedUuidV7Column() diff --git a/server/src/schema/tables/user-metadata.table.ts b/server/src/schema/tables/user-metadata.table.ts index 6d03acaf80..04b457867f 100644 --- a/server/src/schema/tables/user-metadata.table.ts +++ b/server/src/schema/tables/user-metadata.table.ts @@ -5,7 +5,13 @@ import { UserMetadata, UserMetadataItem } from 'src/types'; @Table('user_metadata') export class UserMetadataTable implements UserMetadataItem { - @ForeignKeyColumn(() => UserTable, { onUpdate: 'CASCADE', onDelete: 'CASCADE', primary: true }) + @ForeignKeyColumn(() => UserTable, { + onUpdate: 'CASCADE', + onDelete: 'CASCADE', + primary: true, + // [userId, key] is the PK constraint + index: false, + }) userId!: string; @PrimaryColumn({ type: 'character varying' }) diff --git a/server/src/schema/tables/user.table.ts b/server/src/schema/tables/user.table.ts index 5160f979b9..eeef923796 100644 --- a/server/src/schema/tables/user.table.ts +++ b/server/src/schema/tables/user.table.ts @@ -5,7 +5,6 @@ import { users_delete_audit } from 'src/schema/functions'; import { AfterDeleteTrigger, Column, - ColumnIndex, CreateDateColumn, DeleteDateColumn, Index, @@ -77,7 +76,6 @@ export class UserTable { @Column({ type: 'timestamp with time zone', default: () => 'now()' }) profileChangedAt!: Generated; - @ColumnIndex({ name: 'IDX_users_update_id' }) - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_users_update_id' }) updateId!: Generated; } diff --git a/server/src/services/album.service.ts b/server/src/services/album.service.ts index eac000005b..1c612de8c0 100644 --- a/server/src/services/album.service.ts +++ b/server/src/services/album.service.ts @@ -6,15 +6,15 @@ import { AlbumStatisticsResponseDto, CreateAlbumDto, GetAlbumsDto, - UpdateAlbumDto, - UpdateAlbumUserDto, mapAlbum, + MapAlbumDto, mapAlbumWithAssets, mapAlbumWithoutAssets, + UpdateAlbumDto, + UpdateAlbumUserDto, } from 'src/dtos/album.dto'; import { BulkIdResponseDto, BulkIdsDto } from 'src/dtos/asset-ids.response.dto'; import { AuthDto } from 'src/dtos/auth.dto'; -import { AlbumEntity } from 'src/entities/album.entity'; import { Permission } from 'src/enum'; import { AlbumAssetCount, AlbumInfoOptions } from 'src/repositories/album.repository'; import { BaseService } from 'src/services/base.service'; @@ -39,7 +39,7 @@ export class AlbumService extends BaseService { async getAll({ user: { id: ownerId } }: AuthDto, { assetId, shared }: GetAlbumsDto): Promise { await this.albumRepository.updateThumbnails(); - let albums: AlbumEntity[]; + let albums: MapAlbumDto[]; if (assetId) { albums = await this.albumRepository.getByAssetId(ownerId, assetId); } else if (shared === true) { diff --git a/server/src/services/asset-media.service.spec.ts b/server/src/services/asset-media.service.spec.ts index 0c1bbc3cee..d25067f1c9 100644 --- a/server/src/services/asset-media.service.spec.ts +++ b/server/src/services/asset-media.service.spec.ts @@ -8,10 +8,11 @@ import { Stats } from 'node:fs'; import { AssetFile } from 'src/database'; import { AssetMediaStatus, AssetRejectReason, AssetUploadAction } from 'src/dtos/asset-media-response.dto'; import { AssetMediaCreateDto, AssetMediaReplaceDto, AssetMediaSize, UploadFieldName } from 'src/dtos/asset-media.dto'; -import { ASSET_CHECKSUM_CONSTRAINT, AssetEntity } from 'src/entities/asset.entity'; +import { MapAsset } from 'src/dtos/asset-response.dto'; import { AssetFileType, AssetStatus, AssetType, CacheControl, JobName } from 'src/enum'; import { AuthRequest } from 'src/middleware/auth.guard'; import { AssetMediaService } from 'src/services/asset-media.service'; +import { ASSET_CHECKSUM_CONSTRAINT } from 'src/utils/database'; import { ImmichFileResponse } from 'src/utils/file'; import { assetStub } from 'test/fixtures/asset.stub'; import { authStub } from 'test/fixtures/auth.stub'; @@ -173,7 +174,7 @@ const assetEntity = Object.freeze({ }, livePhotoVideoId: null, sidecarPath: null, -}) as AssetEntity; +} as MapAsset); const existingAsset = Object.freeze({ ...assetEntity, @@ -182,18 +183,18 @@ const existingAsset = Object.freeze({ checksum: Buffer.from('_getExistingAsset', 'utf8'), libraryId: 'libraryId', originalFileName: 'existing-filename.jpeg', -}) as AssetEntity; +}) as MapAsset; const sidecarAsset = Object.freeze({ ...existingAsset, sidecarPath: 'sidecar-path', checksum: Buffer.from('_getExistingAssetWithSideCar', 'utf8'), -}) as AssetEntity; +}) as MapAsset; const copiedAsset = Object.freeze({ id: 'copied-asset', originalPath: 'copied-path', -}) as AssetEntity; +}) as MapAsset; describe(AssetMediaService.name, () => { let sut: AssetMediaService; @@ -819,8 +820,8 @@ describe(AssetMediaService.name, () => { const file2 = Buffer.from('53be335e99f18a66ff12e9a901c7a6171dd76573', 'hex'); mocks.asset.getByChecksums.mockResolvedValue([ - { id: 'asset-1', checksum: file1 } as AssetEntity, - { id: 'asset-2', checksum: file2 } as AssetEntity, + { id: 'asset-1', checksum: file1, deletedAt: null }, + { id: 'asset-2', checksum: file2, deletedAt: null }, ]); await expect( @@ -856,7 +857,7 @@ describe(AssetMediaService.name, () => { const file1 = Buffer.from('d2947b871a706081be194569951b7db246907957', 'hex'); const file2 = Buffer.from('53be335e99f18a66ff12e9a901c7a6171dd76573', 'hex'); - mocks.asset.getByChecksums.mockResolvedValue([{ id: 'asset-1', checksum: file1 } as AssetEntity]); + mocks.asset.getByChecksums.mockResolvedValue([{ id: 'asset-1', checksum: file1, deletedAt: null }]); await expect( sut.bulkUploadCheck(authStub.admin, { diff --git a/server/src/services/asset-media.service.ts b/server/src/services/asset-media.service.ts index 2929950f4d..78e23fa802 100644 --- a/server/src/services/asset-media.service.ts +++ b/server/src/services/asset-media.service.ts @@ -2,6 +2,7 @@ import { BadRequestException, Injectable, InternalServerErrorException, NotFound import { extname } from 'node:path'; import sanitize from 'sanitize-filename'; import { StorageCore } from 'src/cores/storage.core'; +import { Asset } from 'src/database'; import { AssetBulkUploadCheckResponseDto, AssetMediaResponseDto, @@ -20,13 +21,13 @@ import { UploadFieldName, } from 'src/dtos/asset-media.dto'; import { AuthDto } from 'src/dtos/auth.dto'; -import { ASSET_CHECKSUM_CONSTRAINT, AssetEntity } from 'src/entities/asset.entity'; import { AssetStatus, AssetType, CacheControl, JobName, Permission, StorageFolder } from 'src/enum'; import { AuthRequest } from 'src/middleware/auth.guard'; import { BaseService } from 'src/services/base.service'; import { UploadFile } from 'src/types'; import { requireUploadAccess } from 'src/utils/access'; import { asRequest, getAssetFiles, onBeforeLink } from 'src/utils/asset.util'; +import { ASSET_CHECKSUM_CONSTRAINT } from 'src/utils/database'; import { getFilenameExtension, getFileNameWithoutExtension, ImmichFileResponse } from 'src/utils/file'; import { mimeTypes } from 'src/utils/mime-types'; import { fromChecksum } from 'src/utils/request'; @@ -212,7 +213,7 @@ export class AssetMediaService extends BaseService { const asset = await this.findOrFail(id); const size = dto.size ?? AssetMediaSize.THUMBNAIL; - const { thumbnailFile, previewFile, fullsizeFile } = getAssetFiles(asset.files); + const { thumbnailFile, previewFile, fullsizeFile } = getAssetFiles(asset.files ?? []); let filepath = previewFile?.path; if (size === AssetMediaSize.THUMBNAIL && thumbnailFile) { filepath = thumbnailFile.path; @@ -375,7 +376,7 @@ export class AssetMediaService extends BaseService { * Uses only vital properties excluding things like: stacks, faces, smart search info, etc, * and then queues a METADATA_EXTRACTION job. */ - private async createCopy(asset: AssetEntity): Promise { + private async createCopy(asset: Omit) { const created = await this.assetRepository.create({ ownerId: asset.ownerId, originalPath: asset.originalPath, @@ -398,12 +399,7 @@ export class AssetMediaService extends BaseService { return created; } - private async create( - ownerId: string, - dto: AssetMediaCreateDto, - file: UploadFile, - sidecarFile?: UploadFile, - ): Promise { + private async create(ownerId: string, dto: AssetMediaCreateDto, file: UploadFile, sidecarFile?: UploadFile) { const asset = await this.assetRepository.create({ ownerId, libraryId: null, @@ -444,7 +440,7 @@ export class AssetMediaService extends BaseService { } } - private async findOrFail(id: string): Promise { + private async findOrFail(id: string) { const asset = await this.assetRepository.getById(id, { files: true }); if (!asset) { throw new NotFoundException('Asset not found'); diff --git a/server/src/services/asset.service.spec.ts b/server/src/services/asset.service.spec.ts index 5fc4984b62..a3f536ac33 100755 --- a/server/src/services/asset.service.spec.ts +++ b/server/src/services/asset.service.spec.ts @@ -1,8 +1,7 @@ import { BadRequestException } from '@nestjs/common'; import { DateTime } from 'luxon'; -import { mapAsset } from 'src/dtos/asset-response.dto'; +import { MapAsset, mapAsset } from 'src/dtos/asset-response.dto'; import { AssetJobName, AssetStatsResponseDto } from 'src/dtos/asset.dto'; -import { AssetEntity } from 'src/entities/asset.entity'; import { AssetStatus, AssetType, JobName, JobStatus } from 'src/enum'; import { AssetStats } from 'src/repositories/asset.repository'; import { AssetService } from 'src/services/asset.service'; @@ -35,7 +34,7 @@ describe(AssetService.name, () => { expect(sut).toBeDefined(); }); - const mockGetById = (assets: AssetEntity[]) => { + const mockGetById = (assets: MapAsset[]) => { mocks.asset.getById.mockImplementation((assetId) => Promise.resolve(assets.find((asset) => asset.id === assetId))); }; @@ -608,7 +607,7 @@ describe(AssetService.name, () => { mocks.asset.getById.mockResolvedValue({ ...assetStub.primaryImage, stack: { ...assetStub.primaryImage.stack, assets: assetStub.primaryImage.stack!.assets.slice(0, 2) }, - } as AssetEntity); + }); await sut.handleAssetDeletion({ id: assetStub.primaryImage.id, deleteOnDisk: true }); diff --git a/server/src/services/asset.service.ts b/server/src/services/asset.service.ts index 1ded79680b..16f60907ba 100644 --- a/server/src/services/asset.service.ts +++ b/server/src/services/asset.service.ts @@ -5,6 +5,7 @@ import { JOBS_ASSET_PAGINATION_SIZE } from 'src/constants'; import { OnJob } from 'src/decorators'; import { AssetResponseDto, + MapAsset, MemoryLaneResponseDto, SanitizedAssetResponseDto, mapAsset, @@ -20,7 +21,6 @@ import { } from 'src/dtos/asset.dto'; import { AuthDto } from 'src/dtos/auth.dto'; import { MemoryLaneDto } from 'src/dtos/search.dto'; -import { AssetEntity } from 'src/entities/asset.entity'; import { AssetStatus, JobName, JobStatus, Permission, QueueName } from 'src/enum'; import { BaseService } from 'src/services/base.service'; import { ISidecarWriteJob, JobItem, JobOf } from 'src/types'; @@ -43,7 +43,7 @@ export class AssetService extends BaseService { yearsAgo, // TODO move this to clients title: `${yearsAgo} year${yearsAgo > 1 ? 's' : ''} ago`, - assets: assets.map((asset) => mapAsset(asset as unknown as AssetEntity, { auth })), + assets: assets.map((asset) => mapAsset(asset, { auth })), }; }); } @@ -105,7 +105,7 @@ export class AssetService extends BaseService { const { description, dateTimeOriginal, latitude, longitude, rating, ...rest } = dto; const repos = { asset: this.assetRepository, event: this.eventRepository }; - let previousMotion: AssetEntity | null = null; + let previousMotion: MapAsset | null = null; if (rest.livePhotoVideoId) { await onBeforeLink(repos, { userId: auth.user.id, livePhotoVideoId: rest.livePhotoVideoId }); } else if (rest.livePhotoVideoId === null) { @@ -233,7 +233,7 @@ export class AssetService extends BaseService { } } - const { fullsizeFile, previewFile, thumbnailFile } = getAssetFiles(asset.files); + const { fullsizeFile, previewFile, thumbnailFile } = getAssetFiles(asset.files ?? []); const files = [thumbnailFile?.path, previewFile?.path, fullsizeFile?.path, asset.encodedVideoPath]; if (deleteOnDisk) { diff --git a/server/src/services/duplicate.service.ts b/server/src/services/duplicate.service.ts index 9f1ac3d4ce..c504b1a305 100644 --- a/server/src/services/duplicate.service.ts +++ b/server/src/services/duplicate.service.ts @@ -68,7 +68,7 @@ export class DuplicateService extends BaseService { return JobStatus.SKIPPED; } - const previewFile = getAssetFile(asset.files, AssetFileType.PREVIEW); + const previewFile = getAssetFile(asset.files || [], AssetFileType.PREVIEW); if (!previewFile) { this.logger.warn(`Asset ${id} is missing preview image`); return JobStatus.FAILED; diff --git a/server/src/services/job.service.spec.ts b/server/src/services/job.service.spec.ts index 134a86b69f..baac0af428 100644 --- a/server/src/services/job.service.spec.ts +++ b/server/src/services/job.service.spec.ts @@ -285,9 +285,9 @@ describe(JobService.name, () => { it(`should queue ${jobs.length} jobs when a ${item.name} job finishes successfully`, async () => { if (item.name === JobName.GENERATE_THUMBNAILS && item.data.source === 'upload') { if (item.data.id === 'asset-live-image') { - mocks.asset.getByIdsWithAllRelations.mockResolvedValue([assetStub.livePhotoStillAsset]); + mocks.asset.getByIdsWithAllRelationsButStacks.mockResolvedValue([assetStub.livePhotoStillAsset as any]); } else { - mocks.asset.getByIdsWithAllRelations.mockResolvedValue([assetStub.livePhotoMotionAsset]); + mocks.asset.getByIdsWithAllRelationsButStacks.mockResolvedValue([assetStub.livePhotoMotionAsset as any]); } } diff --git a/server/src/services/job.service.ts b/server/src/services/job.service.ts index 2f180edd40..edd018d7b1 100644 --- a/server/src/services/job.service.ts +++ b/server/src/services/job.service.ts @@ -254,7 +254,7 @@ export class JobService extends BaseService { case JobName.METADATA_EXTRACTION: { if (item.data.source === 'sidecar-write') { - const [asset] = await this.assetRepository.getByIdsWithAllRelations([item.data.id]); + const [asset] = await this.assetRepository.getByIdsWithAllRelationsButStacks([item.data.id]); if (asset) { this.eventRepository.clientSend('on_asset_update', asset.ownerId, mapAsset(asset)); } @@ -284,7 +284,7 @@ export class JobService extends BaseService { break; } - const [asset] = await this.assetRepository.getByIdsWithAllRelations([item.data.id]); + const [asset] = await this.assetRepository.getByIdsWithAllRelationsButStacks([item.data.id]); if (!asset) { this.logger.warn(`Could not find asset ${item.data.id} after generating thumbnails`); break; diff --git a/server/src/services/library.service.spec.ts b/server/src/services/library.service.spec.ts index aef02b7244..15b150f551 100644 --- a/server/src/services/library.service.spec.ts +++ b/server/src/services/library.service.spec.ts @@ -350,7 +350,7 @@ describe(LibraryService.name, () => { progressCounter: 0, }; - mocks.asset.getByIds.mockResolvedValue([assetStub.external]); + mocks.assetJob.getForSyncAssets.mockResolvedValue([assetStub.external]); mocks.storage.stat.mockRejectedValue(new Error('ENOENT, no such file or directory')); await expect(sut.handleSyncAssets(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); @@ -371,7 +371,7 @@ describe(LibraryService.name, () => { progressCounter: 0, }; - mocks.asset.getByIds.mockResolvedValue([assetStub.external]); + mocks.assetJob.getForSyncAssets.mockResolvedValue([assetStub.external]); mocks.storage.stat.mockRejectedValue(new Error('Could not read file')); await expect(sut.handleSyncAssets(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); @@ -392,7 +392,7 @@ describe(LibraryService.name, () => { progressCounter: 0, }; - mocks.asset.getByIds.mockResolvedValue([assetStub.trashedOffline]); + mocks.assetJob.getForSyncAssets.mockResolvedValue([assetStub.trashedOffline]); mocks.storage.stat.mockRejectedValue(new Error('Could not read file')); await expect(sut.handleSyncAssets(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); @@ -410,7 +410,7 @@ describe(LibraryService.name, () => { progressCounter: 0, }; - mocks.asset.getByIds.mockResolvedValue([assetStub.trashedOffline]); + mocks.assetJob.getForSyncAssets.mockResolvedValue([assetStub.trashedOffline]); mocks.storage.stat.mockResolvedValue({ mtime: assetStub.external.fileModifiedAt } as Stats); await expect(sut.handleSyncAssets(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); @@ -431,7 +431,7 @@ describe(LibraryService.name, () => { progressCounter: 0, }; - mocks.asset.getByIds.mockResolvedValue([assetStub.trashedOffline]); + mocks.assetJob.getForSyncAssets.mockResolvedValue([assetStub.trashedOffline]); mocks.storage.stat.mockResolvedValue({ mtime: assetStub.external.fileModifiedAt } as Stats); await expect(sut.handleSyncAssets(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); @@ -451,7 +451,7 @@ describe(LibraryService.name, () => { progressCounter: 0, }; - mocks.asset.getByIds.mockResolvedValue([assetStub.trashedOffline]); + mocks.assetJob.getForSyncAssets.mockResolvedValue([assetStub.trashedOffline]); mocks.storage.stat.mockResolvedValue({ mtime: assetStub.external.fileModifiedAt } as Stats); await expect(sut.handleSyncAssets(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); @@ -471,7 +471,7 @@ describe(LibraryService.name, () => { progressCounter: 0, }; - mocks.asset.getByIds.mockResolvedValue([assetStub.external]); + mocks.assetJob.getForSyncAssets.mockResolvedValue([assetStub.external]); mocks.storage.stat.mockResolvedValue({ mtime: assetStub.external.fileModifiedAt } as Stats); await expect(sut.handleSyncAssets(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); @@ -489,7 +489,7 @@ describe(LibraryService.name, () => { progressCounter: 0, }; - mocks.asset.getByIds.mockResolvedValue([assetStub.trashedOffline]); + mocks.assetJob.getForSyncAssets.mockResolvedValue([assetStub.trashedOffline]); mocks.storage.stat.mockResolvedValue({ mtime: assetStub.trashedOffline.fileModifiedAt } as Stats); await expect(sut.handleSyncAssets(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); @@ -518,7 +518,7 @@ describe(LibraryService.name, () => { const mtime = new Date(assetStub.external.fileModifiedAt.getDate() + 1); - mocks.asset.getByIds.mockResolvedValue([assetStub.external]); + mocks.assetJob.getForSyncAssets.mockResolvedValue([assetStub.external]); mocks.storage.stat.mockResolvedValue({ mtime } as Stats); await expect(sut.handleSyncAssets(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); diff --git a/server/src/services/library.service.ts b/server/src/services/library.service.ts index 8cc2cf48ff..2add5f484b 100644 --- a/server/src/services/library.service.ts +++ b/server/src/services/library.service.ts @@ -18,7 +18,6 @@ import { ValidateLibraryImportPathResponseDto, ValidateLibraryResponseDto, } from 'src/dtos/library.dto'; -import { AssetEntity } from 'src/entities/asset.entity'; import { AssetStatus, AssetType, DatabaseLock, ImmichWorker, JobName, JobStatus, QueueName } from 'src/enum'; import { ArgOf } from 'src/repositories/event.repository'; import { AssetSyncResult } from 'src/repositories/library.repository'; @@ -467,7 +466,7 @@ export class LibraryService extends BaseService { @OnJob({ name: JobName.LIBRARY_SYNC_ASSETS, queue: QueueName.LIBRARY }) async handleSyncAssets(job: JobOf): Promise { - const assets = await this.assetRepository.getByIds(job.assetIds); + const assets = await this.assetJobRepository.getForSyncAssets(job.assetIds); const assetIdsToOffline: string[] = []; const trashedAssetIdsToOffline: string[] = []; @@ -561,7 +560,16 @@ export class LibraryService extends BaseService { return JobStatus.SUCCESS; } - private checkExistingAsset(asset: AssetEntity, stat: Stats | null): AssetSyncResult { + private checkExistingAsset( + asset: { + isOffline: boolean; + libraryId: string | null; + originalPath: string; + status: AssetStatus; + fileModifiedAt: Date; + }, + stat: Stats | null, + ): AssetSyncResult { if (!stat) { // File not found on disk or permission error if (asset.isOffline) { diff --git a/server/src/services/memory.service.spec.ts b/server/src/services/memory.service.spec.ts index 7ce8b1ab46..d55c58d9af 100644 --- a/server/src/services/memory.service.spec.ts +++ b/server/src/services/memory.service.spec.ts @@ -15,6 +15,14 @@ describe(MemoryService.name, () => { expect(sut).toBeDefined(); }); + describe('onMemoryCleanup', () => { + it('should clean up memories', async () => { + mocks.memory.cleanup.mockResolvedValue([]); + await sut.onMemoriesCleanup(); + expect(mocks.memory.cleanup).toHaveBeenCalled(); + }); + }); + describe('search', () => { it('should search memories', async () => { const [userId] = newUuids(); diff --git a/server/src/services/metadata.service.spec.ts b/server/src/services/metadata.service.spec.ts index ca1277a8c8..e412b1c31f 100644 --- a/server/src/services/metadata.service.spec.ts +++ b/server/src/services/metadata.service.spec.ts @@ -3,7 +3,7 @@ import { randomBytes } from 'node:crypto'; import { Stats } from 'node:fs'; import { constants } from 'node:fs/promises'; import { defaults } from 'src/config'; -import { AssetEntity } from 'src/entities/asset.entity'; +import { MapAsset } from 'src/dtos/asset-response.dto'; import { AssetType, ExifOrientation, ImmichWorker, JobName, JobStatus, SourceType } from 'src/enum'; import { WithoutProperty } from 'src/repositories/asset.repository'; import { ImmichTags } from 'src/repositories/metadata.repository'; @@ -549,7 +549,6 @@ describe(MetadataService.name, () => { livePhotoVideoId: null, libraryId: null, }); - mocks.asset.getByIds.mockResolvedValue([{ ...assetStub.livePhotoWithOriginalFileName, livePhotoVideoId: null }]); mocks.storage.stat.mockResolvedValue({ size: 123_456, mtime: assetStub.livePhotoWithOriginalFileName.fileModifiedAt, @@ -719,7 +718,7 @@ describe(MetadataService.name, () => { }); mocks.crypto.hashSha1.mockReturnValue(randomBytes(512)); mocks.asset.create.mockImplementation( - (asset) => Promise.resolve({ ...assetStub.livePhotoMotionAsset, ...asset }) as Promise, + (asset) => Promise.resolve({ ...assetStub.livePhotoMotionAsset, ...asset }) as Promise, ); const video = randomBytes(512); mocks.storage.readFile.mockResolvedValue(video); @@ -1394,7 +1393,7 @@ describe(MetadataService.name, () => { }); it('should set sidecar path if exists (sidecar named photo.xmp)', async () => { - mocks.asset.getByIds.mockResolvedValue([assetStub.sidecarWithoutExt]); + mocks.asset.getByIds.mockResolvedValue([assetStub.sidecarWithoutExt as any]); mocks.storage.checkFileExists.mockResolvedValueOnce(false); mocks.storage.checkFileExists.mockResolvedValueOnce(true); @@ -1446,7 +1445,7 @@ describe(MetadataService.name, () => { describe('handleSidecarDiscovery', () => { it('should skip hidden assets', async () => { - mocks.asset.getByIds.mockResolvedValue([assetStub.livePhotoMotionAsset]); + mocks.asset.getByIds.mockResolvedValue([assetStub.livePhotoMotionAsset as any]); await sut.handleSidecarDiscovery({ id: assetStub.livePhotoMotionAsset.id }); expect(mocks.storage.checkFileExists).not.toHaveBeenCalled(); }); diff --git a/server/src/services/metadata.service.ts b/server/src/services/metadata.service.ts index ab62c38ed0..faf146a2be 100644 --- a/server/src/services/metadata.service.ts +++ b/server/src/services/metadata.service.ts @@ -271,7 +271,7 @@ export class MetadataService extends BaseService { ]; if (this.isMotionPhoto(asset, exifTags)) { - promises.push(this.applyMotionPhotos(asset as unknown as Asset, exifTags, dates, stats)); + promises.push(this.applyMotionPhotos(asset, exifTags, dates, stats)); } if (isFaceImportEnabled(metadata) && this.hasTaggedFaces(exifTags)) { diff --git a/server/src/services/person.service.ts b/server/src/services/person.service.ts index a413c688f0..66d68857a0 100644 --- a/server/src/services/person.service.ts +++ b/server/src/services/person.service.ts @@ -2,7 +2,8 @@ import { BadRequestException, Injectable, NotFoundException } from '@nestjs/comm import { Insertable, Updateable } from 'kysely'; import { FACE_THUMBNAIL_SIZE, JOBS_ASSET_PAGINATION_SIZE } from 'src/constants'; import { StorageCore } from 'src/cores/storage.core'; -import { AssetFaces, FaceSearch, Person } from 'src/db'; +import { Person } from 'src/database'; +import { AssetFaces, FaceSearch } from 'src/db'; import { Chunked, OnJob } from 'src/decorators'; import { BulkIdErrorReason, BulkIdResponseDto } from 'src/dtos/asset-ids.response.dto'; import { AuthDto } from 'src/dtos/auth.dto'; @@ -315,6 +316,7 @@ export class PersonService extends BaseService { const facesToAdd: (Insertable & { id: string })[] = []; const embeddings: FaceSearch[] = []; const mlFaceIds = new Set(); + for (const face of asset.faces) { if (face.sourceType === SourceType.MACHINE_LEARNING) { mlFaceIds.add(face.id); @@ -477,7 +479,7 @@ export class PersonService extends BaseService { embedding: face.faceSearch.embedding, maxDistance: machineLearning.facialRecognition.maxDistance, numResults: machineLearning.facialRecognition.minFaces, - minBirthDate: face.asset.fileCreatedAt, + minBirthDate: face.asset.fileCreatedAt ?? undefined, }); // `matches` also includes the face itself @@ -503,7 +505,7 @@ export class PersonService extends BaseService { maxDistance: machineLearning.facialRecognition.maxDistance, numResults: 1, hasPerson: true, - minBirthDate: face.asset.fileCreatedAt, + minBirthDate: face.asset.fileCreatedAt ?? undefined, }); if (matchWithPerson.length > 0) { diff --git a/server/src/services/search.service.spec.ts b/server/src/services/search.service.spec.ts index 51c6b55e11..d87ccbde1d 100644 --- a/server/src/services/search.service.spec.ts +++ b/server/src/services/search.service.spec.ts @@ -39,13 +39,36 @@ describe(SearchService.name, () => { }); }); + describe('searchPlaces', () => { + it('should search places', async () => { + mocks.search.searchPlaces.mockResolvedValue([ + { + id: 42, + name: 'my place', + latitude: 420, + longitude: 69, + admin1Code: null, + admin1Name: null, + admin2Code: null, + admin2Name: null, + alternateNames: null, + countryCode: 'US', + modificationDate: new Date(), + }, + ]); + + await sut.searchPlaces({ name: 'place' }); + expect(mocks.search.searchPlaces).toHaveBeenCalledWith('place'); + }); + }); + describe('getExploreData', () => { it('should get assets by city and tag', async () => { mocks.asset.getAssetIdByCity.mockResolvedValue({ fieldName: 'exifInfo.city', items: [{ value: 'test-city', data: assetStub.withLocation.id }], }); - mocks.asset.getByIdsWithAllRelations.mockResolvedValue([assetStub.withLocation]); + mocks.asset.getByIdsWithAllRelationsButStacks.mockResolvedValue([assetStub.withLocation]); const expectedResponse = [ { fieldName: 'exifInfo.city', items: [{ value: 'test-city', data: mapAsset(assetStub.withLocation) }] }, ]; diff --git a/server/src/services/search.service.ts b/server/src/services/search.service.ts index 1c0c0ad490..442d49136c 100644 --- a/server/src/services/search.service.ts +++ b/server/src/services/search.service.ts @@ -1,5 +1,5 @@ import { BadRequestException, Injectable } from '@nestjs/common'; -import { AssetMapOptions, AssetResponseDto, mapAsset } from 'src/dtos/asset-response.dto'; +import { AssetMapOptions, AssetResponseDto, MapAsset, mapAsset } from 'src/dtos/asset-response.dto'; import { AuthDto } from 'src/dtos/auth.dto'; import { mapPerson, PersonResponseDto } from 'src/dtos/person.dto'; import { @@ -14,7 +14,6 @@ import { SearchSuggestionType, SmartSearchDto, } from 'src/dtos/search.dto'; -import { AssetEntity } from 'src/entities/asset.entity'; import { AssetOrder } from 'src/enum'; import { SearchExploreItem } from 'src/repositories/search.repository'; import { BaseService } from 'src/services/base.service'; @@ -36,7 +35,7 @@ export class SearchService extends BaseService { async getExploreData(auth: AuthDto): Promise[]> { const options = { maxFields: 12, minAssetsPerField: 5 }; const cities = await this.assetRepository.getAssetIdByCity(auth.user.id, options); - const assets = await this.assetRepository.getByIdsWithAllRelations(cities.items.map(({ data }) => data)); + const assets = await this.assetRepository.getByIdsWithAllRelationsButStacks(cities.items.map(({ data }) => data)); const items = assets.map((asset) => ({ value: asset.exifInfo!.city!, data: mapAsset(asset, { auth }) })); return [{ fieldName: cities.fieldName, items }]; } @@ -139,7 +138,7 @@ export class SearchService extends BaseService { return [auth.user.id, ...partnerIds]; } - private mapResponse(assets: AssetEntity[], nextPage: string | null, options: AssetMapOptions): SearchResponseDto { + private mapResponse(assets: MapAsset[], nextPage: string | null, options: AssetMapOptions): SearchResponseDto { return { albums: { total: 0, count: 0, items: [], facets: [] }, assets: { diff --git a/server/src/services/shared-link.service.spec.ts b/server/src/services/shared-link.service.spec.ts index 4d084d6e67..66a0a925c7 100644 --- a/server/src/services/shared-link.service.spec.ts +++ b/server/src/services/shared-link.service.spec.ts @@ -244,7 +244,7 @@ describe(SharedLinkService.name, () => { await sut.remove(authStub.user1, sharedLinkStub.valid.id); expect(mocks.sharedLink.get).toHaveBeenCalledWith(authStub.user1.user.id, sharedLinkStub.valid.id); - expect(mocks.sharedLink.remove).toHaveBeenCalledWith(sharedLinkStub.valid); + expect(mocks.sharedLink.remove).toHaveBeenCalledWith(sharedLinkStub.valid.id); }); }); @@ -333,8 +333,7 @@ describe(SharedLinkService.name, () => { }); it('should return metadata tags with a default image path if the asset id is not set', async () => { - mocks.sharedLink.get.mockResolvedValue({ ...sharedLinkStub.individual, album: undefined, assets: [] }); - + mocks.sharedLink.get.mockResolvedValue({ ...sharedLinkStub.individual, album: null, assets: [] }); await expect(sut.getMetadataTags(authStub.adminSharedLink)).resolves.toEqual({ description: '0 shared photos & videos', imageUrl: `https://my.immich.app/feature-panel.png`, diff --git a/server/src/services/shared-link.service.ts b/server/src/services/shared-link.service.ts index 95f8cef5f8..17f1b974d2 100644 --- a/server/src/services/shared-link.service.ts +++ b/server/src/services/shared-link.service.ts @@ -1,4 +1,5 @@ import { BadRequestException, ForbiddenException, Injectable, UnauthorizedException } from '@nestjs/common'; +import { SharedLink } from 'src/database'; import { AssetIdErrorReason, AssetIdsResponseDto } from 'src/dtos/asset-ids.response.dto'; import { AssetIdsDto } from 'src/dtos/asset.dto'; import { AuthDto } from 'src/dtos/auth.dto'; @@ -11,7 +12,6 @@ import { SharedLinkResponseDto, SharedLinkSearchDto, } from 'src/dtos/shared-link.dto'; -import { SharedLinkEntity } from 'src/entities/shared-link.entity'; import { Permission, SharedLinkType } from 'src/enum'; import { BaseService } from 'src/services/base.service'; import { getExternalDomain, OpenGraphTags } from 'src/utils/misc'; @@ -98,7 +98,7 @@ export class SharedLinkService extends BaseService { async remove(auth: AuthDto, id: string): Promise { const sharedLink = await this.findOrFail(auth.user.id, id); - await this.sharedLinkRepository.remove(sharedLink); + await this.sharedLinkRepository.remove(sharedLink.id); } // TODO: replace `userId` with permissions and access control checks @@ -182,7 +182,7 @@ export class SharedLinkService extends BaseService { const config = await this.getConfig({ withCache: true }); const sharedLink = await this.findOrFail(auth.sharedLink.userId, auth.sharedLink.id); const assetId = sharedLink.album?.albumThumbnailAssetId || sharedLink.assets[0]?.id; - const assetCount = sharedLink.assets.length > 0 ? sharedLink.assets.length : sharedLink.album?.assets.length || 0; + const assetCount = sharedLink.assets.length > 0 ? sharedLink.assets.length : sharedLink.album?.assets?.length || 0; const imagePath = assetId ? `/api/assets/${assetId}/thumbnail?key=${sharedLink.key.toString('base64url')}` : '/feature-panel.png'; @@ -194,11 +194,11 @@ export class SharedLinkService extends BaseService { }; } - private mapToSharedLink(sharedLink: SharedLinkEntity, { withExif }: { withExif: boolean }) { + private mapToSharedLink(sharedLink: SharedLink, { withExif }: { withExif: boolean }) { return withExif ? mapSharedLink(sharedLink) : mapSharedLinkWithoutMetadata(sharedLink); } - private validateAndRefreshToken(sharedLink: SharedLinkEntity, dto: SharedLinkPasswordDto): string { + private validateAndRefreshToken(sharedLink: SharedLink, dto: SharedLinkPasswordDto): string { const token = this.cryptoRepository.hashSha256(`${sharedLink.id}-${sharedLink.password}`); const sharedLinkTokens = dto.token?.split(',') || []; if (sharedLink.password !== dto.password && !sharedLinkTokens.includes(token)) { diff --git a/server/src/services/sync.service.spec.ts b/server/src/services/sync.service.spec.ts index 5f7357c64d..5b50340a9f 100644 --- a/server/src/services/sync.service.spec.ts +++ b/server/src/services/sync.service.spec.ts @@ -1,5 +1,4 @@ import { mapAsset } from 'src/dtos/asset-response.dto'; -import { AssetEntity } from 'src/entities/asset.entity'; import { SyncService } from 'src/services/sync.service'; import { assetStub } from 'test/fixtures/asset.stub'; import { authStub } from 'test/fixtures/auth.stub'; @@ -63,7 +62,7 @@ describe(SyncService.name, () => { it('should return a response requiring a full sync when there are too many changes', async () => { mocks.partner.getAll.mockResolvedValue([]); mocks.asset.getChangedDeltaSync.mockResolvedValue( - Array.from({ length: 10_000 }).fill(assetStub.image), + Array.from({ length: 10_000 }).fill(assetStub.image), ); await expect( sut.getDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }), diff --git a/server/src/services/view.service.ts b/server/src/services/view.service.ts index 5871b04b32..9d1ee3cf89 100644 --- a/server/src/services/view.service.ts +++ b/server/src/services/view.service.ts @@ -1,7 +1,6 @@ import { Injectable } from '@nestjs/common'; import { AssetResponseDto, mapAsset } from 'src/dtos/asset-response.dto'; import { AuthDto } from 'src/dtos/auth.dto'; -import { AssetEntity } from 'src/entities/asset.entity'; import { BaseService } from 'src/services/base.service'; @Injectable() @@ -12,6 +11,6 @@ export class ViewService extends BaseService { async getAssetsByOriginalPath(auth: AuthDto, path: string): Promise { const assets = await this.viewRepository.getAssetsByOriginalPath(auth.user.id, path); - return assets.map((asset) => mapAsset(asset as unknown as AssetEntity, { auth })); + return assets.map((asset) => mapAsset(asset, { auth })); } } diff --git a/server/src/sql-tools/from-code/decorators/column-index.decorator.ts b/server/src/sql-tools/from-code/decorators/column-index.decorator.ts deleted file mode 100644 index ab15292612..0000000000 --- a/server/src/sql-tools/from-code/decorators/column-index.decorator.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { register } from 'src/sql-tools/from-code/register'; -import { asOptions } from 'src/sql-tools/helpers'; - -export type ColumnIndexOptions = { - name?: string; - unique?: boolean; - expression?: string; - using?: string; - with?: string; - where?: string; - synchronize?: boolean; -}; -export const ColumnIndex = (options: string | ColumnIndexOptions = {}): PropertyDecorator => { - return (object: object, propertyName: string | symbol) => - void register({ type: 'columnIndex', item: { object, propertyName, options: asOptions(options) } }); -}; diff --git a/server/src/sql-tools/from-code/decorators/column.decorator.ts b/server/src/sql-tools/from-code/decorators/column.decorator.ts index 74a83cbcf3..7b00af80cc 100644 --- a/server/src/sql-tools/from-code/decorators/column.decorator.ts +++ b/server/src/sql-tools/from-code/decorators/column.decorator.ts @@ -15,13 +15,15 @@ export type ColumnBaseOptions = { synchronize?: boolean; storage?: ColumnStorage; identity?: boolean; + index?: boolean; + indexName?: string; + unique?: boolean; + uniqueConstraintName?: string; }; export type ColumnOptions = ColumnBaseOptions & { enum?: DatabaseEnum; array?: boolean; - unique?: boolean; - uniqueConstraintName?: string; }; export const Column = (options: string | ColumnOptions = {}): PropertyDecorator => { diff --git a/server/src/sql-tools/from-code/decorators/foreign-key-column.decorator.ts b/server/src/sql-tools/from-code/decorators/foreign-key-column.decorator.ts index 070aa5cb51..beb3aa6fd6 100644 --- a/server/src/sql-tools/from-code/decorators/foreign-key-column.decorator.ts +++ b/server/src/sql-tools/from-code/decorators/foreign-key-column.decorator.ts @@ -7,8 +7,6 @@ export type ForeignKeyColumnOptions = ColumnBaseOptions & { onUpdate?: Action; onDelete?: Action; constraintName?: string; - unique?: boolean; - uniqueConstraintName?: string; }; export const ForeignKeyColumn = (target: () => object, options: ForeignKeyColumnOptions): PropertyDecorator => { diff --git a/server/src/sql-tools/from-code/decorators/index.decorator.ts b/server/src/sql-tools/from-code/decorators/index.decorator.ts index cd76b5e36d..5d90c4f58d 100644 --- a/server/src/sql-tools/from-code/decorators/index.decorator.ts +++ b/server/src/sql-tools/from-code/decorators/index.decorator.ts @@ -1,8 +1,13 @@ -import { ColumnIndexOptions } from 'src/sql-tools/from-code/decorators/column-index.decorator'; import { register } from 'src/sql-tools/from-code/register'; import { asOptions } from 'src/sql-tools/helpers'; -export type IndexOptions = ColumnIndexOptions & { +export type IndexOptions = { + name?: string; + unique?: boolean; + expression?: string; + using?: string; + with?: string; + where?: string; columns?: string[]; synchronize?: boolean; }; diff --git a/server/src/sql-tools/from-code/index.ts b/server/src/sql-tools/from-code/index.ts index 3c74d2763c..95f1dbb22d 100644 --- a/server/src/sql-tools/from-code/index.ts +++ b/server/src/sql-tools/from-code/index.ts @@ -1,6 +1,5 @@ import 'reflect-metadata'; import { processCheckConstraints } from 'src/sql-tools/from-code/processors/check-constraint.processor'; -import { processColumnIndexes } from 'src/sql-tools/from-code/processors/column-index.processor'; import { processColumns } from 'src/sql-tools/from-code/processors/column.processor'; import { processConfigurationParameters } from 'src/sql-tools/from-code/processors/configuration-parameter.processor'; import { processDatabases } from 'src/sql-tools/from-code/processors/database.processor'; @@ -36,14 +35,21 @@ const processors: Processor[] = [ processUniqueConstraints, processCheckConstraints, processPrimaryKeyConstraints, - processIndexes, - processColumnIndexes, processForeignKeyConstraints, + processIndexes, processTriggers, ]; -export const schemaFromCode = () => { +export type SchemaFromCodeOptions = { + /** automatically create indexes on foreign key columns */ + createForeignKeyIndexes?: boolean; +}; +export const schemaFromCode = (options: SchemaFromCodeOptions = {}) => { if (!initialized) { + const globalOptions = { + createForeignKeyIndexes: options.createForeignKeyIndexes ?? true, + }; + const builder: SchemaBuilder = { name: 'postgres', schemaName: 'public', @@ -58,7 +64,7 @@ export const schemaFromCode = () => { const items = getRegisteredItems(); for (const processor of processors) { - processor(builder, items); + processor(builder, items, globalOptions); } schema = { ...builder, tables: builder.tables.map(({ metadata: _, ...table }) => table) }; diff --git a/server/src/sql-tools/from-code/processors/check-constraint.processor.ts b/server/src/sql-tools/from-code/processors/check-constraint.processor.ts index d61ee18277..feb21b9894 100644 --- a/server/src/sql-tools/from-code/processors/check-constraint.processor.ts +++ b/server/src/sql-tools/from-code/processors/check-constraint.processor.ts @@ -1,6 +1,6 @@ import { onMissingTable, resolveTable } from 'src/sql-tools/from-code/processors/table.processor'; import { Processor } from 'src/sql-tools/from-code/processors/type'; -import { asCheckConstraintName } from 'src/sql-tools/helpers'; +import { asKey } from 'src/sql-tools/helpers'; import { DatabaseConstraintType } from 'src/sql-tools/types'; export const processCheckConstraints: Processor = (builder, items) => { @@ -24,3 +24,5 @@ export const processCheckConstraints: Processor = (builder, items) => { }); } }; + +const asCheckConstraintName = (table: string, expression: string) => asKey('CHK_', table, [expression]); diff --git a/server/src/sql-tools/from-code/processors/column-index.processor.ts b/server/src/sql-tools/from-code/processors/column-index.processor.ts deleted file mode 100644 index 0e40fa1ee3..0000000000 --- a/server/src/sql-tools/from-code/processors/column-index.processor.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { onMissingColumn, resolveColumn } from 'src/sql-tools/from-code/processors/column.processor'; -import { onMissingTable } from 'src/sql-tools/from-code/processors/table.processor'; -import { Processor } from 'src/sql-tools/from-code/processors/type'; -import { asIndexName } from 'src/sql-tools/helpers'; - -export const processColumnIndexes: Processor = (builder, items) => { - for (const { - item: { object, propertyName, options }, - } of items.filter((item) => item.type === 'columnIndex')) { - const { table, column } = resolveColumn(builder, object, propertyName); - if (!table) { - onMissingTable(builder, '@ColumnIndex', object); - continue; - } - - if (!column) { - onMissingColumn(builder, `@ColumnIndex`, object, propertyName); - continue; - } - - table.indexes.push({ - name: options.name || asIndexName(table.name, [column.name], options.where), - tableName: table.name, - unique: options.unique ?? false, - expression: options.expression, - using: options.using, - where: options.where, - columnNames: [column.name], - synchronize: options.synchronize ?? true, - }); - } -}; diff --git a/server/src/sql-tools/from-code/processors/column.processor.ts b/server/src/sql-tools/from-code/processors/column.processor.ts index 37f3f5d082..e8c2544f87 100644 --- a/server/src/sql-tools/from-code/processors/column.processor.ts +++ b/server/src/sql-tools/from-code/processors/column.processor.ts @@ -1,8 +1,8 @@ import { ColumnOptions } from 'src/sql-tools/from-code/decorators/column.decorator'; import { onMissingTable, resolveTable } from 'src/sql-tools/from-code/processors/table.processor'; import { Processor, SchemaBuilder } from 'src/sql-tools/from-code/processors/type'; -import { asMetadataKey, asUniqueConstraintName, fromColumnValue } from 'src/sql-tools/helpers'; -import { DatabaseColumn, DatabaseConstraintType } from 'src/sql-tools/types'; +import { asMetadataKey, fromColumnValue } from 'src/sql-tools/helpers'; +import { DatabaseColumn } from 'src/sql-tools/types'; export const processColumns: Processor = (builder, items) => { for (const { @@ -54,16 +54,6 @@ export const processColumns: Processor = (builder, items) => { writeMetadata(object, propertyName, { name: column.name, options }); table.columns.push(column); - - if (type === 'column' && !options.primary && options.unique) { - table.constraints.push({ - type: DatabaseConstraintType.UNIQUE, - name: options.uniqueConstraintName || asUniqueConstraintName(table.name, [column.name]), - tableName: table.name, - columnNames: [column.name], - synchronize: options.synchronize ?? true, - }); - } } }; diff --git a/server/src/sql-tools/from-code/processors/foreign-key-constriant.processor.ts b/server/src/sql-tools/from-code/processors/foreign-key-constriant.processor.ts index 784a8b8e99..612b74c30f 100644 --- a/server/src/sql-tools/from-code/processors/foreign-key-constriant.processor.ts +++ b/server/src/sql-tools/from-code/processors/foreign-key-constriant.processor.ts @@ -1,7 +1,7 @@ import { onMissingColumn, resolveColumn } from 'src/sql-tools/from-code/processors/column.processor'; import { onMissingTable, resolveTable } from 'src/sql-tools/from-code/processors/table.processor'; import { Processor } from 'src/sql-tools/from-code/processors/type'; -import { asForeignKeyConstraintName, asRelationKeyConstraintName } from 'src/sql-tools/helpers'; +import { asKey } from 'src/sql-tools/helpers'; import { DatabaseActionType, DatabaseConstraintType } from 'src/sql-tools/types'; export const processForeignKeyConstraints: Processor = (builder, items) => { @@ -46,7 +46,7 @@ export const processForeignKeyConstraints: Processor = (builder, items) => { synchronize: options.synchronize ?? true, }); - if (options.unique) { + if (options.unique || options.uniqueConstraintName) { table.constraints.push({ name: options.uniqueConstraintName || asRelationKeyConstraintName(table.name, columnNames), tableName: table.name, @@ -57,3 +57,6 @@ export const processForeignKeyConstraints: Processor = (builder, items) => { } } }; + +const asForeignKeyConstraintName = (table: string, columns: string[]) => asKey('FK_', table, columns); +const asRelationKeyConstraintName = (table: string, columns: string[]) => asKey('REL_', table, columns); diff --git a/server/src/sql-tools/from-code/processors/index.processor.ts b/server/src/sql-tools/from-code/processors/index.processor.ts index 3625bf9784..f4c9c7cec1 100644 --- a/server/src/sql-tools/from-code/processors/index.processor.ts +++ b/server/src/sql-tools/from-code/processors/index.processor.ts @@ -1,8 +1,9 @@ +import { onMissingColumn, resolveColumn } from 'src/sql-tools/from-code/processors/column.processor'; import { onMissingTable, resolveTable } from 'src/sql-tools/from-code/processors/table.processor'; import { Processor } from 'src/sql-tools/from-code/processors/type'; -import { asIndexName } from 'src/sql-tools/helpers'; +import { asKey } from 'src/sql-tools/helpers'; -export const processIndexes: Processor = (builder, items) => { +export const processIndexes: Processor = (builder, items, config) => { for (const { item: { object, options }, } of items.filter((item) => item.type === 'index')) { @@ -24,4 +25,66 @@ export const processIndexes: Processor = (builder, items) => { synchronize: options.synchronize ?? true, }); } + + // column indexes + for (const { + type, + item: { object, propertyName, options }, + } of items.filter((item) => item.type === 'column' || item.type === 'foreignKeyColumn')) { + const { table, column } = resolveColumn(builder, object, propertyName); + if (!table) { + onMissingTable(builder, '@Column', object); + continue; + } + + if (!column) { + // should be impossible since they are created in `column.processor.ts` + onMissingColumn(builder, '@Column', object, propertyName); + continue; + } + + if (options.index === false) { + continue; + } + + const isIndexRequested = + options.indexName || options.index || (type === 'foreignKeyColumn' && config.createForeignKeyIndexes); + if (!isIndexRequested) { + continue; + } + + const indexName = options.indexName || asIndexName(table.name, [column.name]); + + const isIndexPresent = table.indexes.some((index) => index.name === indexName); + if (isIndexPresent) { + continue; + } + + const isOnlyPrimaryColumn = options.primary && table.columns.filter(({ primary }) => primary === true).length === 1; + if (isOnlyPrimaryColumn) { + // will have an index created by the primary key constraint + continue; + } + + table.indexes.push({ + name: indexName, + tableName: table.name, + unique: false, + columnNames: [column.name], + synchronize: options.synchronize ?? true, + }); + } +}; + +const asIndexName = (table: string, columns?: string[], where?: string) => { + const items: string[] = []; + for (const columnName of columns ?? []) { + items.push(columnName); + } + + if (where) { + items.push(where); + } + + return asKey('IDX_', table, items); }; diff --git a/server/src/sql-tools/from-code/processors/primary-key-contraint.processor.ts b/server/src/sql-tools/from-code/processors/primary-key-contraint.processor.ts index f123f2e495..74aecc5ea0 100644 --- a/server/src/sql-tools/from-code/processors/primary-key-contraint.processor.ts +++ b/server/src/sql-tools/from-code/processors/primary-key-contraint.processor.ts @@ -1,5 +1,5 @@ import { Processor } from 'src/sql-tools/from-code/processors/type'; -import { asPrimaryKeyConstraintName } from 'src/sql-tools/helpers'; +import { asKey } from 'src/sql-tools/helpers'; import { DatabaseConstraintType } from 'src/sql-tools/types'; export const processPrimaryKeyConstraints: Processor = (builder) => { @@ -22,3 +22,5 @@ export const processPrimaryKeyConstraints: Processor = (builder) => { } } }; + +const asPrimaryKeyConstraintName = (table: string, columns: string[]) => asKey('PK_', table, columns); diff --git a/server/src/sql-tools/from-code/processors/table.processor.ts b/server/src/sql-tools/from-code/processors/table.processor.ts index eb4b414576..4ef4e82020 100644 --- a/server/src/sql-tools/from-code/processors/table.processor.ts +++ b/server/src/sql-tools/from-code/processors/table.processor.ts @@ -6,6 +6,13 @@ export const processTables: Processor = (builder, items) => { for (const { item: { options, object }, } of items.filter((item) => item.type === 'table')) { + const test = readMetadata(object); + if (test) { + throw new Error( + `Table ${test.name} has already been registered. Does ${object.name} have two @Table() decorators?`, + ); + } + const tableName = options.name || asSnakeCase(object.name); writeMetadata(object, { name: tableName, options }); diff --git a/server/src/sql-tools/from-code/processors/trigger.processor.ts b/server/src/sql-tools/from-code/processors/trigger.processor.ts index 2f4cc04326..4b875f353b 100644 --- a/server/src/sql-tools/from-code/processors/trigger.processor.ts +++ b/server/src/sql-tools/from-code/processors/trigger.processor.ts @@ -1,6 +1,7 @@ +import { TriggerOptions } from 'src/sql-tools/from-code/decorators/trigger.decorator'; import { onMissingTable, resolveTable } from 'src/sql-tools/from-code/processors/table.processor'; import { Processor } from 'src/sql-tools/from-code/processors/type'; -import { asTriggerName } from 'src/sql-tools/helpers'; +import { asKey } from 'src/sql-tools/helpers'; export const processTriggers: Processor = (builder, items) => { for (const { @@ -26,3 +27,6 @@ export const processTriggers: Processor = (builder, items) => { }); } }; + +const asTriggerName = (table: string, trigger: TriggerOptions) => + asKey('TR_', table, [...trigger.actions, trigger.scope, trigger.timing, trigger.functionName]); diff --git a/server/src/sql-tools/from-code/processors/type.ts b/server/src/sql-tools/from-code/processors/type.ts index 5a69efbcf0..deb142d278 100644 --- a/server/src/sql-tools/from-code/processors/type.ts +++ b/server/src/sql-tools/from-code/processors/type.ts @@ -1,3 +1,4 @@ +import { SchemaFromCodeOptions } from 'src/sql-tools/from-code'; import { TableOptions } from 'src/sql-tools/from-code/decorators/table.decorator'; import { RegisterItem } from 'src/sql-tools/from-code/register-item'; import { DatabaseSchema, DatabaseTable } from 'src/sql-tools/types'; @@ -6,4 +7,4 @@ import { DatabaseSchema, DatabaseTable } from 'src/sql-tools/types'; export type TableWithMetadata = DatabaseTable & { metadata: { options: TableOptions; object: Function } }; export type SchemaBuilder = Omit & { tables: TableWithMetadata[] }; -export type Processor = (builder: SchemaBuilder, items: RegisterItem[]) => void; +export type Processor = (builder: SchemaBuilder, items: RegisterItem[], options: SchemaFromCodeOptions) => void; diff --git a/server/src/sql-tools/from-code/processors/unique-constraint.processor.ts b/server/src/sql-tools/from-code/processors/unique-constraint.processor.ts index 74c0504f7e..9014378085 100644 --- a/server/src/sql-tools/from-code/processors/unique-constraint.processor.ts +++ b/server/src/sql-tools/from-code/processors/unique-constraint.processor.ts @@ -1,6 +1,7 @@ +import { onMissingColumn, resolveColumn } from 'src/sql-tools/from-code/processors/column.processor'; import { onMissingTable, resolveTable } from 'src/sql-tools/from-code/processors/table.processor'; import { Processor } from 'src/sql-tools/from-code/processors/type'; -import { asUniqueConstraintName } from 'src/sql-tools/helpers'; +import { asKey } from 'src/sql-tools/helpers'; import { DatabaseConstraintType } from 'src/sql-tools/types'; export const processUniqueConstraints: Processor = (builder, items) => { @@ -24,4 +25,34 @@ export const processUniqueConstraints: Processor = (builder, items) => { synchronize: options.synchronize ?? true, }); } + + // column level constraints + for (const { + type, + item: { object, propertyName, options }, + } of items.filter((item) => item.type === 'column' || item.type === 'foreignKeyColumn')) { + const { table, column } = resolveColumn(builder, object, propertyName); + if (!table) { + onMissingTable(builder, '@Column', object); + continue; + } + + if (!column) { + // should be impossible since they are created in `column.processor.ts` + onMissingColumn(builder, '@Column', object, propertyName); + continue; + } + + if (type === 'column' && !options.primary && (options.unique || options.uniqueConstraintName)) { + table.constraints.push({ + type: DatabaseConstraintType.UNIQUE, + name: options.uniqueConstraintName || asUniqueConstraintName(table.name, [column.name]), + tableName: table.name, + columnNames: [column.name], + synchronize: options.synchronize ?? true, + }); + } + } }; + +const asUniqueConstraintName = (table: string, columns: string[]) => asKey('UQ_', table, columns); diff --git a/server/src/sql-tools/from-code/register-function.ts b/server/src/sql-tools/from-code/register-function.ts index 69e1a0f8f3..3e1e7054be 100644 --- a/server/src/sql-tools/from-code/register-function.ts +++ b/server/src/sql-tools/from-code/register-function.ts @@ -1,5 +1,4 @@ import { register } from 'src/sql-tools/from-code/register'; -import { asFunctionExpression } from 'src/sql-tools/helpers'; import { ColumnType, DatabaseFunction } from 'src/sql-tools/types'; export type FunctionOptions = { @@ -27,3 +26,33 @@ export const registerFunction = (options: FunctionOptions) => { return item; }; + +const asFunctionExpression = (options: FunctionOptions) => { + const name = options.name; + const sql: string[] = [ + `CREATE OR REPLACE FUNCTION ${name}(${(options.arguments || []).join(', ')})`, + `RETURNS ${options.returnType}`, + ]; + + const flags = [ + options.parallel ? `PARALLEL ${options.parallel.toUpperCase()}` : undefined, + options.strict ? 'STRICT' : undefined, + options.behavior ? options.behavior.toUpperCase() : undefined, + `LANGUAGE ${options.language ?? 'SQL'}`, + ].filter((x) => x !== undefined); + + if (flags.length > 0) { + sql.push(flags.join(' ')); + } + + if ('return' in options) { + sql.push(` RETURN ${options.return}`); + } + + if ('body' in options) { + const body = options.body; + sql.push(...(body.includes('\n') ? [`AS $$`, ' ' + body.trim(), `$$;`] : [`AS $$${body}$$;`])); + } + + return sql.join('\n ').trim(); +}; diff --git a/server/src/sql-tools/from-code/register-item.ts b/server/src/sql-tools/from-code/register-item.ts index 08200cbc4f..4889ae34b9 100644 --- a/server/src/sql-tools/from-code/register-item.ts +++ b/server/src/sql-tools/from-code/register-item.ts @@ -1,5 +1,4 @@ import { CheckOptions } from 'src/sql-tools/from-code/decorators/check.decorator'; -import { ColumnIndexOptions } from 'src/sql-tools/from-code/decorators/column-index.decorator'; import { ColumnOptions } from 'src/sql-tools/from-code/decorators/column.decorator'; import { ConfigurationParameterOptions } from 'src/sql-tools/from-code/decorators/configuration-parameter.decorator'; import { DatabaseOptions } from 'src/sql-tools/from-code/decorators/database.decorator'; @@ -21,7 +20,6 @@ export type RegisterItem = | { type: 'uniqueConstraint'; item: ClassBased<{ options: UniqueOptions }> } | { type: 'checkConstraint'; item: ClassBased<{ options: CheckOptions }> } | { type: 'column'; item: PropertyBased<{ options: ColumnOptions }> } - | { type: 'columnIndex'; item: PropertyBased<{ options: ColumnIndexOptions }> } | { type: 'function'; item: DatabaseFunction } | { type: 'enum'; item: DatabaseEnum } | { type: 'trigger'; item: ClassBased<{ options: TriggerOptions }> } diff --git a/server/src/sql-tools/helpers.ts b/server/src/sql-tools/helpers.ts index 364b695194..2802407ea6 100644 --- a/server/src/sql-tools/helpers.ts +++ b/server/src/sql-tools/helpers.ts @@ -1,7 +1,5 @@ import { createHash } from 'node:crypto'; import { ColumnValue } from 'src/sql-tools/from-code/decorators/column.decorator'; -import { TriggerOptions } from 'src/sql-tools/from-code/decorators/trigger.decorator'; -import { FunctionOptions } from 'src/sql-tools/from-code/register-function'; import { Comparer, DatabaseColumn, @@ -18,25 +16,6 @@ export const asSnakeCase = (name: string): string => name.replaceAll(/([a-z])([A // match TypeORM export const asKey = (prefix: string, tableName: string, values: string[]) => (prefix + sha1(`${tableName}_${values.toSorted().join('_')}`)).slice(0, 30); -export const asPrimaryKeyConstraintName = (table: string, columns: string[]) => asKey('PK_', table, columns); -export const asForeignKeyConstraintName = (table: string, columns: string[]) => asKey('FK_', table, columns); -export const asTriggerName = (table: string, trigger: TriggerOptions) => - asKey('TR_', table, [...trigger.actions, trigger.scope, trigger.timing, trigger.functionName]); -export const asRelationKeyConstraintName = (table: string, columns: string[]) => asKey('REL_', table, columns); -export const asUniqueConstraintName = (table: string, columns: string[]) => asKey('UQ_', table, columns); -export const asCheckConstraintName = (table: string, expression: string) => asKey('CHK_', table, [expression]); -export const asIndexName = (table: string, columns: string[] | undefined, where: string | undefined) => { - const items: string[] = []; - for (const columnName of columns ?? []) { - items.push(columnName); - } - - if (where) { - items.push(where); - } - - return asKey('IDX_', table, items); -}; export const asOptions = (options: string | T): T => { if (typeof options === 'string') { @@ -46,40 +25,6 @@ export const asOptions = (options: string | T): T = return options; }; -export const asFunctionExpression = (options: FunctionOptions) => { - const name = options.name; - const sql: string[] = [ - `CREATE OR REPLACE FUNCTION ${name}(${(options.arguments || []).join(', ')})`, - `RETURNS ${options.returnType}`, - ]; - - const flags = [ - options.parallel ? `PARALLEL ${options.parallel.toUpperCase()}` : undefined, - options.strict ? 'STRICT' : undefined, - options.behavior ? options.behavior.toUpperCase() : undefined, - `LANGUAGE ${options.language ?? 'SQL'}`, - ].filter((x) => x !== undefined); - - if (flags.length > 0) { - sql.push(flags.join(' ')); - } - - if ('return' in options) { - sql.push(` RETURN ${options.return}`); - } - - if ('body' in options) { - sql.push( - // - `AS $$`, - ' ' + options.body.trim(), - `$$;`, - ); - } - - return sql.join('\n ').trim(); -}; - export const sha1 = (value: string) => createHash('sha1').update(value).digest('hex'); export const hasMask = (input: number, mask: number) => (input & mask) === mask; diff --git a/server/src/sql-tools/public_api.ts b/server/src/sql-tools/public_api.ts index d916678d4a..b41cce4ab5 100644 --- a/server/src/sql-tools/public_api.ts +++ b/server/src/sql-tools/public_api.ts @@ -3,7 +3,6 @@ export { schemaFromCode } from 'src/sql-tools/from-code'; export * from 'src/sql-tools/from-code/decorators/after-delete.decorator'; export * from 'src/sql-tools/from-code/decorators/before-update.decorator'; export * from 'src/sql-tools/from-code/decorators/check.decorator'; -export * from 'src/sql-tools/from-code/decorators/column-index.decorator'; export * from 'src/sql-tools/from-code/decorators/column.decorator'; export * from 'src/sql-tools/from-code/decorators/configuration-parameter.decorator'; export * from 'src/sql-tools/from-code/decorators/create-date-column.decorator'; diff --git a/server/src/utils/asset.util.ts b/server/src/utils/asset.util.ts index 29db9dd574..058746f388 100644 --- a/server/src/utils/asset.util.ts +++ b/server/src/utils/asset.util.ts @@ -13,11 +13,8 @@ import { PartnerRepository } from 'src/repositories/partner.repository'; import { IBulkAsset, ImmichFile, UploadFile } from 'src/types'; import { checkAccess } from 'src/utils/access'; -export const getAssetFile = ( - files: T[], - type: AssetFileType | GeneratedImageType, -) => { - return (files || []).find((file) => file.type === type); +export const getAssetFile = (files: AssetFile[], type: AssetFileType | GeneratedImageType) => { + return files.find((file) => file.type === type); }; export const getAssetFiles = (files: AssetFile[]) => ({ diff --git a/server/src/utils/database.ts b/server/src/utils/database.ts index a9c7b09c5a..17b8f3235f 100644 --- a/server/src/utils/database.ts +++ b/server/src/utils/database.ts @@ -1,15 +1,25 @@ import { + DeduplicateJoinsPlugin, Expression, + expressionBuilder, ExpressionBuilder, ExpressionWrapper, + Kysely, KyselyConfig, Nullable, Selectable, + SelectQueryBuilder, Simplify, sql, } from 'kysely'; import { PostgresJSDialect } from 'kysely-postgres-js'; +import { jsonArrayFrom, jsonObjectFrom } from 'kysely/helpers/postgres'; import postgres, { Notice } from 'postgres'; +import { columns, Exif, Person } from 'src/database'; +import { DB } from 'src/db'; +import { AssetFileType } from 'src/enum'; +import { TimeBucketSize } from 'src/repositories/asset.repository'; +import { AssetSearchBuilderOptions } from 'src/repositories/search.repository'; type Ssl = 'require' | 'allow' | 'prefer' | 'verify-full' | boolean | object; @@ -112,3 +122,230 @@ export function toJson >; } + +export const ASSET_CHECKSUM_CONSTRAINT = 'UQ_assets_owner_checksum'; +// TODO come up with a better query that only selects the fields we need + +export function withExif(qb: SelectQueryBuilder) { + return qb + .leftJoin('exif', 'assets.id', 'exif.assetId') + .select((eb) => eb.fn.toJson(eb.table('exif')).$castTo().as('exifInfo')); +} + +export function withExifInner(qb: SelectQueryBuilder) { + return qb + .innerJoin('exif', 'assets.id', 'exif.assetId') + .select((eb) => eb.fn.toJson(eb.table('exif')).$castTo().as('exifInfo')); +} + +export function withSmartSearch(qb: SelectQueryBuilder) { + return qb + .leftJoin('smart_search', 'assets.id', 'smart_search.assetId') + .select((eb) => toJson(eb, 'smart_search').as('smartSearch')); +} + +export function withFaces(eb: ExpressionBuilder, withDeletedFace?: boolean) { + return jsonArrayFrom( + eb + .selectFrom('asset_faces') + .selectAll('asset_faces') + .whereRef('asset_faces.assetId', '=', 'assets.id') + .$if(!withDeletedFace, (qb) => qb.where('asset_faces.deletedAt', 'is', null)), + ).as('faces'); +} + +export function withFiles(eb: ExpressionBuilder, type?: AssetFileType) { + return jsonArrayFrom( + eb + .selectFrom('asset_files') + .select(columns.assetFiles) + .whereRef('asset_files.assetId', '=', 'assets.id') + .$if(!!type, (qb) => qb.where('asset_files.type', '=', type!)), + ).as('files'); +} + +export function withFacesAndPeople(eb: ExpressionBuilder, withDeletedFace?: boolean) { + return jsonArrayFrom( + eb + .selectFrom('asset_faces') + .leftJoinLateral( + (eb) => + eb.selectFrom('person').selectAll('person').whereRef('asset_faces.personId', '=', 'person.id').as('person'), + (join) => join.onTrue(), + ) + .selectAll('asset_faces') + .select((eb) => eb.table('person').$castTo().as('person')) + .whereRef('asset_faces.assetId', '=', 'assets.id') + .$if(!withDeletedFace, (qb) => qb.where('asset_faces.deletedAt', 'is', null)), + ).as('faces'); +} + +export function hasPeople(qb: SelectQueryBuilder, personIds: string[]) { + return qb.innerJoin(hasPeopleNoJoin(personIds), (join) => join.onRef('has_people.assetId', '=', 'assets.id')); +} + +export function hasPeopleNoJoin(personIds: string[]) { + const eb = expressionBuilder(); + return eb + .selectFrom('asset_faces') + .select('assetId') + .where('personId', '=', anyUuid(personIds!)) + .where('deletedAt', 'is', null) + .groupBy('assetId') + .having((eb) => eb.fn.count('personId').distinct(), '=', personIds.length) + .as('has_people'); +} + +export function hasTags(qb: SelectQueryBuilder, tagIds: string[]) { + return qb.innerJoin( + (eb) => + eb + .selectFrom('tag_asset') + .select('assetsId') + .innerJoin('tags_closure', 'tag_asset.tagsId', 'tags_closure.id_descendant') + .where('tags_closure.id_ancestor', '=', anyUuid(tagIds)) + .groupBy('assetsId') + .having((eb) => eb.fn.count('tags_closure.id_ancestor').distinct(), '>=', tagIds.length) + .as('has_tags'), + (join) => join.onRef('has_tags.assetsId', '=', 'assets.id'), + ); +} + +export function withOwner(eb: ExpressionBuilder) { + return jsonObjectFrom(eb.selectFrom('users').select(columns.user).whereRef('users.id', '=', 'assets.ownerId')).as( + 'owner', + ); +} + +export function withLibrary(eb: ExpressionBuilder) { + return jsonObjectFrom( + eb.selectFrom('libraries').selectAll('libraries').whereRef('libraries.id', '=', 'assets.libraryId'), + ).as('library'); +} + +export function withTags(eb: ExpressionBuilder) { + return jsonArrayFrom( + eb + .selectFrom('tags') + .select(columns.tag) + .innerJoin('tag_asset', 'tags.id', 'tag_asset.tagsId') + .whereRef('assets.id', '=', 'tag_asset.assetsId'), + ).as('tags'); +} + +export function truncatedDate(size: TimeBucketSize) { + return sql`date_trunc(${size}, "localDateTime" at time zone 'UTC') at time zone 'UTC'`; +} + +export function withTagId(qb: SelectQueryBuilder, tagId: string) { + return qb.where((eb) => withTagIdNoWhere(tagId, eb.ref('assets.id'))); +} + +export function withTagIdNoWhere(tagId: string, assetId: Expression) { + const eb = expressionBuilder(); + return eb.exists( + eb + .selectFrom('tags_closure') + .innerJoin('tag_asset', 'tag_asset.tagsId', 'tags_closure.id_descendant') + .whereRef('tag_asset.assetsId', '=', assetId) + .where('tags_closure.id_ancestor', '=', tagId), + ); +} + +const joinDeduplicationPlugin = new DeduplicateJoinsPlugin(); +/** TODO: This should only be used for search-related queries, not as a general purpose query builder */ + +export function searchAssetBuilder(kysely: Kysely, options: AssetSearchBuilderOptions) { + options.isArchived ??= options.withArchived ? undefined : false; + options.withDeleted ||= !!(options.trashedAfter || options.trashedBefore || options.isOffline); + return kysely + .withPlugin(joinDeduplicationPlugin) + .selectFrom('assets') + .selectAll('assets') + .$if(!!options.tagIds && options.tagIds.length > 0, (qb) => hasTags(qb, options.tagIds!)) + .$if(!!options.personIds && options.personIds.length > 0, (qb) => hasPeople(qb, options.personIds!)) + .$if(!!options.createdBefore, (qb) => qb.where('assets.createdAt', '<=', options.createdBefore!)) + .$if(!!options.createdAfter, (qb) => qb.where('assets.createdAt', '>=', options.createdAfter!)) + .$if(!!options.updatedBefore, (qb) => qb.where('assets.updatedAt', '<=', options.updatedBefore!)) + .$if(!!options.updatedAfter, (qb) => qb.where('assets.updatedAt', '>=', options.updatedAfter!)) + .$if(!!options.trashedBefore, (qb) => qb.where('assets.deletedAt', '<=', options.trashedBefore!)) + .$if(!!options.trashedAfter, (qb) => qb.where('assets.deletedAt', '>=', options.trashedAfter!)) + .$if(!!options.takenBefore, (qb) => qb.where('assets.fileCreatedAt', '<=', options.takenBefore!)) + .$if(!!options.takenAfter, (qb) => qb.where('assets.fileCreatedAt', '>=', options.takenAfter!)) + .$if(options.city !== undefined, (qb) => + qb + .innerJoin('exif', 'assets.id', 'exif.assetId') + .where('exif.city', options.city === null ? 'is' : '=', options.city!), + ) + .$if(options.state !== undefined, (qb) => + qb + .innerJoin('exif', 'assets.id', 'exif.assetId') + .where('exif.state', options.state === null ? 'is' : '=', options.state!), + ) + .$if(options.country !== undefined, (qb) => + qb + .innerJoin('exif', 'assets.id', 'exif.assetId') + .where('exif.country', options.country === null ? 'is' : '=', options.country!), + ) + .$if(options.make !== undefined, (qb) => + qb + .innerJoin('exif', 'assets.id', 'exif.assetId') + .where('exif.make', options.make === null ? 'is' : '=', options.make!), + ) + .$if(options.model !== undefined, (qb) => + qb + .innerJoin('exif', 'assets.id', 'exif.assetId') + .where('exif.model', options.model === null ? 'is' : '=', options.model!), + ) + .$if(options.lensModel !== undefined, (qb) => + qb + .innerJoin('exif', 'assets.id', 'exif.assetId') + .where('exif.lensModel', options.lensModel === null ? 'is' : '=', options.lensModel!), + ) + .$if(options.rating !== undefined, (qb) => + qb + .innerJoin('exif', 'assets.id', 'exif.assetId') + .where('exif.rating', options.rating === null ? 'is' : '=', options.rating!), + ) + .$if(!!options.checksum, (qb) => qb.where('assets.checksum', '=', options.checksum!)) + .$if(!!options.deviceAssetId, (qb) => qb.where('assets.deviceAssetId', '=', options.deviceAssetId!)) + .$if(!!options.deviceId, (qb) => qb.where('assets.deviceId', '=', options.deviceId!)) + .$if(!!options.id, (qb) => qb.where('assets.id', '=', asUuid(options.id!))) + .$if(!!options.libraryId, (qb) => qb.where('assets.libraryId', '=', asUuid(options.libraryId!))) + .$if(!!options.userIds, (qb) => qb.where('assets.ownerId', '=', anyUuid(options.userIds!))) + .$if(!!options.encodedVideoPath, (qb) => qb.where('assets.encodedVideoPath', '=', options.encodedVideoPath!)) + .$if(!!options.originalPath, (qb) => + qb.where(sql`f_unaccent(assets."originalPath")`, 'ilike', sql`'%' || f_unaccent(${options.originalPath}) || '%'`), + ) + .$if(!!options.originalFileName, (qb) => + qb.where( + sql`f_unaccent(assets."originalFileName")`, + 'ilike', + sql`'%' || f_unaccent(${options.originalFileName}) || '%'`, + ), + ) + .$if(!!options.description, (qb) => + qb + .innerJoin('exif', 'assets.id', 'exif.assetId') + .where(sql`f_unaccent(exif.description)`, 'ilike', sql`'%' || f_unaccent(${options.description}) || '%'`), + ) + .$if(!!options.type, (qb) => qb.where('assets.type', '=', options.type!)) + .$if(options.isFavorite !== undefined, (qb) => qb.where('assets.isFavorite', '=', options.isFavorite!)) + .$if(options.isOffline !== undefined, (qb) => qb.where('assets.isOffline', '=', options.isOffline!)) + .$if(options.isVisible !== undefined, (qb) => qb.where('assets.isVisible', '=', options.isVisible!)) + .$if(options.isArchived !== undefined, (qb) => qb.where('assets.isArchived', '=', options.isArchived!)) + .$if(options.isEncoded !== undefined, (qb) => + qb.where('assets.encodedVideoPath', options.isEncoded ? 'is not' : 'is', null), + ) + .$if(options.isMotion !== undefined, (qb) => + qb.where('assets.livePhotoVideoId', options.isMotion ? 'is not' : 'is', null), + ) + .$if(!!options.isNotInAlbum, (qb) => + qb.where((eb) => + eb.not(eb.exists((eb) => eb.selectFrom('albums_assets_assets').whereRef('assetsId', '=', 'assets.id'))), + ), + ) + .$if(!!options.withExif, withExifInner) + .$if(!!(options.withFaces || options.withPeople || options.personIds), (qb) => qb.select(withFacesAndPeople)) + .$if(!options.withDeleted, (qb) => qb.where('assets.deletedAt', 'is', null)); +} diff --git a/server/test/fixtures/album.stub.ts b/server/test/fixtures/album.stub.ts index 5a1c141512..fd6a8678a0 100644 --- a/server/test/fixtures/album.stub.ts +++ b/server/test/fixtures/album.stub.ts @@ -1,11 +1,10 @@ -import { AlbumEntity } from 'src/entities/album.entity'; import { AlbumUserRole, AssetOrder } from 'src/enum'; import { assetStub } from 'test/fixtures/asset.stub'; import { authStub } from 'test/fixtures/auth.stub'; import { userStub } from 'test/fixtures/user.stub'; export const albumStub = { - empty: Object.freeze({ + empty: Object.freeze({ id: 'album-1', albumName: 'Empty album', description: '', @@ -21,8 +20,9 @@ export const albumStub = { albumUsers: [], isActivityEnabled: true, order: AssetOrder.DESC, + updateId: '42', }), - sharedWithUser: Object.freeze({ + sharedWithUser: Object.freeze({ id: 'album-2', albumName: 'Empty album shared with user', description: '', @@ -43,8 +43,9 @@ export const albumStub = { ], isActivityEnabled: true, order: AssetOrder.DESC, + updateId: '42', }), - sharedWithMultiple: Object.freeze({ + sharedWithMultiple: Object.freeze({ id: 'album-3', albumName: 'Empty album shared with users', description: '', @@ -69,8 +70,9 @@ export const albumStub = { ], isActivityEnabled: true, order: AssetOrder.DESC, + updateId: '42', }), - sharedWithAdmin: Object.freeze({ + sharedWithAdmin: Object.freeze({ id: 'album-3', albumName: 'Empty album shared with admin', description: '', @@ -91,8 +93,9 @@ export const albumStub = { ], isActivityEnabled: true, order: AssetOrder.DESC, + updateId: '42', }), - oneAsset: Object.freeze({ + oneAsset: Object.freeze({ id: 'album-4', albumName: 'Album with one asset', description: '', @@ -108,8 +111,9 @@ export const albumStub = { albumUsers: [], isActivityEnabled: true, order: AssetOrder.DESC, + updateId: '42', }), - twoAssets: Object.freeze({ + twoAssets: Object.freeze({ id: 'album-4a', albumName: 'Album with two assets', description: '', @@ -125,8 +129,9 @@ export const albumStub = { albumUsers: [], isActivityEnabled: true, order: AssetOrder.DESC, + updateId: '42', }), - emptyWithValidThumbnail: Object.freeze({ + emptyWithValidThumbnail: Object.freeze({ id: 'album-5', albumName: 'Empty album with valid thumbnail', description: '', @@ -142,5 +147,6 @@ export const albumStub = { albumUsers: [], isActivityEnabled: true, order: AssetOrder.DESC, + updateId: '42', }), }; diff --git a/server/test/fixtures/asset.stub.ts b/server/test/fixtures/asset.stub.ts index 16e4f20bb3..d1b8e7cf28 100644 --- a/server/test/fixtures/asset.stub.ts +++ b/server/test/fixtures/asset.stub.ts @@ -1,5 +1,5 @@ -import { AssetFile, Exif } from 'src/database'; -import { AssetEntity } from 'src/entities/asset.entity'; +import { AssetFace, AssetFile, Exif } from 'src/database'; +import { MapAsset } from 'src/dtos/asset-response.dto'; import { AssetFileType, AssetStatus, AssetType } from 'src/enum'; import { StorageAsset } from 'src/types'; import { authStub } from 'test/fixtures/auth.stub'; @@ -26,13 +26,15 @@ const fullsizeFile: AssetFile = { const files: AssetFile[] = [fullsizeFile, previewFile, thumbnailFile]; -export const stackStub = (stackId: string, assets: AssetEntity[]) => { +export const stackStub = (stackId: string, assets: (MapAsset & { exifInfo: Exif })[]) => { return { id: stackId, assets, ownerId: assets[0].ownerId, primaryAsset: assets[0], primaryAssetId: assets[0].id, + createdAt: new Date('2023-02-23T05:06:29.716Z'), + updatedAt: new Date('2023-02-23T05:06:29.716Z'), }; }; @@ -85,9 +87,12 @@ export const assetStub = { isExternal: false, duplicateId: null, isOffline: false, + libraryId: null, + stackId: null, + updateId: '42', }), - noWebpPath: Object.freeze({ + noWebpPath: Object.freeze({ id: 'asset-id', status: AssetStatus.ACTIVE, deviceAssetId: 'device-asset-id', @@ -122,9 +127,12 @@ export const assetStub = { deletedAt: null, duplicateId: null, isOffline: false, + libraryId: null, + stackId: null, + updateId: '42', }), - noThumbhash: Object.freeze({ + noThumbhash: Object.freeze({ id: 'asset-id', status: AssetStatus.ACTIVE, deviceAssetId: 'device-asset-id', @@ -156,6 +164,9 @@ export const assetStub = { deletedAt: null, duplicateId: null, isOffline: false, + libraryId: null, + stackId: null, + updateId: '42', }), primaryImage: Object.freeze({ @@ -195,12 +206,13 @@ export const assetStub = { } as Exif, stackId: 'stack-1', stack: stackStub('stack-1', [ - { id: 'primary-asset-id' } as AssetEntity, - { id: 'stack-child-asset-1' } as AssetEntity, - { id: 'stack-child-asset-2' } as AssetEntity, + { id: 'primary-asset-id' } as MapAsset & { exifInfo: Exif }, + { id: 'stack-child-asset-1' } as MapAsset & { exifInfo: Exif }, + { id: 'stack-child-asset-2' } as MapAsset & { exifInfo: Exif }, ]), duplicateId: null, isOffline: false, + updateId: '42', libraryId: null, }), @@ -229,6 +241,9 @@ export const assetStub = { isExternal: false, livePhotoVideo: null, livePhotoVideoId: null, + updateId: 'foo', + libraryId: null, + stackId: null, sharedLinks: [], originalFileName: 'asset-id.jpg', faces: [], @@ -241,10 +256,10 @@ export const assetStub = { } as Exif, duplicateId: null, isOffline: false, - libraryId: null, + stack: null, }), - trashed: Object.freeze({ + trashed: Object.freeze({ id: 'asset-id', deviceAssetId: 'device-asset-id', fileModifiedAt: new Date('2023-02-23T05:06:29.716Z'), @@ -281,9 +296,12 @@ export const assetStub = { duplicateId: null, isOffline: false, status: AssetStatus.TRASHED, + libraryId: null, + stackId: null, + updateId: '42', }), - trashedOffline: Object.freeze({ + trashedOffline: Object.freeze({ id: 'asset-id', status: AssetStatus.ACTIVE, deviceAssetId: 'device-asset-id', @@ -321,8 +339,10 @@ export const assetStub = { } as Exif, duplicateId: null, isOffline: true, + stackId: null, + updateId: '42', }), - archived: Object.freeze({ + archived: Object.freeze({ id: 'asset-id', status: AssetStatus.ACTIVE, deviceAssetId: 'device-asset-id', @@ -359,9 +379,12 @@ export const assetStub = { } as Exif, duplicateId: null, isOffline: false, + libraryId: null, + stackId: null, + updateId: '42', }), - external: Object.freeze({ + external: Object.freeze({ id: 'asset-id', status: AssetStatus.ACTIVE, deviceAssetId: 'device-asset-id', @@ -397,9 +420,12 @@ export const assetStub = { } as Exif, duplicateId: null, isOffline: false, + updateId: '42', + stackId: null, + stack: null, }), - image1: Object.freeze({ + image1: Object.freeze({ id: 'asset-id-1', status: AssetStatus.ACTIVE, deviceAssetId: 'device-asset-id', @@ -434,9 +460,13 @@ export const assetStub = { } as Exif, duplicateId: null, isOffline: false, + updateId: '42', + stackId: null, + libraryId: null, + stack: null, }), - imageFrom2015: Object.freeze({ + imageFrom2015: Object.freeze({ id: 'asset-id-1', status: AssetStatus.ACTIVE, deviceAssetId: 'device-asset-id', @@ -510,7 +540,9 @@ export const assetStub = { deletedAt: null, duplicateId: null, isOffline: false, + updateId: '42', libraryId: null, + stackId: null, }), livePhotoMotionAsset: Object.freeze({ @@ -527,7 +559,7 @@ export const assetStub = { timeZone: `America/New_York`, }, libraryId: null, - } as AssetEntity & { libraryId: string | null; files: AssetFile[]; exifInfo: Exif }), + } as MapAsset & { faces: AssetFace[]; files: AssetFile[]; exifInfo: Exif }), livePhotoStillAsset: Object.freeze({ id: 'live-photo-still-asset', @@ -544,7 +576,8 @@ export const assetStub = { timeZone: `America/New_York`, }, files, - } as AssetEntity & { libraryId: string | null }), + faces: [] as AssetFace[], + } as MapAsset & { faces: AssetFace[] }), livePhotoWithOriginalFileName: Object.freeze({ id: 'live-photo-still-asset', @@ -562,7 +595,8 @@ export const assetStub = { timeZone: `America/New_York`, }, libraryId: null, - } as AssetEntity & { libraryId: string | null }), + faces: [] as AssetFace[], + } as MapAsset & { faces: AssetFace[] }), withLocation: Object.freeze({ id: 'asset-with-favorite-id', @@ -590,6 +624,9 @@ export const assetStub = { isVisible: true, livePhotoVideo: null, livePhotoVideoId: null, + updateId: 'foo', + libraryId: null, + stackId: null, sharedLinks: [], originalFileName: 'asset-id.ext', faces: [], @@ -604,7 +641,7 @@ export const assetStub = { deletedAt: null, duplicateId: null, isOffline: false, - libraryId: null, + tags: [], }), sidecar: Object.freeze({ @@ -639,10 +676,12 @@ export const assetStub = { deletedAt: null, duplicateId: null, isOffline: false, + updateId: 'foo', libraryId: null, + stackId: null, }), - sidecarWithoutExt: Object.freeze({ + sidecarWithoutExt: Object.freeze({ id: 'asset-id', status: AssetStatus.ACTIVE, deviceAssetId: 'device-asset-id', @@ -676,7 +715,7 @@ export const assetStub = { isOffline: false, }), - hasEncodedVideo: Object.freeze({ + hasEncodedVideo: Object.freeze({ id: 'asset-id', status: AssetStatus.ACTIVE, originalFileName: 'asset-id.ext', @@ -711,9 +750,13 @@ export const assetStub = { deletedAt: null, duplicateId: null, isOffline: false, + updateId: '42', + libraryId: null, + stackId: null, + stack: null, }), - hasFileExtension: Object.freeze({ + hasFileExtension: Object.freeze({ id: 'asset-id', status: AssetStatus.ACTIVE, deviceAssetId: 'device-asset-id', @@ -788,6 +831,9 @@ export const assetStub = { } as Exif, duplicateId: null, isOffline: false, + updateId: '42', + libraryId: null, + stackId: null, }), imageHif: Object.freeze({ @@ -827,5 +873,8 @@ export const assetStub = { } as Exif, duplicateId: null, isOffline: false, + updateId: '42', + libraryId: null, + stackId: null, }), }; diff --git a/server/test/fixtures/auth.stub.ts b/server/test/fixtures/auth.stub.ts index dfa21fc707..9ef55398d3 100644 --- a/server/test/fixtures/auth.stub.ts +++ b/server/test/fixtures/auth.stub.ts @@ -1,6 +1,5 @@ import { Session } from 'src/database'; import { AuthDto } from 'src/dtos/auth.dto'; -import { SharedLinkEntity } from 'src/entities/shared-link.entity'; const authUser = { admin: { @@ -42,14 +41,16 @@ export const authStub = { id: 'token-id', } as Session, }), - adminSharedLink: Object.freeze({ + adminSharedLink: Object.freeze({ user: authUser.admin, sharedLink: { id: '123', showExif: true, allowDownload: true, allowUpload: true, - key: Buffer.from('shared-link-key'), - } as SharedLinkEntity, + expiresAt: null, + password: null, + userId: '42', + }, }), }; diff --git a/server/test/fixtures/shared-link.stub.ts b/server/test/fixtures/shared-link.stub.ts index 4eba0de845..a4d83863c7 100644 --- a/server/test/fixtures/shared-link.stub.ts +++ b/server/test/fixtures/shared-link.stub.ts @@ -1,10 +1,9 @@ import { UserAdmin } from 'src/database'; import { AlbumResponseDto } from 'src/dtos/album.dto'; -import { AssetResponseDto } from 'src/dtos/asset-response.dto'; +import { AssetResponseDto, MapAsset } from 'src/dtos/asset-response.dto'; import { ExifResponseDto } from 'src/dtos/exif.dto'; import { SharedLinkResponseDto } from 'src/dtos/shared-link.dto'; import { mapUser } from 'src/dtos/user.dto'; -import { SharedLinkEntity } from 'src/entities/shared-link.entity'; import { AssetOrder, AssetStatus, AssetType, SharedLinkType } from 'src/enum'; import { assetStub } from 'test/fixtures/asset.stub'; import { authStub } from 'test/fixtures/auth.stub'; @@ -113,12 +112,12 @@ export const sharedLinkStub = { allowUpload: true, allowDownload: true, showExif: true, - album: undefined, + albumId: null, + album: null, description: null, assets: [assetStub.image], password: 'password', - albumId: null, - } as SharedLinkEntity), + }), valid: Object.freeze({ id: '123', userId: authStub.admin.user.id, @@ -130,12 +129,12 @@ export const sharedLinkStub = { allowUpload: true, allowDownload: true, showExif: true, - album: undefined, albumId: null, description: null, password: null, - assets: [], - } as SharedLinkEntity), + assets: [] as MapAsset[], + album: null, + }), expired: Object.freeze({ id: '123', userId: authStub.admin.user.id, @@ -150,9 +149,10 @@ export const sharedLinkStub = { description: null, password: null, albumId: null, - assets: [], - } as SharedLinkEntity), - readonlyNoExif: Object.freeze({ + assets: [] as MapAsset[], + album: null, + }), + readonlyNoExif: Object.freeze({ id: '123', userId: authStub.admin.user.id, key: sharedLinkBytes, @@ -168,6 +168,7 @@ export const sharedLinkStub = { albumId: 'album-123', album: { id: 'album-123', + updateId: '42', ownerId: authStub.admin.user.id, owner: userStub.admin, albumName: 'Test Album', @@ -239,17 +240,22 @@ export const sharedLinkStub = { colorspace: 'sRGB', autoStackId: null, rating: 3, + updatedAt: today, + updateId: '42', }, sharedLinks: [], faces: [], sidecarPath: null, deletedAt: null, duplicateId: null, + updateId: '42', + libraryId: null, + stackId: null, }, ], }, }), - passwordRequired: Object.freeze({ + passwordRequired: Object.freeze({ id: '123', userId: authStub.admin.user.id, key: sharedLinkBytes, @@ -263,6 +269,7 @@ export const sharedLinkStub = { password: 'password', assets: [], albumId: null, + album: null, }), }; diff --git a/server/test/medium/globalSetup.ts b/server/test/medium/globalSetup.ts index 89b25da766..46eb1a733f 100644 --- a/server/test/medium/globalSetup.ts +++ b/server/test/medium/globalSetup.ts @@ -1,10 +1,11 @@ -import { FileMigrationProvider, Kysely, Migrator } from 'kysely'; -import { mkdir, readdir } from 'node:fs/promises'; -import { join } from 'node:path'; +import { Kysely } from 'kysely'; import { parse } from 'pg-connection-string'; +import { DB } from 'src/db'; +import { ConfigRepository } from 'src/repositories/config.repository'; +import { DatabaseRepository } from 'src/repositories/database.repository'; +import { LoggingRepository } from 'src/repositories/logging.repository'; import { getKyselyConfig } from 'src/utils/database'; import { GenericContainer, Wait } from 'testcontainers'; -import { DataSource } from 'typeorm'; const globalSetup = async () => { const postgresContainer = await new GenericContainer('tensorchord/pgvecto-rs:pg14-v0.2.0') @@ -36,66 +37,23 @@ const globalSetup = async () => { const postgresPort = postgresContainer.getMappedPort(5432); const postgresUrl = `postgres://postgres:postgres@localhost:${postgresPort}/immich`; + const parsed = parse(postgresUrl); + process.env.IMMICH_TEST_POSTGRES_URL = postgresUrl; - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-expect-error - const modules = import.meta.glob('/src/migrations/*.ts', { eager: true }); - - const config = { - type: 'postgres' as const, - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-expect-error - migrations: Object.values(modules).map((module) => Object.values(module)[0]), - migrationsRun: false, - synchronize: false, - connectTimeoutMS: 10_000, // 10 seconds - parseInt8: true, - url: postgresUrl, - }; - - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-expect-error - const dataSource = new DataSource(config); - await dataSource.initialize(); - await dataSource.runMigrations(); - await dataSource.destroy(); - - // for whatever reason, importing from test/utils causes vitest to crash - // eslint-disable-next-line unicorn/prefer-module - const migrationFolder = join(__dirname, '..', 'schema/migrations'); - // TODO remove after we have at least one kysely migration - await mkdir(migrationFolder, { recursive: true }); - - const parsed = parse(process.env.IMMICH_TEST_POSTGRES_URL!); - - const parsedOptions = { - ...parsed, - ssl: false, - host: parsed.host ?? undefined, - port: parsed.port ? Number(parsed.port) : undefined, - database: parsed.database ?? undefined, - }; - - const db = new Kysely(getKyselyConfig(parsedOptions)); - - // TODO just call `databaseRepository.migrate()` (probably have to wait until TypeOrm is gone) - const migrator = new Migrator({ - db, - migrationLockTableName: 'kysely_migrations_lock', - migrationTableName: 'kysely_migrations', - provider: new FileMigrationProvider({ - fs: { readdir }, - path: { join }, - migrationFolder, + const db = new Kysely( + getKyselyConfig({ + ...parsed, + ssl: false, + host: parsed.host ?? undefined, + port: parsed.port ? Number(parsed.port) : undefined, + database: parsed.database ?? undefined, }), - }); + ); - const { error } = await migrator.migrateToLatest(); - if (error) { - console.error('Unable to run kysely migrations', error); - throw error; - } + const configRepository = new ConfigRepository(); + const logger = new LoggingRepository(undefined, configRepository); + await new DatabaseRepository(db, logger, configRepository).runMigrations(); await db.destroy(); }; diff --git a/server/test/medium/specs/services/memory.service.spec.ts b/server/test/medium/specs/services/memory.service.spec.ts index 172c48ca5b..445434d60a 100644 --- a/server/test/medium/specs/services/memory.service.spec.ts +++ b/server/test/medium/specs/services/memory.service.spec.ts @@ -39,9 +39,12 @@ describe(MemoryService.name, () => { it('should create a memory from an asset', async () => { const { sut, repos, getRepository } = createSut(); - const now = DateTime.fromObject({ year: 2025, month: 2, day: 25 }, { zone: 'utc' }); + const now = DateTime.fromObject({ year: 2025, month: 2, day: 25 }, { zone: 'utc' }) as DateTime; const user = mediumFactory.userInsert(); - const asset = mediumFactory.assetInsert({ ownerId: user.id, localDateTime: now.minus({ years: 1 }).toISO() }); + const asset = mediumFactory.assetInsert({ + ownerId: user.id, + localDateTime: now.minus({ years: 1 }).toISO(), + }); const jobStatus = mediumFactory.assetJobStatusInsert({ assetId: asset.id }); const userRepo = getRepository('user'); @@ -86,7 +89,7 @@ describe(MemoryService.name, () => { it('should not generate a memory twice for the same day', async () => { const { sut, repos, getRepository } = createSut(); - const now = DateTime.fromObject({ year: 2025, month: 2, day: 20 }, { zone: 'utc' }); + const now = DateTime.fromObject({ year: 2025, month: 2, day: 20 }, { zone: 'utc' }) as DateTime; const assetRepo = getRepository('asset'); const memoryRepo = getRepository('memory'); diff --git a/server/test/medium/specs/services/metadata.service.spec.ts b/server/test/medium/specs/services/metadata.service.spec.ts index b25cce2724..13b9867373 100644 --- a/server/test/medium/specs/services/metadata.service.spec.ts +++ b/server/test/medium/specs/services/metadata.service.spec.ts @@ -118,7 +118,7 @@ describe(MetadataService.name, () => { process.env.TZ = serverTimeZone ?? undefined; const { filePath } = await createTestFile(exifData); - mocks.assetJob.getForMetadataExtraction.mockResolvedValue({ id: 'asset-1', originalPath: filePath } as never); + mocks.assetJob.getForMetadataExtraction.mockResolvedValue({ id: 'asset-1', originalPath: filePath } as any); await sut.handleMetadataExtraction({ id: 'asset-1' }); diff --git a/server/test/repositories/asset.repository.mock.ts b/server/test/repositories/asset.repository.mock.ts index 2418b6aa64..d540e55b2a 100644 --- a/server/test/repositories/asset.repository.mock.ts +++ b/server/test/repositories/asset.repository.mock.ts @@ -11,7 +11,7 @@ export const newAssetRepositoryMock = (): Mocked = {}) => { }; }; -const assetFactory = (asset: Partial = {}) => ({ +const assetFactory = (asset: Partial = {}) => ({ id: newUuid(), createdAt: newDate(), updatedAt: newDate(), diff --git a/server/test/sql-tools/column-index-name-default.ts b/server/test/sql-tools/column-index-name-default.ts index e8b36ec119..cedae006be 100644 --- a/server/test/sql-tools/column-index-name-default.ts +++ b/server/test/sql-tools/column-index-name-default.ts @@ -1,9 +1,8 @@ -import { Column, ColumnIndex, DatabaseSchema, Table } from 'src/sql-tools'; +import { Column, DatabaseSchema, Table } from 'src/sql-tools'; @Table() export class Table1 { - @ColumnIndex() - @Column() + @Column({ index: true }) column1!: string; } diff --git a/server/test/sql-tools/column-index-name.ts b/server/test/sql-tools/column-index-name.ts new file mode 100644 index 0000000000..8ba18a8851 --- /dev/null +++ b/server/test/sql-tools/column-index-name.ts @@ -0,0 +1,46 @@ +import { Column, DatabaseSchema, Table } from 'src/sql-tools'; + +@Table() +export class Table1 { + @Column({ indexName: 'IDX_test' }) + column1!: string; +} + +export const description = 'should create a column with an index if a name is provided'; +export const schema: DatabaseSchema = { + name: 'postgres', + schemaName: 'public', + functions: [], + enums: [], + extensions: [], + parameters: [], + tables: [ + { + name: 'table1', + columns: [ + { + name: 'column1', + tableName: 'table1', + type: 'character varying', + nullable: false, + isArray: false, + primary: false, + synchronize: true, + }, + ], + indexes: [ + { + name: 'IDX_test', + columnNames: ['column1'], + tableName: 'table1', + unique: false, + synchronize: true, + }, + ], + triggers: [], + constraints: [], + synchronize: true, + }, + ], + warnings: [], +}; diff --git a/server/test/sql-tools/foreign-key-inferred-type.stub.ts b/server/test/sql-tools/foreign-key-inferred-type.stub.ts index 2ecaafdcad..0b66a1acd4 100644 --- a/server/test/sql-tools/foreign-key-inferred-type.stub.ts +++ b/server/test/sql-tools/foreign-key-inferred-type.stub.ts @@ -60,7 +60,15 @@ export const schema: DatabaseSchema = { synchronize: true, }, ], - indexes: [], + indexes: [ + { + name: 'IDX_3fcca5cc563abf256fc346e3ff', + tableName: 'table2', + columnNames: ['parentId'], + unique: false, + synchronize: true, + }, + ], triggers: [], constraints: [ { diff --git a/server/test/sql-tools/foreign-key-with-unique-constraint.stub.ts b/server/test/sql-tools/foreign-key-with-unique-constraint.stub.ts index 0601a02d42..109a3dfc85 100644 --- a/server/test/sql-tools/foreign-key-with-unique-constraint.stub.ts +++ b/server/test/sql-tools/foreign-key-with-unique-constraint.stub.ts @@ -60,7 +60,15 @@ export const schema: DatabaseSchema = { synchronize: true, }, ], - indexes: [], + indexes: [ + { + name: 'IDX_3fcca5cc563abf256fc346e3ff', + tableName: 'table2', + columnNames: ['parentId'], + unique: false, + synchronize: true, + }, + ], triggers: [], constraints: [ { diff --git a/server/test/sql-tools/index-with-where.stub copy.ts b/server/test/sql-tools/index-with-expression.ts similarity index 100% rename from server/test/sql-tools/index-with-where.stub copy.ts rename to server/test/sql-tools/index-with-expression.ts diff --git a/web/src/lib/components/assets/thumbnail/thumbnail.svelte b/web/src/lib/components/assets/thumbnail/thumbnail.svelte index 802c04cf16..0d1142fbfb 100644 --- a/web/src/lib/components/assets/thumbnail/thumbnail.svelte +++ b/web/src/lib/components/assets/thumbnail/thumbnail.svelte @@ -131,18 +131,30 @@ }; let timer: ReturnType; - const clearLongPressTimer = () => clearTimeout(timer); + + const preventContextMenu = (evt: Event) => evt.preventDefault(); + let disposeables: (() => void)[] = []; + + const clearLongPressTimer = () => { + clearTimeout(timer); + for (const dispose of disposeables) { + dispose(); + } + disposeables = []; + }; let startX: number = 0; let startY: number = 0; function longPress(element: HTMLElement, { onLongPress }: { onLongPress: () => void }) { let didPress = false; - const start = (evt: TouchEvent) => { - startX = evt.changedTouches[0].clientX; - startY = evt.changedTouches[0].clientY; + const start = (evt: PointerEvent) => { + startX = evt.clientX; + startY = evt.clientY; didPress = false; timer = setTimeout(() => { onLongPress(); + element.addEventListener('contextmenu', preventContextMenu, { once: true }); + disposeables.push(() => element.removeEventListener('contextmenu', preventContextMenu)); didPress = true; }, 350); }; @@ -154,13 +166,13 @@ e.preventDefault(); }; element.addEventListener('click', click); - element.addEventListener('touchstart', start, true); - element.addEventListener('touchend', clearLongPressTimer, true); + element.addEventListener('pointerdown', start, true); + element.addEventListener('pointerup', clearLongPressTimer, true); return { destroy: () => { element.removeEventListener('click', click); - element.removeEventListener('touchstart', start, true); - element.removeEventListener('touchend', clearLongPressTimer, true); + element.removeEventListener('pointerdown', start, true); + element.removeEventListener('pointerup', clearLongPressTimer, true); }, }; }