diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 79126fd658..9563be6125 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -5,7 +5,8 @@ "immich-server", "redis", "database", - "immich-machine-learning" + "immich-machine-learning", + "init" ], "dockerComposeFile": [ "../docker/docker-compose.dev.yml", diff --git a/.devcontainer/server/container-compose-overrides.yml b/.devcontainer/server/container-compose-overrides.yml index 869b6c2624..59975baad3 100644 --- a/.devcontainer/server/container-compose-overrides.yml +++ b/.devcontainer/server/container-compose-overrides.yml @@ -11,6 +11,18 @@ services: - ${UPLOAD_LOCATION:-upload1-devcontainer-volume}${UPLOAD_LOCATION:+/photos}:/data - ${UPLOAD_LOCATION:-upload2-devcontainer-volume}${UPLOAD_LOCATION:+/photos/upload}:/data/upload - /etc/localtime:/etc/localtime:ro + - pnpm-store:/usr/src/app/.pnpm-store + - server-node-modules:/usr/src/app/server/node_modules + - server-dist:/usr/src/app/server/dist + - web-node_modules:/usr/src/app/web/node_modules + - github-node_modules:/usr/src/app/.github/node_modules + - cli-node_modules:/usr/src/app/cli/node_modules + - docs-node_modules:/usr/src/app/docs/node_modules + - e2e-node_modules:/usr/src/app/e2e/node_modules + - sdk-node_modules:/usr/src/app/open-api/typescript-sdk/node_modules + - app-node_modules:/usr/src/app/node_modules + - sveltekit:/usr/src/app/web/.svelte-kit + - coverage:/usr/src/app/web/coverage immich-web: env_file: !reset [] immich-machine-learning: diff --git a/.pnpmfile.cjs b/.pnpmfile.cjs index feda217821..0e76dabe66 100644 --- a/.pnpmfile.cjs +++ b/.pnpmfile.cjs @@ -4,34 +4,13 @@ module.exports = { if (!pkg.name) { return pkg; } - switch (pkg.name) { - case "exiftool-vendored": - if (pkg.optionalDependencies["exiftool-vendored.pl"]) { - // make exiftool-vendored.pl a regular dependency - pkg.dependencies["exiftool-vendored.pl"] = - pkg.optionalDependencies["exiftool-vendored.pl"]; - delete pkg.optionalDependencies["exiftool-vendored.pl"]; - } - break; - case "sharp": - const optionalDeps = Object.keys(pkg.optionalDependencies).filter( - (dep) => dep.startsWith("@img") - ); - for (const dep of optionalDeps) { - // remove all optionalDependencies from sharp (they will be compiled from source), except: - // include the precompiled musl version of sharp, for web - // include precompiled linux-x64 version of sharp, for server (stage: web-prod) - // include precompiled linux-arm64 version of sharp, for server (stage: web-prod) - if ( - dep.includes("musl") || - dep.includes("linux-x64") || - dep.includes("linux-arm64") - ) { - continue; - } - delete pkg.optionalDependencies[dep]; - } - break; + if (pkg.name === "exiftool-vendored") { + if (pkg.optionalDependencies["exiftool-vendored.pl"]) { + // make exiftool-vendored.pl a regular dependency + pkg.dependencies["exiftool-vendored.pl"] = + pkg.optionalDependencies["exiftool-vendored.pl"]; + delete pkg.optionalDependencies["exiftool-vendored.pl"]; + } } return pkg; }, diff --git a/Makefile b/Makefile index a9faceadf4..31a00ee6be 100644 --- a/Makefile +++ b/Makefile @@ -1,29 +1,29 @@ -dev: +dev: prepare-volumes @trap 'make dev-down' EXIT; COMPOSE_BAKE=true docker compose -f ./docker/docker-compose.dev.yml up --remove-orphans dev-down: docker compose -f ./docker/docker-compose.dev.yml down --remove-orphans -dev-update: +dev-update: prepare-volumes @trap 'make dev-down' EXIT; COMPOSE_BAKE=true docker compose -f ./docker/docker-compose.dev.yml up --build -V --remove-orphans -dev-scale: +dev-scale: prepare-volumes @trap 'make dev-down' EXIT; COMPOSE_BAKE=true docker compose -f ./docker/docker-compose.dev.yml up --build -V --scale immich-server=3 --remove-orphans -dev-docs: +dev-docs: prepare-volumes npm --prefix docs run start .PHONY: e2e -e2e: - @trap 'make e2e-down' EXIT; COMPOSE_BAKE=true docker compose -f ./e2e/docker-compose.yml up --build -V --remove-orphans +e2e: prepare-volumes + @trap 'make e2e-down' EXIT; COMPOSE_BAKE=true docker compose -f ./e2e/docker-compose.yml up --remove-orphans -e2e-update: +e2e-update: prepare-volumes @trap 'make e2e-down' EXIT; COMPOSE_BAKE=true docker compose -f ./e2e/docker-compose.yml up --build -V --remove-orphans e2e-down: docker compose -f ./e2e/docker-compose.yml down --remove-orphans -prod: +prod: @trap 'make prod-down' EXIT; COMPOSE_BAKE=true docker compose -f ./docker/docker-compose.prod.yml up --build -V --remove-orphans prod-down: @@ -33,16 +33,16 @@ prod-scale: @trap 'make prod-down' EXIT; COMPOSE_BAKE=true docker compose -f ./docker/docker-compose.prod.yml up --build -V --scale immich-server=3 --scale immich-microservices=3 --remove-orphans .PHONY: open-api -open-api: +open-api: prepare-volumes cd ./open-api && bash ./bin/generate-open-api.sh -open-api-dart: +open-api-dart: prepare-volumes cd ./open-api && bash ./bin/generate-open-api.sh dart -open-api-typescript: +open-api-typescript: prepare-volumes cd ./open-api && bash ./bin/generate-open-api.sh typescript -sql: +sql: prepare-volumes pnpm --filter immich run sync:sql attach-server: @@ -51,6 +51,30 @@ attach-server: renovate: LOG_LEVEL=debug npx renovate --platform=local --repository-cache=reset +# Directories that need to be created for volumes or build output +VOLUME_DIRS = \ + ./.pnpm-store \ + ./web/.svelte-kit \ + ./web/node_modules \ + ./web/coverage \ + ./e2e/node_modules \ + ./docs/node_modules \ + ./server/node_modules \ + ./server/dist \ + ./open-api/typescript-sdk/node_modules \ + ./.github/node_modules \ + ./node_modules \ + ./cli/node_modules + +# create empty directories and chown to current user +prepare-volumes: + @for dir in $(VOLUME_DIRS); do \ + mkdir -p $$dir; \ + done + @if [ -n "$(VOLUME_DIRS)" ]; then \ + chown -R $$(id -u):$$(id -g) $(VOLUME_DIRS); \ + fi + MODULES = e2e server web cli sdk docs .github # directory to package name mapping function diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index c6393fd132..fe621c768f 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -32,6 +32,18 @@ services: - ${UPLOAD_LOCATION}/photos:/data - ${UPLOAD_LOCATION}/photos/upload:/data/upload - /etc/localtime:/etc/localtime:ro + - pnpm-store:/usr/src/app/.pnpm-store + - server-node-modules:/usr/src/app/server/node_modules + - server-dist:/usr/src/app/server/dist + - web-node_modules:/usr/src/app/web/node_modules + - github-node_modules:/usr/src/app/.github/node_modules + - cli-node_modules:/usr/src/app/cli/node_modules + - docs-node_modules:/usr/src/app/docs/node_modules + - e2e-node_modules:/usr/src/app/e2e/node_modules + - sdk-node_modules:/usr/src/app/open-api/typescript-sdk/node_modules + - app-node_modules:/usr/src/app/node_modules + - sveltekit:/usr/src/app/web/.svelte-kit + - coverage:/usr/src/app/web/coverage env_file: - .env environment: @@ -84,6 +96,18 @@ services: - 24678:24678 volumes: - ..:/usr/src/app + - pnpm-store:/usr/src/app/.pnpm-store + - server-node-modules:/usr/src/app/server/node_modules + - server-dist:/usr/src/app/server/dist + - web-node_modules:/usr/src/app/web/node_modules + - github-node_modules:/usr/src/app/.github/node_modules + - cli-node_modules:/usr/src/app/cli/node_modules + - docs-node_modules:/usr/src/app/docs/node_modules + - e2e-node_modules:/usr/src/app/e2e/node_modules + - sdk-node_modules:/usr/src/app/open-api/typescript-sdk/node_modules + - app-node_modules:/usr/src/app/node_modules + - sveltekit:/usr/src/app/web/.svelte-kit + - coverage:/usr/src/app/web/coverage ulimits: nofile: soft: 1048576 @@ -167,9 +191,33 @@ services: env_file: - .env user: 0:0 - command: sh -c 'for path in /usr/src/app/.pnpm-store /usr/src/app/server/node_modules /usr/src/app/.github/node_modules /usr/src/app/cli/node_modules /usr/src/app/docs/node_modules /usr/src/app/e2e/node_modules /usr/src/app/open-api/typescript-sdk/node_modules /usr/src/app/web/.svelte-kit /usr/src/app/web/coverage /usr/src/app/node_modules /usr/src/app/web/node_modules; do [ -e "$$path" ] && chown -R ${UID:-1000}:${GID:-1000} "$$path" || true; done' - + command: sh -c 'for path in /usr/src/app/.pnpm-store /usr/src/app/server/node_modules /usr/src/app/server/dist /usr/src/app/.github/node_modules /usr/src/app/cli/node_modules /usr/src/app/docs/node_modules /usr/src/app/e2e/node_modules /usr/src/app/open-api/typescript-sdk/node_modules /usr/src/app/web/.svelte-kit /usr/src/app/web/coverage /usr/src/app/node_modules /usr/src/app/web/node_modules; do [ -e "$$path" ] && chown -R ${UID:-1000}:${GID:-1000} "$$path" || true; done' + volumes: + - pnpm-store:/usr/src/app/.pnpm-store + - server-node-modules:/usr/src/app/server/node_modules + - server-dist:/usr/src/app/server/dist + - web-node_modules:/usr/src/app/web/node_modules + - github-node_modules:/usr/src/app/.github/node_modules + - cli-node_modules:/usr/src/app/cli/node_modules + - docs-node_modules:/usr/src/app/docs/node_modules + - e2e-node_modules:/usr/src/app/e2e/node_modules + - sdk-node_modules:/usr/src/app/open-api/typescript-sdk/node_modules + - app-node_modules:/usr/src/app/node_modules + - sveltekit:/usr/src/app/web/.svelte-kit + - coverage:/usr/src/app/web/coverage volumes: model-cache: prometheus-data: grafana-data: + pnpm-store: + server-node-modules: + server-dist: + web-node_modules: + github-node_modules: + cli-node_modules: + docs-node_modules: + e2e-node_modules: + sdk-node_modules: + app-node_modules: + sveltekit: + coverage: diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 103cacfdc0..fa24e5b31f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,7 +11,7 @@ overrides: packageExtensionsChecksum: sha256-DAYr0FTkvKYnvBH4muAER9UE1FVGKhqfRU4/QwA2xPQ= -pnpmfileChecksum: sha256-7GOLcTtuczNumtarIG1mbRinBOSpiOOVzgbeV3Xp4X4= +pnpmfileChecksum: sha256-AG/qwrPNpmy9q60PZwCpecoYVptglTHgH+N6RKQHOM0= importers: @@ -2124,6 +2124,9 @@ packages: resolution: {integrity: sha512-P1ml0nvOmEFdmu0smSXOqTS1sxU5tqvnc0dA4MTKV39kye+bhQnjkIKEE18fNOvxjyB86k8esoCIFM3x4RykOQ==} engines: {node: '>=18.0'} + '@emnapi/runtime@1.4.5': + resolution: {integrity: sha512-++LApOtY0pEEz1zrd9vy1/zXVaVJJ/EbAF3u0fXIzPJEDtnITsBGbbK0EkM72amhl/R5b+5xx0Y/QhcVOpuulg==} + '@esbuild/aix-ppc64@0.19.12': resolution: {integrity: sha512-bmoCYyWdEL3wDQIVbcyzRyeKLgk2WtWLTWz1ZIAZF/EGbNOwSA6ew3PftJ1PqMiOOGu0OyFMzG53L0zqIpPeNA==} engines: {node: '>=12'} @@ -2553,11 +2556,48 @@ packages: resolution: {integrity: sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==} engines: {node: '>=18.18'} + '@img/sharp-darwin-arm64@0.34.2': + resolution: {integrity: sha512-OfXHZPppddivUJnqyKoi5YVeHRkkNE2zUFT2gbpKxp/JZCFYEYubnMg+gOp6lWfasPrTS+KPosKqdI+ELYVDtg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [darwin] + + '@img/sharp-darwin-x64@0.34.2': + resolution: {integrity: sha512-dYvWqmjU9VxqXmjEtjmvHnGqF8GrVjM2Epj9rJ6BUIXvk8slvNDJbhGFvIoXzkDhrJC2jUxNLz/GUjjvSzfw+g==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [darwin] + + '@img/sharp-libvips-darwin-arm64@1.1.0': + resolution: {integrity: sha512-HZ/JUmPwrJSoM4DIQPv/BfNh9yrOA8tlBbqbLz4JZ5uew2+o22Ik+tHQJcih7QJuSa0zo5coHTfD5J8inqj9DA==} + cpu: [arm64] + os: [darwin] + + '@img/sharp-libvips-darwin-x64@1.1.0': + resolution: {integrity: sha512-Xzc2ToEmHN+hfvsl9wja0RlnXEgpKNmftriQp6XzY/RaSfwD9th+MSh0WQKzUreLKKINb3afirxW7A0fz2YWuQ==} + cpu: [x64] + os: [darwin] + '@img/sharp-libvips-linux-arm64@1.1.0': resolution: {integrity: sha512-IVfGJa7gjChDET1dK9SekxFFdflarnUB8PwW8aGwEoF3oAsSDuNUTYS+SKDOyOJxQyDC1aPFMuRYLoDInyV9Ew==} cpu: [arm64] os: [linux] + '@img/sharp-libvips-linux-arm@1.1.0': + resolution: {integrity: sha512-s8BAd0lwUIvYCJyRdFqvsj+BJIpDBSxs6ivrOPm/R7piTs5UIwY5OjXrP2bqXC9/moGsyRa37eYWYCOGVXxVrA==} + cpu: [arm] + os: [linux] + + '@img/sharp-libvips-linux-ppc64@1.1.0': + resolution: {integrity: sha512-tiXxFZFbhnkWE2LA8oQj7KYR+bWBkiV2nilRldT7bqoEZ4HiDOcePr9wVDAZPi/Id5fT1oY9iGnDq20cwUz8lQ==} + cpu: [ppc64] + os: [linux] + + '@img/sharp-libvips-linux-s390x@1.1.0': + resolution: {integrity: sha512-xukSwvhguw7COyzvmjydRb3x/09+21HykyapcZchiCUkTThEQEOMtBj9UhkaBRLuBrgLFzQ2wbxdeCCJW/jgJA==} + cpu: [s390x] + os: [linux] + '@img/sharp-libvips-linux-x64@1.1.0': resolution: {integrity: sha512-yRj2+reB8iMg9W5sULM3S74jVS7zqSzHG3Ol/twnAAkAhnGQnpjj6e4ayUz7V+FpKypwgs82xbRdYtchTTUB+Q==} cpu: [x64] @@ -2579,6 +2619,18 @@ packages: cpu: [arm64] os: [linux] + '@img/sharp-linux-arm@0.34.2': + resolution: {integrity: sha512-0DZzkvuEOqQUP9mo2kjjKNok5AmnOr1jB2XYjkaoNRwpAYMDzRmAqUIa1nRi58S2WswqSfPOWLNOr0FDT3H5RQ==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm] + os: [linux] + + '@img/sharp-linux-s390x@0.34.2': + resolution: {integrity: sha512-EGZ1xwhBI7dNISwxjChqBGELCWMGDvmxZXKjQRuqMrakhO8QoMgqCrdjnAqJq/CScxfRn+Bb7suXBElKQpPDiw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [s390x] + os: [linux] + '@img/sharp-linux-x64@0.34.2': resolution: {integrity: sha512-sD7J+h5nFLMMmOXYH4DD9UtSNBD05tWSSdWAcEyzqW8Cn5UxXvsHAxmxSesYUsTOBmUnjtxghKDl15EvfqLFbQ==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} @@ -2597,6 +2649,29 @@ packages: cpu: [x64] os: [linux] + '@img/sharp-wasm32@0.34.2': + resolution: {integrity: sha512-/VI4mdlJ9zkaq53MbIG6rZY+QRN3MLbR6usYlgITEzi4Rpx5S6LFKsycOQjkOGmqTNmkIdLjEvooFKwww6OpdQ==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [wasm32] + + '@img/sharp-win32-arm64@0.34.2': + resolution: {integrity: sha512-cfP/r9FdS63VA5k0xiqaNaEoGxBg9k7uE+RQGzuK9fHt7jib4zAVVseR9LsE4gJcNWgT6APKMNnCcnyOtmSEUQ==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [win32] + + '@img/sharp-win32-ia32@0.34.2': + resolution: {integrity: sha512-QLjGGvAbj0X/FXl8n1WbtQ6iVBpWU7JO94u/P2M4a8CFYsvQi4GW2mRy/JqkRx0qpBzaOdKJKw8uc930EX2AHw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [ia32] + os: [win32] + + '@img/sharp-win32-x64@0.34.2': + resolution: {integrity: sha512-aUdT6zEYtDKCaxkofmmJDJYGCf0+pJg3eU9/oBuqvEeoB9dKI6ZLc/1iLJCTuJQDO4ptntAlkUmHgGjyuobZbw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [win32] + '@immich/ui@0.24.1': resolution: {integrity: sha512-phJ9BHV0+OnKsxXD+5+Te5Amnb1N4ExYpRGSJPYFqutd5WXeN7kZGKZXd3CfcQ1e31SXRy4DsHSGdM1pY7AUgA==} peerDependencies: @@ -13888,6 +13963,11 @@ snapshots: - uglify-js - webpack-cli + '@emnapi/runtime@1.4.5': + dependencies: + tslib: 2.8.1 + optional: true + '@esbuild/aix-ppc64@0.19.12': optional: true @@ -14184,9 +14264,34 @@ snapshots: '@humanwhocodes/retry@0.4.3': {} + '@img/sharp-darwin-arm64@0.34.2': + optionalDependencies: + '@img/sharp-libvips-darwin-arm64': 1.1.0 + optional: true + + '@img/sharp-darwin-x64@0.34.2': + optionalDependencies: + '@img/sharp-libvips-darwin-x64': 1.1.0 + optional: true + + '@img/sharp-libvips-darwin-arm64@1.1.0': + optional: true + + '@img/sharp-libvips-darwin-x64@1.1.0': + optional: true + '@img/sharp-libvips-linux-arm64@1.1.0': optional: true + '@img/sharp-libvips-linux-arm@1.1.0': + optional: true + + '@img/sharp-libvips-linux-ppc64@1.1.0': + optional: true + + '@img/sharp-libvips-linux-s390x@1.1.0': + optional: true + '@img/sharp-libvips-linux-x64@1.1.0': optional: true @@ -14201,6 +14306,16 @@ snapshots: '@img/sharp-libvips-linux-arm64': 1.1.0 optional: true + '@img/sharp-linux-arm@0.34.2': + optionalDependencies: + '@img/sharp-libvips-linux-arm': 1.1.0 + optional: true + + '@img/sharp-linux-s390x@0.34.2': + optionalDependencies: + '@img/sharp-libvips-linux-s390x': 1.1.0 + optional: true + '@img/sharp-linux-x64@0.34.2': optionalDependencies: '@img/sharp-libvips-linux-x64': 1.1.0 @@ -14216,6 +14331,20 @@ snapshots: '@img/sharp-libvips-linuxmusl-x64': 1.1.0 optional: true + '@img/sharp-wasm32@0.34.2': + dependencies: + '@emnapi/runtime': 1.4.5 + optional: true + + '@img/sharp-win32-arm64@0.34.2': + optional: true + + '@img/sharp-win32-ia32@0.34.2': + optional: true + + '@img/sharp-win32-x64@0.34.2': + optional: true + '@immich/ui@0.24.1(@internationalized/date@3.8.2)(svelte@5.35.5)': dependencies: '@mdi/js': 7.4.47 @@ -23758,14 +23887,27 @@ snapshots: node-gyp: 11.2.0 semver: 7.7.2 optionalDependencies: + '@img/sharp-darwin-arm64': 0.34.2 + '@img/sharp-darwin-x64': 0.34.2 + '@img/sharp-libvips-darwin-arm64': 1.1.0 + '@img/sharp-libvips-darwin-x64': 1.1.0 + '@img/sharp-libvips-linux-arm': 1.1.0 '@img/sharp-libvips-linux-arm64': 1.1.0 + '@img/sharp-libvips-linux-ppc64': 1.1.0 + '@img/sharp-libvips-linux-s390x': 1.1.0 '@img/sharp-libvips-linux-x64': 1.1.0 '@img/sharp-libvips-linuxmusl-arm64': 1.1.0 '@img/sharp-libvips-linuxmusl-x64': 1.1.0 + '@img/sharp-linux-arm': 0.34.2 '@img/sharp-linux-arm64': 0.34.2 + '@img/sharp-linux-s390x': 0.34.2 '@img/sharp-linux-x64': 0.34.2 '@img/sharp-linuxmusl-arm64': 0.34.2 '@img/sharp-linuxmusl-x64': 0.34.2 + '@img/sharp-wasm32': 0.34.2 + '@img/sharp-win32-arm64': 0.34.2 + '@img/sharp-win32-ia32': 0.34.2 + '@img/sharp-win32-x64': 0.34.2 transitivePeerDependencies: - supports-color diff --git a/server/Dockerfile b/server/Dockerfile index ae3198c5d3..bd5c55402e 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -91,9 +91,8 @@ FROM prod-builder-base AS server-prod WORKDIR /usr/src/app COPY ./package* ./pnpm* .pnpmfile.cjs ./ COPY ./server ./server/ -# SHARP_IGNORE_GLOBAL_LIBVIPS because 'deploy' will always build sharp bindings from source RUN SHARP_IGNORE_GLOBAL_LIBVIPS=true pnpm --filter immich --frozen-lockfile build && \ - pnpm --filter immich --frozen-lockfile --prod --no-optional deploy /output/server-pruned + SHARP_FORCE_GLOBAL_LIBVIPS=true pnpm --filter immich --frozen-lockfile --prod --no-optional deploy /output/server-pruned # web production build FROM prod-builder-base AS web-prod diff --git a/server/nest-cli.json b/server/nest-cli.json index 1eaf1888d5..16a8b8a09b 100644 --- a/server/nest-cli.json +++ b/server/nest-cli.json @@ -3,7 +3,7 @@ "collection": "@nestjs/schematics", "sourceRoot": "src", "compilerOptions": { - "deleteOutDir": true, + "deleteOutDir": false, "webpack": false, "plugins": [ {