From 0dbb0aabc943cc81e2e4a0afe99cfd4304e8d88e Mon Sep 17 00:00:00 2001 From: Jonathan Jogenfors Date: Fri, 11 Oct 2024 01:05:19 +0200 Subject: [PATCH] feat(server): xxhash --- server/Dockerfile | 1 + server/package-lock.json | 444 ++++++++++++++++++ server/package.json | 1 + server/src/cores/storage.core.ts | 5 +- server/src/entities/asset-files.entity.ts | 4 + server/src/enum.ts | 1 + server/src/interfaces/asset.interface.ts | 1 + server/src/interfaces/crypto.interface.ts | 1 + .../src/middleware/file-upload.interceptor.ts | 19 +- .../1728632095015-asset-file-checksum.ts | 15 + server/src/queries/asset.repository.sql | 16 +- server/src/repositories/asset.repository.ts | 2 +- server/src/repositories/crypto.repository.ts | 5 + server/src/services/asset-media.service.ts | 21 +- server/src/services/library.service.ts | 1 - 15 files changed, 523 insertions(+), 14 deletions(-) create mode 100644 server/src/migrations/1728632095015-asset-file-checksum.ts diff --git a/server/Dockerfile b/server/Dockerfile index 2eb1435664..e4a4d2ea61 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -23,6 +23,7 @@ RUN npm run build RUN npm prune --omit=dev --omit=optional COPY --from=dev /usr/src/app/node_modules/@img ./node_modules/@img COPY --from=dev /usr/src/app/node_modules/exiftool-vendored.pl ./node_modules/exiftool-vendored.pl +COPY --from=dev /usr/src/app/node_modules/@node-rs ./node_modules/@node-rs # web build FROM node:20.17.0-alpine3.20@sha256:2d07db07a2df6830718ae2a47db6fedce6745f5bcd174c398f2acdda90a11c03 AS web diff --git a/server/package-lock.json b/server/package-lock.json index 5e2c7fd662..4ffb7ce290 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -20,6 +20,7 @@ "@nestjs/swagger": "^7.1.8", "@nestjs/typeorm": "^10.0.0", "@nestjs/websockets": "^10.2.2", + "@node-rs/xxhash": "^1.7.4", "@opentelemetry/auto-instrumentations-node": "^0.50.0", "@opentelemetry/context-async-hooks": "^1.24.0", "@opentelemetry/exporter-prometheus": "^0.53.0", @@ -725,6 +726,17 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@emnapi/core": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.3.0.tgz", + "integrity": "sha512-9hRqVlhwqBqCoToZ3hFcNVqL+uyHV06Y47ax4UB8L6XgVRqYz7MFnfessojo6+5TK89pKwJnpophwjTMOeKI9Q==", + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.0.1", + "tslib": "^2.4.0" + } + }, "node_modules/@emnapi/runtime": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.2.0.tgz", @@ -734,6 +746,16 @@ "tslib": "^2.4.0" } }, + "node_modules/@emnapi/wasi-threads": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.0.1.tgz", + "integrity": "sha512-iIBu7mwkq4UQGeMEM8bLwNK962nXdhodeScX4slfQnRhEMMzvYivHhutCIk8uojvmASXXPC2WNEjwxFWk72Oqw==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.21.5", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz", @@ -1969,6 +1991,18 @@ "darwin" ] }, + "node_modules/@napi-rs/wasm-runtime": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-0.2.5.tgz", + "integrity": "sha512-kwUxR7J9WLutBbulqg1dfOrMTwhMdXLdcGUhcbCcGwnPLt3gz19uHVdwH1syKVDbE022ZS2vZxOWflFLS0YTjw==", + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "^1.1.0", + "@emnapi/runtime": "^1.1.0", + "@tybys/wasm-util": "^0.9.0" + } + }, "node_modules/@nestjs/bull-shared": { "version": "10.2.1", "resolved": "https://registry.npmjs.org/@nestjs/bull-shared/-/bull-shared-10.2.1.tgz", @@ -2515,6 +2549,259 @@ "node": ">= 10" } }, + "node_modules/@node-rs/xxhash": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/@node-rs/xxhash/-/xxhash-1.7.4.tgz", + "integrity": "sha512-NU1YQx1IUlehoHEH2j/SAyVALBAVgI2Btp9//GL816/6Wgd79nz0XxbYG88iFv43T3tRff3i9qE9drHHMTDMFw==", + "license": "MIT", + "engines": { + "node": ">= 12" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "optionalDependencies": { + "@node-rs/xxhash-android-arm-eabi": "1.7.4", + "@node-rs/xxhash-android-arm64": "1.7.4", + "@node-rs/xxhash-darwin-arm64": "1.7.4", + "@node-rs/xxhash-darwin-x64": "1.7.4", + "@node-rs/xxhash-freebsd-x64": "1.7.4", + "@node-rs/xxhash-linux-arm-gnueabihf": "1.7.4", + "@node-rs/xxhash-linux-arm64-gnu": "1.7.4", + "@node-rs/xxhash-linux-arm64-musl": "1.7.4", + "@node-rs/xxhash-linux-x64-gnu": "1.7.4", + "@node-rs/xxhash-linux-x64-musl": "1.7.4", + "@node-rs/xxhash-wasm32-wasi": "1.7.4", + "@node-rs/xxhash-win32-arm64-msvc": "1.7.4", + "@node-rs/xxhash-win32-ia32-msvc": "1.7.4", + "@node-rs/xxhash-win32-x64-msvc": "1.7.4" + } + }, + "node_modules/@node-rs/xxhash-android-arm-eabi": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/@node-rs/xxhash-android-arm-eabi/-/xxhash-android-arm-eabi-1.7.4.tgz", + "integrity": "sha512-+Zhpx5X3taDeEPUA20gkWDm2JAQusYyZNVhZLXr+rIgsSUhA8pgc5kJ1jn7NlKnn++ilCrJhHW9T8DRqVDRuZA==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 12" + } + }, + "node_modules/@node-rs/xxhash-android-arm64": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/@node-rs/xxhash-android-arm64/-/xxhash-android-arm64-1.7.4.tgz", + "integrity": "sha512-BZjCr0xfbzX4S1XsM3pxC+PFXKGo05YiQ74AE7+YnXNCJNvIBhDZ7wytYj/Lnx5/azjxKtWFlZzmTV7GwNZBbQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 12" + } + }, + "node_modules/@node-rs/xxhash-darwin-arm64": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/@node-rs/xxhash-darwin-arm64/-/xxhash-darwin-arm64-1.7.4.tgz", + "integrity": "sha512-v4KXiKLpHNe86mTlR/6ManCuDabCZ2c8NfkbXuFaXzqUxOt6eU34NFQqpIGo9+P6M+ncrwaEBS1EtNR0DcPQrA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12" + } + }, + "node_modules/@node-rs/xxhash-darwin-x64": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/@node-rs/xxhash-darwin-x64/-/xxhash-darwin-x64-1.7.4.tgz", + "integrity": "sha512-E8k4cJSo2lpbloGRU3g4MocezxNMXSGSrQa1yGiTH59Zb1TDZArWlsFIALD8pVyQnjn1JNtINLyjN11Ym92YFg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12" + } + }, + "node_modules/@node-rs/xxhash-freebsd-x64": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/@node-rs/xxhash-freebsd-x64/-/xxhash-freebsd-x64-1.7.4.tgz", + "integrity": "sha512-xlqDNdofr7fKFwMrOtIcDWhNMgnCZZ4/mnrxxMuzK0YU5onEp253wYOCdcENzFpnXednpJrsmVhtElBBh9buiw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 12" + } + }, + "node_modules/@node-rs/xxhash-linux-arm-gnueabihf": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/@node-rs/xxhash-linux-arm-gnueabihf/-/xxhash-linux-arm-gnueabihf-1.7.4.tgz", + "integrity": "sha512-4ZM/257gBX8dOICXlCj8xig/Oc22KOZTbmg24qOGeBXLCPLgHrT83XuSJRnCo73nJ66LdhdiKpnS+9ZDZP5XVQ==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12" + } + }, + "node_modules/@node-rs/xxhash-linux-arm64-gnu": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/@node-rs/xxhash-linux-arm64-gnu/-/xxhash-linux-arm64-gnu-1.7.4.tgz", + "integrity": "sha512-3YkhVyGWuIuOOzgr517uEHUfDFIbVRg2iBLVi/qTyyGdZlvziZAkiMU6FY7bXu3eTLpfUXHb9R53TNGkDDBM1Q==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12" + } + }, + "node_modules/@node-rs/xxhash-linux-arm64-musl": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/@node-rs/xxhash-linux-arm64-musl/-/xxhash-linux-arm64-musl-1.7.4.tgz", + "integrity": "sha512-0ULea3A/77VLQhBR70REhfYx9ghGl9LZrVeba2/ANzIL9j6F55tjN8c4nN8G/wJNp8kuw8FrJL7V7tOsbZ/tjQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12" + } + }, + "node_modules/@node-rs/xxhash-linux-x64-gnu": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/@node-rs/xxhash-linux-x64-gnu/-/xxhash-linux-x64-gnu-1.7.4.tgz", + "integrity": "sha512-6zrEQQAM30HYku/ET2xLoI1L4q6X1Si9wZtsmc3ZPPmrPCPXksqYKcXJe41M5End+HhBPSU7iIvnBXny8TTmrg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12" + } + }, + "node_modules/@node-rs/xxhash-linux-x64-musl": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/@node-rs/xxhash-linux-x64-musl/-/xxhash-linux-x64-musl-1.7.4.tgz", + "integrity": "sha512-Wa+BzU4uD262NvxQL5mdqmLNWY1yzkKDHItjsA79G73357MNCg6QP+TGhYwTiPYlN1eQa9h0APYNDbBRBXpowg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12" + } + }, + "node_modules/@node-rs/xxhash-wasm32-wasi": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/@node-rs/xxhash-wasm32-wasi/-/xxhash-wasm32-wasi-1.7.4.tgz", + "integrity": "sha512-aQdGSlSGkkBsSXpTnGT97CrmHjBrzagTcp39jAzKpGiTLkZalxHrK3KA3SSFot6OSOBhQLqUAfXvXkEcqPWhsg==", + "cpu": [ + "wasm32" + ], + "license": "MIT", + "optional": true, + "dependencies": { + "@napi-rs/wasm-runtime": "^0.2.3" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@node-rs/xxhash-win32-arm64-msvc": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/@node-rs/xxhash-win32-arm64-msvc/-/xxhash-win32-arm64-msvc-1.7.4.tgz", + "integrity": "sha512-15rssDOpcJPMqUKgb5BmEIQJSohYcX9aWaBVl+RjrRs9m+Pn/gxdWB+HgWy/pfq6zbXhtlnZ/s3BlRjKBKE/EA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12" + } + }, + "node_modules/@node-rs/xxhash-win32-ia32-msvc": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/@node-rs/xxhash-win32-ia32-msvc/-/xxhash-win32-ia32-msvc-1.7.4.tgz", + "integrity": "sha512-gX4hvPeIVxrEluqNCUnYh0h/3vC/VEpBxAYUo8mI33wSt6u6qiDssTe5P/1/06KCOpmASSEkSdmBaaEyJZOaSg==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12" + } + }, + "node_modules/@node-rs/xxhash-win32-x64-msvc": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/@node-rs/xxhash-win32-x64-msvc/-/xxhash-win32-x64-msvc-1.7.4.tgz", + "integrity": "sha512-qoOPEJBOq8FJiCWxU1werr4a5efG+tl6HB4L5KXSMhk28ayz0Wjkd3Ld03S4/24WcVia6Q7k4ZeRph0XBcVzVg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -5154,6 +5441,16 @@ "url": "https://opencollective.com/turf" } }, + "node_modules/@tybys/wasm-util": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.9.0.tgz", + "integrity": "sha512-6+7nlbMVX/PVDCwaIQ8nTOPveOcFLSt8GcXdx8hD0bt39uWxYT88uXzqTd4fTvqta7oeUJqudepapKNt2DYJFw==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@types/archiver": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/@types/archiver/-/archiver-6.0.2.tgz", @@ -15805,6 +16102,16 @@ } } }, + "@emnapi/core": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.3.0.tgz", + "integrity": "sha512-9hRqVlhwqBqCoToZ3hFcNVqL+uyHV06Y47ax4UB8L6XgVRqYz7MFnfessojo6+5TK89pKwJnpophwjTMOeKI9Q==", + "optional": true, + "requires": { + "@emnapi/wasi-threads": "1.0.1", + "tslib": "^2.4.0" + } + }, "@emnapi/runtime": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.2.0.tgz", @@ -15814,6 +16121,15 @@ "tslib": "^2.4.0" } }, + "@emnapi/wasi-threads": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.0.1.tgz", + "integrity": "sha512-iIBu7mwkq4UQGeMEM8bLwNK962nXdhodeScX4slfQnRhEMMzvYivHhutCIk8uojvmASXXPC2WNEjwxFWk72Oqw==", + "optional": true, + "requires": { + "tslib": "^2.4.0" + } + }, "@esbuild/aix-ppc64": { "version": "0.21.5", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz", @@ -16489,6 +16805,17 @@ "integrity": "sha512-9bfjwDxIDWmmOKusUcqdS4Rw+SETlp9Dy39Xui9BEGEk19dDwH0jhipwFzEff/pFg95NKymc6TOTbRKcWeRqyQ==", "optional": true }, + "@napi-rs/wasm-runtime": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-0.2.5.tgz", + "integrity": "sha512-kwUxR7J9WLutBbulqg1dfOrMTwhMdXLdcGUhcbCcGwnPLt3gz19uHVdwH1syKVDbE022ZS2vZxOWflFLS0YTjw==", + "optional": true, + "requires": { + "@emnapi/core": "^1.1.0", + "@emnapi/runtime": "^1.1.0", + "@tybys/wasm-util": "^0.9.0" + } + }, "@nestjs/bull-shared": { "version": "10.2.1", "resolved": "https://registry.npmjs.org/@nestjs/bull-shared/-/bull-shared-10.2.1.tgz", @@ -16788,6 +17115,114 @@ "integrity": "sha512-Q1/zm43RWynxrO7lW4ehciQVj+5ePBhOK+/K2P7pLFX3JaJ/IZVC69SHidrmZSOkqz7ECIOhhy7XhAFG4JYyHA==", "optional": true }, + "@node-rs/xxhash": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/@node-rs/xxhash/-/xxhash-1.7.4.tgz", + "integrity": "sha512-NU1YQx1IUlehoHEH2j/SAyVALBAVgI2Btp9//GL816/6Wgd79nz0XxbYG88iFv43T3tRff3i9qE9drHHMTDMFw==", + "requires": { + "@node-rs/xxhash-android-arm-eabi": "1.7.4", + "@node-rs/xxhash-android-arm64": "1.7.4", + "@node-rs/xxhash-darwin-arm64": "1.7.4", + "@node-rs/xxhash-darwin-x64": "1.7.4", + "@node-rs/xxhash-freebsd-x64": "1.7.4", + "@node-rs/xxhash-linux-arm-gnueabihf": "1.7.4", + "@node-rs/xxhash-linux-arm64-gnu": "1.7.4", + "@node-rs/xxhash-linux-arm64-musl": "1.7.4", + "@node-rs/xxhash-linux-x64-gnu": "1.7.4", + "@node-rs/xxhash-linux-x64-musl": "1.7.4", + "@node-rs/xxhash-wasm32-wasi": "1.7.4", + "@node-rs/xxhash-win32-arm64-msvc": "1.7.4", + "@node-rs/xxhash-win32-ia32-msvc": "1.7.4", + "@node-rs/xxhash-win32-x64-msvc": "1.7.4" + } + }, + "@node-rs/xxhash-android-arm-eabi": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/@node-rs/xxhash-android-arm-eabi/-/xxhash-android-arm-eabi-1.7.4.tgz", + "integrity": "sha512-+Zhpx5X3taDeEPUA20gkWDm2JAQusYyZNVhZLXr+rIgsSUhA8pgc5kJ1jn7NlKnn++ilCrJhHW9T8DRqVDRuZA==", + "optional": true + }, + "@node-rs/xxhash-android-arm64": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/@node-rs/xxhash-android-arm64/-/xxhash-android-arm64-1.7.4.tgz", + "integrity": "sha512-BZjCr0xfbzX4S1XsM3pxC+PFXKGo05YiQ74AE7+YnXNCJNvIBhDZ7wytYj/Lnx5/azjxKtWFlZzmTV7GwNZBbQ==", + "optional": true + }, + "@node-rs/xxhash-darwin-arm64": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/@node-rs/xxhash-darwin-arm64/-/xxhash-darwin-arm64-1.7.4.tgz", + "integrity": "sha512-v4KXiKLpHNe86mTlR/6ManCuDabCZ2c8NfkbXuFaXzqUxOt6eU34NFQqpIGo9+P6M+ncrwaEBS1EtNR0DcPQrA==", + "optional": true + }, + "@node-rs/xxhash-darwin-x64": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/@node-rs/xxhash-darwin-x64/-/xxhash-darwin-x64-1.7.4.tgz", + "integrity": "sha512-E8k4cJSo2lpbloGRU3g4MocezxNMXSGSrQa1yGiTH59Zb1TDZArWlsFIALD8pVyQnjn1JNtINLyjN11Ym92YFg==", + "optional": true + }, + "@node-rs/xxhash-freebsd-x64": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/@node-rs/xxhash-freebsd-x64/-/xxhash-freebsd-x64-1.7.4.tgz", + "integrity": "sha512-xlqDNdofr7fKFwMrOtIcDWhNMgnCZZ4/mnrxxMuzK0YU5onEp253wYOCdcENzFpnXednpJrsmVhtElBBh9buiw==", + "optional": true + }, + "@node-rs/xxhash-linux-arm-gnueabihf": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/@node-rs/xxhash-linux-arm-gnueabihf/-/xxhash-linux-arm-gnueabihf-1.7.4.tgz", + "integrity": "sha512-4ZM/257gBX8dOICXlCj8xig/Oc22KOZTbmg24qOGeBXLCPLgHrT83XuSJRnCo73nJ66LdhdiKpnS+9ZDZP5XVQ==", + "optional": true + }, + "@node-rs/xxhash-linux-arm64-gnu": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/@node-rs/xxhash-linux-arm64-gnu/-/xxhash-linux-arm64-gnu-1.7.4.tgz", + "integrity": "sha512-3YkhVyGWuIuOOzgr517uEHUfDFIbVRg2iBLVi/qTyyGdZlvziZAkiMU6FY7bXu3eTLpfUXHb9R53TNGkDDBM1Q==", + "optional": true + }, + "@node-rs/xxhash-linux-arm64-musl": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/@node-rs/xxhash-linux-arm64-musl/-/xxhash-linux-arm64-musl-1.7.4.tgz", + "integrity": "sha512-0ULea3A/77VLQhBR70REhfYx9ghGl9LZrVeba2/ANzIL9j6F55tjN8c4nN8G/wJNp8kuw8FrJL7V7tOsbZ/tjQ==", + "optional": true + }, + "@node-rs/xxhash-linux-x64-gnu": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/@node-rs/xxhash-linux-x64-gnu/-/xxhash-linux-x64-gnu-1.7.4.tgz", + "integrity": "sha512-6zrEQQAM30HYku/ET2xLoI1L4q6X1Si9wZtsmc3ZPPmrPCPXksqYKcXJe41M5End+HhBPSU7iIvnBXny8TTmrg==", + "optional": true + }, + "@node-rs/xxhash-linux-x64-musl": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/@node-rs/xxhash-linux-x64-musl/-/xxhash-linux-x64-musl-1.7.4.tgz", + "integrity": "sha512-Wa+BzU4uD262NvxQL5mdqmLNWY1yzkKDHItjsA79G73357MNCg6QP+TGhYwTiPYlN1eQa9h0APYNDbBRBXpowg==", + "optional": true + }, + "@node-rs/xxhash-wasm32-wasi": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/@node-rs/xxhash-wasm32-wasi/-/xxhash-wasm32-wasi-1.7.4.tgz", + "integrity": "sha512-aQdGSlSGkkBsSXpTnGT97CrmHjBrzagTcp39jAzKpGiTLkZalxHrK3KA3SSFot6OSOBhQLqUAfXvXkEcqPWhsg==", + "optional": true, + "requires": { + "@napi-rs/wasm-runtime": "^0.2.3" + } + }, + "@node-rs/xxhash-win32-arm64-msvc": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/@node-rs/xxhash-win32-arm64-msvc/-/xxhash-win32-arm64-msvc-1.7.4.tgz", + "integrity": "sha512-15rssDOpcJPMqUKgb5BmEIQJSohYcX9aWaBVl+RjrRs9m+Pn/gxdWB+HgWy/pfq6zbXhtlnZ/s3BlRjKBKE/EA==", + "optional": true + }, + "@node-rs/xxhash-win32-ia32-msvc": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/@node-rs/xxhash-win32-ia32-msvc/-/xxhash-win32-ia32-msvc-1.7.4.tgz", + "integrity": "sha512-gX4hvPeIVxrEluqNCUnYh0h/3vC/VEpBxAYUo8mI33wSt6u6qiDssTe5P/1/06KCOpmASSEkSdmBaaEyJZOaSg==", + "optional": true + }, + "@node-rs/xxhash-win32-x64-msvc": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/@node-rs/xxhash-win32-x64-msvc/-/xxhash-win32-x64-msvc-1.7.4.tgz", + "integrity": "sha512-qoOPEJBOq8FJiCWxU1werr4a5efG+tl6HB4L5KXSMhk28ayz0Wjkd3Ld03S4/24WcVia6Q7k4ZeRph0XBcVzVg==", + "optional": true + }, "@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -18485,6 +18920,15 @@ "tslib": "^2.6.2" } }, + "@tybys/wasm-util": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.9.0.tgz", + "integrity": "sha512-6+7nlbMVX/PVDCwaIQ8nTOPveOcFLSt8GcXdx8hD0bt39uWxYT88uXzqTd4fTvqta7oeUJqudepapKNt2DYJFw==", + "optional": true, + "requires": { + "tslib": "^2.4.0" + } + }, "@types/archiver": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/@types/archiver/-/archiver-6.0.2.tgz", diff --git a/server/package.json b/server/package.json index 78922609d9..0743a944e0 100644 --- a/server/package.json +++ b/server/package.json @@ -45,6 +45,7 @@ "@nestjs/swagger": "^7.1.8", "@nestjs/typeorm": "^10.0.0", "@nestjs/websockets": "^10.2.2", + "@node-rs/xxhash": "^1.7.4", "@opentelemetry/auto-instrumentations-node": "^0.50.0", "@opentelemetry/context-async-hooks": "^1.24.0", "@opentelemetry/exporter-prometheus": "^0.53.0", diff --git a/server/src/cores/storage.core.ts b/server/src/cores/storage.core.ts index 8e42cd1076..4fd7d3d0c6 100644 --- a/server/src/cores/storage.core.ts +++ b/server/src/cores/storage.core.ts @@ -282,7 +282,10 @@ export class StorageCore { private savePath(pathType: PathType, id: string, newPath: string) { switch (pathType) { case AssetPathType.ORIGINAL: { - return this.assetRepository.update({ id, originalPath: newPath }); + return Promise.all([ + this.assetRepository.upsertFile({ assetId: id, type: AssetFileType.ORIGINAL, path: newPath }), + this.assetRepository.update({ id, originalPath: newPath }), + ]); } case AssetPathType.PREVIEW: { return this.assetRepository.upsertFile({ assetId: id, type: AssetFileType.PREVIEW, path: newPath }); diff --git a/server/src/entities/asset-files.entity.ts b/server/src/entities/asset-files.entity.ts index a8a6ddfee1..1a7656f3ad 100644 --- a/server/src/entities/asset-files.entity.ts +++ b/server/src/entities/asset-files.entity.ts @@ -35,4 +35,8 @@ export class AssetFileEntity { @Column() path!: string; + + @Column({ type: 'bigint' }) + @Index() + checksum!: BigInt | null; } diff --git a/server/src/enum.ts b/server/src/enum.ts index 109e9a90b7..ca4571dc47 100644 --- a/server/src/enum.ts +++ b/server/src/enum.ts @@ -11,6 +11,7 @@ export enum AssetType { } export enum AssetFileType { + ORIGINAL = 'original', PREVIEW = 'preview', THUMBNAIL = 'thumbnail', } diff --git a/server/src/interfaces/asset.interface.ts b/server/src/interfaces/asset.interface.ts index 750a852094..3a3add51ab 100644 --- a/server/src/interfaces/asset.interface.ts +++ b/server/src/interfaces/asset.interface.ts @@ -145,6 +145,7 @@ export interface UpsertFileOptions { assetId: string; type: AssetFileType; path: string; + checksum?: BigInt; } export type AssetPathEntity = Pick; diff --git a/server/src/interfaces/crypto.interface.ts b/server/src/interfaces/crypto.interface.ts index c661695cf7..aa02f55864 100644 --- a/server/src/interfaces/crypto.interface.ts +++ b/server/src/interfaces/crypto.interface.ts @@ -5,6 +5,7 @@ export interface ICryptoRepository { randomUUID(): string; hashFile(filePath: string | Buffer): Promise; hashSha256(data: string): string; + xxHash(value: string): BigInt; verifySha256(data: string, encrypted: string, publicKey: string): boolean; hashSha1(data: string | Buffer): Buffer; hashBcrypt(data: string | Buffer, saltOrRounds: string | number): Promise; diff --git a/server/src/middleware/file-upload.interceptor.ts b/server/src/middleware/file-upload.interceptor.ts index 075a7f5046..777239c817 100644 --- a/server/src/middleware/file-upload.interceptor.ts +++ b/server/src/middleware/file-upload.interceptor.ts @@ -2,6 +2,7 @@ import { CallHandler, ExecutionContext, Inject, Injectable, NestInterceptor } fr import { PATH_METADATA } from '@nestjs/common/constants'; import { Reflector } from '@nestjs/core'; import { transformException } from '@nestjs/platform-express/multer/multer/multer.utils'; +import { xxh3 } from '@node-rs/xxhash'; import { NextFunction, RequestHandler } from 'express'; import multer, { StorageEngine, diskStorage } from 'multer'; import { createHash, randomUUID } from 'node:crypto'; @@ -33,12 +34,14 @@ export interface ImmichFile extends Express.Multer.File { /** sha1 hash of file */ uuid: string; checksum: Buffer; + xxhash: BigInt; } export function mapToUploadFile(file: ImmichFile): UploadFile { return { uuid: file.uuid, checksum: file.checksum, + xxhash: file.xxhash, originalPath: file.path, originalName: Buffer.from(file.originalname, 'latin1').toString('utf8'), size: file.size, @@ -146,14 +149,22 @@ export class FileUploadInterceptor implements NestInterceptor { return; } - const hash = createHash('sha1'); - file.stream.on('data', (chunk) => hash.update(chunk)); + this.logger.debug(`Handling asset upload file: ${file.originalname}`); + const xxhash = new xxh3.Xxh3(); + const sha1hash = createHash('sha1'); + + file.stream.on('data', (chunk) => { + xxhash.update(chunk); + sha1hash.update(chunk); + }); + this.defaultStorage._handleFile(request, file, (error, info) => { if (error) { - hash.destroy(); + sha1hash.destroy(); + xxhash.reset(); callback(error); } else { - callback(null, { ...info, checksum: hash.digest() }); + callback(null, { ...info, checksum: sha1hash.digest(), xxhash: xxhash.digest() }); } }); } diff --git a/server/src/migrations/1728632095015-asset-file-checksum.ts b/server/src/migrations/1728632095015-asset-file-checksum.ts new file mode 100644 index 0000000000..004f153e69 --- /dev/null +++ b/server/src/migrations/1728632095015-asset-file-checksum.ts @@ -0,0 +1,15 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AssetFileChecksum1728632095015 implements MigrationInterface { + name = 'AssetFileChecksum1728632095015'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "asset_files" ADD "checksum" bigint`); + await queryRunner.query(`CREATE INDEX "IDX_c946066edd16cfa5c25a26aa8e" ON "asset_files" ("checksum") `); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX "public"."IDX_c946066edd16cfa5c25a26aa8e"`); + await queryRunner.query(`ALTER TABLE "asset_files" DROP COLUMN "checksum"`); + } +} diff --git a/server/src/queries/asset.repository.sql b/server/src/queries/asset.repository.sql index eda91482bb..ca5acf5de4 100644 --- a/server/src/queries/asset.repository.sql +++ b/server/src/queries/asset.repository.sql @@ -64,7 +64,8 @@ SELECT "files"."createdAt" AS "files_createdAt", "files"."updatedAt" AS "files_updatedAt", "files"."type" AS "files_type", - "files"."path" AS "files_path" + "files"."path" AS "files_path", + "files"."checksum" AS "files_checksum" FROM "assets" "entity" LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "entity"."id" @@ -248,7 +249,8 @@ SELECT "AssetEntity__AssetEntity_files"."createdAt" AS "AssetEntity__AssetEntity_files_createdAt", "AssetEntity__AssetEntity_files"."updatedAt" AS "AssetEntity__AssetEntity_files_updatedAt", "AssetEntity__AssetEntity_files"."type" AS "AssetEntity__AssetEntity_files_type", - "AssetEntity__AssetEntity_files"."path" AS "AssetEntity__AssetEntity_files_path" + "AssetEntity__AssetEntity_files"."path" AS "AssetEntity__AssetEntity_files_path", + "AssetEntity__AssetEntity_files"."checksum" AS "AssetEntity__AssetEntity_files_checksum" FROM "assets" "AssetEntity" LEFT JOIN "exif" "AssetEntity__AssetEntity_exifInfo" ON "AssetEntity__AssetEntity_exifInfo"."assetId" = "AssetEntity"."id" @@ -1117,10 +1119,11 @@ INSERT INTO "createdAt", "updatedAt", "type", - "path" + "path", + "checksum" ) VALUES - (DEFAULT, $1, DEFAULT, DEFAULT, $2, $3) + (DEFAULT, $1, DEFAULT, DEFAULT, $2, $3, DEFAULT) ON CONFLICT ("assetId", "type") DO UPDATE SET @@ -1141,10 +1144,11 @@ INSERT INTO "createdAt", "updatedAt", "type", - "path" + "path", + "checksum" ) VALUES - (DEFAULT, $1, DEFAULT, DEFAULT, $2, $3) + (DEFAULT, $1, DEFAULT, DEFAULT, $2, $3, DEFAULT) ON CONFLICT ("assetId", "type") DO UPDATE SET diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts index 8bca755c32..a7cbc68a62 100644 --- a/server/src/repositories/asset.repository.ts +++ b/server/src/repositories/asset.repository.ts @@ -801,7 +801,7 @@ export class AssetRepository implements IAssetRepository { } @GenerateSql({ params: [{ assetId: DummyValue.UUID, type: AssetFileType.PREVIEW, path: '/path/to/file' }] }) - async upsertFile(file: { assetId: string; type: AssetFileType; path: string }): Promise { + async upsertFile(file: { assetId: string; type: AssetFileType; path: string; checksum?: BigInt }): Promise { await this.fileRepository.upsert(file, { conflictPaths: ['assetId', 'type'] }); } diff --git a/server/src/repositories/crypto.repository.ts b/server/src/repositories/crypto.repository.ts index 72e75ef174..4684851244 100644 --- a/server/src/repositories/crypto.repository.ts +++ b/server/src/repositories/crypto.repository.ts @@ -1,4 +1,5 @@ import { Injectable } from '@nestjs/common'; +import { xxh3 } from '@node-rs/xxhash'; import { compareSync, hash } from 'bcrypt'; import { createHash, createPublicKey, createVerify, randomBytes, randomUUID } from 'node:crypto'; import { createReadStream } from 'node:fs'; @@ -28,6 +29,10 @@ export class CryptoRepository implements ICryptoRepository { return createHash('sha256').update(value).digest('base64'); } + xxHash(value: string) { + return xxh3.Xxh3.withSeed().update(value).digest(); + } + verifySha256(value: string, encryptedValue: string, publicKey: string) { const publicKeyBuffer = Buffer.from(publicKey, 'base64'); const cryptoPublicKey = createPublicKey({ diff --git a/server/src/services/asset-media.service.ts b/server/src/services/asset-media.service.ts index b320c32a21..3c19e62ea7 100644 --- a/server/src/services/asset-media.service.ts +++ b/server/src/services/asset-media.service.ts @@ -21,7 +21,7 @@ import { } from 'src/dtos/asset-media.dto'; import { AuthDto } from 'src/dtos/auth.dto'; import { ASSET_CHECKSUM_CONSTRAINT, AssetEntity } from 'src/entities/asset.entity'; -import { AssetStatus, AssetType, CacheControl, Permission, StorageFolder } from 'src/enum'; +import { AssetFileType, AssetStatus, AssetType, CacheControl, Permission, StorageFolder } from 'src/enum'; import { JobName } from 'src/interfaces/job.interface'; import { BaseService } from 'src/services/base.service'; import { requireAccess, requireUploadAccess } from 'src/utils/access'; @@ -39,6 +39,7 @@ export interface UploadRequest { export interface UploadFile { uuid: string; checksum: Buffer; + xxhash: BigInt; originalPath: string; originalName: string; size: number; @@ -334,6 +335,15 @@ export class AssetMediaService extends BaseService { sidecarPath: sidecarPath || null, }); + await this.assetRepository.upsertFile({ + assetId, + type: AssetFileType.ORIGINAL, + path: file.originalPath, + checksum: file.xxhash, + }); + + console.log('xxhash', file.xxhash); + await this.storageRepository.utimes(file.originalPath, new Date(), new Date(dto.fileModifiedAt)); await this.assetRepository.upsertExif({ assetId, fileSizeInByte: file.size }); await this.jobRepository.queue({ @@ -364,6 +374,8 @@ export class AssetMediaService extends BaseService { sidecarPath: asset.sidecarPath, }); + // TODO: asset file original + const { size } = await this.storageRepository.stat(created.originalPath); await this.assetRepository.upsertExif({ assetId: created.id, fileSizeInByte: size }); await this.jobRepository.queue({ name: JobName.METADATA_EXTRACTION, data: { id: created.id, source: 'copy' } }); @@ -400,6 +412,13 @@ export class AssetMediaService extends BaseService { sidecarPath: sidecarFile?.originalPath, }); + await this.assetRepository.upsertFile({ + assetId: asset.id, + type: AssetFileType.ORIGINAL, + path: asset.originalPath, + checksum: file.xxhash, + }); + if (sidecarFile) { await this.storageRepository.utimes(sidecarFile.originalPath, new Date(), new Date(dto.fileModifiedAt)); } diff --git a/server/src/services/library.service.ts b/server/src/services/library.service.ts index 84a8a277f5..8ebe58d00c 100644 --- a/server/src/services/library.service.ts +++ b/server/src/services/library.service.ts @@ -417,7 +417,6 @@ export class LibraryService extends BaseService { localDateTime: mtime, type: assetType, originalFileName: parse(assetPath).base, - sidecarPath, isExternal: true, });