diff --git a/server/src/database.ts b/server/src/database.ts index f4878b2cee..4f339624e6 100644 --- a/server/src/database.ts +++ b/server/src/database.ts @@ -5,6 +5,7 @@ import { AssetFileType, AssetType, AssetVisibility, + ChecksumAlgorithm, MemoryType, Permission, PluginContext, @@ -112,6 +113,7 @@ export type Memory = { export type Asset = { id: string; checksum: Buffer; + checksumAlgorithm: ChecksumAlgorithm; deviceAssetId: string; deviceId: string; fileCreatedAt: Date; @@ -330,6 +332,7 @@ export const columns = { asset: [ 'asset.id', 'asset.checksum', + 'asset.checksumAlgorithm', 'asset.deviceAssetId', 'asset.deviceId', 'asset.fileCreatedAt', diff --git a/server/src/dtos/asset-response.dto.ts b/server/src/dtos/asset-response.dto.ts index 8b38b2e124..2c2f57bbb2 100644 --- a/server/src/dtos/asset-response.dto.ts +++ b/server/src/dtos/asset-response.dto.ts @@ -13,7 +13,7 @@ import { } from 'src/dtos/person.dto'; import { TagResponseDto, mapTag } from 'src/dtos/tag.dto'; import { UserResponseDto, mapUser } from 'src/dtos/user.dto'; -import { AssetStatus, AssetType, AssetVisibility } from 'src/enum'; +import { AssetStatus, AssetType, AssetVisibility, ChecksumAlgorithm } from 'src/enum'; import { ImageDimensions, MaybeDehydrated } from 'src/types'; import { getDimensions } from 'src/utils/asset.util'; import { hexOrBufferToBase64 } from 'src/utils/bytes'; @@ -148,6 +148,7 @@ export type MapAsset = { updateId: string; status: AssetStatus; checksum: Buffer; + checksumAlgorithm: ChecksumAlgorithm; deviceAssetId: string; deviceId: string; duplicateId: string | null; diff --git a/server/src/enum.ts b/server/src/enum.ts index 60f45efd6e..1277a39036 100644 --- a/server/src/enum.ts +++ b/server/src/enum.ts @@ -37,6 +37,11 @@ export enum AssetType { Other = 'OTHER', } +export enum ChecksumAlgorithm { + sha1File = 'sha1', // sha1 checksum of the whole file contents + sha1Path = 'sha1-path', // sha1 checksum of "path:" plus the file path, currently used in external libraries, deprecated +} + export enum AssetFileType { /** * An full/large-size image extracted/converted from RAW photos diff --git a/server/src/queries/asset.job.repository.sql b/server/src/queries/asset.job.repository.sql index cf5b8f02dc..554aa3b195 100644 --- a/server/src/queries/asset.job.repository.sql +++ b/server/src/queries/asset.job.repository.sql @@ -249,6 +249,7 @@ where select "asset"."id", "asset"."checksum", + "asset"."checksumAlgorithm", "asset"."deviceAssetId", "asset"."deviceId", "asset"."fileCreatedAt", diff --git a/server/src/schema/enums.ts b/server/src/schema/enums.ts index c68f152779..f63a09c462 100644 --- a/server/src/schema/enums.ts +++ b/server/src/schema/enums.ts @@ -1,5 +1,5 @@ import { registerEnum } from '@immich/sql-tools'; -import { AssetStatus, AssetVisibility, SourceType } from 'src/enum'; +import { AssetStatus, AssetVisibility, ChecksumAlgorithm, SourceType } from 'src/enum'; export const assets_status_enum = registerEnum({ name: 'assets_status_enum', @@ -15,3 +15,8 @@ export const asset_visibility_enum = registerEnum({ name: 'asset_visibility_enum', values: Object.values(AssetVisibility), }); + +export const asset_checksum_algorithm_enum = registerEnum({ + name: 'asset_checksum_algorithm_enum', + values: Object.values(ChecksumAlgorithm), +}); diff --git a/server/src/schema/migrations/1774548649115-AddChecksumAlgorithm.ts.ts b/server/src/schema/migrations/1774548649115-AddChecksumAlgorithm.ts.ts new file mode 100644 index 0000000000..477e7d0e48 --- /dev/null +++ b/server/src/schema/migrations/1774548649115-AddChecksumAlgorithm.ts.ts @@ -0,0 +1,21 @@ +import { Kysely, sql } from 'kysely'; + +export async function up(db: Kysely): Promise { + await sql`CREATE TYPE "asset_checksum_algorithm_enum" AS ENUM ('sha1','sha1-path');`.execute(db); + await sql`ALTER TABLE "asset" ADD "checksumAlgorithm" asset_checksum_algorithm_enum;`.execute(db); + + await sql` + UPDATE "asset" + SET "checksumAlgorithm" = CASE + WHEN "isExternal" = true THEN 'sha1-path'::asset_checksum_algorithm_enum + ELSE 'sha1'::asset_checksum_algorithm_enum + END + `.execute(db); + + await sql`ALTER TABLE "asset" ALTER COLUMN "checksumAlgorithm" SET NOT NULL;`.execute(db); +} + +export async function down(db: Kysely): Promise { + await sql`ALTER TABLE "asset" DROP COLUMN "checksumAlgorithm";`.execute(db); + await sql`DROP TYPE "asset_checksum_algorithm_enum";`.execute(db); +} diff --git a/server/src/schema/tables/asset.table.ts b/server/src/schema/tables/asset.table.ts index 8bdaa59bc6..7418dab102 100644 --- a/server/src/schema/tables/asset.table.ts +++ b/server/src/schema/tables/asset.table.ts @@ -12,8 +12,8 @@ import { UpdateDateColumn, } from '@immich/sql-tools'; import { UpdatedAtTrigger, UpdateIdColumn } from 'src/decorators'; -import { AssetStatus, AssetType, AssetVisibility } from 'src/enum'; -import { asset_visibility_enum, assets_status_enum } from 'src/schema/enums'; +import { AssetStatus, AssetType, AssetVisibility, ChecksumAlgorithm } from 'src/enum'; +import { asset_checksum_algorithm_enum, asset_visibility_enum, assets_status_enum } from 'src/schema/enums'; import { asset_delete_audit } from 'src/schema/functions'; import { LibraryTable } from 'src/schema/tables/library.table'; import { StackTable } from 'src/schema/tables/stack.table'; @@ -95,6 +95,9 @@ export class AssetTable { @Column({ type: 'bytea', index: true }) checksum!: Buffer; // sha1 checksum + @Column({ enum: asset_checksum_algorithm_enum }) + checksumAlgorithm!: ChecksumAlgorithm; + @ForeignKeyColumn(() => AssetTable, { nullable: true, onUpdate: 'CASCADE', onDelete: 'SET NULL' }) livePhotoVideoId!: string | null; diff --git a/server/src/services/asset-media.service.ts b/server/src/services/asset-media.service.ts index 03677054ec..8b8efd19c5 100644 --- a/server/src/services/asset-media.service.ts +++ b/server/src/services/asset-media.service.ts @@ -27,6 +27,7 @@ import { AssetStatus, AssetVisibility, CacheControl, + ChecksumAlgorithm, JobName, Permission, StorageFolder, @@ -425,6 +426,7 @@ export class AssetMediaService extends BaseService { deviceId: asset.deviceId, type: asset.type, checksum: asset.checksum, + checksumAlgorithm: asset.checksumAlgorithm, fileCreatedAt: asset.fileCreatedAt, localDateTime: asset.localDateTime, fileModifiedAt: asset.fileModifiedAt, @@ -446,6 +448,7 @@ export class AssetMediaService extends BaseService { libraryId: null, checksum: file.checksum, + checksumAlgorithm: ChecksumAlgorithm.sha1File, originalPath: file.originalPath, deviceAssetId: dto.deviceAssetId, diff --git a/server/src/services/library.service.ts b/server/src/services/library.service.ts index 841fa4743c..9f2d69bab5 100644 --- a/server/src/services/library.service.ts +++ b/server/src/services/library.service.ts @@ -17,7 +17,17 @@ import { ValidateLibraryImportPathResponseDto, ValidateLibraryResponseDto, } from 'src/dtos/library.dto'; -import { AssetStatus, AssetType, CronJob, DatabaseLock, ImmichWorker, JobName, JobStatus, QueueName } from 'src/enum'; +import { + AssetStatus, + AssetType, + ChecksumAlgorithm, + CronJob, + DatabaseLock, + ImmichWorker, + JobName, + JobStatus, + QueueName, +} from 'src/enum'; import { ArgOf } from 'src/repositories/event.repository'; import { AssetSyncResult } from 'src/repositories/library.repository'; import { AssetTable } from 'src/schema/tables/asset.table'; @@ -400,6 +410,7 @@ export class LibraryService extends BaseService { ownerId, libraryId, checksum: this.cryptoRepository.hashSha1(`path:${assetPath}`), + checksumAlgorithm: ChecksumAlgorithm.sha1Path, originalPath: assetPath, fileCreatedAt: stat.mtime, diff --git a/server/src/services/metadata.service.spec.ts b/server/src/services/metadata.service.spec.ts index cb35e21d0a..f9d17079e3 100644 --- a/server/src/services/metadata.service.spec.ts +++ b/server/src/services/metadata.service.spec.ts @@ -7,6 +7,7 @@ import { AssetFileType, AssetType, AssetVisibility, + ChecksumAlgorithm, ExifOrientation, ImmichWorker, JobName, @@ -652,6 +653,7 @@ describe(MetadataService.name, () => { expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(asset.id); expect(mocks.asset.create).toHaveBeenCalledWith({ checksum: expect.any(Buffer), + checksumAlgorithm: ChecksumAlgorithm.sha1File, deviceAssetId: 'NONE', deviceId: 'NONE', fileCreatedAt: asset.fileCreatedAt, @@ -705,6 +707,7 @@ describe(MetadataService.name, () => { expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(asset.id); expect(mocks.asset.create).toHaveBeenCalledWith({ checksum: expect.any(Buffer), + checksumAlgorithm: ChecksumAlgorithm.sha1File, deviceAssetId: 'NONE', deviceId: 'NONE', fileCreatedAt: asset.fileCreatedAt, @@ -758,6 +761,7 @@ describe(MetadataService.name, () => { expect(mocks.storage.readFile).toHaveBeenCalledWith(asset.originalPath, expect.any(Object)); expect(mocks.asset.create).toHaveBeenCalledWith({ checksum: expect.any(Buffer), + checksumAlgorithm: ChecksumAlgorithm.sha1File, deviceAssetId: 'NONE', deviceId: 'NONE', fileCreatedAt: asset.fileCreatedAt, diff --git a/server/src/services/metadata.service.ts b/server/src/services/metadata.service.ts index bfbcb413c0..c2cf66ad57 100644 --- a/server/src/services/metadata.service.ts +++ b/server/src/services/metadata.service.ts @@ -14,6 +14,7 @@ import { AssetFileType, AssetType, AssetVisibility, + ChecksumAlgorithm, DatabaseLock, ExifOrientation, ImmichWorker, @@ -675,6 +676,7 @@ export class MetadataService extends BaseService { fileModifiedAt: stats.mtime, localDateTime: dates.localDateTime, checksum, + checksumAlgorithm: ChecksumAlgorithm.sha1File, ownerId: asset.ownerId, originalPath: StorageCore.getAndroidMotionPath(asset, motionAssetId), originalFileName: `${parse(asset.originalFileName).name}.mp4`, diff --git a/server/test/factories/asset.factory.ts b/server/test/factories/asset.factory.ts index 321a6f8ddd..9caec31d6f 100644 --- a/server/test/factories/asset.factory.ts +++ b/server/test/factories/asset.factory.ts @@ -1,5 +1,5 @@ import { Selectable } from 'kysely'; -import { AssetFileType, AssetStatus, AssetType, AssetVisibility } from 'src/enum'; +import { AssetFileType, AssetStatus, AssetType, AssetVisibility, ChecksumAlgorithm } from 'src/enum'; import { AssetTable } from 'src/schema/tables/asset.table'; import { StackTable } from 'src/schema/tables/stack.table'; import { AssetEditFactory } from 'test/factories/asset-edit.factory'; @@ -53,6 +53,7 @@ export class AssetFactory { updateId: newUuidV7(), status: AssetStatus.Active, checksum: newSha1(), + checksumAlgorithm: ChecksumAlgorithm.sha1File, deviceAssetId: '', deviceId: '', duplicateId: null, diff --git a/server/test/mappers.ts b/server/test/mappers.ts index 2f3b248576..3d5b34e9c0 100644 --- a/server/test/mappers.ts +++ b/server/test/mappers.ts @@ -125,6 +125,7 @@ export const getForMemory = (memory: ReturnType) => ({ export const getForMetadataExtraction = (asset: ReturnType) => ({ id: asset.id, checksum: asset.checksum, + checksumAlgorithm: asset.checksumAlgorithm, deviceAssetId: asset.deviceAssetId, deviceId: asset.deviceId, fileCreatedAt: asset.fileCreatedAt, diff --git a/server/test/medium.factory.ts b/server/test/medium.factory.ts index a8aa00c2a3..1906fc0ccb 100644 --- a/server/test/medium.factory.ts +++ b/server/test/medium.factory.ts @@ -12,6 +12,7 @@ import { AlbumUserRole, AssetType, AssetVisibility, + ChecksumAlgorithm, MemoryType, SourceType, SyncEntityType, @@ -547,6 +548,7 @@ const assetInsert = (asset: Partial> = {}) => { deviceId: '', originalFileName: '', checksum: randomBytes(32), + checksumAlgorithm: ChecksumAlgorithm.sha1File, type: AssetType.Image, originalPath: '/path/to/something.jpg', ownerId: 'not-a-valid-uuid',