diff --git a/server/src/bin/migrations.ts b/server/src/bin/migrations.ts index 222ad96858..5fdb925cab 100644 --- a/server/src/bin/migrations.ts +++ b/server/src/bin/migrations.ts @@ -93,7 +93,7 @@ const create = (path: string, up: string[], down: string[]) => { const filename = `${timestamp}-${name}.ts`; const folder = dirname(path); const fullPath = join(folder, filename); - writeFileSync(fullPath, asMigration('kysely', { name, timestamp, up, down })); + writeFileSync(fullPath, asMigration('typeorm', { name, timestamp, up, down })); console.log(`Wrote ${fullPath}`); }; diff --git a/server/src/entities/asset.entity.ts b/server/src/entities/asset.entity.ts index c285961e64..55e795b6de 100644 --- a/server/src/entities/asset.entity.ts +++ b/server/src/entities/asset.entity.ts @@ -313,8 +313,5 @@ export function searchAssetBuilder(kysely: Kysely, options: AssetSearchBuild ) .$if(!!options.withExif, withExifInner) .$if(!!(options.withFaces || options.withPeople || options.personIds), (qb) => qb.select(withFacesAndPeople)) - .$if(!options.withDeleted, (qb) => qb.where('assets.deletedAt', 'is', null)) - .where('assets.fileCreatedAt', 'is not', null) - .where('assets.fileModifiedAt', 'is not', null) - .where('assets.localDateTime', 'is not', null); + .$if(!options.withDeleted, (qb) => qb.where('assets.deletedAt', 'is', null)); } diff --git a/server/src/migrations/1744662638410-MakeFileMetadataNonNullable.ts b/server/src/migrations/1744662638410-MakeFileMetadataNonNullable.ts new file mode 100644 index 0000000000..1ba4df01cd --- /dev/null +++ b/server/src/migrations/1744662638410-MakeFileMetadataNonNullable.ts @@ -0,0 +1,22 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class MakeFileMetadataNonNullable1744662638410 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `DELETE FROM assets WHERE "fileCreatedAt" IS NULL OR "fileModifiedAt" IS NULL OR "localDateTime" IS NULL`, + ); + await queryRunner.query(` + ALTER TABLE assets + ALTER COLUMN "fileCreatedAt" SET NOT NULL, + ALTER COLUMN "fileModifiedAt" SET NOT NULL, + ALTER COLUMN "localDateTime" SET NOT NULL`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + ALTER TABLE assets + ALTER COLUMN "fileCreatedAt" DROP NOT NULL, + ALTER COLUMN "fileModifiedAt" DROP NOT NULL, + ALTER COLUMN "localDateTime" DROP NOT NULL`); + } +} diff --git a/server/src/queries/activity.repository.sql b/server/src/queries/activity.repository.sql index 3d4d667de6..c6e4c60a19 100644 --- a/server/src/queries/activity.repository.sql +++ b/server/src/queries/activity.repository.sql @@ -71,6 +71,3 @@ where and "activity"."albumId" = $2 and "activity"."isLiked" = $3 and "assets"."deletedAt" is null - and "assets"."fileCreatedAt" is not null - and "assets"."fileModifiedAt" is not null - and "assets"."localDateTime" is not null diff --git a/server/src/queries/asset.repository.sql b/server/src/queries/asset.repository.sql index a72e6ee8cf..b59cee5864 100644 --- a/server/src/queries/asset.repository.sql +++ b/server/src/queries/asset.repository.sql @@ -171,9 +171,6 @@ where "ownerId" = $1::uuid and "deviceId" = $2 and "isVisible" = $3 - and "assets"."fileCreatedAt" is not null - and "assets"."fileModifiedAt" is not null - and "assets"."localDateTime" is not null and "deletedAt" is null -- AssetRepository.getLivePhotoCount @@ -334,9 +331,6 @@ with where "assets"."deletedAt" is null and "assets"."isVisible" = $2 - and "assets"."fileCreatedAt" is not null - and "assets"."fileModifiedAt" is not null - and "assets"."localDateTime" is not null ) select "timeBucket", @@ -490,9 +484,6 @@ from where "assets"."ownerId" = $1::uuid and "assets"."isVisible" = $2 - and "assets"."fileCreatedAt" is not null - and "assets"."fileModifiedAt" is not null - and "assets"."localDateTime" is not null and "assets"."updatedAt" <= $3 and "assets"."id" > $4 order by @@ -523,9 +514,6 @@ from where "assets"."ownerId" = any ($1::uuid[]) and "assets"."isVisible" = $2 - and "assets"."fileCreatedAt" is not null - and "assets"."fileModifiedAt" is not null - and "assets"."localDateTime" is not null and "assets"."updatedAt" > $3 limit $4 diff --git a/server/src/queries/search.repository.sql b/server/src/queries/search.repository.sql index 73f276a7fb..4fce272365 100644 --- a/server/src/queries/search.repository.sql +++ b/server/src/queries/search.repository.sql @@ -13,9 +13,6 @@ where and "assets"."isFavorite" = $4 and "assets"."isArchived" = $5 and "assets"."deletedAt" is null - and "assets"."fileCreatedAt" is not null - and "assets"."fileModifiedAt" is not null - and "assets"."localDateTime" is not null order by "assets"."fileCreatedAt" desc limit @@ -37,9 +34,6 @@ offset and "assets"."isFavorite" = $4 and "assets"."isArchived" = $5 and "assets"."deletedAt" is null - and "assets"."fileCreatedAt" is not null - and "assets"."fileModifiedAt" is not null - and "assets"."localDateTime" is not null and "assets"."id" < $6 order by random() @@ -60,9 +54,6 @@ union all and "assets"."isFavorite" = $11 and "assets"."isArchived" = $12 and "assets"."deletedAt" is null - and "assets"."fileCreatedAt" is not null - and "assets"."fileModifiedAt" is not null - and "assets"."localDateTime" is not null and "assets"."id" > $13 order by random() @@ -86,9 +77,6 @@ where and "assets"."isFavorite" = $4 and "assets"."isArchived" = $5 and "assets"."deletedAt" is null - and "assets"."fileCreatedAt" is not null - and "assets"."fileModifiedAt" is not null - and "assets"."localDateTime" is not null order by smart_search.embedding <=> $6 limit diff --git a/server/src/repositories/activity.repository.ts b/server/src/repositories/activity.repository.ts index 18128fb087..d030a99f4f 100644 --- a/server/src/repositories/activity.repository.ts +++ b/server/src/repositories/activity.repository.ts @@ -76,9 +76,6 @@ export class ActivityRepository { .where('activity.albumId', '=', albumId) .where('activity.isLiked', '=', false) .where('assets.deletedAt', 'is', null) - .where('assets.fileCreatedAt', 'is not', null) - .where('assets.fileModifiedAt', 'is not', null) - .where('assets.localDateTime', 'is not', null) .executeTakeFirstOrThrow(); return count; diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts index cdca1fa9c8..8b31f47fd1 100644 --- a/server/src/repositories/asset.repository.ts +++ b/server/src/repositories/asset.repository.ts @@ -457,9 +457,6 @@ export class AssetRepository { .where('ownerId', '=', asUuid(ownerId)) .where('deviceId', '=', deviceId) .where('isVisible', '=', true) - .where('assets.fileCreatedAt', 'is not', null) - .where('assets.fileModifiedAt', 'is not', null) - .where('assets.localDateTime', 'is not', null) .where('deletedAt', 'is', null) .execute(); @@ -674,8 +671,7 @@ export class AssetRepository { 'exif.timeZone', 'exif.fileSizeInByte', ]) - .where('assets.deletedAt', 'is', null) - .where('assets.fileCreatedAt', 'is not', null); + .where('assets.deletedAt', 'is', null); } getStorageTemplateAsset(id: string): Promise { @@ -712,10 +708,7 @@ export class AssetRepository { .where('job_status.duplicatesDetectedAt', 'is', null) .where('job_status.previewAt', 'is not', null) .where((eb) => eb.exists(eb.selectFrom('smart_search').where('assetId', '=', eb.ref('assets.id')))) - .where('assets.isVisible', '=', true) - .where('assets.fileCreatedAt', 'is not', null) - .where('assets.fileModifiedAt', 'is not', null) - .where('assets.localDateTime', 'is not', null), + .where('assets.isVisible', '=', true), ) .$if(property === WithoutProperty.ENCODED_VIDEO, (qb) => qb @@ -778,9 +771,6 @@ export class AssetRepository { .select((eb) => eb.fn.countAll().filterWhere('type', '=', AssetType.VIDEO).as(AssetType.VIDEO)) .select((eb) => eb.fn.countAll().filterWhere('type', '=', AssetType.OTHER).as(AssetType.OTHER)) .where('ownerId', '=', asUuid(ownerId)) - .where('assets.fileCreatedAt', 'is not', null) - .where('assets.fileModifiedAt', 'is not', null) - .where('assets.localDateTime', 'is not', null) .where('isVisible', '=', true) .$if(isArchived !== undefined, (qb) => qb.where('isArchived', '=', isArchived!)) .$if(isFavorite !== undefined, (qb) => qb.where('isFavorite', '=', isFavorite!)) @@ -813,9 +803,6 @@ export class AssetRepository { .$if(!!options.isTrashed, (qb) => qb.where('assets.status', '!=', AssetStatus.DELETED)) .where('assets.deletedAt', options.isTrashed ? 'is not' : 'is', null) .where('assets.isVisible', '=', true) - .where('assets.fileCreatedAt', 'is not', null) - .where('assets.fileModifiedAt', 'is not', null) - .where('assets.localDateTime', 'is not', null) .$if(!!options.albumId, (qb) => qb .innerJoin('albums_assets_assets', 'assets.id', 'albums_assets_assets.assetsId') @@ -1009,9 +996,6 @@ export class AssetRepository { .select((eb) => eb.fn.toJson(eb.table('stacked_assets')).as('stack')) .where('assets.ownerId', '=', asUuid(ownerId)) .where('assets.isVisible', '=', true) - .where('assets.fileCreatedAt', 'is not', null) - .where('assets.fileModifiedAt', 'is not', null) - .where('assets.localDateTime', 'is not', null) .where('assets.updatedAt', '<=', updatedUntil) .$if(!!lastId, (qb) => qb.where('assets.id', '>', lastId!)) .orderBy('assets.id') @@ -1040,9 +1024,6 @@ export class AssetRepository { .select((eb) => eb.fn.toJson(eb.table('stacked_assets')).as('stack')) .where('assets.ownerId', '=', anyUuid(options.userIds)) .where('assets.isVisible', '=', true) - .where('assets.fileCreatedAt', 'is not', null) - .where('assets.fileModifiedAt', 'is not', null) - .where('assets.localDateTime', 'is not', null) .where('assets.updatedAt', '>', options.updatedAfter) .limit(options.limit) .execute() as any as Promise; diff --git a/server/src/schema/tables/asset.table.ts b/server/src/schema/tables/asset.table.ts index 0fcbf4f9b1..250c3546a2 100644 --- a/server/src/schema/tables/asset.table.ts +++ b/server/src/schema/tables/asset.table.ts @@ -79,10 +79,10 @@ export class AssetTable { originalPath!: string; @ColumnIndex('idx_asset_file_created_at') - @Column({ type: 'timestamp with time zone', default: null }) + @Column({ type: 'timestamp with time zone' }) fileCreatedAt!: Date; - @Column({ type: 'timestamp with time zone', default: null }) + @Column({ type: 'timestamp with time zone' }) fileModifiedAt!: Date; @Column({ type: 'boolean', default: false }) @@ -135,7 +135,7 @@ export class AssetTable { @DeleteDateColumn() deletedAt!: Date | null; - @Column({ type: 'timestamp with time zone', default: null }) + @Column({ type: 'timestamp with time zone' }) localDateTime!: Date; @ForeignKeyColumn(() => StackTable, { nullable: true, onDelete: 'SET NULL', onUpdate: 'CASCADE' }) diff --git a/server/src/services/library.service.ts b/server/src/services/library.service.ts index 1039cff761..0c98288d2d 100644 --- a/server/src/services/library.service.ts +++ b/server/src/services/library.service.ts @@ -1,10 +1,12 @@ import { BadRequestException, Injectable } from '@nestjs/common'; +import { Insertable } from 'kysely'; import { R_OK } from 'node:constants'; import { Stats } from 'node:fs'; import path, { basename, isAbsolute, parse } from 'node:path'; import picomatch from 'picomatch'; import { JOBS_LIBRARY_PAGINATION_SIZE } from 'src/constants'; import { StorageCore } from 'src/cores/storage.core'; +import { Assets } from 'src/db'; import { OnEvent, OnJob } from 'src/decorators'; import { CreateLibraryDto, @@ -236,7 +238,14 @@ export class LibraryService extends BaseService { return JobStatus.FAILED; } - const assetImports = job.paths.map((assetPath) => this.processEntity(assetPath, library.ownerId, job.libraryId)); + const assetImports: Insertable[] = []; + await Promise.all( + job.paths.map((path) => + this.processEntity(path, library.ownerId, job.libraryId) + .then((asset) => assetImports.push(asset)) + .catch((error: any) => this.logger.error(`Error processing ${path} for library ${job.libraryId}`, error)), + ), + ); const assetIds: string[] = []; @@ -374,8 +383,9 @@ export class LibraryService extends BaseService { return JobStatus.SUCCESS; } - private processEntity(filePath: string, ownerId: string, libraryId: string) { + private async processEntity(filePath: string, ownerId: string, libraryId: string) { const assetPath = path.normalize(filePath); + const stat = await this.storageRepository.stat(assetPath); return { ownerId, @@ -383,9 +393,9 @@ export class LibraryService extends BaseService { checksum: this.cryptoRepository.hashSha1(`path:${assetPath}`), originalPath: assetPath, - fileCreatedAt: null, - fileModifiedAt: null, - localDateTime: null, + fileCreatedAt: stat.mtime, + fileModifiedAt: stat.mtime, + localDateTime: stat.mtime, // TODO: device asset id is deprecated, remove it deviceAssetId: `${basename(assetPath)}`.replaceAll(/\s+/g, ''), deviceId: 'Library Import', diff --git a/server/test/factory.ts b/server/test/factory.ts index 970410f10e..0e4d272b70 100644 --- a/server/test/factory.ts +++ b/server/test/factory.ts @@ -95,6 +95,9 @@ export class TestFactory { originalPath: '/path/to/something.jpg', ownerId: '@immich.cloud', isVisible: true, + fileCreatedAt: new Date('2000-01-01T00:00:00Z'), + fileModifiedAt: new Date('2000-01-01T00:00:00Z'), + localDateTime: new Date('2000-01-01T00:00:00Z'), }; return { diff --git a/server/test/medium/specs/sync.service.spec.ts b/server/test/medium/specs/sync.service.spec.ts index 574ddde93c..12ec4079fe 100644 --- a/server/test/medium/specs/sync.service.spec.ts +++ b/server/test/medium/specs/sync.service.spec.ts @@ -418,7 +418,7 @@ describe(SyncService.name, () => { fileModifiedAt: date, isFavorite: false, isVisible: true, - localDateTime: null, + localDateTime: '2000-01-01T00:00:00.000Z', type: asset.type, }, type: 'AssetV1', @@ -521,7 +521,7 @@ describe(SyncService.name, () => { fileModifiedAt: date, isFavorite: false, isVisible: true, - localDateTime: null, + localDateTime: '2000-01-01T00:00:00.000Z', type: asset.type, }, type: SyncEntityType.PartnerAssetV1,