refactor(server): non-nullable file metadata (#17598)

server/src/bin/migrations.ts

@@ -93,7 +93,7 @@ const create = (path: string, up: string[], down: string[]) => {
   const filename = `${timestamp}-${name}.ts`;
   const folder = dirname(path);
   const fullPath = join(folder, filename);
-  writeFileSync(fullPath, asMigration('kysely', { name, timestamp, up, down }));
+  writeFileSync(fullPath, asMigration('typeorm', { name, timestamp, up, down }));
   console.log(`Wrote ${fullPath}`);
 };
 

server/src/entities/asset.entity.ts

@@ -313,8 +313,5 @@ export function searchAssetBuilder(kysely: Kysely<DB>, options: AssetSearchBuild
     )
     .$if(!!options.withExif, withExifInner)
     .$if(!!(options.withFaces || options.withPeople || options.personIds), (qb) => qb.select(withFacesAndPeople))
-    .$if(!options.withDeleted, (qb) => qb.where('assets.deletedAt', 'is', null))
-    .where('assets.fileCreatedAt', 'is not', null)
-    .where('assets.fileModifiedAt', 'is not', null)
-    .where('assets.localDateTime', 'is not', null);
+    .$if(!options.withDeleted, (qb) => qb.where('assets.deletedAt', 'is', null));
 }

server/src/migrations/1744662638410-MakeFileMetadataNonNullable.ts

@@ -0,0 +1,22 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class MakeFileMetadataNonNullable1744662638410 implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `DELETE FROM assets WHERE "fileCreatedAt" IS NULL OR "fileModifiedAt" IS NULL OR "localDateTime" IS NULL`,
+    );
+    await queryRunner.query(`
+      ALTER TABLE assets
+        ALTER COLUMN "fileCreatedAt" SET NOT NULL,
+        ALTER COLUMN "fileModifiedAt" SET NOT NULL,
+        ALTER COLUMN "localDateTime" SET NOT NULL`);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`
+      ALTER TABLE assets
+        ALTER COLUMN "fileCreatedAt" DROP NOT NULL,
+        ALTER COLUMN "fileModifiedAt" DROP NOT NULL,
+        ALTER COLUMN "localDateTime" DROP NOT NULL`);
+  }
+}

server/src/queries/activity.repository.sql

@@ -71,6 +71,3 @@ where
   and "activity"."albumId" = $2
   and "activity"."isLiked" = $3
   and "assets"."deletedAt" is null
-  and "assets"."fileCreatedAt" is not null
-  and "assets"."fileModifiedAt" is not null
-  and "assets"."localDateTime" is not null

server/src/queries/asset.repository.sql

@@ -171,9 +171,6 @@ where
   "ownerId" = $1::uuid
   and "deviceId" = $2
   and "isVisible" = $3
-  and "assets"."fileCreatedAt" is not null
-  and "assets"."fileModifiedAt" is not null
-  and "assets"."localDateTime" is not null
   and "deletedAt" is null
 
 -- AssetRepository.getLivePhotoCount
@@ -334,9 +331,6 @@ with
     where
       "assets"."deletedAt" is null
       and "assets"."isVisible" = $2
-      and "assets"."fileCreatedAt" is not null
-      and "assets"."fileModifiedAt" is not null
-      and "assets"."localDateTime" is not null
   )
 select
   "timeBucket",
@@ -490,9 +484,6 @@ from
 where
   "assets"."ownerId" = $1::uuid
   and "assets"."isVisible" = $2
-  and "assets"."fileCreatedAt" is not null
-  and "assets"."fileModifiedAt" is not null
-  and "assets"."localDateTime" is not null
   and "assets"."updatedAt" <= $3
   and "assets"."id" > $4
 order by
@@ -523,9 +514,6 @@ from
 where
   "assets"."ownerId" = any ($1::uuid[])
   and "assets"."isVisible" = $2
-  and "assets"."fileCreatedAt" is not null
-  and "assets"."fileModifiedAt" is not null
-  and "assets"."localDateTime" is not null
   and "assets"."updatedAt" > $3
 limit
   $4

server/src/queries/search.repository.sql

@@ -13,9 +13,6 @@ where
   and "assets"."isFavorite" = $4
   and "assets"."isArchived" = $5
   and "assets"."deletedAt" is null
-  and "assets"."fileCreatedAt" is not null
-  and "assets"."fileModifiedAt" is not null
-  and "assets"."localDateTime" is not null
 order by
   "assets"."fileCreatedAt" desc
 limit
@@ -37,9 +34,6 @@ offset
     and "assets"."isFavorite" = $4
     and "assets"."isArchived" = $5
     and "assets"."deletedAt" is null
-    and "assets"."fileCreatedAt" is not null
-    and "assets"."fileModifiedAt" is not null
-    and "assets"."localDateTime" is not null
     and "assets"."id" < $6
   order by
     random()
@@ -60,9 +54,6 @@ union all
     and "assets"."isFavorite" = $11
     and "assets"."isArchived" = $12
     and "assets"."deletedAt" is null
-    and "assets"."fileCreatedAt" is not null
-    and "assets"."fileModifiedAt" is not null
-    and "assets"."localDateTime" is not null
     and "assets"."id" > $13
   order by
     random()
@@ -86,9 +77,6 @@ where
   and "assets"."isFavorite" = $4
   and "assets"."isArchived" = $5
   and "assets"."deletedAt" is null
-  and "assets"."fileCreatedAt" is not null
-  and "assets"."fileModifiedAt" is not null
-  and "assets"."localDateTime" is not null
 order by
   smart_search.embedding <=> $6
 limit

server/src/repositories/activity.repository.ts

@@ -76,9 +76,6 @@ export class ActivityRepository {
       .where('activity.albumId', '=', albumId)
       .where('activity.isLiked', '=', false)
       .where('assets.deletedAt', 'is', null)
-      .where('assets.fileCreatedAt', 'is not', null)
-      .where('assets.fileModifiedAt', 'is not', null)
-      .where('assets.localDateTime', 'is not', null)
       .executeTakeFirstOrThrow();
 
     return count;

server/src/repositories/asset.repository.ts

@@ -457,9 +457,6 @@ export class AssetRepository {
       .where('ownerId', '=', asUuid(ownerId))
       .where('deviceId', '=', deviceId)
       .where('isVisible', '=', true)
-      .where('assets.fileCreatedAt', 'is not', null)
-      .where('assets.fileModifiedAt', 'is not', null)
-      .where('assets.localDateTime', 'is not', null)
       .where('deletedAt', 'is', null)
       .execute();
 
@@ -674,8 +671,7 @@ export class AssetRepository {
         'exif.timeZone',
         'exif.fileSizeInByte',
       ])
-      .where('assets.deletedAt', 'is', null)
-      .where('assets.fileCreatedAt', 'is not', null);
+      .where('assets.deletedAt', 'is', null);
   }
 
   getStorageTemplateAsset(id: string): Promise<StorageAsset | undefined> {
@@ -712,10 +708,7 @@ export class AssetRepository {
           .where('job_status.duplicatesDetectedAt', 'is', null)
           .where('job_status.previewAt', 'is not', null)
           .where((eb) => eb.exists(eb.selectFrom('smart_search').where('assetId', '=', eb.ref('assets.id'))))
-          .where('assets.isVisible', '=', true)
-          .where('assets.fileCreatedAt', 'is not', null)
-          .where('assets.fileModifiedAt', 'is not', null)
-          .where('assets.localDateTime', 'is not', null),
+          .where('assets.isVisible', '=', true),
       )
       .$if(property === WithoutProperty.ENCODED_VIDEO, (qb) =>
         qb
@@ -778,9 +771,6 @@ export class AssetRepository {
       .select((eb) => eb.fn.countAll<number>().filterWhere('type', '=', AssetType.VIDEO).as(AssetType.VIDEO))
       .select((eb) => eb.fn.countAll<number>().filterWhere('type', '=', AssetType.OTHER).as(AssetType.OTHER))
       .where('ownerId', '=', asUuid(ownerId))
-      .where('assets.fileCreatedAt', 'is not', null)
-      .where('assets.fileModifiedAt', 'is not', null)
-      .where('assets.localDateTime', 'is not', null)
       .where('isVisible', '=', true)
       .$if(isArchived !== undefined, (qb) => qb.where('isArchived', '=', isArchived!))
       .$if(isFavorite !== undefined, (qb) => qb.where('isFavorite', '=', isFavorite!))
@@ -813,9 +803,6 @@ export class AssetRepository {
             .$if(!!options.isTrashed, (qb) => qb.where('assets.status', '!=', AssetStatus.DELETED))
             .where('assets.deletedAt', options.isTrashed ? 'is not' : 'is', null)
             .where('assets.isVisible', '=', true)
-            .where('assets.fileCreatedAt', 'is not', null)
-            .where('assets.fileModifiedAt', 'is not', null)
-            .where('assets.localDateTime', 'is not', null)
             .$if(!!options.albumId, (qb) =>
               qb
                 .innerJoin('albums_assets_assets', 'assets.id', 'albums_assets_assets.assetsId')
@@ -1009,9 +996,6 @@ export class AssetRepository {
       .select((eb) => eb.fn.toJson(eb.table('stacked_assets')).as('stack'))
       .where('assets.ownerId', '=', asUuid(ownerId))
       .where('assets.isVisible', '=', true)
-      .where('assets.fileCreatedAt', 'is not', null)
-      .where('assets.fileModifiedAt', 'is not', null)
-      .where('assets.localDateTime', 'is not', null)
       .where('assets.updatedAt', '<=', updatedUntil)
       .$if(!!lastId, (qb) => qb.where('assets.id', '>', lastId!))
       .orderBy('assets.id')
@@ -1040,9 +1024,6 @@ export class AssetRepository {
       .select((eb) => eb.fn.toJson(eb.table('stacked_assets')).as('stack'))
       .where('assets.ownerId', '=', anyUuid(options.userIds))
       .where('assets.isVisible', '=', true)
-      .where('assets.fileCreatedAt', 'is not', null)
-      .where('assets.fileModifiedAt', 'is not', null)
-      .where('assets.localDateTime', 'is not', null)
       .where('assets.updatedAt', '>', options.updatedAfter)
       .limit(options.limit)
       .execute() as any as Promise<AssetEntity[]>;

server/src/schema/tables/asset.table.ts

@@ -79,10 +79,10 @@ export class AssetTable {
   originalPath!: string;
 
   @ColumnIndex('idx_asset_file_created_at')
-  @Column({ type: 'timestamp with time zone', default: null })
+  @Column({ type: 'timestamp with time zone' })
   fileCreatedAt!: Date;
 
-  @Column({ type: 'timestamp with time zone', default: null })
+  @Column({ type: 'timestamp with time zone' })
   fileModifiedAt!: Date;
 
   @Column({ type: 'boolean', default: false })
@@ -135,7 +135,7 @@ export class AssetTable {
   @DeleteDateColumn()
   deletedAt!: Date | null;
 
-  @Column({ type: 'timestamp with time zone', default: null })
+  @Column({ type: 'timestamp with time zone' })
   localDateTime!: Date;
 
   @ForeignKeyColumn(() => StackTable, { nullable: true, onDelete: 'SET NULL', onUpdate: 'CASCADE' })

server/src/services/library.service.ts

@@ -1,10 +1,12 @@
 import { BadRequestException, Injectable } from '@nestjs/common';
+import { Insertable } from 'kysely';
 import { R_OK } from 'node:constants';
 import { Stats } from 'node:fs';
 import path, { basename, isAbsolute, parse } from 'node:path';
 import picomatch from 'picomatch';
 import { JOBS_LIBRARY_PAGINATION_SIZE } from 'src/constants';
 import { StorageCore } from 'src/cores/storage.core';
+import { Assets } from 'src/db';
 import { OnEvent, OnJob } from 'src/decorators';
 import {
   CreateLibraryDto,
@@ -236,7 +238,14 @@ export class LibraryService extends BaseService {
       return JobStatus.FAILED;
     }
 
-    const assetImports = job.paths.map((assetPath) => this.processEntity(assetPath, library.ownerId, job.libraryId));
+    const assetImports: Insertable<Assets>[] = [];
+    await Promise.all(
+      job.paths.map((path) =>
+        this.processEntity(path, library.ownerId, job.libraryId)
+          .then((asset) => assetImports.push(asset))
+          .catch((error: any) => this.logger.error(`Error processing ${path} for library ${job.libraryId}`, error)),
+      ),
+    );
 
     const assetIds: string[] = [];
 
@@ -374,8 +383,9 @@ export class LibraryService extends BaseService {
     return JobStatus.SUCCESS;
   }
 
-  private processEntity(filePath: string, ownerId: string, libraryId: string) {
+  private async processEntity(filePath: string, ownerId: string, libraryId: string) {
     const assetPath = path.normalize(filePath);
+    const stat = await this.storageRepository.stat(assetPath);
 
     return {
       ownerId,
@@ -383,9 +393,9 @@ export class LibraryService extends BaseService {
       checksum: this.cryptoRepository.hashSha1(`path:${assetPath}`),
       originalPath: assetPath,
 
-      fileCreatedAt: null,
-      fileModifiedAt: null,
-      localDateTime: null,
+      fileCreatedAt: stat.mtime,
+      fileModifiedAt: stat.mtime,
+      localDateTime: stat.mtime,
       // TODO: device asset id is deprecated, remove it
       deviceAssetId: `${basename(assetPath)}`.replaceAll(/\s+/g, ''),
       deviceId: 'Library Import',

server/test/factory.ts

@@ -95,6 +95,9 @@ export class TestFactory {
       originalPath: '/path/to/something.jpg',
       ownerId: '@immich.cloud',
       isVisible: true,
+      fileCreatedAt: new Date('2000-01-01T00:00:00Z'),
+      fileModifiedAt: new Date('2000-01-01T00:00:00Z'),
+      localDateTime: new Date('2000-01-01T00:00:00Z'),
     };
 
     return {

server/test/medium/specs/sync.service.spec.ts

@@ -418,7 +418,7 @@ describe(SyncService.name, () => {
               fileModifiedAt: date,
               isFavorite: false,
               isVisible: true,
-              localDateTime: null,
+              localDateTime: '2000-01-01T00:00:00.000Z',
               type: asset.type,
             },
             type: 'AssetV1',
@@ -521,7 +521,7 @@ describe(SyncService.name, () => {
               fileModifiedAt: date,
               isFavorite: false,
               isVisible: true,
-              localDateTime: null,
+              localDateTime: '2000-01-01T00:00:00.000Z',
               type: asset.type,
             },
             type: SyncEntityType.PartnerAssetV1,