refactor: new asset-job repository (#17622)

* refactor: new asset-job repository

* fix: broken medium tests on main

server/src/queries/asset.job.repository.sql

@@ -0,0 +1,60 @@
+-- NOTE: This file is auto generated by ./sql-generator
+
+-- AssetJobRepository.getForSearchDuplicatesJob
+select
+  "id",
+  "type",
+  "ownerId",
+  "duplicateId",
+  "stackId",
+  "isVisible",
+  "smart_search"."embedding",
+  (
+    select
+      coalesce(json_agg(agg), '[]')
+    from
+      (
+        select
+          "asset_files"."id",
+          "asset_files"."path",
+          "asset_files"."type"
+        from
+          "asset_files"
+        where
+          "asset_files"."assetId" = "assets"."id"
+          and "asset_files"."type" = $1
+      ) as agg
+  ) as "files"
+from
+  "assets"
+  left join "smart_search" on "assets"."id" = "smart_search"."assetId"
+where
+  "assets"."id" = $2::uuid
+limit
+  $3
+
+-- AssetJobRepository.getForSidecarWriteJob
+select
+  "id",
+  "sidecarPath",
+  "originalPath",
+  (
+    select
+      coalesce(json_agg(agg), '[]')
+    from
+      (
+        select
+          "tags"."value"
+        from
+          "tags"
+          inner join "tag_asset" on "tags"."id" = "tag_asset"."tagsId"
+        where
+          "assets"."id" = "tag_asset"."assetsId"
+      ) as agg
+  ) as "tags"
+from
+  "assets"
+where
+  "assets"."id" = $1::uuid
+limit
+  $2

server/src/queries/asset.repository.sql

@@ -181,65 +181,6 @@ from
 where
   "livePhotoVideoId" = $1::uuid
 
--- AssetRepository.getAssetForSearchDuplicatesJob
-select
-  "id",
-  "type",
-  "ownerId",
-  "duplicateId",
-  "stackId",
-  "isVisible",
-  "smart_search"."embedding",
-  (
-    select
-      coalesce(json_agg(agg), '[]')
-    from
-      (
-        select
-          "asset_files"."id",
-          "asset_files"."path",
-          "asset_files"."type"
-        from
-          "asset_files"
-        where
-          "asset_files"."assetId" = "assets"."id"
-          and "asset_files"."type" = $1
-      ) as agg
-  ) as "files"
-from
-  "assets"
-  left join "smart_search" on "assets"."id" = "smart_search"."assetId"
-where
-  "assets"."id" = $2::uuid
-limit
-  $3
-
--- AssetRepository.getAssetForSidecarWriteJob
-select
-  "id",
-  "sidecarPath",
-  "originalPath",
-  (
-    select
-      coalesce(json_agg(agg), '[]')
-    from
-      (
-        select
-          "tags"."value"
-        from
-          "tags"
-          inner join "tag_asset" on "tags"."id" = "tag_asset"."tagsId"
-        where
-          "assets"."id" = "tag_asset"."assetsId"
-      ) as agg
-  ) as "tags"
-from
-  "assets"
-where
-  "assets"."id" = $1::uuid
-limit
-  $2
-
 -- AssetRepository.getById
 select
   "assets".*

server/src/repositories/asset-job.repository.ts

@@ -0,0 +1,95 @@
+import { Injectable } from '@nestjs/common';
+import { Kysely } from 'kysely';
+import { jsonArrayFrom } from 'kysely/helpers/postgres';
+import { InjectKysely } from 'nestjs-kysely';
+import { DB } from 'src/db';
+import { DummyValue, GenerateSql } from 'src/decorators';
+import { withFiles } from 'src/entities/asset.entity';
+import { AssetFileType } from 'src/enum';
+import { StorageAsset } from 'src/types';
+import { asUuid } from 'src/utils/database';
+
+@Injectable()
+export class AssetJobRepository {
+  constructor(@InjectKysely() private db: Kysely<DB>) {}
+
+  @GenerateSql({ params: [DummyValue.UUID] })
+  getForSearchDuplicatesJob(id: string) {
+    return this.db
+      .selectFrom('assets')
+      .where('assets.id', '=', asUuid(id))
+      .leftJoin('smart_search', 'assets.id', 'smart_search.assetId')
+      .select((eb) => [
+        'id',
+        'type',
+        'ownerId',
+        'duplicateId',
+        'stackId',
+        'isVisible',
+        'smart_search.embedding',
+        withFiles(eb, AssetFileType.PREVIEW),
+      ])
+      .limit(1)
+      .executeTakeFirst();
+  }
+
+  @GenerateSql({ params: [DummyValue.UUID] })
+  getForSidecarWriteJob(id: string) {
+    return this.db
+      .selectFrom('assets')
+      .where('assets.id', '=', asUuid(id))
+      .select((eb) => [
+        'id',
+        'sidecarPath',
+        'originalPath',
+        jsonArrayFrom(
+          eb
+            .selectFrom('tags')
+            .select(['tags.value'])
+            .innerJoin('tag_asset', 'tags.id', 'tag_asset.tagsId')
+            .whereRef('assets.id', '=', 'tag_asset.assetsId'),
+        ).as('tags'),
+      ])
+      .limit(1)
+      .executeTakeFirst();
+  }
+
+  private storageTemplateAssetQuery() {
+    return this.db
+      .selectFrom('assets')
+      .innerJoin('exif', 'assets.id', 'exif.assetId')
+      .select([
+        'assets.id',
+        'assets.ownerId',
+        'assets.type',
+        'assets.checksum',
+        'assets.originalPath',
+        'assets.isExternal',
+        'assets.sidecarPath',
+        'assets.originalFileName',
+        'assets.livePhotoVideoId',
+        'assets.fileCreatedAt',
+        'exif.timeZone',
+        'exif.fileSizeInByte',
+      ])
+      .where('assets.deletedAt', 'is', null);
+  }
+
+  getForStorageTemplateJob(id: string): Promise<StorageAsset | undefined> {
+    return this.storageTemplateAssetQuery().where('assets.id', '=', id).executeTakeFirst() as Promise<
+      StorageAsset | undefined
+    >;
+  }
+
+  streamForStorageTemplateJob() {
+    return this.storageTemplateAssetQuery().stream() as AsyncIterableIterator<StorageAsset>;
+  }
+
+  streamForDeletedJob(trashedBefore: Date) {
+    return this.db
+      .selectFrom('assets')
+      .select(['id', 'isOffline'])
+      .where('assets.deletedAt', '<=', trashedBefore)
+      .stream();
+  }
+}

server/src/repositories/asset.repository.ts

@@ -1,6 +1,5 @@
 import { Injectable } from '@nestjs/common';
 import { Insertable, Kysely, Selectable, UpdateResult, Updateable, sql } from 'kysely';
-import { jsonArrayFrom } from 'kysely/helpers/postgres';
 import { isEmpty, isUndefined, omitBy } from 'lodash';
 import { InjectKysely } from 'nestjs-kysely';
 import { AssetFiles, AssetJobStatus, Assets, DB, Exif } from 'src/db';
@@ -24,7 +23,6 @@ import {
 } from 'src/entities/asset.entity';
 import { AssetFileType, AssetOrder, AssetStatus, AssetType } from 'src/enum';
 import { AssetSearchOptions, SearchExploreItem, SearchExploreItemSet } from 'src/repositories/search.repository';
-import { StorageAsset } from 'src/types';
 import { anyUuid, asUuid, removeUndefinedKeys, unnest } from 'src/utils/database';
 import { globToSqlPattern } from 'src/utils/misc';
 import { Paginated, PaginationOptions, paginationHelper } from 'src/utils/pagination';
@@ -473,47 +471,6 @@ export class AssetRepository {
     return count;
   }
 
-  @GenerateSql({ params: [DummyValue.UUID] })
-  getAssetForSearchDuplicatesJob(id: string) {
-    return this.db
-      .selectFrom('assets')
-      .where('assets.id', '=', asUuid(id))
-      .leftJoin('smart_search', 'assets.id', 'smart_search.assetId')
-      .select((eb) => [
-        'id',
-        'type',
-        'ownerId',
-        'duplicateId',
-        'stackId',
-        'isVisible',
-        'smart_search.embedding',
-        withFiles(eb, AssetFileType.PREVIEW),
-      ])
-      .limit(1)
-      .executeTakeFirst();
-  }
-
-  @GenerateSql({ params: [DummyValue.UUID] })
-  getAssetForSidecarWriteJob(id: string) {
-    return this.db
-      .selectFrom('assets')
-      .where('assets.id', '=', asUuid(id))
-      .select((eb) => [
-        'id',
-        'sidecarPath',
-        'originalPath',
-        jsonArrayFrom(
-          eb
-            .selectFrom('tags')
-            .select(['tags.value'])
-            .innerJoin('tag_asset', 'tags.id', 'tag_asset.tagsId')
-            .whereRef('assets.id', '=', 'tag_asset.assetsId'),
-        ).as('tags'),
-      ])
-      .limit(1)
-      .executeTakeFirst();
-  }
-
   @GenerateSql({ params: [DummyValue.UUID] })
   getById(
     id: string,
@@ -653,45 +610,6 @@ export class AssetRepository {
       .executeTakeFirst() as Promise<AssetEntity | undefined>;
   }
 
-  private storageTemplateAssetQuery() {
-    return this.db
-      .selectFrom('assets')
-      .innerJoin('exif', 'assets.id', 'exif.assetId')
-      .select([
-        'assets.id',
-        'assets.ownerId',
-        'assets.type',
-        'assets.checksum',
-        'assets.originalPath',
-        'assets.isExternal',
-        'assets.sidecarPath',
-        'assets.originalFileName',
-        'assets.livePhotoVideoId',
-        'assets.fileCreatedAt',
-        'exif.timeZone',
-        'exif.fileSizeInByte',
-      ])
-      .where('assets.deletedAt', 'is', null);
-  }
-
-  getStorageTemplateAsset(id: string): Promise<StorageAsset | undefined> {
-    return this.storageTemplateAssetQuery().where('assets.id', '=', id).executeTakeFirst() as Promise<
-      StorageAsset | undefined
-    >;
-  }
-
-  streamStorageTemplateAssets() {
-    return this.storageTemplateAssetQuery().stream() as AsyncIterableIterator<StorageAsset>;
-  }
-
-  streamDeletedAssets(trashedBefore: Date) {
-    return this.db
-      .selectFrom('assets')
-      .select(['id', 'isOffline'])
-      .where('assets.deletedAt', '<=', trashedBefore)
-      .stream();
-  }
-
   @GenerateSql(
     ...Object.values(WithProperty).map((property) => ({
       name: property,

server/src/repositories/index.ts

@@ -3,6 +3,7 @@ import { ActivityRepository } from 'src/repositories/activity.repository';
 import { AlbumUserRepository } from 'src/repositories/album-user.repository';
 import { AlbumRepository } from 'src/repositories/album.repository';
 import { ApiKeyRepository } from 'src/repositories/api-key.repository';
+import { AssetJobRepository } from 'src/repositories/asset-job.repository';
 import { AssetRepository } from 'src/repositories/asset.repository';
 import { AuditRepository } from 'src/repositories/audit.repository';
 import { ConfigRepository } from 'src/repositories/config.repository';
@@ -48,6 +49,7 @@ export const repositories = [
   AuditRepository,
   ApiKeyRepository,
   AssetRepository,
+  AssetJobRepository,
   ConfigRepository,
   CronRepository,
   CryptoRepository,

server/src/services/asset.service.spec.ts

@@ -536,12 +536,12 @@ describe(AssetService.name, () => {
     it('should immediately queue assets for deletion if trash is disabled', async () => {
       const asset = factory.asset({ isOffline: false });
 
-      mocks.asset.streamDeletedAssets.mockReturnValue(makeStream([asset]));
+      mocks.assetJob.streamForDeletedJob.mockReturnValue(makeStream([asset]));
       mocks.systemMetadata.get.mockResolvedValue({ trash: { enabled: false } });
 
       await expect(sut.handleAssetDeletionCheck()).resolves.toBe(JobStatus.SUCCESS);
 
-      expect(mocks.asset.streamDeletedAssets).toHaveBeenCalledWith(new Date());
+      expect(mocks.assetJob.streamForDeletedJob).toHaveBeenCalledWith(new Date());
       expect(mocks.job.queueAll).toHaveBeenCalledWith([
         { name: JobName.ASSET_DELETION, data: { id: asset.id, deleteOnDisk: true } },
       ]);
@@ -550,12 +550,12 @@ describe(AssetService.name, () => {
     it('should queue assets for deletion after trash duration', async () => {
       const asset = factory.asset({ isOffline: false });
 
-      mocks.asset.streamDeletedAssets.mockReturnValue(makeStream([asset]));
+      mocks.assetJob.streamForDeletedJob.mockReturnValue(makeStream([asset]));
       mocks.systemMetadata.get.mockResolvedValue({ trash: { enabled: true, days: 7 } });
 
       await expect(sut.handleAssetDeletionCheck()).resolves.toBe(JobStatus.SUCCESS);
 
-      expect(mocks.asset.streamDeletedAssets).toHaveBeenCalledWith(DateTime.now().minus({ days: 7 }).toJSDate());
+      expect(mocks.assetJob.streamForDeletedJob).toHaveBeenCalledWith(DateTime.now().minus({ days: 7 }).toJSDate());
       expect(mocks.job.queueAll).toHaveBeenCalledWith([
         { name: JobName.ASSET_DELETION, data: { id: asset.id, deleteOnDisk: true } },
       ]);

server/src/services/asset.service.ts

@@ -172,7 +172,7 @@ export class AssetService extends BaseService {
       }
     };
 
-    const assets = this.assetRepository.streamDeletedAssets(trashedBefore);
+    const assets = this.assetJobRepository.streamForDeletedJob(trashedBefore);
     for await (const asset of assets) {
       chunk.push(asset);
       if (chunk.length >= JOBS_ASSET_PAGINATION_SIZE) {

server/src/services/base.service.ts

@@ -10,6 +10,7 @@ import { ActivityRepository } from 'src/repositories/activity.repository';
 import { AlbumUserRepository } from 'src/repositories/album-user.repository';
 import { AlbumRepository } from 'src/repositories/album.repository';
 import { ApiKeyRepository } from 'src/repositories/api-key.repository';
+import { AssetJobRepository } from 'src/repositories/asset-job.repository';
 import { AssetRepository } from 'src/repositories/asset.repository';
 import { AuditRepository } from 'src/repositories/audit.repository';
 import { ConfigRepository } from 'src/repositories/config.repository';
@@ -62,6 +63,7 @@ export class BaseService {
     protected albumUserRepository: AlbumUserRepository,
     protected apiKeyRepository: ApiKeyRepository,
     protected assetRepository: AssetRepository,
+    protected assetJobRepository: AssetJobRepository,
     protected auditRepository: AuditRepository,
     protected configRepository: ConfigRepository,
     protected cronRepository: CronRepository,

server/src/services/duplicate.service.spec.ts

@@ -194,6 +194,8 @@ describe(SearchService.name, () => {
     });
 
     it('should fail if asset is not found', async () => {
+      mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue(void 0);
+
       const result = await sut.handleSearchDuplicates({ id: assetStub.image.id });
 
       expect(result).toBe(JobStatus.FAILED);
@@ -202,7 +204,7 @@ describe(SearchService.name, () => {
 
     it('should skip if asset is part of stack', async () => {
       const id = assetStub.primaryImage.id;
-      mocks.asset.getAssetForSearchDuplicatesJob.mockResolvedValue({ ...hasEmbedding, stackId: 'stack-id' });
+      mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue({ ...hasEmbedding, stackId: 'stack-id' });
 
       const result = await sut.handleSearchDuplicates({ id });
 
@@ -212,7 +214,7 @@ describe(SearchService.name, () => {
 
     it('should skip if asset is not visible', async () => {
       const id = assetStub.livePhotoMotionAsset.id;
-      mocks.asset.getAssetForSearchDuplicatesJob.mockResolvedValue({ ...hasEmbedding, isVisible: false });
+      mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue({ ...hasEmbedding, isVisible: false });
 
       const result = await sut.handleSearchDuplicates({ id });
 
@@ -221,7 +223,7 @@ describe(SearchService.name, () => {
     });
 
     it('should fail if asset is missing preview image', async () => {
-      mocks.asset.getAssetForSearchDuplicatesJob.mockResolvedValue({ ...hasEmbedding, files: [] });
+      mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue({ ...hasEmbedding, files: [] });
 
       const result = await sut.handleSearchDuplicates({ id: assetStub.noResizePath.id });
 
@@ -230,7 +232,7 @@ describe(SearchService.name, () => {
     });
 
     it('should fail if asset is missing embedding', async () => {
-      mocks.asset.getAssetForSearchDuplicatesJob.mockResolvedValue({ ...hasEmbedding, embedding: null });
+      mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue({ ...hasEmbedding, embedding: null });
 
       const result = await sut.handleSearchDuplicates({ id: assetStub.image.id });
 
@@ -239,7 +241,7 @@ describe(SearchService.name, () => {
     });
 
     it('should search for duplicates and update asset with duplicateId', async () => {
-      mocks.asset.getAssetForSearchDuplicatesJob.mockResolvedValue(hasEmbedding);
+      mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue(hasEmbedding);
       mocks.search.searchDuplicates.mockResolvedValue([
         { assetId: assetStub.image.id, distance: 0.01, duplicateId: null },
       ]);
@@ -267,7 +269,7 @@ describe(SearchService.name, () => {
 
     it('should use existing duplicate ID among matched duplicates', async () => {
       const duplicateId = hasDupe.duplicateId;
-      mocks.asset.getAssetForSearchDuplicatesJob.mockResolvedValue(hasEmbedding);
+      mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue(hasEmbedding);
       mocks.search.searchDuplicates.mockResolvedValue([{ assetId: hasDupe.id, distance: 0.01, duplicateId }]);
       const expectedAssetIds = [hasEmbedding.id];
 
@@ -292,7 +294,7 @@ describe(SearchService.name, () => {
     });
 
     it('should remove duplicateId if no duplicates found and asset has duplicateId', async () => {
-      mocks.asset.getAssetForSearchDuplicatesJob.mockResolvedValue(hasDupe);
+      mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue(hasDupe);
       mocks.search.searchDuplicates.mockResolvedValue([]);
 
       const result = await sut.handleSearchDuplicates({ id: hasDupe.id });

server/src/services/duplicate.service.ts

@@ -52,7 +52,7 @@ export class DuplicateService extends BaseService {
       return JobStatus.SKIPPED;
     }
 
-    const asset = await this.assetRepository.getAssetForSearchDuplicatesJob(id);
+    const asset = await this.assetJobRepository.getForSearchDuplicatesJob(id);
     if (!asset) {
       this.logger.error(`Asset ${id} not found`);
       return JobStatus.FAILED;

server/src/services/metadata.service.spec.ts

@@ -1486,14 +1486,14 @@ describe(MetadataService.name, () => {
 
   describe('handleSidecarWrite', () => {
     it('should skip assets that do not exist anymore', async () => {
-      mocks.asset.getAssetForSidecarWriteJob.mockResolvedValue(void 0);
+      mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(void 0);
       await expect(sut.handleSidecarWrite({ id: 'asset-123' })).resolves.toBe(JobStatus.FAILED);
       expect(mocks.metadata.writeTags).not.toHaveBeenCalled();
     });
 
     it('should skip jobs with no metadata', async () => {
       const asset = factory.jobAssets.sidecarWrite();
-      mocks.asset.getAssetForSidecarWriteJob.mockResolvedValue(asset);
+      mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(asset);
       await expect(sut.handleSidecarWrite({ id: asset.id })).resolves.toBe(JobStatus.SKIPPED);
       expect(mocks.metadata.writeTags).not.toHaveBeenCalled();
     });
@@ -1504,7 +1504,7 @@ describe(MetadataService.name, () => {
       const gps = 12;
       const date = '2023-11-22T04:56:12.196Z';
 
-      mocks.asset.getAssetForSidecarWriteJob.mockResolvedValue(asset);
+      mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(asset);
       await expect(
         sut.handleSidecarWrite({
           id: asset.id,

server/src/services/metadata.service.ts

@@ -329,7 +329,7 @@ export class MetadataService extends BaseService {
   @OnJob({ name: JobName.SIDECAR_WRITE, queue: QueueName.SIDECAR })
   async handleSidecarWrite(job: JobOf<JobName.SIDECAR_WRITE>): Promise<JobStatus> {
     const { id, description, dateTimeOriginal, latitude, longitude, rating, tags } = job;
-    const asset = await this.assetRepository.getAssetForSidecarWriteJob(id);
+    const asset = await this.assetJobRepository.getForSidecarWriteJob(id);
     if (!asset) {
       return JobStatus.FAILED;
     }

server/src/services/storage-template.service.spec.ts

@@ -112,8 +112,8 @@ describe(StorageTemplateService.name, () => {
       const newMotionPicturePath = `upload/library/${motionAsset.ownerId}/2022/2022-06-19/${motionAsset.originalFileName}`;
       const newStillPicturePath = `upload/library/${stillAsset.ownerId}/2022/2022-06-19/${stillAsset.originalFileName}`;
 
-      mocks.asset.getStorageTemplateAsset.mockResolvedValueOnce(stillAsset);
+      mocks.assetJob.getForStorageTemplateJob.mockResolvedValueOnce(stillAsset);
-      mocks.asset.getStorageTemplateAsset.mockResolvedValueOnce(motionAsset);
+      mocks.assetJob.getForStorageTemplateJob.mockResolvedValueOnce(motionAsset);
 
       mocks.move.create.mockResolvedValueOnce({
         id: '123',
@@ -148,7 +148,7 @@ describe(StorageTemplateService.name, () => {
       sut.onConfigInit({ newConfig: config });
 
       mocks.user.get.mockResolvedValue(user);
-      mocks.asset.getStorageTemplateAsset.mockResolvedValueOnce(asset);
+      mocks.assetJob.getForStorageTemplateJob.mockResolvedValueOnce(asset);
       mocks.album.getByAssetId.mockResolvedValueOnce([album]);
 
       expect(await sut.handleMigrationSingle({ id: asset.id })).toBe(JobStatus.SUCCESS);
@@ -169,7 +169,7 @@ describe(StorageTemplateService.name, () => {
       sut.onConfigInit({ newConfig: config });
 
       mocks.user.get.mockResolvedValue(user);
-      mocks.asset.getStorageTemplateAsset.mockResolvedValueOnce(asset);
+      mocks.assetJob.getForStorageTemplateJob.mockResolvedValueOnce(asset);
 
       expect(await sut.handleMigrationSingle({ id: asset.id })).toBe(JobStatus.SUCCESS);
 
@@ -197,7 +197,7 @@ describe(StorageTemplateService.name, () => {
         oldPath: asset.originalPath,
         newPath: previousFailedNewPath,
       });
-      mocks.asset.getStorageTemplateAsset.mockResolvedValue(asset);
+      mocks.assetJob.getForStorageTemplateJob.mockResolvedValue(asset);
       mocks.move.update.mockResolvedValue({
         id: '123',
         entityId: asset.id,
@@ -208,7 +208,7 @@ describe(StorageTemplateService.name, () => {
 
       await expect(sut.handleMigrationSingle({ id: asset.id })).resolves.toBe(JobStatus.SUCCESS);
 
-      expect(mocks.asset.getStorageTemplateAsset).toHaveBeenCalledWith(asset.id);
+      expect(mocks.assetJob.getForStorageTemplateJob).toHaveBeenCalledWith(asset.id);
       expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(3);
       expect(mocks.storage.rename).toHaveBeenCalledWith(asset.originalPath, newPath);
       expect(mocks.move.update).toHaveBeenCalledWith('123', {
@@ -239,7 +239,7 @@ describe(StorageTemplateService.name, () => {
         oldPath: asset.originalPath,
         newPath: previousFailedNewPath,
       });
-      mocks.asset.getStorageTemplateAsset.mockResolvedValue(asset);
+      mocks.assetJob.getForStorageTemplateJob.mockResolvedValue(asset);
       mocks.move.update.mockResolvedValue({
         id: '123',
         entityId: asset.id,
@@ -250,7 +250,7 @@ describe(StorageTemplateService.name, () => {
 
       await expect(sut.handleMigrationSingle({ id: asset.id })).resolves.toBe(JobStatus.SUCCESS);
 
-      expect(mocks.asset.getStorageTemplateAsset).toHaveBeenCalledWith(asset.id);
+      expect(mocks.assetJob.getForStorageTemplateJob).toHaveBeenCalledWith(asset.id);
       expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(3);
       expect(mocks.storage.stat).toHaveBeenCalledWith(previousFailedNewPath);
       expect(mocks.storage.rename).toHaveBeenCalledWith(previousFailedNewPath, newPath);
@@ -266,7 +266,7 @@ describe(StorageTemplateService.name, () => {
       mocks.storage.rename.mockRejectedValue({ code: 'EXDEV' });
       mocks.storage.stat.mockResolvedValue({ size: 5000 } as Stats);
       mocks.crypto.hashFile.mockResolvedValue(Buffer.from('different-hash', 'utf8'));
-      mocks.asset.getStorageTemplateAsset.mockResolvedValue(testAsset);
+      mocks.assetJob.getForStorageTemplateJob.mockResolvedValue(testAsset);
       mocks.move.create.mockResolvedValue({
         id: '123',
         entityId: testAsset.id,
@@ -277,7 +277,7 @@ describe(StorageTemplateService.name, () => {
 
       await expect(sut.handleMigrationSingle({ id: testAsset.id })).resolves.toBe(JobStatus.SUCCESS);
 
-      expect(mocks.asset.getStorageTemplateAsset).toHaveBeenCalledWith(testAsset.id);
+      expect(mocks.assetJob.getForStorageTemplateJob).toHaveBeenCalledWith(testAsset.id);
       expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(1);
       expect(mocks.storage.stat).toHaveBeenCalledWith(newPath);
       expect(mocks.move.create).toHaveBeenCalledWith({
@@ -316,7 +316,7 @@ describe(StorageTemplateService.name, () => {
           oldPath: testAsset.originalPath,
           newPath: previousFailedNewPath,
         });
-        mocks.asset.getStorageTemplateAsset.mockResolvedValue(testAsset);
+        mocks.assetJob.getForStorageTemplateJob.mockResolvedValue(testAsset);
         mocks.move.update.mockResolvedValue({
           id: '123',
           entityId: testAsset.id,
@@ -327,7 +327,7 @@ describe(StorageTemplateService.name, () => {
 
         await expect(sut.handleMigrationSingle({ id: testAsset.id })).resolves.toBe(JobStatus.SUCCESS);
 
-        expect(mocks.asset.getStorageTemplateAsset).toHaveBeenCalledWith(testAsset.id);
+        expect(mocks.assetJob.getForStorageTemplateJob).toHaveBeenCalledWith(testAsset.id);
         expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(3);
         expect(mocks.storage.stat).toHaveBeenCalledWith(previousFailedNewPath);
         expect(mocks.storage.rename).not.toHaveBeenCalled();
@@ -340,12 +340,12 @@ describe(StorageTemplateService.name, () => {
 
   describe('handle template migration', () => {
     it('should handle no assets', async () => {
-      mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([]));
+      mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([]));
       mocks.user.getList.mockResolvedValue([]);
 
       await sut.handleMigration();
 
-      expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
+      expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();
     });
 
     it('should handle an asset with a duplicate destination', async () => {
@@ -354,7 +354,7 @@ describe(StorageTemplateService.name, () => {
       const newPath = `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`;
       const newPath2 = newPath.replace('.jpg', '+1.jpg');
 
-      mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
+      mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset]));
       mocks.user.getList.mockResolvedValue([userStub.user1]);
       mocks.move.create.mockResolvedValue({
         id: '123',
@@ -369,7 +369,7 @@ describe(StorageTemplateService.name, () => {
 
       await sut.handleMigration();
 
-      expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
+      expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();
       expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(2);
       expect(mocks.asset.update).toHaveBeenCalledWith({ id: asset.id, originalPath: newPath2 });
       expect(mocks.user.getList).toHaveBeenCalled();
@@ -378,12 +378,12 @@ describe(StorageTemplateService.name, () => {
     it('should skip when an asset already matches the template', async () => {
       const asset = assetStub.storageAsset({ originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg' });
 
-      mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
+      mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset]));
       mocks.user.getList.mockResolvedValue([userStub.user1]);
 
       await sut.handleMigration();
 
-      expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
+      expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();
       expect(mocks.storage.rename).not.toHaveBeenCalled();
       expect(mocks.storage.copyFile).not.toHaveBeenCalled();
       expect(mocks.storage.checkFileExists).not.toHaveBeenCalledTimes(2);
@@ -393,12 +393,12 @@ describe(StorageTemplateService.name, () => {
     it('should skip when an asset is probably a duplicate', async () => {
       const asset = assetStub.storageAsset({ originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id+1.jpg' });
 
-      mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
+      mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset]));
       mocks.user.getList.mockResolvedValue([userStub.user1]);
 
       await sut.handleMigration();
 
-      expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
+      expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();
       expect(mocks.storage.rename).not.toHaveBeenCalled();
       expect(mocks.storage.copyFile).not.toHaveBeenCalled();
       expect(mocks.storage.checkFileExists).not.toHaveBeenCalledTimes(2);
@@ -409,7 +409,7 @@ describe(StorageTemplateService.name, () => {
       const asset = assetStub.storageAsset();
       const oldPath = asset.originalPath;
       const newPath = `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`;
-      mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
+      mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset]));
       mocks.user.getList.mockResolvedValue([userStub.user1]);
       mocks.move.create.mockResolvedValue({
         id: '123',
@@ -421,7 +421,7 @@ describe(StorageTemplateService.name, () => {
 
       await sut.handleMigration();
 
-      expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
+      expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();
       expect(mocks.storage.rename).toHaveBeenCalledWith(oldPath, newPath);
       expect(mocks.asset.update).toHaveBeenCalledWith({ id: asset.id, originalPath: newPath });
     });
@@ -429,7 +429,7 @@ describe(StorageTemplateService.name, () => {
     it('should use the user storage label', async () => {
       const user = factory.userAdmin({ storageLabel: 'label-1' });
       const asset = assetStub.storageAsset({ ownerId: user.id });
-      mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
+      mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset]));
       mocks.user.getList.mockResolvedValue([user]);
       mocks.move.create.mockResolvedValue({
         id: '123',
@@ -441,7 +441,7 @@ describe(StorageTemplateService.name, () => {
 
       await sut.handleMigration();
 
-      expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
+      expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();
       expect(mocks.storage.rename).toHaveBeenCalledWith(
         '/original/path.jpg',
         `upload/library/${user.storageLabel}/2022/2022-06-19/${asset.originalFileName}`,
@@ -456,7 +456,7 @@ describe(StorageTemplateService.name, () => {
       const asset = assetStub.storageAsset({ originalPath: '/path/to/original.jpg', fileSizeInByte: 5000 });
       const oldPath = asset.originalPath;
       const newPath = `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`;
-      mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
+      mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset]));
       mocks.storage.rename.mockRejectedValue({ code: 'EXDEV' });
       mocks.user.getList.mockResolvedValue([userStub.user1]);
       mocks.move.create.mockResolvedValue({
@@ -482,7 +482,7 @@ describe(StorageTemplateService.name, () => {
 
       await sut.handleMigration();
 
-      expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
+      expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();
       expect(mocks.storage.rename).toHaveBeenCalledWith(oldPath, newPath);
       expect(mocks.storage.copyFile).toHaveBeenCalledWith(oldPath, newPath);
       expect(mocks.storage.stat).toHaveBeenCalledWith(oldPath);
@@ -495,7 +495,7 @@ describe(StorageTemplateService.name, () => {
 
     it('should not update the database if the move fails due to incorrect newPath filesize', async () => {
       const asset = assetStub.storageAsset();
-      mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
+      mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset]));
       mocks.storage.rename.mockRejectedValue({ code: 'EXDEV' });
       mocks.user.getList.mockResolvedValue([userStub.user1]);
       mocks.move.create.mockResolvedValue({
@@ -511,7 +511,7 @@ describe(StorageTemplateService.name, () => {
 
       await sut.handleMigration();
 
-      expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
+      expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();
       expect(mocks.storage.rename).toHaveBeenCalledWith(
         '/original/path.jpg',
         `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`,
@@ -528,7 +528,7 @@ describe(StorageTemplateService.name, () => {
 
     it('should not update the database if the move fails', async () => {
       const asset = assetStub.storageAsset();
-      mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
+      mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset]));
       mocks.storage.rename.mockRejectedValue(new Error('Read only system'));
       mocks.storage.copyFile.mockRejectedValue(new Error('Read only system'));
       mocks.move.create.mockResolvedValue({
@@ -542,7 +542,7 @@ describe(StorageTemplateService.name, () => {
 
       await sut.handleMigration();
 
-      expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
+      expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();
       expect(mocks.storage.rename).toHaveBeenCalledWith(
         '/original/path.jpg',
         `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`,
@@ -559,7 +559,7 @@ describe(StorageTemplateService.name, () => {
         originalPath: `upload/library/${user.id}/2022/2022-06-19/IMG_7065.heic`,
         originalFileName: 'IMG_7065.HEIC',
       });
-      mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
+      mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset]));
       mocks.user.getList.mockResolvedValue([user]);
       mocks.move.create.mockResolvedValue({
         id: '123',
@@ -571,7 +571,7 @@ describe(StorageTemplateService.name, () => {
 
       await sut.handleMigration();
 
-      expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
+      expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();
       expect(mocks.storage.rename).toHaveBeenCalledWith(
         `upload/library/${user.id}/2022/2022-06-19/IMG_7065.heic`,
         `upload/library/${user.storageLabel}/2022/2022-06-19/IMG_7065.heic`,
@@ -585,7 +585,7 @@ describe(StorageTemplateService.name, () => {
         originalPath: `upload/library/${user.id}/2022/2022-06-19/IMG_7065.HEIC`,
         originalFileName: 'IMG_7065.HEIC',
       });
-      mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
+      mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset]));
       mocks.user.getList.mockResolvedValue([user]);
       mocks.move.create.mockResolvedValue({
         id: '123',
@@ -597,7 +597,7 @@ describe(StorageTemplateService.name, () => {
 
       await sut.handleMigration();
 
-      expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
+      expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();
       expect(mocks.storage.rename).toHaveBeenCalledWith(
         `upload/library/${user.id}/2022/2022-06-19/IMG_7065.HEIC`,
         `upload/library/${user.id}/2022/2022-06-19/IMG_7065.heic`,
@@ -611,7 +611,7 @@ describe(StorageTemplateService.name, () => {
         originalPath: `upload/library/${user.id}/2022/2022-06-19/IMG_7065.JPEG`,
         originalFileName: 'IMG_7065.JPEG',
       });
-      mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
+      mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset]));
       mocks.user.getList.mockResolvedValue([user]);
       mocks.move.create.mockResolvedValue({
         id: '123',
@@ -623,7 +623,7 @@ describe(StorageTemplateService.name, () => {
 
       await sut.handleMigration();
 
-      expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
+      expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();
       expect(mocks.storage.rename).toHaveBeenCalledWith(
         `upload/library/${user.id}/2022/2022-06-19/IMG_7065.JPEG`,
         `upload/library/${user.id}/2022/2022-06-19/IMG_7065.jpg`,
@@ -637,7 +637,7 @@ describe(StorageTemplateService.name, () => {
         originalPath: 'upload/library/user-id/2022/2022-06-19/IMG_7065.JPG',
         originalFileName: 'IMG_7065.JPG',
       });
-      mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
+      mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset]));
       mocks.user.getList.mockResolvedValue([user]);
       mocks.move.create.mockResolvedValue({
         id: '123',
@@ -649,7 +649,7 @@ describe(StorageTemplateService.name, () => {
 
       await sut.handleMigration();
 
-      expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
+      expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();
       expect(mocks.storage.rename).toHaveBeenCalledWith(
         `upload/library/${user.id}/2022/2022-06-19/IMG_7065.JPG`,
         `upload/library/${user.id}/2022/2022-06-19/IMG_7065.jpg`,

server/src/services/storage-template.service.ts

@@ -118,7 +118,7 @@ export class StorageTemplateService extends BaseService {
       return JobStatus.SKIPPED;
     }
 
-    const asset = await this.assetRepository.getStorageTemplateAsset(id);
+    const asset = await this.assetJobRepository.getForStorageTemplateJob(id);
     if (!asset) {
       return JobStatus.FAILED;
     }
@@ -130,7 +130,7 @@ export class StorageTemplateService extends BaseService {
 
     // move motion part of live photo
     if (asset.livePhotoVideoId) {
-      const livePhotoVideo = await this.assetRepository.getStorageTemplateAsset(asset.livePhotoVideoId);
+      const livePhotoVideo = await this.assetJobRepository.getForStorageTemplateJob(asset.livePhotoVideoId);
       if (!livePhotoVideo) {
         return JobStatus.FAILED;
       }
@@ -152,7 +152,7 @@ export class StorageTemplateService extends BaseService {
 
     await this.moveRepository.cleanMoveHistory();
 
-    const assets = this.assetRepository.streamStorageTemplateAssets();
+    const assets = this.assetJobRepository.streamForStorageTemplateJob();
     const users = await this.userRepository.getList();
 
     for await (const asset of assets) {

server/test/medium.factory.ts

@@ -6,6 +6,7 @@ import { AssetJobStatus, Assets, DB } from 'src/db';
 import { AssetType } from 'src/enum';
 import { ActivityRepository } from 'src/repositories/activity.repository';
 import { AlbumRepository } from 'src/repositories/album.repository';
+import { AssetJobRepository } from 'src/repositories/asset-job.repository';
 import { AssetRepository } from 'src/repositories/asset.repository';
 import { ConfigRepository } from 'src/repositories/config.repository';
 import { CryptoRepository } from 'src/repositories/crypto.repository';
@@ -21,7 +22,7 @@ import { VersionHistoryRepository } from 'src/repositories/version-history.repos
 import { UserTable } from 'src/schema/tables/user.table';
 import { BaseService } from 'src/services/base.service';
 import { RepositoryInterface } from 'src/types';
-import { newUuid } from 'test/small.factory';
+import { newDate, newUuid } from 'test/small.factory';
 import { automock, ServiceOverrides } from 'test/utils';
 import { Mocked } from 'vitest';
 
@@ -30,6 +31,7 @@ type Repositories = {
   activity: ActivityRepository;
   album: AlbumRepository;
   asset: AssetRepository;
+  assetJob: AssetJobRepository;
   config: ConfigRepository;
   crypto: CryptoRepository;
   database: DatabaseRepository;
@@ -113,6 +115,10 @@ export const getRepository = <K extends keyof Repositories>(key: K, db: Kysely<D
       return new AssetRepository(db);
     }
 
+    case 'assetJob': {
+      return new AssetJobRepository(db);
+    }
+
     case 'config': {
       return new ConfigRepository();
     }
@@ -175,6 +181,10 @@ const getRepositoryMock = <K extends keyof Repositories>(key: K) => {
       return automock(AssetRepository);
     }
 
+    case 'assetJob': {
+      return automock(AssetJobRepository);
+    }
+
     case 'config': {
       return automock(ConfigRepository);
     }
@@ -237,6 +247,7 @@ export const asDeps = (repositories: ServiceOverrides) => {
     repositories.albumUser,
     repositories.apiKey,
     repositories.asset || getRepositoryMock('asset'),
+    repositories.assetJob || getRepositoryMock('assetJob'),
     repositories.audit,
     repositories.config || getRepositoryMock('config'),
     repositories.cron,
@@ -276,6 +287,7 @@ export const asDeps = (repositories: ServiceOverrides) => {
 
 const assetInsert = (asset: Partial<Insertable<Assets>> = {}) => {
   const id = asset.id || newUuid();
+  const now = newDate();
   const defaults: Insertable<Assets> = {
     deviceAssetId: '',
     deviceId: '',
@@ -285,6 +297,9 @@ const assetInsert = (asset: Partial<Insertable<Assets>> = {}) => {
     originalPath: '/path/to/something.jpg',
     ownerId: '@immich.cloud',
     isVisible: true,
+    fileCreatedAt: now,
+    fileModifiedAt: now,
+    localDateTime: now,
   };
 
   return {

server/test/repositories/asset.repository.mock.ts

@@ -11,8 +11,6 @@ export const newAssetRepositoryMock = (): Mocked<RepositoryInterface<AssetReposi
     upsertJobStatus: vitest.fn(),
     getByDayOfYear: vitest.fn(),
     getByIds: vitest.fn().mockResolvedValue([]),
-    getAssetForSearchDuplicatesJob: vitest.fn(),
-    getAssetForSidecarWriteJob: vitest.fn(),
     getByIdsWithAllRelations: vitest.fn().mockResolvedValue([]),
     getByAlbumId: vitest.fn(),
     getByDeviceIds: vitest.fn(),
@@ -47,8 +45,5 @@ export const newAssetRepositoryMock = (): Mocked<RepositoryInterface<AssetReposi
     detectOfflineExternalAssets: vitest.fn(),
     filterNewExternalAssetPaths: vitest.fn(),
     updateByLibraryId: vitest.fn(),
-    streamStorageTemplateAssets: vitest.fn(),
-    getStorageTemplateAsset: vitest.fn(),
-    streamDeletedAssets: vitest.fn(),
   };
 };

server/test/utils.ts

@@ -12,6 +12,7 @@ import { ActivityRepository } from 'src/repositories/activity.repository';
 import { AlbumUserRepository } from 'src/repositories/album-user.repository';
 import { AlbumRepository } from 'src/repositories/album.repository';
 import { ApiKeyRepository } from 'src/repositories/api-key.repository';
+import { AssetJobRepository } from 'src/repositories/asset-job.repository';
 import { AssetRepository } from 'src/repositories/asset.repository';
 import { AuditRepository } from 'src/repositories/audit.repository';
 import { ConfigRepository } from 'src/repositories/config.repository';
@@ -114,6 +115,7 @@ export type ServiceOverrides = {
   apiKey: ApiKeyRepository;
   audit: AuditRepository;
   asset: AssetRepository;
+  assetJob: AssetJobRepository;
   config: ConfigRepository;
   cron: CronRepository;
   crypto: CryptoRepository;
@@ -181,6 +183,7 @@ export const newTestService = <T extends BaseService>(
     album: automock(AlbumRepository, { strict: false }),
     albumUser: automock(AlbumUserRepository),
     asset: newAssetRepositoryMock(),
+    assetJob: automock(AssetJobRepository),
     config: newConfigRepositoryMock(),
     database: newDatabaseRepositoryMock(),
     downloadRepository: automock(DownloadRepository, { strict: false }),
@@ -227,6 +230,7 @@ export const newTestService = <T extends BaseService>(
     overrides.albumUser || (mocks.albumUser as As<AlbumUserRepository>),
     overrides.apiKey || (mocks.apiKey as As<ApiKeyRepository>),
     overrides.asset || (mocks.asset as As<AssetRepository>),
+    overrides.assetJob || (mocks.assetJob as As<AssetJobRepository>),
     overrides.audit || (mocks.audit as As<AuditRepository>),
     overrides.config || (mocks.config as As<ConfigRepository> as ConfigRepository),
     overrides.cron || (mocks.cron as As<CronRepository>),