refactor: new asset-job repository (#17622)

* refactor: new asset-job repository

* fix: broken medium tests on main
This commit is contained in:
Jason Rasmussen 2025-04-15 10:24:51 -04:00 committed by GitHub
parent a522130122
commit 17e720440d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 239 additions and 205 deletions

View File

@ -0,0 +1,60 @@
-- NOTE: This file is auto generated by ./sql-generator
-- AssetJobRepository.getForSearchDuplicatesJob
select
"id",
"type",
"ownerId",
"duplicateId",
"stackId",
"isVisible",
"smart_search"."embedding",
(
select
coalesce(json_agg(agg), '[]')
from
(
select
"asset_files"."id",
"asset_files"."path",
"asset_files"."type"
from
"asset_files"
where
"asset_files"."assetId" = "assets"."id"
and "asset_files"."type" = $1
) as agg
) as "files"
from
"assets"
left join "smart_search" on "assets"."id" = "smart_search"."assetId"
where
"assets"."id" = $2::uuid
limit
$3
-- AssetJobRepository.getForSidecarWriteJob
select
"id",
"sidecarPath",
"originalPath",
(
select
coalesce(json_agg(agg), '[]')
from
(
select
"tags"."value"
from
"tags"
inner join "tag_asset" on "tags"."id" = "tag_asset"."tagsId"
where
"assets"."id" = "tag_asset"."assetsId"
) as agg
) as "tags"
from
"assets"
where
"assets"."id" = $1::uuid
limit
$2

View File

@ -181,65 +181,6 @@ from
where
"livePhotoVideoId" = $1::uuid
-- AssetRepository.getAssetForSearchDuplicatesJob
select
"id",
"type",
"ownerId",
"duplicateId",
"stackId",
"isVisible",
"smart_search"."embedding",
(
select
coalesce(json_agg(agg), '[]')
from
(
select
"asset_files"."id",
"asset_files"."path",
"asset_files"."type"
from
"asset_files"
where
"asset_files"."assetId" = "assets"."id"
and "asset_files"."type" = $1
) as agg
) as "files"
from
"assets"
left join "smart_search" on "assets"."id" = "smart_search"."assetId"
where
"assets"."id" = $2::uuid
limit
$3
-- AssetRepository.getAssetForSidecarWriteJob
select
"id",
"sidecarPath",
"originalPath",
(
select
coalesce(json_agg(agg), '[]')
from
(
select
"tags"."value"
from
"tags"
inner join "tag_asset" on "tags"."id" = "tag_asset"."tagsId"
where
"assets"."id" = "tag_asset"."assetsId"
) as agg
) as "tags"
from
"assets"
where
"assets"."id" = $1::uuid
limit
$2
-- AssetRepository.getById
select
"assets".*

View File

@ -0,0 +1,95 @@
import { Injectable } from '@nestjs/common';
import { Kysely } from 'kysely';
import { jsonArrayFrom } from 'kysely/helpers/postgres';
import { InjectKysely } from 'nestjs-kysely';
import { DB } from 'src/db';
import { DummyValue, GenerateSql } from 'src/decorators';
import { withFiles } from 'src/entities/asset.entity';
import { AssetFileType } from 'src/enum';
import { StorageAsset } from 'src/types';
import { asUuid } from 'src/utils/database';
@Injectable()
export class AssetJobRepository {
constructor(@InjectKysely() private db: Kysely<DB>) {}
@GenerateSql({ params: [DummyValue.UUID] })
getForSearchDuplicatesJob(id: string) {
return this.db
.selectFrom('assets')
.where('assets.id', '=', asUuid(id))
.leftJoin('smart_search', 'assets.id', 'smart_search.assetId')
.select((eb) => [
'id',
'type',
'ownerId',
'duplicateId',
'stackId',
'isVisible',
'smart_search.embedding',
withFiles(eb, AssetFileType.PREVIEW),
])
.limit(1)
.executeTakeFirst();
}
@GenerateSql({ params: [DummyValue.UUID] })
getForSidecarWriteJob(id: string) {
return this.db
.selectFrom('assets')
.where('assets.id', '=', asUuid(id))
.select((eb) => [
'id',
'sidecarPath',
'originalPath',
jsonArrayFrom(
eb
.selectFrom('tags')
.select(['tags.value'])
.innerJoin('tag_asset', 'tags.id', 'tag_asset.tagsId')
.whereRef('assets.id', '=', 'tag_asset.assetsId'),
).as('tags'),
])
.limit(1)
.executeTakeFirst();
}
private storageTemplateAssetQuery() {
return this.db
.selectFrom('assets')
.innerJoin('exif', 'assets.id', 'exif.assetId')
.select([
'assets.id',
'assets.ownerId',
'assets.type',
'assets.checksum',
'assets.originalPath',
'assets.isExternal',
'assets.sidecarPath',
'assets.originalFileName',
'assets.livePhotoVideoId',
'assets.fileCreatedAt',
'exif.timeZone',
'exif.fileSizeInByte',
])
.where('assets.deletedAt', 'is', null);
}
getForStorageTemplateJob(id: string): Promise<StorageAsset | undefined> {
return this.storageTemplateAssetQuery().where('assets.id', '=', id).executeTakeFirst() as Promise<
StorageAsset | undefined
>;
}
streamForStorageTemplateJob() {
return this.storageTemplateAssetQuery().stream() as AsyncIterableIterator<StorageAsset>;
}
streamForDeletedJob(trashedBefore: Date) {
return this.db
.selectFrom('assets')
.select(['id', 'isOffline'])
.where('assets.deletedAt', '<=', trashedBefore)
.stream();
}
}

View File

@ -1,6 +1,5 @@
import { Injectable } from '@nestjs/common';
import { Insertable, Kysely, Selectable, UpdateResult, Updateable, sql } from 'kysely';
import { jsonArrayFrom } from 'kysely/helpers/postgres';
import { isEmpty, isUndefined, omitBy } from 'lodash';
import { InjectKysely } from 'nestjs-kysely';
import { AssetFiles, AssetJobStatus, Assets, DB, Exif } from 'src/db';
@ -24,7 +23,6 @@ import {
} from 'src/entities/asset.entity';
import { AssetFileType, AssetOrder, AssetStatus, AssetType } from 'src/enum';
import { AssetSearchOptions, SearchExploreItem, SearchExploreItemSet } from 'src/repositories/search.repository';
import { StorageAsset } from 'src/types';
import { anyUuid, asUuid, removeUndefinedKeys, unnest } from 'src/utils/database';
import { globToSqlPattern } from 'src/utils/misc';
import { Paginated, PaginationOptions, paginationHelper } from 'src/utils/pagination';
@ -473,47 +471,6 @@ export class AssetRepository {
return count;
}
@GenerateSql({ params: [DummyValue.UUID] })
getAssetForSearchDuplicatesJob(id: string) {
return this.db
.selectFrom('assets')
.where('assets.id', '=', asUuid(id))
.leftJoin('smart_search', 'assets.id', 'smart_search.assetId')
.select((eb) => [
'id',
'type',
'ownerId',
'duplicateId',
'stackId',
'isVisible',
'smart_search.embedding',
withFiles(eb, AssetFileType.PREVIEW),
])
.limit(1)
.executeTakeFirst();
}
@GenerateSql({ params: [DummyValue.UUID] })
getAssetForSidecarWriteJob(id: string) {
return this.db
.selectFrom('assets')
.where('assets.id', '=', asUuid(id))
.select((eb) => [
'id',
'sidecarPath',
'originalPath',
jsonArrayFrom(
eb
.selectFrom('tags')
.select(['tags.value'])
.innerJoin('tag_asset', 'tags.id', 'tag_asset.tagsId')
.whereRef('assets.id', '=', 'tag_asset.assetsId'),
).as('tags'),
])
.limit(1)
.executeTakeFirst();
}
@GenerateSql({ params: [DummyValue.UUID] })
getById(
id: string,
@ -653,45 +610,6 @@ export class AssetRepository {
.executeTakeFirst() as Promise<AssetEntity | undefined>;
}
private storageTemplateAssetQuery() {
return this.db
.selectFrom('assets')
.innerJoin('exif', 'assets.id', 'exif.assetId')
.select([
'assets.id',
'assets.ownerId',
'assets.type',
'assets.checksum',
'assets.originalPath',
'assets.isExternal',
'assets.sidecarPath',
'assets.originalFileName',
'assets.livePhotoVideoId',
'assets.fileCreatedAt',
'exif.timeZone',
'exif.fileSizeInByte',
])
.where('assets.deletedAt', 'is', null);
}
getStorageTemplateAsset(id: string): Promise<StorageAsset | undefined> {
return this.storageTemplateAssetQuery().where('assets.id', '=', id).executeTakeFirst() as Promise<
StorageAsset | undefined
>;
}
streamStorageTemplateAssets() {
return this.storageTemplateAssetQuery().stream() as AsyncIterableIterator<StorageAsset>;
}
streamDeletedAssets(trashedBefore: Date) {
return this.db
.selectFrom('assets')
.select(['id', 'isOffline'])
.where('assets.deletedAt', '<=', trashedBefore)
.stream();
}
@GenerateSql(
...Object.values(WithProperty).map((property) => ({
name: property,

View File

@ -3,6 +3,7 @@ import { ActivityRepository } from 'src/repositories/activity.repository';
import { AlbumUserRepository } from 'src/repositories/album-user.repository';
import { AlbumRepository } from 'src/repositories/album.repository';
import { ApiKeyRepository } from 'src/repositories/api-key.repository';
import { AssetJobRepository } from 'src/repositories/asset-job.repository';
import { AssetRepository } from 'src/repositories/asset.repository';
import { AuditRepository } from 'src/repositories/audit.repository';
import { ConfigRepository } from 'src/repositories/config.repository';
@ -48,6 +49,7 @@ export const repositories = [
AuditRepository,
ApiKeyRepository,
AssetRepository,
AssetJobRepository,
ConfigRepository,
CronRepository,
CryptoRepository,

View File

@ -536,12 +536,12 @@ describe(AssetService.name, () => {
it('should immediately queue assets for deletion if trash is disabled', async () => {
const asset = factory.asset({ isOffline: false });
mocks.asset.streamDeletedAssets.mockReturnValue(makeStream([asset]));
mocks.assetJob.streamForDeletedJob.mockReturnValue(makeStream([asset]));
mocks.systemMetadata.get.mockResolvedValue({ trash: { enabled: false } });
await expect(sut.handleAssetDeletionCheck()).resolves.toBe(JobStatus.SUCCESS);
expect(mocks.asset.streamDeletedAssets).toHaveBeenCalledWith(new Date());
expect(mocks.assetJob.streamForDeletedJob).toHaveBeenCalledWith(new Date());
expect(mocks.job.queueAll).toHaveBeenCalledWith([
{ name: JobName.ASSET_DELETION, data: { id: asset.id, deleteOnDisk: true } },
]);
@ -550,12 +550,12 @@ describe(AssetService.name, () => {
it('should queue assets for deletion after trash duration', async () => {
const asset = factory.asset({ isOffline: false });
mocks.asset.streamDeletedAssets.mockReturnValue(makeStream([asset]));
mocks.assetJob.streamForDeletedJob.mockReturnValue(makeStream([asset]));
mocks.systemMetadata.get.mockResolvedValue({ trash: { enabled: true, days: 7 } });
await expect(sut.handleAssetDeletionCheck()).resolves.toBe(JobStatus.SUCCESS);
expect(mocks.asset.streamDeletedAssets).toHaveBeenCalledWith(DateTime.now().minus({ days: 7 }).toJSDate());
expect(mocks.assetJob.streamForDeletedJob).toHaveBeenCalledWith(DateTime.now().minus({ days: 7 }).toJSDate());
expect(mocks.job.queueAll).toHaveBeenCalledWith([
{ name: JobName.ASSET_DELETION, data: { id: asset.id, deleteOnDisk: true } },
]);

View File

@ -172,7 +172,7 @@ export class AssetService extends BaseService {
}
};
const assets = this.assetRepository.streamDeletedAssets(trashedBefore);
const assets = this.assetJobRepository.streamForDeletedJob(trashedBefore);
for await (const asset of assets) {
chunk.push(asset);
if (chunk.length >= JOBS_ASSET_PAGINATION_SIZE) {

View File

@ -10,6 +10,7 @@ import { ActivityRepository } from 'src/repositories/activity.repository';
import { AlbumUserRepository } from 'src/repositories/album-user.repository';
import { AlbumRepository } from 'src/repositories/album.repository';
import { ApiKeyRepository } from 'src/repositories/api-key.repository';
import { AssetJobRepository } from 'src/repositories/asset-job.repository';
import { AssetRepository } from 'src/repositories/asset.repository';
import { AuditRepository } from 'src/repositories/audit.repository';
import { ConfigRepository } from 'src/repositories/config.repository';
@ -62,6 +63,7 @@ export class BaseService {
protected albumUserRepository: AlbumUserRepository,
protected apiKeyRepository: ApiKeyRepository,
protected assetRepository: AssetRepository,
protected assetJobRepository: AssetJobRepository,
protected auditRepository: AuditRepository,
protected configRepository: ConfigRepository,
protected cronRepository: CronRepository,

View File

@ -194,6 +194,8 @@ describe(SearchService.name, () => {
});
it('should fail if asset is not found', async () => {
mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue(void 0);
const result = await sut.handleSearchDuplicates({ id: assetStub.image.id });
expect(result).toBe(JobStatus.FAILED);
@ -202,7 +204,7 @@ describe(SearchService.name, () => {
it('should skip if asset is part of stack', async () => {
const id = assetStub.primaryImage.id;
mocks.asset.getAssetForSearchDuplicatesJob.mockResolvedValue({ ...hasEmbedding, stackId: 'stack-id' });
mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue({ ...hasEmbedding, stackId: 'stack-id' });
const result = await sut.handleSearchDuplicates({ id });
@ -212,7 +214,7 @@ describe(SearchService.name, () => {
it('should skip if asset is not visible', async () => {
const id = assetStub.livePhotoMotionAsset.id;
mocks.asset.getAssetForSearchDuplicatesJob.mockResolvedValue({ ...hasEmbedding, isVisible: false });
mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue({ ...hasEmbedding, isVisible: false });
const result = await sut.handleSearchDuplicates({ id });
@ -221,7 +223,7 @@ describe(SearchService.name, () => {
});
it('should fail if asset is missing preview image', async () => {
mocks.asset.getAssetForSearchDuplicatesJob.mockResolvedValue({ ...hasEmbedding, files: [] });
mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue({ ...hasEmbedding, files: [] });
const result = await sut.handleSearchDuplicates({ id: assetStub.noResizePath.id });
@ -230,7 +232,7 @@ describe(SearchService.name, () => {
});
it('should fail if asset is missing embedding', async () => {
mocks.asset.getAssetForSearchDuplicatesJob.mockResolvedValue({ ...hasEmbedding, embedding: null });
mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue({ ...hasEmbedding, embedding: null });
const result = await sut.handleSearchDuplicates({ id: assetStub.image.id });
@ -239,7 +241,7 @@ describe(SearchService.name, () => {
});
it('should search for duplicates and update asset with duplicateId', async () => {
mocks.asset.getAssetForSearchDuplicatesJob.mockResolvedValue(hasEmbedding);
mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue(hasEmbedding);
mocks.search.searchDuplicates.mockResolvedValue([
{ assetId: assetStub.image.id, distance: 0.01, duplicateId: null },
]);
@ -267,7 +269,7 @@ describe(SearchService.name, () => {
it('should use existing duplicate ID among matched duplicates', async () => {
const duplicateId = hasDupe.duplicateId;
mocks.asset.getAssetForSearchDuplicatesJob.mockResolvedValue(hasEmbedding);
mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue(hasEmbedding);
mocks.search.searchDuplicates.mockResolvedValue([{ assetId: hasDupe.id, distance: 0.01, duplicateId }]);
const expectedAssetIds = [hasEmbedding.id];
@ -292,7 +294,7 @@ describe(SearchService.name, () => {
});
it('should remove duplicateId if no duplicates found and asset has duplicateId', async () => {
mocks.asset.getAssetForSearchDuplicatesJob.mockResolvedValue(hasDupe);
mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue(hasDupe);
mocks.search.searchDuplicates.mockResolvedValue([]);
const result = await sut.handleSearchDuplicates({ id: hasDupe.id });

View File

@ -52,7 +52,7 @@ export class DuplicateService extends BaseService {
return JobStatus.SKIPPED;
}
const asset = await this.assetRepository.getAssetForSearchDuplicatesJob(id);
const asset = await this.assetJobRepository.getForSearchDuplicatesJob(id);
if (!asset) {
this.logger.error(`Asset ${id} not found`);
return JobStatus.FAILED;

View File

@ -1486,14 +1486,14 @@ describe(MetadataService.name, () => {
describe('handleSidecarWrite', () => {
it('should skip assets that do not exist anymore', async () => {
mocks.asset.getAssetForSidecarWriteJob.mockResolvedValue(void 0);
mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(void 0);
await expect(sut.handleSidecarWrite({ id: 'asset-123' })).resolves.toBe(JobStatus.FAILED);
expect(mocks.metadata.writeTags).not.toHaveBeenCalled();
});
it('should skip jobs with no metadata', async () => {
const asset = factory.jobAssets.sidecarWrite();
mocks.asset.getAssetForSidecarWriteJob.mockResolvedValue(asset);
mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(asset);
await expect(sut.handleSidecarWrite({ id: asset.id })).resolves.toBe(JobStatus.SKIPPED);
expect(mocks.metadata.writeTags).not.toHaveBeenCalled();
});
@ -1504,7 +1504,7 @@ describe(MetadataService.name, () => {
const gps = 12;
const date = '2023-11-22T04:56:12.196Z';
mocks.asset.getAssetForSidecarWriteJob.mockResolvedValue(asset);
mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(asset);
await expect(
sut.handleSidecarWrite({
id: asset.id,

View File

@ -329,7 +329,7 @@ export class MetadataService extends BaseService {
@OnJob({ name: JobName.SIDECAR_WRITE, queue: QueueName.SIDECAR })
async handleSidecarWrite(job: JobOf<JobName.SIDECAR_WRITE>): Promise<JobStatus> {
const { id, description, dateTimeOriginal, latitude, longitude, rating, tags } = job;
const asset = await this.assetRepository.getAssetForSidecarWriteJob(id);
const asset = await this.assetJobRepository.getForSidecarWriteJob(id);
if (!asset) {
return JobStatus.FAILED;
}

View File

@ -112,8 +112,8 @@ describe(StorageTemplateService.name, () => {
const newMotionPicturePath = `upload/library/${motionAsset.ownerId}/2022/2022-06-19/${motionAsset.originalFileName}`;
const newStillPicturePath = `upload/library/${stillAsset.ownerId}/2022/2022-06-19/${stillAsset.originalFileName}`;
mocks.asset.getStorageTemplateAsset.mockResolvedValueOnce(stillAsset);
mocks.asset.getStorageTemplateAsset.mockResolvedValueOnce(motionAsset);
mocks.assetJob.getForStorageTemplateJob.mockResolvedValueOnce(stillAsset);
mocks.assetJob.getForStorageTemplateJob.mockResolvedValueOnce(motionAsset);
mocks.move.create.mockResolvedValueOnce({
id: '123',
@ -148,7 +148,7 @@ describe(StorageTemplateService.name, () => {
sut.onConfigInit({ newConfig: config });
mocks.user.get.mockResolvedValue(user);
mocks.asset.getStorageTemplateAsset.mockResolvedValueOnce(asset);
mocks.assetJob.getForStorageTemplateJob.mockResolvedValueOnce(asset);
mocks.album.getByAssetId.mockResolvedValueOnce([album]);
expect(await sut.handleMigrationSingle({ id: asset.id })).toBe(JobStatus.SUCCESS);
@ -169,7 +169,7 @@ describe(StorageTemplateService.name, () => {
sut.onConfigInit({ newConfig: config });
mocks.user.get.mockResolvedValue(user);
mocks.asset.getStorageTemplateAsset.mockResolvedValueOnce(asset);
mocks.assetJob.getForStorageTemplateJob.mockResolvedValueOnce(asset);
expect(await sut.handleMigrationSingle({ id: asset.id })).toBe(JobStatus.SUCCESS);
@ -197,7 +197,7 @@ describe(StorageTemplateService.name, () => {
oldPath: asset.originalPath,
newPath: previousFailedNewPath,
});
mocks.asset.getStorageTemplateAsset.mockResolvedValue(asset);
mocks.assetJob.getForStorageTemplateJob.mockResolvedValue(asset);
mocks.move.update.mockResolvedValue({
id: '123',
entityId: asset.id,
@ -208,7 +208,7 @@ describe(StorageTemplateService.name, () => {
await expect(sut.handleMigrationSingle({ id: asset.id })).resolves.toBe(JobStatus.SUCCESS);
expect(mocks.asset.getStorageTemplateAsset).toHaveBeenCalledWith(asset.id);
expect(mocks.assetJob.getForStorageTemplateJob).toHaveBeenCalledWith(asset.id);
expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(3);
expect(mocks.storage.rename).toHaveBeenCalledWith(asset.originalPath, newPath);
expect(mocks.move.update).toHaveBeenCalledWith('123', {
@ -239,7 +239,7 @@ describe(StorageTemplateService.name, () => {
oldPath: asset.originalPath,
newPath: previousFailedNewPath,
});
mocks.asset.getStorageTemplateAsset.mockResolvedValue(asset);
mocks.assetJob.getForStorageTemplateJob.mockResolvedValue(asset);
mocks.move.update.mockResolvedValue({
id: '123',
entityId: asset.id,
@ -250,7 +250,7 @@ describe(StorageTemplateService.name, () => {
await expect(sut.handleMigrationSingle({ id: asset.id })).resolves.toBe(JobStatus.SUCCESS);
expect(mocks.asset.getStorageTemplateAsset).toHaveBeenCalledWith(asset.id);
expect(mocks.assetJob.getForStorageTemplateJob).toHaveBeenCalledWith(asset.id);
expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(3);
expect(mocks.storage.stat).toHaveBeenCalledWith(previousFailedNewPath);
expect(mocks.storage.rename).toHaveBeenCalledWith(previousFailedNewPath, newPath);
@ -266,7 +266,7 @@ describe(StorageTemplateService.name, () => {
mocks.storage.rename.mockRejectedValue({ code: 'EXDEV' });
mocks.storage.stat.mockResolvedValue({ size: 5000 } as Stats);
mocks.crypto.hashFile.mockResolvedValue(Buffer.from('different-hash', 'utf8'));
mocks.asset.getStorageTemplateAsset.mockResolvedValue(testAsset);
mocks.assetJob.getForStorageTemplateJob.mockResolvedValue(testAsset);
mocks.move.create.mockResolvedValue({
id: '123',
entityId: testAsset.id,
@ -277,7 +277,7 @@ describe(StorageTemplateService.name, () => {
await expect(sut.handleMigrationSingle({ id: testAsset.id })).resolves.toBe(JobStatus.SUCCESS);
expect(mocks.asset.getStorageTemplateAsset).toHaveBeenCalledWith(testAsset.id);
expect(mocks.assetJob.getForStorageTemplateJob).toHaveBeenCalledWith(testAsset.id);
expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(1);
expect(mocks.storage.stat).toHaveBeenCalledWith(newPath);
expect(mocks.move.create).toHaveBeenCalledWith({
@ -316,7 +316,7 @@ describe(StorageTemplateService.name, () => {
oldPath: testAsset.originalPath,
newPath: previousFailedNewPath,
});
mocks.asset.getStorageTemplateAsset.mockResolvedValue(testAsset);
mocks.assetJob.getForStorageTemplateJob.mockResolvedValue(testAsset);
mocks.move.update.mockResolvedValue({
id: '123',
entityId: testAsset.id,
@ -327,7 +327,7 @@ describe(StorageTemplateService.name, () => {
await expect(sut.handleMigrationSingle({ id: testAsset.id })).resolves.toBe(JobStatus.SUCCESS);
expect(mocks.asset.getStorageTemplateAsset).toHaveBeenCalledWith(testAsset.id);
expect(mocks.assetJob.getForStorageTemplateJob).toHaveBeenCalledWith(testAsset.id);
expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(3);
expect(mocks.storage.stat).toHaveBeenCalledWith(previousFailedNewPath);
expect(mocks.storage.rename).not.toHaveBeenCalled();
@ -340,12 +340,12 @@ describe(StorageTemplateService.name, () => {
describe('handle template migration', () => {
it('should handle no assets', async () => {
mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([]));
mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([]));
mocks.user.getList.mockResolvedValue([]);
await sut.handleMigration();
expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();
});
it('should handle an asset with a duplicate destination', async () => {
@ -354,7 +354,7 @@ describe(StorageTemplateService.name, () => {
const newPath = `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`;
const newPath2 = newPath.replace('.jpg', '+1.jpg');
mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset]));
mocks.user.getList.mockResolvedValue([userStub.user1]);
mocks.move.create.mockResolvedValue({
id: '123',
@ -369,7 +369,7 @@ describe(StorageTemplateService.name, () => {
await sut.handleMigration();
expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();
expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(2);
expect(mocks.asset.update).toHaveBeenCalledWith({ id: asset.id, originalPath: newPath2 });
expect(mocks.user.getList).toHaveBeenCalled();
@ -378,12 +378,12 @@ describe(StorageTemplateService.name, () => {
it('should skip when an asset already matches the template', async () => {
const asset = assetStub.storageAsset({ originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg' });
mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset]));
mocks.user.getList.mockResolvedValue([userStub.user1]);
await sut.handleMigration();
expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();
expect(mocks.storage.rename).not.toHaveBeenCalled();
expect(mocks.storage.copyFile).not.toHaveBeenCalled();
expect(mocks.storage.checkFileExists).not.toHaveBeenCalledTimes(2);
@ -393,12 +393,12 @@ describe(StorageTemplateService.name, () => {
it('should skip when an asset is probably a duplicate', async () => {
const asset = assetStub.storageAsset({ originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id+1.jpg' });
mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset]));
mocks.user.getList.mockResolvedValue([userStub.user1]);
await sut.handleMigration();
expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();
expect(mocks.storage.rename).not.toHaveBeenCalled();
expect(mocks.storage.copyFile).not.toHaveBeenCalled();
expect(mocks.storage.checkFileExists).not.toHaveBeenCalledTimes(2);
@ -409,7 +409,7 @@ describe(StorageTemplateService.name, () => {
const asset = assetStub.storageAsset();
const oldPath = asset.originalPath;
const newPath = `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`;
mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset]));
mocks.user.getList.mockResolvedValue([userStub.user1]);
mocks.move.create.mockResolvedValue({
id: '123',
@ -421,7 +421,7 @@ describe(StorageTemplateService.name, () => {
await sut.handleMigration();
expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();
expect(mocks.storage.rename).toHaveBeenCalledWith(oldPath, newPath);
expect(mocks.asset.update).toHaveBeenCalledWith({ id: asset.id, originalPath: newPath });
});
@ -429,7 +429,7 @@ describe(StorageTemplateService.name, () => {
it('should use the user storage label', async () => {
const user = factory.userAdmin({ storageLabel: 'label-1' });
const asset = assetStub.storageAsset({ ownerId: user.id });
mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset]));
mocks.user.getList.mockResolvedValue([user]);
mocks.move.create.mockResolvedValue({
id: '123',
@ -441,7 +441,7 @@ describe(StorageTemplateService.name, () => {
await sut.handleMigration();
expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();
expect(mocks.storage.rename).toHaveBeenCalledWith(
'/original/path.jpg',
`upload/library/${user.storageLabel}/2022/2022-06-19/${asset.originalFileName}`,
@ -456,7 +456,7 @@ describe(StorageTemplateService.name, () => {
const asset = assetStub.storageAsset({ originalPath: '/path/to/original.jpg', fileSizeInByte: 5000 });
const oldPath = asset.originalPath;
const newPath = `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`;
mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset]));
mocks.storage.rename.mockRejectedValue({ code: 'EXDEV' });
mocks.user.getList.mockResolvedValue([userStub.user1]);
mocks.move.create.mockResolvedValue({
@ -482,7 +482,7 @@ describe(StorageTemplateService.name, () => {
await sut.handleMigration();
expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();
expect(mocks.storage.rename).toHaveBeenCalledWith(oldPath, newPath);
expect(mocks.storage.copyFile).toHaveBeenCalledWith(oldPath, newPath);
expect(mocks.storage.stat).toHaveBeenCalledWith(oldPath);
@ -495,7 +495,7 @@ describe(StorageTemplateService.name, () => {
it('should not update the database if the move fails due to incorrect newPath filesize', async () => {
const asset = assetStub.storageAsset();
mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset]));
mocks.storage.rename.mockRejectedValue({ code: 'EXDEV' });
mocks.user.getList.mockResolvedValue([userStub.user1]);
mocks.move.create.mockResolvedValue({
@ -511,7 +511,7 @@ describe(StorageTemplateService.name, () => {
await sut.handleMigration();
expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();
expect(mocks.storage.rename).toHaveBeenCalledWith(
'/original/path.jpg',
`upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`,
@ -528,7 +528,7 @@ describe(StorageTemplateService.name, () => {
it('should not update the database if the move fails', async () => {
const asset = assetStub.storageAsset();
mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset]));
mocks.storage.rename.mockRejectedValue(new Error('Read only system'));
mocks.storage.copyFile.mockRejectedValue(new Error('Read only system'));
mocks.move.create.mockResolvedValue({
@ -542,7 +542,7 @@ describe(StorageTemplateService.name, () => {
await sut.handleMigration();
expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();
expect(mocks.storage.rename).toHaveBeenCalledWith(
'/original/path.jpg',
`upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`,
@ -559,7 +559,7 @@ describe(StorageTemplateService.name, () => {
originalPath: `upload/library/${user.id}/2022/2022-06-19/IMG_7065.heic`,
originalFileName: 'IMG_7065.HEIC',
});
mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset]));
mocks.user.getList.mockResolvedValue([user]);
mocks.move.create.mockResolvedValue({
id: '123',
@ -571,7 +571,7 @@ describe(StorageTemplateService.name, () => {
await sut.handleMigration();
expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();
expect(mocks.storage.rename).toHaveBeenCalledWith(
`upload/library/${user.id}/2022/2022-06-19/IMG_7065.heic`,
`upload/library/${user.storageLabel}/2022/2022-06-19/IMG_7065.heic`,
@ -585,7 +585,7 @@ describe(StorageTemplateService.name, () => {
originalPath: `upload/library/${user.id}/2022/2022-06-19/IMG_7065.HEIC`,
originalFileName: 'IMG_7065.HEIC',
});
mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset]));
mocks.user.getList.mockResolvedValue([user]);
mocks.move.create.mockResolvedValue({
id: '123',
@ -597,7 +597,7 @@ describe(StorageTemplateService.name, () => {
await sut.handleMigration();
expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();
expect(mocks.storage.rename).toHaveBeenCalledWith(
`upload/library/${user.id}/2022/2022-06-19/IMG_7065.HEIC`,
`upload/library/${user.id}/2022/2022-06-19/IMG_7065.heic`,
@ -611,7 +611,7 @@ describe(StorageTemplateService.name, () => {
originalPath: `upload/library/${user.id}/2022/2022-06-19/IMG_7065.JPEG`,
originalFileName: 'IMG_7065.JPEG',
});
mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset]));
mocks.user.getList.mockResolvedValue([user]);
mocks.move.create.mockResolvedValue({
id: '123',
@ -623,7 +623,7 @@ describe(StorageTemplateService.name, () => {
await sut.handleMigration();
expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();
expect(mocks.storage.rename).toHaveBeenCalledWith(
`upload/library/${user.id}/2022/2022-06-19/IMG_7065.JPEG`,
`upload/library/${user.id}/2022/2022-06-19/IMG_7065.jpg`,
@ -637,7 +637,7 @@ describe(StorageTemplateService.name, () => {
originalPath: 'upload/library/user-id/2022/2022-06-19/IMG_7065.JPG',
originalFileName: 'IMG_7065.JPG',
});
mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
mocks.assetJob.streamForStorageTemplateJob.mockReturnValue(makeStream([asset]));
mocks.user.getList.mockResolvedValue([user]);
mocks.move.create.mockResolvedValue({
id: '123',
@ -649,7 +649,7 @@ describe(StorageTemplateService.name, () => {
await sut.handleMigration();
expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
expect(mocks.assetJob.streamForStorageTemplateJob).toHaveBeenCalled();
expect(mocks.storage.rename).toHaveBeenCalledWith(
`upload/library/${user.id}/2022/2022-06-19/IMG_7065.JPG`,
`upload/library/${user.id}/2022/2022-06-19/IMG_7065.jpg`,

View File

@ -118,7 +118,7 @@ export class StorageTemplateService extends BaseService {
return JobStatus.SKIPPED;
}
const asset = await this.assetRepository.getStorageTemplateAsset(id);
const asset = await this.assetJobRepository.getForStorageTemplateJob(id);
if (!asset) {
return JobStatus.FAILED;
}
@ -130,7 +130,7 @@ export class StorageTemplateService extends BaseService {
// move motion part of live photo
if (asset.livePhotoVideoId) {
const livePhotoVideo = await this.assetRepository.getStorageTemplateAsset(asset.livePhotoVideoId);
const livePhotoVideo = await this.assetJobRepository.getForStorageTemplateJob(asset.livePhotoVideoId);
if (!livePhotoVideo) {
return JobStatus.FAILED;
}
@ -152,7 +152,7 @@ export class StorageTemplateService extends BaseService {
await this.moveRepository.cleanMoveHistory();
const assets = this.assetRepository.streamStorageTemplateAssets();
const assets = this.assetJobRepository.streamForStorageTemplateJob();
const users = await this.userRepository.getList();
for await (const asset of assets) {

View File

@ -6,6 +6,7 @@ import { AssetJobStatus, Assets, DB } from 'src/db';
import { AssetType } from 'src/enum';
import { ActivityRepository } from 'src/repositories/activity.repository';
import { AlbumRepository } from 'src/repositories/album.repository';
import { AssetJobRepository } from 'src/repositories/asset-job.repository';
import { AssetRepository } from 'src/repositories/asset.repository';
import { ConfigRepository } from 'src/repositories/config.repository';
import { CryptoRepository } from 'src/repositories/crypto.repository';
@ -21,7 +22,7 @@ import { VersionHistoryRepository } from 'src/repositories/version-history.repos
import { UserTable } from 'src/schema/tables/user.table';
import { BaseService } from 'src/services/base.service';
import { RepositoryInterface } from 'src/types';
import { newUuid } from 'test/small.factory';
import { newDate, newUuid } from 'test/small.factory';
import { automock, ServiceOverrides } from 'test/utils';
import { Mocked } from 'vitest';
@ -30,6 +31,7 @@ type Repositories = {
activity: ActivityRepository;
album: AlbumRepository;
asset: AssetRepository;
assetJob: AssetJobRepository;
config: ConfigRepository;
crypto: CryptoRepository;
database: DatabaseRepository;
@ -113,6 +115,10 @@ export const getRepository = <K extends keyof Repositories>(key: K, db: Kysely<D
return new AssetRepository(db);
}
case 'assetJob': {
return new AssetJobRepository(db);
}
case 'config': {
return new ConfigRepository();
}
@ -175,6 +181,10 @@ const getRepositoryMock = <K extends keyof Repositories>(key: K) => {
return automock(AssetRepository);
}
case 'assetJob': {
return automock(AssetJobRepository);
}
case 'config': {
return automock(ConfigRepository);
}
@ -237,6 +247,7 @@ export const asDeps = (repositories: ServiceOverrides) => {
repositories.albumUser,
repositories.apiKey,
repositories.asset || getRepositoryMock('asset'),
repositories.assetJob || getRepositoryMock('assetJob'),
repositories.audit,
repositories.config || getRepositoryMock('config'),
repositories.cron,
@ -276,6 +287,7 @@ export const asDeps = (repositories: ServiceOverrides) => {
const assetInsert = (asset: Partial<Insertable<Assets>> = {}) => {
const id = asset.id || newUuid();
const now = newDate();
const defaults: Insertable<Assets> = {
deviceAssetId: '',
deviceId: '',
@ -285,6 +297,9 @@ const assetInsert = (asset: Partial<Insertable<Assets>> = {}) => {
originalPath: '/path/to/something.jpg',
ownerId: '@immich.cloud',
isVisible: true,
fileCreatedAt: now,
fileModifiedAt: now,
localDateTime: now,
};
return {

View File

@ -11,8 +11,6 @@ export const newAssetRepositoryMock = (): Mocked<RepositoryInterface<AssetReposi
upsertJobStatus: vitest.fn(),
getByDayOfYear: vitest.fn(),
getByIds: vitest.fn().mockResolvedValue([]),
getAssetForSearchDuplicatesJob: vitest.fn(),
getAssetForSidecarWriteJob: vitest.fn(),
getByIdsWithAllRelations: vitest.fn().mockResolvedValue([]),
getByAlbumId: vitest.fn(),
getByDeviceIds: vitest.fn(),
@ -47,8 +45,5 @@ export const newAssetRepositoryMock = (): Mocked<RepositoryInterface<AssetReposi
detectOfflineExternalAssets: vitest.fn(),
filterNewExternalAssetPaths: vitest.fn(),
updateByLibraryId: vitest.fn(),
streamStorageTemplateAssets: vitest.fn(),
getStorageTemplateAsset: vitest.fn(),
streamDeletedAssets: vitest.fn(),
};
};

View File

@ -12,6 +12,7 @@ import { ActivityRepository } from 'src/repositories/activity.repository';
import { AlbumUserRepository } from 'src/repositories/album-user.repository';
import { AlbumRepository } from 'src/repositories/album.repository';
import { ApiKeyRepository } from 'src/repositories/api-key.repository';
import { AssetJobRepository } from 'src/repositories/asset-job.repository';
import { AssetRepository } from 'src/repositories/asset.repository';
import { AuditRepository } from 'src/repositories/audit.repository';
import { ConfigRepository } from 'src/repositories/config.repository';
@ -114,6 +115,7 @@ export type ServiceOverrides = {
apiKey: ApiKeyRepository;
audit: AuditRepository;
asset: AssetRepository;
assetJob: AssetJobRepository;
config: ConfigRepository;
cron: CronRepository;
crypto: CryptoRepository;
@ -181,6 +183,7 @@ export const newTestService = <T extends BaseService>(
album: automock(AlbumRepository, { strict: false }),
albumUser: automock(AlbumUserRepository),
asset: newAssetRepositoryMock(),
assetJob: automock(AssetJobRepository),
config: newConfigRepositoryMock(),
database: newDatabaseRepositoryMock(),
downloadRepository: automock(DownloadRepository, { strict: false }),
@ -227,6 +230,7 @@ export const newTestService = <T extends BaseService>(
overrides.albumUser || (mocks.albumUser as As<AlbumUserRepository>),
overrides.apiKey || (mocks.apiKey as As<ApiKeyRepository>),
overrides.asset || (mocks.asset as As<AssetRepository>),
overrides.assetJob || (mocks.assetJob as As<AssetJobRepository>),
overrides.audit || (mocks.audit as As<AuditRepository>),
overrides.config || (mocks.config as As<ConfigRepository> as ConfigRepository),
overrides.cron || (mocks.cron as As<CronRepository>),