Cutlery/immich
1
0
Fork 1
mirror of https://github.com/immich-app/immich.git synced 2025-06-03 05:34:32 -04:00
Code Issues Packages Projects Releases Wiki Activity

feat: use stream for template migrations (#16700)

Browse Source
This commit is contained in:
Jason Rasmussen 2025-03-07 14:30:01 -05:00 committed by GitHub
parent c12986d38c
commit f82786a297
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 226 additions and 208 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
server/src/repositories/asset.repository.ts
View File

@ -23,6 +23,7 @@ import {
} from 'src/entities/asset.entity'; } from 'src/entities/asset.entity';
import { AssetFileType, AssetOrder, AssetStatus, AssetType } from 'src/enum'; import { AssetFileType, AssetOrder, AssetStatus, AssetType } from 'src/enum';
import { AssetSearchOptions, SearchExploreItem, SearchExploreItemSet } from 'src/repositories/search.repository'; import { AssetSearchOptions, SearchExploreItem, SearchExploreItemSet } from 'src/repositories/search.repository';
import { StorageAsset } from 'src/types';
import { anyUuid, asUuid, removeUndefinedKeys, unnest } from 'src/utils/database'; import { anyUuid, asUuid, removeUndefinedKeys, unnest } from 'src/utils/database';
import { globToSqlPattern } from 'src/utils/misc'; import { globToSqlPattern } from 'src/utils/misc';
import { Paginated, PaginationOptions, paginationHelper } from 'src/utils/pagination'; import { Paginated, PaginationOptions, paginationHelper } from 'src/utils/pagination';
@ -604,6 +605,38 @@ export class AssetRepository {
.executeTakeFirst() as Promise<AssetEntity | undefined>; .executeTakeFirst() as Promise<AssetEntity | undefined>;
} }
private storageTemplateAssetQuery() {
return this.db
.selectFrom('assets')
.innerJoin('exif', 'assets.id', 'exif.assetId')
.select([
'assets.id',
'assets.ownerId',
'assets.type',
'assets.checksum',
'assets.originalPath',
'assets.isExternal',
'assets.sidecarPath',
'assets.originalFileName',
'assets.livePhotoVideoId',
'assets.fileCreatedAt',
'exif.timeZone',
'exif.fileSizeInByte',
])
.where('assets.deletedAt', 'is', null)
.where('assets.fileCreatedAt', 'is not', null);
}
getStorageTemplateAsset(id: string): Promise<StorageAsset | undefined> {
return this.storageTemplateAssetQuery().where('assets.id', '=', id).executeTakeFirst() as Promise<
StorageAsset | undefined
>;
}
streamStorageTemplateAssets() {
return this.storageTemplateAssetQuery().stream() as AsyncIterableIterator<StorageAsset>;
}
@GenerateSql( @GenerateSql(
...Object.values(WithProperty).map((property) => ({ ...Object.values(WithProperty).map((property) => ({
name: property, name: property,

324
server/src/services/storage-template.service.spec.ts
View File

@ -1,12 +1,14 @@
import { Stats } from 'node:fs'; import { Stats } from 'node:fs';
import { defaults, SystemConfig } from 'src/config'; import { defaults, SystemConfig } from 'src/config';
import { AssetEntity } from 'src/entities/asset.entity';
import { AssetPathType, JobStatus } from 'src/enum'; import { AssetPathType, JobStatus } from 'src/enum';
import { StorageTemplateService } from 'src/services/storage-template.service'; import { StorageTemplateService } from 'src/services/storage-template.service';
import { albumStub } from 'test/fixtures/album.stub'; import { albumStub } from 'test/fixtures/album.stub';
import { assetStub } from 'test/fixtures/asset.stub'; import { assetStub } from 'test/fixtures/asset.stub';
import { userStub } from 'test/fixtures/user.stub'; import { userStub } from 'test/fixtures/user.stub';
import { newTestService, ServiceMocks } from 'test/utils'; import { makeStream, newTestService, ServiceMocks } from 'test/utils';
const motionAsset = assetStub.storageAsset({});
const stillAsset = assetStub.storageAsset({ livePhotoVideoId: motionAsset.id });
describe(StorageTemplateService.name, () => { describe(StorageTemplateService.name, () => {
let sut: StorageTemplateService; let sut: StorageTemplateService;
@ -91,7 +93,7 @@ describe(StorageTemplateService.name, () => {
describe('handleMigrationSingle', () => { describe('handleMigrationSingle', () => {
it('should skip when storage template is disabled', async () => { it('should skip when storage template is disabled', async () => {
mocks.systemMetadata.get.mockResolvedValue({ storageTemplate: { enabled: false } }); mocks.systemMetadata.get.mockResolvedValue({ storageTemplate: { enabled: false } });
await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(JobStatus.SKIPPED); await expect(sut.handleMigrationSingle({ id: testAsset.id })).resolves.toBe(JobStatus.SKIPPED);
expect(mocks.asset.getByIds).not.toHaveBeenCalled(); expect(mocks.asset.getByIds).not.toHaveBeenCalled();
expect(mocks.storage.checkFileExists).not.toHaveBeenCalled(); expect(mocks.storage.checkFileExists).not.toHaveBeenCalled();
expect(mocks.storage.rename).not.toHaveBeenCalled(); expect(mocks.storage.rename).not.toHaveBeenCalled();
@ -104,51 +106,37 @@ describe(StorageTemplateService.name, () => {
it('should migrate single moving picture', async () => { it('should migrate single moving picture', async () => {
mocks.user.get.mockResolvedValue(userStub.user1); mocks.user.get.mockResolvedValue(userStub.user1);
const newMotionPicturePath = `upload/library/${userStub.user1.id}/2022/2022-06-19/${assetStub.livePhotoStillAsset.id}.mp4`; const newMotionPicturePath = `upload/library/${motionAsset.ownerId}/2022/2022-06-19/${motionAsset.originalFileName}`;
const newStillPicturePath = `upload/library/${userStub.user1.id}/2022/2022-06-19/${assetStub.livePhotoStillAsset.id}.jpg`; const newStillPicturePath = `upload/library/${stillAsset.ownerId}/2022/2022-06-19/${stillAsset.originalFileName}`;
mocks.asset.getByIds.mockImplementation((ids) => { mocks.asset.getStorageTemplateAsset.mockResolvedValueOnce(stillAsset);
const assets = [assetStub.livePhotoStillAsset, assetStub.livePhotoMotionAsset]; mocks.asset.getStorageTemplateAsset.mockResolvedValueOnce(motionAsset);
return Promise.resolve(
ids.map((id) => assets.find((asset) => asset.id === id)).filter((asset) => !!asset),
) as Promise<AssetEntity[]>;
});
mocks.move.create.mockResolvedValueOnce({ mocks.move.create.mockResolvedValueOnce({
id: '123', id: '123',
entityId: assetStub.livePhotoStillAsset.id, entityId: stillAsset.id,
pathType: AssetPathType.ORIGINAL, pathType: AssetPathType.ORIGINAL,
oldPath: assetStub.livePhotoStillAsset.originalPath, oldPath: stillAsset.originalPath,
newPath: newStillPicturePath, newPath: newStillPicturePath,
}); });
mocks.move.create.mockResolvedValueOnce({ mocks.move.create.mockResolvedValueOnce({
id: '124', id: '124',
entityId: assetStub.livePhotoMotionAsset.id, entityId: motionAsset.id,
pathType: AssetPathType.ORIGINAL, pathType: AssetPathType.ORIGINAL,
oldPath: assetStub.livePhotoMotionAsset.originalPath, oldPath: motionAsset.originalPath,
newPath: newMotionPicturePath, newPath: newMotionPicturePath,
}); });
await expect(sut.handleMigrationSingle({ id: assetStub.livePhotoStillAsset.id })).resolves.toBe( await expect(sut.handleMigrationSingle({ id: stillAsset.id })).resolves.toBe(JobStatus.SUCCESS);
JobStatus.SUCCESS,
);
expect(mocks.asset.getByIds).toHaveBeenCalledWith([assetStub.livePhotoStillAsset.id], { exifInfo: true });
expect(mocks.asset.getByIds).toHaveBeenCalledWith([assetStub.livePhotoMotionAsset.id], { exifInfo: true });
expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(2); expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(2);
expect(mocks.asset.update).toHaveBeenCalledWith({ expect(mocks.asset.update).toHaveBeenCalledWith({ id: stillAsset.id, originalPath: newStillPicturePath });
id: assetStub.livePhotoStillAsset.id, expect(mocks.asset.update).toHaveBeenCalledWith({ id: motionAsset.id, originalPath: newMotionPicturePath });
originalPath: newStillPicturePath,
});
expect(mocks.asset.update).toHaveBeenCalledWith({
id: assetStub.livePhotoMotionAsset.id,
originalPath: newMotionPicturePath,
});
}); });
it('should use handlebar if condition for album', async () => { it('should use handlebar if condition for album', async () => {
const asset = assetStub.image; const asset = assetStub.storageAsset();
const user = userStub.user1; const user = userStub.user1;
const album = albumStub.oneAsset; const album = albumStub.oneAsset;
const config = structuredClone(defaults); const config = structuredClone(defaults);
@ -157,7 +145,7 @@ describe(StorageTemplateService.name, () => {
sut.onConfigInit({ newConfig: config }); sut.onConfigInit({ newConfig: config });
mocks.user.get.mockResolvedValue(user); mocks.user.get.mockResolvedValue(user);
mocks.asset.getByIds.mockResolvedValueOnce([asset]); mocks.asset.getStorageTemplateAsset.mockResolvedValueOnce(asset);
mocks.album.getByAssetId.mockResolvedValueOnce([album]); mocks.album.getByAssetId.mockResolvedValueOnce([album]);
expect(await sut.handleMigrationSingle({ id: asset.id })).toBe(JobStatus.SUCCESS); expect(await sut.handleMigrationSingle({ id: asset.id })).toBe(JobStatus.SUCCESS);
@ -171,14 +159,14 @@ describe(StorageTemplateService.name, () => {
}); });
it('should use handlebar else condition for album', async () => { it('should use handlebar else condition for album', async () => {
const asset = assetStub.image; const asset = assetStub.storageAsset();
const user = userStub.user1; const user = userStub.user1;
const config = structuredClone(defaults); const config = structuredClone(defaults);
config.storageTemplate.template = '{{y}}/{{#if album}}{{album}}{{else}}other//{{MM}}{{/if}}/{{filename}}'; config.storageTemplate.template = '{{y}}/{{#if album}}{{album}}{{else}}other//{{MM}}{{/if}}/{{filename}}';
sut.onConfigInit({ newConfig: config }); sut.onConfigInit({ newConfig: config });
mocks.user.get.mockResolvedValue(user); mocks.user.get.mockResolvedValue(user);
mocks.asset.getByIds.mockResolvedValueOnce([asset]); mocks.asset.getStorageTemplateAsset.mockResolvedValueOnce(asset);
expect(await sut.handleMigrationSingle({ id: asset.id })).toBe(JobStatus.SUCCESS); expect(await sut.handleMigrationSingle({ id: asset.id })).toBe(JobStatus.SUCCESS);
@ -193,153 +181,150 @@ describe(StorageTemplateService.name, () => {
it('should migrate previously failed move from original path when it still exists', async () => { it('should migrate previously failed move from original path when it still exists', async () => {
mocks.user.get.mockResolvedValue(userStub.user1); mocks.user.get.mockResolvedValue(userStub.user1);
const previousFailedNewPath = `upload/library/${userStub.user1.id}/2023/Feb/${assetStub.image.id}.jpg`;
const newPath = `upload/library/${userStub.user1.id}/2023/2023-02-23/${assetStub.image.id}.jpg`;
mocks.storage.checkFileExists.mockImplementation((path) => const asset = assetStub.storageAsset();
Promise.resolve(path === assetStub.image.originalPath), const previousFailedNewPath = `upload/library/${userStub.user1.id}/2023/Feb/${asset.originalFileName}`;
); const newPath = `upload/library/${userStub.user1.id}/2022/2022-06-19/${asset.originalFileName}`;
mocks.storage.checkFileExists.mockImplementation((path) => Promise.resolve(path === asset.originalPath));
mocks.move.getByEntity.mockResolvedValue({ mocks.move.getByEntity.mockResolvedValue({
id: '123', id: '123',
entityId: assetStub.image.id, entityId: asset.id,
pathType: AssetPathType.ORIGINAL, pathType: AssetPathType.ORIGINAL,
oldPath: assetStub.image.originalPath, oldPath: asset.originalPath,
newPath: previousFailedNewPath, newPath: previousFailedNewPath,
}); });
mocks.asset.getByIds.mockResolvedValue([assetStub.image]); mocks.asset.getStorageTemplateAsset.mockResolvedValue(asset);
mocks.move.update.mockResolvedValue({ mocks.move.update.mockResolvedValue({
id: '123', id: '123',
entityId: assetStub.image.id, entityId: asset.id,
pathType: AssetPathType.ORIGINAL, pathType: AssetPathType.ORIGINAL,
oldPath: assetStub.image.originalPath, oldPath: asset.originalPath,
newPath, newPath,
}); });
await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(JobStatus.SUCCESS); await expect(sut.handleMigrationSingle({ id: asset.id })).resolves.toBe(JobStatus.SUCCESS);
expect(mocks.asset.getByIds).toHaveBeenCalledWith([assetStub.image.id], { exifInfo: true }); expect(mocks.asset.getStorageTemplateAsset).toHaveBeenCalledWith(asset.id);
expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(3); expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(3);
expect(mocks.storage.rename).toHaveBeenCalledWith(assetStub.image.originalPath, newPath); expect(mocks.storage.rename).toHaveBeenCalledWith(asset.originalPath, newPath);
expect(mocks.move.update).toHaveBeenCalledWith('123', { expect(mocks.move.update).toHaveBeenCalledWith('123', {
id: '123', id: '123',
oldPath: assetStub.image.originalPath, oldPath: asset.originalPath,
newPath, newPath,
}); });
expect(mocks.asset.update).toHaveBeenCalledWith({ expect(mocks.asset.update).toHaveBeenCalledWith({
id: assetStub.image.id, id: asset.id,
originalPath: newPath, originalPath: newPath,
}); });
}); });
it('should migrate previously failed move from previous new path when old path no longer exists, should validate file size still matches before moving', async () => { it('should migrate previously failed move from previous new path when old path no longer exists, should validate file size still matches before moving', async () => {
mocks.user.get.mockResolvedValue(userStub.user1); mocks.user.get.mockResolvedValue(userStub.user1);
const previousFailedNewPath = `upload/library/${userStub.user1.id}/2023/Feb/${assetStub.image.id}.jpg`;
const newPath = `upload/library/${userStub.user1.id}/2023/2023-02-23/${assetStub.image.id}.jpg`; const asset = assetStub.storageAsset({ fileSizeInByte: 5000 });
const previousFailedNewPath = `upload/library/${asset.ownerId}/2022/June/${asset.originalFileName}`;
const newPath = `upload/library/${asset.ownerId}/2022/2022-06-19/${asset.originalFileName}`;
mocks.storage.checkFileExists.mockImplementation((path) => Promise.resolve(path === previousFailedNewPath)); mocks.storage.checkFileExists.mockImplementation((path) => Promise.resolve(path === previousFailedNewPath));
mocks.storage.stat.mockResolvedValue({ size: 5000 } as Stats); mocks.storage.stat.mockResolvedValue({ size: 5000 } as Stats);
mocks.crypto.hashFile.mockResolvedValue(assetStub.image.checksum); mocks.crypto.hashFile.mockResolvedValue(asset.checksum);
mocks.move.getByEntity.mockResolvedValue({ mocks.move.getByEntity.mockResolvedValue({
id: '123', id: '123',
entityId: assetStub.image.id, entityId: asset.id,
pathType: AssetPathType.ORIGINAL, pathType: AssetPathType.ORIGINAL,
oldPath: assetStub.image.originalPath, oldPath: asset.originalPath,
newPath: previousFailedNewPath, newPath: previousFailedNewPath,
}); });
mocks.asset.getByIds.mockResolvedValue([assetStub.image]); mocks.asset.getStorageTemplateAsset.mockResolvedValue(asset);
mocks.move.update.mockResolvedValue({ mocks.move.update.mockResolvedValue({
id: '123', id: '123',
entityId: assetStub.image.id, entityId: asset.id,
pathType: AssetPathType.ORIGINAL, pathType: AssetPathType.ORIGINAL,
oldPath: previousFailedNewPath, oldPath: previousFailedNewPath,
newPath, newPath,
}); });
await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(JobStatus.SUCCESS); await expect(sut.handleMigrationSingle({ id: asset.id })).resolves.toBe(JobStatus.SUCCESS);
expect(mocks.asset.getByIds).toHaveBeenCalledWith([assetStub.image.id], { exifInfo: true }); expect(mocks.asset.getStorageTemplateAsset).toHaveBeenCalledWith(asset.id);
expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(3); expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(3);
expect(mocks.storage.stat).toHaveBeenCalledWith(previousFailedNewPath); expect(mocks.storage.stat).toHaveBeenCalledWith(previousFailedNewPath);
expect(mocks.storage.rename).toHaveBeenCalledWith(previousFailedNewPath, newPath); expect(mocks.storage.rename).toHaveBeenCalledWith(previousFailedNewPath, newPath);
expect(mocks.storage.copyFile).not.toHaveBeenCalled(); expect(mocks.storage.copyFile).not.toHaveBeenCalled();
expect(mocks.move.update).toHaveBeenCalledWith('123', { expect(mocks.move.update).toHaveBeenCalledWith('123', { id: '123', oldPath: previousFailedNewPath, newPath });
id: '123', expect(mocks.asset.update).toHaveBeenCalledWith({ id: asset.id, originalPath: newPath });
oldPath: previousFailedNewPath,
newPath,
});
expect(mocks.asset.update).toHaveBeenCalledWith({
id: assetStub.image.id,
originalPath: newPath,
});
}); });
it('should fail move if copying and hash of asset and the new file do not match', async () => { it('should fail move if copying and hash of asset and the new file do not match', async () => {
mocks.user.get.mockResolvedValue(userStub.user1); mocks.user.get.mockResolvedValue(userStub.user1);
const newPath = `upload/library/${userStub.user1.id}/2023/2023-02-23/${assetStub.image.id}.jpg`; const newPath = `upload/library/${userStub.user1.id}/2022/2022-06-19/${testAsset.originalFileName}`;
mocks.storage.rename.mockRejectedValue({ code: 'EXDEV' }); mocks.storage.rename.mockRejectedValue({ code: 'EXDEV' });
mocks.storage.stat.mockResolvedValue({ size: 5000 } as Stats); mocks.storage.stat.mockResolvedValue({ size: 5000 } as Stats);
mocks.crypto.hashFile.mockResolvedValue(Buffer.from('different-hash', 'utf8')); mocks.crypto.hashFile.mockResolvedValue(Buffer.from('different-hash', 'utf8'));
mocks.asset.getByIds.mockResolvedValue([assetStub.image]); mocks.asset.getStorageTemplateAsset.mockResolvedValue(testAsset);
mocks.move.create.mockResolvedValue({ mocks.move.create.mockResolvedValue({
id: '123', id: '123',
entityId: assetStub.image.id, entityId: testAsset.id,
pathType: AssetPathType.ORIGINAL, pathType: AssetPathType.ORIGINAL,
oldPath: assetStub.image.originalPath, oldPath: testAsset.originalPath,
newPath, newPath,
}); });
await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(JobStatus.SUCCESS); await expect(sut.handleMigrationSingle({ id: testAsset.id })).resolves.toBe(JobStatus.SUCCESS);
expect(mocks.asset.getByIds).toHaveBeenCalledWith([assetStub.image.id], { exifInfo: true }); expect(mocks.asset.getStorageTemplateAsset).toHaveBeenCalledWith(testAsset.id);
expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(1); expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(1);
expect(mocks.storage.stat).toHaveBeenCalledWith(newPath); expect(mocks.storage.stat).toHaveBeenCalledWith(newPath);
expect(mocks.move.create).toHaveBeenCalledWith({ expect(mocks.move.create).toHaveBeenCalledWith({
entityId: assetStub.image.id, entityId: testAsset.id,
pathType: AssetPathType.ORIGINAL, pathType: AssetPathType.ORIGINAL,
oldPath: assetStub.image.originalPath, oldPath: testAsset.originalPath,
newPath, newPath,
}); });
expect(mocks.storage.rename).toHaveBeenCalledWith(assetStub.image.originalPath, newPath); expect(mocks.storage.rename).toHaveBeenCalledWith(testAsset.originalPath, newPath);
expect(mocks.storage.copyFile).toHaveBeenCalledWith(assetStub.image.originalPath, newPath); expect(mocks.storage.copyFile).toHaveBeenCalledWith(testAsset.originalPath, newPath);
expect(mocks.storage.unlink).toHaveBeenCalledWith(newPath); expect(mocks.storage.unlink).toHaveBeenCalledWith(newPath);
expect(mocks.storage.unlink).toHaveBeenCalledTimes(1); expect(mocks.storage.unlink).toHaveBeenCalledTimes(1);
expect(mocks.asset.update).not.toHaveBeenCalled(); expect(mocks.asset.update).not.toHaveBeenCalled();
}); });
const testAsset = assetStub.storageAsset();
it.each` it.each`
failedPathChecksum | failedPathSize | reason failedPathChecksum | failedPathSize | reason
${assetStub.image.checksum} | ${500} | ${'file size'} ${testAsset.checksum} | ${500} | ${'file size'}
${Buffer.from('bad checksum', 'utf8')} | ${assetStub.image.exifInfo?.fileSizeInByte} | ${'checksum'} ${Buffer.from('bad checksum', 'utf8')} | ${testAsset.fileSizeInByte} | ${'checksum'}
`( `(
'should fail to migrate previously failed move from previous new path when old path no longer exists if $reason validation fails', 'should fail to migrate previously failed move from previous new path when old path no longer exists if $reason validation fails',
async ({ failedPathChecksum, failedPathSize }) => { async ({ failedPathChecksum, failedPathSize }) => {
mocks.user.get.mockResolvedValue(userStub.user1); mocks.user.get.mockResolvedValue(userStub.user1);
const previousFailedNewPath = `upload/library/${userStub.user1.id}/2023/Feb/${assetStub.image.id}.jpg`; const previousFailedNewPath = `upload/library/${userStub.user1.id}/2023/Feb/${testAsset.originalFileName}`;
const newPath = `upload/library/${userStub.user1.id}/2023/2023-02-23/${assetStub.image.id}.jpg`; const newPath = `upload/library/${userStub.user1.id}/2023/2023-02-23/${testAsset.originalFileName}`;
mocks.storage.checkFileExists.mockImplementation((path) => Promise.resolve(previousFailedNewPath === path)); mocks.storage.checkFileExists.mockImplementation((path) => Promise.resolve(previousFailedNewPath === path));
mocks.storage.stat.mockResolvedValue({ size: failedPathSize } as Stats); mocks.storage.stat.mockResolvedValue({ size: failedPathSize } as Stats);
mocks.crypto.hashFile.mockResolvedValue(failedPathChecksum); mocks.crypto.hashFile.mockResolvedValue(failedPathChecksum);
mocks.move.getByEntity.mockResolvedValue({ mocks.move.getByEntity.mockResolvedValue({
id: '123', id: '123',
entityId: assetStub.image.id, entityId: testAsset.id,
pathType: AssetPathType.ORIGINAL, pathType: AssetPathType.ORIGINAL,
oldPath: assetStub.image.originalPath, oldPath: testAsset.originalPath,
newPath: previousFailedNewPath, newPath: previousFailedNewPath,
}); });
mocks.asset.getByIds.mockResolvedValue([assetStub.image]); mocks.asset.getStorageTemplateAsset.mockResolvedValue(testAsset);
mocks.move.update.mockResolvedValue({ mocks.move.update.mockResolvedValue({
id: '123', id: '123',
entityId: assetStub.image.id, entityId: testAsset.id,
pathType: AssetPathType.ORIGINAL, pathType: AssetPathType.ORIGINAL,
oldPath: previousFailedNewPath, oldPath: previousFailedNewPath,
newPath, newPath,
}); });
await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(JobStatus.SUCCESS); await expect(sut.handleMigrationSingle({ id: testAsset.id })).resolves.toBe(JobStatus.SUCCESS);
expect(mocks.asset.getByIds).toHaveBeenCalledWith([assetStub.image.id], { exifInfo: true }); expect(mocks.asset.getStorageTemplateAsset).toHaveBeenCalledWith(testAsset.id);
expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(3); expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(3);
expect(mocks.storage.stat).toHaveBeenCalledWith(previousFailedNewPath); expect(mocks.storage.stat).toHaveBeenCalledWith(previousFailedNewPath);
expect(mocks.storage.rename).not.toHaveBeenCalled(); expect(mocks.storage.rename).not.toHaveBeenCalled();
@ -352,29 +337,28 @@ describe(StorageTemplateService.name, () => {
describe('handle template migration', () => { describe('handle template migration', () => {
it('should handle no assets', async () => { it('should handle no assets', async () => {
mocks.asset.getAll.mockResolvedValue({ mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([]));
items: [],
hasNextPage: false,
});
mocks.user.getList.mockResolvedValue([]); mocks.user.getList.mockResolvedValue([]);
await sut.handleMigration(); await sut.handleMigration();
expect(mocks.asset.getAll).toHaveBeenCalled(); expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
}); });
it('should handle an asset with a duplicate destination', async () => { it('should handle an asset with a duplicate destination', async () => {
mocks.asset.getAll.mockResolvedValue({ const asset = assetStub.storageAsset();
items: [assetStub.image], const oldPath = asset.originalPath;
hasNextPage: false, const newPath = `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`;
}); const newPath2 = newPath.replace('.jpg', '+1.jpg');
mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
mocks.user.getList.mockResolvedValue([userStub.user1]); mocks.user.getList.mockResolvedValue([userStub.user1]);
mocks.move.create.mockResolvedValue({ mocks.move.create.mockResolvedValue({
id: '123', id: '123',
entityId: assetStub.image.id, entityId: asset.id,
pathType: AssetPathType.ORIGINAL, pathType: AssetPathType.ORIGINAL,
oldPath: assetStub.image.originalPath, oldPath,
newPath: 'upload/library/user-id/2023/2023-02-23/asset-id+1.jpg', newPath,
}); });
mocks.storage.checkFileExists.mockResolvedValueOnce(true); mocks.storage.checkFileExists.mockResolvedValueOnce(true);
@ -382,30 +366,21 @@ describe(StorageTemplateService.name, () => {
await sut.handleMigration(); await sut.handleMigration();
expect(mocks.asset.getAll).toHaveBeenCalled(); expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(2); expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(2);
expect(mocks.asset.update).toHaveBeenCalledWith({ expect(mocks.asset.update).toHaveBeenCalledWith({ id: asset.id, originalPath: newPath2 });
id: assetStub.image.id,
originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id+1.jpg',
});
expect(mocks.user.getList).toHaveBeenCalled(); expect(mocks.user.getList).toHaveBeenCalled();
}); });
it('should skip when an asset already matches the template', async () => { it('should skip when an asset already matches the template', async () => {
mocks.asset.getAll.mockResolvedValue({ const asset = assetStub.storageAsset({ originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg' });
items: [
{ mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
...assetStub.image,
originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg',
},
],
hasNextPage: false,
});
mocks.user.getList.mockResolvedValue([userStub.user1]); mocks.user.getList.mockResolvedValue([userStub.user1]);
await sut.handleMigration(); await sut.handleMigration();
expect(mocks.asset.getAll).toHaveBeenCalled(); expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
expect(mocks.storage.rename).not.toHaveBeenCalled(); expect(mocks.storage.rename).not.toHaveBeenCalled();
expect(mocks.storage.copyFile).not.toHaveBeenCalled(); expect(mocks.storage.copyFile).not.toHaveBeenCalled();
expect(mocks.storage.checkFileExists).not.toHaveBeenCalledTimes(2); expect(mocks.storage.checkFileExists).not.toHaveBeenCalledTimes(2);
@ -413,20 +388,14 @@ describe(StorageTemplateService.name, () => {
}); });
it('should skip when an asset is probably a duplicate', async () => { it('should skip when an asset is probably a duplicate', async () => {
mocks.asset.getAll.mockResolvedValue({ const asset = assetStub.storageAsset({ originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id+1.jpg' });
items: [
{ mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
...assetStub.image,
originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id+1.jpg',
},
],
hasNextPage: false,
});
mocks.user.getList.mockResolvedValue([userStub.user1]); mocks.user.getList.mockResolvedValue([userStub.user1]);
await sut.handleMigration(); await sut.handleMigration();
expect(mocks.asset.getAll).toHaveBeenCalled(); expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
expect(mocks.storage.rename).not.toHaveBeenCalled(); expect(mocks.storage.rename).not.toHaveBeenCalled();
expect(mocks.storage.copyFile).not.toHaveBeenCalled(); expect(mocks.storage.copyFile).not.toHaveBeenCalled();
expect(mocks.storage.checkFileExists).not.toHaveBeenCalledTimes(2); expect(mocks.storage.checkFileExists).not.toHaveBeenCalledTimes(2);
@ -434,72 +403,63 @@ describe(StorageTemplateService.name, () => {
}); });
it('should move an asset', async () => { it('should move an asset', async () => {
mocks.asset.getAll.mockResolvedValue({ const asset = assetStub.storageAsset();
items: [assetStub.image], const oldPath = asset.originalPath;
hasNextPage: false, const newPath = `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`;
}); mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
mocks.user.getList.mockResolvedValue([userStub.user1]); mocks.user.getList.mockResolvedValue([userStub.user1]);
mocks.move.create.mockResolvedValue({ mocks.move.create.mockResolvedValue({
id: '123', id: '123',
entityId: assetStub.image.id, entityId: assetStub.image.id,
pathType: AssetPathType.ORIGINAL, pathType: AssetPathType.ORIGINAL,
oldPath: assetStub.image.originalPath, oldPath: assetStub.image.originalPath,
newPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg', newPath,
}); });
await sut.handleMigration(); await sut.handleMigration();
expect(mocks.asset.getAll).toHaveBeenCalled(); expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
expect(mocks.storage.rename).toHaveBeenCalledWith( expect(mocks.storage.rename).toHaveBeenCalledWith(oldPath, newPath);
'/original/path.jpg', expect(mocks.asset.update).toHaveBeenCalledWith({ id: asset.id, originalPath: newPath });
'upload/library/user-id/2023/2023-02-23/asset-id.jpg',
);
expect(mocks.asset.update).toHaveBeenCalledWith({
id: assetStub.image.id,
originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg',
});
}); });
it('should use the user storage label', async () => { it('should use the user storage label', async () => {
mocks.asset.getAll.mockResolvedValue({ const asset = assetStub.storageAsset();
items: [assetStub.image], mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
hasNextPage: false,
});
mocks.user.getList.mockResolvedValue([userStub.storageLabel]); mocks.user.getList.mockResolvedValue([userStub.storageLabel]);
mocks.move.create.mockResolvedValue({ mocks.move.create.mockResolvedValue({
id: '123', id: '123',
entityId: assetStub.image.id, entityId: asset.id,
pathType: AssetPathType.ORIGINAL, pathType: AssetPathType.ORIGINAL,
oldPath: assetStub.image.originalPath, oldPath: asset.originalPath,
newPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg', newPath: `upload/library/user-id/2023/2023-02-23/${asset.originalFileName}`,
}); });
await sut.handleMigration(); await sut.handleMigration();
expect(mocks.asset.getAll).toHaveBeenCalled(); expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
expect(mocks.storage.rename).toHaveBeenCalledWith( expect(mocks.storage.rename).toHaveBeenCalledWith(
'/original/path.jpg', '/original/path.jpg',
'upload/library/label-1/2023/2023-02-23/asset-id.jpg', `upload/library/label-1/2022/2022-06-19/${asset.originalFileName}`,
); );
expect(mocks.asset.update).toHaveBeenCalledWith({ expect(mocks.asset.update).toHaveBeenCalledWith({
id: assetStub.image.id, id: asset.id,
originalPath: 'upload/library/label-1/2023/2023-02-23/asset-id.jpg', originalPath: `upload/library/label-1/2022/2022-06-19/${asset.originalFileName}`,
}); });
}); });
it('should copy the file if rename fails due to EXDEV (rename across filesystems)', async () => { it('should copy the file if rename fails due to EXDEV (rename across filesystems)', async () => {
const newPath = 'upload/library/user-id/2023/2023-02-23/asset-id.jpg'; const asset = assetStub.storageAsset({ originalPath: '/path/to/original.jpg', fileSizeInByte: 5000 });
mocks.asset.getAll.mockResolvedValue({ const oldPath = asset.originalPath;
items: [assetStub.image], const newPath = `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`;
hasNextPage: false, mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
});
mocks.storage.rename.mockRejectedValue({ code: 'EXDEV' }); mocks.storage.rename.mockRejectedValue({ code: 'EXDEV' });
mocks.user.getList.mockResolvedValue([userStub.user1]); mocks.user.getList.mockResolvedValue([userStub.user1]);
mocks.move.create.mockResolvedValue({ mocks.move.create.mockResolvedValue({
id: '123', id: '123',
entityId: assetStub.image.id, entityId: asset.id,
pathType: AssetPathType.ORIGINAL, pathType: AssetPathType.ORIGINAL,
oldPath: assetStub.image.originalPath, oldPath,
newPath, newPath,
}); });
mocks.storage.stat.mockResolvedValueOnce({ mocks.storage.stat.mockResolvedValueOnce({
@ -510,40 +470,36 @@ describe(StorageTemplateService.name, () => {
size: 5000, size: 5000,
} as Stats); } as Stats);
mocks.storage.stat.mockResolvedValueOnce({ mocks.storage.stat.mockResolvedValueOnce({
size: 5000,
atime: new Date(), atime: new Date(),
mtime: new Date(), mtime: new Date(),
} as Stats); } as Stats);
mocks.crypto.hashFile.mockResolvedValue(assetStub.image.checksum); mocks.crypto.hashFile.mockResolvedValue(asset.checksum);
await sut.handleMigration(); await sut.handleMigration();
expect(mocks.asset.getAll).toHaveBeenCalled(); expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
expect(mocks.storage.rename).toHaveBeenCalledWith('/original/path.jpg', newPath); expect(mocks.storage.rename).toHaveBeenCalledWith(oldPath, newPath);
expect(mocks.storage.copyFile).toHaveBeenCalledWith('/original/path.jpg', newPath); expect(mocks.storage.copyFile).toHaveBeenCalledWith(oldPath, newPath);
expect(mocks.storage.stat).toHaveBeenCalledWith(oldPath);
expect(mocks.storage.stat).toHaveBeenCalledWith(newPath); expect(mocks.storage.stat).toHaveBeenCalledWith(newPath);
expect(mocks.storage.stat).toHaveBeenCalledWith(assetStub.image.originalPath);
expect(mocks.storage.utimes).toHaveBeenCalledWith(newPath, expect.any(Date), expect.any(Date)); expect(mocks.storage.utimes).toHaveBeenCalledWith(newPath, expect.any(Date), expect.any(Date));
expect(mocks.storage.unlink).toHaveBeenCalledWith(assetStub.image.originalPath); expect(mocks.storage.unlink).toHaveBeenCalledWith(oldPath);
expect(mocks.storage.unlink).toHaveBeenCalledTimes(1); expect(mocks.storage.unlink).toHaveBeenCalledTimes(1);
expect(mocks.asset.update).toHaveBeenCalledWith({ expect(mocks.asset.update).toHaveBeenCalledWith({ id: asset.id, originalPath: newPath });
id: assetStub.image.id,
originalPath: newPath,
});
}); });
it('should not update the database if the move fails due to incorrect newPath filesize', async () => { it('should not update the database if the move fails due to incorrect newPath filesize', async () => {
mocks.asset.getAll.mockResolvedValue({ const asset = assetStub.storageAsset();
items: [assetStub.image], mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
hasNextPage: false,
});
mocks.storage.rename.mockRejectedValue({ code: 'EXDEV' }); mocks.storage.rename.mockRejectedValue({ code: 'EXDEV' });
mocks.user.getList.mockResolvedValue([userStub.user1]); mocks.user.getList.mockResolvedValue([userStub.user1]);
mocks.move.create.mockResolvedValue({ mocks.move.create.mockResolvedValue({
id: '123', id: '123',
entityId: assetStub.image.id, entityId: asset.id,
pathType: AssetPathType.ORIGINAL, pathType: AssetPathType.ORIGINAL,
oldPath: assetStub.image.originalPath, oldPath: asset.originalPath,
newPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg', newPath: `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`,
}); });
mocks.storage.stat.mockResolvedValue({ mocks.storage.stat.mockResolvedValue({
size: 100, size: 100,
@ -551,41 +507,41 @@ describe(StorageTemplateService.name, () => {
await sut.handleMigration(); await sut.handleMigration();
expect(mocks.asset.getAll).toHaveBeenCalled(); expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
expect(mocks.storage.rename).toHaveBeenCalledWith( expect(mocks.storage.rename).toHaveBeenCalledWith(
'/original/path.jpg', '/original/path.jpg',
'upload/library/user-id/2023/2023-02-23/asset-id.jpg', `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`,
); );
expect(mocks.storage.copyFile).toHaveBeenCalledWith( expect(mocks.storage.copyFile).toHaveBeenCalledWith(
'/original/path.jpg', '/original/path.jpg',
'upload/library/user-id/2023/2023-02-23/asset-id.jpg', `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`,
);
expect(mocks.storage.stat).toHaveBeenCalledWith(
`upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`,
); );
expect(mocks.storage.stat).toHaveBeenCalledWith('upload/library/user-id/2023/2023-02-23/asset-id.jpg');
expect(mocks.asset.update).not.toHaveBeenCalled(); expect(mocks.asset.update).not.toHaveBeenCalled();
}); });
it('should not update the database if the move fails', async () => { it('should not update the database if the move fails', async () => {
mocks.asset.getAll.mockResolvedValue({ const asset = assetStub.storageAsset();
items: [assetStub.image], mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
hasNextPage: false,
});
mocks.storage.rename.mockRejectedValue(new Error('Read only system')); mocks.storage.rename.mockRejectedValue(new Error('Read only system'));
mocks.storage.copyFile.mockRejectedValue(new Error('Read only system')); mocks.storage.copyFile.mockRejectedValue(new Error('Read only system'));
mocks.move.create.mockResolvedValue({ mocks.move.create.mockResolvedValue({
id: 'move-123', id: 'move-123',
entityId: '123', entityId: asset.id,
pathType: AssetPathType.ORIGINAL, pathType: AssetPathType.ORIGINAL,
oldPath: assetStub.image.originalPath, oldPath: asset.originalPath,
newPath: '', newPath: '',
}); });
mocks.user.getList.mockResolvedValue([userStub.user1]); mocks.user.getList.mockResolvedValue([userStub.user1]);
await sut.handleMigration(); await sut.handleMigration();
expect(mocks.asset.getAll).toHaveBeenCalled(); expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
expect(mocks.storage.rename).toHaveBeenCalledWith( expect(mocks.storage.rename).toHaveBeenCalledWith(
'/original/path.jpg', '/original/path.jpg',
'upload/library/user-id/2023/2023-02-23/asset-id.jpg', `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`,
); );
expect(mocks.asset.update).not.toHaveBeenCalled(); expect(mocks.asset.update).not.toHaveBeenCalled();
}); });

43
server/src/services/storage-template.service.ts
View File

@ -3,17 +3,14 @@ import handlebar from 'handlebars';
import { DateTime } from 'luxon'; import { DateTime } from 'luxon';
import path from 'node:path'; import path from 'node:path';
import sanitize from 'sanitize-filename'; import sanitize from 'sanitize-filename';
import { JOBS_ASSET_PAGINATION_SIZE } from 'src/constants';
import { StorageCore } from 'src/cores/storage.core'; import { StorageCore } from 'src/cores/storage.core';
import { OnEvent, OnJob } from 'src/decorators'; import { OnEvent, OnJob } from 'src/decorators';
import { SystemConfigTemplateStorageOptionDto } from 'src/dtos/system-config.dto'; import { SystemConfigTemplateStorageOptionDto } from 'src/dtos/system-config.dto';
import { AssetEntity } from 'src/entities/asset.entity';
import { AssetPathType, AssetType, DatabaseLock, JobName, JobStatus, QueueName, StorageFolder } from 'src/enum'; import { AssetPathType, AssetType, DatabaseLock, JobName, JobStatus, QueueName, StorageFolder } from 'src/enum';
import { ArgOf } from 'src/repositories/event.repository'; import { ArgOf } from 'src/repositories/event.repository';
import { BaseService } from 'src/services/base.service'; import { BaseService } from 'src/services/base.service';
import { JobOf } from 'src/types'; import { JobOf, StorageAsset } from 'src/types';
import { getLivePhotoMotionFilename } from 'src/utils/file'; import { getLivePhotoMotionFilename } from 'src/utils/file';
import { usePagination } from 'src/utils/pagination';
const storageTokens = { const storageTokens = {
secondOptions: ['s', 'ss', 'SSS'], secondOptions: ['s', 'ss', 'SSS'],
@ -53,7 +50,7 @@ export interface MoveAssetMetadata {
} }
interface RenderMetadata { interface RenderMetadata {
asset: AssetEntity; asset: StorageAsset;
filename: string; filename: string;
extension: string; extension: string;
albumName: string | null; albumName: string | null;
@ -98,7 +95,7 @@ export class StorageTemplateService extends BaseService {
originalPath: '/upload/test/IMG_123.jpg', originalPath: '/upload/test/IMG_123.jpg',
type: AssetType.IMAGE, type: AssetType.IMAGE,
id: 'd587e44b-f8c0-4832-9ba3-43268bbf5d4e', id: 'd587e44b-f8c0-4832-9ba3-43268bbf5d4e',
} as AssetEntity, } as StorageAsset,
filename: 'IMG_123', filename: 'IMG_123',
extension: 'jpg', extension: 'jpg',
albumName: 'album', albumName: 'album',
@ -121,7 +118,7 @@ export class StorageTemplateService extends BaseService {
return JobStatus.SKIPPED; return JobStatus.SKIPPED;
} }
const [asset] = await this.assetRepository.getByIds([id], { exifInfo: true }); const asset = await this.assetRepository.getStorageTemplateAsset(id);
if (!asset) { if (!asset) {
return JobStatus.FAILED; return JobStatus.FAILED;
} }
@ -133,7 +130,7 @@ export class StorageTemplateService extends BaseService {
// move motion part of live photo // move motion part of live photo
if (asset.livePhotoVideoId) { if (asset.livePhotoVideoId) {
const [livePhotoVideo] = await this.assetRepository.getByIds([asset.livePhotoVideoId], { exifInfo: true }); const livePhotoVideo = await this.assetRepository.getStorageTemplateAsset(asset.livePhotoVideoId);
if (!livePhotoVideo) { if (!livePhotoVideo) {
return JobStatus.FAILED; return JobStatus.FAILED;
} }
@ -152,19 +149,17 @@ export class StorageTemplateService extends BaseService {
this.logger.log('Storage template migration disabled, skipping'); this.logger.log('Storage template migration disabled, skipping');
return JobStatus.SKIPPED; return JobStatus.SKIPPED;
} }
await this.moveRepository.cleanMoveHistory(); await this.moveRepository.cleanMoveHistory();
const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
this.assetRepository.getAll(pagination, { withExif: true, withArchived: true }), const assets = this.assetRepository.streamStorageTemplateAssets();
);
const users = await this.userRepository.getList(); const users = await this.userRepository.getList();
for await (const assets of assetPagination) { for await (const asset of assets) {
for (const asset of assets) { const user = users.find((user) => user.id === asset.ownerId);
const user = users.find((user) => user.id === asset.ownerId); const storageLabel = user?.storageLabel || null;
const storageLabel = user?.storageLabel || null; const filename = asset.originalFileName || asset.id;
const filename = asset.originalFileName || asset.id; await this.moveAsset(asset, { storageLabel, filename });
await this.moveAsset(asset, { storageLabel, filename });
}
} }
this.logger.debug('Cleaning up empty directories...'); this.logger.debug('Cleaning up empty directories...');
@ -182,7 +177,7 @@ export class StorageTemplateService extends BaseService {
await this.moveRepository.cleanMoveHistorySingle(assetId); await this.moveRepository.cleanMoveHistorySingle(assetId);
} }
async moveAsset(asset: AssetEntity, metadata: MoveAssetMetadata) { async moveAsset(asset: StorageAsset, metadata: MoveAssetMetadata) {
if (asset.isExternal || StorageCore.isAndroidMotionPath(asset.originalPath)) { if (asset.isExternal || StorageCore.isAndroidMotionPath(asset.originalPath)) {
// External assets are not affected by storage template // External assets are not affected by storage template
// TODO: shouldn't this only apply to external assets? // TODO: shouldn't this only apply to external assets?
@ -190,11 +185,11 @@ export class StorageTemplateService extends BaseService {
} }
return this.databaseRepository.withLock(DatabaseLock.StorageTemplateMigration, async () => { return this.databaseRepository.withLock(DatabaseLock.StorageTemplateMigration, async () => {
const { id, sidecarPath, originalPath, exifInfo, checksum } = asset; const { id, sidecarPath, originalPath, checksum, fileSizeInByte } = asset;
const oldPath = originalPath; const oldPath = originalPath;
const newPath = await this.getTemplatePath(asset, metadata); const newPath = await this.getTemplatePath(asset, metadata);
if (!exifInfo || !exifInfo.fileSizeInByte) { if (!fileSizeInByte) {
this.logger.error(`Asset ${id} missing exif info, skipping storage template migration`); this.logger.error(`Asset ${id} missing exif info, skipping storage template migration`);
return; return;
} }
@ -205,7 +200,7 @@ export class StorageTemplateService extends BaseService {
pathType: AssetPathType.ORIGINAL, pathType: AssetPathType.ORIGINAL,
oldPath, oldPath,
newPath, newPath,
assetInfo: { sizeInBytes: exifInfo.fileSizeInByte, checksum }, assetInfo: { sizeInBytes: fileSizeInByte, checksum },
}); });
if (sidecarPath) { if (sidecarPath) {
await this.storageCore.moveFile({ await this.storageCore.moveFile({
@ -221,7 +216,7 @@ export class StorageTemplateService extends BaseService {
}); });
} }
private async getTemplatePath(asset: AssetEntity, metadata: MoveAssetMetadata): Promise<string> { private async getTemplatePath(asset: StorageAsset, metadata: MoveAssetMetadata): Promise<string> {
const { storageLabel, filename } = metadata; const { storageLabel, filename } = metadata;
try { try {
@ -344,7 +339,7 @@ export class StorageTemplateService extends BaseService {
}; };
const systemTimeZone = Intl.DateTimeFormat().resolvedOptions().timeZone; const systemTimeZone = Intl.DateTimeFormat().resolvedOptions().timeZone;
const zone = asset.exifInfo?.timeZone || systemTimeZone; const zone = asset.timeZone || systemTimeZone;
const dt = DateTime.fromJSDate(asset.fileCreatedAt, { zone }); const dt = DateTime.fromJSDate(asset.fileCreatedAt, { zone });
for (const token of Object.values(storageTokens).flat()) { for (const token of Object.values(storageTokens).flat()) {

16
server/src/types.ts
View File

@ -1,4 +1,5 @@
import { import {
AssetType,
DatabaseExtension, DatabaseExtension,
ExifOrientation, ExifOrientation,
ImageFormat, ImageFormat,
@ -438,3 +439,18 @@ export type SyncAck = {
type: SyncEntityType; type: SyncEntityType;
updateId: string; updateId: string;
}; };
export type StorageAsset = {
id: string;
ownerId: string;
livePhotoVideoId: string | null;
type: AssetType;
isExternal: boolean;
checksum: Buffer;
timeZone: string | null;
fileCreatedAt: Date;
originalPath: string;
originalFileName: string;
sidecarPath: string | null;
fileSizeInByte: number | null;
};

16
server/test/fixtures/asset.stub.ts vendored
View File

@ -3,6 +3,7 @@ import { AssetEntity } from 'src/entities/asset.entity';
import { ExifEntity } from 'src/entities/exif.entity'; import { ExifEntity } from 'src/entities/exif.entity';
import { StackEntity } from 'src/entities/stack.entity'; import { StackEntity } from 'src/entities/stack.entity';
import { AssetFileType, AssetStatus, AssetType } from 'src/enum'; import { AssetFileType, AssetStatus, AssetType } from 'src/enum';
import { StorageAsset } from 'src/types';
import { authStub } from 'test/fixtures/auth.stub'; import { authStub } from 'test/fixtures/auth.stub';
import { fileStub } from 'test/fixtures/file.stub'; import { fileStub } from 'test/fixtures/file.stub';
import { libraryStub } from 'test/fixtures/library.stub'; import { libraryStub } from 'test/fixtures/library.stub';
@ -40,6 +41,21 @@ export const stackStub = (stackId: string, assets: AssetEntity[]): StackEntity =
}; };
export const assetStub = { export const assetStub = {
storageAsset: (asset: Partial<StorageAsset> = {}) => ({
id: 'asset-id',
ownerId: 'user-id',
livePhotoVideoId: null,
type: AssetType.IMAGE,
isExternal: false,
checksum: Buffer.from('file hash'),
timeZone: null,
fileCreatedAt: new Date('2022-06-19T23:41:36.910Z'),
originalPath: '/original/path.jpg',
originalFileName: 'IMG_123.jpg',
sidecarPath: null,
fileSizeInByte: 12_345,
...asset,
}),
noResizePath: Object.freeze<AssetEntity>({ noResizePath: Object.freeze<AssetEntity>({
id: 'asset-id', id: 'asset-id',
status: AssetStatus.ACTIVE, status: AssetStatus.ACTIVE,

2
server/test/repositories/asset.repository.mock.ts
View File

@ -44,5 +44,7 @@ export const newAssetRepositoryMock = (): Mocked<RepositoryInterface<AssetReposi
detectOfflineExternalAssets: vitest.fn(), detectOfflineExternalAssets: vitest.fn(),
filterNewExternalAssetPaths: vitest.fn(), filterNewExternalAssetPaths: vitest.fn(),
updateByLibraryId: vitest.fn(), updateByLibraryId: vitest.fn(),
streamStorageTemplateAssets: vitest.fn(),
getStorageTemplateAsset: vitest.fn(),
}; };
}; };