diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts index 6274c88dfe..1efcb3875b 100644 --- a/server/src/repositories/asset.repository.ts +++ b/server/src/repositories/asset.repository.ts @@ -23,6 +23,7 @@ import { } from 'src/entities/asset.entity'; import { AssetFileType, AssetOrder, AssetStatus, AssetType } from 'src/enum'; import { AssetSearchOptions, SearchExploreItem, SearchExploreItemSet } from 'src/repositories/search.repository'; +import { StorageAsset } from 'src/types'; import { anyUuid, asUuid, removeUndefinedKeys, unnest } from 'src/utils/database'; import { globToSqlPattern } from 'src/utils/misc'; import { Paginated, PaginationOptions, paginationHelper } from 'src/utils/pagination'; @@ -604,6 +605,38 @@ export class AssetRepository { .executeTakeFirst() as Promise; } + private storageTemplateAssetQuery() { + return this.db + .selectFrom('assets') + .innerJoin('exif', 'assets.id', 'exif.assetId') + .select([ + 'assets.id', + 'assets.ownerId', + 'assets.type', + 'assets.checksum', + 'assets.originalPath', + 'assets.isExternal', + 'assets.sidecarPath', + 'assets.originalFileName', + 'assets.livePhotoVideoId', + 'assets.fileCreatedAt', + 'exif.timeZone', + 'exif.fileSizeInByte', + ]) + .where('assets.deletedAt', 'is', null) + .where('assets.fileCreatedAt', 'is not', null); + } + + getStorageTemplateAsset(id: string): Promise { + return this.storageTemplateAssetQuery().where('assets.id', '=', id).executeTakeFirst() as Promise< + StorageAsset | undefined + >; + } + + streamStorageTemplateAssets() { + return this.storageTemplateAssetQuery().stream() as AsyncIterableIterator; + } + @GenerateSql( ...Object.values(WithProperty).map((property) => ({ name: property, diff --git a/server/src/services/storage-template.service.spec.ts b/server/src/services/storage-template.service.spec.ts index 9b5f5a483d..1133d25db4 100644 --- a/server/src/services/storage-template.service.spec.ts +++ b/server/src/services/storage-template.service.spec.ts @@ -1,12 +1,14 @@ import { Stats } from 'node:fs'; import { defaults, SystemConfig } from 'src/config'; -import { AssetEntity } from 'src/entities/asset.entity'; import { AssetPathType, JobStatus } from 'src/enum'; import { StorageTemplateService } from 'src/services/storage-template.service'; import { albumStub } from 'test/fixtures/album.stub'; import { assetStub } from 'test/fixtures/asset.stub'; import { userStub } from 'test/fixtures/user.stub'; -import { newTestService, ServiceMocks } from 'test/utils'; +import { makeStream, newTestService, ServiceMocks } from 'test/utils'; + +const motionAsset = assetStub.storageAsset({}); +const stillAsset = assetStub.storageAsset({ livePhotoVideoId: motionAsset.id }); describe(StorageTemplateService.name, () => { let sut: StorageTemplateService; @@ -91,7 +93,7 @@ describe(StorageTemplateService.name, () => { describe('handleMigrationSingle', () => { it('should skip when storage template is disabled', async () => { mocks.systemMetadata.get.mockResolvedValue({ storageTemplate: { enabled: false } }); - await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(JobStatus.SKIPPED); + await expect(sut.handleMigrationSingle({ id: testAsset.id })).resolves.toBe(JobStatus.SKIPPED); expect(mocks.asset.getByIds).not.toHaveBeenCalled(); expect(mocks.storage.checkFileExists).not.toHaveBeenCalled(); expect(mocks.storage.rename).not.toHaveBeenCalled(); @@ -104,51 +106,37 @@ describe(StorageTemplateService.name, () => { it('should migrate single moving picture', async () => { mocks.user.get.mockResolvedValue(userStub.user1); - const newMotionPicturePath = `upload/library/${userStub.user1.id}/2022/2022-06-19/${assetStub.livePhotoStillAsset.id}.mp4`; - const newStillPicturePath = `upload/library/${userStub.user1.id}/2022/2022-06-19/${assetStub.livePhotoStillAsset.id}.jpg`; + const newMotionPicturePath = `upload/library/${motionAsset.ownerId}/2022/2022-06-19/${motionAsset.originalFileName}`; + const newStillPicturePath = `upload/library/${stillAsset.ownerId}/2022/2022-06-19/${stillAsset.originalFileName}`; - mocks.asset.getByIds.mockImplementation((ids) => { - const assets = [assetStub.livePhotoStillAsset, assetStub.livePhotoMotionAsset]; - return Promise.resolve( - ids.map((id) => assets.find((asset) => asset.id === id)).filter((asset) => !!asset), - ) as Promise; - }); + mocks.asset.getStorageTemplateAsset.mockResolvedValueOnce(stillAsset); + mocks.asset.getStorageTemplateAsset.mockResolvedValueOnce(motionAsset); mocks.move.create.mockResolvedValueOnce({ id: '123', - entityId: assetStub.livePhotoStillAsset.id, + entityId: stillAsset.id, pathType: AssetPathType.ORIGINAL, - oldPath: assetStub.livePhotoStillAsset.originalPath, + oldPath: stillAsset.originalPath, newPath: newStillPicturePath, }); mocks.move.create.mockResolvedValueOnce({ id: '124', - entityId: assetStub.livePhotoMotionAsset.id, + entityId: motionAsset.id, pathType: AssetPathType.ORIGINAL, - oldPath: assetStub.livePhotoMotionAsset.originalPath, + oldPath: motionAsset.originalPath, newPath: newMotionPicturePath, }); - await expect(sut.handleMigrationSingle({ id: assetStub.livePhotoStillAsset.id })).resolves.toBe( - JobStatus.SUCCESS, - ); + await expect(sut.handleMigrationSingle({ id: stillAsset.id })).resolves.toBe(JobStatus.SUCCESS); - expect(mocks.asset.getByIds).toHaveBeenCalledWith([assetStub.livePhotoStillAsset.id], { exifInfo: true }); - expect(mocks.asset.getByIds).toHaveBeenCalledWith([assetStub.livePhotoMotionAsset.id], { exifInfo: true }); expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(2); - expect(mocks.asset.update).toHaveBeenCalledWith({ - id: assetStub.livePhotoStillAsset.id, - originalPath: newStillPicturePath, - }); - expect(mocks.asset.update).toHaveBeenCalledWith({ - id: assetStub.livePhotoMotionAsset.id, - originalPath: newMotionPicturePath, - }); + expect(mocks.asset.update).toHaveBeenCalledWith({ id: stillAsset.id, originalPath: newStillPicturePath }); + expect(mocks.asset.update).toHaveBeenCalledWith({ id: motionAsset.id, originalPath: newMotionPicturePath }); }); it('should use handlebar if condition for album', async () => { - const asset = assetStub.image; + const asset = assetStub.storageAsset(); const user = userStub.user1; const album = albumStub.oneAsset; const config = structuredClone(defaults); @@ -157,7 +145,7 @@ describe(StorageTemplateService.name, () => { sut.onConfigInit({ newConfig: config }); mocks.user.get.mockResolvedValue(user); - mocks.asset.getByIds.mockResolvedValueOnce([asset]); + mocks.asset.getStorageTemplateAsset.mockResolvedValueOnce(asset); mocks.album.getByAssetId.mockResolvedValueOnce([album]); expect(await sut.handleMigrationSingle({ id: asset.id })).toBe(JobStatus.SUCCESS); @@ -171,14 +159,14 @@ describe(StorageTemplateService.name, () => { }); it('should use handlebar else condition for album', async () => { - const asset = assetStub.image; + const asset = assetStub.storageAsset(); const user = userStub.user1; const config = structuredClone(defaults); config.storageTemplate.template = '{{y}}/{{#if album}}{{album}}{{else}}other//{{MM}}{{/if}}/{{filename}}'; sut.onConfigInit({ newConfig: config }); mocks.user.get.mockResolvedValue(user); - mocks.asset.getByIds.mockResolvedValueOnce([asset]); + mocks.asset.getStorageTemplateAsset.mockResolvedValueOnce(asset); expect(await sut.handleMigrationSingle({ id: asset.id })).toBe(JobStatus.SUCCESS); @@ -193,153 +181,150 @@ describe(StorageTemplateService.name, () => { it('should migrate previously failed move from original path when it still exists', async () => { mocks.user.get.mockResolvedValue(userStub.user1); - const previousFailedNewPath = `upload/library/${userStub.user1.id}/2023/Feb/${assetStub.image.id}.jpg`; - const newPath = `upload/library/${userStub.user1.id}/2023/2023-02-23/${assetStub.image.id}.jpg`; - mocks.storage.checkFileExists.mockImplementation((path) => - Promise.resolve(path === assetStub.image.originalPath), - ); + const asset = assetStub.storageAsset(); + const previousFailedNewPath = `upload/library/${userStub.user1.id}/2023/Feb/${asset.originalFileName}`; + const newPath = `upload/library/${userStub.user1.id}/2022/2022-06-19/${asset.originalFileName}`; + + mocks.storage.checkFileExists.mockImplementation((path) => Promise.resolve(path === asset.originalPath)); mocks.move.getByEntity.mockResolvedValue({ id: '123', - entityId: assetStub.image.id, + entityId: asset.id, pathType: AssetPathType.ORIGINAL, - oldPath: assetStub.image.originalPath, + oldPath: asset.originalPath, newPath: previousFailedNewPath, }); - mocks.asset.getByIds.mockResolvedValue([assetStub.image]); + mocks.asset.getStorageTemplateAsset.mockResolvedValue(asset); mocks.move.update.mockResolvedValue({ id: '123', - entityId: assetStub.image.id, + entityId: asset.id, pathType: AssetPathType.ORIGINAL, - oldPath: assetStub.image.originalPath, + oldPath: asset.originalPath, newPath, }); - await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(JobStatus.SUCCESS); + await expect(sut.handleMigrationSingle({ id: asset.id })).resolves.toBe(JobStatus.SUCCESS); - expect(mocks.asset.getByIds).toHaveBeenCalledWith([assetStub.image.id], { exifInfo: true }); + expect(mocks.asset.getStorageTemplateAsset).toHaveBeenCalledWith(asset.id); expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(3); - expect(mocks.storage.rename).toHaveBeenCalledWith(assetStub.image.originalPath, newPath); + expect(mocks.storage.rename).toHaveBeenCalledWith(asset.originalPath, newPath); expect(mocks.move.update).toHaveBeenCalledWith('123', { id: '123', - oldPath: assetStub.image.originalPath, + oldPath: asset.originalPath, newPath, }); expect(mocks.asset.update).toHaveBeenCalledWith({ - id: assetStub.image.id, + id: asset.id, originalPath: newPath, }); }); it('should migrate previously failed move from previous new path when old path no longer exists, should validate file size still matches before moving', async () => { mocks.user.get.mockResolvedValue(userStub.user1); - const previousFailedNewPath = `upload/library/${userStub.user1.id}/2023/Feb/${assetStub.image.id}.jpg`; - const newPath = `upload/library/${userStub.user1.id}/2023/2023-02-23/${assetStub.image.id}.jpg`; + + const asset = assetStub.storageAsset({ fileSizeInByte: 5000 }); + const previousFailedNewPath = `upload/library/${asset.ownerId}/2022/June/${asset.originalFileName}`; + const newPath = `upload/library/${asset.ownerId}/2022/2022-06-19/${asset.originalFileName}`; mocks.storage.checkFileExists.mockImplementation((path) => Promise.resolve(path === previousFailedNewPath)); mocks.storage.stat.mockResolvedValue({ size: 5000 } as Stats); - mocks.crypto.hashFile.mockResolvedValue(assetStub.image.checksum); + mocks.crypto.hashFile.mockResolvedValue(asset.checksum); mocks.move.getByEntity.mockResolvedValue({ id: '123', - entityId: assetStub.image.id, + entityId: asset.id, pathType: AssetPathType.ORIGINAL, - oldPath: assetStub.image.originalPath, + oldPath: asset.originalPath, newPath: previousFailedNewPath, }); - mocks.asset.getByIds.mockResolvedValue([assetStub.image]); + mocks.asset.getStorageTemplateAsset.mockResolvedValue(asset); mocks.move.update.mockResolvedValue({ id: '123', - entityId: assetStub.image.id, + entityId: asset.id, pathType: AssetPathType.ORIGINAL, oldPath: previousFailedNewPath, newPath, }); - await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(JobStatus.SUCCESS); + await expect(sut.handleMigrationSingle({ id: asset.id })).resolves.toBe(JobStatus.SUCCESS); - expect(mocks.asset.getByIds).toHaveBeenCalledWith([assetStub.image.id], { exifInfo: true }); + expect(mocks.asset.getStorageTemplateAsset).toHaveBeenCalledWith(asset.id); expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(3); expect(mocks.storage.stat).toHaveBeenCalledWith(previousFailedNewPath); expect(mocks.storage.rename).toHaveBeenCalledWith(previousFailedNewPath, newPath); expect(mocks.storage.copyFile).not.toHaveBeenCalled(); - expect(mocks.move.update).toHaveBeenCalledWith('123', { - id: '123', - oldPath: previousFailedNewPath, - newPath, - }); - expect(mocks.asset.update).toHaveBeenCalledWith({ - id: assetStub.image.id, - originalPath: newPath, - }); + expect(mocks.move.update).toHaveBeenCalledWith('123', { id: '123', oldPath: previousFailedNewPath, newPath }); + expect(mocks.asset.update).toHaveBeenCalledWith({ id: asset.id, originalPath: newPath }); }); it('should fail move if copying and hash of asset and the new file do not match', async () => { mocks.user.get.mockResolvedValue(userStub.user1); - const newPath = `upload/library/${userStub.user1.id}/2023/2023-02-23/${assetStub.image.id}.jpg`; + const newPath = `upload/library/${userStub.user1.id}/2022/2022-06-19/${testAsset.originalFileName}`; mocks.storage.rename.mockRejectedValue({ code: 'EXDEV' }); mocks.storage.stat.mockResolvedValue({ size: 5000 } as Stats); mocks.crypto.hashFile.mockResolvedValue(Buffer.from('different-hash', 'utf8')); - mocks.asset.getByIds.mockResolvedValue([assetStub.image]); + mocks.asset.getStorageTemplateAsset.mockResolvedValue(testAsset); mocks.move.create.mockResolvedValue({ id: '123', - entityId: assetStub.image.id, + entityId: testAsset.id, pathType: AssetPathType.ORIGINAL, - oldPath: assetStub.image.originalPath, + oldPath: testAsset.originalPath, newPath, }); - await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(JobStatus.SUCCESS); + await expect(sut.handleMigrationSingle({ id: testAsset.id })).resolves.toBe(JobStatus.SUCCESS); - expect(mocks.asset.getByIds).toHaveBeenCalledWith([assetStub.image.id], { exifInfo: true }); + expect(mocks.asset.getStorageTemplateAsset).toHaveBeenCalledWith(testAsset.id); expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(1); expect(mocks.storage.stat).toHaveBeenCalledWith(newPath); expect(mocks.move.create).toHaveBeenCalledWith({ - entityId: assetStub.image.id, + entityId: testAsset.id, pathType: AssetPathType.ORIGINAL, - oldPath: assetStub.image.originalPath, + oldPath: testAsset.originalPath, newPath, }); - expect(mocks.storage.rename).toHaveBeenCalledWith(assetStub.image.originalPath, newPath); - expect(mocks.storage.copyFile).toHaveBeenCalledWith(assetStub.image.originalPath, newPath); + expect(mocks.storage.rename).toHaveBeenCalledWith(testAsset.originalPath, newPath); + expect(mocks.storage.copyFile).toHaveBeenCalledWith(testAsset.originalPath, newPath); expect(mocks.storage.unlink).toHaveBeenCalledWith(newPath); expect(mocks.storage.unlink).toHaveBeenCalledTimes(1); expect(mocks.asset.update).not.toHaveBeenCalled(); }); + const testAsset = assetStub.storageAsset(); + it.each` - failedPathChecksum | failedPathSize | reason - ${assetStub.image.checksum} | ${500} | ${'file size'} - ${Buffer.from('bad checksum', 'utf8')} | ${assetStub.image.exifInfo?.fileSizeInByte} | ${'checksum'} + failedPathChecksum | failedPathSize | reason + ${testAsset.checksum} | ${500} | ${'file size'} + ${Buffer.from('bad checksum', 'utf8')} | ${testAsset.fileSizeInByte} | ${'checksum'} `( 'should fail to migrate previously failed move from previous new path when old path no longer exists if $reason validation fails', async ({ failedPathChecksum, failedPathSize }) => { mocks.user.get.mockResolvedValue(userStub.user1); - const previousFailedNewPath = `upload/library/${userStub.user1.id}/2023/Feb/${assetStub.image.id}.jpg`; - const newPath = `upload/library/${userStub.user1.id}/2023/2023-02-23/${assetStub.image.id}.jpg`; + const previousFailedNewPath = `upload/library/${userStub.user1.id}/2023/Feb/${testAsset.originalFileName}`; + const newPath = `upload/library/${userStub.user1.id}/2023/2023-02-23/${testAsset.originalFileName}`; mocks.storage.checkFileExists.mockImplementation((path) => Promise.resolve(previousFailedNewPath === path)); mocks.storage.stat.mockResolvedValue({ size: failedPathSize } as Stats); mocks.crypto.hashFile.mockResolvedValue(failedPathChecksum); mocks.move.getByEntity.mockResolvedValue({ id: '123', - entityId: assetStub.image.id, + entityId: testAsset.id, pathType: AssetPathType.ORIGINAL, - oldPath: assetStub.image.originalPath, + oldPath: testAsset.originalPath, newPath: previousFailedNewPath, }); - mocks.asset.getByIds.mockResolvedValue([assetStub.image]); + mocks.asset.getStorageTemplateAsset.mockResolvedValue(testAsset); mocks.move.update.mockResolvedValue({ id: '123', - entityId: assetStub.image.id, + entityId: testAsset.id, pathType: AssetPathType.ORIGINAL, oldPath: previousFailedNewPath, newPath, }); - await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(JobStatus.SUCCESS); + await expect(sut.handleMigrationSingle({ id: testAsset.id })).resolves.toBe(JobStatus.SUCCESS); - expect(mocks.asset.getByIds).toHaveBeenCalledWith([assetStub.image.id], { exifInfo: true }); + expect(mocks.asset.getStorageTemplateAsset).toHaveBeenCalledWith(testAsset.id); expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(3); expect(mocks.storage.stat).toHaveBeenCalledWith(previousFailedNewPath); expect(mocks.storage.rename).not.toHaveBeenCalled(); @@ -352,29 +337,28 @@ describe(StorageTemplateService.name, () => { describe('handle template migration', () => { it('should handle no assets', async () => { - mocks.asset.getAll.mockResolvedValue({ - items: [], - hasNextPage: false, - }); + mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([])); mocks.user.getList.mockResolvedValue([]); await sut.handleMigration(); - expect(mocks.asset.getAll).toHaveBeenCalled(); + expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled(); }); it('should handle an asset with a duplicate destination', async () => { - mocks.asset.getAll.mockResolvedValue({ - items: [assetStub.image], - hasNextPage: false, - }); + const asset = assetStub.storageAsset(); + const oldPath = asset.originalPath; + const newPath = `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`; + const newPath2 = newPath.replace('.jpg', '+1.jpg'); + + mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset])); mocks.user.getList.mockResolvedValue([userStub.user1]); mocks.move.create.mockResolvedValue({ id: '123', - entityId: assetStub.image.id, + entityId: asset.id, pathType: AssetPathType.ORIGINAL, - oldPath: assetStub.image.originalPath, - newPath: 'upload/library/user-id/2023/2023-02-23/asset-id+1.jpg', + oldPath, + newPath, }); mocks.storage.checkFileExists.mockResolvedValueOnce(true); @@ -382,30 +366,21 @@ describe(StorageTemplateService.name, () => { await sut.handleMigration(); - expect(mocks.asset.getAll).toHaveBeenCalled(); + expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled(); expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(2); - expect(mocks.asset.update).toHaveBeenCalledWith({ - id: assetStub.image.id, - originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id+1.jpg', - }); + expect(mocks.asset.update).toHaveBeenCalledWith({ id: asset.id, originalPath: newPath2 }); expect(mocks.user.getList).toHaveBeenCalled(); }); it('should skip when an asset already matches the template', async () => { - mocks.asset.getAll.mockResolvedValue({ - items: [ - { - ...assetStub.image, - originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg', - }, - ], - hasNextPage: false, - }); + const asset = assetStub.storageAsset({ originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg' }); + + mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset])); mocks.user.getList.mockResolvedValue([userStub.user1]); await sut.handleMigration(); - expect(mocks.asset.getAll).toHaveBeenCalled(); + expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled(); expect(mocks.storage.rename).not.toHaveBeenCalled(); expect(mocks.storage.copyFile).not.toHaveBeenCalled(); expect(mocks.storage.checkFileExists).not.toHaveBeenCalledTimes(2); @@ -413,20 +388,14 @@ describe(StorageTemplateService.name, () => { }); it('should skip when an asset is probably a duplicate', async () => { - mocks.asset.getAll.mockResolvedValue({ - items: [ - { - ...assetStub.image, - originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id+1.jpg', - }, - ], - hasNextPage: false, - }); + const asset = assetStub.storageAsset({ originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id+1.jpg' }); + + mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset])); mocks.user.getList.mockResolvedValue([userStub.user1]); await sut.handleMigration(); - expect(mocks.asset.getAll).toHaveBeenCalled(); + expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled(); expect(mocks.storage.rename).not.toHaveBeenCalled(); expect(mocks.storage.copyFile).not.toHaveBeenCalled(); expect(mocks.storage.checkFileExists).not.toHaveBeenCalledTimes(2); @@ -434,72 +403,63 @@ describe(StorageTemplateService.name, () => { }); it('should move an asset', async () => { - mocks.asset.getAll.mockResolvedValue({ - items: [assetStub.image], - hasNextPage: false, - }); + const asset = assetStub.storageAsset(); + const oldPath = asset.originalPath; + const newPath = `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`; + mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset])); mocks.user.getList.mockResolvedValue([userStub.user1]); mocks.move.create.mockResolvedValue({ id: '123', entityId: assetStub.image.id, pathType: AssetPathType.ORIGINAL, oldPath: assetStub.image.originalPath, - newPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg', + newPath, }); await sut.handleMigration(); - expect(mocks.asset.getAll).toHaveBeenCalled(); - expect(mocks.storage.rename).toHaveBeenCalledWith( - '/original/path.jpg', - 'upload/library/user-id/2023/2023-02-23/asset-id.jpg', - ); - expect(mocks.asset.update).toHaveBeenCalledWith({ - id: assetStub.image.id, - originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg', - }); + expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled(); + expect(mocks.storage.rename).toHaveBeenCalledWith(oldPath, newPath); + expect(mocks.asset.update).toHaveBeenCalledWith({ id: asset.id, originalPath: newPath }); }); it('should use the user storage label', async () => { - mocks.asset.getAll.mockResolvedValue({ - items: [assetStub.image], - hasNextPage: false, - }); + const asset = assetStub.storageAsset(); + mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset])); mocks.user.getList.mockResolvedValue([userStub.storageLabel]); mocks.move.create.mockResolvedValue({ id: '123', - entityId: assetStub.image.id, + entityId: asset.id, pathType: AssetPathType.ORIGINAL, - oldPath: assetStub.image.originalPath, - newPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg', + oldPath: asset.originalPath, + newPath: `upload/library/user-id/2023/2023-02-23/${asset.originalFileName}`, }); await sut.handleMigration(); - expect(mocks.asset.getAll).toHaveBeenCalled(); + expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled(); expect(mocks.storage.rename).toHaveBeenCalledWith( '/original/path.jpg', - 'upload/library/label-1/2023/2023-02-23/asset-id.jpg', + `upload/library/label-1/2022/2022-06-19/${asset.originalFileName}`, ); expect(mocks.asset.update).toHaveBeenCalledWith({ - id: assetStub.image.id, - originalPath: 'upload/library/label-1/2023/2023-02-23/asset-id.jpg', + id: asset.id, + originalPath: `upload/library/label-1/2022/2022-06-19/${asset.originalFileName}`, }); }); it('should copy the file if rename fails due to EXDEV (rename across filesystems)', async () => { - const newPath = 'upload/library/user-id/2023/2023-02-23/asset-id.jpg'; - mocks.asset.getAll.mockResolvedValue({ - items: [assetStub.image], - hasNextPage: false, - }); + const asset = assetStub.storageAsset({ originalPath: '/path/to/original.jpg', fileSizeInByte: 5000 }); + const oldPath = asset.originalPath; + const newPath = `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`; + mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset])); mocks.storage.rename.mockRejectedValue({ code: 'EXDEV' }); mocks.user.getList.mockResolvedValue([userStub.user1]); mocks.move.create.mockResolvedValue({ id: '123', - entityId: assetStub.image.id, + entityId: asset.id, pathType: AssetPathType.ORIGINAL, - oldPath: assetStub.image.originalPath, + oldPath, newPath, }); mocks.storage.stat.mockResolvedValueOnce({ @@ -510,40 +470,36 @@ describe(StorageTemplateService.name, () => { size: 5000, } as Stats); mocks.storage.stat.mockResolvedValueOnce({ + size: 5000, atime: new Date(), mtime: new Date(), } as Stats); - mocks.crypto.hashFile.mockResolvedValue(assetStub.image.checksum); + mocks.crypto.hashFile.mockResolvedValue(asset.checksum); await sut.handleMigration(); - expect(mocks.asset.getAll).toHaveBeenCalled(); - expect(mocks.storage.rename).toHaveBeenCalledWith('/original/path.jpg', newPath); - expect(mocks.storage.copyFile).toHaveBeenCalledWith('/original/path.jpg', newPath); + expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled(); + expect(mocks.storage.rename).toHaveBeenCalledWith(oldPath, newPath); + expect(mocks.storage.copyFile).toHaveBeenCalledWith(oldPath, newPath); + expect(mocks.storage.stat).toHaveBeenCalledWith(oldPath); expect(mocks.storage.stat).toHaveBeenCalledWith(newPath); - expect(mocks.storage.stat).toHaveBeenCalledWith(assetStub.image.originalPath); expect(mocks.storage.utimes).toHaveBeenCalledWith(newPath, expect.any(Date), expect.any(Date)); - expect(mocks.storage.unlink).toHaveBeenCalledWith(assetStub.image.originalPath); + expect(mocks.storage.unlink).toHaveBeenCalledWith(oldPath); expect(mocks.storage.unlink).toHaveBeenCalledTimes(1); - expect(mocks.asset.update).toHaveBeenCalledWith({ - id: assetStub.image.id, - originalPath: newPath, - }); + expect(mocks.asset.update).toHaveBeenCalledWith({ id: asset.id, originalPath: newPath }); }); it('should not update the database if the move fails due to incorrect newPath filesize', async () => { - mocks.asset.getAll.mockResolvedValue({ - items: [assetStub.image], - hasNextPage: false, - }); + const asset = assetStub.storageAsset(); + mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset])); mocks.storage.rename.mockRejectedValue({ code: 'EXDEV' }); mocks.user.getList.mockResolvedValue([userStub.user1]); mocks.move.create.mockResolvedValue({ id: '123', - entityId: assetStub.image.id, + entityId: asset.id, pathType: AssetPathType.ORIGINAL, - oldPath: assetStub.image.originalPath, - newPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg', + oldPath: asset.originalPath, + newPath: `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`, }); mocks.storage.stat.mockResolvedValue({ size: 100, @@ -551,41 +507,41 @@ describe(StorageTemplateService.name, () => { await sut.handleMigration(); - expect(mocks.asset.getAll).toHaveBeenCalled(); + expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled(); expect(mocks.storage.rename).toHaveBeenCalledWith( '/original/path.jpg', - 'upload/library/user-id/2023/2023-02-23/asset-id.jpg', + `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`, ); expect(mocks.storage.copyFile).toHaveBeenCalledWith( '/original/path.jpg', - 'upload/library/user-id/2023/2023-02-23/asset-id.jpg', + `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`, + ); + expect(mocks.storage.stat).toHaveBeenCalledWith( + `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`, ); - expect(mocks.storage.stat).toHaveBeenCalledWith('upload/library/user-id/2023/2023-02-23/asset-id.jpg'); expect(mocks.asset.update).not.toHaveBeenCalled(); }); it('should not update the database if the move fails', async () => { - mocks.asset.getAll.mockResolvedValue({ - items: [assetStub.image], - hasNextPage: false, - }); + const asset = assetStub.storageAsset(); + mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset])); mocks.storage.rename.mockRejectedValue(new Error('Read only system')); mocks.storage.copyFile.mockRejectedValue(new Error('Read only system')); mocks.move.create.mockResolvedValue({ id: 'move-123', - entityId: '123', + entityId: asset.id, pathType: AssetPathType.ORIGINAL, - oldPath: assetStub.image.originalPath, + oldPath: asset.originalPath, newPath: '', }); mocks.user.getList.mockResolvedValue([userStub.user1]); await sut.handleMigration(); - expect(mocks.asset.getAll).toHaveBeenCalled(); + expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled(); expect(mocks.storage.rename).toHaveBeenCalledWith( '/original/path.jpg', - 'upload/library/user-id/2023/2023-02-23/asset-id.jpg', + `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`, ); expect(mocks.asset.update).not.toHaveBeenCalled(); }); diff --git a/server/src/services/storage-template.service.ts b/server/src/services/storage-template.service.ts index 53d93b709b..1a0d4f4644 100644 --- a/server/src/services/storage-template.service.ts +++ b/server/src/services/storage-template.service.ts @@ -3,17 +3,14 @@ import handlebar from 'handlebars'; import { DateTime } from 'luxon'; import path from 'node:path'; import sanitize from 'sanitize-filename'; -import { JOBS_ASSET_PAGINATION_SIZE } from 'src/constants'; import { StorageCore } from 'src/cores/storage.core'; import { OnEvent, OnJob } from 'src/decorators'; import { SystemConfigTemplateStorageOptionDto } from 'src/dtos/system-config.dto'; -import { AssetEntity } from 'src/entities/asset.entity'; import { AssetPathType, AssetType, DatabaseLock, JobName, JobStatus, QueueName, StorageFolder } from 'src/enum'; import { ArgOf } from 'src/repositories/event.repository'; import { BaseService } from 'src/services/base.service'; -import { JobOf } from 'src/types'; +import { JobOf, StorageAsset } from 'src/types'; import { getLivePhotoMotionFilename } from 'src/utils/file'; -import { usePagination } from 'src/utils/pagination'; const storageTokens = { secondOptions: ['s', 'ss', 'SSS'], @@ -53,7 +50,7 @@ export interface MoveAssetMetadata { } interface RenderMetadata { - asset: AssetEntity; + asset: StorageAsset; filename: string; extension: string; albumName: string | null; @@ -98,7 +95,7 @@ export class StorageTemplateService extends BaseService { originalPath: '/upload/test/IMG_123.jpg', type: AssetType.IMAGE, id: 'd587e44b-f8c0-4832-9ba3-43268bbf5d4e', - } as AssetEntity, + } as StorageAsset, filename: 'IMG_123', extension: 'jpg', albumName: 'album', @@ -121,7 +118,7 @@ export class StorageTemplateService extends BaseService { return JobStatus.SKIPPED; } - const [asset] = await this.assetRepository.getByIds([id], { exifInfo: true }); + const asset = await this.assetRepository.getStorageTemplateAsset(id); if (!asset) { return JobStatus.FAILED; } @@ -133,7 +130,7 @@ export class StorageTemplateService extends BaseService { // move motion part of live photo if (asset.livePhotoVideoId) { - const [livePhotoVideo] = await this.assetRepository.getByIds([asset.livePhotoVideoId], { exifInfo: true }); + const livePhotoVideo = await this.assetRepository.getStorageTemplateAsset(asset.livePhotoVideoId); if (!livePhotoVideo) { return JobStatus.FAILED; } @@ -152,19 +149,17 @@ export class StorageTemplateService extends BaseService { this.logger.log('Storage template migration disabled, skipping'); return JobStatus.SKIPPED; } + await this.moveRepository.cleanMoveHistory(); - const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => - this.assetRepository.getAll(pagination, { withExif: true, withArchived: true }), - ); + + const assets = this.assetRepository.streamStorageTemplateAssets(); const users = await this.userRepository.getList(); - for await (const assets of assetPagination) { - for (const asset of assets) { - const user = users.find((user) => user.id === asset.ownerId); - const storageLabel = user?.storageLabel || null; - const filename = asset.originalFileName || asset.id; - await this.moveAsset(asset, { storageLabel, filename }); - } + for await (const asset of assets) { + const user = users.find((user) => user.id === asset.ownerId); + const storageLabel = user?.storageLabel || null; + const filename = asset.originalFileName || asset.id; + await this.moveAsset(asset, { storageLabel, filename }); } this.logger.debug('Cleaning up empty directories...'); @@ -182,7 +177,7 @@ export class StorageTemplateService extends BaseService { await this.moveRepository.cleanMoveHistorySingle(assetId); } - async moveAsset(asset: AssetEntity, metadata: MoveAssetMetadata) { + async moveAsset(asset: StorageAsset, metadata: MoveAssetMetadata) { if (asset.isExternal || StorageCore.isAndroidMotionPath(asset.originalPath)) { // External assets are not affected by storage template // TODO: shouldn't this only apply to external assets? @@ -190,11 +185,11 @@ export class StorageTemplateService extends BaseService { } return this.databaseRepository.withLock(DatabaseLock.StorageTemplateMigration, async () => { - const { id, sidecarPath, originalPath, exifInfo, checksum } = asset; + const { id, sidecarPath, originalPath, checksum, fileSizeInByte } = asset; const oldPath = originalPath; const newPath = await this.getTemplatePath(asset, metadata); - if (!exifInfo || !exifInfo.fileSizeInByte) { + if (!fileSizeInByte) { this.logger.error(`Asset ${id} missing exif info, skipping storage template migration`); return; } @@ -205,7 +200,7 @@ export class StorageTemplateService extends BaseService { pathType: AssetPathType.ORIGINAL, oldPath, newPath, - assetInfo: { sizeInBytes: exifInfo.fileSizeInByte, checksum }, + assetInfo: { sizeInBytes: fileSizeInByte, checksum }, }); if (sidecarPath) { await this.storageCore.moveFile({ @@ -221,7 +216,7 @@ export class StorageTemplateService extends BaseService { }); } - private async getTemplatePath(asset: AssetEntity, metadata: MoveAssetMetadata): Promise { + private async getTemplatePath(asset: StorageAsset, metadata: MoveAssetMetadata): Promise { const { storageLabel, filename } = metadata; try { @@ -344,7 +339,7 @@ export class StorageTemplateService extends BaseService { }; const systemTimeZone = Intl.DateTimeFormat().resolvedOptions().timeZone; - const zone = asset.exifInfo?.timeZone || systemTimeZone; + const zone = asset.timeZone || systemTimeZone; const dt = DateTime.fromJSDate(asset.fileCreatedAt, { zone }); for (const token of Object.values(storageTokens).flat()) { diff --git a/server/src/types.ts b/server/src/types.ts index ece5704a7a..1c0a61b259 100644 --- a/server/src/types.ts +++ b/server/src/types.ts @@ -1,4 +1,5 @@ import { + AssetType, DatabaseExtension, ExifOrientation, ImageFormat, @@ -438,3 +439,18 @@ export type SyncAck = { type: SyncEntityType; updateId: string; }; + +export type StorageAsset = { + id: string; + ownerId: string; + livePhotoVideoId: string | null; + type: AssetType; + isExternal: boolean; + checksum: Buffer; + timeZone: string | null; + fileCreatedAt: Date; + originalPath: string; + originalFileName: string; + sidecarPath: string | null; + fileSizeInByte: number | null; +}; diff --git a/server/test/fixtures/asset.stub.ts b/server/test/fixtures/asset.stub.ts index 4230c9350e..b7a8686b1f 100644 --- a/server/test/fixtures/asset.stub.ts +++ b/server/test/fixtures/asset.stub.ts @@ -3,6 +3,7 @@ import { AssetEntity } from 'src/entities/asset.entity'; import { ExifEntity } from 'src/entities/exif.entity'; import { StackEntity } from 'src/entities/stack.entity'; import { AssetFileType, AssetStatus, AssetType } from 'src/enum'; +import { StorageAsset } from 'src/types'; import { authStub } from 'test/fixtures/auth.stub'; import { fileStub } from 'test/fixtures/file.stub'; import { libraryStub } from 'test/fixtures/library.stub'; @@ -40,6 +41,21 @@ export const stackStub = (stackId: string, assets: AssetEntity[]): StackEntity = }; export const assetStub = { + storageAsset: (asset: Partial = {}) => ({ + id: 'asset-id', + ownerId: 'user-id', + livePhotoVideoId: null, + type: AssetType.IMAGE, + isExternal: false, + checksum: Buffer.from('file hash'), + timeZone: null, + fileCreatedAt: new Date('2022-06-19T23:41:36.910Z'), + originalPath: '/original/path.jpg', + originalFileName: 'IMG_123.jpg', + sidecarPath: null, + fileSizeInByte: 12_345, + ...asset, + }), noResizePath: Object.freeze({ id: 'asset-id', status: AssetStatus.ACTIVE, diff --git a/server/test/repositories/asset.repository.mock.ts b/server/test/repositories/asset.repository.mock.ts index bc6a172e05..66b56b0ecc 100644 --- a/server/test/repositories/asset.repository.mock.ts +++ b/server/test/repositories/asset.repository.mock.ts @@ -44,5 +44,7 @@ export const newAssetRepositoryMock = (): Mocked