feat(server): apply storage migration after exif completes (#2093)

* feat(server): apply storage migraiton after exif completes

* feat: same for videos

* fix: migration for live photos

server/apps/immich/src/api-v1/asset/asset.core.ts

@@ -1,29 +1,10 @@
-import {
-  AuthUserDto,
-  IJobRepository,
-  IStorageRepository,
-  ISystemConfigRepository,
-  JobName,
-  StorageTemplateCore,
-} from '@app/domain';
-import { AssetEntity, SystemConfig, UserEntity } from '@app/infra/db/entities';
-import { Logger } from '@nestjs/common';
+import { AuthUserDto, IJobRepository, JobName } from '@app/domain';
+import { AssetEntity, UserEntity } from '@app/infra/db/entities';
 import { IAssetRepository } from './asset-repository';
 import { CreateAssetDto, UploadFile } from './dto/create-asset.dto';
 
 export class AssetCore {
-  private templateCore: StorageTemplateCore;
-  private logger = new Logger(AssetCore.name);
-
-  constructor(
-    private repository: IAssetRepository,
-    private jobRepository: IJobRepository,
-    configRepository: ISystemConfigRepository,
-    config: SystemConfig,
-    private storageRepository: IStorageRepository,
-  ) {
-    this.templateCore = new StorageTemplateCore(configRepository, config, storageRepository);
-  }
+  constructor(private repository: IAssetRepository, private jobRepository: IJobRepository) {}
 
   async create(
     authUser: AuthUserDto,
@@ -31,7 +12,7 @@ export class AssetCore {
     file: UploadFile,
     livePhotoAssetId?: string,
   ): Promise<AssetEntity> {
-    let asset = await this.repository.create({
+    const asset = await this.repository.create({
       owner: { id: authUser.id } as UserEntity,
 
       mimeType: file.mimeType,
@@ -56,31 +37,8 @@ export class AssetCore {
       sharedLinks: [],
     });
 
-    asset = await this.moveAsset(asset, file.originalName);
-
     await this.jobRepository.queue({ name: JobName.ASSET_UPLOADED, data: { asset, fileName: file.originalName } });
 
     return asset;
   }
-
-  async moveAsset(asset: AssetEntity, originalName: string) {
-    const destination = await this.templateCore.getTemplatePath(asset, originalName);
-    if (asset.originalPath !== destination) {
-      const source = asset.originalPath;
-
-      try {
-        await this.storageRepository.moveFile(asset.originalPath, destination);
-        try {
-          await this.repository.save({ id: asset.id, originalPath: destination });
-          asset.originalPath = destination;
-        } catch (error: any) {
-          this.logger.warn('Unable to save new originalPath to database, undoing move', error?.stack);
-          await this.storageRepository.moveFile(destination, source);
-        }
-      } catch (error: any) {
-        this.logger.error(`Problem applying storage template`, error?.stack, { id: asset.id, source, destination });
-      }
-    }
-    return asset;
-  }
 }

server/apps/immich/src/api-v1/asset/asset.service.spec.ts

@@ -8,14 +8,7 @@ import { TimeGroupEnum } from './dto/get-asset-count-by-time-bucket.dto';
 import { AssetCountByUserIdResponseDto } from './response-dto/asset-count-by-user-id-response.dto';
 import { DownloadService } from '../../modules/download/download.service';
 import { AlbumRepository, IAlbumRepository } from '../album/album-repository';
-import {
-  ICryptoRepository,
-  IJobRepository,
-  ISharedLinkRepository,
-  IStorageRepository,
-  ISystemConfigRepository,
-  JobName,
-} from '@app/domain';
+import { ICryptoRepository, IJobRepository, ISharedLinkRepository, IStorageRepository, JobName } from '@app/domain';
 import {
   assetEntityStub,
   authStub,
@@ -24,10 +17,8 @@ import {
   newJobRepositoryMock,
   newSharedLinkRepositoryMock,
   newStorageRepositoryMock,
-  newSystemConfigRepositoryMock,
   sharedLinkResponseStub,
   sharedLinkStub,
-  systemConfigStub,
 } from '@app/domain/../test';
 import { CreateAssetsShareLinkDto } from './dto/create-asset-shared-link.dto';
 import { BadRequestException, ForbiddenException } from '@nestjs/common';
@@ -121,7 +112,6 @@ describe('AssetService', () => {
   let albumRepositoryMock: jest.Mocked<IAlbumRepository>;
   let downloadServiceMock: jest.Mocked<Partial<DownloadService>>;
   let sharedLinkRepositoryMock: jest.Mocked<ISharedLinkRepository>;
-  let configMock: jest.Mocked<ISystemConfigRepository>;
   let cryptoMock: jest.Mocked<ICryptoRepository>;
   let jobMock: jest.Mocked<IJobRepository>;
   let storageMock: jest.Mocked<IStorageRepository>;
@@ -160,7 +150,6 @@ describe('AssetService', () => {
 
     sharedLinkRepositoryMock = newSharedLinkRepositoryMock();
     jobMock = newJobRepositoryMock();
-    configMock = newSystemConfigRepositoryMock();
     cryptoMock = newCryptoRepositoryMock();
     storageMock = newStorageRepositoryMock();
 
@@ -171,8 +160,6 @@ describe('AssetService', () => {
       downloadServiceMock as DownloadService,
       sharedLinkRepositoryMock,
       jobMock,
-      configMock,
-      systemConfigStub.defaults,
       cryptoMock,
       storageMock,
     );
@@ -273,10 +260,6 @@ describe('AssetService', () => {
       await expect(sut.uploadFile(authStub.user1, dto, file)).resolves.toEqual({ duplicate: false, id: 'id_1' });
 
       expect(assetRepositoryMock.create).toHaveBeenCalled();
-      expect(assetRepositoryMock.save).toHaveBeenCalledWith({
-        id: 'id_1',
-        originalPath: 'upload/library/user_id_1/2022/2022-06-19/asset_1.jpeg',
-      });
     });
 
     it('should handle a duplicate', async () => {

server/apps/immich/src/api-v1/asset/asset.service.ts

@@ -12,7 +12,7 @@ import {
 import { InjectRepository } from '@nestjs/typeorm';
 import { QueryFailedError, Repository } from 'typeorm';
 import { AuthUserDto } from '../../decorators/auth-user.decorator';
-import { AssetEntity, AssetType, SharedLinkType, SystemConfig } from '@app/infra/db/entities';
+import { AssetEntity, AssetType, SharedLinkType } from '@app/infra/db/entities';
 import { constants, createReadStream, stat } from 'fs';
 import { ServeFileDto } from './dto/serve-file.dto';
 import { Response as Res } from 'express';
@@ -24,10 +24,9 @@ import { CheckDuplicateAssetDto } from './dto/check-duplicate-asset.dto';
 import { CuratedObjectsResponseDto } from './response-dto/curated-objects-response.dto';
 import {
   AssetResponseDto,
+  getLivePhotoMotionFilename,
   ImmichReadStream,
-  INITIAL_SYSTEM_CONFIG,
   IStorageRepository,
-  ISystemConfigRepository,
   JobName,
   mapAsset,
   mapAssetWithoutExif,
@@ -62,8 +61,6 @@ import { mapSharedLink, SharedLinkResponseDto } from '@app/domain';
 import { AssetSearchDto } from './dto/asset-search.dto';
 import { AddAssetsDto } from '../album/dto/add-assets.dto';
 import { RemoveAssetsDto } from '../album/dto/remove-assets.dto';
-import path from 'path';
-import { getFileNameWithoutExtension } from '@app/domain';
 
 const fileInfo = promisify(stat);
 
@@ -86,12 +83,10 @@ export class AssetService {
     private downloadService: DownloadService,
     @Inject(ISharedLinkRepository) sharedLinkRepository: ISharedLinkRepository,
     @Inject(IJobRepository) private jobRepository: IJobRepository,
-    @Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
-    @Inject(INITIAL_SYSTEM_CONFIG) config: SystemConfig,
     @Inject(ICryptoRepository) cryptoRepository: ICryptoRepository,
     @Inject(IStorageRepository) private storageRepository: IStorageRepository,
   ) {
-    this.assetCore = new AssetCore(_assetRepository, jobRepository, configRepository, config, storageRepository);
+    this.assetCore = new AssetCore(_assetRepository, jobRepository);
     this.shareCore = new ShareCore(sharedLinkRepository, cryptoRepository);
   }
 
@@ -104,7 +99,7 @@ export class AssetService {
     if (livePhotoFile) {
       livePhotoFile = {
         ...livePhotoFile,
-        originalName: getFileNameWithoutExtension(file.originalName) + path.extname(livePhotoFile.originalName),
+        originalName: getLivePhotoMotionFilename(file.originalName, livePhotoFile.originalName),
       };
     }
 

server/apps/microservices/src/processors.ts

@@ -127,6 +127,11 @@ export class StorageTemplateMigrationProcessor {
   async onTemplateMigration() {
     await this.storageTemplateService.handleTemplateMigration();
   }
+
+  @Process({ name: JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE })
+  async onTemplateMigrationSingle(job: Job<IAssetJob>) {
+    await this.storageTemplateService.handleTemplateMigrationSingle(job.data);
+  }
 }
 
 @Processor(QueueName.THUMBNAIL_GENERATION)

server/apps/microservices/src/processors/metadata-extraction.processor.ts

@@ -10,7 +10,7 @@ import {
   QueueName,
   WithoutProperty,
 } from '@app/domain';
-import { AssetEntity, AssetType, ExifEntity } from '@app/infra/db/entities';
+import { AssetType, ExifEntity } from '@app/infra/db/entities';
 import { Process, Processor } from '@nestjs/bull';
 import { Inject, Logger } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
@@ -173,7 +173,8 @@ export class MetadataExtractionProcessor {
   @Process(JobName.EXIF_EXTRACTION)
   async extractExifInfo(job: Job<IAssetUploadedJob>) {
     try {
-      const { asset, fileName }: { asset: AssetEntity; fileName: string } = job.data;
+      let asset = job.data.asset;
+      const fileName = job.data.fileName;
       const exifData = await exiftool.read<ImmichTags>(asset.originalPath).catch((e) => {
         this.logger.warn(`The exifData parsing failed due to: ${e} on file ${asset.originalPath}`);
         return null;
@@ -256,7 +257,8 @@ export class MetadataExtractionProcessor {
       }
 
       await this.exifRepository.upsert(newExif, { conflictPaths: ['assetId'] });
-      await this.assetCore.save({ id: asset.id, fileCreatedAt: fileCreatedAt?.toISOString() });
+      asset = await this.assetCore.save({ id: asset.id, fileCreatedAt: fileCreatedAt?.toISOString() });
+      await this.jobRepository.queue({ name: JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE, data: { asset } });
     } catch (error: any) {
       this.logger.error(`Error extracting EXIF ${error}`, error?.stack);
     }
@@ -273,7 +275,8 @@ export class MetadataExtractionProcessor {
 
   @Process({ name: JobName.EXTRACT_VIDEO_METADATA, concurrency: 2 })
   async extractVideoMetadata(job: Job<IAssetUploadedJob>) {
-    const { asset, fileName } = job.data;
+    let asset = job.data.asset;
+    const fileName = job.data.fileName;
 
     if (!asset.isVisible) {
       return;
@@ -318,6 +321,7 @@ export class MetadataExtractionProcessor {
         if (photoAsset) {
           await this.assetCore.save({ id: photoAsset.id, livePhotoVideoId: asset.id });
           await this.assetCore.save({ id: asset.id, isVisible: false });
+          newExif.imageName = (photoAsset.exifInfo as ExifEntity).imageName;
         }
       }
 
@@ -373,7 +377,8 @@ export class MetadataExtractionProcessor {
       }
 
       await this.exifRepository.upsert(newExif, { conflictPaths: ['assetId'] });
-      await this.assetCore.save({ id: asset.id, duration: durationString, fileCreatedAt });
+      asset = await this.assetCore.save({ id: asset.id, duration: durationString, fileCreatedAt });
+      await this.jobRepository.queue({ name: JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE, data: { asset } });
     } catch (err) {
       ``;
       // do nothing

server/libs/domain/src/domain.util.ts

@@ -4,6 +4,10 @@ export function getFileNameWithoutExtension(path: string): string {
   return basename(path, extname(path));
 }
 
+export function getLivePhotoMotionFilename(stillName: string, motionName: string) {
+  return getFileNameWithoutExtension(stillName) + extname(motionName);
+}
+
 const KiB = Math.pow(1024, 1);
 const MiB = Math.pow(1024, 2);
 const GiB = Math.pow(1024, 3);

server/libs/domain/src/job/job.constants.ts

@@ -41,6 +41,7 @@ export enum JobName {
 
   // storage template
   STORAGE_TEMPLATE_MIGRATION = 'storage-template-migration',
+  STORAGE_TEMPLATE_MIGRATION_SINGLE = 'storage-template-migration-single',
   SYSTEM_CONFIG_CHANGE = 'system-config-change',
 
   // object tagging

server/libs/domain/src/job/job.repository.ts

@@ -36,6 +36,7 @@ export type JobItem =
 
   // Storage Template
   | { name: JobName.STORAGE_TEMPLATE_MIGRATION }
+  | { name: JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE; data: IAssetJob }
   | { name: JobName.SYSTEM_CONFIG_CHANGE }
 
   // Metadata Extraction

server/libs/domain/src/storage-template/storage-template.service.ts

@@ -2,6 +2,8 @@ import { AssetEntity, SystemConfig } from '@app/infra/db/entities';
 import { Inject, Injectable, Logger } from '@nestjs/common';
 import { IAssetRepository } from '../asset/asset.repository';
 import { APP_MEDIA_LOCATION } from '../domain.constant';
+import { getLivePhotoMotionFilename } from '../domain.util';
+import { IAssetJob } from '../job';
 import { IStorageRepository } from '../storage/storage.repository';
 import { INITIAL_SYSTEM_CONFIG, ISystemConfigRepository } from '../system-config';
 import { StorageTemplateCore } from './storage-template.core';
@@ -20,6 +22,24 @@ export class StorageTemplateService {
     this.core = new StorageTemplateCore(configRepository, config, storageRepository);
   }
 
+  async handleTemplateMigrationSingle(data: IAssetJob) {
+    const { asset } = data;
+
+    try {
+      const filename = asset.exifInfo?.imageName || asset.id;
+      await this.moveAsset(asset, filename);
+
+      // move motion part of live photo
+      if (asset.livePhotoVideoId) {
+        const [livePhotoVideo] = await this.assetRepository.getByIds([asset.livePhotoVideoId]);
+        const motionFilename = getLivePhotoMotionFilename(filename, livePhotoVideo.originalPath);
+        await this.moveAsset(livePhotoVideo, motionFilename);
+      }
+    } catch (error: any) {
+      this.logger.error('Error running single template migration', error);
+    }
+  }
+
   async handleTemplateMigration() {
     try {
       console.time('migrating-time');

server/libs/infra/src/job/job.repository.ts

@@ -99,6 +99,10 @@ export class JobRepository implements IJobRepository {
         await this.storageTemplateMigration.add(item.name);
         break;
 
+      case JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE:
+        await this.storageTemplateMigration.add(item.name, item.data);
+        break;
+
       case JobName.SYSTEM_CONFIG_CHANGE:
         await this.backgroundTask.add(item.name, {});
         break;