refactor: move encoded video to asset files table (#26863)

* refactor: move encoded video to asset files table

* chore: update

server/src/cores/storage.core.ts

@@ -154,10 +154,11 @@ export class StorageCore {
   }
 
   async moveAssetVideo(asset: StorageAsset) {
+    const encodedVideoFile = getAssetFile(asset.files, AssetFileType.EncodedVideo, { isEdited: false });
     return this.moveFile({
       entityId: asset.id,
       pathType: AssetPathType.EncodedVideo,
-      oldPath: asset.encodedVideoPath,
+      oldPath: encodedVideoFile?.path || null,
       newPath: StorageCore.getEncodedVideoPath(asset),
     });
   }
@@ -303,21 +304,15 @@ export class StorageCore {
       case AssetPathType.Original: {
         return this.assetRepository.update({ id, originalPath: newPath });
       }
-      case AssetFileType.FullSize: {
-        return this.assetRepository.upsertFile({ assetId: id, type: AssetFileType.FullSize, path: newPath });
-      }
-      case AssetFileType.Preview: {
-        return this.assetRepository.upsertFile({ assetId: id, type: AssetFileType.Preview, path: newPath });
-      }
-      case AssetFileType.Thumbnail: {
-        return this.assetRepository.upsertFile({ assetId: id, type: AssetFileType.Thumbnail, path: newPath });
-      }
-      case AssetPathType.EncodedVideo: {
-        return this.assetRepository.update({ id, encodedVideoPath: newPath });
-      }
+
+      case AssetFileType.FullSize:
+      case AssetFileType.EncodedVideo:
+      case AssetFileType.Thumbnail:
+      case AssetFileType.Preview:
       case AssetFileType.Sidecar: {
-        return this.assetRepository.upsertFile({ assetId: id, type: AssetFileType.Sidecar, path: newPath });
+        return this.assetRepository.upsertFile({ assetId: id, type: pathType as AssetFileType, path: newPath });
       }
+
       case PersonPathType.Face: {
         return this.personRepository.update({ id, thumbnailPath: newPath });
       }

server/src/database.ts

@@ -154,7 +154,6 @@ export type StorageAsset = {
   id: string;
   ownerId: string;
   files: AssetFile[];
-  encodedVideoPath: string | null;
 };
 
 export type Stack = {

server/src/dtos/asset-response.dto.ts

@@ -153,7 +153,6 @@ export type MapAsset = {
   duplicateId: string | null;
   duration: string | null;
   edits?: ShallowDehydrateObject<AssetEditActionItem>[];
-  encodedVideoPath: string | null;
   exifInfo?: ShallowDehydrateObject<Selectable<Exif>> | null;
   faces?: ShallowDehydrateObject<AssetFace>[];
   fileCreatedAt: Date;

server/src/enum.ts

@@ -45,6 +45,7 @@ export enum AssetFileType {
   Preview = 'preview',
   Thumbnail = 'thumbnail',
   Sidecar = 'sidecar',
+  EncodedVideo = 'encoded_video',
 }
 
 export enum AlbumUserRole {

server/src/queries/asset.job.repository.sql

@@ -175,7 +175,6 @@ where
 select
   "asset"."id",
   "asset"."ownerId",
-  "asset"."encodedVideoPath",
   (
     select
       coalesce(json_agg(agg), '[]')
@@ -463,7 +462,6 @@ select
   "asset"."libraryId",
   "asset"."ownerId",
   "asset"."livePhotoVideoId",
-  "asset"."encodedVideoPath",
   "asset"."originalPath",
   "asset"."isOffline",
   to_json("asset_exif") as "exifInfo",
@@ -521,12 +519,17 @@ select
 from
   "asset"
 where
-  "asset"."type" = $1
-  and (
-    "asset"."encodedVideoPath" is null
-    or "asset"."encodedVideoPath" = $2
+  "asset"."type" = 'VIDEO'
+  and not exists (
+    select
+      "asset_file"."id"
+    from
+      "asset_file"
+    where
+      "asset_file"."assetId" = "asset"."id"
+      and "asset_file"."type" = 'encoded_video'
   )
-  and "asset"."visibility" != $3
+  and "asset"."visibility" != 'hidden'
   and "asset"."deletedAt" is null
 
 -- AssetJobRepository.getForVideoConversion
@@ -534,12 +537,27 @@ select
   "asset"."id",
   "asset"."ownerId",
   "asset"."originalPath",
-  "asset"."encodedVideoPath"
+  (
+    select
+      coalesce(json_agg(agg), '[]')
+    from
+      (
+        select
+          "asset_file"."id",
+          "asset_file"."path",
+          "asset_file"."type",
+          "asset_file"."isEdited"
+        from
+          "asset_file"
+        where
+          "asset_file"."assetId" = "asset"."id"
+      ) as agg
+  ) as "files"
 from
   "asset"
 where
   "asset"."id" = $1
-  and "asset"."type" = $2
+  and "asset"."type" = 'VIDEO'
 
 -- AssetJobRepository.streamForMetadataExtraction
 select

server/src/queries/asset.repository.sql

@@ -629,13 +629,21 @@ order by
 
 -- AssetRepository.getForVideo
 select
-  "asset"."encodedVideoPath",
-  "asset"."originalPath"
+  "asset"."originalPath",
+  (
+    select
+      "asset_file"."path"
+    from
+      "asset_file"
+    where
+      "asset_file"."assetId" = "asset"."id"
+      and "asset_file"."type" = $1
+  ) as "encodedVideoPath"
 from
   "asset"
 where
-  "asset"."id" = $1
-  and "asset"."type" = $2
+  "asset"."id" = $2
+  and "asset"."type" = $3
 
 -- AssetRepository.getForOcr
 select

server/src/repositories/asset-job.repository.ts

@@ -104,7 +104,7 @@ export class AssetJobRepository {
   getForMigrationJob(id: string) {
     return this.db
       .selectFrom('asset')
-      .select(['asset.id', 'asset.ownerId', 'asset.encodedVideoPath'])
+      .select(['asset.id', 'asset.ownerId'])
       .select(withFiles)
       .where('asset.id', '=', id)
       .executeTakeFirst();
@@ -268,7 +268,6 @@ export class AssetJobRepository {
         'asset.libraryId',
         'asset.ownerId',
         'asset.livePhotoVideoId',
-        'asset.encodedVideoPath',
         'asset.originalPath',
         'asset.isOffline',
       ])
@@ -310,11 +309,21 @@ export class AssetJobRepository {
     return this.db
       .selectFrom('asset')
       .select(['asset.id'])
-      .where('asset.type', '=', AssetType.Video)
+      .where('asset.type', '=', sql.lit(AssetType.Video))
       .$if(!force, (qb) =>
         qb
-          .where((eb) => eb.or([eb('asset.encodedVideoPath', 'is', null), eb('asset.encodedVideoPath', '=', '')]))
-          .where('asset.visibility', '!=', AssetVisibility.Hidden),
+          .where((eb) =>
+            eb.not(
+              eb.exists(
+                eb
+                  .selectFrom('asset_file')
+                  .select('asset_file.id')
+                  .whereRef('asset_file.assetId', '=', 'asset.id')
+                  .where('asset_file.type', '=', sql.lit(AssetFileType.EncodedVideo)),
+              ),
+            ),
+          )
+          .where('asset.visibility', '!=', sql.lit(AssetVisibility.Hidden)),
       )
       .where('asset.deletedAt', 'is', null)
       .stream();
@@ -324,9 +333,10 @@ export class AssetJobRepository {
   getForVideoConversion(id: string) {
     return this.db
       .selectFrom('asset')
-      .select(['asset.id', 'asset.ownerId', 'asset.originalPath', 'asset.encodedVideoPath'])
+      .select(['asset.id', 'asset.ownerId', 'asset.originalPath'])
+      .select(withFiles)
       .where('asset.id', '=', id)
-      .where('asset.type', '=', AssetType.Video)
+      .where('asset.type', '=', sql.lit(AssetType.Video))
       .executeTakeFirst();
   }
 

server/src/repositories/asset.repository.ts

@@ -36,6 +36,7 @@ import {
   withExif,
   withFaces,
   withFacesAndPeople,
+  withFilePath,
   withFiles,
   withLibrary,
   withOwner,
@@ -1019,8 +1020,21 @@ export class AssetRepository {
       .execute();
   }
 
-  async deleteFile({ assetId, type }: { assetId: string; type: AssetFileType }): Promise<void> {
-    await this.db.deleteFrom('asset_file').where('assetId', '=', asUuid(assetId)).where('type', '=', type).execute();
+  async deleteFile({
+    assetId,
+    type,
+    edited,
+  }: {
+    assetId: string;
+    type: AssetFileType;
+    edited?: boolean;
+  }): Promise<void> {
+    await this.db
+      .deleteFrom('asset_file')
+      .where('assetId', '=', asUuid(assetId))
+      .where('type', '=', type)
+      .$if(edited !== undefined, (qb) => qb.where('isEdited', '=', edited!))
+      .execute();
   }
 
   async deleteFiles(files: Pick<Selectable<AssetFileTable>, 'id'>[]): Promise<void> {
@@ -1139,7 +1153,8 @@ export class AssetRepository {
   async getForVideo(id: string) {
     return this.db
       .selectFrom('asset')
-      .select(['asset.encodedVideoPath', 'asset.originalPath'])
+      .select(['asset.originalPath'])
+      .select((eb) => withFilePath(eb, AssetFileType.EncodedVideo).as('encodedVideoPath'))
       .where('asset.id', '=', id)
       .where('asset.type', '=', AssetType.Video)
       .executeTakeFirst();

server/src/repositories/database.repository.ts

@@ -431,7 +431,6 @@ export class DatabaseRepository {
         .updateTable('asset')
         .set((eb) => ({
           originalPath: eb.fn('REGEXP_REPLACE', ['originalPath', source, target]),
-          encodedVideoPath: eb.fn('REGEXP_REPLACE', ['encodedVideoPath', source, target]),
         }))
         .execute();
 

server/src/schema/migrations/1773242919341-EncodedVideoAssetFiles.ts

@@ -0,0 +1,25 @@
+import { Kysely, sql } from 'kysely';
+
+export async function up(db: Kysely<any>): Promise<void> {
+  await sql`
+    INSERT INTO "asset_file" ("assetId", "type", "path")
+    SELECT "id", 'encoded_video', "encodedVideoPath"
+    FROM "asset"
+    WHERE "encodedVideoPath" IS NOT NULL AND "encodedVideoPath" != '';
+  `.execute(db);
+
+  await sql`ALTER TABLE "asset" DROP COLUMN "encodedVideoPath";`.execute(db);
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await sql`ALTER TABLE "asset" ADD "encodedVideoPath" character varying DEFAULT '';`.execute(db);
+
+  await sql`
+    UPDATE "asset"
+    SET "encodedVideoPath" = af."path"
+    FROM "asset_file" af
+    WHERE "asset"."id" = af."assetId"
+      AND af."type" = 'encoded_video'
+      AND af."isEdited" = false;
+  `.execute(db);
+}

server/src/schema/tables/asset.table.ts

@@ -92,9 +92,6 @@ export class AssetTable {
   @Column({ type: 'character varying', nullable: true })
   duration!: string | null;
 
-  @Column({ type: 'character varying', nullable: true, default: '' })
-  encodedVideoPath!: string | null;
-
   @Column({ type: 'bytea', index: true })
   checksum!: Buffer; // sha1 checksum
 

server/src/services/asset-media.service.spec.ts

@@ -163,7 +163,6 @@ const assetEntity = Object.freeze({
   fileCreatedAt: new Date('2022-06-19T23:41:36.910Z'),
   updatedAt: new Date('2022-06-19T23:41:36.910Z'),
   isFavorite: false,
-  encodedVideoPath: '',
   duration: '0:00:00.000000',
   files: [] as AssetFile[],
   exifInfo: {
@@ -711,13 +710,18 @@ describe(AssetMediaService.name, () => {
     });
 
     it('should return the encoded video path if available', async () => {
-      const asset = AssetFactory.create({ encodedVideoPath: '/path/to/encoded/video.mp4' });
+      const asset = AssetFactory.from()
+        .file({ type: AssetFileType.EncodedVideo, path: '/path/to/encoded/video.mp4' })
+        .build();
       mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([asset.id]));
-      mocks.asset.getForVideo.mockResolvedValue(asset);
+      mocks.asset.getForVideo.mockResolvedValue({
+        originalPath: asset.originalPath,
+        encodedVideoPath: asset.files[0].path,
+      });
 
       await expect(sut.playbackVideo(authStub.admin, asset.id)).resolves.toEqual(
         new ImmichFileResponse({
-          path: asset.encodedVideoPath!,
+          path: '/path/to/encoded/video.mp4',
           cacheControl: CacheControl.PrivateWithCache,
           contentType: 'video/mp4',
         }),
@@ -727,7 +731,10 @@ describe(AssetMediaService.name, () => {
     it('should fall back to the original path', async () => {
       const asset = AssetFactory.create({ type: AssetType.Video, originalPath: '/original/path.ext' });
       mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([asset.id]));
-      mocks.asset.getForVideo.mockResolvedValue(asset);
+      mocks.asset.getForVideo.mockResolvedValue({
+        originalPath: asset.originalPath,
+        encodedVideoPath: null,
+      });
 
       await expect(sut.playbackVideo(authStub.admin, asset.id)).resolves.toEqual(
         new ImmichFileResponse({

server/src/services/asset.service.ts

@@ -370,7 +370,7 @@ export class AssetService extends BaseService {
       assetFiles.editedFullsizeFile?.path,
       assetFiles.editedPreviewFile?.path,
       assetFiles.editedThumbnailFile?.path,
-      asset.encodedVideoPath,
+      assetFiles.encodedVideoFile?.path,
     ];
 
     if (deleteOnDisk && !asset.isOffline) {

server/src/services/media.service.spec.ts

@@ -2254,7 +2254,9 @@ describe(MediaService.name, () => {
     });
 
     it('should delete existing transcode if current policy does not require transcoding', async () => {
-      const asset = AssetFactory.create({ type: AssetType.Video, encodedVideoPath: '/encoded/video/path.mp4' });
+      const asset = AssetFactory.from({ type: AssetType.Video })
+        .file({ type: AssetFileType.EncodedVideo, path: '/encoded/video/path.mp4' })
+        .build();
       mocks.media.probe.mockResolvedValue(probeStub.videoStream2160p);
       mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { transcode: TranscodePolicy.Disabled } });
       mocks.assetJob.getForVideoConversion.mockResolvedValue(asset);
@@ -2264,7 +2266,7 @@ describe(MediaService.name, () => {
       expect(mocks.media.transcode).not.toHaveBeenCalled();
       expect(mocks.job.queue).toHaveBeenCalledWith({
         name: JobName.FileDelete,
-        data: { files: [asset.encodedVideoPath] },
+        data: { files: ['/encoded/video/path.mp4'] },
       });
     });
 

server/src/services/media.service.ts

@@ -39,7 +39,7 @@ import {
   VideoInterfaces,
   VideoStreamInfo,
 } from 'src/types';
-import { getDimensions } from 'src/utils/asset.util';
+import { getAssetFile, getDimensions } from 'src/utils/asset.util';
 import { checkFaceVisibility, checkOcrVisibility } from 'src/utils/editor';
 import { BaseConfig, ThumbnailConfig } from 'src/utils/media';
 import { mimeTypes } from 'src/utils/mime-types';
@@ -605,10 +605,11 @@ export class MediaService extends BaseService {
     let { ffmpeg } = await this.getConfig({ withCache: true });
     const target = this.getTranscodeTarget(ffmpeg, videoStream, audioStream);
     if (target === TranscodeTarget.None && !this.isRemuxRequired(ffmpeg, format)) {
-      if (asset.encodedVideoPath) {
+      const encodedVideo = getAssetFile(asset.files, AssetFileType.EncodedVideo, { isEdited: false });
+      if (encodedVideo) {
         this.logger.log(`Transcoded video exists for asset ${asset.id}, but is no longer required. Deleting...`);
-        await this.jobRepository.queue({ name: JobName.FileDelete, data: { files: [asset.encodedVideoPath] } });
-        await this.assetRepository.update({ id: asset.id, encodedVideoPath: null });
+        await this.jobRepository.queue({ name: JobName.FileDelete, data: { files: [encodedVideo.path] } });
+        await this.assetRepository.deleteFiles([encodedVideo]);
       } else {
         this.logger.verbose(`Asset ${asset.id} does not require transcoding based on current policy, skipping`);
       }
@@ -656,7 +657,12 @@ export class MediaService extends BaseService {
 
     this.logger.log(`Successfully encoded ${asset.id}`);
 
-    await this.assetRepository.update({ id: asset.id, encodedVideoPath: output });
+    await this.assetRepository.upsertFile({
+      assetId: asset.id,
+      type: AssetFileType.EncodedVideo,
+      path: output,
+      isEdited: false,
+    });
 
     return JobStatus.Success;
   }

server/src/utils/asset.util.ts

@@ -26,6 +26,8 @@ export const getAssetFiles = (files: AssetFile[]) => ({
   editedFullsizeFile: getAssetFile(files, AssetFileType.FullSize, { isEdited: true }),
   editedPreviewFile: getAssetFile(files, AssetFileType.Preview, { isEdited: true }),
   editedThumbnailFile: getAssetFile(files, AssetFileType.Thumbnail, { isEdited: true }),
+
+  encodedVideoFile: getAssetFile(files, AssetFileType.EncodedVideo, { isEdited: false }),
 });
 
 export const addAssets = async (

server/src/utils/database.ts

@@ -355,7 +355,16 @@ export function searchAssetBuilder(kysely: Kysely<DB>, options: AssetSearchBuild
     .$if(!!options.id, (qb) => qb.where('asset.id', '=', asUuid(options.id!)))
     .$if(!!options.libraryId, (qb) => qb.where('asset.libraryId', '=', asUuid(options.libraryId!)))
     .$if(!!options.userIds, (qb) => qb.where('asset.ownerId', '=', anyUuid(options.userIds!)))
-    .$if(!!options.encodedVideoPath, (qb) => qb.where('asset.encodedVideoPath', '=', options.encodedVideoPath!))
+    .$if(!!options.encodedVideoPath, (qb) =>
+      qb
+        .innerJoin('asset_file', (join) =>
+          join
+            .onRef('asset.id', '=', 'asset_file.assetId')
+            .on('asset_file.type', '=', AssetFileType.EncodedVideo)
+            .on('asset_file.isEdited', '=', false),
+        )
+        .where('asset_file.path', '=', options.encodedVideoPath!),
+    )
     .$if(!!options.originalPath, (qb) =>
       qb.where(sql`f_unaccent(asset."originalPath")`, 'ilike', sql`'%' || f_unaccent(${options.originalPath}) || '%'`),
     )
@@ -380,7 +389,15 @@ export function searchAssetBuilder(kysely: Kysely<DB>, options: AssetSearchBuild
     .$if(options.isFavorite !== undefined, (qb) => qb.where('asset.isFavorite', '=', options.isFavorite!))
     .$if(options.isOffline !== undefined, (qb) => qb.where('asset.isOffline', '=', options.isOffline!))
     .$if(options.isEncoded !== undefined, (qb) =>
-      qb.where('asset.encodedVideoPath', options.isEncoded ? 'is not' : 'is', null),
+      qb.where((eb) => {
+        const exists = eb.exists((eb) =>
+          eb
+            .selectFrom('asset_file')
+            .whereRef('assetId', '=', 'asset.id')
+            .where('type', '=', AssetFileType.EncodedVideo),
+        );
+        return options.isEncoded ? exists : eb.not(exists);
+      }),
     )
     .$if(options.isMotion !== undefined, (qb) =>
       qb.where('asset.livePhotoVideoId', options.isMotion ? 'is not' : 'is', null),

server/test/factories/asset.factory.ts

@@ -55,7 +55,6 @@ export class AssetFactory {
       deviceId: '',
       duplicateId: null,
       duration: null,
-      encodedVideoPath: null,
       fileCreatedAt: newDate(),
       fileModifiedAt: newDate(),
       isExternal: false,

server/test/mappers.ts

@@ -183,7 +183,6 @@ export const getForAssetDeletion = (asset: ReturnType<AssetFactory['build']>) =>
   libraryId: asset.libraryId,
   ownerId: asset.ownerId,
   livePhotoVideoId: asset.livePhotoVideoId,
-  encodedVideoPath: asset.encodedVideoPath,
   originalPath: asset.originalPath,
   isOffline: asset.isOffline,
   exifInfo: asset.exifInfo ? getDehydrated(asset.exifInfo) : null,