fix(server): link live photos after metadata extraction finishes (#3702)

* fix(server): link live photos after metadata extraction finishes

* chore: fix test

---------

Co-authored-by: Alex <alex.tran1502@gmail.com>

server/src/domain/album/album.repository.ts

@@ -16,6 +16,8 @@ export interface IAlbumRepository {
   getByIds(ids: string[]): Promise<AlbumEntity[]>;
   getByAssetId(ownerId: string, assetId: string): Promise<AlbumEntity[]>;
   hasAsset(id: string, assetId: string): Promise<boolean>;
+  /** Remove an asset from _all_ albums */
+  removeAsset(id: string): Promise<void>;
   getAssetCountForIds(ids: string[]): Promise<AlbumAssetCount[]>;
   getInvalidThumbnail(): Promise<string[]>;
   getOwned(ownerId: string): Promise<AlbumEntity[]>;

server/src/domain/job/job.constants.ts

@@ -32,6 +32,7 @@ export enum JobName {
   // metadata
   QUEUE_METADATA_EXTRACTION = 'queue-metadata-extraction',
   METADATA_EXTRACTION = 'metadata-extraction',
+  LINK_LIVE_PHOTOS = 'link-live-photos',
 
   // user deletion
   USER_DELETION = 'user-deletion',
@@ -98,6 +99,7 @@ export const JOBS_TO_QUEUE: Record<JobName, QueueName> = {
   // metadata
   [JobName.QUEUE_METADATA_EXTRACTION]: QueueName.METADATA_EXTRACTION,
   [JobName.METADATA_EXTRACTION]: QueueName.METADATA_EXTRACTION,
+  [JobName.LINK_LIVE_PHOTOS]: QueueName.METADATA_EXTRACTION,
 
   // storage template
   [JobName.STORAGE_TEMPLATE_MIGRATION]: QueueName.STORAGE_TEMPLATE_MIGRATION,

server/src/domain/job/job.repository.ts

@@ -45,6 +45,7 @@ export type JobItem =
   // Metadata Extraction
   | { name: JobName.QUEUE_METADATA_EXTRACTION; data: IBaseJob }
   | { name: JobName.METADATA_EXTRACTION; data: IEntityJob }
+  | { name: JobName.LINK_LIVE_PHOTOS; data: IEntityJob }
 
   // Sidecar Scanning
   | { name: JobName.QUEUE_SIDECAR; data: IBaseJob }

server/src/domain/job/job.service.spec.ts

@@ -252,6 +252,10 @@ describe(JobService.name, () => {
       },
       {
         item: { name: JobName.METADATA_EXTRACTION, data: { id: 'asset-1' } },
+        jobs: [JobName.LINK_LIVE_PHOTOS],
+      },
+      {
+        item: { name: JobName.LINK_LIVE_PHOTOS, data: { id: 'asset-1' } },
         jobs: [JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE, JobName.SEARCH_INDEX_ASSET],
       },
       {

server/src/domain/job/job.service.ts

@@ -149,6 +149,10 @@ export class JobService {
         break;
 
       case JobName.METADATA_EXTRACTION:
+        await this.jobRepository.queue({ name: JobName.LINK_LIVE_PHOTOS, data: item.data });
+        break;
+
+      case JobName.LINK_LIVE_PHOTOS:
         await this.jobRepository.queue({ name: JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE, data: item.data });
         break;
 
@@ -186,7 +190,7 @@ export class JobService {
       case JobName.CLASSIFY_IMAGE:
       case JobName.ENCODE_CLIP:
       case JobName.RECOGNIZE_FACES:
-      case JobName.METADATA_EXTRACTION:
+      case JobName.LINK_LIVE_PHOTOS:
         await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ASSET, data: { ids: [item.data.id] } });
         break;
     }

server/src/infra/repositories/album.repository.ts

@@ -1,7 +1,7 @@
 import { AlbumAssetCount, AlbumInfoOptions, IAlbumRepository } from '@app/domain';
 import { Injectable } from '@nestjs/common';
-import { InjectRepository } from '@nestjs/typeorm';
+import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';
-import { FindOptionsOrder, FindOptionsRelations, In, IsNull, Not, Repository } from 'typeorm';
+import { DataSource, FindOptionsOrder, FindOptionsRelations, In, IsNull, Not, Repository } from 'typeorm';
 import { dataSource } from '../database.config';
 import { AlbumEntity, AssetEntity } from '../entities';
 
@@ -10,6 +10,7 @@ export class AlbumRepository implements IAlbumRepository {
   constructor(
     @InjectRepository(AssetEntity) private assetRepository: Repository<AssetEntity>,
     @InjectRepository(AlbumEntity) private repository: Repository<AlbumEntity>,
+    @InjectDataSource() private dataSource: DataSource,
   ) {}
 
   getById(id: string, options: AlbumInfoOptions): Promise<AlbumEntity | null> {
@@ -84,7 +85,7 @@ export class AlbumRepository implements IAlbumRepository {
    */
   async getInvalidThumbnail(): Promise<string[]> {
     // Using dataSource, because there is no direct access to albums_assets_assets.
-    const albumHasAssets = dataSource
+    const albumHasAssets = this.dataSource
       .createQueryBuilder()
       .select('1')
       .from('albums_assets_assets', 'albums_assets')
@@ -150,6 +151,16 @@ export class AlbumRepository implements IAlbumRepository {
     });
   }
 
+  async removeAsset(assetId: string): Promise<void> {
+    // Using dataSource, because there is no direct access to albums_assets_assets.
+    await this.dataSource
+      .createQueryBuilder()
+      .delete()
+      .from('albums_assets_assets')
+      .where('"albums_assets_assets"."assetsId" = :assetId', { assetId })
+      .execute();
+  }
+
   hasAsset(id: string, assetId: string): Promise<boolean> {
     return this.repository.exist({
       where: {

server/src/microservices/app.service.ts

@@ -66,6 +66,7 @@ export class AppService {
       [JobName.VIDEO_CONVERSION]: (data) => this.mediaService.handleVideoConversion(data),
       [JobName.QUEUE_METADATA_EXTRACTION]: (data) => this.metadataProcessor.handleQueueMetadataExtraction(data),
       [JobName.METADATA_EXTRACTION]: (data) => this.metadataProcessor.handleMetadataExtraction(data),
+      [JobName.LINK_LIVE_PHOTOS]: (data) => this.metadataProcessor.handleLivePhotoLinking(data),
       [JobName.QUEUE_RECOGNIZE_FACES]: (data) => this.facialRecognitionService.handleQueueRecognizeFaces(data),
       [JobName.RECOGNIZE_FACES]: (data) => this.facialRecognitionService.handleRecognizeFaces(data),
       [JobName.GENERATE_FACE_THUMBNAIL]: (data) => this.facialRecognitionService.handleGenerateFaceThumbnail(data),

server/src/microservices/processors/metadata-extraction.processor.ts

@@ -1,4 +1,5 @@
 import {
+  IAlbumRepository,
   IAssetRepository,
   IBaseJob,
   ICryptoRepository,
@@ -59,6 +60,7 @@ export class MetadataExtractionProcessor {
 
   constructor(
     @Inject(IAssetRepository) private assetRepository: IAssetRepository,
+    @Inject(IAlbumRepository) private albumRepository: IAlbumRepository,
     @Inject(IJobRepository) private jobRepository: IJobRepository,
     @Inject(IGeocodingRepository) private geocodingRepository: IGeocodingRepository,
     @Inject(ICryptoRepository) private cryptoRepository: ICryptoRepository,
@@ -92,6 +94,38 @@ export class MetadataExtractionProcessor {
     }
   }
 
+  async handleLivePhotoLinking(job: IEntityJob) {
+    const { id } = job;
+    const [asset] = await this.assetRepository.getByIds([id]);
+    if (!asset?.exifInfo) {
+      return false;
+    }
+
+    if (!asset.exifInfo.livePhotoCID) {
+      return true;
+    }
+
+    const otherType = asset.type === AssetType.VIDEO ? AssetType.IMAGE : AssetType.VIDEO;
+    const match = await this.assetRepository.findLivePhotoMatch({
+      livePhotoCID: asset.exifInfo.livePhotoCID,
+      ownerId: asset.ownerId,
+      otherAssetId: asset.id,
+      type: otherType,
+    });
+
+    if (!match) {
+      return true;
+    }
+
+    const [photoAsset, motionAsset] = asset.type === AssetType.IMAGE ? [asset, match] : [match, asset];
+
+    await this.assetRepository.save({ id: photoAsset.id, livePhotoVideoId: motionAsset.id });
+    await this.assetRepository.save({ id: motionAsset.id, isVisible: false });
+    await this.albumRepository.removeAsset(motionAsset.id);
+
+    return true;
+  }
+
   async handleQueueMetadataExtraction(job: IBaseJob) {
     const { force } = job;
     const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => {
@@ -351,19 +385,6 @@ export class MetadataExtractionProcessor {
     }
 
     newExif.livePhotoCID = getExifProperty('MediaGroupUUID');
-    if (newExif.livePhotoCID && !asset.livePhotoVideoId) {
-      const motionAsset = await this.assetRepository.findLivePhotoMatch({
-        livePhotoCID: newExif.livePhotoCID,
-        otherAssetId: asset.id,
-        ownerId: asset.ownerId,
-        type: AssetType.VIDEO,
-      });
-      if (motionAsset) {
-        await this.assetRepository.save({ id: asset.id, livePhotoVideoId: motionAsset.id });
-        await this.assetRepository.save({ id: motionAsset.id, isVisible: false });
-      }
-    }
-
     await this.applyReverseGeocoding(asset, newExif);
 
     /**
@@ -428,19 +449,6 @@ export class MetadataExtractionProcessor {
     newExif.fps = null;
     newExif.livePhotoCID = exifData?.ContentIdentifier || null;
 
-    if (newExif.livePhotoCID) {
-      const photoAsset = await this.assetRepository.findLivePhotoMatch({
-        livePhotoCID: newExif.livePhotoCID,
-        ownerId: asset.ownerId,
-        otherAssetId: asset.id,
-        type: AssetType.IMAGE,
-      });
-      if (photoAsset) {
-        await this.assetRepository.save({ id: photoAsset.id, livePhotoVideoId: asset.id });
-        await this.assetRepository.save({ id: asset.id, isVisible: false });
-      }
-    }
-
     if (videoTags && videoTags['location']) {
       const location = videoTags['location'] as string;
       const locationRegex = /([+-][0-9]+\.[0-9]+)([+-][0-9]+\.[0-9]+)\/$/;

server/test/repositories/album.repository.mock.ts

@@ -12,6 +12,7 @@ export const newAlbumRepositoryMock = (): jest.Mocked<IAlbumRepository> => {
     getNotShared: jest.fn(),
     deleteAll: jest.fn(),
     getAll: jest.fn(),
+    removeAsset: jest.fn(),
     hasAsset: jest.fn(),
     create: jest.fn(),
     update: jest.fn(),