feat(server): auto-link live photos (#1761)

* feat(server): auto-link live photos

* fix: video extraction and linking

server/apps/microservices/src/processors/metadata-extraction.processor.ts

@@ -1,4 +1,4 @@
-import { AssetEntity, ExifEntity } from '@app/infra';
+import { AssetEntity, AssetType, ExifEntity } from '@app/infra';
 import {
   IExifExtractionProcessor,
   IReverseGeocodingProcessor,
@@ -18,7 +18,12 @@ import { Repository } from 'typeorm/repository/Repository';
 import geocoder, { InitOptions } from 'local-reverse-geocoder';
 import { getName } from 'i18n-iso-countries';
 import fs from 'node:fs';
-import { ExifDateTime, exiftool } from 'exiftool-vendored';
+import { ExifDateTime, exiftool, Tags } from 'exiftool-vendored';
+import { IsNull, Not } from 'typeorm';
+
+interface ImmichTags extends Tags {
+  ContentIdentifier?: string;
+}
 
 function geocoderInit(init: InitOptions) {
   return new Promise<void>(function (resolve) {
@@ -139,7 +144,7 @@ export class MetadataExtractionProcessor {
   async extractExifInfo(job: Job<IExifExtractionProcessor>) {
     try {
       const { asset, fileName }: { asset: AssetEntity; fileName: string } = job.data;
-      const exifData = await exiftool.read(asset.originalPath).catch((e) => {
+      const exifData = await exiftool.read<ImmichTags>(asset.originalPath).catch((e) => {
         this.logger.warn(`The exifData parsing failed due to: ${e} on file ${asset.originalPath}`);
         return null;
       });
@@ -177,12 +182,33 @@ export class MetadataExtractionProcessor {
       newExif.iso = exifData?.ISO || null;
       newExif.latitude = exifData?.GPSLatitude || null;
       newExif.longitude = exifData?.GPSLongitude || null;
+      newExif.livePhotoCID = exifData?.MediaGroupUUID || null;
 
       await this.assetRepository.save({
         id: asset.id,
         createdAt: createdAt?.toISOString(),
       });
 
+      if (newExif.livePhotoCID && !asset.livePhotoVideoId) {
+        const motionAsset = await this.assetRepository.findOne({
+          where: {
+            id: Not(asset.id),
+            type: AssetType.VIDEO,
+            exifInfo: {
+              livePhotoCID: newExif.livePhotoCID,
+            },
+          },
+          relations: {
+            exifInfo: true,
+          },
+        });
+
+        if (motionAsset) {
+          await this.assetRepository.update(asset.id, { livePhotoVideoId: motionAsset.id });
+          await this.assetRepository.update(motionAsset.id, { isVisible: false });
+        }
+      }
+
       /**
        * Reverse Geocoding
        *
@@ -266,6 +292,11 @@ export class MetadataExtractionProcessor {
         createdAt = asset.createdAt;
       }
 
+      const exifData = await exiftool.read<ImmichTags>(asset.originalPath).catch((e) => {
+        this.logger.warn(`The exifData parsing failed due to: ${e} on file ${asset.originalPath}`);
+        return null;
+      });
+
       const newExif = new ExifEntity();
       newExif.assetId = asset.id;
       newExif.description = '';
@@ -279,6 +310,25 @@ export class MetadataExtractionProcessor {
       newExif.state = null;
       newExif.country = null;
       newExif.fps = null;
+      newExif.livePhotoCID = exifData?.ContentIdentifier || null;
+
+      if (newExif.livePhotoCID) {
+        const photoAsset = await this.assetRepository.findOne({
+          where: {
+            id: Not(asset.id),
+            type: AssetType.IMAGE,
+            livePhotoVideoId: IsNull(),
+            exifInfo: {
+              livePhotoCID: newExif.livePhotoCID,
+            },
+          },
+        });
+
+        if (photoAsset) {
+          await this.assetRepository.update(photoAsset.id, { livePhotoVideoId: asset.id });
+          await this.assetRepository.update(asset.id, { isVisible: false });
+        }
+      }
 
       if (videoTags && videoTags['location']) {
         const location = videoTags['location'] as string;

server/libs/domain/test/fixtures.ts

@@ -378,6 +378,7 @@ export const sharedLinkStub = {
             isVisible: true,
             livePhotoVideoId: null,
             exifInfo: {
+              livePhotoCID: null,
               id: 1,
               assetId: 'id_1',
               description: 'description',

server/libs/infra/src/db/entities/exif.entity.ts

@@ -44,6 +44,10 @@ export class ExifEntity {
   @Column({ type: 'varchar', nullable: true })
   city!: string | null;
 
+  @Index('IDX_live_photo_cid')
+  @Column({ type: 'varchar', nullable: true })
+  livePhotoCID!: string | null;
+
   @Column({ type: 'varchar', nullable: true })
   state!: string | null;
 

server/libs/infra/src/db/migrations/1676437878377-AppleContentIdentifier.ts

@@ -0,0 +1,15 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class AppleContentIdentifier1676437878377 implements MigrationInterface {
+  name = 'AppleContentIdentifier1676437878377';
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`ALTER TABLE "exif" ADD "livePhotoCID" character varying`);
+    await queryRunner.query(`CREATE INDEX "IDX_live_photo_cid" ON "exif" ("livePhotoCID") `);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`DROP INDEX "public"."IDX_live_photo_cid"`);
+    await queryRunner.query(`ALTER TABLE "exif" DROP COLUMN "livePhotoCID"`);
+  }
+}