fix: sidecar check job (#21312)

server/src/enum.ts

@@ -571,8 +571,7 @@ export enum JobName {
   SendMail = 'SendMail',
 
   SidecarQueueAll = 'SidecarQueueAll',
-  SidecarDiscovery = 'SidecarDiscovery',
+  SidecarCheck = 'SidecarCheck',
-  SidecarSync = 'SidecarSync',
   SidecarWrite = 'SidecarWrite',
 
   SmartSearchQueueAll = 'SmartSearchQueueAll',

server/src/queries/asset.job.repository.sql

@@ -43,6 +43,18 @@ where
 limit
   $2
 
+-- AssetJobRepository.getForSidecarCheckJob
+select
+  "id",
+  "sidecarPath",
+  "originalPath"
+from
+  "asset"
+where
+  "asset"."id" = $1::uuid
+limit
+  $2
+
 -- AssetJobRepository.streamForThumbnailJob
 select
   "asset"."id",

server/src/repositories/asset-job.repository.ts

@@ -39,10 +39,8 @@ export class AssetJobRepository {
     return this.db
       .selectFrom('asset')
       .where('asset.id', '=', asUuid(id))
-      .select((eb) => [
+      .select(['id', 'sidecarPath', 'originalPath'])
-        'id',
+      .select((eb) =>
-        'sidecarPath',
-        'originalPath',
         jsonArrayFrom(
           eb
             .selectFrom('tag')
@@ -50,7 +48,17 @@ export class AssetJobRepository {
             .innerJoin('tag_asset', 'tag.id', 'tag_asset.tagsId')
             .whereRef('asset.id', '=', 'tag_asset.assetsId'),
         ).as('tags'),
-      ])
+      )
+      .limit(1)
+      .executeTakeFirst();
+  }
+
+  @GenerateSql({ params: [DummyValue.UUID] })
+  getForSidecarCheckJob(id: string) {
+    return this.db
+      .selectFrom('asset')
+      .where('asset.id', '=', asUuid(id))
+      .select(['id', 'sidecarPath', 'originalPath'])
       .limit(1)
       .executeTakeFirst();
   }

server/src/services/job.service.spec.ts

@@ -239,11 +239,11 @@ describe(JobService.name, () => {
 
     const tests: Array<{ item: JobItem; jobs: JobName[]; stub?: any }> = [
       {
-        item: { name: JobName.SidecarSync, data: { id: 'asset-1' } },
+        item: { name: JobName.SidecarCheck, data: { id: 'asset-1' } },
         jobs: [JobName.AssetExtractMetadata],
       },
       {
-        item: { name: JobName.SidecarDiscovery, data: { id: 'asset-1' } },
+        item: { name: JobName.SidecarCheck, data: { id: 'asset-1' } },
         jobs: [JobName.AssetExtractMetadata],
       },
       {

server/src/services/job.service.ts

@@ -310,8 +310,7 @@ export class JobService extends BaseService {
    */
   private async onDone(item: JobItem) {
     switch (item.name) {
-      case JobName.SidecarSync:
+      case JobName.SidecarCheck: {
-      case JobName.SidecarDiscovery: {
         await this.jobRepository.queue({ name: JobName.AssetExtractMetadata, data: item.data });
         break;
       }

server/src/services/library.service.spec.ts

@@ -527,7 +527,7 @@ describe(LibraryService.name, () => {
 
       expect(mocks.job.queueAll).toHaveBeenCalledWith([
         {
-          name: JobName.SidecarDiscovery,
+          name: JobName.SidecarCheck,
           data: {
             id: assetStub.external.id,
             source: 'upload',
@@ -573,7 +573,7 @@ describe(LibraryService.name, () => {
 
       expect(mocks.job.queueAll).toHaveBeenCalledWith([
         {
-          name: JobName.SidecarDiscovery,
+          name: JobName.SidecarCheck,
           data: {
             id: assetStub.image.id,
             source: 'upload',

server/src/services/library.service.ts

@@ -414,7 +414,7 @@ export class LibraryService extends BaseService {
     // We queue a sidecar discovery which, in turn, queues metadata extraction
     await this.jobRepository.queueAll(
       assetIds.map((assetId) => ({
-        name: JobName.SidecarDiscovery,
+        name: JobName.SidecarCheck,
         data: { id: assetId, source: 'upload' },
       })),
     );

server/src/services/metadata.service.spec.ts

@@ -1,7 +1,6 @@
 import { BinaryField, ExifDateTime } from 'exiftool-vendored';
 import { randomBytes } from 'node:crypto';
 import { Stats } from 'node:fs';
-import { constants } from 'node:fs/promises';
 import { defaults } from 'src/config';
 import { MapAsset } from 'src/dtos/asset-response.dto';
 import { AssetType, AssetVisibility, ExifOrientation, ImmichWorker, JobName, JobStatus, SourceType } from 'src/enum';
@@ -15,6 +14,21 @@ import { tagStub } from 'test/fixtures/tag.stub';
 import { factory } from 'test/small.factory';
 import { makeStream, newTestService, ServiceMocks } from 'test/utils';
 
+const forSidecarJob = (
+  asset: {
+    id?: string;
+    originalPath?: string;
+    sidecarPath?: string | null;
+  } = {},
+) => {
+  return {
+    id: factory.uuid(),
+    originalPath: '/path/to/IMG_123.jpg',
+    sidecarPath: null,
+    ...asset,
+  };
+};
+
 const makeFaceTags = (face: Partial<{ Name: string }> = {}, orientation?: ImmichTags['Orientation']) => ({
   Orientation: orientation,
   RegionInfo: {
@@ -1457,7 +1471,7 @@ describe(MetadataService.name, () => {
 
       expect(mocks.job.queueAll).toHaveBeenCalledWith([
         {
-          name: JobName.SidecarSync,
+          name: JobName.SidecarCheck,
           data: { id: assetStub.sidecar.id },
         },
       ]);
@@ -1471,133 +1485,65 @@ describe(MetadataService.name, () => {
       expect(mocks.assetJob.streamForSidecar).toHaveBeenCalledWith(false);
       expect(mocks.job.queueAll).toHaveBeenCalledWith([
         {
-          name: JobName.SidecarDiscovery,
+          name: JobName.SidecarCheck,
           data: { id: assetStub.image.id },
         },
       ]);
     });
   });
 
-  describe('handleSidecarSync', () => {
+  describe('handleSidecarCheck', () => {
     it('should do nothing if asset could not be found', async () => {
-      mocks.asset.getByIds.mockResolvedValue([]);
+      mocks.assetJob.getForSidecarCheckJob.mockResolvedValue(void 0);
-      await expect(sut.handleSidecarSync({ id: assetStub.image.id })).resolves.toBe(JobStatus.Failed);
+
+      await expect(sut.handleSidecarCheck({ id: assetStub.image.id })).resolves.toBeUndefined();
+
       expect(mocks.asset.update).not.toHaveBeenCalled();
     });
 
-    it('should do nothing if asset has no sidecar path', async () => {
+    it('should detect a new sidecar at .jpg.xmp', async () => {
-      mocks.asset.getByIds.mockResolvedValue([assetStub.image]);
+      const asset = forSidecarJob({ originalPath: '/path/to/IMG_123.jpg' });
-      await expect(sut.handleSidecarSync({ id: assetStub.image.id })).resolves.toBe(JobStatus.Failed);
+
-      expect(mocks.asset.update).not.toHaveBeenCalled();
+      mocks.assetJob.getForSidecarCheckJob.mockResolvedValue(asset);
+      mocks.storage.checkFileExists.mockResolvedValueOnce(true);
+
+      await expect(sut.handleSidecarCheck({ id: asset.id })).resolves.toBe(JobStatus.Success);
+
+      expect(mocks.asset.update).toHaveBeenCalledWith({ id: asset.id, sidecarPath: `/path/to/IMG_123.jpg.xmp` });
     });
 
-    it('should set sidecar path if exists (sidecar named photo.ext.xmp)', async () => {
+    it('should detect a new sidecar at .xmp', async () => {
-      mocks.asset.getByIds.mockResolvedValue([assetStub.sidecar]);
+      const asset = forSidecarJob({ originalPath: '/path/to/IMG_123.jpg' });
-      mocks.storage.checkFileExists.mockResolvedValue(true);
 
-      await expect(sut.handleSidecarSync({ id: assetStub.sidecar.id })).resolves.toBe(JobStatus.Success);
+      mocks.assetJob.getForSidecarCheckJob.mockResolvedValue(asset);
-      expect(mocks.storage.checkFileExists).toHaveBeenCalledWith(
-        `${assetStub.sidecar.originalPath}.xmp`,
-        constants.R_OK,
-      );
-      expect(mocks.asset.update).toHaveBeenCalledWith({
-        id: assetStub.sidecar.id,
-        sidecarPath: assetStub.sidecar.sidecarPath,
-      });
-    });
-
-    it('should set sidecar path if exists (sidecar named photo.xmp)', async () => {
-      mocks.asset.getByIds.mockResolvedValue([assetStub.sidecarWithoutExt as any]);
       mocks.storage.checkFileExists.mockResolvedValueOnce(false);
       mocks.storage.checkFileExists.mockResolvedValueOnce(true);
 
-      await expect(sut.handleSidecarSync({ id: assetStub.sidecarWithoutExt.id })).resolves.toBe(JobStatus.Success);
+      await expect(sut.handleSidecarCheck({ id: asset.id })).resolves.toBe(JobStatus.Success);
-      expect(mocks.storage.checkFileExists).toHaveBeenNthCalledWith(
-        2,
-        assetStub.sidecarWithoutExt.sidecarPath,
-        constants.R_OK,
-      );
-      expect(mocks.asset.update).toHaveBeenCalledWith({
-        id: assetStub.sidecarWithoutExt.id,
-        sidecarPath: assetStub.sidecarWithoutExt.sidecarPath,
-      });
-    });
 
-    it('should set sidecar path if exists (two sidecars named photo.ext.xmp and photo.xmp, should pick photo.ext.xmp)', async () => {
+      expect(mocks.asset.update).toHaveBeenCalledWith({ id: asset.id, sidecarPath: '/path/to/IMG_123.xmp' });
-      mocks.asset.getByIds.mockResolvedValue([assetStub.sidecar]);
-      mocks.storage.checkFileExists.mockResolvedValueOnce(true);
-      mocks.storage.checkFileExists.mockResolvedValueOnce(true);
-
-      await expect(sut.handleSidecarSync({ id: assetStub.sidecar.id })).resolves.toBe(JobStatus.Success);
-      expect(mocks.storage.checkFileExists).toHaveBeenNthCalledWith(1, assetStub.sidecar.sidecarPath, constants.R_OK);
-      expect(mocks.storage.checkFileExists).toHaveBeenNthCalledWith(
-        2,
-        assetStub.sidecarWithoutExt.sidecarPath,
-        constants.R_OK,
-      );
-      expect(mocks.asset.update).toHaveBeenCalledWith({
-        id: assetStub.sidecar.id,
-        sidecarPath: assetStub.sidecar.sidecarPath,
-      });
     });
 
     it('should unset sidecar path if file does not exist anymore', async () => {
-      mocks.asset.getByIds.mockResolvedValue([assetStub.sidecar]);
+      const asset = forSidecarJob({ originalPath: '/path/to/IMG_123.jpg', sidecarPath: '/path/to/IMG_123.jpg.xmp' });
+      mocks.assetJob.getForSidecarCheckJob.mockResolvedValue(asset);
       mocks.storage.checkFileExists.mockResolvedValue(false);
 
-      await expect(sut.handleSidecarSync({ id: assetStub.sidecar.id })).resolves.toBe(JobStatus.Success);
+      await expect(sut.handleSidecarCheck({ id: asset.id })).resolves.toBe(JobStatus.Success);
-      expect(mocks.storage.checkFileExists).toHaveBeenCalledWith(
-        `${assetStub.sidecar.originalPath}.xmp`,
-        constants.R_OK,
-      );
-      expect(mocks.asset.update).toHaveBeenCalledWith({
-        id: assetStub.sidecar.id,
-        sidecarPath: null,
-      });
-    });
-  });
 
-  describe('handleSidecarDiscovery', () => {
+      expect(mocks.asset.update).toHaveBeenCalledWith({ id: asset.id, sidecarPath: null });
-    it('should skip hidden assets', async () => {
-      mocks.asset.getByIds.mockResolvedValue([assetStub.livePhotoMotionAsset as any]);
-      await sut.handleSidecarDiscovery({ id: assetStub.livePhotoMotionAsset.id });
-      expect(mocks.storage.checkFileExists).not.toHaveBeenCalled();
     });
 
-    it('should skip assets with a sidecar path', async () => {
+    it('should do nothing if the sidecar file still exists', async () => {
-      mocks.asset.getByIds.mockResolvedValue([assetStub.sidecar]);
+      const asset = forSidecarJob({ originalPath: '/path/to/IMG_123.jpg', sidecarPath: '/path/to/IMG_123.jpg' });
-      await sut.handleSidecarDiscovery({ id: assetStub.sidecar.id });
+
-      expect(mocks.storage.checkFileExists).not.toHaveBeenCalled();
+      mocks.assetJob.getForSidecarCheckJob.mockResolvedValue(asset);
-    });
+      mocks.storage.checkFileExists.mockResolvedValueOnce(true);
+
+      await expect(sut.handleSidecarCheck({ id: asset.id })).resolves.toBe(JobStatus.Skipped);
 
-    it('should do nothing when a sidecar is not found ', async () => {
-      mocks.asset.getByIds.mockResolvedValue([assetStub.image]);
-      mocks.storage.checkFileExists.mockResolvedValue(false);
-      await sut.handleSidecarDiscovery({ id: assetStub.image.id });
       expect(mocks.asset.update).not.toHaveBeenCalled();
     });
-
-    it('should update a image asset when a sidecar is found', async () => {
-      mocks.asset.getByIds.mockResolvedValue([assetStub.image]);
-      mocks.storage.checkFileExists.mockResolvedValue(true);
-      await sut.handleSidecarDiscovery({ id: assetStub.image.id });
-      expect(mocks.storage.checkFileExists).toHaveBeenCalledWith('/original/path.jpg.xmp', constants.R_OK);
-      expect(mocks.asset.update).toHaveBeenCalledWith({
-        id: assetStub.image.id,
-        sidecarPath: '/original/path.jpg.xmp',
-      });
-    });
-
-    it('should update a video asset when a sidecar is found', async () => {
-      mocks.asset.getByIds.mockResolvedValue([assetStub.video]);
-      mocks.storage.checkFileExists.mockResolvedValue(true);
-      await sut.handleSidecarDiscovery({ id: assetStub.video.id });
-      expect(mocks.storage.checkFileExists).toHaveBeenCalledWith('/original/path.ext.xmp', constants.R_OK);
-      expect(mocks.asset.update).toHaveBeenCalledWith({
-        id: assetStub.image.id,
-        sidecarPath: '/original/path.ext.xmp',
-      });
-    });
   });
 
   describe('handleSidecarWrite', () => {

server/src/services/metadata.service.ts

@@ -5,7 +5,7 @@ import _ from 'lodash';
 import { Duration } from 'luxon';
 import { Stats } from 'node:fs';
 import { constants } from 'node:fs/promises';
-import path from 'node:path';
+import { join, parse } from 'node:path';
 import { JOBS_ASSET_PAGINATION_SIZE } from 'src/constants';
 import { StorageCore } from 'src/cores/storage.core';
 import { Asset, AssetFace } from 'src/database';
@@ -331,7 +331,7 @@ export class MetadataService extends BaseService {
 
     const assets = this.assetJobRepository.streamForSidecar(force);
     for await (const asset of assets) {
-      jobs.push({ name: force ? JobName.SidecarSync : JobName.SidecarDiscovery, data: { id: asset.id } });
+      jobs.push({ name: JobName.SidecarCheck, data: { id: asset.id } });
       if (jobs.length >= JOBS_ASSET_PAGINATION_SIZE) {
         await queueAll();
       }
@@ -342,14 +342,37 @@ export class MetadataService extends BaseService {
     return JobStatus.Success;
   }
 
-  @OnJob({ name: JobName.SidecarSync, queue: QueueName.Sidecar })
+  @OnJob({ name: JobName.SidecarCheck, queue: QueueName.Sidecar })
-  handleSidecarSync({ id }: JobOf<JobName.SidecarSync>): Promise<JobStatus> {
+  async handleSidecarCheck({ id }: JobOf<JobName.SidecarCheck>): Promise<JobStatus | undefined> {
-    return this.processSidecar(id, true);
+    const asset = await this.assetJobRepository.getForSidecarCheckJob(id);
-  }
+    if (!asset) {
+      return;
+    }
 
-  @OnJob({ name: JobName.SidecarDiscovery, queue: QueueName.Sidecar })
+    let sidecarPath = null;
-  handleSidecarDiscovery({ id }: JobOf<JobName.SidecarDiscovery>): Promise<JobStatus> {
+    for (const candidate of this.getSidecarCandidates(asset)) {
-    return this.processSidecar(id, false);
+      const exists = await this.storageRepository.checkFileExists(candidate, constants.R_OK);
+      if (!exists) {
+        continue;
+      }
+
+      sidecarPath = candidate;
+      break;
+    }
+
+    const isChanged = sidecarPath !== asset.sidecarPath;
+
+    this.logger.debug(
+      `Sidecar check found old=${asset.sidecarPath}, new=${sidecarPath} will ${isChanged ? 'update' : 'do nothing for'}  asset ${asset.id}: ${asset.originalPath}`,
+    );
+
+    if (!isChanged) {
+      return JobStatus.Skipped;
+    }
+
+    await this.assetRepository.update({ id: asset.id, sidecarPath });
+
+    return JobStatus.Success;
   }
 
   @OnEvent({ name: 'AssetTag' })
@@ -399,6 +422,25 @@ export class MetadataService extends BaseService {
     return JobStatus.Success;
   }
 
+  private getSidecarCandidates({ sidecarPath, originalPath }: { sidecarPath: string | null; originalPath: string }) {
+    const candidates: string[] = [];
+
+    if (sidecarPath) {
+      candidates.push(sidecarPath);
+    }
+
+    const assetPath = parse(originalPath);
+
+    candidates.push(
+      // IMG_123.jpg.xmp
+      `${originalPath}.xmp`,
+      // IMG_123.xmp
+      `${join(assetPath.dir, assetPath.name)}.xmp`,
+    );
+
+    return candidates;
+  }
+
   private getImageDimensions(exifTags: ImmichTags): { width?: number; height?: number } {
     /*
      * The "true" values for width and height are a bit hidden, depending on the camera model and file format.
@@ -564,7 +606,7 @@ export class MetadataService extends BaseService {
             checksum,
             ownerId: asset.ownerId,
             originalPath: StorageCore.getAndroidMotionPath(asset, motionAssetId),
-            originalFileName: `${path.parse(asset.originalFileName).name}.mp4`,
+            originalFileName: `${parse(asset.originalFileName).name}.mp4`,
             visibility: AssetVisibility.Hidden,
             deviceAssetId: 'NONE',
             deviceId: 'NONE',
@@ -905,60 +947,4 @@ export class MetadataService extends BaseService {
 
     return tags;
   }
-
-  private async processSidecar(id: string, isSync: boolean): Promise<JobStatus> {
-    const [asset] = await this.assetRepository.getByIds([id]);
-
-    if (!asset) {
-      return JobStatus.Failed;
-    }
-
-    if (isSync && !asset.sidecarPath) {
-      return JobStatus.Failed;
-    }
-
-    if (!isSync && (asset.visibility === AssetVisibility.Hidden || asset.sidecarPath) && !asset.isExternal) {
-      return JobStatus.Failed;
-    }
-
-    // XMP sidecars can come in two filename formats. For a photo named photo.ext, the filenames are photo.ext.xmp and photo.xmp
-    const assetPath = path.parse(asset.originalPath);
-    const assetPathWithoutExt = path.join(assetPath.dir, assetPath.name);
-    const sidecarPathWithoutExt = `${assetPathWithoutExt}.xmp`;
-    const sidecarPathWithExt = `${asset.originalPath}.xmp`;
-
-    const [sidecarPathWithExtExists, sidecarPathWithoutExtExists] = await Promise.all([
-      this.storageRepository.checkFileExists(sidecarPathWithExt, constants.R_OK),
-      this.storageRepository.checkFileExists(sidecarPathWithoutExt, constants.R_OK),
-    ]);
-
-    let sidecarPath = null;
-    if (sidecarPathWithExtExists) {
-      sidecarPath = sidecarPathWithExt;
-    } else if (sidecarPathWithoutExtExists) {
-      sidecarPath = sidecarPathWithoutExt;
-    }
-
-    if (asset.isExternal) {
-      if (sidecarPath !== asset.sidecarPath) {
-        await this.assetRepository.update({ id: asset.id, sidecarPath });
-      }
-      return JobStatus.Success;
-    }
-
-    if (sidecarPath) {
-      this.logger.debug(`Detected sidecar at '${sidecarPath}' for asset ${asset.id}: ${asset.originalPath}`);
-      await this.assetRepository.update({ id: asset.id, sidecarPath });
-      return JobStatus.Success;
-    }
-
-    if (!isSync) {
-      return JobStatus.Failed;
-    }
-
-    this.logger.debug(`No sidecar found for asset ${asset.id}: ${asset.originalPath}`);
-    await this.assetRepository.update({ id: asset.id, sidecarPath: null });
-
-    return JobStatus.Success;
-  }
 }

server/src/types.ts

@@ -312,8 +312,7 @@ export type JobItem =
 
   // Sidecar Scanning
   | { name: JobName.SidecarQueueAll; data: IBaseJob }
-  | { name: JobName.SidecarDiscovery; data: IEntityJob }
+  | { name: JobName.SidecarCheck; data: IEntityJob }
-  | { name: JobName.SidecarSync; data: IEntityJob }
   | { name: JobName.SidecarWrite; data: ISidecarWriteJob }
 
   // Facial Recognition
@@ -400,8 +399,8 @@ export interface VectorUpdateResult {
 }
 
 export interface ImmichFile extends Express.Multer.File {
-  /** sha1 hash of file */
   uuid: string;
+  /** sha1 hash of file */
   checksum: Buffer;
 }