fix(server): use stat instead of exifinfo for file date metadata (#17311)

* use stat instead of filecreatedate

* update tests

* unused import

server/src/services/metadata.service.spec.ts

@@ -1,5 +1,6 @@
 import { BinaryField, ExifDateTime } from 'exiftool-vendored';
 import { randomBytes } from 'node:crypto';
+import { Stats } from 'node:fs';
 import { constants } from 'node:fs/promises';
 import { defaults } from 'src/config';
 import { AssetEntity } from 'src/entities/asset.entity';
@@ -21,14 +22,8 @@ describe(MetadataService.name, () => {
   let mocks: ServiceMocks;
 
   const mockReadTags = (exifData?: Partial<ImmichTags>, sidecarData?: Partial<ImmichTags>) => {
-    exifData = {
-      FileSize: '123456',
-      FileCreateDate: '2024-01-01T00:00:00.000Z',
-      FileModifyDate: '2024-01-01T00:00:00.000Z',
-      ...exifData,
-    };
     mocks.metadata.readTags.mockReset();
-    mocks.metadata.readTags.mockResolvedValueOnce(exifData);
+    mocks.metadata.readTags.mockResolvedValueOnce(exifData ?? {});
     mocks.metadata.readTags.mockResolvedValueOnce(sidecarData ?? {});
   };
 
@@ -114,6 +109,17 @@ describe(MetadataService.name, () => {
   });
 
   describe('handleMetadataExtraction', () => {
+    beforeEach(() => {
+      const time = new Date('2022-01-01T00:00:00.000Z');
+      const timeMs = time.valueOf();
+      mocks.storage.stat.mockResolvedValue({
+        size: 123_456,
+        mtime: time,
+        mtimeMs: timeMs,
+        birthtimeMs: timeMs,
+      } as Stats);
+    });
+
     it('should handle an asset that could not be found', async () => {
       await expect(sut.handleMetadataExtraction({ id: assetStub.image.id })).resolves.toBe(JobStatus.FAILED);
 
@@ -145,10 +151,13 @@ describe(MetadataService.name, () => {
       const fileCreatedAt = new Date('2022-01-01T00:00:00.000Z');
       const fileModifiedAt = new Date('2021-01-01T00:00:00.000Z');
       mocks.asset.getByIds.mockResolvedValue([assetStub.image]);
-      mockReadTags({
+      mocks.storage.stat.mockResolvedValue({
-        FileCreateDate: fileCreatedAt.toISOString(),
+        size: 123_456,
-        FileModifyDate: fileModifiedAt.toISOString(),
+        mtime: fileModifiedAt,
-      });
+        mtimeMs: fileModifiedAt.valueOf(),
+        birthtimeMs: fileCreatedAt.valueOf(),
+      } as Stats);
+      mockReadTags();
 
       await sut.handleMetadataExtraction({ id: assetStub.image.id });
       expect(mocks.asset.getByIds).toHaveBeenCalledWith([assetStub.image.id], { faces: { person: false } });
@@ -168,10 +177,13 @@ describe(MetadataService.name, () => {
       const fileCreatedAt = new Date('2021-01-01T00:00:00.000Z');
       const fileModifiedAt = new Date('2022-01-01T00:00:00.000Z');
       mocks.asset.getByIds.mockResolvedValue([assetStub.image]);
-      mockReadTags({
+      mocks.storage.stat.mockResolvedValue({
-        FileCreateDate: fileCreatedAt.toISOString(),
+        size: 123_456,
-        FileModifyDate: fileModifiedAt.toISOString(),
+        mtime: fileModifiedAt,
-      });
+        mtimeMs: fileModifiedAt.valueOf(),
+        birthtimeMs: fileCreatedAt.valueOf(),
+      } as Stats);
+      mockReadTags();
 
       await sut.handleMetadataExtraction({ id: assetStub.image.id });
       expect(mocks.asset.getByIds).toHaveBeenCalledWith([assetStub.image.id], { faces: { person: false } });
@@ -206,10 +218,14 @@ describe(MetadataService.name, () => {
 
     it('should handle lists of numbers', async () => {
       mocks.asset.getByIds.mockResolvedValue([assetStub.image]);
+      mocks.storage.stat.mockResolvedValue({
+        size: 123_456,
+        mtime: assetStub.image.fileModifiedAt,
+        mtimeMs: assetStub.image.fileModifiedAt.valueOf(),
+        birthtimeMs: assetStub.image.fileCreatedAt.valueOf(),
+      } as Stats);
       mockReadTags({
         ISO: [160],
-        FileCreateDate: assetStub.image.fileCreatedAt.toISOString(),
-        FileModifyDate: assetStub.image.fileModifiedAt.toISOString(),
       });
 
       await sut.handleMetadataExtraction({ id: assetStub.image.id });
@@ -228,11 +244,15 @@ describe(MetadataService.name, () => {
       mocks.asset.getByIds.mockResolvedValue([assetStub.withLocation]);
       mocks.systemMetadata.get.mockResolvedValue({ reverseGeocoding: { enabled: true } });
       mocks.map.reverseGeocode.mockResolvedValue({ city: 'City', state: 'State', country: 'Country' });
+      mocks.storage.stat.mockResolvedValue({
+        size: 123_456,
+        mtime: assetStub.withLocation.fileModifiedAt,
+        mtimeMs: assetStub.withLocation.fileModifiedAt.valueOf(),
+        birthtimeMs: assetStub.withLocation.fileCreatedAt.valueOf(),
+      } as Stats);
       mockReadTags({
         GPSLatitude: assetStub.withLocation.exifInfo!.latitude!,
         GPSLongitude: assetStub.withLocation.exifInfo!.longitude!,
-        FileCreateDate: assetStub.withLocation.fileCreatedAt.toISOString(),
-        FileModifyDate: assetStub.withLocation.fileModifiedAt.toISOString(),
       });
 
       await sut.handleMetadataExtraction({ id: assetStub.image.id });
@@ -475,6 +495,12 @@ describe(MetadataService.name, () => {
 
     it('should extract the MotionPhotoVideo tag from Samsung HEIC motion photos', async () => {
       mocks.asset.getByIds.mockResolvedValue([{ ...assetStub.livePhotoWithOriginalFileName, livePhotoVideoId: null }]);
+      mocks.storage.stat.mockResolvedValue({
+        size: 123_456,
+        mtime: assetStub.livePhotoWithOriginalFileName.fileModifiedAt,
+        mtimeMs: assetStub.livePhotoWithOriginalFileName.fileModifiedAt.valueOf(),
+        birthtimeMs: assetStub.livePhotoWithOriginalFileName.fileCreatedAt.valueOf(),
+      } as Stats);
       mockReadTags({
         Directory: 'foo/bar/',
         MotionPhoto: 1,
@@ -483,8 +509,6 @@ describe(MetadataService.name, () => {
         // instead of the EmbeddedVideoFile, since HEIC MotionPhotos include both
         EmbeddedVideoFile: new BinaryField(0, ''),
         EmbeddedVideoType: 'MotionPhoto_Data',
-        FileCreateDate: assetStub.livePhotoWithOriginalFileName.fileCreatedAt.toISOString(),
-        FileModifyDate: assetStub.livePhotoWithOriginalFileName.fileModifiedAt.toISOString(),
       });
       mocks.crypto.hashSha1.mockReturnValue(randomBytes(512));
       mocks.asset.create.mockResolvedValue(assetStub.livePhotoMotionAsset);
@@ -525,14 +549,18 @@ describe(MetadataService.name, () => {
     });
 
     it('should extract the EmbeddedVideo tag from Samsung JPEG motion photos', async () => {
+      mocks.storage.stat.mockResolvedValue({
+        size: 123_456,
+        mtime: assetStub.livePhotoWithOriginalFileName.fileModifiedAt,
+        mtimeMs: assetStub.livePhotoWithOriginalFileName.fileModifiedAt.valueOf(),
+        birthtimeMs: assetStub.livePhotoWithOriginalFileName.fileCreatedAt.valueOf(),
+      } as Stats);
       mocks.asset.getByIds.mockResolvedValue([{ ...assetStub.livePhotoWithOriginalFileName, livePhotoVideoId: null }]);
       mockReadTags({
         Directory: 'foo/bar/',
         EmbeddedVideoFile: new BinaryField(0, ''),
         EmbeddedVideoType: 'MotionPhoto_Data',
         MotionPhoto: 1,
-        FileCreateDate: assetStub.livePhotoWithOriginalFileName.fileCreatedAt.toISOString(),
-        FileModifyDate: assetStub.livePhotoWithOriginalFileName.fileModifiedAt.toISOString(),
       });
       mocks.crypto.hashSha1.mockReturnValue(randomBytes(512));
       mocks.asset.create.mockResolvedValue(assetStub.livePhotoMotionAsset);
@@ -574,13 +602,17 @@ describe(MetadataService.name, () => {
 
     it('should extract the motion photo video from the XMP directory entry ', async () => {
       mocks.asset.getByIds.mockResolvedValue([{ ...assetStub.livePhotoWithOriginalFileName, livePhotoVideoId: null }]);
+      mocks.storage.stat.mockResolvedValue({
+        size: 123_456,
+        mtime: assetStub.livePhotoWithOriginalFileName.fileModifiedAt,
+        mtimeMs: assetStub.livePhotoWithOriginalFileName.fileModifiedAt.valueOf(),
+        birthtimeMs: assetStub.livePhotoWithOriginalFileName.fileCreatedAt.valueOf(),
+      } as Stats);
       mockReadTags({
         Directory: 'foo/bar/',
         MotionPhoto: 1,
         MicroVideo: 1,
         MicroVideoOffset: 1,
-        FileCreateDate: assetStub.livePhotoWithOriginalFileName.fileCreatedAt.toISOString(),
-        FileModifyDate: assetStub.livePhotoWithOriginalFileName.fileModifiedAt.toISOString(),
       });
       mocks.crypto.hashSha1.mockReturnValue(randomBytes(512));
       mocks.asset.create.mockResolvedValue(assetStub.livePhotoMotionAsset);

server/src/services/metadata.service.ts

@@ -1,9 +1,10 @@
 import { Injectable } from '@nestjs/common';
-import { ContainerDirectoryItem, ExifDateTime, Maybe, Tags } from 'exiftool-vendored';
+import { ContainerDirectoryItem, Maybe, Tags } from 'exiftool-vendored';
 import { firstDateTime } from 'exiftool-vendored/dist/FirstDateTime';
 import { Insertable } from 'kysely';
 import _ from 'lodash';
 import { Duration } from 'luxon';
+import { Stats } from 'node:fs';
 import { constants } from 'node:fs/promises';
 import path from 'node:path';
 import { JOBS_ASSET_PAGINATION_SIZE } from 'src/constants';
@@ -77,6 +78,11 @@ const validateRange = (value: number | undefined, min: number, max: number): Non
 
 type ImmichTagsWithFaces = ImmichTags & { RegionInfo: NonNullable<ImmichTags['RegionInfo']> };
 
+type Dates = {
+  dateTimeOriginal: Date;
+  localDateTime: Date;
+};
+
 @Injectable()
 export class MetadataService extends BaseService {
   @OnEvent({ name: 'app.bootstrap', workers: [ImmichWorker.MICROSERVICES] })
@@ -171,18 +177,13 @@ export class MetadataService extends BaseService {
       return JobStatus.FAILED;
     }
 
-    const exifTags = await this.getExifTags(asset);
+    const [exifTags, stats] = await Promise.all([
-    if (!exifTags.FileCreateDate || !exifTags.FileModifyDate || exifTags.FileSize === undefined) {
+      this.getExifTags(asset),
-      this.logger.warn(`Missing file creation or modification date for asset ${asset.id}: ${asset.originalPath}`);
+      this.storageRepository.stat(asset.originalPath),
-      const stat = await this.storageRepository.stat(asset.originalPath);
+    ]);
-      exifTags.FileCreateDate = stat.ctime.toISOString();
-      exifTags.FileModifyDate = stat.mtime.toISOString();
-      exifTags.FileSize = stat.size.toString();
-    }
-
     this.logger.verbose('Exif Tags', exifTags);
 
-    const { dateTimeOriginal, localDateTime, timeZone, modifyDate } = this.getDates(asset, exifTags);
+    const dates = this.getDates(asset, exifTags, stats);
 
     const { width, height } = this.getImageDimensions(exifTags);
     let geo: ReverseGeocodeResult, latitude: number | null, longitude: number | null;
@@ -200,9 +201,9 @@ export class MetadataService extends BaseService {
       assetId: asset.id,
 
       // dates
-      dateTimeOriginal,
+      dateTimeOriginal: dates.dateTimeOriginal,
-      modifyDate,
+      modifyDate: stats.mtime,
-      timeZone,
+      timeZone: dates.timeZone,
 
       // gps
       latitude,
@@ -212,7 +213,7 @@ export class MetadataService extends BaseService {
       city: geo.city,
 
       // image/file
-      fileSizeInByte: Number.parseInt(exifTags.FileSize!),
+      fileSizeInByte: stats.size,
       exifImageHeight: validate(height),
       exifImageWidth: validate(width),
       orientation: validate(exifTags.Orientation)?.toString() ?? null,
@@ -245,15 +246,15 @@ export class MetadataService extends BaseService {
       this.assetRepository.update({
         id: asset.id,
         duration: exifTags.Duration?.toString() ?? null,
-        localDateTime,
+        localDateTime: dates.localDateTime,
-        fileCreatedAt: exifData.dateTimeOriginal ?? undefined,
+        fileCreatedAt: dates.dateTimeOriginal ?? undefined,
-        fileModifiedAt: exifData.modifyDate ?? undefined,
+        fileModifiedAt: stats.mtime,
       }),
       this.applyTagList(asset, exifTags),
     ];
 
     if (this.isMotionPhoto(asset, exifTags)) {
-      promises.push(this.applyMotionPhotos(asset, exifTags, exifData.fileSizeInByte!));
+      promises.push(this.applyMotionPhotos(asset, exifTags, dates, stats));
     }
 
     if (isFaceImportEnabled(metadata) && this.hasTaggedFaces(exifTags)) {
@@ -432,7 +433,7 @@ export class MetadataService extends BaseService {
     return asset.type === AssetType.IMAGE && !!(tags.MotionPhoto || tags.MicroVideo);
   }
 
-  private async applyMotionPhotos(asset: AssetEntity, tags: ImmichTags, fileSize: number) {
+  private async applyMotionPhotos(asset: AssetEntity, tags: ImmichTags, dates: Dates, stats: Stats) {
     const isMotionPhoto = tags.MotionPhoto;
     const isMicroVideo = tags.MicroVideo;
     const videoOffset = tags.MicroVideoOffset;
@@ -466,7 +467,7 @@ export class MetadataService extends BaseService {
     this.logger.debug(`Starting motion photo video extraction for asset ${asset.id}: ${asset.originalPath}`);
 
     try {
-      const position = fileSize - length - padding;
+      const position = stats.size - length - padding;
       let video: Buffer;
       // Samsung MotionPhoto video extraction
       //     HEIC-encoded
@@ -505,13 +506,12 @@ export class MetadataService extends BaseService {
         }
       } else {
         const motionAssetId = this.cryptoRepository.randomUUID();
-        const dates = this.getDates(asset, tags);
         motionAsset = await this.assetRepository.create({
           id: motionAssetId,
           libraryId: asset.libraryId,
           type: AssetType.VIDEO,
           fileCreatedAt: dates.dateTimeOriginal,
-          fileModifiedAt: dates.modifyDate,
+          fileModifiedAt: stats.mtime,
           localDateTime: dates.localDateTime,
           checksum,
           ownerId: asset.ownerId,
@@ -634,7 +634,7 @@ export class MetadataService extends BaseService {
     }
   }
 
-  private getDates(asset: AssetEntity, exifTags: ImmichTags) {
+  private getDates(asset: AssetEntity, exifTags: ImmichTags, stats: Stats) {
     const dateTime = firstDateTime(exifTags as Maybe<Tags>, EXIF_DATE_TAGS);
     this.logger.verbose(`Date and time is ${dateTime} for asset ${asset.id}: ${asset.originalPath}`);
 
@@ -654,17 +654,16 @@ export class MetadataService extends BaseService {
       this.logger.debug(`No timezone information found for asset ${asset.id}: ${asset.originalPath}`);
     }
 
-    const modifyDate = this.toDate(exifTags.FileModifyDate!);
     let dateTimeOriginal = dateTime?.toDate();
     let localDateTime = dateTime?.toDateTime().setZone('UTC', { keepLocalTime: true }).toJSDate();
     if (!localDateTime || !dateTimeOriginal) {
-      const fileCreatedAt = this.toDate(exifTags.FileCreateDate!);
+      // FileCreateDate is not available on linux, likely because exiftool hasn't integrated the statx syscall yet
-      const earliestDate = this.earliestDate(fileCreatedAt, modifyDate);
+      // birthtime is not available in Docker on macOS, so it appears as 0
+      const earliestDate = stats.birthtimeMs ? new Date(Math.min(stats.mtimeMs, stats.birthtimeMs)) : stats.mtime;
       this.logger.debug(
-        `No exif date time found, falling back on ${earliestDate.toISOString()}, earliest of file creation and modification for assset ${asset.id}: ${asset.originalPath}`,
+        `No exif date time found, falling back on ${earliestDate.toISOString()}, earliest of file creation and modification for asset ${asset.id}: ${asset.originalPath}`,
       );
-      dateTimeOriginal = earliestDate;
+      dateTimeOriginal = localDateTime = earliestDate;
-      localDateTime = earliestDate;
     }
 
     this.logger.verbose(
@@ -675,18 +674,9 @@ export class MetadataService extends BaseService {
       dateTimeOriginal,
       timeZone,
       localDateTime,
-      modifyDate,
     };
   }
 
-  private toDate(date: string | ExifDateTime): Date {
-    return typeof date === 'string' ? new Date(date) : date.toDate();
-  }
-
-  private earliestDate(a: Date, b: Date) {
-    return new Date(Math.min(a.valueOf(), b.valueOf()));
-  }
-
   private hasGeo(tags: ImmichTags): tags is ImmichTags & { GPSLatitude: number; GPSLongitude: number } {
     return (
       tags.GPSLatitude !== undefined &&

server/test/medium/specs/metadata.service.spec.ts

@@ -39,7 +39,12 @@ describe(MetadataService.name, () => {
   beforeEach(() => {
     ({ sut, mocks } = newTestService(MetadataService, { metadata: metadataRepository }));
 
-    mocks.storage.stat.mockResolvedValue({ size: 123_456, ctime: new Date(), mtime: new Date() } as Stats);
+    mocks.storage.stat.mockResolvedValue({
+      size: 123_456,
+      mtime: new Date(654_321),
+      mtimeMs: 654_321,
+      birthtimeMs: 654_322,
+    } as Stats);
 
     delete process.env.TZ;
   });
@@ -54,8 +59,6 @@ describe(MetadataService.name, () => {
         description: 'should handle no time zone information',
         exifData: {
           DateTimeOriginal: '2022:01:01 00:00:00',
-          FileCreateDate: '2022:01:01 00:00:00',
-          FileModifyDate: '2022:01:01 00:00:00',
         },
         expected: {
           localDateTime: '2022-01-01T00:00:00.000Z',
@@ -68,8 +71,6 @@ describe(MetadataService.name, () => {
         serverTimeZone: 'America/Los_Angeles',
         exifData: {
           DateTimeOriginal: '2022:01:01 00:00:00',
-          FileCreateDate: '2022:01:01 00:00:00',
-          FileModifyDate: '2022:01:01 00:00:00',
         },
         expected: {
           localDateTime: '2022-01-01T00:00:00.000Z',
@@ -82,8 +83,6 @@ describe(MetadataService.name, () => {
         serverTimeZone: 'Europe/Brussels',
         exifData: {
           DateTimeOriginal: '2022:01:01 00:00:00',
-          FileCreateDate: '2022:01:01 00:00:00',
-          FileModifyDate: '2022:01:01 00:00:00',
         },
         expected: {
           localDateTime: '2022-01-01T00:00:00.000Z',
@@ -96,8 +95,6 @@ describe(MetadataService.name, () => {
         serverTimeZone: 'Europe/Brussels',
         exifData: {
           DateTimeOriginal: '2022:06:01 00:00:00',
-          FileCreateDate: '2022:06:01 00:00:00',
-          FileModifyDate: '2022:06:01 00:00:00',
         },
         expected: {
           localDateTime: '2022-06-01T00:00:00.000Z',
@@ -109,8 +106,6 @@ describe(MetadataService.name, () => {
         description: 'should handle a +13:00 time zone',
         exifData: {
           DateTimeOriginal: '2022:01:01 00:00:00+13:00',
-          FileCreateDate: '2022:01:01 00:00:00+13:00',
-          FileModifyDate: '2022:01:01 00:00:00+13:00',
         },
         expected: {
           localDateTime: '2022-01-01T00:00:00.000Z',