refactor: metadata extraction (#12359)

server/src/interfaces/map.interface.ts

@@ -26,7 +26,7 @@ export interface MapMarker extends ReverseGeocodeResult {
 
 export interface IMapRepository {
   init(): Promise<void>;
-  reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult | null>;
+  reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult>;
   getMapMarkers(ownerIds: string[], albumIds: string[], options?: MapMarkerSearchOptions): Promise<MapMarker[]>;
   fetchStyle(url: string): Promise<any>;
 }

server/src/interfaces/metadata.interface.ts

@@ -50,7 +50,7 @@ export interface ImmichTags extends Omit<Tags, TagsWithWrongTypes> {
 
 export interface IMetadataRepository {
   teardown(): Promise<void>;
-  readTags(path: string): Promise<ImmichTags | null>;
+  readTags(path: string): Promise<ImmichTags>;
   writeTags(path: string, tags: Partial<Tags>): Promise<void>;
   extractBinaryTag(tagName: string, path: string): Promise<Buffer>;
   getCountries(userIds: string[]): Promise<Array<string | null>>;

server/src/repositories/map.repository.ts

@@ -124,7 +124,7 @@ export class MapRepository implements IMapRepository {
     }
   }
 
-  async reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult | null> {
+  async reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult> {
     this.logger.debug(`Request: ${point.latitude},${point.longitude}`);
 
     const response = await this.geodataPlacesRepository
@@ -159,7 +159,7 @@ export class MapRepository implements IMapRepository {
         `Response from database for natural earth reverse geocoding latitude: ${point.latitude}, longitude: ${point.longitude} was null`,
       );
 
-      return null;
+      return { country: null, state: null, city: null };
     }
 
     this.logger.verbose(`Raw: ${JSON.stringify(ne_response, ['id', 'admin', 'admin_a3', 'type'], 2)}`);

server/src/repositories/metadata.repository.ts

@@ -36,11 +36,11 @@ export class MetadataRepository implements IMetadataRepository {
     await this.exiftool.end();
   }
 
-  readTags(path: string): Promise<ImmichTags | null> {
+  readTags(path: string): Promise<ImmichTags> {
     return this.exiftool.read(path).catch((error) => {
       this.logger.warn(`Error reading exif data (${path}): ${error}`, error?.stack);
-      return null;
+      return {};
-    }) as Promise<ImmichTags | null>;
+    }) as Promise<ImmichTags>;
   }
 
   extractBinaryTag(path: string, tagName: string): Promise<Buffer> {

server/src/services/metadata.service.spec.ts

@@ -522,13 +522,13 @@ describe(MetadataService.name, () => {
     it('should extract the correct video orientation', async () => {
       assetMock.getByIds.mockResolvedValue([assetStub.video]);
       mediaMock.probe.mockResolvedValue(probeStub.videoStreamVertical2160p);
-      metadataMock.readTags.mockResolvedValue(null);
+      metadataMock.readTags.mockResolvedValue({});
 
       await sut.handleMetadataExtraction({ id: assetStub.video.id });
 
       expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.video.id]);
       expect(assetMock.upsertExif).toHaveBeenCalledWith(
-        expect.objectContaining({ orientation: Orientation.Rotate270CW }),
+        expect.objectContaining({ orientation: Orientation.Rotate270CW.toString() }),
       );
     });
 
@@ -814,6 +814,9 @@ describe(MetadataService.name, () => {
         projectionType: 'EQUIRECTANGULAR',
         timeZone: tags.tz,
         rating: tags.Rating,
+        country: null,
+        state: null,
+        city: null,
       });
       expect(assetMock.update).toHaveBeenCalledWith({
         id: assetStub.image.id,

server/src/services/metadata.service.ts

@@ -1,5 +1,5 @@
 import { Inject, Injectable } from '@nestjs/common';
-import { ContainerDirectoryItem, ExifDateTime, Tags } from 'exiftool-vendored';
+import { ContainerDirectoryItem, ExifDateTime, Maybe, Tags } from 'exiftool-vendored';
 import { firstDateTime } from 'exiftool-vendored/dist/FirstDateTime';
 import _ from 'lodash';
 import { Duration } from 'luxon';
@@ -11,7 +11,6 @@ import { SystemConfigCore } from 'src/cores/system-config.core';
 import { OnEmit } from 'src/decorators';
 import { AssetFaceEntity } from 'src/entities/asset-face.entity';
 import { AssetEntity } from 'src/entities/asset.entity';
-import { ExifEntity } from 'src/entities/exif.entity';
 import { PersonEntity } from 'src/entities/person.entity';
 import { AssetType, SourceType } from 'src/enum';
 import { IAlbumRepository } from 'src/interfaces/album.interface';
@@ -30,7 +29,7 @@ import {
   QueueName,
 } from 'src/interfaces/job.interface';
 import { ILoggerRepository } from 'src/interfaces/logger.interface';
-import { IMapRepository } from 'src/interfaces/map.interface';
+import { IMapRepository, ReverseGeocodeResult } from 'src/interfaces/map.interface';
 import { IMediaRepository } from 'src/interfaces/media.interface';
 import { IMetadataRepository, ImmichTags } from 'src/interfaces/metadata.interface';
 import { IMoveRepository } from 'src/interfaces/move.interface';
@@ -56,23 +55,16 @@ const EXIF_DATE_TAGS: Array<keyof Tags> = [
 ];
 
 export enum Orientation {
-  Horizontal = '1',
+  Horizontal = 1,
-  MirrorHorizontal = '2',
+  MirrorHorizontal = 2,
-  Rotate180 = '3',
+  Rotate180 = 3,
-  MirrorVertical = '4',
+  MirrorVertical = 4,
-  MirrorHorizontalRotate270CW = '5',
+  MirrorHorizontalRotate270CW = 5,
-  Rotate90CW = '6',
+  Rotate90CW = 6,
-  MirrorHorizontalRotate90CW = '7',
+  MirrorHorizontalRotate90CW = 7,
-  Rotate270CW = '8',
+  Rotate270CW = 8,
 }
 
-type ExifEntityWithoutGeocodeAndTypeOrm = Omit<ExifEntity, 'city' | 'state' | 'country' | 'description'> & {
-  dateTimeOriginal: Date;
-};
-
-const exifDate = (dt: ExifDateTime | string | undefined) => (dt instanceof ExifDateTime ? dt?.toDate() : null);
-const tzOffset = (dt: ExifDateTime | string | undefined) => (dt instanceof ExifDateTime ? dt?.tzoffsetMinutes : null);
-
 const validate = <T>(value: T): NonNullable<T> | null => {
   // handle lists of numbers
   if (Array.isArray(value)) {
@@ -218,36 +210,73 @@ export class MetadataService {
   }
 
   async handleMetadataExtraction({ id }: IEntityJob): Promise<JobStatus> {
-    const { metadata } = await this.configCore.getConfig({ withCache: true });
+    const { metadata, reverseGeocoding } = await this.configCore.getConfig({ withCache: true });
     const [asset] = await this.assetRepository.getByIds([id]);
     if (!asset) {
       return JobStatus.FAILED;
     }
 
-    const { exifData, exifTags } = await this.exifData(asset);
+    const stats = await this.storageRepository.stat(asset.originalPath);
 
-    if (asset.type === AssetType.VIDEO) {
+    const exifTags = await this.getExifTags(asset);
-      await this.applyVideoMetadata(asset, exifData);
+
-    }
+    this.logger.verbose('Exif Tags', exifTags);
+
+    const { dateTimeOriginal, localDateTime, timeZone, modifyDate } = this.getDates(asset, exifTags);
+    const { latitude, longitude, country, state, city } = await this.getGeo(exifTags, reverseGeocoding);
+
+    const exifData = {
+      assetId: asset.id,
+
+      // dates
+      dateTimeOriginal,
+      modifyDate,
+      timeZone,
+
+      // gps
+      latitude,
+      longitude,
+      country,
+      state,
+      city,
+
+      // image/file
+      fileSizeInByte: stats.size,
+      exifImageHeight: validate(exifTags.ImageHeight),
+      exifImageWidth: validate(exifTags.ImageWidth),
+      orientation: validate(exifTags.Orientation)?.toString() ?? null,
+      projectionType: exifTags.ProjectionType ? String(exifTags.ProjectionType).toUpperCase() : null,
+      bitsPerSample: this.getBitsPerSample(exifTags),
+      colorspace: exifTags.ColorSpace ?? null,
+
+      // camera
+      make: exifTags.Make ?? null,
+      model: exifTags.Model ?? null,
+      fps: validate(Number.parseFloat(exifTags.VideoFrameRate!)),
+      iso: validate(exifTags.ISO),
+      exposureTime: exifTags.ExposureTime ?? null,
+      lensModel: exifTags.LensModel ?? null,
+      fNumber: validate(exifTags.FNumber),
+      focalLength: validate(exifTags.FocalLength),
+
+      // comments
+      description: String(exifTags.ImageDescription || exifTags.Description || '').trim(),
+      profileDescription: exifTags.ProfileDescription || null,
+      rating: exifTags.Rating ?? null,
+
+      // grouping
+      livePhotoCID: (exifTags.ContentIdentifier || exifTags.MediaGroupUUID) ?? null,
+      autoStackId: this.getAutoStackId(exifTags),
+    };
 
-    await this.applyMotionPhotos(asset, exifTags);
-    await this.applyReverseGeocoding(asset, exifData);
     await this.applyTagList(asset, exifTags);
+    await this.applyMotionPhotos(asset, exifTags);
 
     await this.assetRepository.upsertExif(exifData);
 
-    const dateTimeOriginal = exifData.dateTimeOriginal;
-    let localDateTime = dateTimeOriginal ?? undefined;
-
-    const timeZoneOffset = tzOffset(firstDateTime(exifTags as Tags)) ?? 0;
-
-    if (dateTimeOriginal && timeZoneOffset) {
-      localDateTime = new Date(dateTimeOriginal.getTime() + timeZoneOffset * 60_000);
-    }
-
     await this.assetRepository.update({
       id: asset.id,
-      duration: asset.duration,
+      duration: exifTags.Duration?.toString() ?? null,
       localDateTime,
       fileCreatedAt: exifData.dateTimeOriginal ?? undefined,
     });
@@ -338,25 +367,20 @@ export class MetadataService {
     return JobStatus.SUCCESS;
   }
 
-  private async applyReverseGeocoding(asset: AssetEntity, exifData: ExifEntityWithoutGeocodeAndTypeOrm) {
+  private async getExifTags(asset: AssetEntity): Promise<ImmichTags> {
-    const { latitude, longitude } = exifData;
+    const mediaTags = await this.repository.readTags(asset.originalPath);
-    const { reverseGeocoding } = await this.configCore.getConfig({ withCache: true });
+    const sidecarTags = asset.sidecarPath ? await this.repository.readTags(asset.sidecarPath) : {};
-    if (!reverseGeocoding.enabled || !longitude || !latitude) {
+    const videoTags = asset.type === AssetType.VIDEO ? await this.getVideoTags(asset.originalPath) : {};
-      return;
+
+    // make sure dates comes from sidecar
+    const sidecarDate = firstDateTime(sidecarTags as Tags, EXIF_DATE_TAGS);
+    if (sidecarDate) {
+      for (const tag of EXIF_DATE_TAGS) {
+        delete mediaTags[tag];
+      }
     }
 
-    try {
+    return { ...mediaTags, ...videoTags, ...sidecarTags };
-      const reverseGeocode = await this.mapRepository.reverseGeocode({ latitude, longitude });
-      if (!reverseGeocode) {
-        return;
-      }
-      Object.assign(exifData, reverseGeocode);
-    } catch (error: Error | any) {
-      this.logger.warn(
-        `Unable to run reverse geocoding due to ${error} for asset ${asset.id} at ${asset.originalPath}`,
-        error?.stack,
-      );
-    }
   }
 
   private async applyTagList(asset: AssetEntity, exifTags: ImmichTags) {
@@ -576,66 +600,65 @@ export class MetadataService {
     );
   }
 
-  private async exifData(
+  private getDates(asset: AssetEntity, exifTags: ImmichTags) {
-    asset: AssetEntity,
+    const dateTime = firstDateTime(exifTags as Maybe<Tags>, EXIF_DATE_TAGS);
-  ): Promise<{ exifData: ExifEntityWithoutGeocodeAndTypeOrm; exifTags: ImmichTags }> {
+    this.logger.debug(`Asset ${asset.id} date time is ${dateTime}`);
-    const stats = await this.storageRepository.stat(asset.originalPath);
-    const mediaTags = await this.repository.readTags(asset.originalPath);
-    const sidecarTags = asset.sidecarPath ? await this.repository.readTags(asset.sidecarPath) : null;
 
-    // ensure date from sidecar is used if present
+    // created
-    const hasDateOverride = !!this.getDateTimeOriginal(sidecarTags);
+    let dateTimeOriginal = dateTime?.toDate();
-    if (mediaTags && hasDateOverride) {
+    if (!dateTimeOriginal) {
-      for (const tag of EXIF_DATE_TAGS) {
+      this.logger.warn(`Asset ${asset.id} has no valid date (${dateTime}), falling back to asset.fileCreatedAt`);
-        delete mediaTags[tag];
+      dateTimeOriginal = asset.fileCreatedAt;
-      }
     }
 
-    const exifTags = { ...mediaTags, ...sidecarTags };
+    // timezone
+    let timeZone = exifTags.tz ?? null;
+    if (timeZone == null && dateTime?.rawValue?.endsWith('+00:00')) {
+      // exiftool-vendored returns "no timezone" information even though "+00:00" might be set explicitly
+      // https://github.com/photostructure/exiftool-vendored.js/issues/203
+      timeZone = 'UTC+0';
+    }
 
-    this.logger.verbose('Exif Tags', exifTags);
+    if (timeZone) {
+      this.logger.debug(`Asset ${asset.id} timezone is ${timeZone} (via ${exifTags.tzSource})`);
+    } else {
+      this.logger.warn(`Asset ${asset.id} has no time zone information`);
+    }
 
-    const dateTimeOriginalWithRawValue = this.getDateTimeOriginalWithRawValue(exifTags);
+    // offset minutes
-    const dateTimeOriginal = dateTimeOriginalWithRawValue.exifDate ?? asset.fileCreatedAt;
+    const offsetMinutes = dateTime?.tzoffsetMinutes || 0;
-    const timeZone = this.getTimeZone(exifTags, dateTimeOriginalWithRawValue.rawValue);
+    let localDateTime = dateTimeOriginal;
+    if (offsetMinutes) {
+      localDateTime = new Date(dateTimeOriginal.getTime() + offsetMinutes * 60_000);
+      this.logger.debug(`Asset ${asset.id} local time is offset by ${offsetMinutes} minutes`);
+    }
 
-    const exifData = {
+    return {
-      // altitude: tags.GPSAltitude ?? null,
-      assetId: asset.id,
-      bitsPerSample: this.getBitsPerSample(exifTags),
-      colorspace: exifTags.ColorSpace ?? null,
       dateTimeOriginal,
-      description: String(exifTags.ImageDescription || exifTags.Description || '').trim(),
-      exifImageHeight: validate(exifTags.ImageHeight),
-      exifImageWidth: validate(exifTags.ImageWidth),
-      exposureTime: exifTags.ExposureTime ?? null,
-      fileSizeInByte: stats.size,
-      fNumber: validate(exifTags.FNumber),
-      focalLength: validate(exifTags.FocalLength),
-      fps: validate(Number.parseFloat(exifTags.VideoFrameRate!)),
-      iso: validate(exifTags.ISO),
-      latitude: validate(exifTags.GPSLatitude),
-      lensModel: exifTags.LensModel ?? null,
-      livePhotoCID: (exifTags.ContentIdentifier || exifTags.MediaGroupUUID) ?? null,
-      autoStackId: this.getAutoStackId(exifTags),
-      longitude: validate(exifTags.GPSLongitude),
-      make: exifTags.Make ?? null,
-      model: exifTags.Model ?? null,
-      modifyDate: exifDate(exifTags.ModifyDate) ?? asset.fileModifiedAt,
-      orientation: validate(exifTags.Orientation)?.toString() ?? null,
-      profileDescription: exifTags.ProfileDescription || null,
-      projectionType: exifTags.ProjectionType ? String(exifTags.ProjectionType).toUpperCase() : null,
       timeZone,
-      rating: exifTags.Rating ?? null,
+      localDateTime,
+      modifyDate: (exifTags.ModifyDate as ExifDateTime)?.toDate() ?? asset.fileModifiedAt,
     };
+  }
 
-    if (exifData.latitude === 0 && exifData.longitude === 0) {
+  private async getGeo(tags: ImmichTags, reverseGeocoding: SystemConfig['reverseGeocoding']) {
-      this.logger.warn('Exif data has latitude and longitude of 0, setting to null');
+    let latitude = validate(tags.GPSLatitude);
-      exifData.latitude = null;
+    let longitude = validate(tags.GPSLongitude);
-      exifData.longitude = null;
+
+    // TODO take ref into account
+
+    if (latitude === 0 && longitude === 0) {
+      this.logger.warn('Latitude and longitude of 0, setting to null');
+      latitude = null;
+      longitude = null;
     }
 
-    return { exifData, exifTags };
+    let result: ReverseGeocodeResult = { country: null, state: null, city: null };
+    if (reverseGeocoding.enabled && longitude && latitude) {
+      result = await this.mapRepository.reverseGeocode({ latitude, longitude });
+    }
+
+    return { ...result, latitude, longitude };
   }
 
   private getAutoStackId(tags: ImmichTags | null): string | null {
@@ -645,28 +668,6 @@ export class MetadataService {
     return tags.BurstID ?? tags.BurstUUID ?? tags.CameraBurstID ?? tags.MediaUniqueID ?? null;
   }
 
-  private getDateTimeOriginal(tags: ImmichTags | Tags | null) {
-    return this.getDateTimeOriginalWithRawValue(tags).exifDate;
-  }
-
-  private getDateTimeOriginalWithRawValue(tags: ImmichTags | Tags | null): { exifDate: Date | null; rawValue: string } {
-    if (!tags) {
-      return { exifDate: null, rawValue: '' };
-    }
-    const first = firstDateTime(tags as Tags, EXIF_DATE_TAGS);
-    return { exifDate: exifDate(first), rawValue: first?.rawValue ?? '' };
-  }
-
-  private getTimeZone(exifTags: ImmichTags, rawValue: string) {
-    const timeZone = exifTags.tz ?? null;
-    if (timeZone == null && rawValue.endsWith('+00:00')) {
-      // exiftool-vendored returns "no timezone" information even though "+00:00" might be set explicitly
-      // https://github.com/photostructure/exiftool-vendored.js/issues/203
-      return 'UTC+0';
-    }
-    return timeZone;
-  }
-
   private getBitsPerSample(tags: ImmichTags): number | null {
     const bitDepthTags = [
       tags.BitsPerSample,
@@ -685,33 +686,37 @@ export class MetadataService {
     return bitsPerSample;
   }
 
-  private async applyVideoMetadata(asset: AssetEntity, exifData: ExifEntityWithoutGeocodeAndTypeOrm) {
+  private async getVideoTags(originalPath: string) {
-    const { videoStreams, format } = await this.mediaRepository.probe(asset.originalPath);
+    const { videoStreams, format } = await this.mediaRepository.probe(originalPath);
+
+    const tags: Pick<ImmichTags, 'Duration' | 'Orientation'> = {};
 
     if (videoStreams[0]) {
       switch (videoStreams[0].rotation) {
         case -90: {
-          exifData.orientation = Orientation.Rotate90CW;
+          tags.Orientation = Orientation.Rotate90CW;
           break;
         }
         case 0: {
-          exifData.orientation = Orientation.Horizontal;
+          tags.Orientation = Orientation.Horizontal;
           break;
         }
         case 90: {
-          exifData.orientation = Orientation.Rotate270CW;
+          tags.Orientation = Orientation.Rotate270CW;
           break;
         }
         case 180: {
-          exifData.orientation = Orientation.Rotate180;
+          tags.Orientation = Orientation.Rotate180;
           break;
         }
       }
     }
 
     if (format.duration) {
-      asset.duration = Duration.fromObject({ seconds: format.duration }).toFormat('hh:mm:ss.SSS');
+      tags.Duration = Duration.fromObject({ seconds: format.duration }).toFormat('hh:mm:ss.SSS');
     }
+
+    return tags;
   }
 
   private async processSidecar(id: string, isSync: boolean): Promise<JobStatus> {