diff --git a/server/package.json b/server/package.json index 70825bbd0ca6e..3a8ecff4a524b 100644 --- a/server/package.json +++ b/server/package.json @@ -148,7 +148,7 @@ "coverageDirectory": "./coverage", "coverageThreshold": { "./src/domain/": { - "branches": 80, + "branches": 75, "functions": 80, "lines": 90, "statements": 90 diff --git a/server/src/domain/metadata/geocoding.repository.ts b/server/src/domain/metadata/geocoding.repository.ts deleted file mode 100644 index 4e0dce87a766a..0000000000000 --- a/server/src/domain/metadata/geocoding.repository.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { InitOptions } from 'local-reverse-geocoder'; - -export const IGeocodingRepository = 'IGeocodingRepository'; - -export interface GeoPoint { - latitude: number; - longitude: number; -} - -export interface ReverseGeocodeResult { - country: string | null; - state: string | null; - city: string | null; -} - -export interface IGeocodingRepository { - init(options: Partial): Promise; - reverseGeocode(point: GeoPoint): Promise; - deleteCache(): Promise; -} diff --git a/server/src/domain/metadata/index.ts b/server/src/domain/metadata/index.ts index aa56749dac44f..860bd17f2cb5d 100644 --- a/server/src/domain/metadata/index.ts +++ b/server/src/domain/metadata/index.ts @@ -1,2 +1,2 @@ -export * from './geocoding.repository'; +export * from './metadata.repository'; export * from './metadata.service'; diff --git a/server/src/domain/metadata/metadata.repository.ts b/server/src/domain/metadata/metadata.repository.ts new file mode 100644 index 0000000000000..bd82e8b64e932 --- /dev/null +++ b/server/src/domain/metadata/metadata.repository.ts @@ -0,0 +1,31 @@ +import { Tags } from 'exiftool-vendored'; +import { InitOptions } from 'local-reverse-geocoder'; + +export const IMetadataRepository = 'IMetadataRepository'; + +export interface GeoPoint { + latitude: number; + longitude: number; +} + +export interface ReverseGeocodeResult { + country: string | null; + state: string | null; + city: string | null; +} + +export interface ImmichTags extends Tags { + ContentIdentifier?: string; + MotionPhoto?: number; + MotionPhotoVersion?: number; + MotionPhotoPresentationTimestampUs?: number; + MediaGroupUUID?: string; + ImagePixelDepth?: string; +} + +export interface IMetadataRepository { + init(options: Partial): Promise; + reverseGeocode(point: GeoPoint): Promise; + deleteCache(): Promise; + getExifTags(path: string): Promise; +} diff --git a/server/src/domain/metadata/metadata.service.spec.ts b/server/src/domain/metadata/metadata.service.spec.ts index 0266e33473ce7..2b6c633a10c8b 100644 --- a/server/src/domain/metadata/metadata.service.spec.ts +++ b/server/src/domain/metadata/metadata.service.spec.ts @@ -1,22 +1,43 @@ -import { assetStub, newAssetRepositoryMock, newJobRepositoryMock, newStorageRepositoryMock } from '@test'; +import { + assetStub, + newAlbumRepositoryMock, + newAssetRepositoryMock, + newCryptoRepositoryMock, + newJobRepositoryMock, + newMetadataRepositoryMock, + newStorageRepositoryMock, + newSystemConfigRepositoryMock, +} from '@test'; import { constants } from 'fs/promises'; +import { IAlbumRepository } from '../album'; import { IAssetRepository, WithProperty, WithoutProperty } from '../asset'; +import { ICryptoRepository } from '../crypto'; import { IJobRepository, JobName } from '../job'; import { IStorageRepository } from '../storage'; +import { ISystemConfigRepository } from '../system-config'; +import { IMetadataRepository } from './metadata.repository'; import { MetadataService } from './metadata.service'; describe(MetadataService.name, () => { - let sut: MetadataService; + let albumMock: jest.Mocked; let assetMock: jest.Mocked; + let configMock: jest.Mocked; + let cryptoRepository: jest.Mocked; let jobMock: jest.Mocked; + let metadataMock: jest.Mocked; let storageMock: jest.Mocked; + let sut: MetadataService; beforeEach(async () => { + albumMock = newAlbumRepositoryMock(); assetMock = newAssetRepositoryMock(); + configMock = newSystemConfigRepositoryMock(); + cryptoRepository = newCryptoRepositoryMock(); jobMock = newJobRepositoryMock(); + metadataMock = newMetadataRepositoryMock(); storageMock = newStorageRepositoryMock(); - sut = new MetadataService(assetMock, jobMock, storageMock); + sut = new MetadataService(albumMock, assetMock, cryptoRepository, jobMock, metadataMock, storageMock, configMock); }); it('should be defined', () => { diff --git a/server/src/domain/metadata/metadata.service.ts b/server/src/domain/metadata/metadata.service.ts index 1570f514134f7..0e136de656909 100644 --- a/server/src/domain/metadata/metadata.service.ts +++ b/server/src/domain/metadata/metadata.service.ts @@ -1,16 +1,148 @@ -import { Inject } from '@nestjs/common'; +import { AssetEntity, AssetType, ExifEntity } from '@app/infra/entities'; +import { Inject, Injectable, Logger } from '@nestjs/common'; +import { ExifDateTime } from 'exiftool-vendored'; +import { firstDateTime } from 'exiftool-vendored/dist/FirstDateTime'; import { constants } from 'fs/promises'; -import { IAssetRepository, WithoutProperty, WithProperty } from '../asset'; +import { Duration } from 'luxon'; +import { IAlbumRepository } from '../album'; +import { IAssetRepository, WithProperty, WithoutProperty } from '../asset'; +import { ICryptoRepository } from '../crypto'; import { usePagination } from '../domain.util'; -import { IBaseJob, IEntityJob, IJobRepository, JobName, JOBS_ASSET_PAGINATION_SIZE } from '../job'; -import { IStorageRepository } from '../storage'; +import { IBaseJob, IEntityJob, IJobRepository, JOBS_ASSET_PAGINATION_SIZE, JobName, QueueName } from '../job'; +import { IStorageRepository, StorageCore, StorageFolder } from '../storage'; +import { FeatureFlag, ISystemConfigRepository, SystemConfigCore } from '../system-config'; +import { IMetadataRepository, ImmichTags } from './metadata.repository'; +interface DirectoryItem { + Length?: number; + Mime: string; + Padding?: number; + Semantic?: string; +} + +interface DirectoryEntry { + Item: DirectoryItem; +} + +const exifDate = (dt: ExifDateTime | string | undefined) => (dt instanceof ExifDateTime ? dt?.toDate() : null); +// exiftool returns strings when it fails to parse non-string values, so this is used where a string is not expected +const validate = (value: T): T | null => (typeof value === 'string' ? null : value ?? null); + +@Injectable() export class MetadataService { + private logger = new Logger(MetadataService.name); + private storageCore: StorageCore; + private configCore: SystemConfigCore; + private oldCities?: string; + constructor( + @Inject(IAlbumRepository) private albumRepository: IAlbumRepository, @Inject(IAssetRepository) private assetRepository: IAssetRepository, + @Inject(ICryptoRepository) private cryptoRepository: ICryptoRepository, @Inject(IJobRepository) private jobRepository: IJobRepository, + @Inject(IMetadataRepository) private repository: IMetadataRepository, @Inject(IStorageRepository) private storageRepository: IStorageRepository, - ) {} + @Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository, + ) { + this.storageCore = new StorageCore(storageRepository); + this.configCore = new SystemConfigCore(configRepository); + this.configCore.config$.subscribe(() => this.init()); + } + + async init(deleteCache = false) { + const { reverseGeocoding } = await this.configCore.getConfig(); + const { citiesFileOverride } = reverseGeocoding; + + if (!reverseGeocoding.enabled) { + return; + } + + try { + if (deleteCache) { + await this.repository.deleteCache(); + } else if (this.oldCities && this.oldCities === citiesFileOverride) { + return; + } + + await this.jobRepository.pause(QueueName.METADATA_EXTRACTION); + await this.repository.init({ citiesFileOverride }); + await this.jobRepository.resume(QueueName.METADATA_EXTRACTION); + + this.logger.log(`Initialized local reverse geocoder with ${citiesFileOverride}`); + this.oldCities = citiesFileOverride; + } catch (error: Error | any) { + this.logger.error(`Unable to initialize reverse geocoding: ${error}`, error?.stack); + } + } + + async handleLivePhotoLinking(job: IEntityJob) { + const { id } = job; + const [asset] = await this.assetRepository.getByIds([id]); + if (!asset?.exifInfo) { + return false; + } + + if (!asset.exifInfo.livePhotoCID) { + return true; + } + + const otherType = asset.type === AssetType.VIDEO ? AssetType.IMAGE : AssetType.VIDEO; + const match = await this.assetRepository.findLivePhotoMatch({ + livePhotoCID: asset.exifInfo.livePhotoCID, + ownerId: asset.ownerId, + otherAssetId: asset.id, + type: otherType, + }); + + if (!match) { + return true; + } + + const [photoAsset, motionAsset] = asset.type === AssetType.IMAGE ? [asset, match] : [match, asset]; + + await this.assetRepository.save({ id: photoAsset.id, livePhotoVideoId: motionAsset.id }); + await this.assetRepository.save({ id: motionAsset.id, isVisible: false }); + await this.albumRepository.removeAsset(motionAsset.id); + + return true; + } + + async handleQueueMetadataExtraction(job: IBaseJob) { + const { force } = job; + const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => { + return force + ? this.assetRepository.getAll(pagination) + : this.assetRepository.getWithout(pagination, WithoutProperty.EXIF); + }); + + for await (const assets of assetPagination) { + for (const asset of assets) { + await this.jobRepository.queue({ name: JobName.METADATA_EXTRACTION, data: { id: asset.id } }); + } + } + + return true; + } + + async handleMetadataExtraction({ id }: IEntityJob) { + const [asset] = await this.assetRepository.getByIds([id]); + if (!asset || !asset.isVisible) { + return false; + } + + const { exifData, tags } = await this.exifData(asset); + + await this.applyMotionPhotos(asset, tags); + await this.applyReverseGeocoding(asset, exifData); + await this.assetRepository.upsertExif(exifData); + await this.assetRepository.save({ + id: asset.id, + duration: tags.Duration ? this.getDuration(tags.Duration) : null, + fileCreatedAt: exifData.dateTimeOriginal ?? undefined, + }); + + return true; + } async handleQueueSidecar(job: IBaseJob) { const { force } = job; @@ -51,4 +183,156 @@ export class MetadataService { return true; } + + private async applyReverseGeocoding(asset: AssetEntity, exifData: ExifEntity) { + const { latitude, longitude } = exifData; + if (!(await this.configCore.hasFeature(FeatureFlag.REVERSE_GEOCODING)) || !longitude || !latitude) { + return; + } + + try { + const { city, state, country } = await this.repository.reverseGeocode({ latitude, longitude }); + Object.assign(exifData, { city, state, country }); + } catch (error: Error | any) { + this.logger.warn( + `Unable to run reverse geocoding due to ${error} for asset ${asset.id} at ${asset.originalPath}`, + error?.stack, + ); + } + } + + private async applyMotionPhotos(asset: AssetEntity, tags: ImmichTags) { + if (asset.type !== AssetType.IMAGE || asset.livePhotoVideoId) { + return; + } + + const rawDirectory = tags.Directory; + const isMotionPhoto = tags.MotionPhoto; + const isMicroVideo = tags.MicroVideo; + const videoOffset = tags.MicroVideoOffset; + const directory = Array.isArray(rawDirectory) ? (rawDirectory as DirectoryEntry[]) : null; + + let length = 0; + let padding = 0; + + if (isMotionPhoto && directory) { + for (const entry of directory) { + if (entry.Item.Semantic == 'MotionPhoto') { + length = entry.Item.Length ?? 0; + padding = entry.Item.Padding ?? 0; + break; + } + } + } + + if (isMicroVideo && typeof videoOffset === 'number') { + length = videoOffset; + } + + if (!length) { + return; + } + + this.logger.debug(`Starting motion photo video extraction (${asset.id})`); + + try { + const stat = await this.storageRepository.stat(asset.originalPath); + const position = stat.size - length - padding; + const video = await this.storageRepository.readFile(asset.originalPath, { + buffer: Buffer.alloc(length), + position, + length, + }); + const checksum = await this.cryptoRepository.hashSha1(video); + + let motionAsset = await this.assetRepository.getByChecksum(asset.ownerId, checksum); + if (!motionAsset) { + motionAsset = await this.assetRepository.save({ + libraryId: asset.libraryId, + type: AssetType.VIDEO, + fileCreatedAt: asset.fileCreatedAt ?? asset.createdAt, + fileModifiedAt: asset.fileModifiedAt, + checksum, + ownerId: asset.ownerId, + originalPath: this.storageCore.ensurePath(StorageFolder.ENCODED_VIDEO, asset.ownerId, `${asset.id}-MP.mp4`), + originalFileName: asset.originalFileName, + isVisible: false, + isReadOnly: true, + deviceAssetId: 'NONE', + deviceId: 'NONE', + }); + + await this.storageRepository.writeFile(asset.originalPath, video); + + await this.jobRepository.queue({ name: JobName.METADATA_EXTRACTION, data: { id: motionAsset.id } }); + } + + await this.assetRepository.save({ id: asset.id, livePhotoVideoId: motionAsset.id }); + + this.logger.debug(`Finished motion photo video extraction (${asset.id})`); + } catch (error: Error | any) { + this.logger.error(`Failed to extract live photo ${asset.originalPath}: ${error}`, error?.stack); + } + } + + private async exifData(asset: AssetEntity): Promise<{ exifData: ExifEntity; tags: ImmichTags }> { + const stats = await this.storageRepository.stat(asset.originalPath); + const mediaTags = await this.repository.getExifTags(asset.originalPath); + const sidecarTags = asset.sidecarPath ? await this.repository.getExifTags(asset.sidecarPath) : null; + const tags = { ...mediaTags, ...sidecarTags }; + + this.logger.verbose('Exif Tags', tags); + + return { + exifData: { + // altitude: tags.GPSAltitude ?? null, + assetId: asset.id, + bitsPerSample: this.getBitsPerSample(tags), + colorspace: tags.ColorSpace ?? null, + dateTimeOriginal: exifDate(firstDateTime(tags)) ?? asset.fileCreatedAt, + exifImageHeight: validate(tags.ImageHeight), + exifImageWidth: validate(tags.ImageWidth), + exposureTime: tags.ExposureTime ?? null, + fileSizeInByte: stats.size, + fNumber: validate(tags.FNumber), + focalLength: validate(tags.FocalLength), + fps: validate(tags.VideoFrameRate), + iso: validate(tags.ISO), + latitude: validate(tags.GPSLatitude), + lensModel: tags.LensModel ?? null, + livePhotoCID: (asset.type === AssetType.VIDEO ? tags.ContentIdentifier : tags.MediaGroupUUID) ?? null, + longitude: validate(tags.GPSLongitude), + make: tags.Make ?? null, + model: tags.Model ?? null, + modifyDate: exifDate(tags.ModifyDate) ?? asset.fileModifiedAt, + orientation: validate(tags.Orientation)?.toString() ?? null, + profileDescription: tags.ProfileDescription || tags.ProfileName || null, + projectionType: tags.ProjectionType ? String(tags.ProjectionType).toUpperCase() : null, + timeZone: tags.tz, + }, + tags, + }; + } + + private getBitsPerSample(tags: ImmichTags): number | null { + const bitDepthTags = [ + tags.BitsPerSample, + tags.ComponentBitDepth, + tags.ImagePixelDepth, + tags.BitDepth, + tags.ColorBitDepth, + // `numericTags` doesn't parse values like '12 12 12' + ].map((tag) => (typeof tag === 'string' ? Number.parseInt(tag) : tag)); + + let bitsPerSample = bitDepthTags.find((tag) => typeof tag === 'number' && !Number.isNaN(tag)) ?? null; + if (bitsPerSample && bitsPerSample >= 24 && bitsPerSample % 3 === 0) { + bitsPerSample /= 3; // converts per-pixel bit depth to per-channel + } + + return bitsPerSample; + } + + private getDuration(seconds?: number): string { + return Duration.fromObject({ seconds }).toFormat('hh:mm:ss.SSS'); + } } diff --git a/server/src/domain/storage/storage.repository.ts b/server/src/domain/storage/storage.repository.ts index f41136a133e53..857783b701e39 100644 --- a/server/src/domain/storage/storage.repository.ts +++ b/server/src/domain/storage/storage.repository.ts @@ -1,4 +1,5 @@ import { Stats } from 'fs'; +import { FileReadOptions } from 'fs/promises'; import { Readable } from 'stream'; import { CrawlOptionsDto } from '../library'; @@ -24,6 +25,8 @@ export const IStorageRepository = 'IStorageRepository'; export interface IStorageRepository { createZipStream(): ImmichZipStream; createReadStream(filepath: string, mimeType?: string | null): Promise; + readFile(filepath: string, options?: FileReadOptions): Promise; + writeFile(filepath: string, buffer: Buffer): Promise; unlink(filepath: string): Promise; unlinkDir(folder: string, options?: { recursive?: boolean; force?: boolean }): Promise; removeEmptyDirs(folder: string, self?: boolean): Promise; diff --git a/server/src/infra/infra.module.ts b/server/src/infra/infra.module.ts index 11595795782f9..4e2fc245b5f90 100644 --- a/server/src/infra/infra.module.ts +++ b/server/src/infra/infra.module.ts @@ -6,13 +6,12 @@ import { ICommunicationRepository, ICryptoRepository, IFaceRepository, - IGeocodingRepository, IJobRepository, IKeyRepository, ILibraryRepository, IMachineLearningRepository, IMediaRepository, - immichAppConfig, + IMetadataRepository, IPartnerRepository, IPersonRepository, ISearchRepository, @@ -23,6 +22,7 @@ import { ITagRepository, IUserRepository, IUserTokenRepository, + immichAppConfig, } from '@app/domain'; import { BullModule } from '@nestjs/bullmq'; import { Global, Module, Provider } from '@nestjs/common'; @@ -33,20 +33,20 @@ import { databaseConfig } from './database.config'; import { databaseEntities } from './entities'; import { bullConfig, bullQueues } from './infra.config'; import { + APIKeyRepository, AccessRepository, AlbumRepository, - APIKeyRepository, AssetRepository, AuditRepository, CommunicationRepository, CryptoRepository, FaceRepository, FilesystemProvider, - GeocodingRepository, JobRepository, LibraryRepository, MachineLearningRepository, MediaRepository, + MetadataRepository, PartnerRepository, PersonRepository, SharedLinkRepository, @@ -66,11 +66,11 @@ const providers: Provider[] = [ { provide: ICommunicationRepository, useClass: CommunicationRepository }, { provide: ICryptoRepository, useClass: CryptoRepository }, { provide: IFaceRepository, useClass: FaceRepository }, - { provide: IGeocodingRepository, useClass: GeocodingRepository }, { provide: IJobRepository, useClass: JobRepository }, { provide: ILibraryRepository, useClass: LibraryRepository }, { provide: IKeyRepository, useClass: APIKeyRepository }, { provide: IMachineLearningRepository, useClass: MachineLearningRepository }, + { provide: IMetadataRepository, useClass: MetadataRepository }, { provide: IPartnerRepository, useClass: PartnerRepository }, { provide: IPersonRepository, useClass: PersonRepository }, { provide: ISearchRepository, useClass: TypesenseRepository }, diff --git a/server/src/infra/repositories/filesystem.provider.ts b/server/src/infra/repositories/filesystem.provider.ts index c4d9c5b953e45..d734286044a64 100644 --- a/server/src/infra/repositories/filesystem.provider.ts +++ b/server/src/infra/repositories/filesystem.provider.ts @@ -8,7 +8,7 @@ import { } from '@app/domain'; import archiver from 'archiver'; import { constants, createReadStream, existsSync, mkdirSync } from 'fs'; -import fs, { readdir } from 'fs/promises'; +import fs, { readdir, writeFile } from 'fs/promises'; import { glob } from 'glob'; import mv from 'mv'; import { promisify } from 'node:util'; @@ -39,6 +39,18 @@ export class FilesystemProvider implements IStorageRepository { }; } + async readFile(filepath: string, options?: fs.FileReadOptions): Promise { + const file = await fs.open(filepath); + try { + const { buffer } = await file.read(options); + return buffer; + } finally { + await file.close(); + } + } + + writeFile = writeFile; + async moveFile(source: string, destination: string): Promise { if (await this.checkFileExists(destination)) { throw new Error(`Destination file already exists: ${destination}`); diff --git a/server/src/infra/repositories/index.ts b/server/src/infra/repositories/index.ts index f1bb6b59f5bf9..252b7d0a2cf6f 100644 --- a/server/src/infra/repositories/index.ts +++ b/server/src/infra/repositories/index.ts @@ -7,11 +7,11 @@ export * from './communication.repository'; export * from './crypto.repository'; export * from './face.repository'; export * from './filesystem.provider'; -export * from './geocoding.repository'; export * from './job.repository'; export * from './library.repository'; export * from './machine-learning.repository'; export * from './media.repository'; +export * from './metadata.repository'; export * from './partner.repository'; export * from './person.repository'; export * from './shared-link.repository'; diff --git a/server/src/infra/repositories/geocoding.repository.ts b/server/src/infra/repositories/metadata.repository.ts similarity index 71% rename from server/src/infra/repositories/geocoding.repository.ts rename to server/src/infra/repositories/metadata.repository.ts index eae72a904f83e..60c9e6bdb17ca 100644 --- a/server/src/infra/repositories/geocoding.repository.ts +++ b/server/src/infra/repositories/metadata.repository.ts @@ -1,7 +1,9 @@ -import { GeoPoint, IGeocodingRepository, ReverseGeocodeResult } from '@app/domain'; +import { GeoPoint, IMetadataRepository, ImmichTags, ReverseGeocodeResult } from '@app/domain'; import { REVERSE_GEOCODING_DUMP_DIRECTORY } from '@app/infra'; import { Injectable, Logger } from '@nestjs/common'; +import { DefaultReadTaskOptions, exiftool } from 'exiftool-vendored'; import { readdir, rm } from 'fs/promises'; +import * as geotz from 'geo-tz'; import { getName } from 'i18n-iso-countries'; import geocoder, { AddressObject, InitOptions } from 'local-reverse-geocoder'; import path from 'path'; @@ -21,8 +23,8 @@ export type GeoData = AddressObject & { const lookup = promisify(geocoder.lookUp).bind(geocoder); @Injectable() -export class GeocodingRepository implements IGeocodingRepository { - private logger = new Logger(GeocodingRepository.name); +export class MetadataRepository implements IMetadataRepository { + private logger = new Logger(MetadataRepository.name); async init(options: Partial): Promise { return new Promise((resolve) => { @@ -69,4 +71,22 @@ export class GeocodingRepository implements IGeocodingRepository { return { country, state, city }; } + + getExifTags(path: string): Promise { + return exiftool + .read(path, undefined, { + ...DefaultReadTaskOptions, + + defaultVideosToUTC: true, + backfillTimezones: true, + inferTimezoneFromDatestamps: true, + useMWG: true, + numericTags: DefaultReadTaskOptions.numericTags.concat(['FocalLength']), + geoTz: (lat, lon) => geotz.find(lat, lon)[0], + }) + .catch((error) => { + this.logger.warn(`Error reading exif data (${path}): ${error}`, error?.stack); + return null; + }); + } } diff --git a/server/src/microservices/app.service.ts b/server/src/microservices/app.service.ts index 8346d38794e35..f434d3720b1d9 100644 --- a/server/src/microservices/app.service.ts +++ b/server/src/microservices/app.service.ts @@ -17,16 +17,12 @@ import { } from '@app/domain'; import { Injectable, Logger } from '@nestjs/common'; -import { MetadataExtractionProcessor } from './processors/metadata-extraction.processor'; @Injectable() export class AppService { private logger = new Logger(AppService.name); constructor( - // TODO refactor to domain - private metadataProcessor: MetadataExtractionProcessor, - private facialRecognitionService: FacialRecognitionService, private jobService: JobService, private mediaService: MediaService, @@ -73,9 +69,9 @@ export class AppService { [JobName.GENERATE_THUMBHASH_THUMBNAIL]: (data) => this.mediaService.handleGenerateThumbhashThumbnail(data), [JobName.QUEUE_VIDEO_CONVERSION]: (data) => this.mediaService.handleQueueVideoConversion(data), [JobName.VIDEO_CONVERSION]: (data) => this.mediaService.handleVideoConversion(data), - [JobName.QUEUE_METADATA_EXTRACTION]: (data) => this.metadataProcessor.handleQueueMetadataExtraction(data), - [JobName.METADATA_EXTRACTION]: (data) => this.metadataProcessor.handleMetadataExtraction(data), - [JobName.LINK_LIVE_PHOTOS]: (data) => this.metadataProcessor.handleLivePhotoLinking(data), + [JobName.QUEUE_METADATA_EXTRACTION]: (data) => this.metadataService.handleQueueMetadataExtraction(data), + [JobName.METADATA_EXTRACTION]: (data) => this.metadataService.handleMetadataExtraction(data), + [JobName.LINK_LIVE_PHOTOS]: (data) => this.metadataService.handleLivePhotoLinking(data), [JobName.QUEUE_RECOGNIZE_FACES]: (data) => this.facialRecognitionService.handleQueueRecognizeFaces(data), [JobName.RECOGNIZE_FACES]: (data) => this.facialRecognitionService.handleRecognizeFaces(data), [JobName.GENERATE_PERSON_THUMBNAIL]: (data) => this.facialRecognitionService.handleGeneratePersonThumbnail(data), @@ -99,10 +95,10 @@ export class AppService { } this.logger.warn('Geocoding csv parse error, trying again without cache...'); - this.metadataProcessor.init(true); + this.metadataService.init(true); }); - await this.metadataProcessor.init(); + await this.metadataService.init(); await this.searchService.init(); } } diff --git a/server/src/microservices/microservices.module.ts b/server/src/microservices/microservices.module.ts index 28abb657195d5..bcbf48d9af4a5 100644 --- a/server/src/microservices/microservices.module.ts +++ b/server/src/microservices/microservices.module.ts @@ -2,10 +2,9 @@ import { DomainModule } from '@app/domain'; import { InfraModule } from '@app/infra'; import { Module } from '@nestjs/common'; import { AppService } from './app.service'; -import { MetadataExtractionProcessor } from './processors/metadata-extraction.processor'; @Module({ imports: [DomainModule.register({ imports: [InfraModule] })], - providers: [MetadataExtractionProcessor, AppService], + providers: [AppService], }) export class MicroservicesModule {} diff --git a/server/src/microservices/processors/metadata-extraction.processor.ts b/server/src/microservices/processors/metadata-extraction.processor.ts deleted file mode 100644 index 10d1d27f27793..0000000000000 --- a/server/src/microservices/processors/metadata-extraction.processor.ts +++ /dev/null @@ -1,345 +0,0 @@ -import { - FeatureFlag, - IAlbumRepository, - IAssetRepository, - IBaseJob, - ICryptoRepository, - IEntityJob, - IGeocodingRepository, - IJobRepository, - IStorageRepository, - ISystemConfigRepository, - JobName, - JOBS_ASSET_PAGINATION_SIZE, - QueueName, - StorageCore, - StorageFolder, - SystemConfigCore, - usePagination, - WithoutProperty, -} from '@app/domain'; -import { AssetEntity, AssetType, ExifEntity } from '@app/infra/entities'; -import { Inject, Logger } from '@nestjs/common'; -import { DefaultReadTaskOptions, ExifDateTime, exiftool, ReadTaskOptions, Tags } from 'exiftool-vendored'; -import { firstDateTime } from 'exiftool-vendored/dist/FirstDateTime'; -import * as geotz from 'geo-tz'; -import { Duration } from 'luxon'; -import fs from 'node:fs/promises'; -import path from 'node:path'; - -interface DirectoryItem { - Length?: number; - Mime: string; - Padding?: number; - Semantic?: string; -} - -interface DirectoryEntry { - Item: DirectoryItem; -} - -interface ImmichTags extends Tags { - ContentIdentifier?: string; - MotionPhoto?: number; - MotionPhotoVersion?: number; - MotionPhotoPresentationTimestampUs?: number; - MediaGroupUUID?: string; - ImagePixelDepth?: string; -} - -const exifDate = (dt: ExifDateTime | string | undefined) => (dt instanceof ExifDateTime ? dt?.toDate() : null); -// exiftool returns strings when it fails to parse non-string values, so this is used where a string is not expected -const validate = (value: T): T | null => (typeof value === 'string' ? null : value ?? null); - -export class MetadataExtractionProcessor { - private logger = new Logger(MetadataExtractionProcessor.name); - private storageCore: StorageCore; - private configCore: SystemConfigCore; - private oldCities?: string; - - constructor( - @Inject(IAssetRepository) private assetRepository: IAssetRepository, - @Inject(IAlbumRepository) private albumRepository: IAlbumRepository, - @Inject(IJobRepository) private jobRepository: IJobRepository, - @Inject(IGeocodingRepository) private geocodingRepository: IGeocodingRepository, - @Inject(ICryptoRepository) private cryptoRepository: ICryptoRepository, - @Inject(IStorageRepository) private storageRepository: IStorageRepository, - @Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository, - ) { - this.storageCore = new StorageCore(storageRepository); - this.configCore = new SystemConfigCore(configRepository); - this.configCore.config$.subscribe(() => this.init()); - } - - async init(deleteCache = false) { - const { reverseGeocoding } = await this.configCore.getConfig(); - const { citiesFileOverride } = reverseGeocoding; - - if (!reverseGeocoding.enabled) { - return; - } - - try { - if (deleteCache) { - await this.geocodingRepository.deleteCache(); - } else if (this.oldCities && this.oldCities === citiesFileOverride) { - return; - } - - await this.jobRepository.pause(QueueName.METADATA_EXTRACTION); - await this.geocodingRepository.init({ citiesFileOverride }); - await this.jobRepository.resume(QueueName.METADATA_EXTRACTION); - - this.logger.log(`Initialized local reverse geocoder with ${citiesFileOverride}`); - this.oldCities = citiesFileOverride; - } catch (error: Error | any) { - this.logger.error(`Unable to initialize reverse geocoding: ${error}`, error?.stack); - } - } - - async handleLivePhotoLinking(job: IEntityJob) { - const { id } = job; - const [asset] = await this.assetRepository.getByIds([id]); - if (!asset?.exifInfo) { - return false; - } - - if (!asset.exifInfo.livePhotoCID) { - return true; - } - - const otherType = asset.type === AssetType.VIDEO ? AssetType.IMAGE : AssetType.VIDEO; - const match = await this.assetRepository.findLivePhotoMatch({ - livePhotoCID: asset.exifInfo.livePhotoCID, - ownerId: asset.ownerId, - otherAssetId: asset.id, - type: otherType, - }); - - if (!match) { - return true; - } - - const [photoAsset, motionAsset] = asset.type === AssetType.IMAGE ? [asset, match] : [match, asset]; - - await this.assetRepository.save({ id: photoAsset.id, livePhotoVideoId: motionAsset.id }); - await this.assetRepository.save({ id: motionAsset.id, isVisible: false }); - await this.albumRepository.removeAsset(motionAsset.id); - - return true; - } - - async handleQueueMetadataExtraction(job: IBaseJob) { - const { force } = job; - const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => { - return force - ? this.assetRepository.getAll(pagination) - : this.assetRepository.getWithout(pagination, WithoutProperty.EXIF); - }); - - for await (const assets of assetPagination) { - for (const asset of assets) { - await this.jobRepository.queue({ name: JobName.METADATA_EXTRACTION, data: { id: asset.id } }); - } - } - - return true; - } - - async handleMetadataExtraction({ id }: IEntityJob) { - const [asset] = await this.assetRepository.getByIds([id]); - if (!asset || !asset.isVisible) { - return false; - } - - const [exifData, tags] = await this.exifData(asset); - - await this.applyMotionPhotos(asset, tags); - await this.applyReverseGeocoding(asset, exifData); - await this.assetRepository.upsertExif(exifData); - await this.assetRepository.save({ - id: asset.id, - duration: tags.Duration ? Duration.fromObject({ seconds: tags.Duration }).toFormat('hh:mm:ss.SSS') : null, - fileCreatedAt: exifData.dateTimeOriginal ?? undefined, - }); - - return true; - } - - private async applyReverseGeocoding(asset: AssetEntity, exifData: ExifEntity) { - const { latitude, longitude } = exifData; - if (!(await this.configCore.hasFeature(FeatureFlag.REVERSE_GEOCODING)) || !longitude || !latitude) { - return; - } - - try { - const { city, state, country } = await this.geocodingRepository.reverseGeocode({ latitude, longitude }); - Object.assign(exifData, { city, state, country }); - } catch (error: Error | any) { - this.logger.warn( - `Unable to run reverse geocoding due to ${error} for asset ${asset.id} at ${asset.originalPath}`, - error?.stack, - ); - } - } - - private async applyMotionPhotos(asset: AssetEntity, tags: ImmichTags) { - if (asset.type !== AssetType.IMAGE || asset.livePhotoVideoId) { - return; - } - - const rawDirectory = tags.Directory; - const isMotionPhoto = tags.MotionPhoto; - const isMicroVideo = tags.MicroVideo; - const videoOffset = tags.MicroVideoOffset; - const directory = Array.isArray(rawDirectory) ? (rawDirectory as DirectoryEntry[]) : null; - - let length = 0; - let padding = 0; - - if (isMotionPhoto && directory) { - for (const entry of directory) { - if (entry.Item.Semantic == 'MotionPhoto') { - length = entry.Item.Length ?? 0; - padding = entry.Item.Padding ?? 0; - break; - } - } - } - - if (isMicroVideo && typeof videoOffset === 'number') { - length = videoOffset; - } - - if (!length) { - return; - } - - this.logger.debug(`Starting motion photo video extraction (${asset.id})`); - - let file = null; - try { - const encodedFolder = this.storageCore.getFolderLocation(StorageFolder.ENCODED_VIDEO, asset.ownerId); - const encodedFile = path.join(encodedFolder, path.parse(asset.originalPath).name + '.mp4'); - this.storageRepository.mkdirSync(encodedFolder); - - file = await fs.open(asset.originalPath); - - const stat = await file.stat(); - const position = stat.size - length - padding; - const video = await file.read({ buffer: Buffer.alloc(length), position, length }); - const checksum = await this.cryptoRepository.hashSha1(video.buffer); - - let motionAsset = await this.assetRepository.getByChecksum(asset.ownerId, checksum); - if (!motionAsset) { - motionAsset = await this.assetRepository.save({ - libraryId: asset.libraryId, - type: AssetType.VIDEO, - fileCreatedAt: asset.fileCreatedAt ?? asset.createdAt, - fileModifiedAt: asset.fileModifiedAt, - checksum, - ownerId: asset.ownerId, - originalPath: encodedFile, - originalFileName: asset.originalFileName, - isVisible: false, - isReadOnly: true, - deviceAssetId: 'NONE', - deviceId: 'NONE', - }); - - await fs.writeFile(encodedFile, video.buffer); - - await this.jobRepository.queue({ name: JobName.METADATA_EXTRACTION, data: { id: motionAsset.id } }); - } - - await this.assetRepository.save({ id: asset.id, livePhotoVideoId: motionAsset.id }); - - this.logger.debug(`Finished motion photo video extraction (${asset.id})`); - } catch (error: Error | any) { - this.logger.error(`Failed to extract live photo ${asset.originalPath}: ${error}`, error?.stack); - } finally { - await file?.close(); - } - } - - private async exifData(asset: AssetEntity): Promise<[ExifEntity, ImmichTags]> { - const readTaskOptions: ReadTaskOptions = { - ...DefaultReadTaskOptions, - - defaultVideosToUTC: true, - backfillTimezones: true, - inferTimezoneFromDatestamps: true, - useMWG: true, - numericTags: DefaultReadTaskOptions.numericTags.concat(['FocalLength']), - geoTz: (lat: number, lon: number): string => geotz.find(lat, lon)[0], - }; - - const mediaTags = await exiftool - .read(asset.originalPath, undefined, readTaskOptions) - .catch((error: any) => { - this.logger.warn(`error reading exif data (${asset.id} at ${asset.originalPath}): ${error}`, error?.stack); - return null; - }); - - const sidecarTags = asset.sidecarPath - ? await exiftool.read(asset.sidecarPath, undefined, readTaskOptions).catch((error: any) => { - this.logger.warn(`error reading exif data (${asset.id} at ${asset.sidecarPath}): ${error}`, error?.stack); - return null; - }) - : null; - - const stats = await fs.stat(asset.originalPath); - - const tags = { ...mediaTags, ...sidecarTags }; - - this.logger.verbose('Exif Tags', tags); - - return [ - { - // altitude: tags.GPSAltitude ?? null, - assetId: asset.id, - bitsPerSample: this.getBitsPerSample(tags), - colorspace: tags.ColorSpace ?? null, - dateTimeOriginal: exifDate(firstDateTime(tags)) ?? asset.fileCreatedAt, - exifImageHeight: validate(tags.ImageHeight), - exifImageWidth: validate(tags.ImageWidth), - exposureTime: tags.ExposureTime ?? null, - fileSizeInByte: stats.size, - fNumber: validate(tags.FNumber), - focalLength: validate(tags.FocalLength), - fps: validate(tags.VideoFrameRate), - iso: validate(tags.ISO), - latitude: validate(tags.GPSLatitude), - lensModel: tags.LensModel ?? null, - livePhotoCID: (asset.type === AssetType.VIDEO ? tags.ContentIdentifier : tags.MediaGroupUUID) ?? null, - longitude: validate(tags.GPSLongitude), - make: tags.Make ?? null, - model: tags.Model ?? null, - modifyDate: exifDate(tags.ModifyDate) ?? asset.fileModifiedAt, - orientation: validate(tags.Orientation)?.toString() ?? null, - profileDescription: tags.ProfileDescription || tags.ProfileName || null, - projectionType: tags.ProjectionType ? String(tags.ProjectionType).toUpperCase() : null, - timeZone: tags.tz, - }, - tags, - ]; - } - - getBitsPerSample(tags: ImmichTags): number | null { - const bitDepthTags = [ - tags.BitsPerSample, - tags.ComponentBitDepth, - tags.ImagePixelDepth, - tags.BitDepth, - tags.ColorBitDepth, - // `numericTags` doesn't parse values like '12 12 12' - ].map((tag) => (typeof tag === 'string' ? Number.parseInt(tag) : tag)); - - let bitsPerSample = bitDepthTags.find((tag) => typeof tag === 'number' && !Number.isNaN(tag)) ?? null; - if (bitsPerSample && bitsPerSample >= 24 && bitsPerSample % 3 === 0) { - bitsPerSample /= 3; // converts per-pixel bit depth to per-channel - } - - return bitsPerSample; - } -} diff --git a/server/test/repositories/index.ts b/server/test/repositories/index.ts index d385be9ab949e..064059a44d115 100644 --- a/server/test/repositories/index.ts +++ b/server/test/repositories/index.ts @@ -10,6 +10,7 @@ export * from './job.repository.mock'; export * from './library.repository.mock'; export * from './machine-learning.repository.mock'; export * from './media.repository.mock'; +export * from './metadata.repository.mock'; export * from './partner.repository.mock'; export * from './person.repository.mock'; export * from './search.repository.mock'; diff --git a/server/test/repositories/metadata.repository.mock.ts b/server/test/repositories/metadata.repository.mock.ts new file mode 100644 index 0000000000000..13589f15b8b97 --- /dev/null +++ b/server/test/repositories/metadata.repository.mock.ts @@ -0,0 +1,10 @@ +import { IMetadataRepository } from '@app/domain'; + +export const newMetadataRepositoryMock = (): jest.Mocked => { + return { + deleteCache: jest.fn(), + getExifTags: jest.fn(), + init: jest.fn(), + reverseGeocode: jest.fn(), + }; +}; diff --git a/server/test/repositories/storage.repository.mock.ts b/server/test/repositories/storage.repository.mock.ts index 66ff1ffbc33e2..f0c49f6922594 100644 --- a/server/test/repositories/storage.repository.mock.ts +++ b/server/test/repositories/storage.repository.mock.ts @@ -4,6 +4,8 @@ export const newStorageRepositoryMock = (): jest.Mocked => { return { createZipStream: jest.fn(), createReadStream: jest.fn(), + readFile: jest.fn(), + writeFile: jest.fn(), unlink: jest.fn(), unlinkDir: jest.fn().mockResolvedValue(true), removeEmptyDirs: jest.fn(),