From 4762fd83d414cc239124a9d2902c18be99a1d2f5 Mon Sep 17 00:00:00 2001 From: Jason Rasmussen Date: Tue, 15 Aug 2023 21:34:57 -0400 Subject: [PATCH] fix(server): link live photos after metadata extraction finishes (#3702) * fix(server): link live photos after metadata extraction finishes * chore: fix test --------- Co-authored-by: Alex --- server/src/domain/album/album.repository.ts | 2 + server/src/domain/job/job.constants.ts | 2 + server/src/domain/job/job.repository.ts | 1 + server/src/domain/job/job.service.spec.ts | 4 ++ server/src/domain/job/job.service.ts | 6 +- .../infra/repositories/album.repository.ts | 17 +++++- server/src/microservices/app.service.ts | 1 + .../metadata-extraction.processor.ts | 60 +++++++++++-------- .../repositories/album.repository.mock.ts | 1 + 9 files changed, 64 insertions(+), 30 deletions(-) diff --git a/server/src/domain/album/album.repository.ts b/server/src/domain/album/album.repository.ts index bc6fa37524e55..d501964ef5900 100644 --- a/server/src/domain/album/album.repository.ts +++ b/server/src/domain/album/album.repository.ts @@ -16,6 +16,8 @@ export interface IAlbumRepository { getByIds(ids: string[]): Promise; getByAssetId(ownerId: string, assetId: string): Promise; hasAsset(id: string, assetId: string): Promise; + /** Remove an asset from _all_ albums */ + removeAsset(id: string): Promise; getAssetCountForIds(ids: string[]): Promise; getInvalidThumbnail(): Promise; getOwned(ownerId: string): Promise; diff --git a/server/src/domain/job/job.constants.ts b/server/src/domain/job/job.constants.ts index a02248b306271..02fa588c9b0e7 100644 --- a/server/src/domain/job/job.constants.ts +++ b/server/src/domain/job/job.constants.ts @@ -32,6 +32,7 @@ export enum JobName { // metadata QUEUE_METADATA_EXTRACTION = 'queue-metadata-extraction', METADATA_EXTRACTION = 'metadata-extraction', + LINK_LIVE_PHOTOS = 'link-live-photos', // user deletion USER_DELETION = 'user-deletion', @@ -98,6 +99,7 @@ export const JOBS_TO_QUEUE: Record = { // metadata [JobName.QUEUE_METADATA_EXTRACTION]: QueueName.METADATA_EXTRACTION, [JobName.METADATA_EXTRACTION]: QueueName.METADATA_EXTRACTION, + [JobName.LINK_LIVE_PHOTOS]: QueueName.METADATA_EXTRACTION, // storage template [JobName.STORAGE_TEMPLATE_MIGRATION]: QueueName.STORAGE_TEMPLATE_MIGRATION, diff --git a/server/src/domain/job/job.repository.ts b/server/src/domain/job/job.repository.ts index c088a2eedf867..f605bef4b4d97 100644 --- a/server/src/domain/job/job.repository.ts +++ b/server/src/domain/job/job.repository.ts @@ -45,6 +45,7 @@ export type JobItem = // Metadata Extraction | { name: JobName.QUEUE_METADATA_EXTRACTION; data: IBaseJob } | { name: JobName.METADATA_EXTRACTION; data: IEntityJob } + | { name: JobName.LINK_LIVE_PHOTOS; data: IEntityJob } // Sidecar Scanning | { name: JobName.QUEUE_SIDECAR; data: IBaseJob } diff --git a/server/src/domain/job/job.service.spec.ts b/server/src/domain/job/job.service.spec.ts index a8c6ec9dc6dbe..503440a5cf7f8 100644 --- a/server/src/domain/job/job.service.spec.ts +++ b/server/src/domain/job/job.service.spec.ts @@ -252,6 +252,10 @@ describe(JobService.name, () => { }, { item: { name: JobName.METADATA_EXTRACTION, data: { id: 'asset-1' } }, + jobs: [JobName.LINK_LIVE_PHOTOS], + }, + { + item: { name: JobName.LINK_LIVE_PHOTOS, data: { id: 'asset-1' } }, jobs: [JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE, JobName.SEARCH_INDEX_ASSET], }, { diff --git a/server/src/domain/job/job.service.ts b/server/src/domain/job/job.service.ts index df7949136767d..1c82908911cb4 100644 --- a/server/src/domain/job/job.service.ts +++ b/server/src/domain/job/job.service.ts @@ -149,6 +149,10 @@ export class JobService { break; case JobName.METADATA_EXTRACTION: + await this.jobRepository.queue({ name: JobName.LINK_LIVE_PHOTOS, data: item.data }); + break; + + case JobName.LINK_LIVE_PHOTOS: await this.jobRepository.queue({ name: JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE, data: item.data }); break; @@ -186,7 +190,7 @@ export class JobService { case JobName.CLASSIFY_IMAGE: case JobName.ENCODE_CLIP: case JobName.RECOGNIZE_FACES: - case JobName.METADATA_EXTRACTION: + case JobName.LINK_LIVE_PHOTOS: await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ASSET, data: { ids: [item.data.id] } }); break; } diff --git a/server/src/infra/repositories/album.repository.ts b/server/src/infra/repositories/album.repository.ts index 5e4ac06ca0828..c59589710b142 100644 --- a/server/src/infra/repositories/album.repository.ts +++ b/server/src/infra/repositories/album.repository.ts @@ -1,7 +1,7 @@ import { AlbumAssetCount, AlbumInfoOptions, IAlbumRepository } from '@app/domain'; import { Injectable } from '@nestjs/common'; -import { InjectRepository } from '@nestjs/typeorm'; -import { FindOptionsOrder, FindOptionsRelations, In, IsNull, Not, Repository } from 'typeorm'; +import { InjectDataSource, InjectRepository } from '@nestjs/typeorm'; +import { DataSource, FindOptionsOrder, FindOptionsRelations, In, IsNull, Not, Repository } from 'typeorm'; import { dataSource } from '../database.config'; import { AlbumEntity, AssetEntity } from '../entities'; @@ -10,6 +10,7 @@ export class AlbumRepository implements IAlbumRepository { constructor( @InjectRepository(AssetEntity) private assetRepository: Repository, @InjectRepository(AlbumEntity) private repository: Repository, + @InjectDataSource() private dataSource: DataSource, ) {} getById(id: string, options: AlbumInfoOptions): Promise { @@ -84,7 +85,7 @@ export class AlbumRepository implements IAlbumRepository { */ async getInvalidThumbnail(): Promise { // Using dataSource, because there is no direct access to albums_assets_assets. - const albumHasAssets = dataSource + const albumHasAssets = this.dataSource .createQueryBuilder() .select('1') .from('albums_assets_assets', 'albums_assets') @@ -150,6 +151,16 @@ export class AlbumRepository implements IAlbumRepository { }); } + async removeAsset(assetId: string): Promise { + // Using dataSource, because there is no direct access to albums_assets_assets. + await this.dataSource + .createQueryBuilder() + .delete() + .from('albums_assets_assets') + .where('"albums_assets_assets"."assetsId" = :assetId', { assetId }) + .execute(); + } + hasAsset(id: string, assetId: string): Promise { return this.repository.exist({ where: { diff --git a/server/src/microservices/app.service.ts b/server/src/microservices/app.service.ts index a8f30e1888280..1204a6ebdd553 100644 --- a/server/src/microservices/app.service.ts +++ b/server/src/microservices/app.service.ts @@ -66,6 +66,7 @@ export class AppService { [JobName.VIDEO_CONVERSION]: (data) => this.mediaService.handleVideoConversion(data), [JobName.QUEUE_METADATA_EXTRACTION]: (data) => this.metadataProcessor.handleQueueMetadataExtraction(data), [JobName.METADATA_EXTRACTION]: (data) => this.metadataProcessor.handleMetadataExtraction(data), + [JobName.LINK_LIVE_PHOTOS]: (data) => this.metadataProcessor.handleLivePhotoLinking(data), [JobName.QUEUE_RECOGNIZE_FACES]: (data) => this.facialRecognitionService.handleQueueRecognizeFaces(data), [JobName.RECOGNIZE_FACES]: (data) => this.facialRecognitionService.handleRecognizeFaces(data), [JobName.GENERATE_FACE_THUMBNAIL]: (data) => this.facialRecognitionService.handleGenerateFaceThumbnail(data), diff --git a/server/src/microservices/processors/metadata-extraction.processor.ts b/server/src/microservices/processors/metadata-extraction.processor.ts index a5f3ed4dd3879..7c58f7102e78d 100644 --- a/server/src/microservices/processors/metadata-extraction.processor.ts +++ b/server/src/microservices/processors/metadata-extraction.processor.ts @@ -1,4 +1,5 @@ import { + IAlbumRepository, IAssetRepository, IBaseJob, ICryptoRepository, @@ -59,6 +60,7 @@ export class MetadataExtractionProcessor { constructor( @Inject(IAssetRepository) private assetRepository: IAssetRepository, + @Inject(IAlbumRepository) private albumRepository: IAlbumRepository, @Inject(IJobRepository) private jobRepository: IJobRepository, @Inject(IGeocodingRepository) private geocodingRepository: IGeocodingRepository, @Inject(ICryptoRepository) private cryptoRepository: ICryptoRepository, @@ -92,6 +94,38 @@ export class MetadataExtractionProcessor { } } + async handleLivePhotoLinking(job: IEntityJob) { + const { id } = job; + const [asset] = await this.assetRepository.getByIds([id]); + if (!asset?.exifInfo) { + return false; + } + + if (!asset.exifInfo.livePhotoCID) { + return true; + } + + const otherType = asset.type === AssetType.VIDEO ? AssetType.IMAGE : AssetType.VIDEO; + const match = await this.assetRepository.findLivePhotoMatch({ + livePhotoCID: asset.exifInfo.livePhotoCID, + ownerId: asset.ownerId, + otherAssetId: asset.id, + type: otherType, + }); + + if (!match) { + return true; + } + + const [photoAsset, motionAsset] = asset.type === AssetType.IMAGE ? [asset, match] : [match, asset]; + + await this.assetRepository.save({ id: photoAsset.id, livePhotoVideoId: motionAsset.id }); + await this.assetRepository.save({ id: motionAsset.id, isVisible: false }); + await this.albumRepository.removeAsset(motionAsset.id); + + return true; + } + async handleQueueMetadataExtraction(job: IBaseJob) { const { force } = job; const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => { @@ -351,19 +385,6 @@ export class MetadataExtractionProcessor { } newExif.livePhotoCID = getExifProperty('MediaGroupUUID'); - if (newExif.livePhotoCID && !asset.livePhotoVideoId) { - const motionAsset = await this.assetRepository.findLivePhotoMatch({ - livePhotoCID: newExif.livePhotoCID, - otherAssetId: asset.id, - ownerId: asset.ownerId, - type: AssetType.VIDEO, - }); - if (motionAsset) { - await this.assetRepository.save({ id: asset.id, livePhotoVideoId: motionAsset.id }); - await this.assetRepository.save({ id: motionAsset.id, isVisible: false }); - } - } - await this.applyReverseGeocoding(asset, newExif); /** @@ -428,19 +449,6 @@ export class MetadataExtractionProcessor { newExif.fps = null; newExif.livePhotoCID = exifData?.ContentIdentifier || null; - if (newExif.livePhotoCID) { - const photoAsset = await this.assetRepository.findLivePhotoMatch({ - livePhotoCID: newExif.livePhotoCID, - ownerId: asset.ownerId, - otherAssetId: asset.id, - type: AssetType.IMAGE, - }); - if (photoAsset) { - await this.assetRepository.save({ id: photoAsset.id, livePhotoVideoId: asset.id }); - await this.assetRepository.save({ id: asset.id, isVisible: false }); - } - } - if (videoTags && videoTags['location']) { const location = videoTags['location'] as string; const locationRegex = /([+-][0-9]+\.[0-9]+)([+-][0-9]+\.[0-9]+)\/$/; diff --git a/server/test/repositories/album.repository.mock.ts b/server/test/repositories/album.repository.mock.ts index 8656fa64a1daa..3d42630dfb0df 100644 --- a/server/test/repositories/album.repository.mock.ts +++ b/server/test/repositories/album.repository.mock.ts @@ -12,6 +12,7 @@ export const newAlbumRepositoryMock = (): jest.Mocked => { getNotShared: jest.fn(), deleteAll: jest.fn(), getAll: jest.fn(), + removeAsset: jest.fn(), hasAsset: jest.fn(), create: jest.fn(), update: jest.fn(),