From 5cd186d3d435aeadb6462430d9eef19695469821 Mon Sep 17 00:00:00 2001 From: Jason Rasmussen Date: Thu, 12 Jun 2025 14:23:02 -0400 Subject: [PATCH] refactor: duplicate queries (#19136) --- server/src/queries/asset.repository.sql | 70 --------- server/src/queries/duplicate.repository.sql | 103 ++++++++++++++ server/src/queries/search.repository.sql | 33 ----- server/src/repositories/asset.repository.ts | 108 ++------------ .../src/repositories/duplicate.repository.ts | 133 ++++++++++++++++++ server/src/repositories/index.ts | 2 + server/src/repositories/search.repository.ts | 50 +------ server/src/services/base.service.ts | 2 + server/src/services/duplicate.service.spec.ts | 26 ++-- server/src/services/duplicate.service.ts | 10 +- server/src/services/media.service.ts | 6 +- server/test/medium.factory.ts | 1 + .../repositories/asset.repository.mock.ts | 2 - server/test/utils.ts | 4 + 14 files changed, 280 insertions(+), 270 deletions(-) create mode 100644 server/src/queries/duplicate.repository.sql create mode 100644 server/src/repositories/duplicate.repository.ts diff --git a/server/src/queries/asset.repository.sql b/server/src/queries/asset.repository.sql index b6c5d4bea8..9792257652 100644 --- a/server/src/queries/asset.repository.sql +++ b/server/src/queries/asset.repository.sql @@ -185,16 +185,6 @@ set where "id" = any ($2::uuid[]) --- AssetRepository.updateDuplicates -update "assets" -set - "duplicateId" = $1 -where - ( - "duplicateId" = any ($2::uuid[]) - or "id" = any ($3::uuid[]) - ) - -- AssetRepository.getByChecksum select "assets".* @@ -349,66 +339,6 @@ select from "agg" --- AssetRepository.getDuplicates -with - "duplicates" as ( - select - "assets"."duplicateId", - json_agg( - "asset" - order by - "assets"."localDateTime" asc - ) as "assets" - from - "assets" - left join lateral ( - select - "assets".*, - "exif" as "exifInfo" - from - "exif" - where - "exif"."assetId" = "assets"."id" - ) as "asset" on true - where - "assets"."visibility" in ('archive', 'timeline') - and "assets"."ownerId" = $1::uuid - and "assets"."duplicateId" is not null - and "assets"."deletedAt" is null - and "assets"."stackId" is null - group by - "assets"."duplicateId" - ), - "unique" as ( - select - "duplicateId" - from - "duplicates" - where - json_array_length("assets") = $2 - ), - "removed_unique" as ( - update "assets" - set - "duplicateId" = $3 - from - "unique" - where - "assets"."duplicateId" = "unique"."duplicateId" - ) -select - * -from - "duplicates" -where - not exists ( - select - from - "unique" - where - "unique"."duplicateId" = "duplicates"."duplicateId" - ) - -- AssetRepository.getAssetIdByCity with "cities" as ( diff --git a/server/src/queries/duplicate.repository.sql b/server/src/queries/duplicate.repository.sql new file mode 100644 index 0000000000..f008e57963 --- /dev/null +++ b/server/src/queries/duplicate.repository.sql @@ -0,0 +1,103 @@ +-- NOTE: This file is auto generated by ./sql-generator + +-- DuplicateRepository.getAll +with + "duplicates" as ( + select + "assets"."duplicateId", + json_agg( + "asset" + order by + "assets"."localDateTime" asc + ) as "assets" + from + "assets" + left join lateral ( + select + "assets".*, + "exif" as "exifInfo" + from + "exif" + where + "exif"."assetId" = "assets"."id" + ) as "asset" on true + where + "assets"."visibility" in ('archive', 'timeline') + and "assets"."ownerId" = $1::uuid + and "assets"."duplicateId" is not null + and "assets"."deletedAt" is null + and "assets"."stackId" is null + group by + "assets"."duplicateId" + ), + "unique" as ( + select + "duplicateId" + from + "duplicates" + where + json_array_length("assets") = $2 + ), + "removed_unique" as ( + update "assets" + set + "duplicateId" = $3 + from + "unique" + where + "assets"."duplicateId" = "unique"."duplicateId" + ) +select + * +from + "duplicates" +where + not exists ( + select + from + "unique" + where + "unique"."duplicateId" = "duplicates"."duplicateId" + ) + +-- DuplicateRepository.search +begin +set + local vchordrq.probes = 1 +with + "cte" as ( + select + "assets"."id" as "assetId", + "assets"."duplicateId", + smart_search.embedding <=> $1 as "distance" + from + "assets" + inner join "smart_search" on "assets"."id" = "smart_search"."assetId" + where + "assets"."visibility" in ('archive', 'timeline') + and "assets"."ownerId" = any ($2::uuid[]) + and "assets"."deletedAt" is null + and "assets"."type" = $3 + and "assets"."id" != $4::uuid + and "assets"."stackId" is null + order by + "distance" + limit + $5 + ) +select + * +from + "cte" +where + "cte"."distance" <= $6 +commit + +-- DuplicateRepository.merge +update "assets" +set +where + ( + "duplicateId" = any ($1::uuid[]) + or "id" = any ($2::uuid[]) + ) diff --git a/server/src/queries/search.repository.sql b/server/src/queries/search.repository.sql index b68e0e177e..abbc063e8f 100644 --- a/server/src/queries/search.repository.sql +++ b/server/src/queries/search.repository.sql @@ -102,39 +102,6 @@ offset $8 commit --- SearchRepository.searchDuplicates -begin -set - local vchordrq.probes = 1 -with - "cte" as ( - select - "assets"."id" as "assetId", - "assets"."duplicateId", - smart_search.embedding <=> $1 as "distance" - from - "assets" - inner join "smart_search" on "assets"."id" = "smart_search"."assetId" - where - "assets"."visibility" in ('archive', 'timeline') - and "assets"."ownerId" = any ($2::uuid[]) - and "assets"."deletedAt" is null - and "assets"."type" = $3 - and "assets"."id" != $4::uuid - and "assets"."stackId" is null - order by - "distance" - limit - $5 - ) -select - * -from - "cte" -where - "cte"."distance" <= $6 -commit - -- SearchRepository.searchFaces begin set diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts index af5239ed70..8cb971775c 100644 --- a/server/src/repositories/asset.repository.ts +++ b/server/src/repositories/asset.repository.ts @@ -5,7 +5,6 @@ import { InjectKysely } from 'nestjs-kysely'; import { Stack } from 'src/database'; import { AssetFiles, AssetJobStatus, Assets, DB, Exif } from 'src/db'; import { Chunked, ChunkedArray, DummyValue, GenerateSql } from 'src/decorators'; -import { MapAsset } from 'src/dtos/asset-response.dto'; import { AssetFileType, AssetOrder, AssetStatus, AssetType, AssetVisibility } from 'src/enum'; import { anyUuid, @@ -29,13 +28,13 @@ import { globToSqlPattern } from 'src/utils/misc'; export type AssetStats = Record; -export interface AssetStatsOptions { +interface AssetStatsOptions { isFavorite?: boolean; isTrashed?: boolean; visibility?: AssetVisibility; } -export interface LivePhotoSearchOptions { +interface LivePhotoSearchOptions { ownerId: string; libraryId?: string | null; livePhotoCID: string; @@ -43,16 +42,12 @@ export interface LivePhotoSearchOptions { type: AssetType; } -export enum WithProperty { - SIDECAR = 'sidecar', -} - export enum TimeBucketSize { DAY = 'DAY', MONTH = 'MONTH', } -export interface AssetBuilderOptions { +interface AssetBuilderOptions { isFavorite?: boolean; isTrashed?: boolean; isDuplicate?: boolean; @@ -81,43 +76,31 @@ export interface MonthDay { month: number; } -export interface AssetExploreFieldOptions { +interface AssetExploreFieldOptions { maxFields: number; minAssetsPerField: number; } -export interface AssetFullSyncOptions { +interface AssetFullSyncOptions { ownerId: string; lastId?: string; updatedUntil: Date; limit: number; } -export interface AssetDeltaSyncOptions { +interface AssetDeltaSyncOptions { userIds: string[]; updatedAfter: Date; limit: number; } -export interface AssetUpdateDuplicateOptions { - targetDuplicateId: string | null; - assetIds: string[]; - duplicateIds: string[]; -} - -export interface UpsertFileOptions { - assetId: string; - type: AssetFileType; - path: string; -} - -export interface AssetGetByChecksumOptions { +interface AssetGetByChecksumOptions { ownerId: string; checksum: Buffer; libraryId?: string; } -export interface GetByIdsRelations { +interface GetByIdsRelations { exifInfo?: boolean; faces?: { person?: boolean; withDeleted?: boolean }; files?: boolean; @@ -128,16 +111,6 @@ export interface GetByIdsRelations { tags?: boolean; } -export interface DuplicateGroup { - duplicateId: string; - assets: MapAsset[]; -} - -export interface DayOfYearAssets { - yearsAgo: number; - assets: MapAsset[]; -} - @Injectable() export class AssetRepository { constructor(@InjectKysely() private db: Kysely) {} @@ -418,19 +391,6 @@ export class AssetRepository { await this.db.updateTable('assets').set(options).where('libraryId', '=', asUuid(libraryId)).execute(); } - @GenerateSql({ - params: [{ targetDuplicateId: DummyValue.UUID, duplicateIds: [DummyValue.UUID], assetIds: [DummyValue.UUID] }], - }) - async updateDuplicates(options: AssetUpdateDuplicateOptions): Promise { - await this.db - .updateTable('assets') - .set({ duplicateId: options.targetDuplicateId }) - .where((eb) => - eb.or([eb('duplicateId', '=', anyUuid(options.duplicateIds)), eb('id', '=', anyUuid(options.assetIds))]), - ) - .execute(); - } - async update(asset: Updateable & { id: string }) { const value = omitBy(asset, isUndefined); delete value.id; @@ -696,58 +656,6 @@ export class AssetRepository { return query.executeTakeFirstOrThrow(); } - @GenerateSql({ params: [DummyValue.UUID] }) - getDuplicates(userId: string) { - return ( - this.db - .with('duplicates', (qb) => - qb - .selectFrom('assets') - .$call(withDefaultVisibility) - .leftJoinLateral( - (qb) => - qb - .selectFrom('exif') - .selectAll('assets') - .select((eb) => eb.table('exif').as('exifInfo')) - .whereRef('exif.assetId', '=', 'assets.id') - .as('asset'), - (join) => join.onTrue(), - ) - .select('assets.duplicateId') - .select((eb) => - eb.fn.jsonAgg('asset').orderBy('assets.localDateTime', 'asc').$castTo().as('assets'), - ) - .where('assets.ownerId', '=', asUuid(userId)) - .where('assets.duplicateId', 'is not', null) - .$narrowType<{ duplicateId: NotNull }>() - .where('assets.deletedAt', 'is', null) - .where('assets.stackId', 'is', null) - .groupBy('assets.duplicateId'), - ) - .with('unique', (qb) => - qb - .selectFrom('duplicates') - .select('duplicateId') - .where((eb) => eb(eb.fn('json_array_length', ['assets']), '=', 1)), - ) - .with('removed_unique', (qb) => - qb - .updateTable('assets') - .set({ duplicateId: null }) - .from('unique') - .whereRef('assets.duplicateId', '=', 'unique.duplicateId'), - ) - .selectFrom('duplicates') - .selectAll() - // TODO: compare with filtering by json_array_length > 1 - .where(({ not, exists }) => - not(exists((eb) => eb.selectFrom('unique').whereRef('unique.duplicateId', '=', 'duplicates.duplicateId'))), - ) - .execute() - ); - } - @GenerateSql({ params: [DummyValue.UUID, { minAssetsPerField: 5, maxFields: 12 }] }) async getAssetIdByCity(ownerId: string, { minAssetsPerField, maxFields }: AssetExploreFieldOptions) { const items = await this.db diff --git a/server/src/repositories/duplicate.repository.ts b/server/src/repositories/duplicate.repository.ts new file mode 100644 index 0000000000..b3329e4ca7 --- /dev/null +++ b/server/src/repositories/duplicate.repository.ts @@ -0,0 +1,133 @@ +import { Injectable } from '@nestjs/common'; +import { Kysely, NotNull, sql } from 'kysely'; +import { InjectKysely } from 'nestjs-kysely'; +import { DB } from 'src/db'; +import { DummyValue, GenerateSql } from 'src/decorators'; +import { MapAsset } from 'src/dtos/asset-response.dto'; +import { AssetType, VectorIndex } from 'src/enum'; +import { probes } from 'src/repositories/database.repository'; +import { anyUuid, asUuid, withDefaultVisibility } from 'src/utils/database'; + +interface DuplicateSearch { + assetId: string; + embedding: string; + maxDistance: number; + type: AssetType; + userIds: string[]; +} + +interface DuplicateMergeOptions { + targetId: string | null; + assetIds: string[]; + sourceIds: string[]; +} + +@Injectable() +export class DuplicateRepository { + constructor(@InjectKysely() private db: Kysely) {} + + @GenerateSql({ params: [DummyValue.UUID] }) + getAll(userId: string) { + return ( + this.db + .with('duplicates', (qb) => + qb + .selectFrom('assets') + .$call(withDefaultVisibility) + .leftJoinLateral( + (qb) => + qb + .selectFrom('exif') + .selectAll('assets') + .select((eb) => eb.table('exif').as('exifInfo')) + .whereRef('exif.assetId', '=', 'assets.id') + .as('asset'), + (join) => join.onTrue(), + ) + .select('assets.duplicateId') + .select((eb) => + eb.fn.jsonAgg('asset').orderBy('assets.localDateTime', 'asc').$castTo().as('assets'), + ) + .where('assets.ownerId', '=', asUuid(userId)) + .where('assets.duplicateId', 'is not', null) + .$narrowType<{ duplicateId: NotNull }>() + .where('assets.deletedAt', 'is', null) + .where('assets.stackId', 'is', null) + .groupBy('assets.duplicateId'), + ) + .with('unique', (qb) => + qb + .selectFrom('duplicates') + .select('duplicateId') + .where((eb) => eb(eb.fn('json_array_length', ['assets']), '=', 1)), + ) + .with('removed_unique', (qb) => + qb + .updateTable('assets') + .set({ duplicateId: null }) + .from('unique') + .whereRef('assets.duplicateId', '=', 'unique.duplicateId'), + ) + .selectFrom('duplicates') + .selectAll() + // TODO: compare with filtering by json_array_length > 1 + .where(({ not, exists }) => + not(exists((eb) => eb.selectFrom('unique').whereRef('unique.duplicateId', '=', 'duplicates.duplicateId'))), + ) + .execute() + ); + } + + @GenerateSql({ + params: [ + { + assetId: DummyValue.UUID, + embedding: DummyValue.VECTOR, + maxDistance: 0.6, + type: AssetType.IMAGE, + userIds: [DummyValue.UUID], + }, + ], + }) + search({ assetId, embedding, maxDistance, type, userIds }: DuplicateSearch) { + return this.db.transaction().execute(async (trx) => { + await sql`set local vchordrq.probes = ${sql.lit(probes[VectorIndex.CLIP])}`.execute(trx); + return await trx + .with('cte', (qb) => + qb + .selectFrom('assets') + .$call(withDefaultVisibility) + .select([ + 'assets.id as assetId', + 'assets.duplicateId', + sql`smart_search.embedding <=> ${embedding}`.as('distance'), + ]) + .innerJoin('smart_search', 'assets.id', 'smart_search.assetId') + .where('assets.ownerId', '=', anyUuid(userIds)) + .where('assets.deletedAt', 'is', null) + .where('assets.type', '=', type) + .where('assets.id', '!=', asUuid(assetId)) + .where('assets.stackId', 'is', null) + .orderBy('distance') + .limit(64), + ) + .selectFrom('cte') + .selectAll() + .where('cte.distance', '<=', maxDistance as number) + .execute(); + }); + } + + @GenerateSql({ + params: [{ targetDuplicateId: DummyValue.UUID, duplicateIds: [DummyValue.UUID], assetIds: [DummyValue.UUID] }], + }) + async merge(options: DuplicateMergeOptions): Promise { + await this.db + .updateTable('assets') + .set({ duplicateId: options.targetId }) + .where((eb) => + eb.or([eb('duplicateId', '=', anyUuid(options.sourceIds)), eb('id', '=', anyUuid(options.assetIds))]), + ) + .execute(); + } +} diff --git a/server/src/repositories/index.ts b/server/src/repositories/index.ts index 453e515fe0..879385f9fd 100644 --- a/server/src/repositories/index.ts +++ b/server/src/repositories/index.ts @@ -11,6 +11,7 @@ import { CronRepository } from 'src/repositories/cron.repository'; import { CryptoRepository } from 'src/repositories/crypto.repository'; import { DatabaseRepository } from 'src/repositories/database.repository'; import { DownloadRepository } from 'src/repositories/download.repository'; +import { DuplicateRepository } from 'src/repositories/duplicate.repository'; import { EmailRepository } from 'src/repositories/email.repository'; import { EventRepository } from 'src/repositories/event.repository'; import { JobRepository } from 'src/repositories/job.repository'; @@ -56,6 +57,7 @@ export const repositories = [ CryptoRepository, DatabaseRepository, DownloadRepository, + DuplicateRepository, EmailRepository, EventRepository, JobRepository, diff --git a/server/src/repositories/search.repository.ts b/server/src/repositories/search.repository.ts index 068de1eb4d..2455b65b20 100644 --- a/server/src/repositories/search.repository.ts +++ b/server/src/repositories/search.repository.ts @@ -7,7 +7,7 @@ import { DummyValue, GenerateSql } from 'src/decorators'; import { MapAsset } from 'src/dtos/asset-response.dto'; import { AssetStatus, AssetType, AssetVisibility, VectorIndex } from 'src/enum'; import { probes } from 'src/repositories/database.repository'; -import { anyUuid, asUuid, searchAssetBuilder, withDefaultVisibility } from 'src/utils/database'; +import { anyUuid, searchAssetBuilder } from 'src/utils/database'; import { paginationHelper } from 'src/utils/pagination'; import { isValidInteger } from 'src/validation'; @@ -135,14 +135,6 @@ export interface FaceEmbeddingSearch extends SearchEmbeddingOptions { minBirthDate?: Date | null; } -export interface AssetDuplicateSearch { - assetId: string; - embedding: string; - maxDistance: number; - type: AssetType; - userIds: string[]; -} - export interface FaceSearchResult { distance: number; id: string; @@ -275,46 +267,6 @@ export class SearchRepository { }); } - @GenerateSql({ - params: [ - { - assetId: DummyValue.UUID, - embedding: DummyValue.VECTOR, - maxDistance: 0.6, - type: AssetType.IMAGE, - userIds: [DummyValue.UUID], - }, - ], - }) - searchDuplicates({ assetId, embedding, maxDistance, type, userIds }: AssetDuplicateSearch) { - return this.db.transaction().execute(async (trx) => { - await sql`set local vchordrq.probes = ${sql.lit(probes[VectorIndex.CLIP])}`.execute(trx); - return await trx - .with('cte', (qb) => - qb - .selectFrom('assets') - .$call(withDefaultVisibility) - .select([ - 'assets.id as assetId', - 'assets.duplicateId', - sql`smart_search.embedding <=> ${embedding}`.as('distance'), - ]) - .innerJoin('smart_search', 'assets.id', 'smart_search.assetId') - .where('assets.ownerId', '=', anyUuid(userIds)) - .where('assets.deletedAt', 'is', null) - .where('assets.type', '=', type) - .where('assets.id', '!=', asUuid(assetId)) - .where('assets.stackId', 'is', null) - .orderBy('distance') - .limit(64), - ) - .selectFrom('cte') - .selectAll() - .where('cte.distance', '<=', maxDistance as number) - .execute(); - }); - } - @GenerateSql({ params: [ { diff --git a/server/src/services/base.service.ts b/server/src/services/base.service.ts index 3381ad7222..1c44abfd92 100644 --- a/server/src/services/base.service.ts +++ b/server/src/services/base.service.ts @@ -18,6 +18,7 @@ import { CronRepository } from 'src/repositories/cron.repository'; import { CryptoRepository } from 'src/repositories/crypto.repository'; import { DatabaseRepository } from 'src/repositories/database.repository'; import { DownloadRepository } from 'src/repositories/download.repository'; +import { DuplicateRepository } from 'src/repositories/duplicate.repository'; import { EmailRepository } from 'src/repositories/email.repository'; import { EventRepository } from 'src/repositories/event.repository'; import { JobRepository } from 'src/repositories/job.repository'; @@ -71,6 +72,7 @@ export class BaseService { protected cryptoRepository: CryptoRepository, protected databaseRepository: DatabaseRepository, protected downloadRepository: DownloadRepository, + protected duplicateRepository: DuplicateRepository, protected emailRepository: EmailRepository, protected eventRepository: EventRepository, protected jobRepository: JobRepository, diff --git a/server/src/services/duplicate.service.spec.ts b/server/src/services/duplicate.service.spec.ts index d23144babe..4524a7cb68 100644 --- a/server/src/services/duplicate.service.spec.ts +++ b/server/src/services/duplicate.service.spec.ts @@ -38,7 +38,7 @@ describe(SearchService.name, () => { describe('getDuplicates', () => { it('should get duplicates', async () => { - mocks.asset.getDuplicates.mockResolvedValue([ + mocks.duplicateRepository.getAll.mockResolvedValue([ { duplicateId: 'duplicate-id', assets: [assetStub.image, assetStub.image], @@ -218,25 +218,26 @@ describe(SearchService.name, () => { it('should search for duplicates and update asset with duplicateId', async () => { mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue(hasEmbedding); - mocks.search.searchDuplicates.mockResolvedValue([ + mocks.duplicateRepository.search.mockResolvedValue([ { assetId: assetStub.image.id, distance: 0.01, duplicateId: null }, ]); + mocks.duplicateRepository.merge.mockResolvedValue(); const expectedAssetIds = [assetStub.image.id, hasEmbedding.id]; const result = await sut.handleSearchDuplicates({ id: hasEmbedding.id }); expect(result).toBe(JobStatus.SUCCESS); - expect(mocks.search.searchDuplicates).toHaveBeenCalledWith({ + expect(mocks.duplicateRepository.search).toHaveBeenCalledWith({ assetId: hasEmbedding.id, embedding: hasEmbedding.embedding, maxDistance: 0.01, type: hasEmbedding.type, userIds: [hasEmbedding.ownerId], }); - expect(mocks.asset.updateDuplicates).toHaveBeenCalledWith({ + expect(mocks.duplicateRepository.merge).toHaveBeenCalledWith({ assetIds: expectedAssetIds, - targetDuplicateId: expect.any(String), - duplicateIds: [], + targetId: expect.any(String), + sourceIds: [], }); expect(mocks.asset.upsertJobStatus).toHaveBeenCalledWith( ...expectedAssetIds.map((assetId) => ({ assetId, duplicatesDetectedAt: expect.any(Date) })), @@ -246,23 +247,24 @@ describe(SearchService.name, () => { it('should use existing duplicate ID among matched duplicates', async () => { const duplicateId = hasDupe.duplicateId; mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue(hasEmbedding); - mocks.search.searchDuplicates.mockResolvedValue([{ assetId: hasDupe.id, distance: 0.01, duplicateId }]); + mocks.duplicateRepository.search.mockResolvedValue([{ assetId: hasDupe.id, distance: 0.01, duplicateId }]); + mocks.duplicateRepository.merge.mockResolvedValue(); const expectedAssetIds = [hasEmbedding.id]; const result = await sut.handleSearchDuplicates({ id: hasEmbedding.id }); expect(result).toBe(JobStatus.SUCCESS); - expect(mocks.search.searchDuplicates).toHaveBeenCalledWith({ + expect(mocks.duplicateRepository.search).toHaveBeenCalledWith({ assetId: hasEmbedding.id, embedding: hasEmbedding.embedding, maxDistance: 0.01, type: hasEmbedding.type, userIds: [hasEmbedding.ownerId], }); - expect(mocks.asset.updateDuplicates).toHaveBeenCalledWith({ + expect(mocks.duplicateRepository.merge).toHaveBeenCalledWith({ assetIds: expectedAssetIds, - targetDuplicateId: duplicateId, - duplicateIds: [], + targetId: duplicateId, + sourceIds: [], }); expect(mocks.asset.upsertJobStatus).toHaveBeenCalledWith( ...expectedAssetIds.map((assetId) => ({ assetId, duplicatesDetectedAt: expect.any(Date) })), @@ -271,7 +273,7 @@ describe(SearchService.name, () => { it('should remove duplicateId if no duplicates found and asset has duplicateId', async () => { mocks.assetJob.getForSearchDuplicatesJob.mockResolvedValue(hasDupe); - mocks.search.searchDuplicates.mockResolvedValue([]); + mocks.duplicateRepository.search.mockResolvedValue([]); const result = await sut.handleSearchDuplicates({ id: hasDupe.id }); diff --git a/server/src/services/duplicate.service.ts b/server/src/services/duplicate.service.ts index 617f5c5d0d..ed6e5f16e2 100644 --- a/server/src/services/duplicate.service.ts +++ b/server/src/services/duplicate.service.ts @@ -13,7 +13,7 @@ import { isDuplicateDetectionEnabled } from 'src/utils/misc'; @Injectable() export class DuplicateService extends BaseService { async getDuplicates(auth: AuthDto): Promise { - const duplicates = await this.assetRepository.getDuplicates(auth.user.id); + const duplicates = await this.duplicateRepository.getAll(auth.user.id); return duplicates.map(({ duplicateId, assets }) => ({ duplicateId, assets: assets.map((asset) => mapAsset(asset, { auth })), @@ -74,7 +74,7 @@ export class DuplicateService extends BaseService { return JobStatus.FAILED; } - const duplicateAssets = await this.searchRepository.searchDuplicates({ + const duplicateAssets = await this.duplicateRepository.search({ assetId: asset.id, embedding: asset.embedding, maxDistance: machineLearning.duplicateDetection.maxDistance, @@ -117,7 +117,11 @@ export class DuplicateService extends BaseService { .map((duplicate) => duplicate.assetId); assetIdsToUpdate.push(asset.id); - await this.assetRepository.updateDuplicates({ targetDuplicateId, assetIds: assetIdsToUpdate, duplicateIds }); + await this.duplicateRepository.merge({ + targetId: targetDuplicateId, + assetIds: assetIdsToUpdate, + sourceIds: duplicateIds, + }); return assetIdsToUpdate; } } diff --git a/server/src/services/media.service.ts b/server/src/services/media.service.ts index bd419f0b34..d11ba6d60a 100644 --- a/server/src/services/media.service.ts +++ b/server/src/services/media.service.ts @@ -24,7 +24,6 @@ import { VideoCodec, VideoContainer, } from 'src/enum'; -import { UpsertFileOptions } from 'src/repositories/asset.repository'; import { BoundingBox } from 'src/repositories/machine-learning.repository'; import { BaseService } from 'src/services/base.service'; import { @@ -42,6 +41,11 @@ import { getAssetFiles } from 'src/utils/asset.util'; import { BaseConfig, ThumbnailConfig } from 'src/utils/media'; import { mimeTypes } from 'src/utils/mime-types'; import { clamp, isFaceImportEnabled, isFacialRecognitionEnabled } from 'src/utils/misc'; +interface UpsertFileOptions { + assetId: string; + type: AssetFileType; + path: string; +} @Injectable() export class MediaService extends BaseService { diff --git a/server/test/medium.factory.ts b/server/test/medium.factory.ts index cab74f70fb..9a4065aa9d 100644 --- a/server/test/medium.factory.ts +++ b/server/test/medium.factory.ts @@ -390,6 +390,7 @@ export const asDeps = (repositories: ServiceOverrides) => { repositories.crypto || getRepositoryMock('crypto'), repositories.database || getRepositoryMock('database'), repositories.downloadRepository, + repositories.duplicateRepository, repositories.email || getRepositoryMock('email'), repositories.event, repositories.job || getRepositoryMock('job'), diff --git a/server/test/repositories/asset.repository.mock.ts b/server/test/repositories/asset.repository.mock.ts index d8230a23f3..a29babbf54 100644 --- a/server/test/repositories/asset.repository.mock.ts +++ b/server/test/repositories/asset.repository.mock.ts @@ -22,7 +22,6 @@ export const newAssetRepositoryMock = (): Mocked( config: newConfigRepositoryMock(), database: newDatabaseRepositoryMock(), downloadRepository: automock(DownloadRepository, { strict: false }), + duplicateRepository: automock(DuplicateRepository), email: automock(EmailRepository, { args: [loggerMock] }), // eslint-disable-next-line no-sparse-arrays event: automock(EventRepository, { args: [, , loggerMock], strict: false }), @@ -311,6 +314,7 @@ export const newTestService = ( overrides.crypto || (mocks.crypto as As), overrides.database || (mocks.database as As), overrides.downloadRepository || (mocks.downloadRepository as As), + overrides.duplicateRepository || (mocks.duplicateRepository as As), overrides.email || (mocks.email as As), overrides.event || (mocks.event as As), overrides.job || (mocks.job as As),