diff --git a/docs/docs/features/duplicates-utility.md b/docs/docs/features/duplicates-utility.md new file mode 100644 index 0000000000..f790c42708 --- /dev/null +++ b/docs/docs/features/duplicates-utility.md @@ -0,0 +1,28 @@ +# Duplicates Utility + +Immich comes with a duplicates utility to help you detect assets that look visually similar. The duplicate detection feature relies on machine learning and is enabled by default. For more information about when the duplicate detection job runs, see [Jobs and Workers](/administration/jobs-workers). Once an asset has been processed and added to a duplicate group, it becomes available to review in the "Review duplicates" utility, which can be found [here](https://my.immich.app/utilities/duplicates). + +## Reviewing duplicates + +The review duplicates page allows the user to individually select which assets should be kept and which ones should be trashed. When more than one asset is kept, there is an option to automatically put the kept assets into a stack. + +### Automatic preselection + +When using "Deduplicate All" or viewing suggestions, Immich automatically preselects which assets to keep based on: + +1. **Image size in bytes** — larger files are preferred as they typically have higher quality. +2. **Count of EXIF data** — assets with more metadata are preferred. + +### Synchronizing metadata + +When resolving duplicates, metadata from trashed assets is automatically synchronized to the kept assets. The following metadata is synchronized: + +| Name | Description | +| ----------- | ------------------------------------------------------------------------------------------------------------------------------- | +| Album | The kept assets will be added to _every_ album that the other assets in the group belong to. | +| Favorite | If any of the assets in the group have been added to favorites, every kept asset will also be added to favorites. | +| Rating | If one or more assets in the duplicate group have a rating, the highest rating is selected and synchronized to the kept assets. | +| Description | Descriptions from each asset are combined together and synchronized to all the kept assets. | +| Visibility | The most restrictive visibility is applied to the kept assets. | +| Location | Latitude and longitude are copied if all assets with geolocation data in the group share the same coordinates. | +| Tag | Tags from all assets in the group are merged and applied to every kept asset. | diff --git a/e2e/src/api/specs/duplicate.e2e-spec.ts b/e2e/src/api/specs/duplicate.e2e-spec.ts new file mode 100644 index 0000000000..d6d0ec1394 --- /dev/null +++ b/e2e/src/api/specs/duplicate.e2e-spec.ts @@ -0,0 +1,651 @@ +import { LoginResponseDto } from '@immich/sdk'; +import { createUserDto, uuidDto } from 'src/fixtures'; +import { errorDto } from 'src/responses'; +import { app, utils } from 'src/utils'; +import request from 'supertest'; +import { beforeAll, beforeEach, describe, expect, it } from 'vitest'; + +describe('/duplicates', () => { + let admin: LoginResponseDto; + let user1: LoginResponseDto; + let user2: LoginResponseDto; + + beforeAll(async () => { + await utils.resetDatabase(); + + admin = await utils.adminSetup(); + + [user1, user2] = await Promise.all([ + utils.userSetup(admin.accessToken, createUserDto.user1), + utils.userSetup(admin.accessToken, createUserDto.user2), + ]); + }); + + beforeEach(async () => { + // Reset assets, albums, tags, and stacks between tests to ensure clean state for repeated test runs + // Note: We don't reset users since they're set up once in beforeAll + // Stack must be reset before asset due to foreign key constraint + await utils.resetDatabase(['stack', 'asset', 'album', 'tag']); + }); + + describe('GET /duplicates', () => { + it('should return empty array when no duplicates', async () => { + const { status, body } = await request(app) + .get('/duplicates') + .set('Authorization', `Bearer ${user1.accessToken}`); + + expect(status).toBe(200); + expect(body).toEqual([]); + }); + + it('should return duplicate groups with suggestedKeepAssetIds', async () => { + // Create assets with different file sizes for duplicate detection + const [asset1, asset2] = await Promise.all([ + utils.createAsset(user1.accessToken), + utils.createAsset(user1.accessToken), + ]); + + // Manually set duplicateId on both assets to create a duplicate group + const duplicateId = '00000000-0000-4000-8000-000000000001'; + await utils.setAssetDuplicateId(user1.accessToken, asset1.id, duplicateId); + await utils.setAssetDuplicateId(user1.accessToken, asset2.id, duplicateId); + + const { status, body } = await request(app) + .get('/duplicates') + .set('Authorization', `Bearer ${user1.accessToken}`); + + expect(status).toBe(200); + expect(body).toEqual([ + { + duplicateId, + assets: expect.arrayContaining([ + expect.objectContaining({ id: asset1.id }), + expect.objectContaining({ id: asset2.id }), + ]), + suggestedKeepAssetIds: expect.any(Array), + }, + ]); + expect(body[0].suggestedKeepAssetIds.length).toBe(1); + }); + }); + + describe('POST /duplicates/resolve', () => { + it('should require authentication', async () => { + const { status, body } = await request(app) + .post('/duplicates/resolve') + .send({ + groups: [{ duplicateId: uuidDto.dummy, keepAssetIds: [], trashAssetIds: [] }], + }); + + expect(status).toBe(401); + expect(body).toEqual(errorDto.unauthorized); + }); + + it('should return failure for non-existent duplicate group', async () => { + const { status, body } = await request(app) + .post('/duplicates/resolve') + .set('Authorization', `Bearer ${user1.accessToken}`) + .send({ + groups: [{ duplicateId: uuidDto.dummy, keepAssetIds: [], trashAssetIds: [] }], + }); + + expect(status).toBe(200); + expect(body).toEqual({ + status: 'COMPLETED', + results: [ + { + duplicateId: uuidDto.dummy, + status: 'FAILED', + reason: expect.stringContaining('not found or access denied'), + }, + ], + }); + }); + + it('should resolve duplicate group with keepers', async () => { + const [asset1, asset2] = await Promise.all([ + utils.createAsset(user1.accessToken), + utils.createAsset(user1.accessToken), + ]); + + const duplicateId = '00000000-0000-4000-8000-000000000002'; + await utils.setAssetDuplicateId(user1.accessToken, asset1.id, duplicateId); + await utils.setAssetDuplicateId(user1.accessToken, asset2.id, duplicateId); + + const { status, body } = await request(app) + .post('/duplicates/resolve') + .set('Authorization', `Bearer ${user1.accessToken}`) + .send({ + groups: [{ duplicateId, keepAssetIds: [asset1.id], trashAssetIds: [asset2.id] }], + }); + + expect(status).toBe(200); + expect(body).toEqual({ + status: 'COMPLETED', + results: [ + { + duplicateId, + status: 'SUCCESS', + }, + ], + }); + + // Verify side effects: duplicateId cleared on kept asset + const keptAsset = await utils.getAssetInfo(user1.accessToken, asset1.id); + expect(keptAsset.duplicateId).toBeNull(); + + // Verify side effects: trashed asset is trashed and duplicateId cleared + const trashedAsset = await utils.getAssetInfo(user1.accessToken, asset2.id); + expect(trashedAsset.isTrashed).toBe(true); + expect(trashedAsset.duplicateId).toBeNull(); + }); + + it('should reject when keepAssetIds and trashAssetIds overlap', async () => { + const [asset1, asset2] = await Promise.all([ + utils.createAsset(user1.accessToken), + utils.createAsset(user1.accessToken), + ]); + + const duplicateId = '00000000-0000-4000-8000-000000000003'; + await utils.setAssetDuplicateId(user1.accessToken, asset1.id, duplicateId); + await utils.setAssetDuplicateId(user1.accessToken, asset2.id, duplicateId); + + const { status, body } = await request(app) + .post('/duplicates/resolve') + .set('Authorization', `Bearer ${user1.accessToken}`) + .send({ + groups: [{ duplicateId, keepAssetIds: [asset1.id], trashAssetIds: [asset1.id] }], + }); + + expect(status).toBe(200); + expect(body.results[0].status).toBe('FAILED'); + expect(body.results[0].reason).toContain('disjoint'); + }); + + it('should require keepAssetIds when partially trashing', async () => { + const [asset1, asset2] = await Promise.all([ + utils.createAsset(user1.accessToken), + utils.createAsset(user1.accessToken), + ]); + + const duplicateId = '00000000-0000-4000-8000-000000000004'; + await utils.setAssetDuplicateId(user1.accessToken, asset1.id, duplicateId); + await utils.setAssetDuplicateId(user1.accessToken, asset2.id, duplicateId); + + const { status, body } = await request(app) + .post('/duplicates/resolve') + .set('Authorization', `Bearer ${user1.accessToken}`) + .send({ + groups: [{ duplicateId, keepAssetIds: [], trashAssetIds: [asset1.id] }], + }); + + expect(status).toBe(200); + expect(body.results[0].status).toBe('FAILED'); + expect(body.results[0].reason).toContain('must cover all assets'); + }); + + it('should reject partial resolution (not all assets covered)', async () => { + const [asset1, asset2, asset3] = await Promise.all([ + utils.createAsset(user1.accessToken), + utils.createAsset(user1.accessToken), + utils.createAsset(user1.accessToken), + ]); + + const duplicateId = '00000000-0000-4000-8000-000000000010'; + await utils.setAssetDuplicateId(user1.accessToken, asset1.id, duplicateId); + await utils.setAssetDuplicateId(user1.accessToken, asset2.id, duplicateId); + await utils.setAssetDuplicateId(user1.accessToken, asset3.id, duplicateId); + + const { status, body } = await request(app) + .post('/duplicates/resolve') + .set('Authorization', `Bearer ${user1.accessToken}`) + .send({ + groups: [{ duplicateId, keepAssetIds: [asset1.id], trashAssetIds: [asset2.id] }], + }); + + expect(status).toBe(200); + expect(body.results[0].status).toBe('FAILED'); + expect(body.results[0].reason).toContain('must cover all assets'); + }); + + it('should reject asset not in duplicate group', async () => { + const [asset1, asset2, outsideAsset] = await Promise.all([ + utils.createAsset(user1.accessToken), + utils.createAsset(user1.accessToken), + utils.createAsset(user1.accessToken), + ]); + + const duplicateId = '00000000-0000-4000-8000-000000000011'; + await utils.setAssetDuplicateId(user1.accessToken, asset1.id, duplicateId); + await utils.setAssetDuplicateId(user1.accessToken, asset2.id, duplicateId); + + const { status, body } = await request(app) + .post('/duplicates/resolve') + .set('Authorization', `Bearer ${user1.accessToken}`) + .send({ + groups: [{ duplicateId, keepAssetIds: [asset1.id], trashAssetIds: [outsideAsset.id] }], + }); + + expect(status).toBe(200); + expect(body.results[0].status).toBe('FAILED'); + expect(body.results[0].reason).toContain('not a member of duplicate group'); + }); + + it('should allow trash-all without keepers', async () => { + const [asset1, asset2] = await Promise.all([ + utils.createAsset(user1.accessToken), + utils.createAsset(user1.accessToken), + ]); + + const duplicateId = '00000000-0000-4000-8000-000000000012'; + await utils.setAssetDuplicateId(user1.accessToken, asset1.id, duplicateId); + await utils.setAssetDuplicateId(user1.accessToken, asset2.id, duplicateId); + + const { status, body } = await request(app) + .post('/duplicates/resolve') + .set('Authorization', `Bearer ${user1.accessToken}`) + .send({ + groups: [{ duplicateId, keepAssetIds: [], trashAssetIds: [asset1.id, asset2.id] }], + }); + + expect(status).toBe(200); + expect(body).toEqual({ + status: 'COMPLETED', + results: [ + { + duplicateId, + status: 'SUCCESS', + }, + ], + }); + + // Verify both assets are trashed + const [asset1Info, asset2Info] = await Promise.all([ + utils.getAssetInfo(user1.accessToken, asset1.id), + utils.getAssetInfo(user1.accessToken, asset2.id), + ]); + + expect(asset1Info.isTrashed).toBe(true); + expect(asset1Info.duplicateId).toBeNull(); + expect(asset2Info.isTrashed).toBe(true); + expect(asset2Info.duplicateId).toBeNull(); + }); + + it('should reject cross-user duplicate group access', async () => { + const asset1 = await utils.createAsset(user1.accessToken); + const asset2 = await utils.createAsset(user2.accessToken); + + const duplicateId = '00000000-0000-4000-8000-000000000013'; + await utils.setAssetDuplicateId(user1.accessToken, asset1.id, duplicateId); + await utils.setAssetDuplicateId(user2.accessToken, asset2.id, duplicateId); + + // User1 tries to resolve a group containing user2's asset + const { status, body } = await request(app) + .post('/duplicates/resolve') + .set('Authorization', `Bearer ${user1.accessToken}`) + .send({ + groups: [{ duplicateId, keepAssetIds: [asset1.id], trashAssetIds: [asset2.id] }], + }); + + expect(status).toBe(200); + expect(body.results[0].status).toBe('FAILED'); + expect(body.results[0].reason).toContain('not a member of duplicate group'); + }); + + it('should synchronize favorites when enabled', async () => { + const [asset1, asset2] = await Promise.all([ + utils.createAsset(user1.accessToken), + utils.createAsset(user1.accessToken), + ]); + + // Mark one asset as favorite + await request(app) + .put('/assets') + .set('Authorization', `Bearer ${user1.accessToken}`) + .send({ ids: [asset2.id], isFavorite: true }); + + const duplicateId = '00000000-0000-4000-8000-000000000020'; + await utils.setAssetDuplicateId(user1.accessToken, asset1.id, duplicateId); + await utils.setAssetDuplicateId(user1.accessToken, asset2.id, duplicateId); + + const { status, body } = await request(app) + .post('/duplicates/resolve') + .set('Authorization', `Bearer ${user1.accessToken}`) + .send({ + groups: [{ duplicateId, keepAssetIds: [asset1.id], trashAssetIds: [asset2.id] }], + }); + + expect(status).toBe(200); + expect(body.results[0].status).toBe('SUCCESS'); + + // Verify favorite was synchronized to keeper + const keptAsset = await utils.getAssetInfo(user1.accessToken, asset1.id); + expect(keptAsset.isFavorite).toBe(true); + expect(keptAsset.duplicateId).toBeNull(); + }); + + it('should synchronize visibility when enabled', async () => { + const [asset1, asset2] = await Promise.all([ + utils.createAsset(user1.accessToken), + utils.createAsset(user1.accessToken), + ]); + + // Archive one asset + await utils.archiveAssets(user1.accessToken, [asset2.id]); + + const duplicateId = '00000000-0000-4000-8000-000000000021'; + await utils.setAssetDuplicateId(user1.accessToken, asset1.id, duplicateId); + await utils.setAssetDuplicateId(user1.accessToken, asset2.id, duplicateId); + + const { status, body } = await request(app) + .post('/duplicates/resolve') + .set('Authorization', `Bearer ${user1.accessToken}`) + .send({ + groups: [{ duplicateId, keepAssetIds: [asset1.id], trashAssetIds: [asset2.id] }], + }); + + expect(status).toBe(200); + expect(body.results[0].status).toBe('SUCCESS'); + + // Verify visibility was synchronized to keeper + const keptAsset = await utils.getAssetInfo(user1.accessToken, asset1.id); + expect(keptAsset.visibility).toBe('archive'); + expect(keptAsset.duplicateId).toBeNull(); + }); + + it('should synchronize rating when enabled', async () => { + const [asset1, asset2] = await Promise.all([ + utils.createAsset(user1.accessToken), + utils.createAsset(user1.accessToken), + ]); + + // Set rating on one asset + await request(app) + .put('/assets') + .set('Authorization', `Bearer ${user1.accessToken}`) + .send({ ids: [asset2.id], rating: 5 }); + + const duplicateId = '00000000-0000-4000-8000-000000000022'; + await utils.setAssetDuplicateId(user1.accessToken, asset1.id, duplicateId); + await utils.setAssetDuplicateId(user1.accessToken, asset2.id, duplicateId); + + const { status, body } = await request(app) + .post('/duplicates/resolve') + .set('Authorization', `Bearer ${user1.accessToken}`) + .send({ + groups: [{ duplicateId, keepAssetIds: [asset1.id], trashAssetIds: [asset2.id] }], + }); + + expect(status).toBe(200); + expect(body.results[0].status).toBe('SUCCESS'); + + // Verify rating was synchronized to keeper + const keptAsset = await utils.getAssetInfo(user1.accessToken, asset1.id); + expect(keptAsset.exifInfo?.rating).toBe(5); + expect(keptAsset.duplicateId).toBeNull(); + }); + + it('should synchronize description when enabled', async () => { + const [asset1, asset2] = await Promise.all([ + utils.createAsset(user1.accessToken), + utils.createAsset(user1.accessToken), + ]); + + // Set description on one asset + await request(app) + .put('/assets') + .set('Authorization', `Bearer ${user1.accessToken}`) + .send({ ids: [asset2.id], description: 'Test description for duplicate' }); + + const duplicateId = '00000000-0000-4000-8000-000000000023'; + await utils.setAssetDuplicateId(user1.accessToken, asset1.id, duplicateId); + await utils.setAssetDuplicateId(user1.accessToken, asset2.id, duplicateId); + + const { status, body } = await request(app) + .post('/duplicates/resolve') + .set('Authorization', `Bearer ${user1.accessToken}`) + .send({ + groups: [{ duplicateId, keepAssetIds: [asset1.id], trashAssetIds: [asset2.id] }], + }); + + expect(status).toBe(200); + expect(body.results[0].status).toBe('SUCCESS'); + + // Verify description was synchronized to keeper + const keptAsset = await utils.getAssetInfo(user1.accessToken, asset1.id); + expect(keptAsset.exifInfo?.description).toBe('Test description for duplicate'); + expect(keptAsset.duplicateId).toBeNull(); + }); + + it('should synchronize location when enabled', async () => { + const [asset1, asset2] = await Promise.all([ + utils.createAsset(user1.accessToken), + utils.createAsset(user1.accessToken), + ]); + + // Set location on one asset + await request(app) + .put('/assets') + .set('Authorization', `Bearer ${user1.accessToken}`) + .send({ ids: [asset2.id], latitude: 40.7128, longitude: -74.006 }); + + const duplicateId = '00000000-0000-4000-8000-000000000024'; + await utils.setAssetDuplicateId(user1.accessToken, asset1.id, duplicateId); + await utils.setAssetDuplicateId(user1.accessToken, asset2.id, duplicateId); + + const { status, body } = await request(app) + .post('/duplicates/resolve') + .set('Authorization', `Bearer ${user1.accessToken}`) + .send({ + groups: [{ duplicateId, keepAssetIds: [asset1.id], trashAssetIds: [asset2.id] }], + }); + + expect(status).toBe(200); + expect(body.results[0].status).toBe('SUCCESS'); + + // Verify location was synchronized to keeper + const keptAsset = await utils.getAssetInfo(user1.accessToken, asset1.id); + expect(keptAsset.exifInfo?.latitude).toBe(40.7128); + expect(keptAsset.exifInfo?.longitude).toBe(-74.006); + expect(keptAsset.duplicateId).toBeNull(); + }); + + it('should synchronize albums when enabled', async () => { + const [asset1, asset2] = await Promise.all([ + utils.createAsset(user1.accessToken), + utils.createAsset(user1.accessToken), + ]); + + // Create albums and add assets to different albums + const album1 = await utils.createAlbum(user1.accessToken, { + albumName: 'Album 1', + assetIds: [asset1.id], + }); + const album2 = await utils.createAlbum(user1.accessToken, { + albumName: 'Album 2', + assetIds: [asset2.id], + }); + + const duplicateId = '00000000-0000-4000-8000-000000000025'; + await utils.setAssetDuplicateId(user1.accessToken, asset1.id, duplicateId); + await utils.setAssetDuplicateId(user1.accessToken, asset2.id, duplicateId); + + const { status, body } = await request(app) + .post('/duplicates/resolve') + .set('Authorization', `Bearer ${user1.accessToken}`) + .send({ + groups: [{ duplicateId, keepAssetIds: [asset1.id], trashAssetIds: [asset2.id] }], + }); + + expect(status).toBe(200); + expect(body.results[0].status).toBe('SUCCESS'); + + // Verify keeper is now in both albums + const keptAsset = await utils.getAssetInfo(user1.accessToken, asset1.id); + expect(keptAsset.duplicateId).toBeNull(); + + // Check albums directly + const { status: album1Status, body: album1Body } = await request(app) + .get(`/albums/${album1.id}`) + .set('Authorization', `Bearer ${user1.accessToken}`); + const { status: album2Status, body: album2Body } = await request(app) + .get(`/albums/${album2.id}`) + .set('Authorization', `Bearer ${user1.accessToken}`); + + expect(album1Status).toBe(200); + expect(album2Status).toBe(200); + expect(album1Body.assets.map((a: any) => a.id)).toContain(asset1.id); + expect(album2Body.assets.map((a: any) => a.id)).toContain(asset1.id); + }); + + it('should synchronize tags when enabled', async () => { + const [asset1, asset2] = await Promise.all([ + utils.createAsset(user1.accessToken), + utils.createAsset(user1.accessToken), + ]); + + // Wait for metadata extraction to complete before adding tags + // Otherwise, metadata jobs will race and overwrite our tags + await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction'); + + // Create tags and tag assets differently + const tags = await utils.upsertTags(user1.accessToken, ['tag1', 'tag2']); + await utils.tagAssets(user1.accessToken, tags[0].id, [asset1.id]); + await utils.tagAssets(user1.accessToken, tags[1].id, [asset2.id]); + + const duplicateId = '00000000-0000-4000-8000-000000000026'; + await utils.setAssetDuplicateId(user1.accessToken, asset1.id, duplicateId); + await utils.setAssetDuplicateId(user1.accessToken, asset2.id, duplicateId); + + const { status, body } = await request(app) + .post('/duplicates/resolve') + .set('Authorization', `Bearer ${user1.accessToken}`) + .send({ + groups: [{ duplicateId, keepAssetIds: [asset1.id], trashAssetIds: [asset2.id] }], + }); + + expect(status).toBe(200); + expect(body.results[0].status).toBe('SUCCESS'); + + // Verify keeper has both tags + const keptAsset = await utils.getAssetInfo(user1.accessToken, asset1.id); + expect(keptAsset.duplicateId).toBeNull(); + expect(keptAsset.tags).toBeDefined(); + const tagIds = keptAsset.tags?.map((t) => t.id) || []; + expect(tagIds).toContain(tags[0].id); + expect(tagIds).toContain(tags[1].id); + }); + + it('should handle batch resolve with mixed success and failure', async () => { + // Create first group that will succeed + const [asset1, asset2] = await Promise.all([ + utils.createAsset(user1.accessToken), + utils.createAsset(user1.accessToken), + ]); + const duplicateId1 = '00000000-0000-4000-8000-000000000027'; + await utils.setAssetDuplicateId(user1.accessToken, asset1.id, duplicateId1); + await utils.setAssetDuplicateId(user1.accessToken, asset2.id, duplicateId1); + + // Create second group with non-existent duplicate ID (will fail) + const fakeId = '00000000-0000-4000-8000-000000000099'; + + const { status, body } = await request(app) + .post('/duplicates/resolve') + .set('Authorization', `Bearer ${user1.accessToken}`) + .send({ + groups: [ + { duplicateId: duplicateId1, keepAssetIds: [asset1.id], trashAssetIds: [asset2.id] }, + { duplicateId: fakeId, keepAssetIds: [], trashAssetIds: [] }, + ], + }); + + expect(status).toBe(200); + expect(body.status).toBe('COMPLETED'); + expect(body.results).toHaveLength(2); + + // First group should succeed + expect(body.results[0].duplicateId).toBe(duplicateId1); + expect(body.results[0].status).toBe('SUCCESS'); + + // Second group should fail + expect(body.results[1].duplicateId).toBe(fakeId); + expect(body.results[1].status).toBe('FAILED'); + expect(body.results[1].reason).toContain('not found or access denied'); + + // Verify first group was actually resolved despite second failure + const asset1Info = await utils.getAssetInfo(user1.accessToken, asset1.id); + expect(asset1Info.duplicateId).toBeNull(); + const asset2Info = await utils.getAssetInfo(user1.accessToken, asset2.id); + expect(asset2Info.isTrashed).toBe(true); + }); + + it('should trash assets when trash is enabled', async () => { + const [asset1, asset2] = await Promise.all([ + utils.createAsset(user1.accessToken), + utils.createAsset(user1.accessToken), + ]); + + const duplicateId = '00000000-0000-4000-8000-000000000028'; + await utils.setAssetDuplicateId(user1.accessToken, asset1.id, duplicateId); + await utils.setAssetDuplicateId(user1.accessToken, asset2.id, duplicateId); + + // Ensure trash is enabled (default) + const config = await utils.getSystemConfig(admin.accessToken); + expect(config.trash.enabled).toBe(true); + + const { status, body } = await request(app) + .post('/duplicates/resolve') + .set('Authorization', `Bearer ${user1.accessToken}`) + .send({ + groups: [{ duplicateId, keepAssetIds: [asset1.id], trashAssetIds: [asset2.id] }], + }); + + expect(status).toBe(200); + expect(body.results[0].status).toBe('SUCCESS'); + + // Verify asset is trashed (not deleted) + const trashedAsset = await utils.getAssetInfo(user1.accessToken, asset2.id); + expect(trashedAsset.isTrashed).toBe(true); + }); + + it('should delete assets when trash is disabled', async () => { + const [asset1, asset2] = await Promise.all([ + utils.createAsset(user1.accessToken), + utils.createAsset(user1.accessToken), + ]); + + const duplicateId = '00000000-0000-4000-8000-000000000029'; + await utils.setAssetDuplicateId(user1.accessToken, asset1.id, duplicateId); + await utils.setAssetDuplicateId(user1.accessToken, asset2.id, duplicateId); + + // Disable trash + await request(app) + .put('/system-config') + .set('Authorization', `Bearer ${admin.accessToken}`) + .send({ + trash: { enabled: false, days: 30 }, + }); + + const { status, body } = await request(app) + .post('/duplicates/resolve') + .set('Authorization', `Bearer ${user1.accessToken}`) + .send({ + groups: [{ duplicateId, keepAssetIds: [asset1.id], trashAssetIds: [asset2.id] }], + }); + + expect(status).toBe(200); + expect(body.results[0].status).toBe('SUCCESS'); + + // Asset should be marked as deleted (force delete) + const { status: getStatus } = await request(app) + .get(`/assets/${asset2.id}`) + .set('Authorization', `Bearer ${user1.accessToken}`); + + // Asset should still be accessible (soft deleted) but marked as deleted + expect(getStatus).toBe(200); + + // Re-enable trash for other tests + await utils.resetAdminConfig(admin.accessToken); + }); + }); +}); diff --git a/e2e/src/fixtures.ts b/e2e/src/fixtures.ts index 9e311c896d..1e03ad6d24 100644 --- a/e2e/src/fixtures.ts +++ b/e2e/src/fixtures.ts @@ -2,6 +2,8 @@ export const uuidDto = { invalid: 'invalid-uuid', // valid uuid v4 notFound: '00000000-0000-4000-a000-000000000000', + dummy: '00000000-0000-4000-a000-000000000001', + dummy2: '00000000-0000-4000-a000-000000000002', }; const adminLoginDto = { diff --git a/e2e/src/utils.ts b/e2e/src/utils.ts index a5567f0778..4d44d99e2f 100644 --- a/e2e/src/utils.ts +++ b/e2e/src/utils.ts @@ -510,6 +510,9 @@ export const utils = { createStack: (accessToken: string, assetIds: string[]) => createStack({ stackCreateDto: { assetIds } }, { headers: asBearerAuth(accessToken) }), + setAssetDuplicateId: (accessToken: string, assetId: string, duplicateId: string | null) => + updateAssets({ assetBulkUpdateDto: { ids: [assetId], duplicateId } }, { headers: asBearerAuth(accessToken) }), + upsertTags: (accessToken: string, tags: string[]) => upsertTags({ tagUpsertDto: { tags } }, { headers: asBearerAuth(accessToken) }), diff --git a/i18n/en.json b/i18n/en.json index 252664653c..42a89586f9 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -892,10 +892,8 @@ "day": "Day", "days": "Days", "deduplicate_all": "Deduplicate All", - "deduplication_criteria_1": "Image size in bytes", - "deduplication_criteria_2": "Count of EXIF data", - "deduplication_info": "Deduplication Info", - "deduplication_info_description": "To automatically preselect assets and remove duplicates in bulk, we look at:", + "default_locale": "Default Locale", + "default_locale_description": "Format dates and numbers based on your browser locale", "delete": "Delete", "delete_action_confirmation_message": "Are you sure you want to delete this asset? This action will move the asset to the server's trash and will prompt if you want to delete it locally", "delete_action_prompt": "{count} deleted", @@ -971,7 +969,7 @@ "downloading_media": "Downloading media", "drop_files_to_upload": "Drop files anywhere to upload", "duplicates": "Duplicates", - "duplicates_description": "Resolve each group by indicating which, if any, are duplicates", + "duplicates_description": "Resolve each group by indicating which, if any, are duplicates.", "duration": "Duration", "edit": "Edit", "edit_album": "Edit album", @@ -1392,6 +1390,7 @@ "like": "Like", "like_deleted": "Like deleted", "link_motion_video": "Link motion video", + "link_to_docs": "For more information, refer to the documentation.", "link_to_oauth": "Link to OAuth", "linked_oauth_account": "Linked OAuth account", "list": "List", @@ -2396,6 +2395,7 @@ "viewer_remove_from_stack": "Remove from Stack", "viewer_stack_use_as_main_asset": "Use as Main Asset", "viewer_unstack": "Un-Stack", + "visibility": "Visibility", "visibility_changed": "Visibility changed for {count, plural, one {# person} other {# people}}", "visual": "Visual", "visual_builder": "Visual builder", diff --git a/mobile/openapi/README.md b/mobile/openapi/README.md index 500de51622..e79a60f98b 100644 --- a/mobile/openapi/README.md +++ b/mobile/openapi/README.md @@ -156,6 +156,7 @@ Class | Method | HTTP request | Description *DuplicatesApi* | [**deleteDuplicate**](doc//DuplicatesApi.md#deleteduplicate) | **DELETE** /duplicates/{id} | Delete a duplicate *DuplicatesApi* | [**deleteDuplicates**](doc//DuplicatesApi.md#deleteduplicates) | **DELETE** /duplicates | Delete duplicates *DuplicatesApi* | [**getAssetDuplicates**](doc//DuplicatesApi.md#getassetduplicates) | **GET** /duplicates | Retrieve duplicates +*DuplicatesApi* | [**resolveDuplicates**](doc//DuplicatesApi.md#resolveduplicates) | **POST** /duplicates/resolve | Resolve duplicate groups *FacesApi* | [**createFace**](doc//FacesApi.md#createface) | **POST** /faces | Create a face *FacesApi* | [**deleteFace**](doc//FacesApi.md#deleteface) | **DELETE** /faces/{id} | Delete a face *FacesApi* | [**getFaces**](doc//FacesApi.md#getfaces) | **GET** /faces | Retrieve faces for asset @@ -422,6 +423,8 @@ Class | Method | HTTP request | Description - [DownloadResponseDto](doc//DownloadResponseDto.md) - [DownloadUpdate](doc//DownloadUpdate.md) - [DuplicateDetectionConfig](doc//DuplicateDetectionConfig.md) + - [DuplicateResolveDto](doc//DuplicateResolveDto.md) + - [DuplicateResolveGroupDto](doc//DuplicateResolveGroupDto.md) - [DuplicateResponseDto](doc//DuplicateResponseDto.md) - [EmailNotificationsResponse](doc//EmailNotificationsResponse.md) - [EmailNotificationsUpdate](doc//EmailNotificationsUpdate.md) diff --git a/mobile/openapi/lib/api.dart b/mobile/openapi/lib/api.dart index 253e8a6811..6b554fb644 100644 --- a/mobile/openapi/lib/api.dart +++ b/mobile/openapi/lib/api.dart @@ -161,6 +161,8 @@ part 'model/download_response.dart'; part 'model/download_response_dto.dart'; part 'model/download_update.dart'; part 'model/duplicate_detection_config.dart'; +part 'model/duplicate_resolve_dto.dart'; +part 'model/duplicate_resolve_group_dto.dart'; part 'model/duplicate_response_dto.dart'; part 'model/email_notifications_response.dart'; part 'model/email_notifications_update.dart'; diff --git a/mobile/openapi/lib/api/duplicates_api.dart b/mobile/openapi/lib/api/duplicates_api.dart index 7fa7b368b5..e873537592 100644 --- a/mobile/openapi/lib/api/duplicates_api.dart +++ b/mobile/openapi/lib/api/duplicates_api.dart @@ -163,4 +163,63 @@ class DuplicatesApi { } return null; } + + /// Resolve duplicate groups + /// + /// Resolve duplicate groups by synchronizing metadata across assets and deleting/trashing duplicates. + /// + /// Note: This method returns the HTTP [Response]. + /// + /// Parameters: + /// + /// * [DuplicateResolveDto] duplicateResolveDto (required): + Future resolveDuplicatesWithHttpInfo(DuplicateResolveDto duplicateResolveDto,) async { + // ignore: prefer_const_declarations + final apiPath = r'/duplicates/resolve'; + + // ignore: prefer_final_locals + Object? postBody = duplicateResolveDto; + + final queryParams = []; + final headerParams = {}; + final formParams = {}; + + const contentTypes = ['application/json']; + + + return apiClient.invokeAPI( + apiPath, + 'POST', + queryParams, + postBody, + headerParams, + formParams, + contentTypes.isEmpty ? null : contentTypes.first, + ); + } + + /// Resolve duplicate groups + /// + /// Resolve duplicate groups by synchronizing metadata across assets and deleting/trashing duplicates. + /// + /// Parameters: + /// + /// * [DuplicateResolveDto] duplicateResolveDto (required): + Future?> resolveDuplicates(DuplicateResolveDto duplicateResolveDto,) async { + final response = await resolveDuplicatesWithHttpInfo(duplicateResolveDto,); + if (response.statusCode >= HttpStatus.badRequest) { + throw ApiException(response.statusCode, await _decodeBodyBytes(response)); + } + // When a remote server returns no body with a status of 204, we shall not decode it. + // At the time of writing this, `dart:convert` will throw an "Unexpected end of input" + // FormatException when trying to decode an empty string. + if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) { + final responseBody = await _decodeBodyBytes(response); + return (await apiClient.deserializeAsync(responseBody, 'List') as List) + .cast() + .toList(growable: false); + + } + return null; + } } diff --git a/mobile/openapi/lib/api_client.dart b/mobile/openapi/lib/api_client.dart index bfe469e7c0..48e5f5874b 100644 --- a/mobile/openapi/lib/api_client.dart +++ b/mobile/openapi/lib/api_client.dart @@ -368,6 +368,10 @@ class ApiClient { return DownloadUpdate.fromJson(value); case 'DuplicateDetectionConfig': return DuplicateDetectionConfig.fromJson(value); + case 'DuplicateResolveDto': + return DuplicateResolveDto.fromJson(value); + case 'DuplicateResolveGroupDto': + return DuplicateResolveGroupDto.fromJson(value); case 'DuplicateResponseDto': return DuplicateResponseDto.fromJson(value); case 'EmailNotificationsResponse': diff --git a/mobile/openapi/lib/model/bulk_id_error_reason.dart b/mobile/openapi/lib/model/bulk_id_error_reason.dart index ea56e9dbba..fd6c61d6fd 100644 --- a/mobile/openapi/lib/model/bulk_id_error_reason.dart +++ b/mobile/openapi/lib/model/bulk_id_error_reason.dart @@ -27,6 +27,7 @@ class BulkIdErrorReason { static const noPermission = BulkIdErrorReason._(r'no_permission'); static const notFound = BulkIdErrorReason._(r'not_found'); static const unknown = BulkIdErrorReason._(r'unknown'); + static const validation = BulkIdErrorReason._(r'validation'); /// List of all possible values in this [enum][BulkIdErrorReason]. static const values = [ @@ -34,6 +35,7 @@ class BulkIdErrorReason { noPermission, notFound, unknown, + validation, ]; static BulkIdErrorReason? fromJson(dynamic value) => BulkIdErrorReasonTypeTransformer().decode(value); @@ -76,6 +78,7 @@ class BulkIdErrorReasonTypeTransformer { case r'no_permission': return BulkIdErrorReason.noPermission; case r'not_found': return BulkIdErrorReason.notFound; case r'unknown': return BulkIdErrorReason.unknown; + case r'validation': return BulkIdErrorReason.validation; default: if (!allowNull) { throw ArgumentError('Unknown enum value to decode: $data'); diff --git a/mobile/openapi/lib/model/bulk_id_response_dto.dart b/mobile/openapi/lib/model/bulk_id_response_dto.dart index cd122785dd..1fa8536964 100644 --- a/mobile/openapi/lib/model/bulk_id_response_dto.dart +++ b/mobile/openapi/lib/model/bulk_id_response_dto.dart @@ -14,6 +14,7 @@ class BulkIdResponseDto { /// Returns a new [BulkIdResponseDto] instance. BulkIdResponseDto({ this.error, + this.errorMessage, required this.id, required this.success, }); @@ -21,6 +22,14 @@ class BulkIdResponseDto { /// Error reason if failed BulkIdResponseDtoErrorEnum? error; + /// + /// Please note: This property should have been non-nullable! Since the specification file + /// does not include a default value (using the "default:" property), however, the generated + /// source code must fall back to having a nullable type. + /// Consider adding a "default:" property in the specification file to hide this note. + /// + String? errorMessage; + /// ID String id; @@ -30,6 +39,7 @@ class BulkIdResponseDto { @override bool operator ==(Object other) => identical(this, other) || other is BulkIdResponseDto && other.error == error && + other.errorMessage == errorMessage && other.id == id && other.success == success; @@ -37,11 +47,12 @@ class BulkIdResponseDto { int get hashCode => // ignore: unnecessary_parenthesis (error == null ? 0 : error!.hashCode) + + (errorMessage == null ? 0 : errorMessage!.hashCode) + (id.hashCode) + (success.hashCode); @override - String toString() => 'BulkIdResponseDto[error=$error, id=$id, success=$success]'; + String toString() => 'BulkIdResponseDto[error=$error, errorMessage=$errorMessage, id=$id, success=$success]'; Map toJson() { final json = {}; @@ -49,6 +60,11 @@ class BulkIdResponseDto { json[r'error'] = this.error; } else { // json[r'error'] = null; + } + if (this.errorMessage != null) { + json[r'errorMessage'] = this.errorMessage; + } else { + // json[r'errorMessage'] = null; } json[r'id'] = this.id; json[r'success'] = this.success; @@ -65,6 +81,7 @@ class BulkIdResponseDto { return BulkIdResponseDto( error: BulkIdResponseDtoErrorEnum.fromJson(json[r'error']), + errorMessage: mapValueOfType(json, r'errorMessage'), id: mapValueOfType(json, r'id')!, success: mapValueOfType(json, r'success')!, ); @@ -136,6 +153,7 @@ class BulkIdResponseDtoErrorEnum { static const noPermission = BulkIdResponseDtoErrorEnum._(r'no_permission'); static const notFound = BulkIdResponseDtoErrorEnum._(r'not_found'); static const unknown = BulkIdResponseDtoErrorEnum._(r'unknown'); + static const validation = BulkIdResponseDtoErrorEnum._(r'validation'); /// List of all possible values in this [enum][BulkIdResponseDtoErrorEnum]. static const values = [ @@ -143,6 +161,7 @@ class BulkIdResponseDtoErrorEnum { noPermission, notFound, unknown, + validation, ]; static BulkIdResponseDtoErrorEnum? fromJson(dynamic value) => BulkIdResponseDtoErrorEnumTypeTransformer().decode(value); @@ -185,6 +204,7 @@ class BulkIdResponseDtoErrorEnumTypeTransformer { case r'no_permission': return BulkIdResponseDtoErrorEnum.noPermission; case r'not_found': return BulkIdResponseDtoErrorEnum.notFound; case r'unknown': return BulkIdResponseDtoErrorEnum.unknown; + case r'validation': return BulkIdResponseDtoErrorEnum.validation; default: if (!allowNull) { throw ArgumentError('Unknown enum value to decode: $data'); diff --git a/mobile/openapi/lib/model/duplicate_resolve_dto.dart b/mobile/openapi/lib/model/duplicate_resolve_dto.dart new file mode 100644 index 0000000000..3466d3a620 --- /dev/null +++ b/mobile/openapi/lib/model/duplicate_resolve_dto.dart @@ -0,0 +1,100 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class DuplicateResolveDto { + /// Returns a new [DuplicateResolveDto] instance. + DuplicateResolveDto({ + this.groups = const [], + }); + + /// List of duplicate groups to resolve + List groups; + + @override + bool operator ==(Object other) => identical(this, other) || other is DuplicateResolveDto && + _deepEquality.equals(other.groups, groups); + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (groups.hashCode); + + @override + String toString() => 'DuplicateResolveDto[groups=$groups]'; + + Map toJson() { + final json = {}; + json[r'groups'] = this.groups; + return json; + } + + /// Returns a new [DuplicateResolveDto] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static DuplicateResolveDto? fromJson(dynamic value) { + upgradeDto(value, "DuplicateResolveDto"); + if (value is Map) { + final json = value.cast(); + + return DuplicateResolveDto( + groups: DuplicateResolveGroupDto.listFromJson(json[r'groups']), + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = DuplicateResolveDto.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = DuplicateResolveDto.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of DuplicateResolveDto-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = DuplicateResolveDto.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'groups', + }; +} + diff --git a/mobile/openapi/lib/model/duplicate_resolve_group_dto.dart b/mobile/openapi/lib/model/duplicate_resolve_group_dto.dart new file mode 100644 index 0000000000..94ca53eb7d --- /dev/null +++ b/mobile/openapi/lib/model/duplicate_resolve_group_dto.dart @@ -0,0 +1,121 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class DuplicateResolveGroupDto { + /// Returns a new [DuplicateResolveGroupDto] instance. + DuplicateResolveGroupDto({ + required this.duplicateId, + this.keepAssetIds = const [], + this.trashAssetIds = const [], + }); + + String duplicateId; + + /// Asset IDs to keep + List keepAssetIds; + + /// Asset IDs to trash or delete + List trashAssetIds; + + @override + bool operator ==(Object other) => identical(this, other) || other is DuplicateResolveGroupDto && + other.duplicateId == duplicateId && + _deepEquality.equals(other.keepAssetIds, keepAssetIds) && + _deepEquality.equals(other.trashAssetIds, trashAssetIds); + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (duplicateId.hashCode) + + (keepAssetIds.hashCode) + + (trashAssetIds.hashCode); + + @override + String toString() => 'DuplicateResolveGroupDto[duplicateId=$duplicateId, keepAssetIds=$keepAssetIds, trashAssetIds=$trashAssetIds]'; + + Map toJson() { + final json = {}; + json[r'duplicateId'] = this.duplicateId; + json[r'keepAssetIds'] = this.keepAssetIds; + json[r'trashAssetIds'] = this.trashAssetIds; + return json; + } + + /// Returns a new [DuplicateResolveGroupDto] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static DuplicateResolveGroupDto? fromJson(dynamic value) { + upgradeDto(value, "DuplicateResolveGroupDto"); + if (value is Map) { + final json = value.cast(); + + return DuplicateResolveGroupDto( + duplicateId: mapValueOfType(json, r'duplicateId')!, + keepAssetIds: json[r'keepAssetIds'] is Iterable + ? (json[r'keepAssetIds'] as Iterable).cast().toList(growable: false) + : const [], + trashAssetIds: json[r'trashAssetIds'] is Iterable + ? (json[r'trashAssetIds'] as Iterable).cast().toList(growable: false) + : const [], + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = DuplicateResolveGroupDto.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = DuplicateResolveGroupDto.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of DuplicateResolveGroupDto-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = DuplicateResolveGroupDto.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'duplicateId', + 'keepAssetIds', + 'trashAssetIds', + }; +} + diff --git a/mobile/openapi/lib/model/duplicate_response_dto.dart b/mobile/openapi/lib/model/duplicate_response_dto.dart index 6c85dc8013..f0ddbb4fdd 100644 --- a/mobile/openapi/lib/model/duplicate_response_dto.dart +++ b/mobile/openapi/lib/model/duplicate_response_dto.dart @@ -15,6 +15,7 @@ class DuplicateResponseDto { DuplicateResponseDto({ this.assets = const [], required this.duplicateId, + this.suggestedKeepAssetIds = const [], }); /// Duplicate assets @@ -23,24 +24,30 @@ class DuplicateResponseDto { /// Duplicate group ID String duplicateId; + /// Suggested asset IDs to keep based on file size and EXIF data + List suggestedKeepAssetIds; + @override bool operator ==(Object other) => identical(this, other) || other is DuplicateResponseDto && _deepEquality.equals(other.assets, assets) && - other.duplicateId == duplicateId; + other.duplicateId == duplicateId && + _deepEquality.equals(other.suggestedKeepAssetIds, suggestedKeepAssetIds); @override int get hashCode => // ignore: unnecessary_parenthesis (assets.hashCode) + - (duplicateId.hashCode); + (duplicateId.hashCode) + + (suggestedKeepAssetIds.hashCode); @override - String toString() => 'DuplicateResponseDto[assets=$assets, duplicateId=$duplicateId]'; + String toString() => 'DuplicateResponseDto[assets=$assets, duplicateId=$duplicateId, suggestedKeepAssetIds=$suggestedKeepAssetIds]'; Map toJson() { final json = {}; json[r'assets'] = this.assets; json[r'duplicateId'] = this.duplicateId; + json[r'suggestedKeepAssetIds'] = this.suggestedKeepAssetIds; return json; } @@ -55,6 +62,9 @@ class DuplicateResponseDto { return DuplicateResponseDto( assets: AssetResponseDto.listFromJson(json[r'assets']), duplicateId: mapValueOfType(json, r'duplicateId')!, + suggestedKeepAssetIds: json[r'suggestedKeepAssetIds'] is Iterable + ? (json[r'suggestedKeepAssetIds'] as Iterable).cast().toList(growable: false) + : const [], ); } return null; @@ -104,6 +114,7 @@ class DuplicateResponseDto { static const requiredKeys = { 'assets', 'duplicateId', + 'suggestedKeepAssetIds', }; } diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index f9bfc0639f..19427413b0 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -5285,6 +5285,65 @@ "x-immich-state": "Stable" } }, + "/duplicates/resolve": { + "post": { + "description": "Resolve duplicate groups by synchronizing metadata across assets and deleting/trashing duplicates.", + "operationId": "resolveDuplicates", + "parameters": [], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DuplicateResolveDto" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/BulkIdResponseDto" + }, + "type": "array" + } + } + }, + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "summary": "Resolve duplicate groups", + "tags": [ + "Duplicates" + ], + "x-immich-history": [ + { + "version": "v3.0.0", + "state": "Added" + }, + { + "version": "v3.0.0", + "state": "Alpha" + } + ], + "x-immich-permission": "duplicate.delete", + "x-immich-state": "Alpha" + } + }, "/duplicates/{id}": { "delete": { "description": "Delete a single duplicate asset specified by its ID.", @@ -17299,7 +17358,8 @@ "duplicate", "no_permission", "not_found", - "unknown" + "unknown", + "validation" ], "type": "string" }, @@ -17311,10 +17371,14 @@ "duplicate", "no_permission", "not_found", - "unknown" + "unknown", + "validation" ], "type": "string" }, + "errorMessage": { + "type": "string" + }, "id": { "description": "ID", "type": "string" @@ -17828,6 +17892,52 @@ ], "type": "object" }, + "DuplicateResolveDto": { + "properties": { + "groups": { + "description": "List of duplicate groups to resolve", + "items": { + "$ref": "#/components/schemas/DuplicateResolveGroupDto" + }, + "minItems": 1, + "type": "array" + } + }, + "required": [ + "groups" + ], + "type": "object" + }, + "DuplicateResolveGroupDto": { + "properties": { + "duplicateId": { + "format": "uuid", + "type": "string" + }, + "keepAssetIds": { + "description": "Asset IDs to keep", + "items": { + "format": "uuid", + "type": "string" + }, + "type": "array" + }, + "trashAssetIds": { + "description": "Asset IDs to trash or delete", + "items": { + "format": "uuid", + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "duplicateId", + "keepAssetIds", + "trashAssetIds" + ], + "type": "object" + }, "DuplicateResponseDto": { "properties": { "assets": { @@ -17840,11 +17950,20 @@ "duplicateId": { "description": "Duplicate group ID", "type": "string" + }, + "suggestedKeepAssetIds": { + "description": "Suggested asset IDs to keep based on file size and EXIF data", + "items": { + "format": "uuid", + "type": "string" + }, + "type": "array" } }, "required": [ "assets", - "duplicateId" + "duplicateId", + "suggestedKeepAssetIds" ], "type": "object" }, diff --git a/open-api/typescript-sdk/src/fetch-client.ts b/open-api/typescript-sdk/src/fetch-client.ts index 257bf668d0..fc465ce529 100644 --- a/open-api/typescript-sdk/src/fetch-client.ts +++ b/open-api/typescript-sdk/src/fetch-client.ts @@ -725,6 +725,7 @@ export type BulkIdsDto = { export type BulkIdResponseDto = { /** Error reason if failed */ error?: Error; + errorMessage?: string; /** ID */ id: string; /** Whether operation succeeded */ @@ -1163,6 +1164,19 @@ export type DuplicateResponseDto = { assets: AssetResponseDto[]; /** Duplicate group ID */ duplicateId: string; + /** Suggested asset IDs to keep based on file size and EXIF data */ + suggestedKeepAssetIds: string[]; +}; +export type DuplicateResolveGroupDto = { + duplicateId: string; + /** Asset IDs to keep */ + keepAssetIds: string[]; + /** Asset IDs to trash or delete */ + trashAssetIds: string[]; +}; +export type DuplicateResolveDto = { + /** List of duplicate groups to resolve */ + groups: DuplicateResolveGroupDto[]; }; export type PersonResponseDto = { /** Person date of birth */ @@ -4531,6 +4545,21 @@ export function getAssetDuplicates(opts?: Oazapfts.RequestOpts) { ...opts })); } +/** + * Resolve duplicate groups + */ +export function resolveDuplicates({ duplicateResolveDto }: { + duplicateResolveDto: DuplicateResolveDto; +}, opts?: Oazapfts.RequestOpts) { + return oazapfts.ok(oazapfts.fetchJson<{ + status: 200; + data: BulkIdResponseDto[]; + }>("/duplicates/resolve", oazapfts.json({ + ...opts, + method: "POST", + body: duplicateResolveDto + }))); +} /** * Delete a duplicate */ @@ -6893,13 +6922,15 @@ export enum BulkIdErrorReason { Duplicate = "duplicate", NoPermission = "no_permission", NotFound = "not_found", - Unknown = "unknown" + Unknown = "unknown", + Validation = "validation" } export enum Error { Duplicate = "duplicate", NoPermission = "no_permission", NotFound = "not_found", - Unknown = "unknown" + Unknown = "unknown", + Validation = "validation" } export enum Permission { All = "all", diff --git a/server/src/controllers/duplicate.controller.spec.ts b/server/src/controllers/duplicate.controller.spec.ts new file mode 100644 index 0000000000..66598b9920 --- /dev/null +++ b/server/src/controllers/duplicate.controller.spec.ts @@ -0,0 +1,47 @@ +import { DuplicateController } from 'src/controllers/duplicate.controller'; +import { DuplicateService } from 'src/services/duplicate.service'; +import request from 'supertest'; +import { factory } from 'test/small.factory'; +import { ControllerContext, controllerSetup, mockBaseService } from 'test/utils'; + +describe(DuplicateController.name, () => { + let ctx: ControllerContext; + const service = mockBaseService(DuplicateService); + + beforeAll(async () => { + ctx = await controllerSetup(DuplicateController, [{ provide: DuplicateService, useValue: service }]); + return () => ctx.close(); + }); + + beforeEach(() => { + service.resetAllMocks(); + ctx.reset(); + }); + + describe('GET /duplicates', () => { + it('should be an authenticated route', async () => { + await request(ctx.getHttpServer()).get('/duplicates'); + expect(ctx.authenticate).toHaveBeenCalled(); + }); + }); + + describe('DELETE /duplicates', () => { + it('should be an authenticated route', async () => { + await request(ctx.getHttpServer()).delete('/duplicates'); + expect(ctx.authenticate).toHaveBeenCalled(); + }); + }); + + describe('DELETE /duplicates/:id', () => { + it('should be an authenticated route', async () => { + await request(ctx.getHttpServer()).delete(`/duplicates/${factory.uuid()}`); + expect(ctx.authenticate).toHaveBeenCalled(); + }); + + it('should require a valid uuid', async () => { + const { status, body } = await request(ctx.getHttpServer()).delete(`/duplicates/123`); + expect(status).toBe(400); + expect(body).toEqual(factory.responses.badRequest(['id must be a UUID'])); + }); + }); +}); diff --git a/server/src/controllers/duplicate.controller.ts b/server/src/controllers/duplicate.controller.ts index e8c8e5ef80..0a8c451ed4 100644 --- a/server/src/controllers/duplicate.controller.ts +++ b/server/src/controllers/duplicate.controller.ts @@ -1,9 +1,9 @@ -import { Body, Controller, Delete, Get, HttpCode, HttpStatus, Param } from '@nestjs/common'; +import { Body, Controller, Delete, Get, HttpCode, HttpStatus, Param, Post } from '@nestjs/common'; import { ApiTags } from '@nestjs/swagger'; import { Endpoint, HistoryBuilder } from 'src/decorators'; -import { BulkIdsDto } from 'src/dtos/asset-ids.response.dto'; +import { BulkIdResponseDto, BulkIdsDto } from 'src/dtos/asset-ids.response.dto'; import { AuthDto } from 'src/dtos/auth.dto'; -import { DuplicateResponseDto } from 'src/dtos/duplicate.dto'; +import { DuplicateResolveDto, DuplicateResponseDto } from 'src/dtos/duplicate.dto'; import { ApiTag, Permission } from 'src/enum'; import { Auth, Authenticated } from 'src/middleware/auth.guard'; import { DuplicateService } from 'src/services/duplicate.service'; @@ -48,4 +48,16 @@ export class DuplicateController { deleteDuplicate(@Auth() auth: AuthDto, @Param() { id }: UUIDParamDto): Promise { return this.service.delete(auth, id); } + + @Post('resolve') + @HttpCode(HttpStatus.OK) + @Authenticated({ permission: Permission.DuplicateDelete }) + @Endpoint({ + summary: 'Resolve duplicate groups', + description: 'Resolve duplicate groups by synchronizing metadata across assets and deleting/trashing duplicates.', + history: new HistoryBuilder().added('v3.0.0').alpha('v3.0.0'), + }) + resolveDuplicates(@Auth() auth: AuthDto, @Body() dto: DuplicateResolveDto): Promise { + return this.service.resolve(auth, dto); + } } diff --git a/server/src/dtos/asset-ids.response.dto.ts b/server/src/dtos/asset-ids.response.dto.ts index 427117518d..1065d8485e 100644 --- a/server/src/dtos/asset-ids.response.dto.ts +++ b/server/src/dtos/asset-ids.response.dto.ts @@ -23,6 +23,7 @@ export enum BulkIdErrorReason { NO_PERMISSION = 'no_permission', NOT_FOUND = 'not_found', UNKNOWN = 'unknown', + VALIDATION = 'validation', } export class BulkIdsDto { @@ -37,4 +38,5 @@ export class BulkIdResponseDto { success!: boolean; @ApiPropertyOptional({ description: 'Error reason if failed', enum: BulkIdErrorReason }) error?: BulkIdErrorReason; + errorMessage?: string; } diff --git a/server/src/dtos/duplicate.dto.ts b/server/src/dtos/duplicate.dto.ts index 9cd9147ec5..40b1b74c70 100644 --- a/server/src/dtos/duplicate.dto.ts +++ b/server/src/dtos/duplicate.dto.ts @@ -1,9 +1,35 @@ import { ApiProperty } from '@nestjs/swagger'; +import { Type } from 'class-transformer'; +import { ArrayMinSize, IsArray, ValidateNested } from 'class-validator'; import { AssetResponseDto } from 'src/dtos/asset-response.dto'; +import { ValidateUUID } from 'src/validation'; export class DuplicateResponseDto { @ApiProperty({ description: 'Duplicate group ID' }) duplicateId!: string; @ApiProperty({ description: 'Duplicate assets' }) assets!: AssetResponseDto[]; + + @ValidateUUID({ each: true, description: 'Suggested asset IDs to keep based on file size and EXIF data' }) + suggestedKeepAssetIds!: string[]; +} + +export class DuplicateResolveGroupDto { + @ValidateUUID() + duplicateId!: string; + + @ValidateUUID({ each: true, description: 'Asset IDs to keep' }) + keepAssetIds!: string[]; + + @ValidateUUID({ each: true, description: 'Asset IDs to trash or delete' }) + trashAssetIds!: string[]; +} + +export class DuplicateResolveDto { + @ApiProperty({ description: 'List of duplicate groups to resolve' }) + @ValidateNested({ each: true }) + @IsArray() + @Type(() => DuplicateResolveGroupDto) + @ArrayMinSize(1) + groups!: DuplicateResolveGroupDto[]; } diff --git a/server/src/queries/access.repository.sql b/server/src/queries/access.repository.sql index 1239260dce..810229093b 100644 --- a/server/src/queries/access.repository.sql +++ b/server/src/queries/access.repository.sql @@ -160,6 +160,16 @@ where "session"."userId" = $1 and "session"."id" in ($2) +-- AccessRepository.duplicate.checkOwnerAccess +select + "asset"."duplicateId" +from + "asset" +where + "asset"."duplicateId" in ($1) + and "asset"."ownerId" = $2 + and "asset"."deletedAt" is null + -- AccessRepository.memory.checkOwnerAccess select "memory"."id" diff --git a/server/src/queries/album.repository.sql b/server/src/queries/album.repository.sql index e3d7436c30..cc15260bdb 100644 --- a/server/src/queries/album.repository.sql +++ b/server/src/queries/album.repository.sql @@ -164,6 +164,28 @@ order by "album"."createdAt" desc, "album"."createdAt" desc +-- AlbumRepository.getByAssetIds +select + "album"."id", + "album_asset"."assetId" +from + "album" + inner join "album_asset" on "album_asset"."albumId" = "album"."id" +where + ( + "album"."ownerId" = $1 + or exists ( + select + from + "album_user" + where + "album_user"."albumId" = "album"."id" + and "album_user"."userId" = $2 + ) + ) + and "album_asset"."assetId" in ($3) + and "album"."deletedAt" is null + -- AlbumRepository.getMetadataForIds select "album_asset"."albumId" as "albumId", diff --git a/server/src/queries/duplicate.repository.sql b/server/src/queries/duplicate.repository.sql index 3f718f84c2..24a02e0f23 100644 --- a/server/src/queries/duplicate.repository.sql +++ b/server/src/queries/duplicate.repository.sql @@ -15,7 +15,26 @@ with inner join lateral ( select "asset".*, - "asset_exif" as "exifInfo" + to_json("asset_exif") as "exifInfo", + ( + select + coalesce(json_agg(agg), '[]') + from + ( + select + "tag"."id", + "tag"."value", + "tag"."createdAt", + "tag"."updatedAt", + "tag"."color", + "tag"."parentId" + from + "tag" + inner join "tag_asset" on "tag"."id" = "tag_asset"."tagId" + where + "tag_asset"."assetId" = "asset"."id" + ) as agg + ) as "tags" from "asset_exif" where @@ -29,36 +48,84 @@ with and "asset"."stackId" is null group by "asset"."duplicateId" - ), - "unique" as ( - select - "duplicateId" - from - "duplicates" - where - json_array_length("assets") = $2 - ), - "removed_unique" as ( - update "asset" - set - "duplicateId" = $3 - from - "unique" - where - "asset"."duplicateId" = "unique"."duplicateId" ) select * from "duplicates" where - not exists ( + json_array_length("assets") > $2 + +-- DuplicateRepository.cleanupSingletonGroups +with + "singletons" as ( select + "duplicateId" from - "unique" + "asset" where - "unique"."duplicateId" = "duplicates"."duplicateId" + "ownerId" = $1::uuid + and "duplicateId" is not null + and "deletedAt" is null + and "stackId" is null + group by + "duplicateId" + having + count("id") = $2 ) +update "asset" +set + "duplicateId" = $3 +from + "singletons" +where + "asset"."duplicateId" = "singletons"."duplicateId" + +-- DuplicateRepository.get +select + "asset"."duplicateId", + json_agg( + "asset2" + order by + "asset"."localDateTime" asc + ) as "assets" +from + "asset" + inner join lateral ( + select + "asset".*, + to_json("asset_exif") as "exifInfo", + ( + select + coalesce(json_agg(agg), '[]') + from + ( + select + "tag"."id", + "tag"."value", + "tag"."createdAt", + "tag"."updatedAt", + "tag"."color", + "tag"."parentId" + from + "tag" + inner join "tag_asset" on "tag"."id" = "tag_asset"."tagId" + where + "tag_asset"."assetId" = "asset"."id" + ) as agg + ) as "tags" + from + "asset_exif" + where + "asset_exif"."assetId" = "asset"."id" + ) as "asset2" on true +where + "asset"."visibility" in ('archive', 'timeline') + and "asset"."duplicateId" = $1::uuid + and "asset"."deletedAt" is null + and "asset"."stackId" is null +group by + "asset"."duplicateId" -- DuplicateRepository.delete update "asset" diff --git a/server/src/repositories/access.repository.ts b/server/src/repositories/access.repository.ts index 533e74a311..1661e42c14 100644 --- a/server/src/repositories/access.repository.ts +++ b/server/src/repositories/access.repository.ts @@ -1,5 +1,5 @@ import { Injectable } from '@nestjs/common'; -import { Kysely, sql } from 'kysely'; +import { Kysely, NotNull, sql } from 'kysely'; import { InjectKysely } from 'nestjs-kysely'; import { ChunkedSet, DummyValue, GenerateSql } from 'src/decorators'; import { AlbumUserRole, AssetVisibility } from 'src/enum'; @@ -285,6 +285,28 @@ class AuthDeviceAccess { } } +class DuplicateAccess { + constructor(private db: Kysely) {} + + @GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID_SET] }) + @ChunkedSet({ paramIndex: 1 }) + async checkOwnerAccess(userId: string, duplicateIds: Set) { + if (duplicateIds.size === 0) { + return new Set(); + } + + return this.db + .selectFrom('asset') + .select('asset.duplicateId') + .where('asset.duplicateId', 'in', [...duplicateIds]) + .where('asset.ownerId', '=', userId) + .where('asset.deletedAt', 'is', null) + .$narrowType<{ duplicateId: NotNull }>() + .execute() + .then((assets) => new Set(assets.map((asset) => asset.duplicateId))); + } +} + class NotificationAccess { constructor(private db: Kysely) {} @@ -488,6 +510,7 @@ export class AccessRepository { album: AlbumAccess; asset: AssetAccess; authDevice: AuthDeviceAccess; + duplicate: DuplicateAccess; memory: MemoryAccess; notification: NotificationAccess; person: PersonAccess; @@ -503,6 +526,7 @@ export class AccessRepository { this.album = new AlbumAccess(db); this.asset = new AssetAccess(db); this.authDevice = new AuthDeviceAccess(db); + this.duplicate = new DuplicateAccess(db); this.memory = new MemoryAccess(db); this.notification = new NotificationAccess(db); this.person = new PersonAccess(db); diff --git a/server/src/repositories/album.repository.ts b/server/src/repositories/album.repository.ts index f74356c924..e4d802b93c 100644 --- a/server/src/repositories/album.repository.ts +++ b/server/src/repositories/album.repository.ts @@ -125,6 +125,44 @@ export class AlbumRepository { .execute(); } + @GenerateSql({ params: [DummyValue.UUID, [DummyValue.UUID]] }) + @ChunkedSet({ paramIndex: 1 }) + async getByAssetIds(ownerId: string, assetIds: string[]): Promise> { + if (assetIds.length === 0) { + return new Map(); + } + + const results = await this.db + .selectFrom('album') + .select('album.id') + .innerJoin('album_asset', 'album_asset.albumId', 'album.id') + .where((eb) => + eb.or([ + eb('album.ownerId', '=', ownerId), + eb.exists( + eb + .selectFrom('album_user') + .whereRef('album_user.albumId', '=', 'album.id') + .where('album_user.userId', '=', ownerId), + ), + ]), + ) + .where('album_asset.assetId', 'in', assetIds) + .where('album.deletedAt', 'is', null) + .select('album_asset.assetId') + .execute(); + + // Group by assetId + const map = new Map(); + for (const row of results) { + const existing = map.get(row.assetId) ?? []; + existing.push(row.id); + map.set(row.assetId, existing); + } + + return map; + } + @GenerateSql({ params: [[DummyValue.UUID]] }) @ChunkedArray() async getMetadataForIds(ids: string[]): Promise { @@ -339,7 +377,12 @@ export class AlbumRepository { if (values.length === 0) { return; } - await this.db.insertInto('album_asset').values(values).execute(); + await this.db + .insertInto('album_asset') + .values(values) + // Allow idempotent album sync without failing on existing album memberships. + .onConflict((oc) => oc.columns(['albumId', 'assetId']).doNothing()) + .execute(); } /** diff --git a/server/src/repositories/duplicate.repository.ts b/server/src/repositories/duplicate.repository.ts index 7a5931e029..6a9b4e9082 100644 --- a/server/src/repositories/duplicate.repository.ts +++ b/server/src/repositories/duplicate.repository.ts @@ -1,13 +1,19 @@ import { Injectable } from '@nestjs/common'; import { Kysely, NotNull, Selectable, ShallowDehydrateObject, sql } from 'kysely'; +import { jsonArrayFrom } from 'kysely/helpers/postgres'; import { InjectKysely } from 'nestjs-kysely'; +import { columns } from 'src/database'; import { Chunked, DummyValue, GenerateSql } from 'src/decorators'; +import { MapAsset } from 'src/dtos/asset-response.dto'; import { AssetType, VectorIndex } from 'src/enum'; import { probes } from 'src/repositories/database.repository'; import { DB } from 'src/schema'; import { AssetExifTable } from 'src/schema/tables/asset-exif.table'; import { anyUuid, asUuid, withDefaultVisibility } from 'src/utils/database'; +// Maximum number of candidate duplicates to return from vector search +const DUPLICATE_SEARCH_LIMIT = 64; + interface DuplicateSearch { assetId: string; embedding: string; @@ -34,20 +40,39 @@ export class DuplicateRepository { qb .selectFrom('asset') .$call(withDefaultVisibility) + // Use innerJoinLateral to build a composite object per asset that includes + // exifInfo and tags. This "asset2" object is then aggregated via jsonAgg. + // Tags must be included here (not via separate joins) so they appear in the + // final MapAsset[] output - needed for tag synchronization during resolution. .innerJoinLateral( (qb) => qb .selectFrom('asset_exif') .selectAll('asset') .select((eb) => - eb.table('asset_exif').$castTo>>().as('exifInfo'), + eb.fn + .toJson('asset_exif') + .$castTo>>() + .as('exifInfo'), + ) + + .select((eb) => + jsonArrayFrom( + eb + .selectFrom('tag') + .select(columns.tag) + .innerJoin('tag_asset', 'tag.id', 'tag_asset.tagId') + .whereRef('tag_asset.assetId', '=', 'asset.id'), + ).as('tags'), ) .whereRef('asset_exif.assetId', '=', 'asset.id') .as('asset2'), (join) => join.onTrue(), ) .select('asset.duplicateId') - .select((eb) => eb.fn.jsonAgg('asset2').orderBy('asset.localDateTime', 'asc').as('assets')) + .select((eb) => + eb.fn.jsonAgg('asset2').orderBy('asset.localDateTime', 'asc').$castTo().as('assets'), + ) .where('asset.ownerId', '=', asUuid(userId)) .where('asset.duplicateId', 'is not', null) .$narrowType<{ duplicateId: NotNull }>() @@ -55,29 +80,80 @@ export class DuplicateRepository { .where('asset.stackId', 'is', null) .groupBy('asset.duplicateId'), ) - .with('unique', (qb) => - qb - .selectFrom('duplicates') - .select('duplicateId') - .where((eb) => eb(eb.fn('json_array_length', ['assets']), '=', 1)), - ) - .with('removed_unique', (qb) => - qb - .updateTable('asset') - .set({ duplicateId: null }) - .from('unique') - .whereRef('asset.duplicateId', '=', 'unique.duplicateId'), - ) .selectFrom('duplicates') .selectAll() - // TODO: compare with filtering by json_array_length > 1 - .where(({ not, exists }) => - not(exists((eb) => eb.selectFrom('unique').whereRef('unique.duplicateId', '=', 'duplicates.duplicateId'))), - ) + // Filter out singleton groups (only 1 asset) directly in the query + .where((eb) => eb(eb.fn('json_array_length', ['assets']), '>', 1)) .execute() ); } + @GenerateSql({ params: [DummyValue.UUID] }) + async cleanupSingletonGroups(userId: string): Promise { + // Remove duplicateId from assets that are the only member of their duplicate group + await this.db + .with('singletons', (qb) => + qb + .selectFrom('asset') + .select('duplicateId') + .where('ownerId', '=', asUuid(userId)) + .where('duplicateId', 'is not', null) + .$narrowType<{ duplicateId: NotNull }>() + .where('deletedAt', 'is', null) + .where('stackId', 'is', null) + .groupBy('duplicateId') + .having((eb) => eb.fn.count('id'), '=', 1), + ) + .updateTable('asset') + .set({ duplicateId: null }) + .from('singletons') + .whereRef('asset.duplicateId', '=', 'singletons.duplicateId') + .execute(); + } + + @GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID] }) + async get(duplicateId: string): Promise<{ duplicateId: string; assets: MapAsset[] } | undefined> { + const result = await this.db + .selectFrom('asset') + .$call(withDefaultVisibility) + // Use innerJoinLateral to build a composite object per asset that includes + // exifInfo and tags. This "asset2" object is then aggregated via jsonAgg. + // Tags must be included here (not via separate joins) so they appear in the + // final MapAsset[] output - needed for tag synchronization during resolution. + .innerJoinLateral( + (qb) => + qb + .selectFrom('asset_exif') + .selectAll('asset') + .select((eb) => eb.fn.toJson('asset_exif').as('exifInfo')) + .select((eb) => + jsonArrayFrom( + eb + .selectFrom('tag') + .select(columns.tag) + .innerJoin('tag_asset', 'tag.id', 'tag_asset.tagId') + .whereRef('tag_asset.assetId', '=', 'asset.id'), + ).as('tags'), + ) + .whereRef('asset_exif.assetId', '=', 'asset.id') + .as('asset2'), + (join) => join.onTrue(), + ) + .select('asset.duplicateId') + .select((eb) => eb.fn.jsonAgg('asset2').orderBy('asset.localDateTime', 'asc').$castTo().as('assets')) + .where('asset.duplicateId', '=', asUuid(duplicateId)) + .where('asset.deletedAt', 'is', null) + .where('asset.stackId', 'is', null) + .groupBy('asset.duplicateId') + .executeTakeFirst(); + + if (!result || !result.duplicateId) { + return; + } + + return { duplicateId: result.duplicateId, assets: result.assets }; + } + @GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID] }) async delete(userId: string, id: string): Promise { await this.db @@ -134,7 +210,7 @@ export class DuplicateRepository { .where('asset.id', '!=', asUuid(assetId)) .where('asset.stackId', 'is', null) .orderBy('distance') - .limit(64), + .limit(DUPLICATE_SEARCH_LIMIT), ) .selectFrom('cte') .selectAll() diff --git a/server/src/services/duplicate.service.spec.ts b/server/src/services/duplicate.service.spec.ts index 38c6833105..564cffa0bc 100644 --- a/server/src/services/duplicate.service.spec.ts +++ b/server/src/services/duplicate.service.spec.ts @@ -1,12 +1,13 @@ +import { BulkIdErrorReason } from 'src/dtos/asset-ids.response.dto'; +import { MapAsset } from 'src/dtos/asset-response.dto'; import { AssetType, AssetVisibility, JobName, JobStatus } from 'src/enum'; import { DuplicateService } from 'src/services/duplicate.service'; -import { SearchService } from 'src/services/search.service'; import { AssetFactory } from 'test/factories/asset.factory'; import { authStub } from 'test/fixtures/auth.stub'; import { getForDuplicate } from 'test/mappers'; import { newUuid } from 'test/small.factory'; import { makeStream, newTestService, ServiceMocks } from 'test/utils'; -import { beforeEach, vitest } from 'vitest'; +import { beforeEach, describe, expect, it, vitest } from 'vitest'; vitest.useFakeTimers(); @@ -26,7 +27,7 @@ const hasDupe = { duplicateId: 'duplicate-id', }; -describe(SearchService.name, () => { +describe(DuplicateService.name, () => { let sut: DuplicateService; let mocks: ServiceMocks; @@ -41,6 +42,8 @@ describe(SearchService.name, () => { describe('getDuplicates', () => { it('should get duplicates', async () => { const asset = AssetFactory.from().exif().build(); + mocks.access.duplicate.checkOwnerAccess.mockResolvedValue(new Set(['duplicate-id'])); + mocks.duplicateRepository.cleanupSingletonGroups.mockResolvedValue(); mocks.duplicateRepository.getAll.mockResolvedValue([ { duplicateId: 'duplicate-id', @@ -51,9 +54,24 @@ describe(SearchService.name, () => { { duplicateId: 'duplicate-id', assets: [expect.objectContaining({ id: asset.id }), expect.objectContaining({ id: asset.id })], + suggestedKeepAssetIds: [asset.id], }, ]); }); + + it('should return suggestedKeepAssetIds based on file size', async () => { + const smallAsset = AssetFactory.from().exif({ fileSizeInByte: 1000 }).build(); + const largeAsset = AssetFactory.from().exif({ fileSizeInByte: 5000 }).build(); + mocks.duplicateRepository.cleanupSingletonGroups.mockResolvedValue(); + mocks.duplicateRepository.getAll.mockResolvedValue([ + { + duplicateId: 'duplicate-id', + assets: [getForDuplicate(smallAsset), getForDuplicate(largeAsset)], + }, + ]); + const result = await sut.getDuplicates(authStub.admin); + expect(result[0].suggestedKeepAssetIds).toEqual([largeAsset.id]); + }); }); describe('handleQueueSearchDuplicates', () => { @@ -131,6 +149,200 @@ describe(SearchService.name, () => { }); }); + describe('resolve', () => { + it('should handle mixed success and failure', async () => { + const asset = AssetFactory.create(); + mocks.access.duplicate.checkOwnerAccess.mockResolvedValue(new Set(['group-1', 'group-2'])); + mocks.duplicateRepository.get.mockResolvedValueOnce(void 0); + mocks.duplicateRepository.get.mockResolvedValueOnce({ + duplicateId: 'group-2', + assets: [asset as unknown as MapAsset], + }); + + await expect( + sut.resolve(authStub.admin, { + groups: [ + { duplicateId: 'group-1', keepAssetIds: [], trashAssetIds: [] }, + { duplicateId: 'group-2', keepAssetIds: [asset.id], trashAssetIds: [] }, + ], + }), + ).resolves.toEqual([ + { id: 'group-1', success: false, error: BulkIdErrorReason.NOT_FOUND }, + { id: 'group-2', success: true }, + ]); + }); + + it('should catch and report errors', async () => { + mocks.access.duplicate.checkOwnerAccess.mockResolvedValue(new Set(['group-1'])); + mocks.duplicateRepository.get.mockRejectedValue(new Error('Database error')); + + await expect( + sut.resolve(authStub.admin, { + groups: [{ duplicateId: 'group-1', keepAssetIds: [], trashAssetIds: [] }], + }), + ).resolves.toEqual([{ id: 'group-1', success: false, error: BulkIdErrorReason.UNKNOWN }]); + }); + }); + + describe('resolveGroup (via resolve)', () => { + it('should fail if duplicate group not found', async () => { + mocks.access.duplicate.checkOwnerAccess.mockResolvedValue(new Set(['missing-id'])); + mocks.duplicateRepository.get.mockResolvedValue(void 0); + + await expect( + sut.resolve(authStub.admin, { + groups: [{ duplicateId: 'missing-id', keepAssetIds: [], trashAssetIds: [] }], + }), + ).resolves.toEqual([ + { + id: 'missing-id', + success: false, + error: BulkIdErrorReason.NOT_FOUND, + }, + ]); + }); + + it('should skip when keepAssetIds contains non-member', async () => { + const asset = AssetFactory.create(); + mocks.access.duplicate.checkOwnerAccess.mockResolvedValue(new Set(['group-1'])); + mocks.duplicateRepository.get.mockResolvedValue({ + duplicateId: 'group-1', + assets: [asset as unknown as MapAsset], + }); + + await expect( + sut.resolve(authStub.admin, { + groups: [{ duplicateId: 'group-1', keepAssetIds: ['asset-999', asset.id], trashAssetIds: [] }], + }), + ).resolves.toEqual([{ id: 'group-1', success: true }]); + }); + + it('should skip when trashAssetIds contains non-member', async () => { + const asset = AssetFactory.create(); + mocks.access.duplicate.checkOwnerAccess.mockResolvedValue(new Set(['group-1'])); + mocks.duplicateRepository.get.mockResolvedValue({ + duplicateId: 'group-1', + assets: [asset as unknown as MapAsset], + }); + + await expect( + sut.resolve(authStub.admin, { + groups: [{ duplicateId: 'group-1', keepAssetIds: [asset.id], trashAssetIds: ['asset-999'] }], + }), + ).resolves.toEqual([{ id: 'group-1', success: true }]); + }); + + it('should fail if keepAssetIds and trashAssetIds overlap', async () => { + const asset1 = AssetFactory.create(); + const asset2 = AssetFactory.create(); + mocks.access.duplicate.checkOwnerAccess.mockResolvedValue(new Set(['group-1'])); + mocks.duplicateRepository.get.mockResolvedValue({ + duplicateId: 'group-1', + assets: [asset1 as unknown as MapAsset, asset2 as unknown as MapAsset], + }); + + const result = await sut.resolve(authStub.admin, { + groups: [{ duplicateId: 'group-1', keepAssetIds: [asset1.id], trashAssetIds: [asset1.id] }], + }); + + expect(result[0].success).toBe(false); + expect(result[0].errorMessage).toContain('An asset cannot be in both keepAssetIds and trashAssetIds'); + }); + + it('should fail if keepAssetIds and trashAssetIds do not cover all assets', async () => { + const asset1 = AssetFactory.create(); + const asset2 = AssetFactory.create(); + const asset3 = AssetFactory.create(); + mocks.access.duplicate.checkOwnerAccess.mockResolvedValue(new Set(['group-1'])); + mocks.duplicateRepository.get.mockResolvedValue({ + duplicateId: 'group-1', + assets: [asset1 as unknown as MapAsset, asset2 as unknown as MapAsset, asset3 as unknown as MapAsset], + }); + + const result = await sut.resolve(authStub.admin, { + groups: [{ duplicateId: 'group-1', keepAssetIds: [asset1.id], trashAssetIds: [asset2.id] }], + }); + + expect(result[0].success).toBe(false); + expect(result[0].errorMessage).toContain('Every asset must be in either keepAssetIds or trashAssetIds'); + }); + + it('should fail if partial trash without keepers', async () => { + const asset1 = AssetFactory.create(); + const asset2 = AssetFactory.create(); + mocks.access.duplicate.checkOwnerAccess.mockResolvedValue(new Set(['group-1'])); + mocks.duplicateRepository.get.mockResolvedValue({ + duplicateId: 'group-1', + assets: [asset1 as unknown as MapAsset, asset2 as unknown as MapAsset], + }); + + const result = await sut.resolve(authStub.admin, { + groups: [{ duplicateId: 'group-1', keepAssetIds: [], trashAssetIds: [asset1.id] }], + }); + + expect(result[0].success).toBe(false); + expect(result[0].errorMessage).toContain('Every asset must be in either keepAssetIds or trashAssetIds'); + }); + + it('should sync merged tags to asset_exif.tags', async () => { + const asset1 = AssetFactory.create(); + const asset2 = AssetFactory.create(); + mocks.access.duplicate.checkOwnerAccess.mockResolvedValue(new Set(['group-1'])); + mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-2'])); + mocks.access.tag.checkOwnerAccess.mockResolvedValue(new Set(['tag-1', 'tag-2'])); + mocks.duplicateRepository.get.mockResolvedValue({ + duplicateId: 'group-1', + assets: [ + { + ...asset1, + tags: [ + { + id: 'tag-1', + value: 'Work', + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + parentId: null, + color: null, + }, + ], + }, + { + ...asset2, + tags: [ + { + id: 'tag-2', + value: 'Travel', + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + parentId: null, + color: null, + }, + ], + }, + ] as any, + }); + + const result = await sut.resolve(authStub.admin, { + groups: [{ duplicateId: 'group-1', keepAssetIds: [asset1.id], trashAssetIds: [asset2.id] }], + }); + + expect(result[0].success).toBe(true); + + // Verify tags were applied to tag_asset table + expect(mocks.tag.replaceAssetTags).toHaveBeenCalledWith(asset1.id, ['tag-1', 'tag-2']); + + // Verify merged tag values were written to asset_exif.tags so SidecarWrite preserves them + expect(mocks.asset.updateAllExif).toHaveBeenCalledWith([asset1.id], { tags: ['Work', 'Travel'] }); + + // Verify SidecarWrite was queued (to write tags to sidecar) + expect(mocks.job.queueAll).toHaveBeenCalledWith([{ name: JobName.SidecarWrite, data: { id: asset1.id } }]); + }); + + // NOTE: The following integration-style tests are covered by E2E tests instead + // to avoid complex mock setup. The validation and error-handling logic above + // is thoroughly unit tested. + }); + describe('handleSearchDuplicates', () => { beforeEach(() => { mocks.systemMetadata.get.mockResolvedValue({ diff --git a/server/src/services/duplicate.service.ts b/server/src/services/duplicate.service.ts index 618754ff74..39123e031c 100644 --- a/server/src/services/duplicate.service.ts +++ b/server/src/services/duplicate.service.ts @@ -1,24 +1,84 @@ import { Injectable } from '@nestjs/common'; import { JOBS_ASSET_PAGINATION_SIZE } from 'src/constants'; import { OnJob } from 'src/decorators'; -import { BulkIdsDto } from 'src/dtos/asset-ids.response.dto'; -import { mapAsset } from 'src/dtos/asset-response.dto'; +import { BulkIdErrorReason, BulkIdResponseDto, BulkIdsDto } from 'src/dtos/asset-ids.response.dto'; +import { MapAsset, mapAsset } from 'src/dtos/asset-response.dto'; import { AuthDto } from 'src/dtos/auth.dto'; -import { DuplicateResponseDto } from 'src/dtos/duplicate.dto'; -import { AssetVisibility, JobName, JobStatus, QueueName } from 'src/enum'; +import { DuplicateResolveDto, DuplicateResolveGroupDto, DuplicateResponseDto } from 'src/dtos/duplicate.dto'; +import { AssetStatus, AssetVisibility, JobName, JobStatus, Permission, QueueName } from 'src/enum'; import { AssetDuplicateResult } from 'src/repositories/search.repository'; import { BaseService } from 'src/services/base.service'; import { JobItem, JobOf } from 'src/types'; +import { suggestDuplicateKeepAssetIds } from 'src/utils/duplicate'; import { isDuplicateDetectionEnabled } from 'src/utils/misc'; +type ResolveRequest = { + assetUpdate: { + isFavorite?: boolean; + visibility?: AssetVisibility; + }; + + exifUpdate: { + rating?: number; + latitude?: number; + longitude?: number; + description?: string; + }; + + mergedAlbumIds: string[]; + + mergedTagIds: string[]; + + mergedTagValues: string[]; +}; + +const uniqueNonEmptyLines = (values: Array): string[] => { + const unique = new Set(); + const lines: string[] = []; + for (const value of values) { + if (!value) { + continue; + } + for (const line of value.split(/\r?\n/)) { + const trimmed = line.trim(); + if (!trimmed || unique.has(trimmed)) { + continue; + } + unique.add(trimmed); + lines.push(trimmed); + } + } + return lines; +}; + +const getUniqueCoordinate = (assets: MapAsset[], key: 'latitude' | 'longitude'): number | null => { + const values = assets + .map((asset) => asset.exifInfo?.[key]) + .filter((value): value is number => Number.isFinite(value)); + + if (values.length === 0) { + return null; + } + + const unique = new Set(values); + return unique.size === 1 ? [...unique][0] : null; +}; + @Injectable() export class DuplicateService extends BaseService { async getDuplicates(auth: AuthDto): Promise { + // Clean up singleton groups (assets that are the only member of their duplicate group) + await this.duplicateRepository.cleanupSingletonGroups(auth.user.id); + const duplicates = await this.duplicateRepository.getAll(auth.user.id); - return duplicates.map(({ duplicateId, assets }) => ({ - duplicateId, - assets: assets.map((asset) => mapAsset(asset, { auth })), - })); + return duplicates.map(({ duplicateId, assets }) => { + const mappedAssets = assets.map((asset) => mapAsset(asset, { auth })); + return { + duplicateId, + assets: mappedAssets, + suggestedKeepAssetIds: suggestDuplicateKeepAssetIds(mappedAssets), + }; + }); } async delete(auth: AuthDto, id: string): Promise { @@ -29,6 +89,213 @@ export class DuplicateService extends BaseService { await this.duplicateRepository.deleteAll(auth.user.id, dto.ids); } + async resolve(auth: AuthDto, dto: DuplicateResolveDto) { + const duplicateIds = dto.groups.map(({ duplicateId }) => duplicateId); + + await this.requireAccess({ auth, permission: Permission.DuplicateDelete, ids: duplicateIds }); + + const results: BulkIdResponseDto[] = []; + + for (const group of dto.groups) { + try { + results.push(await this.resolveGroup(auth, group)); + } catch (error: Error | any) { + this.logger.error(`Error resolving duplicate group ${group.duplicateId}: ${error}`, error?.stack); + results.push({ id: group.duplicateId, success: false, error: BulkIdErrorReason.UNKNOWN }); + } + } + + return results; + } + + private async resolveGroup(auth: AuthDto, group: DuplicateResolveGroupDto): Promise { + const { duplicateId, keepAssetIds, trashAssetIds } = group; + + const duplicateGroup = await this.duplicateRepository.get(duplicateId); + if (!duplicateGroup) { + return { id: duplicateId, success: false, error: BulkIdErrorReason.NOT_FOUND }; + } + + const groupAssetIds = new Set(duplicateGroup.assets.map((a) => a.id)); + + // ignore/skip asset IDs not in the group + const idsToKeep = keepAssetIds.filter((id) => groupAssetIds.has(id)); + const idsToTrash = trashAssetIds.filter((id) => groupAssetIds.has(id)); + + for (const assetId of groupAssetIds) { + if (idsToKeep.includes(assetId) && idsToTrash.includes(assetId)) { + return { + id: duplicateId, + success: false, + error: BulkIdErrorReason.VALIDATION, + errorMessage: 'An asset cannot be in both keepAssetIds and trashAssetIds', + }; + } + + if (!idsToKeep.includes(assetId) && !idsToTrash.includes(assetId)) { + return { + id: duplicateId, + success: false, + error: BulkIdErrorReason.VALIDATION, + errorMessage: 'Every asset must be in either keepAssetIds or trashAssetIds', + }; + } + } + + if (idsToTrash.length > 0) { + const ids = await this.checkAccess({ auth, permission: Permission.AssetDelete, ids: idsToTrash }); + if (ids.size !== idsToTrash.length) { + return { + id: duplicateId, + success: false, + error: BulkIdErrorReason.NO_PERMISSION, + errorMessage: 'No permission to delete assets', + }; + } + } + + const assetAlbumMap = await this.albumRepository.getByAssetIds(auth.user.id, [...groupAssetIds]); + + const { assetUpdate, exifUpdate, mergedAlbumIds, mergedTagIds, mergedTagValues } = this.getSyncMergeResult( + duplicateGroup.assets, + assetAlbumMap, + ); + + if (mergedAlbumIds.length > 0) { + const allowedAlbumIds = await this.checkAccess({ + auth, + permission: Permission.AlbumAssetCreate, + ids: mergedAlbumIds, + }); + + const allowedShareIds = await this.checkAccess({ + auth, + permission: Permission.AssetShare, + ids: idsToKeep, + }); + + if (allowedAlbumIds.size > 0 && allowedShareIds.size > 0) { + await this.albumRepository.addAssetIdsToAlbums( + [...allowedAlbumIds].flatMap((albumId) => [...allowedShareIds].map((assetId) => ({ albumId, assetId }))), + ); + } + } + + if (mergedTagIds.length > 0) { + const allowedTagIds = await this.checkAccess({ + auth, + permission: Permission.TagAsset, + ids: mergedTagIds, + }); + + if (allowedTagIds.size > 0) { + // Replace tags for each keeper asset to ensure all merged tags are applied + await Promise.all(idsToKeep.map((assetId) => this.tagRepository.replaceAssetTags(assetId, [...allowedTagIds]))); + + // Update asset_exif.tags so the subsequent SidecarWrite + MetadataExtraction + // cycle preserves the merged tags (updateAllExif locks the property automatically) + await this.assetRepository.updateAllExif(idsToKeep, { tags: mergedTagValues }); + } + } + + if (idsToKeep.length > 0) { + const hasExifUpdate = Object.keys(exifUpdate).length > 0; + const hasTagUpdate = mergedTagIds.length > 0; + + if (hasExifUpdate) { + await this.assetRepository.updateAllExif(idsToKeep, exifUpdate); + } + + if (hasExifUpdate || hasTagUpdate) { + await this.jobRepository.queueAll(idsToKeep.map((id) => ({ name: JobName.SidecarWrite, data: { id } }))); + } + + await this.assetRepository.updateAll(idsToKeep, { duplicateId: null, ...assetUpdate }); + } + + if (idsToTrash.length > 0) { + // TODO: this is duplicated with AssetService.deleteAssets + const { trash } = await this.getConfig({ withCache: true }); + const force = !trash.enabled; + + await this.assetRepository.updateAll(idsToTrash, { + deletedAt: new Date(), + status: force ? AssetStatus.Deleted : AssetStatus.Trashed, + duplicateId: null, + }); + + await this.eventRepository.emit(force ? 'AssetDeleteAll' : 'AssetTrashAll', { + assetIds: idsToTrash, + userId: auth.user.id, + }); + } + + return { id: duplicateId, success: true }; + } + + private getSyncMergeResult(assets: MapAsset[], assetAlbumMap: Map = new Map()): ResolveRequest { + const response: ResolveRequest = { + mergedAlbumIds: [], + mergedTagIds: [], + mergedTagValues: [], + assetUpdate: {}, + exifUpdate: {}, + }; + + response.assetUpdate.isFavorite = assets.some((asset) => asset.isFavorite); + + const visibilityOrder = [AssetVisibility.Locked, AssetVisibility.Archive, AssetVisibility.Timeline]; + let visibility = visibilityOrder.find((level) => assets.some((asset) => asset.visibility === level)); + if (!visibility && assets.some((asset) => asset.visibility === AssetVisibility.Hidden)) { + visibility = AssetVisibility.Hidden; + } + if (visibility) { + response.assetUpdate.visibility = visibility; + } + + let rating = 0; + for (const asset of assets) { + const assetRating = asset.exifInfo?.rating ?? 0; + if (assetRating > rating) { + rating = assetRating; + } + } + if (rating > 0) { + response.exifUpdate.rating = rating; + } + + const descriptionLines = uniqueNonEmptyLines(assets.map((asset) => asset.exifInfo?.description)); + const description = descriptionLines.length > 0 ? descriptionLines.join('\n') : null; + if (description !== null) { + response.exifUpdate.description = description; + } + + const latitude = getUniqueCoordinate(assets, 'latitude'); + const longitude = getUniqueCoordinate(assets, 'longitude'); + if (latitude !== null && longitude !== null) { + response.exifUpdate.latitude = latitude; + response.exifUpdate.longitude = longitude; + } + + const albumIdSet = new Set(); + for (const [, albumIds] of assetAlbumMap) { + for (const albumId of albumIds) { + albumIdSet.add(albumId); + } + } + response.mergedAlbumIds = [...albumIdSet]; + + const allTags = assets.flatMap((asset) => asset.tags ?? []); + const tagIds = [...new Set(allTags.map((tag) => tag.id).filter((id): id is string => !!id))]; + const tagValues = [...new Set(allTags.map((tag) => tag.value).filter((v): v is string => !!v))]; + if (tagIds.length > 0) { + response.mergedTagIds = tagIds; + response.mergedTagValues = tagValues; + } + + return response; + } + @OnJob({ name: JobName.AssetDetectDuplicatesQueueAll, queue: QueueName.DuplicateDetection }) async handleQueueSearchDuplicates({ force }: JobOf): Promise { const { machineLearning } = await this.getConfig({ withCache: false }); diff --git a/server/src/utils/access.ts b/server/src/utils/access.ts index 2e0f7d10d0..21e8bdd66e 100644 --- a/server/src/utils/access.ts +++ b/server/src/utils/access.ts @@ -241,6 +241,11 @@ const checkOtherAccess = async (access: AccessRepository, request: OtherAccessRe return ids.has(auth.user.id) ? new Set([auth.user.id]) : new Set(); } + case Permission.DuplicateRead: + case Permission.DuplicateDelete: { + return access.duplicate.checkOwnerAccess(auth.user.id, ids); + } + case Permission.AuthDeviceDelete: { return await access.authDevice.checkOwnerAccess(auth.user.id, ids); } diff --git a/server/src/utils/duplicate.spec.ts b/server/src/utils/duplicate.spec.ts new file mode 100644 index 0000000000..4c5d5ddfc4 --- /dev/null +++ b/server/src/utils/duplicate.spec.ts @@ -0,0 +1,178 @@ +import { AssetResponseDto } from 'src/dtos/asset-response.dto'; +import { AssetType, AssetVisibility } from 'src/enum'; +import { getExifCount, suggestDuplicate, suggestDuplicateKeepAssetIds } from 'src/utils/duplicate'; +import { describe, expect, it } from 'vitest'; + +const createAsset = ( + id: string, + fileSizeInByte: number | null = null, + exifFields: Record = {}, +): AssetResponseDto => ({ + id, + type: AssetType.Image, + thumbhash: null, + localDateTime: new Date().toISOString(), + duration: '0:00:00.00000', + hasMetadata: true, + width: 1920, + height: 1080, + createdAt: new Date().toISOString(), + deviceAssetId: 'device-asset-1', + deviceId: 'device-1', + ownerId: 'owner-1', + originalPath: '/path/to/asset', + originalFileName: 'asset.jpg', + fileCreatedAt: new Date().toISOString(), + fileModifiedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + isFavorite: false, + isArchived: false, + isTrashed: false, + isOffline: false, + isEdited: false, + visibility: AssetVisibility.Timeline, + checksum: 'checksum', + exifInfo: + fileSizeInByte !== null || Object.keys(exifFields).length > 0 ? { fileSizeInByte, ...exifFields } : undefined, +}); + +describe('duplicate utils', () => { + describe('getExifCount', () => { + it('should return 0 for asset without exifInfo', () => { + const asset = createAsset('asset-1'); + asset.exifInfo = undefined; + expect(getExifCount(asset)).toBe(0); + }); + + it('should return 0 for empty exifInfo', () => { + const asset = createAsset('asset-1'); + asset.exifInfo = {}; + expect(getExifCount(asset)).toBe(0); + }); + + it('should count all truthy values in exifInfo', () => { + const asset = createAsset('asset-1', 1000, { + make: 'Canon', + model: 'EOS 5D', + dateTimeOriginal: new Date(), + timeZone: 'UTC', + latitude: 40.7128, + longitude: -74.006, + city: 'New York', + state: 'NY', + country: 'USA', + description: 'A photo', + rating: 5, + }); + // fileSizeInByte (1000) + 11 other truthy fields = 12 + expect(getExifCount(asset)).toBe(12); + }); + + it('should not count null or undefined values', () => { + const asset = createAsset('asset-1', 1000, { + make: 'Canon', + model: null, + latitude: undefined, + city: '', + rating: 0, + }); + // fileSizeInByte (1000) + make ('Canon') = 2 truthy values + // model (null), latitude (undefined), city (''), rating (0) are all falsy + expect(getExifCount(asset)).toBe(2); + }); + }); + + describe('suggestDuplicate', () => { + it('should return undefined for empty list', () => { + expect(suggestDuplicate([])).toBeUndefined(); + }); + + it('should return the single asset for list with one asset', () => { + const asset = createAsset('asset-1', 1000); + expect(suggestDuplicate([asset])).toEqual(asset); + }); + + it('should return asset with largest file size', () => { + const small = createAsset('small', 1000); + const large = createAsset('large', 5000); + const medium = createAsset('medium', 3000); + + expect(suggestDuplicate([small, large, medium])?.id).toBe('large'); + expect(suggestDuplicate([large, small, medium])?.id).toBe('large'); + expect(suggestDuplicate([medium, small, large])?.id).toBe('large'); + }); + + it('should use EXIF count as tie-breaker when file sizes are equal', () => { + const lessExif = createAsset('less-exif', 1000, { make: 'Canon' }); + const moreExif = createAsset('more-exif', 1000, { + make: 'Canon', + model: 'EOS 5D', + dateTimeOriginal: new Date(), + city: 'New York', + }); + + expect(suggestDuplicate([lessExif, moreExif])?.id).toBe('more-exif'); + expect(suggestDuplicate([moreExif, lessExif])?.id).toBe('more-exif'); + }); + + it('should handle assets with no exifInfo (treat as 0 file size)', () => { + const noExif = createAsset('no-exif'); + noExif.exifInfo = undefined; + const withExif = createAsset('with-exif', 1000); + + expect(suggestDuplicate([noExif, withExif])?.id).toBe('with-exif'); + }); + + it('should handle assets with exifInfo but no fileSizeInByte', () => { + const noFileSize = createAsset('no-file-size'); + noFileSize.exifInfo = { make: 'Canon', model: 'EOS 5D' }; + const withFileSize = createAsset('with-file-size', 1000); + + expect(suggestDuplicate([noFileSize, withFileSize])?.id).toBe('with-file-size'); + }); + + it('should return last asset when all have same file size and EXIF count', () => { + const asset1 = createAsset('asset-1', 1000, { make: 'Canon' }); + const asset2 = createAsset('asset-2', 1000, { make: 'Nikon' }); + + // Both have same file size (1000) and same EXIF count (2: fileSizeInByte + make) + // Should return the last one in the sorted array + const result = suggestDuplicate([asset1, asset2]); + // Since they're equal, the last one after sorting should be returned + expect(result).toBeDefined(); + expect(['asset-1', 'asset-2']).toContain(result?.id); + }); + + it('should prioritize file size over EXIF count', () => { + const largeWithLessExif = createAsset('large-less-exif', 5000, { make: 'Canon' }); + const smallWithMoreExif = createAsset('small-more-exif', 1000, { + make: 'Canon', + model: 'EOS 5D', + dateTimeOriginal: new Date(), + city: 'New York', + state: 'NY', + country: 'USA', + }); + + expect(suggestDuplicate([largeWithLessExif, smallWithMoreExif])?.id).toBe('large-less-exif'); + }); + }); + + describe('suggestDuplicateKeepAssetIds', () => { + it('should return empty array for empty list', () => { + expect(suggestDuplicateKeepAssetIds([])).toEqual([]); + }); + + it('should return array with single asset ID', () => { + const asset = createAsset('asset-1', 1000); + expect(suggestDuplicateKeepAssetIds([asset])).toEqual(['asset-1']); + }); + + it('should return array with best asset ID', () => { + const small = createAsset('small', 1000); + const large = createAsset('large', 5000); + + expect(suggestDuplicateKeepAssetIds([small, large])).toEqual(['large']); + }); + }); +}); diff --git a/server/src/utils/duplicate.ts b/server/src/utils/duplicate.ts new file mode 100644 index 0000000000..4f6deb2fce --- /dev/null +++ b/server/src/utils/duplicate.ts @@ -0,0 +1,60 @@ +import { AssetResponseDto } from 'src/dtos/asset-response.dto'; + +/** + * Counts all truthy values in the exifInfo object. + * This matches the client implementation in web/src/lib/utils/exif-utils.ts + * + * @param asset Asset with optional exifInfo + * @returns Count of truthy EXIF values + */ +export const getExifCount = (asset: AssetResponseDto): number => { + return Object.values(asset.exifInfo ?? {}).filter(Boolean).length; +}; + +/** + * Suggests the best duplicate asset to keep from a list of duplicates. + * This is a direct port of the client logic from web/src/lib/utils/duplicate-utils.ts + * + * The best asset is determined by the following criteria: + * 1. Largest image file size in bytes + * 2. Largest count of EXIF data (as tie-breaker) + * + * @param assets List of duplicate assets + * @returns The best asset to keep, or undefined if empty list + */ +export const suggestDuplicate = (assets: AssetResponseDto[]): AssetResponseDto | undefined => { + if (assets.length === 0) { + return undefined; + } + + // Sort by file size ascending (smallest first) + let duplicateAssets = [...assets].toSorted( + (a, b) => (a.exifInfo?.fileSizeInByte ?? 0) - (b.exifInfo?.fileSizeInByte ?? 0), + ); + + // Get the largest file size (last element after sorting) + const largestFileSize = duplicateAssets.at(-1)?.exifInfo?.fileSizeInByte ?? 0; + + // Filter to keep only assets with the largest file size + duplicateAssets = duplicateAssets.filter((asset) => (asset.exifInfo?.fileSizeInByte ?? 0) === largestFileSize); + + // If there are multiple assets with the same file size, sort by EXIF count + if (duplicateAssets.length >= 2) { + duplicateAssets = duplicateAssets.toSorted((a, b) => getExifCount(a) - getExifCount(b)); + } + + // Return the last asset (highest EXIF count among highest file size) + return duplicateAssets.at(-1); +}; + +/** + * Suggests the best duplicate asset IDs to keep from a list of duplicates. + * Returns an array with a single asset ID (the best candidate), or empty if no assets. + * + * @param assets List of duplicate assets + * @returns Array of suggested asset IDs to keep (0 or 1 element) + */ +export const suggestDuplicateKeepAssetIds = (assets: AssetResponseDto[]): string[] => { + const suggested = suggestDuplicate(assets); + return suggested ? [suggested.id] : []; +}; diff --git a/server/test/mappers.ts b/server/test/mappers.ts index 3d5b34e9c0..ed2c9431f3 100644 --- a/server/test/mappers.ts +++ b/server/test/mappers.ts @@ -1,4 +1,5 @@ import { Selectable, ShallowDehydrateObject } from 'kysely'; +import { MapAsset } from 'src/dtos/asset-response.dto'; import { AssetEditActionItem } from 'src/dtos/editing.dto'; import { ActivityTable } from 'src/schema/tables/activity.table'; import { AssetTable } from 'src/schema/tables/asset.table'; @@ -205,10 +206,11 @@ export const getForStack = (stack: ReturnType) => ({ })), }); -export const getForDuplicate = (asset: ReturnType) => ({ - ...getDehydrated(asset), - exifInfo: getDehydrated(asset.exifInfo), -}); +export const getForDuplicate = (asset: ReturnType) => + ({ + ...getDehydrated(asset), + exifInfo: getDehydrated(asset.exifInfo), + }) as unknown as MapAsset; export const getForSharedLink = (sharedLink: ReturnType) => ({ ...sharedLink, diff --git a/server/test/repositories/access.repository.mock.ts b/server/test/repositories/access.repository.mock.ts index 208b09c120..f723113bd1 100644 --- a/server/test/repositories/access.repository.mock.ts +++ b/server/test/repositories/access.repository.mock.ts @@ -33,6 +33,10 @@ export const newAccessRepositoryMock = (): IAccessRepositoryMock => { checkOwnerAccess: vitest.fn().mockResolvedValue(new Set()), }, + duplicate: { + checkOwnerAccess: vitest.fn().mockResolvedValue(new Set()), + }, + memory: { checkOwnerAccess: vitest.fn().mockResolvedValue(new Set()), }, diff --git a/web/src/lib/components/LinkToDocs.svelte b/web/src/lib/components/LinkToDocs.svelte new file mode 100644 index 0000000000..604b1ac14b --- /dev/null +++ b/web/src/lib/components/LinkToDocs.svelte @@ -0,0 +1,16 @@ + + + + {#snippet children({ message })} + {message} + {/snippet} + diff --git a/web/src/lib/components/utilities-page/duplicates/duplicates-compare-control.svelte b/web/src/lib/components/utilities-page/duplicates/duplicates-compare-control.svelte index ab19f12079..094b50813b 100644 --- a/web/src/lib/components/utilities-page/duplicates/duplicates-compare-control.svelte +++ b/web/src/lib/components/utilities-page/duplicates/duplicates-compare-control.svelte @@ -6,7 +6,6 @@ import { authManager } from '$lib/managers/auth-manager.svelte'; import { handlePromiseError } from '$lib/utils'; import { getNextAsset, getPreviousAsset } from '$lib/utils/asset-utils'; - import { suggestDuplicate } from '$lib/utils/duplicate-utils'; import { navigate } from '$lib/utils/navigation'; import { getAssetInfo, type AssetResponseDto } from '@immich/sdk'; import { Button } from '@immich/ui'; @@ -17,24 +16,27 @@ interface Props { assets: AssetResponseDto[]; + suggestedKeepAssetIds: string[]; onResolve: (duplicateAssetIds: string[], trashIds: string[]) => void; onStack: (assets: AssetResponseDto[]) => void; } - let { assets, onResolve, onStack }: Props = $props(); + let { assets, suggestedKeepAssetIds, onResolve, onStack }: Props = $props(); // eslint-disable-next-line svelte/no-unnecessary-state-wrap let selectedAssetIds = $state(new SvelteSet()); let trashCount = $derived(assets.length - selectedAssetIds.size); onMount(() => { - const suggestedAsset = suggestDuplicate(assets); - - if (!suggestedAsset) { - selectedAssetIds = new SvelteSet(assets[0].id); + if (suggestedKeepAssetIds.length > 0) { + for (const id of suggestedKeepAssetIds) { + selectedAssetIds.add(id); + } return; } - selectedAssetIds.add(suggestedAsset.id); + if (assets.length > 0) { + selectedAssetIds.add(assets[0].id); + } }); onDestroy(() => { diff --git a/web/src/lib/modals/DuplicatesInformationModal.svelte b/web/src/lib/modals/DuplicatesInformationModal.svelte deleted file mode 100644 index b32165a1ae..0000000000 --- a/web/src/lib/modals/DuplicatesInformationModal.svelte +++ /dev/null @@ -1,22 +0,0 @@ - - - - -
-

{$t('deduplication_info_description')}

-
    -
  1. {$t('deduplication_criteria_1')}
  2. -
  3. {$t('deduplication_criteria_2')}
  4. -
-
-
-
diff --git a/web/src/lib/route.ts b/web/src/lib/route.ts index 5a59ec704c..4cb1965122 100644 --- a/web/src/lib/route.ts +++ b/web/src/lib/route.ts @@ -42,6 +42,12 @@ const asQueryString = ( return items.length === 0 ? '' : `?${items.join('&')}`; }; +const DOCS_BASE = 'https://docs.immich.app'; + +export const Docs = { + duplicates: () => `${DOCS_BASE}/features/duplicates-utility`, +}; + export const Route = { // auth login: (params?: { continue?: string; autoLaunch?: 0 | 1 }) => '/auth/login' + asQueryString(params), diff --git a/web/src/lib/stores/preferences.store.ts b/web/src/lib/stores/preferences.store.ts index e863d5a446..0873337e1f 100644 --- a/web/src/lib/stores/preferences.store.ts +++ b/web/src/lib/stores/preferences.store.ts @@ -49,7 +49,7 @@ const defaultMapSettings = { const persistedObject = (key: string, defaults: T) => persisted(key, defaults, { serializer: { - parse: (text) => ({ ...defaultMapSettings, ...JSON.parse(text ?? null) }), + parse: (text) => ({ ...defaults, ...JSON.parse(text ?? null) }), stringify: JSON.stringify, }, }); diff --git a/web/src/lib/utils/duplicate-utils.spec.ts b/web/src/lib/utils/duplicate-utils.spec.ts deleted file mode 100644 index 4fa427989a..0000000000 --- a/web/src/lib/utils/duplicate-utils.spec.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { suggestDuplicate } from '$lib/utils/duplicate-utils'; -import type { AssetResponseDto } from '@immich/sdk'; - -describe('choosing a duplicate', () => { - it('picks the asset with the largest file size', () => { - const assets = [ - { exifInfo: { fileSizeInByte: 300 } }, - { exifInfo: { fileSizeInByte: 200 } }, - { exifInfo: { fileSizeInByte: 100 } }, - ]; - expect(suggestDuplicate(assets as AssetResponseDto[])).toEqual(assets[0]); - }); - - it('picks the asset with the most exif data if multiple assets have the same file size', () => { - const assets = [ - { exifInfo: { fileSizeInByte: 200, rating: 5, fNumber: 1 } }, - { exifInfo: { fileSizeInByte: 200, rating: 5 } }, - { exifInfo: { fileSizeInByte: 100, rating: 5 } }, - ]; - expect(suggestDuplicate(assets as AssetResponseDto[])).toEqual(assets[0]); - }); - - it('returns undefined for an empty array', () => { - const assets: AssetResponseDto[] = []; - expect(suggestDuplicate(assets)).toBeUndefined(); - }); - - it('handles assets with no exifInfo', () => { - const assets = [{ exifInfo: { fileSizeInByte: 200 } }, {}]; - expect(suggestDuplicate(assets as AssetResponseDto[])).toEqual(assets[0]); - }); - - it('handles assets with exifInfo but no fileSizeInByte', () => { - const assets = [{ exifInfo: { rating: 5, fNumber: 1 } }, { exifInfo: { rating: 5 } }]; - expect(suggestDuplicate(assets as AssetResponseDto[])).toEqual(assets[0]); - }); -}); diff --git a/web/src/lib/utils/duplicate-utils.ts b/web/src/lib/utils/duplicate-utils.ts deleted file mode 100644 index 1c783a3667..0000000000 --- a/web/src/lib/utils/duplicate-utils.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { getExifCount } from '$lib/utils/exif-utils'; -import type { AssetResponseDto } from '@immich/sdk'; -import { sortBy } from 'lodash-es'; - -/** - * Suggests the best duplicate asset to keep from a list of duplicates. - * - * The best asset is determined by the following criteria: - * - Largest image file size in bytes - * - Largest count of exif data - * - * @param assets List of duplicate assets - * @returns The best asset to keep - */ -export const suggestDuplicate = (assets: AssetResponseDto[]): AssetResponseDto | undefined => { - let duplicateAssets = sortBy(assets, (asset) => asset.exifInfo?.fileSizeInByte ?? 0); - - // Update the list to only include assets with the largest file size - duplicateAssets = duplicateAssets.filter( - (asset) => asset.exifInfo?.fileSizeInByte === duplicateAssets.at(-1)?.exifInfo?.fileSizeInByte, - ); - - // If there are multiple assets with the same file size, sort the list by the count of exif data - if (duplicateAssets.length >= 2) { - duplicateAssets = sortBy(duplicateAssets, getExifCount); - } - - // Return the last asset in the list - return duplicateAssets.pop(); -}; diff --git a/web/src/lib/utils/persisted.ts b/web/src/lib/utils/persisted.ts index 73eb4de5db..008352ac8f 100644 --- a/web/src/lib/utils/persisted.ts +++ b/web/src/lib/utils/persisted.ts @@ -46,11 +46,25 @@ type PersistedLocalStorageOptions = { parse(text: string): T; }; valid?: (value: T | unknown) => value is T; + upgrade?: 'merge' | ((value: T) => T); }; +const merge = (defaultValue: T) => { + return (value: T): T => { + if (typeof value === 'object') { + value = { ...defaultValue, ...value } as T; + } + + return value; + }; +}; + +const identity = (value: T): T => value; + export class PersistedLocalStorage extends PersistedBase { constructor(key: string, defaultValue: T, options: PersistedLocalStorageOptions = {}) { const valid = options.valid || (() => true); + const upgrade = options.upgrade === 'merge' ? merge(defaultValue) : identity; const serializer = options.serializer || JSON; super(key, defaultValue, { @@ -69,7 +83,7 @@ export class PersistedLocalStorage extends PersistedBase { return; } - return parsed; + return upgrade(parsed); }, write: (key: string, value: T) => { if (browser) { diff --git a/web/src/routes/(user)/utilities/duplicates/[[photos=photos]]/[[assetId=id]]/+page.svelte b/web/src/routes/(user)/utilities/duplicates/[[photos=photos]]/[[assetId=id]]/+page.svelte index c7d30febc7..148ad08a02 100644 --- a/web/src/routes/(user)/utilities/duplicates/[[photos=photos]]/[[assetId=id]]/+page.svelte +++ b/web/src/routes/(user)/utilities/duplicates/[[photos=photos]]/[[assetId=id]]/+page.svelte @@ -3,24 +3,21 @@ import { page } from '$app/state'; import { shortcuts } from '$lib/actions/shortcut'; import UserPageLayout from '$lib/components/layouts/user-page-layout.svelte'; + import LinkToDocs from '$lib/components/LinkToDocs.svelte'; import DuplicatesCompareControl from '$lib/components/utilities-page/duplicates/duplicates-compare-control.svelte'; import { assetViewerManager } from '$lib/managers/asset-viewer-manager.svelte'; import { featureFlagsManager } from '$lib/managers/feature-flags-manager.svelte'; - import DuplicatesInformationModal from '$lib/modals/DuplicatesInformationModal.svelte'; import ShortcutsModal from '$lib/modals/ShortcutsModal.svelte'; import { Route } from '$lib/route'; import { locale } from '$lib/stores/preferences.store'; - import { stackAssets } from '$lib/utils/asset-utils'; - import { suggestDuplicate } from '$lib/utils/duplicate-utils'; import { handleError } from '$lib/utils/handle-error'; import type { AssetResponseDto } from '@immich/sdk'; - import { deleteAssets, deleteDuplicates, updateAssets } from '@immich/sdk'; + import { createStack, deleteDuplicates, resolveDuplicates, updateAssets } from '@immich/sdk'; import { Button, HStack, IconButton, modalManager, Text, toastManager } from '@immich/ui'; import { mdiCheckOutline, mdiChevronLeft, mdiChevronRight, - mdiInformationOutline, mdiKeyboard, mdiPageFirst, mdiPageLast, @@ -98,34 +95,48 @@ }; const handleResolve = async (duplicateId: string, duplicateAssetIds: string[], trashIds: string[]) => { + const forceDelete = !featureFlagsManager.value.trash; + const shouldConfirmDelete = trashIds.length > 0 && forceDelete; + return withConfirmation( async () => { - await deleteAssets({ assetBulkDeleteDto: { ids: trashIds, force: !featureFlagsManager.value.trash } }); - await updateAssets({ assetBulkUpdateDto: { ids: duplicateAssetIds, duplicateId: null } }); + const keepAssetIds = duplicateAssetIds.filter((id) => !trashIds.includes(id)); + + const response = await resolveDuplicates({ + duplicateResolveDto: { + groups: [{ duplicateId, keepAssetIds, trashAssetIds: trashIds }], + }, + }); + + const { success, error, errorMessage } = response[0]; + if (!success) { + throw new Error(errorMessage || error); + } duplicates = duplicates.filter((duplicate) => duplicate.duplicateId !== duplicateId); deletedNotification(trashIds.length); await navigateToIndex(duplicatesIndex); }, - trashIds.length > 0 && !featureFlagsManager.value.trash ? $t('delete_duplicates_confirmation') : undefined, - trashIds.length > 0 && !featureFlagsManager.value.trash ? $t('permanently_delete') : undefined, + shouldConfirmDelete ? $t('delete_duplicates_confirmation') : undefined, + shouldConfirmDelete ? $t('permanently_delete') : undefined, ); }; const handleStack = async (duplicateId: string, assets: AssetResponseDto[]) => { - await stackAssets(assets, false); - const duplicateAssetIds = assets.map((asset) => asset.id); - await updateAssets({ assetBulkUpdateDto: { ids: duplicateAssetIds, duplicateId: null } }); + const assetIds = assets.map((asset) => asset.id); + await createStack({ stackCreateDto: { assetIds } }); + await updateAssets({ assetBulkUpdateDto: { ids: assetIds, duplicateId: null } }); duplicates = duplicates.filter((duplicate) => duplicate.duplicateId !== duplicateId); await navigateToIndex(duplicatesIndex); }; const handleDeduplicateAll = async () => { - const idsToKeep = duplicates.map((group) => suggestDuplicate(group.assets)).map((asset) => asset?.id); - const idsToDelete = duplicates.flatMap((group, i) => - group.assets.map((asset) => asset.id).filter((asset) => asset !== idsToKeep[i]), - ); + // Use server-provided suggestedKeepAssetIds from each group + const idsToDelete = duplicates.flatMap((group) => { + const keepIds = new Set(group.suggestedKeepAssetIds); + return group.assets.map((asset) => asset.id).filter((id) => !keepIds.has(id)); + }); let prompt, confirmText; if (featureFlagsManager.value.trash) { @@ -138,14 +149,26 @@ return withConfirmation( async () => { - await deleteAssets({ assetBulkDeleteDto: { ids: idsToDelete, force: !featureFlagsManager.value.trash } }); - await updateAssets({ - assetBulkUpdateDto: { - ids: [...idsToDelete, ...idsToKeep.filter((id): id is string => !!id)], - duplicateId: null, + // Resolve all groups in a single batch request + const response = await resolveDuplicates({ + duplicateResolveDto: { + groups: duplicates.map((group) => { + const keepIds = new Set(group.suggestedKeepAssetIds); + return { + duplicateId: group.duplicateId, + keepAssetIds: group.suggestedKeepAssetIds, + trashAssetIds: group.assets.map((asset) => asset.id).filter((id) => !keepIds.has(id)), + }; + }), }, }); + // Count failures and show appropriate message + const failedCount = response.filter(({ success }) => !success).length; + if (failedCount > 0) { + toastManager.danger($t('errors.unable_to_resolve_duplicate')); + } + duplicates = []; deletedNotification(idsToDelete.length); @@ -228,26 +251,16 @@ {/snippet} -
+
{#if duplicates && duplicates.length > 0} -
-
-

{$t('duplicates_description')}

-
- modalManager.show(DuplicatesInformationModal)} - /> -
+ +

{$t('duplicates_description')}

+
{#key duplicates[duplicatesIndex].duplicateId} handleResolve(duplicates[duplicatesIndex].duplicateId, duplicateAssetIds, trashIds)} onStack={(assets) => handleStack(duplicates[duplicatesIndex].duplicateId, assets)}