feat(server): near-duplicate detection (#8228)

* duplicate detection job, entity, config

* queueing

* job panel, update api

* use embedding in db instead of fetching

* disable concurrency

* only queue visible assets

* handle multiple duplicateIds

* update concurrent queue check

* add provider

* add web placeholder, server endpoint, migration, various fixes

* update sql

* select embedding by default

* rename variable

* simplify

* remove separate entity, handle re-running with different threshold, set default back to 0.02

* fix tests

* add tests

* add index to entity

* formatting

* update asset mock

* fix `upsertJobStatus` signature

* update sql

* formatting

* default to 0.03

* optimize clustering

* use asset's `duplicateId` if present

* update sql

* update tests

* expose admin setting

* refactor

* formatting

* skip if ml is disabled

* debug trash e2e

* remove from web

* remove from sidebar

* test if ml is disabled

* update sql

* separate duplicate detection from clip in config, disable by default for now

* fix doc

* lower minimum `maxDistance`

* update api

* Add and Use Duplicate Detection Feature Flag (#9364)

* Add Duplicate Detection Flag

* Use Duplicate Detection Flag

* Attempt Fixes for Failing Checks

* lower minimum `maxDistance`

* fix tests

---------

Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com>

* chore: fixes and additions after rebase

* chore: update api (remove new Role enum)

* fix: left join smart search so getAll works without machine learning

* test: trash e2e go back to checking length of assets is zero

* chore: regen api after rebase

* test: fix tests after rebase

* redundant join

---------

Co-authored-by: Nicholas Flamy <30300649+NicholasFlamy@users.noreply.github.com>
Co-authored-by: Zack Pollard <zackpollard@ymail.com>
Co-authored-by: Zack Pollard <zack@futo.org>

docs/docs/install/config-file.md

@@ -77,6 +77,10 @@ The default configuration looks like this:
       "enabled": true,
       "modelName": "ViT-B-32__openai"
     },
+    "duplicateDetection": {
+      "enabled": false,
+      "maxDistance": 0.03
+    },
     "facialRecognition": {
       "enabled": true,
       "modelName": "buffalo_l",

e2e/src/api/specs/server-info.e2e-spec.ts

@@ -66,6 +66,7 @@ describe('/server-info', () => {
       expect(body).toEqual({
         smartSearch: false,
         configFile: false,
+        duplicateDetection: false,
         facialRecognition: false,
         map: true,
         reverseGeocoding: true,

e2e/src/api/specs/trash.e2e-spec.ts

@@ -32,8 +32,7 @@ describe('/trash', () => {
       await utils.deleteAssets(admin.accessToken, [assetId]);
 
       const before = await getAllAssets({}, { headers: asBearerAuth(admin.accessToken) });
-
+      expect(before).toStrictEqual([expect.objectContaining({ id: assetId, isTrashed: true })]);
-      expect(before.length).toBeGreaterThanOrEqual(1);
 
       const { status } = await request(app).post('/trash/empty').set('Authorization', `Bearer ${admin.accessToken}`);
       expect(status).toBe(204);
@@ -57,14 +56,14 @@ describe('/trash', () => {
       const { id: assetId } = await utils.createAsset(admin.accessToken);
       await utils.deleteAssets(admin.accessToken, [assetId]);
 
-      const before = await utils.getAssetInfo(admin.accessToken, assetId);
+      const before = await getAllAssets({}, { headers: asBearerAuth(admin.accessToken) });
-      expect(before.isTrashed).toBe(true);
+      expect(before).toStrictEqual([expect.objectContaining({ id: assetId, isTrashed: true })]);
 
       const { status } = await request(app).post('/trash/restore').set('Authorization', `Bearer ${admin.accessToken}`);
       expect(status).toBe(204);
 
-      const after = await utils.getAssetInfo(admin.accessToken, assetId);
+      const after = await getAllAssets({}, { headers: asBearerAuth(admin.accessToken) });
-      expect(after.isTrashed).toBe(false);
+      expect(after).toStrictEqual([expect.objectContaining({ id: assetId, isTrashed: false })]);
     });
   });
 

mobile/openapi/.openapi-generator/FILES

@@ -68,6 +68,7 @@ doc/DownloadApi.md
 doc/DownloadArchiveInfo.md
 doc/DownloadInfoDto.md
 doc/DownloadResponseDto.md
+doc/DuplicateDetectionConfig.md
 doc/EntityType.md
 doc/ExifResponseDto.md
 doc/FaceApi.md
@@ -308,6 +309,7 @@ lib/model/delete_user_dto.dart
 lib/model/download_archive_info.dart
 lib/model/download_info_dto.dart
 lib/model/download_response_dto.dart
+lib/model/duplicate_detection_config.dart
 lib/model/entity_type.dart
 lib/model/exif_response_dto.dart
 lib/model/face_dto.dart
@@ -501,6 +503,7 @@ test/download_api_test.dart
 test/download_archive_info_test.dart
 test/download_info_dto_test.dart
 test/download_response_dto_test.dart
+test/duplicate_detection_config_test.dart
 test/entity_type_test.dart
 test/exif_response_dto_test.dart
 test/face_api_test.dart

mobile/openapi/README.md

@@ -98,6 +98,7 @@ Class | Method | HTTP request | Description
 *AssetApi* | [**deleteAssets**](doc//AssetApi.md#deleteassets) | **DELETE** /asset | 
 *AssetApi* | [**getAllAssets**](doc//AssetApi.md#getallassets) | **GET** /asset | 
 *AssetApi* | [**getAllUserAssetsByDeviceId**](doc//AssetApi.md#getalluserassetsbydeviceid) | **GET** /asset/device/{deviceId} | 
+*AssetApi* | [**getAssetDuplicates**](doc//AssetApi.md#getassetduplicates) | **GET** /asset/duplicates | 
 *AssetApi* | [**getAssetInfo**](doc//AssetApi.md#getassetinfo) | **GET** /asset/{id} | 
 *AssetApi* | [**getAssetStatistics**](doc//AssetApi.md#getassetstatistics) | **GET** /asset/statistics | 
 *AssetApi* | [**getAssetThumbnail**](doc//AssetApi.md#getassetthumbnail) | **GET** /asset/thumbnail/{id} | 
@@ -282,6 +283,7 @@ Class | Method | HTTP request | Description
  - [DownloadArchiveInfo](doc//DownloadArchiveInfo.md)
  - [DownloadInfoDto](doc//DownloadInfoDto.md)
  - [DownloadResponseDto](doc//DownloadResponseDto.md)
+ - [DuplicateDetectionConfig](doc//DuplicateDetectionConfig.md)
  - [EntityType](doc//EntityType.md)
  - [ExifResponseDto](doc//ExifResponseDto.md)
  - [FaceDto](doc//FaceDto.md)

mobile/openapi/doc/AllJobStatusResponseDto.md

@@ -9,6 +9,7 @@ import 'package:openapi/api.dart';
 Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
 **backgroundTask** | [**JobStatusDto**](JobStatusDto.md) |  | 
+**duplicateDetection** | [**JobStatusDto**](JobStatusDto.md) |  | 
 **faceDetection** | [**JobStatusDto**](JobStatusDto.md) |  | 
 **facialRecognition** | [**JobStatusDto**](JobStatusDto.md) |  | 
 **library_** | [**JobStatusDto**](JobStatusDto.md) |  | 

mobile/openapi/doc/AssetApi.md

@@ -14,6 +14,7 @@ Method | HTTP request | Description
 [**deleteAssets**](AssetApi.md#deleteassets) | **DELETE** /asset | 
 [**getAllAssets**](AssetApi.md#getallassets) | **GET** /asset | 
 [**getAllUserAssetsByDeviceId**](AssetApi.md#getalluserassetsbydeviceid) | **GET** /asset/device/{deviceId} | 
+[**getAssetDuplicates**](AssetApi.md#getassetduplicates) | **GET** /asset/duplicates | 
 [**getAssetInfo**](AssetApi.md#getassetinfo) | **GET** /asset/{id} | 
 [**getAssetStatistics**](AssetApi.md#getassetstatistics) | **GET** /asset/statistics | 
 [**getAssetThumbnail**](AssetApi.md#getassetthumbnail) | **GET** /asset/thumbnail/{id} | 
@@ -324,6 +325,43 @@ Name | Type | Description  | Notes
 
 [[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md)
 
+# **getAssetDuplicates**
+> List<AssetResponseDto> getAssetDuplicates()
+
+
+
+### Example
+```dart
+import 'package:openapi/api.dart';
+
+final api_instance = AssetApi();
+
+try {
+    final result = api_instance.getAssetDuplicates();
+    print(result);
+} catch (e) {
+    print('Exception when calling AssetApi->getAssetDuplicates: $e\n');
+}
+```
+
+### Parameters
+This endpoint does not need any parameter.
+
+### Return type
+
+[**List<AssetResponseDto>**](AssetResponseDto.md)
+
+### Authorization
+
+No authorization required
+
+### HTTP request headers
+
+ - **Content-Type**: Not defined
+ - **Accept**: application/json
+
+[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md)
+
 # **getAssetInfo**
 > AssetResponseDto getAssetInfo(id, key)
 

mobile/openapi/doc/DuplicateDetectionConfig.md

@@ -0,0 +1,16 @@
+# openapi.model.DuplicateDetectionConfig
+
+## Load the model package
+```dart
+import 'package:openapi/api.dart';
+```
+
+## Properties
+Name | Type | Description | Notes
+------------ | ------------- | ------------- | -------------
+**enabled** | **bool** |  | 
+**maxDistance** | **double** |  | 
+
+[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
+
+

mobile/openapi/doc/ServerFeaturesDto.md

@@ -9,6 +9,7 @@ import 'package:openapi/api.dart';
 Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
 **configFile** | **bool** |  | 
+**duplicateDetection** | **bool** |  | 
 **email** | **bool** |  | 
 **facialRecognition** | **bool** |  | 
 **map** | **bool** |  | 

mobile/openapi/doc/SystemConfigMachineLearningDto.md

@@ -9,6 +9,7 @@ import 'package:openapi/api.dart';
 Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
 **clip** | [**CLIPConfig**](CLIPConfig.md) |  | 
+**duplicateDetection** | [**DuplicateDetectionConfig**](DuplicateDetectionConfig.md) |  | 
 **enabled** | **bool** |  | 
 **facialRecognition** | [**RecognitionConfig**](RecognitionConfig.md) |  | 
 **url** | **String** |  | 

mobile/openapi/lib/api.dart

@@ -114,6 +114,7 @@ part 'model/delete_user_dto.dart';
 part 'model/download_archive_info.dart';
 part 'model/download_info_dto.dart';
 part 'model/download_response_dto.dart';
+part 'model/duplicate_detection_config.dart';
 part 'model/entity_type.dart';
 part 'model/exif_response_dto.dart';
 part 'model/face_dto.dart';

mobile/openapi/lib/api/asset_api.dart

@@ -326,6 +326,50 @@ class AssetApi {
     return null;
   }
 
+  /// Performs an HTTP 'GET /asset/duplicates' operation and returns the [Response].
+  Future<Response> getAssetDuplicatesWithHttpInfo() async {
+    // ignore: prefer_const_declarations
+    final path = r'/asset/duplicates';
+
+    // ignore: prefer_final_locals
+    Object? postBody;
+
+    final queryParams = <QueryParam>[];
+    final headerParams = <String, String>{};
+    final formParams = <String, String>{};
+
+    const contentTypes = <String>[];
+
+
+    return apiClient.invokeAPI(
+      path,
+      'GET',
+      queryParams,
+      postBody,
+      headerParams,
+      formParams,
+      contentTypes.isEmpty ? null : contentTypes.first,
+    );
+  }
+
+  Future<List<AssetResponseDto>?> getAssetDuplicates() async {
+    final response = await getAssetDuplicatesWithHttpInfo();
+    if (response.statusCode >= HttpStatus.badRequest) {
+      throw ApiException(response.statusCode, await _decodeBodyBytes(response));
+    }
+    // When a remote server returns no body with a status of 204, we shall not decode it.
+    // At the time of writing this, `dart:convert` will throw an "Unexpected end of input"
+    // FormatException when trying to decode an empty string.
+    if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) {
+      final responseBody = await _decodeBodyBytes(response);
+      return (await apiClient.deserializeAsync(responseBody, 'List<AssetResponseDto>') as List)
+        .cast<AssetResponseDto>()
+        .toList(growable: false);
+
+    }
+    return null;
+  }
+
   /// Performs an HTTP 'GET /asset/{id}' operation and returns the [Response].
   /// Parameters:
   ///

mobile/openapi/lib/api_client.dart

@@ -298,6 +298,8 @@ class ApiClient {
           return DownloadInfoDto.fromJson(value);
         case 'DownloadResponseDto':
           return DownloadResponseDto.fromJson(value);
+        case 'DuplicateDetectionConfig':
+          return DuplicateDetectionConfig.fromJson(value);
         case 'EntityType':
           return EntityTypeTypeTransformer().decode(value);
         case 'ExifResponseDto':

mobile/openapi/lib/model/all_job_status_response_dto.dart

@@ -14,6 +14,7 @@ class AllJobStatusResponseDto {
   /// Returns a new [AllJobStatusResponseDto] instance.
   AllJobStatusResponseDto({
     required this.backgroundTask,
+    required this.duplicateDetection,
     required this.faceDetection,
     required this.facialRecognition,
     required this.library_,
@@ -30,6 +31,8 @@ class AllJobStatusResponseDto {
 
   JobStatusDto backgroundTask;
 
+  JobStatusDto duplicateDetection;
+
   JobStatusDto faceDetection;
 
   JobStatusDto facialRecognition;
@@ -57,6 +60,7 @@ class AllJobStatusResponseDto {
   @override
   bool operator ==(Object other) => identical(this, other) || other is AllJobStatusResponseDto &&
     other.backgroundTask == backgroundTask &&
+    other.duplicateDetection == duplicateDetection &&
     other.faceDetection == faceDetection &&
     other.facialRecognition == facialRecognition &&
     other.library_ == library_ &&
@@ -74,6 +78,7 @@ class AllJobStatusResponseDto {
   int get hashCode =>
     // ignore: unnecessary_parenthesis
     (backgroundTask.hashCode) +
+    (duplicateDetection.hashCode) +
     (faceDetection.hashCode) +
     (facialRecognition.hashCode) +
     (library_.hashCode) +
@@ -88,11 +93,12 @@ class AllJobStatusResponseDto {
     (videoConversion.hashCode);
 
   @override
-  String toString() => 'AllJobStatusResponseDto[backgroundTask=$backgroundTask, faceDetection=$faceDetection, facialRecognition=$facialRecognition, library_=$library_, metadataExtraction=$metadataExtraction, migration=$migration, notifications=$notifications, search=$search, sidecar=$sidecar, smartSearch=$smartSearch, storageTemplateMigration=$storageTemplateMigration, thumbnailGeneration=$thumbnailGeneration, videoConversion=$videoConversion]';
+  String toString() => 'AllJobStatusResponseDto[backgroundTask=$backgroundTask, duplicateDetection=$duplicateDetection, faceDetection=$faceDetection, facialRecognition=$facialRecognition, library_=$library_, metadataExtraction=$metadataExtraction, migration=$migration, notifications=$notifications, search=$search, sidecar=$sidecar, smartSearch=$smartSearch, storageTemplateMigration=$storageTemplateMigration, thumbnailGeneration=$thumbnailGeneration, videoConversion=$videoConversion]';
 
   Map<String, dynamic> toJson() {
     final json = <String, dynamic>{};
       json[r'backgroundTask'] = this.backgroundTask;
+      json[r'duplicateDetection'] = this.duplicateDetection;
       json[r'faceDetection'] = this.faceDetection;
       json[r'facialRecognition'] = this.facialRecognition;
       json[r'library'] = this.library_;
@@ -117,6 +123,7 @@ class AllJobStatusResponseDto {
 
       return AllJobStatusResponseDto(
         backgroundTask: JobStatusDto.fromJson(json[r'backgroundTask'])!,
+        duplicateDetection: JobStatusDto.fromJson(json[r'duplicateDetection'])!,
         faceDetection: JobStatusDto.fromJson(json[r'faceDetection'])!,
         facialRecognition: JobStatusDto.fromJson(json[r'facialRecognition'])!,
         library_: JobStatusDto.fromJson(json[r'library'])!,
@@ -177,6 +184,7 @@ class AllJobStatusResponseDto {
   /// The list of required keys that must be present in a JSON.
   static const requiredKeys = <String>{
     'backgroundTask',
+    'duplicateDetection',
     'faceDetection',
     'facialRecognition',
     'library',

mobile/openapi/lib/model/duplicate_detection_config.dart

@@ -0,0 +1,108 @@
+//
+// AUTO-GENERATED FILE, DO NOT MODIFY!
+//
+// @dart=2.18
+
+// ignore_for_file: unused_element, unused_import
+// ignore_for_file: always_put_required_named_parameters_first
+// ignore_for_file: constant_identifier_names
+// ignore_for_file: lines_longer_than_80_chars
+
+part of openapi.api;
+
+class DuplicateDetectionConfig {
+  /// Returns a new [DuplicateDetectionConfig] instance.
+  DuplicateDetectionConfig({
+    required this.enabled,
+    required this.maxDistance,
+  });
+
+  bool enabled;
+
+  /// Minimum value: 0.001
+  /// Maximum value: 0.1
+  double maxDistance;
+
+  @override
+  bool operator ==(Object other) => identical(this, other) || other is DuplicateDetectionConfig &&
+    other.enabled == enabled &&
+    other.maxDistance == maxDistance;
+
+  @override
+  int get hashCode =>
+    // ignore: unnecessary_parenthesis
+    (enabled.hashCode) +
+    (maxDistance.hashCode);
+
+  @override
+  String toString() => 'DuplicateDetectionConfig[enabled=$enabled, maxDistance=$maxDistance]';
+
+  Map<String, dynamic> toJson() {
+    final json = <String, dynamic>{};
+      json[r'enabled'] = this.enabled;
+      json[r'maxDistance'] = this.maxDistance;
+    return json;
+  }
+
+  /// Returns a new [DuplicateDetectionConfig] instance and imports its values from
+  /// [value] if it's a [Map], null otherwise.
+  // ignore: prefer_constructors_over_static_methods
+  static DuplicateDetectionConfig? fromJson(dynamic value) {
+    if (value is Map) {
+      final json = value.cast<String, dynamic>();
+
+      return DuplicateDetectionConfig(
+        enabled: mapValueOfType<bool>(json, r'enabled')!,
+        maxDistance: mapValueOfType<double>(json, r'maxDistance')!,
+      );
+    }
+    return null;
+  }
+
+  static List<DuplicateDetectionConfig> listFromJson(dynamic json, {bool growable = false,}) {
+    final result = <DuplicateDetectionConfig>[];
+    if (json is List && json.isNotEmpty) {
+      for (final row in json) {
+        final value = DuplicateDetectionConfig.fromJson(row);
+        if (value != null) {
+          result.add(value);
+        }
+      }
+    }
+    return result.toList(growable: growable);
+  }
+
+  static Map<String, DuplicateDetectionConfig> mapFromJson(dynamic json) {
+    final map = <String, DuplicateDetectionConfig>{};
+    if (json is Map && json.isNotEmpty) {
+      json = json.cast<String, dynamic>(); // ignore: parameter_assignments
+      for (final entry in json.entries) {
+        final value = DuplicateDetectionConfig.fromJson(entry.value);
+        if (value != null) {
+          map[entry.key] = value;
+        }
+      }
+    }
+    return map;
+  }
+
+  // maps a json object with a list of DuplicateDetectionConfig-objects as value to a dart map
+  static Map<String, List<DuplicateDetectionConfig>> mapListFromJson(dynamic json, {bool growable = false,}) {
+    final map = <String, List<DuplicateDetectionConfig>>{};
+    if (json is Map && json.isNotEmpty) {
+      // ignore: parameter_assignments
+      json = json.cast<String, dynamic>();
+      for (final entry in json.entries) {
+        map[entry.key] = DuplicateDetectionConfig.listFromJson(entry.value, growable: growable,);
+      }
+    }
+    return map;
+  }
+
+  /// The list of required keys that must be present in a JSON.
+  static const requiredKeys = <String>{
+    'enabled',
+    'maxDistance',
+  };
+}
+

mobile/openapi/lib/model/job_name.dart

@@ -29,6 +29,7 @@ class JobName {
   static const faceDetection = JobName._(r'faceDetection');
   static const facialRecognition = JobName._(r'facialRecognition');
   static const smartSearch = JobName._(r'smartSearch');
+  static const duplicateDetection = JobName._(r'duplicateDetection');
   static const backgroundTask = JobName._(r'backgroundTask');
   static const storageTemplateMigration = JobName._(r'storageTemplateMigration');
   static const migration = JobName._(r'migration');
@@ -45,6 +46,7 @@ class JobName {
     faceDetection,
     facialRecognition,
     smartSearch,
+    duplicateDetection,
     backgroundTask,
     storageTemplateMigration,
     migration,
@@ -96,6 +98,7 @@ class JobNameTypeTransformer {
         case r'faceDetection': return JobName.faceDetection;
         case r'facialRecognition': return JobName.facialRecognition;
         case r'smartSearch': return JobName.smartSearch;
+        case r'duplicateDetection': return JobName.duplicateDetection;
         case r'backgroundTask': return JobName.backgroundTask;
         case r'storageTemplateMigration': return JobName.storageTemplateMigration;
         case r'migration': return JobName.migration;

mobile/openapi/lib/model/server_features_dto.dart

@@ -14,6 +14,7 @@ class ServerFeaturesDto {
   /// Returns a new [ServerFeaturesDto] instance.
   ServerFeaturesDto({
     required this.configFile,
+    required this.duplicateDetection,
     required this.email,
     required this.facialRecognition,
     required this.map,
@@ -29,6 +30,8 @@ class ServerFeaturesDto {
 
   bool configFile;
 
+  bool duplicateDetection;
+
   bool email;
 
   bool facialRecognition;
@@ -54,6 +57,7 @@ class ServerFeaturesDto {
   @override
   bool operator ==(Object other) => identical(this, other) || other is ServerFeaturesDto &&
     other.configFile == configFile &&
+    other.duplicateDetection == duplicateDetection &&
     other.email == email &&
     other.facialRecognition == facialRecognition &&
     other.map == map &&
@@ -70,6 +74,7 @@ class ServerFeaturesDto {
   int get hashCode =>
     // ignore: unnecessary_parenthesis
     (configFile.hashCode) +
+    (duplicateDetection.hashCode) +
     (email.hashCode) +
     (facialRecognition.hashCode) +
     (map.hashCode) +
@@ -83,11 +88,12 @@ class ServerFeaturesDto {
     (trash.hashCode);
 
   @override
-  String toString() => 'ServerFeaturesDto[configFile=$configFile, email=$email, facialRecognition=$facialRecognition, map=$map, oauth=$oauth, oauthAutoLaunch=$oauthAutoLaunch, passwordLogin=$passwordLogin, reverseGeocoding=$reverseGeocoding, search=$search, sidecar=$sidecar, smartSearch=$smartSearch, trash=$trash]';
+  String toString() => 'ServerFeaturesDto[configFile=$configFile, duplicateDetection=$duplicateDetection, email=$email, facialRecognition=$facialRecognition, map=$map, oauth=$oauth, oauthAutoLaunch=$oauthAutoLaunch, passwordLogin=$passwordLogin, reverseGeocoding=$reverseGeocoding, search=$search, sidecar=$sidecar, smartSearch=$smartSearch, trash=$trash]';
 
   Map<String, dynamic> toJson() {
     final json = <String, dynamic>{};
       json[r'configFile'] = this.configFile;
+      json[r'duplicateDetection'] = this.duplicateDetection;
       json[r'email'] = this.email;
       json[r'facialRecognition'] = this.facialRecognition;
       json[r'map'] = this.map;
@@ -111,6 +117,7 @@ class ServerFeaturesDto {
 
       return ServerFeaturesDto(
         configFile: mapValueOfType<bool>(json, r'configFile')!,
+        duplicateDetection: mapValueOfType<bool>(json, r'duplicateDetection')!,
         email: mapValueOfType<bool>(json, r'email')!,
         facialRecognition: mapValueOfType<bool>(json, r'facialRecognition')!,
         map: mapValueOfType<bool>(json, r'map')!,
@@ -170,6 +177,7 @@ class ServerFeaturesDto {
   /// The list of required keys that must be present in a JSON.
   static const requiredKeys = <String>{
     'configFile',
+    'duplicateDetection',
     'email',
     'facialRecognition',
     'map',

mobile/openapi/lib/model/system_config_machine_learning_dto.dart

@@ -14,6 +14,7 @@ class SystemConfigMachineLearningDto {
   /// Returns a new [SystemConfigMachineLearningDto] instance.
   SystemConfigMachineLearningDto({
     required this.clip,
+    required this.duplicateDetection,
     required this.enabled,
     required this.facialRecognition,
     required this.url,
@@ -21,6 +22,8 @@ class SystemConfigMachineLearningDto {
 
   CLIPConfig clip;
 
+  DuplicateDetectionConfig duplicateDetection;
+
   bool enabled;
 
   RecognitionConfig facialRecognition;
@@ -30,6 +33,7 @@ class SystemConfigMachineLearningDto {
   @override
   bool operator ==(Object other) => identical(this, other) || other is SystemConfigMachineLearningDto &&
     other.clip == clip &&
+    other.duplicateDetection == duplicateDetection &&
     other.enabled == enabled &&
     other.facialRecognition == facialRecognition &&
     other.url == url;
@@ -38,16 +42,18 @@ class SystemConfigMachineLearningDto {
   int get hashCode =>
     // ignore: unnecessary_parenthesis
     (clip.hashCode) +
+    (duplicateDetection.hashCode) +
     (enabled.hashCode) +
     (facialRecognition.hashCode) +
     (url.hashCode);
 
   @override
-  String toString() => 'SystemConfigMachineLearningDto[clip=$clip, enabled=$enabled, facialRecognition=$facialRecognition, url=$url]';
+  String toString() => 'SystemConfigMachineLearningDto[clip=$clip, duplicateDetection=$duplicateDetection, enabled=$enabled, facialRecognition=$facialRecognition, url=$url]';
 
   Map<String, dynamic> toJson() {
     final json = <String, dynamic>{};
       json[r'clip'] = this.clip;
+      json[r'duplicateDetection'] = this.duplicateDetection;
       json[r'enabled'] = this.enabled;
       json[r'facialRecognition'] = this.facialRecognition;
       json[r'url'] = this.url;
@@ -63,6 +69,7 @@ class SystemConfigMachineLearningDto {
 
       return SystemConfigMachineLearningDto(
         clip: CLIPConfig.fromJson(json[r'clip'])!,
+        duplicateDetection: DuplicateDetectionConfig.fromJson(json[r'duplicateDetection'])!,
         enabled: mapValueOfType<bool>(json, r'enabled')!,
         facialRecognition: RecognitionConfig.fromJson(json[r'facialRecognition'])!,
         url: mapValueOfType<String>(json, r'url')!,
@@ -114,6 +121,7 @@ class SystemConfigMachineLearningDto {
   /// The list of required keys that must be present in a JSON.
   static const requiredKeys = <String>{
     'clip',
+    'duplicateDetection',
     'enabled',
     'facialRecognition',
     'url',

mobile/openapi/test/all_job_status_response_dto_test.dart

@@ -21,6 +21,11 @@ void main() {
       // TODO
     });
 
+    // JobStatusDto duplicateDetection
+    test('to test the property `duplicateDetection`', () async {
+      // TODO
+    });
+
     // JobStatusDto faceDetection
     test('to test the property `faceDetection`', () async {
       // TODO

mobile/openapi/test/asset_api_test.dart

@@ -50,6 +50,11 @@ void main() {
       // TODO
     });
 
+    //Future<List<AssetResponseDto>> getAssetDuplicates() async
+    test('test getAssetDuplicates', () async {
+      // TODO
+    });
+
     //Future<AssetResponseDto> getAssetInfo(String id, { String key }) async
     test('test getAssetInfo', () async {
       // TODO

mobile/openapi/test/duplicate_detection_config_test.dart

@@ -0,0 +1,32 @@
+//
+// AUTO-GENERATED FILE, DO NOT MODIFY!
+//
+// @dart=2.18
+
+// ignore_for_file: unused_element, unused_import
+// ignore_for_file: always_put_required_named_parameters_first
+// ignore_for_file: constant_identifier_names
+// ignore_for_file: lines_longer_than_80_chars
+
+import 'package:openapi/api.dart';
+import 'package:test/test.dart';
+
+// tests for DuplicateDetectionConfig
+void main() {
+  // final instance = DuplicateDetectionConfig();
+
+  group('test DuplicateDetectionConfig', () {
+    // bool enabled
+    test('to test the property `enabled`', () async {
+      // TODO
+    });
+
+    // double maxDistance
+    test('to test the property `maxDistance`', () async {
+      // TODO
+    });
+
+
+  });
+
+}

mobile/openapi/test/server_features_dto_test.dart

@@ -21,6 +21,11 @@ void main() {
       // TODO
     });
 
+    // bool duplicateDetection
+    test('to test the property `duplicateDetection`', () async {
+      // TODO
+    });
+
     // bool email
     test('to test the property `email`', () async {
       // TODO

mobile/openapi/test/system_config_machine_learning_dto_test.dart

@@ -21,6 +21,11 @@ void main() {
       // TODO
     });
 
+    // DuplicateDetectionConfig duplicateDetection
+    test('to test the property `duplicateDetection`', () async {
+      // TODO
+    });
+
     // bool enabled
     test('to test the property `enabled`', () async {
       // TODO

open-api/immich-openapi-specs.json

@@ -1194,6 +1194,30 @@
         ]
       }
     },
+    "/asset/duplicates": {
+      "get": {
+        "operationId": "getAssetDuplicates",
+        "parameters": [],
+        "responses": {
+          "200": {
+            "content": {
+              "application/json": {
+                "schema": {
+                  "items": {
+                    "$ref": "#/components/schemas/AssetResponseDto"
+                  },
+                  "type": "array"
+                }
+              }
+            },
+            "description": ""
+          }
+        },
+        "tags": [
+          "Asset"
+        ]
+      }
+    },
     "/asset/exist": {
       "post": {
         "description": "Checks if multiple assets exist on the server and returns all existing - used by background backup",
@@ -6812,6 +6836,9 @@
           "backgroundTask": {
             "$ref": "#/components/schemas/JobStatusDto"
           },
+          "duplicateDetection": {
+            "$ref": "#/components/schemas/JobStatusDto"
+          },
           "faceDetection": {
             "$ref": "#/components/schemas/JobStatusDto"
           },
@@ -6851,6 +6878,7 @@
         },
         "required": [
           "backgroundTask",
+          "duplicateDetection",
           "faceDetection",
           "facialRecognition",
           "library",
@@ -7873,6 +7901,24 @@
         ],
         "type": "object"
       },
+      "DuplicateDetectionConfig": {
+        "properties": {
+          "enabled": {
+            "type": "boolean"
+          },
+          "maxDistance": {
+            "format": "float",
+            "maximum": 0.1,
+            "minimum": 0.001,
+            "type": "number"
+          }
+        },
+        "required": [
+          "enabled",
+          "maxDistance"
+        ],
+        "type": "object"
+      },
       "EntityType": {
         "enum": [
           "ASSET",
@@ -8167,6 +8213,7 @@
           "faceDetection",
           "facialRecognition",
           "smartSearch",
+          "duplicateDetection",
           "backgroundTask",
           "storageTemplateMigration",
           "migration",
@@ -9379,6 +9426,9 @@
           "configFile": {
             "type": "boolean"
           },
+          "duplicateDetection": {
+            "type": "boolean"
+          },
           "email": {
             "type": "boolean"
           },
@@ -9415,6 +9465,7 @@
         },
         "required": [
           "configFile",
+          "duplicateDetection",
           "email",
           "facialRecognition",
           "map",
@@ -10247,6 +10298,9 @@
           "clip": {
             "$ref": "#/components/schemas/CLIPConfig"
           },
+          "duplicateDetection": {
+            "$ref": "#/components/schemas/DuplicateDetectionConfig"
+          },
           "enabled": {
             "type": "boolean"
           },
@@ -10259,6 +10313,7 @@
         },
         "required": [
           "clip",
+          "duplicateDetection",
           "enabled",
           "facialRecognition",
           "url"

open-api/typescript-sdk/src/fetch-client.ts

@@ -410,6 +410,7 @@ export type JobStatusDto = {
 };
 export type AllJobStatusResponseDto = {
     backgroundTask: JobStatusDto;
+    duplicateDetection: JobStatusDto;
     faceDetection: JobStatusDto;
     facialRecognition: JobStatusDto;
     library: JobStatusDto;
@@ -748,6 +749,7 @@ export type ServerConfigDto = {
 };
 export type ServerFeaturesDto = {
     configFile: boolean;
+    duplicateDetection: boolean;
     email: boolean;
     facialRecognition: boolean;
     map: boolean;
@@ -927,6 +929,10 @@ export type ClipConfig = {
     modelName: string;
     modelType?: ModelType;
 };
+export type DuplicateDetectionConfig = {
+    enabled: boolean;
+    maxDistance: number;
+};
 export type RecognitionConfig = {
     enabled: boolean;
     maxDistance: number;
@@ -937,6 +943,7 @@ export type RecognitionConfig = {
 };
 export type SystemConfigMachineLearningDto = {
     clip: ClipConfig;
+    duplicateDetection: DuplicateDetectionConfig;
     enabled: boolean;
     facialRecognition: RecognitionConfig;
     url: string;
@@ -1399,6 +1406,14 @@ export function getAllUserAssetsByDeviceId({ deviceId }: {
         ...opts
     }));
 }
+export function getAssetDuplicates(opts?: Oazapfts.RequestOpts) {
+    return oazapfts.ok(oazapfts.fetchJson<{
+        status: 200;
+        data: AssetResponseDto[];
+    }>("/asset/duplicates", {
+        ...opts
+    }));
+}
 /**
  * Checks if multiple assets exist on the server and returns all existing - used by background backup
  */
@@ -2876,6 +2891,7 @@ export enum JobName {
     FaceDetection = "faceDetection",
     FacialRecognition = "facialRecognition",
     SmartSearch = "smartSearch",
+    DuplicateDetection = "duplicateDetection",
     BackgroundTask = "backgroundTask",
     StorageTemplateMigration = "storageTemplateMigration",
     Migration = "migration",

server/src/config.ts

@@ -111,6 +111,10 @@ export interface SystemConfig {
       enabled: boolean;
       modelName: string;
     };
+    duplicateDetection: {
+      enabled: boolean;
+      maxDistance: number;
+    };
     facialRecognition: {
       enabled: boolean;
       modelName: string;
@@ -249,6 +253,10 @@ export const defaults = Object.freeze<SystemConfig>({
       enabled: true,
       modelName: 'ViT-B-32__openai',
     },
+    duplicateDetection: {
+      enabled: false,
+      maxDistance: 0.03,
+    },
     facialRecognition: {
       enabled: true,
       modelName: 'buffalo_l',

server/src/controllers/asset.controller.ts

@@ -57,6 +57,11 @@ export class AssetController {
     return this.service.getStatistics(auth, dto);
   }
 
+  @Get('duplicates')
+  getAssetDuplicates(@Auth() auth: AuthDto): Promise<AssetResponseDto[]> {
+    return this.service.getDuplicates(auth);
+  }
+
   @Post('jobs')
   @HttpCode(HttpStatus.NO_CONTENT)
   @Authenticated()

server/src/dtos/job.dto.ts

@@ -73,6 +73,9 @@ export class AllJobStatusResponseDto implements Record<QueueName, JobStatusDto>
   @ApiProperty({ type: JobStatusDto })
   [QueueName.SEARCH]!: JobStatusDto;
 
+  @ApiProperty({ type: JobStatusDto })
+  [QueueName.DUPLICATE_DETECTION]!: JobStatusDto;
+
   @ApiProperty({ type: JobStatusDto })
   [QueueName.FACE_DETECTION]!: JobStatusDto;
 

server/src/dtos/model-config.dto.ts

@@ -4,10 +4,12 @@ import { IsEnum, IsNotEmpty, IsNumber, IsString, Max, Min } from 'class-validato
 import { CLIPMode, ModelType } from 'src/interfaces/machine-learning.interface';
 import { Optional, ValidateBoolean } from 'src/validation';
 
-export class ModelConfig {
+export class TaskConfig {
   @ValidateBoolean()
   enabled!: boolean;
+}
 
+export class ModelConfig extends TaskConfig {
   @IsString()
   @IsNotEmpty()
   modelName!: string;
@@ -25,6 +27,15 @@ export class CLIPConfig extends ModelConfig {
   mode?: CLIPMode;
 }
 
+export class DuplicateDetectionConfig extends TaskConfig {
+  @IsNumber()
+  @Min(0.001)
+  @Max(0.1)
+  @Type(() => Number)
+  @ApiProperty({ type: 'number', format: 'float' })
+  maxDistance!: number;
+}
+
 export class RecognitionConfig extends ModelConfig {
   @IsNumber()
   @Min(0)

server/src/dtos/server-info.dto.ts

@@ -97,6 +97,7 @@ export class ServerConfigDto {
 
 export class ServerFeaturesDto {
   smartSearch!: boolean;
+  duplicateDetection!: boolean;
   configFile!: boolean;
   facialRecognition!: boolean;
   map!: boolean;

server/src/dtos/system-config.dto.ts

@@ -30,7 +30,7 @@ import {
   TranscodePolicy,
   VideoCodec,
 } from 'src/config';
-import { CLIPConfig, RecognitionConfig } from 'src/dtos/model-config.dto';
+import { CLIPConfig, DuplicateDetectionConfig, RecognitionConfig } from 'src/dtos/model-config.dto';
 import { ConcurrentQueueName, QueueName } from 'src/interfaces/job.interface';
 import { ValidateBoolean, validateCronExpression } from 'src/validation';
 
@@ -262,6 +262,11 @@ class SystemConfigMachineLearningDto {
   @IsObject()
   clip!: CLIPConfig;
 
+  @Type(() => DuplicateDetectionConfig)
+  @ValidateNested()
+  @IsObject()
+  duplicateDetection!: DuplicateDetectionConfig;
+
   @Type(() => RecognitionConfig)
   @ValidateNested()
   @IsObject()

server/src/entities/asset-job-status.entity.ts

@@ -15,4 +15,7 @@ export class AssetJobStatusEntity {
 
   @Column({ type: 'timestamptz', nullable: true })
   metadataExtractedAt!: Date | null;
+
+  @Column({ type: 'timestamptz', nullable: true })
+  duplicatesDetectedAt!: Date | null;
 }

server/src/entities/asset.entity.ts

@@ -165,6 +165,10 @@ export class AssetEntity {
 
   @OneToOne(() => AssetJobStatusEntity, (jobStatus) => jobStatus.asset, { nullable: true })
   jobStatus?: AssetJobStatusEntity;
+
+  @Index('IDX_assets_duplicateId')
+  @Column({ type: 'uuid', nullable: true })
+  duplicateId!: string | null;
 }
 
 export enum AssetType {

server/src/entities/smart-search.entity.ts

@@ -11,10 +11,6 @@ export class SmartSearchEntity {
   assetId!: string;
 
   @Index('clip_index', { synchronize: false })
-  @Column({
+  @Column({ type: 'float4', array: true, transformer: { from: (v) => JSON.parse(v), to: (v) => v } })
-    type: 'float4',
-    array: true,
-    select: false,
-  })
   embedding!: number[];
 }

server/src/interfaces/asset.interface.ts

@@ -40,6 +40,7 @@ export enum WithoutProperty {
   ENCODED_VIDEO = 'encoded-video',
   EXIF = 'exif',
   SMART_SEARCH = 'smart-search',
+  DUPLICATE = 'duplicate',
   OBJECT_TAGS = 'object-tags',
   FACES = 'faces',
   PERSON = 'person',
@@ -60,6 +61,7 @@ export interface AssetBuilderOptions {
   isArchived?: boolean;
   isFavorite?: boolean;
   isTrashed?: boolean;
+  isDuplicate?: boolean;
   albumId?: string;
   personId?: string;
   userIds?: string[];
@@ -143,6 +145,12 @@ export interface AssetDeltaSyncOptions {
   limit: number;
 }
 
+export interface AssetUpdateDuplicateOptions {
+  targetDuplicateId: string | null;
+  assetIds: string[];
+  duplicateIds: string[];
+}
+
 export type AssetPathEntity = Pick<AssetEntity, 'id' | 'originalPath' | 'isOffline'>;
 
 export const IAssetRepository = 'IAssetRepository';
@@ -176,6 +184,7 @@ export interface IAssetRepository {
   getAll(pagination: PaginationOptions, options?: AssetSearchOptions): Paginated<AssetEntity>;
   getAllByDeviceId(userId: string, deviceId: string): Promise<string[]>;
   updateAll(ids: string[], options: Partial<AssetUpdateAllOptions>): Promise<void>;
+  updateDuplicates(options: AssetUpdateDuplicateOptions): Promise<void>;
   update(asset: AssetUpdateOptions): Promise<void>;
   remove(asset: AssetEntity): Promise<void>;
   softDeleteAll(ids: string[]): Promise<void>;
@@ -186,9 +195,10 @@ export interface IAssetRepository {
   getTimeBuckets(options: TimeBucketOptions): Promise<TimeBucketItem[]>;
   getTimeBucket(timeBucket: string, options: TimeBucketOptions): Promise<AssetEntity[]>;
   upsertExif(exif: Partial<ExifEntity>): Promise<void>;
-  upsertJobStatus(jobStatus: Partial<AssetJobStatusEntity>): Promise<void>;
+  upsertJobStatus(...jobStatus: Partial<AssetJobStatusEntity>[]): Promise<void>;
   getAssetIdByCity(userId: string, options: AssetExploreFieldOptions): Promise<SearchExploreItem<string>>;
   getAssetIdByTag(userId: string, options: AssetExploreFieldOptions): Promise<SearchExploreItem<string>>;
+  getDuplicates(options: AssetBuilderOptions): Promise<AssetEntity[]>;
   getAllForUserFullSync(options: AssetFullSyncOptions): Promise<AssetEntity[]>;
   getChangedDeltaSync(options: AssetDeltaSyncOptions): Promise<AssetEntity[]>;
 }

server/src/interfaces/job.interface.ts

@@ -5,6 +5,7 @@ export enum QueueName {
   FACE_DETECTION = 'faceDetection',
   FACIAL_RECOGNITION = 'facialRecognition',
   SMART_SEARCH = 'smartSearch',
+  DUPLICATE_DETECTION = 'duplicateDetection',
   BACKGROUND_TASK = 'backgroundTask',
   STORAGE_TEMPLATE_MIGRATION = 'storageTemplateMigration',
   MIGRATION = 'migration',
@@ -16,7 +17,7 @@ export enum QueueName {
 
 export type ConcurrentQueueName = Exclude<
   QueueName,
-  QueueName.STORAGE_TEMPLATE_MIGRATION | QueueName.FACIAL_RECOGNITION
+  QueueName.STORAGE_TEMPLATE_MIGRATION | QueueName.FACIAL_RECOGNITION | QueueName.DUPLICATE_DETECTION
 >;
 
 export enum JobCommand {
@@ -86,6 +87,10 @@ export enum JobName {
   QUEUE_SMART_SEARCH = 'queue-smart-search',
   SMART_SEARCH = 'smart-search',
 
+  // duplicate detection
+  QUEUE_DUPLICATE_DETECTION = 'queue-duplicate-detection',
+  DUPLICATE_DETECTION = 'duplicate-detection',
+
   // XMP sidecars
   QUEUE_SIDECAR = 'queue-sidecar',
   SIDECAR_DISCOVERY = 'sidecar-discovery',
@@ -212,6 +217,10 @@ export type JobItem =
   | { name: JobName.QUEUE_SMART_SEARCH; data: IBaseJob }
   | { name: JobName.SMART_SEARCH; data: IEntityJob }
 
+  // Duplicate Detection
+  | { name: JobName.QUEUE_DUPLICATE_DETECTION; data: IBaseJob }
+  | { name: JobName.DUPLICATE_DETECTION; data: IEntityJob }
+
   // Filesystem
   | { name: JobName.DELETE_FILES; data: IDeleteFilesJob }
 

server/src/interfaces/search.interface.ts

@@ -152,15 +152,29 @@ export interface FaceEmbeddingSearch extends SearchEmbeddingOptions {
   maxDistance?: number;
 }
 
+export interface AssetDuplicateSearch {
+  assetId: string;
+  embedding: Embedding;
+  userIds: string[];
+  maxDistance?: number;
+}
+
 export interface FaceSearchResult {
   distance: number;
   face: AssetFaceEntity;
 }
 
+export interface AssetDuplicateResult {
+  assetId: string;
+  duplicateId: string | null;
+  distance: number;
+}
+
 export interface ISearchRepository {
   init(modelName: string): Promise<void>;
   searchMetadata(pagination: SearchPaginationOptions, options: AssetSearchOptions): Paginated<AssetEntity>;
   searchSmart(pagination: SearchPaginationOptions, options: SmartSearchOptions): Paginated<AssetEntity>;
+  searchDuplicates(options: AssetDuplicateSearch): Promise<AssetDuplicateResult[]>;
   searchFaces(search: FaceEmbeddingSearch): Promise<FaceSearchResult[]>;
   upsert(assetId: string, embedding: number[]): Promise<void>;
   searchPlaces(placeName: string): Promise<GeodataPlacesEntity[]>;

server/src/migrations/1711989989911-AddAssetDuplicateColumns.ts

@@ -0,0 +1,14 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class CreateAssetDuplicateColumns1711989989911 implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`ALTER TABLE assets ADD COLUMN "duplicateId" uuid`);
+    await queryRunner.query(`ALTER TABLE asset_job_status ADD COLUMN "duplicatesDetectedAt" timestamptz`);
+    await queryRunner.query(`CREATE INDEX "IDX_assets_duplicateId" ON assets ("duplicateId")`);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`ALTER TABLE assets DROP COLUMN "duplicateId"`);
+    await queryRunner.query(`ALTER TABLE asset_job_status DROP COLUMN "duplicatesDetectedAt"`);
+  }
+}

server/src/queries/asset.repository.sql

@@ -30,6 +30,7 @@ SELECT
   "entity"."originalFileName" AS "entity_originalFileName",
   "entity"."sidecarPath" AS "entity_sidecarPath",
   "entity"."stackId" AS "entity_stackId",
+  "entity"."duplicateId" AS "entity_duplicateId",
   "exifInfo"."assetId" AS "exifInfo_assetId",
   "exifInfo"."description" AS "exifInfo_description",
   "exifInfo"."exifImageWidth" AS "exifInfo_exifImageWidth",
@@ -111,7 +112,8 @@ SELECT
   "AssetEntity"."livePhotoVideoId" AS "AssetEntity_livePhotoVideoId",
   "AssetEntity"."originalFileName" AS "AssetEntity_originalFileName",
   "AssetEntity"."sidecarPath" AS "AssetEntity_sidecarPath",
-  "AssetEntity"."stackId" AS "AssetEntity_stackId"
+  "AssetEntity"."stackId" AS "AssetEntity_stackId",
+  "AssetEntity"."duplicateId" AS "AssetEntity_duplicateId"
 FROM
   "assets" "AssetEntity"
 WHERE
@@ -147,6 +149,7 @@ SELECT
   "AssetEntity"."originalFileName" AS "AssetEntity_originalFileName",
   "AssetEntity"."sidecarPath" AS "AssetEntity_sidecarPath",
   "AssetEntity"."stackId" AS "AssetEntity_stackId",
+  "AssetEntity"."duplicateId" AS "AssetEntity_duplicateId",
   "AssetEntity__AssetEntity_exifInfo"."assetId" AS "AssetEntity__AssetEntity_exifInfo_assetId",
   "AssetEntity__AssetEntity_exifInfo"."description" AS "AssetEntity__AssetEntity_exifInfo_description",
   "AssetEntity__AssetEntity_exifInfo"."exifImageWidth" AS "AssetEntity__AssetEntity_exifInfo_exifImageWidth",
@@ -230,7 +233,8 @@ SELECT
   "bd93d5747511a4dad4923546c51365bf1a803774"."livePhotoVideoId" AS "bd93d5747511a4dad4923546c51365bf1a803774_livePhotoVideoId",
   "bd93d5747511a4dad4923546c51365bf1a803774"."originalFileName" AS "bd93d5747511a4dad4923546c51365bf1a803774_originalFileName",
   "bd93d5747511a4dad4923546c51365bf1a803774"."sidecarPath" AS "bd93d5747511a4dad4923546c51365bf1a803774_sidecarPath",
-  "bd93d5747511a4dad4923546c51365bf1a803774"."stackId" AS "bd93d5747511a4dad4923546c51365bf1a803774_stackId"
+  "bd93d5747511a4dad4923546c51365bf1a803774"."stackId" AS "bd93d5747511a4dad4923546c51365bf1a803774_stackId",
+  "bd93d5747511a4dad4923546c51365bf1a803774"."duplicateId" AS "bd93d5747511a4dad4923546c51365bf1a803774_duplicateId"
 FROM
   "assets" "AssetEntity"
   LEFT JOIN "exif" "AssetEntity__AssetEntity_exifInfo" ON "AssetEntity__AssetEntity_exifInfo"."assetId" = "AssetEntity"."id"
@@ -311,7 +315,8 @@ FROM
       "AssetEntity"."livePhotoVideoId" AS "AssetEntity_livePhotoVideoId",
       "AssetEntity"."originalFileName" AS "AssetEntity_originalFileName",
       "AssetEntity"."sidecarPath" AS "AssetEntity_sidecarPath",
-      "AssetEntity"."stackId" AS "AssetEntity_stackId"
+      "AssetEntity"."stackId" AS "AssetEntity_stackId",
+      "AssetEntity"."duplicateId" AS "AssetEntity_duplicateId"
     FROM
       "assets" "AssetEntity"
       LEFT JOIN "libraries" "AssetEntity__AssetEntity_library" ON "AssetEntity__AssetEntity_library"."id" = "AssetEntity"."libraryId"
@@ -407,7 +412,8 @@ SELECT
   "AssetEntity"."livePhotoVideoId" AS "AssetEntity_livePhotoVideoId",
   "AssetEntity"."originalFileName" AS "AssetEntity_originalFileName",
   "AssetEntity"."sidecarPath" AS "AssetEntity_sidecarPath",
-  "AssetEntity"."stackId" AS "AssetEntity_stackId"
+  "AssetEntity"."stackId" AS "AssetEntity_stackId",
+  "AssetEntity"."duplicateId" AS "AssetEntity_duplicateId"
 FROM
   "assets" "AssetEntity"
 WHERE
@@ -423,6 +429,15 @@ SET
 WHERE
   "id" IN ($2)
 
+-- AssetRepository.updateDuplicates
+UPDATE "assets"
+SET
+  "duplicateId" = $1,
+  "updatedAt" = CURRENT_TIMESTAMP
+WHERE
+  "duplicateId" IN ($2)
+  OR "id" IN ($3)
+
 -- AssetRepository.getByChecksum
 SELECT
   "AssetEntity"."id" AS "AssetEntity_id",
@@ -452,7 +467,8 @@ SELECT
   "AssetEntity"."livePhotoVideoId" AS "AssetEntity_livePhotoVideoId",
   "AssetEntity"."originalFileName" AS "AssetEntity_originalFileName",
   "AssetEntity"."sidecarPath" AS "AssetEntity_sidecarPath",
-  "AssetEntity"."stackId" AS "AssetEntity_stackId"
+  "AssetEntity"."stackId" AS "AssetEntity_stackId",
+  "AssetEntity"."duplicateId" AS "AssetEntity_duplicateId"
 FROM
   "assets" "AssetEntity"
 WHERE
@@ -519,7 +535,8 @@ SELECT
   "AssetEntity"."livePhotoVideoId" AS "AssetEntity_livePhotoVideoId",
   "AssetEntity"."originalFileName" AS "AssetEntity_originalFileName",
   "AssetEntity"."sidecarPath" AS "AssetEntity_sidecarPath",
-  "AssetEntity"."stackId" AS "AssetEntity_stackId"
+  "AssetEntity"."stackId" AS "AssetEntity_stackId",
+  "AssetEntity"."duplicateId" AS "AssetEntity_duplicateId"
 FROM
   "assets" "AssetEntity"
 WHERE
@@ -575,6 +592,7 @@ SELECT
   "asset"."originalFileName" AS "asset_originalFileName",
   "asset"."sidecarPath" AS "asset_sidecarPath",
   "asset"."stackId" AS "asset_stackId",
+  "asset"."duplicateId" AS "asset_duplicateId",
   "exifInfo"."assetId" AS "exifInfo_assetId",
   "exifInfo"."description" AS "exifInfo_description",
   "exifInfo"."exifImageWidth" AS "exifInfo_exifImageWidth",
@@ -632,7 +650,8 @@ SELECT
   "stackedAssets"."livePhotoVideoId" AS "stackedAssets_livePhotoVideoId",
   "stackedAssets"."originalFileName" AS "stackedAssets_originalFileName",
   "stackedAssets"."sidecarPath" AS "stackedAssets_sidecarPath",
-  "stackedAssets"."stackId" AS "stackedAssets_stackId"
+  "stackedAssets"."stackId" AS "stackedAssets_stackId",
+  "stackedAssets"."duplicateId" AS "stackedAssets_duplicateId"
 FROM
   "assets" "asset"
   LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id"
@@ -713,6 +732,7 @@ SELECT
   "asset"."originalFileName" AS "asset_originalFileName",
   "asset"."sidecarPath" AS "asset_sidecarPath",
   "asset"."stackId" AS "asset_stackId",
+  "asset"."duplicateId" AS "asset_duplicateId",
   "exifInfo"."assetId" AS "exifInfo_assetId",
   "exifInfo"."description" AS "exifInfo_description",
   "exifInfo"."exifImageWidth" AS "exifInfo_exifImageWidth",
@@ -770,7 +790,8 @@ SELECT
   "stackedAssets"."livePhotoVideoId" AS "stackedAssets_livePhotoVideoId",
   "stackedAssets"."originalFileName" AS "stackedAssets_originalFileName",
   "stackedAssets"."sidecarPath" AS "stackedAssets_sidecarPath",
-  "stackedAssets"."stackId" AS "stackedAssets_stackId"
+  "stackedAssets"."stackId" AS "stackedAssets_stackId",
+  "stackedAssets"."duplicateId" AS "stackedAssets_duplicateId"
 FROM
   "assets" "asset"
   LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id"
@@ -797,6 +818,112 @@ ORDER BY
   )::timestamptz DESC,
   "asset"."fileCreatedAt" DESC
 
+-- AssetRepository.getDuplicates
+SELECT
+  "asset"."id" AS "asset_id",
+  "asset"."deviceAssetId" AS "asset_deviceAssetId",
+  "asset"."ownerId" AS "asset_ownerId",
+  "asset"."libraryId" AS "asset_libraryId",
+  "asset"."deviceId" AS "asset_deviceId",
+  "asset"."type" AS "asset_type",
+  "asset"."originalPath" AS "asset_originalPath",
+  "asset"."previewPath" AS "asset_previewPath",
+  "asset"."thumbnailPath" AS "asset_thumbnailPath",
+  "asset"."thumbhash" AS "asset_thumbhash",
+  "asset"."encodedVideoPath" AS "asset_encodedVideoPath",
+  "asset"."createdAt" AS "asset_createdAt",
+  "asset"."updatedAt" AS "asset_updatedAt",
+  "asset"."deletedAt" AS "asset_deletedAt",
+  "asset"."fileCreatedAt" AS "asset_fileCreatedAt",
+  "asset"."localDateTime" AS "asset_localDateTime",
+  "asset"."fileModifiedAt" AS "asset_fileModifiedAt",
+  "asset"."isFavorite" AS "asset_isFavorite",
+  "asset"."isArchived" AS "asset_isArchived",
+  "asset"."isExternal" AS "asset_isExternal",
+  "asset"."isOffline" AS "asset_isOffline",
+  "asset"."checksum" AS "asset_checksum",
+  "asset"."duration" AS "asset_duration",
+  "asset"."isVisible" AS "asset_isVisible",
+  "asset"."livePhotoVideoId" AS "asset_livePhotoVideoId",
+  "asset"."originalFileName" AS "asset_originalFileName",
+  "asset"."sidecarPath" AS "asset_sidecarPath",
+  "asset"."stackId" AS "asset_stackId",
+  "asset"."duplicateId" AS "asset_duplicateId",
+  "exifInfo"."assetId" AS "exifInfo_assetId",
+  "exifInfo"."description" AS "exifInfo_description",
+  "exifInfo"."exifImageWidth" AS "exifInfo_exifImageWidth",
+  "exifInfo"."exifImageHeight" AS "exifInfo_exifImageHeight",
+  "exifInfo"."fileSizeInByte" AS "exifInfo_fileSizeInByte",
+  "exifInfo"."orientation" AS "exifInfo_orientation",
+  "exifInfo"."dateTimeOriginal" AS "exifInfo_dateTimeOriginal",
+  "exifInfo"."modifyDate" AS "exifInfo_modifyDate",
+  "exifInfo"."timeZone" AS "exifInfo_timeZone",
+  "exifInfo"."latitude" AS "exifInfo_latitude",
+  "exifInfo"."longitude" AS "exifInfo_longitude",
+  "exifInfo"."projectionType" AS "exifInfo_projectionType",
+  "exifInfo"."city" AS "exifInfo_city",
+  "exifInfo"."livePhotoCID" AS "exifInfo_livePhotoCID",
+  "exifInfo"."autoStackId" AS "exifInfo_autoStackId",
+  "exifInfo"."state" AS "exifInfo_state",
+  "exifInfo"."country" AS "exifInfo_country",
+  "exifInfo"."make" AS "exifInfo_make",
+  "exifInfo"."model" AS "exifInfo_model",
+  "exifInfo"."lensModel" AS "exifInfo_lensModel",
+  "exifInfo"."fNumber" AS "exifInfo_fNumber",
+  "exifInfo"."focalLength" AS "exifInfo_focalLength",
+  "exifInfo"."iso" AS "exifInfo_iso",
+  "exifInfo"."exposureTime" AS "exifInfo_exposureTime",
+  "exifInfo"."profileDescription" AS "exifInfo_profileDescription",
+  "exifInfo"."colorspace" AS "exifInfo_colorspace",
+  "exifInfo"."bitsPerSample" AS "exifInfo_bitsPerSample",
+  "exifInfo"."fps" AS "exifInfo_fps",
+  "stack"."id" AS "stack_id",
+  "stack"."primaryAssetId" AS "stack_primaryAssetId",
+  "stackedAssets"."id" AS "stackedAssets_id",
+  "stackedAssets"."deviceAssetId" AS "stackedAssets_deviceAssetId",
+  "stackedAssets"."ownerId" AS "stackedAssets_ownerId",
+  "stackedAssets"."libraryId" AS "stackedAssets_libraryId",
+  "stackedAssets"."deviceId" AS "stackedAssets_deviceId",
+  "stackedAssets"."type" AS "stackedAssets_type",
+  "stackedAssets"."originalPath" AS "stackedAssets_originalPath",
+  "stackedAssets"."previewPath" AS "stackedAssets_previewPath",
+  "stackedAssets"."thumbnailPath" AS "stackedAssets_thumbnailPath",
+  "stackedAssets"."thumbhash" AS "stackedAssets_thumbhash",
+  "stackedAssets"."encodedVideoPath" AS "stackedAssets_encodedVideoPath",
+  "stackedAssets"."createdAt" AS "stackedAssets_createdAt",
+  "stackedAssets"."updatedAt" AS "stackedAssets_updatedAt",
+  "stackedAssets"."deletedAt" AS "stackedAssets_deletedAt",
+  "stackedAssets"."fileCreatedAt" AS "stackedAssets_fileCreatedAt",
+  "stackedAssets"."localDateTime" AS "stackedAssets_localDateTime",
+  "stackedAssets"."fileModifiedAt" AS "stackedAssets_fileModifiedAt",
+  "stackedAssets"."isFavorite" AS "stackedAssets_isFavorite",
+  "stackedAssets"."isArchived" AS "stackedAssets_isArchived",
+  "stackedAssets"."isExternal" AS "stackedAssets_isExternal",
+  "stackedAssets"."isOffline" AS "stackedAssets_isOffline",
+  "stackedAssets"."checksum" AS "stackedAssets_checksum",
+  "stackedAssets"."duration" AS "stackedAssets_duration",
+  "stackedAssets"."isVisible" AS "stackedAssets_isVisible",
+  "stackedAssets"."livePhotoVideoId" AS "stackedAssets_livePhotoVideoId",
+  "stackedAssets"."originalFileName" AS "stackedAssets_originalFileName",
+  "stackedAssets"."sidecarPath" AS "stackedAssets_sidecarPath",
+  "stackedAssets"."stackId" AS "stackedAssets_stackId",
+  "stackedAssets"."duplicateId" AS "stackedAssets_duplicateId"
+FROM
+  "assets" "asset"
+  LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id"
+  LEFT JOIN "asset_stack" "stack" ON "stack"."id" = "asset"."stackId"
+  LEFT JOIN "assets" "stackedAssets" ON "stackedAssets"."stackId" = "stack"."id"
+  AND ("stackedAssets"."deletedAt" IS NULL)
+WHERE
+  (
+    "asset"."isVisible" = true
+    AND "asset"."ownerId" IN ($1, $2)
+    AND "asset"."duplicateId" IS NOT NULL
+  )
+  AND ("asset"."deletedAt" IS NULL)
+ORDER BY
+  "asset"."duplicateId" ASC
+
 -- AssetRepository.getAssetIdByCity
 WITH
   "cities" AS (
@@ -887,6 +1014,7 @@ SELECT
   "asset"."originalFileName" AS "asset_originalFileName",
   "asset"."sidecarPath" AS "asset_sidecarPath",
   "asset"."stackId" AS "asset_stackId",
+  "asset"."duplicateId" AS "asset_duplicateId",
   "exifInfo"."assetId" AS "exifInfo_assetId",
   "exifInfo"."description" AS "exifInfo_description",
   "exifInfo"."exifImageWidth" AS "exifInfo_exifImageWidth",
@@ -944,7 +1072,8 @@ SELECT
   "stackedAssets"."livePhotoVideoId" AS "stackedAssets_livePhotoVideoId",
   "stackedAssets"."originalFileName" AS "stackedAssets_originalFileName",
   "stackedAssets"."sidecarPath" AS "stackedAssets_sidecarPath",
-  "stackedAssets"."stackId" AS "stackedAssets_stackId"
+  "stackedAssets"."stackId" AS "stackedAssets_stackId",
+  "stackedAssets"."duplicateId" AS "stackedAssets_duplicateId"
 FROM
   "assets" "asset"
   LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id"
@@ -992,6 +1121,7 @@ SELECT
   "asset"."originalFileName" AS "asset_originalFileName",
   "asset"."sidecarPath" AS "asset_sidecarPath",
   "asset"."stackId" AS "asset_stackId",
+  "asset"."duplicateId" AS "asset_duplicateId",
   "exifInfo"."assetId" AS "exifInfo_assetId",
   "exifInfo"."description" AS "exifInfo_description",
   "exifInfo"."exifImageWidth" AS "exifInfo_exifImageWidth",
@@ -1049,7 +1179,8 @@ SELECT
   "stackedAssets"."livePhotoVideoId" AS "stackedAssets_livePhotoVideoId",
   "stackedAssets"."originalFileName" AS "stackedAssets_originalFileName",
   "stackedAssets"."sidecarPath" AS "stackedAssets_sidecarPath",
-  "stackedAssets"."stackId" AS "stackedAssets_stackId"
+  "stackedAssets"."stackId" AS "stackedAssets_stackId",
+  "stackedAssets"."duplicateId" AS "stackedAssets_duplicateId"
 FROM
   "assets" "asset"
   LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id"

server/src/queries/person.repository.sql

@@ -174,7 +174,8 @@ FROM
       "AssetFaceEntity__AssetFaceEntity_asset"."livePhotoVideoId" AS "AssetFaceEntity__AssetFaceEntity_asset_livePhotoVideoId",
       "AssetFaceEntity__AssetFaceEntity_asset"."originalFileName" AS "AssetFaceEntity__AssetFaceEntity_asset_originalFileName",
       "AssetFaceEntity__AssetFaceEntity_asset"."sidecarPath" AS "AssetFaceEntity__AssetFaceEntity_asset_sidecarPath",
-      "AssetFaceEntity__AssetFaceEntity_asset"."stackId" AS "AssetFaceEntity__AssetFaceEntity_asset_stackId"
+      "AssetFaceEntity__AssetFaceEntity_asset"."stackId" AS "AssetFaceEntity__AssetFaceEntity_asset_stackId",
+      "AssetFaceEntity__AssetFaceEntity_asset"."duplicateId" AS "AssetFaceEntity__AssetFaceEntity_asset_duplicateId"
     FROM
       "asset_faces" "AssetFaceEntity"
       LEFT JOIN "person" "AssetFaceEntity__AssetFaceEntity_person" ON "AssetFaceEntity__AssetFaceEntity_person"."id" = "AssetFaceEntity"."personId"
@@ -272,6 +273,7 @@ FROM
       "AssetEntity"."originalFileName" AS "AssetEntity_originalFileName",
       "AssetEntity"."sidecarPath" AS "AssetEntity_sidecarPath",
       "AssetEntity"."stackId" AS "AssetEntity_stackId",
+      "AssetEntity"."duplicateId" AS "AssetEntity_duplicateId",
       "AssetEntity__AssetEntity_faces"."id" AS "AssetEntity__AssetEntity_faces_id",
       "AssetEntity__AssetEntity_faces"."assetId" AS "AssetEntity__AssetEntity_faces_assetId",
       "AssetEntity__AssetEntity_faces"."personId" AS "AssetEntity__AssetEntity_faces_personId",
@@ -400,7 +402,8 @@ SELECT
   "AssetFaceEntity__AssetFaceEntity_asset"."livePhotoVideoId" AS "AssetFaceEntity__AssetFaceEntity_asset_livePhotoVideoId",
   "AssetFaceEntity__AssetFaceEntity_asset"."originalFileName" AS "AssetFaceEntity__AssetFaceEntity_asset_originalFileName",
   "AssetFaceEntity__AssetFaceEntity_asset"."sidecarPath" AS "AssetFaceEntity__AssetFaceEntity_asset_sidecarPath",
-  "AssetFaceEntity__AssetFaceEntity_asset"."stackId" AS "AssetFaceEntity__AssetFaceEntity_asset_stackId"
+  "AssetFaceEntity__AssetFaceEntity_asset"."stackId" AS "AssetFaceEntity__AssetFaceEntity_asset_stackId",
+  "AssetFaceEntity__AssetFaceEntity_asset"."duplicateId" AS "AssetFaceEntity__AssetFaceEntity_asset_duplicateId"
 FROM
   "asset_faces" "AssetFaceEntity"
   LEFT JOIN "assets" "AssetFaceEntity__AssetFaceEntity_asset" ON "AssetFaceEntity__AssetFaceEntity_asset"."id" = "AssetFaceEntity"."assetId"

server/src/queries/search.repository.sql

@@ -35,6 +35,7 @@ FROM
       "asset"."originalFileName" AS "asset_originalFileName",
       "asset"."sidecarPath" AS "asset_sidecarPath",
       "asset"."stackId" AS "asset_stackId",
+      "asset"."duplicateId" AS "asset_duplicateId",
       "stack"."id" AS "stack_id",
       "stack"."primaryAssetId" AS "stack_primaryAssetId",
       "stackedAssets"."id" AS "stackedAssets_id",
@@ -64,7 +65,8 @@ FROM
       "stackedAssets"."livePhotoVideoId" AS "stackedAssets_livePhotoVideoId",
       "stackedAssets"."originalFileName" AS "stackedAssets_originalFileName",
       "stackedAssets"."sidecarPath" AS "stackedAssets_sidecarPath",
-      "stackedAssets"."stackId" AS "stackedAssets_stackId"
+      "stackedAssets"."stackId" AS "stackedAssets_stackId",
+      "stackedAssets"."duplicateId" AS "stackedAssets_duplicateId"
     FROM
       "assets" "asset"
       LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id"
@@ -129,6 +131,7 @@ SELECT
   "asset"."originalFileName" AS "asset_originalFileName",
   "asset"."sidecarPath" AS "asset_sidecarPath",
   "asset"."stackId" AS "asset_stackId",
+  "asset"."duplicateId" AS "asset_duplicateId",
   "stack"."id" AS "stack_id",
   "stack"."primaryAssetId" AS "stack_primaryAssetId",
   "stackedAssets"."id" AS "stackedAssets_id",
@@ -158,7 +161,8 @@ SELECT
   "stackedAssets"."livePhotoVideoId" AS "stackedAssets_livePhotoVideoId",
   "stackedAssets"."originalFileName" AS "stackedAssets_originalFileName",
   "stackedAssets"."sidecarPath" AS "stackedAssets_sidecarPath",
-  "stackedAssets"."stackId" AS "stackedAssets_stackId"
+  "stackedAssets"."stackId" AS "stackedAssets_stackId",
+  "stackedAssets"."duplicateId" AS "stackedAssets_duplicateId"
 FROM
   "assets" "asset"
   LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id"
@@ -185,6 +189,35 @@ LIMIT
   101
 COMMIT
 
+-- SearchRepository.searchDuplicates
+WITH
+  "cte" AS (
+    SELECT
+      "asset"."duplicateId" AS "duplicateId",
+      "search"."assetId" AS "assetId",
+      "search"."embedding" <= > $1 AS "distance"
+    FROM
+      "assets" "asset"
+      INNER JOIN "smart_search" "search" ON "search"."assetId" = "asset"."id"
+    WHERE
+      (
+        "asset"."ownerId" IN ($2)
+        AND "asset"."id" != $3
+        AND "asset"."isVisible" = $4
+      )
+      AND ("asset"."deletedAt" IS NULL)
+    ORDER BY
+      "search"."embedding" <= > $1 ASC
+    LIMIT
+      64
+  )
+SELECT
+  res.*
+FROM
+  "cte" "res"
+WHERE
+  res.distance <= $5
+
 -- SearchRepository.searchFaces
 START TRANSACTION
 SET
@@ -337,6 +370,7 @@ SELECT
   "asset"."originalFileName" AS "asset_originalFileName",
   "asset"."sidecarPath" AS "asset_sidecarPath",
   "asset"."stackId" AS "asset_stackId",
+  "asset"."duplicateId" AS "asset_duplicateId",
   "exif"."assetId" AS "exif_assetId",
   "exif"."description" AS "exif_description",
   "exif"."exifImageWidth" AS "exif_exifImageWidth",

server/src/queries/shared.link.repository.sql

@@ -49,6 +49,7 @@ FROM
       "SharedLinkEntity__SharedLinkEntity_assets"."originalFileName" AS "SharedLinkEntity__SharedLinkEntity_assets_originalFileName",
       "SharedLinkEntity__SharedLinkEntity_assets"."sidecarPath" AS "SharedLinkEntity__SharedLinkEntity_assets_sidecarPath",
       "SharedLinkEntity__SharedLinkEntity_assets"."stackId" AS "SharedLinkEntity__SharedLinkEntity_assets_stackId",
+      "SharedLinkEntity__SharedLinkEntity_assets"."duplicateId" AS "SharedLinkEntity__SharedLinkEntity_assets_duplicateId",
       "9b1d35b344d838023994a3233afd6ffe098be6d8"."assetId" AS "9b1d35b344d838023994a3233afd6ffe098be6d8_assetId",
       "9b1d35b344d838023994a3233afd6ffe098be6d8"."description" AS "9b1d35b344d838023994a3233afd6ffe098be6d8_description",
       "9b1d35b344d838023994a3233afd6ffe098be6d8"."exifImageWidth" AS "9b1d35b344d838023994a3233afd6ffe098be6d8_exifImageWidth",
@@ -115,6 +116,7 @@ FROM
       "4a35f463ae8c5544ede95c4b6d9ce8c686b6bfe6"."originalFileName" AS "4a35f463ae8c5544ede95c4b6d9ce8c686b6bfe6_originalFileName",
       "4a35f463ae8c5544ede95c4b6d9ce8c686b6bfe6"."sidecarPath" AS "4a35f463ae8c5544ede95c4b6d9ce8c686b6bfe6_sidecarPath",
       "4a35f463ae8c5544ede95c4b6d9ce8c686b6bfe6"."stackId" AS "4a35f463ae8c5544ede95c4b6d9ce8c686b6bfe6_stackId",
+      "4a35f463ae8c5544ede95c4b6d9ce8c686b6bfe6"."duplicateId" AS "4a35f463ae8c5544ede95c4b6d9ce8c686b6bfe6_duplicateId",
       "d9f2f4dd8920bad1d6907cdb1d699732daff3c2f"."assetId" AS "d9f2f4dd8920bad1d6907cdb1d699732daff3c2f_assetId",
       "d9f2f4dd8920bad1d6907cdb1d699732daff3c2f"."description" AS "d9f2f4dd8920bad1d6907cdb1d699732daff3c2f_description",
       "d9f2f4dd8920bad1d6907cdb1d699732daff3c2f"."exifImageWidth" AS "d9f2f4dd8920bad1d6907cdb1d699732daff3c2f_exifImageWidth",
@@ -237,6 +239,7 @@ SELECT
   "SharedLinkEntity__SharedLinkEntity_assets"."originalFileName" AS "SharedLinkEntity__SharedLinkEntity_assets_originalFileName",
   "SharedLinkEntity__SharedLinkEntity_assets"."sidecarPath" AS "SharedLinkEntity__SharedLinkEntity_assets_sidecarPath",
   "SharedLinkEntity__SharedLinkEntity_assets"."stackId" AS "SharedLinkEntity__SharedLinkEntity_assets_stackId",
+  "SharedLinkEntity__SharedLinkEntity_assets"."duplicateId" AS "SharedLinkEntity__SharedLinkEntity_assets_duplicateId",
   "SharedLinkEntity__SharedLinkEntity_album"."id" AS "SharedLinkEntity__SharedLinkEntity_album_id",
   "SharedLinkEntity__SharedLinkEntity_album"."ownerId" AS "SharedLinkEntity__SharedLinkEntity_album_ownerId",
   "SharedLinkEntity__SharedLinkEntity_album"."albumName" AS "SharedLinkEntity__SharedLinkEntity_album_albumName",

server/src/repositories/asset.repository.ts

@@ -18,6 +18,7 @@ import {
   AssetStats,
   AssetStatsOptions,
   AssetUpdateAllOptions,
+  AssetUpdateDuplicateOptions,
   AssetUpdateOptions,
   IAssetRepository,
   LivePhotoSearchOptions,
@@ -73,7 +74,7 @@ export class AssetRepository implements IAssetRepository {
     await this.exifRepository.upsert(exif, { conflictPaths: ['assetId'] });
   }
 
-  async upsertJobStatus(jobStatus: Partial<AssetJobStatusEntity>): Promise<void> {
+  async upsertJobStatus(...jobStatus: Partial<AssetJobStatusEntity>[]): Promise<void> {
     await this.jobStatusRepository.upsert(jobStatus, { conflictPaths: ['assetId'] });
   }
 
@@ -257,6 +258,21 @@ export class AssetRepository implements IAssetRepository {
     await this.repository.update({ id: In(ids) }, options);
   }
 
+  @GenerateSql({
+    params: [{ targetDuplicateId: DummyValue.UUID, duplicateIds: [DummyValue.UUID], assetIds: [DummyValue.UUID] }],
+  })
+  async updateDuplicates(options: AssetUpdateDuplicateOptions): Promise<void> {
+    await this.repository
+      .createQueryBuilder()
+      .update()
+      .set({ duplicateId: options.targetDuplicateId })
+      .where({
+        duplicateId: In(options.duplicateIds),
+      })
+      .orWhere({ id: In(options.assetIds) })
+      .execute();
+  }
+
   @Chunked()
   async softDeleteAll(ids: string[]): Promise<void> {
     await this.repository.softDelete({ id: In(ids) });
@@ -375,6 +391,18 @@ export class AssetRepository implements IAssetRepository {
         break;
       }
 
+      case WithoutProperty.DUPLICATE: {
+        where = {
+          previewPath: Not(IsNull()),
+          isVisible: true,
+          smartSearch: true,
+          jobStatus: {
+            duplicatesDetectedAt: IsNull(),
+          },
+        };
+        break;
+      }
+
       case WithoutProperty.OBJECT_TAGS: {
         relations = {
           smartInfo: true,
@@ -614,6 +642,13 @@ export class AssetRepository implements IAssetRepository {
     );
   }
 
+  @GenerateSql({ params: [{ userIds: [DummyValue.UUID, DummyValue.UUID] }] })
+  getDuplicates(options: AssetBuilderOptions): Promise<AssetEntity[]> {
+    return this.getBuilder({ ...options, isDuplicate: true })
+      .orderBy('asset.duplicateId')
+      .getMany();
+  }
+
   @GenerateSql({ params: [DummyValue.UUID, { minAssetsPerField: 5, maxFields: 12 }] })
   async getAssetIdByCity(
     ownerId: string,
@@ -673,16 +708,14 @@ export class AssetRepository implements IAssetRepository {
   }
 
   private getBuilder(options: AssetBuilderOptions) {
-    const { isArchived, isFavorite, isTrashed, albumId, personId, userIds, withStacked, exifInfo, assetType } = options;
-
     const builder = this.repository.createQueryBuilder('asset').where('asset.isVisible = true');
-    if (assetType !== undefined) {
+    if (options.assetType !== undefined) {
-      builder.andWhere('asset.type = :assetType', { assetType });
+      builder.andWhere('asset.type = :assetType', { assetType: options.assetType });
     }
 
     let stackJoined = false;
 
-    if (exifInfo !== false) {
+    if (options.exifInfo !== false) {
       stackJoined = true;
       builder
         .leftJoinAndSelect('asset.exifInfo', 'exifInfo')
@@ -690,34 +723,38 @@ export class AssetRepository implements IAssetRepository {
         .leftJoinAndSelect('stack.assets', 'stackedAssets');
     }
 
-    if (albumId) {
+    if (options.albumId) {
-      builder.leftJoin('asset.albums', 'album').andWhere('album.id = :albumId', { albumId });
+      builder.leftJoin('asset.albums', 'album').andWhere('album.id = :albumId', { albumId: options.albumId });
     }
 
-    if (userIds) {
+    if (options.userIds) {
-      builder.andWhere('asset.ownerId IN (:...userIds )', { userIds });
+      builder.andWhere('asset.ownerId IN (:...userIds )', { userIds: options.userIds });
     }
 
-    if (isArchived !== undefined) {
+    if (options.isArchived !== undefined) {
-      builder.andWhere('asset.isArchived = :isArchived', { isArchived });
+      builder.andWhere('asset.isArchived = :isArchived', { isArchived: options.isArchived });
     }
 
-    if (isFavorite !== undefined) {
+    if (options.isFavorite !== undefined) {
-      builder.andWhere('asset.isFavorite = :isFavorite', { isFavorite });
+      builder.andWhere('asset.isFavorite = :isFavorite', { isFavorite: options.isFavorite });
     }
 
-    if (isTrashed !== undefined) {
+    if (options.isTrashed !== undefined) {
-      builder.andWhere(`asset.deletedAt ${isTrashed ? 'IS NOT NULL' : 'IS NULL'}`).withDeleted();
+      builder.andWhere(`asset.deletedAt ${options.isTrashed ? 'IS NOT NULL' : 'IS NULL'}`).withDeleted();
     }
 
-    if (personId !== undefined) {
+    if (options.isDuplicate !== undefined) {
+      builder.andWhere(`asset.duplicateId ${options.isDuplicate ? 'IS NOT NULL' : 'IS NULL'}`);
+    }
+
+    if (options.personId !== undefined) {
       builder
         .innerJoin('asset.faces', 'faces')
         .innerJoin('faces.person', 'person')
-        .andWhere('person.id = :personId', { personId });
+        .andWhere('person.id = :personId', { personId: options.personId });
     }
 
-    if (withStacked) {
+    if (options.withStacked) {
       if (!stackJoined) {
         builder.leftJoinAndSelect('asset.stack', 'stack').leftJoinAndSelect('stack.assets', 'stackedAssets');
       }

server/src/repositories/job.repository.ts

@@ -65,6 +65,10 @@ export const JOBS_TO_QUEUE: Record<JobName, QueueName> = {
   [JobName.QUEUE_SMART_SEARCH]: QueueName.SMART_SEARCH,
   [JobName.SMART_SEARCH]: QueueName.SMART_SEARCH,
 
+  // duplicate detection
+  [JobName.QUEUE_DUPLICATE_DETECTION]: QueueName.DUPLICATE_DETECTION,
+  [JobName.DUPLICATE_DETECTION]: QueueName.DUPLICATE_DETECTION,
+
   // XMP sidecars
   [JobName.QUEUE_SIDECAR]: QueueName.SIDECAR,
   [JobName.SIDECAR_DISCOVERY]: QueueName.SIDECAR,

server/src/repositories/search.repository.ts

@@ -10,6 +10,8 @@ import { SmartSearchEntity } from 'src/entities/smart-search.entity';
 import { DatabaseExtension } from 'src/interfaces/database.interface';
 import { ILoggerRepository } from 'src/interfaces/logger.interface';
 import {
+  AssetDuplicateResult,
+  AssetDuplicateSearch,
   AssetSearchOptions,
   FaceEmbeddingSearch,
   FaceSearchResult,
@@ -145,6 +147,44 @@ export class SearchRepository implements ISearchRepository {
     return results;
   }
 
+  @GenerateSql({
+    params: [
+      {
+        embedding: Array.from({ length: 512 }, Math.random),
+        maxDistance: 0.6,
+        userIds: [DummyValue.UUID],
+      },
+    ],
+  })
+  searchDuplicates({
+    assetId,
+    embedding,
+    maxDistance,
+    userIds,
+  }: AssetDuplicateSearch): Promise<AssetDuplicateResult[]> {
+    const cte = this.assetRepository.createQueryBuilder('asset');
+    cte
+      .select('search.assetId', 'assetId')
+      .addSelect('asset.duplicateId', 'duplicateId')
+      .addSelect(`search.embedding <=> :embedding`, 'distance')
+      .innerJoin('asset.smartSearch', 'search')
+      .where('asset.ownerId IN (:...userIds )')
+      .andWhere('asset.id != :assetId')
+      .andWhere('asset.isVisible = :isVisible')
+      .orderBy('search.embedding <=> :embedding')
+      .limit(64)
+      .setParameters({ assetId, embedding: asVector(embedding), isVisible: true, userIds });
+
+    const builder = this.assetRepository.manager
+      .createQueryBuilder()
+      .addCommonTableExpression(cte, 'cte')
+      .from('cte', 'res')
+      .select('res.*')
+      .where('res.distance <= :maxDistance', { maxDistance });
+
+    return builder.getRawMany() as any as Promise<AssetDuplicateResult[]>;
+  }
+
   @GenerateSql({
     params: [
       {

server/src/services/asset.service.ts

@@ -286,6 +286,11 @@ export class AssetService {
     return data;
   }
 
+  async getDuplicates(auth: AuthDto): Promise<AssetResponseDto[]> {
+    const res = await this.assetRepository.getDuplicates({ userIds: [auth.user.id] });
+    return res.map((a) => mapAsset(a, { auth }));
+  }
+
   async update(auth: AuthDto, id: string, dto: UpdateAssetDto): Promise<AssetResponseDto> {
     await this.access.requirePermission(auth, Permission.ASSET_UPDATE, id);
 

server/src/services/job.service.spec.ts

@@ -109,6 +109,7 @@ describe(JobService.name, () => {
 
       await expect(sut.getAllJobsStatus()).resolves.toEqual({
         [QueueName.BACKGROUND_TASK]: expectedJobStatus,
+        [QueueName.DUPLICATE_DETECTION]: expectedJobStatus,
         [QueueName.SMART_SEARCH]: expectedJobStatus,
         [QueueName.METADATA_EXTRACTION]: expectedJobStatus,
         [QueueName.SEARCH]: expectedJobStatus,

server/src/services/job.service.ts

@@ -115,6 +115,10 @@ export class JobService {
         return this.jobRepository.queue({ name: JobName.QUEUE_SMART_SEARCH, data: { force } });
       }
 
+      case QueueName.DUPLICATE_DETECTION: {
+        return this.jobRepository.queue({ name: JobName.QUEUE_DUPLICATE_DETECTION, data: { force } });
+      }
+
       case QueueName.METADATA_EXTRACTION: {
         return this.jobRepository.queue({ name: JobName.QUEUE_METADATA_EXTRACTION, data: { force } });
       }
@@ -191,7 +195,11 @@ export class JobService {
   }
 
   private isConcurrentQueue(name: QueueName): name is ConcurrentQueueName {
-    return ![QueueName.FACIAL_RECOGNITION, QueueName.STORAGE_TEMPLATE_MIGRATION].includes(name);
+    return ![
+      QueueName.FACIAL_RECOGNITION,
+      QueueName.STORAGE_TEMPLATE_MIGRATION,
+      QueueName.DUPLICATE_DETECTION,
+    ].includes(name);
   }
 
   async handleNightlyJobs() {
@@ -294,6 +302,13 @@ export class JobService {
         break;
       }
 
+      case JobName.SMART_SEARCH: {
+        if (item.data.source === 'upload') {
+          await this.jobRepository.queue({ name: JobName.DUPLICATE_DETECTION, data: item.data });
+        }
+        break;
+      }
+
       case JobName.USER_DELETION: {
         this.eventRepository.clientBroadcast(ClientEvent.USER_DELETE, item.data.id);
         break;

server/src/services/microservices.service.ts

@@ -9,6 +9,7 @@ import { MediaService } from 'src/services/media.service';
 import { MetadataService } from 'src/services/metadata.service';
 import { NotificationService } from 'src/services/notification.service';
 import { PersonService } from 'src/services/person.service';
+import { SearchService } from 'src/services/search.service';
 import { SessionService } from 'src/services/session.service';
 import { SmartInfoService } from 'src/services/smart-info.service';
 import { StorageTemplateService } from 'src/services/storage-template.service';
@@ -35,6 +36,7 @@ export class MicroservicesService {
     private storageTemplateService: StorageTemplateService,
     private storageService: StorageService,
     private userService: UserService,
+    private searchService: SearchService,
   ) {}
 
   async init() {
@@ -53,6 +55,8 @@ export class MicroservicesService {
       [JobName.USER_SYNC_USAGE]: () => this.userService.handleUserSyncUsage(),
       [JobName.QUEUE_SMART_SEARCH]: (data) => this.smartInfoService.handleQueueEncodeClip(data),
       [JobName.SMART_SEARCH]: (data) => this.smartInfoService.handleEncodeClip(data),
+      [JobName.QUEUE_DUPLICATE_DETECTION]: (data) => this.searchService.handleQueueSearchDuplicates(data),
+      [JobName.DUPLICATE_DETECTION]: (data) => this.searchService.handleSearchDuplicates(data),
       [JobName.STORAGE_TEMPLATE_MIGRATION]: () => this.storageTemplateService.handleMigration(),
       [JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE]: (data) => this.storageTemplateService.handleMigrationSingle(data),
       [JobName.QUEUE_MIGRATION]: () => this.mediaService.handleQueueMigration(),

server/src/services/search.service.spec.ts

@@ -1,5 +1,7 @@
 import { mapAsset } from 'src/dtos/asset-response.dto';
-import { IAssetRepository } from 'src/interfaces/asset.interface';
+import { IAssetRepository, WithoutProperty } from 'src/interfaces/asset.interface';
+import { ICryptoRepository } from 'src/interfaces/crypto.interface';
+import { IJobRepository, JobName, JobStatus } from 'src/interfaces/job.interface';
 import { ILoggerRepository } from 'src/interfaces/logger.interface';
 import { IMachineLearningRepository } from 'src/interfaces/machine-learning.interface';
 import { IMetadataRepository } from 'src/interfaces/metadata.interface';
@@ -12,6 +14,8 @@ import { assetStub } from 'test/fixtures/asset.stub';
 import { authStub } from 'test/fixtures/auth.stub';
 import { personStub } from 'test/fixtures/person.stub';
 import { newAssetRepositoryMock } from 'test/repositories/asset.repository.mock';
+import { newCryptoRepositoryMock } from 'test/repositories/crypto.repository.mock';
+import { newJobRepositoryMock } from 'test/repositories/job.repository.mock';
 import { newLoggerRepositoryMock } from 'test/repositories/logger.repository.mock';
 import { newMachineLearningRepositoryMock } from 'test/repositories/machine-learning.repository.mock';
 import { newMetadataRepositoryMock } from 'test/repositories/metadata.repository.mock';
@@ -19,7 +23,7 @@ import { newPartnerRepositoryMock } from 'test/repositories/partner.repository.m
 import { newPersonRepositoryMock } from 'test/repositories/person.repository.mock';
 import { newSearchRepositoryMock } from 'test/repositories/search.repository.mock';
 import { newSystemMetadataRepositoryMock } from 'test/repositories/system-metadata.repository.mock';
-import { Mocked, vitest } from 'vitest';
+import { Mocked, beforeEach, vitest } from 'vitest';
 
 vitest.useFakeTimers();
 
@@ -33,6 +37,8 @@ describe(SearchService.name, () => {
   let partnerMock: Mocked<IPartnerRepository>;
   let metadataMock: Mocked<IMetadataRepository>;
   let loggerMock: Mocked<ILoggerRepository>;
+  let cryptoMock: Mocked<ICryptoRepository>;
+  let jobMock: Mocked<IJobRepository>;
 
   beforeEach(() => {
     assetMock = newAssetRepositoryMock();
@@ -43,6 +49,8 @@ describe(SearchService.name, () => {
     partnerMock = newPartnerRepositoryMock();
     metadataMock = newMetadataRepositoryMock();
     loggerMock = newLoggerRepositoryMock();
+    cryptoMock = newCryptoRepositoryMock();
+    jobMock = newJobRepositoryMock();
 
     sut = new SearchService(
       systemMock,
@@ -53,6 +61,8 @@ describe(SearchService.name, () => {
       partnerMock,
       metadataMock,
       loggerMock,
+      cryptoMock,
+      jobMock,
     );
   });
 
@@ -76,15 +86,15 @@ describe(SearchService.name, () => {
 
   describe('getExploreData', () => {
     it('should get assets by city and tag', async () => {
-      assetMock.getAssetIdByCity.mockResolvedValueOnce({
+      assetMock.getAssetIdByCity.mockResolvedValue({
         fieldName: 'exifInfo.city',
         items: [{ value: 'Paris', data: assetStub.image.id }],
       });
-      assetMock.getAssetIdByTag.mockResolvedValueOnce({
+      assetMock.getAssetIdByTag.mockResolvedValue({
         fieldName: 'smartInfo.tags',
         items: [{ value: 'train', data: assetStub.imageFrom2015.id }],
       });
-      assetMock.getByIdsWithAllRelations.mockResolvedValueOnce([assetStub.image, assetStub.imageFrom2015]);
+      assetMock.getByIdsWithAllRelations.mockResolvedValue([assetStub.image, assetStub.imageFrom2015]);
       const expectedResponse = [
         { fieldName: 'exifInfo.city', items: [{ value: 'Paris', data: mapAsset(assetStub.image) }] },
         { fieldName: 'smartInfo.tags', items: [{ value: 'train', data: mapAsset(assetStub.imageFrom2015) }] },
@@ -95,4 +105,234 @@ describe(SearchService.name, () => {
       expect(result).toEqual(expectedResponse);
     });
   });
+
+  describe('handleQueueSearchDuplicates', () => {
+    beforeEach(() => {
+      systemMock.get.mockResolvedValue({
+        machineLearning: {
+          enabled: true,
+          duplicateDetection: {
+            enabled: true,
+          },
+        },
+      });
+    });
+
+    it('should skip if machine learning is disabled', async () => {
+      systemMock.get.mockResolvedValue({
+        machineLearning: {
+          enabled: false,
+          duplicateDetection: {
+            enabled: true,
+          },
+        },
+      });
+
+      await expect(sut.handleQueueSearchDuplicates({})).resolves.toBe(JobStatus.SKIPPED);
+      expect(jobMock.queue).not.toHaveBeenCalled();
+      expect(jobMock.queueAll).not.toHaveBeenCalled();
+      expect(systemMock.get).toHaveBeenCalled();
+    });
+
+    it('should skip if duplicate detection is disabled', async () => {
+      systemMock.get.mockResolvedValue({
+        machineLearning: {
+          enabled: true,
+          duplicateDetection: {
+            enabled: false,
+          },
+        },
+      });
+
+      await expect(sut.handleQueueSearchDuplicates({})).resolves.toBe(JobStatus.SKIPPED);
+      expect(jobMock.queue).not.toHaveBeenCalled();
+      expect(jobMock.queueAll).not.toHaveBeenCalled();
+      expect(systemMock.get).toHaveBeenCalled();
+    });
+
+    it('should queue missing assets', async () => {
+      assetMock.getWithout.mockResolvedValue({
+        items: [assetStub.image],
+        hasNextPage: false,
+      });
+
+      await sut.handleQueueSearchDuplicates({});
+
+      expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.DUPLICATE);
+      expect(jobMock.queueAll).toHaveBeenCalledWith([
+        {
+          name: JobName.DUPLICATE_DETECTION,
+          data: { id: assetStub.image.id },
+        },
+      ]);
+    });
+
+    it('should queue all assets', async () => {
+      assetMock.getAll.mockResolvedValue({
+        items: [assetStub.image],
+        hasNextPage: false,
+      });
+      personMock.getAll.mockResolvedValue({
+        items: [personStub.withName],
+        hasNextPage: false,
+      });
+
+      await sut.handleQueueSearchDuplicates({ force: true });
+
+      expect(assetMock.getAll).toHaveBeenCalled();
+      expect(jobMock.queueAll).toHaveBeenCalledWith([
+        {
+          name: JobName.DUPLICATE_DETECTION,
+          data: { id: assetStub.image.id },
+        },
+      ]);
+    });
+  });
+
+  describe('handleSearchDuplicates', () => {
+    beforeEach(() => {
+      systemMock.get.mockResolvedValue({
+        machineLearning: {
+          enabled: true,
+          duplicateDetection: {
+            enabled: true,
+          },
+        },
+      });
+    });
+
+    it('should skip if machine learning is disabled', async () => {
+      systemMock.get.mockResolvedValue({
+        machineLearning: {
+          enabled: false,
+          duplicateDetection: {
+            enabled: true,
+          },
+        },
+      });
+      const id = assetStub.livePhotoMotionAsset.id;
+      assetMock.getById.mockResolvedValue(assetStub.livePhotoMotionAsset);
+
+      const result = await sut.handleSearchDuplicates({ id });
+
+      expect(result).toBe(JobStatus.SKIPPED);
+    });
+
+    it('should skip if duplicate detection is disabled', async () => {
+      systemMock.get.mockResolvedValue({
+        machineLearning: {
+          enabled: true,
+          duplicateDetection: {
+            enabled: false,
+          },
+        },
+      });
+      const id = assetStub.livePhotoMotionAsset.id;
+      assetMock.getById.mockResolvedValue(assetStub.livePhotoMotionAsset);
+
+      const result = await sut.handleSearchDuplicates({ id });
+
+      expect(result).toBe(JobStatus.SKIPPED);
+    });
+
+    it('should fail if asset is not found', async () => {
+      const result = await sut.handleSearchDuplicates({ id: assetStub.image.id });
+
+      expect(result).toBe(JobStatus.FAILED);
+      expect(loggerMock.error).toHaveBeenCalledWith(`Asset ${assetStub.image.id} not found`);
+    });
+
+    it('should skip if asset is not visible', async () => {
+      const id = assetStub.livePhotoMotionAsset.id;
+      assetMock.getById.mockResolvedValue(assetStub.livePhotoMotionAsset);
+
+      const result = await sut.handleSearchDuplicates({ id });
+
+      expect(result).toBe(JobStatus.SKIPPED);
+      expect(loggerMock.debug).toHaveBeenCalledWith(`Asset ${id} is not visible, skipping`);
+    });
+
+    it('should fail if asset is missing preview image', async () => {
+      assetMock.getById.mockResolvedValue(assetStub.noResizePath);
+
+      const result = await sut.handleSearchDuplicates({ id: assetStub.noResizePath.id });
+
+      expect(result).toBe(JobStatus.FAILED);
+      expect(loggerMock.warn).toHaveBeenCalledWith(`Asset ${assetStub.noResizePath.id} is missing preview image`);
+    });
+
+    it('should fail if asset is missing embedding', async () => {
+      assetMock.getById.mockResolvedValue(assetStub.image);
+
+      const result = await sut.handleSearchDuplicates({ id: assetStub.image.id });
+
+      expect(result).toBe(JobStatus.FAILED);
+      expect(loggerMock.debug).toHaveBeenCalledWith(`Asset ${assetStub.image.id} is missing embedding`);
+    });
+
+    it('should search for duplicates and update asset with duplicateId', async () => {
+      assetMock.getById.mockResolvedValue(assetStub.hasEmbedding);
+      searchMock.searchDuplicates.mockResolvedValue([
+        { assetId: assetStub.image.id, distance: 0.01, duplicateId: null },
+      ]);
+      const expectedAssetIds = [assetStub.image.id, assetStub.hasEmbedding.id];
+
+      const result = await sut.handleSearchDuplicates({ id: assetStub.hasEmbedding.id });
+
+      expect(result).toBe(JobStatus.SUCCESS);
+      expect(searchMock.searchDuplicates).toHaveBeenCalledWith({
+        assetId: assetStub.hasEmbedding.id,
+        embedding: assetStub.hasEmbedding.smartSearch!.embedding,
+        maxDistance: 0.03,
+        userIds: [assetStub.hasEmbedding.ownerId],
+      });
+      expect(assetMock.updateDuplicates).toHaveBeenCalledWith({
+        assetIds: expectedAssetIds,
+        targetDuplicateId: expect.any(String),
+        duplicateIds: [],
+      });
+      expect(assetMock.upsertJobStatus).toHaveBeenCalledWith(
+        ...expectedAssetIds.map((assetId) => ({ assetId, duplicatesDetectedAt: expect.any(Date) })),
+      );
+    });
+
+    it('should use existing duplicate ID among matched duplicates', async () => {
+      const duplicateId = assetStub.hasDupe.duplicateId;
+      assetMock.getById.mockResolvedValue(assetStub.hasEmbedding);
+      searchMock.searchDuplicates.mockResolvedValue([{ assetId: assetStub.hasDupe.id, distance: 0.01, duplicateId }]);
+      const expectedAssetIds = [assetStub.hasEmbedding.id];
+
+      const result = await sut.handleSearchDuplicates({ id: assetStub.hasEmbedding.id });
+
+      expect(result).toBe(JobStatus.SUCCESS);
+      expect(searchMock.searchDuplicates).toHaveBeenCalledWith({
+        assetId: assetStub.hasEmbedding.id,
+        embedding: assetStub.hasEmbedding.smartSearch!.embedding,
+        maxDistance: 0.03,
+        userIds: [assetStub.hasEmbedding.ownerId],
+      });
+      expect(assetMock.updateDuplicates).toHaveBeenCalledWith({
+        assetIds: expectedAssetIds,
+        targetDuplicateId: assetStub.hasDupe.duplicateId,
+        duplicateIds: [],
+      });
+      expect(assetMock.upsertJobStatus).toHaveBeenCalledWith(
+        ...expectedAssetIds.map((assetId) => ({ assetId, duplicatesDetectedAt: expect.any(Date) })),
+      );
+    });
+
+    it('should remove duplicateId if no duplicates found and asset has duplicateId', async () => {
+      assetMock.getById.mockResolvedValue(assetStub.hasDupe);
+      searchMock.searchDuplicates.mockResolvedValue([]);
+
+      const result = await sut.handleSearchDuplicates({ id: assetStub.hasDupe.id });
+
+      expect(result).toBe(JobStatus.SUCCESS);
+      expect(assetMock.update).toHaveBeenCalledWith({ id: assetStub.hasDupe.id, duplicateId: null });
+      expect(assetMock.upsertJobStatus).toHaveBeenCalledWith({
+        assetId: assetStub.hasDupe.id,
+        duplicatesDetectedAt: expect.any(Date),
+      });
+    });
+  });
 });

server/src/services/search.service.ts

@@ -16,15 +16,25 @@ import {
 } from 'src/dtos/search.dto';
 import { AssetOrder } from 'src/entities/album.entity';
 import { AssetEntity } from 'src/entities/asset.entity';
-import { IAssetRepository } from 'src/interfaces/asset.interface';
+import { IAssetRepository, WithoutProperty } from 'src/interfaces/asset.interface';
+import { ICryptoRepository } from 'src/interfaces/crypto.interface';
+import {
+  IBaseJob,
+  IEntityJob,
+  IJobRepository,
+  JOBS_ASSET_PAGINATION_SIZE,
+  JobName,
+  JobStatus,
+} from 'src/interfaces/job.interface';
 import { ILoggerRepository } from 'src/interfaces/logger.interface';
 import { IMachineLearningRepository } from 'src/interfaces/machine-learning.interface';
 import { IMetadataRepository } from 'src/interfaces/metadata.interface';
 import { IPartnerRepository } from 'src/interfaces/partner.interface';
 import { IPersonRepository } from 'src/interfaces/person.interface';
-import { ISearchRepository, SearchExploreItem } from 'src/interfaces/search.interface';
+import { AssetDuplicateResult, ISearchRepository, SearchExploreItem } from 'src/interfaces/search.interface';
 import { ISystemMetadataRepository } from 'src/interfaces/system-metadata.interface';
-import { isSmartSearchEnabled } from 'src/utils/misc';
+import { isDuplicateDetectionEnabled, isSmartSearchEnabled } from 'src/utils/misc';
+import { usePagination } from 'src/utils/pagination';
 
 @Injectable()
 export class SearchService {
@@ -39,6 +49,8 @@ export class SearchService {
     @Inject(IPartnerRepository) private partnerRepository: IPartnerRepository,
     @Inject(IMetadataRepository) private metadataRepository: IMetadataRepository,
     @Inject(ILoggerRepository) private logger: ILoggerRepository,
+    @Inject(ICryptoRepository) private cryptoRepository: ICryptoRepository,
+    @Inject(IJobRepository) private jobRepository: IJobRepository,
   ) {
     this.logger.setContext(SearchService.name);
     this.configCore = SystemConfigCore.create(systemMetadataRepository, logger);
@@ -147,6 +159,97 @@ export class SearchService {
     }
   }
 
+  async handleQueueSearchDuplicates({ force }: IBaseJob): Promise<JobStatus> {
+    const { machineLearning } = await this.configCore.getConfig();
+    if (!isDuplicateDetectionEnabled(machineLearning)) {
+      return JobStatus.SKIPPED;
+    }
+
+    const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => {
+      return force
+        ? this.assetRepository.getAll(pagination, { isVisible: true })
+        : this.assetRepository.getWithout(pagination, WithoutProperty.DUPLICATE);
+    });
+
+    for await (const assets of assetPagination) {
+      await this.jobRepository.queueAll(
+        assets.map((asset) => ({ name: JobName.DUPLICATE_DETECTION, data: { id: asset.id } })),
+      );
+    }
+
+    return JobStatus.SUCCESS;
+  }
+
+  async handleSearchDuplicates({ id }: IEntityJob): Promise<JobStatus> {
+    const { machineLearning } = await this.configCore.getConfig();
+    if (!isDuplicateDetectionEnabled(machineLearning)) {
+      return JobStatus.SKIPPED;
+    }
+
+    const asset = await this.assetRepository.getById(id, { smartSearch: true });
+    if (!asset) {
+      this.logger.error(`Asset ${id} not found`);
+      return JobStatus.FAILED;
+    }
+
+    if (!asset.isVisible) {
+      this.logger.debug(`Asset ${id} is not visible, skipping`);
+      return JobStatus.SKIPPED;
+    }
+
+    if (!asset.previewPath) {
+      this.logger.warn(`Asset ${id} is missing preview image`);
+      return JobStatus.FAILED;
+    }
+
+    if (!asset.smartSearch?.embedding) {
+      this.logger.debug(`Asset ${id} is missing embedding`);
+      return JobStatus.FAILED;
+    }
+
+    const duplicateAssets = await this.searchRepository.searchDuplicates({
+      assetId: asset.id,
+      embedding: asset.smartSearch.embedding,
+      maxDistance: machineLearning.duplicateDetection.maxDistance,
+      userIds: [asset.ownerId],
+    });
+
+    let assetIds = [asset.id];
+    if (duplicateAssets.length > 0) {
+      this.logger.debug(
+        `Found ${duplicateAssets.length} duplicate${duplicateAssets.length === 1 ? '' : 's'} for asset ${asset.id}`,
+      );
+      assetIds = await this.updateDuplicates(asset, duplicateAssets);
+    } else if (asset.duplicateId) {
+      this.logger.debug(`No duplicates found for asset ${asset.id}, removing duplicateId`);
+      await this.assetRepository.update({ id: asset.id, duplicateId: null });
+    }
+
+    const duplicatesDetectedAt = new Date();
+    await this.assetRepository.upsertJobStatus(...assetIds.map((assetId) => ({ assetId, duplicatesDetectedAt })));
+
+    return JobStatus.SUCCESS;
+  }
+
+  private async updateDuplicates(asset: AssetEntity, duplicateAssets: AssetDuplicateResult[]): Promise<string[]> {
+    const duplicateIds = [
+      ...new Set(
+        duplicateAssets
+          .filter((asset): asset is AssetDuplicateResult & { duplicateId: string } => !!asset.duplicateId)
+          .map((duplicate) => duplicate.duplicateId),
+      ),
+    ];
+
+    const targetDuplicateId = asset.duplicateId ?? duplicateIds.shift() ?? this.cryptoRepository.randomUUID();
+    const assetIdsToUpdate = duplicateAssets
+      .filter((asset) => asset.duplicateId !== targetDuplicateId)
+      .map((duplicate) => duplicate.assetId);
+    assetIdsToUpdate.push(asset.id);
+
+    await this.assetRepository.updateDuplicates({ targetDuplicateId, assetIds: assetIdsToUpdate, duplicateIds });
+    return assetIdsToUpdate;
+  }
+
   private async getUserIdsToSearch(auth: AuthDto): Promise<string[]> {
     const userIds: string[] = [auth.user.id];
     const partners = await this.partnerRepository.getAll(auth.user.id);

server/src/services/server-info.service.spec.ts

@@ -164,6 +164,7 @@ describe(ServerInfoService.name, () => {
     it('should respond the server features', async () => {
       await expect(sut.getFeatures()).resolves.toEqual({
         smartSearch: true,
+        duplicateDetection: false,
         facialRecognition: true,
         map: true,
         reverseGeocoding: true,

server/src/services/server-info.service.ts

@@ -22,7 +22,7 @@ import { ISystemMetadataRepository } from 'src/interfaces/system-metadata.interf
 import { IUserRepository, UserStatsQueryResponse } from 'src/interfaces/user.interface';
 import { asHumanReadable } from 'src/utils/bytes';
 import { mimeTypes } from 'src/utils/mime-types';
-import { isFacialRecognitionEnabled, isSmartSearchEnabled } from 'src/utils/misc';
+import { isDuplicateDetectionEnabled, isFacialRecognitionEnabled, isSmartSearchEnabled } from 'src/utils/misc';
 import { Version } from 'src/utils/version';
 
 @Injectable()
@@ -88,6 +88,7 @@ export class ServerInfoService {
     return {
       smartSearch: isSmartSearchEnabled(machineLearning),
       facialRecognition: isFacialRecognitionEnabled(machineLearning),
+      duplicateDetection: isDuplicateDetectionEnabled(machineLearning),
       map: map.enabled,
       reverseGeocoding: reverseGeocoding.enabled,
       sidecar: true,

server/src/services/system-config.service.spec.ts

@@ -79,6 +79,10 @@ const updatedConfig = Object.freeze<SystemConfig>({
       enabled: true,
       modelName: 'ViT-B-32__openai',
     },
+    duplicateDetection: {
+      enabled: false,
+      maxDistance: 0.03,
+    },
     facialRecognition: {
       enabled: true,
       modelName: 'buffalo_l',

server/src/utils/misc.ts

@@ -62,6 +62,8 @@ export const isSmartSearchEnabled = (machineLearning: SystemConfig['machineLearn
   isMachineLearningEnabled(machineLearning) && machineLearning.clip.enabled;
 export const isFacialRecognitionEnabled = (machineLearning: SystemConfig['machineLearning']) =>
   isMachineLearningEnabled(machineLearning) && machineLearning.facialRecognition.enabled;
+export const isDuplicateDetectionEnabled = (machineLearning: SystemConfig['machineLearning']) =>
+  isMachineLearningEnabled(machineLearning) && machineLearning.duplicateDetection.enabled;
 
 export const isConnectionAborted = (error: Error | any) => error.code === 'ECONNABORTED';
 

server/test/fixtures/asset.stub.ts

@@ -50,6 +50,7 @@ export const assetStub = {
     isExternal: false,
     libraryId: 'library-id',
     library: libraryStub.uploadLibrary1,
+    duplicateId: null,
   }),
 
   noWebpPath: Object.freeze<AssetEntity>({
@@ -89,6 +90,7 @@ export const assetStub = {
       fileSizeInByte: 123_000,
     } as ExifEntity,
     deletedAt: null,
+    duplicateId: null,
   }),
 
   noThumbhash: Object.freeze<AssetEntity>({
@@ -125,6 +127,7 @@ export const assetStub = {
     faces: [],
     sidecarPath: null,
     deletedAt: null,
+    duplicateId: null,
   }),
 
   primaryImage: Object.freeze<AssetEntity>({
@@ -171,6 +174,7 @@ export const assetStub = {
       { id: 'stack-child-asset-1' } as AssetEntity,
       { id: 'stack-child-asset-2' } as AssetEntity,
     ]),
+    duplicateId: null,
   }),
 
   image: Object.freeze<AssetEntity>({
@@ -212,6 +216,7 @@ export const assetStub = {
       exifImageHeight: 3840,
       exifImageWidth: 2160,
     } as ExifEntity,
+    duplicateId: null,
   }),
 
   external: Object.freeze<AssetEntity>({
@@ -251,6 +256,7 @@ export const assetStub = {
     exifInfo: {
       fileSizeInByte: 5000,
     } as ExifEntity,
+    duplicateId: null,
   }),
 
   offline: Object.freeze<AssetEntity>({
@@ -290,6 +296,7 @@ export const assetStub = {
       fileSizeInByte: 5000,
     } as ExifEntity,
     deletedAt: null,
+    duplicateId: null,
   }),
 
   externalOffline: Object.freeze<AssetEntity>({
@@ -329,6 +336,7 @@ export const assetStub = {
       fileSizeInByte: 5000,
     } as ExifEntity,
     deletedAt: null,
+    duplicateId: null,
   }),
 
   image1: Object.freeze<AssetEntity>({
@@ -368,6 +376,7 @@ export const assetStub = {
     exifInfo: {
       fileSizeInByte: 5000,
     } as ExifEntity,
+    duplicateId: null,
   }),
 
   imageFrom2015: Object.freeze<AssetEntity>({
@@ -407,6 +416,7 @@ export const assetStub = {
       fileSizeInByte: 5000,
     } as ExifEntity,
     deletedAt: null,
+    duplicateId: null,
   }),
 
   video: Object.freeze<AssetEntity>({
@@ -446,6 +456,7 @@ export const assetStub = {
       fileSizeInByte: 100_000,
     } as ExifEntity,
     deletedAt: null,
+    duplicateId: null,
   }),
 
   livePhotoMotionAsset: Object.freeze({
@@ -541,6 +552,7 @@ export const assetStub = {
       country: 'test-country',
     } as ExifEntity,
     deletedAt: null,
+    duplicateId: null,
   }),
   sidecar: Object.freeze<AssetEntity>({
     id: 'asset-id',
@@ -576,6 +588,7 @@ export const assetStub = {
     faces: [],
     sidecarPath: '/original/path.ext.xmp',
     deletedAt: null,
+    duplicateId: null,
   }),
   sidecarWithoutExt: Object.freeze<AssetEntity>({
     id: 'asset-id',
@@ -611,6 +624,7 @@ export const assetStub = {
     faces: [],
     sidecarPath: '/original/path.xmp',
     deletedAt: null,
+    duplicateId: null,
   }),
 
   readOnly: Object.freeze<AssetEntity>({
@@ -647,6 +661,7 @@ export const assetStub = {
     faces: [],
     sidecarPath: '/original/path.ext.xmp',
     deletedAt: null,
+    duplicateId: null,
   }),
 
   hasEncodedVideo: Object.freeze<AssetEntity>({
@@ -686,6 +701,7 @@ export const assetStub = {
       fileSizeInByte: 100_000,
     } as ExifEntity,
     deletedAt: null,
+    duplicateId: null,
   }),
   missingFileExtension: Object.freeze<AssetEntity>({
     id: 'asset-id',
@@ -724,6 +740,7 @@ export const assetStub = {
     exifInfo: {
       fileSizeInByte: 5000,
     } as ExifEntity,
+    duplicateId: null,
   }),
   hasFileExtension: Object.freeze<AssetEntity>({
     id: 'asset-id',
@@ -762,6 +779,7 @@ export const assetStub = {
     exifInfo: {
       fileSizeInByte: 5000,
     } as ExifEntity,
+    duplicateId: null,
   }),
   imageDng: Object.freeze<AssetEntity>({
     id: 'asset-id',
@@ -802,5 +820,92 @@ export const assetStub = {
       profileDescription: 'Adobe RGB',
       bitsPerSample: 14,
     } as ExifEntity,
+    duplicateId: null,
+  }),
+  hasEmbedding: Object.freeze<AssetEntity>({
+    id: 'asset-id-embedding',
+    deviceAssetId: 'device-asset-id',
+    fileModifiedAt: new Date('2023-02-23T05:06:29.716Z'),
+    fileCreatedAt: new Date('2023-02-23T05:06:29.716Z'),
+    owner: userStub.user1,
+    ownerId: 'user-id',
+    deviceId: 'device-id',
+    originalPath: '/original/path.jpg',
+    previewPath: '/uploads/user-id/thumbs/path.jpg',
+    checksum: Buffer.from('file hash', 'utf8'),
+    type: AssetType.IMAGE,
+    thumbnailPath: '/uploads/user-id/webp/path.ext',
+    thumbhash: Buffer.from('blablabla', 'base64'),
+    encodedVideoPath: null,
+    createdAt: new Date('2023-02-23T05:06:29.716Z'),
+    updatedAt: new Date('2023-02-23T05:06:29.716Z'),
+    localDateTime: new Date('2023-02-23T05:06:29.716Z'),
+    isFavorite: true,
+    isArchived: false,
+    duration: null,
+    isVisible: true,
+    isExternal: false,
+    livePhotoVideo: null,
+    livePhotoVideoId: null,
+    isOffline: false,
+    libraryId: 'library-id',
+    library: libraryStub.uploadLibrary1,
+    tags: [],
+    sharedLinks: [],
+    originalFileName: 'asset-id.jpg',
+    faces: [],
+    deletedAt: null,
+    sidecarPath: null,
+    exifInfo: {
+      fileSizeInByte: 5000,
+    } as ExifEntity,
+    duplicateId: null,
+    smartSearch: {
+      assetId: 'asset-id',
+      embedding: Array.from({ length: 512 }, Math.random),
+    },
+  }),
+  hasDupe: Object.freeze<AssetEntity>({
+    id: 'asset-id-dupe',
+    deviceAssetId: 'device-asset-id',
+    fileModifiedAt: new Date('2023-02-23T05:06:29.716Z'),
+    fileCreatedAt: new Date('2023-02-23T05:06:29.716Z'),
+    owner: userStub.user1,
+    ownerId: 'user-id',
+    deviceId: 'device-id',
+    originalPath: '/original/path.jpg',
+    previewPath: '/uploads/user-id/thumbs/path.jpg',
+    checksum: Buffer.from('file hash', 'utf8'),
+    type: AssetType.IMAGE,
+    thumbnailPath: '/uploads/user-id/webp/path.ext',
+    thumbhash: Buffer.from('blablabla', 'base64'),
+    encodedVideoPath: null,
+    createdAt: new Date('2023-02-23T05:06:29.716Z'),
+    updatedAt: new Date('2023-02-23T05:06:29.716Z'),
+    localDateTime: new Date('2023-02-23T05:06:29.716Z'),
+    isFavorite: true,
+    isArchived: false,
+    duration: null,
+    isVisible: true,
+    isExternal: false,
+    livePhotoVideo: null,
+    livePhotoVideoId: null,
+    isOffline: false,
+    libraryId: 'library-id',
+    library: libraryStub.uploadLibrary1,
+    tags: [],
+    sharedLinks: [],
+    originalFileName: 'asset-id.jpg',
+    faces: [],
+    deletedAt: null,
+    sidecarPath: null,
+    exifInfo: {
+      fileSizeInByte: 5000,
+    } as ExifEntity,
+    duplicateId: 'duplicate-id',
+    smartSearch: {
+      assetId: 'asset-id',
+      embedding: Array.from({ length: 512 }, Math.random),
+    },
   }),
 };

server/test/fixtures/shared-link.stub.ts

@@ -262,6 +262,7 @@ export const sharedLinkStub = {
           faces: [],
           sidecarPath: null,
           deletedAt: null,
+          duplicateId: null,
         },
       ],
     },

server/test/repositories/asset.repository.mock.ts

@@ -22,6 +22,7 @@ export const newAssetRepositoryMock = (): Mocked<IAssetRepository> => {
     getAll: vitest.fn().mockResolvedValue({ items: [], hasNextPage: false }),
     getAllByDeviceId: vitest.fn(),
     updateAll: vitest.fn(),
+    updateDuplicates: vitest.fn(),
     getExternalLibraryAssetPaths: vitest.fn(),
     getByLibraryIdAndOriginalPath: vitest.fn(),
     deleteAll: vitest.fn(),
@@ -38,5 +39,6 @@ export const newAssetRepositoryMock = (): Mocked<IAssetRepository> => {
     getAssetIdByTag: vitest.fn(),
     getAllForUserFullSync: vitest.fn(),
     getChangedDeltaSync: vitest.fn(),
+    getDuplicates: vitest.fn(),
   };
 };

server/test/repositories/search.repository.mock.ts

@@ -6,6 +6,7 @@ export const newSearchRepositoryMock = (): Mocked<ISearchRepository> => {
     init: vitest.fn(),
     searchMetadata: vitest.fn(),
     searchSmart: vitest.fn(),
+    searchDuplicates: vitest.fn(),
     searchFaces: vitest.fn(),
     upsert: vitest.fn(),
     searchPlaces: vitest.fn(),

web/src/lib/stores/server-config.store.ts

@@ -6,6 +6,7 @@ export type FeatureFlags = ServerFeaturesDto & { loaded: boolean };
 export const featureFlags = writable<FeatureFlags>({
   loaded: false,
   smartSearch: true,
+  duplicateDetection: false,
   facialRecognition: true,
   sidecar: true,
   map: true,

web/src/lib/utils.ts

@@ -116,6 +116,7 @@ export const getJobName = (jobName: JobName) => {
     [JobName.MetadataExtraction]: 'Extract Metadata',
     [JobName.Sidecar]: 'Sidecar Metadata',
     [JobName.SmartSearch]: 'Smart Search',
+    [JobName.DuplicateDetection]: 'Duplicate Detection',
     [JobName.FaceDetection]: 'Face Detection',
     [JobName.FacialRecognition]: 'Facial Recognition',
     [JobName.VideoConversion]: 'Transcode Videos',