feat(server): efficient full app sync (#8755)

* feat(server): efficient full app sync

* add SQL, fix test compile issues

* fix linter warning

* new sync controller+service, add tests

* enable new sync controller+service

* Update server/src/services/sync.service.ts

Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>

---------

Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>

mobile/openapi/.openapi-generator/FILES

@@ -24,6 +24,7 @@ doc/AssetBulkUploadCheckDto.md
 doc/AssetBulkUploadCheckItem.md
 doc/AssetBulkUploadCheckResponseDto.md
 doc/AssetBulkUploadCheckResult.md
+doc/AssetDeltaSyncResponseDto.md
 doc/AssetFaceResponseDto.md
 doc/AssetFaceUpdateDto.md
 doc/AssetFaceUpdateItem.md
@@ -149,6 +150,7 @@ doc/SharedLinkType.md
 doc/SignUpDto.md
 doc/SmartInfoResponseDto.md
 doc/SmartSearchDto.md
+doc/SyncApi.md
 doc/SystemConfigApi.md
 doc/SystemConfigDto.md
 doc/SystemConfigFFmpegDto.md
@@ -218,6 +220,7 @@ lib/api/person_api.dart
 lib/api/search_api.dart
 lib/api/server_info_api.dart
 lib/api/shared_link_api.dart
+lib/api/sync_api.dart
 lib/api/system_config_api.dart
 lib/api/tag_api.dart
 lib/api/timeline_api.dart
@@ -248,6 +251,7 @@ lib/model/asset_bulk_upload_check_dto.dart
 lib/model/asset_bulk_upload_check_item.dart
 lib/model/asset_bulk_upload_check_response_dto.dart
 lib/model/asset_bulk_upload_check_result.dart
+lib/model/asset_delta_sync_response_dto.dart
 lib/model/asset_face_response_dto.dart
 lib/model/asset_face_update_dto.dart
 lib/model/asset_face_update_item.dart
@@ -427,6 +431,7 @@ test/asset_bulk_upload_check_dto_test.dart
 test/asset_bulk_upload_check_item_test.dart
 test/asset_bulk_upload_check_response_dto_test.dart
 test/asset_bulk_upload_check_result_test.dart
+test/asset_delta_sync_response_dto_test.dart
 test/asset_face_response_dto_test.dart
 test/asset_face_update_dto_test.dart
 test/asset_face_update_item_test.dart
@@ -552,6 +557,7 @@ test/shared_link_type_test.dart
 test/sign_up_dto_test.dart
 test/smart_info_response_dto_test.dart
 test/smart_search_dto_test.dart
+test/sync_api_test.dart
 test/system_config_api_test.dart
 test/system_config_dto_test.dart
 test/system_config_f_fmpeg_dto_test.dart

mobile/openapi/README.md

@@ -191,6 +191,8 @@ Class | Method | HTTP request | Description
 *SharedLinkApi* | [**removeSharedLink**](doc//SharedLinkApi.md#removesharedlink) | **DELETE** /shared-link/{id} | 
 *SharedLinkApi* | [**removeSharedLinkAssets**](doc//SharedLinkApi.md#removesharedlinkassets) | **DELETE** /shared-link/{id}/assets | 
 *SharedLinkApi* | [**updateSharedLink**](doc//SharedLinkApi.md#updatesharedlink) | **PATCH** /shared-link/{id} | 
+*SyncApi* | [**getAllForUserFullSync**](doc//SyncApi.md#getallforuserfullsync) | **GET** /sync/full-sync | 
+*SyncApi* | [**getDeltaSync**](doc//SyncApi.md#getdeltasync) | **GET** /sync/delta-sync | 
 *SystemConfigApi* | [**getConfig**](doc//SystemConfigApi.md#getconfig) | **GET** /system-config | 
 *SystemConfigApi* | [**getConfigDefaults**](doc//SystemConfigApi.md#getconfigdefaults) | **GET** /system-config/defaults | 
 *SystemConfigApi* | [**getMapStyle**](doc//SystemConfigApi.md#getmapstyle) | **GET** /system-config/map/style.json | 
@@ -240,6 +242,7 @@ Class | Method | HTTP request | Description
  - [AssetBulkUploadCheckItem](doc//AssetBulkUploadCheckItem.md)
  - [AssetBulkUploadCheckResponseDto](doc//AssetBulkUploadCheckResponseDto.md)
  - [AssetBulkUploadCheckResult](doc//AssetBulkUploadCheckResult.md)
+ - [AssetDeltaSyncResponseDto](doc//AssetDeltaSyncResponseDto.md)
  - [AssetFaceResponseDto](doc//AssetFaceResponseDto.md)
  - [AssetFaceUpdateDto](doc//AssetFaceUpdateDto.md)
  - [AssetFaceUpdateItem](doc//AssetFaceUpdateItem.md)

mobile/openapi/doc/AssetDeltaSyncResponseDto.md

@@ -0,0 +1,17 @@
+# openapi.model.AssetDeltaSyncResponseDto
+
+## Load the model package
+```dart
+import 'package:openapi/api.dart';
+```
+
+## Properties
+Name | Type | Description | Notes
+------------ | ------------- | ------------- | -------------
+**deleted** | **List<String>** |  | [default to const []]
+**needsFullSync** | **bool** |  | 
+**upserted** | [**List<AssetResponseDto>**](AssetResponseDto.md) |  | [default to const []]
+
+[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
+
+

mobile/openapi/doc/SyncApi.md

@@ -0,0 +1,135 @@
+# openapi.api.SyncApi
+
+## Load the API package
+```dart
+import 'package:openapi/api.dart';
+```
+
+All URIs are relative to */api*
+
+Method | HTTP request | Description
+------------- | ------------- | -------------
+[**getAllForUserFullSync**](SyncApi.md#getallforuserfullsync) | **GET** /sync/full-sync | 
+[**getDeltaSync**](SyncApi.md#getdeltasync) | **GET** /sync/delta-sync | 
+
+
+# **getAllForUserFullSync**
+> List<AssetResponseDto> getAllForUserFullSync(limit, updatedUntil, lastCreationDate, lastId, userId)
+
+
+
+### Example
+```dart
+import 'package:openapi/api.dart';
+// TODO Configure API key authorization: cookie
+//defaultApiClient.getAuthentication<ApiKeyAuth>('cookie').apiKey = 'YOUR_API_KEY';
+// uncomment below to setup prefix (e.g. Bearer) for API key, if needed
+//defaultApiClient.getAuthentication<ApiKeyAuth>('cookie').apiKeyPrefix = 'Bearer';
+// TODO Configure API key authorization: api_key
+//defaultApiClient.getAuthentication<ApiKeyAuth>('api_key').apiKey = 'YOUR_API_KEY';
+// uncomment below to setup prefix (e.g. Bearer) for API key, if needed
+//defaultApiClient.getAuthentication<ApiKeyAuth>('api_key').apiKeyPrefix = 'Bearer';
+// TODO Configure HTTP Bearer authorization: bearer
+// Case 1. Use String Token
+//defaultApiClient.getAuthentication<HttpBearerAuth>('bearer').setAccessToken('YOUR_ACCESS_TOKEN');
+// Case 2. Use Function which generate token.
+// String yourTokenGeneratorFunction() { ... }
+//defaultApiClient.getAuthentication<HttpBearerAuth>('bearer').setAccessToken(yourTokenGeneratorFunction);
+
+final api_instance = SyncApi();
+final limit = 56; // int | 
+final updatedUntil = 2013-10-20T19:20:30+01:00; // DateTime | 
+final lastCreationDate = 2013-10-20T19:20:30+01:00; // DateTime | 
+final lastId = 38400000-8cf0-11bd-b23e-10b96e4ef00d; // String | 
+final userId = 38400000-8cf0-11bd-b23e-10b96e4ef00d; // String | 
+
+try {
+    final result = api_instance.getAllForUserFullSync(limit, updatedUntil, lastCreationDate, lastId, userId);
+    print(result);
+} catch (e) {
+    print('Exception when calling SyncApi->getAllForUserFullSync: $e\n');
+}
+```
+
+### Parameters
+
+Name | Type | Description  | Notes
+------------- | ------------- | ------------- | -------------
+ **limit** | **int**|  | 
+ **updatedUntil** | **DateTime**|  | 
+ **lastCreationDate** | **DateTime**|  | [optional] 
+ **lastId** | **String**|  | [optional] 
+ **userId** | **String**|  | [optional] 
+
+### Return type
+
+[**List<AssetResponseDto>**](AssetResponseDto.md)
+
+### Authorization
+
+[cookie](../README.md#cookie), [api_key](../README.md#api_key), [bearer](../README.md#bearer)
+
+### HTTP request headers
+
+ - **Content-Type**: Not defined
+ - **Accept**: application/json
+
+[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md)
+
+# **getDeltaSync**
+> AssetDeltaSyncResponseDto getDeltaSync(updatedAfter, userIds)
+
+
+
+### Example
+```dart
+import 'package:openapi/api.dart';
+// TODO Configure API key authorization: cookie
+//defaultApiClient.getAuthentication<ApiKeyAuth>('cookie').apiKey = 'YOUR_API_KEY';
+// uncomment below to setup prefix (e.g. Bearer) for API key, if needed
+//defaultApiClient.getAuthentication<ApiKeyAuth>('cookie').apiKeyPrefix = 'Bearer';
+// TODO Configure API key authorization: api_key
+//defaultApiClient.getAuthentication<ApiKeyAuth>('api_key').apiKey = 'YOUR_API_KEY';
+// uncomment below to setup prefix (e.g. Bearer) for API key, if needed
+//defaultApiClient.getAuthentication<ApiKeyAuth>('api_key').apiKeyPrefix = 'Bearer';
+// TODO Configure HTTP Bearer authorization: bearer
+// Case 1. Use String Token
+//defaultApiClient.getAuthentication<HttpBearerAuth>('bearer').setAccessToken('YOUR_ACCESS_TOKEN');
+// Case 2. Use Function which generate token.
+// String yourTokenGeneratorFunction() { ... }
+//defaultApiClient.getAuthentication<HttpBearerAuth>('bearer').setAccessToken(yourTokenGeneratorFunction);
+
+final api_instance = SyncApi();
+final updatedAfter = 2013-10-20T19:20:30+01:00; // DateTime | 
+final userIds = []; // List<String> | 
+
+try {
+    final result = api_instance.getDeltaSync(updatedAfter, userIds);
+    print(result);
+} catch (e) {
+    print('Exception when calling SyncApi->getDeltaSync: $e\n');
+}
+```
+
+### Parameters
+
+Name | Type | Description  | Notes
+------------- | ------------- | ------------- | -------------
+ **updatedAfter** | **DateTime**|  | 
+ **userIds** | [**List<String>**](String.md)|  | [default to const []]
+
+### Return type
+
+[**AssetDeltaSyncResponseDto**](AssetDeltaSyncResponseDto.md)
+
+### Authorization
+
+[cookie](../README.md#cookie), [api_key](../README.md#api_key), [bearer](../README.md#bearer)
+
+### HTTP request headers
+
+ - **Content-Type**: Not defined
+ - **Accept**: application/json
+
+[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md)
+

mobile/openapi/lib/api.dart

@@ -46,6 +46,7 @@ part 'api/person_api.dart';
 part 'api/search_api.dart';
 part 'api/server_info_api.dart';
 part 'api/shared_link_api.dart';
+part 'api/sync_api.dart';
 part 'api/system_config_api.dart';
 part 'api/tag_api.dart';
 part 'api/timeline_api.dart';
@@ -69,6 +70,7 @@ part 'model/asset_bulk_upload_check_dto.dart';
 part 'model/asset_bulk_upload_check_item.dart';
 part 'model/asset_bulk_upload_check_response_dto.dart';
 part 'model/asset_bulk_upload_check_result.dart';
+part 'model/asset_delta_sync_response_dto.dart';
 part 'model/asset_face_response_dto.dart';
 part 'model/asset_face_update_dto.dart';
 part 'model/asset_face_update_item.dart';

mobile/openapi/lib/api/sync_api.dart

@@ -0,0 +1,150 @@
+//
+// AUTO-GENERATED FILE, DO NOT MODIFY!
+//
+// @dart=2.12
+
+// ignore_for_file: unused_element, unused_import
+// ignore_for_file: always_put_required_named_parameters_first
+// ignore_for_file: constant_identifier_names
+// ignore_for_file: lines_longer_than_80_chars
+
+part of openapi.api;
+
+
+class SyncApi {
+  SyncApi([ApiClient? apiClient]) : apiClient = apiClient ?? defaultApiClient;
+
+  final ApiClient apiClient;
+
+  /// Performs an HTTP 'GET /sync/full-sync' operation and returns the [Response].
+  /// Parameters:
+  ///
+  /// * [int] limit (required):
+  ///
+  /// * [DateTime] updatedUntil (required):
+  ///
+  /// * [DateTime] lastCreationDate:
+  ///
+  /// * [String] lastId:
+  ///
+  /// * [String] userId:
+  Future<Response> getAllForUserFullSyncWithHttpInfo(int limit, DateTime updatedUntil, { DateTime? lastCreationDate, String? lastId, String? userId, }) async {
+    // ignore: prefer_const_declarations
+    final path = r'/sync/full-sync';
+
+    // ignore: prefer_final_locals
+    Object? postBody;
+
+    final queryParams = <QueryParam>[];
+    final headerParams = <String, String>{};
+    final formParams = <String, String>{};
+
+    if (lastCreationDate != null) {
+      queryParams.addAll(_queryParams('', 'lastCreationDate', lastCreationDate));
+    }
+    if (lastId != null) {
+      queryParams.addAll(_queryParams('', 'lastId', lastId));
+    }
+      queryParams.addAll(_queryParams('', 'limit', limit));
+      queryParams.addAll(_queryParams('', 'updatedUntil', updatedUntil));
+    if (userId != null) {
+      queryParams.addAll(_queryParams('', 'userId', userId));
+    }
+
+    const contentTypes = <String>[];
+
+
+    return apiClient.invokeAPI(
+      path,
+      'GET',
+      queryParams,
+      postBody,
+      headerParams,
+      formParams,
+      contentTypes.isEmpty ? null : contentTypes.first,
+    );
+  }
+
+  /// Parameters:
+  ///
+  /// * [int] limit (required):
+  ///
+  /// * [DateTime] updatedUntil (required):
+  ///
+  /// * [DateTime] lastCreationDate:
+  ///
+  /// * [String] lastId:
+  ///
+  /// * [String] userId:
+  Future<List<AssetResponseDto>?> getAllForUserFullSync(int limit, DateTime updatedUntil, { DateTime? lastCreationDate, String? lastId, String? userId, }) async {
+    final response = await getAllForUserFullSyncWithHttpInfo(limit, updatedUntil,  lastCreationDate: lastCreationDate, lastId: lastId, userId: userId, );
+    if (response.statusCode >= HttpStatus.badRequest) {
+      throw ApiException(response.statusCode, await _decodeBodyBytes(response));
+    }
+    // When a remote server returns no body with a status of 204, we shall not decode it.
+    // At the time of writing this, `dart:convert` will throw an "Unexpected end of input"
+    // FormatException when trying to decode an empty string.
+    if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) {
+      final responseBody = await _decodeBodyBytes(response);
+      return (await apiClient.deserializeAsync(responseBody, 'List<AssetResponseDto>') as List)
+        .cast<AssetResponseDto>()
+        .toList(growable: false);
+
+    }
+    return null;
+  }
+
+  /// Performs an HTTP 'GET /sync/delta-sync' operation and returns the [Response].
+  /// Parameters:
+  ///
+  /// * [DateTime] updatedAfter (required):
+  ///
+  /// * [List<String>] userIds (required):
+  Future<Response> getDeltaSyncWithHttpInfo(DateTime updatedAfter, List<String> userIds,) async {
+    // ignore: prefer_const_declarations
+    final path = r'/sync/delta-sync';
+
+    // ignore: prefer_final_locals
+    Object? postBody;
+
+    final queryParams = <QueryParam>[];
+    final headerParams = <String, String>{};
+    final formParams = <String, String>{};
+
+      queryParams.addAll(_queryParams('', 'updatedAfter', updatedAfter));
+      queryParams.addAll(_queryParams('multi', 'userIds', userIds));
+
+    const contentTypes = <String>[];
+
+
+    return apiClient.invokeAPI(
+      path,
+      'GET',
+      queryParams,
+      postBody,
+      headerParams,
+      formParams,
+      contentTypes.isEmpty ? null : contentTypes.first,
+    );
+  }
+
+  /// Parameters:
+  ///
+  /// * [DateTime] updatedAfter (required):
+  ///
+  /// * [List<String>] userIds (required):
+  Future<AssetDeltaSyncResponseDto?> getDeltaSync(DateTime updatedAfter, List<String> userIds,) async {
+    final response = await getDeltaSyncWithHttpInfo(updatedAfter, userIds,);
+    if (response.statusCode >= HttpStatus.badRequest) {
+      throw ApiException(response.statusCode, await _decodeBodyBytes(response));
+    }
+    // When a remote server returns no body with a status of 204, we shall not decode it.
+    // At the time of writing this, `dart:convert` will throw an "Unexpected end of input"
+    // FormatException when trying to decode an empty string.
+    if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) {
+      return await apiClient.deserializeAsync(await _decodeBodyBytes(response), 'AssetDeltaSyncResponseDto',) as AssetDeltaSyncResponseDto;
+    
+    }
+    return null;
+  }
+}

mobile/openapi/lib/api_client.dart

@@ -216,6 +216,8 @@ class ApiClient {
           return AssetBulkUploadCheckResponseDto.fromJson(value);
         case 'AssetBulkUploadCheckResult':
           return AssetBulkUploadCheckResult.fromJson(value);
+        case 'AssetDeltaSyncResponseDto':
+          return AssetDeltaSyncResponseDto.fromJson(value);
         case 'AssetFaceResponseDto':
           return AssetFaceResponseDto.fromJson(value);
         case 'AssetFaceUpdateDto':

mobile/openapi/lib/model/asset_delta_sync_response_dto.dart

@@ -0,0 +1,116 @@
+//
+// AUTO-GENERATED FILE, DO NOT MODIFY!
+//
+// @dart=2.12
+
+// ignore_for_file: unused_element, unused_import
+// ignore_for_file: always_put_required_named_parameters_first
+// ignore_for_file: constant_identifier_names
+// ignore_for_file: lines_longer_than_80_chars
+
+part of openapi.api;
+
+class AssetDeltaSyncResponseDto {
+  /// Returns a new [AssetDeltaSyncResponseDto] instance.
+  AssetDeltaSyncResponseDto({
+    this.deleted = const [],
+    required this.needsFullSync,
+    this.upserted = const [],
+  });
+
+  List<String> deleted;
+
+  bool needsFullSync;
+
+  List<AssetResponseDto> upserted;
+
+  @override
+  bool operator ==(Object other) => identical(this, other) || other is AssetDeltaSyncResponseDto &&
+    _deepEquality.equals(other.deleted, deleted) &&
+    other.needsFullSync == needsFullSync &&
+    _deepEquality.equals(other.upserted, upserted);
+
+  @override
+  int get hashCode =>
+    // ignore: unnecessary_parenthesis
+    (deleted.hashCode) +
+    (needsFullSync.hashCode) +
+    (upserted.hashCode);
+
+  @override
+  String toString() => 'AssetDeltaSyncResponseDto[deleted=$deleted, needsFullSync=$needsFullSync, upserted=$upserted]';
+
+  Map<String, dynamic> toJson() {
+    final json = <String, dynamic>{};
+      json[r'deleted'] = this.deleted;
+      json[r'needsFullSync'] = this.needsFullSync;
+      json[r'upserted'] = this.upserted;
+    return json;
+  }
+
+  /// Returns a new [AssetDeltaSyncResponseDto] instance and imports its values from
+  /// [value] if it's a [Map], null otherwise.
+  // ignore: prefer_constructors_over_static_methods
+  static AssetDeltaSyncResponseDto? fromJson(dynamic value) {
+    if (value is Map) {
+      final json = value.cast<String, dynamic>();
+
+      return AssetDeltaSyncResponseDto(
+        deleted: json[r'deleted'] is Iterable
+            ? (json[r'deleted'] as Iterable).cast<String>().toList(growable: false)
+            : const [],
+        needsFullSync: mapValueOfType<bool>(json, r'needsFullSync')!,
+        upserted: AssetResponseDto.listFromJson(json[r'upserted']),
+      );
+    }
+    return null;
+  }
+
+  static List<AssetDeltaSyncResponseDto> listFromJson(dynamic json, {bool growable = false,}) {
+    final result = <AssetDeltaSyncResponseDto>[];
+    if (json is List && json.isNotEmpty) {
+      for (final row in json) {
+        final value = AssetDeltaSyncResponseDto.fromJson(row);
+        if (value != null) {
+          result.add(value);
+        }
+      }
+    }
+    return result.toList(growable: growable);
+  }
+
+  static Map<String, AssetDeltaSyncResponseDto> mapFromJson(dynamic json) {
+    final map = <String, AssetDeltaSyncResponseDto>{};
+    if (json is Map && json.isNotEmpty) {
+      json = json.cast<String, dynamic>(); // ignore: parameter_assignments
+      for (final entry in json.entries) {
+        final value = AssetDeltaSyncResponseDto.fromJson(entry.value);
+        if (value != null) {
+          map[entry.key] = value;
+        }
+      }
+    }
+    return map;
+  }
+
+  // maps a json object with a list of AssetDeltaSyncResponseDto-objects as value to a dart map
+  static Map<String, List<AssetDeltaSyncResponseDto>> mapListFromJson(dynamic json, {bool growable = false,}) {
+    final map = <String, List<AssetDeltaSyncResponseDto>>{};
+    if (json is Map && json.isNotEmpty) {
+      // ignore: parameter_assignments
+      json = json.cast<String, dynamic>();
+      for (final entry in json.entries) {
+        map[entry.key] = AssetDeltaSyncResponseDto.listFromJson(entry.value, growable: growable,);
+      }
+    }
+    return map;
+  }
+
+  /// The list of required keys that must be present in a JSON.
+  static const requiredKeys = <String>{
+    'deleted',
+    'needsFullSync',
+    'upserted',
+  };
+}
+

mobile/openapi/test/asset_delta_sync_response_dto_test.dart

@@ -0,0 +1,37 @@
+//
+// AUTO-GENERATED FILE, DO NOT MODIFY!
+//
+// @dart=2.12
+
+// ignore_for_file: unused_element, unused_import
+// ignore_for_file: always_put_required_named_parameters_first
+// ignore_for_file: constant_identifier_names
+// ignore_for_file: lines_longer_than_80_chars
+
+import 'package:openapi/api.dart';
+import 'package:test/test.dart';
+
+// tests for AssetDeltaSyncResponseDto
+void main() {
+  // final instance = AssetDeltaSyncResponseDto();
+
+  group('test AssetDeltaSyncResponseDto', () {
+    // List<String> deleted (default value: const [])
+    test('to test the property `deleted`', () async {
+      // TODO
+    });
+
+    // bool needsFullSync
+    test('to test the property `needsFullSync`', () async {
+      // TODO
+    });
+
+    // List<AssetResponseDto> upserted (default value: const [])
+    test('to test the property `upserted`', () async {
+      // TODO
+    });
+
+
+  });
+
+}

mobile/openapi/test/sync_api_test.dart

@@ -0,0 +1,31 @@
+//
+// AUTO-GENERATED FILE, DO NOT MODIFY!
+//
+// @dart=2.12
+
+// ignore_for_file: unused_element, unused_import
+// ignore_for_file: always_put_required_named_parameters_first
+// ignore_for_file: constant_identifier_names
+// ignore_for_file: lines_longer_than_80_chars
+
+import 'package:openapi/api.dart';
+import 'package:test/test.dart';
+
+
+/// tests for SyncApi
+void main() {
+  // final instance = SyncApi();
+
+  group('tests for SyncApi', () {
+    //Future<List<AssetResponseDto>> getAllForUserFullSync(int limit, DateTime updatedUntil, { DateTime lastCreationDate, String lastId, String userId }) async
+    test('test getAllForUserFullSync', () async {
+      // TODO
+    });
+
+    //Future<AssetDeltaSyncResponseDto> getDeltaSync(DateTime updatedAfter, List<String> userIds) async
+    test('test getDeltaSync', () async {
+      // TODO
+    });
+
+  });
+}

open-api/immich-openapi-specs.json

@@ -5566,6 +5566,140 @@
         ]
       }
     },
+    "/sync/delta-sync": {
+      "get": {
+        "operationId": "getDeltaSync",
+        "parameters": [
+          {
+            "name": "updatedAfter",
+            "required": true,
+            "in": "query",
+            "schema": {
+              "format": "date-time",
+              "type": "string"
+            }
+          },
+          {
+            "name": "userIds",
+            "required": true,
+            "in": "query",
+            "schema": {
+              "format": "uuid",
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            }
+          }
+        ],
+        "responses": {
+          "200": {
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/AssetDeltaSyncResponseDto"
+                }
+              }
+            },
+            "description": ""
+          }
+        },
+        "security": [
+          {
+            "bearer": []
+          },
+          {
+            "cookie": []
+          },
+          {
+            "api_key": []
+          }
+        ],
+        "tags": [
+          "Sync"
+        ]
+      }
+    },
+    "/sync/full-sync": {
+      "get": {
+        "operationId": "getAllForUserFullSync",
+        "parameters": [
+          {
+            "name": "lastCreationDate",
+            "required": false,
+            "in": "query",
+            "schema": {
+              "format": "date-time",
+              "type": "string"
+            }
+          },
+          {
+            "name": "lastId",
+            "required": false,
+            "in": "query",
+            "schema": {
+              "format": "uuid",
+              "type": "string"
+            }
+          },
+          {
+            "name": "limit",
+            "required": true,
+            "in": "query",
+            "schema": {
+              "type": "integer"
+            }
+          },
+          {
+            "name": "updatedUntil",
+            "required": true,
+            "in": "query",
+            "schema": {
+              "format": "date-time",
+              "type": "string"
+            }
+          },
+          {
+            "name": "userId",
+            "required": false,
+            "in": "query",
+            "schema": {
+              "format": "uuid",
+              "type": "string"
+            }
+          }
+        ],
+        "responses": {
+          "200": {
+            "content": {
+              "application/json": {
+                "schema": {
+                  "items": {
+                    "$ref": "#/components/schemas/AssetResponseDto"
+                  },
+                  "type": "array"
+                }
+              }
+            },
+            "description": ""
+          }
+        },
+        "security": [
+          {
+            "bearer": []
+          },
+          {
+            "cookie": []
+          },
+          {
+            "api_key": []
+          }
+        ],
+        "tags": [
+          "Sync"
+        ]
+      }
+    },
     "/system-config": {
       "get": {
         "operationId": "getConfig",
@@ -7335,6 +7469,31 @@
         ],
         "type": "object"
       },
+      "AssetDeltaSyncResponseDto": {
+        "properties": {
+          "deleted": {
+            "items": {
+              "type": "string"
+            },
+            "type": "array"
+          },
+          "needsFullSync": {
+            "type": "boolean"
+          },
+          "upserted": {
+            "items": {
+              "$ref": "#/components/schemas/AssetResponseDto"
+            },
+            "type": "array"
+          }
+        },
+        "required": [
+          "deleted",
+          "needsFullSync",
+          "upserted"
+        ],
+        "type": "object"
+      },
       "AssetFaceResponseDto": {
         "properties": {
           "boundingBoxX1": {

open-api/typescript-sdk/src/fetch-client.ts

@@ -835,6 +835,11 @@ export type AssetIdsResponseDto = {
     error?: Error2;
     success: boolean;
 };
+export type AssetDeltaSyncResponseDto = {
+    deleted: string[];
+    needsFullSync: boolean;
+    upserted: AssetResponseDto[];
+};
 export type SystemConfigFFmpegDto = {
     accel: TranscodeHWAccel;
     acceptedAudioCodecs: AudioCodec[];
@@ -2507,6 +2512,40 @@ export function addSharedLinkAssets({ id, key, assetIdsDto }: {
         body: assetIdsDto
     })));
 }
+export function getDeltaSync({ updatedAfter, userIds }: {
+    updatedAfter: string;
+    userIds: string[];
+}, opts?: Oazapfts.RequestOpts) {
+    return oazapfts.ok(oazapfts.fetchJson<{
+        status: 200;
+        data: AssetDeltaSyncResponseDto;
+    }>(`/sync/delta-sync${QS.query(QS.explode({
+        updatedAfter,
+        userIds
+    }))}`, {
+        ...opts
+    }));
+}
+export function getAllForUserFullSync({ lastCreationDate, lastId, limit, updatedUntil, userId }: {
+    lastCreationDate?: string;
+    lastId?: string;
+    limit: number;
+    updatedUntil: string;
+    userId?: string;
+}, opts?: Oazapfts.RequestOpts) {
+    return oazapfts.ok(oazapfts.fetchJson<{
+        status: 200;
+        data: AssetResponseDto[];
+    }>(`/sync/full-sync${QS.query(QS.explode({
+        lastCreationDate,
+        lastId,
+        limit,
+        updatedUntil,
+        userId
+    }))}`, {
+        ...opts
+    }));
+}
 export function getConfig(opts?: Oazapfts.RequestOpts) {
     return oazapfts.ok(oazapfts.fetchJson<{
         status: 200;

server/src/controllers/index.ts

@@ -17,6 +17,7 @@ import { PersonController } from 'src/controllers/person.controller';
 import { SearchController } from 'src/controllers/search.controller';
 import { ServerInfoController } from 'src/controllers/server-info.controller';
 import { SharedLinkController } from 'src/controllers/shared-link.controller';
+import { SyncController } from 'src/controllers/sync.controller';
 import { SystemConfigController } from 'src/controllers/system-config.controller';
 import { TagController } from 'src/controllers/tag.controller';
 import { TimelineController } from 'src/controllers/timeline.controller';
@@ -43,6 +44,7 @@ export const controllers = [
   SearchController,
   ServerInfoController,
   SharedLinkController,
+  SyncController,
   SystemConfigController,
   TagController,
   TimelineController,

server/src/controllers/sync.controller.ts

@@ -0,0 +1,24 @@
+import { Controller, Get, Query } from '@nestjs/common';
+import { ApiTags } from '@nestjs/swagger';
+import { AssetResponseDto } from 'src/dtos/asset-response.dto';
+import { AuthDto } from 'src/dtos/auth.dto';
+import { AssetDeltaSyncDto, AssetDeltaSyncResponseDto, AssetFullSyncDto } from 'src/dtos/sync.dto';
+import { Auth, Authenticated } from 'src/middleware/auth.guard';
+import { SyncService } from 'src/services/sync.service';
+
+@ApiTags('Sync')
+@Controller('sync')
+@Authenticated()
+export class SyncController {
+  constructor(private service: SyncService) {}
+
+  @Get('full-sync')
+  getAllForUserFullSync(@Auth() auth: AuthDto, @Query() dto: AssetFullSyncDto): Promise<AssetResponseDto[]> {
+    return this.service.getAllAssetsForUserFullSync(auth, dto);
+  }
+
+  @Get('delta-sync')
+  getDeltaSync(@Auth() auth: AuthDto, @Query() dto: AssetDeltaSyncDto): Promise<AssetDeltaSyncResponseDto> {
+    return this.service.getChangesForDeltaSync(auth, dto);
+  }
+}

server/src/dtos/sync.dto.ts

@@ -0,0 +1,38 @@
+import { ApiProperty } from '@nestjs/swagger';
+import { Type } from 'class-transformer';
+import { IsInt, IsPositive } from 'class-validator';
+import { AssetResponseDto } from 'src/dtos/asset-response.dto';
+import { ValidateDate, ValidateUUID } from 'src/validation';
+
+export class AssetFullSyncDto {
+  @ValidateUUID({ optional: true })
+  lastId?: string;
+
+  @ValidateDate({ optional: true })
+  lastCreationDate?: Date;
+
+  @ValidateDate()
+  updatedUntil!: Date;
+
+  @IsInt()
+  @IsPositive()
+  @Type(() => Number)
+  @ApiProperty({ type: 'integer' })
+  limit!: number;
+
+  @ValidateUUID({ optional: true })
+  userId?: string;
+}
+
+export class AssetDeltaSyncDto {
+  @ValidateDate()
+  updatedAfter!: Date;
+  @ValidateUUID({ each: true })
+  userIds!: string[];
+}
+
+export class AssetDeltaSyncResponseDto {
+  needsFullSync!: boolean;
+  upserted!: AssetResponseDto[];
+  deleted!: string[];
+}

server/src/interfaces/asset.interface.ts

@@ -133,6 +133,20 @@ export interface MetadataSearchOptions {
   numResults: number;
 }
 
+export interface AssetFullSyncOptions {
+  ownerId: string;
+  lastCreationDate?: Date;
+  lastId?: string;
+  updatedUntil: Date;
+  limit: number;
+}
+
+export interface AssetDeltaSyncOptions {
+  userIds: string[];
+  updatedAfter: Date;
+  limit: number;
+}
+
 export type AssetPathEntity = Pick<AssetEntity, 'id' | 'originalPath' | 'isOffline'>;
 
 export const IAssetRepository = 'IAssetRepository';
@@ -175,4 +189,6 @@ export interface IAssetRepository {
   getAssetIdByCity(userId: string, options: AssetExploreFieldOptions): Promise<SearchExploreItem<string>>;
   getAssetIdByTag(userId: string, options: AssetExploreFieldOptions): Promise<SearchExploreItem<string>>;
   searchMetadata(query: string, userIds: string[], options: MetadataSearchOptions): Promise<AssetEntity[]>;
+  getAllForUserFullSync(options: AssetFullSyncOptions): Promise<AssetEntity[]>;
+  getChangedDeltaSync(options: AssetDeltaSyncOptions): Promise<AssetEntity[]>;
 }

server/src/interfaces/audit.interface.ts

@@ -1,14 +1,14 @@
-import { AuditEntity, DatabaseAction, EntityType } from 'src/entities/audit.entity';
+import { DatabaseAction, EntityType } from 'src/entities/audit.entity';
 
 export const IAuditRepository = 'IAuditRepository';
 
 export interface AuditSearch {
   action?: DatabaseAction;
   entityType?: EntityType;
-  ownerId?: string;
+  userIds: string[];
 }
 
 export interface IAuditRepository {
-  getAfter(since: Date, options: AuditSearch): Promise<AuditEntity[]>;
+  getAfter(since: Date, options: AuditSearch): Promise<string[]>;
   removeBefore(before: Date): Promise<void>;
 }

server/src/queries/asset.repository.sql

@@ -768,3 +768,151 @@ ORDER BY
   "asset"."fileCreatedAt" DESC
 LIMIT
   250
+
+-- AssetRepository.getAllForUserFullSync
+SELECT
+  "asset"."id" AS "asset_id",
+  "asset"."deviceAssetId" AS "asset_deviceAssetId",
+  "asset"."ownerId" AS "asset_ownerId",
+  "asset"."libraryId" AS "asset_libraryId",
+  "asset"."deviceId" AS "asset_deviceId",
+  "asset"."type" AS "asset_type",
+  "asset"."originalPath" AS "asset_originalPath",
+  "asset"."previewPath" AS "asset_previewPath",
+  "asset"."thumbnailPath" AS "asset_thumbnailPath",
+  "asset"."thumbhash" AS "asset_thumbhash",
+  "asset"."encodedVideoPath" AS "asset_encodedVideoPath",
+  "asset"."createdAt" AS "asset_createdAt",
+  "asset"."updatedAt" AS "asset_updatedAt",
+  "asset"."deletedAt" AS "asset_deletedAt",
+  "asset"."fileCreatedAt" AS "asset_fileCreatedAt",
+  "asset"."localDateTime" AS "asset_localDateTime",
+  "asset"."fileModifiedAt" AS "asset_fileModifiedAt",
+  "asset"."isFavorite" AS "asset_isFavorite",
+  "asset"."isArchived" AS "asset_isArchived",
+  "asset"."isExternal" AS "asset_isExternal",
+  "asset"."isReadOnly" AS "asset_isReadOnly",
+  "asset"."isOffline" AS "asset_isOffline",
+  "asset"."checksum" AS "asset_checksum",
+  "asset"."duration" AS "asset_duration",
+  "asset"."isVisible" AS "asset_isVisible",
+  "asset"."livePhotoVideoId" AS "asset_livePhotoVideoId",
+  "asset"."originalFileName" AS "asset_originalFileName",
+  "asset"."sidecarPath" AS "asset_sidecarPath",
+  "asset"."stackId" AS "asset_stackId",
+  "exifInfo"."assetId" AS "exifInfo_assetId",
+  "exifInfo"."description" AS "exifInfo_description",
+  "exifInfo"."exifImageWidth" AS "exifInfo_exifImageWidth",
+  "exifInfo"."exifImageHeight" AS "exifInfo_exifImageHeight",
+  "exifInfo"."fileSizeInByte" AS "exifInfo_fileSizeInByte",
+  "exifInfo"."orientation" AS "exifInfo_orientation",
+  "exifInfo"."dateTimeOriginal" AS "exifInfo_dateTimeOriginal",
+  "exifInfo"."modifyDate" AS "exifInfo_modifyDate",
+  "exifInfo"."timeZone" AS "exifInfo_timeZone",
+  "exifInfo"."latitude" AS "exifInfo_latitude",
+  "exifInfo"."longitude" AS "exifInfo_longitude",
+  "exifInfo"."projectionType" AS "exifInfo_projectionType",
+  "exifInfo"."city" AS "exifInfo_city",
+  "exifInfo"."livePhotoCID" AS "exifInfo_livePhotoCID",
+  "exifInfo"."autoStackId" AS "exifInfo_autoStackId",
+  "exifInfo"."state" AS "exifInfo_state",
+  "exifInfo"."country" AS "exifInfo_country",
+  "exifInfo"."make" AS "exifInfo_make",
+  "exifInfo"."model" AS "exifInfo_model",
+  "exifInfo"."lensModel" AS "exifInfo_lensModel",
+  "exifInfo"."fNumber" AS "exifInfo_fNumber",
+  "exifInfo"."focalLength" AS "exifInfo_focalLength",
+  "exifInfo"."iso" AS "exifInfo_iso",
+  "exifInfo"."exposureTime" AS "exifInfo_exposureTime",
+  "exifInfo"."profileDescription" AS "exifInfo_profileDescription",
+  "exifInfo"."colorspace" AS "exifInfo_colorspace",
+  "exifInfo"."bitsPerSample" AS "exifInfo_bitsPerSample",
+  "exifInfo"."fps" AS "exifInfo_fps",
+  "stack"."id" AS "stack_id",
+  "stack"."primaryAssetId" AS "stack_primaryAssetId"
+FROM
+  "assets" "asset"
+  LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id"
+  LEFT JOIN "asset_stack" "stack" ON "stack"."id" = "asset"."stackId"
+WHERE
+  "asset"."ownerId" = $1
+  AND ("asset"."fileCreatedAt", "asset"."id") < ($2, $3)
+  AND "asset"."updatedAt" <= $4
+  AND "asset"."isVisible" = true
+ORDER BY
+  "asset"."fileCreatedAt" DESC,
+  "asset"."id" DESC
+LIMIT
+  10
+
+-- AssetRepository.getChangedDeltaSync
+SELECT
+  "AssetEntity"."id" AS "AssetEntity_id",
+  "AssetEntity"."deviceAssetId" AS "AssetEntity_deviceAssetId",
+  "AssetEntity"."ownerId" AS "AssetEntity_ownerId",
+  "AssetEntity"."libraryId" AS "AssetEntity_libraryId",
+  "AssetEntity"."deviceId" AS "AssetEntity_deviceId",
+  "AssetEntity"."type" AS "AssetEntity_type",
+  "AssetEntity"."originalPath" AS "AssetEntity_originalPath",
+  "AssetEntity"."previewPath" AS "AssetEntity_previewPath",
+  "AssetEntity"."thumbnailPath" AS "AssetEntity_thumbnailPath",
+  "AssetEntity"."thumbhash" AS "AssetEntity_thumbhash",
+  "AssetEntity"."encodedVideoPath" AS "AssetEntity_encodedVideoPath",
+  "AssetEntity"."createdAt" AS "AssetEntity_createdAt",
+  "AssetEntity"."updatedAt" AS "AssetEntity_updatedAt",
+  "AssetEntity"."deletedAt" AS "AssetEntity_deletedAt",
+  "AssetEntity"."fileCreatedAt" AS "AssetEntity_fileCreatedAt",
+  "AssetEntity"."localDateTime" AS "AssetEntity_localDateTime",
+  "AssetEntity"."fileModifiedAt" AS "AssetEntity_fileModifiedAt",
+  "AssetEntity"."isFavorite" AS "AssetEntity_isFavorite",
+  "AssetEntity"."isArchived" AS "AssetEntity_isArchived",
+  "AssetEntity"."isExternal" AS "AssetEntity_isExternal",
+  "AssetEntity"."isReadOnly" AS "AssetEntity_isReadOnly",
+  "AssetEntity"."isOffline" AS "AssetEntity_isOffline",
+  "AssetEntity"."checksum" AS "AssetEntity_checksum",
+  "AssetEntity"."duration" AS "AssetEntity_duration",
+  "AssetEntity"."isVisible" AS "AssetEntity_isVisible",
+  "AssetEntity"."livePhotoVideoId" AS "AssetEntity_livePhotoVideoId",
+  "AssetEntity"."originalFileName" AS "AssetEntity_originalFileName",
+  "AssetEntity"."sidecarPath" AS "AssetEntity_sidecarPath",
+  "AssetEntity"."stackId" AS "AssetEntity_stackId",
+  "AssetEntity__AssetEntity_exifInfo"."assetId" AS "AssetEntity__AssetEntity_exifInfo_assetId",
+  "AssetEntity__AssetEntity_exifInfo"."description" AS "AssetEntity__AssetEntity_exifInfo_description",
+  "AssetEntity__AssetEntity_exifInfo"."exifImageWidth" AS "AssetEntity__AssetEntity_exifInfo_exifImageWidth",
+  "AssetEntity__AssetEntity_exifInfo"."exifImageHeight" AS "AssetEntity__AssetEntity_exifInfo_exifImageHeight",
+  "AssetEntity__AssetEntity_exifInfo"."fileSizeInByte" AS "AssetEntity__AssetEntity_exifInfo_fileSizeInByte",
+  "AssetEntity__AssetEntity_exifInfo"."orientation" AS "AssetEntity__AssetEntity_exifInfo_orientation",
+  "AssetEntity__AssetEntity_exifInfo"."dateTimeOriginal" AS "AssetEntity__AssetEntity_exifInfo_dateTimeOriginal",
+  "AssetEntity__AssetEntity_exifInfo"."modifyDate" AS "AssetEntity__AssetEntity_exifInfo_modifyDate",
+  "AssetEntity__AssetEntity_exifInfo"."timeZone" AS "AssetEntity__AssetEntity_exifInfo_timeZone",
+  "AssetEntity__AssetEntity_exifInfo"."latitude" AS "AssetEntity__AssetEntity_exifInfo_latitude",
+  "AssetEntity__AssetEntity_exifInfo"."longitude" AS "AssetEntity__AssetEntity_exifInfo_longitude",
+  "AssetEntity__AssetEntity_exifInfo"."projectionType" AS "AssetEntity__AssetEntity_exifInfo_projectionType",
+  "AssetEntity__AssetEntity_exifInfo"."city" AS "AssetEntity__AssetEntity_exifInfo_city",
+  "AssetEntity__AssetEntity_exifInfo"."livePhotoCID" AS "AssetEntity__AssetEntity_exifInfo_livePhotoCID",
+  "AssetEntity__AssetEntity_exifInfo"."autoStackId" AS "AssetEntity__AssetEntity_exifInfo_autoStackId",
+  "AssetEntity__AssetEntity_exifInfo"."state" AS "AssetEntity__AssetEntity_exifInfo_state",
+  "AssetEntity__AssetEntity_exifInfo"."country" AS "AssetEntity__AssetEntity_exifInfo_country",
+  "AssetEntity__AssetEntity_exifInfo"."make" AS "AssetEntity__AssetEntity_exifInfo_make",
+  "AssetEntity__AssetEntity_exifInfo"."model" AS "AssetEntity__AssetEntity_exifInfo_model",
+  "AssetEntity__AssetEntity_exifInfo"."lensModel" AS "AssetEntity__AssetEntity_exifInfo_lensModel",
+  "AssetEntity__AssetEntity_exifInfo"."fNumber" AS "AssetEntity__AssetEntity_exifInfo_fNumber",
+  "AssetEntity__AssetEntity_exifInfo"."focalLength" AS "AssetEntity__AssetEntity_exifInfo_focalLength",
+  "AssetEntity__AssetEntity_exifInfo"."iso" AS "AssetEntity__AssetEntity_exifInfo_iso",
+  "AssetEntity__AssetEntity_exifInfo"."exposureTime" AS "AssetEntity__AssetEntity_exifInfo_exposureTime",
+  "AssetEntity__AssetEntity_exifInfo"."profileDescription" AS "AssetEntity__AssetEntity_exifInfo_profileDescription",
+  "AssetEntity__AssetEntity_exifInfo"."colorspace" AS "AssetEntity__AssetEntity_exifInfo_colorspace",
+  "AssetEntity__AssetEntity_exifInfo"."bitsPerSample" AS "AssetEntity__AssetEntity_exifInfo_bitsPerSample",
+  "AssetEntity__AssetEntity_exifInfo"."fps" AS "AssetEntity__AssetEntity_exifInfo_fps",
+  "AssetEntity__AssetEntity_stack"."id" AS "AssetEntity__AssetEntity_stack_id",
+  "AssetEntity__AssetEntity_stack"."primaryAssetId" AS "AssetEntity__AssetEntity_stack_primaryAssetId"
+FROM
+  "assets" "AssetEntity"
+  LEFT JOIN "exif" "AssetEntity__AssetEntity_exifInfo" ON "AssetEntity__AssetEntity_exifInfo"."assetId" = "AssetEntity"."id"
+  LEFT JOIN "asset_stack" "AssetEntity__AssetEntity_stack" ON "AssetEntity__AssetEntity_stack"."id" = "AssetEntity"."stackId"
+WHERE
+  (
+    ("AssetEntity"."ownerId" IN ($1))
+    AND ("AssetEntity"."isVisible" = $2)
+    AND ("AssetEntity"."updatedAt" > $3)
+  )

server/src/repositories/asset.repository.ts

@@ -2,15 +2,18 @@ import { Injectable } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
 import path from 'node:path';
 import { Chunked, ChunkedArray, DummyValue, GenerateSql } from 'src/decorators';
-import { AssetOrder } from 'src/entities/album.entity';
+import { AlbumEntity, AssetOrder } from 'src/entities/album.entity';
 import { AssetJobStatusEntity } from 'src/entities/asset-job-status.entity';
 import { AssetEntity, AssetType } from 'src/entities/asset.entity';
 import { ExifEntity } from 'src/entities/exif.entity';
+import { PartnerEntity } from 'src/entities/partner.entity';
 import { SmartInfoEntity } from 'src/entities/smart-info.entity';
 import {
   AssetBuilderOptions,
   AssetCreate,
+  AssetDeltaSyncOptions,
   AssetExploreFieldOptions,
+  AssetFullSyncOptions,
   AssetPathEntity,
   AssetStats,
   AssetStatsOptions,
@@ -39,6 +42,7 @@ import {
   FindOptionsWhere,
   In,
   IsNull,
+  MoreThan,
   Not,
   Repository,
 } from 'typeorm';
@@ -61,6 +65,8 @@ export class AssetRepository implements IAssetRepository {
     @InjectRepository(ExifEntity) private exifRepository: Repository<ExifEntity>,
     @InjectRepository(AssetJobStatusEntity) private jobStatusRepository: Repository<AssetJobStatusEntity>,
     @InjectRepository(SmartInfoEntity) private smartInfoRepository: Repository<SmartInfoEntity>,
+    @InjectRepository(PartnerEntity) private partnerRepository: Repository<PartnerEntity>,
+    @InjectRepository(AlbumEntity) private albumRepository: Repository<AlbumEntity>,
   ) {}
 
   async upsertExif(exif: Partial<ExifEntity>): Promise<void> {
@@ -781,4 +787,55 @@ export class AssetRepository implements IAssetRepository {
         }) as AssetEntity,
     );
   }
+
+  @GenerateSql({
+    params: [
+      {
+        ownerId: DummyValue.UUID,
+        lastCreationDate: DummyValue.DATE,
+        lastId: DummyValue.STRING,
+        updatedUntil: DummyValue.DATE,
+        limit: 10,
+      },
+    ],
+  })
+  getAllForUserFullSync(options: AssetFullSyncOptions): Promise<AssetEntity[]> {
+    const { ownerId, lastCreationDate, lastId, updatedUntil, limit } = options;
+    let builder = this.repository
+      .createQueryBuilder('asset')
+      .leftJoinAndSelect('asset.exifInfo', 'exifInfo')
+      .leftJoinAndSelect('asset.stack', 'stack')
+      .where('asset.ownerId = :ownerId', { ownerId });
+    if (lastCreationDate !== undefined && lastId !== undefined) {
+      builder = builder.andWhere('(asset.fileCreatedAt, asset.id) < (:lastCreationDate, :lastId)', {
+        lastCreationDate,
+        lastId,
+      });
+    }
+    return builder
+      .andWhere('asset.updatedAt <= :updatedUntil', { updatedUntil })
+      .andWhere('asset.isVisible = true')
+      .orderBy('asset.fileCreatedAt', 'DESC')
+      .addOrderBy('asset.id', 'DESC')
+      .limit(limit)
+      .withDeleted()
+      .getMany();
+  }
+
+  @GenerateSql({ params: [{ userIds: [DummyValue.UUID], updatedAfter: DummyValue.DATE }] })
+  getChangedDeltaSync(options: AssetDeltaSyncOptions): Promise<AssetEntity[]> {
+    return this.repository.find({
+      where: {
+        ownerId: In(options.userIds),
+        isVisible: true,
+        updatedAt: MoreThan(options.updatedAfter),
+      },
+      relations: {
+        exifInfo: true,
+        stack: true,
+      },
+      take: options.limit,
+      withDeleted: true,
+    });
+  }
 }

server/src/repositories/audit.repository.ts

@@ -2,24 +2,25 @@ import { InjectRepository } from '@nestjs/typeorm';
 import { AuditEntity } from 'src/entities/audit.entity';
 import { AuditSearch, IAuditRepository } from 'src/interfaces/audit.interface';
 import { Instrumentation } from 'src/utils/instrumentation';
-import { LessThan, MoreThan, Repository } from 'typeorm';
+import { In, LessThan, MoreThan, Repository } from 'typeorm';
 
 @Instrumentation()
 export class AuditRepository implements IAuditRepository {
   constructor(@InjectRepository(AuditEntity) private repository: Repository<AuditEntity>) {}
 
-  getAfter(since: Date, options: AuditSearch): Promise<AuditEntity[]> {
+  getAfter(since: Date, options: AuditSearch): Promise<string[]> {
     return this.repository
       .createQueryBuilder('audit')
       .where({
         createdAt: MoreThan(since),
         action: options.action,
         entityType: options.entityType,
-        ownerId: options.ownerId,
+        ownerId: In(options.userIds),
       })
       .distinctOn(['audit.entityId', 'audit.entityType'])
       .orderBy('audit.entityId, audit.entityType, audit.createdAt', 'DESC')
-      .getMany();
+      .select('audit.entityId')
+      .getRawMany();
   }
 
   async removeBefore(before: Date): Promise<void> {

server/src/services/audit.service.spec.ts

@@ -61,13 +61,13 @@ describe(AuditService.name, () => {
 
       expect(auditMock.getAfter).toHaveBeenCalledWith(date, {
         action: DatabaseAction.DELETE,
-        ownerId: authStub.admin.user.id,
+        userIds: [authStub.admin.user.id],
         entityType: EntityType.ASSET,
       });
     });
 
     it('should get any new or updated assets and deleted ids', async () => {
-      auditMock.getAfter.mockResolvedValue([auditStub.delete]);
+      auditMock.getAfter.mockResolvedValue([auditStub.delete.entityId]);
 
       const date = new Date();
       await expect(sut.getDeletes(authStub.admin, { after: date, entityType: EntityType.ASSET })).resolves.toEqual({
@@ -77,7 +77,7 @@ describe(AuditService.name, () => {
 
       expect(auditMock.getAfter).toHaveBeenCalledWith(date, {
         action: DatabaseAction.DELETE,
-        ownerId: authStub.admin.user.id,
+        userIds: [authStub.admin.user.id],
         entityType: EntityType.ASSET,
       });
     });

server/src/services/audit.service.ts

@@ -53,7 +53,7 @@ export class AuditService {
     await this.access.requirePermission(auth, Permission.TIMELINE_READ, userId);
 
     const audits = await this.repository.getAfter(dto.after, {
-      ownerId: userId,
+      userIds: [userId],
       entityType: dto.entityType,
       action: DatabaseAction.DELETE,
     });
@@ -62,7 +62,7 @@ export class AuditService {
 
     return {
       needsFullSync: duration > AUDIT_LOG_MAX_DURATION,
-      ids: audits.map(({ entityId }) => entityId),
+      ids: audits,
     };
   }
 

server/src/services/index.ts

@@ -22,6 +22,7 @@ import { SharedLinkService } from 'src/services/shared-link.service';
 import { SmartInfoService } from 'src/services/smart-info.service';
 import { StorageTemplateService } from 'src/services/storage-template.service';
 import { StorageService } from 'src/services/storage.service';
+import { SyncService } from 'src/services/sync.service';
 import { SystemConfigService } from 'src/services/system-config.service';
 import { TagService } from 'src/services/tag.service';
 import { TimelineService } from 'src/services/timeline.service';
@@ -53,6 +54,7 @@ export const services = [
   SmartInfoService,
   StorageService,
   StorageTemplateService,
+  SyncService,
   SystemConfigService,
   TagService,
   TimelineService,

server/src/services/sync.service.spec.ts

@@ -0,0 +1,101 @@
+import { mapAsset } from 'src/dtos/asset-response.dto';
+import { AssetEntity } from 'src/entities/asset.entity';
+import { IAccessRepository } from 'src/interfaces/access.interface';
+import { IAssetRepository } from 'src/interfaces/asset.interface';
+import { IAuditRepository } from 'src/interfaces/audit.interface';
+import { IPartnerRepository } from 'src/interfaces/partner.interface';
+import { SyncService } from 'src/services/sync.service';
+import { assetStub } from 'test/fixtures/asset.stub';
+import { authStub } from 'test/fixtures/auth.stub';
+import { partnerStub } from 'test/fixtures/partner.stub';
+import { newAccessRepositoryMock } from 'test/repositories/access.repository.mock';
+import { newAssetRepositoryMock } from 'test/repositories/asset.repository.mock';
+import { newAuditRepositoryMock } from 'test/repositories/audit.repository.mock';
+import { newPartnerRepositoryMock } from 'test/repositories/partner.repository.mock';
+
+const untilDate = new Date(2024);
+const mapAssetOpts = { auth: authStub.user1, stripMetadata: false, withStack: true };
+
+describe(SyncService.name, () => {
+  let sut: SyncService;
+  let accessMock: jest.Mocked<IAccessRepository>;
+  let assetMock: jest.Mocked<IAssetRepository>;
+  let partnerMock: jest.Mocked<IPartnerRepository>;
+  let auditMock: jest.Mocked<IAuditRepository>;
+
+  beforeEach(() => {
+    partnerMock = newPartnerRepositoryMock();
+    assetMock = newAssetRepositoryMock();
+    accessMock = newAccessRepositoryMock();
+    auditMock = newAuditRepositoryMock();
+    sut = new SyncService(accessMock, assetMock, partnerMock, auditMock);
+  });
+
+  it('should exist', () => {
+    expect(sut).toBeDefined();
+  });
+
+  describe('getAllAssetsForUserFullSync', () => {
+    it('should return a list of all assets owned by the user', async () => {
+      assetMock.getAllForUserFullSync.mockResolvedValue([assetStub.external, assetStub.hasEncodedVideo]);
+      await expect(
+        sut.getAllAssetsForUserFullSync(authStub.user1, { limit: 2, updatedUntil: untilDate }),
+      ).resolves.toEqual([
+        mapAsset(assetStub.external, mapAssetOpts),
+        mapAsset(assetStub.hasEncodedVideo, mapAssetOpts),
+      ]);
+      expect(assetMock.getAllForUserFullSync).toHaveBeenCalledWith({
+        ownerId: authStub.user1.user.id,
+        updatedUntil: untilDate,
+        limit: 2,
+      });
+    });
+  });
+
+  describe('getChangesForDeltaSync', () => {
+    it('should return a response requiring a full sync when partners are out of sync', async () => {
+      partnerMock.getAll.mockResolvedValue([partnerStub.adminToUser1]);
+      await expect(
+        sut.getChangesForDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }),
+      ).resolves.toEqual({ needsFullSync: true, upserted: [], deleted: [] });
+      expect(assetMock.getChangedDeltaSync).toHaveBeenCalledTimes(0);
+      expect(auditMock.getAfter).toHaveBeenCalledTimes(0);
+    });
+
+    it('should return a response requiring a full sync when last sync was too long ago', async () => {
+      partnerMock.getAll.mockResolvedValue([]);
+      await expect(
+        sut.getChangesForDeltaSync(authStub.user1, { updatedAfter: new Date(2000), userIds: [authStub.user1.user.id] }),
+      ).resolves.toEqual({ needsFullSync: true, upserted: [], deleted: [] });
+      expect(assetMock.getChangedDeltaSync).toHaveBeenCalledTimes(0);
+      expect(auditMock.getAfter).toHaveBeenCalledTimes(0);
+    });
+
+    it('should return a response requiring a full sync when there are too many changes', async () => {
+      partnerMock.getAll.mockResolvedValue([]);
+      assetMock.getChangedDeltaSync.mockResolvedValue(
+        Array.from<AssetEntity>({ length: 10_000 }).fill(assetStub.image),
+      );
+      await expect(
+        sut.getChangesForDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }),
+      ).resolves.toEqual({ needsFullSync: true, upserted: [], deleted: [] });
+      expect(assetMock.getChangedDeltaSync).toHaveBeenCalledTimes(1);
+      expect(auditMock.getAfter).toHaveBeenCalledTimes(0);
+    });
+
+    it('should return a response with changes and deletions', async () => {
+      partnerMock.getAll.mockResolvedValue([]);
+      assetMock.getChangedDeltaSync.mockResolvedValue([assetStub.image1]);
+      auditMock.getAfter.mockResolvedValue([assetStub.external.id]);
+      await expect(
+        sut.getChangesForDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }),
+      ).resolves.toEqual({
+        needsFullSync: false,
+        upserted: [mapAsset(assetStub.image1, mapAssetOpts)],
+        deleted: [assetStub.external.id],
+      });
+      expect(assetMock.getChangedDeltaSync).toHaveBeenCalledTimes(1);
+      expect(auditMock.getAfter).toHaveBeenCalledTimes(1);
+    });
+  });
+});

server/src/services/sync.service.ts

@@ -0,0 +1,77 @@
+import { Inject } from '@nestjs/common';
+import _ from 'lodash';
+import { DateTime } from 'luxon';
+import { AUDIT_LOG_MAX_DURATION } from 'src/constants';
+import { AccessCore, Permission } from 'src/cores/access.core';
+import { AssetResponseDto, mapAsset } from 'src/dtos/asset-response.dto';
+import { AuthDto } from 'src/dtos/auth.dto';
+import { AssetDeltaSyncDto, AssetDeltaSyncResponseDto, AssetFullSyncDto } from 'src/dtos/sync.dto';
+import { DatabaseAction, EntityType } from 'src/entities/audit.entity';
+import { IAccessRepository } from 'src/interfaces/access.interface';
+import { IAssetRepository } from 'src/interfaces/asset.interface';
+import { IAuditRepository } from 'src/interfaces/audit.interface';
+import { IPartnerRepository } from 'src/interfaces/partner.interface';
+
+export class SyncService {
+  private access: AccessCore;
+
+  constructor(
+    @Inject(IAccessRepository) accessRepository: IAccessRepository,
+    @Inject(IAssetRepository) private assetRepository: IAssetRepository,
+    @Inject(IPartnerRepository) private partnerRepository: IPartnerRepository,
+    @Inject(IAuditRepository) private auditRepository: IAuditRepository,
+  ) {
+    this.access = AccessCore.create(accessRepository);
+  }
+
+  async getAllAssetsForUserFullSync(auth: AuthDto, dto: AssetFullSyncDto): Promise<AssetResponseDto[]> {
+    const userId = dto.userId || auth.user.id;
+    await this.access.requirePermission(auth, Permission.TIMELINE_READ, userId);
+    const assets = await this.assetRepository.getAllForUserFullSync({
+      ownerId: userId,
+      lastCreationDate: dto.lastCreationDate,
+      updatedUntil: dto.updatedUntil,
+      lastId: dto.lastId,
+      limit: dto.limit,
+    });
+    const options = { auth, stripMetadata: false, withStack: true };
+    return assets.map((a) => mapAsset(a, options));
+  }
+
+  async getChangesForDeltaSync(auth: AuthDto, dto: AssetDeltaSyncDto): Promise<AssetDeltaSyncResponseDto> {
+    await this.access.requirePermission(auth, Permission.TIMELINE_READ, dto.userIds);
+    const partner = await this.partnerRepository.getAll(auth.user.id);
+    const userIds = [auth.user.id, ...partner.filter((p) => p.sharedWithId == auth.user.id).map((p) => p.sharedById)];
+    userIds.sort();
+    dto.userIds.sort();
+    const duration = DateTime.now().diff(DateTime.fromJSDate(dto.updatedAfter));
+
+    if (!_.isEqual(userIds, dto.userIds) || duration > AUDIT_LOG_MAX_DURATION) {
+      // app does not have the correct partners synced
+      // or app has not synced in the last 100 days
+      return { needsFullSync: true, deleted: [], upserted: [] };
+    }
+
+    const limit = 10_000;
+    const upserted = await this.assetRepository.getChangedDeltaSync({ limit, updatedAfter: dto.updatedAfter, userIds });
+
+    if (upserted.length === limit) {
+      // too many changes -> do a full sync (paginated) instead
+      return { needsFullSync: true, deleted: [], upserted: [] };
+    }
+
+    const deleted = await this.auditRepository.getAfter(dto.updatedAfter, {
+      userIds: userIds,
+      entityType: EntityType.ASSET,
+      action: DatabaseAction.DELETE,
+    });
+
+    const options = { auth, stripMetadata: false, withStack: true };
+    const result = {
+      needsFullSync: false,
+      upserted: upserted.map((a) => mapAsset(a, options)),
+      deleted,
+    };
+    return result;
+  }
+}

server/test/repositories/asset.repository.mock.ts

@@ -35,5 +35,7 @@ export const newAssetRepositoryMock = (): jest.Mocked<IAssetRepository> => {
     getAssetIdByCity: jest.fn(),
     getAssetIdByTag: jest.fn(),
     searchMetadata: jest.fn(),
+    getAllForUserFullSync: jest.fn(),
+    getChangedDeltaSync: jest.fn(),
   };
 };