diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a8a8f0e5c157..9d89063f246c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -504,6 +504,7 @@ jobs: run: | echo "ERROR: Generated migration files not up to date!" echo "Changed files: ${{ steps.verify-changed-files.outputs.changed_files }}" + cat ./src/migrations/*-TestMigration.ts exit 1 - name: Run SQL generation diff --git a/mobile/openapi/README.md b/mobile/openapi/README.md index d006ef38bb3e..e86ac9335080 100644 --- a/mobile/openapi/README.md +++ b/mobile/openapi/README.md @@ -201,8 +201,12 @@ Class | Method | HTTP request | Description *StacksApi* | [**getStack**](doc//StacksApi.md#getstack) | **GET** /stacks/{id} | *StacksApi* | [**searchStacks**](doc//StacksApi.md#searchstacks) | **GET** /stacks | *StacksApi* | [**updateStack**](doc//StacksApi.md#updatestack) | **PUT** /stacks/{id} | +*SyncApi* | [**deleteSyncAck**](doc//SyncApi.md#deletesyncack) | **DELETE** /sync/ack | *SyncApi* | [**getDeltaSync**](doc//SyncApi.md#getdeltasync) | **POST** /sync/delta-sync | *SyncApi* | [**getFullSyncForUser**](doc//SyncApi.md#getfullsyncforuser) | **POST** /sync/full-sync | +*SyncApi* | [**getSyncAck**](doc//SyncApi.md#getsyncack) | **GET** /sync/ack | +*SyncApi* | [**getSyncStream**](doc//SyncApi.md#getsyncstream) | **POST** /sync/stream | +*SyncApi* | [**sendSyncAck**](doc//SyncApi.md#sendsyncack) | **POST** /sync/ack | *SystemConfigApi* | [**getConfig**](doc//SystemConfigApi.md#getconfig) | **GET** /system-config | *SystemConfigApi* | [**getConfigDefaults**](doc//SystemConfigApi.md#getconfigdefaults) | **GET** /system-config/defaults | *SystemConfigApi* | [**getStorageTemplateOptions**](doc//SystemConfigApi.md#getstoragetemplateoptions) | **GET** /system-config/storage-template-options | @@ -413,6 +417,14 @@ Class | Method | HTTP request | Description - [StackCreateDto](doc//StackCreateDto.md) - [StackResponseDto](doc//StackResponseDto.md) - [StackUpdateDto](doc//StackUpdateDto.md) + - [SyncAckDeleteDto](doc//SyncAckDeleteDto.md) + - [SyncAckDto](doc//SyncAckDto.md) + - [SyncAckSetDto](doc//SyncAckSetDto.md) + - [SyncEntityType](doc//SyncEntityType.md) + - [SyncRequestType](doc//SyncRequestType.md) + - [SyncStreamDto](doc//SyncStreamDto.md) + - [SyncUserDeleteV1](doc//SyncUserDeleteV1.md) + - [SyncUserV1](doc//SyncUserV1.md) - [SystemConfigBackupsDto](doc//SystemConfigBackupsDto.md) - [SystemConfigDto](doc//SystemConfigDto.md) - [SystemConfigFFmpegDto](doc//SystemConfigFFmpegDto.md) diff --git a/mobile/openapi/lib/api.dart b/mobile/openapi/lib/api.dart index 2a2b6d46a496..e5794a26940b 100644 --- a/mobile/openapi/lib/api.dart +++ b/mobile/openapi/lib/api.dart @@ -226,6 +226,14 @@ part 'model/source_type.dart'; part 'model/stack_create_dto.dart'; part 'model/stack_response_dto.dart'; part 'model/stack_update_dto.dart'; +part 'model/sync_ack_delete_dto.dart'; +part 'model/sync_ack_dto.dart'; +part 'model/sync_ack_set_dto.dart'; +part 'model/sync_entity_type.dart'; +part 'model/sync_request_type.dart'; +part 'model/sync_stream_dto.dart'; +part 'model/sync_user_delete_v1.dart'; +part 'model/sync_user_v1.dart'; part 'model/system_config_backups_dto.dart'; part 'model/system_config_dto.dart'; part 'model/system_config_f_fmpeg_dto.dart'; diff --git a/mobile/openapi/lib/api/sync_api.dart b/mobile/openapi/lib/api/sync_api.dart index f94eb88081a1..49a4963bffd3 100644 --- a/mobile/openapi/lib/api/sync_api.dart +++ b/mobile/openapi/lib/api/sync_api.dart @@ -16,6 +16,45 @@ class SyncApi { final ApiClient apiClient; + /// Performs an HTTP 'DELETE /sync/ack' operation and returns the [Response]. + /// Parameters: + /// + /// * [SyncAckDeleteDto] syncAckDeleteDto (required): + Future deleteSyncAckWithHttpInfo(SyncAckDeleteDto syncAckDeleteDto,) async { + // ignore: prefer_const_declarations + final path = r'/sync/ack'; + + // ignore: prefer_final_locals + Object? postBody = syncAckDeleteDto; + + final queryParams = []; + final headerParams = {}; + final formParams = {}; + + const contentTypes = ['application/json']; + + + return apiClient.invokeAPI( + path, + 'DELETE', + queryParams, + postBody, + headerParams, + formParams, + contentTypes.isEmpty ? null : contentTypes.first, + ); + } + + /// Parameters: + /// + /// * [SyncAckDeleteDto] syncAckDeleteDto (required): + Future deleteSyncAck(SyncAckDeleteDto syncAckDeleteDto,) async { + final response = await deleteSyncAckWithHttpInfo(syncAckDeleteDto,); + if (response.statusCode >= HttpStatus.badRequest) { + throw ApiException(response.statusCode, await _decodeBodyBytes(response)); + } + } + /// Performs an HTTP 'POST /sync/delta-sync' operation and returns the [Response]. /// Parameters: /// @@ -112,4 +151,126 @@ class SyncApi { } return null; } + + /// Performs an HTTP 'GET /sync/ack' operation and returns the [Response]. + Future getSyncAckWithHttpInfo() async { + // ignore: prefer_const_declarations + final path = r'/sync/ack'; + + // ignore: prefer_final_locals + Object? postBody; + + final queryParams = []; + final headerParams = {}; + final formParams = {}; + + const contentTypes = []; + + + return apiClient.invokeAPI( + path, + 'GET', + queryParams, + postBody, + headerParams, + formParams, + contentTypes.isEmpty ? null : contentTypes.first, + ); + } + + Future?> getSyncAck() async { + final response = await getSyncAckWithHttpInfo(); + if (response.statusCode >= HttpStatus.badRequest) { + throw ApiException(response.statusCode, await _decodeBodyBytes(response)); + } + // When a remote server returns no body with a status of 204, we shall not decode it. + // At the time of writing this, `dart:convert` will throw an "Unexpected end of input" + // FormatException when trying to decode an empty string. + if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) { + final responseBody = await _decodeBodyBytes(response); + return (await apiClient.deserializeAsync(responseBody, 'List') as List) + .cast() + .toList(growable: false); + + } + return null; + } + + /// Performs an HTTP 'POST /sync/stream' operation and returns the [Response]. + /// Parameters: + /// + /// * [SyncStreamDto] syncStreamDto (required): + Future getSyncStreamWithHttpInfo(SyncStreamDto syncStreamDto,) async { + // ignore: prefer_const_declarations + final path = r'/sync/stream'; + + // ignore: prefer_final_locals + Object? postBody = syncStreamDto; + + final queryParams = []; + final headerParams = {}; + final formParams = {}; + + const contentTypes = ['application/json']; + + + return apiClient.invokeAPI( + path, + 'POST', + queryParams, + postBody, + headerParams, + formParams, + contentTypes.isEmpty ? null : contentTypes.first, + ); + } + + /// Parameters: + /// + /// * [SyncStreamDto] syncStreamDto (required): + Future getSyncStream(SyncStreamDto syncStreamDto,) async { + final response = await getSyncStreamWithHttpInfo(syncStreamDto,); + if (response.statusCode >= HttpStatus.badRequest) { + throw ApiException(response.statusCode, await _decodeBodyBytes(response)); + } + } + + /// Performs an HTTP 'POST /sync/ack' operation and returns the [Response]. + /// Parameters: + /// + /// * [SyncAckSetDto] syncAckSetDto (required): + Future sendSyncAckWithHttpInfo(SyncAckSetDto syncAckSetDto,) async { + // ignore: prefer_const_declarations + final path = r'/sync/ack'; + + // ignore: prefer_final_locals + Object? postBody = syncAckSetDto; + + final queryParams = []; + final headerParams = {}; + final formParams = {}; + + const contentTypes = ['application/json']; + + + return apiClient.invokeAPI( + path, + 'POST', + queryParams, + postBody, + headerParams, + formParams, + contentTypes.isEmpty ? null : contentTypes.first, + ); + } + + /// Parameters: + /// + /// * [SyncAckSetDto] syncAckSetDto (required): + Future sendSyncAck(SyncAckSetDto syncAckSetDto,) async { + final response = await sendSyncAckWithHttpInfo(syncAckSetDto,); + if (response.statusCode >= HttpStatus.badRequest) { + throw ApiException(response.statusCode, await _decodeBodyBytes(response)); + } + } } diff --git a/mobile/openapi/lib/api_client.dart b/mobile/openapi/lib/api_client.dart index 49fbe9464b7d..54a8959f6a11 100644 --- a/mobile/openapi/lib/api_client.dart +++ b/mobile/openapi/lib/api_client.dart @@ -508,6 +508,22 @@ class ApiClient { return StackResponseDto.fromJson(value); case 'StackUpdateDto': return StackUpdateDto.fromJson(value); + case 'SyncAckDeleteDto': + return SyncAckDeleteDto.fromJson(value); + case 'SyncAckDto': + return SyncAckDto.fromJson(value); + case 'SyncAckSetDto': + return SyncAckSetDto.fromJson(value); + case 'SyncEntityType': + return SyncEntityTypeTypeTransformer().decode(value); + case 'SyncRequestType': + return SyncRequestTypeTypeTransformer().decode(value); + case 'SyncStreamDto': + return SyncStreamDto.fromJson(value); + case 'SyncUserDeleteV1': + return SyncUserDeleteV1.fromJson(value); + case 'SyncUserV1': + return SyncUserV1.fromJson(value); case 'SystemConfigBackupsDto': return SystemConfigBackupsDto.fromJson(value); case 'SystemConfigDto': diff --git a/mobile/openapi/lib/api_helper.dart b/mobile/openapi/lib/api_helper.dart index 6a917201aabb..1ebf8314ad41 100644 --- a/mobile/openapi/lib/api_helper.dart +++ b/mobile/openapi/lib/api_helper.dart @@ -127,6 +127,12 @@ String parameterToString(dynamic value) { if (value is SourceType) { return SourceTypeTypeTransformer().encode(value).toString(); } + if (value is SyncEntityType) { + return SyncEntityTypeTypeTransformer().encode(value).toString(); + } + if (value is SyncRequestType) { + return SyncRequestTypeTypeTransformer().encode(value).toString(); + } if (value is TimeBucketSize) { return TimeBucketSizeTypeTransformer().encode(value).toString(); } diff --git a/mobile/openapi/lib/model/sync_ack_delete_dto.dart b/mobile/openapi/lib/model/sync_ack_delete_dto.dart new file mode 100644 index 000000000000..998f812f2e40 --- /dev/null +++ b/mobile/openapi/lib/model/sync_ack_delete_dto.dart @@ -0,0 +1,98 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class SyncAckDeleteDto { + /// Returns a new [SyncAckDeleteDto] instance. + SyncAckDeleteDto({ + this.types = const [], + }); + + List types; + + @override + bool operator ==(Object other) => identical(this, other) || other is SyncAckDeleteDto && + _deepEquality.equals(other.types, types); + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (types.hashCode); + + @override + String toString() => 'SyncAckDeleteDto[types=$types]'; + + Map toJson() { + final json = {}; + json[r'types'] = this.types; + return json; + } + + /// Returns a new [SyncAckDeleteDto] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static SyncAckDeleteDto? fromJson(dynamic value) { + upgradeDto(value, "SyncAckDeleteDto"); + if (value is Map) { + final json = value.cast(); + + return SyncAckDeleteDto( + types: SyncEntityType.listFromJson(json[r'types']), + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = SyncAckDeleteDto.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = SyncAckDeleteDto.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of SyncAckDeleteDto-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = SyncAckDeleteDto.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + }; +} + diff --git a/mobile/openapi/lib/model/sync_ack_dto.dart b/mobile/openapi/lib/model/sync_ack_dto.dart new file mode 100644 index 000000000000..c7fafa17d20d --- /dev/null +++ b/mobile/openapi/lib/model/sync_ack_dto.dart @@ -0,0 +1,107 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class SyncAckDto { + /// Returns a new [SyncAckDto] instance. + SyncAckDto({ + required this.ack, + required this.type, + }); + + String ack; + + SyncEntityType type; + + @override + bool operator ==(Object other) => identical(this, other) || other is SyncAckDto && + other.ack == ack && + other.type == type; + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (ack.hashCode) + + (type.hashCode); + + @override + String toString() => 'SyncAckDto[ack=$ack, type=$type]'; + + Map toJson() { + final json = {}; + json[r'ack'] = this.ack; + json[r'type'] = this.type; + return json; + } + + /// Returns a new [SyncAckDto] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static SyncAckDto? fromJson(dynamic value) { + upgradeDto(value, "SyncAckDto"); + if (value is Map) { + final json = value.cast(); + + return SyncAckDto( + ack: mapValueOfType(json, r'ack')!, + type: SyncEntityType.fromJson(json[r'type'])!, + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = SyncAckDto.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = SyncAckDto.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of SyncAckDto-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = SyncAckDto.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'ack', + 'type', + }; +} + diff --git a/mobile/openapi/lib/model/sync_ack_set_dto.dart b/mobile/openapi/lib/model/sync_ack_set_dto.dart new file mode 100644 index 000000000000..0d9eedc38929 --- /dev/null +++ b/mobile/openapi/lib/model/sync_ack_set_dto.dart @@ -0,0 +1,101 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class SyncAckSetDto { + /// Returns a new [SyncAckSetDto] instance. + SyncAckSetDto({ + this.acks = const [], + }); + + List acks; + + @override + bool operator ==(Object other) => identical(this, other) || other is SyncAckSetDto && + _deepEquality.equals(other.acks, acks); + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (acks.hashCode); + + @override + String toString() => 'SyncAckSetDto[acks=$acks]'; + + Map toJson() { + final json = {}; + json[r'acks'] = this.acks; + return json; + } + + /// Returns a new [SyncAckSetDto] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static SyncAckSetDto? fromJson(dynamic value) { + upgradeDto(value, "SyncAckSetDto"); + if (value is Map) { + final json = value.cast(); + + return SyncAckSetDto( + acks: json[r'acks'] is Iterable + ? (json[r'acks'] as Iterable).cast().toList(growable: false) + : const [], + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = SyncAckSetDto.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = SyncAckSetDto.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of SyncAckSetDto-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = SyncAckSetDto.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'acks', + }; +} + diff --git a/mobile/openapi/lib/model/sync_entity_type.dart b/mobile/openapi/lib/model/sync_entity_type.dart new file mode 100644 index 000000000000..ed82205a37a1 --- /dev/null +++ b/mobile/openapi/lib/model/sync_entity_type.dart @@ -0,0 +1,85 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + + +class SyncEntityType { + /// Instantiate a new enum with the provided [value]. + const SyncEntityType._(this.value); + + /// The underlying value of this enum member. + final String value; + + @override + String toString() => value; + + String toJson() => value; + + static const userV1 = SyncEntityType._(r'UserV1'); + static const userDeleteV1 = SyncEntityType._(r'UserDeleteV1'); + + /// List of all possible values in this [enum][SyncEntityType]. + static const values = [ + userV1, + userDeleteV1, + ]; + + static SyncEntityType? fromJson(dynamic value) => SyncEntityTypeTypeTransformer().decode(value); + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = SyncEntityType.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } +} + +/// Transformation class that can [encode] an instance of [SyncEntityType] to String, +/// and [decode] dynamic data back to [SyncEntityType]. +class SyncEntityTypeTypeTransformer { + factory SyncEntityTypeTypeTransformer() => _instance ??= const SyncEntityTypeTypeTransformer._(); + + const SyncEntityTypeTypeTransformer._(); + + String encode(SyncEntityType data) => data.value; + + /// Decodes a [dynamic value][data] to a SyncEntityType. + /// + /// If [allowNull] is true and the [dynamic value][data] cannot be decoded successfully, + /// then null is returned. However, if [allowNull] is false and the [dynamic value][data] + /// cannot be decoded successfully, then an [UnimplementedError] is thrown. + /// + /// The [allowNull] is very handy when an API changes and a new enum value is added or removed, + /// and users are still using an old app with the old code. + SyncEntityType? decode(dynamic data, {bool allowNull = true}) { + if (data != null) { + switch (data) { + case r'UserV1': return SyncEntityType.userV1; + case r'UserDeleteV1': return SyncEntityType.userDeleteV1; + default: + if (!allowNull) { + throw ArgumentError('Unknown enum value to decode: $data'); + } + } + } + return null; + } + + /// Singleton [SyncEntityTypeTypeTransformer] instance. + static SyncEntityTypeTypeTransformer? _instance; +} + diff --git a/mobile/openapi/lib/model/sync_request_type.dart b/mobile/openapi/lib/model/sync_request_type.dart new file mode 100644 index 000000000000..d7f1bde54cc2 --- /dev/null +++ b/mobile/openapi/lib/model/sync_request_type.dart @@ -0,0 +1,82 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + + +class SyncRequestType { + /// Instantiate a new enum with the provided [value]. + const SyncRequestType._(this.value); + + /// The underlying value of this enum member. + final String value; + + @override + String toString() => value; + + String toJson() => value; + + static const usersV1 = SyncRequestType._(r'UsersV1'); + + /// List of all possible values in this [enum][SyncRequestType]. + static const values = [ + usersV1, + ]; + + static SyncRequestType? fromJson(dynamic value) => SyncRequestTypeTypeTransformer().decode(value); + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = SyncRequestType.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } +} + +/// Transformation class that can [encode] an instance of [SyncRequestType] to String, +/// and [decode] dynamic data back to [SyncRequestType]. +class SyncRequestTypeTypeTransformer { + factory SyncRequestTypeTypeTransformer() => _instance ??= const SyncRequestTypeTypeTransformer._(); + + const SyncRequestTypeTypeTransformer._(); + + String encode(SyncRequestType data) => data.value; + + /// Decodes a [dynamic value][data] to a SyncRequestType. + /// + /// If [allowNull] is true and the [dynamic value][data] cannot be decoded successfully, + /// then null is returned. However, if [allowNull] is false and the [dynamic value][data] + /// cannot be decoded successfully, then an [UnimplementedError] is thrown. + /// + /// The [allowNull] is very handy when an API changes and a new enum value is added or removed, + /// and users are still using an old app with the old code. + SyncRequestType? decode(dynamic data, {bool allowNull = true}) { + if (data != null) { + switch (data) { + case r'UsersV1': return SyncRequestType.usersV1; + default: + if (!allowNull) { + throw ArgumentError('Unknown enum value to decode: $data'); + } + } + } + return null; + } + + /// Singleton [SyncRequestTypeTypeTransformer] instance. + static SyncRequestTypeTypeTransformer? _instance; +} + diff --git a/mobile/openapi/lib/model/sync_stream_dto.dart b/mobile/openapi/lib/model/sync_stream_dto.dart new file mode 100644 index 000000000000..28fd3dfaeed9 --- /dev/null +++ b/mobile/openapi/lib/model/sync_stream_dto.dart @@ -0,0 +1,99 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class SyncStreamDto { + /// Returns a new [SyncStreamDto] instance. + SyncStreamDto({ + this.types = const [], + }); + + List types; + + @override + bool operator ==(Object other) => identical(this, other) || other is SyncStreamDto && + _deepEquality.equals(other.types, types); + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (types.hashCode); + + @override + String toString() => 'SyncStreamDto[types=$types]'; + + Map toJson() { + final json = {}; + json[r'types'] = this.types; + return json; + } + + /// Returns a new [SyncStreamDto] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static SyncStreamDto? fromJson(dynamic value) { + upgradeDto(value, "SyncStreamDto"); + if (value is Map) { + final json = value.cast(); + + return SyncStreamDto( + types: SyncRequestType.listFromJson(json[r'types']), + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = SyncStreamDto.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = SyncStreamDto.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of SyncStreamDto-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = SyncStreamDto.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'types', + }; +} + diff --git a/mobile/openapi/lib/model/sync_user_delete_v1.dart b/mobile/openapi/lib/model/sync_user_delete_v1.dart new file mode 100644 index 000000000000..09411cb79d7d --- /dev/null +++ b/mobile/openapi/lib/model/sync_user_delete_v1.dart @@ -0,0 +1,99 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class SyncUserDeleteV1 { + /// Returns a new [SyncUserDeleteV1] instance. + SyncUserDeleteV1({ + required this.userId, + }); + + String userId; + + @override + bool operator ==(Object other) => identical(this, other) || other is SyncUserDeleteV1 && + other.userId == userId; + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (userId.hashCode); + + @override + String toString() => 'SyncUserDeleteV1[userId=$userId]'; + + Map toJson() { + final json = {}; + json[r'userId'] = this.userId; + return json; + } + + /// Returns a new [SyncUserDeleteV1] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static SyncUserDeleteV1? fromJson(dynamic value) { + upgradeDto(value, "SyncUserDeleteV1"); + if (value is Map) { + final json = value.cast(); + + return SyncUserDeleteV1( + userId: mapValueOfType(json, r'userId')!, + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = SyncUserDeleteV1.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = SyncUserDeleteV1.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of SyncUserDeleteV1-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = SyncUserDeleteV1.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'userId', + }; +} + diff --git a/mobile/openapi/lib/model/sync_user_v1.dart b/mobile/openapi/lib/model/sync_user_v1.dart new file mode 100644 index 000000000000..b9b41bb72396 --- /dev/null +++ b/mobile/openapi/lib/model/sync_user_v1.dart @@ -0,0 +1,127 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class SyncUserV1 { + /// Returns a new [SyncUserV1] instance. + SyncUserV1({ + required this.deletedAt, + required this.email, + required this.id, + required this.name, + }); + + DateTime? deletedAt; + + String email; + + String id; + + String name; + + @override + bool operator ==(Object other) => identical(this, other) || other is SyncUserV1 && + other.deletedAt == deletedAt && + other.email == email && + other.id == id && + other.name == name; + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (deletedAt == null ? 0 : deletedAt!.hashCode) + + (email.hashCode) + + (id.hashCode) + + (name.hashCode); + + @override + String toString() => 'SyncUserV1[deletedAt=$deletedAt, email=$email, id=$id, name=$name]'; + + Map toJson() { + final json = {}; + if (this.deletedAt != null) { + json[r'deletedAt'] = this.deletedAt!.toUtc().toIso8601String(); + } else { + // json[r'deletedAt'] = null; + } + json[r'email'] = this.email; + json[r'id'] = this.id; + json[r'name'] = this.name; + return json; + } + + /// Returns a new [SyncUserV1] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static SyncUserV1? fromJson(dynamic value) { + upgradeDto(value, "SyncUserV1"); + if (value is Map) { + final json = value.cast(); + + return SyncUserV1( + deletedAt: mapDateTime(json, r'deletedAt', r''), + email: mapValueOfType(json, r'email')!, + id: mapValueOfType(json, r'id')!, + name: mapValueOfType(json, r'name')!, + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = SyncUserV1.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = SyncUserV1.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of SyncUserV1-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = SyncUserV1.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'deletedAt', + 'email', + 'id', + 'name', + }; +} + diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index 14245e11bdeb..1d0f065992d0 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -5802,6 +5802,107 @@ ] } }, + "/sync/ack": { + "delete": { + "operationId": "deleteSyncAck", + "parameters": [], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SyncAckDeleteDto" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "tags": [ + "Sync" + ] + }, + "get": { + "operationId": "getSyncAck", + "parameters": [], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/SyncAckDto" + }, + "type": "array" + } + } + }, + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "tags": [ + "Sync" + ] + }, + "post": { + "operationId": "sendSyncAck", + "parameters": [], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SyncAckSetDto" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "tags": [ + "Sync" + ] + } + }, "/sync/delta-sync": { "post": { "operationId": "getDeltaSync", @@ -5889,6 +5990,41 @@ ] } }, + "/sync/stream": { + "post": { + "operationId": "getSyncStream", + "parameters": [], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SyncStreamDto" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "tags": [ + "Sync" + ] + } + }, "/system-config": { "get": { "operationId": "getConfig", @@ -11696,6 +11832,113 @@ }, "type": "object" }, + "SyncAckDeleteDto": { + "properties": { + "types": { + "items": { + "$ref": "#/components/schemas/SyncEntityType" + }, + "type": "array" + } + }, + "type": "object" + }, + "SyncAckDto": { + "properties": { + "ack": { + "type": "string" + }, + "type": { + "allOf": [ + { + "$ref": "#/components/schemas/SyncEntityType" + } + ] + } + }, + "required": [ + "ack", + "type" + ], + "type": "object" + }, + "SyncAckSetDto": { + "properties": { + "acks": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "acks" + ], + "type": "object" + }, + "SyncEntityType": { + "enum": [ + "UserV1", + "UserDeleteV1" + ], + "type": "string" + }, + "SyncRequestType": { + "enum": [ + "UsersV1" + ], + "type": "string" + }, + "SyncStreamDto": { + "properties": { + "types": { + "items": { + "$ref": "#/components/schemas/SyncRequestType" + }, + "type": "array" + } + }, + "required": [ + "types" + ], + "type": "object" + }, + "SyncUserDeleteV1": { + "properties": { + "userId": { + "type": "string" + } + }, + "required": [ + "userId" + ], + "type": "object" + }, + "SyncUserV1": { + "properties": { + "deletedAt": { + "format": "date-time", + "nullable": true, + "type": "string" + }, + "email": { + "type": "string" + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + } + }, + "required": [ + "deletedAt", + "email", + "id", + "name" + ], + "type": "object" + }, "SystemConfigBackupsDto": { "properties": { "database": { diff --git a/open-api/typescript-sdk/src/fetch-client.ts b/open-api/typescript-sdk/src/fetch-client.ts index 9ff35331fb1a..8b2e88183094 100644 --- a/open-api/typescript-sdk/src/fetch-client.ts +++ b/open-api/typescript-sdk/src/fetch-client.ts @@ -1104,6 +1104,16 @@ export type StackCreateDto = { export type StackUpdateDto = { primaryAssetId?: string; }; +export type SyncAckDeleteDto = { + types?: SyncEntityType[]; +}; +export type SyncAckDto = { + ack: string; + "type": SyncEntityType; +}; +export type SyncAckSetDto = { + acks: string[]; +}; export type AssetDeltaSyncDto = { updatedAfter: string; userIds: string[]; @@ -1119,6 +1129,9 @@ export type AssetFullSyncDto = { updatedUntil: string; userId?: string; }; +export type SyncStreamDto = { + types: SyncRequestType[]; +}; export type DatabaseBackupConfig = { cronExpression: string; enabled: boolean; @@ -2912,6 +2925,32 @@ export function updateStack({ id, stackUpdateDto }: { body: stackUpdateDto }))); } +export function deleteSyncAck({ syncAckDeleteDto }: { + syncAckDeleteDto: SyncAckDeleteDto; +}, opts?: Oazapfts.RequestOpts) { + return oazapfts.ok(oazapfts.fetchText("/sync/ack", oazapfts.json({ + ...opts, + method: "DELETE", + body: syncAckDeleteDto + }))); +} +export function getSyncAck(opts?: Oazapfts.RequestOpts) { + return oazapfts.ok(oazapfts.fetchJson<{ + status: 200; + data: SyncAckDto[]; + }>("/sync/ack", { + ...opts + })); +} +export function sendSyncAck({ syncAckSetDto }: { + syncAckSetDto: SyncAckSetDto; +}, opts?: Oazapfts.RequestOpts) { + return oazapfts.ok(oazapfts.fetchText("/sync/ack", oazapfts.json({ + ...opts, + method: "POST", + body: syncAckSetDto + }))); +} export function getDeltaSync({ assetDeltaSyncDto }: { assetDeltaSyncDto: AssetDeltaSyncDto; }, opts?: Oazapfts.RequestOpts) { @@ -2936,6 +2975,15 @@ export function getFullSyncForUser({ assetFullSyncDto }: { body: assetFullSyncDto }))); } +export function getSyncStream({ syncStreamDto }: { + syncStreamDto: SyncStreamDto; +}, opts?: Oazapfts.RequestOpts) { + return oazapfts.ok(oazapfts.fetchText("/sync/stream", oazapfts.json({ + ...opts, + method: "POST", + body: syncStreamDto + }))); +} export function getConfig(opts?: Oazapfts.RequestOpts) { return oazapfts.ok(oazapfts.fetchJson<{ status: 200; @@ -3548,6 +3596,13 @@ export enum Error2 { NoPermission = "no_permission", NotFound = "not_found" } +export enum SyncEntityType { + UserV1 = "UserV1", + UserDeleteV1 = "UserDeleteV1" +} +export enum SyncRequestType { + UsersV1 = "UsersV1" +} export enum TranscodeHWAccel { Nvenc = "nvenc", Qsv = "qsv", diff --git a/server/src/app.module.ts b/server/src/app.module.ts index a4518598a367..b02d869a1eb8 100644 --- a/server/src/app.module.ts +++ b/server/src/app.module.ts @@ -29,7 +29,7 @@ import { AuthService } from 'src/services/auth.service'; import { CliService } from 'src/services/cli.service'; import { DatabaseService } from 'src/services/database.service'; -const common = [...repositories, ...services]; +const common = [...repositories, ...services, GlobalExceptionFilter]; const middleware = [ FileUploadInterceptor, diff --git a/server/src/controllers/sync.controller.ts b/server/src/controllers/sync.controller.ts index 4d970a7102a8..0945810be768 100644 --- a/server/src/controllers/sync.controller.ts +++ b/server/src/controllers/sync.controller.ts @@ -1,15 +1,28 @@ -import { Body, Controller, HttpCode, HttpStatus, Post } from '@nestjs/common'; +import { Body, Controller, Delete, Get, Header, HttpCode, HttpStatus, Post, Res } from '@nestjs/common'; import { ApiTags } from '@nestjs/swagger'; +import { Response } from 'express'; import { AssetResponseDto } from 'src/dtos/asset-response.dto'; import { AuthDto } from 'src/dtos/auth.dto'; -import { AssetDeltaSyncDto, AssetDeltaSyncResponseDto, AssetFullSyncDto } from 'src/dtos/sync.dto'; +import { + AssetDeltaSyncDto, + AssetDeltaSyncResponseDto, + AssetFullSyncDto, + SyncAckDeleteDto, + SyncAckDto, + SyncAckSetDto, + SyncStreamDto, +} from 'src/dtos/sync.dto'; import { Auth, Authenticated } from 'src/middleware/auth.guard'; +import { GlobalExceptionFilter } from 'src/middleware/global-exception.filter'; import { SyncService } from 'src/services/sync.service'; @ApiTags('Sync') @Controller('sync') export class SyncController { - constructor(private service: SyncService) {} + constructor( + private service: SyncService, + private errorService: GlobalExceptionFilter, + ) {} @Post('full-sync') @HttpCode(HttpStatus.OK) @@ -24,4 +37,37 @@ export class SyncController { getDeltaSync(@Auth() auth: AuthDto, @Body() dto: AssetDeltaSyncDto): Promise { return this.service.getDeltaSync(auth, dto); } + + @Post('stream') + @Header('Content-Type', 'application/jsonlines+json') + @HttpCode(HttpStatus.OK) + @Authenticated() + async getSyncStream(@Auth() auth: AuthDto, @Res() res: Response, @Body() dto: SyncStreamDto) { + try { + await this.service.stream(auth, res, dto); + } catch (error: Error | any) { + res.setHeader('Content-Type', 'application/json'); + this.errorService.handleError(res, error); + } + } + + @Get('ack') + @Authenticated() + getSyncAck(@Auth() auth: AuthDto): Promise { + return this.service.getAcks(auth); + } + + @Post('ack') + @HttpCode(HttpStatus.NO_CONTENT) + @Authenticated() + sendSyncAck(@Auth() auth: AuthDto, @Body() dto: SyncAckSetDto) { + return this.service.setAcks(auth, dto); + } + + @Delete('ack') + @HttpCode(HttpStatus.NO_CONTENT) + @Authenticated() + deleteSyncAck(@Auth() auth: AuthDto, @Body() dto: SyncAckDeleteDto) { + return this.service.deleteAcks(auth, dto); + } } diff --git a/server/src/database.ts b/server/src/database.ts index 4fcab0fd6db5..c3fb4cbab4cd 100644 --- a/server/src/database.ts +++ b/server/src/database.ts @@ -1,3 +1,4 @@ +import { sql } from 'kysely'; import { Permission } from 'src/enum'; export type AuthUser = { @@ -29,6 +30,8 @@ export type AuthSession = { }; export const columns = { + ackEpoch: (columnName: 'createdAt' | 'updatedAt' | 'deletedAt') => + sql.raw(`extract(epoch from "${columnName}")::text`).as('ackEpoch'), authUser: [ 'users.id', 'users.name', diff --git a/server/src/db.d.ts b/server/src/db.d.ts index 2e10e1adeda6..255ac8cd200f 100644 --- a/server/src/db.d.ts +++ b/server/src/db.d.ts @@ -4,7 +4,7 @@ */ import type { ColumnType } from 'kysely'; -import { Permission } from 'src/enum'; +import { Permission, SyncEntityType } from 'src/enum'; export type ArrayType = ArrayTypeImpl extends (infer U)[] ? U[] : ArrayTypeImpl; @@ -294,6 +294,15 @@ export interface Sessions { userId: string; } +export interface SessionSyncCheckpoints { + ack: string; + createdAt: Generated; + sessionId: string; + type: SyncEntityType; + updatedAt: Generated; +} + + export interface SharedLinkAsset { assetsId: string; sharedLinksId: string; @@ -384,6 +393,11 @@ export interface Users { updatedAt: Generated; } +export interface UsersAudit { + userId: string; + deletedAt: Generated; +} + export interface VectorsPgVectorIndexStat { idx_growing: ArrayType | null; idx_indexing: boolean | null; @@ -429,6 +443,7 @@ export interface DB { partners: Partners; person: Person; sessions: Sessions; + session_sync_checkpoints: SessionSyncCheckpoints; shared_link__asset: SharedLinkAsset; shared_links: SharedLinks; smart_search: SmartSearch; @@ -440,6 +455,7 @@ export interface DB { typeorm_metadata: TypeormMetadata; user_metadata: UserMetadata; users: Users; + users_audit: UsersAudit; 'vectors.pg_vector_index_stat': VectorsPgVectorIndexStat; version_history: VersionHistory; } diff --git a/server/src/dtos/sync.dto.ts b/server/src/dtos/sync.dto.ts index 820de8d6c332..0628a566cd67 100644 --- a/server/src/dtos/sync.dto.ts +++ b/server/src/dtos/sync.dto.ts @@ -1,7 +1,8 @@ import { ApiProperty } from '@nestjs/swagger'; -import { IsInt, IsPositive } from 'class-validator'; +import { IsEnum, IsInt, IsPositive, IsString } from 'class-validator'; import { AssetResponseDto } from 'src/dtos/asset-response.dto'; -import { ValidateDate, ValidateUUID } from 'src/validation'; +import { SyncEntityType, SyncRequestType } from 'src/enum'; +import { Optional, ValidateDate, ValidateUUID } from 'src/validation'; export class AssetFullSyncDto { @ValidateUUID({ optional: true }) @@ -32,3 +33,51 @@ export class AssetDeltaSyncResponseDto { upserted!: AssetResponseDto[]; deleted!: string[]; } + +export class SyncUserV1 { + id!: string; + name!: string; + email!: string; + deletedAt!: Date | null; +} + +export class SyncUserDeleteV1 { + userId!: string; +} + +export type SyncItem = { + [SyncEntityType.UserV1]: SyncUserV1; + [SyncEntityType.UserDeleteV1]: SyncUserDeleteV1; +}; + +const responseDtos = [ + // + SyncUserV1, + SyncUserDeleteV1, +]; + +export const extraSyncModels = responseDtos; + +export class SyncStreamDto { + @IsEnum(SyncRequestType, { each: true }) + @ApiProperty({ enumName: 'SyncRequestType', enum: SyncRequestType, isArray: true }) + types!: SyncRequestType[]; +} + +export class SyncAckDto { + @ApiProperty({ enumName: 'SyncEntityType', enum: SyncEntityType }) + type!: SyncEntityType; + ack!: string; +} + +export class SyncAckSetDto { + @IsString({ each: true }) + acks!: string[]; +} + +export class SyncAckDeleteDto { + @IsEnum(SyncEntityType, { each: true }) + @ApiProperty({ enumName: 'SyncEntityType', enum: SyncEntityType, isArray: true }) + @Optional() + types?: SyncEntityType[]; +} diff --git a/server/src/entities/index.ts b/server/src/entities/index.ts index 75e92038acb2..a1df269c0989 100644 --- a/server/src/entities/index.ts +++ b/server/src/entities/index.ts @@ -20,8 +20,10 @@ import { SessionEntity } from 'src/entities/session.entity'; import { SharedLinkEntity } from 'src/entities/shared-link.entity'; import { SmartSearchEntity } from 'src/entities/smart-search.entity'; import { StackEntity } from 'src/entities/stack.entity'; +import { SessionSyncCheckpointEntity } from 'src/entities/sync-checkpoint.entity'; import { SystemMetadataEntity } from 'src/entities/system-metadata.entity'; import { TagEntity } from 'src/entities/tag.entity'; +import { UserAuditEntity } from 'src/entities/user-audit.entity'; import { UserMetadataEntity } from 'src/entities/user-metadata.entity'; import { UserEntity } from 'src/entities/user.entity'; import { VersionHistoryEntity } from 'src/entities/version-history.entity'; @@ -44,12 +46,14 @@ export const entities = [ MoveEntity, PartnerEntity, PersonEntity, + SessionSyncCheckpointEntity, SharedLinkEntity, SmartSearchEntity, StackEntity, SystemMetadataEntity, TagEntity, UserEntity, + UserAuditEntity, UserMetadataEntity, SessionEntity, LibraryEntity, diff --git a/server/src/entities/sync-checkpoint.entity.ts b/server/src/entities/sync-checkpoint.entity.ts new file mode 100644 index 000000000000..2a91d2386c61 --- /dev/null +++ b/server/src/entities/sync-checkpoint.entity.ts @@ -0,0 +1,24 @@ +import { SessionEntity } from 'src/entities/session.entity'; +import { SyncEntityType } from 'src/enum'; +import { Column, CreateDateColumn, Entity, ManyToOne, PrimaryColumn, UpdateDateColumn } from 'typeorm'; + +@Entity('session_sync_checkpoints') +export class SessionSyncCheckpointEntity { + @ManyToOne(() => SessionEntity, { onDelete: 'CASCADE', onUpdate: 'CASCADE' }) + session?: SessionEntity; + + @PrimaryColumn() + sessionId!: string; + + @PrimaryColumn({ type: 'varchar' }) + type!: SyncEntityType; + + @CreateDateColumn({ type: 'timestamptz' }) + createdAt!: Date; + + @UpdateDateColumn({ type: 'timestamptz' }) + updatedAt!: Date; + + @Column() + ack!: string; +} diff --git a/server/src/entities/user-audit.entity.ts b/server/src/entities/user-audit.entity.ts new file mode 100644 index 000000000000..305994a6d6b0 --- /dev/null +++ b/server/src/entities/user-audit.entity.ts @@ -0,0 +1,14 @@ +import { Column, CreateDateColumn, Entity, Index, PrimaryGeneratedColumn } from 'typeorm'; + +@Entity('users_audit') +@Index('IDX_users_audit_deleted_at_asc_user_id_asc', ['deletedAt', 'userId']) +export class UserAuditEntity { + @PrimaryGeneratedColumn('increment') + id!: number; + + @Column({ type: 'uuid' }) + userId!: string; + + @CreateDateColumn({ type: 'timestamptz' }) + deletedAt!: Date; +} diff --git a/server/src/entities/user.entity.ts b/server/src/entities/user.entity.ts index 3f5b470ce467..b597d15cf91c 100644 --- a/server/src/entities/user.entity.ts +++ b/server/src/entities/user.entity.ts @@ -10,12 +10,14 @@ import { CreateDateColumn, DeleteDateColumn, Entity, + Index, OneToMany, PrimaryGeneratedColumn, UpdateDateColumn, } from 'typeorm'; @Entity('users') +@Index('IDX_users_updated_at_asc_id_asc', ['updatedAt', 'id']) export class UserEntity { @PrimaryGeneratedColumn('uuid') id!: string; diff --git a/server/src/enum.ts b/server/src/enum.ts index 0c1fb01a1231..b99518c4ff8b 100644 --- a/server/src/enum.ts +++ b/server/src/enum.ts @@ -537,3 +537,12 @@ export enum DatabaseLock { GetSystemConfig = 69, BackupDatabase = 42, } + +export enum SyncRequestType { + UsersV1 = 'UsersV1', +} + +export enum SyncEntityType { + UserV1 = 'UserV1', + UserDeleteV1 = 'UserDeleteV1', +} diff --git a/server/src/middleware/global-exception.filter.ts b/server/src/middleware/global-exception.filter.ts index 7d7ade471e73..a8afa91cbcad 100644 --- a/server/src/middleware/global-exception.filter.ts +++ b/server/src/middleware/global-exception.filter.ts @@ -22,6 +22,13 @@ export class GlobalExceptionFilter implements ExceptionFilter { } } + handleError(res: Response, error: Error) { + const { status, body } = this.fromError(error); + if (!res.headersSent) { + res.status(status).json({ ...body, statusCode: status, correlationId: this.cls.getId() }); + } + } + private fromError(error: Error) { logGlobalError(this.logger, error); diff --git a/server/src/migrations/1740001232576-AddSessionSyncCheckpointTable.ts b/server/src/migrations/1740001232576-AddSessionSyncCheckpointTable.ts new file mode 100644 index 000000000000..ef75dd7c0d3b --- /dev/null +++ b/server/src/migrations/1740001232576-AddSessionSyncCheckpointTable.ts @@ -0,0 +1,22 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class AddSessionSyncCheckpointTable1740001232576 implements MigrationInterface { + name = 'AddSessionSyncCheckpointTable1740001232576' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`CREATE TABLE "session_sync_checkpoints" ("sessionId" uuid NOT NULL, "type" character varying NOT NULL, "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "ack" character varying NOT NULL, CONSTRAINT "PK_b846ab547a702863ef7cd9412fb" PRIMARY KEY ("sessionId", "type"))`); + await queryRunner.query(`ALTER TABLE "session_sync_checkpoints" ADD CONSTRAINT "FK_d8ddd9d687816cc490432b3d4bc" FOREIGN KEY ("sessionId") REFERENCES "sessions"("id") ON DELETE CASCADE ON UPDATE CASCADE`); + await queryRunner.query(` + create trigger session_sync_checkpoints_updated_at + before update on session_sync_checkpoints + for each row execute procedure updated_at() + `); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`drop trigger session_sync_checkpoints_updated_at on session_sync_checkpoints`); + await queryRunner.query(`ALTER TABLE "session_sync_checkpoints" DROP CONSTRAINT "FK_d8ddd9d687816cc490432b3d4bc"`); + await queryRunner.query(`DROP TABLE "session_sync_checkpoints"`); + } + +} diff --git a/server/src/migrations/1740064899123-AddUsersAuditTable.ts b/server/src/migrations/1740064899123-AddUsersAuditTable.ts new file mode 100644 index 000000000000..b8f2ce5e3a72 --- /dev/null +++ b/server/src/migrations/1740064899123-AddUsersAuditTable.ts @@ -0,0 +1,34 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class AddUsersAuditTable1740064899123 implements MigrationInterface { + name = 'AddUsersAuditTable1740064899123' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`CREATE INDEX IF NOT EXISTS "IDX_users_updated_at_asc_id_asc" ON "users" ("updatedAt" ASC, "id" ASC);`) + await queryRunner.query(`CREATE TABLE "users_audit" ("id" SERIAL NOT NULL, "userId" uuid NOT NULL, "deletedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), CONSTRAINT "PK_e9b2bdfd90e7eb5961091175180" PRIMARY KEY ("id"))`); + await queryRunner.query(`CREATE INDEX IF NOT EXISTS "IDX_users_audit_deleted_at_asc_user_id_asc" ON "users_audit" ("deletedAt" ASC, "userId" ASC);`) + await queryRunner.query(`CREATE OR REPLACE FUNCTION users_delete_audit() RETURNS TRIGGER AS + $$ + BEGIN + INSERT INTO users_audit ("userId") + SELECT "id" + FROM OLD; + RETURN NULL; + END; + $$ LANGUAGE plpgsql` + ); + await queryRunner.query(`CREATE OR REPLACE TRIGGER users_delete_audit + AFTER DELETE ON users + REFERENCING OLD TABLE AS OLD + FOR EACH STATEMENT + EXECUTE FUNCTION users_delete_audit(); + `); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP TRIGGER users_delete_audit`); + await queryRunner.query(`DROP FUNCTION users_delete_audit`); + await queryRunner.query(`DROP TABLE "users_audit"`); + } + +} diff --git a/server/src/repositories/index.ts b/server/src/repositories/index.ts index d3a8aeeb69ac..180d8ccd4f29 100644 --- a/server/src/repositories/index.ts +++ b/server/src/repositories/index.ts @@ -30,6 +30,7 @@ import { SessionRepository } from 'src/repositories/session.repository'; import { SharedLinkRepository } from 'src/repositories/shared-link.repository'; import { StackRepository } from 'src/repositories/stack.repository'; import { StorageRepository } from 'src/repositories/storage.repository'; +import { SyncRepository } from 'src/repositories/sync.repository'; import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository'; import { TagRepository } from 'src/repositories/tag.repository'; import { TelemetryRepository } from 'src/repositories/telemetry.repository'; @@ -71,6 +72,7 @@ export const repositories = [ SharedLinkRepository, StackRepository, StorageRepository, + SyncRepository, SystemMetadataRepository, TagRepository, TelemetryRepository, diff --git a/server/src/repositories/sync.repository.ts b/server/src/repositories/sync.repository.ts new file mode 100644 index 000000000000..4023bf890e36 --- /dev/null +++ b/server/src/repositories/sync.repository.ts @@ -0,0 +1,80 @@ +import { Injectable } from '@nestjs/common'; +import { Insertable, Kysely, sql } from 'kysely'; +import { InjectKysely } from 'nestjs-kysely'; +import { columns } from 'src/database'; +import { DB, SessionSyncCheckpoints } from 'src/db'; +import { SyncEntityType } from 'src/enum'; +import { SyncAck } from 'src/types'; + +@Injectable() +export class SyncRepository { + constructor(@InjectKysely() private db: Kysely) {} + + getCheckpoints(sessionId: string) { + return this.db + .selectFrom('session_sync_checkpoints') + .select(['type', 'ack']) + .where('sessionId', '=', sessionId) + .execute(); + } + + upsertCheckpoints(items: Insertable[]) { + return this.db + .insertInto('session_sync_checkpoints') + .values(items) + .onConflict((oc) => + oc.columns(['sessionId', 'type']).doUpdateSet((eb) => ({ + ack: eb.ref('excluded.ack'), + })), + ) + .execute(); + } + + deleteCheckpoints(sessionId: string, types?: SyncEntityType[]) { + return this.db + .deleteFrom('session_sync_checkpoints') + .where('sessionId', '=', sessionId) + .$if(!!types, (qb) => qb.where('type', 'in', types!)) + .execute(); + } + + getUserUpserts(ack?: SyncAck) { + return this.db + .selectFrom('users') + .select(['id', 'name', 'email', 'deletedAt']) + .select(columns.ackEpoch('updatedAt')) + .$if(!!ack, (qb) => + qb.where((eb) => + eb.or([ + eb(eb.fn('to_timestamp', [sql.val(ack!.ackEpoch)]), '<', eb.ref('updatedAt')), + eb.and([ + eb(eb.fn('to_timestamp', [sql.val(ack!.ackEpoch)]), '<=', eb.ref('updatedAt')), + eb('id', '>', ack!.ids[0]), + ]), + ]), + ), + ) + .orderBy(['updatedAt asc', 'id asc']) + .stream(); + } + + getUserDeletes(ack?: SyncAck) { + return this.db + .selectFrom('users_audit') + .select(['userId']) + .select(columns.ackEpoch('deletedAt')) + .$if(!!ack, (qb) => + qb.where((eb) => + eb.or([ + eb(eb.fn('to_timestamp', [sql.val(ack!.ackEpoch)]), '<', eb.ref('deletedAt')), + eb.and([ + eb(eb.fn('to_timestamp', [sql.val(ack!.ackEpoch)]), '<=', eb.ref('deletedAt')), + eb('userId', '>', ack!.ids[0]), + ]), + ]), + ), + ) + .orderBy(['deletedAt asc', 'userId asc']) + .stream(); + } +} diff --git a/server/src/services/base.service.ts b/server/src/services/base.service.ts index f476adba1139..63cca43cc2ac 100644 --- a/server/src/services/base.service.ts +++ b/server/src/services/base.service.ts @@ -38,6 +38,7 @@ import { SessionRepository } from 'src/repositories/session.repository'; import { SharedLinkRepository } from 'src/repositories/shared-link.repository'; import { StackRepository } from 'src/repositories/stack.repository'; import { StorageRepository } from 'src/repositories/storage.repository'; +import { SyncRepository } from 'src/repositories/sync.repository'; import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository'; import { TagRepository } from 'src/repositories/tag.repository'; import { TelemetryRepository } from 'src/repositories/telemetry.repository'; @@ -85,6 +86,7 @@ export class BaseService { protected sharedLinkRepository: SharedLinkRepository, protected stackRepository: StackRepository, protected storageRepository: StorageRepository, + protected syncRepository: SyncRepository, protected systemMetadataRepository: SystemMetadataRepository, protected tagRepository: TagRepository, protected telemetryRepository: TelemetryRepository, diff --git a/server/src/services/sync.service.ts b/server/src/services/sync.service.ts index fe967e37e07b..b94e8cfcbfd2 100644 --- a/server/src/services/sync.service.ts +++ b/server/src/services/sync.service.ts @@ -1,18 +1,112 @@ -import { Injectable } from '@nestjs/common'; +import { ForbiddenException, Injectable } from '@nestjs/common'; +import { Insertable } from 'kysely'; import { DateTime } from 'luxon'; +import { Writable } from 'node:stream'; import { AUDIT_LOG_MAX_DURATION } from 'src/constants'; +import { SessionSyncCheckpoints } from 'src/db'; import { AssetResponseDto, mapAsset } from 'src/dtos/asset-response.dto'; import { AuthDto } from 'src/dtos/auth.dto'; -import { AssetDeltaSyncDto, AssetDeltaSyncResponseDto, AssetFullSyncDto } from 'src/dtos/sync.dto'; -import { DatabaseAction, EntityType, Permission } from 'src/enum'; +import { + AssetDeltaSyncDto, + AssetDeltaSyncResponseDto, + AssetFullSyncDto, + SyncAckDeleteDto, + SyncAckSetDto, + SyncStreamDto, +} from 'src/dtos/sync.dto'; +import { DatabaseAction, EntityType, Permission, SyncEntityType, SyncRequestType } from 'src/enum'; import { BaseService } from 'src/services/base.service'; +import { SyncAck } from 'src/types'; import { getMyPartnerIds } from 'src/utils/asset.util'; import { setIsEqual } from 'src/utils/set'; +import { fromAck, serialize } from 'src/utils/sync'; const FULL_SYNC = { needsFullSync: true, deleted: [], upserted: [] }; +const SYNC_TYPES_ORDER = [ + // + SyncRequestType.UsersV1, +]; + +const throwSessionRequired = () => { + throw new ForbiddenException('Sync endpoints cannot be used with API keys'); +}; @Injectable() export class SyncService extends BaseService { + getAcks(auth: AuthDto) { + const sessionId = auth.session?.id; + if (!sessionId) { + return throwSessionRequired(); + } + + return this.syncRepository.getCheckpoints(sessionId); + } + + async setAcks(auth: AuthDto, dto: SyncAckSetDto) { + // TODO ack validation + + const sessionId = auth.session?.id; + if (!sessionId) { + return throwSessionRequired(); + } + + const checkpoints: Insertable[] = []; + for (const ack of dto.acks) { + const { type } = fromAck(ack); + checkpoints.push({ sessionId, type, ack }); + } + + await this.syncRepository.upsertCheckpoints(checkpoints); + } + + async deleteAcks(auth: AuthDto, dto: SyncAckDeleteDto) { + const sessionId = auth.session?.id; + if (!sessionId) { + return throwSessionRequired(); + } + + await this.syncRepository.deleteCheckpoints(sessionId, dto.types); + } + + async stream(auth: AuthDto, response: Writable, dto: SyncStreamDto) { + const sessionId = auth.session?.id; + if (!sessionId) { + return throwSessionRequired(); + } + + const checkpoints = await this.syncRepository.getCheckpoints(sessionId); + const checkpointMap: Partial> = Object.fromEntries( + checkpoints.map(({ type, ack }) => [type, fromAck(ack)]), + ); + + // TODO pre-filter/sort list based on optimal sync order + + for (const type of SYNC_TYPES_ORDER.filter((type) => dto.types.includes(type))) { + switch (type) { + case SyncRequestType.UsersV1: { + const deletes = this.syncRepository.getUserDeletes(checkpointMap[SyncEntityType.UserDeleteV1]); + for await (const { ackEpoch, ...data } of deletes) { + response.write(serialize({ type: SyncEntityType.UserDeleteV1, ackEpoch, ids: [data.userId], data })); + } + + const upserts = this.syncRepository.getUserUpserts(checkpointMap[SyncEntityType.UserV1]); + for await (const { ackEpoch, ...data } of upserts) { + response.write(serialize({ type: SyncEntityType.UserV1, ackEpoch, ids: [data.id], data })); + } + + break; + } + + default: { + this.logger.warn(`Unsupported sync type: ${type}`); + break; + } + } + } + + response.end(); + } + async getFullSync(auth: AuthDto, dto: AssetFullSyncDto): Promise { // mobile implementation is faster if this is a single id const userId = dto.userId || auth.user.id; diff --git a/server/src/types.ts b/server/src/types.ts index 3a331127e6c4..544d35524e91 100644 --- a/server/src/types.ts +++ b/server/src/types.ts @@ -4,6 +4,7 @@ import { ImageFormat, JobName, QueueName, + SyncEntityType, TranscodeTarget, VideoCodec, } from 'src/enum'; @@ -409,3 +410,9 @@ export interface IBulkAsset { addAssetIds: (id: string, assetIds: string[]) => Promise; removeAssetIds: (id: string, assetIds: string[]) => Promise; } + +export type SyncAck = { + type: SyncEntityType; + ackEpoch: string; + ids: string[]; +}; diff --git a/server/src/utils/misc.ts b/server/src/utils/misc.ts index 13969543efa4..e07d0fe03f3a 100644 --- a/server/src/utils/misc.ts +++ b/server/src/utils/misc.ts @@ -12,6 +12,7 @@ import { writeFileSync } from 'node:fs'; import path from 'node:path'; import { SystemConfig } from 'src/config'; import { CLIP_MODEL_INFO, serverVersion } from 'src/constants'; +import { extraSyncModels } from 'src/dtos/sync.dto'; import { ImmichCookie, ImmichHeader, MetadataKey } from 'src/enum'; import { LoggingRepository } from 'src/repositories/logging.repository'; @@ -245,6 +246,7 @@ export const useSwagger = (app: INestApplication, { write }: { write: boolean }) const options: SwaggerDocumentOptions = { operationIdFactory: (controllerKey: string, methodKey: string) => methodKey, + extraModels: extraSyncModels, }; const specification = SwaggerModule.createDocument(app, config, options); diff --git a/server/src/utils/sync.ts b/server/src/utils/sync.ts new file mode 100644 index 000000000000..8e426ab8609e --- /dev/null +++ b/server/src/utils/sync.ts @@ -0,0 +1,30 @@ +import { SyncItem } from 'src/dtos/sync.dto'; +import { SyncEntityType } from 'src/enum'; +import { SyncAck } from 'src/types'; + +type Impossible = { + [P in K]: never; +}; + +type Exact = U & Impossible>; + +export const fromAck = (ack: string): SyncAck => { + const [type, timestamp, ...ids] = ack.split('|'); + return { type: type as SyncEntityType, ackEpoch: timestamp, ids }; +}; + +export const toAck = ({ type, ackEpoch, ids }: SyncAck) => [type, ackEpoch, ...ids].join('|'); + +export const mapJsonLine = (object: unknown) => JSON.stringify(object) + '\n'; + +export const serialize = ({ + type, + ackEpoch, + ids, + data, +}: { + type: T; + ackEpoch: string; + ids: string[]; + data: Exact; +}) => mapJsonLine({ type, data, ack: toAck({ type, ackEpoch, ids }) }); diff --git a/server/test/repositories/sync.repository.mock.ts b/server/test/repositories/sync.repository.mock.ts new file mode 100644 index 000000000000..fbb8ec2f62e7 --- /dev/null +++ b/server/test/repositories/sync.repository.mock.ts @@ -0,0 +1,13 @@ +import { SyncRepository } from 'src/repositories/sync.repository'; +import { RepositoryInterface } from 'src/types'; +import { Mocked, vitest } from 'vitest'; + +export const newSyncRepositoryMock = (): Mocked> => { + return { + getCheckpoints: vitest.fn(), + upsertCheckpoints: vitest.fn(), + deleteCheckpoints: vitest.fn(), + getUserUpserts: vitest.fn(), + getUserDeletes: vitest.fn(), + }; +}; diff --git a/server/test/utils.ts b/server/test/utils.ts index d1dda3eedf5d..ca2272f6b8ae 100644 --- a/server/test/utils.ts +++ b/server/test/utils.ts @@ -34,6 +34,7 @@ import { SessionRepository } from 'src/repositories/session.repository'; import { SharedLinkRepository } from 'src/repositories/shared-link.repository'; import { StackRepository } from 'src/repositories/stack.repository'; import { StorageRepository } from 'src/repositories/storage.repository'; +import { SyncRepository } from 'src/repositories/sync.repository'; import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository'; import { TagRepository } from 'src/repositories/tag.repository'; import { TelemetryRepository } from 'src/repositories/telemetry.repository'; @@ -75,6 +76,7 @@ import { newSessionRepositoryMock } from 'test/repositories/session.repository.m import { newSharedLinkRepositoryMock } from 'test/repositories/shared-link.repository.mock'; import { newStackRepositoryMock } from 'test/repositories/stack.repository.mock'; import { newStorageRepositoryMock } from 'test/repositories/storage.repository.mock'; +import { newSyncRepositoryMock } from 'test/repositories/sync.repository.mock'; import { newSystemMetadataRepositoryMock } from 'test/repositories/system-metadata.repository.mock'; import { newTagRepositoryMock } from 'test/repositories/tag.repository.mock'; import { ITelemetryRepositoryMock, newTelemetryRepositoryMock } from 'test/repositories/telemetry.repository.mock'; @@ -178,6 +180,7 @@ export const newTestService = ( const sharedLinkMock = newSharedLinkRepositoryMock(); const stackMock = newStackRepositoryMock(); const storageMock = newStorageRepositoryMock(); + const syncMock = newSyncRepositoryMock(); const systemMock = newSystemMetadataRepositoryMock(); const tagMock = newTagRepositoryMock(); const telemetryMock = newTelemetryRepositoryMock(); @@ -219,6 +222,7 @@ export const newTestService = ( sharedLinkMock as RepositoryInterface as SharedLinkRepository, stackMock as RepositoryInterface as StackRepository, storageMock as RepositoryInterface as StorageRepository, + syncMock as RepositoryInterface as SyncRepository, systemMock as RepositoryInterface as SystemMetadataRepository, tagMock as RepositoryInterface as TagRepository, telemetryMock as unknown as TelemetryRepository,