From 5e68f8c51940c48daab2ffcf4f16c3c691d5ee8b Mon Sep 17 00:00:00 2001 From: Min Idzelis Date: Wed, 16 Apr 2025 19:24:26 -0400 Subject: [PATCH 01/12] fix: longpress triggers contextmenu (#17602) --- .../assets/thumbnail/thumbnail.svelte | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/web/src/lib/components/assets/thumbnail/thumbnail.svelte b/web/src/lib/components/assets/thumbnail/thumbnail.svelte index 93a4e3c6cc..c21acd8f86 100644 --- a/web/src/lib/components/assets/thumbnail/thumbnail.svelte +++ b/web/src/lib/components/assets/thumbnail/thumbnail.svelte @@ -130,18 +130,30 @@ }; let timer: ReturnType; - const clearLongPressTimer = () => clearTimeout(timer); + + const preventContextMenu = (evt: Event) => evt.preventDefault(); + let disposeables: (() => void)[] = []; + + const clearLongPressTimer = () => { + clearTimeout(timer); + for (const dispose of disposeables) { + dispose(); + } + disposeables = []; + }; let startX: number = 0; let startY: number = 0; function longPress(element: HTMLElement, { onLongPress }: { onLongPress: () => void }) { let didPress = false; - const start = (evt: TouchEvent) => { - startX = evt.changedTouches[0].clientX; - startY = evt.changedTouches[0].clientY; + const start = (evt: PointerEvent) => { + startX = evt.clientX; + startY = evt.clientY; didPress = false; timer = setTimeout(() => { onLongPress(); + element.addEventListener('contextmenu', preventContextMenu, { once: true }); + disposeables.push(() => element.removeEventListener('contextmenu', preventContextMenu)); didPress = true; }, 350); }; @@ -153,13 +165,13 @@ e.preventDefault(); }; element.addEventListener('click', click); - element.addEventListener('touchstart', start, true); - element.addEventListener('touchend', clearLongPressTimer, true); + element.addEventListener('pointerdown', start, true); + element.addEventListener('pointerup', clearLongPressTimer, true); return { destroy: () => { element.removeEventListener('click', click); - element.removeEventListener('touchstart', start, true); - element.removeEventListener('touchend', clearLongPressTimer, true); + element.removeEventListener('pointerdown', start, true); + element.removeEventListener('pointerup', clearLongPressTimer, true); }, }; } From 067338b0edd8e1ed9e3c3c4245c31c1a55978f6b Mon Sep 17 00:00:00 2001 From: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com> Date: Thu, 17 Apr 2025 16:46:52 +0200 Subject: [PATCH 02/12] chore: remove transfer encoding header (#17671) --- mobile/lib/services/backup.service.dart | 1 - 1 file changed, 1 deletion(-) diff --git a/mobile/lib/services/backup.service.dart b/mobile/lib/services/backup.service.dart index a6468f249b..596ad8dc2e 100644 --- a/mobile/lib/services/backup.service.dart +++ b/mobile/lib/services/backup.service.dart @@ -351,7 +351,6 @@ class BackupService { ); baseRequest.headers.addAll(ApiService.getRequestHeaders()); - baseRequest.headers["Transfer-Encoding"] = "chunked"; baseRequest.fields['deviceAssetId'] = asset.localId!; baseRequest.fields['deviceId'] = deviceId; baseRequest.fields['fileCreatedAt'] = From 81ed54aa61a0ff6e94de6e52709d04c1db0a0d44 Mon Sep 17 00:00:00 2001 From: shenlong <139912620+shenlong-tanwen@users.noreply.github.com> Date: Thu, 17 Apr 2025 20:55:27 +0530 Subject: [PATCH 03/12] feat: user sync stream (#16862) * refactor: user entity * chore: rebase fixes * refactor: remove int user Id * refactor: migrate store userId from int to string * refactor: rename uid to id * feat: drift * pr feedback * refactor: move common overrides to mixin * refactor: remove int user Id * refactor: migrate store userId from int to string * refactor: rename uid to id * feat: user & partner sync stream * pr changes * refactor: sync service and add tests * chore: remove generated change * chore: move sync model * rebase: convert string ids to byte uuids * rebase * add processing logs * batch db calls * rewrite isolate manager * rewrite with worker_manager * misc fixes * add sync order test --------- Co-authored-by: shenlong-tanwen <139912620+shalong-tanwen@users.noreply.github.com> Co-authored-by: Alex --- mobile/analysis_options.yaml | 8 +- mobile/devtools_options.yaml | 1 + mobile/lib/constants/constants.dart | 4 + .../domain/interfaces/sync_api.interface.dart | 7 +- .../interfaces/sync_stream.interface.dart | 10 + .../domain/models/sync/sync_event.model.dart | 14 - .../lib/domain/models/sync_event.model.dart | 13 + .../domain/services/sync_stream.service.dart | 217 +++++++-- mobile/lib/domain/utils/background_sync.dart | 37 ++ mobile/lib/extensions/string_extensions.dart | 9 + .../repositories/sync_api.repository.dart | 93 ++-- .../repositories/sync_stream.repository.dart | 104 ++++ mobile/lib/main.dart | 5 +- .../providers/background_sync.provider.dart | 8 + .../infrastructure/cancel.provider.dart | 12 + .../providers/infrastructure/db.provider.dart | 9 + .../infrastructure/sync_stream.provider.dart | 27 +- mobile/lib/services/auth.service.dart | 11 +- mobile/lib/utils/bootstrap.dart | 6 +- mobile/lib/utils/isolate.dart | 69 +++ mobile/pubspec.lock | 10 +- mobile/pubspec.yaml | 2 + mobile/test/domain/service.mock.dart | 3 + .../services/sync_stream_service_test.dart | 443 ++++++++++++++++++ mobile/test/fixtures/sync_stream.stub.dart | 45 ++ .../test/infrastructure/repository.mock.dart | 6 + mobile/test/service.mocks.dart | 1 - mobile/test/services/auth.service_test.dart | 8 + 28 files changed, 1065 insertions(+), 117 deletions(-) create mode 100644 mobile/lib/domain/interfaces/sync_stream.interface.dart delete mode 100644 mobile/lib/domain/models/sync/sync_event.model.dart create mode 100644 mobile/lib/domain/models/sync_event.model.dart create mode 100644 mobile/lib/domain/utils/background_sync.dart create mode 100644 mobile/lib/infrastructure/repositories/sync_stream.repository.dart create mode 100644 mobile/lib/providers/background_sync.provider.dart create mode 100644 mobile/lib/providers/infrastructure/cancel.provider.dart create mode 100644 mobile/lib/utils/isolate.dart create mode 100644 mobile/test/domain/services/sync_stream_service_test.dart create mode 100644 mobile/test/fixtures/sync_stream.stub.dart diff --git a/mobile/analysis_options.yaml b/mobile/analysis_options.yaml index 04f3145908..854f852e3c 100644 --- a/mobile/analysis_options.yaml +++ b/mobile/analysis_options.yaml @@ -35,6 +35,7 @@ linter: analyzer: exclude: - openapi/** + - build/** - lib/generated_plugin_registrant.dart - lib/**/*.g.dart - lib/**/*.drift.dart @@ -92,6 +93,9 @@ custom_lint: allowed: # required / wanted - lib/repositories/*_api.repository.dart + - lib/domain/models/sync_event.model.dart + - lib/{domain,infrastructure}/**/sync_stream.* + - lib/{domain,infrastructure}/**/sync_api.* - lib/infrastructure/repositories/*_api.repository.dart - lib/infrastructure/utils/*.converter.dart # acceptable exceptions for the time being @@ -144,7 +148,9 @@ dart_code_metrics: - avoid-global-state - avoid-inverted-boolean-checks - avoid-late-final-reassignment - - avoid-local-functions + - avoid-local-functions: + exclude: + - test/**.dart - avoid-negated-conditions - avoid-nested-streams-and-futures - avoid-referencing-subclasses diff --git a/mobile/devtools_options.yaml b/mobile/devtools_options.yaml index fa0b357c4f..f592d85a9b 100644 --- a/mobile/devtools_options.yaml +++ b/mobile/devtools_options.yaml @@ -1,3 +1,4 @@ description: This file stores settings for Dart & Flutter DevTools. documentation: https://docs.flutter.dev/tools/devtools/extensions#configure-extension-enablement-states extensions: + - drift: true \ No newline at end of file diff --git a/mobile/lib/constants/constants.dart b/mobile/lib/constants/constants.dart index 83d540d54c..a91e0a715d 100644 --- a/mobile/lib/constants/constants.dart +++ b/mobile/lib/constants/constants.dart @@ -5,5 +5,9 @@ const double downloadFailed = -2; // Number of log entries to retain on app start const int kLogTruncateLimit = 250; +// Sync +const int kSyncEventBatchSize = 5000; + +// Hash batch limits const int kBatchHashFileLimit = 128; const int kBatchHashSizeLimit = 1024 * 1024 * 1024; // 1GB diff --git a/mobile/lib/domain/interfaces/sync_api.interface.dart b/mobile/lib/domain/interfaces/sync_api.interface.dart index fb8f1aa46e..44e22c5894 100644 --- a/mobile/lib/domain/interfaces/sync_api.interface.dart +++ b/mobile/lib/domain/interfaces/sync_api.interface.dart @@ -1,7 +1,8 @@ -import 'package:immich_mobile/domain/models/sync/sync_event.model.dart'; +import 'package:immich_mobile/domain/models/sync_event.model.dart'; +import 'package:openapi/api.dart'; abstract interface class ISyncApiRepository { - Future ack(String data); + Future ack(List data); - Stream> watchUserSyncEvent(); + Stream> getSyncEvents(List type); } diff --git a/mobile/lib/domain/interfaces/sync_stream.interface.dart b/mobile/lib/domain/interfaces/sync_stream.interface.dart new file mode 100644 index 0000000000..f9c52d7ee0 --- /dev/null +++ b/mobile/lib/domain/interfaces/sync_stream.interface.dart @@ -0,0 +1,10 @@ +import 'package:immich_mobile/domain/interfaces/db.interface.dart'; +import 'package:openapi/api.dart'; + +abstract interface class ISyncStreamRepository implements IDatabaseRepository { + Future updateUsersV1(Iterable data); + Future deleteUsersV1(Iterable data); + + Future updatePartnerV1(Iterable data); + Future deletePartnerV1(Iterable data); +} diff --git a/mobile/lib/domain/models/sync/sync_event.model.dart b/mobile/lib/domain/models/sync/sync_event.model.dart deleted file mode 100644 index f4642d59cf..0000000000 --- a/mobile/lib/domain/models/sync/sync_event.model.dart +++ /dev/null @@ -1,14 +0,0 @@ -class SyncEvent { - // dynamic - final dynamic data; - - final String ack; - - SyncEvent({ - required this.data, - required this.ack, - }); - - @override - String toString() => 'SyncEvent(data: $data, ack: $ack)'; -} diff --git a/mobile/lib/domain/models/sync_event.model.dart b/mobile/lib/domain/models/sync_event.model.dart new file mode 100644 index 0000000000..2ad8a75fe1 --- /dev/null +++ b/mobile/lib/domain/models/sync_event.model.dart @@ -0,0 +1,13 @@ +import 'package:openapi/api.dart'; + +class SyncEvent { + final SyncEntityType type; + // ignore: avoid-dynamic + final dynamic data; + final String ack; + + const SyncEvent({required this.type, required this.data, required this.ack}); + + @override + String toString() => 'SyncEvent(type: $type, data: $data, ack: $ack)'; +} diff --git a/mobile/lib/domain/services/sync_stream.service.dart b/mobile/lib/domain/services/sync_stream.service.dart index 72e29b3677..8d7d87e35e 100644 --- a/mobile/lib/domain/services/sync_stream.service.dart +++ b/mobile/lib/domain/services/sync_stream.service.dart @@ -1,49 +1,200 @@ +// ignore_for_file: avoid-passing-async-when-sync-expected + import 'dart:async'; -import 'package:flutter/foundation.dart'; +import 'package:collection/collection.dart'; import 'package:immich_mobile/domain/interfaces/sync_api.interface.dart'; +import 'package:immich_mobile/domain/interfaces/sync_stream.interface.dart'; +import 'package:logging/logging.dart'; import 'package:openapi/api.dart'; +import 'package:worker_manager/worker_manager.dart'; + +const _kSyncTypeOrder = [ + SyncEntityType.userDeleteV1, + SyncEntityType.userV1, + SyncEntityType.partnerDeleteV1, + SyncEntityType.partnerV1, + SyncEntityType.assetDeleteV1, + SyncEntityType.assetV1, + SyncEntityType.assetExifV1, + SyncEntityType.partnerAssetDeleteV1, + SyncEntityType.partnerAssetV1, + SyncEntityType.partnerAssetExifV1, +]; class SyncStreamService { + final Logger _logger = Logger('SyncStreamService'); + final ISyncApiRepository _syncApiRepository; + final ISyncStreamRepository _syncStreamRepository; + final bool Function()? _cancelChecker; - SyncStreamService(this._syncApiRepository); + SyncStreamService({ + required ISyncApiRepository syncApiRepository, + required ISyncStreamRepository syncStreamRepository, + bool Function()? cancelChecker, + }) : _syncApiRepository = syncApiRepository, + _syncStreamRepository = syncStreamRepository, + _cancelChecker = cancelChecker; - StreamSubscription? _userSyncSubscription; + Future _handleSyncData( + SyncEntityType type, + // ignore: avoid-dynamic + Iterable data, + ) async { + if (data.isEmpty) { + _logger.warning("Received empty sync data for $type"); + return false; + } - void syncUsers() { - _userSyncSubscription = - _syncApiRepository.watchUserSyncEvent().listen((events) async { - for (final event in events) { - if (event.data is SyncUserV1) { - final data = event.data as SyncUserV1; - debugPrint("User Update: $data"); + _logger.fine("Processing sync data for $type of length ${data.length}"); - // final user = await _userRepository.get(data.id); - - // if (user == null) { - // continue; - // } - - // user.name = data.name; - // user.email = data.email; - // user.updatedAt = DateTime.now(); - - // await _userRepository.update(user); - // await _syncApiRepository.ack(event.ack); - } - - if (event.data is SyncUserDeleteV1) { - final data = event.data as SyncUserDeleteV1; - - debugPrint("User delete: $data"); - // await _syncApiRepository.ack(event.ack); - } + try { + if (type == SyncEntityType.partnerV1) { + return await _syncStreamRepository.updatePartnerV1(data.cast()); } + + if (type == SyncEntityType.partnerDeleteV1) { + return await _syncStreamRepository.deletePartnerV1(data.cast()); + } + + if (type == SyncEntityType.userV1) { + return await _syncStreamRepository.updateUsersV1(data.cast()); + } + + if (type == SyncEntityType.userDeleteV1) { + return await _syncStreamRepository.deleteUsersV1(data.cast()); + } + } catch (error, stack) { + _logger.severe("Error processing sync data for $type", error, stack); + return false; + } + + _logger.warning("Unknown sync data type: $type"); + return false; + } + + Future _syncEvent(List types) { + _logger.info("Syncing Events: $types"); + final streamCompleter = Completer(); + bool shouldComplete = false; + // the onDone callback might fire before the events are processed + // the following flag ensures that the onDone callback is not called + // before the events are processed and also that events are processed sequentially + Completer? mutex; + StreamSubscription? subscription; + try { + subscription = _syncApiRepository.getSyncEvents(types).listen( + (events) async { + if (events.isEmpty) { + _logger.warning("Received empty sync events"); + return; + } + + // If previous events are still being processed, wait for them to finish + if (mutex != null) { + await mutex!.future; + } + + if (_cancelChecker?.call() ?? false) { + _logger.info("Sync cancelled, stopping stream"); + subscription?.cancel(); + if (!streamCompleter.isCompleted) { + streamCompleter.completeError( + CanceledError(), + StackTrace.current, + ); + } + return; + } + + // Take control of the mutex and process the events + mutex = Completer(); + + try { + final eventsMap = events.groupListsBy((event) => event.type); + final Map acks = {}; + + for (final type in _kSyncTypeOrder) { + final data = eventsMap[type]; + if (data == null) { + continue; + } + + if (_cancelChecker?.call() ?? false) { + _logger.info("Sync cancelled, stopping stream"); + mutex?.complete(); + mutex = null; + if (!streamCompleter.isCompleted) { + streamCompleter.completeError( + CanceledError(), + StackTrace.current, + ); + } + + return; + } + + if (data.isEmpty) { + _logger.warning("Received empty sync events for $type"); + continue; + } + + if (await _handleSyncData(type, data.map((e) => e.data))) { + // ignore: avoid-unsafe-collection-methods + acks[type] = data.last.ack; + } else { + _logger.warning("Failed to handle sync events for $type"); + } + } + + if (acks.isNotEmpty) { + await _syncApiRepository.ack(acks.values.toList()); + } + _logger.info("$types events processed"); + } catch (error, stack) { + _logger.warning("Error handling sync events", error, stack); + } finally { + mutex?.complete(); + mutex = null; + } + + if (shouldComplete) { + _logger.info("Sync done, completing stream"); + if (!streamCompleter.isCompleted) streamCompleter.complete(); + } + }, + onError: (error, stack) { + _logger.warning("Error in sync stream for $types", error, stack); + // Do not proceed if the stream errors + if (!streamCompleter.isCompleted) { + // ignore: avoid-missing-completer-stack-trace + streamCompleter.completeError(error, stack); + } + }, + onDone: () { + _logger.info("$types stream done"); + if (mutex == null && !streamCompleter.isCompleted) { + streamCompleter.complete(); + } else { + // Marks the stream as done but does not complete the completer + // until the events are processed + shouldComplete = true; + } + }, + ); + } catch (error, stack) { + _logger.severe("Error starting sync stream", error, stack); + if (!streamCompleter.isCompleted) { + streamCompleter.completeError(error, stack); + } + } + return streamCompleter.future.whenComplete(() { + _logger.info("Sync stream completed"); + return subscription?.cancel(); }); } - Future dispose() async { - await _userSyncSubscription?.cancel(); - } + Future syncUsers() => + _syncEvent([SyncRequestType.usersV1, SyncRequestType.partnersV1]); } diff --git a/mobile/lib/domain/utils/background_sync.dart b/mobile/lib/domain/utils/background_sync.dart new file mode 100644 index 0000000000..0bd456f0bb --- /dev/null +++ b/mobile/lib/domain/utils/background_sync.dart @@ -0,0 +1,37 @@ +// ignore_for_file: avoid-passing-async-when-sync-expected + +import 'dart:async'; + +import 'package:immich_mobile/providers/infrastructure/sync_stream.provider.dart'; +import 'package:immich_mobile/utils/isolate.dart'; +import 'package:worker_manager/worker_manager.dart'; + +class BackgroundSyncManager { + Cancelable? _userSyncTask; + + BackgroundSyncManager(); + + Future cancel() { + final futures = []; + if (_userSyncTask != null) { + futures.add(_userSyncTask!.future); + } + _userSyncTask?.cancel(); + _userSyncTask = null; + return Future.wait(futures); + } + + Future syncUsers() { + if (_userSyncTask != null) { + return _userSyncTask!.future; + } + + _userSyncTask = runInIsolateGentle( + computation: (ref) => ref.read(syncStreamServiceProvider).syncUsers(), + ); + _userSyncTask!.whenComplete(() { + _userSyncTask = null; + }); + return _userSyncTask!.future; + } +} diff --git a/mobile/lib/extensions/string_extensions.dart b/mobile/lib/extensions/string_extensions.dart index 67411013ee..73c8c2d34c 100644 --- a/mobile/lib/extensions/string_extensions.dart +++ b/mobile/lib/extensions/string_extensions.dart @@ -1,3 +1,7 @@ +import 'dart:typed_data'; + +import 'package:uuid/parsing.dart'; + extension StringExtension on String { String capitalize() { return split(" ") @@ -29,3 +33,8 @@ extension DurationExtension on String { return int.parse(this); } } + +extension UUIDExtension on String { + Uint8List toUuidByte({bool shouldValidate = false}) => + UuidParsing.parseAsByteList(this, validate: shouldValidate); +} diff --git a/mobile/lib/infrastructure/repositories/sync_api.repository.dart b/mobile/lib/infrastructure/repositories/sync_api.repository.dart index 88a6838c44..a26b867df6 100644 --- a/mobile/lib/infrastructure/repositories/sync_api.repository.dart +++ b/mobile/lib/infrastructure/repositories/sync_api.repository.dart @@ -1,37 +1,36 @@ import 'dart:async'; import 'dart:convert'; -import 'package:flutter/foundation.dart'; -import 'package:immich_mobile/domain/interfaces/sync_api.interface.dart'; -import 'package:immich_mobile/domain/models/sync/sync_event.model.dart'; -import 'package:immich_mobile/services/api.service.dart'; -import 'package:openapi/api.dart'; import 'package:http/http.dart' as http; +import 'package:immich_mobile/constants/constants.dart'; +import 'package:immich_mobile/domain/interfaces/sync_api.interface.dart'; +import 'package:immich_mobile/domain/models/sync_event.model.dart'; +import 'package:immich_mobile/services/api.service.dart'; +import 'package:logging/logging.dart'; +import 'package:openapi/api.dart'; class SyncApiRepository implements ISyncApiRepository { + final Logger _logger = Logger('SyncApiRepository'); final ApiService _api; - const SyncApiRepository(this._api); + final int _batchSize; + SyncApiRepository(this._api, {int batchSize = kSyncEventBatchSize}) + : _batchSize = batchSize; @override - Stream> watchUserSyncEvent() { - return _getSyncStream( - SyncStreamDto(types: [SyncRequestType.usersV1]), - ); + Stream> getSyncEvents(List type) { + return _getSyncStream(SyncStreamDto(types: type)); } @override - Future ack(String data) { - return _api.syncApi.sendSyncAck(SyncAckSetDto(acks: [data])); + Future ack(List data) { + return _api.syncApi.sendSyncAck(SyncAckSetDto(acks: data)); } - Stream> _getSyncStream( - SyncStreamDto dto, { - int batchSize = 5000, - }) async* { + Stream> _getSyncStream(SyncStreamDto dto) async* { final client = http.Client(); final endpoint = "${_api.apiClient.basePath}/sync/stream"; - final headers = { + final headers = { 'Content-Type': 'application/json', 'Accept': 'application/jsonlines+json', }; @@ -61,52 +60,54 @@ class SyncApiRepository implements ISyncApiRepository { await for (final chunk in response.stream.transform(utf8.decoder)) { previousChunk += chunk; - final parts = previousChunk.split('\n'); + final parts = previousChunk.toString().split('\n'); previousChunk = parts.removeLast(); lines.addAll(parts); - if (lines.length < batchSize) { + if (lines.length < _batchSize) { continue; } - yield await compute(_parseSyncResponse, lines); + yield _parseSyncResponse(lines); lines.clear(); } } finally { if (lines.isNotEmpty) { - yield await compute(_parseSyncResponse, lines); + yield _parseSyncResponse(lines); } client.close(); } } + + List _parseSyncResponse(List lines) { + final List data = []; + + for (final line in lines) { + try { + final jsonData = jsonDecode(line); + final type = SyncEntityType.fromJson(jsonData['type'])!; + final dataJson = jsonData['data']; + final ack = jsonData['ack']; + final converter = _kResponseMap[type]; + if (converter == null) { + _logger.warning("[_parseSyncResponse] Unknown type $type"); + continue; + } + + data.add(SyncEvent(type: type, data: converter(dataJson), ack: ack)); + } catch (error, stack) { + _logger.severe("[_parseSyncResponse] Error parsing json", error, stack); + } + } + + return data; + } } +// ignore: avoid-dynamic const _kResponseMap = { SyncEntityType.userV1: SyncUserV1.fromJson, SyncEntityType.userDeleteV1: SyncUserDeleteV1.fromJson, + SyncEntityType.partnerV1: SyncPartnerV1.fromJson, + SyncEntityType.partnerDeleteV1: SyncPartnerDeleteV1.fromJson, }; - -// Need to be outside of the class to be able to use compute -List _parseSyncResponse(List lines) { - final List data = []; - - for (var line in lines) { - try { - final jsonData = jsonDecode(line); - final type = SyncEntityType.fromJson(jsonData['type'])!; - final dataJson = jsonData['data']; - final ack = jsonData['ack']; - final converter = _kResponseMap[type]; - if (converter == null) { - debugPrint("[_parseSyncReponse] Unknown type $type"); - continue; - } - - data.add(SyncEvent(data: converter(dataJson), ack: ack)); - } catch (error, stack) { - debugPrint("[_parseSyncReponse] Error parsing json $error $stack"); - } - } - - return data; -} diff --git a/mobile/lib/infrastructure/repositories/sync_stream.repository.dart b/mobile/lib/infrastructure/repositories/sync_stream.repository.dart new file mode 100644 index 0000000000..a947a9a66b --- /dev/null +++ b/mobile/lib/infrastructure/repositories/sync_stream.repository.dart @@ -0,0 +1,104 @@ +import 'package:drift/drift.dart'; +import 'package:immich_mobile/domain/interfaces/sync_stream.interface.dart'; +import 'package:immich_mobile/extensions/string_extensions.dart'; +import 'package:immich_mobile/infrastructure/entities/partner.entity.drift.dart'; +import 'package:immich_mobile/infrastructure/entities/user.entity.drift.dart'; +import 'package:immich_mobile/infrastructure/repositories/db.repository.dart'; +import 'package:logging/logging.dart'; +import 'package:openapi/api.dart'; + +class DriftSyncStreamRepository extends DriftDatabaseRepository + implements ISyncStreamRepository { + final Logger _logger = Logger('DriftSyncStreamRepository'); + final Drift _db; + + DriftSyncStreamRepository(super.db) : _db = db; + + @override + Future deleteUsersV1(Iterable data) async { + try { + await _db.batch((batch) { + for (final user in data) { + batch.delete( + _db.userEntity, + UserEntityCompanion(id: Value(user.userId.toUuidByte())), + ); + } + }); + return true; + } catch (e, s) { + _logger.severe('Error while processing SyncUserDeleteV1', e, s); + return false; + } + } + + @override + Future updateUsersV1(Iterable data) async { + try { + await _db.batch((batch) { + for (final user in data) { + final companion = UserEntityCompanion( + name: Value(user.name), + email: Value(user.email), + ); + + batch.insert( + _db.userEntity, + companion.copyWith(id: Value(user.id.toUuidByte())), + onConflict: DoUpdate((_) => companion), + ); + } + }); + return true; + } catch (e, s) { + _logger.severe('Error while processing SyncUserV1', e, s); + return false; + } + } + + @override + Future deletePartnerV1(Iterable data) async { + try { + await _db.batch((batch) { + for (final partner in data) { + batch.delete( + _db.partnerEntity, + PartnerEntityCompanion( + sharedById: Value(partner.sharedById.toUuidByte()), + sharedWithId: Value(partner.sharedWithId.toUuidByte()), + ), + ); + } + }); + return true; + } catch (e, s) { + _logger.severe('Error while processing SyncPartnerDeleteV1', e, s); + return false; + } + } + + @override + Future updatePartnerV1(Iterable data) async { + try { + await _db.batch((batch) { + for (final partner in data) { + final companion = + PartnerEntityCompanion(inTimeline: Value(partner.inTimeline)); + + batch.insert( + _db.partnerEntity, + companion.copyWith( + sharedById: Value(partner.sharedById.toUuidByte()), + sharedWithId: Value(partner.sharedWithId.toUuidByte()), + ), + onConflict: DoUpdate((_) => companion), + ); + } + }); + return true; + } catch (e, s) { + _logger.severe('Error while processing SyncPartnerV1', e, s); + return false; + } + } +} diff --git a/mobile/lib/main.dart b/mobile/lib/main.dart index 1a434aa359..73af81d69d 100644 --- a/mobile/lib/main.dart +++ b/mobile/lib/main.dart @@ -11,6 +11,7 @@ import 'package:flutter_displaymode/flutter_displaymode.dart'; import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/constants/locales.dart'; import 'package:immich_mobile/extensions/build_context_extensions.dart'; +import 'package:immich_mobile/generated/codegen_loader.g.dart'; import 'package:immich_mobile/providers/app_life_cycle.provider.dart'; import 'package:immich_mobile/providers/asset_viewer/share_intent_upload.provider.dart'; import 'package:immich_mobile/providers/db.provider.dart'; @@ -31,13 +32,15 @@ import 'package:immich_mobile/utils/migration.dart'; import 'package:intl/date_symbol_data_local.dart'; import 'package:logging/logging.dart'; import 'package:timezone/data/latest.dart'; -import 'package:immich_mobile/generated/codegen_loader.g.dart'; +import 'package:worker_manager/worker_manager.dart'; void main() async { ImmichWidgetsBinding(); final db = await Bootstrap.initIsar(); await Bootstrap.initDomain(db); await initApp(); + // Warm-up isolate pool for worker manager + await workerManager.init(dynamicSpawning: true); await migrateDatabaseIfNeeded(db); HttpOverrides.global = HttpSSLCertOverride(); diff --git a/mobile/lib/providers/background_sync.provider.dart b/mobile/lib/providers/background_sync.provider.dart new file mode 100644 index 0000000000..83d103bb3b --- /dev/null +++ b/mobile/lib/providers/background_sync.provider.dart @@ -0,0 +1,8 @@ +import 'package:hooks_riverpod/hooks_riverpod.dart'; +import 'package:immich_mobile/domain/utils/background_sync.dart'; + +final backgroundSyncProvider = Provider((ref) { + final manager = BackgroundSyncManager(); + ref.onDispose(manager.cancel); + return manager; +}); diff --git a/mobile/lib/providers/infrastructure/cancel.provider.dart b/mobile/lib/providers/infrastructure/cancel.provider.dart new file mode 100644 index 0000000000..6851861e1a --- /dev/null +++ b/mobile/lib/providers/infrastructure/cancel.provider.dart @@ -0,0 +1,12 @@ +import 'package:hooks_riverpod/hooks_riverpod.dart'; + +/// Provider holding a boolean function that returns true when cancellation is requested. +/// A computation running in the isolate uses the function to implement cooperative cancellation. +final cancellationProvider = Provider( + // This will be overridden in the isolate's container. + // Throwing ensures it's not used without an override. + (ref) => throw UnimplementedError( + "cancellationProvider must be overridden in the isolate's ProviderContainer and not to be used in the root isolate", + ), + name: 'cancellationProvider', +); diff --git a/mobile/lib/providers/infrastructure/db.provider.dart b/mobile/lib/providers/infrastructure/db.provider.dart index 84010b3b96..4eefbc556c 100644 --- a/mobile/lib/providers/infrastructure/db.provider.dart +++ b/mobile/lib/providers/infrastructure/db.provider.dart @@ -1,4 +1,7 @@ +import 'dart:async'; + import 'package:hooks_riverpod/hooks_riverpod.dart'; +import 'package:immich_mobile/infrastructure/repositories/db.repository.dart'; import 'package:isar/isar.dart'; import 'package:riverpod_annotation/riverpod_annotation.dart'; @@ -6,3 +9,9 @@ part 'db.provider.g.dart'; @Riverpod(keepAlive: true) Isar isar(Ref ref) => throw UnimplementedError('isar'); + +final driftProvider = Provider((ref) { + final drift = Drift(); + ref.onDispose(() => unawaited(drift.close())); + return drift; +}); diff --git a/mobile/lib/providers/infrastructure/sync_stream.provider.dart b/mobile/lib/providers/infrastructure/sync_stream.provider.dart index 64f1a6cb05..e313982a30 100644 --- a/mobile/lib/providers/infrastructure/sync_stream.provider.dart +++ b/mobile/lib/providers/infrastructure/sync_stream.provider.dart @@ -1,24 +1,23 @@ -import 'dart:async'; - import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/domain/services/sync_stream.service.dart'; import 'package:immich_mobile/infrastructure/repositories/sync_api.repository.dart'; +import 'package:immich_mobile/infrastructure/repositories/sync_stream.repository.dart'; import 'package:immich_mobile/providers/api.provider.dart'; +import 'package:immich_mobile/providers/infrastructure/cancel.provider.dart'; +import 'package:immich_mobile/providers/infrastructure/db.provider.dart'; final syncStreamServiceProvider = Provider( - (ref) { - final instance = SyncStreamService( - ref.watch(syncApiRepositoryProvider), - ); - - ref.onDispose(() => unawaited(instance.dispose())); - - return instance; - }, + (ref) => SyncStreamService( + syncApiRepository: ref.watch(syncApiRepositoryProvider), + syncStreamRepository: ref.watch(syncStreamRepositoryProvider), + cancelChecker: ref.watch(cancellationProvider), + ), ); final syncApiRepositoryProvider = Provider( - (ref) => SyncApiRepository( - ref.watch(apiServiceProvider), - ), + (ref) => SyncApiRepository(ref.watch(apiServiceProvider)), +); + +final syncStreamRepositoryProvider = Provider( + (ref) => DriftSyncStreamRepository(ref.watch(driftProvider)), ); diff --git a/mobile/lib/services/auth.service.dart b/mobile/lib/services/auth.service.dart index 20fa62dc4b..ec053c078b 100644 --- a/mobile/lib/services/auth.service.dart +++ b/mobile/lib/services/auth.service.dart @@ -3,12 +3,14 @@ import 'dart:io'; import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/domain/models/store.model.dart'; +import 'package:immich_mobile/domain/utils/background_sync.dart'; import 'package:immich_mobile/entities/store.entity.dart'; import 'package:immich_mobile/interfaces/auth.interface.dart'; import 'package:immich_mobile/interfaces/auth_api.interface.dart'; import 'package:immich_mobile/models/auth/auxilary_endpoint.model.dart'; import 'package:immich_mobile/models/auth/login_response.model.dart'; import 'package:immich_mobile/providers/api.provider.dart'; +import 'package:immich_mobile/providers/background_sync.provider.dart'; import 'package:immich_mobile/repositories/auth.repository.dart'; import 'package:immich_mobile/repositories/auth_api.repository.dart'; import 'package:immich_mobile/services/api.service.dart'; @@ -22,6 +24,7 @@ final authServiceProvider = Provider( ref.watch(authRepositoryProvider), ref.watch(apiServiceProvider), ref.watch(networkServiceProvider), + ref.watch(backgroundSyncProvider), ), ); @@ -30,6 +33,7 @@ class AuthService { final IAuthRepository _authRepository; final ApiService _apiService; final NetworkService _networkService; + final BackgroundSyncManager _backgroundSyncManager; final _log = Logger("AuthService"); @@ -38,6 +42,7 @@ class AuthService { this._authRepository, this._apiService, this._networkService, + this._backgroundSyncManager, ); /// Validates the provided server URL by resolving and setting the endpoint. @@ -115,8 +120,10 @@ class AuthService { /// - Asset ETag /// /// All deletions are executed in parallel using [Future.wait]. - Future clearLocalData() { - return Future.wait([ + Future clearLocalData() async { + // Cancel any ongoing background sync operations before clearing data + await _backgroundSyncManager.cancel(); + await Future.wait([ _authRepository.clearLocalData(), Store.delete(StoreKey.currentUser), Store.delete(StoreKey.accessToken), diff --git a/mobile/lib/utils/bootstrap.dart b/mobile/lib/utils/bootstrap.dart index 570752c6d9..26f3b49242 100644 --- a/mobile/lib/utils/bootstrap.dart +++ b/mobile/lib/utils/bootstrap.dart @@ -48,11 +48,15 @@ abstract final class Bootstrap { ); } - static Future initDomain(Isar db) async { + static Future initDomain( + Isar db, { + bool shouldBufferLogs = true, + }) async { await StoreService.init(storeRepository: IsarStoreRepository(db)); await LogService.init( logRepository: IsarLogRepository(db), storeRepository: IsarStoreRepository(db), + shouldBuffer: shouldBufferLogs, ); } } diff --git a/mobile/lib/utils/isolate.dart b/mobile/lib/utils/isolate.dart new file mode 100644 index 0000000000..cfbb1b544f --- /dev/null +++ b/mobile/lib/utils/isolate.dart @@ -0,0 +1,69 @@ +import 'dart:async'; +import 'dart:ui'; + +import 'package:flutter/services.dart'; +import 'package:hooks_riverpod/hooks_riverpod.dart'; +import 'package:immich_mobile/providers/db.provider.dart'; +import 'package:immich_mobile/providers/infrastructure/cancel.provider.dart'; +import 'package:immich_mobile/providers/infrastructure/db.provider.dart'; +import 'package:immich_mobile/utils/bootstrap.dart'; +import 'package:logging/logging.dart'; +import 'package:worker_manager/worker_manager.dart'; + +class InvalidIsolateUsageException implements Exception { + const InvalidIsolateUsageException(); + + @override + String toString() => + "IsolateHelper should only be used from the root isolate"; +} + +// !! Should be used only from the root isolate +Cancelable runInIsolateGentle({ + required Future Function(ProviderContainer ref) computation, + String? debugLabel, +}) { + final token = RootIsolateToken.instance; + if (token == null) { + throw const InvalidIsolateUsageException(); + } + + return workerManager.executeGentle((cancelledChecker) async { + BackgroundIsolateBinaryMessenger.ensureInitialized(token); + DartPluginRegistrant.ensureInitialized(); + + final db = await Bootstrap.initIsar(); + await Bootstrap.initDomain(db, shouldBufferLogs: false); + final ref = ProviderContainer( + overrides: [ + // TODO: Remove once isar is removed + dbProvider.overrideWithValue(db), + isarProvider.overrideWithValue(db), + cancellationProvider.overrideWithValue(cancelledChecker), + ], + ); + + Logger log = Logger("IsolateLogger"); + + try { + return await computation(ref); + } on CanceledError { + log.warning( + "Computation cancelled ${debugLabel == null ? '' : ' for $debugLabel'}", + ); + } catch (error, stack) { + log.severe( + "Error in runInIsolateGentle ${debugLabel == null ? '' : ' for $debugLabel'}", + error, + stack, + ); + } finally { + // Wait for the logs to flush + await Future.delayed(const Duration(seconds: 2)); + // Always close the new db connection on Isolate end + ref.read(driftProvider).close(); + ref.read(isarProvider).close(); + } + return null; + }); +} diff --git a/mobile/pubspec.lock b/mobile/pubspec.lock index 3731832296..235b3f71c3 100644 --- a/mobile/pubspec.lock +++ b/mobile/pubspec.lock @@ -1806,7 +1806,7 @@ packages: source: hosted version: "3.1.4" uuid: - dependency: transitive + dependency: "direct main" description: name: uuid sha256: a5be9ef6618a7ac1e964353ef476418026db906c4facdedaa299b7a2e71690ff @@ -1933,6 +1933,14 @@ packages: url: "https://pub.dev" source: hosted version: "0.0.3" + worker_manager: + dependency: "direct main" + description: + name: worker_manager + sha256: "086ed63e9b36266e851404ca90fd44e37c0f4c9bbf819e5f8d7c87f9741c0591" + url: "https://pub.dev" + source: hosted + version: "7.2.3" xdg_directories: dependency: transitive description: diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index 03c39810f6..fdd91e1f87 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -60,7 +60,9 @@ dependencies: thumbhash: 0.1.0+1 timezone: ^0.9.4 url_launcher: ^6.3.1 + uuid: ^4.5.1 wakelock_plus: ^1.2.10 + worker_manager: ^7.2.3 native_video_player: git: diff --git a/mobile/test/domain/service.mock.dart b/mobile/test/domain/service.mock.dart index 53a173fc28..97a3f30294 100644 --- a/mobile/test/domain/service.mock.dart +++ b/mobile/test/domain/service.mock.dart @@ -1,7 +1,10 @@ import 'package:immich_mobile/domain/services/store.service.dart'; import 'package:immich_mobile/domain/services/user.service.dart'; +import 'package:immich_mobile/domain/utils/background_sync.dart'; import 'package:mocktail/mocktail.dart'; class MockStoreService extends Mock implements StoreService {} class MockUserService extends Mock implements UserService {} + +class MockBackgroundSyncManager extends Mock implements BackgroundSyncManager {} diff --git a/mobile/test/domain/services/sync_stream_service_test.dart b/mobile/test/domain/services/sync_stream_service_test.dart new file mode 100644 index 0000000000..e1d8e6987f --- /dev/null +++ b/mobile/test/domain/services/sync_stream_service_test.dart @@ -0,0 +1,443 @@ +// ignore_for_file: avoid-unnecessary-futures, avoid-async-call-in-sync-function + +import 'dart:async'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:immich_mobile/domain/interfaces/sync_api.interface.dart'; +import 'package:immich_mobile/domain/interfaces/sync_stream.interface.dart'; +import 'package:immich_mobile/domain/models/sync_event.model.dart'; +import 'package:immich_mobile/domain/services/sync_stream.service.dart'; +import 'package:mocktail/mocktail.dart'; +import 'package:openapi/api.dart'; +import 'package:worker_manager/worker_manager.dart'; + +import '../../fixtures/sync_stream.stub.dart'; +import '../../infrastructure/repository.mock.dart'; + +class _CancellationWrapper { + const _CancellationWrapper(); + + bool isCancelled() => false; +} + +class _MockCancellationWrapper extends Mock implements _CancellationWrapper {} + +void main() { + late SyncStreamService sut; + late ISyncStreamRepository mockSyncStreamRepo; + late ISyncApiRepository mockSyncApiRepo; + late StreamController> streamController; + + successHandler(Invocation _) async => true; + failureHandler(Invocation _) async => false; + + setUp(() { + mockSyncStreamRepo = MockSyncStreamRepository(); + mockSyncApiRepo = MockSyncApiRepository(); + streamController = StreamController>.broadcast(); + + sut = SyncStreamService( + syncApiRepository: mockSyncApiRepo, + syncStreamRepository: mockSyncStreamRepo, + ); + + // Default stream setup - emits one batch and closes + when(() => mockSyncApiRepo.getSyncEvents(any())) + .thenAnswer((_) => streamController.stream); + + // Default ack setup + when(() => mockSyncApiRepo.ack(any())).thenAnswer((_) async => {}); + + // Register fallbacks for mocktail verification + registerFallbackValue([]); + registerFallbackValue([]); + registerFallbackValue([]); + registerFallbackValue([]); + + // Default successful repository calls + when(() => mockSyncStreamRepo.updateUsersV1(any())) + .thenAnswer(successHandler); + when(() => mockSyncStreamRepo.deleteUsersV1(any())) + .thenAnswer(successHandler); + when(() => mockSyncStreamRepo.updatePartnerV1(any())) + .thenAnswer(successHandler); + when(() => mockSyncStreamRepo.deletePartnerV1(any())) + .thenAnswer(successHandler); + }); + + tearDown(() async { + if (!streamController.isClosed) { + await streamController.close(); + } + }); + + // Helper to trigger sync and add events to the stream + Future triggerSyncAndEmit(List events) async { + final future = sut.syncUsers(); // Start listening + await Future.delayed(Duration.zero); // Allow listener to attach + if (!streamController.isClosed) { + streamController.add(events); + await streamController.close(); // Close after emitting + } + await future; // Wait for processing to complete + } + + group("SyncStreamService", () { + test( + "completes successfully when stream emits data and handlers succeed", + () async { + final events = [ + ...SyncStreamStub.userEvents, + ...SyncStreamStub.partnerEvents, + ]; + final future = triggerSyncAndEmit(events); + await expectLater(future, completes); + // Verify ack includes last ack from each successfully handled type + verify( + () => + mockSyncApiRepo.ack(any(that: containsAll(["5", "2", "4", "3"]))), + ).called(1); + }, + ); + + test("completes successfully when stream emits an error", () async { + when(() => mockSyncApiRepo.getSyncEvents(any())) + .thenAnswer((_) => Stream.error(Exception("Stream Error"))); + // Should complete gracefully without throwing + await expectLater(sut.syncUsers(), throwsException); + verifyNever(() => mockSyncApiRepo.ack(any())); // No ack on stream error + }); + + test("throws when initial getSyncEvents call fails", () async { + final apiException = Exception("API Error"); + when(() => mockSyncApiRepo.getSyncEvents(any())).thenThrow(apiException); + // Should rethrow the exception from the initial call + await expectLater(sut.syncUsers(), throwsA(apiException)); + verifyNever(() => mockSyncApiRepo.ack(any())); + }); + + test( + "completes successfully when a repository handler throws an exception", + () async { + when(() => mockSyncStreamRepo.updateUsersV1(any())) + .thenThrow(Exception("Repo Error")); + final events = [ + ...SyncStreamStub.userEvents, + ...SyncStreamStub.partnerEvents, + ]; + // Should complete, but ack only for the successful types + await triggerSyncAndEmit(events); + // Only partner delete was successful by default setup + verify(() => mockSyncApiRepo.ack(["2", "4", "3"])).called(1); + }, + ); + + test( + "completes successfully but sends no ack when all handlers fail", + () async { + when(() => mockSyncStreamRepo.updateUsersV1(any())) + .thenAnswer(failureHandler); + when(() => mockSyncStreamRepo.deleteUsersV1(any())) + .thenAnswer(failureHandler); + when(() => mockSyncStreamRepo.updatePartnerV1(any())) + .thenAnswer(failureHandler); + when(() => mockSyncStreamRepo.deletePartnerV1(any())) + .thenAnswer(failureHandler); + + final events = [ + ...SyncStreamStub.userEvents, + ...SyncStreamStub.partnerEvents, + ]; + await triggerSyncAndEmit(events); + verifyNever(() => mockSyncApiRepo.ack(any())); + }, + ); + + test("sends ack only for types where handler returns true", () async { + // Mock specific handlers: user update fails, user delete succeeds + when(() => mockSyncStreamRepo.updateUsersV1(any())) + .thenAnswer(failureHandler); + when(() => mockSyncStreamRepo.deleteUsersV1(any())) + .thenAnswer(successHandler); + // partner update fails, partner delete succeeds + when(() => mockSyncStreamRepo.updatePartnerV1(any())) + .thenAnswer(failureHandler); + + final events = [ + ...SyncStreamStub.userEvents, + ...SyncStreamStub.partnerEvents, + ]; + await triggerSyncAndEmit(events); + + // Expect ack only for userDeleteV1 (ack: "2") and partnerDeleteV1 (ack: "4") + verify(() => mockSyncApiRepo.ack(any(that: containsAll(["2", "4"])))) + .called(1); + }); + + test("does not process or ack when stream emits an empty list", () async { + final future = sut.syncUsers(); + streamController.add([]); // Emit empty list + await streamController.close(); + await future; // Wait for completion + + verifyNever(() => mockSyncStreamRepo.updateUsersV1(any())); + verifyNever(() => mockSyncStreamRepo.deleteUsersV1(any())); + verifyNever(() => mockSyncStreamRepo.updatePartnerV1(any())); + verifyNever(() => mockSyncStreamRepo.deletePartnerV1(any())); + verifyNever(() => mockSyncApiRepo.ack(any())); + }); + + test("processes multiple batches sequentially using mutex", () async { + final completer1 = Completer(); + final completer2 = Completer(); + int callOrder = 0; + int handler1StartOrder = -1; + int handler2StartOrder = -1; + int handler1Calls = 0; + int handler2Calls = 0; + + when(() => mockSyncStreamRepo.updateUsersV1(any())).thenAnswer((_) async { + handler1Calls++; + handler1StartOrder = ++callOrder; + await completer1.future; + return true; + }); + when(() => mockSyncStreamRepo.updatePartnerV1(any())) + .thenAnswer((_) async { + handler2Calls++; + handler2StartOrder = ++callOrder; + await completer2.future; + return true; + }); + + final batch1 = SyncStreamStub.userEvents; + final batch2 = SyncStreamStub.partnerEvents; + + final syncFuture = sut.syncUsers(); + await pumpEventQueue(); + + streamController.add(batch1); + await pumpEventQueue(); + // Small delay to ensure the first handler starts + await Future.delayed(const Duration(milliseconds: 20)); + + expect(handler1StartOrder, 1, reason: "Handler 1 should start first"); + expect(handler1Calls, 1); + + streamController.add(batch2); + await pumpEventQueue(); + // Small delay + await Future.delayed(const Duration(milliseconds: 20)); + + expect(handler2StartOrder, -1, reason: "Handler 2 should wait"); + expect(handler2Calls, 0); + + completer1.complete(); + await pumpEventQueue(times: 40); + // Small delay to ensure the second handler starts + await Future.delayed(const Duration(milliseconds: 20)); + + expect(handler2StartOrder, 2, reason: "Handler 2 should start after H1"); + expect(handler2Calls, 1); + + completer2.complete(); + await pumpEventQueue(times: 40); + // Small delay before closing the stream + await Future.delayed(const Duration(milliseconds: 20)); + + if (!streamController.isClosed) { + await streamController.close(); + } + await pumpEventQueue(times: 40); + // Small delay to ensure the sync completes + await Future.delayed(const Duration(milliseconds: 20)); + + await syncFuture; + + verify(() => mockSyncStreamRepo.updateUsersV1(any())).called(1); + verify(() => mockSyncStreamRepo.updatePartnerV1(any())).called(1); + verify(() => mockSyncApiRepo.ack(any())).called(2); + }); + + test( + "stops processing and ack when cancel checker is completed", + () async { + final cancellationChecker = _MockCancellationWrapper(); + when(() => cancellationChecker.isCancelled()).thenAnswer((_) => false); + + sut = SyncStreamService( + syncApiRepository: mockSyncApiRepo, + syncStreamRepository: mockSyncStreamRepo, + cancelChecker: cancellationChecker.isCancelled, + ); + + final processingCompleter = Completer(); + bool handlerStarted = false; + + // Make handler wait so we can cancel it mid-flight + when(() => mockSyncStreamRepo.deleteUsersV1(any())) + .thenAnswer((_) async { + handlerStarted = true; + await processingCompleter + .future; // Wait indefinitely until test completes it + return true; + }); + + final syncFuture = sut.syncUsers(); + await pumpEventQueue(times: 30); + + streamController.add(SyncStreamStub.userEvents); + // Ensure processing starts + await Future.delayed(const Duration(milliseconds: 10)); + + expect(handlerStarted, isTrue, reason: "Handler should have started"); + + when(() => cancellationChecker.isCancelled()).thenAnswer((_) => true); + + // Allow cancellation logic to propagate + await Future.delayed(const Duration(milliseconds: 10)); + + // Complete the handler's completer after cancellation signal + // to ensure the cancellation logic itself isn't blocked by the handler. + processingCompleter.complete(); + + await expectLater(syncFuture, throwsA(isA())); + + // Verify that ack was NOT called because processing was cancelled + verifyNever(() => mockSyncApiRepo.ack(any())); + }, + ); + + test("completes successfully when ack call throws an exception", () async { + when(() => mockSyncApiRepo.ack(any())).thenThrow(Exception("Ack Error")); + final events = [ + ...SyncStreamStub.userEvents, + ...SyncStreamStub.partnerEvents, + ]; + + // Should still complete even if ack fails + await triggerSyncAndEmit(events); + verify(() => mockSyncApiRepo.ack(any())) + .called(1); // Verify ack was attempted + }); + + test("waits for processing to finish if onDone called early", () async { + final processingCompleter = Completer(); + bool handlerFinished = false; + + when(() => mockSyncStreamRepo.updateUsersV1(any())).thenAnswer((_) async { + await processingCompleter.future; // Wait inside handler + handlerFinished = true; + return true; + }); + + final syncFuture = sut.syncUsers(); + // Allow listener to attach + // This is necessary to ensure the stream is ready to receive events + await Future.delayed(Duration.zero); + + streamController.add(SyncStreamStub.userEvents); // Emit batch + await Future.delayed( + const Duration(milliseconds: 10), + ); // Ensure processing starts + + await streamController + .close(); // Close stream (triggers onDone internally) + await Future.delayed( + const Duration(milliseconds: 10), + ); // Give onDone a chance to fire + + // At this point, onDone was called, but processing is blocked + expect(handlerFinished, isFalse); + + processingCompleter.complete(); // Allow processing to finish + await syncFuture; // Now the main future should complete + + expect(handlerFinished, isTrue); + verify(() => mockSyncApiRepo.ack(any())).called(1); + }); + + test("processes events in the defined _kSyncTypeOrder", () async { + final future = sut.syncUsers(); + await pumpEventQueue(); + if (!streamController.isClosed) { + final events = [ + SyncEvent( + type: SyncEntityType.partnerV1, + data: SyncStreamStub.partnerV1, + ack: "1", + ), // Should be processed last + SyncEvent( + type: SyncEntityType.userV1, + data: SyncStreamStub.userV1Admin, + ack: "2", + ), // Should be processed second + SyncEvent( + type: SyncEntityType.partnerDeleteV1, + data: SyncStreamStub.partnerDeleteV1, + ack: "3", + ), // Should be processed third + SyncEvent( + type: SyncEntityType.userDeleteV1, + data: SyncStreamStub.userDeleteV1, + ack: "4", + ), // Should be processed first + ]; + + streamController.add(events); + await streamController.close(); + } + await future; + + verifyInOrder([ + () => mockSyncStreamRepo.deleteUsersV1(any()), + () => mockSyncStreamRepo.updateUsersV1(any()), + () => mockSyncStreamRepo.deletePartnerV1(any()), + () => mockSyncStreamRepo.updatePartnerV1(any()), + // Verify ack happens after all processing + () => mockSyncApiRepo.ack(any()), + ]); + }); + }); + + group("syncUsers", () { + test("calls getSyncEvents with correct types", () async { + // Need to close the stream for the future to complete + final future = sut.syncUsers(); + await streamController.close(); + await future; + + verify( + () => mockSyncApiRepo.getSyncEvents([ + SyncRequestType.usersV1, + SyncRequestType.partnersV1, + ]), + ).called(1); + }); + + test("calls repository methods with correctly grouped data", () async { + final events = [ + ...SyncStreamStub.userEvents, + ...SyncStreamStub.partnerEvents, + ]; + await triggerSyncAndEmit(events); + + // Verify each handler was called with the correct list of data payloads + verify( + () => mockSyncStreamRepo.updateUsersV1( + [SyncStreamStub.userV1Admin, SyncStreamStub.userV1User], + ), + ).called(1); + verify( + () => mockSyncStreamRepo.deleteUsersV1([SyncStreamStub.userDeleteV1]), + ).called(1); + verify( + () => mockSyncStreamRepo.updatePartnerV1([SyncStreamStub.partnerV1]), + ).called(1); + verify( + () => mockSyncStreamRepo + .deletePartnerV1([SyncStreamStub.partnerDeleteV1]), + ).called(1); + }); + }); +} diff --git a/mobile/test/fixtures/sync_stream.stub.dart b/mobile/test/fixtures/sync_stream.stub.dart new file mode 100644 index 0000000000..781e63a2bb --- /dev/null +++ b/mobile/test/fixtures/sync_stream.stub.dart @@ -0,0 +1,45 @@ +import 'package:immich_mobile/domain/models/sync_event.model.dart'; +import 'package:openapi/api.dart'; + +abstract final class SyncStreamStub { + static final userV1Admin = SyncUserV1( + deletedAt: DateTime(2020), + email: "admin@admin", + id: "1", + name: "Admin", + ); + static final userV1User = SyncUserV1( + deletedAt: DateTime(2021), + email: "user@user", + id: "2", + name: "User", + ); + static final userDeleteV1 = SyncUserDeleteV1(userId: "2"); + static final userEvents = [ + SyncEvent(type: SyncEntityType.userV1, data: userV1Admin, ack: "1"), + SyncEvent( + type: SyncEntityType.userDeleteV1, + data: userDeleteV1, + ack: "2", + ), + SyncEvent(type: SyncEntityType.userV1, data: userV1User, ack: "5"), + ]; + + static final partnerV1 = SyncPartnerV1( + inTimeline: true, + sharedById: "1", + sharedWithId: "2", + ); + static final partnerDeleteV1 = SyncPartnerDeleteV1( + sharedById: "3", + sharedWithId: "4", + ); + static final partnerEvents = [ + SyncEvent( + type: SyncEntityType.partnerDeleteV1, + data: partnerDeleteV1, + ack: "4", + ), + SyncEvent(type: SyncEntityType.partnerV1, data: partnerV1, ack: "3"), + ]; +} diff --git a/mobile/test/infrastructure/repository.mock.dart b/mobile/test/infrastructure/repository.mock.dart index 192858adff..c4a5680f71 100644 --- a/mobile/test/infrastructure/repository.mock.dart +++ b/mobile/test/infrastructure/repository.mock.dart @@ -1,6 +1,8 @@ import 'package:immich_mobile/domain/interfaces/device_asset.interface.dart'; import 'package:immich_mobile/domain/interfaces/log.interface.dart'; import 'package:immich_mobile/domain/interfaces/store.interface.dart'; +import 'package:immich_mobile/domain/interfaces/sync_api.interface.dart'; +import 'package:immich_mobile/domain/interfaces/sync_stream.interface.dart'; import 'package:immich_mobile/domain/interfaces/user.interface.dart'; import 'package:immich_mobile/domain/interfaces/user_api.interface.dart'; import 'package:mocktail/mocktail.dart'; @@ -14,5 +16,9 @@ class MockUserRepository extends Mock implements IUserRepository {} class MockDeviceAssetRepository extends Mock implements IDeviceAssetRepository {} +class MockSyncStreamRepository extends Mock implements ISyncStreamRepository {} + // API Repos class MockUserApiRepository extends Mock implements IUserApiRepository {} + +class MockSyncApiRepository extends Mock implements ISyncApiRepository {} diff --git a/mobile/test/service.mocks.dart b/mobile/test/service.mocks.dart index e1b8df40a3..87a8c01cf0 100644 --- a/mobile/test/service.mocks.dart +++ b/mobile/test/service.mocks.dart @@ -29,4 +29,3 @@ class MockSearchApi extends Mock implements SearchApi {} class MockAppSettingService extends Mock implements AppSettingsService {} class MockBackgroundService extends Mock implements BackgroundService {} - diff --git a/mobile/test/services/auth.service_test.dart b/mobile/test/services/auth.service_test.dart index e4f011d940..4ada98a6c9 100644 --- a/mobile/test/services/auth.service_test.dart +++ b/mobile/test/services/auth.service_test.dart @@ -8,6 +8,7 @@ import 'package:isar/isar.dart'; import 'package:mocktail/mocktail.dart'; import 'package:openapi/api.dart'; +import '../domain/service.mock.dart'; import '../repository.mocks.dart'; import '../service.mocks.dart'; import '../test_utils.dart'; @@ -18,6 +19,7 @@ void main() { late MockAuthRepository authRepository; late MockApiService apiService; late MockNetworkService networkService; + late MockBackgroundSyncManager backgroundSyncManager; late Isar db; setUp(() async { @@ -25,12 +27,14 @@ void main() { authRepository = MockAuthRepository(); apiService = MockApiService(); networkService = MockNetworkService(); + backgroundSyncManager = MockBackgroundSyncManager(); sut = AuthService( authApiRepository, authRepository, apiService, networkService, + backgroundSyncManager, ); registerFallbackValue(Uri()); @@ -116,24 +120,28 @@ void main() { group('logout', () { test('Should logout user', () async { when(() => authApiRepository.logout()).thenAnswer((_) async => {}); + when(() => backgroundSyncManager.cancel()).thenAnswer((_) async => {}); when(() => authRepository.clearLocalData()) .thenAnswer((_) => Future.value(null)); await sut.logout(); verify(() => authApiRepository.logout()).called(1); + verify(() => backgroundSyncManager.cancel()).called(1); verify(() => authRepository.clearLocalData()).called(1); }); test('Should clear local data even on server error', () async { when(() => authApiRepository.logout()) .thenThrow(Exception('Server error')); + when(() => backgroundSyncManager.cancel()).thenAnswer((_) async => {}); when(() => authRepository.clearLocalData()) .thenAnswer((_) => Future.value(null)); await sut.logout(); verify(() => authApiRepository.logout()).called(1); + verify(() => backgroundSyncManager.cancel()).called(1); verify(() => authRepository.clearLocalData()).called(1); }); }); From e275f2d8b390cb95cb861118747be27507e34493 Mon Sep 17 00:00:00 2001 From: Jason Rasmussen Date: Thu, 17 Apr 2025 14:41:06 -0400 Subject: [PATCH 04/12] feat: add foreign key indexes (#17672) --- server/src/decorators.ts | 3 +- .../1744900200559-AddForeignKeyIndexes.ts | 51 ++++++++++++++ server/src/schema/tables/activity.table.ts | 4 +- server/src/schema/tables/album-asset.table.ts | 18 +---- server/src/schema/tables/album.table.ts | 4 +- server/src/schema/tables/api-key.table.ts | 4 +- server/src/schema/tables/asset-audit.table.ts | 11 ++- server/src/schema/tables/asset-face.table.ts | 15 ++++- server/src/schema/tables/asset-files.table.ts | 11 +-- server/src/schema/tables/asset.table.ts | 16 ++--- server/src/schema/tables/exif.table.ts | 14 ++-- server/src/schema/tables/library.table.ts | 4 +- server/src/schema/tables/memory.table.ts | 4 +- .../src/schema/tables/memory_asset.table.ts | 4 +- .../src/schema/tables/partner-audit.table.ts | 11 ++- server/src/schema/tables/partner.table.ts | 20 +++--- server/src/schema/tables/person.table.ts | 4 +- server/src/schema/tables/session.table.ts | 4 +- .../schema/tables/shared-link-asset.table.ts | 4 +- server/src/schema/tables/shared-link.table.ts | 21 +++--- .../schema/tables/sync-checkpoint.table.ts | 13 +--- server/src/schema/tables/tag-asset.table.ts | 8 +-- server/src/schema/tables/tag-closure.table.ts | 8 +-- server/src/schema/tables/tag.table.ts | 11 +-- server/src/schema/tables/user-audit.table.ts | 5 +- .../src/schema/tables/user-metadata.table.ts | 8 ++- server/src/schema/tables/user.table.ts | 4 +- .../decorators/column-index.decorator.ts | 16 ----- .../from-code/decorators/column.decorator.ts | 6 +- .../foreign-key-column.decorator.ts | 2 - .../from-code/decorators/index.decorator.ts | 9 ++- server/src/sql-tools/from-code/index.ts | 16 +++-- .../processors/check-constraint.processor.ts | 4 +- .../processors/column-index.processor.ts | 32 --------- .../from-code/processors/column.processor.ts | 14 +--- .../foreign-key-constriant.processor.ts | 7 +- .../from-code/processors/index.processor.ts | 67 ++++++++++++++++++- .../primary-key-contraint.processor.ts | 4 +- .../from-code/processors/trigger.processor.ts | 6 +- .../sql-tools/from-code/processors/type.ts | 3 +- .../processors/unique-constraint.processor.ts | 33 ++++++++- .../sql-tools/from-code/register-function.ts | 35 +++++++++- .../src/sql-tools/from-code/register-item.ts | 2 - server/src/sql-tools/helpers.ts | 55 --------------- server/src/sql-tools/public_api.ts | 1 - .../sql-tools/column-index-name-default.ts | 5 +- server/test/sql-tools/column-index-name.ts | 46 +++++++++++++ .../foreign-key-inferred-type.stub.ts | 10 ++- ...foreign-key-with-unique-constraint.stub.ts | 10 ++- 49 files changed, 382 insertions(+), 285 deletions(-) create mode 100644 server/src/migrations/1744900200559-AddForeignKeyIndexes.ts delete mode 100644 server/src/sql-tools/from-code/decorators/column-index.decorator.ts delete mode 100644 server/src/sql-tools/from-code/processors/column-index.processor.ts create mode 100644 server/test/sql-tools/column-index-name.ts diff --git a/server/src/decorators.ts b/server/src/decorators.ts index 7085899af7..1af9342e0b 100644 --- a/server/src/decorators.ts +++ b/server/src/decorators.ts @@ -11,7 +11,8 @@ import { setUnion } from 'src/utils/set'; const GeneratedUuidV7Column = (options: Omit = {}) => Column({ ...options, type: 'uuid', nullable: false, default: () => `${immich_uuid_v7.name}()` }); -export const UpdateIdColumn = () => GeneratedUuidV7Column(); +export const UpdateIdColumn = (options: Omit = {}) => + GeneratedUuidV7Column(options); export const PrimaryGeneratedUuidV7Column = () => GeneratedUuidV7Column({ primary: true }); diff --git a/server/src/migrations/1744900200559-AddForeignKeyIndexes.ts b/server/src/migrations/1744900200559-AddForeignKeyIndexes.ts new file mode 100644 index 0000000000..db351d5bab --- /dev/null +++ b/server/src/migrations/1744900200559-AddForeignKeyIndexes.ts @@ -0,0 +1,51 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AddForeignKeyIndexes1744900200559 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`CREATE INDEX "IDX_0f6fc2fb195f24d19b0fb0d57c" ON "libraries" ("ownerId")`); + await queryRunner.query(`CREATE INDEX "IDX_91704e101438fd0653f582426d" ON "asset_stack" ("primaryAssetId")`); + await queryRunner.query(`CREATE INDEX "IDX_c05079e542fd74de3b5ecb5c1c" ON "asset_stack" ("ownerId")`); + await queryRunner.query(`CREATE INDEX "IDX_2c5ac0d6fb58b238fd2068de67" ON "assets" ("ownerId")`); + await queryRunner.query(`CREATE INDEX "IDX_16294b83fa8c0149719a1f631e" ON "assets" ("livePhotoVideoId")`); + await queryRunner.query(`CREATE INDEX "IDX_9977c3c1de01c3d848039a6b90" ON "assets" ("libraryId")`); + await queryRunner.query(`CREATE INDEX "IDX_f15d48fa3ea5e4bda05ca8ab20" ON "assets" ("stackId")`); + await queryRunner.query(`CREATE INDEX "IDX_b22c53f35ef20c28c21637c85f" ON "albums" ("ownerId")`); + await queryRunner.query(`CREATE INDEX "IDX_05895aa505a670300d4816debc" ON "albums" ("albumThumbnailAssetId")`); + await queryRunner.query(`CREATE INDEX "IDX_1af8519996fbfb3684b58df280" ON "activity" ("albumId")`); + await queryRunner.query(`CREATE INDEX "IDX_3571467bcbe021f66e2bdce96e" ON "activity" ("userId")`); + await queryRunner.query(`CREATE INDEX "IDX_8091ea76b12338cb4428d33d78" ON "activity" ("assetId")`); + await queryRunner.query(`CREATE INDEX "IDX_6c2e267ae764a9413b863a2934" ON "api_keys" ("userId")`); + await queryRunner.query(`CREATE INDEX "IDX_5527cc99f530a547093f9e577b" ON "person" ("ownerId")`); + await queryRunner.query(`CREATE INDEX "IDX_2bbabe31656b6778c6b87b6102" ON "person" ("faceAssetId")`); + await queryRunner.query(`CREATE INDEX "IDX_575842846f0c28fa5da46c99b1" ON "memories" ("ownerId")`); + await queryRunner.query(`CREATE INDEX "IDX_d7e875c6c60e661723dbf372fd" ON "partners" ("sharedWithId")`); + await queryRunner.query(`CREATE INDEX "IDX_57de40bc620f456c7311aa3a1e" ON "sessions" ("userId")`); + await queryRunner.query(`CREATE INDEX "IDX_66fe3837414c5a9f1c33ca4934" ON "shared_links" ("userId")`); + await queryRunner.query(`CREATE INDEX "IDX_d8ddd9d687816cc490432b3d4b" ON "session_sync_checkpoints" ("sessionId")`); + await queryRunner.query(`CREATE INDEX "IDX_9f9590cc11561f1f48ff034ef9" ON "tags" ("parentId")`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX "IDX_66fe3837414c5a9f1c33ca4934";`); + await queryRunner.query(`DROP INDEX "IDX_91704e101438fd0653f582426d";`); + await queryRunner.query(`DROP INDEX "IDX_c05079e542fd74de3b5ecb5c1c";`); + await queryRunner.query(`DROP INDEX "IDX_5527cc99f530a547093f9e577b";`); + await queryRunner.query(`DROP INDEX "IDX_2bbabe31656b6778c6b87b6102";`); + await queryRunner.query(`DROP INDEX "IDX_0f6fc2fb195f24d19b0fb0d57c";`); + await queryRunner.query(`DROP INDEX "IDX_9f9590cc11561f1f48ff034ef9";`); + await queryRunner.query(`DROP INDEX "IDX_2c5ac0d6fb58b238fd2068de67";`); + await queryRunner.query(`DROP INDEX "IDX_16294b83fa8c0149719a1f631e";`); + await queryRunner.query(`DROP INDEX "IDX_9977c3c1de01c3d848039a6b90";`); + await queryRunner.query(`DROP INDEX "IDX_f15d48fa3ea5e4bda05ca8ab20";`); + await queryRunner.query(`DROP INDEX "IDX_b22c53f35ef20c28c21637c85f";`); + await queryRunner.query(`DROP INDEX "IDX_05895aa505a670300d4816debc";`); + await queryRunner.query(`DROP INDEX "IDX_57de40bc620f456c7311aa3a1e";`); + await queryRunner.query(`DROP INDEX "IDX_d8ddd9d687816cc490432b3d4b";`); + await queryRunner.query(`DROP INDEX "IDX_d7e875c6c60e661723dbf372fd";`); + await queryRunner.query(`DROP INDEX "IDX_575842846f0c28fa5da46c99b1";`); + await queryRunner.query(`DROP INDEX "IDX_6c2e267ae764a9413b863a2934";`); + await queryRunner.query(`DROP INDEX "IDX_1af8519996fbfb3684b58df280";`); + await queryRunner.query(`DROP INDEX "IDX_3571467bcbe021f66e2bdce96e";`); + await queryRunner.query(`DROP INDEX "IDX_8091ea76b12338cb4428d33d78";`); + } +} diff --git a/server/src/schema/tables/activity.table.ts b/server/src/schema/tables/activity.table.ts index e7a144722c..802a86a303 100644 --- a/server/src/schema/tables/activity.table.ts +++ b/server/src/schema/tables/activity.table.ts @@ -5,7 +5,6 @@ import { UserTable } from 'src/schema/tables/user.table'; import { Check, Column, - ColumnIndex, CreateDateColumn, ForeignKeyColumn, Index, @@ -51,7 +50,6 @@ export class ActivityTable { @Column({ type: 'boolean', default: false }) isLiked!: boolean; - @ColumnIndex('IDX_activity_update_id') - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_activity_update_id' }) updateId!: string; } diff --git a/server/src/schema/tables/album-asset.table.ts b/server/src/schema/tables/album-asset.table.ts index 1b931e3116..8054009c39 100644 --- a/server/src/schema/tables/album-asset.table.ts +++ b/server/src/schema/tables/album-asset.table.ts @@ -1,25 +1,13 @@ import { AlbumTable } from 'src/schema/tables/album.table'; import { AssetTable } from 'src/schema/tables/asset.table'; -import { ColumnIndex, CreateDateColumn, ForeignKeyColumn, Table } from 'src/sql-tools'; +import { CreateDateColumn, ForeignKeyColumn, Table } from 'src/sql-tools'; @Table({ name: 'albums_assets_assets', primaryConstraintName: 'PK_c67bc36fa845fb7b18e0e398180' }) export class AlbumAssetTable { - @ForeignKeyColumn(() => AlbumTable, { - onDelete: 'CASCADE', - onUpdate: 'CASCADE', - nullable: false, - primary: true, - }) - @ColumnIndex() + @ForeignKeyColumn(() => AlbumTable, { onDelete: 'CASCADE', onUpdate: 'CASCADE', nullable: false, primary: true }) albumsId!: string; - @ForeignKeyColumn(() => AssetTable, { - onDelete: 'CASCADE', - onUpdate: 'CASCADE', - nullable: false, - primary: true, - }) - @ColumnIndex() + @ForeignKeyColumn(() => AssetTable, { onDelete: 'CASCADE', onUpdate: 'CASCADE', nullable: false, primary: true }) assetsId!: string; @CreateDateColumn() diff --git a/server/src/schema/tables/album.table.ts b/server/src/schema/tables/album.table.ts index cdfd092b1b..428947fa51 100644 --- a/server/src/schema/tables/album.table.ts +++ b/server/src/schema/tables/album.table.ts @@ -4,7 +4,6 @@ import { AssetTable } from 'src/schema/tables/asset.table'; import { UserTable } from 'src/schema/tables/user.table'; import { Column, - ColumnIndex, CreateDateColumn, DeleteDateColumn, ForeignKeyColumn, @@ -51,7 +50,6 @@ export class AlbumTable { @Column({ default: AssetOrder.DESC }) order!: AssetOrder; - @ColumnIndex('IDX_albums_update_id') - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_albums_update_id' }) updateId?: string; } diff --git a/server/src/schema/tables/api-key.table.ts b/server/src/schema/tables/api-key.table.ts index 29c4ad2b0f..1d4cc83172 100644 --- a/server/src/schema/tables/api-key.table.ts +++ b/server/src/schema/tables/api-key.table.ts @@ -3,7 +3,6 @@ import { Permission } from 'src/enum'; import { UserTable } from 'src/schema/tables/user.table'; import { Column, - ColumnIndex, CreateDateColumn, ForeignKeyColumn, PrimaryGeneratedColumn, @@ -35,7 +34,6 @@ export class APIKeyTable { @Column({ array: true, type: 'character varying' }) permissions!: Permission[]; - @ColumnIndex({ name: 'IDX_api_keys_update_id' }) - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_api_keys_update_id' }) updateId?: string; } diff --git a/server/src/schema/tables/asset-audit.table.ts b/server/src/schema/tables/asset-audit.table.ts index 55d6f5c911..030256480c 100644 --- a/server/src/schema/tables/asset-audit.table.ts +++ b/server/src/schema/tables/asset-audit.table.ts @@ -1,20 +1,17 @@ import { PrimaryGeneratedUuidV7Column } from 'src/decorators'; -import { Column, ColumnIndex, CreateDateColumn, Table } from 'src/sql-tools'; +import { Column, CreateDateColumn, Table } from 'src/sql-tools'; @Table('assets_audit') export class AssetAuditTable { @PrimaryGeneratedUuidV7Column() id!: string; - @ColumnIndex('IDX_assets_audit_asset_id') - @Column({ type: 'uuid' }) + @Column({ type: 'uuid', indexName: 'IDX_assets_audit_asset_id' }) assetId!: string; - @ColumnIndex('IDX_assets_audit_owner_id') - @Column({ type: 'uuid' }) + @Column({ type: 'uuid', indexName: 'IDX_assets_audit_owner_id' }) ownerId!: string; - @ColumnIndex('IDX_assets_audit_deleted_at') - @CreateDateColumn({ default: () => 'clock_timestamp()' }) + @CreateDateColumn({ default: () => 'clock_timestamp()', indexName: 'IDX_assets_audit_deleted_at' }) deletedAt!: Date; } diff --git a/server/src/schema/tables/asset-face.table.ts b/server/src/schema/tables/asset-face.table.ts index 0ae99f44bf..52f4364a93 100644 --- a/server/src/schema/tables/asset-face.table.ts +++ b/server/src/schema/tables/asset-face.table.ts @@ -8,10 +8,21 @@ import { Column, DeleteDateColumn, ForeignKeyColumn, Index, PrimaryGeneratedColu @Index({ name: 'IDX_asset_faces_assetId_personId', columns: ['assetId', 'personId'] }) @Index({ columns: ['personId', 'assetId'] }) export class AssetFaceTable { - @ForeignKeyColumn(() => AssetTable, { onDelete: 'CASCADE', onUpdate: 'CASCADE' }) + @ForeignKeyColumn(() => AssetTable, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE', + // [assetId, personId] is the PK constraint + index: false, + }) assetId!: string; - @ForeignKeyColumn(() => PersonTable, { onDelete: 'SET NULL', onUpdate: 'CASCADE', nullable: true }) + @ForeignKeyColumn(() => PersonTable, { + onDelete: 'SET NULL', + onUpdate: 'CASCADE', + nullable: true, + // [personId, assetId] makes this redundant + index: false, + }) personId!: string | null; @Column({ default: 0, type: 'integer' }) diff --git a/server/src/schema/tables/asset-files.table.ts b/server/src/schema/tables/asset-files.table.ts index fb8750a8ef..0859bd5cf0 100644 --- a/server/src/schema/tables/asset-files.table.ts +++ b/server/src/schema/tables/asset-files.table.ts @@ -3,7 +3,6 @@ import { AssetFileType } from 'src/enum'; import { AssetTable } from 'src/schema/tables/asset.table'; import { Column, - ColumnIndex, CreateDateColumn, ForeignKeyColumn, PrimaryGeneratedColumn, @@ -19,8 +18,11 @@ export class AssetFileTable { @PrimaryGeneratedColumn() id!: string; - @ColumnIndex('IDX_asset_files_assetId') - @ForeignKeyColumn(() => AssetTable, { onDelete: 'CASCADE', onUpdate: 'CASCADE' }) + @ForeignKeyColumn(() => AssetTable, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE', + indexName: 'IDX_asset_files_assetId', + }) assetId?: string; @CreateDateColumn() @@ -35,7 +37,6 @@ export class AssetFileTable { @Column() path!: string; - @ColumnIndex('IDX_asset_files_update_id') - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_asset_files_update_id' }) updateId?: string; } diff --git a/server/src/schema/tables/asset.table.ts b/server/src/schema/tables/asset.table.ts index 250c3546a2..9a9670cd0e 100644 --- a/server/src/schema/tables/asset.table.ts +++ b/server/src/schema/tables/asset.table.ts @@ -9,7 +9,6 @@ import { UserTable } from 'src/schema/tables/user.table'; import { AfterDeleteTrigger, Column, - ColumnIndex, CreateDateColumn, DeleteDateColumn, ForeignKeyColumn, @@ -78,8 +77,7 @@ export class AssetTable { @Column() originalPath!: string; - @ColumnIndex('idx_asset_file_created_at') - @Column({ type: 'timestamp with time zone' }) + @Column({ type: 'timestamp with time zone', indexName: 'idx_asset_file_created_at' }) fileCreatedAt!: Date; @Column({ type: 'timestamp with time zone' }) @@ -94,8 +92,7 @@ export class AssetTable { @Column({ type: 'character varying', nullable: true, default: '' }) encodedVideoPath!: string | null; - @Column({ type: 'bytea' }) - @ColumnIndex() + @Column({ type: 'bytea', index: true }) checksum!: Buffer; // sha1 checksum @Column({ type: 'boolean', default: true }) @@ -113,8 +110,7 @@ export class AssetTable { @Column({ type: 'boolean', default: false }) isArchived!: boolean; - @Column() - @ColumnIndex() + @Column({ index: true }) originalFileName!: string; @Column({ nullable: true }) @@ -141,14 +137,12 @@ export class AssetTable { @ForeignKeyColumn(() => StackTable, { nullable: true, onDelete: 'SET NULL', onUpdate: 'CASCADE' }) stackId?: string | null; - @ColumnIndex('IDX_assets_duplicateId') - @Column({ type: 'uuid', nullable: true }) + @Column({ type: 'uuid', nullable: true, indexName: 'IDX_assets_duplicateId' }) duplicateId!: string | null; @Column({ enum: assets_status_enum, default: AssetStatus.ACTIVE }) status!: AssetStatus; - @ColumnIndex('IDX_assets_update_id') - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_assets_update_id' }) updateId?: string; } diff --git a/server/src/schema/tables/exif.table.ts b/server/src/schema/tables/exif.table.ts index e40ce94b4f..ca300945c3 100644 --- a/server/src/schema/tables/exif.table.ts +++ b/server/src/schema/tables/exif.table.ts @@ -1,6 +1,6 @@ import { UpdatedAtTrigger, UpdateIdColumn } from 'src/decorators'; import { AssetTable } from 'src/schema/tables/asset.table'; -import { Column, ColumnIndex, ForeignKeyColumn, Table, UpdateDateColumn } from 'src/sql-tools'; +import { Column, ForeignKeyColumn, Table, UpdateDateColumn } from 'src/sql-tools'; @Table('exif') @UpdatedAtTrigger('asset_exif_updated_at') @@ -50,8 +50,7 @@ export class ExifTable { @Column({ type: 'double precision', nullable: true }) longitude!: number | null; - @ColumnIndex('exif_city') - @Column({ type: 'character varying', nullable: true }) + @Column({ type: 'character varying', nullable: true, indexName: 'exif_city' }) city!: string | null; @Column({ type: 'character varying', nullable: true }) @@ -69,8 +68,7 @@ export class ExifTable { @Column({ type: 'character varying', nullable: true }) exposureTime!: string | null; - @ColumnIndex('IDX_live_photo_cid') - @Column({ type: 'character varying', nullable: true }) + @Column({ type: 'character varying', nullable: true, indexName: 'IDX_live_photo_cid' }) livePhotoCID!: string | null; @Column({ type: 'character varying', nullable: true }) @@ -88,8 +86,7 @@ export class ExifTable { @Column({ type: 'integer', nullable: true }) bitsPerSample!: number | null; - @ColumnIndex('IDX_auto_stack_id') - @Column({ type: 'character varying', nullable: true }) + @Column({ type: 'character varying', nullable: true, indexName: 'IDX_auto_stack_id' }) autoStackId!: string | null; @Column({ type: 'integer', nullable: true }) @@ -98,7 +95,6 @@ export class ExifTable { @UpdateDateColumn({ default: () => 'clock_timestamp()' }) updatedAt?: Date; - @ColumnIndex('IDX_asset_exif_update_id') - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_asset_exif_update_id' }) updateId?: string; } diff --git a/server/src/schema/tables/library.table.ts b/server/src/schema/tables/library.table.ts index 54b3752f41..8b21d5feb0 100644 --- a/server/src/schema/tables/library.table.ts +++ b/server/src/schema/tables/library.table.ts @@ -2,7 +2,6 @@ import { UpdatedAtTrigger, UpdateIdColumn } from 'src/decorators'; import { UserTable } from 'src/schema/tables/user.table'; import { Column, - ColumnIndex, CreateDateColumn, DeleteDateColumn, ForeignKeyColumn, @@ -41,7 +40,6 @@ export class LibraryTable { @Column({ type: 'timestamp with time zone', nullable: true }) refreshedAt!: Date | null; - @ColumnIndex('IDX_libraries_update_id') - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_libraries_update_id' }) updateId?: string; } diff --git a/server/src/schema/tables/memory.table.ts b/server/src/schema/tables/memory.table.ts index 1926405565..32dafe3384 100644 --- a/server/src/schema/tables/memory.table.ts +++ b/server/src/schema/tables/memory.table.ts @@ -3,7 +3,6 @@ import { MemoryType } from 'src/enum'; import { UserTable } from 'src/schema/tables/user.table'; import { Column, - ColumnIndex, CreateDateColumn, DeleteDateColumn, ForeignKeyColumn, @@ -55,7 +54,6 @@ export class MemoryTable { @Column({ type: 'timestamp with time zone', nullable: true }) hideAt?: Date; - @ColumnIndex('IDX_memories_update_id') - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_memories_update_id' }) updateId?: string; } diff --git a/server/src/schema/tables/memory_asset.table.ts b/server/src/schema/tables/memory_asset.table.ts index 864e6291c7..0e5ca29a08 100644 --- a/server/src/schema/tables/memory_asset.table.ts +++ b/server/src/schema/tables/memory_asset.table.ts @@ -1,14 +1,12 @@ import { AssetTable } from 'src/schema/tables/asset.table'; import { MemoryTable } from 'src/schema/tables/memory.table'; -import { ColumnIndex, ForeignKeyColumn, Table } from 'src/sql-tools'; +import { ForeignKeyColumn, Table } from 'src/sql-tools'; @Table('memories_assets_assets') export class MemoryAssetTable { - @ColumnIndex() @ForeignKeyColumn(() => MemoryTable, { onUpdate: 'CASCADE', onDelete: 'CASCADE', primary: true }) memoriesId!: string; - @ColumnIndex() @ForeignKeyColumn(() => AssetTable, { onUpdate: 'CASCADE', onDelete: 'CASCADE', primary: true }) assetsId!: string; } diff --git a/server/src/schema/tables/partner-audit.table.ts b/server/src/schema/tables/partner-audit.table.ts index 08b6e94626..da5243dc75 100644 --- a/server/src/schema/tables/partner-audit.table.ts +++ b/server/src/schema/tables/partner-audit.table.ts @@ -1,20 +1,17 @@ import { PrimaryGeneratedUuidV7Column } from 'src/decorators'; -import { Column, ColumnIndex, CreateDateColumn, Table } from 'src/sql-tools'; +import { Column, CreateDateColumn, Table } from 'src/sql-tools'; @Table('partners_audit') export class PartnerAuditTable { @PrimaryGeneratedUuidV7Column() id!: string; - @ColumnIndex('IDX_partners_audit_shared_by_id') - @Column({ type: 'uuid' }) + @Column({ type: 'uuid', indexName: 'IDX_partners_audit_shared_by_id' }) sharedById!: string; - @ColumnIndex('IDX_partners_audit_shared_with_id') - @Column({ type: 'uuid' }) + @Column({ type: 'uuid', indexName: 'IDX_partners_audit_shared_with_id' }) sharedWithId!: string; - @ColumnIndex('IDX_partners_audit_deleted_at') - @CreateDateColumn({ default: () => 'clock_timestamp()' }) + @CreateDateColumn({ default: () => 'clock_timestamp()', indexName: 'IDX_partners_audit_deleted_at' }) deletedAt!: Date; } diff --git a/server/src/schema/tables/partner.table.ts b/server/src/schema/tables/partner.table.ts index 770107fe7a..0da60cfc0c 100644 --- a/server/src/schema/tables/partner.table.ts +++ b/server/src/schema/tables/partner.table.ts @@ -1,15 +1,7 @@ import { UpdatedAtTrigger, UpdateIdColumn } from 'src/decorators'; import { partners_delete_audit } from 'src/schema/functions'; import { UserTable } from 'src/schema/tables/user.table'; -import { - AfterDeleteTrigger, - Column, - ColumnIndex, - CreateDateColumn, - ForeignKeyColumn, - Table, - UpdateDateColumn, -} from 'src/sql-tools'; +import { AfterDeleteTrigger, Column, CreateDateColumn, ForeignKeyColumn, Table, UpdateDateColumn } from 'src/sql-tools'; @Table('partners') @UpdatedAtTrigger('partners_updated_at') @@ -21,7 +13,12 @@ import { when: 'pg_trigger_depth() = 0', }) export class PartnerTable { - @ForeignKeyColumn(() => UserTable, { onDelete: 'CASCADE', primary: true }) + @ForeignKeyColumn(() => UserTable, { + onDelete: 'CASCADE', + primary: true, + // [sharedById, sharedWithId] is the PK constraint + index: false, + }) sharedById!: string; @ForeignKeyColumn(() => UserTable, { onDelete: 'CASCADE', primary: true }) @@ -36,7 +33,6 @@ export class PartnerTable { @Column({ type: 'boolean', default: false }) inTimeline!: boolean; - @ColumnIndex('IDX_partners_update_id') - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_partners_update_id' }) updateId!: string; } diff --git a/server/src/schema/tables/person.table.ts b/server/src/schema/tables/person.table.ts index b96fc5b709..1320b91f18 100644 --- a/server/src/schema/tables/person.table.ts +++ b/server/src/schema/tables/person.table.ts @@ -4,7 +4,6 @@ import { UserTable } from 'src/schema/tables/user.table'; import { Check, Column, - ColumnIndex, CreateDateColumn, ForeignKeyColumn, PrimaryGeneratedColumn, @@ -49,7 +48,6 @@ export class PersonTable { @Column({ type: 'character varying', nullable: true, default: null }) color?: string | null; - @ColumnIndex('IDX_person_update_id') - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_person_update_id' }) updateId!: string; } diff --git a/server/src/schema/tables/session.table.ts b/server/src/schema/tables/session.table.ts index a66732a7d9..ad43d0d6e4 100644 --- a/server/src/schema/tables/session.table.ts +++ b/server/src/schema/tables/session.table.ts @@ -2,7 +2,6 @@ import { UpdatedAtTrigger, UpdateIdColumn } from 'src/decorators'; import { UserTable } from 'src/schema/tables/user.table'; import { Column, - ColumnIndex, CreateDateColumn, ForeignKeyColumn, PrimaryGeneratedColumn, @@ -35,7 +34,6 @@ export class SessionTable { @Column({ default: '' }) deviceOS!: string; - @ColumnIndex('IDX_sessions_update_id') - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_sessions_update_id' }) updateId!: string; } diff --git a/server/src/schema/tables/shared-link-asset.table.ts b/server/src/schema/tables/shared-link-asset.table.ts index 1eb294c1e8..66c9068441 100644 --- a/server/src/schema/tables/shared-link-asset.table.ts +++ b/server/src/schema/tables/shared-link-asset.table.ts @@ -1,14 +1,12 @@ import { AssetTable } from 'src/schema/tables/asset.table'; import { SharedLinkTable } from 'src/schema/tables/shared-link.table'; -import { ColumnIndex, ForeignKeyColumn, Table } from 'src/sql-tools'; +import { ForeignKeyColumn, Table } from 'src/sql-tools'; @Table('shared_link__asset') export class SharedLinkAssetTable { - @ColumnIndex() @ForeignKeyColumn(() => AssetTable, { onUpdate: 'CASCADE', onDelete: 'CASCADE', primary: true }) assetsId!: string; - @ColumnIndex() @ForeignKeyColumn(() => SharedLinkTable, { onUpdate: 'CASCADE', onDelete: 'CASCADE', primary: true }) sharedLinksId!: string; } diff --git a/server/src/schema/tables/shared-link.table.ts b/server/src/schema/tables/shared-link.table.ts index 36237c58ef..3bb36b36ed 100644 --- a/server/src/schema/tables/shared-link.table.ts +++ b/server/src/schema/tables/shared-link.table.ts @@ -1,15 +1,7 @@ import { SharedLinkType } from 'src/enum'; import { AlbumTable } from 'src/schema/tables/album.table'; import { UserTable } from 'src/schema/tables/user.table'; -import { - Column, - ColumnIndex, - CreateDateColumn, - ForeignKeyColumn, - PrimaryGeneratedColumn, - Table, - Unique, -} from 'src/sql-tools'; +import { Column, CreateDateColumn, ForeignKeyColumn, PrimaryGeneratedColumn, Table, Unique } from 'src/sql-tools'; @Table('shared_links') @Unique({ name: 'UQ_sharedlink_key', columns: ['key'] }) @@ -23,8 +15,7 @@ export class SharedLinkTable { @ForeignKeyColumn(() => UserTable, { onDelete: 'CASCADE', onUpdate: 'CASCADE' }) userId!: string; - @ColumnIndex('IDX_sharedlink_key') - @Column({ type: 'bytea' }) + @Column({ type: 'bytea', indexName: 'IDX_sharedlink_key' }) key!: Buffer; // use to access the inidividual asset @Column() @@ -39,8 +30,12 @@ export class SharedLinkTable { @Column({ type: 'boolean', default: false }) allowUpload!: boolean; - @ColumnIndex('IDX_sharedlink_albumId') - @ForeignKeyColumn(() => AlbumTable, { nullable: true, onDelete: 'CASCADE', onUpdate: 'CASCADE' }) + @ForeignKeyColumn(() => AlbumTable, { + nullable: true, + onDelete: 'CASCADE', + onUpdate: 'CASCADE', + indexName: 'IDX_sharedlink_albumId', + }) albumId!: string; @Column({ type: 'boolean', default: true }) diff --git a/server/src/schema/tables/sync-checkpoint.table.ts b/server/src/schema/tables/sync-checkpoint.table.ts index 831205ce7a..21fd7983ac 100644 --- a/server/src/schema/tables/sync-checkpoint.table.ts +++ b/server/src/schema/tables/sync-checkpoint.table.ts @@ -1,15 +1,7 @@ import { UpdatedAtTrigger, UpdateIdColumn } from 'src/decorators'; import { SyncEntityType } from 'src/enum'; import { SessionTable } from 'src/schema/tables/session.table'; -import { - Column, - ColumnIndex, - CreateDateColumn, - ForeignKeyColumn, - PrimaryColumn, - Table, - UpdateDateColumn, -} from 'src/sql-tools'; +import { Column, CreateDateColumn, ForeignKeyColumn, PrimaryColumn, Table, UpdateDateColumn } from 'src/sql-tools'; @Table('session_sync_checkpoints') @UpdatedAtTrigger('session_sync_checkpoints_updated_at') @@ -29,7 +21,6 @@ export class SessionSyncCheckpointTable { @Column() ack!: string; - @ColumnIndex('IDX_session_sync_checkpoints_update_id') - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_session_sync_checkpoints_update_id' }) updateId!: string; } diff --git a/server/src/schema/tables/tag-asset.table.ts b/server/src/schema/tables/tag-asset.table.ts index 5f24799cec..8793af0a8a 100644 --- a/server/src/schema/tables/tag-asset.table.ts +++ b/server/src/schema/tables/tag-asset.table.ts @@ -1,15 +1,13 @@ import { AssetTable } from 'src/schema/tables/asset.table'; import { TagTable } from 'src/schema/tables/tag.table'; -import { ColumnIndex, ForeignKeyColumn, Index, Table } from 'src/sql-tools'; +import { ForeignKeyColumn, Index, Table } from 'src/sql-tools'; @Index({ name: 'IDX_tag_asset_assetsId_tagsId', columns: ['assetsId', 'tagsId'] }) @Table('tag_asset') export class TagAssetTable { - @ColumnIndex() - @ForeignKeyColumn(() => AssetTable, { onUpdate: 'CASCADE', onDelete: 'CASCADE', primary: true }) + @ForeignKeyColumn(() => AssetTable, { onUpdate: 'CASCADE', onDelete: 'CASCADE', primary: true, index: true }) assetsId!: string; - @ColumnIndex() - @ForeignKeyColumn(() => TagTable, { onUpdate: 'CASCADE', onDelete: 'CASCADE', primary: true }) + @ForeignKeyColumn(() => TagTable, { onUpdate: 'CASCADE', onDelete: 'CASCADE', primary: true, index: true }) tagsId!: string; } diff --git a/server/src/schema/tables/tag-closure.table.ts b/server/src/schema/tables/tag-closure.table.ts index acde84b91d..8829e802e1 100644 --- a/server/src/schema/tables/tag-closure.table.ts +++ b/server/src/schema/tables/tag-closure.table.ts @@ -1,13 +1,11 @@ import { TagTable } from 'src/schema/tables/tag.table'; -import { ColumnIndex, ForeignKeyColumn, Table } from 'src/sql-tools'; +import { ForeignKeyColumn, Table } from 'src/sql-tools'; @Table('tags_closure') export class TagClosureTable { - @ColumnIndex() - @ForeignKeyColumn(() => TagTable, { primary: true, onDelete: 'CASCADE', onUpdate: 'NO ACTION' }) + @ForeignKeyColumn(() => TagTable, { primary: true, onDelete: 'CASCADE', onUpdate: 'NO ACTION', index: true }) id_ancestor!: string; - @ColumnIndex() - @ForeignKeyColumn(() => TagTable, { primary: true, onDelete: 'CASCADE', onUpdate: 'NO ACTION' }) + @ForeignKeyColumn(() => TagTable, { primary: true, onDelete: 'CASCADE', onUpdate: 'NO ACTION', index: true }) id_descendant!: string; } diff --git a/server/src/schema/tables/tag.table.ts b/server/src/schema/tables/tag.table.ts index 5042e2eb0e..a9f2a57f27 100644 --- a/server/src/schema/tables/tag.table.ts +++ b/server/src/schema/tables/tag.table.ts @@ -2,7 +2,6 @@ import { UpdatedAtTrigger, UpdateIdColumn } from 'src/decorators'; import { UserTable } from 'src/schema/tables/user.table'; import { Column, - ColumnIndex, CreateDateColumn, ForeignKeyColumn, PrimaryGeneratedColumn, @@ -18,7 +17,12 @@ export class TagTable { @PrimaryGeneratedColumn() id!: string; - @ForeignKeyColumn(() => UserTable, { onUpdate: 'CASCADE', onDelete: 'CASCADE' }) + @ForeignKeyColumn(() => UserTable, { + onUpdate: 'CASCADE', + onDelete: 'CASCADE', + // [userId, value] makes this redundant + index: false, + }) userId!: string; @Column() @@ -36,7 +40,6 @@ export class TagTable { @ForeignKeyColumn(() => TagTable, { nullable: true, onDelete: 'CASCADE' }) parentId?: string; - @ColumnIndex('IDX_tags_update_id') - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_tags_update_id' }) updateId!: string; } diff --git a/server/src/schema/tables/user-audit.table.ts b/server/src/schema/tables/user-audit.table.ts index 0f881ccc9a..e0c9afcdc3 100644 --- a/server/src/schema/tables/user-audit.table.ts +++ b/server/src/schema/tables/user-audit.table.ts @@ -1,13 +1,12 @@ import { PrimaryGeneratedUuidV7Column } from 'src/decorators'; -import { Column, ColumnIndex, CreateDateColumn, Table } from 'src/sql-tools'; +import { Column, CreateDateColumn, Table } from 'src/sql-tools'; @Table('users_audit') export class UserAuditTable { @Column({ type: 'uuid' }) userId!: string; - @ColumnIndex('IDX_users_audit_deleted_at') - @CreateDateColumn({ default: () => 'clock_timestamp()' }) + @CreateDateColumn({ default: () => 'clock_timestamp()', indexName: 'IDX_users_audit_deleted_at' }) deletedAt!: Date; @PrimaryGeneratedUuidV7Column() diff --git a/server/src/schema/tables/user-metadata.table.ts b/server/src/schema/tables/user-metadata.table.ts index 6d03acaf80..04b457867f 100644 --- a/server/src/schema/tables/user-metadata.table.ts +++ b/server/src/schema/tables/user-metadata.table.ts @@ -5,7 +5,13 @@ import { UserMetadata, UserMetadataItem } from 'src/types'; @Table('user_metadata') export class UserMetadataTable implements UserMetadataItem { - @ForeignKeyColumn(() => UserTable, { onUpdate: 'CASCADE', onDelete: 'CASCADE', primary: true }) + @ForeignKeyColumn(() => UserTable, { + onUpdate: 'CASCADE', + onDelete: 'CASCADE', + primary: true, + // [userId, key] is the PK constraint + index: false, + }) userId!: string; @PrimaryColumn({ type: 'character varying' }) diff --git a/server/src/schema/tables/user.table.ts b/server/src/schema/tables/user.table.ts index 5160f979b9..eeef923796 100644 --- a/server/src/schema/tables/user.table.ts +++ b/server/src/schema/tables/user.table.ts @@ -5,7 +5,6 @@ import { users_delete_audit } from 'src/schema/functions'; import { AfterDeleteTrigger, Column, - ColumnIndex, CreateDateColumn, DeleteDateColumn, Index, @@ -77,7 +76,6 @@ export class UserTable { @Column({ type: 'timestamp with time zone', default: () => 'now()' }) profileChangedAt!: Generated; - @ColumnIndex({ name: 'IDX_users_update_id' }) - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_users_update_id' }) updateId!: Generated; } diff --git a/server/src/sql-tools/from-code/decorators/column-index.decorator.ts b/server/src/sql-tools/from-code/decorators/column-index.decorator.ts deleted file mode 100644 index ab15292612..0000000000 --- a/server/src/sql-tools/from-code/decorators/column-index.decorator.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { register } from 'src/sql-tools/from-code/register'; -import { asOptions } from 'src/sql-tools/helpers'; - -export type ColumnIndexOptions = { - name?: string; - unique?: boolean; - expression?: string; - using?: string; - with?: string; - where?: string; - synchronize?: boolean; -}; -export const ColumnIndex = (options: string | ColumnIndexOptions = {}): PropertyDecorator => { - return (object: object, propertyName: string | symbol) => - void register({ type: 'columnIndex', item: { object, propertyName, options: asOptions(options) } }); -}; diff --git a/server/src/sql-tools/from-code/decorators/column.decorator.ts b/server/src/sql-tools/from-code/decorators/column.decorator.ts index 74a83cbcf3..7b00af80cc 100644 --- a/server/src/sql-tools/from-code/decorators/column.decorator.ts +++ b/server/src/sql-tools/from-code/decorators/column.decorator.ts @@ -15,13 +15,15 @@ export type ColumnBaseOptions = { synchronize?: boolean; storage?: ColumnStorage; identity?: boolean; + index?: boolean; + indexName?: string; + unique?: boolean; + uniqueConstraintName?: string; }; export type ColumnOptions = ColumnBaseOptions & { enum?: DatabaseEnum; array?: boolean; - unique?: boolean; - uniqueConstraintName?: string; }; export const Column = (options: string | ColumnOptions = {}): PropertyDecorator => { diff --git a/server/src/sql-tools/from-code/decorators/foreign-key-column.decorator.ts b/server/src/sql-tools/from-code/decorators/foreign-key-column.decorator.ts index 070aa5cb51..beb3aa6fd6 100644 --- a/server/src/sql-tools/from-code/decorators/foreign-key-column.decorator.ts +++ b/server/src/sql-tools/from-code/decorators/foreign-key-column.decorator.ts @@ -7,8 +7,6 @@ export type ForeignKeyColumnOptions = ColumnBaseOptions & { onUpdate?: Action; onDelete?: Action; constraintName?: string; - unique?: boolean; - uniqueConstraintName?: string; }; export const ForeignKeyColumn = (target: () => object, options: ForeignKeyColumnOptions): PropertyDecorator => { diff --git a/server/src/sql-tools/from-code/decorators/index.decorator.ts b/server/src/sql-tools/from-code/decorators/index.decorator.ts index cd76b5e36d..5d90c4f58d 100644 --- a/server/src/sql-tools/from-code/decorators/index.decorator.ts +++ b/server/src/sql-tools/from-code/decorators/index.decorator.ts @@ -1,8 +1,13 @@ -import { ColumnIndexOptions } from 'src/sql-tools/from-code/decorators/column-index.decorator'; import { register } from 'src/sql-tools/from-code/register'; import { asOptions } from 'src/sql-tools/helpers'; -export type IndexOptions = ColumnIndexOptions & { +export type IndexOptions = { + name?: string; + unique?: boolean; + expression?: string; + using?: string; + with?: string; + where?: string; columns?: string[]; synchronize?: boolean; }; diff --git a/server/src/sql-tools/from-code/index.ts b/server/src/sql-tools/from-code/index.ts index 3c74d2763c..95f1dbb22d 100644 --- a/server/src/sql-tools/from-code/index.ts +++ b/server/src/sql-tools/from-code/index.ts @@ -1,6 +1,5 @@ import 'reflect-metadata'; import { processCheckConstraints } from 'src/sql-tools/from-code/processors/check-constraint.processor'; -import { processColumnIndexes } from 'src/sql-tools/from-code/processors/column-index.processor'; import { processColumns } from 'src/sql-tools/from-code/processors/column.processor'; import { processConfigurationParameters } from 'src/sql-tools/from-code/processors/configuration-parameter.processor'; import { processDatabases } from 'src/sql-tools/from-code/processors/database.processor'; @@ -36,14 +35,21 @@ const processors: Processor[] = [ processUniqueConstraints, processCheckConstraints, processPrimaryKeyConstraints, - processIndexes, - processColumnIndexes, processForeignKeyConstraints, + processIndexes, processTriggers, ]; -export const schemaFromCode = () => { +export type SchemaFromCodeOptions = { + /** automatically create indexes on foreign key columns */ + createForeignKeyIndexes?: boolean; +}; +export const schemaFromCode = (options: SchemaFromCodeOptions = {}) => { if (!initialized) { + const globalOptions = { + createForeignKeyIndexes: options.createForeignKeyIndexes ?? true, + }; + const builder: SchemaBuilder = { name: 'postgres', schemaName: 'public', @@ -58,7 +64,7 @@ export const schemaFromCode = () => { const items = getRegisteredItems(); for (const processor of processors) { - processor(builder, items); + processor(builder, items, globalOptions); } schema = { ...builder, tables: builder.tables.map(({ metadata: _, ...table }) => table) }; diff --git a/server/src/sql-tools/from-code/processors/check-constraint.processor.ts b/server/src/sql-tools/from-code/processors/check-constraint.processor.ts index d61ee18277..feb21b9894 100644 --- a/server/src/sql-tools/from-code/processors/check-constraint.processor.ts +++ b/server/src/sql-tools/from-code/processors/check-constraint.processor.ts @@ -1,6 +1,6 @@ import { onMissingTable, resolveTable } from 'src/sql-tools/from-code/processors/table.processor'; import { Processor } from 'src/sql-tools/from-code/processors/type'; -import { asCheckConstraintName } from 'src/sql-tools/helpers'; +import { asKey } from 'src/sql-tools/helpers'; import { DatabaseConstraintType } from 'src/sql-tools/types'; export const processCheckConstraints: Processor = (builder, items) => { @@ -24,3 +24,5 @@ export const processCheckConstraints: Processor = (builder, items) => { }); } }; + +const asCheckConstraintName = (table: string, expression: string) => asKey('CHK_', table, [expression]); diff --git a/server/src/sql-tools/from-code/processors/column-index.processor.ts b/server/src/sql-tools/from-code/processors/column-index.processor.ts deleted file mode 100644 index 0e40fa1ee3..0000000000 --- a/server/src/sql-tools/from-code/processors/column-index.processor.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { onMissingColumn, resolveColumn } from 'src/sql-tools/from-code/processors/column.processor'; -import { onMissingTable } from 'src/sql-tools/from-code/processors/table.processor'; -import { Processor } from 'src/sql-tools/from-code/processors/type'; -import { asIndexName } from 'src/sql-tools/helpers'; - -export const processColumnIndexes: Processor = (builder, items) => { - for (const { - item: { object, propertyName, options }, - } of items.filter((item) => item.type === 'columnIndex')) { - const { table, column } = resolveColumn(builder, object, propertyName); - if (!table) { - onMissingTable(builder, '@ColumnIndex', object); - continue; - } - - if (!column) { - onMissingColumn(builder, `@ColumnIndex`, object, propertyName); - continue; - } - - table.indexes.push({ - name: options.name || asIndexName(table.name, [column.name], options.where), - tableName: table.name, - unique: options.unique ?? false, - expression: options.expression, - using: options.using, - where: options.where, - columnNames: [column.name], - synchronize: options.synchronize ?? true, - }); - } -}; diff --git a/server/src/sql-tools/from-code/processors/column.processor.ts b/server/src/sql-tools/from-code/processors/column.processor.ts index 37f3f5d082..e8c2544f87 100644 --- a/server/src/sql-tools/from-code/processors/column.processor.ts +++ b/server/src/sql-tools/from-code/processors/column.processor.ts @@ -1,8 +1,8 @@ import { ColumnOptions } from 'src/sql-tools/from-code/decorators/column.decorator'; import { onMissingTable, resolveTable } from 'src/sql-tools/from-code/processors/table.processor'; import { Processor, SchemaBuilder } from 'src/sql-tools/from-code/processors/type'; -import { asMetadataKey, asUniqueConstraintName, fromColumnValue } from 'src/sql-tools/helpers'; -import { DatabaseColumn, DatabaseConstraintType } from 'src/sql-tools/types'; +import { asMetadataKey, fromColumnValue } from 'src/sql-tools/helpers'; +import { DatabaseColumn } from 'src/sql-tools/types'; export const processColumns: Processor = (builder, items) => { for (const { @@ -54,16 +54,6 @@ export const processColumns: Processor = (builder, items) => { writeMetadata(object, propertyName, { name: column.name, options }); table.columns.push(column); - - if (type === 'column' && !options.primary && options.unique) { - table.constraints.push({ - type: DatabaseConstraintType.UNIQUE, - name: options.uniqueConstraintName || asUniqueConstraintName(table.name, [column.name]), - tableName: table.name, - columnNames: [column.name], - synchronize: options.synchronize ?? true, - }); - } } }; diff --git a/server/src/sql-tools/from-code/processors/foreign-key-constriant.processor.ts b/server/src/sql-tools/from-code/processors/foreign-key-constriant.processor.ts index 784a8b8e99..612b74c30f 100644 --- a/server/src/sql-tools/from-code/processors/foreign-key-constriant.processor.ts +++ b/server/src/sql-tools/from-code/processors/foreign-key-constriant.processor.ts @@ -1,7 +1,7 @@ import { onMissingColumn, resolveColumn } from 'src/sql-tools/from-code/processors/column.processor'; import { onMissingTable, resolveTable } from 'src/sql-tools/from-code/processors/table.processor'; import { Processor } from 'src/sql-tools/from-code/processors/type'; -import { asForeignKeyConstraintName, asRelationKeyConstraintName } from 'src/sql-tools/helpers'; +import { asKey } from 'src/sql-tools/helpers'; import { DatabaseActionType, DatabaseConstraintType } from 'src/sql-tools/types'; export const processForeignKeyConstraints: Processor = (builder, items) => { @@ -46,7 +46,7 @@ export const processForeignKeyConstraints: Processor = (builder, items) => { synchronize: options.synchronize ?? true, }); - if (options.unique) { + if (options.unique || options.uniqueConstraintName) { table.constraints.push({ name: options.uniqueConstraintName || asRelationKeyConstraintName(table.name, columnNames), tableName: table.name, @@ -57,3 +57,6 @@ export const processForeignKeyConstraints: Processor = (builder, items) => { } } }; + +const asForeignKeyConstraintName = (table: string, columns: string[]) => asKey('FK_', table, columns); +const asRelationKeyConstraintName = (table: string, columns: string[]) => asKey('REL_', table, columns); diff --git a/server/src/sql-tools/from-code/processors/index.processor.ts b/server/src/sql-tools/from-code/processors/index.processor.ts index 3625bf9784..f4c9c7cec1 100644 --- a/server/src/sql-tools/from-code/processors/index.processor.ts +++ b/server/src/sql-tools/from-code/processors/index.processor.ts @@ -1,8 +1,9 @@ +import { onMissingColumn, resolveColumn } from 'src/sql-tools/from-code/processors/column.processor'; import { onMissingTable, resolveTable } from 'src/sql-tools/from-code/processors/table.processor'; import { Processor } from 'src/sql-tools/from-code/processors/type'; -import { asIndexName } from 'src/sql-tools/helpers'; +import { asKey } from 'src/sql-tools/helpers'; -export const processIndexes: Processor = (builder, items) => { +export const processIndexes: Processor = (builder, items, config) => { for (const { item: { object, options }, } of items.filter((item) => item.type === 'index')) { @@ -24,4 +25,66 @@ export const processIndexes: Processor = (builder, items) => { synchronize: options.synchronize ?? true, }); } + + // column indexes + for (const { + type, + item: { object, propertyName, options }, + } of items.filter((item) => item.type === 'column' || item.type === 'foreignKeyColumn')) { + const { table, column } = resolveColumn(builder, object, propertyName); + if (!table) { + onMissingTable(builder, '@Column', object); + continue; + } + + if (!column) { + // should be impossible since they are created in `column.processor.ts` + onMissingColumn(builder, '@Column', object, propertyName); + continue; + } + + if (options.index === false) { + continue; + } + + const isIndexRequested = + options.indexName || options.index || (type === 'foreignKeyColumn' && config.createForeignKeyIndexes); + if (!isIndexRequested) { + continue; + } + + const indexName = options.indexName || asIndexName(table.name, [column.name]); + + const isIndexPresent = table.indexes.some((index) => index.name === indexName); + if (isIndexPresent) { + continue; + } + + const isOnlyPrimaryColumn = options.primary && table.columns.filter(({ primary }) => primary === true).length === 1; + if (isOnlyPrimaryColumn) { + // will have an index created by the primary key constraint + continue; + } + + table.indexes.push({ + name: indexName, + tableName: table.name, + unique: false, + columnNames: [column.name], + synchronize: options.synchronize ?? true, + }); + } +}; + +const asIndexName = (table: string, columns?: string[], where?: string) => { + const items: string[] = []; + for (const columnName of columns ?? []) { + items.push(columnName); + } + + if (where) { + items.push(where); + } + + return asKey('IDX_', table, items); }; diff --git a/server/src/sql-tools/from-code/processors/primary-key-contraint.processor.ts b/server/src/sql-tools/from-code/processors/primary-key-contraint.processor.ts index f123f2e495..74aecc5ea0 100644 --- a/server/src/sql-tools/from-code/processors/primary-key-contraint.processor.ts +++ b/server/src/sql-tools/from-code/processors/primary-key-contraint.processor.ts @@ -1,5 +1,5 @@ import { Processor } from 'src/sql-tools/from-code/processors/type'; -import { asPrimaryKeyConstraintName } from 'src/sql-tools/helpers'; +import { asKey } from 'src/sql-tools/helpers'; import { DatabaseConstraintType } from 'src/sql-tools/types'; export const processPrimaryKeyConstraints: Processor = (builder) => { @@ -22,3 +22,5 @@ export const processPrimaryKeyConstraints: Processor = (builder) => { } } }; + +const asPrimaryKeyConstraintName = (table: string, columns: string[]) => asKey('PK_', table, columns); diff --git a/server/src/sql-tools/from-code/processors/trigger.processor.ts b/server/src/sql-tools/from-code/processors/trigger.processor.ts index 2f4cc04326..4b875f353b 100644 --- a/server/src/sql-tools/from-code/processors/trigger.processor.ts +++ b/server/src/sql-tools/from-code/processors/trigger.processor.ts @@ -1,6 +1,7 @@ +import { TriggerOptions } from 'src/sql-tools/from-code/decorators/trigger.decorator'; import { onMissingTable, resolveTable } from 'src/sql-tools/from-code/processors/table.processor'; import { Processor } from 'src/sql-tools/from-code/processors/type'; -import { asTriggerName } from 'src/sql-tools/helpers'; +import { asKey } from 'src/sql-tools/helpers'; export const processTriggers: Processor = (builder, items) => { for (const { @@ -26,3 +27,6 @@ export const processTriggers: Processor = (builder, items) => { }); } }; + +const asTriggerName = (table: string, trigger: TriggerOptions) => + asKey('TR_', table, [...trigger.actions, trigger.scope, trigger.timing, trigger.functionName]); diff --git a/server/src/sql-tools/from-code/processors/type.ts b/server/src/sql-tools/from-code/processors/type.ts index 5a69efbcf0..deb142d278 100644 --- a/server/src/sql-tools/from-code/processors/type.ts +++ b/server/src/sql-tools/from-code/processors/type.ts @@ -1,3 +1,4 @@ +import { SchemaFromCodeOptions } from 'src/sql-tools/from-code'; import { TableOptions } from 'src/sql-tools/from-code/decorators/table.decorator'; import { RegisterItem } from 'src/sql-tools/from-code/register-item'; import { DatabaseSchema, DatabaseTable } from 'src/sql-tools/types'; @@ -6,4 +7,4 @@ import { DatabaseSchema, DatabaseTable } from 'src/sql-tools/types'; export type TableWithMetadata = DatabaseTable & { metadata: { options: TableOptions; object: Function } }; export type SchemaBuilder = Omit & { tables: TableWithMetadata[] }; -export type Processor = (builder: SchemaBuilder, items: RegisterItem[]) => void; +export type Processor = (builder: SchemaBuilder, items: RegisterItem[], options: SchemaFromCodeOptions) => void; diff --git a/server/src/sql-tools/from-code/processors/unique-constraint.processor.ts b/server/src/sql-tools/from-code/processors/unique-constraint.processor.ts index 74c0504f7e..9014378085 100644 --- a/server/src/sql-tools/from-code/processors/unique-constraint.processor.ts +++ b/server/src/sql-tools/from-code/processors/unique-constraint.processor.ts @@ -1,6 +1,7 @@ +import { onMissingColumn, resolveColumn } from 'src/sql-tools/from-code/processors/column.processor'; import { onMissingTable, resolveTable } from 'src/sql-tools/from-code/processors/table.processor'; import { Processor } from 'src/sql-tools/from-code/processors/type'; -import { asUniqueConstraintName } from 'src/sql-tools/helpers'; +import { asKey } from 'src/sql-tools/helpers'; import { DatabaseConstraintType } from 'src/sql-tools/types'; export const processUniqueConstraints: Processor = (builder, items) => { @@ -24,4 +25,34 @@ export const processUniqueConstraints: Processor = (builder, items) => { synchronize: options.synchronize ?? true, }); } + + // column level constraints + for (const { + type, + item: { object, propertyName, options }, + } of items.filter((item) => item.type === 'column' || item.type === 'foreignKeyColumn')) { + const { table, column } = resolveColumn(builder, object, propertyName); + if (!table) { + onMissingTable(builder, '@Column', object); + continue; + } + + if (!column) { + // should be impossible since they are created in `column.processor.ts` + onMissingColumn(builder, '@Column', object, propertyName); + continue; + } + + if (type === 'column' && !options.primary && (options.unique || options.uniqueConstraintName)) { + table.constraints.push({ + type: DatabaseConstraintType.UNIQUE, + name: options.uniqueConstraintName || asUniqueConstraintName(table.name, [column.name]), + tableName: table.name, + columnNames: [column.name], + synchronize: options.synchronize ?? true, + }); + } + } }; + +const asUniqueConstraintName = (table: string, columns: string[]) => asKey('UQ_', table, columns); diff --git a/server/src/sql-tools/from-code/register-function.ts b/server/src/sql-tools/from-code/register-function.ts index 69e1a0f8f3..be71e0dfd7 100644 --- a/server/src/sql-tools/from-code/register-function.ts +++ b/server/src/sql-tools/from-code/register-function.ts @@ -1,5 +1,4 @@ import { register } from 'src/sql-tools/from-code/register'; -import { asFunctionExpression } from 'src/sql-tools/helpers'; import { ColumnType, DatabaseFunction } from 'src/sql-tools/types'; export type FunctionOptions = { @@ -27,3 +26,37 @@ export const registerFunction = (options: FunctionOptions) => { return item; }; + +const asFunctionExpression = (options: FunctionOptions) => { + const name = options.name; + const sql: string[] = [ + `CREATE OR REPLACE FUNCTION ${name}(${(options.arguments || []).join(', ')})`, + `RETURNS ${options.returnType}`, + ]; + + const flags = [ + options.parallel ? `PARALLEL ${options.parallel.toUpperCase()}` : undefined, + options.strict ? 'STRICT' : undefined, + options.behavior ? options.behavior.toUpperCase() : undefined, + `LANGUAGE ${options.language ?? 'SQL'}`, + ].filter((x) => x !== undefined); + + if (flags.length > 0) { + sql.push(flags.join(' ')); + } + + if ('return' in options) { + sql.push(` RETURN ${options.return}`); + } + + if ('body' in options) { + sql.push( + // + `AS $$`, + ' ' + options.body.trim(), + `$$;`, + ); + } + + return sql.join('\n ').trim(); +}; diff --git a/server/src/sql-tools/from-code/register-item.ts b/server/src/sql-tools/from-code/register-item.ts index 08200cbc4f..4889ae34b9 100644 --- a/server/src/sql-tools/from-code/register-item.ts +++ b/server/src/sql-tools/from-code/register-item.ts @@ -1,5 +1,4 @@ import { CheckOptions } from 'src/sql-tools/from-code/decorators/check.decorator'; -import { ColumnIndexOptions } from 'src/sql-tools/from-code/decorators/column-index.decorator'; import { ColumnOptions } from 'src/sql-tools/from-code/decorators/column.decorator'; import { ConfigurationParameterOptions } from 'src/sql-tools/from-code/decorators/configuration-parameter.decorator'; import { DatabaseOptions } from 'src/sql-tools/from-code/decorators/database.decorator'; @@ -21,7 +20,6 @@ export type RegisterItem = | { type: 'uniqueConstraint'; item: ClassBased<{ options: UniqueOptions }> } | { type: 'checkConstraint'; item: ClassBased<{ options: CheckOptions }> } | { type: 'column'; item: PropertyBased<{ options: ColumnOptions }> } - | { type: 'columnIndex'; item: PropertyBased<{ options: ColumnIndexOptions }> } | { type: 'function'; item: DatabaseFunction } | { type: 'enum'; item: DatabaseEnum } | { type: 'trigger'; item: ClassBased<{ options: TriggerOptions }> } diff --git a/server/src/sql-tools/helpers.ts b/server/src/sql-tools/helpers.ts index 364b695194..2802407ea6 100644 --- a/server/src/sql-tools/helpers.ts +++ b/server/src/sql-tools/helpers.ts @@ -1,7 +1,5 @@ import { createHash } from 'node:crypto'; import { ColumnValue } from 'src/sql-tools/from-code/decorators/column.decorator'; -import { TriggerOptions } from 'src/sql-tools/from-code/decorators/trigger.decorator'; -import { FunctionOptions } from 'src/sql-tools/from-code/register-function'; import { Comparer, DatabaseColumn, @@ -18,25 +16,6 @@ export const asSnakeCase = (name: string): string => name.replaceAll(/([a-z])([A // match TypeORM export const asKey = (prefix: string, tableName: string, values: string[]) => (prefix + sha1(`${tableName}_${values.toSorted().join('_')}`)).slice(0, 30); -export const asPrimaryKeyConstraintName = (table: string, columns: string[]) => asKey('PK_', table, columns); -export const asForeignKeyConstraintName = (table: string, columns: string[]) => asKey('FK_', table, columns); -export const asTriggerName = (table: string, trigger: TriggerOptions) => - asKey('TR_', table, [...trigger.actions, trigger.scope, trigger.timing, trigger.functionName]); -export const asRelationKeyConstraintName = (table: string, columns: string[]) => asKey('REL_', table, columns); -export const asUniqueConstraintName = (table: string, columns: string[]) => asKey('UQ_', table, columns); -export const asCheckConstraintName = (table: string, expression: string) => asKey('CHK_', table, [expression]); -export const asIndexName = (table: string, columns: string[] | undefined, where: string | undefined) => { - const items: string[] = []; - for (const columnName of columns ?? []) { - items.push(columnName); - } - - if (where) { - items.push(where); - } - - return asKey('IDX_', table, items); -}; export const asOptions = (options: string | T): T => { if (typeof options === 'string') { @@ -46,40 +25,6 @@ export const asOptions = (options: string | T): T = return options; }; -export const asFunctionExpression = (options: FunctionOptions) => { - const name = options.name; - const sql: string[] = [ - `CREATE OR REPLACE FUNCTION ${name}(${(options.arguments || []).join(', ')})`, - `RETURNS ${options.returnType}`, - ]; - - const flags = [ - options.parallel ? `PARALLEL ${options.parallel.toUpperCase()}` : undefined, - options.strict ? 'STRICT' : undefined, - options.behavior ? options.behavior.toUpperCase() : undefined, - `LANGUAGE ${options.language ?? 'SQL'}`, - ].filter((x) => x !== undefined); - - if (flags.length > 0) { - sql.push(flags.join(' ')); - } - - if ('return' in options) { - sql.push(` RETURN ${options.return}`); - } - - if ('body' in options) { - sql.push( - // - `AS $$`, - ' ' + options.body.trim(), - `$$;`, - ); - } - - return sql.join('\n ').trim(); -}; - export const sha1 = (value: string) => createHash('sha1').update(value).digest('hex'); export const hasMask = (input: number, mask: number) => (input & mask) === mask; diff --git a/server/src/sql-tools/public_api.ts b/server/src/sql-tools/public_api.ts index d916678d4a..b41cce4ab5 100644 --- a/server/src/sql-tools/public_api.ts +++ b/server/src/sql-tools/public_api.ts @@ -3,7 +3,6 @@ export { schemaFromCode } from 'src/sql-tools/from-code'; export * from 'src/sql-tools/from-code/decorators/after-delete.decorator'; export * from 'src/sql-tools/from-code/decorators/before-update.decorator'; export * from 'src/sql-tools/from-code/decorators/check.decorator'; -export * from 'src/sql-tools/from-code/decorators/column-index.decorator'; export * from 'src/sql-tools/from-code/decorators/column.decorator'; export * from 'src/sql-tools/from-code/decorators/configuration-parameter.decorator'; export * from 'src/sql-tools/from-code/decorators/create-date-column.decorator'; diff --git a/server/test/sql-tools/column-index-name-default.ts b/server/test/sql-tools/column-index-name-default.ts index e8b36ec119..cedae006be 100644 --- a/server/test/sql-tools/column-index-name-default.ts +++ b/server/test/sql-tools/column-index-name-default.ts @@ -1,9 +1,8 @@ -import { Column, ColumnIndex, DatabaseSchema, Table } from 'src/sql-tools'; +import { Column, DatabaseSchema, Table } from 'src/sql-tools'; @Table() export class Table1 { - @ColumnIndex() - @Column() + @Column({ index: true }) column1!: string; } diff --git a/server/test/sql-tools/column-index-name.ts b/server/test/sql-tools/column-index-name.ts new file mode 100644 index 0000000000..8ba18a8851 --- /dev/null +++ b/server/test/sql-tools/column-index-name.ts @@ -0,0 +1,46 @@ +import { Column, DatabaseSchema, Table } from 'src/sql-tools'; + +@Table() +export class Table1 { + @Column({ indexName: 'IDX_test' }) + column1!: string; +} + +export const description = 'should create a column with an index if a name is provided'; +export const schema: DatabaseSchema = { + name: 'postgres', + schemaName: 'public', + functions: [], + enums: [], + extensions: [], + parameters: [], + tables: [ + { + name: 'table1', + columns: [ + { + name: 'column1', + tableName: 'table1', + type: 'character varying', + nullable: false, + isArray: false, + primary: false, + synchronize: true, + }, + ], + indexes: [ + { + name: 'IDX_test', + columnNames: ['column1'], + tableName: 'table1', + unique: false, + synchronize: true, + }, + ], + triggers: [], + constraints: [], + synchronize: true, + }, + ], + warnings: [], +}; diff --git a/server/test/sql-tools/foreign-key-inferred-type.stub.ts b/server/test/sql-tools/foreign-key-inferred-type.stub.ts index 2ecaafdcad..0b66a1acd4 100644 --- a/server/test/sql-tools/foreign-key-inferred-type.stub.ts +++ b/server/test/sql-tools/foreign-key-inferred-type.stub.ts @@ -60,7 +60,15 @@ export const schema: DatabaseSchema = { synchronize: true, }, ], - indexes: [], + indexes: [ + { + name: 'IDX_3fcca5cc563abf256fc346e3ff', + tableName: 'table2', + columnNames: ['parentId'], + unique: false, + synchronize: true, + }, + ], triggers: [], constraints: [ { diff --git a/server/test/sql-tools/foreign-key-with-unique-constraint.stub.ts b/server/test/sql-tools/foreign-key-with-unique-constraint.stub.ts index 0601a02d42..109a3dfc85 100644 --- a/server/test/sql-tools/foreign-key-with-unique-constraint.stub.ts +++ b/server/test/sql-tools/foreign-key-with-unique-constraint.stub.ts @@ -60,7 +60,15 @@ export const schema: DatabaseSchema = { synchronize: true, }, ], - indexes: [], + indexes: [ + { + name: 'IDX_3fcca5cc563abf256fc346e3ff', + tableName: 'table2', + columnNames: ['parentId'], + unique: false, + synchronize: true, + }, + ], triggers: [], constraints: [ { From 6474a78b8be31cf6a3c2fabf2f3177ffd40624e1 Mon Sep 17 00:00:00 2001 From: Jason Rasmussen Date: Thu, 17 Apr 2025 17:38:47 -0400 Subject: [PATCH 05/12] feat: initial kysely migration file (#17678) --- .github/workflows/test.yml | 2 +- docs/docs/developer/database-migrations.md | 6 +- server/package.json | 5 +- server/src/bin/database.ts | 11 - server/src/bin/migrations.ts | 37 +- .../1744910873956-AddMissingIndex.ts | 13 + .../src/repositories/database.repository.ts | 75 ++-- .../1744910873969-InitialMigration.ts | 391 ++++++++++++++++++ .../src/schema/tables/geodata-places.table.ts | 5 +- .../from-code/processors/table.processor.ts | 7 + .../sql-tools/from-code/register-function.ts | 8 +- server/test/medium/globalSetup.ts | 78 +--- ....stub copy.ts => index-with-expression.ts} | 0 13 files changed, 499 insertions(+), 139 deletions(-) delete mode 100644 server/src/bin/database.ts create mode 100644 server/src/migrations/1744910873956-AddMissingIndex.ts create mode 100644 server/src/schema/migrations/1744910873969-InitialMigration.ts rename server/test/sql-tools/{index-with-where.stub copy.ts => index-with-expression.ts} (100%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5273698e4e..f5cb6c8d30 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -521,7 +521,7 @@ jobs: run: npm run migrations:run - name: Test npm run schema:reset command works - run: npm run typeorm:schema:reset + run: npm run schema:reset - name: Generate new migrations continue-on-error: true diff --git a/docs/docs/developer/database-migrations.md b/docs/docs/developer/database-migrations.md index 2cddf5f386..c0c61340cb 100644 --- a/docs/docs/developer/database-migrations.md +++ b/docs/docs/developer/database-migrations.md @@ -1,14 +1,14 @@ # Database Migrations -After making any changes in the `server/src/entities`, a database migration need to run in order to register the changes in the database. Follow the steps below to create a new migration. +After making any changes in the `server/src/schema`, a database migration need to run in order to register the changes in the database. Follow the steps below to create a new migration. 1. Run the command ```bash -npm run typeorm:migrations:generate +npm run migrations:generate ``` 2. Check if the migration file makes sense. -3. Move the migration file to folder `./server/src/migrations` in your code editor. +3. Move the migration file to folder `./server/src/schema/migrations` in your code editor. The server will automatically detect `*.ts` file changes and restart. Part of the server start-up process includes running any new migrations, so it will be applied immediately. diff --git a/server/package.json b/server/package.json index 76415da7c8..8e149d961e 100644 --- a/server/package.json +++ b/server/package.json @@ -26,9 +26,8 @@ "migrations:generate": "node ./dist/bin/migrations.js generate", "migrations:create": "node ./dist/bin/migrations.js create", "migrations:run": "node ./dist/bin/migrations.js run", - "typeorm:migrations:revert": "typeorm migration:revert -d ./dist/bin/database.js", - "typeorm:schema:drop": "typeorm query -d ./dist/bin/database.js 'DROP schema public cascade; CREATE schema public;'", - "typeorm:schema:reset": "npm run typeorm:schema:drop && npm run migrations:run", + "schema:drop": "node ./dist/bin/migrations.js query 'DROP schema public cascade; CREATE schema public;'", + "schema:reset": "npm run schema:drop && npm run migrations:run", "kysely:codegen": "npx kysely-codegen --include-pattern=\"(public|vectors).*\" --dialect postgres --url postgres://postgres:postgres@localhost/immich --log-level debug --out-file=./src/db.d.ts", "sync:open-api": "node ./dist/bin/sync-open-api.js", "sync:sql": "node ./dist/bin/sync-sql.js", diff --git a/server/src/bin/database.ts b/server/src/bin/database.ts deleted file mode 100644 index 7ea56e0fc0..0000000000 --- a/server/src/bin/database.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { ConfigRepository } from 'src/repositories/config.repository'; -import { DataSource } from 'typeorm'; - -const { database } = new ConfigRepository().getEnv(); - -/** - * @deprecated - DO NOT USE THIS - * - * this export is ONLY to be used for TypeORM commands in package.json#scripts - */ -export const dataSource = new DataSource({ ...database.config.typeorm, host: 'localhost' }); diff --git a/server/src/bin/migrations.ts b/server/src/bin/migrations.ts index fb50323dff..2ddc6776fb 100644 --- a/server/src/bin/migrations.ts +++ b/server/src/bin/migrations.ts @@ -1,8 +1,8 @@ #!/usr/bin/env node process.env.DB_URL = process.env.DB_URL || 'postgres://postgres:postgres@localhost:5432/immich'; -import { Kysely } from 'kysely'; -import { writeFileSync } from 'node:fs'; +import { Kysely, sql } from 'kysely'; +import { mkdirSync, writeFileSync } from 'node:fs'; import { basename, dirname, extname, join } from 'node:path'; import postgres from 'postgres'; import { ConfigRepository } from 'src/repositories/config.repository'; @@ -23,8 +23,13 @@ const main = async () => { } case 'run': { - const only = process.argv[3] as 'kysely' | 'typeorm' | undefined; - await run(only); + await runMigrations(); + return; + } + + case 'query': { + const query = process.argv[3]; + await runQuery(query); return; } @@ -48,14 +53,25 @@ const main = async () => { } }; -const run = async (only?: 'kysely' | 'typeorm') => { +const getDatabaseClient = () => { const configRepository = new ConfigRepository(); const { database } = configRepository.getEnv(); - const logger = new LoggingRepository(undefined, configRepository); - const db = new Kysely(getKyselyConfig(database.config.kysely)); - const databaseRepository = new DatabaseRepository(db, logger, configRepository); + return new Kysely(getKyselyConfig(database.config.kysely)); +}; - await databaseRepository.runMigrations({ only }); +const runQuery = async (query: string) => { + const db = getDatabaseClient(); + await sql.raw(query).execute(db); + await db.destroy(); +}; + +const runMigrations = async () => { + const configRepository = new ConfigRepository(); + const logger = new LoggingRepository(undefined, configRepository); + const db = getDatabaseClient(); + const databaseRepository = new DatabaseRepository(db, logger, configRepository); + await databaseRepository.runMigrations(); + await db.destroy(); }; const debug = async () => { @@ -81,7 +97,8 @@ const create = (path: string, up: string[], down: string[]) => { const filename = `${timestamp}-${name}.ts`; const folder = dirname(path); const fullPath = join(folder, filename); - writeFileSync(fullPath, asMigration('typeorm', { name, timestamp, up, down })); + mkdirSync(folder, { recursive: true }); + writeFileSync(fullPath, asMigration('kysely', { name, timestamp, up, down })); console.log(`Wrote ${fullPath}`); }; diff --git a/server/src/migrations/1744910873956-AddMissingIndex.ts b/server/src/migrations/1744910873956-AddMissingIndex.ts new file mode 100644 index 0000000000..38dd6f4958 --- /dev/null +++ b/server/src/migrations/1744910873956-AddMissingIndex.ts @@ -0,0 +1,13 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AddMissingIndex1744910873956 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE INDEX IF NOT EXISTS "IDX_geodata_gist_earthcoord" ON "geodata_places" (ll_to_earth_public(latitude, longitude))`, + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX "IDX_geodata_gist_earthcoord";`); + } +} diff --git a/server/src/repositories/database.repository.ts b/server/src/repositories/database.repository.ts index ec0b263408..c70c2cbdd4 100644 --- a/server/src/repositories/database.repository.ts +++ b/server/src/repositories/database.repository.ts @@ -2,7 +2,6 @@ import { Injectable } from '@nestjs/common'; import AsyncLock from 'async-lock'; import { FileMigrationProvider, Kysely, Migrator, sql, Transaction } from 'kysely'; import { InjectKysely } from 'nestjs-kysely'; -import { existsSync } from 'node:fs'; import { readdir } from 'node:fs/promises'; import { join } from 'node:path'; import semver from 'semver'; @@ -197,62 +196,54 @@ export class DatabaseRepository { return dimSize; } - async runMigrations(options?: { transaction?: 'all' | 'none' | 'each'; only?: 'kysely' | 'typeorm' }): Promise { + async runMigrations(options?: { transaction?: 'all' | 'none' | 'each' }): Promise { const { database } = this.configRepository.getEnv(); - if (options?.only !== 'kysely') { - const dataSource = new DataSource(database.config.typeorm); - this.logger.log('Running migrations, this may take a while'); + this.logger.log('Running migrations, this may take a while'); + const tableExists = sql<{ result: string | null }>`select to_regclass('migrations') as "result"`; + const { rows } = await tableExists.execute(this.db); + const hasTypeOrmMigrations = !!rows[0]?.result; + if (hasTypeOrmMigrations) { this.logger.debug('Running typeorm migrations'); - + const dataSource = new DataSource(database.config.typeorm); await dataSource.initialize(); await dataSource.runMigrations(options); await dataSource.destroy(); - this.logger.debug('Finished running typeorm migrations'); } - if (options?.only !== 'typeorm') { - // eslint-disable-next-line unicorn/prefer-module - const migrationFolder = join(__dirname, '..', 'schema/migrations'); + this.logger.debug('Running kysely migrations'); + const migrator = new Migrator({ + db: this.db, + migrationLockTableName: 'kysely_migrations_lock', + migrationTableName: 'kysely_migrations', + provider: new FileMigrationProvider({ + fs: { readdir }, + path: { join }, + // eslint-disable-next-line unicorn/prefer-module + migrationFolder: join(__dirname, '..', 'schema/migrations'), + }), + }); - // TODO remove after we have at least one kysely migration - if (!existsSync(migrationFolder)) { - return; + const { error, results } = await migrator.migrateToLatest(); + + for (const result of results ?? []) { + if (result.status === 'Success') { + this.logger.log(`Migration "${result.migrationName}" succeeded`); } - this.logger.debug('Running kysely migrations'); - const migrator = new Migrator({ - db: this.db, - migrationLockTableName: 'kysely_migrations_lock', - migrationTableName: 'kysely_migrations', - provider: new FileMigrationProvider({ - fs: { readdir }, - path: { join }, - migrationFolder, - }), - }); - - const { error, results } = await migrator.migrateToLatest(); - - for (const result of results ?? []) { - if (result.status === 'Success') { - this.logger.log(`Migration "${result.migrationName}" succeeded`); - } - - if (result.status === 'Error') { - this.logger.warn(`Migration "${result.migrationName}" failed`); - } + if (result.status === 'Error') { + this.logger.warn(`Migration "${result.migrationName}" failed`); } - - if (error) { - this.logger.error(`Kysely migrations failed: ${error}`); - throw error; - } - - this.logger.debug('Finished running kysely migrations'); } + + if (error) { + this.logger.error(`Kysely migrations failed: ${error}`); + throw error; + } + + this.logger.debug('Finished running kysely migrations'); } async withLock(lock: DatabaseLock, callback: () => Promise): Promise { diff --git a/server/src/schema/migrations/1744910873969-InitialMigration.ts b/server/src/schema/migrations/1744910873969-InitialMigration.ts new file mode 100644 index 0000000000..e157607681 --- /dev/null +++ b/server/src/schema/migrations/1744910873969-InitialMigration.ts @@ -0,0 +1,391 @@ +import { Kysely, sql } from 'kysely'; +import { DatabaseExtension } from 'src/enum'; +import { ConfigRepository } from 'src/repositories/config.repository'; + +const vectorExtension = new ConfigRepository().getEnv().database.vectorExtension; + +export async function up(db: Kysely): Promise { + await sql`CREATE EXTENSION IF NOT EXISTS "uuid-ossp";`.execute(db); + await sql`CREATE EXTENSION IF NOT EXISTS "unaccent";`.execute(db); + await sql`CREATE EXTENSION IF NOT EXISTS "cube";`.execute(db); + await sql`CREATE EXTENSION IF NOT EXISTS "earthdistance";`.execute(db); + await sql`CREATE EXTENSION IF NOT EXISTS "pg_trgm";`.execute(db); + await sql`CREATE EXTENSION IF NOT EXISTS "vectors";`.execute(db); + await sql`CREATE OR REPLACE FUNCTION immich_uuid_v7(p_timestamp timestamp with time zone default clock_timestamp()) + RETURNS uuid + VOLATILE LANGUAGE SQL + AS $$ + select encode( + set_bit( + set_bit( + overlay(uuid_send(gen_random_uuid()) + placing substring(int8send(floor(extract(epoch from p_timestamp) * 1000)::bigint) from 3) + from 1 for 6 + ), + 52, 1 + ), + 53, 1 + ), + 'hex')::uuid; + $$;`.execute(db); + await sql`CREATE OR REPLACE FUNCTION updated_at() + RETURNS TRIGGER + LANGUAGE PLPGSQL + AS $$ + DECLARE + clock_timestamp TIMESTAMP := clock_timestamp(); + BEGIN + new."updatedAt" = clock_timestamp; + new."updateId" = immich_uuid_v7(clock_timestamp); + return new; + END; + $$;`.execute(db); + await sql`CREATE OR REPLACE FUNCTION f_concat_ws(text, text[]) + RETURNS text + PARALLEL SAFE IMMUTABLE LANGUAGE SQL + AS $$SELECT array_to_string($2, $1)$$;`.execute(db); + await sql`CREATE OR REPLACE FUNCTION f_unaccent(text) + RETURNS text + PARALLEL SAFE STRICT IMMUTABLE LANGUAGE SQL + RETURN unaccent('unaccent', $1)`.execute(db); + await sql`CREATE OR REPLACE FUNCTION ll_to_earth_public(latitude double precision, longitude double precision) + RETURNS public.earth + PARALLEL SAFE STRICT IMMUTABLE LANGUAGE SQL + AS $$ + SELECT public.cube(public.cube(public.cube(public.earth()*cos(radians(latitude))*cos(radians(longitude))),public.earth()*cos(radians(latitude))*sin(radians(longitude))),public.earth()*sin(radians(latitude)))::public.earth + $$;`.execute(db); + await sql`CREATE OR REPLACE FUNCTION users_delete_audit() + RETURNS TRIGGER + LANGUAGE PLPGSQL + AS $$ + BEGIN + INSERT INTO users_audit ("userId") + SELECT "id" + FROM OLD; + RETURN NULL; + END; + $$;`.execute(db); + await sql`CREATE OR REPLACE FUNCTION partners_delete_audit() + RETURNS TRIGGER + LANGUAGE PLPGSQL + AS $$ + BEGIN + INSERT INTO partners_audit ("sharedById", "sharedWithId") + SELECT "sharedById", "sharedWithId" + FROM OLD; + RETURN NULL; + END; + $$;`.execute(db); + await sql`CREATE OR REPLACE FUNCTION assets_delete_audit() + RETURNS TRIGGER + LANGUAGE PLPGSQL + AS $$ + BEGIN + INSERT INTO assets_audit ("assetId", "ownerId") + SELECT "id", "ownerId" + FROM OLD; + RETURN NULL; + END; + $$;`.execute(db); + if (vectorExtension === DatabaseExtension.VECTORS) { + await sql`SET search_path TO "$user", public, vectors`.execute(db); + await sql`SET vectors.pgvector_compatibility=on`.execute(db); + } + await sql`CREATE TYPE "assets_status_enum" AS ENUM ('active','trashed','deleted');`.execute(db); + await sql`CREATE TYPE "sourcetype" AS ENUM ('machine-learning','exif','manual');`.execute(db); + await sql`CREATE TABLE "users" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "email" character varying NOT NULL, "password" character varying NOT NULL DEFAULT '', "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "profileImagePath" character varying NOT NULL DEFAULT '', "isAdmin" boolean NOT NULL DEFAULT false, "shouldChangePassword" boolean NOT NULL DEFAULT true, "deletedAt" timestamp with time zone, "oauthId" character varying NOT NULL DEFAULT '', "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "storageLabel" character varying, "name" character varying NOT NULL DEFAULT '', "quotaSizeInBytes" bigint, "quotaUsageInBytes" bigint NOT NULL DEFAULT 0, "status" character varying NOT NULL DEFAULT 'active', "profileChangedAt" timestamp with time zone NOT NULL DEFAULT now(), "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "libraries" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "name" character varying NOT NULL, "ownerId" uuid NOT NULL, "importPaths" text[] NOT NULL, "exclusionPatterns" text[] NOT NULL, "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "deletedAt" timestamp with time zone, "refreshedAt" timestamp with time zone, "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "asset_stack" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "primaryAssetId" uuid NOT NULL, "ownerId" uuid NOT NULL);`.execute(db); + await sql`CREATE TABLE "assets" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "deviceAssetId" character varying NOT NULL, "ownerId" uuid NOT NULL, "deviceId" character varying NOT NULL, "type" character varying NOT NULL, "originalPath" character varying NOT NULL, "fileCreatedAt" timestamp with time zone NOT NULL, "fileModifiedAt" timestamp with time zone NOT NULL, "isFavorite" boolean NOT NULL DEFAULT false, "duration" character varying, "encodedVideoPath" character varying DEFAULT '', "checksum" bytea NOT NULL, "isVisible" boolean NOT NULL DEFAULT true, "livePhotoVideoId" uuid, "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "isArchived" boolean NOT NULL DEFAULT false, "originalFileName" character varying NOT NULL, "sidecarPath" character varying, "thumbhash" bytea, "isOffline" boolean NOT NULL DEFAULT false, "libraryId" uuid, "isExternal" boolean NOT NULL DEFAULT false, "deletedAt" timestamp with time zone, "localDateTime" timestamp with time zone NOT NULL, "stackId" uuid, "duplicateId" uuid, "status" assets_status_enum NOT NULL DEFAULT 'active', "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "albums" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "ownerId" uuid NOT NULL, "albumName" character varying NOT NULL DEFAULT 'Untitled Album', "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "albumThumbnailAssetId" uuid, "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "description" text NOT NULL DEFAULT '', "deletedAt" timestamp with time zone, "isActivityEnabled" boolean NOT NULL DEFAULT true, "order" character varying NOT NULL DEFAULT 'desc', "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`COMMENT ON COLUMN "albums"."albumThumbnailAssetId" IS 'Asset ID to be used as thumbnail';`.execute(db); + await sql`CREATE TABLE "activity" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "albumId" uuid NOT NULL, "userId" uuid NOT NULL, "assetId" uuid, "comment" text, "isLiked" boolean NOT NULL DEFAULT false, "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "albums_assets_assets" ("albumsId" uuid NOT NULL, "assetsId" uuid NOT NULL, "createdAt" timestamp with time zone NOT NULL DEFAULT now());`.execute(db); + await sql`CREATE TABLE "albums_shared_users_users" ("albumsId" uuid NOT NULL, "usersId" uuid NOT NULL, "role" character varying NOT NULL DEFAULT 'editor');`.execute(db); + await sql`CREATE TABLE "api_keys" ("name" character varying NOT NULL, "key" character varying NOT NULL, "userId" uuid NOT NULL, "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "id" uuid NOT NULL DEFAULT uuid_generate_v4(), "permissions" character varying[] NOT NULL, "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "assets_audit" ("id" uuid NOT NULL DEFAULT immich_uuid_v7(), "assetId" uuid NOT NULL, "ownerId" uuid NOT NULL, "deletedAt" timestamp with time zone NOT NULL DEFAULT clock_timestamp());`.execute(db); + await sql`CREATE TABLE "person" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "ownerId" uuid NOT NULL, "name" character varying NOT NULL DEFAULT '', "thumbnailPath" character varying NOT NULL DEFAULT '', "isHidden" boolean NOT NULL DEFAULT false, "birthDate" date, "faceAssetId" uuid, "isFavorite" boolean NOT NULL DEFAULT false, "color" character varying, "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "asset_faces" ("assetId" uuid NOT NULL, "personId" uuid, "imageWidth" integer NOT NULL DEFAULT 0, "imageHeight" integer NOT NULL DEFAULT 0, "boundingBoxX1" integer NOT NULL DEFAULT 0, "boundingBoxY1" integer NOT NULL DEFAULT 0, "boundingBoxX2" integer NOT NULL DEFAULT 0, "boundingBoxY2" integer NOT NULL DEFAULT 0, "id" uuid NOT NULL DEFAULT uuid_generate_v4(), "sourceType" sourcetype NOT NULL DEFAULT 'machine-learning', "deletedAt" timestamp with time zone);`.execute(db); + await sql`CREATE TABLE "asset_files" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "assetId" uuid NOT NULL, "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "type" character varying NOT NULL, "path" character varying NOT NULL, "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "asset_job_status" ("assetId" uuid NOT NULL, "facesRecognizedAt" timestamp with time zone, "metadataExtractedAt" timestamp with time zone, "duplicatesDetectedAt" timestamp with time zone, "previewAt" timestamp with time zone, "thumbnailAt" timestamp with time zone);`.execute(db); + await sql`CREATE TABLE "audit" ("id" serial NOT NULL, "entityType" character varying NOT NULL, "entityId" uuid NOT NULL, "action" character varying NOT NULL, "ownerId" uuid NOT NULL, "createdAt" timestamp with time zone NOT NULL DEFAULT now());`.execute(db); + await sql`CREATE TABLE "exif" ("assetId" uuid NOT NULL, "make" character varying, "model" character varying, "exifImageWidth" integer, "exifImageHeight" integer, "fileSizeInByte" bigint, "orientation" character varying, "dateTimeOriginal" timestamp with time zone, "modifyDate" timestamp with time zone, "lensModel" character varying, "fNumber" double precision, "focalLength" double precision, "iso" integer, "latitude" double precision, "longitude" double precision, "city" character varying, "state" character varying, "country" character varying, "description" text NOT NULL DEFAULT '', "fps" double precision, "exposureTime" character varying, "livePhotoCID" character varying, "timeZone" character varying, "projectionType" character varying, "profileDescription" character varying, "colorspace" character varying, "bitsPerSample" integer, "autoStackId" character varying, "rating" integer, "updatedAt" timestamp with time zone NOT NULL DEFAULT clock_timestamp(), "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "face_search" ("faceId" uuid NOT NULL, "embedding" vector(512) NOT NULL);`.execute(db); + await sql`CREATE TABLE "geodata_places" ("id" integer NOT NULL, "name" character varying(200) NOT NULL, "longitude" double precision NOT NULL, "latitude" double precision NOT NULL, "countryCode" character(2) NOT NULL, "admin1Code" character varying(20), "admin2Code" character varying(80), "modificationDate" date NOT NULL, "admin1Name" character varying, "admin2Name" character varying, "alternateNames" character varying);`.execute(db); + await sql`CREATE TABLE "memories" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "deletedAt" timestamp with time zone, "ownerId" uuid NOT NULL, "type" character varying NOT NULL, "data" jsonb NOT NULL, "isSaved" boolean NOT NULL DEFAULT false, "memoryAt" timestamp with time zone NOT NULL, "seenAt" timestamp with time zone, "showAt" timestamp with time zone, "hideAt" timestamp with time zone, "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "memories_assets_assets" ("memoriesId" uuid NOT NULL, "assetsId" uuid NOT NULL);`.execute(db); + await sql`CREATE TABLE "move_history" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "entityId" uuid NOT NULL, "pathType" character varying NOT NULL, "oldPath" character varying NOT NULL, "newPath" character varying NOT NULL);`.execute(db); + await sql`CREATE TABLE "naturalearth_countries" ("id" integer NOT NULL GENERATED ALWAYS AS IDENTITY, "admin" character varying(50) NOT NULL, "admin_a3" character varying(3) NOT NULL, "type" character varying(50) NOT NULL, "coordinates" polygon NOT NULL);`.execute(db); + await sql`CREATE TABLE "partners_audit" ("id" uuid NOT NULL DEFAULT immich_uuid_v7(), "sharedById" uuid NOT NULL, "sharedWithId" uuid NOT NULL, "deletedAt" timestamp with time zone NOT NULL DEFAULT clock_timestamp());`.execute(db); + await sql`CREATE TABLE "partners" ("sharedById" uuid NOT NULL, "sharedWithId" uuid NOT NULL, "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "inTimeline" boolean NOT NULL DEFAULT false, "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "sessions" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "token" character varying NOT NULL, "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "userId" uuid NOT NULL, "deviceType" character varying NOT NULL DEFAULT '', "deviceOS" character varying NOT NULL DEFAULT '', "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "shared_links" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "description" character varying, "userId" uuid NOT NULL, "key" bytea NOT NULL, "type" character varying NOT NULL, "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "expiresAt" timestamp with time zone, "allowUpload" boolean NOT NULL DEFAULT false, "albumId" uuid, "allowDownload" boolean NOT NULL DEFAULT true, "showExif" boolean NOT NULL DEFAULT true, "password" character varying);`.execute(db); + await sql`CREATE TABLE "shared_link__asset" ("assetsId" uuid NOT NULL, "sharedLinksId" uuid NOT NULL);`.execute(db); + await sql`CREATE TABLE "smart_search" ("assetId" uuid NOT NULL, "embedding" vector(512) NOT NULL);`.execute(db); + await sql`ALTER TABLE "smart_search" ALTER COLUMN "embedding" SET STORAGE EXTERNAL;`.execute(db); + await sql`CREATE TABLE "session_sync_checkpoints" ("sessionId" uuid NOT NULL, "type" character varying NOT NULL, "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "ack" character varying NOT NULL, "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "system_metadata" ("key" character varying NOT NULL, "value" jsonb NOT NULL);`.execute(db); + await sql`CREATE TABLE "tags" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "userId" uuid NOT NULL, "value" character varying NOT NULL, "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "color" character varying, "parentId" uuid, "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "tag_asset" ("assetsId" uuid NOT NULL, "tagsId" uuid NOT NULL);`.execute(db); + await sql`CREATE TABLE "tags_closure" ("id_ancestor" uuid NOT NULL, "id_descendant" uuid NOT NULL);`.execute(db); + await sql`CREATE TABLE "users_audit" ("userId" uuid NOT NULL, "deletedAt" timestamp with time zone NOT NULL DEFAULT clock_timestamp(), "id" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "user_metadata" ("userId" uuid NOT NULL, "key" character varying NOT NULL, "value" jsonb NOT NULL);`.execute(db); + await sql`CREATE TABLE "version_history" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "version" character varying NOT NULL);`.execute(db); + await sql`ALTER TABLE "users" ADD CONSTRAINT "PK_a3ffb1c0c8416b9fc6f907b7433" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "libraries" ADD CONSTRAINT "PK_505fedfcad00a09b3734b4223de" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "asset_stack" ADD CONSTRAINT "PK_74a27e7fcbd5852463d0af3034b" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "assets" ADD CONSTRAINT "PK_da96729a8b113377cfb6a62439c" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "albums" ADD CONSTRAINT "PK_7f71c7b5bc7c87b8f94c9a93a00" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "activity" ADD CONSTRAINT "PK_24625a1d6b1b089c8ae206fe467" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "albums_assets_assets" ADD CONSTRAINT "PK_c67bc36fa845fb7b18e0e398180" PRIMARY KEY ("albumsId", "assetsId");`.execute(db); + await sql`ALTER TABLE "albums_shared_users_users" ADD CONSTRAINT "PK_7df55657e0b2e8b626330a0ebc8" PRIMARY KEY ("albumsId", "usersId");`.execute(db); + await sql`ALTER TABLE "api_keys" ADD CONSTRAINT "PK_5c8a79801b44bd27b79228e1dad" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "assets_audit" ADD CONSTRAINT "PK_99bd5c015f81a641927a32b4212" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "person" ADD CONSTRAINT "PK_5fdaf670315c4b7e70cce85daa3" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "asset_faces" ADD CONSTRAINT "PK_6df76ab2eb6f5b57b7c2f1fc684" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "asset_files" ADD CONSTRAINT "PK_c41dc3e9ef5e1c57ca5a08a0004" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "asset_job_status" ADD CONSTRAINT "PK_420bec36fc02813bddf5c8b73d4" PRIMARY KEY ("assetId");`.execute(db); + await sql`ALTER TABLE "audit" ADD CONSTRAINT "PK_1d3d120ddaf7bc9b1ed68ed463a" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "exif" ADD CONSTRAINT "PK_c0117fdbc50b917ef9067740c44" PRIMARY KEY ("assetId");`.execute(db); + await sql`ALTER TABLE "face_search" ADD CONSTRAINT "face_search_pkey" PRIMARY KEY ("faceId");`.execute(db); + await sql`ALTER TABLE "geodata_places" ADD CONSTRAINT "PK_c29918988912ef4036f3d7fbff4" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "memories" ADD CONSTRAINT "PK_aaa0692d9496fe827b0568612f8" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "memories_assets_assets" ADD CONSTRAINT "PK_fcaf7112a013d1703c011c6793d" PRIMARY KEY ("memoriesId", "assetsId");`.execute(db); + await sql`ALTER TABLE "move_history" ADD CONSTRAINT "PK_af608f132233acf123f2949678d" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "naturalearth_countries" ADD CONSTRAINT "PK_21a6d86d1ab5d841648212e5353" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "partners_audit" ADD CONSTRAINT "PK_952b50217ff78198a7e380f0359" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "partners" ADD CONSTRAINT "PK_f1cc8f73d16b367f426261a8736" PRIMARY KEY ("sharedById", "sharedWithId");`.execute(db); + await sql`ALTER TABLE "sessions" ADD CONSTRAINT "PK_48cb6b5c20faa63157b3c1baf7f" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "shared_links" ADD CONSTRAINT "PK_642e2b0f619e4876e5f90a43465" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "shared_link__asset" ADD CONSTRAINT "PK_9b4f3687f9b31d1e311336b05e3" PRIMARY KEY ("assetsId", "sharedLinksId");`.execute(db); + await sql`ALTER TABLE "smart_search" ADD CONSTRAINT "smart_search_pkey" PRIMARY KEY ("assetId");`.execute(db); + await sql`ALTER TABLE "session_sync_checkpoints" ADD CONSTRAINT "PK_b846ab547a702863ef7cd9412fb" PRIMARY KEY ("sessionId", "type");`.execute(db); + await sql`ALTER TABLE "system_metadata" ADD CONSTRAINT "PK_fa94f6857470fb5b81ec6084465" PRIMARY KEY ("key");`.execute(db); + await sql`ALTER TABLE "tags" ADD CONSTRAINT "PK_e7dc17249a1148a1970748eda99" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "tag_asset" ADD CONSTRAINT "PK_ef5346fe522b5fb3bc96454747e" PRIMARY KEY ("assetsId", "tagsId");`.execute(db); + await sql`ALTER TABLE "tags_closure" ADD CONSTRAINT "PK_eab38eb12a3ec6df8376c95477c" PRIMARY KEY ("id_ancestor", "id_descendant");`.execute(db); + await sql`ALTER TABLE "users_audit" ADD CONSTRAINT "PK_e9b2bdfd90e7eb5961091175180" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "user_metadata" ADD CONSTRAINT "PK_5931462150b3438cbc83277fe5a" PRIMARY KEY ("userId", "key");`.execute(db); + await sql`ALTER TABLE "version_history" ADD CONSTRAINT "PK_5db259cbb09ce82c0d13cfd1b23" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "libraries" ADD CONSTRAINT "FK_0f6fc2fb195f24d19b0fb0d57c1" FOREIGN KEY ("ownerId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "asset_stack" ADD CONSTRAINT "FK_91704e101438fd0653f582426dc" FOREIGN KEY ("primaryAssetId") REFERENCES "assets" ("id") ON UPDATE NO ACTION ON DELETE NO ACTION;`.execute(db); + await sql`ALTER TABLE "asset_stack" ADD CONSTRAINT "FK_c05079e542fd74de3b5ecb5c1c8" FOREIGN KEY ("ownerId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "assets" ADD CONSTRAINT "FK_2c5ac0d6fb58b238fd2068de67d" FOREIGN KEY ("ownerId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "assets" ADD CONSTRAINT "FK_16294b83fa8c0149719a1f631ef" FOREIGN KEY ("livePhotoVideoId") REFERENCES "assets" ("id") ON UPDATE CASCADE ON DELETE SET NULL;`.execute(db); + await sql`ALTER TABLE "assets" ADD CONSTRAINT "FK_9977c3c1de01c3d848039a6b90c" FOREIGN KEY ("libraryId") REFERENCES "libraries" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "assets" ADD CONSTRAINT "FK_f15d48fa3ea5e4bda05ca8ab207" FOREIGN KEY ("stackId") REFERENCES "asset_stack" ("id") ON UPDATE CASCADE ON DELETE SET NULL;`.execute(db); + await sql`ALTER TABLE "albums" ADD CONSTRAINT "FK_b22c53f35ef20c28c21637c85f4" FOREIGN KEY ("ownerId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "albums" ADD CONSTRAINT "FK_05895aa505a670300d4816debce" FOREIGN KEY ("albumThumbnailAssetId") REFERENCES "assets" ("id") ON UPDATE CASCADE ON DELETE SET NULL;`.execute(db); + await sql`ALTER TABLE "activity" ADD CONSTRAINT "FK_1af8519996fbfb3684b58df280b" FOREIGN KEY ("albumId") REFERENCES "albums" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "activity" ADD CONSTRAINT "FK_3571467bcbe021f66e2bdce96ea" FOREIGN KEY ("userId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "activity" ADD CONSTRAINT "FK_8091ea76b12338cb4428d33d782" FOREIGN KEY ("assetId") REFERENCES "assets" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "albums_assets_assets" ADD CONSTRAINT "FK_e590fa396c6898fcd4a50e40927" FOREIGN KEY ("albumsId") REFERENCES "albums" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "albums_assets_assets" ADD CONSTRAINT "FK_4bd1303d199f4e72ccdf998c621" FOREIGN KEY ("assetsId") REFERENCES "assets" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "albums_shared_users_users" ADD CONSTRAINT "FK_427c350ad49bd3935a50baab737" FOREIGN KEY ("albumsId") REFERENCES "albums" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "albums_shared_users_users" ADD CONSTRAINT "FK_f48513bf9bccefd6ff3ad30bd06" FOREIGN KEY ("usersId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "api_keys" ADD CONSTRAINT "FK_6c2e267ae764a9413b863a29342" FOREIGN KEY ("userId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "person" ADD CONSTRAINT "FK_5527cc99f530a547093f9e577b6" FOREIGN KEY ("ownerId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "person" ADD CONSTRAINT "FK_2bbabe31656b6778c6b87b61023" FOREIGN KEY ("faceAssetId") REFERENCES "asset_faces" ("id") ON UPDATE NO ACTION ON DELETE SET NULL;`.execute(db); + await sql`ALTER TABLE "asset_faces" ADD CONSTRAINT "FK_02a43fd0b3c50fb6d7f0cb7282c" FOREIGN KEY ("assetId") REFERENCES "assets" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "asset_faces" ADD CONSTRAINT "FK_95ad7106dd7b484275443f580f9" FOREIGN KEY ("personId") REFERENCES "person" ("id") ON UPDATE CASCADE ON DELETE SET NULL;`.execute(db); + await sql`ALTER TABLE "asset_files" ADD CONSTRAINT "FK_e3e103a5f1d8bc8402999286040" FOREIGN KEY ("assetId") REFERENCES "assets" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "asset_job_status" ADD CONSTRAINT "FK_420bec36fc02813bddf5c8b73d4" FOREIGN KEY ("assetId") REFERENCES "assets" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "exif" ADD CONSTRAINT "FK_c0117fdbc50b917ef9067740c44" FOREIGN KEY ("assetId") REFERENCES "assets" ("id") ON UPDATE NO ACTION ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "face_search" ADD CONSTRAINT "face_search_faceId_fkey" FOREIGN KEY ("faceId") REFERENCES "asset_faces" ("id") ON UPDATE NO ACTION ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "memories" ADD CONSTRAINT "FK_575842846f0c28fa5da46c99b19" FOREIGN KEY ("ownerId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "memories_assets_assets" ADD CONSTRAINT "FK_984e5c9ab1f04d34538cd32334e" FOREIGN KEY ("memoriesId") REFERENCES "memories" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "memories_assets_assets" ADD CONSTRAINT "FK_6942ecf52d75d4273de19d2c16f" FOREIGN KEY ("assetsId") REFERENCES "assets" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "partners" ADD CONSTRAINT "FK_7e077a8b70b3530138610ff5e04" FOREIGN KEY ("sharedById") REFERENCES "users" ("id") ON UPDATE NO ACTION ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "partners" ADD CONSTRAINT "FK_d7e875c6c60e661723dbf372fd3" FOREIGN KEY ("sharedWithId") REFERENCES "users" ("id") ON UPDATE NO ACTION ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "sessions" ADD CONSTRAINT "FK_57de40bc620f456c7311aa3a1e6" FOREIGN KEY ("userId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "shared_links" ADD CONSTRAINT "FK_66fe3837414c5a9f1c33ca49340" FOREIGN KEY ("userId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "shared_links" ADD CONSTRAINT "FK_0c6ce9058c29f07cdf7014eac66" FOREIGN KEY ("albumId") REFERENCES "albums" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "shared_link__asset" ADD CONSTRAINT "FK_5b7decce6c8d3db9593d6111a66" FOREIGN KEY ("assetsId") REFERENCES "assets" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "shared_link__asset" ADD CONSTRAINT "FK_c9fab4aa97ffd1b034f3d6581ab" FOREIGN KEY ("sharedLinksId") REFERENCES "shared_links" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "smart_search" ADD CONSTRAINT "smart_search_assetId_fkey" FOREIGN KEY ("assetId") REFERENCES "assets" ("id") ON UPDATE NO ACTION ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "session_sync_checkpoints" ADD CONSTRAINT "FK_d8ddd9d687816cc490432b3d4bc" FOREIGN KEY ("sessionId") REFERENCES "sessions" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "tags" ADD CONSTRAINT "FK_92e67dc508c705dd66c94615576" FOREIGN KEY ("userId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "tags" ADD CONSTRAINT "FK_9f9590cc11561f1f48ff034ef99" FOREIGN KEY ("parentId") REFERENCES "tags" ("id") ON UPDATE NO ACTION ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "tag_asset" ADD CONSTRAINT "FK_f8e8a9e893cb5c54907f1b798e9" FOREIGN KEY ("assetsId") REFERENCES "assets" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "tag_asset" ADD CONSTRAINT "FK_e99f31ea4cdf3a2c35c7287eb42" FOREIGN KEY ("tagsId") REFERENCES "tags" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "tags_closure" ADD CONSTRAINT "FK_15fbcbc67663c6bfc07b354c22c" FOREIGN KEY ("id_ancestor") REFERENCES "tags" ("id") ON UPDATE NO ACTION ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "tags_closure" ADD CONSTRAINT "FK_b1a2a7ed45c29179b5ad51548a1" FOREIGN KEY ("id_descendant") REFERENCES "tags" ("id") ON UPDATE NO ACTION ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "user_metadata" ADD CONSTRAINT "FK_6afb43681a21cf7815932bc38ac" FOREIGN KEY ("userId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "users" ADD CONSTRAINT "UQ_97672ac88f789774dd47f7c8be3" UNIQUE ("email");`.execute(db); + await sql`ALTER TABLE "users" ADD CONSTRAINT "UQ_b309cf34fa58137c416b32cea3a" UNIQUE ("storageLabel");`.execute(db); + await sql`ALTER TABLE "asset_stack" ADD CONSTRAINT "REL_91704e101438fd0653f582426d" UNIQUE ("primaryAssetId");`.execute(db); + await sql`ALTER TABLE "asset_files" ADD CONSTRAINT "UQ_assetId_type" UNIQUE ("assetId", "type");`.execute(db); + await sql`ALTER TABLE "move_history" ADD CONSTRAINT "UQ_newPath" UNIQUE ("newPath");`.execute(db); + await sql`ALTER TABLE "move_history" ADD CONSTRAINT "UQ_entityId_pathType" UNIQUE ("entityId", "pathType");`.execute(db); + await sql`ALTER TABLE "shared_links" ADD CONSTRAINT "UQ_sharedlink_key" UNIQUE ("key");`.execute(db); + await sql`ALTER TABLE "tags" ADD CONSTRAINT "UQ_79d6f16e52bb2c7130375246793" UNIQUE ("userId", "value");`.execute(db); + await sql`ALTER TABLE "activity" ADD CONSTRAINT "CHK_2ab1e70f113f450eb40c1e3ec8" CHECK (("comment" IS NULL AND "isLiked" = true) OR ("comment" IS NOT NULL AND "isLiked" = false));`.execute(db); + await sql`ALTER TABLE "person" ADD CONSTRAINT "CHK_b0f82b0ed662bfc24fbb58bb45" CHECK ("birthDate" <= CURRENT_DATE);`.execute(db); + await sql`CREATE INDEX "IDX_users_updated_at_asc_id_asc" ON "users" ("updatedAt", "id")`.execute(db); + await sql`CREATE INDEX "IDX_users_update_id" ON "users" ("updateId")`.execute(db); + await sql`CREATE INDEX "IDX_0f6fc2fb195f24d19b0fb0d57c" ON "libraries" ("ownerId")`.execute(db); + await sql`CREATE INDEX "IDX_libraries_update_id" ON "libraries" ("updateId")`.execute(db); + await sql`CREATE INDEX "IDX_91704e101438fd0653f582426d" ON "asset_stack" ("primaryAssetId")`.execute(db); + await sql`CREATE INDEX "IDX_c05079e542fd74de3b5ecb5c1c" ON "asset_stack" ("ownerId")`.execute(db); + await sql`CREATE INDEX "idx_originalfilename_trigram" ON "assets" USING gin (f_unaccent("originalFileName") gin_trgm_ops)`.execute(db); + await sql`CREATE INDEX "IDX_asset_id_stackId" ON "assets" ("id", "stackId")`.execute(db); + await sql`CREATE INDEX "IDX_originalPath_libraryId" ON "assets" ("originalPath", "libraryId")`.execute(db); + await sql`CREATE INDEX "idx_local_date_time_month" ON "assets" ((date_trunc('MONTH'::text, ("localDateTime" AT TIME ZONE 'UTC'::text)) AT TIME ZONE 'UTC'::text))`.execute(db); + await sql`CREATE INDEX "idx_local_date_time" ON "assets" ((("localDateTime" at time zone 'UTC')::date))`.execute(db); + await sql`CREATE UNIQUE INDEX "UQ_assets_owner_library_checksum" ON "assets" ("ownerId", "libraryId", "checksum") WHERE ("libraryId" IS NOT NULL)`.execute(db); + await sql`CREATE UNIQUE INDEX "UQ_assets_owner_checksum" ON "assets" ("ownerId", "checksum") WHERE ("libraryId" IS NULL)`.execute(db); + await sql`CREATE INDEX "IDX_2c5ac0d6fb58b238fd2068de67" ON "assets" ("ownerId")`.execute(db); + await sql`CREATE INDEX "idx_asset_file_created_at" ON "assets" ("fileCreatedAt")`.execute(db); + await sql`CREATE INDEX "IDX_8d3efe36c0755849395e6ea866" ON "assets" ("checksum")`.execute(db); + await sql`CREATE INDEX "IDX_16294b83fa8c0149719a1f631e" ON "assets" ("livePhotoVideoId")`.execute(db); + await sql`CREATE INDEX "IDX_4d66e76dada1ca180f67a205dc" ON "assets" ("originalFileName")`.execute(db); + await sql`CREATE INDEX "IDX_9977c3c1de01c3d848039a6b90" ON "assets" ("libraryId")`.execute(db); + await sql`CREATE INDEX "IDX_f15d48fa3ea5e4bda05ca8ab20" ON "assets" ("stackId")`.execute(db); + await sql`CREATE INDEX "IDX_assets_duplicateId" ON "assets" ("duplicateId")`.execute(db); + await sql`CREATE INDEX "IDX_assets_update_id" ON "assets" ("updateId")`.execute(db); + await sql`CREATE INDEX "IDX_b22c53f35ef20c28c21637c85f" ON "albums" ("ownerId")`.execute(db); + await sql`CREATE INDEX "IDX_05895aa505a670300d4816debc" ON "albums" ("albumThumbnailAssetId")`.execute(db); + await sql`CREATE INDEX "IDX_albums_update_id" ON "albums" ("updateId")`.execute(db); + await sql`CREATE UNIQUE INDEX "IDX_activity_like" ON "activity" ("assetId", "userId", "albumId") WHERE ("isLiked" = true)`.execute(db); + await sql`CREATE INDEX "IDX_1af8519996fbfb3684b58df280" ON "activity" ("albumId")`.execute(db); + await sql`CREATE INDEX "IDX_3571467bcbe021f66e2bdce96e" ON "activity" ("userId")`.execute(db); + await sql`CREATE INDEX "IDX_8091ea76b12338cb4428d33d78" ON "activity" ("assetId")`.execute(db); + await sql`CREATE INDEX "IDX_activity_update_id" ON "activity" ("updateId")`.execute(db); + await sql`CREATE INDEX "IDX_e590fa396c6898fcd4a50e4092" ON "albums_assets_assets" ("albumsId")`.execute(db); + await sql`CREATE INDEX "IDX_4bd1303d199f4e72ccdf998c62" ON "albums_assets_assets" ("assetsId")`.execute(db); + await sql`CREATE INDEX "IDX_f48513bf9bccefd6ff3ad30bd0" ON "albums_shared_users_users" ("usersId")`.execute(db); + await sql`CREATE INDEX "IDX_427c350ad49bd3935a50baab73" ON "albums_shared_users_users" ("albumsId")`.execute(db); + await sql`CREATE INDEX "IDX_6c2e267ae764a9413b863a2934" ON "api_keys" ("userId")`.execute(db); + await sql`CREATE INDEX "IDX_api_keys_update_id" ON "api_keys" ("updateId")`.execute(db); + await sql`CREATE INDEX "IDX_assets_audit_asset_id" ON "assets_audit" ("assetId")`.execute(db); + await sql`CREATE INDEX "IDX_assets_audit_owner_id" ON "assets_audit" ("ownerId")`.execute(db); + await sql`CREATE INDEX "IDX_assets_audit_deleted_at" ON "assets_audit" ("deletedAt")`.execute(db); + await sql`CREATE INDEX "IDX_5527cc99f530a547093f9e577b" ON "person" ("ownerId")`.execute(db); + await sql`CREATE INDEX "IDX_2bbabe31656b6778c6b87b6102" ON "person" ("faceAssetId")`.execute(db); + await sql`CREATE INDEX "IDX_person_update_id" ON "person" ("updateId")`.execute(db); + await sql`CREATE INDEX "IDX_bf339a24070dac7e71304ec530" ON "asset_faces" ("personId", "assetId")`.execute(db); + await sql`CREATE INDEX "IDX_asset_faces_assetId_personId" ON "asset_faces" ("assetId", "personId")`.execute(db); + await sql`CREATE INDEX "IDX_02a43fd0b3c50fb6d7f0cb7282" ON "asset_faces" ("assetId")`.execute(db); + await sql`CREATE INDEX "IDX_95ad7106dd7b484275443f580f" ON "asset_faces" ("personId")`.execute(db); + await sql`CREATE INDEX "IDX_asset_files_assetId" ON "asset_files" ("assetId")`.execute(db); + await sql`CREATE INDEX "IDX_asset_files_update_id" ON "asset_files" ("updateId")`.execute(db); + await sql`CREATE INDEX "IDX_ownerId_createdAt" ON "audit" ("ownerId", "createdAt")`.execute(db); + await sql`CREATE INDEX "exif_city" ON "exif" ("city")`.execute(db); + await sql`CREATE INDEX "IDX_live_photo_cid" ON "exif" ("livePhotoCID")`.execute(db); + await sql`CREATE INDEX "IDX_auto_stack_id" ON "exif" ("autoStackId")`.execute(db); + await sql`CREATE INDEX "IDX_asset_exif_update_id" ON "exif" ("updateId")`.execute(db); + await sql`CREATE INDEX "face_index" ON "face_search" USING hnsw (embedding vector_cosine_ops) WITH (ef_construction = 300, m = 16)`.execute(db); + await sql`CREATE INDEX "IDX_geodata_gist_earthcoord" ON "geodata_places" (ll_to_earth_public(latitude, longitude))`.execute(db); + await sql`CREATE INDEX "idx_geodata_places_name" ON "geodata_places" USING gin (f_unaccent("name") gin_trgm_ops)`.execute(db); + await sql`CREATE INDEX "idx_geodata_places_admin2_name" ON "geodata_places" USING gin (f_unaccent("admin2Name") gin_trgm_ops)`.execute(db); + await sql`CREATE INDEX "idx_geodata_places_admin1_name" ON "geodata_places" USING gin (f_unaccent("admin1Name") gin_trgm_ops)`.execute(db); + await sql`CREATE INDEX "idx_geodata_places_alternate_names" ON "geodata_places" USING gin (f_unaccent("alternateNames") gin_trgm_ops)`.execute(db); + await sql`CREATE INDEX "IDX_575842846f0c28fa5da46c99b1" ON "memories" ("ownerId")`.execute(db); + await sql`CREATE INDEX "IDX_memories_update_id" ON "memories" ("updateId")`.execute(db); + await sql`CREATE INDEX "IDX_984e5c9ab1f04d34538cd32334" ON "memories_assets_assets" ("memoriesId")`.execute(db); + await sql`CREATE INDEX "IDX_6942ecf52d75d4273de19d2c16" ON "memories_assets_assets" ("assetsId")`.execute(db); + await sql`CREATE INDEX "IDX_partners_audit_shared_by_id" ON "partners_audit" ("sharedById")`.execute(db); + await sql`CREATE INDEX "IDX_partners_audit_shared_with_id" ON "partners_audit" ("sharedWithId")`.execute(db); + await sql`CREATE INDEX "IDX_partners_audit_deleted_at" ON "partners_audit" ("deletedAt")`.execute(db); + await sql`CREATE INDEX "IDX_7e077a8b70b3530138610ff5e0" ON "partners" ("sharedById")`.execute(db); + await sql`CREATE INDEX "IDX_d7e875c6c60e661723dbf372fd" ON "partners" ("sharedWithId")`.execute(db); + await sql`CREATE INDEX "IDX_partners_update_id" ON "partners" ("updateId")`.execute(db); + await sql`CREATE INDEX "IDX_57de40bc620f456c7311aa3a1e" ON "sessions" ("userId")`.execute(db); + await sql`CREATE INDEX "IDX_sessions_update_id" ON "sessions" ("updateId")`.execute(db); + await sql`CREATE INDEX "IDX_66fe3837414c5a9f1c33ca4934" ON "shared_links" ("userId")`.execute(db); + await sql`CREATE INDEX "IDX_sharedlink_key" ON "shared_links" ("key")`.execute(db); + await sql`CREATE INDEX "IDX_sharedlink_albumId" ON "shared_links" ("albumId")`.execute(db); + await sql`CREATE INDEX "IDX_5b7decce6c8d3db9593d6111a6" ON "shared_link__asset" ("assetsId")`.execute(db); + await sql`CREATE INDEX "IDX_c9fab4aa97ffd1b034f3d6581a" ON "shared_link__asset" ("sharedLinksId")`.execute(db); + await sql`CREATE INDEX "clip_index" ON "smart_search" USING hnsw (embedding vector_cosine_ops) WITH (ef_construction = 300, m = 16)`.execute(db); + await sql`CREATE INDEX "IDX_d8ddd9d687816cc490432b3d4b" ON "session_sync_checkpoints" ("sessionId")`.execute(db); + await sql`CREATE INDEX "IDX_session_sync_checkpoints_update_id" ON "session_sync_checkpoints" ("updateId")`.execute(db); + await sql`CREATE INDEX "IDX_92e67dc508c705dd66c9461557" ON "tags" ("userId")`.execute(db); + await sql`CREATE INDEX "IDX_9f9590cc11561f1f48ff034ef9" ON "tags" ("parentId")`.execute(db); + await sql`CREATE INDEX "IDX_tags_update_id" ON "tags" ("updateId")`.execute(db); + await sql`CREATE INDEX "IDX_tag_asset_assetsId_tagsId" ON "tag_asset" ("assetsId", "tagsId")`.execute(db); + await sql`CREATE INDEX "IDX_f8e8a9e893cb5c54907f1b798e" ON "tag_asset" ("assetsId")`.execute(db); + await sql`CREATE INDEX "IDX_e99f31ea4cdf3a2c35c7287eb4" ON "tag_asset" ("tagsId")`.execute(db); + await sql`CREATE INDEX "IDX_15fbcbc67663c6bfc07b354c22" ON "tags_closure" ("id_ancestor")`.execute(db); + await sql`CREATE INDEX "IDX_b1a2a7ed45c29179b5ad51548a" ON "tags_closure" ("id_descendant")`.execute(db); + await sql`CREATE INDEX "IDX_users_audit_deleted_at" ON "users_audit" ("deletedAt")`.execute(db); + await sql`CREATE INDEX "IDX_6afb43681a21cf7815932bc38a" ON "user_metadata" ("userId")`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "users_delete_audit" + AFTER DELETE ON "users" + REFERENCING OLD TABLE AS "old" + FOR EACH STATEMENT + WHEN (pg_trigger_depth() = 0) + EXECUTE FUNCTION users_delete_audit();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "users_updated_at" + BEFORE UPDATE ON "users" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "libraries_updated_at" + BEFORE UPDATE ON "libraries" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "assets_delete_audit" + AFTER DELETE ON "assets" + REFERENCING OLD TABLE AS "old" + FOR EACH STATEMENT + WHEN (pg_trigger_depth() = 0) + EXECUTE FUNCTION assets_delete_audit();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "assets_updated_at" + BEFORE UPDATE ON "assets" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "albums_updated_at" + BEFORE UPDATE ON "albums" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "activity_updated_at" + BEFORE UPDATE ON "activity" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "api_keys_updated_at" + BEFORE UPDATE ON "api_keys" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "person_updated_at" + BEFORE UPDATE ON "person" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "asset_files_updated_at" + BEFORE UPDATE ON "asset_files" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "asset_exif_updated_at" + BEFORE UPDATE ON "exif" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "memories_updated_at" + BEFORE UPDATE ON "memories" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "partners_delete_audit" + AFTER DELETE ON "partners" + REFERENCING OLD TABLE AS "old" + FOR EACH STATEMENT + WHEN (pg_trigger_depth() = 0) + EXECUTE FUNCTION partners_delete_audit();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "partners_updated_at" + BEFORE UPDATE ON "partners" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "sessions_updated_at" + BEFORE UPDATE ON "sessions" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "session_sync_checkpoints_updated_at" + BEFORE UPDATE ON "session_sync_checkpoints" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "tags_updated_at" + BEFORE UPDATE ON "tags" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); +} + +export async function down(): Promise { +// not implemented +} diff --git a/server/src/schema/tables/geodata-places.table.ts b/server/src/schema/tables/geodata-places.table.ts index 631cfdff08..3e78b4cfcf 100644 --- a/server/src/schema/tables/geodata-places.table.ts +++ b/server/src/schema/tables/geodata-places.table.ts @@ -1,6 +1,6 @@ import { Column, Index, PrimaryColumn, Table } from 'src/sql-tools'; -@Table({ name: 'geodata_places' }) +@Table({ name: 'geodata_places', synchronize: false }) @Index({ name: 'idx_geodata_places_alternate_names', using: 'gin', @@ -26,11 +26,10 @@ import { Column, Index, PrimaryColumn, Table } from 'src/sql-tools'; synchronize: false, }) @Index({ - name: 'idx_geodata_places_gist_earthcoord', + name: 'IDX_geodata_gist_earthcoord', expression: 'll_to_earth_public(latitude, longitude)', synchronize: false, }) -@Table({ name: 'idx_geodata_places', synchronize: false }) export class GeodataPlacesTable { @PrimaryColumn({ type: 'integer' }) id!: number; diff --git a/server/src/sql-tools/from-code/processors/table.processor.ts b/server/src/sql-tools/from-code/processors/table.processor.ts index eb4b414576..4ef4e82020 100644 --- a/server/src/sql-tools/from-code/processors/table.processor.ts +++ b/server/src/sql-tools/from-code/processors/table.processor.ts @@ -6,6 +6,13 @@ export const processTables: Processor = (builder, items) => { for (const { item: { options, object }, } of items.filter((item) => item.type === 'table')) { + const test = readMetadata(object); + if (test) { + throw new Error( + `Table ${test.name} has already been registered. Does ${object.name} have two @Table() decorators?`, + ); + } + const tableName = options.name || asSnakeCase(object.name); writeMetadata(object, { name: tableName, options }); diff --git a/server/src/sql-tools/from-code/register-function.ts b/server/src/sql-tools/from-code/register-function.ts index be71e0dfd7..3e1e7054be 100644 --- a/server/src/sql-tools/from-code/register-function.ts +++ b/server/src/sql-tools/from-code/register-function.ts @@ -50,12 +50,8 @@ const asFunctionExpression = (options: FunctionOptions) => { } if ('body' in options) { - sql.push( - // - `AS $$`, - ' ' + options.body.trim(), - `$$;`, - ); + const body = options.body; + sql.push(...(body.includes('\n') ? [`AS $$`, ' ' + body.trim(), `$$;`] : [`AS $$${body}$$;`])); } return sql.join('\n ').trim(); diff --git a/server/test/medium/globalSetup.ts b/server/test/medium/globalSetup.ts index 89b25da766..46eb1a733f 100644 --- a/server/test/medium/globalSetup.ts +++ b/server/test/medium/globalSetup.ts @@ -1,10 +1,11 @@ -import { FileMigrationProvider, Kysely, Migrator } from 'kysely'; -import { mkdir, readdir } from 'node:fs/promises'; -import { join } from 'node:path'; +import { Kysely } from 'kysely'; import { parse } from 'pg-connection-string'; +import { DB } from 'src/db'; +import { ConfigRepository } from 'src/repositories/config.repository'; +import { DatabaseRepository } from 'src/repositories/database.repository'; +import { LoggingRepository } from 'src/repositories/logging.repository'; import { getKyselyConfig } from 'src/utils/database'; import { GenericContainer, Wait } from 'testcontainers'; -import { DataSource } from 'typeorm'; const globalSetup = async () => { const postgresContainer = await new GenericContainer('tensorchord/pgvecto-rs:pg14-v0.2.0') @@ -36,66 +37,23 @@ const globalSetup = async () => { const postgresPort = postgresContainer.getMappedPort(5432); const postgresUrl = `postgres://postgres:postgres@localhost:${postgresPort}/immich`; + const parsed = parse(postgresUrl); + process.env.IMMICH_TEST_POSTGRES_URL = postgresUrl; - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-expect-error - const modules = import.meta.glob('/src/migrations/*.ts', { eager: true }); - - const config = { - type: 'postgres' as const, - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-expect-error - migrations: Object.values(modules).map((module) => Object.values(module)[0]), - migrationsRun: false, - synchronize: false, - connectTimeoutMS: 10_000, // 10 seconds - parseInt8: true, - url: postgresUrl, - }; - - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-expect-error - const dataSource = new DataSource(config); - await dataSource.initialize(); - await dataSource.runMigrations(); - await dataSource.destroy(); - - // for whatever reason, importing from test/utils causes vitest to crash - // eslint-disable-next-line unicorn/prefer-module - const migrationFolder = join(__dirname, '..', 'schema/migrations'); - // TODO remove after we have at least one kysely migration - await mkdir(migrationFolder, { recursive: true }); - - const parsed = parse(process.env.IMMICH_TEST_POSTGRES_URL!); - - const parsedOptions = { - ...parsed, - ssl: false, - host: parsed.host ?? undefined, - port: parsed.port ? Number(parsed.port) : undefined, - database: parsed.database ?? undefined, - }; - - const db = new Kysely(getKyselyConfig(parsedOptions)); - - // TODO just call `databaseRepository.migrate()` (probably have to wait until TypeOrm is gone) - const migrator = new Migrator({ - db, - migrationLockTableName: 'kysely_migrations_lock', - migrationTableName: 'kysely_migrations', - provider: new FileMigrationProvider({ - fs: { readdir }, - path: { join }, - migrationFolder, + const db = new Kysely( + getKyselyConfig({ + ...parsed, + ssl: false, + host: parsed.host ?? undefined, + port: parsed.port ? Number(parsed.port) : undefined, + database: parsed.database ?? undefined, }), - }); + ); - const { error } = await migrator.migrateToLatest(); - if (error) { - console.error('Unable to run kysely migrations', error); - throw error; - } + const configRepository = new ConfigRepository(); + const logger = new LoggingRepository(undefined, configRepository); + await new DatabaseRepository(db, logger, configRepository).runMigrations(); await db.destroy(); }; diff --git a/server/test/sql-tools/index-with-where.stub copy.ts b/server/test/sql-tools/index-with-expression.ts similarity index 100% rename from server/test/sql-tools/index-with-where.stub copy.ts rename to server/test/sql-tools/index-with-expression.ts From 160bb492a2ae1344718578281756f79bb33f0489 Mon Sep 17 00:00:00 2001 From: Jason Rasmussen Date: Fri, 18 Apr 2025 12:19:11 -0400 Subject: [PATCH 06/12] fix: skip initial kysely migration for existing installs (#17690) * fix: skip initial kysely migration for existing installs * Update docs/src/pages/errors.md --------- Co-authored-by: Alex --- docs/src/pages/errors.md | 5 +++++ server/src/repositories/logging.repository.ts | 14 ++++++++++++-- .../1744910873969-InitialMigration.ts | 19 +++++++++++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 docs/src/pages/errors.md diff --git a/docs/src/pages/errors.md b/docs/src/pages/errors.md new file mode 100644 index 0000000000..e9bf09770c --- /dev/null +++ b/docs/src/pages/errors.md @@ -0,0 +1,5 @@ +# Errors + +## TypeORM Upgrade + +The upgrade to Immich `v2.x.x` has a required upgrade path to `v1.132.0+`. This means it is required to start up the application at least once on version `1.132.0` (or later). Doing so will complete database schema upgrades that are required for `v2.0.0`. After Immich has successfully booted on this version, shut the system down and try the `v2.x.x` upgrade again. diff --git a/server/src/repositories/logging.repository.ts b/server/src/repositories/logging.repository.ts index 3f809db41e..05d2d45f4d 100644 --- a/server/src/repositories/logging.repository.ts +++ b/server/src/repositories/logging.repository.ts @@ -74,11 +74,21 @@ export class MyConsoleLogger extends ConsoleLogger { export class LoggingRepository { private logger: MyConsoleLogger; - constructor(@Inject(ClsService) cls: ClsService | undefined, configRepository: ConfigRepository) { - const { noColor } = configRepository.getEnv(); + constructor( + @Inject(ClsService) cls: ClsService | undefined, + @Inject(ConfigRepository) configRepository: ConfigRepository | undefined, + ) { + let noColor = false; + if (configRepository) { + noColor = configRepository.getEnv().noColor; + } this.logger = new MyConsoleLogger(cls, { context: LoggingRepository.name, color: !noColor }); } + static create() { + return new LoggingRepository(undefined, undefined); + } + setAppName(name: string): void { appName = name.charAt(0).toUpperCase() + name.slice(1); } diff --git a/server/src/schema/migrations/1744910873969-InitialMigration.ts b/server/src/schema/migrations/1744910873969-InitialMigration.ts index e157607681..459534a26a 100644 --- a/server/src/schema/migrations/1744910873969-InitialMigration.ts +++ b/server/src/schema/migrations/1744910873969-InitialMigration.ts @@ -1,10 +1,29 @@ import { Kysely, sql } from 'kysely'; import { DatabaseExtension } from 'src/enum'; import { ConfigRepository } from 'src/repositories/config.repository'; +import { LoggingRepository } from 'src/repositories/logging.repository'; const vectorExtension = new ConfigRepository().getEnv().database.vectorExtension; +const lastMigrationSql = sql<{ name: string }>`SELECT "name" FROM "migrations" ORDER BY "timestamp" DESC LIMIT 1;`; +const tableExists = sql<{ result: string | null }>`select to_regclass('migrations') as "result"`; +const logger = LoggingRepository.create(); export async function up(db: Kysely): Promise { + const { rows } = await tableExists.execute(db); + const hasTypeOrmMigrations = !!rows[0]?.result; + if (hasTypeOrmMigrations) { + const { + rows: [lastMigration], + } = await lastMigrationSql.execute(db); + if (lastMigration?.name !== 'AddMissingIndex1744910873956') { + throw new Error( + 'Invalid upgrade path. For more information, see https://immich.app/errors#typeorm-upgrade', + ); + } + logger.log('Database has up to date TypeORM migrations, skipping initial Kysely migration'); + return; + } + await sql`CREATE EXTENSION IF NOT EXISTS "uuid-ossp";`.execute(db); await sql`CREATE EXTENSION IF NOT EXISTS "unaccent";`.execute(db); await sql`CREATE EXTENSION IF NOT EXISTS "cube";`.execute(db); From bd2deda50c4481903a04d51a5dc7042fdeea095a Mon Sep 17 00:00:00 2001 From: Yaros Date: Fri, 18 Apr 2025 18:19:51 +0200 Subject: [PATCH 07/12] feat(mobile): search on places page (#17679) * feat: search on places page * chore: use searchfield on people page --- i18n/en.json | 1 + .../people/people_collection.page.dart | 43 ++----------- .../places/places_collection.page.dart | 62 ++++++++++++++----- 3 files changed, 51 insertions(+), 55 deletions(-) diff --git a/i18n/en.json b/i18n/en.json index 3b52c2019e..c4b4746871 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -996,6 +996,7 @@ "filetype": "Filetype", "filter": "Filter", "filter_people": "Filter people", + "filter_places": "Filter places", "find_them_fast": "Find them fast by name with search", "fix_incorrect_match": "Fix incorrect match", "folder": "Folder", diff --git a/mobile/lib/pages/library/people/people_collection.page.dart b/mobile/lib/pages/library/people/people_collection.page.dart index 5f587c0c76..27daf0a887 100644 --- a/mobile/lib/pages/library/people/people_collection.page.dart +++ b/mobile/lib/pages/library/people/people_collection.page.dart @@ -4,11 +4,11 @@ import 'package:flutter/material.dart'; import 'package:flutter_hooks/flutter_hooks.dart'; import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/extensions/build_context_extensions.dart'; -import 'package:immich_mobile/extensions/theme_extensions.dart'; import 'package:immich_mobile/providers/search/people.provider.dart'; import 'package:immich_mobile/routing/router.dart'; import 'package:immich_mobile/services/api.service.dart'; import 'package:immich_mobile/utils/image_url_builder.dart'; +import 'package:immich_mobile/widgets/common/search_field.dart'; import 'package:immich_mobile/widgets/search/person_name_edit_form.dart'; @RoutePage() @@ -42,47 +42,12 @@ class PeopleCollectionPage extends HookConsumerWidget { appBar: AppBar( automaticallyImplyLeading: search.value == null, title: search.value != null - ? TextField( + ? SearchField( focusNode: formFocus, onTapOutside: (_) => formFocus.unfocus(), onChanged: (value) => search.value = value, - decoration: InputDecoration( - contentPadding: const EdgeInsets.only(left: 24), - filled: true, - fillColor: context.primaryColor.withValues(alpha: 0.1), - hintStyle: context.textTheme.bodyLarge?.copyWith( - color: context.themeData.colorScheme.onSurfaceSecondary, - ), - border: OutlineInputBorder( - borderRadius: BorderRadius.circular(25), - borderSide: BorderSide( - color: context.colorScheme.surfaceContainerHighest, - ), - ), - enabledBorder: OutlineInputBorder( - borderRadius: BorderRadius.circular(25), - borderSide: BorderSide( - color: context.colorScheme.surfaceContainerHighest, - ), - ), - disabledBorder: OutlineInputBorder( - borderRadius: BorderRadius.circular(25), - borderSide: BorderSide( - color: context.colorScheme.surfaceContainerHighest, - ), - ), - focusedBorder: OutlineInputBorder( - borderRadius: BorderRadius.circular(25), - borderSide: BorderSide( - color: context.colorScheme.primary.withAlpha(150), - ), - ), - prefixIcon: Icon( - Icons.search_rounded, - color: context.colorScheme.primary, - ), - hintText: 'filter_people'.tr(), - ), + filled: true, + hintText: 'filter_people'.tr(), autofocus: true, ) : Text('people'.tr()), diff --git a/mobile/lib/pages/library/places/places_collection.page.dart b/mobile/lib/pages/library/places/places_collection.page.dart index d4da3ff37e..f9a2d4292c 100644 --- a/mobile/lib/pages/library/places/places_collection.page.dart +++ b/mobile/lib/pages/library/places/places_collection.page.dart @@ -2,6 +2,7 @@ import 'package:auto_route/auto_route.dart'; import 'package:cached_network_image/cached_network_image.dart'; import 'package:easy_localization/easy_localization.dart'; import 'package:flutter/material.dart'; +import 'package:flutter_hooks/flutter_hooks.dart' hide Store; import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/domain/models/store.model.dart'; import 'package:immich_mobile/entities/asset.entity.dart'; @@ -12,6 +13,7 @@ import 'package:immich_mobile/pages/common/large_leading_tile.dart'; import 'package:immich_mobile/providers/search/search_page_state.provider.dart'; import 'package:immich_mobile/routing/router.dart'; import 'package:immich_mobile/services/api.service.dart'; +import 'package:immich_mobile/widgets/common/search_field.dart'; import 'package:immich_mobile/widgets/map/map_thumbnail.dart'; import 'package:maplibre_gl/maplibre_gl.dart'; @@ -21,34 +23,62 @@ class PlacesCollectionPage extends HookConsumerWidget { @override Widget build(BuildContext context, WidgetRef ref) { final places = ref.watch(getAllPlacesProvider); + final formFocus = useFocusNode(); + final ValueNotifier search = useState(null); return Scaffold( appBar: AppBar( - title: Text('places'.tr()), + automaticallyImplyLeading: search.value == null, + title: search.value != null + ? SearchField( + autofocus: true, + filled: true, + focusNode: formFocus, + onChanged: (value) => search.value = value, + onTapOutside: (_) => formFocus.unfocus(), + hintText: 'filter_places'.tr(), + ) + : Text('places'.tr()), + actions: [ + IconButton( + icon: Icon(search.value != null ? Icons.close : Icons.search), + onPressed: () { + search.value = search.value == null ? '' : null; + }, + ), + ], ), body: ListView( shrinkWrap: true, children: [ - Padding( - padding: const EdgeInsets.all(16.0), - child: SizedBox( - height: 200, - width: context.width, - child: MapThumbnail( - onTap: (_, __) => context.pushRoute(const MapRoute()), - zoom: 8, - centre: const LatLng( - 21.44950, - -157.91959, + if (search.value == null) + Padding( + padding: const EdgeInsets.all(16.0), + child: SizedBox( + height: 200, + width: context.width, + child: MapThumbnail( + onTap: (_, __) => context.pushRoute(const MapRoute()), + zoom: 8, + centre: const LatLng( + 21.44950, + -157.91959, + ), + showAttribution: false, + themeMode: + context.isDarkTheme ? ThemeMode.dark : ThemeMode.light, ), - showAttribution: false, - themeMode: - context.isDarkTheme ? ThemeMode.dark : ThemeMode.light, ), ), - ), places.when( data: (places) { + if (search.value != null) { + places = places.where((place) { + return place.label + .toLowerCase() + .contains(search.value!.toLowerCase()); + }).toList(); + } return ListView.builder( shrinkWrap: true, physics: const NeverScrollableScrollPhysics(), From 0e6ac876450c6e6ba1072cf644abf395912e0fef Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 18 Apr 2025 14:01:16 -0500 Subject: [PATCH 08/12] feat(mobile): assets + exif stream sync placeholder (#17677) * feat(mobile): assets + exif stream sync placeholder * feat(mobile): assets + exif stream sync placeholder * refactor * fix: test * fix:test * refactor(mobile): sync stream service (#17687) * refactor: sync stream to use callbacks * pr feedback * pr feedback * pr feedback * fix: test --------- Co-authored-by: shenlong-tanwen <139912620+shalong-tanwen@users.noreply.github.com> Co-authored-by: Alex Tran --------- Co-authored-by: shenlong <139912620+shenlong-tanwen@users.noreply.github.com> Co-authored-by: shenlong-tanwen <139912620+shalong-tanwen@users.noreply.github.com> --- .../domain/interfaces/sync_api.interface.dart | 8 +- .../interfaces/sync_stream.interface.dart | 16 +- .../domain/services/sync_stream.service.dart | 228 +++------ mobile/lib/domain/utils/background_sync.dart | 28 +- .../repositories/sync_api.repository.dart | 75 ++- .../repositories/sync_stream.repository.dart | 62 ++- mobile/lib/widgets/common/immich_app_bar.dart | 7 + mobile/test/api.mocks.dart | 2 + .../services/sync_stream_service_test.dart | 475 +++++------------- mobile/test/fixtures/sync_stream.stub.dart | 74 +-- .../sync_api_repository_test.dart | 299 +++++++++++ 11 files changed, 666 insertions(+), 608 deletions(-) create mode 100644 mobile/test/infrastructure/repositories/sync_api_repository_test.dart diff --git a/mobile/lib/domain/interfaces/sync_api.interface.dart b/mobile/lib/domain/interfaces/sync_api.interface.dart index 44e22c5894..57abed2e7f 100644 --- a/mobile/lib/domain/interfaces/sync_api.interface.dart +++ b/mobile/lib/domain/interfaces/sync_api.interface.dart @@ -1,8 +1,12 @@ +import 'package:http/http.dart' as http; import 'package:immich_mobile/domain/models/sync_event.model.dart'; -import 'package:openapi/api.dart'; abstract interface class ISyncApiRepository { Future ack(List data); - Stream> getSyncEvents(List type); + Future streamChanges( + Function(List, Function() abort) onData, { + int batchSize, + http.Client? httpClient, + }); } diff --git a/mobile/lib/domain/interfaces/sync_stream.interface.dart b/mobile/lib/domain/interfaces/sync_stream.interface.dart index f9c52d7ee0..5f61d6b52f 100644 --- a/mobile/lib/domain/interfaces/sync_stream.interface.dart +++ b/mobile/lib/domain/interfaces/sync_stream.interface.dart @@ -2,9 +2,17 @@ import 'package:immich_mobile/domain/interfaces/db.interface.dart'; import 'package:openapi/api.dart'; abstract interface class ISyncStreamRepository implements IDatabaseRepository { - Future updateUsersV1(Iterable data); - Future deleteUsersV1(Iterable data); + Future updateUsersV1(Iterable data); + Future deleteUsersV1(Iterable data); - Future updatePartnerV1(Iterable data); - Future deletePartnerV1(Iterable data); + Future updatePartnerV1(Iterable data); + Future deletePartnerV1(Iterable data); + + Future updateAssetsV1(Iterable data); + Future deleteAssetsV1(Iterable data); + Future updateAssetsExifV1(Iterable data); + + Future updatePartnerAssetsV1(Iterable data); + Future deletePartnerAssetsV1(Iterable data); + Future updatePartnerAssetsExifV1(Iterable data); } diff --git a/mobile/lib/domain/services/sync_stream.service.dart b/mobile/lib/domain/services/sync_stream.service.dart index 8d7d87e35e..ac63734b07 100644 --- a/mobile/lib/domain/services/sync_stream.service.dart +++ b/mobile/lib/domain/services/sync_stream.service.dart @@ -2,25 +2,11 @@ import 'dart:async'; -import 'package:collection/collection.dart'; import 'package:immich_mobile/domain/interfaces/sync_api.interface.dart'; import 'package:immich_mobile/domain/interfaces/sync_stream.interface.dart'; +import 'package:immich_mobile/domain/models/sync_event.model.dart'; import 'package:logging/logging.dart'; import 'package:openapi/api.dart'; -import 'package:worker_manager/worker_manager.dart'; - -const _kSyncTypeOrder = [ - SyncEntityType.userDeleteV1, - SyncEntityType.userV1, - SyncEntityType.partnerDeleteV1, - SyncEntityType.partnerV1, - SyncEntityType.assetDeleteV1, - SyncEntityType.assetV1, - SyncEntityType.assetExifV1, - SyncEntityType.partnerAssetDeleteV1, - SyncEntityType.partnerAssetV1, - SyncEntityType.partnerAssetExifV1, -]; class SyncStreamService { final Logger _logger = Logger('SyncStreamService'); @@ -37,164 +23,70 @@ class SyncStreamService { _syncStreamRepository = syncStreamRepository, _cancelChecker = cancelChecker; - Future _handleSyncData( + bool get isCancelled => _cancelChecker?.call() ?? false; + + Future sync() => _syncApiRepository.streamChanges(_handleEvents); + + Future _handleEvents(List events, Function() abort) async { + List items = []; + for (final event in events) { + if (isCancelled) { + _logger.warning("Sync stream cancelled"); + abort(); + return; + } + + if (event.type != items.firstOrNull?.type) { + await _processBatch(items); + } + + items.add(event); + } + + await _processBatch(items); + } + + Future _processBatch(List batch) async { + if (batch.isEmpty) { + return; + } + + final type = batch.first.type; + await _handleSyncData(type, batch.map((e) => e.data)); + await _syncApiRepository.ack([batch.last.ack]); + batch.clear(); + } + + Future _handleSyncData( SyncEntityType type, // ignore: avoid-dynamic Iterable data, ) async { - if (data.isEmpty) { - _logger.warning("Received empty sync data for $type"); - return false; - } - _logger.fine("Processing sync data for $type of length ${data.length}"); - - try { - if (type == SyncEntityType.partnerV1) { - return await _syncStreamRepository.updatePartnerV1(data.cast()); - } - - if (type == SyncEntityType.partnerDeleteV1) { - return await _syncStreamRepository.deletePartnerV1(data.cast()); - } - - if (type == SyncEntityType.userV1) { - return await _syncStreamRepository.updateUsersV1(data.cast()); - } - - if (type == SyncEntityType.userDeleteV1) { - return await _syncStreamRepository.deleteUsersV1(data.cast()); - } - } catch (error, stack) { - _logger.severe("Error processing sync data for $type", error, stack); - return false; + // ignore: prefer-switch-expression + switch (type) { + case SyncEntityType.userV1: + return _syncStreamRepository.updateUsersV1(data.cast()); + case SyncEntityType.userDeleteV1: + return _syncStreamRepository.deleteUsersV1(data.cast()); + case SyncEntityType.partnerV1: + return _syncStreamRepository.updatePartnerV1(data.cast()); + case SyncEntityType.partnerDeleteV1: + return _syncStreamRepository.deletePartnerV1(data.cast()); + case SyncEntityType.assetV1: + return _syncStreamRepository.updateAssetsV1(data.cast()); + case SyncEntityType.assetDeleteV1: + return _syncStreamRepository.deleteAssetsV1(data.cast()); + case SyncEntityType.assetExifV1: + return _syncStreamRepository.updateAssetsExifV1(data.cast()); + case SyncEntityType.partnerAssetV1: + return _syncStreamRepository.updatePartnerAssetsV1(data.cast()); + case SyncEntityType.partnerAssetDeleteV1: + return _syncStreamRepository.deletePartnerAssetsV1(data.cast()); + case SyncEntityType.partnerAssetExifV1: + return _syncStreamRepository.updatePartnerAssetsExifV1(data.cast()); + default: + _logger.warning("Unknown sync data type: $type"); } - - _logger.warning("Unknown sync data type: $type"); - return false; } - - Future _syncEvent(List types) { - _logger.info("Syncing Events: $types"); - final streamCompleter = Completer(); - bool shouldComplete = false; - // the onDone callback might fire before the events are processed - // the following flag ensures that the onDone callback is not called - // before the events are processed and also that events are processed sequentially - Completer? mutex; - StreamSubscription? subscription; - try { - subscription = _syncApiRepository.getSyncEvents(types).listen( - (events) async { - if (events.isEmpty) { - _logger.warning("Received empty sync events"); - return; - } - - // If previous events are still being processed, wait for them to finish - if (mutex != null) { - await mutex!.future; - } - - if (_cancelChecker?.call() ?? false) { - _logger.info("Sync cancelled, stopping stream"); - subscription?.cancel(); - if (!streamCompleter.isCompleted) { - streamCompleter.completeError( - CanceledError(), - StackTrace.current, - ); - } - return; - } - - // Take control of the mutex and process the events - mutex = Completer(); - - try { - final eventsMap = events.groupListsBy((event) => event.type); - final Map acks = {}; - - for (final type in _kSyncTypeOrder) { - final data = eventsMap[type]; - if (data == null) { - continue; - } - - if (_cancelChecker?.call() ?? false) { - _logger.info("Sync cancelled, stopping stream"); - mutex?.complete(); - mutex = null; - if (!streamCompleter.isCompleted) { - streamCompleter.completeError( - CanceledError(), - StackTrace.current, - ); - } - - return; - } - - if (data.isEmpty) { - _logger.warning("Received empty sync events for $type"); - continue; - } - - if (await _handleSyncData(type, data.map((e) => e.data))) { - // ignore: avoid-unsafe-collection-methods - acks[type] = data.last.ack; - } else { - _logger.warning("Failed to handle sync events for $type"); - } - } - - if (acks.isNotEmpty) { - await _syncApiRepository.ack(acks.values.toList()); - } - _logger.info("$types events processed"); - } catch (error, stack) { - _logger.warning("Error handling sync events", error, stack); - } finally { - mutex?.complete(); - mutex = null; - } - - if (shouldComplete) { - _logger.info("Sync done, completing stream"); - if (!streamCompleter.isCompleted) streamCompleter.complete(); - } - }, - onError: (error, stack) { - _logger.warning("Error in sync stream for $types", error, stack); - // Do not proceed if the stream errors - if (!streamCompleter.isCompleted) { - // ignore: avoid-missing-completer-stack-trace - streamCompleter.completeError(error, stack); - } - }, - onDone: () { - _logger.info("$types stream done"); - if (mutex == null && !streamCompleter.isCompleted) { - streamCompleter.complete(); - } else { - // Marks the stream as done but does not complete the completer - // until the events are processed - shouldComplete = true; - } - }, - ); - } catch (error, stack) { - _logger.severe("Error starting sync stream", error, stack); - if (!streamCompleter.isCompleted) { - streamCompleter.completeError(error, stack); - } - } - return streamCompleter.future.whenComplete(() { - _logger.info("Sync stream completed"); - return subscription?.cancel(); - }); - } - - Future syncUsers() => - _syncEvent([SyncRequestType.usersV1, SyncRequestType.partnersV1]); } diff --git a/mobile/lib/domain/utils/background_sync.dart b/mobile/lib/domain/utils/background_sync.dart index 0bd456f0bb..f63dc81ba9 100644 --- a/mobile/lib/domain/utils/background_sync.dart +++ b/mobile/lib/domain/utils/background_sync.dart @@ -7,31 +7,33 @@ import 'package:immich_mobile/utils/isolate.dart'; import 'package:worker_manager/worker_manager.dart'; class BackgroundSyncManager { - Cancelable? _userSyncTask; + Cancelable? _syncTask; BackgroundSyncManager(); Future cancel() { final futures = []; - if (_userSyncTask != null) { - futures.add(_userSyncTask!.future); + + if (_syncTask != null) { + futures.add(_syncTask!.future); } - _userSyncTask?.cancel(); - _userSyncTask = null; + _syncTask?.cancel(); + _syncTask = null; + return Future.wait(futures); } - Future syncUsers() { - if (_userSyncTask != null) { - return _userSyncTask!.future; + Future sync() { + if (_syncTask != null) { + return _syncTask!.future; } - _userSyncTask = runInIsolateGentle( - computation: (ref) => ref.read(syncStreamServiceProvider).syncUsers(), + _syncTask = runInIsolateGentle( + computation: (ref) => ref.read(syncStreamServiceProvider).sync(), ); - _userSyncTask!.whenComplete(() { - _userSyncTask = null; + _syncTask!.whenComplete(() { + _syncTask = null; }); - return _userSyncTask!.future; + return _syncTask!.future; } } diff --git a/mobile/lib/infrastructure/repositories/sync_api.repository.dart b/mobile/lib/infrastructure/repositories/sync_api.repository.dart index a26b867df6..dd1ea208ba 100644 --- a/mobile/lib/infrastructure/repositories/sync_api.repository.dart +++ b/mobile/lib/infrastructure/repositories/sync_api.repository.dart @@ -12,22 +12,22 @@ import 'package:openapi/api.dart'; class SyncApiRepository implements ISyncApiRepository { final Logger _logger = Logger('SyncApiRepository'); final ApiService _api; - final int _batchSize; - SyncApiRepository(this._api, {int batchSize = kSyncEventBatchSize}) - : _batchSize = batchSize; - - @override - Stream> getSyncEvents(List type) { - return _getSyncStream(SyncStreamDto(types: type)); - } + SyncApiRepository(this._api); @override Future ack(List data) { return _api.syncApi.sendSyncAck(SyncAckSetDto(acks: data)); } - Stream> _getSyncStream(SyncStreamDto dto) async* { - final client = http.Client(); + @override + Future streamChanges( + Function(List, Function() abort) onData, { + int batchSize = kSyncEventBatchSize, + http.Client? httpClient, + }) async { + // ignore: avoid-unused-assignment + final stopwatch = Stopwatch()..start(); + final client = httpClient ?? http.Client(); final endpoint = "${_api.apiClient.basePath}/sync/stream"; final headers = { @@ -35,20 +35,38 @@ class SyncApiRepository implements ISyncApiRepository { 'Accept': 'application/jsonlines+json', }; - final queryParams = []; final headerParams = {}; - await _api.applyToParams(queryParams, headerParams); + await _api.applyToParams([], headerParams); headers.addAll(headerParams); final request = http.Request('POST', Uri.parse(endpoint)); request.headers.addAll(headers); - request.body = jsonEncode(dto.toJson()); + request.body = jsonEncode( + SyncStreamDto( + types: [ + SyncRequestType.usersV1, + SyncRequestType.partnersV1, + SyncRequestType.assetsV1, + SyncRequestType.partnerAssetsV1, + SyncRequestType.assetExifsV1, + SyncRequestType.partnerAssetExifsV1, + ], + ).toJson(), + ); String previousChunk = ''; List lines = []; + bool shouldAbort = false; + + void abort() { + _logger.warning("Abort requested, stopping sync stream"); + shouldAbort = true; + } + try { - final response = await client.send(request); + final response = + await client.send(request).timeout(const Duration(seconds: 20)); if (response.statusCode != 200) { final errorBody = await response.stream.bytesToString(); @@ -59,27 +77,38 @@ class SyncApiRepository implements ISyncApiRepository { } await for (final chunk in response.stream.transform(utf8.decoder)) { + if (shouldAbort) { + break; + } + previousChunk += chunk; final parts = previousChunk.toString().split('\n'); previousChunk = parts.removeLast(); lines.addAll(parts); - if (lines.length < _batchSize) { + if (lines.length < batchSize) { continue; } - yield _parseSyncResponse(lines); + await onData(_parseLines(lines), abort); lines.clear(); } - } finally { - if (lines.isNotEmpty) { - yield _parseSyncResponse(lines); + + if (lines.isNotEmpty && !shouldAbort) { + await onData(_parseLines(lines), abort); } + } catch (error, stack) { + _logger.severe("error processing stream", error, stack); + return Future.error(error, stack); + } finally { client.close(); } + stopwatch.stop(); + _logger + .info("Remote Sync completed in ${stopwatch.elapsed.inMilliseconds}ms"); } - List _parseSyncResponse(List lines) { + List _parseLines(List lines) { final List data = []; for (final line in lines) { @@ -110,4 +139,10 @@ const _kResponseMap = { SyncEntityType.userDeleteV1: SyncUserDeleteV1.fromJson, SyncEntityType.partnerV1: SyncPartnerV1.fromJson, SyncEntityType.partnerDeleteV1: SyncPartnerDeleteV1.fromJson, + SyncEntityType.assetV1: SyncAssetV1.fromJson, + SyncEntityType.assetDeleteV1: SyncAssetDeleteV1.fromJson, + SyncEntityType.assetExifV1: SyncAssetExifV1.fromJson, + SyncEntityType.partnerAssetV1: SyncAssetV1.fromJson, + SyncEntityType.partnerAssetDeleteV1: SyncAssetDeleteV1.fromJson, + SyncEntityType.partnerAssetExifV1: SyncAssetExifV1.fromJson, }; diff --git a/mobile/lib/infrastructure/repositories/sync_stream.repository.dart b/mobile/lib/infrastructure/repositories/sync_stream.repository.dart index a947a9a66b..5ad9a369df 100644 --- a/mobile/lib/infrastructure/repositories/sync_stream.repository.dart +++ b/mobile/lib/infrastructure/repositories/sync_stream.repository.dart @@ -1,4 +1,5 @@ import 'package:drift/drift.dart'; +import 'package:flutter/foundation.dart'; import 'package:immich_mobile/domain/interfaces/sync_stream.interface.dart'; import 'package:immich_mobile/extensions/string_extensions.dart'; import 'package:immich_mobile/infrastructure/entities/partner.entity.drift.dart'; @@ -15,7 +16,7 @@ class DriftSyncStreamRepository extends DriftDatabaseRepository DriftSyncStreamRepository(super.db) : _db = db; @override - Future deleteUsersV1(Iterable data) async { + Future deleteUsersV1(Iterable data) async { try { await _db.batch((batch) { for (final user in data) { @@ -25,15 +26,14 @@ class DriftSyncStreamRepository extends DriftDatabaseRepository ); } }); - return true; - } catch (e, s) { - _logger.severe('Error while processing SyncUserDeleteV1', e, s); - return false; + } catch (error, stack) { + _logger.severe('Error while processing SyncUserDeleteV1', error, stack); + rethrow; } } @override - Future updateUsersV1(Iterable data) async { + Future updateUsersV1(Iterable data) async { try { await _db.batch((batch) { for (final user in data) { @@ -49,15 +49,14 @@ class DriftSyncStreamRepository extends DriftDatabaseRepository ); } }); - return true; - } catch (e, s) { - _logger.severe('Error while processing SyncUserV1', e, s); - return false; + } catch (error, stack) { + _logger.severe('Error while processing SyncUserV1', error, stack); + rethrow; } } @override - Future deletePartnerV1(Iterable data) async { + Future deletePartnerV1(Iterable data) async { try { await _db.batch((batch) { for (final partner in data) { @@ -70,15 +69,14 @@ class DriftSyncStreamRepository extends DriftDatabaseRepository ); } }); - return true; } catch (e, s) { _logger.severe('Error while processing SyncPartnerDeleteV1', e, s); - return false; + rethrow; } } @override - Future updatePartnerV1(Iterable data) async { + Future updatePartnerV1(Iterable data) async { try { await _db.batch((batch) { for (final partner in data) { @@ -95,10 +93,42 @@ class DriftSyncStreamRepository extends DriftDatabaseRepository ); } }); - return true; } catch (e, s) { _logger.severe('Error while processing SyncPartnerV1', e, s); - return false; + rethrow; } } + + // Assets + @override + Future updateAssetsV1(Iterable data) async { + debugPrint("updateAssetsV1 - ${data.length}"); + } + + @override + Future deleteAssetsV1(Iterable data) async { + debugPrint("deleteAssetsV1 - ${data.length}"); + } + + // Partner Assets + @override + Future updatePartnerAssetsV1(Iterable data) async { + debugPrint("updatePartnerAssetsV1 - ${data.length}"); + } + + @override + Future deletePartnerAssetsV1(Iterable data) async { + debugPrint("deletePartnerAssetsV1 - ${data.length}"); + } + + // EXIF + @override + Future updateAssetsExifV1(Iterable data) async { + debugPrint("updateAssetsExifV1 - ${data.length}"); + } + + @override + Future updatePartnerAssetsExifV1(Iterable data) async { + debugPrint("updatePartnerAssetsExifV1 - ${data.length}"); + } } diff --git a/mobile/lib/widgets/common/immich_app_bar.dart b/mobile/lib/widgets/common/immich_app_bar.dart index 51b4faa014..4f95e657d9 100644 --- a/mobile/lib/widgets/common/immich_app_bar.dart +++ b/mobile/lib/widgets/common/immich_app_bar.dart @@ -1,11 +1,13 @@ import 'package:auto_route/auto_route.dart'; import 'package:easy_localization/easy_localization.dart'; +import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; import 'package:flutter_svg/svg.dart'; import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/extensions/build_context_extensions.dart'; import 'package:immich_mobile/models/backup/backup_state.model.dart'; import 'package:immich_mobile/models/server_info/server_info.model.dart'; +import 'package:immich_mobile/providers/background_sync.provider.dart'; import 'package:immich_mobile/providers/backup/backup.provider.dart'; import 'package:immich_mobile/providers/server_info.provider.dart'; import 'package:immich_mobile/providers/user.provider.dart'; @@ -178,6 +180,11 @@ class ImmichAppBar extends ConsumerWidget implements PreferredSizeWidget { child: action, ), ), + if (kDebugMode) + IconButton( + onPressed: () => ref.read(backgroundSyncProvider).sync(), + icon: const Icon(Icons.sync), + ), if (showUploadButton) Padding( padding: const EdgeInsets.only(right: 20), diff --git a/mobile/test/api.mocks.dart b/mobile/test/api.mocks.dart index d502ea0675..b0a4e9b8fd 100644 --- a/mobile/test/api.mocks.dart +++ b/mobile/test/api.mocks.dart @@ -2,3 +2,5 @@ import 'package:mocktail/mocktail.dart'; import 'package:openapi/api.dart'; class MockAssetsApi extends Mock implements AssetsApi {} + +class MockSyncApi extends Mock implements SyncApi {} diff --git a/mobile/test/domain/services/sync_stream_service_test.dart b/mobile/test/domain/services/sync_stream_service_test.dart index e1d8e6987f..b78a44342b 100644 --- a/mobile/test/domain/services/sync_stream_service_test.dart +++ b/mobile/test/domain/services/sync_stream_service_test.dart @@ -1,4 +1,4 @@ -// ignore_for_file: avoid-unnecessary-futures, avoid-async-call-in-sync-function +// ignore_for_file: avoid-declaring-call-method, avoid-unnecessary-futures import 'dart:async'; @@ -8,16 +8,22 @@ import 'package:immich_mobile/domain/interfaces/sync_stream.interface.dart'; import 'package:immich_mobile/domain/models/sync_event.model.dart'; import 'package:immich_mobile/domain/services/sync_stream.service.dart'; import 'package:mocktail/mocktail.dart'; -import 'package:openapi/api.dart'; -import 'package:worker_manager/worker_manager.dart'; import '../../fixtures/sync_stream.stub.dart'; import '../../infrastructure/repository.mock.dart'; +class _AbortCallbackWrapper { + const _AbortCallbackWrapper(); + + bool call() => false; +} + +class _MockAbortCallbackWrapper extends Mock implements _AbortCallbackWrapper {} + class _CancellationWrapper { const _CancellationWrapper(); - bool isCancelled() => false; + bool call() => false; } class _MockCancellationWrapper extends Mock implements _CancellationWrapper {} @@ -26,35 +32,26 @@ void main() { late SyncStreamService sut; late ISyncStreamRepository mockSyncStreamRepo; late ISyncApiRepository mockSyncApiRepo; - late StreamController> streamController; + late Function(List, Function()) handleEventsCallback; + late _MockAbortCallbackWrapper mockAbortCallbackWrapper; successHandler(Invocation _) async => true; - failureHandler(Invocation _) async => false; setUp(() { mockSyncStreamRepo = MockSyncStreamRepository(); mockSyncApiRepo = MockSyncApiRepository(); - streamController = StreamController>.broadcast(); + mockAbortCallbackWrapper = _MockAbortCallbackWrapper(); - sut = SyncStreamService( - syncApiRepository: mockSyncApiRepo, - syncStreamRepository: mockSyncStreamRepo, - ); + when(() => mockAbortCallbackWrapper()).thenReturn(false); - // Default stream setup - emits one batch and closes - when(() => mockSyncApiRepo.getSyncEvents(any())) - .thenAnswer((_) => streamController.stream); + when(() => mockSyncApiRepo.streamChanges(any())) + .thenAnswer((invocation) async { + // ignore: avoid-unsafe-collection-methods + handleEventsCallback = invocation.positionalArguments.first; + }); - // Default ack setup when(() => mockSyncApiRepo.ack(any())).thenAnswer((_) async => {}); - // Register fallbacks for mocktail verification - registerFallbackValue([]); - registerFallbackValue([]); - registerFallbackValue([]); - registerFallbackValue([]); - - // Default successful repository calls when(() => mockSyncStreamRepo.updateUsersV1(any())) .thenAnswer(successHandler); when(() => mockSyncStreamRepo.deleteUsersV1(any())) @@ -63,381 +60,163 @@ void main() { .thenAnswer(successHandler); when(() => mockSyncStreamRepo.deletePartnerV1(any())) .thenAnswer(successHandler); + when(() => mockSyncStreamRepo.updateAssetsV1(any())) + .thenAnswer(successHandler); + when(() => mockSyncStreamRepo.deleteAssetsV1(any())) + .thenAnswer(successHandler); + when(() => mockSyncStreamRepo.updateAssetsExifV1(any())) + .thenAnswer(successHandler); + when(() => mockSyncStreamRepo.updatePartnerAssetsV1(any())) + .thenAnswer(successHandler); + when(() => mockSyncStreamRepo.deletePartnerAssetsV1(any())) + .thenAnswer(successHandler); + when(() => mockSyncStreamRepo.updatePartnerAssetsExifV1(any())) + .thenAnswer(successHandler); + + sut = SyncStreamService( + syncApiRepository: mockSyncApiRepo, + syncStreamRepository: mockSyncStreamRepo, + ); }); - tearDown(() async { - if (!streamController.isClosed) { - await streamController.close(); - } - }); - - // Helper to trigger sync and add events to the stream - Future triggerSyncAndEmit(List events) async { - final future = sut.syncUsers(); // Start listening - await Future.delayed(Duration.zero); // Allow listener to attach - if (!streamController.isClosed) { - streamController.add(events); - await streamController.close(); // Close after emitting - } - await future; // Wait for processing to complete + Future simulateEvents(List events) async { + await sut.sync(); + await handleEventsCallback(events, mockAbortCallbackWrapper.call); } - group("SyncStreamService", () { + group("SyncStreamService - _handleEvents", () { test( - "completes successfully when stream emits data and handlers succeed", + "processes events and acks successfully when handlers succeed", () async { final events = [ - ...SyncStreamStub.userEvents, - ...SyncStreamStub.partnerEvents, + SyncStreamStub.userDeleteV1, + SyncStreamStub.userV1Admin, + SyncStreamStub.userV1User, + SyncStreamStub.partnerDeleteV1, + SyncStreamStub.partnerV1, ]; - final future = triggerSyncAndEmit(events); - await expectLater(future, completes); - // Verify ack includes last ack from each successfully handled type - verify( - () => - mockSyncApiRepo.ack(any(that: containsAll(["5", "2", "4", "3"]))), - ).called(1); + + await simulateEvents(events); + + verifyInOrder([ + () => mockSyncStreamRepo.deleteUsersV1(any()), + () => mockSyncApiRepo.ack(["2"]), + () => mockSyncStreamRepo.updateUsersV1(any()), + () => mockSyncApiRepo.ack(["5"]), + () => mockSyncStreamRepo.deletePartnerV1(any()), + () => mockSyncApiRepo.ack(["4"]), + () => mockSyncStreamRepo.updatePartnerV1(any()), + () => mockSyncApiRepo.ack(["3"]), + ]); + verifyNever(() => mockAbortCallbackWrapper()); }, ); - test("completes successfully when stream emits an error", () async { - when(() => mockSyncApiRepo.getSyncEvents(any())) - .thenAnswer((_) => Stream.error(Exception("Stream Error"))); - // Should complete gracefully without throwing - await expectLater(sut.syncUsers(), throwsException); - verifyNever(() => mockSyncApiRepo.ack(any())); // No ack on stream error - }); - - test("throws when initial getSyncEvents call fails", () async { - final apiException = Exception("API Error"); - when(() => mockSyncApiRepo.getSyncEvents(any())).thenThrow(apiException); - // Should rethrow the exception from the initial call - await expectLater(sut.syncUsers(), throwsA(apiException)); - verifyNever(() => mockSyncApiRepo.ack(any())); - }); - - test( - "completes successfully when a repository handler throws an exception", - () async { - when(() => mockSyncStreamRepo.updateUsersV1(any())) - .thenThrow(Exception("Repo Error")); - final events = [ - ...SyncStreamStub.userEvents, - ...SyncStreamStub.partnerEvents, - ]; - // Should complete, but ack only for the successful types - await triggerSyncAndEmit(events); - // Only partner delete was successful by default setup - verify(() => mockSyncApiRepo.ack(["2", "4", "3"])).called(1); - }, - ); - - test( - "completes successfully but sends no ack when all handlers fail", - () async { - when(() => mockSyncStreamRepo.updateUsersV1(any())) - .thenAnswer(failureHandler); - when(() => mockSyncStreamRepo.deleteUsersV1(any())) - .thenAnswer(failureHandler); - when(() => mockSyncStreamRepo.updatePartnerV1(any())) - .thenAnswer(failureHandler); - when(() => mockSyncStreamRepo.deletePartnerV1(any())) - .thenAnswer(failureHandler); - - final events = [ - ...SyncStreamStub.userEvents, - ...SyncStreamStub.partnerEvents, - ]; - await triggerSyncAndEmit(events); - verifyNever(() => mockSyncApiRepo.ack(any())); - }, - ); - - test("sends ack only for types where handler returns true", () async { - // Mock specific handlers: user update fails, user delete succeeds - when(() => mockSyncStreamRepo.updateUsersV1(any())) - .thenAnswer(failureHandler); - when(() => mockSyncStreamRepo.deleteUsersV1(any())) - .thenAnswer(successHandler); - // partner update fails, partner delete succeeds - when(() => mockSyncStreamRepo.updatePartnerV1(any())) - .thenAnswer(failureHandler); - + test("processes final batch correctly", () async { final events = [ - ...SyncStreamStub.userEvents, - ...SyncStreamStub.partnerEvents, + SyncStreamStub.userDeleteV1, + SyncStreamStub.userV1Admin, ]; - await triggerSyncAndEmit(events); - // Expect ack only for userDeleteV1 (ack: "2") and partnerDeleteV1 (ack: "4") - verify(() => mockSyncApiRepo.ack(any(that: containsAll(["2", "4"])))) - .called(1); + await simulateEvents(events); + + verifyInOrder([ + () => mockSyncStreamRepo.deleteUsersV1(any()), + () => mockSyncApiRepo.ack(["2"]), + () => mockSyncStreamRepo.updateUsersV1(any()), + () => mockSyncApiRepo.ack(["1"]), + ]); + verifyNever(() => mockAbortCallbackWrapper()); }); - test("does not process or ack when stream emits an empty list", () async { - final future = sut.syncUsers(); - streamController.add([]); // Emit empty list - await streamController.close(); - await future; // Wait for completion + test("does not process or ack when event list is empty", () async { + await simulateEvents([]); verifyNever(() => mockSyncStreamRepo.updateUsersV1(any())); verifyNever(() => mockSyncStreamRepo.deleteUsersV1(any())); verifyNever(() => mockSyncStreamRepo.updatePartnerV1(any())); verifyNever(() => mockSyncStreamRepo.deletePartnerV1(any())); + verifyNever(() => mockAbortCallbackWrapper()); verifyNever(() => mockSyncApiRepo.ack(any())); }); - test("processes multiple batches sequentially using mutex", () async { - final completer1 = Completer(); - final completer2 = Completer(); - int callOrder = 0; - int handler1StartOrder = -1; - int handler2StartOrder = -1; - int handler1Calls = 0; - int handler2Calls = 0; + test("aborts and stops processing if cancelled during iteration", () async { + final cancellationChecker = _MockCancellationWrapper(); + when(() => cancellationChecker()).thenReturn(false); - when(() => mockSyncStreamRepo.updateUsersV1(any())).thenAnswer((_) async { - handler1Calls++; - handler1StartOrder = ++callOrder; - await completer1.future; - return true; - }); - when(() => mockSyncStreamRepo.updatePartnerV1(any())) - .thenAnswer((_) async { - handler2Calls++; - handler2StartOrder = ++callOrder; - await completer2.future; - return true; + sut = SyncStreamService( + syncApiRepository: mockSyncApiRepo, + syncStreamRepository: mockSyncStreamRepo, + cancelChecker: cancellationChecker.call, + ); + await sut.sync(); + + final events = [ + SyncStreamStub.userDeleteV1, + SyncStreamStub.userV1Admin, + SyncStreamStub.partnerDeleteV1, + ]; + + when(() => mockSyncStreamRepo.deleteUsersV1(any())).thenAnswer((_) async { + when(() => cancellationChecker()).thenReturn(true); }); - final batch1 = SyncStreamStub.userEvents; - final batch2 = SyncStreamStub.partnerEvents; + await handleEventsCallback(events, mockAbortCallbackWrapper.call); - final syncFuture = sut.syncUsers(); - await pumpEventQueue(); + verify(() => mockSyncStreamRepo.deleteUsersV1(any())).called(1); + verifyNever(() => mockSyncStreamRepo.updateUsersV1(any())); + verifyNever(() => mockSyncStreamRepo.deletePartnerV1(any())); - streamController.add(batch1); - await pumpEventQueue(); - // Small delay to ensure the first handler starts - await Future.delayed(const Duration(milliseconds: 20)); + verify(() => mockAbortCallbackWrapper()).called(1); - expect(handler1StartOrder, 1, reason: "Handler 1 should start first"); - expect(handler1Calls, 1); - - streamController.add(batch2); - await pumpEventQueue(); - // Small delay - await Future.delayed(const Duration(milliseconds: 20)); - - expect(handler2StartOrder, -1, reason: "Handler 2 should wait"); - expect(handler2Calls, 0); - - completer1.complete(); - await pumpEventQueue(times: 40); - // Small delay to ensure the second handler starts - await Future.delayed(const Duration(milliseconds: 20)); - - expect(handler2StartOrder, 2, reason: "Handler 2 should start after H1"); - expect(handler2Calls, 1); - - completer2.complete(); - await pumpEventQueue(times: 40); - // Small delay before closing the stream - await Future.delayed(const Duration(milliseconds: 20)); - - if (!streamController.isClosed) { - await streamController.close(); - } - await pumpEventQueue(times: 40); - // Small delay to ensure the sync completes - await Future.delayed(const Duration(milliseconds: 20)); - - await syncFuture; - - verify(() => mockSyncStreamRepo.updateUsersV1(any())).called(1); - verify(() => mockSyncStreamRepo.updatePartnerV1(any())).called(1); - verify(() => mockSyncApiRepo.ack(any())).called(2); + verify(() => mockSyncApiRepo.ack(["2"])).called(1); }); test( - "stops processing and ack when cancel checker is completed", + "aborts and stops processing if cancelled before processing batch", () async { final cancellationChecker = _MockCancellationWrapper(); - when(() => cancellationChecker.isCancelled()).thenAnswer((_) => false); + when(() => cancellationChecker()).thenReturn(false); + + final processingCompleter = Completer(); + bool handler1Started = false; + when(() => mockSyncStreamRepo.deleteUsersV1(any())) + .thenAnswer((_) async { + handler1Started = true; + return processingCompleter.future; + }); sut = SyncStreamService( syncApiRepository: mockSyncApiRepo, syncStreamRepository: mockSyncStreamRepo, - cancelChecker: cancellationChecker.isCancelled, + cancelChecker: cancellationChecker.call, ); - final processingCompleter = Completer(); - bool handlerStarted = false; + await sut.sync(); - // Make handler wait so we can cancel it mid-flight - when(() => mockSyncStreamRepo.deleteUsersV1(any())) - .thenAnswer((_) async { - handlerStarted = true; - await processingCompleter - .future; // Wait indefinitely until test completes it - return true; - }); - - final syncFuture = sut.syncUsers(); - await pumpEventQueue(times: 30); - - streamController.add(SyncStreamStub.userEvents); - // Ensure processing starts - await Future.delayed(const Duration(milliseconds: 10)); - - expect(handlerStarted, isTrue, reason: "Handler should have started"); - - when(() => cancellationChecker.isCancelled()).thenAnswer((_) => true); - - // Allow cancellation logic to propagate - await Future.delayed(const Duration(milliseconds: 10)); - - // Complete the handler's completer after cancellation signal - // to ensure the cancellation logic itself isn't blocked by the handler. - processingCompleter.complete(); - - await expectLater(syncFuture, throwsA(isA())); - - // Verify that ack was NOT called because processing was cancelled - verifyNever(() => mockSyncApiRepo.ack(any())); - }, - ); - - test("completes successfully when ack call throws an exception", () async { - when(() => mockSyncApiRepo.ack(any())).thenThrow(Exception("Ack Error")); - final events = [ - ...SyncStreamStub.userEvents, - ...SyncStreamStub.partnerEvents, - ]; - - // Should still complete even if ack fails - await triggerSyncAndEmit(events); - verify(() => mockSyncApiRepo.ack(any())) - .called(1); // Verify ack was attempted - }); - - test("waits for processing to finish if onDone called early", () async { - final processingCompleter = Completer(); - bool handlerFinished = false; - - when(() => mockSyncStreamRepo.updateUsersV1(any())).thenAnswer((_) async { - await processingCompleter.future; // Wait inside handler - handlerFinished = true; - return true; - }); - - final syncFuture = sut.syncUsers(); - // Allow listener to attach - // This is necessary to ensure the stream is ready to receive events - await Future.delayed(Duration.zero); - - streamController.add(SyncStreamStub.userEvents); // Emit batch - await Future.delayed( - const Duration(milliseconds: 10), - ); // Ensure processing starts - - await streamController - .close(); // Close stream (triggers onDone internally) - await Future.delayed( - const Duration(milliseconds: 10), - ); // Give onDone a chance to fire - - // At this point, onDone was called, but processing is blocked - expect(handlerFinished, isFalse); - - processingCompleter.complete(); // Allow processing to finish - await syncFuture; // Now the main future should complete - - expect(handlerFinished, isTrue); - verify(() => mockSyncApiRepo.ack(any())).called(1); - }); - - test("processes events in the defined _kSyncTypeOrder", () async { - final future = sut.syncUsers(); - await pumpEventQueue(); - if (!streamController.isClosed) { final events = [ - SyncEvent( - type: SyncEntityType.partnerV1, - data: SyncStreamStub.partnerV1, - ack: "1", - ), // Should be processed last - SyncEvent( - type: SyncEntityType.userV1, - data: SyncStreamStub.userV1Admin, - ack: "2", - ), // Should be processed second - SyncEvent( - type: SyncEntityType.partnerDeleteV1, - data: SyncStreamStub.partnerDeleteV1, - ack: "3", - ), // Should be processed third - SyncEvent( - type: SyncEntityType.userDeleteV1, - data: SyncStreamStub.userDeleteV1, - ack: "4", - ), // Should be processed first + SyncStreamStub.userDeleteV1, + SyncStreamStub.userV1Admin, + SyncStreamStub.partnerDeleteV1, ]; - streamController.add(events); - await streamController.close(); - } - await future; + final processingFuture = + handleEventsCallback(events, mockAbortCallbackWrapper.call); + await pumpEventQueue(); - verifyInOrder([ - () => mockSyncStreamRepo.deleteUsersV1(any()), - () => mockSyncStreamRepo.updateUsersV1(any()), - () => mockSyncStreamRepo.deletePartnerV1(any()), - () => mockSyncStreamRepo.updatePartnerV1(any()), - // Verify ack happens after all processing - () => mockSyncApiRepo.ack(any()), - ]); - }); - }); + expect(handler1Started, isTrue); - group("syncUsers", () { - test("calls getSyncEvents with correct types", () async { - // Need to close the stream for the future to complete - final future = sut.syncUsers(); - await streamController.close(); - await future; + // Signal cancellation while handler 1 is waiting + when(() => cancellationChecker()).thenReturn(true); + await pumpEventQueue(); - verify( - () => mockSyncApiRepo.getSyncEvents([ - SyncRequestType.usersV1, - SyncRequestType.partnersV1, - ]), - ).called(1); - }); + processingCompleter.complete(); + await processingFuture; - test("calls repository methods with correctly grouped data", () async { - final events = [ - ...SyncStreamStub.userEvents, - ...SyncStreamStub.partnerEvents, - ]; - await triggerSyncAndEmit(events); + verifyNever(() => mockSyncStreamRepo.updateUsersV1(any())); - // Verify each handler was called with the correct list of data payloads - verify( - () => mockSyncStreamRepo.updateUsersV1( - [SyncStreamStub.userV1Admin, SyncStreamStub.userV1User], - ), - ).called(1); - verify( - () => mockSyncStreamRepo.deleteUsersV1([SyncStreamStub.userDeleteV1]), - ).called(1); - verify( - () => mockSyncStreamRepo.updatePartnerV1([SyncStreamStub.partnerV1]), - ).called(1); - verify( - () => mockSyncStreamRepo - .deletePartnerV1([SyncStreamStub.partnerDeleteV1]), - ).called(1); - }); + verify(() => mockSyncApiRepo.ack(["2"])).called(1); + }, + ); }); } diff --git a/mobile/test/fixtures/sync_stream.stub.dart b/mobile/test/fixtures/sync_stream.stub.dart index 781e63a2bb..ba97f1434a 100644 --- a/mobile/test/fixtures/sync_stream.stub.dart +++ b/mobile/test/fixtures/sync_stream.stub.dart @@ -2,44 +2,44 @@ import 'package:immich_mobile/domain/models/sync_event.model.dart'; import 'package:openapi/api.dart'; abstract final class SyncStreamStub { - static final userV1Admin = SyncUserV1( - deletedAt: DateTime(2020), - email: "admin@admin", - id: "1", - name: "Admin", - ); - static final userV1User = SyncUserV1( - deletedAt: DateTime(2021), - email: "user@user", - id: "2", - name: "User", - ); - static final userDeleteV1 = SyncUserDeleteV1(userId: "2"); - static final userEvents = [ - SyncEvent(type: SyncEntityType.userV1, data: userV1Admin, ack: "1"), - SyncEvent( - type: SyncEntityType.userDeleteV1, - data: userDeleteV1, - ack: "2", + static final userV1Admin = SyncEvent( + type: SyncEntityType.userV1, + data: SyncUserV1( + deletedAt: DateTime(2020), + email: "admin@admin", + id: "1", + name: "Admin", ), - SyncEvent(type: SyncEntityType.userV1, data: userV1User, ack: "5"), - ]; + ack: "1", + ); + static final userV1User = SyncEvent( + type: SyncEntityType.userV1, + data: SyncUserV1( + deletedAt: DateTime(2021), + email: "user@user", + id: "5", + name: "User", + ), + ack: "5", + ); + static final userDeleteV1 = SyncEvent( + type: SyncEntityType.userDeleteV1, + data: SyncUserDeleteV1(userId: "2"), + ack: "2", + ); - static final partnerV1 = SyncPartnerV1( - inTimeline: true, - sharedById: "1", - sharedWithId: "2", - ); - static final partnerDeleteV1 = SyncPartnerDeleteV1( - sharedById: "3", - sharedWithId: "4", - ); - static final partnerEvents = [ - SyncEvent( - type: SyncEntityType.partnerDeleteV1, - data: partnerDeleteV1, - ack: "4", + static final partnerV1 = SyncEvent( + type: SyncEntityType.partnerV1, + data: SyncPartnerV1( + inTimeline: true, + sharedById: "1", + sharedWithId: "2", ), - SyncEvent(type: SyncEntityType.partnerV1, data: partnerV1, ack: "3"), - ]; + ack: "3", + ); + static final partnerDeleteV1 = SyncEvent( + type: SyncEntityType.partnerDeleteV1, + data: SyncPartnerDeleteV1(sharedById: "3", sharedWithId: "4"), + ack: "4", + ); } diff --git a/mobile/test/infrastructure/repositories/sync_api_repository_test.dart b/mobile/test/infrastructure/repositories/sync_api_repository_test.dart new file mode 100644 index 0000000000..55b03a8116 --- /dev/null +++ b/mobile/test/infrastructure/repositories/sync_api_repository_test.dart @@ -0,0 +1,299 @@ +import 'dart:async'; +import 'dart:convert'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:http/http.dart' as http; +import 'package:immich_mobile/domain/models/sync_event.model.dart'; +import 'package:immich_mobile/infrastructure/repositories/sync_api.repository.dart'; +import 'package:mocktail/mocktail.dart'; +import 'package:openapi/api.dart'; + +import '../../api.mocks.dart'; +import '../../service.mocks.dart'; + +class MockHttpClient extends Mock implements http.Client {} + +class MockApiClient extends Mock implements ApiClient {} + +class MockStreamedResponse extends Mock implements http.StreamedResponse {} + +class FakeBaseRequest extends Fake implements http.BaseRequest {} + +String _createJsonLine(String type, Map data, String ack) { + return '${jsonEncode({'type': type, 'data': data, 'ack': ack})}\n'; +} + +void main() { + late SyncApiRepository sut; + late MockApiService mockApiService; + late MockApiClient mockApiClient; + late MockSyncApi mockSyncApi; + late MockHttpClient mockHttpClient; + late MockStreamedResponse mockStreamedResponse; + late StreamController> responseStreamController; + late int testBatchSize = 3; + + setUp(() { + mockApiService = MockApiService(); + mockApiClient = MockApiClient(); + mockSyncApi = MockSyncApi(); + mockHttpClient = MockHttpClient(); + mockStreamedResponse = MockStreamedResponse(); + responseStreamController = + StreamController>.broadcast(sync: true); + + registerFallbackValue(FakeBaseRequest()); + + when(() => mockApiService.apiClient).thenReturn(mockApiClient); + when(() => mockApiService.syncApi).thenReturn(mockSyncApi); + when(() => mockApiClient.basePath).thenReturn('http://demo.immich.app/api'); + when(() => mockApiService.applyToParams(any(), any())) + .thenAnswer((_) async => {}); + + // Mock HTTP client behavior + when(() => mockHttpClient.send(any())) + .thenAnswer((_) async => mockStreamedResponse); + when(() => mockStreamedResponse.statusCode).thenReturn(200); + when(() => mockStreamedResponse.stream) + .thenAnswer((_) => http.ByteStream(responseStreamController.stream)); + when(() => mockHttpClient.close()).thenAnswer((_) => {}); + + sut = SyncApiRepository(mockApiService); + }); + + tearDown(() async { + if (!responseStreamController.isClosed) { + await responseStreamController.close(); + } + }); + + Future streamChanges( + Function(List, Function() abort) onDataCallback, + ) { + return sut.streamChanges( + onDataCallback, + batchSize: testBatchSize, + httpClient: mockHttpClient, + ); + } + + test('streamChanges stops processing stream when abort is called', () async { + int onDataCallCount = 0; + bool abortWasCalledInCallback = false; + List receivedEventsBatch1 = []; + + onDataCallback(List events, Function() abort) { + onDataCallCount++; + if (onDataCallCount == 1) { + receivedEventsBatch1 = events; + abort(); + abortWasCalledInCallback = true; + } else { + fail("onData called more than once after abort was invoked"); + } + } + + final streamChangesFuture = streamChanges(onDataCallback); + + await pumpEventQueue(); + + for (int i = 0; i < testBatchSize; i++) { + responseStreamController.add( + utf8.encode( + _createJsonLine( + SyncEntityType.userDeleteV1.toString(), + SyncUserDeleteV1(userId: "user$i").toJson(), + 'ack$i', + ), + ), + ); + } + + for (int i = testBatchSize; i < testBatchSize * 2; i++) { + responseStreamController.add( + utf8.encode( + _createJsonLine( + SyncEntityType.userDeleteV1.toString(), + SyncUserDeleteV1(userId: "user$i").toJson(), + 'ack$i', + ), + ), + ); + } + + await responseStreamController.close(); + await expectLater(streamChangesFuture, completes); + + expect(onDataCallCount, 1); + expect(abortWasCalledInCallback, isTrue); + expect(receivedEventsBatch1.length, testBatchSize); + verify(() => mockHttpClient.close()).called(1); + }); + + test( + 'streamChanges does not process remaining lines in finally block if aborted', + () async { + int onDataCallCount = 0; + bool abortWasCalledInCallback = false; + + onDataCallback(List events, Function() abort) { + onDataCallCount++; + if (onDataCallCount == 1) { + abort(); + abortWasCalledInCallback = true; + } else { + fail("onData called more than once after abort was invoked"); + } + } + + final streamChangesFuture = streamChanges(onDataCallback); + + await pumpEventQueue(); + + for (int i = 0; i < testBatchSize; i++) { + responseStreamController.add( + utf8.encode( + _createJsonLine( + SyncEntityType.userDeleteV1.toString(), + SyncUserDeleteV1(userId: "user$i").toJson(), + 'ack$i', + ), + ), + ); + } + + // emit a single event to skip batching and trigger finally + responseStreamController.add( + utf8.encode( + _createJsonLine( + SyncEntityType.userDeleteV1.toString(), + SyncUserDeleteV1(userId: "user100").toJson(), + 'ack100', + ), + ), + ); + + await responseStreamController.close(); + await expectLater(streamChangesFuture, completes); + + expect(onDataCallCount, 1); + expect(abortWasCalledInCallback, isTrue); + verify(() => mockHttpClient.close()).called(1); + }, + ); + + test( + 'streamChanges processes remaining lines in finally block if not aborted', + () async { + int onDataCallCount = 0; + List receivedEventsBatch1 = []; + List receivedEventsBatch2 = []; + + onDataCallback(List events, Function() _) { + onDataCallCount++; + if (onDataCallCount == 1) { + receivedEventsBatch1 = events; + } else if (onDataCallCount == 2) { + receivedEventsBatch2 = events; + } else { + fail("onData called more than expected"); + } + } + + final streamChangesFuture = streamChanges(onDataCallback); + + await pumpEventQueue(); + + // Batch 1 + for (int i = 0; i < testBatchSize; i++) { + responseStreamController.add( + utf8.encode( + _createJsonLine( + SyncEntityType.userDeleteV1.toString(), + SyncUserDeleteV1(userId: "user$i").toJson(), + 'ack$i', + ), + ), + ); + } + + // Partial Batch 2 + responseStreamController.add( + utf8.encode( + _createJsonLine( + SyncEntityType.userDeleteV1.toString(), + SyncUserDeleteV1(userId: "user100").toJson(), + 'ack100', + ), + ), + ); + + await responseStreamController.close(); + await expectLater(streamChangesFuture, completes); + + expect(onDataCallCount, 2); + expect(receivedEventsBatch1.length, testBatchSize); + expect(receivedEventsBatch2.length, 1); + verify(() => mockHttpClient.close()).called(1); + }, + ); + + test('streamChanges handles stream error gracefully', () async { + final streamError = Exception("Network Error"); + int onDataCallCount = 0; + + onDataCallback(List events, Function() _) { + onDataCallCount++; + } + + final streamChangesFuture = streamChanges(onDataCallback); + + await pumpEventQueue(); + + responseStreamController.add( + utf8.encode( + _createJsonLine( + SyncEntityType.userDeleteV1.toString(), + SyncUserDeleteV1(userId: "user1").toJson(), + 'ack1', + ), + ), + ); + + responseStreamController.addError(streamError); + await expectLater(streamChangesFuture, throwsA(streamError)); + + expect(onDataCallCount, 0); + verify(() => mockHttpClient.close()).called(1); + }); + + test('streamChanges throws ApiException on non-200 status code', () async { + when(() => mockStreamedResponse.statusCode).thenReturn(401); + final errorBodyController = StreamController>(sync: true); + when(() => mockStreamedResponse.stream) + .thenAnswer((_) => http.ByteStream(errorBodyController.stream)); + + int onDataCallCount = 0; + + onDataCallback(List events, Function() _) { + onDataCallCount++; + } + + final future = streamChanges(onDataCallback); + + errorBodyController.add(utf8.encode('{"error":"Unauthorized"}')); + await errorBodyController.close(); + + await expectLater( + future, + throwsA( + isA() + .having((e) => e.code, 'code', 401) + .having((e) => e.message, 'message', contains('Unauthorized')), + ), + ); + + expect(onDataCallCount, 0); + verify(() => mockHttpClient.close()).called(1); + }); +} From 504930947d62194ab4cf2e9c219d9d0bc5f4fde0 Mon Sep 17 00:00:00 2001 From: bo0tzz Date: Fri, 18 Apr 2025 22:10:27 +0200 Subject: [PATCH 09/12] fix: various actions workflow security improvements (#17651) * fix: set persist-credentials explicitly for checkout https://woodruffw.github.io/zizmor/audits/#artipacked * fix: minimize permissions scope for workflows https://woodruffw.github.io/zizmor/audits/#excessive-permissions * fix: remove potential template injections https://woodruffw.github.io/zizmor/audits/#template-injection * fix: only pass needed secrets in workflow_call https://woodruffw.github.io/zizmor/audits/#secrets-inherit * fix: push perm for single-arch build jobs I hadn't realised these push to the registry too :x * chore: fix formatting * fix: $ * fix: retag job quoting --------- Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- .github/workflows/build-mobile.yml | 29 +++++-- .github/workflows/cache-cleanup.yml | 17 ++-- .github/workflows/cli.yml | 13 ++- .github/workflows/codeql-analysis.yml | 4 + .github/workflows/docker.yml | 85 +++++++++++++------ .github/workflows/docs-build.yml | 10 +++ .github/workflows/docs-deploy.yml | 20 ++++- .github/workflows/docs-destroy.yml | 7 ++ .github/workflows/fix-format.yml | 4 + .github/workflows/pr-label-validation.yml | 2 + .github/workflows/pr-labeler.yml | 2 + .../pr-require-conventional-commit.yml | 4 + .github/workflows/prepare-release.yml | 18 +++- .github/workflows/preview-label.yaml | 2 + .github/workflows/sdk.yml | 8 +- .github/workflows/static_analysis.yml | 16 +++- .github/workflows/test.yml | 78 ++++++++++++++++- .github/workflows/weblate-lock.yml | 13 +-- 18 files changed, 269 insertions(+), 63 deletions(-) diff --git a/.github/workflows/build-mobile.yml b/.github/workflows/build-mobile.yml index 406e8f89e1..7217b5267e 100644 --- a/.github/workflows/build-mobile.yml +++ b/.github/workflows/build-mobile.yml @@ -7,6 +7,15 @@ on: ref: required: false type: string + secrets: + KEY_JKS: + required: true + ALIAS: + required: true + ANDROID_KEY_PASSWORD: + required: true + ANDROID_STORE_PASSWORD: + required: true pull_request: push: branches: [main] @@ -15,14 +24,21 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: {} + jobs: pre-job: runs-on: ubuntu-latest + permissions: + contents: read outputs: should_run: ${{ steps.found_paths.outputs.mobile == 'true' || steps.should_force.outputs.should_force == 'true' }} steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false + - id: found_paths uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 with: @@ -38,22 +54,17 @@ jobs: build-sign-android: name: Build and sign Android needs: pre-job + permissions: + contents: read # Skip when PR from a fork if: ${{ !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]' && needs.pre-job.outputs.should_run == 'true' }} runs-on: macos-14 steps: - - name: Determine ref - id: get-ref - run: | - input_ref="${{ inputs.ref }}" - github_ref="${{ github.sha }}" - ref="${input_ref:-$github_ref}" - echo "ref=$ref" >> $GITHUB_OUTPUT - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: - ref: ${{ steps.get-ref.outputs.ref }} + ref: ${{ inputs.ref || github.sha }} + persist-credentials: false - uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4 with: diff --git a/.github/workflows/cache-cleanup.yml b/.github/workflows/cache-cleanup.yml index 0cc73c46c3..84adde08cf 100644 --- a/.github/workflows/cache-cleanup.yml +++ b/.github/workflows/cache-cleanup.yml @@ -8,31 +8,38 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: {} + jobs: cleanup: name: Cleanup runs-on: ubuntu-latest + permissions: + contents: read + actions: write steps: - name: Check out code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Cleanup + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REF: ${{ github.ref }} run: | gh extension install actions/gh-actions-cache REPO=${{ github.repository }} - BRANCH=${{ github.ref }} echo "Fetching list of cache keys" - cacheKeysForPR=$(gh actions-cache list -R $REPO -B $BRANCH -L 100 | cut -f 1 ) + cacheKeysForPR=$(gh actions-cache list -R $REPO -B ${REF} -L 100 | cut -f 1 ) ## Setting this to not fail the workflow while deleting cache keys. set +e echo "Deleting caches..." for cacheKey in $cacheKeysForPR do - gh actions-cache delete $cacheKey -R $REPO -B $BRANCH --confirm + gh actions-cache delete $cacheKey -R "$REPO" -B "${REF}" --confirm done echo "Done" - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.yml index 05326d063e..231ad141e4 100644 --- a/.github/workflows/cli.yml +++ b/.github/workflows/cli.yml @@ -16,19 +16,23 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true -permissions: - packages: write +permissions: {} jobs: publish: name: CLI Publish runs-on: ubuntu-latest + permissions: + contents: read defaults: run: working-directory: ./cli steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false + # Setup .npmrc file to publish to npm - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: @@ -48,11 +52,16 @@ jobs: docker: name: Docker runs-on: ubuntu-latest + permissions: + contents: read + packages: write needs: publish steps: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Set up QEMU uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 99ffee2e88..7d07785de7 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -24,6 +24,8 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: {} + jobs: analyze: name: Analyze @@ -43,6 +45,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d1bdb5e8e7..a78d3c25dc 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -12,18 +12,21 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true -permissions: - packages: write +permissions: {} jobs: pre-job: runs-on: ubuntu-latest + permissions: + contents: read outputs: should_run_server: ${{ steps.found_paths.outputs.server == 'true' || steps.should_force.outputs.should_force == 'true' }} should_run_ml: ${{ steps.found_paths.outputs.machine-learning == 'true' || steps.should_force.outputs.should_force == 'true' }} steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - id: found_paths uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 with: @@ -45,6 +48,9 @@ jobs: retag_ml: name: Re-Tag ML needs: pre-job + permissions: + contents: read + packages: write if: ${{ needs.pre-job.outputs.should_run_ml == 'false' && !github.event.pull_request.head.repo.fork }} runs-on: ubuntu-latest strategy: @@ -58,18 +64,22 @@ jobs: username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Re-tag image + env: + REGISTRY_NAME: 'ghcr.io' + REPOSITORY: ${{ github.repository_owner }}/immich-machine-learning + TAG_OLD: main${{ matrix.suffix }} + TAG_PR: ${{ github.event.number == 0 && github.ref_name || format('pr-{0}', github.event.number) }}${{ matrix.suffix }} + TAG_COMMIT: commit-${{ github.event_name != 'pull_request' && github.sha || github.event.pull_request.head.sha }}${{ matrix.suffix }} run: | - REGISTRY_NAME="ghcr.io" - REPOSITORY=${{ github.repository_owner }}/immich-machine-learning - TAG_OLD=main${{ matrix.suffix }} - TAG_PR=${{ github.event.number == 0 && github.ref_name || format('pr-{0}', github.event.number) }}${{ matrix.suffix }} - TAG_COMMIT=commit-${{ github.event_name != 'pull_request' && github.sha || github.event.pull_request.head.sha }}${{ matrix.suffix }} - docker buildx imagetools create -t $REGISTRY_NAME/$REPOSITORY:$TAG_PR $REGISTRY_NAME/$REPOSITORY:$TAG_OLD - docker buildx imagetools create -t $REGISTRY_NAME/$REPOSITORY:$TAG_COMMIT $REGISTRY_NAME/$REPOSITORY:$TAG_OLD + docker buildx imagetools create -t "${REGISTRY_NAME}/${REPOSITORY}:${TAG_PR}" "${REGISTRY_NAME}/${REPOSITORY}:${TAG_OLD}" + docker buildx imagetools create -t "${REGISTRY_NAME}/${REPOSITORY}:${TAG_COMMIT}" "${REGISTRY_NAME}/${REPOSITORY}:${TAG_OLD}" retag_server: name: Re-Tag Server needs: pre-job + permissions: + contents: read + packages: write if: ${{ needs.pre-job.outputs.should_run_server == 'false' && !github.event.pull_request.head.repo.fork }} runs-on: ubuntu-latest strategy: @@ -83,18 +93,22 @@ jobs: username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Re-tag image + env: + REGISTRY_NAME: 'ghcr.io' + REPOSITORY: ${{ github.repository_owner }}/immich-server + TAG_OLD: main${{ matrix.suffix }} + TAG_PR: ${{ github.event.number == 0 && github.ref_name || format('pr-{0}', github.event.number) }}${{ matrix.suffix }} + TAG_COMMIT: commit-${{ github.event_name != 'pull_request' && github.sha || github.event.pull_request.head.sha }}${{ matrix.suffix }} run: | - REGISTRY_NAME="ghcr.io" - REPOSITORY=${{ github.repository_owner }}/immich-server - TAG_OLD=main${{ matrix.suffix }} - TAG_PR=${{ github.event.number == 0 && github.ref_name || format('pr-{0}', github.event.number) }}${{ matrix.suffix }} - TAG_COMMIT=commit-${{ github.event_name != 'pull_request' && github.sha || github.event.pull_request.head.sha }}${{ matrix.suffix }} - docker buildx imagetools create -t $REGISTRY_NAME/$REPOSITORY:$TAG_PR $REGISTRY_NAME/$REPOSITORY:$TAG_OLD - docker buildx imagetools create -t $REGISTRY_NAME/$REPOSITORY:$TAG_COMMIT $REGISTRY_NAME/$REPOSITORY:$TAG_OLD + docker buildx imagetools create -t "${REGISTRY_NAME}/${REPOSITORY}:${TAG_PR}" "${REGISTRY_NAME}/${REPOSITORY}:${TAG_OLD}" + docker buildx imagetools create -t "${REGISTRY_NAME}/${REPOSITORY}:${TAG_COMMIT}" "${REGISTRY_NAME}/${REPOSITORY}:${TAG_OLD}" build_and_push_ml: name: Build and Push ML needs: pre-job + permissions: + contents: read + packages: write if: ${{ needs.pre-job.outputs.should_run_ml == 'true' }} runs-on: ${{ matrix.runner }} env: @@ -148,6 +162,8 @@ jobs: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Set up Docker Buildx uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 @@ -161,11 +177,14 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Generate cache key suffix + env: + REF: ${{ github.ref_name }} run: | if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo "CACHE_KEY_SUFFIX=pr-${{ github.event.number }}" >> $GITHUB_ENV else - echo "CACHE_KEY_SUFFIX=$(echo ${{ github.ref_name }} | sed 's/[^a-zA-Z0-9]/-/g')" >> $GITHUB_ENV + SUFFIX=$(echo "${REF}" | sed 's/[^a-zA-Z0-9]/-/g') + echo "CACHE_KEY_SUFFIX=${SUFFIX}" >> $GITHUB_ENV fi - name: Generate cache target @@ -175,7 +194,7 @@ jobs: # Essentially just ignore the cache output (forks can't write to registry cache) echo "cache-to=type=local,dest=/tmp/discard,ignore-error=true" >> $GITHUB_OUTPUT else - echo "cache-to=type=registry,ref=${{ env.GHCR_REPO }}-build-cache:${{ env.PLATFORM_PAIR }}-${{ matrix.device }}-${{ env.CACHE_KEY_SUFFIX }},mode=max,compression=zstd" >> $GITHUB_OUTPUT + echo "cache-to=type=registry,ref=${GHCR_REPO}-build-cache:${PLATFORM_PAIR}-${{ matrix.device }}-${CACHE_KEY_SUFFIX},mode=max,compression=zstd" >> $GITHUB_OUTPUT fi - name: Generate docker image tags @@ -221,6 +240,10 @@ jobs: merge_ml: name: Merge & Push ML runs-on: ubuntu-latest + permissions: + contents: read + actions: read + packages: write if: ${{ needs.pre-job.outputs.should_run_ml == 'true' && !github.event.pull_request.head.repo.fork }} env: GHCR_REPO: ghcr.io/${{ github.repository_owner }}/immich-machine-learning @@ -308,15 +331,16 @@ jobs: fi TAGS=$(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - SOURCE_ARGS=$(printf '${{ env.GHCR_REPO }}@sha256:%s ' *) - - echo "docker buildx imagetools create $TAGS "${ANNOTATIONS[@]}" $SOURCE_ARGS" + SOURCE_ARGS=$(printf "${GHCR_REPO}@sha256:%s " *) docker buildx imagetools create $TAGS "${ANNOTATIONS[@]}" $SOURCE_ARGS build_and_push_server: name: Build and Push Server runs-on: ${{ matrix.runner }} + permissions: + contents: read + packages: write needs: pre-job if: ${{ needs.pre-job.outputs.should_run_server == 'true' }} env: @@ -340,6 +364,8 @@ jobs: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Set up Docker Buildx uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3 @@ -353,11 +379,14 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Generate cache key suffix + env: + REF: ${{ github.ref_name }} run: | if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo "CACHE_KEY_SUFFIX=pr-${{ github.event.number }}" >> $GITHUB_ENV else - echo "CACHE_KEY_SUFFIX=$(echo ${{ github.ref_name }} | sed 's/[^a-zA-Z0-9]/-/g')" >> $GITHUB_ENV + SUFFIX=$(echo "${REF}" | sed 's/[^a-zA-Z0-9]/-/g') + echo "CACHE_KEY_SUFFIX=${SUFFIX}" >> $GITHUB_ENV fi - name: Generate cache target @@ -367,7 +396,7 @@ jobs: # Essentially just ignore the cache output (forks can't write to registry cache) echo "cache-to=type=local,dest=/tmp/discard,ignore-error=true" >> $GITHUB_OUTPUT else - echo "cache-to=type=registry,ref=${{ env.GHCR_REPO }}-build-cache:${{ env.PLATFORM_PAIR }}-${{ env.CACHE_KEY_SUFFIX }},mode=max,compression=zstd" >> $GITHUB_OUTPUT + echo "cache-to=type=registry,ref=${GHCR_REPO}-build-cache:${PLATFORM_PAIR}-${CACHE_KEY_SUFFIX},mode=max,compression=zstd" >> $GITHUB_OUTPUT fi - name: Generate docker image tags @@ -413,6 +442,10 @@ jobs: merge_server: name: Merge & Push Server runs-on: ubuntu-latest + permissions: + contents: read + actions: read + packages: write if: ${{ needs.pre-job.outputs.should_run_server == 'true' && !github.event.pull_request.head.repo.fork }} env: GHCR_REPO: ghcr.io/${{ github.repository_owner }}/immich-server @@ -486,15 +519,14 @@ jobs: fi TAGS=$(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - SOURCE_ARGS=$(printf '${{ env.GHCR_REPO }}@sha256:%s ' *) - - echo "docker buildx imagetools create $TAGS "${ANNOTATIONS[@]}" $SOURCE_ARGS" + SOURCE_ARGS=$(printf "${GHCR_REPO}@sha256:%s " *) docker buildx imagetools create $TAGS "${ANNOTATIONS[@]}" $SOURCE_ARGS success-check-server: name: Docker Build & Push Server Success needs: [merge_server, retag_server] + permissions: {} runs-on: ubuntu-latest if: always() steps: @@ -508,6 +540,7 @@ jobs: success-check-ml: name: Docker Build & Push ML Success needs: [merge_ml, retag_ml] + permissions: {} runs-on: ubuntu-latest if: always() steps: diff --git a/.github/workflows/docs-build.yml b/.github/workflows/docs-build.yml index fdd30034ee..ece3bbd248 100644 --- a/.github/workflows/docs-build.yml +++ b/.github/workflows/docs-build.yml @@ -10,14 +10,20 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: {} + jobs: pre-job: runs-on: ubuntu-latest + permissions: + contents: read outputs: should_run: ${{ steps.found_paths.outputs.docs == 'true' || steps.should_force.outputs.should_force == 'true' }} steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - id: found_paths uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 with: @@ -33,6 +39,8 @@ jobs: build: name: Docs Build needs: pre-job + permissions: + contents: read if: ${{ needs.pre-job.outputs.should_run == 'true' }} runs-on: ubuntu-latest defaults: @@ -42,6 +50,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Setup Node uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 diff --git a/.github/workflows/docs-deploy.yml b/.github/workflows/docs-deploy.yml index f33c0c4c03..10277a0c5e 100644 --- a/.github/workflows/docs-deploy.yml +++ b/.github/workflows/docs-deploy.yml @@ -9,6 +9,9 @@ jobs: checks: name: Docs Deploy Checks runs-on: ubuntu-latest + permissions: + actions: read + pull-requests: read outputs: parameters: ${{ steps.parameters.outputs.result }} artifact: ${{ steps.get-artifact.outputs.result }} @@ -36,6 +39,8 @@ jobs: - name: Determine deploy parameters id: parameters uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 + env: + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} with: script: | const eventType = context.payload.workflow_run.event; @@ -57,7 +62,8 @@ jobs: } else if (eventType == "pull_request") { let pull_number = context.payload.workflow_run.pull_requests[0]?.number; if(!pull_number) { - const response = await github.rest.search.issuesAndPullRequests({q: 'repo:${{ github.repository }} is:pr sha:${{ github.event.workflow_run.head_sha }}',per_page: 1,}) + const {HEAD_SHA} = process.env; + const response = await github.rest.search.issuesAndPullRequests({q: `repo:${{ github.repository }} is:pr sha:${HEAD_SHA}`,per_page: 1,}) const items = response.data.items if (items.length < 1) { throw new Error("No pull request found for the commit") @@ -95,10 +101,16 @@ jobs: name: Docs Deploy runs-on: ubuntu-latest needs: checks + permissions: + contents: read + actions: read + pull-requests: write if: ${{ fromJson(needs.checks.outputs.artifact).found && fromJson(needs.checks.outputs.parameters).shouldDeploy }} steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Load parameters id: parameters @@ -162,9 +174,11 @@ jobs: - name: Output Cleaning id: clean + env: + TG_OUTPUT: ${{ steps.docs-output.outputs.tg_action_output }} run: | - TG_OUT=$(echo '${{ steps.docs-output.outputs.tg_action_output }}' | sed 's|%0A|\n|g ; s|%3C|<|g' | jq -c .) - echo "output=$TG_OUT" >> $GITHUB_OUTPUT + CLEANED=$(echo "$TG_OUTPUT" | sed 's|%0A|\n|g ; s|%3C|<|g' | jq -c .) + echo "output=$CLEANED" >> $GITHUB_OUTPUT - name: Publish to Cloudflare Pages uses: cloudflare/pages-action@f0a1cd58cd66095dee69bfa18fa5efd1dde93bca # v1 diff --git a/.github/workflows/docs-destroy.yml b/.github/workflows/docs-destroy.yml index 99499528b6..9d1e4b6612 100644 --- a/.github/workflows/docs-destroy.yml +++ b/.github/workflows/docs-destroy.yml @@ -3,13 +3,20 @@ on: pull_request_target: types: [closed] +permissions: {} + jobs: deploy: name: Docs Destroy runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Destroy Docs Subdomain env: diff --git a/.github/workflows/fix-format.yml b/.github/workflows/fix-format.yml index 9c52691a52..77b86cb0b8 100644 --- a/.github/workflows/fix-format.yml +++ b/.github/workflows/fix-format.yml @@ -4,11 +4,14 @@ on: pull_request: types: [labeled] +permissions: {} + jobs: fix-formatting: runs-on: ubuntu-latest if: ${{ github.event.label.name == 'fix:formatting' }} permissions: + contents: write pull-requests: write steps: - name: Generate a token @@ -23,6 +26,7 @@ jobs: with: ref: ${{ github.event.pull_request.head.ref }} token: ${{ steps.generate-token.outputs.token }} + persist-credentials: true - name: Setup Node uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 diff --git a/.github/workflows/pr-label-validation.yml b/.github/workflows/pr-label-validation.yml index 247c625a96..8d34597a08 100644 --- a/.github/workflows/pr-label-validation.yml +++ b/.github/workflows/pr-label-validation.yml @@ -4,6 +4,8 @@ on: pull_request_target: types: [opened, labeled, unlabeled, synchronize] +permissions: {} + jobs: validate-release-label: runs-on: ubuntu-latest diff --git a/.github/workflows/pr-labeler.yml b/.github/workflows/pr-labeler.yml index 1b43c89889..5704f4275f 100644 --- a/.github/workflows/pr-labeler.yml +++ b/.github/workflows/pr-labeler.yml @@ -2,6 +2,8 @@ name: 'Pull Request Labeler' on: - pull_request_target +permissions: {} + jobs: labeler: permissions: diff --git a/.github/workflows/pr-require-conventional-commit.yml b/.github/workflows/pr-require-conventional-commit.yml index 20dd0492f4..78ba77495c 100644 --- a/.github/workflows/pr-require-conventional-commit.yml +++ b/.github/workflows/pr-require-conventional-commit.yml @@ -4,9 +4,13 @@ on: pull_request: types: [opened, synchronize, reopened, edited] +permissions: {} + jobs: validate-pr-title: runs-on: ubuntu-latest + permissions: + pull-requests: write steps: - name: PR Conventional Commit Validation uses: ytanikin/PRConventionalCommits@b628c5a234cc32513014b7bfdd1e47b532124d98 # 1.3.0 diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml index ffb24d8952..7971f7574a 100644 --- a/.github/workflows/prepare-release.yml +++ b/.github/workflows/prepare-release.yml @@ -21,13 +21,14 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }}-root cancel-in-progress: true +permissions: {} + jobs: bump_version: runs-on: ubuntu-latest - outputs: ref: ${{ steps.push-tag.outputs.commit_long_sha }} - + permissions: {} # No job-level permissions are needed because it uses the app-token steps: - name: Generate a token id: generate-token @@ -40,6 +41,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: token: ${{ steps.generate-token.outputs.token }} + persist-credentials: true - name: Install uv uses: astral-sh/setup-uv@0c5e2b8115b80b4c7c5ddf6ffdd634974642d182 # v5 @@ -59,14 +61,20 @@ jobs: build_mobile: uses: ./.github/workflows/build-mobile.yml needs: bump_version - secrets: inherit + secrets: + KEY_JKS: ${{ secrets.KEY_JKS }} + ALIAS: ${{ secrets.ALIAS }} + ANDROID_KEY_PASSWORD: ${{ secrets.ANDROID_KEY_PASSWORD }} + ANDROID_STORE_PASSWORD: ${{ secrets.ANDROID_STORE_PASSWORD }} with: ref: ${{ needs.bump_version.outputs.ref }} prepare_release: runs-on: ubuntu-latest needs: build_mobile - + permissions: + actions: read # To download the app artifact + # No content permissions are needed because it uses the app-token steps: - name: Generate a token id: generate-token @@ -79,6 +87,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: token: ${{ steps.generate-token.outputs.token }} + persist-credentials: false - name: Download APK uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4 @@ -90,6 +99,7 @@ jobs: with: draft: true tag_name: ${{ env.IMMICH_VERSION }} + token: ${{ steps.generate-token.outputs.token }} generate_release_notes: true body_path: misc/release/notes.tmpl files: | diff --git a/.github/workflows/preview-label.yaml b/.github/workflows/preview-label.yaml index 447a309a2e..4c445f13d0 100644 --- a/.github/workflows/preview-label.yaml +++ b/.github/workflows/preview-label.yaml @@ -4,6 +4,8 @@ on: pull_request: types: [labeled, closed] +permissions: {} + jobs: comment-status: runs-on: ubuntu-latest diff --git a/.github/workflows/sdk.yml b/.github/workflows/sdk.yml index cde2075423..482faf29f4 100644 --- a/.github/workflows/sdk.yml +++ b/.github/workflows/sdk.yml @@ -4,18 +4,22 @@ on: release: types: [published] -permissions: - packages: write +permissions: {} jobs: publish: name: Publish `@immich/sdk` runs-on: ubuntu-latest + permissions: + contents: read defaults: run: working-directory: ./open-api/typescript-sdk steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false + # Setup .npmrc file to publish to npm - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: diff --git a/.github/workflows/static_analysis.yml b/.github/workflows/static_analysis.yml index 615082f86a..1a3c11d3d5 100644 --- a/.github/workflows/static_analysis.yml +++ b/.github/workflows/static_analysis.yml @@ -9,14 +9,20 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: {} + jobs: pre-job: runs-on: ubuntu-latest + permissions: + contents: read outputs: should_run: ${{ steps.found_paths.outputs.mobile == 'true' || steps.should_force.outputs.should_force == 'true' }} steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - id: found_paths uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 with: @@ -33,12 +39,14 @@ jobs: name: Run Dart Code Analysis needs: pre-job if: ${{ needs.pre-job.outputs.should_run == 'true' }} - runs-on: ubuntu-latest - + permissions: + contents: read steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Setup Flutter SDK uses: subosito/flutter-action@e938fdf56512cc96ef2f93601a5a40bde3801046 # v2 @@ -69,9 +77,11 @@ jobs: - name: Verify files have not changed if: steps.verify-changed-files.outputs.files_changed == 'true' + env: + CHANGED_FILES: ${{ steps.verify-changed-files.outputs.changed_files }} run: | echo "ERROR: Generated files not up to date! Run make_build inside the mobile directory" - echo "Changed files: ${{ steps.verify-changed-files.outputs.changed_files }}" + echo "Changed files: ${CHANGED_FILES}" exit 1 - name: Run dart analyze diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f5cb6c8d30..91389c25ff 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,9 +9,13 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: {} + jobs: pre-job: runs-on: ubuntu-latest + permissions: + contents: read outputs: should_run_web: ${{ steps.found_paths.outputs.web == 'true' || steps.should_force.outputs.should_force == 'true' }} should_run_server: ${{ steps.found_paths.outputs.server == 'true' || steps.should_force.outputs.should_force == 'true' }} @@ -25,6 +29,9 @@ jobs: steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false + - id: found_paths uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 with: @@ -58,6 +65,8 @@ jobs: needs: pre-job if: ${{ needs.pre-job.outputs.should_run_server == 'true' }} runs-on: ubuntu-latest + permissions: + contents: read defaults: run: working-directory: ./server @@ -65,6 +74,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Setup Node uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 @@ -95,6 +106,8 @@ jobs: needs: pre-job if: ${{ needs.pre-job.outputs.should_run_cli == 'true' }} runs-on: ubuntu-latest + permissions: + contents: read defaults: run: working-directory: ./cli @@ -102,6 +115,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Setup Node uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 @@ -136,6 +151,8 @@ jobs: needs: pre-job if: ${{ needs.pre-job.outputs.should_run_cli == 'true' }} runs-on: windows-latest + permissions: + contents: read defaults: run: working-directory: ./cli @@ -143,6 +160,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Setup Node uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 @@ -170,6 +189,8 @@ jobs: needs: pre-job if: ${{ needs.pre-job.outputs.should_run_web == 'true' }} runs-on: ubuntu-latest + permissions: + contents: read defaults: run: working-directory: ./web @@ -177,6 +198,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Setup Node uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 @@ -215,6 +238,8 @@ jobs: needs: pre-job if: ${{ needs.pre-job.outputs.should_run_e2e == 'true' }} runs-on: ubuntu-latest + permissions: + contents: read defaults: run: working-directory: ./e2e @@ -222,6 +247,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Setup Node uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 @@ -254,6 +281,8 @@ jobs: needs: pre-job if: ${{ needs.pre-job.outputs.should_run_server == 'true' }} runs-on: ubuntu-latest + permissions: + contents: read defaults: run: working-directory: ./server @@ -261,6 +290,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Setup Node uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 @@ -279,6 +310,8 @@ jobs: needs: pre-job if: ${{ needs.pre-job.outputs.should_run_e2e_server_cli == 'true' }} runs-on: mich + permissions: + contents: read defaults: run: working-directory: ./e2e @@ -287,6 +320,7 @@ jobs: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: + persist-credentials: false submodules: 'recursive' - name: Setup Node @@ -321,6 +355,8 @@ jobs: needs: pre-job if: ${{ needs.pre-job.outputs.should_run_e2e_web == 'true' }} runs-on: mich + permissions: + contents: read defaults: run: working-directory: ./e2e @@ -329,6 +365,7 @@ jobs: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: + persist-credentials: false submodules: 'recursive' - name: Setup Node @@ -362,8 +399,13 @@ jobs: needs: pre-job if: ${{ needs.pre-job.outputs.should_run_mobile == 'true' }} runs-on: ubuntu-latest + permissions: + contents: read steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false + - name: Setup Flutter SDK uses: subosito/flutter-action@e938fdf56512cc96ef2f93601a5a40bde3801046 # v2 with: @@ -378,11 +420,16 @@ jobs: needs: pre-job if: ${{ needs.pre-job.outputs.should_run_ml == 'true' }} runs-on: ubuntu-latest + permissions: + contents: read defaults: run: working-directory: ./machine-learning steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false + - name: Install uv uses: astral-sh/setup-uv@0c5e2b8115b80b4c7c5ddf6ffdd634974642d182 # v5 - uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5 @@ -411,6 +458,8 @@ jobs: needs: pre-job if: ${{ needs.pre-job.outputs['should_run_.github'] == 'true' }} runs-on: ubuntu-latest + permissions: + contents: read defaults: run: working-directory: ./.github @@ -418,6 +467,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Setup Node uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 @@ -434,22 +485,31 @@ jobs: shellcheck: name: ShellCheck runs-on: ubuntu-latest + permissions: + contents: read steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false + - name: Run ShellCheck uses: ludeeus/action-shellcheck@master with: ignore_paths: >- **/open-api/** - **/openapi/** + **/openapi** **/node_modules/** generated-api-up-to-date: name: OpenAPI Clients runs-on: ubuntu-latest + permissions: + contents: read steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Setup Node uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 @@ -476,14 +536,18 @@ jobs: - name: Verify files have not changed if: steps.verify-changed-files.outputs.files_changed == 'true' + env: + CHANGED_FILES: ${{ steps.verify-changed-files.outputs.changed_files }} run: | echo "ERROR: Generated files not up to date!" - echo "Changed files: ${{ steps.verify-changed-files.outputs.changed_files }}" + echo "Changed files: ${CHANGED_FILES}" exit 1 generated-typeorm-migrations-up-to-date: name: TypeORM Checks runs-on: ubuntu-latest + permissions: + contents: read services: postgres: image: tensorchord/pgvecto-rs:pg14-v0.2.0@sha256:739cdd626151ff1f796dc95a6591b55a714f341c737e27f045019ceabf8e8c52 @@ -505,6 +569,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Setup Node uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 @@ -535,9 +601,11 @@ jobs: server/src - name: Verify migration files have not changed if: steps.verify-changed-files.outputs.files_changed == 'true' + env: + CHANGED_FILES: ${{ steps.verify-changed-files.outputs.changed_files }} run: | echo "ERROR: Generated migration files not up to date!" - echo "Changed files: ${{ steps.verify-changed-files.outputs.changed_files }}" + echo "Changed files: ${CHANGED_FILES}" cat ./src/*-TestMigration.ts exit 1 @@ -555,9 +623,11 @@ jobs: - name: Verify SQL files have not changed if: steps.verify-changed-sql-files.outputs.files_changed == 'true' + env: + CHANGED_FILES: ${{ steps.verify-changed-sql-files.outputs.changed_files }} run: | echo "ERROR: Generated SQL files not up to date!" - echo "Changed files: ${{ steps.verify-changed-sql-files.outputs.changed_files }}" + echo "Changed files: ${CHANGED_FILES}" exit 1 # mobile-integration-tests: diff --git a/.github/workflows/weblate-lock.yml b/.github/workflows/weblate-lock.yml index 69dce3ac41..2aef5c472a 100644 --- a/.github/workflows/weblate-lock.yml +++ b/.github/workflows/weblate-lock.yml @@ -4,30 +4,32 @@ on: pull_request: branches: [main] +permissions: {} + jobs: pre-job: runs-on: ubuntu-latest + permissions: + contents: read outputs: should_run: ${{ steps.found_paths.outputs.i18n == 'true' && github.head_ref != 'chore/translations'}} steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - id: found_paths uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 with: filters: | i18n: - 'i18n/!(en)**\.json' - - name: Debug - run: | - echo "Should run: ${{ steps.found_paths.outputs.i18n == 'true' && github.head_ref != 'chore/translations'}}" - echo "Found i18n paths: ${{ steps.found_paths.outputs.i18n }}" - echo "Head ref: ${{ github.head_ref }}" enforce-lock: name: Check Weblate Lock needs: [pre-job] runs-on: ubuntu-latest + permissions: {} if: ${{ needs.pre-job.outputs.should_run == 'true' }} steps: - name: Check weblate lock @@ -47,6 +49,7 @@ jobs: name: Weblate Lock Check Success needs: [enforce-lock] runs-on: ubuntu-latest + permissions: {} if: always() steps: - name: Any jobs failed? From 854ea13d6abcdb031e4f63974be049733c717f81 Mon Sep 17 00:00:00 2001 From: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com> Date: Fri, 18 Apr 2025 22:52:41 +0200 Subject: [PATCH 10/12] chore: simplify asset getByIds (#17699) --- server/src/repositories/asset.repository.ts | 54 ++++----------------- 1 file changed, 9 insertions(+), 45 deletions(-) diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts index 52c390c162..af79fb7c5f 100644 --- a/server/src/repositories/asset.repository.ts +++ b/server/src/repositories/asset.repository.ts @@ -299,51 +299,15 @@ export class AssetRepository { @GenerateSql({ params: [[DummyValue.UUID]] }) @ChunkedArray() - async getByIds( - ids: string[], - { exifInfo, faces, files, library, owner, smartSearch, stack, tags }: GetByIdsRelations = {}, - ): Promise { - const res = await this.db - .selectFrom('assets') - .selectAll('assets') - .where('assets.id', '=', anyUuid(ids)) - .$if(!!exifInfo, withExif) - .$if(!!faces, (qb) => - qb.select((eb) => - faces?.person ? withFacesAndPeople(eb, faces.withDeleted) : withFaces(eb, faces?.withDeleted), - ), - ) - .$if(!!files, (qb) => qb.select(withFiles)) - .$if(!!library, (qb) => qb.select(withLibrary)) - .$if(!!owner, (qb) => qb.select(withOwner)) - .$if(!!smartSearch, withSmartSearch) - .$if(!!stack, (qb) => - qb - .leftJoin('asset_stack', 'asset_stack.id', 'assets.stackId') - .$if(!stack!.assets, (qb) => qb.select((eb) => eb.fn.toJson(eb.table('asset_stack')).as('stack'))) - .$if(!!stack!.assets, (qb) => - qb - .leftJoinLateral( - (eb) => - eb - .selectFrom('assets as stacked') - .selectAll('asset_stack') - .select((eb) => eb.fn('array_agg', [eb.table('stacked')]).as('assets')) - .whereRef('stacked.stackId', '=', 'asset_stack.id') - .whereRef('stacked.id', '!=', 'asset_stack.primaryAssetId') - .where('stacked.deletedAt', 'is', null) - .where('stacked.isArchived', '=', false) - .groupBy('asset_stack.id') - .as('stacked_assets'), - (join) => join.on('asset_stack.id', 'is not', null), - ) - .select((eb) => eb.fn.toJson(eb.table('stacked_assets')).as('stack')), - ), - ) - .$if(!!tags, (qb) => qb.select(withTags)) - .execute(); - - return res as any as AssetEntity[]; + getByIds(ids: string[]): Promise { + return ( + this.db + // + .selectFrom('assets') + .selectAll('assets') + .where('assets.id', '=', anyUuid(ids)) + .execute() as Promise + ); } @GenerateSql({ params: [[DummyValue.UUID]] }) From 52ae06c1194e992df6407e67bdebf9fd9b41238f Mon Sep 17 00:00:00 2001 From: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com> Date: Fri, 18 Apr 2025 23:10:34 +0200 Subject: [PATCH 11/12] refactor: remove album entity, update types (#17450) --- server/src/database.ts | 85 ++++++----- server/src/db.d.ts | 6 +- server/src/dtos/album.dto.ts | 28 +++- server/src/dtos/asset-response.dto.ts | 50 ++++++- server/src/dtos/memory.dto.ts | 3 +- server/src/dtos/shared-link.dto.ts | 8 +- server/src/entities/album.entity.ts | 23 --- server/src/entities/asset.entity.ts | 24 ++-- server/src/entities/shared-link.entity.ts | 20 --- server/src/queries/asset.repository.sql | 21 +-- server/src/repositories/album.repository.ts | 53 +++---- .../src/repositories/asset-job.repository.ts | 17 ++- server/src/repositories/asset.repository.ts | 132 ++++++++---------- server/src/repositories/search.repository.ts | 35 +++-- .../repositories/shared-link.repository.ts | 41 +++--- server/src/repositories/stack.repository.ts | 5 +- server/src/services/album.service.ts | 8 +- .../src/services/asset-media.service.spec.ts | 9 +- server/src/services/asset-media.service.ts | 16 +-- server/src/services/asset.service.spec.ts | 7 +- server/src/services/asset.service.ts | 8 +- server/src/services/duplicate.service.ts | 2 +- server/src/services/job.service.spec.ts | 4 +- server/src/services/job.service.ts | 4 +- server/src/services/library.service.spec.ts | 18 +-- server/src/services/library.service.ts | 14 +- server/src/services/metadata.service.spec.ts | 9 +- server/src/services/metadata.service.ts | 2 +- server/src/services/person.service.ts | 8 +- server/src/services/search.service.spec.ts | 2 +- server/src/services/search.service.ts | 7 +- .../src/services/shared-link.service.spec.ts | 5 +- server/src/services/shared-link.service.ts | 10 +- server/src/services/sync.service.spec.ts | 3 +- server/src/services/view.service.ts | 3 +- server/src/utils/asset.util.ts | 7 +- server/test/fixtures/album.stub.ts | 22 +-- server/test/fixtures/asset.stub.ts | 93 +++++++++--- server/test/fixtures/auth.stub.ts | 9 +- server/test/fixtures/shared-link.stub.ts | 31 ++-- .../specs/services/memory.service.spec.ts | 9 +- .../specs/services/metadata.service.spec.ts | 2 +- .../repositories/asset.repository.mock.ts | 2 +- server/test/small.factory.ts | 4 +- 44 files changed, 473 insertions(+), 396 deletions(-) delete mode 100644 server/src/entities/album.entity.ts delete mode 100644 server/src/entities/shared-link.entity.ts diff --git a/server/src/database.ts b/server/src/database.ts index b504d39579..6e66cb4d3d 100644 --- a/server/src/database.ts +++ b/server/src/database.ts @@ -1,13 +1,14 @@ import { Selectable } from 'kysely'; -import { AssetJobStatus as DatabaseAssetJobStatus, Exif as DatabaseExif } from 'src/db'; +import { Albums, AssetJobStatus as DatabaseAssetJobStatus, Exif as DatabaseExif } from 'src/db'; +import { MapAsset } from 'src/dtos/asset-response.dto'; import { AssetEntity } from 'src/entities/asset.entity'; import { AlbumUserRole, AssetFileType, - AssetStatus, AssetType, MemoryType, Permission, + SharedLinkType, SourceType, UserStatus, } from 'src/enum'; @@ -44,7 +45,7 @@ export type Library = { exclusionPatterns: string[]; deletedAt: Date | null; refreshedAt: Date | null; - assets?: Asset[]; + assets?: MapAsset[]; }; export type AuthApiKey = { @@ -96,7 +97,26 @@ export type Memory = { data: OnThisDayData; ownerId: string; isSaved: boolean; - assets: Asset[]; + assets: MapAsset[]; +}; + +export type Asset = { + id: string; + checksum: Buffer; + deviceAssetId: string; + deviceId: string; + fileCreatedAt: Date; + fileModifiedAt: Date; + isExternal: boolean; + isVisible: boolean; + libraryId: string | null; + livePhotoVideoId: string | null; + localDateTime: Date; + originalFileName: string; + originalPath: string; + ownerId: string; + sidecarPath: string | null; + type: AssetType; }; export type User = { @@ -128,39 +148,6 @@ export type StorageAsset = { encodedVideoPath: string | null; }; -export type Asset = { - createdAt: Date; - updatedAt: Date; - deletedAt: Date | null; - id: string; - updateId: string; - status: AssetStatus; - checksum: Buffer; - deviceAssetId: string; - deviceId: string; - duplicateId: string | null; - duration: string | null; - encodedVideoPath: string | null; - fileCreatedAt: Date | null; - fileModifiedAt: Date | null; - isArchived: boolean; - isExternal: boolean; - isFavorite: boolean; - isOffline: boolean; - isVisible: boolean; - libraryId: string | null; - livePhotoVideoId: string | null; - localDateTime: Date | null; - originalFileName: string; - originalPath: string; - ownerId: string; - sidecarPath: string | null; - stack?: Stack | null; - stackId: string | null; - thumbhash: Buffer | null; - type: AssetType; -}; - export type SidecarWriteAsset = { id: string; sidecarPath: string | null; @@ -173,7 +160,7 @@ export type Stack = { primaryAssetId: string; owner?: User; ownerId: string; - assets: AssetEntity[]; + assets: MapAsset[]; assetCount?: number; }; @@ -187,6 +174,28 @@ export type AuthSharedLink = { password: string | null; }; +export type SharedLink = { + id: string; + album?: Album | null; + albumId: string | null; + allowDownload: boolean; + allowUpload: boolean; + assets: MapAsset[]; + createdAt: Date; + description: string | null; + expiresAt: Date | null; + key: Buffer; + password: string | null; + showExif: boolean; + type: SharedLinkType; + userId: string; +}; + +export type Album = Selectable & { + owner: User; + assets: MapAsset[]; +}; + export type AuthSession = { id: string; }; diff --git a/server/src/db.d.ts b/server/src/db.d.ts index 7115b701ce..4e9738ecec 100644 --- a/server/src/db.d.ts +++ b/server/src/db.d.ts @@ -143,8 +143,8 @@ export interface Assets { duplicateId: string | null; duration: string | null; encodedVideoPath: Generated; - fileCreatedAt: Timestamp | null; - fileModifiedAt: Timestamp | null; + fileCreatedAt: Timestamp; + fileModifiedAt: Timestamp; id: Generated; isArchived: Generated; isExternal: Generated; @@ -153,7 +153,7 @@ export interface Assets { isVisible: Generated; libraryId: string | null; livePhotoVideoId: string | null; - localDateTime: Timestamp | null; + localDateTime: Timestamp; originalFileName: string; originalPath: string; ownerId: string; diff --git a/server/src/dtos/album.dto.ts b/server/src/dtos/album.dto.ts index c9934ec909..40e51ef729 100644 --- a/server/src/dtos/album.dto.ts +++ b/server/src/dtos/album.dto.ts @@ -2,10 +2,10 @@ import { ApiProperty } from '@nestjs/swagger'; import { Type } from 'class-transformer'; import { ArrayNotEmpty, IsArray, IsEnum, IsString, ValidateNested } from 'class-validator'; import _ from 'lodash'; -import { AssetResponseDto, mapAsset } from 'src/dtos/asset-response.dto'; +import { AlbumUser, AuthSharedLink, User } from 'src/database'; +import { AssetResponseDto, MapAsset, mapAsset } from 'src/dtos/asset-response.dto'; import { AuthDto } from 'src/dtos/auth.dto'; import { UserResponseDto, mapUser } from 'src/dtos/user.dto'; -import { AlbumEntity } from 'src/entities/album.entity'; import { AlbumUserRole, AssetOrder } from 'src/enum'; import { Optional, ValidateBoolean, ValidateUUID } from 'src/validation'; @@ -142,7 +142,23 @@ export class AlbumResponseDto { order?: AssetOrder; } -export const mapAlbum = (entity: AlbumEntity, withAssets: boolean, auth?: AuthDto): AlbumResponseDto => { +export type MapAlbumDto = { + albumUsers?: AlbumUser[]; + assets?: MapAsset[]; + sharedLinks?: AuthSharedLink[]; + albumName: string; + description: string; + albumThumbnailAssetId: string | null; + createdAt: Date; + updatedAt: Date; + id: string; + ownerId: string; + owner: User; + isActivityEnabled: boolean; + order: AssetOrder; +}; + +export const mapAlbum = (entity: MapAlbumDto, withAssets: boolean, auth?: AuthDto): AlbumResponseDto => { const albumUsers: AlbumUserResponseDto[] = []; if (entity.albumUsers) { @@ -159,7 +175,7 @@ export const mapAlbum = (entity: AlbumEntity, withAssets: boolean, auth?: AuthDt const assets = entity.assets || []; - const hasSharedLink = entity.sharedLinks?.length > 0; + const hasSharedLink = !!entity.sharedLinks && entity.sharedLinks.length > 0; const hasSharedUser = albumUsers.length > 0; let startDate = assets.at(0)?.localDateTime; @@ -190,5 +206,5 @@ export const mapAlbum = (entity: AlbumEntity, withAssets: boolean, auth?: AuthDt }; }; -export const mapAlbumWithAssets = (entity: AlbumEntity) => mapAlbum(entity, true); -export const mapAlbumWithoutAssets = (entity: AlbumEntity) => mapAlbum(entity, false); +export const mapAlbumWithAssets = (entity: MapAlbumDto) => mapAlbum(entity, true); +export const mapAlbumWithoutAssets = (entity: MapAlbumDto) => mapAlbum(entity, false); diff --git a/server/src/dtos/asset-response.dto.ts b/server/src/dtos/asset-response.dto.ts index 985ad04729..c0e589f380 100644 --- a/server/src/dtos/asset-response.dto.ts +++ b/server/src/dtos/asset-response.dto.ts @@ -1,5 +1,6 @@ import { ApiProperty } from '@nestjs/swagger'; -import { AssetFace } from 'src/database'; +import { Selectable } from 'kysely'; +import { AssetFace, AssetFile, Exif, Stack, Tag, User } from 'src/database'; import { PropertyLifecycle } from 'src/decorators'; import { AuthDto } from 'src/dtos/auth.dto'; import { ExifResponseDto, mapExif } from 'src/dtos/exif.dto'; @@ -11,8 +12,7 @@ import { } from 'src/dtos/person.dto'; import { TagResponseDto, mapTag } from 'src/dtos/tag.dto'; import { UserResponseDto, mapUser } from 'src/dtos/user.dto'; -import { AssetEntity } from 'src/entities/asset.entity'; -import { AssetType } from 'src/enum'; +import { AssetStatus, AssetType } from 'src/enum'; import { mimeTypes } from 'src/utils/mime-types'; export class SanitizedAssetResponseDto { @@ -56,6 +56,44 @@ export class AssetResponseDto extends SanitizedAssetResponseDto { resized?: boolean; } +export type MapAsset = { + createdAt: Date; + updatedAt: Date; + deletedAt: Date | null; + id: string; + updateId: string; + status: AssetStatus; + checksum: Buffer; + deviceAssetId: string; + deviceId: string; + duplicateId: string | null; + duration: string | null; + encodedVideoPath: string | null; + exifInfo?: Selectable | null; + faces?: AssetFace[]; + fileCreatedAt: Date; + fileModifiedAt: Date; + files?: AssetFile[]; + isArchived: boolean; + isExternal: boolean; + isFavorite: boolean; + isOffline: boolean; + isVisible: boolean; + libraryId: string | null; + livePhotoVideoId: string | null; + localDateTime: Date; + originalFileName: string; + originalPath: string; + owner?: User | null; + ownerId: string; + sidecarPath: string | null; + stack?: Stack | null; + stackId: string | null; + tags?: Tag[]; + thumbhash: Buffer | null; + type: AssetType; +}; + export class AssetStackResponseDto { id!: string; @@ -72,7 +110,7 @@ export type AssetMapOptions = { }; // TODO: this is inefficient -const peopleWithFaces = (faces: AssetFace[]): PersonWithFacesResponseDto[] => { +const peopleWithFaces = (faces?: AssetFace[]): PersonWithFacesResponseDto[] => { const result: PersonWithFacesResponseDto[] = []; if (faces) { for (const face of faces) { @@ -90,7 +128,7 @@ const peopleWithFaces = (faces: AssetFace[]): PersonWithFacesResponseDto[] => { return result; }; -const mapStack = (entity: AssetEntity) => { +const mapStack = (entity: { stack?: Stack | null }) => { if (!entity.stack) { return null; } @@ -111,7 +149,7 @@ export const hexOrBufferToBase64 = (encoded: string | Buffer) => { return encoded.toString('base64'); }; -export function mapAsset(entity: AssetEntity, options: AssetMapOptions = {}): AssetResponseDto { +export function mapAsset(entity: MapAsset, options: AssetMapOptions = {}): AssetResponseDto { const { stripMetadata = false, withStack = false } = options; if (stripMetadata) { diff --git a/server/src/dtos/memory.dto.ts b/server/src/dtos/memory.dto.ts index b3054d7a4c..98231a9035 100644 --- a/server/src/dtos/memory.dto.ts +++ b/server/src/dtos/memory.dto.ts @@ -4,7 +4,6 @@ import { IsEnum, IsInt, IsObject, IsPositive, ValidateNested } from 'class-valid import { Memory } from 'src/database'; import { AssetResponseDto, mapAsset } from 'src/dtos/asset-response.dto'; import { AuthDto } from 'src/dtos/auth.dto'; -import { AssetEntity } from 'src/entities/asset.entity'; import { MemoryType } from 'src/enum'; import { Optional, ValidateBoolean, ValidateDate, ValidateUUID } from 'src/validation'; @@ -103,6 +102,6 @@ export const mapMemory = (entity: Memory, auth: AuthDto): MemoryResponseDto => { type: entity.type as MemoryType, data: entity.data as unknown as MemoryData, isSaved: entity.isSaved, - assets: ('assets' in entity ? entity.assets : []).map((asset) => mapAsset(asset as AssetEntity, { auth })), + assets: ('assets' in entity ? entity.assets : []).map((asset) => mapAsset(asset, { auth })), }; }; diff --git a/server/src/dtos/shared-link.dto.ts b/server/src/dtos/shared-link.dto.ts index 6bb8ab1f0d..8d373b40b6 100644 --- a/server/src/dtos/shared-link.dto.ts +++ b/server/src/dtos/shared-link.dto.ts @@ -1,9 +1,9 @@ import { ApiProperty } from '@nestjs/swagger'; import { IsEnum, IsString } from 'class-validator'; import _ from 'lodash'; +import { SharedLink } from 'src/database'; import { AlbumResponseDto, mapAlbumWithoutAssets } from 'src/dtos/album.dto'; import { AssetResponseDto, mapAsset } from 'src/dtos/asset-response.dto'; -import { SharedLinkEntity } from 'src/entities/shared-link.entity'; import { SharedLinkType } from 'src/enum'; import { Optional, ValidateBoolean, ValidateDate, ValidateUUID } from 'src/validation'; @@ -102,7 +102,7 @@ export class SharedLinkResponseDto { showMetadata!: boolean; } -export function mapSharedLink(sharedLink: SharedLinkEntity): SharedLinkResponseDto { +export function mapSharedLink(sharedLink: SharedLink): SharedLinkResponseDto { const linkAssets = sharedLink.assets || []; return { @@ -122,7 +122,7 @@ export function mapSharedLink(sharedLink: SharedLinkEntity): SharedLinkResponseD }; } -export function mapSharedLinkWithoutMetadata(sharedLink: SharedLinkEntity): SharedLinkResponseDto { +export function mapSharedLinkWithoutMetadata(sharedLink: SharedLink): SharedLinkResponseDto { const linkAssets = sharedLink.assets || []; const albumAssets = (sharedLink?.album?.assets || []).map((asset) => asset); @@ -137,7 +137,7 @@ export function mapSharedLinkWithoutMetadata(sharedLink: SharedLinkEntity): Shar type: sharedLink.type, createdAt: sharedLink.createdAt, expiresAt: sharedLink.expiresAt, - assets: assets.map((asset) => mapAsset(asset, { stripMetadata: true })) as AssetResponseDto[], + assets: assets.map((asset) => mapAsset(asset, { stripMetadata: true })), album: sharedLink.album ? mapAlbumWithoutAssets(sharedLink.album) : undefined, allowUpload: sharedLink.allowUpload, allowDownload: sharedLink.allowDownload, diff --git a/server/src/entities/album.entity.ts b/server/src/entities/album.entity.ts deleted file mode 100644 index eb20c1afdd..0000000000 --- a/server/src/entities/album.entity.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { AlbumUser, User } from 'src/database'; -import { AssetEntity } from 'src/entities/asset.entity'; -import { SharedLinkEntity } from 'src/entities/shared-link.entity'; -import { AssetOrder } from 'src/enum'; - -export class AlbumEntity { - id!: string; - owner!: User; - ownerId!: string; - albumName!: string; - description!: string; - createdAt!: Date; - updatedAt!: Date; - updateId?: string; - deletedAt!: Date | null; - albumThumbnailAsset!: AssetEntity | null; - albumThumbnailAssetId!: string | null; - albumUsers!: AlbumUser[]; - assets!: AssetEntity[]; - sharedLinks!: SharedLinkEntity[]; - isActivityEnabled!: boolean; - order!: AssetOrder; -} diff --git a/server/src/entities/asset.entity.ts b/server/src/entities/asset.entity.ts index 64c038a689..da291292e7 100644 --- a/server/src/entities/asset.entity.ts +++ b/server/src/entities/asset.entity.ts @@ -1,12 +1,12 @@ import { DeduplicateJoinsPlugin, ExpressionBuilder, Kysely, SelectQueryBuilder, sql } from 'kysely'; import { jsonArrayFrom, jsonObjectFrom } from 'kysely/helpers/postgres'; -import { AssetFace, AssetFile, AssetJobStatus, columns, Exif, Stack, Tag, User } from 'src/database'; +import { AssetFace, AssetFile, AssetJobStatus, columns, Exif, Person, Stack, Tag, User } from 'src/database'; import { DB } from 'src/db'; -import { SharedLinkEntity } from 'src/entities/shared-link.entity'; +import { MapAsset } from 'src/dtos/asset-response.dto'; import { AssetFileType, AssetStatus, AssetType } from 'src/enum'; import { TimeBucketSize } from 'src/repositories/asset.repository'; import { AssetSearchBuilderOptions } from 'src/repositories/search.repository'; -import { anyUuid, asUuid } from 'src/utils/database'; +import { anyUuid, asUuid, toJson } from 'src/utils/database'; export const ASSET_CHECKSUM_CONSTRAINT = 'UQ_assets_owner_checksum'; @@ -37,13 +37,12 @@ export class AssetEntity { checksum!: Buffer; // sha1 checksum duration!: string | null; isVisible!: boolean; - livePhotoVideo!: AssetEntity | null; + livePhotoVideo!: MapAsset | null; livePhotoVideoId!: string | null; originalFileName!: string; sidecarPath!: string | null; exifInfo?: Exif; tags?: Tag[]; - sharedLinks!: SharedLinkEntity[]; faces!: AssetFace[]; stackId?: string | null; stack?: Stack | null; @@ -51,6 +50,7 @@ export class AssetEntity { duplicateId!: string | null; } +// TODO come up with a better query that only selects the fields we need export function withExif(qb: SelectQueryBuilder) { return qb .leftJoin('exif', 'assets.id', 'exif.assetId') @@ -66,7 +66,7 @@ export function withExifInner(qb: SelectQueryBuilder) { export function withSmartSearch(qb: SelectQueryBuilder) { return qb .leftJoin('smart_search', 'assets.id', 'smart_search.assetId') - .select((eb) => eb.fn.toJson(eb.table('smart_search')).as('smartSearch')); + .select((eb) => toJson(eb, 'smart_search').as('smartSearch')); } export function withFaces(eb: ExpressionBuilder, withDeletedFace?: boolean) { @@ -99,7 +99,7 @@ export function withFacesAndPeople(eb: ExpressionBuilder, withDele (join) => join.onTrue(), ) .selectAll('asset_faces') - .select((eb) => eb.table('person').as('person')) + .select((eb) => eb.table('person').$castTo().as('person')) .whereRef('asset_faces.assetId', '=', 'assets.id') .$if(!withDeletedFace, (qb) => qb.where('asset_faces.deletedAt', 'is', null)), ).as('faces'); @@ -136,13 +136,15 @@ export function hasTags(qb: SelectQueryBuilder, tagIds: stri } export function withOwner(eb: ExpressionBuilder) { - return jsonObjectFrom(eb.selectFrom('users').selectAll().whereRef('users.id', '=', 'assets.ownerId')).as('owner'); + return jsonObjectFrom(eb.selectFrom('users').select(columns.user).whereRef('users.id', '=', 'assets.ownerId')).as( + 'owner', + ); } export function withLibrary(eb: ExpressionBuilder) { - return jsonObjectFrom(eb.selectFrom('libraries').selectAll().whereRef('libraries.id', '=', 'assets.libraryId')).as( - 'library', - ); + return jsonObjectFrom( + eb.selectFrom('libraries').selectAll('libraries').whereRef('libraries.id', '=', 'assets.libraryId'), + ).as('library'); } export function withTags(eb: ExpressionBuilder) { diff --git a/server/src/entities/shared-link.entity.ts b/server/src/entities/shared-link.entity.ts deleted file mode 100644 index 720ba424d1..0000000000 --- a/server/src/entities/shared-link.entity.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { AlbumEntity } from 'src/entities/album.entity'; -import { AssetEntity } from 'src/entities/asset.entity'; -import { SharedLinkType } from 'src/enum'; - -export class SharedLinkEntity { - id!: string; - description!: string | null; - password!: string | null; - userId!: string; - key!: Buffer; // use to access the inidividual asset - type!: SharedLinkType; - createdAt!: Date; - expiresAt!: Date | null; - allowUpload!: boolean; - allowDownload!: boolean; - showExif!: boolean; - assets!: AssetEntity[]; - album?: AlbumEntity; - albumId!: string | null; -} diff --git a/server/src/queries/asset.repository.sql b/server/src/queries/asset.repository.sql index cf17bb0276..a3dcb08c1e 100644 --- a/server/src/queries/asset.repository.sql +++ b/server/src/queries/asset.repository.sql @@ -82,7 +82,7 @@ from where "assets"."id" = any ($1::uuid[]) --- AssetRepository.getByIdsWithAllRelations +-- AssetRepository.getByIdsWithAllRelationsButStacks select "assets".*, ( @@ -127,28 +127,13 @@ select "assets"."id" = "tag_asset"."assetsId" ) as agg ) as "tags", - to_json("exif") as "exifInfo", - to_json("stacked_assets") as "stack" + to_json("exif") as "exifInfo" from "assets" left join "exif" on "assets"."id" = "exif"."assetId" left join "asset_stack" on "asset_stack"."id" = "assets"."stackId" - left join lateral ( - select - "asset_stack".*, - array_agg("stacked") as "assets" - from - "assets" as "stacked" - where - "stacked"."stackId" = "asset_stack"."id" - and "stacked"."id" != "asset_stack"."primaryAssetId" - and "stacked"."deletedAt" is null - and "stacked"."isArchived" = $1 - group by - "asset_stack"."id" - ) as "stacked_assets" on "asset_stack"."id" is not null where - "assets"."id" = any ($2::uuid[]) + "assets"."id" = any ($1::uuid[]) -- AssetRepository.deleteAll delete from "assets" diff --git a/server/src/repositories/album.repository.ts b/server/src/repositories/album.repository.ts index e21d5d73cd..1768135210 100644 --- a/server/src/repositories/album.repository.ts +++ b/server/src/repositories/album.repository.ts @@ -1,12 +1,11 @@ import { Injectable } from '@nestjs/common'; -import { ExpressionBuilder, Insertable, Kysely, sql, Updateable } from 'kysely'; +import { ExpressionBuilder, Insertable, Kysely, NotNull, sql, Updateable } from 'kysely'; import { jsonArrayFrom, jsonObjectFrom } from 'kysely/helpers/postgres'; import { InjectKysely } from 'nestjs-kysely'; -import { columns } from 'src/database'; +import { columns, Exif } from 'src/database'; import { Albums, DB } from 'src/db'; import { Chunked, ChunkedArray, ChunkedSet, DummyValue, GenerateSql } from 'src/decorators'; import { AlbumUserCreateDto } from 'src/dtos/album.dto'; -import { AlbumEntity } from 'src/entities/album.entity'; export interface AlbumAssetCount { albumId: string; @@ -21,9 +20,9 @@ export interface AlbumInfoOptions { } const withOwner = (eb: ExpressionBuilder) => { - return jsonObjectFrom(eb.selectFrom('users').select(columns.user).whereRef('users.id', '=', 'albums.ownerId')).as( - 'owner', - ); + return jsonObjectFrom(eb.selectFrom('users').select(columns.user).whereRef('users.id', '=', 'albums.ownerId')) + .$notNull() + .as('owner'); }; const withAlbumUsers = (eb: ExpressionBuilder) => { @@ -32,12 +31,14 @@ const withAlbumUsers = (eb: ExpressionBuilder) => { .selectFrom('albums_shared_users_users as album_users') .select('album_users.role') .select((eb) => - jsonObjectFrom(eb.selectFrom('users').select(columns.user).whereRef('users.id', '=', 'album_users.usersId')).as( - 'user', - ), + jsonObjectFrom(eb.selectFrom('users').select(columns.user).whereRef('users.id', '=', 'album_users.usersId')) + .$notNull() + .as('user'), ) .whereRef('album_users.albumsId', '=', 'albums.id'), - ).as('albumUsers'); + ) + .$notNull() + .as('albumUsers'); }; const withSharedLink = (eb: ExpressionBuilder) => { @@ -53,7 +54,7 @@ const withAssets = (eb: ExpressionBuilder) => { .selectFrom('assets') .selectAll('assets') .leftJoin('exif', 'assets.id', 'exif.assetId') - .select((eb) => eb.table('exif').as('exifInfo')) + .select((eb) => eb.table('exif').$castTo().as('exifInfo')) .innerJoin('albums_assets_assets', 'albums_assets_assets.assetsId', 'assets.id') .whereRef('albums_assets_assets.albumsId', '=', 'albums.id') .where('assets.deletedAt', 'is', null) @@ -69,7 +70,7 @@ export class AlbumRepository { constructor(@InjectKysely() private db: Kysely) {} @GenerateSql({ params: [DummyValue.UUID, { withAssets: true }] }) - async getById(id: string, options: AlbumInfoOptions): Promise { + async getById(id: string, options: AlbumInfoOptions) { return this.db .selectFrom('albums') .selectAll('albums') @@ -79,11 +80,12 @@ export class AlbumRepository { .select(withAlbumUsers) .select(withSharedLink) .$if(options.withAssets, (eb) => eb.select(withAssets)) - .executeTakeFirst() as Promise; + .$narrowType<{ assets: NotNull }>() + .executeTakeFirst(); } @GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID] }) - async getByAssetId(ownerId: string, assetId: string): Promise { + async getByAssetId(ownerId: string, assetId: string) { return this.db .selectFrom('albums') .selectAll('albums') @@ -105,7 +107,7 @@ export class AlbumRepository { .select(withOwner) .select(withAlbumUsers) .orderBy('albums.createdAt', 'desc') - .execute() as unknown as Promise; + .execute(); } @GenerateSql({ params: [[DummyValue.UUID]] }) @@ -134,7 +136,7 @@ export class AlbumRepository { } @GenerateSql({ params: [DummyValue.UUID] }) - async getOwned(ownerId: string): Promise { + async getOwned(ownerId: string) { return this.db .selectFrom('albums') .selectAll('albums') @@ -144,14 +146,14 @@ export class AlbumRepository { .where('albums.ownerId', '=', ownerId) .where('albums.deletedAt', 'is', null) .orderBy('albums.createdAt', 'desc') - .execute() as unknown as Promise; + .execute(); } /** * Get albums shared with and shared by owner. */ @GenerateSql({ params: [DummyValue.UUID] }) - async getShared(ownerId: string): Promise { + async getShared(ownerId: string) { return this.db .selectFrom('albums') .selectAll('albums') @@ -176,14 +178,14 @@ export class AlbumRepository { .select(withOwner) .select(withSharedLink) .orderBy('albums.createdAt', 'desc') - .execute() as unknown as Promise; + .execute(); } /** * Get albums of owner that are _not_ shared */ @GenerateSql({ params: [DummyValue.UUID] }) - async getNotShared(ownerId: string): Promise { + async getNotShared(ownerId: string) { return this.db .selectFrom('albums') .selectAll('albums') @@ -203,7 +205,7 @@ export class AlbumRepository { ) .select(withOwner) .orderBy('albums.createdAt', 'desc') - .execute() as unknown as Promise; + .execute(); } async restoreAll(userId: string): Promise { @@ -262,7 +264,7 @@ export class AlbumRepository { await this.addAssets(this.db, albumId, assetIds); } - create(album: Insertable, assetIds: string[], albumUsers: AlbumUserCreateDto[]): Promise { + create(album: Insertable, assetIds: string[], albumUsers: AlbumUserCreateDto[]) { return this.db.transaction().execute(async (tx) => { const newAlbum = await tx.insertInto('albums').values(album).returning('albums.id').executeTakeFirst(); @@ -290,11 +292,12 @@ export class AlbumRepository { .select(withOwner) .select(withAssets) .select(withAlbumUsers) - .executeTakeFirst() as unknown as Promise; + .$narrowType<{ assets: NotNull }>() + .executeTakeFirstOrThrow(); }); } - update(id: string, album: Updateable): Promise { + update(id: string, album: Updateable) { return this.db .updateTable('albums') .set(album) @@ -303,7 +306,7 @@ export class AlbumRepository { .returning(withOwner) .returning(withSharedLink) .returning(withAlbumUsers) - .executeTakeFirst() as unknown as Promise; + .executeTakeFirstOrThrow(); } async delete(id: string): Promise { diff --git a/server/src/repositories/asset-job.repository.ts b/server/src/repositories/asset-job.repository.ts index b507fa5445..1bf08e81f5 100644 --- a/server/src/repositories/asset-job.repository.ts +++ b/server/src/repositories/asset-job.repository.ts @@ -8,7 +8,7 @@ import { DummyValue, GenerateSql } from 'src/decorators'; import { withExifInner, withFaces, withFiles } from 'src/entities/asset.entity'; import { AssetFileType } from 'src/enum'; import { StorageAsset } from 'src/types'; -import { asUuid } from 'src/utils/database'; +import { anyUuid, asUuid } from 'src/utils/database'; @Injectable() export class AssetJobRepository { @@ -149,6 +149,21 @@ export class AssetJobRepository { .executeTakeFirst(); } + getForSyncAssets(ids: string[]) { + return this.db + .selectFrom('assets') + .select([ + 'assets.id', + 'assets.isOffline', + 'assets.libraryId', + 'assets.originalPath', + 'assets.status', + 'assets.fileModifiedAt', + ]) + .where('assets.id', '=', anyUuid(ids)) + .execute(); + } + private storageTemplateAssetQuery() { return this.db .selectFrom('assets') diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts index af79fb7c5f..7fb7056ba7 100644 --- a/server/src/repositories/asset.repository.ts +++ b/server/src/repositories/asset.repository.ts @@ -1,9 +1,11 @@ import { Injectable } from '@nestjs/common'; -import { Insertable, Kysely, Selectable, UpdateResult, Updateable, sql } from 'kysely'; +import { Insertable, Kysely, NotNull, Selectable, UpdateResult, Updateable, sql } from 'kysely'; import { isEmpty, isUndefined, omitBy } from 'lodash'; import { InjectKysely } from 'nestjs-kysely'; +import { Stack } from 'src/database'; import { AssetFiles, AssetJobStatus, Assets, DB, Exif } from 'src/db'; import { Chunked, ChunkedArray, DummyValue, GenerateSql } from 'src/decorators'; +import { MapAsset } from 'src/dtos/asset-response.dto'; import { AssetEntity, hasPeople, @@ -23,7 +25,7 @@ import { AssetFileType, AssetOrder, AssetStatus, AssetType } from 'src/enum'; import { AssetSearchOptions, SearchExploreItem, SearchExploreItemSet } from 'src/repositories/search.repository'; import { anyUuid, asUuid, removeUndefinedKeys, unnest } from 'src/utils/database'; import { globToSqlPattern } from 'src/utils/misc'; -import { Paginated, PaginationOptions, paginationHelper } from 'src/utils/pagination'; +import { PaginationOptions, paginationHelper } from 'src/utils/pagination'; export type AssetStats = Record; @@ -141,12 +143,12 @@ export interface GetByIdsRelations { export interface DuplicateGroup { duplicateId: string; - assets: AssetEntity[]; + assets: MapAsset[]; } export interface DayOfYearAssets { yearsAgo: number; - assets: AssetEntity[]; + assets: MapAsset[]; } @Injectable() @@ -234,12 +236,12 @@ export class AssetRepository { .execute(); } - create(asset: Insertable): Promise { - return this.db.insertInto('assets').values(asset).returningAll().executeTakeFirst() as any as Promise; + create(asset: Insertable) { + return this.db.insertInto('assets').values(asset).returningAll().executeTakeFirstOrThrow(); } - createAll(assets: Insertable[]): Promise { - return this.db.insertInto('assets').values(assets).returningAll().execute() as any as Promise; + createAll(assets: Insertable[]) { + return this.db.insertInto('assets').values(assets).returningAll().execute(); } @GenerateSql({ params: [DummyValue.UUID, { day: 1, month: 1 }] }) @@ -299,20 +301,13 @@ export class AssetRepository { @GenerateSql({ params: [[DummyValue.UUID]] }) @ChunkedArray() - getByIds(ids: string[]): Promise { - return ( - this.db - // - .selectFrom('assets') - .selectAll('assets') - .where('assets.id', '=', anyUuid(ids)) - .execute() as Promise - ); + getByIds(ids: string[]) { + return this.db.selectFrom('assets').selectAll('assets').where('assets.id', '=', anyUuid(ids)).execute(); } @GenerateSql({ params: [[DummyValue.UUID]] }) @ChunkedArray() - getByIdsWithAllRelations(ids: string[]): Promise { + getByIdsWithAllRelationsButStacks(ids: string[]) { return this.db .selectFrom('assets') .selectAll('assets') @@ -320,23 +315,8 @@ export class AssetRepository { .select(withTags) .$call(withExif) .leftJoin('asset_stack', 'asset_stack.id', 'assets.stackId') - .leftJoinLateral( - (eb) => - eb - .selectFrom('assets as stacked') - .selectAll('asset_stack') - .select((eb) => eb.fn('array_agg', [eb.table('stacked')]).as('assets')) - .whereRef('stacked.stackId', '=', 'asset_stack.id') - .whereRef('stacked.id', '!=', 'asset_stack.primaryAssetId') - .where('stacked.deletedAt', 'is', null) - .where('stacked.isArchived', '=', false) - .groupBy('asset_stack.id') - .as('stacked_assets'), - (join) => join.on('asset_stack.id', 'is not', null), - ) - .select((eb) => eb.fn.toJson(eb.table('stacked_assets')).as('stack')) .where('assets.id', '=', anyUuid(ids)) - .execute() as any as Promise; + .execute(); } @GenerateSql({ params: [DummyValue.UUID] }) @@ -356,36 +336,29 @@ export class AssetRepository { return assets.map((asset) => asset.deviceAssetId); } - getByUserId( - pagination: PaginationOptions, - userId: string, - options: Omit = {}, - ): Paginated { + getByUserId(pagination: PaginationOptions, userId: string, options: Omit = {}) { return this.getAll(pagination, { ...options, userIds: [userId] }); } @GenerateSql({ params: [DummyValue.UUID, DummyValue.STRING] }) - getByLibraryIdAndOriginalPath(libraryId: string, originalPath: string): Promise { + getByLibraryIdAndOriginalPath(libraryId: string, originalPath: string) { return this.db .selectFrom('assets') .selectAll('assets') .where('libraryId', '=', asUuid(libraryId)) .where('originalPath', '=', originalPath) .limit(1) - .executeTakeFirst() as any as Promise; + .executeTakeFirst(); } - async getAll( - pagination: PaginationOptions, - { orderDirection, ...options }: AssetSearchOptions = {}, - ): Paginated { + async getAll(pagination: PaginationOptions, { orderDirection, ...options }: AssetSearchOptions = {}) { const builder = searchAssetBuilder(this.db, options) .select(withFiles) .orderBy('assets.createdAt', orderDirection ?? 'asc') .limit(pagination.take + 1) .offset(pagination.skip ?? 0); const items = await builder.execute(); - return paginationHelper(items as any as AssetEntity[], pagination.take); + return paginationHelper(items, pagination.take); } /** @@ -420,23 +393,22 @@ export class AssetRepository { } @GenerateSql({ params: [DummyValue.UUID] }) - getById( - id: string, - { exifInfo, faces, files, library, owner, smartSearch, stack, tags }: GetByIdsRelations = {}, - ): Promise { + getById(id: string, { exifInfo, faces, files, library, owner, smartSearch, stack, tags }: GetByIdsRelations = {}) { return this.db .selectFrom('assets') .selectAll('assets') .where('assets.id', '=', asUuid(id)) .$if(!!exifInfo, withExif) - .$if(!!faces, (qb) => qb.select(faces?.person ? withFacesAndPeople : withFaces)) + .$if(!!faces, (qb) => qb.select(faces?.person ? withFacesAndPeople : withFaces).$narrowType<{ faces: NotNull }>()) .$if(!!library, (qb) => qb.select(withLibrary)) .$if(!!owner, (qb) => qb.select(withOwner)) .$if(!!smartSearch, withSmartSearch) .$if(!!stack, (qb) => qb .leftJoin('asset_stack', 'asset_stack.id', 'assets.stackId') - .$if(!stack!.assets, (qb) => qb.select((eb) => eb.fn.toJson(eb.table('asset_stack')).as('stack'))) + .$if(!stack!.assets, (qb) => + qb.select((eb) => eb.fn.toJson(eb.table('asset_stack')).$castTo().as('stack')), + ) .$if(!!stack!.assets, (qb) => qb .leftJoinLateral( @@ -453,13 +425,13 @@ export class AssetRepository { .as('stacked_assets'), (join) => join.on('asset_stack.id', 'is not', null), ) - .select((eb) => eb.fn.toJson(eb.table('stacked_assets')).as('stack')), + .select((eb) => eb.fn.toJson(eb.table('stacked_assets')).$castTo().as('stack')), ), ) .$if(!!files, (qb) => qb.select(withFiles)) .$if(!!tags, (qb) => qb.select(withTags)) .limit(1) - .executeTakeFirst() as any as Promise; + .executeTakeFirst(); } @GenerateSql({ params: [[DummyValue.UUID], { deviceId: DummyValue.STRING }] }) @@ -488,7 +460,7 @@ export class AssetRepository { .execute(); } - async update(asset: Updateable & { id: string }): Promise { + async update(asset: Updateable & { id: string }) { const value = omitBy(asset, isUndefined); delete value.id; if (!isEmpty(value)) { @@ -498,10 +470,10 @@ export class AssetRepository { .selectAll('assets') .$call(withExif) .$call((qb) => qb.select(withFacesAndPeople)) - .executeTakeFirst() as Promise; + .executeTakeFirst(); } - return this.getById(asset.id, { exifInfo: true, faces: { person: true } }) as Promise; + return this.getById(asset.id, { exifInfo: true, faces: { person: true } }); } async remove(asset: { id: string }): Promise { @@ -509,7 +481,7 @@ export class AssetRepository { } @GenerateSql({ params: [{ ownerId: DummyValue.UUID, libraryId: DummyValue.UUID, checksum: DummyValue.BUFFER }] }) - getByChecksum({ ownerId, libraryId, checksum }: AssetGetByChecksumOptions): Promise { + getByChecksum({ ownerId, libraryId, checksum }: AssetGetByChecksumOptions) { return this.db .selectFrom('assets') .selectAll('assets') @@ -517,7 +489,7 @@ export class AssetRepository { .where('checksum', '=', checksum) .$call((qb) => (libraryId ? qb.where('libraryId', '=', asUuid(libraryId)) : qb.where('libraryId', 'is', null))) .limit(1) - .executeTakeFirst() as Promise; + .executeTakeFirst(); } @GenerateSql({ params: [DummyValue.UUID, [DummyValue.BUFFER]] }) @@ -544,7 +516,7 @@ export class AssetRepository { return asset?.id; } - findLivePhotoMatch(options: LivePhotoSearchOptions): Promise { + findLivePhotoMatch(options: LivePhotoSearchOptions) { const { ownerId, otherAssetId, livePhotoCID, type } = options; return this.db .selectFrom('assets') @@ -555,7 +527,7 @@ export class AssetRepository { .where('type', '=', type) .where('exif.livePhotoCID', '=', livePhotoCID) .limit(1) - .executeTakeFirst() as Promise; + .executeTakeFirst(); } @GenerateSql( @@ -564,7 +536,7 @@ export class AssetRepository { params: [DummyValue.PAGINATION, property], })), ) - async getWithout(pagination: PaginationOptions, property: WithoutProperty): Paginated { + async getWithout(pagination: PaginationOptions, property: WithoutProperty) { const items = await this.db .selectFrom('assets') .selectAll('assets') @@ -626,7 +598,7 @@ export class AssetRepository { .orderBy('createdAt') .execute(); - return paginationHelper(items as any as AssetEntity[], pagination.take); + return paginationHelper(items, pagination.take); } getStatistics(ownerId: string, { isArchived, isFavorite, isTrashed }: AssetStatsOptions): Promise { @@ -645,7 +617,7 @@ export class AssetRepository { .executeTakeFirstOrThrow(); } - getRandom(userIds: string[], take: number): Promise { + getRandom(userIds: string[], take: number) { return this.db .selectFrom('assets') .selectAll('assets') @@ -655,7 +627,7 @@ export class AssetRepository { .where('deletedAt', 'is', null) .orderBy((eb) => eb.fn('random')) .limit(take) - .execute() as any as Promise; + .execute(); } @GenerateSql({ params: [{ size: TimeBucketSize.MONTH }] }) @@ -708,7 +680,7 @@ export class AssetRepository { } @GenerateSql({ params: [DummyValue.TIME_BUCKET, { size: TimeBucketSize.MONTH, withStacked: true }] }) - async getTimeBucket(timeBucket: string, options: TimeBucketOptions): Promise { + async getTimeBucket(timeBucket: string, options: TimeBucketOptions) { return this.db .selectFrom('assets') .selectAll('assets') @@ -741,7 +713,7 @@ export class AssetRepository { .as('stacked_assets'), (join) => join.on('asset_stack.id', 'is not', null), ) - .select((eb) => eb.fn.toJson(eb.table('stacked_assets')).as('stack')), + .select((eb) => eb.fn.toJson(eb.table('stacked_assets').$castTo()).as('stack')), ) .$if(!!options.assetType, (qb) => qb.where('assets.type', '=', options.assetType!)) .$if(options.isDuplicate !== undefined, (qb) => @@ -753,11 +725,11 @@ export class AssetRepository { .where('assets.isVisible', '=', true) .where(truncatedDate(options.size), '=', timeBucket.replace(/^[+-]/, '')) .orderBy('assets.localDateTime', options.order ?? 'desc') - .execute() as any as Promise; + .execute(); } @GenerateSql({ params: [DummyValue.UUID] }) - getDuplicates(userId: string): Promise { + getDuplicates(userId: string) { return ( this.db .with('duplicates', (qb) => @@ -774,9 +746,15 @@ export class AssetRepository { (join) => join.onTrue(), ) .select('assets.duplicateId') - .select((eb) => eb.fn('jsonb_agg', [eb.table('asset')]).as('assets')) + .select((eb) => + eb + .fn('jsonb_agg', [eb.table('asset')]) + .$castTo() + .as('assets'), + ) .where('assets.ownerId', '=', asUuid(userId)) .where('assets.duplicateId', 'is not', null) + .$narrowType<{ duplicateId: NotNull }>() .where('assets.deletedAt', 'is', null) .where('assets.isVisible', '=', true) .where('assets.stackId', 'is', null) @@ -801,7 +779,7 @@ export class AssetRepository { .where(({ not, exists }) => not(exists((eb) => eb.selectFrom('unique').whereRef('unique.duplicateId', '=', 'duplicates.duplicateId'))), ) - .execute() as any as Promise + .execute() ); } @@ -845,7 +823,7 @@ export class AssetRepository { }, ], }) - getAllForUserFullSync(options: AssetFullSyncOptions): Promise { + getAllForUserFullSync(options: AssetFullSyncOptions) { const { ownerId, lastId, updatedUntil, limit } = options; return this.db .selectFrom('assets') @@ -863,18 +841,18 @@ export class AssetRepository { .as('stacked_assets'), (join) => join.on('asset_stack.id', 'is not', null), ) - .select((eb) => eb.fn.toJson(eb.table('stacked_assets')).as('stack')) + .select((eb) => eb.fn.toJson(eb.table('stacked_assets')).$castTo().as('stack')) .where('assets.ownerId', '=', asUuid(ownerId)) .where('assets.isVisible', '=', true) .where('assets.updatedAt', '<=', updatedUntil) .$if(!!lastId, (qb) => qb.where('assets.id', '>', lastId!)) .orderBy('assets.id') .limit(limit) - .execute() as any as Promise; + .execute(); } @GenerateSql({ params: [{ userIds: [DummyValue.UUID], updatedAfter: DummyValue.DATE, limit: 100 }] }) - async getChangedDeltaSync(options: AssetDeltaSyncOptions): Promise { + async getChangedDeltaSync(options: AssetDeltaSyncOptions) { return this.db .selectFrom('assets') .selectAll('assets') @@ -891,12 +869,12 @@ export class AssetRepository { .as('stacked_assets'), (join) => join.on('asset_stack.id', 'is not', null), ) - .select((eb) => eb.fn.toJson(eb.table('stacked_assets')).as('stack')) + .select((eb) => eb.fn.toJson(eb.table('stacked_assets').$castTo()).as('stack')) .where('assets.ownerId', '=', anyUuid(options.userIds)) .where('assets.isVisible', '=', true) .where('assets.updatedAt', '>', options.updatedAfter) .limit(options.limit) - .execute() as any as Promise; + .execute(); } async upsertFile(file: Pick, 'assetId' | 'path' | 'type'>): Promise { diff --git a/server/src/repositories/search.repository.ts b/server/src/repositories/search.repository.ts index c86ae8f60e..95c350fe34 100644 --- a/server/src/repositories/search.repository.ts +++ b/server/src/repositories/search.repository.ts @@ -1,13 +1,13 @@ import { Injectable } from '@nestjs/common'; -import { Kysely, OrderByDirection, sql } from 'kysely'; +import { Kysely, OrderByDirection, Selectable, sql } from 'kysely'; import { InjectKysely } from 'nestjs-kysely'; import { randomUUID } from 'node:crypto'; -import { DB } from 'src/db'; +import { DB, Exif } from 'src/db'; import { DummyValue, GenerateSql } from 'src/decorators'; -import { AssetEntity, searchAssetBuilder } from 'src/entities/asset.entity'; +import { MapAsset } from 'src/dtos/asset-response.dto'; +import { searchAssetBuilder } from 'src/entities/asset.entity'; import { AssetStatus, AssetType } from 'src/enum'; import { anyUuid, asUuid } from 'src/utils/database'; -import { Paginated } from 'src/utils/pagination'; import { isValidInteger } from 'src/validation'; export interface SearchResult { @@ -216,7 +216,7 @@ export class SearchRepository { }, ], }) - async searchMetadata(pagination: SearchPaginationOptions, options: AssetSearchOptions): Paginated { + async searchMetadata(pagination: SearchPaginationOptions, options: AssetSearchOptions) { const orderDirection = (options.orderDirection?.toLowerCase() || 'desc') as OrderByDirection; const items = await searchAssetBuilder(this.db, options) .orderBy('assets.fileCreatedAt', orderDirection) @@ -225,7 +225,7 @@ export class SearchRepository { .execute(); const hasNextPage = items.length > pagination.size; items.splice(pagination.size); - return { items: items as any as AssetEntity[], hasNextPage }; + return { items, hasNextPage }; } @GenerateSql({ @@ -240,7 +240,7 @@ export class SearchRepository { }, ], }) - async searchRandom(size: number, options: AssetSearchOptions): Promise { + async searchRandom(size: number, options: AssetSearchOptions) { const uuid = randomUUID(); const builder = searchAssetBuilder(this.db, options); const lessThan = builder @@ -251,8 +251,8 @@ export class SearchRepository { .where('assets.id', '>', uuid) .orderBy(sql`random()`) .limit(size); - const { rows } = await sql`${lessThan} union all ${greaterThan} limit ${size}`.execute(this.db); - return rows as any as AssetEntity[]; + const { rows } = await sql`${lessThan} union all ${greaterThan} limit ${size}`.execute(this.db); + return rows; } @GenerateSql({ @@ -268,17 +268,17 @@ export class SearchRepository { }, ], }) - async searchSmart(pagination: SearchPaginationOptions, options: SmartSearchOptions): Paginated { + async searchSmart(pagination: SearchPaginationOptions, options: SmartSearchOptions) { if (!isValidInteger(pagination.size, { min: 1, max: 1000 })) { throw new Error(`Invalid value for 'size': ${pagination.size}`); } - const items = (await searchAssetBuilder(this.db, options) + const items = await searchAssetBuilder(this.db, options) .innerJoin('smart_search', 'assets.id', 'smart_search.assetId') .orderBy(sql`smart_search.embedding <=> ${options.embedding}`) .limit(pagination.size + 1) .offset((pagination.page - 1) * pagination.size) - .execute()) as any as AssetEntity[]; + .execute(); const hasNextPage = items.length > pagination.size; items.splice(pagination.size); @@ -392,7 +392,7 @@ export class SearchRepository { } @GenerateSql({ params: [[DummyValue.UUID]] }) - getAssetsByCity(userIds: string[]): Promise { + getAssetsByCity(userIds: string[]) { return this.db .withRecursive('cte', (qb) => { const base = qb @@ -434,9 +434,14 @@ export class SearchRepository { .innerJoin('exif', 'assets.id', 'exif.assetId') .innerJoin('cte', 'assets.id', 'cte.assetId') .selectAll('assets') - .select((eb) => eb.fn('to_jsonb', [eb.table('exif')]).as('exifInfo')) + .select((eb) => + eb + .fn('to_jsonb', [eb.table('exif')]) + .$castTo>() + .as('exifInfo'), + ) .orderBy('exif.city') - .execute() as any as Promise; + .execute(); } async upsert(assetId: string, embedding: string): Promise { diff --git a/server/src/repositories/shared-link.repository.ts b/server/src/repositories/shared-link.repository.ts index 272d7f3794..67a97dc2d5 100644 --- a/server/src/repositories/shared-link.repository.ts +++ b/server/src/repositories/shared-link.repository.ts @@ -1,12 +1,12 @@ import { Injectable } from '@nestjs/common'; -import { Insertable, Kysely, sql, Updateable } from 'kysely'; +import { Insertable, Kysely, NotNull, sql, Updateable } from 'kysely'; import { jsonObjectFrom } from 'kysely/helpers/postgres'; import _ from 'lodash'; import { InjectKysely } from 'nestjs-kysely'; -import { columns } from 'src/database'; +import { Album, columns } from 'src/database'; import { DB, SharedLinks } from 'src/db'; import { DummyValue, GenerateSql } from 'src/decorators'; -import { SharedLinkEntity } from 'src/entities/shared-link.entity'; +import { MapAsset } from 'src/dtos/asset-response.dto'; import { SharedLinkType } from 'src/enum'; export type SharedLinkSearchOptions = { @@ -19,7 +19,7 @@ export class SharedLinkRepository { constructor(@InjectKysely() private db: Kysely) {} @GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID] }) - get(userId: string, id: string): Promise { + get(userId: string, id: string) { return this.db .selectFrom('shared_links') .selectAll('shared_links') @@ -87,18 +87,23 @@ export class SharedLinkRepository { .as('album'), (join) => join.onTrue(), ) - .select((eb) => eb.fn.coalesce(eb.fn.jsonAgg('a').filterWhere('a.id', 'is not', null), sql`'[]'`).as('assets')) + .select((eb) => + eb.fn + .coalesce(eb.fn.jsonAgg('a').filterWhere('a.id', 'is not', null), sql`'[]'`) + .$castTo() + .as('assets'), + ) .groupBy(['shared_links.id', sql`"album".*`]) - .select((eb) => eb.fn.toJson('album').as('album')) + .select((eb) => eb.fn.toJson('album').$castTo().as('album')) .where('shared_links.id', '=', id) .where('shared_links.userId', '=', userId) .where((eb) => eb.or([eb('shared_links.type', '=', SharedLinkType.INDIVIDUAL), eb('album.id', 'is not', null)])) .orderBy('shared_links.createdAt', 'desc') - .executeTakeFirst() as Promise; + .executeTakeFirst(); } @GenerateSql({ params: [{ userId: DummyValue.UUID, albumId: DummyValue.UUID }] }) - getAll({ userId, albumId }: SharedLinkSearchOptions): Promise { + getAll({ userId, albumId }: SharedLinkSearchOptions) { return this.db .selectFrom('shared_links') .selectAll('shared_links') @@ -115,6 +120,7 @@ export class SharedLinkRepository { (join) => join.onTrue(), ) .select('assets.assets') + .$narrowType<{ assets: NotNull }>() .leftJoinLateral( (eb) => eb @@ -152,12 +158,12 @@ export class SharedLinkRepository { .as('album'), (join) => join.onTrue(), ) - .select((eb) => eb.fn.toJson('album').as('album')) + .select((eb) => eb.fn.toJson('album').$castTo().as('album')) .where((eb) => eb.or([eb('shared_links.type', '=', SharedLinkType.INDIVIDUAL), eb('album.id', 'is not', null)])) .$if(!!albumId, (eb) => eb.where('shared_links.albumId', '=', albumId!)) .orderBy('shared_links.createdAt', 'desc') .distinctOn(['shared_links.createdAt']) - .execute() as unknown as Promise; + .execute(); } @GenerateSql({ params: [DummyValue.BUFFER] }) @@ -177,7 +183,7 @@ export class SharedLinkRepository { .executeTakeFirst(); } - async create(entity: Insertable & { assetIds?: string[] }): Promise { + async create(entity: Insertable & { assetIds?: string[] }) { const { id } = await this.db .insertInto('shared_links') .values(_.omit(entity, 'assetIds')) @@ -194,7 +200,7 @@ export class SharedLinkRepository { return this.getSharedLinks(id); } - async update(entity: Updateable & { id: string; assetIds?: string[] }): Promise { + async update(entity: Updateable & { id: string; assetIds?: string[] }) { const { id } = await this.db .updateTable('shared_links') .set(_.omit(entity, 'assets', 'album', 'assetIds')) @@ -212,8 +218,8 @@ export class SharedLinkRepository { return this.getSharedLinks(id); } - async remove(entity: SharedLinkEntity): Promise { - await this.db.deleteFrom('shared_links').where('shared_links.id', '=', entity.id).execute(); + async remove(id: string): Promise { + await this.db.deleteFrom('shared_links').where('shared_links.id', '=', id).execute(); } private getSharedLinks(id: string) { @@ -236,9 +242,12 @@ export class SharedLinkRepository { (join) => join.onTrue(), ) .select((eb) => - eb.fn.coalesce(eb.fn.jsonAgg('assets').filterWhere('assets.id', 'is not', null), sql`'[]'`).as('assets'), + eb.fn + .coalesce(eb.fn.jsonAgg('assets').filterWhere('assets.id', 'is not', null), sql`'[]'`) + .$castTo() + .as('assets'), ) .groupBy('shared_links.id') - .executeTakeFirstOrThrow() as Promise; + .executeTakeFirstOrThrow(); } } diff --git a/server/src/repositories/stack.repository.ts b/server/src/repositories/stack.repository.ts index 75dd9b497f..c9d69fb37f 100644 --- a/server/src/repositories/stack.repository.ts +++ b/server/src/repositories/stack.repository.ts @@ -5,7 +5,6 @@ import { InjectKysely } from 'nestjs-kysely'; import { columns } from 'src/database'; import { AssetStack, DB } from 'src/db'; import { DummyValue, GenerateSql } from 'src/decorators'; -import { AssetEntity } from 'src/entities/asset.entity'; import { asUuid } from 'src/utils/database'; export interface StackSearch { @@ -36,9 +35,7 @@ const withAssets = (eb: ExpressionBuilder, withTags = false) .select((eb) => eb.fn.toJson('exifInfo').as('exifInfo')) .where('assets.deletedAt', 'is', null) .whereRef('assets.stackId', '=', 'asset_stack.id'), - ) - .$castTo() - .as('assets'); + ).as('assets'); }; @Injectable() diff --git a/server/src/services/album.service.ts b/server/src/services/album.service.ts index eac000005b..1c612de8c0 100644 --- a/server/src/services/album.service.ts +++ b/server/src/services/album.service.ts @@ -6,15 +6,15 @@ import { AlbumStatisticsResponseDto, CreateAlbumDto, GetAlbumsDto, - UpdateAlbumDto, - UpdateAlbumUserDto, mapAlbum, + MapAlbumDto, mapAlbumWithAssets, mapAlbumWithoutAssets, + UpdateAlbumDto, + UpdateAlbumUserDto, } from 'src/dtos/album.dto'; import { BulkIdResponseDto, BulkIdsDto } from 'src/dtos/asset-ids.response.dto'; import { AuthDto } from 'src/dtos/auth.dto'; -import { AlbumEntity } from 'src/entities/album.entity'; import { Permission } from 'src/enum'; import { AlbumAssetCount, AlbumInfoOptions } from 'src/repositories/album.repository'; import { BaseService } from 'src/services/base.service'; @@ -39,7 +39,7 @@ export class AlbumService extends BaseService { async getAll({ user: { id: ownerId } }: AuthDto, { assetId, shared }: GetAlbumsDto): Promise { await this.albumRepository.updateThumbnails(); - let albums: AlbumEntity[]; + let albums: MapAlbumDto[]; if (assetId) { albums = await this.albumRepository.getByAssetId(ownerId, assetId); } else if (shared === true) { diff --git a/server/src/services/asset-media.service.spec.ts b/server/src/services/asset-media.service.spec.ts index 0c1bbc3cee..a49230f852 100644 --- a/server/src/services/asset-media.service.spec.ts +++ b/server/src/services/asset-media.service.spec.ts @@ -8,6 +8,7 @@ import { Stats } from 'node:fs'; import { AssetFile } from 'src/database'; import { AssetMediaStatus, AssetRejectReason, AssetUploadAction } from 'src/dtos/asset-media-response.dto'; import { AssetMediaCreateDto, AssetMediaReplaceDto, AssetMediaSize, UploadFieldName } from 'src/dtos/asset-media.dto'; +import { MapAsset } from 'src/dtos/asset-response.dto'; import { ASSET_CHECKSUM_CONSTRAINT, AssetEntity } from 'src/entities/asset.entity'; import { AssetFileType, AssetStatus, AssetType, CacheControl, JobName } from 'src/enum'; import { AuthRequest } from 'src/middleware/auth.guard'; @@ -173,7 +174,7 @@ const assetEntity = Object.freeze({ }, livePhotoVideoId: null, sidecarPath: null, -}) as AssetEntity; +} as MapAsset); const existingAsset = Object.freeze({ ...assetEntity, @@ -182,18 +183,18 @@ const existingAsset = Object.freeze({ checksum: Buffer.from('_getExistingAsset', 'utf8'), libraryId: 'libraryId', originalFileName: 'existing-filename.jpeg', -}) as AssetEntity; +}) as MapAsset; const sidecarAsset = Object.freeze({ ...existingAsset, sidecarPath: 'sidecar-path', checksum: Buffer.from('_getExistingAssetWithSideCar', 'utf8'), -}) as AssetEntity; +}) as MapAsset; const copiedAsset = Object.freeze({ id: 'copied-asset', originalPath: 'copied-path', -}) as AssetEntity; +}) as MapAsset; describe(AssetMediaService.name, () => { let sut: AssetMediaService; diff --git a/server/src/services/asset-media.service.ts b/server/src/services/asset-media.service.ts index 2929950f4d..de40d8b304 100644 --- a/server/src/services/asset-media.service.ts +++ b/server/src/services/asset-media.service.ts @@ -2,6 +2,7 @@ import { BadRequestException, Injectable, InternalServerErrorException, NotFound import { extname } from 'node:path'; import sanitize from 'sanitize-filename'; import { StorageCore } from 'src/cores/storage.core'; +import { Asset } from 'src/database'; import { AssetBulkUploadCheckResponseDto, AssetMediaResponseDto, @@ -20,7 +21,7 @@ import { UploadFieldName, } from 'src/dtos/asset-media.dto'; import { AuthDto } from 'src/dtos/auth.dto'; -import { ASSET_CHECKSUM_CONSTRAINT, AssetEntity } from 'src/entities/asset.entity'; +import { ASSET_CHECKSUM_CONSTRAINT } from 'src/entities/asset.entity'; import { AssetStatus, AssetType, CacheControl, JobName, Permission, StorageFolder } from 'src/enum'; import { AuthRequest } from 'src/middleware/auth.guard'; import { BaseService } from 'src/services/base.service'; @@ -212,7 +213,7 @@ export class AssetMediaService extends BaseService { const asset = await this.findOrFail(id); const size = dto.size ?? AssetMediaSize.THUMBNAIL; - const { thumbnailFile, previewFile, fullsizeFile } = getAssetFiles(asset.files); + const { thumbnailFile, previewFile, fullsizeFile } = getAssetFiles(asset.files ?? []); let filepath = previewFile?.path; if (size === AssetMediaSize.THUMBNAIL && thumbnailFile) { filepath = thumbnailFile.path; @@ -375,7 +376,7 @@ export class AssetMediaService extends BaseService { * Uses only vital properties excluding things like: stacks, faces, smart search info, etc, * and then queues a METADATA_EXTRACTION job. */ - private async createCopy(asset: AssetEntity): Promise { + private async createCopy(asset: Omit) { const created = await this.assetRepository.create({ ownerId: asset.ownerId, originalPath: asset.originalPath, @@ -398,12 +399,7 @@ export class AssetMediaService extends BaseService { return created; } - private async create( - ownerId: string, - dto: AssetMediaCreateDto, - file: UploadFile, - sidecarFile?: UploadFile, - ): Promise { + private async create(ownerId: string, dto: AssetMediaCreateDto, file: UploadFile, sidecarFile?: UploadFile) { const asset = await this.assetRepository.create({ ownerId, libraryId: null, @@ -444,7 +440,7 @@ export class AssetMediaService extends BaseService { } } - private async findOrFail(id: string): Promise { + private async findOrFail(id: string) { const asset = await this.assetRepository.getById(id, { files: true }); if (!asset) { throw new NotFoundException('Asset not found'); diff --git a/server/src/services/asset.service.spec.ts b/server/src/services/asset.service.spec.ts index 5fc4984b62..a3f536ac33 100755 --- a/server/src/services/asset.service.spec.ts +++ b/server/src/services/asset.service.spec.ts @@ -1,8 +1,7 @@ import { BadRequestException } from '@nestjs/common'; import { DateTime } from 'luxon'; -import { mapAsset } from 'src/dtos/asset-response.dto'; +import { MapAsset, mapAsset } from 'src/dtos/asset-response.dto'; import { AssetJobName, AssetStatsResponseDto } from 'src/dtos/asset.dto'; -import { AssetEntity } from 'src/entities/asset.entity'; import { AssetStatus, AssetType, JobName, JobStatus } from 'src/enum'; import { AssetStats } from 'src/repositories/asset.repository'; import { AssetService } from 'src/services/asset.service'; @@ -35,7 +34,7 @@ describe(AssetService.name, () => { expect(sut).toBeDefined(); }); - const mockGetById = (assets: AssetEntity[]) => { + const mockGetById = (assets: MapAsset[]) => { mocks.asset.getById.mockImplementation((assetId) => Promise.resolve(assets.find((asset) => asset.id === assetId))); }; @@ -608,7 +607,7 @@ describe(AssetService.name, () => { mocks.asset.getById.mockResolvedValue({ ...assetStub.primaryImage, stack: { ...assetStub.primaryImage.stack, assets: assetStub.primaryImage.stack!.assets.slice(0, 2) }, - } as AssetEntity); + }); await sut.handleAssetDeletion({ id: assetStub.primaryImage.id, deleteOnDisk: true }); diff --git a/server/src/services/asset.service.ts b/server/src/services/asset.service.ts index 1ded79680b..16f60907ba 100644 --- a/server/src/services/asset.service.ts +++ b/server/src/services/asset.service.ts @@ -5,6 +5,7 @@ import { JOBS_ASSET_PAGINATION_SIZE } from 'src/constants'; import { OnJob } from 'src/decorators'; import { AssetResponseDto, + MapAsset, MemoryLaneResponseDto, SanitizedAssetResponseDto, mapAsset, @@ -20,7 +21,6 @@ import { } from 'src/dtos/asset.dto'; import { AuthDto } from 'src/dtos/auth.dto'; import { MemoryLaneDto } from 'src/dtos/search.dto'; -import { AssetEntity } from 'src/entities/asset.entity'; import { AssetStatus, JobName, JobStatus, Permission, QueueName } from 'src/enum'; import { BaseService } from 'src/services/base.service'; import { ISidecarWriteJob, JobItem, JobOf } from 'src/types'; @@ -43,7 +43,7 @@ export class AssetService extends BaseService { yearsAgo, // TODO move this to clients title: `${yearsAgo} year${yearsAgo > 1 ? 's' : ''} ago`, - assets: assets.map((asset) => mapAsset(asset as unknown as AssetEntity, { auth })), + assets: assets.map((asset) => mapAsset(asset, { auth })), }; }); } @@ -105,7 +105,7 @@ export class AssetService extends BaseService { const { description, dateTimeOriginal, latitude, longitude, rating, ...rest } = dto; const repos = { asset: this.assetRepository, event: this.eventRepository }; - let previousMotion: AssetEntity | null = null; + let previousMotion: MapAsset | null = null; if (rest.livePhotoVideoId) { await onBeforeLink(repos, { userId: auth.user.id, livePhotoVideoId: rest.livePhotoVideoId }); } else if (rest.livePhotoVideoId === null) { @@ -233,7 +233,7 @@ export class AssetService extends BaseService { } } - const { fullsizeFile, previewFile, thumbnailFile } = getAssetFiles(asset.files); + const { fullsizeFile, previewFile, thumbnailFile } = getAssetFiles(asset.files ?? []); const files = [thumbnailFile?.path, previewFile?.path, fullsizeFile?.path, asset.encodedVideoPath]; if (deleteOnDisk) { diff --git a/server/src/services/duplicate.service.ts b/server/src/services/duplicate.service.ts index 9f1ac3d4ce..c504b1a305 100644 --- a/server/src/services/duplicate.service.ts +++ b/server/src/services/duplicate.service.ts @@ -68,7 +68,7 @@ export class DuplicateService extends BaseService { return JobStatus.SKIPPED; } - const previewFile = getAssetFile(asset.files, AssetFileType.PREVIEW); + const previewFile = getAssetFile(asset.files || [], AssetFileType.PREVIEW); if (!previewFile) { this.logger.warn(`Asset ${id} is missing preview image`); return JobStatus.FAILED; diff --git a/server/src/services/job.service.spec.ts b/server/src/services/job.service.spec.ts index 134a86b69f..baac0af428 100644 --- a/server/src/services/job.service.spec.ts +++ b/server/src/services/job.service.spec.ts @@ -285,9 +285,9 @@ describe(JobService.name, () => { it(`should queue ${jobs.length} jobs when a ${item.name} job finishes successfully`, async () => { if (item.name === JobName.GENERATE_THUMBNAILS && item.data.source === 'upload') { if (item.data.id === 'asset-live-image') { - mocks.asset.getByIdsWithAllRelations.mockResolvedValue([assetStub.livePhotoStillAsset]); + mocks.asset.getByIdsWithAllRelationsButStacks.mockResolvedValue([assetStub.livePhotoStillAsset as any]); } else { - mocks.asset.getByIdsWithAllRelations.mockResolvedValue([assetStub.livePhotoMotionAsset]); + mocks.asset.getByIdsWithAllRelationsButStacks.mockResolvedValue([assetStub.livePhotoMotionAsset as any]); } } diff --git a/server/src/services/job.service.ts b/server/src/services/job.service.ts index 2f180edd40..edd018d7b1 100644 --- a/server/src/services/job.service.ts +++ b/server/src/services/job.service.ts @@ -254,7 +254,7 @@ export class JobService extends BaseService { case JobName.METADATA_EXTRACTION: { if (item.data.source === 'sidecar-write') { - const [asset] = await this.assetRepository.getByIdsWithAllRelations([item.data.id]); + const [asset] = await this.assetRepository.getByIdsWithAllRelationsButStacks([item.data.id]); if (asset) { this.eventRepository.clientSend('on_asset_update', asset.ownerId, mapAsset(asset)); } @@ -284,7 +284,7 @@ export class JobService extends BaseService { break; } - const [asset] = await this.assetRepository.getByIdsWithAllRelations([item.data.id]); + const [asset] = await this.assetRepository.getByIdsWithAllRelationsButStacks([item.data.id]); if (!asset) { this.logger.warn(`Could not find asset ${item.data.id} after generating thumbnails`); break; diff --git a/server/src/services/library.service.spec.ts b/server/src/services/library.service.spec.ts index aef02b7244..15b150f551 100644 --- a/server/src/services/library.service.spec.ts +++ b/server/src/services/library.service.spec.ts @@ -350,7 +350,7 @@ describe(LibraryService.name, () => { progressCounter: 0, }; - mocks.asset.getByIds.mockResolvedValue([assetStub.external]); + mocks.assetJob.getForSyncAssets.mockResolvedValue([assetStub.external]); mocks.storage.stat.mockRejectedValue(new Error('ENOENT, no such file or directory')); await expect(sut.handleSyncAssets(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); @@ -371,7 +371,7 @@ describe(LibraryService.name, () => { progressCounter: 0, }; - mocks.asset.getByIds.mockResolvedValue([assetStub.external]); + mocks.assetJob.getForSyncAssets.mockResolvedValue([assetStub.external]); mocks.storage.stat.mockRejectedValue(new Error('Could not read file')); await expect(sut.handleSyncAssets(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); @@ -392,7 +392,7 @@ describe(LibraryService.name, () => { progressCounter: 0, }; - mocks.asset.getByIds.mockResolvedValue([assetStub.trashedOffline]); + mocks.assetJob.getForSyncAssets.mockResolvedValue([assetStub.trashedOffline]); mocks.storage.stat.mockRejectedValue(new Error('Could not read file')); await expect(sut.handleSyncAssets(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); @@ -410,7 +410,7 @@ describe(LibraryService.name, () => { progressCounter: 0, }; - mocks.asset.getByIds.mockResolvedValue([assetStub.trashedOffline]); + mocks.assetJob.getForSyncAssets.mockResolvedValue([assetStub.trashedOffline]); mocks.storage.stat.mockResolvedValue({ mtime: assetStub.external.fileModifiedAt } as Stats); await expect(sut.handleSyncAssets(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); @@ -431,7 +431,7 @@ describe(LibraryService.name, () => { progressCounter: 0, }; - mocks.asset.getByIds.mockResolvedValue([assetStub.trashedOffline]); + mocks.assetJob.getForSyncAssets.mockResolvedValue([assetStub.trashedOffline]); mocks.storage.stat.mockResolvedValue({ mtime: assetStub.external.fileModifiedAt } as Stats); await expect(sut.handleSyncAssets(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); @@ -451,7 +451,7 @@ describe(LibraryService.name, () => { progressCounter: 0, }; - mocks.asset.getByIds.mockResolvedValue([assetStub.trashedOffline]); + mocks.assetJob.getForSyncAssets.mockResolvedValue([assetStub.trashedOffline]); mocks.storage.stat.mockResolvedValue({ mtime: assetStub.external.fileModifiedAt } as Stats); await expect(sut.handleSyncAssets(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); @@ -471,7 +471,7 @@ describe(LibraryService.name, () => { progressCounter: 0, }; - mocks.asset.getByIds.mockResolvedValue([assetStub.external]); + mocks.assetJob.getForSyncAssets.mockResolvedValue([assetStub.external]); mocks.storage.stat.mockResolvedValue({ mtime: assetStub.external.fileModifiedAt } as Stats); await expect(sut.handleSyncAssets(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); @@ -489,7 +489,7 @@ describe(LibraryService.name, () => { progressCounter: 0, }; - mocks.asset.getByIds.mockResolvedValue([assetStub.trashedOffline]); + mocks.assetJob.getForSyncAssets.mockResolvedValue([assetStub.trashedOffline]); mocks.storage.stat.mockResolvedValue({ mtime: assetStub.trashedOffline.fileModifiedAt } as Stats); await expect(sut.handleSyncAssets(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); @@ -518,7 +518,7 @@ describe(LibraryService.name, () => { const mtime = new Date(assetStub.external.fileModifiedAt.getDate() + 1); - mocks.asset.getByIds.mockResolvedValue([assetStub.external]); + mocks.assetJob.getForSyncAssets.mockResolvedValue([assetStub.external]); mocks.storage.stat.mockResolvedValue({ mtime } as Stats); await expect(sut.handleSyncAssets(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); diff --git a/server/src/services/library.service.ts b/server/src/services/library.service.ts index 8cc2cf48ff..2add5f484b 100644 --- a/server/src/services/library.service.ts +++ b/server/src/services/library.service.ts @@ -18,7 +18,6 @@ import { ValidateLibraryImportPathResponseDto, ValidateLibraryResponseDto, } from 'src/dtos/library.dto'; -import { AssetEntity } from 'src/entities/asset.entity'; import { AssetStatus, AssetType, DatabaseLock, ImmichWorker, JobName, JobStatus, QueueName } from 'src/enum'; import { ArgOf } from 'src/repositories/event.repository'; import { AssetSyncResult } from 'src/repositories/library.repository'; @@ -467,7 +466,7 @@ export class LibraryService extends BaseService { @OnJob({ name: JobName.LIBRARY_SYNC_ASSETS, queue: QueueName.LIBRARY }) async handleSyncAssets(job: JobOf): Promise { - const assets = await this.assetRepository.getByIds(job.assetIds); + const assets = await this.assetJobRepository.getForSyncAssets(job.assetIds); const assetIdsToOffline: string[] = []; const trashedAssetIdsToOffline: string[] = []; @@ -561,7 +560,16 @@ export class LibraryService extends BaseService { return JobStatus.SUCCESS; } - private checkExistingAsset(asset: AssetEntity, stat: Stats | null): AssetSyncResult { + private checkExistingAsset( + asset: { + isOffline: boolean; + libraryId: string | null; + originalPath: string; + status: AssetStatus; + fileModifiedAt: Date; + }, + stat: Stats | null, + ): AssetSyncResult { if (!stat) { // File not found on disk or permission error if (asset.isOffline) { diff --git a/server/src/services/metadata.service.spec.ts b/server/src/services/metadata.service.spec.ts index ca1277a8c8..e412b1c31f 100644 --- a/server/src/services/metadata.service.spec.ts +++ b/server/src/services/metadata.service.spec.ts @@ -3,7 +3,7 @@ import { randomBytes } from 'node:crypto'; import { Stats } from 'node:fs'; import { constants } from 'node:fs/promises'; import { defaults } from 'src/config'; -import { AssetEntity } from 'src/entities/asset.entity'; +import { MapAsset } from 'src/dtos/asset-response.dto'; import { AssetType, ExifOrientation, ImmichWorker, JobName, JobStatus, SourceType } from 'src/enum'; import { WithoutProperty } from 'src/repositories/asset.repository'; import { ImmichTags } from 'src/repositories/metadata.repository'; @@ -549,7 +549,6 @@ describe(MetadataService.name, () => { livePhotoVideoId: null, libraryId: null, }); - mocks.asset.getByIds.mockResolvedValue([{ ...assetStub.livePhotoWithOriginalFileName, livePhotoVideoId: null }]); mocks.storage.stat.mockResolvedValue({ size: 123_456, mtime: assetStub.livePhotoWithOriginalFileName.fileModifiedAt, @@ -719,7 +718,7 @@ describe(MetadataService.name, () => { }); mocks.crypto.hashSha1.mockReturnValue(randomBytes(512)); mocks.asset.create.mockImplementation( - (asset) => Promise.resolve({ ...assetStub.livePhotoMotionAsset, ...asset }) as Promise, + (asset) => Promise.resolve({ ...assetStub.livePhotoMotionAsset, ...asset }) as Promise, ); const video = randomBytes(512); mocks.storage.readFile.mockResolvedValue(video); @@ -1394,7 +1393,7 @@ describe(MetadataService.name, () => { }); it('should set sidecar path if exists (sidecar named photo.xmp)', async () => { - mocks.asset.getByIds.mockResolvedValue([assetStub.sidecarWithoutExt]); + mocks.asset.getByIds.mockResolvedValue([assetStub.sidecarWithoutExt as any]); mocks.storage.checkFileExists.mockResolvedValueOnce(false); mocks.storage.checkFileExists.mockResolvedValueOnce(true); @@ -1446,7 +1445,7 @@ describe(MetadataService.name, () => { describe('handleSidecarDiscovery', () => { it('should skip hidden assets', async () => { - mocks.asset.getByIds.mockResolvedValue([assetStub.livePhotoMotionAsset]); + mocks.asset.getByIds.mockResolvedValue([assetStub.livePhotoMotionAsset as any]); await sut.handleSidecarDiscovery({ id: assetStub.livePhotoMotionAsset.id }); expect(mocks.storage.checkFileExists).not.toHaveBeenCalled(); }); diff --git a/server/src/services/metadata.service.ts b/server/src/services/metadata.service.ts index ab62c38ed0..faf146a2be 100644 --- a/server/src/services/metadata.service.ts +++ b/server/src/services/metadata.service.ts @@ -271,7 +271,7 @@ export class MetadataService extends BaseService { ]; if (this.isMotionPhoto(asset, exifTags)) { - promises.push(this.applyMotionPhotos(asset as unknown as Asset, exifTags, dates, stats)); + promises.push(this.applyMotionPhotos(asset, exifTags, dates, stats)); } if (isFaceImportEnabled(metadata) && this.hasTaggedFaces(exifTags)) { diff --git a/server/src/services/person.service.ts b/server/src/services/person.service.ts index a413c688f0..66d68857a0 100644 --- a/server/src/services/person.service.ts +++ b/server/src/services/person.service.ts @@ -2,7 +2,8 @@ import { BadRequestException, Injectable, NotFoundException } from '@nestjs/comm import { Insertable, Updateable } from 'kysely'; import { FACE_THUMBNAIL_SIZE, JOBS_ASSET_PAGINATION_SIZE } from 'src/constants'; import { StorageCore } from 'src/cores/storage.core'; -import { AssetFaces, FaceSearch, Person } from 'src/db'; +import { Person } from 'src/database'; +import { AssetFaces, FaceSearch } from 'src/db'; import { Chunked, OnJob } from 'src/decorators'; import { BulkIdErrorReason, BulkIdResponseDto } from 'src/dtos/asset-ids.response.dto'; import { AuthDto } from 'src/dtos/auth.dto'; @@ -315,6 +316,7 @@ export class PersonService extends BaseService { const facesToAdd: (Insertable & { id: string })[] = []; const embeddings: FaceSearch[] = []; const mlFaceIds = new Set(); + for (const face of asset.faces) { if (face.sourceType === SourceType.MACHINE_LEARNING) { mlFaceIds.add(face.id); @@ -477,7 +479,7 @@ export class PersonService extends BaseService { embedding: face.faceSearch.embedding, maxDistance: machineLearning.facialRecognition.maxDistance, numResults: machineLearning.facialRecognition.minFaces, - minBirthDate: face.asset.fileCreatedAt, + minBirthDate: face.asset.fileCreatedAt ?? undefined, }); // `matches` also includes the face itself @@ -503,7 +505,7 @@ export class PersonService extends BaseService { maxDistance: machineLearning.facialRecognition.maxDistance, numResults: 1, hasPerson: true, - minBirthDate: face.asset.fileCreatedAt, + minBirthDate: face.asset.fileCreatedAt ?? undefined, }); if (matchWithPerson.length > 0) { diff --git a/server/src/services/search.service.spec.ts b/server/src/services/search.service.spec.ts index 51c6b55e11..18699edf9a 100644 --- a/server/src/services/search.service.spec.ts +++ b/server/src/services/search.service.spec.ts @@ -45,7 +45,7 @@ describe(SearchService.name, () => { fieldName: 'exifInfo.city', items: [{ value: 'test-city', data: assetStub.withLocation.id }], }); - mocks.asset.getByIdsWithAllRelations.mockResolvedValue([assetStub.withLocation]); + mocks.asset.getByIdsWithAllRelationsButStacks.mockResolvedValue([assetStub.withLocation]); const expectedResponse = [ { fieldName: 'exifInfo.city', items: [{ value: 'test-city', data: mapAsset(assetStub.withLocation) }] }, ]; diff --git a/server/src/services/search.service.ts b/server/src/services/search.service.ts index 1c0c0ad490..442d49136c 100644 --- a/server/src/services/search.service.ts +++ b/server/src/services/search.service.ts @@ -1,5 +1,5 @@ import { BadRequestException, Injectable } from '@nestjs/common'; -import { AssetMapOptions, AssetResponseDto, mapAsset } from 'src/dtos/asset-response.dto'; +import { AssetMapOptions, AssetResponseDto, MapAsset, mapAsset } from 'src/dtos/asset-response.dto'; import { AuthDto } from 'src/dtos/auth.dto'; import { mapPerson, PersonResponseDto } from 'src/dtos/person.dto'; import { @@ -14,7 +14,6 @@ import { SearchSuggestionType, SmartSearchDto, } from 'src/dtos/search.dto'; -import { AssetEntity } from 'src/entities/asset.entity'; import { AssetOrder } from 'src/enum'; import { SearchExploreItem } from 'src/repositories/search.repository'; import { BaseService } from 'src/services/base.service'; @@ -36,7 +35,7 @@ export class SearchService extends BaseService { async getExploreData(auth: AuthDto): Promise[]> { const options = { maxFields: 12, minAssetsPerField: 5 }; const cities = await this.assetRepository.getAssetIdByCity(auth.user.id, options); - const assets = await this.assetRepository.getByIdsWithAllRelations(cities.items.map(({ data }) => data)); + const assets = await this.assetRepository.getByIdsWithAllRelationsButStacks(cities.items.map(({ data }) => data)); const items = assets.map((asset) => ({ value: asset.exifInfo!.city!, data: mapAsset(asset, { auth }) })); return [{ fieldName: cities.fieldName, items }]; } @@ -139,7 +138,7 @@ export class SearchService extends BaseService { return [auth.user.id, ...partnerIds]; } - private mapResponse(assets: AssetEntity[], nextPage: string | null, options: AssetMapOptions): SearchResponseDto { + private mapResponse(assets: MapAsset[], nextPage: string | null, options: AssetMapOptions): SearchResponseDto { return { albums: { total: 0, count: 0, items: [], facets: [] }, assets: { diff --git a/server/src/services/shared-link.service.spec.ts b/server/src/services/shared-link.service.spec.ts index 4d084d6e67..66a0a925c7 100644 --- a/server/src/services/shared-link.service.spec.ts +++ b/server/src/services/shared-link.service.spec.ts @@ -244,7 +244,7 @@ describe(SharedLinkService.name, () => { await sut.remove(authStub.user1, sharedLinkStub.valid.id); expect(mocks.sharedLink.get).toHaveBeenCalledWith(authStub.user1.user.id, sharedLinkStub.valid.id); - expect(mocks.sharedLink.remove).toHaveBeenCalledWith(sharedLinkStub.valid); + expect(mocks.sharedLink.remove).toHaveBeenCalledWith(sharedLinkStub.valid.id); }); }); @@ -333,8 +333,7 @@ describe(SharedLinkService.name, () => { }); it('should return metadata tags with a default image path if the asset id is not set', async () => { - mocks.sharedLink.get.mockResolvedValue({ ...sharedLinkStub.individual, album: undefined, assets: [] }); - + mocks.sharedLink.get.mockResolvedValue({ ...sharedLinkStub.individual, album: null, assets: [] }); await expect(sut.getMetadataTags(authStub.adminSharedLink)).resolves.toEqual({ description: '0 shared photos & videos', imageUrl: `https://my.immich.app/feature-panel.png`, diff --git a/server/src/services/shared-link.service.ts b/server/src/services/shared-link.service.ts index 95f8cef5f8..17f1b974d2 100644 --- a/server/src/services/shared-link.service.ts +++ b/server/src/services/shared-link.service.ts @@ -1,4 +1,5 @@ import { BadRequestException, ForbiddenException, Injectable, UnauthorizedException } from '@nestjs/common'; +import { SharedLink } from 'src/database'; import { AssetIdErrorReason, AssetIdsResponseDto } from 'src/dtos/asset-ids.response.dto'; import { AssetIdsDto } from 'src/dtos/asset.dto'; import { AuthDto } from 'src/dtos/auth.dto'; @@ -11,7 +12,6 @@ import { SharedLinkResponseDto, SharedLinkSearchDto, } from 'src/dtos/shared-link.dto'; -import { SharedLinkEntity } from 'src/entities/shared-link.entity'; import { Permission, SharedLinkType } from 'src/enum'; import { BaseService } from 'src/services/base.service'; import { getExternalDomain, OpenGraphTags } from 'src/utils/misc'; @@ -98,7 +98,7 @@ export class SharedLinkService extends BaseService { async remove(auth: AuthDto, id: string): Promise { const sharedLink = await this.findOrFail(auth.user.id, id); - await this.sharedLinkRepository.remove(sharedLink); + await this.sharedLinkRepository.remove(sharedLink.id); } // TODO: replace `userId` with permissions and access control checks @@ -182,7 +182,7 @@ export class SharedLinkService extends BaseService { const config = await this.getConfig({ withCache: true }); const sharedLink = await this.findOrFail(auth.sharedLink.userId, auth.sharedLink.id); const assetId = sharedLink.album?.albumThumbnailAssetId || sharedLink.assets[0]?.id; - const assetCount = sharedLink.assets.length > 0 ? sharedLink.assets.length : sharedLink.album?.assets.length || 0; + const assetCount = sharedLink.assets.length > 0 ? sharedLink.assets.length : sharedLink.album?.assets?.length || 0; const imagePath = assetId ? `/api/assets/${assetId}/thumbnail?key=${sharedLink.key.toString('base64url')}` : '/feature-panel.png'; @@ -194,11 +194,11 @@ export class SharedLinkService extends BaseService { }; } - private mapToSharedLink(sharedLink: SharedLinkEntity, { withExif }: { withExif: boolean }) { + private mapToSharedLink(sharedLink: SharedLink, { withExif }: { withExif: boolean }) { return withExif ? mapSharedLink(sharedLink) : mapSharedLinkWithoutMetadata(sharedLink); } - private validateAndRefreshToken(sharedLink: SharedLinkEntity, dto: SharedLinkPasswordDto): string { + private validateAndRefreshToken(sharedLink: SharedLink, dto: SharedLinkPasswordDto): string { const token = this.cryptoRepository.hashSha256(`${sharedLink.id}-${sharedLink.password}`); const sharedLinkTokens = dto.token?.split(',') || []; if (sharedLink.password !== dto.password && !sharedLinkTokens.includes(token)) { diff --git a/server/src/services/sync.service.spec.ts b/server/src/services/sync.service.spec.ts index 5f7357c64d..5b50340a9f 100644 --- a/server/src/services/sync.service.spec.ts +++ b/server/src/services/sync.service.spec.ts @@ -1,5 +1,4 @@ import { mapAsset } from 'src/dtos/asset-response.dto'; -import { AssetEntity } from 'src/entities/asset.entity'; import { SyncService } from 'src/services/sync.service'; import { assetStub } from 'test/fixtures/asset.stub'; import { authStub } from 'test/fixtures/auth.stub'; @@ -63,7 +62,7 @@ describe(SyncService.name, () => { it('should return a response requiring a full sync when there are too many changes', async () => { mocks.partner.getAll.mockResolvedValue([]); mocks.asset.getChangedDeltaSync.mockResolvedValue( - Array.from({ length: 10_000 }).fill(assetStub.image), + Array.from({ length: 10_000 }).fill(assetStub.image), ); await expect( sut.getDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }), diff --git a/server/src/services/view.service.ts b/server/src/services/view.service.ts index 5871b04b32..9d1ee3cf89 100644 --- a/server/src/services/view.service.ts +++ b/server/src/services/view.service.ts @@ -1,7 +1,6 @@ import { Injectable } from '@nestjs/common'; import { AssetResponseDto, mapAsset } from 'src/dtos/asset-response.dto'; import { AuthDto } from 'src/dtos/auth.dto'; -import { AssetEntity } from 'src/entities/asset.entity'; import { BaseService } from 'src/services/base.service'; @Injectable() @@ -12,6 +11,6 @@ export class ViewService extends BaseService { async getAssetsByOriginalPath(auth: AuthDto, path: string): Promise { const assets = await this.viewRepository.getAssetsByOriginalPath(auth.user.id, path); - return assets.map((asset) => mapAsset(asset as unknown as AssetEntity, { auth })); + return assets.map((asset) => mapAsset(asset, { auth })); } } diff --git a/server/src/utils/asset.util.ts b/server/src/utils/asset.util.ts index a15f006cda..8905f84165 100644 --- a/server/src/utils/asset.util.ts +++ b/server/src/utils/asset.util.ts @@ -13,11 +13,8 @@ import { PartnerRepository } from 'src/repositories/partner.repository'; import { IBulkAsset, ImmichFile, UploadFile } from 'src/types'; import { checkAccess } from 'src/utils/access'; -export const getAssetFile = ( - files: T[], - type: AssetFileType | GeneratedImageType, -) => { - return (files || []).find((file) => file.type === type); +export const getAssetFile = (files: AssetFile[], type: AssetFileType | GeneratedImageType) => { + return files.find((file) => file.type === type); }; export const getAssetFiles = (files: AssetFile[]) => ({ diff --git a/server/test/fixtures/album.stub.ts b/server/test/fixtures/album.stub.ts index 5a1c141512..fd6a8678a0 100644 --- a/server/test/fixtures/album.stub.ts +++ b/server/test/fixtures/album.stub.ts @@ -1,11 +1,10 @@ -import { AlbumEntity } from 'src/entities/album.entity'; import { AlbumUserRole, AssetOrder } from 'src/enum'; import { assetStub } from 'test/fixtures/asset.stub'; import { authStub } from 'test/fixtures/auth.stub'; import { userStub } from 'test/fixtures/user.stub'; export const albumStub = { - empty: Object.freeze({ + empty: Object.freeze({ id: 'album-1', albumName: 'Empty album', description: '', @@ -21,8 +20,9 @@ export const albumStub = { albumUsers: [], isActivityEnabled: true, order: AssetOrder.DESC, + updateId: '42', }), - sharedWithUser: Object.freeze({ + sharedWithUser: Object.freeze({ id: 'album-2', albumName: 'Empty album shared with user', description: '', @@ -43,8 +43,9 @@ export const albumStub = { ], isActivityEnabled: true, order: AssetOrder.DESC, + updateId: '42', }), - sharedWithMultiple: Object.freeze({ + sharedWithMultiple: Object.freeze({ id: 'album-3', albumName: 'Empty album shared with users', description: '', @@ -69,8 +70,9 @@ export const albumStub = { ], isActivityEnabled: true, order: AssetOrder.DESC, + updateId: '42', }), - sharedWithAdmin: Object.freeze({ + sharedWithAdmin: Object.freeze({ id: 'album-3', albumName: 'Empty album shared with admin', description: '', @@ -91,8 +93,9 @@ export const albumStub = { ], isActivityEnabled: true, order: AssetOrder.DESC, + updateId: '42', }), - oneAsset: Object.freeze({ + oneAsset: Object.freeze({ id: 'album-4', albumName: 'Album with one asset', description: '', @@ -108,8 +111,9 @@ export const albumStub = { albumUsers: [], isActivityEnabled: true, order: AssetOrder.DESC, + updateId: '42', }), - twoAssets: Object.freeze({ + twoAssets: Object.freeze({ id: 'album-4a', albumName: 'Album with two assets', description: '', @@ -125,8 +129,9 @@ export const albumStub = { albumUsers: [], isActivityEnabled: true, order: AssetOrder.DESC, + updateId: '42', }), - emptyWithValidThumbnail: Object.freeze({ + emptyWithValidThumbnail: Object.freeze({ id: 'album-5', albumName: 'Empty album with valid thumbnail', description: '', @@ -142,5 +147,6 @@ export const albumStub = { albumUsers: [], isActivityEnabled: true, order: AssetOrder.DESC, + updateId: '42', }), }; diff --git a/server/test/fixtures/asset.stub.ts b/server/test/fixtures/asset.stub.ts index 16e4f20bb3..d1b8e7cf28 100644 --- a/server/test/fixtures/asset.stub.ts +++ b/server/test/fixtures/asset.stub.ts @@ -1,5 +1,5 @@ -import { AssetFile, Exif } from 'src/database'; -import { AssetEntity } from 'src/entities/asset.entity'; +import { AssetFace, AssetFile, Exif } from 'src/database'; +import { MapAsset } from 'src/dtos/asset-response.dto'; import { AssetFileType, AssetStatus, AssetType } from 'src/enum'; import { StorageAsset } from 'src/types'; import { authStub } from 'test/fixtures/auth.stub'; @@ -26,13 +26,15 @@ const fullsizeFile: AssetFile = { const files: AssetFile[] = [fullsizeFile, previewFile, thumbnailFile]; -export const stackStub = (stackId: string, assets: AssetEntity[]) => { +export const stackStub = (stackId: string, assets: (MapAsset & { exifInfo: Exif })[]) => { return { id: stackId, assets, ownerId: assets[0].ownerId, primaryAsset: assets[0], primaryAssetId: assets[0].id, + createdAt: new Date('2023-02-23T05:06:29.716Z'), + updatedAt: new Date('2023-02-23T05:06:29.716Z'), }; }; @@ -85,9 +87,12 @@ export const assetStub = { isExternal: false, duplicateId: null, isOffline: false, + libraryId: null, + stackId: null, + updateId: '42', }), - noWebpPath: Object.freeze({ + noWebpPath: Object.freeze({ id: 'asset-id', status: AssetStatus.ACTIVE, deviceAssetId: 'device-asset-id', @@ -122,9 +127,12 @@ export const assetStub = { deletedAt: null, duplicateId: null, isOffline: false, + libraryId: null, + stackId: null, + updateId: '42', }), - noThumbhash: Object.freeze({ + noThumbhash: Object.freeze({ id: 'asset-id', status: AssetStatus.ACTIVE, deviceAssetId: 'device-asset-id', @@ -156,6 +164,9 @@ export const assetStub = { deletedAt: null, duplicateId: null, isOffline: false, + libraryId: null, + stackId: null, + updateId: '42', }), primaryImage: Object.freeze({ @@ -195,12 +206,13 @@ export const assetStub = { } as Exif, stackId: 'stack-1', stack: stackStub('stack-1', [ - { id: 'primary-asset-id' } as AssetEntity, - { id: 'stack-child-asset-1' } as AssetEntity, - { id: 'stack-child-asset-2' } as AssetEntity, + { id: 'primary-asset-id' } as MapAsset & { exifInfo: Exif }, + { id: 'stack-child-asset-1' } as MapAsset & { exifInfo: Exif }, + { id: 'stack-child-asset-2' } as MapAsset & { exifInfo: Exif }, ]), duplicateId: null, isOffline: false, + updateId: '42', libraryId: null, }), @@ -229,6 +241,9 @@ export const assetStub = { isExternal: false, livePhotoVideo: null, livePhotoVideoId: null, + updateId: 'foo', + libraryId: null, + stackId: null, sharedLinks: [], originalFileName: 'asset-id.jpg', faces: [], @@ -241,10 +256,10 @@ export const assetStub = { } as Exif, duplicateId: null, isOffline: false, - libraryId: null, + stack: null, }), - trashed: Object.freeze({ + trashed: Object.freeze({ id: 'asset-id', deviceAssetId: 'device-asset-id', fileModifiedAt: new Date('2023-02-23T05:06:29.716Z'), @@ -281,9 +296,12 @@ export const assetStub = { duplicateId: null, isOffline: false, status: AssetStatus.TRASHED, + libraryId: null, + stackId: null, + updateId: '42', }), - trashedOffline: Object.freeze({ + trashedOffline: Object.freeze({ id: 'asset-id', status: AssetStatus.ACTIVE, deviceAssetId: 'device-asset-id', @@ -321,8 +339,10 @@ export const assetStub = { } as Exif, duplicateId: null, isOffline: true, + stackId: null, + updateId: '42', }), - archived: Object.freeze({ + archived: Object.freeze({ id: 'asset-id', status: AssetStatus.ACTIVE, deviceAssetId: 'device-asset-id', @@ -359,9 +379,12 @@ export const assetStub = { } as Exif, duplicateId: null, isOffline: false, + libraryId: null, + stackId: null, + updateId: '42', }), - external: Object.freeze({ + external: Object.freeze({ id: 'asset-id', status: AssetStatus.ACTIVE, deviceAssetId: 'device-asset-id', @@ -397,9 +420,12 @@ export const assetStub = { } as Exif, duplicateId: null, isOffline: false, + updateId: '42', + stackId: null, + stack: null, }), - image1: Object.freeze({ + image1: Object.freeze({ id: 'asset-id-1', status: AssetStatus.ACTIVE, deviceAssetId: 'device-asset-id', @@ -434,9 +460,13 @@ export const assetStub = { } as Exif, duplicateId: null, isOffline: false, + updateId: '42', + stackId: null, + libraryId: null, + stack: null, }), - imageFrom2015: Object.freeze({ + imageFrom2015: Object.freeze({ id: 'asset-id-1', status: AssetStatus.ACTIVE, deviceAssetId: 'device-asset-id', @@ -510,7 +540,9 @@ export const assetStub = { deletedAt: null, duplicateId: null, isOffline: false, + updateId: '42', libraryId: null, + stackId: null, }), livePhotoMotionAsset: Object.freeze({ @@ -527,7 +559,7 @@ export const assetStub = { timeZone: `America/New_York`, }, libraryId: null, - } as AssetEntity & { libraryId: string | null; files: AssetFile[]; exifInfo: Exif }), + } as MapAsset & { faces: AssetFace[]; files: AssetFile[]; exifInfo: Exif }), livePhotoStillAsset: Object.freeze({ id: 'live-photo-still-asset', @@ -544,7 +576,8 @@ export const assetStub = { timeZone: `America/New_York`, }, files, - } as AssetEntity & { libraryId: string | null }), + faces: [] as AssetFace[], + } as MapAsset & { faces: AssetFace[] }), livePhotoWithOriginalFileName: Object.freeze({ id: 'live-photo-still-asset', @@ -562,7 +595,8 @@ export const assetStub = { timeZone: `America/New_York`, }, libraryId: null, - } as AssetEntity & { libraryId: string | null }), + faces: [] as AssetFace[], + } as MapAsset & { faces: AssetFace[] }), withLocation: Object.freeze({ id: 'asset-with-favorite-id', @@ -590,6 +624,9 @@ export const assetStub = { isVisible: true, livePhotoVideo: null, livePhotoVideoId: null, + updateId: 'foo', + libraryId: null, + stackId: null, sharedLinks: [], originalFileName: 'asset-id.ext', faces: [], @@ -604,7 +641,7 @@ export const assetStub = { deletedAt: null, duplicateId: null, isOffline: false, - libraryId: null, + tags: [], }), sidecar: Object.freeze({ @@ -639,10 +676,12 @@ export const assetStub = { deletedAt: null, duplicateId: null, isOffline: false, + updateId: 'foo', libraryId: null, + stackId: null, }), - sidecarWithoutExt: Object.freeze({ + sidecarWithoutExt: Object.freeze({ id: 'asset-id', status: AssetStatus.ACTIVE, deviceAssetId: 'device-asset-id', @@ -676,7 +715,7 @@ export const assetStub = { isOffline: false, }), - hasEncodedVideo: Object.freeze({ + hasEncodedVideo: Object.freeze({ id: 'asset-id', status: AssetStatus.ACTIVE, originalFileName: 'asset-id.ext', @@ -711,9 +750,13 @@ export const assetStub = { deletedAt: null, duplicateId: null, isOffline: false, + updateId: '42', + libraryId: null, + stackId: null, + stack: null, }), - hasFileExtension: Object.freeze({ + hasFileExtension: Object.freeze({ id: 'asset-id', status: AssetStatus.ACTIVE, deviceAssetId: 'device-asset-id', @@ -788,6 +831,9 @@ export const assetStub = { } as Exif, duplicateId: null, isOffline: false, + updateId: '42', + libraryId: null, + stackId: null, }), imageHif: Object.freeze({ @@ -827,5 +873,8 @@ export const assetStub = { } as Exif, duplicateId: null, isOffline: false, + updateId: '42', + libraryId: null, + stackId: null, }), }; diff --git a/server/test/fixtures/auth.stub.ts b/server/test/fixtures/auth.stub.ts index dfa21fc707..9ef55398d3 100644 --- a/server/test/fixtures/auth.stub.ts +++ b/server/test/fixtures/auth.stub.ts @@ -1,6 +1,5 @@ import { Session } from 'src/database'; import { AuthDto } from 'src/dtos/auth.dto'; -import { SharedLinkEntity } from 'src/entities/shared-link.entity'; const authUser = { admin: { @@ -42,14 +41,16 @@ export const authStub = { id: 'token-id', } as Session, }), - adminSharedLink: Object.freeze({ + adminSharedLink: Object.freeze({ user: authUser.admin, sharedLink: { id: '123', showExif: true, allowDownload: true, allowUpload: true, - key: Buffer.from('shared-link-key'), - } as SharedLinkEntity, + expiresAt: null, + password: null, + userId: '42', + }, }), }; diff --git a/server/test/fixtures/shared-link.stub.ts b/server/test/fixtures/shared-link.stub.ts index 4eba0de845..a4d83863c7 100644 --- a/server/test/fixtures/shared-link.stub.ts +++ b/server/test/fixtures/shared-link.stub.ts @@ -1,10 +1,9 @@ import { UserAdmin } from 'src/database'; import { AlbumResponseDto } from 'src/dtos/album.dto'; -import { AssetResponseDto } from 'src/dtos/asset-response.dto'; +import { AssetResponseDto, MapAsset } from 'src/dtos/asset-response.dto'; import { ExifResponseDto } from 'src/dtos/exif.dto'; import { SharedLinkResponseDto } from 'src/dtos/shared-link.dto'; import { mapUser } from 'src/dtos/user.dto'; -import { SharedLinkEntity } from 'src/entities/shared-link.entity'; import { AssetOrder, AssetStatus, AssetType, SharedLinkType } from 'src/enum'; import { assetStub } from 'test/fixtures/asset.stub'; import { authStub } from 'test/fixtures/auth.stub'; @@ -113,12 +112,12 @@ export const sharedLinkStub = { allowUpload: true, allowDownload: true, showExif: true, - album: undefined, + albumId: null, + album: null, description: null, assets: [assetStub.image], password: 'password', - albumId: null, - } as SharedLinkEntity), + }), valid: Object.freeze({ id: '123', userId: authStub.admin.user.id, @@ -130,12 +129,12 @@ export const sharedLinkStub = { allowUpload: true, allowDownload: true, showExif: true, - album: undefined, albumId: null, description: null, password: null, - assets: [], - } as SharedLinkEntity), + assets: [] as MapAsset[], + album: null, + }), expired: Object.freeze({ id: '123', userId: authStub.admin.user.id, @@ -150,9 +149,10 @@ export const sharedLinkStub = { description: null, password: null, albumId: null, - assets: [], - } as SharedLinkEntity), - readonlyNoExif: Object.freeze({ + assets: [] as MapAsset[], + album: null, + }), + readonlyNoExif: Object.freeze({ id: '123', userId: authStub.admin.user.id, key: sharedLinkBytes, @@ -168,6 +168,7 @@ export const sharedLinkStub = { albumId: 'album-123', album: { id: 'album-123', + updateId: '42', ownerId: authStub.admin.user.id, owner: userStub.admin, albumName: 'Test Album', @@ -239,17 +240,22 @@ export const sharedLinkStub = { colorspace: 'sRGB', autoStackId: null, rating: 3, + updatedAt: today, + updateId: '42', }, sharedLinks: [], faces: [], sidecarPath: null, deletedAt: null, duplicateId: null, + updateId: '42', + libraryId: null, + stackId: null, }, ], }, }), - passwordRequired: Object.freeze({ + passwordRequired: Object.freeze({ id: '123', userId: authStub.admin.user.id, key: sharedLinkBytes, @@ -263,6 +269,7 @@ export const sharedLinkStub = { password: 'password', assets: [], albumId: null, + album: null, }), }; diff --git a/server/test/medium/specs/services/memory.service.spec.ts b/server/test/medium/specs/services/memory.service.spec.ts index 172c48ca5b..445434d60a 100644 --- a/server/test/medium/specs/services/memory.service.spec.ts +++ b/server/test/medium/specs/services/memory.service.spec.ts @@ -39,9 +39,12 @@ describe(MemoryService.name, () => { it('should create a memory from an asset', async () => { const { sut, repos, getRepository } = createSut(); - const now = DateTime.fromObject({ year: 2025, month: 2, day: 25 }, { zone: 'utc' }); + const now = DateTime.fromObject({ year: 2025, month: 2, day: 25 }, { zone: 'utc' }) as DateTime; const user = mediumFactory.userInsert(); - const asset = mediumFactory.assetInsert({ ownerId: user.id, localDateTime: now.minus({ years: 1 }).toISO() }); + const asset = mediumFactory.assetInsert({ + ownerId: user.id, + localDateTime: now.minus({ years: 1 }).toISO(), + }); const jobStatus = mediumFactory.assetJobStatusInsert({ assetId: asset.id }); const userRepo = getRepository('user'); @@ -86,7 +89,7 @@ describe(MemoryService.name, () => { it('should not generate a memory twice for the same day', async () => { const { sut, repos, getRepository } = createSut(); - const now = DateTime.fromObject({ year: 2025, month: 2, day: 20 }, { zone: 'utc' }); + const now = DateTime.fromObject({ year: 2025, month: 2, day: 20 }, { zone: 'utc' }) as DateTime; const assetRepo = getRepository('asset'); const memoryRepo = getRepository('memory'); diff --git a/server/test/medium/specs/services/metadata.service.spec.ts b/server/test/medium/specs/services/metadata.service.spec.ts index b25cce2724..13b9867373 100644 --- a/server/test/medium/specs/services/metadata.service.spec.ts +++ b/server/test/medium/specs/services/metadata.service.spec.ts @@ -118,7 +118,7 @@ describe(MetadataService.name, () => { process.env.TZ = serverTimeZone ?? undefined; const { filePath } = await createTestFile(exifData); - mocks.assetJob.getForMetadataExtraction.mockResolvedValue({ id: 'asset-1', originalPath: filePath } as never); + mocks.assetJob.getForMetadataExtraction.mockResolvedValue({ id: 'asset-1', originalPath: filePath } as any); await sut.handleMetadataExtraction({ id: 'asset-1' }); diff --git a/server/test/repositories/asset.repository.mock.ts b/server/test/repositories/asset.repository.mock.ts index 2418b6aa64..d540e55b2a 100644 --- a/server/test/repositories/asset.repository.mock.ts +++ b/server/test/repositories/asset.repository.mock.ts @@ -11,7 +11,7 @@ export const newAssetRepositoryMock = (): Mocked = {}) => { }; }; -const assetFactory = (asset: Partial = {}) => ({ +const assetFactory = (asset: Partial = {}) => ({ id: newUuid(), createdAt: newDate(), updatedAt: newDate(), From dd1fcd5be50a3b39dba55ff5610883997674d3d7 Mon Sep 17 00:00:00 2001 From: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com> Date: Fri, 18 Apr 2025 23:39:56 +0200 Subject: [PATCH 12/12] chore: remove asset entity (#17703) --- server/src/database.ts | 7 +- server/src/entities/asset.entity.ts | 272 ------------------ .../src/repositories/asset-job.repository.ts | 3 +- server/src/repositories/asset.repository.ts | 18 +- server/src/repositories/search.repository.ts | 3 +- server/src/repositories/view-repository.ts | 3 +- server/src/schema/tables/asset.table.ts | 2 +- .../src/services/asset-media.service.spec.ts | 8 +- server/src/services/asset-media.service.ts | 2 +- server/src/services/memory.service.spec.ts | 8 + server/src/services/search.service.spec.ts | 23 ++ server/src/utils/database.ts | 231 +++++++++++++++ 12 files changed, 281 insertions(+), 299 deletions(-) delete mode 100644 server/src/entities/asset.entity.ts diff --git a/server/src/database.ts b/server/src/database.ts index 6e66cb4d3d..27094958ed 100644 --- a/server/src/database.ts +++ b/server/src/database.ts @@ -1,7 +1,6 @@ import { Selectable } from 'kysely'; -import { Albums, AssetJobStatus as DatabaseAssetJobStatus, Exif as DatabaseExif } from 'src/db'; +import { Albums, Exif as DatabaseExif } from 'src/db'; import { MapAsset } from 'src/dtos/asset-response.dto'; -import { AssetEntity } from 'src/entities/asset.entity'; import { AlbumUserRole, AssetFileType, @@ -265,10 +264,6 @@ export type AssetFace = { person?: Person | null; }; -export type AssetJobStatus = Selectable & { - asset: AssetEntity; -}; - const userColumns = ['id', 'name', 'email', 'profileImagePath', 'profileChangedAt'] as const; export const columns = { diff --git a/server/src/entities/asset.entity.ts b/server/src/entities/asset.entity.ts deleted file mode 100644 index da291292e7..0000000000 --- a/server/src/entities/asset.entity.ts +++ /dev/null @@ -1,272 +0,0 @@ -import { DeduplicateJoinsPlugin, ExpressionBuilder, Kysely, SelectQueryBuilder, sql } from 'kysely'; -import { jsonArrayFrom, jsonObjectFrom } from 'kysely/helpers/postgres'; -import { AssetFace, AssetFile, AssetJobStatus, columns, Exif, Person, Stack, Tag, User } from 'src/database'; -import { DB } from 'src/db'; -import { MapAsset } from 'src/dtos/asset-response.dto'; -import { AssetFileType, AssetStatus, AssetType } from 'src/enum'; -import { TimeBucketSize } from 'src/repositories/asset.repository'; -import { AssetSearchBuilderOptions } from 'src/repositories/search.repository'; -import { anyUuid, asUuid, toJson } from 'src/utils/database'; - -export const ASSET_CHECKSUM_CONSTRAINT = 'UQ_assets_owner_checksum'; - -export class AssetEntity { - id!: string; - deviceAssetId!: string; - owner!: User; - ownerId!: string; - libraryId?: string | null; - deviceId!: string; - type!: AssetType; - status!: AssetStatus; - originalPath!: string; - files!: AssetFile[]; - thumbhash!: Buffer | null; - encodedVideoPath!: string | null; - createdAt!: Date; - updatedAt!: Date; - updateId?: string; - deletedAt!: Date | null; - fileCreatedAt!: Date; - localDateTime!: Date; - fileModifiedAt!: Date; - isFavorite!: boolean; - isArchived!: boolean; - isExternal!: boolean; - isOffline!: boolean; - checksum!: Buffer; // sha1 checksum - duration!: string | null; - isVisible!: boolean; - livePhotoVideo!: MapAsset | null; - livePhotoVideoId!: string | null; - originalFileName!: string; - sidecarPath!: string | null; - exifInfo?: Exif; - tags?: Tag[]; - faces!: AssetFace[]; - stackId?: string | null; - stack?: Stack | null; - jobStatus?: AssetJobStatus; - duplicateId!: string | null; -} - -// TODO come up with a better query that only selects the fields we need -export function withExif(qb: SelectQueryBuilder) { - return qb - .leftJoin('exif', 'assets.id', 'exif.assetId') - .select((eb) => eb.fn.toJson(eb.table('exif')).$castTo().as('exifInfo')); -} - -export function withExifInner(qb: SelectQueryBuilder) { - return qb - .innerJoin('exif', 'assets.id', 'exif.assetId') - .select((eb) => eb.fn.toJson(eb.table('exif')).$castTo().as('exifInfo')); -} - -export function withSmartSearch(qb: SelectQueryBuilder) { - return qb - .leftJoin('smart_search', 'assets.id', 'smart_search.assetId') - .select((eb) => toJson(eb, 'smart_search').as('smartSearch')); -} - -export function withFaces(eb: ExpressionBuilder, withDeletedFace?: boolean) { - return jsonArrayFrom( - eb - .selectFrom('asset_faces') - .selectAll('asset_faces') - .whereRef('asset_faces.assetId', '=', 'assets.id') - .$if(!withDeletedFace, (qb) => qb.where('asset_faces.deletedAt', 'is', null)), - ).as('faces'); -} - -export function withFiles(eb: ExpressionBuilder, type?: AssetFileType) { - return jsonArrayFrom( - eb - .selectFrom('asset_files') - .select(columns.assetFiles) - .whereRef('asset_files.assetId', '=', 'assets.id') - .$if(!!type, (qb) => qb.where('asset_files.type', '=', type!)), - ).as('files'); -} - -export function withFacesAndPeople(eb: ExpressionBuilder, withDeletedFace?: boolean) { - return jsonArrayFrom( - eb - .selectFrom('asset_faces') - .leftJoinLateral( - (eb) => - eb.selectFrom('person').selectAll('person').whereRef('asset_faces.personId', '=', 'person.id').as('person'), - (join) => join.onTrue(), - ) - .selectAll('asset_faces') - .select((eb) => eb.table('person').$castTo().as('person')) - .whereRef('asset_faces.assetId', '=', 'assets.id') - .$if(!withDeletedFace, (qb) => qb.where('asset_faces.deletedAt', 'is', null)), - ).as('faces'); -} - -export function hasPeople(qb: SelectQueryBuilder, personIds: string[]) { - return qb.innerJoin( - (eb) => - eb - .selectFrom('asset_faces') - .select('assetId') - .where('personId', '=', anyUuid(personIds!)) - .where('deletedAt', 'is', null) - .groupBy('assetId') - .having((eb) => eb.fn.count('personId').distinct(), '=', personIds.length) - .as('has_people'), - (join) => join.onRef('has_people.assetId', '=', 'assets.id'), - ); -} - -export function hasTags(qb: SelectQueryBuilder, tagIds: string[]) { - return qb.innerJoin( - (eb) => - eb - .selectFrom('tag_asset') - .select('assetsId') - .innerJoin('tags_closure', 'tag_asset.tagsId', 'tags_closure.id_descendant') - .where('tags_closure.id_ancestor', '=', anyUuid(tagIds)) - .groupBy('assetsId') - .having((eb) => eb.fn.count('tags_closure.id_ancestor').distinct(), '>=', tagIds.length) - .as('has_tags'), - (join) => join.onRef('has_tags.assetsId', '=', 'assets.id'), - ); -} - -export function withOwner(eb: ExpressionBuilder) { - return jsonObjectFrom(eb.selectFrom('users').select(columns.user).whereRef('users.id', '=', 'assets.ownerId')).as( - 'owner', - ); -} - -export function withLibrary(eb: ExpressionBuilder) { - return jsonObjectFrom( - eb.selectFrom('libraries').selectAll('libraries').whereRef('libraries.id', '=', 'assets.libraryId'), - ).as('library'); -} - -export function withTags(eb: ExpressionBuilder) { - return jsonArrayFrom( - eb - .selectFrom('tags') - .select(columns.tag) - .innerJoin('tag_asset', 'tags.id', 'tag_asset.tagsId') - .whereRef('assets.id', '=', 'tag_asset.assetsId'), - ).as('tags'); -} - -export function truncatedDate(size: TimeBucketSize) { - return sql`date_trunc(${size}, "localDateTime" at time zone 'UTC') at time zone 'UTC'`; -} - -export function withTagId(qb: SelectQueryBuilder, tagId: string) { - return qb.where((eb) => - eb.exists( - eb - .selectFrom('tags_closure') - .innerJoin('tag_asset', 'tag_asset.tagsId', 'tags_closure.id_descendant') - .whereRef('tag_asset.assetsId', '=', 'assets.id') - .where('tags_closure.id_ancestor', '=', tagId), - ), - ); -} - -const joinDeduplicationPlugin = new DeduplicateJoinsPlugin(); - -/** TODO: This should only be used for search-related queries, not as a general purpose query builder */ -export function searchAssetBuilder(kysely: Kysely, options: AssetSearchBuilderOptions) { - options.isArchived ??= options.withArchived ? undefined : false; - options.withDeleted ||= !!(options.trashedAfter || options.trashedBefore || options.isOffline); - return kysely - .withPlugin(joinDeduplicationPlugin) - .selectFrom('assets') - .selectAll('assets') - .$if(!!options.tagIds && options.tagIds.length > 0, (qb) => hasTags(qb, options.tagIds!)) - .$if(!!options.personIds && options.personIds.length > 0, (qb) => hasPeople(qb, options.personIds!)) - .$if(!!options.createdBefore, (qb) => qb.where('assets.createdAt', '<=', options.createdBefore!)) - .$if(!!options.createdAfter, (qb) => qb.where('assets.createdAt', '>=', options.createdAfter!)) - .$if(!!options.updatedBefore, (qb) => qb.where('assets.updatedAt', '<=', options.updatedBefore!)) - .$if(!!options.updatedAfter, (qb) => qb.where('assets.updatedAt', '>=', options.updatedAfter!)) - .$if(!!options.trashedBefore, (qb) => qb.where('assets.deletedAt', '<=', options.trashedBefore!)) - .$if(!!options.trashedAfter, (qb) => qb.where('assets.deletedAt', '>=', options.trashedAfter!)) - .$if(!!options.takenBefore, (qb) => qb.where('assets.fileCreatedAt', '<=', options.takenBefore!)) - .$if(!!options.takenAfter, (qb) => qb.where('assets.fileCreatedAt', '>=', options.takenAfter!)) - .$if(options.city !== undefined, (qb) => - qb - .innerJoin('exif', 'assets.id', 'exif.assetId') - .where('exif.city', options.city === null ? 'is' : '=', options.city!), - ) - .$if(options.state !== undefined, (qb) => - qb - .innerJoin('exif', 'assets.id', 'exif.assetId') - .where('exif.state', options.state === null ? 'is' : '=', options.state!), - ) - .$if(options.country !== undefined, (qb) => - qb - .innerJoin('exif', 'assets.id', 'exif.assetId') - .where('exif.country', options.country === null ? 'is' : '=', options.country!), - ) - .$if(options.make !== undefined, (qb) => - qb - .innerJoin('exif', 'assets.id', 'exif.assetId') - .where('exif.make', options.make === null ? 'is' : '=', options.make!), - ) - .$if(options.model !== undefined, (qb) => - qb - .innerJoin('exif', 'assets.id', 'exif.assetId') - .where('exif.model', options.model === null ? 'is' : '=', options.model!), - ) - .$if(options.lensModel !== undefined, (qb) => - qb - .innerJoin('exif', 'assets.id', 'exif.assetId') - .where('exif.lensModel', options.lensModel === null ? 'is' : '=', options.lensModel!), - ) - .$if(options.rating !== undefined, (qb) => - qb - .innerJoin('exif', 'assets.id', 'exif.assetId') - .where('exif.rating', options.rating === null ? 'is' : '=', options.rating!), - ) - .$if(!!options.checksum, (qb) => qb.where('assets.checksum', '=', options.checksum!)) - .$if(!!options.deviceAssetId, (qb) => qb.where('assets.deviceAssetId', '=', options.deviceAssetId!)) - .$if(!!options.deviceId, (qb) => qb.where('assets.deviceId', '=', options.deviceId!)) - .$if(!!options.id, (qb) => qb.where('assets.id', '=', asUuid(options.id!))) - .$if(!!options.libraryId, (qb) => qb.where('assets.libraryId', '=', asUuid(options.libraryId!))) - .$if(!!options.userIds, (qb) => qb.where('assets.ownerId', '=', anyUuid(options.userIds!))) - .$if(!!options.encodedVideoPath, (qb) => qb.where('assets.encodedVideoPath', '=', options.encodedVideoPath!)) - .$if(!!options.originalPath, (qb) => - qb.where(sql`f_unaccent(assets."originalPath")`, 'ilike', sql`'%' || f_unaccent(${options.originalPath}) || '%'`), - ) - .$if(!!options.originalFileName, (qb) => - qb.where( - sql`f_unaccent(assets."originalFileName")`, - 'ilike', - sql`'%' || f_unaccent(${options.originalFileName}) || '%'`, - ), - ) - .$if(!!options.description, (qb) => - qb - .innerJoin('exif', 'assets.id', 'exif.assetId') - .where(sql`f_unaccent(exif.description)`, 'ilike', sql`'%' || f_unaccent(${options.description}) || '%'`), - ) - .$if(!!options.type, (qb) => qb.where('assets.type', '=', options.type!)) - .$if(options.isFavorite !== undefined, (qb) => qb.where('assets.isFavorite', '=', options.isFavorite!)) - .$if(options.isOffline !== undefined, (qb) => qb.where('assets.isOffline', '=', options.isOffline!)) - .$if(options.isVisible !== undefined, (qb) => qb.where('assets.isVisible', '=', options.isVisible!)) - .$if(options.isArchived !== undefined, (qb) => qb.where('assets.isArchived', '=', options.isArchived!)) - .$if(options.isEncoded !== undefined, (qb) => - qb.where('assets.encodedVideoPath', options.isEncoded ? 'is not' : 'is', null), - ) - .$if(options.isMotion !== undefined, (qb) => - qb.where('assets.livePhotoVideoId', options.isMotion ? 'is not' : 'is', null), - ) - .$if(!!options.isNotInAlbum, (qb) => - qb.where((eb) => - eb.not(eb.exists((eb) => eb.selectFrom('albums_assets_assets').whereRef('assetsId', '=', 'assets.id'))), - ), - ) - .$if(!!options.withExif, withExifInner) - .$if(!!(options.withFaces || options.withPeople || options.personIds), (qb) => qb.select(withFacesAndPeople)) - .$if(!options.withDeleted, (qb) => qb.where('assets.deletedAt', 'is', null)); -} diff --git a/server/src/repositories/asset-job.repository.ts b/server/src/repositories/asset-job.repository.ts index 1bf08e81f5..4a2d52566f 100644 --- a/server/src/repositories/asset-job.repository.ts +++ b/server/src/repositories/asset-job.repository.ts @@ -5,10 +5,9 @@ import { InjectKysely } from 'nestjs-kysely'; import { columns } from 'src/database'; import { DB } from 'src/db'; import { DummyValue, GenerateSql } from 'src/decorators'; -import { withExifInner, withFaces, withFiles } from 'src/entities/asset.entity'; import { AssetFileType } from 'src/enum'; import { StorageAsset } from 'src/types'; -import { anyUuid, asUuid } from 'src/utils/database'; +import { anyUuid, asUuid, withExifInner, withFaces, withFiles } from 'src/utils/database'; @Injectable() export class AssetJobRepository { diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts index 7fb7056ba7..7a68ba907f 100644 --- a/server/src/repositories/asset.repository.ts +++ b/server/src/repositories/asset.repository.ts @@ -6,11 +6,16 @@ import { Stack } from 'src/database'; import { AssetFiles, AssetJobStatus, Assets, DB, Exif } from 'src/db'; import { Chunked, ChunkedArray, DummyValue, GenerateSql } from 'src/decorators'; import { MapAsset } from 'src/dtos/asset-response.dto'; +import { AssetFileType, AssetOrder, AssetStatus, AssetType } from 'src/enum'; +import { AssetSearchOptions, SearchExploreItem, SearchExploreItemSet } from 'src/repositories/search.repository'; import { - AssetEntity, + anyUuid, + asUuid, hasPeople, + removeUndefinedKeys, searchAssetBuilder, truncatedDate, + unnest, withExif, withFaces, withFacesAndPeople, @@ -20,10 +25,7 @@ import { withSmartSearch, withTagId, withTags, -} from 'src/entities/asset.entity'; -import { AssetFileType, AssetOrder, AssetStatus, AssetType } from 'src/enum'; -import { AssetSearchOptions, SearchExploreItem, SearchExploreItemSet } from 'src/repositories/search.repository'; -import { anyUuid, asUuid, removeUndefinedKeys, unnest } from 'src/utils/database'; +} from 'src/utils/database'; import { globToSqlPattern } from 'src/utils/misc'; import { PaginationOptions, paginationHelper } from 'src/utils/pagination'; @@ -128,8 +130,6 @@ export interface AssetGetByChecksumOptions { libraryId?: string; } -export type AssetPathEntity = Pick; - export interface GetByIdsRelations { exifInfo?: boolean; faces?: { person?: boolean; withDeleted?: boolean }; @@ -493,13 +493,13 @@ export class AssetRepository { } @GenerateSql({ params: [DummyValue.UUID, [DummyValue.BUFFER]] }) - getByChecksums(userId: string, checksums: Buffer[]): Promise { + getByChecksums(userId: string, checksums: Buffer[]) { return this.db .selectFrom('assets') .select(['id', 'checksum', 'deletedAt']) .where('ownerId', '=', asUuid(userId)) .where('checksum', 'in', checksums) - .execute() as any as Promise; + .execute(); } @GenerateSql({ params: [DummyValue.UUID, DummyValue.BUFFER] }) diff --git a/server/src/repositories/search.repository.ts b/server/src/repositories/search.repository.ts index 95c350fe34..5c1ebae69d 100644 --- a/server/src/repositories/search.repository.ts +++ b/server/src/repositories/search.repository.ts @@ -5,9 +5,8 @@ import { randomUUID } from 'node:crypto'; import { DB, Exif } from 'src/db'; import { DummyValue, GenerateSql } from 'src/decorators'; import { MapAsset } from 'src/dtos/asset-response.dto'; -import { searchAssetBuilder } from 'src/entities/asset.entity'; import { AssetStatus, AssetType } from 'src/enum'; -import { anyUuid, asUuid } from 'src/utils/database'; +import { anyUuid, asUuid, searchAssetBuilder } from 'src/utils/database'; import { isValidInteger } from 'src/validation'; export interface SearchResult { diff --git a/server/src/repositories/view-repository.ts b/server/src/repositories/view-repository.ts index ae2303e9e2..e32933065c 100644 --- a/server/src/repositories/view-repository.ts +++ b/server/src/repositories/view-repository.ts @@ -2,8 +2,7 @@ import { Kysely } from 'kysely'; import { InjectKysely } from 'nestjs-kysely'; import { DB } from 'src/db'; import { DummyValue, GenerateSql } from 'src/decorators'; -import { withExif } from 'src/entities/asset.entity'; -import { asUuid } from 'src/utils/database'; +import { asUuid, withExif } from 'src/utils/database'; export class ViewRepository { constructor(@InjectKysely() private db: Kysely) {} diff --git a/server/src/schema/tables/asset.table.ts b/server/src/schema/tables/asset.table.ts index 9a9670cd0e..19ec8d2ef4 100644 --- a/server/src/schema/tables/asset.table.ts +++ b/server/src/schema/tables/asset.table.ts @@ -1,5 +1,4 @@ import { UpdatedAtTrigger, UpdateIdColumn } from 'src/decorators'; -import { ASSET_CHECKSUM_CONSTRAINT } from 'src/entities/asset.entity'; import { AssetStatus, AssetType } from 'src/enum'; import { assets_status_enum } from 'src/schema/enums'; import { assets_delete_audit } from 'src/schema/functions'; @@ -17,6 +16,7 @@ import { Table, UpdateDateColumn, } from 'src/sql-tools'; +import { ASSET_CHECKSUM_CONSTRAINT } from 'src/utils/database'; @Table('assets') @UpdatedAtTrigger('assets_updated_at') diff --git a/server/src/services/asset-media.service.spec.ts b/server/src/services/asset-media.service.spec.ts index a49230f852..d25067f1c9 100644 --- a/server/src/services/asset-media.service.spec.ts +++ b/server/src/services/asset-media.service.spec.ts @@ -9,10 +9,10 @@ import { AssetFile } from 'src/database'; import { AssetMediaStatus, AssetRejectReason, AssetUploadAction } from 'src/dtos/asset-media-response.dto'; import { AssetMediaCreateDto, AssetMediaReplaceDto, AssetMediaSize, UploadFieldName } from 'src/dtos/asset-media.dto'; import { MapAsset } from 'src/dtos/asset-response.dto'; -import { ASSET_CHECKSUM_CONSTRAINT, AssetEntity } from 'src/entities/asset.entity'; import { AssetFileType, AssetStatus, AssetType, CacheControl, JobName } from 'src/enum'; import { AuthRequest } from 'src/middleware/auth.guard'; import { AssetMediaService } from 'src/services/asset-media.service'; +import { ASSET_CHECKSUM_CONSTRAINT } from 'src/utils/database'; import { ImmichFileResponse } from 'src/utils/file'; import { assetStub } from 'test/fixtures/asset.stub'; import { authStub } from 'test/fixtures/auth.stub'; @@ -820,8 +820,8 @@ describe(AssetMediaService.name, () => { const file2 = Buffer.from('53be335e99f18a66ff12e9a901c7a6171dd76573', 'hex'); mocks.asset.getByChecksums.mockResolvedValue([ - { id: 'asset-1', checksum: file1 } as AssetEntity, - { id: 'asset-2', checksum: file2 } as AssetEntity, + { id: 'asset-1', checksum: file1, deletedAt: null }, + { id: 'asset-2', checksum: file2, deletedAt: null }, ]); await expect( @@ -857,7 +857,7 @@ describe(AssetMediaService.name, () => { const file1 = Buffer.from('d2947b871a706081be194569951b7db246907957', 'hex'); const file2 = Buffer.from('53be335e99f18a66ff12e9a901c7a6171dd76573', 'hex'); - mocks.asset.getByChecksums.mockResolvedValue([{ id: 'asset-1', checksum: file1 } as AssetEntity]); + mocks.asset.getByChecksums.mockResolvedValue([{ id: 'asset-1', checksum: file1, deletedAt: null }]); await expect( sut.bulkUploadCheck(authStub.admin, { diff --git a/server/src/services/asset-media.service.ts b/server/src/services/asset-media.service.ts index de40d8b304..78e23fa802 100644 --- a/server/src/services/asset-media.service.ts +++ b/server/src/services/asset-media.service.ts @@ -21,13 +21,13 @@ import { UploadFieldName, } from 'src/dtos/asset-media.dto'; import { AuthDto } from 'src/dtos/auth.dto'; -import { ASSET_CHECKSUM_CONSTRAINT } from 'src/entities/asset.entity'; import { AssetStatus, AssetType, CacheControl, JobName, Permission, StorageFolder } from 'src/enum'; import { AuthRequest } from 'src/middleware/auth.guard'; import { BaseService } from 'src/services/base.service'; import { UploadFile } from 'src/types'; import { requireUploadAccess } from 'src/utils/access'; import { asRequest, getAssetFiles, onBeforeLink } from 'src/utils/asset.util'; +import { ASSET_CHECKSUM_CONSTRAINT } from 'src/utils/database'; import { getFilenameExtension, getFileNameWithoutExtension, ImmichFileResponse } from 'src/utils/file'; import { mimeTypes } from 'src/utils/mime-types'; import { fromChecksum } from 'src/utils/request'; diff --git a/server/src/services/memory.service.spec.ts b/server/src/services/memory.service.spec.ts index 7ce8b1ab46..d55c58d9af 100644 --- a/server/src/services/memory.service.spec.ts +++ b/server/src/services/memory.service.spec.ts @@ -15,6 +15,14 @@ describe(MemoryService.name, () => { expect(sut).toBeDefined(); }); + describe('onMemoryCleanup', () => { + it('should clean up memories', async () => { + mocks.memory.cleanup.mockResolvedValue([]); + await sut.onMemoriesCleanup(); + expect(mocks.memory.cleanup).toHaveBeenCalled(); + }); + }); + describe('search', () => { it('should search memories', async () => { const [userId] = newUuids(); diff --git a/server/src/services/search.service.spec.ts b/server/src/services/search.service.spec.ts index 18699edf9a..d87ccbde1d 100644 --- a/server/src/services/search.service.spec.ts +++ b/server/src/services/search.service.spec.ts @@ -39,6 +39,29 @@ describe(SearchService.name, () => { }); }); + describe('searchPlaces', () => { + it('should search places', async () => { + mocks.search.searchPlaces.mockResolvedValue([ + { + id: 42, + name: 'my place', + latitude: 420, + longitude: 69, + admin1Code: null, + admin1Name: null, + admin2Code: null, + admin2Name: null, + alternateNames: null, + countryCode: 'US', + modificationDate: new Date(), + }, + ]); + + await sut.searchPlaces({ name: 'place' }); + expect(mocks.search.searchPlaces).toHaveBeenCalledWith('place'); + }); + }); + describe('getExploreData', () => { it('should get assets by city and tag', async () => { mocks.asset.getAssetIdByCity.mockResolvedValue({ diff --git a/server/src/utils/database.ts b/server/src/utils/database.ts index a9c7b09c5a..1af0aa4b4e 100644 --- a/server/src/utils/database.ts +++ b/server/src/utils/database.ts @@ -1,15 +1,24 @@ import { + DeduplicateJoinsPlugin, Expression, ExpressionBuilder, ExpressionWrapper, + Kysely, KyselyConfig, Nullable, Selectable, + SelectQueryBuilder, Simplify, sql, } from 'kysely'; import { PostgresJSDialect } from 'kysely-postgres-js'; +import { jsonArrayFrom, jsonObjectFrom } from 'kysely/helpers/postgres'; import postgres, { Notice } from 'postgres'; +import { columns, Exif, Person } from 'src/database'; +import { DB } from 'src/db'; +import { AssetFileType } from 'src/enum'; +import { TimeBucketSize } from 'src/repositories/asset.repository'; +import { AssetSearchBuilderOptions } from 'src/repositories/search.repository'; type Ssl = 'require' | 'allow' | 'prefer' | 'verify-full' | boolean | object; @@ -112,3 +121,225 @@ export function toJson >; } + +export const ASSET_CHECKSUM_CONSTRAINT = 'UQ_assets_owner_checksum'; +// TODO come up with a better query that only selects the fields we need + +export function withExif(qb: SelectQueryBuilder) { + return qb + .leftJoin('exif', 'assets.id', 'exif.assetId') + .select((eb) => eb.fn.toJson(eb.table('exif')).$castTo().as('exifInfo')); +} + +export function withExifInner(qb: SelectQueryBuilder) { + return qb + .innerJoin('exif', 'assets.id', 'exif.assetId') + .select((eb) => eb.fn.toJson(eb.table('exif')).$castTo().as('exifInfo')); +} + +export function withSmartSearch(qb: SelectQueryBuilder) { + return qb + .leftJoin('smart_search', 'assets.id', 'smart_search.assetId') + .select((eb) => toJson(eb, 'smart_search').as('smartSearch')); +} + +export function withFaces(eb: ExpressionBuilder, withDeletedFace?: boolean) { + return jsonArrayFrom( + eb + .selectFrom('asset_faces') + .selectAll('asset_faces') + .whereRef('asset_faces.assetId', '=', 'assets.id') + .$if(!withDeletedFace, (qb) => qb.where('asset_faces.deletedAt', 'is', null)), + ).as('faces'); +} + +export function withFiles(eb: ExpressionBuilder, type?: AssetFileType) { + return jsonArrayFrom( + eb + .selectFrom('asset_files') + .select(columns.assetFiles) + .whereRef('asset_files.assetId', '=', 'assets.id') + .$if(!!type, (qb) => qb.where('asset_files.type', '=', type!)), + ).as('files'); +} + +export function withFacesAndPeople(eb: ExpressionBuilder, withDeletedFace?: boolean) { + return jsonArrayFrom( + eb + .selectFrom('asset_faces') + .leftJoinLateral( + (eb) => + eb.selectFrom('person').selectAll('person').whereRef('asset_faces.personId', '=', 'person.id').as('person'), + (join) => join.onTrue(), + ) + .selectAll('asset_faces') + .select((eb) => eb.table('person').$castTo().as('person')) + .whereRef('asset_faces.assetId', '=', 'assets.id') + .$if(!withDeletedFace, (qb) => qb.where('asset_faces.deletedAt', 'is', null)), + ).as('faces'); +} + +export function hasPeople(qb: SelectQueryBuilder, personIds: string[]) { + return qb.innerJoin( + (eb) => + eb + .selectFrom('asset_faces') + .select('assetId') + .where('personId', '=', anyUuid(personIds!)) + .where('deletedAt', 'is', null) + .groupBy('assetId') + .having((eb) => eb.fn.count('personId').distinct(), '=', personIds.length) + .as('has_people'), + (join) => join.onRef('has_people.assetId', '=', 'assets.id'), + ); +} + +export function hasTags(qb: SelectQueryBuilder, tagIds: string[]) { + return qb.innerJoin( + (eb) => + eb + .selectFrom('tag_asset') + .select('assetsId') + .innerJoin('tags_closure', 'tag_asset.tagsId', 'tags_closure.id_descendant') + .where('tags_closure.id_ancestor', '=', anyUuid(tagIds)) + .groupBy('assetsId') + .having((eb) => eb.fn.count('tags_closure.id_ancestor').distinct(), '>=', tagIds.length) + .as('has_tags'), + (join) => join.onRef('has_tags.assetsId', '=', 'assets.id'), + ); +} + +export function withOwner(eb: ExpressionBuilder) { + return jsonObjectFrom(eb.selectFrom('users').select(columns.user).whereRef('users.id', '=', 'assets.ownerId')).as( + 'owner', + ); +} + +export function withLibrary(eb: ExpressionBuilder) { + return jsonObjectFrom( + eb.selectFrom('libraries').selectAll('libraries').whereRef('libraries.id', '=', 'assets.libraryId'), + ).as('library'); +} + +export function withTags(eb: ExpressionBuilder) { + return jsonArrayFrom( + eb + .selectFrom('tags') + .select(columns.tag) + .innerJoin('tag_asset', 'tags.id', 'tag_asset.tagsId') + .whereRef('assets.id', '=', 'tag_asset.assetsId'), + ).as('tags'); +} + +export function truncatedDate(size: TimeBucketSize) { + return sql`date_trunc(${size}, "localDateTime" at time zone 'UTC') at time zone 'UTC'`; +} + +export function withTagId(qb: SelectQueryBuilder, tagId: string) { + return qb.where((eb) => + eb.exists( + eb + .selectFrom('tags_closure') + .innerJoin('tag_asset', 'tag_asset.tagsId', 'tags_closure.id_descendant') + .whereRef('tag_asset.assetsId', '=', 'assets.id') + .where('tags_closure.id_ancestor', '=', tagId), + ), + ); +} +const joinDeduplicationPlugin = new DeduplicateJoinsPlugin(); +/** TODO: This should only be used for search-related queries, not as a general purpose query builder */ + +export function searchAssetBuilder(kysely: Kysely, options: AssetSearchBuilderOptions) { + options.isArchived ??= options.withArchived ? undefined : false; + options.withDeleted ||= !!(options.trashedAfter || options.trashedBefore || options.isOffline); + return kysely + .withPlugin(joinDeduplicationPlugin) + .selectFrom('assets') + .selectAll('assets') + .$if(!!options.tagIds && options.tagIds.length > 0, (qb) => hasTags(qb, options.tagIds!)) + .$if(!!options.personIds && options.personIds.length > 0, (qb) => hasPeople(qb, options.personIds!)) + .$if(!!options.createdBefore, (qb) => qb.where('assets.createdAt', '<=', options.createdBefore!)) + .$if(!!options.createdAfter, (qb) => qb.where('assets.createdAt', '>=', options.createdAfter!)) + .$if(!!options.updatedBefore, (qb) => qb.where('assets.updatedAt', '<=', options.updatedBefore!)) + .$if(!!options.updatedAfter, (qb) => qb.where('assets.updatedAt', '>=', options.updatedAfter!)) + .$if(!!options.trashedBefore, (qb) => qb.where('assets.deletedAt', '<=', options.trashedBefore!)) + .$if(!!options.trashedAfter, (qb) => qb.where('assets.deletedAt', '>=', options.trashedAfter!)) + .$if(!!options.takenBefore, (qb) => qb.where('assets.fileCreatedAt', '<=', options.takenBefore!)) + .$if(!!options.takenAfter, (qb) => qb.where('assets.fileCreatedAt', '>=', options.takenAfter!)) + .$if(options.city !== undefined, (qb) => + qb + .innerJoin('exif', 'assets.id', 'exif.assetId') + .where('exif.city', options.city === null ? 'is' : '=', options.city!), + ) + .$if(options.state !== undefined, (qb) => + qb + .innerJoin('exif', 'assets.id', 'exif.assetId') + .where('exif.state', options.state === null ? 'is' : '=', options.state!), + ) + .$if(options.country !== undefined, (qb) => + qb + .innerJoin('exif', 'assets.id', 'exif.assetId') + .where('exif.country', options.country === null ? 'is' : '=', options.country!), + ) + .$if(options.make !== undefined, (qb) => + qb + .innerJoin('exif', 'assets.id', 'exif.assetId') + .where('exif.make', options.make === null ? 'is' : '=', options.make!), + ) + .$if(options.model !== undefined, (qb) => + qb + .innerJoin('exif', 'assets.id', 'exif.assetId') + .where('exif.model', options.model === null ? 'is' : '=', options.model!), + ) + .$if(options.lensModel !== undefined, (qb) => + qb + .innerJoin('exif', 'assets.id', 'exif.assetId') + .where('exif.lensModel', options.lensModel === null ? 'is' : '=', options.lensModel!), + ) + .$if(options.rating !== undefined, (qb) => + qb + .innerJoin('exif', 'assets.id', 'exif.assetId') + .where('exif.rating', options.rating === null ? 'is' : '=', options.rating!), + ) + .$if(!!options.checksum, (qb) => qb.where('assets.checksum', '=', options.checksum!)) + .$if(!!options.deviceAssetId, (qb) => qb.where('assets.deviceAssetId', '=', options.deviceAssetId!)) + .$if(!!options.deviceId, (qb) => qb.where('assets.deviceId', '=', options.deviceId!)) + .$if(!!options.id, (qb) => qb.where('assets.id', '=', asUuid(options.id!))) + .$if(!!options.libraryId, (qb) => qb.where('assets.libraryId', '=', asUuid(options.libraryId!))) + .$if(!!options.userIds, (qb) => qb.where('assets.ownerId', '=', anyUuid(options.userIds!))) + .$if(!!options.encodedVideoPath, (qb) => qb.where('assets.encodedVideoPath', '=', options.encodedVideoPath!)) + .$if(!!options.originalPath, (qb) => + qb.where(sql`f_unaccent(assets."originalPath")`, 'ilike', sql`'%' || f_unaccent(${options.originalPath}) || '%'`), + ) + .$if(!!options.originalFileName, (qb) => + qb.where( + sql`f_unaccent(assets."originalFileName")`, + 'ilike', + sql`'%' || f_unaccent(${options.originalFileName}) || '%'`, + ), + ) + .$if(!!options.description, (qb) => + qb + .innerJoin('exif', 'assets.id', 'exif.assetId') + .where(sql`f_unaccent(exif.description)`, 'ilike', sql`'%' || f_unaccent(${options.description}) || '%'`), + ) + .$if(!!options.type, (qb) => qb.where('assets.type', '=', options.type!)) + .$if(options.isFavorite !== undefined, (qb) => qb.where('assets.isFavorite', '=', options.isFavorite!)) + .$if(options.isOffline !== undefined, (qb) => qb.where('assets.isOffline', '=', options.isOffline!)) + .$if(options.isVisible !== undefined, (qb) => qb.where('assets.isVisible', '=', options.isVisible!)) + .$if(options.isArchived !== undefined, (qb) => qb.where('assets.isArchived', '=', options.isArchived!)) + .$if(options.isEncoded !== undefined, (qb) => + qb.where('assets.encodedVideoPath', options.isEncoded ? 'is not' : 'is', null), + ) + .$if(options.isMotion !== undefined, (qb) => + qb.where('assets.livePhotoVideoId', options.isMotion ? 'is not' : 'is', null), + ) + .$if(!!options.isNotInAlbum, (qb) => + qb.where((eb) => + eb.not(eb.exists((eb) => eb.selectFrom('albums_assets_assets').whereRef('assetsId', '=', 'assets.id'))), + ), + ) + .$if(!!options.withExif, withExifInner) + .$if(!!(options.withFaces || options.withPeople || options.personIds), (qb) => qb.select(withFacesAndPeople)) + .$if(!options.withDeleted, (qb) => qb.where('assets.deletedAt', 'is', null)); +}