From 5e68f8c51940c48daab2ffcf4f16c3c691d5ee8b Mon Sep 17 00:00:00 2001 From: Min Idzelis Date: Wed, 16 Apr 2025 19:24:26 -0400 Subject: [PATCH 01/29] fix: longpress triggers contextmenu (#17602) --- .../assets/thumbnail/thumbnail.svelte | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/web/src/lib/components/assets/thumbnail/thumbnail.svelte b/web/src/lib/components/assets/thumbnail/thumbnail.svelte index 93a4e3c6cc..c21acd8f86 100644 --- a/web/src/lib/components/assets/thumbnail/thumbnail.svelte +++ b/web/src/lib/components/assets/thumbnail/thumbnail.svelte @@ -130,18 +130,30 @@ }; let timer: ReturnType; - const clearLongPressTimer = () => clearTimeout(timer); + + const preventContextMenu = (evt: Event) => evt.preventDefault(); + let disposeables: (() => void)[] = []; + + const clearLongPressTimer = () => { + clearTimeout(timer); + for (const dispose of disposeables) { + dispose(); + } + disposeables = []; + }; let startX: number = 0; let startY: number = 0; function longPress(element: HTMLElement, { onLongPress }: { onLongPress: () => void }) { let didPress = false; - const start = (evt: TouchEvent) => { - startX = evt.changedTouches[0].clientX; - startY = evt.changedTouches[0].clientY; + const start = (evt: PointerEvent) => { + startX = evt.clientX; + startY = evt.clientY; didPress = false; timer = setTimeout(() => { onLongPress(); + element.addEventListener('contextmenu', preventContextMenu, { once: true }); + disposeables.push(() => element.removeEventListener('contextmenu', preventContextMenu)); didPress = true; }, 350); }; @@ -153,13 +165,13 @@ e.preventDefault(); }; element.addEventListener('click', click); - element.addEventListener('touchstart', start, true); - element.addEventListener('touchend', clearLongPressTimer, true); + element.addEventListener('pointerdown', start, true); + element.addEventListener('pointerup', clearLongPressTimer, true); return { destroy: () => { element.removeEventListener('click', click); - element.removeEventListener('touchstart', start, true); - element.removeEventListener('touchend', clearLongPressTimer, true); + element.removeEventListener('pointerdown', start, true); + element.removeEventListener('pointerup', clearLongPressTimer, true); }, }; } From 067338b0edd8e1ed9e3c3c4245c31c1a55978f6b Mon Sep 17 00:00:00 2001 From: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com> Date: Thu, 17 Apr 2025 16:46:52 +0200 Subject: [PATCH 02/29] chore: remove transfer encoding header (#17671) --- mobile/lib/services/backup.service.dart | 1 - 1 file changed, 1 deletion(-) diff --git a/mobile/lib/services/backup.service.dart b/mobile/lib/services/backup.service.dart index a6468f249b..596ad8dc2e 100644 --- a/mobile/lib/services/backup.service.dart +++ b/mobile/lib/services/backup.service.dart @@ -351,7 +351,6 @@ class BackupService { ); baseRequest.headers.addAll(ApiService.getRequestHeaders()); - baseRequest.headers["Transfer-Encoding"] = "chunked"; baseRequest.fields['deviceAssetId'] = asset.localId!; baseRequest.fields['deviceId'] = deviceId; baseRequest.fields['fileCreatedAt'] = From 81ed54aa61a0ff6e94de6e52709d04c1db0a0d44 Mon Sep 17 00:00:00 2001 From: shenlong <139912620+shenlong-tanwen@users.noreply.github.com> Date: Thu, 17 Apr 2025 20:55:27 +0530 Subject: [PATCH 03/29] feat: user sync stream (#16862) * refactor: user entity * chore: rebase fixes * refactor: remove int user Id * refactor: migrate store userId from int to string * refactor: rename uid to id * feat: drift * pr feedback * refactor: move common overrides to mixin * refactor: remove int user Id * refactor: migrate store userId from int to string * refactor: rename uid to id * feat: user & partner sync stream * pr changes * refactor: sync service and add tests * chore: remove generated change * chore: move sync model * rebase: convert string ids to byte uuids * rebase * add processing logs * batch db calls * rewrite isolate manager * rewrite with worker_manager * misc fixes * add sync order test --------- Co-authored-by: shenlong-tanwen <139912620+shalong-tanwen@users.noreply.github.com> Co-authored-by: Alex --- mobile/analysis_options.yaml | 8 +- mobile/devtools_options.yaml | 1 + mobile/lib/constants/constants.dart | 4 + .../domain/interfaces/sync_api.interface.dart | 7 +- .../interfaces/sync_stream.interface.dart | 10 + .../domain/models/sync/sync_event.model.dart | 14 - .../lib/domain/models/sync_event.model.dart | 13 + .../domain/services/sync_stream.service.dart | 217 +++++++-- mobile/lib/domain/utils/background_sync.dart | 37 ++ mobile/lib/extensions/string_extensions.dart | 9 + .../repositories/sync_api.repository.dart | 93 ++-- .../repositories/sync_stream.repository.dart | 104 ++++ mobile/lib/main.dart | 5 +- .../providers/background_sync.provider.dart | 8 + .../infrastructure/cancel.provider.dart | 12 + .../providers/infrastructure/db.provider.dart | 9 + .../infrastructure/sync_stream.provider.dart | 27 +- mobile/lib/services/auth.service.dart | 11 +- mobile/lib/utils/bootstrap.dart | 6 +- mobile/lib/utils/isolate.dart | 69 +++ mobile/pubspec.lock | 10 +- mobile/pubspec.yaml | 2 + mobile/test/domain/service.mock.dart | 3 + .../services/sync_stream_service_test.dart | 443 ++++++++++++++++++ mobile/test/fixtures/sync_stream.stub.dart | 45 ++ .../test/infrastructure/repository.mock.dart | 6 + mobile/test/service.mocks.dart | 1 - mobile/test/services/auth.service_test.dart | 8 + 28 files changed, 1065 insertions(+), 117 deletions(-) create mode 100644 mobile/lib/domain/interfaces/sync_stream.interface.dart delete mode 100644 mobile/lib/domain/models/sync/sync_event.model.dart create mode 100644 mobile/lib/domain/models/sync_event.model.dart create mode 100644 mobile/lib/domain/utils/background_sync.dart create mode 100644 mobile/lib/infrastructure/repositories/sync_stream.repository.dart create mode 100644 mobile/lib/providers/background_sync.provider.dart create mode 100644 mobile/lib/providers/infrastructure/cancel.provider.dart create mode 100644 mobile/lib/utils/isolate.dart create mode 100644 mobile/test/domain/services/sync_stream_service_test.dart create mode 100644 mobile/test/fixtures/sync_stream.stub.dart diff --git a/mobile/analysis_options.yaml b/mobile/analysis_options.yaml index 04f3145908..854f852e3c 100644 --- a/mobile/analysis_options.yaml +++ b/mobile/analysis_options.yaml @@ -35,6 +35,7 @@ linter: analyzer: exclude: - openapi/** + - build/** - lib/generated_plugin_registrant.dart - lib/**/*.g.dart - lib/**/*.drift.dart @@ -92,6 +93,9 @@ custom_lint: allowed: # required / wanted - lib/repositories/*_api.repository.dart + - lib/domain/models/sync_event.model.dart + - lib/{domain,infrastructure}/**/sync_stream.* + - lib/{domain,infrastructure}/**/sync_api.* - lib/infrastructure/repositories/*_api.repository.dart - lib/infrastructure/utils/*.converter.dart # acceptable exceptions for the time being @@ -144,7 +148,9 @@ dart_code_metrics: - avoid-global-state - avoid-inverted-boolean-checks - avoid-late-final-reassignment - - avoid-local-functions + - avoid-local-functions: + exclude: + - test/**.dart - avoid-negated-conditions - avoid-nested-streams-and-futures - avoid-referencing-subclasses diff --git a/mobile/devtools_options.yaml b/mobile/devtools_options.yaml index fa0b357c4f..f592d85a9b 100644 --- a/mobile/devtools_options.yaml +++ b/mobile/devtools_options.yaml @@ -1,3 +1,4 @@ description: This file stores settings for Dart & Flutter DevTools. documentation: https://docs.flutter.dev/tools/devtools/extensions#configure-extension-enablement-states extensions: + - drift: true \ No newline at end of file diff --git a/mobile/lib/constants/constants.dart b/mobile/lib/constants/constants.dart index 83d540d54c..a91e0a715d 100644 --- a/mobile/lib/constants/constants.dart +++ b/mobile/lib/constants/constants.dart @@ -5,5 +5,9 @@ const double downloadFailed = -2; // Number of log entries to retain on app start const int kLogTruncateLimit = 250; +// Sync +const int kSyncEventBatchSize = 5000; + +// Hash batch limits const int kBatchHashFileLimit = 128; const int kBatchHashSizeLimit = 1024 * 1024 * 1024; // 1GB diff --git a/mobile/lib/domain/interfaces/sync_api.interface.dart b/mobile/lib/domain/interfaces/sync_api.interface.dart index fb8f1aa46e..44e22c5894 100644 --- a/mobile/lib/domain/interfaces/sync_api.interface.dart +++ b/mobile/lib/domain/interfaces/sync_api.interface.dart @@ -1,7 +1,8 @@ -import 'package:immich_mobile/domain/models/sync/sync_event.model.dart'; +import 'package:immich_mobile/domain/models/sync_event.model.dart'; +import 'package:openapi/api.dart'; abstract interface class ISyncApiRepository { - Future ack(String data); + Future ack(List data); - Stream> watchUserSyncEvent(); + Stream> getSyncEvents(List type); } diff --git a/mobile/lib/domain/interfaces/sync_stream.interface.dart b/mobile/lib/domain/interfaces/sync_stream.interface.dart new file mode 100644 index 0000000000..f9c52d7ee0 --- /dev/null +++ b/mobile/lib/domain/interfaces/sync_stream.interface.dart @@ -0,0 +1,10 @@ +import 'package:immich_mobile/domain/interfaces/db.interface.dart'; +import 'package:openapi/api.dart'; + +abstract interface class ISyncStreamRepository implements IDatabaseRepository { + Future updateUsersV1(Iterable data); + Future deleteUsersV1(Iterable data); + + Future updatePartnerV1(Iterable data); + Future deletePartnerV1(Iterable data); +} diff --git a/mobile/lib/domain/models/sync/sync_event.model.dart b/mobile/lib/domain/models/sync/sync_event.model.dart deleted file mode 100644 index f4642d59cf..0000000000 --- a/mobile/lib/domain/models/sync/sync_event.model.dart +++ /dev/null @@ -1,14 +0,0 @@ -class SyncEvent { - // dynamic - final dynamic data; - - final String ack; - - SyncEvent({ - required this.data, - required this.ack, - }); - - @override - String toString() => 'SyncEvent(data: $data, ack: $ack)'; -} diff --git a/mobile/lib/domain/models/sync_event.model.dart b/mobile/lib/domain/models/sync_event.model.dart new file mode 100644 index 0000000000..2ad8a75fe1 --- /dev/null +++ b/mobile/lib/domain/models/sync_event.model.dart @@ -0,0 +1,13 @@ +import 'package:openapi/api.dart'; + +class SyncEvent { + final SyncEntityType type; + // ignore: avoid-dynamic + final dynamic data; + final String ack; + + const SyncEvent({required this.type, required this.data, required this.ack}); + + @override + String toString() => 'SyncEvent(type: $type, data: $data, ack: $ack)'; +} diff --git a/mobile/lib/domain/services/sync_stream.service.dart b/mobile/lib/domain/services/sync_stream.service.dart index 72e29b3677..8d7d87e35e 100644 --- a/mobile/lib/domain/services/sync_stream.service.dart +++ b/mobile/lib/domain/services/sync_stream.service.dart @@ -1,49 +1,200 @@ +// ignore_for_file: avoid-passing-async-when-sync-expected + import 'dart:async'; -import 'package:flutter/foundation.dart'; +import 'package:collection/collection.dart'; import 'package:immich_mobile/domain/interfaces/sync_api.interface.dart'; +import 'package:immich_mobile/domain/interfaces/sync_stream.interface.dart'; +import 'package:logging/logging.dart'; import 'package:openapi/api.dart'; +import 'package:worker_manager/worker_manager.dart'; + +const _kSyncTypeOrder = [ + SyncEntityType.userDeleteV1, + SyncEntityType.userV1, + SyncEntityType.partnerDeleteV1, + SyncEntityType.partnerV1, + SyncEntityType.assetDeleteV1, + SyncEntityType.assetV1, + SyncEntityType.assetExifV1, + SyncEntityType.partnerAssetDeleteV1, + SyncEntityType.partnerAssetV1, + SyncEntityType.partnerAssetExifV1, +]; class SyncStreamService { + final Logger _logger = Logger('SyncStreamService'); + final ISyncApiRepository _syncApiRepository; + final ISyncStreamRepository _syncStreamRepository; + final bool Function()? _cancelChecker; - SyncStreamService(this._syncApiRepository); + SyncStreamService({ + required ISyncApiRepository syncApiRepository, + required ISyncStreamRepository syncStreamRepository, + bool Function()? cancelChecker, + }) : _syncApiRepository = syncApiRepository, + _syncStreamRepository = syncStreamRepository, + _cancelChecker = cancelChecker; - StreamSubscription? _userSyncSubscription; + Future _handleSyncData( + SyncEntityType type, + // ignore: avoid-dynamic + Iterable data, + ) async { + if (data.isEmpty) { + _logger.warning("Received empty sync data for $type"); + return false; + } - void syncUsers() { - _userSyncSubscription = - _syncApiRepository.watchUserSyncEvent().listen((events) async { - for (final event in events) { - if (event.data is SyncUserV1) { - final data = event.data as SyncUserV1; - debugPrint("User Update: $data"); + _logger.fine("Processing sync data for $type of length ${data.length}"); - // final user = await _userRepository.get(data.id); - - // if (user == null) { - // continue; - // } - - // user.name = data.name; - // user.email = data.email; - // user.updatedAt = DateTime.now(); - - // await _userRepository.update(user); - // await _syncApiRepository.ack(event.ack); - } - - if (event.data is SyncUserDeleteV1) { - final data = event.data as SyncUserDeleteV1; - - debugPrint("User delete: $data"); - // await _syncApiRepository.ack(event.ack); - } + try { + if (type == SyncEntityType.partnerV1) { + return await _syncStreamRepository.updatePartnerV1(data.cast()); } + + if (type == SyncEntityType.partnerDeleteV1) { + return await _syncStreamRepository.deletePartnerV1(data.cast()); + } + + if (type == SyncEntityType.userV1) { + return await _syncStreamRepository.updateUsersV1(data.cast()); + } + + if (type == SyncEntityType.userDeleteV1) { + return await _syncStreamRepository.deleteUsersV1(data.cast()); + } + } catch (error, stack) { + _logger.severe("Error processing sync data for $type", error, stack); + return false; + } + + _logger.warning("Unknown sync data type: $type"); + return false; + } + + Future _syncEvent(List types) { + _logger.info("Syncing Events: $types"); + final streamCompleter = Completer(); + bool shouldComplete = false; + // the onDone callback might fire before the events are processed + // the following flag ensures that the onDone callback is not called + // before the events are processed and also that events are processed sequentially + Completer? mutex; + StreamSubscription? subscription; + try { + subscription = _syncApiRepository.getSyncEvents(types).listen( + (events) async { + if (events.isEmpty) { + _logger.warning("Received empty sync events"); + return; + } + + // If previous events are still being processed, wait for them to finish + if (mutex != null) { + await mutex!.future; + } + + if (_cancelChecker?.call() ?? false) { + _logger.info("Sync cancelled, stopping stream"); + subscription?.cancel(); + if (!streamCompleter.isCompleted) { + streamCompleter.completeError( + CanceledError(), + StackTrace.current, + ); + } + return; + } + + // Take control of the mutex and process the events + mutex = Completer(); + + try { + final eventsMap = events.groupListsBy((event) => event.type); + final Map acks = {}; + + for (final type in _kSyncTypeOrder) { + final data = eventsMap[type]; + if (data == null) { + continue; + } + + if (_cancelChecker?.call() ?? false) { + _logger.info("Sync cancelled, stopping stream"); + mutex?.complete(); + mutex = null; + if (!streamCompleter.isCompleted) { + streamCompleter.completeError( + CanceledError(), + StackTrace.current, + ); + } + + return; + } + + if (data.isEmpty) { + _logger.warning("Received empty sync events for $type"); + continue; + } + + if (await _handleSyncData(type, data.map((e) => e.data))) { + // ignore: avoid-unsafe-collection-methods + acks[type] = data.last.ack; + } else { + _logger.warning("Failed to handle sync events for $type"); + } + } + + if (acks.isNotEmpty) { + await _syncApiRepository.ack(acks.values.toList()); + } + _logger.info("$types events processed"); + } catch (error, stack) { + _logger.warning("Error handling sync events", error, stack); + } finally { + mutex?.complete(); + mutex = null; + } + + if (shouldComplete) { + _logger.info("Sync done, completing stream"); + if (!streamCompleter.isCompleted) streamCompleter.complete(); + } + }, + onError: (error, stack) { + _logger.warning("Error in sync stream for $types", error, stack); + // Do not proceed if the stream errors + if (!streamCompleter.isCompleted) { + // ignore: avoid-missing-completer-stack-trace + streamCompleter.completeError(error, stack); + } + }, + onDone: () { + _logger.info("$types stream done"); + if (mutex == null && !streamCompleter.isCompleted) { + streamCompleter.complete(); + } else { + // Marks the stream as done but does not complete the completer + // until the events are processed + shouldComplete = true; + } + }, + ); + } catch (error, stack) { + _logger.severe("Error starting sync stream", error, stack); + if (!streamCompleter.isCompleted) { + streamCompleter.completeError(error, stack); + } + } + return streamCompleter.future.whenComplete(() { + _logger.info("Sync stream completed"); + return subscription?.cancel(); }); } - Future dispose() async { - await _userSyncSubscription?.cancel(); - } + Future syncUsers() => + _syncEvent([SyncRequestType.usersV1, SyncRequestType.partnersV1]); } diff --git a/mobile/lib/domain/utils/background_sync.dart b/mobile/lib/domain/utils/background_sync.dart new file mode 100644 index 0000000000..0bd456f0bb --- /dev/null +++ b/mobile/lib/domain/utils/background_sync.dart @@ -0,0 +1,37 @@ +// ignore_for_file: avoid-passing-async-when-sync-expected + +import 'dart:async'; + +import 'package:immich_mobile/providers/infrastructure/sync_stream.provider.dart'; +import 'package:immich_mobile/utils/isolate.dart'; +import 'package:worker_manager/worker_manager.dart'; + +class BackgroundSyncManager { + Cancelable? _userSyncTask; + + BackgroundSyncManager(); + + Future cancel() { + final futures = []; + if (_userSyncTask != null) { + futures.add(_userSyncTask!.future); + } + _userSyncTask?.cancel(); + _userSyncTask = null; + return Future.wait(futures); + } + + Future syncUsers() { + if (_userSyncTask != null) { + return _userSyncTask!.future; + } + + _userSyncTask = runInIsolateGentle( + computation: (ref) => ref.read(syncStreamServiceProvider).syncUsers(), + ); + _userSyncTask!.whenComplete(() { + _userSyncTask = null; + }); + return _userSyncTask!.future; + } +} diff --git a/mobile/lib/extensions/string_extensions.dart b/mobile/lib/extensions/string_extensions.dart index 67411013ee..73c8c2d34c 100644 --- a/mobile/lib/extensions/string_extensions.dart +++ b/mobile/lib/extensions/string_extensions.dart @@ -1,3 +1,7 @@ +import 'dart:typed_data'; + +import 'package:uuid/parsing.dart'; + extension StringExtension on String { String capitalize() { return split(" ") @@ -29,3 +33,8 @@ extension DurationExtension on String { return int.parse(this); } } + +extension UUIDExtension on String { + Uint8List toUuidByte({bool shouldValidate = false}) => + UuidParsing.parseAsByteList(this, validate: shouldValidate); +} diff --git a/mobile/lib/infrastructure/repositories/sync_api.repository.dart b/mobile/lib/infrastructure/repositories/sync_api.repository.dart index 88a6838c44..a26b867df6 100644 --- a/mobile/lib/infrastructure/repositories/sync_api.repository.dart +++ b/mobile/lib/infrastructure/repositories/sync_api.repository.dart @@ -1,37 +1,36 @@ import 'dart:async'; import 'dart:convert'; -import 'package:flutter/foundation.dart'; -import 'package:immich_mobile/domain/interfaces/sync_api.interface.dart'; -import 'package:immich_mobile/domain/models/sync/sync_event.model.dart'; -import 'package:immich_mobile/services/api.service.dart'; -import 'package:openapi/api.dart'; import 'package:http/http.dart' as http; +import 'package:immich_mobile/constants/constants.dart'; +import 'package:immich_mobile/domain/interfaces/sync_api.interface.dart'; +import 'package:immich_mobile/domain/models/sync_event.model.dart'; +import 'package:immich_mobile/services/api.service.dart'; +import 'package:logging/logging.dart'; +import 'package:openapi/api.dart'; class SyncApiRepository implements ISyncApiRepository { + final Logger _logger = Logger('SyncApiRepository'); final ApiService _api; - const SyncApiRepository(this._api); + final int _batchSize; + SyncApiRepository(this._api, {int batchSize = kSyncEventBatchSize}) + : _batchSize = batchSize; @override - Stream> watchUserSyncEvent() { - return _getSyncStream( - SyncStreamDto(types: [SyncRequestType.usersV1]), - ); + Stream> getSyncEvents(List type) { + return _getSyncStream(SyncStreamDto(types: type)); } @override - Future ack(String data) { - return _api.syncApi.sendSyncAck(SyncAckSetDto(acks: [data])); + Future ack(List data) { + return _api.syncApi.sendSyncAck(SyncAckSetDto(acks: data)); } - Stream> _getSyncStream( - SyncStreamDto dto, { - int batchSize = 5000, - }) async* { + Stream> _getSyncStream(SyncStreamDto dto) async* { final client = http.Client(); final endpoint = "${_api.apiClient.basePath}/sync/stream"; - final headers = { + final headers = { 'Content-Type': 'application/json', 'Accept': 'application/jsonlines+json', }; @@ -61,52 +60,54 @@ class SyncApiRepository implements ISyncApiRepository { await for (final chunk in response.stream.transform(utf8.decoder)) { previousChunk += chunk; - final parts = previousChunk.split('\n'); + final parts = previousChunk.toString().split('\n'); previousChunk = parts.removeLast(); lines.addAll(parts); - if (lines.length < batchSize) { + if (lines.length < _batchSize) { continue; } - yield await compute(_parseSyncResponse, lines); + yield _parseSyncResponse(lines); lines.clear(); } } finally { if (lines.isNotEmpty) { - yield await compute(_parseSyncResponse, lines); + yield _parseSyncResponse(lines); } client.close(); } } + + List _parseSyncResponse(List lines) { + final List data = []; + + for (final line in lines) { + try { + final jsonData = jsonDecode(line); + final type = SyncEntityType.fromJson(jsonData['type'])!; + final dataJson = jsonData['data']; + final ack = jsonData['ack']; + final converter = _kResponseMap[type]; + if (converter == null) { + _logger.warning("[_parseSyncResponse] Unknown type $type"); + continue; + } + + data.add(SyncEvent(type: type, data: converter(dataJson), ack: ack)); + } catch (error, stack) { + _logger.severe("[_parseSyncResponse] Error parsing json", error, stack); + } + } + + return data; + } } +// ignore: avoid-dynamic const _kResponseMap = { SyncEntityType.userV1: SyncUserV1.fromJson, SyncEntityType.userDeleteV1: SyncUserDeleteV1.fromJson, + SyncEntityType.partnerV1: SyncPartnerV1.fromJson, + SyncEntityType.partnerDeleteV1: SyncPartnerDeleteV1.fromJson, }; - -// Need to be outside of the class to be able to use compute -List _parseSyncResponse(List lines) { - final List data = []; - - for (var line in lines) { - try { - final jsonData = jsonDecode(line); - final type = SyncEntityType.fromJson(jsonData['type'])!; - final dataJson = jsonData['data']; - final ack = jsonData['ack']; - final converter = _kResponseMap[type]; - if (converter == null) { - debugPrint("[_parseSyncReponse] Unknown type $type"); - continue; - } - - data.add(SyncEvent(data: converter(dataJson), ack: ack)); - } catch (error, stack) { - debugPrint("[_parseSyncReponse] Error parsing json $error $stack"); - } - } - - return data; -} diff --git a/mobile/lib/infrastructure/repositories/sync_stream.repository.dart b/mobile/lib/infrastructure/repositories/sync_stream.repository.dart new file mode 100644 index 0000000000..a947a9a66b --- /dev/null +++ b/mobile/lib/infrastructure/repositories/sync_stream.repository.dart @@ -0,0 +1,104 @@ +import 'package:drift/drift.dart'; +import 'package:immich_mobile/domain/interfaces/sync_stream.interface.dart'; +import 'package:immich_mobile/extensions/string_extensions.dart'; +import 'package:immich_mobile/infrastructure/entities/partner.entity.drift.dart'; +import 'package:immich_mobile/infrastructure/entities/user.entity.drift.dart'; +import 'package:immich_mobile/infrastructure/repositories/db.repository.dart'; +import 'package:logging/logging.dart'; +import 'package:openapi/api.dart'; + +class DriftSyncStreamRepository extends DriftDatabaseRepository + implements ISyncStreamRepository { + final Logger _logger = Logger('DriftSyncStreamRepository'); + final Drift _db; + + DriftSyncStreamRepository(super.db) : _db = db; + + @override + Future deleteUsersV1(Iterable data) async { + try { + await _db.batch((batch) { + for (final user in data) { + batch.delete( + _db.userEntity, + UserEntityCompanion(id: Value(user.userId.toUuidByte())), + ); + } + }); + return true; + } catch (e, s) { + _logger.severe('Error while processing SyncUserDeleteV1', e, s); + return false; + } + } + + @override + Future updateUsersV1(Iterable data) async { + try { + await _db.batch((batch) { + for (final user in data) { + final companion = UserEntityCompanion( + name: Value(user.name), + email: Value(user.email), + ); + + batch.insert( + _db.userEntity, + companion.copyWith(id: Value(user.id.toUuidByte())), + onConflict: DoUpdate((_) => companion), + ); + } + }); + return true; + } catch (e, s) { + _logger.severe('Error while processing SyncUserV1', e, s); + return false; + } + } + + @override + Future deletePartnerV1(Iterable data) async { + try { + await _db.batch((batch) { + for (final partner in data) { + batch.delete( + _db.partnerEntity, + PartnerEntityCompanion( + sharedById: Value(partner.sharedById.toUuidByte()), + sharedWithId: Value(partner.sharedWithId.toUuidByte()), + ), + ); + } + }); + return true; + } catch (e, s) { + _logger.severe('Error while processing SyncPartnerDeleteV1', e, s); + return false; + } + } + + @override + Future updatePartnerV1(Iterable data) async { + try { + await _db.batch((batch) { + for (final partner in data) { + final companion = + PartnerEntityCompanion(inTimeline: Value(partner.inTimeline)); + + batch.insert( + _db.partnerEntity, + companion.copyWith( + sharedById: Value(partner.sharedById.toUuidByte()), + sharedWithId: Value(partner.sharedWithId.toUuidByte()), + ), + onConflict: DoUpdate((_) => companion), + ); + } + }); + return true; + } catch (e, s) { + _logger.severe('Error while processing SyncPartnerV1', e, s); + return false; + } + } +} diff --git a/mobile/lib/main.dart b/mobile/lib/main.dart index 1a434aa359..73af81d69d 100644 --- a/mobile/lib/main.dart +++ b/mobile/lib/main.dart @@ -11,6 +11,7 @@ import 'package:flutter_displaymode/flutter_displaymode.dart'; import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/constants/locales.dart'; import 'package:immich_mobile/extensions/build_context_extensions.dart'; +import 'package:immich_mobile/generated/codegen_loader.g.dart'; import 'package:immich_mobile/providers/app_life_cycle.provider.dart'; import 'package:immich_mobile/providers/asset_viewer/share_intent_upload.provider.dart'; import 'package:immich_mobile/providers/db.provider.dart'; @@ -31,13 +32,15 @@ import 'package:immich_mobile/utils/migration.dart'; import 'package:intl/date_symbol_data_local.dart'; import 'package:logging/logging.dart'; import 'package:timezone/data/latest.dart'; -import 'package:immich_mobile/generated/codegen_loader.g.dart'; +import 'package:worker_manager/worker_manager.dart'; void main() async { ImmichWidgetsBinding(); final db = await Bootstrap.initIsar(); await Bootstrap.initDomain(db); await initApp(); + // Warm-up isolate pool for worker manager + await workerManager.init(dynamicSpawning: true); await migrateDatabaseIfNeeded(db); HttpOverrides.global = HttpSSLCertOverride(); diff --git a/mobile/lib/providers/background_sync.provider.dart b/mobile/lib/providers/background_sync.provider.dart new file mode 100644 index 0000000000..83d103bb3b --- /dev/null +++ b/mobile/lib/providers/background_sync.provider.dart @@ -0,0 +1,8 @@ +import 'package:hooks_riverpod/hooks_riverpod.dart'; +import 'package:immich_mobile/domain/utils/background_sync.dart'; + +final backgroundSyncProvider = Provider((ref) { + final manager = BackgroundSyncManager(); + ref.onDispose(manager.cancel); + return manager; +}); diff --git a/mobile/lib/providers/infrastructure/cancel.provider.dart b/mobile/lib/providers/infrastructure/cancel.provider.dart new file mode 100644 index 0000000000..6851861e1a --- /dev/null +++ b/mobile/lib/providers/infrastructure/cancel.provider.dart @@ -0,0 +1,12 @@ +import 'package:hooks_riverpod/hooks_riverpod.dart'; + +/// Provider holding a boolean function that returns true when cancellation is requested. +/// A computation running in the isolate uses the function to implement cooperative cancellation. +final cancellationProvider = Provider( + // This will be overridden in the isolate's container. + // Throwing ensures it's not used without an override. + (ref) => throw UnimplementedError( + "cancellationProvider must be overridden in the isolate's ProviderContainer and not to be used in the root isolate", + ), + name: 'cancellationProvider', +); diff --git a/mobile/lib/providers/infrastructure/db.provider.dart b/mobile/lib/providers/infrastructure/db.provider.dart index 84010b3b96..4eefbc556c 100644 --- a/mobile/lib/providers/infrastructure/db.provider.dart +++ b/mobile/lib/providers/infrastructure/db.provider.dart @@ -1,4 +1,7 @@ +import 'dart:async'; + import 'package:hooks_riverpod/hooks_riverpod.dart'; +import 'package:immich_mobile/infrastructure/repositories/db.repository.dart'; import 'package:isar/isar.dart'; import 'package:riverpod_annotation/riverpod_annotation.dart'; @@ -6,3 +9,9 @@ part 'db.provider.g.dart'; @Riverpod(keepAlive: true) Isar isar(Ref ref) => throw UnimplementedError('isar'); + +final driftProvider = Provider((ref) { + final drift = Drift(); + ref.onDispose(() => unawaited(drift.close())); + return drift; +}); diff --git a/mobile/lib/providers/infrastructure/sync_stream.provider.dart b/mobile/lib/providers/infrastructure/sync_stream.provider.dart index 64f1a6cb05..e313982a30 100644 --- a/mobile/lib/providers/infrastructure/sync_stream.provider.dart +++ b/mobile/lib/providers/infrastructure/sync_stream.provider.dart @@ -1,24 +1,23 @@ -import 'dart:async'; - import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/domain/services/sync_stream.service.dart'; import 'package:immich_mobile/infrastructure/repositories/sync_api.repository.dart'; +import 'package:immich_mobile/infrastructure/repositories/sync_stream.repository.dart'; import 'package:immich_mobile/providers/api.provider.dart'; +import 'package:immich_mobile/providers/infrastructure/cancel.provider.dart'; +import 'package:immich_mobile/providers/infrastructure/db.provider.dart'; final syncStreamServiceProvider = Provider( - (ref) { - final instance = SyncStreamService( - ref.watch(syncApiRepositoryProvider), - ); - - ref.onDispose(() => unawaited(instance.dispose())); - - return instance; - }, + (ref) => SyncStreamService( + syncApiRepository: ref.watch(syncApiRepositoryProvider), + syncStreamRepository: ref.watch(syncStreamRepositoryProvider), + cancelChecker: ref.watch(cancellationProvider), + ), ); final syncApiRepositoryProvider = Provider( - (ref) => SyncApiRepository( - ref.watch(apiServiceProvider), - ), + (ref) => SyncApiRepository(ref.watch(apiServiceProvider)), +); + +final syncStreamRepositoryProvider = Provider( + (ref) => DriftSyncStreamRepository(ref.watch(driftProvider)), ); diff --git a/mobile/lib/services/auth.service.dart b/mobile/lib/services/auth.service.dart index 20fa62dc4b..ec053c078b 100644 --- a/mobile/lib/services/auth.service.dart +++ b/mobile/lib/services/auth.service.dart @@ -3,12 +3,14 @@ import 'dart:io'; import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/domain/models/store.model.dart'; +import 'package:immich_mobile/domain/utils/background_sync.dart'; import 'package:immich_mobile/entities/store.entity.dart'; import 'package:immich_mobile/interfaces/auth.interface.dart'; import 'package:immich_mobile/interfaces/auth_api.interface.dart'; import 'package:immich_mobile/models/auth/auxilary_endpoint.model.dart'; import 'package:immich_mobile/models/auth/login_response.model.dart'; import 'package:immich_mobile/providers/api.provider.dart'; +import 'package:immich_mobile/providers/background_sync.provider.dart'; import 'package:immich_mobile/repositories/auth.repository.dart'; import 'package:immich_mobile/repositories/auth_api.repository.dart'; import 'package:immich_mobile/services/api.service.dart'; @@ -22,6 +24,7 @@ final authServiceProvider = Provider( ref.watch(authRepositoryProvider), ref.watch(apiServiceProvider), ref.watch(networkServiceProvider), + ref.watch(backgroundSyncProvider), ), ); @@ -30,6 +33,7 @@ class AuthService { final IAuthRepository _authRepository; final ApiService _apiService; final NetworkService _networkService; + final BackgroundSyncManager _backgroundSyncManager; final _log = Logger("AuthService"); @@ -38,6 +42,7 @@ class AuthService { this._authRepository, this._apiService, this._networkService, + this._backgroundSyncManager, ); /// Validates the provided server URL by resolving and setting the endpoint. @@ -115,8 +120,10 @@ class AuthService { /// - Asset ETag /// /// All deletions are executed in parallel using [Future.wait]. - Future clearLocalData() { - return Future.wait([ + Future clearLocalData() async { + // Cancel any ongoing background sync operations before clearing data + await _backgroundSyncManager.cancel(); + await Future.wait([ _authRepository.clearLocalData(), Store.delete(StoreKey.currentUser), Store.delete(StoreKey.accessToken), diff --git a/mobile/lib/utils/bootstrap.dart b/mobile/lib/utils/bootstrap.dart index 570752c6d9..26f3b49242 100644 --- a/mobile/lib/utils/bootstrap.dart +++ b/mobile/lib/utils/bootstrap.dart @@ -48,11 +48,15 @@ abstract final class Bootstrap { ); } - static Future initDomain(Isar db) async { + static Future initDomain( + Isar db, { + bool shouldBufferLogs = true, + }) async { await StoreService.init(storeRepository: IsarStoreRepository(db)); await LogService.init( logRepository: IsarLogRepository(db), storeRepository: IsarStoreRepository(db), + shouldBuffer: shouldBufferLogs, ); } } diff --git a/mobile/lib/utils/isolate.dart b/mobile/lib/utils/isolate.dart new file mode 100644 index 0000000000..cfbb1b544f --- /dev/null +++ b/mobile/lib/utils/isolate.dart @@ -0,0 +1,69 @@ +import 'dart:async'; +import 'dart:ui'; + +import 'package:flutter/services.dart'; +import 'package:hooks_riverpod/hooks_riverpod.dart'; +import 'package:immich_mobile/providers/db.provider.dart'; +import 'package:immich_mobile/providers/infrastructure/cancel.provider.dart'; +import 'package:immich_mobile/providers/infrastructure/db.provider.dart'; +import 'package:immich_mobile/utils/bootstrap.dart'; +import 'package:logging/logging.dart'; +import 'package:worker_manager/worker_manager.dart'; + +class InvalidIsolateUsageException implements Exception { + const InvalidIsolateUsageException(); + + @override + String toString() => + "IsolateHelper should only be used from the root isolate"; +} + +// !! Should be used only from the root isolate +Cancelable runInIsolateGentle({ + required Future Function(ProviderContainer ref) computation, + String? debugLabel, +}) { + final token = RootIsolateToken.instance; + if (token == null) { + throw const InvalidIsolateUsageException(); + } + + return workerManager.executeGentle((cancelledChecker) async { + BackgroundIsolateBinaryMessenger.ensureInitialized(token); + DartPluginRegistrant.ensureInitialized(); + + final db = await Bootstrap.initIsar(); + await Bootstrap.initDomain(db, shouldBufferLogs: false); + final ref = ProviderContainer( + overrides: [ + // TODO: Remove once isar is removed + dbProvider.overrideWithValue(db), + isarProvider.overrideWithValue(db), + cancellationProvider.overrideWithValue(cancelledChecker), + ], + ); + + Logger log = Logger("IsolateLogger"); + + try { + return await computation(ref); + } on CanceledError { + log.warning( + "Computation cancelled ${debugLabel == null ? '' : ' for $debugLabel'}", + ); + } catch (error, stack) { + log.severe( + "Error in runInIsolateGentle ${debugLabel == null ? '' : ' for $debugLabel'}", + error, + stack, + ); + } finally { + // Wait for the logs to flush + await Future.delayed(const Duration(seconds: 2)); + // Always close the new db connection on Isolate end + ref.read(driftProvider).close(); + ref.read(isarProvider).close(); + } + return null; + }); +} diff --git a/mobile/pubspec.lock b/mobile/pubspec.lock index 3731832296..235b3f71c3 100644 --- a/mobile/pubspec.lock +++ b/mobile/pubspec.lock @@ -1806,7 +1806,7 @@ packages: source: hosted version: "3.1.4" uuid: - dependency: transitive + dependency: "direct main" description: name: uuid sha256: a5be9ef6618a7ac1e964353ef476418026db906c4facdedaa299b7a2e71690ff @@ -1933,6 +1933,14 @@ packages: url: "https://pub.dev" source: hosted version: "0.0.3" + worker_manager: + dependency: "direct main" + description: + name: worker_manager + sha256: "086ed63e9b36266e851404ca90fd44e37c0f4c9bbf819e5f8d7c87f9741c0591" + url: "https://pub.dev" + source: hosted + version: "7.2.3" xdg_directories: dependency: transitive description: diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index 03c39810f6..fdd91e1f87 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -60,7 +60,9 @@ dependencies: thumbhash: 0.1.0+1 timezone: ^0.9.4 url_launcher: ^6.3.1 + uuid: ^4.5.1 wakelock_plus: ^1.2.10 + worker_manager: ^7.2.3 native_video_player: git: diff --git a/mobile/test/domain/service.mock.dart b/mobile/test/domain/service.mock.dart index 53a173fc28..97a3f30294 100644 --- a/mobile/test/domain/service.mock.dart +++ b/mobile/test/domain/service.mock.dart @@ -1,7 +1,10 @@ import 'package:immich_mobile/domain/services/store.service.dart'; import 'package:immich_mobile/domain/services/user.service.dart'; +import 'package:immich_mobile/domain/utils/background_sync.dart'; import 'package:mocktail/mocktail.dart'; class MockStoreService extends Mock implements StoreService {} class MockUserService extends Mock implements UserService {} + +class MockBackgroundSyncManager extends Mock implements BackgroundSyncManager {} diff --git a/mobile/test/domain/services/sync_stream_service_test.dart b/mobile/test/domain/services/sync_stream_service_test.dart new file mode 100644 index 0000000000..e1d8e6987f --- /dev/null +++ b/mobile/test/domain/services/sync_stream_service_test.dart @@ -0,0 +1,443 @@ +// ignore_for_file: avoid-unnecessary-futures, avoid-async-call-in-sync-function + +import 'dart:async'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:immich_mobile/domain/interfaces/sync_api.interface.dart'; +import 'package:immich_mobile/domain/interfaces/sync_stream.interface.dart'; +import 'package:immich_mobile/domain/models/sync_event.model.dart'; +import 'package:immich_mobile/domain/services/sync_stream.service.dart'; +import 'package:mocktail/mocktail.dart'; +import 'package:openapi/api.dart'; +import 'package:worker_manager/worker_manager.dart'; + +import '../../fixtures/sync_stream.stub.dart'; +import '../../infrastructure/repository.mock.dart'; + +class _CancellationWrapper { + const _CancellationWrapper(); + + bool isCancelled() => false; +} + +class _MockCancellationWrapper extends Mock implements _CancellationWrapper {} + +void main() { + late SyncStreamService sut; + late ISyncStreamRepository mockSyncStreamRepo; + late ISyncApiRepository mockSyncApiRepo; + late StreamController> streamController; + + successHandler(Invocation _) async => true; + failureHandler(Invocation _) async => false; + + setUp(() { + mockSyncStreamRepo = MockSyncStreamRepository(); + mockSyncApiRepo = MockSyncApiRepository(); + streamController = StreamController>.broadcast(); + + sut = SyncStreamService( + syncApiRepository: mockSyncApiRepo, + syncStreamRepository: mockSyncStreamRepo, + ); + + // Default stream setup - emits one batch and closes + when(() => mockSyncApiRepo.getSyncEvents(any())) + .thenAnswer((_) => streamController.stream); + + // Default ack setup + when(() => mockSyncApiRepo.ack(any())).thenAnswer((_) async => {}); + + // Register fallbacks for mocktail verification + registerFallbackValue([]); + registerFallbackValue([]); + registerFallbackValue([]); + registerFallbackValue([]); + + // Default successful repository calls + when(() => mockSyncStreamRepo.updateUsersV1(any())) + .thenAnswer(successHandler); + when(() => mockSyncStreamRepo.deleteUsersV1(any())) + .thenAnswer(successHandler); + when(() => mockSyncStreamRepo.updatePartnerV1(any())) + .thenAnswer(successHandler); + when(() => mockSyncStreamRepo.deletePartnerV1(any())) + .thenAnswer(successHandler); + }); + + tearDown(() async { + if (!streamController.isClosed) { + await streamController.close(); + } + }); + + // Helper to trigger sync and add events to the stream + Future triggerSyncAndEmit(List events) async { + final future = sut.syncUsers(); // Start listening + await Future.delayed(Duration.zero); // Allow listener to attach + if (!streamController.isClosed) { + streamController.add(events); + await streamController.close(); // Close after emitting + } + await future; // Wait for processing to complete + } + + group("SyncStreamService", () { + test( + "completes successfully when stream emits data and handlers succeed", + () async { + final events = [ + ...SyncStreamStub.userEvents, + ...SyncStreamStub.partnerEvents, + ]; + final future = triggerSyncAndEmit(events); + await expectLater(future, completes); + // Verify ack includes last ack from each successfully handled type + verify( + () => + mockSyncApiRepo.ack(any(that: containsAll(["5", "2", "4", "3"]))), + ).called(1); + }, + ); + + test("completes successfully when stream emits an error", () async { + when(() => mockSyncApiRepo.getSyncEvents(any())) + .thenAnswer((_) => Stream.error(Exception("Stream Error"))); + // Should complete gracefully without throwing + await expectLater(sut.syncUsers(), throwsException); + verifyNever(() => mockSyncApiRepo.ack(any())); // No ack on stream error + }); + + test("throws when initial getSyncEvents call fails", () async { + final apiException = Exception("API Error"); + when(() => mockSyncApiRepo.getSyncEvents(any())).thenThrow(apiException); + // Should rethrow the exception from the initial call + await expectLater(sut.syncUsers(), throwsA(apiException)); + verifyNever(() => mockSyncApiRepo.ack(any())); + }); + + test( + "completes successfully when a repository handler throws an exception", + () async { + when(() => mockSyncStreamRepo.updateUsersV1(any())) + .thenThrow(Exception("Repo Error")); + final events = [ + ...SyncStreamStub.userEvents, + ...SyncStreamStub.partnerEvents, + ]; + // Should complete, but ack only for the successful types + await triggerSyncAndEmit(events); + // Only partner delete was successful by default setup + verify(() => mockSyncApiRepo.ack(["2", "4", "3"])).called(1); + }, + ); + + test( + "completes successfully but sends no ack when all handlers fail", + () async { + when(() => mockSyncStreamRepo.updateUsersV1(any())) + .thenAnswer(failureHandler); + when(() => mockSyncStreamRepo.deleteUsersV1(any())) + .thenAnswer(failureHandler); + when(() => mockSyncStreamRepo.updatePartnerV1(any())) + .thenAnswer(failureHandler); + when(() => mockSyncStreamRepo.deletePartnerV1(any())) + .thenAnswer(failureHandler); + + final events = [ + ...SyncStreamStub.userEvents, + ...SyncStreamStub.partnerEvents, + ]; + await triggerSyncAndEmit(events); + verifyNever(() => mockSyncApiRepo.ack(any())); + }, + ); + + test("sends ack only for types where handler returns true", () async { + // Mock specific handlers: user update fails, user delete succeeds + when(() => mockSyncStreamRepo.updateUsersV1(any())) + .thenAnswer(failureHandler); + when(() => mockSyncStreamRepo.deleteUsersV1(any())) + .thenAnswer(successHandler); + // partner update fails, partner delete succeeds + when(() => mockSyncStreamRepo.updatePartnerV1(any())) + .thenAnswer(failureHandler); + + final events = [ + ...SyncStreamStub.userEvents, + ...SyncStreamStub.partnerEvents, + ]; + await triggerSyncAndEmit(events); + + // Expect ack only for userDeleteV1 (ack: "2") and partnerDeleteV1 (ack: "4") + verify(() => mockSyncApiRepo.ack(any(that: containsAll(["2", "4"])))) + .called(1); + }); + + test("does not process or ack when stream emits an empty list", () async { + final future = sut.syncUsers(); + streamController.add([]); // Emit empty list + await streamController.close(); + await future; // Wait for completion + + verifyNever(() => mockSyncStreamRepo.updateUsersV1(any())); + verifyNever(() => mockSyncStreamRepo.deleteUsersV1(any())); + verifyNever(() => mockSyncStreamRepo.updatePartnerV1(any())); + verifyNever(() => mockSyncStreamRepo.deletePartnerV1(any())); + verifyNever(() => mockSyncApiRepo.ack(any())); + }); + + test("processes multiple batches sequentially using mutex", () async { + final completer1 = Completer(); + final completer2 = Completer(); + int callOrder = 0; + int handler1StartOrder = -1; + int handler2StartOrder = -1; + int handler1Calls = 0; + int handler2Calls = 0; + + when(() => mockSyncStreamRepo.updateUsersV1(any())).thenAnswer((_) async { + handler1Calls++; + handler1StartOrder = ++callOrder; + await completer1.future; + return true; + }); + when(() => mockSyncStreamRepo.updatePartnerV1(any())) + .thenAnswer((_) async { + handler2Calls++; + handler2StartOrder = ++callOrder; + await completer2.future; + return true; + }); + + final batch1 = SyncStreamStub.userEvents; + final batch2 = SyncStreamStub.partnerEvents; + + final syncFuture = sut.syncUsers(); + await pumpEventQueue(); + + streamController.add(batch1); + await pumpEventQueue(); + // Small delay to ensure the first handler starts + await Future.delayed(const Duration(milliseconds: 20)); + + expect(handler1StartOrder, 1, reason: "Handler 1 should start first"); + expect(handler1Calls, 1); + + streamController.add(batch2); + await pumpEventQueue(); + // Small delay + await Future.delayed(const Duration(milliseconds: 20)); + + expect(handler2StartOrder, -1, reason: "Handler 2 should wait"); + expect(handler2Calls, 0); + + completer1.complete(); + await pumpEventQueue(times: 40); + // Small delay to ensure the second handler starts + await Future.delayed(const Duration(milliseconds: 20)); + + expect(handler2StartOrder, 2, reason: "Handler 2 should start after H1"); + expect(handler2Calls, 1); + + completer2.complete(); + await pumpEventQueue(times: 40); + // Small delay before closing the stream + await Future.delayed(const Duration(milliseconds: 20)); + + if (!streamController.isClosed) { + await streamController.close(); + } + await pumpEventQueue(times: 40); + // Small delay to ensure the sync completes + await Future.delayed(const Duration(milliseconds: 20)); + + await syncFuture; + + verify(() => mockSyncStreamRepo.updateUsersV1(any())).called(1); + verify(() => mockSyncStreamRepo.updatePartnerV1(any())).called(1); + verify(() => mockSyncApiRepo.ack(any())).called(2); + }); + + test( + "stops processing and ack when cancel checker is completed", + () async { + final cancellationChecker = _MockCancellationWrapper(); + when(() => cancellationChecker.isCancelled()).thenAnswer((_) => false); + + sut = SyncStreamService( + syncApiRepository: mockSyncApiRepo, + syncStreamRepository: mockSyncStreamRepo, + cancelChecker: cancellationChecker.isCancelled, + ); + + final processingCompleter = Completer(); + bool handlerStarted = false; + + // Make handler wait so we can cancel it mid-flight + when(() => mockSyncStreamRepo.deleteUsersV1(any())) + .thenAnswer((_) async { + handlerStarted = true; + await processingCompleter + .future; // Wait indefinitely until test completes it + return true; + }); + + final syncFuture = sut.syncUsers(); + await pumpEventQueue(times: 30); + + streamController.add(SyncStreamStub.userEvents); + // Ensure processing starts + await Future.delayed(const Duration(milliseconds: 10)); + + expect(handlerStarted, isTrue, reason: "Handler should have started"); + + when(() => cancellationChecker.isCancelled()).thenAnswer((_) => true); + + // Allow cancellation logic to propagate + await Future.delayed(const Duration(milliseconds: 10)); + + // Complete the handler's completer after cancellation signal + // to ensure the cancellation logic itself isn't blocked by the handler. + processingCompleter.complete(); + + await expectLater(syncFuture, throwsA(isA())); + + // Verify that ack was NOT called because processing was cancelled + verifyNever(() => mockSyncApiRepo.ack(any())); + }, + ); + + test("completes successfully when ack call throws an exception", () async { + when(() => mockSyncApiRepo.ack(any())).thenThrow(Exception("Ack Error")); + final events = [ + ...SyncStreamStub.userEvents, + ...SyncStreamStub.partnerEvents, + ]; + + // Should still complete even if ack fails + await triggerSyncAndEmit(events); + verify(() => mockSyncApiRepo.ack(any())) + .called(1); // Verify ack was attempted + }); + + test("waits for processing to finish if onDone called early", () async { + final processingCompleter = Completer(); + bool handlerFinished = false; + + when(() => mockSyncStreamRepo.updateUsersV1(any())).thenAnswer((_) async { + await processingCompleter.future; // Wait inside handler + handlerFinished = true; + return true; + }); + + final syncFuture = sut.syncUsers(); + // Allow listener to attach + // This is necessary to ensure the stream is ready to receive events + await Future.delayed(Duration.zero); + + streamController.add(SyncStreamStub.userEvents); // Emit batch + await Future.delayed( + const Duration(milliseconds: 10), + ); // Ensure processing starts + + await streamController + .close(); // Close stream (triggers onDone internally) + await Future.delayed( + const Duration(milliseconds: 10), + ); // Give onDone a chance to fire + + // At this point, onDone was called, but processing is blocked + expect(handlerFinished, isFalse); + + processingCompleter.complete(); // Allow processing to finish + await syncFuture; // Now the main future should complete + + expect(handlerFinished, isTrue); + verify(() => mockSyncApiRepo.ack(any())).called(1); + }); + + test("processes events in the defined _kSyncTypeOrder", () async { + final future = sut.syncUsers(); + await pumpEventQueue(); + if (!streamController.isClosed) { + final events = [ + SyncEvent( + type: SyncEntityType.partnerV1, + data: SyncStreamStub.partnerV1, + ack: "1", + ), // Should be processed last + SyncEvent( + type: SyncEntityType.userV1, + data: SyncStreamStub.userV1Admin, + ack: "2", + ), // Should be processed second + SyncEvent( + type: SyncEntityType.partnerDeleteV1, + data: SyncStreamStub.partnerDeleteV1, + ack: "3", + ), // Should be processed third + SyncEvent( + type: SyncEntityType.userDeleteV1, + data: SyncStreamStub.userDeleteV1, + ack: "4", + ), // Should be processed first + ]; + + streamController.add(events); + await streamController.close(); + } + await future; + + verifyInOrder([ + () => mockSyncStreamRepo.deleteUsersV1(any()), + () => mockSyncStreamRepo.updateUsersV1(any()), + () => mockSyncStreamRepo.deletePartnerV1(any()), + () => mockSyncStreamRepo.updatePartnerV1(any()), + // Verify ack happens after all processing + () => mockSyncApiRepo.ack(any()), + ]); + }); + }); + + group("syncUsers", () { + test("calls getSyncEvents with correct types", () async { + // Need to close the stream for the future to complete + final future = sut.syncUsers(); + await streamController.close(); + await future; + + verify( + () => mockSyncApiRepo.getSyncEvents([ + SyncRequestType.usersV1, + SyncRequestType.partnersV1, + ]), + ).called(1); + }); + + test("calls repository methods with correctly grouped data", () async { + final events = [ + ...SyncStreamStub.userEvents, + ...SyncStreamStub.partnerEvents, + ]; + await triggerSyncAndEmit(events); + + // Verify each handler was called with the correct list of data payloads + verify( + () => mockSyncStreamRepo.updateUsersV1( + [SyncStreamStub.userV1Admin, SyncStreamStub.userV1User], + ), + ).called(1); + verify( + () => mockSyncStreamRepo.deleteUsersV1([SyncStreamStub.userDeleteV1]), + ).called(1); + verify( + () => mockSyncStreamRepo.updatePartnerV1([SyncStreamStub.partnerV1]), + ).called(1); + verify( + () => mockSyncStreamRepo + .deletePartnerV1([SyncStreamStub.partnerDeleteV1]), + ).called(1); + }); + }); +} diff --git a/mobile/test/fixtures/sync_stream.stub.dart b/mobile/test/fixtures/sync_stream.stub.dart new file mode 100644 index 0000000000..781e63a2bb --- /dev/null +++ b/mobile/test/fixtures/sync_stream.stub.dart @@ -0,0 +1,45 @@ +import 'package:immich_mobile/domain/models/sync_event.model.dart'; +import 'package:openapi/api.dart'; + +abstract final class SyncStreamStub { + static final userV1Admin = SyncUserV1( + deletedAt: DateTime(2020), + email: "admin@admin", + id: "1", + name: "Admin", + ); + static final userV1User = SyncUserV1( + deletedAt: DateTime(2021), + email: "user@user", + id: "2", + name: "User", + ); + static final userDeleteV1 = SyncUserDeleteV1(userId: "2"); + static final userEvents = [ + SyncEvent(type: SyncEntityType.userV1, data: userV1Admin, ack: "1"), + SyncEvent( + type: SyncEntityType.userDeleteV1, + data: userDeleteV1, + ack: "2", + ), + SyncEvent(type: SyncEntityType.userV1, data: userV1User, ack: "5"), + ]; + + static final partnerV1 = SyncPartnerV1( + inTimeline: true, + sharedById: "1", + sharedWithId: "2", + ); + static final partnerDeleteV1 = SyncPartnerDeleteV1( + sharedById: "3", + sharedWithId: "4", + ); + static final partnerEvents = [ + SyncEvent( + type: SyncEntityType.partnerDeleteV1, + data: partnerDeleteV1, + ack: "4", + ), + SyncEvent(type: SyncEntityType.partnerV1, data: partnerV1, ack: "3"), + ]; +} diff --git a/mobile/test/infrastructure/repository.mock.dart b/mobile/test/infrastructure/repository.mock.dart index 192858adff..c4a5680f71 100644 --- a/mobile/test/infrastructure/repository.mock.dart +++ b/mobile/test/infrastructure/repository.mock.dart @@ -1,6 +1,8 @@ import 'package:immich_mobile/domain/interfaces/device_asset.interface.dart'; import 'package:immich_mobile/domain/interfaces/log.interface.dart'; import 'package:immich_mobile/domain/interfaces/store.interface.dart'; +import 'package:immich_mobile/domain/interfaces/sync_api.interface.dart'; +import 'package:immich_mobile/domain/interfaces/sync_stream.interface.dart'; import 'package:immich_mobile/domain/interfaces/user.interface.dart'; import 'package:immich_mobile/domain/interfaces/user_api.interface.dart'; import 'package:mocktail/mocktail.dart'; @@ -14,5 +16,9 @@ class MockUserRepository extends Mock implements IUserRepository {} class MockDeviceAssetRepository extends Mock implements IDeviceAssetRepository {} +class MockSyncStreamRepository extends Mock implements ISyncStreamRepository {} + // API Repos class MockUserApiRepository extends Mock implements IUserApiRepository {} + +class MockSyncApiRepository extends Mock implements ISyncApiRepository {} diff --git a/mobile/test/service.mocks.dart b/mobile/test/service.mocks.dart index e1b8df40a3..87a8c01cf0 100644 --- a/mobile/test/service.mocks.dart +++ b/mobile/test/service.mocks.dart @@ -29,4 +29,3 @@ class MockSearchApi extends Mock implements SearchApi {} class MockAppSettingService extends Mock implements AppSettingsService {} class MockBackgroundService extends Mock implements BackgroundService {} - diff --git a/mobile/test/services/auth.service_test.dart b/mobile/test/services/auth.service_test.dart index e4f011d940..4ada98a6c9 100644 --- a/mobile/test/services/auth.service_test.dart +++ b/mobile/test/services/auth.service_test.dart @@ -8,6 +8,7 @@ import 'package:isar/isar.dart'; import 'package:mocktail/mocktail.dart'; import 'package:openapi/api.dart'; +import '../domain/service.mock.dart'; import '../repository.mocks.dart'; import '../service.mocks.dart'; import '../test_utils.dart'; @@ -18,6 +19,7 @@ void main() { late MockAuthRepository authRepository; late MockApiService apiService; late MockNetworkService networkService; + late MockBackgroundSyncManager backgroundSyncManager; late Isar db; setUp(() async { @@ -25,12 +27,14 @@ void main() { authRepository = MockAuthRepository(); apiService = MockApiService(); networkService = MockNetworkService(); + backgroundSyncManager = MockBackgroundSyncManager(); sut = AuthService( authApiRepository, authRepository, apiService, networkService, + backgroundSyncManager, ); registerFallbackValue(Uri()); @@ -116,24 +120,28 @@ void main() { group('logout', () { test('Should logout user', () async { when(() => authApiRepository.logout()).thenAnswer((_) async => {}); + when(() => backgroundSyncManager.cancel()).thenAnswer((_) async => {}); when(() => authRepository.clearLocalData()) .thenAnswer((_) => Future.value(null)); await sut.logout(); verify(() => authApiRepository.logout()).called(1); + verify(() => backgroundSyncManager.cancel()).called(1); verify(() => authRepository.clearLocalData()).called(1); }); test('Should clear local data even on server error', () async { when(() => authApiRepository.logout()) .thenThrow(Exception('Server error')); + when(() => backgroundSyncManager.cancel()).thenAnswer((_) async => {}); when(() => authRepository.clearLocalData()) .thenAnswer((_) => Future.value(null)); await sut.logout(); verify(() => authApiRepository.logout()).called(1); + verify(() => backgroundSyncManager.cancel()).called(1); verify(() => authRepository.clearLocalData()).called(1); }); }); From e275f2d8b390cb95cb861118747be27507e34493 Mon Sep 17 00:00:00 2001 From: Jason Rasmussen Date: Thu, 17 Apr 2025 14:41:06 -0400 Subject: [PATCH 04/29] feat: add foreign key indexes (#17672) --- server/src/decorators.ts | 3 +- .../1744900200559-AddForeignKeyIndexes.ts | 51 ++++++++++++++ server/src/schema/tables/activity.table.ts | 4 +- server/src/schema/tables/album-asset.table.ts | 18 +---- server/src/schema/tables/album.table.ts | 4 +- server/src/schema/tables/api-key.table.ts | 4 +- server/src/schema/tables/asset-audit.table.ts | 11 ++- server/src/schema/tables/asset-face.table.ts | 15 ++++- server/src/schema/tables/asset-files.table.ts | 11 +-- server/src/schema/tables/asset.table.ts | 16 ++--- server/src/schema/tables/exif.table.ts | 14 ++-- server/src/schema/tables/library.table.ts | 4 +- server/src/schema/tables/memory.table.ts | 4 +- .../src/schema/tables/memory_asset.table.ts | 4 +- .../src/schema/tables/partner-audit.table.ts | 11 ++- server/src/schema/tables/partner.table.ts | 20 +++--- server/src/schema/tables/person.table.ts | 4 +- server/src/schema/tables/session.table.ts | 4 +- .../schema/tables/shared-link-asset.table.ts | 4 +- server/src/schema/tables/shared-link.table.ts | 21 +++--- .../schema/tables/sync-checkpoint.table.ts | 13 +--- server/src/schema/tables/tag-asset.table.ts | 8 +-- server/src/schema/tables/tag-closure.table.ts | 8 +-- server/src/schema/tables/tag.table.ts | 11 +-- server/src/schema/tables/user-audit.table.ts | 5 +- .../src/schema/tables/user-metadata.table.ts | 8 ++- server/src/schema/tables/user.table.ts | 4 +- .../decorators/column-index.decorator.ts | 16 ----- .../from-code/decorators/column.decorator.ts | 6 +- .../foreign-key-column.decorator.ts | 2 - .../from-code/decorators/index.decorator.ts | 9 ++- server/src/sql-tools/from-code/index.ts | 16 +++-- .../processors/check-constraint.processor.ts | 4 +- .../processors/column-index.processor.ts | 32 --------- .../from-code/processors/column.processor.ts | 14 +--- .../foreign-key-constriant.processor.ts | 7 +- .../from-code/processors/index.processor.ts | 67 ++++++++++++++++++- .../primary-key-contraint.processor.ts | 4 +- .../from-code/processors/trigger.processor.ts | 6 +- .../sql-tools/from-code/processors/type.ts | 3 +- .../processors/unique-constraint.processor.ts | 33 ++++++++- .../sql-tools/from-code/register-function.ts | 35 +++++++++- .../src/sql-tools/from-code/register-item.ts | 2 - server/src/sql-tools/helpers.ts | 55 --------------- server/src/sql-tools/public_api.ts | 1 - .../sql-tools/column-index-name-default.ts | 5 +- server/test/sql-tools/column-index-name.ts | 46 +++++++++++++ .../foreign-key-inferred-type.stub.ts | 10 ++- ...foreign-key-with-unique-constraint.stub.ts | 10 ++- 49 files changed, 382 insertions(+), 285 deletions(-) create mode 100644 server/src/migrations/1744900200559-AddForeignKeyIndexes.ts delete mode 100644 server/src/sql-tools/from-code/decorators/column-index.decorator.ts delete mode 100644 server/src/sql-tools/from-code/processors/column-index.processor.ts create mode 100644 server/test/sql-tools/column-index-name.ts diff --git a/server/src/decorators.ts b/server/src/decorators.ts index 7085899af7..1af9342e0b 100644 --- a/server/src/decorators.ts +++ b/server/src/decorators.ts @@ -11,7 +11,8 @@ import { setUnion } from 'src/utils/set'; const GeneratedUuidV7Column = (options: Omit = {}) => Column({ ...options, type: 'uuid', nullable: false, default: () => `${immich_uuid_v7.name}()` }); -export const UpdateIdColumn = () => GeneratedUuidV7Column(); +export const UpdateIdColumn = (options: Omit = {}) => + GeneratedUuidV7Column(options); export const PrimaryGeneratedUuidV7Column = () => GeneratedUuidV7Column({ primary: true }); diff --git a/server/src/migrations/1744900200559-AddForeignKeyIndexes.ts b/server/src/migrations/1744900200559-AddForeignKeyIndexes.ts new file mode 100644 index 0000000000..db351d5bab --- /dev/null +++ b/server/src/migrations/1744900200559-AddForeignKeyIndexes.ts @@ -0,0 +1,51 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AddForeignKeyIndexes1744900200559 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`CREATE INDEX "IDX_0f6fc2fb195f24d19b0fb0d57c" ON "libraries" ("ownerId")`); + await queryRunner.query(`CREATE INDEX "IDX_91704e101438fd0653f582426d" ON "asset_stack" ("primaryAssetId")`); + await queryRunner.query(`CREATE INDEX "IDX_c05079e542fd74de3b5ecb5c1c" ON "asset_stack" ("ownerId")`); + await queryRunner.query(`CREATE INDEX "IDX_2c5ac0d6fb58b238fd2068de67" ON "assets" ("ownerId")`); + await queryRunner.query(`CREATE INDEX "IDX_16294b83fa8c0149719a1f631e" ON "assets" ("livePhotoVideoId")`); + await queryRunner.query(`CREATE INDEX "IDX_9977c3c1de01c3d848039a6b90" ON "assets" ("libraryId")`); + await queryRunner.query(`CREATE INDEX "IDX_f15d48fa3ea5e4bda05ca8ab20" ON "assets" ("stackId")`); + await queryRunner.query(`CREATE INDEX "IDX_b22c53f35ef20c28c21637c85f" ON "albums" ("ownerId")`); + await queryRunner.query(`CREATE INDEX "IDX_05895aa505a670300d4816debc" ON "albums" ("albumThumbnailAssetId")`); + await queryRunner.query(`CREATE INDEX "IDX_1af8519996fbfb3684b58df280" ON "activity" ("albumId")`); + await queryRunner.query(`CREATE INDEX "IDX_3571467bcbe021f66e2bdce96e" ON "activity" ("userId")`); + await queryRunner.query(`CREATE INDEX "IDX_8091ea76b12338cb4428d33d78" ON "activity" ("assetId")`); + await queryRunner.query(`CREATE INDEX "IDX_6c2e267ae764a9413b863a2934" ON "api_keys" ("userId")`); + await queryRunner.query(`CREATE INDEX "IDX_5527cc99f530a547093f9e577b" ON "person" ("ownerId")`); + await queryRunner.query(`CREATE INDEX "IDX_2bbabe31656b6778c6b87b6102" ON "person" ("faceAssetId")`); + await queryRunner.query(`CREATE INDEX "IDX_575842846f0c28fa5da46c99b1" ON "memories" ("ownerId")`); + await queryRunner.query(`CREATE INDEX "IDX_d7e875c6c60e661723dbf372fd" ON "partners" ("sharedWithId")`); + await queryRunner.query(`CREATE INDEX "IDX_57de40bc620f456c7311aa3a1e" ON "sessions" ("userId")`); + await queryRunner.query(`CREATE INDEX "IDX_66fe3837414c5a9f1c33ca4934" ON "shared_links" ("userId")`); + await queryRunner.query(`CREATE INDEX "IDX_d8ddd9d687816cc490432b3d4b" ON "session_sync_checkpoints" ("sessionId")`); + await queryRunner.query(`CREATE INDEX "IDX_9f9590cc11561f1f48ff034ef9" ON "tags" ("parentId")`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX "IDX_66fe3837414c5a9f1c33ca4934";`); + await queryRunner.query(`DROP INDEX "IDX_91704e101438fd0653f582426d";`); + await queryRunner.query(`DROP INDEX "IDX_c05079e542fd74de3b5ecb5c1c";`); + await queryRunner.query(`DROP INDEX "IDX_5527cc99f530a547093f9e577b";`); + await queryRunner.query(`DROP INDEX "IDX_2bbabe31656b6778c6b87b6102";`); + await queryRunner.query(`DROP INDEX "IDX_0f6fc2fb195f24d19b0fb0d57c";`); + await queryRunner.query(`DROP INDEX "IDX_9f9590cc11561f1f48ff034ef9";`); + await queryRunner.query(`DROP INDEX "IDX_2c5ac0d6fb58b238fd2068de67";`); + await queryRunner.query(`DROP INDEX "IDX_16294b83fa8c0149719a1f631e";`); + await queryRunner.query(`DROP INDEX "IDX_9977c3c1de01c3d848039a6b90";`); + await queryRunner.query(`DROP INDEX "IDX_f15d48fa3ea5e4bda05ca8ab20";`); + await queryRunner.query(`DROP INDEX "IDX_b22c53f35ef20c28c21637c85f";`); + await queryRunner.query(`DROP INDEX "IDX_05895aa505a670300d4816debc";`); + await queryRunner.query(`DROP INDEX "IDX_57de40bc620f456c7311aa3a1e";`); + await queryRunner.query(`DROP INDEX "IDX_d8ddd9d687816cc490432b3d4b";`); + await queryRunner.query(`DROP INDEX "IDX_d7e875c6c60e661723dbf372fd";`); + await queryRunner.query(`DROP INDEX "IDX_575842846f0c28fa5da46c99b1";`); + await queryRunner.query(`DROP INDEX "IDX_6c2e267ae764a9413b863a2934";`); + await queryRunner.query(`DROP INDEX "IDX_1af8519996fbfb3684b58df280";`); + await queryRunner.query(`DROP INDEX "IDX_3571467bcbe021f66e2bdce96e";`); + await queryRunner.query(`DROP INDEX "IDX_8091ea76b12338cb4428d33d78";`); + } +} diff --git a/server/src/schema/tables/activity.table.ts b/server/src/schema/tables/activity.table.ts index e7a144722c..802a86a303 100644 --- a/server/src/schema/tables/activity.table.ts +++ b/server/src/schema/tables/activity.table.ts @@ -5,7 +5,6 @@ import { UserTable } from 'src/schema/tables/user.table'; import { Check, Column, - ColumnIndex, CreateDateColumn, ForeignKeyColumn, Index, @@ -51,7 +50,6 @@ export class ActivityTable { @Column({ type: 'boolean', default: false }) isLiked!: boolean; - @ColumnIndex('IDX_activity_update_id') - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_activity_update_id' }) updateId!: string; } diff --git a/server/src/schema/tables/album-asset.table.ts b/server/src/schema/tables/album-asset.table.ts index 1b931e3116..8054009c39 100644 --- a/server/src/schema/tables/album-asset.table.ts +++ b/server/src/schema/tables/album-asset.table.ts @@ -1,25 +1,13 @@ import { AlbumTable } from 'src/schema/tables/album.table'; import { AssetTable } from 'src/schema/tables/asset.table'; -import { ColumnIndex, CreateDateColumn, ForeignKeyColumn, Table } from 'src/sql-tools'; +import { CreateDateColumn, ForeignKeyColumn, Table } from 'src/sql-tools'; @Table({ name: 'albums_assets_assets', primaryConstraintName: 'PK_c67bc36fa845fb7b18e0e398180' }) export class AlbumAssetTable { - @ForeignKeyColumn(() => AlbumTable, { - onDelete: 'CASCADE', - onUpdate: 'CASCADE', - nullable: false, - primary: true, - }) - @ColumnIndex() + @ForeignKeyColumn(() => AlbumTable, { onDelete: 'CASCADE', onUpdate: 'CASCADE', nullable: false, primary: true }) albumsId!: string; - @ForeignKeyColumn(() => AssetTable, { - onDelete: 'CASCADE', - onUpdate: 'CASCADE', - nullable: false, - primary: true, - }) - @ColumnIndex() + @ForeignKeyColumn(() => AssetTable, { onDelete: 'CASCADE', onUpdate: 'CASCADE', nullable: false, primary: true }) assetsId!: string; @CreateDateColumn() diff --git a/server/src/schema/tables/album.table.ts b/server/src/schema/tables/album.table.ts index cdfd092b1b..428947fa51 100644 --- a/server/src/schema/tables/album.table.ts +++ b/server/src/schema/tables/album.table.ts @@ -4,7 +4,6 @@ import { AssetTable } from 'src/schema/tables/asset.table'; import { UserTable } from 'src/schema/tables/user.table'; import { Column, - ColumnIndex, CreateDateColumn, DeleteDateColumn, ForeignKeyColumn, @@ -51,7 +50,6 @@ export class AlbumTable { @Column({ default: AssetOrder.DESC }) order!: AssetOrder; - @ColumnIndex('IDX_albums_update_id') - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_albums_update_id' }) updateId?: string; } diff --git a/server/src/schema/tables/api-key.table.ts b/server/src/schema/tables/api-key.table.ts index 29c4ad2b0f..1d4cc83172 100644 --- a/server/src/schema/tables/api-key.table.ts +++ b/server/src/schema/tables/api-key.table.ts @@ -3,7 +3,6 @@ import { Permission } from 'src/enum'; import { UserTable } from 'src/schema/tables/user.table'; import { Column, - ColumnIndex, CreateDateColumn, ForeignKeyColumn, PrimaryGeneratedColumn, @@ -35,7 +34,6 @@ export class APIKeyTable { @Column({ array: true, type: 'character varying' }) permissions!: Permission[]; - @ColumnIndex({ name: 'IDX_api_keys_update_id' }) - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_api_keys_update_id' }) updateId?: string; } diff --git a/server/src/schema/tables/asset-audit.table.ts b/server/src/schema/tables/asset-audit.table.ts index 55d6f5c911..030256480c 100644 --- a/server/src/schema/tables/asset-audit.table.ts +++ b/server/src/schema/tables/asset-audit.table.ts @@ -1,20 +1,17 @@ import { PrimaryGeneratedUuidV7Column } from 'src/decorators'; -import { Column, ColumnIndex, CreateDateColumn, Table } from 'src/sql-tools'; +import { Column, CreateDateColumn, Table } from 'src/sql-tools'; @Table('assets_audit') export class AssetAuditTable { @PrimaryGeneratedUuidV7Column() id!: string; - @ColumnIndex('IDX_assets_audit_asset_id') - @Column({ type: 'uuid' }) + @Column({ type: 'uuid', indexName: 'IDX_assets_audit_asset_id' }) assetId!: string; - @ColumnIndex('IDX_assets_audit_owner_id') - @Column({ type: 'uuid' }) + @Column({ type: 'uuid', indexName: 'IDX_assets_audit_owner_id' }) ownerId!: string; - @ColumnIndex('IDX_assets_audit_deleted_at') - @CreateDateColumn({ default: () => 'clock_timestamp()' }) + @CreateDateColumn({ default: () => 'clock_timestamp()', indexName: 'IDX_assets_audit_deleted_at' }) deletedAt!: Date; } diff --git a/server/src/schema/tables/asset-face.table.ts b/server/src/schema/tables/asset-face.table.ts index 0ae99f44bf..52f4364a93 100644 --- a/server/src/schema/tables/asset-face.table.ts +++ b/server/src/schema/tables/asset-face.table.ts @@ -8,10 +8,21 @@ import { Column, DeleteDateColumn, ForeignKeyColumn, Index, PrimaryGeneratedColu @Index({ name: 'IDX_asset_faces_assetId_personId', columns: ['assetId', 'personId'] }) @Index({ columns: ['personId', 'assetId'] }) export class AssetFaceTable { - @ForeignKeyColumn(() => AssetTable, { onDelete: 'CASCADE', onUpdate: 'CASCADE' }) + @ForeignKeyColumn(() => AssetTable, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE', + // [assetId, personId] is the PK constraint + index: false, + }) assetId!: string; - @ForeignKeyColumn(() => PersonTable, { onDelete: 'SET NULL', onUpdate: 'CASCADE', nullable: true }) + @ForeignKeyColumn(() => PersonTable, { + onDelete: 'SET NULL', + onUpdate: 'CASCADE', + nullable: true, + // [personId, assetId] makes this redundant + index: false, + }) personId!: string | null; @Column({ default: 0, type: 'integer' }) diff --git a/server/src/schema/tables/asset-files.table.ts b/server/src/schema/tables/asset-files.table.ts index fb8750a8ef..0859bd5cf0 100644 --- a/server/src/schema/tables/asset-files.table.ts +++ b/server/src/schema/tables/asset-files.table.ts @@ -3,7 +3,6 @@ import { AssetFileType } from 'src/enum'; import { AssetTable } from 'src/schema/tables/asset.table'; import { Column, - ColumnIndex, CreateDateColumn, ForeignKeyColumn, PrimaryGeneratedColumn, @@ -19,8 +18,11 @@ export class AssetFileTable { @PrimaryGeneratedColumn() id!: string; - @ColumnIndex('IDX_asset_files_assetId') - @ForeignKeyColumn(() => AssetTable, { onDelete: 'CASCADE', onUpdate: 'CASCADE' }) + @ForeignKeyColumn(() => AssetTable, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE', + indexName: 'IDX_asset_files_assetId', + }) assetId?: string; @CreateDateColumn() @@ -35,7 +37,6 @@ export class AssetFileTable { @Column() path!: string; - @ColumnIndex('IDX_asset_files_update_id') - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_asset_files_update_id' }) updateId?: string; } diff --git a/server/src/schema/tables/asset.table.ts b/server/src/schema/tables/asset.table.ts index 250c3546a2..9a9670cd0e 100644 --- a/server/src/schema/tables/asset.table.ts +++ b/server/src/schema/tables/asset.table.ts @@ -9,7 +9,6 @@ import { UserTable } from 'src/schema/tables/user.table'; import { AfterDeleteTrigger, Column, - ColumnIndex, CreateDateColumn, DeleteDateColumn, ForeignKeyColumn, @@ -78,8 +77,7 @@ export class AssetTable { @Column() originalPath!: string; - @ColumnIndex('idx_asset_file_created_at') - @Column({ type: 'timestamp with time zone' }) + @Column({ type: 'timestamp with time zone', indexName: 'idx_asset_file_created_at' }) fileCreatedAt!: Date; @Column({ type: 'timestamp with time zone' }) @@ -94,8 +92,7 @@ export class AssetTable { @Column({ type: 'character varying', nullable: true, default: '' }) encodedVideoPath!: string | null; - @Column({ type: 'bytea' }) - @ColumnIndex() + @Column({ type: 'bytea', index: true }) checksum!: Buffer; // sha1 checksum @Column({ type: 'boolean', default: true }) @@ -113,8 +110,7 @@ export class AssetTable { @Column({ type: 'boolean', default: false }) isArchived!: boolean; - @Column() - @ColumnIndex() + @Column({ index: true }) originalFileName!: string; @Column({ nullable: true }) @@ -141,14 +137,12 @@ export class AssetTable { @ForeignKeyColumn(() => StackTable, { nullable: true, onDelete: 'SET NULL', onUpdate: 'CASCADE' }) stackId?: string | null; - @ColumnIndex('IDX_assets_duplicateId') - @Column({ type: 'uuid', nullable: true }) + @Column({ type: 'uuid', nullable: true, indexName: 'IDX_assets_duplicateId' }) duplicateId!: string | null; @Column({ enum: assets_status_enum, default: AssetStatus.ACTIVE }) status!: AssetStatus; - @ColumnIndex('IDX_assets_update_id') - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_assets_update_id' }) updateId?: string; } diff --git a/server/src/schema/tables/exif.table.ts b/server/src/schema/tables/exif.table.ts index e40ce94b4f..ca300945c3 100644 --- a/server/src/schema/tables/exif.table.ts +++ b/server/src/schema/tables/exif.table.ts @@ -1,6 +1,6 @@ import { UpdatedAtTrigger, UpdateIdColumn } from 'src/decorators'; import { AssetTable } from 'src/schema/tables/asset.table'; -import { Column, ColumnIndex, ForeignKeyColumn, Table, UpdateDateColumn } from 'src/sql-tools'; +import { Column, ForeignKeyColumn, Table, UpdateDateColumn } from 'src/sql-tools'; @Table('exif') @UpdatedAtTrigger('asset_exif_updated_at') @@ -50,8 +50,7 @@ export class ExifTable { @Column({ type: 'double precision', nullable: true }) longitude!: number | null; - @ColumnIndex('exif_city') - @Column({ type: 'character varying', nullable: true }) + @Column({ type: 'character varying', nullable: true, indexName: 'exif_city' }) city!: string | null; @Column({ type: 'character varying', nullable: true }) @@ -69,8 +68,7 @@ export class ExifTable { @Column({ type: 'character varying', nullable: true }) exposureTime!: string | null; - @ColumnIndex('IDX_live_photo_cid') - @Column({ type: 'character varying', nullable: true }) + @Column({ type: 'character varying', nullable: true, indexName: 'IDX_live_photo_cid' }) livePhotoCID!: string | null; @Column({ type: 'character varying', nullable: true }) @@ -88,8 +86,7 @@ export class ExifTable { @Column({ type: 'integer', nullable: true }) bitsPerSample!: number | null; - @ColumnIndex('IDX_auto_stack_id') - @Column({ type: 'character varying', nullable: true }) + @Column({ type: 'character varying', nullable: true, indexName: 'IDX_auto_stack_id' }) autoStackId!: string | null; @Column({ type: 'integer', nullable: true }) @@ -98,7 +95,6 @@ export class ExifTable { @UpdateDateColumn({ default: () => 'clock_timestamp()' }) updatedAt?: Date; - @ColumnIndex('IDX_asset_exif_update_id') - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_asset_exif_update_id' }) updateId?: string; } diff --git a/server/src/schema/tables/library.table.ts b/server/src/schema/tables/library.table.ts index 54b3752f41..8b21d5feb0 100644 --- a/server/src/schema/tables/library.table.ts +++ b/server/src/schema/tables/library.table.ts @@ -2,7 +2,6 @@ import { UpdatedAtTrigger, UpdateIdColumn } from 'src/decorators'; import { UserTable } from 'src/schema/tables/user.table'; import { Column, - ColumnIndex, CreateDateColumn, DeleteDateColumn, ForeignKeyColumn, @@ -41,7 +40,6 @@ export class LibraryTable { @Column({ type: 'timestamp with time zone', nullable: true }) refreshedAt!: Date | null; - @ColumnIndex('IDX_libraries_update_id') - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_libraries_update_id' }) updateId?: string; } diff --git a/server/src/schema/tables/memory.table.ts b/server/src/schema/tables/memory.table.ts index 1926405565..32dafe3384 100644 --- a/server/src/schema/tables/memory.table.ts +++ b/server/src/schema/tables/memory.table.ts @@ -3,7 +3,6 @@ import { MemoryType } from 'src/enum'; import { UserTable } from 'src/schema/tables/user.table'; import { Column, - ColumnIndex, CreateDateColumn, DeleteDateColumn, ForeignKeyColumn, @@ -55,7 +54,6 @@ export class MemoryTable { @Column({ type: 'timestamp with time zone', nullable: true }) hideAt?: Date; - @ColumnIndex('IDX_memories_update_id') - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_memories_update_id' }) updateId?: string; } diff --git a/server/src/schema/tables/memory_asset.table.ts b/server/src/schema/tables/memory_asset.table.ts index 864e6291c7..0e5ca29a08 100644 --- a/server/src/schema/tables/memory_asset.table.ts +++ b/server/src/schema/tables/memory_asset.table.ts @@ -1,14 +1,12 @@ import { AssetTable } from 'src/schema/tables/asset.table'; import { MemoryTable } from 'src/schema/tables/memory.table'; -import { ColumnIndex, ForeignKeyColumn, Table } from 'src/sql-tools'; +import { ForeignKeyColumn, Table } from 'src/sql-tools'; @Table('memories_assets_assets') export class MemoryAssetTable { - @ColumnIndex() @ForeignKeyColumn(() => MemoryTable, { onUpdate: 'CASCADE', onDelete: 'CASCADE', primary: true }) memoriesId!: string; - @ColumnIndex() @ForeignKeyColumn(() => AssetTable, { onUpdate: 'CASCADE', onDelete: 'CASCADE', primary: true }) assetsId!: string; } diff --git a/server/src/schema/tables/partner-audit.table.ts b/server/src/schema/tables/partner-audit.table.ts index 08b6e94626..da5243dc75 100644 --- a/server/src/schema/tables/partner-audit.table.ts +++ b/server/src/schema/tables/partner-audit.table.ts @@ -1,20 +1,17 @@ import { PrimaryGeneratedUuidV7Column } from 'src/decorators'; -import { Column, ColumnIndex, CreateDateColumn, Table } from 'src/sql-tools'; +import { Column, CreateDateColumn, Table } from 'src/sql-tools'; @Table('partners_audit') export class PartnerAuditTable { @PrimaryGeneratedUuidV7Column() id!: string; - @ColumnIndex('IDX_partners_audit_shared_by_id') - @Column({ type: 'uuid' }) + @Column({ type: 'uuid', indexName: 'IDX_partners_audit_shared_by_id' }) sharedById!: string; - @ColumnIndex('IDX_partners_audit_shared_with_id') - @Column({ type: 'uuid' }) + @Column({ type: 'uuid', indexName: 'IDX_partners_audit_shared_with_id' }) sharedWithId!: string; - @ColumnIndex('IDX_partners_audit_deleted_at') - @CreateDateColumn({ default: () => 'clock_timestamp()' }) + @CreateDateColumn({ default: () => 'clock_timestamp()', indexName: 'IDX_partners_audit_deleted_at' }) deletedAt!: Date; } diff --git a/server/src/schema/tables/partner.table.ts b/server/src/schema/tables/partner.table.ts index 770107fe7a..0da60cfc0c 100644 --- a/server/src/schema/tables/partner.table.ts +++ b/server/src/schema/tables/partner.table.ts @@ -1,15 +1,7 @@ import { UpdatedAtTrigger, UpdateIdColumn } from 'src/decorators'; import { partners_delete_audit } from 'src/schema/functions'; import { UserTable } from 'src/schema/tables/user.table'; -import { - AfterDeleteTrigger, - Column, - ColumnIndex, - CreateDateColumn, - ForeignKeyColumn, - Table, - UpdateDateColumn, -} from 'src/sql-tools'; +import { AfterDeleteTrigger, Column, CreateDateColumn, ForeignKeyColumn, Table, UpdateDateColumn } from 'src/sql-tools'; @Table('partners') @UpdatedAtTrigger('partners_updated_at') @@ -21,7 +13,12 @@ import { when: 'pg_trigger_depth() = 0', }) export class PartnerTable { - @ForeignKeyColumn(() => UserTable, { onDelete: 'CASCADE', primary: true }) + @ForeignKeyColumn(() => UserTable, { + onDelete: 'CASCADE', + primary: true, + // [sharedById, sharedWithId] is the PK constraint + index: false, + }) sharedById!: string; @ForeignKeyColumn(() => UserTable, { onDelete: 'CASCADE', primary: true }) @@ -36,7 +33,6 @@ export class PartnerTable { @Column({ type: 'boolean', default: false }) inTimeline!: boolean; - @ColumnIndex('IDX_partners_update_id') - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_partners_update_id' }) updateId!: string; } diff --git a/server/src/schema/tables/person.table.ts b/server/src/schema/tables/person.table.ts index b96fc5b709..1320b91f18 100644 --- a/server/src/schema/tables/person.table.ts +++ b/server/src/schema/tables/person.table.ts @@ -4,7 +4,6 @@ import { UserTable } from 'src/schema/tables/user.table'; import { Check, Column, - ColumnIndex, CreateDateColumn, ForeignKeyColumn, PrimaryGeneratedColumn, @@ -49,7 +48,6 @@ export class PersonTable { @Column({ type: 'character varying', nullable: true, default: null }) color?: string | null; - @ColumnIndex('IDX_person_update_id') - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_person_update_id' }) updateId!: string; } diff --git a/server/src/schema/tables/session.table.ts b/server/src/schema/tables/session.table.ts index a66732a7d9..ad43d0d6e4 100644 --- a/server/src/schema/tables/session.table.ts +++ b/server/src/schema/tables/session.table.ts @@ -2,7 +2,6 @@ import { UpdatedAtTrigger, UpdateIdColumn } from 'src/decorators'; import { UserTable } from 'src/schema/tables/user.table'; import { Column, - ColumnIndex, CreateDateColumn, ForeignKeyColumn, PrimaryGeneratedColumn, @@ -35,7 +34,6 @@ export class SessionTable { @Column({ default: '' }) deviceOS!: string; - @ColumnIndex('IDX_sessions_update_id') - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_sessions_update_id' }) updateId!: string; } diff --git a/server/src/schema/tables/shared-link-asset.table.ts b/server/src/schema/tables/shared-link-asset.table.ts index 1eb294c1e8..66c9068441 100644 --- a/server/src/schema/tables/shared-link-asset.table.ts +++ b/server/src/schema/tables/shared-link-asset.table.ts @@ -1,14 +1,12 @@ import { AssetTable } from 'src/schema/tables/asset.table'; import { SharedLinkTable } from 'src/schema/tables/shared-link.table'; -import { ColumnIndex, ForeignKeyColumn, Table } from 'src/sql-tools'; +import { ForeignKeyColumn, Table } from 'src/sql-tools'; @Table('shared_link__asset') export class SharedLinkAssetTable { - @ColumnIndex() @ForeignKeyColumn(() => AssetTable, { onUpdate: 'CASCADE', onDelete: 'CASCADE', primary: true }) assetsId!: string; - @ColumnIndex() @ForeignKeyColumn(() => SharedLinkTable, { onUpdate: 'CASCADE', onDelete: 'CASCADE', primary: true }) sharedLinksId!: string; } diff --git a/server/src/schema/tables/shared-link.table.ts b/server/src/schema/tables/shared-link.table.ts index 36237c58ef..3bb36b36ed 100644 --- a/server/src/schema/tables/shared-link.table.ts +++ b/server/src/schema/tables/shared-link.table.ts @@ -1,15 +1,7 @@ import { SharedLinkType } from 'src/enum'; import { AlbumTable } from 'src/schema/tables/album.table'; import { UserTable } from 'src/schema/tables/user.table'; -import { - Column, - ColumnIndex, - CreateDateColumn, - ForeignKeyColumn, - PrimaryGeneratedColumn, - Table, - Unique, -} from 'src/sql-tools'; +import { Column, CreateDateColumn, ForeignKeyColumn, PrimaryGeneratedColumn, Table, Unique } from 'src/sql-tools'; @Table('shared_links') @Unique({ name: 'UQ_sharedlink_key', columns: ['key'] }) @@ -23,8 +15,7 @@ export class SharedLinkTable { @ForeignKeyColumn(() => UserTable, { onDelete: 'CASCADE', onUpdate: 'CASCADE' }) userId!: string; - @ColumnIndex('IDX_sharedlink_key') - @Column({ type: 'bytea' }) + @Column({ type: 'bytea', indexName: 'IDX_sharedlink_key' }) key!: Buffer; // use to access the inidividual asset @Column() @@ -39,8 +30,12 @@ export class SharedLinkTable { @Column({ type: 'boolean', default: false }) allowUpload!: boolean; - @ColumnIndex('IDX_sharedlink_albumId') - @ForeignKeyColumn(() => AlbumTable, { nullable: true, onDelete: 'CASCADE', onUpdate: 'CASCADE' }) + @ForeignKeyColumn(() => AlbumTable, { + nullable: true, + onDelete: 'CASCADE', + onUpdate: 'CASCADE', + indexName: 'IDX_sharedlink_albumId', + }) albumId!: string; @Column({ type: 'boolean', default: true }) diff --git a/server/src/schema/tables/sync-checkpoint.table.ts b/server/src/schema/tables/sync-checkpoint.table.ts index 831205ce7a..21fd7983ac 100644 --- a/server/src/schema/tables/sync-checkpoint.table.ts +++ b/server/src/schema/tables/sync-checkpoint.table.ts @@ -1,15 +1,7 @@ import { UpdatedAtTrigger, UpdateIdColumn } from 'src/decorators'; import { SyncEntityType } from 'src/enum'; import { SessionTable } from 'src/schema/tables/session.table'; -import { - Column, - ColumnIndex, - CreateDateColumn, - ForeignKeyColumn, - PrimaryColumn, - Table, - UpdateDateColumn, -} from 'src/sql-tools'; +import { Column, CreateDateColumn, ForeignKeyColumn, PrimaryColumn, Table, UpdateDateColumn } from 'src/sql-tools'; @Table('session_sync_checkpoints') @UpdatedAtTrigger('session_sync_checkpoints_updated_at') @@ -29,7 +21,6 @@ export class SessionSyncCheckpointTable { @Column() ack!: string; - @ColumnIndex('IDX_session_sync_checkpoints_update_id') - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_session_sync_checkpoints_update_id' }) updateId!: string; } diff --git a/server/src/schema/tables/tag-asset.table.ts b/server/src/schema/tables/tag-asset.table.ts index 5f24799cec..8793af0a8a 100644 --- a/server/src/schema/tables/tag-asset.table.ts +++ b/server/src/schema/tables/tag-asset.table.ts @@ -1,15 +1,13 @@ import { AssetTable } from 'src/schema/tables/asset.table'; import { TagTable } from 'src/schema/tables/tag.table'; -import { ColumnIndex, ForeignKeyColumn, Index, Table } from 'src/sql-tools'; +import { ForeignKeyColumn, Index, Table } from 'src/sql-tools'; @Index({ name: 'IDX_tag_asset_assetsId_tagsId', columns: ['assetsId', 'tagsId'] }) @Table('tag_asset') export class TagAssetTable { - @ColumnIndex() - @ForeignKeyColumn(() => AssetTable, { onUpdate: 'CASCADE', onDelete: 'CASCADE', primary: true }) + @ForeignKeyColumn(() => AssetTable, { onUpdate: 'CASCADE', onDelete: 'CASCADE', primary: true, index: true }) assetsId!: string; - @ColumnIndex() - @ForeignKeyColumn(() => TagTable, { onUpdate: 'CASCADE', onDelete: 'CASCADE', primary: true }) + @ForeignKeyColumn(() => TagTable, { onUpdate: 'CASCADE', onDelete: 'CASCADE', primary: true, index: true }) tagsId!: string; } diff --git a/server/src/schema/tables/tag-closure.table.ts b/server/src/schema/tables/tag-closure.table.ts index acde84b91d..8829e802e1 100644 --- a/server/src/schema/tables/tag-closure.table.ts +++ b/server/src/schema/tables/tag-closure.table.ts @@ -1,13 +1,11 @@ import { TagTable } from 'src/schema/tables/tag.table'; -import { ColumnIndex, ForeignKeyColumn, Table } from 'src/sql-tools'; +import { ForeignKeyColumn, Table } from 'src/sql-tools'; @Table('tags_closure') export class TagClosureTable { - @ColumnIndex() - @ForeignKeyColumn(() => TagTable, { primary: true, onDelete: 'CASCADE', onUpdate: 'NO ACTION' }) + @ForeignKeyColumn(() => TagTable, { primary: true, onDelete: 'CASCADE', onUpdate: 'NO ACTION', index: true }) id_ancestor!: string; - @ColumnIndex() - @ForeignKeyColumn(() => TagTable, { primary: true, onDelete: 'CASCADE', onUpdate: 'NO ACTION' }) + @ForeignKeyColumn(() => TagTable, { primary: true, onDelete: 'CASCADE', onUpdate: 'NO ACTION', index: true }) id_descendant!: string; } diff --git a/server/src/schema/tables/tag.table.ts b/server/src/schema/tables/tag.table.ts index 5042e2eb0e..a9f2a57f27 100644 --- a/server/src/schema/tables/tag.table.ts +++ b/server/src/schema/tables/tag.table.ts @@ -2,7 +2,6 @@ import { UpdatedAtTrigger, UpdateIdColumn } from 'src/decorators'; import { UserTable } from 'src/schema/tables/user.table'; import { Column, - ColumnIndex, CreateDateColumn, ForeignKeyColumn, PrimaryGeneratedColumn, @@ -18,7 +17,12 @@ export class TagTable { @PrimaryGeneratedColumn() id!: string; - @ForeignKeyColumn(() => UserTable, { onUpdate: 'CASCADE', onDelete: 'CASCADE' }) + @ForeignKeyColumn(() => UserTable, { + onUpdate: 'CASCADE', + onDelete: 'CASCADE', + // [userId, value] makes this redundant + index: false, + }) userId!: string; @Column() @@ -36,7 +40,6 @@ export class TagTable { @ForeignKeyColumn(() => TagTable, { nullable: true, onDelete: 'CASCADE' }) parentId?: string; - @ColumnIndex('IDX_tags_update_id') - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_tags_update_id' }) updateId!: string; } diff --git a/server/src/schema/tables/user-audit.table.ts b/server/src/schema/tables/user-audit.table.ts index 0f881ccc9a..e0c9afcdc3 100644 --- a/server/src/schema/tables/user-audit.table.ts +++ b/server/src/schema/tables/user-audit.table.ts @@ -1,13 +1,12 @@ import { PrimaryGeneratedUuidV7Column } from 'src/decorators'; -import { Column, ColumnIndex, CreateDateColumn, Table } from 'src/sql-tools'; +import { Column, CreateDateColumn, Table } from 'src/sql-tools'; @Table('users_audit') export class UserAuditTable { @Column({ type: 'uuid' }) userId!: string; - @ColumnIndex('IDX_users_audit_deleted_at') - @CreateDateColumn({ default: () => 'clock_timestamp()' }) + @CreateDateColumn({ default: () => 'clock_timestamp()', indexName: 'IDX_users_audit_deleted_at' }) deletedAt!: Date; @PrimaryGeneratedUuidV7Column() diff --git a/server/src/schema/tables/user-metadata.table.ts b/server/src/schema/tables/user-metadata.table.ts index 6d03acaf80..04b457867f 100644 --- a/server/src/schema/tables/user-metadata.table.ts +++ b/server/src/schema/tables/user-metadata.table.ts @@ -5,7 +5,13 @@ import { UserMetadata, UserMetadataItem } from 'src/types'; @Table('user_metadata') export class UserMetadataTable implements UserMetadataItem { - @ForeignKeyColumn(() => UserTable, { onUpdate: 'CASCADE', onDelete: 'CASCADE', primary: true }) + @ForeignKeyColumn(() => UserTable, { + onUpdate: 'CASCADE', + onDelete: 'CASCADE', + primary: true, + // [userId, key] is the PK constraint + index: false, + }) userId!: string; @PrimaryColumn({ type: 'character varying' }) diff --git a/server/src/schema/tables/user.table.ts b/server/src/schema/tables/user.table.ts index 5160f979b9..eeef923796 100644 --- a/server/src/schema/tables/user.table.ts +++ b/server/src/schema/tables/user.table.ts @@ -5,7 +5,6 @@ import { users_delete_audit } from 'src/schema/functions'; import { AfterDeleteTrigger, Column, - ColumnIndex, CreateDateColumn, DeleteDateColumn, Index, @@ -77,7 +76,6 @@ export class UserTable { @Column({ type: 'timestamp with time zone', default: () => 'now()' }) profileChangedAt!: Generated; - @ColumnIndex({ name: 'IDX_users_update_id' }) - @UpdateIdColumn() + @UpdateIdColumn({ indexName: 'IDX_users_update_id' }) updateId!: Generated; } diff --git a/server/src/sql-tools/from-code/decorators/column-index.decorator.ts b/server/src/sql-tools/from-code/decorators/column-index.decorator.ts deleted file mode 100644 index ab15292612..0000000000 --- a/server/src/sql-tools/from-code/decorators/column-index.decorator.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { register } from 'src/sql-tools/from-code/register'; -import { asOptions } from 'src/sql-tools/helpers'; - -export type ColumnIndexOptions = { - name?: string; - unique?: boolean; - expression?: string; - using?: string; - with?: string; - where?: string; - synchronize?: boolean; -}; -export const ColumnIndex = (options: string | ColumnIndexOptions = {}): PropertyDecorator => { - return (object: object, propertyName: string | symbol) => - void register({ type: 'columnIndex', item: { object, propertyName, options: asOptions(options) } }); -}; diff --git a/server/src/sql-tools/from-code/decorators/column.decorator.ts b/server/src/sql-tools/from-code/decorators/column.decorator.ts index 74a83cbcf3..7b00af80cc 100644 --- a/server/src/sql-tools/from-code/decorators/column.decorator.ts +++ b/server/src/sql-tools/from-code/decorators/column.decorator.ts @@ -15,13 +15,15 @@ export type ColumnBaseOptions = { synchronize?: boolean; storage?: ColumnStorage; identity?: boolean; + index?: boolean; + indexName?: string; + unique?: boolean; + uniqueConstraintName?: string; }; export type ColumnOptions = ColumnBaseOptions & { enum?: DatabaseEnum; array?: boolean; - unique?: boolean; - uniqueConstraintName?: string; }; export const Column = (options: string | ColumnOptions = {}): PropertyDecorator => { diff --git a/server/src/sql-tools/from-code/decorators/foreign-key-column.decorator.ts b/server/src/sql-tools/from-code/decorators/foreign-key-column.decorator.ts index 070aa5cb51..beb3aa6fd6 100644 --- a/server/src/sql-tools/from-code/decorators/foreign-key-column.decorator.ts +++ b/server/src/sql-tools/from-code/decorators/foreign-key-column.decorator.ts @@ -7,8 +7,6 @@ export type ForeignKeyColumnOptions = ColumnBaseOptions & { onUpdate?: Action; onDelete?: Action; constraintName?: string; - unique?: boolean; - uniqueConstraintName?: string; }; export const ForeignKeyColumn = (target: () => object, options: ForeignKeyColumnOptions): PropertyDecorator => { diff --git a/server/src/sql-tools/from-code/decorators/index.decorator.ts b/server/src/sql-tools/from-code/decorators/index.decorator.ts index cd76b5e36d..5d90c4f58d 100644 --- a/server/src/sql-tools/from-code/decorators/index.decorator.ts +++ b/server/src/sql-tools/from-code/decorators/index.decorator.ts @@ -1,8 +1,13 @@ -import { ColumnIndexOptions } from 'src/sql-tools/from-code/decorators/column-index.decorator'; import { register } from 'src/sql-tools/from-code/register'; import { asOptions } from 'src/sql-tools/helpers'; -export type IndexOptions = ColumnIndexOptions & { +export type IndexOptions = { + name?: string; + unique?: boolean; + expression?: string; + using?: string; + with?: string; + where?: string; columns?: string[]; synchronize?: boolean; }; diff --git a/server/src/sql-tools/from-code/index.ts b/server/src/sql-tools/from-code/index.ts index 3c74d2763c..95f1dbb22d 100644 --- a/server/src/sql-tools/from-code/index.ts +++ b/server/src/sql-tools/from-code/index.ts @@ -1,6 +1,5 @@ import 'reflect-metadata'; import { processCheckConstraints } from 'src/sql-tools/from-code/processors/check-constraint.processor'; -import { processColumnIndexes } from 'src/sql-tools/from-code/processors/column-index.processor'; import { processColumns } from 'src/sql-tools/from-code/processors/column.processor'; import { processConfigurationParameters } from 'src/sql-tools/from-code/processors/configuration-parameter.processor'; import { processDatabases } from 'src/sql-tools/from-code/processors/database.processor'; @@ -36,14 +35,21 @@ const processors: Processor[] = [ processUniqueConstraints, processCheckConstraints, processPrimaryKeyConstraints, - processIndexes, - processColumnIndexes, processForeignKeyConstraints, + processIndexes, processTriggers, ]; -export const schemaFromCode = () => { +export type SchemaFromCodeOptions = { + /** automatically create indexes on foreign key columns */ + createForeignKeyIndexes?: boolean; +}; +export const schemaFromCode = (options: SchemaFromCodeOptions = {}) => { if (!initialized) { + const globalOptions = { + createForeignKeyIndexes: options.createForeignKeyIndexes ?? true, + }; + const builder: SchemaBuilder = { name: 'postgres', schemaName: 'public', @@ -58,7 +64,7 @@ export const schemaFromCode = () => { const items = getRegisteredItems(); for (const processor of processors) { - processor(builder, items); + processor(builder, items, globalOptions); } schema = { ...builder, tables: builder.tables.map(({ metadata: _, ...table }) => table) }; diff --git a/server/src/sql-tools/from-code/processors/check-constraint.processor.ts b/server/src/sql-tools/from-code/processors/check-constraint.processor.ts index d61ee18277..feb21b9894 100644 --- a/server/src/sql-tools/from-code/processors/check-constraint.processor.ts +++ b/server/src/sql-tools/from-code/processors/check-constraint.processor.ts @@ -1,6 +1,6 @@ import { onMissingTable, resolveTable } from 'src/sql-tools/from-code/processors/table.processor'; import { Processor } from 'src/sql-tools/from-code/processors/type'; -import { asCheckConstraintName } from 'src/sql-tools/helpers'; +import { asKey } from 'src/sql-tools/helpers'; import { DatabaseConstraintType } from 'src/sql-tools/types'; export const processCheckConstraints: Processor = (builder, items) => { @@ -24,3 +24,5 @@ export const processCheckConstraints: Processor = (builder, items) => { }); } }; + +const asCheckConstraintName = (table: string, expression: string) => asKey('CHK_', table, [expression]); diff --git a/server/src/sql-tools/from-code/processors/column-index.processor.ts b/server/src/sql-tools/from-code/processors/column-index.processor.ts deleted file mode 100644 index 0e40fa1ee3..0000000000 --- a/server/src/sql-tools/from-code/processors/column-index.processor.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { onMissingColumn, resolveColumn } from 'src/sql-tools/from-code/processors/column.processor'; -import { onMissingTable } from 'src/sql-tools/from-code/processors/table.processor'; -import { Processor } from 'src/sql-tools/from-code/processors/type'; -import { asIndexName } from 'src/sql-tools/helpers'; - -export const processColumnIndexes: Processor = (builder, items) => { - for (const { - item: { object, propertyName, options }, - } of items.filter((item) => item.type === 'columnIndex')) { - const { table, column } = resolveColumn(builder, object, propertyName); - if (!table) { - onMissingTable(builder, '@ColumnIndex', object); - continue; - } - - if (!column) { - onMissingColumn(builder, `@ColumnIndex`, object, propertyName); - continue; - } - - table.indexes.push({ - name: options.name || asIndexName(table.name, [column.name], options.where), - tableName: table.name, - unique: options.unique ?? false, - expression: options.expression, - using: options.using, - where: options.where, - columnNames: [column.name], - synchronize: options.synchronize ?? true, - }); - } -}; diff --git a/server/src/sql-tools/from-code/processors/column.processor.ts b/server/src/sql-tools/from-code/processors/column.processor.ts index 37f3f5d082..e8c2544f87 100644 --- a/server/src/sql-tools/from-code/processors/column.processor.ts +++ b/server/src/sql-tools/from-code/processors/column.processor.ts @@ -1,8 +1,8 @@ import { ColumnOptions } from 'src/sql-tools/from-code/decorators/column.decorator'; import { onMissingTable, resolveTable } from 'src/sql-tools/from-code/processors/table.processor'; import { Processor, SchemaBuilder } from 'src/sql-tools/from-code/processors/type'; -import { asMetadataKey, asUniqueConstraintName, fromColumnValue } from 'src/sql-tools/helpers'; -import { DatabaseColumn, DatabaseConstraintType } from 'src/sql-tools/types'; +import { asMetadataKey, fromColumnValue } from 'src/sql-tools/helpers'; +import { DatabaseColumn } from 'src/sql-tools/types'; export const processColumns: Processor = (builder, items) => { for (const { @@ -54,16 +54,6 @@ export const processColumns: Processor = (builder, items) => { writeMetadata(object, propertyName, { name: column.name, options }); table.columns.push(column); - - if (type === 'column' && !options.primary && options.unique) { - table.constraints.push({ - type: DatabaseConstraintType.UNIQUE, - name: options.uniqueConstraintName || asUniqueConstraintName(table.name, [column.name]), - tableName: table.name, - columnNames: [column.name], - synchronize: options.synchronize ?? true, - }); - } } }; diff --git a/server/src/sql-tools/from-code/processors/foreign-key-constriant.processor.ts b/server/src/sql-tools/from-code/processors/foreign-key-constriant.processor.ts index 784a8b8e99..612b74c30f 100644 --- a/server/src/sql-tools/from-code/processors/foreign-key-constriant.processor.ts +++ b/server/src/sql-tools/from-code/processors/foreign-key-constriant.processor.ts @@ -1,7 +1,7 @@ import { onMissingColumn, resolveColumn } from 'src/sql-tools/from-code/processors/column.processor'; import { onMissingTable, resolveTable } from 'src/sql-tools/from-code/processors/table.processor'; import { Processor } from 'src/sql-tools/from-code/processors/type'; -import { asForeignKeyConstraintName, asRelationKeyConstraintName } from 'src/sql-tools/helpers'; +import { asKey } from 'src/sql-tools/helpers'; import { DatabaseActionType, DatabaseConstraintType } from 'src/sql-tools/types'; export const processForeignKeyConstraints: Processor = (builder, items) => { @@ -46,7 +46,7 @@ export const processForeignKeyConstraints: Processor = (builder, items) => { synchronize: options.synchronize ?? true, }); - if (options.unique) { + if (options.unique || options.uniqueConstraintName) { table.constraints.push({ name: options.uniqueConstraintName || asRelationKeyConstraintName(table.name, columnNames), tableName: table.name, @@ -57,3 +57,6 @@ export const processForeignKeyConstraints: Processor = (builder, items) => { } } }; + +const asForeignKeyConstraintName = (table: string, columns: string[]) => asKey('FK_', table, columns); +const asRelationKeyConstraintName = (table: string, columns: string[]) => asKey('REL_', table, columns); diff --git a/server/src/sql-tools/from-code/processors/index.processor.ts b/server/src/sql-tools/from-code/processors/index.processor.ts index 3625bf9784..f4c9c7cec1 100644 --- a/server/src/sql-tools/from-code/processors/index.processor.ts +++ b/server/src/sql-tools/from-code/processors/index.processor.ts @@ -1,8 +1,9 @@ +import { onMissingColumn, resolveColumn } from 'src/sql-tools/from-code/processors/column.processor'; import { onMissingTable, resolveTable } from 'src/sql-tools/from-code/processors/table.processor'; import { Processor } from 'src/sql-tools/from-code/processors/type'; -import { asIndexName } from 'src/sql-tools/helpers'; +import { asKey } from 'src/sql-tools/helpers'; -export const processIndexes: Processor = (builder, items) => { +export const processIndexes: Processor = (builder, items, config) => { for (const { item: { object, options }, } of items.filter((item) => item.type === 'index')) { @@ -24,4 +25,66 @@ export const processIndexes: Processor = (builder, items) => { synchronize: options.synchronize ?? true, }); } + + // column indexes + for (const { + type, + item: { object, propertyName, options }, + } of items.filter((item) => item.type === 'column' || item.type === 'foreignKeyColumn')) { + const { table, column } = resolveColumn(builder, object, propertyName); + if (!table) { + onMissingTable(builder, '@Column', object); + continue; + } + + if (!column) { + // should be impossible since they are created in `column.processor.ts` + onMissingColumn(builder, '@Column', object, propertyName); + continue; + } + + if (options.index === false) { + continue; + } + + const isIndexRequested = + options.indexName || options.index || (type === 'foreignKeyColumn' && config.createForeignKeyIndexes); + if (!isIndexRequested) { + continue; + } + + const indexName = options.indexName || asIndexName(table.name, [column.name]); + + const isIndexPresent = table.indexes.some((index) => index.name === indexName); + if (isIndexPresent) { + continue; + } + + const isOnlyPrimaryColumn = options.primary && table.columns.filter(({ primary }) => primary === true).length === 1; + if (isOnlyPrimaryColumn) { + // will have an index created by the primary key constraint + continue; + } + + table.indexes.push({ + name: indexName, + tableName: table.name, + unique: false, + columnNames: [column.name], + synchronize: options.synchronize ?? true, + }); + } +}; + +const asIndexName = (table: string, columns?: string[], where?: string) => { + const items: string[] = []; + for (const columnName of columns ?? []) { + items.push(columnName); + } + + if (where) { + items.push(where); + } + + return asKey('IDX_', table, items); }; diff --git a/server/src/sql-tools/from-code/processors/primary-key-contraint.processor.ts b/server/src/sql-tools/from-code/processors/primary-key-contraint.processor.ts index f123f2e495..74aecc5ea0 100644 --- a/server/src/sql-tools/from-code/processors/primary-key-contraint.processor.ts +++ b/server/src/sql-tools/from-code/processors/primary-key-contraint.processor.ts @@ -1,5 +1,5 @@ import { Processor } from 'src/sql-tools/from-code/processors/type'; -import { asPrimaryKeyConstraintName } from 'src/sql-tools/helpers'; +import { asKey } from 'src/sql-tools/helpers'; import { DatabaseConstraintType } from 'src/sql-tools/types'; export const processPrimaryKeyConstraints: Processor = (builder) => { @@ -22,3 +22,5 @@ export const processPrimaryKeyConstraints: Processor = (builder) => { } } }; + +const asPrimaryKeyConstraintName = (table: string, columns: string[]) => asKey('PK_', table, columns); diff --git a/server/src/sql-tools/from-code/processors/trigger.processor.ts b/server/src/sql-tools/from-code/processors/trigger.processor.ts index 2f4cc04326..4b875f353b 100644 --- a/server/src/sql-tools/from-code/processors/trigger.processor.ts +++ b/server/src/sql-tools/from-code/processors/trigger.processor.ts @@ -1,6 +1,7 @@ +import { TriggerOptions } from 'src/sql-tools/from-code/decorators/trigger.decorator'; import { onMissingTable, resolveTable } from 'src/sql-tools/from-code/processors/table.processor'; import { Processor } from 'src/sql-tools/from-code/processors/type'; -import { asTriggerName } from 'src/sql-tools/helpers'; +import { asKey } from 'src/sql-tools/helpers'; export const processTriggers: Processor = (builder, items) => { for (const { @@ -26,3 +27,6 @@ export const processTriggers: Processor = (builder, items) => { }); } }; + +const asTriggerName = (table: string, trigger: TriggerOptions) => + asKey('TR_', table, [...trigger.actions, trigger.scope, trigger.timing, trigger.functionName]); diff --git a/server/src/sql-tools/from-code/processors/type.ts b/server/src/sql-tools/from-code/processors/type.ts index 5a69efbcf0..deb142d278 100644 --- a/server/src/sql-tools/from-code/processors/type.ts +++ b/server/src/sql-tools/from-code/processors/type.ts @@ -1,3 +1,4 @@ +import { SchemaFromCodeOptions } from 'src/sql-tools/from-code'; import { TableOptions } from 'src/sql-tools/from-code/decorators/table.decorator'; import { RegisterItem } from 'src/sql-tools/from-code/register-item'; import { DatabaseSchema, DatabaseTable } from 'src/sql-tools/types'; @@ -6,4 +7,4 @@ import { DatabaseSchema, DatabaseTable } from 'src/sql-tools/types'; export type TableWithMetadata = DatabaseTable & { metadata: { options: TableOptions; object: Function } }; export type SchemaBuilder = Omit & { tables: TableWithMetadata[] }; -export type Processor = (builder: SchemaBuilder, items: RegisterItem[]) => void; +export type Processor = (builder: SchemaBuilder, items: RegisterItem[], options: SchemaFromCodeOptions) => void; diff --git a/server/src/sql-tools/from-code/processors/unique-constraint.processor.ts b/server/src/sql-tools/from-code/processors/unique-constraint.processor.ts index 74c0504f7e..9014378085 100644 --- a/server/src/sql-tools/from-code/processors/unique-constraint.processor.ts +++ b/server/src/sql-tools/from-code/processors/unique-constraint.processor.ts @@ -1,6 +1,7 @@ +import { onMissingColumn, resolveColumn } from 'src/sql-tools/from-code/processors/column.processor'; import { onMissingTable, resolveTable } from 'src/sql-tools/from-code/processors/table.processor'; import { Processor } from 'src/sql-tools/from-code/processors/type'; -import { asUniqueConstraintName } from 'src/sql-tools/helpers'; +import { asKey } from 'src/sql-tools/helpers'; import { DatabaseConstraintType } from 'src/sql-tools/types'; export const processUniqueConstraints: Processor = (builder, items) => { @@ -24,4 +25,34 @@ export const processUniqueConstraints: Processor = (builder, items) => { synchronize: options.synchronize ?? true, }); } + + // column level constraints + for (const { + type, + item: { object, propertyName, options }, + } of items.filter((item) => item.type === 'column' || item.type === 'foreignKeyColumn')) { + const { table, column } = resolveColumn(builder, object, propertyName); + if (!table) { + onMissingTable(builder, '@Column', object); + continue; + } + + if (!column) { + // should be impossible since they are created in `column.processor.ts` + onMissingColumn(builder, '@Column', object, propertyName); + continue; + } + + if (type === 'column' && !options.primary && (options.unique || options.uniqueConstraintName)) { + table.constraints.push({ + type: DatabaseConstraintType.UNIQUE, + name: options.uniqueConstraintName || asUniqueConstraintName(table.name, [column.name]), + tableName: table.name, + columnNames: [column.name], + synchronize: options.synchronize ?? true, + }); + } + } }; + +const asUniqueConstraintName = (table: string, columns: string[]) => asKey('UQ_', table, columns); diff --git a/server/src/sql-tools/from-code/register-function.ts b/server/src/sql-tools/from-code/register-function.ts index 69e1a0f8f3..be71e0dfd7 100644 --- a/server/src/sql-tools/from-code/register-function.ts +++ b/server/src/sql-tools/from-code/register-function.ts @@ -1,5 +1,4 @@ import { register } from 'src/sql-tools/from-code/register'; -import { asFunctionExpression } from 'src/sql-tools/helpers'; import { ColumnType, DatabaseFunction } from 'src/sql-tools/types'; export type FunctionOptions = { @@ -27,3 +26,37 @@ export const registerFunction = (options: FunctionOptions) => { return item; }; + +const asFunctionExpression = (options: FunctionOptions) => { + const name = options.name; + const sql: string[] = [ + `CREATE OR REPLACE FUNCTION ${name}(${(options.arguments || []).join(', ')})`, + `RETURNS ${options.returnType}`, + ]; + + const flags = [ + options.parallel ? `PARALLEL ${options.parallel.toUpperCase()}` : undefined, + options.strict ? 'STRICT' : undefined, + options.behavior ? options.behavior.toUpperCase() : undefined, + `LANGUAGE ${options.language ?? 'SQL'}`, + ].filter((x) => x !== undefined); + + if (flags.length > 0) { + sql.push(flags.join(' ')); + } + + if ('return' in options) { + sql.push(` RETURN ${options.return}`); + } + + if ('body' in options) { + sql.push( + // + `AS $$`, + ' ' + options.body.trim(), + `$$;`, + ); + } + + return sql.join('\n ').trim(); +}; diff --git a/server/src/sql-tools/from-code/register-item.ts b/server/src/sql-tools/from-code/register-item.ts index 08200cbc4f..4889ae34b9 100644 --- a/server/src/sql-tools/from-code/register-item.ts +++ b/server/src/sql-tools/from-code/register-item.ts @@ -1,5 +1,4 @@ import { CheckOptions } from 'src/sql-tools/from-code/decorators/check.decorator'; -import { ColumnIndexOptions } from 'src/sql-tools/from-code/decorators/column-index.decorator'; import { ColumnOptions } from 'src/sql-tools/from-code/decorators/column.decorator'; import { ConfigurationParameterOptions } from 'src/sql-tools/from-code/decorators/configuration-parameter.decorator'; import { DatabaseOptions } from 'src/sql-tools/from-code/decorators/database.decorator'; @@ -21,7 +20,6 @@ export type RegisterItem = | { type: 'uniqueConstraint'; item: ClassBased<{ options: UniqueOptions }> } | { type: 'checkConstraint'; item: ClassBased<{ options: CheckOptions }> } | { type: 'column'; item: PropertyBased<{ options: ColumnOptions }> } - | { type: 'columnIndex'; item: PropertyBased<{ options: ColumnIndexOptions }> } | { type: 'function'; item: DatabaseFunction } | { type: 'enum'; item: DatabaseEnum } | { type: 'trigger'; item: ClassBased<{ options: TriggerOptions }> } diff --git a/server/src/sql-tools/helpers.ts b/server/src/sql-tools/helpers.ts index 364b695194..2802407ea6 100644 --- a/server/src/sql-tools/helpers.ts +++ b/server/src/sql-tools/helpers.ts @@ -1,7 +1,5 @@ import { createHash } from 'node:crypto'; import { ColumnValue } from 'src/sql-tools/from-code/decorators/column.decorator'; -import { TriggerOptions } from 'src/sql-tools/from-code/decorators/trigger.decorator'; -import { FunctionOptions } from 'src/sql-tools/from-code/register-function'; import { Comparer, DatabaseColumn, @@ -18,25 +16,6 @@ export const asSnakeCase = (name: string): string => name.replaceAll(/([a-z])([A // match TypeORM export const asKey = (prefix: string, tableName: string, values: string[]) => (prefix + sha1(`${tableName}_${values.toSorted().join('_')}`)).slice(0, 30); -export const asPrimaryKeyConstraintName = (table: string, columns: string[]) => asKey('PK_', table, columns); -export const asForeignKeyConstraintName = (table: string, columns: string[]) => asKey('FK_', table, columns); -export const asTriggerName = (table: string, trigger: TriggerOptions) => - asKey('TR_', table, [...trigger.actions, trigger.scope, trigger.timing, trigger.functionName]); -export const asRelationKeyConstraintName = (table: string, columns: string[]) => asKey('REL_', table, columns); -export const asUniqueConstraintName = (table: string, columns: string[]) => asKey('UQ_', table, columns); -export const asCheckConstraintName = (table: string, expression: string) => asKey('CHK_', table, [expression]); -export const asIndexName = (table: string, columns: string[] | undefined, where: string | undefined) => { - const items: string[] = []; - for (const columnName of columns ?? []) { - items.push(columnName); - } - - if (where) { - items.push(where); - } - - return asKey('IDX_', table, items); -}; export const asOptions = (options: string | T): T => { if (typeof options === 'string') { @@ -46,40 +25,6 @@ export const asOptions = (options: string | T): T = return options; }; -export const asFunctionExpression = (options: FunctionOptions) => { - const name = options.name; - const sql: string[] = [ - `CREATE OR REPLACE FUNCTION ${name}(${(options.arguments || []).join(', ')})`, - `RETURNS ${options.returnType}`, - ]; - - const flags = [ - options.parallel ? `PARALLEL ${options.parallel.toUpperCase()}` : undefined, - options.strict ? 'STRICT' : undefined, - options.behavior ? options.behavior.toUpperCase() : undefined, - `LANGUAGE ${options.language ?? 'SQL'}`, - ].filter((x) => x !== undefined); - - if (flags.length > 0) { - sql.push(flags.join(' ')); - } - - if ('return' in options) { - sql.push(` RETURN ${options.return}`); - } - - if ('body' in options) { - sql.push( - // - `AS $$`, - ' ' + options.body.trim(), - `$$;`, - ); - } - - return sql.join('\n ').trim(); -}; - export const sha1 = (value: string) => createHash('sha1').update(value).digest('hex'); export const hasMask = (input: number, mask: number) => (input & mask) === mask; diff --git a/server/src/sql-tools/public_api.ts b/server/src/sql-tools/public_api.ts index d916678d4a..b41cce4ab5 100644 --- a/server/src/sql-tools/public_api.ts +++ b/server/src/sql-tools/public_api.ts @@ -3,7 +3,6 @@ export { schemaFromCode } from 'src/sql-tools/from-code'; export * from 'src/sql-tools/from-code/decorators/after-delete.decorator'; export * from 'src/sql-tools/from-code/decorators/before-update.decorator'; export * from 'src/sql-tools/from-code/decorators/check.decorator'; -export * from 'src/sql-tools/from-code/decorators/column-index.decorator'; export * from 'src/sql-tools/from-code/decorators/column.decorator'; export * from 'src/sql-tools/from-code/decorators/configuration-parameter.decorator'; export * from 'src/sql-tools/from-code/decorators/create-date-column.decorator'; diff --git a/server/test/sql-tools/column-index-name-default.ts b/server/test/sql-tools/column-index-name-default.ts index e8b36ec119..cedae006be 100644 --- a/server/test/sql-tools/column-index-name-default.ts +++ b/server/test/sql-tools/column-index-name-default.ts @@ -1,9 +1,8 @@ -import { Column, ColumnIndex, DatabaseSchema, Table } from 'src/sql-tools'; +import { Column, DatabaseSchema, Table } from 'src/sql-tools'; @Table() export class Table1 { - @ColumnIndex() - @Column() + @Column({ index: true }) column1!: string; } diff --git a/server/test/sql-tools/column-index-name.ts b/server/test/sql-tools/column-index-name.ts new file mode 100644 index 0000000000..8ba18a8851 --- /dev/null +++ b/server/test/sql-tools/column-index-name.ts @@ -0,0 +1,46 @@ +import { Column, DatabaseSchema, Table } from 'src/sql-tools'; + +@Table() +export class Table1 { + @Column({ indexName: 'IDX_test' }) + column1!: string; +} + +export const description = 'should create a column with an index if a name is provided'; +export const schema: DatabaseSchema = { + name: 'postgres', + schemaName: 'public', + functions: [], + enums: [], + extensions: [], + parameters: [], + tables: [ + { + name: 'table1', + columns: [ + { + name: 'column1', + tableName: 'table1', + type: 'character varying', + nullable: false, + isArray: false, + primary: false, + synchronize: true, + }, + ], + indexes: [ + { + name: 'IDX_test', + columnNames: ['column1'], + tableName: 'table1', + unique: false, + synchronize: true, + }, + ], + triggers: [], + constraints: [], + synchronize: true, + }, + ], + warnings: [], +}; diff --git a/server/test/sql-tools/foreign-key-inferred-type.stub.ts b/server/test/sql-tools/foreign-key-inferred-type.stub.ts index 2ecaafdcad..0b66a1acd4 100644 --- a/server/test/sql-tools/foreign-key-inferred-type.stub.ts +++ b/server/test/sql-tools/foreign-key-inferred-type.stub.ts @@ -60,7 +60,15 @@ export const schema: DatabaseSchema = { synchronize: true, }, ], - indexes: [], + indexes: [ + { + name: 'IDX_3fcca5cc563abf256fc346e3ff', + tableName: 'table2', + columnNames: ['parentId'], + unique: false, + synchronize: true, + }, + ], triggers: [], constraints: [ { diff --git a/server/test/sql-tools/foreign-key-with-unique-constraint.stub.ts b/server/test/sql-tools/foreign-key-with-unique-constraint.stub.ts index 0601a02d42..109a3dfc85 100644 --- a/server/test/sql-tools/foreign-key-with-unique-constraint.stub.ts +++ b/server/test/sql-tools/foreign-key-with-unique-constraint.stub.ts @@ -60,7 +60,15 @@ export const schema: DatabaseSchema = { synchronize: true, }, ], - indexes: [], + indexes: [ + { + name: 'IDX_3fcca5cc563abf256fc346e3ff', + tableName: 'table2', + columnNames: ['parentId'], + unique: false, + synchronize: true, + }, + ], triggers: [], constraints: [ { From 6474a78b8be31cf6a3c2fabf2f3177ffd40624e1 Mon Sep 17 00:00:00 2001 From: Jason Rasmussen Date: Thu, 17 Apr 2025 17:38:47 -0400 Subject: [PATCH 05/29] feat: initial kysely migration file (#17678) --- .github/workflows/test.yml | 2 +- docs/docs/developer/database-migrations.md | 6 +- server/package.json | 5 +- server/src/bin/database.ts | 11 - server/src/bin/migrations.ts | 37 +- .../1744910873956-AddMissingIndex.ts | 13 + .../src/repositories/database.repository.ts | 75 ++-- .../1744910873969-InitialMigration.ts | 391 ++++++++++++++++++ .../src/schema/tables/geodata-places.table.ts | 5 +- .../from-code/processors/table.processor.ts | 7 + .../sql-tools/from-code/register-function.ts | 8 +- server/test/medium/globalSetup.ts | 78 +--- ....stub copy.ts => index-with-expression.ts} | 0 13 files changed, 499 insertions(+), 139 deletions(-) delete mode 100644 server/src/bin/database.ts create mode 100644 server/src/migrations/1744910873956-AddMissingIndex.ts create mode 100644 server/src/schema/migrations/1744910873969-InitialMigration.ts rename server/test/sql-tools/{index-with-where.stub copy.ts => index-with-expression.ts} (100%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5273698e4e..f5cb6c8d30 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -521,7 +521,7 @@ jobs: run: npm run migrations:run - name: Test npm run schema:reset command works - run: npm run typeorm:schema:reset + run: npm run schema:reset - name: Generate new migrations continue-on-error: true diff --git a/docs/docs/developer/database-migrations.md b/docs/docs/developer/database-migrations.md index 2cddf5f386..c0c61340cb 100644 --- a/docs/docs/developer/database-migrations.md +++ b/docs/docs/developer/database-migrations.md @@ -1,14 +1,14 @@ # Database Migrations -After making any changes in the `server/src/entities`, a database migration need to run in order to register the changes in the database. Follow the steps below to create a new migration. +After making any changes in the `server/src/schema`, a database migration need to run in order to register the changes in the database. Follow the steps below to create a new migration. 1. Run the command ```bash -npm run typeorm:migrations:generate +npm run migrations:generate ``` 2. Check if the migration file makes sense. -3. Move the migration file to folder `./server/src/migrations` in your code editor. +3. Move the migration file to folder `./server/src/schema/migrations` in your code editor. The server will automatically detect `*.ts` file changes and restart. Part of the server start-up process includes running any new migrations, so it will be applied immediately. diff --git a/server/package.json b/server/package.json index 76415da7c8..8e149d961e 100644 --- a/server/package.json +++ b/server/package.json @@ -26,9 +26,8 @@ "migrations:generate": "node ./dist/bin/migrations.js generate", "migrations:create": "node ./dist/bin/migrations.js create", "migrations:run": "node ./dist/bin/migrations.js run", - "typeorm:migrations:revert": "typeorm migration:revert -d ./dist/bin/database.js", - "typeorm:schema:drop": "typeorm query -d ./dist/bin/database.js 'DROP schema public cascade; CREATE schema public;'", - "typeorm:schema:reset": "npm run typeorm:schema:drop && npm run migrations:run", + "schema:drop": "node ./dist/bin/migrations.js query 'DROP schema public cascade; CREATE schema public;'", + "schema:reset": "npm run schema:drop && npm run migrations:run", "kysely:codegen": "npx kysely-codegen --include-pattern=\"(public|vectors).*\" --dialect postgres --url postgres://postgres:postgres@localhost/immich --log-level debug --out-file=./src/db.d.ts", "sync:open-api": "node ./dist/bin/sync-open-api.js", "sync:sql": "node ./dist/bin/sync-sql.js", diff --git a/server/src/bin/database.ts b/server/src/bin/database.ts deleted file mode 100644 index 7ea56e0fc0..0000000000 --- a/server/src/bin/database.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { ConfigRepository } from 'src/repositories/config.repository'; -import { DataSource } from 'typeorm'; - -const { database } = new ConfigRepository().getEnv(); - -/** - * @deprecated - DO NOT USE THIS - * - * this export is ONLY to be used for TypeORM commands in package.json#scripts - */ -export const dataSource = new DataSource({ ...database.config.typeorm, host: 'localhost' }); diff --git a/server/src/bin/migrations.ts b/server/src/bin/migrations.ts index fb50323dff..2ddc6776fb 100644 --- a/server/src/bin/migrations.ts +++ b/server/src/bin/migrations.ts @@ -1,8 +1,8 @@ #!/usr/bin/env node process.env.DB_URL = process.env.DB_URL || 'postgres://postgres:postgres@localhost:5432/immich'; -import { Kysely } from 'kysely'; -import { writeFileSync } from 'node:fs'; +import { Kysely, sql } from 'kysely'; +import { mkdirSync, writeFileSync } from 'node:fs'; import { basename, dirname, extname, join } from 'node:path'; import postgres from 'postgres'; import { ConfigRepository } from 'src/repositories/config.repository'; @@ -23,8 +23,13 @@ const main = async () => { } case 'run': { - const only = process.argv[3] as 'kysely' | 'typeorm' | undefined; - await run(only); + await runMigrations(); + return; + } + + case 'query': { + const query = process.argv[3]; + await runQuery(query); return; } @@ -48,14 +53,25 @@ const main = async () => { } }; -const run = async (only?: 'kysely' | 'typeorm') => { +const getDatabaseClient = () => { const configRepository = new ConfigRepository(); const { database } = configRepository.getEnv(); - const logger = new LoggingRepository(undefined, configRepository); - const db = new Kysely(getKyselyConfig(database.config.kysely)); - const databaseRepository = new DatabaseRepository(db, logger, configRepository); + return new Kysely(getKyselyConfig(database.config.kysely)); +}; - await databaseRepository.runMigrations({ only }); +const runQuery = async (query: string) => { + const db = getDatabaseClient(); + await sql.raw(query).execute(db); + await db.destroy(); +}; + +const runMigrations = async () => { + const configRepository = new ConfigRepository(); + const logger = new LoggingRepository(undefined, configRepository); + const db = getDatabaseClient(); + const databaseRepository = new DatabaseRepository(db, logger, configRepository); + await databaseRepository.runMigrations(); + await db.destroy(); }; const debug = async () => { @@ -81,7 +97,8 @@ const create = (path: string, up: string[], down: string[]) => { const filename = `${timestamp}-${name}.ts`; const folder = dirname(path); const fullPath = join(folder, filename); - writeFileSync(fullPath, asMigration('typeorm', { name, timestamp, up, down })); + mkdirSync(folder, { recursive: true }); + writeFileSync(fullPath, asMigration('kysely', { name, timestamp, up, down })); console.log(`Wrote ${fullPath}`); }; diff --git a/server/src/migrations/1744910873956-AddMissingIndex.ts b/server/src/migrations/1744910873956-AddMissingIndex.ts new file mode 100644 index 0000000000..38dd6f4958 --- /dev/null +++ b/server/src/migrations/1744910873956-AddMissingIndex.ts @@ -0,0 +1,13 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AddMissingIndex1744910873956 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE INDEX IF NOT EXISTS "IDX_geodata_gist_earthcoord" ON "geodata_places" (ll_to_earth_public(latitude, longitude))`, + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX "IDX_geodata_gist_earthcoord";`); + } +} diff --git a/server/src/repositories/database.repository.ts b/server/src/repositories/database.repository.ts index ec0b263408..c70c2cbdd4 100644 --- a/server/src/repositories/database.repository.ts +++ b/server/src/repositories/database.repository.ts @@ -2,7 +2,6 @@ import { Injectable } from '@nestjs/common'; import AsyncLock from 'async-lock'; import { FileMigrationProvider, Kysely, Migrator, sql, Transaction } from 'kysely'; import { InjectKysely } from 'nestjs-kysely'; -import { existsSync } from 'node:fs'; import { readdir } from 'node:fs/promises'; import { join } from 'node:path'; import semver from 'semver'; @@ -197,62 +196,54 @@ export class DatabaseRepository { return dimSize; } - async runMigrations(options?: { transaction?: 'all' | 'none' | 'each'; only?: 'kysely' | 'typeorm' }): Promise { + async runMigrations(options?: { transaction?: 'all' | 'none' | 'each' }): Promise { const { database } = this.configRepository.getEnv(); - if (options?.only !== 'kysely') { - const dataSource = new DataSource(database.config.typeorm); - this.logger.log('Running migrations, this may take a while'); + this.logger.log('Running migrations, this may take a while'); + const tableExists = sql<{ result: string | null }>`select to_regclass('migrations') as "result"`; + const { rows } = await tableExists.execute(this.db); + const hasTypeOrmMigrations = !!rows[0]?.result; + if (hasTypeOrmMigrations) { this.logger.debug('Running typeorm migrations'); - + const dataSource = new DataSource(database.config.typeorm); await dataSource.initialize(); await dataSource.runMigrations(options); await dataSource.destroy(); - this.logger.debug('Finished running typeorm migrations'); } - if (options?.only !== 'typeorm') { - // eslint-disable-next-line unicorn/prefer-module - const migrationFolder = join(__dirname, '..', 'schema/migrations'); + this.logger.debug('Running kysely migrations'); + const migrator = new Migrator({ + db: this.db, + migrationLockTableName: 'kysely_migrations_lock', + migrationTableName: 'kysely_migrations', + provider: new FileMigrationProvider({ + fs: { readdir }, + path: { join }, + // eslint-disable-next-line unicorn/prefer-module + migrationFolder: join(__dirname, '..', 'schema/migrations'), + }), + }); - // TODO remove after we have at least one kysely migration - if (!existsSync(migrationFolder)) { - return; + const { error, results } = await migrator.migrateToLatest(); + + for (const result of results ?? []) { + if (result.status === 'Success') { + this.logger.log(`Migration "${result.migrationName}" succeeded`); } - this.logger.debug('Running kysely migrations'); - const migrator = new Migrator({ - db: this.db, - migrationLockTableName: 'kysely_migrations_lock', - migrationTableName: 'kysely_migrations', - provider: new FileMigrationProvider({ - fs: { readdir }, - path: { join }, - migrationFolder, - }), - }); - - const { error, results } = await migrator.migrateToLatest(); - - for (const result of results ?? []) { - if (result.status === 'Success') { - this.logger.log(`Migration "${result.migrationName}" succeeded`); - } - - if (result.status === 'Error') { - this.logger.warn(`Migration "${result.migrationName}" failed`); - } + if (result.status === 'Error') { + this.logger.warn(`Migration "${result.migrationName}" failed`); } - - if (error) { - this.logger.error(`Kysely migrations failed: ${error}`); - throw error; - } - - this.logger.debug('Finished running kysely migrations'); } + + if (error) { + this.logger.error(`Kysely migrations failed: ${error}`); + throw error; + } + + this.logger.debug('Finished running kysely migrations'); } async withLock(lock: DatabaseLock, callback: () => Promise): Promise { diff --git a/server/src/schema/migrations/1744910873969-InitialMigration.ts b/server/src/schema/migrations/1744910873969-InitialMigration.ts new file mode 100644 index 0000000000..e157607681 --- /dev/null +++ b/server/src/schema/migrations/1744910873969-InitialMigration.ts @@ -0,0 +1,391 @@ +import { Kysely, sql } from 'kysely'; +import { DatabaseExtension } from 'src/enum'; +import { ConfigRepository } from 'src/repositories/config.repository'; + +const vectorExtension = new ConfigRepository().getEnv().database.vectorExtension; + +export async function up(db: Kysely): Promise { + await sql`CREATE EXTENSION IF NOT EXISTS "uuid-ossp";`.execute(db); + await sql`CREATE EXTENSION IF NOT EXISTS "unaccent";`.execute(db); + await sql`CREATE EXTENSION IF NOT EXISTS "cube";`.execute(db); + await sql`CREATE EXTENSION IF NOT EXISTS "earthdistance";`.execute(db); + await sql`CREATE EXTENSION IF NOT EXISTS "pg_trgm";`.execute(db); + await sql`CREATE EXTENSION IF NOT EXISTS "vectors";`.execute(db); + await sql`CREATE OR REPLACE FUNCTION immich_uuid_v7(p_timestamp timestamp with time zone default clock_timestamp()) + RETURNS uuid + VOLATILE LANGUAGE SQL + AS $$ + select encode( + set_bit( + set_bit( + overlay(uuid_send(gen_random_uuid()) + placing substring(int8send(floor(extract(epoch from p_timestamp) * 1000)::bigint) from 3) + from 1 for 6 + ), + 52, 1 + ), + 53, 1 + ), + 'hex')::uuid; + $$;`.execute(db); + await sql`CREATE OR REPLACE FUNCTION updated_at() + RETURNS TRIGGER + LANGUAGE PLPGSQL + AS $$ + DECLARE + clock_timestamp TIMESTAMP := clock_timestamp(); + BEGIN + new."updatedAt" = clock_timestamp; + new."updateId" = immich_uuid_v7(clock_timestamp); + return new; + END; + $$;`.execute(db); + await sql`CREATE OR REPLACE FUNCTION f_concat_ws(text, text[]) + RETURNS text + PARALLEL SAFE IMMUTABLE LANGUAGE SQL + AS $$SELECT array_to_string($2, $1)$$;`.execute(db); + await sql`CREATE OR REPLACE FUNCTION f_unaccent(text) + RETURNS text + PARALLEL SAFE STRICT IMMUTABLE LANGUAGE SQL + RETURN unaccent('unaccent', $1)`.execute(db); + await sql`CREATE OR REPLACE FUNCTION ll_to_earth_public(latitude double precision, longitude double precision) + RETURNS public.earth + PARALLEL SAFE STRICT IMMUTABLE LANGUAGE SQL + AS $$ + SELECT public.cube(public.cube(public.cube(public.earth()*cos(radians(latitude))*cos(radians(longitude))),public.earth()*cos(radians(latitude))*sin(radians(longitude))),public.earth()*sin(radians(latitude)))::public.earth + $$;`.execute(db); + await sql`CREATE OR REPLACE FUNCTION users_delete_audit() + RETURNS TRIGGER + LANGUAGE PLPGSQL + AS $$ + BEGIN + INSERT INTO users_audit ("userId") + SELECT "id" + FROM OLD; + RETURN NULL; + END; + $$;`.execute(db); + await sql`CREATE OR REPLACE FUNCTION partners_delete_audit() + RETURNS TRIGGER + LANGUAGE PLPGSQL + AS $$ + BEGIN + INSERT INTO partners_audit ("sharedById", "sharedWithId") + SELECT "sharedById", "sharedWithId" + FROM OLD; + RETURN NULL; + END; + $$;`.execute(db); + await sql`CREATE OR REPLACE FUNCTION assets_delete_audit() + RETURNS TRIGGER + LANGUAGE PLPGSQL + AS $$ + BEGIN + INSERT INTO assets_audit ("assetId", "ownerId") + SELECT "id", "ownerId" + FROM OLD; + RETURN NULL; + END; + $$;`.execute(db); + if (vectorExtension === DatabaseExtension.VECTORS) { + await sql`SET search_path TO "$user", public, vectors`.execute(db); + await sql`SET vectors.pgvector_compatibility=on`.execute(db); + } + await sql`CREATE TYPE "assets_status_enum" AS ENUM ('active','trashed','deleted');`.execute(db); + await sql`CREATE TYPE "sourcetype" AS ENUM ('machine-learning','exif','manual');`.execute(db); + await sql`CREATE TABLE "users" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "email" character varying NOT NULL, "password" character varying NOT NULL DEFAULT '', "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "profileImagePath" character varying NOT NULL DEFAULT '', "isAdmin" boolean NOT NULL DEFAULT false, "shouldChangePassword" boolean NOT NULL DEFAULT true, "deletedAt" timestamp with time zone, "oauthId" character varying NOT NULL DEFAULT '', "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "storageLabel" character varying, "name" character varying NOT NULL DEFAULT '', "quotaSizeInBytes" bigint, "quotaUsageInBytes" bigint NOT NULL DEFAULT 0, "status" character varying NOT NULL DEFAULT 'active', "profileChangedAt" timestamp with time zone NOT NULL DEFAULT now(), "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "libraries" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "name" character varying NOT NULL, "ownerId" uuid NOT NULL, "importPaths" text[] NOT NULL, "exclusionPatterns" text[] NOT NULL, "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "deletedAt" timestamp with time zone, "refreshedAt" timestamp with time zone, "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "asset_stack" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "primaryAssetId" uuid NOT NULL, "ownerId" uuid NOT NULL);`.execute(db); + await sql`CREATE TABLE "assets" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "deviceAssetId" character varying NOT NULL, "ownerId" uuid NOT NULL, "deviceId" character varying NOT NULL, "type" character varying NOT NULL, "originalPath" character varying NOT NULL, "fileCreatedAt" timestamp with time zone NOT NULL, "fileModifiedAt" timestamp with time zone NOT NULL, "isFavorite" boolean NOT NULL DEFAULT false, "duration" character varying, "encodedVideoPath" character varying DEFAULT '', "checksum" bytea NOT NULL, "isVisible" boolean NOT NULL DEFAULT true, "livePhotoVideoId" uuid, "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "isArchived" boolean NOT NULL DEFAULT false, "originalFileName" character varying NOT NULL, "sidecarPath" character varying, "thumbhash" bytea, "isOffline" boolean NOT NULL DEFAULT false, "libraryId" uuid, "isExternal" boolean NOT NULL DEFAULT false, "deletedAt" timestamp with time zone, "localDateTime" timestamp with time zone NOT NULL, "stackId" uuid, "duplicateId" uuid, "status" assets_status_enum NOT NULL DEFAULT 'active', "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "albums" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "ownerId" uuid NOT NULL, "albumName" character varying NOT NULL DEFAULT 'Untitled Album', "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "albumThumbnailAssetId" uuid, "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "description" text NOT NULL DEFAULT '', "deletedAt" timestamp with time zone, "isActivityEnabled" boolean NOT NULL DEFAULT true, "order" character varying NOT NULL DEFAULT 'desc', "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`COMMENT ON COLUMN "albums"."albumThumbnailAssetId" IS 'Asset ID to be used as thumbnail';`.execute(db); + await sql`CREATE TABLE "activity" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "albumId" uuid NOT NULL, "userId" uuid NOT NULL, "assetId" uuid, "comment" text, "isLiked" boolean NOT NULL DEFAULT false, "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "albums_assets_assets" ("albumsId" uuid NOT NULL, "assetsId" uuid NOT NULL, "createdAt" timestamp with time zone NOT NULL DEFAULT now());`.execute(db); + await sql`CREATE TABLE "albums_shared_users_users" ("albumsId" uuid NOT NULL, "usersId" uuid NOT NULL, "role" character varying NOT NULL DEFAULT 'editor');`.execute(db); + await sql`CREATE TABLE "api_keys" ("name" character varying NOT NULL, "key" character varying NOT NULL, "userId" uuid NOT NULL, "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "id" uuid NOT NULL DEFAULT uuid_generate_v4(), "permissions" character varying[] NOT NULL, "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "assets_audit" ("id" uuid NOT NULL DEFAULT immich_uuid_v7(), "assetId" uuid NOT NULL, "ownerId" uuid NOT NULL, "deletedAt" timestamp with time zone NOT NULL DEFAULT clock_timestamp());`.execute(db); + await sql`CREATE TABLE "person" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "ownerId" uuid NOT NULL, "name" character varying NOT NULL DEFAULT '', "thumbnailPath" character varying NOT NULL DEFAULT '', "isHidden" boolean NOT NULL DEFAULT false, "birthDate" date, "faceAssetId" uuid, "isFavorite" boolean NOT NULL DEFAULT false, "color" character varying, "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "asset_faces" ("assetId" uuid NOT NULL, "personId" uuid, "imageWidth" integer NOT NULL DEFAULT 0, "imageHeight" integer NOT NULL DEFAULT 0, "boundingBoxX1" integer NOT NULL DEFAULT 0, "boundingBoxY1" integer NOT NULL DEFAULT 0, "boundingBoxX2" integer NOT NULL DEFAULT 0, "boundingBoxY2" integer NOT NULL DEFAULT 0, "id" uuid NOT NULL DEFAULT uuid_generate_v4(), "sourceType" sourcetype NOT NULL DEFAULT 'machine-learning', "deletedAt" timestamp with time zone);`.execute(db); + await sql`CREATE TABLE "asset_files" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "assetId" uuid NOT NULL, "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "type" character varying NOT NULL, "path" character varying NOT NULL, "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "asset_job_status" ("assetId" uuid NOT NULL, "facesRecognizedAt" timestamp with time zone, "metadataExtractedAt" timestamp with time zone, "duplicatesDetectedAt" timestamp with time zone, "previewAt" timestamp with time zone, "thumbnailAt" timestamp with time zone);`.execute(db); + await sql`CREATE TABLE "audit" ("id" serial NOT NULL, "entityType" character varying NOT NULL, "entityId" uuid NOT NULL, "action" character varying NOT NULL, "ownerId" uuid NOT NULL, "createdAt" timestamp with time zone NOT NULL DEFAULT now());`.execute(db); + await sql`CREATE TABLE "exif" ("assetId" uuid NOT NULL, "make" character varying, "model" character varying, "exifImageWidth" integer, "exifImageHeight" integer, "fileSizeInByte" bigint, "orientation" character varying, "dateTimeOriginal" timestamp with time zone, "modifyDate" timestamp with time zone, "lensModel" character varying, "fNumber" double precision, "focalLength" double precision, "iso" integer, "latitude" double precision, "longitude" double precision, "city" character varying, "state" character varying, "country" character varying, "description" text NOT NULL DEFAULT '', "fps" double precision, "exposureTime" character varying, "livePhotoCID" character varying, "timeZone" character varying, "projectionType" character varying, "profileDescription" character varying, "colorspace" character varying, "bitsPerSample" integer, "autoStackId" character varying, "rating" integer, "updatedAt" timestamp with time zone NOT NULL DEFAULT clock_timestamp(), "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "face_search" ("faceId" uuid NOT NULL, "embedding" vector(512) NOT NULL);`.execute(db); + await sql`CREATE TABLE "geodata_places" ("id" integer NOT NULL, "name" character varying(200) NOT NULL, "longitude" double precision NOT NULL, "latitude" double precision NOT NULL, "countryCode" character(2) NOT NULL, "admin1Code" character varying(20), "admin2Code" character varying(80), "modificationDate" date NOT NULL, "admin1Name" character varying, "admin2Name" character varying, "alternateNames" character varying);`.execute(db); + await sql`CREATE TABLE "memories" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "deletedAt" timestamp with time zone, "ownerId" uuid NOT NULL, "type" character varying NOT NULL, "data" jsonb NOT NULL, "isSaved" boolean NOT NULL DEFAULT false, "memoryAt" timestamp with time zone NOT NULL, "seenAt" timestamp with time zone, "showAt" timestamp with time zone, "hideAt" timestamp with time zone, "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "memories_assets_assets" ("memoriesId" uuid NOT NULL, "assetsId" uuid NOT NULL);`.execute(db); + await sql`CREATE TABLE "move_history" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "entityId" uuid NOT NULL, "pathType" character varying NOT NULL, "oldPath" character varying NOT NULL, "newPath" character varying NOT NULL);`.execute(db); + await sql`CREATE TABLE "naturalearth_countries" ("id" integer NOT NULL GENERATED ALWAYS AS IDENTITY, "admin" character varying(50) NOT NULL, "admin_a3" character varying(3) NOT NULL, "type" character varying(50) NOT NULL, "coordinates" polygon NOT NULL);`.execute(db); + await sql`CREATE TABLE "partners_audit" ("id" uuid NOT NULL DEFAULT immich_uuid_v7(), "sharedById" uuid NOT NULL, "sharedWithId" uuid NOT NULL, "deletedAt" timestamp with time zone NOT NULL DEFAULT clock_timestamp());`.execute(db); + await sql`CREATE TABLE "partners" ("sharedById" uuid NOT NULL, "sharedWithId" uuid NOT NULL, "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "inTimeline" boolean NOT NULL DEFAULT false, "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "sessions" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "token" character varying NOT NULL, "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "userId" uuid NOT NULL, "deviceType" character varying NOT NULL DEFAULT '', "deviceOS" character varying NOT NULL DEFAULT '', "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "shared_links" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "description" character varying, "userId" uuid NOT NULL, "key" bytea NOT NULL, "type" character varying NOT NULL, "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "expiresAt" timestamp with time zone, "allowUpload" boolean NOT NULL DEFAULT false, "albumId" uuid, "allowDownload" boolean NOT NULL DEFAULT true, "showExif" boolean NOT NULL DEFAULT true, "password" character varying);`.execute(db); + await sql`CREATE TABLE "shared_link__asset" ("assetsId" uuid NOT NULL, "sharedLinksId" uuid NOT NULL);`.execute(db); + await sql`CREATE TABLE "smart_search" ("assetId" uuid NOT NULL, "embedding" vector(512) NOT NULL);`.execute(db); + await sql`ALTER TABLE "smart_search" ALTER COLUMN "embedding" SET STORAGE EXTERNAL;`.execute(db); + await sql`CREATE TABLE "session_sync_checkpoints" ("sessionId" uuid NOT NULL, "type" character varying NOT NULL, "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "ack" character varying NOT NULL, "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "system_metadata" ("key" character varying NOT NULL, "value" jsonb NOT NULL);`.execute(db); + await sql`CREATE TABLE "tags" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "userId" uuid NOT NULL, "value" character varying NOT NULL, "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "updatedAt" timestamp with time zone NOT NULL DEFAULT now(), "color" character varying, "parentId" uuid, "updateId" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "tag_asset" ("assetsId" uuid NOT NULL, "tagsId" uuid NOT NULL);`.execute(db); + await sql`CREATE TABLE "tags_closure" ("id_ancestor" uuid NOT NULL, "id_descendant" uuid NOT NULL);`.execute(db); + await sql`CREATE TABLE "users_audit" ("userId" uuid NOT NULL, "deletedAt" timestamp with time zone NOT NULL DEFAULT clock_timestamp(), "id" uuid NOT NULL DEFAULT immich_uuid_v7());`.execute(db); + await sql`CREATE TABLE "user_metadata" ("userId" uuid NOT NULL, "key" character varying NOT NULL, "value" jsonb NOT NULL);`.execute(db); + await sql`CREATE TABLE "version_history" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "createdAt" timestamp with time zone NOT NULL DEFAULT now(), "version" character varying NOT NULL);`.execute(db); + await sql`ALTER TABLE "users" ADD CONSTRAINT "PK_a3ffb1c0c8416b9fc6f907b7433" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "libraries" ADD CONSTRAINT "PK_505fedfcad00a09b3734b4223de" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "asset_stack" ADD CONSTRAINT "PK_74a27e7fcbd5852463d0af3034b" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "assets" ADD CONSTRAINT "PK_da96729a8b113377cfb6a62439c" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "albums" ADD CONSTRAINT "PK_7f71c7b5bc7c87b8f94c9a93a00" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "activity" ADD CONSTRAINT "PK_24625a1d6b1b089c8ae206fe467" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "albums_assets_assets" ADD CONSTRAINT "PK_c67bc36fa845fb7b18e0e398180" PRIMARY KEY ("albumsId", "assetsId");`.execute(db); + await sql`ALTER TABLE "albums_shared_users_users" ADD CONSTRAINT "PK_7df55657e0b2e8b626330a0ebc8" PRIMARY KEY ("albumsId", "usersId");`.execute(db); + await sql`ALTER TABLE "api_keys" ADD CONSTRAINT "PK_5c8a79801b44bd27b79228e1dad" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "assets_audit" ADD CONSTRAINT "PK_99bd5c015f81a641927a32b4212" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "person" ADD CONSTRAINT "PK_5fdaf670315c4b7e70cce85daa3" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "asset_faces" ADD CONSTRAINT "PK_6df76ab2eb6f5b57b7c2f1fc684" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "asset_files" ADD CONSTRAINT "PK_c41dc3e9ef5e1c57ca5a08a0004" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "asset_job_status" ADD CONSTRAINT "PK_420bec36fc02813bddf5c8b73d4" PRIMARY KEY ("assetId");`.execute(db); + await sql`ALTER TABLE "audit" ADD CONSTRAINT "PK_1d3d120ddaf7bc9b1ed68ed463a" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "exif" ADD CONSTRAINT "PK_c0117fdbc50b917ef9067740c44" PRIMARY KEY ("assetId");`.execute(db); + await sql`ALTER TABLE "face_search" ADD CONSTRAINT "face_search_pkey" PRIMARY KEY ("faceId");`.execute(db); + await sql`ALTER TABLE "geodata_places" ADD CONSTRAINT "PK_c29918988912ef4036f3d7fbff4" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "memories" ADD CONSTRAINT "PK_aaa0692d9496fe827b0568612f8" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "memories_assets_assets" ADD CONSTRAINT "PK_fcaf7112a013d1703c011c6793d" PRIMARY KEY ("memoriesId", "assetsId");`.execute(db); + await sql`ALTER TABLE "move_history" ADD CONSTRAINT "PK_af608f132233acf123f2949678d" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "naturalearth_countries" ADD CONSTRAINT "PK_21a6d86d1ab5d841648212e5353" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "partners_audit" ADD CONSTRAINT "PK_952b50217ff78198a7e380f0359" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "partners" ADD CONSTRAINT "PK_f1cc8f73d16b367f426261a8736" PRIMARY KEY ("sharedById", "sharedWithId");`.execute(db); + await sql`ALTER TABLE "sessions" ADD CONSTRAINT "PK_48cb6b5c20faa63157b3c1baf7f" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "shared_links" ADD CONSTRAINT "PK_642e2b0f619e4876e5f90a43465" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "shared_link__asset" ADD CONSTRAINT "PK_9b4f3687f9b31d1e311336b05e3" PRIMARY KEY ("assetsId", "sharedLinksId");`.execute(db); + await sql`ALTER TABLE "smart_search" ADD CONSTRAINT "smart_search_pkey" PRIMARY KEY ("assetId");`.execute(db); + await sql`ALTER TABLE "session_sync_checkpoints" ADD CONSTRAINT "PK_b846ab547a702863ef7cd9412fb" PRIMARY KEY ("sessionId", "type");`.execute(db); + await sql`ALTER TABLE "system_metadata" ADD CONSTRAINT "PK_fa94f6857470fb5b81ec6084465" PRIMARY KEY ("key");`.execute(db); + await sql`ALTER TABLE "tags" ADD CONSTRAINT "PK_e7dc17249a1148a1970748eda99" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "tag_asset" ADD CONSTRAINT "PK_ef5346fe522b5fb3bc96454747e" PRIMARY KEY ("assetsId", "tagsId");`.execute(db); + await sql`ALTER TABLE "tags_closure" ADD CONSTRAINT "PK_eab38eb12a3ec6df8376c95477c" PRIMARY KEY ("id_ancestor", "id_descendant");`.execute(db); + await sql`ALTER TABLE "users_audit" ADD CONSTRAINT "PK_e9b2bdfd90e7eb5961091175180" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "user_metadata" ADD CONSTRAINT "PK_5931462150b3438cbc83277fe5a" PRIMARY KEY ("userId", "key");`.execute(db); + await sql`ALTER TABLE "version_history" ADD CONSTRAINT "PK_5db259cbb09ce82c0d13cfd1b23" PRIMARY KEY ("id");`.execute(db); + await sql`ALTER TABLE "libraries" ADD CONSTRAINT "FK_0f6fc2fb195f24d19b0fb0d57c1" FOREIGN KEY ("ownerId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "asset_stack" ADD CONSTRAINT "FK_91704e101438fd0653f582426dc" FOREIGN KEY ("primaryAssetId") REFERENCES "assets" ("id") ON UPDATE NO ACTION ON DELETE NO ACTION;`.execute(db); + await sql`ALTER TABLE "asset_stack" ADD CONSTRAINT "FK_c05079e542fd74de3b5ecb5c1c8" FOREIGN KEY ("ownerId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "assets" ADD CONSTRAINT "FK_2c5ac0d6fb58b238fd2068de67d" FOREIGN KEY ("ownerId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "assets" ADD CONSTRAINT "FK_16294b83fa8c0149719a1f631ef" FOREIGN KEY ("livePhotoVideoId") REFERENCES "assets" ("id") ON UPDATE CASCADE ON DELETE SET NULL;`.execute(db); + await sql`ALTER TABLE "assets" ADD CONSTRAINT "FK_9977c3c1de01c3d848039a6b90c" FOREIGN KEY ("libraryId") REFERENCES "libraries" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "assets" ADD CONSTRAINT "FK_f15d48fa3ea5e4bda05ca8ab207" FOREIGN KEY ("stackId") REFERENCES "asset_stack" ("id") ON UPDATE CASCADE ON DELETE SET NULL;`.execute(db); + await sql`ALTER TABLE "albums" ADD CONSTRAINT "FK_b22c53f35ef20c28c21637c85f4" FOREIGN KEY ("ownerId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "albums" ADD CONSTRAINT "FK_05895aa505a670300d4816debce" FOREIGN KEY ("albumThumbnailAssetId") REFERENCES "assets" ("id") ON UPDATE CASCADE ON DELETE SET NULL;`.execute(db); + await sql`ALTER TABLE "activity" ADD CONSTRAINT "FK_1af8519996fbfb3684b58df280b" FOREIGN KEY ("albumId") REFERENCES "albums" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "activity" ADD CONSTRAINT "FK_3571467bcbe021f66e2bdce96ea" FOREIGN KEY ("userId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "activity" ADD CONSTRAINT "FK_8091ea76b12338cb4428d33d782" FOREIGN KEY ("assetId") REFERENCES "assets" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "albums_assets_assets" ADD CONSTRAINT "FK_e590fa396c6898fcd4a50e40927" FOREIGN KEY ("albumsId") REFERENCES "albums" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "albums_assets_assets" ADD CONSTRAINT "FK_4bd1303d199f4e72ccdf998c621" FOREIGN KEY ("assetsId") REFERENCES "assets" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "albums_shared_users_users" ADD CONSTRAINT "FK_427c350ad49bd3935a50baab737" FOREIGN KEY ("albumsId") REFERENCES "albums" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "albums_shared_users_users" ADD CONSTRAINT "FK_f48513bf9bccefd6ff3ad30bd06" FOREIGN KEY ("usersId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "api_keys" ADD CONSTRAINT "FK_6c2e267ae764a9413b863a29342" FOREIGN KEY ("userId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "person" ADD CONSTRAINT "FK_5527cc99f530a547093f9e577b6" FOREIGN KEY ("ownerId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "person" ADD CONSTRAINT "FK_2bbabe31656b6778c6b87b61023" FOREIGN KEY ("faceAssetId") REFERENCES "asset_faces" ("id") ON UPDATE NO ACTION ON DELETE SET NULL;`.execute(db); + await sql`ALTER TABLE "asset_faces" ADD CONSTRAINT "FK_02a43fd0b3c50fb6d7f0cb7282c" FOREIGN KEY ("assetId") REFERENCES "assets" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "asset_faces" ADD CONSTRAINT "FK_95ad7106dd7b484275443f580f9" FOREIGN KEY ("personId") REFERENCES "person" ("id") ON UPDATE CASCADE ON DELETE SET NULL;`.execute(db); + await sql`ALTER TABLE "asset_files" ADD CONSTRAINT "FK_e3e103a5f1d8bc8402999286040" FOREIGN KEY ("assetId") REFERENCES "assets" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "asset_job_status" ADD CONSTRAINT "FK_420bec36fc02813bddf5c8b73d4" FOREIGN KEY ("assetId") REFERENCES "assets" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "exif" ADD CONSTRAINT "FK_c0117fdbc50b917ef9067740c44" FOREIGN KEY ("assetId") REFERENCES "assets" ("id") ON UPDATE NO ACTION ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "face_search" ADD CONSTRAINT "face_search_faceId_fkey" FOREIGN KEY ("faceId") REFERENCES "asset_faces" ("id") ON UPDATE NO ACTION ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "memories" ADD CONSTRAINT "FK_575842846f0c28fa5da46c99b19" FOREIGN KEY ("ownerId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "memories_assets_assets" ADD CONSTRAINT "FK_984e5c9ab1f04d34538cd32334e" FOREIGN KEY ("memoriesId") REFERENCES "memories" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "memories_assets_assets" ADD CONSTRAINT "FK_6942ecf52d75d4273de19d2c16f" FOREIGN KEY ("assetsId") REFERENCES "assets" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "partners" ADD CONSTRAINT "FK_7e077a8b70b3530138610ff5e04" FOREIGN KEY ("sharedById") REFERENCES "users" ("id") ON UPDATE NO ACTION ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "partners" ADD CONSTRAINT "FK_d7e875c6c60e661723dbf372fd3" FOREIGN KEY ("sharedWithId") REFERENCES "users" ("id") ON UPDATE NO ACTION ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "sessions" ADD CONSTRAINT "FK_57de40bc620f456c7311aa3a1e6" FOREIGN KEY ("userId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "shared_links" ADD CONSTRAINT "FK_66fe3837414c5a9f1c33ca49340" FOREIGN KEY ("userId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "shared_links" ADD CONSTRAINT "FK_0c6ce9058c29f07cdf7014eac66" FOREIGN KEY ("albumId") REFERENCES "albums" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "shared_link__asset" ADD CONSTRAINT "FK_5b7decce6c8d3db9593d6111a66" FOREIGN KEY ("assetsId") REFERENCES "assets" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "shared_link__asset" ADD CONSTRAINT "FK_c9fab4aa97ffd1b034f3d6581ab" FOREIGN KEY ("sharedLinksId") REFERENCES "shared_links" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "smart_search" ADD CONSTRAINT "smart_search_assetId_fkey" FOREIGN KEY ("assetId") REFERENCES "assets" ("id") ON UPDATE NO ACTION ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "session_sync_checkpoints" ADD CONSTRAINT "FK_d8ddd9d687816cc490432b3d4bc" FOREIGN KEY ("sessionId") REFERENCES "sessions" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "tags" ADD CONSTRAINT "FK_92e67dc508c705dd66c94615576" FOREIGN KEY ("userId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "tags" ADD CONSTRAINT "FK_9f9590cc11561f1f48ff034ef99" FOREIGN KEY ("parentId") REFERENCES "tags" ("id") ON UPDATE NO ACTION ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "tag_asset" ADD CONSTRAINT "FK_f8e8a9e893cb5c54907f1b798e9" FOREIGN KEY ("assetsId") REFERENCES "assets" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "tag_asset" ADD CONSTRAINT "FK_e99f31ea4cdf3a2c35c7287eb42" FOREIGN KEY ("tagsId") REFERENCES "tags" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "tags_closure" ADD CONSTRAINT "FK_15fbcbc67663c6bfc07b354c22c" FOREIGN KEY ("id_ancestor") REFERENCES "tags" ("id") ON UPDATE NO ACTION ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "tags_closure" ADD CONSTRAINT "FK_b1a2a7ed45c29179b5ad51548a1" FOREIGN KEY ("id_descendant") REFERENCES "tags" ("id") ON UPDATE NO ACTION ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "user_metadata" ADD CONSTRAINT "FK_6afb43681a21cf7815932bc38ac" FOREIGN KEY ("userId") REFERENCES "users" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(db); + await sql`ALTER TABLE "users" ADD CONSTRAINT "UQ_97672ac88f789774dd47f7c8be3" UNIQUE ("email");`.execute(db); + await sql`ALTER TABLE "users" ADD CONSTRAINT "UQ_b309cf34fa58137c416b32cea3a" UNIQUE ("storageLabel");`.execute(db); + await sql`ALTER TABLE "asset_stack" ADD CONSTRAINT "REL_91704e101438fd0653f582426d" UNIQUE ("primaryAssetId");`.execute(db); + await sql`ALTER TABLE "asset_files" ADD CONSTRAINT "UQ_assetId_type" UNIQUE ("assetId", "type");`.execute(db); + await sql`ALTER TABLE "move_history" ADD CONSTRAINT "UQ_newPath" UNIQUE ("newPath");`.execute(db); + await sql`ALTER TABLE "move_history" ADD CONSTRAINT "UQ_entityId_pathType" UNIQUE ("entityId", "pathType");`.execute(db); + await sql`ALTER TABLE "shared_links" ADD CONSTRAINT "UQ_sharedlink_key" UNIQUE ("key");`.execute(db); + await sql`ALTER TABLE "tags" ADD CONSTRAINT "UQ_79d6f16e52bb2c7130375246793" UNIQUE ("userId", "value");`.execute(db); + await sql`ALTER TABLE "activity" ADD CONSTRAINT "CHK_2ab1e70f113f450eb40c1e3ec8" CHECK (("comment" IS NULL AND "isLiked" = true) OR ("comment" IS NOT NULL AND "isLiked" = false));`.execute(db); + await sql`ALTER TABLE "person" ADD CONSTRAINT "CHK_b0f82b0ed662bfc24fbb58bb45" CHECK ("birthDate" <= CURRENT_DATE);`.execute(db); + await sql`CREATE INDEX "IDX_users_updated_at_asc_id_asc" ON "users" ("updatedAt", "id")`.execute(db); + await sql`CREATE INDEX "IDX_users_update_id" ON "users" ("updateId")`.execute(db); + await sql`CREATE INDEX "IDX_0f6fc2fb195f24d19b0fb0d57c" ON "libraries" ("ownerId")`.execute(db); + await sql`CREATE INDEX "IDX_libraries_update_id" ON "libraries" ("updateId")`.execute(db); + await sql`CREATE INDEX "IDX_91704e101438fd0653f582426d" ON "asset_stack" ("primaryAssetId")`.execute(db); + await sql`CREATE INDEX "IDX_c05079e542fd74de3b5ecb5c1c" ON "asset_stack" ("ownerId")`.execute(db); + await sql`CREATE INDEX "idx_originalfilename_trigram" ON "assets" USING gin (f_unaccent("originalFileName") gin_trgm_ops)`.execute(db); + await sql`CREATE INDEX "IDX_asset_id_stackId" ON "assets" ("id", "stackId")`.execute(db); + await sql`CREATE INDEX "IDX_originalPath_libraryId" ON "assets" ("originalPath", "libraryId")`.execute(db); + await sql`CREATE INDEX "idx_local_date_time_month" ON "assets" ((date_trunc('MONTH'::text, ("localDateTime" AT TIME ZONE 'UTC'::text)) AT TIME ZONE 'UTC'::text))`.execute(db); + await sql`CREATE INDEX "idx_local_date_time" ON "assets" ((("localDateTime" at time zone 'UTC')::date))`.execute(db); + await sql`CREATE UNIQUE INDEX "UQ_assets_owner_library_checksum" ON "assets" ("ownerId", "libraryId", "checksum") WHERE ("libraryId" IS NOT NULL)`.execute(db); + await sql`CREATE UNIQUE INDEX "UQ_assets_owner_checksum" ON "assets" ("ownerId", "checksum") WHERE ("libraryId" IS NULL)`.execute(db); + await sql`CREATE INDEX "IDX_2c5ac0d6fb58b238fd2068de67" ON "assets" ("ownerId")`.execute(db); + await sql`CREATE INDEX "idx_asset_file_created_at" ON "assets" ("fileCreatedAt")`.execute(db); + await sql`CREATE INDEX "IDX_8d3efe36c0755849395e6ea866" ON "assets" ("checksum")`.execute(db); + await sql`CREATE INDEX "IDX_16294b83fa8c0149719a1f631e" ON "assets" ("livePhotoVideoId")`.execute(db); + await sql`CREATE INDEX "IDX_4d66e76dada1ca180f67a205dc" ON "assets" ("originalFileName")`.execute(db); + await sql`CREATE INDEX "IDX_9977c3c1de01c3d848039a6b90" ON "assets" ("libraryId")`.execute(db); + await sql`CREATE INDEX "IDX_f15d48fa3ea5e4bda05ca8ab20" ON "assets" ("stackId")`.execute(db); + await sql`CREATE INDEX "IDX_assets_duplicateId" ON "assets" ("duplicateId")`.execute(db); + await sql`CREATE INDEX "IDX_assets_update_id" ON "assets" ("updateId")`.execute(db); + await sql`CREATE INDEX "IDX_b22c53f35ef20c28c21637c85f" ON "albums" ("ownerId")`.execute(db); + await sql`CREATE INDEX "IDX_05895aa505a670300d4816debc" ON "albums" ("albumThumbnailAssetId")`.execute(db); + await sql`CREATE INDEX "IDX_albums_update_id" ON "albums" ("updateId")`.execute(db); + await sql`CREATE UNIQUE INDEX "IDX_activity_like" ON "activity" ("assetId", "userId", "albumId") WHERE ("isLiked" = true)`.execute(db); + await sql`CREATE INDEX "IDX_1af8519996fbfb3684b58df280" ON "activity" ("albumId")`.execute(db); + await sql`CREATE INDEX "IDX_3571467bcbe021f66e2bdce96e" ON "activity" ("userId")`.execute(db); + await sql`CREATE INDEX "IDX_8091ea76b12338cb4428d33d78" ON "activity" ("assetId")`.execute(db); + await sql`CREATE INDEX "IDX_activity_update_id" ON "activity" ("updateId")`.execute(db); + await sql`CREATE INDEX "IDX_e590fa396c6898fcd4a50e4092" ON "albums_assets_assets" ("albumsId")`.execute(db); + await sql`CREATE INDEX "IDX_4bd1303d199f4e72ccdf998c62" ON "albums_assets_assets" ("assetsId")`.execute(db); + await sql`CREATE INDEX "IDX_f48513bf9bccefd6ff3ad30bd0" ON "albums_shared_users_users" ("usersId")`.execute(db); + await sql`CREATE INDEX "IDX_427c350ad49bd3935a50baab73" ON "albums_shared_users_users" ("albumsId")`.execute(db); + await sql`CREATE INDEX "IDX_6c2e267ae764a9413b863a2934" ON "api_keys" ("userId")`.execute(db); + await sql`CREATE INDEX "IDX_api_keys_update_id" ON "api_keys" ("updateId")`.execute(db); + await sql`CREATE INDEX "IDX_assets_audit_asset_id" ON "assets_audit" ("assetId")`.execute(db); + await sql`CREATE INDEX "IDX_assets_audit_owner_id" ON "assets_audit" ("ownerId")`.execute(db); + await sql`CREATE INDEX "IDX_assets_audit_deleted_at" ON "assets_audit" ("deletedAt")`.execute(db); + await sql`CREATE INDEX "IDX_5527cc99f530a547093f9e577b" ON "person" ("ownerId")`.execute(db); + await sql`CREATE INDEX "IDX_2bbabe31656b6778c6b87b6102" ON "person" ("faceAssetId")`.execute(db); + await sql`CREATE INDEX "IDX_person_update_id" ON "person" ("updateId")`.execute(db); + await sql`CREATE INDEX "IDX_bf339a24070dac7e71304ec530" ON "asset_faces" ("personId", "assetId")`.execute(db); + await sql`CREATE INDEX "IDX_asset_faces_assetId_personId" ON "asset_faces" ("assetId", "personId")`.execute(db); + await sql`CREATE INDEX "IDX_02a43fd0b3c50fb6d7f0cb7282" ON "asset_faces" ("assetId")`.execute(db); + await sql`CREATE INDEX "IDX_95ad7106dd7b484275443f580f" ON "asset_faces" ("personId")`.execute(db); + await sql`CREATE INDEX "IDX_asset_files_assetId" ON "asset_files" ("assetId")`.execute(db); + await sql`CREATE INDEX "IDX_asset_files_update_id" ON "asset_files" ("updateId")`.execute(db); + await sql`CREATE INDEX "IDX_ownerId_createdAt" ON "audit" ("ownerId", "createdAt")`.execute(db); + await sql`CREATE INDEX "exif_city" ON "exif" ("city")`.execute(db); + await sql`CREATE INDEX "IDX_live_photo_cid" ON "exif" ("livePhotoCID")`.execute(db); + await sql`CREATE INDEX "IDX_auto_stack_id" ON "exif" ("autoStackId")`.execute(db); + await sql`CREATE INDEX "IDX_asset_exif_update_id" ON "exif" ("updateId")`.execute(db); + await sql`CREATE INDEX "face_index" ON "face_search" USING hnsw (embedding vector_cosine_ops) WITH (ef_construction = 300, m = 16)`.execute(db); + await sql`CREATE INDEX "IDX_geodata_gist_earthcoord" ON "geodata_places" (ll_to_earth_public(latitude, longitude))`.execute(db); + await sql`CREATE INDEX "idx_geodata_places_name" ON "geodata_places" USING gin (f_unaccent("name") gin_trgm_ops)`.execute(db); + await sql`CREATE INDEX "idx_geodata_places_admin2_name" ON "geodata_places" USING gin (f_unaccent("admin2Name") gin_trgm_ops)`.execute(db); + await sql`CREATE INDEX "idx_geodata_places_admin1_name" ON "geodata_places" USING gin (f_unaccent("admin1Name") gin_trgm_ops)`.execute(db); + await sql`CREATE INDEX "idx_geodata_places_alternate_names" ON "geodata_places" USING gin (f_unaccent("alternateNames") gin_trgm_ops)`.execute(db); + await sql`CREATE INDEX "IDX_575842846f0c28fa5da46c99b1" ON "memories" ("ownerId")`.execute(db); + await sql`CREATE INDEX "IDX_memories_update_id" ON "memories" ("updateId")`.execute(db); + await sql`CREATE INDEX "IDX_984e5c9ab1f04d34538cd32334" ON "memories_assets_assets" ("memoriesId")`.execute(db); + await sql`CREATE INDEX "IDX_6942ecf52d75d4273de19d2c16" ON "memories_assets_assets" ("assetsId")`.execute(db); + await sql`CREATE INDEX "IDX_partners_audit_shared_by_id" ON "partners_audit" ("sharedById")`.execute(db); + await sql`CREATE INDEX "IDX_partners_audit_shared_with_id" ON "partners_audit" ("sharedWithId")`.execute(db); + await sql`CREATE INDEX "IDX_partners_audit_deleted_at" ON "partners_audit" ("deletedAt")`.execute(db); + await sql`CREATE INDEX "IDX_7e077a8b70b3530138610ff5e0" ON "partners" ("sharedById")`.execute(db); + await sql`CREATE INDEX "IDX_d7e875c6c60e661723dbf372fd" ON "partners" ("sharedWithId")`.execute(db); + await sql`CREATE INDEX "IDX_partners_update_id" ON "partners" ("updateId")`.execute(db); + await sql`CREATE INDEX "IDX_57de40bc620f456c7311aa3a1e" ON "sessions" ("userId")`.execute(db); + await sql`CREATE INDEX "IDX_sessions_update_id" ON "sessions" ("updateId")`.execute(db); + await sql`CREATE INDEX "IDX_66fe3837414c5a9f1c33ca4934" ON "shared_links" ("userId")`.execute(db); + await sql`CREATE INDEX "IDX_sharedlink_key" ON "shared_links" ("key")`.execute(db); + await sql`CREATE INDEX "IDX_sharedlink_albumId" ON "shared_links" ("albumId")`.execute(db); + await sql`CREATE INDEX "IDX_5b7decce6c8d3db9593d6111a6" ON "shared_link__asset" ("assetsId")`.execute(db); + await sql`CREATE INDEX "IDX_c9fab4aa97ffd1b034f3d6581a" ON "shared_link__asset" ("sharedLinksId")`.execute(db); + await sql`CREATE INDEX "clip_index" ON "smart_search" USING hnsw (embedding vector_cosine_ops) WITH (ef_construction = 300, m = 16)`.execute(db); + await sql`CREATE INDEX "IDX_d8ddd9d687816cc490432b3d4b" ON "session_sync_checkpoints" ("sessionId")`.execute(db); + await sql`CREATE INDEX "IDX_session_sync_checkpoints_update_id" ON "session_sync_checkpoints" ("updateId")`.execute(db); + await sql`CREATE INDEX "IDX_92e67dc508c705dd66c9461557" ON "tags" ("userId")`.execute(db); + await sql`CREATE INDEX "IDX_9f9590cc11561f1f48ff034ef9" ON "tags" ("parentId")`.execute(db); + await sql`CREATE INDEX "IDX_tags_update_id" ON "tags" ("updateId")`.execute(db); + await sql`CREATE INDEX "IDX_tag_asset_assetsId_tagsId" ON "tag_asset" ("assetsId", "tagsId")`.execute(db); + await sql`CREATE INDEX "IDX_f8e8a9e893cb5c54907f1b798e" ON "tag_asset" ("assetsId")`.execute(db); + await sql`CREATE INDEX "IDX_e99f31ea4cdf3a2c35c7287eb4" ON "tag_asset" ("tagsId")`.execute(db); + await sql`CREATE INDEX "IDX_15fbcbc67663c6bfc07b354c22" ON "tags_closure" ("id_ancestor")`.execute(db); + await sql`CREATE INDEX "IDX_b1a2a7ed45c29179b5ad51548a" ON "tags_closure" ("id_descendant")`.execute(db); + await sql`CREATE INDEX "IDX_users_audit_deleted_at" ON "users_audit" ("deletedAt")`.execute(db); + await sql`CREATE INDEX "IDX_6afb43681a21cf7815932bc38a" ON "user_metadata" ("userId")`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "users_delete_audit" + AFTER DELETE ON "users" + REFERENCING OLD TABLE AS "old" + FOR EACH STATEMENT + WHEN (pg_trigger_depth() = 0) + EXECUTE FUNCTION users_delete_audit();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "users_updated_at" + BEFORE UPDATE ON "users" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "libraries_updated_at" + BEFORE UPDATE ON "libraries" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "assets_delete_audit" + AFTER DELETE ON "assets" + REFERENCING OLD TABLE AS "old" + FOR EACH STATEMENT + WHEN (pg_trigger_depth() = 0) + EXECUTE FUNCTION assets_delete_audit();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "assets_updated_at" + BEFORE UPDATE ON "assets" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "albums_updated_at" + BEFORE UPDATE ON "albums" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "activity_updated_at" + BEFORE UPDATE ON "activity" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "api_keys_updated_at" + BEFORE UPDATE ON "api_keys" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "person_updated_at" + BEFORE UPDATE ON "person" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "asset_files_updated_at" + BEFORE UPDATE ON "asset_files" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "asset_exif_updated_at" + BEFORE UPDATE ON "exif" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "memories_updated_at" + BEFORE UPDATE ON "memories" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "partners_delete_audit" + AFTER DELETE ON "partners" + REFERENCING OLD TABLE AS "old" + FOR EACH STATEMENT + WHEN (pg_trigger_depth() = 0) + EXECUTE FUNCTION partners_delete_audit();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "partners_updated_at" + BEFORE UPDATE ON "partners" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "sessions_updated_at" + BEFORE UPDATE ON "sessions" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "session_sync_checkpoints_updated_at" + BEFORE UPDATE ON "session_sync_checkpoints" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "tags_updated_at" + BEFORE UPDATE ON "tags" + FOR EACH ROW + EXECUTE FUNCTION updated_at();`.execute(db); +} + +export async function down(): Promise { +// not implemented +} diff --git a/server/src/schema/tables/geodata-places.table.ts b/server/src/schema/tables/geodata-places.table.ts index 631cfdff08..3e78b4cfcf 100644 --- a/server/src/schema/tables/geodata-places.table.ts +++ b/server/src/schema/tables/geodata-places.table.ts @@ -1,6 +1,6 @@ import { Column, Index, PrimaryColumn, Table } from 'src/sql-tools'; -@Table({ name: 'geodata_places' }) +@Table({ name: 'geodata_places', synchronize: false }) @Index({ name: 'idx_geodata_places_alternate_names', using: 'gin', @@ -26,11 +26,10 @@ import { Column, Index, PrimaryColumn, Table } from 'src/sql-tools'; synchronize: false, }) @Index({ - name: 'idx_geodata_places_gist_earthcoord', + name: 'IDX_geodata_gist_earthcoord', expression: 'll_to_earth_public(latitude, longitude)', synchronize: false, }) -@Table({ name: 'idx_geodata_places', synchronize: false }) export class GeodataPlacesTable { @PrimaryColumn({ type: 'integer' }) id!: number; diff --git a/server/src/sql-tools/from-code/processors/table.processor.ts b/server/src/sql-tools/from-code/processors/table.processor.ts index eb4b414576..4ef4e82020 100644 --- a/server/src/sql-tools/from-code/processors/table.processor.ts +++ b/server/src/sql-tools/from-code/processors/table.processor.ts @@ -6,6 +6,13 @@ export const processTables: Processor = (builder, items) => { for (const { item: { options, object }, } of items.filter((item) => item.type === 'table')) { + const test = readMetadata(object); + if (test) { + throw new Error( + `Table ${test.name} has already been registered. Does ${object.name} have two @Table() decorators?`, + ); + } + const tableName = options.name || asSnakeCase(object.name); writeMetadata(object, { name: tableName, options }); diff --git a/server/src/sql-tools/from-code/register-function.ts b/server/src/sql-tools/from-code/register-function.ts index be71e0dfd7..3e1e7054be 100644 --- a/server/src/sql-tools/from-code/register-function.ts +++ b/server/src/sql-tools/from-code/register-function.ts @@ -50,12 +50,8 @@ const asFunctionExpression = (options: FunctionOptions) => { } if ('body' in options) { - sql.push( - // - `AS $$`, - ' ' + options.body.trim(), - `$$;`, - ); + const body = options.body; + sql.push(...(body.includes('\n') ? [`AS $$`, ' ' + body.trim(), `$$;`] : [`AS $$${body}$$;`])); } return sql.join('\n ').trim(); diff --git a/server/test/medium/globalSetup.ts b/server/test/medium/globalSetup.ts index 89b25da766..46eb1a733f 100644 --- a/server/test/medium/globalSetup.ts +++ b/server/test/medium/globalSetup.ts @@ -1,10 +1,11 @@ -import { FileMigrationProvider, Kysely, Migrator } from 'kysely'; -import { mkdir, readdir } from 'node:fs/promises'; -import { join } from 'node:path'; +import { Kysely } from 'kysely'; import { parse } from 'pg-connection-string'; +import { DB } from 'src/db'; +import { ConfigRepository } from 'src/repositories/config.repository'; +import { DatabaseRepository } from 'src/repositories/database.repository'; +import { LoggingRepository } from 'src/repositories/logging.repository'; import { getKyselyConfig } from 'src/utils/database'; import { GenericContainer, Wait } from 'testcontainers'; -import { DataSource } from 'typeorm'; const globalSetup = async () => { const postgresContainer = await new GenericContainer('tensorchord/pgvecto-rs:pg14-v0.2.0') @@ -36,66 +37,23 @@ const globalSetup = async () => { const postgresPort = postgresContainer.getMappedPort(5432); const postgresUrl = `postgres://postgres:postgres@localhost:${postgresPort}/immich`; + const parsed = parse(postgresUrl); + process.env.IMMICH_TEST_POSTGRES_URL = postgresUrl; - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-expect-error - const modules = import.meta.glob('/src/migrations/*.ts', { eager: true }); - - const config = { - type: 'postgres' as const, - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-expect-error - migrations: Object.values(modules).map((module) => Object.values(module)[0]), - migrationsRun: false, - synchronize: false, - connectTimeoutMS: 10_000, // 10 seconds - parseInt8: true, - url: postgresUrl, - }; - - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-expect-error - const dataSource = new DataSource(config); - await dataSource.initialize(); - await dataSource.runMigrations(); - await dataSource.destroy(); - - // for whatever reason, importing from test/utils causes vitest to crash - // eslint-disable-next-line unicorn/prefer-module - const migrationFolder = join(__dirname, '..', 'schema/migrations'); - // TODO remove after we have at least one kysely migration - await mkdir(migrationFolder, { recursive: true }); - - const parsed = parse(process.env.IMMICH_TEST_POSTGRES_URL!); - - const parsedOptions = { - ...parsed, - ssl: false, - host: parsed.host ?? undefined, - port: parsed.port ? Number(parsed.port) : undefined, - database: parsed.database ?? undefined, - }; - - const db = new Kysely(getKyselyConfig(parsedOptions)); - - // TODO just call `databaseRepository.migrate()` (probably have to wait until TypeOrm is gone) - const migrator = new Migrator({ - db, - migrationLockTableName: 'kysely_migrations_lock', - migrationTableName: 'kysely_migrations', - provider: new FileMigrationProvider({ - fs: { readdir }, - path: { join }, - migrationFolder, + const db = new Kysely( + getKyselyConfig({ + ...parsed, + ssl: false, + host: parsed.host ?? undefined, + port: parsed.port ? Number(parsed.port) : undefined, + database: parsed.database ?? undefined, }), - }); + ); - const { error } = await migrator.migrateToLatest(); - if (error) { - console.error('Unable to run kysely migrations', error); - throw error; - } + const configRepository = new ConfigRepository(); + const logger = new LoggingRepository(undefined, configRepository); + await new DatabaseRepository(db, logger, configRepository).runMigrations(); await db.destroy(); }; diff --git a/server/test/sql-tools/index-with-where.stub copy.ts b/server/test/sql-tools/index-with-expression.ts similarity index 100% rename from server/test/sql-tools/index-with-where.stub copy.ts rename to server/test/sql-tools/index-with-expression.ts From 160bb492a2ae1344718578281756f79bb33f0489 Mon Sep 17 00:00:00 2001 From: Jason Rasmussen Date: Fri, 18 Apr 2025 12:19:11 -0400 Subject: [PATCH 06/29] fix: skip initial kysely migration for existing installs (#17690) * fix: skip initial kysely migration for existing installs * Update docs/src/pages/errors.md --------- Co-authored-by: Alex --- docs/src/pages/errors.md | 5 +++++ server/src/repositories/logging.repository.ts | 14 ++++++++++++-- .../1744910873969-InitialMigration.ts | 19 +++++++++++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 docs/src/pages/errors.md diff --git a/docs/src/pages/errors.md b/docs/src/pages/errors.md new file mode 100644 index 0000000000..e9bf09770c --- /dev/null +++ b/docs/src/pages/errors.md @@ -0,0 +1,5 @@ +# Errors + +## TypeORM Upgrade + +The upgrade to Immich `v2.x.x` has a required upgrade path to `v1.132.0+`. This means it is required to start up the application at least once on version `1.132.0` (or later). Doing so will complete database schema upgrades that are required for `v2.0.0`. After Immich has successfully booted on this version, shut the system down and try the `v2.x.x` upgrade again. diff --git a/server/src/repositories/logging.repository.ts b/server/src/repositories/logging.repository.ts index 3f809db41e..05d2d45f4d 100644 --- a/server/src/repositories/logging.repository.ts +++ b/server/src/repositories/logging.repository.ts @@ -74,11 +74,21 @@ export class MyConsoleLogger extends ConsoleLogger { export class LoggingRepository { private logger: MyConsoleLogger; - constructor(@Inject(ClsService) cls: ClsService | undefined, configRepository: ConfigRepository) { - const { noColor } = configRepository.getEnv(); + constructor( + @Inject(ClsService) cls: ClsService | undefined, + @Inject(ConfigRepository) configRepository: ConfigRepository | undefined, + ) { + let noColor = false; + if (configRepository) { + noColor = configRepository.getEnv().noColor; + } this.logger = new MyConsoleLogger(cls, { context: LoggingRepository.name, color: !noColor }); } + static create() { + return new LoggingRepository(undefined, undefined); + } + setAppName(name: string): void { appName = name.charAt(0).toUpperCase() + name.slice(1); } diff --git a/server/src/schema/migrations/1744910873969-InitialMigration.ts b/server/src/schema/migrations/1744910873969-InitialMigration.ts index e157607681..459534a26a 100644 --- a/server/src/schema/migrations/1744910873969-InitialMigration.ts +++ b/server/src/schema/migrations/1744910873969-InitialMigration.ts @@ -1,10 +1,29 @@ import { Kysely, sql } from 'kysely'; import { DatabaseExtension } from 'src/enum'; import { ConfigRepository } from 'src/repositories/config.repository'; +import { LoggingRepository } from 'src/repositories/logging.repository'; const vectorExtension = new ConfigRepository().getEnv().database.vectorExtension; +const lastMigrationSql = sql<{ name: string }>`SELECT "name" FROM "migrations" ORDER BY "timestamp" DESC LIMIT 1;`; +const tableExists = sql<{ result: string | null }>`select to_regclass('migrations') as "result"`; +const logger = LoggingRepository.create(); export async function up(db: Kysely): Promise { + const { rows } = await tableExists.execute(db); + const hasTypeOrmMigrations = !!rows[0]?.result; + if (hasTypeOrmMigrations) { + const { + rows: [lastMigration], + } = await lastMigrationSql.execute(db); + if (lastMigration?.name !== 'AddMissingIndex1744910873956') { + throw new Error( + 'Invalid upgrade path. For more information, see https://immich.app/errors#typeorm-upgrade', + ); + } + logger.log('Database has up to date TypeORM migrations, skipping initial Kysely migration'); + return; + } + await sql`CREATE EXTENSION IF NOT EXISTS "uuid-ossp";`.execute(db); await sql`CREATE EXTENSION IF NOT EXISTS "unaccent";`.execute(db); await sql`CREATE EXTENSION IF NOT EXISTS "cube";`.execute(db); From bd2deda50c4481903a04d51a5dc7042fdeea095a Mon Sep 17 00:00:00 2001 From: Yaros Date: Fri, 18 Apr 2025 18:19:51 +0200 Subject: [PATCH 07/29] feat(mobile): search on places page (#17679) * feat: search on places page * chore: use searchfield on people page --- i18n/en.json | 1 + .../people/people_collection.page.dart | 43 ++----------- .../places/places_collection.page.dart | 62 ++++++++++++++----- 3 files changed, 51 insertions(+), 55 deletions(-) diff --git a/i18n/en.json b/i18n/en.json index 3b52c2019e..c4b4746871 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -996,6 +996,7 @@ "filetype": "Filetype", "filter": "Filter", "filter_people": "Filter people", + "filter_places": "Filter places", "find_them_fast": "Find them fast by name with search", "fix_incorrect_match": "Fix incorrect match", "folder": "Folder", diff --git a/mobile/lib/pages/library/people/people_collection.page.dart b/mobile/lib/pages/library/people/people_collection.page.dart index 5f587c0c76..27daf0a887 100644 --- a/mobile/lib/pages/library/people/people_collection.page.dart +++ b/mobile/lib/pages/library/people/people_collection.page.dart @@ -4,11 +4,11 @@ import 'package:flutter/material.dart'; import 'package:flutter_hooks/flutter_hooks.dart'; import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/extensions/build_context_extensions.dart'; -import 'package:immich_mobile/extensions/theme_extensions.dart'; import 'package:immich_mobile/providers/search/people.provider.dart'; import 'package:immich_mobile/routing/router.dart'; import 'package:immich_mobile/services/api.service.dart'; import 'package:immich_mobile/utils/image_url_builder.dart'; +import 'package:immich_mobile/widgets/common/search_field.dart'; import 'package:immich_mobile/widgets/search/person_name_edit_form.dart'; @RoutePage() @@ -42,47 +42,12 @@ class PeopleCollectionPage extends HookConsumerWidget { appBar: AppBar( automaticallyImplyLeading: search.value == null, title: search.value != null - ? TextField( + ? SearchField( focusNode: formFocus, onTapOutside: (_) => formFocus.unfocus(), onChanged: (value) => search.value = value, - decoration: InputDecoration( - contentPadding: const EdgeInsets.only(left: 24), - filled: true, - fillColor: context.primaryColor.withValues(alpha: 0.1), - hintStyle: context.textTheme.bodyLarge?.copyWith( - color: context.themeData.colorScheme.onSurfaceSecondary, - ), - border: OutlineInputBorder( - borderRadius: BorderRadius.circular(25), - borderSide: BorderSide( - color: context.colorScheme.surfaceContainerHighest, - ), - ), - enabledBorder: OutlineInputBorder( - borderRadius: BorderRadius.circular(25), - borderSide: BorderSide( - color: context.colorScheme.surfaceContainerHighest, - ), - ), - disabledBorder: OutlineInputBorder( - borderRadius: BorderRadius.circular(25), - borderSide: BorderSide( - color: context.colorScheme.surfaceContainerHighest, - ), - ), - focusedBorder: OutlineInputBorder( - borderRadius: BorderRadius.circular(25), - borderSide: BorderSide( - color: context.colorScheme.primary.withAlpha(150), - ), - ), - prefixIcon: Icon( - Icons.search_rounded, - color: context.colorScheme.primary, - ), - hintText: 'filter_people'.tr(), - ), + filled: true, + hintText: 'filter_people'.tr(), autofocus: true, ) : Text('people'.tr()), diff --git a/mobile/lib/pages/library/places/places_collection.page.dart b/mobile/lib/pages/library/places/places_collection.page.dart index d4da3ff37e..f9a2d4292c 100644 --- a/mobile/lib/pages/library/places/places_collection.page.dart +++ b/mobile/lib/pages/library/places/places_collection.page.dart @@ -2,6 +2,7 @@ import 'package:auto_route/auto_route.dart'; import 'package:cached_network_image/cached_network_image.dart'; import 'package:easy_localization/easy_localization.dart'; import 'package:flutter/material.dart'; +import 'package:flutter_hooks/flutter_hooks.dart' hide Store; import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/domain/models/store.model.dart'; import 'package:immich_mobile/entities/asset.entity.dart'; @@ -12,6 +13,7 @@ import 'package:immich_mobile/pages/common/large_leading_tile.dart'; import 'package:immich_mobile/providers/search/search_page_state.provider.dart'; import 'package:immich_mobile/routing/router.dart'; import 'package:immich_mobile/services/api.service.dart'; +import 'package:immich_mobile/widgets/common/search_field.dart'; import 'package:immich_mobile/widgets/map/map_thumbnail.dart'; import 'package:maplibre_gl/maplibre_gl.dart'; @@ -21,34 +23,62 @@ class PlacesCollectionPage extends HookConsumerWidget { @override Widget build(BuildContext context, WidgetRef ref) { final places = ref.watch(getAllPlacesProvider); + final formFocus = useFocusNode(); + final ValueNotifier search = useState(null); return Scaffold( appBar: AppBar( - title: Text('places'.tr()), + automaticallyImplyLeading: search.value == null, + title: search.value != null + ? SearchField( + autofocus: true, + filled: true, + focusNode: formFocus, + onChanged: (value) => search.value = value, + onTapOutside: (_) => formFocus.unfocus(), + hintText: 'filter_places'.tr(), + ) + : Text('places'.tr()), + actions: [ + IconButton( + icon: Icon(search.value != null ? Icons.close : Icons.search), + onPressed: () { + search.value = search.value == null ? '' : null; + }, + ), + ], ), body: ListView( shrinkWrap: true, children: [ - Padding( - padding: const EdgeInsets.all(16.0), - child: SizedBox( - height: 200, - width: context.width, - child: MapThumbnail( - onTap: (_, __) => context.pushRoute(const MapRoute()), - zoom: 8, - centre: const LatLng( - 21.44950, - -157.91959, + if (search.value == null) + Padding( + padding: const EdgeInsets.all(16.0), + child: SizedBox( + height: 200, + width: context.width, + child: MapThumbnail( + onTap: (_, __) => context.pushRoute(const MapRoute()), + zoom: 8, + centre: const LatLng( + 21.44950, + -157.91959, + ), + showAttribution: false, + themeMode: + context.isDarkTheme ? ThemeMode.dark : ThemeMode.light, ), - showAttribution: false, - themeMode: - context.isDarkTheme ? ThemeMode.dark : ThemeMode.light, ), ), - ), places.when( data: (places) { + if (search.value != null) { + places = places.where((place) { + return place.label + .toLowerCase() + .contains(search.value!.toLowerCase()); + }).toList(); + } return ListView.builder( shrinkWrap: true, physics: const NeverScrollableScrollPhysics(), From 0e6ac876450c6e6ba1072cf644abf395912e0fef Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 18 Apr 2025 14:01:16 -0500 Subject: [PATCH 08/29] feat(mobile): assets + exif stream sync placeholder (#17677) * feat(mobile): assets + exif stream sync placeholder * feat(mobile): assets + exif stream sync placeholder * refactor * fix: test * fix:test * refactor(mobile): sync stream service (#17687) * refactor: sync stream to use callbacks * pr feedback * pr feedback * pr feedback * fix: test --------- Co-authored-by: shenlong-tanwen <139912620+shalong-tanwen@users.noreply.github.com> Co-authored-by: Alex Tran --------- Co-authored-by: shenlong <139912620+shenlong-tanwen@users.noreply.github.com> Co-authored-by: shenlong-tanwen <139912620+shalong-tanwen@users.noreply.github.com> --- .../domain/interfaces/sync_api.interface.dart | 8 +- .../interfaces/sync_stream.interface.dart | 16 +- .../domain/services/sync_stream.service.dart | 228 +++------ mobile/lib/domain/utils/background_sync.dart | 28 +- .../repositories/sync_api.repository.dart | 75 ++- .../repositories/sync_stream.repository.dart | 62 ++- mobile/lib/widgets/common/immich_app_bar.dart | 7 + mobile/test/api.mocks.dart | 2 + .../services/sync_stream_service_test.dart | 475 +++++------------- mobile/test/fixtures/sync_stream.stub.dart | 74 +-- .../sync_api_repository_test.dart | 299 +++++++++++ 11 files changed, 666 insertions(+), 608 deletions(-) create mode 100644 mobile/test/infrastructure/repositories/sync_api_repository_test.dart diff --git a/mobile/lib/domain/interfaces/sync_api.interface.dart b/mobile/lib/domain/interfaces/sync_api.interface.dart index 44e22c5894..57abed2e7f 100644 --- a/mobile/lib/domain/interfaces/sync_api.interface.dart +++ b/mobile/lib/domain/interfaces/sync_api.interface.dart @@ -1,8 +1,12 @@ +import 'package:http/http.dart' as http; import 'package:immich_mobile/domain/models/sync_event.model.dart'; -import 'package:openapi/api.dart'; abstract interface class ISyncApiRepository { Future ack(List data); - Stream> getSyncEvents(List type); + Future streamChanges( + Function(List, Function() abort) onData, { + int batchSize, + http.Client? httpClient, + }); } diff --git a/mobile/lib/domain/interfaces/sync_stream.interface.dart b/mobile/lib/domain/interfaces/sync_stream.interface.dart index f9c52d7ee0..5f61d6b52f 100644 --- a/mobile/lib/domain/interfaces/sync_stream.interface.dart +++ b/mobile/lib/domain/interfaces/sync_stream.interface.dart @@ -2,9 +2,17 @@ import 'package:immich_mobile/domain/interfaces/db.interface.dart'; import 'package:openapi/api.dart'; abstract interface class ISyncStreamRepository implements IDatabaseRepository { - Future updateUsersV1(Iterable data); - Future deleteUsersV1(Iterable data); + Future updateUsersV1(Iterable data); + Future deleteUsersV1(Iterable data); - Future updatePartnerV1(Iterable data); - Future deletePartnerV1(Iterable data); + Future updatePartnerV1(Iterable data); + Future deletePartnerV1(Iterable data); + + Future updateAssetsV1(Iterable data); + Future deleteAssetsV1(Iterable data); + Future updateAssetsExifV1(Iterable data); + + Future updatePartnerAssetsV1(Iterable data); + Future deletePartnerAssetsV1(Iterable data); + Future updatePartnerAssetsExifV1(Iterable data); } diff --git a/mobile/lib/domain/services/sync_stream.service.dart b/mobile/lib/domain/services/sync_stream.service.dart index 8d7d87e35e..ac63734b07 100644 --- a/mobile/lib/domain/services/sync_stream.service.dart +++ b/mobile/lib/domain/services/sync_stream.service.dart @@ -2,25 +2,11 @@ import 'dart:async'; -import 'package:collection/collection.dart'; import 'package:immich_mobile/domain/interfaces/sync_api.interface.dart'; import 'package:immich_mobile/domain/interfaces/sync_stream.interface.dart'; +import 'package:immich_mobile/domain/models/sync_event.model.dart'; import 'package:logging/logging.dart'; import 'package:openapi/api.dart'; -import 'package:worker_manager/worker_manager.dart'; - -const _kSyncTypeOrder = [ - SyncEntityType.userDeleteV1, - SyncEntityType.userV1, - SyncEntityType.partnerDeleteV1, - SyncEntityType.partnerV1, - SyncEntityType.assetDeleteV1, - SyncEntityType.assetV1, - SyncEntityType.assetExifV1, - SyncEntityType.partnerAssetDeleteV1, - SyncEntityType.partnerAssetV1, - SyncEntityType.partnerAssetExifV1, -]; class SyncStreamService { final Logger _logger = Logger('SyncStreamService'); @@ -37,164 +23,70 @@ class SyncStreamService { _syncStreamRepository = syncStreamRepository, _cancelChecker = cancelChecker; - Future _handleSyncData( + bool get isCancelled => _cancelChecker?.call() ?? false; + + Future sync() => _syncApiRepository.streamChanges(_handleEvents); + + Future _handleEvents(List events, Function() abort) async { + List items = []; + for (final event in events) { + if (isCancelled) { + _logger.warning("Sync stream cancelled"); + abort(); + return; + } + + if (event.type != items.firstOrNull?.type) { + await _processBatch(items); + } + + items.add(event); + } + + await _processBatch(items); + } + + Future _processBatch(List batch) async { + if (batch.isEmpty) { + return; + } + + final type = batch.first.type; + await _handleSyncData(type, batch.map((e) => e.data)); + await _syncApiRepository.ack([batch.last.ack]); + batch.clear(); + } + + Future _handleSyncData( SyncEntityType type, // ignore: avoid-dynamic Iterable data, ) async { - if (data.isEmpty) { - _logger.warning("Received empty sync data for $type"); - return false; - } - _logger.fine("Processing sync data for $type of length ${data.length}"); - - try { - if (type == SyncEntityType.partnerV1) { - return await _syncStreamRepository.updatePartnerV1(data.cast()); - } - - if (type == SyncEntityType.partnerDeleteV1) { - return await _syncStreamRepository.deletePartnerV1(data.cast()); - } - - if (type == SyncEntityType.userV1) { - return await _syncStreamRepository.updateUsersV1(data.cast()); - } - - if (type == SyncEntityType.userDeleteV1) { - return await _syncStreamRepository.deleteUsersV1(data.cast()); - } - } catch (error, stack) { - _logger.severe("Error processing sync data for $type", error, stack); - return false; + // ignore: prefer-switch-expression + switch (type) { + case SyncEntityType.userV1: + return _syncStreamRepository.updateUsersV1(data.cast()); + case SyncEntityType.userDeleteV1: + return _syncStreamRepository.deleteUsersV1(data.cast()); + case SyncEntityType.partnerV1: + return _syncStreamRepository.updatePartnerV1(data.cast()); + case SyncEntityType.partnerDeleteV1: + return _syncStreamRepository.deletePartnerV1(data.cast()); + case SyncEntityType.assetV1: + return _syncStreamRepository.updateAssetsV1(data.cast()); + case SyncEntityType.assetDeleteV1: + return _syncStreamRepository.deleteAssetsV1(data.cast()); + case SyncEntityType.assetExifV1: + return _syncStreamRepository.updateAssetsExifV1(data.cast()); + case SyncEntityType.partnerAssetV1: + return _syncStreamRepository.updatePartnerAssetsV1(data.cast()); + case SyncEntityType.partnerAssetDeleteV1: + return _syncStreamRepository.deletePartnerAssetsV1(data.cast()); + case SyncEntityType.partnerAssetExifV1: + return _syncStreamRepository.updatePartnerAssetsExifV1(data.cast()); + default: + _logger.warning("Unknown sync data type: $type"); } - - _logger.warning("Unknown sync data type: $type"); - return false; } - - Future _syncEvent(List types) { - _logger.info("Syncing Events: $types"); - final streamCompleter = Completer(); - bool shouldComplete = false; - // the onDone callback might fire before the events are processed - // the following flag ensures that the onDone callback is not called - // before the events are processed and also that events are processed sequentially - Completer? mutex; - StreamSubscription? subscription; - try { - subscription = _syncApiRepository.getSyncEvents(types).listen( - (events) async { - if (events.isEmpty) { - _logger.warning("Received empty sync events"); - return; - } - - // If previous events are still being processed, wait for them to finish - if (mutex != null) { - await mutex!.future; - } - - if (_cancelChecker?.call() ?? false) { - _logger.info("Sync cancelled, stopping stream"); - subscription?.cancel(); - if (!streamCompleter.isCompleted) { - streamCompleter.completeError( - CanceledError(), - StackTrace.current, - ); - } - return; - } - - // Take control of the mutex and process the events - mutex = Completer(); - - try { - final eventsMap = events.groupListsBy((event) => event.type); - final Map acks = {}; - - for (final type in _kSyncTypeOrder) { - final data = eventsMap[type]; - if (data == null) { - continue; - } - - if (_cancelChecker?.call() ?? false) { - _logger.info("Sync cancelled, stopping stream"); - mutex?.complete(); - mutex = null; - if (!streamCompleter.isCompleted) { - streamCompleter.completeError( - CanceledError(), - StackTrace.current, - ); - } - - return; - } - - if (data.isEmpty) { - _logger.warning("Received empty sync events for $type"); - continue; - } - - if (await _handleSyncData(type, data.map((e) => e.data))) { - // ignore: avoid-unsafe-collection-methods - acks[type] = data.last.ack; - } else { - _logger.warning("Failed to handle sync events for $type"); - } - } - - if (acks.isNotEmpty) { - await _syncApiRepository.ack(acks.values.toList()); - } - _logger.info("$types events processed"); - } catch (error, stack) { - _logger.warning("Error handling sync events", error, stack); - } finally { - mutex?.complete(); - mutex = null; - } - - if (shouldComplete) { - _logger.info("Sync done, completing stream"); - if (!streamCompleter.isCompleted) streamCompleter.complete(); - } - }, - onError: (error, stack) { - _logger.warning("Error in sync stream for $types", error, stack); - // Do not proceed if the stream errors - if (!streamCompleter.isCompleted) { - // ignore: avoid-missing-completer-stack-trace - streamCompleter.completeError(error, stack); - } - }, - onDone: () { - _logger.info("$types stream done"); - if (mutex == null && !streamCompleter.isCompleted) { - streamCompleter.complete(); - } else { - // Marks the stream as done but does not complete the completer - // until the events are processed - shouldComplete = true; - } - }, - ); - } catch (error, stack) { - _logger.severe("Error starting sync stream", error, stack); - if (!streamCompleter.isCompleted) { - streamCompleter.completeError(error, stack); - } - } - return streamCompleter.future.whenComplete(() { - _logger.info("Sync stream completed"); - return subscription?.cancel(); - }); - } - - Future syncUsers() => - _syncEvent([SyncRequestType.usersV1, SyncRequestType.partnersV1]); } diff --git a/mobile/lib/domain/utils/background_sync.dart b/mobile/lib/domain/utils/background_sync.dart index 0bd456f0bb..f63dc81ba9 100644 --- a/mobile/lib/domain/utils/background_sync.dart +++ b/mobile/lib/domain/utils/background_sync.dart @@ -7,31 +7,33 @@ import 'package:immich_mobile/utils/isolate.dart'; import 'package:worker_manager/worker_manager.dart'; class BackgroundSyncManager { - Cancelable? _userSyncTask; + Cancelable? _syncTask; BackgroundSyncManager(); Future cancel() { final futures = []; - if (_userSyncTask != null) { - futures.add(_userSyncTask!.future); + + if (_syncTask != null) { + futures.add(_syncTask!.future); } - _userSyncTask?.cancel(); - _userSyncTask = null; + _syncTask?.cancel(); + _syncTask = null; + return Future.wait(futures); } - Future syncUsers() { - if (_userSyncTask != null) { - return _userSyncTask!.future; + Future sync() { + if (_syncTask != null) { + return _syncTask!.future; } - _userSyncTask = runInIsolateGentle( - computation: (ref) => ref.read(syncStreamServiceProvider).syncUsers(), + _syncTask = runInIsolateGentle( + computation: (ref) => ref.read(syncStreamServiceProvider).sync(), ); - _userSyncTask!.whenComplete(() { - _userSyncTask = null; + _syncTask!.whenComplete(() { + _syncTask = null; }); - return _userSyncTask!.future; + return _syncTask!.future; } } diff --git a/mobile/lib/infrastructure/repositories/sync_api.repository.dart b/mobile/lib/infrastructure/repositories/sync_api.repository.dart index a26b867df6..dd1ea208ba 100644 --- a/mobile/lib/infrastructure/repositories/sync_api.repository.dart +++ b/mobile/lib/infrastructure/repositories/sync_api.repository.dart @@ -12,22 +12,22 @@ import 'package:openapi/api.dart'; class SyncApiRepository implements ISyncApiRepository { final Logger _logger = Logger('SyncApiRepository'); final ApiService _api; - final int _batchSize; - SyncApiRepository(this._api, {int batchSize = kSyncEventBatchSize}) - : _batchSize = batchSize; - - @override - Stream> getSyncEvents(List type) { - return _getSyncStream(SyncStreamDto(types: type)); - } + SyncApiRepository(this._api); @override Future ack(List data) { return _api.syncApi.sendSyncAck(SyncAckSetDto(acks: data)); } - Stream> _getSyncStream(SyncStreamDto dto) async* { - final client = http.Client(); + @override + Future streamChanges( + Function(List, Function() abort) onData, { + int batchSize = kSyncEventBatchSize, + http.Client? httpClient, + }) async { + // ignore: avoid-unused-assignment + final stopwatch = Stopwatch()..start(); + final client = httpClient ?? http.Client(); final endpoint = "${_api.apiClient.basePath}/sync/stream"; final headers = { @@ -35,20 +35,38 @@ class SyncApiRepository implements ISyncApiRepository { 'Accept': 'application/jsonlines+json', }; - final queryParams = []; final headerParams = {}; - await _api.applyToParams(queryParams, headerParams); + await _api.applyToParams([], headerParams); headers.addAll(headerParams); final request = http.Request('POST', Uri.parse(endpoint)); request.headers.addAll(headers); - request.body = jsonEncode(dto.toJson()); + request.body = jsonEncode( + SyncStreamDto( + types: [ + SyncRequestType.usersV1, + SyncRequestType.partnersV1, + SyncRequestType.assetsV1, + SyncRequestType.partnerAssetsV1, + SyncRequestType.assetExifsV1, + SyncRequestType.partnerAssetExifsV1, + ], + ).toJson(), + ); String previousChunk = ''; List lines = []; + bool shouldAbort = false; + + void abort() { + _logger.warning("Abort requested, stopping sync stream"); + shouldAbort = true; + } + try { - final response = await client.send(request); + final response = + await client.send(request).timeout(const Duration(seconds: 20)); if (response.statusCode != 200) { final errorBody = await response.stream.bytesToString(); @@ -59,27 +77,38 @@ class SyncApiRepository implements ISyncApiRepository { } await for (final chunk in response.stream.transform(utf8.decoder)) { + if (shouldAbort) { + break; + } + previousChunk += chunk; final parts = previousChunk.toString().split('\n'); previousChunk = parts.removeLast(); lines.addAll(parts); - if (lines.length < _batchSize) { + if (lines.length < batchSize) { continue; } - yield _parseSyncResponse(lines); + await onData(_parseLines(lines), abort); lines.clear(); } - } finally { - if (lines.isNotEmpty) { - yield _parseSyncResponse(lines); + + if (lines.isNotEmpty && !shouldAbort) { + await onData(_parseLines(lines), abort); } + } catch (error, stack) { + _logger.severe("error processing stream", error, stack); + return Future.error(error, stack); + } finally { client.close(); } + stopwatch.stop(); + _logger + .info("Remote Sync completed in ${stopwatch.elapsed.inMilliseconds}ms"); } - List _parseSyncResponse(List lines) { + List _parseLines(List lines) { final List data = []; for (final line in lines) { @@ -110,4 +139,10 @@ const _kResponseMap = { SyncEntityType.userDeleteV1: SyncUserDeleteV1.fromJson, SyncEntityType.partnerV1: SyncPartnerV1.fromJson, SyncEntityType.partnerDeleteV1: SyncPartnerDeleteV1.fromJson, + SyncEntityType.assetV1: SyncAssetV1.fromJson, + SyncEntityType.assetDeleteV1: SyncAssetDeleteV1.fromJson, + SyncEntityType.assetExifV1: SyncAssetExifV1.fromJson, + SyncEntityType.partnerAssetV1: SyncAssetV1.fromJson, + SyncEntityType.partnerAssetDeleteV1: SyncAssetDeleteV1.fromJson, + SyncEntityType.partnerAssetExifV1: SyncAssetExifV1.fromJson, }; diff --git a/mobile/lib/infrastructure/repositories/sync_stream.repository.dart b/mobile/lib/infrastructure/repositories/sync_stream.repository.dart index a947a9a66b..5ad9a369df 100644 --- a/mobile/lib/infrastructure/repositories/sync_stream.repository.dart +++ b/mobile/lib/infrastructure/repositories/sync_stream.repository.dart @@ -1,4 +1,5 @@ import 'package:drift/drift.dart'; +import 'package:flutter/foundation.dart'; import 'package:immich_mobile/domain/interfaces/sync_stream.interface.dart'; import 'package:immich_mobile/extensions/string_extensions.dart'; import 'package:immich_mobile/infrastructure/entities/partner.entity.drift.dart'; @@ -15,7 +16,7 @@ class DriftSyncStreamRepository extends DriftDatabaseRepository DriftSyncStreamRepository(super.db) : _db = db; @override - Future deleteUsersV1(Iterable data) async { + Future deleteUsersV1(Iterable data) async { try { await _db.batch((batch) { for (final user in data) { @@ -25,15 +26,14 @@ class DriftSyncStreamRepository extends DriftDatabaseRepository ); } }); - return true; - } catch (e, s) { - _logger.severe('Error while processing SyncUserDeleteV1', e, s); - return false; + } catch (error, stack) { + _logger.severe('Error while processing SyncUserDeleteV1', error, stack); + rethrow; } } @override - Future updateUsersV1(Iterable data) async { + Future updateUsersV1(Iterable data) async { try { await _db.batch((batch) { for (final user in data) { @@ -49,15 +49,14 @@ class DriftSyncStreamRepository extends DriftDatabaseRepository ); } }); - return true; - } catch (e, s) { - _logger.severe('Error while processing SyncUserV1', e, s); - return false; + } catch (error, stack) { + _logger.severe('Error while processing SyncUserV1', error, stack); + rethrow; } } @override - Future deletePartnerV1(Iterable data) async { + Future deletePartnerV1(Iterable data) async { try { await _db.batch((batch) { for (final partner in data) { @@ -70,15 +69,14 @@ class DriftSyncStreamRepository extends DriftDatabaseRepository ); } }); - return true; } catch (e, s) { _logger.severe('Error while processing SyncPartnerDeleteV1', e, s); - return false; + rethrow; } } @override - Future updatePartnerV1(Iterable data) async { + Future updatePartnerV1(Iterable data) async { try { await _db.batch((batch) { for (final partner in data) { @@ -95,10 +93,42 @@ class DriftSyncStreamRepository extends DriftDatabaseRepository ); } }); - return true; } catch (e, s) { _logger.severe('Error while processing SyncPartnerV1', e, s); - return false; + rethrow; } } + + // Assets + @override + Future updateAssetsV1(Iterable data) async { + debugPrint("updateAssetsV1 - ${data.length}"); + } + + @override + Future deleteAssetsV1(Iterable data) async { + debugPrint("deleteAssetsV1 - ${data.length}"); + } + + // Partner Assets + @override + Future updatePartnerAssetsV1(Iterable data) async { + debugPrint("updatePartnerAssetsV1 - ${data.length}"); + } + + @override + Future deletePartnerAssetsV1(Iterable data) async { + debugPrint("deletePartnerAssetsV1 - ${data.length}"); + } + + // EXIF + @override + Future updateAssetsExifV1(Iterable data) async { + debugPrint("updateAssetsExifV1 - ${data.length}"); + } + + @override + Future updatePartnerAssetsExifV1(Iterable data) async { + debugPrint("updatePartnerAssetsExifV1 - ${data.length}"); + } } diff --git a/mobile/lib/widgets/common/immich_app_bar.dart b/mobile/lib/widgets/common/immich_app_bar.dart index 51b4faa014..4f95e657d9 100644 --- a/mobile/lib/widgets/common/immich_app_bar.dart +++ b/mobile/lib/widgets/common/immich_app_bar.dart @@ -1,11 +1,13 @@ import 'package:auto_route/auto_route.dart'; import 'package:easy_localization/easy_localization.dart'; +import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; import 'package:flutter_svg/svg.dart'; import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/extensions/build_context_extensions.dart'; import 'package:immich_mobile/models/backup/backup_state.model.dart'; import 'package:immich_mobile/models/server_info/server_info.model.dart'; +import 'package:immich_mobile/providers/background_sync.provider.dart'; import 'package:immich_mobile/providers/backup/backup.provider.dart'; import 'package:immich_mobile/providers/server_info.provider.dart'; import 'package:immich_mobile/providers/user.provider.dart'; @@ -178,6 +180,11 @@ class ImmichAppBar extends ConsumerWidget implements PreferredSizeWidget { child: action, ), ), + if (kDebugMode) + IconButton( + onPressed: () => ref.read(backgroundSyncProvider).sync(), + icon: const Icon(Icons.sync), + ), if (showUploadButton) Padding( padding: const EdgeInsets.only(right: 20), diff --git a/mobile/test/api.mocks.dart b/mobile/test/api.mocks.dart index d502ea0675..b0a4e9b8fd 100644 --- a/mobile/test/api.mocks.dart +++ b/mobile/test/api.mocks.dart @@ -2,3 +2,5 @@ import 'package:mocktail/mocktail.dart'; import 'package:openapi/api.dart'; class MockAssetsApi extends Mock implements AssetsApi {} + +class MockSyncApi extends Mock implements SyncApi {} diff --git a/mobile/test/domain/services/sync_stream_service_test.dart b/mobile/test/domain/services/sync_stream_service_test.dart index e1d8e6987f..b78a44342b 100644 --- a/mobile/test/domain/services/sync_stream_service_test.dart +++ b/mobile/test/domain/services/sync_stream_service_test.dart @@ -1,4 +1,4 @@ -// ignore_for_file: avoid-unnecessary-futures, avoid-async-call-in-sync-function +// ignore_for_file: avoid-declaring-call-method, avoid-unnecessary-futures import 'dart:async'; @@ -8,16 +8,22 @@ import 'package:immich_mobile/domain/interfaces/sync_stream.interface.dart'; import 'package:immich_mobile/domain/models/sync_event.model.dart'; import 'package:immich_mobile/domain/services/sync_stream.service.dart'; import 'package:mocktail/mocktail.dart'; -import 'package:openapi/api.dart'; -import 'package:worker_manager/worker_manager.dart'; import '../../fixtures/sync_stream.stub.dart'; import '../../infrastructure/repository.mock.dart'; +class _AbortCallbackWrapper { + const _AbortCallbackWrapper(); + + bool call() => false; +} + +class _MockAbortCallbackWrapper extends Mock implements _AbortCallbackWrapper {} + class _CancellationWrapper { const _CancellationWrapper(); - bool isCancelled() => false; + bool call() => false; } class _MockCancellationWrapper extends Mock implements _CancellationWrapper {} @@ -26,35 +32,26 @@ void main() { late SyncStreamService sut; late ISyncStreamRepository mockSyncStreamRepo; late ISyncApiRepository mockSyncApiRepo; - late StreamController> streamController; + late Function(List, Function()) handleEventsCallback; + late _MockAbortCallbackWrapper mockAbortCallbackWrapper; successHandler(Invocation _) async => true; - failureHandler(Invocation _) async => false; setUp(() { mockSyncStreamRepo = MockSyncStreamRepository(); mockSyncApiRepo = MockSyncApiRepository(); - streamController = StreamController>.broadcast(); + mockAbortCallbackWrapper = _MockAbortCallbackWrapper(); - sut = SyncStreamService( - syncApiRepository: mockSyncApiRepo, - syncStreamRepository: mockSyncStreamRepo, - ); + when(() => mockAbortCallbackWrapper()).thenReturn(false); - // Default stream setup - emits one batch and closes - when(() => mockSyncApiRepo.getSyncEvents(any())) - .thenAnswer((_) => streamController.stream); + when(() => mockSyncApiRepo.streamChanges(any())) + .thenAnswer((invocation) async { + // ignore: avoid-unsafe-collection-methods + handleEventsCallback = invocation.positionalArguments.first; + }); - // Default ack setup when(() => mockSyncApiRepo.ack(any())).thenAnswer((_) async => {}); - // Register fallbacks for mocktail verification - registerFallbackValue([]); - registerFallbackValue([]); - registerFallbackValue([]); - registerFallbackValue([]); - - // Default successful repository calls when(() => mockSyncStreamRepo.updateUsersV1(any())) .thenAnswer(successHandler); when(() => mockSyncStreamRepo.deleteUsersV1(any())) @@ -63,381 +60,163 @@ void main() { .thenAnswer(successHandler); when(() => mockSyncStreamRepo.deletePartnerV1(any())) .thenAnswer(successHandler); + when(() => mockSyncStreamRepo.updateAssetsV1(any())) + .thenAnswer(successHandler); + when(() => mockSyncStreamRepo.deleteAssetsV1(any())) + .thenAnswer(successHandler); + when(() => mockSyncStreamRepo.updateAssetsExifV1(any())) + .thenAnswer(successHandler); + when(() => mockSyncStreamRepo.updatePartnerAssetsV1(any())) + .thenAnswer(successHandler); + when(() => mockSyncStreamRepo.deletePartnerAssetsV1(any())) + .thenAnswer(successHandler); + when(() => mockSyncStreamRepo.updatePartnerAssetsExifV1(any())) + .thenAnswer(successHandler); + + sut = SyncStreamService( + syncApiRepository: mockSyncApiRepo, + syncStreamRepository: mockSyncStreamRepo, + ); }); - tearDown(() async { - if (!streamController.isClosed) { - await streamController.close(); - } - }); - - // Helper to trigger sync and add events to the stream - Future triggerSyncAndEmit(List events) async { - final future = sut.syncUsers(); // Start listening - await Future.delayed(Duration.zero); // Allow listener to attach - if (!streamController.isClosed) { - streamController.add(events); - await streamController.close(); // Close after emitting - } - await future; // Wait for processing to complete + Future simulateEvents(List events) async { + await sut.sync(); + await handleEventsCallback(events, mockAbortCallbackWrapper.call); } - group("SyncStreamService", () { + group("SyncStreamService - _handleEvents", () { test( - "completes successfully when stream emits data and handlers succeed", + "processes events and acks successfully when handlers succeed", () async { final events = [ - ...SyncStreamStub.userEvents, - ...SyncStreamStub.partnerEvents, + SyncStreamStub.userDeleteV1, + SyncStreamStub.userV1Admin, + SyncStreamStub.userV1User, + SyncStreamStub.partnerDeleteV1, + SyncStreamStub.partnerV1, ]; - final future = triggerSyncAndEmit(events); - await expectLater(future, completes); - // Verify ack includes last ack from each successfully handled type - verify( - () => - mockSyncApiRepo.ack(any(that: containsAll(["5", "2", "4", "3"]))), - ).called(1); + + await simulateEvents(events); + + verifyInOrder([ + () => mockSyncStreamRepo.deleteUsersV1(any()), + () => mockSyncApiRepo.ack(["2"]), + () => mockSyncStreamRepo.updateUsersV1(any()), + () => mockSyncApiRepo.ack(["5"]), + () => mockSyncStreamRepo.deletePartnerV1(any()), + () => mockSyncApiRepo.ack(["4"]), + () => mockSyncStreamRepo.updatePartnerV1(any()), + () => mockSyncApiRepo.ack(["3"]), + ]); + verifyNever(() => mockAbortCallbackWrapper()); }, ); - test("completes successfully when stream emits an error", () async { - when(() => mockSyncApiRepo.getSyncEvents(any())) - .thenAnswer((_) => Stream.error(Exception("Stream Error"))); - // Should complete gracefully without throwing - await expectLater(sut.syncUsers(), throwsException); - verifyNever(() => mockSyncApiRepo.ack(any())); // No ack on stream error - }); - - test("throws when initial getSyncEvents call fails", () async { - final apiException = Exception("API Error"); - when(() => mockSyncApiRepo.getSyncEvents(any())).thenThrow(apiException); - // Should rethrow the exception from the initial call - await expectLater(sut.syncUsers(), throwsA(apiException)); - verifyNever(() => mockSyncApiRepo.ack(any())); - }); - - test( - "completes successfully when a repository handler throws an exception", - () async { - when(() => mockSyncStreamRepo.updateUsersV1(any())) - .thenThrow(Exception("Repo Error")); - final events = [ - ...SyncStreamStub.userEvents, - ...SyncStreamStub.partnerEvents, - ]; - // Should complete, but ack only for the successful types - await triggerSyncAndEmit(events); - // Only partner delete was successful by default setup - verify(() => mockSyncApiRepo.ack(["2", "4", "3"])).called(1); - }, - ); - - test( - "completes successfully but sends no ack when all handlers fail", - () async { - when(() => mockSyncStreamRepo.updateUsersV1(any())) - .thenAnswer(failureHandler); - when(() => mockSyncStreamRepo.deleteUsersV1(any())) - .thenAnswer(failureHandler); - when(() => mockSyncStreamRepo.updatePartnerV1(any())) - .thenAnswer(failureHandler); - when(() => mockSyncStreamRepo.deletePartnerV1(any())) - .thenAnswer(failureHandler); - - final events = [ - ...SyncStreamStub.userEvents, - ...SyncStreamStub.partnerEvents, - ]; - await triggerSyncAndEmit(events); - verifyNever(() => mockSyncApiRepo.ack(any())); - }, - ); - - test("sends ack only for types where handler returns true", () async { - // Mock specific handlers: user update fails, user delete succeeds - when(() => mockSyncStreamRepo.updateUsersV1(any())) - .thenAnswer(failureHandler); - when(() => mockSyncStreamRepo.deleteUsersV1(any())) - .thenAnswer(successHandler); - // partner update fails, partner delete succeeds - when(() => mockSyncStreamRepo.updatePartnerV1(any())) - .thenAnswer(failureHandler); - + test("processes final batch correctly", () async { final events = [ - ...SyncStreamStub.userEvents, - ...SyncStreamStub.partnerEvents, + SyncStreamStub.userDeleteV1, + SyncStreamStub.userV1Admin, ]; - await triggerSyncAndEmit(events); - // Expect ack only for userDeleteV1 (ack: "2") and partnerDeleteV1 (ack: "4") - verify(() => mockSyncApiRepo.ack(any(that: containsAll(["2", "4"])))) - .called(1); + await simulateEvents(events); + + verifyInOrder([ + () => mockSyncStreamRepo.deleteUsersV1(any()), + () => mockSyncApiRepo.ack(["2"]), + () => mockSyncStreamRepo.updateUsersV1(any()), + () => mockSyncApiRepo.ack(["1"]), + ]); + verifyNever(() => mockAbortCallbackWrapper()); }); - test("does not process or ack when stream emits an empty list", () async { - final future = sut.syncUsers(); - streamController.add([]); // Emit empty list - await streamController.close(); - await future; // Wait for completion + test("does not process or ack when event list is empty", () async { + await simulateEvents([]); verifyNever(() => mockSyncStreamRepo.updateUsersV1(any())); verifyNever(() => mockSyncStreamRepo.deleteUsersV1(any())); verifyNever(() => mockSyncStreamRepo.updatePartnerV1(any())); verifyNever(() => mockSyncStreamRepo.deletePartnerV1(any())); + verifyNever(() => mockAbortCallbackWrapper()); verifyNever(() => mockSyncApiRepo.ack(any())); }); - test("processes multiple batches sequentially using mutex", () async { - final completer1 = Completer(); - final completer2 = Completer(); - int callOrder = 0; - int handler1StartOrder = -1; - int handler2StartOrder = -1; - int handler1Calls = 0; - int handler2Calls = 0; + test("aborts and stops processing if cancelled during iteration", () async { + final cancellationChecker = _MockCancellationWrapper(); + when(() => cancellationChecker()).thenReturn(false); - when(() => mockSyncStreamRepo.updateUsersV1(any())).thenAnswer((_) async { - handler1Calls++; - handler1StartOrder = ++callOrder; - await completer1.future; - return true; - }); - when(() => mockSyncStreamRepo.updatePartnerV1(any())) - .thenAnswer((_) async { - handler2Calls++; - handler2StartOrder = ++callOrder; - await completer2.future; - return true; + sut = SyncStreamService( + syncApiRepository: mockSyncApiRepo, + syncStreamRepository: mockSyncStreamRepo, + cancelChecker: cancellationChecker.call, + ); + await sut.sync(); + + final events = [ + SyncStreamStub.userDeleteV1, + SyncStreamStub.userV1Admin, + SyncStreamStub.partnerDeleteV1, + ]; + + when(() => mockSyncStreamRepo.deleteUsersV1(any())).thenAnswer((_) async { + when(() => cancellationChecker()).thenReturn(true); }); - final batch1 = SyncStreamStub.userEvents; - final batch2 = SyncStreamStub.partnerEvents; + await handleEventsCallback(events, mockAbortCallbackWrapper.call); - final syncFuture = sut.syncUsers(); - await pumpEventQueue(); + verify(() => mockSyncStreamRepo.deleteUsersV1(any())).called(1); + verifyNever(() => mockSyncStreamRepo.updateUsersV1(any())); + verifyNever(() => mockSyncStreamRepo.deletePartnerV1(any())); - streamController.add(batch1); - await pumpEventQueue(); - // Small delay to ensure the first handler starts - await Future.delayed(const Duration(milliseconds: 20)); + verify(() => mockAbortCallbackWrapper()).called(1); - expect(handler1StartOrder, 1, reason: "Handler 1 should start first"); - expect(handler1Calls, 1); - - streamController.add(batch2); - await pumpEventQueue(); - // Small delay - await Future.delayed(const Duration(milliseconds: 20)); - - expect(handler2StartOrder, -1, reason: "Handler 2 should wait"); - expect(handler2Calls, 0); - - completer1.complete(); - await pumpEventQueue(times: 40); - // Small delay to ensure the second handler starts - await Future.delayed(const Duration(milliseconds: 20)); - - expect(handler2StartOrder, 2, reason: "Handler 2 should start after H1"); - expect(handler2Calls, 1); - - completer2.complete(); - await pumpEventQueue(times: 40); - // Small delay before closing the stream - await Future.delayed(const Duration(milliseconds: 20)); - - if (!streamController.isClosed) { - await streamController.close(); - } - await pumpEventQueue(times: 40); - // Small delay to ensure the sync completes - await Future.delayed(const Duration(milliseconds: 20)); - - await syncFuture; - - verify(() => mockSyncStreamRepo.updateUsersV1(any())).called(1); - verify(() => mockSyncStreamRepo.updatePartnerV1(any())).called(1); - verify(() => mockSyncApiRepo.ack(any())).called(2); + verify(() => mockSyncApiRepo.ack(["2"])).called(1); }); test( - "stops processing and ack when cancel checker is completed", + "aborts and stops processing if cancelled before processing batch", () async { final cancellationChecker = _MockCancellationWrapper(); - when(() => cancellationChecker.isCancelled()).thenAnswer((_) => false); + when(() => cancellationChecker()).thenReturn(false); + + final processingCompleter = Completer(); + bool handler1Started = false; + when(() => mockSyncStreamRepo.deleteUsersV1(any())) + .thenAnswer((_) async { + handler1Started = true; + return processingCompleter.future; + }); sut = SyncStreamService( syncApiRepository: mockSyncApiRepo, syncStreamRepository: mockSyncStreamRepo, - cancelChecker: cancellationChecker.isCancelled, + cancelChecker: cancellationChecker.call, ); - final processingCompleter = Completer(); - bool handlerStarted = false; + await sut.sync(); - // Make handler wait so we can cancel it mid-flight - when(() => mockSyncStreamRepo.deleteUsersV1(any())) - .thenAnswer((_) async { - handlerStarted = true; - await processingCompleter - .future; // Wait indefinitely until test completes it - return true; - }); - - final syncFuture = sut.syncUsers(); - await pumpEventQueue(times: 30); - - streamController.add(SyncStreamStub.userEvents); - // Ensure processing starts - await Future.delayed(const Duration(milliseconds: 10)); - - expect(handlerStarted, isTrue, reason: "Handler should have started"); - - when(() => cancellationChecker.isCancelled()).thenAnswer((_) => true); - - // Allow cancellation logic to propagate - await Future.delayed(const Duration(milliseconds: 10)); - - // Complete the handler's completer after cancellation signal - // to ensure the cancellation logic itself isn't blocked by the handler. - processingCompleter.complete(); - - await expectLater(syncFuture, throwsA(isA())); - - // Verify that ack was NOT called because processing was cancelled - verifyNever(() => mockSyncApiRepo.ack(any())); - }, - ); - - test("completes successfully when ack call throws an exception", () async { - when(() => mockSyncApiRepo.ack(any())).thenThrow(Exception("Ack Error")); - final events = [ - ...SyncStreamStub.userEvents, - ...SyncStreamStub.partnerEvents, - ]; - - // Should still complete even if ack fails - await triggerSyncAndEmit(events); - verify(() => mockSyncApiRepo.ack(any())) - .called(1); // Verify ack was attempted - }); - - test("waits for processing to finish if onDone called early", () async { - final processingCompleter = Completer(); - bool handlerFinished = false; - - when(() => mockSyncStreamRepo.updateUsersV1(any())).thenAnswer((_) async { - await processingCompleter.future; // Wait inside handler - handlerFinished = true; - return true; - }); - - final syncFuture = sut.syncUsers(); - // Allow listener to attach - // This is necessary to ensure the stream is ready to receive events - await Future.delayed(Duration.zero); - - streamController.add(SyncStreamStub.userEvents); // Emit batch - await Future.delayed( - const Duration(milliseconds: 10), - ); // Ensure processing starts - - await streamController - .close(); // Close stream (triggers onDone internally) - await Future.delayed( - const Duration(milliseconds: 10), - ); // Give onDone a chance to fire - - // At this point, onDone was called, but processing is blocked - expect(handlerFinished, isFalse); - - processingCompleter.complete(); // Allow processing to finish - await syncFuture; // Now the main future should complete - - expect(handlerFinished, isTrue); - verify(() => mockSyncApiRepo.ack(any())).called(1); - }); - - test("processes events in the defined _kSyncTypeOrder", () async { - final future = sut.syncUsers(); - await pumpEventQueue(); - if (!streamController.isClosed) { final events = [ - SyncEvent( - type: SyncEntityType.partnerV1, - data: SyncStreamStub.partnerV1, - ack: "1", - ), // Should be processed last - SyncEvent( - type: SyncEntityType.userV1, - data: SyncStreamStub.userV1Admin, - ack: "2", - ), // Should be processed second - SyncEvent( - type: SyncEntityType.partnerDeleteV1, - data: SyncStreamStub.partnerDeleteV1, - ack: "3", - ), // Should be processed third - SyncEvent( - type: SyncEntityType.userDeleteV1, - data: SyncStreamStub.userDeleteV1, - ack: "4", - ), // Should be processed first + SyncStreamStub.userDeleteV1, + SyncStreamStub.userV1Admin, + SyncStreamStub.partnerDeleteV1, ]; - streamController.add(events); - await streamController.close(); - } - await future; + final processingFuture = + handleEventsCallback(events, mockAbortCallbackWrapper.call); + await pumpEventQueue(); - verifyInOrder([ - () => mockSyncStreamRepo.deleteUsersV1(any()), - () => mockSyncStreamRepo.updateUsersV1(any()), - () => mockSyncStreamRepo.deletePartnerV1(any()), - () => mockSyncStreamRepo.updatePartnerV1(any()), - // Verify ack happens after all processing - () => mockSyncApiRepo.ack(any()), - ]); - }); - }); + expect(handler1Started, isTrue); - group("syncUsers", () { - test("calls getSyncEvents with correct types", () async { - // Need to close the stream for the future to complete - final future = sut.syncUsers(); - await streamController.close(); - await future; + // Signal cancellation while handler 1 is waiting + when(() => cancellationChecker()).thenReturn(true); + await pumpEventQueue(); - verify( - () => mockSyncApiRepo.getSyncEvents([ - SyncRequestType.usersV1, - SyncRequestType.partnersV1, - ]), - ).called(1); - }); + processingCompleter.complete(); + await processingFuture; - test("calls repository methods with correctly grouped data", () async { - final events = [ - ...SyncStreamStub.userEvents, - ...SyncStreamStub.partnerEvents, - ]; - await triggerSyncAndEmit(events); + verifyNever(() => mockSyncStreamRepo.updateUsersV1(any())); - // Verify each handler was called with the correct list of data payloads - verify( - () => mockSyncStreamRepo.updateUsersV1( - [SyncStreamStub.userV1Admin, SyncStreamStub.userV1User], - ), - ).called(1); - verify( - () => mockSyncStreamRepo.deleteUsersV1([SyncStreamStub.userDeleteV1]), - ).called(1); - verify( - () => mockSyncStreamRepo.updatePartnerV1([SyncStreamStub.partnerV1]), - ).called(1); - verify( - () => mockSyncStreamRepo - .deletePartnerV1([SyncStreamStub.partnerDeleteV1]), - ).called(1); - }); + verify(() => mockSyncApiRepo.ack(["2"])).called(1); + }, + ); }); } diff --git a/mobile/test/fixtures/sync_stream.stub.dart b/mobile/test/fixtures/sync_stream.stub.dart index 781e63a2bb..ba97f1434a 100644 --- a/mobile/test/fixtures/sync_stream.stub.dart +++ b/mobile/test/fixtures/sync_stream.stub.dart @@ -2,44 +2,44 @@ import 'package:immich_mobile/domain/models/sync_event.model.dart'; import 'package:openapi/api.dart'; abstract final class SyncStreamStub { - static final userV1Admin = SyncUserV1( - deletedAt: DateTime(2020), - email: "admin@admin", - id: "1", - name: "Admin", - ); - static final userV1User = SyncUserV1( - deletedAt: DateTime(2021), - email: "user@user", - id: "2", - name: "User", - ); - static final userDeleteV1 = SyncUserDeleteV1(userId: "2"); - static final userEvents = [ - SyncEvent(type: SyncEntityType.userV1, data: userV1Admin, ack: "1"), - SyncEvent( - type: SyncEntityType.userDeleteV1, - data: userDeleteV1, - ack: "2", + static final userV1Admin = SyncEvent( + type: SyncEntityType.userV1, + data: SyncUserV1( + deletedAt: DateTime(2020), + email: "admin@admin", + id: "1", + name: "Admin", ), - SyncEvent(type: SyncEntityType.userV1, data: userV1User, ack: "5"), - ]; + ack: "1", + ); + static final userV1User = SyncEvent( + type: SyncEntityType.userV1, + data: SyncUserV1( + deletedAt: DateTime(2021), + email: "user@user", + id: "5", + name: "User", + ), + ack: "5", + ); + static final userDeleteV1 = SyncEvent( + type: SyncEntityType.userDeleteV1, + data: SyncUserDeleteV1(userId: "2"), + ack: "2", + ); - static final partnerV1 = SyncPartnerV1( - inTimeline: true, - sharedById: "1", - sharedWithId: "2", - ); - static final partnerDeleteV1 = SyncPartnerDeleteV1( - sharedById: "3", - sharedWithId: "4", - ); - static final partnerEvents = [ - SyncEvent( - type: SyncEntityType.partnerDeleteV1, - data: partnerDeleteV1, - ack: "4", + static final partnerV1 = SyncEvent( + type: SyncEntityType.partnerV1, + data: SyncPartnerV1( + inTimeline: true, + sharedById: "1", + sharedWithId: "2", ), - SyncEvent(type: SyncEntityType.partnerV1, data: partnerV1, ack: "3"), - ]; + ack: "3", + ); + static final partnerDeleteV1 = SyncEvent( + type: SyncEntityType.partnerDeleteV1, + data: SyncPartnerDeleteV1(sharedById: "3", sharedWithId: "4"), + ack: "4", + ); } diff --git a/mobile/test/infrastructure/repositories/sync_api_repository_test.dart b/mobile/test/infrastructure/repositories/sync_api_repository_test.dart new file mode 100644 index 0000000000..55b03a8116 --- /dev/null +++ b/mobile/test/infrastructure/repositories/sync_api_repository_test.dart @@ -0,0 +1,299 @@ +import 'dart:async'; +import 'dart:convert'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:http/http.dart' as http; +import 'package:immich_mobile/domain/models/sync_event.model.dart'; +import 'package:immich_mobile/infrastructure/repositories/sync_api.repository.dart'; +import 'package:mocktail/mocktail.dart'; +import 'package:openapi/api.dart'; + +import '../../api.mocks.dart'; +import '../../service.mocks.dart'; + +class MockHttpClient extends Mock implements http.Client {} + +class MockApiClient extends Mock implements ApiClient {} + +class MockStreamedResponse extends Mock implements http.StreamedResponse {} + +class FakeBaseRequest extends Fake implements http.BaseRequest {} + +String _createJsonLine(String type, Map data, String ack) { + return '${jsonEncode({'type': type, 'data': data, 'ack': ack})}\n'; +} + +void main() { + late SyncApiRepository sut; + late MockApiService mockApiService; + late MockApiClient mockApiClient; + late MockSyncApi mockSyncApi; + late MockHttpClient mockHttpClient; + late MockStreamedResponse mockStreamedResponse; + late StreamController> responseStreamController; + late int testBatchSize = 3; + + setUp(() { + mockApiService = MockApiService(); + mockApiClient = MockApiClient(); + mockSyncApi = MockSyncApi(); + mockHttpClient = MockHttpClient(); + mockStreamedResponse = MockStreamedResponse(); + responseStreamController = + StreamController>.broadcast(sync: true); + + registerFallbackValue(FakeBaseRequest()); + + when(() => mockApiService.apiClient).thenReturn(mockApiClient); + when(() => mockApiService.syncApi).thenReturn(mockSyncApi); + when(() => mockApiClient.basePath).thenReturn('http://demo.immich.app/api'); + when(() => mockApiService.applyToParams(any(), any())) + .thenAnswer((_) async => {}); + + // Mock HTTP client behavior + when(() => mockHttpClient.send(any())) + .thenAnswer((_) async => mockStreamedResponse); + when(() => mockStreamedResponse.statusCode).thenReturn(200); + when(() => mockStreamedResponse.stream) + .thenAnswer((_) => http.ByteStream(responseStreamController.stream)); + when(() => mockHttpClient.close()).thenAnswer((_) => {}); + + sut = SyncApiRepository(mockApiService); + }); + + tearDown(() async { + if (!responseStreamController.isClosed) { + await responseStreamController.close(); + } + }); + + Future streamChanges( + Function(List, Function() abort) onDataCallback, + ) { + return sut.streamChanges( + onDataCallback, + batchSize: testBatchSize, + httpClient: mockHttpClient, + ); + } + + test('streamChanges stops processing stream when abort is called', () async { + int onDataCallCount = 0; + bool abortWasCalledInCallback = false; + List receivedEventsBatch1 = []; + + onDataCallback(List events, Function() abort) { + onDataCallCount++; + if (onDataCallCount == 1) { + receivedEventsBatch1 = events; + abort(); + abortWasCalledInCallback = true; + } else { + fail("onData called more than once after abort was invoked"); + } + } + + final streamChangesFuture = streamChanges(onDataCallback); + + await pumpEventQueue(); + + for (int i = 0; i < testBatchSize; i++) { + responseStreamController.add( + utf8.encode( + _createJsonLine( + SyncEntityType.userDeleteV1.toString(), + SyncUserDeleteV1(userId: "user$i").toJson(), + 'ack$i', + ), + ), + ); + } + + for (int i = testBatchSize; i < testBatchSize * 2; i++) { + responseStreamController.add( + utf8.encode( + _createJsonLine( + SyncEntityType.userDeleteV1.toString(), + SyncUserDeleteV1(userId: "user$i").toJson(), + 'ack$i', + ), + ), + ); + } + + await responseStreamController.close(); + await expectLater(streamChangesFuture, completes); + + expect(onDataCallCount, 1); + expect(abortWasCalledInCallback, isTrue); + expect(receivedEventsBatch1.length, testBatchSize); + verify(() => mockHttpClient.close()).called(1); + }); + + test( + 'streamChanges does not process remaining lines in finally block if aborted', + () async { + int onDataCallCount = 0; + bool abortWasCalledInCallback = false; + + onDataCallback(List events, Function() abort) { + onDataCallCount++; + if (onDataCallCount == 1) { + abort(); + abortWasCalledInCallback = true; + } else { + fail("onData called more than once after abort was invoked"); + } + } + + final streamChangesFuture = streamChanges(onDataCallback); + + await pumpEventQueue(); + + for (int i = 0; i < testBatchSize; i++) { + responseStreamController.add( + utf8.encode( + _createJsonLine( + SyncEntityType.userDeleteV1.toString(), + SyncUserDeleteV1(userId: "user$i").toJson(), + 'ack$i', + ), + ), + ); + } + + // emit a single event to skip batching and trigger finally + responseStreamController.add( + utf8.encode( + _createJsonLine( + SyncEntityType.userDeleteV1.toString(), + SyncUserDeleteV1(userId: "user100").toJson(), + 'ack100', + ), + ), + ); + + await responseStreamController.close(); + await expectLater(streamChangesFuture, completes); + + expect(onDataCallCount, 1); + expect(abortWasCalledInCallback, isTrue); + verify(() => mockHttpClient.close()).called(1); + }, + ); + + test( + 'streamChanges processes remaining lines in finally block if not aborted', + () async { + int onDataCallCount = 0; + List receivedEventsBatch1 = []; + List receivedEventsBatch2 = []; + + onDataCallback(List events, Function() _) { + onDataCallCount++; + if (onDataCallCount == 1) { + receivedEventsBatch1 = events; + } else if (onDataCallCount == 2) { + receivedEventsBatch2 = events; + } else { + fail("onData called more than expected"); + } + } + + final streamChangesFuture = streamChanges(onDataCallback); + + await pumpEventQueue(); + + // Batch 1 + for (int i = 0; i < testBatchSize; i++) { + responseStreamController.add( + utf8.encode( + _createJsonLine( + SyncEntityType.userDeleteV1.toString(), + SyncUserDeleteV1(userId: "user$i").toJson(), + 'ack$i', + ), + ), + ); + } + + // Partial Batch 2 + responseStreamController.add( + utf8.encode( + _createJsonLine( + SyncEntityType.userDeleteV1.toString(), + SyncUserDeleteV1(userId: "user100").toJson(), + 'ack100', + ), + ), + ); + + await responseStreamController.close(); + await expectLater(streamChangesFuture, completes); + + expect(onDataCallCount, 2); + expect(receivedEventsBatch1.length, testBatchSize); + expect(receivedEventsBatch2.length, 1); + verify(() => mockHttpClient.close()).called(1); + }, + ); + + test('streamChanges handles stream error gracefully', () async { + final streamError = Exception("Network Error"); + int onDataCallCount = 0; + + onDataCallback(List events, Function() _) { + onDataCallCount++; + } + + final streamChangesFuture = streamChanges(onDataCallback); + + await pumpEventQueue(); + + responseStreamController.add( + utf8.encode( + _createJsonLine( + SyncEntityType.userDeleteV1.toString(), + SyncUserDeleteV1(userId: "user1").toJson(), + 'ack1', + ), + ), + ); + + responseStreamController.addError(streamError); + await expectLater(streamChangesFuture, throwsA(streamError)); + + expect(onDataCallCount, 0); + verify(() => mockHttpClient.close()).called(1); + }); + + test('streamChanges throws ApiException on non-200 status code', () async { + when(() => mockStreamedResponse.statusCode).thenReturn(401); + final errorBodyController = StreamController>(sync: true); + when(() => mockStreamedResponse.stream) + .thenAnswer((_) => http.ByteStream(errorBodyController.stream)); + + int onDataCallCount = 0; + + onDataCallback(List events, Function() _) { + onDataCallCount++; + } + + final future = streamChanges(onDataCallback); + + errorBodyController.add(utf8.encode('{"error":"Unauthorized"}')); + await errorBodyController.close(); + + await expectLater( + future, + throwsA( + isA() + .having((e) => e.code, 'code', 401) + .having((e) => e.message, 'message', contains('Unauthorized')), + ), + ); + + expect(onDataCallCount, 0); + verify(() => mockHttpClient.close()).called(1); + }); +} From 504930947d62194ab4cf2e9c219d9d0bc5f4fde0 Mon Sep 17 00:00:00 2001 From: bo0tzz Date: Fri, 18 Apr 2025 22:10:27 +0200 Subject: [PATCH 09/29] fix: various actions workflow security improvements (#17651) * fix: set persist-credentials explicitly for checkout https://woodruffw.github.io/zizmor/audits/#artipacked * fix: minimize permissions scope for workflows https://woodruffw.github.io/zizmor/audits/#excessive-permissions * fix: remove potential template injections https://woodruffw.github.io/zizmor/audits/#template-injection * fix: only pass needed secrets in workflow_call https://woodruffw.github.io/zizmor/audits/#secrets-inherit * fix: push perm for single-arch build jobs I hadn't realised these push to the registry too :x * chore: fix formatting * fix: $ * fix: retag job quoting --------- Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- .github/workflows/build-mobile.yml | 29 +++++-- .github/workflows/cache-cleanup.yml | 17 ++-- .github/workflows/cli.yml | 13 ++- .github/workflows/codeql-analysis.yml | 4 + .github/workflows/docker.yml | 85 +++++++++++++------ .github/workflows/docs-build.yml | 10 +++ .github/workflows/docs-deploy.yml | 20 ++++- .github/workflows/docs-destroy.yml | 7 ++ .github/workflows/fix-format.yml | 4 + .github/workflows/pr-label-validation.yml | 2 + .github/workflows/pr-labeler.yml | 2 + .../pr-require-conventional-commit.yml | 4 + .github/workflows/prepare-release.yml | 18 +++- .github/workflows/preview-label.yaml | 2 + .github/workflows/sdk.yml | 8 +- .github/workflows/static_analysis.yml | 16 +++- .github/workflows/test.yml | 78 ++++++++++++++++- .github/workflows/weblate-lock.yml | 13 +-- 18 files changed, 269 insertions(+), 63 deletions(-) diff --git a/.github/workflows/build-mobile.yml b/.github/workflows/build-mobile.yml index 406e8f89e1..7217b5267e 100644 --- a/.github/workflows/build-mobile.yml +++ b/.github/workflows/build-mobile.yml @@ -7,6 +7,15 @@ on: ref: required: false type: string + secrets: + KEY_JKS: + required: true + ALIAS: + required: true + ANDROID_KEY_PASSWORD: + required: true + ANDROID_STORE_PASSWORD: + required: true pull_request: push: branches: [main] @@ -15,14 +24,21 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: {} + jobs: pre-job: runs-on: ubuntu-latest + permissions: + contents: read outputs: should_run: ${{ steps.found_paths.outputs.mobile == 'true' || steps.should_force.outputs.should_force == 'true' }} steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false + - id: found_paths uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 with: @@ -38,22 +54,17 @@ jobs: build-sign-android: name: Build and sign Android needs: pre-job + permissions: + contents: read # Skip when PR from a fork if: ${{ !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]' && needs.pre-job.outputs.should_run == 'true' }} runs-on: macos-14 steps: - - name: Determine ref - id: get-ref - run: | - input_ref="${{ inputs.ref }}" - github_ref="${{ github.sha }}" - ref="${input_ref:-$github_ref}" - echo "ref=$ref" >> $GITHUB_OUTPUT - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: - ref: ${{ steps.get-ref.outputs.ref }} + ref: ${{ inputs.ref || github.sha }} + persist-credentials: false - uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4 with: diff --git a/.github/workflows/cache-cleanup.yml b/.github/workflows/cache-cleanup.yml index 0cc73c46c3..84adde08cf 100644 --- a/.github/workflows/cache-cleanup.yml +++ b/.github/workflows/cache-cleanup.yml @@ -8,31 +8,38 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: {} + jobs: cleanup: name: Cleanup runs-on: ubuntu-latest + permissions: + contents: read + actions: write steps: - name: Check out code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Cleanup + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REF: ${{ github.ref }} run: | gh extension install actions/gh-actions-cache REPO=${{ github.repository }} - BRANCH=${{ github.ref }} echo "Fetching list of cache keys" - cacheKeysForPR=$(gh actions-cache list -R $REPO -B $BRANCH -L 100 | cut -f 1 ) + cacheKeysForPR=$(gh actions-cache list -R $REPO -B ${REF} -L 100 | cut -f 1 ) ## Setting this to not fail the workflow while deleting cache keys. set +e echo "Deleting caches..." for cacheKey in $cacheKeysForPR do - gh actions-cache delete $cacheKey -R $REPO -B $BRANCH --confirm + gh actions-cache delete $cacheKey -R "$REPO" -B "${REF}" --confirm done echo "Done" - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.yml index 05326d063e..231ad141e4 100644 --- a/.github/workflows/cli.yml +++ b/.github/workflows/cli.yml @@ -16,19 +16,23 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true -permissions: - packages: write +permissions: {} jobs: publish: name: CLI Publish runs-on: ubuntu-latest + permissions: + contents: read defaults: run: working-directory: ./cli steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false + # Setup .npmrc file to publish to npm - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: @@ -48,11 +52,16 @@ jobs: docker: name: Docker runs-on: ubuntu-latest + permissions: + contents: read + packages: write needs: publish steps: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Set up QEMU uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 99ffee2e88..7d07785de7 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -24,6 +24,8 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: {} + jobs: analyze: name: Analyze @@ -43,6 +45,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d1bdb5e8e7..a78d3c25dc 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -12,18 +12,21 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true -permissions: - packages: write +permissions: {} jobs: pre-job: runs-on: ubuntu-latest + permissions: + contents: read outputs: should_run_server: ${{ steps.found_paths.outputs.server == 'true' || steps.should_force.outputs.should_force == 'true' }} should_run_ml: ${{ steps.found_paths.outputs.machine-learning == 'true' || steps.should_force.outputs.should_force == 'true' }} steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - id: found_paths uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 with: @@ -45,6 +48,9 @@ jobs: retag_ml: name: Re-Tag ML needs: pre-job + permissions: + contents: read + packages: write if: ${{ needs.pre-job.outputs.should_run_ml == 'false' && !github.event.pull_request.head.repo.fork }} runs-on: ubuntu-latest strategy: @@ -58,18 +64,22 @@ jobs: username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Re-tag image + env: + REGISTRY_NAME: 'ghcr.io' + REPOSITORY: ${{ github.repository_owner }}/immich-machine-learning + TAG_OLD: main${{ matrix.suffix }} + TAG_PR: ${{ github.event.number == 0 && github.ref_name || format('pr-{0}', github.event.number) }}${{ matrix.suffix }} + TAG_COMMIT: commit-${{ github.event_name != 'pull_request' && github.sha || github.event.pull_request.head.sha }}${{ matrix.suffix }} run: | - REGISTRY_NAME="ghcr.io" - REPOSITORY=${{ github.repository_owner }}/immich-machine-learning - TAG_OLD=main${{ matrix.suffix }} - TAG_PR=${{ github.event.number == 0 && github.ref_name || format('pr-{0}', github.event.number) }}${{ matrix.suffix }} - TAG_COMMIT=commit-${{ github.event_name != 'pull_request' && github.sha || github.event.pull_request.head.sha }}${{ matrix.suffix }} - docker buildx imagetools create -t $REGISTRY_NAME/$REPOSITORY:$TAG_PR $REGISTRY_NAME/$REPOSITORY:$TAG_OLD - docker buildx imagetools create -t $REGISTRY_NAME/$REPOSITORY:$TAG_COMMIT $REGISTRY_NAME/$REPOSITORY:$TAG_OLD + docker buildx imagetools create -t "${REGISTRY_NAME}/${REPOSITORY}:${TAG_PR}" "${REGISTRY_NAME}/${REPOSITORY}:${TAG_OLD}" + docker buildx imagetools create -t "${REGISTRY_NAME}/${REPOSITORY}:${TAG_COMMIT}" "${REGISTRY_NAME}/${REPOSITORY}:${TAG_OLD}" retag_server: name: Re-Tag Server needs: pre-job + permissions: + contents: read + packages: write if: ${{ needs.pre-job.outputs.should_run_server == 'false' && !github.event.pull_request.head.repo.fork }} runs-on: ubuntu-latest strategy: @@ -83,18 +93,22 @@ jobs: username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Re-tag image + env: + REGISTRY_NAME: 'ghcr.io' + REPOSITORY: ${{ github.repository_owner }}/immich-server + TAG_OLD: main${{ matrix.suffix }} + TAG_PR: ${{ github.event.number == 0 && github.ref_name || format('pr-{0}', github.event.number) }}${{ matrix.suffix }} + TAG_COMMIT: commit-${{ github.event_name != 'pull_request' && github.sha || github.event.pull_request.head.sha }}${{ matrix.suffix }} run: | - REGISTRY_NAME="ghcr.io" - REPOSITORY=${{ github.repository_owner }}/immich-server - TAG_OLD=main${{ matrix.suffix }} - TAG_PR=${{ github.event.number == 0 && github.ref_name || format('pr-{0}', github.event.number) }}${{ matrix.suffix }} - TAG_COMMIT=commit-${{ github.event_name != 'pull_request' && github.sha || github.event.pull_request.head.sha }}${{ matrix.suffix }} - docker buildx imagetools create -t $REGISTRY_NAME/$REPOSITORY:$TAG_PR $REGISTRY_NAME/$REPOSITORY:$TAG_OLD - docker buildx imagetools create -t $REGISTRY_NAME/$REPOSITORY:$TAG_COMMIT $REGISTRY_NAME/$REPOSITORY:$TAG_OLD + docker buildx imagetools create -t "${REGISTRY_NAME}/${REPOSITORY}:${TAG_PR}" "${REGISTRY_NAME}/${REPOSITORY}:${TAG_OLD}" + docker buildx imagetools create -t "${REGISTRY_NAME}/${REPOSITORY}:${TAG_COMMIT}" "${REGISTRY_NAME}/${REPOSITORY}:${TAG_OLD}" build_and_push_ml: name: Build and Push ML needs: pre-job + permissions: + contents: read + packages: write if: ${{ needs.pre-job.outputs.should_run_ml == 'true' }} runs-on: ${{ matrix.runner }} env: @@ -148,6 +162,8 @@ jobs: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Set up Docker Buildx uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 @@ -161,11 +177,14 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Generate cache key suffix + env: + REF: ${{ github.ref_name }} run: | if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo "CACHE_KEY_SUFFIX=pr-${{ github.event.number }}" >> $GITHUB_ENV else - echo "CACHE_KEY_SUFFIX=$(echo ${{ github.ref_name }} | sed 's/[^a-zA-Z0-9]/-/g')" >> $GITHUB_ENV + SUFFIX=$(echo "${REF}" | sed 's/[^a-zA-Z0-9]/-/g') + echo "CACHE_KEY_SUFFIX=${SUFFIX}" >> $GITHUB_ENV fi - name: Generate cache target @@ -175,7 +194,7 @@ jobs: # Essentially just ignore the cache output (forks can't write to registry cache) echo "cache-to=type=local,dest=/tmp/discard,ignore-error=true" >> $GITHUB_OUTPUT else - echo "cache-to=type=registry,ref=${{ env.GHCR_REPO }}-build-cache:${{ env.PLATFORM_PAIR }}-${{ matrix.device }}-${{ env.CACHE_KEY_SUFFIX }},mode=max,compression=zstd" >> $GITHUB_OUTPUT + echo "cache-to=type=registry,ref=${GHCR_REPO}-build-cache:${PLATFORM_PAIR}-${{ matrix.device }}-${CACHE_KEY_SUFFIX},mode=max,compression=zstd" >> $GITHUB_OUTPUT fi - name: Generate docker image tags @@ -221,6 +240,10 @@ jobs: merge_ml: name: Merge & Push ML runs-on: ubuntu-latest + permissions: + contents: read + actions: read + packages: write if: ${{ needs.pre-job.outputs.should_run_ml == 'true' && !github.event.pull_request.head.repo.fork }} env: GHCR_REPO: ghcr.io/${{ github.repository_owner }}/immich-machine-learning @@ -308,15 +331,16 @@ jobs: fi TAGS=$(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - SOURCE_ARGS=$(printf '${{ env.GHCR_REPO }}@sha256:%s ' *) - - echo "docker buildx imagetools create $TAGS "${ANNOTATIONS[@]}" $SOURCE_ARGS" + SOURCE_ARGS=$(printf "${GHCR_REPO}@sha256:%s " *) docker buildx imagetools create $TAGS "${ANNOTATIONS[@]}" $SOURCE_ARGS build_and_push_server: name: Build and Push Server runs-on: ${{ matrix.runner }} + permissions: + contents: read + packages: write needs: pre-job if: ${{ needs.pre-job.outputs.should_run_server == 'true' }} env: @@ -340,6 +364,8 @@ jobs: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Set up Docker Buildx uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3 @@ -353,11 +379,14 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Generate cache key suffix + env: + REF: ${{ github.ref_name }} run: | if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo "CACHE_KEY_SUFFIX=pr-${{ github.event.number }}" >> $GITHUB_ENV else - echo "CACHE_KEY_SUFFIX=$(echo ${{ github.ref_name }} | sed 's/[^a-zA-Z0-9]/-/g')" >> $GITHUB_ENV + SUFFIX=$(echo "${REF}" | sed 's/[^a-zA-Z0-9]/-/g') + echo "CACHE_KEY_SUFFIX=${SUFFIX}" >> $GITHUB_ENV fi - name: Generate cache target @@ -367,7 +396,7 @@ jobs: # Essentially just ignore the cache output (forks can't write to registry cache) echo "cache-to=type=local,dest=/tmp/discard,ignore-error=true" >> $GITHUB_OUTPUT else - echo "cache-to=type=registry,ref=${{ env.GHCR_REPO }}-build-cache:${{ env.PLATFORM_PAIR }}-${{ env.CACHE_KEY_SUFFIX }},mode=max,compression=zstd" >> $GITHUB_OUTPUT + echo "cache-to=type=registry,ref=${GHCR_REPO}-build-cache:${PLATFORM_PAIR}-${CACHE_KEY_SUFFIX},mode=max,compression=zstd" >> $GITHUB_OUTPUT fi - name: Generate docker image tags @@ -413,6 +442,10 @@ jobs: merge_server: name: Merge & Push Server runs-on: ubuntu-latest + permissions: + contents: read + actions: read + packages: write if: ${{ needs.pre-job.outputs.should_run_server == 'true' && !github.event.pull_request.head.repo.fork }} env: GHCR_REPO: ghcr.io/${{ github.repository_owner }}/immich-server @@ -486,15 +519,14 @@ jobs: fi TAGS=$(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - SOURCE_ARGS=$(printf '${{ env.GHCR_REPO }}@sha256:%s ' *) - - echo "docker buildx imagetools create $TAGS "${ANNOTATIONS[@]}" $SOURCE_ARGS" + SOURCE_ARGS=$(printf "${GHCR_REPO}@sha256:%s " *) docker buildx imagetools create $TAGS "${ANNOTATIONS[@]}" $SOURCE_ARGS success-check-server: name: Docker Build & Push Server Success needs: [merge_server, retag_server] + permissions: {} runs-on: ubuntu-latest if: always() steps: @@ -508,6 +540,7 @@ jobs: success-check-ml: name: Docker Build & Push ML Success needs: [merge_ml, retag_ml] + permissions: {} runs-on: ubuntu-latest if: always() steps: diff --git a/.github/workflows/docs-build.yml b/.github/workflows/docs-build.yml index fdd30034ee..ece3bbd248 100644 --- a/.github/workflows/docs-build.yml +++ b/.github/workflows/docs-build.yml @@ -10,14 +10,20 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: {} + jobs: pre-job: runs-on: ubuntu-latest + permissions: + contents: read outputs: should_run: ${{ steps.found_paths.outputs.docs == 'true' || steps.should_force.outputs.should_force == 'true' }} steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - id: found_paths uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 with: @@ -33,6 +39,8 @@ jobs: build: name: Docs Build needs: pre-job + permissions: + contents: read if: ${{ needs.pre-job.outputs.should_run == 'true' }} runs-on: ubuntu-latest defaults: @@ -42,6 +50,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Setup Node uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 diff --git a/.github/workflows/docs-deploy.yml b/.github/workflows/docs-deploy.yml index f33c0c4c03..10277a0c5e 100644 --- a/.github/workflows/docs-deploy.yml +++ b/.github/workflows/docs-deploy.yml @@ -9,6 +9,9 @@ jobs: checks: name: Docs Deploy Checks runs-on: ubuntu-latest + permissions: + actions: read + pull-requests: read outputs: parameters: ${{ steps.parameters.outputs.result }} artifact: ${{ steps.get-artifact.outputs.result }} @@ -36,6 +39,8 @@ jobs: - name: Determine deploy parameters id: parameters uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 + env: + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} with: script: | const eventType = context.payload.workflow_run.event; @@ -57,7 +62,8 @@ jobs: } else if (eventType == "pull_request") { let pull_number = context.payload.workflow_run.pull_requests[0]?.number; if(!pull_number) { - const response = await github.rest.search.issuesAndPullRequests({q: 'repo:${{ github.repository }} is:pr sha:${{ github.event.workflow_run.head_sha }}',per_page: 1,}) + const {HEAD_SHA} = process.env; + const response = await github.rest.search.issuesAndPullRequests({q: `repo:${{ github.repository }} is:pr sha:${HEAD_SHA}`,per_page: 1,}) const items = response.data.items if (items.length < 1) { throw new Error("No pull request found for the commit") @@ -95,10 +101,16 @@ jobs: name: Docs Deploy runs-on: ubuntu-latest needs: checks + permissions: + contents: read + actions: read + pull-requests: write if: ${{ fromJson(needs.checks.outputs.artifact).found && fromJson(needs.checks.outputs.parameters).shouldDeploy }} steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Load parameters id: parameters @@ -162,9 +174,11 @@ jobs: - name: Output Cleaning id: clean + env: + TG_OUTPUT: ${{ steps.docs-output.outputs.tg_action_output }} run: | - TG_OUT=$(echo '${{ steps.docs-output.outputs.tg_action_output }}' | sed 's|%0A|\n|g ; s|%3C|<|g' | jq -c .) - echo "output=$TG_OUT" >> $GITHUB_OUTPUT + CLEANED=$(echo "$TG_OUTPUT" | sed 's|%0A|\n|g ; s|%3C|<|g' | jq -c .) + echo "output=$CLEANED" >> $GITHUB_OUTPUT - name: Publish to Cloudflare Pages uses: cloudflare/pages-action@f0a1cd58cd66095dee69bfa18fa5efd1dde93bca # v1 diff --git a/.github/workflows/docs-destroy.yml b/.github/workflows/docs-destroy.yml index 99499528b6..9d1e4b6612 100644 --- a/.github/workflows/docs-destroy.yml +++ b/.github/workflows/docs-destroy.yml @@ -3,13 +3,20 @@ on: pull_request_target: types: [closed] +permissions: {} + jobs: deploy: name: Docs Destroy runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Destroy Docs Subdomain env: diff --git a/.github/workflows/fix-format.yml b/.github/workflows/fix-format.yml index 9c52691a52..77b86cb0b8 100644 --- a/.github/workflows/fix-format.yml +++ b/.github/workflows/fix-format.yml @@ -4,11 +4,14 @@ on: pull_request: types: [labeled] +permissions: {} + jobs: fix-formatting: runs-on: ubuntu-latest if: ${{ github.event.label.name == 'fix:formatting' }} permissions: + contents: write pull-requests: write steps: - name: Generate a token @@ -23,6 +26,7 @@ jobs: with: ref: ${{ github.event.pull_request.head.ref }} token: ${{ steps.generate-token.outputs.token }} + persist-credentials: true - name: Setup Node uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 diff --git a/.github/workflows/pr-label-validation.yml b/.github/workflows/pr-label-validation.yml index 247c625a96..8d34597a08 100644 --- a/.github/workflows/pr-label-validation.yml +++ b/.github/workflows/pr-label-validation.yml @@ -4,6 +4,8 @@ on: pull_request_target: types: [opened, labeled, unlabeled, synchronize] +permissions: {} + jobs: validate-release-label: runs-on: ubuntu-latest diff --git a/.github/workflows/pr-labeler.yml b/.github/workflows/pr-labeler.yml index 1b43c89889..5704f4275f 100644 --- a/.github/workflows/pr-labeler.yml +++ b/.github/workflows/pr-labeler.yml @@ -2,6 +2,8 @@ name: 'Pull Request Labeler' on: - pull_request_target +permissions: {} + jobs: labeler: permissions: diff --git a/.github/workflows/pr-require-conventional-commit.yml b/.github/workflows/pr-require-conventional-commit.yml index 20dd0492f4..78ba77495c 100644 --- a/.github/workflows/pr-require-conventional-commit.yml +++ b/.github/workflows/pr-require-conventional-commit.yml @@ -4,9 +4,13 @@ on: pull_request: types: [opened, synchronize, reopened, edited] +permissions: {} + jobs: validate-pr-title: runs-on: ubuntu-latest + permissions: + pull-requests: write steps: - name: PR Conventional Commit Validation uses: ytanikin/PRConventionalCommits@b628c5a234cc32513014b7bfdd1e47b532124d98 # 1.3.0 diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml index ffb24d8952..7971f7574a 100644 --- a/.github/workflows/prepare-release.yml +++ b/.github/workflows/prepare-release.yml @@ -21,13 +21,14 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }}-root cancel-in-progress: true +permissions: {} + jobs: bump_version: runs-on: ubuntu-latest - outputs: ref: ${{ steps.push-tag.outputs.commit_long_sha }} - + permissions: {} # No job-level permissions are needed because it uses the app-token steps: - name: Generate a token id: generate-token @@ -40,6 +41,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: token: ${{ steps.generate-token.outputs.token }} + persist-credentials: true - name: Install uv uses: astral-sh/setup-uv@0c5e2b8115b80b4c7c5ddf6ffdd634974642d182 # v5 @@ -59,14 +61,20 @@ jobs: build_mobile: uses: ./.github/workflows/build-mobile.yml needs: bump_version - secrets: inherit + secrets: + KEY_JKS: ${{ secrets.KEY_JKS }} + ALIAS: ${{ secrets.ALIAS }} + ANDROID_KEY_PASSWORD: ${{ secrets.ANDROID_KEY_PASSWORD }} + ANDROID_STORE_PASSWORD: ${{ secrets.ANDROID_STORE_PASSWORD }} with: ref: ${{ needs.bump_version.outputs.ref }} prepare_release: runs-on: ubuntu-latest needs: build_mobile - + permissions: + actions: read # To download the app artifact + # No content permissions are needed because it uses the app-token steps: - name: Generate a token id: generate-token @@ -79,6 +87,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: token: ${{ steps.generate-token.outputs.token }} + persist-credentials: false - name: Download APK uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4 @@ -90,6 +99,7 @@ jobs: with: draft: true tag_name: ${{ env.IMMICH_VERSION }} + token: ${{ steps.generate-token.outputs.token }} generate_release_notes: true body_path: misc/release/notes.tmpl files: | diff --git a/.github/workflows/preview-label.yaml b/.github/workflows/preview-label.yaml index 447a309a2e..4c445f13d0 100644 --- a/.github/workflows/preview-label.yaml +++ b/.github/workflows/preview-label.yaml @@ -4,6 +4,8 @@ on: pull_request: types: [labeled, closed] +permissions: {} + jobs: comment-status: runs-on: ubuntu-latest diff --git a/.github/workflows/sdk.yml b/.github/workflows/sdk.yml index cde2075423..482faf29f4 100644 --- a/.github/workflows/sdk.yml +++ b/.github/workflows/sdk.yml @@ -4,18 +4,22 @@ on: release: types: [published] -permissions: - packages: write +permissions: {} jobs: publish: name: Publish `@immich/sdk` runs-on: ubuntu-latest + permissions: + contents: read defaults: run: working-directory: ./open-api/typescript-sdk steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false + # Setup .npmrc file to publish to npm - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: diff --git a/.github/workflows/static_analysis.yml b/.github/workflows/static_analysis.yml index 615082f86a..1a3c11d3d5 100644 --- a/.github/workflows/static_analysis.yml +++ b/.github/workflows/static_analysis.yml @@ -9,14 +9,20 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: {} + jobs: pre-job: runs-on: ubuntu-latest + permissions: + contents: read outputs: should_run: ${{ steps.found_paths.outputs.mobile == 'true' || steps.should_force.outputs.should_force == 'true' }} steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - id: found_paths uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 with: @@ -33,12 +39,14 @@ jobs: name: Run Dart Code Analysis needs: pre-job if: ${{ needs.pre-job.outputs.should_run == 'true' }} - runs-on: ubuntu-latest - + permissions: + contents: read steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Setup Flutter SDK uses: subosito/flutter-action@e938fdf56512cc96ef2f93601a5a40bde3801046 # v2 @@ -69,9 +77,11 @@ jobs: - name: Verify files have not changed if: steps.verify-changed-files.outputs.files_changed == 'true' + env: + CHANGED_FILES: ${{ steps.verify-changed-files.outputs.changed_files }} run: | echo "ERROR: Generated files not up to date! Run make_build inside the mobile directory" - echo "Changed files: ${{ steps.verify-changed-files.outputs.changed_files }}" + echo "Changed files: ${CHANGED_FILES}" exit 1 - name: Run dart analyze diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f5cb6c8d30..91389c25ff 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,9 +9,13 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: {} + jobs: pre-job: runs-on: ubuntu-latest + permissions: + contents: read outputs: should_run_web: ${{ steps.found_paths.outputs.web == 'true' || steps.should_force.outputs.should_force == 'true' }} should_run_server: ${{ steps.found_paths.outputs.server == 'true' || steps.should_force.outputs.should_force == 'true' }} @@ -25,6 +29,9 @@ jobs: steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false + - id: found_paths uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 with: @@ -58,6 +65,8 @@ jobs: needs: pre-job if: ${{ needs.pre-job.outputs.should_run_server == 'true' }} runs-on: ubuntu-latest + permissions: + contents: read defaults: run: working-directory: ./server @@ -65,6 +74,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Setup Node uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 @@ -95,6 +106,8 @@ jobs: needs: pre-job if: ${{ needs.pre-job.outputs.should_run_cli == 'true' }} runs-on: ubuntu-latest + permissions: + contents: read defaults: run: working-directory: ./cli @@ -102,6 +115,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Setup Node uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 @@ -136,6 +151,8 @@ jobs: needs: pre-job if: ${{ needs.pre-job.outputs.should_run_cli == 'true' }} runs-on: windows-latest + permissions: + contents: read defaults: run: working-directory: ./cli @@ -143,6 +160,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Setup Node uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 @@ -170,6 +189,8 @@ jobs: needs: pre-job if: ${{ needs.pre-job.outputs.should_run_web == 'true' }} runs-on: ubuntu-latest + permissions: + contents: read defaults: run: working-directory: ./web @@ -177,6 +198,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Setup Node uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 @@ -215,6 +238,8 @@ jobs: needs: pre-job if: ${{ needs.pre-job.outputs.should_run_e2e == 'true' }} runs-on: ubuntu-latest + permissions: + contents: read defaults: run: working-directory: ./e2e @@ -222,6 +247,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Setup Node uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 @@ -254,6 +281,8 @@ jobs: needs: pre-job if: ${{ needs.pre-job.outputs.should_run_server == 'true' }} runs-on: ubuntu-latest + permissions: + contents: read defaults: run: working-directory: ./server @@ -261,6 +290,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Setup Node uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 @@ -279,6 +310,8 @@ jobs: needs: pre-job if: ${{ needs.pre-job.outputs.should_run_e2e_server_cli == 'true' }} runs-on: mich + permissions: + contents: read defaults: run: working-directory: ./e2e @@ -287,6 +320,7 @@ jobs: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: + persist-credentials: false submodules: 'recursive' - name: Setup Node @@ -321,6 +355,8 @@ jobs: needs: pre-job if: ${{ needs.pre-job.outputs.should_run_e2e_web == 'true' }} runs-on: mich + permissions: + contents: read defaults: run: working-directory: ./e2e @@ -329,6 +365,7 @@ jobs: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: + persist-credentials: false submodules: 'recursive' - name: Setup Node @@ -362,8 +399,13 @@ jobs: needs: pre-job if: ${{ needs.pre-job.outputs.should_run_mobile == 'true' }} runs-on: ubuntu-latest + permissions: + contents: read steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false + - name: Setup Flutter SDK uses: subosito/flutter-action@e938fdf56512cc96ef2f93601a5a40bde3801046 # v2 with: @@ -378,11 +420,16 @@ jobs: needs: pre-job if: ${{ needs.pre-job.outputs.should_run_ml == 'true' }} runs-on: ubuntu-latest + permissions: + contents: read defaults: run: working-directory: ./machine-learning steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false + - name: Install uv uses: astral-sh/setup-uv@0c5e2b8115b80b4c7c5ddf6ffdd634974642d182 # v5 - uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5 @@ -411,6 +458,8 @@ jobs: needs: pre-job if: ${{ needs.pre-job.outputs['should_run_.github'] == 'true' }} runs-on: ubuntu-latest + permissions: + contents: read defaults: run: working-directory: ./.github @@ -418,6 +467,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Setup Node uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 @@ -434,22 +485,31 @@ jobs: shellcheck: name: ShellCheck runs-on: ubuntu-latest + permissions: + contents: read steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false + - name: Run ShellCheck uses: ludeeus/action-shellcheck@master with: ignore_paths: >- **/open-api/** - **/openapi/** + **/openapi** **/node_modules/** generated-api-up-to-date: name: OpenAPI Clients runs-on: ubuntu-latest + permissions: + contents: read steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Setup Node uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 @@ -476,14 +536,18 @@ jobs: - name: Verify files have not changed if: steps.verify-changed-files.outputs.files_changed == 'true' + env: + CHANGED_FILES: ${{ steps.verify-changed-files.outputs.changed_files }} run: | echo "ERROR: Generated files not up to date!" - echo "Changed files: ${{ steps.verify-changed-files.outputs.changed_files }}" + echo "Changed files: ${CHANGED_FILES}" exit 1 generated-typeorm-migrations-up-to-date: name: TypeORM Checks runs-on: ubuntu-latest + permissions: + contents: read services: postgres: image: tensorchord/pgvecto-rs:pg14-v0.2.0@sha256:739cdd626151ff1f796dc95a6591b55a714f341c737e27f045019ceabf8e8c52 @@ -505,6 +569,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Setup Node uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 @@ -535,9 +601,11 @@ jobs: server/src - name: Verify migration files have not changed if: steps.verify-changed-files.outputs.files_changed == 'true' + env: + CHANGED_FILES: ${{ steps.verify-changed-files.outputs.changed_files }} run: | echo "ERROR: Generated migration files not up to date!" - echo "Changed files: ${{ steps.verify-changed-files.outputs.changed_files }}" + echo "Changed files: ${CHANGED_FILES}" cat ./src/*-TestMigration.ts exit 1 @@ -555,9 +623,11 @@ jobs: - name: Verify SQL files have not changed if: steps.verify-changed-sql-files.outputs.files_changed == 'true' + env: + CHANGED_FILES: ${{ steps.verify-changed-sql-files.outputs.changed_files }} run: | echo "ERROR: Generated SQL files not up to date!" - echo "Changed files: ${{ steps.verify-changed-sql-files.outputs.changed_files }}" + echo "Changed files: ${CHANGED_FILES}" exit 1 # mobile-integration-tests: diff --git a/.github/workflows/weblate-lock.yml b/.github/workflows/weblate-lock.yml index 69dce3ac41..2aef5c472a 100644 --- a/.github/workflows/weblate-lock.yml +++ b/.github/workflows/weblate-lock.yml @@ -4,30 +4,32 @@ on: pull_request: branches: [main] +permissions: {} + jobs: pre-job: runs-on: ubuntu-latest + permissions: + contents: read outputs: should_run: ${{ steps.found_paths.outputs.i18n == 'true' && github.head_ref != 'chore/translations'}} steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - id: found_paths uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 with: filters: | i18n: - 'i18n/!(en)**\.json' - - name: Debug - run: | - echo "Should run: ${{ steps.found_paths.outputs.i18n == 'true' && github.head_ref != 'chore/translations'}}" - echo "Found i18n paths: ${{ steps.found_paths.outputs.i18n }}" - echo "Head ref: ${{ github.head_ref }}" enforce-lock: name: Check Weblate Lock needs: [pre-job] runs-on: ubuntu-latest + permissions: {} if: ${{ needs.pre-job.outputs.should_run == 'true' }} steps: - name: Check weblate lock @@ -47,6 +49,7 @@ jobs: name: Weblate Lock Check Success needs: [enforce-lock] runs-on: ubuntu-latest + permissions: {} if: always() steps: - name: Any jobs failed? From 854ea13d6abcdb031e4f63974be049733c717f81 Mon Sep 17 00:00:00 2001 From: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com> Date: Fri, 18 Apr 2025 22:52:41 +0200 Subject: [PATCH 10/29] chore: simplify asset getByIds (#17699) --- server/src/repositories/asset.repository.ts | 54 ++++----------------- 1 file changed, 9 insertions(+), 45 deletions(-) diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts index 52c390c162..af79fb7c5f 100644 --- a/server/src/repositories/asset.repository.ts +++ b/server/src/repositories/asset.repository.ts @@ -299,51 +299,15 @@ export class AssetRepository { @GenerateSql({ params: [[DummyValue.UUID]] }) @ChunkedArray() - async getByIds( - ids: string[], - { exifInfo, faces, files, library, owner, smartSearch, stack, tags }: GetByIdsRelations = {}, - ): Promise { - const res = await this.db - .selectFrom('assets') - .selectAll('assets') - .where('assets.id', '=', anyUuid(ids)) - .$if(!!exifInfo, withExif) - .$if(!!faces, (qb) => - qb.select((eb) => - faces?.person ? withFacesAndPeople(eb, faces.withDeleted) : withFaces(eb, faces?.withDeleted), - ), - ) - .$if(!!files, (qb) => qb.select(withFiles)) - .$if(!!library, (qb) => qb.select(withLibrary)) - .$if(!!owner, (qb) => qb.select(withOwner)) - .$if(!!smartSearch, withSmartSearch) - .$if(!!stack, (qb) => - qb - .leftJoin('asset_stack', 'asset_stack.id', 'assets.stackId') - .$if(!stack!.assets, (qb) => qb.select((eb) => eb.fn.toJson(eb.table('asset_stack')).as('stack'))) - .$if(!!stack!.assets, (qb) => - qb - .leftJoinLateral( - (eb) => - eb - .selectFrom('assets as stacked') - .selectAll('asset_stack') - .select((eb) => eb.fn('array_agg', [eb.table('stacked')]).as('assets')) - .whereRef('stacked.stackId', '=', 'asset_stack.id') - .whereRef('stacked.id', '!=', 'asset_stack.primaryAssetId') - .where('stacked.deletedAt', 'is', null) - .where('stacked.isArchived', '=', false) - .groupBy('asset_stack.id') - .as('stacked_assets'), - (join) => join.on('asset_stack.id', 'is not', null), - ) - .select((eb) => eb.fn.toJson(eb.table('stacked_assets')).as('stack')), - ), - ) - .$if(!!tags, (qb) => qb.select(withTags)) - .execute(); - - return res as any as AssetEntity[]; + getByIds(ids: string[]): Promise { + return ( + this.db + // + .selectFrom('assets') + .selectAll('assets') + .where('assets.id', '=', anyUuid(ids)) + .execute() as Promise + ); } @GenerateSql({ params: [[DummyValue.UUID]] }) From 52ae06c1194e992df6407e67bdebf9fd9b41238f Mon Sep 17 00:00:00 2001 From: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com> Date: Fri, 18 Apr 2025 23:10:34 +0200 Subject: [PATCH 11/29] refactor: remove album entity, update types (#17450) --- server/src/database.ts | 85 ++++++----- server/src/db.d.ts | 6 +- server/src/dtos/album.dto.ts | 28 +++- server/src/dtos/asset-response.dto.ts | 50 ++++++- server/src/dtos/memory.dto.ts | 3 +- server/src/dtos/shared-link.dto.ts | 8 +- server/src/entities/album.entity.ts | 23 --- server/src/entities/asset.entity.ts | 24 ++-- server/src/entities/shared-link.entity.ts | 20 --- server/src/queries/asset.repository.sql | 21 +-- server/src/repositories/album.repository.ts | 53 +++---- .../src/repositories/asset-job.repository.ts | 17 ++- server/src/repositories/asset.repository.ts | 132 ++++++++---------- server/src/repositories/search.repository.ts | 35 +++-- .../repositories/shared-link.repository.ts | 41 +++--- server/src/repositories/stack.repository.ts | 5 +- server/src/services/album.service.ts | 8 +- .../src/services/asset-media.service.spec.ts | 9 +- server/src/services/asset-media.service.ts | 16 +-- server/src/services/asset.service.spec.ts | 7 +- server/src/services/asset.service.ts | 8 +- server/src/services/duplicate.service.ts | 2 +- server/src/services/job.service.spec.ts | 4 +- server/src/services/job.service.ts | 4 +- server/src/services/library.service.spec.ts | 18 +-- server/src/services/library.service.ts | 14 +- server/src/services/metadata.service.spec.ts | 9 +- server/src/services/metadata.service.ts | 2 +- server/src/services/person.service.ts | 8 +- server/src/services/search.service.spec.ts | 2 +- server/src/services/search.service.ts | 7 +- .../src/services/shared-link.service.spec.ts | 5 +- server/src/services/shared-link.service.ts | 10 +- server/src/services/sync.service.spec.ts | 3 +- server/src/services/view.service.ts | 3 +- server/src/utils/asset.util.ts | 7 +- server/test/fixtures/album.stub.ts | 22 +-- server/test/fixtures/asset.stub.ts | 93 +++++++++--- server/test/fixtures/auth.stub.ts | 9 +- server/test/fixtures/shared-link.stub.ts | 31 ++-- .../specs/services/memory.service.spec.ts | 9 +- .../specs/services/metadata.service.spec.ts | 2 +- .../repositories/asset.repository.mock.ts | 2 +- server/test/small.factory.ts | 4 +- 44 files changed, 473 insertions(+), 396 deletions(-) delete mode 100644 server/src/entities/album.entity.ts delete mode 100644 server/src/entities/shared-link.entity.ts diff --git a/server/src/database.ts b/server/src/database.ts index b504d39579..6e66cb4d3d 100644 --- a/server/src/database.ts +++ b/server/src/database.ts @@ -1,13 +1,14 @@ import { Selectable } from 'kysely'; -import { AssetJobStatus as DatabaseAssetJobStatus, Exif as DatabaseExif } from 'src/db'; +import { Albums, AssetJobStatus as DatabaseAssetJobStatus, Exif as DatabaseExif } from 'src/db'; +import { MapAsset } from 'src/dtos/asset-response.dto'; import { AssetEntity } from 'src/entities/asset.entity'; import { AlbumUserRole, AssetFileType, - AssetStatus, AssetType, MemoryType, Permission, + SharedLinkType, SourceType, UserStatus, } from 'src/enum'; @@ -44,7 +45,7 @@ export type Library = { exclusionPatterns: string[]; deletedAt: Date | null; refreshedAt: Date | null; - assets?: Asset[]; + assets?: MapAsset[]; }; export type AuthApiKey = { @@ -96,7 +97,26 @@ export type Memory = { data: OnThisDayData; ownerId: string; isSaved: boolean; - assets: Asset[]; + assets: MapAsset[]; +}; + +export type Asset = { + id: string; + checksum: Buffer; + deviceAssetId: string; + deviceId: string; + fileCreatedAt: Date; + fileModifiedAt: Date; + isExternal: boolean; + isVisible: boolean; + libraryId: string | null; + livePhotoVideoId: string | null; + localDateTime: Date; + originalFileName: string; + originalPath: string; + ownerId: string; + sidecarPath: string | null; + type: AssetType; }; export type User = { @@ -128,39 +148,6 @@ export type StorageAsset = { encodedVideoPath: string | null; }; -export type Asset = { - createdAt: Date; - updatedAt: Date; - deletedAt: Date | null; - id: string; - updateId: string; - status: AssetStatus; - checksum: Buffer; - deviceAssetId: string; - deviceId: string; - duplicateId: string | null; - duration: string | null; - encodedVideoPath: string | null; - fileCreatedAt: Date | null; - fileModifiedAt: Date | null; - isArchived: boolean; - isExternal: boolean; - isFavorite: boolean; - isOffline: boolean; - isVisible: boolean; - libraryId: string | null; - livePhotoVideoId: string | null; - localDateTime: Date | null; - originalFileName: string; - originalPath: string; - ownerId: string; - sidecarPath: string | null; - stack?: Stack | null; - stackId: string | null; - thumbhash: Buffer | null; - type: AssetType; -}; - export type SidecarWriteAsset = { id: string; sidecarPath: string | null; @@ -173,7 +160,7 @@ export type Stack = { primaryAssetId: string; owner?: User; ownerId: string; - assets: AssetEntity[]; + assets: MapAsset[]; assetCount?: number; }; @@ -187,6 +174,28 @@ export type AuthSharedLink = { password: string | null; }; +export type SharedLink = { + id: string; + album?: Album | null; + albumId: string | null; + allowDownload: boolean; + allowUpload: boolean; + assets: MapAsset[]; + createdAt: Date; + description: string | null; + expiresAt: Date | null; + key: Buffer; + password: string | null; + showExif: boolean; + type: SharedLinkType; + userId: string; +}; + +export type Album = Selectable & { + owner: User; + assets: MapAsset[]; +}; + export type AuthSession = { id: string; }; diff --git a/server/src/db.d.ts b/server/src/db.d.ts index 7115b701ce..4e9738ecec 100644 --- a/server/src/db.d.ts +++ b/server/src/db.d.ts @@ -143,8 +143,8 @@ export interface Assets { duplicateId: string | null; duration: string | null; encodedVideoPath: Generated; - fileCreatedAt: Timestamp | null; - fileModifiedAt: Timestamp | null; + fileCreatedAt: Timestamp; + fileModifiedAt: Timestamp; id: Generated; isArchived: Generated; isExternal: Generated; @@ -153,7 +153,7 @@ export interface Assets { isVisible: Generated; libraryId: string | null; livePhotoVideoId: string | null; - localDateTime: Timestamp | null; + localDateTime: Timestamp; originalFileName: string; originalPath: string; ownerId: string; diff --git a/server/src/dtos/album.dto.ts b/server/src/dtos/album.dto.ts index c9934ec909..40e51ef729 100644 --- a/server/src/dtos/album.dto.ts +++ b/server/src/dtos/album.dto.ts @@ -2,10 +2,10 @@ import { ApiProperty } from '@nestjs/swagger'; import { Type } from 'class-transformer'; import { ArrayNotEmpty, IsArray, IsEnum, IsString, ValidateNested } from 'class-validator'; import _ from 'lodash'; -import { AssetResponseDto, mapAsset } from 'src/dtos/asset-response.dto'; +import { AlbumUser, AuthSharedLink, User } from 'src/database'; +import { AssetResponseDto, MapAsset, mapAsset } from 'src/dtos/asset-response.dto'; import { AuthDto } from 'src/dtos/auth.dto'; import { UserResponseDto, mapUser } from 'src/dtos/user.dto'; -import { AlbumEntity } from 'src/entities/album.entity'; import { AlbumUserRole, AssetOrder } from 'src/enum'; import { Optional, ValidateBoolean, ValidateUUID } from 'src/validation'; @@ -142,7 +142,23 @@ export class AlbumResponseDto { order?: AssetOrder; } -export const mapAlbum = (entity: AlbumEntity, withAssets: boolean, auth?: AuthDto): AlbumResponseDto => { +export type MapAlbumDto = { + albumUsers?: AlbumUser[]; + assets?: MapAsset[]; + sharedLinks?: AuthSharedLink[]; + albumName: string; + description: string; + albumThumbnailAssetId: string | null; + createdAt: Date; + updatedAt: Date; + id: string; + ownerId: string; + owner: User; + isActivityEnabled: boolean; + order: AssetOrder; +}; + +export const mapAlbum = (entity: MapAlbumDto, withAssets: boolean, auth?: AuthDto): AlbumResponseDto => { const albumUsers: AlbumUserResponseDto[] = []; if (entity.albumUsers) { @@ -159,7 +175,7 @@ export const mapAlbum = (entity: AlbumEntity, withAssets: boolean, auth?: AuthDt const assets = entity.assets || []; - const hasSharedLink = entity.sharedLinks?.length > 0; + const hasSharedLink = !!entity.sharedLinks && entity.sharedLinks.length > 0; const hasSharedUser = albumUsers.length > 0; let startDate = assets.at(0)?.localDateTime; @@ -190,5 +206,5 @@ export const mapAlbum = (entity: AlbumEntity, withAssets: boolean, auth?: AuthDt }; }; -export const mapAlbumWithAssets = (entity: AlbumEntity) => mapAlbum(entity, true); -export const mapAlbumWithoutAssets = (entity: AlbumEntity) => mapAlbum(entity, false); +export const mapAlbumWithAssets = (entity: MapAlbumDto) => mapAlbum(entity, true); +export const mapAlbumWithoutAssets = (entity: MapAlbumDto) => mapAlbum(entity, false); diff --git a/server/src/dtos/asset-response.dto.ts b/server/src/dtos/asset-response.dto.ts index 985ad04729..c0e589f380 100644 --- a/server/src/dtos/asset-response.dto.ts +++ b/server/src/dtos/asset-response.dto.ts @@ -1,5 +1,6 @@ import { ApiProperty } from '@nestjs/swagger'; -import { AssetFace } from 'src/database'; +import { Selectable } from 'kysely'; +import { AssetFace, AssetFile, Exif, Stack, Tag, User } from 'src/database'; import { PropertyLifecycle } from 'src/decorators'; import { AuthDto } from 'src/dtos/auth.dto'; import { ExifResponseDto, mapExif } from 'src/dtos/exif.dto'; @@ -11,8 +12,7 @@ import { } from 'src/dtos/person.dto'; import { TagResponseDto, mapTag } from 'src/dtos/tag.dto'; import { UserResponseDto, mapUser } from 'src/dtos/user.dto'; -import { AssetEntity } from 'src/entities/asset.entity'; -import { AssetType } from 'src/enum'; +import { AssetStatus, AssetType } from 'src/enum'; import { mimeTypes } from 'src/utils/mime-types'; export class SanitizedAssetResponseDto { @@ -56,6 +56,44 @@ export class AssetResponseDto extends SanitizedAssetResponseDto { resized?: boolean; } +export type MapAsset = { + createdAt: Date; + updatedAt: Date; + deletedAt: Date | null; + id: string; + updateId: string; + status: AssetStatus; + checksum: Buffer; + deviceAssetId: string; + deviceId: string; + duplicateId: string | null; + duration: string | null; + encodedVideoPath: string | null; + exifInfo?: Selectable | null; + faces?: AssetFace[]; + fileCreatedAt: Date; + fileModifiedAt: Date; + files?: AssetFile[]; + isArchived: boolean; + isExternal: boolean; + isFavorite: boolean; + isOffline: boolean; + isVisible: boolean; + libraryId: string | null; + livePhotoVideoId: string | null; + localDateTime: Date; + originalFileName: string; + originalPath: string; + owner?: User | null; + ownerId: string; + sidecarPath: string | null; + stack?: Stack | null; + stackId: string | null; + tags?: Tag[]; + thumbhash: Buffer | null; + type: AssetType; +}; + export class AssetStackResponseDto { id!: string; @@ -72,7 +110,7 @@ export type AssetMapOptions = { }; // TODO: this is inefficient -const peopleWithFaces = (faces: AssetFace[]): PersonWithFacesResponseDto[] => { +const peopleWithFaces = (faces?: AssetFace[]): PersonWithFacesResponseDto[] => { const result: PersonWithFacesResponseDto[] = []; if (faces) { for (const face of faces) { @@ -90,7 +128,7 @@ const peopleWithFaces = (faces: AssetFace[]): PersonWithFacesResponseDto[] => { return result; }; -const mapStack = (entity: AssetEntity) => { +const mapStack = (entity: { stack?: Stack | null }) => { if (!entity.stack) { return null; } @@ -111,7 +149,7 @@ export const hexOrBufferToBase64 = (encoded: string | Buffer) => { return encoded.toString('base64'); }; -export function mapAsset(entity: AssetEntity, options: AssetMapOptions = {}): AssetResponseDto { +export function mapAsset(entity: MapAsset, options: AssetMapOptions = {}): AssetResponseDto { const { stripMetadata = false, withStack = false } = options; if (stripMetadata) { diff --git a/server/src/dtos/memory.dto.ts b/server/src/dtos/memory.dto.ts index b3054d7a4c..98231a9035 100644 --- a/server/src/dtos/memory.dto.ts +++ b/server/src/dtos/memory.dto.ts @@ -4,7 +4,6 @@ import { IsEnum, IsInt, IsObject, IsPositive, ValidateNested } from 'class-valid import { Memory } from 'src/database'; import { AssetResponseDto, mapAsset } from 'src/dtos/asset-response.dto'; import { AuthDto } from 'src/dtos/auth.dto'; -import { AssetEntity } from 'src/entities/asset.entity'; import { MemoryType } from 'src/enum'; import { Optional, ValidateBoolean, ValidateDate, ValidateUUID } from 'src/validation'; @@ -103,6 +102,6 @@ export const mapMemory = (entity: Memory, auth: AuthDto): MemoryResponseDto => { type: entity.type as MemoryType, data: entity.data as unknown as MemoryData, isSaved: entity.isSaved, - assets: ('assets' in entity ? entity.assets : []).map((asset) => mapAsset(asset as AssetEntity, { auth })), + assets: ('assets' in entity ? entity.assets : []).map((asset) => mapAsset(asset, { auth })), }; }; diff --git a/server/src/dtos/shared-link.dto.ts b/server/src/dtos/shared-link.dto.ts index 6bb8ab1f0d..8d373b40b6 100644 --- a/server/src/dtos/shared-link.dto.ts +++ b/server/src/dtos/shared-link.dto.ts @@ -1,9 +1,9 @@ import { ApiProperty } from '@nestjs/swagger'; import { IsEnum, IsString } from 'class-validator'; import _ from 'lodash'; +import { SharedLink } from 'src/database'; import { AlbumResponseDto, mapAlbumWithoutAssets } from 'src/dtos/album.dto'; import { AssetResponseDto, mapAsset } from 'src/dtos/asset-response.dto'; -import { SharedLinkEntity } from 'src/entities/shared-link.entity'; import { SharedLinkType } from 'src/enum'; import { Optional, ValidateBoolean, ValidateDate, ValidateUUID } from 'src/validation'; @@ -102,7 +102,7 @@ export class SharedLinkResponseDto { showMetadata!: boolean; } -export function mapSharedLink(sharedLink: SharedLinkEntity): SharedLinkResponseDto { +export function mapSharedLink(sharedLink: SharedLink): SharedLinkResponseDto { const linkAssets = sharedLink.assets || []; return { @@ -122,7 +122,7 @@ export function mapSharedLink(sharedLink: SharedLinkEntity): SharedLinkResponseD }; } -export function mapSharedLinkWithoutMetadata(sharedLink: SharedLinkEntity): SharedLinkResponseDto { +export function mapSharedLinkWithoutMetadata(sharedLink: SharedLink): SharedLinkResponseDto { const linkAssets = sharedLink.assets || []; const albumAssets = (sharedLink?.album?.assets || []).map((asset) => asset); @@ -137,7 +137,7 @@ export function mapSharedLinkWithoutMetadata(sharedLink: SharedLinkEntity): Shar type: sharedLink.type, createdAt: sharedLink.createdAt, expiresAt: sharedLink.expiresAt, - assets: assets.map((asset) => mapAsset(asset, { stripMetadata: true })) as AssetResponseDto[], + assets: assets.map((asset) => mapAsset(asset, { stripMetadata: true })), album: sharedLink.album ? mapAlbumWithoutAssets(sharedLink.album) : undefined, allowUpload: sharedLink.allowUpload, allowDownload: sharedLink.allowDownload, diff --git a/server/src/entities/album.entity.ts b/server/src/entities/album.entity.ts deleted file mode 100644 index eb20c1afdd..0000000000 --- a/server/src/entities/album.entity.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { AlbumUser, User } from 'src/database'; -import { AssetEntity } from 'src/entities/asset.entity'; -import { SharedLinkEntity } from 'src/entities/shared-link.entity'; -import { AssetOrder } from 'src/enum'; - -export class AlbumEntity { - id!: string; - owner!: User; - ownerId!: string; - albumName!: string; - description!: string; - createdAt!: Date; - updatedAt!: Date; - updateId?: string; - deletedAt!: Date | null; - albumThumbnailAsset!: AssetEntity | null; - albumThumbnailAssetId!: string | null; - albumUsers!: AlbumUser[]; - assets!: AssetEntity[]; - sharedLinks!: SharedLinkEntity[]; - isActivityEnabled!: boolean; - order!: AssetOrder; -} diff --git a/server/src/entities/asset.entity.ts b/server/src/entities/asset.entity.ts index 64c038a689..da291292e7 100644 --- a/server/src/entities/asset.entity.ts +++ b/server/src/entities/asset.entity.ts @@ -1,12 +1,12 @@ import { DeduplicateJoinsPlugin, ExpressionBuilder, Kysely, SelectQueryBuilder, sql } from 'kysely'; import { jsonArrayFrom, jsonObjectFrom } from 'kysely/helpers/postgres'; -import { AssetFace, AssetFile, AssetJobStatus, columns, Exif, Stack, Tag, User } from 'src/database'; +import { AssetFace, AssetFile, AssetJobStatus, columns, Exif, Person, Stack, Tag, User } from 'src/database'; import { DB } from 'src/db'; -import { SharedLinkEntity } from 'src/entities/shared-link.entity'; +import { MapAsset } from 'src/dtos/asset-response.dto'; import { AssetFileType, AssetStatus, AssetType } from 'src/enum'; import { TimeBucketSize } from 'src/repositories/asset.repository'; import { AssetSearchBuilderOptions } from 'src/repositories/search.repository'; -import { anyUuid, asUuid } from 'src/utils/database'; +import { anyUuid, asUuid, toJson } from 'src/utils/database'; export const ASSET_CHECKSUM_CONSTRAINT = 'UQ_assets_owner_checksum'; @@ -37,13 +37,12 @@ export class AssetEntity { checksum!: Buffer; // sha1 checksum duration!: string | null; isVisible!: boolean; - livePhotoVideo!: AssetEntity | null; + livePhotoVideo!: MapAsset | null; livePhotoVideoId!: string | null; originalFileName!: string; sidecarPath!: string | null; exifInfo?: Exif; tags?: Tag[]; - sharedLinks!: SharedLinkEntity[]; faces!: AssetFace[]; stackId?: string | null; stack?: Stack | null; @@ -51,6 +50,7 @@ export class AssetEntity { duplicateId!: string | null; } +// TODO come up with a better query that only selects the fields we need export function withExif(qb: SelectQueryBuilder) { return qb .leftJoin('exif', 'assets.id', 'exif.assetId') @@ -66,7 +66,7 @@ export function withExifInner(qb: SelectQueryBuilder) { export function withSmartSearch(qb: SelectQueryBuilder) { return qb .leftJoin('smart_search', 'assets.id', 'smart_search.assetId') - .select((eb) => eb.fn.toJson(eb.table('smart_search')).as('smartSearch')); + .select((eb) => toJson(eb, 'smart_search').as('smartSearch')); } export function withFaces(eb: ExpressionBuilder, withDeletedFace?: boolean) { @@ -99,7 +99,7 @@ export function withFacesAndPeople(eb: ExpressionBuilder, withDele (join) => join.onTrue(), ) .selectAll('asset_faces') - .select((eb) => eb.table('person').as('person')) + .select((eb) => eb.table('person').$castTo().as('person')) .whereRef('asset_faces.assetId', '=', 'assets.id') .$if(!withDeletedFace, (qb) => qb.where('asset_faces.deletedAt', 'is', null)), ).as('faces'); @@ -136,13 +136,15 @@ export function hasTags(qb: SelectQueryBuilder, tagIds: stri } export function withOwner(eb: ExpressionBuilder) { - return jsonObjectFrom(eb.selectFrom('users').selectAll().whereRef('users.id', '=', 'assets.ownerId')).as('owner'); + return jsonObjectFrom(eb.selectFrom('users').select(columns.user).whereRef('users.id', '=', 'assets.ownerId')).as( + 'owner', + ); } export function withLibrary(eb: ExpressionBuilder) { - return jsonObjectFrom(eb.selectFrom('libraries').selectAll().whereRef('libraries.id', '=', 'assets.libraryId')).as( - 'library', - ); + return jsonObjectFrom( + eb.selectFrom('libraries').selectAll('libraries').whereRef('libraries.id', '=', 'assets.libraryId'), + ).as('library'); } export function withTags(eb: ExpressionBuilder) { diff --git a/server/src/entities/shared-link.entity.ts b/server/src/entities/shared-link.entity.ts deleted file mode 100644 index 720ba424d1..0000000000 --- a/server/src/entities/shared-link.entity.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { AlbumEntity } from 'src/entities/album.entity'; -import { AssetEntity } from 'src/entities/asset.entity'; -import { SharedLinkType } from 'src/enum'; - -export class SharedLinkEntity { - id!: string; - description!: string | null; - password!: string | null; - userId!: string; - key!: Buffer; // use to access the inidividual asset - type!: SharedLinkType; - createdAt!: Date; - expiresAt!: Date | null; - allowUpload!: boolean; - allowDownload!: boolean; - showExif!: boolean; - assets!: AssetEntity[]; - album?: AlbumEntity; - albumId!: string | null; -} diff --git a/server/src/queries/asset.repository.sql b/server/src/queries/asset.repository.sql index cf17bb0276..a3dcb08c1e 100644 --- a/server/src/queries/asset.repository.sql +++ b/server/src/queries/asset.repository.sql @@ -82,7 +82,7 @@ from where "assets"."id" = any ($1::uuid[]) --- AssetRepository.getByIdsWithAllRelations +-- AssetRepository.getByIdsWithAllRelationsButStacks select "assets".*, ( @@ -127,28 +127,13 @@ select "assets"."id" = "tag_asset"."assetsId" ) as agg ) as "tags", - to_json("exif") as "exifInfo", - to_json("stacked_assets") as "stack" + to_json("exif") as "exifInfo" from "assets" left join "exif" on "assets"."id" = "exif"."assetId" left join "asset_stack" on "asset_stack"."id" = "assets"."stackId" - left join lateral ( - select - "asset_stack".*, - array_agg("stacked") as "assets" - from - "assets" as "stacked" - where - "stacked"."stackId" = "asset_stack"."id" - and "stacked"."id" != "asset_stack"."primaryAssetId" - and "stacked"."deletedAt" is null - and "stacked"."isArchived" = $1 - group by - "asset_stack"."id" - ) as "stacked_assets" on "asset_stack"."id" is not null where - "assets"."id" = any ($2::uuid[]) + "assets"."id" = any ($1::uuid[]) -- AssetRepository.deleteAll delete from "assets" diff --git a/server/src/repositories/album.repository.ts b/server/src/repositories/album.repository.ts index e21d5d73cd..1768135210 100644 --- a/server/src/repositories/album.repository.ts +++ b/server/src/repositories/album.repository.ts @@ -1,12 +1,11 @@ import { Injectable } from '@nestjs/common'; -import { ExpressionBuilder, Insertable, Kysely, sql, Updateable } from 'kysely'; +import { ExpressionBuilder, Insertable, Kysely, NotNull, sql, Updateable } from 'kysely'; import { jsonArrayFrom, jsonObjectFrom } from 'kysely/helpers/postgres'; import { InjectKysely } from 'nestjs-kysely'; -import { columns } from 'src/database'; +import { columns, Exif } from 'src/database'; import { Albums, DB } from 'src/db'; import { Chunked, ChunkedArray, ChunkedSet, DummyValue, GenerateSql } from 'src/decorators'; import { AlbumUserCreateDto } from 'src/dtos/album.dto'; -import { AlbumEntity } from 'src/entities/album.entity'; export interface AlbumAssetCount { albumId: string; @@ -21,9 +20,9 @@ export interface AlbumInfoOptions { } const withOwner = (eb: ExpressionBuilder) => { - return jsonObjectFrom(eb.selectFrom('users').select(columns.user).whereRef('users.id', '=', 'albums.ownerId')).as( - 'owner', - ); + return jsonObjectFrom(eb.selectFrom('users').select(columns.user).whereRef('users.id', '=', 'albums.ownerId')) + .$notNull() + .as('owner'); }; const withAlbumUsers = (eb: ExpressionBuilder) => { @@ -32,12 +31,14 @@ const withAlbumUsers = (eb: ExpressionBuilder) => { .selectFrom('albums_shared_users_users as album_users') .select('album_users.role') .select((eb) => - jsonObjectFrom(eb.selectFrom('users').select(columns.user).whereRef('users.id', '=', 'album_users.usersId')).as( - 'user', - ), + jsonObjectFrom(eb.selectFrom('users').select(columns.user).whereRef('users.id', '=', 'album_users.usersId')) + .$notNull() + .as('user'), ) .whereRef('album_users.albumsId', '=', 'albums.id'), - ).as('albumUsers'); + ) + .$notNull() + .as('albumUsers'); }; const withSharedLink = (eb: ExpressionBuilder) => { @@ -53,7 +54,7 @@ const withAssets = (eb: ExpressionBuilder) => { .selectFrom('assets') .selectAll('assets') .leftJoin('exif', 'assets.id', 'exif.assetId') - .select((eb) => eb.table('exif').as('exifInfo')) + .select((eb) => eb.table('exif').$castTo().as('exifInfo')) .innerJoin('albums_assets_assets', 'albums_assets_assets.assetsId', 'assets.id') .whereRef('albums_assets_assets.albumsId', '=', 'albums.id') .where('assets.deletedAt', 'is', null) @@ -69,7 +70,7 @@ export class AlbumRepository { constructor(@InjectKysely() private db: Kysely) {} @GenerateSql({ params: [DummyValue.UUID, { withAssets: true }] }) - async getById(id: string, options: AlbumInfoOptions): Promise { + async getById(id: string, options: AlbumInfoOptions) { return this.db .selectFrom('albums') .selectAll('albums') @@ -79,11 +80,12 @@ export class AlbumRepository { .select(withAlbumUsers) .select(withSharedLink) .$if(options.withAssets, (eb) => eb.select(withAssets)) - .executeTakeFirst() as Promise; + .$narrowType<{ assets: NotNull }>() + .executeTakeFirst(); } @GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID] }) - async getByAssetId(ownerId: string, assetId: string): Promise { + async getByAssetId(ownerId: string, assetId: string) { return this.db .selectFrom('albums') .selectAll('albums') @@ -105,7 +107,7 @@ export class AlbumRepository { .select(withOwner) .select(withAlbumUsers) .orderBy('albums.createdAt', 'desc') - .execute() as unknown as Promise; + .execute(); } @GenerateSql({ params: [[DummyValue.UUID]] }) @@ -134,7 +136,7 @@ export class AlbumRepository { } @GenerateSql({ params: [DummyValue.UUID] }) - async getOwned(ownerId: string): Promise { + async getOwned(ownerId: string) { return this.db .selectFrom('albums') .selectAll('albums') @@ -144,14 +146,14 @@ export class AlbumRepository { .where('albums.ownerId', '=', ownerId) .where('albums.deletedAt', 'is', null) .orderBy('albums.createdAt', 'desc') - .execute() as unknown as Promise; + .execute(); } /** * Get albums shared with and shared by owner. */ @GenerateSql({ params: [DummyValue.UUID] }) - async getShared(ownerId: string): Promise { + async getShared(ownerId: string) { return this.db .selectFrom('albums') .selectAll('albums') @@ -176,14 +178,14 @@ export class AlbumRepository { .select(withOwner) .select(withSharedLink) .orderBy('albums.createdAt', 'desc') - .execute() as unknown as Promise; + .execute(); } /** * Get albums of owner that are _not_ shared */ @GenerateSql({ params: [DummyValue.UUID] }) - async getNotShared(ownerId: string): Promise { + async getNotShared(ownerId: string) { return this.db .selectFrom('albums') .selectAll('albums') @@ -203,7 +205,7 @@ export class AlbumRepository { ) .select(withOwner) .orderBy('albums.createdAt', 'desc') - .execute() as unknown as Promise; + .execute(); } async restoreAll(userId: string): Promise { @@ -262,7 +264,7 @@ export class AlbumRepository { await this.addAssets(this.db, albumId, assetIds); } - create(album: Insertable, assetIds: string[], albumUsers: AlbumUserCreateDto[]): Promise { + create(album: Insertable, assetIds: string[], albumUsers: AlbumUserCreateDto[]) { return this.db.transaction().execute(async (tx) => { const newAlbum = await tx.insertInto('albums').values(album).returning('albums.id').executeTakeFirst(); @@ -290,11 +292,12 @@ export class AlbumRepository { .select(withOwner) .select(withAssets) .select(withAlbumUsers) - .executeTakeFirst() as unknown as Promise; + .$narrowType<{ assets: NotNull }>() + .executeTakeFirstOrThrow(); }); } - update(id: string, album: Updateable): Promise { + update(id: string, album: Updateable) { return this.db .updateTable('albums') .set(album) @@ -303,7 +306,7 @@ export class AlbumRepository { .returning(withOwner) .returning(withSharedLink) .returning(withAlbumUsers) - .executeTakeFirst() as unknown as Promise; + .executeTakeFirstOrThrow(); } async delete(id: string): Promise { diff --git a/server/src/repositories/asset-job.repository.ts b/server/src/repositories/asset-job.repository.ts index b507fa5445..1bf08e81f5 100644 --- a/server/src/repositories/asset-job.repository.ts +++ b/server/src/repositories/asset-job.repository.ts @@ -8,7 +8,7 @@ import { DummyValue, GenerateSql } from 'src/decorators'; import { withExifInner, withFaces, withFiles } from 'src/entities/asset.entity'; import { AssetFileType } from 'src/enum'; import { StorageAsset } from 'src/types'; -import { asUuid } from 'src/utils/database'; +import { anyUuid, asUuid } from 'src/utils/database'; @Injectable() export class AssetJobRepository { @@ -149,6 +149,21 @@ export class AssetJobRepository { .executeTakeFirst(); } + getForSyncAssets(ids: string[]) { + return this.db + .selectFrom('assets') + .select([ + 'assets.id', + 'assets.isOffline', + 'assets.libraryId', + 'assets.originalPath', + 'assets.status', + 'assets.fileModifiedAt', + ]) + .where('assets.id', '=', anyUuid(ids)) + .execute(); + } + private storageTemplateAssetQuery() { return this.db .selectFrom('assets') diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts index af79fb7c5f..7fb7056ba7 100644 --- a/server/src/repositories/asset.repository.ts +++ b/server/src/repositories/asset.repository.ts @@ -1,9 +1,11 @@ import { Injectable } from '@nestjs/common'; -import { Insertable, Kysely, Selectable, UpdateResult, Updateable, sql } from 'kysely'; +import { Insertable, Kysely, NotNull, Selectable, UpdateResult, Updateable, sql } from 'kysely'; import { isEmpty, isUndefined, omitBy } from 'lodash'; import { InjectKysely } from 'nestjs-kysely'; +import { Stack } from 'src/database'; import { AssetFiles, AssetJobStatus, Assets, DB, Exif } from 'src/db'; import { Chunked, ChunkedArray, DummyValue, GenerateSql } from 'src/decorators'; +import { MapAsset } from 'src/dtos/asset-response.dto'; import { AssetEntity, hasPeople, @@ -23,7 +25,7 @@ import { AssetFileType, AssetOrder, AssetStatus, AssetType } from 'src/enum'; import { AssetSearchOptions, SearchExploreItem, SearchExploreItemSet } from 'src/repositories/search.repository'; import { anyUuid, asUuid, removeUndefinedKeys, unnest } from 'src/utils/database'; import { globToSqlPattern } from 'src/utils/misc'; -import { Paginated, PaginationOptions, paginationHelper } from 'src/utils/pagination'; +import { PaginationOptions, paginationHelper } from 'src/utils/pagination'; export type AssetStats = Record; @@ -141,12 +143,12 @@ export interface GetByIdsRelations { export interface DuplicateGroup { duplicateId: string; - assets: AssetEntity[]; + assets: MapAsset[]; } export interface DayOfYearAssets { yearsAgo: number; - assets: AssetEntity[]; + assets: MapAsset[]; } @Injectable() @@ -234,12 +236,12 @@ export class AssetRepository { .execute(); } - create(asset: Insertable): Promise { - return this.db.insertInto('assets').values(asset).returningAll().executeTakeFirst() as any as Promise; + create(asset: Insertable) { + return this.db.insertInto('assets').values(asset).returningAll().executeTakeFirstOrThrow(); } - createAll(assets: Insertable[]): Promise { - return this.db.insertInto('assets').values(assets).returningAll().execute() as any as Promise; + createAll(assets: Insertable[]) { + return this.db.insertInto('assets').values(assets).returningAll().execute(); } @GenerateSql({ params: [DummyValue.UUID, { day: 1, month: 1 }] }) @@ -299,20 +301,13 @@ export class AssetRepository { @GenerateSql({ params: [[DummyValue.UUID]] }) @ChunkedArray() - getByIds(ids: string[]): Promise { - return ( - this.db - // - .selectFrom('assets') - .selectAll('assets') - .where('assets.id', '=', anyUuid(ids)) - .execute() as Promise - ); + getByIds(ids: string[]) { + return this.db.selectFrom('assets').selectAll('assets').where('assets.id', '=', anyUuid(ids)).execute(); } @GenerateSql({ params: [[DummyValue.UUID]] }) @ChunkedArray() - getByIdsWithAllRelations(ids: string[]): Promise { + getByIdsWithAllRelationsButStacks(ids: string[]) { return this.db .selectFrom('assets') .selectAll('assets') @@ -320,23 +315,8 @@ export class AssetRepository { .select(withTags) .$call(withExif) .leftJoin('asset_stack', 'asset_stack.id', 'assets.stackId') - .leftJoinLateral( - (eb) => - eb - .selectFrom('assets as stacked') - .selectAll('asset_stack') - .select((eb) => eb.fn('array_agg', [eb.table('stacked')]).as('assets')) - .whereRef('stacked.stackId', '=', 'asset_stack.id') - .whereRef('stacked.id', '!=', 'asset_stack.primaryAssetId') - .where('stacked.deletedAt', 'is', null) - .where('stacked.isArchived', '=', false) - .groupBy('asset_stack.id') - .as('stacked_assets'), - (join) => join.on('asset_stack.id', 'is not', null), - ) - .select((eb) => eb.fn.toJson(eb.table('stacked_assets')).as('stack')) .where('assets.id', '=', anyUuid(ids)) - .execute() as any as Promise; + .execute(); } @GenerateSql({ params: [DummyValue.UUID] }) @@ -356,36 +336,29 @@ export class AssetRepository { return assets.map((asset) => asset.deviceAssetId); } - getByUserId( - pagination: PaginationOptions, - userId: string, - options: Omit = {}, - ): Paginated { + getByUserId(pagination: PaginationOptions, userId: string, options: Omit = {}) { return this.getAll(pagination, { ...options, userIds: [userId] }); } @GenerateSql({ params: [DummyValue.UUID, DummyValue.STRING] }) - getByLibraryIdAndOriginalPath(libraryId: string, originalPath: string): Promise { + getByLibraryIdAndOriginalPath(libraryId: string, originalPath: string) { return this.db .selectFrom('assets') .selectAll('assets') .where('libraryId', '=', asUuid(libraryId)) .where('originalPath', '=', originalPath) .limit(1) - .executeTakeFirst() as any as Promise; + .executeTakeFirst(); } - async getAll( - pagination: PaginationOptions, - { orderDirection, ...options }: AssetSearchOptions = {}, - ): Paginated { + async getAll(pagination: PaginationOptions, { orderDirection, ...options }: AssetSearchOptions = {}) { const builder = searchAssetBuilder(this.db, options) .select(withFiles) .orderBy('assets.createdAt', orderDirection ?? 'asc') .limit(pagination.take + 1) .offset(pagination.skip ?? 0); const items = await builder.execute(); - return paginationHelper(items as any as AssetEntity[], pagination.take); + return paginationHelper(items, pagination.take); } /** @@ -420,23 +393,22 @@ export class AssetRepository { } @GenerateSql({ params: [DummyValue.UUID] }) - getById( - id: string, - { exifInfo, faces, files, library, owner, smartSearch, stack, tags }: GetByIdsRelations = {}, - ): Promise { + getById(id: string, { exifInfo, faces, files, library, owner, smartSearch, stack, tags }: GetByIdsRelations = {}) { return this.db .selectFrom('assets') .selectAll('assets') .where('assets.id', '=', asUuid(id)) .$if(!!exifInfo, withExif) - .$if(!!faces, (qb) => qb.select(faces?.person ? withFacesAndPeople : withFaces)) + .$if(!!faces, (qb) => qb.select(faces?.person ? withFacesAndPeople : withFaces).$narrowType<{ faces: NotNull }>()) .$if(!!library, (qb) => qb.select(withLibrary)) .$if(!!owner, (qb) => qb.select(withOwner)) .$if(!!smartSearch, withSmartSearch) .$if(!!stack, (qb) => qb .leftJoin('asset_stack', 'asset_stack.id', 'assets.stackId') - .$if(!stack!.assets, (qb) => qb.select((eb) => eb.fn.toJson(eb.table('asset_stack')).as('stack'))) + .$if(!stack!.assets, (qb) => + qb.select((eb) => eb.fn.toJson(eb.table('asset_stack')).$castTo().as('stack')), + ) .$if(!!stack!.assets, (qb) => qb .leftJoinLateral( @@ -453,13 +425,13 @@ export class AssetRepository { .as('stacked_assets'), (join) => join.on('asset_stack.id', 'is not', null), ) - .select((eb) => eb.fn.toJson(eb.table('stacked_assets')).as('stack')), + .select((eb) => eb.fn.toJson(eb.table('stacked_assets')).$castTo().as('stack')), ), ) .$if(!!files, (qb) => qb.select(withFiles)) .$if(!!tags, (qb) => qb.select(withTags)) .limit(1) - .executeTakeFirst() as any as Promise; + .executeTakeFirst(); } @GenerateSql({ params: [[DummyValue.UUID], { deviceId: DummyValue.STRING }] }) @@ -488,7 +460,7 @@ export class AssetRepository { .execute(); } - async update(asset: Updateable & { id: string }): Promise { + async update(asset: Updateable & { id: string }) { const value = omitBy(asset, isUndefined); delete value.id; if (!isEmpty(value)) { @@ -498,10 +470,10 @@ export class AssetRepository { .selectAll('assets') .$call(withExif) .$call((qb) => qb.select(withFacesAndPeople)) - .executeTakeFirst() as Promise; + .executeTakeFirst(); } - return this.getById(asset.id, { exifInfo: true, faces: { person: true } }) as Promise; + return this.getById(asset.id, { exifInfo: true, faces: { person: true } }); } async remove(asset: { id: string }): Promise { @@ -509,7 +481,7 @@ export class AssetRepository { } @GenerateSql({ params: [{ ownerId: DummyValue.UUID, libraryId: DummyValue.UUID, checksum: DummyValue.BUFFER }] }) - getByChecksum({ ownerId, libraryId, checksum }: AssetGetByChecksumOptions): Promise { + getByChecksum({ ownerId, libraryId, checksum }: AssetGetByChecksumOptions) { return this.db .selectFrom('assets') .selectAll('assets') @@ -517,7 +489,7 @@ export class AssetRepository { .where('checksum', '=', checksum) .$call((qb) => (libraryId ? qb.where('libraryId', '=', asUuid(libraryId)) : qb.where('libraryId', 'is', null))) .limit(1) - .executeTakeFirst() as Promise; + .executeTakeFirst(); } @GenerateSql({ params: [DummyValue.UUID, [DummyValue.BUFFER]] }) @@ -544,7 +516,7 @@ export class AssetRepository { return asset?.id; } - findLivePhotoMatch(options: LivePhotoSearchOptions): Promise { + findLivePhotoMatch(options: LivePhotoSearchOptions) { const { ownerId, otherAssetId, livePhotoCID, type } = options; return this.db .selectFrom('assets') @@ -555,7 +527,7 @@ export class AssetRepository { .where('type', '=', type) .where('exif.livePhotoCID', '=', livePhotoCID) .limit(1) - .executeTakeFirst() as Promise; + .executeTakeFirst(); } @GenerateSql( @@ -564,7 +536,7 @@ export class AssetRepository { params: [DummyValue.PAGINATION, property], })), ) - async getWithout(pagination: PaginationOptions, property: WithoutProperty): Paginated { + async getWithout(pagination: PaginationOptions, property: WithoutProperty) { const items = await this.db .selectFrom('assets') .selectAll('assets') @@ -626,7 +598,7 @@ export class AssetRepository { .orderBy('createdAt') .execute(); - return paginationHelper(items as any as AssetEntity[], pagination.take); + return paginationHelper(items, pagination.take); } getStatistics(ownerId: string, { isArchived, isFavorite, isTrashed }: AssetStatsOptions): Promise { @@ -645,7 +617,7 @@ export class AssetRepository { .executeTakeFirstOrThrow(); } - getRandom(userIds: string[], take: number): Promise { + getRandom(userIds: string[], take: number) { return this.db .selectFrom('assets') .selectAll('assets') @@ -655,7 +627,7 @@ export class AssetRepository { .where('deletedAt', 'is', null) .orderBy((eb) => eb.fn('random')) .limit(take) - .execute() as any as Promise; + .execute(); } @GenerateSql({ params: [{ size: TimeBucketSize.MONTH }] }) @@ -708,7 +680,7 @@ export class AssetRepository { } @GenerateSql({ params: [DummyValue.TIME_BUCKET, { size: TimeBucketSize.MONTH, withStacked: true }] }) - async getTimeBucket(timeBucket: string, options: TimeBucketOptions): Promise { + async getTimeBucket(timeBucket: string, options: TimeBucketOptions) { return this.db .selectFrom('assets') .selectAll('assets') @@ -741,7 +713,7 @@ export class AssetRepository { .as('stacked_assets'), (join) => join.on('asset_stack.id', 'is not', null), ) - .select((eb) => eb.fn.toJson(eb.table('stacked_assets')).as('stack')), + .select((eb) => eb.fn.toJson(eb.table('stacked_assets').$castTo()).as('stack')), ) .$if(!!options.assetType, (qb) => qb.where('assets.type', '=', options.assetType!)) .$if(options.isDuplicate !== undefined, (qb) => @@ -753,11 +725,11 @@ export class AssetRepository { .where('assets.isVisible', '=', true) .where(truncatedDate(options.size), '=', timeBucket.replace(/^[+-]/, '')) .orderBy('assets.localDateTime', options.order ?? 'desc') - .execute() as any as Promise; + .execute(); } @GenerateSql({ params: [DummyValue.UUID] }) - getDuplicates(userId: string): Promise { + getDuplicates(userId: string) { return ( this.db .with('duplicates', (qb) => @@ -774,9 +746,15 @@ export class AssetRepository { (join) => join.onTrue(), ) .select('assets.duplicateId') - .select((eb) => eb.fn('jsonb_agg', [eb.table('asset')]).as('assets')) + .select((eb) => + eb + .fn('jsonb_agg', [eb.table('asset')]) + .$castTo() + .as('assets'), + ) .where('assets.ownerId', '=', asUuid(userId)) .where('assets.duplicateId', 'is not', null) + .$narrowType<{ duplicateId: NotNull }>() .where('assets.deletedAt', 'is', null) .where('assets.isVisible', '=', true) .where('assets.stackId', 'is', null) @@ -801,7 +779,7 @@ export class AssetRepository { .where(({ not, exists }) => not(exists((eb) => eb.selectFrom('unique').whereRef('unique.duplicateId', '=', 'duplicates.duplicateId'))), ) - .execute() as any as Promise + .execute() ); } @@ -845,7 +823,7 @@ export class AssetRepository { }, ], }) - getAllForUserFullSync(options: AssetFullSyncOptions): Promise { + getAllForUserFullSync(options: AssetFullSyncOptions) { const { ownerId, lastId, updatedUntil, limit } = options; return this.db .selectFrom('assets') @@ -863,18 +841,18 @@ export class AssetRepository { .as('stacked_assets'), (join) => join.on('asset_stack.id', 'is not', null), ) - .select((eb) => eb.fn.toJson(eb.table('stacked_assets')).as('stack')) + .select((eb) => eb.fn.toJson(eb.table('stacked_assets')).$castTo().as('stack')) .where('assets.ownerId', '=', asUuid(ownerId)) .where('assets.isVisible', '=', true) .where('assets.updatedAt', '<=', updatedUntil) .$if(!!lastId, (qb) => qb.where('assets.id', '>', lastId!)) .orderBy('assets.id') .limit(limit) - .execute() as any as Promise; + .execute(); } @GenerateSql({ params: [{ userIds: [DummyValue.UUID], updatedAfter: DummyValue.DATE, limit: 100 }] }) - async getChangedDeltaSync(options: AssetDeltaSyncOptions): Promise { + async getChangedDeltaSync(options: AssetDeltaSyncOptions) { return this.db .selectFrom('assets') .selectAll('assets') @@ -891,12 +869,12 @@ export class AssetRepository { .as('stacked_assets'), (join) => join.on('asset_stack.id', 'is not', null), ) - .select((eb) => eb.fn.toJson(eb.table('stacked_assets')).as('stack')) + .select((eb) => eb.fn.toJson(eb.table('stacked_assets').$castTo()).as('stack')) .where('assets.ownerId', '=', anyUuid(options.userIds)) .where('assets.isVisible', '=', true) .where('assets.updatedAt', '>', options.updatedAfter) .limit(options.limit) - .execute() as any as Promise; + .execute(); } async upsertFile(file: Pick, 'assetId' | 'path' | 'type'>): Promise { diff --git a/server/src/repositories/search.repository.ts b/server/src/repositories/search.repository.ts index c86ae8f60e..95c350fe34 100644 --- a/server/src/repositories/search.repository.ts +++ b/server/src/repositories/search.repository.ts @@ -1,13 +1,13 @@ import { Injectable } from '@nestjs/common'; -import { Kysely, OrderByDirection, sql } from 'kysely'; +import { Kysely, OrderByDirection, Selectable, sql } from 'kysely'; import { InjectKysely } from 'nestjs-kysely'; import { randomUUID } from 'node:crypto'; -import { DB } from 'src/db'; +import { DB, Exif } from 'src/db'; import { DummyValue, GenerateSql } from 'src/decorators'; -import { AssetEntity, searchAssetBuilder } from 'src/entities/asset.entity'; +import { MapAsset } from 'src/dtos/asset-response.dto'; +import { searchAssetBuilder } from 'src/entities/asset.entity'; import { AssetStatus, AssetType } from 'src/enum'; import { anyUuid, asUuid } from 'src/utils/database'; -import { Paginated } from 'src/utils/pagination'; import { isValidInteger } from 'src/validation'; export interface SearchResult { @@ -216,7 +216,7 @@ export class SearchRepository { }, ], }) - async searchMetadata(pagination: SearchPaginationOptions, options: AssetSearchOptions): Paginated { + async searchMetadata(pagination: SearchPaginationOptions, options: AssetSearchOptions) { const orderDirection = (options.orderDirection?.toLowerCase() || 'desc') as OrderByDirection; const items = await searchAssetBuilder(this.db, options) .orderBy('assets.fileCreatedAt', orderDirection) @@ -225,7 +225,7 @@ export class SearchRepository { .execute(); const hasNextPage = items.length > pagination.size; items.splice(pagination.size); - return { items: items as any as AssetEntity[], hasNextPage }; + return { items, hasNextPage }; } @GenerateSql({ @@ -240,7 +240,7 @@ export class SearchRepository { }, ], }) - async searchRandom(size: number, options: AssetSearchOptions): Promise { + async searchRandom(size: number, options: AssetSearchOptions) { const uuid = randomUUID(); const builder = searchAssetBuilder(this.db, options); const lessThan = builder @@ -251,8 +251,8 @@ export class SearchRepository { .where('assets.id', '>', uuid) .orderBy(sql`random()`) .limit(size); - const { rows } = await sql`${lessThan} union all ${greaterThan} limit ${size}`.execute(this.db); - return rows as any as AssetEntity[]; + const { rows } = await sql`${lessThan} union all ${greaterThan} limit ${size}`.execute(this.db); + return rows; } @GenerateSql({ @@ -268,17 +268,17 @@ export class SearchRepository { }, ], }) - async searchSmart(pagination: SearchPaginationOptions, options: SmartSearchOptions): Paginated { + async searchSmart(pagination: SearchPaginationOptions, options: SmartSearchOptions) { if (!isValidInteger(pagination.size, { min: 1, max: 1000 })) { throw new Error(`Invalid value for 'size': ${pagination.size}`); } - const items = (await searchAssetBuilder(this.db, options) + const items = await searchAssetBuilder(this.db, options) .innerJoin('smart_search', 'assets.id', 'smart_search.assetId') .orderBy(sql`smart_search.embedding <=> ${options.embedding}`) .limit(pagination.size + 1) .offset((pagination.page - 1) * pagination.size) - .execute()) as any as AssetEntity[]; + .execute(); const hasNextPage = items.length > pagination.size; items.splice(pagination.size); @@ -392,7 +392,7 @@ export class SearchRepository { } @GenerateSql({ params: [[DummyValue.UUID]] }) - getAssetsByCity(userIds: string[]): Promise { + getAssetsByCity(userIds: string[]) { return this.db .withRecursive('cte', (qb) => { const base = qb @@ -434,9 +434,14 @@ export class SearchRepository { .innerJoin('exif', 'assets.id', 'exif.assetId') .innerJoin('cte', 'assets.id', 'cte.assetId') .selectAll('assets') - .select((eb) => eb.fn('to_jsonb', [eb.table('exif')]).as('exifInfo')) + .select((eb) => + eb + .fn('to_jsonb', [eb.table('exif')]) + .$castTo>() + .as('exifInfo'), + ) .orderBy('exif.city') - .execute() as any as Promise; + .execute(); } async upsert(assetId: string, embedding: string): Promise { diff --git a/server/src/repositories/shared-link.repository.ts b/server/src/repositories/shared-link.repository.ts index 272d7f3794..67a97dc2d5 100644 --- a/server/src/repositories/shared-link.repository.ts +++ b/server/src/repositories/shared-link.repository.ts @@ -1,12 +1,12 @@ import { Injectable } from '@nestjs/common'; -import { Insertable, Kysely, sql, Updateable } from 'kysely'; +import { Insertable, Kysely, NotNull, sql, Updateable } from 'kysely'; import { jsonObjectFrom } from 'kysely/helpers/postgres'; import _ from 'lodash'; import { InjectKysely } from 'nestjs-kysely'; -import { columns } from 'src/database'; +import { Album, columns } from 'src/database'; import { DB, SharedLinks } from 'src/db'; import { DummyValue, GenerateSql } from 'src/decorators'; -import { SharedLinkEntity } from 'src/entities/shared-link.entity'; +import { MapAsset } from 'src/dtos/asset-response.dto'; import { SharedLinkType } from 'src/enum'; export type SharedLinkSearchOptions = { @@ -19,7 +19,7 @@ export class SharedLinkRepository { constructor(@InjectKysely() private db: Kysely) {} @GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID] }) - get(userId: string, id: string): Promise { + get(userId: string, id: string) { return this.db .selectFrom('shared_links') .selectAll('shared_links') @@ -87,18 +87,23 @@ export class SharedLinkRepository { .as('album'), (join) => join.onTrue(), ) - .select((eb) => eb.fn.coalesce(eb.fn.jsonAgg('a').filterWhere('a.id', 'is not', null), sql`'[]'`).as('assets')) + .select((eb) => + eb.fn + .coalesce(eb.fn.jsonAgg('a').filterWhere('a.id', 'is not', null), sql`'[]'`) + .$castTo() + .as('assets'), + ) .groupBy(['shared_links.id', sql`"album".*`]) - .select((eb) => eb.fn.toJson('album').as('album')) + .select((eb) => eb.fn.toJson('album').$castTo().as('album')) .where('shared_links.id', '=', id) .where('shared_links.userId', '=', userId) .where((eb) => eb.or([eb('shared_links.type', '=', SharedLinkType.INDIVIDUAL), eb('album.id', 'is not', null)])) .orderBy('shared_links.createdAt', 'desc') - .executeTakeFirst() as Promise; + .executeTakeFirst(); } @GenerateSql({ params: [{ userId: DummyValue.UUID, albumId: DummyValue.UUID }] }) - getAll({ userId, albumId }: SharedLinkSearchOptions): Promise { + getAll({ userId, albumId }: SharedLinkSearchOptions) { return this.db .selectFrom('shared_links') .selectAll('shared_links') @@ -115,6 +120,7 @@ export class SharedLinkRepository { (join) => join.onTrue(), ) .select('assets.assets') + .$narrowType<{ assets: NotNull }>() .leftJoinLateral( (eb) => eb @@ -152,12 +158,12 @@ export class SharedLinkRepository { .as('album'), (join) => join.onTrue(), ) - .select((eb) => eb.fn.toJson('album').as('album')) + .select((eb) => eb.fn.toJson('album').$castTo().as('album')) .where((eb) => eb.or([eb('shared_links.type', '=', SharedLinkType.INDIVIDUAL), eb('album.id', 'is not', null)])) .$if(!!albumId, (eb) => eb.where('shared_links.albumId', '=', albumId!)) .orderBy('shared_links.createdAt', 'desc') .distinctOn(['shared_links.createdAt']) - .execute() as unknown as Promise; + .execute(); } @GenerateSql({ params: [DummyValue.BUFFER] }) @@ -177,7 +183,7 @@ export class SharedLinkRepository { .executeTakeFirst(); } - async create(entity: Insertable & { assetIds?: string[] }): Promise { + async create(entity: Insertable & { assetIds?: string[] }) { const { id } = await this.db .insertInto('shared_links') .values(_.omit(entity, 'assetIds')) @@ -194,7 +200,7 @@ export class SharedLinkRepository { return this.getSharedLinks(id); } - async update(entity: Updateable & { id: string; assetIds?: string[] }): Promise { + async update(entity: Updateable & { id: string; assetIds?: string[] }) { const { id } = await this.db .updateTable('shared_links') .set(_.omit(entity, 'assets', 'album', 'assetIds')) @@ -212,8 +218,8 @@ export class SharedLinkRepository { return this.getSharedLinks(id); } - async remove(entity: SharedLinkEntity): Promise { - await this.db.deleteFrom('shared_links').where('shared_links.id', '=', entity.id).execute(); + async remove(id: string): Promise { + await this.db.deleteFrom('shared_links').where('shared_links.id', '=', id).execute(); } private getSharedLinks(id: string) { @@ -236,9 +242,12 @@ export class SharedLinkRepository { (join) => join.onTrue(), ) .select((eb) => - eb.fn.coalesce(eb.fn.jsonAgg('assets').filterWhere('assets.id', 'is not', null), sql`'[]'`).as('assets'), + eb.fn + .coalesce(eb.fn.jsonAgg('assets').filterWhere('assets.id', 'is not', null), sql`'[]'`) + .$castTo() + .as('assets'), ) .groupBy('shared_links.id') - .executeTakeFirstOrThrow() as Promise; + .executeTakeFirstOrThrow(); } } diff --git a/server/src/repositories/stack.repository.ts b/server/src/repositories/stack.repository.ts index 75dd9b497f..c9d69fb37f 100644 --- a/server/src/repositories/stack.repository.ts +++ b/server/src/repositories/stack.repository.ts @@ -5,7 +5,6 @@ import { InjectKysely } from 'nestjs-kysely'; import { columns } from 'src/database'; import { AssetStack, DB } from 'src/db'; import { DummyValue, GenerateSql } from 'src/decorators'; -import { AssetEntity } from 'src/entities/asset.entity'; import { asUuid } from 'src/utils/database'; export interface StackSearch { @@ -36,9 +35,7 @@ const withAssets = (eb: ExpressionBuilder, withTags = false) .select((eb) => eb.fn.toJson('exifInfo').as('exifInfo')) .where('assets.deletedAt', 'is', null) .whereRef('assets.stackId', '=', 'asset_stack.id'), - ) - .$castTo() - .as('assets'); + ).as('assets'); }; @Injectable() diff --git a/server/src/services/album.service.ts b/server/src/services/album.service.ts index eac000005b..1c612de8c0 100644 --- a/server/src/services/album.service.ts +++ b/server/src/services/album.service.ts @@ -6,15 +6,15 @@ import { AlbumStatisticsResponseDto, CreateAlbumDto, GetAlbumsDto, - UpdateAlbumDto, - UpdateAlbumUserDto, mapAlbum, + MapAlbumDto, mapAlbumWithAssets, mapAlbumWithoutAssets, + UpdateAlbumDto, + UpdateAlbumUserDto, } from 'src/dtos/album.dto'; import { BulkIdResponseDto, BulkIdsDto } from 'src/dtos/asset-ids.response.dto'; import { AuthDto } from 'src/dtos/auth.dto'; -import { AlbumEntity } from 'src/entities/album.entity'; import { Permission } from 'src/enum'; import { AlbumAssetCount, AlbumInfoOptions } from 'src/repositories/album.repository'; import { BaseService } from 'src/services/base.service'; @@ -39,7 +39,7 @@ export class AlbumService extends BaseService { async getAll({ user: { id: ownerId } }: AuthDto, { assetId, shared }: GetAlbumsDto): Promise { await this.albumRepository.updateThumbnails(); - let albums: AlbumEntity[]; + let albums: MapAlbumDto[]; if (assetId) { albums = await this.albumRepository.getByAssetId(ownerId, assetId); } else if (shared === true) { diff --git a/server/src/services/asset-media.service.spec.ts b/server/src/services/asset-media.service.spec.ts index 0c1bbc3cee..a49230f852 100644 --- a/server/src/services/asset-media.service.spec.ts +++ b/server/src/services/asset-media.service.spec.ts @@ -8,6 +8,7 @@ import { Stats } from 'node:fs'; import { AssetFile } from 'src/database'; import { AssetMediaStatus, AssetRejectReason, AssetUploadAction } from 'src/dtos/asset-media-response.dto'; import { AssetMediaCreateDto, AssetMediaReplaceDto, AssetMediaSize, UploadFieldName } from 'src/dtos/asset-media.dto'; +import { MapAsset } from 'src/dtos/asset-response.dto'; import { ASSET_CHECKSUM_CONSTRAINT, AssetEntity } from 'src/entities/asset.entity'; import { AssetFileType, AssetStatus, AssetType, CacheControl, JobName } from 'src/enum'; import { AuthRequest } from 'src/middleware/auth.guard'; @@ -173,7 +174,7 @@ const assetEntity = Object.freeze({ }, livePhotoVideoId: null, sidecarPath: null, -}) as AssetEntity; +} as MapAsset); const existingAsset = Object.freeze({ ...assetEntity, @@ -182,18 +183,18 @@ const existingAsset = Object.freeze({ checksum: Buffer.from('_getExistingAsset', 'utf8'), libraryId: 'libraryId', originalFileName: 'existing-filename.jpeg', -}) as AssetEntity; +}) as MapAsset; const sidecarAsset = Object.freeze({ ...existingAsset, sidecarPath: 'sidecar-path', checksum: Buffer.from('_getExistingAssetWithSideCar', 'utf8'), -}) as AssetEntity; +}) as MapAsset; const copiedAsset = Object.freeze({ id: 'copied-asset', originalPath: 'copied-path', -}) as AssetEntity; +}) as MapAsset; describe(AssetMediaService.name, () => { let sut: AssetMediaService; diff --git a/server/src/services/asset-media.service.ts b/server/src/services/asset-media.service.ts index 2929950f4d..de40d8b304 100644 --- a/server/src/services/asset-media.service.ts +++ b/server/src/services/asset-media.service.ts @@ -2,6 +2,7 @@ import { BadRequestException, Injectable, InternalServerErrorException, NotFound import { extname } from 'node:path'; import sanitize from 'sanitize-filename'; import { StorageCore } from 'src/cores/storage.core'; +import { Asset } from 'src/database'; import { AssetBulkUploadCheckResponseDto, AssetMediaResponseDto, @@ -20,7 +21,7 @@ import { UploadFieldName, } from 'src/dtos/asset-media.dto'; import { AuthDto } from 'src/dtos/auth.dto'; -import { ASSET_CHECKSUM_CONSTRAINT, AssetEntity } from 'src/entities/asset.entity'; +import { ASSET_CHECKSUM_CONSTRAINT } from 'src/entities/asset.entity'; import { AssetStatus, AssetType, CacheControl, JobName, Permission, StorageFolder } from 'src/enum'; import { AuthRequest } from 'src/middleware/auth.guard'; import { BaseService } from 'src/services/base.service'; @@ -212,7 +213,7 @@ export class AssetMediaService extends BaseService { const asset = await this.findOrFail(id); const size = dto.size ?? AssetMediaSize.THUMBNAIL; - const { thumbnailFile, previewFile, fullsizeFile } = getAssetFiles(asset.files); + const { thumbnailFile, previewFile, fullsizeFile } = getAssetFiles(asset.files ?? []); let filepath = previewFile?.path; if (size === AssetMediaSize.THUMBNAIL && thumbnailFile) { filepath = thumbnailFile.path; @@ -375,7 +376,7 @@ export class AssetMediaService extends BaseService { * Uses only vital properties excluding things like: stacks, faces, smart search info, etc, * and then queues a METADATA_EXTRACTION job. */ - private async createCopy(asset: AssetEntity): Promise { + private async createCopy(asset: Omit) { const created = await this.assetRepository.create({ ownerId: asset.ownerId, originalPath: asset.originalPath, @@ -398,12 +399,7 @@ export class AssetMediaService extends BaseService { return created; } - private async create( - ownerId: string, - dto: AssetMediaCreateDto, - file: UploadFile, - sidecarFile?: UploadFile, - ): Promise { + private async create(ownerId: string, dto: AssetMediaCreateDto, file: UploadFile, sidecarFile?: UploadFile) { const asset = await this.assetRepository.create({ ownerId, libraryId: null, @@ -444,7 +440,7 @@ export class AssetMediaService extends BaseService { } } - private async findOrFail(id: string): Promise { + private async findOrFail(id: string) { const asset = await this.assetRepository.getById(id, { files: true }); if (!asset) { throw new NotFoundException('Asset not found'); diff --git a/server/src/services/asset.service.spec.ts b/server/src/services/asset.service.spec.ts index 5fc4984b62..a3f536ac33 100755 --- a/server/src/services/asset.service.spec.ts +++ b/server/src/services/asset.service.spec.ts @@ -1,8 +1,7 @@ import { BadRequestException } from '@nestjs/common'; import { DateTime } from 'luxon'; -import { mapAsset } from 'src/dtos/asset-response.dto'; +import { MapAsset, mapAsset } from 'src/dtos/asset-response.dto'; import { AssetJobName, AssetStatsResponseDto } from 'src/dtos/asset.dto'; -import { AssetEntity } from 'src/entities/asset.entity'; import { AssetStatus, AssetType, JobName, JobStatus } from 'src/enum'; import { AssetStats } from 'src/repositories/asset.repository'; import { AssetService } from 'src/services/asset.service'; @@ -35,7 +34,7 @@ describe(AssetService.name, () => { expect(sut).toBeDefined(); }); - const mockGetById = (assets: AssetEntity[]) => { + const mockGetById = (assets: MapAsset[]) => { mocks.asset.getById.mockImplementation((assetId) => Promise.resolve(assets.find((asset) => asset.id === assetId))); }; @@ -608,7 +607,7 @@ describe(AssetService.name, () => { mocks.asset.getById.mockResolvedValue({ ...assetStub.primaryImage, stack: { ...assetStub.primaryImage.stack, assets: assetStub.primaryImage.stack!.assets.slice(0, 2) }, - } as AssetEntity); + }); await sut.handleAssetDeletion({ id: assetStub.primaryImage.id, deleteOnDisk: true }); diff --git a/server/src/services/asset.service.ts b/server/src/services/asset.service.ts index 1ded79680b..16f60907ba 100644 --- a/server/src/services/asset.service.ts +++ b/server/src/services/asset.service.ts @@ -5,6 +5,7 @@ import { JOBS_ASSET_PAGINATION_SIZE } from 'src/constants'; import { OnJob } from 'src/decorators'; import { AssetResponseDto, + MapAsset, MemoryLaneResponseDto, SanitizedAssetResponseDto, mapAsset, @@ -20,7 +21,6 @@ import { } from 'src/dtos/asset.dto'; import { AuthDto } from 'src/dtos/auth.dto'; import { MemoryLaneDto } from 'src/dtos/search.dto'; -import { AssetEntity } from 'src/entities/asset.entity'; import { AssetStatus, JobName, JobStatus, Permission, QueueName } from 'src/enum'; import { BaseService } from 'src/services/base.service'; import { ISidecarWriteJob, JobItem, JobOf } from 'src/types'; @@ -43,7 +43,7 @@ export class AssetService extends BaseService { yearsAgo, // TODO move this to clients title: `${yearsAgo} year${yearsAgo > 1 ? 's' : ''} ago`, - assets: assets.map((asset) => mapAsset(asset as unknown as AssetEntity, { auth })), + assets: assets.map((asset) => mapAsset(asset, { auth })), }; }); } @@ -105,7 +105,7 @@ export class AssetService extends BaseService { const { description, dateTimeOriginal, latitude, longitude, rating, ...rest } = dto; const repos = { asset: this.assetRepository, event: this.eventRepository }; - let previousMotion: AssetEntity | null = null; + let previousMotion: MapAsset | null = null; if (rest.livePhotoVideoId) { await onBeforeLink(repos, { userId: auth.user.id, livePhotoVideoId: rest.livePhotoVideoId }); } else if (rest.livePhotoVideoId === null) { @@ -233,7 +233,7 @@ export class AssetService extends BaseService { } } - const { fullsizeFile, previewFile, thumbnailFile } = getAssetFiles(asset.files); + const { fullsizeFile, previewFile, thumbnailFile } = getAssetFiles(asset.files ?? []); const files = [thumbnailFile?.path, previewFile?.path, fullsizeFile?.path, asset.encodedVideoPath]; if (deleteOnDisk) { diff --git a/server/src/services/duplicate.service.ts b/server/src/services/duplicate.service.ts index 9f1ac3d4ce..c504b1a305 100644 --- a/server/src/services/duplicate.service.ts +++ b/server/src/services/duplicate.service.ts @@ -68,7 +68,7 @@ export class DuplicateService extends BaseService { return JobStatus.SKIPPED; } - const previewFile = getAssetFile(asset.files, AssetFileType.PREVIEW); + const previewFile = getAssetFile(asset.files || [], AssetFileType.PREVIEW); if (!previewFile) { this.logger.warn(`Asset ${id} is missing preview image`); return JobStatus.FAILED; diff --git a/server/src/services/job.service.spec.ts b/server/src/services/job.service.spec.ts index 134a86b69f..baac0af428 100644 --- a/server/src/services/job.service.spec.ts +++ b/server/src/services/job.service.spec.ts @@ -285,9 +285,9 @@ describe(JobService.name, () => { it(`should queue ${jobs.length} jobs when a ${item.name} job finishes successfully`, async () => { if (item.name === JobName.GENERATE_THUMBNAILS && item.data.source === 'upload') { if (item.data.id === 'asset-live-image') { - mocks.asset.getByIdsWithAllRelations.mockResolvedValue([assetStub.livePhotoStillAsset]); + mocks.asset.getByIdsWithAllRelationsButStacks.mockResolvedValue([assetStub.livePhotoStillAsset as any]); } else { - mocks.asset.getByIdsWithAllRelations.mockResolvedValue([assetStub.livePhotoMotionAsset]); + mocks.asset.getByIdsWithAllRelationsButStacks.mockResolvedValue([assetStub.livePhotoMotionAsset as any]); } } diff --git a/server/src/services/job.service.ts b/server/src/services/job.service.ts index 2f180edd40..edd018d7b1 100644 --- a/server/src/services/job.service.ts +++ b/server/src/services/job.service.ts @@ -254,7 +254,7 @@ export class JobService extends BaseService { case JobName.METADATA_EXTRACTION: { if (item.data.source === 'sidecar-write') { - const [asset] = await this.assetRepository.getByIdsWithAllRelations([item.data.id]); + const [asset] = await this.assetRepository.getByIdsWithAllRelationsButStacks([item.data.id]); if (asset) { this.eventRepository.clientSend('on_asset_update', asset.ownerId, mapAsset(asset)); } @@ -284,7 +284,7 @@ export class JobService extends BaseService { break; } - const [asset] = await this.assetRepository.getByIdsWithAllRelations([item.data.id]); + const [asset] = await this.assetRepository.getByIdsWithAllRelationsButStacks([item.data.id]); if (!asset) { this.logger.warn(`Could not find asset ${item.data.id} after generating thumbnails`); break; diff --git a/server/src/services/library.service.spec.ts b/server/src/services/library.service.spec.ts index aef02b7244..15b150f551 100644 --- a/server/src/services/library.service.spec.ts +++ b/server/src/services/library.service.spec.ts @@ -350,7 +350,7 @@ describe(LibraryService.name, () => { progressCounter: 0, }; - mocks.asset.getByIds.mockResolvedValue([assetStub.external]); + mocks.assetJob.getForSyncAssets.mockResolvedValue([assetStub.external]); mocks.storage.stat.mockRejectedValue(new Error('ENOENT, no such file or directory')); await expect(sut.handleSyncAssets(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); @@ -371,7 +371,7 @@ describe(LibraryService.name, () => { progressCounter: 0, }; - mocks.asset.getByIds.mockResolvedValue([assetStub.external]); + mocks.assetJob.getForSyncAssets.mockResolvedValue([assetStub.external]); mocks.storage.stat.mockRejectedValue(new Error('Could not read file')); await expect(sut.handleSyncAssets(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); @@ -392,7 +392,7 @@ describe(LibraryService.name, () => { progressCounter: 0, }; - mocks.asset.getByIds.mockResolvedValue([assetStub.trashedOffline]); + mocks.assetJob.getForSyncAssets.mockResolvedValue([assetStub.trashedOffline]); mocks.storage.stat.mockRejectedValue(new Error('Could not read file')); await expect(sut.handleSyncAssets(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); @@ -410,7 +410,7 @@ describe(LibraryService.name, () => { progressCounter: 0, }; - mocks.asset.getByIds.mockResolvedValue([assetStub.trashedOffline]); + mocks.assetJob.getForSyncAssets.mockResolvedValue([assetStub.trashedOffline]); mocks.storage.stat.mockResolvedValue({ mtime: assetStub.external.fileModifiedAt } as Stats); await expect(sut.handleSyncAssets(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); @@ -431,7 +431,7 @@ describe(LibraryService.name, () => { progressCounter: 0, }; - mocks.asset.getByIds.mockResolvedValue([assetStub.trashedOffline]); + mocks.assetJob.getForSyncAssets.mockResolvedValue([assetStub.trashedOffline]); mocks.storage.stat.mockResolvedValue({ mtime: assetStub.external.fileModifiedAt } as Stats); await expect(sut.handleSyncAssets(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); @@ -451,7 +451,7 @@ describe(LibraryService.name, () => { progressCounter: 0, }; - mocks.asset.getByIds.mockResolvedValue([assetStub.trashedOffline]); + mocks.assetJob.getForSyncAssets.mockResolvedValue([assetStub.trashedOffline]); mocks.storage.stat.mockResolvedValue({ mtime: assetStub.external.fileModifiedAt } as Stats); await expect(sut.handleSyncAssets(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); @@ -471,7 +471,7 @@ describe(LibraryService.name, () => { progressCounter: 0, }; - mocks.asset.getByIds.mockResolvedValue([assetStub.external]); + mocks.assetJob.getForSyncAssets.mockResolvedValue([assetStub.external]); mocks.storage.stat.mockResolvedValue({ mtime: assetStub.external.fileModifiedAt } as Stats); await expect(sut.handleSyncAssets(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); @@ -489,7 +489,7 @@ describe(LibraryService.name, () => { progressCounter: 0, }; - mocks.asset.getByIds.mockResolvedValue([assetStub.trashedOffline]); + mocks.assetJob.getForSyncAssets.mockResolvedValue([assetStub.trashedOffline]); mocks.storage.stat.mockResolvedValue({ mtime: assetStub.trashedOffline.fileModifiedAt } as Stats); await expect(sut.handleSyncAssets(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); @@ -518,7 +518,7 @@ describe(LibraryService.name, () => { const mtime = new Date(assetStub.external.fileModifiedAt.getDate() + 1); - mocks.asset.getByIds.mockResolvedValue([assetStub.external]); + mocks.assetJob.getForSyncAssets.mockResolvedValue([assetStub.external]); mocks.storage.stat.mockResolvedValue({ mtime } as Stats); await expect(sut.handleSyncAssets(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); diff --git a/server/src/services/library.service.ts b/server/src/services/library.service.ts index 8cc2cf48ff..2add5f484b 100644 --- a/server/src/services/library.service.ts +++ b/server/src/services/library.service.ts @@ -18,7 +18,6 @@ import { ValidateLibraryImportPathResponseDto, ValidateLibraryResponseDto, } from 'src/dtos/library.dto'; -import { AssetEntity } from 'src/entities/asset.entity'; import { AssetStatus, AssetType, DatabaseLock, ImmichWorker, JobName, JobStatus, QueueName } from 'src/enum'; import { ArgOf } from 'src/repositories/event.repository'; import { AssetSyncResult } from 'src/repositories/library.repository'; @@ -467,7 +466,7 @@ export class LibraryService extends BaseService { @OnJob({ name: JobName.LIBRARY_SYNC_ASSETS, queue: QueueName.LIBRARY }) async handleSyncAssets(job: JobOf): Promise { - const assets = await this.assetRepository.getByIds(job.assetIds); + const assets = await this.assetJobRepository.getForSyncAssets(job.assetIds); const assetIdsToOffline: string[] = []; const trashedAssetIdsToOffline: string[] = []; @@ -561,7 +560,16 @@ export class LibraryService extends BaseService { return JobStatus.SUCCESS; } - private checkExistingAsset(asset: AssetEntity, stat: Stats | null): AssetSyncResult { + private checkExistingAsset( + asset: { + isOffline: boolean; + libraryId: string | null; + originalPath: string; + status: AssetStatus; + fileModifiedAt: Date; + }, + stat: Stats | null, + ): AssetSyncResult { if (!stat) { // File not found on disk or permission error if (asset.isOffline) { diff --git a/server/src/services/metadata.service.spec.ts b/server/src/services/metadata.service.spec.ts index ca1277a8c8..e412b1c31f 100644 --- a/server/src/services/metadata.service.spec.ts +++ b/server/src/services/metadata.service.spec.ts @@ -3,7 +3,7 @@ import { randomBytes } from 'node:crypto'; import { Stats } from 'node:fs'; import { constants } from 'node:fs/promises'; import { defaults } from 'src/config'; -import { AssetEntity } from 'src/entities/asset.entity'; +import { MapAsset } from 'src/dtos/asset-response.dto'; import { AssetType, ExifOrientation, ImmichWorker, JobName, JobStatus, SourceType } from 'src/enum'; import { WithoutProperty } from 'src/repositories/asset.repository'; import { ImmichTags } from 'src/repositories/metadata.repository'; @@ -549,7 +549,6 @@ describe(MetadataService.name, () => { livePhotoVideoId: null, libraryId: null, }); - mocks.asset.getByIds.mockResolvedValue([{ ...assetStub.livePhotoWithOriginalFileName, livePhotoVideoId: null }]); mocks.storage.stat.mockResolvedValue({ size: 123_456, mtime: assetStub.livePhotoWithOriginalFileName.fileModifiedAt, @@ -719,7 +718,7 @@ describe(MetadataService.name, () => { }); mocks.crypto.hashSha1.mockReturnValue(randomBytes(512)); mocks.asset.create.mockImplementation( - (asset) => Promise.resolve({ ...assetStub.livePhotoMotionAsset, ...asset }) as Promise, + (asset) => Promise.resolve({ ...assetStub.livePhotoMotionAsset, ...asset }) as Promise, ); const video = randomBytes(512); mocks.storage.readFile.mockResolvedValue(video); @@ -1394,7 +1393,7 @@ describe(MetadataService.name, () => { }); it('should set sidecar path if exists (sidecar named photo.xmp)', async () => { - mocks.asset.getByIds.mockResolvedValue([assetStub.sidecarWithoutExt]); + mocks.asset.getByIds.mockResolvedValue([assetStub.sidecarWithoutExt as any]); mocks.storage.checkFileExists.mockResolvedValueOnce(false); mocks.storage.checkFileExists.mockResolvedValueOnce(true); @@ -1446,7 +1445,7 @@ describe(MetadataService.name, () => { describe('handleSidecarDiscovery', () => { it('should skip hidden assets', async () => { - mocks.asset.getByIds.mockResolvedValue([assetStub.livePhotoMotionAsset]); + mocks.asset.getByIds.mockResolvedValue([assetStub.livePhotoMotionAsset as any]); await sut.handleSidecarDiscovery({ id: assetStub.livePhotoMotionAsset.id }); expect(mocks.storage.checkFileExists).not.toHaveBeenCalled(); }); diff --git a/server/src/services/metadata.service.ts b/server/src/services/metadata.service.ts index ab62c38ed0..faf146a2be 100644 --- a/server/src/services/metadata.service.ts +++ b/server/src/services/metadata.service.ts @@ -271,7 +271,7 @@ export class MetadataService extends BaseService { ]; if (this.isMotionPhoto(asset, exifTags)) { - promises.push(this.applyMotionPhotos(asset as unknown as Asset, exifTags, dates, stats)); + promises.push(this.applyMotionPhotos(asset, exifTags, dates, stats)); } if (isFaceImportEnabled(metadata) && this.hasTaggedFaces(exifTags)) { diff --git a/server/src/services/person.service.ts b/server/src/services/person.service.ts index a413c688f0..66d68857a0 100644 --- a/server/src/services/person.service.ts +++ b/server/src/services/person.service.ts @@ -2,7 +2,8 @@ import { BadRequestException, Injectable, NotFoundException } from '@nestjs/comm import { Insertable, Updateable } from 'kysely'; import { FACE_THUMBNAIL_SIZE, JOBS_ASSET_PAGINATION_SIZE } from 'src/constants'; import { StorageCore } from 'src/cores/storage.core'; -import { AssetFaces, FaceSearch, Person } from 'src/db'; +import { Person } from 'src/database'; +import { AssetFaces, FaceSearch } from 'src/db'; import { Chunked, OnJob } from 'src/decorators'; import { BulkIdErrorReason, BulkIdResponseDto } from 'src/dtos/asset-ids.response.dto'; import { AuthDto } from 'src/dtos/auth.dto'; @@ -315,6 +316,7 @@ export class PersonService extends BaseService { const facesToAdd: (Insertable & { id: string })[] = []; const embeddings: FaceSearch[] = []; const mlFaceIds = new Set(); + for (const face of asset.faces) { if (face.sourceType === SourceType.MACHINE_LEARNING) { mlFaceIds.add(face.id); @@ -477,7 +479,7 @@ export class PersonService extends BaseService { embedding: face.faceSearch.embedding, maxDistance: machineLearning.facialRecognition.maxDistance, numResults: machineLearning.facialRecognition.minFaces, - minBirthDate: face.asset.fileCreatedAt, + minBirthDate: face.asset.fileCreatedAt ?? undefined, }); // `matches` also includes the face itself @@ -503,7 +505,7 @@ export class PersonService extends BaseService { maxDistance: machineLearning.facialRecognition.maxDistance, numResults: 1, hasPerson: true, - minBirthDate: face.asset.fileCreatedAt, + minBirthDate: face.asset.fileCreatedAt ?? undefined, }); if (matchWithPerson.length > 0) { diff --git a/server/src/services/search.service.spec.ts b/server/src/services/search.service.spec.ts index 51c6b55e11..18699edf9a 100644 --- a/server/src/services/search.service.spec.ts +++ b/server/src/services/search.service.spec.ts @@ -45,7 +45,7 @@ describe(SearchService.name, () => { fieldName: 'exifInfo.city', items: [{ value: 'test-city', data: assetStub.withLocation.id }], }); - mocks.asset.getByIdsWithAllRelations.mockResolvedValue([assetStub.withLocation]); + mocks.asset.getByIdsWithAllRelationsButStacks.mockResolvedValue([assetStub.withLocation]); const expectedResponse = [ { fieldName: 'exifInfo.city', items: [{ value: 'test-city', data: mapAsset(assetStub.withLocation) }] }, ]; diff --git a/server/src/services/search.service.ts b/server/src/services/search.service.ts index 1c0c0ad490..442d49136c 100644 --- a/server/src/services/search.service.ts +++ b/server/src/services/search.service.ts @@ -1,5 +1,5 @@ import { BadRequestException, Injectable } from '@nestjs/common'; -import { AssetMapOptions, AssetResponseDto, mapAsset } from 'src/dtos/asset-response.dto'; +import { AssetMapOptions, AssetResponseDto, MapAsset, mapAsset } from 'src/dtos/asset-response.dto'; import { AuthDto } from 'src/dtos/auth.dto'; import { mapPerson, PersonResponseDto } from 'src/dtos/person.dto'; import { @@ -14,7 +14,6 @@ import { SearchSuggestionType, SmartSearchDto, } from 'src/dtos/search.dto'; -import { AssetEntity } from 'src/entities/asset.entity'; import { AssetOrder } from 'src/enum'; import { SearchExploreItem } from 'src/repositories/search.repository'; import { BaseService } from 'src/services/base.service'; @@ -36,7 +35,7 @@ export class SearchService extends BaseService { async getExploreData(auth: AuthDto): Promise[]> { const options = { maxFields: 12, minAssetsPerField: 5 }; const cities = await this.assetRepository.getAssetIdByCity(auth.user.id, options); - const assets = await this.assetRepository.getByIdsWithAllRelations(cities.items.map(({ data }) => data)); + const assets = await this.assetRepository.getByIdsWithAllRelationsButStacks(cities.items.map(({ data }) => data)); const items = assets.map((asset) => ({ value: asset.exifInfo!.city!, data: mapAsset(asset, { auth }) })); return [{ fieldName: cities.fieldName, items }]; } @@ -139,7 +138,7 @@ export class SearchService extends BaseService { return [auth.user.id, ...partnerIds]; } - private mapResponse(assets: AssetEntity[], nextPage: string | null, options: AssetMapOptions): SearchResponseDto { + private mapResponse(assets: MapAsset[], nextPage: string | null, options: AssetMapOptions): SearchResponseDto { return { albums: { total: 0, count: 0, items: [], facets: [] }, assets: { diff --git a/server/src/services/shared-link.service.spec.ts b/server/src/services/shared-link.service.spec.ts index 4d084d6e67..66a0a925c7 100644 --- a/server/src/services/shared-link.service.spec.ts +++ b/server/src/services/shared-link.service.spec.ts @@ -244,7 +244,7 @@ describe(SharedLinkService.name, () => { await sut.remove(authStub.user1, sharedLinkStub.valid.id); expect(mocks.sharedLink.get).toHaveBeenCalledWith(authStub.user1.user.id, sharedLinkStub.valid.id); - expect(mocks.sharedLink.remove).toHaveBeenCalledWith(sharedLinkStub.valid); + expect(mocks.sharedLink.remove).toHaveBeenCalledWith(sharedLinkStub.valid.id); }); }); @@ -333,8 +333,7 @@ describe(SharedLinkService.name, () => { }); it('should return metadata tags with a default image path if the asset id is not set', async () => { - mocks.sharedLink.get.mockResolvedValue({ ...sharedLinkStub.individual, album: undefined, assets: [] }); - + mocks.sharedLink.get.mockResolvedValue({ ...sharedLinkStub.individual, album: null, assets: [] }); await expect(sut.getMetadataTags(authStub.adminSharedLink)).resolves.toEqual({ description: '0 shared photos & videos', imageUrl: `https://my.immich.app/feature-panel.png`, diff --git a/server/src/services/shared-link.service.ts b/server/src/services/shared-link.service.ts index 95f8cef5f8..17f1b974d2 100644 --- a/server/src/services/shared-link.service.ts +++ b/server/src/services/shared-link.service.ts @@ -1,4 +1,5 @@ import { BadRequestException, ForbiddenException, Injectable, UnauthorizedException } from '@nestjs/common'; +import { SharedLink } from 'src/database'; import { AssetIdErrorReason, AssetIdsResponseDto } from 'src/dtos/asset-ids.response.dto'; import { AssetIdsDto } from 'src/dtos/asset.dto'; import { AuthDto } from 'src/dtos/auth.dto'; @@ -11,7 +12,6 @@ import { SharedLinkResponseDto, SharedLinkSearchDto, } from 'src/dtos/shared-link.dto'; -import { SharedLinkEntity } from 'src/entities/shared-link.entity'; import { Permission, SharedLinkType } from 'src/enum'; import { BaseService } from 'src/services/base.service'; import { getExternalDomain, OpenGraphTags } from 'src/utils/misc'; @@ -98,7 +98,7 @@ export class SharedLinkService extends BaseService { async remove(auth: AuthDto, id: string): Promise { const sharedLink = await this.findOrFail(auth.user.id, id); - await this.sharedLinkRepository.remove(sharedLink); + await this.sharedLinkRepository.remove(sharedLink.id); } // TODO: replace `userId` with permissions and access control checks @@ -182,7 +182,7 @@ export class SharedLinkService extends BaseService { const config = await this.getConfig({ withCache: true }); const sharedLink = await this.findOrFail(auth.sharedLink.userId, auth.sharedLink.id); const assetId = sharedLink.album?.albumThumbnailAssetId || sharedLink.assets[0]?.id; - const assetCount = sharedLink.assets.length > 0 ? sharedLink.assets.length : sharedLink.album?.assets.length || 0; + const assetCount = sharedLink.assets.length > 0 ? sharedLink.assets.length : sharedLink.album?.assets?.length || 0; const imagePath = assetId ? `/api/assets/${assetId}/thumbnail?key=${sharedLink.key.toString('base64url')}` : '/feature-panel.png'; @@ -194,11 +194,11 @@ export class SharedLinkService extends BaseService { }; } - private mapToSharedLink(sharedLink: SharedLinkEntity, { withExif }: { withExif: boolean }) { + private mapToSharedLink(sharedLink: SharedLink, { withExif }: { withExif: boolean }) { return withExif ? mapSharedLink(sharedLink) : mapSharedLinkWithoutMetadata(sharedLink); } - private validateAndRefreshToken(sharedLink: SharedLinkEntity, dto: SharedLinkPasswordDto): string { + private validateAndRefreshToken(sharedLink: SharedLink, dto: SharedLinkPasswordDto): string { const token = this.cryptoRepository.hashSha256(`${sharedLink.id}-${sharedLink.password}`); const sharedLinkTokens = dto.token?.split(',') || []; if (sharedLink.password !== dto.password && !sharedLinkTokens.includes(token)) { diff --git a/server/src/services/sync.service.spec.ts b/server/src/services/sync.service.spec.ts index 5f7357c64d..5b50340a9f 100644 --- a/server/src/services/sync.service.spec.ts +++ b/server/src/services/sync.service.spec.ts @@ -1,5 +1,4 @@ import { mapAsset } from 'src/dtos/asset-response.dto'; -import { AssetEntity } from 'src/entities/asset.entity'; import { SyncService } from 'src/services/sync.service'; import { assetStub } from 'test/fixtures/asset.stub'; import { authStub } from 'test/fixtures/auth.stub'; @@ -63,7 +62,7 @@ describe(SyncService.name, () => { it('should return a response requiring a full sync when there are too many changes', async () => { mocks.partner.getAll.mockResolvedValue([]); mocks.asset.getChangedDeltaSync.mockResolvedValue( - Array.from({ length: 10_000 }).fill(assetStub.image), + Array.from({ length: 10_000 }).fill(assetStub.image), ); await expect( sut.getDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }), diff --git a/server/src/services/view.service.ts b/server/src/services/view.service.ts index 5871b04b32..9d1ee3cf89 100644 --- a/server/src/services/view.service.ts +++ b/server/src/services/view.service.ts @@ -1,7 +1,6 @@ import { Injectable } from '@nestjs/common'; import { AssetResponseDto, mapAsset } from 'src/dtos/asset-response.dto'; import { AuthDto } from 'src/dtos/auth.dto'; -import { AssetEntity } from 'src/entities/asset.entity'; import { BaseService } from 'src/services/base.service'; @Injectable() @@ -12,6 +11,6 @@ export class ViewService extends BaseService { async getAssetsByOriginalPath(auth: AuthDto, path: string): Promise { const assets = await this.viewRepository.getAssetsByOriginalPath(auth.user.id, path); - return assets.map((asset) => mapAsset(asset as unknown as AssetEntity, { auth })); + return assets.map((asset) => mapAsset(asset, { auth })); } } diff --git a/server/src/utils/asset.util.ts b/server/src/utils/asset.util.ts index a15f006cda..8905f84165 100644 --- a/server/src/utils/asset.util.ts +++ b/server/src/utils/asset.util.ts @@ -13,11 +13,8 @@ import { PartnerRepository } from 'src/repositories/partner.repository'; import { IBulkAsset, ImmichFile, UploadFile } from 'src/types'; import { checkAccess } from 'src/utils/access'; -export const getAssetFile = ( - files: T[], - type: AssetFileType | GeneratedImageType, -) => { - return (files || []).find((file) => file.type === type); +export const getAssetFile = (files: AssetFile[], type: AssetFileType | GeneratedImageType) => { + return files.find((file) => file.type === type); }; export const getAssetFiles = (files: AssetFile[]) => ({ diff --git a/server/test/fixtures/album.stub.ts b/server/test/fixtures/album.stub.ts index 5a1c141512..fd6a8678a0 100644 --- a/server/test/fixtures/album.stub.ts +++ b/server/test/fixtures/album.stub.ts @@ -1,11 +1,10 @@ -import { AlbumEntity } from 'src/entities/album.entity'; import { AlbumUserRole, AssetOrder } from 'src/enum'; import { assetStub } from 'test/fixtures/asset.stub'; import { authStub } from 'test/fixtures/auth.stub'; import { userStub } from 'test/fixtures/user.stub'; export const albumStub = { - empty: Object.freeze({ + empty: Object.freeze({ id: 'album-1', albumName: 'Empty album', description: '', @@ -21,8 +20,9 @@ export const albumStub = { albumUsers: [], isActivityEnabled: true, order: AssetOrder.DESC, + updateId: '42', }), - sharedWithUser: Object.freeze({ + sharedWithUser: Object.freeze({ id: 'album-2', albumName: 'Empty album shared with user', description: '', @@ -43,8 +43,9 @@ export const albumStub = { ], isActivityEnabled: true, order: AssetOrder.DESC, + updateId: '42', }), - sharedWithMultiple: Object.freeze({ + sharedWithMultiple: Object.freeze({ id: 'album-3', albumName: 'Empty album shared with users', description: '', @@ -69,8 +70,9 @@ export const albumStub = { ], isActivityEnabled: true, order: AssetOrder.DESC, + updateId: '42', }), - sharedWithAdmin: Object.freeze({ + sharedWithAdmin: Object.freeze({ id: 'album-3', albumName: 'Empty album shared with admin', description: '', @@ -91,8 +93,9 @@ export const albumStub = { ], isActivityEnabled: true, order: AssetOrder.DESC, + updateId: '42', }), - oneAsset: Object.freeze({ + oneAsset: Object.freeze({ id: 'album-4', albumName: 'Album with one asset', description: '', @@ -108,8 +111,9 @@ export const albumStub = { albumUsers: [], isActivityEnabled: true, order: AssetOrder.DESC, + updateId: '42', }), - twoAssets: Object.freeze({ + twoAssets: Object.freeze({ id: 'album-4a', albumName: 'Album with two assets', description: '', @@ -125,8 +129,9 @@ export const albumStub = { albumUsers: [], isActivityEnabled: true, order: AssetOrder.DESC, + updateId: '42', }), - emptyWithValidThumbnail: Object.freeze({ + emptyWithValidThumbnail: Object.freeze({ id: 'album-5', albumName: 'Empty album with valid thumbnail', description: '', @@ -142,5 +147,6 @@ export const albumStub = { albumUsers: [], isActivityEnabled: true, order: AssetOrder.DESC, + updateId: '42', }), }; diff --git a/server/test/fixtures/asset.stub.ts b/server/test/fixtures/asset.stub.ts index 16e4f20bb3..d1b8e7cf28 100644 --- a/server/test/fixtures/asset.stub.ts +++ b/server/test/fixtures/asset.stub.ts @@ -1,5 +1,5 @@ -import { AssetFile, Exif } from 'src/database'; -import { AssetEntity } from 'src/entities/asset.entity'; +import { AssetFace, AssetFile, Exif } from 'src/database'; +import { MapAsset } from 'src/dtos/asset-response.dto'; import { AssetFileType, AssetStatus, AssetType } from 'src/enum'; import { StorageAsset } from 'src/types'; import { authStub } from 'test/fixtures/auth.stub'; @@ -26,13 +26,15 @@ const fullsizeFile: AssetFile = { const files: AssetFile[] = [fullsizeFile, previewFile, thumbnailFile]; -export const stackStub = (stackId: string, assets: AssetEntity[]) => { +export const stackStub = (stackId: string, assets: (MapAsset & { exifInfo: Exif })[]) => { return { id: stackId, assets, ownerId: assets[0].ownerId, primaryAsset: assets[0], primaryAssetId: assets[0].id, + createdAt: new Date('2023-02-23T05:06:29.716Z'), + updatedAt: new Date('2023-02-23T05:06:29.716Z'), }; }; @@ -85,9 +87,12 @@ export const assetStub = { isExternal: false, duplicateId: null, isOffline: false, + libraryId: null, + stackId: null, + updateId: '42', }), - noWebpPath: Object.freeze({ + noWebpPath: Object.freeze({ id: 'asset-id', status: AssetStatus.ACTIVE, deviceAssetId: 'device-asset-id', @@ -122,9 +127,12 @@ export const assetStub = { deletedAt: null, duplicateId: null, isOffline: false, + libraryId: null, + stackId: null, + updateId: '42', }), - noThumbhash: Object.freeze({ + noThumbhash: Object.freeze({ id: 'asset-id', status: AssetStatus.ACTIVE, deviceAssetId: 'device-asset-id', @@ -156,6 +164,9 @@ export const assetStub = { deletedAt: null, duplicateId: null, isOffline: false, + libraryId: null, + stackId: null, + updateId: '42', }), primaryImage: Object.freeze({ @@ -195,12 +206,13 @@ export const assetStub = { } as Exif, stackId: 'stack-1', stack: stackStub('stack-1', [ - { id: 'primary-asset-id' } as AssetEntity, - { id: 'stack-child-asset-1' } as AssetEntity, - { id: 'stack-child-asset-2' } as AssetEntity, + { id: 'primary-asset-id' } as MapAsset & { exifInfo: Exif }, + { id: 'stack-child-asset-1' } as MapAsset & { exifInfo: Exif }, + { id: 'stack-child-asset-2' } as MapAsset & { exifInfo: Exif }, ]), duplicateId: null, isOffline: false, + updateId: '42', libraryId: null, }), @@ -229,6 +241,9 @@ export const assetStub = { isExternal: false, livePhotoVideo: null, livePhotoVideoId: null, + updateId: 'foo', + libraryId: null, + stackId: null, sharedLinks: [], originalFileName: 'asset-id.jpg', faces: [], @@ -241,10 +256,10 @@ export const assetStub = { } as Exif, duplicateId: null, isOffline: false, - libraryId: null, + stack: null, }), - trashed: Object.freeze({ + trashed: Object.freeze({ id: 'asset-id', deviceAssetId: 'device-asset-id', fileModifiedAt: new Date('2023-02-23T05:06:29.716Z'), @@ -281,9 +296,12 @@ export const assetStub = { duplicateId: null, isOffline: false, status: AssetStatus.TRASHED, + libraryId: null, + stackId: null, + updateId: '42', }), - trashedOffline: Object.freeze({ + trashedOffline: Object.freeze({ id: 'asset-id', status: AssetStatus.ACTIVE, deviceAssetId: 'device-asset-id', @@ -321,8 +339,10 @@ export const assetStub = { } as Exif, duplicateId: null, isOffline: true, + stackId: null, + updateId: '42', }), - archived: Object.freeze({ + archived: Object.freeze({ id: 'asset-id', status: AssetStatus.ACTIVE, deviceAssetId: 'device-asset-id', @@ -359,9 +379,12 @@ export const assetStub = { } as Exif, duplicateId: null, isOffline: false, + libraryId: null, + stackId: null, + updateId: '42', }), - external: Object.freeze({ + external: Object.freeze({ id: 'asset-id', status: AssetStatus.ACTIVE, deviceAssetId: 'device-asset-id', @@ -397,9 +420,12 @@ export const assetStub = { } as Exif, duplicateId: null, isOffline: false, + updateId: '42', + stackId: null, + stack: null, }), - image1: Object.freeze({ + image1: Object.freeze({ id: 'asset-id-1', status: AssetStatus.ACTIVE, deviceAssetId: 'device-asset-id', @@ -434,9 +460,13 @@ export const assetStub = { } as Exif, duplicateId: null, isOffline: false, + updateId: '42', + stackId: null, + libraryId: null, + stack: null, }), - imageFrom2015: Object.freeze({ + imageFrom2015: Object.freeze({ id: 'asset-id-1', status: AssetStatus.ACTIVE, deviceAssetId: 'device-asset-id', @@ -510,7 +540,9 @@ export const assetStub = { deletedAt: null, duplicateId: null, isOffline: false, + updateId: '42', libraryId: null, + stackId: null, }), livePhotoMotionAsset: Object.freeze({ @@ -527,7 +559,7 @@ export const assetStub = { timeZone: `America/New_York`, }, libraryId: null, - } as AssetEntity & { libraryId: string | null; files: AssetFile[]; exifInfo: Exif }), + } as MapAsset & { faces: AssetFace[]; files: AssetFile[]; exifInfo: Exif }), livePhotoStillAsset: Object.freeze({ id: 'live-photo-still-asset', @@ -544,7 +576,8 @@ export const assetStub = { timeZone: `America/New_York`, }, files, - } as AssetEntity & { libraryId: string | null }), + faces: [] as AssetFace[], + } as MapAsset & { faces: AssetFace[] }), livePhotoWithOriginalFileName: Object.freeze({ id: 'live-photo-still-asset', @@ -562,7 +595,8 @@ export const assetStub = { timeZone: `America/New_York`, }, libraryId: null, - } as AssetEntity & { libraryId: string | null }), + faces: [] as AssetFace[], + } as MapAsset & { faces: AssetFace[] }), withLocation: Object.freeze({ id: 'asset-with-favorite-id', @@ -590,6 +624,9 @@ export const assetStub = { isVisible: true, livePhotoVideo: null, livePhotoVideoId: null, + updateId: 'foo', + libraryId: null, + stackId: null, sharedLinks: [], originalFileName: 'asset-id.ext', faces: [], @@ -604,7 +641,7 @@ export const assetStub = { deletedAt: null, duplicateId: null, isOffline: false, - libraryId: null, + tags: [], }), sidecar: Object.freeze({ @@ -639,10 +676,12 @@ export const assetStub = { deletedAt: null, duplicateId: null, isOffline: false, + updateId: 'foo', libraryId: null, + stackId: null, }), - sidecarWithoutExt: Object.freeze({ + sidecarWithoutExt: Object.freeze({ id: 'asset-id', status: AssetStatus.ACTIVE, deviceAssetId: 'device-asset-id', @@ -676,7 +715,7 @@ export const assetStub = { isOffline: false, }), - hasEncodedVideo: Object.freeze({ + hasEncodedVideo: Object.freeze({ id: 'asset-id', status: AssetStatus.ACTIVE, originalFileName: 'asset-id.ext', @@ -711,9 +750,13 @@ export const assetStub = { deletedAt: null, duplicateId: null, isOffline: false, + updateId: '42', + libraryId: null, + stackId: null, + stack: null, }), - hasFileExtension: Object.freeze({ + hasFileExtension: Object.freeze({ id: 'asset-id', status: AssetStatus.ACTIVE, deviceAssetId: 'device-asset-id', @@ -788,6 +831,9 @@ export const assetStub = { } as Exif, duplicateId: null, isOffline: false, + updateId: '42', + libraryId: null, + stackId: null, }), imageHif: Object.freeze({ @@ -827,5 +873,8 @@ export const assetStub = { } as Exif, duplicateId: null, isOffline: false, + updateId: '42', + libraryId: null, + stackId: null, }), }; diff --git a/server/test/fixtures/auth.stub.ts b/server/test/fixtures/auth.stub.ts index dfa21fc707..9ef55398d3 100644 --- a/server/test/fixtures/auth.stub.ts +++ b/server/test/fixtures/auth.stub.ts @@ -1,6 +1,5 @@ import { Session } from 'src/database'; import { AuthDto } from 'src/dtos/auth.dto'; -import { SharedLinkEntity } from 'src/entities/shared-link.entity'; const authUser = { admin: { @@ -42,14 +41,16 @@ export const authStub = { id: 'token-id', } as Session, }), - adminSharedLink: Object.freeze({ + adminSharedLink: Object.freeze({ user: authUser.admin, sharedLink: { id: '123', showExif: true, allowDownload: true, allowUpload: true, - key: Buffer.from('shared-link-key'), - } as SharedLinkEntity, + expiresAt: null, + password: null, + userId: '42', + }, }), }; diff --git a/server/test/fixtures/shared-link.stub.ts b/server/test/fixtures/shared-link.stub.ts index 4eba0de845..a4d83863c7 100644 --- a/server/test/fixtures/shared-link.stub.ts +++ b/server/test/fixtures/shared-link.stub.ts @@ -1,10 +1,9 @@ import { UserAdmin } from 'src/database'; import { AlbumResponseDto } from 'src/dtos/album.dto'; -import { AssetResponseDto } from 'src/dtos/asset-response.dto'; +import { AssetResponseDto, MapAsset } from 'src/dtos/asset-response.dto'; import { ExifResponseDto } from 'src/dtos/exif.dto'; import { SharedLinkResponseDto } from 'src/dtos/shared-link.dto'; import { mapUser } from 'src/dtos/user.dto'; -import { SharedLinkEntity } from 'src/entities/shared-link.entity'; import { AssetOrder, AssetStatus, AssetType, SharedLinkType } from 'src/enum'; import { assetStub } from 'test/fixtures/asset.stub'; import { authStub } from 'test/fixtures/auth.stub'; @@ -113,12 +112,12 @@ export const sharedLinkStub = { allowUpload: true, allowDownload: true, showExif: true, - album: undefined, + albumId: null, + album: null, description: null, assets: [assetStub.image], password: 'password', - albumId: null, - } as SharedLinkEntity), + }), valid: Object.freeze({ id: '123', userId: authStub.admin.user.id, @@ -130,12 +129,12 @@ export const sharedLinkStub = { allowUpload: true, allowDownload: true, showExif: true, - album: undefined, albumId: null, description: null, password: null, - assets: [], - } as SharedLinkEntity), + assets: [] as MapAsset[], + album: null, + }), expired: Object.freeze({ id: '123', userId: authStub.admin.user.id, @@ -150,9 +149,10 @@ export const sharedLinkStub = { description: null, password: null, albumId: null, - assets: [], - } as SharedLinkEntity), - readonlyNoExif: Object.freeze({ + assets: [] as MapAsset[], + album: null, + }), + readonlyNoExif: Object.freeze({ id: '123', userId: authStub.admin.user.id, key: sharedLinkBytes, @@ -168,6 +168,7 @@ export const sharedLinkStub = { albumId: 'album-123', album: { id: 'album-123', + updateId: '42', ownerId: authStub.admin.user.id, owner: userStub.admin, albumName: 'Test Album', @@ -239,17 +240,22 @@ export const sharedLinkStub = { colorspace: 'sRGB', autoStackId: null, rating: 3, + updatedAt: today, + updateId: '42', }, sharedLinks: [], faces: [], sidecarPath: null, deletedAt: null, duplicateId: null, + updateId: '42', + libraryId: null, + stackId: null, }, ], }, }), - passwordRequired: Object.freeze({ + passwordRequired: Object.freeze({ id: '123', userId: authStub.admin.user.id, key: sharedLinkBytes, @@ -263,6 +269,7 @@ export const sharedLinkStub = { password: 'password', assets: [], albumId: null, + album: null, }), }; diff --git a/server/test/medium/specs/services/memory.service.spec.ts b/server/test/medium/specs/services/memory.service.spec.ts index 172c48ca5b..445434d60a 100644 --- a/server/test/medium/specs/services/memory.service.spec.ts +++ b/server/test/medium/specs/services/memory.service.spec.ts @@ -39,9 +39,12 @@ describe(MemoryService.name, () => { it('should create a memory from an asset', async () => { const { sut, repos, getRepository } = createSut(); - const now = DateTime.fromObject({ year: 2025, month: 2, day: 25 }, { zone: 'utc' }); + const now = DateTime.fromObject({ year: 2025, month: 2, day: 25 }, { zone: 'utc' }) as DateTime; const user = mediumFactory.userInsert(); - const asset = mediumFactory.assetInsert({ ownerId: user.id, localDateTime: now.minus({ years: 1 }).toISO() }); + const asset = mediumFactory.assetInsert({ + ownerId: user.id, + localDateTime: now.minus({ years: 1 }).toISO(), + }); const jobStatus = mediumFactory.assetJobStatusInsert({ assetId: asset.id }); const userRepo = getRepository('user'); @@ -86,7 +89,7 @@ describe(MemoryService.name, () => { it('should not generate a memory twice for the same day', async () => { const { sut, repos, getRepository } = createSut(); - const now = DateTime.fromObject({ year: 2025, month: 2, day: 20 }, { zone: 'utc' }); + const now = DateTime.fromObject({ year: 2025, month: 2, day: 20 }, { zone: 'utc' }) as DateTime; const assetRepo = getRepository('asset'); const memoryRepo = getRepository('memory'); diff --git a/server/test/medium/specs/services/metadata.service.spec.ts b/server/test/medium/specs/services/metadata.service.spec.ts index b25cce2724..13b9867373 100644 --- a/server/test/medium/specs/services/metadata.service.spec.ts +++ b/server/test/medium/specs/services/metadata.service.spec.ts @@ -118,7 +118,7 @@ describe(MetadataService.name, () => { process.env.TZ = serverTimeZone ?? undefined; const { filePath } = await createTestFile(exifData); - mocks.assetJob.getForMetadataExtraction.mockResolvedValue({ id: 'asset-1', originalPath: filePath } as never); + mocks.assetJob.getForMetadataExtraction.mockResolvedValue({ id: 'asset-1', originalPath: filePath } as any); await sut.handleMetadataExtraction({ id: 'asset-1' }); diff --git a/server/test/repositories/asset.repository.mock.ts b/server/test/repositories/asset.repository.mock.ts index 2418b6aa64..d540e55b2a 100644 --- a/server/test/repositories/asset.repository.mock.ts +++ b/server/test/repositories/asset.repository.mock.ts @@ -11,7 +11,7 @@ export const newAssetRepositoryMock = (): Mocked = {}) => { }; }; -const assetFactory = (asset: Partial = {}) => ({ +const assetFactory = (asset: Partial = {}) => ({ id: newUuid(), createdAt: newDate(), updatedAt: newDate(), From dd1fcd5be50a3b39dba55ff5610883997674d3d7 Mon Sep 17 00:00:00 2001 From: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com> Date: Fri, 18 Apr 2025 23:39:56 +0200 Subject: [PATCH 12/29] chore: remove asset entity (#17703) --- server/src/database.ts | 7 +- server/src/entities/asset.entity.ts | 272 ------------------ .../src/repositories/asset-job.repository.ts | 3 +- server/src/repositories/asset.repository.ts | 18 +- server/src/repositories/search.repository.ts | 3 +- server/src/repositories/view-repository.ts | 3 +- server/src/schema/tables/asset.table.ts | 2 +- .../src/services/asset-media.service.spec.ts | 8 +- server/src/services/asset-media.service.ts | 2 +- server/src/services/memory.service.spec.ts | 8 + server/src/services/search.service.spec.ts | 23 ++ server/src/utils/database.ts | 231 +++++++++++++++ 12 files changed, 281 insertions(+), 299 deletions(-) delete mode 100644 server/src/entities/asset.entity.ts diff --git a/server/src/database.ts b/server/src/database.ts index 6e66cb4d3d..27094958ed 100644 --- a/server/src/database.ts +++ b/server/src/database.ts @@ -1,7 +1,6 @@ import { Selectable } from 'kysely'; -import { Albums, AssetJobStatus as DatabaseAssetJobStatus, Exif as DatabaseExif } from 'src/db'; +import { Albums, Exif as DatabaseExif } from 'src/db'; import { MapAsset } from 'src/dtos/asset-response.dto'; -import { AssetEntity } from 'src/entities/asset.entity'; import { AlbumUserRole, AssetFileType, @@ -265,10 +264,6 @@ export type AssetFace = { person?: Person | null; }; -export type AssetJobStatus = Selectable & { - asset: AssetEntity; -}; - const userColumns = ['id', 'name', 'email', 'profileImagePath', 'profileChangedAt'] as const; export const columns = { diff --git a/server/src/entities/asset.entity.ts b/server/src/entities/asset.entity.ts deleted file mode 100644 index da291292e7..0000000000 --- a/server/src/entities/asset.entity.ts +++ /dev/null @@ -1,272 +0,0 @@ -import { DeduplicateJoinsPlugin, ExpressionBuilder, Kysely, SelectQueryBuilder, sql } from 'kysely'; -import { jsonArrayFrom, jsonObjectFrom } from 'kysely/helpers/postgres'; -import { AssetFace, AssetFile, AssetJobStatus, columns, Exif, Person, Stack, Tag, User } from 'src/database'; -import { DB } from 'src/db'; -import { MapAsset } from 'src/dtos/asset-response.dto'; -import { AssetFileType, AssetStatus, AssetType } from 'src/enum'; -import { TimeBucketSize } from 'src/repositories/asset.repository'; -import { AssetSearchBuilderOptions } from 'src/repositories/search.repository'; -import { anyUuid, asUuid, toJson } from 'src/utils/database'; - -export const ASSET_CHECKSUM_CONSTRAINT = 'UQ_assets_owner_checksum'; - -export class AssetEntity { - id!: string; - deviceAssetId!: string; - owner!: User; - ownerId!: string; - libraryId?: string | null; - deviceId!: string; - type!: AssetType; - status!: AssetStatus; - originalPath!: string; - files!: AssetFile[]; - thumbhash!: Buffer | null; - encodedVideoPath!: string | null; - createdAt!: Date; - updatedAt!: Date; - updateId?: string; - deletedAt!: Date | null; - fileCreatedAt!: Date; - localDateTime!: Date; - fileModifiedAt!: Date; - isFavorite!: boolean; - isArchived!: boolean; - isExternal!: boolean; - isOffline!: boolean; - checksum!: Buffer; // sha1 checksum - duration!: string | null; - isVisible!: boolean; - livePhotoVideo!: MapAsset | null; - livePhotoVideoId!: string | null; - originalFileName!: string; - sidecarPath!: string | null; - exifInfo?: Exif; - tags?: Tag[]; - faces!: AssetFace[]; - stackId?: string | null; - stack?: Stack | null; - jobStatus?: AssetJobStatus; - duplicateId!: string | null; -} - -// TODO come up with a better query that only selects the fields we need -export function withExif(qb: SelectQueryBuilder) { - return qb - .leftJoin('exif', 'assets.id', 'exif.assetId') - .select((eb) => eb.fn.toJson(eb.table('exif')).$castTo().as('exifInfo')); -} - -export function withExifInner(qb: SelectQueryBuilder) { - return qb - .innerJoin('exif', 'assets.id', 'exif.assetId') - .select((eb) => eb.fn.toJson(eb.table('exif')).$castTo().as('exifInfo')); -} - -export function withSmartSearch(qb: SelectQueryBuilder) { - return qb - .leftJoin('smart_search', 'assets.id', 'smart_search.assetId') - .select((eb) => toJson(eb, 'smart_search').as('smartSearch')); -} - -export function withFaces(eb: ExpressionBuilder, withDeletedFace?: boolean) { - return jsonArrayFrom( - eb - .selectFrom('asset_faces') - .selectAll('asset_faces') - .whereRef('asset_faces.assetId', '=', 'assets.id') - .$if(!withDeletedFace, (qb) => qb.where('asset_faces.deletedAt', 'is', null)), - ).as('faces'); -} - -export function withFiles(eb: ExpressionBuilder, type?: AssetFileType) { - return jsonArrayFrom( - eb - .selectFrom('asset_files') - .select(columns.assetFiles) - .whereRef('asset_files.assetId', '=', 'assets.id') - .$if(!!type, (qb) => qb.where('asset_files.type', '=', type!)), - ).as('files'); -} - -export function withFacesAndPeople(eb: ExpressionBuilder, withDeletedFace?: boolean) { - return jsonArrayFrom( - eb - .selectFrom('asset_faces') - .leftJoinLateral( - (eb) => - eb.selectFrom('person').selectAll('person').whereRef('asset_faces.personId', '=', 'person.id').as('person'), - (join) => join.onTrue(), - ) - .selectAll('asset_faces') - .select((eb) => eb.table('person').$castTo().as('person')) - .whereRef('asset_faces.assetId', '=', 'assets.id') - .$if(!withDeletedFace, (qb) => qb.where('asset_faces.deletedAt', 'is', null)), - ).as('faces'); -} - -export function hasPeople(qb: SelectQueryBuilder, personIds: string[]) { - return qb.innerJoin( - (eb) => - eb - .selectFrom('asset_faces') - .select('assetId') - .where('personId', '=', anyUuid(personIds!)) - .where('deletedAt', 'is', null) - .groupBy('assetId') - .having((eb) => eb.fn.count('personId').distinct(), '=', personIds.length) - .as('has_people'), - (join) => join.onRef('has_people.assetId', '=', 'assets.id'), - ); -} - -export function hasTags(qb: SelectQueryBuilder, tagIds: string[]) { - return qb.innerJoin( - (eb) => - eb - .selectFrom('tag_asset') - .select('assetsId') - .innerJoin('tags_closure', 'tag_asset.tagsId', 'tags_closure.id_descendant') - .where('tags_closure.id_ancestor', '=', anyUuid(tagIds)) - .groupBy('assetsId') - .having((eb) => eb.fn.count('tags_closure.id_ancestor').distinct(), '>=', tagIds.length) - .as('has_tags'), - (join) => join.onRef('has_tags.assetsId', '=', 'assets.id'), - ); -} - -export function withOwner(eb: ExpressionBuilder) { - return jsonObjectFrom(eb.selectFrom('users').select(columns.user).whereRef('users.id', '=', 'assets.ownerId')).as( - 'owner', - ); -} - -export function withLibrary(eb: ExpressionBuilder) { - return jsonObjectFrom( - eb.selectFrom('libraries').selectAll('libraries').whereRef('libraries.id', '=', 'assets.libraryId'), - ).as('library'); -} - -export function withTags(eb: ExpressionBuilder) { - return jsonArrayFrom( - eb - .selectFrom('tags') - .select(columns.tag) - .innerJoin('tag_asset', 'tags.id', 'tag_asset.tagsId') - .whereRef('assets.id', '=', 'tag_asset.assetsId'), - ).as('tags'); -} - -export function truncatedDate(size: TimeBucketSize) { - return sql`date_trunc(${size}, "localDateTime" at time zone 'UTC') at time zone 'UTC'`; -} - -export function withTagId(qb: SelectQueryBuilder, tagId: string) { - return qb.where((eb) => - eb.exists( - eb - .selectFrom('tags_closure') - .innerJoin('tag_asset', 'tag_asset.tagsId', 'tags_closure.id_descendant') - .whereRef('tag_asset.assetsId', '=', 'assets.id') - .where('tags_closure.id_ancestor', '=', tagId), - ), - ); -} - -const joinDeduplicationPlugin = new DeduplicateJoinsPlugin(); - -/** TODO: This should only be used for search-related queries, not as a general purpose query builder */ -export function searchAssetBuilder(kysely: Kysely, options: AssetSearchBuilderOptions) { - options.isArchived ??= options.withArchived ? undefined : false; - options.withDeleted ||= !!(options.trashedAfter || options.trashedBefore || options.isOffline); - return kysely - .withPlugin(joinDeduplicationPlugin) - .selectFrom('assets') - .selectAll('assets') - .$if(!!options.tagIds && options.tagIds.length > 0, (qb) => hasTags(qb, options.tagIds!)) - .$if(!!options.personIds && options.personIds.length > 0, (qb) => hasPeople(qb, options.personIds!)) - .$if(!!options.createdBefore, (qb) => qb.where('assets.createdAt', '<=', options.createdBefore!)) - .$if(!!options.createdAfter, (qb) => qb.where('assets.createdAt', '>=', options.createdAfter!)) - .$if(!!options.updatedBefore, (qb) => qb.where('assets.updatedAt', '<=', options.updatedBefore!)) - .$if(!!options.updatedAfter, (qb) => qb.where('assets.updatedAt', '>=', options.updatedAfter!)) - .$if(!!options.trashedBefore, (qb) => qb.where('assets.deletedAt', '<=', options.trashedBefore!)) - .$if(!!options.trashedAfter, (qb) => qb.where('assets.deletedAt', '>=', options.trashedAfter!)) - .$if(!!options.takenBefore, (qb) => qb.where('assets.fileCreatedAt', '<=', options.takenBefore!)) - .$if(!!options.takenAfter, (qb) => qb.where('assets.fileCreatedAt', '>=', options.takenAfter!)) - .$if(options.city !== undefined, (qb) => - qb - .innerJoin('exif', 'assets.id', 'exif.assetId') - .where('exif.city', options.city === null ? 'is' : '=', options.city!), - ) - .$if(options.state !== undefined, (qb) => - qb - .innerJoin('exif', 'assets.id', 'exif.assetId') - .where('exif.state', options.state === null ? 'is' : '=', options.state!), - ) - .$if(options.country !== undefined, (qb) => - qb - .innerJoin('exif', 'assets.id', 'exif.assetId') - .where('exif.country', options.country === null ? 'is' : '=', options.country!), - ) - .$if(options.make !== undefined, (qb) => - qb - .innerJoin('exif', 'assets.id', 'exif.assetId') - .where('exif.make', options.make === null ? 'is' : '=', options.make!), - ) - .$if(options.model !== undefined, (qb) => - qb - .innerJoin('exif', 'assets.id', 'exif.assetId') - .where('exif.model', options.model === null ? 'is' : '=', options.model!), - ) - .$if(options.lensModel !== undefined, (qb) => - qb - .innerJoin('exif', 'assets.id', 'exif.assetId') - .where('exif.lensModel', options.lensModel === null ? 'is' : '=', options.lensModel!), - ) - .$if(options.rating !== undefined, (qb) => - qb - .innerJoin('exif', 'assets.id', 'exif.assetId') - .where('exif.rating', options.rating === null ? 'is' : '=', options.rating!), - ) - .$if(!!options.checksum, (qb) => qb.where('assets.checksum', '=', options.checksum!)) - .$if(!!options.deviceAssetId, (qb) => qb.where('assets.deviceAssetId', '=', options.deviceAssetId!)) - .$if(!!options.deviceId, (qb) => qb.where('assets.deviceId', '=', options.deviceId!)) - .$if(!!options.id, (qb) => qb.where('assets.id', '=', asUuid(options.id!))) - .$if(!!options.libraryId, (qb) => qb.where('assets.libraryId', '=', asUuid(options.libraryId!))) - .$if(!!options.userIds, (qb) => qb.where('assets.ownerId', '=', anyUuid(options.userIds!))) - .$if(!!options.encodedVideoPath, (qb) => qb.where('assets.encodedVideoPath', '=', options.encodedVideoPath!)) - .$if(!!options.originalPath, (qb) => - qb.where(sql`f_unaccent(assets."originalPath")`, 'ilike', sql`'%' || f_unaccent(${options.originalPath}) || '%'`), - ) - .$if(!!options.originalFileName, (qb) => - qb.where( - sql`f_unaccent(assets."originalFileName")`, - 'ilike', - sql`'%' || f_unaccent(${options.originalFileName}) || '%'`, - ), - ) - .$if(!!options.description, (qb) => - qb - .innerJoin('exif', 'assets.id', 'exif.assetId') - .where(sql`f_unaccent(exif.description)`, 'ilike', sql`'%' || f_unaccent(${options.description}) || '%'`), - ) - .$if(!!options.type, (qb) => qb.where('assets.type', '=', options.type!)) - .$if(options.isFavorite !== undefined, (qb) => qb.where('assets.isFavorite', '=', options.isFavorite!)) - .$if(options.isOffline !== undefined, (qb) => qb.where('assets.isOffline', '=', options.isOffline!)) - .$if(options.isVisible !== undefined, (qb) => qb.where('assets.isVisible', '=', options.isVisible!)) - .$if(options.isArchived !== undefined, (qb) => qb.where('assets.isArchived', '=', options.isArchived!)) - .$if(options.isEncoded !== undefined, (qb) => - qb.where('assets.encodedVideoPath', options.isEncoded ? 'is not' : 'is', null), - ) - .$if(options.isMotion !== undefined, (qb) => - qb.where('assets.livePhotoVideoId', options.isMotion ? 'is not' : 'is', null), - ) - .$if(!!options.isNotInAlbum, (qb) => - qb.where((eb) => - eb.not(eb.exists((eb) => eb.selectFrom('albums_assets_assets').whereRef('assetsId', '=', 'assets.id'))), - ), - ) - .$if(!!options.withExif, withExifInner) - .$if(!!(options.withFaces || options.withPeople || options.personIds), (qb) => qb.select(withFacesAndPeople)) - .$if(!options.withDeleted, (qb) => qb.where('assets.deletedAt', 'is', null)); -} diff --git a/server/src/repositories/asset-job.repository.ts b/server/src/repositories/asset-job.repository.ts index 1bf08e81f5..4a2d52566f 100644 --- a/server/src/repositories/asset-job.repository.ts +++ b/server/src/repositories/asset-job.repository.ts @@ -5,10 +5,9 @@ import { InjectKysely } from 'nestjs-kysely'; import { columns } from 'src/database'; import { DB } from 'src/db'; import { DummyValue, GenerateSql } from 'src/decorators'; -import { withExifInner, withFaces, withFiles } from 'src/entities/asset.entity'; import { AssetFileType } from 'src/enum'; import { StorageAsset } from 'src/types'; -import { anyUuid, asUuid } from 'src/utils/database'; +import { anyUuid, asUuid, withExifInner, withFaces, withFiles } from 'src/utils/database'; @Injectable() export class AssetJobRepository { diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts index 7fb7056ba7..7a68ba907f 100644 --- a/server/src/repositories/asset.repository.ts +++ b/server/src/repositories/asset.repository.ts @@ -6,11 +6,16 @@ import { Stack } from 'src/database'; import { AssetFiles, AssetJobStatus, Assets, DB, Exif } from 'src/db'; import { Chunked, ChunkedArray, DummyValue, GenerateSql } from 'src/decorators'; import { MapAsset } from 'src/dtos/asset-response.dto'; +import { AssetFileType, AssetOrder, AssetStatus, AssetType } from 'src/enum'; +import { AssetSearchOptions, SearchExploreItem, SearchExploreItemSet } from 'src/repositories/search.repository'; import { - AssetEntity, + anyUuid, + asUuid, hasPeople, + removeUndefinedKeys, searchAssetBuilder, truncatedDate, + unnest, withExif, withFaces, withFacesAndPeople, @@ -20,10 +25,7 @@ import { withSmartSearch, withTagId, withTags, -} from 'src/entities/asset.entity'; -import { AssetFileType, AssetOrder, AssetStatus, AssetType } from 'src/enum'; -import { AssetSearchOptions, SearchExploreItem, SearchExploreItemSet } from 'src/repositories/search.repository'; -import { anyUuid, asUuid, removeUndefinedKeys, unnest } from 'src/utils/database'; +} from 'src/utils/database'; import { globToSqlPattern } from 'src/utils/misc'; import { PaginationOptions, paginationHelper } from 'src/utils/pagination'; @@ -128,8 +130,6 @@ export interface AssetGetByChecksumOptions { libraryId?: string; } -export type AssetPathEntity = Pick; - export interface GetByIdsRelations { exifInfo?: boolean; faces?: { person?: boolean; withDeleted?: boolean }; @@ -493,13 +493,13 @@ export class AssetRepository { } @GenerateSql({ params: [DummyValue.UUID, [DummyValue.BUFFER]] }) - getByChecksums(userId: string, checksums: Buffer[]): Promise { + getByChecksums(userId: string, checksums: Buffer[]) { return this.db .selectFrom('assets') .select(['id', 'checksum', 'deletedAt']) .where('ownerId', '=', asUuid(userId)) .where('checksum', 'in', checksums) - .execute() as any as Promise; + .execute(); } @GenerateSql({ params: [DummyValue.UUID, DummyValue.BUFFER] }) diff --git a/server/src/repositories/search.repository.ts b/server/src/repositories/search.repository.ts index 95c350fe34..5c1ebae69d 100644 --- a/server/src/repositories/search.repository.ts +++ b/server/src/repositories/search.repository.ts @@ -5,9 +5,8 @@ import { randomUUID } from 'node:crypto'; import { DB, Exif } from 'src/db'; import { DummyValue, GenerateSql } from 'src/decorators'; import { MapAsset } from 'src/dtos/asset-response.dto'; -import { searchAssetBuilder } from 'src/entities/asset.entity'; import { AssetStatus, AssetType } from 'src/enum'; -import { anyUuid, asUuid } from 'src/utils/database'; +import { anyUuid, asUuid, searchAssetBuilder } from 'src/utils/database'; import { isValidInteger } from 'src/validation'; export interface SearchResult { diff --git a/server/src/repositories/view-repository.ts b/server/src/repositories/view-repository.ts index ae2303e9e2..e32933065c 100644 --- a/server/src/repositories/view-repository.ts +++ b/server/src/repositories/view-repository.ts @@ -2,8 +2,7 @@ import { Kysely } from 'kysely'; import { InjectKysely } from 'nestjs-kysely'; import { DB } from 'src/db'; import { DummyValue, GenerateSql } from 'src/decorators'; -import { withExif } from 'src/entities/asset.entity'; -import { asUuid } from 'src/utils/database'; +import { asUuid, withExif } from 'src/utils/database'; export class ViewRepository { constructor(@InjectKysely() private db: Kysely) {} diff --git a/server/src/schema/tables/asset.table.ts b/server/src/schema/tables/asset.table.ts index 9a9670cd0e..19ec8d2ef4 100644 --- a/server/src/schema/tables/asset.table.ts +++ b/server/src/schema/tables/asset.table.ts @@ -1,5 +1,4 @@ import { UpdatedAtTrigger, UpdateIdColumn } from 'src/decorators'; -import { ASSET_CHECKSUM_CONSTRAINT } from 'src/entities/asset.entity'; import { AssetStatus, AssetType } from 'src/enum'; import { assets_status_enum } from 'src/schema/enums'; import { assets_delete_audit } from 'src/schema/functions'; @@ -17,6 +16,7 @@ import { Table, UpdateDateColumn, } from 'src/sql-tools'; +import { ASSET_CHECKSUM_CONSTRAINT } from 'src/utils/database'; @Table('assets') @UpdatedAtTrigger('assets_updated_at') diff --git a/server/src/services/asset-media.service.spec.ts b/server/src/services/asset-media.service.spec.ts index a49230f852..d25067f1c9 100644 --- a/server/src/services/asset-media.service.spec.ts +++ b/server/src/services/asset-media.service.spec.ts @@ -9,10 +9,10 @@ import { AssetFile } from 'src/database'; import { AssetMediaStatus, AssetRejectReason, AssetUploadAction } from 'src/dtos/asset-media-response.dto'; import { AssetMediaCreateDto, AssetMediaReplaceDto, AssetMediaSize, UploadFieldName } from 'src/dtos/asset-media.dto'; import { MapAsset } from 'src/dtos/asset-response.dto'; -import { ASSET_CHECKSUM_CONSTRAINT, AssetEntity } from 'src/entities/asset.entity'; import { AssetFileType, AssetStatus, AssetType, CacheControl, JobName } from 'src/enum'; import { AuthRequest } from 'src/middleware/auth.guard'; import { AssetMediaService } from 'src/services/asset-media.service'; +import { ASSET_CHECKSUM_CONSTRAINT } from 'src/utils/database'; import { ImmichFileResponse } from 'src/utils/file'; import { assetStub } from 'test/fixtures/asset.stub'; import { authStub } from 'test/fixtures/auth.stub'; @@ -820,8 +820,8 @@ describe(AssetMediaService.name, () => { const file2 = Buffer.from('53be335e99f18a66ff12e9a901c7a6171dd76573', 'hex'); mocks.asset.getByChecksums.mockResolvedValue([ - { id: 'asset-1', checksum: file1 } as AssetEntity, - { id: 'asset-2', checksum: file2 } as AssetEntity, + { id: 'asset-1', checksum: file1, deletedAt: null }, + { id: 'asset-2', checksum: file2, deletedAt: null }, ]); await expect( @@ -857,7 +857,7 @@ describe(AssetMediaService.name, () => { const file1 = Buffer.from('d2947b871a706081be194569951b7db246907957', 'hex'); const file2 = Buffer.from('53be335e99f18a66ff12e9a901c7a6171dd76573', 'hex'); - mocks.asset.getByChecksums.mockResolvedValue([{ id: 'asset-1', checksum: file1 } as AssetEntity]); + mocks.asset.getByChecksums.mockResolvedValue([{ id: 'asset-1', checksum: file1, deletedAt: null }]); await expect( sut.bulkUploadCheck(authStub.admin, { diff --git a/server/src/services/asset-media.service.ts b/server/src/services/asset-media.service.ts index de40d8b304..78e23fa802 100644 --- a/server/src/services/asset-media.service.ts +++ b/server/src/services/asset-media.service.ts @@ -21,13 +21,13 @@ import { UploadFieldName, } from 'src/dtos/asset-media.dto'; import { AuthDto } from 'src/dtos/auth.dto'; -import { ASSET_CHECKSUM_CONSTRAINT } from 'src/entities/asset.entity'; import { AssetStatus, AssetType, CacheControl, JobName, Permission, StorageFolder } from 'src/enum'; import { AuthRequest } from 'src/middleware/auth.guard'; import { BaseService } from 'src/services/base.service'; import { UploadFile } from 'src/types'; import { requireUploadAccess } from 'src/utils/access'; import { asRequest, getAssetFiles, onBeforeLink } from 'src/utils/asset.util'; +import { ASSET_CHECKSUM_CONSTRAINT } from 'src/utils/database'; import { getFilenameExtension, getFileNameWithoutExtension, ImmichFileResponse } from 'src/utils/file'; import { mimeTypes } from 'src/utils/mime-types'; import { fromChecksum } from 'src/utils/request'; diff --git a/server/src/services/memory.service.spec.ts b/server/src/services/memory.service.spec.ts index 7ce8b1ab46..d55c58d9af 100644 --- a/server/src/services/memory.service.spec.ts +++ b/server/src/services/memory.service.spec.ts @@ -15,6 +15,14 @@ describe(MemoryService.name, () => { expect(sut).toBeDefined(); }); + describe('onMemoryCleanup', () => { + it('should clean up memories', async () => { + mocks.memory.cleanup.mockResolvedValue([]); + await sut.onMemoriesCleanup(); + expect(mocks.memory.cleanup).toHaveBeenCalled(); + }); + }); + describe('search', () => { it('should search memories', async () => { const [userId] = newUuids(); diff --git a/server/src/services/search.service.spec.ts b/server/src/services/search.service.spec.ts index 18699edf9a..d87ccbde1d 100644 --- a/server/src/services/search.service.spec.ts +++ b/server/src/services/search.service.spec.ts @@ -39,6 +39,29 @@ describe(SearchService.name, () => { }); }); + describe('searchPlaces', () => { + it('should search places', async () => { + mocks.search.searchPlaces.mockResolvedValue([ + { + id: 42, + name: 'my place', + latitude: 420, + longitude: 69, + admin1Code: null, + admin1Name: null, + admin2Code: null, + admin2Name: null, + alternateNames: null, + countryCode: 'US', + modificationDate: new Date(), + }, + ]); + + await sut.searchPlaces({ name: 'place' }); + expect(mocks.search.searchPlaces).toHaveBeenCalledWith('place'); + }); + }); + describe('getExploreData', () => { it('should get assets by city and tag', async () => { mocks.asset.getAssetIdByCity.mockResolvedValue({ diff --git a/server/src/utils/database.ts b/server/src/utils/database.ts index a9c7b09c5a..1af0aa4b4e 100644 --- a/server/src/utils/database.ts +++ b/server/src/utils/database.ts @@ -1,15 +1,24 @@ import { + DeduplicateJoinsPlugin, Expression, ExpressionBuilder, ExpressionWrapper, + Kysely, KyselyConfig, Nullable, Selectable, + SelectQueryBuilder, Simplify, sql, } from 'kysely'; import { PostgresJSDialect } from 'kysely-postgres-js'; +import { jsonArrayFrom, jsonObjectFrom } from 'kysely/helpers/postgres'; import postgres, { Notice } from 'postgres'; +import { columns, Exif, Person } from 'src/database'; +import { DB } from 'src/db'; +import { AssetFileType } from 'src/enum'; +import { TimeBucketSize } from 'src/repositories/asset.repository'; +import { AssetSearchBuilderOptions } from 'src/repositories/search.repository'; type Ssl = 'require' | 'allow' | 'prefer' | 'verify-full' | boolean | object; @@ -112,3 +121,225 @@ export function toJson >; } + +export const ASSET_CHECKSUM_CONSTRAINT = 'UQ_assets_owner_checksum'; +// TODO come up with a better query that only selects the fields we need + +export function withExif(qb: SelectQueryBuilder) { + return qb + .leftJoin('exif', 'assets.id', 'exif.assetId') + .select((eb) => eb.fn.toJson(eb.table('exif')).$castTo().as('exifInfo')); +} + +export function withExifInner(qb: SelectQueryBuilder) { + return qb + .innerJoin('exif', 'assets.id', 'exif.assetId') + .select((eb) => eb.fn.toJson(eb.table('exif')).$castTo().as('exifInfo')); +} + +export function withSmartSearch(qb: SelectQueryBuilder) { + return qb + .leftJoin('smart_search', 'assets.id', 'smart_search.assetId') + .select((eb) => toJson(eb, 'smart_search').as('smartSearch')); +} + +export function withFaces(eb: ExpressionBuilder, withDeletedFace?: boolean) { + return jsonArrayFrom( + eb + .selectFrom('asset_faces') + .selectAll('asset_faces') + .whereRef('asset_faces.assetId', '=', 'assets.id') + .$if(!withDeletedFace, (qb) => qb.where('asset_faces.deletedAt', 'is', null)), + ).as('faces'); +} + +export function withFiles(eb: ExpressionBuilder, type?: AssetFileType) { + return jsonArrayFrom( + eb + .selectFrom('asset_files') + .select(columns.assetFiles) + .whereRef('asset_files.assetId', '=', 'assets.id') + .$if(!!type, (qb) => qb.where('asset_files.type', '=', type!)), + ).as('files'); +} + +export function withFacesAndPeople(eb: ExpressionBuilder, withDeletedFace?: boolean) { + return jsonArrayFrom( + eb + .selectFrom('asset_faces') + .leftJoinLateral( + (eb) => + eb.selectFrom('person').selectAll('person').whereRef('asset_faces.personId', '=', 'person.id').as('person'), + (join) => join.onTrue(), + ) + .selectAll('asset_faces') + .select((eb) => eb.table('person').$castTo().as('person')) + .whereRef('asset_faces.assetId', '=', 'assets.id') + .$if(!withDeletedFace, (qb) => qb.where('asset_faces.deletedAt', 'is', null)), + ).as('faces'); +} + +export function hasPeople(qb: SelectQueryBuilder, personIds: string[]) { + return qb.innerJoin( + (eb) => + eb + .selectFrom('asset_faces') + .select('assetId') + .where('personId', '=', anyUuid(personIds!)) + .where('deletedAt', 'is', null) + .groupBy('assetId') + .having((eb) => eb.fn.count('personId').distinct(), '=', personIds.length) + .as('has_people'), + (join) => join.onRef('has_people.assetId', '=', 'assets.id'), + ); +} + +export function hasTags(qb: SelectQueryBuilder, tagIds: string[]) { + return qb.innerJoin( + (eb) => + eb + .selectFrom('tag_asset') + .select('assetsId') + .innerJoin('tags_closure', 'tag_asset.tagsId', 'tags_closure.id_descendant') + .where('tags_closure.id_ancestor', '=', anyUuid(tagIds)) + .groupBy('assetsId') + .having((eb) => eb.fn.count('tags_closure.id_ancestor').distinct(), '>=', tagIds.length) + .as('has_tags'), + (join) => join.onRef('has_tags.assetsId', '=', 'assets.id'), + ); +} + +export function withOwner(eb: ExpressionBuilder) { + return jsonObjectFrom(eb.selectFrom('users').select(columns.user).whereRef('users.id', '=', 'assets.ownerId')).as( + 'owner', + ); +} + +export function withLibrary(eb: ExpressionBuilder) { + return jsonObjectFrom( + eb.selectFrom('libraries').selectAll('libraries').whereRef('libraries.id', '=', 'assets.libraryId'), + ).as('library'); +} + +export function withTags(eb: ExpressionBuilder) { + return jsonArrayFrom( + eb + .selectFrom('tags') + .select(columns.tag) + .innerJoin('tag_asset', 'tags.id', 'tag_asset.tagsId') + .whereRef('assets.id', '=', 'tag_asset.assetsId'), + ).as('tags'); +} + +export function truncatedDate(size: TimeBucketSize) { + return sql`date_trunc(${size}, "localDateTime" at time zone 'UTC') at time zone 'UTC'`; +} + +export function withTagId(qb: SelectQueryBuilder, tagId: string) { + return qb.where((eb) => + eb.exists( + eb + .selectFrom('tags_closure') + .innerJoin('tag_asset', 'tag_asset.tagsId', 'tags_closure.id_descendant') + .whereRef('tag_asset.assetsId', '=', 'assets.id') + .where('tags_closure.id_ancestor', '=', tagId), + ), + ); +} +const joinDeduplicationPlugin = new DeduplicateJoinsPlugin(); +/** TODO: This should only be used for search-related queries, not as a general purpose query builder */ + +export function searchAssetBuilder(kysely: Kysely, options: AssetSearchBuilderOptions) { + options.isArchived ??= options.withArchived ? undefined : false; + options.withDeleted ||= !!(options.trashedAfter || options.trashedBefore || options.isOffline); + return kysely + .withPlugin(joinDeduplicationPlugin) + .selectFrom('assets') + .selectAll('assets') + .$if(!!options.tagIds && options.tagIds.length > 0, (qb) => hasTags(qb, options.tagIds!)) + .$if(!!options.personIds && options.personIds.length > 0, (qb) => hasPeople(qb, options.personIds!)) + .$if(!!options.createdBefore, (qb) => qb.where('assets.createdAt', '<=', options.createdBefore!)) + .$if(!!options.createdAfter, (qb) => qb.where('assets.createdAt', '>=', options.createdAfter!)) + .$if(!!options.updatedBefore, (qb) => qb.where('assets.updatedAt', '<=', options.updatedBefore!)) + .$if(!!options.updatedAfter, (qb) => qb.where('assets.updatedAt', '>=', options.updatedAfter!)) + .$if(!!options.trashedBefore, (qb) => qb.where('assets.deletedAt', '<=', options.trashedBefore!)) + .$if(!!options.trashedAfter, (qb) => qb.where('assets.deletedAt', '>=', options.trashedAfter!)) + .$if(!!options.takenBefore, (qb) => qb.where('assets.fileCreatedAt', '<=', options.takenBefore!)) + .$if(!!options.takenAfter, (qb) => qb.where('assets.fileCreatedAt', '>=', options.takenAfter!)) + .$if(options.city !== undefined, (qb) => + qb + .innerJoin('exif', 'assets.id', 'exif.assetId') + .where('exif.city', options.city === null ? 'is' : '=', options.city!), + ) + .$if(options.state !== undefined, (qb) => + qb + .innerJoin('exif', 'assets.id', 'exif.assetId') + .where('exif.state', options.state === null ? 'is' : '=', options.state!), + ) + .$if(options.country !== undefined, (qb) => + qb + .innerJoin('exif', 'assets.id', 'exif.assetId') + .where('exif.country', options.country === null ? 'is' : '=', options.country!), + ) + .$if(options.make !== undefined, (qb) => + qb + .innerJoin('exif', 'assets.id', 'exif.assetId') + .where('exif.make', options.make === null ? 'is' : '=', options.make!), + ) + .$if(options.model !== undefined, (qb) => + qb + .innerJoin('exif', 'assets.id', 'exif.assetId') + .where('exif.model', options.model === null ? 'is' : '=', options.model!), + ) + .$if(options.lensModel !== undefined, (qb) => + qb + .innerJoin('exif', 'assets.id', 'exif.assetId') + .where('exif.lensModel', options.lensModel === null ? 'is' : '=', options.lensModel!), + ) + .$if(options.rating !== undefined, (qb) => + qb + .innerJoin('exif', 'assets.id', 'exif.assetId') + .where('exif.rating', options.rating === null ? 'is' : '=', options.rating!), + ) + .$if(!!options.checksum, (qb) => qb.where('assets.checksum', '=', options.checksum!)) + .$if(!!options.deviceAssetId, (qb) => qb.where('assets.deviceAssetId', '=', options.deviceAssetId!)) + .$if(!!options.deviceId, (qb) => qb.where('assets.deviceId', '=', options.deviceId!)) + .$if(!!options.id, (qb) => qb.where('assets.id', '=', asUuid(options.id!))) + .$if(!!options.libraryId, (qb) => qb.where('assets.libraryId', '=', asUuid(options.libraryId!))) + .$if(!!options.userIds, (qb) => qb.where('assets.ownerId', '=', anyUuid(options.userIds!))) + .$if(!!options.encodedVideoPath, (qb) => qb.where('assets.encodedVideoPath', '=', options.encodedVideoPath!)) + .$if(!!options.originalPath, (qb) => + qb.where(sql`f_unaccent(assets."originalPath")`, 'ilike', sql`'%' || f_unaccent(${options.originalPath}) || '%'`), + ) + .$if(!!options.originalFileName, (qb) => + qb.where( + sql`f_unaccent(assets."originalFileName")`, + 'ilike', + sql`'%' || f_unaccent(${options.originalFileName}) || '%'`, + ), + ) + .$if(!!options.description, (qb) => + qb + .innerJoin('exif', 'assets.id', 'exif.assetId') + .where(sql`f_unaccent(exif.description)`, 'ilike', sql`'%' || f_unaccent(${options.description}) || '%'`), + ) + .$if(!!options.type, (qb) => qb.where('assets.type', '=', options.type!)) + .$if(options.isFavorite !== undefined, (qb) => qb.where('assets.isFavorite', '=', options.isFavorite!)) + .$if(options.isOffline !== undefined, (qb) => qb.where('assets.isOffline', '=', options.isOffline!)) + .$if(options.isVisible !== undefined, (qb) => qb.where('assets.isVisible', '=', options.isVisible!)) + .$if(options.isArchived !== undefined, (qb) => qb.where('assets.isArchived', '=', options.isArchived!)) + .$if(options.isEncoded !== undefined, (qb) => + qb.where('assets.encodedVideoPath', options.isEncoded ? 'is not' : 'is', null), + ) + .$if(options.isMotion !== undefined, (qb) => + qb.where('assets.livePhotoVideoId', options.isMotion ? 'is not' : 'is', null), + ) + .$if(!!options.isNotInAlbum, (qb) => + qb.where((eb) => + eb.not(eb.exists((eb) => eb.selectFrom('albums_assets_assets').whereRef('assetsId', '=', 'assets.id'))), + ), + ) + .$if(!!options.withExif, withExifInner) + .$if(!!(options.withFaces || options.withPeople || options.personIds), (qb) => qb.select(withFacesAndPeople)) + .$if(!options.withDeleted, (qb) => qb.where('assets.deletedAt', 'is', null)); +} From 9e063c993c36f814c48ee0dd1fbebfaa24a0536d Mon Sep 17 00:00:00 2001 From: Matthew Momjian <50788000+mmomjian@users.noreply.github.com> Date: Mon, 21 Apr 2025 00:54:37 -0400 Subject: [PATCH 13/29] fix(docs): Database dump warnings (#17676) * docs * admin page * roadmap * whitespace * whitespace * no danger --- .../docs/administration/backup-and-restore.md | 31 ++++++++++++------- docs/src/pages/roadmap.tsx | 4 +-- i18n/en.json | 10 +++--- 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/docs/docs/administration/backup-and-restore.md b/docs/docs/administration/backup-and-restore.md index 817a7dca6d..7e55e4e88f 100644 --- a/docs/docs/administration/backup-and-restore.md +++ b/docs/docs/administration/backup-and-restore.md @@ -23,23 +23,32 @@ Refer to the official [postgres documentation](https://www.postgresql.org/docs/c It is not recommended to directly backup the `DB_DATA_LOCATION` folder. Doing so while the database is running can lead to a corrupted backup that cannot be restored. ::: -### Automatic Database Backups +### Automatic Database Dumps -For convenience, Immich will automatically create database backups by default. The backups are stored in `UPLOAD_LOCATION/backups`. -As mentioned above, you should make your own backup of these together with the asset folders as noted below. -You can adjust the schedule and amount of kept backups in the [admin settings](http://my.immich.app/admin/system-settings?isOpen=backup). -By default, Immich will keep the last 14 backups and create a new backup every day at 2:00 AM. +:::warning +The automatic database dumps can be used to restore the database in the event of damage to the Postgres database files. +There is no monitoring for these dumps and you will not be notified if they are unsuccessful. +::: -#### Trigger Backup +:::caution +The database dumps do **NOT** contain any pictures or videos, only metadata. They are only usable with a copy of the other files in `UPLOAD_LOCATION` as outlined below. +::: -You are able to trigger a backup in the [admin job status page](http://my.immich.app/admin/jobs-status). -Visit the page, open the "Create job" modal from the top right, select "Backup Database" and click "Confirm". -A job will run and trigger a backup, you can verify this worked correctly by checking the logs or the backup folder. -This backup will count towards the last X backups that will be kept based on your settings. +For disaster-recovery purposes, Immich will automatically create database dumps. The dumps are stored in `UPLOAD_LOCATION/backups`. +Please be sure to make your own, independent backup of the database together with the asset folders as noted below. +You can adjust the schedule and amount of kept database dumps in the [admin settings](http://my.immich.app/admin/system-settings?isOpen=backup). +By default, Immich will keep the last 14 database dumps and create a new dump every day at 2:00 AM. + +#### Trigger Dump + +You are able to trigger a database dump in the [admin job status page](http://my.immich.app/admin/jobs-status). +Visit the page, open the "Create job" modal from the top right, select "Create Database Dump" and click "Confirm". +A job will run and trigger a dump, you can verify this worked correctly by checking the logs or the `backups/` folder. +This dumps will count towards the last `X` dumps that will be kept based on your settings. #### Restoring -We hope to make restoring simpler in future versions, for now you can find the backups in the `UPLOAD_LOCATION/backups` folder on your host. +We hope to make restoring simpler in future versions, for now you can find the database dumps in the `UPLOAD_LOCATION/backups` folder on your host. Then please follow the steps in the following section for restoring the database. ### Manual Backup and Restore diff --git a/docs/src/pages/roadmap.tsx b/docs/src/pages/roadmap.tsx index b7ded8e8c9..205b976aed 100644 --- a/docs/src/pages/roadmap.tsx +++ b/docs/src/pages/roadmap.tsx @@ -266,8 +266,8 @@ const milestones: Item[] = [ withRelease({ icon: mdiDatabaseOutline, iconColor: 'brown', - title: 'Automatic database backups', - description: 'Database backups are now integrated into the Immich server', + title: 'Automatic database dumps', + description: 'Database dumps are now integrated into the Immich server', release: 'v1.120.0', }), { diff --git a/i18n/en.json b/i18n/en.json index c4b4746871..9951717de6 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -39,11 +39,11 @@ "authentication_settings_disable_all": "Are you sure you want to disable all login methods? Login will be completely disabled.", "authentication_settings_reenable": "To re-enable, use a Server Command.", "background_task_job": "Background Tasks", - "backup_database": "Backup Database", - "backup_database_enable_description": "Enable database backups", - "backup_keep_last_amount": "Amount of previous backups to keep", - "backup_settings": "Backup Settings", - "backup_settings_description": "Manage database backup settings", + "backup_database": "Create Database Dump", + "backup_database_enable_description": "Enable database dumps", + "backup_keep_last_amount": "Amount of previous dumps to keep", + "backup_settings": "Database Dump Settings", + "backup_settings_description": "Manage database dump settings. Note: These jobs are not monitored and you will not be notified of failure.", "check_all": "Check All", "cleanup": "Cleanup", "cleared_jobs": "Cleared jobs for: {job}", From 21a6eb30ff7333dbdf4899cd05686c0bd6cfe535 Mon Sep 17 00:00:00 2001 From: aviv926 <51673860+aviv926@users.noreply.github.com> Date: Mon, 21 Apr 2025 07:55:58 +0300 Subject: [PATCH 14/29] feat(docs): documentation update (#17720) Documentation update --- .../docs/features/ml-hardware-acceleration.md | 2 +- docs/src/pages/roadmap.tsx | 20 ++++++++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/docs/docs/features/ml-hardware-acceleration.md b/docs/docs/features/ml-hardware-acceleration.md index 8371e726b9..a94f8c8c64 100644 --- a/docs/docs/features/ml-hardware-acceleration.md +++ b/docs/docs/features/ml-hardware-acceleration.md @@ -42,7 +42,7 @@ You do not need to redo any machine learning jobs after enabling hardware accele - The GPU must have compute capability 5.2 or greater. - The server must have the official NVIDIA driver installed. -- The installed driver must be >= 535 (it must support CUDA 12.2). +- The installed driver must be >= 545 (it must support CUDA 12.3). - On Linux (except for WSL2), you also need to have [NVIDIA Container Toolkit][nvct] installed. #### ROCm diff --git a/docs/src/pages/roadmap.tsx b/docs/src/pages/roadmap.tsx index 205b976aed..4dc391cb27 100644 --- a/docs/src/pages/roadmap.tsx +++ b/docs/src/pages/roadmap.tsx @@ -76,6 +76,7 @@ import { mdiWeb, mdiDatabaseOutline, mdiLinkEdit, + mdiTagFaces, mdiMovieOpenPlayOutline, } from '@mdi/js'; import Layout from '@theme/Layout'; @@ -83,6 +84,8 @@ import React from 'react'; import { Item, Timeline } from '../components/timeline'; const releases = { + 'v1.130.0': new Date(2025, 2, 25), + 'v1.127.0': new Date(2025, 1, 26), 'v1.122.0': new Date(2024, 11, 5), 'v1.120.0': new Date(2024, 10, 6), 'v1.114.0': new Date(2024, 8, 6), @@ -242,6 +245,21 @@ const roadmap: Item[] = [ ]; const milestones: Item[] = [ + withRelease({ + icon: mdiFolderMultiple, + iconColor: 'brown', + title: 'Folders view in the mobile app', + description: 'Browse your photos and videos in their folder structure inside the mobile app', + release: 'v1.130.0', + }), + withRelease({ + icon: mdiTagFaces, + iconColor: 'teal', + title: 'Manual face tagging', + description: + 'Manually tag or remove faces in photos and videos, even when automatic detection misses or misidentifies them.', + release: 'v1.127.0', + }), { icon: mdiStar, iconColor: 'gold', @@ -300,7 +318,7 @@ const milestones: Item[] = [ withRelease({ icon: mdiFolderMultiple, iconColor: 'brown', - title: 'Folders', + title: 'Folders view', description: 'Browse your photos and videos in their folder structure', release: 'v1.113.0', }), From c49fd2065bcce54415066bb49a604f4a5fabb17e Mon Sep 17 00:00:00 2001 From: Yaros Date: Mon, 21 Apr 2025 07:18:25 +0200 Subject: [PATCH 15/29] chore(mobile): bump ios deployment target (#17715) * chore: bump ios deployment target * podfile --------- Co-authored-by: Alex --- mobile/ios/Podfile | 4 ++-- mobile/ios/Podfile.lock | 4 ++-- mobile/ios/Runner.xcodeproj/project.pbxproj | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/mobile/ios/Podfile b/mobile/ios/Podfile index a98032db20..ca0166a382 100644 --- a/mobile/ios/Podfile +++ b/mobile/ios/Podfile @@ -1,5 +1,5 @@ # Uncomment this line to define a global platform for your project -# platform :ios, '12.0' +platform :ios, '14.0' # CocoaPods analytics sends network stats synchronously affecting flutter build latency. ENV['COCOAPODS_DISABLE_STATS'] = 'true' @@ -45,7 +45,7 @@ post_install do |installer| installer.generated_projects.each do |project| project.targets.each do |target| target.build_configurations.each do |config| - config.build_settings['IPHONEOS_DEPLOYMENT_TARGET'] = '13.0' + config.build_settings['IPHONEOS_DEPLOYMENT_TARGET'] = '14.0' end end end diff --git a/mobile/ios/Podfile.lock b/mobile/ios/Podfile.lock index 908ee84aed..9740d6aa52 100644 --- a/mobile/ios/Podfile.lock +++ b/mobile/ios/Podfile.lock @@ -224,7 +224,7 @@ EXTERNAL SOURCES: :path: ".symlinks/plugins/wakelock_plus/ios" SPEC CHECKSUMS: - background_downloader: b42a56120f5348bff70e74222f0e9e6f7f1a1537 + background_downloader: 50e91d979067b82081aba359d7d916b3ba5fadad connectivity_plus: cb623214f4e1f6ef8fe7403d580fdad517d2f7dd device_info_plus: 21fcca2080fbcd348be798aa36c3e5ed849eefbe DKImagePickerController: 946cec48c7873164274ecc4624d19e3da4c1ef3c @@ -261,6 +261,6 @@ SPEC CHECKSUMS: url_launcher_ios: 694010445543906933d732453a59da0a173ae33d wakelock_plus: 04623e3f525556020ebd4034310f20fe7fda8b49 -PODFILE CHECKSUM: 03b7eead4ee77b9e778179eeb0f3b5513617451c +PODFILE CHECKSUM: 7ce312f2beab01395db96f6969d90a447279cf45 COCOAPODS: 1.16.2 diff --git a/mobile/ios/Runner.xcodeproj/project.pbxproj b/mobile/ios/Runner.xcodeproj/project.pbxproj index 83c231d741..69f122cf17 100644 --- a/mobile/ios/Runner.xcodeproj/project.pbxproj +++ b/mobile/ios/Runner.xcodeproj/project.pbxproj @@ -546,7 +546,7 @@ DEVELOPMENT_TEAM = 2F67MQ8R79; ENABLE_BITCODE = NO; INFOPLIST_FILE = Runner/Info.plist; - IPHONEOS_DEPLOYMENT_TARGET = 13.0; + IPHONEOS_DEPLOYMENT_TARGET = 14.0; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", @@ -690,7 +690,7 @@ DEVELOPMENT_TEAM = 2F67MQ8R79; ENABLE_BITCODE = NO; INFOPLIST_FILE = Runner/Info.plist; - IPHONEOS_DEPLOYMENT_TARGET = 13.0; + IPHONEOS_DEPLOYMENT_TARGET = 14.0; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", @@ -720,7 +720,7 @@ DEVELOPMENT_TEAM = 2F67MQ8R79; ENABLE_BITCODE = NO; INFOPLIST_FILE = Runner/Info.plist; - IPHONEOS_DEPLOYMENT_TARGET = 13.0; + IPHONEOS_DEPLOYMENT_TARGET = 14.0; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", From f0ff8581da6a8e67338f2123c8c24f7557058bf1 Mon Sep 17 00:00:00 2001 From: Yaros Date: Mon, 21 Apr 2025 07:55:13 +0200 Subject: [PATCH 16/29] feat(mobile): map improvements (#17714) * fix: remove unnecessary db operations in map * feat: use user's location for map thumbnails * chore: refactored handleMapEvents * fix: location fails fetching & update geolocator * chore: minor refactor * chore: small style tweak --------- Co-authored-by: Alex --- mobile/lib/pages/library/library.page.dart | 145 +++++++++++------- .../places/places_collection.page.dart | 15 +- mobile/lib/pages/search/map/map.page.dart | 16 +- .../search/map/map_location_picker.page.dart | 2 +- mobile/lib/routing/router.gr.dart | 72 ++++++++- mobile/lib/utils/map_utils.dart | 21 ++- mobile/lib/widgets/map/map_asset_grid.dart | 57 +++++-- .../widgets/search/search_map_thumbnail.dart | 2 +- mobile/pubspec.lock | 12 +- mobile/pubspec.yaml | 2 +- 10 files changed, 240 insertions(+), 104 deletions(-) diff --git a/mobile/lib/pages/library/library.page.dart b/mobile/lib/pages/library/library.page.dart index 1852fb7877..c08a1c715d 100644 --- a/mobile/lib/pages/library/library.page.dart +++ b/mobile/lib/pages/library/library.page.dart @@ -1,6 +1,7 @@ import 'package:auto_route/auto_route.dart'; import 'package:easy_localization/easy_localization.dart'; import 'package:flutter/material.dart'; +import 'package:geolocator/geolocator.dart'; import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/domain/models/user.model.dart'; import 'package:immich_mobile/extensions/asyncvalue_extensions.dart'; @@ -12,6 +13,7 @@ import 'package:immich_mobile/providers/server_info.provider.dart'; import 'package:immich_mobile/routing/router.dart'; import 'package:immich_mobile/services/api.service.dart'; import 'package:immich_mobile/utils/image_url_builder.dart'; +import 'package:immich_mobile/utils/map_utils.dart'; import 'package:immich_mobile/widgets/album/album_thumbnail_card.dart'; import 'package:immich_mobile/widgets/common/immich_app_bar.dart'; import 'package:immich_mobile/widgets/common/user_avatar.dart'; @@ -297,32 +299,34 @@ class LocalAlbumsCollectionCard extends HookConsumerWidget { child: Column( crossAxisAlignment: CrossAxisAlignment.start, children: [ - Container( + SizedBox( height: size, width: size, - decoration: BoxDecoration( - borderRadius: BorderRadius.circular(20), - gradient: LinearGradient( - colors: [ - context.colorScheme.primary.withAlpha(30), - context.colorScheme.primary.withAlpha(25), - ], - begin: Alignment.topCenter, - end: Alignment.bottomCenter, + child: DecoratedBox( + decoration: BoxDecoration( + borderRadius: const BorderRadius.all(Radius.circular(20)), + gradient: LinearGradient( + colors: [ + context.colorScheme.primary.withAlpha(30), + context.colorScheme.primary.withAlpha(25), + ], + begin: Alignment.topCenter, + end: Alignment.bottomCenter, + ), + ), + child: GridView.count( + crossAxisCount: 2, + padding: const EdgeInsets.all(12), + crossAxisSpacing: 8, + mainAxisSpacing: 8, + physics: const NeverScrollableScrollPhysics(), + children: albums.take(4).map((album) { + return AlbumThumbnailCard( + album: album, + showTitle: false, + ); + }).toList(), ), - ), - child: GridView.count( - crossAxisCount: 2, - padding: const EdgeInsets.all(12), - crossAxisSpacing: 8, - mainAxisSpacing: 8, - physics: const NeverScrollableScrollPhysics(), - children: albums.take(4).map((album) { - return AlbumThumbnailCard( - album: album, - showTitle: false, - ); - }).toList(), ), ), Padding( @@ -353,43 +357,66 @@ class PlacesCollectionCard extends StatelessWidget { final widthFactor = isTablet ? 0.25 : 0.5; final size = context.width * widthFactor - 20.0; - return GestureDetector( - onTap: () => context.pushRoute(const PlacesCollectionRoute()), - child: Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - Container( - height: size, - width: size, - decoration: BoxDecoration( - borderRadius: BorderRadius.circular(20), - color: context.colorScheme.secondaryContainer.withAlpha(100), - ), - child: IgnorePointer( - child: MapThumbnail( - zoom: 8, - centre: const LatLng( - 21.44950, - -157.91959, - ), - showAttribution: false, - themeMode: - context.isDarkTheme ? ThemeMode.dark : ThemeMode.light, - ), - ), - ), - Padding( - padding: const EdgeInsets.all(8.0), - child: Text( - 'places'.tr(), - style: context.textTheme.titleSmall?.copyWith( - color: context.colorScheme.onSurface, - fontWeight: FontWeight.w500, - ), - ), - ), - ], + return FutureBuilder<(Position?, LocationPermission?)>( + future: MapUtils.checkPermAndGetLocation( + context: context, + silent: true, ), + builder: (context, snapshot) { + var position = snapshot.data?.$1; + return GestureDetector( + onTap: () => context.pushRoute( + PlacesCollectionRoute( + currentLocation: position != null + ? LatLng(position.latitude, position.longitude) + : null, + ), + ), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + SizedBox( + height: size, + width: size, + child: DecoratedBox( + decoration: BoxDecoration( + borderRadius: + const BorderRadius.all(Radius.circular(20)), + color: context.colorScheme.secondaryContainer + .withAlpha(100), + ), + child: IgnorePointer( + child: snapshot.connectionState == + ConnectionState.waiting + ? const Center(child: CircularProgressIndicator()) + : MapThumbnail( + zoom: 8, + centre: LatLng( + position?.latitude ?? 21.44950, + position?.longitude ?? -157.91959, + ), + showAttribution: false, + themeMode: context.isDarkTheme + ? ThemeMode.dark + : ThemeMode.light, + ), + ), + ), + ), + Padding( + padding: const EdgeInsets.all(8.0), + child: Text( + 'places'.tr(), + style: context.textTheme.titleSmall?.copyWith( + color: context.colorScheme.onSurface, + fontWeight: FontWeight.w500, + ), + ), + ), + ], + ), + ); + }, ); }, ); diff --git a/mobile/lib/pages/library/places/places_collection.page.dart b/mobile/lib/pages/library/places/places_collection.page.dart index f9a2d4292c..5f2dea0dec 100644 --- a/mobile/lib/pages/library/places/places_collection.page.dart +++ b/mobile/lib/pages/library/places/places_collection.page.dart @@ -19,7 +19,8 @@ import 'package:maplibre_gl/maplibre_gl.dart'; @RoutePage() class PlacesCollectionPage extends HookConsumerWidget { - const PlacesCollectionPage({super.key}); + const PlacesCollectionPage({super.key, this.currentLocation}); + final LatLng? currentLocation; @override Widget build(BuildContext context, WidgetRef ref) { final places = ref.watch(getAllPlacesProvider); @@ -58,12 +59,14 @@ class PlacesCollectionPage extends HookConsumerWidget { height: 200, width: context.width, child: MapThumbnail( - onTap: (_, __) => context.pushRoute(const MapRoute()), + onTap: (_, __) => context + .pushRoute(MapRoute(initialLocation: currentLocation)), zoom: 8, - centre: const LatLng( - 21.44950, - -157.91959, - ), + centre: currentLocation ?? + const LatLng( + 21.44950, + -157.91959, + ), showAttribution: false, themeMode: context.isDarkTheme ? ThemeMode.dark : ThemeMode.light, diff --git a/mobile/lib/pages/search/map/map.page.dart b/mobile/lib/pages/search/map/map.page.dart index 0e64759241..b80b96f94f 100644 --- a/mobile/lib/pages/search/map/map.page.dart +++ b/mobile/lib/pages/search/map/map.page.dart @@ -34,7 +34,8 @@ import 'package:maplibre_gl/maplibre_gl.dart'; @RoutePage() class MapPage extends HookConsumerWidget { - const MapPage({super.key}); + const MapPage({super.key, this.initialLocation}); + final LatLng? initialLocation; @override Widget build(BuildContext context, WidgetRef ref) { @@ -235,7 +236,8 @@ class MapPage extends HookConsumerWidget { } void onZoomToLocation() async { - final (location, error) = await MapUtils.checkPermAndGetLocation(context); + final (location, error) = + await MapUtils.checkPermAndGetLocation(context: context); if (error != null) { if (error == LocationPermission.unableToDetermine && context.mounted) { ImmichToast.show( @@ -272,6 +274,7 @@ class MapPage extends HookConsumerWidget { body: Stack( children: [ _MapWithMarker( + initialLocation: initialLocation, style: style, selectedMarker: selectedMarker, onMapCreated: onMapCreated, @@ -303,6 +306,7 @@ class MapPage extends HookConsumerWidget { body: Stack( children: [ _MapWithMarker( + initialLocation: initialLocation, style: style, selectedMarker: selectedMarker, onMapCreated: onMapCreated, @@ -368,6 +372,7 @@ class _MapWithMarker extends StatelessWidget { final OnStyleLoadedCallback onStyleLoaded; final Function()? onMarkerTapped; final ValueNotifier<_AssetMarkerMeta?> selectedMarker; + final LatLng? initialLocation; const _MapWithMarker({ required this.style, @@ -377,6 +382,7 @@ class _MapWithMarker extends StatelessWidget { required this.onStyleLoaded, required this.selectedMarker, this.onMarkerTapped, + this.initialLocation, }); @override @@ -389,8 +395,10 @@ class _MapWithMarker extends StatelessWidget { children: [ style.widgetWhen( onData: (style) => MapLibreMap( - initialCameraPosition: - const CameraPosition(target: LatLng(0, 0)), + initialCameraPosition: CameraPosition( + target: initialLocation ?? const LatLng(0, 0), + zoom: initialLocation != null ? 12 : 0, + ), styleString: style, // This is needed to update the selectedMarker's position on map camera updates // The changes are notified through the mapController ValueListener which is added in [onMapCreated] diff --git a/mobile/lib/pages/search/map/map_location_picker.page.dart b/mobile/lib/pages/search/map/map_location_picker.page.dart index 9d526d8080..f27deae052 100644 --- a/mobile/lib/pages/search/map/map_location_picker.page.dart +++ b/mobile/lib/pages/search/map/map_location_picker.page.dart @@ -46,7 +46,7 @@ class MapLocationPickerPage extends HookConsumerWidget { Future getCurrentLocation() async { var (currentLocation, _) = - await MapUtils.checkPermAndGetLocation(context); + await MapUtils.checkPermAndGetLocation(context: context); if (currentLocation == null) { return; diff --git a/mobile/lib/routing/router.gr.dart b/mobile/lib/routing/router.gr.dart index a78371e05e..89e83e8159 100644 --- a/mobile/lib/routing/router.gr.dart +++ b/mobile/lib/routing/router.gr.dart @@ -1024,10 +1024,17 @@ class MapLocationPickerRouteArgs { /// generated route for /// [MapPage] -class MapRoute extends PageRouteInfo { - const MapRoute({List? children}) - : super( +class MapRoute extends PageRouteInfo { + MapRoute({ + Key? key, + LatLng? initialLocation, + List? children, + }) : super( MapRoute.name, + args: MapRouteArgs( + key: key, + initialLocation: initialLocation, + ), initialChildren: children, ); @@ -1036,11 +1043,32 @@ class MapRoute extends PageRouteInfo { static PageInfo page = PageInfo( name, builder: (data) { - return const MapPage(); + final args = + data.argsAs(orElse: () => const MapRouteArgs()); + return MapPage( + key: args.key, + initialLocation: args.initialLocation, + ); }, ); } +class MapRouteArgs { + const MapRouteArgs({ + this.key, + this.initialLocation, + }); + + final Key? key; + + final LatLng? initialLocation; + + @override + String toString() { + return 'MapRouteArgs{key: $key, initialLocation: $initialLocation}'; + } +} + /// generated route for /// [MemoryPage] class MemoryRoute extends PageRouteInfo { @@ -1333,10 +1361,17 @@ class PhotosRoute extends PageRouteInfo { /// generated route for /// [PlacesCollectionPage] -class PlacesCollectionRoute extends PageRouteInfo { - const PlacesCollectionRoute({List? children}) - : super( +class PlacesCollectionRoute extends PageRouteInfo { + PlacesCollectionRoute({ + Key? key, + LatLng? currentLocation, + List? children, + }) : super( PlacesCollectionRoute.name, + args: PlacesCollectionRouteArgs( + key: key, + currentLocation: currentLocation, + ), initialChildren: children, ); @@ -1345,11 +1380,32 @@ class PlacesCollectionRoute extends PageRouteInfo { static PageInfo page = PageInfo( name, builder: (data) { - return const PlacesCollectionPage(); + final args = data.argsAs( + orElse: () => const PlacesCollectionRouteArgs()); + return PlacesCollectionPage( + key: args.key, + currentLocation: args.currentLocation, + ); }, ); } +class PlacesCollectionRouteArgs { + const PlacesCollectionRouteArgs({ + this.key, + this.currentLocation, + }); + + final Key? key; + + final LatLng? currentLocation; + + @override + String toString() { + return 'PlacesCollectionRouteArgs{key: $key, currentLocation: $currentLocation}'; + } +} + /// generated route for /// [RecentlyAddedPage] class RecentlyAddedRoute extends PageRouteInfo { diff --git a/mobile/lib/utils/map_utils.dart b/mobile/lib/utils/map_utils.dart index 44f7ebf271..df1ff28d8f 100644 --- a/mobile/lib/utils/map_utils.dart +++ b/mobile/lib/utils/map_utils.dart @@ -64,12 +64,13 @@ class MapUtils { 'features': markers.map(_addFeature).toList(), }; - static Future<(Position?, LocationPermission?)> checkPermAndGetLocation( - BuildContext context, - ) async { + static Future<(Position?, LocationPermission?)> checkPermAndGetLocation({ + required BuildContext context, + bool silent = false, + }) async { try { bool serviceEnabled = await Geolocator.isLocationServiceEnabled(); - if (!serviceEnabled) { + if (!serviceEnabled && !silent) { showDialog( context: context, builder: (context) => _LocationServiceDisabledDialog(), @@ -80,7 +81,7 @@ class MapUtils { LocationPermission permission = await Geolocator.checkPermission(); bool shouldRequestPermission = false; - if (permission == LocationPermission.denied) { + if (permission == LocationPermission.denied && !silent) { shouldRequestPermission = await showDialog( context: context, builder: (context) => _LocationPermissionDisabledDialog(), @@ -94,15 +95,19 @@ class MapUtils { permission == LocationPermission.deniedForever) { // Open app settings only if you did not request for permission before if (permission == LocationPermission.deniedForever && - !shouldRequestPermission) { + !shouldRequestPermission && + !silent) { await Geolocator.openAppSettings(); } return (null, LocationPermission.deniedForever); } Position currentUserLocation = await Geolocator.getCurrentPosition( - desiredAccuracy: LocationAccuracy.medium, - timeLimit: const Duration(seconds: 5), + locationSettings: const LocationSettings( + accuracy: LocationAccuracy.high, + distanceFilter: 0, + timeLimit: Duration(seconds: 5), + ), ); return (currentUserLocation, null); } catch (error, stack) { diff --git a/mobile/lib/widgets/map/map_asset_grid.dart b/mobile/lib/widgets/map/map_asset_grid.dart index 18003cf293..a9ddc86df9 100644 --- a/mobile/lib/widgets/map/map_asset_grid.dart +++ b/mobile/lib/widgets/map/map_asset_grid.dart @@ -46,12 +46,39 @@ class MapAssetGrid extends HookConsumerWidget { final gridScrollThrottler = useThrottler(interval: const Duration(milliseconds: 300)); + // Add a cache for assets we've already loaded + final assetCache = useRef>({}); + void handleMapEvents(MapEvent event) async { if (event is MapAssetsInBoundsUpdated) { - assetsInBounds.value = await ref - .read(dbProvider) - .assets - .getAllByRemoteId(event.assetRemoteIds); + final assetIds = event.assetRemoteIds; + final missingIds = []; + final currentAssets = []; + + for (final id in assetIds) { + final asset = assetCache.value[id]; + if (asset != null) { + currentAssets.add(asset); + } else { + missingIds.add(id); + } + } + + // Only fetch missing assets + if (missingIds.isNotEmpty) { + final newAssets = + await ref.read(dbProvider).assets.getAllByRemoteId(missingIds); + + // Add new assets to cache and current list + for (final asset in newAssets) { + if (asset.remoteId != null) { + assetCache.value[asset.remoteId!] = asset; + currentAssets.add(asset); + } + } + } + + assetsInBounds.value = currentAssets; return; } } @@ -124,7 +151,7 @@ class MapAssetGrid extends HookConsumerWidget { alignment: Alignment.bottomCenter, child: FractionallySizedBox( // Place it just below the drag handle - heightFactor: 0.80, + heightFactor: 0.87, child: assetsInBounds.value.isNotEmpty ? ref .watch(assetsTimelineProvider(assetsInBounds.value)) @@ -251,8 +278,18 @@ class _MapSheetDragRegion extends StatelessWidget { const SizedBox(height: 15), const CustomDraggingHandle(), const SizedBox(height: 15), - Text(assetsInBoundsText, style: context.textTheme.bodyLarge), - const Divider(height: 35), + Center( + child: Text( + assetsInBoundsText, + style: TextStyle( + fontSize: 20, + color: context.textTheme.displayLarge?.color + ?.withValues(alpha: 0.75), + fontWeight: FontWeight.w500, + ), + ), + ), + const SizedBox(height: 8), ], ), ValueListenableBuilder( @@ -260,14 +297,14 @@ class _MapSheetDragRegion extends StatelessWidget { builder: (_, value, __) => Visibility( visible: value != null, child: Positioned( - right: 15, - top: 15, + right: 18, + top: 24, child: IconButton( icon: Icon( Icons.map_outlined, color: context.textTheme.displayLarge?.color, ), - iconSize: 20, + iconSize: 24, tooltip: 'Zoom to bounds', onPressed: () => onZoomToAsset?.call(value!), ), diff --git a/mobile/lib/widgets/search/search_map_thumbnail.dart b/mobile/lib/widgets/search/search_map_thumbnail.dart index b4a12ab826..78af8f936b 100644 --- a/mobile/lib/widgets/search/search_map_thumbnail.dart +++ b/mobile/lib/widgets/search/search_map_thumbnail.dart @@ -20,7 +20,7 @@ class SearchMapThumbnail extends StatelessWidget { return ThumbnailWithInfoContainer( label: 'search_page_your_map'.tr(), onTap: () { - context.pushRoute(const MapRoute()); + context.pushRoute(MapRoute()); }, child: IgnorePointer( child: MapThumbnail( diff --git a/mobile/pubspec.lock b/mobile/pubspec.lock index 235b3f71c3..9e8aced11c 100644 --- a/mobile/pubspec.lock +++ b/mobile/pubspec.lock @@ -696,18 +696,18 @@ packages: dependency: "direct main" description: name: geolocator - sha256: "6cb9fb6e5928b58b9a84bdf85012d757fd07aab8215c5205337021c4999bad27" + sha256: e7ebfa04ce451daf39b5499108c973189a71a919aa53c1204effda1c5b93b822 url: "https://pub.dev" source: hosted - version: "11.1.0" + version: "14.0.0" geolocator_android: dependency: transitive description: name: geolocator_android - sha256: "7aefc530db47d90d0580b552df3242440a10fe60814496a979aa67aa98b1fd47" + sha256: "114072db5d1dce0ec0b36af2697f55c133bc89a2c8dd513e137c0afe59696ed4" url: "https://pub.dev" source: hosted - version: "4.6.1" + version: "5.0.1+1" geolocator_apple: dependency: transitive description: @@ -728,10 +728,10 @@ packages: dependency: transitive description: name: geolocator_web - sha256: "49d8f846ebeb5e2b6641fe477a7e97e5dd73f03cbfef3fd5c42177b7300fb0ed" + sha256: b1ae9bdfd90f861fde8fd4f209c37b953d65e92823cb73c7dee1fa021b06f172 url: "https://pub.dev" source: hosted - version: "3.0.0" + version: "4.1.3" geolocator_windows: dependency: transitive description: diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index fdd91e1f87..44d2e7e5d1 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -35,7 +35,7 @@ dependencies: flutter_udid: ^3.0.0 flutter_web_auth_2: ^5.0.0-alpha.0 fluttertoast: ^8.2.12 - geolocator: ^11.0.0 + geolocator: ^14.0.0 hooks_riverpod: ^2.6.1 http: ^1.3.0 image_picker: ^1.1.2 From 488dc4efbdcb64fd84d4cf67f8b145160313a1c4 Mon Sep 17 00:00:00 2001 From: Jason Rasmussen Date: Mon, 21 Apr 2025 10:49:26 -0400 Subject: [PATCH 17/29] refactor: notification-admin controller (#17748) --- mobile/openapi/README.md | 4 ++-- mobile/openapi/lib/api.dart | 2 +- ..._api.dart => notifications_admin_api.dart} | 24 +++++++++---------- open-api/immich-openapi-specs.json | 12 +++++----- open-api/typescript-sdk/src/fetch-client.ts | 8 +++---- server/src/controllers/index.ts | 4 ++-- ...er.ts => notification-admin.controller.ts} | 10 ++++---- .../notification-settings.svelte | 4 ++-- .../template-settings.svelte | 4 ++-- 9 files changed, 36 insertions(+), 36 deletions(-) rename mobile/openapi/lib/api/{notifications_api.dart => notifications_admin_api.dart} (75%) rename server/src/controllers/{notification.controller.ts => notification-admin.controller.ts} (80%) diff --git a/mobile/openapi/README.md b/mobile/openapi/README.md index 0ae07e9efd..fef299b5af 100644 --- a/mobile/openapi/README.md +++ b/mobile/openapi/README.md @@ -145,8 +145,8 @@ Class | Method | HTTP request | Description *MemoriesApi* | [**removeMemoryAssets**](doc//MemoriesApi.md#removememoryassets) | **DELETE** /memories/{id}/assets | *MemoriesApi* | [**searchMemories**](doc//MemoriesApi.md#searchmemories) | **GET** /memories | *MemoriesApi* | [**updateMemory**](doc//MemoriesApi.md#updatememory) | **PUT** /memories/{id} | -*NotificationsApi* | [**getNotificationTemplate**](doc//NotificationsApi.md#getnotificationtemplate) | **POST** /notifications/templates/{name} | -*NotificationsApi* | [**sendTestEmail**](doc//NotificationsApi.md#sendtestemail) | **POST** /notifications/test-email | +*NotificationsAdminApi* | [**getNotificationTemplateAdmin**](doc//NotificationsAdminApi.md#getnotificationtemplateadmin) | **POST** /notifications/admin/templates/{name} | +*NotificationsAdminApi* | [**sendTestEmailAdmin**](doc//NotificationsAdminApi.md#sendtestemailadmin) | **POST** /notifications/admin/test-email | *OAuthApi* | [**finishOAuth**](doc//OAuthApi.md#finishoauth) | **POST** /oauth/callback | *OAuthApi* | [**linkOAuthAccount**](doc//OAuthApi.md#linkoauthaccount) | **POST** /oauth/link | *OAuthApi* | [**redirectOAuthToMobile**](doc//OAuthApi.md#redirectoauthtomobile) | **GET** /oauth/mobile-redirect | diff --git a/mobile/openapi/lib/api.dart b/mobile/openapi/lib/api.dart index 3986362c96..ff5a95bbbc 100644 --- a/mobile/openapi/lib/api.dart +++ b/mobile/openapi/lib/api.dart @@ -44,7 +44,7 @@ part 'api/jobs_api.dart'; part 'api/libraries_api.dart'; part 'api/map_api.dart'; part 'api/memories_api.dart'; -part 'api/notifications_api.dart'; +part 'api/notifications_admin_api.dart'; part 'api/o_auth_api.dart'; part 'api/partners_api.dart'; part 'api/people_api.dart'; diff --git a/mobile/openapi/lib/api/notifications_api.dart b/mobile/openapi/lib/api/notifications_admin_api.dart similarity index 75% rename from mobile/openapi/lib/api/notifications_api.dart rename to mobile/openapi/lib/api/notifications_admin_api.dart index 518a1baa4a..c58bf8978d 100644 --- a/mobile/openapi/lib/api/notifications_api.dart +++ b/mobile/openapi/lib/api/notifications_admin_api.dart @@ -11,20 +11,20 @@ part of openapi.api; -class NotificationsApi { - NotificationsApi([ApiClient? apiClient]) : apiClient = apiClient ?? defaultApiClient; +class NotificationsAdminApi { + NotificationsAdminApi([ApiClient? apiClient]) : apiClient = apiClient ?? defaultApiClient; final ApiClient apiClient; - /// Performs an HTTP 'POST /notifications/templates/{name}' operation and returns the [Response]. + /// Performs an HTTP 'POST /notifications/admin/templates/{name}' operation and returns the [Response]. /// Parameters: /// /// * [String] name (required): /// /// * [TemplateDto] templateDto (required): - Future getNotificationTemplateWithHttpInfo(String name, TemplateDto templateDto,) async { + Future getNotificationTemplateAdminWithHttpInfo(String name, TemplateDto templateDto,) async { // ignore: prefer_const_declarations - final apiPath = r'/notifications/templates/{name}' + final apiPath = r'/notifications/admin/templates/{name}' .replaceAll('{name}', name); // ignore: prefer_final_locals @@ -53,8 +53,8 @@ class NotificationsApi { /// * [String] name (required): /// /// * [TemplateDto] templateDto (required): - Future getNotificationTemplate(String name, TemplateDto templateDto,) async { - final response = await getNotificationTemplateWithHttpInfo(name, templateDto,); + Future getNotificationTemplateAdmin(String name, TemplateDto templateDto,) async { + final response = await getNotificationTemplateAdminWithHttpInfo(name, templateDto,); if (response.statusCode >= HttpStatus.badRequest) { throw ApiException(response.statusCode, await _decodeBodyBytes(response)); } @@ -68,13 +68,13 @@ class NotificationsApi { return null; } - /// Performs an HTTP 'POST /notifications/test-email' operation and returns the [Response]. + /// Performs an HTTP 'POST /notifications/admin/test-email' operation and returns the [Response]. /// Parameters: /// /// * [SystemConfigSmtpDto] systemConfigSmtpDto (required): - Future sendTestEmailWithHttpInfo(SystemConfigSmtpDto systemConfigSmtpDto,) async { + Future sendTestEmailAdminWithHttpInfo(SystemConfigSmtpDto systemConfigSmtpDto,) async { // ignore: prefer_const_declarations - final apiPath = r'/notifications/test-email'; + final apiPath = r'/notifications/admin/test-email'; // ignore: prefer_final_locals Object? postBody = systemConfigSmtpDto; @@ -100,8 +100,8 @@ class NotificationsApi { /// Parameters: /// /// * [SystemConfigSmtpDto] systemConfigSmtpDto (required): - Future sendTestEmail(SystemConfigSmtpDto systemConfigSmtpDto,) async { - final response = await sendTestEmailWithHttpInfo(systemConfigSmtpDto,); + Future sendTestEmailAdmin(SystemConfigSmtpDto systemConfigSmtpDto,) async { + final response = await sendTestEmailAdminWithHttpInfo(systemConfigSmtpDto,); if (response.statusCode >= HttpStatus.badRequest) { throw ApiException(response.statusCode, await _decodeBodyBytes(response)); } diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index 4e8e7ab834..169c076341 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -3485,9 +3485,9 @@ ] } }, - "/notifications/templates/{name}": { + "/notifications/admin/templates/{name}": { "post": { - "operationId": "getNotificationTemplate", + "operationId": "getNotificationTemplateAdmin", "parameters": [ { "name": "name", @@ -3532,13 +3532,13 @@ } ], "tags": [ - "Notifications" + "Notifications (Admin)" ] } }, - "/notifications/test-email": { + "/notifications/admin/test-email": { "post": { - "operationId": "sendTestEmail", + "operationId": "sendTestEmailAdmin", "parameters": [], "requestBody": { "content": { @@ -3574,7 +3574,7 @@ } ], "tags": [ - "Notifications" + "Notifications (Admin)" ] } }, diff --git a/open-api/typescript-sdk/src/fetch-client.ts b/open-api/typescript-sdk/src/fetch-client.ts index f82f5bc9a7..e45449c9cd 100644 --- a/open-api/typescript-sdk/src/fetch-client.ts +++ b/open-api/typescript-sdk/src/fetch-client.ts @@ -2318,26 +2318,26 @@ export function addMemoryAssets({ id, bulkIdsDto }: { body: bulkIdsDto }))); } -export function getNotificationTemplate({ name, templateDto }: { +export function getNotificationTemplateAdmin({ name, templateDto }: { name: string; templateDto: TemplateDto; }, opts?: Oazapfts.RequestOpts) { return oazapfts.ok(oazapfts.fetchJson<{ status: 200; data: TemplateResponseDto; - }>(`/notifications/templates/${encodeURIComponent(name)}`, oazapfts.json({ + }>(`/notifications/admin/templates/${encodeURIComponent(name)}`, oazapfts.json({ ...opts, method: "POST", body: templateDto }))); } -export function sendTestEmail({ systemConfigSmtpDto }: { +export function sendTestEmailAdmin({ systemConfigSmtpDto }: { systemConfigSmtpDto: SystemConfigSmtpDto; }, opts?: Oazapfts.RequestOpts) { return oazapfts.ok(oazapfts.fetchJson<{ status: 200; data: TestEmailResponseDto; - }>("/notifications/test-email", oazapfts.json({ + }>("/notifications/admin/test-email", oazapfts.json({ ...opts, method: "POST", body: systemConfigSmtpDto diff --git a/server/src/controllers/index.ts b/server/src/controllers/index.ts index c9d63f8bcd..0da0aac8b1 100644 --- a/server/src/controllers/index.ts +++ b/server/src/controllers/index.ts @@ -13,7 +13,7 @@ import { JobController } from 'src/controllers/job.controller'; import { LibraryController } from 'src/controllers/library.controller'; import { MapController } from 'src/controllers/map.controller'; import { MemoryController } from 'src/controllers/memory.controller'; -import { NotificationController } from 'src/controllers/notification.controller'; +import { NotificationAdminController } from 'src/controllers/notification-admin.controller'; import { OAuthController } from 'src/controllers/oauth.controller'; import { PartnerController } from 'src/controllers/partner.controller'; import { PersonController } from 'src/controllers/person.controller'; @@ -47,7 +47,7 @@ export const controllers = [ LibraryController, MapController, MemoryController, - NotificationController, + NotificationAdminController, OAuthController, PartnerController, PersonController, diff --git a/server/src/controllers/notification.controller.ts b/server/src/controllers/notification-admin.controller.ts similarity index 80% rename from server/src/controllers/notification.controller.ts rename to server/src/controllers/notification-admin.controller.ts index 39946a9fc9..f3ce4cdac9 100644 --- a/server/src/controllers/notification.controller.ts +++ b/server/src/controllers/notification-admin.controller.ts @@ -7,22 +7,22 @@ import { Auth, Authenticated } from 'src/middleware/auth.guard'; import { EmailTemplate } from 'src/repositories/notification.repository'; import { NotificationService } from 'src/services/notification.service'; -@ApiTags('Notifications') -@Controller('notifications') -export class NotificationController { +@ApiTags('Notifications (Admin)') +@Controller('notifications/admin') +export class NotificationAdminController { constructor(private service: NotificationService) {} @Post('test-email') @HttpCode(HttpStatus.OK) @Authenticated({ admin: true }) - sendTestEmail(@Auth() auth: AuthDto, @Body() dto: SystemConfigSmtpDto): Promise { + sendTestEmailAdmin(@Auth() auth: AuthDto, @Body() dto: SystemConfigSmtpDto): Promise { return this.service.sendTestEmail(auth.user.id, dto); } @Post('templates/:name') @HttpCode(HttpStatus.OK) @Authenticated({ admin: true }) - getNotificationTemplate( + getNotificationTemplateAdmin( @Auth() auth: AuthDto, @Param('name') name: EmailTemplate, @Body() dto: TemplateDto, diff --git a/web/src/lib/components/admin-page/settings/notification-settings/notification-settings.svelte b/web/src/lib/components/admin-page/settings/notification-settings/notification-settings.svelte index a2b6305f76..24e672607d 100644 --- a/web/src/lib/components/admin-page/settings/notification-settings/notification-settings.svelte +++ b/web/src/lib/components/admin-page/settings/notification-settings/notification-settings.svelte @@ -12,7 +12,7 @@ import { SettingInputFieldType } from '$lib/constants'; import { user } from '$lib/stores/user.store'; import { handleError } from '$lib/utils/handle-error'; - import { sendTestEmail, type SystemConfigDto } from '@immich/sdk'; + import { sendTestEmailAdmin, type SystemConfigDto } from '@immich/sdk'; import { Button } from '@immich/ui'; import { isEqual } from 'lodash-es'; import { t } from 'svelte-i18n'; @@ -40,7 +40,7 @@ isSending = true; try { - await sendTestEmail({ + await sendTestEmailAdmin({ systemConfigSmtpDto: { enabled: config.notifications.smtp.enabled, transport: { diff --git a/web/src/lib/components/admin-page/settings/template-settings/template-settings.svelte b/web/src/lib/components/admin-page/settings/template-settings/template-settings.svelte index b418aebb2b..06d8196935 100644 --- a/web/src/lib/components/admin-page/settings/template-settings/template-settings.svelte +++ b/web/src/lib/components/admin-page/settings/template-settings/template-settings.svelte @@ -6,7 +6,7 @@ import SettingAccordion from '$lib/components/shared-components/settings/setting-accordion.svelte'; import SettingTextarea from '$lib/components/shared-components/settings/setting-textarea.svelte'; import { handleError } from '$lib/utils/handle-error'; - import { type SystemConfigDto, type SystemConfigTemplateEmailsDto, getNotificationTemplate } from '@immich/sdk'; + import { type SystemConfigDto, type SystemConfigTemplateEmailsDto, getNotificationTemplateAdmin } from '@immich/sdk'; import { Button } from '@immich/ui'; import { mdiEyeOutline } from '@mdi/js'; import { t } from 'svelte-i18n'; @@ -25,7 +25,7 @@ const getTemplate = async (name: string, template: string) => { try { loadingPreview = true; - const result = await getNotificationTemplate({ name, templateDto: { template } }); + const result = await getNotificationTemplateAdmin({ name, templateDto: { template } }); htmlPreview = result.html; } catch (error) { handleError(error, 'Could not load template.'); From 56a4aa9ffe61eb5bf83fe511067e789123d9464d Mon Sep 17 00:00:00 2001 From: Jason Rasmussen Date: Mon, 21 Apr 2025 12:53:37 -0400 Subject: [PATCH 18/29] refactor: email repository (#17746) --- .../notification-admin.controller.ts | 2 +- server/src/emails/album-invite.email.tsx | 2 +- server/src/emails/album-update.email.tsx | 2 +- server/src/emails/test.email.tsx | 2 +- server/src/emails/welcome.email.tsx | 2 +- ...itory.spec.ts => email.repository.spec.ts} | 8 +- ...tion.repository.ts => email.repository.ts} | 4 +- server/src/repositories/index.ts | 4 +- server/src/services/base.service.ts | 4 +- .../src/services/notification.service.spec.ts | 88 +++++++++---------- server/src/services/notification.service.ts | 24 ++--- server/test/medium.factory.ts | 2 +- server/test/utils.ts | 8 +- 13 files changed, 74 insertions(+), 78 deletions(-) rename server/src/repositories/{notification.repository.spec.ts => email.repository.spec.ts} (87%) rename server/src/repositories/{notification.repository.ts => email.repository.ts} (97%) diff --git a/server/src/controllers/notification-admin.controller.ts b/server/src/controllers/notification-admin.controller.ts index f3ce4cdac9..937244fc56 100644 --- a/server/src/controllers/notification-admin.controller.ts +++ b/server/src/controllers/notification-admin.controller.ts @@ -4,7 +4,7 @@ import { AuthDto } from 'src/dtos/auth.dto'; import { TemplateDto, TemplateResponseDto, TestEmailResponseDto } from 'src/dtos/notification.dto'; import { SystemConfigSmtpDto } from 'src/dtos/system-config.dto'; import { Auth, Authenticated } from 'src/middleware/auth.guard'; -import { EmailTemplate } from 'src/repositories/notification.repository'; +import { EmailTemplate } from 'src/repositories/email.repository'; import { NotificationService } from 'src/services/notification.service'; @ApiTags('Notifications (Admin)') diff --git a/server/src/emails/album-invite.email.tsx b/server/src/emails/album-invite.email.tsx index 4bd7abc305..fdc189af97 100644 --- a/server/src/emails/album-invite.email.tsx +++ b/server/src/emails/album-invite.email.tsx @@ -2,7 +2,7 @@ import { Img, Link, Section, Text } from '@react-email/components'; import * as React from 'react'; import { ImmichButton } from 'src/emails/components/button.component'; import ImmichLayout from 'src/emails/components/immich.layout'; -import { AlbumInviteEmailProps } from 'src/repositories/notification.repository'; +import { AlbumInviteEmailProps } from 'src/repositories/email.repository'; import { replaceTemplateTags } from 'src/utils/replace-template-tags'; export const AlbumInviteEmail = ({ diff --git a/server/src/emails/album-update.email.tsx b/server/src/emails/album-update.email.tsx index 2311e896e1..3bed3a5b36 100644 --- a/server/src/emails/album-update.email.tsx +++ b/server/src/emails/album-update.email.tsx @@ -2,7 +2,7 @@ import { Img, Link, Section, Text } from '@react-email/components'; import * as React from 'react'; import { ImmichButton } from 'src/emails/components/button.component'; import ImmichLayout from 'src/emails/components/immich.layout'; -import { AlbumUpdateEmailProps } from 'src/repositories/notification.repository'; +import { AlbumUpdateEmailProps } from 'src/repositories/email.repository'; import { replaceTemplateTags } from 'src/utils/replace-template-tags'; export const AlbumUpdateEmail = ({ diff --git a/server/src/emails/test.email.tsx b/server/src/emails/test.email.tsx index ac9bdbe0ea..0d87307080 100644 --- a/server/src/emails/test.email.tsx +++ b/server/src/emails/test.email.tsx @@ -1,7 +1,7 @@ import { Link, Row, Text } from '@react-email/components'; import * as React from 'react'; import ImmichLayout from 'src/emails/components/immich.layout'; -import { TestEmailProps } from 'src/repositories/notification.repository'; +import { TestEmailProps } from 'src/repositories/email.repository'; export const TestEmail = ({ baseUrl, displayName }: TestEmailProps) => ( diff --git a/server/src/emails/welcome.email.tsx b/server/src/emails/welcome.email.tsx index 11a6602711..57e86ab252 100644 --- a/server/src/emails/welcome.email.tsx +++ b/server/src/emails/welcome.email.tsx @@ -2,7 +2,7 @@ import { Link, Section, Text } from '@react-email/components'; import * as React from 'react'; import { ImmichButton } from 'src/emails/components/button.component'; import ImmichLayout from 'src/emails/components/immich.layout'; -import { WelcomeEmailProps } from 'src/repositories/notification.repository'; +import { WelcomeEmailProps } from 'src/repositories/email.repository'; import { replaceTemplateTags } from 'src/utils/replace-template-tags'; export const WelcomeEmail = ({ baseUrl, displayName, username, password, customTemplate }: WelcomeEmailProps) => { diff --git a/server/src/repositories/notification.repository.spec.ts b/server/src/repositories/email.repository.spec.ts similarity index 87% rename from server/src/repositories/notification.repository.spec.ts rename to server/src/repositories/email.repository.spec.ts index 1d0770af6b..5640b26bf6 100644 --- a/server/src/repositories/notification.repository.spec.ts +++ b/server/src/repositories/email.repository.spec.ts @@ -1,13 +1,13 @@ +import { EmailRenderRequest, EmailRepository, EmailTemplate } from 'src/repositories/email.repository'; import { LoggingRepository } from 'src/repositories/logging.repository'; -import { EmailRenderRequest, EmailTemplate, NotificationRepository } from 'src/repositories/notification.repository'; import { automock } from 'test/utils'; -describe(NotificationRepository.name, () => { - let sut: NotificationRepository; +describe(EmailRepository.name, () => { + let sut: EmailRepository; beforeEach(() => { // eslint-disable-next-line no-sparse-arrays - sut = new NotificationRepository(automock(LoggingRepository, { args: [, { getEnv: () => ({}) }], strict: false })); + sut = new EmailRepository(automock(LoggingRepository, { args: [, { getEnv: () => ({}) }], strict: false })); }); describe('renderEmail', () => { diff --git a/server/src/repositories/notification.repository.ts b/server/src/repositories/email.repository.ts similarity index 97% rename from server/src/repositories/notification.repository.ts rename to server/src/repositories/email.repository.ts index 91f03b928b..78c89b4a9d 100644 --- a/server/src/repositories/notification.repository.ts +++ b/server/src/repositories/email.repository.ts @@ -98,9 +98,9 @@ export type SendEmailResponse = { }; @Injectable() -export class NotificationRepository { +export class EmailRepository { constructor(private logger: LoggingRepository) { - this.logger.setContext(NotificationRepository.name); + this.logger.setContext(EmailRepository.name); } verifySmtp(options: SmtpOptions): Promise { diff --git a/server/src/repositories/index.ts b/server/src/repositories/index.ts index ef36a2b3f8..bd2e5c6774 100644 --- a/server/src/repositories/index.ts +++ b/server/src/repositories/index.ts @@ -11,6 +11,7 @@ import { CronRepository } from 'src/repositories/cron.repository'; import { CryptoRepository } from 'src/repositories/crypto.repository'; import { DatabaseRepository } from 'src/repositories/database.repository'; import { DownloadRepository } from 'src/repositories/download.repository'; +import { EmailRepository } from 'src/repositories/email.repository'; import { EventRepository } from 'src/repositories/event.repository'; import { JobRepository } from 'src/repositories/job.repository'; import { LibraryRepository } from 'src/repositories/library.repository'; @@ -21,7 +22,6 @@ import { MediaRepository } from 'src/repositories/media.repository'; import { MemoryRepository } from 'src/repositories/memory.repository'; import { MetadataRepository } from 'src/repositories/metadata.repository'; import { MoveRepository } from 'src/repositories/move.repository'; -import { NotificationRepository } from 'src/repositories/notification.repository'; import { OAuthRepository } from 'src/repositories/oauth.repository'; import { PartnerRepository } from 'src/repositories/partner.repository'; import { PersonRepository } from 'src/repositories/person.repository'; @@ -65,7 +65,7 @@ export const repositories = [ MemoryRepository, MetadataRepository, MoveRepository, - NotificationRepository, + EmailRepository, OAuthRepository, PartnerRepository, PersonRepository, diff --git a/server/src/services/base.service.ts b/server/src/services/base.service.ts index 2fbdd6e4c0..23ddb1b63e 100644 --- a/server/src/services/base.service.ts +++ b/server/src/services/base.service.ts @@ -18,6 +18,7 @@ import { CronRepository } from 'src/repositories/cron.repository'; import { CryptoRepository } from 'src/repositories/crypto.repository'; import { DatabaseRepository } from 'src/repositories/database.repository'; import { DownloadRepository } from 'src/repositories/download.repository'; +import { EmailRepository } from 'src/repositories/email.repository'; import { EventRepository } from 'src/repositories/event.repository'; import { JobRepository } from 'src/repositories/job.repository'; import { LibraryRepository } from 'src/repositories/library.repository'; @@ -28,7 +29,6 @@ import { MediaRepository } from 'src/repositories/media.repository'; import { MemoryRepository } from 'src/repositories/memory.repository'; import { MetadataRepository } from 'src/repositories/metadata.repository'; import { MoveRepository } from 'src/repositories/move.repository'; -import { NotificationRepository } from 'src/repositories/notification.repository'; import { OAuthRepository } from 'src/repositories/oauth.repository'; import { PartnerRepository } from 'src/repositories/partner.repository'; import { PersonRepository } from 'src/repositories/person.repository'; @@ -70,6 +70,7 @@ export class BaseService { protected cryptoRepository: CryptoRepository, protected databaseRepository: DatabaseRepository, protected downloadRepository: DownloadRepository, + protected emailRepository: EmailRepository, protected eventRepository: EventRepository, protected jobRepository: JobRepository, protected libraryRepository: LibraryRepository, @@ -79,7 +80,6 @@ export class BaseService { protected memoryRepository: MemoryRepository, protected metadataRepository: MetadataRepository, protected moveRepository: MoveRepository, - protected notificationRepository: NotificationRepository, protected oauthRepository: OAuthRepository, protected partnerRepository: PartnerRepository, protected personRepository: PersonRepository, diff --git a/server/src/services/notification.service.spec.ts b/server/src/services/notification.service.spec.ts index 85e425b11f..5830260753 100644 --- a/server/src/services/notification.service.spec.ts +++ b/server/src/services/notification.service.spec.ts @@ -3,7 +3,7 @@ import { defaults, SystemConfig } from 'src/config'; import { AlbumUser } from 'src/database'; import { SystemConfigDto } from 'src/dtos/system-config.dto'; import { AssetFileType, JobName, JobStatus, UserMetadataKey } from 'src/enum'; -import { EmailTemplate } from 'src/repositories/notification.repository'; +import { EmailTemplate } from 'src/repositories/email.repository'; import { NotificationService } from 'src/services/notification.service'; import { INotifyAlbumUpdateJob } from 'src/types'; import { albumStub } from 'test/fixtures/album.stub'; @@ -74,18 +74,18 @@ describe(NotificationService.name, () => { const oldConfig = configs.smtpDisabled; const newConfig = configs.smtpEnabled; - mocks.notification.verifySmtp.mockResolvedValue(true); + mocks.email.verifySmtp.mockResolvedValue(true); await expect(sut.onConfigValidate({ oldConfig, newConfig })).resolves.not.toThrow(); - expect(mocks.notification.verifySmtp).toHaveBeenCalledWith(newConfig.notifications.smtp.transport); + expect(mocks.email.verifySmtp).toHaveBeenCalledWith(newConfig.notifications.smtp.transport); }); it('validates smtp config when transport changes', async () => { const oldConfig = configs.smtpEnabled; const newConfig = configs.smtpTransport; - mocks.notification.verifySmtp.mockResolvedValue(true); + mocks.email.verifySmtp.mockResolvedValue(true); await expect(sut.onConfigValidate({ oldConfig, newConfig })).resolves.not.toThrow(); - expect(mocks.notification.verifySmtp).toHaveBeenCalledWith(newConfig.notifications.smtp.transport); + expect(mocks.email.verifySmtp).toHaveBeenCalledWith(newConfig.notifications.smtp.transport); }); it('skips smtp validation when there are no changes', async () => { @@ -93,7 +93,7 @@ describe(NotificationService.name, () => { const newConfig = { ...configs.smtpEnabled }; await expect(sut.onConfigValidate({ oldConfig, newConfig })).resolves.not.toThrow(); - expect(mocks.notification.verifySmtp).not.toHaveBeenCalled(); + expect(mocks.email.verifySmtp).not.toHaveBeenCalled(); }); it('skips smtp validation with DTO when there are no changes', async () => { @@ -101,7 +101,7 @@ describe(NotificationService.name, () => { const newConfig = plainToInstance(SystemConfigDto, configs.smtpEnabled); await expect(sut.onConfigValidate({ oldConfig, newConfig })).resolves.not.toThrow(); - expect(mocks.notification.verifySmtp).not.toHaveBeenCalled(); + expect(mocks.email.verifySmtp).not.toHaveBeenCalled(); }); it('skips smtp validation when smtp is disabled', async () => { @@ -109,14 +109,14 @@ describe(NotificationService.name, () => { const newConfig = { ...configs.smtpDisabled }; await expect(sut.onConfigValidate({ oldConfig, newConfig })).resolves.not.toThrow(); - expect(mocks.notification.verifySmtp).not.toHaveBeenCalled(); + expect(mocks.email.verifySmtp).not.toHaveBeenCalled(); }); it('should fail if smtp configuration is invalid', async () => { const oldConfig = configs.smtpDisabled; const newConfig = configs.smtpEnabled; - mocks.notification.verifySmtp.mockRejectedValue(new Error('Failed validating smtp')); + mocks.email.verifySmtp.mockRejectedValue(new Error('Failed validating smtp')); await expect(sut.onConfigValidate({ oldConfig, newConfig })).rejects.toBeInstanceOf(Error); }); }); @@ -248,7 +248,7 @@ describe(NotificationService.name, () => { it('should throw error if smtp validation fails', async () => { mocks.user.get.mockResolvedValue(userStub.admin); - mocks.notification.verifySmtp.mockRejectedValue(''); + mocks.email.verifySmtp.mockRejectedValue(''); await expect(sut.sendTestEmail('', configs.smtpTransport.notifications.smtp)).rejects.toThrow( 'Failed to verify SMTP configuration', @@ -257,16 +257,16 @@ describe(NotificationService.name, () => { it('should send email to default domain', async () => { mocks.user.get.mockResolvedValue(userStub.admin); - mocks.notification.verifySmtp.mockResolvedValue(true); - mocks.notification.renderEmail.mockResolvedValue({ html: '', text: '' }); - mocks.notification.sendEmail.mockResolvedValue({ messageId: 'message-1', response: '' }); + mocks.email.verifySmtp.mockResolvedValue(true); + mocks.email.renderEmail.mockResolvedValue({ html: '', text: '' }); + mocks.email.sendEmail.mockResolvedValue({ messageId: 'message-1', response: '' }); await expect(sut.sendTestEmail('', configs.smtpTransport.notifications.smtp)).resolves.not.toThrow(); - expect(mocks.notification.renderEmail).toHaveBeenCalledWith({ + expect(mocks.email.renderEmail).toHaveBeenCalledWith({ template: EmailTemplate.TEST_EMAIL, data: { baseUrl: 'https://my.immich.app', displayName: userStub.admin.name }, }); - expect(mocks.notification.sendEmail).toHaveBeenCalledWith( + expect(mocks.email.sendEmail).toHaveBeenCalledWith( expect.objectContaining({ subject: 'Test email from Immich', smtp: configs.smtpTransport.notifications.smtp.transport, @@ -276,17 +276,17 @@ describe(NotificationService.name, () => { it('should send email to external domain', async () => { mocks.user.get.mockResolvedValue(userStub.admin); - mocks.notification.verifySmtp.mockResolvedValue(true); - mocks.notification.renderEmail.mockResolvedValue({ html: '', text: '' }); + mocks.email.verifySmtp.mockResolvedValue(true); + mocks.email.renderEmail.mockResolvedValue({ html: '', text: '' }); mocks.systemMetadata.get.mockResolvedValue({ server: { externalDomain: 'https://demo.immich.app' } }); - mocks.notification.sendEmail.mockResolvedValue({ messageId: 'message-1', response: '' }); + mocks.email.sendEmail.mockResolvedValue({ messageId: 'message-1', response: '' }); await expect(sut.sendTestEmail('', configs.smtpTransport.notifications.smtp)).resolves.not.toThrow(); - expect(mocks.notification.renderEmail).toHaveBeenCalledWith({ + expect(mocks.email.renderEmail).toHaveBeenCalledWith({ template: EmailTemplate.TEST_EMAIL, data: { baseUrl: 'https://demo.immich.app', displayName: userStub.admin.name }, }); - expect(mocks.notification.sendEmail).toHaveBeenCalledWith( + expect(mocks.email.sendEmail).toHaveBeenCalledWith( expect.objectContaining({ subject: 'Test email from Immich', smtp: configs.smtpTransport.notifications.smtp.transport, @@ -296,18 +296,18 @@ describe(NotificationService.name, () => { it('should send email with replyTo', async () => { mocks.user.get.mockResolvedValue(userStub.admin); - mocks.notification.verifySmtp.mockResolvedValue(true); - mocks.notification.renderEmail.mockResolvedValue({ html: '', text: '' }); - mocks.notification.sendEmail.mockResolvedValue({ messageId: 'message-1', response: '' }); + mocks.email.verifySmtp.mockResolvedValue(true); + mocks.email.renderEmail.mockResolvedValue({ html: '', text: '' }); + mocks.email.sendEmail.mockResolvedValue({ messageId: 'message-1', response: '' }); await expect( sut.sendTestEmail('', { ...configs.smtpTransport.notifications.smtp, replyTo: 'demo@immich.app' }), ).resolves.not.toThrow(); - expect(mocks.notification.renderEmail).toHaveBeenCalledWith({ + expect(mocks.email.renderEmail).toHaveBeenCalledWith({ template: EmailTemplate.TEST_EMAIL, data: { baseUrl: 'https://my.immich.app', displayName: userStub.admin.name }, }); - expect(mocks.notification.sendEmail).toHaveBeenCalledWith( + expect(mocks.email.sendEmail).toHaveBeenCalledWith( expect.objectContaining({ subject: 'Test email from Immich', smtp: configs.smtpTransport.notifications.smtp.transport, @@ -325,7 +325,7 @@ describe(NotificationService.name, () => { it('should be successful', async () => { mocks.user.get.mockResolvedValue(userStub.admin); mocks.systemMetadata.get.mockResolvedValue({ server: {} }); - mocks.notification.renderEmail.mockResolvedValue({ html: '', text: '' }); + mocks.email.renderEmail.mockResolvedValue({ html: '', text: '' }); await expect(sut.handleUserSignup({ id: '' })).resolves.toBe(JobStatus.SUCCESS); expect(mocks.job.queue).toHaveBeenCalledWith({ @@ -390,7 +390,7 @@ describe(NotificationService.name, () => { ], }); mocks.systemMetadata.get.mockResolvedValue({ server: {} }); - mocks.notification.renderEmail.mockResolvedValue({ html: '', text: '' }); + mocks.email.renderEmail.mockResolvedValue({ html: '', text: '' }); await expect(sut.handleAlbumInvite({ id: '', recipientId: '' })).resolves.toBe(JobStatus.SUCCESS); expect(mocks.job.queue).toHaveBeenCalledWith({ @@ -411,7 +411,7 @@ describe(NotificationService.name, () => { ], }); mocks.systemMetadata.get.mockResolvedValue({ server: {} }); - mocks.notification.renderEmail.mockResolvedValue({ html: '', text: '' }); + mocks.email.renderEmail.mockResolvedValue({ html: '', text: '' }); mocks.assetJob.getAlbumThumbnailFiles.mockResolvedValue([]); await expect(sut.handleAlbumInvite({ id: '', recipientId: '' })).resolves.toBe(JobStatus.SUCCESS); @@ -440,7 +440,7 @@ describe(NotificationService.name, () => { ], }); mocks.systemMetadata.get.mockResolvedValue({ server: {} }); - mocks.notification.renderEmail.mockResolvedValue({ html: '', text: '' }); + mocks.email.renderEmail.mockResolvedValue({ html: '', text: '' }); mocks.assetJob.getAlbumThumbnailFiles.mockResolvedValue([ { id: '1', type: AssetFileType.THUMBNAIL, path: 'path-to-thumb.jpg' }, ]); @@ -471,7 +471,7 @@ describe(NotificationService.name, () => { ], }); mocks.systemMetadata.get.mockResolvedValue({ server: {} }); - mocks.notification.renderEmail.mockResolvedValue({ html: '', text: '' }); + mocks.email.renderEmail.mockResolvedValue({ html: '', text: '' }); mocks.assetJob.getAlbumThumbnailFiles.mockResolvedValue([assetStub.image.files[2]]); await expect(sut.handleAlbumInvite({ id: '', recipientId: '' })).resolves.toBe(JobStatus.SUCCESS); @@ -508,12 +508,12 @@ describe(NotificationService.name, () => { albumUsers: [{ user: { id: userStub.user1.id } } as AlbumUser], }); mocks.user.get.mockResolvedValueOnce(userStub.user1); - mocks.notification.renderEmail.mockResolvedValue({ html: '', text: '' }); + mocks.email.renderEmail.mockResolvedValue({ html: '', text: '' }); mocks.assetJob.getAlbumThumbnailFiles.mockResolvedValue([]); await sut.handleAlbumUpdate({ id: '', recipientIds: [userStub.user1.id] }); expect(mocks.user.get).toHaveBeenCalledWith(userStub.user1.id, { withDeleted: false }); - expect(mocks.notification.renderEmail).not.toHaveBeenCalled(); + expect(mocks.email.renderEmail).not.toHaveBeenCalled(); }); it('should skip recipient with disabled email notifications', async () => { @@ -530,12 +530,12 @@ describe(NotificationService.name, () => { }, ], }); - mocks.notification.renderEmail.mockResolvedValue({ html: '', text: '' }); + mocks.email.renderEmail.mockResolvedValue({ html: '', text: '' }); mocks.assetJob.getAlbumThumbnailFiles.mockResolvedValue([]); await sut.handleAlbumUpdate({ id: '', recipientIds: [userStub.user1.id] }); expect(mocks.user.get).toHaveBeenCalledWith(userStub.user1.id, { withDeleted: false }); - expect(mocks.notification.renderEmail).not.toHaveBeenCalled(); + expect(mocks.email.renderEmail).not.toHaveBeenCalled(); }); it('should skip recipient with disabled email notifications for the album update event', async () => { @@ -552,12 +552,12 @@ describe(NotificationService.name, () => { }, ], }); - mocks.notification.renderEmail.mockResolvedValue({ html: '', text: '' }); + mocks.email.renderEmail.mockResolvedValue({ html: '', text: '' }); mocks.assetJob.getAlbumThumbnailFiles.mockResolvedValue([]); await sut.handleAlbumUpdate({ id: '', recipientIds: [userStub.user1.id] }); expect(mocks.user.get).toHaveBeenCalledWith(userStub.user1.id, { withDeleted: false }); - expect(mocks.notification.renderEmail).not.toHaveBeenCalled(); + expect(mocks.email.renderEmail).not.toHaveBeenCalled(); }); it('should send email', async () => { @@ -566,12 +566,12 @@ describe(NotificationService.name, () => { albumUsers: [{ user: { id: userStub.user1.id } } as AlbumUser], }); mocks.user.get.mockResolvedValue(userStub.user1); - mocks.notification.renderEmail.mockResolvedValue({ html: '', text: '' }); + mocks.email.renderEmail.mockResolvedValue({ html: '', text: '' }); mocks.assetJob.getAlbumThumbnailFiles.mockResolvedValue([]); await sut.handleAlbumUpdate({ id: '', recipientIds: [userStub.user1.id] }); expect(mocks.user.get).toHaveBeenCalledWith(userStub.user1.id, { withDeleted: false }); - expect(mocks.notification.renderEmail).toHaveBeenCalled(); + expect(mocks.email.renderEmail).toHaveBeenCalled(); expect(mocks.job.queue).toHaveBeenCalled(); }); @@ -599,24 +599,20 @@ describe(NotificationService.name, () => { mocks.systemMetadata.get.mockResolvedValue({ notifications: { smtp: { enabled: true, from: 'test@immich.app' } }, }); - mocks.notification.sendEmail.mockResolvedValue({ messageId: '', response: '' }); + mocks.email.sendEmail.mockResolvedValue({ messageId: '', response: '' }); await expect(sut.handleSendEmail({ html: '', subject: '', text: '', to: '' })).resolves.toBe(JobStatus.SUCCESS); - expect(mocks.notification.sendEmail).toHaveBeenCalledWith( - expect.objectContaining({ replyTo: 'test@immich.app' }), - ); + expect(mocks.email.sendEmail).toHaveBeenCalledWith(expect.objectContaining({ replyTo: 'test@immich.app' })); }); it('should send mail with replyTo successfully', async () => { mocks.systemMetadata.get.mockResolvedValue({ notifications: { smtp: { enabled: true, from: 'test@immich.app', replyTo: 'demo@immich.app' } }, }); - mocks.notification.sendEmail.mockResolvedValue({ messageId: '', response: '' }); + mocks.email.sendEmail.mockResolvedValue({ messageId: '', response: '' }); await expect(sut.handleSendEmail({ html: '', subject: '', text: '', to: '' })).resolves.toBe(JobStatus.SUCCESS); - expect(mocks.notification.sendEmail).toHaveBeenCalledWith( - expect.objectContaining({ replyTo: 'demo@immich.app' }), - ); + expect(mocks.email.sendEmail).toHaveBeenCalledWith(expect.objectContaining({ replyTo: 'demo@immich.app' })); }); }); }); diff --git a/server/src/services/notification.service.ts b/server/src/services/notification.service.ts index 2c4cc76756..2e456718ca 100644 --- a/server/src/services/notification.service.ts +++ b/server/src/services/notification.service.ts @@ -2,8 +2,8 @@ import { BadRequestException, Injectable } from '@nestjs/common'; import { OnEvent, OnJob } from 'src/decorators'; import { SystemConfigSmtpDto } from 'src/dtos/system-config.dto'; import { AssetFileType, JobName, JobStatus, QueueName } from 'src/enum'; +import { EmailTemplate } from 'src/repositories/email.repository'; import { ArgOf } from 'src/repositories/event.repository'; -import { EmailTemplate } from 'src/repositories/notification.repository'; import { BaseService } from 'src/services/base.service'; import { EmailImageAttachment, IEntityJob, INotifyAlbumUpdateJob, JobItem, JobOf } from 'src/types'; import { getFilenameExtension } from 'src/utils/file'; @@ -28,7 +28,7 @@ export class NotificationService extends BaseService { newConfig.notifications.smtp.enabled && !isEqualObject(oldConfig.notifications.smtp, newConfig.notifications.smtp) ) { - await this.notificationRepository.verifySmtp(newConfig.notifications.smtp.transport); + await this.emailRepository.verifySmtp(newConfig.notifications.smtp.transport); } } catch (error: Error | any) { this.logger.error(`Failed to validate SMTP configuration: ${error}`, error?.stack); @@ -138,13 +138,13 @@ export class NotificationService extends BaseService { } try { - await this.notificationRepository.verifySmtp(dto.transport); + await this.emailRepository.verifySmtp(dto.transport); } catch (error) { throw new BadRequestException('Failed to verify SMTP configuration', { cause: error }); } const { server } = await this.getConfig({ withCache: false }); - const { html, text } = await this.notificationRepository.renderEmail({ + const { html, text } = await this.emailRepository.renderEmail({ template: EmailTemplate.TEST_EMAIL, data: { baseUrl: getExternalDomain(server), @@ -152,7 +152,7 @@ export class NotificationService extends BaseService { }, customTemplate: tempTemplate!, }); - const { messageId } = await this.notificationRepository.sendEmail({ + const { messageId } = await this.emailRepository.sendEmail({ to: user.email, subject: 'Test email from Immich', html, @@ -172,7 +172,7 @@ export class NotificationService extends BaseService { switch (name) { case EmailTemplate.WELCOME: { - const { html: _welcomeHtml } = await this.notificationRepository.renderEmail({ + const { html: _welcomeHtml } = await this.emailRepository.renderEmail({ template: EmailTemplate.WELCOME, data: { baseUrl: getExternalDomain(server), @@ -187,7 +187,7 @@ export class NotificationService extends BaseService { break; } case EmailTemplate.ALBUM_UPDATE: { - const { html: _updateAlbumHtml } = await this.notificationRepository.renderEmail({ + const { html: _updateAlbumHtml } = await this.emailRepository.renderEmail({ template: EmailTemplate.ALBUM_UPDATE, data: { baseUrl: getExternalDomain(server), @@ -203,7 +203,7 @@ export class NotificationService extends BaseService { } case EmailTemplate.ALBUM_INVITE: { - const { html } = await this.notificationRepository.renderEmail({ + const { html } = await this.emailRepository.renderEmail({ template: EmailTemplate.ALBUM_INVITE, data: { baseUrl: getExternalDomain(server), @@ -235,7 +235,7 @@ export class NotificationService extends BaseService { } const { server, templates } = await this.getConfig({ withCache: true }); - const { html, text } = await this.notificationRepository.renderEmail({ + const { html, text } = await this.emailRepository.renderEmail({ template: EmailTemplate.WELCOME, data: { baseUrl: getExternalDomain(server), @@ -280,7 +280,7 @@ export class NotificationService extends BaseService { const attachment = await this.getAlbumThumbnailAttachment(album); const { server, templates } = await this.getConfig({ withCache: false }); - const { html, text } = await this.notificationRepository.renderEmail({ + const { html, text } = await this.emailRepository.renderEmail({ template: EmailTemplate.ALBUM_INVITE, data: { baseUrl: getExternalDomain(server), @@ -339,7 +339,7 @@ export class NotificationService extends BaseService { continue; } - const { html, text } = await this.notificationRepository.renderEmail({ + const { html, text } = await this.emailRepository.renderEmail({ template: EmailTemplate.ALBUM_UPDATE, data: { baseUrl: getExternalDomain(server), @@ -374,7 +374,7 @@ export class NotificationService extends BaseService { } const { to, subject, html, text: plain } = data; - const response = await this.notificationRepository.sendEmail({ + const response = await this.emailRepository.sendEmail({ to, subject, html, diff --git a/server/test/medium.factory.ts b/server/test/medium.factory.ts index d3ab876e07..671a8a50ca 100644 --- a/server/test/medium.factory.ts +++ b/server/test/medium.factory.ts @@ -284,6 +284,7 @@ export const asDeps = (repositories: ServiceOverrides) => { repositories.crypto || getRepositoryMock('crypto'), repositories.database || getRepositoryMock('database'), repositories.downloadRepository, + repositories.email, repositories.event, repositories.job || getRepositoryMock('job'), repositories.library, @@ -293,7 +294,6 @@ export const asDeps = (repositories: ServiceOverrides) => { repositories.memory || getRepositoryMock('memory'), repositories.metadata, repositories.move, - repositories.notification, repositories.oauth, repositories.partner || getRepositoryMock('partner'), repositories.person || getRepositoryMock('person'), diff --git a/server/test/utils.ts b/server/test/utils.ts index 52984d97a2..e1d979fbfe 100644 --- a/server/test/utils.ts +++ b/server/test/utils.ts @@ -18,6 +18,7 @@ import { CronRepository } from 'src/repositories/cron.repository'; import { CryptoRepository } from 'src/repositories/crypto.repository'; import { DatabaseRepository } from 'src/repositories/database.repository'; import { DownloadRepository } from 'src/repositories/download.repository'; +import { EmailRepository } from 'src/repositories/email.repository'; import { EventRepository } from 'src/repositories/event.repository'; import { JobRepository } from 'src/repositories/job.repository'; import { LibraryRepository } from 'src/repositories/library.repository'; @@ -28,7 +29,6 @@ import { MediaRepository } from 'src/repositories/media.repository'; import { MemoryRepository } from 'src/repositories/memory.repository'; import { MetadataRepository } from 'src/repositories/metadata.repository'; import { MoveRepository } from 'src/repositories/move.repository'; -import { NotificationRepository } from 'src/repositories/notification.repository'; import { OAuthRepository } from 'src/repositories/oauth.repository'; import { PartnerRepository } from 'src/repositories/partner.repository'; import { PersonRepository } from 'src/repositories/person.repository'; @@ -124,6 +124,7 @@ export type ServiceOverrides = { crypto: CryptoRepository; database: DatabaseRepository; downloadRepository: DownloadRepository; + email: EmailRepository; event: EventRepository; job: JobRepository; library: LibraryRepository; @@ -134,7 +135,6 @@ export type ServiceOverrides = { memory: MemoryRepository; metadata: MetadataRepository; move: MoveRepository; - notification: NotificationRepository; oauth: OAuthRepository; partner: PartnerRepository; person: PersonRepository; @@ -190,6 +190,7 @@ export const newTestService = ( config: newConfigRepositoryMock(), database: newDatabaseRepositoryMock(), downloadRepository: automock(DownloadRepository, { strict: false }), + email: automock(EmailRepository, { args: [loggerMock] }), // eslint-disable-next-line no-sparse-arrays event: automock(EventRepository, { args: [, , loggerMock], strict: false }), job: newJobRepositoryMock(), @@ -201,7 +202,6 @@ export const newTestService = ( memory: automock(MemoryRepository), metadata: newMetadataRepositoryMock(), move: automock(MoveRepository, { strict: false }), - notification: automock(NotificationRepository, { args: [loggerMock] }), oauth: automock(OAuthRepository, { args: [loggerMock] }), partner: automock(PartnerRepository, { strict: false }), person: newPersonRepositoryMock(), @@ -240,6 +240,7 @@ export const newTestService = ( overrides.crypto || (mocks.crypto as As), overrides.database || (mocks.database as As), overrides.downloadRepository || (mocks.downloadRepository as As), + overrides.email || (mocks.email as As), overrides.event || (mocks.event as As), overrides.job || (mocks.job as As), overrides.library || (mocks.library as As), @@ -249,7 +250,6 @@ export const newTestService = ( overrides.memory || (mocks.memory as As), overrides.metadata || (mocks.metadata as As), overrides.move || (mocks.move as As), - overrides.notification || (mocks.notification as As), overrides.oauth || (mocks.oauth as As), overrides.partner || (mocks.partner as As), overrides.person || (mocks.person as As), From b71039e83c2497f9829f1592573dbc2e85bb2f2c Mon Sep 17 00:00:00 2001 From: Toni <51962051+EinToni@users.noreply.github.com> Date: Mon, 21 Apr 2025 19:51:37 +0200 Subject: [PATCH 19/29] perf(mobile): remove small thumbnail and cache generated thumbnails (#17682) * Remove small thumbnail and cache generated thumbnails * Creating the small thumbnails takes quite some time, which should not be underestimated. * The time needed to generate the small or big thumbnail is not too different from each other. Therefore there is no real benefit of the small thumbnail and it only adds frustration to the end user experience. That is because the image appeared to have loaded (the visual move from blur to something better) but it's still so bad that it is basically a blur. The better solution is therefore to stay at the blur until the actual thumbnail has loaded. * Additionaly to the faster generation of the thumbnail, it now also gets cached similarly to the remote thumbnail which already gets cached. This further speeds up the all over usage of the app and prevents a repeatet thumbnail generation when opening the app. * Decrease quality and use try catch * Decreased the quality from the default 95 to 80 to provide similar quality with much reduces thumbnail size. * Use try catch around the read of the cache file. * Replace ImmutableBuffer.fromUint8List with ImmutableBuffer.fromFilePath * Removed unnecessary comment * Replace debugPrint with log.severe for catch of error --------- Co-authored-by: Alex --- .../immich_local_thumbnail_provider.dart | 49 +++++++++++-------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/mobile/lib/providers/image/immich_local_thumbnail_provider.dart b/mobile/lib/providers/image/immich_local_thumbnail_provider.dart index 69cdb105c0..93245bd78e 100644 --- a/mobile/lib/providers/image/immich_local_thumbnail_provider.dart +++ b/mobile/lib/providers/image/immich_local_thumbnail_provider.dart @@ -2,11 +2,14 @@ import 'dart:async'; import 'dart:ui' as ui; import 'package:cached_network_image/cached_network_image.dart'; +import 'package:flutter_cache_manager/flutter_cache_manager.dart'; +import 'package:immich_mobile/providers/image/cache/thumbnail_image_cache_manager.dart'; import 'package:flutter/foundation.dart'; import 'package:flutter/painting.dart'; import 'package:immich_mobile/entities/asset.entity.dart'; import 'package:photo_manager/photo_manager.dart' show ThumbnailSize; +import 'package:logging/logging.dart'; /// The local image provider for an asset /// Only viable @@ -15,11 +18,14 @@ class ImmichLocalThumbnailProvider final Asset asset; final int height; final int width; + final CacheManager? cacheManager; + final Logger log = Logger("ImmichLocalThumbnailProvider"); ImmichLocalThumbnailProvider({ required this.asset, this.height = 256, this.width = 256, + this.cacheManager, }) : assert(asset.local != null, 'Only usable when asset.local is set'); /// Converts an [ImageProvider]'s settings plus an [ImageConfiguration] to a key @@ -36,11 +42,10 @@ class ImmichLocalThumbnailProvider ImmichLocalThumbnailProvider key, ImageDecoderCallback decode, ) { - final chunkEvents = StreamController(); + final cache = cacheManager ?? ThumbnailImageCacheManager(); return MultiImageStreamCompleter( - codec: _codec(key.asset, decode, chunkEvents), + codec: _codec(key.asset, cache, decode), scale: 1.0, - chunkEvents: chunkEvents.stream, informationCollector: () sync* { yield ErrorDescription(asset.fileName); }, @@ -50,34 +55,36 @@ class ImmichLocalThumbnailProvider // Streams in each stage of the image as we ask for it Stream _codec( Asset key, + CacheManager cache, ImageDecoderCallback decode, - StreamController chunkEvents, ) async* { - // Load a small thumbnail - final thumbBytes = await asset.local?.thumbnailDataWithSize( - const ThumbnailSize.square(32), - quality: 75, - ); - if (thumbBytes != null) { - final buffer = await ui.ImmutableBuffer.fromUint8List(thumbBytes); - final codec = await decode(buffer); - yield codec; - } else { - debugPrint("Loading thumb for ${asset.fileName} failed"); + final cacheKey = '${key.id}_${width}x$height'; + final fileFromCache = await cache.getFileFromCache(cacheKey); + if (fileFromCache != null) { + try { + final buffer = + await ui.ImmutableBuffer.fromFilePath(fileFromCache.file.path); + final codec = await decode(buffer); + yield codec; + return; + } catch (error) { + log.severe('Found thumbnail in cache, but loading it failed', error); + } } - final normalThumbBytes = - await asset.local?.thumbnailDataWithSize(ThumbnailSize(width, height)); - if (normalThumbBytes == null) { + final thumbnailBytes = await asset.local?.thumbnailDataWithSize( + ThumbnailSize(width, height), + quality: 80, + ); + if (thumbnailBytes == null) { throw StateError( "Loading thumb for local photo ${asset.fileName} failed", ); } - final buffer = await ui.ImmutableBuffer.fromUint8List(normalThumbBytes); + final buffer = await ui.ImmutableBuffer.fromUint8List(thumbnailBytes); final codec = await decode(buffer); yield codec; - - chunkEvents.close(); + await cache.putFile(cacheKey, thumbnailBytes); } @override From 010b144754680f39979a65668311b5a5ffee2416 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Wawrzyk?= Date: Mon, 21 Apr 2025 20:00:46 +0200 Subject: [PATCH 20/29] fix(mobile): use immutable cache keys for local images (#17736) fix(mobile): usse immutable cache keys for local images --- .../lib/providers/image/immich_local_image_provider.dart | 4 ++-- .../providers/image/immich_local_thumbnail_provider.dart | 9 ++++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/mobile/lib/providers/image/immich_local_image_provider.dart b/mobile/lib/providers/image/immich_local_image_provider.dart index 36fd3334b9..34c3c94ded 100644 --- a/mobile/lib/providers/image/immich_local_image_provider.dart +++ b/mobile/lib/providers/image/immich_local_image_provider.dart @@ -106,12 +106,12 @@ class ImmichLocalImageProvider extends ImageProvider { bool operator ==(Object other) { if (identical(this, other)) return true; if (other is ImmichLocalImageProvider) { - return asset == other.asset; + return asset.id == other.asset.id; } return false; } @override - int get hashCode => asset.hashCode; + int get hashCode => asset.id.hashCode; } diff --git a/mobile/lib/providers/image/immich_local_thumbnail_provider.dart b/mobile/lib/providers/image/immich_local_thumbnail_provider.dart index 93245bd78e..9ccb127626 100644 --- a/mobile/lib/providers/image/immich_local_thumbnail_provider.dart +++ b/mobile/lib/providers/image/immich_local_thumbnail_provider.dart @@ -89,11 +89,14 @@ class ImmichLocalThumbnailProvider @override bool operator ==(Object other) { - if (other is! ImmichLocalThumbnailProvider) return false; if (identical(this, other)) return true; - return asset == other.asset; + if (other is ImmichLocalThumbnailProvider) { + return asset.id == other.asset.id; + } + + return false; } @override - int get hashCode => asset.hashCode; + int get hashCode => asset.id.hashCode; } From c70140e7078e585db8fe65e5dbbb92d19f11c2c4 Mon Sep 17 00:00:00 2001 From: Yaros Date: Mon, 21 Apr 2025 20:01:38 +0200 Subject: [PATCH 21/29] fix(web): map marker positioning in details pane (#17754) fix: map marker positioning in details pane --- web/src/lib/components/shared-components/map/map.svelte | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/web/src/lib/components/shared-components/map/map.svelte b/web/src/lib/components/shared-components/map/map.svelte index dc594bae3f..e21a73ab43 100644 --- a/web/src/lib/components/shared-components/map/map.svelte +++ b/web/src/lib/components/shared-components/map/map.svelte @@ -249,7 +249,11 @@ > {#snippet children({ feature }: { feature: Feature })} {#if useLocationPin} - + {:else} Date: Mon, 21 Apr 2025 16:54:33 -0700 Subject: [PATCH 22/29] feat: add album start and end dates for storage template (#17188) --- .../services/storage-template.service.spec.ts | 58 +++++++++++++++++++ .../src/services/storage-template.service.ts | 35 ++++++++++- .../storage-template-settings.svelte | 4 ++ .../supported-variables-panel.svelte | 8 +++ 4 files changed, 102 insertions(+), 3 deletions(-) diff --git a/server/src/services/storage-template.service.spec.ts b/server/src/services/storage-template.service.spec.ts index 971a9e8302..d9ac89952e 100644 --- a/server/src/services/storage-template.service.spec.ts +++ b/server/src/services/storage-template.service.spec.ts @@ -69,6 +69,7 @@ describe(StorageTemplateService.name, () => { '{{y}}/{{MMMM}}-{{dd}}/{{filename}}', '{{y}}/{{MM}}/{{filename}}', '{{y}}/{{#if album}}{{album}}{{else}}Other/{{MM}}{{/if}}/{{filename}}', + '{{#if album}}{{album-startDate-y}}/{{album}}{{else}}{{y}}/Other/{{MM}}{{/if}}/{{filename}}', '{{y}}/{{MMM}}/{{filename}}', '{{y}}/{{MMMM}}/{{filename}}', '{{y}}/{{MM}}/{{dd}}/{{filename}}', @@ -182,6 +183,63 @@ describe(StorageTemplateService.name, () => { }); }); + it('should handle album startDate', async () => { + const asset = assetStub.storageAsset(); + const user = userStub.user1; + const album = albumStub.oneAsset; + const config = structuredClone(defaults); + config.storageTemplate.template = + '{{#if album}}{{album-startDate-y}}/{{album-startDate-MM}} - {{album}}{{else}}{{y}}/{{MM}}/{{/if}}/{{filename}}'; + + sut.onConfigInit({ newConfig: config }); + + mocks.user.get.mockResolvedValue(user); + mocks.asset.getStorageTemplateAsset.mockResolvedValueOnce(asset); + mocks.album.getByAssetId.mockResolvedValueOnce([album]); + mocks.album.getMetadataForIds.mockResolvedValueOnce([ + { + startDate: asset.fileCreatedAt, + endDate: asset.fileCreatedAt, + albumId: album.id, + assetCount: 1, + lastModifiedAssetTimestamp: null, + }, + ]); + + expect(await sut.handleMigrationSingle({ id: asset.id })).toBe(JobStatus.SUCCESS); + + const month = (asset.fileCreatedAt.getMonth() + 1).toString().padStart(2, '0'); + expect(mocks.move.create).toHaveBeenCalledWith({ + entityId: asset.id, + newPath: `upload/library/${user.id}/${asset.fileCreatedAt.getFullYear()}/${month} - ${album.albumName}/${asset.originalFileName}`, + oldPath: asset.originalPath, + pathType: AssetPathType.ORIGINAL, + }); + }); + + it('should handle else condition from album startDate', async () => { + const asset = assetStub.storageAsset(); + const user = userStub.user1; + const config = structuredClone(defaults); + config.storageTemplate.template = + '{{#if album}}{{album-startDate-y}}/{{album-startDate-MM}} - {{album}}{{else}}{{y}}/{{MM}}/{{/if}}/{{filename}}'; + + sut.onConfigInit({ newConfig: config }); + + mocks.user.get.mockResolvedValue(user); + mocks.asset.getStorageTemplateAsset.mockResolvedValueOnce(asset); + + expect(await sut.handleMigrationSingle({ id: asset.id })).toBe(JobStatus.SUCCESS); + + const month = (asset.fileCreatedAt.getMonth() + 1).toString().padStart(2, '0'); + expect(mocks.move.create).toHaveBeenCalledWith({ + entityId: asset.id, + newPath: `upload/library/${user.id}/${asset.fileCreatedAt.getFullYear()}/${month}/${asset.originalFileName}`, + oldPath: asset.originalPath, + pathType: AssetPathType.ORIGINAL, + }); + }); + it('should migrate previously failed move from original path when it still exists', async () => { mocks.user.get.mockResolvedValue(userStub.user1); diff --git a/server/src/services/storage-template.service.ts b/server/src/services/storage-template.service.ts index 71a0160ee2..542633a03f 100644 --- a/server/src/services/storage-template.service.ts +++ b/server/src/services/storage-template.service.ts @@ -28,6 +28,7 @@ const storagePresets = [ '{{y}}/{{MMMM}}-{{dd}}/{{filename}}', '{{y}}/{{MM}}/{{filename}}', '{{y}}/{{#if album}}{{album}}{{else}}Other/{{MM}}{{/if}}/{{filename}}', + '{{#if album}}{{album-startDate-y}}/{{album}}{{else}}{{y}}/Other/{{MM}}{{/if}}/{{filename}}', '{{y}}/{{MMM}}/{{filename}}', '{{y}}/{{MMMM}}/{{filename}}', '{{y}}/{{MM}}/{{dd}}/{{filename}}', @@ -54,6 +55,8 @@ interface RenderMetadata { filename: string; extension: string; albumName: string | null; + albumStartDate: Date | null; + albumEndDate: Date | null; } @Injectable() @@ -62,6 +65,7 @@ export class StorageTemplateService extends BaseService { compiled: HandlebarsTemplateDelegate; raw: string; needsAlbum: boolean; + needsAlbumMetadata: boolean; } | null = null; private get template() { @@ -99,6 +103,8 @@ export class StorageTemplateService extends BaseService { filename: 'IMG_123', extension: 'jpg', albumName: 'album', + albumStartDate: new Date(), + albumEndDate: new Date(), }); } catch (error) { this.logger.warn(`Storage template validation failed: ${JSON.stringify(error)}`); @@ -255,9 +261,20 @@ export class StorageTemplateService extends BaseService { } let albumName = null; + let albumStartDate = null; + let albumEndDate = null; if (this.template.needsAlbum) { const albums = await this.albumRepository.getByAssetId(asset.ownerId, asset.id); - albumName = albums?.[0]?.albumName || null; + const album = albums?.[0]; + if (album) { + albumName = album.albumName || null; + + if (this.template.needsAlbumMetadata) { + const [metadata] = await this.albumRepository.getMetadataForIds([album.id]); + albumStartDate = metadata?.startDate || null; + albumEndDate = metadata?.endDate || null; + } + } } const storagePath = this.render(this.template.compiled, { @@ -265,6 +282,8 @@ export class StorageTemplateService extends BaseService { filename: sanitized, extension, albumName, + albumStartDate, + albumEndDate, }); const fullPath = path.normalize(path.join(rootPath, storagePath)); let destination = `${fullPath}.${extension}`; @@ -323,12 +342,13 @@ export class StorageTemplateService extends BaseService { return { raw: template, compiled: handlebar.compile(template, { knownHelpers: undefined, strict: true }), - needsAlbum: template.includes('{{album}}'), + needsAlbum: template.includes('album'), + needsAlbumMetadata: template.includes('album-startDate') || template.includes('album-endDate'), }; } private render(template: HandlebarsTemplateDelegate, options: RenderMetadata) { - const { filename, extension, asset, albumName } = options; + const { filename, extension, asset, albumName, albumStartDate, albumEndDate } = options; const substitutions: Record = { filename, ext: extension, @@ -346,6 +366,15 @@ export class StorageTemplateService extends BaseService { for (const token of Object.values(storageTokens).flat()) { substitutions[token] = dt.toFormat(token); + if (albumName) { + // Use system time zone for album dates to ensure all assets get the exact same date. + substitutions['album-startDate-' + token] = albumStartDate + ? DateTime.fromJSDate(albumStartDate, { zone: systemTimeZone }).toFormat(token) + : ''; + substitutions['album-endDate-' + token] = albumEndDate + ? DateTime.fromJSDate(albumEndDate, { zone: systemTimeZone }).toFormat(token) + : ''; + } } return template(substitutions).replaceAll(/\/{2,}/gm, '/'); diff --git a/web/src/lib/components/admin-page/settings/storage-template/storage-template-settings.svelte b/web/src/lib/components/admin-page/settings/storage-template/storage-template-settings.svelte index 9b4aa5e934..67299d8f6b 100644 --- a/web/src/lib/components/admin-page/settings/storage-template/storage-template-settings.svelte +++ b/web/src/lib/components/admin-page/settings/storage-template/storage-template-settings.svelte @@ -78,6 +78,8 @@ }; const dt = luxon.DateTime.fromISO(new Date('2022-02-03T04:56:05.250').toISOString()); + const albumStartTime = luxon.DateTime.fromISO(new Date('2021-12-31T05:32:41.750').toISOString()); + const albumEndTime = luxon.DateTime.fromISO(new Date('2023-05-06T09:15:17.100').toISOString()); const dateTokens = [ ...templateOptions.yearOptions, @@ -91,6 +93,8 @@ for (const token of dateTokens) { substitutions[token] = dt.toFormat(token); + substitutions['album-startDate-' + token] = albumStartTime.toFormat(token); + substitutions['album-endDate-' + token] = albumEndTime.toFormat(token); } return template(substitutions); diff --git a/web/src/lib/components/admin-page/settings/storage-template/supported-variables-panel.svelte b/web/src/lib/components/admin-page/settings/storage-template/supported-variables-panel.svelte index fc8f913281..c1255c252d 100644 --- a/web/src/lib/components/admin-page/settings/storage-template/supported-variables-panel.svelte +++ b/web/src/lib/components/admin-page/settings/storage-template/supported-variables-panel.svelte @@ -29,6 +29,14 @@
  • {`{{assetId}}`} - Asset ID
  • {`{{assetIdShort}}`} - Asset ID (last 12 characters)
  • {`{{album}}`} - Album Name
    • +
  • + {`{{album-startDate-x}}`} - Album Start Date and Time (e.g. album-startDate-yy). + {$t('admin.storage_template_date_time_sample', { values: { date: '2021-12-31T05:32:41.750' } })} +
    • +
  • + {`{{album-endDate-x}}`} - Album End Date and Time (e.g. album-endDate-MM). + {$t('admin.storage_template_date_time_sample', { values: { date: '2023-05-06T09:15:17.100' } })} +
    • From ad8511c9788a7c248d6f06186d7090f0978c3ddb Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 21 Apr 2025 23:27:00 -0500 Subject: [PATCH 23/29] feat(docs): APK download button (#17768) --- docs/src/pages/index.tsx | 8 ++++++++ docs/static/img/download-apk-github.svg | 13 +++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 docs/static/img/download-apk-github.svg diff --git a/docs/src/pages/index.tsx b/docs/src/pages/index.tsx index 2ffe1debc7..db299271aa 100644 --- a/docs/src/pages/index.tsx +++ b/docs/src/pages/index.tsx @@ -4,6 +4,7 @@ import Layout from '@theme/Layout'; import { discordPath, discordViewBox } from '@site/src/components/svg-paths'; import ThemedImage from '@theme/ThemedImage'; import Icon from '@mdi/react'; +import { mdiAndroid } from '@mdi/js'; function HomepageHeader() { return (
    @@ -88,11 +89,18 @@ function HomepageHeader() { Get it on Google Play + + +
    + + Download APK + +
    + + + + + + + + + + + + From a8eec92da7c3bb846654553156fa7da6868940c7 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 22 Apr 2025 10:18:44 +0000 Subject: [PATCH 24/29] chore(deps): update dependency @types/node to ^22.14.1 (#17770) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: Daniel Dietzler --- cli/package-lock.json | 4 ++-- cli/package.json | 2 +- e2e/package-lock.json | 6 +++--- e2e/package.json | 2 +- open-api/typescript-sdk/package-lock.json | 8 ++++---- open-api/typescript-sdk/package.json | 2 +- server/package-lock.json | 2 +- server/package.json | 2 +- server/src/services/storage-template.service.spec.ts | 4 ++-- 9 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cli/package-lock.json b/cli/package-lock.json index 22f8980754..daac54a552 100644 --- a/cli/package-lock.json +++ b/cli/package-lock.json @@ -27,7 +27,7 @@ "@types/lodash-es": "^4.17.12", "@types/micromatch": "^4.0.9", "@types/mock-fs": "^4.13.1", - "@types/node": "^22.14.0", + "@types/node": "^22.14.1", "@vitest/coverage-v8": "^3.0.0", "byte-size": "^9.0.0", "cli-progress": "^3.12.0", @@ -61,7 +61,7 @@ "@oazapfts/runtime": "^1.0.2" }, "devDependencies": { - "@types/node": "^22.14.0", + "@types/node": "^22.14.1", "typescript": "^5.3.3" } }, diff --git a/cli/package.json b/cli/package.json index 304c2acfbd..0759eb13ee 100644 --- a/cli/package.json +++ b/cli/package.json @@ -21,7 +21,7 @@ "@types/lodash-es": "^4.17.12", "@types/micromatch": "^4.0.9", "@types/mock-fs": "^4.13.1", - "@types/node": "^22.14.0", + "@types/node": "^22.14.1", "@vitest/coverage-v8": "^3.0.0", "byte-size": "^9.0.0", "cli-progress": "^3.12.0", diff --git a/e2e/package-lock.json b/e2e/package-lock.json index d09a7e9701..07234a770f 100644 --- a/e2e/package-lock.json +++ b/e2e/package-lock.json @@ -15,7 +15,7 @@ "@immich/sdk": "file:../open-api/typescript-sdk", "@playwright/test": "^1.44.1", "@types/luxon": "^3.4.2", - "@types/node": "^22.14.0", + "@types/node": "^22.14.1", "@types/oidc-provider": "^8.5.1", "@types/pg": "^8.11.0", "@types/pngjs": "^6.0.4", @@ -66,7 +66,7 @@ "@types/lodash-es": "^4.17.12", "@types/micromatch": "^4.0.9", "@types/mock-fs": "^4.13.1", - "@types/node": "^22.14.0", + "@types/node": "^22.14.1", "@vitest/coverage-v8": "^3.0.0", "byte-size": "^9.0.0", "cli-progress": "^3.12.0", @@ -100,7 +100,7 @@ "@oazapfts/runtime": "^1.0.2" }, "devDependencies": { - "@types/node": "^22.14.0", + "@types/node": "^22.14.1", "typescript": "^5.3.3" } }, diff --git a/e2e/package.json b/e2e/package.json index f141430c97..8027cb420e 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -25,7 +25,7 @@ "@immich/sdk": "file:../open-api/typescript-sdk", "@playwright/test": "^1.44.1", "@types/luxon": "^3.4.2", - "@types/node": "^22.14.0", + "@types/node": "^22.14.1", "@types/oidc-provider": "^8.5.1", "@types/pg": "^8.11.0", "@types/pngjs": "^6.0.4", diff --git a/open-api/typescript-sdk/package-lock.json b/open-api/typescript-sdk/package-lock.json index 761a228de0..1fe3ea7587 100644 --- a/open-api/typescript-sdk/package-lock.json +++ b/open-api/typescript-sdk/package-lock.json @@ -12,7 +12,7 @@ "@oazapfts/runtime": "^1.0.2" }, "devDependencies": { - "@types/node": "^22.14.0", + "@types/node": "^22.14.1", "typescript": "^5.3.3" } }, @@ -23,9 +23,9 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "22.14.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.14.0.tgz", - "integrity": "sha512-Kmpl+z84ILoG+3T/zQFyAJsU6EPTmOCj8/2+83fSN6djd6I4o7uOuGIH6vq3PrjY5BGitSbFuMN18j3iknubbA==", + "version": "22.14.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.14.1.tgz", + "integrity": "sha512-u0HuPQwe/dHrItgHHpmw3N2fYCR6x4ivMNbPHRkBVP4CvN+kiRrKHWk3i8tXiO/joPwXLMYvF9TTF0eqgHIuOw==", "dev": true, "license": "MIT", "dependencies": { diff --git a/open-api/typescript-sdk/package.json b/open-api/typescript-sdk/package.json index 29fe50dcd9..a3d5d9d224 100644 --- a/open-api/typescript-sdk/package.json +++ b/open-api/typescript-sdk/package.json @@ -19,7 +19,7 @@ "@oazapfts/runtime": "^1.0.2" }, "devDependencies": { - "@types/node": "^22.14.0", + "@types/node": "^22.14.1", "typescript": "^5.3.3" }, "repository": { diff --git a/server/package-lock.json b/server/package-lock.json index 8045976b3c..85edb9426b 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -90,7 +90,7 @@ "@types/lodash": "^4.14.197", "@types/mock-fs": "^4.13.1", "@types/multer": "^1.4.7", - "@types/node": "^22.14.0", + "@types/node": "^22.14.1", "@types/nodemailer": "^6.4.14", "@types/picomatch": "^3.0.0", "@types/pngjs": "^6.0.5", diff --git a/server/package.json b/server/package.json index 8e149d961e..a641c3b29a 100644 --- a/server/package.json +++ b/server/package.json @@ -115,7 +115,7 @@ "@types/lodash": "^4.14.197", "@types/mock-fs": "^4.13.1", "@types/multer": "^1.4.7", - "@types/node": "^22.14.0", + "@types/node": "^22.14.1", "@types/nodemailer": "^6.4.14", "@types/picomatch": "^3.0.0", "@types/pngjs": "^6.0.5", diff --git a/server/src/services/storage-template.service.spec.ts b/server/src/services/storage-template.service.spec.ts index d9ac89952e..9c4fe02f3e 100644 --- a/server/src/services/storage-template.service.spec.ts +++ b/server/src/services/storage-template.service.spec.ts @@ -194,7 +194,7 @@ describe(StorageTemplateService.name, () => { sut.onConfigInit({ newConfig: config }); mocks.user.get.mockResolvedValue(user); - mocks.asset.getStorageTemplateAsset.mockResolvedValueOnce(asset); + mocks.assetJob.getForStorageTemplateJob.mockResolvedValueOnce(asset); mocks.album.getByAssetId.mockResolvedValueOnce([album]); mocks.album.getMetadataForIds.mockResolvedValueOnce([ { @@ -227,7 +227,7 @@ describe(StorageTemplateService.name, () => { sut.onConfigInit({ newConfig: config }); mocks.user.get.mockResolvedValue(user); - mocks.asset.getStorageTemplateAsset.mockResolvedValueOnce(asset); + mocks.assetJob.getForStorageTemplateJob.mockResolvedValueOnce(asset); expect(await sut.handleMigrationSingle({ id: asset.id })).toBe(JobStatus.SUCCESS); From fda68f972ff1f0e1f010f7083de2a4a75b59bce0 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 22 Apr 2025 08:25:27 -0500 Subject: [PATCH 25/29] fix(web): forceDark control app bar doesn't work (#17759) --- .../lib/components/shared-components/control-app-bar.svelte | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/web/src/lib/components/shared-components/control-app-bar.svelte b/web/src/lib/components/shared-components/control-app-bar.svelte index 6b52adff10..7c630875c0 100644 --- a/web/src/lib/components/shared-components/control-app-bar.svelte +++ b/web/src/lib/components/shared-components/control-app-bar.svelte @@ -77,8 +77,7 @@ appBarBorder, 'mx-2 my-2 place-items-center rounded-lg p-2 max-md:p-0 transition-all', tailwindClasses, - 'bg-immich-gray dark:bg-immich-dark-gray', - forceDark && 'bg-immich-dark-gray text-white', + forceDark ? 'bg-immich-dark-gray text-white' : 'bg-immich-gray dark:bg-immich-dark-gray', ]} >
    From af36eaa61bfffe5f83b47877c5a66feb66dd2492 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 22 Apr 2025 10:51:20 -0500 Subject: [PATCH 26/29] fix(mobile): video player initialization (#17778) * fix(mobile): video player initialization * nit --- mobile/lib/pages/common/gallery_viewer.page.dart | 11 +++++++---- mobile/lib/widgets/common/immich_logo.dart | 13 +++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/mobile/lib/pages/common/gallery_viewer.page.dart b/mobile/lib/pages/common/gallery_viewer.page.dart index 7392a4d340..420b699730 100644 --- a/mobile/lib/pages/common/gallery_viewer.page.dart +++ b/mobile/lib/pages/common/gallery_viewer.page.dart @@ -63,9 +63,12 @@ class GalleryViewerPage extends HookConsumerWidget { final loadAsset = renderList.loadAsset; final isPlayingMotionVideo = ref.watch(isPlayingMotionVideoProvider); - // This key is to prevent the video player from being re-initialized during - // hero animation or device rotation. - final videoPlayerKey = useMemoized(() => GlobalKey()); + final videoPlayerKeys = useRef>({}); + + GlobalKey getVideoPlayerKey(int id) { + videoPlayerKeys.value.putIfAbsent(id, () => GlobalKey()); + return videoPlayerKeys.value[id]!; + } Future precacheNextImage(int index) async { if (!context.mounted) { @@ -243,7 +246,7 @@ class GalleryViewerPage extends HookConsumerWidget { width: context.width, height: context.height, child: NativeVideoViewerPage( - key: videoPlayerKey, + key: getVideoPlayerKey(asset.id), asset: asset, image: Image( key: ValueKey(asset), diff --git a/mobile/lib/widgets/common/immich_logo.dart b/mobile/lib/widgets/common/immich_logo.dart index 9f7725aa12..43987878cb 100644 --- a/mobile/lib/widgets/common/immich_logo.dart +++ b/mobile/lib/widgets/common/immich_logo.dart @@ -12,14 +12,11 @@ class ImmichLogo extends StatelessWidget { @override Widget build(BuildContext context) { - return Hero( - tag: heroTag, - child: Image( - image: const AssetImage('assets/immich-logo.png'), - width: size, - filterQuality: FilterQuality.high, - isAntiAlias: true, - ), + return Image( + image: const AssetImage('assets/immich-logo.png'), + width: size, + filterQuality: FilterQuality.high, + isAntiAlias: true, ); } } From 0986a71ce3dfc78c547a3cd99c12da0a7f9d96e8 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 22 Apr 2025 12:15:54 -0500 Subject: [PATCH 27/29] fix(mobile): revert cache fixes (#17786) * Revert "fix(mobile): use immutable cache keys for local images (#17736)" This reverts commit 010b144754680f39979a65668311b5a5ffee2416. * Revert "perf(mobile): remove small thumbnail and cache generated thumbnails (#17682)" This reverts commit b71039e83c2497f9829f1592573dbc2e85bb2f2c. --- .../image/immich_local_image_provider.dart | 4 +- .../immich_local_thumbnail_provider.dart | 58 ++++++++----------- 2 files changed, 26 insertions(+), 36 deletions(-) diff --git a/mobile/lib/providers/image/immich_local_image_provider.dart b/mobile/lib/providers/image/immich_local_image_provider.dart index 34c3c94ded..36fd3334b9 100644 --- a/mobile/lib/providers/image/immich_local_image_provider.dart +++ b/mobile/lib/providers/image/immich_local_image_provider.dart @@ -106,12 +106,12 @@ class ImmichLocalImageProvider extends ImageProvider { bool operator ==(Object other) { if (identical(this, other)) return true; if (other is ImmichLocalImageProvider) { - return asset.id == other.asset.id; + return asset == other.asset; } return false; } @override - int get hashCode => asset.id.hashCode; + int get hashCode => asset.hashCode; } diff --git a/mobile/lib/providers/image/immich_local_thumbnail_provider.dart b/mobile/lib/providers/image/immich_local_thumbnail_provider.dart index 9ccb127626..69cdb105c0 100644 --- a/mobile/lib/providers/image/immich_local_thumbnail_provider.dart +++ b/mobile/lib/providers/image/immich_local_thumbnail_provider.dart @@ -2,14 +2,11 @@ import 'dart:async'; import 'dart:ui' as ui; import 'package:cached_network_image/cached_network_image.dart'; -import 'package:flutter_cache_manager/flutter_cache_manager.dart'; -import 'package:immich_mobile/providers/image/cache/thumbnail_image_cache_manager.dart'; import 'package:flutter/foundation.dart'; import 'package:flutter/painting.dart'; import 'package:immich_mobile/entities/asset.entity.dart'; import 'package:photo_manager/photo_manager.dart' show ThumbnailSize; -import 'package:logging/logging.dart'; /// The local image provider for an asset /// Only viable @@ -18,14 +15,11 @@ class ImmichLocalThumbnailProvider final Asset asset; final int height; final int width; - final CacheManager? cacheManager; - final Logger log = Logger("ImmichLocalThumbnailProvider"); ImmichLocalThumbnailProvider({ required this.asset, this.height = 256, this.width = 256, - this.cacheManager, }) : assert(asset.local != null, 'Only usable when asset.local is set'); /// Converts an [ImageProvider]'s settings plus an [ImageConfiguration] to a key @@ -42,10 +36,11 @@ class ImmichLocalThumbnailProvider ImmichLocalThumbnailProvider key, ImageDecoderCallback decode, ) { - final cache = cacheManager ?? ThumbnailImageCacheManager(); + final chunkEvents = StreamController(); return MultiImageStreamCompleter( - codec: _codec(key.asset, cache, decode), + codec: _codec(key.asset, decode, chunkEvents), scale: 1.0, + chunkEvents: chunkEvents.stream, informationCollector: () sync* { yield ErrorDescription(asset.fileName); }, @@ -55,48 +50,43 @@ class ImmichLocalThumbnailProvider // Streams in each stage of the image as we ask for it Stream _codec( Asset key, - CacheManager cache, ImageDecoderCallback decode, + StreamController chunkEvents, ) async* { - final cacheKey = '${key.id}_${width}x$height'; - final fileFromCache = await cache.getFileFromCache(cacheKey); - if (fileFromCache != null) { - try { - final buffer = - await ui.ImmutableBuffer.fromFilePath(fileFromCache.file.path); - final codec = await decode(buffer); - yield codec; - return; - } catch (error) { - log.severe('Found thumbnail in cache, but loading it failed', error); - } + // Load a small thumbnail + final thumbBytes = await asset.local?.thumbnailDataWithSize( + const ThumbnailSize.square(32), + quality: 75, + ); + if (thumbBytes != null) { + final buffer = await ui.ImmutableBuffer.fromUint8List(thumbBytes); + final codec = await decode(buffer); + yield codec; + } else { + debugPrint("Loading thumb for ${asset.fileName} failed"); } - final thumbnailBytes = await asset.local?.thumbnailDataWithSize( - ThumbnailSize(width, height), - quality: 80, - ); - if (thumbnailBytes == null) { + final normalThumbBytes = + await asset.local?.thumbnailDataWithSize(ThumbnailSize(width, height)); + if (normalThumbBytes == null) { throw StateError( "Loading thumb for local photo ${asset.fileName} failed", ); } - final buffer = await ui.ImmutableBuffer.fromUint8List(thumbnailBytes); + final buffer = await ui.ImmutableBuffer.fromUint8List(normalThumbBytes); final codec = await decode(buffer); yield codec; - await cache.putFile(cacheKey, thumbnailBytes); + + chunkEvents.close(); } @override bool operator ==(Object other) { + if (other is! ImmichLocalThumbnailProvider) return false; if (identical(this, other)) return true; - if (other is ImmichLocalThumbnailProvider) { - return asset.id == other.asset.id; - } - - return false; + return asset == other.asset; } @override - int get hashCode => asset.id.hashCode; + int get hashCode => asset.hashCode; } From ee017803bf3b01ba27935b740bab09360efa868d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Wawrzyk?= Date: Wed, 23 Apr 2025 04:32:03 +0200 Subject: [PATCH 28/29] fix(mobile): use immutable cache keys for local images (#17794) --- .../lib/providers/image/immich_local_image_provider.dart | 5 ++--- .../providers/image/immich_local_thumbnail_provider.dart | 8 +++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/mobile/lib/providers/image/immich_local_image_provider.dart b/mobile/lib/providers/image/immich_local_image_provider.dart index 36fd3334b9..c152934333 100644 --- a/mobile/lib/providers/image/immich_local_image_provider.dart +++ b/mobile/lib/providers/image/immich_local_image_provider.dart @@ -106,12 +106,11 @@ class ImmichLocalImageProvider extends ImageProvider { bool operator ==(Object other) { if (identical(this, other)) return true; if (other is ImmichLocalImageProvider) { - return asset == other.asset; + return asset.id == other.asset.id && asset.localId == other.asset.localId; } - return false; } @override - int get hashCode => asset.hashCode; + int get hashCode => Object.hash(asset.id, asset.localId); } diff --git a/mobile/lib/providers/image/immich_local_thumbnail_provider.dart b/mobile/lib/providers/image/immich_local_thumbnail_provider.dart index 69cdb105c0..54dfd97983 100644 --- a/mobile/lib/providers/image/immich_local_thumbnail_provider.dart +++ b/mobile/lib/providers/image/immich_local_thumbnail_provider.dart @@ -82,11 +82,13 @@ class ImmichLocalThumbnailProvider @override bool operator ==(Object other) { - if (other is! ImmichLocalThumbnailProvider) return false; if (identical(this, other)) return true; - return asset == other.asset; + if (other is ImmichLocalThumbnailProvider) { + return asset.id == other.asset.id && asset.localId == other.asset.localId; + } + return false; } @override - int get hashCode => asset.hashCode; + int get hashCode => Object.hash(asset.id, asset.localId); } From 8c0b416dc3d85864a2fc893c9a52a4cf4946b75d Mon Sep 17 00:00:00 2001 From: Min Idzelis Date: Wed, 23 Apr 2025 02:35:24 +0000 Subject: [PATCH 29/29] feat: improve focus --- .../lib/actions/__test__/focus-trap.spec.ts | 4 ++ web/src/lib/actions/focus-trap.ts | 19 +++--- .../thumbnail/__test__/thumbnail.spec.ts | 39 ++++++------ .../assets/thumbnail/thumbnail.svelte | 35 ++--------- .../elements/buttons/skip-link.svelte | 8 ++- .../photos-page/asset-date-group.svelte | 5 -- .../components/photos-page/asset-grid.svelte | 31 +--------- .../scrubber/scrubber.svelte | 4 +- .../lib/stores/asset-interaction.svelte.ts | 5 -- web/src/lib/utils/focus-util.ts | 62 ++++++++++++++++++- 10 files changed, 109 insertions(+), 103 deletions(-) diff --git a/web/src/lib/actions/__test__/focus-trap.spec.ts b/web/src/lib/actions/__test__/focus-trap.spec.ts index d92d8e037d..b03064a91d 100644 --- a/web/src/lib/actions/__test__/focus-trap.spec.ts +++ b/web/src/lib/actions/__test__/focus-trap.spec.ts @@ -1,8 +1,11 @@ import FocusTrapTest from '$lib/actions/__test__/focus-trap-test.svelte'; +import { setDefaultTabbleOptions } from '$lib/utils/focus-util'; import { render, screen } from '@testing-library/svelte'; import userEvent from '@testing-library/user-event'; import { tick } from 'svelte'; +setDefaultTabbleOptions({ displayCheck: 'none' }); + describe('focusTrap action', () => { const user = userEvent.setup(); @@ -38,6 +41,7 @@ describe('focusTrap action', () => { const openButton = screen.getByText('Open'); await user.click(openButton); + await tick(); expect(document.activeElement).toEqual(screen.getByTestId('one')); screen.getByText('Close').click(); diff --git a/web/src/lib/actions/focus-trap.ts b/web/src/lib/actions/focus-trap.ts index 1564dd90d0..2b03282c2d 100644 --- a/web/src/lib/actions/focus-trap.ts +++ b/web/src/lib/actions/focus-trap.ts @@ -1,5 +1,5 @@ import { shortcuts } from '$lib/actions/shortcut'; -import { getFocusable } from '$lib/utils/focus-util'; +import { getTabbable } from '$lib/utils/focus-util'; import { tick } from 'svelte'; interface Options { @@ -18,18 +18,21 @@ export function focusTrap(container: HTMLElement, options?: Options) { }; }; - const setInitialFocus = () => { - const focusableElement = getFocusable(container)[0]; - // Use tick() to ensure focus trap works correctly inside - void tick().then(() => focusableElement?.focus()); + const setInitialFocus = async () => { + const focusableElement = getTabbable(container, false)[0]; + if (focusableElement) { + // Use tick() to ensure focus trap works correctly inside + await tick(); + focusableElement?.focus(); + } }; if (withDefaults(options).active) { - setInitialFocus(); + void setInitialFocus(); } const getFocusableElements = () => { - const focusableElements = getFocusable(container); + const focusableElements = getTabbable(container); return [ focusableElements.at(0), // focusableElements.at(-1), @@ -67,7 +70,7 @@ export function focusTrap(container: HTMLElement, options?: Options) { update(newOptions?: Options) { options = newOptions; if (withDefaults(options).active) { - setInitialFocus(); + void setInitialFocus(); } }, destroy() { diff --git a/web/src/lib/components/assets/thumbnail/__test__/thumbnail.spec.ts b/web/src/lib/components/assets/thumbnail/__test__/thumbnail.spec.ts index f7447551f0..b6e053da12 100644 --- a/web/src/lib/components/assets/thumbnail/__test__/thumbnail.spec.ts +++ b/web/src/lib/components/assets/thumbnail/__test__/thumbnail.spec.ts @@ -1,5 +1,6 @@ import { getIntersectionObserverMock } from '$lib/__mocks__/intersection-observer.mock'; import Thumbnail from '$lib/components/assets/thumbnail/thumbnail.svelte'; +import { getTabbable } from '$lib/utils/focus-util'; import { assetFactory } from '@test-data/factories/asset-factory'; import { fireEvent, render, screen } from '@testing-library/svelte'; @@ -30,51 +31,47 @@ describe('Thumbnail component', () => { it('should only contain a single tabbable element (the container)', () => { const asset = assetFactory.build({ originalPath: 'image.jpg', originalMimeType: 'image/jpeg' }); - render(Thumbnail, { + const { baseElement } = render(Thumbnail, { asset, - focussed: false, selected: true, }); - const container = screen.getByTestId('container-with-tabindex'); - expect(container.getAttribute('tabindex')).toBe('0'); + const container = baseElement.querySelector('[data-thumbnail-focus-container]'); + expect(container).not.toBeNull(); + expect(container!.getAttribute('tabindex')).toBe('0'); - // This isn't capturing all tabbable elements, but should be the most likely ones. Mainly guarding against - // inserting extra tabbable elments in future in - let allTabbableElements = screen.queryAllByRole('link'); - allTabbableElements = allTabbableElements.concat(screen.queryAllByRole('checkbox')); - expect(allTabbableElements.length).toBeGreaterThan(0); - for (const tabbableElement of allTabbableElements) { - const testIdValue = tabbableElement.dataset.testid; - if (testIdValue === null || testIdValue !== 'container-with-tabindex') { - expect(tabbableElement.getAttribute('tabindex')).toBe('-1'); - } - } + // Guarding against inserting extra tabbable elments in future in + const tabbables = getTabbable(container!); + expect(tabbables.length).toBe(0); }); it('handleFocus should be called on focus of container', async () => { const asset = assetFactory.build({ originalPath: 'image.jpg', originalMimeType: 'image/jpeg' }); const handleFocusSpy = vi.fn(); - render(Thumbnail, { + const { baseElement } = render(Thumbnail, { asset, handleFocus: handleFocusSpy, }); - const container = screen.getByTestId('container-with-tabindex'); - await fireEvent(container, new FocusEvent('focus')); + const container = baseElement.querySelector('[data-thumbnail-focus-container]'); + expect(container).not.toBeNull(); + await fireEvent(container as HTMLElement, new FocusEvent('focus')); expect(handleFocusSpy).toBeCalled(); }); - it('element will be focussed if not already', () => { + it('element will be focussed if not already', async () => { const asset = assetFactory.build({ originalPath: 'image.jpg', originalMimeType: 'image/jpeg' }); const handleFocusSpy = vi.fn(); - render(Thumbnail, { + const { baseElement } = render(Thumbnail, { asset, - focussed: true, handleFocus: handleFocusSpy, }); + const container = baseElement.querySelector('[data-thumbnail-focus-container]'); + expect(container).not.toBeNull(); + await fireEvent(container as HTMLElement, new FocusEvent('focus')); + expect(handleFocusSpy).toBeCalled(); }); }); diff --git a/web/src/lib/components/assets/thumbnail/thumbnail.svelte b/web/src/lib/components/assets/thumbnail/thumbnail.svelte index 93a4e3c6cc..c01c1c9bea 100644 --- a/web/src/lib/components/assets/thumbnail/thumbnail.svelte +++ b/web/src/lib/components/assets/thumbnail/thumbnail.svelte @@ -25,7 +25,7 @@ import ImageThumbnail from './image-thumbnail.svelte'; import VideoThumbnail from './video-thumbnail.svelte'; import { onMount } from 'svelte'; - import { getFocusable } from '$lib/utils/focus-util'; + import { focusNext } from '$lib/utils/focus-util'; interface Props { asset: AssetResponseDto; @@ -34,7 +34,6 @@ thumbnailWidth?: number | undefined; thumbnailHeight?: number | undefined; selected?: boolean; - focussed?: boolean; selectionCandidate?: boolean; disabled?: boolean; disableLinkMouseOver?: boolean; @@ -57,7 +56,6 @@ thumbnailWidth = undefined, thumbnailHeight = undefined, selected = false, - focussed = false, selectionCandidate = false, disabled = false, disableLinkMouseOver = false, @@ -78,17 +76,11 @@ } = TUNABLES; let usingMobileDevice = $derived(mobileDevice.pointerCoarse); - let focussableElement: HTMLElement | undefined = $state(); + let element: HTMLElement | undefined = $state(); let mouseOver = $state(false); let loaded = $state(false); let thumbError = $state(false); - $effect(() => { - if (focussed && document.activeElement !== focussableElement) { - focussableElement?.focus(); - } - }); - let width = $derived(thumbnailSize || thumbnailWidth || 235); let height = $derived(thumbnailSize || thumbnailHeight || 235); @@ -223,31 +215,14 @@ if (evt.key === 'x') { onSelect?.(asset); } - if (document.activeElement === focussableElement && evt.key === 'Escape') { - const focusable = getFocusable(document); - const index = focusable.indexOf(focussableElement); - - let i = index + 1; - while (i !== index) { - const next = focusable[i]; - if (next.dataset.thumbnailFocusContainer !== undefined) { - if (i === focusable.length - 1) { - i = 0; - } else { - i++; - } - continue; - } - next.focus(); - break; - } + if (document.activeElement === element && evt.key === 'Escape') { + focusNext((element) => element.dataset.thumbnailFocusContainer === undefined, true); } }} onclick={handleClick} - bind:this={focussableElement} + bind:this={element} onfocus={handleFocus} data-thumbnail-focus-container - data-testid="container-with-tabindex" tabindex={0} role="link" > diff --git a/web/src/lib/components/elements/buttons/skip-link.svelte b/web/src/lib/components/elements/buttons/skip-link.svelte index a1a24634c4..331814813c 100644 --- a/web/src/lib/components/elements/buttons/skip-link.svelte +++ b/web/src/lib/components/elements/buttons/skip-link.svelte @@ -1,6 +1,7 @@