diff --git a/mobile/lib/pages/photos/photos.page.dart b/mobile/lib/pages/photos/photos.page.dart index 2ef738aaf1..aecac56065 100644 --- a/mobile/lib/pages/photos/photos.page.dart +++ b/mobile/lib/pages/photos/photos.page.dart @@ -33,7 +33,6 @@ class PhotosPage extends HookConsumerWidget { () { ref.read(websocketProvider.notifier).connect(); Future(() => ref.read(assetProvider.notifier).getAllAsset()); - ref.read(assetProvider.notifier).getPartnerAssets(); ref.read(albumProvider.notifier).getAllAlbums(); ref.read(sharedAlbumProvider.notifier).getAllSharedAlbums(); ref.read(serverInfoProvider.notifier).getServerInfo(); @@ -85,9 +84,6 @@ class PhotosPage extends HookConsumerWidget { Future refreshAssets() async { final fullRefresh = refreshCount.value > 0; await ref.read(assetProvider.notifier).getAllAsset(clear: fullRefresh); - if (timelineUsers.length > 1) { - await ref.read(assetProvider.notifier).getPartnerAssets(); - } if (fullRefresh) { // refresh was forced: user requested another refresh within 2 seconds refreshCount.value = 0; diff --git a/mobile/lib/pages/sharing/partner/partner_detail.page.dart b/mobile/lib/pages/sharing/partner/partner_detail.page.dart index a858320b12..8a2dd4b820 100644 --- a/mobile/lib/pages/sharing/partner/partner_detail.page.dart +++ b/mobile/lib/pages/sharing/partner/partner_detail.page.dart @@ -22,7 +22,7 @@ class PartnerDetailPage extends HookConsumerWidget { useEffect( () { - ref.read(assetProvider.notifier).getPartnerAssets(partner); + ref.read(assetProvider.notifier).getAllAsset(); return null; }, [], @@ -78,8 +78,7 @@ class PartnerDetailPage extends HookConsumerWidget { ), body: MultiselectGrid( renderListProvider: assetsProvider(partner.isarId), - onRefresh: () => - ref.read(assetProvider.notifier).getPartnerAssets(partner), + onRefresh: () => ref.read(assetProvider.notifier).getAllAsset(), deleteEnabled: false, favoriteEnabled: false, ), diff --git a/mobile/lib/providers/app_life_cycle.provider.dart b/mobile/lib/providers/app_life_cycle.provider.dart index bb6c9722cc..938961efb6 100644 --- a/mobile/lib/providers/app_life_cycle.provider.dart +++ b/mobile/lib/providers/app_life_cycle.provider.dart @@ -56,11 +56,10 @@ class AppLifeCycleNotifier extends StateNotifier { switch (_ref.read(tabProvider)) { case TabEnum.home: _ref.read(assetProvider.notifier).getAllAsset(); - _ref.read(assetProvider.notifier).getPartnerAssets(); case TabEnum.search: // nothing to do case TabEnum.sharing: - _ref.read(assetProvider.notifier).getPartnerAssets(); + _ref.read(assetProvider.notifier).getAllAsset(); _ref.read(sharedAlbumProvider.notifier).getAllSharedAlbums(); case TabEnum.library: _ref.read(albumProvider.notifier).getAllAlbums(); diff --git a/mobile/lib/providers/asset.provider.dart b/mobile/lib/providers/asset.provider.dart index 4c12ed2f10..a0a3879db5 100644 --- a/mobile/lib/providers/asset.provider.dart +++ b/mobile/lib/providers/asset.provider.dart @@ -1,9 +1,9 @@ import 'package:flutter/material.dart'; import 'package:hooks_riverpod/hooks_riverpod.dart'; +import 'package:immich_mobile/providers/memory.provider.dart'; import 'package:immich_mobile/services/album.service.dart'; import 'package:immich_mobile/entities/exif_info.entity.dart'; import 'package:immich_mobile/entities/store.entity.dart'; -import 'package:immich_mobile/entities/user.entity.dart'; import 'package:immich_mobile/providers/db.provider.dart'; import 'package:immich_mobile/providers/user.provider.dart'; import 'package:immich_mobile/services/asset.service.dart'; @@ -23,10 +23,10 @@ class AssetNotifier extends StateNotifier { final UserService _userService; final SyncService _syncService; final Isar _db; + final StateNotifierProviderRef _ref; final log = Logger('AssetNotifier'); bool _getAllAssetInProgress = false; bool _deleteInProgress = false; - bool _getPartnerAssetsInProgress = false; AssetNotifier( this._assetService, @@ -34,6 +34,7 @@ class AssetNotifier extends StateNotifier { this._userService, this._syncService, this._db, + this._ref, ) : super(false); Future getAllAsset({bool clear = false}) async { @@ -49,9 +50,15 @@ class AssetNotifier extends StateNotifier { await clearAssetsAndAlbums(_db); log.info("Manual refresh requested, cleared assets and albums from db"); } + final bool changedUsers = await _userService.refreshUsers(); final bool newRemote = await _assetService.refreshRemoteAssets(); final bool newLocal = await _albumService.refreshDeviceAlbums(); - debugPrint("newRemote: $newRemote, newLocal: $newLocal"); + debugPrint( + "changedUsers: $changedUsers, newRemote: $newRemote, newLocal: $newLocal", + ); + if (newRemote) { + _ref.invalidate(memoryFutureProvider); + } log.info("Load assets: ${stopwatch.elapsedMilliseconds}ms"); } finally { @@ -60,27 +67,6 @@ class AssetNotifier extends StateNotifier { } } - Future getPartnerAssets([User? partner]) async { - if (_getPartnerAssetsInProgress) return; - try { - final stopwatch = Stopwatch()..start(); - _getPartnerAssetsInProgress = true; - if (partner == null) { - await _userService.refreshUsers(); - final List partners = - await _db.users.filter().isPartnerSharedWithEqualTo(true).findAll(); - for (User u in partners) { - await _assetService.refreshRemoteAssets(u); - } - } else { - await _assetService.refreshRemoteAssets(partner); - } - log.info("Load partner assets: ${stopwatch.elapsedMilliseconds}ms"); - } finally { - _getPartnerAssetsInProgress = false; - } - } - Future clearAllAsset() { return clearAssetsAndAlbums(_db); } @@ -321,6 +307,7 @@ final assetProvider = StateNotifierProvider((ref) { ref.watch(userServiceProvider), ref.watch(syncServiceProvider), ref.watch(dbProvider), + ref, ); }); diff --git a/mobile/lib/routing/tab_navigation_observer.dart b/mobile/lib/routing/tab_navigation_observer.dart index d60cdb683f..f88adbda91 100644 --- a/mobile/lib/routing/tab_navigation_observer.dart +++ b/mobile/lib/routing/tab_navigation_observer.dart @@ -43,7 +43,7 @@ class TabNavigationObserver extends AutoRouterObserver { if (route.name == 'SharingRoute') { ref.read(sharedAlbumProvider.notifier).getAllSharedAlbums(); - ref.read(assetProvider.notifier).getPartnerAssets(); + Future(() => ref.read(assetProvider.notifier).getAllAsset()); } if (route.name == 'LibraryRoute') { diff --git a/mobile/lib/services/api.service.dart b/mobile/lib/services/api.service.dart index cc57d6b26e..07b9a6e177 100644 --- a/mobile/lib/services/api.service.dart +++ b/mobile/lib/services/api.service.dart @@ -23,6 +23,7 @@ class ApiService { late PersonApi personApi; late AuditApi auditApi; late SharedLinkApi sharedLinkApi; + late SyncApi syncApi; late SystemConfigApi systemConfigApi; late ActivityApi activityApi; late DownloadApi downloadApi; @@ -53,6 +54,7 @@ class ApiService { personApi = PersonApi(_apiClient); auditApi = AuditApi(_apiClient); sharedLinkApi = SharedLinkApi(_apiClient); + syncApi = SyncApi(_apiClient); systemConfigApi = SystemConfigApi(_apiClient); activityApi = ActivityApi(_apiClient); downloadApi = DownloadApi(_apiClient); diff --git a/mobile/lib/services/asset.service.dart b/mobile/lib/services/asset.service.dart index 5610dc435d..aa156a9586 100644 --- a/mobile/lib/services/asset.service.dart +++ b/mobile/lib/services/asset.service.dart @@ -5,13 +5,14 @@ import 'dart:async'; import 'package:flutter/material.dart'; import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/entities/asset.entity.dart'; +import 'package:immich_mobile/entities/etag.entity.dart'; import 'package:immich_mobile/entities/exif_info.entity.dart'; -import 'package:immich_mobile/entities/store.entity.dart'; import 'package:immich_mobile/entities/user.entity.dart'; import 'package:immich_mobile/providers/api.provider.dart'; import 'package:immich_mobile/providers/db.provider.dart'; import 'package:immich_mobile/services/api.service.dart'; import 'package:immich_mobile/services/sync.service.dart'; +import 'package:immich_mobile/services/user.service.dart'; import 'package:isar/isar.dart'; import 'package:logging/logging.dart'; import 'package:maplibre_gl/maplibre_gl.dart'; @@ -21,6 +22,7 @@ final assetServiceProvider = Provider( (ref) => AssetService( ref.watch(apiServiceProvider), ref.watch(syncServiceProvider), + ref.watch(userServiceProvider), ref.watch(dbProvider), ), ); @@ -28,24 +30,33 @@ final assetServiceProvider = Provider( class AssetService { final ApiService _apiService; final SyncService _syncService; + final UserService _userService; final log = Logger('AssetService'); final Isar _db; AssetService( this._apiService, this._syncService, + this._userService, this._db, ); /// Checks the server for updated assets and updates the local database if /// required. Returns `true` if there were any changes. - Future refreshRemoteAssets([User? user]) async { - user ??= Store.get(StoreKey.currentUser); + Future refreshRemoteAssets() async { + final syncedUserIds = await _db.eTags.where().idProperty().findAll(); + final List syncedUsers = syncedUserIds.isEmpty + ? [] + : await _db.users + .where() + .anyOf(syncedUserIds, (q, id) => q.idEqualTo(id)) + .findAll(); final Stopwatch sw = Stopwatch()..start(); final bool changes = await _syncService.syncRemoteAssetsToDb( - user, - _getRemoteAssetChanges, - _getRemoteAssets, + users: syncedUsers, + getChangedAssets: _getRemoteAssetChanges, + loadAssets: _getRemoteAssets, + refreshUsers: _userService.getUsersFromServer, ); debugPrint("refreshRemoteAssets full took ${sw.elapsedMilliseconds}ms"); return changes; @@ -53,14 +64,15 @@ class AssetService { /// Returns `(null, null)` if changes are invalid -> requires full sync Future<(List? toUpsert, List? toDelete)> - _getRemoteAssetChanges(User user, DateTime since) async { - final deleted = await _apiService.auditApi - .getAuditDeletes(since, EntityType.ASSET, userId: user.id); - if (deleted == null || deleted.needsFullSync) return (null, null); - final assetDto = await _apiService.assetApi - .getAllAssets(userId: user.id, updatedAfter: since); - if (assetDto == null) return (null, null); - return (assetDto.map(Asset.remote).toList(), deleted.ids); + _getRemoteAssetChanges(List users, DateTime since) async { + final dto = AssetDeltaSyncDto( + updatedAfter: since, + userIds: users.map((e) => e.id).toList(), + ); + final changes = await _apiService.syncApi.getDeltaSync(dto); + return changes == null || changes.needsFullSync + ? (null, null) + : (changes.upserted.map(Asset.remote).toList(), changes.deleted); } /// Returns the list of people of the given asset id. @@ -85,38 +97,32 @@ class AssetService { } /// Returns `null` if the server state did not change, else list of assets - Future?> _getRemoteAssets(User user) async { + Future?> _getRemoteAssets(User user, DateTime until) async { const int chunkSize = 10000; try { - final DateTime now = DateTime.now().toUtc(); final List allAssets = []; - for (int i = 0;; i += chunkSize) { - final List? assets = - await _apiService.assetApi.getAllAssets( + DateTime? lastCreationDate; + String? lastId; + // will break on error or once all assets are loaded + while (true) { + final dto = AssetFullSyncDto( + limit: chunkSize, + updatedUntil: until, + lastId: lastId, + lastCreationDate: lastCreationDate, userId: user.id, - // updatedBefore is important! without it we could - // a) get the same Asset multiple times in different versions (when - // the asset is modified while the chunks are loaded from the server) - // b) miss assets when new assets are inserted in between the calls - updatedBefore: now, - skip: i, - take: chunkSize, ); - if (assets == null) { - return null; - } + final List? assets = + await _apiService.syncApi.getFullSyncForUser(dto); + if (assets == null) return null; allAssets.addAll(assets.map(Asset.remote)); - if (assets.length < chunkSize) { - break; - } + if (assets.isEmpty) break; + lastCreationDate = assets.last.fileCreatedAt; + lastId = assets.last.id; } return allAssets; } catch (error, stack) { - log.severe( - 'Error while getting remote assets', - error, - stack, - ); + log.severe('Error while getting remote assets', error, stack); return null; } } diff --git a/mobile/lib/services/memory.service.dart b/mobile/lib/services/memory.service.dart index 0a1ceecf16..96b8c6900e 100644 --- a/mobile/lib/services/memory.service.dart +++ b/mobile/lib/services/memory.service.dart @@ -37,12 +37,16 @@ class MemoryService { List memories = []; for (final MemoryLaneResponseDto(:title, :assets) in data) { - memories.add( - Memory( - title: title, - assets: await _db.assets.getAllByRemoteId(assets.map((e) => e.id)), - ), - ); + final dbAssets = + await _db.assets.getAllByRemoteId(assets.map((e) => e.id)); + if (dbAssets.isNotEmpty) { + memories.add( + Memory( + title: title, + assets: dbAssets, + ), + ); + } } return memories.isNotEmpty ? memories : null; diff --git a/mobile/lib/services/sync.service.dart b/mobile/lib/services/sync.service.dart index ba4336c59d..acbd34d183 100644 --- a/mobile/lib/services/sync.service.dart +++ b/mobile/lib/services/sync.service.dart @@ -40,18 +40,20 @@ class SyncService { /// Syncs remote assets owned by the logged-in user to the DB /// Returns `true` if there were any changes - Future syncRemoteAssetsToDb( - User user, - Future<(List? toUpsert, List? toDelete)> Function( - User user, + Future syncRemoteAssetsToDb({ + required List users, + required Future<(List? toUpsert, List? toDelete)> Function( + List users, DateTime since, ) getChangedAssets, - FutureOr?> Function(User user) loadAssets, - ) => + required FutureOr?> Function(User user, DateTime until) + loadAssets, + required FutureOr?> Function() refreshUsers, + }) => _lock.run( () async => - await _syncRemoteAssetChanges(user, getChangedAssets) ?? - await _syncRemoteAssetsFull(user, loadAssets), + await _syncRemoteAssetChanges(users, getChangedAssets) ?? + await _syncRemoteAssetsFull(refreshUsers, loadAssets), ); /// Syncs remote albums to the database @@ -111,7 +113,8 @@ class SyncService { both: (User a, User b) { if (!a.updatedAt.isAtSameMomentAs(b.updatedAt) || a.isPartnerSharedBy != b.isPartnerSharedBy || - a.isPartnerSharedWith != b.isPartnerSharedWith) { + a.isPartnerSharedWith != b.isPartnerSharedWith || + a.inTimeline != b.inTimeline) { toUpsert.add(a); return true; } @@ -149,17 +152,22 @@ class SyncService { /// Efficiently syncs assets via changes. Returns `null` when a full sync is required. Future _syncRemoteAssetChanges( - User user, + List users, Future<(List? toUpsert, List? toDelete)> Function( - User user, + List users, DateTime since, ) getChangedAssets, ) async { - final DateTime? since = _db.eTags.getByIdSync(user.id)?.time?.toUtc(); + final currentUser = Store.get(StoreKey.currentUser); + final DateTime? since = + _db.eTags.getSync(currentUser.isarId)?.time?.toUtc(); if (since == null) return null; final DateTime now = DateTime.now(); - final (toUpsert, toDelete) = await getChangedAssets(user, since); - if (toUpsert == null || toDelete == null) return null; + final (toUpsert, toDelete) = await getChangedAssets(users, since); + if (toUpsert == null || toDelete == null) { + await _clearUserAssetsETag(users); + return null; + } try { if (toDelete.isNotEmpty) { await handleRemoteAssetRemoval(toDelete); @@ -169,7 +177,7 @@ class SyncService { await upsertAssetsWithExif(updated); } if (toUpsert.isNotEmpty || toDelete.isNotEmpty) { - await _updateUserAssetsETag(user, now); + await _updateUserAssetsETag(users, now); return true; } return false; @@ -203,11 +211,34 @@ class SyncService { /// Syncs assets by loading and comparing all assets from the server. Future _syncRemoteAssetsFull( + FutureOr?> Function() refreshUsers, + FutureOr?> Function(User user, DateTime until) loadAssets, + ) async { + final serverUsers = await refreshUsers(); + if (serverUsers == null) { + _log.warning("_syncRemoteAssetsFull aborted because user refresh failed"); + return false; + } + await _syncUsersFromServer(serverUsers); + final List users = await _db.users + .filter() + .isPartnerSharedWithEqualTo(true) + .or() + .isarIdEqualTo(Store.get(StoreKey.currentUser).isarId) + .findAll(); + bool changes = false; + for (User u in users) { + changes |= await _syncRemoteAssetsForUser(u, loadAssets); + } + return changes; + } + + Future _syncRemoteAssetsForUser( User user, - FutureOr?> Function(User user) loadAssets, + FutureOr?> Function(User user, DateTime until) loadAssets, ) async { final DateTime now = DateTime.now().toUtc(); - final List? remote = await loadAssets(user); + final List? remote = await loadAssets(user, now); if (remote == null) { return false; } @@ -225,7 +256,7 @@ class SyncService { final (toAdd, toUpdate, toRemove) = _diffAssets(remote, inDb, remote: true); if (toAdd.isEmpty && toUpdate.isEmpty && toRemove.isEmpty) { - await _updateUserAssetsETag(user, now); + await _updateUserAssetsETag([user], now); return false; } final idsToDelete = toRemove.map((e) => e.id).toList(); @@ -235,12 +266,19 @@ class SyncService { } on IsarError catch (e) { _log.severe("Failed to sync remote assets to db", e); } - await _updateUserAssetsETag(user, now); + await _updateUserAssetsETag([user], now); return true; } - Future _updateUserAssetsETag(User user, DateTime time) => - _db.writeTxn(() => _db.eTags.put(ETag(id: user.id, time: time))); + Future _updateUserAssetsETag(List users, DateTime time) { + final etags = users.map((u) => ETag(id: u.id, time: time)).toList(); + return _db.writeTxn(() => _db.eTags.putAll(etags)); + } + + Future _clearUserAssetsETag(List users) { + final ids = users.map((u) => u.id).toList(); + return _db.writeTxn(() => _db.eTags.deleteAllById(ids)); + } /// Syncs remote albums to the database /// returns `true` if there were any changes diff --git a/mobile/lib/services/user.service.dart b/mobile/lib/services/user.service.dart index b6e5521def..81100f1624 100644 --- a/mobile/lib/services/user.service.dart +++ b/mobile/lib/services/user.service.dart @@ -70,7 +70,7 @@ class UserService { } } - Future refreshUsers() async { + Future?> getUsersFromServer() async { final List? users = await _getAllUsers(isAll: true); final List? sharedBy = await _partnerService.getPartners(PartnerDirection.sharedBy); @@ -79,7 +79,7 @@ class UserService { if (users == null || sharedBy == null || sharedWith == null) { _log.warning("Failed to refresh users"); - return false; + return null; } users.sortBy((u) => u.id); @@ -108,6 +108,12 @@ class UserService { onlySecond: (_) {}, ); + return users; + } + + Future refreshUsers() async { + final users = await getUsersFromServer(); + if (users == null) return false; return _syncService.syncUsersFromServer(users); } } diff --git a/mobile/lib/utils/migration.dart b/mobile/lib/utils/migration.dart index d73a4cd317..2b02a5ff8f 100644 --- a/mobile/lib/utils/migration.dart +++ b/mobile/lib/utils/migration.dart @@ -4,17 +4,12 @@ import 'package:immich_mobile/entities/store.entity.dart'; import 'package:immich_mobile/utils/db.dart'; import 'package:isar/isar.dart'; +const int targetVersion = 6; + Future migrateDatabaseIfNeeded(Isar db) async { final int version = Store.get(StoreKey.version, 1); - switch (version) { - case 1: - await _migrateTo(db, 2); - case 2: - await _migrateTo(db, 3); - case 3: - await _migrateTo(db, 4); - case 4: - await _migrateTo(db, 5); + if (version < targetVersion) { + _migrateTo(db, targetVersion); } } diff --git a/mobile/test/modules/shared/sync_service_test.dart b/mobile/test/modules/shared/sync_service_test.dart index 735cc57a1b..24f0c443ba 100644 --- a/mobile/test/modules/shared/sync_service_test.dart +++ b/mobile/test/modules/shared/sync_service_test.dart @@ -75,8 +75,12 @@ void main() { makeAsset(checksum: "c", remoteId: "1-1"), ]; expect(db.assets.countSync(), 5); - final bool c1 = - await s.syncRemoteAssetsToDb(owner, _failDiff, (u) => remoteAssets); + final bool c1 = await s.syncRemoteAssetsToDb( + users: [owner], + getChangedAssets: _failDiff, + loadAssets: (u, d) => remoteAssets, + refreshUsers: () => [owner], + ); expect(c1, isFalse); expect(db.assets.countSync(), 5); }); @@ -92,8 +96,12 @@ void main() { makeAsset(checksum: "g", remoteId: "3-1"), ]; expect(db.assets.countSync(), 5); - final bool c1 = - await s.syncRemoteAssetsToDb(owner, _failDiff, (u) => remoteAssets); + final bool c1 = await s.syncRemoteAssetsToDb( + users: [owner], + getChangedAssets: _failDiff, + loadAssets: (u, d) => remoteAssets, + refreshUsers: () => [owner], + ); expect(c1, isTrue); expect(db.assets.countSync(), 7); }); @@ -109,23 +117,39 @@ void main() { makeAsset(checksum: "j", remoteId: "2-1d"), ]; expect(db.assets.countSync(), 5); - final bool c1 = - await s.syncRemoteAssetsToDb(owner, _failDiff, (u) => remoteAssets); + final bool c1 = await s.syncRemoteAssetsToDb( + users: [owner], + getChangedAssets: _failDiff, + loadAssets: (u, d) => remoteAssets, + refreshUsers: () => [owner], + ); expect(c1, isTrue); expect(db.assets.countSync(), 8); - final bool c2 = - await s.syncRemoteAssetsToDb(owner, _failDiff, (u) => remoteAssets); + final bool c2 = await s.syncRemoteAssetsToDb( + users: [owner], + getChangedAssets: _failDiff, + loadAssets: (u, d) => remoteAssets, + refreshUsers: () => [owner], + ); expect(c2, isFalse); expect(db.assets.countSync(), 8); remoteAssets.removeAt(4); - final bool c3 = - await s.syncRemoteAssetsToDb(owner, _failDiff, (u) => remoteAssets); + final bool c3 = await s.syncRemoteAssetsToDb( + users: [owner], + getChangedAssets: _failDiff, + loadAssets: (u, d) => remoteAssets, + refreshUsers: () => [owner], + ); expect(c3, isTrue); expect(db.assets.countSync(), 7); remoteAssets.add(makeAsset(checksum: "k", remoteId: "2-1e")); remoteAssets.add(makeAsset(checksum: "l", remoteId: "2-2")); - final bool c4 = - await s.syncRemoteAssetsToDb(owner, _failDiff, (u) => remoteAssets); + final bool c4 = await s.syncRemoteAssetsToDb( + users: [owner], + getChangedAssets: _failDiff, + loadAssets: (u, d) => remoteAssets, + refreshUsers: () => [owner], + ); expect(c4, isTrue); expect(db.assets.countSync(), 9); }); @@ -140,9 +164,10 @@ void main() { toUpsert[0].isFavorite = true; final List toDelete = ["2-1", "1-1"]; final bool c = await s.syncRemoteAssetsToDb( - owner, - (user, since) async => (toUpsert, toDelete), - (user) => throw Exception(), + users: [owner], + getChangedAssets: (user, since) async => (toUpsert, toDelete), + loadAssets: (user, date) => throw Exception(), + refreshUsers: () => throw Exception(), ); expect(c, isTrue); expect(db.assets.countSync(), 6); @@ -150,5 +175,8 @@ void main() { }); } -Future<(List?, List?)> _failDiff(User user, DateTime time) => +Future<(List?, List?)> _failDiff( + List user, + DateTime time, +) => Future.value((null, null));