From 3ccde454b1e47c69ccdd25bb48195275f7d4ac5f Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 9 Jun 2025 14:35:37 -0500 Subject: [PATCH] wip --- .../domain/interfaces/backup.interface.dart | 8 +- .../domain/interfaces/storage.interface.dart | 3 + .../repositories/backup.repository.dart | 140 ++++++++++++++++-- .../repositories/storage.repository.dart | 20 +++ mobile/lib/interfaces/upload.interface.dart | 2 +- .../backup/exp_backup_controller.page.dart | 94 ++++++------ .../providers/backup/exp_backup.provider.dart | 103 +++++++++++-- .../lib/repositories/upload.repository.dart | 7 +- mobile/lib/services/backup.service.dart | 2 +- mobile/lib/services/exp_backup.service.dart | 76 ++++++++-- mobile/lib/services/upload.service.dart | 14 +- 11 files changed, 374 insertions(+), 95 deletions(-) diff --git a/mobile/lib/domain/interfaces/backup.interface.dart b/mobile/lib/domain/interfaces/backup.interface.dart index 61ab41d16e..d678a0e138 100644 --- a/mobile/lib/domain/interfaces/backup.interface.dart +++ b/mobile/lib/domain/interfaces/backup.interface.dart @@ -7,8 +7,10 @@ abstract interface class IBackupRepository implements IDatabaseRepository { Future> getAssetIds(String albumId); - /// Returns the total number of assets that are selected for backup. - Future getTotalCount(BackupSelection selection); - + Future getTotalCount(); + Future getRemainderCount(); Future getBackupCount(); + + Future> getBackupAlbums(BackupSelection selectionType); + Future> getCandidates(); } diff --git a/mobile/lib/domain/interfaces/storage.interface.dart b/mobile/lib/domain/interfaces/storage.interface.dart index ea6513e7f2..c70aba08e4 100644 --- a/mobile/lib/domain/interfaces/storage.interface.dart +++ b/mobile/lib/domain/interfaces/storage.interface.dart @@ -1,7 +1,10 @@ import 'dart:io'; import 'package:immich_mobile/domain/models/asset/base_asset.model.dart'; +// ignore: import_rule_photo_manager +import 'package:photo_manager/photo_manager.dart'; abstract interface class IStorageRepository { Future getFileForAsset(LocalAsset asset); + Future getAssetEntityForAsset(LocalAsset asset); } diff --git a/mobile/lib/infrastructure/repositories/backup.repository.dart b/mobile/lib/infrastructure/repositories/backup.repository.dart index f23293ef6c..59c73a2ba3 100644 --- a/mobile/lib/infrastructure/repositories/backup.repository.dart +++ b/mobile/lib/infrastructure/repositories/backup.repository.dart @@ -51,7 +51,9 @@ class DriftBackupRepository extends DriftDatabaseRepository } @override - Future getTotalCount(BackupSelection selection) { + Future getTotalCount() async { + final excludedAssetIds = await _getExcludedAssetIds(); + final query = _db.localAlbumAssetEntity.selectOnly(distinct: true) ..addColumns([_db.localAlbumAssetEntity.assetId]) ..join([ @@ -61,29 +63,147 @@ class DriftBackupRepository extends DriftDatabaseRepository ), ]) ..where( - _db.localAlbumEntity.backupSelection.equals(selection.index), + _db.localAlbumEntity.backupSelection + .equals(BackupSelection.selected.index) & + (excludedAssetIds.isEmpty + ? const Constant(true) + : _db.localAlbumAssetEntity.assetId.isNotIn(excludedAssetIds)), ); return query.get().then((rows) => rows.length); } @override - Future getBackupCount() { - final query = _db.localAlbumEntity.select().join( - [ + Future getRemainderCount() async { + final excludedAssetIds = await _getExcludedAssetIds(); + + final query = _db.localAlbumAssetEntity.selectOnly(distinct: true) + ..addColumns( + [_db.localAlbumAssetEntity.assetId], + ) + ..join([ innerJoin( - _db.localAlbumAssetEntity, + _db.localAlbumEntity, _db.localAlbumAssetEntity.albumId.equalsExp(_db.localAlbumEntity.id), ), - ], - )..where( - _db.localAlbumEntity.backupSelection.equals( - BackupSelection.selected.index, + innerJoin( + _db.localAssetEntity, + _db.localAlbumAssetEntity.assetId.equalsExp(_db.localAssetEntity.id), ), + leftOuterJoin( + _db.remoteAssetEntity, + _db.localAssetEntity.checksum + .equalsExp(_db.remoteAssetEntity.checksum), + ), + ]) + ..where( + _db.localAlbumEntity.backupSelection + .equals(BackupSelection.selected.index) & + _db.remoteAssetEntity.checksum.isNull() & + (excludedAssetIds.isEmpty + ? const Constant(true) + : _db.localAlbumAssetEntity.assetId.isNotIn(excludedAssetIds)), ); return query.get().then((rows) => rows.length); } + + @override + Future getBackupCount() async { + final excludedAssetIds = await _getExcludedAssetIds(); + final query = _db.localAlbumAssetEntity.selectOnly(distinct: true) + ..addColumns( + [_db.localAlbumAssetEntity.assetId], + ) + ..join([ + innerJoin( + _db.localAlbumEntity, + _db.localAlbumAssetEntity.albumId.equalsExp(_db.localAlbumEntity.id), + ), + innerJoin( + _db.localAssetEntity, + _db.localAlbumAssetEntity.assetId.equalsExp(_db.localAssetEntity.id), + ), + innerJoin( + _db.remoteAssetEntity, + _db.localAssetEntity.checksum + .equalsExp(_db.remoteAssetEntity.checksum), + ), + ]) + ..where( + _db.localAlbumEntity.backupSelection + .equals(BackupSelection.selected.index) & + _db.remoteAssetEntity.checksum.isNotNull() & + (excludedAssetIds.isEmpty + ? const Constant(true) + : _db.localAlbumAssetEntity.assetId.isNotIn(excludedAssetIds)), + ); + + return query.get().then((rows) => rows.length); + } + + Future> _getExcludedAssetIds() async { + final query = _db.localAlbumAssetEntity.selectOnly() + ..addColumns([_db.localAlbumAssetEntity.assetId]) + ..join([ + innerJoin( + _db.localAlbumEntity, + _db.localAlbumAssetEntity.albumId.equalsExp(_db.localAlbumEntity.id), + ), + ]) + ..where( + _db.localAlbumEntity.backupSelection + .equals(BackupSelection.excluded.index), + ); + + return query + .map((row) => row.read(_db.localAlbumAssetEntity.assetId)!) + .get(); + } + + @override + Future> getBackupAlbums(BackupSelection selectionType) { + final query = _db.localAlbumEntity.select() + ..where( + (tbl) => tbl.backupSelection.equals(selectionType.index), + ); + + return query.map((localAlbum) => localAlbum.toDto(assetCount: 0)).get(); + } + + @override + Future> getCandidates() async { + final excludedAssetIds = await _getExcludedAssetIds(); + + final query = _db.localAlbumAssetEntity.select(distinct: true).join( + [ + innerJoin( + _db.localAlbumEntity, + _db.localAlbumAssetEntity.albumId.equalsExp(_db.localAlbumEntity.id), + ), + innerJoin( + _db.localAssetEntity, + _db.localAlbumAssetEntity.assetId.equalsExp(_db.localAssetEntity.id), + ), + leftOuterJoin( + _db.remoteAssetEntity, + _db.localAssetEntity.checksum + .equalsExp(_db.remoteAssetEntity.checksum), + ), + ], + )..where( + _db.localAlbumEntity.backupSelection + .equals(BackupSelection.selected.index) & + _db.remoteAssetEntity.checksum.isNull() & + (excludedAssetIds.isEmpty + ? const Constant(true) + : _db.localAlbumAssetEntity.assetId.isNotIn(excludedAssetIds)), + ); + + return query + .map((row) => row.readTable(_db.localAssetEntity).toDto()) + .get(); + } } extension on LocalAlbumEntityData { diff --git a/mobile/lib/infrastructure/repositories/storage.repository.dart b/mobile/lib/infrastructure/repositories/storage.repository.dart index 57dfc42135..c4f999f882 100644 --- a/mobile/lib/infrastructure/repositories/storage.repository.dart +++ b/mobile/lib/infrastructure/repositories/storage.repository.dart @@ -28,4 +28,24 @@ class StorageRepository implements IStorageRepository { } return file; } + + @override + Future getAssetEntityForAsset(LocalAsset asset) async { + AssetEntity? entity; + try { + entity = await AssetEntity.fromId(asset.id); + if (entity == null) { + _log.warning( + "Cannot get AssetEntity for asset ${asset.id}, name: ${asset.name}, created on: ${asset.createdAt}", + ); + } + } catch (error, stackTrace) { + _log.warning( + "Error getting AssetEntity for asset ${asset.id}, name: ${asset.name}, created on: ${asset.createdAt}", + error, + stackTrace, + ); + } + return entity; + } } diff --git a/mobile/lib/interfaces/upload.interface.dart b/mobile/lib/interfaces/upload.interface.dart index 455e588cbc..4959b277e5 100644 --- a/mobile/lib/interfaces/upload.interface.dart +++ b/mobile/lib/interfaces/upload.interface.dart @@ -6,7 +6,7 @@ abstract interface class IUploadRepository { void enqueueAll(List tasks); Future cancel(String id); - void cancelAll(); + Future cancelAll(); Future pauseAll(); Future deleteAllTrackingRecords(); Future deleteRecordsWithIds(List id); diff --git a/mobile/lib/pages/backup/exp_backup_controller.page.dart b/mobile/lib/pages/backup/exp_backup_controller.page.dart index db6ca16522..648838f37c 100644 --- a/mobile/lib/pages/backup/exp_backup_controller.page.dart +++ b/mobile/lib/pages/backup/exp_backup_controller.page.dart @@ -95,55 +95,34 @@ class ExpBackupPage extends HookConsumerWidget { [backupState.backupProgress], ); - void startBackup() { - ref.watch(errorBackupListProvider.notifier).empty(); - if (ref.watch(backupProvider).backupProgress != - BackUpProgressEnum.inBackground) { - ref.watch(backupProvider.notifier).startBackupProcess(); - } - } - - Widget buildBackupButton() { + Widget buildControlButtons() { return Padding( padding: const EdgeInsets.only( top: 24, ), - child: Container( - child: backupState.backupProgress == BackUpProgressEnum.inProgress || - backupState.backupProgress == - BackUpProgressEnum.manualInProgress - ? ElevatedButton( - style: ElevatedButton.styleFrom( - foregroundColor: Colors.grey[50], - backgroundColor: Colors.red[300], - // padding: const EdgeInsets.all(14), - ), - onPressed: () { - if (backupState.backupProgress == - BackUpProgressEnum.manualInProgress) { - ref.read(manualUploadProvider.notifier).cancelBackup(); - } else { - ref.read(backupProvider.notifier).cancelBackup(); - } - }, - child: const Text( - "cancel", - style: TextStyle( - fontSize: 14, - fontWeight: FontWeight.bold, - ), - ).tr(), - ) - : ElevatedButton( - onPressed: shouldBackup ? startBackup : null, - child: const Text( - "backup_controller_page_start_backup", - style: TextStyle( - fontSize: 16, - fontWeight: FontWeight.bold, - ), - ).tr(), + child: Column( + children: [ + ElevatedButton( + onPressed: () => ref.read(expBackupProvider.notifier).backup(), + child: const Text( + "backup_controller_page_start_backup", + style: TextStyle( + fontSize: 16, + fontWeight: FontWeight.bold, ), + ).tr(), + ), + OutlinedButton( + onPressed: () => ref.read(expBackupProvider.notifier).cancel(), + child: const Text( + "cancel", + style: TextStyle( + fontSize: 16, + fontWeight: FontWeight.bold, + ), + ).tr(), + ), + ], ), ); } @@ -214,11 +193,11 @@ class ExpBackupPage extends HookConsumerWidget { const SizedBox(height: 8), const BackupAlbumSelectionCard(), const TotalCard(), + const BackupCard(), const RemainderCard(), const Divider(), + buildControlButtons(), const CurrentUploadingAssetInfoBox(), - if (!hasExclusiveAccess) buildBackgroundBackupInfo(), - buildBackupButton(), ] : [ const BackupAlbumSelectionCard(), @@ -369,18 +348,33 @@ class TotalCard extends ConsumerWidget { } } +class BackupCard extends ConsumerWidget { + const BackupCard({super.key}); + + @override + Widget build(BuildContext context, WidgetRef ref) { + final backupCount = + ref.watch(expBackupProvider.select((p) => p.backupCount)); + + return BackupInfoCard( + title: "backup_controller_page_backup".tr(), + subtitle: "backup_controller_page_backup_sub".tr(), + info: backupCount.toString(), + ); + } +} + class RemainderCard extends ConsumerWidget { const RemainderCard({super.key}); @override Widget build(BuildContext context, WidgetRef ref) { - final backupState = ref.watch(backupProvider); + final remainderCount = + ref.watch(expBackupProvider.select((p) => p.remainderCount)); return BackupInfoCard( title: "backup_controller_page_remainder".tr(), subtitle: "backup_controller_page_remainder_sub".tr(), - info: backupState.availableAlbums.isEmpty - ? "..." - : "${max(0, backupState.allUniqueAssets.length - backupState.selectedAlbumsBackupAssetsIds.length)}", + info: remainderCount.toString(), ); } } diff --git a/mobile/lib/providers/backup/exp_backup.provider.dart b/mobile/lib/providers/backup/exp_backup.provider.dart index 080317d0d5..b07a606d45 100644 --- a/mobile/lib/providers/backup/exp_backup.provider.dart +++ b/mobile/lib/providers/backup/exp_backup.provider.dart @@ -1,33 +1,52 @@ // ignore_for_file: public_member_api_docs, sort_constructors_first import 'dart:convert'; +import 'package:background_downloader/background_downloader.dart'; +import 'package:flutter/cupertino.dart'; import 'package:hooks_riverpod/hooks_riverpod.dart'; +import 'package:immich_mobile/constants/constants.dart'; +import 'package:immich_mobile/domain/utils/background_sync.dart'; +import 'package:immich_mobile/providers/background_sync.provider.dart'; import 'package:immich_mobile/services/exp_backup.service.dart'; +import 'package:immich_mobile/services/upload.service.dart'; class ExpBackupState { final int totalCount; + final int backupCount; + final int remainderCount; + ExpBackupState({ required this.totalCount, + required this.backupCount, + required this.remainderCount, }); ExpBackupState copyWith({ int? totalCount, + int? backupCount, + int? remainderCount, }) { return ExpBackupState( totalCount: totalCount ?? this.totalCount, + backupCount: backupCount ?? this.backupCount, + remainderCount: remainderCount ?? this.remainderCount, ); } Map toMap() { return { 'totalCount': totalCount, + 'backupCount': backupCount, + 'remainderCount': remainderCount, }; } factory ExpBackupState.fromMap(Map map) { return ExpBackupState( totalCount: map['totalCount'] as int, + backupCount: map['backupCount'] as int, + remainderCount: map['remainderCount'] as int, ); } @@ -37,37 +56,103 @@ class ExpBackupState { ExpBackupState.fromMap(json.decode(source) as Map); @override - String toString() => 'ExpBackupState(totalCount: $totalCount)'; + String toString() => + 'ExpBackupState(totalCount: $totalCount, backupCount: $backupCount, remainderCount: $remainderCount)'; @override bool operator ==(covariant ExpBackupState other) { if (identical(this, other)) return true; - return other.totalCount == totalCount; + return other.totalCount == totalCount && + other.backupCount == backupCount && + other.remainderCount == remainderCount; } @override - int get hashCode => totalCount.hashCode; + int get hashCode => + totalCount.hashCode ^ backupCount.hashCode ^ remainderCount.hashCode; } final expBackupProvider = StateNotifierProvider((ref) { - return ExpBackupNotifier(ref.watch(expBackupServiceProvider)); + return ExpBackupNotifier( + ref.watch(expBackupServiceProvider), + ref.watch(uploadServiceProvider), + ref.watch(backgroundSyncProvider), + ); }); class ExpBackupNotifier extends StateNotifier { - ExpBackupNotifier(this._backupService) - : super( + ExpBackupNotifier( + this._backupService, + this._uploadService, + this._backgroundSyncManager, + ) : super( ExpBackupState( totalCount: 0, + backupCount: 0, + remainderCount: 0, ), - ); + ) { + { + _uploadService.onUploadStatus = _uploadStatusCallback; + _uploadService.onTaskProgress = _taskProgressCallback; + } + } final ExpBackupService _backupService; + final UploadService _uploadService; + final BackgroundSyncManager _backgroundSyncManager; + + void _updateUploadStatus(TaskStatusUpdate task, TaskStatus status) async { + if (status == TaskStatus.canceled) { + return; + } + } + + void _uploadStatusCallback(TaskStatusUpdate update) { + _updateUploadStatus(update, update.status); + + switch (update.status) { + case TaskStatus.complete: + state = state.copyWith( + backupCount: state.backupCount + 1, + remainderCount: state.remainderCount - 1, + ); + + // TODO: find a better place to call this. + _backgroundSyncManager.syncRemote(); + break; + + default: + break; + } + } + + void _taskProgressCallback(TaskProgressUpdate update) { + debugPrint("[_taskProgressCallback] $update"); + } Future getBackupStatus() async { - final totalCount = await _backupService.getTotalCount(); + final [totalCount, backupCount, remainderCount] = await Future.wait([ + _backupService.getTotalCount(), + _backupService.getBackupCount(), + _backupService.getRemainderCount(), + ]); - state = state.copyWith(totalCount: totalCount); + state = state.copyWith( + totalCount: totalCount, + backupCount: backupCount, + remainderCount: remainderCount, + ); + } + + Future backup() async { + await _backupService.backup(); + } + + Future cancel() async { + await _uploadService.cancel(); + debugPrint("Cancel uploads"); } } diff --git a/mobile/lib/repositories/upload.repository.dart b/mobile/lib/repositories/upload.repository.dart index b08c107ed3..0278ba5137 100644 --- a/mobile/lib/repositories/upload.repository.dart +++ b/mobile/lib/repositories/upload.repository.dart @@ -33,7 +33,7 @@ class UploadRepository implements IUploadRepository { @override Future deleteAllTrackingRecords() { - return FileDownloader().database.deleteAllRecords(); + return FileDownloader().database.deleteAllRecords(group: kUploadGroup); } @override @@ -42,8 +42,9 @@ class UploadRepository implements IUploadRepository { } @override - void cancelAll() { - return taskQueue.removeAll(); + Future cancelAll() { + taskQueue.removeTasksWithGroup(kUploadGroup); + return FileDownloader().cancelAll(group: kUploadGroup); } @override diff --git a/mobile/lib/services/backup.service.dart b/mobile/lib/services/backup.service.dart index 3678c573b3..67b719bd12 100644 --- a/mobile/lib/services/backup.service.dart +++ b/mobile/lib/services/backup.service.dart @@ -298,7 +298,7 @@ class BackupService { print("Uploading ${uploadTasks.length} assets"); - _uploadService.upload(uploadTasks); + _uploadService.enqueueTasks(uploadTasks); } Future backupAsset( diff --git a/mobile/lib/services/exp_backup.service.dart b/mobile/lib/services/exp_backup.service.dart index c551badb96..e795ab9da9 100644 --- a/mobile/lib/services/exp_backup.service.dart +++ b/mobile/lib/services/exp_backup.service.dart @@ -1,31 +1,87 @@ +import 'package:background_downloader/background_downloader.dart'; +import 'package:flutter/material.dart'; import 'package:immich_mobile/domain/interfaces/backup.interface.dart'; -import 'package:immich_mobile/domain/interfaces/local_album.interface.dart'; -import 'package:immich_mobile/domain/models/local_album.model.dart'; +import 'package:immich_mobile/domain/interfaces/storage.interface.dart'; +import 'package:immich_mobile/domain/models/asset/base_asset.model.dart'; import 'package:immich_mobile/infrastructure/repositories/backup.repository.dart'; -import 'package:immich_mobile/providers/infrastructure/album.provider.dart'; +import 'package:immich_mobile/providers/infrastructure/storage.provider.dart'; +import 'package:immich_mobile/services/upload.service.dart'; import 'package:riverpod_annotation/riverpod_annotation.dart'; final expBackupServiceProvider = Provider( (ref) => ExpBackupService( ref.watch(backupRepositoryProvider), + ref.watch(storageRepositoryProvider), + ref.watch(uploadServiceProvider), ), ); class ExpBackupService { - ExpBackupService(this._backupRepository); + ExpBackupService( + this._backupRepository, + this._storageRepository, + this._uploadService, + ); final IBackupRepository _backupRepository; + final IStorageRepository _storageRepository; + final UploadService _uploadService; - Future getTotalCount() async { - final [selectedCount, excludedCount] = await Future.wait([ - _backupRepository.getTotalCount(BackupSelection.selected), - _backupRepository.getTotalCount(BackupSelection.excluded), - ]); + Future getTotalCount() { + return _backupRepository.getTotalCount(); + } - return selectedCount - excludedCount; + Future getRemainderCount() { + return _backupRepository.getRemainderCount(); } Future getBackupCount() { return _backupRepository.getBackupCount(); } + + Future backup() async { + final candidates = await _backupRepository.getCandidates(); + if (candidates.isEmpty) { + return; + } + + const batchSize = 5; + for (int i = 0; i < candidates.length; i += batchSize) { + final batch = candidates.skip(i).take(batchSize).toList(); + + List tasks = []; + for (final asset in batch) { + final task = await _getUploadTask(asset); + if (task != null) { + tasks.add(task); + } + } + + if (tasks.isNotEmpty) { + _uploadService.enqueueTasks(tasks); + } + } + } + + Future _getUploadTask(LocalAsset asset) async { + final entity = await _storageRepository.getAssetEntityForAsset(asset); + if (entity == null) { + return null; + } + + final file = await _storageRepository.getFileForAsset(asset); + if (file == null) { + return null; + } + + return _uploadService.buildUploadTask( + file, + originalFileName: asset.name, + deviceAssetId: asset.id, + ); + } + + Future cancel() async { + await _uploadService.cancel(); + } } diff --git a/mobile/lib/services/upload.service.dart b/mobile/lib/services/upload.service.dart index 4f5673dee9..02d39763b6 100644 --- a/mobile/lib/services/upload.service.dart +++ b/mobile/lib/services/upload.service.dart @@ -42,18 +42,15 @@ class UploadService { return FileDownloader().cancelTaskWithId(id); } - void cancelAllUpload() { - return _uploadRepository.cancelAll(); + Future cancel() async { + await _uploadRepository.cancelAll(); + await _uploadRepository.deleteAllTrackingRecords(); } - Future pauseAllUploads() { + Future pause() { return _uploadRepository.pauseAll(); } - Future deleteAllUploadTasks() { - return _uploadRepository.deleteAllTrackingRecords(); - } - Future> getRecords() async { final all = await _uploadRepository.getRecords(); print('all record: all: ${all.length} records found'); @@ -64,7 +61,7 @@ class UploadService { return all; } - void upload(List tasks) { + void enqueueTasks(List tasks) { _uploadRepository.enqueueAll(tasks); } @@ -111,6 +108,7 @@ class UploadService { return UploadTask( taskId: id, + displayName: filename, httpRequestMethod: 'POST', url: url, headers: headers,