chore: refactor upload service (#20130)

* chore: refactor upload service

* fix: cancel upload queue on logout (#20131)

* fix: cancel upload on logout

* fix: test

---------

Co-authored-by: shenlong-tanwen <139912620+shalong-tanwen@users.noreply.github.com>
Co-authored-by: Alex <alex.tran1502@gmail.com>

---------

Co-authored-by: shenlong <139912620+shenlong-tanwen@users.noreply.github.com>
Co-authored-by: shenlong-tanwen <139912620+shalong-tanwen@users.noreply.github.com>

mobile/lib/main.dart

@@ -97,7 +97,7 @@ Future<void> initApp() async {
 
   await FileDownloader().configure(
     // maxConcurrent: 6, maxConcurrentByHost(server):6, maxConcurrentByGroup: 3
-    globalConfig: (Config.holdingQueue, (6, 6, 3)),
+    globalConfig: (Config.holdingQueue, (1000, 1000, 1000)),
   );
 
   await FileDownloader().trackTasksInGroup(

mobile/lib/pages/backup/drift_backup.page.dart

@@ -40,7 +40,7 @@ class _DriftBackupPageState extends ConsumerState<DriftBackupPage> {
     }
 
     await ref.read(driftBackupProvider.notifier).getBackupStatus(currentUser.id);
-    await ref.read(driftBackupProvider.notifier).backup(currentUser.id);
+    await ref.read(driftBackupProvider.notifier).startBackup(currentUser.id);
   }
 
   Future<void> stopBackup() async {

mobile/lib/pages/backup/drift_backup_album_selection.page.dart

@@ -101,7 +101,7 @@ class _DriftBackupAlbumSelectionPageState extends ConsumerState<DriftBackupAlbum
             final backupNotifier = ref.read(driftBackupProvider.notifier);
 
             backupNotifier.cancel().then((_) {
-              backupNotifier.backup(currentUser.id);
+              backupNotifier.startBackup(currentUser.id);
             });
           }
         }

mobile/lib/providers/asset_viewer/share_intent_upload.provider.dart

@@ -27,8 +27,8 @@ class ShareIntentUploadStateNotifier extends StateNotifier<List<ShareIntentAttac
     this._uploadService,
     this._shareIntentService,
   ) : super([]) {
-    _uploadService.onUploadStatus = _updateUploadStatus;
+    _uploadService.taskStatusStream.listen(_updateUploadStatus);
-    _uploadService.onTaskProgress = _taskProgressCallback;
+    _uploadService.taskProgressStream.listen(_taskProgressCallback);
   }
 
   void init() {

mobile/lib/providers/auth.provider.dart

@@ -13,6 +13,7 @@ import 'package:immich_mobile/providers/infrastructure/user.provider.dart';
 import 'package:immich_mobile/services/api.service.dart';
 import 'package:immich_mobile/services/auth.service.dart';
 import 'package:immich_mobile/services/secure_storage.service.dart';
+import 'package:immich_mobile/services/upload.service.dart';
 import 'package:immich_mobile/services/widget.service.dart';
 import 'package:immich_mobile/utils/hash.dart';
 import 'package:logging/logging.dart';
@@ -23,6 +24,7 @@ final authProvider = StateNotifierProvider<AuthNotifier, AuthState>((ref) {
     ref.watch(authServiceProvider),
     ref.watch(apiServiceProvider),
     ref.watch(userServiceProvider),
+    ref.watch(uploadServiceProvider),
     ref.watch(secureStorageServiceProvider),
     ref.watch(widgetServiceProvider),
   );
@@ -32,6 +34,7 @@ class AuthNotifier extends StateNotifier<AuthState> {
   final AuthService _authService;
   final ApiService _apiService;
   final UserService _userService;
+  final UploadService _uploadService;
   final SecureStorageService _secureStorageService;
   final WidgetService _widgetService;
   final _log = Logger("AuthenticationNotifier");
@@ -42,6 +45,7 @@ class AuthNotifier extends StateNotifier<AuthState> {
     this._authService,
     this._apiService,
     this._userService,
+    this._uploadService,
     this._secureStorageService,
     this._widgetService,
   ) : super(
@@ -83,6 +87,7 @@ class AuthNotifier extends StateNotifier<AuthState> {
       await _widgetService.clearCredentials();
 
       await _authService.logout();
+      await _uploadService.cancelBackup();
     } finally {
       await _cleanUp();
     }

mobile/lib/providers/backup/drift_backup.provider.dart

@@ -8,7 +8,6 @@ import 'package:flutter/widgets.dart';
 import 'package:hooks_riverpod/hooks_riverpod.dart';
 
 import 'package:immich_mobile/constants/constants.dart';
-import 'package:immich_mobile/services/drift_backup.service.dart';
 import 'package:immich_mobile/services/upload.service.dart';
 
 class EnqueueStatus {
@@ -199,14 +198,12 @@ class DriftBackupState {
 
 final driftBackupProvider = StateNotifierProvider<ExpBackupNotifier, DriftBackupState>((ref) {
   return ExpBackupNotifier(
-    ref.watch(driftBackupServiceProvider),
     ref.watch(uploadServiceProvider),
   );
 });
 
 class ExpBackupNotifier extends StateNotifier<DriftBackupState> {
   ExpBackupNotifier(
-    this._backupService,
     this._uploadService,
   ) : super(
           const DriftBackupState(
@@ -225,7 +222,6 @@ class ExpBackupNotifier extends StateNotifier<DriftBackupState> {
     }
   }
 
-  final DriftBackupService _backupService;
   final UploadService _uploadService;
   StreamSubscription<TaskStatusUpdate>? _statusSubscription;
   StreamSubscription<TaskProgressUpdate>? _progressSubscription;
@@ -328,9 +324,9 @@ class ExpBackupNotifier extends StateNotifier<DriftBackupState> {
 
   Future<void> getBackupStatus(String userId) async {
     final [totalCount, backupCount, remainderCount] = await Future.wait([
-      _backupService.getTotalCount(),
+      _uploadService.getBackupTotalCount(),
-      _backupService.getBackupCount(userId),
+      _uploadService.getBackupFinishedCount(userId),
-      _backupService.getRemainderCount(userId),
+      _uploadService.getBackupRemainderCount(userId),
     ]);
 
     state = state.copyWith(
@@ -340,8 +336,8 @@ class ExpBackupNotifier extends StateNotifier<DriftBackupState> {
     );
   }
 
-  Future<void> backup(String userId) {
+  Future<void> startBackup(String userId) {
-    return _backupService.backup(userId, _updateEnqueueCount);
+    return _uploadService.startBackup(userId, _updateEnqueueCount);
   }
 
   void _updateEnqueueCount(EnqueueStatus status) {
@@ -352,22 +348,22 @@ class ExpBackupNotifier extends StateNotifier<DriftBackupState> {
   }
 
   Future<void> cancel() async {
+    debugPrint("Canceling backup tasks...");
     state = state.copyWith(
       enqueueCount: 0,
       enqueueTotalCount: 0,
       isCanceling: true,
     );
 
-    await _backupService.cancel();
+    final activeTaskCount = await _uploadService.cancelBackup();
 
-    // Check if there are any tasks left in the queue
+    if (activeTaskCount > 0) {
-    final tasks = await FileDownloader().allTasks(group: kBackupGroup);
+      debugPrint(
-
+        "$activeTaskCount tasks left, continuing to cancel...",
-    debugPrint("Tasks left to cancel: ${tasks.length}");
+      );
-
-    if (tasks.isNotEmpty) {
       await cancel();
     } else {
+      debugPrint("All tasks canceled successfully.");
       // Clear all upload items when cancellation is complete
       state = state.copyWith(
         isCanceling: false,
@@ -377,14 +373,18 @@ class ExpBackupNotifier extends StateNotifier<DriftBackupState> {
   }
 
   Future<void> handleBackupResume(String userId) async {
-    final tasks = await FileDownloader().allTasks(group: kBackupGroup);
+    debugPrint("handleBackupResume");
+    final tasks = await _uploadService.getActiveTasks(kBackupGroup);
+    debugPrint("Found ${tasks.length} tasks");
+
     if (tasks.isEmpty) {
       // Start a new backup queue
-      await backup(userId);
+      debugPrint("Start a new backup queue");
+      await startBackup(userId);
     }
 
     debugPrint("Tasks to resume: ${tasks.length}");
-    await FileDownloader().start();
+    await _uploadService.resumeBackup();
   }
 
   @override

mobile/lib/providers/infrastructure/action.provider.dart

@@ -5,8 +5,8 @@ import 'package:immich_mobile/providers/infrastructure/asset_viewer/current_asse
 import 'package:immich_mobile/providers/timeline/multiselect.provider.dart';
 import 'package:immich_mobile/providers/user.provider.dart';
 import 'package:immich_mobile/services/action.service.dart';
-import 'package:immich_mobile/services/drift_backup.service.dart';
 import 'package:immich_mobile/services/timeline.service.dart';
+import 'package:immich_mobile/services/upload.service.dart';
 import 'package:logging/logging.dart';
 import 'package:riverpod_annotation/riverpod_annotation.dart';
 
@@ -32,14 +32,14 @@ class ActionResult {
 class ActionNotifier extends Notifier<void> {
   final Logger _logger = Logger('ActionNotifier');
   late ActionService _service;
-  late DriftBackupService _backupService;
+  late UploadService _uploadService;
 
   ActionNotifier() : super();
 
   @override
   void build() {
+    _uploadService = ref.watch(uploadServiceProvider);
     _service = ref.watch(actionServiceProvider);
-    _backupService = ref.watch(driftBackupServiceProvider);
   }
 
   List<String> _getRemoteIdsForSource(ActionSource source) {
@@ -366,7 +366,7 @@ class ActionNotifier extends Notifier<void> {
   Future<ActionResult> upload(ActionSource source) async {
     final assets = _getAssets(source).whereType<LocalAsset>().toList();
     try {
-      await _backupService.manualBackup(assets);
+      await _uploadService.manualBackup(assets);
       return ActionResult(count: assets.length, success: true);
     } catch (error, stack) {
       _logger.severe('Failed manually upload assets', error, stack);

mobile/lib/repositories/upload.repository.dart

@@ -1,4 +1,5 @@
 import 'package:background_downloader/background_downloader.dart';
+import 'package:flutter/material.dart';
 import 'package:hooks_riverpod/hooks_riverpod.dart';
 import 'package:immich_mobile/constants/constants.dart';
 
@@ -6,7 +7,6 @@ final uploadRepositoryProvider = Provider((ref) => UploadRepository());
 
 class UploadRepository {
   void Function(TaskStatusUpdate)? onUploadStatus;
-
   void Function(TaskProgressUpdate)? onTaskProgress;
 
   UploadRepository() {
@@ -27,11 +27,11 @@ class UploadRepository {
     );
   }
 
-  void enqueueAll(List<UploadTask> tasks) {
+  void enqueueBackgroundAll(List<UploadTask> tasks) {
     FileDownloader().enqueueAll(tasks);
   }
 
-  Future<void> deleteAllTrackingRecords(String group) {
+  Future<void> deleteDatabaseRecords(String group) {
     return FileDownloader().database.deleteAllRecords(group: group);
   }
 
@@ -42,4 +42,47 @@ class UploadRepository {
   Future<int> reset(String group) {
     return FileDownloader().reset(group: group);
   }
+
+  /// Get a list of tasks that are ENQUEUED or RUNNING
+  Future<List<Task>> getActiveTasks(String group) {
+    return FileDownloader().allTasks(group: group);
+  }
+
+  Future<void> start() {
+    return FileDownloader().start();
+  }
+
+  Future<void> getUploadInfo() async {
+    final [enqueuedTasks, runningTasks, canceledTasks, waitingTasks, pausedTasks] = await Future.wait([
+      FileDownloader().database.allRecordsWithStatus(
+            TaskStatus.enqueued,
+            group: kBackupGroup,
+          ),
+      FileDownloader().database.allRecordsWithStatus(
+            TaskStatus.running,
+            group: kBackupGroup,
+          ),
+      FileDownloader().database.allRecordsWithStatus(
+            TaskStatus.canceled,
+            group: kBackupGroup,
+          ),
+      FileDownloader().database.allRecordsWithStatus(
+            TaskStatus.waitingToRetry,
+            group: kBackupGroup,
+          ),
+      FileDownloader().database.allRecordsWithStatus(
+            TaskStatus.paused,
+            group: kBackupGroup,
+          ),
+    ]);
+
+    debugPrint("""
+      Upload Info:
+      Enqueued: ${enqueuedTasks.length}
+      Running: ${runningTasks.length}
+      Canceled: ${canceledTasks.length}
+      Waiting: ${waitingTasks.length}
+      Paused: ${pausedTasks.length}
+    """);
+  }
 }

mobile/lib/services/auth.service.dart

@@ -8,10 +8,12 @@ import 'package:immich_mobile/entities/store.entity.dart';
 import 'package:immich_mobile/models/auth/auxilary_endpoint.model.dart';
 import 'package:immich_mobile/models/auth/login_response.model.dart';
 import 'package:immich_mobile/providers/api.provider.dart';
+import 'package:immich_mobile/providers/app_settings.provider.dart';
 import 'package:immich_mobile/providers/background_sync.provider.dart';
 import 'package:immich_mobile/repositories/auth.repository.dart';
 import 'package:immich_mobile/repositories/auth_api.repository.dart';
 import 'package:immich_mobile/services/api.service.dart';
+import 'package:immich_mobile/services/app_settings.service.dart';
 import 'package:immich_mobile/services/network.service.dart';
 import 'package:logging/logging.dart';
 import 'package:openapi/api.dart';
@@ -23,6 +25,7 @@ final authServiceProvider = Provider(
     ref.watch(apiServiceProvider),
     ref.watch(networkServiceProvider),
     ref.watch(backgroundSyncProvider),
+    ref.watch(appSettingsServiceProvider),
   ),
 );
 
@@ -32,7 +35,7 @@ class AuthService {
   final ApiService _apiService;
   final NetworkService _networkService;
   final BackgroundSyncManager _backgroundSyncManager;
-
+  final AppSettingsService _appSettingsService;
   final _log = Logger("AuthService");
 
   AuthService(
@@ -41,6 +44,7 @@ class AuthService {
     this._apiService,
     this._networkService,
     this._backgroundSyncManager,
+    this._appSettingsService,
   );
 
   /// Validates the provided server URL by resolving and setting the endpoint.
@@ -106,6 +110,11 @@ class AuthService {
       await clearLocalData().catchError((error, stackTrace) {
         _log.severe("Error clearing local data", error, stackTrace);
       });
+
+      await _appSettingsService.setSetting(
+        AppSettingsEnum.enableBackup,
+        false,
+      );
     }
   }
 

mobile/lib/services/drift_backup.service.dart

@@ -1,316 +0,0 @@
-import 'dart:async';
-import 'dart:convert';
-import 'dart:io';
-
-import 'package:background_downloader/background_downloader.dart';
-import 'package:flutter/material.dart';
-import 'package:immich_mobile/constants/constants.dart';
-import 'package:immich_mobile/domain/models/asset/base_asset.model.dart';
-import 'package:immich_mobile/infrastructure/repositories/backup.repository.dart';
-import 'package:immich_mobile/infrastructure/repositories/local_asset.repository.dart';
-import 'package:immich_mobile/infrastructure/repositories/storage.repository.dart';
-import 'package:immich_mobile/providers/backup/drift_backup.provider.dart';
-import 'package:immich_mobile/providers/infrastructure/asset.provider.dart';
-import 'package:immich_mobile/providers/infrastructure/storage.provider.dart';
-import 'package:immich_mobile/services/upload.service.dart';
-import 'package:logging/logging.dart';
-import 'package:path/path.dart' as p;
-import 'package:riverpod_annotation/riverpod_annotation.dart';
-
-final driftBackupServiceProvider = Provider<DriftBackupService>(
-  (ref) => DriftBackupService(
-    ref.watch(backupRepositoryProvider),
-    ref.watch(storageRepositoryProvider),
-    ref.watch(uploadServiceProvider),
-    ref.watch(localAssetRepository),
-  ),
-);
-
-// TODO: Rename to UploadService after removing Isar
-class DriftBackupService {
-  DriftBackupService(
-    this._backupRepository,
-    this._storageRepository,
-    this._uploadService,
-    this._localAssetRepository,
-  ) {
-    _uploadService.taskStatusStream.listen(_handleTaskStatusUpdate);
-  }
-
-  final DriftBackupRepository _backupRepository;
-  final StorageRepository _storageRepository;
-  final DriftLocalAssetRepository _localAssetRepository;
-  final UploadService _uploadService;
-  final _log = Logger("DriftBackupService");
-
-  bool shouldCancel = false;
-
-  Future<int> getTotalCount() {
-    return _backupRepository.getTotalCount();
-  }
-
-  Future<int> getRemainderCount(String userId) {
-    return _backupRepository.getRemainderCount(userId);
-  }
-
-  Future<int> getBackupCount(String userId) {
-    return _backupRepository.getBackupCount(userId);
-  }
-
-  Future<void> manualBackup(List<LocalAsset> localAssets) async {
-    List<UploadTask> tasks = [];
-    for (final asset in localAssets) {
-      final task = await _getUploadTask(
-        asset,
-        group: kManualUploadGroup,
-        priority: 1, // High priority after upload motion photo part
-      );
-      if (task != null) {
-        tasks.add(task);
-      }
-    }
-
-    if (tasks.isNotEmpty) {
-      _uploadService.enqueueTasks(tasks);
-    }
-  }
-
-  Future<void> backup(
-    String userId,
-    void Function(EnqueueStatus status) onEnqueueTasks,
-  ) async {
-    shouldCancel = false;
-
-    final candidates = await _backupRepository.getCandidates(userId);
-    if (candidates.isEmpty) {
-      return;
-    }
-
-    const batchSize = 100;
-    int count = 0;
-    for (int i = 0; i < candidates.length; i += batchSize) {
-      if (shouldCancel) {
-        break;
-      }
-
-      final batch = candidates.skip(i).take(batchSize).toList();
-
-      List<UploadTask> tasks = [];
-      for (final asset in batch) {
-        final task = await _getUploadTask(asset);
-        if (task != null) {
-          tasks.add(task);
-        }
-      }
-
-      if (tasks.isNotEmpty && !shouldCancel) {
-        count += tasks.length;
-        _uploadService.enqueueTasks(tasks);
-
-        onEnqueueTasks(
-          EnqueueStatus(
-            enqueueCount: count,
-            totalCount: candidates.length,
-          ),
-        );
-      }
-    }
-  }
-
-  void _handleTaskStatusUpdate(TaskStatusUpdate update) {
-    switch (update.status) {
-      case TaskStatus.complete:
-        _handleLivePhoto(update);
-        break;
-
-      default:
-        break;
-    }
-  }
-
-  Future<void> _handleLivePhoto(TaskStatusUpdate update) async {
-    try {
-      if (update.task.metaData.isEmpty || update.task.metaData == '') {
-        return;
-      }
-
-      final metadata = UploadTaskMetadata.fromJson(update.task.metaData);
-      if (!metadata.isLivePhotos) {
-        return;
-      }
-
-      if (update.responseBody == null || update.responseBody!.isEmpty) {
-        return;
-      }
-      final response = jsonDecode(update.responseBody!);
-
-      final localAsset = await _localAssetRepository.getById(metadata.localAssetId);
-      if (localAsset == null) {
-        return;
-      }
-
-      final uploadTask = await _getLivePhotoUploadTask(
-        localAsset,
-        response['id'] as String,
-      );
-
-      if (uploadTask == null) {
-        return;
-      }
-
-      _uploadService.enqueueTasks([uploadTask]);
-    } catch (error, stackTrace) {
-      _log.severe("Error handling live photo upload task", error, stackTrace);
-      debugPrint("Error handling live photo upload task: $error $stackTrace");
-    }
-  }
-
-  Future<UploadTask?> _getUploadTask(
-    LocalAsset asset, {
-    String group = kBackupGroup,
-    int? priority,
-  }) async {
-    final entity = await _storageRepository.getAssetEntityForAsset(asset);
-    if (entity == null) {
-      return null;
-    }
-
-    File? file;
-
-    /// iOS LivePhoto has two files: a photo and a video.
-    /// They are uploaded separately, with video file being upload first, then returned with the assetId
-    /// The assetId is then used as a metadata for the photo file upload task.
-    ///
-    /// We implement two separate upload groups for this, the normal one for the video file
-    /// and the higher priority group for the photo file because the video file is already uploaded.
-    ///
-    /// The cancel operation will only cancel the video group (normal group), the photo group will not
-    /// be touched, as the video file is already uploaded.
-
-    if (entity.isLivePhoto) {
-      file = await _storageRepository.getMotionFileForAsset(asset);
-    } else {
-      file = await _storageRepository.getFileForAsset(asset.id);
-    }
-
-    if (file == null) {
-      return null;
-    }
-
-    final originalFileName = entity.isLivePhoto
-        ? p.setExtension(
-            asset.name,
-            p.extension(file.path),
-          )
-        : asset.name;
-
-    String metadata = UploadTaskMetadata(
-      localAssetId: asset.id,
-      isLivePhotos: entity.isLivePhoto,
-      livePhotoVideoId: '',
-    ).toJson();
-
-    return _uploadService.buildUploadTask(
-      file,
-      originalFileName: originalFileName,
-      deviceAssetId: asset.id,
-      metadata: metadata,
-      group: group,
-      priority: priority,
-    );
-  }
-
-  Future<UploadTask?> _getLivePhotoUploadTask(
-    LocalAsset asset,
-    String livePhotoVideoId,
-  ) async {
-    final entity = await _storageRepository.getAssetEntityForAsset(asset);
-    if (entity == null) {
-      return null;
-    }
-
-    final file = await _storageRepository.getFileForAsset(asset.id);
-    if (file == null) {
-      return null;
-    }
-
-    final fields = {
-      'livePhotoVideoId': livePhotoVideoId,
-    };
-
-    return _uploadService.buildUploadTask(
-      file,
-      originalFileName: asset.name,
-      deviceAssetId: asset.id,
-      fields: fields,
-      group: kBackupLivePhotoGroup,
-      priority: 0, // Highest priority to get upload immediately
-    );
-  }
-
-  Future<void> cancel() async {
-    shouldCancel = true;
-    await _uploadService.cancelAllForGroup(kBackupGroup);
-  }
-}
-
-class UploadTaskMetadata {
-  final String localAssetId;
-  final bool isLivePhotos;
-  final String livePhotoVideoId;
-
-  const UploadTaskMetadata({
-    required this.localAssetId,
-    required this.isLivePhotos,
-    required this.livePhotoVideoId,
-  });
-
-  UploadTaskMetadata copyWith({
-    String? localAssetId,
-    bool? isLivePhotos,
-    String? livePhotoVideoId,
-  }) {
-    return UploadTaskMetadata(
-      localAssetId: localAssetId ?? this.localAssetId,
-      isLivePhotos: isLivePhotos ?? this.isLivePhotos,
-      livePhotoVideoId: livePhotoVideoId ?? this.livePhotoVideoId,
-    );
-  }
-
-  Map<String, dynamic> toMap() {
-    return <String, dynamic>{
-      'localAssetId': localAssetId,
-      'isLivePhotos': isLivePhotos,
-      'livePhotoVideoId': livePhotoVideoId,
-    };
-  }
-
-  factory UploadTaskMetadata.fromMap(Map<String, dynamic> map) {
-    return UploadTaskMetadata(
-      localAssetId: map['localAssetId'] as String,
-      isLivePhotos: map['isLivePhotos'] as bool,
-      livePhotoVideoId: map['livePhotoVideoId'] as String,
-    );
-  }
-
-  String toJson() => json.encode(toMap());
-
-  factory UploadTaskMetadata.fromJson(String source) =>
-      UploadTaskMetadata.fromMap(json.decode(source) as Map<String, dynamic>);
-
-  @override
-  String toString() =>
-      'UploadTaskMetadata(localAssetId: $localAssetId, isLivePhotos: $isLivePhotos, livePhotoVideoId: $livePhotoVideoId)';
-
-  @override
-  bool operator ==(covariant UploadTaskMetadata other) {
-    if (identical(this, other)) return true;
-
-    return other.localAssetId == localAssetId &&
-        other.isLivePhotos == isLivePhotos &&
-        other.livePhotoVideoId == livePhotoVideoId;
-  }
-
-  @override
-  int get hashCode => localAssetId.hashCode ^ isLivePhotos.hashCode ^ livePhotoVideoId.hashCode;
-}

mobile/lib/services/upload.service.dart

@@ -1,24 +1,51 @@
 import 'dart:async';
+import 'dart:convert';
 import 'dart:io';
 
 import 'package:background_downloader/background_downloader.dart';
+import 'package:flutter/material.dart';
 import 'package:hooks_riverpod/hooks_riverpod.dart';
+import 'package:immich_mobile/constants/constants.dart';
+import 'package:immich_mobile/domain/models/asset/base_asset.model.dart';
 import 'package:immich_mobile/domain/models/store.model.dart';
 import 'package:immich_mobile/entities/store.entity.dart';
+import 'package:immich_mobile/infrastructure/repositories/backup.repository.dart';
+import 'package:immich_mobile/infrastructure/repositories/local_asset.repository.dart';
+import 'package:immich_mobile/infrastructure/repositories/storage.repository.dart';
+import 'package:immich_mobile/providers/backup/drift_backup.provider.dart';
+import 'package:immich_mobile/providers/infrastructure/asset.provider.dart';
+import 'package:immich_mobile/providers/infrastructure/storage.provider.dart';
 import 'package:immich_mobile/repositories/upload.repository.dart';
 import 'package:immich_mobile/services/api.service.dart';
-import 'package:path/path.dart';
+import 'package:path/path.dart' as p;
 
 final uploadServiceProvider = Provider((ref) {
-  final service = UploadService(ref.watch(uploadRepositoryProvider));
+  final service = UploadService(
+    ref.watch(uploadRepositoryProvider),
+    ref.watch(backupRepositoryProvider),
+    ref.watch(storageRepositoryProvider),
+    ref.watch(localAssetRepository),
+  );
+
   ref.onDispose(service.dispose);
   return service;
 });
 
 class UploadService {
+  UploadService(
+    this._uploadRepository,
+    this._backupRepository,
+    this._storageRepository,
+    this._localAssetRepository,
+  ) {
+    _uploadRepository.onUploadStatus = _onUploadCallback;
+    _uploadRepository.onTaskProgress = _onTaskProgressCallback;
+  }
+
   final UploadRepository _uploadRepository;
-  void Function(TaskStatusUpdate)? onUploadStatus;
+  final DriftBackupRepository _backupRepository;
-  void Function(TaskProgressUpdate)? onTaskProgress;
+  final StorageRepository _storageRepository;
+  final DriftLocalAssetRepository _localAssetRepository;
 
   final StreamController<TaskStatusUpdate> _taskStatusController = StreamController<TaskStatusUpdate>.broadcast();
   final StreamController<TaskProgressUpdate> _taskProgressController = StreamController<TaskProgressUpdate>.broadcast();
@@ -26,25 +53,19 @@ class UploadService {
   Stream<TaskStatusUpdate> get taskStatusStream => _taskStatusController.stream;
   Stream<TaskProgressUpdate> get taskProgressStream => _taskProgressController.stream;
 
-  UploadService(
+  bool shouldAbortQueuingTasks = false;
-    this._uploadRepository,
-  ) {
-    _uploadRepository.onUploadStatus = _onUploadCallback;
-    _uploadRepository.onTaskProgress = _onTaskProgressCallback;
-  }
 
   void _onTaskProgressCallback(TaskProgressUpdate update) {
-    onTaskProgress?.call(update);
     if (!_taskProgressController.isClosed) {
       _taskProgressController.add(update);
     }
   }
 
   void _onUploadCallback(TaskStatusUpdate update) {
-    onUploadStatus?.call(update);
     if (!_taskStatusController.isClosed) {
       _taskStatusController.add(update);
     }
+    _handleTaskStatusUpdate(update);
   }
 
   void dispose() {
@@ -52,18 +73,234 @@ class UploadService {
     _taskProgressController.close();
   }
 
-  Future<bool> cancelUpload(String id) {
-    return FileDownloader().cancelTaskWithId(id);
-  }
-
-  Future<void> cancelAllForGroup(String group) async {
-    await _uploadRepository.cancelAll(group);
-    await _uploadRepository.reset(group);
-    await _uploadRepository.deleteAllTrackingRecords(group);
-  }
-
   void enqueueTasks(List<UploadTask> tasks) {
-    _uploadRepository.enqueueAll(tasks);
+    _uploadRepository.enqueueBackgroundAll(tasks);
+  }
+
+  Future<List<Task>> getActiveTasks(String group) {
+    return _uploadRepository.getActiveTasks(group);
+  }
+
+  Future<int> getBackupTotalCount() {
+    return _backupRepository.getTotalCount();
+  }
+
+  Future<int> getBackupRemainderCount(String userId) {
+    return _backupRepository.getRemainderCount(userId);
+  }
+
+  Future<int> getBackupFinishedCount(String userId) {
+    return _backupRepository.getBackupCount(userId);
+  }
+
+  Future<void> manualBackup(List<LocalAsset> localAssets) async {
+    List<UploadTask> tasks = [];
+    for (final asset in localAssets) {
+      final task = await _getUploadTask(
+        asset,
+        group: kManualUploadGroup,
+        priority: 1, // High priority after upload motion photo part
+      );
+      if (task != null) {
+        tasks.add(task);
+      }
+    }
+
+    if (tasks.isNotEmpty) {
+      enqueueTasks(tasks);
+    }
+  }
+
+  /// Find backup candidates
+  /// Build the upload tasks
+  /// Enqueue the tasks
+  Future<void> startBackup(
+    String userId,
+    void Function(EnqueueStatus status) onEnqueueTasks,
+  ) async {
+    shouldAbortQueuingTasks = false;
+
+    final candidates = await _backupRepository.getCandidates(userId);
+    if (candidates.isEmpty) {
+      return;
+    }
+
+    const batchSize = 100;
+    int count = 0;
+    for (int i = 0; i < candidates.length; i += batchSize) {
+      if (shouldAbortQueuingTasks) {
+        break;
+      }
+
+      final batch = candidates.skip(i).take(batchSize).toList();
+
+      List<UploadTask> tasks = [];
+      for (final asset in batch) {
+        final task = await _getUploadTask(asset);
+        if (task != null) {
+          tasks.add(task);
+        }
+      }
+
+      if (tasks.isNotEmpty && !shouldAbortQueuingTasks) {
+        count += tasks.length;
+        enqueueTasks(tasks);
+
+        onEnqueueTasks(
+          EnqueueStatus(
+            enqueueCount: count,
+            totalCount: candidates.length,
+          ),
+        );
+      }
+    }
+  }
+
+  /// Cancel all ongoing uploads and reset the upload queue
+  ///
+  /// Return the number of left over tasks in the queue
+  Future<int> cancelBackup() async {
+    shouldAbortQueuingTasks = true;
+
+    await _uploadRepository.reset(kBackupGroup);
+    await _uploadRepository.deleteDatabaseRecords(kBackupGroup);
+
+    final activeTasks = await _uploadRepository.getActiveTasks(kBackupGroup);
+    return activeTasks.length;
+  }
+
+  Future<void> resumeBackup() {
+    return _uploadRepository.start();
+  }
+
+  void _handleTaskStatusUpdate(TaskStatusUpdate update) {
+    switch (update.status) {
+      case TaskStatus.complete:
+        _handleLivePhoto(update);
+        break;
+
+      default:
+        break;
+    }
+  }
+
+  Future<void> _handleLivePhoto(TaskStatusUpdate update) async {
+    try {
+      if (update.task.metaData.isEmpty || update.task.metaData == '') {
+        return;
+      }
+
+      final metadata = UploadTaskMetadata.fromJson(update.task.metaData);
+      if (!metadata.isLivePhotos) {
+        return;
+      }
+
+      if (update.responseBody == null || update.responseBody!.isEmpty) {
+        return;
+      }
+      final response = jsonDecode(update.responseBody!);
+
+      final localAsset = await _localAssetRepository.getById(metadata.localAssetId);
+      if (localAsset == null) {
+        return;
+      }
+
+      final uploadTask = await _getLivePhotoUploadTask(
+        localAsset,
+        response['id'] as String,
+      );
+
+      if (uploadTask == null) {
+        return;
+      }
+
+      enqueueTasks([uploadTask]);
+    } catch (error, stackTrace) {
+      debugPrint("Error handling live photo upload task: $error $stackTrace");
+    }
+  }
+
+  Future<UploadTask?> _getUploadTask(
+    LocalAsset asset, {
+    String group = kBackupGroup,
+    int? priority,
+  }) async {
+    final entity = await _storageRepository.getAssetEntityForAsset(asset);
+    if (entity == null) {
+      return null;
+    }
+
+    File? file;
+
+    /// iOS LivePhoto has two files: a photo and a video.
+    /// They are uploaded separately, with video file being upload first, then returned with the assetId
+    /// The assetId is then used as a metadata for the photo file upload task.
+    ///
+    /// We implement two separate upload groups for this, the normal one for the video file
+    /// and the higher priority group for the photo file because the video file is already uploaded.
+    ///
+    /// The cancel operation will only cancel the video group (normal group), the photo group will not
+    /// be touched, as the video file is already uploaded.
+
+    if (entity.isLivePhoto) {
+      file = await _storageRepository.getMotionFileForAsset(asset);
+    } else {
+      file = await _storageRepository.getFileForAsset(asset.id);
+    }
+
+    if (file == null) {
+      return null;
+    }
+
+    final originalFileName = entity.isLivePhoto
+        ? p.setExtension(
+            asset.name,
+            p.extension(file.path),
+          )
+        : asset.name;
+
+    String metadata = UploadTaskMetadata(
+      localAssetId: asset.id,
+      isLivePhotos: entity.isLivePhoto,
+      livePhotoVideoId: '',
+    ).toJson();
+
+    return buildUploadTask(
+      file,
+      originalFileName: originalFileName,
+      deviceAssetId: asset.id,
+      metadata: metadata,
+      group: group,
+      priority: priority,
+    );
+  }
+
+  Future<UploadTask?> _getLivePhotoUploadTask(
+    LocalAsset asset,
+    String livePhotoVideoId,
+  ) async {
+    final entity = await _storageRepository.getAssetEntityForAsset(asset);
+    if (entity == null) {
+      return null;
+    }
+
+    final file = await _storageRepository.getFileForAsset(asset.id);
+    if (file == null) {
+      return null;
+    }
+
+    final fields = {
+      'livePhotoVideoId': livePhotoVideoId,
+    };
+
+    return buildUploadTask(
+      file,
+      originalFileName: asset.name,
+      deviceAssetId: asset.id,
+      fields: fields,
+      group: kBackupLivePhotoGroup,
+      priority: 0, // Highest priority to get upload immediately
+    );
   }
 
   Future<UploadTask> buildUploadTask(
@@ -74,26 +311,6 @@ class UploadService {
     String? deviceAssetId,
     String? metadata,
     int? priority,
-  }) async {
-    return _buildTask(
-      deviceAssetId ?? hash(file.path).toString(),
-      file,
-      fields: fields,
-      originalFileName: originalFileName,
-      metadata: metadata,
-      group: group,
-      priority: priority,
-    );
-  }
-
-  Future<UploadTask> _buildTask(
-    String id,
-    File file, {
-    required String group,
-    Map<String, String>? fields,
-    String? originalFileName,
-    String? metadata,
-    int? priority,
   }) async {
     final serverEndpoint = Store.get(StoreKey.serverEndpoint);
     final url = Uri.parse('$serverEndpoint/assets').toString();
@@ -106,7 +323,7 @@ class UploadService {
     final fileModifiedAt = stats.modified;
     final fieldsMap = {
       'filename': originalFileName ?? filename,
-      'deviceAssetId': id,
+      'deviceAssetId': deviceAssetId ?? '',
       'deviceId': deviceId,
       'fileCreatedAt': fileCreatedAt.toUtc().toIso8601String(),
       'fileModifiedAt': fileModifiedAt.toUtc().toIso8601String(),
@@ -116,7 +333,7 @@ class UploadService {
     };
 
     return UploadTask(
-      taskId: id,
+      taskId: deviceAssetId,
       displayName: originalFileName ?? filename,
       httpRequestMethod: 'POST',
       url: url,
@@ -134,3 +351,64 @@ class UploadService {
     );
   }
 }
+
+class UploadTaskMetadata {
+  final String localAssetId;
+  final bool isLivePhotos;
+  final String livePhotoVideoId;
+
+  const UploadTaskMetadata({
+    required this.localAssetId,
+    required this.isLivePhotos,
+    required this.livePhotoVideoId,
+  });
+
+  UploadTaskMetadata copyWith({
+    String? localAssetId,
+    bool? isLivePhotos,
+    String? livePhotoVideoId,
+  }) {
+    return UploadTaskMetadata(
+      localAssetId: localAssetId ?? this.localAssetId,
+      isLivePhotos: isLivePhotos ?? this.isLivePhotos,
+      livePhotoVideoId: livePhotoVideoId ?? this.livePhotoVideoId,
+    );
+  }
+
+  Map<String, dynamic> toMap() {
+    return <String, dynamic>{
+      'localAssetId': localAssetId,
+      'isLivePhotos': isLivePhotos,
+      'livePhotoVideoId': livePhotoVideoId,
+    };
+  }
+
+  factory UploadTaskMetadata.fromMap(Map<String, dynamic> map) {
+    return UploadTaskMetadata(
+      localAssetId: map['localAssetId'] as String,
+      isLivePhotos: map['isLivePhotos'] as bool,
+      livePhotoVideoId: map['livePhotoVideoId'] as String,
+    );
+  }
+
+  String toJson() => json.encode(toMap());
+
+  factory UploadTaskMetadata.fromJson(String source) =>
+      UploadTaskMetadata.fromMap(json.decode(source) as Map<String, dynamic>);
+
+  @override
+  String toString() =>
+      'UploadTaskMetadata(localAssetId: $localAssetId, isLivePhotos: $isLivePhotos, livePhotoVideoId: $livePhotoVideoId)';
+
+  @override
+  bool operator ==(covariant UploadTaskMetadata other) {
+    if (identical(this, other)) return true;
+
+    return other.localAssetId == localAssetId &&
+        other.isLivePhotos == isLivePhotos &&
+        other.livePhotoVideoId == livePhotoVideoId;
+  }
+
+  @override
+  int get hashCode => localAssetId.hashCode ^ isLivePhotos.hashCode ^ livePhotoVideoId.hashCode;
+}

mobile/test/domain/service.mock.dart

@@ -2,6 +2,8 @@ import 'package:immich_mobile/domain/services/store.service.dart';
 import 'package:immich_mobile/domain/services/user.service.dart';
 import 'package:immich_mobile/domain/utils/background_sync.dart';
 import 'package:immich_mobile/platform/native_sync_api.g.dart';
+import 'package:immich_mobile/services/app_settings.service.dart';
+import 'package:immich_mobile/services/upload.service.dart';
 import 'package:mocktail/mocktail.dart';
 
 class MockStoreService extends Mock implements StoreService {}
@@ -11,3 +13,7 @@ class MockUserService extends Mock implements UserService {}
 class MockBackgroundSyncManager extends Mock implements BackgroundSyncManager {}
 
 class MockNativeSyncApi extends Mock implements NativeSyncApi {}
+
+class MockAppSettingsService extends Mock implements AppSettingsService {}
+
+class MockUploadService extends Mock implements UploadService {}

mobile/test/services/auth.service_test.dart

@@ -3,6 +3,7 @@ import 'package:flutter_test/flutter_test.dart';
 import 'package:immich_mobile/domain/services/store.service.dart';
 import 'package:immich_mobile/infrastructure/repositories/store.repository.dart';
 import 'package:immich_mobile/models/auth/auxilary_endpoint.model.dart';
+import 'package:immich_mobile/services/app_settings.service.dart';
 import 'package:immich_mobile/services/auth.service.dart';
 import 'package:isar/isar.dart';
 import 'package:mocktail/mocktail.dart';
@@ -20,6 +21,8 @@ void main() {
   late MockApiService apiService;
   late MockNetworkService networkService;
   late MockBackgroundSyncManager backgroundSyncManager;
+  late MockUploadService uploadService;
+  late MockAppSettingService appSettingsService;
   late Isar db;
 
   setUp(() async {
@@ -28,6 +31,8 @@ void main() {
     apiService = MockApiService();
     networkService = MockNetworkService();
     backgroundSyncManager = MockBackgroundSyncManager();
+    uploadService = MockUploadService();
+    appSettingsService = MockAppSettingService();
 
     sut = AuthService(
       authApiRepository,
@@ -35,6 +40,7 @@ void main() {
       apiService,
       networkService,
       backgroundSyncManager,
+      appSettingsService,
     );
 
     registerFallbackValue(Uri());
@@ -118,7 +124,13 @@ void main() {
       when(() => authApiRepository.logout()).thenAnswer((_) async => {});
       when(() => backgroundSyncManager.cancel()).thenAnswer((_) async => {});
       when(() => authRepository.clearLocalData()).thenAnswer((_) => Future.value(null));
-
+      when(() => uploadService.cancelBackup()).thenAnswer((_) => Future.value(1));
+      when(
+        () => appSettingsService.setSetting(
+          AppSettingsEnum.enableBackup,
+          false,
+        ),
+      ).thenAnswer((_) => Future.value(null));
       await sut.logout();
 
       verify(() => authApiRepository.logout()).called(1);
@@ -130,7 +142,13 @@ void main() {
       when(() => authApiRepository.logout()).thenThrow(Exception('Server error'));
       when(() => backgroundSyncManager.cancel()).thenAnswer((_) async => {});
       when(() => authRepository.clearLocalData()).thenAnswer((_) => Future.value(null));
-
+      when(() => uploadService.cancelBackup()).thenAnswer((_) => Future.value(1));
+      when(
+        () => appSettingsService.setSetting(
+          AppSettingsEnum.enableBackup,
+          false,
+        ),
+      ).thenAnswer((_) => Future.value(null));
       await sut.logout();
 
       verify(() => authApiRepository.logout()).called(1);