add job to check for offline files

mobile/openapi/doc/ScanLibraryDto.md

@@ -8,6 +8,7 @@ import 'package:openapi/api.dart';
 ## Properties
 Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
+**checkForOffline** | **bool** |  | [optional] 
 **refreshAllFiles** | **bool** |  | [optional] 
 **refreshModifiedFiles** | **bool** |  | [optional] 
 

mobile/openapi/lib/model/scan_library_dto.dart

@@ -13,10 +13,19 @@ part of openapi.api;
 class ScanLibraryDto {
   /// Returns a new [ScanLibraryDto] instance.
   ScanLibraryDto({
+    this.checkForOffline,
     this.refreshAllFiles,
     this.refreshModifiedFiles,
   });
 
+  ///
+  /// Please note: This property should have been non-nullable! Since the specification file
+  /// does not include a default value (using the "default:" property), however, the generated
+  /// source code must fall back to having a nullable type.
+  /// Consider adding a "default:" property in the specification file to hide this note.
+  ///
+  bool? checkForOffline;
+
   ///
   /// Please note: This property should have been non-nullable! Since the specification file
   /// does not include a default value (using the "default:" property), however, the generated
@@ -35,20 +44,27 @@ class ScanLibraryDto {
 
   @override
   bool operator ==(Object other) => identical(this, other) || other is ScanLibraryDto &&
+    other.checkForOffline == checkForOffline &&
     other.refreshAllFiles == refreshAllFiles &&
     other.refreshModifiedFiles == refreshModifiedFiles;
 
   @override
   int get hashCode =>
     // ignore: unnecessary_parenthesis
+    (checkForOffline == null ? 0 : checkForOffline!.hashCode) +
     (refreshAllFiles == null ? 0 : refreshAllFiles!.hashCode) +
     (refreshModifiedFiles == null ? 0 : refreshModifiedFiles!.hashCode);
 
   @override
-  String toString() => 'ScanLibraryDto[refreshAllFiles=$refreshAllFiles, refreshModifiedFiles=$refreshModifiedFiles]';
+  String toString() => 'ScanLibraryDto[checkForOffline=$checkForOffline, refreshAllFiles=$refreshAllFiles, refreshModifiedFiles=$refreshModifiedFiles]';
 
   Map<String, dynamic> toJson() {
     final json = <String, dynamic>{};
+    if (this.checkForOffline != null) {
+      json[r'checkForOffline'] = this.checkForOffline;
+    } else {
+    //  json[r'checkForOffline'] = null;
+    }
     if (this.refreshAllFiles != null) {
       json[r'refreshAllFiles'] = this.refreshAllFiles;
     } else {
@@ -70,6 +86,7 @@ class ScanLibraryDto {
       final json = value.cast<String, dynamic>();
 
       return ScanLibraryDto(
+        checkForOffline: mapValueOfType<bool>(json, r'checkForOffline'),
         refreshAllFiles: mapValueOfType<bool>(json, r'refreshAllFiles'),
         refreshModifiedFiles: mapValueOfType<bool>(json, r'refreshModifiedFiles'),
       );

mobile/openapi/test/scan_library_dto_test.dart

@@ -16,6 +16,11 @@ void main() {
   // final instance = ScanLibraryDto();
 
   group('test ScanLibraryDto', () {
+    // bool checkForOffline
+    test('to test the property `checkForOffline`', () async {
+      // TODO
+    });
+
     // bool refreshAllFiles
     test('to test the property `refreshAllFiles`', () async {
       // TODO

open-api/immich-openapi-specs.json

@@ -8959,6 +8959,9 @@
       },
       "ScanLibraryDto": {
         "properties": {
+          "checkForOffline": {
+            "type": "boolean"
+          },
           "refreshAllFiles": {
             "type": "boolean"
           },

open-api/typescript-sdk/src/fetch-client.ts

@@ -474,6 +474,7 @@ export type UpdateLibraryDto = {
     name?: string;
 };
 export type ScanLibraryDto = {
+    checkForOffline?: boolean;
     refreshAllFiles?: boolean;
     refreshModifiedFiles?: boolean;
 };

server/src/domain/job/job.constants.ts

@@ -71,10 +71,12 @@ export enum JobName {
   // library managment
   LIBRARY_SCAN = 'library-refresh',
   LIBRARY_SCAN_ASSET = 'library-refresh-asset',
-  LIBRARY_REMOVE_OFFLINE = 'library-remove-offline',
   LIBRARY_DELETE = 'library-delete',
   LIBRARY_QUEUE_SCAN_ALL = 'library-queue-all-refresh',
   LIBRARY_QUEUE_CLEANUP = 'library-queue-cleanup',
+  LIBRARY_SCAN_OFFLINE = 'library-scan-offline',
+  LIBRARY_CHECK_IF_ASSET_ONLINE = 'asset-check-if-online',
+  LIBRARY_REMOVE_OFFLINE = 'library-remove-offline',
 
   // cleanup
   DELETE_FILES = 'delete-files',
@@ -149,7 +151,9 @@ export const JOBS_TO_QUEUE: Record<JobName, QueueName> = {
   [JobName.LIBRARY_SCAN_ASSET]: QueueName.LIBRARY,
   [JobName.LIBRARY_SCAN]: QueueName.LIBRARY,
   [JobName.LIBRARY_DELETE]: QueueName.LIBRARY,
+  [JobName.LIBRARY_SCAN_OFFLINE]: QueueName.LIBRARY,
   [JobName.LIBRARY_REMOVE_OFFLINE]: QueueName.LIBRARY,
+  [JobName.LIBRARY_CHECK_IF_ASSET_ONLINE]: QueueName.LIBRARY,
   [JobName.LIBRARY_QUEUE_SCAN_ALL]: QueueName.LIBRARY,
   [JobName.LIBRARY_QUEUE_CLEANUP]: QueueName.LIBRARY,
 };

server/src/domain/library/library.dto.ts

@@ -104,6 +104,9 @@ export class ScanLibraryDto {
 
   @ValidateBoolean({ optional: true })
   refreshAllFiles?: boolean;
+
+  @ValidateBoolean({ optional: true })
+  checkForOffline?: boolean;
 }
 
 export class SearchLibraryDto {

server/src/domain/library/library.service.ts

@@ -551,14 +551,21 @@ export class LibraryService extends EventEmitter {
       throw new BadRequestException('Can only refresh external libraries');
     }
 
-    await this.jobRepository.queue({
+    if (dto.checkForOffline) {
-      name: JobName.LIBRARY_SCAN,
+      if (dto.refreshAllFiles || dto.refreshModifiedFiles) {
-      data: {
+        throw new BadRequestException('Cannot use checkForOffline with refreshAllFiles or refreshModifiedFiles');
-        id,
+      }
-        refreshModifiedFiles: dto.refreshModifiedFiles ?? false,
+      await this.jobRepository.queue({ name: JobName.LIBRARY_SCAN_OFFLINE, data: { id } });
-        refreshAllFiles: dto.refreshAllFiles ?? false,
+    } else {
-      },
+      await this.jobRepository.queue({
-    });
+        name: JobName.LIBRARY_SCAN,
+        data: {
+          id,
+          refreshModifiedFiles: dto.refreshModifiedFiles ?? false,
+          refreshAllFiles: dto.refreshAllFiles ?? false,
+        },
+      });
+    }
   }
 
   async queueRemoveOffline(auth: AuthDto, id: string) {
@@ -594,6 +601,46 @@ export class LibraryService extends EventEmitter {
     return true;
   }
 
+  async handleQueueOfflineScan(job: IEntityJob): Promise<boolean> {
+    this.logger.log(`Checking for offline files in library: ${job.id}`);
+    const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
+      this.assetRepository.getWith(pagination, WithProperty.IS_ONLINE, job.id),
+    );
+
+    for await (const assets of assetPagination) {
+      this.logger.debug(`Checking if ${assets.length} assets are still online`);
+      await this.jobRepository.queueAll(
+        assets.map((asset) => ({
+          name: JobName.LIBRARY_CHECK_IF_ASSET_ONLINE,
+          data: { id: asset.id },
+        })),
+      );
+    }
+
+    return true;
+  }
+
+  // Check if an online asset is offline
+  async handleAssetOnlineCheck(job: IEntityJob) {
+    const asset = await this.assetRepository.getById(job.id);
+
+    if (!asset || asset.isOffline) {
+      // We only care about online assets, we exit here if offline
+      return false;
+    }
+
+    const exists = await this.storageRepository.checkFileExists(asset.originalPath, R_OK);
+
+    if (!exists) {
+      this.logger.debug(`Marking asset as offline: ${asset.originalPath}`);
+      await this.assetRepository.save({ id: asset.id, isOffline: true });
+    } else {
+      this.logger.verbose(`Asset is still online: ${asset.originalPath}`);
+    }
+
+    return true;
+  }
+
   async handleOfflineRemoval(job: IEntityJob): Promise<boolean> {
     const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
       this.assetRepository.getWith(pagination, WithProperty.IS_OFFLINE, job.id),

server/src/domain/repositories/asset.repository.ts

@@ -44,6 +44,7 @@ export enum WithoutProperty {
 
 export enum WithProperty {
   SIDECAR = 'sidecar',
+  IS_ONLINE = 'isOnline',
   IS_OFFLINE = 'isOffline',
 }
 

server/src/domain/repositories/job.repository.ts

@@ -92,6 +92,8 @@ export type JobItem =
   | { name: JobName.LIBRARY_REMOVE_OFFLINE; data: IEntityJob }
   | { name: JobName.LIBRARY_DELETE; data: IEntityJob }
   | { name: JobName.LIBRARY_QUEUE_SCAN_ALL; data: IBaseJob }
+  | { name: JobName.LIBRARY_SCAN_OFFLINE; data: IEntityJob }
+  | { name: JobName.LIBRARY_CHECK_IF_ASSET_ONLINE; data: IEntityJob }
   | { name: JobName.LIBRARY_QUEUE_CLEANUP; data: IBaseJob };
 
 export type JobHandler<T = any> = (data: T) => boolean | Promise<boolean>;

server/src/infra/repositories/asset.repository.ts

@@ -473,6 +473,13 @@ export class AssetRepository implements IAssetRepository {
         where = [{ isOffline: true, libraryId: libraryId }];
         break;
       }
+      case WithProperty.IS_ONLINE: {
+        if (!libraryId) {
+          throw new Error('Library id is required when finding online assets');
+        }
+        where = [{ isOffline: false, libraryId: libraryId }];
+        break;
+      }
 
       default: {
         throw new Error(`Invalid getWith property: ${property}`);

server/src/microservices/app.service.ts

@@ -77,6 +77,8 @@ export class AppService {
       [JobName.LIBRARY_SCAN_ASSET]: (data) => this.libraryService.handleAssetRefresh(data),
       [JobName.LIBRARY_SCAN]: (data) => this.libraryService.handleQueueAssetRefresh(data),
       [JobName.LIBRARY_DELETE]: (data) => this.libraryService.handleDeleteLibrary(data),
+      [JobName.LIBRARY_SCAN_OFFLINE]: (data) => this.libraryService.handleQueueOfflineScan(data),
+      [JobName.LIBRARY_CHECK_IF_ASSET_ONLINE]: (data) => this.libraryService.handleAssetOnlineCheck(data),
       [JobName.LIBRARY_REMOVE_OFFLINE]: (data) => this.libraryService.handleOfflineRemoval(data),
       [JobName.LIBRARY_QUEUE_SCAN_ALL]: (data) => this.libraryService.handleQueueAllScan(data),
       [JobName.LIBRARY_QUEUE_CLEANUP]: () => this.libraryService.handleQueueCleanup(),

web/src/routes/admin/library-management/+page.svelte

@@ -205,6 +205,18 @@
     }
   };
 
+  const handleScanDeleted = async (libraryId: string) => {
+    try {
+      await scanLibrary({ id: libraryId, scanLibraryDto: { checkForOffline: true } });
+      notificationController.show({
+        message: `Scanning library for deleted files`,
+        type: NotificationType.Info,
+      });
+    } catch (error) {
+      handleError(error, 'Unable to scan library');
+    }
+  };
+
   const handleScanChanges = async (libraryId: string) => {
     try {
       await scanLibrary({ id: libraryId, scanLibraryDto: { refreshModifiedFiles: true } });
@@ -261,6 +273,14 @@
     }
   };
 
+  const onScanDeletedLibraryClicked = async () => {
+    closeAll();
+
+    if (selectedLibrary) {
+      await handleScanDeleted(selectedLibrary.id);
+    }
+  };
+
   const onScanSettingClicked = () => {
     closeAll();
     editScanSettings = selectedLibraryIndex;
@@ -406,6 +426,11 @@
                           <MenuOption on:click={() => onScanSettingClicked()} text="Scan Settings" />
                           <hr />
                           <MenuOption on:click={() => onScanNewLibraryClicked()} text="Scan New Library Files" />
+                          <MenuOption
+                            on:click={() => onScanDeletedLibraryClicked()}
+                            text="Scan Deleted Library Files"
+                          />
+
                           <MenuOption
                             on:click={() => onScanAllLibraryFilesClicked()}
                             text="Re-scan All Library Files"