From 5e497e516606d4765458edd00356e0cf6472e32f Mon Sep 17 00:00:00 2001 From: Jonathan Jogenfors Date: Wed, 13 Mar 2024 21:55:27 +0100 Subject: [PATCH] add job to check for offline files --- mobile/openapi/doc/ScanLibraryDto.md | 1 + .../openapi/lib/model/scan_library_dto.dart | 19 +++++- .../openapi/test/scan_library_dto_test.dart | 5 ++ open-api/immich-openapi-specs.json | 3 + open-api/typescript-sdk/src/fetch-client.ts | 1 + server/src/domain/job/job.constants.ts | 6 +- server/src/domain/library/library.dto.ts | 3 + server/src/domain/library/library.service.ts | 63 ++++++++++++++++--- .../domain/repositories/asset.repository.ts | 1 + .../src/domain/repositories/job.repository.ts | 2 + .../infra/repositories/asset.repository.ts | 7 +++ server/src/microservices/app.service.ts | 2 + .../admin/library-management/+page.svelte | 25 ++++++++ 13 files changed, 128 insertions(+), 10 deletions(-) diff --git a/mobile/openapi/doc/ScanLibraryDto.md b/mobile/openapi/doc/ScanLibraryDto.md index e2c489d85..cb105c5f8 100644 --- a/mobile/openapi/doc/ScanLibraryDto.md +++ b/mobile/openapi/doc/ScanLibraryDto.md @@ -8,6 +8,7 @@ import 'package:openapi/api.dart'; ## Properties Name | Type | Description | Notes ------------ | ------------- | ------------- | ------------- +**checkForOffline** | **bool** | | [optional] **refreshAllFiles** | **bool** | | [optional] **refreshModifiedFiles** | **bool** | | [optional] diff --git a/mobile/openapi/lib/model/scan_library_dto.dart b/mobile/openapi/lib/model/scan_library_dto.dart index 0f5dedf64..6a857dad1 100644 --- a/mobile/openapi/lib/model/scan_library_dto.dart +++ b/mobile/openapi/lib/model/scan_library_dto.dart @@ -13,10 +13,19 @@ part of openapi.api; class ScanLibraryDto { /// Returns a new [ScanLibraryDto] instance. ScanLibraryDto({ + this.checkForOffline, this.refreshAllFiles, this.refreshModifiedFiles, }); + /// + /// Please note: This property should have been non-nullable! Since the specification file + /// does not include a default value (using the "default:" property), however, the generated + /// source code must fall back to having a nullable type. + /// Consider adding a "default:" property in the specification file to hide this note. + /// + bool? checkForOffline; + /// /// Please note: This property should have been non-nullable! Since the specification file /// does not include a default value (using the "default:" property), however, the generated @@ -35,20 +44,27 @@ class ScanLibraryDto { @override bool operator ==(Object other) => identical(this, other) || other is ScanLibraryDto && + other.checkForOffline == checkForOffline && other.refreshAllFiles == refreshAllFiles && other.refreshModifiedFiles == refreshModifiedFiles; @override int get hashCode => // ignore: unnecessary_parenthesis + (checkForOffline == null ? 0 : checkForOffline!.hashCode) + (refreshAllFiles == null ? 0 : refreshAllFiles!.hashCode) + (refreshModifiedFiles == null ? 0 : refreshModifiedFiles!.hashCode); @override - String toString() => 'ScanLibraryDto[refreshAllFiles=$refreshAllFiles, refreshModifiedFiles=$refreshModifiedFiles]'; + String toString() => 'ScanLibraryDto[checkForOffline=$checkForOffline, refreshAllFiles=$refreshAllFiles, refreshModifiedFiles=$refreshModifiedFiles]'; Map toJson() { final json = {}; + if (this.checkForOffline != null) { + json[r'checkForOffline'] = this.checkForOffline; + } else { + // json[r'checkForOffline'] = null; + } if (this.refreshAllFiles != null) { json[r'refreshAllFiles'] = this.refreshAllFiles; } else { @@ -70,6 +86,7 @@ class ScanLibraryDto { final json = value.cast(); return ScanLibraryDto( + checkForOffline: mapValueOfType(json, r'checkForOffline'), refreshAllFiles: mapValueOfType(json, r'refreshAllFiles'), refreshModifiedFiles: mapValueOfType(json, r'refreshModifiedFiles'), ); diff --git a/mobile/openapi/test/scan_library_dto_test.dart b/mobile/openapi/test/scan_library_dto_test.dart index 2b3c75867..8482dcd2b 100644 --- a/mobile/openapi/test/scan_library_dto_test.dart +++ b/mobile/openapi/test/scan_library_dto_test.dart @@ -16,6 +16,11 @@ void main() { // final instance = ScanLibraryDto(); group('test ScanLibraryDto', () { + // bool checkForOffline + test('to test the property `checkForOffline`', () async { + // TODO + }); + // bool refreshAllFiles test('to test the property `refreshAllFiles`', () async { // TODO diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index 2540baf77..1ede69c47 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -8959,6 +8959,9 @@ }, "ScanLibraryDto": { "properties": { + "checkForOffline": { + "type": "boolean" + }, "refreshAllFiles": { "type": "boolean" }, diff --git a/open-api/typescript-sdk/src/fetch-client.ts b/open-api/typescript-sdk/src/fetch-client.ts index acf540aff..a762c7dcf 100644 --- a/open-api/typescript-sdk/src/fetch-client.ts +++ b/open-api/typescript-sdk/src/fetch-client.ts @@ -474,6 +474,7 @@ export type UpdateLibraryDto = { name?: string; }; export type ScanLibraryDto = { + checkForOffline?: boolean; refreshAllFiles?: boolean; refreshModifiedFiles?: boolean; }; diff --git a/server/src/domain/job/job.constants.ts b/server/src/domain/job/job.constants.ts index fe3c81727..a2ec21836 100644 --- a/server/src/domain/job/job.constants.ts +++ b/server/src/domain/job/job.constants.ts @@ -71,10 +71,12 @@ export enum JobName { // library managment LIBRARY_SCAN = 'library-refresh', LIBRARY_SCAN_ASSET = 'library-refresh-asset', - LIBRARY_REMOVE_OFFLINE = 'library-remove-offline', LIBRARY_DELETE = 'library-delete', LIBRARY_QUEUE_SCAN_ALL = 'library-queue-all-refresh', LIBRARY_QUEUE_CLEANUP = 'library-queue-cleanup', + LIBRARY_SCAN_OFFLINE = 'library-scan-offline', + LIBRARY_CHECK_IF_ASSET_ONLINE = 'asset-check-if-online', + LIBRARY_REMOVE_OFFLINE = 'library-remove-offline', // cleanup DELETE_FILES = 'delete-files', @@ -149,7 +151,9 @@ export const JOBS_TO_QUEUE: Record = { [JobName.LIBRARY_SCAN_ASSET]: QueueName.LIBRARY, [JobName.LIBRARY_SCAN]: QueueName.LIBRARY, [JobName.LIBRARY_DELETE]: QueueName.LIBRARY, + [JobName.LIBRARY_SCAN_OFFLINE]: QueueName.LIBRARY, [JobName.LIBRARY_REMOVE_OFFLINE]: QueueName.LIBRARY, + [JobName.LIBRARY_CHECK_IF_ASSET_ONLINE]: QueueName.LIBRARY, [JobName.LIBRARY_QUEUE_SCAN_ALL]: QueueName.LIBRARY, [JobName.LIBRARY_QUEUE_CLEANUP]: QueueName.LIBRARY, }; diff --git a/server/src/domain/library/library.dto.ts b/server/src/domain/library/library.dto.ts index b11bc9998..0d7ac7c4d 100644 --- a/server/src/domain/library/library.dto.ts +++ b/server/src/domain/library/library.dto.ts @@ -104,6 +104,9 @@ export class ScanLibraryDto { @ValidateBoolean({ optional: true }) refreshAllFiles?: boolean; + + @ValidateBoolean({ optional: true }) + checkForOffline?: boolean; } export class SearchLibraryDto { diff --git a/server/src/domain/library/library.service.ts b/server/src/domain/library/library.service.ts index 25894c9b5..3040cbfa5 100644 --- a/server/src/domain/library/library.service.ts +++ b/server/src/domain/library/library.service.ts @@ -551,14 +551,21 @@ export class LibraryService extends EventEmitter { throw new BadRequestException('Can only refresh external libraries'); } - await this.jobRepository.queue({ - name: JobName.LIBRARY_SCAN, - data: { - id, - refreshModifiedFiles: dto.refreshModifiedFiles ?? false, - refreshAllFiles: dto.refreshAllFiles ?? false, - }, - }); + if (dto.checkForOffline) { + if (dto.refreshAllFiles || dto.refreshModifiedFiles) { + throw new BadRequestException('Cannot use checkForOffline with refreshAllFiles or refreshModifiedFiles'); + } + await this.jobRepository.queue({ name: JobName.LIBRARY_SCAN_OFFLINE, data: { id } }); + } else { + await this.jobRepository.queue({ + name: JobName.LIBRARY_SCAN, + data: { + id, + refreshModifiedFiles: dto.refreshModifiedFiles ?? false, + refreshAllFiles: dto.refreshAllFiles ?? false, + }, + }); + } } async queueRemoveOffline(auth: AuthDto, id: string) { @@ -594,6 +601,46 @@ export class LibraryService extends EventEmitter { return true; } + async handleQueueOfflineScan(job: IEntityJob): Promise { + this.logger.log(`Checking for offline files in library: ${job.id}`); + const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => + this.assetRepository.getWith(pagination, WithProperty.IS_ONLINE, job.id), + ); + + for await (const assets of assetPagination) { + this.logger.debug(`Checking if ${assets.length} assets are still online`); + await this.jobRepository.queueAll( + assets.map((asset) => ({ + name: JobName.LIBRARY_CHECK_IF_ASSET_ONLINE, + data: { id: asset.id }, + })), + ); + } + + return true; + } + + // Check if an online asset is offline + async handleAssetOnlineCheck(job: IEntityJob) { + const asset = await this.assetRepository.getById(job.id); + + if (!asset || asset.isOffline) { + // We only care about online assets, we exit here if offline + return false; + } + + const exists = await this.storageRepository.checkFileExists(asset.originalPath, R_OK); + + if (!exists) { + this.logger.debug(`Marking asset as offline: ${asset.originalPath}`); + await this.assetRepository.save({ id: asset.id, isOffline: true }); + } else { + this.logger.verbose(`Asset is still online: ${asset.originalPath}`); + } + + return true; + } + async handleOfflineRemoval(job: IEntityJob): Promise { const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => this.assetRepository.getWith(pagination, WithProperty.IS_OFFLINE, job.id), diff --git a/server/src/domain/repositories/asset.repository.ts b/server/src/domain/repositories/asset.repository.ts index b779c8b8c..2947924eb 100644 --- a/server/src/domain/repositories/asset.repository.ts +++ b/server/src/domain/repositories/asset.repository.ts @@ -44,6 +44,7 @@ export enum WithoutProperty { export enum WithProperty { SIDECAR = 'sidecar', + IS_ONLINE = 'isOnline', IS_OFFLINE = 'isOffline', } diff --git a/server/src/domain/repositories/job.repository.ts b/server/src/domain/repositories/job.repository.ts index 232040f7a..ada5e84b4 100644 --- a/server/src/domain/repositories/job.repository.ts +++ b/server/src/domain/repositories/job.repository.ts @@ -92,6 +92,8 @@ export type JobItem = | { name: JobName.LIBRARY_REMOVE_OFFLINE; data: IEntityJob } | { name: JobName.LIBRARY_DELETE; data: IEntityJob } | { name: JobName.LIBRARY_QUEUE_SCAN_ALL; data: IBaseJob } + | { name: JobName.LIBRARY_SCAN_OFFLINE; data: IEntityJob } + | { name: JobName.LIBRARY_CHECK_IF_ASSET_ONLINE; data: IEntityJob } | { name: JobName.LIBRARY_QUEUE_CLEANUP; data: IBaseJob }; export type JobHandler = (data: T) => boolean | Promise; diff --git a/server/src/infra/repositories/asset.repository.ts b/server/src/infra/repositories/asset.repository.ts index c91ef5e0b..a0b0862a1 100644 --- a/server/src/infra/repositories/asset.repository.ts +++ b/server/src/infra/repositories/asset.repository.ts @@ -473,6 +473,13 @@ export class AssetRepository implements IAssetRepository { where = [{ isOffline: true, libraryId: libraryId }]; break; } + case WithProperty.IS_ONLINE: { + if (!libraryId) { + throw new Error('Library id is required when finding online assets'); + } + where = [{ isOffline: false, libraryId: libraryId }]; + break; + } default: { throw new Error(`Invalid getWith property: ${property}`); diff --git a/server/src/microservices/app.service.ts b/server/src/microservices/app.service.ts index 9fabd5855..68de5fbcd 100644 --- a/server/src/microservices/app.service.ts +++ b/server/src/microservices/app.service.ts @@ -77,6 +77,8 @@ export class AppService { [JobName.LIBRARY_SCAN_ASSET]: (data) => this.libraryService.handleAssetRefresh(data), [JobName.LIBRARY_SCAN]: (data) => this.libraryService.handleQueueAssetRefresh(data), [JobName.LIBRARY_DELETE]: (data) => this.libraryService.handleDeleteLibrary(data), + [JobName.LIBRARY_SCAN_OFFLINE]: (data) => this.libraryService.handleQueueOfflineScan(data), + [JobName.LIBRARY_CHECK_IF_ASSET_ONLINE]: (data) => this.libraryService.handleAssetOnlineCheck(data), [JobName.LIBRARY_REMOVE_OFFLINE]: (data) => this.libraryService.handleOfflineRemoval(data), [JobName.LIBRARY_QUEUE_SCAN_ALL]: (data) => this.libraryService.handleQueueAllScan(data), [JobName.LIBRARY_QUEUE_CLEANUP]: () => this.libraryService.handleQueueCleanup(), diff --git a/web/src/routes/admin/library-management/+page.svelte b/web/src/routes/admin/library-management/+page.svelte index e6f4332ee..32791c048 100644 --- a/web/src/routes/admin/library-management/+page.svelte +++ b/web/src/routes/admin/library-management/+page.svelte @@ -205,6 +205,18 @@ } }; + const handleScanDeleted = async (libraryId: string) => { + try { + await scanLibrary({ id: libraryId, scanLibraryDto: { checkForOffline: true } }); + notificationController.show({ + message: `Scanning library for deleted files`, + type: NotificationType.Info, + }); + } catch (error) { + handleError(error, 'Unable to scan library'); + } + }; + const handleScanChanges = async (libraryId: string) => { try { await scanLibrary({ id: libraryId, scanLibraryDto: { refreshModifiedFiles: true } }); @@ -261,6 +273,14 @@ } }; + const onScanDeletedLibraryClicked = async () => { + closeAll(); + + if (selectedLibrary) { + await handleScanDeleted(selectedLibrary.id); + } + }; + const onScanSettingClicked = () => { closeAll(); editScanSettings = selectedLibraryIndex; @@ -406,6 +426,11 @@ onScanSettingClicked()} text="Scan Settings" />
onScanNewLibraryClicked()} text="Scan New Library Files" /> + onScanDeletedLibraryClicked()} + text="Scan Deleted Library Files" + /> + onScanAllLibraryFilesClicked()} text="Re-scan All Library Files"