tbelway/immich-quadlet
1
0
Fork 0
forked from Cutlery/immich
Code Pull Requests Activity

refactor batches

Browse Source
This commit is contained in:
Jonathan Jogenfors 2024-03-14 11:45:21 +01:00
parent 3b0d993f12
commit 4ba95bb0a6
1 changed files with 25 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
server/src/domain/library/library.service.ts
View File

@ -1,4 +1,4 @@
import { AssetType, LibraryEntity, LibraryType } from '@app/infra/entities'; import { AssetType, LibraryType } from '@app/infra/entities';
import { ImmichLogger } from '@app/infra/logger'; import { ImmichLogger } from '@app/infra/logger';
import { BadRequestException, Inject, Injectable } from '@nestjs/common'; import { BadRequestException, Inject, Injectable } from '@nestjs/common';
import { R_OK } from 'node:constants'; import { R_OK } from 'node:constants';
@ -685,19 +685,40 @@ export class LibraryService extends EventEmitter {
exclusionPatterns: library.exclusionPatterns, exclusionPatterns: library.exclusionPatterns,
}); });
const shouldScanAll = job.refreshAllFiles || job.refreshModifiedFiles;
let crawledAssetPaths: string[] = []; let crawledAssetPaths: string[] = [];
let assetIdsToMarkOnline: string[] = [];
let pathCounter = 0; let pathCounter = 0;
const processAssetBatch = async () => {
if (assetIdsToMarkOnline.length > 0) {
this.logger.debug(`Found ${assetIdsToMarkOnline.length} online asset(s) previously marked as offline`);
await this.assetRepository.updateAll(assetIdsToMarkOnline, { isOffline: false });
}
await this.scanAssets(job.id, crawledAssetPaths, library.ownerId, job.refreshAllFiles ?? false);
};
for await (const filePath of generator) { for await (const filePath of generator) {
const asset = await this.assetRepository.getByLibraryIdAndOriginalPath(job.id, filePath);
if (asset && asset.isOffline) {
// Mark this asset as online
assetIdsToMarkOnline.push(asset.id);
}
crawledAssetPaths.push(filePath); crawledAssetPaths.push(filePath);
pathCounter++; pathCounter++;
if (pathCounter % LIBRARY_SCAN_BATCH_SIZE === 0) { if (crawledAssetPaths.length % LIBRARY_SCAN_BATCH_SIZE === 0) {
await this.scanAssets(job.id, crawledAssetPaths, library.ownerId, job.refreshAllFiles ?? false); await processAssetBatch();
assetIdsToMarkOnline = [];
crawledAssetPaths = []; crawledAssetPaths = [];
} }
} }
await this.scanAssets(job.id, crawledAssetPaths, library.ownerId, job.refreshAllFiles ?? false);
await processAssetBatch();
this.logger.log(`Found ${pathCounter} asset(s) when crawling import paths ${library.importPaths}`); this.logger.log(`Found ${pathCounter} asset(s) when crawling import paths ${library.importPaths}`);