tbelway/immich-quadlet
1
0
Fork 0
forked from Cutlery/immich
Code Pull Requests Activity

wip

Browse Source
This commit is contained in:
Jonathan Jogenfors 2024-03-14 09:48:08 +01:00
parent d09d4d3f29
commit 5b581cee6a
1 changed files with 60 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
server/src/domain/library/library.service.ts
View File

@ -667,9 +667,36 @@ export class LibraryService extends EventEmitter {
this.logger.verbose(`Refreshing library: ${job.id}`); this.logger.verbose(`Refreshing library: ${job.id}`);
const crawledAssetPaths = await this.getPathTrie(library); const pathValidation = await Promise.all(
this.logger.debug(`Found ${crawledAssetPaths.size} asset(s) when crawling import paths ${library.importPaths}`); library.importPaths.map(async (importPath) => await this.validateImportPath(importPath)),
);
const validImportPaths = pathValidation
.map((validation) => {
if (!validation.isValid) {
this.logger.error(`Skipping invalid import path: ${validation.importPath}. Reason: ${validation.message}`);
}
return validation;
})
.filter((validation) => validation.isValid)
.map((validation) => validation.importPath);
const generator = this.storageRepository.walk({
pathsToCrawl: validImportPaths,
exclusionPatterns: library.exclusionPatterns,
});
const crawledAssetPaths = new Trie<string>();
let pathCounter = 0;
for await (const filePath of generator) {
crawledAssetPaths.add(filePath);
pathCounter++;
// Print status message every 50000 counter
if (pathCounter % 50000 === 0) {
this.logger.debug(`Crawled ${pathCounter} paths`);
}
if (pathCounter % 5000 === 0) {
const assetIdsToMarkOnline = []; const assetIdsToMarkOnline = [];
const pagination = usePagination(LIBRARY_SCAN_BATCH_SIZE, (pagination) => const pagination = usePagination(LIBRARY_SCAN_BATCH_SIZE, (pagination) =>
this.assetRepository.getLibraryAssetPaths(pagination, library.id), this.assetRepository.getLibraryAssetPaths(pagination, library.id),
@ -713,39 +740,17 @@ export class LibraryService extends EventEmitter {
} }
} }
crawledAssetPaths.clear();
}
}
// this.logger.debug(`Found ${crawledAssetPaths.size} asset(s) when crawling import paths ${library.importPaths}`);
await this.repository.update({ id: job.id, refreshedAt: new Date() }); await this.repository.update({ id: job.id, refreshedAt: new Date() });
return true; return true;
} }
private async getPathTrie(library: LibraryEntity): Promise<Trie<string>> {
const pathValidation = await Promise.all(
library.importPaths.map(async (importPath) => await this.validateImportPath(importPath)),
);
const validImportPaths = pathValidation
.map((validation) => {
if (!validation.isValid) {
this.logger.error(`Skipping invalid import path: ${validation.importPath}. Reason: ${validation.message}`);
}
return validation;
})
.filter((validation) => validation.isValid)
.map((validation) => validation.importPath);
const generator = this.storageRepository.walk({
pathsToCrawl: validImportPaths,
exclusionPatterns: library.exclusionPatterns,
});
const trie = new Trie<string>();
for await (const filePath of generator) {
trie.add(filePath);
}
return trie;
}
private async findOrFail(id: string) { private async findOrFail(id: string) {
const library = await this.repository.get(id); const library = await this.repository.get(id);
if (!library) { if (!library) {