Cutlery/immich
1
0
Fork 1
mirror of https://github.com/immich-app/immich.git synced 2025-11-03 19:17:11 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix(server): loop on checksum generation (#662)

Browse Source
This commit is contained in:
Thanh Pham 2022-09-10 23:52:39 +07:00 committed by GitHub
parent 4ed647c43d
commit 8f2e7b6f65
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 23 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
server/apps/microservices/src/processors/generate-checksum.processor.ts
View File

@ -5,7 +5,7 @@ import { Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm'; import { InjectRepository } from '@nestjs/typeorm';
import { createHash } from 'node:crypto'; import { createHash } from 'node:crypto';
import fs from 'node:fs'; import fs from 'node:fs';
import { IsNull, Repository } from 'typeorm'; import { FindOptionsWhere, IsNull, MoreThan, QueryFailedError, Repository } from 'typeorm';
// TODO: just temporary task to generate previous uploaded assets. // TODO: just temporary task to generate previous uploaded assets.
@Processor(generateChecksumQueueName) @Processor(generateChecksumQueueName)
@ -17,15 +17,23 @@ export class GenerateChecksumProcessor {
@Process() @Process()
async generateChecksum() { async generateChecksum() {
let hasNext = true;
const pageSize = 200; const pageSize = 200;
let hasNext = true;
let lastErrAssetId: string | undefined = undefined;
while (hasNext) { while (hasNext) {
const whereStat: FindOptionsWhere<AssetEntity> = {
checksum: IsNull(),
};
if (lastErrAssetId) {
whereStat.id = MoreThan(lastErrAssetId);
}
const assets = await this.assetRepository.find({ const assets = await this.assetRepository.find({
where: { where: whereStat,
checksum: IsNull()
},
take: pageSize, take: pageSize,
order: { id: 'ASC' }
}); });
if (!assets?.length) { if (!assets?.length) {
@ -35,15 +43,24 @@ export class GenerateChecksumProcessor {
try { try {
await this.generateAssetChecksum(asset); await this.generateAssetChecksum(asset);
} catch (err: any) { } catch (err: any) {
Logger.error(`Error generate checksum ${err}`); lastErrAssetId = asset.id;
if (err instanceof QueryFailedError && (err as any).constraint === 'UQ_userid_checksum') {
Logger.error(`${asset.originalPath} duplicated`);
} else {
Logger.error(`checksum generation ${err}`);
}
} }
} }
// break when reach to the last page
if (assets.length < pageSize) { if (assets.length < pageSize) {
hasNext = false; hasNext = false;
} }
} }
} }
Logger.log(`checksum generation done!`);
} }
private async generateAssetChecksum(asset: AssetEntity) { private async generateAssetChecksum(asset: AssetEntity) {