import { BadRequestException, Injectable } from '@nestjs/common'; import { snakeCase } from 'lodash'; import { OnEvent } from 'src/decorators'; import { mapAsset } from 'src/dtos/asset-response.dto'; import { AllJobStatusResponseDto, JobCommandDto, JobCreateDto, JobStatusDto } from 'src/dtos/job.dto'; import { AssetType, ImmichWorker, ManualJobName } from 'src/enum'; import { ArgOf, ArgsOf } from 'src/interfaces/event.interface'; import { ConcurrentQueueName, JobCommand, JobItem, JobName, JobStatus, QueueCleanType, QueueName, } from 'src/interfaces/job.interface'; import { BaseService } from 'src/services/base.service'; const asJobItem = (dto: JobCreateDto): JobItem => { switch (dto.name) { case ManualJobName.TAG_CLEANUP: { return { name: JobName.TAG_CLEANUP }; } case ManualJobName.PERSON_CLEANUP: { return { name: JobName.PERSON_CLEANUP }; } case ManualJobName.USER_CLEANUP: { return { name: JobName.USER_DELETE_CHECK }; } default: { throw new BadRequestException('Invalid job name'); } } }; @Injectable() export class JobService extends BaseService { @OnEvent({ name: 'config.init', workers: [ImmichWorker.MICROSERVICES] }) onConfigInit({ newConfig: config }: ArgOf<'config.init'>) { this.logger.debug(`Updating queue concurrency settings`); for (const queueName of Object.values(QueueName)) { let concurrency = 1; if (this.isConcurrentQueue(queueName)) { concurrency = config.job[queueName].concurrency; } this.logger.debug(`Setting ${queueName} concurrency to ${concurrency}`); this.jobRepository.setConcurrency(queueName, concurrency); } } @OnEvent({ name: 'config.update', server: true, workers: [ImmichWorker.MICROSERVICES] }) onConfigUpdate({ newConfig: config }: ArgOf<'config.update'>) { this.onConfigInit({ newConfig: config }); } async create(dto: JobCreateDto): Promise { await this.jobRepository.queue(asJobItem(dto)); } async handleCommand(queueName: QueueName, dto: JobCommandDto): Promise { this.logger.debug(`Handling command: queue=${queueName},force=${dto.force}`); switch (dto.command) { case JobCommand.START: { await this.start(queueName, dto); break; } case JobCommand.PAUSE: { await this.jobRepository.pause(queueName); break; } case JobCommand.RESUME: { await this.jobRepository.resume(queueName); break; } case JobCommand.EMPTY: { await this.jobRepository.empty(queueName); break; } case JobCommand.CLEAR_FAILED: { const failedJobs = await this.jobRepository.clear(queueName, QueueCleanType.FAILED); this.logger.debug(`Cleared failed jobs: ${failedJobs}`); break; } } return this.getJobStatus(queueName); } async getJobStatus(queueName: QueueName): Promise { const [jobCounts, queueStatus] = await Promise.all([ this.jobRepository.getJobCounts(queueName), this.jobRepository.getQueueStatus(queueName), ]); return { jobCounts, queueStatus }; } async getAllJobsStatus(): Promise { const response = new AllJobStatusResponseDto(); for (const queueName of Object.values(QueueName)) { response[queueName] = await this.getJobStatus(queueName); } return response; } private async start(name: QueueName, { force }: JobCommandDto): Promise { const { isActive } = await this.jobRepository.getQueueStatus(name); if (isActive) { throw new BadRequestException(`Job is already running`); } this.telemetryRepository.jobs.addToCounter(`immich.queues.${snakeCase(name)}.started`, 1); switch (name) { case QueueName.VIDEO_CONVERSION: { return this.jobRepository.queue({ name: JobName.QUEUE_VIDEO_CONVERSION, data: { force } }); } case QueueName.STORAGE_TEMPLATE_MIGRATION: { return this.jobRepository.queue({ name: JobName.STORAGE_TEMPLATE_MIGRATION }); } case QueueName.MIGRATION: { return this.jobRepository.queue({ name: JobName.QUEUE_MIGRATION }); } case QueueName.SMART_SEARCH: { return this.jobRepository.queue({ name: JobName.QUEUE_SMART_SEARCH, data: { force } }); } case QueueName.DUPLICATE_DETECTION: { return this.jobRepository.queue({ name: JobName.QUEUE_DUPLICATE_DETECTION, data: { force } }); } case QueueName.METADATA_EXTRACTION: { return this.jobRepository.queue({ name: JobName.QUEUE_METADATA_EXTRACTION, data: { force } }); } case QueueName.SIDECAR: { return this.jobRepository.queue({ name: JobName.QUEUE_SIDECAR, data: { force } }); } case QueueName.THUMBNAIL_GENERATION: { return this.jobRepository.queue({ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force } }); } case QueueName.FACE_DETECTION: { return this.jobRepository.queue({ name: JobName.QUEUE_FACE_DETECTION, data: { force } }); } case QueueName.FACIAL_RECOGNITION: { return this.jobRepository.queue({ name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force } }); } case QueueName.LIBRARY: { return this.jobRepository.queue({ name: JobName.LIBRARY_QUEUE_SYNC_ALL, data: { force } }); } case QueueName.BACKUP_DATABASE: { return this.jobRepository.queue({ name: JobName.BACKUP_DATABASE, data: { force } }); } default: { throw new BadRequestException(`Invalid job name: ${name}`); } } } @OnEvent({ name: 'job.start' }) async onJobStart(...[queueName, job]: ArgsOf<'job.start'>) { const queueMetric = `immich.queues.${snakeCase(queueName)}.active`; this.telemetryRepository.jobs.addToGauge(queueMetric, 1); try { const status = await this.jobRepository.run(job); const jobMetric = `immich.jobs.${job.name.replaceAll('-', '_')}.${status}`; this.telemetryRepository.jobs.addToCounter(jobMetric, 1); if (status === JobStatus.SUCCESS || status == JobStatus.SKIPPED) { await this.onDone(job); } } catch (error: Error | any) { this.logger.error(`Unable to run job handler (${queueName}/${job.name}): ${error}`, error?.stack, job.data); } finally { this.telemetryRepository.jobs.addToGauge(queueMetric, -1); } } private isConcurrentQueue(name: QueueName): name is ConcurrentQueueName { return ![ QueueName.FACIAL_RECOGNITION, QueueName.STORAGE_TEMPLATE_MIGRATION, QueueName.DUPLICATE_DETECTION, QueueName.BACKUP_DATABASE, ].includes(name); } async handleNightlyJobs() { await this.jobRepository.queueAll([ { name: JobName.ASSET_DELETION_CHECK }, { name: JobName.USER_DELETE_CHECK }, { name: JobName.PERSON_CLEANUP }, { name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } }, { name: JobName.CLEAN_OLD_AUDIT_LOGS }, { name: JobName.USER_SYNC_USAGE }, { name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false, nightly: true } }, { name: JobName.CLEAN_OLD_SESSION_TOKENS }, ]); } /** * Queue follow up jobs */ private async onDone(item: JobItem) { switch (item.name) { case JobName.SIDECAR_SYNC: case JobName.SIDECAR_DISCOVERY: { await this.jobRepository.queue({ name: JobName.METADATA_EXTRACTION, data: item.data }); break; } case JobName.SIDECAR_WRITE: { await this.jobRepository.queue({ name: JobName.METADATA_EXTRACTION, data: { id: item.data.id, source: 'sidecar-write' }, }); break; } case JobName.METADATA_EXTRACTION: { if (item.data.source === 'sidecar-write') { const [asset] = await this.assetRepository.getByIdsWithAllRelations([item.data.id]); if (asset) { this.eventRepository.clientSend('on_asset_update', asset.ownerId, mapAsset(asset)); } } await this.jobRepository.queue({ name: JobName.LINK_LIVE_PHOTOS, data: item.data }); break; } case JobName.LINK_LIVE_PHOTOS: { await this.jobRepository.queue({ name: JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE, data: item.data }); break; } case JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE: { if (item.data.source === 'upload' || item.data.source === 'copy') { await this.jobRepository.queue({ name: JobName.GENERATE_THUMBNAILS, data: item.data }); } break; } case JobName.GENERATE_PERSON_THUMBNAIL: { const { id } = item.data; const person = await this.personRepository.getById(id); if (person) { this.eventRepository.clientSend('on_person_thumbnail', person.ownerId, person.id); } break; } case JobName.GENERATE_THUMBNAILS: { if (!item.data.notify && item.data.source !== 'upload') { break; } const [asset] = await this.assetRepository.getByIdsWithAllRelations([item.data.id]); if (!asset) { this.logger.warn(`Could not find asset ${item.data.id} after generating thumbnails`); break; } const jobs: JobItem[] = [ { name: JobName.SMART_SEARCH, data: item.data }, { name: JobName.FACE_DETECTION, data: item.data }, ]; if (asset.type === AssetType.VIDEO) { jobs.push({ name: JobName.VIDEO_CONVERSION, data: item.data }); } else if (asset.livePhotoVideoId) { jobs.push({ name: JobName.VIDEO_CONVERSION, data: { id: asset.livePhotoVideoId } }); } await this.jobRepository.queueAll(jobs); if (asset.isVisible) { this.eventRepository.clientSend('on_upload_success', asset.ownerId, mapAsset(asset)); } break; } case JobName.SMART_SEARCH: { if (item.data.source === 'upload') { await this.jobRepository.queue({ name: JobName.DUPLICATE_DETECTION, data: item.data }); } break; } case JobName.USER_DELETION: { this.eventRepository.clientBroadcast('on_user_delete', item.data.id); break; } } } }