feat(server): OpenTelemetry integration (#7356)

* wip

* span class decorator

fix typing

* improvements

* noisy postgres logs

formatting

* add source

* strict string comparison

Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>

* remove debug code

* execution time histogram

* remove prometheus stuff

remove prometheus data

* disable by default

disable nestjs-otel stuff by default

update imports

* re-add postgres instrumentation

formatting

formatting

* refactor: execution time histogram

* decorator alias

* formatting

* keep original method order in filesystem repo

* linting

* enable otel sdk in e2e

* actually enable otel sdk in e2e

* share exclude paths

* formatting

* fix rebase

* more buckets

* add example setup

* add envs

fix

actual fix

* linting

* update comments

* update docker env

* use more specific env

---------

Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>
Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>

docker/docker-compose.dev.yml

@@ -115,5 +115,28 @@ services:
     ports:
       - 5432:5432
   
+  # set IMMICH_METRICS=true in .env to enable metrics
+  # immich-prometheus:
+  #   container_name: immich_prometheus
+  #   ports:
+  #     - 9090:9090
+  #   image: prom/prometheus
+  #   volumes:
+  #     - ./prometheus.yml:/etc/prometheus/prometheus.yml
+  #     - prometheus-data:/prometheus
+
+  # first login uses admin/admin
+  # add data source for http://immich-prometheus:9090 to get started
+  # immich-grafana:
+  #   container_name: immich_grafana
+  #   command: ['./run.sh', '-disable-reporting']
+  #   ports:
+  #     - 3000:3000
+  #   image: grafana/grafana:10.3.3-ubuntu
+  #   volumes:
+  #     - grafana-data:/var/lib/grafana
+
 volumes:
   model-cache:
+  prometheus-data:
+  grafana-data:

docker/docker-compose.prod.yml

@@ -73,5 +73,28 @@ services:
     ports:
       - 5432:5432
 
+  # set IMMICH_METRICS=true in .env to enable metrics
+  immich-prometheus:
+    container_name: immich_prometheus
+    ports:
+      - 9090:9090
+    image: prom/prometheus
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+      - prometheus-data:/prometheus
+
+  # first login uses admin/admin
+  # add data source for http://immich-prometheus:9090 to get started
+  immich-grafana:
+    container_name: immich_grafana
+    command: ['./run.sh', '-disable-reporting']
+    ports:
+      - 3000:3000
+    image: grafana/grafana:10.3.3-ubuntu
+    volumes:
+      - grafana-data:/var/lib/grafana
+
 volumes:
   model-cache:
+  prometheus-data:
+  grafana-data:

docker/prometheus.yml

@@ -0,0 +1,12 @@
+global:
+  scrape_interval:     15s
+  evaluation_interval: 15s
+
+scrape_configs:
+  - job_name: immich_server
+    static_configs:
+      - targets: ['immich-server:8081']
+  
+  - job_name: immich_microservices
+    static_configs:
+      - targets: ['immich-microservices:8081']

e2e/docker-compose.yml

@@ -14,6 +14,7 @@ x-server-build: &server-common
     - DB_DATABASE_NAME=immich
     - REDIS_HOSTNAME=redis
     - IMMICH_MACHINE_LEARNING_ENABLED=false
+    - IMMICH_METRICS=true
   volumes:
     - upload:/usr/src/app/upload
     - ../server/test/assets:/data/assets

server/e2e/docker-compose.server-e2e.yml

@@ -18,6 +18,7 @@ services:
       - DB_USERNAME=postgres
       - DB_PASSWORD=postgres
       - DB_DATABASE_NAME=e2e_test
+      - IMMICH_METRICS=true
     depends_on:
       - database
 

server/package.json

@@ -45,6 +45,9 @@
     "@nestjs/swagger": "^7.1.8",
     "@nestjs/typeorm": "^10.0.0",
     "@nestjs/websockets": "^10.2.2",
+    "@opentelemetry/auto-instrumentations-node": "^0.41.1",
+    "@opentelemetry/exporter-prometheus": "^0.48.0",
+    "@opentelemetry/sdk-node": "^0.48.0",
     "@socket.io/postgres-adapter": "^0.3.1",
     "@types/picomatch": "^2.3.3",
     "archiver": "^7.0.0",
@@ -68,6 +71,7 @@
     "lodash": "^4.17.21",
     "luxon": "^3.4.2",
     "nest-commander": "^3.11.1",
+    "nestjs-otel": "^5.1.5",
     "node-addon-api": "^7.0.0",
     "openid-client": "^5.4.3",
     "pg": "^8.11.3",

server/src/domain/domain.config.ts

@@ -14,16 +14,23 @@ export const immichAppConfig: ConfigModuleOptions = {
   isGlobal: true,
   validationSchema: Joi.object({
     NODE_ENV: Joi.string().optional().valid('development', 'production', 'staging').default('development'),
+    LOG_LEVEL: Joi.string()
+      .optional()
+      .valid(...Object.values(LogLevel)),
+
     DB_USERNAME: WHEN_DB_URL_SET,
     DB_PASSWORD: WHEN_DB_URL_SET,
     DB_DATABASE_NAME: WHEN_DB_URL_SET,
     DB_URL: Joi.string().optional(),
     DB_VECTOR_EXTENSION: Joi.string().optional().valid('pgvector', 'pgvecto.rs').default('pgvecto.rs'),
-    LOG_LEVEL: Joi.string()
+
-      .optional()
-      .valid(...Object.values(LogLevel)),
     MACHINE_LEARNING_PORT: Joi.number().optional(),
     MICROSERVICES_PORT: Joi.number().optional(),
-    SERVER_PORT: Joi.number().optional(),
+    IMMICH_METRICS_PORT: Joi.number().optional(),
+
+    IMMICH_METRICS: Joi.boolean().optional().default(false),
+    IMMICH_HOST_METRICS: Joi.boolean().optional().default(false),
+    IMMICH_API_METRICS: Joi.boolean().optional().default(false),
+    IMMICH_IO_METRICS: Joi.boolean().optional().default(false),
   }),
 };

server/src/immich/main.ts

@@ -1,5 +1,6 @@
 import { WEB_ROOT, envName, isDev, serverVersion } from '@app/domain';
-import { WebSocketAdapter } from '@app/infra';
+import { WebSocketAdapter, excludePaths } from '@app/infra';
+import { otelSDK } from '@app/infra/instrumentation';
 import { ImmichLogger } from '@app/infra/logger';
 import { NestFactory } from '@nestjs/core';
 import { NestExpressApplication } from '@nestjs/platform-express';
@@ -15,6 +16,7 @@ const logger = new ImmichLogger('ImmichServer');
 const port = Number(process.env.SERVER_PORT) || 3001;
 
 export async function bootstrap() {
+  otelSDK.start();
   const app = await NestFactory.create<NestExpressApplication>(AppModule, { bufferLogs: true });
 
   app.useLogger(app.get(ImmichLogger));
@@ -28,7 +30,6 @@ export async function bootstrap() {
   app.useWebSocketAdapter(new WebSocketAdapter(app));
   useSwagger(app, isDev);
 
-  const excludePaths = ['/.well-known/immich', '/custom.css'];
   app.setGlobalPrefix('api', { exclude: excludePaths });
   if (existsSync(WEB_ROOT)) {
     // copied from https://github.com/sveltejs/kit/blob/679b5989fe62e3964b9a73b712d7b41831aa1f07/packages/adapter-node/src/handler.js#L46

server/src/infra/infra.config.ts

@@ -34,3 +34,5 @@ export const bullConfig: QueueOptions = {
 };
 
 export const bullQueues: RegisterQueueOptions[] = Object.values(QueueName).map((name) => ({ name }));
+
+export const excludePaths = ['/.well-known/immich', '/custom.css', '/favicon.ico'];

server/src/infra/infra.module.ts

@@ -33,9 +33,11 @@ import { Global, Module, Provider } from '@nestjs/common';
 import { ConfigModule } from '@nestjs/config';
 import { ScheduleModule, SchedulerRegistry } from '@nestjs/schedule';
 import { TypeOrmModule } from '@nestjs/typeorm';
+import { OpenTelemetryModule } from 'nestjs-otel';
 import { databaseConfig } from './database.config';
 import { databaseEntities } from './entities';
 import { bullConfig, bullQueues } from './infra.config';
+import { otelConfig } from './instrumentation';
 import {
   AccessRepository,
   ActivityRepository,
@@ -106,6 +108,7 @@ const providers: Provider[] = [
     ScheduleModule,
     BullModule.forRoot(bullConfig),
     BullModule.registerQueue(...bullQueues),
+    OpenTelemetryModule.forRoot(otelConfig),
   ],
   providers: [...providers],
   exports: [...providers, BullModule],

server/src/infra/infra.utils.ts

@@ -121,6 +121,27 @@ export function ChunkedSet(options?: { paramIndex?: number }): MethodDecorator {
   return Chunked({ ...options, mergeFn: setUnion });
 }
 
+// https://stackoverflow.com/a/74898678
+export function DecorateAll(
+  decorator: <T>(
+    target: any,
+    propertyKey: string,
+    descriptor: TypedPropertyDescriptor<T>,
+  ) => TypedPropertyDescriptor<T> | void,
+) {
+  return (target: any) => {
+    const descriptors = Object.getOwnPropertyDescriptors(target.prototype);
+    for (const [propName, descriptor] of Object.entries(descriptors)) {
+      const isMethod = typeof descriptor.value == 'function' && propName !== 'constructor';
+      if (!isMethod) {
+        continue;
+      }
+      decorator({ ...target, constructor: { ...target.constructor, name: target.name } as any }, propName, descriptor);
+      Object.defineProperty(target.prototype, propName, descriptor);
+    }
+  };
+}
+
 export function searchAssetBuilder(
   builder: SelectQueryBuilder<AssetEntity>,
   options: AssetSearchBuilderOptions,

server/src/infra/instrumentation.ts

@@ -0,0 +1,106 @@
+import { serverVersion } from '@app/domain/domain.constant';
+import { Histogram, MetricOptions, ValueType, metrics } from '@opentelemetry/api';
+import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks';
+import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
+import { HttpInstrumentation } from '@opentelemetry/instrumentation-http';
+import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis';
+import { NestInstrumentation } from '@opentelemetry/instrumentation-nestjs-core';
+import { PgInstrumentation } from '@opentelemetry/instrumentation-pg';
+import { Resource } from '@opentelemetry/resources';
+import { ExplicitBucketHistogramAggregation, View } from '@opentelemetry/sdk-metrics';
+import { NodeSDK } from '@opentelemetry/sdk-node';
+import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions';
+import { snakeCase, startCase } from 'lodash';
+import { OpenTelemetryModuleOptions } from 'nestjs-otel/lib/interfaces';
+import { copyMetadataFromFunctionToFunction } from 'nestjs-otel/lib/opentelemetry.utils';
+import { performance } from 'node:perf_hooks';
+import { excludePaths } from './infra.config';
+import { DecorateAll } from './infra.utils';
+
+let metricsEnabled = process.env.IMMICH_METRICS === 'true';
+const hostMetrics =
+  process.env.IMMICH_HOST_METRICS == null ? metricsEnabled : process.env.IMMICH_HOST_METRICS === 'true';
+const apiMetrics = process.env.IMMICH_API_METRICS == null ? metricsEnabled : process.env.IMMICH_API_METRICS === 'true';
+const repoMetrics = process.env.IMMICH_IO_METRICS == null ? metricsEnabled : process.env.IMMICH_IO_METRICS === 'true';
+
+metricsEnabled ||= hostMetrics || apiMetrics || repoMetrics;
+if (!metricsEnabled && process.env.OTEL_SDK_DISABLED === undefined) {
+  process.env.OTEL_SDK_DISABLED = 'true';
+}
+
+const aggregation = new ExplicitBucketHistogramAggregation(
+  [0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10, 25, 50, 75, 100, 250, 500, 750, 1000, 2500, 5000, 7500, 10_000],
+  true,
+);
+
+const metricsPort = Number.parseInt(process.env.IMMICH_METRICS_PORT ?? '8081');
+
+export const otelSDK = new NodeSDK({
+  resource: new Resource({
+    [SemanticResourceAttributes.SERVICE_NAME]: `immich`,
+    [SemanticResourceAttributes.SERVICE_VERSION]: serverVersion.toString(),
+  }),
+  metricReader: new PrometheusExporter({ port: metricsPort }),
+  contextManager: new AsyncLocalStorageContextManager(),
+  instrumentations: [
+    new HttpInstrumentation(),
+    new IORedisInstrumentation(),
+    new NestInstrumentation(),
+    new PgInstrumentation(),
+  ],
+  views: [new View({ aggregation, instrumentName: '*', instrumentUnit: 'ms' })],
+});
+
+export const otelConfig: OpenTelemetryModuleOptions = {
+  metrics: {
+    hostMetrics,
+    apiMetrics: {
+      enable: apiMetrics,
+      ignoreRoutes: excludePaths,
+    },
+  },
+};
+
+function ExecutionTimeHistogram({ description, unit = 'ms', valueType = ValueType.DOUBLE }: MetricOptions = {}) {
+  return (target: any, propertyKey: string | symbol, descriptor: PropertyDescriptor) => {
+    if (!repoMetrics || process.env.OTEL_SDK_DISABLED) {
+      return;
+    }
+
+    const method = descriptor.value;
+    const className = target.constructor.name as string;
+    const propertyName = String(propertyKey);
+    const metricName = `${snakeCase(className).replaceAll(/_(?=(repository)|(controller)|(provider)|(service)|(module))/g, '.')}.${snakeCase(propertyName)}.duration`;
+
+    const metricDescription =
+      description ??
+      `The elapsed time in ${unit} for the ${startCase(className)} to ${startCase(propertyName).toLowerCase()}`;
+
+    let histogram: Histogram | undefined;
+
+    descriptor.value = function (...args: any[]) {
+      const start = performance.now();
+      const result = method.apply(this, args);
+
+      void Promise.resolve(result)
+        .then(() => {
+          const end = performance.now();
+          if (!histogram) {
+            histogram = metrics
+              .getMeter('immich')
+              .createHistogram(metricName, { description: metricDescription, unit, valueType });
+          }
+          histogram.record(end - start, {});
+        })
+        .catch(() => {
+          // noop
+        });
+
+      return result;
+    };
+
+    copyMetadataFromFunctionToFunction(method, descriptor.value);
+  };
+}
+
+export const Instrumentation = () => DecorateAll(ExecutionTimeHistogram());

server/src/infra/repositories/access.repository.ts

@@ -14,6 +14,7 @@ import {
 } from '../entities';
 import { DummyValue, GenerateSql } from '../infra.util';
 import { ChunkedSet } from '../infra.utils';
+import { Instrumentation } from '../instrumentation';
 
 type IActivityAccess = IAccessRepository['activity'];
 type IAlbumAccess = IAccessRepository['album'];
@@ -24,6 +25,7 @@ type ITimelineAccess = IAccessRepository['timeline'];
 type IPersonAccess = IAccessRepository['person'];
 type IPartnerAccess = IAccessRepository['partner'];
 
+@Instrumentation()
 class ActivityAccess implements IActivityAccess {
   constructor(
     private activityRepository: Repository<ActivityEntity>,

server/src/infra/repositories/activity.repository.ts

@@ -4,6 +4,7 @@ import { InjectRepository } from '@nestjs/typeorm';
 import { IsNull, Repository } from 'typeorm';
 import { ActivityEntity } from '../entities/activity.entity';
 import { DummyValue, GenerateSql } from '../infra.util';
+import { Instrumentation } from '../instrumentation';
 
 export interface ActivitySearch {
   albumId?: string;
@@ -12,6 +13,7 @@ export interface ActivitySearch {
   isLiked?: boolean;
 }
 
+@Instrumentation()
 @Injectable()
 export class ActivityRepository implements IActivityRepository {
   constructor(@InjectRepository(ActivityEntity) private repository: Repository<ActivityEntity>) {}

server/src/infra/repositories/album.repository.ts

@@ -8,7 +8,9 @@ import { dataSource } from '../database.config';
 import { AlbumEntity, AssetEntity } from '../entities';
 import { DATABASE_PARAMETER_CHUNK_SIZE, DummyValue, GenerateSql } from '../infra.util';
 import { Chunked, ChunkedArray } from '../infra.utils';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 @Injectable()
 export class AlbumRepository implements IAlbumRepository {
   constructor(

server/src/infra/repositories/api-key.repository.ts

@@ -4,7 +4,9 @@ import { InjectRepository } from '@nestjs/typeorm';
 import { Repository } from 'typeorm';
 import { APIKeyEntity } from '../entities';
 import { DummyValue, GenerateSql } from '../infra.util';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 @Injectable()
 export class ApiKeyRepository implements IKeyRepository {
   constructor(@InjectRepository(APIKeyEntity) private repository: Repository<APIKeyEntity>) {}

server/src/infra/repositories/asset-stack.repository.ts

@@ -3,7 +3,9 @@ import { Injectable } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
 import { Repository } from 'typeorm';
 import { AssetStackEntity } from '../entities';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 @Injectable()
 export class AssetStackRepository implements IAssetStackRepository {
   constructor(@InjectRepository(AssetStackEntity) private repository: Repository<AssetStackEntity>) {}

server/src/infra/repositories/asset.repository.ts

@@ -39,6 +39,7 @@ import {
 import { AssetEntity, AssetJobStatusEntity, AssetType, ExifEntity, SmartInfoEntity } from '../entities';
 import { DummyValue, GenerateSql } from '../infra.util';
 import { Chunked, ChunkedArray, OptionalBetween, paginate, paginatedBuilder, searchAssetBuilder } from '../infra.utils';
+import { Instrumentation } from '../instrumentation';
 
 const truncateMap: Record<TimeBucketSize, string> = {
   [TimeBucketSize.DAY]: 'day',
@@ -50,6 +51,7 @@ const dateTrunc = (options: TimeBucketOptions) =>
     truncateMap[options.size]
   }', (asset."localDateTime" at time zone 'UTC')) at time zone 'UTC')::timestamptz`;
 
+@Instrumentation()
 @Injectable()
 export class AssetRepository implements IAssetRepository {
   constructor(

server/src/infra/repositories/audit.repository.ts

@@ -2,7 +2,9 @@ import { AuditSearch, IAuditRepository } from '@app/domain';
 import { InjectRepository } from '@nestjs/typeorm';
 import { LessThan, MoreThan, Repository } from 'typeorm';
 import { AuditEntity } from '../entities';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 export class AuditRepository implements IAuditRepository {
   constructor(@InjectRepository(AuditEntity) private repository: Repository<AuditEntity>) {}
 

server/src/infra/repositories/communication.repository.ts

@@ -15,7 +15,9 @@ import {
   WebSocketServer,
 } from '@nestjs/websockets';
 import { Server, Socket } from 'socket.io';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 @WebSocketGateway({
   cors: true,
   path: '/api/socket.io',

server/src/infra/repositories/crypto.repository.ts

@@ -3,14 +3,26 @@ import { Injectable } from '@nestjs/common';
 import { compareSync, hash } from 'bcrypt';
 import { createHash, randomBytes, randomUUID } from 'node:crypto';
 import { createReadStream } from 'node:fs';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 @Injectable()
 export class CryptoRepository implements ICryptoRepository {
-  randomUUID = randomUUID;
+  randomUUID() {
-  randomBytes = randomBytes;
+    return randomUUID();
+  }
 
-  hashBcrypt = hash;
+  randomBytes(size: number) {
-  compareBcrypt = compareSync;
+    return randomBytes(size);
+  }
+
+  hashBcrypt(data: string | Buffer, saltOrRounds: string | number) {
+    return hash(data, saltOrRounds);
+  }
+
+  compareBcrypt(data: string | Buffer, encrypted: string) {
+    return compareSync(data, encrypted);
+  }
 
   hashSha256(value: string) {
     return createHash('sha256').update(value).digest('base64');

server/src/infra/repositories/database.repository.ts

@@ -15,8 +15,10 @@ import { InjectDataSource } from '@nestjs/typeorm';
 import AsyncLock from 'async-lock';
 import { DataSource, EntityManager, QueryRunner } from 'typeorm';
 import { isValidInteger } from '../infra.utils';
+import { Instrumentation } from '../instrumentation';
 import { ImmichLogger } from '../logger';
 
+@Instrumentation()
 @Injectable()
 export class DatabaseRepository implements IDatabaseRepository {
   private logger = new ImmichLogger(DatabaseRepository.name);

server/src/infra/repositories/filesystem.provider.ts

@@ -13,23 +13,37 @@ import archiver from 'archiver';
 import chokidar, { WatchOptions } from 'chokidar';
 import { glob } from 'fast-glob';
 import { constants, createReadStream, existsSync, mkdirSync } from 'node:fs';
-import fs, { copyFile, readdir, rename, stat, utimes, writeFile } from 'node:fs/promises';
+import fs from 'node:fs/promises';
 import path from 'node:path';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 export class FilesystemProvider implements IStorageRepository {
   private logger = new ImmichLogger(FilesystemProvider.name);
 
-  readdir = readdir;
+  readdir(folder: string): Promise<string[]> {
+    return fs.readdir(folder);
+  }
 
-  copyFile = copyFile;
+  copyFile(source: string, target: string) {
+    return fs.copyFile(source, target);
+  }
 
-  stat = stat;
+  stat(filepath: string) {
+    return fs.stat(filepath);
+  }
 
-  writeFile = writeFile;
+  writeFile(filepath: string, buffer: Buffer) {
+    return fs.writeFile(filepath, buffer);
+  }
 
-  rename = rename;
+  rename(source: string, target: string) {
+    return fs.rename(source, target);
+  }
 
-  utimes = utimes;
+  utimes(filepath: string, atime: Date, mtime: Date) {
+    return fs.utimes(filepath, atime, mtime);
+  }
 
   createZipStream(): ImmichZipStream {
     const archive = archiver('zip', { store: true });

server/src/infra/repositories/job.repository.ts

@@ -17,7 +17,9 @@ import { Job, JobsOptions, Processor, Queue, Worker, WorkerOptions } from 'bullm
 import { CronJob, CronTime } from 'cron';
 import { setTimeout } from 'node:timers/promises';
 import { bullConfig } from '../infra.config';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 @Injectable()
 export class JobRepository implements IJobRepository {
   private workers: Partial<Record<QueueName, Worker>> = {};

server/src/infra/repositories/library.repository.ts

@@ -5,7 +5,9 @@ import { IsNull, Not } from 'typeorm';
 import { Repository } from 'typeorm/repository/Repository.js';
 import { LibraryEntity, LibraryType } from '../entities';
 import { DummyValue, GenerateSql } from '../infra.util';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 @Injectable()
 export class LibraryRepository implements ILibraryRepository {
   constructor(@InjectRepository(LibraryEntity) private repository: Repository<LibraryEntity>) {}

server/src/infra/repositories/machine-learning.repository.ts

@@ -11,9 +11,11 @@ import {
 } from '@app/domain';
 import { Injectable } from '@nestjs/common';
 import { readFile } from 'node:fs/promises';
+import { Instrumentation } from '../instrumentation';
 
 const errorPrefix = 'Machine learning request';
 
+@Instrumentation()
 @Injectable()
 export class MachineLearningRepository implements IMachineLearningRepository {
   private async predict<T>(url: string, input: TextModelInput | VisionModelInput, config: ModelConfig): Promise<T> {
@@ -50,7 +52,7 @@ export class MachineLearningRepository implements IMachineLearningRepository {
     } as CLIPConfig);
   }
 
-  async getFormData(input: TextModelInput | VisionModelInput, config: ModelConfig): Promise<FormData> {
+  private async getFormData(input: TextModelInput | VisionModelInput, config: ModelConfig): Promise<FormData> {
     const formData = new FormData();
     const { enabled, modelName, modelType, ...options } = config;
     if (!enabled) {

server/src/infra/repositories/media.repository.ts

@@ -13,11 +13,13 @@ import fs from 'node:fs/promises';
 import { Writable } from 'node:stream';
 import { promisify } from 'node:util';
 import sharp from 'sharp';
+import { Instrumentation } from '../instrumentation';
 
 const probe = promisify<string, FfprobeData>(ffmpeg.ffprobe);
 sharp.concurrency(0);
 sharp.cache({ files: 0 });
 
+@Instrumentation()
 export class MediaRepository implements IMediaRepository {
   private logger = new ImmichLogger(MediaRepository.name);
 
@@ -115,19 +117,6 @@ export class MediaRepository implements IMediaRepository {
     });
   }
 
-  configureFfmpegCall(input: string, output: string | Writable, options: TranscodeOptions) {
-    return ffmpeg(input, { niceness: 10 })
-      .inputOptions(options.inputOptions)
-      .outputOptions(options.outputOptions)
-      .output(output)
-      .on('error', (error, stdout, stderr) => this.logger.error(stderr || error));
-  }
-
-  chainPath(existing: string, path: string) {
-    const separator = existing.endsWith(':') ? '' : ':';
-    return `${existing}${separator}${path}`;
-  }
-
   async generateThumbhash(imagePath: string): Promise<Buffer> {
     const maxSize = 100;
 
@@ -140,4 +129,17 @@ export class MediaRepository implements IMediaRepository {
     const thumbhash = await import('thumbhash');
     return Buffer.from(thumbhash.rgbaToThumbHash(info.width, info.height, data));
   }
+
+  private configureFfmpegCall(input: string, output: string | Writable, options: TranscodeOptions) {
+    return ffmpeg(input, { niceness: 10 })
+      .inputOptions(options.inputOptions)
+      .outputOptions(options.outputOptions)
+      .output(output)
+      .on('error', (error, stdout, stderr) => this.logger.error(stderr || error));
+  }
+
+  private chainPath(existing: string, path: string) {
+    const separator = existing.endsWith(':') ? '' : ':';
+    return `${existing}${separator}${path}`;
+  }
 }

server/src/infra/repositories/metadata.repository.ts

@@ -23,7 +23,9 @@ import * as readLine from 'node:readline';
 import { DataSource, QueryRunner, Repository } from 'typeorm';
 import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity.js';
 import { DummyValue, GenerateSql } from '../infra.util';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 export class MetadataRepository implements IMetadataRepository {
   constructor(
     @InjectRepository(ExifEntity) private exifRepository: Repository<ExifEntity>,

server/src/infra/repositories/move.repository.ts

@@ -4,7 +4,9 @@ import { InjectRepository } from '@nestjs/typeorm';
 import { Repository } from 'typeorm';
 import { MoveEntity, PathType } from '../entities';
 import { DummyValue, GenerateSql } from '../infra.util';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 @Injectable()
 export class MoveRepository implements IMoveRepository {
   constructor(@InjectRepository(MoveEntity) private repository: Repository<MoveEntity>) {}

server/src/infra/repositories/partner.repository.ts

@@ -3,7 +3,9 @@ import { Injectable } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
 import { DeepPartial, Repository } from 'typeorm';
 import { PartnerEntity } from '../entities';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 @Injectable()
 export class PartnerRepository implements IPartnerRepository {
   constructor(@InjectRepository(PartnerEntity) private readonly repository: Repository<PartnerEntity>) {}

server/src/infra/repositories/person.repository.ts

@@ -15,7 +15,9 @@ import { FindManyOptions, FindOptionsRelations, FindOptionsSelect, In, Repositor
 import { AssetEntity, AssetFaceEntity, PersonEntity } from '../entities';
 import { DummyValue, GenerateSql } from '../infra.util';
 import { ChunkedArray, asVector, paginate } from '../infra.utils';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 export class PersonRepository implements IPersonRepository {
   constructor(
     @InjectRepository(AssetEntity) private assetRepository: Repository<AssetEntity>,

server/src/infra/repositories/search.repository.ts

@@ -26,7 +26,9 @@ import { Repository, SelectQueryBuilder } from 'typeorm';
 import { vectorExt } from '../database.config';
 import { DummyValue, GenerateSql } from '../infra.util';
 import { asVector, isValidInteger, paginatedBuilder, searchAssetBuilder } from '../infra.utils';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 @Injectable()
 export class SearchRepository implements ISearchRepository {
   private logger = new ImmichLogger(SearchRepository.name);

server/src/infra/repositories/server-info.repository.ts

@@ -1,6 +1,8 @@
 import { GitHubRelease, IServerInfoRepository } from '@app/domain';
 import { Injectable } from '@nestjs/common';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 @Injectable()
 export class ServerInfoRepository implements IServerInfoRepository {
   async getGitHubRelease(): Promise<GitHubRelease> {

server/src/infra/repositories/shared-link.repository.ts

@@ -4,7 +4,9 @@ import { InjectRepository } from '@nestjs/typeorm';
 import { Repository } from 'typeorm';
 import { SharedLinkEntity } from '../entities';
 import { DummyValue, GenerateSql } from '../infra.util';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 @Injectable()
 export class SharedLinkRepository implements ISharedLinkRepository {
   constructor(@InjectRepository(SharedLinkEntity) private repository: Repository<SharedLinkEntity>) {}

server/src/infra/repositories/system-config.repository.ts

@@ -5,12 +5,15 @@ import { In, Repository } from 'typeorm';
 import { SystemConfigEntity } from '../entities';
 import { DummyValue, GenerateSql } from '../infra.util';
 import { Chunked } from '../infra.utils';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 export class SystemConfigRepository implements ISystemConfigRepository {
   constructor(
     @InjectRepository(SystemConfigEntity)
     private repository: Repository<SystemConfigEntity>,
   ) {}
+
   async fetchStyle(url: string) {
     try {
       const response = await fetch(url);

server/src/infra/repositories/system-metadata.repository.ts

@@ -2,12 +2,15 @@ import { ISystemMetadataRepository } from '@app/domain/repositories/system-metad
 import { InjectRepository } from '@nestjs/typeorm';
 import { Repository } from 'typeorm';
 import { SystemMetadata, SystemMetadataEntity } from '../entities';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 export class SystemMetadataRepository implements ISystemMetadataRepository {
   constructor(
     @InjectRepository(SystemMetadataEntity)
     private repository: Repository<SystemMetadataEntity>,
   ) {}
+
   async get<T extends keyof SystemMetadata>(key: T): Promise<SystemMetadata[T] | null> {
     const metadata = await this.repository.findOne({ where: { key } });
     if (!metadata) {

server/src/infra/repositories/tag.repository.ts

@@ -3,7 +3,9 @@ import { AssetEntity, TagEntity } from '@app/infra/entities';
 import { Injectable } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
 import { Repository } from 'typeorm';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 @Injectable()
 export class TagRepository implements ITagRepository {
   constructor(

server/src/infra/repositories/user-token.repository.ts

@@ -4,7 +4,9 @@ import { InjectRepository } from '@nestjs/typeorm';
 import { Repository } from 'typeorm';
 import { UserTokenEntity } from '../entities';
 import { DummyValue, GenerateSql } from '../infra.util';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 @Injectable()
 export class UserTokenRepository implements IUserTokenRepository {
   constructor(@InjectRepository(UserTokenEntity) private repository: Repository<UserTokenEntity>) {}

server/src/infra/repositories/user.repository.ts

@@ -4,7 +4,9 @@ import { InjectRepository } from '@nestjs/typeorm';
 import { IsNull, Not, Repository } from 'typeorm';
 import { AssetEntity, UserEntity } from '../entities';
 import { DummyValue, GenerateSql } from '../infra.util';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 @Injectable()
 export class UserRepository implements IUserRepository {
   constructor(

server/src/microservices/app.service.ts

@@ -15,6 +15,7 @@ import {
   SystemConfigService,
   UserService,
 } from '@app/domain';
+import { otelSDK } from '@app/infra/instrumentation';
 import { Injectable } from '@nestjs/common';
 
 @Injectable()
@@ -87,5 +88,6 @@ export class AppService {
   async teardown() {
     await this.libraryService.teardown();
     await this.metadataService.teardown();
+    await otelSDK.shutdown();
   }
 }

server/src/microservices/main.ts

@@ -1,5 +1,6 @@
 import { envName, serverVersion } from '@app/domain';
 import { WebSocketAdapter } from '@app/infra';
+import { otelSDK } from '@app/infra/instrumentation';
 import { ImmichLogger } from '@app/infra/logger';
 import { NestFactory } from '@nestjs/core';
 import { MicroservicesModule } from './microservices.module';
@@ -8,8 +9,8 @@ const logger = new ImmichLogger('ImmichMicroservice');
 const port = Number(process.env.MICROSERVICES_PORT) || 3002;
 
 export async function bootstrap() {
+  otelSDK.start();
   const app = await NestFactory.create(MicroservicesModule, { bufferLogs: true });
-
   app.useLogger(app.get(ImmichLogger));
   app.useWebSocketAdapter(new WebSocketAdapter(app));