perf: cache getConfig (#9377)

* cache `getConfig`

* critical fix

Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>

---------

Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>

server/src/cores/system-config.core.ts

@@ -1,5 +1,6 @@
 import { BadRequestException, ForbiddenException, Injectable } from '@nestjs/common';
 import { CronExpression } from '@nestjs/schedule';
+import AsyncLock from 'async-lock';
 import { plainToInstance } from 'class-transformer';
 import { validate } from 'class-validator';
 import { load as loadYaml } from 'js-yaml';
@@ -21,6 +22,7 @@ import {
   TranscodePolicy,
   VideoCodec,
 } from 'src/entities/system-config.entity';
+import { DatabaseLock } from 'src/interfaces/database.interface';
 import { QueueName } from 'src/interfaces/job.interface';
 import { ILoggerRepository } from 'src/interfaces/logger.interface';
 import { ISystemConfigRepository } from 'src/interfaces/system-config.interface';
@@ -186,7 +188,9 @@ let instance: SystemConfigCore | null;
 
 @Injectable()
 export class SystemConfigCore {
-  private configCache: SystemConfigEntity<SystemConfigValue>[] | null = null;
+  private readonly asyncLock = new AsyncLock();
+  private config: SystemConfig | null = null;
+  private lastUpdated: number | null = null;
 
   public config$ = new Subject<SystemConfig>();
 
@@ -268,32 +272,17 @@ export class SystemConfigCore {
   }
 
   public async getConfig(force = false): Promise<SystemConfig> {
-    const configFilePath = process.env.IMMICH_CONFIG_FILE;
+    if (force || !this.config) {
-    const config = _.cloneDeep(defaults);
+      const lastUpdated = this.lastUpdated;
-    const overrides = configFilePath ? await this.loadFromFile(configFilePath, force) : await this.repository.load();
+      await this.asyncLock.acquire(DatabaseLock[DatabaseLock.GetSystemConfig], async () => {
-
+        if (lastUpdated === this.lastUpdated) {
-    for (const { key, value } of overrides) {
+          this.config = await this.buildConfig();
-      // set via dot notation
+          this.lastUpdated = Date.now();
-      _.set(config, key, value);
+        }
+      });
     }
 
-    const errors = await validate(plainToInstance(SystemConfigDto, config));
+    return this.config!;
-    if (errors.length > 0) {
-      this.logger.error('Validation error', errors);
-      if (configFilePath) {
-        throw new Error(`Invalid value(s) in file: ${errors}`);
-      }
-    }
-
-    if (!config.ffmpeg.acceptedVideoCodecs.includes(config.ffmpeg.targetVideoCodec)) {
-      config.ffmpeg.acceptedVideoCodecs.unshift(config.ffmpeg.targetVideoCodec);
-    }
-
-    if (!config.ffmpeg.acceptedAudioCodecs.includes(config.ffmpeg.targetAudioCodec)) {
-      config.ffmpeg.acceptedAudioCodecs.unshift(config.ffmpeg.targetAudioCodec);
-    }
-
-    return config;
   }
 
   public async updateConfig(newConfig: SystemConfig): Promise<SystemConfig> {
@@ -345,33 +334,60 @@ export class SystemConfigCore {
     this.config$.next(newConfig);
   }
 
-  private async loadFromFile(filepath: string, force = false) {
+  private async buildConfig() {
-    if (force || !this.configCache) {
+    const config = _.cloneDeep(defaults);
-      try {
+    const overrides = process.env.IMMICH_CONFIG_FILE
-        const file = await this.repository.readFile(filepath);
+      ? await this.loadFromFile(process.env.IMMICH_CONFIG_FILE)
-        const config = loadYaml(file.toString()) as any;
+      : await this.repository.load();
-        const overrides: SystemConfigEntity<SystemConfigValue>[] = [];
 
-        for (const key of Object.values(SystemConfigKey)) {
+    for (const { key, value } of overrides) {
-          const value = _.get(config, key);
+      // set via dot notation
-          this.unsetDeep(config, key);
+      _.set(config, key, value);
-          if (value !== undefined) {
+    }
-            overrides.push({ key, value });
-          }
-        }
 
-        if (!_.isEmpty(config)) {
+    const errors = await validate(plainToInstance(SystemConfigDto, config));
-          this.logger.warn(`Unknown keys found: ${JSON.stringify(config, null, 2)}`);
+    if (errors.length > 0) {
-        }
+      if (process.env.IMMICH_CONFIG_FILE) {
-
+        throw new Error(`Invalid value(s) in file: ${errors}`);
-        this.configCache = overrides;
+      } else {
-      } catch (error: Error | any) {
+        this.logger.error('Validation error', errors);
-        this.logger.error(`Unable to load configuration file: ${filepath}`);
-        throw error;
       }
     }
 
-    return this.configCache;
+    if (!config.ffmpeg.acceptedVideoCodecs.includes(config.ffmpeg.targetVideoCodec)) {
+      config.ffmpeg.acceptedVideoCodecs.push(config.ffmpeg.targetVideoCodec);
+    }
+
+    if (!config.ffmpeg.acceptedAudioCodecs.includes(config.ffmpeg.targetAudioCodec)) {
+      config.ffmpeg.acceptedAudioCodecs.push(config.ffmpeg.targetAudioCodec);
+    }
+
+    return config;
+  }
+
+  private async loadFromFile(filepath: string) {
+    try {
+      const file = await this.repository.readFile(filepath);
+      const config = loadYaml(file.toString()) as any;
+      const overrides: SystemConfigEntity<SystemConfigValue>[] = [];
+
+      for (const key of Object.values(SystemConfigKey)) {
+        const value = _.get(config, key);
+        this.unsetDeep(config, key);
+        if (value !== undefined) {
+          overrides.push({ key, value });
+        }
+      }
+
+      if (!_.isEmpty(config)) {
+        this.logger.warn(`Unknown keys found: ${JSON.stringify(config, null, 2)}`);
+      }
+
+      return overrides;
+    } catch (error: Error | any) {
+      this.logger.error(`Unable to load configuration file: ${filepath}`);
+      throw error;
+    }
   }
 
   private unsetDeep(object: object, key: string) {

server/src/interfaces/database.interface.ts

@@ -20,6 +20,7 @@ export enum DatabaseLock {
   StorageTemplateMigration = 420,
   CLIPDimSize = 512,
   LibraryWatch = 1337,
+  GetSystemConfig = 69,
 }
 
 export const extName: Record<DatabaseExtension, string> = {

server/src/services/metadata.service.spec.ts

@@ -99,19 +99,22 @@ describe(MetadataService.name, () => {
   });
 
   describe('init', () => {
-    beforeEach(async () => {
+    it('should pause and resume queue during init', async () => {
-      configMock.load.mockResolvedValue([{ key: SystemConfigKey.REVERSE_GEOCODING_ENABLED, value: true }]);
-
       await sut.init();
+
+      expect(jobMock.pause).toHaveBeenCalledTimes(1);
+      expect(metadataMock.init).toHaveBeenCalledTimes(1);
+      expect(jobMock.resume).toHaveBeenCalledTimes(1);
     });
 
     it('should return if reverse geocoding is disabled', async () => {
       configMock.load.mockResolvedValue([{ key: SystemConfigKey.REVERSE_GEOCODING_ENABLED, value: false }]);
 
       await sut.init();
-      expect(jobMock.pause).toHaveBeenCalledTimes(1);
+
-      expect(metadataMock.init).toHaveBeenCalledTimes(1);
+      expect(jobMock.pause).not.toHaveBeenCalled();
-      expect(jobMock.resume).toHaveBeenCalledTimes(1);
+      expect(metadataMock.init).not.toHaveBeenCalled();
+      expect(jobMock.resume).not.toHaveBeenCalled();
     });
   });