refactor: database connection parsing (#17852)

server/src/app.module.ts

@@ -44,7 +44,7 @@ const imports = [
   BullModule.registerQueue(...bull.queues),
   ClsModule.forRoot(cls.config),
   OpenTelemetryModule.forRoot(otel),
-  KyselyModule.forRoot(getKyselyConfig(database.config.kysely)),
+  KyselyModule.forRoot(getKyselyConfig(database.config)),
 ];
 
 class BaseModule implements OnModuleInit, OnModuleDestroy {

server/src/bin/migrations.ts

@@ -10,7 +10,7 @@ import { DatabaseRepository } from 'src/repositories/database.repository';
 import { LoggingRepository } from 'src/repositories/logging.repository';
 import 'src/schema';
 import { schemaDiff, schemaFromCode, schemaFromDatabase } from 'src/sql-tools';
-import { getKyselyConfig } from 'src/utils/database';
+import { asPostgresConnectionConfig, getKyselyConfig } from 'src/utils/database';
 
 const main = async () => {
   const command = process.argv[2];
@@ -56,7 +56,7 @@ const main = async () => {
 const getDatabaseClient = () => {
   const configRepository = new ConfigRepository();
   const { database } = configRepository.getEnv();
-  return new Kysely<any>(getKyselyConfig(database.config.kysely));
+  return new Kysely<any>(getKyselyConfig(database.config));
 };
 
 const runQuery = async (query: string) => {
@@ -105,7 +105,7 @@ const create = (path: string, up: string[], down: string[]) => {
 const compare = async () => {
   const configRepository = new ConfigRepository();
   const { database } = configRepository.getEnv();
-  const db = postgres(database.config.kysely);
+  const db = postgres(asPostgresConnectionConfig(database.config));
 
   const source = schemaFromCode();
   const target = await schemaFromDatabase(db, {});

server/src/bin/sync-sql.ts

@@ -78,7 +78,7 @@ class SqlGenerator {
     const moduleFixture = await Test.createTestingModule({
       imports: [
         KyselyModule.forRoot({
-          ...getKyselyConfig(database.config.kysely),
+          ...getKyselyConfig(database.config),
           log: (event) => {
             if (event.level === 'query') {
               this.sqlLogger.logQuery(event.query.sql);

server/src/repositories/config.repository.spec.ts

@@ -80,21 +80,12 @@ describe('getEnv', () => {
       const { database } = getEnv();
       expect(database).toEqual({
         config: {
-          kysely: expect.objectContaining({
+          connectionType: 'parts',
           host: 'database',
           port: 5432,
           database: 'immich',
           username: 'postgres',
           password: 'postgres',
-          }),
-          typeorm: expect.objectContaining({
-            type: 'postgres',
-            host: 'database',
-            port: 5432,
-            database: 'immich',
-            username: 'postgres',
-            password: 'postgres',
-          }),
         },
         skipMigrations: false,
         vectorExtension: 'vectors',
@@ -110,88 +101,9 @@ describe('getEnv', () => {
     it('should use DB_URL', () => {
       process.env.DB_URL = 'postgres://postgres1:postgres2@database1:54320/immich';
       const { database } = getEnv();
-      expect(database.config.kysely).toMatchObject({
+      expect(database.config).toMatchObject({
-        host: 'database1',
+        connectionType: 'url',
-        password: 'postgres2',
+        url: 'postgres://postgres1:postgres2@database1:54320/immich',
-        user: 'postgres1',
-        port: 54_320,
-        database: 'immich',
-      });
-    });
-
-    it('should handle sslmode=require', () => {
-      process.env.DB_URL = 'postgres://postgres1:postgres2@database1:54320/immich?sslmode=require';
-
-      const { database } = getEnv();
-
-      expect(database.config.kysely).toMatchObject({ ssl: {} });
-    });
-
-    it('should handle sslmode=prefer', () => {
-      process.env.DB_URL = 'postgres://postgres1:postgres2@database1:54320/immich?sslmode=prefer';
-
-      const { database } = getEnv();
-
-      expect(database.config.kysely).toMatchObject({ ssl: {} });
-    });
-
-    it('should handle sslmode=verify-ca', () => {
-      process.env.DB_URL = 'postgres://postgres1:postgres2@database1:54320/immich?sslmode=verify-ca';
-
-      const { database } = getEnv();
-
-      expect(database.config.kysely).toMatchObject({ ssl: {} });
-    });
-
-    it('should handle sslmode=verify-full', () => {
-      process.env.DB_URL = 'postgres://postgres1:postgres2@database1:54320/immich?sslmode=verify-full';
-
-      const { database } = getEnv();
-
-      expect(database.config.kysely).toMatchObject({ ssl: {} });
-    });
-
-    it('should handle sslmode=no-verify', () => {
-      process.env.DB_URL = 'postgres://postgres1:postgres2@database1:54320/immich?sslmode=no-verify';
-
-      const { database } = getEnv();
-
-      expect(database.config.kysely).toMatchObject({ ssl: { rejectUnauthorized: false } });
-    });
-
-    it('should handle ssl=true', () => {
-      process.env.DB_URL = 'postgres://postgres1:postgres2@database1:54320/immich?ssl=true';
-
-      const { database } = getEnv();
-
-      expect(database.config.kysely).toMatchObject({ ssl: true });
-    });
-
-    it('should reject invalid ssl', () => {
-      process.env.DB_URL = 'postgres://postgres1:postgres2@database1:54320/immich?ssl=invalid';
-
-      expect(() => getEnv()).toThrowError('Invalid ssl option: invalid');
-    });
-
-    it('should handle socket: URLs', () => {
-      process.env.DB_URL = 'socket:/run/postgresql?db=database1';
-
-      const { database } = getEnv();
-
-      expect(database.config.kysely).toMatchObject({
-        host: '/run/postgresql',
-        database: 'database1',
-      });
-    });
-
-    it('should handle sockets in postgres: URLs', () => {
-      process.env.DB_URL = 'postgres:///database2?host=/path/to/socket';
-
-      const { database } = getEnv();
-
-      expect(database.config.kysely).toMatchObject({
-        host: '/path/to/socket',
-        database: 'database2',
       });
     });
   });

server/src/repositories/config.repository.ts

@@ -7,8 +7,7 @@ import { Request, Response } from 'express';
 import { RedisOptions } from 'ioredis';
 import { CLS_ID, ClsModuleOptions } from 'nestjs-cls';
 import { OpenTelemetryModuleOptions } from 'nestjs-otel/lib/interfaces';
-import { join, resolve } from 'node:path';
+import { join } from 'node:path';
-import { parse } from 'pg-connection-string';
 import { citiesFile, excludePaths, IWorker } from 'src/constants';
 import { Telemetry } from 'src/decorators';
 import { EnvDto } from 'src/dtos/env.dto';
@@ -22,9 +21,7 @@ import {
   QueueName,
 } from 'src/enum';
 import { DatabaseConnectionParams, VectorExtension } from 'src/types';
-import { isValidSsl, PostgresConnectionConfig } from 'src/utils/database';
 import { setDifference } from 'src/utils/set';
-import { PostgresConnectionOptions } from 'typeorm/driver/postgres/PostgresConnectionOptions.js';
 
 export interface EnvData {
   host?: string;
@@ -59,7 +56,7 @@ export interface EnvData {
   };
 
   database: {
-    config: { typeorm: PostgresConnectionOptions & DatabaseConnectionParams; kysely: PostgresConnectionConfig };
+    config: DatabaseConnectionParams;
     skipMigrations: boolean;
     vectorExtension: VectorExtension;
   };
@@ -152,14 +149,10 @@ const getEnv = (): EnvData => {
   const isProd = environment === ImmichEnvironment.PRODUCTION;
   const buildFolder = dto.IMMICH_BUILD_DATA || '/build';
   const folders = {
-    // eslint-disable-next-line unicorn/prefer-module
-    dist: resolve(`${__dirname}/..`),
     geodata: join(buildFolder, 'geodata'),
     web: join(buildFolder, 'www'),
   };
 
-  const databaseUrl = dto.DB_URL;
-
   let redisConfig = {
     host: dto.REDIS_HOSTNAME || 'redis',
     port: dto.REDIS_PORT || 6379,
@@ -191,30 +184,16 @@ const getEnv = (): EnvData => {
     }
   }
 
-  const parts = {
+  const databaseConnection: DatabaseConnectionParams = dto.DB_URL
+    ? { connectionType: 'url', url: dto.DB_URL }
+    : {
         connectionType: 'parts',
         host: dto.DB_HOSTNAME || 'database',
         port: dto.DB_PORT || 5432,
         username: dto.DB_USERNAME || 'postgres',
         password: dto.DB_PASSWORD || 'postgres',
         database: dto.DB_DATABASE_NAME || 'immich',
-  } as const;
-
-  let parsedOptions: PostgresConnectionConfig = parts;
-  if (dto.DB_URL) {
-    const parsed = parse(dto.DB_URL);
-    if (!isValidSsl(parsed.ssl)) {
-      throw new Error(`Invalid ssl option: ${parsed.ssl}`);
-    }
-
-    parsedOptions = {
-      ...parsed,
-      ssl: parsed.ssl,
-      host: parsed.host ?? undefined,
-      port: parsed.port ? Number(parsed.port) : undefined,
-      database: parsed.database ?? undefined,
       };
-  }
 
   return {
     host: dto.IMMICH_HOST,
@@ -269,21 +248,7 @@ const getEnv = (): EnvData => {
     },
 
     database: {
-      config: {
+      config: databaseConnection,
-        typeorm: {
-          type: 'postgres',
-          entities: [],
-          migrations: [`${folders.dist}/migrations` + '/*.{js,ts}'],
-          subscribers: [],
-          migrationsRun: false,
-          synchronize: false,
-          connectTimeoutMS: 10_000, // 10 seconds
-          parseInt8: true,
-          ...(databaseUrl ? { connectionType: 'url', url: databaseUrl } : parts),
-        },
-        kysely: parsedOptions,
-      },
-
       skipMigrations: dto.DB_SKIP_MIGRATIONS ?? false,
       vectorExtension: dto.DB_VECTOR_EXTENSION === 'pgvector' ? DatabaseExtension.VECTOR : DatabaseExtension.VECTORS,
     },

server/src/repositories/database.repository.ts

@@ -3,7 +3,7 @@ import AsyncLock from 'async-lock';
 import { FileMigrationProvider, Kysely, Migrator, sql, Transaction } from 'kysely';
 import { InjectKysely } from 'nestjs-kysely';
 import { readdir } from 'node:fs/promises';
-import { join } from 'node:path';
+import { join, resolve } from 'node:path';
 import semver from 'semver';
 import { EXTENSION_NAMES, POSTGRES_VERSION_RANGE, VECTOR_VERSION_RANGE, VECTORS_VERSION_RANGE } from 'src/constants';
 import { DB } from 'src/db';
@@ -205,8 +205,29 @@ export class DatabaseRepository {
     const { rows } = await tableExists.execute(this.db);
     const hasTypeOrmMigrations = !!rows[0]?.result;
     if (hasTypeOrmMigrations) {
+      // eslint-disable-next-line unicorn/prefer-module
+      const dist = resolve(`${__dirname}/..`);
+
       this.logger.debug('Running typeorm migrations');
-      const dataSource = new DataSource(database.config.typeorm);
+      const dataSource = new DataSource({
+        type: 'postgres',
+        entities: [],
+        subscribers: [],
+        migrations: [`${dist}/migrations` + '/*.{js,ts}'],
+        migrationsRun: false,
+        synchronize: false,
+        connectTimeoutMS: 10_000, // 10 seconds
+        parseInt8: true,
+        ...(database.config.connectionType === 'url'
+          ? { url: database.config.url }
+          : {
+              host: database.config.host,
+              port: database.config.port,
+              username: database.config.username,
+              password: database.config.password,
+              database: database.config.database,
+            }),
+      });
       await dataSource.initialize();
       await dataSource.runMigrations(options);
       await dataSource.destroy();

server/src/services/backup.service.ts

@@ -70,7 +70,7 @@ export class BackupService extends BaseService {
   async handleBackupDatabase(): Promise<JobStatus> {
     this.logger.debug(`Database Backup Started`);
     const { database } = this.configRepository.getEnv();
-    const config = database.config.typeorm;
+    const config = database.config;
 
     const isUrlConnection = config.connectionType === 'url';
 

server/src/services/database.service.spec.ts

@@ -53,23 +53,13 @@ describe(DatabaseService.name, () => {
           mockEnvData({
             database: {
               config: {
-                kysely: {
-                  host: 'database',
-                  port: 5432,
-                  user: 'postgres',
-                  password: 'postgres',
-                  database: 'immich',
-                },
-                typeorm: {
                 connectionType: 'parts',
-                  type: 'postgres',
                 host: 'database',
                 port: 5432,
                 username: 'postgres',
                 password: 'postgres',
                 database: 'immich',
               },
-              },
               skipMigrations: false,
               vectorExtension: extension,
             },
@@ -292,23 +282,13 @@ describe(DatabaseService.name, () => {
         mockEnvData({
           database: {
             config: {
-              kysely: {
-                host: 'database',
-                port: 5432,
-                user: 'postgres',
-                password: 'postgres',
-                database: 'immich',
-              },
-              typeorm: {
               connectionType: 'parts',
-                type: 'postgres',
               host: 'database',
               port: 5432,
               username: 'postgres',
               password: 'postgres',
               database: 'immich',
             },
-            },
             skipMigrations: true,
             vectorExtension: DatabaseExtension.VECTORS,
           },
@@ -325,23 +305,13 @@ describe(DatabaseService.name, () => {
         mockEnvData({
           database: {
             config: {
-              kysely: {
-                host: 'database',
-                port: 5432,
-                user: 'postgres',
-                password: 'postgres',
-                database: 'immich',
-              },
-              typeorm: {
               connectionType: 'parts',
-                type: 'postgres',
               host: 'database',
               port: 5432,
               username: 'postgres',
               password: 'postgres',
               database: 'immich',
             },
-            },
             skipMigrations: true,
             vectorExtension: DatabaseExtension.VECTOR,
           },

server/src/utils/database.spec.ts

@@ -0,0 +1,83 @@
+import { asPostgresConnectionConfig } from 'src/utils/database';
+
+describe('database  utils', () => {
+  describe('asPostgresConnectionConfig', () => {
+    it('should handle sslmode=require', () => {
+      expect(
+        asPostgresConnectionConfig({
+          connectionType: 'url',
+          url: 'postgres://postgres1:postgres2@database1:54320/immich?sslmode=require',
+        }),
+      ).toMatchObject({ ssl: {} });
+    });
+
+    it('should handle sslmode=prefer', () => {
+      expect(
+        asPostgresConnectionConfig({
+          connectionType: 'url',
+          url: 'postgres://postgres1:postgres2@database1:54320/immich?sslmode=prefer',
+        }),
+      ).toMatchObject({ ssl: {} });
+    });
+
+    it('should handle sslmode=verify-ca', () => {
+      expect(
+        asPostgresConnectionConfig({
+          connectionType: 'url',
+          url: 'postgres://postgres1:postgres2@database1:54320/immich?sslmode=verify-ca',
+        }),
+      ).toMatchObject({ ssl: {} });
+    });
+
+    it('should handle sslmode=verify-full', () => {
+      expect(
+        asPostgresConnectionConfig({
+          connectionType: 'url',
+          url: 'postgres://postgres1:postgres2@database1:54320/immich?sslmode=verify-full',
+        }),
+      ).toMatchObject({ ssl: {} });
+    });
+
+    it('should handle sslmode=no-verify', () => {
+      expect(
+        asPostgresConnectionConfig({
+          connectionType: 'url',
+          url: 'postgres://postgres1:postgres2@database1:54320/immich?sslmode=no-verify',
+        }),
+      ).toMatchObject({ ssl: { rejectUnauthorized: false } });
+    });
+
+    it('should handle ssl=true', () => {
+      expect(
+        asPostgresConnectionConfig({
+          connectionType: 'url',
+          url: 'postgres://postgres1:postgres2@database1:54320/immich?ssl=true',
+        }),
+      ).toMatchObject({ ssl: true });
+    });
+
+    it('should reject invalid ssl', () => {
+      expect(() =>
+        asPostgresConnectionConfig({
+          connectionType: 'url',
+          url: 'postgres://postgres1:postgres2@database1:54320/immich?ssl=invalid',
+        }),
+      ).toThrowError('Invalid ssl option');
+    });
+
+    it('should handle socket: URLs', () => {
+      expect(
+        asPostgresConnectionConfig({ connectionType: 'url', url: 'socket:/run/postgresql?db=database1' }),
+      ).toMatchObject({ host: '/run/postgresql', database: 'database1' });
+    });
+
+    it('should handle sockets in postgres: URLs', () => {
+      expect(
+        asPostgresConnectionConfig({ connectionType: 'url', url: 'postgres:///database2?host=/path/to/socket' }),
+      ).toMatchObject({
+        host: '/path/to/socket',
+        database: 'database2',
+      });
+    });
+  });
+});

server/src/utils/database.ts

@@ -13,33 +13,57 @@ import {
 } from 'kysely';
 import { PostgresJSDialect } from 'kysely-postgres-js';
 import { jsonArrayFrom, jsonObjectFrom } from 'kysely/helpers/postgres';
+import { parse } from 'pg-connection-string';
 import postgres, { Notice } from 'postgres';
 import { columns, Exif, Person } from 'src/database';
 import { DB } from 'src/db';
 import { AssetFileType } from 'src/enum';
 import { TimeBucketSize } from 'src/repositories/asset.repository';
 import { AssetSearchBuilderOptions } from 'src/repositories/search.repository';
+import { DatabaseConnectionParams } from 'src/types';
 
 type Ssl = 'require' | 'allow' | 'prefer' | 'verify-full' | boolean | object;
 
-export type PostgresConnectionConfig = {
+const isValidSsl = (ssl?: string | boolean | object): ssl is Ssl =>
-  host?: string;
-  password?: string;
-  user?: string;
-  port?: number;
-  database?: string;
-  max?: number;
-  client_encoding?: string;
-  ssl?: Ssl;
-  application_name?: string;
-  fallback_application_name?: string;
-  options?: string;
-};
-
-export const isValidSsl = (ssl?: string | boolean | object): ssl is Ssl =>
   typeof ssl !== 'string' || ssl === 'require' || ssl === 'allow' || ssl === 'prefer' || ssl === 'verify-full';
 
-export const getKyselyConfig = (options: PostgresConnectionConfig): KyselyConfig => {
+export const asPostgresConnectionConfig = (params: DatabaseConnectionParams) => {
+  if (params.connectionType === 'parts') {
+    return {
+      host: params.host,
+      port: params.port,
+      username: params.username,
+      password: params.password,
+      database: params.database,
+      ssl: undefined,
+    };
+  }
+
+  const { host, port, user, password, database, ...rest } = parse(params.url);
+  let ssl: Ssl | undefined;
+  if (rest.ssl) {
+    if (!isValidSsl(rest.ssl)) {
+      throw new Error(`Invalid ssl option: ${rest.ssl}`);
+    }
+    ssl = rest.ssl;
+  }
+
+  return {
+    host: host ?? undefined,
+    port: port ? Number(port) : undefined,
+    username: user,
+    password,
+    database: database ?? undefined,
+    ssl,
+  };
+};
+
+export const getKyselyConfig = (
+  params: DatabaseConnectionParams,
+  options: Partial<postgres.Options<Record<string, postgres.PostgresType>>> = {},
+): KyselyConfig => {
+  const config = asPostgresConnectionConfig(params);
+
   return {
     dialect: new PostgresJSDialect({
       postgres: postgres({
@@ -66,6 +90,12 @@ export const getKyselyConfig = (options: PostgresConnectionConfig): KyselyConfig
         connection: {
           TimeZone: 'UTC',
         },
+        host: config.host,
+        port: config.port,
+        username: config.username,
+        password: config.password,
+        database: config.database,
+        ssl: config.ssl,
         ...options,
       }),
     }),

server/test/medium/globalSetup.ts

@@ -1,5 +1,4 @@
 import { Kysely } from 'kysely';
-import { parse } from 'pg-connection-string';
 import { DB } from 'src/db';
 import { ConfigRepository } from 'src/repositories/config.repository';
 import { DatabaseRepository } from 'src/repositories/database.repository';
@@ -37,19 +36,10 @@ const globalSetup = async () => {
 
   const postgresPort = postgresContainer.getMappedPort(5432);
   const postgresUrl = `postgres://postgres:postgres@localhost:${postgresPort}/immich`;
-  const parsed = parse(postgresUrl);
 
   process.env.IMMICH_TEST_POSTGRES_URL = postgresUrl;
 
-  const db = new Kysely<DB>(
+  const db = new Kysely<DB>(getKyselyConfig({ connectionType: 'url', url: postgresUrl }));
-    getKyselyConfig({
-      ...parsed,
-      ssl: false,
-      host: parsed.host ?? undefined,
-      port: parsed.port ? Number(parsed.port) : undefined,
-      database: parsed.database ?? undefined,
-    }),
-  );
 
   const configRepository = new ConfigRepository();
   const logger = new LoggingRepository(undefined, configRepository);

server/test/repositories/config.repository.mock.ts

@@ -21,19 +21,12 @@ const envData: EnvData = {
 
   database: {
     config: {
-      kysely: { database: 'immich', host: 'database', port: 5432 },
-      typeorm: {
       connectionType: 'parts',
       database: 'immich',
-        type: 'postgres',
       host: 'database',
       port: 5432,
       username: 'postgres',
       password: 'postgres',
-        name: 'immich',
-        synchronize: false,
-        migrationsRun: true,
-      },
     },
 
     skipMigrations: false,

server/test/utils.ts

@@ -1,9 +1,9 @@
 import { ClassConstructor } from 'class-transformer';
-import { Kysely, sql } from 'kysely';
+import { Kysely } from 'kysely';
 import { ChildProcessWithoutNullStreams } from 'node:child_process';
 import { Writable } from 'node:stream';
-import { parse } from 'pg-connection-string';
 import { PNG } from 'pngjs';
+import postgres from 'postgres';
 import { DB } from 'src/db';
 import { AccessRepository } from 'src/repositories/access.repository';
 import { ActivityRepository } from 'src/repositories/activity.repository';
@@ -49,7 +49,7 @@ import { VersionHistoryRepository } from 'src/repositories/version-history.repos
 import { ViewRepository } from 'src/repositories/view-repository';
 import { BaseService } from 'src/services/base.service';
 import { RepositoryInterface } from 'src/types';
-import { getKyselyConfig } from 'src/utils/database';
+import { asPostgresConnectionConfig, getKyselyConfig } from 'src/utils/database';
 import { IAccessRepositoryMock, newAccessRepositoryMock } from 'test/repositories/access.repository.mock';
 import { newAssetRepositoryMock } from 'test/repositories/asset.repository.mock';
 import { newConfigRepositoryMock } from 'test/repositories/config.repository.mock';
@@ -297,24 +297,20 @@ function* newPngFactory() {
 
 const pngFactory = newPngFactory();
 
+const withDatabase = (url: string, name: string) => url.replace('/immich', `/${name}`);
+
 export const getKyselyDB = async (suffix?: string): Promise<Kysely<DB>> => {
-  const parsed = parse(process.env.IMMICH_TEST_POSTGRES_URL!);
+  const testUrl = process.env.IMMICH_TEST_POSTGRES_URL!;
+  const sql = postgres({
+    ...asPostgresConnectionConfig({ connectionType: 'url', url: withDatabase(testUrl, 'postgres') }),
+    max: 1,
+  });
 
-  const parsedOptions = {
-    ...parsed,
-    ssl: false,
-    host: parsed.host ?? undefined,
-    port: parsed.port ? Number(parsed.port) : undefined,
-    database: parsed.database ?? undefined,
-  };
-
-  const kysely = new Kysely<DB>(getKyselyConfig({ ...parsedOptions, max: 1, database: 'postgres' }));
   const randomSuffix = Math.random().toString(36).slice(2, 7);
   const dbName = `immich_${suffix ?? randomSuffix}`;
+  await sql.unsafe(`CREATE DATABASE ${dbName} WITH TEMPLATE immich OWNER postgres;`);
 
-  await sql.raw(`CREATE DATABASE ${dbName} WITH TEMPLATE immich OWNER postgres;`).execute(kysely);
+  return new Kysely<DB>(getKyselyConfig({ connectionType: 'url', url: withDatabase(testUrl, dbName) }));
-
-  return new Kysely<DB>(getKyselyConfig({ ...parsedOptions, database: dbName }));
 };
 
 export const newRandomImage = () => {