Cutlery/immich
1
0
Fork 1
mirror of https://github.com/immich-app/immich.git synced 2025-05-24 01:12:58 -04:00
Code Issues Packages Projects Releases Wiki Activity

refactor: database connection parsing (#17852)

Browse Source
This commit is contained in:
Jason Rasmussen 2025-04-24 12:58:29 -04:00 committed by GitHub
parent dab4870fed
commit 1d610ad9cb
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 217 additions and 257 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
server/src/app.module.ts
View File

@ -44,7 +44,7 @@ const imports = [
BullModule.registerQueue(...bull.queues),
ClsModule.forRoot(cls.config),
OpenTelemetryModule.forRoot(otel),
KyselyModule.forRoot(getKyselyConfig(database.config.kysely)),
KyselyModule.forRoot(getKyselyConfig(database.config)),
];
class BaseModule implements OnModuleInit, OnModuleDestroy {

6
server/src/bin/migrations.ts
View File

@ -10,7 +10,7 @@ import { DatabaseRepository } from 'src/repositories/database.repository';
import { LoggingRepository } from 'src/repositories/logging.repository';
import 'src/schema';
import { schemaDiff, schemaFromCode, schemaFromDatabase } from 'src/sql-tools';
import { getKyselyConfig } from 'src/utils/database';
import { asPostgresConnectionConfig, getKyselyConfig } from 'src/utils/database';
const main = async () => {
const command = process.argv[2];
@ -56,7 +56,7 @@ const main = async () => {
const getDatabaseClient = () => {
const configRepository = new ConfigRepository();
const { database } = configRepository.getEnv();
return new Kysely<any>(getKyselyConfig(database.config.kysely));
return new Kysely<any>(getKyselyConfig(database.config));
};
const runQuery = async (query: string) => {
@ -105,7 +105,7 @@ const create = (path: string, up: string[], down: string[]) => {
const compare = async () => {
const configRepository = new ConfigRepository();
const { database } = configRepository.getEnv();
const db = postgres(database.config.kysely);
const db = postgres(asPostgresConnectionConfig(database.config));
const source = schemaFromCode();
const target = await schemaFromDatabase(db, {});

2
server/src/bin/sync-sql.ts
View File

@ -78,7 +78,7 @@ class SqlGenerator {
const moduleFixture = await Test.createTestingModule({
imports: [
KyselyModule.forRoot({
...getKyselyConfig(database.config.kysely),
...getKyselyConfig(database.config),
log: (event) => {
if (event.level === 'query') {
this.sqlLogger.logQuery(event.query.sql);

106
server/src/repositories/config.repository.spec.ts
View File

@ -80,21 +80,12 @@ describe('getEnv', () => {
const { database } = getEnv();
expect(database).toEqual({
config: {
kysely: expect.objectContaining({
host: 'database',
port: 5432,
database: 'immich',
username: 'postgres',
password: 'postgres',
}),
typeorm: expect.objectContaining({
type: 'postgres',
host: 'database',
port: 5432,
database: 'immich',
username: 'postgres',
password: 'postgres',
}),
connectionType: 'parts',
host: 'database',
port: 5432,
database: 'immich',
username: 'postgres',
password: 'postgres',
},
skipMigrations: false,
vectorExtension: 'vectors',
@ -110,88 +101,9 @@ describe('getEnv', () => {
it('should use DB_URL', () => {
process.env.DB_URL = 'postgres://postgres1:postgres2@database1:54320/immich';
const { database } = getEnv();
expect(database.config.kysely).toMatchObject({
host: 'database1',
password: 'postgres2',
user: 'postgres1',
port: 54_320,
database: 'immich',
});
});
it('should handle sslmode=require', () => {
process.env.DB_URL = 'postgres://postgres1:postgres2@database1:54320/immich?sslmode=require';
const { database } = getEnv();
expect(database.config.kysely).toMatchObject({ ssl: {} });
});
it('should handle sslmode=prefer', () => {
process.env.DB_URL = 'postgres://postgres1:postgres2@database1:54320/immich?sslmode=prefer';
const { database } = getEnv();
expect(database.config.kysely).toMatchObject({ ssl: {} });
});
it('should handle sslmode=verify-ca', () => {
process.env.DB_URL = 'postgres://postgres1:postgres2@database1:54320/immich?sslmode=verify-ca';
const { database } = getEnv();
expect(database.config.kysely).toMatchObject({ ssl: {} });
});
it('should handle sslmode=verify-full', () => {
process.env.DB_URL = 'postgres://postgres1:postgres2@database1:54320/immich?sslmode=verify-full';
const { database } = getEnv();
expect(database.config.kysely).toMatchObject({ ssl: {} });
});
it('should handle sslmode=no-verify', () => {
process.env.DB_URL = 'postgres://postgres1:postgres2@database1:54320/immich?sslmode=no-verify';
const { database } = getEnv();
expect(database.config.kysely).toMatchObject({ ssl: { rejectUnauthorized: false } });
});
it('should handle ssl=true', () => {
process.env.DB_URL = 'postgres://postgres1:postgres2@database1:54320/immich?ssl=true';
const { database } = getEnv();
expect(database.config.kysely).toMatchObject({ ssl: true });
});
it('should reject invalid ssl', () => {
process.env.DB_URL = 'postgres://postgres1:postgres2@database1:54320/immich?ssl=invalid';
expect(() => getEnv()).toThrowError('Invalid ssl option: invalid');
});
it('should handle socket: URLs', () => {
process.env.DB_URL = 'socket:/run/postgresql?db=database1';
const { database } = getEnv();
expect(database.config.kysely).toMatchObject({
host: '/run/postgresql',
database: 'database1',
});
});
it('should handle sockets in postgres: URLs', () => {
process.env.DB_URL = 'postgres:///database2?host=/path/to/socket';
const { database } = getEnv();
expect(database.config.kysely).toMatchObject({
host: '/path/to/socket',
database: 'database2',
expect(database.config).toMatchObject({
connectionType: 'url',
url: 'postgres://postgres1:postgres2@database1:54320/immich',
});
});
});

61
server/src/repositories/config.repository.ts
View File

@ -7,8 +7,7 @@ import { Request, Response } from 'express';
import { RedisOptions } from 'ioredis';
import { CLS_ID, ClsModuleOptions } from 'nestjs-cls';
import { OpenTelemetryModuleOptions } from 'nestjs-otel/lib/interfaces';
import { join, resolve } from 'node:path';
import { parse } from 'pg-connection-string';
import { join } from 'node:path';
import { citiesFile, excludePaths, IWorker } from 'src/constants';
import { Telemetry } from 'src/decorators';
import { EnvDto } from 'src/dtos/env.dto';
@ -22,9 +21,7 @@ import {
QueueName,
} from 'src/enum';
import { DatabaseConnectionParams, VectorExtension } from 'src/types';
import { isValidSsl, PostgresConnectionConfig } from 'src/utils/database';
import { setDifference } from 'src/utils/set';
import { PostgresConnectionOptions } from 'typeorm/driver/postgres/PostgresConnectionOptions.js';
export interface EnvData {
host?: string;
@ -59,7 +56,7 @@ export interface EnvData {
};
database: {
config: { typeorm: PostgresConnectionOptions & DatabaseConnectionParams; kysely: PostgresConnectionConfig };
config: DatabaseConnectionParams;
skipMigrations: boolean;
vectorExtension: VectorExtension;
};
@ -152,14 +149,10 @@ const getEnv = (): EnvData => {
const isProd = environment === ImmichEnvironment.PRODUCTION;
const buildFolder = dto.IMMICH_BUILD_DATA || '/build';
const folders = {
// eslint-disable-next-line unicorn/prefer-module
dist: resolve(`${__dirname}/..`),
geodata: join(buildFolder, 'geodata'),
web: join(buildFolder, 'www'),
};
const databaseUrl = dto.DB_URL;
let redisConfig = {
host: dto.REDIS_HOSTNAME || 'redis',
port: dto.REDIS_PORT || 6379,
@ -191,30 +184,16 @@ const getEnv = (): EnvData => {
}
}
const parts = {
connectionType: 'parts',
host: dto.DB_HOSTNAME || 'database',
port: dto.DB_PORT || 5432,
username: dto.DB_USERNAME || 'postgres',
password: dto.DB_PASSWORD || 'postgres',
database: dto.DB_DATABASE_NAME || 'immich',
} as const;
let parsedOptions: PostgresConnectionConfig = parts;
if (dto.DB_URL) {
const parsed = parse(dto.DB_URL);
if (!isValidSsl(parsed.ssl)) {
throw new Error(`Invalid ssl option: ${parsed.ssl}`);
}
parsedOptions = {
...parsed,
ssl: parsed.ssl,
host: parsed.host ?? undefined,
port: parsed.port ? Number(parsed.port) : undefined,
database: parsed.database ?? undefined,
};
}
const databaseConnection: DatabaseConnectionParams = dto.DB_URL
? { connectionType: 'url', url: dto.DB_URL }
: {
connectionType: 'parts',
host: dto.DB_HOSTNAME || 'database',
port: dto.DB_PORT || 5432,
username: dto.DB_USERNAME || 'postgres',
password: dto.DB_PASSWORD || 'postgres',
database: dto.DB_DATABASE_NAME || 'immich',
};
return {
host: dto.IMMICH_HOST,
@ -269,21 +248,7 @@ const getEnv = (): EnvData => {
},
database: {
config: {
typeorm: {
type: 'postgres',
entities: [],
migrations: [`${folders.dist}/migrations` + '/*.{js,ts}'],
subscribers: [],
migrationsRun: false,
synchronize: false,
connectTimeoutMS: 10_000, // 10 seconds
parseInt8: true,
...(databaseUrl ? { connectionType: 'url', url: databaseUrl } : parts),
},
kysely: parsedOptions,
},
config: databaseConnection,
skipMigrations: dto.DB_SKIP_MIGRATIONS ?? false,
vectorExtension: dto.DB_VECTOR_EXTENSION === 'pgvector' ? DatabaseExtension.VECTOR : DatabaseExtension.VECTORS,
},

25
server/src/repositories/database.repository.ts
View File

@ -3,7 +3,7 @@ import AsyncLock from 'async-lock';
import { FileMigrationProvider, Kysely, Migrator, sql, Transaction } from 'kysely';
import { InjectKysely } from 'nestjs-kysely';
import { readdir } from 'node:fs/promises';
import { join } from 'node:path';
import { join, resolve } from 'node:path';
import semver from 'semver';
import { EXTENSION_NAMES, POSTGRES_VERSION_RANGE, VECTOR_VERSION_RANGE, VECTORS_VERSION_RANGE } from 'src/constants';
import { DB } from 'src/db';
@ -205,8 +205,29 @@ export class DatabaseRepository {
const { rows } = await tableExists.execute(this.db);
const hasTypeOrmMigrations = !!rows[0]?.result;
if (hasTypeOrmMigrations) {
// eslint-disable-next-line unicorn/prefer-module
const dist = resolve(`${__dirname}/..`);
this.logger.debug('Running typeorm migrations');
const dataSource = new DataSource(database.config.typeorm);
const dataSource = new DataSource({
type: 'postgres',
entities: [],
subscribers: [],
migrations: [`${dist}/migrations` + '/*.{js,ts}'],
migrationsRun: false,
synchronize: false,
connectTimeoutMS: 10_000, // 10 seconds
parseInt8: true,
...(database.config.connectionType === 'url'
? { url: database.config.url }
: {
host: database.config.host,
port: database.config.port,
username: database.config.username,
password: database.config.password,
database: database.config.database,
}),
});
await dataSource.initialize();
await dataSource.runMigrations(options);
await dataSource.destroy();

2
server/src/services/backup.service.ts
View File

@ -70,7 +70,7 @@ export class BackupService extends BaseService {
async handleBackupDatabase(): Promise<JobStatus> {
this.logger.debug(`Database Backup Started`);
const { database } = this.configRepository.getEnv();
const config = database.config.typeorm;
const config = database.config;
const isUrlConnection = config.connectionType === 'url';

66
server/src/services/database.service.spec.ts
View File

@ -53,22 +53,12 @@ describe(DatabaseService.name, () => {
mockEnvData({
database: {
config: {
kysely: {
host: 'database',
port: 5432,
user: 'postgres',
password: 'postgres',
database: 'immich',
},
typeorm: {
connectionType: 'parts',
type: 'postgres',
host: 'database',
port: 5432,
username: 'postgres',
password: 'postgres',
database: 'immich',
},
connectionType: 'parts',
host: 'database',
port: 5432,
username: 'postgres',
password: 'postgres',
database: 'immich',
},
skipMigrations: false,
vectorExtension: extension,
@ -292,22 +282,12 @@ describe(DatabaseService.name, () => {
mockEnvData({
database: {
config: {
kysely: {
host: 'database',
port: 5432,
user: 'postgres',
password: 'postgres',
database: 'immich',
},
typeorm: {
connectionType: 'parts',
type: 'postgres',
host: 'database',
port: 5432,
username: 'postgres',
password: 'postgres',
database: 'immich',
},
connectionType: 'parts',
host: 'database',
port: 5432,
username: 'postgres',
password: 'postgres',
database: 'immich',
},
skipMigrations: true,
vectorExtension: DatabaseExtension.VECTORS,
@ -325,22 +305,12 @@ describe(DatabaseService.name, () => {
mockEnvData({
database: {
config: {
kysely: {
host: 'database',
port: 5432,
user: 'postgres',
password: 'postgres',
database: 'immich',
},
typeorm: {
connectionType: 'parts',
type: 'postgres',
host: 'database',
port: 5432,
username: 'postgres',
password: 'postgres',
database: 'immich',
},
connectionType: 'parts',
host: 'database',
port: 5432,
username: 'postgres',
password: 'postgres',
database: 'immich',
},
skipMigrations: true,
vectorExtension: DatabaseExtension.VECTOR,

83
server/src/utils/database.spec.ts Normal file
View File

@ -0,0 +1,83 @@
import { asPostgresConnectionConfig } from 'src/utils/database';
describe('database utils', () => {
describe('asPostgresConnectionConfig', () => {
it('should handle sslmode=require', () => {
expect(
asPostgresConnectionConfig({
connectionType: 'url',
url: 'postgres://postgres1:postgres2@database1:54320/immich?sslmode=require',
}),
).toMatchObject({ ssl: {} });
});
it('should handle sslmode=prefer', () => {
expect(
asPostgresConnectionConfig({
connectionType: 'url',
url: 'postgres://postgres1:postgres2@database1:54320/immich?sslmode=prefer',
}),
).toMatchObject({ ssl: {} });
});
it('should handle sslmode=verify-ca', () => {
expect(
asPostgresConnectionConfig({
connectionType: 'url',
url: 'postgres://postgres1:postgres2@database1:54320/immich?sslmode=verify-ca',
}),
).toMatchObject({ ssl: {} });
});
it('should handle sslmode=verify-full', () => {
expect(
asPostgresConnectionConfig({
connectionType: 'url',
url: 'postgres://postgres1:postgres2@database1:54320/immich?sslmode=verify-full',
}),
).toMatchObject({ ssl: {} });
});
it('should handle sslmode=no-verify', () => {
expect(
asPostgresConnectionConfig({
connectionType: 'url',
url: 'postgres://postgres1:postgres2@database1:54320/immich?sslmode=no-verify',
}),
).toMatchObject({ ssl: { rejectUnauthorized: false } });
});
it('should handle ssl=true', () => {
expect(
asPostgresConnectionConfig({
connectionType: 'url',
url: 'postgres://postgres1:postgres2@database1:54320/immich?ssl=true',
}),
).toMatchObject({ ssl: true });
});
it('should reject invalid ssl', () => {
expect(() =>
asPostgresConnectionConfig({
connectionType: 'url',
url: 'postgres://postgres1:postgres2@database1:54320/immich?ssl=invalid',
}),
).toThrowError('Invalid ssl option');
});
it('should handle socket: URLs', () => {
expect(
asPostgresConnectionConfig({ connectionType: 'url', url: 'socket:/run/postgresql?db=database1' }),
).toMatchObject({ host: '/run/postgresql', database: 'database1' });
});
it('should handle sockets in postgres: URLs', () => {
expect(
asPostgresConnectionConfig({ connectionType: 'url', url: 'postgres:///database2?host=/path/to/socket' }),
).toMatchObject({
host: '/path/to/socket',
database: 'database2',
});
});
});
});

62
server/src/utils/database.ts
View File

@ -13,33 +13,57 @@ import {
} from 'kysely';
import { PostgresJSDialect } from 'kysely-postgres-js';
import { jsonArrayFrom, jsonObjectFrom } from 'kysely/helpers/postgres';
import { parse } from 'pg-connection-string';
import postgres, { Notice } from 'postgres';
import { columns, Exif, Person } from 'src/database';
import { DB } from 'src/db';
import { AssetFileType } from 'src/enum';
import { TimeBucketSize } from 'src/repositories/asset.repository';
import { AssetSearchBuilderOptions } from 'src/repositories/search.repository';
import { DatabaseConnectionParams } from 'src/types';
type Ssl = 'require' | 'allow' | 'prefer' | 'verify-full' | boolean | object;
export type PostgresConnectionConfig = {
host?: string;
password?: string;
user?: string;
port?: number;
database?: string;
max?: number;
client_encoding?: string;
ssl?: Ssl;
application_name?: string;
fallback_application_name?: string;
options?: string;
};
export const isValidSsl = (ssl?: string | boolean | object): ssl is Ssl =>
const isValidSsl = (ssl?: string | boolean | object): ssl is Ssl =>
typeof ssl !== 'string' || ssl === 'require' || ssl === 'allow' || ssl === 'prefer' || ssl === 'verify-full';
export const getKyselyConfig = (options: PostgresConnectionConfig): KyselyConfig => {
export const asPostgresConnectionConfig = (params: DatabaseConnectionParams) => {
if (params.connectionType === 'parts') {
return {
host: params.host,
port: params.port,
username: params.username,
password: params.password,
database: params.database,
ssl: undefined,
};
}
const { host, port, user, password, database, ...rest } = parse(params.url);
let ssl: Ssl | undefined;
if (rest.ssl) {
if (!isValidSsl(rest.ssl)) {
throw new Error(`Invalid ssl option: ${rest.ssl}`);
}
ssl = rest.ssl;
}
return {
host: host ?? undefined,
port: port ? Number(port) : undefined,
username: user,
password,
database: database ?? undefined,
ssl,
};
};
export const getKyselyConfig = (
params: DatabaseConnectionParams,
options: Partial<postgres.Options<Record<string, postgres.PostgresType>>> = {},
): KyselyConfig => {
const config = asPostgresConnectionConfig(params);
return {
dialect: new PostgresJSDialect({
postgres: postgres({
@ -66,6 +90,12 @@ export const getKyselyConfig = (options: PostgresConnectionConfig): KyselyConfig
connection: {
TimeZone: 'UTC',
},
host: config.host,
port: config.port,
username: config.username,
password: config.password,
database: config.database,
ssl: config.ssl,
...options,
}),
}),

12
server/test/medium/globalSetup.ts
View File

@ -1,5 +1,4 @@
import { Kysely } from 'kysely';
import { parse } from 'pg-connection-string';
import { DB } from 'src/db';
import { ConfigRepository } from 'src/repositories/config.repository';
import { DatabaseRepository } from 'src/repositories/database.repository';
@ -37,19 +36,10 @@ const globalSetup = async () => {
const postgresPort = postgresContainer.getMappedPort(5432);
const postgresUrl = `postgres://postgres:postgres@localhost:${postgresPort}/immich`;
const parsed = parse(postgresUrl);
process.env.IMMICH_TEST_POSTGRES_URL = postgresUrl;
const db = new Kysely<DB>(
getKyselyConfig({
...parsed,
ssl: false,
host: parsed.host ?? undefined,
port: parsed.port ? Number(parsed.port) : undefined,
database: parsed.database ?? undefined,
}),
);
const db = new Kysely<DB>(getKyselyConfig({ connectionType: 'url', url: postgresUrl }));
const configRepository = new ConfigRepository();
const logger = new LoggingRepository(undefined, configRepository);

19
server/test/repositories/config.repository.mock.ts
View File

@ -21,19 +21,12 @@ const envData: EnvData = {
database: {
config: {
kysely: { database: 'immich', host: 'database', port: 5432 },
typeorm: {
connectionType: 'parts',
database: 'immich',
type: 'postgres',
host: 'database',
port: 5432,
username: 'postgres',
password: 'postgres',
name: 'immich',
synchronize: false,
migrationsRun: true,
},
connectionType: 'parts',
database: 'immich',
host: 'database',
port: 5432,
username: 'postgres',
password: 'postgres',
},
skipMigrations: false,

28
server/test/utils.ts
View File

@ -1,9 +1,9 @@
import { ClassConstructor } from 'class-transformer';
import { Kysely, sql } from 'kysely';
import { Kysely } from 'kysely';
import { ChildProcessWithoutNullStreams } from 'node:child_process';
import { Writable } from 'node:stream';
import { parse } from 'pg-connection-string';
import { PNG } from 'pngjs';
import postgres from 'postgres';
import { DB } from 'src/db';
import { AccessRepository } from 'src/repositories/access.repository';
import { ActivityRepository } from 'src/repositories/activity.repository';
@ -49,7 +49,7 @@ import { VersionHistoryRepository } from 'src/repositories/version-history.repos
import { ViewRepository } from 'src/repositories/view-repository';
import { BaseService } from 'src/services/base.service';
import { RepositoryInterface } from 'src/types';
import { getKyselyConfig } from 'src/utils/database';
import { asPostgresConnectionConfig, getKyselyConfig } from 'src/utils/database';
import { IAccessRepositoryMock, newAccessRepositoryMock } from 'test/repositories/access.repository.mock';
import { newAssetRepositoryMock } from 'test/repositories/asset.repository.mock';
import { newConfigRepositoryMock } from 'test/repositories/config.repository.mock';
@ -297,24 +297,20 @@ function* newPngFactory() {
const pngFactory = newPngFactory();
const withDatabase = (url: string, name: string) => url.replace('/immich', `/${name}`);
export const getKyselyDB = async (suffix?: string): Promise<Kysely<DB>> => {
const parsed = parse(process.env.IMMICH_TEST_POSTGRES_URL!);
const testUrl = process.env.IMMICH_TEST_POSTGRES_URL!;
const sql = postgres({
...asPostgresConnectionConfig({ connectionType: 'url', url: withDatabase(testUrl, 'postgres') }),
max: 1,
});
const parsedOptions = {
...parsed,
ssl: false,
host: parsed.host ?? undefined,
port: parsed.port ? Number(parsed.port) : undefined,
database: parsed.database ?? undefined,
};
const kysely = new Kysely<DB>(getKyselyConfig({ ...parsedOptions, max: 1, database: 'postgres' }));
const randomSuffix = Math.random().toString(36).slice(2, 7);
const dbName = `immich_${suffix ?? randomSuffix}`;
await sql.unsafe(`CREATE DATABASE ${dbName} WITH TEMPLATE immich OWNER postgres;`);
await sql.raw(`CREATE DATABASE ${dbName} WITH TEMPLATE immich OWNER postgres;`).execute(kysely);
return new Kysely<DB>(getKyselyConfig({ ...parsedOptions, database: dbName }));
return new Kysely<DB>(getKyselyConfig({ connectionType: 'url', url: withDatabase(testUrl, dbName) }));
};
export const newRandomImage = () => {