From 634c8808a1ce84906774ecb1ad20616f16bfd5d2 Mon Sep 17 00:00:00 2001 From: Fred Heinecke Date: Tue, 22 Apr 2025 21:55:44 +0000 Subject: [PATCH 1/3] [v5] Added support for storing images in S3 Signed-off-by: Fred Heinecke --- api/src/db/index.ts | 165 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 154 insertions(+), 11 deletions(-) diff --git a/api/src/db/index.ts b/api/src/db/index.ts index 0935a8c9..036cabeb 100644 --- a/api/src/db/index.ts +++ b/api/src/db/index.ts @@ -1,19 +1,162 @@ +import dns from "node:dns"; +import net from "node:net"; +import os from "node:os"; +import path from "node:path"; +import tls, { type ConnectionOptions } from "node:tls"; import { sql } from "drizzle-orm"; import { drizzle } from "drizzle-orm/node-postgres"; import { migrate as migrateDb } from "drizzle-orm/node-postgres/migrator"; +import type { PoolConfig } from "pg"; import * as schema from "./schema"; -const dbConfig = { - user: process.env.POSTGRES_USER ?? "kyoo", - password: process.env.POSTGRES_PASSWORD ?? "password", - database: process.env.POSTGRES_DB ?? "kyoo", - host: process.env.POSTGRES_SERVER ?? "postgres", - port: Number(process.env.POSTGRES_PORT) || 5432, - ssl: false, -}; +async function getPostgresConfig(): Promise { + const config: PoolConfig = { + connectionString: process.env.POSTGRES_URL, + host: process.env.PGHOST ?? process.env.POSTGRES_SERVER ?? "postgres", + port: Number(process.env.PGPORT ?? process.env.POSTGRES_PORT) || 5432, + database: process.env.PGDATABASE ?? process.env.POSTGRES_DB ?? "kyoo", + user: process.env.PGUSER ?? process.env.POSTGRES_USER ?? "kyoo", + password: + process.env.PGPASSWORD ?? process.env.POSTGRES_PASSWORD ?? "password", + options: process.env.PGOPTIONS, + application_name: process.env.PGAPPNAME ?? "kyoo", + }; + + // Due to an upstream bug, if `ssl` is not falsey, an SSL connection will always be attempted. This means + // that non-SSL connection options under `ssl` (which is incorrectly named) cannot be set unless SSL is enabled. + if (!process.env.PGSSLMODE || process.env.PGSSLMODE === "disable") + return config; + + // Despite this field's name, it is used to configure everything below the application layer. + const ssl: ConnectionOptions = { + timeout: + (process.env.PGCONNECT_TIMEOUT && + Number(process.env.PGCONNECT_TIMEOUT)) || + undefined, + minVersion: process.env.PGSSLMINPROTOCOLVERSION as tls.SecureVersion, + maxVersion: process.env.PGSSLMAXPROTOCOLVERSION as tls.SecureVersion, + }; + + // If the config is a hostname and the host address is set, use a custom lookup function + if (net.isIP(config.host ?? "") === 0 && process.env.PGHOSTADDR) { + const ipVersion = net.isIP(process.env.PGHOSTADDR); + if (ipVersion === 0) { + throw new Error( + `PGHOSTADDR is not a valid IP address: ${process.env.PGHOSTADDR}`, + ); + } + + (config.ssl as ConnectionOptions).lookup = ( + hostname: string, + options: dns.LookupOptions, + callback: ( + err: NodeJS.ErrnoException | null, + address: string | dns.LookupAddress[], + family?: number, + ) => void, + ) => { + if (hostname !== config.host) { + dns.lookup(hostname, options, callback); + return; + } + + return callback(null, process.env.PGHOSTADDR as string, ipVersion); + }; + } + + if (process.env.PGPASSFILE || !process.env.PGPASSWORD) { + const file = Bun.file( + process.env.PGPASSFILE ?? path.join(os.homedir(), ".pgpass"), + ); + if (await file.exists()) { + config.password = await file.text(); + } + } + + // Handle https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLROOTCERT + if (process.env.PGSSLROOTCERT === "system") { + // Bun does not currently support getCACertificates. Until this is supported, + // use the closest equivalent, which loads the bundled CA certs rather than the system CA certs. + // ssl.ca = tls.getCACertificates("system"); + ssl.ca = [...tls.rootCertificates]; + + if (!process.env.PGSSLMODE) { + process.env.PGSSLMODE = "verify-full"; + } + if (process.env.PGSSLMODE && process.env.PGSSLMODE !== "verify-full") { + throw new Error( + "PGSSLROOTCERT=system is only supported with PGSSLMODE=verify-full. See Postgres docs for details.", + ); + } + } else { + const file = Bun.file( + process.env.PGSSLROOTCERT ?? + path.join(os.homedir(), ".postgresql", "root.crt"), + ); + if (await file.exists()) { + ssl.ca = await file.text(); + } + } + + // TODO support CRLs. This requires verifying the contents against symlink hashes prepared by `openssl c_rehash`, + // as described in the postgres docs for PGSSLCRL and PGSSLCRLDIR. This isn't terribly common, so it's not currently + // implemented. + + let file = Bun.file( + process.env.PGSSLCERT ?? + path.join(os.homedir(), ".postgresql", "postgresql.crt"), + ); + if (await file.exists()) { + ssl.cert = await file.text(); + } + + file = Bun.file( + process.env.PGSSLKEY ?? + path.join(os.homedir(), ".postgresql", "postgresql.key"), + ); + if (await file.exists()) { + ssl.key = [ + { + pem: await file.text(), + }, + ]; + } + + if (process.env.PGSSLMODE) { + switch (process.env.PGSSLMODE) { + // Disable is handled above, gateing the configurating of any SSL options. + // Allow and prefer are not currently supported. Supporting them would require + // either mulitiple attempted connections, or changes upstream to the postgres driver. + case "require": + ssl.checkServerIdentity = (_host, _cert) => { + return undefined; + }; + break; + case "verify-ca": + ssl.rejectUnauthorized = true; + ssl.checkServerIdentity = (_host, _cert) => { + return undefined; + }; + break; + case "verify-full": + ssl.rejectUnauthorized = true; + break; + default: + ssl.rejectUnauthorized = false; + } + } + + if (process.env.PGSSLSNI !== "0") { + ssl.servername = config.host; + } + + config.ssl = ssl; + return config; +} + export const db = drizzle({ schema, - connection: dbConfig, + connection: await getPostgresConfig(), casing: "snake_case", }); @@ -22,14 +165,14 @@ export const migrate = async () => { sql.raw(` create extension if not exists pg_trgm; SET pg_trgm.word_similarity_threshold = 0.4; - ALTER DATABASE "${dbConfig.database}" SET pg_trgm.word_similarity_threshold = 0.4; + ALTER DATABASE "${(await getPostgresConfig()).database}" SET pg_trgm.word_similarity_threshold = 0.4; `), ); await migrateDb(db, { migrationsSchema: "kyoo", migrationsFolder: "./drizzle", }); - console.log(`Database ${dbConfig.database} migrated!`); + console.log(`Database ${(await getPostgresConfig()).database} migrated!`); }; export type Transaction = From d0a1ee848f0627e272afc340a57a39bdc66e5f83 Mon Sep 17 00:00:00 2001 From: Fred Heinecke Date: Wed, 23 Apr 2025 19:44:02 +0000 Subject: [PATCH 2/3] bug fixes, PR feedback, remove some vars Signed-off-by: Fred Heinecke --- .github/workflows/api-test.yml | 2 +- api/.env.example | 20 ++++++--- api/src/db/index.ts | 75 +++++++--------------------------- 3 files changed, 31 insertions(+), 66 deletions(-) diff --git a/.github/workflows/api-test.yml b/.github/workflows/api-test.yml index 46e170f0..a8238949 100644 --- a/.github/workflows/api-test.yml +++ b/.github/workflows/api-test.yml @@ -36,4 +36,4 @@ jobs: working-directory: ./api run: bun test env: - POSTGRES_SERVER: localhost + PGHOST: localhost diff --git a/api/.env.example b/api/.env.example index 7ef45abf..cd4fc3cc 100644 --- a/api/.env.example +++ b/api/.env.example @@ -14,8 +14,18 @@ AUTH_SERVER=http://auth:4568 IMAGES_PATH=./images -POSTGRES_USER=kyoo -POSTGRES_PASSWORD=password -POSTGRES_DB=kyooDB -POSTGRES_SERVER=postgres -POSTGRES_PORT=5432 +# It is recommended to use the below PG environment variables when possible. +POSTGRES_URL=postgres://user:password@hostname:port/dbname?sslmode=verify-full&sslrootcert=/path/to/server.crt&sslcert=/path/to/client.crt&sslkey=/path/to/client.key +# The behavior of the below variables match what is documented here: +# https://www.postgresql.org/docs/current/libpq-envars.html +PGUSER=kyoo +PGPASSWORD=password +PGDB=kyooDB +PGSERVER=postgres +PGPORT=5432 +PGOPTIONS=-c search_path=kyoo,public +PGPASSFILE=/my/password # Takes precedence over PGPASSWORD. New line characters are not trimmed. +PGSSLMODE=verify-full +PGSSLROOTCERT=/my/serving.crt +PGSSLCERT=/my/client.crt +PGSSLKEY=/my/client.key diff --git a/api/src/db/index.ts b/api/src/db/index.ts index 036cabeb..48b9e195 100644 --- a/api/src/db/index.ts +++ b/api/src/db/index.ts @@ -1,5 +1,3 @@ -import dns from "node:dns"; -import net from "node:net"; import os from "node:os"; import path from "node:path"; import tls, { type ConnectionOptions } from "node:tls"; @@ -12,12 +10,11 @@ import * as schema from "./schema"; async function getPostgresConfig(): Promise { const config: PoolConfig = { connectionString: process.env.POSTGRES_URL, - host: process.env.PGHOST ?? process.env.POSTGRES_SERVER ?? "postgres", - port: Number(process.env.PGPORT ?? process.env.POSTGRES_PORT) || 5432, - database: process.env.PGDATABASE ?? process.env.POSTGRES_DB ?? "kyoo", - user: process.env.PGUSER ?? process.env.POSTGRES_USER ?? "kyoo", - password: - process.env.PGPASSWORD ?? process.env.POSTGRES_PASSWORD ?? "password", + host: process.env.PGHOST ?? "postgres", + port: Number(process.env.PGPORT) || 5432, + database: process.env.PGDATABASE ?? "kyoo", + user: process.env.PGUSER ?? "kyoo", + password: process.env.PGPASSWORD ?? "password", options: process.env.PGOPTIONS, application_name: process.env.PGAPPNAME ?? "kyoo", }; @@ -28,41 +25,7 @@ async function getPostgresConfig(): Promise { return config; // Despite this field's name, it is used to configure everything below the application layer. - const ssl: ConnectionOptions = { - timeout: - (process.env.PGCONNECT_TIMEOUT && - Number(process.env.PGCONNECT_TIMEOUT)) || - undefined, - minVersion: process.env.PGSSLMINPROTOCOLVERSION as tls.SecureVersion, - maxVersion: process.env.PGSSLMAXPROTOCOLVERSION as tls.SecureVersion, - }; - - // If the config is a hostname and the host address is set, use a custom lookup function - if (net.isIP(config.host ?? "") === 0 && process.env.PGHOSTADDR) { - const ipVersion = net.isIP(process.env.PGHOSTADDR); - if (ipVersion === 0) { - throw new Error( - `PGHOSTADDR is not a valid IP address: ${process.env.PGHOSTADDR}`, - ); - } - - (config.ssl as ConnectionOptions).lookup = ( - hostname: string, - options: dns.LookupOptions, - callback: ( - err: NodeJS.ErrnoException | null, - address: string | dns.LookupAddress[], - family?: number, - ) => void, - ) => { - if (hostname !== config.host) { - dns.lookup(hostname, options, callback); - return; - } - - return callback(null, process.env.PGHOSTADDR as string, ipVersion); - }; - } + const ssl: ConnectionOptions = {}; if (process.env.PGPASSFILE || !process.env.PGPASSWORD) { const file = Bun.file( @@ -115,11 +78,7 @@ async function getPostgresConfig(): Promise { path.join(os.homedir(), ".postgresql", "postgresql.key"), ); if (await file.exists()) { - ssl.key = [ - { - pem: await file.text(), - }, - ]; + ssl.key = await file.text(); } if (process.env.PGSSLMODE) { @@ -127,11 +86,6 @@ async function getPostgresConfig(): Promise { // Disable is handled above, gateing the configurating of any SSL options. // Allow and prefer are not currently supported. Supporting them would require // either mulitiple attempted connections, or changes upstream to the postgres driver. - case "require": - ssl.checkServerIdentity = (_host, _cert) => { - return undefined; - }; - break; case "verify-ca": ssl.rejectUnauthorized = true; ssl.checkServerIdentity = (_host, _cert) => { @@ -142,21 +96,22 @@ async function getPostgresConfig(): Promise { ssl.rejectUnauthorized = true; break; default: + ssl.checkServerIdentity = (_host, _cert) => { + return undefined; + }; ssl.rejectUnauthorized = false; } } - if (process.env.PGSSLSNI !== "0") { - ssl.servername = config.host; - } - config.ssl = ssl; return config; } +const postgresConfig = await getPostgresConfig(); + export const db = drizzle({ schema, - connection: await getPostgresConfig(), + connection: postgresConfig, casing: "snake_case", }); @@ -165,14 +120,14 @@ export const migrate = async () => { sql.raw(` create extension if not exists pg_trgm; SET pg_trgm.word_similarity_threshold = 0.4; - ALTER DATABASE "${(await getPostgresConfig()).database}" SET pg_trgm.word_similarity_threshold = 0.4; + ALTER DATABASE "${postgresConfig.database}" SET pg_trgm.word_similarity_threshold = 0.4; `), ); await migrateDb(db, { migrationsSchema: "kyoo", migrationsFolder: "./drizzle", }); - console.log(`Database ${(await getPostgresConfig()).database} migrated!`); + console.log(`Database ${postgresConfig.database} migrated!`); }; export type Transaction = From 9945d49be91478fe03fc7cf03109bdafebe74035 Mon Sep 17 00:00:00 2001 From: solidDoWant Date: Wed, 23 Apr 2025 21:08:18 +0000 Subject: [PATCH 3/3] comment out env vars Signed-off-by: solidDoWant --- api/.env.example | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/api/.env.example b/api/.env.example index cd4fc3cc..ac9db9df 100644 --- a/api/.env.example +++ b/api/.env.example @@ -15,7 +15,7 @@ AUTH_SERVER=http://auth:4568 IMAGES_PATH=./images # It is recommended to use the below PG environment variables when possible. -POSTGRES_URL=postgres://user:password@hostname:port/dbname?sslmode=verify-full&sslrootcert=/path/to/server.crt&sslcert=/path/to/client.crt&sslkey=/path/to/client.key +# POSTGRES_URL=postgres://user:password@hostname:port/dbname?sslmode=verify-full&sslrootcert=/path/to/server.crt&sslcert=/path/to/client.crt&sslkey=/path/to/client.key # The behavior of the below variables match what is documented here: # https://www.postgresql.org/docs/current/libpq-envars.html PGUSER=kyoo @@ -23,9 +23,9 @@ PGPASSWORD=password PGDB=kyooDB PGSERVER=postgres PGPORT=5432 -PGOPTIONS=-c search_path=kyoo,public -PGPASSFILE=/my/password # Takes precedence over PGPASSWORD. New line characters are not trimmed. -PGSSLMODE=verify-full -PGSSLROOTCERT=/my/serving.crt -PGSSLCERT=/my/client.crt -PGSSLKEY=/my/client.key +# PGOPTIONS=-c search_path=kyoo,public +# PGPASSFILE=/my/password # Takes precedence over PGPASSWORD. New line characters are not trimmed. +# PGSSLMODE=verify-full +# PGSSLROOTCERT=/my/serving.crt +# PGSSLCERT=/my/client.crt +# PGSSLKEY=/my/client.key