Cutlery/immich
1
0
Fork 1
mirror of https://github.com/immich-app/immich.git synced 2025-05-24 01:12:58 -04:00
Code Issues Packages Projects Releases Wiki Activity

feat: vectorchord (#18042)

Browse Source 
* wip

auto-detect available extensions

auto-recovery, fix reindexing check

use original image for ml

* set probes

* update image for sql checker

update images for gha

* cascade

* fix new instance

* accurate dummy vector

* simplify dummy

* preexisiting pg docs

* handle different db name

* maybe fix sql generation

* revert refreshfaces sql change

* redundant switch

* outdated message

* update docker compose files

* Update docs/docs/administration/postgres-standalone.md

Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>

* tighten range

* avoid always printing "vector reindexing complete"

* remove nesting

* use new images

* add vchord to unit tests

* debug e2e image

* mention 1.107.2 in startup error

* support new vchord versions

---------

Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>
This commit is contained in:
Mert 2025-05-20 09:36:43 -04:00 committed by GitHub
parent fe71894308
commit 0d773af6c3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
35 changed files with 572 additions and 444 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/test.yml vendored
View File

@ -643,7 +643,7 @@ jobs:
contents: read
services:
postgres:
image: tensorchord/pgvecto-rs:pg14-v0.2.0@sha256:739cdd626151ff1f796dc95a6591b55a714f341c737e27f045019ceabf8e8c52
image: ghcr.io/immich-app/postgres:14
env:
POSTGRES_PASSWORD: postgres
POSTGRES_USER: postgres

20
docker/docker-compose.dev.yml
View File

@ -122,7 +122,7 @@ services:
database:
container_name: immich_postgres
image: tensorchord/pgvecto-rs:pg14-v0.2.0@sha256:739cdd626151ff1f796dc95a6591b55a714f341c737e27f045019ceabf8e8c52
image: ghcr.io/immich-app/postgres:14-vectorchord0.3.0-pgvectors0.2.0
env_file:
- .env
environment:
@ -134,24 +134,6 @@ services:
- ${UPLOAD_LOCATION}/postgres:/var/lib/postgresql/data
ports:
- 5432:5432
healthcheck:
test: >-
pg_isready --dbname="$${POSTGRES_DB}" --username="$${POSTGRES_USER}" || exit 1;
Chksum="$$(psql --dbname="$${POSTGRES_DB}" --username="$${POSTGRES_USER}" --tuples-only --no-align
--command='SELECT COALESCE(SUM(checksum_failures), 0) FROM pg_stat_database')";
echo "checksum failure count is $$Chksum";
[ "$$Chksum" = '0' ] || exit 1
interval: 5m
start_interval: 30s
start_period: 5m
command: >-
postgres
-c shared_preload_libraries=vectors.so
-c 'search_path="$$user", public, vectors'
-c logging_collector=on
-c max_wal_size=2GB
-c shared_buffers=512MB
-c wal_compression=on
# set IMMICH_TELEMETRY_INCLUDE=all in .env to enable metrics
# immich-prometheus:

20
docker/docker-compose.prod.yml
View File

@ -63,7 +63,7 @@ services:
database:
container_name: immich_postgres
image: tensorchord/pgvecto-rs:pg14-v0.2.0@sha256:739cdd626151ff1f796dc95a6591b55a714f341c737e27f045019ceabf8e8c52
image: ghcr.io/immich-app/postgres:14-vectorchord0.3.0-pgvectors0.2.0
env_file:
- .env
environment:
@ -75,24 +75,6 @@ services:
- ${UPLOAD_LOCATION}/postgres:/var/lib/postgresql/data
ports:
- 5432:5432
healthcheck:
test: >-
pg_isready --dbname="$${POSTGRES_DB}" --username="$${POSTGRES_USER}" || exit 1;
Chksum="$$(psql --dbname="$${POSTGRES_DB}" --username="$${POSTGRES_USER}" --tuples-only --no-align
--command='SELECT COALESCE(SUM(checksum_failures), 0) FROM pg_stat_database')";
echo "checksum failure count is $$Chksum";
[ "$$Chksum" = '0' ] || exit 1
interval: 5m
start_interval: 30s
start_period: 5m
command: >-
postgres
-c shared_preload_libraries=vectors.so
-c 'search_path="$$user", public, vectors'
-c logging_collector=on
-c max_wal_size=2GB
-c shared_buffers=512MB
-c wal_compression=on
restart: always
# set IMMICH_TELEMETRY_INCLUDE=all in .env to enable metrics

22
docker/docker-compose.yml
View File

@ -56,7 +56,7 @@ services:
database:
container_name: immich_postgres
image: docker.io/tensorchord/pgvecto-rs:pg14-v0.2.0@sha256:739cdd626151ff1f796dc95a6591b55a714f341c737e27f045019ceabf8e8c52
image: ghcr.io/immich-app/postgres:14-vectorchord0.3.0-pgvectors0.2.0
environment:
POSTGRES_PASSWORD: ${DB_PASSWORD}
POSTGRES_USER: ${DB_USERNAME}
@ -65,24 +65,8 @@ services:
volumes:
# Do not edit the next line. If you want to change the database storage location on your system, edit the value of DB_DATA_LOCATION in the .env file
- ${DB_DATA_LOCATION}:/var/lib/postgresql/data
healthcheck:
test: >-
pg_isready --dbname="$${POSTGRES_DB}" --username="$${POSTGRES_USER}" || exit 1;
Chksum="$$(psql --dbname="$${POSTGRES_DB}" --username="$${POSTGRES_USER}" --tuples-only --no-align
--command='SELECT COALESCE(SUM(checksum_failures), 0) FROM pg_stat_database')";
echo "checksum failure count is $$Chksum";
[ "$$Chksum" = '0' ] || exit 1
interval: 5m
start_interval: 30s
start_period: 5m
command: >-
postgres
-c shared_preload_libraries=vectors.so
-c 'search_path="$$user", public, vectors'
-c logging_collector=on
-c max_wal_size=2GB
-c shared_buffers=512MB
-c wal_compression=on
# change ssd below to hdd if you are using a hard disk drive or other slow storage
command: postgres -c config_file=/etc/postgresql/postgresql.ssd.conf
restart: always
volumes:

77
docs/docs/administration/postgres-standalone.md
View File

@ -10,12 +10,12 @@ Running with a pre-existing Postgres server can unlock powerful administrative f
## Prerequisites
You must install pgvecto.rs into your instance of Postgres using their [instructions][vectors-install]. After installation, add `shared_preload_libraries = 'vectors.so'` to your `postgresql.conf`. If you already have some `shared_preload_libraries` set, you can separate each extension with a comma. For example, `shared_preload_libraries = 'pg_stat_statements, vectors.so'`.
You must install VectorChord into your instance of Postgres using their [instructions][vchord-install]. After installation, add `shared_preload_libraries = 'vchord.so'` to your `postgresql.conf`. If you already have some `shared_preload_libraries` set, you can separate each extension with a comma. For example, `shared_preload_libraries = 'pg_stat_statements, vchord.so'`.
:::note
Immich is known to work with Postgres versions 14, 15, and 16. Earlier versions are unsupported. Postgres 17 is nominally compatible, but pgvecto.rs does not have prebuilt images or packages for it as of writing.
Immich is known to work with Postgres versions 14, 15, 16 and 17. Earlier versions are unsupported.
Make sure the installed version of pgvecto.rs is compatible with your version of Immich. The current accepted range for pgvecto.rs is `>= 0.2.0, < 0.4.0`.
Make sure the installed version of VectorChord is compatible with your version of Immich. The current accepted range for VectorChord is `>= 0.3.0, < 1.0.0`.
:::
## Specifying the connection URL
@ -53,16 +53,75 @@ CREATE DATABASE <immichdatabasename>;
\c <immichdatabasename>
BEGIN;
ALTER DATABASE <immichdatabasename> OWNER TO <immichdbusername>;
CREATE EXTENSION vectors;
CREATE EXTENSION vchord CASCADE;
CREATE EXTENSION earthdistance CASCADE;
ALTER DATABASE <immichdatabasename> SET search_path TO "$user", public, vectors;
ALTER SCHEMA vectors OWNER TO <immichdbusername>;
COMMIT;
```
### Updating pgvecto.rs
### Updating VectorChord
When installing a new version of pgvecto.rs, you will need to manually update the extension by connecting to the Immich database and running `ALTER EXTENSION vectors UPDATE;`.
When installing a new version of VectorChord, you will need to manually update the extension by connecting to the Immich database and running `ALTER EXTENSION vchord UPDATE;`.
## Migrating to VectorChord
VectorChord is the successor extension to pgvecto.rs, allowing for higher performance, lower memory usage and higher quality results for smart search and facial recognition.
### Migrating from pgvecto.rs
Support for pgvecto.rs will be dropped in a later release, hence we recommend all users currently using pgvecto.rs to migrate to VectorChord at their convenience. There are two primary approaches to do so.
The easiest option is to have both extensions installed during the migration:
1. Ensure you still have pgvecto.rs installed
2. [Install VectorChord][vchord-install]
3. Add `shared_preload_libraries= 'vchord.so, vectors.so'` to your `postgresql.conf`, making sure to include _both_ `vchord.so` and `vectors.so`. You may include other libraries here as well if needed
4. If Immich does not have superuser permissions, run the SQL command `CREATE EXTENSION vchord CASCADE;` using psql or your choice of database client
5. Start Immich and wait for the logs `Reindexed face_index` and `Reindexed clip_index` to be output
6. Remove the `vectors.so` entry from the `shared_preload_libraries` setting
7. Uninstall pgvecto.rs (e.g. `apt-get purge vectors-pg14` on Debian-based environments, replacing `pg14` as appropriate)
If it is not possible to have both VectorChord and pgvector.s installed at the same time, you can perform the migration with more manual steps:
1. While pgvecto.rs is still installed, run the following SQL command using psql or your choice of database client. Take note of the number outputted by this command as you will need it later
```sql
SELECT atttypmod as dimsize
FROM pg_attribute f
JOIN pg_class c ON c.oid = f.attrelid
WHERE c.relkind = 'r'::char
AND f.attnum > 0
AND c.relname = 'smart_search'::text
AND f.attname = 'embedding'::text;
```
2. Remove references to pgvecto.rs using the below SQL commands
```sql
DROP INDEX IF EXISTS clip_index;
DROP INDEX IF EXISTS face_index;
ALTER TABLE smart_search ALTER COLUMN embedding SET DATA TYPE real[];
ALTER TABLE face_search ALTER COLUMN embedding SET DATA TYPE real[];
```
3. [Install VectorChord][vchord-install]
4. Change the columns back to the appropriate vector types, replacing `<number>` with the number from step 1
```sql
CREATE EXTENSION IF NOT EXISTS vchord CASCADE;
ALTER TABLE smart_search ALTER COLUMN embedding SET DATA TYPE vector(<number>);
ALTER TABLE face_search ALTER COLUMN embedding SET DATA TYPE vector(512);
```
5. Start Immich and let it create new indices using VectorChord
### Migrating from pgvector
1. Ensure you have at least 0.7.0 of pgvector installed. If it is below that, please upgrade it and run the SQL command `ALTER EXTENSION vector UPDATE;` using psql or your choice of database client
2. Follow the Prerequisites to install VectorChord
3. If Immich does not have superuser permissions, run the SQL command `CREATE EXTENSION vchord CASCADE;`
4. Start Immich and let it create new indices using VectorChord
Note that VectorChord itself uses pgvector types, so you should not uninstall pgvector after following these steps.
### Common errors
@ -70,4 +129,4 @@ When installing a new version of pgvecto.rs, you will need to manually update th
If you get the error `driverError: error: permission denied for view pg_vector_index_stat`, you can fix this by connecting to the Immich database and running `GRANT SELECT ON TABLE pg_vector_index_stat TO <immichdbusername>;`.
[vectors-install]: https://docs.vectorchord.ai/getting-started/installation.html
[vchord-install]: https://docs.vectorchord.ai/vectorchord/getting-started/installation.html

2
docs/docs/features/searching.md
View File

@ -5,7 +5,7 @@ import TabItem from '@theme/TabItem';
Immich uses Postgres as its search database for both metadata and contextual CLIP search.
Contextual CLIP search is powered by the [pgvecto.rs](https://github.com/tensorchord/pgvecto.rs) extension, utilizing machine learning models like [CLIP](https://openai.com/research/clip) to provide relevant search results. This allows for freeform searches without requiring specific keywords in the image or video metadata.
Contextual CLIP search is powered by the [VectorChord](https://github.com/tensorchord/VectorChord) extension, utilizing machine learning models like [CLIP](https://openai.com/research/clip) to provide relevant search results. This allows for freeform searches without requiring specific keywords in the image or video metadata.
## Advanced Search Filters

6
docs/docs/install/environment-variables.md
View File

@ -73,7 +73,7 @@ Information on the current workers can be found [here](/docs/administration/jobs
## Database
| Variable | Description | Default | Containers |
| :---------------------------------- | :----------------------------------------------------------------------- | :----------: | :----------------------------- |
| :---------------------------------- | :--------------------------------------------------------------------------- | :--------: | :----------------------------- |
| `DB_URL` | Database URL | | server |
| `DB_HOSTNAME` | Database host | `database` | server |
| `DB_PORT` | Database port | `5432` | server |
@ -81,12 +81,12 @@ Information on the current workers can be found [here](/docs/administration/jobs
| `DB_PASSWORD` | Database password | `postgres` | server, database<sup>\*1</sup> |
| `DB_DATABASE_NAME` | Database name | `immich` | server, database<sup>\*1</sup> |
| `DB_SSL_MODE` | Database SSL mode | | server |
| `DB_VECTOR_EXTENSION`<sup>\*2</sup> | Database vector extension (one of [`pgvector`, `pgvecto.rs`]) | `pgvecto.rs` | server |
| `DB_VECTOR_EXTENSION`<sup>\*2</sup> | Database vector extension (one of [`vectorchord`, `pgvector`, `pgvecto.rs`]) | | server |
| `DB_SKIP_MIGRATIONS` | Whether to skip running migrations on startup (one of [`true`, `false`]) | `false` | server |
\*1: The values of `DB_USERNAME`, `DB_PASSWORD`, and `DB_DATABASE_NAME` are passed to the Postgres container as the variables `POSTGRES_USER`, `POSTGRES_PASSWORD`, and `POSTGRES_DB` in `docker-compose.yml`.
\*2: This setting cannot be changed after the server has successfully started up.
\*2: If not provided, the appropriate extension to use is auto-detected at startup by introspecting the database. When multiple extensions are installed, the order of preference is VectorChord, pgvecto.rs, pgvector.
:::info

4
e2e/docker-compose.yml
View File

@ -37,8 +37,8 @@ services:
image: redis:6.2-alpine@sha256:3211c33a618c457e5d241922c975dbc4f446d0bdb2dc75694f5573ef8e2d01fa
database:
image: tensorchord/pgvecto-rs:pg14-v0.2.0@sha256:739cdd626151ff1f796dc95a6591b55a714f341c737e27f045019ceabf8e8c52
command: -c fsync=off -c shared_preload_libraries=vectors.so
image: ghcr.io/immich-app/postgres:14
command: -c fsync=off -c shared_preload_libraries=vchord.so -c config_file=/var/lib/postgresql/data/postgresql.conf
environment:
POSTGRES_PASSWORD: postgres
POSTGRES_USER: postgres

17
server/src/constants.ts
View File

@ -1,9 +1,10 @@
import { Duration } from 'luxon';
import { readFileSync } from 'node:fs';
import { SemVer } from 'semver';
import { DatabaseExtension, ExifOrientation } from 'src/enum';
import { DatabaseExtension, ExifOrientation, VectorIndex } from 'src/enum';
export const POSTGRES_VERSION_RANGE = '>=14.0.0';
export const VECTORCHORD_VERSION_RANGE = '>=0.3 <1';
export const VECTORS_VERSION_RANGE = '>=0.2 <0.4';
export const VECTOR_VERSION_RANGE = '>=0.5 <1';
@ -20,8 +21,22 @@ export const EXTENSION_NAMES: Record<DatabaseExtension, string> = {
earthdistance: 'earthdistance',
vector: 'pgvector',
vectors: 'pgvecto.rs',
vchord: 'VectorChord',
} as const;
export const VECTOR_EXTENSIONS = [
DatabaseExtension.VECTORCHORD,
DatabaseExtension.VECTORS,
DatabaseExtension.VECTOR,
] as const;
export const VECTOR_INDEX_TABLES = {
[VectorIndex.CLIP]: 'smart_search',
[VectorIndex.FACE]: 'face_search',
} as const;
export const VECTORCHORD_LIST_SLACK_FACTOR = 1.2;
export const SALT_ROUNDS = 10;
export const IWorker = 'IWorker';

2
server/src/decorators.ts
View File

@ -116,7 +116,7 @@ export const DummyValue = {
DATE: new Date(),
TIME_BUCKET: '2024-01-01T00:00:00.000Z',
BOOLEAN: true,
VECTOR: '[1, 2, 3]',
VECTOR: JSON.stringify(Array.from({ length: 512 }, () => 0)),
};
export const GENERATE_SQL_KEY = 'generate-sql-key';

4
server/src/dtos/env.dto.ts
View File

@ -154,9 +154,9 @@ export class EnvDto {
@Optional()
DB_USERNAME?: string;
@IsEnum(['pgvector', 'pgvecto.rs'])
@IsEnum(['pgvector', 'pgvecto.rs', 'vectorchord'])
@Optional()
DB_VECTOR_EXTENSION?: 'pgvector' | 'pgvecto.rs';
DB_VECTOR_EXTENSION?: 'pgvector' | 'pgvecto.rs' | 'vectorchord';
@IsString()
@Optional()

1
server/src/enum.ts
View File

@ -414,6 +414,7 @@ export enum DatabaseExtension {
EARTH_DISTANCE = 'earthdistance',
VECTOR = 'vector',
VECTORS = 'vectors',
VECTORCHORD = 'vchord',
}
export enum BootstrapEventPriority {

6
server/src/migrations/1700713871511-UsePgVectors.ts
View File

@ -1,15 +1,13 @@
import { ConfigRepository } from 'src/repositories/config.repository';
import { getVectorExtension } from 'src/repositories/database.repository';
import { getCLIPModelInfo } from 'src/utils/misc';
import { MigrationInterface, QueryRunner } from 'typeorm';
const vectorExtension = new ConfigRepository().getEnv().database.vectorExtension;
export class UsePgVectors1700713871511 implements MigrationInterface {
name = 'UsePgVectors1700713871511';
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`SET search_path TO "$user", public, vectors`);
await queryRunner.query(`CREATE EXTENSION IF NOT EXISTS ${vectorExtension}`);
await queryRunner.query(`CREATE EXTENSION IF NOT EXISTS ${await getVectorExtension(queryRunner)}`);
const faceDimQuery = await queryRunner.query(`
SELECT CARDINALITY(embedding::real[]) as dimsize
FROM asset_faces

5
server/src/migrations/1700713994428-AddCLIPEmbeddingIndex.ts
View File

@ -1,13 +1,12 @@
import { ConfigRepository } from 'src/repositories/config.repository';
import { getVectorExtension } from 'src/repositories/database.repository';
import { vectorIndexQuery } from 'src/utils/database';
import { MigrationInterface, QueryRunner } from 'typeorm';
const vectorExtension = new ConfigRepository().getEnv().database.vectorExtension;
export class AddCLIPEmbeddingIndex1700713994428 implements MigrationInterface {
name = 'AddCLIPEmbeddingIndex1700713994428';
public async up(queryRunner: QueryRunner): Promise<void> {
const vectorExtension = await getVectorExtension(queryRunner);
await queryRunner.query(`SET search_path TO "$user", public, vectors`);
await queryRunner.query(vectorIndexQuery({ vectorExtension, table: 'smart_search', indexName: 'clip_index' }));

5
server/src/migrations/1700714033632-AddFaceEmbeddingIndex.ts
View File

@ -1,13 +1,12 @@
import { ConfigRepository } from 'src/repositories/config.repository';
import { getVectorExtension } from 'src/repositories/database.repository';
import { vectorIndexQuery } from 'src/utils/database';
import { MigrationInterface, QueryRunner } from 'typeorm';
const vectorExtension = new ConfigRepository().getEnv().database.vectorExtension;
export class AddFaceEmbeddingIndex1700714033632 implements MigrationInterface {
name = 'AddFaceEmbeddingIndex1700714033632';
public async up(queryRunner: QueryRunner): Promise<void> {
const vectorExtension = await getVectorExtension(queryRunner);
await queryRunner.query(`SET search_path TO "$user", public, vectors`);
await queryRunner.query(vectorIndexQuery({ vectorExtension, table: 'asset_faces', indexName: 'face_index' }));

7
server/src/migrations/1718486162779-AddFaceSearchRelation.ts
View File

@ -1,12 +1,11 @@
import { DatabaseExtension } from 'src/enum';
import { ConfigRepository } from 'src/repositories/config.repository';
import { getVectorExtension } from 'src/repositories/database.repository';
import { vectorIndexQuery } from 'src/utils/database';
import { MigrationInterface, QueryRunner } from 'typeorm';
const vectorExtension = new ConfigRepository().getEnv().database.vectorExtension;
export class AddFaceSearchRelation1718486162779 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
const vectorExtension = await getVectorExtension(queryRunner);
if (vectorExtension === DatabaseExtension.VECTORS) {
await queryRunner.query(`SET search_path TO "$user", public, vectors`);
}
@ -48,11 +47,11 @@ export class AddFaceSearchRelation1718486162779 implements MigrationInterface {
await queryRunner.query(`ALTER TABLE face_search ALTER COLUMN embedding SET DATA TYPE vector(512)`);
await queryRunner.query(vectorIndexQuery({ vectorExtension, table: 'smart_search', indexName: 'clip_index' }));
await queryRunner.query(vectorIndexQuery({ vectorExtension, table: 'face_search', indexName: 'face_index' }));
}
public async down(queryRunner: QueryRunner): Promise<void> {
const vectorExtension = await getVectorExtension(queryRunner);
if (vectorExtension === DatabaseExtension.VECTORS) {
await queryRunner.query(`SET search_path TO "$user", public, vectors`);
}

8
server/src/queries/database.repository.sql
View File

@ -11,11 +11,3 @@ WHERE
-- DatabaseRepository.getPostgresVersion
SHOW server_version
-- DatabaseRepository.shouldReindex
SELECT
idx_status
FROM
pg_vector_index_stat
WHERE
indexname = $1

15
server/src/queries/person.repository.sql
View File

@ -204,6 +204,21 @@ where
"person"."ownerId" = $3
and "asset_faces"."deletedAt" is null
-- PersonRepository.refreshFaces
with
"added_embeddings" as (
insert into
"face_search" ("faceId", "embedding")
values
($1, $2)
)
select
from
(
select
1
) as "dummy"
-- PersonRepository.getFacesByIds
select
"asset_faces".*,

24
server/src/queries/search.repository.sql
View File

@ -64,6 +64,9 @@ limit
$15
-- SearchRepository.searchSmart
begin
set
local vchordrq.probes = 1
select
"assets".*
from
@ -83,8 +86,12 @@ limit
$7
offset
$8
commit
-- SearchRepository.searchDuplicates
begin
set
local vchordrq.probes = 1
with
"cte" as (
select
@ -102,18 +109,22 @@ with
and "assets"."id" != $5::uuid
and "assets"."stackId" is null
order by
smart_search.embedding <=> $6
"distance"
limit
$7
$6
)
select
*
from
"cte"
where
"cte"."distance" <= $8
"cte"."distance" <= $7
commit
-- SearchRepository.searchFaces
begin
set
local vchordrq.probes = 1
with
"cte" as (
select
@ -129,16 +140,17 @@ with
"assets"."ownerId" = any ($2::uuid[])
and "assets"."deletedAt" is null
order by
face_search.embedding <=> $3
"distance"
limit
$4
$3
)
select
*
from
"cte"
where
"cte"."distance" <= $5
"cte"."distance" <= $4
commit
-- SearchRepository.searchPlaces
select

2
server/src/repositories/config.repository.spec.ts
View File

@ -89,7 +89,7 @@ describe('getEnv', () => {
password: 'postgres',
},
skipMigrations: false,
vectorExtension: 'vectors',
vectorExtension: undefined,
});
});

20
server/src/repositories/config.repository.ts
View File

@ -58,7 +58,7 @@ export interface EnvData {
database: {
config: DatabaseConnectionParams;
skipMigrations: boolean;
vectorExtension: VectorExtension;
vectorExtension?: VectorExtension;
};
licensePublicKey: {
@ -196,6 +196,22 @@ const getEnv = (): EnvData => {
ssl: dto.DB_SSL_MODE || undefined,
};
let vectorExtension: VectorExtension | undefined;
switch (dto.DB_VECTOR_EXTENSION) {
case 'pgvector': {
vectorExtension = DatabaseExtension.VECTOR;
break;
}
case 'pgvecto.rs': {
vectorExtension = DatabaseExtension.VECTORS;
break;
}
case 'vectorchord': {
vectorExtension = DatabaseExtension.VECTORCHORD;
break;
}
}
return {
host: dto.IMMICH_HOST,
port: dto.IMMICH_PORT || 2283,
@ -251,7 +267,7 @@ const getEnv = (): EnvData => {
database: {
config: databaseConnection,
skipMigrations: dto.DB_SKIP_MIGRATIONS ?? false,
vectorExtension: dto.DB_VECTOR_EXTENSION === 'pgvector' ? DatabaseExtension.VECTOR : DatabaseExtension.VECTORS,
vectorExtension,
},
licensePublicKey: isProd ? productionKeys : stagingKeys,

306
server/src/repositories/database.repository.ts
View File

@ -5,7 +5,16 @@ import { InjectKysely } from 'nestjs-kysely';
import { readdir } from 'node:fs/promises';
import { join, resolve } from 'node:path';
import semver from 'semver';
import { EXTENSION_NAMES, POSTGRES_VERSION_RANGE, VECTOR_VERSION_RANGE, VECTORS_VERSION_RANGE } from 'src/constants';
import {
EXTENSION_NAMES,
POSTGRES_VERSION_RANGE,
VECTOR_EXTENSIONS,
VECTOR_INDEX_TABLES,
VECTOR_VERSION_RANGE,
VECTORCHORD_LIST_SLACK_FACTOR,
VECTORCHORD_VERSION_RANGE,
VECTORS_VERSION_RANGE,
} from 'src/constants';
import { DB } from 'src/db';
import { GenerateSql } from 'src/decorators';
import { DatabaseExtension, DatabaseLock, VectorIndex } from 'src/enum';
@ -14,11 +23,42 @@ import { LoggingRepository } from 'src/repositories/logging.repository';
import { ExtensionVersion, VectorExtension, VectorUpdateResult } from 'src/types';
import { vectorIndexQuery } from 'src/utils/database';
import { isValidInteger } from 'src/validation';
import { DataSource } from 'typeorm';
import { DataSource, QueryRunner } from 'typeorm';
export let cachedVectorExtension: VectorExtension | undefined;
export async function getVectorExtension(runner: Kysely<DB> | QueryRunner): Promise<VectorExtension> {
if (cachedVectorExtension) {
return cachedVectorExtension;
}
cachedVectorExtension = new ConfigRepository().getEnv().database.vectorExtension;
if (cachedVectorExtension) {
return cachedVectorExtension;
}
let availableExtensions: { name: VectorExtension }[];
const query = `SELECT name FROM pg_available_extensions WHERE name IN (${VECTOR_EXTENSIONS.map((ext) => `'${ext}'`).join(', ')})`;
if (runner instanceof Kysely) {
const { rows } = await sql.raw<{ name: VectorExtension }>(query).execute(runner);
availableExtensions = rows;
} else {
availableExtensions = (await runner.query(query)) as { name: VectorExtension }[];
}
const extensionNames = new Set(availableExtensions.map((row) => row.name));
cachedVectorExtension = VECTOR_EXTENSIONS.find((ext) => extensionNames.has(ext));
if (!cachedVectorExtension) {
throw new Error(`No vector extension found. Available extensions: ${VECTOR_EXTENSIONS.join(', ')}`);
}
return cachedVectorExtension;
}
export const probes: Record<VectorIndex, number> = {
[VectorIndex.CLIP]: 1,
[VectorIndex.FACE]: 1,
};
@Injectable()
export class DatabaseRepository {
private vectorExtension: VectorExtension;
private readonly asyncLock = new AsyncLock();
constructor(
@ -26,7 +66,6 @@ export class DatabaseRepository {
private logger: LoggingRepository,
private configRepository: ConfigRepository,
) {
this.vectorExtension = configRepository.getEnv().database.vectorExtension;
this.logger.setContext(DatabaseRepository.name);
}
@ -34,6 +73,10 @@ export class DatabaseRepository {
await this.db.destroy();
}
getVectorExtension(): Promise<VectorExtension> {
return getVectorExtension(this.db);
}
@GenerateSql({ params: [DatabaseExtension.VECTORS] })
async getExtensionVersion(extension: DatabaseExtension): Promise<ExtensionVersion> {
const { rows } = await sql<ExtensionVersion>`
@ -45,7 +88,20 @@ export class DatabaseRepository {
}
getExtensionVersionRange(extension: VectorExtension): string {
return extension === DatabaseExtension.VECTORS ? VECTORS_VERSION_RANGE : VECTOR_VERSION_RANGE;
switch (extension) {
case DatabaseExtension.VECTORCHORD: {
return VECTORCHORD_VERSION_RANGE;
}
case DatabaseExtension.VECTORS: {
return VECTORS_VERSION_RANGE;
}
case DatabaseExtension.VECTOR: {
return VECTOR_VERSION_RANGE;
}
default: {
throw new Error(`Unsupported vector extension: '${extension}'`);
}
}
}
@GenerateSql()
@ -59,7 +115,14 @@ export class DatabaseRepository {
}
async createExtension(extension: DatabaseExtension): Promise<void> {
await sql`CREATE EXTENSION IF NOT EXISTS ${sql.raw(extension)}`.execute(this.db);
await sql`CREATE EXTENSION IF NOT EXISTS ${sql.raw(extension)} CASCADE`.execute(this.db);
if (extension === DatabaseExtension.VECTORCHORD) {
const dbName = sql.table(await this.getDatabaseName());
await sql`ALTER DATABASE ${dbName} SET vchordrq.prewarm_dim = '512,640,768,1024,1152,1536'`.execute(this.db);
await sql`SET vchordrq.prewarm_dim = '512,640,768,1024,1152,1536'`.execute(this.db);
await sql`ALTER DATABASE ${dbName} SET vchordrq.probes = 1`.execute(this.db);
await sql`SET vchordrq.probes = 1`.execute(this.db);
}
}
async updateVectorExtension(extension: VectorExtension, targetVersion?: string): Promise<VectorUpdateResult> {
@ -78,120 +141,201 @@ export class DatabaseRepository {
await this.db.transaction().execute(async (tx) => {
await this.setSearchPath(tx);
if (isVectors && installedVersion === '0.1.1') {
await this.setExtVersion(tx, DatabaseExtension.VECTORS, '0.1.11');
}
const isSchemaUpgrade = semver.satisfies(installedVersion, '0.1.1 || 0.1.11');
if (isSchemaUpgrade && isVectors) {
await this.updateVectorsSchema(tx);
}
await sql`ALTER EXTENSION ${sql.raw(extension)} UPDATE TO ${sql.lit(targetVersion)}`.execute(tx);
const diff = semver.diff(installedVersion, targetVersion);
if (isVectors && diff && ['minor', 'major'].includes(diff)) {
if (isVectors && (diff === 'major' || diff === 'minor')) {
await sql`SELECT pgvectors_upgrade()`.execute(tx);
restartRequired = true;
} else {
await this.reindex(VectorIndex.CLIP);
await this.reindex(VectorIndex.FACE);
} else if (diff) {
await Promise.all([this.reindexVectors(VectorIndex.CLIP), this.reindexVectors(VectorIndex.FACE)]);
}
});
return { restartRequired };
}
async reindex(index: VectorIndex): Promise<void> {
try {
await sql`REINDEX INDEX ${sql.raw(index)}`.execute(this.db);
} catch (error) {
if (this.vectorExtension !== DatabaseExtension.VECTORS) {
throw error;
}
this.logger.warn(`Could not reindex index ${index}. Attempting to auto-fix.`);
const table = await this.getIndexTable(index);
const dimSize = await this.getDimSize(table);
await this.db.transaction().execute(async (tx) => {
await this.setSearchPath(tx);
await sql`DROP INDEX IF EXISTS ${sql.raw(index)}`.execute(tx);
await sql`ALTER TABLE ${sql.raw(table)} ALTER COLUMN embedding SET DATA TYPE real[]`.execute(tx);
await sql`ALTER TABLE ${sql.raw(table)} ALTER COLUMN embedding SET DATA TYPE vector(${sql.raw(String(dimSize))})`.execute(
tx,
);
await sql.raw(vectorIndexQuery({ vectorExtension: this.vectorExtension, table, indexName: index })).execute(tx);
});
async prewarm(index: VectorIndex): Promise<void> {
const vectorExtension = await getVectorExtension(this.db);
if (vectorExtension !== DatabaseExtension.VECTORCHORD) {
return;
}
this.logger.debug(`Prewarming ${index}`);
await sql`SELECT vchordrq_prewarm(${index})`.execute(this.db);
}
@GenerateSql({ params: [VectorIndex.CLIP] })
async shouldReindex(name: VectorIndex): Promise<boolean> {
if (this.vectorExtension !== DatabaseExtension.VECTORS) {
return false;
}
try {
async reindexVectorsIfNeeded(names: VectorIndex[]): Promise<void> {
const { rows } = await sql<{
idx_status: string;
}>`SELECT idx_status FROM pg_vector_index_stat WHERE indexname = ${name}`.execute(this.db);
return rows[0]?.idx_status === 'UPGRADE';
} catch (error) {
const message: string = (error as any).message;
if (message.includes('index is not existing')) {
return true;
} else if (message.includes('relation "pg_vector_index_stat" does not exist')) {
return false;
indexdef: string;
indexname: string;
}>`SELECT indexdef, indexname FROM pg_indexes WHERE indexname = ANY(ARRAY[${sql.join(names)}])`.execute(this.db);
const vectorExtension = await getVectorExtension(this.db);
const promises = [];
for (const indexName of names) {
const row = rows.find((index) => index.indexname === indexName);
const table = VECTOR_INDEX_TABLES[indexName];
if (!row) {
promises.push(this.reindexVectors(indexName));
continue;
}
throw error;
switch (vectorExtension) {
case DatabaseExtension.VECTOR: {
if (!row.indexdef.toLowerCase().includes('using hnsw')) {
promises.push(this.reindexVectors(indexName));
}
break;
}
case DatabaseExtension.VECTORS: {
if (!row.indexdef.toLowerCase().includes('using vectors')) {
promises.push(this.reindexVectors(indexName));
}
break;
}
case DatabaseExtension.VECTORCHORD: {
const matches = row.indexdef.match(/(?<=lists = \[)\d+/g);
const lists = matches && matches.length > 0 ? Number(matches[0]) : 1;
promises.push(
this.db
.selectFrom(this.db.dynamic.table(table).as('t'))
.select((eb) => eb.fn.countAll<number>().as('count'))
.executeTakeFirstOrThrow()
.then(({ count }) => {
const targetLists = this.targetListCount(count);
this.logger.log(`targetLists=${targetLists}, current=${lists} for ${indexName} of ${count} rows`);
if (
!row.indexdef.toLowerCase().includes('using vchordrq') ||
// slack factor is to avoid frequent reindexing if the count is borderline
(lists !== targetLists && lists !== this.targetListCount(count * VECTORCHORD_LIST_SLACK_FACTOR))
) {
probes[indexName] = this.targetProbeCount(targetLists);
return this.reindexVectors(indexName, { lists: targetLists });
} else {
probes[indexName] = this.targetProbeCount(lists);
}
}),
);
break;
}
}
}
if (promises.length > 0) {
await Promise.all(promises);
}
}
private async reindexVectors(indexName: VectorIndex, { lists }: { lists?: number } = {}): Promise<void> {
this.logger.log(`Reindexing ${indexName}`);
const table = VECTOR_INDEX_TABLES[indexName];
const vectorExtension = await getVectorExtension(this.db);
const { rows } = await sql<{
columnName: string;
}>`SELECT column_name as "columnName" FROM information_schema.columns WHERE table_name = ${table}`.execute(this.db);
if (rows.length === 0) {
this.logger.warn(
`Table ${table} does not exist, skipping reindexing. This is only normal if this is a new Immich instance.`,
);
return;
}
const dimSize = await this.getDimensionSize(table);
await this.db.transaction().execute(async (tx) => {
await sql`DROP INDEX IF EXISTS ${sql.raw(indexName)}`.execute(tx);
if (!rows.some((row) => row.columnName === 'embedding')) {
this.logger.warn(`Column 'embedding' does not exist in table '${table}', truncating and adding column.`);
await sql`TRUNCATE TABLE ${sql.raw(table)}`.execute(tx);
await sql`ALTER TABLE ${sql.raw(table)} ADD COLUMN embedding real[] NOT NULL`.execute(tx);
}
await sql`ALTER TABLE ${sql.raw(table)} ALTER COLUMN embedding SET DATA TYPE real[]`.execute(tx);
const schema = vectorExtension === DatabaseExtension.VECTORS ? 'vectors.' : '';
await sql`
ALTER TABLE ${sql.raw(table)}
ALTER COLUMN embedding
SET DATA TYPE ${sql.raw(schema)}vector(${sql.raw(String(dimSize))})`.execute(tx);
await sql.raw(vectorIndexQuery({ vectorExtension, table, indexName, lists })).execute(tx);
});
this.logger.log(`Reindexed ${indexName}`);
}
private async setSearchPath(tx: Transaction<DB>): Promise<void> {
await sql`SET search_path TO "$user", public, vectors`.execute(tx);
}
private async setExtVersion(tx: Transaction<DB>, extName: DatabaseExtension, version: string): Promise<void> {
await sql`UPDATE pg_catalog.pg_extension SET extversion = ${version} WHERE extname = ${extName}`.execute(tx);
private async getDatabaseName(): Promise<string> {
const { rows } = await sql<{ db: string }>`SELECT current_database() as db`.execute(this.db);
return rows[0].db;
}
private async getIndexTable(index: VectorIndex): Promise<string> {
const { rows } = await sql<{
relname: string | null;
}>`SELECT relname FROM pg_stat_all_indexes WHERE indexrelname = ${index}`.execute(this.db);
const table = rows[0]?.relname;
if (!table) {
throw new Error(`Could not find table for index ${index}`);
}
return table;
}
private async updateVectorsSchema(tx: Transaction<DB>): Promise<void> {
const extension = DatabaseExtension.VECTORS;
await sql`CREATE SCHEMA IF NOT EXISTS ${extension}`.execute(tx);
await sql`UPDATE pg_catalog.pg_extension SET extrelocatable = true WHERE extname = ${extension}`.execute(tx);
await sql`ALTER EXTENSION vectors SET SCHEMA vectors`.execute(tx);
await sql`UPDATE pg_catalog.pg_extension SET extrelocatable = false WHERE extname = ${extension}`.execute(tx);
}
private async getDimSize(table: string, column = 'embedding'): Promise<number> {
async getDimensionSize(table: string, column = 'embedding'): Promise<number> {
const { rows } = await sql<{ dimsize: number }>`
SELECT atttypmod as dimsize
FROM pg_attribute f
JOIN pg_class c ON c.oid = f.attrelid
WHERE c.relkind = 'r'::char
AND f.attnum > 0
AND c.relname = ${table}
AND f.attname = '${column}'
AND c.relname = ${table}::text
AND f.attname = ${column}::text
`.execute(this.db);
const dimSize = rows[0]?.dimsize;
if (!isValidInteger(dimSize, { min: 1, max: 2 ** 16 })) {
throw new Error(`Could not retrieve dimension size`);
this.logger.warn(`Could not retrieve dimension size of column '${column}' in table '${table}', assuming 512`);
return 512;
}
return dimSize;
}
async setDimensionSize(dimSize: number): Promise<void> {
if (!isValidInteger(dimSize, { min: 1, max: 2 ** 16 })) {
throw new Error(`Invalid CLIP dimension size: ${dimSize}`);
}
// this is done in two transactions to handle concurrent writes
await this.db.transaction().execute(async (trx) => {
await sql`delete from ${sql.table('smart_search')}`.execute(trx);
await trx.schema.alterTable('smart_search').dropConstraint('dim_size_constraint').ifExists().execute();
await sql`alter table ${sql.table('smart_search')} add constraint dim_size_constraint check (array_length(embedding::real[], 1) = ${sql.lit(dimSize)})`.execute(
trx,
);
});
const vectorExtension = await this.getVectorExtension();
await this.db.transaction().execute(async (trx) => {
await sql`drop index if exists clip_index`.execute(trx);
await trx.schema
.alterTable('smart_search')
.alterColumn('embedding', (col) => col.setDataType(sql.raw(`vector(${dimSize})`)))
.execute();
await sql
.raw(vectorIndexQuery({ vectorExtension, table: 'smart_search', indexName: VectorIndex.CLIP }))
.execute(trx);
await trx.schema.alterTable('smart_search').dropConstraint('dim_size_constraint').ifExists().execute();
});
probes[VectorIndex.CLIP] = 1;
await sql`vacuum analyze ${sql.table('smart_search')}`.execute(this.db);
}
async deleteAllSearchEmbeddings(): Promise<void> {
await sql`truncate ${sql.table('smart_search')}`.execute(this.db);
}
private targetListCount(count: number) {
if (count < 128_000) {
return 1;
} else if (count < 2_048_000) {
return 1 << (32 - Math.clz32(count / 1000));
} else {
return 1 << (33 - Math.clz32(Math.sqrt(count)));
}
}
private targetProbeCount(lists: number) {
return Math.ceil(lists / 8);
}
async runMigrations(options?: { transaction?: 'all' | 'none' | 'each' }): Promise<void> {
const { database } = this.configRepository.getEnv();

1
server/src/repositories/person.repository.ts
View File

@ -398,6 +398,7 @@ export class PersonRepository {
return results.map(({ id }) => id);
}
@GenerateSql({ params: [[], [], [{ faceId: DummyValue.UUID, embedding: DummyValue.VECTOR }]] })
async refreshFaces(
facesToAdd: (Insertable<AssetFaces> & { assetId: string })[],
faceIdsToRemove: string[],

87
server/src/repositories/search.repository.ts
View File

@ -5,9 +5,9 @@ import { randomUUID } from 'node:crypto';
import { DB, Exif } from 'src/db';
import { DummyValue, GenerateSql } from 'src/decorators';
import { MapAsset } from 'src/dtos/asset-response.dto';
import { AssetStatus, AssetType, AssetVisibility } from 'src/enum';
import { ConfigRepository } from 'src/repositories/config.repository';
import { anyUuid, asUuid, searchAssetBuilder, vectorIndexQuery } from 'src/utils/database';
import { AssetStatus, AssetType, AssetVisibility, VectorIndex } from 'src/enum';
import { probes } from 'src/repositories/database.repository';
import { anyUuid, asUuid, searchAssetBuilder } from 'src/utils/database';
import { paginationHelper } from 'src/utils/pagination';
import { isValidInteger } from 'src/validation';
@ -168,10 +168,7 @@ export interface GetCameraMakesOptions {
@Injectable()
export class SearchRepository {
constructor(
@InjectKysely() private db: Kysely<DB>,
private configRepository: ConfigRepository,
) {}
constructor(@InjectKysely() private db: Kysely<DB>) {}
@GenerateSql({
params: [
@ -236,19 +233,21 @@ export class SearchRepository {
},
],
})
async searchSmart(pagination: SearchPaginationOptions, options: SmartSearchOptions) {
searchSmart(pagination: SearchPaginationOptions, options: SmartSearchOptions) {
if (!isValidInteger(pagination.size, { min: 1, max: 1000 })) {
throw new Error(`Invalid value for 'size': ${pagination.size}`);
}
const items = await searchAssetBuilder(this.db, options)
return this.db.transaction().execute(async (trx) => {
await sql`set local vchordrq.probes = ${sql.lit(probes[VectorIndex.CLIP])}`.execute(trx);
const items = await searchAssetBuilder(trx, options)
.innerJoin('smart_search', 'assets.id', 'smart_search.assetId')
.orderBy(sql`smart_search.embedding <=> ${options.embedding}`)
.limit(pagination.size + 1)
.offset((pagination.page - 1) * pagination.size)
.execute();
return paginationHelper(items, pagination.size);
});
}
@GenerateSql({
@ -263,7 +262,9 @@ export class SearchRepository {
],
})
searchDuplicates({ assetId, embedding, maxDistance, type, userIds }: AssetDuplicateSearch) {
return this.db
return this.db.transaction().execute(async (trx) => {
await sql`set local vchordrq.probes = ${sql.lit(probes[VectorIndex.CLIP])}`.execute(trx);
return await trx
.with('cte', (qb) =>
qb
.selectFrom('assets')
@ -279,13 +280,14 @@ export class SearchRepository {
.where('assets.type', '=', type)
.where('assets.id', '!=', asUuid(assetId))
.where('assets.stackId', 'is', null)
.orderBy(sql`smart_search.embedding <=> ${embedding}`)
.orderBy('distance')
.limit(64),
)
.selectFrom('cte')
.selectAll()
.where('cte.distance', '<=', maxDistance as number)
.execute();
});
}
@GenerateSql({
@ -303,7 +305,9 @@ export class SearchRepository {
throw new Error(`Invalid value for 'numResults': ${numResults}`);
}
return this.db
return this.db.transaction().execute(async (trx) => {
await sql`set local vchordrq.probes = ${sql.lit(probes[VectorIndex.FACE])}`.execute(trx);
return await trx
.with('cte', (qb) =>
qb
.selectFrom('asset_faces')
@ -319,15 +323,18 @@ export class SearchRepository {
.where('assets.deletedAt', 'is', null)
.$if(!!hasPerson, (qb) => qb.where('asset_faces.personId', 'is not', null))
.$if(!!minBirthDate, (qb) =>
qb.where((eb) => eb.or([eb('person.birthDate', 'is', null), eb('person.birthDate', '<=', minBirthDate!)])),
qb.where((eb) =>
eb.or([eb('person.birthDate', 'is', null), eb('person.birthDate', '<=', minBirthDate!)]),
),
)
.orderBy(sql`face_search.embedding <=> ${embedding}`)
.orderBy('distance')
.limit(numResults),
)
.selectFrom('cte')
.selectAll()
.where('cte.distance', '<=', maxDistance)
.execute();
});
}
@GenerateSql({ params: [DummyValue.STRING] })
@ -416,56 +423,6 @@ export class SearchRepository {
.execute();
}
async getDimensionSize(): Promise<number> {
const { rows } = await sql<{ dimsize: number }>`
select atttypmod as dimsize
from pg_attribute f
join pg_class c ON c.oid = f.attrelid
where c.relkind = 'r'::char
and f.attnum > 0
and c.relname = 'smart_search'
and f.attname = 'embedding'
`.execute(this.db);
const dimSize = rows[0]['dimsize'];
if (!isValidInteger(dimSize, { min: 1, max: 2 ** 16 })) {
throw new Error(`Could not retrieve CLIP dimension size`);
}
return dimSize;
}
async setDimensionSize(dimSize: number): Promise<void> {
if (!isValidInteger(dimSize, { min: 1, max: 2 ** 16 })) {
throw new Error(`Invalid CLIP dimension size: ${dimSize}`);
}
// this is done in two transactions to handle concurrent writes
await this.db.transaction().execute(async (trx) => {
await sql`delete from ${sql.table('smart_search')}`.execute(trx);
await trx.schema.alterTable('smart_search').dropConstraint('dim_size_constraint').ifExists().execute();
await sql`alter table ${sql.table('smart_search')} add constraint dim_size_constraint check (array_length(embedding::real[], 1) = ${sql.lit(dimSize)})`.execute(
trx,
);
});
const vectorExtension = this.configRepository.getEnv().database.vectorExtension;
await this.db.transaction().execute(async (trx) => {
await sql`drop index if exists clip_index`.execute(trx);
await trx.schema
.alterTable('smart_search')
.alterColumn('embedding', (col) => col.setDataType(sql.raw(`vector(${dimSize})`)))
.execute();
await sql.raw(vectorIndexQuery({ vectorExtension, table: 'smart_search', indexName: 'clip_index' })).execute(trx);
await trx.schema.alterTable('smart_search').dropConstraint('dim_size_constraint').ifExists().execute();
});
await sql`vacuum analyze ${sql.table('smart_search')}`.execute(this.db);
}
async deleteAllSearchEmbeddings(): Promise<void> {
await sql`truncate ${sql.table('smart_search')}`.execute(this.db);
}
async getCountries(userIds: string[]): Promise<string[]> {
const res = await this.getExifField('country', userIds).execute();
return res.map((row) => row.country!);

7
server/src/schema/migrations/1744910873969-InitialMigration.ts
View File

@ -1,10 +1,9 @@
import { Kysely, sql } from 'kysely';
import { DatabaseExtension } from 'src/enum';
import { ConfigRepository } from 'src/repositories/config.repository';
import { getVectorExtension } from 'src/repositories/database.repository';
import { LoggingRepository } from 'src/repositories/logging.repository';
import { vectorIndexQuery } from 'src/utils/database';
const vectorExtension = new ConfigRepository().getEnv().database.vectorExtension;
const lastMigrationSql = sql<{ name: string }>`SELECT "name" FROM "migrations" ORDER BY "timestamp" DESC LIMIT 1;`;
const tableExists = sql<{ result: string | null }>`select to_regclass('migrations') as "result"`;
const logger = LoggingRepository.create();
@ -25,12 +24,14 @@ export async function up(db: Kysely<any>): Promise<void> {
return;
}
const vectorExtension = await getVectorExtension(db);
await sql`CREATE EXTENSION IF NOT EXISTS "uuid-ossp";`.execute(db);
await sql`CREATE EXTENSION IF NOT EXISTS "unaccent";`.execute(db);
await sql`CREATE EXTENSION IF NOT EXISTS "cube";`.execute(db);
await sql`CREATE EXTENSION IF NOT EXISTS "earthdistance";`.execute(db);
await sql`CREATE EXTENSION IF NOT EXISTS "pg_trgm";`.execute(db);
await sql`CREATE EXTENSION IF NOT EXISTS ${sql.raw(vectorExtension)}`.execute(db);
await sql`CREATE EXTENSION IF NOT EXISTS ${sql.raw(vectorExtension)} CASCADE`.execute(db);
await sql`CREATE OR REPLACE FUNCTION immich_uuid_v7(p_timestamp timestamp with time zone default clock_timestamp())
RETURNS uuid
VOLATILE LANGUAGE SQL

72
server/src/services/database.service.spec.ts
View File

@ -1,5 +1,5 @@
import { EXTENSION_NAMES } from 'src/constants';
import { DatabaseExtension } from 'src/enum';
import { DatabaseExtension, VectorIndex } from 'src/enum';
import { DatabaseService } from 'src/services/database.service';
import { VectorExtension } from 'src/types';
import { mockEnvData } from 'test/repositories/config.repository.mock';
@ -47,8 +47,10 @@ describe(DatabaseService.name, () => {
describe.each(<Array<{ extension: VectorExtension; extensionName: string }>>[
{ extension: DatabaseExtension.VECTOR, extensionName: EXTENSION_NAMES[DatabaseExtension.VECTOR] },
{ extension: DatabaseExtension.VECTORS, extensionName: EXTENSION_NAMES[DatabaseExtension.VECTORS] },
{ extension: DatabaseExtension.VECTORCHORD, extensionName: EXTENSION_NAMES[DatabaseExtension.VECTORCHORD] },
])('should work with $extensionName', ({ extension, extensionName }) => {
beforeEach(() => {
mocks.database.getVectorExtension.mockResolvedValue(extension);
mocks.config.getEnv.mockReturnValue(
mockEnvData({
database: {
@ -240,41 +242,32 @@ describe(DatabaseService.name, () => {
});
it(`should reindex ${extension} indices if needed`, async () => {
mocks.database.shouldReindex.mockResolvedValue(true);
await expect(sut.onBootstrap()).resolves.toBeUndefined();
expect(mocks.database.shouldReindex).toHaveBeenCalledTimes(2);
expect(mocks.database.reindex).toHaveBeenCalledTimes(2);
expect(mocks.database.reindexVectorsIfNeeded).toHaveBeenCalledExactlyOnceWith([
VectorIndex.CLIP,
VectorIndex.FACE,
]);
expect(mocks.database.reindexVectorsIfNeeded).toHaveBeenCalledTimes(1);
expect(mocks.database.runMigrations).toHaveBeenCalledTimes(1);
expect(mocks.logger.fatal).not.toHaveBeenCalled();
});
it(`should throw an error if reindexing fails`, async () => {
mocks.database.shouldReindex.mockResolvedValue(true);
mocks.database.reindex.mockRejectedValue(new Error('Error reindexing'));
mocks.database.reindexVectorsIfNeeded.mockRejectedValue(new Error('Error reindexing'));
await expect(sut.onBootstrap()).rejects.toBeDefined();
expect(mocks.database.shouldReindex).toHaveBeenCalledTimes(1);
expect(mocks.database.reindex).toHaveBeenCalledTimes(1);
expect(mocks.database.reindexVectorsIfNeeded).toHaveBeenCalledExactlyOnceWith([
VectorIndex.CLIP,
VectorIndex.FACE,
]);
expect(mocks.database.runMigrations).not.toHaveBeenCalled();
expect(mocks.logger.fatal).not.toHaveBeenCalled();
expect(mocks.logger.warn).toHaveBeenCalledWith(
expect.stringContaining('Could not run vector reindexing checks.'),
);
});
it(`should not reindex ${extension} indices if not needed`, async () => {
mocks.database.shouldReindex.mockResolvedValue(false);
await expect(sut.onBootstrap()).resolves.toBeUndefined();
expect(mocks.database.shouldReindex).toHaveBeenCalledTimes(2);
expect(mocks.database.reindex).toHaveBeenCalledTimes(0);
expect(mocks.database.runMigrations).toHaveBeenCalledTimes(1);
expect(mocks.logger.fatal).not.toHaveBeenCalled();
});
});
it('should skip migrations if DB_SKIP_MIGRATIONS=true', async () => {
@ -300,23 +293,7 @@ describe(DatabaseService.name, () => {
expect(mocks.database.runMigrations).not.toHaveBeenCalled();
});
it(`should throw error if pgvector extension could not be created`, async () => {
mocks.config.getEnv.mockReturnValue(
mockEnvData({
database: {
config: {
connectionType: 'parts',
host: 'database',
port: 5432,
username: 'postgres',
password: 'postgres',
database: 'immich',
},
skipMigrations: true,
vectorExtension: DatabaseExtension.VECTOR,
},
}),
);
it(`should throw error if extension could not be created`, async () => {
mocks.database.getExtensionVersion.mockResolvedValue({
installedVersion: null,
availableVersion: minVersionInRange,
@ -328,26 +305,7 @@ describe(DatabaseService.name, () => {
expect(mocks.logger.fatal).toHaveBeenCalledTimes(1);
expect(mocks.logger.fatal.mock.calls[0][0]).toContain(
`Alternatively, if your Postgres instance has pgvecto.rs, you may use this instead`,
);
expect(mocks.database.createExtension).toHaveBeenCalledTimes(1);
expect(mocks.database.updateVectorExtension).not.toHaveBeenCalled();
expect(mocks.database.runMigrations).not.toHaveBeenCalled();
});
it(`should throw error if pgvecto.rs extension could not be created`, async () => {
mocks.database.getExtensionVersion.mockResolvedValue({
installedVersion: null,
availableVersion: minVersionInRange,
});
mocks.database.updateVectorExtension.mockResolvedValue({ restartRequired: false });
mocks.database.createExtension.mockRejectedValue(new Error('Failed to create extension'));
await expect(sut.onBootstrap()).rejects.toThrow('Failed to create extension');
expect(mocks.logger.fatal).toHaveBeenCalledTimes(1);
expect(mocks.logger.fatal.mock.calls[0][0]).toContain(
`Alternatively, if your Postgres instance has pgvector, you may use this instead`,
`Alternatively, if your Postgres instance has any of vector, vectors, vchord, you may use one of them instead by setting the environment variable 'DB_VECTOR_EXTENSION=<extension name>'`,
);
expect(mocks.database.createExtension).toHaveBeenCalledTimes(1);
expect(mocks.database.updateVectorExtension).not.toHaveBeenCalled();

53
server/src/services/database.service.ts
View File

@ -6,7 +6,7 @@ import { BootstrapEventPriority, DatabaseExtension, DatabaseLock, VectorIndex }
import { BaseService } from 'src/services/base.service';
import { VectorExtension } from 'src/types';
type CreateFailedArgs = { name: string; extension: string; otherName: string };
type CreateFailedArgs = { name: string; extension: string; otherExtensions: string[] };
type UpdateFailedArgs = { name: string; extension: string; availableVersion: string };
type RestartRequiredArgs = { name: string; availableVersion: string };
type NightlyVersionArgs = { name: string; extension: string; version: string };
@ -25,18 +25,15 @@ const messages = {
outOfRange: ({ name, version, range }: OutOfRangeArgs) =>
`The ${name} extension version is ${version}, but Immich only supports ${range}.
Please change ${name} to a compatible version in the Postgres instance.`,
createFailed: ({ name, extension, otherName }: CreateFailedArgs) =>
createFailed: ({ name, extension, otherExtensions }: CreateFailedArgs) =>
`Failed to activate ${name} extension.
Please ensure the Postgres instance has ${name} installed.
If the Postgres instance already has ${name} installed, Immich may not have the necessary permissions to activate it.
In this case, please run 'CREATE EXTENSION IF NOT EXISTS ${extension}' manually as a superuser.
In this case, please run 'CREATE EXTENSION IF NOT EXISTS ${extension} CASCADE' manually as a superuser.
See https://immich.app/docs/guides/database-queries for how to query the database.
Alternatively, if your Postgres instance has ${otherName}, you may use this instead by setting the environment variable 'DB_VECTOR_EXTENSION=${otherName}'.
Note that switching between the two extensions after a successful startup is not supported.
The exception is if your version of Immich prior to upgrading was 1.90.2 or earlier.
In this case, you may set either extension now, but you will not be able to switch to the other extension following a successful startup.`,
Alternatively, if your Postgres instance has any of ${otherExtensions.join(', ')}, you may use one of them instead by setting the environment variable 'DB_VECTOR_EXTENSION=<extension name>'.`,
updateFailed: ({ name, extension, availableVersion }: UpdateFailedArgs) =>
`The ${name} extension can be updated to ${availableVersion}.
Immich attempted to update the extension, but failed to do so.
@ -67,8 +64,7 @@ export class DatabaseService extends BaseService {
}
await this.databaseRepository.withLock(DatabaseLock.Migrations, async () => {
const envData = this.configRepository.getEnv();
const extension = envData.database.vectorExtension;
const extension = await this.databaseRepository.getVectorExtension();
const name = EXTENSION_NAMES[extension];
const extensionRange = this.databaseRepository.getExtensionVersionRange(extension);
@ -97,12 +93,23 @@ export class DatabaseService extends BaseService {
throw new Error(messages.invalidDowngrade({ name, extension, availableVersion, installedVersion }));
}
await this.checkReindexing();
try {
await this.databaseRepository.reindexVectorsIfNeeded([VectorIndex.CLIP, VectorIndex.FACE]);
} catch (error) {
this.logger.warn(
'Could not run vector reindexing checks. If the extension was updated, please restart the Postgres instance. If you are upgrading directly from a version below 1.107.2, please upgrade to 1.107.2 first.',
);
throw error;
}
const { database } = this.configRepository.getEnv();
if (!database.skipMigrations) {
await this.databaseRepository.runMigrations();
}
await Promise.all([
this.databaseRepository.prewarm(VectorIndex.CLIP),
this.databaseRepository.prewarm(VectorIndex.FACE),
]);
});
}
@ -110,10 +117,13 @@ export class DatabaseService extends BaseService {
try {
await this.databaseRepository.createExtension(extension);
} catch (error) {
const otherExtension =
extension === DatabaseExtension.VECTORS ? DatabaseExtension.VECTOR : DatabaseExtension.VECTORS;
const otherExtensions = [
DatabaseExtension.VECTOR,
DatabaseExtension.VECTORS,
DatabaseExtension.VECTORCHORD,
].filter((ext) => ext !== extension);
const name = EXTENSION_NAMES[extension];
this.logger.fatal(messages.createFailed({ name, extension, otherName: EXTENSION_NAMES[otherExtension] }));
this.logger.fatal(messages.createFailed({ name, extension, otherExtensions }));
throw error;
}
}
@ -130,21 +140,4 @@ export class DatabaseService extends BaseService {
throw error;
}
}
private async checkReindexing() {
try {
if (await this.databaseRepository.shouldReindex(VectorIndex.CLIP)) {
await this.databaseRepository.reindex(VectorIndex.CLIP);
}
if (await this.databaseRepository.shouldReindex(VectorIndex.FACE)) {
await this.databaseRepository.reindex(VectorIndex.FACE);
}
} catch (error) {
this.logger.warn(
'Could not run vector reindexing checks. If the extension was updated, please restart the Postgres instance.',
);
throw error;
}
}
}

3
server/src/services/person.service.ts
View File

@ -33,6 +33,7 @@ import {
QueueName,
SourceType,
SystemMetadataKey,
VectorIndex,
} from 'src/enum';
import { BoundingBox } from 'src/repositories/machine-learning.repository';
import { UpdateFacesData } from 'src/repositories/person.repository';
@ -418,6 +419,8 @@ export class PersonService extends BaseService {
return JobStatus.SKIPPED;
}
await this.databaseRepository.prewarm(VectorIndex.FACE);
const lastRun = new Date().toISOString();
const facePagination = this.personRepository.getAllFaces(
force ? undefined : { personId: null, sourceType: SourceType.MACHINE_LEARNING },

54
server/src/services/smart-info.service.spec.ts
View File

@ -54,28 +54,28 @@ describe(SmartInfoService.name, () => {
it('should return if machine learning is disabled', async () => {
await sut.onConfigInit({ newConfig: systemConfigStub.machineLearningDisabled as SystemConfig });
expect(mocks.search.getDimensionSize).not.toHaveBeenCalled();
expect(mocks.search.setDimensionSize).not.toHaveBeenCalled();
expect(mocks.search.deleteAllSearchEmbeddings).not.toHaveBeenCalled();
expect(mocks.database.getDimensionSize).not.toHaveBeenCalled();
expect(mocks.database.setDimensionSize).not.toHaveBeenCalled();
expect(mocks.database.deleteAllSearchEmbeddings).not.toHaveBeenCalled();
});
it('should return if model and DB dimension size are equal', async () => {
mocks.search.getDimensionSize.mockResolvedValue(512);
mocks.database.getDimensionSize.mockResolvedValue(512);
await sut.onConfigInit({ newConfig: systemConfigStub.machineLearningEnabled as SystemConfig });
expect(mocks.search.getDimensionSize).toHaveBeenCalledTimes(1);
expect(mocks.search.setDimensionSize).not.toHaveBeenCalled();
expect(mocks.search.deleteAllSearchEmbeddings).not.toHaveBeenCalled();
expect(mocks.database.getDimensionSize).toHaveBeenCalledTimes(1);
expect(mocks.database.setDimensionSize).not.toHaveBeenCalled();
expect(mocks.database.deleteAllSearchEmbeddings).not.toHaveBeenCalled();
});
it('should update DB dimension size if model and DB have different values', async () => {
mocks.search.getDimensionSize.mockResolvedValue(768);
mocks.database.getDimensionSize.mockResolvedValue(768);
await sut.onConfigInit({ newConfig: systemConfigStub.machineLearningEnabled as SystemConfig });
expect(mocks.search.getDimensionSize).toHaveBeenCalledTimes(1);
expect(mocks.search.setDimensionSize).toHaveBeenCalledWith(512);
expect(mocks.database.getDimensionSize).toHaveBeenCalledTimes(1);
expect(mocks.database.setDimensionSize).toHaveBeenCalledWith(512);
});
});
@ -89,13 +89,13 @@ describe(SmartInfoService.name, () => {
});
expect(mocks.systemMetadata.get).not.toHaveBeenCalled();
expect(mocks.search.getDimensionSize).not.toHaveBeenCalled();
expect(mocks.search.setDimensionSize).not.toHaveBeenCalled();
expect(mocks.search.deleteAllSearchEmbeddings).not.toHaveBeenCalled();
expect(mocks.database.getDimensionSize).not.toHaveBeenCalled();
expect(mocks.database.setDimensionSize).not.toHaveBeenCalled();
expect(mocks.database.deleteAllSearchEmbeddings).not.toHaveBeenCalled();
});
it('should return if model and DB dimension size are equal', async () => {
mocks.search.getDimensionSize.mockResolvedValue(512);
mocks.database.getDimensionSize.mockResolvedValue(512);
await sut.onConfigUpdate({
newConfig: {
@ -106,13 +106,13 @@ describe(SmartInfoService.name, () => {
} as SystemConfig,
});
expect(mocks.search.getDimensionSize).toHaveBeenCalledTimes(1);
expect(mocks.search.setDimensionSize).not.toHaveBeenCalled();
expect(mocks.search.deleteAllSearchEmbeddings).not.toHaveBeenCalled();
expect(mocks.database.getDimensionSize).toHaveBeenCalledTimes(1);
expect(mocks.database.setDimensionSize).not.toHaveBeenCalled();
expect(mocks.database.deleteAllSearchEmbeddings).not.toHaveBeenCalled();
});
it('should update DB dimension size if model and DB have different values', async () => {
mocks.search.getDimensionSize.mockResolvedValue(512);
mocks.database.getDimensionSize.mockResolvedValue(512);
await sut.onConfigUpdate({
newConfig: {
@ -123,12 +123,12 @@ describe(SmartInfoService.name, () => {
} as SystemConfig,
});
expect(mocks.search.getDimensionSize).toHaveBeenCalledTimes(1);
expect(mocks.search.setDimensionSize).toHaveBeenCalledWith(768);
expect(mocks.database.getDimensionSize).toHaveBeenCalledTimes(1);
expect(mocks.database.setDimensionSize).toHaveBeenCalledWith(768);
});
it('should clear embeddings if old and new models are different', async () => {
mocks.search.getDimensionSize.mockResolvedValue(512);
mocks.database.getDimensionSize.mockResolvedValue(512);
await sut.onConfigUpdate({
newConfig: {
@ -139,9 +139,9 @@ describe(SmartInfoService.name, () => {
} as SystemConfig,
});
expect(mocks.search.deleteAllSearchEmbeddings).toHaveBeenCalled();
expect(mocks.search.getDimensionSize).toHaveBeenCalledTimes(1);
expect(mocks.search.setDimensionSize).not.toHaveBeenCalled();
expect(mocks.database.deleteAllSearchEmbeddings).toHaveBeenCalled();
expect(mocks.database.getDimensionSize).toHaveBeenCalledTimes(1);
expect(mocks.database.setDimensionSize).not.toHaveBeenCalled();
});
});
@ -151,7 +151,7 @@ describe(SmartInfoService.name, () => {
await sut.handleQueueEncodeClip({});
expect(mocks.search.setDimensionSize).not.toHaveBeenCalled();
expect(mocks.database.setDimensionSize).not.toHaveBeenCalled();
});
it('should queue the assets without clip embeddings', async () => {
@ -163,7 +163,7 @@ describe(SmartInfoService.name, () => {
{ name: JobName.SMART_SEARCH, data: { id: assetStub.image.id } },
]);
expect(mocks.assetJob.streamForEncodeClip).toHaveBeenCalledWith(false);
expect(mocks.search.setDimensionSize).not.toHaveBeenCalled();
expect(mocks.database.setDimensionSize).not.toHaveBeenCalled();
});
it('should queue all the assets', async () => {
@ -175,7 +175,7 @@ describe(SmartInfoService.name, () => {
{ name: JobName.SMART_SEARCH, data: { id: assetStub.image.id } },
]);
expect(mocks.assetJob.streamForEncodeClip).toHaveBeenCalledWith(true);
expect(mocks.search.setDimensionSize).toHaveBeenCalledExactlyOnceWith(512);
expect(mocks.database.setDimensionSize).toHaveBeenCalledExactlyOnceWith(512);
});
});

8
server/src/services/smart-info.service.ts
View File

@ -38,7 +38,7 @@ export class SmartInfoService extends BaseService {
await this.databaseRepository.withLock(DatabaseLock.CLIPDimSize, async () => {
const { dimSize } = getCLIPModelInfo(newConfig.machineLearning.clip.modelName);
const dbDimSize = await this.searchRepository.getDimensionSize();
const dbDimSize = await this.databaseRepository.getDimensionSize('smart_search');
this.logger.verbose(`Current database CLIP dimension size is ${dbDimSize}`);
const modelChange =
@ -53,10 +53,10 @@ export class SmartInfoService extends BaseService {
`Dimension size of model ${newConfig.machineLearning.clip.modelName} is ${dimSize}, but database expects ${dbDimSize}.`,
);
this.logger.log(`Updating database CLIP dimension size to ${dimSize}.`);
await this.searchRepository.setDimensionSize(dimSize);
await this.databaseRepository.setDimensionSize(dimSize);
this.logger.log(`Successfully updated database CLIP dimension size from ${dbDimSize} to ${dimSize}.`);
} else {
await this.searchRepository.deleteAllSearchEmbeddings();
await this.databaseRepository.deleteAllSearchEmbeddings();
}
// TODO: A job to reindex all assets should be scheduled, though user
@ -74,7 +74,7 @@ export class SmartInfoService extends BaseService {
if (force) {
const { dimSize } = getCLIPModelInfo(machineLearning.clip.modelName);
// in addition to deleting embeddings, update the dimension size in case it failed earlier
await this.searchRepository.setDimensionSize(dimSize);
await this.databaseRepository.setDimensionSize(dimSize);
}
let queue: JobItem[] = [];

4
server/src/types.ts
View File

@ -1,7 +1,7 @@
import { SystemConfig } from 'src/config';
import { VECTOR_EXTENSIONS } from 'src/constants';
import {
AssetType,
DatabaseExtension,
DatabaseSslMode,
ExifOrientation,
ImageFormat,
@ -363,7 +363,7 @@ export type JobItem =
// Version check
| { name: JobName.VERSION_CHECK; data: IBaseJob };
export type VectorExtension = DatabaseExtension.VECTOR | DatabaseExtension.VECTORS;
export type VectorExtension = (typeof VECTOR_EXTENSIONS)[number];
export type DatabaseConnectionURL = {
connectionType: 'url';

18
server/src/utils/database.ts
View File

@ -384,14 +384,28 @@ export function searchAssetBuilder(kysely: Kysely<DB>, options: AssetSearchBuild
.$if(!options.withDeleted, (qb) => qb.where('assets.deletedAt', 'is', null));
}
type VectorIndexOptions = { vectorExtension: VectorExtension; table: string; indexName: string };
export type ReindexVectorIndexOptions = { indexName: string; lists?: number };
export function vectorIndexQuery({ vectorExtension, table, indexName }: VectorIndexOptions): string {
type VectorIndexQueryOptions = { table: string; vectorExtension: VectorExtension } & ReindexVectorIndexOptions;
export function vectorIndexQuery({ vectorExtension, table, indexName, lists }: VectorIndexQueryOptions): string {
switch (vectorExtension) {
case DatabaseExtension.VECTORCHORD: {
return `
CREATE INDEX IF NOT EXISTS ${indexName} ON ${table} USING vchordrq (embedding vector_cosine_ops) WITH (options = $$
residual_quantization = false
[build.internal]
lists = [${lists ?? 1}]
spherical_centroids = true
build_threads = 4
sampling_factor = 1024
$$)`;
}
case DatabaseExtension.VECTORS: {
return `
CREATE INDEX IF NOT EXISTS ${indexName} ON ${table}
USING vectors (embedding vector_cos_ops) WITH (options = $$
optimizing.optimizing_threads = 4
[indexing.hnsw]
m = 16
ef_construction = 300

2
server/test/medium.factory.ts
View File

@ -170,7 +170,7 @@ export const getRepository = <K extends keyof RepositoriesTypes>(key: K, db: Kys
}
case 'search': {
return new SearchRepository(db, new ConfigRepository());
return new SearchRepository(db);
}
case 'session': {

8
server/test/medium/globalSetup.ts
View File

@ -7,7 +7,7 @@ import { getKyselyConfig } from 'src/utils/database';
import { GenericContainer, Wait } from 'testcontainers';
const globalSetup = async () => {
const postgresContainer = await new GenericContainer('tensorchord/pgvecto-rs:pg14-v0.2.0')
const postgresContainer = await new GenericContainer('ghcr.io/immich-app/postgres:14')
.withExposedPorts(5432)
.withEnvironment({
POSTGRES_PASSWORD: 'postgres',
@ -17,9 +17,7 @@ const globalSetup = async () => {
.withCommand([
'postgres',
'-c',
'shared_preload_libraries=vectors.so',
'-c',
'search_path="$$user", public, vectors',
'shared_preload_libraries=vchord.so',
'-c',
'max_wal_size=2GB',
'-c',
@ -30,6 +28,8 @@ const globalSetup = async () => {
'full_page_writes=off',
'-c',
'synchronous_commit=off',
'-c',
'config_file=/var/lib/postgresql/data/postgresql.conf',
])
.withWaitStrategy(Wait.forAll([Wait.forLogMessage('database system is ready to accept connections', 2)]))
.start();

8
server/test/repositories/database.repository.mock.ts
View File

@ -6,13 +6,17 @@ export const newDatabaseRepositoryMock = (): Mocked<RepositoryInterface<Database
return {
shutdown: vitest.fn(),
getExtensionVersion: vitest.fn(),
getVectorExtension: vitest.fn(),
getExtensionVersionRange: vitest.fn(),
getPostgresVersion: vitest.fn().mockResolvedValue('14.10 (Debian 14.10-1.pgdg120+1)'),
getPostgresVersionRange: vitest.fn().mockReturnValue('>=14.0.0'),
createExtension: vitest.fn().mockResolvedValue(void 0),
updateVectorExtension: vitest.fn(),
reindex: vitest.fn(),
shouldReindex: vitest.fn(),
reindexVectorsIfNeeded: vitest.fn(),
getDimensionSize: vitest.fn(),
setDimensionSize: vitest.fn(),
deleteAllSearchEmbeddings: vitest.fn(),
prewarm: vitest.fn(),
runMigrations: vitest.fn(),
withLock: vitest.fn().mockImplementation((_, function_: <R>() => Promise<R>) => function_()),
tryLock: vitest.fn(),