From 55a3c30664464a72e2afc27fedc66e83869e7218 Mon Sep 17 00:00:00 2001 From: Jason Rasmussen Date: Sat, 29 Mar 2025 09:26:24 -0400 Subject: [PATCH] feat: kysely migrations (#17198) --- server/src/bin/migrations.ts | 43 ++++++++--- server/src/db.d.ts | 2 +- .../src/repositories/database.repository.ts | 44 ++++++++++- server/src/repositories/user.repository.ts | 2 +- .../src/{ => schema}/tables/activity.table.ts | 6 +- .../{ => schema}/tables/album-asset.table.ts | 4 +- .../{ => schema}/tables/album-user.table.ts | 4 +- server/src/{ => schema}/tables/album.table.ts | 4 +- .../src/{ => schema}/tables/api-key.table.ts | 2 +- .../{ => schema}/tables/asset-audit.table.ts | 0 .../{ => schema}/tables/asset-face.table.ts | 4 +- .../{ => schema}/tables/asset-files.table.ts | 0 .../tables/asset-job-status.table.ts | 2 +- server/src/{ => schema}/tables/asset.table.ts | 6 +- server/src/{ => schema}/tables/audit.table.ts | 0 server/src/{ => schema}/tables/exif.table.ts | 2 +- .../{ => schema}/tables/face-search.table.ts | 2 +- .../tables/geodata-places.table.ts | 0 server/src/schema/tables/index.ts | 73 ++++++++++++++++++ .../src/{ => schema}/tables/library.table.ts | 2 +- .../src/{ => schema}/tables/memory.table.ts | 2 +- .../{ => schema}/tables/memory_asset.table.ts | 4 +- server/src/{ => schema}/tables/move.table.ts | 0 .../tables/natural-earth-countries.table.ts | 0 .../tables/partner-audit.table.ts | 0 .../src/{ => schema}/tables/partner.table.ts | 2 +- .../src/{ => schema}/tables/person.table.ts | 4 +- .../src/{ => schema}/tables/session.table.ts | 2 +- .../tables/shared-link-asset.table.ts | 4 +- .../{ => schema}/tables/shared-link.table.ts | 4 +- .../{ => schema}/tables/smart-search.table.ts | 2 +- server/src/{ => schema}/tables/stack.table.ts | 4 +- .../tables/sync-checkpoint.table.ts | 2 +- .../tables/system-metadata.table.ts | 0 .../{ => schema}/tables/tag-asset.table.ts | 4 +- .../{ => schema}/tables/tag-closure.table.ts | 2 +- server/src/{ => schema}/tables/tag.table.ts | 2 +- .../{ => schema}/tables/user-audit.table.ts | 0 .../tables/user-metadata.table.ts | 2 +- server/src/{ => schema}/tables/user.table.ts | 0 .../tables/version-history.table.ts | 0 server/src/services/base.service.ts | 2 +- server/src/tables/index.ts | 70 ----------------- server/test/factory.ts | 2 +- server/test/medium/globalSetup.ts | 77 ++++++++++++++++++- 45 files changed, 267 insertions(+), 126 deletions(-) rename server/src/{ => schema}/tables/activity.table.ts (87%) rename server/src/{ => schema}/tables/album-asset.table.ts (83%) rename server/src/{ => schema}/tables/album-user.table.ts (88%) rename server/src/{ => schema}/tables/album.table.ts (90%) rename server/src/{ => schema}/tables/api-key.table.ts (92%) rename server/src/{ => schema}/tables/asset-audit.table.ts (100%) rename server/src/{ => schema}/tables/asset-face.table.ts (90%) rename server/src/{ => schema}/tables/asset-files.table.ts (100%) rename server/src/{ => schema}/tables/asset-job-status.table.ts (92%) rename server/src/{ => schema}/tables/asset.table.ts (95%) rename server/src/{ => schema}/tables/audit.table.ts (100%) rename server/src/{ => schema}/tables/exif.table.ts (98%) rename server/src/{ => schema}/tables/face-search.table.ts (87%) rename server/src/{ => schema}/tables/geodata-places.table.ts (100%) create mode 100644 server/src/schema/tables/index.ts rename server/src/{ => schema}/tables/library.table.ts (93%) rename server/src/{ => schema}/tables/memory.table.ts (95%) rename server/src/{ => schema}/tables/memory_asset.table.ts (77%) rename server/src/{ => schema}/tables/move.table.ts (100%) rename server/src/{ => schema}/tables/natural-earth-countries.table.ts (100%) rename server/src/{ => schema}/tables/partner-audit.table.ts (100%) rename server/src/{ => schema}/tables/partner.table.ts (91%) rename server/src/{ => schema}/tables/person.table.ts (90%) rename server/src/{ => schema}/tables/session.table.ts (92%) rename server/src/{ => schema}/tables/shared-link-asset.table.ts (76%) rename server/src/{ => schema}/tables/shared-link.table.ts (91%) rename server/src/{ => schema}/tables/smart-search.table.ts (89%) rename server/src/{ => schema}/tables/stack.table.ts (79%) rename server/src/{ => schema}/tables/sync-checkpoint.table.ts (91%) rename server/src/{ => schema}/tables/system-metadata.table.ts (100%) rename server/src/{ => schema}/tables/tag-asset.table.ts (80%) rename server/src/{ => schema}/tables/tag-closure.table.ts (88%) rename server/src/{ => schema}/tables/tag.table.ts (93%) rename server/src/{ => schema}/tables/user-audit.table.ts (100%) rename server/src/{ => schema}/tables/user-metadata.table.ts (90%) rename server/src/{ => schema}/tables/user.table.ts (100%) rename server/src/{ => schema}/tables/version-history.table.ts (100%) delete mode 100644 server/src/tables/index.ts diff --git a/server/src/bin/migrations.ts b/server/src/bin/migrations.ts index 13a149a1a1..b553ff7fa7 100644 --- a/server/src/bin/migrations.ts +++ b/server/src/bin/migrations.ts @@ -4,8 +4,8 @@ process.env.DB_URL = 'postgres://postgres:postgres@localhost:5432/immich'; import { writeFileSync } from 'node:fs'; import postgres from 'postgres'; import { ConfigRepository } from 'src/repositories/config.repository'; +import 'src/schema/tables'; import { DatabaseTable, schemaDiff, schemaFromDatabase, schemaFromDecorators } from 'src/sql-tools'; -import 'src/tables'; const main = async () => { const command = process.argv[2]; @@ -54,9 +54,10 @@ const generate = async (name: string) => { }; const create = (name: string, up: string[], down: string[]) => { - const { filename, code } = asMigration(name, up, down); + const timestamp = Date.now(); + const filename = `${timestamp}-${name}.ts`; const fullPath = `./src/${filename}`; - writeFileSync(fullPath, code); + writeFileSync(fullPath, asMigration('kysely', { name, timestamp, up, down })); console.log(`Wrote ${fullPath}`); }; @@ -79,14 +80,21 @@ const compare = async () => { return { up, down }; }; -const asMigration = (name: string, up: string[], down: string[]) => { - const timestamp = Date.now(); +type MigrationProps = { + name: string; + timestamp: number; + up: string[]; + down: string[]; +}; +const asMigration = (type: 'kysely' | 'typeorm', options: MigrationProps) => + type === 'typeorm' ? asTypeOrmMigration(options) : asKyselyMigration(options); + +const asTypeOrmMigration = ({ timestamp, name, up, down }: MigrationProps) => { const upSql = up.map((sql) => ` await queryRunner.query(\`${sql}\`);`).join('\n'); const downSql = down.map((sql) => ` await queryRunner.query(\`${sql}\`);`).join('\n'); - return { - filename: `${timestamp}-${name}.ts`, - code: `import { MigrationInterface, QueryRunner } from 'typeorm'; + + return `import { MigrationInterface, QueryRunner } from 'typeorm'; export class ${name}${timestamp} implements MigrationInterface { public async up(queryRunner: QueryRunner): Promise { @@ -97,8 +105,23 @@ ${upSql} ${downSql} } } -`, - }; +`; +}; + +const asKyselyMigration = ({ up, down }: MigrationProps) => { + const upSql = up.map((sql) => ` await sql\`${sql}\`.execute(db);`).join('\n'); + const downSql = down.map((sql) => ` await sql\`${sql}\`.execute(db);`).join('\n'); + + return `import { Kysely, sql } from 'kysely'; + +export async function up(db: Kysely): Promise { +${upSql} +} + +export async function down(db: Kysely): Promise { +${downSql} +} +`; }; main() diff --git a/server/src/db.d.ts b/server/src/db.d.ts index e315a266cf..ca6d1813e4 100644 --- a/server/src/db.d.ts +++ b/server/src/db.d.ts @@ -5,7 +5,7 @@ import type { ColumnType } from 'kysely'; import { AssetType, MemoryType, Permission, SyncEntityType } from 'src/enum'; -import { UserTable } from 'src/tables/user.table'; +import { UserTable } from 'src/schema/tables/user.table'; import { OnThisDayData } from 'src/types'; export type ArrayType = ArrayTypeImpl extends (infer U)[] ? U[] : ArrayTypeImpl; diff --git a/server/src/repositories/database.repository.ts b/server/src/repositories/database.repository.ts index c4aeb74028..917a89b47e 100644 --- a/server/src/repositories/database.repository.ts +++ b/server/src/repositories/database.repository.ts @@ -1,7 +1,9 @@ import { Injectable } from '@nestjs/common'; import AsyncLock from 'async-lock'; -import { Kysely, sql, Transaction } from 'kysely'; +import { FileMigrationProvider, Kysely, Migrator, sql, Transaction } from 'kysely'; import { InjectKysely } from 'nestjs-kysely'; +import { mkdir, readdir } from 'node:fs/promises'; +import { join } from 'node:path'; import semver from 'semver'; import { EXTENSION_NAMES, POSTGRES_VERSION_RANGE, VECTOR_VERSION_RANGE, VECTORS_VERSION_RANGE } from 'src/constants'; import { DB } from 'src/db'; @@ -200,9 +202,49 @@ export class DatabaseRepository { this.logger.log('Running migrations, this may take a while'); + this.logger.debug('Running typeorm migrations'); + await dataSource.initialize(); await dataSource.runMigrations(options); await dataSource.destroy(); + + this.logger.debug('Finished running typeorm migrations'); + + // eslint-disable-next-line unicorn/prefer-module + const migrationFolder = join(__dirname, '..', 'schema/migrations'); + // TODO remove after we have at least one kysely migration + await mkdir(migrationFolder, { recursive: true }); + + this.logger.debug('Running kysely migrations'); + const migrator = new Migrator({ + db: this.db, + migrationLockTableName: 'kysely_migrations_lock', + migrationTableName: 'kysely_migrations', + provider: new FileMigrationProvider({ + fs: { readdir }, + path: { join }, + migrationFolder, + }), + }); + + const { error, results } = await migrator.migrateToLatest(); + + for (const result of results ?? []) { + if (result.status === 'Success') { + this.logger.log(`Migration "${result.migrationName}" succeeded`); + } + + if (result.status === 'Error') { + this.logger.warn(`Migration "${result.migrationName}" failed`); + } + } + + if (error) { + this.logger.error(`Kysely migrations failed: ${error}`); + throw error; + } + + this.logger.debug('Finished running kysely migrations'); } async withLock(lock: DatabaseLock, callback: () => Promise): Promise { diff --git a/server/src/repositories/user.repository.ts b/server/src/repositories/user.repository.ts index 758f99eec1..c619063d04 100644 --- a/server/src/repositories/user.repository.ts +++ b/server/src/repositories/user.repository.ts @@ -8,7 +8,7 @@ import { DummyValue, GenerateSql } from 'src/decorators'; import { UserMetadata, UserMetadataItem } from 'src/entities/user-metadata.entity'; import { UserEntity, withMetadata } from 'src/entities/user.entity'; import { AssetType, UserStatus } from 'src/enum'; -import { UserTable } from 'src/tables/user.table'; +import { UserTable } from 'src/schema/tables/user.table'; import { asUuid } from 'src/utils/database'; type Upsert = Insertable; diff --git a/server/src/tables/activity.table.ts b/server/src/schema/tables/activity.table.ts similarity index 87% rename from server/src/tables/activity.table.ts rename to server/src/schema/tables/activity.table.ts index d7bc7a7bc0..87597838c7 100644 --- a/server/src/tables/activity.table.ts +++ b/server/src/schema/tables/activity.table.ts @@ -1,3 +1,6 @@ +import { AlbumTable } from 'src/schema/tables/album.table'; +import { AssetTable } from 'src/schema/tables/asset.table'; +import { UserTable } from 'src/schema/tables/user.table'; import { Check, Column, @@ -10,9 +13,6 @@ import { UpdateDateColumn, UpdateIdColumn, } from 'src/sql-tools'; -import { AlbumTable } from 'src/tables/album.table'; -import { AssetTable } from 'src/tables/asset.table'; -import { UserTable } from 'src/tables/user.table'; @Table('activity') @Index({ diff --git a/server/src/tables/album-asset.table.ts b/server/src/schema/tables/album-asset.table.ts similarity index 83% rename from server/src/tables/album-asset.table.ts rename to server/src/schema/tables/album-asset.table.ts index 7c51ee9ac2..ccd7fda5fd 100644 --- a/server/src/tables/album-asset.table.ts +++ b/server/src/schema/tables/album-asset.table.ts @@ -1,6 +1,6 @@ +import { AlbumTable } from 'src/schema/tables/album.table'; +import { AssetTable } from 'src/schema/tables/asset.table'; import { ColumnIndex, CreateDateColumn, ForeignKeyColumn, Table } from 'src/sql-tools'; -import { AlbumTable } from 'src/tables/album.table'; -import { AssetTable } from 'src/tables/asset.table'; @Table({ name: 'albums_assets_assets', primaryConstraintName: 'PK_c67bc36fa845fb7b18e0e398180' }) export class AlbumAssetTable { diff --git a/server/src/tables/album-user.table.ts b/server/src/schema/tables/album-user.table.ts similarity index 88% rename from server/src/tables/album-user.table.ts rename to server/src/schema/tables/album-user.table.ts index 3f9df51723..8bd05df2ee 100644 --- a/server/src/tables/album-user.table.ts +++ b/server/src/schema/tables/album-user.table.ts @@ -1,7 +1,7 @@ import { AlbumUserRole } from 'src/enum'; +import { AlbumTable } from 'src/schema/tables/album.table'; +import { UserTable } from 'src/schema/tables/user.table'; import { Column, ForeignKeyColumn, Index, Table } from 'src/sql-tools'; -import { AlbumTable } from 'src/tables/album.table'; -import { UserTable } from 'src/tables/user.table'; @Table({ name: 'albums_shared_users_users', primaryConstraintName: 'PK_7df55657e0b2e8b626330a0ebc8' }) // Pre-existing indices from original album <--> user ManyToMany mapping diff --git a/server/src/tables/album.table.ts b/server/src/schema/tables/album.table.ts similarity index 90% rename from server/src/tables/album.table.ts rename to server/src/schema/tables/album.table.ts index 4f2f7d88f9..cf2f2e1cb4 100644 --- a/server/src/tables/album.table.ts +++ b/server/src/schema/tables/album.table.ts @@ -1,4 +1,6 @@ import { AssetOrder } from 'src/enum'; +import { AssetTable } from 'src/schema/tables/asset.table'; +import { UserTable } from 'src/schema/tables/user.table'; import { Column, ColumnIndex, @@ -10,8 +12,6 @@ import { UpdateDateColumn, UpdateIdColumn, } from 'src/sql-tools'; -import { AssetTable } from 'src/tables/asset.table'; -import { UserTable } from 'src/tables/user.table'; @Table({ name: 'albums', primaryConstraintName: 'PK_7f71c7b5bc7c87b8f94c9a93a00' }) export class AlbumTable { diff --git a/server/src/tables/api-key.table.ts b/server/src/schema/tables/api-key.table.ts similarity index 92% rename from server/src/tables/api-key.table.ts rename to server/src/schema/tables/api-key.table.ts index dd4100e86f..42b98ab957 100644 --- a/server/src/tables/api-key.table.ts +++ b/server/src/schema/tables/api-key.table.ts @@ -1,4 +1,5 @@ import { Permission } from 'src/enum'; +import { UserTable } from 'src/schema/tables/user.table'; import { Column, ColumnIndex, @@ -9,7 +10,6 @@ import { UpdateDateColumn, UpdateIdColumn, } from 'src/sql-tools'; -import { UserTable } from 'src/tables/user.table'; @Table('api_keys') export class APIKeyTable { diff --git a/server/src/tables/asset-audit.table.ts b/server/src/schema/tables/asset-audit.table.ts similarity index 100% rename from server/src/tables/asset-audit.table.ts rename to server/src/schema/tables/asset-audit.table.ts diff --git a/server/src/tables/asset-face.table.ts b/server/src/schema/tables/asset-face.table.ts similarity index 90% rename from server/src/tables/asset-face.table.ts rename to server/src/schema/tables/asset-face.table.ts index 623df937af..56f22cf9a7 100644 --- a/server/src/tables/asset-face.table.ts +++ b/server/src/schema/tables/asset-face.table.ts @@ -1,7 +1,7 @@ import { SourceType } from 'src/enum'; +import { AssetTable } from 'src/schema/tables/asset.table'; +import { PersonTable } from 'src/schema/tables/person.table'; import { Column, DeleteDateColumn, ForeignKeyColumn, Index, PrimaryGeneratedColumn, Table } from 'src/sql-tools'; -import { AssetTable } from 'src/tables/asset.table'; -import { PersonTable } from 'src/tables/person.table'; @Table({ name: 'asset_faces' }) @Index({ name: 'IDX_asset_faces_assetId_personId', columns: ['assetId', 'personId'] }) diff --git a/server/src/tables/asset-files.table.ts b/server/src/schema/tables/asset-files.table.ts similarity index 100% rename from server/src/tables/asset-files.table.ts rename to server/src/schema/tables/asset-files.table.ts diff --git a/server/src/tables/asset-job-status.table.ts b/server/src/schema/tables/asset-job-status.table.ts similarity index 92% rename from server/src/tables/asset-job-status.table.ts rename to server/src/schema/tables/asset-job-status.table.ts index d996577ae4..669ea0a20d 100644 --- a/server/src/tables/asset-job-status.table.ts +++ b/server/src/schema/tables/asset-job-status.table.ts @@ -1,5 +1,5 @@ +import { AssetTable } from 'src/schema/tables/asset.table'; import { Column, ForeignKeyColumn, Table } from 'src/sql-tools'; -import { AssetTable } from 'src/tables/asset.table'; @Table('asset_job_status') export class AssetJobStatusTable { diff --git a/server/src/tables/asset.table.ts b/server/src/schema/tables/asset.table.ts similarity index 95% rename from server/src/tables/asset.table.ts rename to server/src/schema/tables/asset.table.ts index 7e857b8423..bd79d48149 100644 --- a/server/src/tables/asset.table.ts +++ b/server/src/schema/tables/asset.table.ts @@ -1,5 +1,8 @@ import { ASSET_CHECKSUM_CONSTRAINT } from 'src/entities/asset.entity'; import { AssetStatus, AssetType } from 'src/enum'; +import { LibraryTable } from 'src/schema/tables/library.table'; +import { StackTable } from 'src/schema/tables/stack.table'; +import { UserTable } from 'src/schema/tables/user.table'; import { Column, ColumnIndex, @@ -12,9 +15,6 @@ import { UpdateDateColumn, UpdateIdColumn, } from 'src/sql-tools'; -import { LibraryTable } from 'src/tables/library.table'; -import { StackTable } from 'src/tables/stack.table'; -import { UserTable } from 'src/tables/user.table'; @Table('assets') // Checksums must be unique per user and library diff --git a/server/src/tables/audit.table.ts b/server/src/schema/tables/audit.table.ts similarity index 100% rename from server/src/tables/audit.table.ts rename to server/src/schema/tables/audit.table.ts diff --git a/server/src/tables/exif.table.ts b/server/src/schema/tables/exif.table.ts similarity index 98% rename from server/src/tables/exif.table.ts rename to server/src/schema/tables/exif.table.ts index e06659d811..8eddafecc2 100644 --- a/server/src/tables/exif.table.ts +++ b/server/src/schema/tables/exif.table.ts @@ -1,5 +1,5 @@ +import { AssetTable } from 'src/schema/tables/asset.table'; import { Column, ColumnIndex, ForeignKeyColumn, Table, UpdateDateColumn, UpdateIdColumn } from 'src/sql-tools'; -import { AssetTable } from 'src/tables/asset.table'; @Table('exif') export class ExifTable { diff --git a/server/src/tables/face-search.table.ts b/server/src/schema/tables/face-search.table.ts similarity index 87% rename from server/src/tables/face-search.table.ts rename to server/src/schema/tables/face-search.table.ts index 286d09c677..d4da6a69ba 100644 --- a/server/src/tables/face-search.table.ts +++ b/server/src/schema/tables/face-search.table.ts @@ -1,5 +1,5 @@ +import { AssetFaceTable } from 'src/schema/tables/asset-face.table'; import { Column, ColumnIndex, ForeignKeyColumn, Table } from 'src/sql-tools'; -import { AssetFaceTable } from 'src/tables/asset-face.table'; @Table({ name: 'face_search', primaryConstraintName: 'face_search_pkey' }) export class FaceSearchTable { diff --git a/server/src/tables/geodata-places.table.ts b/server/src/schema/tables/geodata-places.table.ts similarity index 100% rename from server/src/tables/geodata-places.table.ts rename to server/src/schema/tables/geodata-places.table.ts diff --git a/server/src/schema/tables/index.ts b/server/src/schema/tables/index.ts new file mode 100644 index 0000000000..6991d957ae --- /dev/null +++ b/server/src/schema/tables/index.ts @@ -0,0 +1,73 @@ +import { ActivityTable } from 'src/schema/tables/activity.table'; +import { AlbumAssetTable } from 'src/schema/tables/album-asset.table'; +import { AlbumUserTable } from 'src/schema/tables/album-user.table'; +import { AlbumTable } from 'src/schema/tables/album.table'; +import { APIKeyTable } from 'src/schema/tables/api-key.table'; +import { AssetAuditTable } from 'src/schema/tables/asset-audit.table'; +import { AssetFaceTable } from 'src/schema/tables/asset-face.table'; +import { AssetJobStatusTable } from 'src/schema/tables/asset-job-status.table'; +import { AssetTable } from 'src/schema/tables/asset.table'; +import { AuditTable } from 'src/schema/tables/audit.table'; +import { ExifTable } from 'src/schema/tables/exif.table'; +import { FaceSearchTable } from 'src/schema/tables/face-search.table'; +import { GeodataPlacesTable } from 'src/schema/tables/geodata-places.table'; +import { LibraryTable } from 'src/schema/tables/library.table'; +import { MemoryTable } from 'src/schema/tables/memory.table'; +import { MemoryAssetTable } from 'src/schema/tables/memory_asset.table'; +import { MoveTable } from 'src/schema/tables/move.table'; +import { + NaturalEarthCountriesTable, + NaturalEarthCountriesTempTable, +} from 'src/schema/tables/natural-earth-countries.table'; +import { PartnerAuditTable } from 'src/schema/tables/partner-audit.table'; +import { PartnerTable } from 'src/schema/tables/partner.table'; +import { PersonTable } from 'src/schema/tables/person.table'; +import { SessionTable } from 'src/schema/tables/session.table'; +import { SharedLinkAssetTable } from 'src/schema/tables/shared-link-asset.table'; +import { SharedLinkTable } from 'src/schema/tables/shared-link.table'; +import { SmartSearchTable } from 'src/schema/tables/smart-search.table'; +import { StackTable } from 'src/schema/tables/stack.table'; +import { SessionSyncCheckpointTable } from 'src/schema/tables/sync-checkpoint.table'; +import { SystemMetadataTable } from 'src/schema/tables/system-metadata.table'; +import { TagAssetTable } from 'src/schema/tables/tag-asset.table'; +import { UserAuditTable } from 'src/schema/tables/user-audit.table'; +import { UserMetadataTable } from 'src/schema/tables/user-metadata.table'; +import { UserTable } from 'src/schema/tables/user.table'; +import { VersionHistoryTable } from 'src/schema/tables/version-history.table'; + +export const tables = [ + ActivityTable, + AlbumAssetTable, + AlbumUserTable, + AlbumTable, + APIKeyTable, + AssetAuditTable, + AssetFaceTable, + AssetJobStatusTable, + AssetTable, + AuditTable, + ExifTable, + FaceSearchTable, + GeodataPlacesTable, + LibraryTable, + MemoryAssetTable, + MemoryTable, + MoveTable, + NaturalEarthCountriesTable, + NaturalEarthCountriesTempTable, + PartnerAuditTable, + PartnerTable, + PersonTable, + SessionTable, + SharedLinkAssetTable, + SharedLinkTable, + SmartSearchTable, + StackTable, + SessionSyncCheckpointTable, + SystemMetadataTable, + TagAssetTable, + UserAuditTable, + UserMetadataTable, + UserTable, + VersionHistoryTable, +]; diff --git a/server/src/tables/library.table.ts b/server/src/schema/tables/library.table.ts similarity index 93% rename from server/src/tables/library.table.ts rename to server/src/schema/tables/library.table.ts index 9119c517ea..ff0bfd64f7 100644 --- a/server/src/tables/library.table.ts +++ b/server/src/schema/tables/library.table.ts @@ -1,3 +1,4 @@ +import { UserTable } from 'src/schema/tables/user.table'; import { Column, ColumnIndex, @@ -9,7 +10,6 @@ import { UpdateDateColumn, UpdateIdColumn, } from 'src/sql-tools'; -import { UserTable } from 'src/tables/user.table'; @Table('libraries') export class LibraryTable { diff --git a/server/src/tables/memory.table.ts b/server/src/schema/tables/memory.table.ts similarity index 95% rename from server/src/tables/memory.table.ts rename to server/src/schema/tables/memory.table.ts index 9523e72610..91a0412649 100644 --- a/server/src/tables/memory.table.ts +++ b/server/src/schema/tables/memory.table.ts @@ -1,4 +1,5 @@ import { MemoryType } from 'src/enum'; +import { UserTable } from 'src/schema/tables/user.table'; import { Column, ColumnIndex, @@ -10,7 +11,6 @@ import { UpdateDateColumn, UpdateIdColumn, } from 'src/sql-tools'; -import { UserTable } from 'src/tables/user.table'; import { MemoryData } from 'src/types'; @Table('memories') diff --git a/server/src/tables/memory_asset.table.ts b/server/src/schema/tables/memory_asset.table.ts similarity index 77% rename from server/src/tables/memory_asset.table.ts rename to server/src/schema/tables/memory_asset.table.ts index 543c81c597..08cdcea442 100644 --- a/server/src/tables/memory_asset.table.ts +++ b/server/src/schema/tables/memory_asset.table.ts @@ -1,6 +1,6 @@ +import { AssetTable } from 'src/schema/tables/asset.table'; +import { MemoryTable } from 'src/schema/tables/memory.table'; import { ColumnIndex, ForeignKeyColumn, Table } from 'src/sql-tools'; -import { AssetTable } from 'src/tables/asset.table'; -import { MemoryTable } from 'src/tables/memory.table'; @Table('memories_assets_assets') export class MemoryAssetTable { diff --git a/server/src/tables/move.table.ts b/server/src/schema/tables/move.table.ts similarity index 100% rename from server/src/tables/move.table.ts rename to server/src/schema/tables/move.table.ts diff --git a/server/src/tables/natural-earth-countries.table.ts b/server/src/schema/tables/natural-earth-countries.table.ts similarity index 100% rename from server/src/tables/natural-earth-countries.table.ts rename to server/src/schema/tables/natural-earth-countries.table.ts diff --git a/server/src/tables/partner-audit.table.ts b/server/src/schema/tables/partner-audit.table.ts similarity index 100% rename from server/src/tables/partner-audit.table.ts rename to server/src/schema/tables/partner-audit.table.ts diff --git a/server/src/tables/partner.table.ts b/server/src/schema/tables/partner.table.ts similarity index 91% rename from server/src/tables/partner.table.ts rename to server/src/schema/tables/partner.table.ts index 900f5fa834..6406b48277 100644 --- a/server/src/tables/partner.table.ts +++ b/server/src/schema/tables/partner.table.ts @@ -1,3 +1,4 @@ +import { UserTable } from 'src/schema/tables/user.table'; import { Column, ColumnIndex, @@ -7,7 +8,6 @@ import { UpdateDateColumn, UpdateIdColumn, } from 'src/sql-tools'; -import { UserTable } from 'src/tables/user.table'; @Table('partners') export class PartnerTable { diff --git a/server/src/tables/person.table.ts b/server/src/schema/tables/person.table.ts similarity index 90% rename from server/src/tables/person.table.ts rename to server/src/schema/tables/person.table.ts index 206e91e68c..91a05d8d76 100644 --- a/server/src/tables/person.table.ts +++ b/server/src/schema/tables/person.table.ts @@ -1,3 +1,5 @@ +import { AssetFaceTable } from 'src/schema/tables/asset-face.table'; +import { UserTable } from 'src/schema/tables/user.table'; import { Check, Column, @@ -9,8 +11,6 @@ import { UpdateDateColumn, UpdateIdColumn, } from 'src/sql-tools'; -import { AssetFaceTable } from 'src/tables/asset-face.table'; -import { UserTable } from 'src/tables/user.table'; @Table('person') @Check({ name: 'CHK_b0f82b0ed662bfc24fbb58bb45', expression: `"birthDate" <= CURRENT_DATE` }) diff --git a/server/src/tables/session.table.ts b/server/src/schema/tables/session.table.ts similarity index 92% rename from server/src/tables/session.table.ts rename to server/src/schema/tables/session.table.ts index 4b6afef099..287f13de7f 100644 --- a/server/src/tables/session.table.ts +++ b/server/src/schema/tables/session.table.ts @@ -1,3 +1,4 @@ +import { UserTable } from 'src/schema/tables/user.table'; import { Column, ColumnIndex, @@ -8,7 +9,6 @@ import { UpdateDateColumn, UpdateIdColumn, } from 'src/sql-tools'; -import { UserTable } from 'src/tables/user.table'; @Table({ name: 'sessions', primaryConstraintName: 'PK_48cb6b5c20faa63157b3c1baf7f' }) export class SessionTable { diff --git a/server/src/tables/shared-link-asset.table.ts b/server/src/schema/tables/shared-link-asset.table.ts similarity index 76% rename from server/src/tables/shared-link-asset.table.ts rename to server/src/schema/tables/shared-link-asset.table.ts index da6526dfc8..1eb294c1e8 100644 --- a/server/src/tables/shared-link-asset.table.ts +++ b/server/src/schema/tables/shared-link-asset.table.ts @@ -1,6 +1,6 @@ +import { AssetTable } from 'src/schema/tables/asset.table'; +import { SharedLinkTable } from 'src/schema/tables/shared-link.table'; import { ColumnIndex, ForeignKeyColumn, Table } from 'src/sql-tools'; -import { AssetTable } from 'src/tables/asset.table'; -import { SharedLinkTable } from 'src/tables/shared-link.table'; @Table('shared_link__asset') export class SharedLinkAssetTable { diff --git a/server/src/tables/shared-link.table.ts b/server/src/schema/tables/shared-link.table.ts similarity index 91% rename from server/src/tables/shared-link.table.ts rename to server/src/schema/tables/shared-link.table.ts index 3a41f5a8f5..4372a5760a 100644 --- a/server/src/tables/shared-link.table.ts +++ b/server/src/schema/tables/shared-link.table.ts @@ -1,4 +1,6 @@ import { SharedLinkType } from 'src/enum'; +import { AlbumTable } from 'src/schema/tables/album.table'; +import { UserTable } from 'src/schema/tables/user.table'; import { Column, ColumnIndex, @@ -8,8 +10,6 @@ import { Table, Unique, } from 'src/sql-tools'; -import { AlbumTable } from 'src/tables/album.table'; -import { UserTable } from 'src/tables/user.table'; @Table('shared_links') @Unique({ name: 'UQ_sharedlink_key', columns: ['key'] }) diff --git a/server/src/tables/smart-search.table.ts b/server/src/schema/tables/smart-search.table.ts similarity index 89% rename from server/src/tables/smart-search.table.ts rename to server/src/schema/tables/smart-search.table.ts index 8647756550..a71eb9ae99 100644 --- a/server/src/tables/smart-search.table.ts +++ b/server/src/schema/tables/smart-search.table.ts @@ -1,5 +1,5 @@ +import { AssetTable } from 'src/schema/tables/asset.table'; import { Column, ColumnIndex, ForeignKeyColumn, Table } from 'src/sql-tools'; -import { AssetTable } from 'src/tables/asset.table'; @Table({ name: 'smart_search', primaryConstraintName: 'smart_search_pkey' }) export class SmartSearchTable { diff --git a/server/src/tables/stack.table.ts b/server/src/schema/tables/stack.table.ts similarity index 79% rename from server/src/tables/stack.table.ts rename to server/src/schema/tables/stack.table.ts index fc711233a4..ea58ccb425 100644 --- a/server/src/tables/stack.table.ts +++ b/server/src/schema/tables/stack.table.ts @@ -1,6 +1,6 @@ +import { AssetTable } from 'src/schema/tables/asset.table'; +import { UserTable } from 'src/schema/tables/user.table'; import { ForeignKeyColumn, PrimaryGeneratedColumn, Table } from 'src/sql-tools'; -import { AssetTable } from 'src/tables/asset.table'; -import { UserTable } from 'src/tables/user.table'; @Table('asset_stack') export class StackTable { diff --git a/server/src/tables/sync-checkpoint.table.ts b/server/src/schema/tables/sync-checkpoint.table.ts similarity index 91% rename from server/src/tables/sync-checkpoint.table.ts rename to server/src/schema/tables/sync-checkpoint.table.ts index 3fbffccb6c..190cd81ffe 100644 --- a/server/src/tables/sync-checkpoint.table.ts +++ b/server/src/schema/tables/sync-checkpoint.table.ts @@ -1,4 +1,5 @@ import { SyncEntityType } from 'src/enum'; +import { SessionTable } from 'src/schema/tables/session.table'; import { Column, ColumnIndex, @@ -9,7 +10,6 @@ import { UpdateDateColumn, UpdateIdColumn, } from 'src/sql-tools'; -import { SessionTable } from 'src/tables/session.table'; @Table('session_sync_checkpoints') export class SessionSyncCheckpointTable { diff --git a/server/src/tables/system-metadata.table.ts b/server/src/schema/tables/system-metadata.table.ts similarity index 100% rename from server/src/tables/system-metadata.table.ts rename to server/src/schema/tables/system-metadata.table.ts diff --git a/server/src/tables/tag-asset.table.ts b/server/src/schema/tables/tag-asset.table.ts similarity index 80% rename from server/src/tables/tag-asset.table.ts rename to server/src/schema/tables/tag-asset.table.ts index 6080c432b5..5f24799cec 100644 --- a/server/src/tables/tag-asset.table.ts +++ b/server/src/schema/tables/tag-asset.table.ts @@ -1,6 +1,6 @@ +import { AssetTable } from 'src/schema/tables/asset.table'; +import { TagTable } from 'src/schema/tables/tag.table'; import { ColumnIndex, ForeignKeyColumn, Index, Table } from 'src/sql-tools'; -import { AssetTable } from 'src/tables/asset.table'; -import { TagTable } from 'src/tables/tag.table'; @Index({ name: 'IDX_tag_asset_assetsId_tagsId', columns: ['assetsId', 'tagsId'] }) @Table('tag_asset') diff --git a/server/src/tables/tag-closure.table.ts b/server/src/schema/tables/tag-closure.table.ts similarity index 88% rename from server/src/tables/tag-closure.table.ts rename to server/src/schema/tables/tag-closure.table.ts index a661904741..079dd4dcc5 100644 --- a/server/src/tables/tag-closure.table.ts +++ b/server/src/schema/tables/tag-closure.table.ts @@ -1,5 +1,5 @@ +import { TagTable } from 'src/schema/tables/tag.table'; import { ColumnIndex, ForeignKeyColumn, PrimaryColumn, Table } from 'src/sql-tools'; -import { TagTable } from 'src/tables/tag.table'; @Table('tags_closure') export class TagClosureTable { diff --git a/server/src/tables/tag.table.ts b/server/src/schema/tables/tag.table.ts similarity index 93% rename from server/src/tables/tag.table.ts rename to server/src/schema/tables/tag.table.ts index 5b74075647..1c6b8cb205 100644 --- a/server/src/tables/tag.table.ts +++ b/server/src/schema/tables/tag.table.ts @@ -1,3 +1,4 @@ +import { UserTable } from 'src/schema/tables/user.table'; import { Column, ColumnIndex, @@ -9,7 +10,6 @@ import { UpdateDateColumn, UpdateIdColumn, } from 'src/sql-tools'; -import { UserTable } from 'src/tables/user.table'; @Table('tags') @Unique({ columns: ['userId', 'value'] }) diff --git a/server/src/tables/user-audit.table.ts b/server/src/schema/tables/user-audit.table.ts similarity index 100% rename from server/src/tables/user-audit.table.ts rename to server/src/schema/tables/user-audit.table.ts diff --git a/server/src/tables/user-metadata.table.ts b/server/src/schema/tables/user-metadata.table.ts similarity index 90% rename from server/src/tables/user-metadata.table.ts rename to server/src/schema/tables/user-metadata.table.ts index 2f83287b6c..e71b3bf9f9 100644 --- a/server/src/tables/user-metadata.table.ts +++ b/server/src/schema/tables/user-metadata.table.ts @@ -1,7 +1,7 @@ import { UserMetadata, UserMetadataItem } from 'src/entities/user-metadata.entity'; import { UserMetadataKey } from 'src/enum'; +import { UserTable } from 'src/schema/tables/user.table'; import { Column, ForeignKeyColumn, PrimaryColumn, Table } from 'src/sql-tools'; -import { UserTable } from 'src/tables/user.table'; @Table('user_metadata') export class UserMetadataTable implements UserMetadataItem { diff --git a/server/src/tables/user.table.ts b/server/src/schema/tables/user.table.ts similarity index 100% rename from server/src/tables/user.table.ts rename to server/src/schema/tables/user.table.ts diff --git a/server/src/tables/version-history.table.ts b/server/src/schema/tables/version-history.table.ts similarity index 100% rename from server/src/tables/version-history.table.ts rename to server/src/schema/tables/version-history.table.ts diff --git a/server/src/services/base.service.ts b/server/src/services/base.service.ts index efdff0e480..6739678561 100644 --- a/server/src/services/base.service.ts +++ b/server/src/services/base.service.ts @@ -46,7 +46,7 @@ import { TrashRepository } from 'src/repositories/trash.repository'; import { UserRepository } from 'src/repositories/user.repository'; import { VersionHistoryRepository } from 'src/repositories/version-history.repository'; import { ViewRepository } from 'src/repositories/view-repository'; -import { UserTable } from 'src/tables/user.table'; +import { UserTable } from 'src/schema/tables/user.table'; import { AccessRequest, checkAccess, requireAccess } from 'src/utils/access'; import { getConfig, updateConfig } from 'src/utils/config'; diff --git a/server/src/tables/index.ts b/server/src/tables/index.ts deleted file mode 100644 index 8b92b55187..0000000000 --- a/server/src/tables/index.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { ActivityTable } from 'src/tables/activity.table'; -import { AlbumAssetTable } from 'src/tables/album-asset.table'; -import { AlbumUserTable } from 'src/tables/album-user.table'; -import { AlbumTable } from 'src/tables/album.table'; -import { APIKeyTable } from 'src/tables/api-key.table'; -import { AssetAuditTable } from 'src/tables/asset-audit.table'; -import { AssetFaceTable } from 'src/tables/asset-face.table'; -import { AssetJobStatusTable } from 'src/tables/asset-job-status.table'; -import { AssetTable } from 'src/tables/asset.table'; -import { AuditTable } from 'src/tables/audit.table'; -import { ExifTable } from 'src/tables/exif.table'; -import { FaceSearchTable } from 'src/tables/face-search.table'; -import { GeodataPlacesTable } from 'src/tables/geodata-places.table'; -import { LibraryTable } from 'src/tables/library.table'; -import { MemoryTable } from 'src/tables/memory.table'; -import { MemoryAssetTable } from 'src/tables/memory_asset.table'; -import { MoveTable } from 'src/tables/move.table'; -import { NaturalEarthCountriesTable, NaturalEarthCountriesTempTable } from 'src/tables/natural-earth-countries.table'; -import { PartnerAuditTable } from 'src/tables/partner-audit.table'; -import { PartnerTable } from 'src/tables/partner.table'; -import { PersonTable } from 'src/tables/person.table'; -import { SessionTable } from 'src/tables/session.table'; -import { SharedLinkAssetTable } from 'src/tables/shared-link-asset.table'; -import { SharedLinkTable } from 'src/tables/shared-link.table'; -import { SmartSearchTable } from 'src/tables/smart-search.table'; -import { StackTable } from 'src/tables/stack.table'; -import { SessionSyncCheckpointTable } from 'src/tables/sync-checkpoint.table'; -import { SystemMetadataTable } from 'src/tables/system-metadata.table'; -import { TagAssetTable } from 'src/tables/tag-asset.table'; -import { UserAuditTable } from 'src/tables/user-audit.table'; -import { UserMetadataTable } from 'src/tables/user-metadata.table'; -import { UserTable } from 'src/tables/user.table'; -import { VersionHistoryTable } from 'src/tables/version-history.table'; - -export const tables = [ - ActivityTable, - AlbumAssetTable, - AlbumUserTable, - AlbumTable, - APIKeyTable, - AssetAuditTable, - AssetFaceTable, - AssetJobStatusTable, - AssetTable, - AuditTable, - ExifTable, - FaceSearchTable, - GeodataPlacesTable, - LibraryTable, - MemoryAssetTable, - MemoryTable, - MoveTable, - NaturalEarthCountriesTable, - NaturalEarthCountriesTempTable, - PartnerAuditTable, - PartnerTable, - PersonTable, - SessionTable, - SharedLinkAssetTable, - SharedLinkTable, - SmartSearchTable, - StackTable, - SessionSyncCheckpointTable, - SystemMetadataTable, - TagAssetTable, - UserAuditTable, - UserMetadataTable, - UserTable, - VersionHistoryTable, -]; diff --git a/server/test/factory.ts b/server/test/factory.ts index 0becc705bc..ce10095d58 100644 --- a/server/test/factory.ts +++ b/server/test/factory.ts @@ -35,7 +35,7 @@ import { TrashRepository } from 'src/repositories/trash.repository'; import { UserRepository } from 'src/repositories/user.repository'; import { VersionHistoryRepository } from 'src/repositories/version-history.repository'; import { ViewRepository } from 'src/repositories/view-repository'; -import { UserTable } from 'src/tables/user.table'; +import { UserTable } from 'src/schema/tables/user.table'; import { newTelemetryRepositoryMock } from 'test/repositories/telemetry.repository.mock'; import { newUuid } from 'test/small.factory'; import { automock } from 'test/utils'; diff --git a/server/test/medium/globalSetup.ts b/server/test/medium/globalSetup.ts index 3c25142073..e8aa8f9ee8 100644 --- a/server/test/medium/globalSetup.ts +++ b/server/test/medium/globalSetup.ts @@ -1,8 +1,14 @@ +import { FileMigrationProvider, Kysely, Migrator } from 'kysely'; +import { PostgresJSDialect } from 'kysely-postgres-js'; +import { mkdir, readdir } from 'node:fs/promises'; +import { join } from 'node:path'; +import { parse } from 'pg-connection-string'; +import postgres, { Notice } from 'postgres'; import { GenericContainer, Wait } from 'testcontainers'; import { DataSource } from 'typeorm'; const globalSetup = async () => { - const postgres = await new GenericContainer('tensorchord/pgvecto-rs:pg14-v0.2.0') + const postgresContainer = await new GenericContainer('tensorchord/pgvecto-rs:pg14-v0.2.0') .withExposedPorts(5432) .withEnvironment({ POSTGRES_PASSWORD: 'postgres', @@ -29,7 +35,7 @@ const globalSetup = async () => { .withWaitStrategy(Wait.forAll([Wait.forLogMessage('database system is ready to accept connections', 2)])) .start(); - const postgresPort = postgres.getMappedPort(5432); + const postgresPort = postgresContainer.getMappedPort(5432); const postgresUrl = `postgres://postgres:postgres@localhost:${postgresPort}/immich`; process.env.IMMICH_TEST_POSTGRES_URL = postgresUrl; @@ -55,6 +61,73 @@ const globalSetup = async () => { await dataSource.initialize(); await dataSource.runMigrations(); await dataSource.destroy(); + + // for whatever reason, importing from test/utils causes vitest to crash + // eslint-disable-next-line unicorn/prefer-module + const migrationFolder = join(__dirname, '..', 'schema/migrations'); + // TODO remove after we have at least one kysely migration + await mkdir(migrationFolder, { recursive: true }); + + const parsed = parse(process.env.IMMICH_TEST_POSTGRES_URL!); + + const parsedOptions = { + ...parsed, + ssl: false, + host: parsed.host ?? undefined, + port: parsed.port ? Number(parsed.port) : undefined, + database: parsed.database ?? undefined, + }; + + const driverOptions = { + ...parsedOptions, + onnotice: (notice: Notice) => { + if (notice['severity'] !== 'NOTICE') { + console.warn('Postgres notice:', notice); + } + }, + max: 10, + types: { + date: { + to: 1184, + from: [1082, 1114, 1184], + serialize: (x: Date | string) => (x instanceof Date ? x.toISOString() : x), + parse: (x: string) => new Date(x), + }, + bigint: { + to: 20, + from: [20], + parse: (value: string) => Number.parseInt(value), + serialize: (value: number) => value.toString(), + }, + }, + connection: { + TimeZone: 'UTC', + }, + }; + + const db = new Kysely({ + dialect: new PostgresJSDialect({ postgres: postgres({ ...driverOptions, max: 1, database: 'postgres' }) }), + }); + + // TODO just call `databaseRepository.migrate()` (probably have to wait until TypeOrm is gone) + const migrator = new Migrator({ + db, + migrationLockTableName: 'kysely_migrations_lock', + migrationTableName: 'kysely_migrations', + provider: new FileMigrationProvider({ + fs: { readdir }, + path: { join }, + migrationFolder, + }), + }); + + const { error } = await migrator.migrateToLatest(); + if (error) { + console.error('Unable to run kysely migrations', error); + throw error; + } + + await db.destroy(); }; export default globalSetup;