diff --git a/server/src/migrations/1743611339000-GeodataCleanup.ts b/server/src/migrations/1743611339000-GeodataCleanup.ts new file mode 100644 index 0000000000..0e25a1268e --- /dev/null +++ b/server/src/migrations/1743611339000-GeodataCleanup.ts @@ -0,0 +1,19 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class GeodataCleanup1743611339000 implements MigrationInterface { + name = 'GeodataCleanup1743611339000'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER INDEX IF EXISTS "idx_geodata_places_admin2_alternate_names" RENAME TO "idx_geodata_places_alternate_names"`, + ); + await queryRunner.query(`DROP TABLE IF EXISTS "geodata_places_tmp"`); + await queryRunner.query(`DROP TABLE IF EXISTS "naturalearth_countries_tmp"`) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER INDEX IF EXISTS "idx_geodata_places_alternate_names" RENAME TO "idx_geodata_places_admin2_alternate_names"`, + ); + } +} diff --git a/server/src/repositories/map.repository.ts b/server/src/repositories/map.repository.ts index 442225f7c8..e6a2d51b7b 100644 --- a/server/src/repositories/map.repository.ts +++ b/server/src/repositories/map.repository.ts @@ -2,7 +2,6 @@ import { Injectable } from '@nestjs/common'; import { getName } from 'i18n-iso-countries'; import { Expression, Kysely, sql, SqlBool } from 'kysely'; import { InjectKysely } from 'nestjs-kysely'; -import { randomUUID } from 'node:crypto'; import { createReadStream, existsSync } from 'node:fs'; import { readFile } from 'node:fs/promises'; import readLine from 'node:readline'; @@ -183,11 +182,6 @@ export class MapRepository { return; } - await this.db.schema.dropTable('naturalearth_countries_tmp').ifExists().execute(); - await sql`CREATE TABLE naturalearth_countries_tmp (LIKE naturalearth_countries INCLUDING ALL EXCLUDING INDEXES)`.execute( - this.db, - ); - const entities: Omit[] = []; for (const feature of geoJSONData.features) { for (const entry of feature.geometry.coordinates) { @@ -204,15 +198,18 @@ export class MapRepository { } } } - await this.db.insertInto('naturalearth_countries_tmp').values(entities).execute(); - - await sql`ALTER TABLE naturalearth_countries_tmp ADD PRIMARY KEY (id) WITH (FILLFACTOR = 100)`.execute(this.db); await this.db.transaction().execute(async (manager) => { - await manager.schema.alterTable('naturalearth_countries').renameTo('naturalearth_countries_old').execute(); + await sql`CREATE TABLE naturalearth_countries_tmp + ( + LIKE naturalearth_countries INCLUDING ALL EXCLUDING INDEXES + )`.execute(manager); + await manager.schema.dropTable('naturalearth_countries').execute(); await manager.schema.alterTable('naturalearth_countries_tmp').renameTo('naturalearth_countries').execute(); - await manager.schema.dropTable('naturalearth_countries_old').execute(); }); + + await this.db.insertInto('naturalearth_countries').values(entities).execute(); + await sql`ALTER TABLE naturalearth_countries ADD PRIMARY KEY (id) WITH (FILLFACTOR = 100)`.execute(this.db); } private async importGeodata() { @@ -222,16 +219,17 @@ export class MapRepository { this.loadAdmin(resourcePaths.geodata.admin2), ]); - await this.db.schema.dropTable('geodata_places_tmp').ifExists().execute(); - await sql`CREATE TABLE geodata_places_tmp (LIKE geodata_places INCLUDING ALL EXCLUDING INDEXES)`.execute(this.db); + await this.db.transaction().execute(async (manager) => { + await sql`CREATE TABLE geodata_places_tmp + ( + LIKE geodata_places INCLUDING ALL EXCLUDING INDEXES + )`.execute(manager); + await manager.schema.dropTable('geodata_places').execute(); + await manager.schema.alterTable('geodata_places_tmp').renameTo('geodata_places').execute(); + }); + await this.loadCities500(admin1, admin2); await this.createGeodataIndices(); - - await this.db.transaction().execute(async (manager) => { - await manager.schema.alterTable('geodata_places').renameTo('geodata_places_old').execute(); - await manager.schema.alterTable('geodata_places_tmp').renameTo('geodata_places').execute(); - await manager.schema.dropTable('geodata_places_old').execute(); - }); } private async loadCities500(admin1Map: Map, admin2Map: Map) { @@ -271,7 +269,7 @@ export class MapRepository { const curLength = bufferGeodata.length; futures.push( this.db - .insertInto('geodata_places_tmp') + .insertInto('geodata_places') .values(bufferGeodata) .execute() .then(() => { @@ -290,7 +288,7 @@ export class MapRepository { } } - await this.db.insertInto('geodata_places_tmp').values(bufferGeodata).execute(); + await this.db.insertInto('geodata_places').values(bufferGeodata).execute(); } private async loadAdmin(filePath: string) { @@ -313,26 +311,31 @@ export class MapRepository { private createGeodataIndices() { return Promise.all([ - sql`ALTER TABLE geodata_places_tmp ADD PRIMARY KEY (id) WITH (FILLFACTOR = 100)`.execute(this.db), + sql`ALTER TABLE geodata_places ADD PRIMARY KEY (id) WITH (FILLFACTOR = 100)`.execute(this.db), sql` - CREATE INDEX IDX_geodata_gist_earthcoord_${sql.raw(randomUUID().replaceAll('-', '_'))} - ON geodata_places_tmp + CREATE INDEX IDX_geodata_gist_earthcoord + ON geodata_places USING gist (ll_to_earth_public(latitude, longitude)) WITH (fillfactor = 100) `.execute(this.db), this.db.schema - .createIndex(`idx_geodata_places_country_code_${randomUUID().replaceAll('-', '_')}`) - .on('geodata_places_tmp') + .createIndex(`idx_geodata_places_alternate_names`) + .on('geodata_places') + .using('gin (f_unaccent("alternateNames") gin_trgm_ops)') + .execute(), + this.db.schema + .createIndex(`idx_geodata_places_name`) + .on('geodata_places') .using('gin (f_unaccent(name) gin_trgm_ops)') .execute(), this.db.schema - .createIndex(`idx_geodata_places_country_code_${randomUUID().replaceAll('-', '_')}`) - .on('geodata_places_tmp') + .createIndex(`idx_geodata_places_admin1_name`) + .on('geodata_places') .using('gin (f_unaccent("admin1Name") gin_trgm_ops)') .execute(), this.db.schema - .createIndex(`idx_geodata_places_admin2_name_${randomUUID().replaceAll('-', '_')}`) - .on('geodata_places_tmp') + .createIndex(`idx_geodata_places_admin2_name`) + .on('geodata_places') .using('gin (f_unaccent("admin2Name") gin_trgm_ops)') .execute(), ]); diff --git a/server/src/schema/tables/geodata-places.table.ts b/server/src/schema/tables/geodata-places.table.ts index 5216a295cb..2ac4ab2780 100644 --- a/server/src/schema/tables/geodata-places.table.ts +++ b/server/src/schema/tables/geodata-places.table.ts @@ -1,6 +1,11 @@ -import { Column, PrimaryColumn, Table } from 'src/sql-tools'; +import { Column, Index, PrimaryColumn, Table } from 'src/sql-tools'; -@Table({ name: 'geodata_places', synchronize: false }) +@Index({ name: 'idx_geodata_places_alternate_names', expression: 'f_unaccent("alternateNames") gin_trgm_ops' }) +@Index({ name: 'idx_geodata_places_admin1_name', expression: 'f_unaccent("admin1Name") gin_trgm_ops' }) +@Index({ name: 'idx_geodata_places_admin2_name', expression: 'f_unaccent("admin2Name") gin_trgm_ops' }) +@Index({ name: 'idx_geodata_places_name', expression: 'f_unaccent("name") gin_trgm_ops' }) +@Index({ name: 'idx_geodata_places_gist_earthcoord', expression: 'll_to_earth_public(latitude, longitude)' }) +@Table({ name: 'idx_geodata_places', synchronize: false }) export class GeodataPlacesTable { @PrimaryColumn({ type: 'integer' }) id!: number;