From 68f249bc03a44f486a238c0657d9e9554c38aa19 Mon Sep 17 00:00:00 2001 From: Min Idzelis Date: Tue, 15 Jul 2025 12:23:41 -0400 Subject: [PATCH] feat: improve geodata import speed (#19906) chore(deps): update dependency vite to v7 (#19657) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: Daniel Dietzler --- server/src/repositories/map.repository.ts | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/server/src/repositories/map.repository.ts b/server/src/repositories/map.repository.ts index ff2e28b8d7..64c8a6229d 100644 --- a/server/src/repositories/map.repository.ts +++ b/server/src/repositories/map.repository.ts @@ -265,6 +265,9 @@ export class MapRepository { throw new Error(`Geodata file ${cities500} not found`); } + this.logger.log(`Starting geodata import`); + const startTime = performance.now(); + const input = createReadStream(cities500, { highWaterMark: 512 * 1024 * 1024 }); let bufferGeodata = []; const lineReader = readLine.createInterface({ input }); @@ -314,7 +317,20 @@ export class MapRepository { } } - await this.db.insertInto('geodata_places').values(bufferGeodata).execute(); + if (bufferGeodata.length > 0) { + await this.db.insertInto('geodata_places').values(bufferGeodata).execute(); + count += bufferGeodata.length; + } + + await Promise.all(futures); + + const duration = performance.now() - startTime; + const seconds = duration / 1000; + const recordsPerSecond = Math.round(count / seconds); + + this.logger.log( + `Successfully imported ${count} geodata records in ${seconds.toFixed(2)}s (${recordsPerSecond} records/second)`, + ); } private async loadAdmin(filePath: string) {