From 8fc279d2ed34ff8578b74cca5a071250d8a6336a Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Wed, 26 Nov 2025 19:42:17 +0100 Subject: [PATCH] Use unnest everywhere --- api/src/controllers/seed/images.ts | 10 ++++-- api/src/controllers/seed/insert/collection.ts | 4 +-- api/src/controllers/seed/insert/entries.ts | 7 ++-- api/src/controllers/seed/insert/seasons.ts | 6 ++-- api/src/controllers/seed/insert/shows.ts | 4 +-- api/src/controllers/seed/insert/staff.ts | 6 ++-- api/src/controllers/seed/insert/studios.ts | 16 ++++++--- api/src/controllers/videos.ts | 9 ++--- api/src/db/utils.ts | 33 ++++++++++++++++--- 9 files changed, 67 insertions(+), 28 deletions(-) diff --git a/api/src/controllers/seed/images.ts b/api/src/controllers/seed/images.ts index 552df70f..743b0a6e 100644 --- a/api/src/controllers/seed/images.ts +++ b/api/src/controllers/seed/images.ts @@ -11,6 +11,7 @@ import { db, type Transaction } from "~/db"; import { mqueue } from "~/db/schema/mqueue"; import type { Image } from "~/models/utils"; import { getFile } from "~/utils"; +import { unnestValues } from "~/db/utils"; export const imageDir = process.env.IMAGES_PATH ?? "/images"; export const defaultBlurhash = "000000"; @@ -83,9 +84,12 @@ export const flushImageQueue = async ( ) => { if (!imgQueue.length) return; record("enqueue images", async () => { - await tx - .insert(mqueue) - .values(imgQueue.map((x) => ({ kind: "image", message: x, priority }))); + await tx.insert(mqueue).select( + unnestValues( + imgQueue.map((x) => ({ kind: "image", message: x, priority })), + mqueue, + ), + ); await tx.execute(sql`notify kyoo_image`); }); }; diff --git a/api/src/controllers/seed/insert/collection.ts b/api/src/controllers/seed/insert/collection.ts index 0b1a9b9e..b93c8b5a 100644 --- a/api/src/controllers/seed/insert/collection.ts +++ b/api/src/controllers/seed/insert/collection.ts @@ -1,7 +1,7 @@ import { sql } from "drizzle-orm"; import { db } from "~/db"; import { shows, showTranslations } from "~/db/schema"; -import { conflictUpdateAllExcept } from "~/db/utils"; +import { conflictUpdateAllExcept, unnestValues } from "~/db/utils"; import type { SeedCollection } from "~/models/collections"; import type { SeedMovie } from "~/models/movie"; import type { SeedSerie } from "~/models/serie"; @@ -75,7 +75,7 @@ export const insertCollection = async ( await flushImageQueue(tx, imgQueue, 100); await tx .insert(showTranslations) - .values(trans) + .select(unnestValues(trans, showTranslations)) .onConflictDoUpdate({ target: [showTranslations.pk, showTranslations.language], set: conflictUpdateAllExcept(showTranslations, ["pk", "language"]), diff --git a/api/src/controllers/seed/insert/entries.ts b/api/src/controllers/seed/insert/entries.ts index fb5c9e60..c6dcbc5c 100644 --- a/api/src/controllers/seed/insert/entries.ts +++ b/api/src/controllers/seed/insert/entries.ts @@ -6,7 +6,7 @@ import { entryVideoJoin, videos, } from "~/db/schema"; -import { conflictUpdateAllExcept, unnestValues, values } from "~/db/utils"; +import { conflictUpdateAllExcept, unnest, unnestValues } from "~/db/utils"; import type { SeedEntry as SEntry, SeedExtra as SExtra } from "~/models/entry"; import { enqueueOptImage, flushImageQueue, type ImageTask } from "../images"; import { guessNextRefresh } from "../refresh"; @@ -169,11 +169,12 @@ export const insertEntries = async ( ), }) .from( - values(vids, { + unnest(vids, "vids", { entryPk: "integer", + entrySlug: "string", needRendering: "boolean", videoId: "uuid", - }).as("vids"), + }), ) .innerJoin(videos, eq(videos.id, sql`vids.videoId`)), ) diff --git a/api/src/controllers/seed/insert/seasons.ts b/api/src/controllers/seed/insert/seasons.ts index c0520da4..8caf947c 100644 --- a/api/src/controllers/seed/insert/seasons.ts +++ b/api/src/controllers/seed/insert/seasons.ts @@ -1,6 +1,6 @@ import { db } from "~/db"; import { seasons, seasonTranslations } from "~/db/schema"; -import { conflictUpdateAllExcept } from "~/db/utils"; +import { conflictUpdateAllExcept, unnestValues } from "~/db/utils"; import type { SeedSeason } from "~/models/season"; import { enqueueOptImage, flushImageQueue, type ImageTask } from "../images"; import { guessNextRefresh } from "../refresh"; @@ -30,7 +30,7 @@ export const insertSeasons = async ( }); const ret = await tx .insert(seasons) - .values(vals) + .select(unnestValues(vals, seasons)) .onConflictDoUpdate({ target: seasons.slug, set: conflictUpdateAllExcept(seasons, [ @@ -66,7 +66,7 @@ export const insertSeasons = async ( await flushImageQueue(tx, imgQueue, -10); await tx .insert(seasonTranslations) - .values(trans) + .select(unnestValues(trans, seasonTranslations)) .onConflictDoUpdate({ target: [seasonTranslations.pk, seasonTranslations.language], set: conflictUpdateAllExcept(seasonTranslations, ["pk", "language"]), diff --git a/api/src/controllers/seed/insert/shows.ts b/api/src/controllers/seed/insert/shows.ts index 88cc8eab..5bb23af6 100644 --- a/api/src/controllers/seed/insert/shows.ts +++ b/api/src/controllers/seed/insert/shows.ts @@ -16,7 +16,7 @@ import { shows, showTranslations, } from "~/db/schema"; -import { conflictUpdateAllExcept, sqlarr } from "~/db/utils"; +import { conflictUpdateAllExcept, sqlarr, unnestValues } from "~/db/utils"; import type { SeedCollection } from "~/models/collections"; import type { SeedMovie } from "~/models/movie"; import type { SeedSerie } from "~/models/serie"; @@ -95,7 +95,7 @@ export const insertShow = async ( await flushImageQueue(tx, imgQueue, 200); await tx .insert(showTranslations) - .values(trans) + .select(unnestValues(trans, showTranslations)) .onConflictDoUpdate({ target: [showTranslations.pk, showTranslations.language], set: conflictUpdateAllExcept(showTranslations, ["pk", "language"]), diff --git a/api/src/controllers/seed/insert/staff.ts b/api/src/controllers/seed/insert/staff.ts index 11cd689b..8ebb6be9 100644 --- a/api/src/controllers/seed/insert/staff.ts +++ b/api/src/controllers/seed/insert/staff.ts @@ -1,7 +1,7 @@ import { eq, sql } from "drizzle-orm"; import { db } from "~/db"; import { roles, staff } from "~/db/schema"; -import { conflictUpdateAllExcept } from "~/db/utils"; +import { conflictUpdateAllExcept, unnestValues } from "~/db/utils"; import type { SeedStaff } from "~/models/staff"; import { enqueueOptImage, flushImageQueue, type ImageTask } from "../images"; @@ -22,7 +22,7 @@ export const insertStaff = async ( })); const ret = await tx .insert(staff) - .values(people) + .select(unnestValues(people, staff)) .onConflictDoUpdate({ target: staff.slug, set: conflictUpdateAllExcept(staff, ["pk", "id", "slug", "createdAt"]), @@ -50,7 +50,7 @@ export const insertStaff = async ( // - we want `order` to stay in sync (& without duplicates) // - we don't have ways to identify a role so we can't onConflict await tx.delete(roles).where(eq(roles.showPk, showPk)); - await tx.insert(roles).values(rval); + await tx.insert(roles).select(unnestValues(rval, roles)); return ret; }); diff --git a/api/src/controllers/seed/insert/studios.ts b/api/src/controllers/seed/insert/studios.ts index e6695af4..c9f8f5dc 100644 --- a/api/src/controllers/seed/insert/studios.ts +++ b/api/src/controllers/seed/insert/studios.ts @@ -1,6 +1,7 @@ +import { sql } from "drizzle-orm"; import { db } from "~/db"; import { showStudioJoin, studios, studioTranslations } from "~/db/schema"; -import { conflictUpdateAllExcept } from "~/db/utils"; +import { conflictUpdateAllExcept, sqlarr, unnestValues } from "~/db/utils"; import type { SeedStudio } from "~/models/studio"; import { enqueueOptImage, flushImageQueue, type ImageTask } from "../images"; @@ -21,7 +22,7 @@ export const insertStudios = async ( const ret = await tx .insert(studios) - .values(vals) + .select(unnestValues(vals, studios)) .onConflictDoUpdate({ target: studios.slug, set: conflictUpdateAllExcept(studios, [ @@ -48,7 +49,7 @@ export const insertStudios = async ( await flushImageQueue(tx, imgQueue, -100); await tx .insert(studioTranslations) - .values(trans) + .select(unnestValues(trans, studioTranslations)) .onConflictDoUpdate({ target: [studioTranslations.pk, studioTranslations.language], set: conflictUpdateAllExcept(studioTranslations, ["pk", "language"]), @@ -56,7 +57,14 @@ export const insertStudios = async ( await tx .insert(showStudioJoin) - .values(ret.map((studio) => ({ showPk: showPk, studioPk: studio.pk }))) + .select( + db + .select({ + showPk: sql`${showPk}`.as("showPk"), + studioPk: sql`v.studioPk`.as("studioPk"), + }) + .from(sql`unnest(${sqlarr(ret.map((x) => x.pk))}) as v("studioPk")`), + ) .onConflictDoNothing(); return ret; }); diff --git a/api/src/controllers/videos.ts b/api/src/controllers/videos.ts index 49a1d6a5..87d35954 100644 --- a/api/src/controllers/videos.ts +++ b/api/src/controllers/videos.ts @@ -35,7 +35,8 @@ import { jsonbBuildObject, jsonbObjectAgg, sqlarr, - values, + unnest, + unnestValues, } from "~/db/utils"; import { Entry } from "~/models/entry"; import { KError } from "~/models/error"; @@ -129,10 +130,10 @@ async function linkVideos( slug: computeVideoSlug(entriesQ.slug, hasRenderingQ), }) .from( - values(links, { + unnest(links, "j", { video: "integer", entry: "jsonb", - }).as("j"), + }), ) .innerJoin(videos, eq(videos.pk, sql`j.video`)) .innerJoin( @@ -835,7 +836,7 @@ export const videosWriteH = new Elysia({ prefix: "/videos", tags: ["videos"] }) try { vids = await tx .insert(videos) - .values(body) + .select(unnestValues(body, videos)) .onConflictDoUpdate({ target: [videos.path], set: conflictUpdateAllExcept(videos, ["pk", "id", "createdAt"]), diff --git a/api/src/db/utils.ts b/api/src/db/utils.ts index b0bb5040..ab31b84c 100644 --- a/api/src/db/utils.ts +++ b/api/src/db/utils.ts @@ -74,14 +74,16 @@ export function conflictUpdateAllExcept< } // drizzle is bugged and doesn't allow js arrays to be used in raw sql. -export function sqlarr(array: unknown[]) { +export function sqlarr(array: unknown[]): string { return `{${array .map((item) => !item || item === "null" ? "null" - : typeof item === "object" - ? `"${JSON.stringify(item).replaceAll('"', '\\"')}"` - : `"${item}"`, + : Array.isArray(item) + ? sqlarr(item) + : typeof item === "object" + ? `"${JSON.stringify(item).replaceAll("\\", "\\\\").replaceAll('"', '\\"')}"` + : `"${item}"`, ) .join(", ")}}`; } @@ -137,6 +139,7 @@ export const unnestValues = < ) => { if (values[0] === undefined) throw new Error("Invalid values, expecting at least one items"); + const columns = getTableColumns(typeInfo); const keys = Object.keys(values[0]).filter((x) => x in columns); // @ts-expect-error: drizzle internal @@ -189,6 +192,28 @@ export const unnestValues = < ); }; +export const unnest = >( + values: T[], + name: string, + typeInfo: Record, +) => { + const keys = Object.keys(typeInfo); + const vals = values.reduce( + (acc, cur) => { + for (const k of keys) { + if (k in cur) acc[k].push(cur[k]); + else acc[k].push(null); + } + return acc; + }, + Object.fromEntries(keys.map((x) => [x, [] as unknown[]])), + ); + return sql`unnest(${sql.join( + keys.map((k) => sql`${sqlarr(vals[k])}${sql.raw(`::${typeInfo[k]}[]`)}`), + sql.raw(", "), + )}) as ${sql.raw(name)}(${sql.raw(keys.map((x) => `"${x}"`).join(", "))})`; +}; + export const coalesce = (val: SQL | SQLWrapper, def: SQL | Column) => { return sql`coalesce(${val}, ${def})`; };