diff --git a/api/.gitignore b/api/.gitignore index ef720b3f..e5ef0f40 100644 --- a/api/.gitignore +++ b/api/.gitignore @@ -1,2 +1,3 @@ node_modules **/*.bun +images diff --git a/api/src/controllers/seed/images.ts b/api/src/controllers/seed/images.ts index aa72a337..b80d744f 100644 --- a/api/src/controllers/seed/images.ts +++ b/api/src/controllers/seed/images.ts @@ -1,18 +1,20 @@ import { mkdir, writeFile } from "node:fs/promises"; import path from "node:path"; import { encode } from "blurhash"; -import { SQL, type SQLWrapper, eq, getTableName, sql } from "drizzle-orm"; -import type { PgColumn, PgTable } from "drizzle-orm/pg-core"; +import { type SQL, eq, is, sql } from "drizzle-orm"; +import { PgColumn, type PgTable } from "drizzle-orm/pg-core"; import { version } from "package.json"; import type { PoolClient } from "pg"; import sharp from "sharp"; import { type Transaction, db } from "~/db"; -import { mqueue } from "~/db/schema/queue"; +import { mqueue } from "~/db/schema/mqueue"; import type { Image } from "~/models/utils"; export const imageDir = process.env.IMAGES_PATH ?? "./images"; await mkdir(imageDir, { recursive: true }); +export const defaultBlurhash = "000000"; + type ImageTask = { id: string; url: string; @@ -35,19 +37,34 @@ export const enqueueOptImage = async ( hasher.update(img.url); const id = hasher.digest().toString("hex"); + const cleanupColumn = (column: SQL) => + // @ts-expect-error dialect is private + db.dialect.sqlToQuery( + sql.join( + column.queryChunks.map((x) => { + if (is(x, PgColumn)) { + return sql.identifier(x.name); + } + return x; + }), + ), + ).sql; + const message: ImageTask = "table" in img ? { id, url: img.url, - table: getTableName(img.table), - column: db.execute(img.column).getQuery().sql, + // @ts-expect-error dialect is private + table: db.dialect.sqlToQuery(sql`${img.table}`).sql, + column: cleanupColumn(img.column), } : { id, url: img.url, - table: getTableName(img.column.table), - column: img.column.name, + // @ts-expect-error dialect is private + table: db.dialect.sqlToQuery(sql`${img.column.table}`).sql, + column: sql.identifier(img.column.name).value, }; await tx.insert(mqueue).values({ kind: "image", @@ -58,7 +75,7 @@ export const enqueueOptImage = async ( return { id, source: img.url, - blurhash: "", + blurhash: defaultBlurhash, }; }; @@ -83,7 +100,7 @@ export const processImages = async () => { const column = sql.raw(img.column); await tx.execute(sql` - update ${table} set ${column} = ${ret} where ${column}->'id' = '${item.id}' + update ${table} set ${column} = ${ret} where ${column}->'id' = ${sql.raw(`'"${img.id}"'::jsonb`)} `); await tx.delete(mqueue).where(eq(mqueue.id, item.id)); @@ -112,6 +129,7 @@ export const processImages = async () => { // start processing old tasks await processAll(); + return () => client.release(true); }; async function downloadImage(id: string, url: string): Promise { @@ -144,6 +162,7 @@ async function downloadImage(id: string, url: string): Promise { const { data, info } = await image .resize(32, 32, { fit: "inside" }) + .ensureAlpha() .raw() .toBuffer({ resolveWithObject: true }); diff --git a/api/src/controllers/seed/series.ts b/api/src/controllers/seed/series.ts index 500aa57a..d8c8818b 100644 --- a/api/src/controllers/seed/series.ts +++ b/api/src/controllers/seed/series.ts @@ -1,7 +1,6 @@ import { t } from "elysia"; import type { SeedSerie } from "~/models/serie"; import { getYear } from "~/utils"; -import { enqueueOptImage } from "./images"; import { insertCollection } from "./insert/collection"; import { insertEntries } from "./insert/entries"; import { insertSeasons } from "./insert/seasons"; diff --git a/api/src/db/schema/index.ts b/api/src/db/schema/index.ts index 3488c4cc..67f4e990 100644 --- a/api/src/db/schema/index.ts +++ b/api/src/db/schema/index.ts @@ -4,3 +4,4 @@ export * from "./shows"; export * from "./studios"; export * from "./staff"; export * from "./videos"; +export * from "./mqueue"; diff --git a/api/src/db/schema/queue.ts b/api/src/db/schema/mqueue.ts similarity index 100% rename from api/src/db/schema/queue.ts rename to api/src/db/schema/mqueue.ts diff --git a/api/src/index.ts b/api/src/index.ts index 4c9a076e..9905c8a2 100644 --- a/api/src/index.ts +++ b/api/src/index.ts @@ -1,5 +1,6 @@ import jwt from "@elysiajs/jwt"; import { swagger } from "@elysiajs/swagger"; +import { processImages } from "./controllers/seed/images"; import { migrate } from "./db"; import { app } from "./elysia"; import { comment } from "./utils"; @@ -23,6 +24,9 @@ if (!secret) { process.exit(1); } +// run image processor task in background +processImages(); + app .use(jwt({ secret })) .use( diff --git a/api/tests/manual.ts b/api/tests/manual.ts index 3835aa2c..4d4cb906 100644 --- a/api/tests/manual.ts +++ b/api/tests/manual.ts @@ -1,5 +1,6 @@ +import { processImages } from "~/controllers/seed/images"; import { db, migrate } from "~/db"; -import { shows, videos } from "~/db/schema"; +import { mqueue, shows, videos } from "~/db/schema"; import { madeInAbyss, madeInAbyssVideo } from "~/models/examples"; import { createSerie, createVideo, getSerie } from "./helpers"; @@ -8,10 +9,16 @@ import { createSerie, createVideo, getSerie } from "./helpers"; await migrate(); await db.delete(shows); await db.delete(videos); +await db.delete(mqueue); const [_, vid] = await createVideo(madeInAbyssVideo); console.log(vid); const [__, ser] = await createSerie(madeInAbyss); console.log(ser); + +await processImages(); + const [___, got] = await getSerie(madeInAbyss.slug, { with: ["translations"] }); console.log(got); + +process.exit(0); diff --git a/api/tests/misc/images.test.ts b/api/tests/misc/images.test.ts new file mode 100644 index 00000000..c59a3a92 --- /dev/null +++ b/api/tests/misc/images.test.ts @@ -0,0 +1,31 @@ +import { beforeAll, describe, expect, it } from "bun:test"; +import { eq } from "drizzle-orm"; +import { defaultBlurhash, processImages } from "~/controllers/seed/images"; +import { db } from "~/db"; +import { mqueue, shows, staff, studios, videos } from "~/db/schema"; +import { madeInAbyss } from "~/models/examples"; +import { createSerie } from "../helpers"; + +beforeAll(async () => { + await db.delete(shows); + await db.delete(studios); + await db.delete(staff); + await db.delete(videos); + await db.delete(mqueue); + + await createSerie(madeInAbyss); + const release = await processImages(); + // remove notifications to prevent other images to be downloaded (do not curl 20000 images for nothing) + release(); +}); + +describe("images", () => { + it("Create a serie download images", async () => { + const ret = await db.query.shows.findFirst({ + where: eq(shows.slug, madeInAbyss.slug), + }); + expect(ret!.slug).toBe(madeInAbyss.slug); + expect(ret!.original.poster!.blurhash).toBeString(); + expect(ret!.original.poster!.blurhash).not.toBe(defaultBlurhash); + }); +});