From 1a11bc3492e94be2bda8c4edb8a53a69482e1833 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sat, 15 Mar 2025 22:24:25 +0100 Subject: [PATCH 1/9] wip: Add image downloading --- api/package.json | 2 +- api/src/controllers/seed/images.ts | 62 +++++++++++++++++-- api/src/controllers/seed/insert/collection.ts | 10 +-- api/src/controllers/seed/insert/entries.ts | 6 +- api/src/controllers/seed/insert/seasons.ts | 8 +-- api/src/controllers/seed/insert/shows.ts | 10 +-- api/src/controllers/seed/insert/staff.ts | 6 +- api/src/controllers/seed/insert/studios.ts | 4 +- api/src/controllers/seed/movies.ts | 10 +-- api/src/controllers/seed/series.ts | 10 +-- api/src/db/schema/queue.ts | 23 +++++++ api/tsconfig.json | 12 +++- 12 files changed, 123 insertions(+), 40 deletions(-) create mode 100644 api/src/db/schema/queue.ts diff --git a/api/package.json b/api/package.json index da151870..b91418d2 100644 --- a/api/package.json +++ b/api/package.json @@ -1,6 +1,6 @@ { "name": "api", - "version": "1.0.50", + "version": "5.0.0", "scripts": { "dev": "bun --watch src/index.ts", "build": "bun build src/index.ts --target bun --outdir ./dist", diff --git a/api/src/controllers/seed/images.ts b/api/src/controllers/seed/images.ts index 12f0d016..d3caba44 100644 --- a/api/src/controllers/seed/images.ts +++ b/api/src/controllers/seed/images.ts @@ -1,22 +1,74 @@ +import { eq, sql } from "drizzle-orm"; +import { version } from "package.json"; +import type { PoolClient } from "pg"; +import { db } from "~/db"; +import * as schema from "~/db/schema"; +import { mqueue } from "~/db/schema/queue"; import type { Image } from "~/models/utils"; +type ImageTask = { + id: string; + url: string; + table: string; + column: string; +}; + // this will only push a task to the image downloader service and not download it instantly. // this is both done to prevent to many requests to be sent at once and to make sure POST // requests are not blocked by image downloading or blurhash calculation -export const processImage = (url: string): Image => { +export const enqueueImage = async ( + tx: typeof db, + url: string, +): Promise => { const hasher = new Bun.CryptoHasher("sha256"); hasher.update(url); + const id = hasher.digest().toString("hex"); - // TODO: download source, save it in multiples qualities & process blurhash + await tx.insert(mqueue).values({ kind: "image", message: { id, url } }); return { - id: hasher.digest().toString("hex"), + id, source: url, blurhash: "", }; }; -export const processOptImage = (url: string | null): Image | null => { +export const enqueueOptImage = async ( + tx: typeof db, + url: string | null, +): Promise => { if (!url) return null; - return processImage(url); + return await enqueueImage(tx, url); +}; + +export const processImages = async () => { + await db.transaction(async (tx) => { + const [item] = await tx + .select() + .from(mqueue) + .for("update", { skipLocked: true }) + .where(eq(mqueue.kind, "image")) + .orderBy(mqueue.createdAt) + .limit(1); + + const img = item.message as ImageTask; + await fetch(img.url, { headers: { "User-Agent": `Kyoo v${version}` }, }); + const blurhash = ""; + + const table = schema[img.table as keyof typeof schema] as any; + + await tx + .update(table) + .set({ + [img.column]: { id: img.id, source: img.url, blurhash } satisfies Image, + }) + .where(eq(sql`${table[img.column]}->'id'`, img.id)); + + await tx.delete(mqueue).where(eq(mqueue.id, item.id)); + }); + + const client = (await db.$client.connect()) as PoolClient; + client.on("notification", (evt) => { + if (evt.channel !== "image") return; + }); }; diff --git a/api/src/controllers/seed/insert/collection.ts b/api/src/controllers/seed/insert/collection.ts index bcdcc589..495a3f33 100644 --- a/api/src/controllers/seed/insert/collection.ts +++ b/api/src/controllers/seed/insert/collection.ts @@ -5,7 +5,7 @@ import { conflictUpdateAllExcept } from "~/db/utils"; import type { SeedCollection } from "~/models/collections"; import type { SeedMovie } from "~/models/movie"; import type { SeedSerie } from "~/models/serie"; -import { processOptImage } from "../images"; +import { enqueueOptImage } from "../images"; type ShowTrans = typeof showTranslations.$inferInsert; @@ -53,10 +53,10 @@ export const insertCollection = async ( pk: ret.pk, language: lang, ...tr, - poster: processOptImage(tr.poster), - thumbnail: processOptImage(tr.thumbnail), - logo: processOptImage(tr.logo), - banner: processOptImage(tr.banner), + poster: enqueueOptImage(tr.poster), + thumbnail: enqueueOptImage(tr.thumbnail), + logo: enqueueOptImage(tr.logo), + banner: enqueueOptImage(tr.banner), }), ); await tx diff --git a/api/src/controllers/seed/insert/entries.ts b/api/src/controllers/seed/insert/entries.ts index 9c2dd804..d57a5b4a 100644 --- a/api/src/controllers/seed/insert/entries.ts +++ b/api/src/controllers/seed/insert/entries.ts @@ -8,7 +8,7 @@ import { } from "~/db/schema"; import { conflictUpdateAllExcept, sqlarr, values } from "~/db/utils"; import type { SeedEntry as SEntry, SeedExtra as SExtra } from "~/models/entry"; -import { processOptImage } from "../images"; +import { enqueueOptImage } from "../images"; import { guessNextRefresh } from "../refresh"; import { updateAvailableCount } from "./shows"; @@ -55,7 +55,7 @@ export const insertEntries = async ( ...entry, showPk: show.pk, slug: generateSlug(show.slug, seed), - thumbnail: processOptImage(seed.thumbnail), + thumbnail: enqueueOptImage(seed.thumbnail), nextRefresh: entry.kind !== "extra" ? guessNextRefresh(entry.airDate ?? new Date()) @@ -103,7 +103,7 @@ export const insertEntries = async ( ...tr, poster: seed.kind === "movie" - ? processOptImage((tr as any).poster) + ? enqueueOptImage((tr as any).poster) : undefined, })); }); diff --git a/api/src/controllers/seed/insert/seasons.ts b/api/src/controllers/seed/insert/seasons.ts index 56d649ae..bdb5ee39 100644 --- a/api/src/controllers/seed/insert/seasons.ts +++ b/api/src/controllers/seed/insert/seasons.ts @@ -2,7 +2,7 @@ import { db } from "~/db"; import { seasonTranslations, seasons } from "~/db/schema"; import { conflictUpdateAllExcept } from "~/db/utils"; import type { SeedSeason } from "~/models/season"; -import { processOptImage } from "../images"; +import { enqueueOptImage } from "../images"; import { guessNextRefresh } from "../refresh"; type SeasonI = typeof seasons.$inferInsert; @@ -43,9 +43,9 @@ export const insertSeasons = async ( pk: ret[i].pk, language: lang, ...tr, - poster: processOptImage(tr.poster), - thumbnail: processOptImage(tr.thumbnail), - banner: processOptImage(tr.banner), + poster: enqueueOptImage(tr.poster), + thumbnail: enqueueOptImage(tr.thumbnail), + banner: enqueueOptImage(tr.banner), })), ); await tx diff --git a/api/src/controllers/seed/insert/shows.ts b/api/src/controllers/seed/insert/shows.ts index e9d8c845..580738b3 100644 --- a/api/src/controllers/seed/insert/shows.ts +++ b/api/src/controllers/seed/insert/shows.ts @@ -6,7 +6,7 @@ import type { SeedCollection } from "~/models/collections"; import type { SeedMovie } from "~/models/movie"; import type { SeedSerie } from "~/models/serie"; import { getYear } from "~/utils"; -import { processOptImage } from "../images"; +import { enqueueOptImage } from "../images"; type Show = typeof shows.$inferInsert; type ShowTrans = typeof showTranslations.$inferInsert; @@ -27,10 +27,10 @@ export const insertShow = async ( pk: ret.pk, language: lang, ...tr, - poster: processOptImage(tr.poster), - thumbnail: processOptImage(tr.thumbnail), - logo: processOptImage(tr.logo), - banner: processOptImage(tr.banner), + poster: enqueueOptImage(tr.poster), + thumbnail: enqueueOptImage(tr.thumbnail), + logo: enqueueOptImage(tr.logo), + banner: enqueueOptImage(tr.banner), }), ); await tx diff --git a/api/src/controllers/seed/insert/staff.ts b/api/src/controllers/seed/insert/staff.ts index 6512953e..d84092d3 100644 --- a/api/src/controllers/seed/insert/staff.ts +++ b/api/src/controllers/seed/insert/staff.ts @@ -3,7 +3,7 @@ import { db } from "~/db"; import { roles, staff } from "~/db/schema"; import { conflictUpdateAllExcept } from "~/db/utils"; import type { SeedStaff } from "~/models/staff"; -import { processOptImage } from "../images"; +import { enqueueOptImage } from "../images"; export const insertStaff = async ( seed: SeedStaff[] | undefined, @@ -14,7 +14,7 @@ export const insertStaff = async ( return await db.transaction(async (tx) => { const people = seed.map((x) => ({ ...x.staff, - image: processOptImage(x.staff.image), + image: enqueueOptImage(x.staff.image), })); const ret = await tx .insert(staff) @@ -32,7 +32,7 @@ export const insertStaff = async ( order: i, character: { ...x.character, - image: processOptImage(x.character.image), + image: enqueueOptImage(x.character.image), }, })); diff --git a/api/src/controllers/seed/insert/studios.ts b/api/src/controllers/seed/insert/studios.ts index f53f93a2..85d18403 100644 --- a/api/src/controllers/seed/insert/studios.ts +++ b/api/src/controllers/seed/insert/studios.ts @@ -2,7 +2,7 @@ import { db } from "~/db"; import { showStudioJoin, studioTranslations, studios } from "~/db/schema"; import { conflictUpdateAllExcept } from "~/db/utils"; import type { SeedStudio } from "~/models/studio"; -import { processOptImage } from "../images"; +import { enqueueOptImage } from "../images"; type StudioI = typeof studios.$inferInsert; type StudioTransI = typeof studioTranslations.$inferInsert; @@ -38,7 +38,7 @@ export const insertStudios = async ( pk: ret[i].pk, language: lang, name: tr.name, - logo: processOptImage(tr.logo), + logo: enqueueOptImage(tr.logo), })), ); await tx diff --git a/api/src/controllers/seed/movies.ts b/api/src/controllers/seed/movies.ts index dcfcb06f..9ab4ac85 100644 --- a/api/src/controllers/seed/movies.ts +++ b/api/src/controllers/seed/movies.ts @@ -1,7 +1,7 @@ import { t } from "elysia"; import type { SeedMovie } from "~/models/movie"; import { getYear } from "~/utils"; -import { processOptImage } from "./images"; +import { enqueueOptImage } from "./images"; import { insertCollection } from "./insert/collection"; import { insertEntries } from "./insert/entries"; import { insertShow, updateAvailableCount } from "./insert/shows"; @@ -80,10 +80,10 @@ export const seedMovie = async ( language: movie.originalLanguage, name: original.name, latinName: original.latinName ?? null, - poster: processOptImage(original.poster), - thumbnail: processOptImage(original.thumbnail), - logo: processOptImage(original.logo), - banner: processOptImage(original.banner), + poster: enqueueOptImage(original.poster), + thumbnail: enqueueOptImage(original.thumbnail), + logo: enqueueOptImage(original.logo), + banner: enqueueOptImage(original.banner), }, ...movie, }, diff --git a/api/src/controllers/seed/series.ts b/api/src/controllers/seed/series.ts index f3d1048e..62b6d453 100644 --- a/api/src/controllers/seed/series.ts +++ b/api/src/controllers/seed/series.ts @@ -1,7 +1,7 @@ import { t } from "elysia"; import type { SeedSerie } from "~/models/serie"; import { getYear } from "~/utils"; -import { processOptImage } from "./images"; +import { enqueueOptImage } from "./images"; import { insertCollection } from "./insert/collection"; import { insertEntries } from "./insert/entries"; import { insertSeasons } from "./insert/seasons"; @@ -115,10 +115,10 @@ export const seedSerie = async ( language: serie.originalLanguage, name: original.name, latinName: original.latinName ?? null, - poster: processOptImage(original.poster), - thumbnail: processOptImage(original.thumbnail), - logo: processOptImage(original.logo), - banner: processOptImage(original.banner), + poster: enqueueOptImage(original.poster), + thumbnail: enqueueOptImage(original.thumbnail), + logo: enqueueOptImage(original.logo), + banner: enqueueOptImage(original.banner), }, ...serie, }, diff --git a/api/src/db/schema/queue.ts b/api/src/db/schema/queue.ts new file mode 100644 index 00000000..f912493b --- /dev/null +++ b/api/src/db/schema/queue.ts @@ -0,0 +1,23 @@ +import { + index, + integer, + jsonb, + timestamp, + uuid, + varchar, +} from "drizzle-orm/pg-core"; +import { schema } from "./utils"; + +export const mqueue = schema.table( + "mqueue", + { + id: uuid().notNull().primaryKey().defaultRandom(), + kind: varchar({ length: 255 }).notNull(), + message: jsonb().notNull(), + attempt: integer().notNull().default(0), + createdAt: timestamp({ withTimezone: true, mode: "string" }) + .notNull() + .defaultNow(), + }, + (t) => [index("mqueue_created").on(t.createdAt)], +); diff --git a/api/tsconfig.json b/api/tsconfig.json index b9c81697..e271deb5 100644 --- a/api/tsconfig.json +++ b/api/tsconfig.json @@ -3,15 +3,23 @@ "target": "ES2021", "module": "ES2022", "moduleResolution": "node", - "types": ["bun-types"], + "types": [ + "bun-types" + ], "esModuleInterop": true, "forceConsistentCasingInFileNames": true, "strict": true, "skipLibCheck": true, "noErrorTruncation": true, + "resolveJsonModule": true, "baseUrl": ".", "paths": { - "~/*": ["./src/*"] + "~/*": [ + "./src/*" + ], + "package.json": [ + "package.json" + ] } } } From 67511a3aa86e2a593402ffbc58e5a37b5fe0379b Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sat, 15 Mar 2025 23:38:53 +0100 Subject: [PATCH 2/9] Add function to download images, resize them & blurhash --- api/.env.example | 2 + api/bun.lock | 62 ++++++++++++++++++++++++++++++ api/package.json | 4 +- api/src/controllers/seed/images.ts | 53 ++++++++++++++++++++++++- 4 files changed, 118 insertions(+), 3 deletions(-) diff --git a/api/.env.example b/api/.env.example index 193ffd37..6d819716 100644 --- a/api/.env.example +++ b/api/.env.example @@ -7,6 +7,8 @@ JWT_SECRET= # keibi's server to retrieve the public jwt secret AUHT_SERVER=http://auth:4568 +IMAGES_PATH=/images + POSTGRES_USER=kyoo POSTGRES_PASSWORD=password POSTGRES_DB=kyooDB diff --git a/api/bun.lock b/api/bun.lock index e5837399..a78af62b 100644 --- a/api/bun.lock +++ b/api/bun.lock @@ -5,11 +5,13 @@ "dependencies": { "@elysiajs/jwt": "^1.2.0", "@elysiajs/swagger": "^1.2.2", + "blurhash": "^2.0.5", "drizzle-kit": "^0.30.4", "drizzle-orm": "0.39.0", "elysia": "^1.2.23", "parjs": "^1.3.9", "pg": "^8.13.3", + "sharp": "^0.33.5", }, "devDependencies": { "@types/pg": "^8.11.11", @@ -27,6 +29,8 @@ "@elysiajs/swagger": ["@elysiajs/swagger@1.2.2", "", { "dependencies": { "@scalar/themes": "^0.9.52", "@scalar/types": "^0.0.12", "openapi-types": "^12.1.3", "pathe": "^1.1.2" }, "peerDependencies": { "elysia": ">= 1.2.0" } }, "sha512-DG0PbX/wzQNQ6kIpFFPCvmkkWTIbNWDS7lVLv3Puy6ONklF14B4NnbDfpYjX1hdSYKeCqKBBOuenh6jKm8tbYA=="], + "@emnapi/runtime": ["@emnapi/runtime@1.3.1", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-kEBmG8KyqtxJZv+ygbEim+KCGtIq1fC22Ms3S4ziXmYKm8uyoLX0MHONVKwp+9opg390VaKRNt4a7A9NwmpNhw=="], + "@esbuild-kit/core-utils": ["@esbuild-kit/core-utils@3.3.2", "", { "dependencies": { "esbuild": "~0.18.20", "source-map-support": "^0.5.21" } }, "sha512-sPRAnw9CdSsRmEtnsl2WXWdyquogVpB3yZ3dgwJfe8zrOzTsV7cJvmwrKVa+0ma5BoiGJ+BoqkMvawbayKUsqQ=="], "@esbuild-kit/esm-loader": ["@esbuild-kit/esm-loader@2.6.5", "", { "dependencies": { "@esbuild-kit/core-utils": "^3.3.2", "get-tsconfig": "^4.7.0" } }, "sha512-FxEMIkJKnodyA1OaCUoEvbYRkoZlLZ4d/eXFu9Fh8CbBBgP5EmZxrfTRyN0qpXZ4vOvqnE5YdRdcrmUUXuU+dA=="], @@ -77,6 +81,44 @@ "@esbuild/win32-x64": ["@esbuild/win32-x64@0.19.12", "", { "os": "win32", "cpu": "x64" }, "sha512-T1QyPSDCyMXaO3pzBkF96E8xMkiRYbUEZADd29SyPGabqxMViNoii+NcK7eWJAEoU6RZyEm5lVSIjTmcdoB9HA=="], + "@img/sharp-darwin-arm64": ["@img/sharp-darwin-arm64@0.33.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-arm64": "1.0.4" }, "os": "darwin", "cpu": "arm64" }, "sha512-UT4p+iz/2H4twwAoLCqfA9UH5pI6DggwKEGuaPy7nCVQ8ZsiY5PIcrRvD1DzuY3qYL07NtIQcWnBSY/heikIFQ=="], + + "@img/sharp-darwin-x64": ["@img/sharp-darwin-x64@0.33.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-x64": "1.0.4" }, "os": "darwin", "cpu": "x64" }, "sha512-fyHac4jIc1ANYGRDxtiqelIbdWkIuQaI84Mv45KvGRRxSAa7o7d1ZKAOBaYbnepLC1WqxfpimdeWfvqqSGwR2Q=="], + + "@img/sharp-libvips-darwin-arm64": ["@img/sharp-libvips-darwin-arm64@1.0.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-XblONe153h0O2zuFfTAbQYAX2JhYmDHeWikp1LM9Hul9gVPjFY427k6dFEcOL72O01QxQsWi761svJ/ev9xEDg=="], + + "@img/sharp-libvips-darwin-x64": ["@img/sharp-libvips-darwin-x64@1.0.4", "", { "os": "darwin", "cpu": "x64" }, "sha512-xnGR8YuZYfJGmWPvmlunFaWJsb9T/AO2ykoP3Fz/0X5XV2aoYBPkX6xqCQvUTKKiLddarLaxpzNe+b1hjeWHAQ=="], + + "@img/sharp-libvips-linux-arm": ["@img/sharp-libvips-linux-arm@1.0.5", "", { "os": "linux", "cpu": "arm" }, "sha512-gvcC4ACAOPRNATg/ov8/MnbxFDJqf/pDePbBnuBDcjsI8PssmjoKMAz4LtLaVi+OnSb5FK/yIOamqDwGmXW32g=="], + + "@img/sharp-libvips-linux-arm64": ["@img/sharp-libvips-linux-arm64@1.0.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-9B+taZ8DlyyqzZQnoeIvDVR/2F4EbMepXMc/NdVbkzsJbzkUjhXv/70GQJ7tdLA4YJgNP25zukcxpX2/SueNrA=="], + + "@img/sharp-libvips-linux-s390x": ["@img/sharp-libvips-linux-s390x@1.0.4", "", { "os": "linux", "cpu": "s390x" }, "sha512-u7Wz6ntiSSgGSGcjZ55im6uvTrOxSIS8/dgoVMoiGE9I6JAfU50yH5BoDlYA1tcuGS7g/QNtetJnxA6QEsCVTA=="], + + "@img/sharp-libvips-linux-x64": ["@img/sharp-libvips-linux-x64@1.0.4", "", { "os": "linux", "cpu": "x64" }, "sha512-MmWmQ3iPFZr0Iev+BAgVMb3ZyC4KeFc3jFxnNbEPas60e1cIfevbtuyf9nDGIzOaW9PdnDciJm+wFFaTlj5xYw=="], + + "@img/sharp-libvips-linuxmusl-arm64": ["@img/sharp-libvips-linuxmusl-arm64@1.0.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-9Ti+BbTYDcsbp4wfYib8Ctm1ilkugkA/uscUn6UXK1ldpC1JjiXbLfFZtRlBhjPZ5o1NCLiDbg8fhUPKStHoTA=="], + + "@img/sharp-libvips-linuxmusl-x64": ["@img/sharp-libvips-linuxmusl-x64@1.0.4", "", { "os": "linux", "cpu": "x64" }, "sha512-viYN1KX9m+/hGkJtvYYp+CCLgnJXwiQB39damAO7WMdKWlIhmYTfHjwSbQeUK/20vY154mwezd9HflVFM1wVSw=="], + + "@img/sharp-linux-arm": ["@img/sharp-linux-arm@0.33.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm": "1.0.5" }, "os": "linux", "cpu": "arm" }, "sha512-JTS1eldqZbJxjvKaAkxhZmBqPRGmxgu+qFKSInv8moZ2AmT5Yib3EQ1c6gp493HvrvV8QgdOXdyaIBrhvFhBMQ=="], + + "@img/sharp-linux-arm64": ["@img/sharp-linux-arm64@0.33.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm64": "1.0.4" }, "os": "linux", "cpu": "arm64" }, "sha512-JMVv+AMRyGOHtO1RFBiJy/MBsgz0x4AWrT6QoEVVTyh1E39TrCUpTRI7mx9VksGX4awWASxqCYLCV4wBZHAYxA=="], + + "@img/sharp-linux-s390x": ["@img/sharp-linux-s390x@0.33.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-s390x": "1.0.4" }, "os": "linux", "cpu": "s390x" }, "sha512-y/5PCd+mP4CA/sPDKl2961b+C9d+vPAveS33s6Z3zfASk2j5upL6fXVPZi7ztePZ5CuH+1kW8JtvxgbuXHRa4Q=="], + + "@img/sharp-linux-x64": ["@img/sharp-linux-x64@0.33.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-x64": "1.0.4" }, "os": "linux", "cpu": "x64" }, "sha512-opC+Ok5pRNAzuvq1AG0ar+1owsu842/Ab+4qvU879ippJBHvyY5n2mxF1izXqkPYlGuP/M556uh53jRLJmzTWA=="], + + "@img/sharp-linuxmusl-arm64": ["@img/sharp-linuxmusl-arm64@0.33.5", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-arm64": "1.0.4" }, "os": "linux", "cpu": "arm64" }, "sha512-XrHMZwGQGvJg2V/oRSUfSAfjfPxO+4DkiRh6p2AFjLQztWUuY/o8Mq0eMQVIY7HJ1CDQUJlxGGZRw1a5bqmd1g=="], + + "@img/sharp-linuxmusl-x64": ["@img/sharp-linuxmusl-x64@0.33.5", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-x64": "1.0.4" }, "os": "linux", "cpu": "x64" }, "sha512-WT+d/cgqKkkKySYmqoZ8y3pxx7lx9vVejxW/W4DOFMYVSkErR+w7mf2u8m/y4+xHe7yY9DAXQMWQhpnMuFfScw=="], + + "@img/sharp-wasm32": ["@img/sharp-wasm32@0.33.5", "", { "dependencies": { "@emnapi/runtime": "^1.2.0" }, "cpu": "none" }, "sha512-ykUW4LVGaMcU9lu9thv85CbRMAwfeadCJHRsg2GmeRa/cJxsVY9Rbd57JcMxBkKHag5U/x7TSBpScF4U8ElVzg=="], + + "@img/sharp-win32-ia32": ["@img/sharp-win32-ia32@0.33.5", "", { "os": "win32", "cpu": "ia32" }, "sha512-T36PblLaTwuVJ/zw/LaH0PdZkRz5rd3SmMHX8GSmR7vtNSP5Z6bQkExdSK7xGWyxLw4sUknBuugTelgw2faBbQ=="], + + "@img/sharp-win32-x64": ["@img/sharp-win32-x64@0.33.5", "", { "os": "win32", "cpu": "x64" }, "sha512-MpY/o8/8kj+EcnxwvrP4aTJSWw/aZ7JIGR4aBeZkZw5B7/Jn+tY9/VNwtcoGmdT7GfggGIU4kygOMSbYnOrAbg=="], + "@petamoriken/float16": ["@petamoriken/float16@3.9.1", "", {}, "sha512-j+ejhYwY6PeB+v1kn7lZFACUIG97u90WxMuGosILFsl9d4Ovi0sjk0GlPfoEcx+FzvXZDAfioD+NGnnPamXgMA=="], "@scalar/openapi-types": ["@scalar/openapi-types@0.1.1", "", {}, "sha512-NMy3QNk6ytcCoPUGJH0t4NNr36OWXgZhA3ormr3TvhX1NDgoF95wFyodGVH8xiHeUyn2/FxtETm8UBLbB5xEmg=="], @@ -95,16 +137,28 @@ "@unhead/schema": ["@unhead/schema@1.11.14", "", { "dependencies": { "hookable": "^5.5.3", "zhead": "^2.2.4" } }, "sha512-V9W9u5tF1/+TiLqxu+Qvh1ShoMDkPEwHoEo4DKdDG6ko7YlbzFfDxV6el9JwCren45U/4Vy/4Xi7j8OH02wsiA=="], + "blurhash": ["blurhash@2.0.5", "", {}, "sha512-cRygWd7kGBQO3VEhPiTgq4Wc43ctsM+o46urrmPOiuAe+07fzlSB9OJVdpgDL0jPqXUVQ9ht7aq7kxOeJHRK+w=="], + "buffer-from": ["buffer-from@1.1.2", "", {}, "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ=="], "bun-types": ["bun-types@1.2.4", "", { "dependencies": { "@types/node": "*", "@types/ws": "~8.5.10" } }, "sha512-nDPymR207ZZEoWD4AavvEaa/KZe/qlrbMSchqpQwovPZCKc7pwMoENjEtHgMKaAjJhy+x6vfqSBA1QU3bJgs0Q=="], "char-info": ["char-info@0.3.5", "", { "dependencies": { "node-interval-tree": "^1.3.3" } }, "sha512-gRslEBFEcuLMGLNO1EFIrdN1MMUfO+aqa7y8iWzNyAzB3mYKnTIvP+ioW3jpyeEvqA5WapVLIPINGtFjEIH4cQ=="], + "color": ["color@4.2.3", "", { "dependencies": { "color-convert": "^2.0.1", "color-string": "^1.9.0" } }, "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A=="], + + "color-convert": ["color-convert@2.0.1", "", { "dependencies": { "color-name": "~1.1.4" } }, "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ=="], + + "color-name": ["color-name@1.1.4", "", {}, "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="], + + "color-string": ["color-string@1.9.1", "", { "dependencies": { "color-name": "^1.0.0", "simple-swizzle": "^0.2.2" } }, "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg=="], + "cookie": ["cookie@1.0.2", "", {}, "sha512-9Kr/j4O16ISv8zBBhJoi4bXOYNTkFLOqSL3UDB0njXxCXNezjeyVrJyGOWtgfs/q2km1gwBcfH8q1yEGoMYunA=="], "debug": ["debug@4.4.0", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA=="], + "detect-libc": ["detect-libc@2.0.3", "", {}, "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw=="], + "drizzle-kit": ["drizzle-kit@0.30.5", "", { "dependencies": { "@drizzle-team/brocli": "^0.10.2", "@esbuild-kit/esm-loader": "^2.5.5", "esbuild": "^0.19.7", "esbuild-register": "^3.5.0", "gel": "^2.0.0" }, "bin": { "drizzle-kit": "bin.cjs" } }, "sha512-l6dMSE100u7sDaTbLczibrQZjA35jLsHNqIV+jmhNVO3O8jzM6kywMOmV9uOz9ZVSCMPQhAZEFjL/qDPVrqpUA=="], "drizzle-orm": ["drizzle-orm@0.39.0", "", { "peerDependencies": { "@aws-sdk/client-rds-data": ">=3", "@cloudflare/workers-types": ">=4", "@electric-sql/pglite": ">=0.2.0", "@libsql/client": ">=0.10.0", "@libsql/client-wasm": ">=0.10.0", "@neondatabase/serverless": ">=0.10.0", "@op-engineering/op-sqlite": ">=2", "@opentelemetry/api": "^1.4.1", "@planetscale/database": ">=1", "@prisma/client": "*", "@tidbcloud/serverless": "*", "@types/better-sqlite3": "*", "@types/pg": "*", "@types/react": ">=18", "@types/sql.js": "*", "@vercel/postgres": ">=0.8.0", "@xata.io/client": "*", "better-sqlite3": ">=7", "bun-types": "*", "expo-sqlite": ">=14.0.0", "knex": "*", "kysely": "*", "mysql2": ">=2", "pg": ">=8", "postgres": ">=3", "react": ">=18", "sql.js": ">=1", "sqlite3": ">=5" }, "optionalPeers": ["@aws-sdk/client-rds-data", "@cloudflare/workers-types", "@electric-sql/pglite", "@libsql/client", "@libsql/client-wasm", "@neondatabase/serverless", "@op-engineering/op-sqlite", "@opentelemetry/api", "@planetscale/database", "@prisma/client", "@tidbcloud/serverless", "@types/better-sqlite3", "@types/pg", "@types/react", "@types/sql.js", "@vercel/postgres", "@xata.io/client", "better-sqlite3", "bun-types", "expo-sqlite", "knex", "kysely", "mysql2", "pg", "postgres", "react", "sql.js", "sqlite3"] }, "sha512-kkZwo3Jvht0fdJD/EWGx0vYcEK0xnGrlNVaY07QYluRZA9N21B9VFbY+54bnb/1xvyzcg97tE65xprSAP/fFGQ=="], @@ -123,6 +177,8 @@ "hookable": ["hookable@5.5.3", "", {}, "sha512-Yc+BQe8SvoXH1643Qez1zqLRmbA5rCL+sSmk6TVos0LWVfNIB7PGncdlId77WzLGSIB5KaWgTaNTs2lNVEI6VQ=="], + "is-arrayish": ["is-arrayish@0.3.2", "", {}, "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ=="], + "isexe": ["isexe@3.1.1", "", {}, "sha512-LpB/54B+/2J5hqQ7imZHfdU31OlgQqx7ZicVlkm9kzg9/w8GKLEcFfJl/t7DCEDueOyBAD6zCCwTO6Fzs0NoEQ=="], "jose": ["jose@4.15.9", "", {}, "sha512-1vUQX+IdDMVPj4k8kOxgUqlcK518yluMuGZwqlr44FS1ppZB/5GWh4rZG89erpOBOJjU/OBsnCVFfapsRz6nEA=="], @@ -175,14 +231,20 @@ "shallowequal": ["shallowequal@1.1.0", "", {}, "sha512-y0m1JoUZSlPAjXVtPPW70aZWfIL/dSP7AFkRnniLCrK/8MDKog3TySTBmckD+RObVxH0v4Tox67+F14PdED2oQ=="], + "sharp": ["sharp@0.33.5", "", { "dependencies": { "color": "^4.2.3", "detect-libc": "^2.0.3", "semver": "^7.6.3" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "0.33.5", "@img/sharp-darwin-x64": "0.33.5", "@img/sharp-libvips-darwin-arm64": "1.0.4", "@img/sharp-libvips-darwin-x64": "1.0.4", "@img/sharp-libvips-linux-arm": "1.0.5", "@img/sharp-libvips-linux-arm64": "1.0.4", "@img/sharp-libvips-linux-s390x": "1.0.4", "@img/sharp-libvips-linux-x64": "1.0.4", "@img/sharp-libvips-linuxmusl-arm64": "1.0.4", "@img/sharp-libvips-linuxmusl-x64": "1.0.4", "@img/sharp-linux-arm": "0.33.5", "@img/sharp-linux-arm64": "0.33.5", "@img/sharp-linux-s390x": "0.33.5", "@img/sharp-linux-x64": "0.33.5", "@img/sharp-linuxmusl-arm64": "0.33.5", "@img/sharp-linuxmusl-x64": "0.33.5", "@img/sharp-wasm32": "0.33.5", "@img/sharp-win32-ia32": "0.33.5", "@img/sharp-win32-x64": "0.33.5" } }, "sha512-haPVm1EkS9pgvHrQ/F3Xy+hgcuMV0Wm9vfIBSiwZ05k+xgb0PkBQpGsAA/oWdDobNaZTH5ppvHtzCFbnSEwHVw=="], + "shell-quote": ["shell-quote@1.8.2", "", {}, "sha512-AzqKpGKjrj7EM6rKVQEPpB288oCfnrEIuyoT9cyF4nmGa7V8Zk6f7RRqYisX8X9m+Q7bd632aZW4ky7EhbQztA=="], + "simple-swizzle": ["simple-swizzle@0.2.2", "", { "dependencies": { "is-arrayish": "^0.3.1" } }, "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg=="], + "source-map": ["source-map@0.6.1", "", {}, "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="], "source-map-support": ["source-map-support@0.5.21", "", { "dependencies": { "buffer-from": "^1.0.0", "source-map": "^0.6.0" } }, "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w=="], "split2": ["split2@4.2.0", "", {}, "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg=="], + "tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], + "undici-types": ["undici-types@6.20.0", "", {}, "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg=="], "which": ["which@4.0.0", "", { "dependencies": { "isexe": "^3.1.1" }, "bin": { "node-which": "bin/which.js" } }, "sha512-GlaYyEb07DPxYCKhKzplCWBJtvxZcZMrL+4UkrTSJHHPyZU4mYYTv3qaOe77H7EODLSSopAUFAc6W8U4yqvscg=="], diff --git a/api/package.json b/api/package.json index b91418d2..ea814f69 100644 --- a/api/package.json +++ b/api/package.json @@ -11,11 +11,13 @@ "dependencies": { "@elysiajs/jwt": "^1.2.0", "@elysiajs/swagger": "^1.2.2", + "blurhash": "^2.0.5", "drizzle-kit": "^0.30.4", "drizzle-orm": "0.39.0", "elysia": "^1.2.23", "parjs": "^1.3.9", - "pg": "^8.13.3" + "pg": "^8.13.3", + "sharp": "^0.33.5" }, "devDependencies": { "@types/pg": "^8.11.11", diff --git a/api/src/controllers/seed/images.ts b/api/src/controllers/seed/images.ts index d3caba44..55b9edec 100644 --- a/api/src/controllers/seed/images.ts +++ b/api/src/controllers/seed/images.ts @@ -1,11 +1,18 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { encode } from "blurhash"; import { eq, sql } from "drizzle-orm"; import { version } from "package.json"; import type { PoolClient } from "pg"; +import sharp from "sharp"; import { db } from "~/db"; import * as schema from "~/db/schema"; import { mqueue } from "~/db/schema/queue"; import type { Image } from "~/models/utils"; +export const imageDir = process.env.IMAGES_PATH ?? "/images"; +await mkdir(imageDir, { recursive: true }); + type ImageTask = { id: string; url: string; @@ -52,8 +59,7 @@ export const processImages = async () => { .limit(1); const img = item.message as ImageTask; - await fetch(img.url, { headers: { "User-Agent": `Kyoo v${version}` }, }); - const blurhash = ""; + const blurhash = await downloadImage(img.id, img.url); const table = schema[img.table as keyof typeof schema] as any; @@ -72,3 +78,46 @@ export const processImages = async () => { if (evt.channel !== "image") return; }); }; + +async function downloadImage(id: string, url: string): Promise { + const resp = await fetch(url, { + headers: { "User-Agent": `Kyoo v${version}` }, + }); + if (!resp.ok) { + throw new Error(`Failed to fetch image: ${resp.status} ${resp.statusText}`); + } + const buf = Buffer.from(await resp.arrayBuffer()); + + const image = sharp(buf); + const metadata = await image.metadata(); + + if (!metadata.width || !metadata.height) { + throw new Error("Could not determine image dimensions"); + } + const resolutions = { + low: { width: 320 }, + medium: { width: 640 }, + high: { width: 1280 }, + }; + await Promise.all( + Object.entries(resolutions).map(async ([resolution, dimensions]) => { + const buffer = await image.clone().resize(dimensions.width).toBuffer(); + await writeFile(path.join(imageDir, `${id}.${resolution}.jpg`), buffer); + }), + ); + + const { data, info } = await image + .resize(32, 32, { fit: "inside" }) + .raw() + .toBuffer({ resolveWithObject: true }); + + const blurHash = encode( + new Uint8ClampedArray(data), + info.width, + info.height, + 4, + 3, + ); + + return blurHash; +} From 71b8cbca4a8b0c39a2affe5ed0e8cf0bbcc2082a Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 16 Mar 2025 14:51:58 +0100 Subject: [PATCH 3/9] Create task runner to download images --- api/src/controllers/seed/images.ts | 79 ++++++++++++++++++++---------- 1 file changed, 53 insertions(+), 26 deletions(-) diff --git a/api/src/controllers/seed/images.ts b/api/src/controllers/seed/images.ts index 55b9edec..18e5faa2 100644 --- a/api/src/controllers/seed/images.ts +++ b/api/src/controllers/seed/images.ts @@ -25,58 +25,85 @@ type ImageTask = { // requests are not blocked by image downloading or blurhash calculation export const enqueueImage = async ( tx: typeof db, - url: string, + img: Omit, ): Promise => { const hasher = new Bun.CryptoHasher("sha256"); - hasher.update(url); + hasher.update(img.url); const id = hasher.digest().toString("hex"); - await tx.insert(mqueue).values({ kind: "image", message: { id, url } }); + await tx.insert(mqueue).values({ kind: "image", message: { id, ...img } }); + await tx.execute(sql`notify image`); return { id, - source: url, + source: img.url, blurhash: "", }; }; export const enqueueOptImage = async ( tx: typeof db, - url: string | null, + img: Omit, ): Promise => { - if (!url) return null; - return await enqueueImage(tx, url); + if (!img.url) return null; + return await enqueueImage(tx, img); }; export const processImages = async () => { - await db.transaction(async (tx) => { - const [item] = await tx - .select() - .from(mqueue) - .for("update", { skipLocked: true }) - .where(eq(mqueue.kind, "image")) - .orderBy(mqueue.createdAt) - .limit(1); + async function processOne() { + return await db.transaction(async (tx) => { + const [item] = await tx + .select() + .from(mqueue) + .for("update", { skipLocked: true }) + .where(eq(mqueue.kind, "image")) + .orderBy(mqueue.createdAt) + .limit(1); - const img = item.message as ImageTask; - const blurhash = await downloadImage(img.id, img.url); + if (!item) return false; - const table = schema[img.table as keyof typeof schema] as any; + const img = item.message as ImageTask; + const blurhash = await downloadImage(img.id, img.url); - await tx - .update(table) - .set({ - [img.column]: { id: img.id, source: img.url, blurhash } satisfies Image, - }) - .where(eq(sql`${table[img.column]}->'id'`, img.id)); + const table = schema[img.table as keyof typeof schema] as any; - await tx.delete(mqueue).where(eq(mqueue.id, item.id)); - }); + await tx + .update(table) + .set({ + [img.column]: { + id: img.id, + source: img.url, + blurhash, + } satisfies Image, + }) + .where(eq(sql`${table[img.column]}->'id'`, img.id)); + + await tx.delete(mqueue).where(eq(mqueue.id, item.id)); + return true; + }); + } + + let running = false; + async function processAll() { + if (running) return; + running = true; + + let found = true; + while (found) { + found = await processOne(); + } + running = false; + } const client = (await db.$client.connect()) as PoolClient; client.on("notification", (evt) => { if (evt.channel !== "image") return; + processAll(); }); + await client.query("listen image"); + + // start processing old tasks + await processAll(); }; async function downloadImage(id: string, url: string): Promise { From 9ef114b91a629978905fe3ee702aada963f049a6 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 16 Mar 2025 17:58:13 +0100 Subject: [PATCH 4/9] Use reworked image queue in every insert --- api/src/controllers/seed/images.ts | 29 +++-- api/src/controllers/seed/insert/collection.ts | 26 +++-- api/src/controllers/seed/insert/entries.ts | 101 ++++++++++-------- api/src/controllers/seed/insert/seasons.ts | 38 +++++-- api/src/controllers/seed/insert/shows.ts | 60 +++++++---- api/src/controllers/seed/insert/staff.ts | 38 ++++--- api/src/controllers/seed/insert/studios.ts | 26 +++-- api/src/controllers/seed/movies.ts | 17 --- api/src/controllers/seed/series.ts | 18 +--- api/src/db/index.ts | 4 + api/src/db/schema/shows.ts | 8 +- 11 files changed, 218 insertions(+), 147 deletions(-) diff --git a/api/src/controllers/seed/images.ts b/api/src/controllers/seed/images.ts index 18e5faa2..9bce15e1 100644 --- a/api/src/controllers/seed/images.ts +++ b/api/src/controllers/seed/images.ts @@ -2,10 +2,11 @@ import { mkdir, writeFile } from "node:fs/promises"; import path from "node:path"; import { encode } from "blurhash"; import { eq, sql } from "drizzle-orm"; +import type { PgColumn } from "drizzle-orm/pg-core"; import { version } from "package.json"; import type { PoolClient } from "pg"; import sharp from "sharp"; -import { db } from "~/db"; +import { type Transaction, db } from "~/db"; import * as schema from "~/db/schema"; import { mqueue } from "~/db/schema/queue"; import type { Image } from "~/models/utils"; @@ -20,18 +21,31 @@ type ImageTask = { column: string; }; +type ImageTaskC = { + url: string; + column: PgColumn; +}; + // this will only push a task to the image downloader service and not download it instantly. // this is both done to prevent to many requests to be sent at once and to make sure POST // requests are not blocked by image downloading or blurhash calculation export const enqueueImage = async ( - tx: typeof db, - img: Omit, + tx: Transaction, + img: ImageTaskC, ): Promise => { const hasher = new Bun.CryptoHasher("sha256"); hasher.update(img.url); const id = hasher.digest().toString("hex"); - await tx.insert(mqueue).values({ kind: "image", message: { id, ...img } }); + await tx.insert(mqueue).values({ + kind: "image", + message: { + id, + url: img.url, + table: img.column.table._.name, + column: img.column.name, + } satisfies ImageTask, + }); await tx.execute(sql`notify image`); return { @@ -42,11 +56,11 @@ export const enqueueImage = async ( }; export const enqueueOptImage = async ( - tx: typeof db, - img: Omit, + tx: Transaction, + img: { url: string | null; column: PgColumn }, ): Promise => { if (!img.url) return null; - return await enqueueImage(tx, img); + return await enqueueImage(tx, { url: img.url, column: img.column }); }; export const processImages = async () => { @@ -107,6 +121,7 @@ export const processImages = async () => { }; async function downloadImage(id: string, url: string): Promise { + // TODO: check if file exists before downloading const resp = await fetch(url, { headers: { "User-Agent": `Kyoo v${version}` }, }); diff --git a/api/src/controllers/seed/insert/collection.ts b/api/src/controllers/seed/insert/collection.ts index 495a3f33..b5a1e65a 100644 --- a/api/src/controllers/seed/insert/collection.ts +++ b/api/src/controllers/seed/insert/collection.ts @@ -48,16 +48,28 @@ export const insertCollection = async ( }) .returning({ pk: shows.pk, id: shows.id, slug: shows.slug }); - const trans: ShowTrans[] = Object.entries(translations).map( - ([lang, tr]) => ({ + const trans: ShowTrans[] = await Promise.all( + Object.entries(translations).map(async ([lang, tr]) => ({ pk: ret.pk, language: lang, ...tr, - poster: enqueueOptImage(tr.poster), - thumbnail: enqueueOptImage(tr.thumbnail), - logo: enqueueOptImage(tr.logo), - banner: enqueueOptImage(tr.banner), - }), + poster: await enqueueOptImage(tx, { + url: tr.poster, + column: showTranslations.poster, + }), + thumbnail: await enqueueOptImage(tx, { + url: tr.thumbnail, + column: showTranslations.thumbnail, + }), + logo: await enqueueOptImage(tx, { + url: tr.logo, + column: showTranslations.logo, + }), + banner: await enqueueOptImage(tx, { + url: tr.banner, + column: showTranslations.banner, + }), + })), ); await tx .insert(showTranslations) diff --git a/api/src/controllers/seed/insert/entries.ts b/api/src/controllers/seed/insert/entries.ts index d57a5b4a..10ce07fa 100644 --- a/api/src/controllers/seed/insert/entries.ts +++ b/api/src/controllers/seed/insert/entries.ts @@ -23,6 +23,7 @@ type SeedExtra = Omit & { }; type EntryI = typeof entries.$inferInsert; +type EntryTransI = typeof entryTranslations.$inferInsert; const generateSlug = ( showSlug: string, @@ -49,25 +50,30 @@ export const insertEntries = async ( if (!items) return []; const retEntries = await db.transaction(async (tx) => { - const vals: EntryI[] = items.map((seed) => { - const { translations, videos, video, ...entry } = seed; - return { - ...entry, - showPk: show.pk, - slug: generateSlug(show.slug, seed), - thumbnail: enqueueOptImage(seed.thumbnail), - nextRefresh: - entry.kind !== "extra" - ? guessNextRefresh(entry.airDate ?? new Date()) - : guessNextRefresh(new Date()), - episodeNumber: - entry.kind === "episode" - ? entry.episodeNumber - : entry.kind === "special" - ? entry.number - : undefined, - }; - }); + const vals: EntryI[] = await Promise.all( + items.map(async (seed) => { + const { translations, videos, video, ...entry } = seed; + return { + ...entry, + showPk: show.pk, + slug: generateSlug(show.slug, seed), + thumbnail: await enqueueOptImage(tx, { + url: seed.thumbnail, + column: entries.thumbnail, + }), + nextRefresh: + entry.kind !== "extra" + ? guessNextRefresh(entry.airDate ?? new Date()) + : guessNextRefresh(new Date()), + episodeNumber: + entry.kind === "episode" + ? entry.episodeNumber + : entry.kind === "special" + ? entry.number + : undefined, + }; + }), + ); const ret = await tx .insert(entries) .values(vals) @@ -83,30 +89,41 @@ export const insertEntries = async ( }) .returning({ pk: entries.pk, id: entries.id, slug: entries.slug }); - const trans = items.flatMap((seed, i) => { - if (seed.kind === "extra") { - return { - pk: ret[i].pk, - // yeah we hardcode the language to extra because if we want to support - // translations one day it won't be awkward - language: "extra", - name: seed.name, - description: null, - poster: undefined, - }; - } + const trans: EntryTransI[] = ( + await Promise.all( + items.map(async (seed, i) => { + if (seed.kind === "extra") { + return [ + { + pk: ret[i].pk, + // yeah we hardcode the language to extra because if we want to support + // translations one day it won't be awkward + language: "extra", + name: seed.name, + description: null, + poster: undefined, + }, + ]; + } - return Object.entries(seed.translations).map(([lang, tr]) => ({ - // assumes ret is ordered like items. - pk: ret[i].pk, - language: lang, - ...tr, - poster: - seed.kind === "movie" - ? enqueueOptImage((tr as any).poster) - : undefined, - })); - }); + return await Promise.all( + Object.entries(seed.translations).map(async ([lang, tr]) => ({ + // assumes ret is ordered like items. + pk: ret[i].pk, + language: lang, + ...tr, + poster: + seed.kind === "movie" + ? await enqueueOptImage(tx, { + url: (tr as any).poster, + column: entryTranslations.poster, + }) + : undefined, + })), + ); + }), + ) + ).flat(); await tx .insert(entryTranslations) .values(trans) diff --git a/api/src/controllers/seed/insert/seasons.ts b/api/src/controllers/seed/insert/seasons.ts index bdb5ee39..5f43b8a0 100644 --- a/api/src/controllers/seed/insert/seasons.ts +++ b/api/src/controllers/seed/insert/seasons.ts @@ -37,17 +37,33 @@ export const insertSeasons = async ( }) .returning({ pk: seasons.pk, id: seasons.id, slug: seasons.slug }); - const trans: SeasonTransI[] = items.flatMap((seed, i) => - Object.entries(seed.translations).map(([lang, tr]) => ({ - // assumes ret is ordered like items. - pk: ret[i].pk, - language: lang, - ...tr, - poster: enqueueOptImage(tr.poster), - thumbnail: enqueueOptImage(tr.thumbnail), - banner: enqueueOptImage(tr.banner), - })), - ); + const trans: SeasonTransI[] = ( + await Promise.all( + items.map( + async (seed, i) => + await Promise.all( + Object.entries(seed.translations).map(async ([lang, tr]) => ({ + // assumes ret is ordered like items. + pk: ret[i].pk, + language: lang, + ...tr, + poster: await enqueueOptImage(tx, { + url: tr.poster, + column: seasonTranslations.poster, + }), + thumbnail: await enqueueOptImage(tx, { + url: tr.thumbnail, + column: seasonTranslations.thumbnail, + }), + banner: await enqueueOptImage(tx, { + url: tr.banner, + column: seasonTranslations.banner, + }), + })), + ), + ), + ) + ).flat(); await tx .insert(seasonTranslations) .values(trans) diff --git a/api/src/controllers/seed/insert/shows.ts b/api/src/controllers/seed/insert/shows.ts index 580738b3..f84e72eb 100644 --- a/api/src/controllers/seed/insert/shows.ts +++ b/api/src/controllers/seed/insert/shows.ts @@ -1,5 +1,5 @@ import { and, count, eq, exists, ne, sql } from "drizzle-orm"; -import { db } from "~/db"; +import { type Transaction, db } from "~/db"; import { entries, entryVideoJoin, showTranslations, shows } from "~/db/schema"; import { conflictUpdateAllExcept, sqlarr } from "~/db/utils"; import type { SeedCollection } from "~/models/collections"; @@ -12,30 +12,53 @@ type Show = typeof shows.$inferInsert; type ShowTrans = typeof showTranslations.$inferInsert; export const insertShow = async ( - show: Show, + show: Omit & { originalLanguage: string }, translations: | SeedMovie["translations"] | SeedSerie["translations"] | SeedCollection["translations"], ) => { return await db.transaction(async (tx) => { - const ret = await insertBaseShow(tx, show); + const trans: (Omit & { latinName: string | null })[] = + await Promise.all( + Object.entries(translations).map(async ([lang, tr]) => ({ + language: lang, + ...tr, + latinName: tr.latinName ?? null, + poster: await enqueueOptImage(tx, { + url: tr.poster, + column: showTranslations.poster, + }), + thumbnail: await enqueueOptImage(tx, { + url: tr.thumbnail, + column: showTranslations.thumbnail, + }), + logo: await enqueueOptImage(tx, { + url: tr.logo, + column: showTranslations.logo, + }), + banner: await enqueueOptImage(tx, { + url: tr.banner, + column: showTranslations.banner, + }), + })), + ); + const original = trans.find((x) => x.language === show.originalLanguage); + + if (!original) { + tx.rollback(); + return { + status: 422 as const, + message: "No translation available in the original language.", + }; + } + + const ret = await insertBaseShow(tx, { ...show, original }); if ("status" in ret) return ret; - const trans: ShowTrans[] = Object.entries(translations).map( - ([lang, tr]) => ({ - pk: ret.pk, - language: lang, - ...tr, - poster: enqueueOptImage(tr.poster), - thumbnail: enqueueOptImage(tr.thumbnail), - logo: enqueueOptImage(tr.logo), - banner: enqueueOptImage(tr.banner), - }), - ); await tx .insert(showTranslations) - .values(trans) + .values(trans.map((x) => ({ ...x, pk: ret.pk }))) .onConflictDoUpdate({ target: [showTranslations.pk, showTranslations.language], set: conflictUpdateAllExcept(showTranslations, ["pk", "language"]), @@ -44,10 +67,7 @@ export const insertShow = async ( }); }; -async function insertBaseShow( - tx: Parameters[0]>[0], - show: Show, -) { +async function insertBaseShow(tx: Transaction, show: Show) { function insert() { return tx .insert(shows) @@ -97,7 +117,7 @@ async function insertBaseShow( } export async function updateAvailableCount( - tx: typeof db | Parameters[0]>[0], + tx: Transaction, showPks: number[], updateEntryCount = true, ) { diff --git a/api/src/controllers/seed/insert/staff.ts b/api/src/controllers/seed/insert/staff.ts index d84092d3..8fa96d25 100644 --- a/api/src/controllers/seed/insert/staff.ts +++ b/api/src/controllers/seed/insert/staff.ts @@ -12,10 +12,15 @@ export const insertStaff = async ( if (!seed?.length) return []; return await db.transaction(async (tx) => { - const people = seed.map((x) => ({ - ...x.staff, - image: enqueueOptImage(x.staff.image), - })); + const people = await Promise.all( + seed.map(async (x) => ({ + ...x.staff, + image: await enqueueOptImage(tx, { + url: x.staff.image, + column: staff.image, + }), + })), + ); const ret = await tx .insert(staff) .values(people) @@ -25,16 +30,21 @@ export const insertStaff = async ( }) .returning({ pk: staff.pk, id: staff.id, slug: staff.slug }); - const rval = seed.map((x, i) => ({ - showPk, - staffPk: ret[i].pk, - kind: x.kind, - order: i, - character: { - ...x.character, - image: enqueueOptImage(x.character.image), - }, - })); + const rval = await Promise.all( + seed.map(async (x, i) => ({ + showPk, + staffPk: ret[i].pk, + kind: x.kind, + order: i, + character: { + ...x.character, + image: await enqueueOptImage(tx, { + url: x.character.image, + column: roles.character.image, + }), + }, + })), + ); // always replace all roles. this is because: // - we want `order` to stay in sync (& without duplicates) diff --git a/api/src/controllers/seed/insert/studios.ts b/api/src/controllers/seed/insert/studios.ts index 85d18403..7eef1bd1 100644 --- a/api/src/controllers/seed/insert/studios.ts +++ b/api/src/controllers/seed/insert/studios.ts @@ -33,14 +33,24 @@ export const insertStudios = async ( }) .returning({ pk: studios.pk, id: studios.id, slug: studios.slug }); - const trans: StudioTransI[] = seed.flatMap((x, i) => - Object.entries(x.translations).map(([lang, tr]) => ({ - pk: ret[i].pk, - language: lang, - name: tr.name, - logo: enqueueOptImage(tr.logo), - })), - ); + const trans: StudioTransI[] = ( + await Promise.all( + seed.map( + async (x, i) => + await Promise.all( + Object.entries(x.translations).map(async ([lang, tr]) => ({ + pk: ret[i].pk, + language: lang, + name: tr.name, + logo: await enqueueOptImage(tx, { + url: tr.logo, + column: studioTranslations.logo, + }), + })), + ), + ), + ) + ).flat(); await tx .insert(studioTranslations) .values(trans) diff --git a/api/src/controllers/seed/movies.ts b/api/src/controllers/seed/movies.ts index 9ab4ac85..2102b6ce 100644 --- a/api/src/controllers/seed/movies.ts +++ b/api/src/controllers/seed/movies.ts @@ -1,7 +1,6 @@ import { t } from "elysia"; import type { SeedMovie } from "~/models/movie"; import { getYear } from "~/utils"; -import { enqueueOptImage } from "./images"; import { insertCollection } from "./insert/collection"; import { insertEntries } from "./insert/entries"; import { insertShow, updateAvailableCount } from "./insert/shows"; @@ -55,13 +54,6 @@ export const seedMovie = async ( const { translations, videos, collection, studios, staff, ...movie } = seed; const nextRefresh = guessNextRefresh(movie.airDate ?? new Date()); - const original = translations[movie.originalLanguage]; - if (!original) { - return { - status: 422, - message: "No translation available in the original language.", - }; - } const col = await insertCollection(collection, { kind: "movie", @@ -76,15 +68,6 @@ export const seedMovie = async ( nextRefresh, collectionPk: col?.pk, entriesCount: 1, - original: { - language: movie.originalLanguage, - name: original.name, - latinName: original.latinName ?? null, - poster: enqueueOptImage(original.poster), - thumbnail: enqueueOptImage(original.thumbnail), - logo: enqueueOptImage(original.logo), - banner: enqueueOptImage(original.banner), - }, ...movie, }, translations, diff --git a/api/src/controllers/seed/series.ts b/api/src/controllers/seed/series.ts index 62b6d453..d207439f 100644 --- a/api/src/controllers/seed/series.ts +++ b/api/src/controllers/seed/series.ts @@ -5,7 +5,7 @@ import { enqueueOptImage } from "./images"; import { insertCollection } from "./insert/collection"; import { insertEntries } from "./insert/entries"; import { insertSeasons } from "./insert/seasons"; -import { insertShow, updateAvailableCount } from "./insert/shows"; +import { insertShow } from "./insert/shows"; import { insertStaff } from "./insert/staff"; import { insertStudios } from "./insert/studios"; import { guessNextRefresh } from "./refresh"; @@ -91,13 +91,6 @@ export const seedSerie = async ( ...serie } = seed; const nextRefresh = guessNextRefresh(serie.startAir ?? new Date()); - const original = translations[serie.originalLanguage]; - if (!original) { - return { - status: 422, - message: "No translation available in the original language.", - }; - } const col = await insertCollection(collection, { kind: "serie", @@ -111,15 +104,6 @@ export const seedSerie = async ( nextRefresh, collectionPk: col?.pk, entriesCount: entries.length, - original: { - language: serie.originalLanguage, - name: original.name, - latinName: original.latinName ?? null, - poster: enqueueOptImage(original.poster), - thumbnail: enqueueOptImage(original.thumbnail), - logo: enqueueOptImage(original.logo), - banner: enqueueOptImage(original.banner), - }, ...serie, }, translations, diff --git a/api/src/db/index.ts b/api/src/db/index.ts index 0745761f..0935a8c9 100644 --- a/api/src/db/index.ts +++ b/api/src/db/index.ts @@ -31,3 +31,7 @@ export const migrate = async () => { }); console.log(`Database ${dbConfig.database} migrated!`); }; + +export type Transaction = + | typeof db + | Parameters[0]>[0]; diff --git a/api/src/db/schema/shows.ts b/api/src/db/schema/shows.ts index 6729071f..35a75bec 100644 --- a/api/src/db/schema/shows.ts +++ b/api/src/db/schema/shows.ts @@ -58,10 +58,10 @@ export const genres = schema.enum("genres", [ ]); type OriginalWithImages = Original & { - poster: Image | null; - thumbnail: Image | null; - banner: Image | null; - logo: Image | null; + poster?: Image | null; + thumbnail?: Image | null; + banner?: Image | null; + logo?: Image | null; }; export const shows = schema.table( From 6e642db7db5d59314122cbc5ab6333d1771830c4 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 16 Mar 2025 18:10:21 +0100 Subject: [PATCH 5/9] Cleanup original images handling --- api/src/controllers/seed/insert/shows.ts | 90 ++++++++++++++---------- api/src/controllers/seed/movies.ts | 13 ++++ api/src/controllers/seed/series.ts | 13 ++++ 3 files changed, 79 insertions(+), 37 deletions(-) diff --git a/api/src/controllers/seed/insert/shows.ts b/api/src/controllers/seed/insert/shows.ts index f84e72eb..66cf81dc 100644 --- a/api/src/controllers/seed/insert/shows.ts +++ b/api/src/controllers/seed/insert/shows.ts @@ -5,6 +5,7 @@ import { conflictUpdateAllExcept, sqlarr } from "~/db/utils"; import type { SeedCollection } from "~/models/collections"; import type { SeedMovie } from "~/models/movie"; import type { SeedSerie } from "~/models/serie"; +import type { Original } from "~/models/utils"; import { getYear } from "~/utils"; import { enqueueOptImage } from "../images"; @@ -12,53 +13,68 @@ type Show = typeof shows.$inferInsert; type ShowTrans = typeof showTranslations.$inferInsert; export const insertShow = async ( - show: Omit & { originalLanguage: string }, + show: Omit, + original: Original & { + poster: string | null; + thumbnail: string | null; + banner: string | null; + logo: string | null; + }, translations: | SeedMovie["translations"] | SeedSerie["translations"] | SeedCollection["translations"], ) => { return await db.transaction(async (tx) => { - const trans: (Omit & { latinName: string | null })[] = - await Promise.all( - Object.entries(translations).map(async ([lang, tr]) => ({ - language: lang, - ...tr, - latinName: tr.latinName ?? null, - poster: await enqueueOptImage(tx, { - url: tr.poster, - column: showTranslations.poster, - }), - thumbnail: await enqueueOptImage(tx, { - url: tr.thumbnail, - column: showTranslations.thumbnail, - }), - logo: await enqueueOptImage(tx, { - url: tr.logo, - column: showTranslations.logo, - }), - banner: await enqueueOptImage(tx, { - url: tr.banner, - column: showTranslations.banner, - }), - })), - ); - const original = trans.find((x) => x.language === show.originalLanguage); - - if (!original) { - tx.rollback(); - return { - status: 422 as const, - message: "No translation available in the original language.", - }; - } - - const ret = await insertBaseShow(tx, { ...show, original }); + const orig = { + ...original, + poster: await enqueueOptImage(tx, { + url: original.poster, + column: shows.original.poster, + }), + thumbnail: await enqueueOptImage(tx, { + url: original.thumbnail, + column: shows.original.thumbnail, + }), + banner: await enqueueOptImage(tx, { + url: original.banner, + column: shows.original.banner, + }), + logo: await enqueueOptImage(tx, { + url: original.logo, + column: shows.original.logo, + }), + }; + const ret = await insertBaseShow(tx, { ...show, original: orig }); if ("status" in ret) return ret; + const trans: ShowTrans[] = await Promise.all( + Object.entries(translations).map(async ([lang, tr]) => ({ + pk: ret.pk, + language: lang, + ...tr, + latinName: tr.latinName ?? null, + poster: await enqueueOptImage(tx, { + url: tr.poster, + column: showTranslations.poster, + }), + thumbnail: await enqueueOptImage(tx, { + url: tr.thumbnail, + column: showTranslations.thumbnail, + }), + logo: await enqueueOptImage(tx, { + url: tr.logo, + column: showTranslations.logo, + }), + banner: await enqueueOptImage(tx, { + url: tr.banner, + column: showTranslations.banner, + }), + })), + ); await tx .insert(showTranslations) - .values(trans.map((x) => ({ ...x, pk: ret.pk }))) + .values(trans) .onConflictDoUpdate({ target: [showTranslations.pk, showTranslations.language], set: conflictUpdateAllExcept(showTranslations, ["pk", "language"]), diff --git a/api/src/controllers/seed/movies.ts b/api/src/controllers/seed/movies.ts index 2102b6ce..158c94bb 100644 --- a/api/src/controllers/seed/movies.ts +++ b/api/src/controllers/seed/movies.ts @@ -55,6 +55,14 @@ export const seedMovie = async ( const { translations, videos, collection, studios, staff, ...movie } = seed; const nextRefresh = guessNextRefresh(movie.airDate ?? new Date()); + const original = translations[movie.originalLanguage]; + if (!original) { + return { + status: 422, + message: "No translation available in the original language.", + }; + } + const col = await insertCollection(collection, { kind: "movie", nextRefresh, @@ -70,6 +78,11 @@ export const seedMovie = async ( entriesCount: 1, ...movie, }, + { + ...original, + latinName: original.latinName ?? null, + language: movie.originalLanguage, + }, translations, ); if ("status" in show) return show; diff --git a/api/src/controllers/seed/series.ts b/api/src/controllers/seed/series.ts index d207439f..500aa57a 100644 --- a/api/src/controllers/seed/series.ts +++ b/api/src/controllers/seed/series.ts @@ -92,6 +92,14 @@ export const seedSerie = async ( } = seed; const nextRefresh = guessNextRefresh(serie.startAir ?? new Date()); + const original = translations[serie.originalLanguage]; + if (!original) { + return { + status: 422, + message: "No translation available in the original language.", + }; + } + const col = await insertCollection(collection, { kind: "serie", nextRefresh, @@ -106,6 +114,11 @@ export const seedSerie = async ( entriesCount: entries.length, ...serie, }, + { + ...original, + latinName: original.latinName ?? null, + language: serie.originalLanguage, + }, translations, ); if ("status" in show) return show; From 71b57c50e7ab1df983231c76ef075e6fdbddff05 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 16 Mar 2025 20:31:34 +0100 Subject: [PATCH 6/9] Handle image in jsonb --- api/src/controllers/seed/images.ts | 66 +++++++++++------------- api/src/controllers/seed/insert/shows.ts | 12 +++-- api/src/controllers/seed/insert/staff.ts | 3 +- 3 files changed, 40 insertions(+), 41 deletions(-) diff --git a/api/src/controllers/seed/images.ts b/api/src/controllers/seed/images.ts index 9bce15e1..efaed7c2 100644 --- a/api/src/controllers/seed/images.ts +++ b/api/src/controllers/seed/images.ts @@ -1,13 +1,12 @@ import { mkdir, writeFile } from "node:fs/promises"; import path from "node:path"; import { encode } from "blurhash"; -import { eq, sql } from "drizzle-orm"; -import type { PgColumn } from "drizzle-orm/pg-core"; +import { type SQLWrapper, eq, sql } from "drizzle-orm"; +import type { PgColumn, PgTable } from "drizzle-orm/pg-core"; import { version } from "package.json"; import type { PoolClient } from "pg"; import sharp from "sharp"; import { type Transaction, db } from "~/db"; -import * as schema from "~/db/schema"; import { mqueue } from "~/db/schema/queue"; import type { Image } from "~/models/utils"; @@ -21,30 +20,38 @@ type ImageTask = { column: string; }; -type ImageTaskC = { - url: string; - column: PgColumn; -}; - // this will only push a task to the image downloader service and not download it instantly. // this is both done to prevent to many requests to be sent at once and to make sure POST // requests are not blocked by image downloading or blurhash calculation -export const enqueueImage = async ( +export const enqueueOptImage = async ( tx: Transaction, - img: ImageTaskC, -): Promise => { + img: + | { url: string | null; column: PgColumn } + | { url: string | null; table: PgTable; column: SQLWrapper }, +): Promise => { + if (!img.url) return null; + const hasher = new Bun.CryptoHasher("sha256"); hasher.update(img.url); const id = hasher.digest().toString("hex"); + const message: ImageTask = + "table" in img + ? { + id, + url: img.url, + table: img.table._.name, + column: img.column.getSQL().sql, + } + : { + id, + url: img.url, + table: img.column.table._.name, + column: img.column, + }; await tx.insert(mqueue).values({ kind: "image", - message: { - id, - url: img.url, - table: img.column.table._.name, - column: img.column.name, - } satisfies ImageTask, + message, }); await tx.execute(sql`notify image`); @@ -55,14 +62,6 @@ export const enqueueImage = async ( }; }; -export const enqueueOptImage = async ( - tx: Transaction, - img: { url: string | null; column: PgColumn }, -): Promise => { - if (!img.url) return null; - return await enqueueImage(tx, { url: img.url, column: img.column }); -}; - export const processImages = async () => { async function processOne() { return await db.transaction(async (tx) => { @@ -78,19 +77,14 @@ export const processImages = async () => { const img = item.message as ImageTask; const blurhash = await downloadImage(img.id, img.url); + const ret: Image = { id: img.id, source: img.url, blurhash }; - const table = schema[img.table as keyof typeof schema] as any; + const table = sql.raw(img.table); + const column = sql.raw(img.column); - await tx - .update(table) - .set({ - [img.column]: { - id: img.id, - source: img.url, - blurhash, - } satisfies Image, - }) - .where(eq(sql`${table[img.column]}->'id'`, img.id)); + await tx.execute(sql` + update ${table} set ${column} = ${ret} where ${column}->'id' = '${item.id}' + `); await tx.delete(mqueue).where(eq(mqueue.id, item.id)); return true; diff --git a/api/src/controllers/seed/insert/shows.ts b/api/src/controllers/seed/insert/shows.ts index 66cf81dc..fcc66ea3 100644 --- a/api/src/controllers/seed/insert/shows.ts +++ b/api/src/controllers/seed/insert/shows.ts @@ -30,19 +30,23 @@ export const insertShow = async ( ...original, poster: await enqueueOptImage(tx, { url: original.poster, - column: shows.original.poster, + table: shows, + column: sql`${shows.original}['poster']`, }), thumbnail: await enqueueOptImage(tx, { url: original.thumbnail, - column: shows.original.thumbnail, + table: shows, + column: sql`${shows.original}['thumbnail']`, }), banner: await enqueueOptImage(tx, { url: original.banner, - column: shows.original.banner, + table: shows, + column: sql`${shows.original}['banner']`, }), logo: await enqueueOptImage(tx, { url: original.logo, - column: shows.original.logo, + table: shows, + column: sql`${shows.original}['logo']`, }), }; const ret = await insertBaseShow(tx, { ...show, original: orig }); diff --git a/api/src/controllers/seed/insert/staff.ts b/api/src/controllers/seed/insert/staff.ts index 8fa96d25..fe89c880 100644 --- a/api/src/controllers/seed/insert/staff.ts +++ b/api/src/controllers/seed/insert/staff.ts @@ -40,7 +40,8 @@ export const insertStaff = async ( ...x.character, image: await enqueueOptImage(tx, { url: x.character.image, - column: roles.character.image, + table: roles, + column: `${roles.character}['image']`, }), }, })), From f11e1b56db9d83464a090b7acb754cdbef42690b Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 16 Mar 2025 21:59:43 +0100 Subject: [PATCH 7/9] Fix tests Also fix sharp on nixos --- api/.env.example | 2 +- api/bun.lock | 40 +++++++++++++----------- api/package.json | 1 + api/src/controllers/seed/images.ts | 14 ++++----- api/src/controllers/seed/insert/staff.ts | 4 +-- api/src/controllers/seed/movies.ts | 2 +- api/tsconfig.json | 12 ++----- shell.nix | 5 +++ 8 files changed, 42 insertions(+), 38 deletions(-) diff --git a/api/.env.example b/api/.env.example index 6d819716..9f395c93 100644 --- a/api/.env.example +++ b/api/.env.example @@ -7,7 +7,7 @@ JWT_SECRET= # keibi's server to retrieve the public jwt secret AUHT_SERVER=http://auth:4568 -IMAGES_PATH=/images +IMAGES_PATH=./images POSTGRES_USER=kyoo POSTGRES_PASSWORD=password diff --git a/api/bun.lock b/api/bun.lock index a78af62b..bdb5a132 100644 --- a/api/bun.lock +++ b/api/bun.lock @@ -2,6 +2,7 @@ "lockfileVersion": 1, "workspaces": { "": { + "name": "api", "dependencies": { "@elysiajs/jwt": "^1.2.0", "@elysiajs/swagger": "^1.2.2", @@ -16,6 +17,7 @@ "devDependencies": { "@types/pg": "^8.11.11", "bun-types": "^1.2.4", + "node-addon-api": "^8.3.1", }, }, }, @@ -119,29 +121,29 @@ "@img/sharp-win32-x64": ["@img/sharp-win32-x64@0.33.5", "", { "os": "win32", "cpu": "x64" }, "sha512-MpY/o8/8kj+EcnxwvrP4aTJSWw/aZ7JIGR4aBeZkZw5B7/Jn+tY9/VNwtcoGmdT7GfggGIU4kygOMSbYnOrAbg=="], - "@petamoriken/float16": ["@petamoriken/float16@3.9.1", "", {}, "sha512-j+ejhYwY6PeB+v1kn7lZFACUIG97u90WxMuGosILFsl9d4Ovi0sjk0GlPfoEcx+FzvXZDAfioD+NGnnPamXgMA=="], + "@petamoriken/float16": ["@petamoriken/float16@3.9.2", "", {}, "sha512-VgffxawQde93xKxT3qap3OH+meZf7VaSB5Sqd4Rqc+FP5alWbpOyan/7tRbOAvynjpG3GpdtAuGU/NdhQpmrog=="], "@scalar/openapi-types": ["@scalar/openapi-types@0.1.1", "", {}, "sha512-NMy3QNk6ytcCoPUGJH0t4NNr36OWXgZhA3ormr3TvhX1NDgoF95wFyodGVH8xiHeUyn2/FxtETm8UBLbB5xEmg=="], - "@scalar/themes": ["@scalar/themes@0.9.58", "", { "dependencies": { "@scalar/types": "0.0.25" } }, "sha512-voMgCIq0N19N8Ehjs8rSS0j5P1mpgWbpN5dXIToGUbVj7KcxMnOfkH3P1/cy2CoUd1gRYe0newUBEcI1+tQi1g=="], + "@scalar/themes": ["@scalar/themes@0.9.79", "", { "dependencies": { "@scalar/types": "0.1.1" } }, "sha512-zWiHCZAIjPGa8X9o/NORBPRMTMblLEz2+2RcfW9yIKNO/8H4Gz0rltiGGlJ6vX0o+qHwx7AdgfY+7njmWQR4ng=="], "@scalar/types": ["@scalar/types@0.0.12", "", { "dependencies": { "@scalar/openapi-types": "0.1.1", "@unhead/schema": "^1.9.5" } }, "sha512-XYZ36lSEx87i4gDqopQlGCOkdIITHHEvgkuJFrXFATQs9zHARop0PN0g4RZYWj+ZpCUclOcaOjbCt8JGe22mnQ=="], - "@sinclair/typebox": ["@sinclair/typebox@0.34.27", "", {}, "sha512-C7mxE1VC3WC2McOufZXEU48IfRVI+BcKxk4NOyNn3+JMUNdJHEWGS5CqjuDX+ij2NCCz8/nse1mT7yn8Fv2GHg=="], + "@sinclair/typebox": ["@sinclair/typebox@0.34.30", "", {}, "sha512-gFB3BiqjDxEoadW0zn+xyMVb7cLxPCoblVn2C/BKpI41WPYi2d6fwHAlynPNZ5O/Q4WEiujdnJzVtvG/Jc2CBQ=="], - "@types/node": ["@types/node@22.10.5", "", { "dependencies": { "undici-types": "~6.20.0" } }, "sha512-F8Q+SeGimwOo86fiovQh8qiXfFEh2/ocYv7tU5pJ3EXMSSxk1Joj5wefpFK2fHTf/N6HKGSxIDBT9f3gCxXPkQ=="], + "@types/node": ["@types/node@22.13.10", "", { "dependencies": { "undici-types": "~6.20.0" } }, "sha512-I6LPUvlRH+O6VRUqYOcMudhaIdUVWfsjnZavnsraHvpBwaEyMN29ry+0UVJhImYL16xsscu0aske3yA+uPOWfw=="], "@types/pg": ["@types/pg@8.11.11", "", { "dependencies": { "@types/node": "*", "pg-protocol": "*", "pg-types": "^4.0.1" } }, "sha512-kGT1qKM8wJQ5qlawUrEkXgvMSXoV213KfMGXcwfDwUIfUHXqXYXOfS1nE1LINRJVVVx5wCm70XnFlMHaIcQAfw=="], - "@types/ws": ["@types/ws@8.5.13", "", { "dependencies": { "@types/node": "*" } }, "sha512-osM/gWBTPKgHV8XkTunnegTRIsvF6owmf5w+JtAfOw472dptdm0dlGv4xCt6GwQRcC2XVOvvRE/0bAoQcL2QkA=="], + "@types/ws": ["@types/ws@8.5.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-bd/YFLW+URhBzMXurx7lWByOu+xzU9+kb3RboOteXYDfW+tr+JZa99OyNmPINEGB/ahzKrEuc8rcv4gnpJmxTw=="], - "@unhead/schema": ["@unhead/schema@1.11.14", "", { "dependencies": { "hookable": "^5.5.3", "zhead": "^2.2.4" } }, "sha512-V9W9u5tF1/+TiLqxu+Qvh1ShoMDkPEwHoEo4DKdDG6ko7YlbzFfDxV6el9JwCren45U/4Vy/4Xi7j8OH02wsiA=="], + "@unhead/schema": ["@unhead/schema@1.11.20", "", { "dependencies": { "hookable": "^5.5.3", "zhead": "^2.2.4" } }, "sha512-0zWykKAaJdm+/Y7yi/Yds20PrUK7XabLe9c3IRcjnwYmSWY6z0Cr19VIs3ozCj8P+GhR+/TI2mwtGlueCEYouA=="], "blurhash": ["blurhash@2.0.5", "", {}, "sha512-cRygWd7kGBQO3VEhPiTgq4Wc43ctsM+o46urrmPOiuAe+07fzlSB9OJVdpgDL0jPqXUVQ9ht7aq7kxOeJHRK+w=="], "buffer-from": ["buffer-from@1.1.2", "", {}, "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ=="], - "bun-types": ["bun-types@1.2.4", "", { "dependencies": { "@types/node": "*", "@types/ws": "~8.5.10" } }, "sha512-nDPymR207ZZEoWD4AavvEaa/KZe/qlrbMSchqpQwovPZCKc7pwMoENjEtHgMKaAjJhy+x6vfqSBA1QU3bJgs0Q=="], + "bun-types": ["bun-types@1.2.5", "", { "dependencies": { "@types/node": "*", "@types/ws": "~8.5.10" } }, "sha512-3oO6LVGGRRKI4kHINx5PIdIgnLRb7l/SprhzqXapmoYkFl5m4j6EvALvbDVuuBFaamB46Ap6HCUxIXNLCGy+tg=="], "char-info": ["char-info@0.3.5", "", { "dependencies": { "node-interval-tree": "^1.3.3" } }, "sha512-gRslEBFEcuLMGLNO1EFIrdN1MMUfO+aqa7y8iWzNyAzB3mYKnTIvP+ioW3jpyeEvqA5WapVLIPINGtFjEIH4cQ=="], @@ -163,7 +165,7 @@ "drizzle-orm": ["drizzle-orm@0.39.0", "", { "peerDependencies": { "@aws-sdk/client-rds-data": ">=3", "@cloudflare/workers-types": ">=4", "@electric-sql/pglite": ">=0.2.0", "@libsql/client": ">=0.10.0", "@libsql/client-wasm": ">=0.10.0", "@neondatabase/serverless": ">=0.10.0", "@op-engineering/op-sqlite": ">=2", "@opentelemetry/api": "^1.4.1", "@planetscale/database": ">=1", "@prisma/client": "*", "@tidbcloud/serverless": "*", "@types/better-sqlite3": "*", "@types/pg": "*", "@types/react": ">=18", "@types/sql.js": "*", "@vercel/postgres": ">=0.8.0", "@xata.io/client": "*", "better-sqlite3": ">=7", "bun-types": "*", "expo-sqlite": ">=14.0.0", "knex": "*", "kysely": "*", "mysql2": ">=2", "pg": ">=8", "postgres": ">=3", "react": ">=18", "sql.js": ">=1", "sqlite3": ">=5" }, "optionalPeers": ["@aws-sdk/client-rds-data", "@cloudflare/workers-types", "@electric-sql/pglite", "@libsql/client", "@libsql/client-wasm", "@neondatabase/serverless", "@op-engineering/op-sqlite", "@opentelemetry/api", "@planetscale/database", "@prisma/client", "@tidbcloud/serverless", "@types/better-sqlite3", "@types/pg", "@types/react", "@types/sql.js", "@vercel/postgres", "@xata.io/client", "better-sqlite3", "bun-types", "expo-sqlite", "knex", "kysely", "mysql2", "pg", "postgres", "react", "sql.js", "sqlite3"] }, "sha512-kkZwo3Jvht0fdJD/EWGx0vYcEK0xnGrlNVaY07QYluRZA9N21B9VFbY+54bnb/1xvyzcg97tE65xprSAP/fFGQ=="], - "elysia": ["elysia@1.2.23", "", { "dependencies": { "@sinclair/typebox": "^0.34.27", "cookie": "^1.0.2", "memoirist": "^0.3.0", "openapi-types": "^12.1.3" }, "peerDependencies": { "typescript": ">= 5.0.0" }, "optionalPeers": ["typescript"] }, "sha512-hMezhUkbpzZQduu01tmODsNJXk5CJ8oAQvc5gN1+GLv8cjiOFOnMQdEpTtaMplTKU0lr7MBtwL2duVFXEBPKOg=="], + "elysia": ["elysia@1.2.25", "", { "dependencies": { "@sinclair/typebox": "^0.34.27", "cookie": "^1.0.2", "memoirist": "^0.3.0", "openapi-types": "^12.1.3" }, "peerDependencies": { "typescript": ">= 5.0.0" }, "optionalPeers": ["typescript"] }, "sha512-WsdQpORJvb4uszzeqYT0lg97knw1iBW1NTzJ1Jm57tiHg+DfAotlWXYbjmvQ039ssV0fYELDHinLLoUazZkEHg=="], "env-paths": ["env-paths@3.0.0", "", {}, "sha512-dtJUTepzMW3Lm/NPxRf3wP4642UWhjL2sQxc+ym2YMj1m/H2zDNQOlezafzkHwn6sMstjHTwG6iQQsctDW/b1A=="], @@ -171,9 +173,9 @@ "esbuild-register": ["esbuild-register@3.6.0", "", { "dependencies": { "debug": "^4.3.4" }, "peerDependencies": { "esbuild": ">=0.12 <1" } }, "sha512-H2/S7Pm8a9CL1uhp9OvjwrBh5Pvx0H8qVOxNu8Wed9Y7qv56MPtq+GGM8RJpq6glYJn9Wspr8uw7l55uyinNeg=="], - "gel": ["gel@2.0.0", "", { "dependencies": { "@petamoriken/float16": "^3.8.7", "debug": "^4.3.4", "env-paths": "^3.0.0", "semver": "^7.6.2", "shell-quote": "^1.8.1", "which": "^4.0.0" }, "bin": { "gel": "dist/cli.mjs" } }, "sha512-Oq3Fjay71s00xzDc0BF/mpcLmnA+uRqMEJK8p5K4PaZjUEsxaeo+kR9OHBVAf289/qPd+0OcLOLUN0UhqiUCog=="], + "gel": ["gel@2.0.1", "", { "dependencies": { "@petamoriken/float16": "^3.8.7", "debug": "^4.3.4", "env-paths": "^3.0.0", "semver": "^7.6.2", "shell-quote": "^1.8.1", "which": "^4.0.0" }, "bin": { "gel": "dist/cli.mjs" } }, "sha512-gfem3IGvqKqXwEq7XseBogyaRwGsQGuE7Cw/yQsjLGdgiyqX92G1xENPCE0ltunPGcsJIa6XBOTx/PK169mOqw=="], - "get-tsconfig": ["get-tsconfig@4.8.1", "", { "dependencies": { "resolve-pkg-maps": "^1.0.0" } }, "sha512-k9PN+cFBmaLWtVz29SkUoqU5O0slLuHJXt/2P+tMVFT+phsSGXGkp9t3rQIqdz0e+06EHNGs3oM6ZX1s2zHxRg=="], + "get-tsconfig": ["get-tsconfig@4.10.0", "", { "dependencies": { "resolve-pkg-maps": "^1.0.0" } }, "sha512-kGzZ3LWWQcGIAmg6iWvXn0ei6WDtV26wzHRMwDSzmAbcXrTEXxHy6IehI6/4eT6VRKyMP1eF1VqwrVUmE/LR7A=="], "hookable": ["hookable@5.5.3", "", {}, "sha512-Yc+BQe8SvoXH1643Qez1zqLRmbA5rCL+sSmk6TVos0LWVfNIB7PGncdlId77WzLGSIB5KaWgTaNTs2lNVEI6VQ=="], @@ -187,6 +189,8 @@ "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + "node-addon-api": ["node-addon-api@8.3.1", "", {}, "sha512-lytcDEdxKjGJPTLEfW4mYMigRezMlyJY8W4wxJK8zE533Jlb8L8dRuObJFWg2P+AuOIxoCgKF+2Oq4d4Zd0OUA=="], + "node-interval-tree": ["node-interval-tree@1.3.3", "", { "dependencies": { "shallowequal": "^1.0.2" } }, "sha512-K9vk96HdTK5fEipJwxSvIIqwTqr4e3HRJeJrNxBSeVMNSC/JWARRaX7etOLOuTmrRMeOI/K5TCJu3aWIwZiNTw=="], "obuf": ["obuf@1.1.2", "", {}, "sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg=="], @@ -197,7 +201,7 @@ "pathe": ["pathe@1.1.2", "", {}, "sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ=="], - "pg": ["pg@8.13.3", "", { "dependencies": { "pg-connection-string": "^2.7.0", "pg-pool": "^3.7.1", "pg-protocol": "^1.7.1", "pg-types": "^2.1.0", "pgpass": "1.x" }, "optionalDependencies": { "pg-cloudflare": "^1.1.1" }, "peerDependencies": { "pg-native": ">=3.0.1" }, "optionalPeers": ["pg-native"] }, "sha512-P6tPt9jXbL9HVu/SSRERNYaYG++MjnscnegFh9pPHihfoBSujsrka0hyuymMzeJKFWrcG8wvCKy8rCe8e5nDUQ=="], + "pg": ["pg@8.14.0", "", { "dependencies": { "pg-connection-string": "^2.7.0", "pg-pool": "^3.8.0", "pg-protocol": "^1.8.0", "pg-types": "^2.1.0", "pgpass": "1.x" }, "optionalDependencies": { "pg-cloudflare": "^1.1.1" }, "peerDependencies": { "pg-native": ">=3.0.1" }, "optionalPeers": ["pg-native"] }, "sha512-nXbVpyoaXVmdqlKEzToFf37qzyeeh7mbiXsnoWvstSqohj88yaa/I/Rq/HEVn2QPSZEuLIJa/jSpRDyzjEx4FQ=="], "pg-cloudflare": ["pg-cloudflare@1.1.1", "", {}, "sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q=="], @@ -207,15 +211,15 @@ "pg-numeric": ["pg-numeric@1.0.2", "", {}, "sha512-BM/Thnrw5jm2kKLE5uJkXqqExRUY/toLHda65XgFTBTFYZyopbKjBe29Ii3RbkvlsMoFwD+tHeGaCjjv0gHlyw=="], - "pg-pool": ["pg-pool@3.7.1", "", { "peerDependencies": { "pg": ">=8.0" } }, "sha512-xIOsFoh7Vdhojas6q3596mXFsR8nwBQBXX5JiV7p9buEVAGqYL4yFzclON5P9vFrpu1u7Zwl2oriyDa89n0wbw=="], + "pg-pool": ["pg-pool@3.8.0", "", { "peerDependencies": { "pg": ">=8.0" } }, "sha512-VBw3jiVm6ZOdLBTIcXLNdSotb6Iy3uOCwDGFAksZCXmi10nyRvnP2v3jl4d+IsLYRyXf6o9hIm/ZtUzlByNUdw=="], - "pg-protocol": ["pg-protocol@1.7.0", "", {}, "sha512-hTK/mE36i8fDDhgDFjy6xNOG+LCorxLG3WO17tku+ij6sVHXh1jQUJ8hYAnRhNla4QVD2H8er/FOjc/+EgC6yQ=="], + "pg-protocol": ["pg-protocol@1.8.0", "", {}, "sha512-jvuYlEkL03NRvOoyoRktBK7+qU5kOvlAwvmrH8sr3wbLrOdVWsRxQfz8mMy9sZFsqJ1hEWNfdWKI4SAmoL+j7g=="], "pg-types": ["pg-types@4.0.2", "", { "dependencies": { "pg-int8": "1.0.1", "pg-numeric": "1.0.2", "postgres-array": "~3.0.1", "postgres-bytea": "~3.0.0", "postgres-date": "~2.1.0", "postgres-interval": "^3.0.0", "postgres-range": "^1.1.1" } }, "sha512-cRL3JpS3lKMGsKaWndugWQoLOCoP+Cic8oseVcbr0qhPzYD5DWXK+RZ9LY9wxRf7RQia4SCwQlXk0q6FCPrVng=="], "pgpass": ["pgpass@1.0.5", "", { "dependencies": { "split2": "^4.1.0" } }, "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug=="], - "postgres-array": ["postgres-array@3.0.2", "", {}, "sha512-6faShkdFugNQCLwucjPcY5ARoW1SlbnrZjmGl0IrrqewpvxvhSLHimCVzqeuULCbG0fQv7Dtk1yDbG3xv7Veog=="], + "postgres-array": ["postgres-array@3.0.4", "", {}, "sha512-nAUSGfSDGOaOAEGwqsRY27GPOea7CNipJPOA7lPbdEpx5Kg3qzdP0AaWC5MlhTWV9s4hFX39nomVZ+C4tnGOJQ=="], "postgres-bytea": ["postgres-bytea@3.0.0", "", { "dependencies": { "obuf": "~1.1.2" } }, "sha512-CNd4jim9RFPkObHSjVHlVrxoVQXz7quwNFpz7RY1okNNme49+sVyiTvTRobiLV548Hx/hb1BG+iE7h9493WzFw=="], @@ -253,11 +257,11 @@ "zhead": ["zhead@2.2.4", "", {}, "sha512-8F0OI5dpWIA5IGG5NHUg9staDwz/ZPxZtvGVf01j7vHqSyZ0raHY+78atOVxRqb73AotX22uV1pXt3gYSstGag=="], + "zod": ["zod@3.24.2", "", {}, "sha512-lY7CDW43ECgW9u1TcT3IoXHflywfVqDYze4waEz812jR/bZ8FHDsl7pFQoSZTz5N+2NqRXs8GBwnAwo3ZNxqhQ=="], + "@esbuild-kit/core-utils/esbuild": ["esbuild@0.18.20", "", { "optionalDependencies": { "@esbuild/android-arm": "0.18.20", "@esbuild/android-arm64": "0.18.20", "@esbuild/android-x64": "0.18.20", "@esbuild/darwin-arm64": "0.18.20", "@esbuild/darwin-x64": "0.18.20", "@esbuild/freebsd-arm64": "0.18.20", "@esbuild/freebsd-x64": "0.18.20", "@esbuild/linux-arm": "0.18.20", "@esbuild/linux-arm64": "0.18.20", "@esbuild/linux-ia32": "0.18.20", "@esbuild/linux-loong64": "0.18.20", "@esbuild/linux-mips64el": "0.18.20", "@esbuild/linux-ppc64": "0.18.20", "@esbuild/linux-riscv64": "0.18.20", "@esbuild/linux-s390x": "0.18.20", "@esbuild/linux-x64": "0.18.20", "@esbuild/netbsd-x64": "0.18.20", "@esbuild/openbsd-x64": "0.18.20", "@esbuild/sunos-x64": "0.18.20", "@esbuild/win32-arm64": "0.18.20", "@esbuild/win32-ia32": "0.18.20", "@esbuild/win32-x64": "0.18.20" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-ceqxoedUrcayh7Y7ZX6NdbbDzGROiyVBgC4PriJThBKSVPWnnFHZAkfI1lJT8QFkOwH4qOS2SJkS4wvpGl8BpA=="], - "@scalar/themes/@scalar/types": ["@scalar/types@0.0.25", "", { "dependencies": { "@scalar/openapi-types": "0.1.5", "@unhead/schema": "^1.11.11" } }, "sha512-sGnOFnfiSn4o23rklU/jrg81hO+630bsFIdHg8MZ/w2Nc6IoUwARA2hbe4d4Fg+D0KBu40Tan/L+WAYDXkTJQg=="], - - "pg/pg-protocol": ["pg-protocol@1.7.1", "", {}, "sha512-gjTHWGYWsEgy9MsY0Gp6ZJxV24IjDqdpTW7Eh0x+WfJLFsm/TJx1MzL6T0D88mBvkpxotCQ6TwW6N+Kko7lhgQ=="], + "@scalar/themes/@scalar/types": ["@scalar/types@0.1.1", "", { "dependencies": { "@scalar/openapi-types": "0.1.9", "@unhead/schema": "^1.11.11", "zod": "^3.23.8" } }, "sha512-LlUX6AmOOGoRqOMoO835V2FezM1KiO5UlvQC3poT/s7oqD6ranqwRNFxyrPz/IxClPYR+SV1yBUSNKely4ZQhQ=="], "pg/pg-types": ["pg-types@2.2.0", "", { "dependencies": { "pg-int8": "1.0.1", "postgres-array": "~2.0.0", "postgres-bytea": "~1.0.0", "postgres-date": "~1.0.4", "postgres-interval": "^1.1.0" } }, "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA=="], @@ -305,7 +309,7 @@ "@esbuild-kit/core-utils/esbuild/@esbuild/win32-x64": ["@esbuild/win32-x64@0.18.20", "", { "os": "win32", "cpu": "x64" }, "sha512-kTdfRcSiDfQca/y9QIkng02avJ+NCaQvrMejlsB3RRv5sE9rRoeBPISaZpKxHELzRxZyLvNts1P27W3wV+8geQ=="], - "@scalar/themes/@scalar/types/@scalar/openapi-types": ["@scalar/openapi-types@0.1.5", "", {}, "sha512-6geH9ehvQ/sG/xUyy3e0lyOw3BaY5s6nn22wHjEJhcobdmWyFER0O6m7AU0ZN4QTjle/gYvFJOjj552l/rsNSw=="], + "@scalar/themes/@scalar/types/@scalar/openapi-types": ["@scalar/openapi-types@0.1.9", "", {}, "sha512-HQQudOSQBU7ewzfnBW9LhDmBE2XOJgSfwrh5PlUB7zJup/kaRkBGNgV2wMjNz9Af/uztiU/xNrO179FysmUT+g=="], "pg/pg-types/postgres-array": ["postgres-array@2.0.0", "", {}, "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA=="], diff --git a/api/package.json b/api/package.json index ea814f69..a0fea57d 100644 --- a/api/package.json +++ b/api/package.json @@ -21,6 +21,7 @@ }, "devDependencies": { "@types/pg": "^8.11.11", + "node-addon-api": "^8.3.1", "bun-types": "^1.2.4" }, "module": "src/index.js", diff --git a/api/src/controllers/seed/images.ts b/api/src/controllers/seed/images.ts index efaed7c2..aa72a337 100644 --- a/api/src/controllers/seed/images.ts +++ b/api/src/controllers/seed/images.ts @@ -1,7 +1,7 @@ import { mkdir, writeFile } from "node:fs/promises"; import path from "node:path"; import { encode } from "blurhash"; -import { type SQLWrapper, eq, sql } from "drizzle-orm"; +import { SQL, type SQLWrapper, eq, getTableName, sql } from "drizzle-orm"; import type { PgColumn, PgTable } from "drizzle-orm/pg-core"; import { version } from "package.json"; import type { PoolClient } from "pg"; @@ -10,7 +10,7 @@ import { type Transaction, db } from "~/db"; import { mqueue } from "~/db/schema/queue"; import type { Image } from "~/models/utils"; -export const imageDir = process.env.IMAGES_PATH ?? "/images"; +export const imageDir = process.env.IMAGES_PATH ?? "./images"; await mkdir(imageDir, { recursive: true }); type ImageTask = { @@ -27,7 +27,7 @@ export const enqueueOptImage = async ( tx: Transaction, img: | { url: string | null; column: PgColumn } - | { url: string | null; table: PgTable; column: SQLWrapper }, + | { url: string | null; table: PgTable; column: SQL }, ): Promise => { if (!img.url) return null; @@ -40,14 +40,14 @@ export const enqueueOptImage = async ( ? { id, url: img.url, - table: img.table._.name, - column: img.column.getSQL().sql, + table: getTableName(img.table), + column: db.execute(img.column).getQuery().sql, } : { id, url: img.url, - table: img.column.table._.name, - column: img.column, + table: getTableName(img.column.table), + column: img.column.name, }; await tx.insert(mqueue).values({ kind: "image", diff --git a/api/src/controllers/seed/insert/staff.ts b/api/src/controllers/seed/insert/staff.ts index fe89c880..50b15467 100644 --- a/api/src/controllers/seed/insert/staff.ts +++ b/api/src/controllers/seed/insert/staff.ts @@ -1,4 +1,4 @@ -import { eq } from "drizzle-orm"; +import { eq, sql } from "drizzle-orm"; import { db } from "~/db"; import { roles, staff } from "~/db/schema"; import { conflictUpdateAllExcept } from "~/db/utils"; @@ -41,7 +41,7 @@ export const insertStaff = async ( image: await enqueueOptImage(tx, { url: x.character.image, table: roles, - column: `${roles.character}['image']`, + column: sql`${roles.character}['image']`, }), }, })), diff --git a/api/src/controllers/seed/movies.ts b/api/src/controllers/seed/movies.ts index 158c94bb..1cb7fc3c 100644 --- a/api/src/controllers/seed/movies.ts +++ b/api/src/controllers/seed/movies.ts @@ -3,7 +3,7 @@ import type { SeedMovie } from "~/models/movie"; import { getYear } from "~/utils"; import { insertCollection } from "./insert/collection"; import { insertEntries } from "./insert/entries"; -import { insertShow, updateAvailableCount } from "./insert/shows"; +import { insertShow } from "./insert/shows"; import { insertStaff } from "./insert/staff"; import { insertStudios } from "./insert/studios"; import { guessNextRefresh } from "./refresh"; diff --git a/api/tsconfig.json b/api/tsconfig.json index e271deb5..12def0d2 100644 --- a/api/tsconfig.json +++ b/api/tsconfig.json @@ -3,9 +3,7 @@ "target": "ES2021", "module": "ES2022", "moduleResolution": "node", - "types": [ - "bun-types" - ], + "types": ["bun-types"], "esModuleInterop": true, "forceConsistentCasingInFileNames": true, "strict": true, @@ -14,12 +12,8 @@ "resolveJsonModule": true, "baseUrl": ".", "paths": { - "~/*": [ - "./src/*" - ], - "package.json": [ - "package.json" - ] + "~/*": ["./src/*"], + "package.json": ["package.json"] } } } diff --git a/shell.nix b/shell.nix index 719afedd..84eefc02 100644 --- a/shell.nix +++ b/shell.nix @@ -42,7 +42,12 @@ in go-swag robotframework-tidy bun + pkg-config + node-gyp + vips ]; DOTNET_ROOT = "${dotnet}"; + + SHARP_FORCE_GLOBAL_LIBVIPS = 1; } From 0a729ccf75c34d4b10a851bd4a566c1afd72e320 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Mon, 17 Mar 2025 11:38:54 +0100 Subject: [PATCH 8/9] Migrate mqueue --- api/drizzle/0016_mqueue.sql | 9 + api/drizzle/meta/0016_snapshot.json | 1555 +++++++++++++++++++++++++++ api/drizzle/meta/_journal.json | 7 + 3 files changed, 1571 insertions(+) create mode 100644 api/drizzle/0016_mqueue.sql create mode 100644 api/drizzle/meta/0016_snapshot.json diff --git a/api/drizzle/0016_mqueue.sql b/api/drizzle/0016_mqueue.sql new file mode 100644 index 00000000..edd6871e --- /dev/null +++ b/api/drizzle/0016_mqueue.sql @@ -0,0 +1,9 @@ +CREATE TABLE "kyoo"."mqueue" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "kind" varchar(255) NOT NULL, + "message" jsonb NOT NULL, + "attempt" integer DEFAULT 0 NOT NULL, + "created_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE INDEX "mqueue_created" ON "kyoo"."mqueue" USING btree ("created_at"); \ No newline at end of file diff --git a/api/drizzle/meta/0016_snapshot.json b/api/drizzle/meta/0016_snapshot.json new file mode 100644 index 00000000..90790bf3 --- /dev/null +++ b/api/drizzle/meta/0016_snapshot.json @@ -0,0 +1,1555 @@ +{ + "id": "c3bd85b9-5370-4689-9a3e-78e5b5488a4a", + "prevId": "3a1cef8c-858f-4a6b-8c78-b91acb421d94", + "version": "7", + "dialect": "postgresql", + "tables": { + "kyoo.entries": { + "name": "entries", + "schema": "kyoo", + "columns": { + "pk": { + "name": "pk", + "type": "integer", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "entries_pk_seq", + "schema": "kyoo", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "2147483647", + "cache": "1", + "cycle": false + } + }, + "id": { + "name": "id", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "slug": { + "name": "slug", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "show_pk": { + "name": "show_pk", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "order": { + "name": "order", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "season_number": { + "name": "season_number", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "episode_number": { + "name": "episode_number", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "kind": { + "name": "kind", + "type": "entry_type", + "typeSchema": "kyoo", + "primaryKey": false, + "notNull": true + }, + "extra_kind": { + "name": "extra_kind", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "air_date": { + "name": "air_date", + "type": "date", + "primaryKey": false, + "notNull": false + }, + "runtime": { + "name": "runtime", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "thumbnail": { + "name": "thumbnail", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "external_id": { + "name": "external_id", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "available_since": { + "name": "available_since", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "next_refresh": { + "name": "next_refresh", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "entry_kind": { + "name": "entry_kind", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hash", + "with": {} + }, + "entry_order": { + "name": "entry_order", + "columns": [ + { + "expression": "order", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "entries_show_pk_shows_pk_fk": { + "name": "entries_show_pk_shows_pk_fk", + "tableFrom": "entries", + "tableTo": "shows", + "schemaTo": "kyoo", + "columnsFrom": ["show_pk"], + "columnsTo": ["pk"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "entries_id_unique": { + "name": "entries_id_unique", + "nullsNotDistinct": false, + "columns": ["id"] + }, + "entries_slug_unique": { + "name": "entries_slug_unique", + "nullsNotDistinct": false, + "columns": ["slug"] + }, + "entries_showPk_seasonNumber_episodeNumber_unique": { + "name": "entries_showPk_seasonNumber_episodeNumber_unique", + "nullsNotDistinct": false, + "columns": ["show_pk", "season_number", "episode_number"] + } + }, + "policies": {}, + "checkConstraints": { + "order_positive": { + "name": "order_positive", + "value": "\"kyoo\".\"entries\".\"order\" >= 0" + } + }, + "isRLSEnabled": false + }, + "kyoo.entry_translations": { + "name": "entry_translations", + "schema": "kyoo", + "columns": { + "pk": { + "name": "pk", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "language": { + "name": "language", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tagline": { + "name": "tagline", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "poster": { + "name": "poster", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "entry_name_trgm": { + "name": "entry_name_trgm", + "columns": [ + { + "expression": "\"name\" gin_trgm_ops", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": { + "entry_translations_pk_entries_pk_fk": { + "name": "entry_translations_pk_entries_pk_fk", + "tableFrom": "entry_translations", + "tableTo": "entries", + "schemaTo": "kyoo", + "columnsFrom": ["pk"], + "columnsTo": ["pk"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "entry_translations_pk_language_pk": { + "name": "entry_translations_pk_language_pk", + "columns": ["pk", "language"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "kyoo.season_translations": { + "name": "season_translations", + "schema": "kyoo", + "columns": { + "pk": { + "name": "pk", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "language": { + "name": "language", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "poster": { + "name": "poster", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "thumbnail": { + "name": "thumbnail", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "banner": { + "name": "banner", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "season_name_trgm": { + "name": "season_name_trgm", + "columns": [ + { + "expression": "\"name\" gin_trgm_ops", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": { + "season_translations_pk_seasons_pk_fk": { + "name": "season_translations_pk_seasons_pk_fk", + "tableFrom": "season_translations", + "tableTo": "seasons", + "schemaTo": "kyoo", + "columnsFrom": ["pk"], + "columnsTo": ["pk"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "season_translations_pk_language_pk": { + "name": "season_translations_pk_language_pk", + "columns": ["pk", "language"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "kyoo.seasons": { + "name": "seasons", + "schema": "kyoo", + "columns": { + "pk": { + "name": "pk", + "type": "integer", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "seasons_pk_seq", + "schema": "kyoo", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "2147483647", + "cache": "1", + "cycle": false + } + }, + "id": { + "name": "id", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "slug": { + "name": "slug", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "show_pk": { + "name": "show_pk", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "season_number": { + "name": "season_number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "start_air": { + "name": "start_air", + "type": "date", + "primaryKey": false, + "notNull": false + }, + "end_air": { + "name": "end_air", + "type": "date", + "primaryKey": false, + "notNull": false + }, + "external_id": { + "name": "external_id", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "next_refresh": { + "name": "next_refresh", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "show_fk": { + "name": "show_fk", + "columns": [ + { + "expression": "show_pk", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hash", + "with": {} + }, + "season_nbr": { + "name": "season_nbr", + "columns": [ + { + "expression": "season_number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "seasons_show_pk_shows_pk_fk": { + "name": "seasons_show_pk_shows_pk_fk", + "tableFrom": "seasons", + "tableTo": "shows", + "schemaTo": "kyoo", + "columnsFrom": ["show_pk"], + "columnsTo": ["pk"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "seasons_id_unique": { + "name": "seasons_id_unique", + "nullsNotDistinct": false, + "columns": ["id"] + }, + "seasons_slug_unique": { + "name": "seasons_slug_unique", + "nullsNotDistinct": false, + "columns": ["slug"] + }, + "seasons_showPk_seasonNumber_unique": { + "name": "seasons_showPk_seasonNumber_unique", + "nullsNotDistinct": false, + "columns": ["show_pk", "season_number"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "kyoo.show_translations": { + "name": "show_translations", + "schema": "kyoo", + "columns": { + "pk": { + "name": "pk", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "language": { + "name": "language", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tagline": { + "name": "tagline", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "aliases": { + "name": "aliases", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "tags": { + "name": "tags", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "poster": { + "name": "poster", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "thumbnail": { + "name": "thumbnail", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "banner": { + "name": "banner", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "logo": { + "name": "logo", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "trailer_url": { + "name": "trailer_url", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "name_trgm": { + "name": "name_trgm", + "columns": [ + { + "expression": "\"name\" gin_trgm_ops", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + }, + "tags": { + "name": "tags", + "columns": [ + { + "expression": "tags", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "show_translations_pk_shows_pk_fk": { + "name": "show_translations_pk_shows_pk_fk", + "tableFrom": "show_translations", + "tableTo": "shows", + "schemaTo": "kyoo", + "columnsFrom": ["pk"], + "columnsTo": ["pk"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "show_translations_pk_language_pk": { + "name": "show_translations_pk_language_pk", + "columns": ["pk", "language"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "kyoo.shows": { + "name": "shows", + "schema": "kyoo", + "columns": { + "pk": { + "name": "pk", + "type": "integer", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "shows_pk_seq", + "schema": "kyoo", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "2147483647", + "cache": "1", + "cycle": false + } + }, + "id": { + "name": "id", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "slug": { + "name": "slug", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "show_kind", + "typeSchema": "kyoo", + "primaryKey": false, + "notNull": true + }, + "genres": { + "name": "genres", + "type": "genres[]", + "primaryKey": false, + "notNull": true + }, + "rating": { + "name": "rating", + "type": "smallint", + "primaryKey": false, + "notNull": false + }, + "runtime": { + "name": "runtime", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "show_status", + "typeSchema": "kyoo", + "primaryKey": false, + "notNull": true + }, + "start_air": { + "name": "start_air", + "type": "date", + "primaryKey": false, + "notNull": false + }, + "end_air": { + "name": "end_air", + "type": "date", + "primaryKey": false, + "notNull": false + }, + "original": { + "name": "original", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "collection_pk": { + "name": "collection_pk", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "entries_count": { + "name": "entries_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "available_count": { + "name": "available_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "external_id": { + "name": "external_id", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "next_refresh": { + "name": "next_refresh", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "kind": { + "name": "kind", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hash", + "with": {} + }, + "rating": { + "name": "rating", + "columns": [ + { + "expression": "rating", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "startAir": { + "name": "startAir", + "columns": [ + { + "expression": "start_air", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "shows_collection_pk_shows_pk_fk": { + "name": "shows_collection_pk_shows_pk_fk", + "tableFrom": "shows", + "tableTo": "shows", + "schemaTo": "kyoo", + "columnsFrom": ["collection_pk"], + "columnsTo": ["pk"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "shows_id_unique": { + "name": "shows_id_unique", + "nullsNotDistinct": false, + "columns": ["id"] + }, + "shows_slug_unique": { + "name": "shows_slug_unique", + "nullsNotDistinct": false, + "columns": ["slug"] + } + }, + "policies": {}, + "checkConstraints": { + "rating_valid": { + "name": "rating_valid", + "value": "\"kyoo\".\"shows\".\"rating\" between 0 and 100" + }, + "runtime_valid": { + "name": "runtime_valid", + "value": "\"kyoo\".\"shows\".\"runtime\" >= 0" + } + }, + "isRLSEnabled": false + }, + "kyoo.show_studio_join": { + "name": "show_studio_join", + "schema": "kyoo", + "columns": { + "show_pk": { + "name": "show_pk", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "studio_pk": { + "name": "studio_pk", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "show_studio_join_show_pk_shows_pk_fk": { + "name": "show_studio_join_show_pk_shows_pk_fk", + "tableFrom": "show_studio_join", + "tableTo": "shows", + "schemaTo": "kyoo", + "columnsFrom": ["show_pk"], + "columnsTo": ["pk"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "show_studio_join_studio_pk_studios_pk_fk": { + "name": "show_studio_join_studio_pk_studios_pk_fk", + "tableFrom": "show_studio_join", + "tableTo": "studios", + "schemaTo": "kyoo", + "columnsFrom": ["studio_pk"], + "columnsTo": ["pk"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "show_studio_join_show_pk_studio_pk_pk": { + "name": "show_studio_join_show_pk_studio_pk_pk", + "columns": ["show_pk", "studio_pk"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "kyoo.studio_translations": { + "name": "studio_translations", + "schema": "kyoo", + "columns": { + "pk": { + "name": "pk", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "language": { + "name": "language", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "logo": { + "name": "logo", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "studio_name_trgm": { + "name": "studio_name_trgm", + "columns": [ + { + "expression": "\"name\" gin_trgm_ops", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": { + "studio_translations_pk_studios_pk_fk": { + "name": "studio_translations_pk_studios_pk_fk", + "tableFrom": "studio_translations", + "tableTo": "studios", + "schemaTo": "kyoo", + "columnsFrom": ["pk"], + "columnsTo": ["pk"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "studio_translations_pk_language_pk": { + "name": "studio_translations_pk_language_pk", + "columns": ["pk", "language"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "kyoo.studios": { + "name": "studios", + "schema": "kyoo", + "columns": { + "pk": { + "name": "pk", + "type": "integer", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "studios_pk_seq", + "schema": "kyoo", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "2147483647", + "cache": "1", + "cycle": false + } + }, + "id": { + "name": "id", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "slug": { + "name": "slug", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "external_id": { + "name": "external_id", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "studios_id_unique": { + "name": "studios_id_unique", + "nullsNotDistinct": false, + "columns": ["id"] + }, + "studios_slug_unique": { + "name": "studios_slug_unique", + "nullsNotDistinct": false, + "columns": ["slug"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "kyoo.roles": { + "name": "roles", + "schema": "kyoo", + "columns": { + "pk": { + "name": "pk", + "type": "integer", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "roles_pk_seq", + "schema": "kyoo", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "2147483647", + "cache": "1", + "cycle": false + } + }, + "show_pk": { + "name": "show_pk", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "staff_pk": { + "name": "staff_pk", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "role_kind", + "typeSchema": "kyoo", + "primaryKey": false, + "notNull": true + }, + "order": { + "name": "order", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "character": { + "name": "character", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "role_kind": { + "name": "role_kind", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hash", + "with": {} + }, + "role_order": { + "name": "role_order", + "columns": [ + { + "expression": "order", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "roles_show_pk_shows_pk_fk": { + "name": "roles_show_pk_shows_pk_fk", + "tableFrom": "roles", + "tableTo": "shows", + "schemaTo": "kyoo", + "columnsFrom": ["show_pk"], + "columnsTo": ["pk"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "roles_staff_pk_staff_pk_fk": { + "name": "roles_staff_pk_staff_pk_fk", + "tableFrom": "roles", + "tableTo": "staff", + "schemaTo": "kyoo", + "columnsFrom": ["staff_pk"], + "columnsTo": ["pk"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "kyoo.staff": { + "name": "staff", + "schema": "kyoo", + "columns": { + "pk": { + "name": "pk", + "type": "integer", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "staff_pk_seq", + "schema": "kyoo", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "2147483647", + "cache": "1", + "cycle": false + } + }, + "id": { + "name": "id", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "slug": { + "name": "slug", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "latin_name": { + "name": "latin_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "image": { + "name": "image", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "external_id": { + "name": "external_id", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "staff_id_unique": { + "name": "staff_id_unique", + "nullsNotDistinct": false, + "columns": ["id"] + }, + "staff_slug_unique": { + "name": "staff_slug_unique", + "nullsNotDistinct": false, + "columns": ["slug"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "kyoo.entry_video_join": { + "name": "entry_video_join", + "schema": "kyoo", + "columns": { + "entry_pk": { + "name": "entry_pk", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "video_pk": { + "name": "video_pk", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "entry_video_join_entry_pk_entries_pk_fk": { + "name": "entry_video_join_entry_pk_entries_pk_fk", + "tableFrom": "entry_video_join", + "tableTo": "entries", + "schemaTo": "kyoo", + "columnsFrom": ["entry_pk"], + "columnsTo": ["pk"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "entry_video_join_video_pk_videos_pk_fk": { + "name": "entry_video_join_video_pk_videos_pk_fk", + "tableFrom": "entry_video_join", + "tableTo": "videos", + "schemaTo": "kyoo", + "columnsFrom": ["video_pk"], + "columnsTo": ["pk"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "entry_video_join_entry_pk_video_pk_pk": { + "name": "entry_video_join_entry_pk_video_pk_pk", + "columns": ["entry_pk", "video_pk"] + } + }, + "uniqueConstraints": { + "entry_video_join_slug_unique": { + "name": "entry_video_join_slug_unique", + "nullsNotDistinct": false, + "columns": ["slug"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "kyoo.videos": { + "name": "videos", + "schema": "kyoo", + "columns": { + "pk": { + "name": "pk", + "type": "integer", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "videos_pk_seq", + "schema": "kyoo", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "2147483647", + "cache": "1", + "cycle": false + } + }, + "id": { + "name": "id", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "rendering": { + "name": "rendering", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "part": { + "name": "part", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "guess": { + "name": "guess", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "videos_id_unique": { + "name": "videos_id_unique", + "nullsNotDistinct": false, + "columns": ["id"] + }, + "videos_path_unique": { + "name": "videos_path_unique", + "nullsNotDistinct": false, + "columns": ["path"] + } + }, + "policies": {}, + "checkConstraints": { + "part_pos": { + "name": "part_pos", + "value": "\"kyoo\".\"videos\".\"part\" >= 0" + }, + "version_pos": { + "name": "version_pos", + "value": "\"kyoo\".\"videos\".\"version\" >= 0" + } + }, + "isRLSEnabled": false + }, + "kyoo.mqueue": { + "name": "mqueue", + "schema": "kyoo", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "kind": { + "name": "kind", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "message": { + "name": "message", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "attempt": { + "name": "attempt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "mqueue_created": { + "name": "mqueue_created", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "kyoo.entry_type": { + "name": "entry_type", + "schema": "kyoo", + "values": ["unknown", "episode", "movie", "special", "extra"] + }, + "kyoo.genres": { + "name": "genres", + "schema": "kyoo", + "values": [ + "action", + "adventure", + "animation", + "comedy", + "crime", + "documentary", + "drama", + "family", + "fantasy", + "history", + "horror", + "music", + "mystery", + "romance", + "science-fiction", + "thriller", + "war", + "western", + "kids", + "reality", + "politics", + "soap", + "talk" + ] + }, + "kyoo.show_kind": { + "name": "show_kind", + "schema": "kyoo", + "values": ["serie", "movie", "collection"] + }, + "kyoo.show_status": { + "name": "show_status", + "schema": "kyoo", + "values": ["unknown", "finished", "airing", "planned"] + }, + "kyoo.role_kind": { + "name": "role_kind", + "schema": "kyoo", + "values": ["actor", "director", "writter", "producer", "music", "other"] + } + }, + "schemas": { + "kyoo": "kyoo" + }, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/api/drizzle/meta/_journal.json b/api/drizzle/meta/_journal.json index 26ad034f..d9f1e294 100644 --- a/api/drizzle/meta/_journal.json +++ b/api/drizzle/meta/_journal.json @@ -113,6 +113,13 @@ "when": 1741623934941, "tag": "0015_news", "breakpoints": true + }, + { + "idx": 16, + "version": "7", + "when": 1742205790510, + "tag": "0016_mqueue", + "breakpoints": true } ] } From 51558db1b2b53a739a45a9535a4799c88a859c88 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Tue, 18 Mar 2025 12:03:37 +0100 Subject: [PATCH 9/9] Add image downloading test --- api/.gitignore | 1 + api/src/controllers/seed/images.ts | 37 +++++++++++++++++------ api/src/controllers/seed/series.ts | 1 - api/src/db/schema/index.ts | 1 + api/src/db/schema/{queue.ts => mqueue.ts} | 0 api/src/index.ts | 4 +++ api/tests/manual.ts | 9 +++++- api/tests/misc/images.test.ts | 31 +++++++++++++++++++ 8 files changed, 73 insertions(+), 11 deletions(-) rename api/src/db/schema/{queue.ts => mqueue.ts} (100%) create mode 100644 api/tests/misc/images.test.ts diff --git a/api/.gitignore b/api/.gitignore index ef720b3f..e5ef0f40 100644 --- a/api/.gitignore +++ b/api/.gitignore @@ -1,2 +1,3 @@ node_modules **/*.bun +images diff --git a/api/src/controllers/seed/images.ts b/api/src/controllers/seed/images.ts index aa72a337..b80d744f 100644 --- a/api/src/controllers/seed/images.ts +++ b/api/src/controllers/seed/images.ts @@ -1,18 +1,20 @@ import { mkdir, writeFile } from "node:fs/promises"; import path from "node:path"; import { encode } from "blurhash"; -import { SQL, type SQLWrapper, eq, getTableName, sql } from "drizzle-orm"; -import type { PgColumn, PgTable } from "drizzle-orm/pg-core"; +import { type SQL, eq, is, sql } from "drizzle-orm"; +import { PgColumn, type PgTable } from "drizzle-orm/pg-core"; import { version } from "package.json"; import type { PoolClient } from "pg"; import sharp from "sharp"; import { type Transaction, db } from "~/db"; -import { mqueue } from "~/db/schema/queue"; +import { mqueue } from "~/db/schema/mqueue"; import type { Image } from "~/models/utils"; export const imageDir = process.env.IMAGES_PATH ?? "./images"; await mkdir(imageDir, { recursive: true }); +export const defaultBlurhash = "000000"; + type ImageTask = { id: string; url: string; @@ -35,19 +37,34 @@ export const enqueueOptImage = async ( hasher.update(img.url); const id = hasher.digest().toString("hex"); + const cleanupColumn = (column: SQL) => + // @ts-expect-error dialect is private + db.dialect.sqlToQuery( + sql.join( + column.queryChunks.map((x) => { + if (is(x, PgColumn)) { + return sql.identifier(x.name); + } + return x; + }), + ), + ).sql; + const message: ImageTask = "table" in img ? { id, url: img.url, - table: getTableName(img.table), - column: db.execute(img.column).getQuery().sql, + // @ts-expect-error dialect is private + table: db.dialect.sqlToQuery(sql`${img.table}`).sql, + column: cleanupColumn(img.column), } : { id, url: img.url, - table: getTableName(img.column.table), - column: img.column.name, + // @ts-expect-error dialect is private + table: db.dialect.sqlToQuery(sql`${img.column.table}`).sql, + column: sql.identifier(img.column.name).value, }; await tx.insert(mqueue).values({ kind: "image", @@ -58,7 +75,7 @@ export const enqueueOptImage = async ( return { id, source: img.url, - blurhash: "", + blurhash: defaultBlurhash, }; }; @@ -83,7 +100,7 @@ export const processImages = async () => { const column = sql.raw(img.column); await tx.execute(sql` - update ${table} set ${column} = ${ret} where ${column}->'id' = '${item.id}' + update ${table} set ${column} = ${ret} where ${column}->'id' = ${sql.raw(`'"${img.id}"'::jsonb`)} `); await tx.delete(mqueue).where(eq(mqueue.id, item.id)); @@ -112,6 +129,7 @@ export const processImages = async () => { // start processing old tasks await processAll(); + return () => client.release(true); }; async function downloadImage(id: string, url: string): Promise { @@ -144,6 +162,7 @@ async function downloadImage(id: string, url: string): Promise { const { data, info } = await image .resize(32, 32, { fit: "inside" }) + .ensureAlpha() .raw() .toBuffer({ resolveWithObject: true }); diff --git a/api/src/controllers/seed/series.ts b/api/src/controllers/seed/series.ts index 500aa57a..d8c8818b 100644 --- a/api/src/controllers/seed/series.ts +++ b/api/src/controllers/seed/series.ts @@ -1,7 +1,6 @@ import { t } from "elysia"; import type { SeedSerie } from "~/models/serie"; import { getYear } from "~/utils"; -import { enqueueOptImage } from "./images"; import { insertCollection } from "./insert/collection"; import { insertEntries } from "./insert/entries"; import { insertSeasons } from "./insert/seasons"; diff --git a/api/src/db/schema/index.ts b/api/src/db/schema/index.ts index 3488c4cc..67f4e990 100644 --- a/api/src/db/schema/index.ts +++ b/api/src/db/schema/index.ts @@ -4,3 +4,4 @@ export * from "./shows"; export * from "./studios"; export * from "./staff"; export * from "./videos"; +export * from "./mqueue"; diff --git a/api/src/db/schema/queue.ts b/api/src/db/schema/mqueue.ts similarity index 100% rename from api/src/db/schema/queue.ts rename to api/src/db/schema/mqueue.ts diff --git a/api/src/index.ts b/api/src/index.ts index 4c9a076e..9905c8a2 100644 --- a/api/src/index.ts +++ b/api/src/index.ts @@ -1,5 +1,6 @@ import jwt from "@elysiajs/jwt"; import { swagger } from "@elysiajs/swagger"; +import { processImages } from "./controllers/seed/images"; import { migrate } from "./db"; import { app } from "./elysia"; import { comment } from "./utils"; @@ -23,6 +24,9 @@ if (!secret) { process.exit(1); } +// run image processor task in background +processImages(); + app .use(jwt({ secret })) .use( diff --git a/api/tests/manual.ts b/api/tests/manual.ts index 3835aa2c..4d4cb906 100644 --- a/api/tests/manual.ts +++ b/api/tests/manual.ts @@ -1,5 +1,6 @@ +import { processImages } from "~/controllers/seed/images"; import { db, migrate } from "~/db"; -import { shows, videos } from "~/db/schema"; +import { mqueue, shows, videos } from "~/db/schema"; import { madeInAbyss, madeInAbyssVideo } from "~/models/examples"; import { createSerie, createVideo, getSerie } from "./helpers"; @@ -8,10 +9,16 @@ import { createSerie, createVideo, getSerie } from "./helpers"; await migrate(); await db.delete(shows); await db.delete(videos); +await db.delete(mqueue); const [_, vid] = await createVideo(madeInAbyssVideo); console.log(vid); const [__, ser] = await createSerie(madeInAbyss); console.log(ser); + +await processImages(); + const [___, got] = await getSerie(madeInAbyss.slug, { with: ["translations"] }); console.log(got); + +process.exit(0); diff --git a/api/tests/misc/images.test.ts b/api/tests/misc/images.test.ts new file mode 100644 index 00000000..c59a3a92 --- /dev/null +++ b/api/tests/misc/images.test.ts @@ -0,0 +1,31 @@ +import { beforeAll, describe, expect, it } from "bun:test"; +import { eq } from "drizzle-orm"; +import { defaultBlurhash, processImages } from "~/controllers/seed/images"; +import { db } from "~/db"; +import { mqueue, shows, staff, studios, videos } from "~/db/schema"; +import { madeInAbyss } from "~/models/examples"; +import { createSerie } from "../helpers"; + +beforeAll(async () => { + await db.delete(shows); + await db.delete(studios); + await db.delete(staff); + await db.delete(videos); + await db.delete(mqueue); + + await createSerie(madeInAbyss); + const release = await processImages(); + // remove notifications to prevent other images to be downloaded (do not curl 20000 images for nothing) + release(); +}); + +describe("images", () => { + it("Create a serie download images", async () => { + const ret = await db.query.shows.findFirst({ + where: eq(shows.slug, madeInAbyss.slug), + }); + expect(ret!.slug).toBe(madeInAbyss.slug); + expect(ret!.original.poster!.blurhash).toBeString(); + expect(ret!.original.poster!.blurhash).not.toBe(defaultBlurhash); + }); +});