diff --git a/api/package.json b/api/package.json index da151870..b91418d2 100644 --- a/api/package.json +++ b/api/package.json @@ -1,6 +1,6 @@ { "name": "api", - "version": "1.0.50", + "version": "5.0.0", "scripts": { "dev": "bun --watch src/index.ts", "build": "bun build src/index.ts --target bun --outdir ./dist", diff --git a/api/src/controllers/seed/images.ts b/api/src/controllers/seed/images.ts index 12f0d016..d3caba44 100644 --- a/api/src/controllers/seed/images.ts +++ b/api/src/controllers/seed/images.ts @@ -1,22 +1,74 @@ +import { eq, sql } from "drizzle-orm"; +import { version } from "package.json"; +import type { PoolClient } from "pg"; +import { db } from "~/db"; +import * as schema from "~/db/schema"; +import { mqueue } from "~/db/schema/queue"; import type { Image } from "~/models/utils"; +type ImageTask = { + id: string; + url: string; + table: string; + column: string; +}; + // this will only push a task to the image downloader service and not download it instantly. // this is both done to prevent to many requests to be sent at once and to make sure POST // requests are not blocked by image downloading or blurhash calculation -export const processImage = (url: string): Image => { +export const enqueueImage = async ( + tx: typeof db, + url: string, +): Promise => { const hasher = new Bun.CryptoHasher("sha256"); hasher.update(url); + const id = hasher.digest().toString("hex"); - // TODO: download source, save it in multiples qualities & process blurhash + await tx.insert(mqueue).values({ kind: "image", message: { id, url } }); return { - id: hasher.digest().toString("hex"), + id, source: url, blurhash: "", }; }; -export const processOptImage = (url: string | null): Image | null => { +export const enqueueOptImage = async ( + tx: typeof db, + url: string | null, +): Promise => { if (!url) return null; - return processImage(url); + return await enqueueImage(tx, url); +}; + +export const processImages = async () => { + await db.transaction(async (tx) => { + const [item] = await tx + .select() + .from(mqueue) + .for("update", { skipLocked: true }) + .where(eq(mqueue.kind, "image")) + .orderBy(mqueue.createdAt) + .limit(1); + + const img = item.message as ImageTask; + await fetch(img.url, { headers: { "User-Agent": `Kyoo v${version}` }, }); + const blurhash = ""; + + const table = schema[img.table as keyof typeof schema] as any; + + await tx + .update(table) + .set({ + [img.column]: { id: img.id, source: img.url, blurhash } satisfies Image, + }) + .where(eq(sql`${table[img.column]}->'id'`, img.id)); + + await tx.delete(mqueue).where(eq(mqueue.id, item.id)); + }); + + const client = (await db.$client.connect()) as PoolClient; + client.on("notification", (evt) => { + if (evt.channel !== "image") return; + }); }; diff --git a/api/src/controllers/seed/insert/collection.ts b/api/src/controllers/seed/insert/collection.ts index bcdcc589..495a3f33 100644 --- a/api/src/controllers/seed/insert/collection.ts +++ b/api/src/controllers/seed/insert/collection.ts @@ -5,7 +5,7 @@ import { conflictUpdateAllExcept } from "~/db/utils"; import type { SeedCollection } from "~/models/collections"; import type { SeedMovie } from "~/models/movie"; import type { SeedSerie } from "~/models/serie"; -import { processOptImage } from "../images"; +import { enqueueOptImage } from "../images"; type ShowTrans = typeof showTranslations.$inferInsert; @@ -53,10 +53,10 @@ export const insertCollection = async ( pk: ret.pk, language: lang, ...tr, - poster: processOptImage(tr.poster), - thumbnail: processOptImage(tr.thumbnail), - logo: processOptImage(tr.logo), - banner: processOptImage(tr.banner), + poster: enqueueOptImage(tr.poster), + thumbnail: enqueueOptImage(tr.thumbnail), + logo: enqueueOptImage(tr.logo), + banner: enqueueOptImage(tr.banner), }), ); await tx diff --git a/api/src/controllers/seed/insert/entries.ts b/api/src/controllers/seed/insert/entries.ts index 9c2dd804..d57a5b4a 100644 --- a/api/src/controllers/seed/insert/entries.ts +++ b/api/src/controllers/seed/insert/entries.ts @@ -8,7 +8,7 @@ import { } from "~/db/schema"; import { conflictUpdateAllExcept, sqlarr, values } from "~/db/utils"; import type { SeedEntry as SEntry, SeedExtra as SExtra } from "~/models/entry"; -import { processOptImage } from "../images"; +import { enqueueOptImage } from "../images"; import { guessNextRefresh } from "../refresh"; import { updateAvailableCount } from "./shows"; @@ -55,7 +55,7 @@ export const insertEntries = async ( ...entry, showPk: show.pk, slug: generateSlug(show.slug, seed), - thumbnail: processOptImage(seed.thumbnail), + thumbnail: enqueueOptImage(seed.thumbnail), nextRefresh: entry.kind !== "extra" ? guessNextRefresh(entry.airDate ?? new Date()) @@ -103,7 +103,7 @@ export const insertEntries = async ( ...tr, poster: seed.kind === "movie" - ? processOptImage((tr as any).poster) + ? enqueueOptImage((tr as any).poster) : undefined, })); }); diff --git a/api/src/controllers/seed/insert/seasons.ts b/api/src/controllers/seed/insert/seasons.ts index 56d649ae..bdb5ee39 100644 --- a/api/src/controllers/seed/insert/seasons.ts +++ b/api/src/controllers/seed/insert/seasons.ts @@ -2,7 +2,7 @@ import { db } from "~/db"; import { seasonTranslations, seasons } from "~/db/schema"; import { conflictUpdateAllExcept } from "~/db/utils"; import type { SeedSeason } from "~/models/season"; -import { processOptImage } from "../images"; +import { enqueueOptImage } from "../images"; import { guessNextRefresh } from "../refresh"; type SeasonI = typeof seasons.$inferInsert; @@ -43,9 +43,9 @@ export const insertSeasons = async ( pk: ret[i].pk, language: lang, ...tr, - poster: processOptImage(tr.poster), - thumbnail: processOptImage(tr.thumbnail), - banner: processOptImage(tr.banner), + poster: enqueueOptImage(tr.poster), + thumbnail: enqueueOptImage(tr.thumbnail), + banner: enqueueOptImage(tr.banner), })), ); await tx diff --git a/api/src/controllers/seed/insert/shows.ts b/api/src/controllers/seed/insert/shows.ts index e9d8c845..580738b3 100644 --- a/api/src/controllers/seed/insert/shows.ts +++ b/api/src/controllers/seed/insert/shows.ts @@ -6,7 +6,7 @@ import type { SeedCollection } from "~/models/collections"; import type { SeedMovie } from "~/models/movie"; import type { SeedSerie } from "~/models/serie"; import { getYear } from "~/utils"; -import { processOptImage } from "../images"; +import { enqueueOptImage } from "../images"; type Show = typeof shows.$inferInsert; type ShowTrans = typeof showTranslations.$inferInsert; @@ -27,10 +27,10 @@ export const insertShow = async ( pk: ret.pk, language: lang, ...tr, - poster: processOptImage(tr.poster), - thumbnail: processOptImage(tr.thumbnail), - logo: processOptImage(tr.logo), - banner: processOptImage(tr.banner), + poster: enqueueOptImage(tr.poster), + thumbnail: enqueueOptImage(tr.thumbnail), + logo: enqueueOptImage(tr.logo), + banner: enqueueOptImage(tr.banner), }), ); await tx diff --git a/api/src/controllers/seed/insert/staff.ts b/api/src/controllers/seed/insert/staff.ts index 6512953e..d84092d3 100644 --- a/api/src/controllers/seed/insert/staff.ts +++ b/api/src/controllers/seed/insert/staff.ts @@ -3,7 +3,7 @@ import { db } from "~/db"; import { roles, staff } from "~/db/schema"; import { conflictUpdateAllExcept } from "~/db/utils"; import type { SeedStaff } from "~/models/staff"; -import { processOptImage } from "../images"; +import { enqueueOptImage } from "../images"; export const insertStaff = async ( seed: SeedStaff[] | undefined, @@ -14,7 +14,7 @@ export const insertStaff = async ( return await db.transaction(async (tx) => { const people = seed.map((x) => ({ ...x.staff, - image: processOptImage(x.staff.image), + image: enqueueOptImage(x.staff.image), })); const ret = await tx .insert(staff) @@ -32,7 +32,7 @@ export const insertStaff = async ( order: i, character: { ...x.character, - image: processOptImage(x.character.image), + image: enqueueOptImage(x.character.image), }, })); diff --git a/api/src/controllers/seed/insert/studios.ts b/api/src/controllers/seed/insert/studios.ts index f53f93a2..85d18403 100644 --- a/api/src/controllers/seed/insert/studios.ts +++ b/api/src/controllers/seed/insert/studios.ts @@ -2,7 +2,7 @@ import { db } from "~/db"; import { showStudioJoin, studioTranslations, studios } from "~/db/schema"; import { conflictUpdateAllExcept } from "~/db/utils"; import type { SeedStudio } from "~/models/studio"; -import { processOptImage } from "../images"; +import { enqueueOptImage } from "../images"; type StudioI = typeof studios.$inferInsert; type StudioTransI = typeof studioTranslations.$inferInsert; @@ -38,7 +38,7 @@ export const insertStudios = async ( pk: ret[i].pk, language: lang, name: tr.name, - logo: processOptImage(tr.logo), + logo: enqueueOptImage(tr.logo), })), ); await tx diff --git a/api/src/controllers/seed/movies.ts b/api/src/controllers/seed/movies.ts index dcfcb06f..9ab4ac85 100644 --- a/api/src/controllers/seed/movies.ts +++ b/api/src/controllers/seed/movies.ts @@ -1,7 +1,7 @@ import { t } from "elysia"; import type { SeedMovie } from "~/models/movie"; import { getYear } from "~/utils"; -import { processOptImage } from "./images"; +import { enqueueOptImage } from "./images"; import { insertCollection } from "./insert/collection"; import { insertEntries } from "./insert/entries"; import { insertShow, updateAvailableCount } from "./insert/shows"; @@ -80,10 +80,10 @@ export const seedMovie = async ( language: movie.originalLanguage, name: original.name, latinName: original.latinName ?? null, - poster: processOptImage(original.poster), - thumbnail: processOptImage(original.thumbnail), - logo: processOptImage(original.logo), - banner: processOptImage(original.banner), + poster: enqueueOptImage(original.poster), + thumbnail: enqueueOptImage(original.thumbnail), + logo: enqueueOptImage(original.logo), + banner: enqueueOptImage(original.banner), }, ...movie, }, diff --git a/api/src/controllers/seed/series.ts b/api/src/controllers/seed/series.ts index f3d1048e..62b6d453 100644 --- a/api/src/controllers/seed/series.ts +++ b/api/src/controllers/seed/series.ts @@ -1,7 +1,7 @@ import { t } from "elysia"; import type { SeedSerie } from "~/models/serie"; import { getYear } from "~/utils"; -import { processOptImage } from "./images"; +import { enqueueOptImage } from "./images"; import { insertCollection } from "./insert/collection"; import { insertEntries } from "./insert/entries"; import { insertSeasons } from "./insert/seasons"; @@ -115,10 +115,10 @@ export const seedSerie = async ( language: serie.originalLanguage, name: original.name, latinName: original.latinName ?? null, - poster: processOptImage(original.poster), - thumbnail: processOptImage(original.thumbnail), - logo: processOptImage(original.logo), - banner: processOptImage(original.banner), + poster: enqueueOptImage(original.poster), + thumbnail: enqueueOptImage(original.thumbnail), + logo: enqueueOptImage(original.logo), + banner: enqueueOptImage(original.banner), }, ...serie, }, diff --git a/api/src/db/schema/queue.ts b/api/src/db/schema/queue.ts new file mode 100644 index 00000000..f912493b --- /dev/null +++ b/api/src/db/schema/queue.ts @@ -0,0 +1,23 @@ +import { + index, + integer, + jsonb, + timestamp, + uuid, + varchar, +} from "drizzle-orm/pg-core"; +import { schema } from "./utils"; + +export const mqueue = schema.table( + "mqueue", + { + id: uuid().notNull().primaryKey().defaultRandom(), + kind: varchar({ length: 255 }).notNull(), + message: jsonb().notNull(), + attempt: integer().notNull().default(0), + createdAt: timestamp({ withTimezone: true, mode: "string" }) + .notNull() + .defaultNow(), + }, + (t) => [index("mqueue_created").on(t.createdAt)], +); diff --git a/api/tsconfig.json b/api/tsconfig.json index b9c81697..e271deb5 100644 --- a/api/tsconfig.json +++ b/api/tsconfig.json @@ -3,15 +3,23 @@ "target": "ES2021", "module": "ES2022", "moduleResolution": "node", - "types": ["bun-types"], + "types": [ + "bun-types" + ], "esModuleInterop": true, "forceConsistentCasingInFileNames": true, "strict": true, "skipLibCheck": true, "noErrorTruncation": true, + "resolveJsonModule": true, "baseUrl": ".", "paths": { - "~/*": ["./src/*"] + "~/*": [ + "./src/*" + ], + "package.json": [ + "package.json" + ] } } }