wip: Add image downloading

api/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "api",
-	"version": "1.0.50",
+	"version": "5.0.0",
 	"scripts": {
 		"dev": "bun --watch src/index.ts",
 		"build": "bun build src/index.ts --target bun --outdir ./dist",

api/src/controllers/seed/images.ts

@@ -1,22 +1,74 @@
+import { eq, sql } from "drizzle-orm";
+import { version } from "package.json";
+import type { PoolClient } from "pg";
+import { db } from "~/db";
+import * as schema from "~/db/schema";
+import { mqueue } from "~/db/schema/queue";
 import type { Image } from "~/models/utils";
 
+type ImageTask = {
+	id: string;
+	url: string;
+	table: string;
+	column: string;
+};
+
 // this will only push a task to the image downloader service and not download it instantly.
 // this is both done to prevent to many requests to be sent at once and to make sure POST
 // requests are not blocked by image downloading or blurhash calculation
-export const processImage = (url: string): Image => {
+export const enqueueImage = async (
+	tx: typeof db,
+	url: string,
+): Promise<Image> => {
 	const hasher = new Bun.CryptoHasher("sha256");
 	hasher.update(url);
+	const id = hasher.digest().toString("hex");
 
-	// TODO: download source, save it in multiples qualities & process blurhash
+	await tx.insert(mqueue).values({ kind: "image", message: { id, url } });
 
 	return {
-		id: hasher.digest().toString("hex"),
+		id,
 		source: url,
 		blurhash: "",
 	};
 };
 
-export const processOptImage = (url: string | null): Image | null => {
+export const enqueueOptImage = async (
+	tx: typeof db,
+	url: string | null,
+): Promise<Image | null> => {
 	if (!url) return null;
-	return processImage(url);
+	return await enqueueImage(tx, url);
+};
+
+export const processImages = async () => {
+	await db.transaction(async (tx) => {
+		const [item] = await tx
+			.select()
+			.from(mqueue)
+			.for("update", { skipLocked: true })
+			.where(eq(mqueue.kind, "image"))
+			.orderBy(mqueue.createdAt)
+			.limit(1);
+
+		const img = item.message as ImageTask;
+		await fetch(img.url, { headers: { "User-Agent": `Kyoo v${version}` }, });
+		const blurhash = "";
+
+		const table = schema[img.table as keyof typeof schema] as any;
+
+		await tx
+			.update(table)
+			.set({
+				[img.column]: { id: img.id, source: img.url, blurhash } satisfies Image,
+			})
+			.where(eq(sql`${table[img.column]}->'id'`, img.id));
+
+		await tx.delete(mqueue).where(eq(mqueue.id, item.id));
+	});
+
+	const client = (await db.$client.connect()) as PoolClient;
+	client.on("notification", (evt) => {
+		if (evt.channel !== "image") return;
+	});
 };

api/src/controllers/seed/insert/collection.ts

@@ -5,7 +5,7 @@ import { conflictUpdateAllExcept } from "~/db/utils";
 import type { SeedCollection } from "~/models/collections";
 import type { SeedMovie } from "~/models/movie";
 import type { SeedSerie } from "~/models/serie";
-import { processOptImage } from "../images";
+import { enqueueOptImage } from "../images";
 
 type ShowTrans = typeof showTranslations.$inferInsert;
 
@@ -53,10 +53,10 @@ export const insertCollection = async (
 				pk: ret.pk,
 				language: lang,
 				...tr,
-				poster: processOptImage(tr.poster),
-				thumbnail: processOptImage(tr.thumbnail),
-				logo: processOptImage(tr.logo),
-				banner: processOptImage(tr.banner),
+				poster: enqueueOptImage(tr.poster),
+				thumbnail: enqueueOptImage(tr.thumbnail),
+				logo: enqueueOptImage(tr.logo),
+				banner: enqueueOptImage(tr.banner),
 			}),
 		);
 		await tx

api/src/controllers/seed/insert/entries.ts

@@ -8,7 +8,7 @@ import {
 } from "~/db/schema";
 import { conflictUpdateAllExcept, sqlarr, values } from "~/db/utils";
 import type { SeedEntry as SEntry, SeedExtra as SExtra } from "~/models/entry";
-import { processOptImage } from "../images";
+import { enqueueOptImage } from "../images";
 import { guessNextRefresh } from "../refresh";
 import { updateAvailableCount } from "./shows";
 
@@ -55,7 +55,7 @@ export const insertEntries = async (
 				...entry,
 				showPk: show.pk,
 				slug: generateSlug(show.slug, seed),
-				thumbnail: processOptImage(seed.thumbnail),
+				thumbnail: enqueueOptImage(seed.thumbnail),
 				nextRefresh:
 					entry.kind !== "extra"
 						? guessNextRefresh(entry.airDate ?? new Date())
@@ -103,7 +103,7 @@ export const insertEntries = async (
 				...tr,
 				poster:
 					seed.kind === "movie"
-						? processOptImage((tr as any).poster)
+						? enqueueOptImage((tr as any).poster)
 						: undefined,
 			}));
 		});

api/src/controllers/seed/insert/seasons.ts

@@ -2,7 +2,7 @@ import { db } from "~/db";
 import { seasonTranslations, seasons } from "~/db/schema";
 import { conflictUpdateAllExcept } from "~/db/utils";
 import type { SeedSeason } from "~/models/season";
-import { processOptImage } from "../images";
+import { enqueueOptImage } from "../images";
 import { guessNextRefresh } from "../refresh";
 
 type SeasonI = typeof seasons.$inferInsert;
@@ -43,9 +43,9 @@ export const insertSeasons = async (
 				pk: ret[i].pk,
 				language: lang,
 				...tr,
-				poster: processOptImage(tr.poster),
-				thumbnail: processOptImage(tr.thumbnail),
-				banner: processOptImage(tr.banner),
+				poster: enqueueOptImage(tr.poster),
+				thumbnail: enqueueOptImage(tr.thumbnail),
+				banner: enqueueOptImage(tr.banner),
 			})),
 		);
 		await tx

api/src/controllers/seed/insert/shows.ts

@@ -6,7 +6,7 @@ import type { SeedCollection } from "~/models/collections";
 import type { SeedMovie } from "~/models/movie";
 import type { SeedSerie } from "~/models/serie";
 import { getYear } from "~/utils";
-import { processOptImage } from "../images";
+import { enqueueOptImage } from "../images";
 
 type Show = typeof shows.$inferInsert;
 type ShowTrans = typeof showTranslations.$inferInsert;
@@ -27,10 +27,10 @@ export const insertShow = async (
 				pk: ret.pk,
 				language: lang,
 				...tr,
-				poster: processOptImage(tr.poster),
-				thumbnail: processOptImage(tr.thumbnail),
-				logo: processOptImage(tr.logo),
-				banner: processOptImage(tr.banner),
+				poster: enqueueOptImage(tr.poster),
+				thumbnail: enqueueOptImage(tr.thumbnail),
+				logo: enqueueOptImage(tr.logo),
+				banner: enqueueOptImage(tr.banner),
 			}),
 		);
 		await tx

api/src/controllers/seed/insert/staff.ts

@@ -3,7 +3,7 @@ import { db } from "~/db";
 import { roles, staff } from "~/db/schema";
 import { conflictUpdateAllExcept } from "~/db/utils";
 import type { SeedStaff } from "~/models/staff";
-import { processOptImage } from "../images";
+import { enqueueOptImage } from "../images";
 
 export const insertStaff = async (
 	seed: SeedStaff[] | undefined,
@@ -14,7 +14,7 @@ export const insertStaff = async (
 	return await db.transaction(async (tx) => {
 		const people = seed.map((x) => ({
 			...x.staff,
-			image: processOptImage(x.staff.image),
+			image: enqueueOptImage(x.staff.image),
 		}));
 		const ret = await tx
 			.insert(staff)
@@ -32,7 +32,7 @@ export const insertStaff = async (
 			order: i,
 			character: {
 				...x.character,
-				image: processOptImage(x.character.image),
+				image: enqueueOptImage(x.character.image),
 			},
 		}));
 

api/src/controllers/seed/insert/studios.ts

@@ -2,7 +2,7 @@ import { db } from "~/db";
 import { showStudioJoin, studioTranslations, studios } from "~/db/schema";
 import { conflictUpdateAllExcept } from "~/db/utils";
 import type { SeedStudio } from "~/models/studio";
-import { processOptImage } from "../images";
+import { enqueueOptImage } from "../images";
 
 type StudioI = typeof studios.$inferInsert;
 type StudioTransI = typeof studioTranslations.$inferInsert;
@@ -38,7 +38,7 @@ export const insertStudios = async (
 				pk: ret[i].pk,
 				language: lang,
 				name: tr.name,
-				logo: processOptImage(tr.logo),
+				logo: enqueueOptImage(tr.logo),
 			})),
 		);
 		await tx

api/src/controllers/seed/movies.ts

@@ -1,7 +1,7 @@
 import { t } from "elysia";
 import type { SeedMovie } from "~/models/movie";
 import { getYear } from "~/utils";
-import { processOptImage } from "./images";
+import { enqueueOptImage } from "./images";
 import { insertCollection } from "./insert/collection";
 import { insertEntries } from "./insert/entries";
 import { insertShow, updateAvailableCount } from "./insert/shows";
@@ -80,10 +80,10 @@ export const seedMovie = async (
 				language: movie.originalLanguage,
 				name: original.name,
 				latinName: original.latinName ?? null,
-				poster: processOptImage(original.poster),
-				thumbnail: processOptImage(original.thumbnail),
-				logo: processOptImage(original.logo),
-				banner: processOptImage(original.banner),
+				poster: enqueueOptImage(original.poster),
+				thumbnail: enqueueOptImage(original.thumbnail),
+				logo: enqueueOptImage(original.logo),
+				banner: enqueueOptImage(original.banner),
 			},
 			...movie,
 		},

api/src/controllers/seed/series.ts

@@ -1,7 +1,7 @@
 import { t } from "elysia";
 import type { SeedSerie } from "~/models/serie";
 import { getYear } from "~/utils";
-import { processOptImage } from "./images";
+import { enqueueOptImage } from "./images";
 import { insertCollection } from "./insert/collection";
 import { insertEntries } from "./insert/entries";
 import { insertSeasons } from "./insert/seasons";
@@ -115,10 +115,10 @@ export const seedSerie = async (
 				language: serie.originalLanguage,
 				name: original.name,
 				latinName: original.latinName ?? null,
-				poster: processOptImage(original.poster),
-				thumbnail: processOptImage(original.thumbnail),
-				logo: processOptImage(original.logo),
-				banner: processOptImage(original.banner),
+				poster: enqueueOptImage(original.poster),
+				thumbnail: enqueueOptImage(original.thumbnail),
+				logo: enqueueOptImage(original.logo),
+				banner: enqueueOptImage(original.banner),
 			},
 			...serie,
 		},

api/src/db/schema/queue.ts

@@ -0,0 +1,23 @@
+import {
+	index,
+	integer,
+	jsonb,
+	timestamp,
+	uuid,
+	varchar,
+} from "drizzle-orm/pg-core";
+import { schema } from "./utils";
+
+export const mqueue = schema.table(
+	"mqueue",
+	{
+		id: uuid().notNull().primaryKey().defaultRandom(),
+		kind: varchar({ length: 255 }).notNull(),
+		message: jsonb().notNull(),
+		attempt: integer().notNull().default(0),
+		createdAt: timestamp({ withTimezone: true, mode: "string" })
+			.notNull()
+			.defaultNow(),
+	},
+	(t) => [index("mqueue_created").on(t.createdAt)],
+);

api/tsconfig.json

@@ -3,15 +3,23 @@
 		"target": "ES2021",
 		"module": "ES2022",
 		"moduleResolution": "node",
-		"types": ["bun-types"],
+		"types": [
+			"bun-types"
+		],
 		"esModuleInterop": true,
 		"forceConsistentCasingInFileNames": true,
 		"strict": true,
 		"skipLibCheck": true,
 		"noErrorTruncation": true,
+		"resolveJsonModule": true,
 		"baseUrl": ".",
 		"paths": {
-			"~/*": ["./src/*"]
+			"~/*": [
+				"./src/*"
+			],
+			"package.json": [
+				"package.json"
+			]
 		}
 	}
 }