Cutlery/Kyoo
1
0
Fork 0
mirror of https://github.com/zoriya/Kyoo.git synced 2025-05-31 12:14:46 -04:00
Code Issues Packages Projects Releases Wiki Activity

Create task runner to download images

Browse Source
This commit is contained in:
Zoe Roux 2025-03-16 14:51:58 +01:00
parent 67511a3aa8
commit 71b8cbca4a
No known key found for this signature in database
1 changed files with 53 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
api/src/controllers/seed/images.ts
View File

@ -25,58 +25,85 @@ type ImageTask = {
// requests are not blocked by image downloading or blurhash calculation // requests are not blocked by image downloading or blurhash calculation
export const enqueueImage = async ( export const enqueueImage = async (
tx: typeof db, tx: typeof db,
url: string, img: Omit<ImageTask, "id">,
): Promise<Image> => { ): Promise<Image> => {
const hasher = new Bun.CryptoHasher("sha256"); const hasher = new Bun.CryptoHasher("sha256");
hasher.update(url); hasher.update(img.url);
const id = hasher.digest().toString("hex"); const id = hasher.digest().toString("hex");
await tx.insert(mqueue).values({ kind: "image", message: { id, url } }); await tx.insert(mqueue).values({ kind: "image", message: { id, ...img } });
await tx.execute(sql`notify image`);
return { return {
id, id,
source: url, source: img.url,
blurhash: "", blurhash: "",
}; };
}; };
export const enqueueOptImage = async ( export const enqueueOptImage = async (
tx: typeof db, tx: typeof db,
url: string | null, img: Omit<ImageTask, "id">,
): Promise<Image | null> => { ): Promise<Image | null> => {
if (!url) return null; if (!img.url) return null;
return await enqueueImage(tx, url); return await enqueueImage(tx, img);
}; };
export const processImages = async () => { export const processImages = async () => {
await db.transaction(async (tx) => { async function processOne() {
const [item] = await tx return await db.transaction(async (tx) => {
.select() const [item] = await tx
.from(mqueue) .select()
.for("update", { skipLocked: true }) .from(mqueue)
.where(eq(mqueue.kind, "image")) .for("update", { skipLocked: true })
.orderBy(mqueue.createdAt) .where(eq(mqueue.kind, "image"))
.limit(1); .orderBy(mqueue.createdAt)
.limit(1);
const img = item.message as ImageTask; if (!item) return false;
const blurhash = await downloadImage(img.id, img.url);
const table = schema[img.table as keyof typeof schema] as any; const img = item.message as ImageTask;
const blurhash = await downloadImage(img.id, img.url);
await tx const table = schema[img.table as keyof typeof schema] as any;
.update(table)
.set({
[img.column]: { id: img.id, source: img.url, blurhash } satisfies Image,
})
.where(eq(sql`${table[img.column]}->'id'`, img.id));
await tx.delete(mqueue).where(eq(mqueue.id, item.id)); await tx
}); .update(table)
.set({
[img.column]: {
id: img.id,
source: img.url,
blurhash,
} satisfies Image,
})
.where(eq(sql`${table[img.column]}->'id'`, img.id));
await tx.delete(mqueue).where(eq(mqueue.id, item.id));
return true;
});
}
let running = false;
async function processAll() {
if (running) return;
running = true;
let found = true;
while (found) {
found = await processOne();
}
running = false;
}
const client = (await db.$client.connect()) as PoolClient; const client = (await db.$client.connect()) as PoolClient;
client.on("notification", (evt) => { client.on("notification", (evt) => {
if (evt.channel !== "image") return; if (evt.channel !== "image") return;
processAll();
}); });
await client.query("listen image");
// start processing old tasks
await processAll();
}; };
async function downloadImage(id: string, url: string): Promise<string> { async function downloadImage(id: string, url: string): Promise<string> {