Cutlery/Kyoo
1
0
Fork 0
mirror of https://github.com/zoriya/Kyoo.git synced 2025-12-03 03:35:11 -05:00
Code Issues Packages Projects Releases Wiki Activity

Properly handle spans of image downloading

Browse Source
This commit is contained in:
Zoe Roux 2025-11-23 15:14:23 +01:00
parent 1f7844b8a5
commit 3602905e86
No known key found for this signature in database
2 changed files with 38 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
api/src/auth.ts
View File

@ -73,7 +73,7 @@ export const auth = new Elysia({ name: "auth" })
.macro({
permissions(perms: string[]) {
return {
beforeHandle: ({ jwt, status }) => {
beforeHandle: function permissionCheck({ jwt, status }) {
for (const perm of perms) {
if (!jwt!.permissions.includes(perm)) {
return status(403, {

88
api/src/controllers/seed/images.ts
View File

@ -1,4 +1,6 @@
import path from "node:path";
import { getCurrentSpan, record, setAttributes } from "@elysiajs/opentelemetry";
import { SpanStatusCode } from "@opentelemetry/api";
import { encode } from "blurhash";
import { and, eq, is, lt, type SQL, sql } from "drizzle-orm";
import { PgColumn, type PgTable } from "drizzle-orm/pg-core";
@ -78,44 +80,7 @@ export const enqueueOptImage = async (
};
export const processImages = async () => {
async function processOne() {
return await db.transaction(async (tx) => {
const [item] = await tx
.select()
.from(mqueue)
.for("update", { skipLocked: true })
.where(and(eq(mqueue.kind, "image"), lt(mqueue.attempt, 5)))
.orderBy(mqueue.attempt, mqueue.createdAt)
.limit(1);
if (!item) return false;
const img = item.message as ImageTask;
try {
const blurhash = await downloadImage(img.id, img.url);
const ret: Image = { id: img.id, source: img.url, blurhash };
const table = sql.raw(img.table);
const column = sql.raw(img.column);
await tx.execute(sql`
update ${table} set ${column} = ${ret}
where ${column}->'id' = ${sql.raw(`'"${img.id}"'::jsonb`)}
`);
await tx.delete(mqueue).where(eq(mqueue.id, item.id));
} catch (err: any) {
console.error("Failed to download image", img.url, err.message);
// don't use the transaction here, it can be aborted.
await db
.update(mqueue)
.set({ attempt: sql`${mqueue.attempt}+1` })
.where(eq(mqueue.id, item.id));
}
return true;
});
}
return record("download images", async () => {
let running = false;
async function processAll() {
if (running) return;
@ -138,8 +103,55 @@ export const processImages = async () => {
// start processing old tasks
await processAll();
return () => client.release(true);
});
};
async function processOne() {
return record("download", async () => {
return await db.transaction(async (tx) => {
const [item] = await tx
.select()
.from(mqueue)
.for("update", { skipLocked: true })
.where(and(eq(mqueue.kind, "image"), lt(mqueue.attempt, 5)))
.orderBy(mqueue.attempt, mqueue.createdAt)
.limit(1);
if (!item) return false;
const img = item.message as ImageTask;
setAttributes({ "item.url": img.url });
try {
const blurhash = await downloadImage(img.id, img.url);
const ret: Image = { id: img.id, source: img.url, blurhash };
const table = sql.raw(img.table);
const column = sql.raw(img.column);
await tx.execute(sql`
update ${table} set ${column} = ${ret}
where ${column}->'id' = ${sql.raw(`'"${img.id}"'::jsonb`)}
`);
await tx.delete(mqueue).where(eq(mqueue.id, item.id));
} catch (err: any) {
const span = getCurrentSpan();
if (span) {
span.recordException(err);
span.setStatus({ code: SpanStatusCode.ERROR });
}
console.error("Failed to download image", img.url, err.message);
// don't use the transaction here, it can be aborted.
await db
.update(mqueue)
.set({ attempt: sql`${mqueue.attempt}+1` })
.where(eq(mqueue.id, item.id));
}
return true;
});
});
}
async function downloadImage(id: string, url: string): Promise<string> {
const low = await getFile(path.join(imageDir, `${id}.low.jpg`))
.arrayBuffer()