Cutlery/Kyoo
1
0
Fork 0
mirror of https://github.com/zoriya/Kyoo.git synced 2025-12-02 19:25:14 -05:00
Code Issues Packages Projects Releases Wiki Activity

Properly handle spans of image downloading

Browse Source
This commit is contained in:
Zoe Roux 2025-11-23 15:14:23 +01:00
parent 1f7844b8a5
commit 3602905e86
No known key found for this signature in database
2 changed files with 38 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
api/src/auth.ts
View File

@ -73,7 +73,7 @@ export const auth = new Elysia({ name: "auth" })
.macro({ .macro({
permissions(perms: string[]) { permissions(perms: string[]) {
return { return {
beforeHandle: ({ jwt, status }) => { beforeHandle: function permissionCheck({ jwt, status }) {
for (const perm of perms) { for (const perm of perms) {
if (!jwt!.permissions.includes(perm)) { if (!jwt!.permissions.includes(perm)) {
return status(403, { return status(403, {

62
api/src/controllers/seed/images.ts
View File

@ -1,4 +1,6 @@
import path from "node:path"; import path from "node:path";
import { getCurrentSpan, record, setAttributes } from "@elysiajs/opentelemetry";
import { SpanStatusCode } from "@opentelemetry/api";
import { encode } from "blurhash"; import { encode } from "blurhash";
import { and, eq, is, lt, type SQL, sql } from "drizzle-orm"; import { and, eq, is, lt, type SQL, sql } from "drizzle-orm";
import { PgColumn, type PgTable } from "drizzle-orm/pg-core"; import { PgColumn, type PgTable } from "drizzle-orm/pg-core";
@ -78,7 +80,34 @@ export const enqueueOptImage = async (
}; };
export const processImages = async () => { export const processImages = async () => {
async function processOne() { return record("download images", async () => {
let running = false;
async function processAll() {
if (running) return;
running = true;
let found = true;
while (found) {
found = await processOne();
}
running = false;
}
const client = (await db.$client.connect()) as PoolClient;
client.on("notification", (evt) => {
if (evt.channel !== "kyoo_image") return;
processAll();
});
await client.query("listen kyoo_image");
// start processing old tasks
await processAll();
return () => client.release(true);
});
};
async function processOne() {
return record("download", async () => {
return await db.transaction(async (tx) => { return await db.transaction(async (tx) => {
const [item] = await tx const [item] = await tx
.select() .select()
@ -91,6 +120,7 @@ export const processImages = async () => {
if (!item) return false; if (!item) return false;
const img = item.message as ImageTask; const img = item.message as ImageTask;
setAttributes({ "item.url": img.url });
try { try {
const blurhash = await downloadImage(img.id, img.url); const blurhash = await downloadImage(img.id, img.url);
const ret: Image = { id: img.id, source: img.url, blurhash }; const ret: Image = { id: img.id, source: img.url, blurhash };
@ -105,6 +135,11 @@ export const processImages = async () => {
await tx.delete(mqueue).where(eq(mqueue.id, item.id)); await tx.delete(mqueue).where(eq(mqueue.id, item.id));
} catch (err: any) { } catch (err: any) {
const span = getCurrentSpan();
if (span) {
span.recordException(err);
span.setStatus({ code: SpanStatusCode.ERROR });
}
console.error("Failed to download image", img.url, err.message); console.error("Failed to download image", img.url, err.message);
// don't use the transaction here, it can be aborted. // don't use the transaction here, it can be aborted.
await db await db
@ -114,31 +149,8 @@ export const processImages = async () => {
} }
return true; return true;
}); });
}
let running = false;
async function processAll() {
if (running) return;
running = true;
let found = true;
while (found) {
found = await processOne();
}
running = false;
}
const client = (await db.$client.connect()) as PoolClient;
client.on("notification", (evt) => {
if (evt.channel !== "kyoo_image") return;
processAll();
}); });
await client.query("listen kyoo_image"); }
// start processing old tasks
await processAll();
return () => client.release(true);
};
async function downloadImage(id: string, url: string): Promise<string> { async function downloadImage(id: string, url: string): Promise<string> {
const low = await getFile(path.join(imageDir, `${id}.low.jpg`)) const low = await getFile(path.join(imageDir, `${id}.low.jpg`))