From 3602905e869c52b89c5c6c86dae5fd3b62e7e8f5 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 23 Nov 2025 15:14:23 +0100 Subject: [PATCH 1/4] Properly handle spans of image downloading --- api/src/auth.ts | 2 +- api/src/controllers/seed/images.ts | 62 ++++++++++++++++++------------ 2 files changed, 38 insertions(+), 26 deletions(-) diff --git a/api/src/auth.ts b/api/src/auth.ts index 2a38bfa4..d7f0d874 100644 --- a/api/src/auth.ts +++ b/api/src/auth.ts @@ -73,7 +73,7 @@ export const auth = new Elysia({ name: "auth" }) .macro({ permissions(perms: string[]) { return { - beforeHandle: ({ jwt, status }) => { + beforeHandle: function permissionCheck({ jwt, status }) { for (const perm of perms) { if (!jwt!.permissions.includes(perm)) { return status(403, { diff --git a/api/src/controllers/seed/images.ts b/api/src/controllers/seed/images.ts index f80b0d53..106047b8 100644 --- a/api/src/controllers/seed/images.ts +++ b/api/src/controllers/seed/images.ts @@ -1,4 +1,6 @@ import path from "node:path"; +import { getCurrentSpan, record, setAttributes } from "@elysiajs/opentelemetry"; +import { SpanStatusCode } from "@opentelemetry/api"; import { encode } from "blurhash"; import { and, eq, is, lt, type SQL, sql } from "drizzle-orm"; import { PgColumn, type PgTable } from "drizzle-orm/pg-core"; @@ -78,7 +80,34 @@ export const enqueueOptImage = async ( }; export const processImages = async () => { - async function processOne() { + return record("download images", async () => { + let running = false; + async function processAll() { + if (running) return; + running = true; + + let found = true; + while (found) { + found = await processOne(); + } + running = false; + } + + const client = (await db.$client.connect()) as PoolClient; + client.on("notification", (evt) => { + if (evt.channel !== "kyoo_image") return; + processAll(); + }); + await client.query("listen kyoo_image"); + + // start processing old tasks + await processAll(); + return () => client.release(true); + }); +}; + +async function processOne() { + return record("download", async () => { return await db.transaction(async (tx) => { const [item] = await tx .select() @@ -91,6 +120,7 @@ export const processImages = async () => { if (!item) return false; const img = item.message as ImageTask; + setAttributes({ "item.url": img.url }); try { const blurhash = await downloadImage(img.id, img.url); const ret: Image = { id: img.id, source: img.url, blurhash }; @@ -105,6 +135,11 @@ export const processImages = async () => { await tx.delete(mqueue).where(eq(mqueue.id, item.id)); } catch (err: any) { + const span = getCurrentSpan(); + if (span) { + span.recordException(err); + span.setStatus({ code: SpanStatusCode.ERROR }); + } console.error("Failed to download image", img.url, err.message); // don't use the transaction here, it can be aborted. await db @@ -114,31 +149,8 @@ export const processImages = async () => { } return true; }); - } - - let running = false; - async function processAll() { - if (running) return; - running = true; - - let found = true; - while (found) { - found = await processOne(); - } - running = false; - } - - const client = (await db.$client.connect()) as PoolClient; - client.on("notification", (evt) => { - if (evt.channel !== "kyoo_image") return; - processAll(); }); - await client.query("listen kyoo_image"); - - // start processing old tasks - await processAll(); - return () => client.release(true); -}; +} async function downloadImage(id: string, url: string): Promise { const low = await getFile(path.join(imageDir, `${id}.low.jpg`)) From c5fa3ecb012abcb1c3af2b7eb7a33c948b14c6b0 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 23 Nov 2025 15:19:04 +0100 Subject: [PATCH 2/4] Cleanup scanner processing span --- scanner/scanner/requests.py | 72 +++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 35 deletions(-) diff --git a/scanner/scanner/requests.py b/scanner/scanner/requests.py index 73db44ce..677234de 100644 --- a/scanner/scanner/requests.py +++ b/scanner/scanner/requests.py @@ -101,6 +101,7 @@ class RequestProcessor: finally: self._processing = False + @tracer.start_as_current_span("process video") async def process_request(self): cur = await self._database.fetchrow( """ @@ -128,43 +129,44 @@ class RequestProcessor: return False request = Request.model_validate(cur) - with tracer.start_as_current_span(f"process {request.title}") as span: - logger.info(f"Starting to process {request.title}") - try: - show = await self._run_request(request) - finished = await self._database.fetchrow( - """ - delete from scanner.requests - where pk = $1 - returning - videos - """, - request.pk, + span = trace.get_current_span() + span.update_name(f"process {request.title}") + logger.info(f"Starting to process {request.title}") + try: + show = await self._run_request(request) + finished = await self._database.fetchrow( + """ + delete from scanner.requests + where pk = $1 + returning + videos + """, + request.pk, + ) + if finished and finished["videos"] != request.videos: + videos = TypeAdapter(list[Request.Video]).validate_python( + finished["videos"] ) - if finished and finished["videos"] != request.videos: - videos = TypeAdapter(list[Request.Video]).validate_python( - finished["videos"] - ) - await self._client.link_videos( - "movie" if request.kind == "movie" else "serie", - show.slug, - videos, - ) - except Exception as e: - span.set_status(trace.Status(trace.StatusCode.ERROR)) - span.record_exception(e) - logger.error("Couldn't process request", exc_info=e) - cur = await self._database.execute( - """ - update - scanner.requests - set - status = 'failed' - where - pk = $1 - """, - request.pk, + await self._client.link_videos( + "movie" if request.kind == "movie" else "serie", + show.slug, + videos, ) + except Exception as e: + span.set_status(trace.Status(trace.StatusCode.ERROR)) + span.record_exception(e) + logger.error("Couldn't process request", exc_info=e) + cur = await self._database.execute( + """ + update + scanner.requests + set + status = 'failed' + where + pk = $1 + """, + request.pk, + ) return True async def _run_request(self, request: Request) -> Resource: From 7b2f1c7a824a3712f23344c9e83de369cc71f14f Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 23 Nov 2025 18:06:55 +0100 Subject: [PATCH 3/4] Fix image test in github --- .github/workflows/api-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/api-test.yml b/.github/workflows/api-test.yml index 8d680569..8fd60b54 100644 --- a/.github/workflows/api-test.yml +++ b/.github/workflows/api-test.yml @@ -37,3 +37,4 @@ jobs: run: bun test env: PGHOST: localhost + IMAGES_PATH: ./images From d4deafe1dc3fb0d7a696b0e0199c7283845cfe50 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 23 Nov 2025 18:58:19 +0100 Subject: [PATCH 4/4] Forward `X-Forward-Host` & proto headers in traefik --- chart/values.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/chart/values.yaml b/chart/values.yaml index 03c3c53f..2a09e0b7 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -429,6 +429,8 @@ traefikproxy: extraArgs: - '--entryPoints.web.address=:80/tcp' - '--entryPoints.websecure.address=:443/tcp' + - '--entryPoints.web.forwardedHeaders.insecure=true' + - '--entryPoints.websecure.forwardedHeaders.insecure=true' - '--api.dashboard=true' - '--api.insecure=true' - '--log.level=INFO'