From 433b90a3fb5e2816ddf8e8955182dc7d772007f0 Mon Sep 17 00:00:00 2001
From: Zoe Roux <zoe.roux@zoriya.dev>
Date: Thu, 4 Dec 2025 12:17:47 +0100
Subject: [PATCH] Add requests errors in db and api

---
 scanner/migrations/000001_request.up.sql |  1 +
 scanner/scanner/client.py                | 24 +++++++++++++++++-------
 scanner/scanner/models/request.py        |  3 ++-
 scanner/scanner/requests.py              | 14 +++++++++++++-
 scanner/scanner/status.py                |  1 +
 5 files changed, 34 insertions(+), 9 deletions(-)

diff --git a/scanner/migrations/000001_request.up.sql b/scanner/migrations/000001_request.up.sql
index f5b13184..c67fe377 100644
--- a/scanner/migrations/000001_request.up.sql
+++ b/scanner/migrations/000001_request.up.sql
@@ -18,6 +18,7 @@ create table scanner.requests(
 	external_id jsonb not null default '{}'::jsonb,
 	videos jsonb not null default '[]'::jsonb,
 	status scanner.request_status not null default 'pending',
+	error jsonb,
 	started_at timestamptz,
 	created_at timestamptz not null default now()::timestamptz,
 	constraint unique_kty unique nulls not distinct (kind, title, year)
diff --git a/scanner/scanner/client.py b/scanner/scanner/client.py
index 37742c0d..aaf05aaf 100644
--- a/scanner/scanner/client.py
+++ b/scanner/scanner/client.py
@@ -3,7 +3,7 @@ from logging import getLogger
 from types import TracebackType
 from typing import Literal
 
-from aiohttp import ClientSession
+from aiohttp import ClientResponse, ClientResponseError, ClientSession
 from pydantic import TypeAdapter
 
 from .models.movie import Movie
@@ -38,9 +38,19 @@ class KyooClient(metaclass=Singleton):
 	):
 		await self._client.close()
 
+	async def raise_for_status(self, r: ClientResponse):
+		if r.status >= 400:
+			raise ClientResponseError(
+				r.request_info,
+				r.history,
+				status=r.status,
+				message=await r.text(),
+				headers=r.headers,
+			)
+
 	async def get_videos_info(self) -> VideoInfo:
 		async with self._client.get("videos") as r:
-			r.raise_for_status()
+			await self.raise_for_status(r)
 			return VideoInfo(**await r.json())
 
 	async def create_videos(self, videos: list[Video]) -> list[VideoCreated]:
@@ -48,7 +58,7 @@ class KyooClient(metaclass=Singleton):
 			"videos",
 			data=TypeAdapter(list[Video]).dump_json(videos, by_alias=True),
 		) as r:
-			r.raise_for_status()
+			await self.raise_for_status(r)
 			return TypeAdapter(list[VideoCreated]).validate_json(await r.text())
 
 	async def delete_videos(self, videos: list[str] | set[str]):
@@ -56,14 +66,14 @@ class KyooClient(metaclass=Singleton):
 			"videos",
 			data=TypeAdapter(list[str] | set[str]).dump_json(videos, by_alias=True),
 		) as r:
-			r.raise_for_status()
+			await self.raise_for_status(r)
 
 	async def create_movie(self, movie: Movie) -> Resource:
 		async with self._client.post(
 			"movies",
 			data=movie.model_dump_json(by_alias=True),
 		) as r:
-			r.raise_for_status()
+			await self.raise_for_status(r)
 			return Resource.model_validate(await r.json())
 
 	async def create_serie(self, serie: Serie) -> Resource:
@@ -71,7 +81,7 @@ class KyooClient(metaclass=Singleton):
 			"series",
 			data=serie.model_dump_json(by_alias=True),
 		) as r:
-			r.raise_for_status()
+			await self.raise_for_status(r)
 			return Resource.model_validate(await r.json())
 
 	async def link_videos(
@@ -100,4 +110,4 @@ class KyooClient(metaclass=Singleton):
 				by_alias=True,
 			),
 		) as r:
-			r.raise_for_status()
+			await self.raise_for_status(r)
diff --git a/scanner/scanner/models/request.py b/scanner/scanner/models/request.py
index 4c6f7d70..74f22a43 100644
--- a/scanner/scanner/models/request.py
+++ b/scanner/scanner/models/request.py
@@ -1,6 +1,6 @@
 from __future__ import annotations
 from datetime import datetime
-from typing import Literal
+from typing import Any, Literal
 
 from pydantic import Field
 
@@ -31,4 +31,5 @@ class RequestRet(Model):
 		"running",
 		"failed",
 	]
+	error: dict[str, Any] | None
 	started_at: datetime | None
diff --git a/scanner/scanner/requests.py b/scanner/scanner/requests.py
index 677234de..f05a1966 100644
--- a/scanner/scanner/requests.py
+++ b/scanner/scanner/requests.py
@@ -1,5 +1,6 @@
 from asyncio import CancelledError, Event, TaskGroup
 from logging import getLogger
+from traceback import TracebackException
 from typing import cast
 
 from asyncpg import Connection, Pool
@@ -161,11 +162,22 @@ class RequestProcessor:
 				update
 					scanner.requests
 				set
-					status = 'failed'
+					status = 'failed',
+					error = $2
 				where
 					pk = $1
 				""",
 				request.pk,
+				{
+					"title": type(e).__name__,
+					"message": str(e),
+					"traceback": [
+						line
+						for part in TracebackException.from_exception(e).format()
+						for line in part.split("\n")
+						if line.strip()
+					],
+				},
 			)
 		return True
 
diff --git a/scanner/scanner/status.py b/scanner/scanner/status.py
index ed0ecb5d..28d6744e 100644
--- a/scanner/scanner/status.py
+++ b/scanner/scanner/status.py
@@ -28,6 +28,7 @@ class StatusService:
 				title,
 				year,
 				status,
+				error,
 				started_at
 			from
 				scanner.requests