Lots of api fixes + error api for scanner (#1201)

api/shell.nix

@@ -13,4 +13,7 @@ pkgs.mkShell {
   ];
 
   SHARP_FORCE_GLOBAL_LIBVIPS = 1;
+  shellHook = ''
+    export LD_LIBRARY_PATH=${pkgs.stdenv.cc.cc.lib}/lib:$LD_LIBRARY_PATH
+  '';
 }

api/src/base.ts

@@ -52,8 +52,7 @@ export const base = new Elysia({ name: "base" })
 		console.error(code, error);
 		return {
 			status: 500,
-			message: "message" in error ? (error?.message ?? code) : code,
-			details: error,
+			message: "Internal server error",
 		} as KError;
 	})
 	.get("/health", () => ({ status: "healthy" }) as const, {

api/src/controllers/seed/images.ts

@@ -30,8 +30,8 @@ export type ImageTask = {
 export const enqueueOptImage = (
 	imgQueue: ImageTask[],
 	img:
-		| { url: string | null; column: PgColumn }
-		| { url: string | null; table: PgTable; column: SQL },
+		| { url?: string | null; column: PgColumn }
+		| { url?: string | null; table: PgTable; column: SQL },
 ): Image | null => {
 	if (!img.url) return null;
 
@@ -139,9 +139,9 @@ const processOne = record("download", async () => {
 			const column = sql.raw(img.column);
 
 			await tx.execute(sql`
-					update ${table} set ${column} = ${ret}
-					where ${column}->'id' = ${sql.raw(`'"${img.id}"'::jsonb`)}
-				`);
+				update ${table} set ${column} = ${ret}
+				where ${column}->'id' = ${sql.raw(`'"${img.id}"'::jsonb`)}
+			`);
 
 			await tx.delete(mqueue).where(eq(mqueue.id, item.id));
 		} catch (err: any) {

api/src/controllers/seed/insert/shows.ts

@@ -33,10 +33,10 @@ export const insertShow = record(
 	async (
 		show: Omit<Show, "original">,
 		original: Original & {
-			poster: string | null;
-			thumbnail: string | null;
-			banner: string | null;
-			logo: string | null;
+			poster?: string | null;
+			thumbnail?: string | null;
+			banner?: string | null;
+			logo?: string | null;
 		},
 		translations:
 			| SeedMovie["translations"]

api/src/controllers/seed/insert/staff.ts

@@ -4,6 +4,7 @@ import { roles, staff } from "~/db/schema";
 import { conflictUpdateAllExcept, unnestValues } from "~/db/utils";
 import type { SeedStaff } from "~/models/staff";
 import { record } from "~/otel";
+import { uniqBy } from "~/utils";
 import { enqueueOptImage, flushImageQueue, type ImageTask } from "../images";
 
 export const insertStaff = record(
@@ -13,13 +14,16 @@ export const insertStaff = record(
 
 		return await db.transaction(async (tx) => {
 			const imgQueue: ImageTask[] = [];
-			const people = seed.map((x) => ({
-				...x.staff,
-				image: enqueueOptImage(imgQueue, {
-					url: x.staff.image,
-					column: staff.image,
-				}),
-			}));
+			const people = uniqBy(
+				seed.map((x) => ({
+					...x.staff,
+					image: enqueueOptImage(imgQueue, {
+						url: x.staff.image,
+						column: staff.image,
+					}),
+				})),
+				(x) => x.slug,
+			);
 			const ret = await tx
 				.insert(staff)
 				.select(unnestValues(people, staff))
@@ -36,7 +40,7 @@ export const insertStaff = record(
 
 			const rval = seed.map((x, i) => ({
 				showPk,
-				staffPk: ret[i].pk,
+				staffPk: ret.find((y) => y.slug === x.staff.slug)!.pk,
 				kind: x.kind,
 				order: i,
 				character: {

api/src/controllers/seed/movies.ts

@@ -55,20 +55,13 @@ export const seedMovie = async (
 	const { translations, videos, collection, studios, staff, ...movie } = seed;
 	const nextRefresh = guessNextRefresh(movie.airDate ?? new Date());
 
-	const original = translations[movie.originalLanguage];
-	if (!original) {
-		return {
-			status: 422,
-			message: "No translation available in the original language.",
-		};
-	}
-
 	const col = await insertCollection(collection, {
 		kind: "movie",
 		nextRefresh,
 		...seed,
 	});
 
+	const original = translations[movie.originalLanguage];
 	const show = await insertShow(
 		{
 			kind: "movie",
@@ -78,11 +71,17 @@ export const seedMovie = async (
 			entriesCount: 1,
 			...movie,
 		},
-		{
-			...original,
-			latinName: original.latinName ?? null,
-			language: movie.originalLanguage,
-		},
+		original
+			? {
+					...original,
+					latinName: original.latinName ?? null,
+					language: movie.originalLanguage,
+				}
+			: {
+					name: null,
+					latinName: null,
+					language: movie.originalLanguage,
+				},
 		translations,
 	);
 	if ("status" in show) return show;

api/src/controllers/seed/series.ts

@@ -91,20 +91,13 @@ export const seedSerie = async (
 	} = seed;
 	const nextRefresh = guessNextRefresh(serie.startAir ?? new Date());
 
-	const original = translations[serie.originalLanguage];
-	if (!original) {
-		return {
-			status: 422,
-			message: "No translation available in the original language.",
-		};
-	}
-
 	const col = await insertCollection(collection, {
 		kind: "serie",
 		nextRefresh,
 		...seed,
 	});
 
+	const original = translations[serie.originalLanguage];
 	const show = await insertShow(
 		{
 			kind: "serie",
@@ -113,11 +106,17 @@ export const seedSerie = async (
 			entriesCount: entries.length,
 			...serie,
 		},
-		{
-			...original,
-			latinName: original.latinName ?? null,
-			language: serie.originalLanguage,
-		},
+		original
+			? {
+					...original,
+					latinName: original.latinName ?? null,
+					language: serie.originalLanguage,
+				}
+			: {
+					name: null,
+					latinName: null,
+					language: serie.originalLanguage,
+				},
 		translations,
 	);
 	if ("status" in show) return show;

api/src/controllers/videos.ts

@@ -831,6 +831,9 @@ export const videosWriteH = new Elysia({ prefix: "/videos", tags: ["videos"] })
 	.post(
 		"",
 		async ({ body, status }) => {
+			if (body.length === 0) {
+				return status(422, { status: 422, message: "No videos" });
+			}
 			return await db.transaction(async (tx) => {
 				let vids: { pk: number; id: string; path: string; guess: Guess }[] = [];
 				try {
@@ -925,6 +928,7 @@ export const videosWriteH = new Elysia({ prefix: "/videos", tags: ["videos"] })
 					description:
 						"Invalid rendering specified. (conflicts with an existing video)",
 				},
+				422: KError,
 			},
 		},
 	)

api/src/db/schema/seasons.ts

@@ -91,7 +91,7 @@ export const seasonRelations = relations(seasons, ({ one, many }) => ({
 
 export const seasonTrRelations = relations(seasonTranslations, ({ one }) => ({
 	season: one(seasons, {
-		relationName: "season_translation",
+		relationName: "season_translations",
 		fields: [seasonTranslations.pk],
 		references: [seasons.pk],
 	}),

api/src/db/utils.ts

@@ -75,6 +75,10 @@ export function conflictUpdateAllExcept<
 
 // drizzle is bugged and doesn't allow js arrays to be used in raw sql.
 export function sqlarr(array: unknown[]): string {
+	function escapeStr(str: string) {
+		return str.replaceAll("\\", "\\\\").replaceAll('"', '\\"');
+	}
+
 	return `{${array
 		.map((item) =>
 			item === "null" || item === null || item === undefined
@@ -82,8 +86,8 @@ export function sqlarr(array: unknown[]): string {
 				: Array.isArray(item)
 					? sqlarr(item)
 					: typeof item === "object"
-						? `"${JSON.stringify(item).replaceAll("\\", "\\\\").replaceAll('"', '\\"')}"`
-						: `"${item?.toString().replaceAll('"', '\\"')}"`,
+						? `"${escapeStr(JSON.stringify(item))}"`
+						: `"${escapeStr(item.toString())}"`,
 		)
 		.join(", ")}}`;
 }

api/src/models/utils/original.ts

@@ -7,10 +7,12 @@ export const Original = t.Object({
 		description: "The language code this was made in.",
 		examples: ["ja"],
 	}),
-	name: t.String({
-		description: "The name in the original language",
-		examples: ["進撃の巨人"],
-	}),
+	name: t.Nullable(
+		t.String({
+			description: "The name in the original language",
+			examples: ["進撃の巨人"],
+		}),
+	),
 	latinName: t.Nullable(
 		t.String({
 			description: comment`

api/src/utils.ts

@@ -28,3 +28,13 @@ export function getFile(path: string): BunFile | S3File {
 
 	return Bun.file(path);
 }
+
+export function uniqBy<T>(a: T[], key: (val: T) => string) {
+	const seen: Record<string, boolean> = {};
+	return a.filter((item) => {
+		const k = key(item);
+		if (seen[k]) return false;
+		seen[k] = true;
+		return true;
+	});
+}

api/tests/series/get-staff.test.ts

@@ -6,9 +6,12 @@ import {
 	getStaffRoles,
 } from "tests/helpers";
 import { expectStatus } from "tests/utils";
+import { db } from "~/db";
+import { staff } from "~/db/schema";
 import { madeInAbyss } from "~/models/examples";
 
 beforeAll(async () => {
+	await db.delete(staff);
 	await createSerie(madeInAbyss);
 });
 

api/tests/series/seed-serie.test.ts

@@ -2,7 +2,7 @@ import { beforeAll, describe, expect, it } from "bun:test";
 import { eq } from "drizzle-orm";
 import { expectStatus } from "tests/utils";
 import { db } from "~/db";
-import { seasons, shows, videos } from "~/db/schema";
+import { entries, seasons, shows, videos } from "~/db/schema";
 import { madeInAbyss, madeInAbyssVideo } from "~/models/examples";
 import { createSerie } from "../helpers";
 
@@ -104,4 +104,61 @@ describe("Serie seeding", () => {
 			],
 		});
 	});
+
+	it("Can create a serie with quotes", async () => {
+		await db.delete(entries);
+		const [resp, body] = await createSerie({
+			...madeInAbyss,
+			slug: "quote-test",
+			seasons: [
+				{
+					...madeInAbyss.seasons[0],
+					translations: {
+						en: {
+							...madeInAbyss.seasons[0].translations.en,
+							name: "Season'1",
+						},
+					},
+				},
+				{
+					...madeInAbyss.seasons[1],
+					translations: {
+						en: {
+							...madeInAbyss.seasons[0].translations.en,
+							name: 'Season"2',
+							description: `This's """""quote, idk'''''`,
+						},
+					},
+				},
+			],
+		});
+
+		expectStatus(resp, body).toBe(201);
+		expect(body.id).toBeString();
+		expect(body.slug).toBe("quote-test");
+
+		const ret = await db.query.shows.findFirst({
+			where: eq(shows.id, body.id),
+			with: {
+				seasons: {
+					orderBy: seasons.seasonNumber,
+					with: { translations: true },
+				},
+				entries: {
+					with: {
+						translations: true,
+						evj: { with: { video: true } },
+					},
+				},
+			},
+		});
+
+		expect(ret).not.toBeNull();
+		expect(ret!.seasons).toBeArrayOfSize(2);
+		expect(ret!.seasons[0].translations[0].name).toBe("Season'1");
+		expect(ret!.seasons[1].translations[0].name).toBe('Season"2');
+		expect(ret!.entries).toBeArrayOfSize(
+			madeInAbyss.entries.length + madeInAbyss.extras.length,
+		);
+	});
 });

api/tsconfig.json

@@ -1,6 +1,6 @@
 {
 	"compilerOptions": {
-		"target": "ES2021",
+		"target": "ES2022",
 		"module": "ES2022",
 		"moduleResolution": "node",
 		"esModuleInterop": true,

auth/otel.go

@@ -88,7 +88,9 @@ func setupOtel(e *echo.Echo) (func(), error) {
 	otel.SetTracerProvider(tp)
 
 	e.Use(otelecho.Middleware("kyoo.auth", otelecho.WithSkipper(func(c echo.Context) bool {
-		return c.Path() == "/auth/health" || c.Path() == "/auth/ready"
+		return (c.Path() == "/auth/health" ||
+			c.Path() == "/auth/ready" ||
+			strings.HasPrefix(c.Path(), "/.well-known/"))
 	})))
 
 	return func() {

docker-compose.dev.yml

@@ -88,7 +88,7 @@ services:
     env_file:
       - ./.env
     volumes:
-      - images:/app/images
+      - images:/images
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.swagger.rule=PathPrefix(`/swagger`)"

docker-compose.yml

@@ -58,7 +58,7 @@ services:
     env_file:
       - ./.env
     volumes:
-      - images:/app/images
+      - images:/images
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.swagger.rule=PathPrefix(`/swagger`)"

front/src/ui/login/logic.tsx

@@ -84,6 +84,7 @@ export const login = async (
 export const logout = async () => {
 	const accounts = readAccounts();
 	const account = accounts.find((x) => x.selected);
+	removeAccounts((x) => x.selected);
 	if (account) {
 		await queryFn({
 			method: "DELETE",
@@ -92,7 +93,6 @@ export const logout = async () => {
 			parser: null,
 		});
 	}
-	removeAccounts((x) => x.selected);
 };
 
 export const deleteAccount = async () => {

scanner/migrations/000001_request.up.sql

@@ -18,6 +18,7 @@ create table scanner.requests(
 	external_id jsonb not null default '{}'::jsonb,
 	videos jsonb not null default '[]'::jsonb,
 	status scanner.request_status not null default 'pending',
+	error jsonb,
 	started_at timestamptz,
 	created_at timestamptz not null default now()::timestamptz,
 	constraint unique_kty unique nulls not distinct (kind, title, year)

scanner/scanner/__init__.py

@@ -26,6 +26,12 @@ async def lifespan(_):
 	):
 		# there's no way someone else used the same id, right?
 		is_master = await db.fetchval("select pg_try_advisory_lock(198347)")
+		is_http = not is_master and await db.fetchval(
+			"select pg_try_advisory_lock(645633)"
+		)
+		if is_http:
+			yield
+			return
 		if is_master:
 			await migrate()
 		processor = RequestProcessor(pool, client, tmdb)

scanner/scanner/client.py

@@ -3,7 +3,7 @@ from logging import getLogger
 from types import TracebackType
 from typing import Literal
 
-from aiohttp import ClientSession
+from aiohttp import ClientResponse, ClientResponseError, ClientSession
 from pydantic import TypeAdapter
 
 from .models.movie import Movie
@@ -38,9 +38,19 @@ class KyooClient(metaclass=Singleton):
 	):
 		await self._client.close()
 
+	async def raise_for_status(self, r: ClientResponse):
+		if r.status >= 400:
+			raise ClientResponseError(
+				r.request_info,
+				r.history,
+				status=r.status,
+				message=await r.text(),
+				headers=r.headers,
+			)
+
 	async def get_videos_info(self) -> VideoInfo:
 		async with self._client.get("videos") as r:
-			r.raise_for_status()
+			await self.raise_for_status(r)
 			return VideoInfo(**await r.json())
 
 	async def create_videos(self, videos: list[Video]) -> list[VideoCreated]:
@@ -48,7 +58,7 @@ class KyooClient(metaclass=Singleton):
 			"videos",
 			data=TypeAdapter(list[Video]).dump_json(videos, by_alias=True),
 		) as r:
-			r.raise_for_status()
+			await self.raise_for_status(r)
 			return TypeAdapter(list[VideoCreated]).validate_json(await r.text())
 
 	async def delete_videos(self, videos: list[str] | set[str]):
@@ -56,14 +66,14 @@ class KyooClient(metaclass=Singleton):
 			"videos",
 			data=TypeAdapter(list[str] | set[str]).dump_json(videos, by_alias=True),
 		) as r:
-			r.raise_for_status()
+			await self.raise_for_status(r)
 
 	async def create_movie(self, movie: Movie) -> Resource:
 		async with self._client.post(
 			"movies",
 			data=movie.model_dump_json(by_alias=True),
 		) as r:
-			r.raise_for_status()
+			await self.raise_for_status(r)
 			return Resource.model_validate(await r.json())
 
 	async def create_serie(self, serie: Serie) -> Resource:
@@ -71,7 +81,7 @@ class KyooClient(metaclass=Singleton):
 			"series",
 			data=serie.model_dump_json(by_alias=True),
 		) as r:
-			r.raise_for_status()
+			await self.raise_for_status(r)
 			return Resource.model_validate(await r.json())
 
 	async def link_videos(
@@ -100,4 +110,4 @@ class KyooClient(metaclass=Singleton):
 				by_alias=True,
 			),
 		) as r:
-			r.raise_for_status()
+			await self.raise_for_status(r)

scanner/scanner/models/request.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 from datetime import datetime
-from typing import Literal
+from typing import Any, Literal
 
 from pydantic import Field
 
@@ -31,4 +31,5 @@ class RequestRet(Model):
 		"running",
 		"failed",
 	]
+	error: dict[str, Any] | None
 	started_at: datetime | None

scanner/scanner/providers/provider.py

@@ -47,7 +47,7 @@ class Provider(ABC):
 		search = await self.search_movies(title, year, language=[])
 		if not any(search):
 			raise ProviderError(
-				f"Couldn't find a movie with title {title}. (year: {year}"
+				f"Couldn't find a movie with title {title}. (year: {year})"
 			)
 		ret = await self.get_movie(
 			{k: v.data_id for k, v in search[0].external_id.items()}
@@ -68,7 +68,7 @@ class Provider(ABC):
 		search = await self.search_series(title, year, language=[])
 		if not any(search):
 			raise ProviderError(
-				f"Couldn't find a serie with title {title}. (year: {year}"
+				f"Couldn't find a serie with title {title}. (year: {year})"
 			)
 		ret = await self.get_serie(
 			{k: v.data_id for k, v in search[0].external_id.items()}

scanner/scanner/providers/themoviedatabase.py

@@ -420,6 +420,8 @@ class TheMovieDatabase(Provider):
 					(x["episode_number"] for x in season["episodes"]), None
 				),
 				"entries_count": len(season["episodes"]),
+				# there can be gaps in episodes (like 1,2,5,6,7)
+				"episodes": [x["episode_number"] for x in season["episodes"]],
 			},
 		)
 
@@ -429,9 +431,9 @@ class TheMovieDatabase(Provider):
 		# TODO: batch those
 		ret = await asyncio.gather(
 			*[
-				self._get_entry(serie_id, s.season_number, s.extra["first_entry"] + e)
+				self._get_entry(serie_id, s.season_number, e)
 				for s in seasons
-				for e in range(0, s.extra["entries_count"])
+				for e in s.extra["episodes"]
 			]
 		)
 

scanner/scanner/requests.py

@@ -1,5 +1,6 @@
 from asyncio import CancelledError, Event, TaskGroup
 from logging import getLogger
+from traceback import TracebackException
 from typing import cast
 
 from asyncpg import Connection, Pool
@@ -40,6 +41,8 @@ class RequestCreator:
 			"""
 			delete from scanner.requests
 			where status = 'failed'
+				or (status = 'running'
+					and now() - started_at > interval '1 hour')
 			"""
 		)
 
@@ -161,11 +164,22 @@ class RequestProcessor:
 				update
 					scanner.requests
 				set
-					status = 'failed'
+					status = 'failed',
+					error = $2
 				where
 					pk = $1
 				""",
 				request.pk,
+				{
+					"title": type(e).__name__,
+					"message": str(e),
+					"traceback": [
+						line
+						for part in TracebackException.from_exception(e).format()
+						for line in part.split("\n")
+						if line.strip()
+					],
+				},
 			)
 		return True
 

scanner/scanner/status.py

@@ -28,6 +28,7 @@ class StatusService:
 				title,
 				year,
 				status,
+				error,
 				started_at
 			from
 				scanner.requests

shell.nix

@@ -16,6 +16,9 @@ pkgs.mkShell {
 
   # env vars aren't inherited from the `inputsFrom`
   SHARP_FORCE_GLOBAL_LIBVIPS = 1;
+  shellHook = ''
+    export LD_LIBRARY_PATH=${pkgs.stdenv.cc.cc.lib}/lib:$LD_LIBRARY_PATH
+  '';
   UV_PYTHON_PREFERENCE = "only-system";
   UV_PYTHON = pkgs.python313;
 }