Handle crew & fix some themoviedb bugs

api/drizzle/0021_crew.sql

@@ -0,0 +1 @@
+ALTER TYPE "kyoo"."role_kind" ADD VALUE 'crew' BEFORE 'other';

api/drizzle/meta/_journal.json

@@ -148,6 +148,13 @@
 			"when": 1746198322219,
 			"tag": "0020_video_unique",
 			"breakpoints": true
+		},
+		{
+			"idx": 21,
+			"version": "7",
+			"when": 1747727831649,
+			"tag": "0021_crew",
+			"breakpoints": true
 		}
 	]
 }

api/src/db/schema/staff.ts

@@ -19,6 +19,7 @@ export const roleKind = schema.enum("role_kind", [
 	"writter",
 	"producer",
 	"music",
+	"crew",
 	"other",
 ]);
 

api/src/models/staff.ts

@@ -16,6 +16,7 @@ export const Role = t.Object({
 		"writter",
 		"producer",
 		"music",
+		"crew",
 		"other",
 	]),
 	character: t.Nullable(Character),

docker-compose.dev.yml

@@ -124,7 +124,7 @@ services:
       - "traefik.http.middlewares.phantom-token.forwardauth.address=http://auth:4568/auth/jwt"
       - "traefik.http.middlewares.phantom-token.forwardauth.authRequestHeaders=Authorization,X-Api-Key"
       - "traefik.http.middlewares.phantom-token.forwardauth.authResponseHeaders=Authorization"
-    command: fastapi dev scanner --host 0.0.0.0 --port 3489
+    command: fastapi dev scanner --host 0.0.0.0 --port 4389
     develop:
       watch:
         - action: sync

scanner/old/implementations/thetvdb.py

@@ -7,6 +7,7 @@ from typing import Optional, Any, Callable, OrderedDict
 from langcodes import Language
 
 from matcher.cache import cache
+from scanner.models.staff import Role
 
 from ..provider import Provider, ProviderError
 from ..utils import normalize_lang
@@ -75,6 +76,19 @@ class TVDB(Provider):
 			"martial-arts": None,
 			"awards-show": None,
 		}
+		self._roles_map = {
+			"Actor": Role.ACTOR,
+			"Creator": Role.OTHER,
+			"Crew": Role.CREW,
+			"Director": Role.DIRECTOR,
+			"Executive Producer": Role.OTHER,
+			"Guest Star": Role.OTHER,
+			"Host": Role.OTHER,
+			"Musical Guest": Role.MUSIC,
+			"Producer": Role.PRODUCER,
+			"Showrunner": Role.OTHER,
+			"Writer": Role.WRITTER,
+		}
 
 	@cache(ttl=timedelta(days=30))
 	async def login(self) -> str:

scanner/scanner/client.py

@@ -58,6 +58,7 @@ class KyooClient(metaclass=Singleton):
 			r.raise_for_status()
 
 	async def create_movie(self, movie: Movie) -> Resource:
+		logger.debug("sending movie %s", movie.model_dump_json())
 		async with self._client.post(
 			"movies",
 			json=movie.model_dump_json(),
@@ -66,6 +67,7 @@ class KyooClient(metaclass=Singleton):
 			return Resource(**await r.json())
 
 	async def create_serie(self, serie: Serie) -> Resource:
+		logger.debug("sending serie %s", serie.model_dump_json())
 		async with self._client.post(
 			"series",
 			json=serie.model_dump_json(),

scanner/scanner/models/staff.py

@@ -12,6 +12,7 @@ class Role(StrEnum):
 	WRITTER = "writter"
 	PRODUCER = "producer"
 	MUSIC = "music"
+	CREW = "crew"
 	OTHER = "other"
 
 

scanner/scanner/providers/themoviedatabase.py

@@ -69,6 +69,20 @@ class TheMovieDatabase(Provider):
 			10767: Genre.TALK,
 			10768: [Genre.WAR, Genre.POLITICS],
 		}
+		self._roles_map = {
+			"Camera": Role.OTHER,
+			"Costume & Make-Up": Role.OTHER,
+			"Lighting": Role.OTHER,
+			"Art": Role.OTHER,
+			"Visual Effects": Role.OTHER,
+			"Crew": Role.CREW,
+			"Writing": Role.WRITTER,
+			"Production": Role.PRODUCER,
+			"Editing": Role.OTHER,
+			"Directing": Role.DIRECTOR,
+			"Sound": Role.MUSIC,
+			"Actors": Role.ACTOR,
+		}
 
 	async def __aenter__(self):
 		return self
@@ -169,7 +183,7 @@ class TheMovieDatabase(Provider):
 				Language.get(
 					f"{trans['iso_639_1']}-{trans['iso_3166_1']}"
 				): MovieTranslation(
-					name=clean(trans["data"]["title"])
+					name=clean(trans["data"]["name"])
 					or (
 						clean(movie["original_title"])
 						if movie["original_language"] == trans["iso_639_1"]
@@ -314,17 +328,17 @@ class TheMovieDatabase(Provider):
 				Language.get(
 					f"{trans['iso_639_1']}-{trans['iso_3166_1']}"
 				): SerieTranslation(
-					name=clean(trans["data"]["title"])
+					name=clean(trans["data"]["name"])
 					or (
-						clean(serie["original_title"])
+						clean(serie["original_name"])
 						if serie["original_language"] == trans["iso_639_1"]
 						else None
 					)
-					or serie["title"],
+					or serie["name"],
 					latin_name=next(
 						(
 							x["title"]
-							for x in serie["alternative_titles"]["titles"]
+							for x in serie["alternative_titles"]["results"]
 							if x["iso_3166_1"] == trans["iso_3166_1"]
 							and x["type"] == "Romaji"
 						),
@@ -334,10 +348,10 @@ class TheMovieDatabase(Provider):
 					tagline=clean(trans["data"]["tagline"]),
 					aliases=[
 						x["title"]
-						for x in serie["alternative_titles"]["titles"]
+						for x in serie["alternative_titles"]["results"]
 						if x["iso_3166_1"] == trans["iso_3166_1"]
 					],
-					tags=[x["name"] for x in serie["keywords"]["keywords"]],
+					tags=[x["name"] for x in serie["keywords"]["results"]],
 					poster=self._pick_image(serie, trans["iso_639_1"], "posters"),
 					logo=self._pick_image(serie, trans["iso_639_1"], "logos"),
 					banner=None,
@@ -366,7 +380,7 @@ class TheMovieDatabase(Provider):
 			staff=[self._map_staff(x) for x in serie["credits"]["cast"]],
 		)
 
-	async def _get_season(self, serie_id: str, season_number: int) -> Season:
+	async def _get_season(self, serie_id: str | int, season_number: int) -> Season:
 		season = await self._get(
 			f"tv/{serie_id}/season/{season_number}",
 			params={
@@ -383,7 +397,7 @@ class TheMovieDatabase(Provider):
 			end_air=None,
 			external_id={
 				self.name: SeasonId(
-					serie_id=serie_id,
+					serie_id=str(serie_id),
 					season=season["season_number"],
 					link=f"https://www.themoviedb.org/tv/{serie_id}/season/{season['season_number']}",
 				)
@@ -403,7 +417,7 @@ class TheMovieDatabase(Provider):
 		)
 
 	async def _get_all_entries(
-		self, serie_id: str, seasons: list[dict[str, Any]]
+		self, serie_id: str | int, seasons: list[dict[str, Any]]
 	) -> list[Entry]:
 		# TODO: batch those
 		ret = await asyncio.gather(
@@ -496,7 +510,7 @@ class TheMovieDatabase(Provider):
 
 		return ret
 
-	async def _get_entry(self, serie_id: str, season: int, episode_nbr: int) -> Entry:
+	async def _get_entry(self, serie_id: str | int, season: int, episode_nbr: int) -> Entry:
 		episode = await self._get(
 			f"tv/{serie_id}/season/{season}/episode/{episode_nbr}",
 			params={
@@ -519,7 +533,7 @@ class TheMovieDatabase(Provider):
 			number=episode["episode_number"],
 			external_id={
 				self.name: EpisodeId(
-					serie_id=serie_id,
+					serie_id=str(serie_id),
 					season=episode["season_number"],
 					episode=episode["episode_number"],
 					link=f"https://www.themoviedb.org/tv/{serie_id}/season/{episode['season_number']}/episode/{episode['episode_number']}",
@@ -538,7 +552,7 @@ class TheMovieDatabase(Provider):
 			},
 		)
 
-	async def _get_collection(self, provider_id: str) -> Collection:
+	async def _get_collection(self, provider_id: str | int) -> Collection:
 		collection = await self._get(
 			f"collection/{provider_id}",
 			params={
@@ -567,7 +581,7 @@ class TheMovieDatabase(Provider):
 				Language.get(
 					f"{trans['iso_639_1']}-{trans['iso_3166_1']}"
 				): CollectionTranslation(
-					name=clean(trans["data"]["title"]) or collection["title"],
+					name=clean(trans["data"]["name"]) or collection["name"],
 					latin_name=None,
 					description=trans["overview"],
 					tagline=None,
@@ -663,8 +677,7 @@ class TheMovieDatabase(Provider):
 
 	def _map_staff(self, person: dict[str, Any]) -> Staff:
 		return Staff(
-			# TODO: map those to Role (see https://developer.themoviedb.org/reference/configuration-jobs for list)
+			kind=self._roles_map.get(person["known_for_department"], Role.OTHER),
-			kind=person["known_for_department"],
 			character=Character(
 				name=person["character"],
 				latin_name=None,

scanner/scanner/requests.py

@@ -77,29 +77,25 @@ class RequestProcessor:
 			_ = tg.create_task(self.process_all())
 
 		def terminated(*_):
-			logger.info("terminated")
 			closed.set()
 
 		while True:
 			closed.clear()
-			logger.info("aquire")
+			# TODO: unsure if timeout actually work, i think not...
-			try:
+			async with self._pool.acquire(timeout=10) as db:
-				async with self._pool.acquire(timeout=10) as db:
+				try:
-					try:
+					self._database = cast(Connection, db)
-						self._database = cast(Connection, db)
+					self._database.add_termination_listener(terminated)
-						self._database.add_termination_listener(terminated)
+					await self._database.add_listener("scanner_requests", process)
-						await self._database.add_listener("scanner_requests", process)
 
-						logger.info("Listening for requestes")
+					logger.info("Listening for requestes")
-						_ = await closed.wait()
+					_ = await closed.wait()
-						logger.info("stopping...")
+					logger.info("stopping...")
-					except CancelledError as e:
+				except CancelledError as e:
-						logger.info("Stopped listening for requsets")
+					logger.info("Stopped listening for requsets")
-						await self._database.remove_listener("scanner_requests", process)
+					await self._database.remove_listener("scanner_requests", process)
-						self._database.remove_termination_listener(terminated)
+					self._database.remove_termination_listener(terminated)
-						raise
+					raise
-			except TimeoutError:
-				logger.info("temiout")
 
 	async def process_all(self):
 		found = True