From 1c483fa14fef0c9f7d659c604374ec6256ddc46b Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Tue, 7 May 2024 01:05:09 +0200 Subject: [PATCH 01/20] wip: Add tvdb as a provider --- scanner/providers/implementations/thetvdb.py | 88 ++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 scanner/providers/implementations/thetvdb.py diff --git a/scanner/providers/implementations/thetvdb.py b/scanner/providers/implementations/thetvdb.py new file mode 100644 index 00000000..e073e573 --- /dev/null +++ b/scanner/providers/implementations/thetvdb.py @@ -0,0 +1,88 @@ +from datetime import timedelta +from aiohttp import ClientSession +from logging import getLogger +from typing import Optional, Any + +from matcher.cache import cache + +from ..provider import Provider, ProviderError +from ..types.movie import Movie, MovieTranslation, Status as MovieStatus +from ..types.season import Season, SeasonTranslation +from ..types.episode import Episode, EpisodeTranslation, PartialShow, EpisodeID +from ..types.studio import Studio +from ..types.genre import Genre +from ..types.metadataid import MetadataID +from ..types.show import Show, ShowTranslation, Status as ShowStatus +from ..types.collection import Collection, CollectionTranslation + +logger = getLogger(__name__) + + +class TVDB(Provider): + def __init__( + self, + client: ClientSession, + api_key: str, + pin: str, + languages: list[str], + ) -> None: + super().__init__() + self._client = client + self.base = "https://api4.thetvdb.com/v4/" + self._api_key = api_key + self._pin = pin + self._languages = languages + + @cache(ttl=timedelta(days=30)) + async def login(self) -> str: + async with self._client.post( + f"{self.base}/login", + json={"apikey": self._api_key, "pin": self._pin}, + ) as r: + r.raise_for_status() + ret = await r.json() + return ret["data"]["token"] + + async def get( + self, + path: str, + *, + params: dict[str, Any] = {}, + not_found_fail: Optional[str] = None, + ): + token = await self.login() + params = {k: v for k, v in params.items() if v is not None} + async with self._client.get( + f"{self.base}/{path}", + params={"api_key": self._api_key, **params}, + headers={"Authorization": f"Bearer {token}"}, + ) as r: + if not_found_fail and r.status == 404: + raise ProviderError(not_found_fail) + r.raise_for_status() + return await r.json() + + @property + def name(self) -> str: + return "tvdb" + + async def search_episode( + self, + name: str, + season: Optional[int], + episode_nbr: Optional[int], + absolute: Optional[int], + year: Optional[int], + ) -> Episode: + show = await self.search_show(name, year) + show_id = show.external_id[self.name].data_id + return await self.identify_episode(show_id, season, episode_nbr, absolute) + + async def identify_episode( + self, + show_id: str, + season: Optional[int], + episode_nbr: Optional[int], + absolute: Optional[int], + ) -> Episode: + return await self.get(f"") From a6b6067b0b443a686b80044ed034b979b5810d21 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Wed, 8 May 2024 15:37:49 +0200 Subject: [PATCH 02/20] Add tvdb episode identify --- scanner/matcher/matcher.py | 2 +- scanner/providers/implementations/thetvdb.py | 115 ++++++++++++++++++- 2 files changed, 111 insertions(+), 6 deletions(-) diff --git a/scanner/matcher/matcher.py b/scanner/matcher/matcher.py index 808fb7fa..a61a7393 100644 --- a/scanner/matcher/matcher.py +++ b/scanner/matcher/matcher.py @@ -111,7 +111,7 @@ class Matcher: episode = await self._provider.search_episode( title, season=season, - episode_nbr=episode_nbr, + episode_nbr=episode_nbr if season is not None else None, absolute=episode_nbr if season is None else None, year=year, ) diff --git a/scanner/providers/implementations/thetvdb.py b/scanner/providers/implementations/thetvdb.py index e073e573..67316297 100644 --- a/scanner/providers/implementations/thetvdb.py +++ b/scanner/providers/implementations/thetvdb.py @@ -1,7 +1,8 @@ -from datetime import timedelta +import asyncio +from datetime import timedelta, datetime from aiohttp import ClientSession from logging import getLogger -from typing import Optional, Any +from typing import Optional, Any, Literal from matcher.cache import cache @@ -33,6 +34,12 @@ class TVDB(Provider): self._pin = pin self._languages = languages + def two_to_three_lang(self, lang: str) -> str: + return lang + + def three_to_two_lang(self, lang: str) -> str: + return lang + @cache(ttl=timedelta(days=30)) async def login(self) -> str: async with self._client.post( @@ -45,15 +52,16 @@ class TVDB(Provider): async def get( self, - path: str, + path: Optional[str] = None, *, + fullPath: Optional[str] = None, params: dict[str, Any] = {}, not_found_fail: Optional[str] = None, ): token = await self.login() params = {k: v for k, v in params.items() if v is not None} async with self._client.get( - f"{self.base}/{path}", + fullPath or f"{self.base}/{path}", params={"api_key": self._api_key, **params}, headers={"Authorization": f"Bearer {token}"}, ) as r: @@ -66,6 +74,30 @@ class TVDB(Provider): def name(self) -> str: return "tvdb" + async def search_show(self, name: str, year: Optional[int]) -> Show: + pass + + @cache(ttl=timedelta(days=1)) + async def get_episodes( + self, + show_id: str, + order: Literal["default", "absolute"], + language: Optional[str] = None, + ): + path = f"/series/{show_id}/episodes/{order}" + if language is not None: + path += f"/{language}" + ret = await self.get( + path, not_found_fail=f"Could not find show with id {show_id}" + ) + episodes = ret["data"]["episodes"] + next = ret["links"]["next"] + while next != None: + ret = await self.get(fullPath=next) + next = ret["links"]["next"] + episodes += ret["data"] + return episodes + async def search_episode( self, name: str, @@ -85,4 +117,77 @@ class TVDB(Provider): episode_nbr: Optional[int], absolute: Optional[int], ) -> Episode: - return await self.get(f"") + flang, slang, *olang = [*self._languages, None] + episodes = await self.get_episodes(show_id, order="default", language=flang) + show = episodes["data"] + ret = next( + filter( + (lambda x: x["seasonNumber"] == 1 and x["number"] == absolute) + if absolute is not None + else ( + lambda x: x["seasonNumber"] == season and x["number"] == episode_nbr + ), + episodes["episodes"], + ), + None, + ) + if ret == None: + raise ProviderError( + f"Could not retrive episode {show['name']} s{season}e{episode_nbr}, absolute {absolute}" + ) + absolutes = await self.get_episodes( + show_id, order="absolute", language=slang or flang + ) + abs = next(filter(lambda x: x["id"] == ret["id"], absolutes["episodes"])) + + otrans = await asyncio.gather( + *( + self.get_episodes(show_id, order="default", language=lang) + for lang in olang + if lang is not None + ) + ) + translations = { + lang: EpisodeTranslation( + name=val["name"], + overview=val["overview"], + ) + for (lang, val) in zip( + self._languages, + [ + ret, + abs, + *( + next(x for x in e["episodes"] if x["id"] == ret["id"]) + for e in otrans + ), + ], + ) + } + + return Episode( + show=PartialShow( + name=show["name"], + original_language=self.three_to_two_lang(show["originalLanguage"]), + external_id={ + self.name: MetadataID( + show_id, f"https://thetvdb.com/series/{show['slug']}" + ), + }, + ), + season_number=ret["seasonNumber"], + episode_number=ret["number"], + absolute_number=abs["number"], + runtime=ret["runtime"], + release_date=datetime.strptime(ret["aired"], "%Y-%m-%d").date(), + thumbnail=f"https://artworks.thetvdb.com{ret['image']}", + external_id={ + self.name: EpisodeID( + show_id, + ret["seasonNumber"], + ret["number"], + f"https://thetvdb.com/series/{show_id}/episodes/{ret['id']}", + ), + }, + translations=translations, + ) From dde38381e2aea59fd8eb6d34f84807809c706a3f Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Thu, 9 May 2024 03:01:28 +0200 Subject: [PATCH 03/20] Add tvdb show identify --- scanner/providers/implementations/thetvdb.py | 112 ++++++++++++++++++- scanner/providers/types/show.py | 2 +- 2 files changed, 111 insertions(+), 3 deletions(-) diff --git a/scanner/providers/implementations/thetvdb.py b/scanner/providers/implementations/thetvdb.py index 67316297..039b017c 100644 --- a/scanner/providers/implementations/thetvdb.py +++ b/scanner/providers/implementations/thetvdb.py @@ -1,8 +1,9 @@ import asyncio from datetime import timedelta, datetime +from math import e from aiohttp import ClientSession from logging import getLogger -from typing import Optional, Any, Literal +from typing import Optional, Any, Literal, Callable from matcher.cache import cache @@ -33,6 +34,43 @@ class TVDB(Provider): self._api_key = api_key self._pin = pin self._languages = languages + self._genre_map = { + "soap": Genre.SOAP, + "science-fiction": Genre.SCIENCE_FICTION, + "reality": Genre.REALITY, + "news": Genre.NEWS, + "mini-series": None, + "horror": Genre.HORROR, + "home-and-garden": None, + "game-show": None, + "food": None, + "fantasy": Genre.FANTASY, + "family": Genre.FAMILY, + "drama": Genre.DRAMA, + "documentary": Genre.DOCUMENTARY, + "crime": Genre.CRIME, + "comedy": Genre.COMEDY, + "children": Genre.KIDS, + "animation": Genre.ANIMATION, + "adventure": Genre.ADVENTURE, + "action": Genre.ACTION, + "sport": None, + "suspense": None, + "talk-show": Genre.TALK, + "thriller": Genre.THRILLER, + "travel": None, + "western": Genre.WESTERN, + "anime": Genre.ANIMATION, + "romance": Genre.ROMANCE, + "musical": Genre.MUSIC, + "podcast": None, + "mystery": Genre.MYSTERY, + "indie": None, + "history": Genre.HISTORY, + "war": Genre.WAR, + "martial-arts": None, + "awards-show": None, + } def two_to_three_lang(self, lang: str) -> str: return lang @@ -75,7 +113,8 @@ class TVDB(Provider): return "tvdb" async def search_show(self, name: str, year: Optional[int]) -> Show: - pass + show_id = "" + return await self.identify_show(show_id) @cache(ttl=timedelta(days=1)) async def get_episodes( @@ -191,3 +230,72 @@ class TVDB(Provider): }, translations=translations, ) + + async def identify_show(self, show_id: str) -> Show: + ret = await self.get( + f"series/{show_id}/extended", + not_found_fail=f"Could not find show with id {show_id}", + ) + translations = await asyncio.gather( + *( + self.get(f"/series/{show_id}/translations/{lang}") + for lang in self._languages + if lang != ret["original_language"] + ) + ) + return Show( + original_language=ret["originalLanguage"], + aliases=[], + start_air=datetime.strptime(ret["firstAired"], "%Y-%m-%d").date(), + end_air=datetime.strptime(ret["lastAired"], "%Y-%m-%d").date(), + status=ShowStatus.FINISHED + if ret["status"]["name"] == "Ended" + else ShowStatus.AIRING + if ret["status"]["name"] == "Continuing" + else ShowStatus.PLANNED, + rating=None, + studios=[ + Studio( + name=x["name"], + logos=[], + external_id={ + self.name: MetadataID( + x["id"], f"https://thetvdb.com/companies/{x['slug']}" + ) + }, + ) + for x in ret["companies"] + if x["companyType"]["companyTypeName"] == "Studio" + ], + genres=[ + self._genre_map[x["slug"]] + for x in ret["genres"] + if self._genre_map[x["slug"]] is not None + ], + external_id={ + self.name: MetadataID( + ret["id"], f"https://thetvdb.com/series/{ret['slug']}" + ), + } + | self.process_remote_id( + ret["remoteIds"], + "themoviedatabase", + lambda x: f"https://www.themoviedb.org/tv/{x}", + "TheMovieDB.com", + ) + | self.process_remote_id( + ret["remoteIds"], + "imdb", + lambda x: f"https://www.imdb.com/title/{x}", + "IMDB", + ), + seasons=[], + ) + + def process_remote_id( + self, ids: dict, name: str, link: Callable[[str], str], tvdb_name: str + ) -> dict: + id = next((x["id"] for x in ids if x["sourceName"] == tvdb_name), None) + if id is None: + return {} + return {name: MetadataID(id, link(id))} diff --git a/scanner/providers/types/show.py b/scanner/providers/types/show.py index b66bd466..4d82a0e6 100644 --- a/scanner/providers/types/show.py +++ b/scanner/providers/types/show.py @@ -38,7 +38,7 @@ class Show: start_air: Optional[date | int] end_air: Optional[date | int] status: Status - rating: int + rating: Optional[int] studios: list[Studio] genres: list[Genre] seasons: list[Season] From 740703a845e20024ab3675ee959c9d1ea95efdbc Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Thu, 9 May 2024 03:21:37 +0200 Subject: [PATCH 04/20] Add tvdb show translations --- scanner/providers/implementations/thetvdb.py | 34 +++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/scanner/providers/implementations/thetvdb.py b/scanner/providers/implementations/thetvdb.py index 039b017c..fd2f78a9 100644 --- a/scanner/providers/implementations/thetvdb.py +++ b/scanner/providers/implementations/thetvdb.py @@ -240,9 +240,40 @@ class TVDB(Provider): *( self.get(f"/series/{show_id}/translations/{lang}") for lang in self._languages - if lang != ret["original_language"] + if lang != ret["originalLanguage"] ) ) + trans = { + lang: ShowTranslation( + name=x["name"], + tagline=None, + tags=[], + overview=x["overview"], + posters=[ + i["image"] + for i in x["artworks"] + if i["type"] == 2 + and (i["language"] == lang or i["language"] is None) + ], + logos=[ + i["image"] + for i in x["artworks"] + if i["type"] == 5 + and (i["language"] == lang or i["language"] is None) + ], + thumbnails=[ + i["image"] + for i in x["artworks"] + if i["type"] == 3 + and (i["language"] == lang or i["language"] is None) + ], + trailers=[x["url"] for t in ret["trailers"] if t["language"] == lang], + ) + for (lang, x) in [ + (ret["originalLanguage"], ret), + *zip(self._languages, translations), + ] + } return Show( original_language=ret["originalLanguage"], aliases=[], @@ -289,6 +320,7 @@ class TVDB(Provider): lambda x: f"https://www.imdb.com/title/{x}", "IMDB", ), + translations=trans, seasons=[], ) From 3d2e8370228a43df804f721ace1ee674d0d6ce76 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Thu, 9 May 2024 03:53:05 +0200 Subject: [PATCH 05/20] Add tvdb show aliases --- scanner/providers/implementations/thetvdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scanner/providers/implementations/thetvdb.py b/scanner/providers/implementations/thetvdb.py index fd2f78a9..fb106374 100644 --- a/scanner/providers/implementations/thetvdb.py +++ b/scanner/providers/implementations/thetvdb.py @@ -276,7 +276,7 @@ class TVDB(Provider): } return Show( original_language=ret["originalLanguage"], - aliases=[], + aliases=[x["name"] for x in ret["aliases"]], start_air=datetime.strptime(ret["firstAired"], "%Y-%m-%d").date(), end_air=datetime.strptime(ret["lastAired"], "%Y-%m-%d").date(), status=ShowStatus.FINISHED From c96a5440218fd2e2bd5ed30d697e35f84aa8c69a Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 12 May 2024 21:36:01 +0200 Subject: [PATCH 06/20] Use new absoluteNumber value from tvdb --- scanner/providers/implementations/thetvdb.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/scanner/providers/implementations/thetvdb.py b/scanner/providers/implementations/thetvdb.py index fb106374..0a0fb1f2 100644 --- a/scanner/providers/implementations/thetvdb.py +++ b/scanner/providers/implementations/thetvdb.py @@ -120,10 +120,9 @@ class TVDB(Provider): async def get_episodes( self, show_id: str, - order: Literal["default", "absolute"], language: Optional[str] = None, ): - path = f"/series/{show_id}/episodes/{order}" + path = f"/series/{show_id}/episodes/default" if language is not None: path += f"/{language}" ret = await self.get( @@ -156,8 +155,8 @@ class TVDB(Provider): episode_nbr: Optional[int], absolute: Optional[int], ) -> Episode: - flang, slang, *olang = [*self._languages, None] - episodes = await self.get_episodes(show_id, order="default", language=flang) + flang, *olang = self._languages + episodes = await self.get_episodes(show_id, language=flang) show = episodes["data"] ret = next( filter( @@ -174,17 +173,9 @@ class TVDB(Provider): raise ProviderError( f"Could not retrive episode {show['name']} s{season}e{episode_nbr}, absolute {absolute}" ) - absolutes = await self.get_episodes( - show_id, order="absolute", language=slang or flang - ) - abs = next(filter(lambda x: x["id"] == ret["id"], absolutes["episodes"])) otrans = await asyncio.gather( - *( - self.get_episodes(show_id, order="default", language=lang) - for lang in olang - if lang is not None - ) + *(self.get_episodes(show_id, language=lang) for lang in olang) ) translations = { lang: EpisodeTranslation( @@ -195,7 +186,6 @@ class TVDB(Provider): self._languages, [ ret, - abs, *( next(x for x in e["episodes"] if x["id"] == ret["id"]) for e in otrans @@ -216,7 +206,7 @@ class TVDB(Provider): ), season_number=ret["seasonNumber"], episode_number=ret["number"], - absolute_number=abs["number"], + absolute_number=ret["absoluteNumber"], runtime=ret["runtime"], release_date=datetime.strptime(ret["aired"], "%Y-%m-%d").date(), thumbnail=f"https://artworks.thetvdb.com{ret['image']}", From a29cfc0123b70520a230adeafd010ecaec1f288a Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 12 May 2024 22:41:20 +0200 Subject: [PATCH 07/20] Add seasons to tvdb --- scanner/providers/implementations/thetvdb.py | 47 +++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/scanner/providers/implementations/thetvdb.py b/scanner/providers/implementations/thetvdb.py index 0a0fb1f2..9d08f866 100644 --- a/scanner/providers/implementations/thetvdb.py +++ b/scanner/providers/implementations/thetvdb.py @@ -311,7 +311,9 @@ class TVDB(Provider): "IMDB", ), translations=trans, - seasons=[], + seasons=await asyncio.gather( + *(self.identify_season(x["id"], x["number"]) for x in ret["seasons"]) + ), ) def process_remote_id( @@ -321,3 +323,46 @@ class TVDB(Provider): if id is None: return {} return {name: MetadataID(id, link(id))} + + async def identify_season(self, show_id: str, season: int) -> Season: + """ + for tvdb, we don't save show_id but the season_id so we don't need to read `season` + """ + season_id = show_id + info = await self.get( + f"seasons/{season_id}/extended", + not_found_fail=f"Invalid season id {season_id}", + ) + + async def process_translation(lang: str) -> SeasonTranslation: + data = await self.get(f"seasons/{season_id}/translations/{lang}") + return SeasonTranslation( + name=data["data"]["name"], + overview=data["data"]["overview"], + posters=[ + i["image"] + for i in data["data"]["artworks"] + if i["type"] == 7 + and (i["language"] == lang or i["language"] is None) + ], + thumbnails=[ + i["image"] + for i in data["data"]["artworks"] + if i["type"] == 8 + and (i["language"] == lang or i["language"] is None) + ], + ) + + trans = await asyncio.gather(*(process_translation(x) for x in self._languages)) + translations = {lang: tl for lang, tl in zip(self._languages, trans)} + + return Season( + season_number=info["data"]["number"], + episodes_count=len(info["data"]["episodes"]), + start_air=min(x["aired"] for x in info["data"]["episodes"]), + end_air=max(x["aired"] for x in info["data"]["episodes"]), + external_id={ + self.name: MetadataID(season_id, None), + }, + translations=translations, + ) From 5f61f4d3a347727f8f96e6b814a7befcaa976b96 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 12 May 2024 23:05:32 +0200 Subject: [PATCH 08/20] Add tvdb search --- scanner/providers/implementations/thetvdb.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/scanner/providers/implementations/thetvdb.py b/scanner/providers/implementations/thetvdb.py index 9d08f866..cd163e66 100644 --- a/scanner/providers/implementations/thetvdb.py +++ b/scanner/providers/implementations/thetvdb.py @@ -1,21 +1,19 @@ import asyncio from datetime import timedelta, datetime -from math import e +from urllib.parse import urlencode from aiohttp import ClientSession from logging import getLogger -from typing import Optional, Any, Literal, Callable +from typing import Optional, Any, Callable, OrderedDict from matcher.cache import cache from ..provider import Provider, ProviderError -from ..types.movie import Movie, MovieTranslation, Status as MovieStatus from ..types.season import Season, SeasonTranslation from ..types.episode import Episode, EpisodeTranslation, PartialShow, EpisodeID from ..types.studio import Studio from ..types.genre import Genre from ..types.metadataid import MetadataID from ..types.show import Show, ShowTranslation, Status as ShowStatus -from ..types.collection import Collection, CollectionTranslation logger = getLogger(__name__) @@ -112,9 +110,14 @@ class TVDB(Provider): def name(self) -> str: return "tvdb" - async def search_show(self, name: str, year: Optional[int]) -> Show: - show_id = "" - return await self.identify_show(show_id) + async def search_show(self, name: str, year: Optional[int]) -> str: + query = OrderedDict( + query=name, + year=year, + type="series", + ) + ret = await self.get(f"search?{urlencode(query)}") + return ret["data"][0]["tvdb_id"] @cache(ttl=timedelta(days=1)) async def get_episodes( @@ -144,8 +147,7 @@ class TVDB(Provider): absolute: Optional[int], year: Optional[int], ) -> Episode: - show = await self.search_show(name, year) - show_id = show.external_id[self.name].data_id + show_id = await self.search_show(name, year) return await self.identify_episode(show_id, season, episode_nbr, absolute) async def identify_episode( From d08a86a72436d5f088162124b401b31528670553 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 12 May 2024 23:40:21 +0200 Subject: [PATCH 09/20] Add tvdb api keys config --- .env.example | 9 +++++++-- scanner/providers/provider.py | 12 ++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index fbd997ee..2298b20a 100644 --- a/.env.example +++ b/.env.example @@ -37,9 +37,14 @@ GOCODER_PRESET=fast # You can input multiple api keys separated by a , KYOO_APIKEYS=t7H5!@4iMNsAaSJQ49pat4jprJgTcF656if#J3 -# Keep this empty to use kyoo's default api key. You can also specify a custom API key if you want. -# To get one, go to https://www.themoviedb.org/settings/api and copy the api key (not the read access token, the api key) +# Keep those empty to use kyoo's default api key. You can also specify a custom API key if you want. +# go to https://www.themoviedb.org/settings/api and copy the api key (not the read access token, the api key) THEMOVIEDB_APIKEY= +# go to https://thetvdb.com/api-information/signup and copy the api key +TVDB_APIKEY= +# you can also input your subscriber's pin to support TVDB +TVDB_PIN= + # The url you can use to reach your kyoo instance. This is used during oidc to redirect users to your instance. PUBLIC_URL=http://localhost:5000 diff --git a/scanner/providers/provider.py b/scanner/providers/provider.py index b6d1c8da..535de39a 100644 --- a/scanner/providers/provider.py +++ b/scanner/providers/provider.py @@ -1,3 +1,4 @@ +from logging import getLogger import os from aiohttp import ClientSession from abc import abstractmethod, abstractproperty @@ -11,6 +12,8 @@ from .types.episode import Episode from .types.movie import Movie from .types.collection import Collection +logger = getLogger(__name__) + class Provider: @classmethod @@ -29,6 +32,14 @@ class Provider: tmdb = TheMovieDatabase(languages, client, tmdb) providers.append(tmdb) + from providers.implementations.thetvdb import TVDB + + tvdb = os.environ.get("TVDB_APIKEY") or TVDB.DEFAULT_API_KEY + if tvdb != "disabled": + pin = os.environ.get("TVDB_PIN") or None + tvdb = TVDB(client, tvdb, pin, languages) + providers.append(tvdb) + if not any(providers): raise ProviderError( "No provider configured. You probably forgot to specify an API Key" @@ -37,6 +48,7 @@ class Provider: from providers.implementations.thexem import TheXem provider = next(iter(providers)) + logger.info(f"Starting with provider: {provider.name}") return TheXem(client, provider) @abstractproperty From 497ba48f1b5cebf2c9d4813bca960b8ea4040645 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 12 May 2024 23:55:21 +0200 Subject: [PATCH 10/20] Cleanup tvdb --- scanner/providers/implementations/thetvdb.py | 35 ++++++++++++-------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/scanner/providers/implementations/thetvdb.py b/scanner/providers/implementations/thetvdb.py index cd163e66..78c2d889 100644 --- a/scanner/providers/implementations/thetvdb.py +++ b/scanner/providers/implementations/thetvdb.py @@ -19,16 +19,18 @@ logger = getLogger(__name__) class TVDB(Provider): + DEFAULT_API_KEY = "3732560f-08b7-41db-9d9a-2966b4d90c10" + def __init__( self, client: ClientSession, api_key: str, - pin: str, + pin: Optional[str], languages: list[str], ) -> None: super().__init__() self._client = client - self.base = "https://api4.thetvdb.com/v4/" + self.base = "https://api4.thetvdb.com/v4" self._api_key = api_key self._pin = pin self._languages = languages @@ -80,7 +82,10 @@ class TVDB(Provider): async def login(self) -> str: async with self._client.post( f"{self.base}/login", - json={"apikey": self._api_key, "pin": self._pin}, + json={ + "apikey": self._api_key, + } + | ({"pin": self._pin} if self._pin else {}), ) as r: r.raise_for_status() ret = await r.json() @@ -110,6 +115,7 @@ class TVDB(Provider): def name(self) -> str: return "tvdb" + @cache(ttl=timedelta(days=1)) async def search_show(self, name: str, year: Optional[int]) -> str: query = OrderedDict( query=name, @@ -125,7 +131,7 @@ class TVDB(Provider): show_id: str, language: Optional[str] = None, ): - path = f"/series/{show_id}/episodes/default" + path = f"series/{show_id}/episodes/default" if language is not None: path += f"/{language}" ret = await self.get( @@ -136,8 +142,8 @@ class TVDB(Provider): while next != None: ret = await self.get(fullPath=next) next = ret["links"]["next"] - episodes += ret["data"] - return episodes + episodes += ret["data"]["episodes"] + return episodes, ret["data"] async def search_episode( self, @@ -150,6 +156,7 @@ class TVDB(Provider): show_id = await self.search_show(name, year) return await self.identify_episode(show_id, season, episode_nbr, absolute) + @cache(ttl=timedelta(days=1)) async def identify_episode( self, show_id: str, @@ -158,8 +165,7 @@ class TVDB(Provider): absolute: Optional[int], ) -> Episode: flang, *olang = self._languages - episodes = await self.get_episodes(show_id, language=flang) - show = episodes["data"] + episodes, show = await self.get_episodes(show_id, language=flang) ret = next( filter( (lambda x: x["seasonNumber"] == 1 and x["number"] == absolute) @@ -167,7 +173,7 @@ class TVDB(Provider): else ( lambda x: x["seasonNumber"] == season and x["number"] == episode_nbr ), - episodes["episodes"], + episodes, ), None, ) @@ -188,10 +194,7 @@ class TVDB(Provider): self._languages, [ ret, - *( - next(x for x in e["episodes"] if x["id"] == ret["id"]) - for e in otrans - ), + *(next(x for x in e[0] if x["id"] == ret["id"]) for e in otrans), ], ) } @@ -223,14 +226,17 @@ class TVDB(Provider): translations=translations, ) + @cache(ttl=timedelta(days=1)) async def identify_show(self, show_id: str) -> Show: ret = await self.get( f"series/{show_id}/extended", not_found_fail=f"Could not find show with id {show_id}", ) + logger.debug("TVDB responded: %s", ret) + ret = ret["data"] translations = await asyncio.gather( *( - self.get(f"/series/{show_id}/translations/{lang}") + self.get(f"series/{show_id}/translations/{lang}") for lang in self._languages if lang != ret["originalLanguage"] ) @@ -326,6 +332,7 @@ class TVDB(Provider): return {} return {name: MetadataID(id, link(id))} + @cache(ttl=timedelta(days=1)) async def identify_season(self, show_id: str, season: int) -> Season: """ for tvdb, we don't save show_id but the season_id so we don't need to read `season` From 8c6b99e31bbcf06d779f929c5f7461d5a100b410 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Mon, 13 May 2024 00:38:56 +0200 Subject: [PATCH 11/20] Normalize tvdb language codes --- scanner/providers/implementations/thetvdb.py | 24 +++++++++++--------- scanner/requirements.txt | 1 + shell.nix | 1 + 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/scanner/providers/implementations/thetvdb.py b/scanner/providers/implementations/thetvdb.py index 78c2d889..bfba0f29 100644 --- a/scanner/providers/implementations/thetvdb.py +++ b/scanner/providers/implementations/thetvdb.py @@ -4,6 +4,7 @@ from urllib.parse import urlencode from aiohttp import ClientSession from logging import getLogger from typing import Optional, Any, Callable, OrderedDict +from langcodes import Language from matcher.cache import cache @@ -33,7 +34,9 @@ class TVDB(Provider): self.base = "https://api4.thetvdb.com/v4" self._api_key = api_key self._pin = pin - self._languages = languages + # tvdb use three letter codes for languages + # (with the terminology code as in 'fra' and not the biblographic code as in 'fre') + self._languages = [Language.get(lang).to_alpha3() for lang in languages] self._genre_map = { "soap": Genre.SOAP, "science-fiction": Genre.SCIENCE_FICTION, @@ -72,11 +75,8 @@ class TVDB(Provider): "awards-show": None, } - def two_to_three_lang(self, lang: str) -> str: - return lang - - def three_to_two_lang(self, lang: str) -> str: - return lang + def normalize_lang(self, lang: str) -> str: + return str(Language.get(lang)) @cache(ttl=timedelta(days=30)) async def login(self) -> str: @@ -186,7 +186,7 @@ class TVDB(Provider): *(self.get_episodes(show_id, language=lang) for lang in olang) ) translations = { - lang: EpisodeTranslation( + self.normalize_lang(lang): EpisodeTranslation( name=val["name"], overview=val["overview"], ) @@ -202,7 +202,7 @@ class TVDB(Provider): return Episode( show=PartialShow( name=show["name"], - original_language=self.three_to_two_lang(show["originalLanguage"]), + original_language=self.normalize_lang(show["originalLanguage"]), external_id={ self.name: MetadataID( show_id, f"https://thetvdb.com/series/{show['slug']}" @@ -242,7 +242,7 @@ class TVDB(Provider): ) ) trans = { - lang: ShowTranslation( + self.normalize_lang(lang): ShowTranslation( name=x["name"], tagline=None, tags=[], @@ -273,7 +273,7 @@ class TVDB(Provider): ] } return Show( - original_language=ret["originalLanguage"], + original_language=self.normalize_lang(ret["originalLanguage"]), aliases=[x["name"] for x in ret["aliases"]], start_air=datetime.strptime(ret["firstAired"], "%Y-%m-%d").date(), end_air=datetime.strptime(ret["lastAired"], "%Y-%m-%d").date(), @@ -363,7 +363,9 @@ class TVDB(Provider): ) trans = await asyncio.gather(*(process_translation(x) for x in self._languages)) - translations = {lang: tl for lang, tl in zip(self._languages, trans)} + translations = { + self.normalize_lang(lang): tl for lang, tl in zip(self._languages, trans) + } return Season( season_number=info["data"]["number"], diff --git a/scanner/requirements.txt b/scanner/requirements.txt index ca23e9ed..8b71b20b 100644 --- a/scanner/requirements.txt +++ b/scanner/requirements.txt @@ -4,3 +4,4 @@ jsons watchfiles aio-pika msgspec +langcodes diff --git a/shell.nix b/shell.nix index e724203c..e9778d1e 100644 --- a/shell.nix +++ b/shell.nix @@ -10,6 +10,7 @@ requests dataclasses-json msgspec + langcodes ]); dotnet = with pkgs.dotnetCorePackages; combinePackages [ From 7577d757f85812a955405e797cbdbe08a218f905 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Mon, 13 May 2024 01:13:39 +0200 Subject: [PATCH 12/20] Tvdb season access cleanup --- scanner/providers/implementations/thetvdb.py | 26 +++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/scanner/providers/implementations/thetvdb.py b/scanner/providers/implementations/thetvdb.py index bfba0f29..36d5d60c 100644 --- a/scanner/providers/implementations/thetvdb.py +++ b/scanner/providers/implementations/thetvdb.py @@ -123,6 +123,10 @@ class TVDB(Provider): type="series", ) ret = await self.get(f"search?{urlencode(query)}") + if not any(ret["data"]): + raise ProviderError( + f"No serie found with the name {name} in the year {year} (on tvdb)" + ) return ret["data"][0]["tvdb_id"] @cache(ttl=timedelta(days=1)) @@ -233,45 +237,47 @@ class TVDB(Provider): not_found_fail=f"Could not find show with id {show_id}", ) logger.debug("TVDB responded: %s", ret) - ret = ret["data"] translations = await asyncio.gather( *( self.get(f"series/{show_id}/translations/{lang}") for lang in self._languages - if lang != ret["originalLanguage"] + if lang != ret["data"]["originalLanguage"] ) ) trans = { self.normalize_lang(lang): ShowTranslation( - name=x["name"], + name=x["data"]["name"], tagline=None, tags=[], - overview=x["overview"], + overview=x["data"]["overview"], posters=[ i["image"] - for i in x["artworks"] + for i in ret["data"]["artworks"] if i["type"] == 2 and (i["language"] == lang or i["language"] is None) ], logos=[ i["image"] - for i in x["artworks"] + for i in ret["data"]["artworks"] if i["type"] == 5 and (i["language"] == lang or i["language"] is None) ], thumbnails=[ i["image"] - for i in x["artworks"] + for i in ret["data"]["artworks"] if i["type"] == 3 and (i["language"] == lang or i["language"] is None) ], - trailers=[x["url"] for t in ret["trailers"] if t["language"] == lang], + trailers=[ + t["url"] for t in ret["data"]["trailers"] if t["language"] == lang + ], ) for (lang, x) in [ - (ret["originalLanguage"], ret), + (ret["data"]["originalLanguage"], ret), *zip(self._languages, translations), ] } + ret = ret["data"] return Show( original_language=self.normalize_lang(ret["originalLanguage"]), aliases=[x["name"] for x in ret["aliases"]], @@ -342,9 +348,11 @@ class TVDB(Provider): f"seasons/{season_id}/extended", not_found_fail=f"Invalid season id {season_id}", ) + logger.debug("TVDB send season (%s) data %s", season_id, info) async def process_translation(lang: str) -> SeasonTranslation: data = await self.get(f"seasons/{season_id}/translations/{lang}") + logger.debug("TVDB send season (%s) translations (%s) data %s", season_id, lang, data) return SeasonTranslation( name=data["data"]["name"], overview=data["data"]["overview"], From fee6faceaa4213aba487b26a7898c2e5af7f2ee7 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Tue, 14 May 2024 01:01:18 +0200 Subject: [PATCH 13/20] Handle tvdb translations missing errors --- scanner/providers/implementations/thetvdb.py | 157 ++++++++++--------- 1 file changed, 87 insertions(+), 70 deletions(-) diff --git a/scanner/providers/implementations/thetvdb.py b/scanner/providers/implementations/thetvdb.py index 36d5d60c..16de3476 100644 --- a/scanner/providers/implementations/thetvdb.py +++ b/scanner/providers/implementations/thetvdb.py @@ -129,26 +129,6 @@ class TVDB(Provider): ) return ret["data"][0]["tvdb_id"] - @cache(ttl=timedelta(days=1)) - async def get_episodes( - self, - show_id: str, - language: Optional[str] = None, - ): - path = f"series/{show_id}/episodes/default" - if language is not None: - path += f"/{language}" - ret = await self.get( - path, not_found_fail=f"Could not find show with id {show_id}" - ) - episodes = ret["data"]["episodes"] - next = ret["links"]["next"] - while next != None: - ret = await self.get(fullPath=next) - next = ret["links"]["next"] - episodes += ret["data"]["episodes"] - return episodes, ret["data"] - async def search_episode( self, name: str, @@ -160,6 +140,27 @@ class TVDB(Provider): show_id = await self.search_show(name, year) return await self.identify_episode(show_id, season, episode_nbr, absolute) + @cache(ttl=timedelta(days=1)) + async def get_episodes( + self, + show_id: str, + language: str, + ): + try: + ret = await self.get( + f"series/{show_id}/episodes/default/{language}", + not_found_fail=f"Could not find show with id {show_id}", + ) + episodes = ret["data"]["episodes"] + next = ret["links"]["next"] + while next != None: + ret = await self.get(fullPath=next) + next = ret["links"]["next"] + episodes += ret["data"]["episodes"] + return episodes, ret["data"] + except ProviderError: + return None + @cache(ttl=timedelta(days=1)) async def identify_episode( self, @@ -168,8 +169,13 @@ class TVDB(Provider): episode_nbr: Optional[int], absolute: Optional[int], ) -> Episode: - flang, *olang = self._languages - episodes, show = await self.get_episodes(show_id, language=flang) + translations = await asyncio.gather( + *(self.get_episodes(show_id, language=lang) for lang in self._languages) + ) + episodes, show = next((x for x in translations if x is not None), (None, None)) + if episodes is None or show is None: + raise ProviderError(f"Could not get episodes for show with id {show_id}") + ret = next( filter( (lambda x: x["seasonNumber"] == 1 and x["number"] == absolute) @@ -186,21 +192,22 @@ class TVDB(Provider): f"Could not retrive episode {show['name']} s{season}e{episode_nbr}, absolute {absolute}" ) - otrans = await asyncio.gather( - *(self.get_episodes(show_id, language=lang) for lang in olang) - ) + trans = [ + ( + next((ep for ep in el[0] if ep["id"] == ret["id"]), None) + if el is not None + else None + ) + for el in translations + ] + translations = { self.normalize_lang(lang): EpisodeTranslation( name=val["name"], overview=val["overview"], ) - for (lang, val) in zip( - self._languages, - [ - ret, - *(next(x for x in e[0] if x["id"] == ret["id"]) for e in otrans), - ], - ) + for lang, val in zip(self._languages, trans) + if val is not None } return Episode( @@ -237,19 +244,14 @@ class TVDB(Provider): not_found_fail=f"Could not find show with id {show_id}", ) logger.debug("TVDB responded: %s", ret) - translations = await asyncio.gather( - *( - self.get(f"series/{show_id}/translations/{lang}") - for lang in self._languages - if lang != ret["data"]["originalLanguage"] - ) - ) - trans = { - self.normalize_lang(lang): ShowTranslation( - name=x["data"]["name"], + + async def process_translation(lang: str) -> Optional[ShowTranslation]: + data = await self.get(f"series/{show_id}/translations/{lang}") + return ShowTranslation( + name=data["data"]["name"], tagline=None, tags=[], - overview=x["data"]["overview"], + overview=data["data"]["overview"], posters=[ i["image"] for i in ret["data"]["artworks"] @@ -272,10 +274,14 @@ class TVDB(Provider): t["url"] for t in ret["data"]["trailers"] if t["language"] == lang ], ) - for (lang, x) in [ - (ret["data"]["originalLanguage"], ret), - *zip(self._languages, translations), - ] + + translations = await asyncio.gather( + *(process_translation(lang) for lang in self._languages) + ) + trans = { + self.normalize_lang(lang): ts + for (lang, ts) in zip(self._languages, translations) + if ts is not None } ret = ret["data"] return Show( @@ -340,9 +346,7 @@ class TVDB(Provider): @cache(ttl=timedelta(days=1)) async def identify_season(self, show_id: str, season: int) -> Season: - """ - for tvdb, we don't save show_id but the season_id so we don't need to read `season` - """ + # for tvdb, we don't save show_id but the season_id so we don't need to read `season` season_id = show_id info = await self.get( f"seasons/{season_id}/extended", @@ -350,29 +354,42 @@ class TVDB(Provider): ) logger.debug("TVDB send season (%s) data %s", season_id, info) - async def process_translation(lang: str) -> SeasonTranslation: - data = await self.get(f"seasons/{season_id}/translations/{lang}") - logger.debug("TVDB send season (%s) translations (%s) data %s", season_id, lang, data) - return SeasonTranslation( - name=data["data"]["name"], - overview=data["data"]["overview"], - posters=[ - i["image"] - for i in data["data"]["artworks"] - if i["type"] == 7 - and (i["language"] == lang or i["language"] is None) - ], - thumbnails=[ - i["image"] - for i in data["data"]["artworks"] - if i["type"] == 8 - and (i["language"] == lang or i["language"] is None) - ], - ) + async def process_translation(lang: str) -> Optional[SeasonTranslation]: + try: + data = await self.get( + f"seasons/{season_id}/translations/{lang}", + not_found_fail="Season translation not found", + ) + logger.debug( + "TVDB send season (%s) translations (%s) data %s", + season_id, + lang, + data, + ) + return SeasonTranslation( + name=data["data"]["name"], + overview=data["data"]["overview"], + posters=[ + i["image"] + for i in data["data"]["artworks"] + if i["type"] == 7 + and (i["language"] == lang or i["language"] is None) + ], + thumbnails=[ + i["image"] + for i in data["data"]["artworks"] + if i["type"] == 8 + and (i["language"] == lang or i["language"] is None) + ], + ) + except ProviderError: + return None trans = await asyncio.gather(*(process_translation(x) for x in self._languages)) translations = { - self.normalize_lang(lang): tl for lang, tl in zip(self._languages, trans) + self.normalize_lang(lang): tl + for lang, tl in zip(self._languages, trans) + if tl is not None } return Season( From 95a2766aeee328b114abccb6b5832db8b56d8ee1 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Tue, 14 May 2024 01:13:29 +0200 Subject: [PATCH 14/20] Add original language data fetching for episodes and series --- scanner/providers/implementations/thetvdb.py | 30 +++++++++++++++----- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/scanner/providers/implementations/thetvdb.py b/scanner/providers/implementations/thetvdb.py index 16de3476..9a4d4505 100644 --- a/scanner/providers/implementations/thetvdb.py +++ b/scanner/providers/implementations/thetvdb.py @@ -144,11 +144,14 @@ class TVDB(Provider): async def get_episodes( self, show_id: str, - language: str, + language: Optional[str] = None, ): try: + path = f"series/{show_id}/episodes/default" + if language is not None: + path += "/{language}" ret = await self.get( - f"series/{show_id}/episodes/default/{language}", + path, not_found_fail=f"Could not find show with id {show_id}", ) episodes = ret["data"]["episodes"] @@ -192,6 +195,10 @@ class TVDB(Provider): f"Could not retrive episode {show['name']} s{season}e{episode_nbr}, absolute {absolute}" ) + languages = self._languages + if show["originalLanguage"] not in languages: + languages = [*self._languages, show["originalLanguage"]] + translations.append(await self.get_episodes(show_id)) trans = [ ( next((ep for ep in el[0] if ep["id"] == ret["id"]), None) @@ -201,12 +208,12 @@ class TVDB(Provider): for el in translations ] - translations = { + ep_trans = { self.normalize_lang(lang): EpisodeTranslation( name=val["name"], overview=val["overview"], ) - for lang, val in zip(self._languages, trans) + for lang, val in zip(languages, trans) if val is not None } @@ -234,7 +241,7 @@ class TVDB(Provider): f"https://thetvdb.com/series/{show_id}/episodes/{ret['id']}", ), }, - translations=translations, + translations=ep_trans, ) @cache(ttl=timedelta(days=1)) @@ -246,7 +253,11 @@ class TVDB(Provider): logger.debug("TVDB responded: %s", ret) async def process_translation(lang: str) -> Optional[ShowTranslation]: - data = await self.get(f"series/{show_id}/translations/{lang}") + data = ( + await self.get(f"series/{show_id}/translations/{lang}") + if lang is not ret["orginalLanguage"] + else ret + ) return ShowTranslation( name=data["data"]["name"], tagline=None, @@ -275,8 +286,13 @@ class TVDB(Provider): ], ) + languages = ( + [*self._languages, ret["originalLanguage"]] + if ret["originalLanguage"] not in self._languages + else self._languages + ) translations = await asyncio.gather( - *(process_translation(lang) for lang in self._languages) + *(process_translation(lang) for lang in languages) ) trans = { self.normalize_lang(lang): ts From 6af791ca72f7c72f8af7a275a5844fc4343f66c3 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Tue, 14 May 2024 02:36:13 +0200 Subject: [PATCH 15/20] Fix original language serie fetch --- scanner/providers/implementations/thetvdb.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scanner/providers/implementations/thetvdb.py b/scanner/providers/implementations/thetvdb.py index 9a4d4505..c55273e8 100644 --- a/scanner/providers/implementations/thetvdb.py +++ b/scanner/providers/implementations/thetvdb.py @@ -255,7 +255,7 @@ class TVDB(Provider): async def process_translation(lang: str) -> Optional[ShowTranslation]: data = ( await self.get(f"series/{show_id}/translations/{lang}") - if lang is not ret["orginalLanguage"] + if lang is not ret["data"]["originalLanguage"] else ret ) return ShowTranslation( @@ -287,8 +287,8 @@ class TVDB(Provider): ) languages = ( - [*self._languages, ret["originalLanguage"]] - if ret["originalLanguage"] not in self._languages + [*self._languages, ret["data"]["originalLanguage"]] + if ret["data"]["originalLanguage"] not in self._languages else self._languages ) translations = await asyncio.gather( From d0a7d5766e077f462d9c6a3353435abc0455715c Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Tue, 14 May 2024 02:52:20 +0200 Subject: [PATCH 16/20] Fix season artwork fetch on tvdb --- scanner/providers/implementations/thetvdb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scanner/providers/implementations/thetvdb.py b/scanner/providers/implementations/thetvdb.py index c55273e8..66ab161b 100644 --- a/scanner/providers/implementations/thetvdb.py +++ b/scanner/providers/implementations/thetvdb.py @@ -387,13 +387,13 @@ class TVDB(Provider): overview=data["data"]["overview"], posters=[ i["image"] - for i in data["data"]["artworks"] + for i in info["data"]["artworks"] if i["type"] == 7 and (i["language"] == lang or i["language"] is None) ], thumbnails=[ i["image"] - for i in data["data"]["artworks"] + for i in info["data"]["artworks"] if i["type"] == 8 and (i["language"] == lang or i["language"] is None) ], From 13e7d87134f44afddcc81fbc0e230c2256813a24 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Tue, 14 May 2024 03:16:26 +0200 Subject: [PATCH 17/20] Fix start/end air compute when no episodes exists in season --- scanner/providers/implementations/thetvdb.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/scanner/providers/implementations/thetvdb.py b/scanner/providers/implementations/thetvdb.py index 66ab161b..b19adca3 100644 --- a/scanner/providers/implementations/thetvdb.py +++ b/scanner/providers/implementations/thetvdb.py @@ -411,8 +411,22 @@ class TVDB(Provider): return Season( season_number=info["data"]["number"], episodes_count=len(info["data"]["episodes"]), - start_air=min(x["aired"] for x in info["data"]["episodes"]), - end_air=max(x["aired"] for x in info["data"]["episodes"]), + start_air=min( + ( + x["aired"] + for x in info["data"]["episodes"] + if x["aired"] is not None + ), + default=None, + ), + end_air=max( + ( + x["aired"] + for x in info["data"]["episodes"] + if x["aired"] is not None + ), + default=None, + ), external_id={ self.name: MetadataID(season_id, None), }, From 47f364840bf1309de18faded566d4bfd9978145f Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Tue, 14 May 2024 03:22:43 +0200 Subject: [PATCH 18/20] Fix season get errors (and use artwork instead of artworkS because tvdb) --- scanner/providers/implementations/thetvdb.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scanner/providers/implementations/thetvdb.py b/scanner/providers/implementations/thetvdb.py index b19adca3..a3e85818 100644 --- a/scanner/providers/implementations/thetvdb.py +++ b/scanner/providers/implementations/thetvdb.py @@ -383,17 +383,17 @@ class TVDB(Provider): data, ) return SeasonTranslation( - name=data["data"]["name"], - overview=data["data"]["overview"], + name=data["data"].get("name"), + overview=data["data"].get("overview"), posters=[ i["image"] - for i in info["data"]["artworks"] + for i in info["data"]["artwork"] if i["type"] == 7 and (i["language"] == lang or i["language"] is None) ], thumbnails=[ i["image"] - for i in info["data"]["artworks"] + for i in info["data"]["artwork"] if i["type"] == 8 and (i["language"] == lang or i["language"] is None) ], From e02912119b1ecbb22e5a28f8c42a32ef0811a372 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Tue, 14 May 2024 23:48:15 +0200 Subject: [PATCH 19/20] Fix missing seasons and episodes translations --- scanner/providers/implementations/thetvdb.py | 22 +++++++------------- scanner/providers/utils.py | 5 +++++ 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/scanner/providers/implementations/thetvdb.py b/scanner/providers/implementations/thetvdb.py index a3e85818..c8bb6e56 100644 --- a/scanner/providers/implementations/thetvdb.py +++ b/scanner/providers/implementations/thetvdb.py @@ -9,6 +9,7 @@ from langcodes import Language from matcher.cache import cache from ..provider import Provider, ProviderError +from ..utils import normalize_lang from ..types.season import Season, SeasonTranslation from ..types.episode import Episode, EpisodeTranslation, PartialShow, EpisodeID from ..types.studio import Studio @@ -75,9 +76,6 @@ class TVDB(Provider): "awards-show": None, } - def normalize_lang(self, lang: str) -> str: - return str(Language.get(lang)) - @cache(ttl=timedelta(days=30)) async def login(self) -> str: async with self._client.post( @@ -149,7 +147,7 @@ class TVDB(Provider): try: path = f"series/{show_id}/episodes/default" if language is not None: - path += "/{language}" + path += f"/{language}" ret = await self.get( path, not_found_fail=f"Could not find show with id {show_id}", @@ -195,10 +193,6 @@ class TVDB(Provider): f"Could not retrive episode {show['name']} s{season}e{episode_nbr}, absolute {absolute}" ) - languages = self._languages - if show["originalLanguage"] not in languages: - languages = [*self._languages, show["originalLanguage"]] - translations.append(await self.get_episodes(show_id)) trans = [ ( next((ep for ep in el[0] if ep["id"] == ret["id"]), None) @@ -209,18 +203,18 @@ class TVDB(Provider): ] ep_trans = { - self.normalize_lang(lang): EpisodeTranslation( + normalize_lang(lang): EpisodeTranslation( name=val["name"], overview=val["overview"], ) - for lang, val in zip(languages, trans) + for lang, val in zip(self._languages, trans) if val is not None } return Episode( show=PartialShow( name=show["name"], - original_language=self.normalize_lang(show["originalLanguage"]), + original_language=normalize_lang(show["originalLanguage"]), external_id={ self.name: MetadataID( show_id, f"https://thetvdb.com/series/{show['slug']}" @@ -295,13 +289,13 @@ class TVDB(Provider): *(process_translation(lang) for lang in languages) ) trans = { - self.normalize_lang(lang): ts + normalize_lang(lang): ts for (lang, ts) in zip(self._languages, translations) if ts is not None } ret = ret["data"] return Show( - original_language=self.normalize_lang(ret["originalLanguage"]), + original_language=normalize_lang(ret["originalLanguage"]), aliases=[x["name"] for x in ret["aliases"]], start_air=datetime.strptime(ret["firstAired"], "%Y-%m-%d").date(), end_air=datetime.strptime(ret["lastAired"], "%Y-%m-%d").date(), @@ -403,7 +397,7 @@ class TVDB(Provider): trans = await asyncio.gather(*(process_translation(x) for x in self._languages)) translations = { - self.normalize_lang(lang): tl + normalize_lang(lang): tl for lang, tl in zip(self._languages, trans) if tl is not None } diff --git a/scanner/providers/utils.py b/scanner/providers/utils.py index f9bc39c0..67938722 100644 --- a/scanner/providers/utils.py +++ b/scanner/providers/utils.py @@ -3,6 +3,7 @@ from __future__ import annotations import os from datetime import date from itertools import chain +from langcodes import Language from typing import TYPE_CHECKING, Literal, Any, Optional if TYPE_CHECKING: @@ -21,6 +22,10 @@ def format_date(date: date | int | None) -> str | None: return date.isoformat() +def normalize_lang(lang: str) -> str: + return str(Language.get(lang)) + + # For now, the API of kyoo only support one language so we remove the others. default_languages = os.environ["LIBRARY_LANGUAGES"].split(",") From 3ce8a49282e34101b2deb39527cf26ce50cb5f5c Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Wed, 15 May 2024 00:06:06 +0200 Subject: [PATCH 20/20] Fix original language missing on series tvdb --- scanner/providers/implementations/thetvdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scanner/providers/implementations/thetvdb.py b/scanner/providers/implementations/thetvdb.py index c8bb6e56..73147a0b 100644 --- a/scanner/providers/implementations/thetvdb.py +++ b/scanner/providers/implementations/thetvdb.py @@ -290,7 +290,7 @@ class TVDB(Provider): ) trans = { normalize_lang(lang): ts - for (lang, ts) in zip(self._languages, translations) + for (lang, ts) in zip(languages, translations) if ts is not None } ret = ret["data"]