From 8c1bb7cc1e1b939f26cec34ad5ddf71c7c6fa57d Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Thu, 25 Apr 2024 00:45:32 +0200 Subject: [PATCH 1/7] Start an anilist provider --- scanner/providers/implementations/anilist.py | 155 +++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 scanner/providers/implementations/anilist.py diff --git a/scanner/providers/implementations/anilist.py b/scanner/providers/implementations/anilist.py new file mode 100644 index 00000000..79912a08 --- /dev/null +++ b/scanner/providers/implementations/anilist.py @@ -0,0 +1,155 @@ +import asyncio +from aiohttp import ClientSession +from datetime import date +from logging import getLogger +from typing import Awaitable, Callable, Dict, List, Optional, Any, TypeVar +from itertools import accumulate, zip_longest + +from providers.utils import ProviderError +from matcher.cache import cache + +from ..provider import Provider +from ..types.movie import Movie, MovieTranslation, Status as MovieStatus +from ..types.season import Season, SeasonTranslation +from ..types.episode import Episode, EpisodeTranslation, PartialShow, EpisodeID +from ..types.studio import Studio +from ..types.genre import Genre +from ..types.metadataid import MetadataID +from ..types.show import Show, ShowTranslation, Status as ShowStatus +from ..types.collection import Collection, CollectionTranslation + +logger = getLogger(__name__) + + +class AniList(Provider): + def __init__( + self, + languages: list[str], + client: ClientSession, + api_key: str, + ) -> None: + super().__init__() + self._languages = languages + self._client = client + self.base = "https://graphql.anilist.co" + self.api_key = api_key + + @property + def name(self) -> str: + return "anilist" + + async def get(self, query: str, **variables: Optional[str]): + async with self._client.post( + self.base, json={"query": query, "variables": variables} + ) as r: + r.raise_for_status() + return await r.json() + + async def queryAnime(self, id: Optional[str], search: Optional[str]) -> Show: + query = """ + { + Media(id: $id, search: $search, type: ANIME) { + id + idMal + title { + romaji + english + native + } + description(asHtml: false) + status + startDate { + year + month + day + } + endDate { + year + month + day + } + countryOfOrigin + trailer { + id + site + } + coverImage { + extraLarge + } + bannerImage + genres + synonyms + averageScore + tags { + name + isMediaSpoiler + isGeneralSpoiler + } + studios(isMain: true) { + nodes { + id + name + siteUrl + } + } + relations { + edges { + id + relationType + node { + id + title { + romaji + english + native + } + } + } + } + } + } + """ + ret = await self.get(query, id=id, search=search) + return Show( + translations={ + "en": ShowTranslation( + name=ret["titles"]["romaji"], + tagline=None, + # TODO: unmarkdown the desc + overview=ret["description"], + # TODO: add spoiler tags + tags=[ + x["name"] + for x in ret["tags"] + if not x["isMediaSpoiler"] and not x["isGeneralSpoiler"] + ], + posters=[ret["coverImage"]["extraLarge"]], + logos=[], + thumbnails=[], + trailers=[f"https://youtube.com/watch?q={ret['trailer']['id']}"] + if ret["trailer"]["site"] == "youtube" + else [], + ) + }, + original_language=ret["countryOfOrigin"], + aliases=[ret["titles"]["english"], ret["titles"]["native"]], + start_air=date( + year=ret["startDate"]["year"], + month=ret["startDate"]["month"], + day=ret["startDate"]["day"], + ), + end_air=date( + year=ret["endDate"]["year"], + month=ret["endDate"]["month"], + day=ret["endDate"]["day"], + ), + status=ShowStatus.FINISHED + if ret["status"] == "FINISHED" + else ShowStatus.AIRING, + rating=ret["averageScore"], + # TODO: fill that + studios=[], + genres=[], + external_id={}, + seasons=[], + ) From 516af3ee9bf940095bbe0a6ca7513835414af1b0 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Thu, 25 Apr 2024 14:05:05 +0200 Subject: [PATCH 2/7] Handle genres, studio, tags for shows --- scanner/providers/implementations/anilist.py | 51 +++++++++++++++++--- 1 file changed, 45 insertions(+), 6 deletions(-) diff --git a/scanner/providers/implementations/anilist.py b/scanner/providers/implementations/anilist.py index 79912a08..2ab7567c 100644 --- a/scanner/providers/implementations/anilist.py +++ b/scanner/providers/implementations/anilist.py @@ -33,6 +33,27 @@ class AniList(Provider): self._client = client self.base = "https://graphql.anilist.co" self.api_key = api_key + self._genre_map = { + "Action": Genre.ACTION, + "Adventure": Genre.ADVENTURE, + "Comedy": Genre.COMEDY, + "Drama": Genre.DRAMA, + "Ecchi": None, + "Fantasy": Genre.FANTASY, + "Hentai": None, + "Horror": Genre.HORROR, + "Mahou Shoujo": None, + "Mecha": None, + "Music": Genre.MUSIC, + "Mystery": Genre.MYSTERY, + "Psychological": None, + "Romance": Genre.ROMANCE, + "Sci-Fi": Genre.SCIENCE_FICTION, + "Slice of Life": None, + "Sports": None, + "Supernatural": None, + "Thriller": Genre.THRILLER, + } @property def name(self) -> str: @@ -45,10 +66,10 @@ class AniList(Provider): r.raise_for_status() return await r.json() - async def queryAnime(self, id: Optional[str], search: Optional[str]) -> Show: + async def query_anime(self, id: Optional[str], search: Optional[str]) -> Show: query = """ { - Media(id: $id, search: $search, type: ANIME) { + Media(id: $id, search: $search, type: ANIME, format_not: MOVIE) { id idMal title { @@ -122,6 +143,11 @@ class AniList(Provider): x["name"] for x in ret["tags"] if not x["isMediaSpoiler"] and not x["isGeneralSpoiler"] + ] + + [ + x + for x in ret["genres"] + if x not in self._genre_map or self._genre_map[x] is None ], posters=[ret["coverImage"]["extraLarge"]], logos=[], @@ -147,9 +173,22 @@ class AniList(Provider): if ret["status"] == "FINISHED" else ShowStatus.AIRING, rating=ret["averageScore"], - # TODO: fill that - studios=[], - genres=[], - external_id={}, + genres=[self._genre_map[x] for x in ret["genres"] if x in self._genre_map], + studios=[ + Studio( + name=x["name"], + external_id={ + self.name: MetadataID(x["id"], x["siteUrl"]), + }, + ) + for x in ret["studios"]["nodes"] + ], + external_id={ + self.name: MetadataID(ret["id"], ret["siteUrl"]), + "mal": MetadataID( + ret["idMal"], f"https://myanimelist.net/anime/{ret['idMal']}" + ), + # TODO: add anidb id (needed for xem lookup and scrubbing) + }, seasons=[], ) From a0a57886830568644521029bda7691dcd9dec8e8 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Fri, 26 Apr 2024 15:39:35 +0200 Subject: [PATCH 3/7] Handle movies --- scanner/providers/implementations/anilist.py | 140 +++++++++++++++++++ 1 file changed, 140 insertions(+) diff --git a/scanner/providers/implementations/anilist.py b/scanner/providers/implementations/anilist.py index 2ab7567c..1686b27d 100644 --- a/scanner/providers/implementations/anilist.py +++ b/scanner/providers/implementations/anilist.py @@ -192,3 +192,143 @@ class AniList(Provider): }, seasons=[], ) + + async def query_movie( + self, + *, + id: Optional[str] = None, + search: Optional[str] = None, + year: Optional[int] = None, + ) -> Movie: + query = """ + { + Media(id: $id, search: $search, type: ANIME, format: MOVIE, seasonYear: $year) { + title { + romaji + english + native + } + description(asHtml: false) + status + startDate { + year + month + day + } + countryOfOrigin + trailer { + id + site + } + coverImage { + extraLarge + } + bannerImage + genres + synonyms + averageScore + tags { + name + isMediaSpoiler + isGeneralSpoiler + } + studios(isMain: true) { + nodes { + id + name + siteUrl + } + } + } + } + """ + ret = await self.get(query, id=id, search=search) + return Movie( + translations={ + "en": MovieTranslation( + name=ret["titles"]["romaji"], + tagline=None, + # TODO: unmarkdown the desc + overview=ret["description"], + # TODO: add spoiler tags + tags=[ + x["name"] + for x in ret["tags"] + if not x["isMediaSpoiler"] and not x["isGeneralSpoiler"] + ] + + [ + x + for x in ret["genres"] + if x not in self._genre_map or self._genre_map[x] is None + ], + posters=[ret["coverImage"]["extraLarge"]], + logos=[], + thumbnails=[], + trailers=[f"https://youtube.com/watch?q={ret['trailer']['id']}"] + if ret["trailer"]["site"] == "youtube" + else [], + ) + }, + original_language=ret["countryOfOrigin"], + aliases=[ret["titles"]["english"], ret["titles"]["native"]], + air_date=date( + year=ret["startDate"]["year"], + month=ret["startDate"]["month"], + day=ret["startDate"]["day"], + ), + status=MovieStatus.FINISHED + if ret["status"] == "FINISHED" + else MovieStatus.PLANNED, + rating=ret["averageScore"], + runtime=ret["runtime"], + genres=[self._genre_map[x] for x in ret["genres"] if x in self._genre_map], + studios=[ + Studio( + name=x["name"], + external_id={ + self.name: MetadataID(x["id"], x["siteUrl"]), + }, + ) + for x in ret["studios"]["nodes"] + ], + external_id={ + self.name: MetadataID(ret["id"], ret["siteUrl"]), + "mal": MetadataID( + ret["idMal"], f"https://myanimelist.net/anime/{ret['idMal']}" + ), + # TODO: add anidb id (needed for xem lookup and scrubbing) + }, + ) + + async def search_movie(self, name: str, year: Optional[int]) -> Movie: + return await self.query_movie(search=name, year=year) + + async def search_episode( + self, + name: str, + season: Optional[int], + episode_nbr: Optional[int], + absolute: Optional[int], + year: Optional[int], + ) -> Episode: + raise NotImplementedError + + async def identify_movie(self, movie_id: str) -> Movie: + return await self.query_movie(id=movie_id) + + async def identify_show(self, show_id: str) -> Show: + raise NotImplementedError + + async def identify_season(self, show_id: str, season: int) -> Season: + raise NotImplementedError + + async def identify_episode( + self, show_id: str, season: Optional[int], episode_nbr: int, absolute: int + ) -> Episode: + raise NotImplementedError + + async def identify_collection(self, provider_id: str) -> Collection: + raise NotImplementedError + + async def get_expected_titles(self) -> list[str]: + return [] From e1169d1d26d1155a675c043208286b5cc7220da6 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Fri, 26 Apr 2024 21:48:06 +0200 Subject: [PATCH 4/7] Fix anilist to use it as a provider --- scanner/providers/implementations/anilist.py | 107 ++++++++++++++----- scanner/providers/provider.py | 4 + scanner/providers/types/episode.py | 2 +- 3 files changed, 86 insertions(+), 27 deletions(-) diff --git a/scanner/providers/implementations/anilist.py b/scanner/providers/implementations/anilist.py index 1686b27d..275658f4 100644 --- a/scanner/providers/implementations/anilist.py +++ b/scanner/providers/implementations/anilist.py @@ -11,7 +11,7 @@ from matcher.cache import cache from ..provider import Provider from ..types.movie import Movie, MovieTranslation, Status as MovieStatus from ..types.season import Season, SeasonTranslation -from ..types.episode import Episode, EpisodeTranslation, PartialShow, EpisodeID +from ..types.episode import Episode, EpisodeTranslation, EpisodeID from ..types.studio import Studio from ..types.genre import Genre from ..types.metadataid import MetadataID @@ -24,15 +24,11 @@ logger = getLogger(__name__) class AniList(Provider): def __init__( self, - languages: list[str], client: ClientSession, - api_key: str, ) -> None: super().__init__() - self._languages = languages self._client = client self.base = "https://graphql.anilist.co" - self.api_key = api_key self._genre_map = { "Action": Genre.ACTION, "Adventure": Genre.ADVENTURE, @@ -59,18 +55,37 @@ class AniList(Provider): def name(self) -> str: return "anilist" - async def get(self, query: str, **variables: Optional[str]): + async def get(self, query: str, not_found: str, **variables: Optional[str | int]): + logger.error(variables) async with self._client.post( - self.base, json={"query": query, "variables": variables} + self.base, + json={ + "query": query, + "variables": {k: v for (k, v) in variables.items() if v is not None}, + }, ) as r: + if r.status == 404: + raise ProviderError(not_found) + ret = await r.json() + logger.error(ret) r.raise_for_status() - return await r.json() + if "errors" in ret: + logger.error(ret) + raise Exception(ret["errors"]) + return ret["data"] - async def query_anime(self, id: Optional[str], search: Optional[str]) -> Show: + async def query_anime( + self, + *, + id: Optional[str] = None, + search: Optional[str] = None, + year: Optional[int] = None, + ) -> Show: query = """ - { - Media(id: $id, search: $search, type: ANIME, format_not: MOVIE) { + query SearchAnime($id: Int, $search: String, $year: Int) { + Media(id: $id, search: $search, type: ANIME, format_not: MOVIE, seasonYear: $year) { id + siteUrl idMal title { romaji @@ -130,11 +145,18 @@ class AniList(Provider): } } """ - ret = await self.get(query, id=id, search=search) + q = await self.get( + query, + id=id, + search=search, + year=year, + not_found=f"Could not find the show {id or ''}{search or ''}", + ) + ret = q["Media"] return Show( translations={ "en": ShowTranslation( - name=ret["titles"]["romaji"], + name=ret["title"]["romaji"], tagline=None, # TODO: unmarkdown the desc overview=ret["description"], @@ -153,12 +175,13 @@ class AniList(Provider): logos=[], thumbnails=[], trailers=[f"https://youtube.com/watch?q={ret['trailer']['id']}"] - if ret["trailer"]["site"] == "youtube" + if ret["trailer"] is not None + and ret["trailer"]["site"] == "youtube" else [], ) }, original_language=ret["countryOfOrigin"], - aliases=[ret["titles"]["english"], ret["titles"]["native"]], + aliases=[ret["title"]["english"], ret["title"]["native"]], start_air=date( year=ret["startDate"]["year"], month=ret["startDate"]["month"], @@ -168,7 +191,9 @@ class AniList(Provider): year=ret["endDate"]["year"], month=ret["endDate"]["month"], day=ret["endDate"]["day"], - ), + ) + if ret["endDate"]["year"] is not None + else None, status=ShowStatus.FINISHED if ret["status"] == "FINISHED" else ShowStatus.AIRING, @@ -201,8 +226,11 @@ class AniList(Provider): year: Optional[int] = None, ) -> Movie: query = """ - { + query SearchMovie($id: Int, $search: String, $year: Int) { Media(id: $id, search: $search, type: ANIME, format: MOVIE, seasonYear: $year) { + id + siteUrl + idMal title { romaji english @@ -242,11 +270,18 @@ class AniList(Provider): } } """ - ret = await self.get(query, id=id, search=search) + q = await self.get( + query, + id=id, + search=search, + year=year, + not_found=f"No movie found for {id or ''}{search or ''}", + ) + ret = q["Media"] return Movie( translations={ "en": MovieTranslation( - name=ret["titles"]["romaji"], + name=ret["title"]["romaji"], tagline=None, # TODO: unmarkdown the desc overview=ret["description"], @@ -265,12 +300,13 @@ class AniList(Provider): logos=[], thumbnails=[], trailers=[f"https://youtube.com/watch?q={ret['trailer']['id']}"] - if ret["trailer"]["site"] == "youtube" + if ret["trailer"] is not None + and ret["trailer"]["site"] == "youtube" else [], ) }, original_language=ret["countryOfOrigin"], - aliases=[ret["titles"]["english"], ret["titles"]["native"]], + aliases=[ret["title"]["english"], ret["title"]["native"]], air_date=date( year=ret["startDate"]["year"], month=ret["startDate"]["month"], @@ -311,13 +347,35 @@ class AniList(Provider): absolute: Optional[int], year: Optional[int], ) -> Episode: - raise NotImplementedError + absolute = absolute or episode_nbr + if absolute is None: + raise ProviderError( + f"Could not guess episode number of the episode {name} {season}-{episode_nbr} ({absolute})" + ) + + show = await self.query_anime(search=name, year=year) + + return Episode( + show=show, + season_number=1, + episode_number=absolute, + absolute_number=absolute, + runtime=None, + release_date=None, + thumbnail=None, + external_id={ + self.name: EpisodeID( + show.external_id[self.name].data_id, None, absolute, None + ), + "mal": EpisodeID(show.external_id["mal"].data_id, None, absolute, None), + }, + ) async def identify_movie(self, movie_id: str) -> Movie: return await self.query_movie(id=movie_id) async def identify_show(self, show_id: str) -> Show: - raise NotImplementedError + return await self.query_anime(id=show_id) async def identify_season(self, show_id: str, season: int) -> Season: raise NotImplementedError @@ -329,6 +387,3 @@ class AniList(Provider): async def identify_collection(self, provider_id: str) -> Collection: raise NotImplementedError - - async def get_expected_titles(self) -> list[str]: - return [] diff --git a/scanner/providers/provider.py b/scanner/providers/provider.py index b6d1c8da..d808828d 100644 --- a/scanner/providers/provider.py +++ b/scanner/providers/provider.py @@ -22,6 +22,9 @@ class Provider: languages = languages.split(",") providers = [] + from providers.implementations.anilist import AniList + return AniList(client) + from providers.implementations.themoviedatabase import TheMovieDatabase tmdb = os.environ.get("THEMOVIEDB_APIKEY") or TheMovieDatabase.DEFAULT_API_KEY @@ -29,6 +32,7 @@ class Provider: tmdb = TheMovieDatabase(languages, client, tmdb) providers.append(tmdb) + if not any(providers): raise ProviderError( "No provider configured. You probably forgot to specify an API Key" diff --git a/scanner/providers/types/episode.py b/scanner/providers/types/episode.py index 0401d4ee..9022ae5d 100644 --- a/scanner/providers/types/episode.py +++ b/scanner/providers/types/episode.py @@ -19,7 +19,7 @@ class EpisodeID: show_id: str season: Optional[int] episode: int - link: str + link: Optional[str] @dataclass From 833775fede373f81cb47583dd2b6c7fb574694c6 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Fri, 26 Apr 2024 21:58:32 +0200 Subject: [PATCH 5/7] Fix matcher serialization when translations are missing --- scanner/providers/utils.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/scanner/providers/utils.py b/scanner/providers/utils.py index 1024de92..ae01a29c 100644 --- a/scanner/providers/utils.py +++ b/scanner/providers/utils.py @@ -18,19 +18,22 @@ def format_date(date: date | int | None) -> str | None: def select_image( self: Movie | Show, - type: Literal["posters"] | Literal["thumbnails"] | Literal["logos"], + kind: Literal["posters"] | Literal["thumbnails"] | Literal["logos"], ) -> str | None: # For now, the API of kyoo only support one language so we remove the others. default_language = os.environ["LIBRARY_LANGUAGES"].split(",")[0] return next( chain( ( - getattr(self.translations[self.original_language], type) + getattr(self.translations[self.original_language], kind) if self.original_language + and self.original_language in self.translations else [] ), - getattr(self.translations[default_language], type), - *(getattr(x, type) for x in self.translations.values()), + getattr(self.translations[default_language], kind) + if default_language in self.translations + else [], + *(getattr(x, kind) for x in self.translations.values()), ), None, ) From fb0af0855c429fc9ff22260b3ba7cf72f1b6736d Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Fri, 26 Apr 2024 22:33:19 +0200 Subject: [PATCH 6/7] Add placeholder seasons for anilist --- scanner/providers/implementations/anilist.py | 69 +++++++++++++------ .../implementations/themoviedatabase.py | 8 ++- scanner/providers/types/season.py | 2 +- 3 files changed, 54 insertions(+), 25 deletions(-) diff --git a/scanner/providers/implementations/anilist.py b/scanner/providers/implementations/anilist.py index 275658f4..8e250885 100644 --- a/scanner/providers/implementations/anilist.py +++ b/scanner/providers/implementations/anilist.py @@ -1,9 +1,8 @@ import asyncio from aiohttp import ClientSession -from datetime import date +from datetime import date, timedelta from logging import getLogger -from typing import Awaitable, Callable, Dict, List, Optional, Any, TypeVar -from itertools import accumulate, zip_longest +from typing import Optional from providers.utils import ProviderError from matcher.cache import cache @@ -56,24 +55,30 @@ class AniList(Provider): return "anilist" async def get(self, query: str, not_found: str, **variables: Optional[str | int]): - logger.error(variables) - async with self._client.post( - self.base, - json={ - "query": query, - "variables": {k: v for (k, v) in variables.items() if v is not None}, - }, - ) as r: - if r.status == 404: - raise ProviderError(not_found) - ret = await r.json() - logger.error(ret) - r.raise_for_status() - if "errors" in ret: + while True: + async with self._client.post( + self.base, + json={ + "query": query, + "variables": { + k: v for (k, v) in variables.items() if v is not None + }, + }, + ) as r: + if r.status == 404: + raise ProviderError(not_found) + if r.status == 429: + await asyncio.sleep(float(r.headers["Retry-After"])) + continue + ret = await r.json() logger.error(ret) - raise Exception(ret["errors"]) - return ret["data"] + r.raise_for_status() + if "errors" in ret: + logger.error(ret) + raise Exception(ret["errors"]) + return ret["data"] + @cache(ttl=timedelta(days=1)) async def query_anime( self, *, @@ -94,6 +99,7 @@ class AniList(Provider): } description(asHtml: false) status + episodes startDate { year month @@ -153,7 +159,7 @@ class AniList(Provider): not_found=f"Could not find the show {id or ''}{search or ''}", ) ret = q["Media"] - return Show( + show = Show( translations={ "en": ShowTranslation( name=ret["title"]["romaji"], @@ -217,7 +223,27 @@ class AniList(Provider): }, seasons=[], ) + show.seasons.append( + Season( + # TODO: fill this approprietly + season_number=1, + episodes_count=ret["episodes"], + start_air=show.start_air, + end_air=show.end_air, + external_id=show.external_id, + translations={ + "en": SeasonTranslation( + name=show.translations["en"].name, + overview=show.translations["en"].overview, + posters=show.translations["en"].posters, + thumbnails=[], + ) + }, + ) + ) + return show + @cache(ttl=timedelta(days=1)) async def query_movie( self, *, @@ -378,7 +404,8 @@ class AniList(Provider): return await self.query_anime(id=show_id) async def identify_season(self, show_id: str, season: int) -> Season: - raise NotImplementedError + show = await self.query_anime(id=show_id) + return next((x for x in show.seasons if x.season_number == season)) async def identify_episode( self, show_id: str, season: Optional[int], episode_nbr: int, absolute: int diff --git a/scanner/providers/implementations/themoviedatabase.py b/scanner/providers/implementations/themoviedatabase.py index 3cf99a88..e48bd3f2 100644 --- a/scanner/providers/implementations/themoviedatabase.py +++ b/scanner/providers/implementations/themoviedatabase.py @@ -2,7 +2,7 @@ import asyncio from aiohttp import ClientSession from datetime import datetime, timedelta from logging import getLogger -from typing import Awaitable, Callable, Dict, List, Optional, Any, TypeVar +from typing import cast, Awaitable, Callable, Dict, List, Optional, Any, TypeVar from itertools import accumulate, zip_longest from providers.utils import ProviderError @@ -635,7 +635,9 @@ class TheMovieDatabase(Provider): show = await self.identify_show(show_id) # Dont forget to ingore the special season (season_number 0) seasons_nbrs = [x.season_number for x in show.seasons if x.season_number != 0] - seasons_eps = [x.episodes_count for x in show.seasons if x.season_number != 0] + seasons_eps = [ + cast(int, x.episodes_count) for x in show.seasons if x.season_number != 0 + ] if not any(seasons_nbrs): return (None, None) @@ -663,7 +665,7 @@ class TheMovieDatabase(Provider): show = await self.identify_show(show_id) return ( sum( - x.episodes_count + cast(int, x.episodes_count) for x in show.seasons if 0 < x.season_number < season ) diff --git a/scanner/providers/types/season.py b/scanner/providers/types/season.py index 0c224ece..a2568ff9 100644 --- a/scanner/providers/types/season.py +++ b/scanner/providers/types/season.py @@ -19,7 +19,7 @@ class Season: season_number: int # This is not used by kyoo, this is just used internaly by the TMDB provider. # maybe this should be moved? - episodes_count: int + episodes_count: Optional[int] start_air: Optional[date | int] = None end_air: Optional[date | int] = None external_id: dict[str, MetadataID] = field(default_factory=dict) From fa7229f6f9e3b282138a9b2bf4579d96a20865aa Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Fri, 26 Apr 2024 23:44:31 +0200 Subject: [PATCH 7/7] Make anilist poc work --- scanner/providers/implementations/anilist.py | 124 ++++++++++++++----- scanner/providers/provider.py | 2 +- scanner/providers/types/season.py | 1 + 3 files changed, 92 insertions(+), 35 deletions(-) diff --git a/scanner/providers/implementations/anilist.py b/scanner/providers/implementations/anilist.py index 8e250885..8f617744 100644 --- a/scanner/providers/implementations/anilist.py +++ b/scanner/providers/implementations/anilist.py @@ -1,6 +1,6 @@ import asyncio from aiohttp import ClientSession -from datetime import date, timedelta +from datetime import date, timedelta, datetime from logging import getLogger from typing import Optional @@ -15,7 +15,7 @@ from ..types.studio import Studio from ..types.genre import Genre from ..types.metadataid import MetadataID from ..types.show import Show, ShowTranslation, Status as ShowStatus -from ..types.collection import Collection, CollectionTranslation +from ..types.collection import Collection logger = getLogger(__name__) @@ -68,7 +68,16 @@ class AniList(Provider): if r.status == 404: raise ProviderError(not_found) if r.status == 429: - await asyncio.sleep(float(r.headers["Retry-After"])) + logger.error(r.headers) + if "Retry-After" in r.headers: + await asyncio.sleep(float(r.headers["Retry-After"])) + elif "X-RateLimit-Reset" in r.headers: + reset = datetime.fromtimestamp( + float(r.headers["X-RateLimit-Reset"]) + ) + await asyncio.sleep((reset - datetime.now()).total_seconds()) + else: + await asyncio.sleep(60) continue ret = await r.json() logger.error(ret) @@ -90,7 +99,7 @@ class AniList(Provider): query SearchAnime($id: Int, $search: String, $year: Int) { Media(id: $id, search: $search, type: ANIME, format_not: MOVIE, seasonYear: $year) { id - siteUrl + siteUrl idMal title { romaji @@ -98,8 +107,8 @@ class AniList(Provider): native } description(asHtml: false) - status - episodes + status + episodes startDate { year month @@ -187,24 +196,34 @@ class AniList(Provider): ) }, original_language=ret["countryOfOrigin"], - aliases=[ret["title"]["english"], ret["title"]["native"]], + aliases=[ + x + for x in [ret["title"]["english"], ret["title"]["native"]] + if x is not None + ], start_air=date( year=ret["startDate"]["year"], - month=ret["startDate"]["month"], - day=ret["startDate"]["day"], - ), + month=ret["startDate"]["month"] or 1, + day=ret["startDate"]["day"] or 1, + ) + if ret["startDate"] is not None + else None, end_air=date( year=ret["endDate"]["year"], - month=ret["endDate"]["month"], - day=ret["endDate"]["day"], + month=ret["endDate"]["month"] or 1, + day=ret["endDate"]["day"] or 1, ) if ret["endDate"]["year"] is not None else None, status=ShowStatus.FINISHED if ret["status"] == "FINISHED" else ShowStatus.AIRING, - rating=ret["averageScore"], - genres=[self._genre_map[x] for x in ret["genres"] if x in self._genre_map], + rating=ret["averageScore"] or 0, + genres=[ + self._genre_map[x] + for x in ret["genres"] + if x in self._genre_map and self._genre_map[x] is not None + ], studios=[ Studio( name=x["name"], @@ -216,11 +235,17 @@ class AniList(Provider): ], external_id={ self.name: MetadataID(ret["id"], ret["siteUrl"]), - "mal": MetadataID( - ret["idMal"], f"https://myanimelist.net/anime/{ret['idMal']}" - ), - # TODO: add anidb id (needed for xem lookup and scrubbing) - }, + } + | ( + { + "mal": MetadataID( + ret["idMal"], f"https://myanimelist.net/anime/{ret['idMal']}" + ) + } + if ret["idMal"] is not None + else {} + ), + # TODO: add anidb id (needed for xem lookup and scrubbing) seasons=[], ) show.seasons.append( @@ -255,7 +280,7 @@ class AniList(Provider): query SearchMovie($id: Int, $search: String, $year: Int) { Media(id: $id, search: $search, type: ANIME, format: MOVIE, seasonYear: $year) { id - siteUrl + siteUrl idMal title { romaji @@ -263,7 +288,8 @@ class AniList(Provider): native } description(asHtml: false) - status + status + duration startDate { year month @@ -332,18 +358,28 @@ class AniList(Provider): ) }, original_language=ret["countryOfOrigin"], - aliases=[ret["title"]["english"], ret["title"]["native"]], + aliases=[ + x + for x in [ret["title"]["english"], ret["title"]["native"]] + if x is not None + ], air_date=date( year=ret["startDate"]["year"], - month=ret["startDate"]["month"], - day=ret["startDate"]["day"], - ), + month=ret["startDate"]["month"] or 1, + day=ret["startDate"]["day"] or 1, + ) + if ret["startDate"] is not None + else None, status=MovieStatus.FINISHED if ret["status"] == "FINISHED" else MovieStatus.PLANNED, - rating=ret["averageScore"], - runtime=ret["runtime"], - genres=[self._genre_map[x] for x in ret["genres"] if x in self._genre_map], + rating=ret["averageScore"] or 0, + runtime=ret["duration"], + genres=[ + self._genre_map[x] + for x in ret["genres"] + if x in self._genre_map and self._genre_map[x] is not None + ], studios=[ Studio( name=x["name"], @@ -355,11 +391,17 @@ class AniList(Provider): ], external_id={ self.name: MetadataID(ret["id"], ret["siteUrl"]), - "mal": MetadataID( - ret["idMal"], f"https://myanimelist.net/anime/{ret['idMal']}" - ), - # TODO: add anidb id (needed for xem lookup and scrubbing) - }, + } + | ( + { + "mal": MetadataID( + ret["idMal"], f"https://myanimelist.net/anime/{ret['idMal']}" + ), + # TODO: add anidb id (needed for xem lookup and scrubbing) + } + if ret["idMal"] is not None + else {} + ), ) async def search_movie(self, name: str, year: Optional[int]) -> Movie: @@ -393,7 +435,21 @@ class AniList(Provider): self.name: EpisodeID( show.external_id[self.name].data_id, None, absolute, None ), - "mal": EpisodeID(show.external_id["mal"].data_id, None, absolute, None), + } + | ( + { + "mal": EpisodeID( + show.external_id["mal"].data_id, None, absolute, None + ), + } + if "mal" in show.external_id + else {} + ), + translations={ + "en": EpisodeTranslation( + name=f"Episode {absolute}", + overview=None, + ), }, ) diff --git a/scanner/providers/provider.py b/scanner/providers/provider.py index d808828d..ad3a0976 100644 --- a/scanner/providers/provider.py +++ b/scanner/providers/provider.py @@ -23,6 +23,7 @@ class Provider: providers = [] from providers.implementations.anilist import AniList + return AniList(client) from providers.implementations.themoviedatabase import TheMovieDatabase @@ -32,7 +33,6 @@ class Provider: tmdb = TheMovieDatabase(languages, client, tmdb) providers.append(tmdb) - if not any(providers): raise ProviderError( "No provider configured. You probably forgot to specify an API Key" diff --git a/scanner/providers/types/season.py b/scanner/providers/types/season.py index a2568ff9..15abbc2d 100644 --- a/scanner/providers/types/season.py +++ b/scanner/providers/types/season.py @@ -33,6 +33,7 @@ class Season: return { **asdict(self), **asdict(self.translations[default_language]), + "episodes_count": 0, "poster": next(iter(self.translations[default_language].posters), None), "thumbnail": next( iter(self.translations[default_language].thumbnails), None