Cutlery/Kyoo
1
0
Fork 0
mirror of https://github.com/zoriya/Kyoo.git synced 2025-05-24 02:02:36 -04:00
Code Issues Packages Projects Releases Wiki Activity

Rework thexem as a provider

Browse Source
This commit is contained in:
Zoe Roux 2024-04-14 19:38:34 +02:00
parent 8392c6ad47
commit c1ecdad916
No known key found for this signature in database
8 changed files with 104 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
scanner/matcher/__init__.py
View File

@ -13,6 +13,6 @@ async def main():
logging.getLogger("rebulk").setLevel(logging.WARNING)
async with KyooClient() as kyoo, Subscriber() as sub:
provider, xem = Provider.get_all(kyoo.client)
scanner = Matcher(kyoo, provider, xem)
provider = Provider.get_default(kyoo.client)
scanner = Matcher(kyoo, provider)
await sub.listen(scanner)

6
scanner/matcher/matcher.py
View File

@ -1,7 +1,6 @@
from datetime import timedelta
import asyncio
from logging import getLogger
from providers.implementations.thexem import TheXem
from providers.provider import Provider, ProviderError
from providers.types.collection import Collection
from providers.types.show import Show
@ -15,10 +14,9 @@ logger = getLogger(__name__)
class Matcher:
def __init__(self, client: KyooClient, provider: Provider, xem: TheXem) -> None:
def __init__(self, client: KyooClient, provider: Provider) -> None:
self._client = client
self._provider = provider
self._xem = xem
self._collection_cache = {}
self._show_cache = {}
@ -48,7 +46,7 @@ class Matcher:
return True
async def _identify(self, path: str):
raw = guessit(path, xem_titles=await self._xem.get_expected_titles())
raw = guessit(path, xem_titles=await self._provider.get_expected_titles())
if "mimetype" not in raw or not raw["mimetype"].startswith("video"):
return

4
scanner/matcher/parser/guess.py
View File

@ -35,14 +35,14 @@ def guessit(name: str, *, xem_titles: List[str] = []):
if __name__ == "__main__":
import sys
import json
from providers.implementations.thexem import TheXem
from providers.implementations.thexem import TheXemClient
from guessit.jsonutils import GuessitEncoder
from aiohttp import ClientSession
import asyncio
async def main():
async with ClientSession() as client:
xem = TheXem(client)
xem = TheXemClient(client)
ret = guessit(sys.argv[1], xem_titles=await xem.get_expected_titles())
print(json.dumps(ret, cls=GuessitEncoder, indent=4))

32
scanner/providers/idmapper.py
View File

@ -1,32 +0,0 @@
from __future__ import annotations
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from providers.implementations.themoviedatabase import TheMovieDatabase
from typing import List, Optional
from providers.types.metadataid import MetadataID
class IdMapper:
def init(self, *, language: str, tmdb: Optional[TheMovieDatabase]):
self.language = language
self._tmdb = tmdb
async def get_show(
self, show: dict[str, MetadataID], *, required: Optional[List[str]] = None
):
ids = show
# Only fetch using tmdb if one of the required ids is not already known.
should_fetch = required is not None and any((x not in ids for x in required))
if self._tmdb and self._tmdb.name in ids and should_fetch:
tmdb_info = await self._tmdb.identify_show(ids[self._tmdb.name].data_id)
return {**ids, **tmdb_info.external_id}
return ids
async def get_movie(
self, movie: dict[str, MetadataID], *, required: Optional[List[str]] = None
):
# TODO: actually do something here
return movie

74
scanner/providers/implementations/themoviedatabase.py
View File

@ -5,8 +5,6 @@ from logging import getLogger
from typing import Awaitable, Callable, Dict, List, Optional, Any, TypeVar
from itertools import accumulate, zip_longest
from providers.idmapper import IdMapper
from providers.implementations.thexem import TheXem
from providers.utils import ProviderError
from matcher.cache import cache
@ -29,14 +27,10 @@ class TheMovieDatabase(Provider):
languages: list[str],
client: ClientSession,
api_key: str,
xem: TheXem,
idmapper: IdMapper,
) -> None:
super().__init__()
self._languages = languages
self._client = client
self._xem = xem
self._idmapper = idmapper
self.base = "https://api.themoviedb.org/3"
self.api_key = api_key
self.genre_map = {
@ -248,8 +242,6 @@ class TheMovieDatabase(Provider):
ret.translations[ret.original_language] = (
await for_language(ret.original_language)
).translations[ret.original_language]
# If we have more external_ids freely available, add them.
ret.external_id = await self._idmapper.get_movie(ret.external_id)
return ret
@cache(ttl=timedelta(days=1))
@ -371,8 +363,6 @@ class TheMovieDatabase(Provider):
ret.translations[ret.original_language] = (
await for_language(ret.original_language)
).translations[ret.original_language]
# If we have more external_ids freely available, add them.
ret.external_id = await self._idmapper.get_show(ret.external_id)
return ret
def to_season(
@ -422,29 +412,7 @@ class TheMovieDatabase(Provider):
)["results"]
if len(search_results) == 0:
(new_name, tvdbid) = await self._xem.get_show_override("tvdb", name)
if new_name is None or tvdbid is None or name.lower() == new_name.lower():
raise ProviderError(f"No result for a tv show named: {name}")
ret = PartialShow(
name=new_name,
original_language=None,
external_id={
"tvdb": MetadataID(tvdbid, link=None),
},
)
ret.external_id = await self._idmapper.get_show(
ret.external_id, required=[self.name]
)
if self.name in ret.external_id:
return ret
logger.warn(
"Could not map xem exception to themoviedb, searching instead for %s",
new_name,
)
nret = await self.search_show(new_name, year)
nret.external_id = {**ret.external_id, **nret.external_id}
return nret
raise ProviderError(f"No result for a tv show named: {name}")
search = self.get_best_result(search_results, name, year)
show_id = search["id"]
@ -467,38 +435,8 @@ class TheMovieDatabase(Provider):
year: Optional[int],
) -> Episode:
show = await self.search_show(name, year)
# Keep it for xem overrides of season/episode
old_name = name
name = show.name
show_id = show.external_id[self.name].data_id
# Handle weird season names overrides from thexem.
# For example when name is "Jojo's bizzare adventure - Stone Ocean", with season None,
# We want something like season 6 ep 3.
if season is None and absolute is not None:
ids = await self._idmapper.get_show(show.external_id, required=["tvdb"])
tvdb_id = (
ids["tvdb"].data_id
if "tvdb" in ids and ids["tvdb"] is not None
else None
)
if tvdb_id is None:
logger.info(
"Tvdb could not be found, trying xem name lookup for %s", name
)
_, tvdb_id = await self._xem.get_show_override("tvdb", old_name)
if tvdb_id is not None:
(
tvdb_season,
tvdb_episode,
absolute,
) = await self._xem.get_episode_override(
"tvdb", tvdb_id, old_name, absolute
)
# Most of the time, tvdb absolute and tmdb absolute are in think so we use that as our souce of truth.
# tvdb_season/episode are not in sync with tmdb so we discard those and use our usual absolute order fetching.
(_, _) = tvdb_season, tvdb_episode
if absolute is not None and (season is None or episode_nbr is None):
(season, episode_nbr) = await self.get_episode_from_absolute(
show_id, absolute
@ -506,7 +444,7 @@ class TheMovieDatabase(Provider):
if season is None or episode_nbr is None:
raise ProviderError(
f"Could not guess season or episode number of the episode {name} {season}-{episode_nbr} ({absolute})",
f"Could not guess season or episode number of the episode {show.name} {season}-{episode_nbr} ({absolute})",
)
if absolute is None:
@ -676,7 +614,9 @@ class TheMovieDatabase(Provider):
(seasons_nbrs[0], absolute),
)
async def get_absolute_number(self, show_id: str, season: int, episode_nbr: int) -> int:
async def get_absolute_number(
self, show_id: str, season: int, episode_nbr: int
) -> int:
absgrp = await self.get_absolute_order(show_id)
if absgrp is None:
# We assume that each season should be played in order with no special episodes.
@ -706,7 +646,9 @@ class TheMovieDatabase(Provider):
(x["episode_number"] for x in absgrp if x["season_number"] == season), None
)
if start is None or start <= episode_nbr:
raise ProviderError(f"Could not guess absolute number of episode {show_id} s{season} e{episode_nbr}")
raise ProviderError(
f"Could not guess absolute number of episode {show_id} s{season} e{episode_nbr}"
)
# add back the continuous number (imagine the user has one piece S21e31
# but tmdb registered it as S21E831 since S21's first ep is 800
return await self.get_absolute_number(show_id, season, episode_nbr + start)

80
scanner/providers/implementations/thexem.py
View File

@ -3,8 +3,15 @@ from typing import Dict, List, Literal
from aiohttp import ClientSession
from logging import getLogger
from datetime import timedelta
from typing import Optional
from providers.provider import Provider
from providers.utils import ProviderError
from providers.types.collection import Collection
from providers.types.movie import Movie
from providers.types.show import Show
from providers.types.season import Season
from providers.types.episode import Episode
from matcher.cache import cache
logger = getLogger(__name__)
@ -21,7 +28,7 @@ def clean(s: str):
return s
class TheXem:
class TheXemClient:
def __init__(self, client: ClientSession) -> None:
self._client = client
self.base = "https://thexem.info"
@ -155,3 +162,74 @@ class TheXem:
for y in x[1:]:
titles.extend(clean(name) for name in y.keys())
return titles
class TheXem(Provider):
def __init__(self, client: ClientSession, base: Provider) -> None:
super().__init__()
self._client = TheXemClient(client)
self._base = base
@property
def name(self) -> str:
return "TheXem"
async def get_expected_titles(self) -> list[str]:
return await self._client.get_expected_titles()
async def search_movie(self, name: str, year: Optional[int]) -> Movie:
return await self._base.search_movie(name, year)
async def search_episode(
self,
name: str,
season: Optional[int],
episode_nbr: Optional[int],
absolute: Optional[int],
year: Optional[int],
) -> Episode:
"""
Handle weird season names overrides from thexem.
For example when name is "Jojo's bizzare adventure - Stone Ocean", with season None,
We want something like season 6 ep 3.
"""
new_name, tvdb_id = await self._client.get_show_override("tvdb", name)
if new_name is None:
return await self._base.search_episode(
name, season, episode_nbr, absolute, year
)
if season is None and absolute is not None:
if tvdb_id is not None:
(
tvdb_season,
tvdb_episode,
absolute,
) = await self._client.get_episode_override(
"tvdb", tvdb_id, name, absolute
)
# Most of the time, tvdb absolute and tmdb absolute are in sync so we use that as our souce of truth.
# tvdb_season/episode are not in sync with tmdb so we discard those and use our usual absolute order fetching.
if self._base == "tvdb":
return await self._base.search_episode(
new_name, tvdb_season, tvdb_episode, absolute, year
)
return await self._base.search_episode(
new_name, season, episode_nbr, absolute, year
)
async def identify_movie(self, movie_id: str) -> Movie:
return await self._base.identify_movie(movie_id)
async def identify_show(self, show_id: str) -> Show:
return await self._base.identify_show(show_id)
async def identify_season(self, show_id: str, season: int) -> Season:
return await self._base.identify_season(show_id, season)
async def identify_episode(self, show_id: str, season: Optional[int], episode_nbr: int, absolute: int) -> Episode:
return await self._base.identify_episode(show_id, season, episode_nbr, absolute)
async def identify_collection(self, provider_id: str) -> Collection:
return await self._base.identify_collection(provider_id)

22
scanner/providers/provider.py
View File

@ -1,7 +1,7 @@
import os
from aiohttp import ClientSession
from abc import abstractmethod, abstractproperty
from typing import Optional, Self
from typing import Optional
from providers.implementations.thexem import TheXem
from providers.utils import ProviderError
@ -15,7 +15,7 @@ from .types.collection import Collection
class Provider:
@classmethod
def get_all(cls, client: ClientSession) -> tuple[Self, TheXem]:
def get_default(cls, client: ClientSession):
languages = os.environ.get("LIBRARY_LANGUAGES")
if not languages:
print("Missing environment variable 'LIBRARY_LANGUAGES'.")
@ -23,28 +23,20 @@ class Provider:
languages = languages.split(",")
providers = []
from providers.idmapper import IdMapper
idmapper = IdMapper()
xem = TheXem(client)
from providers.implementations.themoviedatabase import TheMovieDatabase
tmdb = os.environ.get("THEMOVIEDB_APIKEY")
if tmdb:
tmdb = TheMovieDatabase(languages, client, tmdb, xem, idmapper)
tmdb = TheMovieDatabase(languages, client, tmdb)
providers.append(tmdb)
else:
tmdb = None
if not any(providers):
raise ProviderError(
"No provider configured. You probably forgot to specify an API Key"
)
idmapper.init(tmdb=tmdb, language=languages[0])
return next(iter(providers)), xem
provider = next(iter(providers))
return TheXem(client, provider)
@abstractproperty
def name(self) -> str:
@ -84,3 +76,7 @@ class Provider:
@abstractmethod
async def identify_collection(self, provider_id: str) -> Collection:
raise NotImplementedError
@abstractmethod
async def get_expected_titles(self) -> list[str]:
return []

4
scanner/providers/types/episode.py
View File

@ -30,8 +30,8 @@ class EpisodeTranslation:
@dataclass
class Episode:
show: Show | PartialShow
season_number: Optional[int]
episode_number: Optional[int]
season_number: int
episode_number: int
absolute_number: int
runtime: Optional[int]
release_date: Optional[date | int]