mirror of
https://github.com/zoriya/Kyoo.git
synced 2025-05-24 02:02:36 -04:00
Move old stuff to one dir
This commit is contained in:
Zoe Roux
parent
aaa9a7703a
commit
00f0ec44e4
@ -1,242 +0,0 @@
|
||||
from datetime import timedelta
|
||||
from typing import Literal, Optional
|
||||
import asyncio
|
||||
from logging import getLogger
|
||||
from providers.provider import Provider, ProviderError
|
||||
from providers.types.collection import Collection
|
||||
from providers.types.show import Show
|
||||
from providers.types.episode import Episode, PartialShow
|
||||
from providers.types.season import Season
|
||||
from providers.kyoo_client import KyooClient
|
||||
from .parser.guess import guessit
|
||||
from .cache import cache, exec_as_cache, make_key
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
|
||||
class Matcher:
|
||||
def __init__(self, client: KyooClient, provider: Provider) -> None:
|
||||
self._client = client
|
||||
self._provider = provider
|
||||
|
||||
self._collection_cache = {}
|
||||
self._show_cache = {}
|
||||
self._season_cache = {}
|
||||
|
||||
async def delete(self, path: str):
|
||||
try:
|
||||
await self._client.delete(path)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.exception("Unhandled error", exc_info=e)
|
||||
return False
|
||||
|
||||
async def identify(self, path: str):
|
||||
try:
|
||||
await self._identify(path)
|
||||
await self._client.delete_issue(path)
|
||||
except ProviderError as e:
|
||||
logger.error(e)
|
||||
await self._client.create_issue(path, str(e))
|
||||
except Exception as e:
|
||||
logger.exception("Unhandled error", exc_info=e)
|
||||
await self._client.create_issue(
|
||||
path, "Unknown error", {"type": type(e).__name__, "message": str(e)}
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
async def _identify(self, path: str):
|
||||
raw = guessit(path, xem_titles=await self._provider.get_expected_titles())
|
||||
|
||||
if "mimetype" not in raw or not raw["mimetype"].startswith("video"):
|
||||
return
|
||||
|
||||
logger.info("Identified %s: %s", path, raw)
|
||||
|
||||
title = raw.get("title")
|
||||
if not isinstance(title, str):
|
||||
raise ProviderError(f"Could not guess title, found: {title}")
|
||||
|
||||
year = raw.get("year")
|
||||
if year is not None and not isinstance(year, int):
|
||||
year = None
|
||||
logger.warn(f"Invalid year value. Found {year}. Ignoring")
|
||||
|
||||
if raw["type"] == "movie":
|
||||
await self.search_movie(title, year, path)
|
||||
elif raw["type"] == "episode":
|
||||
season = raw.get("season")
|
||||
if isinstance(season, list):
|
||||
raise ProviderError(
|
||||
f"An episode can't have multiple seasons (found {raw.get('season')} for {path})"
|
||||
)
|
||||
if season is not None and not isinstance(season, int):
|
||||
raise ProviderError(f"Could not guess season, found: {season}")
|
||||
episode = raw.get("episode")
|
||||
if isinstance(episode, list):
|
||||
raise ProviderError(
|
||||
f"Multi-episodes files are not yet supported (for {path})"
|
||||
)
|
||||
if not isinstance(episode, int):
|
||||
raise ProviderError(f"Could not guess episode, found: {episode}")
|
||||
|
||||
await self.search_episode(title, year, season, episode, path)
|
||||
else:
|
||||
logger.warn("Unknown video file type: %s", raw["type"])
|
||||
|
||||
async def search_movie(self, title: str, year: Optional[int], path: str):
|
||||
movie = await self._provider.search_movie(title, year)
|
||||
movie.file_title = title
|
||||
movie.path = path
|
||||
logger.debug("Got movie: %s", movie)
|
||||
movie_id = await self._client.post("movies", data=movie.to_kyoo())
|
||||
|
||||
if any(movie.collections):
|
||||
ids = await asyncio.gather(
|
||||
*(self.create_or_get_collection(x) for x in movie.collections)
|
||||
)
|
||||
await asyncio.gather(
|
||||
*(self._client.link_collection(x, "movie", movie_id) for x in ids)
|
||||
)
|
||||
|
||||
async def search_episode(
|
||||
self,
|
||||
title: str,
|
||||
year: Optional[int],
|
||||
season: Optional[int],
|
||||
episode_nbr: int,
|
||||
path: str,
|
||||
):
|
||||
episode = await self._provider.search_episode(
|
||||
title,
|
||||
season=season,
|
||||
episode_nbr=episode_nbr if season is not None else None,
|
||||
absolute=episode_nbr if season is None else None,
|
||||
year=year,
|
||||
)
|
||||
episode.path = path
|
||||
logger.debug("Got episode: %s", episode)
|
||||
episode.show_id = await self.create_or_get_show(episode, title)
|
||||
|
||||
if episode.season_number is not None:
|
||||
episode.season_id = await self.register_seasons(
|
||||
episode.show, episode.show_id, episode.season_number
|
||||
)
|
||||
await self._client.post("episodes", data=episode.to_kyoo())
|
||||
|
||||
async def create_or_get_collection(self, collection: Collection) -> str:
|
||||
@cache(ttl=timedelta(days=1), cache=self._collection_cache)
|
||||
async def create_collection(provider_id: str):
|
||||
# TODO: Check if a collection with the same metadata id exists already on kyoo.
|
||||
new_collection = (
|
||||
await self._provider.identify_collection(provider_id)
|
||||
if not any(collection.translations.keys())
|
||||
else collection
|
||||
)
|
||||
logger.debug("Got collection: %s", new_collection)
|
||||
return await self._client.post("collection", data=new_collection.to_kyoo())
|
||||
|
||||
# The parameter is only used as a key for the cache.
|
||||
provider_id = collection.external_id[self._provider.name].data_id
|
||||
return await create_collection(provider_id)
|
||||
|
||||
async def create_or_get_show(self, episode: Episode, fallback_name: str) -> str:
|
||||
@cache(ttl=timedelta(days=1), cache=self._show_cache)
|
||||
async def create_show(_: str):
|
||||
# TODO: Check if a show with the same metadata id exists already on kyoo.
|
||||
show = (
|
||||
await self._provider.identify_show(
|
||||
episode.show.external_id[self._provider.name].data_id,
|
||||
)
|
||||
if isinstance(episode.show, PartialShow)
|
||||
else episode.show
|
||||
)
|
||||
show.file_title = fallback_name
|
||||
# TODO: collections
|
||||
logger.debug("Got show: %s", episode)
|
||||
ret = await self._client.post("show", data=show.to_kyoo())
|
||||
|
||||
async def create_season(season: Season, id: str):
|
||||
try:
|
||||
season.show_id = id
|
||||
return await self._client.post("seasons", data=season.to_kyoo())
|
||||
except Exception as e:
|
||||
logger.exception("Unhandled error create a season", exc_info=e)
|
||||
|
||||
season_tasks = map(
|
||||
lambda s: exec_as_cache(
|
||||
self._season_cache,
|
||||
make_key((ret, s.season_number)),
|
||||
lambda: create_season(s, ret),
|
||||
),
|
||||
show.seasons,
|
||||
)
|
||||
await asyncio.gather(*season_tasks)
|
||||
|
||||
return ret
|
||||
|
||||
# The parameter is only used as a key for the cache.
|
||||
provider_id = episode.show.external_id[self._provider.name].data_id
|
||||
return await create_show(provider_id)
|
||||
|
||||
async def register_seasons(
|
||||
self, show: Show | PartialShow, show_id: str, season_number: int
|
||||
) -> str:
|
||||
# We use an external season cache because we want to edit this cache programatically
|
||||
@cache(ttl=timedelta(days=1), cache=self._season_cache)
|
||||
async def create_season(_: str, __: int):
|
||||
season = await self._provider.identify_season(
|
||||
show.external_id[self._provider.name].data_id, season_number
|
||||
)
|
||||
season.show_id = show_id
|
||||
return await self._client.post("seasons", data=season.to_kyoo())
|
||||
|
||||
return await create_season(show_id, season_number)
|
||||
|
||||
async def refresh(
|
||||
self,
|
||||
kind: Literal["collection", "movie", "episode", "show", "season"],
|
||||
kyoo_id: str,
|
||||
):
|
||||
async def id_movie(movie: dict, id: dict):
|
||||
ret = await self._provider.identify_movie(id["dataId"])
|
||||
ret.path = movie["path"]
|
||||
return ret
|
||||
|
||||
async def id_season(season: dict, id: dict):
|
||||
ret = await self._provider.identify_season(
|
||||
id["dataId"], season["seasonNumber"]
|
||||
)
|
||||
ret.show_id = season["showId"]
|
||||
return ret
|
||||
|
||||
async def id_episode(episode: dict, id: dict):
|
||||
ret = await self._provider.identify_episode(
|
||||
id["showId"], id["season"], id["episode"], episode["absoluteNumber"]
|
||||
)
|
||||
ret.show_id = episode["showId"]
|
||||
ret.season_id = episode["seasonId"]
|
||||
ret.path = episode["path"]
|
||||
return ret
|
||||
|
||||
identify_table = {
|
||||
"collection": lambda _, id: self._provider.identify_collection(
|
||||
id["dataId"]
|
||||
),
|
||||
"movie": id_movie,
|
||||
"show": lambda _, id: self._provider.identify_show(id["dataId"]),
|
||||
"season": id_season,
|
||||
"episode": id_episode,
|
||||
}
|
||||
|
||||
current = await self._client.get(f"{kind}/{kyoo_id}")
|
||||
if self._provider.name not in current["externalId"]:
|
||||
logger.error(
|
||||
f"Could not refresh metadata of {kind}/{kyoo_id}. Missing provider id."
|
||||
)
|
||||
return False
|
||||
provider_id = current["externalId"][self._provider.name]
|
||||
new_value = await identify_table[kind](current, provider_id)
|
||||
await self._client.put(f"{kind}/{kyoo_id}", data=new_value.to_kyoo())
|
||||
return True
|
||||
@ -1,62 +0,0 @@
|
||||
import asyncio
|
||||
from typing import Union, Literal
|
||||
from msgspec import Struct, json
|
||||
from logging import getLogger
|
||||
from aio_pika.abc import AbstractIncomingMessage
|
||||
|
||||
from providers.rabbit_base import RabbitBase
|
||||
from matcher.matcher import Matcher
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
|
||||
class Message(Struct, tag_field="action", tag=str.lower):
|
||||
pass
|
||||
|
||||
|
||||
class Scan(Message):
|
||||
path: str
|
||||
|
||||
|
||||
class Delete(Message):
|
||||
path: str
|
||||
|
||||
|
||||
class Refresh(Message):
|
||||
kind: Literal["collection", "show", "movie", "season", "episode"]
|
||||
id: str
|
||||
|
||||
|
||||
decoder = json.Decoder(Union[Scan, Delete, Refresh])
|
||||
|
||||
|
||||
class Subscriber(RabbitBase):
|
||||
async def listen(self, matcher: Matcher):
|
||||
async def on_message(message: AbstractIncomingMessage):
|
||||
try:
|
||||
msg = decoder.decode(message.body)
|
||||
ack = False
|
||||
match msg:
|
||||
case Scan(path):
|
||||
ack = await matcher.identify(path)
|
||||
case Delete(path):
|
||||
ack = await matcher.delete(path)
|
||||
case Refresh(kind, id):
|
||||
ack = await matcher.refresh(kind, id)
|
||||
case _:
|
||||
logger.error(f"Invalid action: {msg.action}")
|
||||
if ack:
|
||||
logger.info("finished processing %s", msg)
|
||||
await message.ack()
|
||||
else:
|
||||
logger.warn("failed to process %s", msg)
|
||||
await message.reject()
|
||||
except Exception as e:
|
||||
logger.exception("Unhandled error", exc_info=e)
|
||||
await message.reject()
|
||||
|
||||
# Allow up to 20 scan requests to run in parallel on the same listener.
|
||||
# Since most work is calling API not doing that is a waste.
|
||||
await self._channel.set_qos(prefetch_count=20)
|
||||
await self._queue.consume(on_message)
|
||||
await asyncio.Future()
|
||||
@ -7,7 +7,7 @@ async def main():
|
||||
from .refresher import refresh
|
||||
from .publisher import Publisher
|
||||
from .subscriber import Subscriber
|
||||
from providers.kyoo_client import KyooClient
|
||||
from old.kyoo_client import KyooClient
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logging.getLogger("watchfiles").setLevel(logging.WARNING)
|
||||
67
scanner/old/matcher.py
Normal file
67
scanner/old/matcher.py
Normal file
@ -0,0 +1,67 @@
|
||||
from datetime import timedelta
|
||||
from typing import Literal, Optional
|
||||
import asyncio
|
||||
from logging import getLogger
|
||||
from old.provider import Provider, ProviderError
|
||||
from old.types.collection import Collection
|
||||
from old.types.show import Show
|
||||
from old.types.episode import Episode, PartialShow
|
||||
from old.types.season import Season
|
||||
from old.kyoo_client import KyooClient
|
||||
from .parser.guess import guessit
|
||||
from .cache import cache, exec_as_cache, make_key
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
|
||||
class Matcher:
|
||||
def __init__(self, client: KyooClient, provider: Provider) -> None:
|
||||
self._client = client
|
||||
self._provider = provider
|
||||
|
||||
async def refresh(
|
||||
self,
|
||||
kind: Literal["collection", "movie", "episode", "show", "season"],
|
||||
kyoo_id: str,
|
||||
):
|
||||
async def id_movie(movie: dict, id: dict):
|
||||
ret = await self._provider.identify_movie(id["dataId"])
|
||||
ret.path = movie["path"]
|
||||
return ret
|
||||
|
||||
async def id_season(season: dict, id: dict):
|
||||
ret = await self._provider.identify_season(
|
||||
id["dataId"], season["seasonNumber"]
|
||||
)
|
||||
ret.show_id = season["showId"]
|
||||
return ret
|
||||
|
||||
async def id_episode(episode: dict, id: dict):
|
||||
ret = await self._provider.identify_episode(
|
||||
id["showId"], id["season"], id["episode"], episode["absoluteNumber"]
|
||||
)
|
||||
ret.show_id = episode["showId"]
|
||||
ret.season_id = episode["seasonId"]
|
||||
ret.path = episode["path"]
|
||||
return ret
|
||||
|
||||
identify_table = {
|
||||
"collection": lambda _, id: self._provider.identify_collection(
|
||||
id["dataId"]
|
||||
),
|
||||
"movie": id_movie,
|
||||
"show": lambda _, id: self._provider.identify_show(id["dataId"]),
|
||||
"season": id_season,
|
||||
"episode": id_episode,
|
||||
}
|
||||
|
||||
current = await self._client.get(f"{kind}/{kyoo_id}")
|
||||
if self._provider.name not in current["externalId"]:
|
||||
logger.error(
|
||||
f"Could not refresh metadata of {kind}/{kyoo_id}. Missing provider id."
|
||||
)
|
||||
return False
|
||||
provider_id = current["externalId"][self._provider.name]
|
||||
new_value = await identify_table[kind](current, provider_id)
|
||||
await self._client.put(f"{kind}/{kyoo_id}", data=new_value.to_kyoo())
|
||||
return True
|
||||
@ -1,7 +1,7 @@
|
||||
import asyncio
|
||||
from logging import getLogger
|
||||
|
||||
from providers.kyoo_client import KyooClient
|
||||
from old.kyoo_client import KyooClient
|
||||
from scanner.publisher import Publisher
|
||||
|
||||
|
||||
@ -5,3 +5,4 @@ aiohttp
|
||||
watchfiles
|
||||
langcodes
|
||||
psycopg[binary]
|
||||
|
||||
|
||||
@ -5,7 +5,7 @@ from typing import Literal
|
||||
|
||||
from .client import KyooClient
|
||||
from .models.videos import Guess
|
||||
from .providers.composite import CompositeProvider
|
||||
from .old.composite import CompositeProvider
|
||||
from .utils import Model
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
@ -7,6 +7,7 @@
|
||||
aiohttp
|
||||
watchfiles
|
||||
langcodes
|
||||
psycopg
|
||||
]);
|
||||
in
|
||||
pkgs.mkShell {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user