Cutlery/Kyoo
1
0
Fork 0
mirror of https://github.com/zoriya/Kyoo.git synced 2025-06-03 05:34:23 -04:00
Code Issues Packages Projects Releases Wiki Activity

Cleanup scanner logger

Browse Source
This commit is contained in:
Zoe Roux 2024-04-09 22:37:26 +02:00
parent 27dfb71838
commit 8d8e984669
No known key found for this signature in database
5 changed files with 43 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docker-compose.dev.yml
View File

@ -86,7 +86,7 @@ services:
matcher: matcher:
build: ./scanner build: ./scanner
command: "matcher" command: matcher
restart: on-failure restart: on-failure
depends_on: depends_on:
back: back:

24
scanner/matcher/matcher.py
View File

@ -1,6 +1,6 @@
from datetime import timedelta from datetime import timedelta
import asyncio import asyncio
import logging from logging import getLogger
from providers.implementations.thexem import TheXem from providers.implementations.thexem import TheXem
from providers.provider import Provider, ProviderError from providers.provider import Provider, ProviderError
from providers.types.collection import Collection from providers.types.collection import Collection
@ -11,6 +11,8 @@ from providers.kyoo_client import KyooClient
from .parser.guess import guessit from .parser.guess import guessit
from .cache import cache, exec_as_cache, make_key from .cache import cache, exec_as_cache, make_key
logger = getLogger(__name__)
class Matcher: class Matcher:
def __init__(self, client: KyooClient, provider: Provider, xem: TheXem) -> None: def __init__(self, client: KyooClient, provider: Provider, xem: TheXem) -> None:
@ -27,7 +29,7 @@ class Matcher:
await self._client.delete(path) await self._client.delete(path)
return True return True
except Exception as e: except Exception as e:
logging.exception("Unhandled error", exc_info=e) logger.exception("Unhandled error", exc_info=e)
return False return False
async def identify(self, path: str): async def identify(self, path: str):
@ -35,10 +37,10 @@ class Matcher:
await self.identify(path) await self.identify(path)
await self._client.delete_issue(path) await self._client.delete_issue(path)
except ProviderError as e: except ProviderError as e:
logging.error(e) logger.error(e)
await self._client.create_issue(path, str(e)) await self._client.create_issue(path, str(e))
except Exception as e: except Exception as e:
logging.exception("Unhandled error", exc_info=e) logger.exception("Unhandled error", exc_info=e)
await self._client.create_issue( await self._client.create_issue(
path, "Unknown error", {"type": type(e).__name__, "message": str(e)} path, "Unknown error", {"type": type(e).__name__, "message": str(e)}
) )
@ -63,12 +65,12 @@ class Matcher:
f"Multi-episodes files are not yet supported (for {path})" f"Multi-episodes files are not yet supported (for {path})"
) )
logging.info("Identied %s: %s", path, raw) logger.info("Identied %s: %s", path, raw)
if raw["type"] == "movie": if raw["type"] == "movie":
movie = await self._provider.identify_movie(raw["title"], raw.get("year")) movie = await self._provider.identify_movie(raw["title"], raw.get("year"))
movie.path = str(path) movie.path = str(path)
logging.debug("Got movie: %s", movie) logger.debug("Got movie: %s", movie)
movie_id = await self._client.post("movies", data=movie.to_kyoo()) movie_id = await self._client.post("movies", data=movie.to_kyoo())
if any(movie.collections): if any(movie.collections):
@ -87,7 +89,7 @@ class Matcher:
year=raw.get("year"), year=raw.get("year"),
) )
episode.path = str(path) episode.path = str(path)
logging.debug("Got episode: %s", episode) logger.debug("Got episode: %s", episode)
episode.show_id = await self.create_or_get_show(episode) episode.show_id = await self.create_or_get_show(episode)
if episode.season_number is not None: if episode.season_number is not None:
@ -96,7 +98,7 @@ class Matcher:
) )
await self._client.post("episodes", data=episode.to_kyoo()) await self._client.post("episodes", data=episode.to_kyoo())
else: else:
logging.warn("Unknown video file type: %s", raw["type"]) logger.warn("Unknown video file type: %s", raw["type"])
async def create_or_get_collection(self, collection: Collection) -> str: async def create_or_get_collection(self, collection: Collection) -> str:
@cache(ttl=timedelta(days=1), cache=self._collection_cache) @cache(ttl=timedelta(days=1), cache=self._collection_cache)
@ -107,7 +109,7 @@ class Matcher:
if not any(collection.translations.keys()) if not any(collection.translations.keys())
else collection else collection
) )
logging.debug("Got collection: %s", new_collection) logger.debug("Got collection: %s", new_collection)
return await self._client.post("collection", data=new_collection.to_kyoo()) return await self._client.post("collection", data=new_collection.to_kyoo())
# The parameter is only used as a key for the cache. # The parameter is only used as a key for the cache.
@ -126,7 +128,7 @@ class Matcher:
else episode.show else episode.show
) )
# TODO: collections # TODO: collections
logging.debug("Got show: %s", episode) logger.debug("Got show: %s", episode)
ret = await self._client.post("show", data=show.to_kyoo()) ret = await self._client.post("show", data=show.to_kyoo())
async def create_season(season: Season, id: str): async def create_season(season: Season, id: str):
@ -134,7 +136,7 @@ class Matcher:
season.show_id = id season.show_id = id
return await self._client.post("seasons", data=season.to_kyoo()) return await self._client.post("seasons", data=season.to_kyoo())
except Exception as e: except Exception as e:
logging.exception("Unhandled error create a season", exc_info=e) logger.exception("Unhandled error create a season", exc_info=e)
season_tasks = map( season_tasks = map(
lambda s: exec_as_cache( lambda s: exec_as_cache(

18
scanner/providers/implementations/themoviedatabase.py
View File

@ -1,7 +1,7 @@
import asyncio import asyncio
import logging
from aiohttp import ClientSession from aiohttp import ClientSession
from datetime import datetime, timedelta from datetime import datetime, timedelta
from logging import getLogger
from typing import Awaitable, Callable, Dict, List, Optional, Any, TypeVar from typing import Awaitable, Callable, Dict, List, Optional, Any, TypeVar
from itertools import accumulate, zip_longest from itertools import accumulate, zip_longest
@ -20,6 +20,8 @@ from ..types.metadataid import MetadataID
from ..types.show import Show, ShowTranslation, Status as ShowStatus from ..types.show import Show, ShowTranslation, Status as ShowStatus
from ..types.collection import Collection, CollectionTranslation from ..types.collection import Collection, CollectionTranslation
logger = getLogger(__name__)
class TheMovieDatabase(Provider): class TheMovieDatabase(Provider):
def __init__( def __init__(
@ -158,7 +160,7 @@ class TheMovieDatabase(Provider):
"append_to_response": "alternative_titles,videos,credits,keywords,images", "append_to_response": "alternative_titles,videos,credits,keywords,images",
}, },
) )
logging.debug("TMDb responded: %s", movie) logger.debug("TMDb responded: %s", movie)
ret = Movie( ret = Movie(
original_language=movie["original_language"], original_language=movie["original_language"],
@ -256,7 +258,7 @@ class TheMovieDatabase(Provider):
"append_to_response": "alternative_titles,videos,credits,keywords,images,external_ids", "append_to_response": "alternative_titles,videos,credits,keywords,images,external_ids",
}, },
) )
logging.debug("TMDb responded: %s", show) logger.debug("TMDb responded: %s", show)
ret = Show( ret = Show(
original_language=show["original_language"], original_language=show["original_language"],
@ -427,7 +429,7 @@ class TheMovieDatabase(Provider):
if self.name in ret.external_id: if self.name in ret.external_id:
return ret return ret
logging.warn( logger.warn(
"Could not map xem exception to themoviedb, searching instead for %s", "Could not map xem exception to themoviedb, searching instead for %s",
new_name, new_name,
) )
@ -473,7 +475,7 @@ class TheMovieDatabase(Provider):
else None else None
) )
if tvdb_id is None: if tvdb_id is None:
logging.info( logger.info(
"Tvdb could not be found, trying xem name lookup for %s", name "Tvdb could not be found, trying xem name lookup for %s", name
) )
_, tvdb_id = await self._xem.get_show_override("tvdb", old_name) _, tvdb_id = await self._xem.get_show_override("tvdb", old_name)
@ -518,7 +520,7 @@ class TheMovieDatabase(Provider):
}, },
not_found_fail=f"Could not find episode {episode_nbr} of season {season} of serie {name} (absolute: {absolute})", not_found_fail=f"Could not find episode {episode_nbr} of season {season} of serie {name} (absolute: {absolute})",
) )
logging.debug("TMDb responded: %s", episode) logger.debug("TMDb responded: %s", episode)
ret = Episode( ret = Episode(
show=show, show=show,
@ -616,7 +618,7 @@ class TheMovieDatabase(Provider):
grp = next(iter(group["groups"]), None) grp = next(iter(group["groups"]), None)
return grp["episodes"] if grp else None return grp["episodes"] if grp else None
except Exception as e: except Exception as e:
logging.exception( logger.exception(
"Could not retrieve absolute ordering information", exc_info=e "Could not retrieve absolute ordering information", exc_info=e
) )
return None return None
@ -697,7 +699,7 @@ class TheMovieDatabase(Provider):
"language": lng, "language": lng,
}, },
) )
logging.debug("TMDb responded: %s", collection) logger.debug("TMDb responded: %s", collection)
ret = Collection( ret = Collection(
external_id={ external_id={

16
scanner/providers/implementations/thexem.py
View File

@ -1,12 +1,14 @@
import re import re
import logging
from typing import Dict, List, Literal from typing import Dict, List, Literal
from aiohttp import ClientSession from aiohttp import ClientSession
from logging import getLogger
from datetime import timedelta from datetime import timedelta
from providers.utils import ProviderError from providers.utils import ProviderError
from matcher.cache import cache from matcher.cache import cache
logger = getLogger(__name__)
def clean(s: str): def clean(s: str):
s = s.lower() s = s.lower()
@ -28,7 +30,7 @@ class TheXem:
async def get_map( async def get_map(
self, provider: Literal["tvdb"] | Literal["anidb"] self, provider: Literal["tvdb"] | Literal["anidb"]
) -> Dict[str, List[Dict[str, int]]]: ) -> Dict[str, List[Dict[str, int]]]:
logging.info("Fetching data from thexem for %s", provider) logger.info("Fetching data from thexem for %s", provider)
async with self._client.get( async with self._client.get(
f"{self.base}/map/allNames", f"{self.base}/map/allNames",
params={ params={
@ -40,7 +42,7 @@ class TheXem:
r.raise_for_status() r.raise_for_status()
ret = await r.json() ret = await r.json()
if "data" not in ret or ret["result"] == "failure": if "data" not in ret or ret["result"] == "failure":
logging.error("Could not fetch xem metadata. Error: %s", ret["message"]) logger.error("Could not fetch xem metadata. Error: %s", ret["message"])
raise ProviderError("Could not fetch xem metadata") raise ProviderError("Could not fetch xem metadata")
return ret["data"] return ret["data"]
@ -53,7 +55,7 @@ class TheXem:
Dict[Literal["season"] | Literal["episode"] | Literal["absolute"], int], Dict[Literal["season"] | Literal["episode"] | Literal["absolute"], int],
] ]
]: ]:
logging.info("Fetching from thexem the map of %s (%s)", id, provider) logger.info("Fetching from thexem the map of %s (%s)", id, provider)
async with self._client.get( async with self._client.get(
f"{self.base}/map/all", f"{self.base}/map/all",
params={ params={
@ -64,7 +66,7 @@ class TheXem:
r.raise_for_status() r.raise_for_status()
ret = await r.json() ret = await r.json()
if "data" not in ret or ret["result"] == "failure": if "data" not in ret or ret["result"] == "failure":
logging.error("Could not fetch xem mapping. Error: %s", ret["message"]) logger.error("Could not fetch xem mapping. Error: %s", ret["message"])
return [] return []
return ret["data"] return ret["data"]
@ -111,7 +113,7 @@ class TheXem:
if master_season is None or master_season == -1: if master_season is None or master_season == -1:
return [None, None, episode] return [None, None, episode]
logging.info( logger.info(
"Fount xem override for show %s, ep %d. Master season: %d", "Fount xem override for show %s, ep %d. Master season: %d",
show_name, show_name,
episode, episode,
@ -130,7 +132,7 @@ class TheXem:
None, None,
) )
if ep is None: if ep is None:
logging.warning( logger.warning(
"Could not get xem mapping for show %s, falling back to identifier mapping.", "Could not get xem mapping for show %s, falling back to identifier mapping.",
show_name, show_name,
) )

18
scanner/providers/kyoo_client.py
View File

@ -1,13 +1,15 @@
import os import os
import logging
import jsons import jsons
from aiohttp import ClientSession from aiohttp import ClientSession
from datetime import date from datetime import date
from logging import getLogger
from typing import List, Literal, Any, Optional from typing import List, Literal, Any, Optional
from urllib.parse import quote from urllib.parse import quote
from .utils import format_date from .utils import format_date
logger = getLogger(__name__)
class KyooClient: class KyooClient:
def __init__(self) -> None: def __init__(self) -> None:
@ -76,11 +78,11 @@ class KyooClient:
) as r: ) as r:
# Allow 409 and continue as if it worked. # Allow 409 and continue as if it worked.
if not r.ok and r.status != 409: if not r.ok and r.status != 409:
logging.error(f"Request error: {await r.text()}") logger.error(f"Request error: {await r.text()}")
r.raise_for_status() r.raise_for_status()
async def post(self, path: str, *, data: dict[str, Any]) -> str: async def post(self, path: str, *, data: dict[str, Any]) -> str:
logging.debug( logger.debug(
"Sending %s: %s", "Sending %s: %s",
path, path,
jsons.dumps( jsons.dumps(
@ -96,7 +98,7 @@ class KyooClient:
) as r: ) as r:
# Allow 409 and continue as if it worked. # Allow 409 and continue as if it worked.
if not r.ok and r.status != 409: if not r.ok and r.status != 409:
logging.error(f"Request error: {await r.text()}") logger.error(f"Request error: {await r.text()}")
r.raise_for_status() r.raise_for_status()
ret = await r.json() ret = await r.json()
@ -107,7 +109,7 @@ class KyooClient:
and ret["airDate"][:4] != str(data["air_date"].year) and ret["airDate"][:4] != str(data["air_date"].year)
) )
): ):
logging.info( logger.info(
f"Found a {path} with the same slug ({ret['slug']}) and a different date, using the date as part of the slug" f"Found a {path} with the same slug ({ret['slug']}) and a different date, using the date as part of the slug"
) )
year = (data["start_air"] if path == "movie" else data["air_date"]).year year = (data["start_air"] if path == "movie" else data["air_date"]).year
@ -120,7 +122,7 @@ class KyooClient:
path: str, path: str,
type: Literal["episode", "movie"] | None = None, type: Literal["episode", "movie"] | None = None,
): ):
logging.info("Deleting %s", path) logger.info("Deleting %s", path)
if type is None or type == "movie": if type is None or type == "movie":
async with self.client.delete( async with self.client.delete(
@ -128,7 +130,7 @@ class KyooClient:
headers={"X-API-Key": self._api_key}, headers={"X-API-Key": self._api_key},
) as r: ) as r:
if not r.ok: if not r.ok:
logging.error(f"Request error: {await r.text()}") logger.error(f"Request error: {await r.text()}")
r.raise_for_status() r.raise_for_status()
if type is None or type == "episode": if type is None or type == "episode":
@ -137,7 +139,7 @@ class KyooClient:
headers={"X-API-Key": self._api_key}, headers={"X-API-Key": self._api_key},
) as r: ) as r:
if not r.ok: if not r.ok:
logging.error(f"Request error: {await r.text()}") logger.error(f"Request error: {await r.text()}")
r.raise_for_status() r.raise_for_status()
await self.delete_issue(path) await self.delete_issue(path)