From 710e3cf10be5d9f12462ca3d6770d0dff0398798 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Tue, 14 May 2024 01:46:51 +0200 Subject: [PATCH 1/6] Handle missing translations for movies --- scanner/matcher/matcher.py | 1 + scanner/providers/types/movie.py | 15 ++++++------ scanner/providers/utils.py | 42 ++++++++++++++++++++------------ 3 files changed, 35 insertions(+), 23 deletions(-) diff --git a/scanner/matcher/matcher.py b/scanner/matcher/matcher.py index 0fd597d2..982a1aa1 100644 --- a/scanner/matcher/matcher.py +++ b/scanner/matcher/matcher.py @@ -87,6 +87,7 @@ class Matcher: async def search_movie(self, title: str, year: Optional[int], path: str): movie = await self._provider.search_movie(title, year) + movie.file_title = title movie.path = path logger.debug("Got movie: %s", movie) movie_id = await self._client.post("movies", data=movie.to_kyoo()) diff --git a/scanner/providers/types/movie.py b/scanner/providers/types/movie.py index 5ae09ce4..b9f3c63c 100644 --- a/scanner/providers/types/movie.py +++ b/scanner/providers/types/movie.py @@ -1,9 +1,10 @@ -import os from dataclasses import asdict, dataclass, field from datetime import date from typing import Optional from enum import Enum +from providers.utils import select_translation, select_image + from .collection import Collection from .genre import Genre from .studio import Studio @@ -44,22 +45,22 @@ class Movie: external_id: dict[str, MetadataID] path: Optional[str] = None + # The title of this show according to it's filename (None only for ease of use in providers) + file_title: Optional[str] = None collections: list[Collection] = field(default_factory=list) translations: dict[str, MovieTranslation] = field(default_factory=dict) def to_kyoo(self): - from ..utils import select_image - - # For now, the API of kyoo only support one language so we remove the others. - default_language = os.environ["LIBRARY_LANGUAGES"].split(",")[0] + trans = select_translation(self) or MovieTranslation(name=self.file_title or "") return { **asdict(self), - **asdict(self.translations[default_language]), + **asdict(trans), "poster": select_image(self, "posters"), "thumbnail": select_image(self, "thumbnails"), "logo": select_image(self, "logos"), - "trailer": next(iter(self.translations[default_language].trailers), None), + "trailer": next(iter(trans.trailers), None), "studio": next((x.to_kyoo() for x in self.studios), None), "genres": [x.to_kyoo() for x in self.genres], "collections": None, + "file_title": None, } diff --git a/scanner/providers/utils.py b/scanner/providers/utils.py index 1024de92..452cc713 100644 --- a/scanner/providers/utils.py +++ b/scanner/providers/utils.py @@ -1,6 +1,7 @@ +from __future__ import annotations + import os from datetime import date -from itertools import chain from typing import Literal @@ -16,24 +17,33 @@ def format_date(date: date | int | None) -> str | None: return date.isoformat() +# For now, the API of kyoo only support one language so we remove the others. +default_languages = os.environ["LIBRARY_LANGUAGES"].split(",") + + +def select_translation(value: Movie | Show, *, prefer_orginal=False): + if ( + prefer_orginal + and value.original_language + and value.original_language in value.translations + ): + return value.translations[value.original_language] + for lang in default_languages: + if lang in value.translations: + return value.translations[lang] + return None + + def select_image( - self: Movie | Show, - type: Literal["posters"] | Literal["thumbnails"] | Literal["logos"], + value: Movie | Show, + kind: Literal["posters"] | Literal["thumbnails"] | Literal["logos"], ) -> str | None: - # For now, the API of kyoo only support one language so we remove the others. - default_language = os.environ["LIBRARY_LANGUAGES"].split(",")[0] - return next( - chain( - ( - getattr(self.translations[self.original_language], type) - if self.original_language - else [] - ), - getattr(self.translations[default_language], type), - *(getattr(x, type) for x in self.translations.values()), - ), - None, + trans = select_translation(value, prefer_orginal=True) or next( + iter(value.translations.values()), None ) + if trans is None: + return None + return getattr(trans, kind) class ProviderError(RuntimeError): From ca7dd81452cff862a603f44c841f89fe878f7de7 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Tue, 14 May 2024 01:52:09 +0200 Subject: [PATCH 2/6] Handle missing translations for series --- scanner/matcher/matcher.py | 5 +++-- scanner/providers/types/show.py | 30 ++++++++++++++++-------------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/scanner/matcher/matcher.py b/scanner/matcher/matcher.py index 982a1aa1..808fb7fa 100644 --- a/scanner/matcher/matcher.py +++ b/scanner/matcher/matcher.py @@ -117,7 +117,7 @@ class Matcher: ) episode.path = path logger.debug("Got episode: %s", episode) - episode.show_id = await self.create_or_get_show(episode) + episode.show_id = await self.create_or_get_show(episode, title) if episode.season_number is not None: episode.season_id = await self.register_seasons( @@ -141,7 +141,7 @@ class Matcher: provider_id = collection.external_id[self._provider.name].data_id return await create_collection(provider_id) - async def create_or_get_show(self, episode: Episode) -> str: + async def create_or_get_show(self, episode: Episode, fallback_name: str) -> str: @cache(ttl=timedelta(days=1), cache=self._show_cache) async def create_show(_: str): # TODO: Check if a show with the same metadata id exists already on kyoo. @@ -152,6 +152,7 @@ class Matcher: if isinstance(episode.show, PartialShow) else episode.show ) + show.file_title = fallback_name # TODO: collections logger.debug("Got show: %s", episode) ret = await self._client.post("show", data=show.to_kyoo()) diff --git a/scanner/providers/types/show.py b/scanner/providers/types/show.py index 3556f628..4f99e57f 100644 --- a/scanner/providers/types/show.py +++ b/scanner/providers/types/show.py @@ -1,9 +1,10 @@ -import os from dataclasses import asdict, dataclass, field from datetime import date from typing import Optional from enum import Enum +from providers.utils import select_translation, select_image + from .genre import Genre from .studio import Studio from .season import Season @@ -20,14 +21,14 @@ class Status(str, Enum): @dataclass class ShowTranslation: name: str - tagline: Optional[str] - tags: list[str] - overview: Optional[str] + tagline: Optional[str] = None + tags: list[str] = [] + overview: Optional[str] = None - posters: list[str] - logos: list[str] - trailers: list[str] - thumbnails: list[str] + posters: list[str] = [] + logos: list[str] = [] + trailers: list[str] = [] + thumbnails: list[str] = [] @dataclass @@ -46,20 +47,21 @@ class Show: external_id: dict[str, MetadataID] translations: dict[str, ShowTranslation] = field(default_factory=dict) + # The title of this show according to it's filename (None only for ease of use in providers) + file_title: Optional[str] = None def to_kyoo(self): - from providers.utils import select_image - - # For now, the API of kyoo only support one language so we remove the others. - default_language = os.environ["LIBRARY_LANGUAGES"].split(",")[0] + trans = select_translation(self) or ShowTranslation(name=self.file_title or "") return { **asdict(self), - **asdict(self.translations[default_language]), + **asdict(trans), + "rating": self.rating or 0, "studio": next((x.to_kyoo() for x in self.studios), None), "seasons": None, "poster": select_image(self, "posters"), "thumbnail": select_image(self, "thumbnails"), "logo": select_image(self, "logos"), - "trailer": next(iter(self.translations[default_language].trailers), None), + "trailer": next(iter(trans.trailers), None), "genres": [x.to_kyoo() for x in self.genres], + "file_title": None, } From f8c66026045b68e2798c403b382555fbc06f2216 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Tue, 14 May 2024 02:11:30 +0200 Subject: [PATCH 3/6] Handle missing translations for episodes or seasons --- scanner/providers/types/episode.py | 12 ++++++------ scanner/providers/types/season.py | 14 ++++++-------- scanner/providers/utils.py | 12 +++++++++--- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/scanner/providers/types/episode.py b/scanner/providers/types/episode.py index 0401d4ee..8da11c44 100644 --- a/scanner/providers/types/episode.py +++ b/scanner/providers/types/episode.py @@ -1,8 +1,9 @@ -import os from datetime import date from dataclasses import dataclass, field, asdict from typing import Optional +from providers.utils import select_translation + from .show import Show from .metadataid import MetadataID @@ -24,8 +25,8 @@ class EpisodeID: @dataclass class EpisodeTranslation: - name: str - overview: Optional[str] + name: Optional[str] + overview: Optional[str] = None @dataclass @@ -45,10 +46,9 @@ class Episode: translations: dict[str, EpisodeTranslation] = field(default_factory=dict) def to_kyoo(self): - # For now, the API of kyoo only support one language so we remove the others. - default_language = os.environ["LIBRARY_LANGUAGES"].split(",")[0] + trans = select_translation(self) or EpisodeTranslation("") return { **asdict(self), - **asdict(self.translations[default_language]), + **asdict(trans), "show": None, } diff --git a/scanner/providers/types/season.py b/scanner/providers/types/season.py index 0c224ece..b553dae0 100644 --- a/scanner/providers/types/season.py +++ b/scanner/providers/types/season.py @@ -1,8 +1,9 @@ -import os from datetime import date from dataclasses import dataclass, field, asdict from typing import Optional +from providers.utils import select_translation, select_image + from .metadataid import MetadataID @@ -28,13 +29,10 @@ class Season: translations: dict[str, SeasonTranslation] = field(default_factory=dict) def to_kyoo(self): - # For now, the API of kyoo only support one language so we remove the others. - default_language = os.environ["LIBRARY_LANGUAGES"].split(",")[0] + trans = select_translation(self) or SeasonTranslation() return { **asdict(self), - **asdict(self.translations[default_language]), - "poster": next(iter(self.translations[default_language].posters), None), - "thumbnail": next( - iter(self.translations[default_language].thumbnails), None - ), + **asdict(trans), + "poster": select_image(self, "posters"), + "thumbnail": select_image(self, "thumbnails"), } diff --git a/scanner/providers/utils.py b/scanner/providers/utils.py index 452cc713..691c98fc 100644 --- a/scanner/providers/utils.py +++ b/scanner/providers/utils.py @@ -3,10 +3,15 @@ from __future__ import annotations import os from datetime import date -from typing import Literal +from typing import Literal, Any from providers.types.movie import Movie from providers.types.show import Show +from providers.types.season import Season +from providers.types.episode import Episode +from providers.types.collection import Collection + +type Resource = Movie | Show | Season | Episode | Collection def format_date(date: date | int | None) -> str | None: @@ -21,9 +26,10 @@ def format_date(date: date | int | None) -> str | None: default_languages = os.environ["LIBRARY_LANGUAGES"].split(",") -def select_translation(value: Movie | Show, *, prefer_orginal=False): +def select_translation(value: Resource, *, prefer_orginal=False) -> Any: if ( prefer_orginal + and (isinstance(value, Movie) or isinstance(value, Show)) and value.original_language and value.original_language in value.translations ): @@ -35,7 +41,7 @@ def select_translation(value: Movie | Show, *, prefer_orginal=False): def select_image( - value: Movie | Show, + value: Resource, kind: Literal["posters"] | Literal["thumbnails"] | Literal["logos"], ) -> str | None: trans = select_translation(value, prefer_orginal=True) or next( From b228fbb35c21fcfb93e97a58a41ff2d907166473 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Tue, 14 May 2024 02:16:33 +0200 Subject: [PATCH 4/6] Add stopgap mesure for collections missing translations --- scanner/providers/types/collection.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/scanner/providers/types/collection.py b/scanner/providers/types/collection.py index c103e998..c6d01603 100644 --- a/scanner/providers/types/collection.py +++ b/scanner/providers/types/collection.py @@ -1,7 +1,8 @@ -import os from dataclasses import asdict, dataclass, field from typing import Optional +from providers.utils import ProviderError, select_translation, select_image + from .metadataid import MetadataID @@ -20,14 +21,15 @@ class Collection: translations: dict[str, CollectionTranslation] = field(default_factory=dict) def to_kyoo(self): - # For now, the API of kyoo only support one language so we remove the others. - default_language = os.environ["LIBRARY_LANGUAGES"].split(",")[0] + trans = select_translation(self) + if trans is None: + raise ProviderError( + "Could not find translations for the collection. Aborting" + ) return { **asdict(self), - **asdict(self.translations[default_language]), - "poster": next(iter(self.translations[default_language].posters), None), - "thumbnail": next( - iter(self.translations[default_language].thumbnails), None - ), - "logo": next(iter(self.translations[default_language].logos), None), + **asdict(trans), + "poster": select_image(self, "posters"), + "thumbnail": select_image(self, "thumbnails"), + "logo": select_image(self, "logos"), } From a34680072441e8772d17f27a3413f6be38628fd2 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Tue, 14 May 2024 02:31:19 +0200 Subject: [PATCH 5/6] Fix runtime issue --- scanner/providers/types/show.py | 10 +++++----- scanner/providers/utils.py | 16 ++++++++++------ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/scanner/providers/types/show.py b/scanner/providers/types/show.py index 4f99e57f..b66bd466 100644 --- a/scanner/providers/types/show.py +++ b/scanner/providers/types/show.py @@ -22,13 +22,13 @@ class Status(str, Enum): class ShowTranslation: name: str tagline: Optional[str] = None - tags: list[str] = [] + tags: list[str] = field(default_factory=list) overview: Optional[str] = None - posters: list[str] = [] - logos: list[str] = [] - trailers: list[str] = [] - thumbnails: list[str] = [] + posters: list[str] = field(default_factory=list) + logos: list[str] = field(default_factory=list) + trailers: list[str] = field(default_factory=list) + thumbnails: list[str] = field(default_factory=list) @dataclass diff --git a/scanner/providers/utils.py b/scanner/providers/utils.py index 691c98fc..2ff79910 100644 --- a/scanner/providers/utils.py +++ b/scanner/providers/utils.py @@ -3,13 +3,14 @@ from __future__ import annotations import os from datetime import date -from typing import Literal, Any +from typing import TYPE_CHECKING, Literal, Any -from providers.types.movie import Movie -from providers.types.show import Show -from providers.types.season import Season -from providers.types.episode import Episode -from providers.types.collection import Collection +if TYPE_CHECKING: + from providers.types.movie import Movie + from providers.types.show import Show + from providers.types.season import Season + from providers.types.episode import Episode + from providers.types.collection import Collection type Resource = Movie | Show | Season | Episode | Collection @@ -27,6 +28,9 @@ default_languages = os.environ["LIBRARY_LANGUAGES"].split(",") def select_translation(value: Resource, *, prefer_orginal=False) -> Any: + from providers.types.movie import Movie + from providers.types.show import Show + if ( prefer_orginal and (isinstance(value, Movie) or isinstance(value, Show)) From 9919f69720ebd670726f984705859f76071025cb Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Tue, 14 May 2024 03:02:29 +0200 Subject: [PATCH 6/6] Fix images chain --- scanner/providers/utils.py | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/scanner/providers/utils.py b/scanner/providers/utils.py index 2ff79910..f9bc39c0 100644 --- a/scanner/providers/utils.py +++ b/scanner/providers/utils.py @@ -2,8 +2,8 @@ from __future__ import annotations import os from datetime import date - -from typing import TYPE_CHECKING, Literal, Any +from itertools import chain +from typing import TYPE_CHECKING, Literal, Any, Optional if TYPE_CHECKING: from providers.types.movie import Movie @@ -12,8 +12,6 @@ if TYPE_CHECKING: from providers.types.episode import Episode from providers.types.collection import Collection -type Resource = Movie | Show | Season | Episode | Collection - def format_date(date: date | int | None) -> str | None: if date is None: @@ -27,7 +25,11 @@ def format_date(date: date | int | None) -> str | None: default_languages = os.environ["LIBRARY_LANGUAGES"].split(",") -def select_translation(value: Resource, *, prefer_orginal=False) -> Any: +def sort_translations( + value: Movie | Show | Season | Episode | Collection, + *, + prefer_orginal=False, +): from providers.types.movie import Movie from providers.types.show import Show @@ -37,23 +39,31 @@ def select_translation(value: Resource, *, prefer_orginal=False) -> Any: and value.original_language and value.original_language in value.translations ): - return value.translations[value.original_language] + yield value.translations[value.original_language] for lang in default_languages: if lang in value.translations: - return value.translations[lang] - return None + yield value.translations[lang] + + +def select_translation( + value: Movie | Show | Season | Episode | Collection, *, prefer_orginal=False +) -> Optional[Any]: + return next(sort_translations(value, prefer_orginal=prefer_orginal), None) def select_image( - value: Resource, + value: Movie | Show | Season | Collection, kind: Literal["posters"] | Literal["thumbnails"] | Literal["logos"], ) -> str | None: - trans = select_translation(value, prefer_orginal=True) or next( - iter(value.translations.values()), None + return next( + chain( + *( + getattr(trans, kind) + for trans in sort_translations(value, prefer_orginal=True) + ) + ), + None, ) - if trans is None: - return None - return getattr(trans, kind) class ProviderError(RuntimeError):