diff --git a/back/src/Kyoo.Abstractions/Models/Resources/Episode.cs b/back/src/Kyoo.Abstractions/Models/Resources/Episode.cs index 10d5f5b6..bb60183f 100644 --- a/back/src/Kyoo.Abstractions/Models/Resources/Episode.cs +++ b/back/src/Kyoo.Abstractions/Models/Resources/Episode.cs @@ -39,7 +39,7 @@ namespace Kyoo.Abstractions.Models { get { - if (ShowSlug != null || Show != null) + if (ShowSlug != null || Show?.Slug != null) return GetSlug(ShowSlug ?? Show.Slug, SeasonNumber, EpisodeNumber, AbsoluteNumber); return ShowID != 0 ? GetSlug(ShowID.ToString(), SeasonNumber, EpisodeNumber, AbsoluteNumber) diff --git a/back/src/Kyoo.Core/Views/Helper/ApiHelper.cs b/back/src/Kyoo.Core/Views/Helper/ApiHelper.cs index 60adae50..f05c4dda 100644 --- a/back/src/Kyoo.Core/Views/Helper/ApiHelper.cs +++ b/back/src/Kyoo.Core/Views/Helper/ApiHelper.cs @@ -97,7 +97,7 @@ namespace Kyoo.Core.Api MemberExpression propertyExpr = Expression.Property(param, property); ConstantExpression valueExpr = null; - bool isList = typeof(IEnumerable).IsAssignableFrom(propertyExpr.Type); + bool isList = typeof(IEnumerable).IsAssignableFrom(propertyExpr.Type) && propertyExpr.Type != typeof(string); if (operand != "ctn" && !typeof(IResource).IsAssignableFrom(propertyExpr.Type) && !isList) { Type propertyType = Nullable.GetUnderlyingType(property.PropertyType) ?? property.PropertyType; diff --git a/back/src/Kyoo.Host/Application.cs b/back/src/Kyoo.Host/Application.cs index a5b958b9..e180aed6 100644 --- a/back/src/Kyoo.Host/Application.cs +++ b/back/src/Kyoo.Host/Application.cs @@ -143,7 +143,7 @@ namespace Kyoo.Host .UseKestrel(options => { options.AddServerHeader = false; }) .UseIIS() .UseIISIntegration() - .UseUrls("http://*:5000") + .UseUrls(Environment.GetEnvironmentVariable("KYOO_BIND_URL") ?? "http://*:5000") .UseStartup(host => PluginsStartup.FromWebHost(host, new LoggerFactory().AddSerilog())) ); } diff --git a/back/src/Kyoo.Postgresql/DatabaseContext.cs b/back/src/Kyoo.Postgresql/DatabaseContext.cs index fb3886df..855ef703 100644 --- a/back/src/Kyoo.Postgresql/DatabaseContext.cs +++ b/back/src/Kyoo.Postgresql/DatabaseContext.cs @@ -480,7 +480,7 @@ namespace Kyoo.Postgresql { try { - return await base.SaveChangesAsync(cancellationToken); + return await SaveChangesAsync(cancellationToken); } catch (DbUpdateException ex) { @@ -489,6 +489,10 @@ namespace Kyoo.Postgresql throw new DuplicatedItemException(await getExisting()); throw; } + catch (DuplicatedItemException) + { + throw new DuplicatedItemException(await getExisting()); + } } /// diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index e5a234c3..4141960b 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -58,6 +58,10 @@ services: - ./.env volumes: - db:/var/lib/postgresql/data + ports: + - "5432:5432" + environment: + - POSTGRES_HOST_AUTH_METHOD=trust healthcheck: test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"] interval: 5s diff --git a/scanner/providers/implementations/themoviedatabase.py b/scanner/providers/implementations/themoviedatabase.py index c162a5e6..1d04f6ec 100644 --- a/scanner/providers/implementations/themoviedatabase.py +++ b/scanner/providers/implementations/themoviedatabase.py @@ -87,7 +87,7 @@ class TheMovieDatabase(Provider): logos=[f"https://image.tmdb.org/t/p/original{company['logo_path']}"] if "logo_path" in company else [], - external_id={ + external_ids={ self.name: MetadataID( company["id"], f"https://www.themoviedb.org/company/{company['id']}" ) @@ -130,7 +130,7 @@ class TheMovieDatabase(Provider): for x in movie["genres"] if x["id"] in self.genre_map ], - external_id={ + external_ids={ self.name: MetadataID( movie["id"], f"https://www.themoviedb.org/movie/{movie['id']}" ), @@ -166,7 +166,7 @@ class TheMovieDatabase(Provider): *, language: list[str], ) -> Show: - show_id = show.external_id[self.name].id + show_id = show.external_ids[self.name].id if show.original_language not in language: language.append(show.original_language) @@ -197,7 +197,7 @@ class TheMovieDatabase(Provider): for x in show["genres"] if x["id"] in self.genre_map ], - external_id={ + external_ids={ self.name: MetadataID( show["id"], f"https://www.themoviedb.org/tv/{show['id']}" ), @@ -260,7 +260,7 @@ class TheMovieDatabase(Provider): season_number=season["season_number"], start_air=datetime.strptime(season["air_date"], "%Y-%m-%d").date(), end_air=None, - external_id={ + external_ids={ self.name: MetadataID( season["id"], f"https://www.themoviedb.org/tv/{show_id}/season/{season['season_number']}", @@ -313,7 +313,7 @@ class TheMovieDatabase(Provider): show=PartialShow( name=search["name"], original_language=search["original_language"], - external_id={ + external_ids={ self.name: MetadataID( show_id, f"https://www.themoviedb.org/tv/{show_id}" ) @@ -327,7 +327,7 @@ class TheMovieDatabase(Provider): thumbnail=f"https://image.tmdb.org/t/p/original{episode['poster_path']}" if "poster_path" in episode else None, - external_id={ + external_ids={ self.name: MetadataID( episode["id"], f"https://www.themoviedb.org/tv/{show_id}/season/{episode['season_number']}/episode/{episode['episode_number']}", diff --git a/scanner/providers/types/episode.py b/scanner/providers/types/episode.py index 7fd9e0ac..416394c6 100644 --- a/scanner/providers/types/episode.py +++ b/scanner/providers/types/episode.py @@ -13,7 +13,7 @@ from .metadataid import MetadataID class PartialShow: name: str original_language: str - external_id: dict[str, MetadataID] + external_ids: dict[str, MetadataID] @dataclass @@ -30,7 +30,7 @@ class Episode: absolute_number: Optional[int] release_date: Optional[date | int] thumbnail: Optional[str] - external_id: dict[str, MetadataID] + external_ids: dict[str, MetadataID] path: Optional[str] = None show_id: Optional[str] = None @@ -43,4 +43,8 @@ class Episode: return { **asdict(self), **asdict(self.translations[default_language]), + "title": self.translations[default_language].name, + # TODO: The back has bad external id support, we disable it for now + "external_ids": None, + "show": None, } diff --git a/scanner/providers/types/movie.py b/scanner/providers/types/movie.py index 964382f4..f9349aa0 100644 --- a/scanner/providers/types/movie.py +++ b/scanner/providers/types/movie.py @@ -40,7 +40,7 @@ class Movie: genres: list[Genre] = field(default_factory=list) # TODO: handle staff # staff: list[Staff] - external_id: dict[str, MetadataID] = field(default_factory=dict) + external_ids: dict[str, MetadataID] = field(default_factory=dict) translations: dict[str, MovieTranslation] = field(default_factory=dict) @@ -56,10 +56,12 @@ class Movie: ), "logo": next(iter(self.translations[default_language].logos), None), "trailer": next(iter(self.translations[default_language].trailers), None), - "studio": next(iter(self.studios), None), + "studio": next((x.to_kyoo() for x in self.studios), None), "release_date": None, "startAir": format_date(self.release_date), "title": self.translations[default_language].name, "genres": [x.to_kyoo() for x in self.genres], "isMovie": True, + # TODO: The back has bad external id support, we disable it for now + "external_ids": None, } diff --git a/scanner/providers/types/season.py b/scanner/providers/types/season.py index 99f04e63..fde20130 100644 --- a/scanner/providers/types/season.py +++ b/scanner/providers/types/season.py @@ -20,8 +20,9 @@ class Season: season_number: int start_air: Optional[date | int] end_air: Optional[date | int] - external_id: dict[str, MetadataID] + external_ids: dict[str, MetadataID] + show_id: Optional[str] = None translations: dict[str, SeasonTranslation] = field(default_factory=dict) def to_kyoo(self): @@ -34,4 +35,7 @@ class Season: "thumbnail": next( iter(self.translations[default_language].thumbnails), None ), + "title": self.translations[default_language].name, + # TODO: The back has bad external id support, we disable it for now + "external_ids": None, } diff --git a/scanner/providers/types/show.py b/scanner/providers/types/show.py index 0c558a7a..0f7ac987 100644 --- a/scanner/providers/types/show.py +++ b/scanner/providers/types/show.py @@ -43,7 +43,7 @@ class Show: seasons: list[Season] # TODO: handle staff # staff: list[Staff] - external_id: dict[str, MetadataID] + external_ids: dict[str, MetadataID] translations: dict[str, ShowTranslation] = field(default_factory=dict) @@ -59,8 +59,10 @@ class Show: ), "logo": next(iter(self.translations[default_language].logos), None), "trailer": next(iter(self.translations[default_language].trailers), None), - "studio": next(iter(self.studios), None), + "studio": next((x.to_kyoo() for x in self.studios), None), "title": self.translations[default_language].name, "genres": [x.to_kyoo() for x in self.genres], - "seasons": [x.to_kyoo() for x in self.seasons], + "seasons": None, + # TODO: The back has bad external id support, we disable it for now + "external_ids": None, } diff --git a/scanner/providers/types/studio.py b/scanner/providers/types/studio.py index 6ca6a7a1..890efd5b 100644 --- a/scanner/providers/types/studio.py +++ b/scanner/providers/types/studio.py @@ -7,4 +7,11 @@ from .metadataid import MetadataID class Studio: name: str logos: list[str] = field(default_factory=list) - external_id: dict[str, MetadataID] = field(default_factory=dict) + external_ids: dict[str, MetadataID] = field(default_factory=dict) + + def to_kyoo(self): + return { + **asdict(self), + # TODO: The back has bad external id support, we disable it for now + "external_ids": None, + } diff --git a/scanner/scanner/scanner.py b/scanner/scanner/scanner.py index 4c9d7f17..22b2df45 100644 --- a/scanner/scanner/scanner.py +++ b/scanner/scanner/scanner.py @@ -8,7 +8,7 @@ from pathlib import Path from guessit import guessit from providers.provider import Provider from providers.types.episode import Episode, PartialShow -from providers.types.show import Show +from providers.types.season import Season def log_errors(f): @@ -28,6 +28,7 @@ class Scanner: ) -> None: self._client = client self._api_key = api_key + self._url = os.environ.get("KYOO_URL", "http://back:5000") self.provider = Provider.get_all(client)[0] self.cache = {"shows": {}} self.languages = languages @@ -36,15 +37,30 @@ class Scanner: videos = filter(lambda p: p.is_file(), Path(path).rglob("*")) await asyncio.gather(*map(self.identify, videos)) + async def is_registered(self, path: Path) -> bool: + # TODO: Once movies are separated from the api, a new endpoint should be created to check for paths. + async with self._client.get( + f"{self._url}/episodes/count", + params={"path": str(path)}, + headers={"X-API-Key": self._api_key}, + ) as r: + r.raise_for_status() + ret = await r.text() + if ret != "0": + return True + return False + + @log_errors async def identify(self, path: Path): + if await self.is_registered(path): + return raw = guessit(path, "--episode-prefer-number") logging.info("Identied %s: %s", path, raw) # TODO: check if episode/movie already exists in kyoo and skip if it does. # TODO: Add collections support if raw["type"] == "movie": - return movie = await self.provider.identify_movie( raw["title"], raw.get("year"), language=self.languages ) @@ -62,21 +78,24 @@ class Scanner: episode.path = str(path) logging.debug("Got episode: %s", episode) episode.show_id = await self.create_or_get_show(episode) + # TODO: Do the same things for seasons and wait for them to be created on the api (else the episode creation will fail) await self.post("episodes", data=episode.to_kyoo()) else: logging.warn("Unknown video file type: %s", raw["type"]) async def create_or_get_show(self, episode: Episode) -> str: - provider_id = episode.show.external_id[self.provider.name].id + provider_id = episode.show.external_ids[self.provider.name].id if provider_id in self.cache["shows"]: ret = self.cache["shows"][provider_id] - print(f"Waiting for {provider_id}") await ret["event"].wait() if not ret["id"]: raise RuntimeError("Provider failed to create the show") return ret["id"] self.cache["shows"][provider_id] = {"id": None, "event": asyncio.Event()} + + # TODO: Check if a show with the same metadata id exists already on kyoo. + show = ( await self.provider.identify_show(episode.show, language=self.languages) if isinstance(episode.show, PartialShow) @@ -89,14 +108,20 @@ class Scanner: # Allow tasks waiting for this show to bail out. self.cache["shows"][provider_id]["event"].set() raise - print(f"setting {provider_id}") self.cache["shows"][provider_id]["id"] = ret self.cache["shows"][provider_id]["event"].set() + + # TODO: Better handling of seasons registrations (maybe a lock also) + await self.register_seasons(ret, show.seasons) return ret + async def register_seasons(self, show_id: str, seasons: list[Season]): + for season in seasons: + season.show_id = show_id + await self.post("seasons", data=season.to_kyoo()) + async def post(self, path: str, *, data: object) -> str: - url = os.environ.get("KYOO_URL", "http://back:5000") - logging.info( + logging.debug( "Sending %s: %s", path, jsons.dumps( @@ -106,12 +131,13 @@ class Scanner: ), ) async with self._client.post( - f"{url}/{path}", + f"{self._url}/{path}", json=data, headers={"X-API-Key": self._api_key}, ) as r: - if not r.ok: - print(await r.text()) - r.raise_for_status() + # Allow 409 and continue as if it worked. + if not r.ok and r.status != 409: + logging.error(f"Request error: {await r.text()}") + r.raise_for_status() ret = await r.json() return ret["id"]