Add identify apis for provider (#419)

back/Dockerfile.migrations

@@ -24,4 +24,4 @@ RUN dotnet ef migrations bundle --no-build --self-contained -r linux-${TARGETARC
 FROM mcr.microsoft.com/dotnet/runtime-deps:8.0
 COPY --from=builder /app/migrate /app/migrate
 
-ENTRYPOINT /app/migrate --connection "USER ID=${POSTGRES_USER};PASSWORD=${POSTGRES_PASSWORD};SERVER=${POSTGRES_SERVER};PORT=${POSTGRES_PORT};DATABASE=${POSTGRES_DB};"
+ENTRYPOINT ["/app/migrate"]

back/src/Kyoo.Abstractions/Models/INews.cs

@@ -24,8 +24,8 @@ namespace Kyoo.Abstractions.Models;
 /// <summary>
 /// A show, a movie or a collection.
 /// </summary>
-[OneOf(Types = new[] { typeof(Episode), typeof(Movie) })]
-public interface INews : IResource, IThumbnails, IMetadata, IAddedDate, IQuery
+[OneOf(Types = [typeof(Episode), typeof(Movie)])]
+public interface INews : IResource, IThumbnails, IAddedDate, IQuery
 {
 	static Sort IQuery.DefaultSort => new Sort<INews>.By(nameof(AddedDate), true);
 }

back/src/Kyoo.Abstractions/Models/MetadataID.cs

@@ -33,3 +33,29 @@ public class MetadataId
 	/// </summary>
 	public string? Link { get; set; }
 }
+
+/// <summary>
+/// ID informations about an episode.
+/// </summary>
+public class EpisodeId
+{
+	/// <summary>
+	/// The Id of the show on the metadata database.
+	/// </summary>
+	public string ShowId { get; set; }
+
+	/// <summary>
+	/// The season number or null if absolute numbering is used in this database.
+	/// </summary>
+	public int? Season { get; set; }
+
+	/// <summary>
+	/// The episode number or absolute number if Season is null.
+	/// </summary>
+	public int Episode { get; set; }
+
+	/// <summary>
+	/// The URL of the resource on the external provider.
+	/// </summary>
+	public string? Link { get; set; }
+}

back/src/Kyoo.Abstractions/Models/Resources/Collection.cs

@@ -28,7 +28,14 @@ namespace Kyoo.Abstractions.Models;
 /// <summary>
 /// A class representing collections of <see cref="Show"/>.
 /// </summary>
-public class Collection : IQuery, IResource, IMetadata, IThumbnails, IAddedDate, ILibraryItem
+public class Collection
+	: IQuery,
+		IResource,
+		IMetadata,
+		IThumbnails,
+		IAddedDate,
+		IRefreshable,
+		ILibraryItem
 {
 	public static Sort DefaultSort => new Sort<Collection>.By(nameof(Collection.Name));
 
@@ -76,6 +83,9 @@ public class Collection : IQuery, IResource, IMetadata, IThumbnails, IAddedDate,
 	/// <inheritdoc />
 	public Dictionary<string, MetadataId> ExternalId { get; set; } = new();
 
+	/// <inheritdoc />
+	public DateTime? NextMetadataRefresh { get; set; }
+
 	public Collection() { }
 
 	[JsonConstructor]

back/src/Kyoo.Abstractions/Models/Resources/Episode.cs

@@ -31,7 +31,7 @@ namespace Kyoo.Abstractions.Models;
 /// <summary>
 /// A class to represent a single show's episode.
 /// </summary>
-public class Episode : IQuery, IResource, IMetadata, IThumbnails, IAddedDate, INews
+public class Episode : IQuery, IResource, IThumbnails, IAddedDate, IRefreshable, INews
 {
 	// Use absolute numbers by default and fallback to season/episodes if it does not exists.
 	public static Sort DefaultSort =>
@@ -166,7 +166,10 @@ public class Episode : IQuery, IResource, IMetadata, IThumbnails, IAddedDate, IN
 	public Image? Logo { get; set; }
 
 	/// <inheritdoc />
-	public Dictionary<string, MetadataId> ExternalId { get; set; } = new();
+	public Dictionary<string, EpisodeId> ExternalId { get; set; } = [];
+
+	/// <inheritdoc />
+	public DateTime? NextMetadataRefresh { get; set; }
 
 	/// <summary>
 	/// The previous episode that should be seen before viewing this one.

back/src/Kyoo.Abstractions/Models/Resources/Interfaces/IRefreshable.cs

@@ -0,0 +1,29 @@
+// Kyoo - A portable and vast media library solution.
+// Copyright (c) Kyoo.
+//
+// See AUTHORS.md and LICENSE file in the project root for full license information.
+//
+// Kyoo is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// any later version.
+//
+// Kyoo is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Kyoo. If not, see <https://www.gnu.org/licenses/>.
+
+using System;
+
+namespace Kyoo.Abstractions.Models;
+
+public interface IRefreshable
+{
+	/// <summary>
+	/// The date of the next metadata refresh. Null if auto-refresh is disabled.
+	/// </summary>
+	public DateTime? NextMetadataRefresh { get; set; }
+}

back/src/Kyoo.Abstractions/Models/Resources/Movie.cs

@@ -38,6 +38,7 @@ public class Movie
 		IMetadata,
 		IThumbnails,
 		IAddedDate,
+		IRefreshable,
 		ILibraryItem,
 		INews,
 		IWatchlist
@@ -134,6 +135,9 @@ public class Movie
 	/// <inheritdoc />
 	public Dictionary<string, MetadataId> ExternalId { get; set; } = new();
 
+	/// <inheritdoc />
+	public DateTime? NextMetadataRefresh { get; set; }
+
 	/// <summary>
 	/// The ID of the Studio that made this show.
 	/// </summary>

back/src/Kyoo.Abstractions/Models/Resources/Season.cs

@@ -31,7 +31,7 @@ namespace Kyoo.Abstractions.Models;
 /// <summary>
 /// A season of a <see cref="Show"/>.
 /// </summary>
-public class Season : IQuery, IResource, IMetadata, IThumbnails, IAddedDate
+public class Season : IQuery, IResource, IMetadata, IThumbnails, IAddedDate, IRefreshable
 {
 	public static Sort DefaultSort => new Sort<Season>.By(x => x.SeasonNumber);
 
@@ -119,6 +119,9 @@ public class Season : IQuery, IResource, IMetadata, IThumbnails, IAddedDate
 	/// <inheritdoc />
 	public Dictionary<string, MetadataId> ExternalId { get; set; } = new();
 
+	/// <inheritdoc />
+	public DateTime? NextMetadataRefresh { get; set; }
+
 	/// <summary>
 	/// The list of episodes that this season contains.
 	/// </summary>

back/src/Kyoo.Abstractions/Models/Resources/Show.cs

@@ -39,6 +39,7 @@ public class Show
 		IOnMerge,
 		IThumbnails,
 		IAddedDate,
+		IRefreshable,
 		ILibraryItem,
 		IWatchlist
 {
@@ -126,6 +127,9 @@ public class Show
 	/// <inheritdoc />
 	public Dictionary<string, MetadataId> ExternalId { get; set; } = new();
 
+	/// <inheritdoc />
+	public DateTime? NextMetadataRefresh { get; set; }
+
 	/// <summary>
 	/// The ID of the Studio that made this show.
 	/// </summary>

back/src/Kyoo.Postgresql/DatabaseContext.cs

@@ -157,21 +157,11 @@ public abstract class DatabaseContext : DbContext
 		optionsBuilder.UseQueryTrackingBehavior(QueryTrackingBehavior.NoTracking);
 	}
 
-	private static ValueComparer<Dictionary<string, T>> _GetComparer<T>()
-	{
-		return new(
-			(c1, c2) => c1!.SequenceEqual(c2!),
-			c => c.Aggregate(0, (a, v) => HashCode.Combine(a, v.GetHashCode()))
-		);
-	}
-
-	/// <summary>
-	/// Build the metadata model for the given type.
-	/// </summary>
-	/// <param name="modelBuilder">The database model builder</param>
-	/// <typeparam name="T">The type to add metadata to.</typeparam>
-	private static void _HasMetadata<T>(ModelBuilder modelBuilder)
-		where T : class, IMetadata
+	private static void _HasJson<T, TVal>(
+		ModelBuilder builder,
+		Expression<Func<T, Dictionary<string, TVal>>> property
+	)
+		where T : class
 	{
 		// TODO: Waiting for https://github.com/dotnet/efcore/issues/29825
 		// modelBuilder.Entity<T>()
@@ -179,22 +169,33 @@ public abstract class DatabaseContext : DbContext
 		// 	{
 		// 		x.ToJson();
 		// 	});
-		modelBuilder
+		builder
 			.Entity<T>()
-			.Property(x => x.ExternalId)
+			.Property(property)
 			.HasConversion(
 				v => JsonSerializer.Serialize(v, (JsonSerializerOptions?)null),
 				v =>
-					JsonSerializer.Deserialize<Dictionary<string, MetadataId>>(
+					JsonSerializer.Deserialize<Dictionary<string, TVal>>(
 						v,
 						(JsonSerializerOptions?)null
 					)!
 			)
 			.HasColumnType("json");
-		modelBuilder
+		builder
 			.Entity<T>()
-			.Property(x => x.ExternalId)
-			.Metadata.SetValueComparer(_GetComparer<MetadataId>());
+			.Property(property)
+			.Metadata.SetValueComparer(
+				new ValueComparer<Dictionary<string, TVal>>(
+					(c1, c2) => c1!.SequenceEqual(c2!),
+					c => c.Aggregate(0, (a, v) => HashCode.Combine(a, v.GetHashCode()))
+				)
+			);
+	}
+
+	private static void _HasMetadata<T>(ModelBuilder modelBuilder)
+		where T : class, IMetadata
+	{
+		_HasJson<T, MetadataId>(modelBuilder, x => x.ExternalId);
 	}
 
 	private static void _HasImages<T>(ModelBuilder modelBuilder)
@@ -215,6 +216,17 @@ public abstract class DatabaseContext : DbContext
 			.ValueGeneratedOnAdd();
 	}
 
+	private static void _HasRefreshDate<T>(ModelBuilder builder)
+		where T : class, IRefreshable
+	{
+		// schedule a refresh soon since metadata can change frequently for recently added items ond online databases
+		builder
+			.Entity<T>()
+			.Property(x => x.NextMetadataRefresh)
+			.HasDefaultValueSql("now() at time zone 'utc' + interval '2 hours'")
+			.ValueGeneratedOnAdd();
+	}
+
 	/// <summary>
 	/// Create a many to many relationship between the two entities.
 	/// The resulting relationship will have an available <see cref="AddLinks{T1,T2}"/> method.
@@ -296,8 +308,8 @@ public abstract class DatabaseContext : DbContext
 		_HasMetadata<Movie>(modelBuilder);
 		_HasMetadata<Show>(modelBuilder);
 		_HasMetadata<Season>(modelBuilder);
-		_HasMetadata<Episode>(modelBuilder);
 		_HasMetadata<Studio>(modelBuilder);
+		_HasJson<Episode, EpisodeId>(modelBuilder, x => x.ExternalId);
 
 		_HasImages<Collection>(modelBuilder);
 		_HasImages<Movie>(modelBuilder);
@@ -313,6 +325,12 @@ public abstract class DatabaseContext : DbContext
 		_HasAddedDate<User>(modelBuilder);
 		_HasAddedDate<Issue>(modelBuilder);
 
+		_HasRefreshDate<Collection>(modelBuilder);
+		_HasRefreshDate<Movie>(modelBuilder);
+		_HasRefreshDate<Show>(modelBuilder);
+		_HasRefreshDate<Season>(modelBuilder);
+		_HasRefreshDate<Episode>(modelBuilder);
+
 		modelBuilder
 			.Entity<MovieWatchStatus>()
 			.HasKey(x => new { User = x.UserId, Movie = x.MovieId });
@@ -389,62 +407,9 @@ public abstract class DatabaseContext : DbContext
 
 		modelBuilder.Entity<Issue>().HasKey(x => new { x.Domain, x.Cause });
 
-		// TODO: Waiting for https://github.com/dotnet/efcore/issues/29825
-		// modelBuilder.Entity<T>()
-		// 	.OwnsOne(x => x.ExternalId, x =>
-		// 	{
-		// 		x.ToJson();
-		// 	});
-		modelBuilder
-			.Entity<User>()
-			.Property(x => x.Settings)
-			.HasConversion(
-				v => JsonSerializer.Serialize(v, (JsonSerializerOptions?)null),
-				v =>
-					JsonSerializer.Deserialize<Dictionary<string, string>>(
-						v,
-						(JsonSerializerOptions?)null
-					)!
-			)
-			.HasColumnType("json");
-		modelBuilder
-			.Entity<User>()
-			.Property(x => x.Settings)
-			.Metadata.SetValueComparer(_GetComparer<string>());
-
-		modelBuilder
-			.Entity<User>()
-			.Property(x => x.ExternalId)
-			.HasConversion(
-				v => JsonSerializer.Serialize(v, (JsonSerializerOptions?)null),
-				v =>
-					JsonSerializer.Deserialize<Dictionary<string, ExternalToken>>(
-						v,
-						(JsonSerializerOptions?)null
-					)!
-			)
-			.HasColumnType("json");
-		modelBuilder
-			.Entity<User>()
-			.Property(x => x.ExternalId)
-			.Metadata.SetValueComparer(_GetComparer<ExternalToken>());
-
-		modelBuilder
-			.Entity<Issue>()
-			.Property(x => x.Extra)
-			.HasConversion(
-				v => JsonSerializer.Serialize(v, (JsonSerializerOptions?)null),
-				v =>
-					JsonSerializer.Deserialize<Dictionary<string, object>>(
-						v,
-						(JsonSerializerOptions?)null
-					)!
-			)
-			.HasColumnType("json");
-		modelBuilder
-			.Entity<Issue>()
-			.Property(x => x.Extra)
-			.Metadata.SetValueComparer(_GetComparer<object>());
+		_HasJson<User, string>(modelBuilder, x => x.Settings);
+		_HasJson<User, ExternalToken>(modelBuilder, x => x.ExternalId);
+		_HasJson<Issue, object>(modelBuilder, x => x.Extra);
 	}
 
 	/// <summary>

back/src/Kyoo.Postgresql/Migrations/20240414212454_AddNextRefresh.cs

@@ -0,0 +1,89 @@
+using System;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace Kyoo.Postgresql.Migrations;
+
+/// <inheritdoc />
+public partial class AddNextRefresh : Migration
+{
+	/// <inheritdoc />
+	protected override void Up(MigrationBuilder migrationBuilder)
+	{
+		migrationBuilder.AddColumn<DateTime>(
+			name: "next_metadata_refresh",
+			table: "shows",
+			type: "timestamp with time zone",
+			nullable: true,
+			defaultValueSql: "now() at time zone 'utc' + interval '2 hours'"
+		);
+
+		migrationBuilder.AddColumn<DateTime>(
+			name: "next_metadata_refresh",
+			table: "seasons",
+			type: "timestamp with time zone",
+			nullable: true,
+			defaultValueSql: "now() at time zone 'utc' + interval '2 hours'"
+		);
+
+		migrationBuilder.AddColumn<DateTime>(
+			name: "next_metadata_refresh",
+			table: "movies",
+			type: "timestamp with time zone",
+			nullable: true,
+			defaultValueSql: "now() at time zone 'utc' + interval '2 hours'"
+		);
+
+		migrationBuilder.AddColumn<DateTime>(
+			name: "next_metadata_refresh",
+			table: "episodes",
+			type: "timestamp with time zone",
+			nullable: true,
+			defaultValueSql: "now() at time zone 'utc' + interval '2 hours'"
+		);
+
+		migrationBuilder.AddColumn<DateTime>(
+			name: "next_metadata_refresh",
+			table: "collections",
+			type: "timestamp with time zone",
+			nullable: true,
+			defaultValueSql: "now() at time zone 'utc' + interval '2 hours'"
+		);
+
+		// language=PostgreSQL
+		migrationBuilder.Sql(
+			"""
+			update episodes as e set external_id = (
+				SELECT jsonb_build_object(
+					'themoviedatabase', jsonb_build_object(
+						'ShowId', s.external_id->'themoviedatabase'->'DataId',
+						'Season', e.season_number,
+						'Episode', e.episode_number,
+						'Link', null
+					)
+				)
+				FROM shows AS s
+				WHERE s.id = e.show_id
+			);
+			"""
+		);
+	}
+
+	/// <inheritdoc />
+	protected override void Down(MigrationBuilder migrationBuilder)
+	{
+		migrationBuilder.DropColumn(name: "next_metadata_refresh", table: "shows");
+
+		migrationBuilder.DropColumn(name: "next_metadata_refresh", table: "seasons");
+
+		migrationBuilder.DropColumn(name: "next_metadata_refresh", table: "movies");
+
+		migrationBuilder.DropColumn(name: "next_metadata_refresh", table: "episodes");
+
+		migrationBuilder.DropColumn(name: "next_metadata_refresh", table: "collections");
+
+		// language=PostgreSQL
+		migrationBuilder.Sql("update episodes as e set external_id = '{}';");
+	}
+}

back/src/Kyoo.Postgresql/Migrations/PostgresContextModelSnapshot.cs

@@ -19,7 +19,7 @@ namespace Kyoo.Postgresql.Migrations
 		{
 #pragma warning disable 612, 618
 			modelBuilder
-				.HasAnnotation("ProductVersion", "8.0.3")
+				.HasAnnotation("ProductVersion", "8.0.4")
 				.HasAnnotation("Relational:MaxIdentifierLength", 63);
 
 			NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "genre", new[] { "action", "adventure", "animation", "comedy", "crime", "documentary", "drama", "family", "fantasy", "history", "horror", "music", "mystery", "romance", "science_fiction", "thriller", "war", "western", "kids", "news", "reality", "soap", "talk", "politics" });
@@ -50,6 +50,12 @@ namespace Kyoo.Postgresql.Migrations
 						.HasColumnType("text")
 						.HasColumnName("name");
 
+					b.Property<DateTime?>("NextMetadataRefresh")
+						.ValueGeneratedOnAdd()
+						.HasColumnType("timestamp with time zone")
+						.HasColumnName("next_metadata_refresh")
+						.HasDefaultValueSql("now() at time zone 'utc' + interval '2 hours'");
+
 					b.Property<string>("Overview")
 						.HasColumnType("text")
 						.HasColumnName("overview");
@@ -100,6 +106,12 @@ namespace Kyoo.Postgresql.Migrations
 						.HasColumnType("text")
 						.HasColumnName("name");
 
+					b.Property<DateTime?>("NextMetadataRefresh")
+						.ValueGeneratedOnAdd()
+						.HasColumnType("timestamp with time zone")
+						.HasColumnName("next_metadata_refresh")
+						.HasDefaultValueSql("now() at time zone 'utc' + interval '2 hours'");
+
 					b.Property<string>("Overview")
 						.HasColumnType("text")
 						.HasColumnName("overview");
@@ -262,6 +274,12 @@ namespace Kyoo.Postgresql.Migrations
 						.HasColumnType("text")
 						.HasColumnName("name");
 
+					b.Property<DateTime?>("NextMetadataRefresh")
+						.ValueGeneratedOnAdd()
+						.HasColumnType("timestamp with time zone")
+						.HasColumnName("next_metadata_refresh")
+						.HasDefaultValueSql("now() at time zone 'utc' + interval '2 hours'");
+
 					b.Property<string>("Overview")
 						.HasColumnType("text")
 						.HasColumnName("overview");
@@ -386,6 +404,12 @@ namespace Kyoo.Postgresql.Migrations
 						.HasColumnType("text")
 						.HasColumnName("name");
 
+					b.Property<DateTime?>("NextMetadataRefresh")
+						.ValueGeneratedOnAdd()
+						.HasColumnType("timestamp with time zone")
+						.HasColumnName("next_metadata_refresh")
+						.HasDefaultValueSql("now() at time zone 'utc' + interval '2 hours'");
+
 					b.Property<string>("Overview")
 						.HasColumnType("text")
 						.HasColumnName("overview");
@@ -459,6 +483,12 @@ namespace Kyoo.Postgresql.Migrations
 						.HasColumnType("text")
 						.HasColumnName("name");
 
+					b.Property<DateTime?>("NextMetadataRefresh")
+						.ValueGeneratedOnAdd()
+						.HasColumnType("timestamp with time zone")
+						.HasColumnName("next_metadata_refresh")
+						.HasDefaultValueSql("now() at time zone 'utc' + interval '2 hours'");
+
 					b.Property<string>("Overview")
 						.HasColumnType("text")
 						.HasColumnName("overview");

back/src/Kyoo.Postgresql/PostgresContext.cs

@@ -40,6 +40,7 @@ public class PostgresContext(DbContextOptions options, IHttpContextAccessor acce
 {
 	protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
 	{
+		optionsBuilder.UseProjectables();
 		optionsBuilder.UseSnakeCaseNamingConvention();
 		base.OnConfiguring(optionsBuilder);
 	}
@@ -81,6 +82,10 @@ public class PostgresContext(DbContextOptions options, IHttpContextAccessor acce
 			typeof(Dictionary<string, MetadataId>),
 			new JsonTypeHandler<Dictionary<string, MetadataId>>()
 		);
+		SqlMapper.AddTypeHandler(
+			typeof(Dictionary<string, EpisodeId>),
+			new JsonTypeHandler<Dictionary<string, EpisodeId>>()
+		);
 		SqlMapper.AddTypeHandler(
 			typeof(Dictionary<string, string>),
 			new JsonTypeHandler<Dictionary<string, string>>()
@@ -128,7 +133,11 @@ public class PostgresContextBuilder : IDesignTimeDbContextFactory<PostgresContex
 {
 	public PostgresContext CreateDbContext(string[] args)
 	{
-		NpgsqlDataSource dataSource = PostgresModule.CreateDataSource(new ConfigurationManager());
+		IConfigurationRoot config = new ConfigurationBuilder()
+			.AddEnvironmentVariables()
+			.AddCommandLine(args)
+			.Build();
+		NpgsqlDataSource dataSource = PostgresModule.CreateDataSource(config);
 		DbContextOptionsBuilder builder = new();
 		builder.UseNpgsql(dataSource);
 

back/src/Kyoo.Postgresql/PostgresModule.cs

@@ -37,7 +37,7 @@ public static class PostgresModule
 			{
 				["USER ID"] = configuration.GetValue("POSTGRES_USER", "KyooUser"),
 				["PASSWORD"] = configuration.GetValue("POSTGRES_PASSWORD", "KyooPassword"),
-				["SERVER"] = configuration.GetValue("POSTGRES_SERVER", "db"),
+				["SERVER"] = configuration.GetValue("POSTGRES_SERVER", "postgres"),
 				["PORT"] = configuration.GetValue("POSTGRES_PORT", "5432"),
 				["DATABASE"] = configuration.GetValue("POSTGRES_DB", "kyooDB"),
 				["POOLING"] = "true",
@@ -55,11 +55,10 @@ public static class PostgresModule
 	public static void ConfigurePostgres(this WebApplicationBuilder builder)
 	{
 		NpgsqlDataSource dataSource = CreateDataSource(builder.Configuration);
-
 		builder.Services.AddDbContext<DatabaseContext, PostgresContext>(
 			x =>
 			{
-				x.UseNpgsql(dataSource).UseProjectables();
+				x.UseNpgsql(dataSource);
 				if (builder.Environment.IsDevelopment())
 					x.EnableDetailedErrors().EnableSensitiveDataLogging();
 			},

scanner/matcher/__init__.py

@@ -13,6 +13,6 @@ async def main():
 	logging.getLogger("rebulk").setLevel(logging.WARNING)
 
 	async with KyooClient() as kyoo, Subscriber() as sub:
-		provider, xem = Provider.get_all(kyoo.client)
-		scanner = Matcher(kyoo, provider, xem)
+		provider = Provider.get_default(kyoo.client)
+		scanner = Matcher(kyoo, provider)
 		await sub.listen(scanner)

scanner/matcher/matcher.py

@@ -1,7 +1,6 @@
 from datetime import timedelta
 import asyncio
 from logging import getLogger
-from providers.implementations.thexem import TheXem
 from providers.provider import Provider, ProviderError
 from providers.types.collection import Collection
 from providers.types.show import Show
@@ -15,10 +14,9 @@ logger = getLogger(__name__)
 
 
 class Matcher:
-	def __init__(self, client: KyooClient, provider: Provider, xem: TheXem) -> None:
+	def __init__(self, client: KyooClient, provider: Provider) -> None:
 		self._client = client
 		self._provider = provider
-		self._xem = xem
 
 		self._collection_cache = {}
 		self._show_cache = {}
@@ -48,7 +46,7 @@ class Matcher:
 		return True
 
 	async def _identify(self, path: str):
-		raw = guessit(path, xem_titles=await self._xem.get_expected_titles())
+		raw = guessit(path, xem_titles=await self._provider.get_expected_titles())
 
 		if "mimetype" not in raw or not raw["mimetype"].startswith("video"):
 			return
@@ -68,7 +66,7 @@ class Matcher:
 		logger.info("Identied %s: %s", path, raw)
 
 		if raw["type"] == "movie":
-			movie = await self._provider.identify_movie(raw["title"], raw.get("year"))
+			movie = await self._provider.search_movie(raw["title"], raw.get("year"))
 			movie.path = str(path)
 			logger.debug("Got movie: %s", movie)
 			movie_id = await self._client.post("movies", data=movie.to_kyoo())
@@ -81,7 +79,7 @@ class Matcher:
 					*(self._client.link_collection(x, "movie", movie_id) for x in ids)
 				)
 		elif raw["type"] == "episode":
-			episode = await self._provider.identify_episode(
+			episode = await self._provider.search_episode(
 				raw["title"],
 				season=raw.get("season"),
 				episode_nbr=raw.get("episode"),

scanner/matcher/parser/guess.py

@@ -35,14 +35,14 @@ def guessit(name: str, *, xem_titles: List[str] = []):
 if __name__ == "__main__":
 	import sys
 	import json
-	from providers.implementations.thexem import TheXem
+	from providers.implementations.thexem import TheXemClient
 	from guessit.jsonutils import GuessitEncoder
 	from aiohttp import ClientSession
 	import asyncio
 
 	async def main():
 		async with ClientSession() as client:
-			xem = TheXem(client)
+			xem = TheXemClient(client)
 
 			ret = guessit(sys.argv[1], xem_titles=await xem.get_expected_titles())
 			print(json.dumps(ret, cls=GuessitEncoder, indent=4))

scanner/providers/idmapper.py

@@ -1,32 +0,0 @@
-from __future__ import annotations
-from typing import TYPE_CHECKING
-
-if TYPE_CHECKING:
-	from providers.implementations.themoviedatabase import TheMovieDatabase
-
-from typing import List, Optional
-from providers.types.metadataid import MetadataID
-
-
-class IdMapper:
-	def init(self, *, language: str, tmdb: Optional[TheMovieDatabase]):
-		self.language = language
-		self._tmdb = tmdb
-
-	async def get_show(
-		self, show: dict[str, MetadataID], *, required: Optional[List[str]] = None
-	):
-		ids = show
-
-		# Only fetch using tmdb if one of the required ids is not already known.
-		should_fetch = required is not None and any((x not in ids for x in required))
-		if self._tmdb and self._tmdb.name in ids and should_fetch:
-			tmdb_info = await self._tmdb.identify_show(ids[self._tmdb.name].data_id)
-			return {**ids, **tmdb_info.external_id}
-		return ids
-
-	async def get_movie(
-		self, movie: dict[str, MetadataID], *, required: Optional[List[str]] = None
-	):
-		# TODO: actually do something here
-		return movie

scanner/providers/implementations/themoviedatabase.py

@@ -5,15 +5,13 @@ from logging import getLogger
 from typing import Awaitable, Callable, Dict, List, Optional, Any, TypeVar
 from itertools import accumulate, zip_longest
 
-from providers.idmapper import IdMapper
-from providers.implementations.thexem import TheXem
 from providers.utils import ProviderError
 from matcher.cache import cache
 
 from ..provider import Provider
 from ..types.movie import Movie, MovieTranslation, Status as MovieStatus
 from ..types.season import Season, SeasonTranslation
-from ..types.episode import Episode, EpisodeTranslation, PartialShow
+from ..types.episode import Episode, EpisodeTranslation, PartialShow, EpisodeID
 from ..types.studio import Studio
 from ..types.genre import Genre
 from ..types.metadataid import MetadataID
@@ -29,14 +27,10 @@ class TheMovieDatabase(Provider):
 		languages: list[str],
 		client: ClientSession,
 		api_key: str,
-		xem: TheXem,
-		idmapper: IdMapper,
 	) -> None:
 		super().__init__()
 		self._languages = languages
 		self._client = client
-		self._xem = xem
-		self._idmapper = idmapper
 		self.base = "https://api.themoviedb.org/3"
 		self.api_key = api_key
 		self.genre_map = {
@@ -142,15 +136,17 @@ class TheMovieDatabase(Provider):
 			},
 		)
 
-	async def identify_movie(self, name: str, year: Optional[int]) -> Movie:
+	async def search_movie(self, name: str, year: Optional[int]) -> Movie:
 		search_results = (
 			await self.get("search/movie", params={"query": name, "year": year})
 		)["results"]
 		if len(search_results) == 0:
 			raise ProviderError(f"No result for a movie named: {name}")
 		search = self.get_best_result(search_results, name, year)
-		movie_id = search["id"]
-		languages = self.get_languages(search["original_language"])
+		return await self.identify_movie(search["id"])
+
+	async def identify_movie(self, movie_id: str) -> Movie:
+		languages = self.get_languages()
 
 		async def for_language(lng: str) -> Movie:
 			movie = await self.get(
@@ -216,7 +212,7 @@ class TheMovieDatabase(Provider):
 					movie["images"]["posters"]
 					+ (
 						[{"file_path": movie["poster_path"]}]
-						if lng == search["original_language"]
+						if lng == movie["original_language"]
 						else []
 					)
 				),
@@ -225,7 +221,7 @@ class TheMovieDatabase(Provider):
 					movie["images"]["backdrops"]
 					+ (
 						[{"file_path": movie["backdrop_path"]}]
-						if lng == search["original_language"]
+						if lng == movie["original_language"]
 						else []
 					)
 				),
@@ -239,8 +235,13 @@ class TheMovieDatabase(Provider):
 			return ret
 
 		ret = await self.process_translations(for_language, languages)
-		# If we have more external_ids freely available, add them.
-		ret.external_id = await self._idmapper.get_movie(ret.external_id)
+		if (
+			ret.original_language is not None
+			and ret.original_language not in ret.translations
+		):
+			ret.translations[ret.original_language] = (
+				await for_language(ret.original_language)
+			).translations[ret.original_language]
 		return ret
 
 	@cache(ttl=timedelta(days=1))
@@ -362,8 +363,6 @@ class TheMovieDatabase(Provider):
 			ret.translations[ret.original_language] = (
 				await for_language(ret.original_language)
 			).translations[ret.original_language]
-		# If we have more external_ids freely available, add them.
-		ret.external_id = await self._idmapper.get_show(ret.external_id)
 		return ret
 
 	def to_season(
@@ -396,13 +395,13 @@ class TheMovieDatabase(Provider):
 			},
 		)
 
-	async def identify_season(self, show_id: str, season_number: int) -> Season:
+	async def identify_season(self, show_id: str, season: int) -> Season:
 		# We already get seasons info in the identify_show and chances are this gets cached already
 		show = await self.identify_show(show_id)
-		ret = next((x for x in show.seasons if x.season_number == season_number), None)
+		ret = next((x for x in show.seasons if x.season_number == season), None)
 		if ret is None:
 			raise ProviderError(
-				f"Could not find season {season_number} for show {show.to_kyoo()['name']}"
+				f"Could not find season {season} for show {show.to_kyoo()['name']}"
 			)
 		return ret
 
@@ -413,29 +412,7 @@ class TheMovieDatabase(Provider):
 		)["results"]
 
 		if len(search_results) == 0:
-			(new_name, tvdbid) = await self._xem.get_show_override("tvdb", name)
-			if new_name is None or tvdbid is None or name.lower() == new_name.lower():
-				raise ProviderError(f"No result for a tv show named: {name}")
-			ret = PartialShow(
-				name=new_name,
-				original_language=None,
-				external_id={
-					"tvdb": MetadataID(tvdbid, link=None),
-				},
-			)
-			ret.external_id = await self._idmapper.get_show(
-				ret.external_id, required=[self.name]
-			)
-
-			if self.name in ret.external_id:
-				return ret
-			logger.warn(
-				"Could not map xem exception to themoviedb, searching instead for %s",
-				new_name,
-			)
-			nret = await self.search_show(new_name, year)
-			nret.external_id = {**ret.external_id, **nret.external_id}
-			return nret
+			raise ProviderError(f"No result for a tv show named: {name}")
 
 		search = self.get_best_result(search_results, name, year)
 		show_id = search["id"]
@@ -449,7 +426,7 @@ class TheMovieDatabase(Provider):
 			},
 		)
 
-	async def identify_episode(
+	async def search_episode(
 		self,
 		name: str,
 		season: Optional[int],
@@ -458,39 +435,8 @@ class TheMovieDatabase(Provider):
 		year: Optional[int],
 	) -> Episode:
 		show = await self.search_show(name, year)
-		languages = self.get_languages(show.original_language)
-		# Keep it for xem overrides of season/episode
-		old_name = name
-		name = show.name
 		show_id = show.external_id[self.name].data_id
 
-		# Handle weird season names overrides from thexem.
-		# For example when name is "Jojo's bizzare adventure - Stone Ocean", with season None,
-		# We want something like season 6 ep 3.
-		if season is None and absolute is not None:
-			ids = await self._idmapper.get_show(show.external_id, required=["tvdb"])
-			tvdb_id = (
-				ids["tvdb"].data_id
-				if "tvdb" in ids and ids["tvdb"] is not None
-				else None
-			)
-			if tvdb_id is None:
-				logger.info(
-					"Tvdb could not be found, trying xem name lookup for %s", name
-				)
-				_, tvdb_id = await self._xem.get_show_override("tvdb", old_name)
-			if tvdb_id is not None:
-				(
-					tvdb_season,
-					tvdb_episode,
-					absolute,
-				) = await self._xem.get_episode_override(
-					"tvdb", tvdb_id, old_name, absolute
-				)
-				# Most of the time, tvdb absolute and tmdb absolute are in think so we use that as our souce of truth.
-				# tvdb_season/episode are not in sync with tmdb so we discard those and use our usual absolute order fetching.
-				(_, _) = tvdb_season, tvdb_episode
-
 		if absolute is not None and (season is None or episode_nbr is None):
 			(season, episode_nbr) = await self.get_episode_from_absolute(
 				show_id, absolute
@@ -498,12 +444,16 @@ class TheMovieDatabase(Provider):
 
 		if season is None or episode_nbr is None:
 			raise ProviderError(
-				f"Could not guess season or episode number of the episode {name} {season}-{episode_nbr} ({absolute})",
+				f"Could not guess season or episode number of the episode {show.name} {season}-{episode_nbr} ({absolute})",
 			)
 
 		if absolute is None:
 			absolute = await self.get_absolute_number(show_id, season, episode_nbr)
+		return await self.identify_episode(show_id, season, episode_nbr, absolute)
 
+	async def identify_episode(
+		self, show_id: str, season: Optional[int], episode_nbr: int, absolute: int
+	) -> Episode:
 		async def for_language(lng: str) -> Episode:
 			try:
 				episode = await self.get(
@@ -518,12 +468,20 @@ class TheMovieDatabase(Provider):
 					params={
 						"language": lng,
 					},
-					not_found_fail=f"Could not find episode {episode_nbr} of season {season} of serie {name} (absolute: {absolute})",
+					not_found_fail=f"Could not find episode {episode_nbr} of season {season} of serie {show_id} (absolute: {absolute})",
 				)
 			logger.debug("TMDb responded: %s", episode)
 
 			ret = Episode(
-				show=show,
+				show=PartialShow(
+					name=show_id,
+					original_language=None,
+					external_id={
+						self.name: MetadataID(
+							show_id, f"https://www.themoviedb.org/tv/{show_id}"
+						)
+					},
+				),
 				season_number=episode["season_number"],
 				episode_number=episode["episode_number"],
 				absolute_number=absolute,
@@ -537,8 +495,10 @@ class TheMovieDatabase(Provider):
 				if "still_path" in episode and episode["still_path"] is not None
 				else None,
 				external_id={
-					self.name: MetadataID(
-						episode["id"],
+					self.name: EpisodeID(
+						show_id,
+						episode["season_number"],
+						episode["episode_number"],
 						f"https://www.themoviedb.org/tv/{show_id}/season/{episode['season_number']}/episode/{episode['episode_number']}",
 					),
 				},
@@ -550,7 +510,7 @@ class TheMovieDatabase(Provider):
 			ret.translations = {lng: translation}
 			return ret
 
-		return await self.process_translations(for_language, languages)
+		return await self.process_translations(for_language, self.get_languages())
 
 	def get_best_result(
 		self, search_results: List[Any], name: str, year: Optional[int]
@@ -654,7 +614,9 @@ class TheMovieDatabase(Provider):
 			(seasons_nbrs[0], absolute),
 		)
 
-	async def get_absolute_number(self, show_id: str, season: int, episode_nbr: int):
+	async def get_absolute_number(
+		self, show_id: str, season: int, episode_nbr: int
+	) -> int:
 		absgrp = await self.get_absolute_order(show_id)
 		if absgrp is None:
 			# We assume that each season should be played in order with no special episodes.
@@ -684,7 +646,9 @@ class TheMovieDatabase(Provider):
 			(x["episode_number"] for x in absgrp if x["season_number"] == season), None
 		)
 		if start is None or start <= episode_nbr:
-			return None
+			raise ProviderError(
+				f"Could not guess absolute number of episode {show_id} s{season} e{episode_nbr}"
+			)
 		# add back the continuous number (imagine the user has one piece S21e31
 		# but tmdb registered it as S21E831 since S21's first ep is 800
 		return await self.get_absolute_number(show_id, season, episode_nbr + start)

scanner/providers/implementations/thexem.py

@@ -3,8 +3,15 @@ from typing import Dict, List, Literal
 from aiohttp import ClientSession
 from logging import getLogger
 from datetime import timedelta
+from typing import Optional
 
-from providers.utils import ProviderError
+from ..provider import Provider
+from ..utils import ProviderError
+from ..types.collection import Collection
+from ..types.movie import Movie
+from ..types.show import Show
+from ..types.season import Season
+from ..types.episode import Episode
 from matcher.cache import cache
 
 logger = getLogger(__name__)
@@ -21,7 +28,7 @@ def clean(s: str):
 	return s
 
 
-class TheXem:
+class TheXemClient:
 	def __init__(self, client: ClientSession) -> None:
 		self._client = client
 		self.base = "https://thexem.info"
@@ -155,3 +162,77 @@ class TheXem:
 			for y in x[1:]:
 				titles.extend(clean(name) for name in y.keys())
 		return titles
+
+
+class TheXem(Provider):
+	def __init__(self, client: ClientSession, base: Provider) -> None:
+		super().__init__()
+		self._client = TheXemClient(client)
+		self._base = base
+
+	@property
+	def name(self) -> str:
+		# Use the base name for id lookup on the matcher.
+		return self._base.name
+
+	async def get_expected_titles(self) -> list[str]:
+		return await self._client.get_expected_titles()
+
+	async def search_movie(self, name: str, year: Optional[int]) -> Movie:
+		return await self._base.search_movie(name, year)
+
+	async def search_episode(
+		self,
+		name: str,
+		season: Optional[int],
+		episode_nbr: Optional[int],
+		absolute: Optional[int],
+		year: Optional[int],
+	) -> Episode:
+		"""
+		Handle weird season names overrides from thexem.
+		For example when name is "Jojo's bizzare adventure - Stone Ocean", with season None,
+		We want something like season 6 ep 3.
+		"""
+		new_name, tvdb_id = await self._client.get_show_override("tvdb", name)
+
+		if new_name is None:
+			return await self._base.search_episode(
+				name, season, episode_nbr, absolute, year
+			)
+
+		if season is None and absolute is not None:
+			if tvdb_id is not None:
+				(
+					tvdb_season,
+					tvdb_episode,
+					absolute,
+				) = await self._client.get_episode_override(
+					"tvdb", tvdb_id, name, absolute
+				)
+				# Most of the time, tvdb absolute and tmdb absolute are in sync so we use that as our souce of truth.
+				# tvdb_season/episode are not in sync with tmdb so we discard those and use our usual absolute order fetching.
+				if self._base == "tvdb":
+					return await self._base.search_episode(
+						new_name, tvdb_season, tvdb_episode, absolute, year
+					)
+		return await self._base.search_episode(
+			new_name, season, episode_nbr, absolute, year
+		)
+
+	async def identify_movie(self, movie_id: str) -> Movie:
+		return await self._base.identify_movie(movie_id)
+
+	async def identify_show(self, show_id: str) -> Show:
+		return await self._base.identify_show(show_id)
+
+	async def identify_season(self, show_id: str, season: int) -> Season:
+		return await self._base.identify_season(show_id, season)
+
+	async def identify_episode(
+		self, show_id: str, season: Optional[int], episode_nbr: int, absolute: int
+	) -> Episode:
+		return await self._base.identify_episode(show_id, season, episode_nbr, absolute)
+
+	async def identify_collection(self, provider_id: str) -> Collection:
+		return await self._base.identify_collection(provider_id)

scanner/providers/provider.py

@@ -1,9 +1,8 @@
 import os
 from aiohttp import ClientSession
 from abc import abstractmethod, abstractproperty
-from typing import Optional, Self
+from typing import Optional
 
-from providers.implementations.thexem import TheXem
 from providers.utils import ProviderError
 
 from .types.show import Show
@@ -15,7 +14,7 @@ from .types.collection import Collection
 
 class Provider:
 	@classmethod
-	def get_all(cls, client: ClientSession) -> tuple[Self, TheXem]:
+	def get_default(cls, client: ClientSession):
 		languages = os.environ.get("LIBRARY_LANGUAGES")
 		if not languages:
 			print("Missing environment variable 'LIBRARY_LANGUAGES'.")
@@ -23,47 +22,33 @@ class Provider:
 		languages = languages.split(",")
 		providers = []
 
-		from providers.idmapper import IdMapper
-
-		idmapper = IdMapper()
-		xem = TheXem(client)
-
 		from providers.implementations.themoviedatabase import TheMovieDatabase
 
 		tmdb = os.environ.get("THEMOVIEDB_APIKEY")
 		if tmdb:
-			tmdb = TheMovieDatabase(languages, client, tmdb, xem, idmapper)
+			tmdb = TheMovieDatabase(languages, client, tmdb)
 			providers.append(tmdb)
-		else:
-			tmdb = None
 
 		if not any(providers):
 			raise ProviderError(
 				"No provider configured. You probably forgot to specify an API Key"
 			)
 
-		idmapper.init(tmdb=tmdb, language=languages[0])
+		from providers.implementations.thexem import TheXem
 
-		return next(iter(providers)), xem
+		provider = next(iter(providers))
+		return TheXem(client, provider)
 
 	@abstractproperty
 	def name(self) -> str:
 		raise NotImplementedError
 
 	@abstractmethod
-	async def identify_movie(self, name: str, year: Optional[int]) -> Movie:
+	async def search_movie(self, name: str, year: Optional[int]) -> Movie:
 		raise NotImplementedError
 
 	@abstractmethod
-	async def identify_show(self, show_id: str) -> Show:
-		raise NotImplementedError
-
-	@abstractmethod
-	async def identify_season(self, show_id: str, season_number: int) -> Season:
-		raise NotImplementedError
-
-	@abstractmethod
-	async def identify_episode(
+	async def search_episode(
 		self,
 		name: str,
 		season: Optional[int],
@@ -73,6 +58,28 @@ class Provider:
 	) -> Episode:
 		raise NotImplementedError
 
+	@abstractmethod
+	async def identify_movie(self, movie_id: str) -> Movie:
+		raise NotImplementedError
+
+	@abstractmethod
+	async def identify_show(self, show_id: str) -> Show:
+		raise NotImplementedError
+
+	@abstractmethod
+	async def identify_season(self, show_id: str, season: int) -> Season:
+		raise NotImplementedError
+
+	@abstractmethod
+	async def identify_episode(
+		self, show_id: str, season: Optional[int], episode_nbr: int, absolute: int
+	) -> Episode:
+		raise NotImplementedError
+
 	@abstractmethod
 	async def identify_collection(self, provider_id: str) -> Collection:
 		raise NotImplementedError
+
+	@abstractmethod
+	async def get_expected_titles(self) -> list[str]:
+		return []

scanner/providers/types/episode.py

@@ -14,6 +14,14 @@ class PartialShow:
 	external_id: dict[str, MetadataID]
 
 
+@dataclass
+class EpisodeID:
+	show_id: str
+	season: Optional[int]
+	episode: int
+	link: str
+
+
 @dataclass
 class EpisodeTranslation:
 	name: str
@@ -23,13 +31,13 @@ class EpisodeTranslation:
 @dataclass
 class Episode:
 	show: Show | PartialShow
-	season_number: Optional[int]
-	episode_number: Optional[int]
-	absolute_number: Optional[int]
+	season_number: int
+	episode_number: int
+	absolute_number: int
 	runtime: Optional[int]
 	release_date: Optional[date | int]
 	thumbnail: Optional[str]
-	external_id: dict[str, MetadataID]
+	external_id: dict[str, EpisodeID]
 
 	path: Optional[str] = None
 	show_id: Optional[str] = None

transcoder/Dockerfile

@@ -59,4 +59,4 @@ ENV NVIDIA_VISIBLE_DEVICES="all"
 ENV NVIDIA_DRIVER_CAPABILITIES="all"
 
 EXPOSE 7666
-CMD ./transcoder
+CMD ["./transcoder"]

transcoder/Dockerfile.dev

@@ -47,4 +47,4 @@ ENV NVIDIA_VISIBLE_DEVICES="all"
 ENV NVIDIA_DRIVER_CAPABILITIES="all"
 
 EXPOSE 7666
-CMD wgo run -race .
+CMD ["wgo", "run", "-race", "."]