From 0c4cab48d7bdd74f073cc98fc605f0e6047b6e21 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Wed, 14 Jul 2021 00:04:55 +0200 Subject: [PATCH 01/31] Adding thetvdb, using autofac --- Kyoo.Common/Controllers/IMetadataProvider.cs | 51 ++++- Kyoo.Common/Controllers/IPlugin.cs | 28 ++- Kyoo.Common/Controllers/IPluginManager.cs | 9 +- Kyoo.Common/Controllers/IProviderManager.cs | 17 -- Kyoo.Common/Kyoo.Common.csproj | 1 + Kyoo.Common/Module.cs | 57 +++--- Kyoo.TheTvdb/Kyoo.TheTvdb.csproj | 33 +++ Kyoo.TheTvdb/PluginTVDB.cs | 41 ++++ Kyoo.TheTvdb/ProviderTVDB.cs | 203 +++++++++++++++++++ Kyoo.sln | 6 + Kyoo/Controllers/PluginManager.cs | 7 + Kyoo/Controllers/ProviderComposite.cs | 182 +++++++++++++++++ Kyoo/Controllers/ProviderManager.cs | 166 --------------- Kyoo/CoreModule.cs | 68 ++++--- Kyoo/Kyoo.csproj | 2 + Kyoo/Program.cs | 45 ++-- Kyoo/Startup.cs | 13 +- Kyoo/Tasks/Crawler.cs | 15 +- 18 files changed, 664 insertions(+), 280 deletions(-) delete mode 100644 Kyoo.Common/Controllers/IProviderManager.cs create mode 100644 Kyoo.TheTvdb/Kyoo.TheTvdb.csproj create mode 100644 Kyoo.TheTvdb/PluginTVDB.cs create mode 100644 Kyoo.TheTvdb/ProviderTVDB.cs create mode 100644 Kyoo/Controllers/ProviderComposite.cs delete mode 100644 Kyoo/Controllers/ProviderManager.cs diff --git a/Kyoo.Common/Controllers/IMetadataProvider.cs b/Kyoo.Common/Controllers/IMetadataProvider.cs index a0c30cbb..9c260030 100644 --- a/Kyoo.Common/Controllers/IMetadataProvider.cs +++ b/Kyoo.Common/Controllers/IMetadataProvider.cs @@ -1,21 +1,56 @@ -using Kyoo.Models; +using System; +using Kyoo.Models; using System.Collections.Generic; using System.Threading.Tasks; +using JetBrains.Annotations; namespace Kyoo.Controllers { + /// + /// An interface to automatically retrieve metadata from external providers. + /// public interface IMetadataProvider { + /// + /// The corresponding to this provider. + /// This allow to map metadata to a provider, keep metadata links and + /// know witch is used for a specific . + /// Provider Provider { get; } - Task GetCollectionFromName(string name); + /// + /// Return a new item with metadata from your provider. + /// + /// + /// The item to retrieve metadata from. Most of the time, only the name will be available but other + /// properties may be filed by other providers before a call to this method. This can allow you to identify + /// the collection on your provider. + /// + /// + /// You must not use metadata from the given . + /// Merging metadata is the job of Kyoo, a complex is given + /// to make a precise search and give you every available properties, not to discard properties. + /// + /// + /// If this metadata provider does not support . + /// + /// A new containing metadata from your provider + [ItemNotNull] + Task Get([NotNull] T item) + where T : class, IResource; - Task GetShowByID(Show show); - Task> SearchShows(string showName, bool isMovie); + /// + /// Search for a specific type of items with a given query. + /// + /// The search query to use. + /// + /// If this metadata provider does not support . + /// + /// The list of items that could be found on this specific provider. + [ItemNotNull] + Task> Search(string query) + where T : class, IResource; + Task> GetPeople(Show show); - - Task GetSeason(Show show, int seasonNumber); - - Task GetEpisode(Show show, int? seasonNumber, int? episodeNumber, int? absoluteNumber); } } diff --git a/Kyoo.Common/Controllers/IPlugin.cs b/Kyoo.Common/Controllers/IPlugin.cs index 3201df83..83cc70a4 100644 --- a/Kyoo.Common/Controllers/IPlugin.cs +++ b/Kyoo.Common/Controllers/IPlugin.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using Autofac; using JetBrains.Annotations; using Microsoft.AspNetCore.Builder; using Microsoft.Extensions.DependencyInjection; @@ -55,6 +56,15 @@ namespace Kyoo.Controllers /// ICollection Requires { get; } + /// + /// A configure method that will be run on plugin's startup. + /// + /// The autofac service container to register services. + void Configure(ContainerBuilder builder) + { + // Skipped + } + /// /// A configure method that will be run on plugin's startup. /// @@ -64,21 +74,31 @@ namespace Kyoo.Controllers /// or > /// You can't simply check on the service collection because some dependencies might be registered after your plugin. /// - void Configure(IServiceCollection services, ICollection availableTypes); + void Configure(IServiceCollection services, ICollection availableTypes) + { + // Skipped + } + /// /// An optional configuration step to allow a plugin to change asp net configurations. /// WARNING: This is only called on Kyoo's startup so you must restart the app to apply this changes. /// /// The Asp.Net application builder. On most case it is not needed but you can use it to add asp net functionalities. - void ConfigureAspNet(IApplicationBuilder app) {} - + void ConfigureAspNet(IApplicationBuilder app) + { + // Skipped + } + /// /// An optional function to execute and initialize your plugin. /// It can be used to initialize a database connection, fill initial data or anything. /// /// A service provider to request services - void Initialize(IServiceProvider provider) {} + void Initialize(IServiceProvider provider) + { + // Skipped + } } /// diff --git a/Kyoo.Common/Controllers/IPluginManager.cs b/Kyoo.Common/Controllers/IPluginManager.cs index bd4ef513..04d308f3 100644 --- a/Kyoo.Common/Controllers/IPluginManager.cs +++ b/Kyoo.Common/Controllers/IPluginManager.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using Autofac; using Kyoo.Models.Exceptions; using Microsoft.AspNetCore.Builder; using Microsoft.Extensions.DependencyInjection; @@ -42,7 +43,13 @@ namespace Kyoo.Controllers public void LoadPlugins(ICollection plugins); /// - /// Configure services adding or removing services as the plugins wants. + /// Configure container adding or removing services as the plugins wants. + /// + /// The container to populate + void ConfigureContainer(ContainerBuilder builder); + + /// + /// Configure services via the microsoft way. This allow libraries to add their services. /// /// The service collection to populate public void ConfigureServices(IServiceCollection services); diff --git a/Kyoo.Common/Controllers/IProviderManager.cs b/Kyoo.Common/Controllers/IProviderManager.cs deleted file mode 100644 index dd83a283..00000000 --- a/Kyoo.Common/Controllers/IProviderManager.cs +++ /dev/null @@ -1,17 +0,0 @@ -using System.Collections.Generic; -using System.Threading.Tasks; -using Kyoo.Models; - -namespace Kyoo.Controllers -{ - public interface IProviderManager - { - Task GetCollectionFromName(string name, Library library); - Task CompleteShow(Show show, Library library); - Task SearchShow(string showName, bool isMovie, Library library); - Task> SearchShows(string showName, bool isMovie, Library library); - Task GetSeason(Show show, int seasonNumber, Library library); - Task GetEpisode(Show show, string episodePath, int? seasonNumber, int? episodeNumber, int? absoluteNumber, Library library); - Task> GetPeople(Show show, Library library); - } -} \ No newline at end of file diff --git a/Kyoo.Common/Kyoo.Common.csproj b/Kyoo.Common/Kyoo.Common.csproj index 349ef6a0..ada2ed60 100644 --- a/Kyoo.Common/Kyoo.Common.csproj +++ b/Kyoo.Common/Kyoo.Common.csproj @@ -21,6 +21,7 @@ + diff --git a/Kyoo.Common/Module.cs b/Kyoo.Common/Module.cs index a8a81b88..d4442d98 100644 --- a/Kyoo.Common/Module.cs +++ b/Kyoo.Common/Module.cs @@ -1,5 +1,6 @@ -using System; using System.Linq; +using Autofac; +using Autofac.Builder; using Kyoo.Controllers; using Kyoo.Models; using Microsoft.Extensions.Configuration; @@ -15,55 +16,63 @@ namespace Kyoo /// /// Register a new task to the container. /// - /// The container + /// The container /// The type of the task - /// The initial container. - public static IServiceCollection AddTask(this IServiceCollection services) + /// The registration builder of this new task. That can be used to edit the registration. + public static IRegistrationBuilder + RegisterTask(this ContainerBuilder builder) where T : class, ITask { - services.AddSingleton(); - return services; + return builder.RegisterType().As().SingleInstance(); + } + + /// + /// Register a new metadata provider to the container. + /// + /// The container + /// The type of the task + /// The registration builder of this new provider. That can be used to edit the registration. + public static IRegistrationBuilder + RegisterProvider(this ContainerBuilder builder) + where T : class, IMetadataProvider + { + return builder.RegisterType().As().InstancePerLifetimeScope(); } /// /// Register a new repository to the container. /// - /// The container - /// The lifetime of the repository. The default is scoped. + /// The container /// The type of the repository. /// - /// If your repository implements a special interface, please use + /// If your repository implements a special interface, please use /// /// The initial container. - public static IServiceCollection AddRepository(this IServiceCollection services, - ServiceLifetime lifetime = ServiceLifetime.Scoped) + public static IRegistrationBuilder + RegisterRepository(this ContainerBuilder builder) where T : IBaseRepository { - Type repository = Utility.GetGenericDefinition(typeof(T), typeof(IRepository<>)); - - if (repository != null) - services.Add(ServiceDescriptor.Describe(repository, typeof(T), lifetime)); - services.Add(ServiceDescriptor.Describe(typeof(IBaseRepository), typeof(T), lifetime)); - return services; + return builder.RegisterType() + .As() + .As(Utility.GetGenericDefinition(typeof(T), typeof(IRepository<>))) + .InstancePerLifetimeScope(); } /// /// Register a new repository with a custom mapping to the container. /// - /// - /// The lifetime of the repository. The default is scoped. + /// The container /// The custom mapping you have for your repository. /// The type of the repository. /// - /// If your repository does not implements a special interface, please use + /// If your repository does not implements a special interface, please use /// /// The initial container. - public static IServiceCollection AddRepository(this IServiceCollection services, - ServiceLifetime lifetime = ServiceLifetime.Scoped) + public static IRegistrationBuilder + RegisterRepository(this ContainerBuilder builder) where T2 : IBaseRepository, T { - services.Add(ServiceDescriptor.Describe(typeof(T), typeof(T2), lifetime)); - return services.AddRepository(lifetime); + return builder.RegisterRepository().As(); } /// diff --git a/Kyoo.TheTvdb/Kyoo.TheTvdb.csproj b/Kyoo.TheTvdb/Kyoo.TheTvdb.csproj new file mode 100644 index 00000000..1b6aee8f --- /dev/null +++ b/Kyoo.TheTvdb/Kyoo.TheTvdb.csproj @@ -0,0 +1,33 @@ + + + net5.0 + + SDG + Zoe Roux + https://github.com/AnonymusRaccoon/Kyoo + default + Kyoo.TheTvdb + + + + ../Kyoo/bin/$(Configuration)/$(TargetFramework)/plugins/the-tvdb + false + false + false + false + true + + + + + + + + + + all + false + runtime + + + diff --git a/Kyoo.TheTvdb/PluginTVDB.cs b/Kyoo.TheTvdb/PluginTVDB.cs new file mode 100644 index 00000000..6484b51d --- /dev/null +++ b/Kyoo.TheTvdb/PluginTVDB.cs @@ -0,0 +1,41 @@ +using System; +using System.Collections.Generic; +using Kyoo.Controllers; +using Microsoft.Extensions.DependencyInjection; + +namespace Kyoo.TheTvdb +{ + /// + /// A plugin that add a for The TVDB. + /// + public class PluginTvdb : IPlugin + { + /// + public string Slug => "the-tvdb"; + + /// + public string Name => "The TVDB Provider"; + + /// + public string Description => "A metadata provider for The TVDB."; + + /// + public ICollection Provides => new [] + { + typeof(IMetadataProvider) + }; + + /// + public ICollection ConditionalProvides => ArraySegment.Empty; + + /// + public ICollection Requires => ArraySegment.Empty; + + + /// + public void Configure(IServiceCollection services, ICollection availableTypes) + { + // services.AddProvider(); + } + } +} \ No newline at end of file diff --git a/Kyoo.TheTvdb/ProviderTVDB.cs b/Kyoo.TheTvdb/ProviderTVDB.cs new file mode 100644 index 00000000..eeb065c9 --- /dev/null +++ b/Kyoo.TheTvdb/ProviderTVDB.cs @@ -0,0 +1,203 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Threading.Tasks; +using Kyoo.Controllers; +using Kyoo.Models; +using TvDbSharper; +using TvDbSharper.Dto; + +namespace Kyoo.TheTvdb +{ + /// + /// A metadata provider for The TVDB. + /// + public class ProviderTvdb : IMetadataProvider + { + public Provider Provider { get; } + public Task Get(T item) where T : class, IResource + { + throw new NotImplementedException(); + } + + public Task> Search(string query) where T : class, IResource + { + throw new NotImplementedException(); + } + + public Task> GetPeople(Show show) + { + throw new NotImplementedException(); + } + } + + + // public class Old + // { + // private static readonly ProviderID _provider = new() + // { + // Slug = "the-tvdb", + // Name = "TheTVDB", + // LogoExtension = "png", + // Logo = "https://www.thetvdb.com/images/logo.png" + // }; + // public ProviderID Provider => _provider; + // + // + // private readonly TvDbClient _client = new(); + // + // private Task Authentificate() + // { + // if (_client.Authentication.Token == null) + // return _client.Authentication.AuthenticateAsync(APIKey); + // return _client.Authentication.RefreshTokenAsync(); + // } + // + // public Task GetCollectionFromName(string name) + // { + // return Task.FromResult(null); + // } + // + // public async Task> SearchShows(string showName, bool isMovie) + // { + // await Authentificate(); + // + // if (isMovie) + // return null; //There is no movie search API for now on TheTVDB. + // TvDbResponse shows = await _client.Search.SearchSeriesAsync(showName, SearchParameter.Name); + // return shows.Data.Select(x => x.ToShow(Provider)).ToArray(); + // } + // + // public async Task GetShowByID(Show show) + // { + // if (!int.TryParse(show?.GetID(Provider.Name), out int id)) + // return await Task.FromResult(null); + // await Authentificate(); + // TvDbResponse serie = await _client.Series.GetAsync(id); + // return serie.Data.ToShow(Provider); + // } + // + // public async Task> GetPeople(Show show) + // { + // if (!int.TryParse(show?.GetID(Provider.Name), out int id)) + // return null; + // await Authentificate(); + // TvDbResponse people = await _client.Series.GetActorsAsync(id); + // return people.Data.Select(x => x.ToPeopleRole(Provider)).ToArray(); + // } + // + // public Task GetSeason(Show show, int seasonNumber) + // { + // return Task.FromResult(null); + // } + // + // public async Task GetEpisode(Show show, int seasonNumber, int episodeNumber, int absoluteNumber) + // { + // if (!int.TryParse(show?.GetID(Provider.Name), out int id)) + // return null; + // await Authentificate(); + // TvDbResponse episodes = absoluteNumber != -1 + // ? await _client.Series.GetEpisodesAsync(id, 0, new EpisodeQuery {AbsoluteNumber = absoluteNumber}) + // : await _client.Series.GetEpisodesAsync(id, 0, new EpisodeQuery {AiredSeason = seasonNumber, AiredEpisode = episodeNumber}); + // EpisodeRecord x = episodes.Data[0]; + // + // if (absoluteNumber == -1) + // absoluteNumber = x.AbsoluteNumber ?? -1; + // else + // { + // seasonNumber = x.AiredSeason ?? -1; + // episodeNumber = x.AiredEpisodeNumber ?? -1; + // } + // + // return new Episode(seasonNumber, + // episodeNumber, + // absoluteNumber, + // x.EpisodeName, + // x.Overview, + // DateTime.ParseExact(x.FirstAired, "yyyy-MM-dd", CultureInfo.InvariantCulture), + // -1, + // x.Filename != null ? "https://www.thetvdb.com/banners/" + x.Filename : null, + // new [] + // { + // new MetadataID(Provider, x.Id.ToString(), $"https://www.thetvdb.com/series/{id}/episodes/{x.Id}") + // }); + // } + // } + + // public static class Convertors + // { + // private static int? GetYear(string firstAired) + // { + // if (firstAired?.Length >= 4 && int.TryParse(firstAired.Substring(0, 4), out int year)) + // return year; + // return null; + // } + // + // private static Status? GetStatus(string status) + // { + // return status switch + // { + // "Ended" => Status.Finished, + // "Continuing" => Status.Airing, + // _ => null + // }; + // } + // + // public static Show ToShow(this SeriesSearchResult x, ProviderID provider) + // { + // Show ret = new(x.Slug, + // x.SeriesName, + // x.Aliases, + // null, + // x.Overview, + // null, + // null, + // GetStatus(x.Status), + // GetYear(x.FirstAired), + // null, + // new[] + // { + // new MetadataID(provider, x.Id.ToString(), $"https://www.thetvdb.com/series/{x.Slug}") + // }); + // if (x.Poster != null) + // Utility.SetImage(ret, $"https://www.thetvdb.com{x.Poster}", ImageType.Poster); + // return ret; + // } + // + // public static Show ToShow(this Series x, ProviderID provider) + // { + // return new(x.Slug, + // x.SeriesName, + // x.Aliases, + // null, + // x.Overview, + // null, + // x.Genre.Select(y => new Genre(Utility.ToSlug(y), y)), + // GetStatus(x.Status), + // GetYear(x.FirstAired), + // null, + // new[] + // { + // new MetadataID(provider, x.Id.ToString(),$"https://www.thetvdb.com/series/{x.Slug}") + // }) + // { + // Poster = x.Poster != null ? $"https://www.thetvdb.com/banners/{x.Poster}" : null, + // Backdrop = x.FanArt != null ? $"https://www.thetvdb.com/banners/{x.FanArt}" : null + // }; + // } + // + // public static PeopleRole ToPeopleRole(this Actor x, ProviderID provider) + // { + // return new (Utility.ToSlug(x.Name), + // x.Name, + // x.Role, + // null, + // x.Image != null ? $"https://www.thetvdb.com/banners/{x.Image}" : null, + // new[] + // { + // new MetadataID(provider, x.Id.ToString(), $"https://www.thetvdb.com/people/{x.Id}") + // }); + // } + // } +} \ No newline at end of file diff --git a/Kyoo.sln b/Kyoo.sln index 60998b55..55aeb44c 100644 --- a/Kyoo.sln +++ b/Kyoo.sln @@ -13,6 +13,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Kyoo.Authentication", "Kyoo EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Kyoo.SqLite", "Kyoo.SqLite\Kyoo.SqLite.csproj", "{6515380E-1E57-42DA-B6E3-E1C8A848818A}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Kyoo.TheTvdb", "Kyoo.TheTvdb\Kyoo.TheTvdb.csproj", "{D06BF829-23F5-40F3-A62D-627D9F4B4D6C}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -47,5 +49,9 @@ Global {6515380E-1E57-42DA-B6E3-E1C8A848818A}.Debug|Any CPU.Build.0 = Debug|Any CPU {6515380E-1E57-42DA-B6E3-E1C8A848818A}.Release|Any CPU.ActiveCfg = Release|Any CPU {6515380E-1E57-42DA-B6E3-E1C8A848818A}.Release|Any CPU.Build.0 = Release|Any CPU + {D06BF829-23F5-40F3-A62D-627D9F4B4D6C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D06BF829-23F5-40F3-A62D-627D9F4B4D6C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D06BF829-23F5-40F3-A62D-627D9F4B4D6C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D06BF829-23F5-40F3-A62D-627D9F4B4D6C}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection EndGlobal diff --git a/Kyoo/Controllers/PluginManager.cs b/Kyoo/Controllers/PluginManager.cs index 7d0c399b..cd0a38f6 100644 --- a/Kyoo/Controllers/PluginManager.cs +++ b/Kyoo/Controllers/PluginManager.cs @@ -4,6 +4,7 @@ using System.IO; using System.Linq; using System.Reflection; using System.Runtime.Loader; +using Autofac; using Kyoo.Models.Options; using Microsoft.AspNetCore.Builder; using Microsoft.Extensions.DependencyInjection; @@ -126,6 +127,12 @@ namespace Kyoo.Controllers else _logger.LogInformation("Plugin enabled: {Plugins}", _plugins.Select(x => x.Name)); } + + public void ConfigureContainer(ContainerBuilder builder) + { + foreach (IPlugin plugin in _plugins) + plugin.Configure(builder); + } /// public void ConfigureServices(IServiceCollection services) diff --git a/Kyoo/Controllers/ProviderComposite.cs b/Kyoo/Controllers/ProviderComposite.cs new file mode 100644 index 00000000..9b7e498b --- /dev/null +++ b/Kyoo/Controllers/ProviderComposite.cs @@ -0,0 +1,182 @@ +using System; +using Kyoo.Models; +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace Kyoo.Controllers +{ + public class ProviderComposite : IMetadataProvider + { + private readonly IEnumerable _providers; + + + public ProviderComposite(IEnumerable providers) + { + _providers = providers; + } + + public Provider Provider { get; } + public Task Get(T item) where T : class, IResource + { + throw new NotImplementedException(); + } + + public Task> Search(string query) where T : class, IResource + { + throw new NotImplementedException(); + } + + public Task> GetPeople(Show show) + { + throw new NotImplementedException(); + } + + // private async Task GetMetadata(Func> providerCall, Library library, string what) + // where T : new() + // { + // T ret = new(); + // + // IEnumerable providers = library?.Providers + // .Select(x => _providers.FirstOrDefault(y => y.Provider.Slug == x.Slug)) + // .Where(x => x != null) + // ?? _providers; + // + // foreach (IMetadataProvider provider in providers) + // { + // try + // { + // ret = Merger.Merge(ret, await providerCall(provider)); + // } catch (Exception ex) + // { + // await Console.Error.WriteLineAsync( + // $"The provider {provider.Provider.Name} could not work for {what}. Exception: {ex.Message}"); + // } + // } + // return ret; + // } + // + // private async Task> GetMetadata( + // Func>> providerCall, + // Library library, + // string what) + // { + // List ret = new(); + // + // IEnumerable providers = library?.Providers + // .Select(x => _providers.FirstOrDefault(y => y.Provider.Slug == x.Slug)) + // .Where(x => x != null) + // ?? _providers; + // + // foreach (IMetadataProvider provider in providers) + // { + // try + // { + // ret.AddRange(await providerCall(provider) ?? new List()); + // } catch (Exception ex) + // { + // await Console.Error.WriteLineAsync( + // $"The provider {provider.Provider.Name} coudln't work for {what}. Exception: {ex.Message}"); + // } + // } + // return ret; + // } + // + // public async Task GetCollectionFromName(string name, Library library) + // { + // Collection collection = await GetMetadata( + // provider => provider.GetCollectionFromName(name), + // library, + // $"the collection {name}"); + // collection.Name ??= name; + // collection.Slug ??= Utility.ToSlug(name); + // return collection; + // } + // + // public async Task CompleteShow(Show show, Library library) + // { + // return await GetMetadata(provider => provider.GetShowByID(show), library, $"the show {show.Title}"); + // } + // + // public async Task SearchShow(string showName, bool isMovie, Library library) + // { + // Show show = await GetMetadata(async provider => + // { + // Show searchResult = (await provider.SearchShows(showName, isMovie))?.FirstOrDefault(); + // if (searchResult == null) + // return null; + // return await provider.GetShowByID(searchResult); + // }, library, $"the show {showName}"); + // show.Slug = Utility.ToSlug(showName); + // show.Title ??= showName; + // show.IsMovie = isMovie; + // show.Genres = show.Genres?.GroupBy(x => x.Slug).Select(x => x.First()).ToList(); + // show.People = show.People?.GroupBy(x => x.Slug).Select(x => x.First()).ToList(); + // return show; + // } + // + // public async Task> SearchShows(string showName, bool isMovie, Library library) + // { + // IEnumerable shows = await GetMetadata( + // provider => provider.SearchShows(showName, isMovie), + // library, + // $"the show {showName}"); + // return shows.Select(show => + // { + // show.Slug = Utility.ToSlug(showName); + // show.Title ??= showName; + // show.IsMovie = isMovie; + // return show; + // }); + // } + // + // public async Task GetSeason(Show show, int seasonNumber, Library library) + // { + // Season season = await GetMetadata( + // provider => provider.GetSeason(show, seasonNumber), + // library, + // $"the season {seasonNumber} of {show.Title}"); + // season.Show = show; + // season.ShowID = show.ID; + // season.ShowSlug = show.Slug; + // season.Title ??= $"Season {season.SeasonNumber}"; + // return season; + // } + // + // public async Task GetEpisode(Show show, + // string episodePath, + // int? seasonNumber, + // int? episodeNumber, + // int? absoluteNumber, + // Library library) + // { + // Episode episode = await GetMetadata( + // provider => provider.GetEpisode(show, seasonNumber, episodeNumber, absoluteNumber), + // library, + // "an episode"); + // episode.Show = show; + // episode.ShowID = show.ID; + // episode.ShowSlug = show.Slug; + // episode.Path = episodePath; + // episode.SeasonNumber ??= seasonNumber; + // episode.EpisodeNumber ??= episodeNumber; + // episode.AbsoluteNumber ??= absoluteNumber; + // return episode; + // } + // + // public async Task> GetPeople(Show show, Library library) + // { + // List people = await GetMetadata( + // provider => provider.GetPeople(show), + // library, + // $"a cast member of {show.Title}"); + // return people?.GroupBy(x => x.Slug) + // .Select(x => x.First()) + // .Select(x => + // { + // x.Show = show; + // x.ShowID = show.ID; + // return x; + // }).ToList(); + // } + } +} diff --git a/Kyoo/Controllers/ProviderManager.cs b/Kyoo/Controllers/ProviderManager.cs deleted file mode 100644 index 6ed5f796..00000000 --- a/Kyoo/Controllers/ProviderManager.cs +++ /dev/null @@ -1,166 +0,0 @@ -using System; -using Kyoo.Models; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; - -namespace Kyoo.Controllers -{ - public class ProviderManager : IProviderManager - { - private readonly IEnumerable _providers; - - public ProviderManager(IPluginManager pluginManager) - { - _providers = pluginManager.GetPlugins(); - } - - private async Task GetMetadata(Func> providerCall, Library library, string what) - where T : new() - { - T ret = new(); - - IEnumerable providers = library?.Providers - .Select(x => _providers.FirstOrDefault(y => y.Provider.Slug == x.Slug)) - .Where(x => x != null) - ?? _providers; - - foreach (IMetadataProvider provider in providers) - { - try - { - ret = Merger.Merge(ret, await providerCall(provider)); - } catch (Exception ex) - { - await Console.Error.WriteLineAsync( - $"The provider {provider.Provider.Name} could not work for {what}. Exception: {ex.Message}"); - } - } - return ret; - } - - private async Task> GetMetadata( - Func>> providerCall, - Library library, - string what) - { - List ret = new(); - - IEnumerable providers = library?.Providers - .Select(x => _providers.FirstOrDefault(y => y.Provider.Slug == x.Slug)) - .Where(x => x != null) - ?? _providers; - - foreach (IMetadataProvider provider in providers) - { - try - { - ret.AddRange(await providerCall(provider) ?? new List()); - } catch (Exception ex) - { - await Console.Error.WriteLineAsync( - $"The provider {provider.Provider.Name} coudln't work for {what}. Exception: {ex.Message}"); - } - } - return ret; - } - - public async Task GetCollectionFromName(string name, Library library) - { - Collection collection = await GetMetadata( - provider => provider.GetCollectionFromName(name), - library, - $"the collection {name}"); - collection.Name ??= name; - collection.Slug ??= Utility.ToSlug(name); - return collection; - } - - public async Task CompleteShow(Show show, Library library) - { - return await GetMetadata(provider => provider.GetShowByID(show), library, $"the show {show.Title}"); - } - - public async Task SearchShow(string showName, bool isMovie, Library library) - { - Show show = await GetMetadata(async provider => - { - Show searchResult = (await provider.SearchShows(showName, isMovie))?.FirstOrDefault(); - if (searchResult == null) - return null; - return await provider.GetShowByID(searchResult); - }, library, $"the show {showName}"); - show.Slug = Utility.ToSlug(showName); - show.Title ??= showName; - show.IsMovie = isMovie; - show.Genres = show.Genres?.GroupBy(x => x.Slug).Select(x => x.First()).ToList(); - show.People = show.People?.GroupBy(x => x.Slug).Select(x => x.First()).ToList(); - return show; - } - - public async Task> SearchShows(string showName, bool isMovie, Library library) - { - IEnumerable shows = await GetMetadata( - provider => provider.SearchShows(showName, isMovie), - library, - $"the show {showName}"); - return shows.Select(show => - { - show.Slug = Utility.ToSlug(showName); - show.Title ??= showName; - show.IsMovie = isMovie; - return show; - }); - } - - public async Task GetSeason(Show show, int seasonNumber, Library library) - { - Season season = await GetMetadata( - provider => provider.GetSeason(show, seasonNumber), - library, - $"the season {seasonNumber} of {show.Title}"); - season.Show = show; - season.ShowID = show.ID; - season.ShowSlug = show.Slug; - season.Title ??= $"Season {season.SeasonNumber}"; - return season; - } - - public async Task GetEpisode(Show show, - string episodePath, - int? seasonNumber, - int? episodeNumber, - int? absoluteNumber, - Library library) - { - Episode episode = await GetMetadata( - provider => provider.GetEpisode(show, seasonNumber, episodeNumber, absoluteNumber), - library, - "an episode"); - episode.Show = show; - episode.ShowID = show.ID; - episode.ShowSlug = show.Slug; - episode.Path = episodePath; - episode.SeasonNumber ??= seasonNumber; - episode.EpisodeNumber ??= episodeNumber; - episode.AbsoluteNumber ??= absoluteNumber; - return episode; - } - - public async Task> GetPeople(Show show, Library library) - { - List people = await GetMetadata( - provider => provider.GetPeople(show), - library, - $"a cast member of {show.Title}"); - return people?.GroupBy(x => x.Slug) - .Select(x => x.First()) - .Select(x => - { - x.Show = show; - x.ShowID = show.ID; - return x; - }).ToList(); - } - } -} diff --git a/Kyoo/CoreModule.cs b/Kyoo/CoreModule.cs index 34e24e75..a0dc0729 100644 --- a/Kyoo/CoreModule.cs +++ b/Kyoo/CoreModule.cs @@ -1,7 +1,9 @@ using System; using System.Collections.Generic; using System.IO; -using System.Linq; +using Autofac; +using Autofac.Core; +using Autofac.Core.Registration; using Kyoo.Controllers; using Kyoo.Models.Options; using Kyoo.Models.Permissions; @@ -34,7 +36,7 @@ namespace Kyoo typeof(IFileManager), typeof(ITranscoder), typeof(IThumbnailsManager), - typeof(IProviderManager), + typeof(IMetadataProvider), typeof(ITaskManager), typeof(ILibraryManager) }; @@ -88,6 +90,39 @@ namespace Kyoo _configuration = configuration; } + /// + public void Configure(ContainerBuilder builder) + { + builder.RegisterType().As().SingleInstance(); + builder.RegisterType().As().SingleInstance(); + builder.RegisterType().As().SingleInstance(); + builder.RegisterType().As().SingleInstance(); + builder.RegisterType().As().SingleInstance(); + builder.RegisterType().As().InstancePerLifetimeScope(); + builder.RegisterComposite().InstancePerLifetimeScope(); + + builder.RegisterTask(); + + static bool DatabaseIsPresent(IComponentRegistryBuilder x) + => x.IsRegistered(new TypedService(typeof(DatabaseContext))); + + builder.RegisterRepository().OnlyIf(DatabaseIsPresent); + builder.RegisterRepository().OnlyIf(DatabaseIsPresent); + builder.RegisterRepository().OnlyIf(DatabaseIsPresent); + builder.RegisterRepository().OnlyIf(DatabaseIsPresent); + builder.RegisterRepository().OnlyIf(DatabaseIsPresent); + builder.RegisterRepository().OnlyIf(DatabaseIsPresent); + builder.RegisterRepository().OnlyIf(DatabaseIsPresent); + builder.RegisterRepository().OnlyIf(DatabaseIsPresent); + builder.RegisterRepository().OnlyIf(DatabaseIsPresent); + builder.RegisterRepository().OnlyIf(DatabaseIsPresent); + builder.RegisterRepository().OnlyIf(DatabaseIsPresent); + builder.RegisterRepository().OnlyIf(DatabaseIsPresent); + + builder.RegisterType().As() + .IfNotRegistered(typeof(IPermissionValidator)); + } + /// public void Configure(IServiceCollection services, ICollection availableTypes) { @@ -109,36 +144,7 @@ namespace Kyoo x.SerializerSettings.Converters.Add(new PeopleRoleConverter()); }); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); services.AddHostedService(x => x.GetService() as TaskManager); - - services.AddScoped(); - - if (ProviderCondition.Has(typeof(DatabaseContext), availableTypes)) - { - services.AddRepository(); - services.AddRepository(); - services.AddRepository(); - services.AddRepository(); - services.AddRepository(); - services.AddRepository(); - services.AddRepository(); - services.AddRepository(); - services.AddRepository(); - services.AddRepository(); - services.AddRepository(); - services.AddRepository(); - } - - services.AddTask(); - - if (services.All(x => x.ServiceType != typeof(IPermissionValidator))) - services.AddSingleton(); } /// diff --git a/Kyoo/Kyoo.csproj b/Kyoo/Kyoo.csproj index 438f3e9b..b1bad964 100644 --- a/Kyoo/Kyoo.csproj +++ b/Kyoo/Kyoo.csproj @@ -34,6 +34,8 @@ + + diff --git a/Kyoo/Program.cs b/Kyoo/Program.cs index 12227266..44b80849 100644 --- a/Kyoo/Program.cs +++ b/Kyoo/Program.cs @@ -2,12 +2,13 @@ using System; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Threading.Tasks; +using Autofac.Extensions.DependencyInjection; using Microsoft.AspNetCore.Hosting; -using Microsoft.AspNetCore.Hosting.StaticWebAssets; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; + namespace Kyoo { /// @@ -30,6 +31,8 @@ namespace Kyoo if (!File.Exists("./settings.json")) File.Copy(Path.Join(AppDomain.CurrentDomain.BaseDirectory, "settings.json"), "settings.json"); + IHostBuilder builder = CreateWebHostBuilder(args); + bool? debug = Environment.GetEnvironmentVariable("ENVIRONMENT")?.ToLowerInvariant() switch { "d" => true, @@ -43,18 +46,21 @@ namespace Kyoo }; if (debug == null && Environment.GetEnvironmentVariable("ENVIRONMENT") != null) - Console.WriteLine($"Invalid ENVIRONMENT variable. Supported values are \"debug\" and \"prod\". Ignoring..."); + { + Console.WriteLine( + $"Invalid ENVIRONMENT variable. Supported values are \"debug\" and \"prod\". Ignoring..."); + } #if DEBUG debug ??= true; #endif - Console.WriteLine($"Running as {Environment.UserName}."); - IWebHostBuilder builder = CreateWebHostBuilder(args); if (debug != null) builder = builder.UseEnvironment(debug == true ? "Development" : "Production"); + try { + Console.WriteLine($"Running as {Environment.UserName}."); await builder.Build().RunAsync(); } catch (Exception ex) @@ -81,13 +87,14 @@ namespace Kyoo /// /// Command line parameters that can be handled by kestrel /// A new web host instance - private static IWebHostBuilder CreateWebHostBuilder(string[] args) + private static IHostBuilder CreateWebHostBuilder(string[] args) { IConfiguration configuration = SetupConfig(new ConfigurationBuilder(), args).Build(); - return new WebHostBuilder() + + return new HostBuilder() + .UseServiceProviderFactory(new AutofacServiceProviderFactory()) .UseContentRoot(AppDomain.CurrentDomain.BaseDirectory) - .UseConfiguration(configuration) .ConfigureAppConfiguration(x => SetupConfig(x, args)) .ConfigureLogging((context, builder) => { @@ -99,18 +106,20 @@ namespace Kyoo .AddDebug() .AddEventSourceLogger(); }) - .UseDefaultServiceProvider((context, options) => - { - options.ValidateScopes = context.HostingEnvironment.IsDevelopment(); - if (context.HostingEnvironment.IsDevelopment()) - StaticWebAssetsLoader.UseStaticWebAssets(context.HostingEnvironment, context.Configuration); - }) + // .UseDefaultServiceProvider((context, options) => + // { + // options.ValidateScopes = context.HostingEnvironment.IsDevelopment(); + // if (context.HostingEnvironment.IsDevelopment()) + // StaticWebAssetsLoader.UseStaticWebAssets(context.HostingEnvironment, context.Configuration); + // }) .ConfigureServices(x => x.AddRouting()) - .UseKestrel(options => { options.AddServerHeader = false; }) - .UseIIS() - .UseIISIntegration() - .UseUrls(configuration.GetValue("basics:url")) - .UseStartup(); + .ConfigureWebHost(x => x + .UseKestrel(options => { options.AddServerHeader = false; }) + .UseIIS() + .UseIISIntegration() + .UseUrls(configuration.GetValue("basics:url")) + .UseStartup() + ); } } } diff --git a/Kyoo/Startup.cs b/Kyoo/Startup.cs index 4d5995ef..5557e872 100644 --- a/Kyoo/Startup.cs +++ b/Kyoo/Startup.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using Autofac; using Kyoo.Authentication; using Kyoo.Controllers; using Kyoo.Models; @@ -71,12 +72,16 @@ namespace Kyoo services.AddHttpClient(); - services.AddTransient(typeof(Lazy<>), typeof(LazyDi<>)); - - services.AddSingleton(_plugins); - services.AddTask(); + // services.AddTransient(typeof(Lazy<>), typeof(LazyDi<>)); _plugins.ConfigureServices(services); } + + public void ConfigureContainer(ContainerBuilder builder) + { + builder.RegisterInstance(_plugins).As().ExternallyOwned(); + builder.RegisterTask(); + _plugins.ConfigureContainer(builder); + } /// /// Configure the asp net host. diff --git a/Kyoo/Tasks/Crawler.cs b/Kyoo/Tasks/Crawler.cs index 9f1185dd..e9d99010 100644 --- a/Kyoo/Tasks/Crawler.cs +++ b/Kyoo/Tasks/Crawler.cs @@ -25,7 +25,7 @@ namespace Kyoo.Tasks [Injected] public IServiceProvider ServiceProvider { private get; set; } [Injected] public IThumbnailsManager ThumbnailsManager { private get; set; } - [Injected] public IProviderManager MetadataProvider { private get; set; } + [Injected] public IMetadataProvider MetadataProvider { private get; set; } [Injected] public ITranscoder Transcoder { private get; set; } [Injected] public IConfiguration Config { private get; set; } @@ -258,7 +258,7 @@ namespace Kyoo.Tasks Collection collection = await libraryManager.GetOrDefault(Utility.ToSlug(collectionName)); if (collection != null) return collection; - collection = await MetadataProvider.GetCollectionFromName(collectionName, library); + // collection = await MetadataProvider.GetCollectionFromName(collectionName, library); try { @@ -283,9 +283,10 @@ namespace Kyoo.Tasks await libraryManager.Load(old, x => x.ExternalIDs); return old; } - Show show = await MetadataProvider.SearchShow(showTitle, isMovie, library); + + Show show = new();//await MetadataProvider.SearchShow(showTitle, isMovie, library); show.Path = showPath; - show.People = await MetadataProvider.GetPeople(show, library); + // show.People = await MetadataProvider.GetPeople(show, library); try { @@ -325,7 +326,7 @@ namespace Kyoo.Tasks } catch (ItemNotFoundException) { - Season season = await MetadataProvider.GetSeason(show, seasonNumber, library); + Season season = new();//await MetadataProvider.GetSeason(show, seasonNumber, library); try { await libraryManager.Create(season); @@ -348,12 +349,12 @@ namespace Kyoo.Tasks string episodePath, Library library) { - Episode episode = await MetadataProvider.GetEpisode(show, + Episode episode = new();/*await MetadataProvider.GetEpisode(show, episodePath, season?.SeasonNumber, episodeNumber, absoluteNumber, - library); + library);*/ if (episode.SeasonNumber != null) { From ecb41019247a0e1d429bfdd95232651b695ffc47 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Wed, 14 Jul 2021 00:33:10 +0200 Subject: [PATCH 02/31] Fixing autofac container creation --- Kyoo/Program.cs | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/Kyoo/Program.cs b/Kyoo/Program.cs index 44b80849..f1adfb40 100644 --- a/Kyoo/Program.cs +++ b/Kyoo/Program.cs @@ -2,11 +2,12 @@ using System; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Threading.Tasks; +using Autofac; using Autofac.Extensions.DependencyInjection; using Microsoft.AspNetCore.Hosting; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; namespace Kyoo @@ -31,7 +32,7 @@ namespace Kyoo if (!File.Exists("./settings.json")) File.Copy(Path.Join(AppDomain.CurrentDomain.BaseDirectory, "settings.json"), "settings.json"); - IHostBuilder builder = CreateWebHostBuilder(args); + IWebHostBuilder builder = CreateWebHostBuilder(args); bool? debug = Environment.GetEnvironmentVariable("ENVIRONMENT")?.ToLowerInvariant() switch { @@ -87,14 +88,18 @@ namespace Kyoo /// /// Command line parameters that can be handled by kestrel /// A new web host instance - private static IHostBuilder CreateWebHostBuilder(string[] args) + private static IWebHostBuilder CreateWebHostBuilder(string[] args) { IConfiguration configuration = SetupConfig(new ConfigurationBuilder(), args).Build(); - - return new HostBuilder() - .UseServiceProviderFactory(new AutofacServiceProviderFactory()) + return new WebHostBuilder() + .ConfigureServices(x => + { + AutofacServiceProviderFactory factory = new(); + x.Replace(ServiceDescriptor.Singleton>(factory)); + }) .UseContentRoot(AppDomain.CurrentDomain.BaseDirectory) + .UseConfiguration(configuration) .ConfigureAppConfiguration(x => SetupConfig(x, args)) .ConfigureLogging((context, builder) => { @@ -106,20 +111,12 @@ namespace Kyoo .AddDebug() .AddEventSourceLogger(); }) - // .UseDefaultServiceProvider((context, options) => - // { - // options.ValidateScopes = context.HostingEnvironment.IsDevelopment(); - // if (context.HostingEnvironment.IsDevelopment()) - // StaticWebAssetsLoader.UseStaticWebAssets(context.HostingEnvironment, context.Configuration); - // }) .ConfigureServices(x => x.AddRouting()) - .ConfigureWebHost(x => x - .UseKestrel(options => { options.AddServerHeader = false; }) - .UseIIS() - .UseIISIntegration() - .UseUrls(configuration.GetValue("basics:url")) - .UseStartup() - ); + .UseKestrel(options => { options.AddServerHeader = false; }) + .UseIIS() + .UseIISIntegration() + .UseUrls(configuration.GetValue("basics:url")) + .UseStartup(); } } } From 2d45d6422d56d082a26502230aca452c60f51b46 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Wed, 14 Jul 2021 01:50:05 +0200 Subject: [PATCH 03/31] Implementing the tvdb provider --- Kyoo.Common/Controllers/IMetadataProvider.cs | 2 +- Kyoo.TheTvdb/Convertors.cs | 157 +++++++++++ Kyoo.TheTvdb/PluginTVDB.cs | 6 +- Kyoo.TheTvdb/ProviderTVDB.cs | 260 ++++++------------- 4 files changed, 245 insertions(+), 180 deletions(-) create mode 100644 Kyoo.TheTvdb/Convertors.cs diff --git a/Kyoo.Common/Controllers/IMetadataProvider.cs b/Kyoo.Common/Controllers/IMetadataProvider.cs index 9c260030..f17e2d8d 100644 --- a/Kyoo.Common/Controllers/IMetadataProvider.cs +++ b/Kyoo.Common/Controllers/IMetadataProvider.cs @@ -35,7 +35,7 @@ namespace Kyoo.Controllers /// If this metadata provider does not support . /// /// A new containing metadata from your provider - [ItemNotNull] + [ItemCanBeNull] Task Get([NotNull] T item) where T : class, IResource; diff --git a/Kyoo.TheTvdb/Convertors.cs b/Kyoo.TheTvdb/Convertors.cs new file mode 100644 index 00000000..b7edd2ea --- /dev/null +++ b/Kyoo.TheTvdb/Convertors.cs @@ -0,0 +1,157 @@ +using System; +using System.Globalization; +using System.Linq; +using Kyoo.Models; +using TvDbSharper.Dto; + +namespace Kyoo.TheTvdb +{ + /// + /// A set of extensions methods used to convert tvdb models to Kyoo models. + /// + public static class Convertors + { + /// + /// Convert the string representation of the status in the tvdb API to a Kyoo's enum. + /// + /// The string representing the status. + /// A kyoo value or null. + private static Status? GetStatus(string status) + { + return status switch + { + "Ended" => Status.Finished, + "Continuing" => Status.Airing, + _ => null + }; + } + + /// + /// Parse a TVDB date and return a or null if the string is invalid. + /// + /// The date string to parse + /// The parsed or null. + private static DateTime ParseDate(string date) + { + DateTime.TryParseExact(date, "yyyy-mm-dd", CultureInfo.InvariantCulture, DateTimeStyles.None, out DateTime parsed); + return parsed; + } + + /// + /// Convert a series search to a show. + /// + /// The search result + /// The provider representing the tvdb inside kyoo + /// A show representing the given search result. + public static Show ToShow(this SeriesSearchResult result, Provider provider) + { + return new() + { + Slug = result.Slug, + Title = result.SeriesName, + Aliases = result.Aliases, + Overview = result.Overview, + Status = GetStatus(result.Status), + StartAir = ParseDate(result.FirstAired), + Poster = result.Poster != null ? $"https://www.thetvdb.com{result.Poster}" : null, + ExternalIDs = new[] + { + new MetadataID + { + DataID = result.Id.ToString(), + Link = $"https://www.thetvdb.com/series/{result.Slug}", + Second = provider + } + } + }; + } + + /// + /// Convert a tvdb series to a kyoo show. + /// + /// The series to convert + /// The provider representing the tvdb inside kyoo + /// A show representing the given series. + public static Show ToShow(this Series series, Provider provider) + { + return new() + { + Slug = series.Slug, + Title = series.SeriesName, + Aliases = series.Aliases, + Overview = series.Overview, + Status = GetStatus(series.Status), + StartAir = ParseDate(series.FirstAired), + Poster = series.Poster != null ? $"https://www.thetvdb.com/banners/{series.Poster}" : null, + Backdrop = series.FanArt != null ? $"https://www.thetvdb.com/banners/{series.FanArt}" : null, + Genres = series.Genre.Select(y => new Genre(y)).ToList(), + ExternalIDs = new[] + { + new MetadataID + { + DataID = series.Id.ToString(), + Link = $"https://www.thetvdb.com/series/{series.Slug}", + Second = provider + } + } + }; + } + + /// + /// Convert a tvdb actor to a kyoo . + /// + /// The actor to convert + /// The provider representing the tvdb inside kyoo + /// A people role representing the given actor in the role they played. + public static PeopleRole ToPeopleRole(this Actor actor, Provider provider) + { + return new() + { + People = new People + { + Slug = Utility.ToSlug(actor.Name), + Name = actor.Name, + Poster = actor.Image != null ? $"https://www.thetvdb.com/banners/{actor.Image}" : null, + ExternalIDs = new [] + { + new MetadataID() + { + DataID = actor.Id.ToString(), + Link = $"https://www.thetvdb.com/people/{actor.Id}", + Second = provider + } + } + }, + Role = actor.Role + }; + } + + /// + /// Convert a tvdb episode to a kyoo . + /// + /// The episode to convert + /// The provider representing the tvdb inside kyoo + /// A episode representing the given tvdb episode. + public static Episode ToEpisode(this EpisodeRecord episode, Provider provider) + { + return new() + { + SeasonNumber = episode.AiredSeason, + EpisodeNumber = episode.AiredEpisodeNumber, + AbsoluteNumber = episode.AbsoluteNumber, + Title = episode.EpisodeName, + Overview = episode.Overview, + Thumb = episode.Filename != null ? $"https://www.thetvdb.com/banners/{episode.Filename}" : null, + ExternalIDs = new[] + { + new MetadataID + { + DataID = episode.Id.ToString(), + Link = $"https://www.thetvdb.com/series/{episode.SeriesId}/episodes/{episode.Id}", + Second = provider + } + } + }; + } + } +} \ No newline at end of file diff --git a/Kyoo.TheTvdb/PluginTVDB.cs b/Kyoo.TheTvdb/PluginTVDB.cs index 6484b51d..ea4cab62 100644 --- a/Kyoo.TheTvdb/PluginTVDB.cs +++ b/Kyoo.TheTvdb/PluginTVDB.cs @@ -1,7 +1,7 @@ using System; using System.Collections.Generic; +using Autofac; using Kyoo.Controllers; -using Microsoft.Extensions.DependencyInjection; namespace Kyoo.TheTvdb { @@ -33,9 +33,9 @@ namespace Kyoo.TheTvdb /// - public void Configure(IServiceCollection services, ICollection availableTypes) + public void Configure(ContainerBuilder builder) { - // services.AddProvider(); + builder.RegisterProvider(); } } } \ No newline at end of file diff --git a/Kyoo.TheTvdb/ProviderTVDB.cs b/Kyoo.TheTvdb/ProviderTVDB.cs index eeb065c9..44841326 100644 --- a/Kyoo.TheTvdb/ProviderTVDB.cs +++ b/Kyoo.TheTvdb/ProviderTVDB.cs @@ -1,8 +1,8 @@ using System; using System.Collections.Generic; -using System.Globalization; using System.Linq; using System.Threading.Tasks; +using JetBrains.Annotations; using Kyoo.Controllers; using Kyoo.Models; using TvDbSharper; @@ -15,189 +15,97 @@ namespace Kyoo.TheTvdb /// public class ProviderTvdb : IMetadataProvider { - public Provider Provider { get; } - public Task Get(T item) where T : class, IResource + /// + /// The internal tvdb client used to make requests. + /// + private readonly TvDbClient _client = new(); + + /// + /// The API key used to authenticate with the tvdb API. + /// + private readonly string _apiKey; + + /// + public Provider Provider => new() { - throw new NotImplementedException(); + Slug = "the-tvdb", + Name = "TheTVDB", + LogoExtension = "png", + Logo = "https://www.thetvdb.com/images/logo.png" + }; + + + public ProviderTvdb(string apiKey) + { + _apiKey = apiKey; } - public Task> Search(string query) where T : class, IResource + private Task _Authenticate() { - throw new NotImplementedException(); + if (_client.Authentication.Token == null) + return _client.Authentication.AuthenticateAsync(_apiKey); + return _client.Authentication.RefreshTokenAsync(); + } + + /// + public async Task Get(T item) + where T : class, IResource + { + await _Authenticate(); + return item switch + { + Show show => await _GetShow(show) as T, + Episode episode => await _GetEpisode(episode) as T, + _ => throw new NotSupportedException() + }; + } + + [ItemCanBeNull] + private async Task _GetShow([NotNull] Show show) + { + if (!int.TryParse(show.GetID(Provider.Slug), out int id)) + return (await _SearchShow(show.Title)).FirstOrDefault(); + TvDbResponse series = await _client.Series.GetAsync(id); + return series.Data.ToShow(Provider); } - public Task> GetPeople(Show show) + [ItemCanBeNull] + private async Task _GetEpisode([NotNull] Episode episode) { + if (!int.TryParse(episode.Show?.GetID(Provider.Slug), out int id)) + return null; + EpisodeQuery query = episode.AbsoluteNumber != null + ? new EpisodeQuery {AbsoluteNumber = episode.AbsoluteNumber} + : new EpisodeQuery {AiredSeason = episode.SeasonNumber, AiredEpisode = episode.EpisodeNumber}; + TvDbResponse episodes = await _client.Series.GetEpisodesAsync(id, 0, query); + return episodes.Data.FirstOrDefault()?.ToEpisode(Provider); + } + + /// + public async Task> Search(string query) + where T : class, IResource + { + await _Authenticate(); + if (typeof(T) == typeof(Show)) + return (await _SearchShow(query) as ICollection)!; throw new NotImplementedException(); } + + [ItemNotNull] + private async Task> _SearchShow(string query) + { + TvDbResponse shows = await _client.Search.SearchSeriesByNameAsync(query); + return shows.Data.Select(x => x.ToShow(Provider)).ToArray(); + } + + /// + public async Task> GetPeople(Show show) + { + if (!int.TryParse(show?.GetID(Provider.Name), out int id)) + return null; + await _Authenticate(); + TvDbResponse people = await _client.Series.GetActorsAsync(id); + return people.Data.Select(x => x.ToPeopleRole(Provider)).ToArray(); + } } - - - // public class Old - // { - // private static readonly ProviderID _provider = new() - // { - // Slug = "the-tvdb", - // Name = "TheTVDB", - // LogoExtension = "png", - // Logo = "https://www.thetvdb.com/images/logo.png" - // }; - // public ProviderID Provider => _provider; - // - // - // private readonly TvDbClient _client = new(); - // - // private Task Authentificate() - // { - // if (_client.Authentication.Token == null) - // return _client.Authentication.AuthenticateAsync(APIKey); - // return _client.Authentication.RefreshTokenAsync(); - // } - // - // public Task GetCollectionFromName(string name) - // { - // return Task.FromResult(null); - // } - // - // public async Task> SearchShows(string showName, bool isMovie) - // { - // await Authentificate(); - // - // if (isMovie) - // return null; //There is no movie search API for now on TheTVDB. - // TvDbResponse shows = await _client.Search.SearchSeriesAsync(showName, SearchParameter.Name); - // return shows.Data.Select(x => x.ToShow(Provider)).ToArray(); - // } - // - // public async Task GetShowByID(Show show) - // { - // if (!int.TryParse(show?.GetID(Provider.Name), out int id)) - // return await Task.FromResult(null); - // await Authentificate(); - // TvDbResponse serie = await _client.Series.GetAsync(id); - // return serie.Data.ToShow(Provider); - // } - // - // public async Task> GetPeople(Show show) - // { - // if (!int.TryParse(show?.GetID(Provider.Name), out int id)) - // return null; - // await Authentificate(); - // TvDbResponse people = await _client.Series.GetActorsAsync(id); - // return people.Data.Select(x => x.ToPeopleRole(Provider)).ToArray(); - // } - // - // public Task GetSeason(Show show, int seasonNumber) - // { - // return Task.FromResult(null); - // } - // - // public async Task GetEpisode(Show show, int seasonNumber, int episodeNumber, int absoluteNumber) - // { - // if (!int.TryParse(show?.GetID(Provider.Name), out int id)) - // return null; - // await Authentificate(); - // TvDbResponse episodes = absoluteNumber != -1 - // ? await _client.Series.GetEpisodesAsync(id, 0, new EpisodeQuery {AbsoluteNumber = absoluteNumber}) - // : await _client.Series.GetEpisodesAsync(id, 0, new EpisodeQuery {AiredSeason = seasonNumber, AiredEpisode = episodeNumber}); - // EpisodeRecord x = episodes.Data[0]; - // - // if (absoluteNumber == -1) - // absoluteNumber = x.AbsoluteNumber ?? -1; - // else - // { - // seasonNumber = x.AiredSeason ?? -1; - // episodeNumber = x.AiredEpisodeNumber ?? -1; - // } - // - // return new Episode(seasonNumber, - // episodeNumber, - // absoluteNumber, - // x.EpisodeName, - // x.Overview, - // DateTime.ParseExact(x.FirstAired, "yyyy-MM-dd", CultureInfo.InvariantCulture), - // -1, - // x.Filename != null ? "https://www.thetvdb.com/banners/" + x.Filename : null, - // new [] - // { - // new MetadataID(Provider, x.Id.ToString(), $"https://www.thetvdb.com/series/{id}/episodes/{x.Id}") - // }); - // } - // } - - // public static class Convertors - // { - // private static int? GetYear(string firstAired) - // { - // if (firstAired?.Length >= 4 && int.TryParse(firstAired.Substring(0, 4), out int year)) - // return year; - // return null; - // } - // - // private static Status? GetStatus(string status) - // { - // return status switch - // { - // "Ended" => Status.Finished, - // "Continuing" => Status.Airing, - // _ => null - // }; - // } - // - // public static Show ToShow(this SeriesSearchResult x, ProviderID provider) - // { - // Show ret = new(x.Slug, - // x.SeriesName, - // x.Aliases, - // null, - // x.Overview, - // null, - // null, - // GetStatus(x.Status), - // GetYear(x.FirstAired), - // null, - // new[] - // { - // new MetadataID(provider, x.Id.ToString(), $"https://www.thetvdb.com/series/{x.Slug}") - // }); - // if (x.Poster != null) - // Utility.SetImage(ret, $"https://www.thetvdb.com{x.Poster}", ImageType.Poster); - // return ret; - // } - // - // public static Show ToShow(this Series x, ProviderID provider) - // { - // return new(x.Slug, - // x.SeriesName, - // x.Aliases, - // null, - // x.Overview, - // null, - // x.Genre.Select(y => new Genre(Utility.ToSlug(y), y)), - // GetStatus(x.Status), - // GetYear(x.FirstAired), - // null, - // new[] - // { - // new MetadataID(provider, x.Id.ToString(),$"https://www.thetvdb.com/series/{x.Slug}") - // }) - // { - // Poster = x.Poster != null ? $"https://www.thetvdb.com/banners/{x.Poster}" : null, - // Backdrop = x.FanArt != null ? $"https://www.thetvdb.com/banners/{x.FanArt}" : null - // }; - // } - // - // public static PeopleRole ToPeopleRole(this Actor x, ProviderID provider) - // { - // return new (Utility.ToSlug(x.Name), - // x.Name, - // x.Role, - // null, - // x.Image != null ? $"https://www.thetvdb.com/banners/{x.Image}" : null, - // new[] - // { - // new MetadataID(provider, x.Id.ToString(), $"https://www.thetvdb.com/people/{x.Id}") - // }); - // } - // } } \ No newline at end of file From 0265c270102b9f376f51d44d1cba12e75a6191ce Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Thu, 15 Jul 2021 18:13:55 +0200 Subject: [PATCH 04/31] Creating the provider composite and testing the merger --- Kyoo.Common/Controllers/IMetadataProvider.cs | 14 ++ Kyoo.Common/Utility/Merger.cs | 17 +- Kyoo.Common/Utility/Utility.cs | 153 ++++++++++++++- Kyoo.Tests/Utility/MergerTests.cs | 191 +++++++++++++++++++ Kyoo.Tests/Utility/UtilityTests.cs | 43 +++++ Kyoo/Controllers/ProviderComposite.cs | 160 ++++++++++------ 6 files changed, 504 insertions(+), 74 deletions(-) diff --git a/Kyoo.Common/Controllers/IMetadataProvider.cs b/Kyoo.Common/Controllers/IMetadataProvider.cs index f17e2d8d..7f1bac18 100644 --- a/Kyoo.Common/Controllers/IMetadataProvider.cs +++ b/Kyoo.Common/Controllers/IMetadataProvider.cs @@ -53,4 +53,18 @@ namespace Kyoo.Controllers Task> GetPeople(Show show); } + + /// + /// A special that merge results. + /// This interface exists to specify witch provider to use but it can be used like any other metadata provider. + /// + public interface IProviderComposite : IMetadataProvider + { + /// + /// Select witch providers to use. + /// The associated with the given will be used. + /// + /// The list of providers to use + void UseProviders(IEnumerable providers); + } } diff --git a/Kyoo.Common/Utility/Merger.cs b/Kyoo.Common/Utility/Merger.cs index 55cc17e3..a1f35756 100644 --- a/Kyoo.Common/Utility/Merger.cs +++ b/Kyoo.Common/Utility/Merger.cs @@ -5,6 +5,7 @@ using System.ComponentModel; using System.Linq; using System.Reflection; using JetBrains.Annotations; +using Kyoo.Models; using Kyoo.Models.Attributes; namespace Kyoo @@ -111,7 +112,8 @@ namespace Kyoo /// Missing fields of first will be completed by fields of this item. If second is null, the function no-op. /// Fields of T will be merged /// - public static T Merge(T first, T second) + [ContractAnnotation("=> null; first:notnull => notnull; second:notnull => notnull")] + public static T Merge([CanBeNull] T first, [CanBeNull] T second) { if (first == null) return second; @@ -127,9 +129,7 @@ namespace Kyoo { object oldValue = property.GetValue(first); object newValue = property.GetValue(second); - object defaultValue = property.PropertyType.IsValueType - ? Activator.CreateInstance(property.PropertyType) - : null; + object defaultValue = property.PropertyType.GetClrDefault(); if (oldValue?.Equals(defaultValue) != false) property.SetValue(first, newValue); @@ -139,11 +139,14 @@ namespace Kyoo Type enumerableType = Utility.GetGenericDefinition(property.PropertyType, typeof(IEnumerable<>)) .GenericTypeArguments .First(); + Func equalityComparer = enumerableType.IsAssignableTo(typeof(IResource)) + ? (x, y) => x.Slug == y.Slug + : null; property.SetValue(first, Utility.RunGenericMethod( - typeof(Utility), + typeof(Merger), nameof(MergeLists), - enumerableType, - oldValue, newValue, null)); + enumerableType, + oldValue, newValue, equalityComparer)); } } diff --git a/Kyoo.Common/Utility/Utility.cs b/Kyoo.Common/Utility/Utility.cs index f7041c51..f7c1a94b 100644 --- a/Kyoo.Common/Utility/Utility.cs +++ b/Kyoo.Common/Utility/Utility.cs @@ -182,15 +182,49 @@ namespace Kyoo return types.FirstOrDefault(x => x.IsGenericType && x.GetGenericTypeDefinition() == genericType); } - public static MethodInfo GetMethod(Type type, BindingFlags flag, string name, Type[] generics, object[] args) + /// + /// Retrieve a method from an with the given name and respect the + /// amount of parameters and generic parameters. This works for polymorphic methods. + /// + /// + /// The type owning the method. For non static methods, this is the this. + /// + /// + /// The binding flags of the method. This allow you to specify public/private and so on. + /// + /// + /// The name of the method. + /// + /// + /// The list of generic parameters. + /// + /// + /// The list of parameters. + /// + /// No method match the given constraints. + /// The method handle of the matching method. + [PublicAPI] + [NotNull] + public static MethodInfo GetMethod([NotNull] Type type, + BindingFlags flag, + string name, + [NotNull] Type[] generics, + [NotNull] object[] args) { + if (type == null) + throw new ArgumentNullException(nameof(type)); + if (generics == null) + throw new ArgumentNullException(nameof(generics)); + if (args == null) + throw new ArgumentNullException(nameof(args)); + MethodInfo[] methods = type.GetMethods(flag | BindingFlags.Public) .Where(x => x.Name == name) .Where(x => x.GetGenericArguments().Length == generics.Length) .Where(x => x.GetParameters().Length == args.Length) - .IfEmpty(() => throw new NullReferenceException($"A method named {name} with " + - $"{args.Length} arguments and {generics.Length} generic " + - $"types could not be found on {type.Name}.")) + .IfEmpty(() => throw new ArgumentException($"A method named {name} with " + + $"{args.Length} arguments and {generics.Length} generic " + + $"types could not be found on {type.Name}.")) // TODO this won't work but I don't know why. // .Where(x => // { @@ -211,9 +245,34 @@ namespace Kyoo if (methods.Length == 1) return methods[0]; - throw new NullReferenceException($"Multiple methods named {name} match the generics and parameters constraints."); + throw new ArgumentException($"Multiple methods named {name} match the generics and parameters constraints."); } + /// + /// Run a generic static method for a runtime . + /// + /// + /// To run for a List where you don't know the type at compile type, + /// you could do: + /// + /// Utility.RunGenericMethod<object>( + /// typeof(Utility), + /// nameof(MergeLists), + /// enumerableType, + /// oldValue, newValue, equalityComparer) + /// + /// + /// The type that owns the method. For non static methods, the type of this. + /// The name of the method. You should use the nameof keyword. + /// The generic type to run the method with. + /// The list of arguments of the method + /// + /// The return type of the method. You can put for an unknown one. + /// + /// No method match the given constraints. + /// The return of the method you wanted to run. + /// + /// public static T RunGenericMethod( [NotNull] Type owner, [NotNull] string methodName, @@ -223,6 +282,34 @@ namespace Kyoo return RunGenericMethod(owner, methodName, new[] {type}, args); } + /// + /// Run a generic static method for a multiple runtime . + /// If your generic method only needs one type, see + /// + /// + /// + /// To run for a List where you don't know the type at compile type, + /// you could do: + /// + /// Utility.RunGenericMethod<object>( + /// typeof(Utility), + /// nameof(MergeLists), + /// enumerableType, + /// oldValue, newValue, equalityComparer) + /// + /// + /// The type that owns the method. For non static methods, the type of this. + /// The name of the method. You should use the nameof keyword. + /// The list of generic types to run the method with. + /// The list of arguments of the method + /// + /// The return type of the method. You can put for an unknown one. + /// + /// No method match the given constraints. + /// The return of the method you wanted to run. + /// + /// + [PublicAPI] public static T RunGenericMethod( [NotNull] Type owner, [NotNull] string methodName, @@ -238,9 +325,34 @@ namespace Kyoo if (types.Length < 1) throw new ArgumentException($"The {nameof(types)} array is empty. At least one type is needed."); MethodInfo method = GetMethod(owner, BindingFlags.Static, methodName, types, args); - return (T)method.MakeGenericMethod(types).Invoke(null, args?.ToArray()); + return (T)method.MakeGenericMethod(types).Invoke(null, args.ToArray()); } + /// + /// Run a generic method for a runtime . + /// + /// + /// To run for a List where you don't know the type at compile type, + /// you could do: + /// + /// Utility.RunGenericMethod<object>( + /// typeof(Utility), + /// nameof(MergeLists), + /// enumerableType, + /// oldValue, newValue, equalityComparer) + /// + /// + /// The this of the method to run. + /// The name of the method. You should use the nameof keyword. + /// The generic type to run the method with. + /// The list of arguments of the method + /// + /// The return type of the method. You can put for an unknown one. + /// + /// No method match the given constraints. + /// The return of the method you wanted to run. + /// + /// public static T RunGenericMethod( [NotNull] object instance, [NotNull] string methodName, @@ -250,6 +362,33 @@ namespace Kyoo return RunGenericMethod(instance, methodName, new[] {type}, args); } + /// + /// Run a generic method for a multiple runtime . + /// If your generic method only needs one type, see + /// + /// + /// + /// To run for a List where you don't know the type at compile type, + /// you could do: + /// + /// Utility.RunGenericMethod<object>( + /// typeof(Utility), + /// nameof(MergeLists), + /// enumerableType, + /// oldValue, newValue, equalityComparer) + /// + /// + /// The this of the method to run. + /// The name of the method. You should use the nameof keyword. + /// The list of generic types to run the method with. + /// The list of arguments of the method + /// + /// The return type of the method. You can put for an unknown one. + /// + /// No method match the given constraints. + /// The return of the method you wanted to run. + /// + /// public static T RunGenericMethod( [NotNull] object instance, [NotNull] string methodName, @@ -263,7 +402,7 @@ namespace Kyoo if (types == null || types.Length == 0) throw new ArgumentNullException(nameof(types)); MethodInfo method = GetMethod(instance.GetType(), BindingFlags.Instance, methodName, types, args); - return (T)method.MakeGenericMethod(types).Invoke(instance, args?.ToArray()); + return (T)method.MakeGenericMethod(types).Invoke(instance, args.ToArray()); } public static string ToQueryString(this Dictionary query) diff --git a/Kyoo.Tests/Utility/MergerTests.cs b/Kyoo.Tests/Utility/MergerTests.cs index 614d328f..008daee1 100644 --- a/Kyoo.Tests/Utility/MergerTests.cs +++ b/Kyoo.Tests/Utility/MergerTests.cs @@ -1,4 +1,8 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using System.Linq; using Kyoo.Models; +using Kyoo.Models.Attributes; using Xunit; namespace Kyoo.Tests @@ -17,5 +21,192 @@ namespace Kyoo.Tests Assert.Null(genre.Name); Assert.Null(genre.Slug); } + + [Fact] + public void MergeTest() + { + Genre genre = new() + { + ID = 5 + }; + Genre genre2 = new() + { + Name = "test" + }; + Genre ret = Merger.Merge(genre, genre2); + Assert.True(ReferenceEquals(genre, ret)); + Assert.Equal(5, ret.ID); + Assert.Equal("test", genre.Name); + Assert.Null(genre.Slug); + } + + [Fact] + [SuppressMessage("ReSharper", "ExpressionIsAlwaysNull")] + public void MergeNullTests() + { + Genre genre = new() + { + ID = 5 + }; + Assert.True(ReferenceEquals(genre, Merger.Merge(genre, null))); + Assert.True(ReferenceEquals(genre, Merger.Merge(null, genre))); + Assert.Null(Merger.Merge(null, null)); + } + + private class TestIOnMerge : IOnMerge + { + public void OnMerge(object other) + { + Exception exception = new(); + exception.Data[0] = other; + throw exception; + } + } + + [Fact] + public void OnMergeTest() + { + TestIOnMerge test = new(); + TestIOnMerge test2 = new(); + Assert.Throws(() => Merger.Merge(test, test2)); + try + { + Merger.Merge(test, test2); + } + catch (Exception ex) + { + Assert.True(ReferenceEquals(test2, ex.Data[0])); + } + } + + private class Test + { + public int ID { get; set; } + + public int[] Numbers { get; set; } + } + + [Fact] + public void GlobalMergeListTest() + { + Test test = new() + { + ID = 5, + Numbers = new [] { 1 } + }; + Test test2 = new() + { + Numbers = new [] { 3 } + }; + Test ret = Merger.Merge(test, test2); + Assert.True(ReferenceEquals(test, ret)); + Assert.Equal(5, ret.ID); + + Assert.Equal(2, ret.Numbers.Length); + Assert.Equal(1, ret.Numbers[0]); + Assert.Equal(3, ret.Numbers[1]); + } + + [Fact] + public void GlobalMergeListDuplicatesTest() + { + Test test = new() + { + ID = 5, + Numbers = new [] { 1 } + }; + Test test2 = new() + { + Numbers = new [] + { + 1, + 3, + 3 + } + }; + Test ret = Merger.Merge(test, test2); + Assert.True(ReferenceEquals(test, ret)); + Assert.Equal(5, ret.ID); + + Assert.Equal(4, ret.Numbers.Length); + Assert.Equal(1, ret.Numbers[0]); + Assert.Equal(1, ret.Numbers[1]); + Assert.Equal(3, ret.Numbers[2]); + Assert.Equal(3, ret.Numbers[3]); + } + + [Fact] + public void GlobalMergeListDuplicatesResourcesTest() + { + Show test = new() + { + ID = 5, + Genres = new [] { new Genre("test") } + }; + Show test2 = new() + { + Genres = new [] + { + new Genre("test"), + new Genre("test2") + } + }; + Show ret = Merger.Merge(test, test2); + Assert.True(ReferenceEquals(test, ret)); + Assert.Equal(5, ret.ID); + + Assert.Equal(2, ret.Genres.Count); + Assert.Equal("test", ret.Genres.ToArray()[0].Slug); + Assert.Equal("test2", ret.Genres.ToArray()[1].Slug); + } + + [Fact] + public void MergeListTest() + { + int[] first = { 1 }; + int[] second = { + 3, + 3 + }; + int[] ret = Merger.MergeLists(first, second); + + Assert.Equal(3, ret.Length); + Assert.Equal(1, ret[0]); + Assert.Equal(3, ret[1]); + Assert.Equal(3, ret[2]); + } + + [Fact] + public void MergeListDuplicateTest() + { + int[] first = { 1 }; + int[] second = { + 1, + 3, + 3 + }; + int[] ret = Merger.MergeLists(first, second); + + Assert.Equal(4, ret.Length); + Assert.Equal(1, ret[0]); + Assert.Equal(1, ret[1]); + Assert.Equal(3, ret[2]); + Assert.Equal(3, ret[3]); + } + + [Fact] + public void MergeListDuplicateCustomEqualityTest() + { + int[] first = { 1 }; + int[] second = { + 3, + 2 + }; + int[] ret = Merger.MergeLists(first, second, (x, y) => x % 2 == y % 2); + + Assert.Equal(2, ret.Length); + Assert.Equal(1, ret[0]); + Assert.Equal(2, ret[1]); + } } } \ No newline at end of file diff --git a/Kyoo.Tests/Utility/UtilityTests.cs b/Kyoo.Tests/Utility/UtilityTests.cs index 15469411..80d233c3 100644 --- a/Kyoo.Tests/Utility/UtilityTests.cs +++ b/Kyoo.Tests/Utility/UtilityTests.cs @@ -1,5 +1,6 @@ using System; using System.Linq.Expressions; +using System.Reflection; using Kyoo.Models; using Xunit; @@ -31,5 +32,47 @@ namespace Kyoo.Tests Assert.Equal("ID", Utility.GetPropertyName(memberCast)); Assert.Throws(() => Utility.GetPropertyName(null)); } + + [Fact] + public void GetMethodTest() + { + MethodInfo method = Utility.GetMethod(typeof(UtilityTests), + BindingFlags.Instance | BindingFlags.Public, + nameof(GetMethodTest), + Array.Empty(), + Array.Empty()); + Assert.Equal(MethodBase.GetCurrentMethod(), method); + } + + [Fact] + public void GetMethodInvalidGenericsTest() + { + Assert.Throws(() => Utility.GetMethod(typeof(UtilityTests), + BindingFlags.Instance | BindingFlags.Public, + nameof(GetMethodTest), + new [] { typeof(Utility) }, + Array.Empty())); + } + + [Fact] + public void GetMethodInvalidParamsTest() + { + Assert.Throws(() => Utility.GetMethod(typeof(UtilityTests), + BindingFlags.Instance | BindingFlags.Public, + nameof(GetMethodTest), + Array.Empty(), + new object[] { this })); + } + + [Fact] + public void GetMethodTest2() + { + MethodInfo method = Utility.GetMethod(typeof(Merger), + BindingFlags.Static | BindingFlags.Public, + nameof(Merger.MergeLists), + new [] { typeof(string) }, + new object[] { "string", "string2", null }); + Assert.Equal(nameof(Merger.MergeLists), method.Name); + } } } \ No newline at end of file diff --git a/Kyoo/Controllers/ProviderComposite.cs b/Kyoo/Controllers/ProviderComposite.cs index 9b7e498b..7d62266e 100644 --- a/Kyoo/Controllers/ProviderComposite.cs +++ b/Kyoo/Controllers/ProviderComposite.cs @@ -1,86 +1,126 @@ using System; using Kyoo.Models; using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; namespace Kyoo.Controllers { - public class ProviderComposite : IMetadataProvider + /// + /// A metadata provider composite that merge results from all available providers. + /// + public class ProviderComposite : IProviderComposite { - private readonly IEnumerable _providers; + /// + /// The list of metadata providers + /// + private readonly ICollection _providers; + /// + /// The list of selected providers. If no provider has been selected, this is null. + /// + private ICollection _selectedProviders; + + /// + /// The logger used to print errors. + /// + private readonly ILogger _logger; - public ProviderComposite(IEnumerable providers) + /// + /// Since this is a composite and not a real provider, no metadata is available. + /// It is not meant to be stored or selected. This class will handle merge based on what is required. + /// + public Provider Provider => null; + + + /// + /// Create a new with a list of available providers. + /// + /// The list of providers to merge. + /// The logger used to print errors. + public ProviderComposite(IEnumerable providers, ILogger logger) { - _providers = providers; + _providers = providers.ToArray(); + _logger = logger; + } + + + /// + public void UseProviders(IEnumerable providers) + { + _selectedProviders = providers.ToArray(); } - public Provider Provider { get; } - public Task Get(T item) where T : class, IResource + /// + /// Return the list of providers that should be used for queries. + /// + /// The list of providers to use, respecting the . + private IEnumerable _GetProviders() { - throw new NotImplementedException(); + return _selectedProviders? + .Select(x => _providers.FirstOrDefault(y => y.Provider.Slug == x.Slug)) + .Where(x => x != null) + ?? _providers; } - public Task> Search(string query) where T : class, IResource + /// + public async Task Get(T item) + where T : class, IResource { - throw new NotImplementedException(); + T ret = null; + + foreach (IMetadataProvider provider in _GetProviders()) + { + try + { + ret = Merger.Merge(ret, await provider.Get(ret ?? item)); + } + catch (NotSupportedException) + { + // Silenced + } + catch (Exception ex) + { + _logger.LogError(ex, "The provider {Provider} could not get a {Type}", + provider.Provider.Name, typeof(T).Name); + } + } + + return Merger.Merge(ret, item); + } + + /// + public async Task> Search(string query) + where T : class, IResource + { + List ret = new(); + + foreach (IMetadataProvider provider in _GetProviders()) + { + try + { + ret.AddRange(await provider.Search(query)); + } + catch (NotSupportedException) + { + // Silenced + } + catch (Exception ex) + { + _logger.LogError(ex, "The provider {Provider} could not search for {Type}", + provider.Provider.Name, typeof(T).Name); + } + } + + return ret; } public Task> GetPeople(Show show) { throw new NotImplementedException(); } - - // private async Task GetMetadata(Func> providerCall, Library library, string what) - // where T : new() - // { - // T ret = new(); - // - // IEnumerable providers = library?.Providers - // .Select(x => _providers.FirstOrDefault(y => y.Provider.Slug == x.Slug)) - // .Where(x => x != null) - // ?? _providers; - // - // foreach (IMetadataProvider provider in providers) - // { - // try - // { - // ret = Merger.Merge(ret, await providerCall(provider)); - // } catch (Exception ex) - // { - // await Console.Error.WriteLineAsync( - // $"The provider {provider.Provider.Name} could not work for {what}. Exception: {ex.Message}"); - // } - // } - // return ret; - // } - // - // private async Task> GetMetadata( - // Func>> providerCall, - // Library library, - // string what) - // { - // List ret = new(); - // - // IEnumerable providers = library?.Providers - // .Select(x => _providers.FirstOrDefault(y => y.Provider.Slug == x.Slug)) - // .Where(x => x != null) - // ?? _providers; - // - // foreach (IMetadataProvider provider in providers) - // { - // try - // { - // ret.AddRange(await providerCall(provider) ?? new List()); - // } catch (Exception ex) - // { - // await Console.Error.WriteLineAsync( - // $"The provider {provider.Provider.Name} coudln't work for {what}. Exception: {ex.Message}"); - // } - // } - // return ret; - // } - // + // public async Task GetCollectionFromName(string name, Library library) // { // Collection collection = await GetMetadata( From 3ba0cffac2d3fb5020970162e1c4411d5d7b2161 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Fri, 16 Jul 2021 00:00:56 +0200 Subject: [PATCH 05/31] Starting to rework the crawler --- Kyoo.Common/Controllers/IFileManager.cs | 4 +- Kyoo.Common/Controllers/IMetadataProvider.cs | 13 +- Kyoo.Common/Controllers/IPlugin.cs | 11 +- Kyoo.Common/Controllers/ITask.cs | 59 ++- Kyoo.Common/Controllers/ITaskManager.cs | 57 ++- Kyoo.Common/Utility/Merger.cs | 2 +- Kyoo.TheTvdb/ProviderTVDB.cs | 20 +- Kyoo/Controllers/FileManager.cs | 9 +- Kyoo/Controllers/ProviderComposite.cs | 119 +---- Kyoo/Controllers/TaskManager.cs | 38 +- Kyoo/Tasks/Crawler.cs | 430 ++++--------------- Kyoo/Tasks/CreateDatabase.cs | 66 --- Kyoo/Tasks/Housekeeping.cs | 80 ++++ Kyoo/Tasks/PluginInitializer.cs | 29 +- Kyoo/Tasks/PluginLoader.cs | 37 -- Kyoo/Tasks/RegisterEpisode.cs | 305 +++++++++++++ Kyoo/Views/LibraryApi.cs | 4 +- Kyoo/Views/TaskApi.cs | 2 +- 18 files changed, 657 insertions(+), 628 deletions(-) delete mode 100644 Kyoo/Tasks/CreateDatabase.cs create mode 100644 Kyoo/Tasks/Housekeeping.cs delete mode 100644 Kyoo/Tasks/PluginLoader.cs create mode 100644 Kyoo/Tasks/RegisterEpisode.cs diff --git a/Kyoo.Common/Controllers/IFileManager.cs b/Kyoo.Common/Controllers/IFileManager.cs index 03f22e79..33d1dd76 100644 --- a/Kyoo.Common/Controllers/IFileManager.cs +++ b/Kyoo.Common/Controllers/IFileManager.cs @@ -54,8 +54,10 @@ namespace Kyoo.Controllers /// List files in a directory. /// /// The path of the directory + /// Should the search be recursive or not. /// A list of files's path. - public Task> ListFiles([NotNull] string path); + public Task> ListFiles([NotNull] string path, + SearchOption options = SearchOption.TopDirectoryOnly); /// /// Check if a file exists at the given path. diff --git a/Kyoo.Common/Controllers/IMetadataProvider.cs b/Kyoo.Common/Controllers/IMetadataProvider.cs index 7f1bac18..33e4cd2b 100644 --- a/Kyoo.Common/Controllers/IMetadataProvider.cs +++ b/Kyoo.Common/Controllers/IMetadataProvider.cs @@ -1,5 +1,4 @@ -using System; -using Kyoo.Models; +using Kyoo.Models; using System.Collections.Generic; using System.Threading.Tasks; using JetBrains.Annotations; @@ -31,10 +30,7 @@ namespace Kyoo.Controllers /// Merging metadata is the job of Kyoo, a complex is given /// to make a precise search and give you every available properties, not to discard properties. /// - /// - /// If this metadata provider does not support . - /// - /// A new containing metadata from your provider + /// A new containing metadata from your provider or null [ItemCanBeNull] Task Get([NotNull] T item) where T : class, IResource; @@ -43,15 +39,10 @@ namespace Kyoo.Controllers /// Search for a specific type of items with a given query. /// /// The search query to use. - /// - /// If this metadata provider does not support . - /// /// The list of items that could be found on this specific provider. [ItemNotNull] Task> Search(string query) where T : class, IResource; - - Task> GetPeople(Show show); } /// diff --git a/Kyoo.Common/Controllers/IPlugin.cs b/Kyoo.Common/Controllers/IPlugin.cs index 83cc70a4..ea072c39 100644 --- a/Kyoo.Common/Controllers/IPlugin.cs +++ b/Kyoo.Common/Controllers/IPlugin.cs @@ -11,8 +11,10 @@ namespace Kyoo.Controllers /// /// A common interface used to discord plugins /// - /// You can inject services in the IPlugin constructor. - /// You should only inject well known services like an ILogger, IConfiguration or IWebHostEnvironment. + /// + /// You can inject services in the IPlugin constructor. + /// You should only inject well known services like an ILogger, IConfiguration or IWebHostEnvironment. + /// [UsedImplicitly(ImplicitUseTargetFlags.WithInheritors)] public interface IPlugin { @@ -84,7 +86,10 @@ namespace Kyoo.Controllers /// An optional configuration step to allow a plugin to change asp net configurations. /// WARNING: This is only called on Kyoo's startup so you must restart the app to apply this changes. /// - /// The Asp.Net application builder. On most case it is not needed but you can use it to add asp net functionalities. + /// + /// The Asp.Net application builder. On most case it is not needed but you can use it to + /// add asp net functionalities. + /// void ConfigureAspNet(IApplicationBuilder app) { // Skipped diff --git a/Kyoo.Common/Controllers/ITask.cs b/Kyoo.Common/Controllers/ITask.cs index 75277dd2..3267837b 100644 --- a/Kyoo.Common/Controllers/ITask.cs +++ b/Kyoo.Common/Controllers/ITask.cs @@ -3,12 +3,13 @@ using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; +using JetBrains.Annotations; using Kyoo.Models.Attributes; namespace Kyoo.Controllers { /// - /// A single task parameter. This struct contains metadata to display and utility functions to get them in the taks. + /// A single task parameter. This struct contains metadata to display and utility functions to get them in the task. /// /// This struct will be used to generate the swagger documentation of the task. public record TaskParameter @@ -60,6 +61,24 @@ namespace Kyoo.Controllers }; } + /// + /// Create a new required task parameter. + /// + /// The name of the parameter + /// The description of the parameter + /// The type of the parameter. + /// A new task parameter. + public static TaskParameter CreateRequired(string name, string description) + { + return new() + { + Name = name, + Description = description, + Type = typeof(T), + IsRequired = true + }; + } + /// /// Create a parameter's value to give to a task. /// @@ -162,27 +181,37 @@ namespace Kyoo.Controllers public int Priority { get; } /// - /// Start this task. + /// true if this task should not be displayed to the user, false otherwise. /// - /// The list of parameters. - /// A token to request the task's cancellation. - /// If this task is not cancelled quickly, it might be killed by the runner. - /// - /// Your task can have any service as a public field and use the , - /// they will be set to an available service from the service container before calling this method. - /// - public Task Run(TaskParameters arguments, CancellationToken cancellationToken); - + public bool IsHidden { get; } + /// /// The list of parameters /// - /// All parameters that this task as. Every one of them will be given to the run function with a value. + /// + /// All parameters that this task as. Every one of them will be given to the run function with a value. + /// public TaskParameters GetParameters(); /// - /// If this task is running, return the percentage of completion of this task or null if no percentage can be given. + /// Start this task. /// - /// The percentage of completion of the task. - public int? Progress(); + /// + /// The list of parameters. + /// + /// + /// The progress reporter. Used to inform the sender the percentage of completion of this task + /// . + /// A token to request the task's cancellation. + /// If this task is not cancelled quickly, it might be killed by the runner. + /// + /// + /// Your task can have any service as a public field and use the , + /// they will be set to an available service from the service container before calling this method. + /// They also will be removed after this method return (or throw) to prevent dangling services. + /// + public Task Run([NotNull] TaskParameters arguments, + [NotNull] IProgress progress, + CancellationToken cancellationToken); } } \ No newline at end of file diff --git a/Kyoo.Common/Controllers/ITaskManager.cs b/Kyoo.Common/Controllers/ITaskManager.cs index 392355d3..ffcb1c75 100644 --- a/Kyoo.Common/Controllers/ITaskManager.cs +++ b/Kyoo.Common/Controllers/ITaskManager.cs @@ -1,5 +1,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Threading; using Kyoo.Models.Exceptions; namespace Kyoo.Controllers @@ -13,11 +15,56 @@ namespace Kyoo.Controllers /// /// Start a new task (or queue it). /// - /// The slug of the task to run - /// A list of arguments to pass to the task. An automatic conversion will be made if arguments to not fit. - /// If the number of arguments is invalid or if an argument can't be converted. - /// The task could not be found. - void StartTask(string taskSlug, Dictionary arguments = null); + /// + /// The slug of the task to run. + /// + /// + /// A progress reporter to know the percentage of completion of the task. + /// + /// + /// A list of arguments to pass to the task. An automatic conversion will be made if arguments to not fit. + /// + /// + /// A custom cancellation token for the task. + /// + /// + /// If the number of arguments is invalid, if an argument can't be converted or if the task finds the argument + /// invalid. + /// + /// + /// The task could not be found. + /// + void StartTask(string taskSlug, + [NotNull] IProgress progress, + Dictionary arguments = null, + CancellationToken? cancellationToken = null); + + /// + /// Start a new task (or queue it). + /// + /// + /// A progress reporter to know the percentage of completion of the task. + /// + /// + /// A list of arguments to pass to the task. An automatic conversion will be made if arguments to not fit. + /// + /// + /// The type of the task to start. + /// + /// + /// A custom cancellation token for the task. + /// + /// + /// If the number of arguments is invalid, if an argument can't be converted or if the task finds the argument + /// invalid. + /// + /// + /// The task could not be found. + /// + void StartTask([NotNull] IProgress progress, + Dictionary arguments = null, + CancellationToken? cancellationToken = null) + where T : ITask, new(); /// /// Get all currently running tasks diff --git a/Kyoo.Common/Utility/Merger.cs b/Kyoo.Common/Utility/Merger.cs index a1f35756..cd860ea3 100644 --- a/Kyoo.Common/Utility/Merger.cs +++ b/Kyoo.Common/Utility/Merger.cs @@ -112,7 +112,7 @@ namespace Kyoo /// Missing fields of first will be completed by fields of this item. If second is null, the function no-op. /// Fields of T will be merged /// - [ContractAnnotation("=> null; first:notnull => notnull; second:notnull => notnull")] + [ContractAnnotation("first:notnull => notnull; second:notnull => notnull", true)] public static T Merge([CanBeNull] T first, [CanBeNull] T second) { if (first == null) diff --git a/Kyoo.TheTvdb/ProviderTVDB.cs b/Kyoo.TheTvdb/ProviderTVDB.cs index 44841326..5497cac1 100644 --- a/Kyoo.TheTvdb/ProviderTVDB.cs +++ b/Kyoo.TheTvdb/ProviderTVDB.cs @@ -56,7 +56,7 @@ namespace Kyoo.TheTvdb { Show show => await _GetShow(show) as T, Episode episode => await _GetEpisode(episode) as T, - _ => throw new NotSupportedException() + _ => null }; } @@ -66,7 +66,11 @@ namespace Kyoo.TheTvdb if (!int.TryParse(show.GetID(Provider.Slug), out int id)) return (await _SearchShow(show.Title)).FirstOrDefault(); TvDbResponse series = await _client.Series.GetAsync(id); - return series.Data.ToShow(Provider); + Show ret = series.Data.ToShow(Provider); + + TvDbResponse people = await _client.Series.GetActorsAsync(id); + ret.People = people.Data.Select(x => x.ToPeopleRole(Provider)).ToArray(); + return ret; } [ItemCanBeNull] @@ -88,7 +92,7 @@ namespace Kyoo.TheTvdb await _Authenticate(); if (typeof(T) == typeof(Show)) return (await _SearchShow(query) as ICollection)!; - throw new NotImplementedException(); + return ArraySegment.Empty; } [ItemNotNull] @@ -97,15 +101,5 @@ namespace Kyoo.TheTvdb TvDbResponse shows = await _client.Search.SearchSeriesByNameAsync(query); return shows.Data.Select(x => x.ToShow(Provider)).ToArray(); } - - /// - public async Task> GetPeople(Show show) - { - if (!int.TryParse(show?.GetID(Provider.Name), out int id)) - return null; - await _Authenticate(); - TvDbResponse people = await _client.Series.GetActorsAsync(id); - return people.Data.Select(x => x.ToPeopleRole(Provider)).ToArray(); - } } } \ No newline at end of file diff --git a/Kyoo/Controllers/FileManager.cs b/Kyoo/Controllers/FileManager.cs index 43b808b8..f6669c32 100644 --- a/Kyoo/Controllers/FileManager.cs +++ b/Kyoo/Controllers/FileManager.cs @@ -70,13 +70,14 @@ namespace Kyoo.Controllers } /// - public Task> ListFiles(string path) + public Task> ListFiles(string path, SearchOption options = SearchOption.TopDirectoryOnly) { if (path == null) throw new ArgumentNullException(nameof(path)); - return Task.FromResult>(Directory.Exists(path) - ? Directory.GetFiles(path) - : Array.Empty()); + string[] ret = Directory.Exists(path) + ? Directory.GetFiles(path, "*", options) + : Array.Empty(); + return Task.FromResult>(ret); } /// diff --git a/Kyoo/Controllers/ProviderComposite.cs b/Kyoo/Controllers/ProviderComposite.cs index 7d62266e..88ad7529 100644 --- a/Kyoo/Controllers/ProviderComposite.cs +++ b/Kyoo/Controllers/ProviderComposite.cs @@ -68,17 +68,13 @@ namespace Kyoo.Controllers public async Task Get(T item) where T : class, IResource { - T ret = null; - + T ret = item; + foreach (IMetadataProvider provider in _GetProviders()) { try { - ret = Merger.Merge(ret, await provider.Get(ret ?? item)); - } - catch (NotSupportedException) - { - // Silenced + ret = Merger.Merge(ret, await provider.Get(ret)); } catch (Exception ex) { @@ -87,7 +83,7 @@ namespace Kyoo.Controllers } } - return Merger.Merge(ret, item); + return ret; } /// @@ -102,10 +98,6 @@ namespace Kyoo.Controllers { ret.AddRange(await provider.Search(query)); } - catch (NotSupportedException) - { - // Silenced - } catch (Exception ex) { _logger.LogError(ex, "The provider {Provider} could not search for {Type}", @@ -115,108 +107,5 @@ namespace Kyoo.Controllers return ret; } - - public Task> GetPeople(Show show) - { - throw new NotImplementedException(); - } - - // public async Task GetCollectionFromName(string name, Library library) - // { - // Collection collection = await GetMetadata( - // provider => provider.GetCollectionFromName(name), - // library, - // $"the collection {name}"); - // collection.Name ??= name; - // collection.Slug ??= Utility.ToSlug(name); - // return collection; - // } - // - // public async Task CompleteShow(Show show, Library library) - // { - // return await GetMetadata(provider => provider.GetShowByID(show), library, $"the show {show.Title}"); - // } - // - // public async Task SearchShow(string showName, bool isMovie, Library library) - // { - // Show show = await GetMetadata(async provider => - // { - // Show searchResult = (await provider.SearchShows(showName, isMovie))?.FirstOrDefault(); - // if (searchResult == null) - // return null; - // return await provider.GetShowByID(searchResult); - // }, library, $"the show {showName}"); - // show.Slug = Utility.ToSlug(showName); - // show.Title ??= showName; - // show.IsMovie = isMovie; - // show.Genres = show.Genres?.GroupBy(x => x.Slug).Select(x => x.First()).ToList(); - // show.People = show.People?.GroupBy(x => x.Slug).Select(x => x.First()).ToList(); - // return show; - // } - // - // public async Task> SearchShows(string showName, bool isMovie, Library library) - // { - // IEnumerable shows = await GetMetadata( - // provider => provider.SearchShows(showName, isMovie), - // library, - // $"the show {showName}"); - // return shows.Select(show => - // { - // show.Slug = Utility.ToSlug(showName); - // show.Title ??= showName; - // show.IsMovie = isMovie; - // return show; - // }); - // } - // - // public async Task GetSeason(Show show, int seasonNumber, Library library) - // { - // Season season = await GetMetadata( - // provider => provider.GetSeason(show, seasonNumber), - // library, - // $"the season {seasonNumber} of {show.Title}"); - // season.Show = show; - // season.ShowID = show.ID; - // season.ShowSlug = show.Slug; - // season.Title ??= $"Season {season.SeasonNumber}"; - // return season; - // } - // - // public async Task GetEpisode(Show show, - // string episodePath, - // int? seasonNumber, - // int? episodeNumber, - // int? absoluteNumber, - // Library library) - // { - // Episode episode = await GetMetadata( - // provider => provider.GetEpisode(show, seasonNumber, episodeNumber, absoluteNumber), - // library, - // "an episode"); - // episode.Show = show; - // episode.ShowID = show.ID; - // episode.ShowSlug = show.Slug; - // episode.Path = episodePath; - // episode.SeasonNumber ??= seasonNumber; - // episode.EpisodeNumber ??= episodeNumber; - // episode.AbsoluteNumber ??= absoluteNumber; - // return episode; - // } - // - // public async Task> GetPeople(Show show, Library library) - // { - // List people = await GetMetadata( - // provider => provider.GetPeople(show), - // library, - // $"a cast member of {show.Title}"); - // return people?.GroupBy(x => x.Slug) - // .Select(x => x.First()) - // .Select(x => - // { - // x.Show = show; - // x.ShowID = show.ID; - // return x; - // }).ToList(); - // } } } diff --git a/Kyoo/Controllers/TaskManager.cs b/Kyoo/Controllers/TaskManager.cs index caa7a01d..c850a516 100644 --- a/Kyoo/Controllers/TaskManager.cs +++ b/Kyoo/Controllers/TaskManager.cs @@ -41,7 +41,7 @@ namespace Kyoo.Controllers /// /// The queue of tasks that should be run as soon as possible. /// - private readonly Queue<(ITask, Dictionary)> _queuedTasks = new(); + private readonly Queue<(ITask, IProgress, Dictionary)> _queuedTasks = new(); /// /// The currently running task. /// @@ -106,11 +106,11 @@ namespace Kyoo.Controllers { if (_queuedTasks.Any()) { - (ITask task, Dictionary arguments) = _queuedTasks.Dequeue(); + (ITask task, IProgress progress, Dictionary args) = _queuedTasks.Dequeue(); _runningTask = task; try { - await RunTask(task, arguments); + await RunTask(task, progress, args); } catch (Exception e) { @@ -129,9 +129,15 @@ namespace Kyoo.Controllers /// Parse parameters, inject a task and run it. /// /// The task to run + /// A progress reporter to know the percentage of completion of the task. /// The arguments to pass to the function - /// There was an invalid argument or a required argument was not found. - private async Task RunTask(ITask task, Dictionary arguments) + /// + /// If the number of arguments is invalid, if an argument can't be converted or if the task finds the argument + /// invalid. + /// + private async Task RunTask(ITask task, + [NotNull] IProgress progress, + Dictionary arguments) { _logger.LogInformation("Task starting: {Task}", task.Name); @@ -160,7 +166,7 @@ namespace Kyoo.Controllers using IServiceScope scope = _provider.CreateScope(); InjectServices(task, x => scope.ServiceProvider.GetRequiredService(x)); - await task.Run(args, _taskToken.Token); + await task.Run(args, progress, _taskToken.Token); InjectServices(task, _ => null); _logger.LogInformation("Task finished: {Task}", task.Name); } @@ -190,7 +196,7 @@ namespace Kyoo.Controllers foreach (string task in tasksToQueue) { _logger.LogDebug("Queuing task scheduled for running: {Task}", task); - StartTask(task, new Dictionary()); + StartTask(task, new Progress(), new Dictionary()); } } @@ -203,21 +209,33 @@ namespace Kyoo.Controllers .Where(x => x.RunOnStartup) .OrderByDescending(x => x.Priority); foreach (ITask task in startupTasks) - _queuedTasks.Enqueue((task, new Dictionary())); + _queuedTasks.Enqueue((task, new Progress(), new Dictionary())); } /// - public void StartTask(string taskSlug, Dictionary arguments = null) + public void StartTask(string taskSlug, + IProgress progress, + Dictionary arguments = null, + CancellationToken? cancellationToken = null) { arguments ??= new Dictionary(); int index = _tasks.FindIndex(x => x.task.Slug == taskSlug); if (index == -1) throw new ItemNotFoundException($"No task found with the slug {taskSlug}"); - _queuedTasks.Enqueue((_tasks[index].task, arguments)); + _queuedTasks.Enqueue((_tasks[index].task, progress, arguments)); _tasks[index] = (_tasks[index].task, GetNextTaskDate(taskSlug)); } + /// + public void StartTask(IProgress progress, + Dictionary arguments = null, + CancellationToken? cancellationToken = null) + where T : ITask, new() + { + StartTask(new T().Slug, progress, arguments, cancellationToken); + } + /// /// Get the next date of the execution of the given task. /// diff --git a/Kyoo/Tasks/Crawler.cs b/Kyoo/Tasks/Crawler.cs index e9d99010..a96f470e 100644 --- a/Kyoo/Tasks/Crawler.cs +++ b/Kyoo/Tasks/Crawler.cs @@ -1,6 +1,5 @@ using System; using Kyoo.Models; -using Microsoft.Extensions.Configuration; using System.Collections.Generic; using System.IO; using System.Linq; @@ -11,26 +10,56 @@ using Kyoo.Controllers; using Kyoo.Models.Attributes; using Kyoo.Models.Exceptions; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; namespace Kyoo.Tasks { + /// + /// A task to add new video files. + /// public class Crawler : ITask { + /// public string Slug => "scan"; + + /// public string Name => "Scan libraries"; - public string Description => "Scan your libraries, load data for new shows and remove shows that don't exist anymore."; - public string HelpMessage => "Reloading all libraries is a long process and may take up to 24 hours if it is the first scan in a while."; + + /// + public string Description => "Scan your libraries and load data for new shows."; + + /// + public string HelpMessage => "Reloading all libraries is a long process and may take up to" + + " 24 hours if it is the first scan in a while."; + + /// public bool RunOnStartup => true; + + /// public int Priority => 0; - - [Injected] public IServiceProvider ServiceProvider { private get; set; } - [Injected] public IThumbnailsManager ThumbnailsManager { private get; set; } - [Injected] public IMetadataProvider MetadataProvider { private get; set; } - [Injected] public ITranscoder Transcoder { private get; set; } - [Injected] public IConfiguration Config { private get; set; } - private int _parallelTasks; + /// + public bool IsHidden => false; + + /// + /// The library manager used to get libraries and providers to use. + /// + [Injected] public ILibraryManager LibraryManager { private get; set; } + /// + /// The file manager used walk inside directories and check they existences. + /// + [Injected] public IFileManager FileManager { private get; set; } + /// + /// A task manager used to create sub tasks for each episode to add to the database. + /// + [Injected] public ITaskManager TaskManager { private get; set; } + /// + /// The logger used to inform the current status to the console. + /// + [Injected] public ILogger Logger { private get; set; } + + /// public TaskParameters GetParameters() { return new() @@ -39,354 +68,83 @@ namespace Kyoo.Tasks }; } - public int? Progress() + /// + public async Task Run(TaskParameters arguments, IProgress progress, CancellationToken cancellationToken) { - // TODO implement this later. - return null; - } - - public async Task Run(TaskParameters parameters, - CancellationToken cancellationToken) - { - string argument = parameters["slug"].As(); - - _parallelTasks = Config.GetValue("parallelTasks"); - if (_parallelTasks <= 0) - _parallelTasks = 30; - - using IServiceScope serviceScope = ServiceProvider.CreateScope(); - ILibraryManager libraryManager = serviceScope.ServiceProvider.GetService(); - - foreach (Show show in await libraryManager!.GetAll()) - if (!Directory.Exists(show.Path)) - await libraryManager.Delete(show); - - ICollection episodes = await libraryManager.GetAll(); - foreach (Episode episode in episodes) - if (!File.Exists(episode.Path)) - await libraryManager.Delete(episode); - - ICollection tracks = await libraryManager.GetAll(); - foreach (Track track in tracks) - if (!File.Exists(track.Path)) - await libraryManager.Delete(track); - + string argument = arguments["slug"].As(); ICollection libraries = argument == null - ? await libraryManager.GetAll() - : new [] { await libraryManager.GetOrDefault(argument)}; - + ? await LibraryManager.GetAll() + : new [] { await LibraryManager.GetOrDefault(argument)}; + if (argument != null && libraries.First() == null) throw new ArgumentException($"No library found with the name {argument}"); - + foreach (Library library in libraries) - await libraryManager.Load(library, x => x.Providers); + await LibraryManager.Load(library, x => x.Providers); + + progress.Report(0); + float percent = 0; + ICollection episodes = await LibraryManager.GetAll(); foreach (Library library in libraries) - await Scan(library, episodes, tracks, cancellationToken); - Console.WriteLine("Scan finished!"); + { + IProgress reporter = new Progress(x => + { + // ReSharper disable once AccessToModifiedClosure + progress.Report(percent + x / libraries.Count); + }); + await Scan(library, episodes, reporter, cancellationToken); + percent += 100f / libraries.Count; + } + + progress.Report(100); } - private async Task Scan(Library library, IEnumerable episodes, IEnumerable tracks, CancellationToken cancellationToken) + private async Task Scan(Library library, + IEnumerable episodes, + IProgress progress, + CancellationToken cancellationToken) { - Console.WriteLine($"Scanning library {library.Name} at {string.Join(", ", library.Paths)}."); + Logger.LogInformation("Scanning library {Library} at {Paths}", library.Name, library.Paths); foreach (string path in library.Paths) { if (cancellationToken.IsCancellationRequested) - continue; - - string[] files; - try - { - files = Directory.GetFiles(path, "*", SearchOption.AllDirectories); - } - catch (DirectoryNotFoundException) - { - await Console.Error.WriteLineAsync($"The library's directory {path} could not be found (library slug: {library.Slug})"); - continue; - } - catch (PathTooLongException) - { - await Console.Error.WriteLineAsync($"The library's directory {path} is too long for this system. (library slug: {library.Slug})"); - continue; - } - catch (ArgumentException) - { - await Console.Error.WriteLineAsync($"The library's directory {path} is invalid. (library slug: {library.Slug})"); - continue; - } - catch (UnauthorizedAccessException ex) - { - await Console.Error.WriteLineAsync($"{ex.Message} (library slug: {library.Slug})"); - continue; - } + return; - List> shows = files - .Where(x => IsVideo(x) && episodes.All(y => y.Path != x)) + ICollection files = await FileManager.ListFiles(path, SearchOption.AllDirectories); + + // We try to group episodes by shows to register one episode of each show first. + // This speeds up the scan process because further episodes of a show are registered when all metadata + // of the show has already been fetched. + List> shows = files + .Where(IsVideo) + .Where(x => episodes.All(y => y.Path != x)) .GroupBy(Path.GetDirectoryName) .ToList(); + string[] paths = shows.Select(x => x.First()) + .Concat(shows.SelectMany(x => x.Skip(1))) + .ToArray(); - // TODO If the library's path end with a /, the regex is broken. - IEnumerable tasks = shows.Select(x => x.First()); - foreach (string[] showTasks in tasks.BatchBy(_parallelTasks)) - await Task.WhenAll(showTasks - .Select(x => RegisterFile(x, x.Substring(path.Length), library, cancellationToken))); - - tasks = shows.SelectMany(x => x.Skip(1)); - foreach (string[] episodeTasks in tasks.BatchBy(_parallelTasks * 2)) - await Task.WhenAll(episodeTasks - .Select(x => RegisterFile(x, x.Substring(path.Length), library, cancellationToken))); + float percent = 0; + IProgress reporter = new Progress(x => + { + // ReSharper disable once AccessToModifiedClosure + progress.Report((percent + x / paths.Length) / library.Paths.Length); + }); - await Task.WhenAll(files.Where(x => IsSubtitle(x) && tracks.All(y => y.Path != x)) - .Select(x => RegisterExternalSubtitle(x, cancellationToken))); - } - } - - private async Task RegisterExternalSubtitle(string path, CancellationToken token) - { - try - { - if (token.IsCancellationRequested || path.Split(Path.DirectorySeparatorChar).Contains("Subtitles")) - return; - using IServiceScope serviceScope = ServiceProvider.CreateScope(); - ILibraryManager libraryManager = serviceScope.ServiceProvider.GetService(); - - string patern = Config.GetValue("subtitleRegex"); - Regex regex = new(patern, RegexOptions.IgnoreCase); - Match match = regex.Match(path); - - if (!match.Success) + foreach (string episodePath in paths) { - await Console.Error.WriteLineAsync($"The subtitle at {path} does not match the subtitle's regex."); - return; + TaskManager.StartTask(reporter, new Dictionary + { + ["path"] = episodePath[path.Length..], + ["library"] = library + }, cancellationToken); + percent += 100f / paths.Length; } - string episodePath = match.Groups["Episode"].Value; - Episode episode = await libraryManager!.Get(x => x.Path.StartsWith(episodePath)); - Track track = new() - { - Type = StreamType.Subtitle, - Language = match.Groups["Language"].Value, - IsDefault = match.Groups["Default"].Value.Length > 0, - IsForced = match.Groups["Forced"].Value.Length > 0, - Codec = SubtitleExtensions[Path.GetExtension(path)], - IsExternal = true, - Path = path, - Episode = episode - }; - - await libraryManager.Create(track); - Console.WriteLine($"Registering subtitle at: {path}."); + // await Task.WhenAll(files.Where(x => IsSubtitle(x) && tracks.All(y => y.Path != x)) + // .Select(x => RegisterExternalSubtitle(x, cancellationToken))); } - catch (ItemNotFoundException) - { - await Console.Error.WriteLineAsync($"No episode found for subtitle at: ${path}."); - } - catch (Exception ex) - { - await Console.Error.WriteLineAsync($"Unknown error while registering subtitle: {ex.Message}"); - } - } - - private async Task RegisterFile(string path, string relativePath, Library library, CancellationToken token) - { - if (token.IsCancellationRequested) - return; - - try - { - using IServiceScope serviceScope = ServiceProvider.CreateScope(); - ILibraryManager libraryManager = serviceScope.ServiceProvider.GetService(); - - string patern = Config.GetValue("regex"); - Regex regex = new(patern, RegexOptions.IgnoreCase); - Match match = regex.Match(relativePath); - - if (!match.Success) - { - await Console.Error.WriteLineAsync($"The episode at {path} does not match the episode's regex."); - return; - } - - string showPath = Path.GetDirectoryName(path); - string collectionName = match.Groups["Collection"].Value; - string showName = match.Groups["Show"].Value; - int? seasonNumber = int.TryParse(match.Groups["Season"].Value, out int tmp) ? tmp : null; - int? episodeNumber = int.TryParse(match.Groups["Episode"].Value, out tmp) ? tmp : null; - int? absoluteNumber = int.TryParse(match.Groups["Absolute"].Value, out tmp) ? tmp : null; - - Collection collection = await GetCollection(libraryManager, collectionName, library); - bool isMovie = seasonNumber == null && episodeNumber == null && absoluteNumber == null; - Show show = await GetShow(libraryManager, showName, showPath, isMovie, library); - if (isMovie) - await libraryManager!.Create(await GetMovie(show, path)); - else - { - Season season = seasonNumber != null - ? await GetSeason(libraryManager, show, seasonNumber.Value, library) - : null; - Episode episode = await GetEpisode(libraryManager, - show, - season, - episodeNumber, - absoluteNumber, - path, - library); - await libraryManager!.Create(episode); - } - - await libraryManager.AddShowLink(show, library, collection); - Console.WriteLine($"Episode at {path} registered."); - } - catch (DuplicatedItemException ex) - { - await Console.Error.WriteLineAsync($"{path}: {ex.Message}"); - } - catch (Exception ex) - { - await Console.Error.WriteLineAsync($"Unknown exception thrown while registering episode at {path}." + - $"\nException: {ex.Message}" + - $"\n{ex.StackTrace}"); - } - } - - private async Task GetCollection(ILibraryManager libraryManager, - string collectionName, - Library library) - { - if (string.IsNullOrEmpty(collectionName)) - return null; - Collection collection = await libraryManager.GetOrDefault(Utility.ToSlug(collectionName)); - if (collection != null) - return collection; - // collection = await MetadataProvider.GetCollectionFromName(collectionName, library); - - try - { - await libraryManager.Create(collection); - return collection; - } - catch (DuplicatedItemException) - { - return await libraryManager.GetOrDefault(collection.Slug); - } - } - - private async Task GetShow(ILibraryManager libraryManager, - string showTitle, - string showPath, - bool isMovie, - Library library) - { - Show old = await libraryManager.GetOrDefault(x => x.Path == showPath); - if (old != null) - { - await libraryManager.Load(old, x => x.ExternalIDs); - return old; - } - - Show show = new();//await MetadataProvider.SearchShow(showTitle, isMovie, library); - show.Path = showPath; - // show.People = await MetadataProvider.GetPeople(show, library); - - try - { - show = await libraryManager.Create(show); - } - catch (DuplicatedItemException) - { - old = await libraryManager.GetOrDefault(show.Slug); - if (old != null && old.Path == showPath) - { - await libraryManager.Load(old, x => x.ExternalIDs); - return old; - } - - if (show.StartAir != null) - { - show.Slug += $"-{show.StartAir.Value.Year}"; - await libraryManager.Create(show); - } - else - throw; - } - await ThumbnailsManager.Validate(show); - return show; - } - - private async Task GetSeason(ILibraryManager libraryManager, - Show show, - int seasonNumber, - Library library) - { - try - { - Season season = await libraryManager.Get(show.Slug, seasonNumber); - season.Show = show; - return season; - } - catch (ItemNotFoundException) - { - Season season = new();//await MetadataProvider.GetSeason(show, seasonNumber, library); - try - { - await libraryManager.Create(season); - await ThumbnailsManager.Validate(season); - } - catch (DuplicatedItemException) - { - season = await libraryManager.Get(show.Slug, seasonNumber); - } - season.Show = show; - return season; - } - } - - private async Task GetEpisode(ILibraryManager libraryManager, - Show show, - Season season, - int? episodeNumber, - int? absoluteNumber, - string episodePath, - Library library) - { - Episode episode = new();/*await MetadataProvider.GetEpisode(show, - episodePath, - season?.SeasonNumber, - episodeNumber, - absoluteNumber, - library);*/ - - if (episode.SeasonNumber != null) - { - season ??= await GetSeason(libraryManager, show, episode.SeasonNumber.Value, library); - episode.Season = season; - episode.SeasonID = season?.ID; - } - await ThumbnailsManager.Validate(episode); - await GetTracks(episode); - return episode; - } - - private async Task GetMovie(Show show, string episodePath) - { - Episode episode = new() - { - Title = show.Title, - Path = episodePath, - Show = show, - ShowID = show.ID, - ShowSlug = show.Slug - }; - episode.Tracks = await GetTracks(episode); - return episode; - } - - private async Task> GetTracks(Episode episode) - { - episode.Tracks = (await Transcoder.ExtractInfos(episode, false)) - .Where(x => x.Type != StreamType.Attachment) - .ToArray(); - return episode.Tracks; } private static readonly string[] VideoExtensions = diff --git a/Kyoo/Tasks/CreateDatabase.cs b/Kyoo/Tasks/CreateDatabase.cs deleted file mode 100644 index 8d54cd87..00000000 --- a/Kyoo/Tasks/CreateDatabase.cs +++ /dev/null @@ -1,66 +0,0 @@ -// using System; -// using System.Collections.Generic; -// using System.Linq; -// using System.Threading; -// using System.Threading.Tasks; -// using IdentityServer4.EntityFramework.DbContexts; -// using IdentityServer4.EntityFramework.Mappers; -// using IdentityServer4.Models; -// using Kyoo.Models; -// using Microsoft.EntityFrameworkCore; -// using Microsoft.Extensions.DependencyInjection; -// -// namespace Kyoo.Tasks -// { -// public class CreateDatabase : ITask -// { -// public string Slug => "create-database"; -// public string Name => "Create the database"; -// public string Description => "Create the database if it does not exit and initialize it with defaults value."; -// public string HelpMessage => null; -// public bool RunOnStartup => true; -// public int Priority => int.MaxValue; -// -// public Task Run(IServiceProvider serviceProvider, CancellationToken cancellationToken, string arguments = null) -// { -// using IServiceScope serviceScope = serviceProvider.CreateScope(); -// DatabaseContext databaseContext = serviceScope.ServiceProvider.GetService(); -// IdentityDatabase identityDatabase = serviceScope.ServiceProvider.GetService(); -// ConfigurationDbContext identityContext = serviceScope.ServiceProvider.GetService(); -// -// databaseContext!.Database.Migrate(); -// identityDatabase!.Database.Migrate(); -// identityContext!.Database.Migrate(); -// -// if (!identityContext.Clients.Any()) -// { -// foreach (Client client in IdentityContext.GetClients()) -// identityContext.Clients.Add(client.ToEntity()); -// identityContext.SaveChanges(); -// } -// if (!identityContext.IdentityResources.Any()) -// { -// foreach (IdentityResource resource in IdentityContext.GetIdentityResources()) -// identityContext.IdentityResources.Add(resource.ToEntity()); -// identityContext.SaveChanges(); -// } -// if (!identityContext.ApiResources.Any()) -// { -// foreach (ApiResource resource in IdentityContext.GetApis()) -// identityContext.ApiResources.Add(resource.ToEntity()); -// identityContext.SaveChanges(); -// } -// return Task.CompletedTask; -// } -// -// public Task> GetPossibleParameters() -// { -// return Task.FromResult>(null); -// } -// -// public int? Progress() -// { -// return null; -// } -// } -// } \ No newline at end of file diff --git a/Kyoo/Tasks/Housekeeping.cs b/Kyoo/Tasks/Housekeeping.cs new file mode 100644 index 00000000..661f0be0 --- /dev/null +++ b/Kyoo/Tasks/Housekeeping.cs @@ -0,0 +1,80 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Kyoo.Controllers; +using Kyoo.Models; +using Kyoo.Models.Attributes; + +namespace Kyoo.Tasks +{ + public class Housekeeping : ITask + { + /// + public string Slug => "housekeeping"; + + /// + public string Name => "Housekeeping"; + + /// + public string Description => "Remove orphaned episode and series."; + + /// + public string HelpMessage => null; + + /// + public bool RunOnStartup => true; + + /// + public int Priority => 0; + + /// + public bool IsHidden => false; + + + /// + /// The library manager used to get libraries or remove deleted episodes + /// + [Injected] public ILibraryManager LibraryManager { private get; set; } + /// + /// The file manager used walk inside directories and check they existences. + /// + [Injected] public IFileManager FileManager { private get; set; } + + + /// + public async Task Run(TaskParameters arguments, IProgress progress, CancellationToken cancellationToken) + { + int count = 0; + int delCount = await LibraryManager.GetCount() + await LibraryManager.GetCount(); + progress.Report(0); + + foreach (Show show in await LibraryManager.GetAll()) + { + progress.Report(count / delCount * 100); + count++; + + if (await FileManager.Exists(show.Path)) + continue; + await LibraryManager.Delete(show); + } + + foreach (Episode episode in await LibraryManager.GetAll()) + { + progress.Report(count / delCount * 100); + count++; + + if (await FileManager.Exists(episode.Path)) + continue; + await LibraryManager.Delete(episode); + } + + progress.Report(100); + } + + /// + public TaskParameters GetParameters() + { + return new(); + } + } +} \ No newline at end of file diff --git a/Kyoo/Tasks/PluginInitializer.cs b/Kyoo/Tasks/PluginInitializer.cs index 55907649..31dd324e 100644 --- a/Kyoo/Tasks/PluginInitializer.cs +++ b/Kyoo/Tasks/PluginInitializer.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Kyoo.Controllers; @@ -28,8 +29,11 @@ namespace Kyoo.Tasks /// public int Priority => int.MaxValue; - - + + /// + public bool IsHidden => true; + + /// /// The plugin manager used to retrieve plugins to initialize them. /// @@ -40,21 +44,28 @@ namespace Kyoo.Tasks [Injected] public IServiceProvider Provider { private get; set; } /// - public Task Run(TaskParameters arguments, CancellationToken cancellationToken) + public Task Run(TaskParameters arguments, IProgress progress, CancellationToken cancellationToken) { - foreach (IPlugin plugin in PluginManager.GetAllPlugins()) + ICollection plugins = PluginManager.GetAllPlugins(); + int count = 0; + progress.Report(0); + + foreach (IPlugin plugin in plugins) + { plugin.Initialize(Provider); + + progress.Report(count / plugins.Count * 100); + count++; + } + + progress.Report(100); return Task.CompletedTask; } + /// public TaskParameters GetParameters() { return new(); } - - public int? Progress() - { - return null; - } } } \ No newline at end of file diff --git a/Kyoo/Tasks/PluginLoader.cs b/Kyoo/Tasks/PluginLoader.cs deleted file mode 100644 index 839e2f1e..00000000 --- a/Kyoo/Tasks/PluginLoader.cs +++ /dev/null @@ -1,37 +0,0 @@ -// using System; -// using System.Collections.Generic; -// using System.Threading; -// using System.Threading.Tasks; -// using Kyoo.Controllers; -// using Kyoo.Models; -// using Microsoft.Extensions.DependencyInjection; -// -// namespace Kyoo.Tasks -// { -// public class PluginLoader : ITask -// { -// public string Slug => "reload-plugin"; -// public string Name => "Reload plugins"; -// public string Description => "Reload all plugins from the plugin folder."; -// public string HelpMessage => null; -// public bool RunOnStartup => true; -// public int Priority => Int32.MaxValue; -// public Task Run(IServiceProvider serviceProvider, CancellationToken cancellationToken, string arguments = null) -// { -// using IServiceScope serviceScope = serviceProvider.CreateScope(); -// IPluginManager pluginManager = serviceScope.ServiceProvider.GetService(); -// pluginManager.ReloadPlugins(); -// return Task.CompletedTask; -// } -// -// public Task> GetPossibleParameters() -// { -// return Task.FromResult>(null); -// } -// -// public int? Progress() -// { -// return null; -// } -// } -// } \ No newline at end of file diff --git a/Kyoo/Tasks/RegisterEpisode.cs b/Kyoo/Tasks/RegisterEpisode.cs new file mode 100644 index 00000000..7b0a4eba --- /dev/null +++ b/Kyoo/Tasks/RegisterEpisode.cs @@ -0,0 +1,305 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Kyoo.Controllers; +using Kyoo.Models; + +namespace Kyoo.Tasks +{ + /// + /// A task to register a new episode + /// + public class RegisterEpisode : ITask + { + /// + public string Slug => "register"; + + /// + public string Name => "Register episode"; + + /// + public string Description => "Register a new episode"; + + /// + public string HelpMessage => null; + + /// + public bool RunOnStartup => false; + + /// + public int Priority => 0; + + /// + public bool IsHidden => false; + + /// + public TaskParameters GetParameters() + { + return new() + { + TaskParameter.CreateRequired("path", "The path of the episode file"), + TaskParameter.CreateRequired("library", "The library in witch the episode is") + }; + } + + /// + public async Task Run(TaskParameters arguments, IProgress progress, CancellationToken cancellationToken) + { + string path = arguments["path"].As(); + Library library = arguments["library"].As(); + + + } + + /* + * private async Task RegisterExternalSubtitle(string path, CancellationToken token) + { + try + { + if (token.IsCancellationRequested || path.Split(Path.DirectorySeparatorChar).Contains("Subtitles")) + return; + using IServiceScope serviceScope = ServiceProvider.CreateScope(); + ILibraryManager libraryManager = serviceScope.ServiceProvider.GetService(); + + string patern = Config.GetValue("subtitleRegex"); + Regex regex = new(patern, RegexOptions.IgnoreCase); + Match match = regex.Match(path); + + if (!match.Success) + { + await Console.Error.WriteLineAsync($"The subtitle at {path} does not match the subtitle's regex."); + return; + } + + string episodePath = match.Groups["Episode"].Value; + Episode episode = await libraryManager!.Get(x => x.Path.StartsWith(episodePath)); + Track track = new() + { + Type = StreamType.Subtitle, + Language = match.Groups["Language"].Value, + IsDefault = match.Groups["Default"].Value.Length > 0, + IsForced = match.Groups["Forced"].Value.Length > 0, + Codec = SubtitleExtensions[Path.GetExtension(path)], + IsExternal = true, + Path = path, + Episode = episode + }; + + await libraryManager.Create(track); + Console.WriteLine($"Registering subtitle at: {path}."); + } + catch (ItemNotFoundException) + { + await Console.Error.WriteLineAsync($"No episode found for subtitle at: ${path}."); + } + catch (Exception ex) + { + await Console.Error.WriteLineAsync($"Unknown error while registering subtitle: {ex.Message}"); + } + } + + private async Task RegisterFile(string path, string relativePath, Library library, CancellationToken token) + { + if (token.IsCancellationRequested) + return; + + try + { + using IServiceScope serviceScope = ServiceProvider.CreateScope(); + ILibraryManager libraryManager = serviceScope.ServiceProvider.GetService(); + + string patern = Config.GetValue("regex"); + Regex regex = new(patern, RegexOptions.IgnoreCase); + Match match = regex.Match(relativePath); + + if (!match.Success) + { + await Console.Error.WriteLineAsync($"The episode at {path} does not match the episode's regex."); + return; + } + + string showPath = Path.GetDirectoryName(path); + string collectionName = match.Groups["Collection"].Value; + string showName = match.Groups["Show"].Value; + int? seasonNumber = int.TryParse(match.Groups["Season"].Value, out int tmp) ? tmp : null; + int? episodeNumber = int.TryParse(match.Groups["Episode"].Value, out tmp) ? tmp : null; + int? absoluteNumber = int.TryParse(match.Groups["Absolute"].Value, out tmp) ? tmp : null; + + Collection collection = await GetCollection(libraryManager, collectionName, library); + bool isMovie = seasonNumber == null && episodeNumber == null && absoluteNumber == null; + Show show = await GetShow(libraryManager, showName, showPath, isMovie, library); + if (isMovie) + await libraryManager!.Create(await GetMovie(show, path)); + else + { + Season season = seasonNumber != null + ? await GetSeason(libraryManager, show, seasonNumber.Value, library) + : null; + Episode episode = await GetEpisode(libraryManager, + show, + season, + episodeNumber, + absoluteNumber, + path, + library); + await libraryManager!.Create(episode); + } + + await libraryManager.AddShowLink(show, library, collection); + Console.WriteLine($"Episode at {path} registered."); + } + catch (DuplicatedItemException ex) + { + await Console.Error.WriteLineAsync($"{path}: {ex.Message}"); + } + catch (Exception ex) + { + await Console.Error.WriteLineAsync($"Unknown exception thrown while registering episode at {path}." + + $"\nException: {ex.Message}" + + $"\n{ex.StackTrace}"); + } + } + + private async Task GetCollection(ILibraryManager libraryManager, + string collectionName, + Library library) + { + if (string.IsNullOrEmpty(collectionName)) + return null; + Collection collection = await libraryManager.GetOrDefault(Utility.ToSlug(collectionName)); + if (collection != null) + return collection; + // collection = await MetadataProvider.GetCollectionFromName(collectionName, library); + + try + { + await libraryManager.Create(collection); + return collection; + } + catch (DuplicatedItemException) + { + return await libraryManager.GetOrDefault(collection.Slug); + } + } + + private async Task GetShow(ILibraryManager libraryManager, + string showTitle, + string showPath, + bool isMovie, + Library library) + { + Show old = await libraryManager.GetOrDefault(x => x.Path == showPath); + if (old != null) + { + await libraryManager.Load(old, x => x.ExternalIDs); + return old; + } + + Show show = new();//await MetadataProvider.SearchShow(showTitle, isMovie, library); + show.Path = showPath; + // show.People = await MetadataProvider.GetPeople(show, library); + + try + { + show = await libraryManager.Create(show); + } + catch (DuplicatedItemException) + { + old = await libraryManager.GetOrDefault(show.Slug); + if (old != null && old.Path == showPath) + { + await libraryManager.Load(old, x => x.ExternalIDs); + return old; + } + + if (show.StartAir != null) + { + show.Slug += $"-{show.StartAir.Value.Year}"; + await libraryManager.Create(show); + } + else + throw; + } + await ThumbnailsManager.Validate(show); + return show; + } + + private async Task GetSeason(ILibraryManager libraryManager, + Show show, + int seasonNumber, + Library library) + { + try + { + Season season = await libraryManager.Get(show.Slug, seasonNumber); + season.Show = show; + return season; + } + catch (ItemNotFoundException) + { + Season season = new();//await MetadataProvider.GetSeason(show, seasonNumber, library); + try + { + await libraryManager.Create(season); + await ThumbnailsManager.Validate(season); + } + catch (DuplicatedItemException) + { + season = await libraryManager.Get(show.Slug, seasonNumber); + } + season.Show = show; + return season; + } + } + + private async Task GetEpisode(ILibraryManager libraryManager, + Show show, + Season season, + int? episodeNumber, + int? absoluteNumber, + string episodePath, + Library library) + { + Episode episode = new(); + //await MetadataProvider.GetEpisode(show, + // episodePath, + // season?.SeasonNumber, + // episodeNumber, + // absoluteNumber, + // library); + + if (episode.SeasonNumber != null) + { + season ??= await GetSeason(libraryManager, show, episode.SeasonNumber.Value, library); + episode.Season = season; + episode.SeasonID = season?.ID; + } + await ThumbnailsManager.Validate(episode); + await GetTracks(episode); + return episode; + } + + private async Task GetMovie(Show show, string episodePath) + { + Episode episode = new() + { + Title = show.Title, + Path = episodePath, + Show = show, + ShowID = show.ID, + ShowSlug = show.Slug + }; + episode.Tracks = await GetTracks(episode); + return episode; + } + + private async Task> GetTracks(Episode episode) + { + episode.Tracks = (await Transcoder.ExtractInfos(episode, false)) + .Where(x => x.Type != StreamType.Attachment) + .ToArray(); + return episode.Tracks; + } + */ + } +} \ No newline at end of file diff --git a/Kyoo/Views/LibraryApi.cs b/Kyoo/Views/LibraryApi.cs index 93cd57d3..83d0e712 100644 --- a/Kyoo/Views/LibraryApi.cs +++ b/Kyoo/Views/LibraryApi.cs @@ -33,7 +33,9 @@ namespace Kyoo.Api { ActionResult result = await base.Create(resource); if (result.Value != null) - _taskManager.StartTask("scan", new Dictionary {{"slug", result.Value.Slug}}); + _taskManager.StartTask("scan", + new Progress(), + new Dictionary {{"slug", result.Value.Slug}}); return result; } diff --git a/Kyoo/Views/TaskApi.cs b/Kyoo/Views/TaskApi.cs index b75033d7..dfad76da 100644 --- a/Kyoo/Views/TaskApi.cs +++ b/Kyoo/Views/TaskApi.cs @@ -34,7 +34,7 @@ namespace Kyoo.Api { try { - _taskManager.StartTask(taskSlug, args); + _taskManager.StartTask(taskSlug, new Progress(), args); return Ok(); } catch (ItemNotFoundException) From 3acc69d602ac211f1ed155d6d07137917941b59e Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Fri, 16 Jul 2021 01:25:01 +0200 Subject: [PATCH 06/31] Starting to implement the register episode task --- Kyoo.Common/Controllers/IIdentifier.cs | 21 +++ Kyoo.Common/Controllers/IMetadataProvider.cs | 19 ++- Kyoo.Common/Utility/Utility.cs | 2 +- Kyoo/Controllers/ProviderComposite.cs | 16 +- Kyoo/Controllers/RegexIdentifier.cs | 79 ++++++++++ Kyoo/Tasks/Crawler.cs | 7 +- Kyoo/Tasks/RegisterEpisode.cs | 154 +++++++++---------- Kyoo/settings.json | 2 +- 8 files changed, 203 insertions(+), 97 deletions(-) create mode 100644 Kyoo.Common/Controllers/IIdentifier.cs create mode 100644 Kyoo/Controllers/RegexIdentifier.cs diff --git a/Kyoo.Common/Controllers/IIdentifier.cs b/Kyoo.Common/Controllers/IIdentifier.cs new file mode 100644 index 00000000..c3b65bf9 --- /dev/null +++ b/Kyoo.Common/Controllers/IIdentifier.cs @@ -0,0 +1,21 @@ +using System.Threading.Tasks; +using Kyoo.Models; + +namespace Kyoo.Controllers +{ + /// + /// An interface to identify episodes, shows and metadata based on the episode file. + /// + public interface IIdentifier + { + /// + /// Identify a path and return the parsed metadata. + /// + /// The path of the episode file to parse. + /// + /// A tuple of models representing parsed metadata. + /// If no metadata could be parsed for a type, null can be returned. + /// + Task<(Collection, Show, Season, Episode)> Identify(string path); + } +} \ No newline at end of file diff --git a/Kyoo.Common/Controllers/IMetadataProvider.cs b/Kyoo.Common/Controllers/IMetadataProvider.cs index 33e4cd2b..ed6ab40d 100644 --- a/Kyoo.Common/Controllers/IMetadataProvider.cs +++ b/Kyoo.Common/Controllers/IMetadataProvider.cs @@ -49,13 +49,28 @@ namespace Kyoo.Controllers /// A special that merge results. /// This interface exists to specify witch provider to use but it can be used like any other metadata provider. /// - public interface IProviderComposite : IMetadataProvider + public abstract class AProviderComposite : IMetadataProvider { + /// + [ItemNotNull] + public abstract Task Get(T item) + where T : class, IResource; + + /// + public abstract Task> Search(string query) + where T : class, IResource; + + /// + /// Since this is a composite and not a real provider, no metadata is available. + /// It is not meant to be stored or selected. This class will handle merge based on what is required. + /// + public Provider Provider => null; + /// /// Select witch providers to use. /// The associated with the given will be used. /// /// The list of providers to use - void UseProviders(IEnumerable providers); + public abstract void UseProviders(IEnumerable providers); } } diff --git a/Kyoo.Common/Utility/Utility.cs b/Kyoo.Common/Utility/Utility.cs index f7c1a94b..6b6dc472 100644 --- a/Kyoo.Common/Utility/Utility.cs +++ b/Kyoo.Common/Utility/Utility.cs @@ -72,7 +72,7 @@ namespace Kyoo /// /// The string to slugify /// The slug version of the given string - public static string ToSlug(string str) + public static string ToSlug([CanBeNull] string str) { if (str == null) return null; diff --git a/Kyoo/Controllers/ProviderComposite.cs b/Kyoo/Controllers/ProviderComposite.cs index 88ad7529..2474016c 100644 --- a/Kyoo/Controllers/ProviderComposite.cs +++ b/Kyoo/Controllers/ProviderComposite.cs @@ -10,7 +10,7 @@ namespace Kyoo.Controllers /// /// A metadata provider composite that merge results from all available providers. /// - public class ProviderComposite : IProviderComposite + public class ProviderComposite : AProviderComposite { /// /// The list of metadata providers @@ -26,12 +26,6 @@ namespace Kyoo.Controllers /// The logger used to print errors. /// private readonly ILogger _logger; - - /// - /// Since this is a composite and not a real provider, no metadata is available. - /// It is not meant to be stored or selected. This class will handle merge based on what is required. - /// - public Provider Provider => null; /// @@ -47,7 +41,7 @@ namespace Kyoo.Controllers /// - public void UseProviders(IEnumerable providers) + public override void UseProviders(IEnumerable providers) { _selectedProviders = providers.ToArray(); } @@ -65,8 +59,7 @@ namespace Kyoo.Controllers } /// - public async Task Get(T item) - where T : class, IResource + public override async Task Get(T item) { T ret = item; @@ -87,8 +80,7 @@ namespace Kyoo.Controllers } /// - public async Task> Search(string query) - where T : class, IResource + public override async Task> Search(string query) { List ret = new(); diff --git a/Kyoo/Controllers/RegexIdentifier.cs b/Kyoo/Controllers/RegexIdentifier.cs new file mode 100644 index 00000000..f46778eb --- /dev/null +++ b/Kyoo/Controllers/RegexIdentifier.cs @@ -0,0 +1,79 @@ +using System; +using System.IO; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using Kyoo.Models; +using Kyoo.Models.Options; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Kyoo.Controllers +{ + /// + /// An identifier that use a regex to extract basics metadata. + /// + public class RegexIdentifier : IIdentifier + { + /// + /// The configuration of kyoo to retrieve the identifier regex. + /// + private readonly IOptions _configuration; + /// + /// A logger to print errors. + /// + private readonly ILogger _logger; + + /// + /// Create a new . + /// + /// The regex patterns to use. + /// The logger to use. + public RegexIdentifier(IOptions configuration, ILogger logger) + { + _configuration = configuration; + _logger = logger; + } + + + /// + public Task<(Collection, Show, Season, Episode)> Identify(string path) + { + string pattern = _configuration.Value.Regex; + Regex regex = new(pattern, RegexOptions.IgnoreCase | RegexOptions.Compiled); + Match match = regex.Match(path); + + if (!match.Success) + { + _logger.LogError("The episode at {Path} does not match the episode's regex", path); + return Task.FromResult<(Collection, Show, Season, Episode)>(default); + } + + (Collection collection, Show show, Season season, Episode episode) ret = new(); + + ret.collection.Name = match.Groups["Collection"].Value; + + ret.show.Title = match.Groups["Show"].Value; + ret.show.Path = Path.GetDirectoryName(path); + + if (match.Groups["StartYear"].Success && int.TryParse(match.Groups["StartYear"].Value, out int tmp)) + ret.show.StartAir = new DateTime(tmp, 1, 1); + + if (match.Groups["Season"].Success && int.TryParse(match.Groups["Season"].Value, out tmp)) + { + ret.season.SeasonNumber = tmp; + ret.episode.SeasonNumber = tmp; + } + + if (match.Groups["Episode"].Success && int.TryParse(match.Groups["Episode"].Value, out tmp)) + ret.episode.EpisodeNumber = tmp; + + if (match.Groups["Absolute"].Success && int.TryParse(match.Groups["Absolute"].Value, out tmp)) + ret.episode.AbsoluteNumber = tmp; + + ret.show.IsMovie = ret.episode.SeasonNumber == null && ret.episode.EpisodeNumber == null + && ret.episode.AbsoluteNumber == null; + + return Task.FromResult(ret); + } + } +} \ No newline at end of file diff --git a/Kyoo/Tasks/Crawler.cs b/Kyoo/Tasks/Crawler.cs index a96f470e..59dd1e35 100644 --- a/Kyoo/Tasks/Crawler.cs +++ b/Kyoo/Tasks/Crawler.cs @@ -95,6 +95,9 @@ namespace Kyoo.Tasks }); await Scan(library, episodes, reporter, cancellationToken); percent += 100f / libraries.Count; + + if (cancellationToken.IsCancellationRequested) + return; } progress.Report(100); @@ -108,11 +111,11 @@ namespace Kyoo.Tasks Logger.LogInformation("Scanning library {Library} at {Paths}", library.Name, library.Paths); foreach (string path in library.Paths) { + ICollection files = await FileManager.ListFiles(path, SearchOption.AllDirectories); + if (cancellationToken.IsCancellationRequested) return; - ICollection files = await FileManager.ListFiles(path, SearchOption.AllDirectories); - // We try to group episodes by shows to register one episode of each show first. // This speeds up the scan process because further episodes of a show are registered when all metadata // of the show has already been fetched. diff --git a/Kyoo/Tasks/RegisterEpisode.cs b/Kyoo/Tasks/RegisterEpisode.cs index 7b0a4eba..74001792 100644 --- a/Kyoo/Tasks/RegisterEpisode.cs +++ b/Kyoo/Tasks/RegisterEpisode.cs @@ -3,6 +3,9 @@ using System.Threading; using System.Threading.Tasks; using Kyoo.Controllers; using Kyoo.Models; +using Kyoo.Models.Attributes; +using Kyoo.Models.Exceptions; +using Microsoft.Extensions.Logging; namespace Kyoo.Tasks { @@ -32,13 +35,30 @@ namespace Kyoo.Tasks /// public bool IsHidden => false; + /// + /// An identifier to extract metadata from paths. + /// + [Injected] public IIdentifier Identifier { private get; set; } + /// + /// The library manager used to register the episode + /// + [Injected] public ILibraryManager LibraryManager { private get; set; } + /// + /// A metadata provider to retrieve the metadata of the new episode (and related items if they do not exist). + /// + [Injected] public AProviderComposite MetadataProvider { private get; set; } + /// + /// The logger used to inform the current status to the console. + /// + [Injected] public ILogger Logger { private get; set; } + /// public TaskParameters GetParameters() { return new() { TaskParameter.CreateRequired("path", "The path of the episode file"), - TaskParameter.CreateRequired("library", "The library in witch the episode is") + TaskParameter.Create("library", "The library in witch the episode is") }; } @@ -48,11 +68,63 @@ namespace Kyoo.Tasks string path = arguments["path"].As(); Library library = arguments["library"].As(); + try + { + (Collection collection, Show show, Season season, Episode episode) = await Identifier.Identify(path); + if (library != null) + MetadataProvider.UseProviders(library.Providers); + + collection = await _RegisterAndFillCollection(collection); + // show = await _RegisterAndFillShow(show); + // if (isMovie) + // await libraryManager!.Create(await GetMovie(show, path)); + // else + // { + // Season season = seasonNumber != null + // ? await GetSeason(libraryManager, show, seasonNumber.Value, library) + // : null; + // Episode episode = await GetEpisode(libraryManager, + // show, + // season, + // episodeNumber, + // absoluteNumber, + // path, + // library); + // await libraryManager!.Create(episode); + // } + // + // await libraryManager.AddShowLink(show, library, collection); + // Console.WriteLine($"Episode at {path} registered."); + } + catch (DuplicatedItemException ex) + { + Logger.LogWarning(ex, "Duplicated found at {Path}", path); + } + catch (Exception ex) + { + Logger.LogCritical(ex, "Unknown exception thrown while registering episode at {Path}", path); + } + } + + private async Task _RegisterAndFillCollection(Collection collection) + { + if (collection == null) + return null; + collection.Slug ??= Utility.ToSlug(collection.Name); + if (string.IsNullOrEmpty(collection.Slug)) + return null; + + Collection existing = await LibraryManager.GetOrDefault(collection.Slug); + if (existing != null) + return existing; + collection = await MetadataProvider.Get(collection); + return await LibraryManager.CreateIfNotExists(collection); } /* - * private async Task RegisterExternalSubtitle(string path, CancellationToken token) + * + private async Task RegisterExternalSubtitle(string path, CancellationToken token) { try { @@ -103,83 +175,7 @@ namespace Kyoo.Tasks if (token.IsCancellationRequested) return; - try - { - using IServiceScope serviceScope = ServiceProvider.CreateScope(); - ILibraryManager libraryManager = serviceScope.ServiceProvider.GetService(); - - string patern = Config.GetValue("regex"); - Regex regex = new(patern, RegexOptions.IgnoreCase); - Match match = regex.Match(relativePath); - - if (!match.Success) - { - await Console.Error.WriteLineAsync($"The episode at {path} does not match the episode's regex."); - return; - } - - string showPath = Path.GetDirectoryName(path); - string collectionName = match.Groups["Collection"].Value; - string showName = match.Groups["Show"].Value; - int? seasonNumber = int.TryParse(match.Groups["Season"].Value, out int tmp) ? tmp : null; - int? episodeNumber = int.TryParse(match.Groups["Episode"].Value, out tmp) ? tmp : null; - int? absoluteNumber = int.TryParse(match.Groups["Absolute"].Value, out tmp) ? tmp : null; - - Collection collection = await GetCollection(libraryManager, collectionName, library); - bool isMovie = seasonNumber == null && episodeNumber == null && absoluteNumber == null; - Show show = await GetShow(libraryManager, showName, showPath, isMovie, library); - if (isMovie) - await libraryManager!.Create(await GetMovie(show, path)); - else - { - Season season = seasonNumber != null - ? await GetSeason(libraryManager, show, seasonNumber.Value, library) - : null; - Episode episode = await GetEpisode(libraryManager, - show, - season, - episodeNumber, - absoluteNumber, - path, - library); - await libraryManager!.Create(episode); - } - - await libraryManager.AddShowLink(show, library, collection); - Console.WriteLine($"Episode at {path} registered."); - } - catch (DuplicatedItemException ex) - { - await Console.Error.WriteLineAsync($"{path}: {ex.Message}"); - } - catch (Exception ex) - { - await Console.Error.WriteLineAsync($"Unknown exception thrown while registering episode at {path}." + - $"\nException: {ex.Message}" + - $"\n{ex.StackTrace}"); - } - } - - private async Task GetCollection(ILibraryManager libraryManager, - string collectionName, - Library library) - { - if (string.IsNullOrEmpty(collectionName)) - return null; - Collection collection = await libraryManager.GetOrDefault(Utility.ToSlug(collectionName)); - if (collection != null) - return collection; - // collection = await MetadataProvider.GetCollectionFromName(collectionName, library); - - try - { - await libraryManager.Create(collection); - return collection; - } - catch (DuplicatedItemException) - { - return await libraryManager.GetOrDefault(collection.Slug); - } + } private async Task GetShow(ILibraryManager libraryManager, diff --git a/Kyoo/settings.json b/Kyoo/settings.json index 2ff0e24b..3233e7f5 100644 --- a/Kyoo/settings.json +++ b/Kyoo/settings.json @@ -44,7 +44,7 @@ }, "media": { - "regex": "(?:\\/(?.*?))?\\/(?.*?)(?: \\(\\d+\\))?\\/\\k(?: \\(\\d+\\))?(?:(?: S(?\\d+)E(?\\d+))| (?\\d+))?.*$", + "regex": "(?:\\/(?.*?))?\\/(?.*?)(?: \\(?\\d+\\))?\\/\\k(?: \\(\\d+\\))?(?:(?: S(?\\d+)E(?\\d+))| (?\\d+))?.*$", "subtitleRegex": "^(?.*)\\.(?\\w{1,3})\\.(?default\\.)?(?forced\\.)?.*$" }, From 5f98484b4ae642c1d999babef38f027750ad7e77 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Fri, 16 Jul 2021 18:58:11 +0200 Subject: [PATCH 07/31] Reworking the thumbnail manager --- Kyoo.Common/Controllers/IFileManager.cs | 14 + Kyoo.Common/Controllers/IThumbnailsManager.cs | 62 +++- Kyoo/Controllers/FileManager.cs | 15 + Kyoo/Controllers/ThumbnailsManager.cs | 277 ++++++++++++------ Kyoo/Tasks/RegisterEpisode.cs | 120 ++++---- Kyoo/Views/EpisodeApi.cs | 10 +- Kyoo/Views/PeopleApi.cs | 4 +- Kyoo/Views/ProviderApi.cs | 4 +- Kyoo/Views/SeasonApi.cs | 12 +- Kyoo/Views/ShowApi.cs | 7 +- 10 files changed, 349 insertions(+), 176 deletions(-) diff --git a/Kyoo.Common/Controllers/IFileManager.cs b/Kyoo.Common/Controllers/IFileManager.cs index 33d1dd76..d39cff6e 100644 --- a/Kyoo.Common/Controllers/IFileManager.cs +++ b/Kyoo.Common/Controllers/IFileManager.cs @@ -50,6 +50,20 @@ namespace Kyoo.Controllers /// A writer to write to the new file. public Stream NewFile([NotNull] string path); + /// + /// Create a new directory at the given path + /// + /// The path of the directory + /// The path of the newly created directory is returned. + public Task CreateDirectory([NotNull] string path); + + /// + /// Combine multiple paths. + /// + /// The paths to combine + /// The combined path. + public string Combine(params string[] paths); + /// /// List files in a directory. /// diff --git a/Kyoo.Common/Controllers/IThumbnailsManager.cs b/Kyoo.Common/Controllers/IThumbnailsManager.cs index ee31498a..465d4c62 100644 --- a/Kyoo.Common/Controllers/IThumbnailsManager.cs +++ b/Kyoo.Common/Controllers/IThumbnailsManager.cs @@ -1,23 +1,59 @@ -using Kyoo.Models; +using System; +using Kyoo.Models; using System.Threading.Tasks; using JetBrains.Annotations; namespace Kyoo.Controllers { + /// + /// Download images and retrieve the path of those images for a resource. + /// public interface IThumbnailsManager { - Task Validate(Show show, bool alwaysDownload = false); - Task Validate(Season season, bool alwaysDownload = false); - Task Validate(Episode episode, bool alwaysDownload = false); - Task Validate(People actors, bool alwaysDownload = false); - Task Validate(Provider actors, bool alwaysDownload = false); + /// + /// Download images of a specified item. + /// If no images is available to download, do nothing and silently return. + /// + /// + /// The item to cache images. + /// + /// + /// true if images should be downloaded even if they already exists locally, false otherwise. + /// + /// The type of the item + /// true if an image has been downloaded, false otherwise. + Task DownloadImages([NotNull] T item, bool alwaysDownload = false) + where T : IResource; + - Task GetShowPoster([NotNull] Show show); - Task GetShowLogo([NotNull] Show show); - Task GetShowBackdrop([NotNull] Show show); - Task GetSeasonPoster([NotNull] Season season); - Task GetEpisodeThumb([NotNull] Episode episode); - Task GetPeoplePoster([NotNull] People people); - Task GetProviderLogo([NotNull] Provider provider); + /// + /// Retrieve the local path of the poster of the given item. + /// + /// The item to retrieve the poster from. + /// The type of the item + /// If the type does not have a poster + /// The path of the poster for the given resource (it might or might not exists). + Task GetPoster([NotNull] T item) + where T : IResource; + + /// + /// Retrieve the local path of the logo of the given item. + /// + /// The item to retrieve the logo from. + /// The type of the item + /// If the type does not have a logo + /// The path of the logo for the given resource (it might or might not exists). + Task GetLogo([NotNull] T item) + where T : IResource; + + /// + /// Retrieve the local path of the thumbnail of the given item. + /// + /// The item to retrieve the thumbnail from. + /// The type of the item + /// If the type does not have a thumbnail + /// The path of the thumbnail for the given resource (it might or might not exists). + Task GetThumbnail([NotNull] T item) + where T : IResource; } } diff --git a/Kyoo/Controllers/FileManager.cs b/Kyoo/Controllers/FileManager.cs index f6669c32..bd7ee4c2 100644 --- a/Kyoo/Controllers/FileManager.cs +++ b/Kyoo/Controllers/FileManager.cs @@ -68,6 +68,21 @@ namespace Kyoo.Controllers throw new ArgumentNullException(nameof(path)); return File.Create(path); } + + /// + public Task CreateDirectory(string path) + { + if (path == null) + throw new ArgumentNullException(nameof(path)); + Directory.CreateDirectory(path); + return Task.FromResult(path); + } + + /// + public string Combine(params string[] paths) + { + return Path.Combine(paths); + } /// public Task> ListFiles(string path, SearchOption options = SearchOption.TopDirectoryOnly) diff --git a/Kyoo/Controllers/ThumbnailsManager.cs b/Kyoo/Controllers/ThumbnailsManager.cs index 1e84cef6..f76123ee 100644 --- a/Kyoo/Controllers/ThumbnailsManager.cs +++ b/Kyoo/Controllers/ThumbnailsManager.cs @@ -1,157 +1,266 @@ using Kyoo.Models; using System; using System.IO; -using System.Net; using System.Threading.Tasks; using JetBrains.Annotations; using Kyoo.Models.Options; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; namespace Kyoo.Controllers { + /// + /// Download images and retrieve the path of those images for a resource. + /// public class ThumbnailsManager : IThumbnailsManager { + /// + /// The file manager used to download the image if the file is distant + /// private readonly IFileManager _files; + /// + /// A logger to report errors. + /// + private readonly ILogger _logger; + /// + /// The options containing the base path of people images and provider logos. + /// private readonly IOptionsMonitor _options; - public ThumbnailsManager(IFileManager files, IOptionsMonitor options) + /// + /// Create a new . + /// + /// The file manager to use. + /// A logger to report errors + /// The options to use. + public ThumbnailsManager(IFileManager files, + ILogger logger, + IOptionsMonitor options) { _files = files; + _logger = logger; _options = options; - Directory.CreateDirectory(_options.CurrentValue.PeoplePath); - Directory.CreateDirectory(_options.CurrentValue.ProviderPath); + + options.OnChange(x => + { + _files.CreateDirectory(x.PeoplePath); + _files.CreateDirectory(x.ProviderPath); + }); } - private static async Task DownloadImage(string url, string localPath, string what) + /// + public Task DownloadImages(T item, bool alwaysDownload = false) + where T : IResource { + if (item == null) + throw new ArgumentNullException(nameof(item)); + return item switch + { + Show show => _Validate(show, alwaysDownload), + Season season => _Validate(season, alwaysDownload), + Episode episode => _Validate(episode, alwaysDownload), + People people => _Validate(people, alwaysDownload), + Provider provider => _Validate(provider, alwaysDownload), + _ => Task.FromResult(false) + }; + } + + /// + /// An helper function to download an image using a . + /// + /// The distant url of the image + /// The local path of the image + /// What is currently downloaded (used for errors) + /// true if an image has been downloaded, false otherwise. + private async Task _DownloadImage(string url, string localPath, string what) + { + if (url == localPath) + return false; + try { - using WebClient client = new(); - await client.DownloadFileTaskAsync(new Uri(url), localPath); + await using Stream reader = _files.GetReader(url); + await using Stream local = _files.NewFile(localPath); + await reader.CopyToAsync(local); + return true; } - catch (WebException exception) + catch (Exception ex) { - await Console.Error.WriteLineAsync($"{what} could not be downloaded. Error: {exception.Message}."); + _logger.LogError(ex, "{What} could not be downloaded", what); + return false; } } - public async Task Validate(Show show, bool alwaysDownload) + /// + /// Download images of a specified show. + /// + /// + /// The item to cache images. + /// + /// + /// true if images should be downloaded even if they already exists locally, false otherwise. + /// + /// true if an image has been downloaded, false otherwise. + private async Task _Validate([NotNull] Show show, bool alwaysDownload) { + bool ret = false; + if (show.Poster != null) { - string posterPath = await GetShowPoster(show); - if (alwaysDownload || !File.Exists(posterPath)) - await DownloadImage(show.Poster, posterPath, $"The poster of {show.Title}"); + string posterPath = await GetPoster(show); + if (alwaysDownload || !await _files.Exists(posterPath)) + ret |= await _DownloadImage(show.Poster, posterPath, $"The poster of {show.Title}"); } if (show.Logo != null) { - string logoPath = await GetShowLogo(show); - if (alwaysDownload || !File.Exists(logoPath)) - await DownloadImage(show.Logo, logoPath, $"The logo of {show.Title}"); + string logoPath = await GetLogo(show); + if (alwaysDownload || !await _files.Exists(logoPath)) + ret |= await _DownloadImage(show.Logo, logoPath, $"The logo of {show.Title}"); } if (show.Backdrop != null) { - string backdropPath = await GetShowBackdrop(show); - if (alwaysDownload || !File.Exists(backdropPath)) - await DownloadImage(show.Backdrop, backdropPath, $"The backdrop of {show.Title}"); + string backdropPath = await GetThumbnail(show); + if (alwaysDownload || !await _files.Exists(backdropPath)) + ret |= await _DownloadImage(show.Backdrop, backdropPath, $"The backdrop of {show.Title}"); } - - foreach (PeopleRole role in show.People) - await Validate(role.People, alwaysDownload); + + return ret; } - public async Task Validate([NotNull] People people, bool alwaysDownload) + /// + /// Download images of a specified person. + /// + /// + /// The item to cache images. + /// + /// + /// true if images should be downloaded even if they already exists locally, false otherwise. + /// + /// true if an image has been downloaded, false otherwise. + private async Task _Validate([NotNull] People people, bool alwaysDownload) { if (people == null) throw new ArgumentNullException(nameof(people)); if (people.Poster == null) - return; - string localPath = await GetPeoplePoster(people); - if (alwaysDownload || !File.Exists(localPath)) - await DownloadImage(people.Poster, localPath, $"The profile picture of {people.Name}"); + return false; + string localPath = await GetPoster(people); + if (alwaysDownload || !await _files.Exists(localPath)) + return await _DownloadImage(people.Poster, localPath, $"The profile picture of {people.Name}"); + return false; } - public async Task Validate(Season season, bool alwaysDownload) + /// + /// Download images of a specified season. + /// + /// + /// The item to cache images. + /// + /// + /// true if images should be downloaded even if they already exists locally, false otherwise. + /// + /// true if an image has been downloaded, false otherwise. + private async Task _Validate([NotNull] Season season, bool alwaysDownload) { - if (season?.Show?.Path == null || season.Poster == null) - return; + if (season.Poster == null) + return false; - string localPath = await GetSeasonPoster(season); - if (alwaysDownload || !File.Exists(localPath)) - await DownloadImage(season.Poster, localPath, $"The poster of {season.Show.Title}'s season {season.SeasonNumber}"); + string localPath = await GetPoster(season); + if (alwaysDownload || !await _files.Exists(localPath)) + return await _DownloadImage(season.Poster, localPath, $"The poster of {season.Slug}"); + return false; } - public async Task Validate(Episode episode, bool alwaysDownload) + /// + /// Download images of a specified episode. + /// + /// + /// The item to cache images. + /// + /// + /// true if images should be downloaded even if they already exists locally, false otherwise. + /// + /// true if an image has been downloaded, false otherwise. + private async Task _Validate([NotNull] Episode episode, bool alwaysDownload) { - if (episode?.Path == null || episode.Thumb == null) - return; + if (episode.Thumb == null) + return false; string localPath = await GetEpisodeThumb(episode); - if (alwaysDownload || !File.Exists(localPath)) - await DownloadImage(episode.Thumb, localPath, $"The thumbnail of {episode.Slug}"); + if (alwaysDownload || !await _files.Exists(localPath)) + return await _DownloadImage(episode.Thumb, localPath, $"The thumbnail of {episode.Slug}"); + return false; } - public async Task Validate(Provider provider, bool alwaysDownload) + /// + /// Download images of a specified provider. + /// + /// + /// The item to cache images. + /// + /// + /// true if images should be downloaded even if they already exists locally, false otherwise. + /// + /// true if an image has been downloaded, false otherwise. + private async Task _Validate([NotNull] Provider provider, bool alwaysDownload) { if (provider.Logo == null) - return; + return false; - string localPath = await GetProviderLogo(provider); - if (alwaysDownload || !File.Exists(localPath)) - await DownloadImage(provider.Logo, localPath, $"The logo of {provider.Slug}"); + string localPath = await GetLogo(provider); + if (alwaysDownload || !await _files.Exists(localPath)) + return await _DownloadImage(provider.Logo, localPath, $"The logo of {provider.Slug}"); + return false; } - public Task GetShowBackdrop(Show show) + /// + public Task GetPoster(T item) + where T : IResource { - if (show?.Path == null) - throw new ArgumentNullException(nameof(show)); - return Task.FromResult(Path.Combine(_files.GetExtraDirectory(show), "backdrop.jpg")); + if (item == null) + throw new ArgumentNullException(nameof(item)); + return Task.FromResult(item switch + { + Show show => _files.Combine(_files.GetExtraDirectory(show), "poster.jpg"), + Season season => _files.Combine(_files.GetExtraDirectory(season), $"season-{season.SeasonNumber}.jpg"), + People people => _files.Combine(_options.CurrentValue.PeoplePath, $"{people.Slug}.jpg"), + _ => throw new NotSupportedException($"The type {typeof(T).Name} does not have a poster.") + }); } - public Task GetShowLogo(Show show) + /// + public Task GetThumbnail(T item) + where T : IResource { - if (show?.Path == null) - throw new ArgumentNullException(nameof(show)); - return Task.FromResult(Path.Combine(_files.GetExtraDirectory(show), "logo.png")); + if (item == null) + throw new ArgumentNullException(nameof(item)); + return item switch + { + Show show => Task.FromResult(_files.Combine(_files.GetExtraDirectory(show), "backdrop.jpg")), + Episode episode => GetEpisodeThumb(episode), + _ => throw new NotSupportedException($"The type {typeof(T).Name} does not have a thumbnail.") + }; } - public Task GetShowPoster(Show show) + private async Task GetEpisodeThumb(Episode episode) { - if (show?.Path == null) - throw new ArgumentNullException(nameof(show)); - return Task.FromResult(Path.Combine(_files.GetExtraDirectory(show), "poster.jpg")); + string dir = _files.Combine(_files.GetExtraDirectory(episode), "Thumbnails"); + await _files.CreateDirectory(dir); + return _files.Combine(dir, $"{Path.GetFileNameWithoutExtension(episode.Path)}.jpg"); } - public Task GetSeasonPoster(Season season) + /// + public Task GetLogo(T item) + where T : IResource { - if (season == null) - throw new ArgumentNullException(nameof(season)); - return Task.FromResult(Path.Combine(_files.GetExtraDirectory(season), $"season-{season.SeasonNumber}.jpg")); - } - - public Task GetEpisodeThumb(Episode episode) - { - string dir = Path.Combine(_files.GetExtraDirectory(episode), "Thumbnails"); - Directory.CreateDirectory(dir); - return Task.FromResult(Path.Combine(dir, $"{Path.GetFileNameWithoutExtension(episode.Path)}.jpg")); - } - - public Task GetPeoplePoster(People people) - { - if (people == null) - throw new ArgumentNullException(nameof(people)); - string peoplePath = _options.CurrentValue.PeoplePath; - string thumbPath = Path.GetFullPath(Path.Combine(peoplePath, $"{people.Slug}.jpg")); - return Task.FromResult(thumbPath.StartsWith(peoplePath) ? thumbPath : null); - } - - public Task GetProviderLogo(Provider provider) - { - if (provider == null) - throw new ArgumentNullException(nameof(provider)); - string providerPath = _options.CurrentValue.ProviderPath; - string thumbPath = Path.GetFullPath(Path.Combine(providerPath, $"{provider.Slug}.{provider.LogoExtension}")); - return Task.FromResult(thumbPath.StartsWith(providerPath) ? thumbPath : null); + if (item == null) + throw new ArgumentNullException(nameof(item)); + return Task.FromResult(item switch + { + Show show => _files.Combine(_files.GetExtraDirectory(show), "logo.png"), + Provider provider => _files.Combine(_options.CurrentValue.ProviderPath, + $"{provider.Slug}.{provider.LogoExtension}"), + _ => throw new NotSupportedException($"The type {typeof(T).Name} does not have a thumbnail.") + }); } } } diff --git a/Kyoo/Tasks/RegisterEpisode.cs b/Kyoo/Tasks/RegisterEpisode.cs index 74001792..51949675 100644 --- a/Kyoo/Tasks/RegisterEpisode.cs +++ b/Kyoo/Tasks/RegisterEpisode.cs @@ -48,6 +48,10 @@ namespace Kyoo.Tasks /// [Injected] public AProviderComposite MetadataProvider { private get; set; } /// + /// The thumbnail manager used to download images. + /// + [Injected] public IThumbnailsManager ThumbnailsManager { private get; set; } + /// /// The logger used to inform the current status to the console. /// [Injected] public ILogger Logger { private get; set; } @@ -74,8 +78,10 @@ namespace Kyoo.Tasks if (library != null) MetadataProvider.UseProviders(library.Providers); - collection = await _RegisterAndFillCollection(collection); - // show = await _RegisterAndFillShow(show); + if (collection != null) + collection.Slug ??= Utility.ToSlug(collection.Name); + collection = await _RegisterAndFill(collection); + show = await _RegisterAndFill(show); // if (isMovie) // await libraryManager!.Create(await GetMovie(show, path)); // else @@ -106,22 +112,62 @@ namespace Kyoo.Tasks } } - private async Task _RegisterAndFillCollection(Collection collection) + private async Task _RegisterAndFill(T item) + where T : class, IResource { - if (collection == null) + if (item == null || string.IsNullOrEmpty(item.Slug)) return null; - - collection.Slug ??= Utility.ToSlug(collection.Name); - if (string.IsNullOrEmpty(collection.Slug)) - return null; - - Collection existing = await LibraryManager.GetOrDefault(collection.Slug); + + T existing = await LibraryManager.GetOrDefault(item.Slug); if (existing != null) return existing; - collection = await MetadataProvider.Get(collection); - return await LibraryManager.CreateIfNotExists(collection); + item = await MetadataProvider.Get(item); + await ThumbnailsManager.DownloadImages(item); + return await LibraryManager.CreateIfNotExists(item); } + // private async Task GetShow(ILibraryManager libraryManager, + // string showTitle, + // string showPath, + // bool isMovie, + // Library library) + // { + // Show old = await libraryManager.GetOrDefault(x => x.Path == showPath); + // if (old != null) + // { + // await libraryManager.Load(old, x => x.ExternalIDs); + // return old; + // } + // + // Show show = await MetadataProvider.SearchShow(showTitle, isMovie, library); + // show.Path = showPath; + // show.People = await MetadataProvider.GetPeople(show, library); + // + // try + // { + // show = await libraryManager.Create(show); + // } + // catch (DuplicatedItemException) + // { + // old = await libraryManager.GetOrDefault(show.Slug); + // if (old != null && old.Path == showPath) + // { + // await libraryManager.Load(old, x => x.ExternalIDs); + // return old; + // } + // + // if (show.StartAir != null) + // { + // show.Slug += $"-{show.StartAir.Value.Year}"; + // await libraryManager.Create(show); + // } + // else + // throw; + // } + // await ThumbnailsManager.Validate(show); + // return show; + // } + // /* * private async Task RegisterExternalSubtitle(string path, CancellationToken token) @@ -170,56 +216,6 @@ namespace Kyoo.Tasks } } - private async Task RegisterFile(string path, string relativePath, Library library, CancellationToken token) - { - if (token.IsCancellationRequested) - return; - - - } - - private async Task GetShow(ILibraryManager libraryManager, - string showTitle, - string showPath, - bool isMovie, - Library library) - { - Show old = await libraryManager.GetOrDefault(x => x.Path == showPath); - if (old != null) - { - await libraryManager.Load(old, x => x.ExternalIDs); - return old; - } - - Show show = new();//await MetadataProvider.SearchShow(showTitle, isMovie, library); - show.Path = showPath; - // show.People = await MetadataProvider.GetPeople(show, library); - - try - { - show = await libraryManager.Create(show); - } - catch (DuplicatedItemException) - { - old = await libraryManager.GetOrDefault(show.Slug); - if (old != null && old.Path == showPath) - { - await libraryManager.Load(old, x => x.ExternalIDs); - return old; - } - - if (show.StartAir != null) - { - show.Slug += $"-{show.StartAir.Value.Year}"; - await libraryManager.Create(show); - } - else - throw; - } - await ThumbnailsManager.Validate(show); - return show; - } - private async Task GetSeason(ILibraryManager libraryManager, Show show, int seasonNumber, diff --git a/Kyoo/Views/EpisodeApi.cs b/Kyoo/Views/EpisodeApi.cs index 490d0b34..a0d92ff8 100644 --- a/Kyoo/Views/EpisodeApi.cs +++ b/Kyoo/Views/EpisodeApi.cs @@ -188,13 +188,14 @@ namespace Kyoo.Api } } - [HttpGet("{id:int}/thumb")] + [HttpGet("{id:int}/thumbnail")] + [HttpGet("{id:int}/backdrop")] public async Task GetThumb(int id) { try { Episode episode = await _libraryManager.Get(id); - return _files.FileResult(await _thumbnails.GetEpisodeThumb(episode)); + return _files.FileResult(await _thumbnails.GetThumbnail(episode)); } catch (ItemNotFoundException) { @@ -202,13 +203,14 @@ namespace Kyoo.Api } } - [HttpGet("{slug}/thumb")] + [HttpGet("{slug}/thumbnail")] + [HttpGet("{slug}/backdrop")] public async Task GetThumb(string slug) { try { Episode episode = await _libraryManager.Get(slug); - return _files.FileResult(await _thumbnails.GetEpisodeThumb(episode)); + return _files.FileResult(await _thumbnails.GetThumbnail(episode)); } catch (ItemNotFoundException) { diff --git a/Kyoo/Views/PeopleApi.cs b/Kyoo/Views/PeopleApi.cs index c421fc16..6a468272 100644 --- a/Kyoo/Views/PeopleApi.cs +++ b/Kyoo/Views/PeopleApi.cs @@ -94,7 +94,7 @@ namespace Kyoo.Api People people = await _libraryManager.GetOrDefault(id); if (people == null) return NotFound(); - return _files.FileResult(await _thumbs.GetPeoplePoster(people)); + return _files.FileResult(await _thumbs.GetPoster(people)); } [HttpGet("{slug}/poster")] @@ -103,7 +103,7 @@ namespace Kyoo.Api People people = await _libraryManager.GetOrDefault(slug); if (people == null) return NotFound(); - return _files.FileResult(await _thumbs.GetPeoplePoster(people)); + return _files.FileResult(await _thumbs.GetPoster(people)); } } } \ No newline at end of file diff --git a/Kyoo/Views/ProviderApi.cs b/Kyoo/Views/ProviderApi.cs index eac22675..ede70dec 100644 --- a/Kyoo/Views/ProviderApi.cs +++ b/Kyoo/Views/ProviderApi.cs @@ -36,7 +36,7 @@ namespace Kyoo.Api Provider provider = await _libraryManager.GetOrDefault(id); if (provider == null) return NotFound(); - return _files.FileResult(await _thumbnails.GetProviderLogo(provider)); + return _files.FileResult(await _thumbnails.GetLogo(provider)); } [HttpGet("{slug}/logo")] @@ -45,7 +45,7 @@ namespace Kyoo.Api Provider provider = await _libraryManager.GetOrDefault(slug); if (provider == null) return NotFound(); - return _files.FileResult(await _thumbnails.GetProviderLogo(provider)); + return _files.FileResult(await _thumbnails.GetLogo(provider)); } } } \ No newline at end of file diff --git a/Kyoo/Views/SeasonApi.cs b/Kyoo/Views/SeasonApi.cs index a32b0b1c..85944c6b 100644 --- a/Kyoo/Views/SeasonApi.cs +++ b/Kyoo/Views/SeasonApi.cs @@ -144,24 +144,24 @@ namespace Kyoo.Api return ret; } - [HttpGet("{id:int}/thumb")] - public async Task GetThumb(int id) + [HttpGet("{id:int}/poster")] + public async Task GetPoster(int id) { Season season = await _libraryManager.GetOrDefault(id); if (season == null) return NotFound(); await _libraryManager.Load(season, x => x.Show); - return _files.FileResult(await _thumbs.GetSeasonPoster(season)); + return _files.FileResult(await _thumbs.GetPoster(season)); } - [HttpGet("{slug}/thumb")] - public async Task GetThumb(string slug) + [HttpGet("{slug}/poster")] + public async Task GetPoster(string slug) { Season season = await _libraryManager.GetOrDefault(slug); if (season == null) return NotFound(); await _libraryManager.Load(season, x => x.Show); - return _files.FileResult(await _thumbs.GetSeasonPoster(season)); + return _files.FileResult(await _thumbs.GetPoster(season)); } } } \ No newline at end of file diff --git a/Kyoo/Views/ShowApi.cs b/Kyoo/Views/ShowApi.cs index 966aad05..605ac28e 100644 --- a/Kyoo/Views/ShowApi.cs +++ b/Kyoo/Views/ShowApi.cs @@ -417,7 +417,7 @@ namespace Kyoo.Api try { Show show = await _libraryManager.Get(slug); - return _files.FileResult(await _thumbs.GetShowPoster(show)); + return _files.FileResult(await _thumbs.GetPoster(show)); } catch (ItemNotFoundException) { @@ -431,7 +431,7 @@ namespace Kyoo.Api try { Show show = await _libraryManager.Get(slug); - return _files.FileResult(await _thumbs.GetShowLogo(show)); + return _files.FileResult(await _thumbs.GetLogo(show)); } catch (ItemNotFoundException) { @@ -440,12 +440,13 @@ namespace Kyoo.Api } [HttpGet("{slug}/backdrop")] + [HttpGet("{slug}/thumbnail")] public async Task GetBackdrop(string slug) { try { Show show = await _libraryManager.Get(slug); - return _files.FileResult(await _thumbs.GetShowBackdrop(show)); + return _files.FileResult(await _thumbs.GetThumbnail(show)); } catch (ItemNotFoundException) { From f71ae0385ce16ddd70db6065a97e8103c56633c2 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sat, 17 Jul 2021 00:51:15 +0200 Subject: [PATCH 08/31] Finishing the register episode task --- Kyoo/Controllers/RegexIdentifier.cs | 16 ++- Kyoo/Startup.cs | 1 - Kyoo/Tasks/RegisterEpisode.cs | 193 +++++++--------------------- 3 files changed, 56 insertions(+), 154 deletions(-) diff --git a/Kyoo/Controllers/RegexIdentifier.cs b/Kyoo/Controllers/RegexIdentifier.cs index f46778eb..e104d70a 100644 --- a/Kyoo/Controllers/RegexIdentifier.cs +++ b/Kyoo/Controllers/RegexIdentifier.cs @@ -38,8 +38,7 @@ namespace Kyoo.Controllers /// public Task<(Collection, Show, Season, Episode)> Identify(string path) { - string pattern = _configuration.Value.Regex; - Regex regex = new(pattern, RegexOptions.IgnoreCase | RegexOptions.Compiled); + Regex regex = new(_configuration.Value.Regex, RegexOptions.IgnoreCase | RegexOptions.Compiled); Match match = regex.Match(path); if (!match.Success) @@ -51,9 +50,12 @@ namespace Kyoo.Controllers (Collection collection, Show show, Season season, Episode episode) ret = new(); ret.collection.Name = match.Groups["Collection"].Value; + ret.collection.Slug = Utility.ToSlug(ret.collection.Name); ret.show.Title = match.Groups["Show"].Value; + ret.show.Slug = Utility.ToSlug(ret.show.Title); ret.show.Path = Path.GetDirectoryName(path); + ret.episode.Path = path; if (match.Groups["StartYear"].Success && int.TryParse(match.Groups["StartYear"].Value, out int tmp)) ret.show.StartAir = new DateTime(tmp, 1, 1); @@ -70,9 +72,13 @@ namespace Kyoo.Controllers if (match.Groups["Absolute"].Success && int.TryParse(match.Groups["Absolute"].Value, out tmp)) ret.episode.AbsoluteNumber = tmp; - ret.show.IsMovie = ret.episode.SeasonNumber == null && ret.episode.EpisodeNumber == null - && ret.episode.AbsoluteNumber == null; - + if (ret.episode.SeasonNumber == null && ret.episode.EpisodeNumber == null + && ret.episode.AbsoluteNumber == null) + { + ret.show.IsMovie = true; + ret.episode.Title = ret.show.Title; + } + return Task.FromResult(ret); } } diff --git a/Kyoo/Startup.cs b/Kyoo/Startup.cs index 5557e872..9d3c19dd 100644 --- a/Kyoo/Startup.cs +++ b/Kyoo/Startup.cs @@ -3,7 +3,6 @@ using System.IO; using Autofac; using Kyoo.Authentication; using Kyoo.Controllers; -using Kyoo.Models; using Kyoo.Models.Options; using Kyoo.Postgresql; using Kyoo.Tasks; diff --git a/Kyoo/Tasks/RegisterEpisode.cs b/Kyoo/Tasks/RegisterEpisode.cs index 51949675..799ac8aa 100644 --- a/Kyoo/Tasks/RegisterEpisode.cs +++ b/Kyoo/Tasks/RegisterEpisode.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using System.Threading; using System.Threading.Tasks; using Kyoo.Controllers; @@ -52,6 +53,10 @@ namespace Kyoo.Tasks /// [Injected] public IThumbnailsManager ThumbnailsManager { private get; set; } /// + /// The transcoder used to extract subtitles and metadata. + /// + [Injected] public ITranscoder Transcoder { private get; set; } + /// /// The logger used to inform the current status to the console. /// [Injected] public ILogger Logger { private get; set; } @@ -62,7 +67,7 @@ namespace Kyoo.Tasks return new() { TaskParameter.CreateRequired("path", "The path of the episode file"), - TaskParameter.Create("library", "The library in witch the episode is") + TaskParameter.CreateRequired("library", "The library in witch the episode is") }; } @@ -74,42 +79,53 @@ namespace Kyoo.Tasks try { - (Collection collection, Show show, Season season, Episode episode) = await Identifier.Identify(path); if (library != null) + { + if (library.Providers == null) + await LibraryManager.Load(library, x => x.Providers); MetadataProvider.UseProviders(library.Providers); + } - if (collection != null) - collection.Slug ??= Utility.ToSlug(collection.Name); + (Collection collection, Show show, Season season, Episode episode) = await Identifier.Identify(path); + collection = await _RegisterAndFill(collection); - show = await _RegisterAndFill(show); - // if (isMovie) - // await libraryManager!.Create(await GetMovie(show, path)); - // else - // { - // Season season = seasonNumber != null - // ? await GetSeason(libraryManager, show, seasonNumber.Value, library) - // : null; - // Episode episode = await GetEpisode(libraryManager, - // show, - // season, - // episodeNumber, - // absoluteNumber, - // path, - // library); - // await libraryManager!.Create(episode); - // } - // - // await libraryManager.AddShowLink(show, library, collection); - // Console.WriteLine($"Episode at {path} registered."); + + Show registeredShow = await _RegisterAndFill(show); + if (registeredShow.Path != show.Path) + { + if (show.StartAir.HasValue) + { + show.Slug += $"-{show.StartAir.Value.Year}"; + show = await LibraryManager.Create(show); + } + else + { + Logger.LogError("Duplicated show found ({Slug}) at {Path1} and {Path2}", + show.Slug, registeredShow.Path, show.Path); + return; + } + } + else + show = registeredShow; + + if (season != null) + season.Show = show; + season = await _RegisterAndFill(season); + + episode = await MetadataProvider.Get(episode); + episode.Season = season; + episode.Tracks = (await Transcoder.ExtractInfos(episode, false)) + .Where(x => x.Type != StreamType.Attachment) + .ToArray(); + await ThumbnailsManager.DownloadImages(episode); + + await LibraryManager.Create(episode); + await LibraryManager.AddShowLink(show, library, collection); } catch (DuplicatedItemException ex) { Logger.LogWarning(ex, "Duplicated found at {Path}", path); } - catch (Exception ex) - { - Logger.LogCritical(ex, "Unknown exception thrown while registering episode at {Path}", path); - } } private async Task _RegisterAndFill(T item) @@ -125,49 +141,7 @@ namespace Kyoo.Tasks await ThumbnailsManager.DownloadImages(item); return await LibraryManager.CreateIfNotExists(item); } - - // private async Task GetShow(ILibraryManager libraryManager, - // string showTitle, - // string showPath, - // bool isMovie, - // Library library) - // { - // Show old = await libraryManager.GetOrDefault(x => x.Path == showPath); - // if (old != null) - // { - // await libraryManager.Load(old, x => x.ExternalIDs); - // return old; - // } - // - // Show show = await MetadataProvider.SearchShow(showTitle, isMovie, library); - // show.Path = showPath; - // show.People = await MetadataProvider.GetPeople(show, library); - // - // try - // { - // show = await libraryManager.Create(show); - // } - // catch (DuplicatedItemException) - // { - // old = await libraryManager.GetOrDefault(show.Slug); - // if (old != null && old.Path == showPath) - // { - // await libraryManager.Load(old, x => x.ExternalIDs); - // return old; - // } - // - // if (show.StartAir != null) - // { - // show.Slug += $"-{show.StartAir.Value.Year}"; - // await libraryManager.Create(show); - // } - // else - // throw; - // } - // await ThumbnailsManager.Validate(show); - // return show; - // } - // + /* * private async Task RegisterExternalSubtitle(string path, CancellationToken token) @@ -215,83 +189,6 @@ namespace Kyoo.Tasks await Console.Error.WriteLineAsync($"Unknown error while registering subtitle: {ex.Message}"); } } - - private async Task GetSeason(ILibraryManager libraryManager, - Show show, - int seasonNumber, - Library library) - { - try - { - Season season = await libraryManager.Get(show.Slug, seasonNumber); - season.Show = show; - return season; - } - catch (ItemNotFoundException) - { - Season season = new();//await MetadataProvider.GetSeason(show, seasonNumber, library); - try - { - await libraryManager.Create(season); - await ThumbnailsManager.Validate(season); - } - catch (DuplicatedItemException) - { - season = await libraryManager.Get(show.Slug, seasonNumber); - } - season.Show = show; - return season; - } - } - - private async Task GetEpisode(ILibraryManager libraryManager, - Show show, - Season season, - int? episodeNumber, - int? absoluteNumber, - string episodePath, - Library library) - { - Episode episode = new(); - //await MetadataProvider.GetEpisode(show, - // episodePath, - // season?.SeasonNumber, - // episodeNumber, - // absoluteNumber, - // library); - - if (episode.SeasonNumber != null) - { - season ??= await GetSeason(libraryManager, show, episode.SeasonNumber.Value, library); - episode.Season = season; - episode.SeasonID = season?.ID; - } - await ThumbnailsManager.Validate(episode); - await GetTracks(episode); - return episode; - } - - private async Task GetMovie(Show show, string episodePath) - { - Episode episode = new() - { - Title = show.Title, - Path = episodePath, - Show = show, - ShowID = show.ID, - ShowSlug = show.Slug - }; - episode.Tracks = await GetTracks(episode); - return episode; - } - - private async Task> GetTracks(Episode episode) - { - episode.Tracks = (await Transcoder.ExtractInfos(episode, false)) - .Where(x => x.Type != StreamType.Attachment) - .ToArray(); - return episode.Tracks; - } */ } } \ No newline at end of file From e66e59cf3206ce0cb37fa04d1e19db39a1921f41 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sat, 17 Jul 2021 01:34:24 +0200 Subject: [PATCH 09/31] Creating a registerr subtitle task --- Kyoo.Common/Controllers/IIdentifier.cs | 12 ++ .../Models/Exceptions/IdentificationFailed.cs | 37 +++++ Kyoo/Controllers/RegexIdentifier.cs | 45 ++++-- Kyoo/Models/FileExtensions.cs | 53 ++++++ Kyoo/Models/LazyDi.cs | 12 -- Kyoo/Tasks/Crawler.cs | 81 ++++------ Kyoo/Tasks/RegisterEpisode.cs | 153 +++++++----------- Kyoo/Tasks/RegisterSubtitle.cs | 85 ++++++++++ 8 files changed, 300 insertions(+), 178 deletions(-) create mode 100644 Kyoo.Common/Models/Exceptions/IdentificationFailed.cs create mode 100644 Kyoo/Models/FileExtensions.cs delete mode 100644 Kyoo/Models/LazyDi.cs create mode 100644 Kyoo/Tasks/RegisterSubtitle.cs diff --git a/Kyoo.Common/Controllers/IIdentifier.cs b/Kyoo.Common/Controllers/IIdentifier.cs index c3b65bf9..b841a63b 100644 --- a/Kyoo.Common/Controllers/IIdentifier.cs +++ b/Kyoo.Common/Controllers/IIdentifier.cs @@ -1,5 +1,6 @@ using System.Threading.Tasks; using Kyoo.Models; +using Kyoo.Models.Exceptions; namespace Kyoo.Controllers { @@ -12,10 +13,21 @@ namespace Kyoo.Controllers /// Identify a path and return the parsed metadata. /// /// The path of the episode file to parse. + /// The identifier could not work for the given path. /// /// A tuple of models representing parsed metadata. /// If no metadata could be parsed for a type, null can be returned. /// Task<(Collection, Show, Season, Episode)> Identify(string path); + + /// + /// Identify an external subtitle or track file from it's path and return the parsed metadata. + /// + /// The path of the external track file to parse. + /// The identifier could not work for the given path. + /// + /// The metadata of the track identified. + /// + Task IdentifyTrack(string path); } } \ No newline at end of file diff --git a/Kyoo.Common/Models/Exceptions/IdentificationFailed.cs b/Kyoo.Common/Models/Exceptions/IdentificationFailed.cs new file mode 100644 index 00000000..a8838fea --- /dev/null +++ b/Kyoo.Common/Models/Exceptions/IdentificationFailed.cs @@ -0,0 +1,37 @@ +using System; +using System.Runtime.Serialization; +using Kyoo.Controllers; + +namespace Kyoo.Models.Exceptions +{ + /// + /// An exception raised when an failed. + /// + [Serializable] + public class IdentificationFailed : Exception + { + /// + /// Create a new with a default message. + /// + public IdentificationFailed() + : base("An identification failed.") + {} + + /// + /// Create a new with a custom message. + /// + /// The message to use. + public IdentificationFailed(string message) + : base(message) + {} + + /// + /// The serialization constructor + /// + /// Serialization infos + /// The serialization context + protected IdentificationFailed(SerializationInfo info, StreamingContext context) + : base(info, context) + { } + } +} \ No newline at end of file diff --git a/Kyoo/Controllers/RegexIdentifier.cs b/Kyoo/Controllers/RegexIdentifier.cs index e104d70a..d99f8255 100644 --- a/Kyoo/Controllers/RegexIdentifier.cs +++ b/Kyoo/Controllers/RegexIdentifier.cs @@ -3,8 +3,9 @@ using System.IO; using System.Text.RegularExpressions; using System.Threading.Tasks; using Kyoo.Models; +using Kyoo.Models.Exceptions; using Kyoo.Models.Options; -using Microsoft.Extensions.Logging; +using Kyoo.Models.Watch; using Microsoft.Extensions.Options; namespace Kyoo.Controllers @@ -18,23 +19,16 @@ namespace Kyoo.Controllers /// The configuration of kyoo to retrieve the identifier regex. /// private readonly IOptions _configuration; - /// - /// A logger to print errors. - /// - private readonly ILogger _logger; /// /// Create a new . /// /// The regex patterns to use. - /// The logger to use. - public RegexIdentifier(IOptions configuration, ILogger logger) + public RegexIdentifier(IOptions configuration) { _configuration = configuration; - _logger = logger; } - - + /// public Task<(Collection, Show, Season, Episode)> Identify(string path) { @@ -42,10 +36,7 @@ namespace Kyoo.Controllers Match match = regex.Match(path); if (!match.Success) - { - _logger.LogError("The episode at {Path} does not match the episode's regex", path); - return Task.FromResult<(Collection, Show, Season, Episode)>(default); - } + throw new IdentificationFailed($"The episode at {path} does not match the episode's regex."); (Collection collection, Show show, Season season, Episode episode) ret = new(); @@ -81,5 +72,31 @@ namespace Kyoo.Controllers return Task.FromResult(ret); } + + /// + public Task IdentifyTrack(string path) + { + Regex regex = new(_configuration.Value.SubtitleRegex, RegexOptions.IgnoreCase | RegexOptions.Compiled); + Match match = regex.Match(path); + + if (!match.Success) + throw new IdentificationFailed($"The subtitle at {path} does not match the subtitle's regex."); + + string episodePath = match.Groups["Episode"].Value; + return Task.FromResult(new Track + { + Type = StreamType.Subtitle, + Language = match.Groups["Language"].Value, + IsDefault = match.Groups["Default"].Value.Length > 0, + IsForced = match.Groups["Forced"].Value.Length > 0, + Codec = FileExtensions.SubtitleExtensions[Path.GetExtension(path)], + IsExternal = true, + Path = path, + Episode = new Episode + { + Path = episodePath + } + }); + } } } \ No newline at end of file diff --git a/Kyoo/Models/FileExtensions.cs b/Kyoo/Models/FileExtensions.cs new file mode 100644 index 00000000..f8058a67 --- /dev/null +++ b/Kyoo/Models/FileExtensions.cs @@ -0,0 +1,53 @@ +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace Kyoo.Models.Watch +{ + public static class FileExtensions + { + public static readonly string[] VideoExtensions = + { + ".webm", + ".mkv", + ".flv", + ".vob", + ".ogg", + ".ogv", + ".avi", + ".mts", + ".m2ts", + ".ts", + ".mov", + ".qt", + ".asf", + ".mp4", + ".m4p", + ".m4v", + ".mpg", + ".mp2", + ".mpeg", + ".mpe", + ".mpv", + ".m2v", + ".3gp", + ".3g2" + }; + + public static bool IsVideo(string filePath) + { + return VideoExtensions.Contains(Path.GetExtension(filePath)); + } + + public static readonly Dictionary SubtitleExtensions = new() + { + {".ass", "ass"}, + {".str", "subrip"} + }; + + public static bool IsSubtitle(string filePath) + { + return SubtitleExtensions.ContainsKey(Path.GetExtension(filePath)); + } + } +} \ No newline at end of file diff --git a/Kyoo/Models/LazyDi.cs b/Kyoo/Models/LazyDi.cs deleted file mode 100644 index 477e1ec4..00000000 --- a/Kyoo/Models/LazyDi.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System; -using Microsoft.Extensions.DependencyInjection; - -namespace Kyoo.Models -{ - public class LazyDi : Lazy - { - public LazyDi(IServiceProvider provider) - : base(provider.GetRequiredService) - { } - } -} \ No newline at end of file diff --git a/Kyoo/Tasks/Crawler.cs b/Kyoo/Tasks/Crawler.cs index 59dd1e35..6f4cf9eb 100644 --- a/Kyoo/Tasks/Crawler.cs +++ b/Kyoo/Tasks/Crawler.cs @@ -3,13 +3,11 @@ using Kyoo.Models; using System.Collections.Generic; using System.IO; using System.Linq; -using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using Kyoo.Controllers; using Kyoo.Models.Attributes; -using Kyoo.Models.Exceptions; -using Microsoft.Extensions.DependencyInjection; +using Kyoo.Models.Watch; using Microsoft.Extensions.Logging; namespace Kyoo.Tasks @@ -86,6 +84,7 @@ namespace Kyoo.Tasks float percent = 0; ICollection episodes = await LibraryManager.GetAll(); + ICollection tracks = await LibraryManager.GetAll(); foreach (Library library in libraries) { IProgress reporter = new Progress(x => @@ -93,7 +92,7 @@ namespace Kyoo.Tasks // ReSharper disable once AccessToModifiedClosure progress.Report(percent + x / libraries.Count); }); - await Scan(library, episodes, reporter, cancellationToken); + await Scan(library, episodes, tracks, reporter, cancellationToken); percent += 100f / libraries.Count; if (cancellationToken.IsCancellationRequested) @@ -105,6 +104,7 @@ namespace Kyoo.Tasks private async Task Scan(Library library, IEnumerable episodes, + IEnumerable tracks, IProgress progress, CancellationToken cancellationToken) { @@ -120,19 +120,20 @@ namespace Kyoo.Tasks // This speeds up the scan process because further episodes of a show are registered when all metadata // of the show has already been fetched. List> shows = files - .Where(IsVideo) + .Where(FileExtensions.IsVideo) .Where(x => episodes.All(y => y.Path != x)) .GroupBy(Path.GetDirectoryName) .ToList(); + + string[] paths = shows.Select(x => x.First()) .Concat(shows.SelectMany(x => x.Skip(1))) .ToArray(); - float percent = 0; IProgress reporter = new Progress(x => { // ReSharper disable once AccessToModifiedClosure - progress.Report((percent + x / paths.Length) / library.Paths.Length); + progress.Report((percent + x / paths.Length - 10) / library.Paths.Length); }); foreach (string episodePath in paths) @@ -145,53 +146,27 @@ namespace Kyoo.Tasks percent += 100f / paths.Length; } - // await Task.WhenAll(files.Where(x => IsSubtitle(x) && tracks.All(y => y.Path != x)) - // .Select(x => RegisterExternalSubtitle(x, cancellationToken))); + + string[] subtitles = files + .Where(FileExtensions.IsSubtitle) + .Where(x => tracks.All(y => y.Path != x)) + .ToArray(); + percent = 0; + reporter = new Progress(x => + { + // ReSharper disable once AccessToModifiedClosure + progress.Report((90 + (percent + x / subtitles.Length)) / library.Paths.Length); + }); + + foreach (string trackPath in subtitles) + { + TaskManager.StartTask(reporter, new Dictionary + { + ["path"] = trackPath + }, cancellationToken); + percent += 100f / subtitles.Length; + } } } - - private static readonly string[] VideoExtensions = - { - ".webm", - ".mkv", - ".flv", - ".vob", - ".ogg", - ".ogv", - ".avi", - ".mts", - ".m2ts", - ".ts", - ".mov", - ".qt", - ".asf", - ".mp4", - ".m4p", - ".m4v", - ".mpg", - ".mp2", - ".mpeg", - ".mpe", - ".mpv", - ".m2v", - ".3gp", - ".3g2" - }; - - private static bool IsVideo(string filePath) - { - return VideoExtensions.Contains(Path.GetExtension(filePath)); - } - - private static readonly Dictionary SubtitleExtensions = new() - { - {".ass", "ass"}, - {".str", "subrip"} - }; - - private static bool IsSubtitle(string filePath) - { - return SubtitleExtensions.ContainsKey(Path.GetExtension(filePath)); - } } } diff --git a/Kyoo/Tasks/RegisterEpisode.cs b/Kyoo/Tasks/RegisterEpisode.cs index 799ac8aa..6ef512e6 100644 --- a/Kyoo/Tasks/RegisterEpisode.cs +++ b/Kyoo/Tasks/RegisterEpisode.cs @@ -6,7 +6,6 @@ using Kyoo.Controllers; using Kyoo.Models; using Kyoo.Models.Attributes; using Kyoo.Models.Exceptions; -using Microsoft.Extensions.Logging; namespace Kyoo.Tasks { @@ -56,11 +55,7 @@ namespace Kyoo.Tasks /// The transcoder used to extract subtitles and metadata. /// [Injected] public ITranscoder Transcoder { private get; set; } - /// - /// The logger used to inform the current status to the console. - /// - [Injected] public ILogger Logger { private get; set; } - + /// public TaskParameters GetParameters() { @@ -76,58 +71,67 @@ namespace Kyoo.Tasks { string path = arguments["path"].As(); Library library = arguments["library"].As(); + progress.Report(0); - try + if (library.Providers == null) + await LibraryManager.Load(library, x => x.Providers); + MetadataProvider.UseProviders(library.Providers); + (Collection collection, Show show, Season season, Episode episode) = await Identifier.Identify(path); + progress.Report(15); + + collection = await _RegisterAndFill(collection); + progress.Report(20); + + Show registeredShow = await _RegisterAndFill(show); + if (registeredShow.Path != show.Path) { - if (library != null) + if (show.StartAir.HasValue) { - if (library.Providers == null) - await LibraryManager.Load(library, x => x.Providers); - MetadataProvider.UseProviders(library.Providers); - } - - (Collection collection, Show show, Season season, Episode episode) = await Identifier.Identify(path); - - collection = await _RegisterAndFill(collection); - - Show registeredShow = await _RegisterAndFill(show); - if (registeredShow.Path != show.Path) - { - if (show.StartAir.HasValue) - { - show.Slug += $"-{show.StartAir.Value.Year}"; - show = await LibraryManager.Create(show); - } - else - { - Logger.LogError("Duplicated show found ({Slug}) at {Path1} and {Path2}", - show.Slug, registeredShow.Path, show.Path); - return; - } + show.Slug += $"-{show.StartAir.Value.Year}"; + show = await LibraryManager.Create(show); } else - show = registeredShow; - - if (season != null) - season.Show = show; - season = await _RegisterAndFill(season); - - episode = await MetadataProvider.Get(episode); - episode.Season = season; - episode.Tracks = (await Transcoder.ExtractInfos(episode, false)) - .Where(x => x.Type != StreamType.Attachment) - .ToArray(); - await ThumbnailsManager.DownloadImages(episode); - - await LibraryManager.Create(episode); - await LibraryManager.AddShowLink(show, library, collection); - } - catch (DuplicatedItemException ex) - { - Logger.LogWarning(ex, "Duplicated found at {Path}", path); + { + throw new DuplicatedItemException($"Duplicated show found ({show.Slug}) " + + $"at {registeredShow.Path} and {show.Path}"); + } } + else + show = registeredShow; + // If they are not already loaded, load external ids to allow metadata providers to use them. + if (show.ExternalIDs == null) + await LibraryManager.Load(show, x => x.ExternalIDs); + progress.Report(50); + + if (season != null) + season.Show = show; + season = await _RegisterAndFill(season); + progress.Report(60); + + episode = await MetadataProvider.Get(episode); + progress.Report(70); + episode.Show = show; + episode.Season = season; + episode.Tracks = (await Transcoder.ExtractInfos(episode, false)) + .Where(x => x.Type != StreamType.Attachment) + .ToArray(); + await ThumbnailsManager.DownloadImages(episode); + progress.Report(90); + + await LibraryManager.Create(episode); + progress.Report(95); + await LibraryManager.AddShowLink(show, library, collection); + progress.Report(100); } + /// + /// Retrieve the equivalent item if it already exists in the database, + /// if it does not, fill metadata using the metadata provider, download images and register the item to the + /// database. + /// + /// The item to retrieve or fill and register + /// The type of the item + /// The existing or filled item. private async Task _RegisterAndFill(T item) where T : class, IResource { @@ -141,54 +145,5 @@ namespace Kyoo.Tasks await ThumbnailsManager.DownloadImages(item); return await LibraryManager.CreateIfNotExists(item); } - - /* - * - private async Task RegisterExternalSubtitle(string path, CancellationToken token) - { - try - { - if (token.IsCancellationRequested || path.Split(Path.DirectorySeparatorChar).Contains("Subtitles")) - return; - using IServiceScope serviceScope = ServiceProvider.CreateScope(); - ILibraryManager libraryManager = serviceScope.ServiceProvider.GetService(); - - string patern = Config.GetValue("subtitleRegex"); - Regex regex = new(patern, RegexOptions.IgnoreCase); - Match match = regex.Match(path); - - if (!match.Success) - { - await Console.Error.WriteLineAsync($"The subtitle at {path} does not match the subtitle's regex."); - return; - } - - string episodePath = match.Groups["Episode"].Value; - Episode episode = await libraryManager!.Get(x => x.Path.StartsWith(episodePath)); - Track track = new() - { - Type = StreamType.Subtitle, - Language = match.Groups["Language"].Value, - IsDefault = match.Groups["Default"].Value.Length > 0, - IsForced = match.Groups["Forced"].Value.Length > 0, - Codec = SubtitleExtensions[Path.GetExtension(path)], - IsExternal = true, - Path = path, - Episode = episode - }; - - await libraryManager.Create(track); - Console.WriteLine($"Registering subtitle at: {path}."); - } - catch (ItemNotFoundException) - { - await Console.Error.WriteLineAsync($"No episode found for subtitle at: ${path}."); - } - catch (Exception ex) - { - await Console.Error.WriteLineAsync($"Unknown error while registering subtitle: {ex.Message}"); - } - } - */ } } \ No newline at end of file diff --git a/Kyoo/Tasks/RegisterSubtitle.cs b/Kyoo/Tasks/RegisterSubtitle.cs new file mode 100644 index 00000000..524977ed --- /dev/null +++ b/Kyoo/Tasks/RegisterSubtitle.cs @@ -0,0 +1,85 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Kyoo.Controllers; +using Kyoo.Models; +using Kyoo.Models.Attributes; +using Kyoo.Models.Exceptions; + +namespace Kyoo.Tasks +{ + /// + /// A task to register a new episode + /// + public class RegisterSubtitle : ITask + { + /// + public string Slug => "register-sub"; + + /// + public string Name => "Register subtitle"; + + /// + public string Description => "Register a new subtitle"; + + /// + public string HelpMessage => null; + + /// + public bool RunOnStartup => false; + + /// + public int Priority => 0; + + /// + public bool IsHidden => false; + + /// + /// An identifier to extract metadata from paths. + /// + [Injected] public IIdentifier Identifier { private get; set; } + /// + /// The library manager used to register the episode + /// + [Injected] public ILibraryManager LibraryManager { private get; set; } + + /// + public TaskParameters GetParameters() + { + return new() + { + TaskParameter.CreateRequired("path", "The path of the episode file"), + }; + } + + /// + public async Task Run(TaskParameters arguments, IProgress progress, CancellationToken cancellationToken) + { + string path = arguments["path"].As(); + + progress.Report(0); + Track track = await Identifier.IdentifyTrack(path); + progress.Report(25); + + if (track.Episode == null) + throw new IdentificationFailed($"No episode identified for the track at {path}"); + if (track.Episode.ID == 0) + { + if (track.Episode.Slug != null) + track.Episode = await LibraryManager.Get(track.Episode.Slug); + else if (track.Episode.Path != null) + { + track.Episode = await LibraryManager.GetOrDefault(x => x.Path == track.Episode.Path); + if (track.Episode == null) + throw new ItemNotFoundException($"No episode found for subtitle at: ${path}."); + } + else + throw new IdentificationFailed($"No episode identified for the track at {path}"); + } + + progress.Report(50); + await LibraryManager.Create(track); + progress.Report(100); + } + } +} \ No newline at end of file From 84458d341337d641dfa1a069cb640e6496940c9f Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sat, 17 Jul 2021 02:32:50 +0200 Subject: [PATCH 10/31] Cleaning uo --- Kyoo.TheTvdb/Kyoo.TheTvdb.csproj | 1 + Kyoo.TheTvdb/{PluginTVDB.cs => PluginTvdb.cs} | 2 ++ .../{ProviderTVDB.cs => ProviderTvdb.cs} | 19 +++++++++++++++---- Kyoo.TheTvdb/TvdbOption.cs | 18 ++++++++++++++++++ Kyoo/Controllers/TaskManager.cs | 2 +- Kyoo/CoreModule.cs | 13 ++++++++++--- Kyoo/Models/FileExtensions.cs | 19 +++++++++++++++++++ 7 files changed, 66 insertions(+), 8 deletions(-) rename Kyoo.TheTvdb/{PluginTVDB.cs => PluginTvdb.cs} (92%) rename Kyoo.TheTvdb/{ProviderTVDB.cs => ProviderTvdb.cs} (81%) create mode 100644 Kyoo.TheTvdb/TvdbOption.cs diff --git a/Kyoo.TheTvdb/Kyoo.TheTvdb.csproj b/Kyoo.TheTvdb/Kyoo.TheTvdb.csproj index 1b6aee8f..8877cb95 100644 --- a/Kyoo.TheTvdb/Kyoo.TheTvdb.csproj +++ b/Kyoo.TheTvdb/Kyoo.TheTvdb.csproj @@ -20,6 +20,7 @@ + diff --git a/Kyoo.TheTvdb/PluginTVDB.cs b/Kyoo.TheTvdb/PluginTvdb.cs similarity index 92% rename from Kyoo.TheTvdb/PluginTVDB.cs rename to Kyoo.TheTvdb/PluginTvdb.cs index ea4cab62..62fc0885 100644 --- a/Kyoo.TheTvdb/PluginTVDB.cs +++ b/Kyoo.TheTvdb/PluginTvdb.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using Autofac; using Kyoo.Controllers; +using TvDbSharper; namespace Kyoo.TheTvdb { @@ -35,6 +36,7 @@ namespace Kyoo.TheTvdb /// public void Configure(ContainerBuilder builder) { + builder.RegisterType().As(); builder.RegisterProvider(); } } diff --git a/Kyoo.TheTvdb/ProviderTVDB.cs b/Kyoo.TheTvdb/ProviderTvdb.cs similarity index 81% rename from Kyoo.TheTvdb/ProviderTVDB.cs rename to Kyoo.TheTvdb/ProviderTvdb.cs index 5497cac1..c6d57fd0 100644 --- a/Kyoo.TheTvdb/ProviderTVDB.cs +++ b/Kyoo.TheTvdb/ProviderTvdb.cs @@ -3,8 +3,10 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using JetBrains.Annotations; +using Kyoo.Authentication.Models; using Kyoo.Controllers; using Kyoo.Models; +using Microsoft.Extensions.Options; using TvDbSharper; using TvDbSharper.Dto; @@ -18,12 +20,12 @@ namespace Kyoo.TheTvdb /// /// The internal tvdb client used to make requests. /// - private readonly TvDbClient _client = new(); + private readonly ITvDbClient _client; /// /// The API key used to authenticate with the tvdb API. /// - private readonly string _apiKey; + private readonly IOptions _apiKey; /// public Provider Provider => new() @@ -35,15 +37,24 @@ namespace Kyoo.TheTvdb }; - public ProviderTvdb(string apiKey) + /// + /// Create a new using a tvdb client and an api key. + /// + /// The tvdb client to use + /// The api key + public ProviderTvdb(ITvDbClient client, IOptions apiKey) { + _client = client; _apiKey = apiKey; } + /// + /// Authenticate and refresh the token of the tvdb client. + /// private Task _Authenticate() { if (_client.Authentication.Token == null) - return _client.Authentication.AuthenticateAsync(_apiKey); + return _client.Authentication.AuthenticateAsync(_apiKey.Value.ApiKey); return _client.Authentication.RefreshTokenAsync(); } diff --git a/Kyoo.TheTvdb/TvdbOption.cs b/Kyoo.TheTvdb/TvdbOption.cs new file mode 100644 index 00000000..3b21ed9e --- /dev/null +++ b/Kyoo.TheTvdb/TvdbOption.cs @@ -0,0 +1,18 @@ +namespace Kyoo.Authentication.Models +{ + /// + /// The option containing the api key for the tvdb. + /// + public class TvdbOption + { + /// + /// The path to get this option from the root configuration. + /// + public const string Path = "tvdb"; + + /// + /// The api key of the tvdb. + /// + public string ApiKey { get; set; } + } +} \ No newline at end of file diff --git a/Kyoo/Controllers/TaskManager.cs b/Kyoo/Controllers/TaskManager.cs index c850a516..a10875c1 100644 --- a/Kyoo/Controllers/TaskManager.cs +++ b/Kyoo/Controllers/TaskManager.cs @@ -144,7 +144,7 @@ namespace Kyoo.Controllers ICollection all = task.GetParameters(); ICollection invalids = arguments.Keys - .Where(x => all.Any(y => x != y.Name)) + .Where(x => all.All(y => x != y.Name)) .ToArray(); if (invalids.Any()) { diff --git a/Kyoo/CoreModule.cs b/Kyoo/CoreModule.cs index a0dc0729..f19ba596 100644 --- a/Kyoo/CoreModule.cs +++ b/Kyoo/CoreModule.cs @@ -38,7 +38,9 @@ namespace Kyoo typeof(IThumbnailsManager), typeof(IMetadataProvider), typeof(ITaskManager), - typeof(ILibraryManager) + typeof(ILibraryManager), + typeof(IIdentifier), + typeof(AProviderComposite) }; /// @@ -99,9 +101,14 @@ namespace Kyoo builder.RegisterType().As().SingleInstance(); builder.RegisterType().As().SingleInstance(); builder.RegisterType().As().InstancePerLifetimeScope(); - builder.RegisterComposite().InstancePerLifetimeScope(); - + builder.RegisterType().As().SingleInstance(); + builder.RegisterComposite(); + builder.Register(x => (AProviderComposite)x.Resolve()); + builder.RegisterTask(); + builder.RegisterTask(); + builder.RegisterTask(); + builder.RegisterTask(); static bool DatabaseIsPresent(IComponentRegistryBuilder x) => x.IsRegistered(new TypedService(typeof(DatabaseContext))); diff --git a/Kyoo/Models/FileExtensions.cs b/Kyoo/Models/FileExtensions.cs index f8058a67..b42b4be2 100644 --- a/Kyoo/Models/FileExtensions.cs +++ b/Kyoo/Models/FileExtensions.cs @@ -4,8 +4,14 @@ using System.Linq; namespace Kyoo.Models.Watch { + /// + /// A static class allowing one to identify files extensions. + /// public static class FileExtensions { + /// + /// The list of known video extensions + /// public static readonly string[] VideoExtensions = { ".webm", @@ -34,17 +40,30 @@ namespace Kyoo.Models.Watch ".3g2" }; + /// + /// Check if a file represent a video file (only by checking the extension of the file) + /// + /// The path of the file to check + /// true if the file is a video file, false otherwise. public static bool IsVideo(string filePath) { return VideoExtensions.Contains(Path.GetExtension(filePath)); } + /// + /// The dictionary of known subtitles extensions and the name of the subtitle codec. + /// public static readonly Dictionary SubtitleExtensions = new() { {".ass", "ass"}, {".str", "subrip"} }; + /// + /// Check if a file represent a subtitle file (only by checking the extension of the file) + /// + /// The path of the file to check + /// true if the file is a subtitle file, false otherwise. public static bool IsSubtitle(string filePath) { return SubtitleExtensions.ContainsKey(Path.GetExtension(filePath)); From 6ebf8a8361be4d317420a4223ad0cf566a396cea Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 18 Jul 2021 17:06:29 +0200 Subject: [PATCH 11/31] Handling configuration --- Kyoo.Authentication/AuthenticationModule.cs | 12 +++-- .../Controllers/IConfigurationManager.cs | 15 ++++++ Kyoo.Common/Module.cs | 34 -------------- Kyoo.TheTvdb/Kyoo.TheTvdb.csproj | 2 +- Kyoo.TheTvdb/PluginTvdb.cs | 38 +++++++++++++++ Kyoo/Controllers/ConfigurationManager.cs | 42 +++++++++++++---- Kyoo/Controllers/PluginManager.cs | 14 +++++- Kyoo/Controllers/TaskManager.cs | 19 +------- Kyoo/CoreModule.cs | 19 +++++--- Kyoo/Helper.cs | 47 +++++++++++++++++++ Kyoo/Startup.cs | 7 +-- 11 files changed, 174 insertions(+), 75 deletions(-) create mode 100644 Kyoo/Helper.cs diff --git a/Kyoo.Authentication/AuthenticationModule.cs b/Kyoo.Authentication/AuthenticationModule.cs index 8e2c78c4..1e9dcd01 100644 --- a/Kyoo.Authentication/AuthenticationModule.cs +++ b/Kyoo.Authentication/AuthenticationModule.cs @@ -9,6 +9,7 @@ using IdentityServer4.Services; using Kyoo.Authentication.Models; using Kyoo.Authentication.Views; using Kyoo.Controllers; +using Kyoo.Models.Attributes; using Kyoo.Models.Permissions; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; @@ -64,6 +65,11 @@ namespace Kyoo.Authentication /// The environment information to check if the app runs in debug mode /// private readonly IWebHostEnvironment _environment; + + /// + /// The configuration manager used to register typed/untyped implementations. + /// + [Injected] public IConfigurationManager ConfigurationManager { private get; set; } /// @@ -98,9 +104,7 @@ namespace Kyoo.Authentication services.Configure(_configuration.GetSection(PermissionOption.Path)); services.Configure(_configuration.GetSection(CertificateOption.Path)); services.Configure(_configuration.GetSection(AuthenticationOption.Path)); - services.AddConfiguration(AuthenticationOption.Path); - - + List clients = new(); _configuration.GetSection("authentication:clients").Bind(clients); CertificateOption certificateOptions = new(); @@ -139,6 +143,8 @@ namespace Kyoo.Authentication /// public void ConfigureAspNet(IApplicationBuilder app) { + ConfigurationManager.AddTyped(AuthenticationOption.Path); + app.UseCookiePolicy(new CookiePolicyOptions { MinimumSameSitePolicy = SameSiteMode.Strict diff --git a/Kyoo.Common/Controllers/IConfigurationManager.cs b/Kyoo.Common/Controllers/IConfigurationManager.cs index 9159d92c..02430b10 100644 --- a/Kyoo.Common/Controllers/IConfigurationManager.cs +++ b/Kyoo.Common/Controllers/IConfigurationManager.cs @@ -12,6 +12,21 @@ namespace Kyoo.Controllers /// public interface IConfigurationManager { + /// + /// Add an editable configuration to the editable configuration list + /// + /// The root path of the editable configuration. It should not be a nested type. + /// The type of the configuration + void AddTyped(string path); + + /// + /// Add an editable configuration to the editable configuration list. + /// WARNING: this method allow you to add an unmanaged type. This type won't be editable. This can be used + /// for external libraries or variable arguments. + /// + /// The root path of the editable configuration. It should not be a nested type. + void AddUntyped(string path); + /// /// Get the value of a setting using it's path. /// diff --git a/Kyoo.Common/Module.cs b/Kyoo.Common/Module.cs index d4442d98..0e8de063 100644 --- a/Kyoo.Common/Module.cs +++ b/Kyoo.Common/Module.cs @@ -1,10 +1,7 @@ -using System.Linq; using Autofac; using Autofac.Builder; using Kyoo.Controllers; -using Kyoo.Models; using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; namespace Kyoo { @@ -75,37 +72,6 @@ namespace Kyoo return builder.RegisterRepository().As(); } - /// - /// Add an editable configuration to the editable configuration list - /// - /// The service collection to edit - /// The root path of the editable configuration. It should not be a nested type. - /// The type of the configuration - /// The given service collection is returned. - public static IServiceCollection AddConfiguration(this IServiceCollection services, string path) - where T : class - { - if (services.Any(x => x.ServiceType == typeof(T))) - return services; - foreach (ConfigurationReference confRef in ConfigurationReference.CreateReference(path)) - services.AddSingleton(confRef); - return services; - } - - /// - /// Add an editable configuration to the editable configuration list. - /// WARNING: this method allow you to add an unmanaged type. This type won't be editable. This can be used - /// for external libraries or variable arguments. - /// - /// The service collection to edit - /// The root path of the editable configuration. It should not be a nested type. - /// The given service collection is returned. - public static IServiceCollection AddUntypedConfiguration(this IServiceCollection services, string path) - { - services.AddSingleton(ConfigurationReference.CreateUntyped(path)); - return services; - } - /// /// Get the public URL of kyoo using the given configuration instance. /// diff --git a/Kyoo.TheTvdb/Kyoo.TheTvdb.csproj b/Kyoo.TheTvdb/Kyoo.TheTvdb.csproj index 8877cb95..ac7dad56 100644 --- a/Kyoo.TheTvdb/Kyoo.TheTvdb.csproj +++ b/Kyoo.TheTvdb/Kyoo.TheTvdb.csproj @@ -19,8 +19,8 @@ - + diff --git a/Kyoo.TheTvdb/PluginTvdb.cs b/Kyoo.TheTvdb/PluginTvdb.cs index 62fc0885..1d008fbf 100644 --- a/Kyoo.TheTvdb/PluginTvdb.cs +++ b/Kyoo.TheTvdb/PluginTvdb.cs @@ -1,7 +1,12 @@ using System; using System.Collections.Generic; using Autofac; +using Kyoo.Authentication.Models; using Kyoo.Controllers; +using Kyoo.Models.Attributes; +using Microsoft.AspNetCore.Builder; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; using TvDbSharper; namespace Kyoo.TheTvdb @@ -33,11 +38,44 @@ namespace Kyoo.TheTvdb public ICollection Requires => ArraySegment.Empty; + /// + /// The configuration to use. + /// + private readonly IConfiguration _configuration; + + /// + /// The configuration manager used to register typed/untyped implementations. + /// + [Injected] public IConfigurationManager ConfigurationManager { private get; set; } + + + /// + /// Create a new tvdb module instance and use the given configuration. + /// + /// The configuration to use + public PluginTvdb(IConfiguration configuration) + { + _configuration = configuration; + } + + /// public void Configure(ContainerBuilder builder) { builder.RegisterType().As(); builder.RegisterProvider(); } + + /// + public void Configure(IServiceCollection services, ICollection availableTypes) + { + services.Configure(_configuration.GetSection(TvdbOption.Path)); + } + + /// + public void ConfigureAspNet(IApplicationBuilder app) + { + ConfigurationManager.AddTyped(TvdbOption.Path); + } } } \ No newline at end of file diff --git a/Kyoo/Controllers/ConfigurationManager.cs b/Kyoo/Controllers/ConfigurationManager.cs index c0a63895..32b2e7f7 100644 --- a/Kyoo/Controllers/ConfigurationManager.cs +++ b/Kyoo/Controllers/ConfigurationManager.cs @@ -36,7 +36,29 @@ namespace Kyoo.Controllers _references = references.ToDictionary(x => x.Path, x => x.Type, StringComparer.OrdinalIgnoreCase); } - private Type GetType(string path) + + /// + public void AddTyped(string path) + { + foreach (ConfigurationReference confRef in ConfigurationReference.CreateReference(path)) + _references.Add(confRef.Path, confRef.Type); + } + + /// + public void AddUntyped(string path) + { + ConfigurationReference config = ConfigurationReference.CreateUntyped(path); + _references.Add(config.Path, config.Type); + } + + /// + /// Get the type of the resource at the given path + /// + /// The path of the resource + /// The path is not editable or readable + /// No configuration exists for the given path + /// The type of the resource at the given path + private Type _GetType(string path) { path = path.Replace("__", ":"); @@ -59,7 +81,7 @@ namespace Kyoo.Controllers { path = path.Replace("__", ":"); // TODO handle lists and dictionaries. - Type type = GetType(path); + Type type = _GetType(path); object ret = _configuration.GetValue(type, path); if (ret != null) return ret; @@ -73,7 +95,7 @@ namespace Kyoo.Controllers { path = path.Replace("__", ":"); // TODO handle lists and dictionaries. - Type type = GetType(path); + Type type = _GetType(path); if (typeof(T).IsAssignableFrom(type)) throw new InvalidCastException($"The type {typeof(T).Name} is not valid for " + $"a resource of type {type.Name}."); @@ -84,12 +106,12 @@ namespace Kyoo.Controllers public async Task EditValue(string path, object value) { path = path.Replace("__", ":"); - Type type = GetType(path); + Type type = _GetType(path); value = JObject.FromObject(value).ToObject(type); if (value == null) throw new ArgumentException("Invalid value format."); - ExpandoObject config = ToObject(_configuration); + ExpandoObject config = _ToObject(_configuration); IDictionary configDic = config; configDic[path] = value; JObject obj = JObject.FromObject(config); @@ -104,7 +126,7 @@ namespace Kyoo.Controllers /// The configuration to transform /// A strongly typed representation of the configuration. [SuppressMessage("ReSharper", "RedundantJumpStatement")] - private ExpandoObject ToObject(IConfiguration config) + private ExpandoObject _ToObject(IConfiguration config) { ExpandoObject obj = new(); @@ -112,12 +134,12 @@ namespace Kyoo.Controllers { try { - Type type = GetType(section.Path); + Type type = _GetType(section.Path); obj.TryAdd(section.Key, section.Get(type)); } catch (ArgumentException) { - obj.TryAdd(section.Key, ToUntyped(section)); + obj.TryAdd(section.Key, _ToUntyped(section)); } catch { @@ -133,13 +155,13 @@ namespace Kyoo.Controllers /// /// The section to convert /// The converted section - private static object ToUntyped(IConfigurationSection config) + private static object _ToUntyped(IConfigurationSection config) { ExpandoObject obj = new(); foreach (IConfigurationSection section in config.GetChildren()) { - obj.TryAdd(section.Key, ToUntyped(section)); + obj.TryAdd(section.Key, _ToUntyped(section)); } if (!obj.Any()) diff --git a/Kyoo/Controllers/PluginManager.cs b/Kyoo/Controllers/PluginManager.cs index cd0a38f6..230480c6 100644 --- a/Kyoo/Controllers/PluginManager.cs +++ b/Kyoo/Controllers/PluginManager.cs @@ -22,7 +22,7 @@ namespace Kyoo.Controllers /// /// The service provider. It allow plugin's activation. /// - private readonly IServiceProvider _provider; + private IServiceProvider _provider; /// /// The configuration to get the plugin's directory. /// @@ -52,6 +52,13 @@ namespace Kyoo.Controllers _logger = logger; } + public void SetProvider(IServiceProvider provider) + { + // TODO temporary bullshit to inject services before the configure asp net. + // TODO should rework this when the host will be reworked, as well as the asp net configure. + _provider = provider; + } + /// public T GetPlugin(string name) @@ -128,6 +135,7 @@ namespace Kyoo.Controllers _logger.LogInformation("Plugin enabled: {Plugins}", _plugins.Select(x => x.Name)); } + /// public void ConfigureContainer(ContainerBuilder builder) { foreach (IPlugin plugin in _plugins) @@ -146,7 +154,11 @@ namespace Kyoo.Controllers public void ConfigureAspnet(IApplicationBuilder app) { foreach (IPlugin plugin in _plugins) + { + using IServiceScope scope = _provider.CreateScope(); + Helper.InjectServices(plugin, x => scope.ServiceProvider.GetRequiredService(x)); plugin.ConfigureAspNet(app); + } } /// diff --git a/Kyoo/Controllers/TaskManager.cs b/Kyoo/Controllers/TaskManager.cs index a10875c1..980bbab0 100644 --- a/Kyoo/Controllers/TaskManager.cs +++ b/Kyoo/Controllers/TaskManager.cs @@ -165,27 +165,12 @@ namespace Kyoo.Controllers })); using IServiceScope scope = _provider.CreateScope(); - InjectServices(task, x => scope.ServiceProvider.GetRequiredService(x)); + Helper.InjectServices(task, x => scope.ServiceProvider.GetRequiredService(x)); await task.Run(args, progress, _taskToken.Token); - InjectServices(task, _ => null); + Helper.InjectServices(task, _ => null); _logger.LogInformation("Task finished: {Task}", task.Name); } - /// - /// Inject services into the marked properties of the given object. - /// - /// The object to inject - /// The function used to retrieve services. (The function is called immediately) - private static void InjectServices(ITask obj, [InstantHandle] Func retrieve) - { - IEnumerable properties = obj.GetType().GetProperties() - .Where(x => x.GetCustomAttribute() != null) - .Where(x => x.CanWrite); - - foreach (PropertyInfo property in properties) - property.SetValue(obj, retrieve(property.PropertyType)); - } - /// /// Start tasks that are scheduled for start. /// diff --git a/Kyoo/CoreModule.cs b/Kyoo/CoreModule.cs index f19ba596..4d4baf7d 100644 --- a/Kyoo/CoreModule.cs +++ b/Kyoo/CoreModule.cs @@ -5,6 +5,7 @@ using Autofac; using Autofac.Core; using Autofac.Core.Registration; using Kyoo.Controllers; +using Kyoo.Models.Attributes; using Kyoo.Models.Options; using Kyoo.Models.Permissions; using Kyoo.Tasks; @@ -81,6 +82,11 @@ namespace Kyoo /// The configuration to use. /// private readonly IConfiguration _configuration; + + /// + /// The configuration manager used to register typed/untyped implementations. + /// + [Injected] public IConfigurationManager ConfigurationManager { private get; set; } /// @@ -136,14 +142,9 @@ namespace Kyoo string publicUrl = _configuration.GetPublicUrl(); services.Configure(_configuration.GetSection(BasicOptions.Path)); - services.AddConfiguration(BasicOptions.Path); services.Configure(_configuration.GetSection(TaskOptions.Path)); - services.AddConfiguration(TaskOptions.Path); services.Configure(_configuration.GetSection(MediaOptions.Path)); - services.AddConfiguration(MediaOptions.Path); - services.AddUntypedConfiguration("database"); - services.AddUntypedConfiguration("logging"); - + services.AddControllers() .AddNewtonsoftJson(x => { @@ -157,6 +158,12 @@ namespace Kyoo /// public void ConfigureAspNet(IApplicationBuilder app) { + ConfigurationManager.AddTyped(BasicOptions.Path); + ConfigurationManager.AddTyped(TaskOptions.Path); + ConfigurationManager.AddTyped(MediaOptions.Path); + ConfigurationManager.AddUntyped("database"); + ConfigurationManager.AddUntyped("logging"); + FileExtensionContentTypeProvider contentTypeProvider = new(); contentTypeProvider.Mappings[".data"] = "application/octet-stream"; app.UseStaticFiles(new StaticFileOptions diff --git a/Kyoo/Helper.cs b/Kyoo/Helper.cs new file mode 100644 index 00000000..244f6eec --- /dev/null +++ b/Kyoo/Helper.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Reflection; +using System.Threading.Tasks; +using JetBrains.Annotations; +using Kyoo.Models.Attributes; +using Newtonsoft.Json; + +namespace Kyoo +{ + public static class Helper + { + /// + /// Inject services into the marked properties of the given object. + /// + /// The object to inject + /// The function used to retrieve services. (The function is called immediately) + public static void InjectServices(object obj, [InstantHandle] Func retrieve) + { + IEnumerable properties = obj.GetType().GetProperties() + .Where(x => x.GetCustomAttribute() != null) + .Where(x => x.CanWrite); + + foreach (PropertyInfo property in properties) + property.SetValue(obj, retrieve(property.PropertyType)); + } + + /// + /// An helper method to get json content from an http server. This is a temporary thing and will probably be + /// replaced by a call to the function of the same name in the System.Net.Http.Json namespace when .net6 + /// gets released. + /// + /// The http server to use. + /// The url to retrieve + /// The type of object to convert + /// A T representing the json contained at the given url. + public static async Task GetFromJsonAsync(this HttpClient client, string url) + { + HttpResponseMessage ret = await client.GetAsync(url); + ret.EnsureSuccessStatusCode(); + string content = await ret.Content.ReadAsStringAsync(); + return JsonConvert.DeserializeObject(content); + } + } +} \ No newline at end of file diff --git a/Kyoo/Startup.cs b/Kyoo/Startup.cs index 9d3c19dd..04a24408 100644 --- a/Kyoo/Startup.cs +++ b/Kyoo/Startup.cs @@ -71,7 +71,6 @@ namespace Kyoo services.AddHttpClient(); - // services.AddTransient(typeof(Lazy<>), typeof(LazyDi<>)); _plugins.ConfigureServices(services); } @@ -87,7 +86,7 @@ namespace Kyoo /// /// The asp net host to configure /// The host environment (is the app in development mode?) - public void Configure(IApplicationBuilder app, IWebHostEnvironment env) + public void Configure(IApplicationBuilder app, IWebHostEnvironment env, IServiceProvider provider) { if (env.IsDevelopment()) app.UseDeveloperExceptionPage(); @@ -114,7 +113,9 @@ namespace Kyoo return next(); }); app.UseResponseCompression(); - + + if (_plugins is PluginManager manager) + manager.SetProvider(provider); _plugins.ConfigureAspnet(app); app.UseSpa(spa => From f1887d1fabd6bd6a6316f6aa230508e1e13c43b6 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 18 Jul 2021 17:31:31 +0200 Subject: [PATCH 12/31] Fixing the regex identifier --- Kyoo/Controllers/RegexIdentifier.cs | 54 +++++++++++++++++------------ 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/Kyoo/Controllers/RegexIdentifier.cs b/Kyoo/Controllers/RegexIdentifier.cs index d99f8255..002ca5d7 100644 --- a/Kyoo/Controllers/RegexIdentifier.cs +++ b/Kyoo/Controllers/RegexIdentifier.cs @@ -38,30 +38,40 @@ namespace Kyoo.Controllers if (!match.Success) throw new IdentificationFailed($"The episode at {path} does not match the episode's regex."); - (Collection collection, Show show, Season season, Episode episode) ret = new(); - - ret.collection.Name = match.Groups["Collection"].Value; - ret.collection.Slug = Utility.ToSlug(ret.collection.Name); - - ret.show.Title = match.Groups["Show"].Value; - ret.show.Slug = Utility.ToSlug(ret.show.Title); - ret.show.Path = Path.GetDirectoryName(path); - ret.episode.Path = path; + (Collection collection, Show show, Season season, Episode episode) ret = ( + collection: new Collection + { + Slug = Utility.ToSlug(match.Groups["Collection"].Value), + Name = match.Groups["Collection"].Value + }, + show: new Show + { + Slug = Utility.ToSlug(match.Groups["Show"].Value), + Title = match.Groups["Show"].Value, + Path = Path.GetDirectoryName(path), + StartAir = match.Groups["StartYear"].Success + ? new DateTime(int.Parse(match.Groups["StartYear"].Value), 1, 1) + : null + }, + season: null, + episode: new Episode + { + SeasonNumber = match.Groups["Season"].Success + ? int.Parse(match.Groups["Season"].Value) + : null, + EpisodeNumber = match.Groups["Episode"].Success + ? int.Parse(match.Groups["Episode"].Value) + : null, + AbsoluteNumber = match.Groups["Absolute"].Success + ? int.Parse(match.Groups["Absolute"].Value) + : null, + Path = path + } + ); - if (match.Groups["StartYear"].Success && int.TryParse(match.Groups["StartYear"].Value, out int tmp)) - ret.show.StartAir = new DateTime(tmp, 1, 1); - - if (match.Groups["Season"].Success && int.TryParse(match.Groups["Season"].Value, out tmp)) - { - ret.season.SeasonNumber = tmp; - ret.episode.SeasonNumber = tmp; - } + if (ret.episode.SeasonNumber.HasValue) + ret.season = new Season { SeasonNumber = ret.episode.SeasonNumber.Value }; - if (match.Groups["Episode"].Success && int.TryParse(match.Groups["Episode"].Value, out tmp)) - ret.episode.EpisodeNumber = tmp; - - if (match.Groups["Absolute"].Success && int.TryParse(match.Groups["Absolute"].Value, out tmp)) - ret.episode.AbsoluteNumber = tmp; if (ret.episode.SeasonNumber == null && ret.episode.EpisodeNumber == null && ret.episode.AbsoluteNumber == null) From 5a480402e197f888e200460900895ab3285a9fc9 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 18 Jul 2021 18:43:14 +0200 Subject: [PATCH 13/31] Fixing track register and directory exist --- Kyoo.Common/Controllers/IIdentifier.cs | 18 +++++++++++---- Kyoo.Common/Models/LibraryItem.cs | 2 +- Kyoo.Common/Models/Resources/Track.cs | 10 +++++---- Kyoo.CommonAPI/JsonSerializer.cs | 22 +++++++++++++++---- .../Migrations/20210627141941_Triggers.cs | 6 ++--- Kyoo.Postgresql/PostgresModule.cs | 5 +++++ .../Migrations/20210626141347_Triggers.cs | 8 +++---- .../Library/SpecificTests/TrackTests.cs | 15 +++++++++++++ Kyoo/Controllers/FileManager.cs | 2 +- Kyoo/Controllers/RegexIdentifier.cs | 8 +++---- .../Repositories/SeasonRepository.cs | 7 +++++- Kyoo/Tasks/Crawler.cs | 6 +++-- Kyoo/Tasks/Housekeeping.cs | 9 ++++++++ Kyoo/Tasks/RegisterEpisode.cs | 7 +++++- Kyoo/Tasks/RegisterSubtitle.cs | 7 ++++-- 15 files changed, 101 insertions(+), 31 deletions(-) diff --git a/Kyoo.Common/Controllers/IIdentifier.cs b/Kyoo.Common/Controllers/IIdentifier.cs index b841a63b..5033bb77 100644 --- a/Kyoo.Common/Controllers/IIdentifier.cs +++ b/Kyoo.Common/Controllers/IIdentifier.cs @@ -12,22 +12,32 @@ namespace Kyoo.Controllers /// /// Identify a path and return the parsed metadata. /// - /// The path of the episode file to parse. + /// + /// The path of the episode file to parse. + /// + /// + /// The path of the episode file relative to the library root. It starts with a /. + /// /// The identifier could not work for the given path. /// /// A tuple of models representing parsed metadata. /// If no metadata could be parsed for a type, null can be returned. /// - Task<(Collection, Show, Season, Episode)> Identify(string path); + Task<(Collection, Show, Season, Episode)> Identify(string path, string relativePath); /// /// Identify an external subtitle or track file from it's path and return the parsed metadata. /// - /// The path of the external track file to parse. + /// + /// The path of the external track file to parse. + /// + /// + /// The path of the episode file relative to the library root. It starts with a /. + /// /// The identifier could not work for the given path. /// /// The metadata of the track identified. /// - Task IdentifyTrack(string path); + Task IdentifyTrack(string path, string relativePath); } } \ No newline at end of file diff --git a/Kyoo.Common/Models/LibraryItem.cs b/Kyoo.Common/Models/LibraryItem.cs index 4df07770..6bc61c2e 100644 --- a/Kyoo.Common/Models/LibraryItem.cs +++ b/Kyoo.Common/Models/LibraryItem.cs @@ -58,7 +58,7 @@ namespace Kyoo.Models /// By default, the http path for this poster is returned from the public API. /// This can be disabled using the internal query flag. /// - [SerializeAs("{HOST}/api/{Type}/{Slug}/poster")] public string Poster { get; set; } + [SerializeAs("{HOST}/api/{Type:l}/{Slug}/poster")] public string Poster { get; set; } /// /// The type of this item (ether a collection, a show or a movie). diff --git a/Kyoo.Common/Models/Resources/Track.cs b/Kyoo.Common/Models/Resources/Track.cs index f093699b..df00e94e 100644 --- a/Kyoo.Common/Models/Resources/Track.cs +++ b/Kyoo.Common/Models/Resources/Track.cs @@ -35,8 +35,8 @@ namespace Kyoo.Models { string type = Type.ToString().ToLower(); string index = TrackIndex != 0 ? $"-{TrackIndex}" : string.Empty; - string episode = EpisodeSlug ?? Episode.Slug ?? EpisodeID.ToString(); - return $"{episode}.{Language}{index}{(IsForced ? ".forced" : "")}.{type}"; + string episode = EpisodeSlug ?? Episode?.Slug ?? EpisodeID.ToString(); + return $"{episode}.{Language ?? "und"}{index}{(IsForced ? ".forced" : "")}.{type}"; } [UsedImplicitly] private set { @@ -47,11 +47,13 @@ namespace Kyoo.Models if (!match.Success) throw new ArgumentException("Invalid track slug. " + - "Format: {episodeSlug}.{language}[-{index}][-forced].{type}[.{extension}]"); + "Format: {episodeSlug}.{language}[-{index}][.forced].{type}[.{extension}]"); EpisodeSlug = match.Groups["ep"].Value; Language = match.Groups["lang"].Value; - TrackIndex = int.Parse(match.Groups["index"].Value); + if (Language == "und") + Language = null; + TrackIndex = match.Groups["index"].Success ? int.Parse(match.Groups["index"].Value) : 0; IsForced = match.Groups["forced"].Success; Type = Enum.Parse(match.Groups["type"].Value, true); } diff --git a/Kyoo.CommonAPI/JsonSerializer.cs b/Kyoo.CommonAPI/JsonSerializer.cs index bf65a32a..3a2caac7 100644 --- a/Kyoo.CommonAPI/JsonSerializer.cs +++ b/Kyoo.CommonAPI/JsonSerializer.cs @@ -115,9 +115,10 @@ namespace Kyoo.Controllers public object GetValue(object target) { - return Regex.Replace(_format, @"(? + return Regex.Replace(_format, @"(? { string value = x.Groups[1].Value; + string modifier = x.Groups[3].Value; if (value == "HOST") return _host; @@ -127,9 +128,22 @@ namespace Kyoo.Controllers .FirstOrDefault(y => y.Name == value); if (properties == null) return null; - if (properties.GetValue(target) is string ret) - return ret; - throw new ArgumentException($"Invalid serializer replacement {value}"); + object objValue = properties.GetValue(target); + if (objValue is not string ret) + ret = objValue?.ToString(); + if (ret == null) + throw new ArgumentException($"Invalid serializer replacement {value}"); + + foreach (char modification in modifier) + { + ret = modification switch + { + 'l' => ret.ToLowerInvariant(), + 'u' => ret.ToUpperInvariant(), + _ => throw new ArgumentException($"Invalid serializer modificator {modification}.") + }; + } + return ret; }); } diff --git a/Kyoo.Postgresql/Migrations/20210627141941_Triggers.cs b/Kyoo.Postgresql/Migrations/20210627141941_Triggers.cs index 16569748..a773e02b 100644 --- a/Kyoo.Postgresql/Migrations/20210627141941_Triggers.cs +++ b/Kyoo.Postgresql/Migrations/20210627141941_Triggers.cs @@ -91,7 +91,7 @@ namespace Kyoo.Postgresql.Migrations END, CASE (is_forced) WHEN false THEN '' - ELSE '-forced' + ELSE '.forced' END, '.', type ) WHERE episode_id = NEW.id; @@ -117,14 +117,14 @@ namespace Kyoo.Postgresql.Migrations END IF; NEW.slug := CONCAT( (SELECT slug FROM episodes WHERE id = NEW.episode_id), - '.', NEW.language, + '.', COALESCE(NEW.language, 'und'), CASE (NEW.track_index) WHEN 0 THEN '' ELSE CONCAT('-', NEW.track_index) END, CASE (NEW.is_forced) WHEN false THEN '' - ELSE '-forced' + ELSE '.forced' END, '.', NEW.type ); diff --git a/Kyoo.Postgresql/PostgresModule.cs b/Kyoo.Postgresql/PostgresModule.cs index f0c8f23c..124df770 100644 --- a/Kyoo.Postgresql/PostgresModule.cs +++ b/Kyoo.Postgresql/PostgresModule.cs @@ -6,6 +6,7 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; +using Npgsql; namespace Kyoo.Postgresql { @@ -73,6 +74,10 @@ namespace Kyoo.Postgresql { DatabaseContext context = provider.GetRequiredService(); context.Database.Migrate(); + + using NpgsqlConnection conn = (NpgsqlConnection)context.Database.GetDbConnection(); + conn.Open(); + conn.ReloadTypes(); } } } \ No newline at end of file diff --git a/Kyoo.SqLite/Migrations/20210626141347_Triggers.cs b/Kyoo.SqLite/Migrations/20210626141347_Triggers.cs index f3ae8325..370fdd37 100644 --- a/Kyoo.SqLite/Migrations/20210626141347_Triggers.cs +++ b/Kyoo.SqLite/Migrations/20210626141347_Triggers.cs @@ -61,14 +61,14 @@ namespace Kyoo.SqLite.Migrations AND Language = new.Language AND IsForced = new.IsForced ) WHERE ID = new.ID AND TrackIndex = 0; UPDATE Tracks SET Slug = (SELECT Slug FROM Episodes WHERE ID = EpisodeID) || - '.' || Language || + '.' || COALESCE(Language, 'und') || CASE (TrackIndex) WHEN 0 THEN '' ELSE '-' || (TrackIndex) END || CASE (IsForced) WHEN false THEN '' - ELSE '-forced' + ELSE '.forced' END || CASE (Type) WHEN 1 THEN '.video' @@ -98,7 +98,7 @@ namespace Kyoo.SqLite.Migrations END || CASE (IsForced) WHEN false THEN '' - ELSE '-forced' + ELSE '.forced' END || CASE (Type) WHEN 1 THEN '.video' @@ -123,7 +123,7 @@ namespace Kyoo.SqLite.Migrations END || CASE (IsForced) WHEN false THEN '' - ELSE '-forced' + ELSE '.forced' END || CASE (Type) WHEN 1 THEN '.video' diff --git a/Kyoo.Tests/Library/SpecificTests/TrackTests.cs b/Kyoo.Tests/Library/SpecificTests/TrackTests.cs index 3c2e2043..3aebaef9 100644 --- a/Kyoo.Tests/Library/SpecificTests/TrackTests.cs +++ b/Kyoo.Tests/Library/SpecificTests/TrackTests.cs @@ -47,5 +47,20 @@ namespace Kyoo.Tests.Library Track track = await _repository.Get(1); Assert.Equal("new-slug-s1e1.eng-1.subtitle", track.Slug); } + + [Fact] + public async Task UndefinedLanguageSlugTest() + { + await _repository.Create(new Track + { + ID = 5, + TrackIndex = 0, + Type = StreamType.Video, + Language = null, + EpisodeID = TestSample.Get().ID + }); + Track track = await _repository.Get(5); + Assert.Equal("anohana-s1e1.und.video", track.Slug); + } } } \ No newline at end of file diff --git a/Kyoo/Controllers/FileManager.cs b/Kyoo/Controllers/FileManager.cs index bd7ee4c2..4aa89467 100644 --- a/Kyoo/Controllers/FileManager.cs +++ b/Kyoo/Controllers/FileManager.cs @@ -98,7 +98,7 @@ namespace Kyoo.Controllers /// public Task Exists(string path) { - return Task.FromResult(File.Exists(path)); + return Task.FromResult(File.Exists(path) || Directory.Exists(path)); } /// diff --git a/Kyoo/Controllers/RegexIdentifier.cs b/Kyoo/Controllers/RegexIdentifier.cs index 002ca5d7..3e5d82fe 100644 --- a/Kyoo/Controllers/RegexIdentifier.cs +++ b/Kyoo/Controllers/RegexIdentifier.cs @@ -30,10 +30,10 @@ namespace Kyoo.Controllers } /// - public Task<(Collection, Show, Season, Episode)> Identify(string path) + public Task<(Collection, Show, Season, Episode)> Identify(string path, string relativePath) { Regex regex = new(_configuration.Value.Regex, RegexOptions.IgnoreCase | RegexOptions.Compiled); - Match match = regex.Match(path); + Match match = regex.Match(relativePath); if (!match.Success) throw new IdentificationFailed($"The episode at {path} does not match the episode's regex."); @@ -84,10 +84,10 @@ namespace Kyoo.Controllers } /// - public Task IdentifyTrack(string path) + public Task IdentifyTrack(string path, string relativePath) { Regex regex = new(_configuration.Value.SubtitleRegex, RegexOptions.IgnoreCase | RegexOptions.Compiled); - Match match = regex.Match(path); + Match match = regex.Match(relativePath); if (!match.Success) throw new IdentificationFailed($"The subtitle at {path} does not match the subtitle's regex."); diff --git a/Kyoo/Controllers/Repositories/SeasonRepository.cs b/Kyoo/Controllers/Repositories/SeasonRepository.cs index fe042e66..e0036982 100644 --- a/Kyoo/Controllers/Repositories/SeasonRepository.cs +++ b/Kyoo/Controllers/Repositories/SeasonRepository.cs @@ -96,7 +96,12 @@ namespace Kyoo.Controllers protected override async Task Validate(Season resource) { if (resource.ShowID <= 0) - throw new InvalidOperationException($"Can't store a season not related to any show (showID: {resource.ShowID})."); + { + if (resource.Show == null) + throw new InvalidOperationException( + $"Can't store a season not related to any show (showID: {resource.ShowID})."); + resource.ShowID = resource.Show.ID; + } await base.Validate(resource); await resource.ExternalIDs.ForEachAsync(async id => diff --git a/Kyoo/Tasks/Crawler.cs b/Kyoo/Tasks/Crawler.cs index 6f4cf9eb..a2e28493 100644 --- a/Kyoo/Tasks/Crawler.cs +++ b/Kyoo/Tasks/Crawler.cs @@ -140,7 +140,8 @@ namespace Kyoo.Tasks { TaskManager.StartTask(reporter, new Dictionary { - ["path"] = episodePath[path.Length..], + ["path"] = episodePath, + ["relativePath"] = episodePath[path.Length..], ["library"] = library }, cancellationToken); percent += 100f / paths.Length; @@ -162,7 +163,8 @@ namespace Kyoo.Tasks { TaskManager.StartTask(reporter, new Dictionary { - ["path"] = trackPath + ["path"] = trackPath, + ["relativePath"] = trackPath[path.Length..] }, cancellationToken); percent += 100f / subtitles.Length; } diff --git a/Kyoo/Tasks/Housekeeping.cs b/Kyoo/Tasks/Housekeeping.cs index 661f0be0..03084aaa 100644 --- a/Kyoo/Tasks/Housekeeping.cs +++ b/Kyoo/Tasks/Housekeeping.cs @@ -4,6 +4,7 @@ using System.Threading.Tasks; using Kyoo.Controllers; using Kyoo.Models; using Kyoo.Models.Attributes; +using Microsoft.Extensions.Logging; namespace Kyoo.Tasks { @@ -39,6 +40,10 @@ namespace Kyoo.Tasks /// The file manager used walk inside directories and check they existences. /// [Injected] public IFileManager FileManager { private get; set; } + /// + /// The logger used to inform the user that episodes has been removed. + /// + [Injected] public ILogger Logger { private get; set; } /// @@ -55,6 +60,8 @@ namespace Kyoo.Tasks if (await FileManager.Exists(show.Path)) continue; + Logger.LogWarning("Show {Name}'s folder has been deleted (was {Path}), removing it from kyoo", + show.Title, show.Path); await LibraryManager.Delete(show); } @@ -65,6 +72,8 @@ namespace Kyoo.Tasks if (await FileManager.Exists(episode.Path)) continue; + Logger.LogWarning("Episode {Slug}'s file has been deleted (was {Path}), removing it from kyoo", + episode.Slug, episode.Path); await LibraryManager.Delete(episode); } diff --git a/Kyoo/Tasks/RegisterEpisode.cs b/Kyoo/Tasks/RegisterEpisode.cs index 6ef512e6..3c01c03d 100644 --- a/Kyoo/Tasks/RegisterEpisode.cs +++ b/Kyoo/Tasks/RegisterEpisode.cs @@ -62,6 +62,8 @@ namespace Kyoo.Tasks return new() { TaskParameter.CreateRequired("path", "The path of the episode file"), + TaskParameter.CreateRequired("relativePath", + "The path of the episode file relative to the library root. It starts with a /."), TaskParameter.CreateRequired("library", "The library in witch the episode is") }; } @@ -70,13 +72,15 @@ namespace Kyoo.Tasks public async Task Run(TaskParameters arguments, IProgress progress, CancellationToken cancellationToken) { string path = arguments["path"].As(); + string relativePath = arguments["relativePath"].As(); Library library = arguments["library"].As(); progress.Report(0); if (library.Providers == null) await LibraryManager.Load(library, x => x.Providers); MetadataProvider.UseProviders(library.Providers); - (Collection collection, Show show, Season season, Episode episode) = await Identifier.Identify(path); + (Collection collection, Show show, Season season, Episode episode) = await Identifier.Identify(path, + relativePath); progress.Report(15); collection = await _RegisterAndFill(collection); @@ -105,6 +109,7 @@ namespace Kyoo.Tasks if (season != null) season.Show = show; + season = await _RegisterAndFill(season); progress.Report(60); diff --git a/Kyoo/Tasks/RegisterSubtitle.cs b/Kyoo/Tasks/RegisterSubtitle.cs index 524977ed..66dd5a48 100644 --- a/Kyoo/Tasks/RegisterSubtitle.cs +++ b/Kyoo/Tasks/RegisterSubtitle.cs @@ -48,7 +48,9 @@ namespace Kyoo.Tasks { return new() { - TaskParameter.CreateRequired("path", "The path of the episode file"), + TaskParameter.CreateRequired("path", "The path of the subtitle file"), + TaskParameter.CreateRequired("relativePath", + "The path of the subtitle file relative to the library root. It starts with a /.") }; } @@ -56,9 +58,10 @@ namespace Kyoo.Tasks public async Task Run(TaskParameters arguments, IProgress progress, CancellationToken cancellationToken) { string path = arguments["path"].As(); + string relativePath = arguments["relativePath"].As(); progress.Report(0); - Track track = await Identifier.IdentifyTrack(path); + Track track = await Identifier.IdentifyTrack(path, relativePath); progress.Report(25); if (track.Episode == null) From a4635866a7df67ed8a2ea11db07fe5c3d05601d1 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Mon, 19 Jul 2021 16:17:51 +0200 Subject: [PATCH 14/31] Handling failed tasks and fixing library provider registration --- .../Models/Exceptions/TaskFailedException.cs | 45 +++++++++ Kyoo.Common/Models/Resources/Episode.cs | 8 +- Kyoo.Common/Models/Resources/Track.cs | 25 ++--- Kyoo.Common/Models/WatchItem.cs | 2 + .../Library/SpecificTests/CollectionsTests.cs | 2 +- .../Library/SpecificTests/EpisodeTests.cs | 2 +- .../Library/SpecificTests/GenreTests.cs | 2 +- .../Library/SpecificTests/LibraryItemTest.cs | 2 +- .../Library/SpecificTests/LibraryTests.cs | 19 +++- .../Library/SpecificTests/PeopleTests.cs | 2 +- .../Library/SpecificTests/ProviderTests.cs | 2 +- .../Library/SpecificTests/SanityTests.cs | 2 +- .../Library/SpecificTests/SeasonTests.cs | 2 +- Kyoo.Tests/Library/SpecificTests/ShowTests.cs | 2 +- .../Library/SpecificTests/StudioTests.cs | 2 +- .../Library/SpecificTests/TrackTests.cs | 2 +- Kyoo.Tests/Library/SpecificTests/UserTests.cs | 2 +- Kyoo.Tests/Library/TestSample.cs | 14 ++- Kyoo.WebApp | 2 +- Kyoo/Controllers/RegexIdentifier.cs | 2 +- .../Repositories/LibraryRepository.cs | 7 +- Kyoo/Controllers/TaskManager.cs | 6 +- Kyoo/Tasks/Crawler.cs | 1 + Kyoo/Tasks/RegisterEpisode.cs | 98 +++++++++++-------- Kyoo/Tasks/RegisterSubtitle.cs | 47 +++++---- Kyoo/Views/ShowApi.cs | 2 +- Kyoo/Views/SubtitleApi.cs | 40 +++++++- Kyoo/Views/TrackApi.cs | 2 - 28 files changed, 228 insertions(+), 116 deletions(-) create mode 100644 Kyoo.Common/Models/Exceptions/TaskFailedException.cs diff --git a/Kyoo.Common/Models/Exceptions/TaskFailedException.cs b/Kyoo.Common/Models/Exceptions/TaskFailedException.cs new file mode 100644 index 00000000..5fe65f7c --- /dev/null +++ b/Kyoo.Common/Models/Exceptions/TaskFailedException.cs @@ -0,0 +1,45 @@ +using System; +using System.Runtime.Serialization; +using Kyoo.Controllers; + +namespace Kyoo.Models.Exceptions +{ + /// + /// An exception raised when an failed. + /// + [Serializable] + public class TaskFailedException : AggregateException + { + /// + /// Create a new with a default message. + /// + public TaskFailedException() + : base("A task failed.") + {} + + /// + /// Create a new with a custom message. + /// + /// The message to use. + public TaskFailedException(string message) + : base(message) + {} + + /// + /// Create a new wrapping another exception. + /// + /// The exception to wrap. + public TaskFailedException(Exception exception) + : base(exception) + {} + + /// + /// The serialization constructor + /// + /// Serialization infos + /// The serialization context + protected TaskFailedException(SerializationInfo info, StreamingContext context) + : base(info, context) + { } + } +} \ No newline at end of file diff --git a/Kyoo.Common/Models/Resources/Episode.cs b/Kyoo.Common/Models/Resources/Episode.cs index cdb7fb8f..e7cf056a 100644 --- a/Kyoo.Common/Models/Resources/Episode.cs +++ b/Kyoo.Common/Models/Resources/Episode.cs @@ -20,9 +20,11 @@ namespace Kyoo.Models { get { - if (ShowSlug == null && Show == null) - return GetSlug(ShowID.ToString(), SeasonNumber, EpisodeNumber, AbsoluteNumber); - return GetSlug(ShowSlug ?? Show.Slug, SeasonNumber, EpisodeNumber, AbsoluteNumber); + if (ShowSlug != null || Show != null) + return GetSlug(ShowSlug ?? Show.Slug, SeasonNumber, EpisodeNumber, AbsoluteNumber); + return ShowID != 0 + ? GetSlug(ShowID.ToString(), SeasonNumber, EpisodeNumber, AbsoluteNumber) + : null; } [UsedImplicitly] [NotNull] private set { diff --git a/Kyoo.Common/Models/Resources/Track.cs b/Kyoo.Common/Models/Resources/Track.cs index df00e94e..d82cdbe1 100644 --- a/Kyoo.Common/Models/Resources/Track.cs +++ b/Kyoo.Common/Models/Resources/Track.cs @@ -156,30 +156,17 @@ namespace Kyoo.Models } /// - /// Utility method to edit a track slug (this only return a slug with the modification, nothing is stored) + /// Utility method to create a track slug from a incomplete slug (only add the type of the track). /// /// The slug to edit /// The new type of this - /// - /// - /// /// - public static string EditSlug(string baseSlug, - StreamType type = StreamType.Unknown, - string language = null, - int? index = null, - bool? forced = null) + public static string BuildSlug(string baseSlug, + StreamType type) { - Track track = new() {Slug = baseSlug}; - if (type != StreamType.Unknown) - track.Type = type; - if (language != null) - track.Language = language; - if (index != null) - track.TrackIndex = index.Value; - if (forced != null) - track.IsForced = forced.Value; - return track.Slug; + return baseSlug.EndsWith($".{type}", StringComparison.InvariantCultureIgnoreCase) + ? baseSlug + : $"{baseSlug}.{type.ToString().ToLowerInvariant()}"; } } } diff --git a/Kyoo.Common/Models/WatchItem.cs b/Kyoo.Common/Models/WatchItem.cs index 3b64d54e..eaf437c7 100644 --- a/Kyoo.Common/Models/WatchItem.cs +++ b/Kyoo.Common/Models/WatchItem.cs @@ -176,6 +176,7 @@ namespace Kyoo.Models return new WatchItem { EpisodeID = ep.ID, + Slug = ep.Slug, ShowSlug = ep.Show.Slug, SeasonNumber = ep.SeasonNumber, EpisodeNumber = ep.EpisodeNumber, @@ -183,6 +184,7 @@ namespace Kyoo.Models Title = ep.Title, ReleaseDate = ep.ReleaseDate, Path = ep.Path, + Container = PathIO.GetExtension(ep.Path)![1..], Video = ep.Tracks.FirstOrDefault(x => x.Type == StreamType.Video), Audios = ep.Tracks.Where(x => x.Type == StreamType.Audio).ToArray(), Subtitles = ep.Tracks.Where(x => x.Type == StreamType.Subtitle).ToArray(), diff --git a/Kyoo.Tests/Library/SpecificTests/CollectionsTests.cs b/Kyoo.Tests/Library/SpecificTests/CollectionsTests.cs index 7a5976de..73691bf7 100644 --- a/Kyoo.Tests/Library/SpecificTests/CollectionsTests.cs +++ b/Kyoo.Tests/Library/SpecificTests/CollectionsTests.cs @@ -3,7 +3,7 @@ using Kyoo.Models; using Xunit; using Xunit.Abstractions; -namespace Kyoo.Tests.Library +namespace Kyoo.Tests.Database { namespace SqLite { diff --git a/Kyoo.Tests/Library/SpecificTests/EpisodeTests.cs b/Kyoo.Tests/Library/SpecificTests/EpisodeTests.cs index 6b1adf27..d9e0e9ff 100644 --- a/Kyoo.Tests/Library/SpecificTests/EpisodeTests.cs +++ b/Kyoo.Tests/Library/SpecificTests/EpisodeTests.cs @@ -4,7 +4,7 @@ using Kyoo.Models; using Xunit; using Xunit.Abstractions; -namespace Kyoo.Tests.Library +namespace Kyoo.Tests.Database { namespace SqLite { diff --git a/Kyoo.Tests/Library/SpecificTests/GenreTests.cs b/Kyoo.Tests/Library/SpecificTests/GenreTests.cs index d79dba5e..dc820187 100644 --- a/Kyoo.Tests/Library/SpecificTests/GenreTests.cs +++ b/Kyoo.Tests/Library/SpecificTests/GenreTests.cs @@ -3,7 +3,7 @@ using Kyoo.Models; using Xunit; using Xunit.Abstractions; -namespace Kyoo.Tests.Library +namespace Kyoo.Tests.Database { namespace SqLite { diff --git a/Kyoo.Tests/Library/SpecificTests/LibraryItemTest.cs b/Kyoo.Tests/Library/SpecificTests/LibraryItemTest.cs index b2db4f66..c5639dbb 100644 --- a/Kyoo.Tests/Library/SpecificTests/LibraryItemTest.cs +++ b/Kyoo.Tests/Library/SpecificTests/LibraryItemTest.cs @@ -5,7 +5,7 @@ using Kyoo.Models; using Xunit; using Xunit.Abstractions; -namespace Kyoo.Tests.Library +namespace Kyoo.Tests.Database { namespace SqLite { diff --git a/Kyoo.Tests/Library/SpecificTests/LibraryTests.cs b/Kyoo.Tests/Library/SpecificTests/LibraryTests.cs index fbed1793..079f50cf 100644 --- a/Kyoo.Tests/Library/SpecificTests/LibraryTests.cs +++ b/Kyoo.Tests/Library/SpecificTests/LibraryTests.cs @@ -1,8 +1,11 @@ +using System.Linq; +using System.Threading.Tasks; using Kyoo.Controllers; +using Kyoo.Models; using Xunit; using Xunit.Abstractions; -namespace Kyoo.Tests.Library +namespace Kyoo.Tests.Database { namespace SqLite { @@ -23,7 +26,7 @@ namespace Kyoo.Tests.Library } } - public abstract class ALibraryTests : RepositoryTests + public abstract class ALibraryTests : RepositoryTests { private readonly ILibraryRepository _repository; @@ -32,5 +35,17 @@ namespace Kyoo.Tests.Library { _repository = Repositories.LibraryManager.LibraryRepository; } + + [Fact] + public async Task CreateWithProvider() + { + Library library = TestSample.GetNew(); + library.Providers = new[] { TestSample.Get() }; + await _repository.Create(library); + Library retrieved = await _repository.Get(2); + await Repositories.LibraryManager.Load(retrieved, x => x.Providers); + Assert.Equal(1, retrieved.Providers.Count); + Assert.Equal(TestSample.Get().Slug, retrieved.Providers.First().Slug); + } } } \ No newline at end of file diff --git a/Kyoo.Tests/Library/SpecificTests/PeopleTests.cs b/Kyoo.Tests/Library/SpecificTests/PeopleTests.cs index fc8b788d..23d40bfe 100644 --- a/Kyoo.Tests/Library/SpecificTests/PeopleTests.cs +++ b/Kyoo.Tests/Library/SpecificTests/PeopleTests.cs @@ -3,7 +3,7 @@ using Kyoo.Models; using Xunit; using Xunit.Abstractions; -namespace Kyoo.Tests.Library +namespace Kyoo.Tests.Database { namespace SqLite { diff --git a/Kyoo.Tests/Library/SpecificTests/ProviderTests.cs b/Kyoo.Tests/Library/SpecificTests/ProviderTests.cs index 853e34a1..9c022875 100644 --- a/Kyoo.Tests/Library/SpecificTests/ProviderTests.cs +++ b/Kyoo.Tests/Library/SpecificTests/ProviderTests.cs @@ -3,7 +3,7 @@ using Kyoo.Models; using Xunit; using Xunit.Abstractions; -namespace Kyoo.Tests.Library +namespace Kyoo.Tests.Database { namespace SqLite { diff --git a/Kyoo.Tests/Library/SpecificTests/SanityTests.cs b/Kyoo.Tests/Library/SpecificTests/SanityTests.cs index 78637d35..933bbf82 100644 --- a/Kyoo.Tests/Library/SpecificTests/SanityTests.cs +++ b/Kyoo.Tests/Library/SpecificTests/SanityTests.cs @@ -5,7 +5,7 @@ using Kyoo.Models; using Xunit; using Xunit.Abstractions; -namespace Kyoo.Tests.Library +namespace Kyoo.Tests.Database { public class GlobalTests : IDisposable, IAsyncDisposable { diff --git a/Kyoo.Tests/Library/SpecificTests/SeasonTests.cs b/Kyoo.Tests/Library/SpecificTests/SeasonTests.cs index 39be8b82..b1692747 100644 --- a/Kyoo.Tests/Library/SpecificTests/SeasonTests.cs +++ b/Kyoo.Tests/Library/SpecificTests/SeasonTests.cs @@ -4,7 +4,7 @@ using Kyoo.Models; using Xunit; using Xunit.Abstractions; -namespace Kyoo.Tests.Library +namespace Kyoo.Tests.Database { namespace SqLite { diff --git a/Kyoo.Tests/Library/SpecificTests/ShowTests.cs b/Kyoo.Tests/Library/SpecificTests/ShowTests.cs index 8940f0c3..63207710 100644 --- a/Kyoo.Tests/Library/SpecificTests/ShowTests.cs +++ b/Kyoo.Tests/Library/SpecificTests/ShowTests.cs @@ -8,7 +8,7 @@ using Microsoft.EntityFrameworkCore; using Xunit; using Xunit.Abstractions; -namespace Kyoo.Tests.Library +namespace Kyoo.Tests.Database { namespace SqLite { diff --git a/Kyoo.Tests/Library/SpecificTests/StudioTests.cs b/Kyoo.Tests/Library/SpecificTests/StudioTests.cs index f5093b19..c727f67a 100644 --- a/Kyoo.Tests/Library/SpecificTests/StudioTests.cs +++ b/Kyoo.Tests/Library/SpecificTests/StudioTests.cs @@ -3,7 +3,7 @@ using Kyoo.Models; using Xunit; using Xunit.Abstractions; -namespace Kyoo.Tests.Library +namespace Kyoo.Tests.Database { namespace SqLite { diff --git a/Kyoo.Tests/Library/SpecificTests/TrackTests.cs b/Kyoo.Tests/Library/SpecificTests/TrackTests.cs index 3aebaef9..0ff0c156 100644 --- a/Kyoo.Tests/Library/SpecificTests/TrackTests.cs +++ b/Kyoo.Tests/Library/SpecificTests/TrackTests.cs @@ -4,7 +4,7 @@ using Kyoo.Models; using Xunit; using Xunit.Abstractions; -namespace Kyoo.Tests.Library +namespace Kyoo.Tests.Database { namespace SqLite { diff --git a/Kyoo.Tests/Library/SpecificTests/UserTests.cs b/Kyoo.Tests/Library/SpecificTests/UserTests.cs index be67296d..24bfc789 100644 --- a/Kyoo.Tests/Library/SpecificTests/UserTests.cs +++ b/Kyoo.Tests/Library/SpecificTests/UserTests.cs @@ -3,7 +3,7 @@ using Kyoo.Models; using Xunit; using Xunit.Abstractions; -namespace Kyoo.Tests.Library +namespace Kyoo.Tests.Database { namespace SqLite { diff --git a/Kyoo.Tests/Library/TestSample.cs b/Kyoo.Tests/Library/TestSample.cs index adbe7d84..96cd63c1 100644 --- a/Kyoo.Tests/Library/TestSample.cs +++ b/Kyoo.Tests/Library/TestSample.cs @@ -9,8 +9,14 @@ namespace Kyoo.Tests private static readonly Dictionary> NewSamples = new() { { - typeof(Show), - () => new Show() + typeof(Library), + () => new Library + { + ID = 2, + Slug = "new-library", + Name = "New Library", + Paths = new [] {"/a/random/path"} + } } }; @@ -18,8 +24,8 @@ namespace Kyoo.Tests private static readonly Dictionary> Samples = new() { { - typeof(Models.Library), - () => new Models.Library + typeof(Library), + () => new Library { ID = 1, Slug = "deck", diff --git a/Kyoo.WebApp b/Kyoo.WebApp index 22a02671..dcdebad1 160000 --- a/Kyoo.WebApp +++ b/Kyoo.WebApp @@ -1 +1 @@ -Subproject commit 22a02671918201d6d9d4e80a76f01b59b216a82d +Subproject commit dcdebad14cbcdf1f9486cb9178e6518d10c0e97f diff --git a/Kyoo/Controllers/RegexIdentifier.cs b/Kyoo/Controllers/RegexIdentifier.cs index 3e5d82fe..f578aa30 100644 --- a/Kyoo/Controllers/RegexIdentifier.cs +++ b/Kyoo/Controllers/RegexIdentifier.cs @@ -87,7 +87,7 @@ namespace Kyoo.Controllers public Task IdentifyTrack(string path, string relativePath) { Regex regex = new(_configuration.Value.SubtitleRegex, RegexOptions.IgnoreCase | RegexOptions.Compiled); - Match match = regex.Match(relativePath); + Match match = regex.Match(path); if (!match.Success) throw new IdentificationFailed($"The subtitle at {path} does not match the subtitle's regex."); diff --git a/Kyoo/Controllers/Repositories/LibraryRepository.cs b/Kyoo/Controllers/Repositories/LibraryRepository.cs index 02194b77..b7b782bc 100644 --- a/Kyoo/Controllers/Repositories/LibraryRepository.cs +++ b/Kyoo/Controllers/Repositories/LibraryRepository.cs @@ -30,7 +30,7 @@ namespace Kyoo.Controllers /// Create a new instance. /// /// The database handle - /// The providere repository + /// The provider repository public LibraryRepository(DatabaseContext database, IProviderRepository providers) : base(database) { @@ -53,8 +53,8 @@ namespace Kyoo.Controllers public override async Task Create(Library obj) { await base.Create(obj); - obj.ProviderLinks = obj.Providers?.Select(x => Link.Create(obj, x)).ToList(); _database.Entry(obj).State = EntityState.Added; + obj.ProviderLinks.ForEach(x => _database.Entry(x).State = EntityState.Added); await _database.SaveChangesAsync($"Trying to insert a duplicated library (slug {obj.Slug} already exists)."); return obj; } @@ -63,6 +63,9 @@ namespace Kyoo.Controllers protected override async Task Validate(Library resource) { await base.Validate(resource); + resource.ProviderLinks = resource.Providers? + .Select(x => Link.Create(resource, x)) + .ToList(); await resource.ProviderLinks.ForEachAsync(async id => { id.Second = await _providers.CreateIfNotExists(id.Second); diff --git a/Kyoo/Controllers/TaskManager.cs b/Kyoo/Controllers/TaskManager.cs index 980bbab0..65d623dc 100644 --- a/Kyoo/Controllers/TaskManager.cs +++ b/Kyoo/Controllers/TaskManager.cs @@ -1,11 +1,9 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Reflection; using System.Threading; using System.Threading.Tasks; using JetBrains.Annotations; -using Kyoo.Models.Attributes; using Kyoo.Models.Exceptions; using Kyoo.Models.Options; using Microsoft.Extensions.DependencyInjection; @@ -112,6 +110,10 @@ namespace Kyoo.Controllers { await RunTask(task, progress, args); } + catch (TaskFailedException ex) + { + _logger.LogWarning("The task \"{Task}\" failed: {Message}", task.Name, ex.Message); + } catch (Exception e) { _logger.LogError(e, "An unhandled exception occured while running the task {Task}", task.Name); diff --git a/Kyoo/Tasks/Crawler.cs b/Kyoo/Tasks/Crawler.cs index a2e28493..e65bdd87 100644 --- a/Kyoo/Tasks/Crawler.cs +++ b/Kyoo/Tasks/Crawler.cs @@ -150,6 +150,7 @@ namespace Kyoo.Tasks string[] subtitles = files .Where(FileExtensions.IsSubtitle) + .Where(x => x.Contains("/Extra/")) .Where(x => tracks.All(y => y.Path != x)) .ToArray(); percent = 0; diff --git a/Kyoo/Tasks/RegisterEpisode.cs b/Kyoo/Tasks/RegisterEpisode.cs index 3c01c03d..0498dd68 100644 --- a/Kyoo/Tasks/RegisterEpisode.cs +++ b/Kyoo/Tasks/RegisterEpisode.cs @@ -79,54 +79,66 @@ namespace Kyoo.Tasks if (library.Providers == null) await LibraryManager.Load(library, x => x.Providers); MetadataProvider.UseProviders(library.Providers); - (Collection collection, Show show, Season season, Episode episode) = await Identifier.Identify(path, - relativePath); - progress.Report(15); - - collection = await _RegisterAndFill(collection); - progress.Report(20); - - Show registeredShow = await _RegisterAndFill(show); - if (registeredShow.Path != show.Path) + try { - if (show.StartAir.HasValue) + (Collection collection, Show show, Season season, Episode episode) = await Identifier.Identify(path, + relativePath); + progress.Report(15); + + collection = await _RegisterAndFill(collection); + progress.Report(20); + + Show registeredShow = await _RegisterAndFill(show); + if (registeredShow.Path != show.Path) { - show.Slug += $"-{show.StartAir.Value.Year}"; - show = await LibraryManager.Create(show); + if (show.StartAir.HasValue) + { + show.Slug += $"-{show.StartAir.Value.Year}"; + show = await LibraryManager.Create(show); + } + else + { + throw new TaskFailedException($"Duplicated show found ({show.Slug}) " + + $"at {registeredShow.Path} and {show.Path}"); + } } else - { - throw new DuplicatedItemException($"Duplicated show found ({show.Slug}) " + - $"at {registeredShow.Path} and {show.Path}"); - } + show = registeredShow; + + // If they are not already loaded, load external ids to allow metadata providers to use them. + if (show.ExternalIDs == null) + await LibraryManager.Load(show, x => x.ExternalIDs); + progress.Report(50); + + if (season != null) + season.Show = show; + + season = await _RegisterAndFill(season); + progress.Report(60); + + episode = await MetadataProvider.Get(episode); + progress.Report(70); + episode.Show = show; + episode.Season = season; + episode.Tracks = (await Transcoder.ExtractInfos(episode, false)) + .Where(x => x.Type != StreamType.Attachment) + .ToArray(); + await ThumbnailsManager.DownloadImages(episode); + progress.Report(90); + + await LibraryManager.Create(episode); + progress.Report(95); + await LibraryManager.AddShowLink(show, library, collection); + progress.Report(100); + } + catch (IdentificationFailed ex) + { + throw new TaskFailedException(ex); + } + catch (DuplicatedItemException ex) + { + throw new TaskFailedException(ex); } - else - show = registeredShow; - // If they are not already loaded, load external ids to allow metadata providers to use them. - if (show.ExternalIDs == null) - await LibraryManager.Load(show, x => x.ExternalIDs); - progress.Report(50); - - if (season != null) - season.Show = show; - - season = await _RegisterAndFill(season); - progress.Report(60); - - episode = await MetadataProvider.Get(episode); - progress.Report(70); - episode.Show = show; - episode.Season = season; - episode.Tracks = (await Transcoder.ExtractInfos(episode, false)) - .Where(x => x.Type != StreamType.Attachment) - .ToArray(); - await ThumbnailsManager.DownloadImages(episode); - progress.Report(90); - - await LibraryManager.Create(episode); - progress.Report(95); - await LibraryManager.AddShowLink(show, library, collection); - progress.Report(100); } /// diff --git a/Kyoo/Tasks/RegisterSubtitle.cs b/Kyoo/Tasks/RegisterSubtitle.cs index 66dd5a48..07620e6c 100644 --- a/Kyoo/Tasks/RegisterSubtitle.cs +++ b/Kyoo/Tasks/RegisterSubtitle.cs @@ -59,30 +59,37 @@ namespace Kyoo.Tasks { string path = arguments["path"].As(); string relativePath = arguments["relativePath"].As(); - - progress.Report(0); - Track track = await Identifier.IdentifyTrack(path, relativePath); - progress.Report(25); - - if (track.Episode == null) - throw new IdentificationFailed($"No episode identified for the track at {path}"); - if (track.Episode.ID == 0) + + try { - if (track.Episode.Slug != null) - track.Episode = await LibraryManager.Get(track.Episode.Slug); - else if (track.Episode.Path != null) + progress.Report(0); + Track track = await Identifier.IdentifyTrack(path, relativePath); + progress.Report(25); + + if (track.Episode == null) + throw new TaskFailedException($"No episode identified for the track at {path}"); + if (track.Episode.ID == 0) { - track.Episode = await LibraryManager.GetOrDefault(x => x.Path == track.Episode.Path); - if (track.Episode == null) - throw new ItemNotFoundException($"No episode found for subtitle at: ${path}."); + if (track.Episode.Slug != null) + track.Episode = await LibraryManager.Get(track.Episode.Slug); + else if (track.Episode.Path != null) + { + track.Episode = await LibraryManager.GetOrDefault(x => x.Path.StartsWith(track.Episode.Path)); + if (track.Episode == null) + throw new TaskFailedException($"No episode found for the track at: {path}."); + } + else + throw new TaskFailedException($"No episode identified for the track at {path}"); } - else - throw new IdentificationFailed($"No episode identified for the track at {path}"); + + progress.Report(50); + await LibraryManager.Create(track); + progress.Report(100); + } + catch (IdentificationFailed ex) + { + throw new TaskFailedException(ex); } - - progress.Report(50); - await LibraryManager.Create(track); - progress.Report(100); } } } \ No newline at end of file diff --git a/Kyoo/Views/ShowApi.cs b/Kyoo/Views/ShowApi.cs index 605ac28e..73822a9c 100644 --- a/Kyoo/Views/ShowApi.cs +++ b/Kyoo/Views/ShowApi.cs @@ -386,7 +386,7 @@ namespace Kyoo.Api string path = Path.Combine(_files.GetExtraDirectory(show), "Attachments"); return (await _files.ListFiles(path)) .ToDictionary(Path.GetFileNameWithoutExtension, - x => $"{BaseURL}/api/shows/{slug}/fonts/{Path.GetFileName(x)}"); + x => $"{BaseURL}api/shows/{slug}/fonts/{Path.GetFileName(x)}"); } catch (ItemNotFoundException) { diff --git a/Kyoo/Views/SubtitleApi.cs b/Kyoo/Views/SubtitleApi.cs index 7f05eedf..5426078d 100644 --- a/Kyoo/Views/SubtitleApi.cs +++ b/Kyoo/Views/SubtitleApi.cs @@ -2,6 +2,7 @@ using Microsoft.AspNetCore.Mvc; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Threading.Tasks; using Kyoo.Controllers; using Kyoo.Models.Permissions; @@ -21,12 +22,43 @@ namespace Kyoo.Api _files = files; } - - [HttpGet("{slug}.{extension}")] + [HttpGet("{id:int}")] [Permission(nameof(SubtitleApi), Kind.Read)] - public async Task GetSubtitle(string slug, string extension) + public async Task GetSubtitle(int id) { - Track subtitle = await _libraryManager.GetOrDefault(Track.EditSlug(slug, StreamType.Subtitle)); + Track subtitle = await _libraryManager.GetOrDefault(id); + return subtitle != null + ? _files.FileResult(subtitle.Path) + : NotFound(); + } + + [HttpGet("{id:int}.{extension}")] + [Permission(nameof(SubtitleApi), Kind.Read)] + public async Task GetSubtitle(int id, string extension) + { + Track subtitle = await _libraryManager.GetOrDefault(id); + if (subtitle == null) + return NotFound(); + if (subtitle.Codec == "subrip" && extension == "vtt") + return new ConvertSubripToVtt(subtitle.Path, _files); + return _files.FileResult(subtitle.Path); + } + + + [HttpGet("{slug}")] + [Permission(nameof(SubtitleApi), Kind.Read)] + public async Task GetSubtitle(string slug) + { + string extension = null; + + if (slug.Count(x => x == '.') == 2) + { + int idx = slug.LastIndexOf('.'); + extension = slug[(idx + 1)..]; + slug = slug[..idx]; + } + + Track subtitle = await _libraryManager.GetOrDefault(Track.BuildSlug(slug, StreamType.Subtitle)); if (subtitle == null) return NotFound(); if (subtitle.Codec == "subrip" && extension == "vtt") diff --git a/Kyoo/Views/TrackApi.cs b/Kyoo/Views/TrackApi.cs index 0eadaf3b..07df38ce 100644 --- a/Kyoo/Views/TrackApi.cs +++ b/Kyoo/Views/TrackApi.cs @@ -45,8 +45,6 @@ namespace Kyoo.Api { try { - // TODO This won't work with the local repository implementation. - // TODO Implement something like this (a dotnet-ef's QueryCompilationContext): https://stackoverflow.com/questions/62687811/how-can-i-convert-a-custom-function-to-a-sql-expression-for-entity-framework-cor return await _libraryManager.Get(x => x.Tracks.Any(y => y.Slug == slug)); } catch (ItemNotFoundException) From 4c0179ad4d572fd3f26a6c46122dff814aec1fb8 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Mon, 19 Jul 2021 17:52:24 +0200 Subject: [PATCH 15/31] Fixing tvdb provider and adding fields=all --- Kyoo.Common/Models/Link.cs | 5 ++-- Kyoo.Common/Models/Resources/Show.cs | 2 ++ Kyoo.CommonAPI/ResourceViewAttribute.cs | 39 +++++++++++++++---------- Kyoo.TheTvdb/Convertors.cs | 6 ++-- Kyoo.TheTvdb/ProviderTvdb.cs | 33 +++++++++++++++++++-- Kyoo/Tasks/Crawler.cs | 2 +- Kyoo/Tasks/RegisterEpisode.cs | 7 +++-- Kyoo/settings.json | 4 +++ 8 files changed, 72 insertions(+), 26 deletions(-) diff --git a/Kyoo.Common/Models/Link.cs b/Kyoo.Common/Models/Link.cs index 6d815af9..09c519da 100644 --- a/Kyoo.Common/Models/Link.cs +++ b/Kyoo.Common/Models/Link.cs @@ -1,5 +1,6 @@ using System; using System.Linq.Expressions; +using Kyoo.Models.Attributes; namespace Kyoo.Models { @@ -107,12 +108,12 @@ namespace Kyoo.Models /// /// A reference of the first resource. /// - public T1 First { get; set; } + [SerializeIgnore] public T1 First { get; set; } /// /// A reference to the second resource. /// - public T2 Second { get; set; } + [SerializeIgnore] public T2 Second { get; set; } /// diff --git a/Kyoo.Common/Models/Resources/Show.cs b/Kyoo.Common/Models/Resources/Show.cs index ffb2ae49..656d6ffb 100644 --- a/Kyoo.Common/Models/Resources/Show.cs +++ b/Kyoo.Common/Models/Resources/Show.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using JetBrains.Annotations; using Kyoo.Common.Models.Attributes; using Kyoo.Controllers; using Kyoo.Models.Attributes; @@ -157,6 +158,7 @@ namespace Kyoo.Models /// This method will never return anything if the are not loaded. /// The slug of the provider /// The field of the asked provider. + [CanBeNull] public string GetID(string provider) { return ExternalIDs?.FirstOrDefault(x => x.Second.Slug == provider)?.DataID; diff --git a/Kyoo.CommonAPI/ResourceViewAttribute.cs b/Kyoo.CommonAPI/ResourceViewAttribute.cs index 06198663..487373c4 100644 --- a/Kyoo.CommonAPI/ResourceViewAttribute.cs +++ b/Kyoo.CommonAPI/ResourceViewAttribute.cs @@ -43,22 +43,31 @@ namespace Kyoo.CommonApi PropertyInfo[] properties = type.GetProperties() .Where(x => x.GetCustomAttribute() != null) .ToArray(); - fields = fields.Select(x => - { - string property = properties - .FirstOrDefault(y => string.Equals(x, y.Name, StringComparison.InvariantCultureIgnoreCase)) - ?.Name; - if (property != null) - return property; - context.Result = new BadRequestObjectResult(new + if (fields.Count == 1 && fields.Contains("all")) + { + fields = properties.Select(x => x.Name).ToList(); + } + else + { + fields = fields + .Select(x => { - Error = $"{x} does not exist on {type.Name}." - }); - return null; - }) - .ToList(); - if (context.Result != null) - return; + string property = properties + .FirstOrDefault(y + => string.Equals(x, y.Name, StringComparison.InvariantCultureIgnoreCase)) + ?.Name; + if (property != null) + return property; + context.Result = new BadRequestObjectResult(new + { + Error = $"{x} does not exist on {type.Name}." + }); + return null; + }) + .ToList(); + if (context.Result != null) + return; + } } context.HttpContext.Items["fields"] = fields; base.OnActionExecuting(context); diff --git a/Kyoo.TheTvdb/Convertors.cs b/Kyoo.TheTvdb/Convertors.cs index b7edd2ea..6aa5c3ef 100644 --- a/Kyoo.TheTvdb/Convertors.cs +++ b/Kyoo.TheTvdb/Convertors.cs @@ -33,7 +33,8 @@ namespace Kyoo.TheTvdb /// The parsed or null. private static DateTime ParseDate(string date) { - DateTime.TryParseExact(date, "yyyy-mm-dd", CultureInfo.InvariantCulture, DateTimeStyles.None, out DateTime parsed); + DateTime.TryParseExact(date, "yyyy-MM-dd", CultureInfo.InvariantCulture, + DateTimeStyles.None, out DateTime parsed); return parsed; } @@ -122,7 +123,8 @@ namespace Kyoo.TheTvdb } } }, - Role = actor.Role + Role = actor.Role, + Type = "Actor" }; } diff --git a/Kyoo.TheTvdb/ProviderTvdb.cs b/Kyoo.TheTvdb/ProviderTvdb.cs index c6d57fd0..78834bfd 100644 --- a/Kyoo.TheTvdb/ProviderTvdb.cs +++ b/Kyoo.TheTvdb/ProviderTvdb.cs @@ -71,11 +71,21 @@ namespace Kyoo.TheTvdb }; } + /// + /// Retrieve metadata about a show. + /// + /// The base show to retrieve metadata for. + /// A new show filled with metadata from the tvdb. [ItemCanBeNull] private async Task _GetShow([NotNull] Show show) { if (!int.TryParse(show.GetID(Provider.Slug), out int id)) - return (await _SearchShow(show.Title)).FirstOrDefault(); + { + Show found = (await _SearchShow(show.Title)).FirstOrDefault(); + if (found == null) + return null; + return await Get(found); + } TvDbResponse series = await _client.Series.GetAsync(id); Show ret = series.Data.ToShow(Provider); @@ -84,6 +94,11 @@ namespace Kyoo.TheTvdb return ret; } + /// + /// Retrieve metadata about an episode. + /// + /// The base episode to retrieve metadata for. + /// A new episode filled with metadata from the tvdb. [ItemCanBeNull] private async Task _GetEpisode([NotNull] Episode episode) { @@ -106,11 +121,23 @@ namespace Kyoo.TheTvdb return ArraySegment.Empty; } + /// + /// Search for shows in the tvdb. + /// + /// The query to ask the tvdb about. + /// A list of shows that could be found on the tvdb. [ItemNotNull] private async Task> _SearchShow(string query) { - TvDbResponse shows = await _client.Search.SearchSeriesByNameAsync(query); - return shows.Data.Select(x => x.ToShow(Provider)).ToArray(); + try + { + TvDbResponse shows = await _client.Search.SearchSeriesByNameAsync(query); + return shows.Data.Select(x => x.ToShow(Provider)).ToArray(); + } + catch (TvDbServerException) + { + return ArraySegment.Empty; + } } } } \ No newline at end of file diff --git a/Kyoo/Tasks/Crawler.cs b/Kyoo/Tasks/Crawler.cs index e65bdd87..0d38e34e 100644 --- a/Kyoo/Tasks/Crawler.cs +++ b/Kyoo/Tasks/Crawler.cs @@ -150,7 +150,7 @@ namespace Kyoo.Tasks string[] subtitles = files .Where(FileExtensions.IsSubtitle) - .Where(x => x.Contains("/Extra/")) + .Where(x => !x.Contains("/Extra/")) .Where(x => tracks.All(y => y.Path != x)) .ToArray(); percent = 0; diff --git a/Kyoo/Tasks/RegisterEpisode.cs b/Kyoo/Tasks/RegisterEpisode.cs index 0498dd68..674ae477 100644 --- a/Kyoo/Tasks/RegisterEpisode.cs +++ b/Kyoo/Tasks/RegisterEpisode.cs @@ -112,14 +112,15 @@ namespace Kyoo.Tasks if (season != null) season.Show = show; - season = await _RegisterAndFill(season); + if (season != null) + season.Title ??= $"Season {season.SeasonNumber}"; progress.Report(60); - episode = await MetadataProvider.Get(episode); - progress.Report(70); episode.Show = show; episode.Season = season; + episode = await MetadataProvider.Get(episode); + progress.Report(70); episode.Tracks = (await Transcoder.ExtractInfos(episode, false)) .Where(x => x.Type != StreamType.Attachment) .ToArray(); diff --git a/Kyoo/settings.json b/Kyoo/settings.json index 3233e7f5..ffc9b446 100644 --- a/Kyoo/settings.json +++ b/Kyoo/settings.json @@ -60,5 +60,9 @@ }, "profilePicturePath": "users/", "clients": [] + }, + + "tvdb": { + "apiKey": "REDACTED" } } From 7875c72dd92dea98948ccadbfa648430629a70ed Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Tue, 20 Jul 2021 00:06:26 +0200 Subject: [PATCH 16/31] Re enabling the metadata provider loader task --- Kyoo.Common/Models/MetadataID.cs | 6 ++ Kyoo.WebApp | 2 +- Kyoo/CoreModule.cs | 1 + Kyoo/Tasks/MetadataProviderLoader.cs | 121 +++++++++++++++++---------- 4 files changed, 83 insertions(+), 47 deletions(-) diff --git a/Kyoo.Common/Models/MetadataID.cs b/Kyoo.Common/Models/MetadataID.cs index 34872314..de6ac817 100644 --- a/Kyoo.Common/Models/MetadataID.cs +++ b/Kyoo.Common/Models/MetadataID.cs @@ -19,6 +19,12 @@ namespace Kyoo.Models /// The URL of the resource on the external provider. /// public string Link { get; set; } + + /// + /// A shortcut to access the provider of this metadata. + /// Unlike the property, this is serializable. + /// + public Provider Provider => Second; /// /// The expression to retrieve the unique ID of a MetadataID. This is an aggregate of the two resources IDs. diff --git a/Kyoo.WebApp b/Kyoo.WebApp index dcdebad1..87783a5b 160000 --- a/Kyoo.WebApp +++ b/Kyoo.WebApp @@ -1 +1 @@ -Subproject commit dcdebad14cbcdf1f9486cb9178e6518d10c0e97f +Subproject commit 87783a5bfdd79f21d72bad13da1d96ec8e08cd42 diff --git a/Kyoo/CoreModule.cs b/Kyoo/CoreModule.cs index 4d4baf7d..1632644f 100644 --- a/Kyoo/CoreModule.cs +++ b/Kyoo/CoreModule.cs @@ -115,6 +115,7 @@ namespace Kyoo builder.RegisterTask(); builder.RegisterTask(); builder.RegisterTask(); + builder.RegisterTask(); static bool DatabaseIsPresent(IComponentRegistryBuilder x) => x.IsRegistered(new TypedService(typeof(DatabaseContext))); diff --git a/Kyoo/Tasks/MetadataProviderLoader.cs b/Kyoo/Tasks/MetadataProviderLoader.cs index 899a2657..db50f9ec 100644 --- a/Kyoo/Tasks/MetadataProviderLoader.cs +++ b/Kyoo/Tasks/MetadataProviderLoader.cs @@ -1,46 +1,75 @@ -// using System; -// using System.Collections.Generic; -// using System.Threading; -// using System.Threading.Tasks; -// using Kyoo.Controllers; -// using Kyoo.Models; -// using Microsoft.Extensions.DependencyInjection; -// -// namespace Kyoo.Tasks -// { -// public class MetadataProviderLoader : ITask -// { -// public string Slug => "reload-metdata"; -// public string Name => "Reload Metadata Providers"; -// public string Description => "Add every loaded metadata provider to the database."; -// public string HelpMessage => null; -// public bool RunOnStartup => true; -// public int Priority => 1000; -// -// public async Task Run(IServiceProvider serviceProvider, CancellationToken cancellationToken, string arguments = null) -// { -// using IServiceScope serviceScope = serviceProvider.CreateScope(); -// IProviderRepository providers = serviceScope.ServiceProvider.GetService(); -// IThumbnailsManager thumbnails = serviceScope.ServiceProvider.GetService(); -// IPluginManager pluginManager = serviceScope.ServiceProvider.GetService(); -// -// foreach (IMetadataProvider provider in pluginManager!.GetPlugins()) -// { -// if (string.IsNullOrEmpty(provider.Provider.Slug)) -// throw new ArgumentException($"Empty provider slug (name: {provider.Provider.Name})."); -// await providers!.CreateIfNotExists(provider.Provider); -// await thumbnails!.Validate(provider.Provider); -// } -// } -// -// public Task> GetPossibleParameters() -// { -// return Task.FromResult>(null); -// } -// -// public int? Progress() -// { -// return null; -// } -// } -// } \ No newline at end of file +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Kyoo.Controllers; +using Kyoo.Models.Attributes; +using Kyoo.Models.Exceptions; + +namespace Kyoo.Tasks +{ + /// + /// A task that download metadata providers images. + /// + public class MetadataProviderLoader : ITask + { + /// + public string Slug => "reload-metdata"; + + /// + public string Name => "Reload Metadata Providers"; + + /// + public string Description => "Add every loaded metadata provider to the database."; + + /// + public string HelpMessage => null; + + /// + public bool RunOnStartup => true; + + /// + public int Priority => 1000; + + /// + public bool IsHidden => true; + + /// + /// The provider repository used to create in-db providers from metadata providers. + /// + [Injected] public IProviderRepository Providers { private get; set; } + /// + /// The thumbnail manager used to download providers logo. + /// + [Injected] public IThumbnailsManager Thumbnails { private get; set; } + /// + /// The list of metadata providers to register. + /// + [Injected] public ICollection MetadataProviders { private get; set; } + + + /// + public TaskParameters GetParameters() + { + return new(); + } + + /// + public async Task Run(TaskParameters arguments, IProgress progress, CancellationToken cancellationToken) + { + float percent = 0; + progress.Report(0); + + foreach (IMetadataProvider provider in MetadataProviders) + { + if (string.IsNullOrEmpty(provider.Provider.Slug)) + throw new TaskFailedException($"Empty provider slug (name: {provider.Provider.Name})."); + await Providers.CreateIfNotExists(provider.Provider); + await Thumbnails.DownloadImages(provider.Provider); + percent += 100f / MetadataProviders.Count; + progress.Report(percent); + } + progress.Report(100); + } + } +} \ No newline at end of file From bd596638a7767b42c67c8d7aea2ae6fbf4d5734f Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Wed, 21 Jul 2021 18:30:43 +0200 Subject: [PATCH 17/31] Creating a file system composite --- Kyoo.Authentication/Views/AccountApi.cs | 8 +- .../{IFileManager.cs => IFileSystem.cs} | 26 +--- Kyoo.Common/Controllers/ITask.cs | 7 + Kyoo.Common/Kyoo.Common.csproj | 1 + .../Attributes/FileSystemMetadataAttribute.cs | 35 +++++ Kyoo.Common/Models/Resources/Episode.cs | 2 +- Kyoo.Common/Models/Resources/Show.cs | 4 +- Kyoo.Common/Models/WatchItem.cs | 2 +- Kyoo.Common/Utility/Utility.cs | 2 +- .../RepositoryActivator.cs | 0 .../{Library => Database}/RepositoryTests.cs | 0 .../SpecificTests/CollectionsTests.cs | 0 .../SpecificTests/EpisodeTests.cs | 0 .../SpecificTests/GenreTests.cs | 0 .../SpecificTests/LibraryItemTest.cs | 0 .../SpecificTests/LibraryTests.cs | 0 .../SpecificTests/PeopleTests.cs | 0 .../SpecificTests/ProviderTests.cs | 0 .../SpecificTests/SanityTests.cs | 0 .../SpecificTests/SeasonTests.cs | 0 .../SpecificTests/ShowTests.cs | 0 .../SpecificTests/StudioTests.cs | 0 .../SpecificTests/TrackTests.cs | 0 .../SpecificTests/UserTests.cs | 0 .../{Library => Database}/TestContext.cs | 0 .../{Library => Database}/TestSample.cs | 0 .../FileSystems/FileSystemComposite.cs | 143 ++++++++++++++++++ .../Controllers/FileSystems/HttpFileSystem.cs | 123 +++++++++++++++ .../LocalFileSystem.cs} | 33 +--- Kyoo/Controllers/ThumbnailsManager.cs | 56 +++++-- Kyoo/Controllers/Transcoder.cs | 15 +- Kyoo/CoreModule.cs | 8 +- Kyoo/Kyoo.csproj | 1 + Kyoo/Startup.cs | 2 + Kyoo/Tasks/Crawler.cs | 4 +- Kyoo/Tasks/Housekeeping.cs | 6 +- Kyoo/Views/EpisodeApi.cs | 4 +- Kyoo/Views/PeopleApi.cs | 4 +- Kyoo/Views/ProviderApi.cs | 4 +- Kyoo/Views/SeasonApi.cs | 4 +- Kyoo/Views/ShowApi.cs | 4 +- Kyoo/Views/SubtitleApi.cs | 10 +- Kyoo/Views/VideoApi.cs | 4 +- 43 files changed, 412 insertions(+), 100 deletions(-) rename Kyoo.Common/Controllers/{IFileManager.cs => IFileSystem.cs} (78%) create mode 100644 Kyoo.Common/Models/Attributes/FileSystemMetadataAttribute.cs rename Kyoo.Tests/{Library => Database}/RepositoryActivator.cs (100%) rename Kyoo.Tests/{Library => Database}/RepositoryTests.cs (100%) rename Kyoo.Tests/{Library => Database}/SpecificTests/CollectionsTests.cs (100%) rename Kyoo.Tests/{Library => Database}/SpecificTests/EpisodeTests.cs (100%) rename Kyoo.Tests/{Library => Database}/SpecificTests/GenreTests.cs (100%) rename Kyoo.Tests/{Library => Database}/SpecificTests/LibraryItemTest.cs (100%) rename Kyoo.Tests/{Library => Database}/SpecificTests/LibraryTests.cs (100%) rename Kyoo.Tests/{Library => Database}/SpecificTests/PeopleTests.cs (100%) rename Kyoo.Tests/{Library => Database}/SpecificTests/ProviderTests.cs (100%) rename Kyoo.Tests/{Library => Database}/SpecificTests/SanityTests.cs (100%) rename Kyoo.Tests/{Library => Database}/SpecificTests/SeasonTests.cs (100%) rename Kyoo.Tests/{Library => Database}/SpecificTests/ShowTests.cs (100%) rename Kyoo.Tests/{Library => Database}/SpecificTests/StudioTests.cs (100%) rename Kyoo.Tests/{Library => Database}/SpecificTests/TrackTests.cs (100%) rename Kyoo.Tests/{Library => Database}/SpecificTests/UserTests.cs (100%) rename Kyoo.Tests/{Library => Database}/TestContext.cs (100%) rename Kyoo.Tests/{Library => Database}/TestSample.cs (100%) create mode 100644 Kyoo/Controllers/FileSystems/FileSystemComposite.cs create mode 100644 Kyoo/Controllers/FileSystems/HttpFileSystem.cs rename Kyoo/Controllers/{FileManager.cs => FileSystems/LocalFileSystem.cs} (77%) diff --git a/Kyoo.Authentication/Views/AccountApi.cs b/Kyoo.Authentication/Views/AccountApi.cs index 59d964f0..4288250e 100644 --- a/Kyoo.Authentication/Views/AccountApi.cs +++ b/Kyoo.Authentication/Views/AccountApi.cs @@ -36,7 +36,7 @@ namespace Kyoo.Authentication.Views /// /// A file manager to send profile pictures /// - private readonly IFileManager _files; + private readonly IFileSystem _files; /// /// Options about authentication. Those options are monitored and reloads are supported. /// @@ -50,7 +50,7 @@ namespace Kyoo.Authentication.Views /// A file manager to send profile pictures /// Authentication options (this may be hot reloaded) public AccountApi(IUserRepository users, - IFileManager files, + IFileSystem files, IOptions options) { _users = users; @@ -205,8 +205,8 @@ namespace Kyoo.Authentication.Views user.Username = data.Username; if (data.Picture?.Length > 0) { - string path = Path.Combine(_options.Value.ProfilePicturePath, user.ID.ToString()); - await using Stream file = _files.NewFile(path); + string path = _files.Combine(_options.Value.ProfilePicturePath, user.ID.ToString()); + await using Stream file = await _files.NewFile(path); await data.Picture.CopyToAsync(file); } return await _users.Edit(user, false); diff --git a/Kyoo.Common/Controllers/IFileManager.cs b/Kyoo.Common/Controllers/IFileSystem.cs similarity index 78% rename from Kyoo.Common/Controllers/IFileManager.cs rename to Kyoo.Common/Controllers/IFileSystem.cs index d39cff6e..2fb1c6d7 100644 --- a/Kyoo.Common/Controllers/IFileManager.cs +++ b/Kyoo.Common/Controllers/IFileSystem.cs @@ -10,7 +10,7 @@ namespace Kyoo.Controllers /// /// A service to abstract the file system to allow custom file systems (like distant file systems or external providers) /// - public interface IFileManager + public interface IFileSystem { // TODO find a way to handle Transmux/Transcode with this system. @@ -41,14 +41,14 @@ namespace Kyoo.Controllers /// The path of the file /// If the file could not be found. /// A reader to read the file. - public Stream GetReader([NotNull] string path); + public Task GetReader([NotNull] string path); /// /// Create a new file at . /// /// The path of the new file. /// A writer to write to the new file. - public Stream NewFile([NotNull] string path); + public Task NewFile([NotNull] string path); /// /// Create a new directory at the given path @@ -87,24 +87,6 @@ namespace Kyoo.Controllers /// /// The show to proceed /// The extra directory of the show - public string GetExtraDirectory(Show show); - - /// - /// Get the extra directory of a season. - /// This method is in this system to allow a filesystem to use a different metadata policy for one. - /// It can be useful if the filesystem is readonly. - /// - /// The season to proceed - /// The extra directory of the season - public string GetExtraDirectory(Season season); - - /// - /// Get the extra directory of an episode. - /// This method is in this system to allow a filesystem to use a different metadata policy for one. - /// It can be useful if the filesystem is readonly. - /// - /// The episode to proceed - /// The extra directory of the episode - public string GetExtraDirectory(Episode episode); + public string GetExtraDirectory([NotNull] Show show); } } \ No newline at end of file diff --git a/Kyoo.Common/Controllers/ITask.cs b/Kyoo.Common/Controllers/ITask.cs index 3267837b..1896bbcd 100644 --- a/Kyoo.Common/Controllers/ITask.cs +++ b/Kyoo.Common/Controllers/ITask.cs @@ -5,6 +5,7 @@ using System.Threading; using System.Threading.Tasks; using JetBrains.Annotations; using Kyoo.Models.Attributes; +using Kyoo.Models.Exceptions; namespace Kyoo.Controllers { @@ -210,6 +211,12 @@ namespace Kyoo.Controllers /// they will be set to an available service from the service container before calling this method. /// They also will be removed after this method return (or throw) to prevent dangling services. /// + /// + /// An exception meaning that the task has failed for handled reasons like invalid arguments, + /// invalid environment, missing plugins or failures not related to a default in the code. + /// This exception allow the task to display a failure message to the end user while others exceptions + /// will be displayed as unhandled exceptions and display a stack trace. + /// public Task Run([NotNull] TaskParameters arguments, [NotNull] IProgress progress, CancellationToken cancellationToken); diff --git a/Kyoo.Common/Kyoo.Common.csproj b/Kyoo.Common/Kyoo.Common.csproj index ada2ed60..844be997 100644 --- a/Kyoo.Common/Kyoo.Common.csproj +++ b/Kyoo.Common/Kyoo.Common.csproj @@ -27,6 +27,7 @@ + diff --git a/Kyoo.Common/Models/Attributes/FileSystemMetadataAttribute.cs b/Kyoo.Common/Models/Attributes/FileSystemMetadataAttribute.cs new file mode 100644 index 00000000..464c44d0 --- /dev/null +++ b/Kyoo.Common/Models/Attributes/FileSystemMetadataAttribute.cs @@ -0,0 +1,35 @@ +using System; +using System.ComponentModel.Composition; +using Kyoo.Controllers; + +namespace Kyoo.Common.Models.Attributes +{ + /// + /// An attribute to inform how a works. + /// + [MetadataAttribute] + [AttributeUsage(AttributeTargets.Class)] + public class FileSystemMetadataAttribute : Attribute + { + /// + /// The scheme(s) used to identify this path. + /// It can be something like http, https, ftp, file and so on. + /// + /// + /// If multiples files with the same schemes exists, an exception will be thrown. + /// + public string[] Scheme { get; } + + /// + /// true if the scheme should be removed from the path before calling + /// methods of this , false otherwise. + /// + public bool StripScheme { get; set; } + + + public FileSystemMetadataAttribute(string[] schemes) + { + Scheme = schemes; + } + } +} \ No newline at end of file diff --git a/Kyoo.Common/Models/Resources/Episode.cs b/Kyoo.Common/Models/Resources/Episode.cs index e7cf056a..7f76a8a4 100644 --- a/Kyoo.Common/Models/Resources/Episode.cs +++ b/Kyoo.Common/Models/Resources/Episode.cs @@ -95,7 +95,7 @@ namespace Kyoo.Models public int? AbsoluteNumber { get; set; } /// - /// The path of the video file for this episode. Any format supported by a is allowed. + /// The path of the video file for this episode. Any format supported by a is allowed. /// [SerializeIgnore] public string Path { get; set; } diff --git a/Kyoo.Common/Models/Resources/Show.cs b/Kyoo.Common/Models/Resources/Show.cs index 656d6ffb..c9d5ca8e 100644 --- a/Kyoo.Common/Models/Resources/Show.cs +++ b/Kyoo.Common/Models/Resources/Show.cs @@ -31,7 +31,7 @@ namespace Kyoo.Models /// /// The path of the root directory of this show. - /// This can be any kind of path supported by + /// This can be any kind of path supported by /// [SerializeIgnore] public string Path { get; set; } @@ -46,7 +46,7 @@ namespace Kyoo.Models public Status? Status { get; set; } /// - /// An URL to a trailer. This could be any path supported by the . + /// An URL to a trailer. This could be any path supported by the . /// /// TODO for now, this is set to a youtube url. It should be cached and converted to a local file. public string TrailerUrl { get; set; } diff --git a/Kyoo.Common/Models/WatchItem.cs b/Kyoo.Common/Models/WatchItem.cs index eaf437c7..241ed9a9 100644 --- a/Kyoo.Common/Models/WatchItem.cs +++ b/Kyoo.Common/Models/WatchItem.cs @@ -62,7 +62,7 @@ namespace Kyoo.Models public DateTime? ReleaseDate { get; set; } /// - /// The path of the video file for this episode. Any format supported by a is allowed. + /// The path of the video file for this episode. Any format supported by a is allowed. /// [SerializeIgnore] public string Path { get; set; } diff --git a/Kyoo.Common/Utility/Utility.cs b/Kyoo.Common/Utility/Utility.cs index 6b6dc472..2f7e14ad 100644 --- a/Kyoo.Common/Utility/Utility.cs +++ b/Kyoo.Common/Utility/Utility.cs @@ -254,7 +254,7 @@ namespace Kyoo /// /// To run for a List where you don't know the type at compile type, /// you could do: - /// + /// /// Utility.RunGenericMethod<object>( /// typeof(Utility), /// nameof(MergeLists), diff --git a/Kyoo.Tests/Library/RepositoryActivator.cs b/Kyoo.Tests/Database/RepositoryActivator.cs similarity index 100% rename from Kyoo.Tests/Library/RepositoryActivator.cs rename to Kyoo.Tests/Database/RepositoryActivator.cs diff --git a/Kyoo.Tests/Library/RepositoryTests.cs b/Kyoo.Tests/Database/RepositoryTests.cs similarity index 100% rename from Kyoo.Tests/Library/RepositoryTests.cs rename to Kyoo.Tests/Database/RepositoryTests.cs diff --git a/Kyoo.Tests/Library/SpecificTests/CollectionsTests.cs b/Kyoo.Tests/Database/SpecificTests/CollectionsTests.cs similarity index 100% rename from Kyoo.Tests/Library/SpecificTests/CollectionsTests.cs rename to Kyoo.Tests/Database/SpecificTests/CollectionsTests.cs diff --git a/Kyoo.Tests/Library/SpecificTests/EpisodeTests.cs b/Kyoo.Tests/Database/SpecificTests/EpisodeTests.cs similarity index 100% rename from Kyoo.Tests/Library/SpecificTests/EpisodeTests.cs rename to Kyoo.Tests/Database/SpecificTests/EpisodeTests.cs diff --git a/Kyoo.Tests/Library/SpecificTests/GenreTests.cs b/Kyoo.Tests/Database/SpecificTests/GenreTests.cs similarity index 100% rename from Kyoo.Tests/Library/SpecificTests/GenreTests.cs rename to Kyoo.Tests/Database/SpecificTests/GenreTests.cs diff --git a/Kyoo.Tests/Library/SpecificTests/LibraryItemTest.cs b/Kyoo.Tests/Database/SpecificTests/LibraryItemTest.cs similarity index 100% rename from Kyoo.Tests/Library/SpecificTests/LibraryItemTest.cs rename to Kyoo.Tests/Database/SpecificTests/LibraryItemTest.cs diff --git a/Kyoo.Tests/Library/SpecificTests/LibraryTests.cs b/Kyoo.Tests/Database/SpecificTests/LibraryTests.cs similarity index 100% rename from Kyoo.Tests/Library/SpecificTests/LibraryTests.cs rename to Kyoo.Tests/Database/SpecificTests/LibraryTests.cs diff --git a/Kyoo.Tests/Library/SpecificTests/PeopleTests.cs b/Kyoo.Tests/Database/SpecificTests/PeopleTests.cs similarity index 100% rename from Kyoo.Tests/Library/SpecificTests/PeopleTests.cs rename to Kyoo.Tests/Database/SpecificTests/PeopleTests.cs diff --git a/Kyoo.Tests/Library/SpecificTests/ProviderTests.cs b/Kyoo.Tests/Database/SpecificTests/ProviderTests.cs similarity index 100% rename from Kyoo.Tests/Library/SpecificTests/ProviderTests.cs rename to Kyoo.Tests/Database/SpecificTests/ProviderTests.cs diff --git a/Kyoo.Tests/Library/SpecificTests/SanityTests.cs b/Kyoo.Tests/Database/SpecificTests/SanityTests.cs similarity index 100% rename from Kyoo.Tests/Library/SpecificTests/SanityTests.cs rename to Kyoo.Tests/Database/SpecificTests/SanityTests.cs diff --git a/Kyoo.Tests/Library/SpecificTests/SeasonTests.cs b/Kyoo.Tests/Database/SpecificTests/SeasonTests.cs similarity index 100% rename from Kyoo.Tests/Library/SpecificTests/SeasonTests.cs rename to Kyoo.Tests/Database/SpecificTests/SeasonTests.cs diff --git a/Kyoo.Tests/Library/SpecificTests/ShowTests.cs b/Kyoo.Tests/Database/SpecificTests/ShowTests.cs similarity index 100% rename from Kyoo.Tests/Library/SpecificTests/ShowTests.cs rename to Kyoo.Tests/Database/SpecificTests/ShowTests.cs diff --git a/Kyoo.Tests/Library/SpecificTests/StudioTests.cs b/Kyoo.Tests/Database/SpecificTests/StudioTests.cs similarity index 100% rename from Kyoo.Tests/Library/SpecificTests/StudioTests.cs rename to Kyoo.Tests/Database/SpecificTests/StudioTests.cs diff --git a/Kyoo.Tests/Library/SpecificTests/TrackTests.cs b/Kyoo.Tests/Database/SpecificTests/TrackTests.cs similarity index 100% rename from Kyoo.Tests/Library/SpecificTests/TrackTests.cs rename to Kyoo.Tests/Database/SpecificTests/TrackTests.cs diff --git a/Kyoo.Tests/Library/SpecificTests/UserTests.cs b/Kyoo.Tests/Database/SpecificTests/UserTests.cs similarity index 100% rename from Kyoo.Tests/Library/SpecificTests/UserTests.cs rename to Kyoo.Tests/Database/SpecificTests/UserTests.cs diff --git a/Kyoo.Tests/Library/TestContext.cs b/Kyoo.Tests/Database/TestContext.cs similarity index 100% rename from Kyoo.Tests/Library/TestContext.cs rename to Kyoo.Tests/Database/TestContext.cs diff --git a/Kyoo.Tests/Library/TestSample.cs b/Kyoo.Tests/Database/TestSample.cs similarity index 100% rename from Kyoo.Tests/Library/TestSample.cs rename to Kyoo.Tests/Database/TestSample.cs diff --git a/Kyoo/Controllers/FileSystems/FileSystemComposite.cs b/Kyoo/Controllers/FileSystems/FileSystemComposite.cs new file mode 100644 index 00000000..f2f87a02 --- /dev/null +++ b/Kyoo/Controllers/FileSystems/FileSystemComposite.cs @@ -0,0 +1,143 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using Autofac.Features.Metadata; +using JetBrains.Annotations; +using Kyoo.Common.Models.Attributes; +using Kyoo.Models; +using Microsoft.AspNetCore.Mvc; + +namespace Kyoo.Controllers +{ + /// + /// A composite that merge every available + /// using . + /// + public class FileSystemComposite : IFileSystem + { + /// + /// The list of mapped to their metadata. + /// + private readonly ICollection, FileSystemMetadataAttribute>> _fileSystems; + + /// + /// Create a new from a list of mapped to their + /// metadata. + /// + /// The list of filesystem mapped to their metadata. + public FileSystemComposite(ICollection, FileSystemMetadataAttribute>> fileSystems) + { + _fileSystems = fileSystems; + } + + + /// + /// Retrieve the file system that should be used for a given path. + /// + /// + /// The path that was requested. + /// + /// + /// The path that the returned file system wants + /// (respecting ). + /// + /// No file system was registered for the given path. + /// The file system that should be used for a given path + [NotNull] + private IFileSystem _GetFileSystemForPath([NotNull] string path, [NotNull] out string usablePath) + { + Regex schemeMatcher = new(@"(.+)://(.*)", RegexOptions.Compiled); + Match match = schemeMatcher.Match(path); + + if (!match.Success) + { + usablePath = path; + Meta, FileSystemMetadataAttribute> defaultFs = _fileSystems + .SingleOrDefault(x => x.Metadata.Scheme.Contains("")); + if (defaultFs == null) + throw new ArgumentException($"No file system registered for the default scheme."); + return defaultFs.Value.Invoke(); + } + string scheme = match.Groups[1].Value; + Meta, FileSystemMetadataAttribute> ret = _fileSystems + .SingleOrDefault(x => x.Metadata.Scheme.Contains(scheme)); + if (ret == null) + throw new ArgumentException($"No file system registered for the scheme: {scheme}."); + usablePath = ret.Metadata.StripScheme ? match.Groups[2].Value : path; + return ret.Value.Invoke(); + } + + /// + public IActionResult FileResult(string path, bool rangeSupport = false, string type = null) + { + if (path == null) + return new NotFoundResult(); + return _GetFileSystemForPath(path, out string relativePath) + .FileResult(relativePath); + } + + /// + public Task GetReader(string path) + { + if (path == null) + throw new ArgumentNullException(nameof(path)); + return _GetFileSystemForPath(path, out string relativePath) + .GetReader(relativePath); + } + + /// + public Task NewFile(string path) + { + if (path == null) + throw new ArgumentNullException(nameof(path)); + return _GetFileSystemForPath(path, out string relativePath) + .NewFile(relativePath); + } + + /// + public Task CreateDirectory(string path) + { + if (path == null) + throw new ArgumentNullException(nameof(path)); + return _GetFileSystemForPath(path, out string relativePath) + .CreateDirectory(relativePath); + } + + /// + public string Combine(params string[] paths) + { + return _GetFileSystemForPath(paths[0], out string relativePath) + .Combine(paths[1..].Prepend(relativePath).ToArray()); + } + + /// + public Task> ListFiles(string path, SearchOption options = SearchOption.TopDirectoryOnly) + { + if (path == null) + throw new ArgumentNullException(nameof(path)); + return _GetFileSystemForPath(path, out string relativePath) + .ListFiles(relativePath, options); + } + + /// + public Task Exists(string path) + { + if (path == null) + throw new ArgumentNullException(nameof(path)); + return _GetFileSystemForPath(path, out string relativePath) + .Exists(relativePath); + } + + /// + public string GetExtraDirectory(Show show) + { + if (show == null) + throw new ArgumentNullException(nameof(show)); + return _GetFileSystemForPath(show.Path, out string _) + .GetExtraDirectory(show); + } + } +} \ No newline at end of file diff --git a/Kyoo/Controllers/FileSystems/HttpFileSystem.cs b/Kyoo/Controllers/FileSystems/HttpFileSystem.cs new file mode 100644 index 00000000..3d2ea991 --- /dev/null +++ b/Kyoo/Controllers/FileSystems/HttpFileSystem.cs @@ -0,0 +1,123 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Net.Http; +using System.Threading.Tasks; +using Kyoo.Common.Models.Attributes; +using Kyoo.Models; +using Microsoft.AspNetCore.Mvc; + +namespace Kyoo.Controllers +{ + /// + /// A for http/https links. + /// + [FileSystemMetadata(new [] {"http", "https"})] + public class HttpFileSystem : IFileSystem + { + /// + /// The http client factory used to create clients. + /// + private readonly IHttpClientFactory _clientFactory; + + /// + /// Create a using the given client factory. + /// + /// The http client factory used to create clients. + public HttpFileSystem(IHttpClientFactory factory) + { + _clientFactory = factory; + } + + + /// + public IActionResult FileResult(string path, bool rangeSupport = false, string type = null) + { + if (path == null) + return new NotFoundResult(); + return new HttpForwardResult(new Uri(path), rangeSupport, type); + } + + /// + public Task GetReader(string path) + { + HttpClient client = _clientFactory.CreateClient(); + return client.GetStreamAsync(path); + } + + /// + public Task NewFile(string path) + { + throw new NotSupportedException("An http filesystem is readonly, a new file can't be created."); + } + + /// + public Task CreateDirectory(string path) + { + throw new NotSupportedException("An http filesystem is readonly, a directory can't be created."); + } + + /// + public string Combine(params string[] paths) + { + return Path.Combine(paths); + } + + /// + public Task> ListFiles(string path, SearchOption options = SearchOption.TopDirectoryOnly) + { + throw new NotSupportedException("Listing files is not supported on an http filesystem."); + } + + /// + public Task Exists(string path) + { + throw new NotSupportedException("Checking if a file exists is not supported on an http filesystem."); + } + + /// + public string GetExtraDirectory(Show show) + { + throw new NotSupportedException("Extras can not be stored inside an http filesystem."); + } + } + + /// + /// An to proxy an http request. + /// + public class HttpForwardResult : IActionResult + { + /// + /// The path of the request to forward. + /// + private readonly Uri _path; + /// + /// Should the proxied result support ranges requests? + /// + private readonly bool _rangeSupport; + /// + /// If not null, override the content type of the resulting request. + /// + private readonly string _type; + + /// + /// Create a new . + /// + /// The path of the request to forward. + /// Should the proxied result support ranges requests? + /// If not null, override the content type of the resulting request. + public HttpForwardResult(Uri path, bool rangeSupport, string type = null) + { + _path = path; + _rangeSupport = rangeSupport; + _type = type; + } + + /// + public Task ExecuteResultAsync(ActionContext context) + { + // TODO implement that, example: https://github.com/twitchax/AspNetCore.Proxy/blob/14dd0f212d7abb43ca1bf8c890d5efb95db66acb/src/Core/Extensions/Http.cs#L15 + throw new NotImplementedException(); + } + } +} \ No newline at end of file diff --git a/Kyoo/Controllers/FileManager.cs b/Kyoo/Controllers/FileSystems/LocalFileSystem.cs similarity index 77% rename from Kyoo/Controllers/FileManager.cs rename to Kyoo/Controllers/FileSystems/LocalFileSystem.cs index 4aa89467..dc1c4b9d 100644 --- a/Kyoo/Controllers/FileManager.cs +++ b/Kyoo/Controllers/FileSystems/LocalFileSystem.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.IO; using System.Threading.Tasks; +using Kyoo.Common.Models.Attributes; using Kyoo.Models; using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.StaticFiles; @@ -9,9 +10,10 @@ using Microsoft.AspNetCore.StaticFiles; namespace Kyoo.Controllers { /// - /// A for the local filesystem (using System.IO). + /// A for the local filesystem (using System.IO). /// - public class FileManager : IFileManager + [FileSystemMetadata(new [] {"", "file"}, StripScheme = true)] + public class LocalFileSystem : IFileSystem { /// /// An extension provider to get content types from files extensions. @@ -54,19 +56,19 @@ namespace Kyoo.Controllers } /// - public Stream GetReader(string path) + public Task GetReader(string path) { if (path == null) throw new ArgumentNullException(nameof(path)); - return File.OpenRead(path); + return Task.FromResult(File.OpenRead(path)); } /// - public Stream NewFile(string path) + public Task NewFile(string path) { if (path == null) throw new ArgumentNullException(nameof(path)); - return File.Create(path); + return Task.FromResult(File.Create(path)); } /// @@ -108,24 +110,5 @@ namespace Kyoo.Controllers Directory.CreateDirectory(path); return path; } - - /// - public string GetExtraDirectory(Season season) - { - if (season.Show == null) - throw new NotImplementedException("Can't get season's extra directory when season.Show == null."); - // TODO use a season.Path here. - string path = Path.Combine(season.Show.Path, "Extra"); - Directory.CreateDirectory(path); - return path; - } - - /// - public string GetExtraDirectory(Episode episode) - { - string path = Path.Combine(Path.GetDirectoryName(episode.Path)!, "Extra"); - Directory.CreateDirectory(path); - return path; - } } } \ No newline at end of file diff --git a/Kyoo/Controllers/ThumbnailsManager.cs b/Kyoo/Controllers/ThumbnailsManager.cs index f76123ee..389891b3 100644 --- a/Kyoo/Controllers/ThumbnailsManager.cs +++ b/Kyoo/Controllers/ThumbnailsManager.cs @@ -17,7 +17,7 @@ namespace Kyoo.Controllers /// /// The file manager used to download the image if the file is distant /// - private readonly IFileManager _files; + private readonly IFileSystem _files; /// /// A logger to report errors. /// @@ -26,6 +26,10 @@ namespace Kyoo.Controllers /// The options containing the base path of people images and provider logos. /// private readonly IOptionsMonitor _options; + /// + /// A library manager used to load episode and seasons shows if they are not loaded. + /// + private readonly Lazy _library; /// /// Create a new . @@ -33,13 +37,16 @@ namespace Kyoo.Controllers /// The file manager to use. /// A logger to report errors /// The options to use. - public ThumbnailsManager(IFileManager files, + /// A library manager used to load shows if they are not loaded. + public ThumbnailsManager(IFileSystem files, ILogger logger, - IOptionsMonitor options) + IOptionsMonitor options, + Lazy library) { _files = files; _logger = logger; _options = options; + _library = library; options.OnChange(x => { @@ -66,7 +73,7 @@ namespace Kyoo.Controllers } /// - /// An helper function to download an image using a . + /// An helper function to download an image using a . /// /// The distant url of the image /// The local path of the image @@ -79,8 +86,8 @@ namespace Kyoo.Controllers try { - await using Stream reader = _files.GetReader(url); - await using Stream local = _files.NewFile(localPath); + await using Stream reader = await _files.GetReader(url); + await using Stream local = await _files.NewFile(localPath); await reader.CopyToAsync(local); return true; } @@ -185,7 +192,7 @@ namespace Kyoo.Controllers if (episode.Thumb == null) return false; - string localPath = await GetEpisodeThumb(episode); + string localPath = await _GetEpisodeThumb(episode); if (alwaysDownload || !await _files.Exists(localPath)) return await _DownloadImage(episode.Thumb, localPath, $"The thumbnail of {episode.Slug}"); return false; @@ -218,13 +225,25 @@ namespace Kyoo.Controllers { if (item == null) throw new ArgumentNullException(nameof(item)); - return Task.FromResult(item switch + return item switch { - Show show => _files.Combine(_files.GetExtraDirectory(show), "poster.jpg"), - Season season => _files.Combine(_files.GetExtraDirectory(season), $"season-{season.SeasonNumber}.jpg"), - People people => _files.Combine(_options.CurrentValue.PeoplePath, $"{people.Slug}.jpg"), + Show show => Task.FromResult(_files.Combine(_files.GetExtraDirectory(show), "poster.jpg")), + Season season => _GetSeasonPoster(season), + People actor => Task.FromResult(_files.Combine(_options.CurrentValue.PeoplePath, $"{actor.Slug}.jpg")), _ => throw new NotSupportedException($"The type {typeof(T).Name} does not have a poster.") - }); + }; + } + + /// + /// Retrieve the path of a season's poster. + /// + /// The season to retrieve the poster from. + /// The path of the season's poster. + private async Task _GetSeasonPoster(Season season) + { + if (season.Show == null) + await _library.Value.Load(season, x => x.Show); + return _files.Combine(_files.GetExtraDirectory(season.Show), $"season-{season.SeasonNumber}.jpg"); } /// @@ -236,14 +255,21 @@ namespace Kyoo.Controllers return item switch { Show show => Task.FromResult(_files.Combine(_files.GetExtraDirectory(show), "backdrop.jpg")), - Episode episode => GetEpisodeThumb(episode), + Episode episode => _GetEpisodeThumb(episode), _ => throw new NotSupportedException($"The type {typeof(T).Name} does not have a thumbnail.") }; } - private async Task GetEpisodeThumb(Episode episode) + /// + /// Get the path for an episode's thumbnail. + /// + /// The episode to retrieve the thumbnail from + /// The path of the given episode's thumbnail. + private async Task _GetEpisodeThumb(Episode episode) { - string dir = _files.Combine(_files.GetExtraDirectory(episode), "Thumbnails"); + if (episode.Show == null) + await _library.Value.Load(episode, x => x.Show); + string dir = _files.Combine(_files.GetExtraDirectory(episode.Show), "Thumbnails"); await _files.CreateDirectory(dir); return _files.Combine(dir, $"{Path.GetFileNameWithoutExtension(episode.Path)}.jpg"); } diff --git a/Kyoo/Controllers/Transcoder.cs b/Kyoo/Controllers/Transcoder.cs index 8f6e006e..47291753 100644 --- a/Kyoo/Controllers/Transcoder.cs +++ b/Kyoo/Controllers/Transcoder.cs @@ -72,24 +72,29 @@ namespace Kyoo.Controllers } } - private readonly IFileManager _files; + private readonly IFileSystem _files; private readonly IOptions _options; + private readonly Lazy _library; - public Transcoder(IFileManager files, IOptions options) + public Transcoder(IFileSystem files, IOptions options, Lazy library) { _files = files; _options = options; + _library = library; if (TranscoderAPI.init() != Marshal.SizeOf()) throw new BadTranscoderException(); } - public Task ExtractInfos(Episode episode, bool reextract) + public async Task ExtractInfos(Episode episode, bool reextract) { - string dir = _files.GetExtraDirectory(episode); + if (episode.Show == null) + await _library.Value.Load(episode, x => x.Show); + + string dir = _files.GetExtraDirectory(episode.Show); if (dir == null) throw new ArgumentException("Invalid path."); - return Task.Factory.StartNew( + return await Task.Factory.StartNew( () => TranscoderAPI.ExtractInfos(episode.Path, dir, reextract), TaskCreationOptions.LongRunning); } diff --git a/Kyoo/CoreModule.cs b/Kyoo/CoreModule.cs index 1632644f..2f5c7cfb 100644 --- a/Kyoo/CoreModule.cs +++ b/Kyoo/CoreModule.cs @@ -34,7 +34,7 @@ namespace Kyoo /// public ICollection Provides => new[] { - typeof(IFileManager), + typeof(IFileSystem), typeof(ITranscoder), typeof(IThumbnailsManager), typeof(IMetadataProvider), @@ -101,13 +101,17 @@ namespace Kyoo /// public void Configure(ContainerBuilder builder) { + builder.RegisterComposite(); + builder.RegisterType().As().SingleInstance(); + builder.RegisterType().As().SingleInstance(); + builder.RegisterType().As().SingleInstance(); - builder.RegisterType().As().SingleInstance(); builder.RegisterType().As().SingleInstance(); builder.RegisterType().As().SingleInstance(); builder.RegisterType().As().SingleInstance(); builder.RegisterType().As().InstancePerLifetimeScope(); builder.RegisterType().As().SingleInstance(); + builder.RegisterComposite(); builder.Register(x => (AProviderComposite)x.Resolve()); diff --git a/Kyoo/Kyoo.csproj b/Kyoo/Kyoo.csproj index b1bad964..205b8458 100644 --- a/Kyoo/Kyoo.csproj +++ b/Kyoo/Kyoo.csproj @@ -36,6 +36,7 @@ + diff --git a/Kyoo/Startup.cs b/Kyoo/Startup.cs index 04a24408..59a14f7d 100644 --- a/Kyoo/Startup.cs +++ b/Kyoo/Startup.cs @@ -1,6 +1,7 @@ using System; using System.IO; using Autofac; +using Autofac.Extras.AttributeMetadata; using Kyoo.Authentication; using Kyoo.Controllers; using Kyoo.Models.Options; @@ -76,6 +77,7 @@ namespace Kyoo public void ConfigureContainer(ContainerBuilder builder) { + builder.RegisterModule(); builder.RegisterInstance(_plugins).As().ExternallyOwned(); builder.RegisterTask(); _plugins.ConfigureContainer(builder); diff --git a/Kyoo/Tasks/Crawler.cs b/Kyoo/Tasks/Crawler.cs index 0d38e34e..43567bec 100644 --- a/Kyoo/Tasks/Crawler.cs +++ b/Kyoo/Tasks/Crawler.cs @@ -46,7 +46,7 @@ namespace Kyoo.Tasks /// /// The file manager used walk inside directories and check they existences. /// - [Injected] public IFileManager FileManager { private get; set; } + [Injected] public IFileSystem FileSystem { private get; set; } /// /// A task manager used to create sub tasks for each episode to add to the database. /// @@ -111,7 +111,7 @@ namespace Kyoo.Tasks Logger.LogInformation("Scanning library {Library} at {Paths}", library.Name, library.Paths); foreach (string path in library.Paths) { - ICollection files = await FileManager.ListFiles(path, SearchOption.AllDirectories); + ICollection files = await FileSystem.ListFiles(path, SearchOption.AllDirectories); if (cancellationToken.IsCancellationRequested) return; diff --git a/Kyoo/Tasks/Housekeeping.cs b/Kyoo/Tasks/Housekeeping.cs index 03084aaa..1d7789aa 100644 --- a/Kyoo/Tasks/Housekeeping.cs +++ b/Kyoo/Tasks/Housekeeping.cs @@ -39,7 +39,7 @@ namespace Kyoo.Tasks /// /// The file manager used walk inside directories and check they existences. /// - [Injected] public IFileManager FileManager { private get; set; } + [Injected] public IFileSystem FileSystem { private get; set; } /// /// The logger used to inform the user that episodes has been removed. /// @@ -58,7 +58,7 @@ namespace Kyoo.Tasks progress.Report(count / delCount * 100); count++; - if (await FileManager.Exists(show.Path)) + if (await FileSystem.Exists(show.Path)) continue; Logger.LogWarning("Show {Name}'s folder has been deleted (was {Path}), removing it from kyoo", show.Title, show.Path); @@ -70,7 +70,7 @@ namespace Kyoo.Tasks progress.Report(count / delCount * 100); count++; - if (await FileManager.Exists(episode.Path)) + if (await FileSystem.Exists(episode.Path)) continue; Logger.LogWarning("Episode {Slug}'s file has been deleted (was {Path}), removing it from kyoo", episode.Slug, episode.Path); diff --git a/Kyoo/Views/EpisodeApi.cs b/Kyoo/Views/EpisodeApi.cs index a0d92ff8..b7d83e27 100644 --- a/Kyoo/Views/EpisodeApi.cs +++ b/Kyoo/Views/EpisodeApi.cs @@ -21,11 +21,11 @@ namespace Kyoo.Api { private readonly ILibraryManager _libraryManager; private readonly IThumbnailsManager _thumbnails; - private readonly IFileManager _files; + private readonly IFileSystem _files; public EpisodeApi(ILibraryManager libraryManager, IOptions options, - IFileManager files, + IFileSystem files, IThumbnailsManager thumbnails) : base(libraryManager.EpisodeRepository, options.Value.PublicUrl) { diff --git a/Kyoo/Views/PeopleApi.cs b/Kyoo/Views/PeopleApi.cs index 6a468272..bb757a60 100644 --- a/Kyoo/Views/PeopleApi.cs +++ b/Kyoo/Views/PeopleApi.cs @@ -18,12 +18,12 @@ namespace Kyoo.Api public class PeopleApi : CrudApi { private readonly ILibraryManager _libraryManager; - private readonly IFileManager _files; + private readonly IFileSystem _files; private readonly IThumbnailsManager _thumbs; public PeopleApi(ILibraryManager libraryManager, IOptions options, - IFileManager files, + IFileSystem files, IThumbnailsManager thumbs) : base(libraryManager.PeopleRepository, options.Value.PublicUrl) { diff --git a/Kyoo/Views/ProviderApi.cs b/Kyoo/Views/ProviderApi.cs index ede70dec..026b79ef 100644 --- a/Kyoo/Views/ProviderApi.cs +++ b/Kyoo/Views/ProviderApi.cs @@ -17,11 +17,11 @@ namespace Kyoo.Api { private readonly IThumbnailsManager _thumbnails; private readonly ILibraryManager _libraryManager; - private readonly IFileManager _files; + private readonly IFileSystem _files; public ProviderApi(ILibraryManager libraryManager, IOptions options, - IFileManager files, + IFileSystem files, IThumbnailsManager thumbnails) : base(libraryManager.ProviderRepository, options.Value.PublicUrl) { diff --git a/Kyoo/Views/SeasonApi.cs b/Kyoo/Views/SeasonApi.cs index 85944c6b..8d08dd0c 100644 --- a/Kyoo/Views/SeasonApi.cs +++ b/Kyoo/Views/SeasonApi.cs @@ -20,12 +20,12 @@ namespace Kyoo.Api { private readonly ILibraryManager _libraryManager; private readonly IThumbnailsManager _thumbs; - private readonly IFileManager _files; + private readonly IFileSystem _files; public SeasonApi(ILibraryManager libraryManager, IOptions options, IThumbnailsManager thumbs, - IFileManager files) + IFileSystem files) : base(libraryManager.SeasonRepository, options.Value.PublicUrl) { _libraryManager = libraryManager; diff --git a/Kyoo/Views/ShowApi.cs b/Kyoo/Views/ShowApi.cs index 73822a9c..04854849 100644 --- a/Kyoo/Views/ShowApi.cs +++ b/Kyoo/Views/ShowApi.cs @@ -23,11 +23,11 @@ namespace Kyoo.Api public class ShowApi : CrudApi { private readonly ILibraryManager _libraryManager; - private readonly IFileManager _files; + private readonly IFileSystem _files; private readonly IThumbnailsManager _thumbs; public ShowApi(ILibraryManager libraryManager, - IFileManager files, + IFileSystem files, IThumbnailsManager thumbs, IOptions options) : base(libraryManager.ShowRepository, options.Value.PublicUrl) diff --git a/Kyoo/Views/SubtitleApi.cs b/Kyoo/Views/SubtitleApi.cs index 5426078d..89d036aa 100644 --- a/Kyoo/Views/SubtitleApi.cs +++ b/Kyoo/Views/SubtitleApi.cs @@ -14,9 +14,9 @@ namespace Kyoo.Api public class SubtitleApi : ControllerBase { private readonly ILibraryManager _libraryManager; - private readonly IFileManager _files; + private readonly IFileSystem _files; - public SubtitleApi(ILibraryManager libraryManager, IFileManager files) + public SubtitleApi(ILibraryManager libraryManager, IFileSystem files) { _libraryManager = libraryManager; _files = files; @@ -71,9 +71,9 @@ namespace Kyoo.Api public class ConvertSubripToVtt : IActionResult { private readonly string _path; - private readonly IFileManager _files; + private readonly IFileSystem _files; - public ConvertSubripToVtt(string subtitlePath, IFileManager files) + public ConvertSubripToVtt(string subtitlePath, IFileSystem files) { _path = subtitlePath; _files = files; @@ -92,7 +92,7 @@ namespace Kyoo.Api await writer.WriteLineAsync(""); await writer.WriteLineAsync(""); - using StreamReader reader = new(_files.GetReader(_path)); + using StreamReader reader = new(await _files.GetReader(_path)); string line; while ((line = await reader.ReadLineAsync()) != null) { diff --git a/Kyoo/Views/VideoApi.cs b/Kyoo/Views/VideoApi.cs index 5c48f7bc..cbc2937b 100644 --- a/Kyoo/Views/VideoApi.cs +++ b/Kyoo/Views/VideoApi.cs @@ -18,12 +18,12 @@ namespace Kyoo.Api private readonly ILibraryManager _libraryManager; private readonly ITranscoder _transcoder; private readonly IOptions _options; - private readonly IFileManager _files; + private readonly IFileSystem _files; public VideoApi(ILibraryManager libraryManager, ITranscoder transcoder, IOptions options, - IFileManager files) + IFileSystem files) { _libraryManager = libraryManager; _transcoder = transcoder; From 50307352a9e537000b41233ce418a81236da1141 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Wed, 21 Jul 2021 18:45:40 +0200 Subject: [PATCH 18/31] Fixing file system metadata registration --- .../Attributes/FileSystemMetadataAttribute.cs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Kyoo.Common/Models/Attributes/FileSystemMetadataAttribute.cs b/Kyoo.Common/Models/Attributes/FileSystemMetadataAttribute.cs index 464c44d0..d2b997a5 100644 --- a/Kyoo.Common/Models/Attributes/FileSystemMetadataAttribute.cs +++ b/Kyoo.Common/Models/Attributes/FileSystemMetadataAttribute.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.ComponentModel.Composition; using Kyoo.Controllers; @@ -27,9 +28,26 @@ namespace Kyoo.Common.Models.Attributes public bool StripScheme { get; set; } + /// + /// Create a new using the specified schemes. + /// + /// The schemes to use. public FileSystemMetadataAttribute(string[] schemes) { Scheme = schemes; } + + /// + /// Create a new using a dictionary of metadata. + /// + /// + /// The dictionary of metadata. This method expect the dictionary to contain a field + /// per property in this attribute, with the same types as the properties of this attribute. + /// + public FileSystemMetadataAttribute(IDictionary metadata) + { + Scheme = (string[])metadata[nameof(Scheme)]; + StripScheme = (bool)metadata[nameof(StripScheme)]; + } } } \ No newline at end of file From 42863bc4031eba6baab2098c10f6de454d28766c Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Wed, 21 Jul 2021 18:50:20 +0200 Subject: [PATCH 19/31] Fixing an error on invalid slug with the crud api --- Kyoo.Common/Controllers/IRepository.cs | 12 ++++++++++++ Kyoo.CommonAPI/CrudApi.cs | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Kyoo.Common/Controllers/IRepository.cs b/Kyoo.Common/Controllers/IRepository.cs index f7be69ad..a83d17fa 100644 --- a/Kyoo.Common/Controllers/IRepository.cs +++ b/Kyoo.Common/Controllers/IRepository.cs @@ -128,6 +128,7 @@ namespace Kyoo.Controllers /// The id of the resource /// If the item could not be found. /// The resource found + [ItemNotNull] Task Get(int id); /// /// Get a resource from it's slug. @@ -135,6 +136,7 @@ namespace Kyoo.Controllers /// The slug of the resource /// If the item could not be found. /// The resource found + [ItemNotNull] Task Get(string slug); /// /// Get the first resource that match the predicate. @@ -142,6 +144,7 @@ namespace Kyoo.Controllers /// A predicate to filter the resource. /// If the item could not be found. /// The resource found + [ItemNotNull] Task Get(Expression> where); /// @@ -149,18 +152,21 @@ namespace Kyoo.Controllers /// /// The id of the resource /// The resource found + [ItemCanBeNull] Task GetOrDefault(int id); /// /// Get a resource from it's slug or null if it is not found. /// /// The slug of the resource /// The resource found + [ItemCanBeNull] Task GetOrDefault(string slug); /// /// Get the first resource that match the predicate or null if it is not found. /// /// A predicate to filter the resource. /// The resource found + [ItemCanBeNull] Task GetOrDefault(Expression> where); /// @@ -168,6 +174,7 @@ namespace Kyoo.Controllers /// /// The query string. /// A list of resources found + [ItemNotNull] Task> Search(string query); /// @@ -177,6 +184,7 @@ namespace Kyoo.Controllers /// Sort information about the query (sort by, sort order) /// How pagination should be done (where to start and how many to return) /// A list of resources that match every filters + [ItemNotNull] Task> GetAll(Expression> where = null, Sort sort = default, Pagination limit = default); @@ -187,6 +195,7 @@ namespace Kyoo.Controllers /// A sort by predicate. The order is ascending. /// How pagination should be done (where to start and how many to return) /// A list of resources that match every filters + [ItemNotNull] Task> GetAll([Optional] Expression> where, Expression> sort, Pagination limit = default @@ -205,6 +214,7 @@ namespace Kyoo.Controllers /// /// The item to register /// The resource registers and completed by database's information (related items & so on) + [ItemNotNull] Task Create([NotNull] T obj); /// @@ -212,6 +222,7 @@ namespace Kyoo.Controllers /// /// The object to create /// The newly created item or the existing value if it existed. + [ItemNotNull] Task CreateIfNotExists([NotNull] T obj); /// @@ -221,6 +232,7 @@ namespace Kyoo.Controllers /// Should old properties of the resource be discarded or should null values considered as not changed? /// If the item is not found /// The resource edited and completed by database's information (related items & so on) + [ItemNotNull] Task Edit([NotNull] T edited, bool resetOld); /// diff --git a/Kyoo.CommonAPI/CrudApi.cs b/Kyoo.CommonAPI/CrudApi.cs index 336d3226..a6834c31 100644 --- a/Kyoo.CommonAPI/CrudApi.cs +++ b/Kyoo.CommonAPI/CrudApi.cs @@ -38,7 +38,7 @@ namespace Kyoo.CommonApi [PartialPermission(Kind.Read)] public virtual async Task> Get(string slug) { - T ret = await _repository.Get(slug); + T ret = await _repository.GetOrDefault(slug); if (ret == null) return NotFound(); return ret; From bb5dc0dcaeb22d9441b5b32d24afcb813c690a04 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Wed, 21 Jul 2021 19:01:31 +0200 Subject: [PATCH 20/31] Fixing the composite of IFileSystem.FileResult --- Kyoo/Controllers/FileSystems/FileSystemComposite.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Kyoo/Controllers/FileSystems/FileSystemComposite.cs b/Kyoo/Controllers/FileSystems/FileSystemComposite.cs index f2f87a02..da98539f 100644 --- a/Kyoo/Controllers/FileSystems/FileSystemComposite.cs +++ b/Kyoo/Controllers/FileSystems/FileSystemComposite.cs @@ -76,7 +76,7 @@ namespace Kyoo.Controllers if (path == null) return new NotFoundResult(); return _GetFileSystemForPath(path, out string relativePath) - .FileResult(relativePath); + .FileResult(relativePath, rangeSupport, type); } /// From fd37cede9d6cd58d24c5303e396975fdd143e48c Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Wed, 21 Jul 2021 19:41:25 +0200 Subject: [PATCH 21/31] Using immuable collections in the file extensions class --- Kyoo/Kyoo.csproj | 1 + Kyoo/Models/FileExtensions.cs | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Kyoo/Kyoo.csproj b/Kyoo/Kyoo.csproj index 205b8458..70fae8ca 100644 --- a/Kyoo/Kyoo.csproj +++ b/Kyoo/Kyoo.csproj @@ -32,6 +32,7 @@ + diff --git a/Kyoo/Models/FileExtensions.cs b/Kyoo/Models/FileExtensions.cs index b42b4be2..d023c039 100644 --- a/Kyoo/Models/FileExtensions.cs +++ b/Kyoo/Models/FileExtensions.cs @@ -1,6 +1,6 @@ using System.Collections.Generic; +using System.Collections.Immutable; using System.IO; -using System.Linq; namespace Kyoo.Models.Watch { @@ -12,7 +12,7 @@ namespace Kyoo.Models.Watch /// /// The list of known video extensions /// - public static readonly string[] VideoExtensions = + public static readonly ImmutableArray VideoExtensions = new() { ".webm", ".mkv", @@ -53,11 +53,11 @@ namespace Kyoo.Models.Watch /// /// The dictionary of known subtitles extensions and the name of the subtitle codec. /// - public static readonly Dictionary SubtitleExtensions = new() + public static readonly ImmutableDictionary SubtitleExtensions = new Dictionary { {".ass", "ass"}, {".str", "subrip"} - }; + }.ToImmutableDictionary(); /// /// Check if a file represent a subtitle file (only by checking the extension of the file) From 48e81dfd92cf5a69aa2a47ed470305da0c3feb46 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Thu, 22 Jul 2021 19:33:21 +0200 Subject: [PATCH 22/31] Using task metadata as an attribute --- Kyoo.Common/Controllers/IIdentifier.cs | 4 +- Kyoo.Common/Controllers/ITask.cs | 38 +--- Kyoo.Common/Controllers/ITaskManager.cs | 11 +- .../Models/Attributes/InjectedAttribute.cs | 4 +- .../Attributes/TaskMetadataAttribute.cs | 77 +++++++ ...ed.cs => IdentificationFailedException.cs} | 12 +- Kyoo.Postgresql/PostgresModule.cs | 2 +- .../FileSystems/LocalFileSystem.cs | 4 +- Kyoo/Controllers/PluginManager.cs | 1 + Kyoo/Controllers/RegexIdentifier.cs | 4 +- .../Repositories/EpisodeRepository.cs | 3 - .../Repositories/LibraryItemRepository.cs | 23 +- Kyoo/Controllers/TaskManager.cs | 203 ++++++++++++------ Kyoo/Models/FileExtensions.cs | 5 +- Kyoo/Tasks/Crawler.cs | 69 +++--- Kyoo/Tasks/Housekeeping.cs | 112 +++++----- Kyoo/Tasks/MetadataProviderLoader.cs | 64 +++--- Kyoo/Tasks/PluginInitializer.cs | 58 +++-- Kyoo/Tasks/RegisterEpisode.cs | 97 +++++---- Kyoo/Tasks/RegisterSubtitle.cs | 51 ++--- 20 files changed, 476 insertions(+), 366 deletions(-) create mode 100644 Kyoo.Common/Models/Attributes/TaskMetadataAttribute.cs rename Kyoo.Common/Models/Exceptions/{IdentificationFailed.cs => IdentificationFailedException.cs} (60%) diff --git a/Kyoo.Common/Controllers/IIdentifier.cs b/Kyoo.Common/Controllers/IIdentifier.cs index 5033bb77..11aeb65d 100644 --- a/Kyoo.Common/Controllers/IIdentifier.cs +++ b/Kyoo.Common/Controllers/IIdentifier.cs @@ -18,7 +18,7 @@ namespace Kyoo.Controllers /// /// The path of the episode file relative to the library root. It starts with a /. /// - /// The identifier could not work for the given path. + /// The identifier could not work for the given path. /// /// A tuple of models representing parsed metadata. /// If no metadata could be parsed for a type, null can be returned. @@ -34,7 +34,7 @@ namespace Kyoo.Controllers /// /// The path of the episode file relative to the library root. It starts with a /. /// - /// The identifier could not work for the given path. + /// The identifier could not work for the given path. /// /// The metadata of the track identified. /// diff --git a/Kyoo.Common/Controllers/ITask.cs b/Kyoo.Common/Controllers/ITask.cs index 1896bbcd..e5fbce29 100644 --- a/Kyoo.Common/Controllers/ITask.cs +++ b/Kyoo.Common/Controllers/ITask.cs @@ -114,6 +114,8 @@ namespace Kyoo.Controllers /// The value of this parameter. public T As() { + if (typeof(T) == typeof(object)) + return (T)Value; return (T)Convert.ChangeType(Value, typeof(T)); } } @@ -150,42 +152,6 @@ namespace Kyoo.Controllers /// public interface ITask { - /// - /// The slug of the task, used to start it. - /// - public string Slug { get; } - - /// - /// The name of the task that will be displayed to the user. - /// - public string Name { get; } - - /// - /// A quick description of what this task will do. - /// - public string Description { get; } - - /// - /// An optional message to display to help the user. - /// - public string HelpMessage { get; } - - /// - /// Should this task be automatically run at app startup? - /// - public bool RunOnStartup { get; } - - /// - /// The priority of this task. Only used if is true. - /// It allow one to specify witch task will be started first as tasked are run on a Priority's descending order. - /// - public int Priority { get; } - - /// - /// true if this task should not be displayed to the user, false otherwise. - /// - public bool IsHidden { get; } - /// /// The list of parameters /// diff --git a/Kyoo.Common/Controllers/ITaskManager.cs b/Kyoo.Common/Controllers/ITaskManager.cs index ffcb1c75..37b40f12 100644 --- a/Kyoo.Common/Controllers/ITaskManager.cs +++ b/Kyoo.Common/Controllers/ITaskManager.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Threading; +using Kyoo.Common.Models.Attributes; using Kyoo.Models.Exceptions; namespace Kyoo.Controllers @@ -38,7 +39,7 @@ namespace Kyoo.Controllers [NotNull] IProgress progress, Dictionary arguments = null, CancellationToken? cancellationToken = null); - + /// /// Start a new task (or queue it). /// @@ -61,21 +62,21 @@ namespace Kyoo.Controllers /// /// The task could not be found. /// - void StartTask([NotNull] IProgress progress, + void StartTask([NotNull] IProgress progress, Dictionary arguments = null, CancellationToken? cancellationToken = null) - where T : ITask, new(); + where T : ITask; /// /// Get all currently running tasks /// /// A list of currently running tasks. - ICollection GetRunningTasks(); + ICollection<(TaskMetadataAttribute, ITask)> GetRunningTasks(); /// /// Get all available tasks /// /// A list of every tasks that this instance know. - ICollection GetAllTasks(); + ICollection GetAllTasks(); } } \ No newline at end of file diff --git a/Kyoo.Common/Models/Attributes/InjectedAttribute.cs b/Kyoo.Common/Models/Attributes/InjectedAttribute.cs index b036acdf..d517300f 100644 --- a/Kyoo.Common/Models/Attributes/InjectedAttribute.cs +++ b/Kyoo.Common/Models/Attributes/InjectedAttribute.cs @@ -8,8 +8,8 @@ namespace Kyoo.Models.Attributes /// An attribute to inform that the service will be injected automatically by a service provider. /// /// - /// It should only be used on and will be injected before calling . - /// It can also be used on and it will be injected before calling . + /// It should only be used on and it will be injected before + /// calling . /// [AttributeUsage(AttributeTargets.Property)] [MeansImplicitUse(ImplicitUseKindFlags.Assign)] diff --git a/Kyoo.Common/Models/Attributes/TaskMetadataAttribute.cs b/Kyoo.Common/Models/Attributes/TaskMetadataAttribute.cs new file mode 100644 index 00000000..0ebe7a34 --- /dev/null +++ b/Kyoo.Common/Models/Attributes/TaskMetadataAttribute.cs @@ -0,0 +1,77 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel.Composition; +using Kyoo.Controllers; + +namespace Kyoo.Common.Models.Attributes +{ + /// + /// An attribute to inform how a works. + /// + [MetadataAttribute] + [AttributeUsage(AttributeTargets.Class)] + public class TaskMetadataAttribute : Attribute + { + /// + /// The slug of the task, used to start it. + /// + public string Slug { get; } + + /// + /// The name of the task that will be displayed to the user. + /// + public string Name { get; } + + /// + /// A quick description of what this task will do. + /// + public string Description { get; } + + /// + /// Should this task be automatically run at app startup? + /// + public bool RunOnStartup { get; set; } + + /// + /// The priority of this task. Only used if is true. + /// It allow one to specify witch task will be started first as tasked are run on a Priority's descending order. + /// + public int Priority { get; set; } + + /// + /// true if this task should not be displayed to the user, false otherwise. + /// + public bool IsHidden { get; set; } + + + /// + /// Create a new with the given slug, name and description. + /// + /// The slug of the task, used to start it. + /// The name of the task that will be displayed to the user. + /// A quick description of what this task will do. + public TaskMetadataAttribute(string slug, string name, string description) + { + Slug = slug; + Name = name; + Description = description; + } + + /// + /// Create a new using a dictionary of metadata. + /// + /// + /// The dictionary of metadata. This method expect the dictionary to contain a field + /// per property in this attribute, with the same types as the properties of this attribute. + /// + public TaskMetadataAttribute(IDictionary metadata) + { + Slug = (string)metadata[nameof(Slug)]; + Name = (string)metadata[nameof(Name)]; + Description = (string)metadata[nameof(Description)]; + RunOnStartup = (bool)metadata[nameof(RunOnStartup)]; + Priority = (int)metadata[nameof(Priority)]; + IsHidden = (bool)metadata[nameof(IsHidden)]; + } + } +} \ No newline at end of file diff --git a/Kyoo.Common/Models/Exceptions/IdentificationFailed.cs b/Kyoo.Common/Models/Exceptions/IdentificationFailedException.cs similarity index 60% rename from Kyoo.Common/Models/Exceptions/IdentificationFailed.cs rename to Kyoo.Common/Models/Exceptions/IdentificationFailedException.cs index a8838fea..7c820f5a 100644 --- a/Kyoo.Common/Models/Exceptions/IdentificationFailed.cs +++ b/Kyoo.Common/Models/Exceptions/IdentificationFailedException.cs @@ -8,20 +8,20 @@ namespace Kyoo.Models.Exceptions /// An exception raised when an failed. /// [Serializable] - public class IdentificationFailed : Exception + public class IdentificationFailedException : Exception { /// - /// Create a new with a default message. + /// Create a new with a default message. /// - public IdentificationFailed() + public IdentificationFailedException() : base("An identification failed.") {} /// - /// Create a new with a custom message. + /// Create a new with a custom message. /// /// The message to use. - public IdentificationFailed(string message) + public IdentificationFailedException(string message) : base(message) {} @@ -30,7 +30,7 @@ namespace Kyoo.Models.Exceptions /// /// Serialization infos /// The serialization context - protected IdentificationFailed(SerializationInfo info, StreamingContext context) + protected IdentificationFailedException(SerializationInfo info, StreamingContext context) : base(info, context) { } } diff --git a/Kyoo.Postgresql/PostgresModule.cs b/Kyoo.Postgresql/PostgresModule.cs index 124df770..70d62f74 100644 --- a/Kyoo.Postgresql/PostgresModule.cs +++ b/Kyoo.Postgresql/PostgresModule.cs @@ -75,7 +75,7 @@ namespace Kyoo.Postgresql DatabaseContext context = provider.GetRequiredService(); context.Database.Migrate(); - using NpgsqlConnection conn = (NpgsqlConnection)context.Database.GetDbConnection(); + NpgsqlConnection conn = (NpgsqlConnection)context.Database.GetDbConnection(); conn.Open(); conn.ReloadTypes(); } diff --git a/Kyoo/Controllers/FileSystems/LocalFileSystem.cs b/Kyoo/Controllers/FileSystems/LocalFileSystem.cs index dc1c4b9d..3239c55f 100644 --- a/Kyoo/Controllers/FileSystems/LocalFileSystem.cs +++ b/Kyoo/Controllers/FileSystems/LocalFileSystem.cs @@ -43,7 +43,7 @@ namespace Kyoo.Controllers } /// - public IActionResult FileResult(string path, bool range = false, string type = null) + public IActionResult FileResult(string path, bool rangeSupport = false, string type = null) { if (path == null) return new NotFoundResult(); @@ -51,7 +51,7 @@ namespace Kyoo.Controllers return new NotFoundResult(); return new PhysicalFileResult(Path.GetFullPath(path), type ?? _GetContentType(path)) { - EnableRangeProcessing = range + EnableRangeProcessing = rangeSupport }; } diff --git a/Kyoo/Controllers/PluginManager.cs b/Kyoo/Controllers/PluginManager.cs index 230480c6..46185f16 100644 --- a/Kyoo/Controllers/PluginManager.cs +++ b/Kyoo/Controllers/PluginManager.cs @@ -158,6 +158,7 @@ namespace Kyoo.Controllers using IServiceScope scope = _provider.CreateScope(); Helper.InjectServices(plugin, x => scope.ServiceProvider.GetRequiredService(x)); plugin.ConfigureAspNet(app); + Helper.InjectServices(plugin, _ => null); } } diff --git a/Kyoo/Controllers/RegexIdentifier.cs b/Kyoo/Controllers/RegexIdentifier.cs index f578aa30..3c09ae4a 100644 --- a/Kyoo/Controllers/RegexIdentifier.cs +++ b/Kyoo/Controllers/RegexIdentifier.cs @@ -36,7 +36,7 @@ namespace Kyoo.Controllers Match match = regex.Match(relativePath); if (!match.Success) - throw new IdentificationFailed($"The episode at {path} does not match the episode's regex."); + throw new IdentificationFailedException($"The episode at {path} does not match the episode's regex."); (Collection collection, Show show, Season season, Episode episode) ret = ( collection: new Collection @@ -90,7 +90,7 @@ namespace Kyoo.Controllers Match match = regex.Match(path); if (!match.Success) - throw new IdentificationFailed($"The subtitle at {path} does not match the subtitle's regex."); + throw new IdentificationFailedException($"The subtitle at {path} does not match the subtitle's regex."); string episodePath = match.Groups["Episode"].Value; return Task.FromResult(new Track diff --git a/Kyoo/Controllers/Repositories/EpisodeRepository.cs b/Kyoo/Controllers/Repositories/EpisodeRepository.cs index 2cf7ff1f..3c1393eb 100644 --- a/Kyoo/Controllers/Repositories/EpisodeRepository.cs +++ b/Kyoo/Controllers/Repositories/EpisodeRepository.cs @@ -114,9 +114,6 @@ namespace Kyoo.Controllers obj.ExternalIDs.ForEach(x => _database.Entry(x).State = EntityState.Added); await _database.SaveChangesAsync($"Trying to insert a duplicated episode (slug {obj.Slug} already exists)."); return await ValidateTracks(obj); - // TODO check if this is needed - // obj.Slug = await _database.Entry(obj).Property(x => x.Slug). - // return obj; } /// diff --git a/Kyoo/Controllers/Repositories/LibraryItemRepository.cs b/Kyoo/Controllers/Repositories/LibraryItemRepository.cs index 37391c5d..b0ec916e 100644 --- a/Kyoo/Controllers/Repositories/LibraryItemRepository.cs +++ b/Kyoo/Controllers/Repositories/LibraryItemRepository.cs @@ -81,17 +81,28 @@ namespace Kyoo.Controllers } /// - public override Task Create(LibraryItem obj) => throw new InvalidOperationException(); + public override Task Create(LibraryItem obj) + => throw new InvalidOperationException(); + /// - public override Task CreateIfNotExists(LibraryItem obj) => throw new InvalidOperationException(); + public override Task CreateIfNotExists(LibraryItem obj) + => throw new InvalidOperationException(); + /// - public override Task Edit(LibraryItem obj, bool reset) => throw new InvalidOperationException(); + public override Task Edit(LibraryItem obj, bool resetOld) + => throw new InvalidOperationException(); + /// - public override Task Delete(int id) => throw new InvalidOperationException(); + public override Task Delete(int id) + => throw new InvalidOperationException(); + /// - public override Task Delete(string slug) => throw new InvalidOperationException(); + public override Task Delete(string slug) + => throw new InvalidOperationException(); + /// - public override Task Delete(LibraryItem obj) => throw new InvalidOperationException(); + public override Task Delete(LibraryItem obj) + => throw new InvalidOperationException(); /// /// Get a basic queryable for a library with the right mapping from shows & collections. diff --git a/Kyoo/Controllers/TaskManager.cs b/Kyoo/Controllers/TaskManager.cs index 65d623dc..54d724f0 100644 --- a/Kyoo/Controllers/TaskManager.cs +++ b/Kyoo/Controllers/TaskManager.cs @@ -1,12 +1,15 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Reflection; using System.Threading; using System.Threading.Tasks; +using Autofac.Features.Metadata; +using Autofac.Features.OwnedInstances; using JetBrains.Annotations; +using Kyoo.Common.Models.Attributes; using Kyoo.Models.Exceptions; using Kyoo.Models.Options; -using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -20,9 +23,52 @@ namespace Kyoo.Controllers public class TaskManager : BackgroundService, ITaskManager { /// - /// The service provider used to activate + /// The class representing task under this jurisdiction. /// - private readonly IServiceProvider _provider; + private class ManagedTask + { + /// + /// The metadata for this task (the slug, and other useful information). + /// + public TaskMetadataAttribute Metadata { get; set; } + + /// + /// The function used to create the task object. + /// + public Func> Factory { get; init; } + + /// + /// The next scheduled date for this task + /// + public DateTime ScheduledDate { get; set; } + } + + /// + /// A class representing a task inside the list. + /// + private class QueuedTask + { + /// + /// The task currently queued. + /// + public ManagedTask Task { get; init; } + + /// + /// The progress reporter that this task should use. + /// + public IProgress ProgressReporter { get; init; } + + /// + /// The arguments to give to run the task with. + /// + public Dictionary Arguments { get; init; } + + /// + /// A token informing the task that it should be cancelled or not. + /// + public CancellationToken? CancellationToken { get; init; } + } + /// /// The configuration instance used to get schedule information /// @@ -35,15 +81,15 @@ namespace Kyoo.Controllers /// /// The list of tasks and their next scheduled run. /// - private readonly List<(ITask task, DateTime scheduledDate)> _tasks; + private readonly List _tasks; /// /// The queue of tasks that should be run as soon as possible. /// - private readonly Queue<(ITask, IProgress, Dictionary)> _queuedTasks = new(); + private readonly Queue _queuedTasks = new(); /// /// The currently running task. /// - private ITask _runningTask; + private (TaskMetadataAttribute, ITask)? _runningTask; /// /// The cancellation token used to cancel the running task when the runner should shutdown. /// @@ -53,22 +99,24 @@ namespace Kyoo.Controllers /// /// Create a new . /// - /// The list of tasks to manage - /// The service provider to request services for tasks + /// The list of tasks to manage with their metadata /// The configuration to load schedule information. /// The logger. - public TaskManager(IEnumerable tasks, - IServiceProvider provider, + public TaskManager(IEnumerable>, TaskMetadataAttribute>> tasks, IOptionsMonitor options, ILogger logger) { - _provider = provider; _options = options; _logger = logger; - _tasks = tasks.Select(x => (x, GetNextTaskDate(x.Slug))).ToList(); + _tasks = tasks.Select(x => new ManagedTask + { + Factory = x.Value, + Metadata = x.Metadata, + ScheduledDate = GetNextTaskDate(x.Metadata.Slug) + }).ToList(); if (_tasks.Any()) - _logger.LogTrace("Task manager initiated with: {Tasks}", _tasks.Select(x => x.task.Name)); + _logger.LogTrace("Task manager initiated with: {Tasks}", _tasks.Select(x => x.Metadata.Name)); else _logger.LogInformation("Task manager initiated without any tasks"); } @@ -98,25 +146,26 @@ namespace Kyoo.Controllers /// A token to stop the runner protected override async Task ExecuteAsync(CancellationToken cancellationToken) { - EnqueueStartupTasks(); + _EnqueueStartupTasks(); while (!cancellationToken.IsCancellationRequested) { if (_queuedTasks.Any()) { - (ITask task, IProgress progress, Dictionary args) = _queuedTasks.Dequeue(); - _runningTask = task; + QueuedTask task = _queuedTasks.Dequeue(); try { - await RunTask(task, progress, args); + await _RunTask(task.Task, task.ProgressReporter, task.Arguments, task.CancellationToken); } catch (TaskFailedException ex) { - _logger.LogWarning("The task \"{Task}\" failed: {Message}", task.Name, ex.Message); + _logger.LogWarning("The task \"{Task}\" failed: {Message}", + task.Task.Metadata.Name, ex.Message); } catch (Exception e) { - _logger.LogError(e, "An unhandled exception occured while running the task {Task}", task.Name); + _logger.LogError(e, "An unhandled exception occured while running the task {Task}", + task.Task.Metadata.Name); } } else @@ -133,44 +182,54 @@ namespace Kyoo.Controllers /// The task to run /// A progress reporter to know the percentage of completion of the task. /// The arguments to pass to the function + /// An optional cancellation token that will be passed to the task. /// /// If the number of arguments is invalid, if an argument can't be converted or if the task finds the argument /// invalid. /// - private async Task RunTask(ITask task, + private async Task _RunTask(ManagedTask task, [NotNull] IProgress progress, - Dictionary arguments) + Dictionary arguments, + CancellationToken? cancellationToken = null) { - _logger.LogInformation("Task starting: {Task}", task.Name); - - ICollection all = task.GetParameters(); - - ICollection invalids = arguments.Keys - .Where(x => all.All(y => x != y.Name)) - .ToArray(); - if (invalids.Any()) + using (_logger.BeginScope("Task: {Task}", task.Metadata.Name)) { - string invalidsStr = string.Join(", ", invalids); - throw new ArgumentException($"{invalidsStr} are invalid arguments for the task {task.Name}"); - } - - TaskParameters args = new(all - .Select(x => - { - object value = arguments - .FirstOrDefault(y => string.Equals(y.Key, x.Name, StringComparison.OrdinalIgnoreCase)) - .Value; - if (value == null && x.IsRequired) - throw new ArgumentException($"The argument {x.Name} is required to run {task.Name}" + - " but it was not specified."); - return x.CreateValue(value ?? x.DefaultValue); - })); + await using Owned taskObj = task.Factory.Invoke(); + ICollection all = taskObj.Value.GetParameters(); - using IServiceScope scope = _provider.CreateScope(); - Helper.InjectServices(task, x => scope.ServiceProvider.GetRequiredService(x)); - await task.Run(args, progress, _taskToken.Token); - Helper.InjectServices(task, _ => null); - _logger.LogInformation("Task finished: {Task}", task.Name); + _runningTask = (task.Metadata, taskObj.Value); + ICollection invalids = arguments.Keys + .Where(x => all.All(y => x != y.Name)) + .ToArray(); + if (invalids.Any()) + { + throw new ArgumentException($"{string.Join(", ", invalids)} are " + + $"invalid arguments for the task {task.Metadata.Name}"); + } + + TaskParameters args = new(all + .Select(x => + { + object value = arguments + .FirstOrDefault(y => string.Equals(y.Key, x.Name, StringComparison.OrdinalIgnoreCase)) + .Value; + if (value == null && x.IsRequired) + throw new ArgumentException($"The argument {x.Name} is required to run " + + $"{task.Metadata.Name} but it was not specified."); + return x.CreateValue(value ?? x.DefaultValue); + })); + + _logger.LogInformation("Task starting: {Task} ({Parameters})", + task.Metadata.Name, args.ToDictionary(x => x.Name, x => x.As())); + + CancellationToken token = cancellationToken != null + ? CancellationTokenSource.CreateLinkedTokenSource(_taskToken.Token, cancellationToken.Value).Token + : _taskToken.Token; + await taskObj.Value.Run(args, progress, token); + + _logger.LogInformation("Task finished: {Task}", task.Metadata.Name); + _runningTask = null; + } } /// @@ -178,8 +237,8 @@ namespace Kyoo.Controllers /// private void QueueScheduledTasks() { - IEnumerable tasksToQueue = _tasks.Where(x => x.scheduledDate <= DateTime.Now) - .Select(x => x.task.Slug); + IEnumerable tasksToQueue = _tasks.Where(x => x.ScheduledDate <= DateTime.Now) + .Select(x => x.Metadata.Slug); foreach (string task in tasksToQueue) { _logger.LogDebug("Queuing task scheduled for running: {Task}", task); @@ -190,13 +249,14 @@ namespace Kyoo.Controllers /// /// Queue startup tasks with respect to the priority rules. /// - private void EnqueueStartupTasks() + private void _EnqueueStartupTasks() { - IEnumerable startupTasks = _tasks.Select(x => x.task) - .Where(x => x.RunOnStartup) - .OrderByDescending(x => x.Priority); - foreach (ITask task in startupTasks) - _queuedTasks.Enqueue((task, new Progress(), new Dictionary())); + IEnumerable startupTasks = _tasks + .Where(x => x.Metadata.RunOnStartup) + .OrderByDescending(x => x.Metadata.Priority) + .Select(x => x.Metadata.Slug); + foreach (string task in startupTasks) + StartTask(task, new Progress(), new Dictionary()); } /// @@ -207,20 +267,29 @@ namespace Kyoo.Controllers { arguments ??= new Dictionary(); - int index = _tasks.FindIndex(x => x.task.Slug == taskSlug); + int index = _tasks.FindIndex(x => x.Metadata.Slug == taskSlug); if (index == -1) throw new ItemNotFoundException($"No task found with the slug {taskSlug}"); - _queuedTasks.Enqueue((_tasks[index].task, progress, arguments)); - _tasks[index] = (_tasks[index].task, GetNextTaskDate(taskSlug)); + _queuedTasks.Enqueue(new QueuedTask + { + Task = _tasks[index], + ProgressReporter = progress, + Arguments = arguments, + CancellationToken = cancellationToken + }); + _tasks[index].ScheduledDate = GetNextTaskDate(taskSlug); } /// public void StartTask(IProgress progress, Dictionary arguments = null, CancellationToken? cancellationToken = null) - where T : ITask, new() + where T : ITask { - StartTask(new T().Slug, progress, arguments, cancellationToken); + TaskMetadataAttribute metadata = typeof(T).GetCustomAttribute(); + if (metadata == null) + throw new ArgumentException($"No metadata found on the given task (type: {typeof(T).Name})."); + StartTask(metadata.Slug, progress, arguments, cancellationToken); } /// @@ -236,15 +305,17 @@ namespace Kyoo.Controllers } /// - public ICollection GetRunningTasks() + public ICollection<(TaskMetadataAttribute, ITask)> GetRunningTasks() { - return new[] {_runningTask}; + return _runningTask == null + ? ArraySegment<(TaskMetadataAttribute, ITask)>.Empty + : new[] { _runningTask.Value }; } /// - public ICollection GetAllTasks() + public ICollection GetAllTasks() { - return _tasks.Select(x => x.task).ToArray(); + return _tasks.Select(x => x.Metadata).ToArray(); } } } \ No newline at end of file diff --git a/Kyoo/Models/FileExtensions.cs b/Kyoo/Models/FileExtensions.cs index d023c039..5b29fe08 100644 --- a/Kyoo/Models/FileExtensions.cs +++ b/Kyoo/Models/FileExtensions.cs @@ -12,8 +12,7 @@ namespace Kyoo.Models.Watch /// /// The list of known video extensions /// - public static readonly ImmutableArray VideoExtensions = new() - { + public static readonly ImmutableArray VideoExtensions = ImmutableArray.Create( ".webm", ".mkv", ".flv", @@ -38,7 +37,7 @@ namespace Kyoo.Models.Watch ".m2v", ".3gp", ".3g2" - }; + ); /// /// Check if a file represent a video file (only by checking the extension of the file) diff --git a/Kyoo/Tasks/Crawler.cs b/Kyoo/Tasks/Crawler.cs index 43567bec..191fa27d 100644 --- a/Kyoo/Tasks/Crawler.cs +++ b/Kyoo/Tasks/Crawler.cs @@ -5,8 +5,8 @@ using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; +using Kyoo.Common.Models.Attributes; using Kyoo.Controllers; -using Kyoo.Models.Attributes; using Kyoo.Models.Watch; using Microsoft.Extensions.Logging; @@ -15,46 +15,43 @@ namespace Kyoo.Tasks /// /// A task to add new video files. /// + [TaskMetadata("scan", "Scan libraries", "Scan your libraries and load data for new shows.", RunOnStartup = true)] public class Crawler : ITask { - /// - public string Slug => "scan"; - - /// - public string Name => "Scan libraries"; - - /// - public string Description => "Scan your libraries and load data for new shows."; - - /// - public string HelpMessage => "Reloading all libraries is a long process and may take up to" + - " 24 hours if it is the first scan in a while."; - - /// - public bool RunOnStartup => true; - - /// - public int Priority => 0; - - /// - public bool IsHidden => false; - /// /// The library manager used to get libraries and providers to use. /// - [Injected] public ILibraryManager LibraryManager { private get; set; } + private readonly ILibraryManager _libraryManager; /// /// The file manager used walk inside directories and check they existences. /// - [Injected] public IFileSystem FileSystem { private get; set; } + private readonly IFileSystem _fileSystem; /// /// A task manager used to create sub tasks for each episode to add to the database. /// - [Injected] public ITaskManager TaskManager { private get; set; } + private readonly ITaskManager _taskManager; /// /// The logger used to inform the current status to the console. /// - [Injected] public ILogger Logger { private get; set; } + private readonly ILogger _logger; + + /// + /// Create a new . + /// + /// The library manager to retrieve existing episodes/library/tracks + /// The file system to glob files + /// The task manager used to start . + /// The logger used print messages. + public Crawler(ILibraryManager libraryManager, + IFileSystem fileSystem, + ITaskManager taskManager, + ILogger logger) + { + _libraryManager = libraryManager; + _fileSystem = fileSystem; + _taskManager = taskManager; + _logger = logger; + } /// @@ -71,20 +68,20 @@ namespace Kyoo.Tasks { string argument = arguments["slug"].As(); ICollection libraries = argument == null - ? await LibraryManager.GetAll() - : new [] { await LibraryManager.GetOrDefault(argument)}; + ? await _libraryManager.GetAll() + : new [] { await _libraryManager.GetOrDefault(argument)}; if (argument != null && libraries.First() == null) throw new ArgumentException($"No library found with the name {argument}"); foreach (Library library in libraries) - await LibraryManager.Load(library, x => x.Providers); + await _libraryManager.Load(library, x => x.Providers); progress.Report(0); float percent = 0; - ICollection episodes = await LibraryManager.GetAll(); - ICollection tracks = await LibraryManager.GetAll(); + ICollection episodes = await _libraryManager.GetAll(); + ICollection tracks = await _libraryManager.GetAll(); foreach (Library library in libraries) { IProgress reporter = new Progress(x => @@ -108,10 +105,10 @@ namespace Kyoo.Tasks IProgress progress, CancellationToken cancellationToken) { - Logger.LogInformation("Scanning library {Library} at {Paths}", library.Name, library.Paths); + _logger.LogInformation("Scanning library {Library} at {Paths}", library.Name, library.Paths); foreach (string path in library.Paths) { - ICollection files = await FileSystem.ListFiles(path, SearchOption.AllDirectories); + ICollection files = await _fileSystem.ListFiles(path, SearchOption.AllDirectories); if (cancellationToken.IsCancellationRequested) return; @@ -138,7 +135,7 @@ namespace Kyoo.Tasks foreach (string episodePath in paths) { - TaskManager.StartTask(reporter, new Dictionary + _taskManager.StartTask(reporter, new Dictionary { ["path"] = episodePath, ["relativePath"] = episodePath[path.Length..], @@ -162,7 +159,7 @@ namespace Kyoo.Tasks foreach (string trackPath in subtitles) { - TaskManager.StartTask(reporter, new Dictionary + _taskManager.StartTask(reporter, new Dictionary { ["path"] = trackPath, ["relativePath"] = trackPath[path.Length..] diff --git a/Kyoo/Tasks/Housekeeping.cs b/Kyoo/Tasks/Housekeeping.cs index 1d7789aa..b98989f5 100644 --- a/Kyoo/Tasks/Housekeeping.cs +++ b/Kyoo/Tasks/Housekeeping.cs @@ -1,83 +1,43 @@ using System; using System.Threading; using System.Threading.Tasks; +using Kyoo.Common.Models.Attributes; using Kyoo.Controllers; using Kyoo.Models; -using Kyoo.Models.Attributes; using Microsoft.Extensions.Logging; namespace Kyoo.Tasks { + /// + /// A task to remove orphaned episode and series. + /// + [TaskMetadata("housekeeping", "Housekeeping", "Remove orphaned episode and series.", RunOnStartup = true)] public class Housekeeping : ITask { - /// - public string Slug => "housekeeping"; - - /// - public string Name => "Housekeeping"; - - /// - public string Description => "Remove orphaned episode and series."; - - /// - public string HelpMessage => null; - - /// - public bool RunOnStartup => true; - - /// - public int Priority => 0; - - /// - public bool IsHidden => false; - - /// - /// The library manager used to get libraries or remove deleted episodes + /// The library manager used to get libraries or remove deleted episodes. /// - [Injected] public ILibraryManager LibraryManager { private get; set; } + private readonly ILibraryManager _libraryManager; /// /// The file manager used walk inside directories and check they existences. /// - [Injected] public IFileSystem FileSystem { private get; set; } + private readonly IFileSystem _fileSystem; /// /// The logger used to inform the user that episodes has been removed. /// - [Injected] public ILogger Logger { private get; set; } + private readonly ILogger _logger; - - /// - public async Task Run(TaskParameters arguments, IProgress progress, CancellationToken cancellationToken) + /// + /// Create a new task. + /// + /// The library manager used to get libraries or remove deleted episodes. + /// The file manager used walk inside directories and check they existences. + /// The logger used to inform the user that episodes has been removed. + public Housekeeping(ILibraryManager libraryManager, IFileSystem fileSystem, ILogger logger) { - int count = 0; - int delCount = await LibraryManager.GetCount() + await LibraryManager.GetCount(); - progress.Report(0); - - foreach (Show show in await LibraryManager.GetAll()) - { - progress.Report(count / delCount * 100); - count++; - - if (await FileSystem.Exists(show.Path)) - continue; - Logger.LogWarning("Show {Name}'s folder has been deleted (was {Path}), removing it from kyoo", - show.Title, show.Path); - await LibraryManager.Delete(show); - } - - foreach (Episode episode in await LibraryManager.GetAll()) - { - progress.Report(count / delCount * 100); - count++; - - if (await FileSystem.Exists(episode.Path)) - continue; - Logger.LogWarning("Episode {Slug}'s file has been deleted (was {Path}), removing it from kyoo", - episode.Slug, episode.Path); - await LibraryManager.Delete(episode); - } - - progress.Report(100); + _libraryManager = libraryManager; + _fileSystem = fileSystem; + _logger = logger; } /// @@ -85,5 +45,39 @@ namespace Kyoo.Tasks { return new(); } + + /// + public async Task Run(TaskParameters arguments, IProgress progress, CancellationToken cancellationToken) + { + int count = 0; + int delCount = await _libraryManager.GetCount() + await _libraryManager.GetCount(); + progress.Report(0); + + foreach (Show show in await _libraryManager.GetAll()) + { + progress.Report(count / delCount * 100); + count++; + + if (await _fileSystem.Exists(show.Path)) + continue; + _logger.LogWarning("Show {Name}'s folder has been deleted (was {Path}), removing it from kyoo", + show.Title, show.Path); + await _libraryManager.Delete(show); + } + + foreach (Episode episode in await _libraryManager.GetAll()) + { + progress.Report(count / delCount * 100); + count++; + + if (await _fileSystem.Exists(episode.Path)) + continue; + _logger.LogWarning("Episode {Slug}'s file has been deleted (was {Path}), removing it from kyoo", + episode.Slug, episode.Path); + await _libraryManager.Delete(episode); + } + + progress.Report(100); + } } } \ No newline at end of file diff --git a/Kyoo/Tasks/MetadataProviderLoader.cs b/Kyoo/Tasks/MetadataProviderLoader.cs index db50f9ec..42342434 100644 --- a/Kyoo/Tasks/MetadataProviderLoader.cs +++ b/Kyoo/Tasks/MetadataProviderLoader.cs @@ -2,8 +2,8 @@ using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; +using Kyoo.Common.Models.Attributes; using Kyoo.Controllers; -using Kyoo.Models.Attributes; using Kyoo.Models.Exceptions; namespace Kyoo.Tasks @@ -11,43 +11,45 @@ namespace Kyoo.Tasks /// /// A task that download metadata providers images. /// + [TaskMetadata("reload-metadata", "Reload Metadata Providers", "Add every loaded metadata provider to the database.", + RunOnStartup = true, Priority = 1000, IsHidden = true)] public class MetadataProviderLoader : ITask { - /// - public string Slug => "reload-metdata"; - - /// - public string Name => "Reload Metadata Providers"; - - /// - public string Description => "Add every loaded metadata provider to the database."; - - /// - public string HelpMessage => null; - - /// - public bool RunOnStartup => true; - - /// - public int Priority => 1000; - - /// - public bool IsHidden => true; - /// /// The provider repository used to create in-db providers from metadata providers. /// - [Injected] public IProviderRepository Providers { private get; set; } + private readonly IProviderRepository _providers; /// /// The thumbnail manager used to download providers logo. /// - [Injected] public IThumbnailsManager Thumbnails { private get; set; } + private readonly IThumbnailsManager _thumbnails; /// /// The list of metadata providers to register. /// - [Injected] public ICollection MetadataProviders { private get; set; } - - + private readonly ICollection _metadataProviders; + + /// + /// Create a new task. + /// + /// + /// The provider repository used to create in-db providers from metadata providers. + /// + /// + /// The thumbnail manager used to download providers logo. + /// + /// + /// The list of metadata providers to register. + /// + public MetadataProviderLoader(IProviderRepository providers, + IThumbnailsManager thumbnails, + ICollection metadataProviders) + { + _providers = providers; + _thumbnails = thumbnails; + _metadataProviders = metadataProviders; + } + + /// public TaskParameters GetParameters() { @@ -60,13 +62,13 @@ namespace Kyoo.Tasks float percent = 0; progress.Report(0); - foreach (IMetadataProvider provider in MetadataProviders) + foreach (IMetadataProvider provider in _metadataProviders) { if (string.IsNullOrEmpty(provider.Provider.Slug)) throw new TaskFailedException($"Empty provider slug (name: {provider.Provider.Name})."); - await Providers.CreateIfNotExists(provider.Provider); - await Thumbnails.DownloadImages(provider.Provider); - percent += 100f / MetadataProviders.Count; + await _providers.CreateIfNotExists(provider.Provider); + await _thumbnails.DownloadImages(provider.Provider); + percent += 100f / _metadataProviders.Count; progress.Report(percent); } progress.Report(100); diff --git a/Kyoo/Tasks/PluginInitializer.cs b/Kyoo/Tasks/PluginInitializer.cs index 31dd324e..c9f814a6 100644 --- a/Kyoo/Tasks/PluginInitializer.cs +++ b/Kyoo/Tasks/PluginInitializer.cs @@ -2,57 +2,55 @@ using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; +using Kyoo.Common.Models.Attributes; using Kyoo.Controllers; -using Kyoo.Models.Attributes; namespace Kyoo.Tasks { /// /// A task run on Kyoo's startup to initialize plugins /// + [TaskMetadata("plugin-init", "Plugin Initializer", "A task to initialize plugins.", + RunOnStartup = true, Priority = int.MaxValue, IsHidden = true)] public class PluginInitializer : ITask { - /// - public string Slug => "plugin-init"; - - /// - public string Name => "PluginInitializer"; - - /// - public string Description => "A task to initialize plugins."; - - /// - public string HelpMessage => null; - - /// - public bool RunOnStartup => true; - - /// - public int Priority => int.MaxValue; - - /// - public bool IsHidden => true; - - /// /// The plugin manager used to retrieve plugins to initialize them. /// - [Injected] public IPluginManager PluginManager { private get; set; } + private readonly IPluginManager _pluginManager; /// /// The service provider given to each method. /// - [Injected] public IServiceProvider Provider { private get; set; } + private readonly IServiceProvider _provider; + + /// + /// Create a new task + /// + /// The plugin manager used to retrieve plugins to initialize them. + /// The service provider given to each method. + public PluginInitializer(IPluginManager pluginManager, IServiceProvider provider) + { + _pluginManager = pluginManager; + _provider = provider; + } + + + /// + public TaskParameters GetParameters() + { + return new(); + } /// public Task Run(TaskParameters arguments, IProgress progress, CancellationToken cancellationToken) { - ICollection plugins = PluginManager.GetAllPlugins(); + ICollection plugins = _pluginManager.GetAllPlugins(); int count = 0; progress.Report(0); foreach (IPlugin plugin in plugins) { - plugin.Initialize(Provider); + plugin.Initialize(_provider); progress.Report(count / plugins.Count * 100); count++; @@ -61,11 +59,5 @@ namespace Kyoo.Tasks progress.Report(100); return Task.CompletedTask; } - - /// - public TaskParameters GetParameters() - { - return new(); - } } } \ No newline at end of file diff --git a/Kyoo/Tasks/RegisterEpisode.cs b/Kyoo/Tasks/RegisterEpisode.cs index 674ae477..dea3bf40 100644 --- a/Kyoo/Tasks/RegisterEpisode.cs +++ b/Kyoo/Tasks/RegisterEpisode.cs @@ -2,9 +2,9 @@ using System; using System.Linq; using System.Threading; using System.Threading.Tasks; +using Kyoo.Common.Models.Attributes; using Kyoo.Controllers; using Kyoo.Models; -using Kyoo.Models.Attributes; using Kyoo.Models.Exceptions; namespace Kyoo.Tasks @@ -12,49 +12,60 @@ namespace Kyoo.Tasks /// /// A task to register a new episode /// + [TaskMetadata("register", "Register episode", "Register a new episode")] public class RegisterEpisode : ITask { - /// - public string Slug => "register"; - - /// - public string Name => "Register episode"; - - /// - public string Description => "Register a new episode"; - - /// - public string HelpMessage => null; - - /// - public bool RunOnStartup => false; - - /// - public int Priority => 0; - - /// - public bool IsHidden => false; - /// /// An identifier to extract metadata from paths. /// - [Injected] public IIdentifier Identifier { private get; set; } + private readonly IIdentifier _identifier; /// - /// The library manager used to register the episode + /// The library manager used to register the episode. /// - [Injected] public ILibraryManager LibraryManager { private get; set; } + private readonly ILibraryManager _libraryManager; /// /// A metadata provider to retrieve the metadata of the new episode (and related items if they do not exist). /// - [Injected] public AProviderComposite MetadataProvider { private get; set; } + private readonly AProviderComposite _metadataProvider; /// /// The thumbnail manager used to download images. /// - [Injected] public IThumbnailsManager ThumbnailsManager { private get; set; } + private readonly IThumbnailsManager _thumbnailsManager; /// /// The transcoder used to extract subtitles and metadata. /// - [Injected] public ITranscoder Transcoder { private get; set; } + private readonly ITranscoder _transcoder; + + /// + /// Create a new task. + /// + /// + /// An identifier to extract metadata from paths. + /// + /// + /// The library manager used to register the episode. + /// + /// + /// A metadata provider to retrieve the metadata of the new episode (and related items if they do not exist). + /// + /// + /// The thumbnail manager used to download images. + /// + /// + /// The transcoder used to extract subtitles and metadata. + /// + public RegisterEpisode(IIdentifier identifier, + ILibraryManager libraryManager, + AProviderComposite metadataProvider, + IThumbnailsManager thumbnailsManager, + ITranscoder transcoder) + { + _identifier = identifier; + _libraryManager = libraryManager; + _metadataProvider = metadataProvider; + _thumbnailsManager = thumbnailsManager; + _transcoder = transcoder; + } /// public TaskParameters GetParameters() @@ -77,11 +88,11 @@ namespace Kyoo.Tasks progress.Report(0); if (library.Providers == null) - await LibraryManager.Load(library, x => x.Providers); - MetadataProvider.UseProviders(library.Providers); + await _libraryManager.Load(library, x => x.Providers); + _metadataProvider.UseProviders(library.Providers); try { - (Collection collection, Show show, Season season, Episode episode) = await Identifier.Identify(path, + (Collection collection, Show show, Season season, Episode episode) = await _identifier.Identify(path, relativePath); progress.Report(15); @@ -94,7 +105,7 @@ namespace Kyoo.Tasks if (show.StartAir.HasValue) { show.Slug += $"-{show.StartAir.Value.Year}"; - show = await LibraryManager.Create(show); + show = await _libraryManager.Create(show); } else { @@ -107,7 +118,7 @@ namespace Kyoo.Tasks // If they are not already loaded, load external ids to allow metadata providers to use them. if (show.ExternalIDs == null) - await LibraryManager.Load(show, x => x.ExternalIDs); + await _libraryManager.Load(show, x => x.ExternalIDs); progress.Report(50); if (season != null) @@ -119,20 +130,20 @@ namespace Kyoo.Tasks episode.Show = show; episode.Season = season; - episode = await MetadataProvider.Get(episode); + episode = await _metadataProvider.Get(episode); progress.Report(70); - episode.Tracks = (await Transcoder.ExtractInfos(episode, false)) + episode.Tracks = (await _transcoder.ExtractInfos(episode, false)) .Where(x => x.Type != StreamType.Attachment) .ToArray(); - await ThumbnailsManager.DownloadImages(episode); + await _thumbnailsManager.DownloadImages(episode); progress.Report(90); - await LibraryManager.Create(episode); + await _libraryManager.Create(episode); progress.Report(95); - await LibraryManager.AddShowLink(show, library, collection); + await _libraryManager.AddShowLink(show, library, collection); progress.Report(100); } - catch (IdentificationFailed ex) + catch (IdentificationFailedException ex) { throw new TaskFailedException(ex); } @@ -156,12 +167,12 @@ namespace Kyoo.Tasks if (item == null || string.IsNullOrEmpty(item.Slug)) return null; - T existing = await LibraryManager.GetOrDefault(item.Slug); + T existing = await _libraryManager.GetOrDefault(item.Slug); if (existing != null) return existing; - item = await MetadataProvider.Get(item); - await ThumbnailsManager.DownloadImages(item); - return await LibraryManager.CreateIfNotExists(item); + item = await _metadataProvider.Get(item); + await _thumbnailsManager.DownloadImages(item); + return await _libraryManager.CreateIfNotExists(item); } } } \ No newline at end of file diff --git a/Kyoo/Tasks/RegisterSubtitle.cs b/Kyoo/Tasks/RegisterSubtitle.cs index 07620e6c..23a7d425 100644 --- a/Kyoo/Tasks/RegisterSubtitle.cs +++ b/Kyoo/Tasks/RegisterSubtitle.cs @@ -1,9 +1,9 @@ using System; using System.Threading; using System.Threading.Tasks; +using Kyoo.Common.Models.Attributes; using Kyoo.Controllers; using Kyoo.Models; -using Kyoo.Models.Attributes; using Kyoo.Models.Exceptions; namespace Kyoo.Tasks @@ -11,37 +11,28 @@ namespace Kyoo.Tasks /// /// A task to register a new episode /// + [TaskMetadata("register-sub", "Register subtitle", "Register a new subtitle")] public class RegisterSubtitle : ITask { - /// - public string Slug => "register-sub"; - - /// - public string Name => "Register subtitle"; - - /// - public string Description => "Register a new subtitle"; - - /// - public string HelpMessage => null; - - /// - public bool RunOnStartup => false; - - /// - public int Priority => 0; - - /// - public bool IsHidden => false; - /// /// An identifier to extract metadata from paths. /// - [Injected] public IIdentifier Identifier { private get; set; } + private readonly IIdentifier _identifier; /// - /// The library manager used to register the episode + /// The library manager used to register the episode. /// - [Injected] public ILibraryManager LibraryManager { private get; set; } + private readonly ILibraryManager _libraryManager; + + /// + /// Create a new task. + /// + /// An identifier to extract metadata from paths. + /// The library manager used to register the episode. + public RegisterSubtitle(IIdentifier identifier, ILibraryManager libraryManager) + { + _identifier = identifier; + _libraryManager = libraryManager; + } /// public TaskParameters GetParameters() @@ -63,7 +54,7 @@ namespace Kyoo.Tasks try { progress.Report(0); - Track track = await Identifier.IdentifyTrack(path, relativePath); + Track track = await _identifier.IdentifyTrack(path, relativePath); progress.Report(25); if (track.Episode == null) @@ -71,10 +62,10 @@ namespace Kyoo.Tasks if (track.Episode.ID == 0) { if (track.Episode.Slug != null) - track.Episode = await LibraryManager.Get(track.Episode.Slug); + track.Episode = await _libraryManager.Get(track.Episode.Slug); else if (track.Episode.Path != null) { - track.Episode = await LibraryManager.GetOrDefault(x => x.Path.StartsWith(track.Episode.Path)); + track.Episode = await _libraryManager.GetOrDefault(x => x.Path.StartsWith(track.Episode.Path)); if (track.Episode == null) throw new TaskFailedException($"No episode found for the track at: {path}."); } @@ -83,10 +74,10 @@ namespace Kyoo.Tasks } progress.Report(50); - await LibraryManager.Create(track); + await _libraryManager.Create(track); progress.Report(100); } - catch (IdentificationFailed ex) + catch (IdentificationFailedException ex) { throw new TaskFailedException(ex); } From e55f60166e70f55de1938d127cde3344312bb52e Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Fri, 23 Jul 2021 18:45:31 +0200 Subject: [PATCH 23/31] Cleaning up and adding tests for the regex identifier --- Kyoo.Common/Controllers/IIdentifier.cs | 26 ++--- Kyoo.Common/Controllers/ITask.cs | 10 ++ Kyoo.Tests/Database/RepositoryTests.cs | 1 + Kyoo.Tests/Identifier/IdentifierTests.cs | 135 +++++++++++++++++++++++ Kyoo.Tests/Kyoo.Tests.csproj | 1 + Kyoo/Controllers/RegexIdentifier.cs | 56 +++++++--- Kyoo/Models/Options/MediaOptions.cs | 4 +- Kyoo/Tasks/RegisterEpisode.cs | 18 +-- Kyoo/Tasks/RegisterSubtitle.cs | 7 +- Kyoo/settings.json | 10 +- 10 files changed, 220 insertions(+), 48 deletions(-) create mode 100644 Kyoo.Tests/Identifier/IdentifierTests.cs diff --git a/Kyoo.Common/Controllers/IIdentifier.cs b/Kyoo.Common/Controllers/IIdentifier.cs index 11aeb65d..5d11c53b 100644 --- a/Kyoo.Common/Controllers/IIdentifier.cs +++ b/Kyoo.Common/Controllers/IIdentifier.cs @@ -12,32 +12,26 @@ namespace Kyoo.Controllers /// /// Identify a path and return the parsed metadata. /// - /// - /// The path of the episode file to parse. - /// - /// - /// The path of the episode file relative to the library root. It starts with a /. - /// - /// The identifier could not work for the given path. + /// The path of the episode file to parse. + /// + /// The identifier could not work for the given path. + /// /// /// A tuple of models representing parsed metadata. /// If no metadata could be parsed for a type, null can be returned. /// - Task<(Collection, Show, Season, Episode)> Identify(string path, string relativePath); + Task<(Collection, Show, Season, Episode)> Identify(string path); /// /// Identify an external subtitle or track file from it's path and return the parsed metadata. /// - /// - /// The path of the external track file to parse. - /// - /// - /// The path of the episode file relative to the library root. It starts with a /. - /// - /// The identifier could not work for the given path. + /// The path of the external track file to parse. + /// + /// The identifier could not work for the given path. + /// /// /// The metadata of the track identified. /// - Task IdentifyTrack(string path, string relativePath); + Task IdentifyTrack(string path); } } \ No newline at end of file diff --git a/Kyoo.Common/Controllers/ITask.cs b/Kyoo.Common/Controllers/ITask.cs index e5fbce29..e8d07000 100644 --- a/Kyoo.Common/Controllers/ITask.cs +++ b/Kyoo.Common/Controllers/ITask.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; using JetBrains.Annotations; +using Kyoo.Models; using Kyoo.Models.Attributes; using Kyoo.Models.Exceptions; @@ -116,6 +117,15 @@ namespace Kyoo.Controllers { if (typeof(T) == typeof(object)) return (T)Value; + + if (Value is IResource resource) + { + if (typeof(T) == typeof(string)) + return (T)(object)resource.Slug; + if (typeof(T) == typeof(int)) + return (T)(object)resource.ID; + } + return (T)Convert.ChangeType(Value, typeof(T)); } } diff --git a/Kyoo.Tests/Database/RepositoryTests.cs b/Kyoo.Tests/Database/RepositoryTests.cs index 51db3061..4cc72b4c 100644 --- a/Kyoo.Tests/Database/RepositoryTests.cs +++ b/Kyoo.Tests/Database/RepositoryTests.cs @@ -25,6 +25,7 @@ namespace Kyoo.Tests public void Dispose() { Repositories.Dispose(); + GC.SuppressFinalize(this); } public ValueTask DisposeAsync() diff --git a/Kyoo.Tests/Identifier/IdentifierTests.cs b/Kyoo.Tests/Identifier/IdentifierTests.cs new file mode 100644 index 00000000..170a1352 --- /dev/null +++ b/Kyoo.Tests/Identifier/IdentifierTests.cs @@ -0,0 +1,135 @@ +using System.Threading.Tasks; +using Kyoo.Controllers; +using Kyoo.Models; +using Kyoo.Models.Options; +using Microsoft.Extensions.Options; +using Moq; +using Xunit; + +namespace Kyoo.Tests.Identifier +{ + public class Identifier + { + private readonly Mock _manager; + private readonly IIdentifier _identifier; + + public Identifier() + { + Mock> options = new(); + options.Setup(x => x.CurrentValue).Returns(new MediaOptions + { + Regex = new [] + { + "^\\/?(?.+)?\\/(?.+?)(?: \\((?\\d+)\\))?\\/\\k(?: \\(\\d+\\))? S(?\\d+)E(?\\d+)\\..*$", + "^\\/?(?.+)?\\/(?.+?)(?: \\((?\\d+)\\))?\\/\\k(?: \\(\\d+\\))? (?\\d+)\\..*$", + "^\\/?(?.+)?\\/(?.+?)(?: \\((?\\d+)\\))?\\/\\k(?: \\(\\d+\\))?\\..*$" + } + }); + + _manager = new Mock(); + _identifier = new RegexIdentifier(options.Object, _manager.Object); + } + + + [Fact] + public async Task EpisodeIdentification() + { + _manager.Setup(x => x.GetAll(null, new Sort(), default)).ReturnsAsync(new[] + { + new Library {Paths = new [] {"/kyoo/Library/"}} + }); + (Collection collection, Show show, Season season, Episode episode) = await _identifier.Identify( + "/kyoo/Library/Collection/Show (2000)/Show S01E01.extension"); + Assert.Equal("Collection", collection.Name); + Assert.Equal("collection", collection.Slug); + Assert.Equal("Show", show.Title); + Assert.Equal("show", show.Slug); + Assert.Equal(2000, show.StartAir!.Value.Year); + Assert.Equal(1, season.SeasonNumber); + Assert.Equal(1, episode.SeasonNumber); + Assert.Equal(1, episode.EpisodeNumber); + Assert.Null(episode.AbsoluteNumber); + } + + [Fact] + public async Task EpisodeIdentificationWithoutLibraryTrailingSlash() + { + _manager.Setup(x => x.GetAll(null, new Sort(), default)).ReturnsAsync(new[] + { + new Library {Paths = new [] {"/kyoo/Library"}} + }); + (Collection collection, Show show, Season season, Episode episode) = await _identifier.Identify( + "/kyoo/Library/Collection/Show (2000)/Show S01E01.extension"); + Assert.Equal("Collection", collection.Name); + Assert.Equal("collection", collection.Slug); + Assert.Equal("Show", show.Title); + Assert.Equal("show", show.Slug); + Assert.Equal(2000, show.StartAir!.Value.Year); + Assert.Equal(1, season.SeasonNumber); + Assert.Equal(1, episode.SeasonNumber); + Assert.Equal(1, episode.EpisodeNumber); + Assert.Null(episode.AbsoluteNumber); + } + + [Fact] + public async Task EpisodeIdentificationMultiplePaths() + { + _manager.Setup(x => x.GetAll(null, new Sort(), default)).ReturnsAsync(new[] + { + new Library {Paths = new [] {"/kyoo", "/kyoo/Library/"}} + }); + (Collection collection, Show show, Season season, Episode episode) = await _identifier.Identify( + "/kyoo/Library/Collection/Show (2000)/Show S01E01.extension"); + Assert.Equal("Collection", collection.Name); + Assert.Equal("collection", collection.Slug); + Assert.Equal("Show", show.Title); + Assert.Equal("show", show.Slug); + Assert.Equal(2000, show.StartAir!.Value.Year); + Assert.Equal(1, season.SeasonNumber); + Assert.Equal(1, episode.SeasonNumber); + Assert.Equal(1, episode.EpisodeNumber); + Assert.Null(episode.AbsoluteNumber); + } + + [Fact] + public async Task AbsoluteEpisodeIdentification() + { + _manager.Setup(x => x.GetAll(null, new Sort(), default)).ReturnsAsync(new[] + { + new Library {Paths = new [] {"/kyoo", "/kyoo/Library/"}} + }); + (Collection collection, Show show, Season season, Episode episode) = await _identifier.Identify( + "/kyoo/Library/Collection/Show (2000)/Show 100.extension"); + Assert.Equal("Collection", collection.Name); + Assert.Equal("collection", collection.Slug); + Assert.Equal("Show", show.Title); + Assert.Equal("show", show.Slug); + Assert.Equal(2000, show.StartAir!.Value.Year); + Assert.Null(season); + Assert.Null(episode.SeasonNumber); + Assert.Null(episode.EpisodeNumber); + Assert.Equal(100, episode.AbsoluteNumber); + } + + [Fact] + public async Task MovieEpisodeIdentification() + { + _manager.Setup(x => x.GetAll(null, new Sort(), default)).ReturnsAsync(new[] + { + new Library {Paths = new [] {"/kyoo", "/kyoo/Library/"}} + }); + (Collection collection, Show show, Season season, Episode episode) = await _identifier.Identify( + "/kyoo/Library/Collection/Show (2000)/Show.extension"); + Assert.Equal("Collection", collection.Name); + Assert.Equal("collection", collection.Slug); + Assert.Equal("Show", show.Title); + Assert.Equal("show", show.Slug); + Assert.Equal(2000, show.StartAir!.Value.Year); + Assert.Null(season); + Assert.True(show.IsMovie); + Assert.Null(episode.SeasonNumber); + Assert.Null(episode.EpisodeNumber); + Assert.Null(episode.AbsoluteNumber); + } + } +} \ No newline at end of file diff --git a/Kyoo.Tests/Kyoo.Tests.csproj b/Kyoo.Tests/Kyoo.Tests.csproj index d198dae6..9dce41fc 100644 --- a/Kyoo.Tests/Kyoo.Tests.csproj +++ b/Kyoo.Tests/Kyoo.Tests.csproj @@ -17,6 +17,7 @@ + runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Kyoo/Controllers/RegexIdentifier.cs b/Kyoo/Controllers/RegexIdentifier.cs index 3c09ae4a..c1a8ab45 100644 --- a/Kyoo/Controllers/RegexIdentifier.cs +++ b/Kyoo/Controllers/RegexIdentifier.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.Linq; using System.Text.RegularExpressions; using System.Threading.Tasks; using Kyoo.Models; @@ -18,24 +19,48 @@ namespace Kyoo.Controllers /// /// The configuration of kyoo to retrieve the identifier regex. /// - private readonly IOptions _configuration; + private readonly IOptionsMonitor _configuration; + /// + /// The library manager used to retrieve libraries paths. + /// + private readonly ILibraryManager _libraryManager; /// /// Create a new . /// /// The regex patterns to use. - public RegexIdentifier(IOptions configuration) + /// The library manager used to retrieve libraries paths. + public RegexIdentifier(IOptionsMonitor configuration, ILibraryManager libraryManager) { _configuration = configuration; + _libraryManager = libraryManager; + } + + /// + /// Retrieve the relative path of an episode or subtitle. + /// + /// The full path of the episode + /// The path relative to the library root. + private async Task _GetRelativePath(string path) + { + string libraryPath = (await _libraryManager.GetAll()) + .SelectMany(x => x.Paths) + .Where(path.StartsWith) + .OrderByDescending(x => x.Length) + .FirstOrDefault(); + return path[(libraryPath?.Length ?? 0)..]; } /// - public Task<(Collection, Show, Season, Episode)> Identify(string path, string relativePath) + public async Task<(Collection, Show, Season, Episode)> Identify(string path) { - Regex regex = new(_configuration.Value.Regex, RegexOptions.IgnoreCase | RegexOptions.Compiled); - Match match = regex.Match(relativePath); + string relativePath = await _GetRelativePath(path); + Match match = _configuration.CurrentValue.Regex + .Select(x => new Regex(x, RegexOptions.IgnoreCase | RegexOptions.Compiled)) + .Select(x => x.Match(relativePath)) + .FirstOrDefault(x => x.Success); - if (!match.Success) + if (match == null) throw new IdentificationFailedException($"The episode at {path} does not match the episode's regex."); (Collection collection, Show show, Season season, Episode episode) ret = ( @@ -80,24 +105,27 @@ namespace Kyoo.Controllers ret.episode.Title = ret.show.Title; } - return Task.FromResult(ret); + return ret; } /// - public Task IdentifyTrack(string path, string relativePath) + public async Task IdentifyTrack(string path) { - Regex regex = new(_configuration.Value.SubtitleRegex, RegexOptions.IgnoreCase | RegexOptions.Compiled); - Match match = regex.Match(path); + string relativePath = await _GetRelativePath(path); + Match match = _configuration.CurrentValue.SubtitleRegex + .Select(x => new Regex(x, RegexOptions.IgnoreCase | RegexOptions.Compiled)) + .Select(x => x.Match(relativePath)) + .FirstOrDefault(x => x.Success); - if (!match.Success) + if (match == null) throw new IdentificationFailedException($"The subtitle at {path} does not match the subtitle's regex."); string episodePath = match.Groups["Episode"].Value; - return Task.FromResult(new Track + return new Track { Type = StreamType.Subtitle, Language = match.Groups["Language"].Value, - IsDefault = match.Groups["Default"].Value.Length > 0, + IsDefault = match.Groups["Default"].Value.Length > 0, IsForced = match.Groups["Forced"].Value.Length > 0, Codec = FileExtensions.SubtitleExtensions[Path.GetExtension(path)], IsExternal = true, @@ -106,7 +134,7 @@ namespace Kyoo.Controllers { Path = episodePath } - }); + }; } } } \ No newline at end of file diff --git a/Kyoo/Models/Options/MediaOptions.cs b/Kyoo/Models/Options/MediaOptions.cs index d53b13d0..4b02e4db 100644 --- a/Kyoo/Models/Options/MediaOptions.cs +++ b/Kyoo/Models/Options/MediaOptions.cs @@ -13,11 +13,11 @@ namespace Kyoo.Models.Options /// /// A regex for episodes /// - public string Regex { get; set; } + public string[] Regex { get; set; } /// /// A regex for subtitles /// - public string SubtitleRegex { get; set; } + public string[] SubtitleRegex { get; set; } } } \ No newline at end of file diff --git a/Kyoo/Tasks/RegisterEpisode.cs b/Kyoo/Tasks/RegisterEpisode.cs index dea3bf40..5541e6fa 100644 --- a/Kyoo/Tasks/RegisterEpisode.cs +++ b/Kyoo/Tasks/RegisterEpisode.cs @@ -73,8 +73,6 @@ namespace Kyoo.Tasks return new() { TaskParameter.CreateRequired("path", "The path of the episode file"), - TaskParameter.CreateRequired("relativePath", - "The path of the episode file relative to the library root. It starts with a /."), TaskParameter.CreateRequired("library", "The library in witch the episode is") }; } @@ -83,17 +81,19 @@ namespace Kyoo.Tasks public async Task Run(TaskParameters arguments, IProgress progress, CancellationToken cancellationToken) { string path = arguments["path"].As(); - string relativePath = arguments["relativePath"].As(); Library library = arguments["library"].As(); progress.Report(0); - - if (library.Providers == null) - await _libraryManager.Load(library, x => x.Providers); - _metadataProvider.UseProviders(library.Providers); + + if (library != null) + { + if (library.Providers == null) + await _libraryManager.Load(library, x => x.Providers); + _metadataProvider.UseProviders(library.Providers); + } + try { - (Collection collection, Show show, Season season, Episode episode) = await _identifier.Identify(path, - relativePath); + (Collection collection, Show show, Season season, Episode episode) = await _identifier.Identify(path); progress.Report(15); collection = await _RegisterAndFill(collection); diff --git a/Kyoo/Tasks/RegisterSubtitle.cs b/Kyoo/Tasks/RegisterSubtitle.cs index 23a7d425..6c36ea58 100644 --- a/Kyoo/Tasks/RegisterSubtitle.cs +++ b/Kyoo/Tasks/RegisterSubtitle.cs @@ -39,9 +39,7 @@ namespace Kyoo.Tasks { return new() { - TaskParameter.CreateRequired("path", "The path of the subtitle file"), - TaskParameter.CreateRequired("relativePath", - "The path of the subtitle file relative to the library root. It starts with a /.") + TaskParameter.CreateRequired("path", "The path of the subtitle file") }; } @@ -49,12 +47,11 @@ namespace Kyoo.Tasks public async Task Run(TaskParameters arguments, IProgress progress, CancellationToken cancellationToken) { string path = arguments["path"].As(); - string relativePath = arguments["relativePath"].As(); try { progress.Report(0); - Track track = await _identifier.IdentifyTrack(path, relativePath); + Track track = await _identifier.IdentifyTrack(path); progress.Report(25); if (track.Episode == null) diff --git a/Kyoo/settings.json b/Kyoo/settings.json index ffc9b446..66ff8566 100644 --- a/Kyoo/settings.json +++ b/Kyoo/settings.json @@ -44,8 +44,14 @@ }, "media": { - "regex": "(?:\\/(?.*?))?\\/(?.*?)(?: \\(?\\d+\\))?\\/\\k(?: \\(\\d+\\))?(?:(?: S(?\\d+)E(?\\d+))| (?\\d+))?.*$", - "subtitleRegex": "^(?.*)\\.(?\\w{1,3})\\.(?default\\.)?(?forced\\.)?.*$" + "regex": [ + "^\\/?(?.+)?\\/(?.+?)(?: \\((?\\d+)\\))?\\/\\k(?: \\(\\d+\\))? S(?\\d+)E(?\\d+)\\..*$", + "^\\/?(?.+)?\\/(?.+?)(?: \\((?\\d+)\\))?\\/\\k(?: \\(\\d+\\))? (?\\d+)\\..*$", + "^\\/?(?.+)?\\/(?.+?)(?: \\((?\\d+)\\))?\\/\\k(?: \\(\\d+\\))?\\..*$" + ], + "subtitleRegex": [ + "^(?.+)\\.(?\\w{1,3})\\.(?default\\.)?(?forced\\.)?.*$" + ] }, "authentication": { From 03bf2c4f10f28c04dd61e259c000dd9b992fb702 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Fri, 23 Jul 2021 19:36:43 +0200 Subject: [PATCH 24/31] Adding more tests and fixing subtitle identification --- Kyoo.Tests/Identifier/IdentifierTests.cs | 57 ++++++++++++++++++++++++ Kyoo/Controllers/RegexIdentifier.cs | 17 +++---- 2 files changed, 66 insertions(+), 8 deletions(-) diff --git a/Kyoo.Tests/Identifier/IdentifierTests.cs b/Kyoo.Tests/Identifier/IdentifierTests.cs index 170a1352..36ad94b9 100644 --- a/Kyoo.Tests/Identifier/IdentifierTests.cs +++ b/Kyoo.Tests/Identifier/IdentifierTests.cs @@ -1,6 +1,7 @@ using System.Threading.Tasks; using Kyoo.Controllers; using Kyoo.Models; +using Kyoo.Models.Exceptions; using Kyoo.Models.Options; using Microsoft.Extensions.Options; using Moq; @@ -23,6 +24,10 @@ namespace Kyoo.Tests.Identifier "^\\/?(?.+)?\\/(?.+?)(?: \\((?\\d+)\\))?\\/\\k(?: \\(\\d+\\))? S(?\\d+)E(?\\d+)\\..*$", "^\\/?(?.+)?\\/(?.+?)(?: \\((?\\d+)\\))?\\/\\k(?: \\(\\d+\\))? (?\\d+)\\..*$", "^\\/?(?.+)?\\/(?.+?)(?: \\((?\\d+)\\))?\\/\\k(?: \\(\\d+\\))?\\..*$" + }, + SubtitleRegex = new[] + { + "^(?.+)\\.(?\\w{1,3})\\.(?default\\.)?(?forced\\.)?.*$" } }); @@ -131,5 +136,57 @@ namespace Kyoo.Tests.Identifier Assert.Null(episode.EpisodeNumber); Assert.Null(episode.AbsoluteNumber); } + + [Fact] + public async Task InvalidEpisodeIdentification() + { + _manager.Setup(x => x.GetAll(null, new Sort(), default)).ReturnsAsync(new[] + { + new Library {Paths = new [] {"/kyoo", "/kyoo/Library/"}} + }); + await Assert.ThrowsAsync(() => _identifier.Identify("/invalid/path")); + } + + [Fact] + public async Task SubtitleIdentification() + { + _manager.Setup(x => x.GetAll(null, new Sort(), default)).ReturnsAsync(new[] + { + new Library {Paths = new [] {"/kyoo", "/kyoo/Library/"}} + }); + Track track = await _identifier.IdentifyTrack("/kyoo/Library/Collection/Show (2000)/Show.eng.default.str"); + Assert.True(track.IsExternal); + Assert.Equal("eng", track.Language); + Assert.Equal("subrip", track.Codec); + Assert.True(track.IsDefault); + Assert.False(track.IsForced); + Assert.StartsWith("/kyoo/Library/Collection/Show (2000)/Show", track.Episode.Path); + } + + [Fact] + public async Task SubtitleIdentificationUnknownCodec() + { + _manager.Setup(x => x.GetAll(null, new Sort(), default)).ReturnsAsync(new[] + { + new Library {Paths = new [] {"/kyoo", "/kyoo/Library/"}} + }); + Track track = await _identifier.IdentifyTrack("/kyoo/Library/Collection/Show (2000)/Show.eng.default.extension"); + Assert.True(track.IsExternal); + Assert.Equal("eng", track.Language); + Assert.Equal("extension", track.Codec); + Assert.True(track.IsDefault); + Assert.False(track.IsForced); + Assert.StartsWith("/kyoo/Library/Collection/Show (2000)/Show", track.Episode.Path); + } + + [Fact] + public async Task InvalidSubtitleIdentification() + { + _manager.Setup(x => x.GetAll(null, new Sort(), default)).ReturnsAsync(new[] + { + new Library {Paths = new [] {"/kyoo", "/kyoo/Library/"}} + }); + await Assert.ThrowsAsync(() => _identifier.IdentifyTrack("/invalid/path")); + } } } \ No newline at end of file diff --git a/Kyoo/Controllers/RegexIdentifier.cs b/Kyoo/Controllers/RegexIdentifier.cs index c1a8ab45..c25e3e4f 100644 --- a/Kyoo/Controllers/RegexIdentifier.cs +++ b/Kyoo/Controllers/RegexIdentifier.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Text.RegularExpressions; @@ -109,32 +110,32 @@ namespace Kyoo.Controllers } /// - public async Task IdentifyTrack(string path) + public Task IdentifyTrack(string path) { - string relativePath = await _GetRelativePath(path); Match match = _configuration.CurrentValue.SubtitleRegex .Select(x => new Regex(x, RegexOptions.IgnoreCase | RegexOptions.Compiled)) - .Select(x => x.Match(relativePath)) + .Select(x => x.Match(path)) .FirstOrDefault(x => x.Success); if (match == null) throw new IdentificationFailedException($"The subtitle at {path} does not match the subtitle's regex."); string episodePath = match.Groups["Episode"].Value; - return new Track + string extension = Path.GetExtension(path); + return Task.FromResult(new Track { Type = StreamType.Subtitle, Language = match.Groups["Language"].Value, - IsDefault = match.Groups["Default"].Value.Length > 0, - IsForced = match.Groups["Forced"].Value.Length > 0, - Codec = FileExtensions.SubtitleExtensions[Path.GetExtension(path)], + IsDefault = match.Groups["Default"].Success, + IsForced = match.Groups["Forced"].Success, + Codec = FileExtensions.SubtitleExtensions.GetValueOrDefault(extension, extension[1..]), IsExternal = true, Path = path, Episode = new Episode { Path = episodePath } - }; + }); } } } \ No newline at end of file From c206512e3b5118adc75796ecf18e7445b963d95c Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sat, 24 Jul 2021 00:36:11 +0200 Subject: [PATCH 25/31] Testing tvdb convertors --- Kyoo.Common/Models/Resources/Show.cs | 2 +- Kyoo.CommonAPI/LocalRepository.cs | 2 +- Kyoo.Tests/Database/TestSample.cs | 2 +- Kyoo.Tests/Identifier/Tvdb/ConvertorTests.cs | 160 +++++++++++++++++++ Kyoo.Tests/Kyoo.Tests.csproj | 2 + Kyoo.Tests/Utility/EnumerableTests.cs | 2 +- Kyoo.Tests/Utility/MergerTests.cs | 2 +- Kyoo.Tests/Utility/TaskTests.cs | 2 +- Kyoo.Tests/Utility/UtilityTests.cs | 28 ++-- Kyoo.TheTvdb/Convertors.cs | 21 +-- Kyoo.TheTvdb/PluginTvdb.cs | 2 +- Kyoo.TheTvdb/ProviderTvdb.cs | 2 +- Kyoo.TheTvdb/TvdbOption.cs | 2 +- Kyoo.WebApp | 2 +- 14 files changed, 198 insertions(+), 33 deletions(-) create mode 100644 Kyoo.Tests/Identifier/Tvdb/ConvertorTests.cs diff --git a/Kyoo.Common/Models/Resources/Show.cs b/Kyoo.Common/Models/Resources/Show.cs index c9d5ca8e..bddc6920 100644 --- a/Kyoo.Common/Models/Resources/Show.cs +++ b/Kyoo.Common/Models/Resources/Show.cs @@ -43,7 +43,7 @@ namespace Kyoo.Models /// /// Is this show airing, not aired yet or finished? /// - public Status? Status { get; set; } + public Status Status { get; set; } /// /// An URL to a trailer. This could be any path supported by the . diff --git a/Kyoo.CommonAPI/LocalRepository.cs b/Kyoo.CommonAPI/LocalRepository.cs index 8ab23443..0187e0dd 100644 --- a/Kyoo.CommonAPI/LocalRepository.cs +++ b/Kyoo.CommonAPI/LocalRepository.cs @@ -208,7 +208,7 @@ namespace Kyoo.Controllers } catch (DuplicatedItemException) { - return await GetOrDefault(obj.Slug); + return await Get(obj.Slug); } } diff --git a/Kyoo.Tests/Database/TestSample.cs b/Kyoo.Tests/Database/TestSample.cs index 96cd63c1..e0ff955f 100644 --- a/Kyoo.Tests/Database/TestSample.cs +++ b/Kyoo.Tests/Database/TestSample.cs @@ -233,7 +233,7 @@ namespace Kyoo.Tests provider.ID = 0; context.Providers.Add(provider); - Models.Library library = Get(); + Library library = Get(); library.ID = 0; library.Collections = new List {collection}; library.Providers = new List {provider}; diff --git a/Kyoo.Tests/Identifier/Tvdb/ConvertorTests.cs b/Kyoo.Tests/Identifier/Tvdb/ConvertorTests.cs new file mode 100644 index 00000000..35e389b6 --- /dev/null +++ b/Kyoo.Tests/Identifier/Tvdb/ConvertorTests.cs @@ -0,0 +1,160 @@ +using System; +using System.Linq; +using Kyoo.Models; +using Kyoo.TheTvdb; +using TvDbSharper.Dto; +using Xunit; + +namespace Kyoo.Tests.Identifier.Tvdb +{ + public class ConvertorTests + { + [Fact] + public void SeriesSearchToShow() + { + SeriesSearchResult result = new() + { + Slug = "slug", + SeriesName = "name", + Aliases = new[] { "Aliases" }, + Overview = "overview", + Status = "Ended", + FirstAired = "2021-07-23", + Poster = "/poster", + Id = 5 + }; + Provider provider = TestSample.Get(); + Show show = result.ToShow(provider); + + Assert.Equal("slug", show.Slug); + Assert.Equal("name", show.Title); + Assert.Single(show.Aliases); + Assert.Equal("Aliases", show.Aliases[0]); + Assert.Equal("overview", show.Overview); + Assert.Equal(new DateTime(2021, 7, 23), show.StartAir); + Assert.Equal("https://www.thetvdb.com/poster", show.Poster); + Assert.Single(show.ExternalIDs); + Assert.Equal("5", show.ExternalIDs.First().DataID); + Assert.Equal(provider, show.ExternalIDs.First().Provider); + Assert.Equal("https://www.thetvdb.com/series/slug", show.ExternalIDs.First().Link); + Assert.Equal(Status.Finished, show.Status); + } + + [Fact] + public void SeriesSearchToShowInvalidDate() + { + SeriesSearchResult result = new() + { + Slug = "slug", + SeriesName = "name", + Aliases = new[] { "Aliases" }, + Overview = "overview", + Status = "ad", + FirstAired = "2e021-07-23", + Poster = "/poster", + Id = 5 + }; + Provider provider = TestSample.Get(); + Show show = result.ToShow(provider); + + Assert.Equal("slug", show.Slug); + Assert.Equal("name", show.Title); + Assert.Single(show.Aliases); + Assert.Equal("Aliases", show.Aliases[0]); + Assert.Equal("overview", show.Overview); + Assert.Null(show.StartAir); + Assert.Equal("https://www.thetvdb.com/poster", show.Poster); + Assert.Single(show.ExternalIDs); + Assert.Equal("5", show.ExternalIDs.First().DataID); + Assert.Equal(provider, show.ExternalIDs.First().Provider); + Assert.Equal("https://www.thetvdb.com/series/slug", show.ExternalIDs.First().Link); + Assert.Equal(Status.Unknown, show.Status); + } + + [Fact] + public void SeriesToShow() + { + Series result = new() + { + Slug = "slug", + SeriesName = "name", + Aliases = new[] { "Aliases" }, + Overview = "overview", + Status = "Continuing", + FirstAired = "2021-07-23", + Poster = "poster", + FanArt = "fanart", + Id = 5, + Genre = new [] + { + "Action", + "Test With Spéàacial characters" + } + }; + Provider provider = TestSample.Get(); + Show show = result.ToShow(provider); + + Assert.Equal("slug", show.Slug); + Assert.Equal("name", show.Title); + Assert.Single(show.Aliases); + Assert.Equal("Aliases", show.Aliases[0]); + Assert.Equal("overview", show.Overview); + Assert.Equal(new DateTime(2021, 7, 23), show.StartAir); + Assert.Equal("https://www.thetvdb.com/banners/poster", show.Poster); + Assert.Equal("https://www.thetvdb.com/banners/fanart", show.Backdrop); + Assert.Single(show.ExternalIDs); + Assert.Equal("5", show.ExternalIDs.First().DataID); + Assert.Equal(provider, show.ExternalIDs.First().Provider); + Assert.Equal("https://www.thetvdb.com/series/slug", show.ExternalIDs.First().Link); + Assert.Equal(Status.Airing, show.Status); + Assert.Equal(2, show.Genres.Count); + Assert.Equal("action", show.Genres.ToArray()[0].Slug); + Assert.Equal("Action", show.Genres.ToArray()[0].Name); + Assert.Equal("Test With Spéàacial characters", show.Genres.ToArray()[1].Name); + Assert.Equal("test-with-speaacial-characters", show.Genres.ToArray()[1].Slug); + } + + [Fact] + public void ActorToPeople() + { + Actor actor = new() + { + Id = 5, + Image = "image", + Name = "Name", + Role = "role" + }; + Provider provider = TestSample.Get(); + PeopleRole people = actor.ToPeopleRole(provider); + + Assert.Equal("name", people.Slug); + Assert.Equal("Name", people.People.Name); + Assert.Equal("role", people.Role); + Assert.Equal("https://www.thetvdb.com/banners/image", people.People.Poster); + } + + [Fact] + public void EpisodeRecordToEpisode() + { + EpisodeRecord record = new() + { + Id = 5, + AiredSeason = 2, + AiredEpisodeNumber = 3, + AbsoluteNumber = 23, + EpisodeName = "title", + Overview = "overview", + Filename = "thumb" + }; + Provider provider = TestSample.Get(); + Episode episode = record.ToEpisode(provider); + + Assert.Equal("title", episode.Title); + Assert.Equal(2, episode.SeasonNumber); + Assert.Equal(3, episode.EpisodeNumber); + Assert.Equal(23, episode.AbsoluteNumber); + Assert.Equal("overview", episode.Overview); + Assert.Equal("https://www.thetvdb.com/banners/thumb", episode.Thumb); + } + } +} \ No newline at end of file diff --git a/Kyoo.Tests/Kyoo.Tests.csproj b/Kyoo.Tests/Kyoo.Tests.csproj index 9dce41fc..120f8ba7 100644 --- a/Kyoo.Tests/Kyoo.Tests.csproj +++ b/Kyoo.Tests/Kyoo.Tests.csproj @@ -18,6 +18,7 @@ + runtime; build; native; contentfiles; analyzers; buildtransitive @@ -33,6 +34,7 @@ + diff --git a/Kyoo.Tests/Utility/EnumerableTests.cs b/Kyoo.Tests/Utility/EnumerableTests.cs index 9cdd8a00..c03efa06 100644 --- a/Kyoo.Tests/Utility/EnumerableTests.cs +++ b/Kyoo.Tests/Utility/EnumerableTests.cs @@ -4,7 +4,7 @@ using System.Linq; using System.Threading.Tasks; using Xunit; -namespace Kyoo.Tests +namespace Kyoo.Tests.Utility { public class EnumerableTests { diff --git a/Kyoo.Tests/Utility/MergerTests.cs b/Kyoo.Tests/Utility/MergerTests.cs index 008daee1..285532c2 100644 --- a/Kyoo.Tests/Utility/MergerTests.cs +++ b/Kyoo.Tests/Utility/MergerTests.cs @@ -5,7 +5,7 @@ using Kyoo.Models; using Kyoo.Models.Attributes; using Xunit; -namespace Kyoo.Tests +namespace Kyoo.Tests.Utility { public class MergerTests { diff --git a/Kyoo.Tests/Utility/TaskTests.cs b/Kyoo.Tests/Utility/TaskTests.cs index 3a7baa48..3bcd723f 100644 --- a/Kyoo.Tests/Utility/TaskTests.cs +++ b/Kyoo.Tests/Utility/TaskTests.cs @@ -3,7 +3,7 @@ using System.Threading; using System.Threading.Tasks; using Xunit; -namespace Kyoo.Tests +namespace Kyoo.Tests.Utility { public class TaskTests { diff --git a/Kyoo.Tests/Utility/UtilityTests.cs b/Kyoo.Tests/Utility/UtilityTests.cs index 80d233c3..f6f279aa 100644 --- a/Kyoo.Tests/Utility/UtilityTests.cs +++ b/Kyoo.Tests/Utility/UtilityTests.cs @@ -4,7 +4,9 @@ using System.Reflection; using Kyoo.Models; using Xunit; -namespace Kyoo.Tests +using Utils = Kyoo.Utility; + +namespace Kyoo.Tests.Utility { public class UtilityTests { @@ -14,12 +16,12 @@ namespace Kyoo.Tests Expression> member = x => x.ID; Expression> memberCast = x => x.ID; - Assert.False(Utility.IsPropertyExpression(null)); - Assert.True(Utility.IsPropertyExpression(member)); - Assert.True(Utility.IsPropertyExpression(memberCast)); + Assert.False(Utils.IsPropertyExpression(null)); + Assert.True(Utils.IsPropertyExpression(member)); + Assert.True(Utils.IsPropertyExpression(memberCast)); Expression> call = x => x.GetID("test"); - Assert.False(Utility.IsPropertyExpression(call)); + Assert.False(Utils.IsPropertyExpression(call)); } [Fact] @@ -28,15 +30,15 @@ namespace Kyoo.Tests Expression> member = x => x.ID; Expression> memberCast = x => x.ID; - Assert.Equal("ID", Utility.GetPropertyName(member)); - Assert.Equal("ID", Utility.GetPropertyName(memberCast)); - Assert.Throws(() => Utility.GetPropertyName(null)); + Assert.Equal("ID", Utils.GetPropertyName(member)); + Assert.Equal("ID", Utils.GetPropertyName(memberCast)); + Assert.Throws(() => Utils.GetPropertyName(null)); } [Fact] public void GetMethodTest() { - MethodInfo method = Utility.GetMethod(typeof(UtilityTests), + MethodInfo method = Utils.GetMethod(typeof(UtilityTests), BindingFlags.Instance | BindingFlags.Public, nameof(GetMethodTest), Array.Empty(), @@ -47,17 +49,17 @@ namespace Kyoo.Tests [Fact] public void GetMethodInvalidGenericsTest() { - Assert.Throws(() => Utility.GetMethod(typeof(UtilityTests), + Assert.Throws(() => Utils.GetMethod(typeof(UtilityTests), BindingFlags.Instance | BindingFlags.Public, nameof(GetMethodTest), - new [] { typeof(Utility) }, + new [] { typeof(Utils) }, Array.Empty())); } [Fact] public void GetMethodInvalidParamsTest() { - Assert.Throws(() => Utility.GetMethod(typeof(UtilityTests), + Assert.Throws(() => Utils.GetMethod(typeof(UtilityTests), BindingFlags.Instance | BindingFlags.Public, nameof(GetMethodTest), Array.Empty(), @@ -67,7 +69,7 @@ namespace Kyoo.Tests [Fact] public void GetMethodTest2() { - MethodInfo method = Utility.GetMethod(typeof(Merger), + MethodInfo method = Utils.GetMethod(typeof(Merger), BindingFlags.Static | BindingFlags.Public, nameof(Merger.MergeLists), new [] { typeof(string) }, diff --git a/Kyoo.TheTvdb/Convertors.cs b/Kyoo.TheTvdb/Convertors.cs index 6aa5c3ef..bc31b5fd 100644 --- a/Kyoo.TheTvdb/Convertors.cs +++ b/Kyoo.TheTvdb/Convertors.cs @@ -16,13 +16,13 @@ namespace Kyoo.TheTvdb /// /// The string representing the status. /// A kyoo value or null. - private static Status? GetStatus(string status) + private static Status _GetStatus(string status) { return status switch { "Ended" => Status.Finished, "Continuing" => Status.Airing, - _ => null + _ => Status.Unknown }; } @@ -31,11 +31,12 @@ namespace Kyoo.TheTvdb /// /// The date string to parse /// The parsed or null. - private static DateTime ParseDate(string date) + private static DateTime? _ParseDate(string date) { - DateTime.TryParseExact(date, "yyyy-MM-dd", CultureInfo.InvariantCulture, - DateTimeStyles.None, out DateTime parsed); - return parsed; + return DateTime.TryParseExact(date, "yyyy-MM-dd", CultureInfo.InvariantCulture, + DateTimeStyles.None, out DateTime parsed) + ? parsed + : null; } /// @@ -52,8 +53,8 @@ namespace Kyoo.TheTvdb Title = result.SeriesName, Aliases = result.Aliases, Overview = result.Overview, - Status = GetStatus(result.Status), - StartAir = ParseDate(result.FirstAired), + Status = _GetStatus(result.Status), + StartAir = _ParseDate(result.FirstAired), Poster = result.Poster != null ? $"https://www.thetvdb.com{result.Poster}" : null, ExternalIDs = new[] { @@ -81,8 +82,8 @@ namespace Kyoo.TheTvdb Title = series.SeriesName, Aliases = series.Aliases, Overview = series.Overview, - Status = GetStatus(series.Status), - StartAir = ParseDate(series.FirstAired), + Status = _GetStatus(series.Status), + StartAir = _ParseDate(series.FirstAired), Poster = series.Poster != null ? $"https://www.thetvdb.com/banners/{series.Poster}" : null, Backdrop = series.FanArt != null ? $"https://www.thetvdb.com/banners/{series.FanArt}" : null, Genres = series.Genre.Select(y => new Genre(y)).ToList(), diff --git a/Kyoo.TheTvdb/PluginTvdb.cs b/Kyoo.TheTvdb/PluginTvdb.cs index 1d008fbf..e0b697ea 100644 --- a/Kyoo.TheTvdb/PluginTvdb.cs +++ b/Kyoo.TheTvdb/PluginTvdb.cs @@ -1,9 +1,9 @@ using System; using System.Collections.Generic; using Autofac; -using Kyoo.Authentication.Models; using Kyoo.Controllers; using Kyoo.Models.Attributes; +using Kyoo.TheTvdb.Models; using Microsoft.AspNetCore.Builder; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; diff --git a/Kyoo.TheTvdb/ProviderTvdb.cs b/Kyoo.TheTvdb/ProviderTvdb.cs index 78834bfd..892e8125 100644 --- a/Kyoo.TheTvdb/ProviderTvdb.cs +++ b/Kyoo.TheTvdb/ProviderTvdb.cs @@ -3,9 +3,9 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using JetBrains.Annotations; -using Kyoo.Authentication.Models; using Kyoo.Controllers; using Kyoo.Models; +using Kyoo.TheTvdb.Models; using Microsoft.Extensions.Options; using TvDbSharper; using TvDbSharper.Dto; diff --git a/Kyoo.TheTvdb/TvdbOption.cs b/Kyoo.TheTvdb/TvdbOption.cs index 3b21ed9e..9a884b24 100644 --- a/Kyoo.TheTvdb/TvdbOption.cs +++ b/Kyoo.TheTvdb/TvdbOption.cs @@ -1,4 +1,4 @@ -namespace Kyoo.Authentication.Models +namespace Kyoo.TheTvdb.Models { /// /// The option containing the api key for the tvdb. diff --git a/Kyoo.WebApp b/Kyoo.WebApp index 87783a5b..c037270d 160000 --- a/Kyoo.WebApp +++ b/Kyoo.WebApp @@ -1 +1 @@ -Subproject commit 87783a5bfdd79f21d72bad13da1d96ec8e08cd42 +Subproject commit c037270d3339fcf0075984a089f353c5c332a751 From d8da21ddec54e0c01508429b0359f5b8830bfc4e Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sat, 24 Jul 2021 01:02:39 +0200 Subject: [PATCH 26/31] Fixing migrations --- Kyoo.Common/Models/Resources/Show.cs | 2 +- ....cs => 20210723224326_Initial.Designer.cs} | 56 +++++++++++++++++-- ...3_Initial.cs => 20210723224326_Initial.cs} | 4 +- ...cs => 20210723224335_Triggers.Designer.cs} | 56 +++++++++++++++++-- ...Triggers.cs => 20210723224335_Triggers.cs} | 0 .../PostgresContextModelSnapshot.cs | 54 ++++++++++++++++-- ....cs => 20210723224542_Initial.Designer.cs} | 44 +++++++++++++-- ...7_Initial.cs => 20210723224542_Initial.cs} | 2 +- ...cs => 20210723224550_Triggers.Designer.cs} | 44 +++++++++++++-- ...Triggers.cs => 20210723224550_Triggers.cs} | 0 .../Migrations/SqLiteContextModelSnapshot.cs | 42 +++++++++++++- 11 files changed, 275 insertions(+), 29 deletions(-) rename Kyoo.Postgresql/Migrations/{20210627141933_Initial.Designer.cs => 20210723224326_Initial.Designer.cs} (95%) rename Kyoo.Postgresql/Migrations/{20210627141933_Initial.cs => 20210723224326_Initial.cs} (99%) rename Kyoo.Postgresql/Migrations/{20210627141941_Triggers.Designer.cs => 20210723224335_Triggers.Designer.cs} (95%) rename Kyoo.Postgresql/Migrations/{20210627141941_Triggers.cs => 20210723224335_Triggers.cs} (100%) rename Kyoo.SqLite/Migrations/{20210626141337_Initial.Designer.cs => 20210723224542_Initial.Designer.cs} (96%) rename Kyoo.SqLite/Migrations/{20210626141337_Initial.cs => 20210723224542_Initial.cs} (99%) rename Kyoo.SqLite/Migrations/{20210626141347_Triggers.Designer.cs => 20210723224550_Triggers.Designer.cs} (96%) rename Kyoo.SqLite/Migrations/{20210626141347_Triggers.cs => 20210723224550_Triggers.cs} (100%) diff --git a/Kyoo.Common/Models/Resources/Show.cs b/Kyoo.Common/Models/Resources/Show.cs index bddc6920..57c9fcef 100644 --- a/Kyoo.Common/Models/Resources/Show.cs +++ b/Kyoo.Common/Models/Resources/Show.cs @@ -185,5 +185,5 @@ namespace Kyoo.Models /// /// The enum containing show's status. /// - public enum Status { Finished, Airing, Planned, Unknown } + public enum Status { Unknown, Finished, Airing, Planned } } diff --git a/Kyoo.Postgresql/Migrations/20210627141933_Initial.Designer.cs b/Kyoo.Postgresql/Migrations/20210723224326_Initial.Designer.cs similarity index 95% rename from Kyoo.Postgresql/Migrations/20210627141933_Initial.Designer.cs rename to Kyoo.Postgresql/Migrations/20210723224326_Initial.Designer.cs index f1733a77..3016a040 100644 --- a/Kyoo.Postgresql/Migrations/20210627141933_Initial.Designer.cs +++ b/Kyoo.Postgresql/Migrations/20210723224326_Initial.Designer.cs @@ -12,7 +12,7 @@ using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; namespace Kyoo.Postgresql.Migrations { [DbContext(typeof(PostgresContext))] - [Migration("20210627141933_Initial")] + [Migration("20210723224326_Initial")] partial class Initial { protected override void BuildTargetModel(ModelBuilder modelBuilder) @@ -20,10 +20,10 @@ namespace Kyoo.Postgresql.Migrations #pragma warning disable 612, 618 modelBuilder .HasPostgresEnum(null, "item_type", new[] { "show", "movie", "collection" }) - .HasPostgresEnum(null, "status", new[] { "finished", "airing", "planned", "unknown" }) + .HasPostgresEnum(null, "status", new[] { "unknown", "finished", "airing", "planned" }) .HasPostgresEnum(null, "stream_type", new[] { "unknown", "video", "audio", "subtitle", "attachment" }) .HasAnnotation("Relational:MaxIdentifierLength", 63) - .HasAnnotation("ProductVersion", "5.0.7") + .HasAnnotation("ProductVersion", "5.0.8") .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); modelBuilder.Entity("Kyoo.Models.Collection", b => @@ -189,6 +189,51 @@ namespace Kyoo.Postgresql.Migrations b.ToTable("libraries"); }); + modelBuilder.Entity("Kyoo.Models.LibraryItem", b => + { + b.Property("ID") + .HasColumnType("integer") + .HasColumnName("id") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("EndAir") + .HasColumnType("timestamp without time zone") + .HasColumnName("end_air"); + + b.Property("Overview") + .HasColumnType("text") + .HasColumnName("overview"); + + b.Property("Poster") + .HasColumnType("text") + .HasColumnName("poster"); + + b.Property("Slug") + .HasColumnType("text") + .HasColumnName("slug"); + + b.Property("StartAir") + .HasColumnType("timestamp without time zone") + .HasColumnName("start_air"); + + b.Property("Status") + .HasColumnType("status") + .HasColumnName("status"); + + b.Property("Title") + .HasColumnType("text") + .HasColumnName("title"); + + b.Property("Type") + .HasColumnType("item_type") + .HasColumnName("type"); + + b.HasKey("ID") + .HasName("pk_library_items"); + + b.ToView("library_items"); + }); + modelBuilder.Entity("Kyoo.Models.Link", b => { b.Property("FirstID") @@ -621,7 +666,7 @@ namespace Kyoo.Postgresql.Migrations .HasColumnType("timestamp without time zone") .HasColumnName("start_air"); - b.Property("Status") + b.Property("Status") .HasColumnType("status") .HasColumnName("status"); @@ -1078,7 +1123,8 @@ namespace Kyoo.Postgresql.Migrations b.HasOne("Kyoo.Models.Studio", "Studio") .WithMany("Shows") .HasForeignKey("StudioID") - .HasConstraintName("fk_shows_studios_studio_id"); + .HasConstraintName("fk_shows_studios_studio_id") + .OnDelete(DeleteBehavior.SetNull); b.Navigation("Studio"); }); diff --git a/Kyoo.Postgresql/Migrations/20210627141933_Initial.cs b/Kyoo.Postgresql/Migrations/20210723224326_Initial.cs similarity index 99% rename from Kyoo.Postgresql/Migrations/20210627141933_Initial.cs rename to Kyoo.Postgresql/Migrations/20210723224326_Initial.cs index 29b51490..2ba22c6a 100644 --- a/Kyoo.Postgresql/Migrations/20210627141933_Initial.cs +++ b/Kyoo.Postgresql/Migrations/20210723224326_Initial.cs @@ -12,7 +12,7 @@ namespace Kyoo.Postgresql.Migrations { migrationBuilder.AlterDatabase() .Annotation("Npgsql:Enum:item_type", "show,movie,collection") - .Annotation("Npgsql:Enum:status", "finished,airing,planned,unknown") + .Annotation("Npgsql:Enum:status", "unknown,finished,airing,planned") .Annotation("Npgsql:Enum:stream_type", "unknown,video,audio,subtitle,attachment"); migrationBuilder.CreateTable( @@ -208,7 +208,7 @@ namespace Kyoo.Postgresql.Migrations aliases = table.Column(type: "text[]", nullable: true), path = table.Column(type: "text", nullable: true), overview = table.Column(type: "text", nullable: true), - status = table.Column(type: "status", nullable: true), + status = table.Column(type: "status", nullable: false), trailer_url = table.Column(type: "text", nullable: true), start_air = table.Column(type: "timestamp without time zone", nullable: true), end_air = table.Column(type: "timestamp without time zone", nullable: true), diff --git a/Kyoo.Postgresql/Migrations/20210627141941_Triggers.Designer.cs b/Kyoo.Postgresql/Migrations/20210723224335_Triggers.Designer.cs similarity index 95% rename from Kyoo.Postgresql/Migrations/20210627141941_Triggers.Designer.cs rename to Kyoo.Postgresql/Migrations/20210723224335_Triggers.Designer.cs index fc019baf..d377a09f 100644 --- a/Kyoo.Postgresql/Migrations/20210627141941_Triggers.Designer.cs +++ b/Kyoo.Postgresql/Migrations/20210723224335_Triggers.Designer.cs @@ -12,7 +12,7 @@ using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; namespace Kyoo.Postgresql.Migrations { [DbContext(typeof(PostgresContext))] - [Migration("20210627141941_Triggers")] + [Migration("20210723224335_Triggers")] partial class Triggers { protected override void BuildTargetModel(ModelBuilder modelBuilder) @@ -20,10 +20,10 @@ namespace Kyoo.Postgresql.Migrations #pragma warning disable 612, 618 modelBuilder .HasPostgresEnum(null, "item_type", new[] { "show", "movie", "collection" }) - .HasPostgresEnum(null, "status", new[] { "finished", "airing", "planned", "unknown" }) + .HasPostgresEnum(null, "status", new[] { "unknown", "finished", "airing", "planned" }) .HasPostgresEnum(null, "stream_type", new[] { "unknown", "video", "audio", "subtitle", "attachment" }) .HasAnnotation("Relational:MaxIdentifierLength", 63) - .HasAnnotation("ProductVersion", "5.0.7") + .HasAnnotation("ProductVersion", "5.0.8") .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); modelBuilder.Entity("Kyoo.Models.Collection", b => @@ -189,6 +189,51 @@ namespace Kyoo.Postgresql.Migrations b.ToTable("libraries"); }); + modelBuilder.Entity("Kyoo.Models.LibraryItem", b => + { + b.Property("ID") + .HasColumnType("integer") + .HasColumnName("id") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("EndAir") + .HasColumnType("timestamp without time zone") + .HasColumnName("end_air"); + + b.Property("Overview") + .HasColumnType("text") + .HasColumnName("overview"); + + b.Property("Poster") + .HasColumnType("text") + .HasColumnName("poster"); + + b.Property("Slug") + .HasColumnType("text") + .HasColumnName("slug"); + + b.Property("StartAir") + .HasColumnType("timestamp without time zone") + .HasColumnName("start_air"); + + b.Property("Status") + .HasColumnType("status") + .HasColumnName("status"); + + b.Property("Title") + .HasColumnType("text") + .HasColumnName("title"); + + b.Property("Type") + .HasColumnType("item_type") + .HasColumnName("type"); + + b.HasKey("ID") + .HasName("pk_library_items"); + + b.ToView("library_items"); + }); + modelBuilder.Entity("Kyoo.Models.Link", b => { b.Property("FirstID") @@ -621,7 +666,7 @@ namespace Kyoo.Postgresql.Migrations .HasColumnType("timestamp without time zone") .HasColumnName("start_air"); - b.Property("Status") + b.Property("Status") .HasColumnType("status") .HasColumnName("status"); @@ -1078,7 +1123,8 @@ namespace Kyoo.Postgresql.Migrations b.HasOne("Kyoo.Models.Studio", "Studio") .WithMany("Shows") .HasForeignKey("StudioID") - .HasConstraintName("fk_shows_studios_studio_id"); + .HasConstraintName("fk_shows_studios_studio_id") + .OnDelete(DeleteBehavior.SetNull); b.Navigation("Studio"); }); diff --git a/Kyoo.Postgresql/Migrations/20210627141941_Triggers.cs b/Kyoo.Postgresql/Migrations/20210723224335_Triggers.cs similarity index 100% rename from Kyoo.Postgresql/Migrations/20210627141941_Triggers.cs rename to Kyoo.Postgresql/Migrations/20210723224335_Triggers.cs diff --git a/Kyoo.Postgresql/Migrations/PostgresContextModelSnapshot.cs b/Kyoo.Postgresql/Migrations/PostgresContextModelSnapshot.cs index e5044c60..f2d55f24 100644 --- a/Kyoo.Postgresql/Migrations/PostgresContextModelSnapshot.cs +++ b/Kyoo.Postgresql/Migrations/PostgresContextModelSnapshot.cs @@ -18,10 +18,10 @@ namespace Kyoo.Postgresql.Migrations #pragma warning disable 612, 618 modelBuilder .HasPostgresEnum(null, "item_type", new[] { "show", "movie", "collection" }) - .HasPostgresEnum(null, "status", new[] { "finished", "airing", "planned", "unknown" }) + .HasPostgresEnum(null, "status", new[] { "unknown", "finished", "airing", "planned" }) .HasPostgresEnum(null, "stream_type", new[] { "unknown", "video", "audio", "subtitle", "attachment" }) .HasAnnotation("Relational:MaxIdentifierLength", 63) - .HasAnnotation("ProductVersion", "5.0.7") + .HasAnnotation("ProductVersion", "5.0.8") .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); modelBuilder.Entity("Kyoo.Models.Collection", b => @@ -187,6 +187,51 @@ namespace Kyoo.Postgresql.Migrations b.ToTable("libraries"); }); + modelBuilder.Entity("Kyoo.Models.LibraryItem", b => + { + b.Property("ID") + .HasColumnType("integer") + .HasColumnName("id") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("EndAir") + .HasColumnType("timestamp without time zone") + .HasColumnName("end_air"); + + b.Property("Overview") + .HasColumnType("text") + .HasColumnName("overview"); + + b.Property("Poster") + .HasColumnType("text") + .HasColumnName("poster"); + + b.Property("Slug") + .HasColumnType("text") + .HasColumnName("slug"); + + b.Property("StartAir") + .HasColumnType("timestamp without time zone") + .HasColumnName("start_air"); + + b.Property("Status") + .HasColumnType("status") + .HasColumnName("status"); + + b.Property("Title") + .HasColumnType("text") + .HasColumnName("title"); + + b.Property("Type") + .HasColumnType("item_type") + .HasColumnName("type"); + + b.HasKey("ID") + .HasName("pk_library_items"); + + b.ToView("library_items"); + }); + modelBuilder.Entity("Kyoo.Models.Link", b => { b.Property("FirstID") @@ -619,7 +664,7 @@ namespace Kyoo.Postgresql.Migrations .HasColumnType("timestamp without time zone") .HasColumnName("start_air"); - b.Property("Status") + b.Property("Status") .HasColumnType("status") .HasColumnName("status"); @@ -1076,7 +1121,8 @@ namespace Kyoo.Postgresql.Migrations b.HasOne("Kyoo.Models.Studio", "Studio") .WithMany("Shows") .HasForeignKey("StudioID") - .HasConstraintName("fk_shows_studios_studio_id"); + .HasConstraintName("fk_shows_studios_studio_id") + .OnDelete(DeleteBehavior.SetNull); b.Navigation("Studio"); }); diff --git a/Kyoo.SqLite/Migrations/20210626141337_Initial.Designer.cs b/Kyoo.SqLite/Migrations/20210723224542_Initial.Designer.cs similarity index 96% rename from Kyoo.SqLite/Migrations/20210626141337_Initial.Designer.cs rename to Kyoo.SqLite/Migrations/20210723224542_Initial.Designer.cs index 1337ce6b..eca501eb 100644 --- a/Kyoo.SqLite/Migrations/20210626141337_Initial.Designer.cs +++ b/Kyoo.SqLite/Migrations/20210723224542_Initial.Designer.cs @@ -9,14 +9,14 @@ using Microsoft.EntityFrameworkCore.Storage.ValueConversion; namespace Kyoo.SqLite.Migrations { [DbContext(typeof(SqLiteContext))] - [Migration("20210626141337_Initial")] + [Migration("20210723224542_Initial")] partial class Initial { protected override void BuildTargetModel(ModelBuilder modelBuilder) { #pragma warning disable 612, 618 modelBuilder - .HasAnnotation("ProductVersion", "5.0.7"); + .HasAnnotation("ProductVersion", "5.0.8"); modelBuilder.Entity("Kyoo.Models.Collection", b => { @@ -143,6 +143,41 @@ namespace Kyoo.SqLite.Migrations b.ToTable("Libraries"); }); + modelBuilder.Entity("Kyoo.Models.LibraryItem", b => + { + b.Property("ID") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("EndAir") + .HasColumnType("TEXT"); + + b.Property("Overview") + .HasColumnType("TEXT"); + + b.Property("Poster") + .HasColumnType("TEXT"); + + b.Property("Slug") + .HasColumnType("TEXT"); + + b.Property("StartAir") + .HasColumnType("TEXT"); + + b.Property("Status") + .HasColumnType("INTEGER"); + + b.Property("Title") + .HasColumnType("TEXT"); + + b.Property("Type") + .HasColumnType("INTEGER"); + + b.HasKey("ID"); + + b.ToView("LibraryItems"); + }); + modelBuilder.Entity("Kyoo.Models.Link", b => { b.Property("FirstID") @@ -477,7 +512,7 @@ namespace Kyoo.SqLite.Migrations b.Property("StartAir") .HasColumnType("TEXT"); - b.Property("Status") + b.Property("Status") .HasColumnType("INTEGER"); b.Property("StudioID") @@ -864,7 +899,8 @@ namespace Kyoo.SqLite.Migrations { b.HasOne("Kyoo.Models.Studio", "Studio") .WithMany("Shows") - .HasForeignKey("StudioID"); + .HasForeignKey("StudioID") + .OnDelete(DeleteBehavior.SetNull); b.Navigation("Studio"); }); diff --git a/Kyoo.SqLite/Migrations/20210626141337_Initial.cs b/Kyoo.SqLite/Migrations/20210723224542_Initial.cs similarity index 99% rename from Kyoo.SqLite/Migrations/20210626141337_Initial.cs rename to Kyoo.SqLite/Migrations/20210723224542_Initial.cs index 88823571..1fa93860 100644 --- a/Kyoo.SqLite/Migrations/20210626141337_Initial.cs +++ b/Kyoo.SqLite/Migrations/20210723224542_Initial.cs @@ -200,7 +200,7 @@ namespace Kyoo.SqLite.Migrations Aliases = table.Column(type: "TEXT", nullable: true), Path = table.Column(type: "TEXT", nullable: true), Overview = table.Column(type: "TEXT", nullable: true), - Status = table.Column(type: "INTEGER", nullable: true), + Status = table.Column(type: "INTEGER", nullable: false), TrailerUrl = table.Column(type: "TEXT", nullable: true), StartAir = table.Column(type: "TEXT", nullable: true), EndAir = table.Column(type: "TEXT", nullable: true), diff --git a/Kyoo.SqLite/Migrations/20210626141347_Triggers.Designer.cs b/Kyoo.SqLite/Migrations/20210723224550_Triggers.Designer.cs similarity index 96% rename from Kyoo.SqLite/Migrations/20210626141347_Triggers.Designer.cs rename to Kyoo.SqLite/Migrations/20210723224550_Triggers.Designer.cs index 02045e3f..059a3aa4 100644 --- a/Kyoo.SqLite/Migrations/20210626141347_Triggers.Designer.cs +++ b/Kyoo.SqLite/Migrations/20210723224550_Triggers.Designer.cs @@ -9,14 +9,14 @@ using Microsoft.EntityFrameworkCore.Storage.ValueConversion; namespace Kyoo.SqLite.Migrations { [DbContext(typeof(SqLiteContext))] - [Migration("20210626141347_Triggers")] + [Migration("20210723224550_Triggers")] partial class Triggers { protected override void BuildTargetModel(ModelBuilder modelBuilder) { #pragma warning disable 612, 618 modelBuilder - .HasAnnotation("ProductVersion", "5.0.7"); + .HasAnnotation("ProductVersion", "5.0.8"); modelBuilder.Entity("Kyoo.Models.Collection", b => { @@ -143,6 +143,41 @@ namespace Kyoo.SqLite.Migrations b.ToTable("Libraries"); }); + modelBuilder.Entity("Kyoo.Models.LibraryItem", b => + { + b.Property("ID") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("EndAir") + .HasColumnType("TEXT"); + + b.Property("Overview") + .HasColumnType("TEXT"); + + b.Property("Poster") + .HasColumnType("TEXT"); + + b.Property("Slug") + .HasColumnType("TEXT"); + + b.Property("StartAir") + .HasColumnType("TEXT"); + + b.Property("Status") + .HasColumnType("INTEGER"); + + b.Property("Title") + .HasColumnType("TEXT"); + + b.Property("Type") + .HasColumnType("INTEGER"); + + b.HasKey("ID"); + + b.ToView("LibraryItems"); + }); + modelBuilder.Entity("Kyoo.Models.Link", b => { b.Property("FirstID") @@ -477,7 +512,7 @@ namespace Kyoo.SqLite.Migrations b.Property("StartAir") .HasColumnType("TEXT"); - b.Property("Status") + b.Property("Status") .HasColumnType("INTEGER"); b.Property("StudioID") @@ -864,7 +899,8 @@ namespace Kyoo.SqLite.Migrations { b.HasOne("Kyoo.Models.Studio", "Studio") .WithMany("Shows") - .HasForeignKey("StudioID"); + .HasForeignKey("StudioID") + .OnDelete(DeleteBehavior.SetNull); b.Navigation("Studio"); }); diff --git a/Kyoo.SqLite/Migrations/20210626141347_Triggers.cs b/Kyoo.SqLite/Migrations/20210723224550_Triggers.cs similarity index 100% rename from Kyoo.SqLite/Migrations/20210626141347_Triggers.cs rename to Kyoo.SqLite/Migrations/20210723224550_Triggers.cs diff --git a/Kyoo.SqLite/Migrations/SqLiteContextModelSnapshot.cs b/Kyoo.SqLite/Migrations/SqLiteContextModelSnapshot.cs index 58ded130..12f5d94b 100644 --- a/Kyoo.SqLite/Migrations/SqLiteContextModelSnapshot.cs +++ b/Kyoo.SqLite/Migrations/SqLiteContextModelSnapshot.cs @@ -14,7 +14,7 @@ namespace Kyoo.SqLite.Migrations { #pragma warning disable 612, 618 modelBuilder - .HasAnnotation("ProductVersion", "5.0.7"); + .HasAnnotation("ProductVersion", "5.0.8"); modelBuilder.Entity("Kyoo.Models.Collection", b => { @@ -141,6 +141,41 @@ namespace Kyoo.SqLite.Migrations b.ToTable("Libraries"); }); + modelBuilder.Entity("Kyoo.Models.LibraryItem", b => + { + b.Property("ID") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("EndAir") + .HasColumnType("TEXT"); + + b.Property("Overview") + .HasColumnType("TEXT"); + + b.Property("Poster") + .HasColumnType("TEXT"); + + b.Property("Slug") + .HasColumnType("TEXT"); + + b.Property("StartAir") + .HasColumnType("TEXT"); + + b.Property("Status") + .HasColumnType("INTEGER"); + + b.Property("Title") + .HasColumnType("TEXT"); + + b.Property("Type") + .HasColumnType("INTEGER"); + + b.HasKey("ID"); + + b.ToView("LibraryItems"); + }); + modelBuilder.Entity("Kyoo.Models.Link", b => { b.Property("FirstID") @@ -475,7 +510,7 @@ namespace Kyoo.SqLite.Migrations b.Property("StartAir") .HasColumnType("TEXT"); - b.Property("Status") + b.Property("Status") .HasColumnType("INTEGER"); b.Property("StudioID") @@ -862,7 +897,8 @@ namespace Kyoo.SqLite.Migrations { b.HasOne("Kyoo.Models.Studio", "Studio") .WithMany("Shows") - .HasForeignKey("StudioID"); + .HasForeignKey("StudioID") + .OnDelete(DeleteBehavior.SetNull); b.Navigation("Studio"); }); From 7e3f6763da517d5a345b955b2f2829e6bad26bea Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sat, 24 Jul 2021 20:36:55 +0200 Subject: [PATCH 27/31] Testing the provider composite --- Kyoo.Tests/Identifier/ProviderTests.cs | 115 +++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 Kyoo.Tests/Identifier/ProviderTests.cs diff --git a/Kyoo.Tests/Identifier/ProviderTests.cs b/Kyoo.Tests/Identifier/ProviderTests.cs new file mode 100644 index 00000000..27ff0042 --- /dev/null +++ b/Kyoo.Tests/Identifier/ProviderTests.cs @@ -0,0 +1,115 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Kyoo.Controllers; +using Kyoo.Models; +using Microsoft.Extensions.Logging; +using Moq; +using Xunit; +using Xunit.Abstractions; + +namespace Kyoo.Tests.Identifier +{ + public class ProviderTests + { + private readonly ILoggerFactory _factory; + + public ProviderTests(ITestOutputHelper output) + { + _factory = LoggerFactory.Create(x => + { + x.ClearProviders(); + x.AddXunit(output); + }); + } + + [Fact] + public async Task NoProviderGetTest() + { + AProviderComposite provider = new ProviderComposite(Array.Empty(), + _factory.CreateLogger()); + Show show = new() + { + ID = 4, + Genres = new[] { new Genre("genre") } + }; + Show ret = await provider.Get(show); + KAssert.DeepEqual(show, ret); + } + + [Fact] + public async Task NoProviderSearchTest() + { + AProviderComposite provider = new ProviderComposite(Array.Empty(), + _factory.CreateLogger()); + ICollection ret = await provider.Search("show"); + Assert.Empty(ret); + } + + [Fact] + public async Task OneProviderGetTest() + { + Show show = new() + { + ID = 4, + Genres = new[] { new Genre("genre") } + }; + Mock mock = new(); + mock.Setup(x => x.Get(show)).ReturnsAsync(new Show + { + Title = "title", + Genres = new[] { new Genre("ToMerge")} + }); + AProviderComposite provider = new ProviderComposite(new [] + { + mock.Object + }, + _factory.CreateLogger()); + + Show ret = await provider.Get(show); + Assert.Equal(4, ret.ID); + Assert.Equal("title", ret.Title); + Assert.Equal(2, ret.Genres.Count); + Assert.Contains("genre", ret.Genres.Select(x => x.Slug)); + Assert.Contains("tomerge", ret.Genres.Select(x => x.Slug)); + } + + [Fact] + public async Task TwoProviderGetTest() + { + Show show = new() + { + ID = 4, + Genres = new[] { new Genre("genre") } + }; + Mock mock = new(); + mock.Setup(x => x.Get(show)).ReturnsAsync(new Show + { + Title = "title", + Genres = new[] { new Genre("ToMerge")} + }); + Mock mockTwo = new(); + mockTwo.Setup(x => x.Get(show)).ReturnsAsync(new Show + { + Title = "title2", + Status = Status.Finished, + Genres = new[] { new Genre("ToMerge")} + }); + AProviderComposite provider = new ProviderComposite(new [] + { + mock.Object, + mockTwo.Object + }, + _factory.CreateLogger()); + + Show ret = await provider.Get(show); + Assert.Equal(4, ret.ID); + Assert.Equal("title", ret.Title); + Assert.Equal(Status.Finished, ret.Status); + Assert.Equal(2, ret.Genres.Count); + Assert.Contains("genre", ret.Genres.Select(x => x.Slug)); + Assert.Contains("tomerge", ret.Genres.Select(x => x.Slug)); + } + } +} \ No newline at end of file From 9d45af3dd2cf979b9a758f749c8491703629d89d Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sat, 24 Jul 2021 21:11:36 +0200 Subject: [PATCH 28/31] Testing failing providers --- Kyoo.Tests/Identifier/ProviderTests.cs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Kyoo.Tests/Identifier/ProviderTests.cs b/Kyoo.Tests/Identifier/ProviderTests.cs index 27ff0042..c49ca3b2 100644 --- a/Kyoo.Tests/Identifier/ProviderTests.cs +++ b/Kyoo.Tests/Identifier/ProviderTests.cs @@ -76,7 +76,7 @@ namespace Kyoo.Tests.Identifier } [Fact] - public async Task TwoProviderGetTest() + public async Task FailingProviderGetTest() { Show show = new() { @@ -84,22 +84,31 @@ namespace Kyoo.Tests.Identifier Genres = new[] { new Genre("genre") } }; Mock mock = new(); + mock.Setup(x => x.Provider).Returns(new Provider("mock", "")); mock.Setup(x => x.Get(show)).ReturnsAsync(new Show { Title = "title", Genres = new[] { new Genre("ToMerge")} }); + Mock mockTwo = new(); + mockTwo.Setup(x => x.Provider).Returns(new Provider("mockTwo", "")); mockTwo.Setup(x => x.Get(show)).ReturnsAsync(new Show { Title = "title2", Status = Status.Finished, Genres = new[] { new Genre("ToMerge")} }); + + Mock mockFailing = new(); + mockFailing.Setup(x => x.Provider).Returns(new Provider("mockFail", "")); + mockFailing.Setup(x => x.Get(show)).Throws(); + AProviderComposite provider = new ProviderComposite(new [] { mock.Object, - mockTwo.Object + mockTwo.Object, + mockFailing.Object }, _factory.CreateLogger()); From 6542062bb8d1b4660485b52da38fd2f2377be4e0 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 25 Jul 2021 01:47:42 +0200 Subject: [PATCH 29/31] Fixing task services resolving --- Kyoo.Common/Models/WatchItem.cs | 1 + Kyoo.Common/Module.cs | 2 +- Kyoo.Postgresql/PostgresModule.cs | 2 +- Kyoo/Controllers/TaskManager.cs | 2 +- Kyoo/Tasks/Crawler.cs | 4 +--- 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Kyoo.Common/Models/WatchItem.cs b/Kyoo.Common/Models/WatchItem.cs index 241ed9a9..158563a8 100644 --- a/Kyoo.Common/Models/WatchItem.cs +++ b/Kyoo.Common/Models/WatchItem.cs @@ -178,6 +178,7 @@ namespace Kyoo.Models EpisodeID = ep.ID, Slug = ep.Slug, ShowSlug = ep.Show.Slug, + ShowTitle = ep.Show.Title, SeasonNumber = ep.SeasonNumber, EpisodeNumber = ep.EpisodeNumber, AbsoluteNumber = ep.AbsoluteNumber, diff --git a/Kyoo.Common/Module.cs b/Kyoo.Common/Module.cs index 0e8de063..7fb9bdbc 100644 --- a/Kyoo.Common/Module.cs +++ b/Kyoo.Common/Module.cs @@ -20,7 +20,7 @@ namespace Kyoo RegisterTask(this ContainerBuilder builder) where T : class, ITask { - return builder.RegisterType().As().SingleInstance(); + return builder.RegisterType().As(); } /// diff --git a/Kyoo.Postgresql/PostgresModule.cs b/Kyoo.Postgresql/PostgresModule.cs index 70d62f74..124df770 100644 --- a/Kyoo.Postgresql/PostgresModule.cs +++ b/Kyoo.Postgresql/PostgresModule.cs @@ -75,7 +75,7 @@ namespace Kyoo.Postgresql DatabaseContext context = provider.GetRequiredService(); context.Database.Migrate(); - NpgsqlConnection conn = (NpgsqlConnection)context.Database.GetDbConnection(); + using NpgsqlConnection conn = (NpgsqlConnection)context.Database.GetDbConnection(); conn.Open(); conn.ReloadTypes(); } diff --git a/Kyoo/Controllers/TaskManager.cs b/Kyoo/Controllers/TaskManager.cs index 54d724f0..8b5650a6 100644 --- a/Kyoo/Controllers/TaskManager.cs +++ b/Kyoo/Controllers/TaskManager.cs @@ -192,7 +192,7 @@ namespace Kyoo.Controllers Dictionary arguments, CancellationToken? cancellationToken = null) { - using (_logger.BeginScope("Task: {Task}", task.Metadata.Name)) + using (_logger.BeginScope("Task: {Task}", task.Metadata.Name)) { await using Owned taskObj = task.Factory.Invoke(); ICollection all = taskObj.Value.GetParameters(); diff --git a/Kyoo/Tasks/Crawler.cs b/Kyoo/Tasks/Crawler.cs index 191fa27d..333710a3 100644 --- a/Kyoo/Tasks/Crawler.cs +++ b/Kyoo/Tasks/Crawler.cs @@ -138,7 +138,6 @@ namespace Kyoo.Tasks _taskManager.StartTask(reporter, new Dictionary { ["path"] = episodePath, - ["relativePath"] = episodePath[path.Length..], ["library"] = library }, cancellationToken); percent += 100f / paths.Length; @@ -161,8 +160,7 @@ namespace Kyoo.Tasks { _taskManager.StartTask(reporter, new Dictionary { - ["path"] = trackPath, - ["relativePath"] = trackPath[path.Length..] + ["path"] = trackPath }, cancellationToken); percent += 100f / subtitles.Length; } From 209a0238f9f44d229f17b4da86a4ee8c3240de3f Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 25 Jul 2021 02:17:03 +0200 Subject: [PATCH 30/31] Fixing the tvdb for movies --- Kyoo.TheTvdb/ProviderTvdb.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Kyoo.TheTvdb/ProviderTvdb.cs b/Kyoo.TheTvdb/ProviderTvdb.cs index 892e8125..47183fb7 100644 --- a/Kyoo.TheTvdb/ProviderTvdb.cs +++ b/Kyoo.TheTvdb/ProviderTvdb.cs @@ -79,6 +79,9 @@ namespace Kyoo.TheTvdb [ItemCanBeNull] private async Task _GetShow([NotNull] Show show) { + if (show.IsMovie) + return null; + if (!int.TryParse(show.GetID(Provider.Slug), out int id)) { Show found = (await _SearchShow(show.Title)).FirstOrDefault(); From a58d2405329c4b80f5d075756879996db6f012c8 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 25 Jul 2021 14:50:31 +0200 Subject: [PATCH 31/31] RegisterEpisoe: Skiping episode metadata for movies --- Kyoo/Tasks/RegisterEpisode.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Kyoo/Tasks/RegisterEpisode.cs b/Kyoo/Tasks/RegisterEpisode.cs index 5541e6fa..9df8b1f7 100644 --- a/Kyoo/Tasks/RegisterEpisode.cs +++ b/Kyoo/Tasks/RegisterEpisode.cs @@ -130,7 +130,8 @@ namespace Kyoo.Tasks episode.Show = show; episode.Season = season; - episode = await _metadataProvider.Get(episode); + if (!show.IsMovie) + episode = await _metadataProvider.Get(episode); progress.Report(70); episode.Tracks = (await _transcoder.ExtractInfos(episode, false)) .Where(x => x.Type != StreamType.Attachment)