mirror of
https://github.com/zoriya/Kyoo.git
synced 2025-07-09 03:04:20 -04:00
Starting to rework the crawler
This commit is contained in:
parent
0265c27010
commit
3ba0cffac2
@ -54,8 +54,10 @@ namespace Kyoo.Controllers
|
||||
/// List files in a directory.
|
||||
/// </summary>
|
||||
/// <param name="path">The path of the directory</param>
|
||||
/// <param name="options">Should the search be recursive or not.</param>
|
||||
/// <returns>A list of files's path.</returns>
|
||||
public Task<ICollection<string>> ListFiles([NotNull] string path);
|
||||
public Task<ICollection<string>> ListFiles([NotNull] string path,
|
||||
SearchOption options = SearchOption.TopDirectoryOnly);
|
||||
|
||||
/// <summary>
|
||||
/// Check if a file exists at the given path.
|
||||
|
@ -1,5 +1,4 @@
|
||||
using System;
|
||||
using Kyoo.Models;
|
||||
using Kyoo.Models;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
using JetBrains.Annotations;
|
||||
@ -31,10 +30,7 @@ namespace Kyoo.Controllers
|
||||
/// Merging metadata is the job of Kyoo, a complex <typeparamref name="T"/> is given
|
||||
/// to make a precise search and give you every available properties, not to discard properties.
|
||||
/// </remarks>
|
||||
/// <exception cref="NotSupportedException">
|
||||
/// If this metadata provider does not support <typeparamref name="T"/>.
|
||||
/// </exception>
|
||||
/// <returns>A new <typeparamref name="T"/> containing metadata from your provider</returns>
|
||||
/// <returns>A new <typeparamref name="T"/> containing metadata from your provider or null</returns>
|
||||
[ItemCanBeNull]
|
||||
Task<T> Get<T>([NotNull] T item)
|
||||
where T : class, IResource;
|
||||
@ -43,15 +39,10 @@ namespace Kyoo.Controllers
|
||||
/// Search for a specific type of items with a given query.
|
||||
/// </summary>
|
||||
/// <param name="query">The search query to use.</param>
|
||||
/// <exception cref="NotSupportedException">
|
||||
/// If this metadata provider does not support <typeparamref name="T"/>.
|
||||
/// </exception>
|
||||
/// <returns>The list of items that could be found on this specific provider.</returns>
|
||||
[ItemNotNull]
|
||||
Task<ICollection<T>> Search<T>(string query)
|
||||
where T : class, IResource;
|
||||
|
||||
Task<ICollection<PeopleRole>> GetPeople(Show show);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
@ -11,8 +11,10 @@ namespace Kyoo.Controllers
|
||||
/// <summary>
|
||||
/// A common interface used to discord plugins
|
||||
/// </summary>
|
||||
/// <remarks>You can inject services in the IPlugin constructor.
|
||||
/// You should only inject well known services like an ILogger, IConfiguration or IWebHostEnvironment.</remarks>
|
||||
/// <remarks>
|
||||
/// You can inject services in the IPlugin constructor.
|
||||
/// You should only inject well known services like an ILogger, IConfiguration or IWebHostEnvironment.
|
||||
/// </remarks>
|
||||
[UsedImplicitly(ImplicitUseTargetFlags.WithInheritors)]
|
||||
public interface IPlugin
|
||||
{
|
||||
@ -84,7 +86,10 @@ namespace Kyoo.Controllers
|
||||
/// An optional configuration step to allow a plugin to change asp net configurations.
|
||||
/// WARNING: This is only called on Kyoo's startup so you must restart the app to apply this changes.
|
||||
/// </summary>
|
||||
/// <param name="app">The Asp.Net application builder. On most case it is not needed but you can use it to add asp net functionalities.</param>
|
||||
/// <param name="app">
|
||||
/// The Asp.Net application builder. On most case it is not needed but you can use it to
|
||||
/// add asp net functionalities.
|
||||
/// </param>
|
||||
void ConfigureAspNet(IApplicationBuilder app)
|
||||
{
|
||||
// Skipped
|
||||
|
@ -3,12 +3,13 @@ using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using JetBrains.Annotations;
|
||||
using Kyoo.Models.Attributes;
|
||||
|
||||
namespace Kyoo.Controllers
|
||||
{
|
||||
/// <summary>
|
||||
/// A single task parameter. This struct contains metadata to display and utility functions to get them in the taks.
|
||||
/// A single task parameter. This struct contains metadata to display and utility functions to get them in the task.
|
||||
/// </summary>
|
||||
/// <remarks>This struct will be used to generate the swagger documentation of the task.</remarks>
|
||||
public record TaskParameter
|
||||
@ -60,6 +61,24 @@ namespace Kyoo.Controllers
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Create a new required task parameter.
|
||||
/// </summary>
|
||||
/// <param name="name">The name of the parameter</param>
|
||||
/// <param name="description">The description of the parameter</param>
|
||||
/// <typeparam name="T">The type of the parameter.</typeparam>
|
||||
/// <returns>A new task parameter.</returns>
|
||||
public static TaskParameter CreateRequired<T>(string name, string description)
|
||||
{
|
||||
return new()
|
||||
{
|
||||
Name = name,
|
||||
Description = description,
|
||||
Type = typeof(T),
|
||||
IsRequired = true
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Create a parameter's value to give to a task.
|
||||
/// </summary>
|
||||
@ -162,27 +181,37 @@ namespace Kyoo.Controllers
|
||||
public int Priority { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Start this task.
|
||||
/// <c>true</c> if this task should not be displayed to the user, <c>false</c> otherwise.
|
||||
/// </summary>
|
||||
/// <param name="arguments">The list of parameters.</param>
|
||||
/// <param name="cancellationToken">A token to request the task's cancellation.
|
||||
/// If this task is not cancelled quickly, it might be killed by the runner.</param>
|
||||
/// <remarks>
|
||||
/// Your task can have any service as a public field and use the <see cref="InjectedAttribute"/>,
|
||||
/// they will be set to an available service from the service container before calling this method.
|
||||
/// </remarks>
|
||||
public Task Run(TaskParameters arguments, CancellationToken cancellationToken);
|
||||
|
||||
public bool IsHidden { get; }
|
||||
|
||||
/// <summary>
|
||||
/// The list of parameters
|
||||
/// </summary>
|
||||
/// <returns>All parameters that this task as. Every one of them will be given to the run function with a value.</returns>
|
||||
/// <returns>
|
||||
/// All parameters that this task as. Every one of them will be given to the run function with a value.
|
||||
/// </returns>
|
||||
public TaskParameters GetParameters();
|
||||
|
||||
/// <summary>
|
||||
/// If this task is running, return the percentage of completion of this task or null if no percentage can be given.
|
||||
/// Start this task.
|
||||
/// </summary>
|
||||
/// <returns>The percentage of completion of the task.</returns>
|
||||
public int? Progress();
|
||||
/// <param name="arguments">
|
||||
/// The list of parameters.
|
||||
/// </param>
|
||||
/// <param name="progress">
|
||||
/// The progress reporter. Used to inform the sender the percentage of completion of this task
|
||||
/// .</param>
|
||||
/// <param name="cancellationToken">A token to request the task's cancellation.
|
||||
/// If this task is not cancelled quickly, it might be killed by the runner.
|
||||
/// </param>
|
||||
/// <remarks>
|
||||
/// Your task can have any service as a public field and use the <see cref="InjectedAttribute"/>,
|
||||
/// they will be set to an available service from the service container before calling this method.
|
||||
/// They also will be removed after this method return (or throw) to prevent dangling services.
|
||||
/// </remarks>
|
||||
public Task Run([NotNull] TaskParameters arguments,
|
||||
[NotNull] IProgress<float> progress,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
}
|
@ -1,5 +1,7 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Threading;
|
||||
using Kyoo.Models.Exceptions;
|
||||
|
||||
namespace Kyoo.Controllers
|
||||
@ -13,11 +15,56 @@ namespace Kyoo.Controllers
|
||||
/// <summary>
|
||||
/// Start a new task (or queue it).
|
||||
/// </summary>
|
||||
/// <param name="taskSlug">The slug of the task to run</param>
|
||||
/// <param name="arguments">A list of arguments to pass to the task. An automatic conversion will be made if arguments to not fit.</param>
|
||||
/// <exception cref="ArgumentException">If the number of arguments is invalid or if an argument can't be converted.</exception>
|
||||
/// <exception cref="ItemNotFoundException">The task could not be found.</exception>
|
||||
void StartTask(string taskSlug, Dictionary<string, object> arguments = null);
|
||||
/// <param name="taskSlug">
|
||||
/// The slug of the task to run.
|
||||
/// </param>
|
||||
/// <param name="progress">
|
||||
/// A progress reporter to know the percentage of completion of the task.
|
||||
/// </param>
|
||||
/// <param name="arguments">
|
||||
/// A list of arguments to pass to the task. An automatic conversion will be made if arguments to not fit.
|
||||
/// </param>
|
||||
/// <param name="cancellationToken">
|
||||
/// A custom cancellation token for the task.
|
||||
/// </param>
|
||||
/// <exception cref="ArgumentException">
|
||||
/// If the number of arguments is invalid, if an argument can't be converted or if the task finds the argument
|
||||
/// invalid.
|
||||
/// </exception>
|
||||
/// <exception cref="ItemNotFoundException">
|
||||
/// The task could not be found.
|
||||
/// </exception>
|
||||
void StartTask(string taskSlug,
|
||||
[NotNull] IProgress<float> progress,
|
||||
Dictionary<string, object> arguments = null,
|
||||
CancellationToken? cancellationToken = null);
|
||||
|
||||
/// <summary>
|
||||
/// Start a new task (or queue it).
|
||||
/// </summary>
|
||||
/// <param name="progress">
|
||||
/// A progress reporter to know the percentage of completion of the task.
|
||||
/// </param>
|
||||
/// <param name="arguments">
|
||||
/// A list of arguments to pass to the task. An automatic conversion will be made if arguments to not fit.
|
||||
/// </param>
|
||||
/// <typeparam name="T">
|
||||
/// The type of the task to start.
|
||||
/// </typeparam>
|
||||
/// <param name="cancellationToken">
|
||||
/// A custom cancellation token for the task.
|
||||
/// </param>
|
||||
/// <exception cref="ArgumentException">
|
||||
/// If the number of arguments is invalid, if an argument can't be converted or if the task finds the argument
|
||||
/// invalid.
|
||||
/// </exception>
|
||||
/// <exception cref="ItemNotFoundException">
|
||||
/// The task could not be found.
|
||||
/// </exception>
|
||||
void StartTask<T>([NotNull] IProgress<float> progress,
|
||||
Dictionary<string, object> arguments = null,
|
||||
CancellationToken? cancellationToken = null)
|
||||
where T : ITask, new();
|
||||
|
||||
/// <summary>
|
||||
/// Get all currently running tasks
|
||||
|
@ -112,7 +112,7 @@ namespace Kyoo
|
||||
/// <param name="second">Missing fields of first will be completed by fields of this item. If second is null, the function no-op.</param>
|
||||
/// <typeparam name="T">Fields of T will be merged</typeparam>
|
||||
/// <returns><see cref="first"/></returns>
|
||||
[ContractAnnotation("=> null; first:notnull => notnull; second:notnull => notnull")]
|
||||
[ContractAnnotation("first:notnull => notnull; second:notnull => notnull", true)]
|
||||
public static T Merge<T>([CanBeNull] T first, [CanBeNull] T second)
|
||||
{
|
||||
if (first == null)
|
||||
|
@ -56,7 +56,7 @@ namespace Kyoo.TheTvdb
|
||||
{
|
||||
Show show => await _GetShow(show) as T,
|
||||
Episode episode => await _GetEpisode(episode) as T,
|
||||
_ => throw new NotSupportedException()
|
||||
_ => null
|
||||
};
|
||||
}
|
||||
|
||||
@ -66,7 +66,11 @@ namespace Kyoo.TheTvdb
|
||||
if (!int.TryParse(show.GetID(Provider.Slug), out int id))
|
||||
return (await _SearchShow(show.Title)).FirstOrDefault();
|
||||
TvDbResponse<Series> series = await _client.Series.GetAsync(id);
|
||||
return series.Data.ToShow(Provider);
|
||||
Show ret = series.Data.ToShow(Provider);
|
||||
|
||||
TvDbResponse<Actor[]> people = await _client.Series.GetActorsAsync(id);
|
||||
ret.People = people.Data.Select(x => x.ToPeopleRole(Provider)).ToArray();
|
||||
return ret;
|
||||
}
|
||||
|
||||
[ItemCanBeNull]
|
||||
@ -88,7 +92,7 @@ namespace Kyoo.TheTvdb
|
||||
await _Authenticate();
|
||||
if (typeof(T) == typeof(Show))
|
||||
return (await _SearchShow(query) as ICollection<T>)!;
|
||||
throw new NotImplementedException();
|
||||
return ArraySegment<T>.Empty;
|
||||
}
|
||||
|
||||
[ItemNotNull]
|
||||
@ -97,15 +101,5 @@ namespace Kyoo.TheTvdb
|
||||
TvDbResponse<SeriesSearchResult[]> shows = await _client.Search.SearchSeriesByNameAsync(query);
|
||||
return shows.Data.Select(x => x.ToShow(Provider)).ToArray();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<ICollection<PeopleRole>> GetPeople(Show show)
|
||||
{
|
||||
if (!int.TryParse(show?.GetID(Provider.Name), out int id))
|
||||
return null;
|
||||
await _Authenticate();
|
||||
TvDbResponse<Actor[]> people = await _client.Series.GetActorsAsync(id);
|
||||
return people.Data.Select(x => x.ToPeopleRole(Provider)).ToArray();
|
||||
}
|
||||
}
|
||||
}
|
@ -70,13 +70,14 @@ namespace Kyoo.Controllers
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<ICollection<string>> ListFiles(string path)
|
||||
public Task<ICollection<string>> ListFiles(string path, SearchOption options = SearchOption.TopDirectoryOnly)
|
||||
{
|
||||
if (path == null)
|
||||
throw new ArgumentNullException(nameof(path));
|
||||
return Task.FromResult<ICollection<string>>(Directory.Exists(path)
|
||||
? Directory.GetFiles(path)
|
||||
: Array.Empty<string>());
|
||||
string[] ret = Directory.Exists(path)
|
||||
? Directory.GetFiles(path, "*", options)
|
||||
: Array.Empty<string>();
|
||||
return Task.FromResult<ICollection<string>>(ret);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
|
@ -68,17 +68,13 @@ namespace Kyoo.Controllers
|
||||
public async Task<T> Get<T>(T item)
|
||||
where T : class, IResource
|
||||
{
|
||||
T ret = null;
|
||||
|
||||
T ret = item;
|
||||
|
||||
foreach (IMetadataProvider provider in _GetProviders())
|
||||
{
|
||||
try
|
||||
{
|
||||
ret = Merger.Merge(ret, await provider.Get(ret ?? item));
|
||||
}
|
||||
catch (NotSupportedException)
|
||||
{
|
||||
// Silenced
|
||||
ret = Merger.Merge(ret, await provider.Get(ret));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@ -87,7 +83,7 @@ namespace Kyoo.Controllers
|
||||
}
|
||||
}
|
||||
|
||||
return Merger.Merge(ret, item);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@ -102,10 +98,6 @@ namespace Kyoo.Controllers
|
||||
{
|
||||
ret.AddRange(await provider.Search<T>(query));
|
||||
}
|
||||
catch (NotSupportedException)
|
||||
{
|
||||
// Silenced
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "The provider {Provider} could not search for {Type}",
|
||||
@ -115,108 +107,5 @@ namespace Kyoo.Controllers
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
public Task<ICollection<PeopleRole>> GetPeople(Show show)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
// public async Task<Collection> GetCollectionFromName(string name, Library library)
|
||||
// {
|
||||
// Collection collection = await GetMetadata(
|
||||
// provider => provider.GetCollectionFromName(name),
|
||||
// library,
|
||||
// $"the collection {name}");
|
||||
// collection.Name ??= name;
|
||||
// collection.Slug ??= Utility.ToSlug(name);
|
||||
// return collection;
|
||||
// }
|
||||
//
|
||||
// public async Task<Show> CompleteShow(Show show, Library library)
|
||||
// {
|
||||
// return await GetMetadata(provider => provider.GetShowByID(show), library, $"the show {show.Title}");
|
||||
// }
|
||||
//
|
||||
// public async Task<Show> SearchShow(string showName, bool isMovie, Library library)
|
||||
// {
|
||||
// Show show = await GetMetadata(async provider =>
|
||||
// {
|
||||
// Show searchResult = (await provider.SearchShows(showName, isMovie))?.FirstOrDefault();
|
||||
// if (searchResult == null)
|
||||
// return null;
|
||||
// return await provider.GetShowByID(searchResult);
|
||||
// }, library, $"the show {showName}");
|
||||
// show.Slug = Utility.ToSlug(showName);
|
||||
// show.Title ??= showName;
|
||||
// show.IsMovie = isMovie;
|
||||
// show.Genres = show.Genres?.GroupBy(x => x.Slug).Select(x => x.First()).ToList();
|
||||
// show.People = show.People?.GroupBy(x => x.Slug).Select(x => x.First()).ToList();
|
||||
// return show;
|
||||
// }
|
||||
//
|
||||
// public async Task<IEnumerable<Show>> SearchShows(string showName, bool isMovie, Library library)
|
||||
// {
|
||||
// IEnumerable<Show> shows = await GetMetadata(
|
||||
// provider => provider.SearchShows(showName, isMovie),
|
||||
// library,
|
||||
// $"the show {showName}");
|
||||
// return shows.Select(show =>
|
||||
// {
|
||||
// show.Slug = Utility.ToSlug(showName);
|
||||
// show.Title ??= showName;
|
||||
// show.IsMovie = isMovie;
|
||||
// return show;
|
||||
// });
|
||||
// }
|
||||
//
|
||||
// public async Task<Season> GetSeason(Show show, int seasonNumber, Library library)
|
||||
// {
|
||||
// Season season = await GetMetadata(
|
||||
// provider => provider.GetSeason(show, seasonNumber),
|
||||
// library,
|
||||
// $"the season {seasonNumber} of {show.Title}");
|
||||
// season.Show = show;
|
||||
// season.ShowID = show.ID;
|
||||
// season.ShowSlug = show.Slug;
|
||||
// season.Title ??= $"Season {season.SeasonNumber}";
|
||||
// return season;
|
||||
// }
|
||||
//
|
||||
// public async Task<Episode> GetEpisode(Show show,
|
||||
// string episodePath,
|
||||
// int? seasonNumber,
|
||||
// int? episodeNumber,
|
||||
// int? absoluteNumber,
|
||||
// Library library)
|
||||
// {
|
||||
// Episode episode = await GetMetadata(
|
||||
// provider => provider.GetEpisode(show, seasonNumber, episodeNumber, absoluteNumber),
|
||||
// library,
|
||||
// "an episode");
|
||||
// episode.Show = show;
|
||||
// episode.ShowID = show.ID;
|
||||
// episode.ShowSlug = show.Slug;
|
||||
// episode.Path = episodePath;
|
||||
// episode.SeasonNumber ??= seasonNumber;
|
||||
// episode.EpisodeNumber ??= episodeNumber;
|
||||
// episode.AbsoluteNumber ??= absoluteNumber;
|
||||
// return episode;
|
||||
// }
|
||||
//
|
||||
// public async Task<ICollection<PeopleRole>> GetPeople(Show show, Library library)
|
||||
// {
|
||||
// List<PeopleRole> people = await GetMetadata(
|
||||
// provider => provider.GetPeople(show),
|
||||
// library,
|
||||
// $"a cast member of {show.Title}");
|
||||
// return people?.GroupBy(x => x.Slug)
|
||||
// .Select(x => x.First())
|
||||
// .Select(x =>
|
||||
// {
|
||||
// x.Show = show;
|
||||
// x.ShowID = show.ID;
|
||||
// return x;
|
||||
// }).ToList();
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
@ -41,7 +41,7 @@ namespace Kyoo.Controllers
|
||||
/// <summary>
|
||||
/// The queue of tasks that should be run as soon as possible.
|
||||
/// </summary>
|
||||
private readonly Queue<(ITask, Dictionary<string, object>)> _queuedTasks = new();
|
||||
private readonly Queue<(ITask, IProgress<float>, Dictionary<string, object>)> _queuedTasks = new();
|
||||
/// <summary>
|
||||
/// The currently running task.
|
||||
/// </summary>
|
||||
@ -106,11 +106,11 @@ namespace Kyoo.Controllers
|
||||
{
|
||||
if (_queuedTasks.Any())
|
||||
{
|
||||
(ITask task, Dictionary<string, object> arguments) = _queuedTasks.Dequeue();
|
||||
(ITask task, IProgress<float> progress, Dictionary<string, object> args) = _queuedTasks.Dequeue();
|
||||
_runningTask = task;
|
||||
try
|
||||
{
|
||||
await RunTask(task, arguments);
|
||||
await RunTask(task, progress, args);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
@ -129,9 +129,15 @@ namespace Kyoo.Controllers
|
||||
/// Parse parameters, inject a task and run it.
|
||||
/// </summary>
|
||||
/// <param name="task">The task to run</param>
|
||||
/// <param name="progress">A progress reporter to know the percentage of completion of the task.</param>
|
||||
/// <param name="arguments">The arguments to pass to the function</param>
|
||||
/// <exception cref="ArgumentException">There was an invalid argument or a required argument was not found.</exception>
|
||||
private async Task RunTask(ITask task, Dictionary<string, object> arguments)
|
||||
/// <exception cref="ArgumentException">
|
||||
/// If the number of arguments is invalid, if an argument can't be converted or if the task finds the argument
|
||||
/// invalid.
|
||||
/// </exception>
|
||||
private async Task RunTask(ITask task,
|
||||
[NotNull] IProgress<float> progress,
|
||||
Dictionary<string, object> arguments)
|
||||
{
|
||||
_logger.LogInformation("Task starting: {Task}", task.Name);
|
||||
|
||||
@ -160,7 +166,7 @@ namespace Kyoo.Controllers
|
||||
|
||||
using IServiceScope scope = _provider.CreateScope();
|
||||
InjectServices(task, x => scope.ServiceProvider.GetRequiredService(x));
|
||||
await task.Run(args, _taskToken.Token);
|
||||
await task.Run(args, progress, _taskToken.Token);
|
||||
InjectServices(task, _ => null);
|
||||
_logger.LogInformation("Task finished: {Task}", task.Name);
|
||||
}
|
||||
@ -190,7 +196,7 @@ namespace Kyoo.Controllers
|
||||
foreach (string task in tasksToQueue)
|
||||
{
|
||||
_logger.LogDebug("Queuing task scheduled for running: {Task}", task);
|
||||
StartTask(task, new Dictionary<string, object>());
|
||||
StartTask(task, new Progress<float>(), new Dictionary<string, object>());
|
||||
}
|
||||
}
|
||||
|
||||
@ -203,21 +209,33 @@ namespace Kyoo.Controllers
|
||||
.Where(x => x.RunOnStartup)
|
||||
.OrderByDescending(x => x.Priority);
|
||||
foreach (ITask task in startupTasks)
|
||||
_queuedTasks.Enqueue((task, new Dictionary<string, object>()));
|
||||
_queuedTasks.Enqueue((task, new Progress<float>(), new Dictionary<string, object>()));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void StartTask(string taskSlug, Dictionary<string, object> arguments = null)
|
||||
public void StartTask(string taskSlug,
|
||||
IProgress<float> progress,
|
||||
Dictionary<string, object> arguments = null,
|
||||
CancellationToken? cancellationToken = null)
|
||||
{
|
||||
arguments ??= new Dictionary<string, object>();
|
||||
|
||||
int index = _tasks.FindIndex(x => x.task.Slug == taskSlug);
|
||||
if (index == -1)
|
||||
throw new ItemNotFoundException($"No task found with the slug {taskSlug}");
|
||||
_queuedTasks.Enqueue((_tasks[index].task, arguments));
|
||||
_queuedTasks.Enqueue((_tasks[index].task, progress, arguments));
|
||||
_tasks[index] = (_tasks[index].task, GetNextTaskDate(taskSlug));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void StartTask<T>(IProgress<float> progress,
|
||||
Dictionary<string, object> arguments = null,
|
||||
CancellationToken? cancellationToken = null)
|
||||
where T : ITask, new()
|
||||
{
|
||||
StartTask(new T().Slug, progress, arguments, cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Get the next date of the execution of the given task.
|
||||
/// </summary>
|
||||
|
@ -1,6 +1,5 @@
|
||||
using System;
|
||||
using Kyoo.Models;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
@ -11,26 +10,56 @@ using Kyoo.Controllers;
|
||||
using Kyoo.Models.Attributes;
|
||||
using Kyoo.Models.Exceptions;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace Kyoo.Tasks
|
||||
{
|
||||
/// <summary>
|
||||
/// A task to add new video files.
|
||||
/// </summary>
|
||||
public class Crawler : ITask
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public string Slug => "scan";
|
||||
|
||||
/// <inheritdoc />
|
||||
public string Name => "Scan libraries";
|
||||
public string Description => "Scan your libraries, load data for new shows and remove shows that don't exist anymore.";
|
||||
public string HelpMessage => "Reloading all libraries is a long process and may take up to 24 hours if it is the first scan in a while.";
|
||||
|
||||
/// <inheritdoc />
|
||||
public string Description => "Scan your libraries and load data for new shows.";
|
||||
|
||||
/// <inheritdoc />
|
||||
public string HelpMessage => "Reloading all libraries is a long process and may take up to" +
|
||||
" 24 hours if it is the first scan in a while.";
|
||||
|
||||
/// <inheritdoc />
|
||||
public bool RunOnStartup => true;
|
||||
|
||||
/// <inheritdoc />
|
||||
public int Priority => 0;
|
||||
|
||||
[Injected] public IServiceProvider ServiceProvider { private get; set; }
|
||||
[Injected] public IThumbnailsManager ThumbnailsManager { private get; set; }
|
||||
[Injected] public IMetadataProvider MetadataProvider { private get; set; }
|
||||
[Injected] public ITranscoder Transcoder { private get; set; }
|
||||
[Injected] public IConfiguration Config { private get; set; }
|
||||
|
||||
private int _parallelTasks;
|
||||
/// <inheritdoc />
|
||||
public bool IsHidden => false;
|
||||
|
||||
/// <summary>
|
||||
/// The library manager used to get libraries and providers to use.
|
||||
/// </summary>
|
||||
[Injected] public ILibraryManager LibraryManager { private get; set; }
|
||||
/// <summary>
|
||||
/// The file manager used walk inside directories and check they existences.
|
||||
/// </summary>
|
||||
[Injected] public IFileManager FileManager { private get; set; }
|
||||
/// <summary>
|
||||
/// A task manager used to create sub tasks for each episode to add to the database.
|
||||
/// </summary>
|
||||
[Injected] public ITaskManager TaskManager { private get; set; }
|
||||
/// <summary>
|
||||
/// The logger used to inform the current status to the console.
|
||||
/// </summary>
|
||||
[Injected] public ILogger<Crawler> Logger { private get; set; }
|
||||
|
||||
|
||||
/// <inheritdoc />
|
||||
public TaskParameters GetParameters()
|
||||
{
|
||||
return new()
|
||||
@ -39,354 +68,83 @@ namespace Kyoo.Tasks
|
||||
};
|
||||
}
|
||||
|
||||
public int? Progress()
|
||||
/// <inheritdoc />
|
||||
public async Task Run(TaskParameters arguments, IProgress<float> progress, CancellationToken cancellationToken)
|
||||
{
|
||||
// TODO implement this later.
|
||||
return null;
|
||||
}
|
||||
|
||||
public async Task Run(TaskParameters parameters,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
string argument = parameters["slug"].As<string>();
|
||||
|
||||
_parallelTasks = Config.GetValue<int>("parallelTasks");
|
||||
if (_parallelTasks <= 0)
|
||||
_parallelTasks = 30;
|
||||
|
||||
using IServiceScope serviceScope = ServiceProvider.CreateScope();
|
||||
ILibraryManager libraryManager = serviceScope.ServiceProvider.GetService<ILibraryManager>();
|
||||
|
||||
foreach (Show show in await libraryManager!.GetAll<Show>())
|
||||
if (!Directory.Exists(show.Path))
|
||||
await libraryManager.Delete(show);
|
||||
|
||||
ICollection<Episode> episodes = await libraryManager.GetAll<Episode>();
|
||||
foreach (Episode episode in episodes)
|
||||
if (!File.Exists(episode.Path))
|
||||
await libraryManager.Delete(episode);
|
||||
|
||||
ICollection<Track> tracks = await libraryManager.GetAll<Track>();
|
||||
foreach (Track track in tracks)
|
||||
if (!File.Exists(track.Path))
|
||||
await libraryManager.Delete(track);
|
||||
|
||||
string argument = arguments["slug"].As<string>();
|
||||
ICollection<Library> libraries = argument == null
|
||||
? await libraryManager.GetAll<Library>()
|
||||
: new [] { await libraryManager.GetOrDefault<Library>(argument)};
|
||||
|
||||
? await LibraryManager.GetAll<Library>()
|
||||
: new [] { await LibraryManager.GetOrDefault<Library>(argument)};
|
||||
|
||||
if (argument != null && libraries.First() == null)
|
||||
throw new ArgumentException($"No library found with the name {argument}");
|
||||
|
||||
|
||||
foreach (Library library in libraries)
|
||||
await libraryManager.Load(library, x => x.Providers);
|
||||
await LibraryManager.Load(library, x => x.Providers);
|
||||
|
||||
progress.Report(0);
|
||||
float percent = 0;
|
||||
|
||||
ICollection<Episode> episodes = await LibraryManager.GetAll<Episode>();
|
||||
foreach (Library library in libraries)
|
||||
await Scan(library, episodes, tracks, cancellationToken);
|
||||
Console.WriteLine("Scan finished!");
|
||||
{
|
||||
IProgress<float> reporter = new Progress<float>(x =>
|
||||
{
|
||||
// ReSharper disable once AccessToModifiedClosure
|
||||
progress.Report(percent + x / libraries.Count);
|
||||
});
|
||||
await Scan(library, episodes, reporter, cancellationToken);
|
||||
percent += 100f / libraries.Count;
|
||||
}
|
||||
|
||||
progress.Report(100);
|
||||
}
|
||||
|
||||
private async Task Scan(Library library, IEnumerable<Episode> episodes, IEnumerable<Track> tracks, CancellationToken cancellationToken)
|
||||
private async Task Scan(Library library,
|
||||
IEnumerable<Episode> episodes,
|
||||
IProgress<float> progress,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
Console.WriteLine($"Scanning library {library.Name} at {string.Join(", ", library.Paths)}.");
|
||||
Logger.LogInformation("Scanning library {Library} at {Paths}", library.Name, library.Paths);
|
||||
foreach (string path in library.Paths)
|
||||
{
|
||||
if (cancellationToken.IsCancellationRequested)
|
||||
continue;
|
||||
|
||||
string[] files;
|
||||
try
|
||||
{
|
||||
files = Directory.GetFiles(path, "*", SearchOption.AllDirectories);
|
||||
}
|
||||
catch (DirectoryNotFoundException)
|
||||
{
|
||||
await Console.Error.WriteLineAsync($"The library's directory {path} could not be found (library slug: {library.Slug})");
|
||||
continue;
|
||||
}
|
||||
catch (PathTooLongException)
|
||||
{
|
||||
await Console.Error.WriteLineAsync($"The library's directory {path} is too long for this system. (library slug: {library.Slug})");
|
||||
continue;
|
||||
}
|
||||
catch (ArgumentException)
|
||||
{
|
||||
await Console.Error.WriteLineAsync($"The library's directory {path} is invalid. (library slug: {library.Slug})");
|
||||
continue;
|
||||
}
|
||||
catch (UnauthorizedAccessException ex)
|
||||
{
|
||||
await Console.Error.WriteLineAsync($"{ex.Message} (library slug: {library.Slug})");
|
||||
continue;
|
||||
}
|
||||
return;
|
||||
|
||||
List<IGrouping<string, string>> shows = files
|
||||
.Where(x => IsVideo(x) && episodes.All(y => y.Path != x))
|
||||
ICollection<string> files = await FileManager.ListFiles(path, SearchOption.AllDirectories);
|
||||
|
||||
// We try to group episodes by shows to register one episode of each show first.
|
||||
// This speeds up the scan process because further episodes of a show are registered when all metadata
|
||||
// of the show has already been fetched.
|
||||
List<IGrouping<string, string>> shows = files
|
||||
.Where(IsVideo)
|
||||
.Where(x => episodes.All(y => y.Path != x))
|
||||
.GroupBy(Path.GetDirectoryName)
|
||||
.ToList();
|
||||
string[] paths = shows.Select(x => x.First())
|
||||
.Concat(shows.SelectMany(x => x.Skip(1)))
|
||||
.ToArray();
|
||||
|
||||
// TODO If the library's path end with a /, the regex is broken.
|
||||
IEnumerable<string> tasks = shows.Select(x => x.First());
|
||||
foreach (string[] showTasks in tasks.BatchBy(_parallelTasks))
|
||||
await Task.WhenAll(showTasks
|
||||
.Select(x => RegisterFile(x, x.Substring(path.Length), library, cancellationToken)));
|
||||
|
||||
tasks = shows.SelectMany(x => x.Skip(1));
|
||||
foreach (string[] episodeTasks in tasks.BatchBy(_parallelTasks * 2))
|
||||
await Task.WhenAll(episodeTasks
|
||||
.Select(x => RegisterFile(x, x.Substring(path.Length), library, cancellationToken)));
|
||||
float percent = 0;
|
||||
IProgress<float> reporter = new Progress<float>(x =>
|
||||
{
|
||||
// ReSharper disable once AccessToModifiedClosure
|
||||
progress.Report((percent + x / paths.Length) / library.Paths.Length);
|
||||
});
|
||||
|
||||
await Task.WhenAll(files.Where(x => IsSubtitle(x) && tracks.All(y => y.Path != x))
|
||||
.Select(x => RegisterExternalSubtitle(x, cancellationToken)));
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RegisterExternalSubtitle(string path, CancellationToken token)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (token.IsCancellationRequested || path.Split(Path.DirectorySeparatorChar).Contains("Subtitles"))
|
||||
return;
|
||||
using IServiceScope serviceScope = ServiceProvider.CreateScope();
|
||||
ILibraryManager libraryManager = serviceScope.ServiceProvider.GetService<ILibraryManager>();
|
||||
|
||||
string patern = Config.GetValue<string>("subtitleRegex");
|
||||
Regex regex = new(patern, RegexOptions.IgnoreCase);
|
||||
Match match = regex.Match(path);
|
||||
|
||||
if (!match.Success)
|
||||
foreach (string episodePath in paths)
|
||||
{
|
||||
await Console.Error.WriteLineAsync($"The subtitle at {path} does not match the subtitle's regex.");
|
||||
return;
|
||||
TaskManager.StartTask<RegisterEpisode>(reporter, new Dictionary<string, object>
|
||||
{
|
||||
["path"] = episodePath[path.Length..],
|
||||
["library"] = library
|
||||
}, cancellationToken);
|
||||
percent += 100f / paths.Length;
|
||||
}
|
||||
|
||||
string episodePath = match.Groups["Episode"].Value;
|
||||
Episode episode = await libraryManager!.Get<Episode>(x => x.Path.StartsWith(episodePath));
|
||||
Track track = new()
|
||||
{
|
||||
Type = StreamType.Subtitle,
|
||||
Language = match.Groups["Language"].Value,
|
||||
IsDefault = match.Groups["Default"].Value.Length > 0,
|
||||
IsForced = match.Groups["Forced"].Value.Length > 0,
|
||||
Codec = SubtitleExtensions[Path.GetExtension(path)],
|
||||
IsExternal = true,
|
||||
Path = path,
|
||||
Episode = episode
|
||||
};
|
||||
|
||||
await libraryManager.Create(track);
|
||||
Console.WriteLine($"Registering subtitle at: {path}.");
|
||||
// await Task.WhenAll(files.Where(x => IsSubtitle(x) && tracks.All(y => y.Path != x))
|
||||
// .Select(x => RegisterExternalSubtitle(x, cancellationToken)));
|
||||
}
|
||||
catch (ItemNotFoundException)
|
||||
{
|
||||
await Console.Error.WriteLineAsync($"No episode found for subtitle at: ${path}.");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
await Console.Error.WriteLineAsync($"Unknown error while registering subtitle: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RegisterFile(string path, string relativePath, Library library, CancellationToken token)
|
||||
{
|
||||
if (token.IsCancellationRequested)
|
||||
return;
|
||||
|
||||
try
|
||||
{
|
||||
using IServiceScope serviceScope = ServiceProvider.CreateScope();
|
||||
ILibraryManager libraryManager = serviceScope.ServiceProvider.GetService<ILibraryManager>();
|
||||
|
||||
string patern = Config.GetValue<string>("regex");
|
||||
Regex regex = new(patern, RegexOptions.IgnoreCase);
|
||||
Match match = regex.Match(relativePath);
|
||||
|
||||
if (!match.Success)
|
||||
{
|
||||
await Console.Error.WriteLineAsync($"The episode at {path} does not match the episode's regex.");
|
||||
return;
|
||||
}
|
||||
|
||||
string showPath = Path.GetDirectoryName(path);
|
||||
string collectionName = match.Groups["Collection"].Value;
|
||||
string showName = match.Groups["Show"].Value;
|
||||
int? seasonNumber = int.TryParse(match.Groups["Season"].Value, out int tmp) ? tmp : null;
|
||||
int? episodeNumber = int.TryParse(match.Groups["Episode"].Value, out tmp) ? tmp : null;
|
||||
int? absoluteNumber = int.TryParse(match.Groups["Absolute"].Value, out tmp) ? tmp : null;
|
||||
|
||||
Collection collection = await GetCollection(libraryManager, collectionName, library);
|
||||
bool isMovie = seasonNumber == null && episodeNumber == null && absoluteNumber == null;
|
||||
Show show = await GetShow(libraryManager, showName, showPath, isMovie, library);
|
||||
if (isMovie)
|
||||
await libraryManager!.Create(await GetMovie(show, path));
|
||||
else
|
||||
{
|
||||
Season season = seasonNumber != null
|
||||
? await GetSeason(libraryManager, show, seasonNumber.Value, library)
|
||||
: null;
|
||||
Episode episode = await GetEpisode(libraryManager,
|
||||
show,
|
||||
season,
|
||||
episodeNumber,
|
||||
absoluteNumber,
|
||||
path,
|
||||
library);
|
||||
await libraryManager!.Create(episode);
|
||||
}
|
||||
|
||||
await libraryManager.AddShowLink(show, library, collection);
|
||||
Console.WriteLine($"Episode at {path} registered.");
|
||||
}
|
||||
catch (DuplicatedItemException ex)
|
||||
{
|
||||
await Console.Error.WriteLineAsync($"{path}: {ex.Message}");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
await Console.Error.WriteLineAsync($"Unknown exception thrown while registering episode at {path}." +
|
||||
$"\nException: {ex.Message}" +
|
||||
$"\n{ex.StackTrace}");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<Collection> GetCollection(ILibraryManager libraryManager,
|
||||
string collectionName,
|
||||
Library library)
|
||||
{
|
||||
if (string.IsNullOrEmpty(collectionName))
|
||||
return null;
|
||||
Collection collection = await libraryManager.GetOrDefault<Collection>(Utility.ToSlug(collectionName));
|
||||
if (collection != null)
|
||||
return collection;
|
||||
// collection = await MetadataProvider.GetCollectionFromName(collectionName, library);
|
||||
|
||||
try
|
||||
{
|
||||
await libraryManager.Create(collection);
|
||||
return collection;
|
||||
}
|
||||
catch (DuplicatedItemException)
|
||||
{
|
||||
return await libraryManager.GetOrDefault<Collection>(collection.Slug);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<Show> GetShow(ILibraryManager libraryManager,
|
||||
string showTitle,
|
||||
string showPath,
|
||||
bool isMovie,
|
||||
Library library)
|
||||
{
|
||||
Show old = await libraryManager.GetOrDefault<Show>(x => x.Path == showPath);
|
||||
if (old != null)
|
||||
{
|
||||
await libraryManager.Load(old, x => x.ExternalIDs);
|
||||
return old;
|
||||
}
|
||||
|
||||
Show show = new();//await MetadataProvider.SearchShow(showTitle, isMovie, library);
|
||||
show.Path = showPath;
|
||||
// show.People = await MetadataProvider.GetPeople(show, library);
|
||||
|
||||
try
|
||||
{
|
||||
show = await libraryManager.Create(show);
|
||||
}
|
||||
catch (DuplicatedItemException)
|
||||
{
|
||||
old = await libraryManager.GetOrDefault<Show>(show.Slug);
|
||||
if (old != null && old.Path == showPath)
|
||||
{
|
||||
await libraryManager.Load(old, x => x.ExternalIDs);
|
||||
return old;
|
||||
}
|
||||
|
||||
if (show.StartAir != null)
|
||||
{
|
||||
show.Slug += $"-{show.StartAir.Value.Year}";
|
||||
await libraryManager.Create(show);
|
||||
}
|
||||
else
|
||||
throw;
|
||||
}
|
||||
await ThumbnailsManager.Validate(show);
|
||||
return show;
|
||||
}
|
||||
|
||||
private async Task<Season> GetSeason(ILibraryManager libraryManager,
|
||||
Show show,
|
||||
int seasonNumber,
|
||||
Library library)
|
||||
{
|
||||
try
|
||||
{
|
||||
Season season = await libraryManager.Get(show.Slug, seasonNumber);
|
||||
season.Show = show;
|
||||
return season;
|
||||
}
|
||||
catch (ItemNotFoundException)
|
||||
{
|
||||
Season season = new();//await MetadataProvider.GetSeason(show, seasonNumber, library);
|
||||
try
|
||||
{
|
||||
await libraryManager.Create(season);
|
||||
await ThumbnailsManager.Validate(season);
|
||||
}
|
||||
catch (DuplicatedItemException)
|
||||
{
|
||||
season = await libraryManager.Get(show.Slug, seasonNumber);
|
||||
}
|
||||
season.Show = show;
|
||||
return season;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<Episode> GetEpisode(ILibraryManager libraryManager,
|
||||
Show show,
|
||||
Season season,
|
||||
int? episodeNumber,
|
||||
int? absoluteNumber,
|
||||
string episodePath,
|
||||
Library library)
|
||||
{
|
||||
Episode episode = new();/*await MetadataProvider.GetEpisode(show,
|
||||
episodePath,
|
||||
season?.SeasonNumber,
|
||||
episodeNumber,
|
||||
absoluteNumber,
|
||||
library);*/
|
||||
|
||||
if (episode.SeasonNumber != null)
|
||||
{
|
||||
season ??= await GetSeason(libraryManager, show, episode.SeasonNumber.Value, library);
|
||||
episode.Season = season;
|
||||
episode.SeasonID = season?.ID;
|
||||
}
|
||||
await ThumbnailsManager.Validate(episode);
|
||||
await GetTracks(episode);
|
||||
return episode;
|
||||
}
|
||||
|
||||
private async Task<Episode> GetMovie(Show show, string episodePath)
|
||||
{
|
||||
Episode episode = new()
|
||||
{
|
||||
Title = show.Title,
|
||||
Path = episodePath,
|
||||
Show = show,
|
||||
ShowID = show.ID,
|
||||
ShowSlug = show.Slug
|
||||
};
|
||||
episode.Tracks = await GetTracks(episode);
|
||||
return episode;
|
||||
}
|
||||
|
||||
private async Task<ICollection<Track>> GetTracks(Episode episode)
|
||||
{
|
||||
episode.Tracks = (await Transcoder.ExtractInfos(episode, false))
|
||||
.Where(x => x.Type != StreamType.Attachment)
|
||||
.ToArray();
|
||||
return episode.Tracks;
|
||||
}
|
||||
|
||||
private static readonly string[] VideoExtensions =
|
||||
|
@ -1,66 +0,0 @@
|
||||
// using System;
|
||||
// using System.Collections.Generic;
|
||||
// using System.Linq;
|
||||
// using System.Threading;
|
||||
// using System.Threading.Tasks;
|
||||
// using IdentityServer4.EntityFramework.DbContexts;
|
||||
// using IdentityServer4.EntityFramework.Mappers;
|
||||
// using IdentityServer4.Models;
|
||||
// using Kyoo.Models;
|
||||
// using Microsoft.EntityFrameworkCore;
|
||||
// using Microsoft.Extensions.DependencyInjection;
|
||||
//
|
||||
// namespace Kyoo.Tasks
|
||||
// {
|
||||
// public class CreateDatabase : ITask
|
||||
// {
|
||||
// public string Slug => "create-database";
|
||||
// public string Name => "Create the database";
|
||||
// public string Description => "Create the database if it does not exit and initialize it with defaults value.";
|
||||
// public string HelpMessage => null;
|
||||
// public bool RunOnStartup => true;
|
||||
// public int Priority => int.MaxValue;
|
||||
//
|
||||
// public Task Run(IServiceProvider serviceProvider, CancellationToken cancellationToken, string arguments = null)
|
||||
// {
|
||||
// using IServiceScope serviceScope = serviceProvider.CreateScope();
|
||||
// DatabaseContext databaseContext = serviceScope.ServiceProvider.GetService<DatabaseContext>();
|
||||
// IdentityDatabase identityDatabase = serviceScope.ServiceProvider.GetService<IdentityDatabase>();
|
||||
// ConfigurationDbContext identityContext = serviceScope.ServiceProvider.GetService<ConfigurationDbContext>();
|
||||
//
|
||||
// databaseContext!.Database.Migrate();
|
||||
// identityDatabase!.Database.Migrate();
|
||||
// identityContext!.Database.Migrate();
|
||||
//
|
||||
// if (!identityContext.Clients.Any())
|
||||
// {
|
||||
// foreach (Client client in IdentityContext.GetClients())
|
||||
// identityContext.Clients.Add(client.ToEntity());
|
||||
// identityContext.SaveChanges();
|
||||
// }
|
||||
// if (!identityContext.IdentityResources.Any())
|
||||
// {
|
||||
// foreach (IdentityResource resource in IdentityContext.GetIdentityResources())
|
||||
// identityContext.IdentityResources.Add(resource.ToEntity());
|
||||
// identityContext.SaveChanges();
|
||||
// }
|
||||
// if (!identityContext.ApiResources.Any())
|
||||
// {
|
||||
// foreach (ApiResource resource in IdentityContext.GetApis())
|
||||
// identityContext.ApiResources.Add(resource.ToEntity());
|
||||
// identityContext.SaveChanges();
|
||||
// }
|
||||
// return Task.CompletedTask;
|
||||
// }
|
||||
//
|
||||
// public Task<IEnumerable<string>> GetPossibleParameters()
|
||||
// {
|
||||
// return Task.FromResult<IEnumerable<string>>(null);
|
||||
// }
|
||||
//
|
||||
// public int? Progress()
|
||||
// {
|
||||
// return null;
|
||||
// }
|
||||
// }
|
||||
// }
|
80
Kyoo/Tasks/Housekeeping.cs
Normal file
80
Kyoo/Tasks/Housekeeping.cs
Normal file
@ -0,0 +1,80 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Kyoo.Controllers;
|
||||
using Kyoo.Models;
|
||||
using Kyoo.Models.Attributes;
|
||||
|
||||
namespace Kyoo.Tasks
|
||||
{
|
||||
public class Housekeeping : ITask
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public string Slug => "housekeeping";
|
||||
|
||||
/// <inheritdoc />
|
||||
public string Name => "Housekeeping";
|
||||
|
||||
/// <inheritdoc />
|
||||
public string Description => "Remove orphaned episode and series.";
|
||||
|
||||
/// <inheritdoc />
|
||||
public string HelpMessage => null;
|
||||
|
||||
/// <inheritdoc />
|
||||
public bool RunOnStartup => true;
|
||||
|
||||
/// <inheritdoc />
|
||||
public int Priority => 0;
|
||||
|
||||
/// <inheritdoc />
|
||||
public bool IsHidden => false;
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// The library manager used to get libraries or remove deleted episodes
|
||||
/// </summary>
|
||||
[Injected] public ILibraryManager LibraryManager { private get; set; }
|
||||
/// <summary>
|
||||
/// The file manager used walk inside directories and check they existences.
|
||||
/// </summary>
|
||||
[Injected] public IFileManager FileManager { private get; set; }
|
||||
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task Run(TaskParameters arguments, IProgress<float> progress, CancellationToken cancellationToken)
|
||||
{
|
||||
int count = 0;
|
||||
int delCount = await LibraryManager.GetCount<Show>() + await LibraryManager.GetCount<Episode>();
|
||||
progress.Report(0);
|
||||
|
||||
foreach (Show show in await LibraryManager.GetAll<Show>())
|
||||
{
|
||||
progress.Report(count / delCount * 100);
|
||||
count++;
|
||||
|
||||
if (await FileManager.Exists(show.Path))
|
||||
continue;
|
||||
await LibraryManager.Delete(show);
|
||||
}
|
||||
|
||||
foreach (Episode episode in await LibraryManager.GetAll<Episode>())
|
||||
{
|
||||
progress.Report(count / delCount * 100);
|
||||
count++;
|
||||
|
||||
if (await FileManager.Exists(episode.Path))
|
||||
continue;
|
||||
await LibraryManager.Delete(episode);
|
||||
}
|
||||
|
||||
progress.Report(100);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public TaskParameters GetParameters()
|
||||
{
|
||||
return new();
|
||||
}
|
||||
}
|
||||
}
|
@ -1,4 +1,5 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Kyoo.Controllers;
|
||||
@ -28,8 +29,11 @@ namespace Kyoo.Tasks
|
||||
|
||||
/// <inheritdoc />
|
||||
public int Priority => int.MaxValue;
|
||||
|
||||
|
||||
|
||||
/// <inheritdoc />
|
||||
public bool IsHidden => true;
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// The plugin manager used to retrieve plugins to initialize them.
|
||||
/// </summary>
|
||||
@ -40,21 +44,28 @@ namespace Kyoo.Tasks
|
||||
[Injected] public IServiceProvider Provider { private get; set; }
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task Run(TaskParameters arguments, CancellationToken cancellationToken)
|
||||
public Task Run(TaskParameters arguments, IProgress<float> progress, CancellationToken cancellationToken)
|
||||
{
|
||||
foreach (IPlugin plugin in PluginManager.GetAllPlugins())
|
||||
ICollection<IPlugin> plugins = PluginManager.GetAllPlugins();
|
||||
int count = 0;
|
||||
progress.Report(0);
|
||||
|
||||
foreach (IPlugin plugin in plugins)
|
||||
{
|
||||
plugin.Initialize(Provider);
|
||||
|
||||
progress.Report(count / plugins.Count * 100);
|
||||
count++;
|
||||
}
|
||||
|
||||
progress.Report(100);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public TaskParameters GetParameters()
|
||||
{
|
||||
return new();
|
||||
}
|
||||
|
||||
public int? Progress()
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
@ -1,37 +0,0 @@
|
||||
// using System;
|
||||
// using System.Collections.Generic;
|
||||
// using System.Threading;
|
||||
// using System.Threading.Tasks;
|
||||
// using Kyoo.Controllers;
|
||||
// using Kyoo.Models;
|
||||
// using Microsoft.Extensions.DependencyInjection;
|
||||
//
|
||||
// namespace Kyoo.Tasks
|
||||
// {
|
||||
// public class PluginLoader : ITask
|
||||
// {
|
||||
// public string Slug => "reload-plugin";
|
||||
// public string Name => "Reload plugins";
|
||||
// public string Description => "Reload all plugins from the plugin folder.";
|
||||
// public string HelpMessage => null;
|
||||
// public bool RunOnStartup => true;
|
||||
// public int Priority => Int32.MaxValue;
|
||||
// public Task Run(IServiceProvider serviceProvider, CancellationToken cancellationToken, string arguments = null)
|
||||
// {
|
||||
// using IServiceScope serviceScope = serviceProvider.CreateScope();
|
||||
// IPluginManager pluginManager = serviceScope.ServiceProvider.GetService<IPluginManager>();
|
||||
// pluginManager.ReloadPlugins();
|
||||
// return Task.CompletedTask;
|
||||
// }
|
||||
//
|
||||
// public Task<IEnumerable<string>> GetPossibleParameters()
|
||||
// {
|
||||
// return Task.FromResult<IEnumerable<string>>(null);
|
||||
// }
|
||||
//
|
||||
// public int? Progress()
|
||||
// {
|
||||
// return null;
|
||||
// }
|
||||
// }
|
||||
// }
|
305
Kyoo/Tasks/RegisterEpisode.cs
Normal file
305
Kyoo/Tasks/RegisterEpisode.cs
Normal file
@ -0,0 +1,305 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Kyoo.Controllers;
|
||||
using Kyoo.Models;
|
||||
|
||||
namespace Kyoo.Tasks
|
||||
{
|
||||
/// <summary>
|
||||
/// A task to register a new episode
|
||||
/// </summary>
|
||||
public class RegisterEpisode : ITask
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public string Slug => "register";
|
||||
|
||||
/// <inheritdoc />
|
||||
public string Name => "Register episode";
|
||||
|
||||
/// <inheritdoc />
|
||||
public string Description => "Register a new episode";
|
||||
|
||||
/// <inheritdoc />
|
||||
public string HelpMessage => null;
|
||||
|
||||
/// <inheritdoc />
|
||||
public bool RunOnStartup => false;
|
||||
|
||||
/// <inheritdoc />
|
||||
public int Priority => 0;
|
||||
|
||||
/// <inheritdoc />
|
||||
public bool IsHidden => false;
|
||||
|
||||
/// <inheritdoc />
|
||||
public TaskParameters GetParameters()
|
||||
{
|
||||
return new()
|
||||
{
|
||||
TaskParameter.CreateRequired<string>("path", "The path of the episode file"),
|
||||
TaskParameter.CreateRequired<Library>("library", "The library in witch the episode is")
|
||||
};
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task Run(TaskParameters arguments, IProgress<float> progress, CancellationToken cancellationToken)
|
||||
{
|
||||
string path = arguments["path"].As<string>();
|
||||
Library library = arguments["library"].As<Library>();
|
||||
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
* private async Task RegisterExternalSubtitle(string path, CancellationToken token)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (token.IsCancellationRequested || path.Split(Path.DirectorySeparatorChar).Contains("Subtitles"))
|
||||
return;
|
||||
using IServiceScope serviceScope = ServiceProvider.CreateScope();
|
||||
ILibraryManager libraryManager = serviceScope.ServiceProvider.GetService<ILibraryManager>();
|
||||
|
||||
string patern = Config.GetValue<string>("subtitleRegex");
|
||||
Regex regex = new(patern, RegexOptions.IgnoreCase);
|
||||
Match match = regex.Match(path);
|
||||
|
||||
if (!match.Success)
|
||||
{
|
||||
await Console.Error.WriteLineAsync($"The subtitle at {path} does not match the subtitle's regex.");
|
||||
return;
|
||||
}
|
||||
|
||||
string episodePath = match.Groups["Episode"].Value;
|
||||
Episode episode = await libraryManager!.Get<Episode>(x => x.Path.StartsWith(episodePath));
|
||||
Track track = new()
|
||||
{
|
||||
Type = StreamType.Subtitle,
|
||||
Language = match.Groups["Language"].Value,
|
||||
IsDefault = match.Groups["Default"].Value.Length > 0,
|
||||
IsForced = match.Groups["Forced"].Value.Length > 0,
|
||||
Codec = SubtitleExtensions[Path.GetExtension(path)],
|
||||
IsExternal = true,
|
||||
Path = path,
|
||||
Episode = episode
|
||||
};
|
||||
|
||||
await libraryManager.Create(track);
|
||||
Console.WriteLine($"Registering subtitle at: {path}.");
|
||||
}
|
||||
catch (ItemNotFoundException)
|
||||
{
|
||||
await Console.Error.WriteLineAsync($"No episode found for subtitle at: ${path}.");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
await Console.Error.WriteLineAsync($"Unknown error while registering subtitle: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RegisterFile(string path, string relativePath, Library library, CancellationToken token)
|
||||
{
|
||||
if (token.IsCancellationRequested)
|
||||
return;
|
||||
|
||||
try
|
||||
{
|
||||
using IServiceScope serviceScope = ServiceProvider.CreateScope();
|
||||
ILibraryManager libraryManager = serviceScope.ServiceProvider.GetService<ILibraryManager>();
|
||||
|
||||
string patern = Config.GetValue<string>("regex");
|
||||
Regex regex = new(patern, RegexOptions.IgnoreCase);
|
||||
Match match = regex.Match(relativePath);
|
||||
|
||||
if (!match.Success)
|
||||
{
|
||||
await Console.Error.WriteLineAsync($"The episode at {path} does not match the episode's regex.");
|
||||
return;
|
||||
}
|
||||
|
||||
string showPath = Path.GetDirectoryName(path);
|
||||
string collectionName = match.Groups["Collection"].Value;
|
||||
string showName = match.Groups["Show"].Value;
|
||||
int? seasonNumber = int.TryParse(match.Groups["Season"].Value, out int tmp) ? tmp : null;
|
||||
int? episodeNumber = int.TryParse(match.Groups["Episode"].Value, out tmp) ? tmp : null;
|
||||
int? absoluteNumber = int.TryParse(match.Groups["Absolute"].Value, out tmp) ? tmp : null;
|
||||
|
||||
Collection collection = await GetCollection(libraryManager, collectionName, library);
|
||||
bool isMovie = seasonNumber == null && episodeNumber == null && absoluteNumber == null;
|
||||
Show show = await GetShow(libraryManager, showName, showPath, isMovie, library);
|
||||
if (isMovie)
|
||||
await libraryManager!.Create(await GetMovie(show, path));
|
||||
else
|
||||
{
|
||||
Season season = seasonNumber != null
|
||||
? await GetSeason(libraryManager, show, seasonNumber.Value, library)
|
||||
: null;
|
||||
Episode episode = await GetEpisode(libraryManager,
|
||||
show,
|
||||
season,
|
||||
episodeNumber,
|
||||
absoluteNumber,
|
||||
path,
|
||||
library);
|
||||
await libraryManager!.Create(episode);
|
||||
}
|
||||
|
||||
await libraryManager.AddShowLink(show, library, collection);
|
||||
Console.WriteLine($"Episode at {path} registered.");
|
||||
}
|
||||
catch (DuplicatedItemException ex)
|
||||
{
|
||||
await Console.Error.WriteLineAsync($"{path}: {ex.Message}");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
await Console.Error.WriteLineAsync($"Unknown exception thrown while registering episode at {path}." +
|
||||
$"\nException: {ex.Message}" +
|
||||
$"\n{ex.StackTrace}");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<Collection> GetCollection(ILibraryManager libraryManager,
|
||||
string collectionName,
|
||||
Library library)
|
||||
{
|
||||
if (string.IsNullOrEmpty(collectionName))
|
||||
return null;
|
||||
Collection collection = await libraryManager.GetOrDefault<Collection>(Utility.ToSlug(collectionName));
|
||||
if (collection != null)
|
||||
return collection;
|
||||
// collection = await MetadataProvider.GetCollectionFromName(collectionName, library);
|
||||
|
||||
try
|
||||
{
|
||||
await libraryManager.Create(collection);
|
||||
return collection;
|
||||
}
|
||||
catch (DuplicatedItemException)
|
||||
{
|
||||
return await libraryManager.GetOrDefault<Collection>(collection.Slug);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<Show> GetShow(ILibraryManager libraryManager,
|
||||
string showTitle,
|
||||
string showPath,
|
||||
bool isMovie,
|
||||
Library library)
|
||||
{
|
||||
Show old = await libraryManager.GetOrDefault<Show>(x => x.Path == showPath);
|
||||
if (old != null)
|
||||
{
|
||||
await libraryManager.Load(old, x => x.ExternalIDs);
|
||||
return old;
|
||||
}
|
||||
|
||||
Show show = new();//await MetadataProvider.SearchShow(showTitle, isMovie, library);
|
||||
show.Path = showPath;
|
||||
// show.People = await MetadataProvider.GetPeople(show, library);
|
||||
|
||||
try
|
||||
{
|
||||
show = await libraryManager.Create(show);
|
||||
}
|
||||
catch (DuplicatedItemException)
|
||||
{
|
||||
old = await libraryManager.GetOrDefault<Show>(show.Slug);
|
||||
if (old != null && old.Path == showPath)
|
||||
{
|
||||
await libraryManager.Load(old, x => x.ExternalIDs);
|
||||
return old;
|
||||
}
|
||||
|
||||
if (show.StartAir != null)
|
||||
{
|
||||
show.Slug += $"-{show.StartAir.Value.Year}";
|
||||
await libraryManager.Create(show);
|
||||
}
|
||||
else
|
||||
throw;
|
||||
}
|
||||
await ThumbnailsManager.Validate(show);
|
||||
return show;
|
||||
}
|
||||
|
||||
private async Task<Season> GetSeason(ILibraryManager libraryManager,
|
||||
Show show,
|
||||
int seasonNumber,
|
||||
Library library)
|
||||
{
|
||||
try
|
||||
{
|
||||
Season season = await libraryManager.Get(show.Slug, seasonNumber);
|
||||
season.Show = show;
|
||||
return season;
|
||||
}
|
||||
catch (ItemNotFoundException)
|
||||
{
|
||||
Season season = new();//await MetadataProvider.GetSeason(show, seasonNumber, library);
|
||||
try
|
||||
{
|
||||
await libraryManager.Create(season);
|
||||
await ThumbnailsManager.Validate(season);
|
||||
}
|
||||
catch (DuplicatedItemException)
|
||||
{
|
||||
season = await libraryManager.Get(show.Slug, seasonNumber);
|
||||
}
|
||||
season.Show = show;
|
||||
return season;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<Episode> GetEpisode(ILibraryManager libraryManager,
|
||||
Show show,
|
||||
Season season,
|
||||
int? episodeNumber,
|
||||
int? absoluteNumber,
|
||||
string episodePath,
|
||||
Library library)
|
||||
{
|
||||
Episode episode = new();
|
||||
//await MetadataProvider.GetEpisode(show,
|
||||
// episodePath,
|
||||
// season?.SeasonNumber,
|
||||
// episodeNumber,
|
||||
// absoluteNumber,
|
||||
// library);
|
||||
|
||||
if (episode.SeasonNumber != null)
|
||||
{
|
||||
season ??= await GetSeason(libraryManager, show, episode.SeasonNumber.Value, library);
|
||||
episode.Season = season;
|
||||
episode.SeasonID = season?.ID;
|
||||
}
|
||||
await ThumbnailsManager.Validate(episode);
|
||||
await GetTracks(episode);
|
||||
return episode;
|
||||
}
|
||||
|
||||
private async Task<Episode> GetMovie(Show show, string episodePath)
|
||||
{
|
||||
Episode episode = new()
|
||||
{
|
||||
Title = show.Title,
|
||||
Path = episodePath,
|
||||
Show = show,
|
||||
ShowID = show.ID,
|
||||
ShowSlug = show.Slug
|
||||
};
|
||||
episode.Tracks = await GetTracks(episode);
|
||||
return episode;
|
||||
}
|
||||
|
||||
private async Task<ICollection<Track>> GetTracks(Episode episode)
|
||||
{
|
||||
episode.Tracks = (await Transcoder.ExtractInfos(episode, false))
|
||||
.Where(x => x.Type != StreamType.Attachment)
|
||||
.ToArray();
|
||||
return episode.Tracks;
|
||||
}
|
||||
*/
|
||||
}
|
||||
}
|
@ -33,7 +33,9 @@ namespace Kyoo.Api
|
||||
{
|
||||
ActionResult<Library> result = await base.Create(resource);
|
||||
if (result.Value != null)
|
||||
_taskManager.StartTask("scan", new Dictionary<string, object> {{"slug", result.Value.Slug}});
|
||||
_taskManager.StartTask("scan",
|
||||
new Progress<float>(),
|
||||
new Dictionary<string, object> {{"slug", result.Value.Slug}});
|
||||
return result;
|
||||
}
|
||||
|
||||
|
@ -34,7 +34,7 @@ namespace Kyoo.Api
|
||||
{
|
||||
try
|
||||
{
|
||||
_taskManager.StartTask(taskSlug, args);
|
||||
_taskManager.StartTask(taskSlug, new Progress<float>(), args);
|
||||
return Ok();
|
||||
}
|
||||
catch (ItemNotFoundException)
|
||||
|
Loading…
x
Reference in New Issue
Block a user