Implementing a task manager

Kyoo.Common/Controllers/ICrawler.cs

@@ -1,10 +0,0 @@
-using System.Threading;
-using System.Threading.Tasks;
-
-namespace Kyoo.Controllers
-{
-	public interface ICrawler
-	{
-		Task StartAsync(CancellationToken cancellationToken);
-	}
-}

Kyoo.Common/Controllers/IPluginManager.cs

@@ -1,4 +1,5 @@
 using System.Collections.Generic;
+using Kyoo.Models;
 
 namespace Kyoo.Controllers
 {
@@ -6,6 +7,7 @@ namespace Kyoo.Controllers
 	{
 		public T GetPlugin<T>(string name);
 		public IEnumerable<T> GetPlugins<T>();
+		public IEnumerable<IPlugin> GetAllPlugins();
 		public void ReloadPlugins();
 	}
 }

Kyoo.Common/Controllers/ITaskManager.cs

@@ -0,0 +1,9 @@
+namespace Kyoo.Controllers
+{
+	public interface ITaskManager
+	{
+		bool StartTask(string taskSlug);
+		
+		void ReloadTask();
+	}
+}

Kyoo.Common/Models/Plugin.cs

@@ -1,7 +1,10 @@
+using System.Collections.Generic;
+
 namespace Kyoo.Models
 {
 	public interface IPlugin
 	{
 		public string Name { get; }
+		public IEnumerable<ITask> Tasks { get; }
 	}
 }

Kyoo.Common/Models/Task.cs

@@ -0,0 +1,17 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Kyoo.Models
+{
+	public interface ITask
+	{
+		public string Slug { get; }
+		public string Name { get; }
+		public string Description { get; }
+		public string HelpMessage { get; }
+		public bool RunOnStartup { get; }
+		public int Priority { get; }
+		public Task Run(IServiceProvider serviceProvider, CancellationToken cancellationToken);
+	}
+}

Kyoo/Controllers/PluginManager.cs

@@ -62,6 +62,11 @@ namespace Kyoo.Controllers
 			return from plugin in _plugins where plugin is T select (T)plugin;
 		}
 
+		public IEnumerable<IPlugin> GetAllPlugins()
+		{
+			return _plugins ?? new  List<IPlugin>();
+		}
+
 		public void ReloadPlugins()
 		{
 			string pluginFolder = _config.GetValue<string>("plugins");
@@ -86,6 +91,12 @@ namespace Kyoo.Controllers
 					return null;
 				}
 			}).Where(x => x != null).ToList();
+			if (!_plugins.Any())
+			{
+				Console.WriteLine("No plugin enabled.");
+				return;
+			}
+
 			Console.WriteLine("Plugin enabled:");
 			foreach (IPlugin plugin in _plugins)
 				Console.WriteLine($"\t{plugin.Name}");

Kyoo/Controllers/StartupCode.cs

@@ -1,58 +0,0 @@
-using System;
-using System.Linq;
-using System.Threading;
-using System.Threading.Tasks;
-using IdentityServer4.EntityFramework.DbContexts;
-using IdentityServer4.EntityFramework.Mappers;
-using IdentityServer4.Models;
-using Microsoft.EntityFrameworkCore;
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Hosting;
-
-namespace Kyoo.Controllers
-{
-	public class StartupCode : BackgroundService
-	{
-		private readonly IServiceProvider _serviceProvider;
-
-		public StartupCode(IServiceProvider serviceProvider)
-		{
-			_serviceProvider = serviceProvider;
-		}
-
-		protected override async Task ExecuteAsync(CancellationToken stoppingToken)
-		{
-			using (IServiceScope serviceScope = _serviceProvider.CreateScope())
-			{
-				serviceScope.ServiceProvider.GetService<DatabaseContext>().Database.Migrate();
-				
-				ConfigurationDbContext identityContext = serviceScope.ServiceProvider.GetService<ConfigurationDbContext>();
-				identityContext.Database.Migrate();
-				if (!identityContext.Clients.Any())
-				{
-					foreach (Client client in IdentityContext.GetClients())
-						identityContext.Clients.Add(client.ToEntity());
-					identityContext.SaveChanges();
-				}
-				if (!identityContext.IdentityResources.Any())
-				{
-					foreach (IdentityResource resource in IdentityContext.GetIdentityResources())
-						identityContext.IdentityResources.Add(resource.ToEntity());
-					identityContext.SaveChanges();
-				}
-				if (!identityContext.ApiResources.Any())
-				{
-					foreach (ApiResource resource in IdentityContext.GetApis())
-						identityContext.ApiResources.Add(resource.ToEntity());
-					identityContext.SaveChanges();
-				}
-
-				IPluginManager pluginManager = serviceScope.ServiceProvider.GetService<IPluginManager>();
-				pluginManager.ReloadPlugins();
-
-				ICrawler crawler = serviceScope.ServiceProvider.GetService<ICrawler>();
-				await crawler.StartAsync(stoppingToken);
-			}
-		}
-	}
-}

Kyoo/Controllers/TaskManager.cs

@@ -0,0 +1,71 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Kyoo.Models;
+using Kyoo.Tasks;
+using Microsoft.Extensions.Hosting;
+
+namespace Kyoo.Controllers
+{
+	public class TaskManager : BackgroundService, ITaskManager
+	{
+		private readonly IServiceProvider _serviceProvider;
+		private readonly IPluginManager _pluginManager;
+		
+		private List<ITask> _tasks = new List<ITask>();
+		private CancellationTokenSource _taskToken = new CancellationTokenSource();
+		private Queue<ITask> _queuedTasks = new Queue<ITask>();
+		
+		public TaskManager(IServiceProvider serviceProvider, IPluginManager pluginManager)
+		{
+			_serviceProvider = serviceProvider;
+			_pluginManager = pluginManager;
+		}
+		
+		protected override async Task ExecuteAsync(CancellationToken cancellationToken)
+		{
+			while (!cancellationToken.IsCancellationRequested)
+			{
+				if (_queuedTasks.Any())
+					await _queuedTasks.Dequeue().Run(_serviceProvider, _taskToken.Token);
+				else
+					await Task.Delay(10);
+			}
+		}
+
+		public override Task StartAsync(CancellationToken cancellationToken)
+		{
+			ReloadTask();
+			foreach (ITask task in _tasks.Where(x => x.RunOnStartup && x.Priority != Int32.MaxValue).OrderByDescending(x => x.Priority))
+				_queuedTasks.Enqueue(task);
+			return base.StartAsync(cancellationToken);
+		}
+
+		public override Task StopAsync(CancellationToken cancellationToken)
+		{
+			_taskToken.Cancel();
+			return base.StopAsync(cancellationToken);
+		}
+
+		public bool StartTask(string taskSlug)
+		{
+			ITask task = _tasks.FirstOrDefault(x => x.Slug == taskSlug);
+			if (task == null)
+				return false;
+			_queuedTasks.Enqueue(task);
+			return true;
+		}
+
+		public void ReloadTask()
+		{
+			_tasks.Clear();
+			_tasks.AddRange(CoreTaskHolder.Tasks);
+			foreach (ITask task in _tasks.Where(x => x.RunOnStartup && x.Priority == Int32.MaxValue))
+				task.Run(_serviceProvider, _taskToken.Token);
+			foreach (IPlugin plugin in _pluginManager.GetAllPlugins())
+				_tasks.AddRange(plugin.Tasks);
+		}
+	}
+}

Kyoo/Startup.cs

@@ -114,13 +114,13 @@ namespace Kyoo
 			});
 
 			services.AddScoped<ILibraryManager, LibraryManager>();
-			services.AddScoped<ICrawler, Crawler>();
 			services.AddSingleton<ITranscoder, Transcoder>();
 			services.AddSingleton<IThumbnailsManager, ThumbnailsManager>();
 			services.AddSingleton<IProviderManager, ProviderManager>();
 			services.AddSingleton<IPluginManager, PluginManager>();
+			services.AddSingleton<ITaskManager, TaskManager>();
 			
-			services.AddHostedService<StartupCode>();
+			services.AddHostedService<TaskManager>(provider => (TaskManager)provider.GetService<ITaskManager>());
 		}
 
 		// This method gets called by the runtime. Use this method to configure the HTTP request pipeline.

Kyoo/Tasks/CoreTaskHolder.cs

@@ -0,0 +1,15 @@
+using Kyoo.Controllers;
+using Kyoo.Models;
+
+namespace Kyoo.Tasks
+{
+	public static class CoreTaskHolder
+	{
+		public static ITask[] Tasks =
+		{
+			new CreateDatabase(),
+			new PluginLoader(),
+			new Crawler()
+		};
+	}
+}

Kyoo/Tasks/Crawler.cs

@@ -8,26 +8,33 @@ using System.Text.RegularExpressions;
 using System.Threading;
 using System.Threading.Tasks;
 using Kyoo.Models.Watch;
+using Microsoft.Extensions.DependencyInjection;
 
 namespace Kyoo.Controllers
 {
-	public class Crawler : ICrawler
+	public class Crawler : ITask
 	{
-		private readonly ILibraryManager _libraryManager;
+		public string Slug => "scan";
-		private readonly IProviderManager _metadataProvider;
+		public string Name => "Scan libraries";
-		private readonly ITranscoder _transcoder;
+		public string Description => "Scan your libraries, load data for new shows and remove shows that don't exist anymore.";
-		private readonly IConfiguration _config;
+		public string HelpMessage => "Reloading all libraries is a long process and may take up to 24 hours if it is the first scan in a while.";
+		public bool RunOnStartup => true;
+		public int Priority => 0;
 		
-		public Crawler(ILibraryManager libraryManager, IProviderManager metadataProvider, ITranscoder transcoder, IConfiguration configuration)
+		private ILibraryManager _libraryManager;
-		{
+		private IProviderManager _metadataProvider;
-			_libraryManager = libraryManager;
+		private ITranscoder _transcoder;
-			_metadataProvider = metadataProvider;
+		private IConfiguration _config;
-			_transcoder = transcoder;
-			_config = configuration;
-		}
 		
-		public async Task StartAsync(CancellationToken cancellationToken)
+		
+		public async Task Run(IServiceProvider serviceProvider, CancellationToken cancellationToken)
 		{
+			using IServiceScope serviceScope = serviceProvider.CreateScope();
+			_libraryManager = serviceScope.ServiceProvider.GetService<ILibraryManager>();
+			_metadataProvider = serviceScope.ServiceProvider.GetService<IProviderManager>();
+			_transcoder = serviceScope.ServiceProvider.GetService<ITranscoder>();
+			_config = serviceScope.ServiceProvider.GetService<IConfiguration>();
+
 			try
 			{
 				IEnumerable<Episode> episodes = _libraryManager.GetAllEpisodes();

Kyoo/Tasks/CreateDatabase.cs

@@ -0,0 +1,53 @@
+using System;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using IdentityServer4.EntityFramework.DbContexts;
+using IdentityServer4.EntityFramework.Mappers;
+using IdentityServer4.Models;
+using Kyoo.Models;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.DependencyInjection;
+
+namespace Kyoo.Tasks
+{
+	public class CreateDatabase : ITask
+	{
+		public string Slug => "create-database";
+		public string Name => "Create the database";
+		public string Description => "Create the database if it does not exit and initialize it with defaults value.";
+		public string HelpMessage => null;
+		public bool RunOnStartup => true;
+		public int Priority => Int32.MaxValue;
+		
+		public Task Run(IServiceProvider serviceProvider, CancellationToken cancellationToken)
+		{
+			using IServiceScope serviceScope = serviceProvider.CreateScope();
+			DatabaseContext databaseContext = serviceScope.ServiceProvider.GetService<DatabaseContext>();
+			ConfigurationDbContext identityContext = serviceScope.ServiceProvider.GetService<ConfigurationDbContext>();
+			
+			databaseContext.Database.Migrate();
+			identityContext.Database.Migrate();
+			
+			if (!identityContext.Clients.Any())
+			{
+				foreach (Client client in IdentityContext.GetClients())
+					identityContext.Clients.Add(client.ToEntity());
+				identityContext.SaveChanges();
+			}
+			if (!identityContext.IdentityResources.Any())
+			{
+				foreach (IdentityResource resource in IdentityContext.GetIdentityResources())
+					identityContext.IdentityResources.Add(resource.ToEntity());
+				identityContext.SaveChanges();
+			}
+			if (!identityContext.ApiResources.Any())
+			{
+				foreach (ApiResource resource in IdentityContext.GetApis())
+					identityContext.ApiResources.Add(resource.ToEntity());
+				identityContext.SaveChanges();
+			}
+			return Task.CompletedTask;
+		}
+	}
+}

Kyoo/Tasks/PluginLoader.cs

@@ -0,0 +1,26 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Kyoo.Controllers;
+using Kyoo.Models;
+using Microsoft.Extensions.DependencyInjection;
+
+namespace Kyoo.Tasks
+{
+	public class PluginLoader : ITask
+	{
+		public string Slug => "reload-plugin";
+		public string Name => "Reload plugins";
+		public string Description => "Reload all plugins from the plugin folder.";
+		public string HelpMessage => null;
+		public bool RunOnStartup => true;
+		public int Priority => Int32.MaxValue;
+		public Task Run(IServiceProvider serviceProvider, CancellationToken cancellationToken)
+		{
+			using IServiceScope serviceScope = serviceProvider.CreateScope();
+			IPluginManager pluginManager = serviceScope.ServiceProvider.GetService<IPluginManager>();
+			pluginManager.ReloadPlugins();
+			return Task.CompletedTask;
+		}
+	}
+}

Kyoo/Views/API/AdminAPI.cs

@@ -1,22 +0,0 @@
-using Microsoft.AspNetCore.Mvc;
-using System.Threading;
-using Kyoo.Controllers;
-using Microsoft.AspNetCore.Authorization;
-
-namespace Kyoo.Api
-{
-	[Route("api/[controller]")]
-	[ApiController]
-	public class AdminController : ControllerBase
-	{
-		[HttpGet("scan")]
-		[Authorize(Policy="Admin")]
-		public IActionResult ScanLibrary([FromServices] ICrawler crawler)
-		{
-			// The crawler is destroyed before the completion of this task.
-			// TODO implement an hosted service that can queue tasks from the controller.
-			crawler.StartAsync(new CancellationToken());
-			return Ok("Scanning");
-		}
-	}
-}

Kyoo/Views/API/TaskAPI.cs

@@ -0,0 +1,28 @@
+using Microsoft.AspNetCore.Mvc;
+using Kyoo.Controllers;
+using Microsoft.AspNetCore.Authorization;
+
+namespace Kyoo.Api
+{
+	[Route("api/[controller]")]
+	[ApiController]
+	public class TaskController : ControllerBase
+	{
+		private readonly ITaskManager _taskManager;
+
+		public TaskController(ITaskManager taskManager)
+		{
+			_taskManager = taskManager;
+		}
+		
+		
+		[HttpGet("{taskSlug}")]
+		[Authorize(Policy="Admin")]
+		public IActionResult RunTask(string taskSlug)
+		{
+			if (_taskManager.StartTask(taskSlug))
+				return Ok();
+			return NotFound();
+		}
+	}
+}