Handling failed tasks and fixing library provider registration

Kyoo.Common/Models/Exceptions/TaskFailedException.cs

@@ -0,0 +1,45 @@
+using System;
+using System.Runtime.Serialization;
+using Kyoo.Controllers;
+
+namespace Kyoo.Models.Exceptions
+{
+	/// <summary>
+	/// An exception raised when an <see cref="ITask"/> failed.
+	/// </summary>
+	[Serializable]
+	public class TaskFailedException : AggregateException
+	{
+		/// <summary>
+		/// Create a new <see cref="TaskFailedException"/> with a default message.
+		/// </summary>
+		public TaskFailedException()
+			: base("A task failed.")
+		{}
+		
+		/// <summary>
+		/// Create a new <see cref="TaskFailedException"/> with a custom message.
+		/// </summary>
+		/// <param name="message">The message to use.</param>
+		public TaskFailedException(string message)
+			: base(message)
+		{}
+		
+		/// <summary>
+		/// Create a new <see cref="TaskFailedException"/> wrapping another exception.
+		/// </summary>
+		/// <param name="exception">The exception to wrap.</param>
+		public TaskFailedException(Exception exception)
+			: base(exception)
+		{}
+		
+		/// <summary>
+		/// The serialization constructor 
+		/// </summary>
+		/// <param name="info">Serialization infos</param>
+		/// <param name="context">The serialization context</param>
+		protected TaskFailedException(SerializationInfo info, StreamingContext context)
+			: base(info, context)
+		{ }
+	}
+}

Kyoo.Common/Models/Resources/Episode.cs

@@ -20,9 +20,11 @@ namespace Kyoo.Models
 		{
 			get
 			{
-				if (ShowSlug == null && Show == null)
+				if (ShowSlug != null || Show != null)
-					return GetSlug(ShowID.ToString(), SeasonNumber, EpisodeNumber, AbsoluteNumber);
+					return GetSlug(ShowSlug ?? Show.Slug, SeasonNumber, EpisodeNumber, AbsoluteNumber);
-				return GetSlug(ShowSlug ?? Show.Slug, SeasonNumber, EpisodeNumber, AbsoluteNumber);
+				return ShowID != 0 
+					? GetSlug(ShowID.ToString(), SeasonNumber, EpisodeNumber, AbsoluteNumber) 
+					: null;
 			}
 			[UsedImplicitly] [NotNull] private set
 			{

Kyoo.Common/Models/Resources/Track.cs

@@ -156,30 +156,17 @@ namespace Kyoo.Models
 		}
 
 		/// <summary>
-		/// Utility method to edit a track slug (this only return a slug with the modification, nothing is stored)
+		/// Utility method to create a track slug from a incomplete slug (only add the type of the track).
 		/// </summary>
 		/// <param name="baseSlug">The slug to edit</param>
 		/// <param name="type">The new type of this </param>
-		/// <param name="language"></param>
-		/// <param name="index"></param>
-		/// <param name="forced"></param>
 		/// <returns></returns>
-		public static string EditSlug(string baseSlug,
+		public static string BuildSlug(string baseSlug,
-			StreamType type = StreamType.Unknown,
+			StreamType type)
-			string language = null,
-			int? index = null,
-			bool? forced = null)
 		{
-			Track track = new() {Slug = baseSlug};
+			return baseSlug.EndsWith($".{type}", StringComparison.InvariantCultureIgnoreCase) 
-			if (type != StreamType.Unknown)
+				? baseSlug 
-				track.Type = type;
+				: $"{baseSlug}.{type.ToString().ToLowerInvariant()}";
-			if (language != null)
-				track.Language = language;
-			if (index != null)
-				track.TrackIndex = index.Value;
-			if (forced != null)
-				track.IsForced = forced.Value;
-			return track.Slug;
 		}
 	}
 }

Kyoo.Common/Models/WatchItem.cs

@@ -176,6 +176,7 @@ namespace Kyoo.Models
 			return new WatchItem
 			{
 				EpisodeID = ep.ID,
+				Slug = ep.Slug,
 				ShowSlug = ep.Show.Slug,
 				SeasonNumber = ep.SeasonNumber,
 				EpisodeNumber = ep.EpisodeNumber,
@@ -183,6 +184,7 @@ namespace Kyoo.Models
 				Title = ep.Title,
 				ReleaseDate = ep.ReleaseDate,
 				Path = ep.Path,
+				Container = PathIO.GetExtension(ep.Path)![1..],
 				Video = ep.Tracks.FirstOrDefault(x => x.Type == StreamType.Video),
 				Audios = ep.Tracks.Where(x => x.Type == StreamType.Audio).ToArray(),
 				Subtitles = ep.Tracks.Where(x => x.Type == StreamType.Subtitle).ToArray(), 

Kyoo.Tests/Library/SpecificTests/CollectionsTests.cs

@@ -3,7 +3,7 @@ using Kyoo.Models;
 using Xunit;
 using Xunit.Abstractions;
 
-namespace Kyoo.Tests.Library
+namespace Kyoo.Tests.Database
 {
 	namespace SqLite
 	{

Kyoo.Tests/Library/SpecificTests/EpisodeTests.cs

@@ -4,7 +4,7 @@ using Kyoo.Models;
 using Xunit;
 using Xunit.Abstractions;
 
-namespace Kyoo.Tests.Library
+namespace Kyoo.Tests.Database
 {
 	namespace SqLite
 	{

Kyoo.Tests/Library/SpecificTests/GenreTests.cs

@@ -3,7 +3,7 @@ using Kyoo.Models;
 using Xunit;
 using Xunit.Abstractions;
 
-namespace Kyoo.Tests.Library
+namespace Kyoo.Tests.Database
 {
 	namespace SqLite
 	{

Kyoo.Tests/Library/SpecificTests/LibraryItemTest.cs

@@ -5,7 +5,7 @@ using Kyoo.Models;
 using Xunit;
 using Xunit.Abstractions;
 
-namespace Kyoo.Tests.Library
+namespace Kyoo.Tests.Database
 {
 	namespace SqLite
 	{

Kyoo.Tests/Library/SpecificTests/LibraryTests.cs

@@ -1,8 +1,11 @@
+using System.Linq;
+using System.Threading.Tasks;
 using Kyoo.Controllers;
+using Kyoo.Models;
 using Xunit;
 using Xunit.Abstractions;
 
-namespace Kyoo.Tests.Library
+namespace Kyoo.Tests.Database
 {
 	namespace SqLite
 	{
@@ -23,7 +26,7 @@ namespace Kyoo.Tests.Library
 		}
 	}
 
-	public abstract class ALibraryTests : RepositoryTests<Models.Library>
+	public abstract class ALibraryTests : RepositoryTests<Library>
 	{
 		private readonly ILibraryRepository _repository;
 
@@ -32,5 +35,17 @@ namespace Kyoo.Tests.Library
 		{
 			_repository = Repositories.LibraryManager.LibraryRepository;
 		}
+
+		[Fact]
+		public async Task CreateWithProvider()
+		{
+			Library library = TestSample.GetNew<Library>();
+			library.Providers = new[] { TestSample.Get<Provider>() };
+			await _repository.Create(library);
+			Library retrieved = await _repository.Get(2);
+			await Repositories.LibraryManager.Load(retrieved, x => x.Providers);
+			Assert.Equal(1, retrieved.Providers.Count);
+			Assert.Equal(TestSample.Get<Provider>().Slug, retrieved.Providers.First().Slug);
+		}
 	}
 }

Kyoo.Tests/Library/SpecificTests/PeopleTests.cs

@@ -3,7 +3,7 @@ using Kyoo.Models;
 using Xunit;
 using Xunit.Abstractions;
 
-namespace Kyoo.Tests.Library
+namespace Kyoo.Tests.Database
 {
 	namespace SqLite
 	{

Kyoo.Tests/Library/SpecificTests/ProviderTests.cs

@@ -3,7 +3,7 @@ using Kyoo.Models;
 using Xunit;
 using Xunit.Abstractions;
 
-namespace Kyoo.Tests.Library
+namespace Kyoo.Tests.Database
 {
 	namespace SqLite
 	{

Kyoo.Tests/Library/SpecificTests/SanityTests.cs

@@ -5,7 +5,7 @@ using Kyoo.Models;
 using Xunit;
 using Xunit.Abstractions;
 
-namespace Kyoo.Tests.Library
+namespace Kyoo.Tests.Database
 {
 	public class GlobalTests : IDisposable, IAsyncDisposable
 	{

Kyoo.Tests/Library/SpecificTests/SeasonTests.cs

@@ -4,7 +4,7 @@ using Kyoo.Models;
 using Xunit;
 using Xunit.Abstractions;
 
-namespace Kyoo.Tests.Library
+namespace Kyoo.Tests.Database
 {
 	namespace SqLite
 	{

Kyoo.Tests/Library/SpecificTests/ShowTests.cs

@@ -8,7 +8,7 @@ using Microsoft.EntityFrameworkCore;
 using Xunit;
 using Xunit.Abstractions;
 
-namespace Kyoo.Tests.Library
+namespace Kyoo.Tests.Database
 {
 	namespace SqLite
 	{

Kyoo.Tests/Library/SpecificTests/StudioTests.cs

@@ -3,7 +3,7 @@ using Kyoo.Models;
 using Xunit;
 using Xunit.Abstractions;
 
-namespace Kyoo.Tests.Library
+namespace Kyoo.Tests.Database
 {
 	namespace SqLite
 	{

Kyoo.Tests/Library/SpecificTests/TrackTests.cs

@@ -4,7 +4,7 @@ using Kyoo.Models;
 using Xunit;
 using Xunit.Abstractions;
 
-namespace Kyoo.Tests.Library
+namespace Kyoo.Tests.Database
 {
 	namespace SqLite
 	{

Kyoo.Tests/Library/SpecificTests/UserTests.cs

@@ -3,7 +3,7 @@ using Kyoo.Models;
 using Xunit;
 using Xunit.Abstractions;
 
-namespace Kyoo.Tests.Library
+namespace Kyoo.Tests.Database
 {
 	namespace SqLite
 	{

Kyoo.Tests/Library/TestSample.cs

@@ -9,8 +9,14 @@ namespace Kyoo.Tests
 		private static readonly Dictionary<Type, Func<object>> NewSamples = new()
 		{
 			{
-				typeof(Show),
+				typeof(Library),
-				() => new Show()
+				() => new Library
+				{
+					ID = 2,
+					Slug = "new-library",
+					Name = "New Library",
+					Paths = new [] {"/a/random/path"}
+				}
 			}
 		};
 		
@@ -18,8 +24,8 @@ namespace Kyoo.Tests
 		private static readonly Dictionary<Type, Func<object>> Samples = new()
 		{
 			{
-				typeof(Models.Library),
+				typeof(Library),
-				() => new Models.Library
+				() => new Library
 				{
 					ID = 1,
 					Slug = "deck",

Kyoo.WebApp

@@ -1 +1 @@
-Subproject commit 22a02671918201d6d9d4e80a76f01b59b216a82d
+Subproject commit dcdebad14cbcdf1f9486cb9178e6518d10c0e97f

Kyoo/Controllers/RegexIdentifier.cs

@@ -87,7 +87,7 @@ namespace Kyoo.Controllers
 		public Task<Track> IdentifyTrack(string path, string relativePath)
 		{
 			Regex regex = new(_configuration.Value.SubtitleRegex, RegexOptions.IgnoreCase | RegexOptions.Compiled);
-			Match match = regex.Match(relativePath);
+			Match match = regex.Match(path);
 
 			if (!match.Success)
 				throw new IdentificationFailed($"The subtitle at {path} does not match the subtitle's regex.");

Kyoo/Controllers/Repositories/LibraryRepository.cs

@@ -30,7 +30,7 @@ namespace Kyoo.Controllers
 		/// Create a new <see cref="LibraryRepository"/> instance.
 		/// </summary>
 		/// <param name="database">The database handle</param>
-		/// <param name="providers">The providere repository</param>
+		/// <param name="providers">The provider repository</param>
 		public LibraryRepository(DatabaseContext database, IProviderRepository providers)
 			: base(database)
 		{
@@ -53,8 +53,8 @@ namespace Kyoo.Controllers
 		public override async Task<Library> Create(Library obj)
 		{
 			await base.Create(obj);
-			obj.ProviderLinks = obj.Providers?.Select(x => Link.Create(obj, x)).ToList();
 			_database.Entry(obj).State = EntityState.Added;
+			obj.ProviderLinks.ForEach(x => _database.Entry(x).State = EntityState.Added);
 			await _database.SaveChangesAsync($"Trying to insert a duplicated library (slug {obj.Slug} already exists).");
 			return obj;
 		}
@@ -63,6 +63,9 @@ namespace Kyoo.Controllers
 		protected override async Task Validate(Library resource)
 		{
 			await base.Validate(resource);
+			resource.ProviderLinks = resource.Providers?
+				.Select(x => Link.Create(resource, x))
+				.ToList();
 			await resource.ProviderLinks.ForEachAsync(async id =>
 			{
 				id.Second = await _providers.CreateIfNotExists(id.Second);

Kyoo/Controllers/TaskManager.cs

@@ -1,11 +1,9 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Reflection;
 using System.Threading;
 using System.Threading.Tasks;
 using JetBrains.Annotations;
-using Kyoo.Models.Attributes;
 using Kyoo.Models.Exceptions;
 using Kyoo.Models.Options;
 using Microsoft.Extensions.DependencyInjection;
@@ -112,6 +110,10 @@ namespace Kyoo.Controllers
 					{
 						await RunTask(task, progress, args);
 					}
+					catch (TaskFailedException ex)
+					{
+						_logger.LogWarning("The task \"{Task}\" failed: {Message}", task.Name, ex.Message);
+					}
 					catch (Exception e)
 					{
 						_logger.LogError(e, "An unhandled exception occured while running the task {Task}", task.Name);

Kyoo/Tasks/Crawler.cs

@@ -150,6 +150,7 @@ namespace Kyoo.Tasks
 				
 				string[] subtitles = files
 					.Where(FileExtensions.IsSubtitle)
+					.Where(x => x.Contains("/Extra/"))
 					.Where(x => tracks.All(y => y.Path != x))
 					.ToArray();
 				percent = 0;

Kyoo/Tasks/RegisterEpisode.cs

@@ -79,54 +79,66 @@ namespace Kyoo.Tasks
 			if (library.Providers == null)
 				await LibraryManager.Load(library, x => x.Providers);
 			MetadataProvider.UseProviders(library.Providers);
-			(Collection collection, Show show, Season season, Episode episode) = await Identifier.Identify(path, 
+			try
-				relativePath);
-			progress.Report(15);
-			
-			collection = await _RegisterAndFill(collection);
-			progress.Report(20);
-			
-			Show registeredShow = await _RegisterAndFill(show);
-			if (registeredShow.Path != show.Path)
 			{
-				if (show.StartAir.HasValue)
+				(Collection collection, Show show, Season season, Episode episode) = await Identifier.Identify(path,
+					relativePath);
+				progress.Report(15);
+
+				collection = await _RegisterAndFill(collection);
+				progress.Report(20);
+
+				Show registeredShow = await _RegisterAndFill(show);
+				if (registeredShow.Path != show.Path)
 				{
-					show.Slug += $"-{show.StartAir.Value.Year}";
+					if (show.StartAir.HasValue)
-					show = await LibraryManager.Create(show);
+					{
+						show.Slug += $"-{show.StartAir.Value.Year}";
+						show = await LibraryManager.Create(show);
+					}
+					else
+					{
+						throw new TaskFailedException($"Duplicated show found ({show.Slug}) " +
+							$"at {registeredShow.Path} and {show.Path}");
+					}
 				}
 				else
-				{
+					show = registeredShow;
-					throw new DuplicatedItemException($"Duplicated show found ({show.Slug}) " +
+
-						$"at {registeredShow.Path} and {show.Path}");
+				// If they are not already loaded, load external ids to allow metadata providers to use them.
-				}
+				if (show.ExternalIDs == null)
+					await LibraryManager.Load(show, x => x.ExternalIDs);
+				progress.Report(50);
+
+				if (season != null)
+					season.Show = show;
+
+				season = await _RegisterAndFill(season);
+				progress.Report(60);
+
+				episode = await MetadataProvider.Get(episode);
+				progress.Report(70);
+				episode.Show = show;
+				episode.Season = season;
+				episode.Tracks = (await Transcoder.ExtractInfos(episode, false))
+					.Where(x => x.Type != StreamType.Attachment)
+					.ToArray();
+				await ThumbnailsManager.DownloadImages(episode);
+				progress.Report(90);
+
+				await LibraryManager.Create(episode);
+				progress.Report(95);
+				await LibraryManager.AddShowLink(show, library, collection);
+				progress.Report(100);
+			}
+			catch (IdentificationFailed ex)
+			{
+				throw new TaskFailedException(ex);
+			}
+			catch (DuplicatedItemException ex)
+			{
+				throw new TaskFailedException(ex);
 			}
-			else
-				show = registeredShow;
-			// If they are not already loaded, load external ids to allow metadata providers to use them.
-			if (show.ExternalIDs == null)
-				await LibraryManager.Load(show, x => x.ExternalIDs);
-			progress.Report(50);
-
-			if (season != null)
-				season.Show = show;
-
-			season = await _RegisterAndFill(season);
-			progress.Report(60);
-
-			episode = await MetadataProvider.Get(episode);
-			progress.Report(70);
-			episode.Show = show;
-			episode.Season = season;
-			episode.Tracks = (await Transcoder.ExtractInfos(episode, false))
-				.Where(x => x.Type != StreamType.Attachment)
-				.ToArray();
-			await ThumbnailsManager.DownloadImages(episode);
-			progress.Report(90);
-
-			await LibraryManager.Create(episode);
-			progress.Report(95);
-			await LibraryManager.AddShowLink(show, library, collection);
-			progress.Report(100);
 		}
 		
 		/// <summary>

Kyoo/Tasks/RegisterSubtitle.cs

@@ -59,30 +59,37 @@ namespace Kyoo.Tasks
 		{
 			string path = arguments["path"].As<string>();
 			string relativePath = arguments["relativePath"].As<string>();
-			
+
-			progress.Report(0);
+			try
-			Track track = await Identifier.IdentifyTrack(path, relativePath);
-			progress.Report(25);
-			
-			if (track.Episode == null)
-				throw new IdentificationFailed($"No episode identified for the track at {path}");
-			if (track.Episode.ID == 0)
 			{
-				if (track.Episode.Slug != null)
+				progress.Report(0);
-					track.Episode = await LibraryManager.Get<Episode>(track.Episode.Slug);
+				Track track = await Identifier.IdentifyTrack(path, relativePath);
-				else if (track.Episode.Path != null)
+				progress.Report(25);
+
+				if (track.Episode == null)
+					throw new TaskFailedException($"No episode identified for the track at {path}");
+				if (track.Episode.ID == 0)
 				{
-					track.Episode = await LibraryManager.GetOrDefault<Episode>(x => x.Path == track.Episode.Path);
+					if (track.Episode.Slug != null)
-					if (track.Episode == null)
+						track.Episode = await LibraryManager.Get<Episode>(track.Episode.Slug);
-						throw new ItemNotFoundException($"No episode found for subtitle at: ${path}.");
+					else if (track.Episode.Path != null)
+					{
+						track.Episode = await LibraryManager.GetOrDefault<Episode>(x => x.Path.StartsWith(track.Episode.Path));
+						if (track.Episode == null)
+							throw new TaskFailedException($"No episode found for the track at: {path}.");
+					}
+					else
+						throw new TaskFailedException($"No episode identified for the track at {path}");
 				}
-				else
+
-					throw new IdentificationFailed($"No episode identified for the track at {path}");
+				progress.Report(50);
+				await LibraryManager.Create(track);
+				progress.Report(100);
+			}
+			catch (IdentificationFailed ex)
+			{
+				throw new TaskFailedException(ex);
 			}
-			
-			progress.Report(50);
-			await LibraryManager.Create(track);
-			progress.Report(100);
 		}
 	}
 }

Kyoo/Views/ShowApi.cs

@@ -386,7 +386,7 @@ namespace Kyoo.Api
 				string path = Path.Combine(_files.GetExtraDirectory(show), "Attachments");
 				return (await _files.ListFiles(path))
 					.ToDictionary(Path.GetFileNameWithoutExtension,
-						x => $"{BaseURL}/api/shows/{slug}/fonts/{Path.GetFileName(x)}");
+						x => $"{BaseURL}api/shows/{slug}/fonts/{Path.GetFileName(x)}");
 			}
 			catch (ItemNotFoundException)
 			{

Kyoo/Views/SubtitleApi.cs

@@ -2,6 +2,7 @@
 using Microsoft.AspNetCore.Mvc;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Threading.Tasks;
 using Kyoo.Controllers;
 using Kyoo.Models.Permissions;
@@ -21,12 +22,43 @@ namespace Kyoo.Api
 			_files = files;
 		}
 		
-		
+		[HttpGet("{id:int}")]
-		[HttpGet("{slug}.{extension}")]
 		[Permission(nameof(SubtitleApi), Kind.Read)]
-		public async Task<IActionResult> GetSubtitle(string slug, string extension)
+		public async Task<IActionResult> GetSubtitle(int id)
 		{
-			Track subtitle = await _libraryManager.GetOrDefault<Track>(Track.EditSlug(slug, StreamType.Subtitle));
+			Track subtitle = await _libraryManager.GetOrDefault<Track>(id);
+			return subtitle != null
+				? _files.FileResult(subtitle.Path)
+				: NotFound();
+		}
+		
+		[HttpGet("{id:int}.{extension}")]
+		[Permission(nameof(SubtitleApi), Kind.Read)]
+		public async Task<IActionResult> GetSubtitle(int id, string extension)
+		{
+			Track subtitle = await _libraryManager.GetOrDefault<Track>(id);
+			if (subtitle == null)
+				return NotFound();
+			if (subtitle.Codec == "subrip" && extension == "vtt")
+				return new ConvertSubripToVtt(subtitle.Path, _files);
+			return _files.FileResult(subtitle.Path);
+		}
+		
+		
+		[HttpGet("{slug}")]
+		[Permission(nameof(SubtitleApi), Kind.Read)]
+		public async Task<IActionResult> GetSubtitle(string slug)
+		{
+			string extension = null;
+			
+			if (slug.Count(x => x == '.') == 2)
+			{
+				int idx = slug.LastIndexOf('.');
+				extension = slug[(idx + 1)..];
+				slug = slug[..idx];
+			}
+
+			Track subtitle = await _libraryManager.GetOrDefault<Track>(Track.BuildSlug(slug, StreamType.Subtitle));
 			if (subtitle == null)
 				return NotFound();
 			if (subtitle.Codec == "subrip" && extension == "vtt")

Kyoo/Views/TrackApi.cs

@@ -45,8 +45,6 @@ namespace Kyoo.Api
 		{
 			try
 			{
-				// TODO This won't work with the local repository implementation.
-				// TODO Implement something like this (a dotnet-ef's QueryCompilationContext): https://stackoverflow.com/questions/62687811/how-can-i-convert-a-custom-function-to-a-sql-expression-for-entity-framework-cor
 				return await _libraryManager.Get<Episode>(x => x.Tracks.Any(y => y.Slug == slug));
 			}
 			catch (ItemNotFoundException)