Cutlery/Kyoo
1
0
Fork 0
mirror of https://github.com/zoriya/Kyoo.git synced 2025-10-31 10:37:13 -04:00
Code Issues Packages Projects Releases Wiki Activity

Creating a registerr subtitle task

Browse Source
This commit is contained in:
Zoe Roux 2021-07-17 01:34:24 +02:00
parent f71ae0385c
commit e66e59cf32
8 changed files with 300 additions and 178 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
Kyoo.Common/Controllers/IIdentifier.cs
View File

@ -1,5 +1,6 @@
using System.Threading.Tasks;
using Kyoo.Models;
using Kyoo.Models.Exceptions;
namespace Kyoo.Controllers
{
@ -12,10 +13,21 @@ namespace Kyoo.Controllers
/// Identify a path and return the parsed metadata.
/// </summary>
/// <param name="path">The path of the episode file to parse.</param>
/// <exception cref="IdentificationFailed">The identifier could not work for the given path.</exception>
/// <returns>
/// A tuple of models representing parsed metadata.
/// If no metadata could be parsed for a type, null can be returned.
/// </returns>
Task<(Collection, Show, Season, Episode)> Identify(string path);
/// <summary>
/// Identify an external subtitle or track file from it's path and return the parsed metadata.
/// </summary>
/// <param name="path">The path of the external track file to parse.</param>
/// <exception cref="IdentificationFailed">The identifier could not work for the given path.</exception>
/// <returns>
/// The metadata of the track identified.
/// </returns>
Task<Track> IdentifyTrack(string path);
}
}

37
Kyoo.Common/Models/Exceptions/IdentificationFailed.cs Normal file
View File

@ -0,0 +1,37 @@
using System;
using System.Runtime.Serialization;
using Kyoo.Controllers;
namespace Kyoo.Models.Exceptions
{
/// <summary>
/// An exception raised when an <see cref="IIdentifier"/> failed.
/// </summary>
[Serializable]
public class IdentificationFailed : Exception
{
/// <summary>
/// Create a new <see cref="IdentificationFailed"/> with a default message.
/// </summary>
public IdentificationFailed()
: base("An identification failed.")
{}
/// <summary>
/// Create a new <see cref="IdentificationFailed"/> with a custom message.
/// </summary>
/// <param name="message">The message to use.</param>
public IdentificationFailed(string message)
: base(message)
{}
/// <summary>
/// The serialization constructor
/// </summary>
/// <param name="info">Serialization infos</param>
/// <param name="context">The serialization context</param>
protected IdentificationFailed(SerializationInfo info, StreamingContext context)
: base(info, context)
{ }
}
}

45
Kyoo/Controllers/RegexIdentifier.cs
View File

@ -3,8 +3,9 @@ using System.IO;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Kyoo.Models;
using Kyoo.Models.Exceptions;
using Kyoo.Models.Options;
using Microsoft.Extensions.Logging;
using Kyoo.Models.Watch;
using Microsoft.Extensions.Options;
namespace Kyoo.Controllers
@ -18,23 +19,16 @@ namespace Kyoo.Controllers
/// The configuration of kyoo to retrieve the identifier regex.
/// </summary>
private readonly IOptions<MediaOptions> _configuration;
/// <summary>
/// A logger to print errors.
/// </summary>
private readonly ILogger<RegexIdentifier> _logger;
/// <summary>
/// Create a new <see cref="RegexIdentifier"/>.
/// </summary>
/// <param name="configuration">The regex patterns to use.</param>
/// <param name="logger">The logger to use.</param>
public RegexIdentifier(IOptions<MediaOptions> configuration, ILogger<RegexIdentifier> logger)
public RegexIdentifier(IOptions<MediaOptions> configuration)
{
_configuration = configuration;
_logger = logger;
}
/// <inheritdoc />
public Task<(Collection, Show, Season, Episode)> Identify(string path)
{
@ -42,10 +36,7 @@ namespace Kyoo.Controllers
Match match = regex.Match(path);
if (!match.Success)
{
_logger.LogError("The episode at {Path} does not match the episode's regex", path);
return Task.FromResult<(Collection, Show, Season, Episode)>(default);
}
throw new IdentificationFailed($"The episode at {path} does not match the episode's regex.");
(Collection collection, Show show, Season season, Episode episode) ret = new();
@ -81,5 +72,31 @@ namespace Kyoo.Controllers
return Task.FromResult(ret);
}
/// <inheritdoc />
public Task<Track> IdentifyTrack(string path)
{
Regex regex = new(_configuration.Value.SubtitleRegex, RegexOptions.IgnoreCase | RegexOptions.Compiled);
Match match = regex.Match(path);
if (!match.Success)
throw new IdentificationFailed($"The subtitle at {path} does not match the subtitle's regex.");
string episodePath = match.Groups["Episode"].Value;
return Task.FromResult(new Track
{
Type = StreamType.Subtitle,
Language = match.Groups["Language"].Value,
IsDefault = match.Groups["Default"].Value.Length > 0,
IsForced = match.Groups["Forced"].Value.Length > 0,
Codec = FileExtensions.SubtitleExtensions[Path.GetExtension(path)],
IsExternal = true,
Path = path,
Episode = new Episode
{
Path = episodePath
}
});
}
}
}

53
Kyoo/Models/FileExtensions.cs Normal file
View File

@ -0,0 +1,53 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
namespace Kyoo.Models.Watch
{
public static class FileExtensions
{
public static readonly string[] VideoExtensions =
{
".webm",
".mkv",
".flv",
".vob",
".ogg",
".ogv",
".avi",
".mts",
".m2ts",
".ts",
".mov",
".qt",
".asf",
".mp4",
".m4p",
".m4v",
".mpg",
".mp2",
".mpeg",
".mpe",
".mpv",
".m2v",
".3gp",
".3g2"
};
public static bool IsVideo(string filePath)
{
return VideoExtensions.Contains(Path.GetExtension(filePath));
}
public static readonly Dictionary<string, string> SubtitleExtensions = new()
{
{".ass", "ass"},
{".str", "subrip"}
};
public static bool IsSubtitle(string filePath)
{
return SubtitleExtensions.ContainsKey(Path.GetExtension(filePath));
}
}
}

12
Kyoo/Models/LazyDi.cs
View File

@ -1,12 +0,0 @@
using System;
using Microsoft.Extensions.DependencyInjection;
namespace Kyoo.Models
{
public class LazyDi<T> : Lazy<T>
{
public LazyDi(IServiceProvider provider)
: base(provider.GetRequiredService<T>)
{ }
}
}

81
Kyoo/Tasks/Crawler.cs
View File

@ -3,13 +3,11 @@ using Kyoo.Models;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using Kyoo.Controllers;
using Kyoo.Models.Attributes;
using Kyoo.Models.Exceptions;
using Microsoft.Extensions.DependencyInjection;
using Kyoo.Models.Watch;
using Microsoft.Extensions.Logging;
namespace Kyoo.Tasks
@ -86,6 +84,7 @@ namespace Kyoo.Tasks
float percent = 0;
ICollection<Episode> episodes = await LibraryManager.GetAll<Episode>();
ICollection<Track> tracks = await LibraryManager.GetAll<Track>();
foreach (Library library in libraries)
{
IProgress<float> reporter = new Progress<float>(x =>
@ -93,7 +92,7 @@ namespace Kyoo.Tasks
// ReSharper disable once AccessToModifiedClosure
progress.Report(percent + x / libraries.Count);
});
await Scan(library, episodes, reporter, cancellationToken);
await Scan(library, episodes, tracks, reporter, cancellationToken);
percent += 100f / libraries.Count;
if (cancellationToken.IsCancellationRequested)
@ -105,6 +104,7 @@ namespace Kyoo.Tasks
private async Task Scan(Library library,
IEnumerable<Episode> episodes,
IEnumerable<Track> tracks,
IProgress<float> progress,
CancellationToken cancellationToken)
{
@ -120,19 +120,20 @@ namespace Kyoo.Tasks
// This speeds up the scan process because further episodes of a show are registered when all metadata
// of the show has already been fetched.
List<IGrouping<string, string>> shows = files
.Where(IsVideo)
.Where(FileExtensions.IsVideo)
.Where(x => episodes.All(y => y.Path != x))
.GroupBy(Path.GetDirectoryName)
.ToList();
string[] paths = shows.Select(x => x.First())
.Concat(shows.SelectMany(x => x.Skip(1)))
.ToArray();
float percent = 0;
IProgress<float> reporter = new Progress<float>(x =>
{
// ReSharper disable once AccessToModifiedClosure
progress.Report((percent + x / paths.Length) / library.Paths.Length);
progress.Report((percent + x / paths.Length - 10) / library.Paths.Length);
});
foreach (string episodePath in paths)
@ -145,53 +146,27 @@ namespace Kyoo.Tasks
percent += 100f / paths.Length;
}
// await Task.WhenAll(files.Where(x => IsSubtitle(x) && tracks.All(y => y.Path != x))
// .Select(x => RegisterExternalSubtitle(x, cancellationToken)));
string[] subtitles = files
.Where(FileExtensions.IsSubtitle)
.Where(x => tracks.All(y => y.Path != x))
.ToArray();
percent = 0;
reporter = new Progress<float>(x =>
{
// ReSharper disable once AccessToModifiedClosure
progress.Report((90 + (percent + x / subtitles.Length)) / library.Paths.Length);
});
foreach (string trackPath in subtitles)
{
TaskManager.StartTask<RegisterSubtitle>(reporter, new Dictionary<string, object>
{
["path"] = trackPath
}, cancellationToken);
percent += 100f / subtitles.Length;
}
}
}
private static readonly string[] VideoExtensions =
{
".webm",
".mkv",
".flv",
".vob",
".ogg",
".ogv",
".avi",
".mts",
".m2ts",
".ts",
".mov",
".qt",
".asf",
".mp4",
".m4p",
".m4v",
".mpg",
".mp2",
".mpeg",
".mpe",
".mpv",
".m2v",
".3gp",
".3g2"
};
private static bool IsVideo(string filePath)
{
return VideoExtensions.Contains(Path.GetExtension(filePath));
}
private static readonly Dictionary<string, string> SubtitleExtensions = new()
{
{".ass", "ass"},
{".str", "subrip"}
};
private static bool IsSubtitle(string filePath)
{
return SubtitleExtensions.ContainsKey(Path.GetExtension(filePath));
}
}
}

153
Kyoo/Tasks/RegisterEpisode.cs
View File

@ -6,7 +6,6 @@ using Kyoo.Controllers;
using Kyoo.Models;
using Kyoo.Models.Attributes;
using Kyoo.Models.Exceptions;
using Microsoft.Extensions.Logging;
namespace Kyoo.Tasks
{
@ -56,11 +55,7 @@ namespace Kyoo.Tasks
/// The transcoder used to extract subtitles and metadata.
/// </summary>
[Injected] public ITranscoder Transcoder { private get; set; }
/// <summary>
/// The logger used to inform the current status to the console.
/// </summary>
[Injected] public ILogger<RegisterEpisode> Logger { private get; set; }
/// <inheritdoc />
public TaskParameters GetParameters()
{
@ -76,58 +71,67 @@ namespace Kyoo.Tasks
{
string path = arguments["path"].As<string>();
Library library = arguments["library"].As<Library>();
progress.Report(0);
try
if (library.Providers == null)
await LibraryManager.Load(library, x => x.Providers);
MetadataProvider.UseProviders(library.Providers);
(Collection collection, Show show, Season season, Episode episode) = await Identifier.Identify(path);
progress.Report(15);
collection = await _RegisterAndFill(collection);
progress.Report(20);
Show registeredShow = await _RegisterAndFill(show);
if (registeredShow.Path != show.Path)
{
if (library != null)
if (show.StartAir.HasValue)
{
if (library.Providers == null)
await LibraryManager.Load(library, x => x.Providers);
MetadataProvider.UseProviders(library.Providers);
}
(Collection collection, Show show, Season season, Episode episode) = await Identifier.Identify(path);
collection = await _RegisterAndFill(collection);
Show registeredShow = await _RegisterAndFill(show);
if (registeredShow.Path != show.Path)
{
if (show.StartAir.HasValue)
{
show.Slug += $"-{show.StartAir.Value.Year}";
show = await LibraryManager.Create(show);
}
else
{
Logger.LogError("Duplicated show found ({Slug}) at {Path1} and {Path2}",
show.Slug, registeredShow.Path, show.Path);
return;
}
show.Slug += $"-{show.StartAir.Value.Year}";
show = await LibraryManager.Create(show);
}
else
show = registeredShow;
if (season != null)
season.Show = show;
season = await _RegisterAndFill(season);
episode = await MetadataProvider.Get(episode);
episode.Season = season;
episode.Tracks = (await Transcoder.ExtractInfos(episode, false))
.Where(x => x.Type != StreamType.Attachment)
.ToArray();
await ThumbnailsManager.DownloadImages(episode);
await LibraryManager.Create(episode);
await LibraryManager.AddShowLink(show, library, collection);
}
catch (DuplicatedItemException ex)
{
Logger.LogWarning(ex, "Duplicated found at {Path}", path);
{
throw new DuplicatedItemException($"Duplicated show found ({show.Slug}) " +
$"at {registeredShow.Path} and {show.Path}");
}
}
else
show = registeredShow;
// If they are not already loaded, load external ids to allow metadata providers to use them.
if (show.ExternalIDs == null)
await LibraryManager.Load(show, x => x.ExternalIDs);
progress.Report(50);
if (season != null)
season.Show = show;
season = await _RegisterAndFill(season);
progress.Report(60);
episode = await MetadataProvider.Get(episode);
progress.Report(70);
episode.Show = show;
episode.Season = season;
episode.Tracks = (await Transcoder.ExtractInfos(episode, false))
.Where(x => x.Type != StreamType.Attachment)
.ToArray();
await ThumbnailsManager.DownloadImages(episode);
progress.Report(90);
await LibraryManager.Create(episode);
progress.Report(95);
await LibraryManager.AddShowLink(show, library, collection);
progress.Report(100);
}
/// <summary>
/// Retrieve the equivalent item if it already exists in the database,
/// if it does not, fill metadata using the metadata provider, download images and register the item to the
/// database.
/// </summary>
/// <param name="item">The item to retrieve or fill and register</param>
/// <typeparam name="T">The type of the item</typeparam>
/// <returns>The existing or filled item.</returns>
private async Task<T> _RegisterAndFill<T>(T item)
where T : class, IResource
{
@ -141,54 +145,5 @@ namespace Kyoo.Tasks
await ThumbnailsManager.DownloadImages(item);
return await LibraryManager.CreateIfNotExists(item);
}
/*
*
private async Task RegisterExternalSubtitle(string path, CancellationToken token)
{
try
{
if (token.IsCancellationRequested || path.Split(Path.DirectorySeparatorChar).Contains("Subtitles"))
return;
using IServiceScope serviceScope = ServiceProvider.CreateScope();
ILibraryManager libraryManager = serviceScope.ServiceProvider.GetService<ILibraryManager>();
string patern = Config.GetValue<string>("subtitleRegex");
Regex regex = new(patern, RegexOptions.IgnoreCase);
Match match = regex.Match(path);
if (!match.Success)
{
await Console.Error.WriteLineAsync($"The subtitle at {path} does not match the subtitle's regex.");
return;
}
string episodePath = match.Groups["Episode"].Value;
Episode episode = await libraryManager!.Get<Episode>(x => x.Path.StartsWith(episodePath));
Track track = new()
{
Type = StreamType.Subtitle,
Language = match.Groups["Language"].Value,
IsDefault = match.Groups["Default"].Value.Length > 0,
IsForced = match.Groups["Forced"].Value.Length > 0,
Codec = SubtitleExtensions[Path.GetExtension(path)],
IsExternal = true,
Path = path,
Episode = episode
};
await libraryManager.Create(track);
Console.WriteLine($"Registering subtitle at: {path}.");
}
catch (ItemNotFoundException)
{
await Console.Error.WriteLineAsync($"No episode found for subtitle at: ${path}.");
}
catch (Exception ex)
{
await Console.Error.WriteLineAsync($"Unknown error while registering subtitle: {ex.Message}");
}
}
*/
}
}

85
Kyoo/Tasks/RegisterSubtitle.cs Normal file
View File

@ -0,0 +1,85 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using Kyoo.Controllers;
using Kyoo.Models;
using Kyoo.Models.Attributes;
using Kyoo.Models.Exceptions;
namespace Kyoo.Tasks
{
/// <summary>
/// A task to register a new episode
/// </summary>
public class RegisterSubtitle : ITask
{
/// <inheritdoc />
public string Slug => "register-sub";
/// <inheritdoc />
public string Name => "Register subtitle";
/// <inheritdoc />
public string Description => "Register a new subtitle";
/// <inheritdoc />
public string HelpMessage => null;
/// <inheritdoc />
public bool RunOnStartup => false;
/// <inheritdoc />
public int Priority => 0;
/// <inheritdoc />
public bool IsHidden => false;
/// <summary>
/// An identifier to extract metadata from paths.
/// </summary>
[Injected] public IIdentifier Identifier { private get; set; }
/// <summary>
/// The library manager used to register the episode
/// </summary>
[Injected] public ILibraryManager LibraryManager { private get; set; }
/// <inheritdoc />
public TaskParameters GetParameters()
{
return new()
{
TaskParameter.CreateRequired<string>("path", "The path of the episode file"),
};
}
/// <inheritdoc />
public async Task Run(TaskParameters arguments, IProgress<float> progress, CancellationToken cancellationToken)
{
string path = arguments["path"].As<string>();
progress.Report(0);
Track track = await Identifier.IdentifyTrack(path);
progress.Report(25);
if (track.Episode == null)
throw new IdentificationFailed($"No episode identified for the track at {path}");
if (track.Episode.ID == 0)
{
if (track.Episode.Slug != null)
track.Episode = await LibraryManager.Get<Episode>(track.Episode.Slug);
else if (track.Episode.Path != null)
{
track.Episode = await LibraryManager.GetOrDefault<Episode>(x => x.Path == track.Episode.Path);
if (track.Episode == null)
throw new ItemNotFoundException($"No episode found for subtitle at: ${path}.");
}
else
throw new IdentificationFailed($"No episode identified for the track at {path}");
}
progress.Report(50);
await LibraryManager.Create(track);
progress.Report(100);
}
}
}