using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; using Kyoo.Abstractions.Controllers; using Kyoo.Abstractions.Models; using Kyoo.Abstractions.Models.Attributes; using Kyoo.Core.Models.Watch; using Microsoft.Extensions.Logging; namespace Kyoo.Core.Tasks { /// /// A task to add new video files. /// [TaskMetadata("scan", "Scan libraries", "Scan your libraries and load data for new shows.", RunOnStartup = true)] public class Crawler : ITask { /// /// The library manager used to get libraries and providers to use. /// private readonly ILibraryManager _libraryManager; /// /// The file manager used walk inside directories and check they existences. /// private readonly IFileSystem _fileSystem; /// /// A task manager used to create sub tasks for each episode to add to the database. /// private readonly ITaskManager _taskManager; /// /// The logger used to inform the current status to the console. /// private readonly ILogger _logger; /// /// Create a new . /// /// The library manager to retrieve existing episodes/library/tracks /// The file system to glob files /// The task manager used to start . /// The logger used print messages. public Crawler(ILibraryManager libraryManager, IFileSystem fileSystem, ITaskManager taskManager, ILogger logger) { _libraryManager = libraryManager; _fileSystem = fileSystem; _taskManager = taskManager; _logger = logger; } /// public TaskParameters GetParameters() { return new() { TaskParameter.Create("slug", "A library slug to restrict the scan to this library.") }; } /// public async Task Run(TaskParameters arguments, IProgress progress, CancellationToken cancellationToken) { string argument = arguments["slug"].As(); ICollection libraries = argument == null ? await _libraryManager.GetAll() : new [] { await _libraryManager.GetOrDefault(argument)}; if (argument != null && libraries.First() == null) throw new ArgumentException($"No library found with the name {argument}"); foreach (Library library in libraries) await _libraryManager.Load(library, x => x.Providers); progress.Report(0); float percent = 0; ICollection episodes = await _libraryManager.GetAll(); ICollection tracks = await _libraryManager.GetAll(); foreach (Library library in libraries) { IProgress reporter = new Progress(x => { // ReSharper disable once AccessToModifiedClosure progress.Report(percent + x / libraries.Count); }); await Scan(library, episodes, tracks, reporter, cancellationToken); percent += 100f / libraries.Count; if (cancellationToken.IsCancellationRequested) return; } progress.Report(100); } private async Task Scan(Library library, IEnumerable episodes, IEnumerable tracks, IProgress progress, CancellationToken cancellationToken) { _logger.LogInformation("Scanning library {Library} at {Paths}", library.Name, library.Paths); foreach (string path in library.Paths) { ICollection files = await _fileSystem.ListFiles(path, SearchOption.AllDirectories); if (cancellationToken.IsCancellationRequested) return; // We try to group episodes by shows to register one episode of each show first. // This speeds up the scan process because further episodes of a show are registered when all metadata // of the show has already been fetched. List> shows = files .Where(FileExtensions.IsVideo) .Where(x => episodes.All(y => y.Path != x)) .GroupBy(Path.GetDirectoryName) .ToList(); string[] paths = shows.Select(x => x.First()) .Concat(shows.SelectMany(x => x.Skip(1))) .ToArray(); float percent = 0; IProgress reporter = new Progress(x => { // ReSharper disable once AccessToModifiedClosure progress.Report((percent + x / paths.Length - 10) / library.Paths.Length); }); foreach (string episodePath in paths) { _taskManager.StartTask(reporter, new Dictionary { ["path"] = episodePath, ["library"] = library }, cancellationToken); percent += 100f / paths.Length; } string[] subtitles = files .Where(FileExtensions.IsSubtitle) .Where(x => !x.Contains("Extra")) .Where(x => tracks.All(y => y.Path != x)) .ToArray(); percent = 0; reporter = new Progress(x => { // ReSharper disable once AccessToModifiedClosure progress.Report((90 + (percent + x / subtitles.Length)) / library.Paths.Length); }); foreach (string trackPath in subtitles) { _taskManager.StartTask(reporter, new Dictionary { ["path"] = trackPath }, cancellationToken); percent += 100f / subtitles.Length; } } } } }