Cutlery/Kyoo
1
0
Fork 0
mirror of https://github.com/zoriya/Kyoo.git synced 2025-05-24 02:02:36 -04:00
Code Issues Packages Projects Releases Wiki Activity

Threading the crawler

Browse Source
This commit is contained in:
Zoe Roux 2020-05-07 04:21:58 +02:00
parent f720545d67
commit 90d0180650
4 changed files with 42 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Kyoo.Common/Controllers/ILibraryManager.cs
View File

@ -23,10 +23,6 @@ namespace Kyoo.Controllers
IEnumerable<Studio> GetStudios();
IEnumerable<Genre> GetGenres();
// Other get helpers
Show GetShowByPath(string path);
IEnumerable<string> GetLibrariesPath();
// Search
IEnumerable<Collection> SearchCollections(string searchQuery);
IEnumerable<Show> SearchShows(string searchQuery);
@ -34,6 +30,10 @@ namespace Kyoo.Controllers
IEnumerable<Genre> SearchGenres(string searchQuery);
IEnumerable<Studio> SearchStudios(string searchQuery);
IEnumerable<People> SearchPeople(string searchQuery);
// Other get helpers
Show GetShowByPath(string path);
IEnumerable<string> GetLibrariesPath();
//Register values
void Register(object obj);

9
Kyoo/Controllers/LibraryManager.cs
View File

@ -153,11 +153,14 @@ namespace Kyoo.Controllers
{
if (collection != null)
{
_database.LibraryLinks.AddIfNotExist(new LibraryLink {LibraryID = library.ID, CollectionID = collection.ID}, x => x.LibraryID == library.ID && x.CollectionID == collection.ID && x.ShowID == null);
_database.CollectionLinks.AddIfNotExist(new CollectionLink { CollectionID = collection.ID, ShowID = show.ID}, x => x.CollectionID == collection.ID && x.ShowID == show.ID);
_database.LibraryLinks.AddIfNotExist(new LibraryLink {Library = library, Collection = collection},
x => x.Library == library && x.Collection == collection && x.ShowID == null);
_database.CollectionLinks.AddIfNotExist(new CollectionLink { Collection = collection, Show = show},
x => x.Collection == collection && x.Show == show);
}
else
_database.LibraryLinks.AddIfNotExist(new LibraryLink {LibraryID = library.ID, ShowID = show.ID}, x => x.LibraryID == library.ID && x.CollectionID == null && x.ShowID == show.ID);
_database.LibraryLinks.AddIfNotExist(new LibraryLink {Library = library, Show = show},
x => x.Library == library && x.Collection == null && x.Show == show);
}
public Task SaveChanges()

63
Kyoo/Tasks/Crawler.cs
View File

@ -57,9 +57,10 @@ namespace Kyoo.Controllers
foreach (Episode episode in episodes)
{
if (!File.Exists(episode.Path))
_libraryManager.RemoveEpisode(episode.ID);
_libraryManager.RemoveEpisode(episode);
}
await _libraryManager.SaveChanges();
foreach (Library library in libraries)
await Scan(library, cancellationToken);
}
@ -75,21 +76,25 @@ namespace Kyoo.Controllers
Console.WriteLine($"Scanning library {library.Name} at {string.Join(", ", library.Paths)}.");
foreach (string path in library.Paths)
{
foreach (string file in Directory.GetFiles(path, "*", SearchOption.AllDirectories))
if (cancellationToken.IsCancellationRequested)
return;
await Task.WhenAll(Directory.GetFiles(path, "*", SearchOption.AllDirectories).Select(file =>
{
if (cancellationToken.IsCancellationRequested)
return;
if (!IsVideo(file) || _libraryManager.IsEpisodeRegistered(file, out long _))
continue;
if (!IsVideo(file) || _libraryManager.GetEpisodes().Any(x => x.Path == file))
return null;
string relativePath = file.Substring(path.Length);
await RegisterFile(file, relativePath, library);
}
return RegisterFile(file, relativePath, library, cancellationToken);
}));
}
}
private async Task RegisterFile(string path, string relativePath, Library library)
private async Task RegisterFile(string path, string relativePath, Library library, CancellationToken token)
{
Console.WriteLine("Registering episode at: " + path);
if (token.IsCancellationRequested)
return;
Console.WriteLine($"Registering episode at: {path}");
string patern = _config.GetValue<string>("regex");
Regex regex = new Regex(patern, RegexOptions.IgnoreCase);
Match match = regex.Match(relativePath);
@ -105,17 +110,17 @@ namespace Kyoo.Controllers
bool isMovie = seasonNumber == -1 && episodeNumber == -1 && absoluteNumber == -1;
Show show = await GetShow(showName, showPath, isMovie, library);
if (isMovie)
_libraryManager.RegisterMovie(await GetMovie(show, path));
_libraryManager.Register(await GetMovie(show, path));
else
{
Season season = await GetSeason(show, seasonNumber, library);
Episode episode = await GetEpisode(show, season, episodeNumber, absoluteNumber, path, library);
if (_libraryManager.RegisterEpisode(episode) == 0)
return;
_libraryManager.Register(episode);
}
if (collection != null)
_libraryManager.RegisterCollection(collection);
_libraryManager.Register(collection);
_libraryManager.RegisterShowLinks(library, collection, show);
await _libraryManager.SaveChanges();
}
private async Task<Collection> GetCollection(string collectionName, Library library)
@ -132,24 +137,10 @@ namespace Kyoo.Controllers
return show;
show = await _metadataProvider.SearchShow(showTitle, isMovie, library);
show.Path = showPath;
show.People = (await _metadataProvider.GetPeople(show, library)).GroupBy(x => x.Slug).Select(x => x.First())
.Select(x =>
{
People existing = _libraryManager.GetPeople(x.Slug);
if (existing != null)
return new PeopleLink(existing, show, x.Role, x.Type);
x.People.ExternalIDs = _libraryManager.ValidateExternalIDs(x.People.ExternalIDs);
return x;
}).ToList();
show.People = await _thumbnailsManager.Validate(show.People);
show.Genres = show.Genres?.Select(x =>
{
Genre existing = _libraryManager.GetGenre(x.Slug);
return existing ?? x;
});
show.ExternalIDs = _libraryManager.ValidateExternalIDs(show.ExternalIDs);
if (show.Studio != null)
show.Studio = _libraryManager.GetStudio(show.Studio.Slug) ?? show.Studio;
show.People = (await _metadataProvider.GetPeople(show, library))
.GroupBy(x => x.Slug)
.Select(x => x.First());
await _thumbnailsManager.Validate(show.People);
await _thumbnailsManager.Validate(show);
return show;
}
@ -162,7 +153,6 @@ namespace Kyoo.Controllers
if (season == null)
{
season = await _metadataProvider.GetSeason(show, seasonNumber, library);
season.ExternalIDs = _libraryManager.ValidateExternalIDs(season.ExternalIDs);
await _thumbnailsManager.Validate(season);
}
season.Show = show;
@ -177,11 +167,10 @@ namespace Kyoo.Controllers
episode.Season = season;
if (season == null)
{
Console.Error.WriteLine("\tError: You don't have any provider that support absolute epiode numbering. Install one and try again.");
return null;
await Console.Error.WriteLineAsync("\tError: You don't have any provider that support absolute epiode numbering. Install one and try again.");
return default;
}
episode.ExternalIDs = _libraryManager.ValidateExternalIDs(episode.ExternalIDs);
await _thumbnailsManager.Validate(episode);
await GetTracks(episode);
return episode;

13
Kyoo/Tasks/ReScan.cs
View File

@ -5,7 +5,6 @@ using System.Threading;
using System.Threading.Tasks;
using Kyoo.Controllers;
using Kyoo.Models;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
namespace Kyoo.Tasks
@ -53,12 +52,12 @@ namespace Kyoo.Tasks
Show old = _database.Shows.FirstOrDefault(x => x.Slug == slug);
if (old == null)
return;
Library library = _libraryManager.GetLibraryForShow(slug);
Library library = _database.LibraryLinks.First(x => x.Show == old && x.Library != null).Library;
Show edited = await _providerManager.CompleteShow(old, library);
edited.ID = old.ID;
edited.Slug = old.Slug;
edited.Path = old.Path;
_libraryManager.EditShow(edited);
_libraryManager.Edit(edited, true);
await _thumbnailsManager.Validate(edited, true);
if (old.Seasons != null)
await Task.WhenAll(old.Seasons.Select(x => ReScanSeason(old, x)));
@ -84,10 +83,10 @@ namespace Kyoo.Tasks
private async Task ReScanSeason(Show show, Season old)
{
Library library = _libraryManager.GetLibraryForShow(show.Slug);
Library library = _database.LibraryLinks.First(x => x.Show == show && x.Library != null).Library;
Season edited = await _providerManager.GetSeason(show, old.SeasonNumber, library);
edited.ID = old.ID;
_libraryManager.EditSeason(edited);
_libraryManager.Edit(edited, true);
await _thumbnailsManager.Validate(edited, true);
if (old.Episodes != null)
await Task.WhenAll(old.Episodes.Select(x => ReScanEpisode(show, x)));
@ -95,10 +94,10 @@ namespace Kyoo.Tasks
private async Task ReScanEpisode(Show show, Episode old)
{
Library library = _libraryManager.GetLibraryForShow(show.Slug);
Library library = _database.LibraryLinks.First(x => x.Show == show && x.Library != null).Library;
Episode edited = await _providerManager.GetEpisode(show, old.Path, old.SeasonNumber, old.EpisodeNumber, old.AbsoluteNumber, library);
edited.ID = old.ID;
_libraryManager.EditEpisode(edited);
_libraryManager.Edit(edited, true);
await _thumbnailsManager.Validate(edited, true);
}