Cutlery/Kyoo
1
0
Fork 0
mirror of https://github.com/zoriya/Kyoo.git synced 2025-06-07 07:34:13 -04:00
Code Issues Packages Projects Releases Wiki Activity

Identifing threading issues with the crawler

Browse Source
This commit is contained in:
Zoe Roux 2020-05-26 04:10:29 +02:00
parent d5439fafe3
commit 3684ea73f3
2 changed files with 21 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Kyoo/Startup.cs
View File

@ -53,8 +53,8 @@ namespace Kyoo
{ {
options.UseLazyLoadingProxies() options.UseLazyLoadingProxies()
.UseSqlite(_configuration.GetConnectionString("Database")); .UseSqlite(_configuration.GetConnectionString("Database"));
//.EnableSensitiveDataLogging() // .EnableSensitiveDataLogging()
//.UseLoggerFactory(LoggerFactory.Create(builder => builder.AddConsole())); // .UseLoggerFactory(LoggerFactory.Create(builder => builder.AddConsole()));
}); });
services.AddDbContext<IdentityDatabase>(options => services.AddDbContext<IdentityDatabase>(options =>

30
Kyoo/Tasks/Crawler.cs
View File

@ -65,6 +65,7 @@ namespace Kyoo.Controllers
} }
await libraryManager.SaveChanges(); await libraryManager.SaveChanges();
// await Task.WhenAll(libraries.Select(x => Scan(x, libraryManager, cancellationToken)));
foreach (Library library in libraries) foreach (Library library in libraries)
await Scan(library, libraryManager, cancellationToken); await Scan(library, libraryManager, cancellationToken);
} }
@ -75,13 +76,15 @@ namespace Kyoo.Controllers
Console.WriteLine("Scan finished!"); Console.WriteLine("Scan finished!");
} }
private async Task Scan(Library library, ILibraryManager libraryManager, CancellationToken cancellationToken) private async Task<Task> Scan(Library library, ILibraryManager libraryManager, CancellationToken cancellationToken)
{ {
Console.WriteLine($"Scanning library {library.Name} at {string.Join(", ", library.Paths)}."); Console.WriteLine($"Scanning library {library.Name} at {string.Join(", ", library.Paths)}.");
// return Task.WhenAll(library.Paths.Select(path =>
foreach (string path in library.Paths) foreach (string path in library.Paths)
{ {
if (cancellationToken.IsCancellationRequested) if (cancellationToken.IsCancellationRequested)
return; return Task.CompletedTask;
string[] files; string[] files;
try try
{ {
@ -90,33 +93,34 @@ namespace Kyoo.Controllers
catch (DirectoryNotFoundException) catch (DirectoryNotFoundException)
{ {
Console.Error.WriteLine($"The library's directory {path} could not be found (library slug: {library.Slug})"); Console.Error.WriteLine($"The library's directory {path} could not be found (library slug: {library.Slug})");
continue; return Task.CompletedTask;
} }
catch (PathTooLongException) catch (PathTooLongException)
{ {
Console.Error.WriteLine($"The library's directory {path} is too long for this system. (library slug: {library.Slug})"); Console.Error.WriteLine($"The library's directory {path} is too long for this system. (library slug: {library.Slug})");
continue; return Task.CompletedTask;
} }
catch (ArgumentException) catch (ArgumentException)
{ {
Console.Error.WriteLine($"The library's directory {path} is invalid. (library slug: {library.Slug})"); Console.Error.WriteLine($"The library's directory {path} is invalid. (library slug: {library.Slug})");
continue; return Task.CompletedTask;
} }
catch (UnauthorizedAccessException) catch (UnauthorizedAccessException)
{ {
Console.Error.WriteLine($"Permission denied: can't access library's directory at {path}. (library slug: {library.Slug})"); Console.Error.WriteLine($"Permission denied: can't access library's directory at {path}. (library slug: {library.Slug})");
continue; return Task.CompletedTask;
} }
// await Task.WhenAll(files.Select(file =>
foreach (var file in files) // return Task.WhenAll(files.Select(file =>
foreach (string file in files)
{ {
if (!IsVideo(file) || libraryManager.GetEpisodes().Any(x => x.Path == file)) if (!IsVideo(file) || libraryManager.GetEpisodes().Any(x => x.Path == file))
continue; continue; //return Task.CompletedTask;
// return Task.CompletedTask;
string relativePath = file.Substring(path.Length); string relativePath = file.Substring(path.Length);
await RegisterFile(file, relativePath, library, cancellationToken); /*return*/ await RegisterFile(file, relativePath, library, cancellationToken);
}//)); }//));
} }//));
return Task.CompletedTask;
} }
private async Task RegisterFile(string path, string relativePath, Library library, CancellationToken token) private async Task RegisterFile(string path, string relativePath, Library library, CancellationToken token)
@ -128,7 +132,6 @@ namespace Kyoo.Controllers
ILibraryManager libraryManager = serviceScope.ServiceProvider.GetService<ILibraryManager>(); ILibraryManager libraryManager = serviceScope.ServiceProvider.GetService<ILibraryManager>();
((DbSet<Library>)libraryManager.GetLibraries()).Attach(library); ((DbSet<Library>)libraryManager.GetLibraries()).Attach(library);
Console.WriteLine($"Registering episode at: {path}");
string patern = _config.GetValue<string>("regex"); string patern = _config.GetValue<string>("regex");
Regex regex = new Regex(patern, RegexOptions.IgnoreCase); Regex regex = new Regex(patern, RegexOptions.IgnoreCase);
Match match = regex.Match(relativePath); Match match = regex.Match(relativePath);
@ -154,6 +157,7 @@ namespace Kyoo.Controllers
if (collection != null) if (collection != null)
libraryManager.Register(collection); libraryManager.Register(collection);
libraryManager.RegisterShowLinks(library, collection, show); libraryManager.RegisterShowLinks(library, collection, show);
Console.WriteLine($"Registering episode at: {path}");
await libraryManager.SaveChanges(); await libraryManager.SaveChanges();
} }