diff --git a/Kyoo/Controllers/Repositories/CollectionRepository.cs b/Kyoo/Controllers/Repositories/CollectionRepository.cs index 6a30f2ab..6e0151ef 100644 --- a/Kyoo/Controllers/Repositories/CollectionRepository.cs +++ b/Kyoo/Controllers/Repositories/CollectionRepository.cs @@ -5,7 +5,6 @@ using System.Threading.Tasks; using Kyoo.Models; using Kyoo.Models.Exceptions; using Microsoft.EntityFrameworkCore; -using Npgsql; namespace Kyoo.Controllers { @@ -65,6 +64,7 @@ namespace Kyoo.Controllers } catch (DbUpdateException ex) { + _database.DiscardChanges(); if (Helper.IsDuplicateException(ex)) throw new DuplicatedItemException($"Trying to insert a duplicated collection (slug {obj.Slug} already exists)."); throw; diff --git a/Kyoo/Controllers/Repositories/EpisodeRepository.cs b/Kyoo/Controllers/Repositories/EpisodeRepository.cs index f3f05c95..4c698bbc 100644 --- a/Kyoo/Controllers/Repositories/EpisodeRepository.cs +++ b/Kyoo/Controllers/Repositories/EpisodeRepository.cs @@ -5,21 +5,19 @@ using System.Threading.Tasks; using Kyoo.Models; using Kyoo.Models.Exceptions; using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.DependencyInjection; -using Npgsql; namespace Kyoo.Controllers { public class EpisodeRepository : IEpisodeRepository { private readonly DatabaseContext _database; - private readonly IServiceProvider _serviceProvider; + private readonly IProviderRepository _providers; - public EpisodeRepository(DatabaseContext database, IServiceProvider serviceProvider) + public EpisodeRepository(DatabaseContext database, IProviderRepository providers) { _database = database; - _serviceProvider = serviceProvider; + _providers = providers; } public void Dispose() @@ -95,6 +93,8 @@ namespace Kyoo.Controllers } catch (DbUpdateException ex) { + _database.DiscardChanges(); + if (Helper.IsDuplicateException(ex)) throw new DuplicatedItemException($"Trying to insert a duplicated episode (slug {obj.Slug} already exists)."); throw; @@ -148,14 +148,8 @@ namespace Kyoo.Controllers if (obj.ExternalIDs != null) { - obj.ExternalIDs = (await Task.WhenAll(obj.ExternalIDs.Select(async x => - { - using IServiceScope serviceScope = _serviceProvider.CreateScope(); - await using IProviderRepository providers = serviceScope.ServiceProvider.GetService(); - - x.ProviderID = await providers.CreateIfNotExists(x.Provider); - return x; - }))).ToList(); + foreach (MetadataID link in obj.ExternalIDs) + link.ProviderID = await _providers.CreateIfNotExists(link.Provider); } } diff --git a/Kyoo/Controllers/Repositories/GenreRepository.cs b/Kyoo/Controllers/Repositories/GenreRepository.cs index 06d73755..a49249a8 100644 --- a/Kyoo/Controllers/Repositories/GenreRepository.cs +++ b/Kyoo/Controllers/Repositories/GenreRepository.cs @@ -64,6 +64,8 @@ namespace Kyoo.Controllers } catch (DbUpdateException ex) { + _database.DiscardChanges(); + if (Helper.IsDuplicateException(ex)) throw new DuplicatedItemException($"Trying to insert a duplicated genre (slug {obj.Slug} already exists)."); throw; diff --git a/Kyoo/Controllers/Repositories/LibraryRepository.cs b/Kyoo/Controllers/Repositories/LibraryRepository.cs index 257399a1..133b47fe 100644 --- a/Kyoo/Controllers/Repositories/LibraryRepository.cs +++ b/Kyoo/Controllers/Repositories/LibraryRepository.cs @@ -5,20 +5,19 @@ using System.Threading.Tasks; using Kyoo.Models; using Kyoo.Models.Exceptions; using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.DependencyInjection; namespace Kyoo.Controllers { public class LibraryRepository : ILibraryRepository { private readonly DatabaseContext _database; - private readonly IServiceProvider _serviceProvider; + private readonly IProviderRepository _providers; - public LibraryRepository(DatabaseContext database, IServiceProvider serviceProvider) + public LibraryRepository(DatabaseContext database, IProviderRepository providers) { _database = database; - _serviceProvider = serviceProvider; + _providers = providers; } public void Dispose() @@ -71,6 +70,7 @@ namespace Kyoo.Controllers } catch (DbUpdateException ex) { + _database.DiscardChanges(); if (Helper.IsDuplicateException(ex)) throw new DuplicatedItemException($"Trying to insert a duplicated library (slug {obj.Slug} already exists)."); throw; @@ -119,14 +119,9 @@ namespace Kyoo.Controllers private async Task Validate(Library obj) { - obj.ProviderLinks = (await Task.WhenAll(obj.ProviderLinks.Select(async x => - { - using IServiceScope serviceScope = _serviceProvider.CreateScope(); - await using IProviderRepository providers = serviceScope.ServiceProvider.GetService(); - - x.ProviderID = await providers.CreateIfNotExists(x.Provider); - return x; - }))).ToList(); + if (obj.ProviderLinks != null) + foreach (ProviderLink link in obj.ProviderLinks) + link.ProviderID = await _providers.CreateIfNotExists(link.Provider); } public async Task Delete(Library obj) diff --git a/Kyoo/Controllers/Repositories/PeopleRepository.cs b/Kyoo/Controllers/Repositories/PeopleRepository.cs index 2cfae2b9..a1689f25 100644 --- a/Kyoo/Controllers/Repositories/PeopleRepository.cs +++ b/Kyoo/Controllers/Repositories/PeopleRepository.cs @@ -5,19 +5,18 @@ using System.Threading.Tasks; using Kyoo.Models; using Kyoo.Models.Exceptions; using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.DependencyInjection; namespace Kyoo.Controllers { public class PeopleRepository : IPeopleRepository { private readonly DatabaseContext _database; - private readonly IServiceProvider _serviceProvider; + private readonly IProviderRepository _providers; - public PeopleRepository(DatabaseContext database, IServiceProvider serviceProvider) + public PeopleRepository(DatabaseContext database, IProviderRepository providers) { _database = database; - _serviceProvider = serviceProvider; + _providers = providers; } public void Dispose() @@ -70,6 +69,7 @@ namespace Kyoo.Controllers } catch (DbUpdateException ex) { + _database.DiscardChanges(); if (Helper.IsDuplicateException(ex)) throw new DuplicatedItemException($"Trying to insert a duplicated people (slug {obj.Slug} already exists)."); throw; @@ -118,14 +118,9 @@ namespace Kyoo.Controllers private async Task Validate(People obj) { - obj.ExternalIDs = (await Task.WhenAll(obj.ExternalIDs.Select(async x => - { - using IServiceScope serviceScope = _serviceProvider.CreateScope(); - await using IProviderRepository providers = serviceScope.ServiceProvider.GetService(); - - x.ProviderID = await providers.CreateIfNotExists(x.Provider); - return x; - }))).ToList(); + if (obj.ExternalIDs != null) + foreach (MetadataID link in obj.ExternalIDs) + link.ProviderID = await _providers.CreateIfNotExists(link.Provider); } public async Task Delete(People obj) diff --git a/Kyoo/Controllers/Repositories/ProviderRepository.cs b/Kyoo/Controllers/Repositories/ProviderRepository.cs index 7d4c3dfb..ab1abee9 100644 --- a/Kyoo/Controllers/Repositories/ProviderRepository.cs +++ b/Kyoo/Controllers/Repositories/ProviderRepository.cs @@ -64,6 +64,7 @@ namespace Kyoo.Controllers } catch (DbUpdateException ex) { + _database.DiscardChanges(); if (Helper.IsDuplicateException(ex)) throw new DuplicatedItemException($"Trying to insert a duplicated provider (name {obj.Name} already exists)."); throw; diff --git a/Kyoo/Controllers/Repositories/SeasonRepository.cs b/Kyoo/Controllers/Repositories/SeasonRepository.cs index 23eb188c..f88c5c2f 100644 --- a/Kyoo/Controllers/Repositories/SeasonRepository.cs +++ b/Kyoo/Controllers/Repositories/SeasonRepository.cs @@ -5,20 +5,19 @@ using System.Threading.Tasks; using Kyoo.Models; using Kyoo.Models.Exceptions; using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.DependencyInjection; namespace Kyoo.Controllers { public class SeasonRepository : ISeasonRepository { private readonly DatabaseContext _database; - private readonly IServiceProvider _serviceProvider; + private readonly IProviderRepository _providers; - public SeasonRepository(DatabaseContext database, IServiceProvider serviceProvider) + public SeasonRepository(DatabaseContext database, IProviderRepository providers) { _database = database; - _serviceProvider = serviceProvider; + _providers = providers; } public void Dispose() @@ -83,6 +82,7 @@ namespace Kyoo.Controllers } catch (DbUpdateException ex) { + _database.DiscardChanges(); if (Helper.IsDuplicateException(ex)) throw new DuplicatedItemException($"Trying to insert a duplicated season (slug {obj.Slug} already exists)."); throw; @@ -137,14 +137,8 @@ namespace Kyoo.Controllers if (obj.ExternalIDs != null) { - obj.ExternalIDs = (await Task.WhenAll(obj.ExternalIDs.Select(async x => - { - using IServiceScope serviceScope = _serviceProvider.CreateScope(); - await using IProviderRepository providers = serviceScope.ServiceProvider.GetService(); - - x.ProviderID = await providers.CreateIfNotExists(x.Provider); - return x; - }))).ToList(); + foreach (MetadataID link in obj.ExternalIDs) + link.ProviderID = await _providers.CreateIfNotExists(link.Provider); } } diff --git a/Kyoo/Controllers/Repositories/ShowRepository.cs b/Kyoo/Controllers/Repositories/ShowRepository.cs index 7d9b41b4..c39741b9 100644 --- a/Kyoo/Controllers/Repositories/ShowRepository.cs +++ b/Kyoo/Controllers/Repositories/ShowRepository.cs @@ -5,23 +5,28 @@ using System.Threading.Tasks; using Kyoo.Models; using Kyoo.Models.Exceptions; using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.DependencyInjection; namespace Kyoo.Controllers { public class ShowRepository : IShowRepository { private readonly DatabaseContext _database; - private readonly IServiceProvider _serviceProvider; private readonly IStudioRepository _studios; + private readonly IPeopleRepository _people; + private readonly IGenreRepository _genres; + private readonly IProviderRepository _providers; public ShowRepository(DatabaseContext database, - IServiceProvider serviceProvider, - IStudioRepository studios) + IStudioRepository studios, + IPeopleRepository people, + IGenreRepository genres, + IProviderRepository providers) { _database = database; - _serviceProvider = serviceProvider; _studios = studios; + _people = people; + _genres = genres; + _providers = providers; } public void Dispose() @@ -87,6 +92,7 @@ namespace Kyoo.Controllers } catch (DbUpdateException ex) { + _database.DiscardChanges(); if (Helper.IsDuplicateException(ex)) throw new DuplicatedItemException($"Trying to insert a duplicated show (slug {obj.Slug} already exists)."); throw; @@ -140,38 +146,20 @@ namespace Kyoo.Controllers if (obj.GenreLinks != null) { - obj.GenreLinks = (await Task.WhenAll(obj.GenreLinks.Select(async x => - { - using IServiceScope serviceScope = _serviceProvider.CreateScope(); - await using IGenreRepository genres = serviceScope.ServiceProvider.GetService(); - - x.GenreID = await genres.CreateIfNotExists(x.Genre); - return x; - }))).ToList(); + foreach (GenreLink link in obj.GenreLinks) + link.GenreID = await _genres.CreateIfNotExists(link.Genre); } if (obj.People != null) { - obj.People = (await Task.WhenAll(obj.People.Select(async x => - { - using IServiceScope serviceScope = _serviceProvider.CreateScope(); - await using IPeopleRepository people = serviceScope.ServiceProvider.GetService(); - - x.PeopleID = await people.CreateIfNotExists(x.People); - return x; - }))).ToList(); + foreach (PeopleLink link in obj.People) + link.PeopleID = await _people.CreateIfNotExists(link.People); } if (obj.ExternalIDs != null) { - obj.ExternalIDs = (await Task.WhenAll(obj.ExternalIDs.Select(async x => - { - using IServiceScope serviceScope = _serviceProvider.CreateScope(); - await using IProviderRepository providers = serviceScope.ServiceProvider.GetService(); - - x.ProviderID = await providers.CreateIfNotExists(x.Provider); - return x; - }))).ToList(); + foreach (MetadataID link in obj.ExternalIDs) + link.ProviderID = await _providers.CreateIfNotExists(link.Provider); } } diff --git a/Kyoo/Controllers/Repositories/StudioRepository.cs b/Kyoo/Controllers/Repositories/StudioRepository.cs index c408234c..8e7c3a02 100644 --- a/Kyoo/Controllers/Repositories/StudioRepository.cs +++ b/Kyoo/Controllers/Repositories/StudioRepository.cs @@ -64,6 +64,7 @@ namespace Kyoo.Controllers } catch (DbUpdateException ex) { + _database.DiscardChanges(); if (Helper.IsDuplicateException(ex)) throw new DuplicatedItemException($"Trying to insert a duplicated studio (slug {obj.Slug} already exists)."); throw; diff --git a/Kyoo/Controllers/Repositories/TrackRepository.cs b/Kyoo/Controllers/Repositories/TrackRepository.cs index 550cee01..711ca9f7 100644 --- a/Kyoo/Controllers/Repositories/TrackRepository.cs +++ b/Kyoo/Controllers/Repositories/TrackRepository.cs @@ -70,6 +70,7 @@ namespace Kyoo.Controllers } catch (DbUpdateException ex) { + _database.DiscardChanges(); if (Helper.IsDuplicateException(ex)) throw new DuplicatedItemException($"Trying to insert a duplicated track (slug {obj.Slug} already exists)."); throw; diff --git a/Kyoo/Models/DatabaseContext.cs b/Kyoo/Models/DatabaseContext.cs index 429ea4ff..e9407992 100644 --- a/Kyoo/Models/DatabaseContext.cs +++ b/Kyoo/Models/DatabaseContext.cs @@ -146,6 +146,15 @@ namespace Kyoo .HasIndex(x => x.Name) .IsUnique(); } + + public void DiscardChanges() + { + foreach (EntityEntry entry in ChangeTracker.Entries().Where(x => x.State != EntityState.Unchanged + && x.State != EntityState.Detached)) + { + entry.State = EntityState.Detached; + } + } } } diff --git a/Kyoo/Startup.cs b/Kyoo/Startup.cs index 952c4e6a..65f99812 100644 --- a/Kyoo/Startup.cs +++ b/Kyoo/Startup.cs @@ -49,7 +49,7 @@ namespace Kyoo .UseNpgsql(_configuration.GetConnectionString("Database")) .EnableSensitiveDataLogging(); // .UseLoggerFactory(LoggerFactory.Create(builder => builder.AddConsole())); - }); + }, ServiceLifetime.Transient); services.AddDbContext(options => { diff --git a/Kyoo/Tasks/Crawler.cs b/Kyoo/Tasks/Crawler.cs index 8947e083..6ffb7df0 100644 --- a/Kyoo/Tasks/Crawler.cs +++ b/Kyoo/Tasks/Crawler.cs @@ -27,8 +27,6 @@ namespace Kyoo.Controllers private IProviderManager _metadataProvider; private ITranscoder _transcoder; private IConfiguration _config; - - private int _parallelRegisters; public async Task> GetPossibleParameters() { @@ -52,10 +50,6 @@ namespace Kyoo.Controllers _metadataProvider = serviceProvider.GetService(); _transcoder = serviceProvider.GetService(); _config = serviceProvider.GetService(); - _parallelRegisters = _config.GetValue("parallelRegisters"); - - if (_parallelRegisters <= 0) - _parallelRegisters = 10; using IServiceScope serviceScope = _serviceProvider.CreateScope(); await using ILibraryManager libraryManager = serviceScope.ServiceProvider.GetService(); @@ -74,17 +68,18 @@ namespace Kyoo.Controllers foreach (Library library in libraries) library.Providers = library.Providers; - await Task.WhenAll(libraries.Select(x => Scan(x, episodes, cancellationToken)).ToArray()); + foreach (Library library in libraries) + await Scan(library, episodes, cancellationToken); Console.WriteLine("Scan finished!"); } - private Task Scan(Library library, IEnumerable episodes, CancellationToken cancellationToken) + private async Task Scan(Library library, IEnumerable episodes, CancellationToken cancellationToken) { Console.WriteLine($"Scanning library {library.Name} at {string.Join(", ", library.Paths)}."); - return Task.WhenAll(library.Paths.Select(async path => + foreach (string path in library.Paths) { if (cancellationToken.IsCancellationRequested) - return; + continue; string[] files; try @@ -94,22 +89,22 @@ namespace Kyoo.Controllers catch (DirectoryNotFoundException) { await Console.Error.WriteLineAsync($"The library's directory {path} could not be found (library slug: {library.Slug})"); - return; + continue; } catch (PathTooLongException) { await Console.Error.WriteLineAsync($"The library's directory {path} is too long for this system. (library slug: {library.Slug})"); - return; + continue; } catch (ArgumentException) { await Console.Error.WriteLineAsync($"The library's directory {path} is invalid. (library slug: {library.Slug})"); - return; + continue; } - catch (UnauthorizedAccessException) + catch (UnauthorizedAccessException ex) { - await Console.Error.WriteLineAsync($"Permission denied: can't access library's directory at {path}. (library slug: {library.Slug})"); - return; + await Console.Error.WriteLineAsync($"{ex.Message} (library slug: {library.Slug})"); + continue; } List> shows = files @@ -117,20 +112,23 @@ namespace Kyoo.Controllers .GroupBy(Path.GetDirectoryName) .ToList(); + // Todo batch wth higher numbers per list once multi-services has been implemented. + List tasks = shows .Select(x => x.First()) .Select(x => RegisterFile(x, x.Substring(path.Length), library, cancellationToken)) .ToList(); - foreach (List showTasks in tasks.BatchBy(_parallelRegisters)) + // TODO EXECUTION OF THE TASKS ARE CALCULATED DURING THE .ToList() CALL, NO BACHING/ASYNC IS DONE. + foreach (List showTasks in tasks.BatchBy(1)) await Task.WhenAll(showTasks); tasks = shows .SelectMany(x => x.Skip(1)) .Select(x => RegisterFile(x, x.Substring(path.Length), library, cancellationToken)) .ToList(); - foreach (List episodeTasks in tasks.BatchBy(_parallelRegisters * 3)) + foreach (List episodeTasks in tasks.BatchBy(1)) await Task.WhenAll(episodeTasks); - }).ToArray()); + } } private async Task RegisterFile(string path, string relativePath, Library library, CancellationToken token) diff --git a/Kyoo/appsettings.json b/Kyoo/appsettings.json index 70b961b1..0e8f69a9 100644 --- a/Kyoo/appsettings.json +++ b/Kyoo/appsettings.json @@ -15,9 +15,8 @@ "AllowedHosts": "*", "ConnectionStrings": { - "Database": "Server=127.0.0.1; Port=5432; Database=kyooDB; User Id=kyoo; Password=kyooPassword; Pooling=true; MaxPoolSize=80; Timeout=30;" + "Database": "Server=127.0.0.1; Port=5432; Database=kyooDB; User Id=kyoo; Password=kyooPassword; Pooling=true; MaxPoolSize=95; Timeout=30;" }, - "parallelRegisters": "30", "scheduledTasks": { "scan": "24:00:00"