Cleaning up repositories

Kyoo/Controllers/Repositories/CollectionRepository.cs

@@ -5,7 +5,6 @@ using System.Threading.Tasks;
 using Kyoo.Models;
 using Kyoo.Models.Exceptions;
 using Microsoft.EntityFrameworkCore;
-using Npgsql;
 
 namespace Kyoo.Controllers
 {
@@ -65,6 +64,7 @@ namespace Kyoo.Controllers
 			}
 			catch (DbUpdateException ex)
 			{
+				_database.DiscardChanges();
 				if (Helper.IsDuplicateException(ex))
 					throw new DuplicatedItemException($"Trying to insert a duplicated collection (slug {obj.Slug} already exists).");
 				throw;

Kyoo/Controllers/Repositories/EpisodeRepository.cs

@@ -5,21 +5,19 @@ using System.Threading.Tasks;
 using Kyoo.Models;
 using Kyoo.Models.Exceptions;
 using Microsoft.EntityFrameworkCore;
-using Microsoft.Extensions.DependencyInjection;
-using Npgsql;
 
 namespace Kyoo.Controllers
 {
 	public class EpisodeRepository : IEpisodeRepository
 	{
 		private readonly DatabaseContext _database;
-		private readonly IServiceProvider _serviceProvider;
+		private readonly IProviderRepository _providers;
 
 
-		public EpisodeRepository(DatabaseContext database, IServiceProvider serviceProvider)
+		public EpisodeRepository(DatabaseContext database, IProviderRepository providers)
 		{
 			_database = database;
-			_serviceProvider = serviceProvider;
+			_providers = providers;
 		}
 		
 		public void Dispose()
@@ -95,6 +93,8 @@ namespace Kyoo.Controllers
 			}
 			catch (DbUpdateException ex)
 			{
+				_database.DiscardChanges();
+				
 				if (Helper.IsDuplicateException(ex))
 					throw new DuplicatedItemException($"Trying to insert a duplicated episode (slug {obj.Slug} already exists).");
 				throw;
@@ -148,14 +148,8 @@ namespace Kyoo.Controllers
 
 			if (obj.ExternalIDs != null)
 			{
-				obj.ExternalIDs = (await Task.WhenAll(obj.ExternalIDs.Select(async x =>
+				foreach (MetadataID link in obj.ExternalIDs)
-				{
+					link.ProviderID = await _providers.CreateIfNotExists(link.Provider);
-					using IServiceScope serviceScope = _serviceProvider.CreateScope();
-					await using IProviderRepository providers = serviceScope.ServiceProvider.GetService<IProviderRepository>();
-
-					x.ProviderID = await providers.CreateIfNotExists(x.Provider);
-					return x;
-				}))).ToList();
 			}
 		}
 		

Kyoo/Controllers/Repositories/GenreRepository.cs

@@ -64,6 +64,8 @@ namespace Kyoo.Controllers
 			}
 			catch (DbUpdateException ex)
 			{
+				_database.DiscardChanges();
+				
 				if (Helper.IsDuplicateException(ex))
 					throw new DuplicatedItemException($"Trying to insert a duplicated genre (slug {obj.Slug} already exists).");
 				throw;

Kyoo/Controllers/Repositories/LibraryRepository.cs

@@ -5,20 +5,19 @@ using System.Threading.Tasks;
 using Kyoo.Models;
 using Kyoo.Models.Exceptions;
 using Microsoft.EntityFrameworkCore;
-using Microsoft.Extensions.DependencyInjection;
 
 namespace Kyoo.Controllers
 {
 	public class LibraryRepository : ILibraryRepository
 	{
 		private readonly DatabaseContext _database;
-		private readonly IServiceProvider _serviceProvider;
+		private readonly IProviderRepository _providers;
 
 
-		public LibraryRepository(DatabaseContext database, IServiceProvider serviceProvider)
+		public LibraryRepository(DatabaseContext database, IProviderRepository providers)
 		{
 			_database = database;
-			_serviceProvider = serviceProvider;
+			_providers = providers;
 		}
 
 		public void Dispose()
@@ -71,6 +70,7 @@ namespace Kyoo.Controllers
 			}
 			catch (DbUpdateException ex)
 			{
+				_database.DiscardChanges();
 				if (Helper.IsDuplicateException(ex))
 					throw new DuplicatedItemException($"Trying to insert a duplicated library (slug {obj.Slug} already exists).");
 				throw;
@@ -119,14 +119,9 @@ namespace Kyoo.Controllers
 
 		private async Task Validate(Library obj)
 		{
-			obj.ProviderLinks = (await Task.WhenAll(obj.ProviderLinks.Select(async x =>
+			if (obj.ProviderLinks != null)
-			{
+				foreach (ProviderLink link in obj.ProviderLinks)
-				using IServiceScope serviceScope = _serviceProvider.CreateScope();
+					link.ProviderID = await _providers.CreateIfNotExists(link.Provider);
-				await using IProviderRepository providers = serviceScope.ServiceProvider.GetService<IProviderRepository>();
-				
-				x.ProviderID = await providers.CreateIfNotExists(x.Provider);
-				return x;
-			}))).ToList();
 		}
 
 		public async Task Delete(Library obj)

Kyoo/Controllers/Repositories/PeopleRepository.cs

@@ -5,19 +5,18 @@ using System.Threading.Tasks;
 using Kyoo.Models;
 using Kyoo.Models.Exceptions;
 using Microsoft.EntityFrameworkCore;
-using Microsoft.Extensions.DependencyInjection;
 
 namespace Kyoo.Controllers
 {
 	public class PeopleRepository : IPeopleRepository
 	{
 		private readonly DatabaseContext _database;
-		private readonly IServiceProvider _serviceProvider;
+		private readonly IProviderRepository _providers;
 
-		public PeopleRepository(DatabaseContext database, IServiceProvider serviceProvider)
+		public PeopleRepository(DatabaseContext database, IProviderRepository providers)
 		{
 			_database = database;
-			_serviceProvider = serviceProvider;
+			_providers = providers;
 		}
 		
 		public void Dispose()
@@ -70,6 +69,7 @@ namespace Kyoo.Controllers
 			}
 			catch (DbUpdateException ex)
 			{
+				_database.DiscardChanges();
 				if (Helper.IsDuplicateException(ex))
 					throw new DuplicatedItemException($"Trying to insert a duplicated people (slug {obj.Slug} already exists).");
 				throw;
@@ -118,14 +118,9 @@ namespace Kyoo.Controllers
 		
 		private async Task Validate(People obj)
 		{
-			obj.ExternalIDs = (await Task.WhenAll(obj.ExternalIDs.Select(async x =>
+			if (obj.ExternalIDs != null)
-			{
+				foreach (MetadataID link in obj.ExternalIDs)
-				using IServiceScope serviceScope = _serviceProvider.CreateScope();
+					link.ProviderID = await _providers.CreateIfNotExists(link.Provider);
-				await using IProviderRepository providers = serviceScope.ServiceProvider.GetService<IProviderRepository>();
-				
-				x.ProviderID = await providers.CreateIfNotExists(x.Provider);
-				return x;
-			}))).ToList();
 		}
 
 		public async Task Delete(People obj)

Kyoo/Controllers/Repositories/ProviderRepository.cs

@@ -64,6 +64,7 @@ namespace Kyoo.Controllers
 			}
 			catch (DbUpdateException ex)
 			{
+				_database.DiscardChanges();
 				if (Helper.IsDuplicateException(ex))
 					throw new DuplicatedItemException($"Trying to insert a duplicated provider (name {obj.Name} already exists).");
 				throw;

Kyoo/Controllers/Repositories/SeasonRepository.cs

@@ -5,20 +5,19 @@ using System.Threading.Tasks;
 using Kyoo.Models;
 using Kyoo.Models.Exceptions;
 using Microsoft.EntityFrameworkCore;
-using Microsoft.Extensions.DependencyInjection;
 
 namespace Kyoo.Controllers
 {
 	public class SeasonRepository : ISeasonRepository
 	{
 		private readonly DatabaseContext _database;
-		private readonly IServiceProvider _serviceProvider;
+		private readonly IProviderRepository _providers;
 
 
-		public SeasonRepository(DatabaseContext database, IServiceProvider serviceProvider)
+		public SeasonRepository(DatabaseContext database, IProviderRepository providers)
 		{
 			_database = database;
-			_serviceProvider = serviceProvider;
+			_providers = providers;
 		}
 		
 		public void Dispose()
@@ -83,6 +82,7 @@ namespace Kyoo.Controllers
 			}
 			catch (DbUpdateException ex)
 			{
+				_database.DiscardChanges();
 				if (Helper.IsDuplicateException(ex))
 					throw new DuplicatedItemException($"Trying to insert a duplicated season (slug {obj.Slug} already exists).");
 				throw;
@@ -137,14 +137,8 @@ namespace Kyoo.Controllers
 
 			if (obj.ExternalIDs != null)
 			{
-				obj.ExternalIDs = (await Task.WhenAll(obj.ExternalIDs.Select(async x =>
+				foreach (MetadataID link in obj.ExternalIDs)
-				{
+					link.ProviderID = await _providers.CreateIfNotExists(link.Provider);
-					using IServiceScope serviceScope = _serviceProvider.CreateScope();
-					await using IProviderRepository providers = serviceScope.ServiceProvider.GetService<IProviderRepository>();
-					
-					x.ProviderID = await providers.CreateIfNotExists(x.Provider);
-					return x;
-				}))).ToList();
 			}
 		}
 

Kyoo/Controllers/Repositories/ShowRepository.cs

@@ -5,23 +5,28 @@ using System.Threading.Tasks;
 using Kyoo.Models;
 using Kyoo.Models.Exceptions;
 using Microsoft.EntityFrameworkCore;
-using Microsoft.Extensions.DependencyInjection;
 
 namespace Kyoo.Controllers
 {
 	public class ShowRepository : IShowRepository
 	{
 		private readonly DatabaseContext _database;
-		private readonly IServiceProvider _serviceProvider;
 		private readonly IStudioRepository _studios;
+		private readonly IPeopleRepository _people;
+		private readonly IGenreRepository _genres;
+		private readonly IProviderRepository _providers;
 
 		public ShowRepository(DatabaseContext database,
-			IServiceProvider serviceProvider, 
+			IStudioRepository studios,
-			IStudioRepository studios)
+			IPeopleRepository people, 
+			IGenreRepository genres, 
+			IProviderRepository providers)
 		{
 			_database = database;
-			_serviceProvider = serviceProvider;
 			_studios = studios;
+			_people = people;
+			_genres = genres;
+			_providers = providers;
 		}
 		
 		public void Dispose()
@@ -87,6 +92,7 @@ namespace Kyoo.Controllers
 			}
 			catch (DbUpdateException ex)
 			{
+				_database.DiscardChanges();
 				if (Helper.IsDuplicateException(ex))
 					throw new DuplicatedItemException($"Trying to insert a duplicated show (slug {obj.Slug} already exists).");
 				throw;
@@ -140,38 +146,20 @@ namespace Kyoo.Controllers
 			
 			if (obj.GenreLinks != null)
 			{
-				obj.GenreLinks = (await Task.WhenAll(obj.GenreLinks.Select(async x =>
+				foreach (GenreLink link in obj.GenreLinks)
-				{
+					link.GenreID = await _genres.CreateIfNotExists(link.Genre);
-					using IServiceScope serviceScope = _serviceProvider.CreateScope();
-					await using IGenreRepository genres = serviceScope.ServiceProvider.GetService<IGenreRepository>();
-					
-					x.GenreID = await genres.CreateIfNotExists(x.Genre);
-					return x;
-				}))).ToList();
 			}
 
 			if (obj.People != null)
 			{
-				obj.People = (await Task.WhenAll(obj.People.Select(async x =>
+				foreach (PeopleLink link in obj.People)
-				{
+					link.PeopleID = await _people.CreateIfNotExists(link.People);
-					using IServiceScope serviceScope = _serviceProvider.CreateScope();
-					await using IPeopleRepository people = serviceScope.ServiceProvider.GetService<IPeopleRepository>();
-					
-					x.PeopleID = await people.CreateIfNotExists(x.People);
-					return x;
-				}))).ToList();
 			}
 
 			if (obj.ExternalIDs != null)
 			{
-				obj.ExternalIDs = (await Task.WhenAll(obj.ExternalIDs.Select(async x =>
+				foreach (MetadataID link in obj.ExternalIDs)
-				{
+					link.ProviderID = await _providers.CreateIfNotExists(link.Provider);
-					using IServiceScope serviceScope = _serviceProvider.CreateScope();
-					await using IProviderRepository providers = serviceScope.ServiceProvider.GetService<IProviderRepository>();
-					
-					x.ProviderID = await providers.CreateIfNotExists(x.Provider);
-					return x;
-				}))).ToList();
 			}
 		}
 

Kyoo/Controllers/Repositories/StudioRepository.cs

@@ -64,6 +64,7 @@ namespace Kyoo.Controllers
 			}
 			catch (DbUpdateException ex)
 			{
+				_database.DiscardChanges();
 				if (Helper.IsDuplicateException(ex))
 					throw new DuplicatedItemException($"Trying to insert a duplicated studio (slug {obj.Slug} already exists).");
 				throw;

Kyoo/Controllers/Repositories/TrackRepository.cs

@@ -70,6 +70,7 @@ namespace Kyoo.Controllers
 			}
 			catch (DbUpdateException ex)
 			{
+				_database.DiscardChanges();
 				if (Helper.IsDuplicateException(ex))
 					throw new DuplicatedItemException($"Trying to insert a duplicated track (slug {obj.Slug} already exists).");
 				throw;

Kyoo/Models/DatabaseContext.cs

@@ -146,6 +146,15 @@ namespace Kyoo
 				.HasIndex(x => x.Name)
 				.IsUnique();
 		}
+			
+		public void DiscardChanges()
+		{
+			foreach (EntityEntry entry in ChangeTracker.Entries().Where(x => x.State != EntityState.Unchanged
+			                                                                 && x.State != EntityState.Detached))
+			{
+				entry.State = EntityState.Detached;
+			}
+		}
 	}
 }
 

Kyoo/Startup.cs

@@ -49,7 +49,7 @@ namespace Kyoo
 					.UseNpgsql(_configuration.GetConnectionString("Database"))
 					.EnableSensitiveDataLogging();
 				// .UseLoggerFactory(LoggerFactory.Create(builder => builder.AddConsole()));
-			});
+			}, ServiceLifetime.Transient);
 			
 			services.AddDbContext<IdentityDatabase>(options =>
 			{

Kyoo/Tasks/Crawler.cs

@@ -28,8 +28,6 @@ namespace Kyoo.Controllers
 		private ITranscoder _transcoder;
 		private IConfiguration _config;
 		
-		private int _parallelRegisters;
-		
 		public async Task<IEnumerable<string>> GetPossibleParameters()
 		{
 			using IServiceScope serviceScope = _serviceProvider.CreateScope();
@@ -52,10 +50,6 @@ namespace Kyoo.Controllers
 			_metadataProvider = serviceProvider.GetService<IProviderManager>();
 			_transcoder = serviceProvider.GetService<ITranscoder>();
 			_config = serviceProvider.GetService<IConfiguration>();
-			_parallelRegisters = _config.GetValue<int>("parallelRegisters");
-
-			if (_parallelRegisters <= 0)
-				_parallelRegisters = 10;
 
 			using IServiceScope serviceScope = _serviceProvider.CreateScope();
 			await using ILibraryManager libraryManager = serviceScope.ServiceProvider.GetService<ILibraryManager>();
@@ -74,17 +68,18 @@ namespace Kyoo.Controllers
 			foreach (Library library in libraries)
 				library.Providers = library.Providers;
 			
-			await Task.WhenAll(libraries.Select(x => Scan(x, episodes, cancellationToken)).ToArray());
+			foreach (Library library in libraries)
+				await Scan(library, episodes, cancellationToken);
 			Console.WriteLine("Scan finished!");
 		}
 
-		private Task Scan(Library library, IEnumerable<Episode> episodes, CancellationToken cancellationToken)
+		private async Task Scan(Library library, IEnumerable<Episode> episodes, CancellationToken cancellationToken)
 		{
 			Console.WriteLine($"Scanning library {library.Name} at {string.Join(", ", library.Paths)}.");
-			return Task.WhenAll(library.Paths.Select(async path =>
+			foreach (string path in library.Paths)
 			{
 				if (cancellationToken.IsCancellationRequested)
-					return;
+					continue;
 				
 				string[] files;
 				try
@@ -94,22 +89,22 @@ namespace Kyoo.Controllers
 				catch (DirectoryNotFoundException)
 				{
 					await Console.Error.WriteLineAsync($"The library's directory {path} could not be found (library slug: {library.Slug})");
-					return;
+					continue;
 				}
 				catch (PathTooLongException)
 				{
 					await Console.Error.WriteLineAsync($"The library's directory {path} is too long for this system. (library slug: {library.Slug})");
-					return;
+					continue;
 				}
 				catch (ArgumentException)
 				{
 					await Console.Error.WriteLineAsync($"The library's directory {path} is invalid. (library slug: {library.Slug})");
-					return;
+					continue;
 				}
-				catch (UnauthorizedAccessException)
+				catch (UnauthorizedAccessException ex)
 				{
-					await Console.Error.WriteLineAsync($"Permission denied: can't access library's directory at {path}. (library slug: {library.Slug})");
+					await Console.Error.WriteLineAsync($"{ex.Message} (library slug: {library.Slug})");
-					return;
+					continue;
 				}
 
 				List<IGrouping<string, string>> shows =  files
@@ -117,20 +112,23 @@ namespace Kyoo.Controllers
 					.GroupBy(Path.GetDirectoryName)
 					.ToList();
 				
+				// Todo batch wth higher numbers per list once multi-services has been implemented.
+				
 				List<Task> tasks = shows
 					.Select(x => x.First())
 					.Select(x => RegisterFile(x, x.Substring(path.Length), library, cancellationToken))
 					.ToList();
-				foreach (List<Task> showTasks in tasks.BatchBy(_parallelRegisters))
+					// TODO EXECUTION OF THE TASKS ARE CALCULATED DURING THE .ToList() CALL, NO BACHING/ASYNC IS DONE.
+				foreach (List<Task> showTasks in tasks.BatchBy(1))
 					await Task.WhenAll(showTasks);
 				
 				tasks = shows
 					.SelectMany(x => x.Skip(1))
 					.Select(x => RegisterFile(x, x.Substring(path.Length), library, cancellationToken))
 					.ToList();
-				foreach (List<Task> episodeTasks in tasks.BatchBy(_parallelRegisters * 3))
+				foreach (List<Task> episodeTasks in tasks.BatchBy(1))
 					await Task.WhenAll(episodeTasks);
-			}).ToArray());
+			}
 		}
 
 		private async Task RegisterFile(string path, string relativePath, Library library, CancellationToken token)

Kyoo/appsettings.json

@@ -15,9 +15,8 @@
   "AllowedHosts": "*",
   
   "ConnectionStrings": {
-    "Database": "Server=127.0.0.1; Port=5432; Database=kyooDB; User Id=kyoo; Password=kyooPassword; Pooling=true; MaxPoolSize=80; Timeout=30;"
+    "Database": "Server=127.0.0.1; Port=5432; Database=kyooDB; User Id=kyoo; Password=kyooPassword; Pooling=true; MaxPoolSize=95; Timeout=30;"
   },
-  "parallelRegisters": "30",
   
   "scheduledTasks": {
     "scan": "24:00:00"