Cutlery/Kyoo
1
0
Fork 0
mirror of https://github.com/zoriya/Kyoo.git synced 2025-07-09 03:04:20 -04:00
Code Issues Packages Projects Releases Wiki Activity

Cleaning up repositories

Browse Source
This commit is contained in:
Zoe Roux 2020-06-18 03:23:23 +02:00
parent 866635fe1f
commit 8e37505b8f
14 changed files with 78 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Kyoo/Controllers/Repositories/CollectionRepository.cs
View File

@ -5,7 +5,6 @@ using System.Threading.Tasks;
using Kyoo.Models;
using Kyoo.Models.Exceptions;
using Microsoft.EntityFrameworkCore;
using Npgsql;
namespace Kyoo.Controllers
{
@ -65,6 +64,7 @@ namespace Kyoo.Controllers
}
catch (DbUpdateException ex)
{
_database.DiscardChanges();
if (Helper.IsDuplicateException(ex))
throw new DuplicatedItemException($"Trying to insert a duplicated collection (slug {obj.Slug} already exists).");
throw;

20
Kyoo/Controllers/Repositories/EpisodeRepository.cs
View File

@ -5,21 +5,19 @@ using System.Threading.Tasks;
using Kyoo.Models;
using Kyoo.Models.Exceptions;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Npgsql;
namespace Kyoo.Controllers
{
public class EpisodeRepository : IEpisodeRepository
{
private readonly DatabaseContext _database;
private readonly IServiceProvider _serviceProvider;
private readonly IProviderRepository _providers;
public EpisodeRepository(DatabaseContext database, IServiceProvider serviceProvider)
public EpisodeRepository(DatabaseContext database, IProviderRepository providers)
{
_database = database;
_serviceProvider = serviceProvider;
_providers = providers;
}
public void Dispose()
@ -95,6 +93,8 @@ namespace Kyoo.Controllers
}
catch (DbUpdateException ex)
{
_database.DiscardChanges();
if (Helper.IsDuplicateException(ex))
throw new DuplicatedItemException($"Trying to insert a duplicated episode (slug {obj.Slug} already exists).");
throw;
@ -148,14 +148,8 @@ namespace Kyoo.Controllers
if (obj.ExternalIDs != null)
{
obj.ExternalIDs = (await Task.WhenAll(obj.ExternalIDs.Select(async x =>
{
using IServiceScope serviceScope = _serviceProvider.CreateScope();
await using IProviderRepository providers = serviceScope.ServiceProvider.GetService<IProviderRepository>();
x.ProviderID = await providers.CreateIfNotExists(x.Provider);
return x;
}))).ToList();
foreach (MetadataID link in obj.ExternalIDs)
link.ProviderID = await _providers.CreateIfNotExists(link.Provider);
}
}

2
Kyoo/Controllers/Repositories/GenreRepository.cs
View File

@ -64,6 +64,8 @@ namespace Kyoo.Controllers
}
catch (DbUpdateException ex)
{
_database.DiscardChanges();
if (Helper.IsDuplicateException(ex))
throw new DuplicatedItemException($"Trying to insert a duplicated genre (slug {obj.Slug} already exists).");
throw;

19
Kyoo/Controllers/Repositories/LibraryRepository.cs
View File

@ -5,20 +5,19 @@ using System.Threading.Tasks;
using Kyoo.Models;
using Kyoo.Models.Exceptions;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
namespace Kyoo.Controllers
{
public class LibraryRepository : ILibraryRepository
{
private readonly DatabaseContext _database;
private readonly IServiceProvider _serviceProvider;
private readonly IProviderRepository _providers;
public LibraryRepository(DatabaseContext database, IServiceProvider serviceProvider)
public LibraryRepository(DatabaseContext database, IProviderRepository providers)
{
_database = database;
_serviceProvider = serviceProvider;
_providers = providers;
}
public void Dispose()
@ -71,6 +70,7 @@ namespace Kyoo.Controllers
}
catch (DbUpdateException ex)
{
_database.DiscardChanges();
if (Helper.IsDuplicateException(ex))
throw new DuplicatedItemException($"Trying to insert a duplicated library (slug {obj.Slug} already exists).");
throw;
@ -119,14 +119,9 @@ namespace Kyoo.Controllers
private async Task Validate(Library obj)
{
obj.ProviderLinks = (await Task.WhenAll(obj.ProviderLinks.Select(async x =>
{
using IServiceScope serviceScope = _serviceProvider.CreateScope();
await using IProviderRepository providers = serviceScope.ServiceProvider.GetService<IProviderRepository>();
x.ProviderID = await providers.CreateIfNotExists(x.Provider);
return x;
}))).ToList();
if (obj.ProviderLinks != null)
foreach (ProviderLink link in obj.ProviderLinks)
link.ProviderID = await _providers.CreateIfNotExists(link.Provider);
}
public async Task Delete(Library obj)

19
Kyoo/Controllers/Repositories/PeopleRepository.cs
View File

@ -5,19 +5,18 @@ using System.Threading.Tasks;
using Kyoo.Models;
using Kyoo.Models.Exceptions;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
namespace Kyoo.Controllers
{
public class PeopleRepository : IPeopleRepository
{
private readonly DatabaseContext _database;
private readonly IServiceProvider _serviceProvider;
private readonly IProviderRepository _providers;
public PeopleRepository(DatabaseContext database, IServiceProvider serviceProvider)
public PeopleRepository(DatabaseContext database, IProviderRepository providers)
{
_database = database;
_serviceProvider = serviceProvider;
_providers = providers;
}
public void Dispose()
@ -70,6 +69,7 @@ namespace Kyoo.Controllers
}
catch (DbUpdateException ex)
{
_database.DiscardChanges();
if (Helper.IsDuplicateException(ex))
throw new DuplicatedItemException($"Trying to insert a duplicated people (slug {obj.Slug} already exists).");
throw;
@ -118,14 +118,9 @@ namespace Kyoo.Controllers
private async Task Validate(People obj)
{
obj.ExternalIDs = (await Task.WhenAll(obj.ExternalIDs.Select(async x =>
{
using IServiceScope serviceScope = _serviceProvider.CreateScope();
await using IProviderRepository providers = serviceScope.ServiceProvider.GetService<IProviderRepository>();
x.ProviderID = await providers.CreateIfNotExists(x.Provider);
return x;
}))).ToList();
if (obj.ExternalIDs != null)
foreach (MetadataID link in obj.ExternalIDs)
link.ProviderID = await _providers.CreateIfNotExists(link.Provider);
}
public async Task Delete(People obj)

1
Kyoo/Controllers/Repositories/ProviderRepository.cs
View File

@ -64,6 +64,7 @@ namespace Kyoo.Controllers
}
catch (DbUpdateException ex)
{
_database.DiscardChanges();
if (Helper.IsDuplicateException(ex))
throw new DuplicatedItemException($"Trying to insert a duplicated provider (name {obj.Name} already exists).");
throw;

18
Kyoo/Controllers/Repositories/SeasonRepository.cs
View File

@ -5,20 +5,19 @@ using System.Threading.Tasks;
using Kyoo.Models;
using Kyoo.Models.Exceptions;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
namespace Kyoo.Controllers
{
public class SeasonRepository : ISeasonRepository
{
private readonly DatabaseContext _database;
private readonly IServiceProvider _serviceProvider;
private readonly IProviderRepository _providers;
public SeasonRepository(DatabaseContext database, IServiceProvider serviceProvider)
public SeasonRepository(DatabaseContext database, IProviderRepository providers)
{
_database = database;
_serviceProvider = serviceProvider;
_providers = providers;
}
public void Dispose()
@ -83,6 +82,7 @@ namespace Kyoo.Controllers
}
catch (DbUpdateException ex)
{
_database.DiscardChanges();
if (Helper.IsDuplicateException(ex))
throw new DuplicatedItemException($"Trying to insert a duplicated season (slug {obj.Slug} already exists).");
throw;
@ -137,14 +137,8 @@ namespace Kyoo.Controllers
if (obj.ExternalIDs != null)
{
obj.ExternalIDs = (await Task.WhenAll(obj.ExternalIDs.Select(async x =>
{
using IServiceScope serviceScope = _serviceProvider.CreateScope();
await using IProviderRepository providers = serviceScope.ServiceProvider.GetService<IProviderRepository>();
x.ProviderID = await providers.CreateIfNotExists(x.Provider);
return x;
}))).ToList();
foreach (MetadataID link in obj.ExternalIDs)
link.ProviderID = await _providers.CreateIfNotExists(link.Provider);
}
}

46
Kyoo/Controllers/Repositories/ShowRepository.cs
View File

@ -5,23 +5,28 @@ using System.Threading.Tasks;
using Kyoo.Models;
using Kyoo.Models.Exceptions;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
namespace Kyoo.Controllers
{
public class ShowRepository : IShowRepository
{
private readonly DatabaseContext _database;
private readonly IServiceProvider _serviceProvider;
private readonly IStudioRepository _studios;
private readonly IPeopleRepository _people;
private readonly IGenreRepository _genres;
private readonly IProviderRepository _providers;
public ShowRepository(DatabaseContext database,
IServiceProvider serviceProvider,
IStudioRepository studios)
IStudioRepository studios,
IPeopleRepository people,
IGenreRepository genres,
IProviderRepository providers)
{
_database = database;
_serviceProvider = serviceProvider;
_studios = studios;
_people = people;
_genres = genres;
_providers = providers;
}
public void Dispose()
@ -87,6 +92,7 @@ namespace Kyoo.Controllers
}
catch (DbUpdateException ex)
{
_database.DiscardChanges();
if (Helper.IsDuplicateException(ex))
throw new DuplicatedItemException($"Trying to insert a duplicated show (slug {obj.Slug} already exists).");
throw;
@ -140,38 +146,20 @@ namespace Kyoo.Controllers
if (obj.GenreLinks != null)
{
obj.GenreLinks = (await Task.WhenAll(obj.GenreLinks.Select(async x =>
{
using IServiceScope serviceScope = _serviceProvider.CreateScope();
await using IGenreRepository genres = serviceScope.ServiceProvider.GetService<IGenreRepository>();
x.GenreID = await genres.CreateIfNotExists(x.Genre);
return x;
}))).ToList();
foreach (GenreLink link in obj.GenreLinks)
link.GenreID = await _genres.CreateIfNotExists(link.Genre);
}
if (obj.People != null)
{
obj.People = (await Task.WhenAll(obj.People.Select(async x =>
{
using IServiceScope serviceScope = _serviceProvider.CreateScope();
await using IPeopleRepository people = serviceScope.ServiceProvider.GetService<IPeopleRepository>();
x.PeopleID = await people.CreateIfNotExists(x.People);
return x;
}))).ToList();
foreach (PeopleLink link in obj.People)
link.PeopleID = await _people.CreateIfNotExists(link.People);
}
if (obj.ExternalIDs != null)
{
obj.ExternalIDs = (await Task.WhenAll(obj.ExternalIDs.Select(async x =>
{
using IServiceScope serviceScope = _serviceProvider.CreateScope();
await using IProviderRepository providers = serviceScope.ServiceProvider.GetService<IProviderRepository>();
x.ProviderID = await providers.CreateIfNotExists(x.Provider);
return x;
}))).ToList();
foreach (MetadataID link in obj.ExternalIDs)
link.ProviderID = await _providers.CreateIfNotExists(link.Provider);
}
}

1
Kyoo/Controllers/Repositories/StudioRepository.cs
View File

@ -64,6 +64,7 @@ namespace Kyoo.Controllers
}
catch (DbUpdateException ex)
{
_database.DiscardChanges();
if (Helper.IsDuplicateException(ex))
throw new DuplicatedItemException($"Trying to insert a duplicated studio (slug {obj.Slug} already exists).");
throw;

1
Kyoo/Controllers/Repositories/TrackRepository.cs
View File

@ -70,6 +70,7 @@ namespace Kyoo.Controllers
}
catch (DbUpdateException ex)
{
_database.DiscardChanges();
if (Helper.IsDuplicateException(ex))
throw new DuplicatedItemException($"Trying to insert a duplicated track (slug {obj.Slug} already exists).");
throw;

9
Kyoo/Models/DatabaseContext.cs
View File

@ -146,6 +146,15 @@ namespace Kyoo
.HasIndex(x => x.Name)
.IsUnique();
}
public void DiscardChanges()
{
foreach (EntityEntry entry in ChangeTracker.Entries().Where(x => x.State != EntityState.Unchanged
&& x.State != EntityState.Detached))
{
entry.State = EntityState.Detached;
}
}
}
}

2
Kyoo/Startup.cs
View File

@ -49,7 +49,7 @@ namespace Kyoo
.UseNpgsql(_configuration.GetConnectionString("Database"))
.EnableSensitiveDataLogging();
// .UseLoggerFactory(LoggerFactory.Create(builder => builder.AddConsole()));
});
}, ServiceLifetime.Transient);
services.AddDbContext<IdentityDatabase>(options =>
{

36
Kyoo/Tasks/Crawler.cs
View File

@ -28,8 +28,6 @@ namespace Kyoo.Controllers
private ITranscoder _transcoder;
private IConfiguration _config;
private int _parallelRegisters;
public async Task<IEnumerable<string>> GetPossibleParameters()
{
using IServiceScope serviceScope = _serviceProvider.CreateScope();
@ -52,10 +50,6 @@ namespace Kyoo.Controllers
_metadataProvider = serviceProvider.GetService<IProviderManager>();
_transcoder = serviceProvider.GetService<ITranscoder>();
_config = serviceProvider.GetService<IConfiguration>();
_parallelRegisters = _config.GetValue<int>("parallelRegisters");
if (_parallelRegisters <= 0)
_parallelRegisters = 10;
using IServiceScope serviceScope = _serviceProvider.CreateScope();
await using ILibraryManager libraryManager = serviceScope.ServiceProvider.GetService<ILibraryManager>();
@ -74,17 +68,18 @@ namespace Kyoo.Controllers
foreach (Library library in libraries)
library.Providers = library.Providers;
await Task.WhenAll(libraries.Select(x => Scan(x, episodes, cancellationToken)).ToArray());
foreach (Library library in libraries)
await Scan(library, episodes, cancellationToken);
Console.WriteLine("Scan finished!");
}
private Task Scan(Library library, IEnumerable<Episode> episodes, CancellationToken cancellationToken)
private async Task Scan(Library library, IEnumerable<Episode> episodes, CancellationToken cancellationToken)
{
Console.WriteLine($"Scanning library {library.Name} at {string.Join(", ", library.Paths)}.");
return Task.WhenAll(library.Paths.Select(async path =>
foreach (string path in library.Paths)
{
if (cancellationToken.IsCancellationRequested)
return;
continue;
string[] files;
try
@ -94,22 +89,22 @@ namespace Kyoo.Controllers
catch (DirectoryNotFoundException)
{
await Console.Error.WriteLineAsync($"The library's directory {path} could not be found (library slug: {library.Slug})");
return;
continue;
}
catch (PathTooLongException)
{
await Console.Error.WriteLineAsync($"The library's directory {path} is too long for this system. (library slug: {library.Slug})");
return;
continue;
}
catch (ArgumentException)
{
await Console.Error.WriteLineAsync($"The library's directory {path} is invalid. (library slug: {library.Slug})");
return;
continue;
}
catch (UnauthorizedAccessException)
catch (UnauthorizedAccessException ex)
{
await Console.Error.WriteLineAsync($"Permission denied: can't access library's directory at {path}. (library slug: {library.Slug})");
return;
await Console.Error.WriteLineAsync($"{ex.Message} (library slug: {library.Slug})");
continue;
}
List<IGrouping<string, string>> shows = files
@ -117,20 +112,23 @@ namespace Kyoo.Controllers
.GroupBy(Path.GetDirectoryName)
.ToList();
// Todo batch wth higher numbers per list once multi-services has been implemented.
List<Task> tasks = shows
.Select(x => x.First())
.Select(x => RegisterFile(x, x.Substring(path.Length), library, cancellationToken))
.ToList();
foreach (List<Task> showTasks in tasks.BatchBy(_parallelRegisters))
// TODO EXECUTION OF THE TASKS ARE CALCULATED DURING THE .ToList() CALL, NO BACHING/ASYNC IS DONE.
foreach (List<Task> showTasks in tasks.BatchBy(1))
await Task.WhenAll(showTasks);
tasks = shows
.SelectMany(x => x.Skip(1))
.Select(x => RegisterFile(x, x.Substring(path.Length), library, cancellationToken))
.ToList();
foreach (List<Task> episodeTasks in tasks.BatchBy(_parallelRegisters * 3))
foreach (List<Task> episodeTasks in tasks.BatchBy(1))
await Task.WhenAll(episodeTasks);
}).ToArray());
}
}
private async Task RegisterFile(string path, string relativePath, Library library, CancellationToken token)

3
Kyoo/appsettings.json
View File

@ -15,9 +15,8 @@
"AllowedHosts": "*",
"ConnectionStrings": {
"Database": "Server=127.0.0.1; Port=5432; Database=kyooDB; User Id=kyoo; Password=kyooPassword; Pooling=true; MaxPoolSize=80; Timeout=30;"
"Database": "Server=127.0.0.1; Port=5432; Database=kyooDB; User Id=kyoo; Password=kyooPassword; Pooling=true; MaxPoolSize=95; Timeout=30;"
},
"parallelRegisters": "30",
"scheduledTasks": {
"scan": "24:00:00"