From d34a6fd75a0aef2875fd5cd820aa876e573cb5b1 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Thu, 11 Jun 2020 16:46:17 +0200 Subject: [PATCH 1/8] Starting to implement a dispose pattern for the library manager/repositories --- Kyoo.Common/Controllers/ILibraryManager.cs | 5 ++-- Kyoo.Common/Controllers/IRepository.cs | 3 +- .../Exceptions/DuplicatedItemException.cs | 14 +++++++++ Kyoo/Controllers/LibraryManager.cs | 22 ++++++++++++++ .../Repositories/CollectionRepository.cs | 29 ++++++++++++++++--- Kyoo/Tasks/Crawler.cs | 18 +++++++----- 6 files changed, 77 insertions(+), 14 deletions(-) create mode 100644 Kyoo.Common/Models/Exceptions/DuplicatedItemException.cs diff --git a/Kyoo.Common/Controllers/ILibraryManager.cs b/Kyoo.Common/Controllers/ILibraryManager.cs index c345bf38..72c211ae 100644 --- a/Kyoo.Common/Controllers/ILibraryManager.cs +++ b/Kyoo.Common/Controllers/ILibraryManager.cs @@ -1,11 +1,12 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Threading.Tasks; using JetBrains.Annotations; using Kyoo.Models; namespace Kyoo.Controllers { - public interface ILibraryManager + public interface ILibraryManager : IDisposable, IAsyncDisposable { // Get by slug Task GetLibrary(string slug); diff --git a/Kyoo.Common/Controllers/IRepository.cs b/Kyoo.Common/Controllers/IRepository.cs index 0993ca69..e1587908 100644 --- a/Kyoo.Common/Controllers/IRepository.cs +++ b/Kyoo.Common/Controllers/IRepository.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.Threading.Tasks; using JetBrains.Annotations; @@ -5,7 +6,7 @@ using Kyoo.Models; namespace Kyoo.Controllers { - public interface IRepository + public interface IRepository : IDisposable, IAsyncDisposable { Task Get(int id); Task Get(string slug); diff --git a/Kyoo.Common/Models/Exceptions/DuplicatedItemException.cs b/Kyoo.Common/Models/Exceptions/DuplicatedItemException.cs new file mode 100644 index 00000000..c8836c0d --- /dev/null +++ b/Kyoo.Common/Models/Exceptions/DuplicatedItemException.cs @@ -0,0 +1,14 @@ +using System; + +namespace Kyoo.Models.Exceptions +{ + public class DuplicatedItemException : Exception + { + public override string Message { get; } + + public DuplicatedItemException(string message) + { + Message = message; + } + } +} \ No newline at end of file diff --git a/Kyoo/Controllers/LibraryManager.cs b/Kyoo/Controllers/LibraryManager.cs index c4f1432a..726675ce 100644 --- a/Kyoo/Controllers/LibraryManager.cs +++ b/Kyoo/Controllers/LibraryManager.cs @@ -40,6 +40,28 @@ namespace Kyoo.Controllers _providers = providers; _people = people; } + + public void Dispose() + { + _libraries?.Dispose(); + _collections?.Dispose(); + _shows?.Dispose(); + _seasons?.Dispose(); + _episodes?.Dispose(); + _tracks?.Dispose(); + _genres?.Dispose(); + _studios?.Dispose(); + _people?.Dispose(); + _providers?.Dispose(); + } + + public async ValueTask DisposeAsync() + { + return ValueTask.(new [] + { + _libraries.DisposeAsync() + }); + } public Task GetLibrary(string slug) { diff --git a/Kyoo/Controllers/Repositories/CollectionRepository.cs b/Kyoo/Controllers/Repositories/CollectionRepository.cs index c09bbc94..69f059ce 100644 --- a/Kyoo/Controllers/Repositories/CollectionRepository.cs +++ b/Kyoo/Controllers/Repositories/CollectionRepository.cs @@ -5,6 +5,7 @@ using System.Threading.Tasks; using Kyoo.Models; using Kyoo.Models.Exceptions; using Microsoft.EntityFrameworkCore; +using Npgsql; namespace Kyoo.Controllers { @@ -45,9 +46,19 @@ namespace Kyoo.Controllers { if (obj == null) throw new ArgumentNullException(nameof(obj)); - - await _database.Collections.AddAsync(obj); - await _database.SaveChangesAsync(); + + try + { + await _database.Collections.AddAsync(obj); + await _database.SaveChangesAsync(); + } + catch (DbUpdateException ex) + { + if (ex.InnerException is PostgresException inner && inner.SqlState == PostgresErrorCodes.UniqueViolation) + throw new DuplicatedItemException($"Trying to insert a duplicated collection (slug {obj.Slug} already exists)."); + throw; + } + return obj.ID; } @@ -59,7 +70,17 @@ namespace Kyoo.Controllers Collection old = await Get(obj.Slug); if (old != null) return old.ID; - return await Create(obj); + try + { + return await Create(obj); + } + catch (DuplicatedItemException) + { + old = await Get(obj.Slug); + if (old == null) + throw new SystemException("Unknown database state."); + return old.ID; + } } public async Task Edit(Collection edited, bool resetOld) diff --git a/Kyoo/Tasks/Crawler.cs b/Kyoo/Tasks/Crawler.cs index 3a5c8ae9..a0388769 100644 --- a/Kyoo/Tasks/Crawler.cs +++ b/Kyoo/Tasks/Crawler.cs @@ -63,6 +63,10 @@ namespace Kyoo.Controllers await libraryManager.DeleteEpisode(episode); } + // TODO replace this grotesque way to load the providers. + foreach (Library library in libraries) + library.Providers = library.Providers; + await Task.WhenAll(libraries.Select(x => Scan(x, episodes, cancellationToken))); } catch (Exception ex) @@ -72,7 +76,7 @@ namespace Kyoo.Controllers Console.WriteLine("Scan finished!"); } - private Task Scan(Library library, ICollection episodes, CancellationToken cancellationToken) + private Task Scan(Library library, IEnumerable episodes, CancellationToken cancellationToken) { Console.WriteLine($"Scanning library {library.Name} at {string.Join(", ", library.Paths)}."); return Task.WhenAll(library.Paths.Select(async path => @@ -106,16 +110,16 @@ namespace Kyoo.Controllers return Task.CompletedTask; } - // return Task.WhenAll(files.Select(file => - foreach (string file in files) + return Task.WhenAll(files.Select(file => + //foreach (string file in files) { if (!IsVideo(file) || episodes.Any(x => x.Path == file)) - continue; //return Task.CompletedTask; + /*continue;*/ return Task.CompletedTask; string relativePath = file.Substring(path.Length); - /*return*/ await RegisterFile(file, relativePath, library, cancellationToken); - }//)); + return /*await*/ RegisterFile(file, relativePath, library, cancellationToken); + })); - return Task.CompletedTask; + // return Task.CompletedTask; })); } From 7b266a2dff984a0b82a43844417bae69754a9b8f Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Thu, 11 Jun 2020 16:56:34 +0200 Subject: [PATCH 2/8] Implementing dispose/asyncdisposes --- Kyoo/Controllers/LibraryManager.cs | 36 +++++++++++-------- .../Repositories/CollectionRepository.cs | 10 ++++++ .../Repositories/EpisodeRepository.cs | 10 ++++++ .../Repositories/GenreRepository.cs | 10 ++++++ .../Repositories/LibraryRepository.cs | 12 ++++++- .../Repositories/PeopleRepository.cs | 10 ++++++ .../Repositories/ProviderRepository.cs | 10 ++++++ .../Repositories/SeasonRepository.cs | 10 ++++++ .../Repositories/ShowRepository.cs | 11 ++++++ .../Repositories/StudioRepository.cs | 10 ++++++ .../Repositories/TrackRepository.cs | 10 ++++++ 11 files changed, 124 insertions(+), 15 deletions(-) diff --git a/Kyoo/Controllers/LibraryManager.cs b/Kyoo/Controllers/LibraryManager.cs index 726675ce..9b1a6590 100644 --- a/Kyoo/Controllers/LibraryManager.cs +++ b/Kyoo/Controllers/LibraryManager.cs @@ -43,24 +43,32 @@ namespace Kyoo.Controllers public void Dispose() { - _libraries?.Dispose(); - _collections?.Dispose(); - _shows?.Dispose(); - _seasons?.Dispose(); - _episodes?.Dispose(); - _tracks?.Dispose(); - _genres?.Dispose(); - _studios?.Dispose(); - _people?.Dispose(); - _providers?.Dispose(); + _libraries.Dispose(); + _collections.Dispose(); + _shows.Dispose(); + _seasons.Dispose(); + _episodes.Dispose(); + _tracks.Dispose(); + _genres.Dispose(); + _studios.Dispose(); + _people.Dispose(); + _providers.Dispose(); } public async ValueTask DisposeAsync() { - return ValueTask.(new [] - { - _libraries.DisposeAsync() - }); + await Task.WhenAll( + _libraries.DisposeAsync().AsTask(), + _collections.DisposeAsync().AsTask(), + _shows.DisposeAsync().AsTask(), + _seasons.DisposeAsync().AsTask(), + _episodes.DisposeAsync().AsTask(), + _tracks.DisposeAsync().AsTask(), + _genres.DisposeAsync().AsTask(), + _studios.DisposeAsync().AsTask(), + _people.DisposeAsync().AsTask(), + _providers.DisposeAsync().AsTask() + ); } public Task GetLibrary(string slug) diff --git a/Kyoo/Controllers/Repositories/CollectionRepository.cs b/Kyoo/Controllers/Repositories/CollectionRepository.cs index 69f059ce..091c2143 100644 --- a/Kyoo/Controllers/Repositories/CollectionRepository.cs +++ b/Kyoo/Controllers/Repositories/CollectionRepository.cs @@ -19,6 +19,16 @@ namespace Kyoo.Controllers _database = database; } + public void Dispose() + { + _database.Dispose(); + } + + public ValueTask DisposeAsync() + { + return _database.DisposeAsync(); + } + public Task Get(int id) { return _database.Collections.FirstOrDefaultAsync(x => x.ID == id); diff --git a/Kyoo/Controllers/Repositories/EpisodeRepository.cs b/Kyoo/Controllers/Repositories/EpisodeRepository.cs index 94cc2889..a6c9b61c 100644 --- a/Kyoo/Controllers/Repositories/EpisodeRepository.cs +++ b/Kyoo/Controllers/Repositories/EpisodeRepository.cs @@ -21,6 +21,16 @@ namespace Kyoo.Controllers _serviceProvider = serviceProvider; } + public void Dispose() + { + _database.Dispose(); + } + + public ValueTask DisposeAsync() + { + return _database.DisposeAsync(); + } + public async Task Get(int id) { return await _database.Episodes.FirstOrDefaultAsync(x => x.ID == id); diff --git a/Kyoo/Controllers/Repositories/GenreRepository.cs b/Kyoo/Controllers/Repositories/GenreRepository.cs index 11931c10..d9299465 100644 --- a/Kyoo/Controllers/Repositories/GenreRepository.cs +++ b/Kyoo/Controllers/Repositories/GenreRepository.cs @@ -17,6 +17,16 @@ namespace Kyoo.Controllers { _database = database; } + + public void Dispose() + { + _database.Dispose(); + } + + public ValueTask DisposeAsync() + { + return _database.DisposeAsync(); + } public async Task Get(int id) { diff --git a/Kyoo/Controllers/Repositories/LibraryRepository.cs b/Kyoo/Controllers/Repositories/LibraryRepository.cs index 4df5a1d0..27715c00 100644 --- a/Kyoo/Controllers/Repositories/LibraryRepository.cs +++ b/Kyoo/Controllers/Repositories/LibraryRepository.cs @@ -20,7 +20,17 @@ namespace Kyoo.Controllers _database = database; _serviceProvider = serviceProvider; } - + + public void Dispose() + { + _database.Dispose(); + } + + public ValueTask DisposeAsync() + { + return _database.DisposeAsync(); + } + public Task Get(int id) { return _database.Libraries.FirstOrDefaultAsync(x => x.ID == id); diff --git a/Kyoo/Controllers/Repositories/PeopleRepository.cs b/Kyoo/Controllers/Repositories/PeopleRepository.cs index 26bcc825..8bc73b42 100644 --- a/Kyoo/Controllers/Repositories/PeopleRepository.cs +++ b/Kyoo/Controllers/Repositories/PeopleRepository.cs @@ -19,6 +19,16 @@ namespace Kyoo.Controllers _database = database; _serviceProvider = serviceProvider; } + + public void Dispose() + { + _database.Dispose(); + } + + public ValueTask DisposeAsync() + { + return _database.DisposeAsync(); + } public Task Get(int id) { diff --git a/Kyoo/Controllers/Repositories/ProviderRepository.cs b/Kyoo/Controllers/Repositories/ProviderRepository.cs index 4b304c45..1cee1d2a 100644 --- a/Kyoo/Controllers/Repositories/ProviderRepository.cs +++ b/Kyoo/Controllers/Repositories/ProviderRepository.cs @@ -18,6 +18,16 @@ namespace Kyoo.Controllers _database = database; } + public void Dispose() + { + _database.Dispose(); + } + + public ValueTask DisposeAsync() + { + return _database.DisposeAsync(); + } + public async Task Get(int id) { return await _database.Providers.FirstOrDefaultAsync(x => x.ID == id); diff --git a/Kyoo/Controllers/Repositories/SeasonRepository.cs b/Kyoo/Controllers/Repositories/SeasonRepository.cs index 6e514871..73ca3ba4 100644 --- a/Kyoo/Controllers/Repositories/SeasonRepository.cs +++ b/Kyoo/Controllers/Repositories/SeasonRepository.cs @@ -21,6 +21,16 @@ namespace Kyoo.Controllers _serviceProvider = serviceProvider; } + public void Dispose() + { + _database.Dispose(); + } + + public ValueTask DisposeAsync() + { + return _database.DisposeAsync(); + } + public async Task Get(int id) { return await _database.Seasons.FirstOrDefaultAsync(x => x.ID == id); diff --git a/Kyoo/Controllers/Repositories/ShowRepository.cs b/Kyoo/Controllers/Repositories/ShowRepository.cs index 8d5a6772..2d76fcfd 100644 --- a/Kyoo/Controllers/Repositories/ShowRepository.cs +++ b/Kyoo/Controllers/Repositories/ShowRepository.cs @@ -24,6 +24,17 @@ namespace Kyoo.Controllers _studios = studios; } + public void Dispose() + { + _database.Dispose(); + _studios.Dispose(); + } + + public async ValueTask DisposeAsync() + { + await Task.WhenAll(_database.DisposeAsync().AsTask(), _studios.DisposeAsync().AsTask()); + } + public async Task Get(int id) { return await _database.Shows.FirstOrDefaultAsync(x => x.ID == id); diff --git a/Kyoo/Controllers/Repositories/StudioRepository.cs b/Kyoo/Controllers/Repositories/StudioRepository.cs index 900f5b79..d8cb7b1c 100644 --- a/Kyoo/Controllers/Repositories/StudioRepository.cs +++ b/Kyoo/Controllers/Repositories/StudioRepository.cs @@ -18,6 +18,16 @@ namespace Kyoo.Controllers _database = database; } + public void Dispose() + { + _database.Dispose(); + } + + public ValueTask DisposeAsync() + { + return _database.DisposeAsync(); + } + public async Task Get(int id) { return await _database.Studios.FirstOrDefaultAsync(x => x.ID == id); diff --git a/Kyoo/Controllers/Repositories/TrackRepository.cs b/Kyoo/Controllers/Repositories/TrackRepository.cs index 8ef1dd8e..199a452b 100644 --- a/Kyoo/Controllers/Repositories/TrackRepository.cs +++ b/Kyoo/Controllers/Repositories/TrackRepository.cs @@ -17,6 +17,16 @@ namespace Kyoo.Controllers _database = database; } + public void Dispose() + { + _database.Dispose(); + } + + public ValueTask DisposeAsync() + { + return _database.DisposeAsync(); + } + public async Task Get(int id) { return await _database.Tracks.FirstOrDefaultAsync(x => x.ID == id); From 19715f795c04b9a56a0e14f334fa970d3dead5d4 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Thu, 11 Jun 2020 22:53:11 +0200 Subject: [PATCH 3/8] Implementing dispose pattern where repositories where used --- Kyoo/Controllers/Repositories/EpisodeRepository.cs | 2 +- Kyoo/Controllers/Repositories/LibraryRepository.cs | 2 +- Kyoo/Controllers/Repositories/PeopleRepository.cs | 2 +- Kyoo/Controllers/Repositories/SeasonRepository.cs | 2 +- Kyoo/Controllers/Repositories/ShowRepository.cs | 6 +++--- Kyoo/Tasks/Crawler.cs | 4 ++-- Kyoo/appsettings.json | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Kyoo/Controllers/Repositories/EpisodeRepository.cs b/Kyoo/Controllers/Repositories/EpisodeRepository.cs index a6c9b61c..56114ae3 100644 --- a/Kyoo/Controllers/Repositories/EpisodeRepository.cs +++ b/Kyoo/Controllers/Repositories/EpisodeRepository.cs @@ -126,7 +126,7 @@ namespace Kyoo.Controllers obj.ExternalIDs = (await Task.WhenAll(obj.ExternalIDs.Select(async x => { using IServiceScope serviceScope = _serviceProvider.CreateScope(); - IProviderRepository providers = serviceScope.ServiceProvider.GetService(); + await using IProviderRepository providers = serviceScope.ServiceProvider.GetService(); x.ProviderID = await providers.CreateIfNotExists(x.Provider); return x; diff --git a/Kyoo/Controllers/Repositories/LibraryRepository.cs b/Kyoo/Controllers/Repositories/LibraryRepository.cs index 27715c00..4c0f0e17 100644 --- a/Kyoo/Controllers/Repositories/LibraryRepository.cs +++ b/Kyoo/Controllers/Repositories/LibraryRepository.cs @@ -101,7 +101,7 @@ namespace Kyoo.Controllers obj.ProviderLinks = (await Task.WhenAll(obj.ProviderLinks.Select(async x => { using IServiceScope serviceScope = _serviceProvider.CreateScope(); - IProviderRepository providers = serviceScope.ServiceProvider.GetService(); + await using IProviderRepository providers = serviceScope.ServiceProvider.GetService(); x.ProviderID = await providers.CreateIfNotExists(x.Provider); return x; diff --git a/Kyoo/Controllers/Repositories/PeopleRepository.cs b/Kyoo/Controllers/Repositories/PeopleRepository.cs index 8bc73b42..5dd5de09 100644 --- a/Kyoo/Controllers/Repositories/PeopleRepository.cs +++ b/Kyoo/Controllers/Repositories/PeopleRepository.cs @@ -100,7 +100,7 @@ namespace Kyoo.Controllers obj.ExternalIDs = (await Task.WhenAll(obj.ExternalIDs.Select(async x => { using IServiceScope serviceScope = _serviceProvider.CreateScope(); - IProviderRepository providers = serviceScope.ServiceProvider.GetService(); + await using IProviderRepository providers = serviceScope.ServiceProvider.GetService(); x.ProviderID = await providers.CreateIfNotExists(x.Provider); return x; diff --git a/Kyoo/Controllers/Repositories/SeasonRepository.cs b/Kyoo/Controllers/Repositories/SeasonRepository.cs index 73ca3ba4..4751b08d 100644 --- a/Kyoo/Controllers/Repositories/SeasonRepository.cs +++ b/Kyoo/Controllers/Repositories/SeasonRepository.cs @@ -119,7 +119,7 @@ namespace Kyoo.Controllers obj.ExternalIDs = (await Task.WhenAll(obj.ExternalIDs.Select(async x => { using IServiceScope serviceScope = _serviceProvider.CreateScope(); - IProviderRepository providers = serviceScope.ServiceProvider.GetService(); + await using IProviderRepository providers = serviceScope.ServiceProvider.GetService(); x.ProviderID = await providers.CreateIfNotExists(x.Provider); return x; diff --git a/Kyoo/Controllers/Repositories/ShowRepository.cs b/Kyoo/Controllers/Repositories/ShowRepository.cs index 2d76fcfd..bea40425 100644 --- a/Kyoo/Controllers/Repositories/ShowRepository.cs +++ b/Kyoo/Controllers/Repositories/ShowRepository.cs @@ -122,7 +122,7 @@ namespace Kyoo.Controllers obj.GenreLinks = (await Task.WhenAll(obj.GenreLinks.Select(async x => { using IServiceScope serviceScope = _serviceProvider.CreateScope(); - IGenreRepository genres = serviceScope.ServiceProvider.GetService(); + await using IGenreRepository genres = serviceScope.ServiceProvider.GetService(); x.GenreID = await genres.CreateIfNotExists(x.Genre); return x; @@ -134,7 +134,7 @@ namespace Kyoo.Controllers obj.People = (await Task.WhenAll(obj.People.Select(async x => { using IServiceScope serviceScope = _serviceProvider.CreateScope(); - IPeopleRepository people = serviceScope.ServiceProvider.GetService(); + await using IPeopleRepository people = serviceScope.ServiceProvider.GetService(); x.PeopleID = await people.CreateIfNotExists(x.People); return x; @@ -146,7 +146,7 @@ namespace Kyoo.Controllers obj.ExternalIDs = (await Task.WhenAll(obj.ExternalIDs.Select(async x => { using IServiceScope serviceScope = _serviceProvider.CreateScope(); - IProviderRepository providers = serviceScope.ServiceProvider.GetService(); + await using IProviderRepository providers = serviceScope.ServiceProvider.GetService(); x.ProviderID = await providers.CreateIfNotExists(x.Provider); return x; diff --git a/Kyoo/Tasks/Crawler.cs b/Kyoo/Tasks/Crawler.cs index a0388769..a12e777b 100644 --- a/Kyoo/Tasks/Crawler.cs +++ b/Kyoo/Tasks/Crawler.cs @@ -51,7 +51,7 @@ namespace Kyoo.Controllers try { using IServiceScope serviceScope = _serviceProvider.CreateScope(); - ILibraryManager libraryManager = serviceScope.ServiceProvider.GetService(); + await using ILibraryManager libraryManager = serviceScope.ServiceProvider.GetService(); ICollection episodes = await libraryManager.GetEpisodes(); ICollection libraries = argument == null ? await libraryManager.GetLibraries() @@ -129,7 +129,7 @@ namespace Kyoo.Controllers return; using IServiceScope serviceScope = _serviceProvider.CreateScope(); - ILibraryManager libraryManager = serviceScope.ServiceProvider.GetService(); + await using ILibraryManager libraryManager = serviceScope.ServiceProvider.GetService(); string patern = _config.GetValue("regex"); Regex regex = new Regex(patern, RegexOptions.IgnoreCase); diff --git a/Kyoo/appsettings.json b/Kyoo/appsettings.json index a3b4f927..836b8edf 100644 --- a/Kyoo/appsettings.json +++ b/Kyoo/appsettings.json @@ -12,7 +12,7 @@ }, "AllowedHosts": "*", "ConnectionStrings": { - "Database": "Server=127.0.0.1; Port=5432; Database=kyooDB; User Id=kyoo; Password=kyooPassword;" + "Database": "Server=127.0.0.1; Port=5432; Database=kyooDB; User Id=kyoo; Password=kyooPassword; Pooling=true;" }, "scheduledTasks": { From 57ade5470f37a95c806bd36a43804c8bf78a1529 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Thu, 11 Jun 2020 23:02:09 +0200 Subject: [PATCH 4/8] Implementing duplicate exceptions for every repository --- .../Repositories/CollectionRepository.cs | 5 +++-- .../Controllers/Repositories/EpisodeRepository.cs | 13 ++++++++++++- Kyoo/Controllers/Repositories/GenreRepository.cs | 13 ++++++++++++- Kyoo/Controllers/Repositories/Helper.cs | 14 ++++++++++++++ .../Controllers/Repositories/LibraryRepository.cs | 13 ++++++++++++- Kyoo/Controllers/Repositories/PeopleRepository.cs | 13 ++++++++++++- .../Repositories/ProviderRepository.cs | 15 +++++++++++++-- Kyoo/Controllers/Repositories/SeasonRepository.cs | 13 ++++++++++++- Kyoo/Controllers/Repositories/ShowRepository.cs | 13 ++++++++++++- Kyoo/Controllers/Repositories/StudioRepository.cs | 14 ++++++++++++-- Kyoo/Controllers/Repositories/TrackRepository.cs | 14 ++++++++++++-- 11 files changed, 126 insertions(+), 14 deletions(-) create mode 100644 Kyoo/Controllers/Repositories/Helper.cs diff --git a/Kyoo/Controllers/Repositories/CollectionRepository.cs b/Kyoo/Controllers/Repositories/CollectionRepository.cs index 091c2143..6a30f2ab 100644 --- a/Kyoo/Controllers/Repositories/CollectionRepository.cs +++ b/Kyoo/Controllers/Repositories/CollectionRepository.cs @@ -57,14 +57,15 @@ namespace Kyoo.Controllers if (obj == null) throw new ArgumentNullException(nameof(obj)); + _database.Entry(obj).State = EntityState.Added; + try { - await _database.Collections.AddAsync(obj); await _database.SaveChangesAsync(); } catch (DbUpdateException ex) { - if (ex.InnerException is PostgresException inner && inner.SqlState == PostgresErrorCodes.UniqueViolation) + if (Helper.IsDuplicateException(ex)) throw new DuplicatedItemException($"Trying to insert a duplicated collection (slug {obj.Slug} already exists)."); throw; } diff --git a/Kyoo/Controllers/Repositories/EpisodeRepository.cs b/Kyoo/Controllers/Repositories/EpisodeRepository.cs index 56114ae3..81650aaa 100644 --- a/Kyoo/Controllers/Repositories/EpisodeRepository.cs +++ b/Kyoo/Controllers/Repositories/EpisodeRepository.cs @@ -6,6 +6,7 @@ using Kyoo.Models; using Kyoo.Models.Exceptions; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; +using Npgsql; namespace Kyoo.Controllers { @@ -83,7 +84,17 @@ namespace Kyoo.Controllers if (obj.Tracks != null) foreach (Track entry in obj.Tracks) _database.Entry(entry).State = EntityState.Added; - await _database.SaveChangesAsync(); + + try + { + await _database.SaveChangesAsync(); + } + catch (DbUpdateException ex) + { + if (Helper.IsDuplicateException(ex)) + throw new DuplicatedItemException($"Trying to insert a duplicated episode (slug {obj.Slug} already exists)."); + throw; + } return obj.ID; } diff --git a/Kyoo/Controllers/Repositories/GenreRepository.cs b/Kyoo/Controllers/Repositories/GenreRepository.cs index d9299465..74442a54 100644 --- a/Kyoo/Controllers/Repositories/GenreRepository.cs +++ b/Kyoo/Controllers/Repositories/GenreRepository.cs @@ -57,7 +57,18 @@ namespace Kyoo.Controllers throw new ArgumentNullException(nameof(obj)); await _database.Genres.AddAsync(obj); - await _database.SaveChangesAsync(); + + try + { + await _database.SaveChangesAsync(); + } + catch (DbUpdateException ex) + { + if (Helper.IsDuplicateException(ex)) + throw new DuplicatedItemException($"Trying to insert a duplicated genre (slug {obj.Slug} already exists)."); + throw; + } + return obj.ID; } diff --git a/Kyoo/Controllers/Repositories/Helper.cs b/Kyoo/Controllers/Repositories/Helper.cs new file mode 100644 index 00000000..2881fe77 --- /dev/null +++ b/Kyoo/Controllers/Repositories/Helper.cs @@ -0,0 +1,14 @@ +using Microsoft.EntityFrameworkCore; +using Npgsql; + +namespace Kyoo.Controllers +{ + public static class Helper + { + public static bool IsDuplicateException(DbUpdateException ex) + { + return ex.InnerException is PostgresException inner + && inner.SqlState == PostgresErrorCodes.UniqueViolation; + } + } +} \ No newline at end of file diff --git a/Kyoo/Controllers/Repositories/LibraryRepository.cs b/Kyoo/Controllers/Repositories/LibraryRepository.cs index 4c0f0e17..8f665922 100644 --- a/Kyoo/Controllers/Repositories/LibraryRepository.cs +++ b/Kyoo/Controllers/Repositories/LibraryRepository.cs @@ -64,7 +64,18 @@ namespace Kyoo.Controllers if (obj.ProviderLinks != null) foreach (ProviderLink entry in obj.ProviderLinks) _database.Entry(entry).State = EntityState.Added; - await _database.SaveChangesAsync(); + + try + { + await _database.SaveChangesAsync(); + } + catch (DbUpdateException ex) + { + if (Helper.IsDuplicateException(ex)) + throw new DuplicatedItemException($"Trying to insert a duplicated library (slug {obj.Slug} already exists)."); + throw; + } + return obj.ID; } diff --git a/Kyoo/Controllers/Repositories/PeopleRepository.cs b/Kyoo/Controllers/Repositories/PeopleRepository.cs index 5dd5de09..444d2836 100644 --- a/Kyoo/Controllers/Repositories/PeopleRepository.cs +++ b/Kyoo/Controllers/Repositories/PeopleRepository.cs @@ -63,7 +63,18 @@ namespace Kyoo.Controllers if (obj.ExternalIDs != null) foreach (MetadataID entry in obj.ExternalIDs) _database.Entry(entry).State = EntityState.Added; - await _database.SaveChangesAsync(); + + try + { + await _database.SaveChangesAsync(); + } + catch (DbUpdateException ex) + { + if (Helper.IsDuplicateException(ex)) + throw new DuplicatedItemException($"Trying to insert a duplicated people (slug {obj.Slug} already exists)."); + throw; + } + return obj.ID; } diff --git a/Kyoo/Controllers/Repositories/ProviderRepository.cs b/Kyoo/Controllers/Repositories/ProviderRepository.cs index 1cee1d2a..fd4c56eb 100644 --- a/Kyoo/Controllers/Repositories/ProviderRepository.cs +++ b/Kyoo/Controllers/Repositories/ProviderRepository.cs @@ -56,8 +56,19 @@ namespace Kyoo.Controllers if (obj == null) throw new ArgumentNullException(nameof(obj)); - await _database.Providers.AddAsync(obj); - await _database.SaveChangesAsync(); + _database.Entry(obj).State = EntityState.Added; + + try + { + await _database.SaveChangesAsync(); + } + catch (DbUpdateException ex) + { + if (Helper.IsDuplicateException(ex)) + throw new DuplicatedItemException($"Trying to insert a duplicated provider (name {obj.Name} already exists)."); + throw; + } + return obj.ID; } diff --git a/Kyoo/Controllers/Repositories/SeasonRepository.cs b/Kyoo/Controllers/Repositories/SeasonRepository.cs index 4751b08d..e7a56faa 100644 --- a/Kyoo/Controllers/Repositories/SeasonRepository.cs +++ b/Kyoo/Controllers/Repositories/SeasonRepository.cs @@ -76,7 +76,18 @@ namespace Kyoo.Controllers if (obj.ExternalIDs != null) foreach (MetadataID entry in obj.ExternalIDs) _database.Entry(entry).State = EntityState.Added; - await _database.SaveChangesAsync(); + + try + { + await _database.SaveChangesAsync(); + } + catch (DbUpdateException ex) + { + if (Helper.IsDuplicateException(ex)) + throw new DuplicatedItemException($"Trying to insert a duplicated season (slug {obj.Slug} already exists)."); + throw; + } + return obj.ID; } diff --git a/Kyoo/Controllers/Repositories/ShowRepository.cs b/Kyoo/Controllers/Repositories/ShowRepository.cs index bea40425..e0973f6b 100644 --- a/Kyoo/Controllers/Repositories/ShowRepository.cs +++ b/Kyoo/Controllers/Repositories/ShowRepository.cs @@ -80,7 +80,18 @@ namespace Kyoo.Controllers if (obj.ExternalIDs != null) foreach (MetadataID entry in obj.ExternalIDs) _database.Entry(entry).State = EntityState.Added; - await _database.SaveChangesAsync(); + + try + { + await _database.SaveChangesAsync(); + } + catch (DbUpdateException ex) + { + if (Helper.IsDuplicateException(ex)) + throw new DuplicatedItemException($"Trying to insert a duplicated show (slug {obj.Slug} already exists)."); + throw; + } + return obj.ID; } diff --git a/Kyoo/Controllers/Repositories/StudioRepository.cs b/Kyoo/Controllers/Repositories/StudioRepository.cs index d8cb7b1c..59731717 100644 --- a/Kyoo/Controllers/Repositories/StudioRepository.cs +++ b/Kyoo/Controllers/Repositories/StudioRepository.cs @@ -56,8 +56,18 @@ namespace Kyoo.Controllers if (obj == null) throw new ArgumentNullException(nameof(obj)); - await _database.Studios.AddAsync(obj); - await _database.SaveChangesAsync(); + _database.Entry(obj).State = EntityState.Added; + + try + { + await _database.SaveChangesAsync(); + } + catch (DbUpdateException ex) + { + if (Helper.IsDuplicateException(ex)) + throw new DuplicatedItemException($"Trying to insert a duplicated studio (slug {obj.Slug} already exists)."); + throw; + } return obj.ID; } diff --git a/Kyoo/Controllers/Repositories/TrackRepository.cs b/Kyoo/Controllers/Repositories/TrackRepository.cs index 199a452b..550cee01 100644 --- a/Kyoo/Controllers/Repositories/TrackRepository.cs +++ b/Kyoo/Controllers/Repositories/TrackRepository.cs @@ -61,9 +61,19 @@ namespace Kyoo.Controllers if (obj.EpisodeID <= 0) throw new InvalidOperationException($"Can't store a track not related to any episode (episodeID: {obj.EpisodeID})."); + + _database.Entry(obj).State = EntityState.Added; - await _database.Tracks.AddAsync(obj); - await _database.SaveChangesAsync(); + try + { + await _database.SaveChangesAsync(); + } + catch (DbUpdateException ex) + { + if (Helper.IsDuplicateException(ex)) + throw new DuplicatedItemException($"Trying to insert a duplicated track (slug {obj.Slug} already exists)."); + throw; + } return obj.ID; } From 7694bd97aeceeb9f3e1b3abd49ba309fc0cafa79 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Thu, 11 Jun 2020 23:04:22 +0200 Subject: [PATCH 5/8] Handling race conditions on create if not exists of every repositories --- Kyoo/Controllers/Repositories/EpisodeRepository.cs | 12 +++++++++++- Kyoo/Controllers/Repositories/GenreRepository.cs | 12 +++++++++++- Kyoo/Controllers/Repositories/LibraryRepository.cs | 14 ++++++++++++-- Kyoo/Controllers/Repositories/PeopleRepository.cs | 12 +++++++++++- .../Controllers/Repositories/ProviderRepository.cs | 12 +++++++++++- Kyoo/Controllers/Repositories/SeasonRepository.cs | 12 +++++++++++- Kyoo/Controllers/Repositories/ShowRepository.cs | 12 +++++++++++- Kyoo/Controllers/Repositories/StudioRepository.cs | 12 +++++++++++- 8 files changed, 89 insertions(+), 9 deletions(-) diff --git a/Kyoo/Controllers/Repositories/EpisodeRepository.cs b/Kyoo/Controllers/Repositories/EpisodeRepository.cs index 81650aaa..e0e4f7f7 100644 --- a/Kyoo/Controllers/Repositories/EpisodeRepository.cs +++ b/Kyoo/Controllers/Repositories/EpisodeRepository.cs @@ -106,7 +106,17 @@ namespace Kyoo.Controllers Episode old = await Get(obj.Slug); if (old != null) return old.ID; - return await Create(obj); + try + { + return await Create(obj); + } + catch (DuplicatedItemException) + { + old = await Get(obj.Slug); + if (old == null) + throw new SystemException("Unknown database state."); + return old.ID; + } } public async Task Edit(Episode edited, bool resetOld) diff --git a/Kyoo/Controllers/Repositories/GenreRepository.cs b/Kyoo/Controllers/Repositories/GenreRepository.cs index 74442a54..06d73755 100644 --- a/Kyoo/Controllers/Repositories/GenreRepository.cs +++ b/Kyoo/Controllers/Repositories/GenreRepository.cs @@ -80,7 +80,17 @@ namespace Kyoo.Controllers Genre old = await Get(obj.Slug); if (old != null) return old.ID; - return await Create(obj); + try + { + return await Create(obj); + } + catch (DuplicatedItemException) + { + old = await Get(obj.Slug); + if (old == null) + throw new SystemException("Unknown database state."); + return old.ID; + } } public async Task Edit(Genre edited, bool resetOld) diff --git a/Kyoo/Controllers/Repositories/LibraryRepository.cs b/Kyoo/Controllers/Repositories/LibraryRepository.cs index 8f665922..257399a1 100644 --- a/Kyoo/Controllers/Repositories/LibraryRepository.cs +++ b/Kyoo/Controllers/Repositories/LibraryRepository.cs @@ -84,10 +84,20 @@ namespace Kyoo.Controllers if (obj == null) throw new ArgumentNullException(nameof(obj)); - Library old = await Get(obj.Name); + Library old = await Get(obj.Slug); if (old != null) return old.ID; - return await Create(obj); + try + { + return await Create(obj); + } + catch (DuplicatedItemException) + { + old = await Get(obj.Slug); + if (old == null) + throw new SystemException("Unknown database state."); + return old.ID; + } } public async Task Edit(Library edited, bool resetOld) diff --git a/Kyoo/Controllers/Repositories/PeopleRepository.cs b/Kyoo/Controllers/Repositories/PeopleRepository.cs index 444d2836..2cfae2b9 100644 --- a/Kyoo/Controllers/Repositories/PeopleRepository.cs +++ b/Kyoo/Controllers/Repositories/PeopleRepository.cs @@ -86,7 +86,17 @@ namespace Kyoo.Controllers People old = await Get(obj.Slug); if (old != null) return old.ID; - return await Create(obj); + try + { + return await Create(obj); + } + catch (DuplicatedItemException) + { + old = await Get(obj.Slug); + if (old == null) + throw new SystemException("Unknown database state."); + return old.ID; + } } public async Task Edit(People edited, bool resetOld) diff --git a/Kyoo/Controllers/Repositories/ProviderRepository.cs b/Kyoo/Controllers/Repositories/ProviderRepository.cs index fd4c56eb..7d4c3dfb 100644 --- a/Kyoo/Controllers/Repositories/ProviderRepository.cs +++ b/Kyoo/Controllers/Repositories/ProviderRepository.cs @@ -80,7 +80,17 @@ namespace Kyoo.Controllers ProviderID old = await Get(obj.Name); if (old != null) return old.ID; - return await Create(obj); + try + { + return await Create(obj); + } + catch (DuplicatedItemException) + { + old = await Get(obj.Name); + if (old == null) + throw new SystemException("Unknown database state."); + return old.ID; + } } public async Task Edit(ProviderID edited, bool resetOld) diff --git a/Kyoo/Controllers/Repositories/SeasonRepository.cs b/Kyoo/Controllers/Repositories/SeasonRepository.cs index e7a56faa..23eb188c 100644 --- a/Kyoo/Controllers/Repositories/SeasonRepository.cs +++ b/Kyoo/Controllers/Repositories/SeasonRepository.cs @@ -99,7 +99,17 @@ namespace Kyoo.Controllers Season old = await Get(obj.Slug); if (old != null) return old.ID; - return await Create(obj); + try + { + return await Create(obj); + } + catch (DuplicatedItemException) + { + old = await Get(obj.Slug); + if (old == null) + throw new SystemException("Unknown database state."); + return old.ID; + } } public async Task Edit(Season edited, bool resetOld) diff --git a/Kyoo/Controllers/Repositories/ShowRepository.cs b/Kyoo/Controllers/Repositories/ShowRepository.cs index e0973f6b..7d9b41b4 100644 --- a/Kyoo/Controllers/Repositories/ShowRepository.cs +++ b/Kyoo/Controllers/Repositories/ShowRepository.cs @@ -103,7 +103,17 @@ namespace Kyoo.Controllers Show old = await Get(obj.Slug); if (old != null) return old.ID; - return await Create(obj); + try + { + return await Create(obj); + } + catch (DuplicatedItemException) + { + old = await Get(obj.Slug); + if (old == null) + throw new SystemException("Unknown database state."); + return old.ID; + } } public async Task Edit(Show edited, bool resetOld) diff --git a/Kyoo/Controllers/Repositories/StudioRepository.cs b/Kyoo/Controllers/Repositories/StudioRepository.cs index 59731717..c408234c 100644 --- a/Kyoo/Controllers/Repositories/StudioRepository.cs +++ b/Kyoo/Controllers/Repositories/StudioRepository.cs @@ -79,7 +79,17 @@ namespace Kyoo.Controllers Studio old = await Get(obj.Slug); if (old != null) return old.ID; - return await Create(obj); + try + { + return await Create(obj); + } + catch (DuplicatedItemException) + { + old = await Get(obj.Slug); + if (old == null) + throw new SystemException("Unknown database state."); + return old.ID; + } } public async Task Edit(Studio edited, bool resetOld) From 51fa75214b01e9ae748dc58985d1819454b937f4 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Fri, 12 Jun 2020 03:39:00 +0200 Subject: [PATCH 6/8] Cleaning up --- Kyoo/Startup.cs | 4 ++-- Kyoo/Tasks/Crawler.cs | 24 +++++++++++------------- Kyoo/appsettings.json | 3 ++- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/Kyoo/Startup.cs b/Kyoo/Startup.cs index bd1a5272..c16e5740 100644 --- a/Kyoo/Startup.cs +++ b/Kyoo/Startup.cs @@ -51,8 +51,8 @@ namespace Kyoo services.AddDbContext(options => { options.UseLazyLoadingProxies() - .UseNpgsql(_configuration.GetConnectionString("Database")); - // .EnableSensitiveDataLogging() + .UseNpgsql(_configuration.GetConnectionString("Database")) + .EnableSensitiveDataLogging(); // .UseLoggerFactory(LoggerFactory.Create(builder => builder.AddConsole())); }); diff --git a/Kyoo/Tasks/Crawler.cs b/Kyoo/Tasks/Crawler.cs index a12e777b..b5c6002f 100644 --- a/Kyoo/Tasks/Crawler.cs +++ b/Kyoo/Tasks/Crawler.cs @@ -67,7 +67,8 @@ namespace Kyoo.Controllers foreach (Library library in libraries) library.Providers = library.Providers; - await Task.WhenAll(libraries.Select(x => Scan(x, episodes, cancellationToken))); + await Task.WhenAll(libraries.Select(x => Scan(x, episodes, cancellationToken)).ToArray()); + Console.WriteLine("Done."); } catch (Exception ex) { @@ -79,7 +80,7 @@ namespace Kyoo.Controllers private Task Scan(Library library, IEnumerable episodes, CancellationToken cancellationToken) { Console.WriteLine($"Scanning library {library.Name} at {string.Join(", ", library.Paths)}."); - return Task.WhenAll(library.Paths.Select(async path => + return Task.WhenAll(library.Paths.Select(path => { if (cancellationToken.IsCancellationRequested) return Task.CompletedTask; @@ -91,36 +92,33 @@ namespace Kyoo.Controllers } catch (DirectoryNotFoundException) { - await Console.Error.WriteLineAsync($"The library's directory {path} could not be found (library slug: {library.Slug})"); + Console.Error.WriteLine($"The library's directory {path} could not be found (library slug: {library.Slug})"); return Task.CompletedTask; } catch (PathTooLongException) { - await Console.Error.WriteLineAsync($"The library's directory {path} is too long for this system. (library slug: {library.Slug})"); + Console.Error.WriteLine($"The library's directory {path} is too long for this system. (library slug: {library.Slug})"); return Task.CompletedTask; } catch (ArgumentException) { - await Console.Error.WriteLineAsync($"The library's directory {path} is invalid. (library slug: {library.Slug})"); + Console.Error.WriteLine($"The library's directory {path} is invalid. (library slug: {library.Slug})"); return Task.CompletedTask; } catch (UnauthorizedAccessException) { - await Console.Error.WriteLineAsync($"Permission denied: can't access library's directory at {path}. (library slug: {library.Slug})"); + Console.Error.WriteLine($"Permission denied: can't access library's directory at {path}. (library slug: {library.Slug})"); return Task.CompletedTask; } return Task.WhenAll(files.Select(file => - //foreach (string file in files) { if (!IsVideo(file) || episodes.Any(x => x.Path == file)) - /*continue;*/ return Task.CompletedTask; + return Task.CompletedTask; string relativePath = file.Substring(path.Length); - return /*await*/ RegisterFile(file, relativePath, library, cancellationToken); - })); - - // return Task.CompletedTask; - })); + return RegisterFile(file, relativePath, library, cancellationToken); + }).ToArray()); + }).ToArray()); } private async Task RegisterFile(string path, string relativePath, Library library, CancellationToken token) diff --git a/Kyoo/appsettings.json b/Kyoo/appsettings.json index 836b8edf..5abce11c 100644 --- a/Kyoo/appsettings.json +++ b/Kyoo/appsettings.json @@ -7,7 +7,8 @@ "LogLevel": { "Default": "Warning", "Microsoft": "Warning", - "Microsoft.Hosting.Lifetime": "Information" + "Microsoft.Hosting.Lifetime": "Information", + "Microsoft.EntityFrameworkCore.DbUpdateException": "None" } }, "AllowedHosts": "*", From 33d9c5d9b4e2c3bbaecc5ddbe687d2e61af6d12d Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Fri, 12 Jun 2020 04:31:27 +0200 Subject: [PATCH 7/8] Handling rush conditions on collection/show/season register --- Kyoo/Models/DatabaseContext.cs | 15 ----------- Kyoo/Startup.cs | 5 ---- Kyoo/Tasks/Crawler.cs | 48 +++++++++++++++++++++++++++------- 3 files changed, 38 insertions(+), 30 deletions(-) diff --git a/Kyoo/Models/DatabaseContext.cs b/Kyoo/Models/DatabaseContext.cs index 9b9513d4..429ea4ff 100644 --- a/Kyoo/Models/DatabaseContext.cs +++ b/Kyoo/Models/DatabaseContext.cs @@ -49,21 +49,6 @@ namespace Kyoo } - public class DatabaseFactory - { - private readonly DbContextOptions _options; - - public DatabaseFactory(DbContextOptions options) - { - _options = options; - } - - public DatabaseContext NewDatabaseConnection() - { - return new DatabaseContext(_options); - } - } - public class DatabaseContext : DbContext { public DatabaseContext(DbContextOptions options) : base(options) { } diff --git a/Kyoo/Startup.cs b/Kyoo/Startup.cs index c16e5740..952c4e6a 100644 --- a/Kyoo/Startup.cs +++ b/Kyoo/Startup.cs @@ -43,11 +43,6 @@ namespace Kyoo services.AddControllers().AddNewtonsoftJson(); services.AddHttpClient(); - services.AddSingleton(x => new DatabaseFactory( - new DbContextOptionsBuilder() - .UseLazyLoadingProxies() - .UseNpgsql(_configuration.GetConnectionString("Database")).Options)); - services.AddDbContext(options => { options.UseLazyLoadingProxies() diff --git a/Kyoo/Tasks/Crawler.cs b/Kyoo/Tasks/Crawler.cs index b5c6002f..0604cd27 100644 --- a/Kyoo/Tasks/Crawler.cs +++ b/Kyoo/Tasks/Crawler.cs @@ -7,6 +7,7 @@ using System.Linq; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; +using Kyoo.Models.Exceptions; using Kyoo.Models.Watch; using Microsoft.Extensions.DependencyInjection; @@ -40,7 +41,9 @@ namespace Kyoo.Controllers return null; } - public async Task Run(IServiceProvider serviceProvider, CancellationToken cancellationToken, string argument = null) + public async Task Run(IServiceProvider serviceProvider, + CancellationToken cancellationToken, + string argument = null) { _serviceProvider = serviceProvider; _thumbnailsManager = serviceProvider.GetService(); @@ -68,7 +71,6 @@ namespace Kyoo.Controllers library.Providers = library.Providers; await Task.WhenAll(libraries.Select(x => Scan(x, episodes, cancellationToken)).ToArray()); - Console.WriteLine("Done."); } catch (Exception ex) { @@ -111,6 +113,8 @@ namespace Kyoo.Controllers return Task.CompletedTask; } + // TODO Should register shows first to prevent all tasks of a same show to register it 150+ times. + return Task.WhenAll(files.Select(file => { if (!IsVideo(file) || episodes.Any(x => x.Path == file)) @@ -118,6 +122,7 @@ namespace Kyoo.Controllers string relativePath = file.Substring(path.Length); return RegisterFile(file, relativePath, library, cancellationToken); }).ToArray()); + }).ToArray()); } @@ -166,8 +171,16 @@ namespace Kyoo.Controllers if (collection != null) return collection; collection = await _metadataProvider.GetCollectionFromName(collectionName, library); - await libraryManager.RegisterCollection(collection); - return collection; + + try + { + await libraryManager.RegisterCollection(collection); + return collection; + } + catch (DuplicatedItemException) + { + return await libraryManager.GetCollection(collection.Slug); + } } private async Task GetShow(ILibraryManager libraryManager, @@ -182,10 +195,18 @@ namespace Kyoo.Controllers show = await _metadataProvider.SearchShow(showTitle, isMovie, library); show.Path = showPath; show.People = await _metadataProvider.GetPeople(show, library); - await libraryManager.RegisterShow(show); - await _thumbnailsManager.Validate(show.People); - await _thumbnailsManager.Validate(show); - return show; + + try + { + await libraryManager.RegisterShow(show); + await _thumbnailsManager.Validate(show.People); + await _thumbnailsManager.Validate(show); + return show; + } + catch (DuplicatedItemException) + { + return await libraryManager.GetShow(show.Slug); + } } private async Task GetSeason(ILibraryManager libraryManager, @@ -199,8 +220,15 @@ namespace Kyoo.Controllers if (season == null) { season = await _metadataProvider.GetSeason(show, seasonNumber, library); - await libraryManager.RegisterSeason(season); - await _thumbnailsManager.Validate(season); + try + { + await libraryManager.RegisterSeason(season); + await _thumbnailsManager.Validate(season); + } + catch (DuplicatedItemException) + { + season = await libraryManager.GetSeason(show.Slug, season.SeasonNumber); + } } season.Show = show; return season; From 80389790ddeece3c49a7f49fe15ad693665ecf4d Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Mon, 15 Jun 2020 02:33:03 +0200 Subject: [PATCH 8/8] Registering shows before episodes to prevent duplicates --- Kyoo/Tasks/Crawler.cs | 42 +++++++++++++++++++++++------------------- Kyoo/appsettings.json | 5 +++-- 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/Kyoo/Tasks/Crawler.cs b/Kyoo/Tasks/Crawler.cs index 0604cd27..6a02005b 100644 --- a/Kyoo/Tasks/Crawler.cs +++ b/Kyoo/Tasks/Crawler.cs @@ -31,7 +31,7 @@ namespace Kyoo.Controllers public async Task> GetPossibleParameters() { using IServiceScope serviceScope = _serviceProvider.CreateScope(); - ILibraryManager libraryManager = serviceScope.ServiceProvider.GetService(); + await using ILibraryManager libraryManager = serviceScope.ServiceProvider.GetService(); return (await libraryManager.GetLibraries()).Select(x => x.Slug); } @@ -82,10 +82,10 @@ namespace Kyoo.Controllers private Task Scan(Library library, IEnumerable episodes, CancellationToken cancellationToken) { Console.WriteLine($"Scanning library {library.Name} at {string.Join(", ", library.Paths)}."); - return Task.WhenAll(library.Paths.Select(path => + return Task.WhenAll(library.Paths.Select(async path => { if (cancellationToken.IsCancellationRequested) - return Task.CompletedTask; + return; string[] files; try @@ -94,35 +94,39 @@ namespace Kyoo.Controllers } catch (DirectoryNotFoundException) { - Console.Error.WriteLine($"The library's directory {path} could not be found (library slug: {library.Slug})"); - return Task.CompletedTask; + await Console.Error.WriteLineAsync($"The library's directory {path} could not be found (library slug: {library.Slug})"); + return; } catch (PathTooLongException) { - Console.Error.WriteLine($"The library's directory {path} is too long for this system. (library slug: {library.Slug})"); - return Task.CompletedTask; + await Console.Error.WriteLineAsync($"The library's directory {path} is too long for this system. (library slug: {library.Slug})"); + return; } catch (ArgumentException) { - Console.Error.WriteLine($"The library's directory {path} is invalid. (library slug: {library.Slug})"); - return Task.CompletedTask; + await Console.Error.WriteLineAsync($"The library's directory {path} is invalid. (library slug: {library.Slug})"); + return; } catch (UnauthorizedAccessException) { - Console.Error.WriteLine($"Permission denied: can't access library's directory at {path}. (library slug: {library.Slug})"); - return Task.CompletedTask; + await Console.Error.WriteLineAsync($"Permission denied: can't access library's directory at {path}. (library slug: {library.Slug})"); + return; } - // TODO Should register shows first to prevent all tasks of a same show to register it 150+ times. + List> shows = files + .Where(x => IsVideo(x) && episodes.All(y => y.Path != x)) + .GroupBy(Path.GetDirectoryName) + .ToList(); - return Task.WhenAll(files.Select(file => - { - if (!IsVideo(file) || episodes.Any(x => x.Path == file)) - return Task.CompletedTask; - string relativePath = file.Substring(path.Length); - return RegisterFile(file, relativePath, library, cancellationToken); - }).ToArray()); + await Task.WhenAll(shows + .Select(x => x.First()) + .Select(x => RegisterFile(x, x.Substring(path.Length), library, cancellationToken)) + .ToArray()); + await Task.WhenAll(shows + .SelectMany(x => x.Skip(1)) + .Select(x => RegisterFile(x, x.Substring(path.Length), library, cancellationToken)) + .ToArray()); }).ToArray()); } diff --git a/Kyoo/appsettings.json b/Kyoo/appsettings.json index 5abce11c..6d669b3b 100644 --- a/Kyoo/appsettings.json +++ b/Kyoo/appsettings.json @@ -8,12 +8,13 @@ "Default": "Warning", "Microsoft": "Warning", "Microsoft.Hosting.Lifetime": "Information", - "Microsoft.EntityFrameworkCore.DbUpdateException": "None" + "Microsoft.EntityFrameworkCore.DbUpdateException": "None", + "Microsoft.EntityFrameworkCore.Update": "None" } }, "AllowedHosts": "*", "ConnectionStrings": { - "Database": "Server=127.0.0.1; Port=5432; Database=kyooDB; User Id=kyoo; Password=kyooPassword; Pooling=true;" + "Database": "Server=127.0.0.1; Port=5432; Database=kyooDB; User Id=kyoo; Password=kyooPassword; Pooling=true; MaxPoolSize=60;" }, "scheduledTasks": {