From 7be4133af90dc9fa611c440206c05c0e56550a07 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Thu, 18 Jun 2020 15:43:25 +0200 Subject: [PATCH] Fixing duplicated seasons --- Kyoo/Controllers/Repositories/SeasonRepository.cs | 8 ++++---- Kyoo/Controllers/Repositories/ShowRepository.cs | 4 ++-- Kyoo/Models/DatabaseContext.cs | 9 ++++++--- ...igner.cs => 20200618133537_Initial.Designer.cs} | 11 +++++------ ...010830_Initial.cs => 20200618133537_Initial.cs} | 14 +++++--------- .../Internal/DatabaseContextModelSnapshot.cs | 9 ++++----- Kyoo/Tasks/Crawler.cs | 11 +++++++---- Kyoo/appsettings.json | 1 + 8 files changed, 34 insertions(+), 33 deletions(-) rename Kyoo/Models/DatabaseMigrations/Internal/{20200607010830_Initial.Designer.cs => 20200618133537_Initial.Designer.cs} (98%) rename Kyoo/Models/DatabaseMigrations/Internal/{20200607010830_Initial.cs => 20200618133537_Initial.cs} (98%) diff --git a/Kyoo/Controllers/Repositories/SeasonRepository.cs b/Kyoo/Controllers/Repositories/SeasonRepository.cs index f88c5c2f..da117095 100644 --- a/Kyoo/Controllers/Repositories/SeasonRepository.cs +++ b/Kyoo/Controllers/Repositories/SeasonRepository.cs @@ -30,9 +30,9 @@ namespace Kyoo.Controllers return _database.DisposeAsync(); } - public async Task Get(int id) + public Task Get(int id) { - return await _database.Seasons.FirstOrDefaultAsync(x => x.ID == id); + return _database.Seasons.FirstOrDefaultAsync(x => x.ID == id); } public Task Get(string slug) @@ -46,9 +46,9 @@ namespace Kyoo.Controllers return Get(showSlug, seasonNumber); } - public async Task Get(string showSlug, int seasonNumber) + public Task Get(string showSlug, int seasonNumber) { - return await _database.Seasons.FirstOrDefaultAsync(x => x.Show.Slug == showSlug + return _database.Seasons.FirstOrDefaultAsync(x => x.Show.Slug == showSlug && x.SeasonNumber == seasonNumber); } diff --git a/Kyoo/Controllers/Repositories/ShowRepository.cs b/Kyoo/Controllers/Repositories/ShowRepository.cs index c39741b9..7783cb82 100644 --- a/Kyoo/Controllers/Repositories/ShowRepository.cs +++ b/Kyoo/Controllers/Repositories/ShowRepository.cs @@ -58,8 +58,8 @@ namespace Kyoo.Controllers public async Task> Search(string query) { return await _database.Shows - .FromSqlInterpolated($@"SELECT * FROM Shows WHERE Shows.Title LIKE {$"%{query}%"} - OR Shows.Aliases LIKE {$"%{query}%"}") + .FromSqlInterpolated($@"SELECT * FROM Shows WHERE 'Shows.Title' LIKE {$"%{query}%"} + OR 'Shows.Aliases' LIKE {$"%{query}%"}") .Take(20) .ToListAsync(); } diff --git a/Kyoo/Models/DatabaseContext.cs b/Kyoo/Models/DatabaseContext.cs index e9407992..86e3aea1 100644 --- a/Kyoo/Models/DatabaseContext.cs +++ b/Kyoo/Models/DatabaseContext.cs @@ -136,15 +136,18 @@ namespace Kyoo modelBuilder.Entity() .HasIndex(x => x.Slug) .IsUnique(); - modelBuilder.Entity() - .HasIndex(x => x.Slug) - .IsUnique(); modelBuilder.Entity() .HasIndex(x => x.Slug) .IsUnique(); modelBuilder.Entity() .HasIndex(x => x.Name) .IsUnique(); + modelBuilder.Entity() + .HasIndex(x => new {x.ShowID, x.SeasonNumber}) + .IsUnique(); + modelBuilder.Entity() + .HasIndex(x => new {x.ShowID, x.SeasonNumber, x.EpisodeNumber, x.AbsoluteNumber}) + .IsUnique(); } public void DiscardChanges() diff --git a/Kyoo/Models/DatabaseMigrations/Internal/20200607010830_Initial.Designer.cs b/Kyoo/Models/DatabaseMigrations/Internal/20200618133537_Initial.Designer.cs similarity index 98% rename from Kyoo/Models/DatabaseMigrations/Internal/20200607010830_Initial.Designer.cs rename to Kyoo/Models/DatabaseMigrations/Internal/20200618133537_Initial.Designer.cs index 1aaddf9b..5b5a5668 100644 --- a/Kyoo/Models/DatabaseMigrations/Internal/20200607010830_Initial.Designer.cs +++ b/Kyoo/Models/DatabaseMigrations/Internal/20200618133537_Initial.Designer.cs @@ -10,7 +10,7 @@ using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; namespace Kyoo.Models.DatabaseMigrations.Internal { [DbContext(typeof(DatabaseContext))] - [Migration("20200607010830_Initial")] + [Migration("20200618133537_Initial")] partial class Initial { protected override void BuildTargetModel(ModelBuilder modelBuilder) @@ -117,7 +117,8 @@ namespace Kyoo.Models.DatabaseMigrations.Internal b.HasIndex("SeasonID"); - b.HasIndex("ShowID"); + b.HasIndex("ShowID", "SeasonNumber", "EpisodeNumber", "AbsoluteNumber") + .IsUnique(); b.ToTable("Episodes"); }); @@ -374,7 +375,8 @@ namespace Kyoo.Models.DatabaseMigrations.Internal b.HasKey("ID"); - b.HasIndex("ShowID"); + b.HasIndex("ShowID", "SeasonNumber") + .IsUnique(); b.ToTable("Seasons"); }); @@ -430,9 +432,6 @@ namespace Kyoo.Models.DatabaseMigrations.Internal b.HasKey("ID"); - b.HasIndex("Slug") - .IsUnique(); - b.HasIndex("StudioID"); b.ToTable("Shows"); diff --git a/Kyoo/Models/DatabaseMigrations/Internal/20200607010830_Initial.cs b/Kyoo/Models/DatabaseMigrations/Internal/20200618133537_Initial.cs similarity index 98% rename from Kyoo/Models/DatabaseMigrations/Internal/20200607010830_Initial.cs rename to Kyoo/Models/DatabaseMigrations/Internal/20200618133537_Initial.cs index f83d1361..6890dbb9 100644 --- a/Kyoo/Models/DatabaseMigrations/Internal/20200607010830_Initial.cs +++ b/Kyoo/Models/DatabaseMigrations/Internal/20200618133537_Initial.cs @@ -423,9 +423,10 @@ namespace Kyoo.Models.DatabaseMigrations.Internal column: "SeasonID"); migrationBuilder.CreateIndex( - name: "IX_Episodes_ShowID", + name: "IX_Episodes_ShowID_SeasonNumber_EpisodeNumber_AbsoluteNumber", table: "Episodes", - column: "ShowID"); + columns: new[] { "ShowID", "SeasonNumber", "EpisodeNumber", "AbsoluteNumber" }, + unique: true); migrationBuilder.CreateIndex( name: "IX_GenreLinks_GenreID", @@ -517,14 +518,9 @@ namespace Kyoo.Models.DatabaseMigrations.Internal unique: true); migrationBuilder.CreateIndex( - name: "IX_Seasons_ShowID", + name: "IX_Seasons_ShowID_SeasonNumber", table: "Seasons", - column: "ShowID"); - - migrationBuilder.CreateIndex( - name: "IX_Shows_Slug", - table: "Shows", - column: "Slug", + columns: new[] { "ShowID", "SeasonNumber" }, unique: true); migrationBuilder.CreateIndex( diff --git a/Kyoo/Models/DatabaseMigrations/Internal/DatabaseContextModelSnapshot.cs b/Kyoo/Models/DatabaseMigrations/Internal/DatabaseContextModelSnapshot.cs index a0e42322..4b4f404d 100644 --- a/Kyoo/Models/DatabaseMigrations/Internal/DatabaseContextModelSnapshot.cs +++ b/Kyoo/Models/DatabaseMigrations/Internal/DatabaseContextModelSnapshot.cs @@ -115,7 +115,8 @@ namespace Kyoo.Models.DatabaseMigrations.Internal b.HasIndex("SeasonID"); - b.HasIndex("ShowID"); + b.HasIndex("ShowID", "SeasonNumber", "EpisodeNumber", "AbsoluteNumber") + .IsUnique(); b.ToTable("Episodes"); }); @@ -372,7 +373,8 @@ namespace Kyoo.Models.DatabaseMigrations.Internal b.HasKey("ID"); - b.HasIndex("ShowID"); + b.HasIndex("ShowID", "SeasonNumber") + .IsUnique(); b.ToTable("Seasons"); }); @@ -428,9 +430,6 @@ namespace Kyoo.Models.DatabaseMigrations.Internal b.HasKey("ID"); - b.HasIndex("Slug") - .IsUnique(); - b.HasIndex("StudioID"); b.ToTable("Shows"); diff --git a/Kyoo/Tasks/Crawler.cs b/Kyoo/Tasks/Crawler.cs index 79238752..17afc242 100644 --- a/Kyoo/Tasks/Crawler.cs +++ b/Kyoo/Tasks/Crawler.cs @@ -27,6 +27,8 @@ namespace Kyoo.Controllers private IProviderManager _metadataProvider; private ITranscoder _transcoder; private IConfiguration _config; + + private int _parallelTasks; public async Task> GetPossibleParameters() { @@ -50,6 +52,9 @@ namespace Kyoo.Controllers _metadataProvider = serviceProvider.GetService(); _transcoder = serviceProvider.GetService(); _config = serviceProvider.GetService(); + _parallelTasks = _config.GetValue("parallelTasks"); + if (_parallelTasks <= 0) + _parallelTasks = 30; using IServiceScope serviceScope = _serviceProvider.CreateScope(); await using ILibraryManager libraryManager = serviceScope.ServiceProvider.GetService(); @@ -112,18 +117,16 @@ namespace Kyoo.Controllers .GroupBy(Path.GetDirectoryName) .ToList(); - // Todo batch wth higher numbers per list once multi-services has been implemented. - IEnumerable tasks = shows .Select(x => x.First()) .Select(x => RegisterFile(x, x.Substring(path.Length), library, cancellationToken)); - foreach (Task[] showTasks in tasks.BatchBy(30)) + foreach (Task[] showTasks in tasks.BatchBy(_parallelTasks)) await Task.WhenAll(showTasks); tasks = shows .SelectMany(x => x.Skip(1)) .Select(x => RegisterFile(x, x.Substring(path.Length), library, cancellationToken)); - foreach (Task[] episodeTasks in tasks.BatchBy(50)) + foreach (Task[] episodeTasks in tasks.BatchBy(_parallelTasks * 2)) await Task.WhenAll(episodeTasks); } } diff --git a/Kyoo/appsettings.json b/Kyoo/appsettings.json index 0e8f69a9..15eb271e 100644 --- a/Kyoo/appsettings.json +++ b/Kyoo/appsettings.json @@ -17,6 +17,7 @@ "ConnectionStrings": { "Database": "Server=127.0.0.1; Port=5432; Database=kyooDB; User Id=kyoo; Password=kyooPassword; Pooling=true; MaxPoolSize=95; Timeout=30;" }, + "parallelTasks": "40", "scheduledTasks": { "scan": "24:00:00"