diff --git a/back/src/Kyoo.Abstractions/Models/Page.cs b/back/src/Kyoo.Abstractions/Models/Page.cs index ca176489..fff8524e 100644 --- a/back/src/Kyoo.Abstractions/Models/Page.cs +++ b/back/src/Kyoo.Abstractions/Models/Page.cs @@ -90,23 +90,18 @@ namespace Kyoo.Abstractions.Models { Items = items; This = url + query.ToQueryString(); - - if (!(query.TryGetValue("sortBy", out string? sort) && sort.Contains("random"))) + if (items.Count > 0 && query.ContainsKey("afterID")) { - if (items.Count > 0 && query.ContainsKey("afterID")) - { - query["afterID"] = items.First().Id.ToString(); - query["reverse"] = "true"; - Previous = url + query.ToQueryString(); - } - query.Remove("reverse"); - if (items.Count == limit && limit > 0) - { - query["afterID"] = items.Last().Id.ToString(); - Next = url + query.ToQueryString(); - } + query["afterID"] = items.First().Id.ToString(); + query["reverse"] = "true"; + Previous = url + query.ToQueryString(); + } + query.Remove("reverse"); + if (items.Count == limit && limit > 0) + { + query["afterID"] = items.Last().Id.ToString(); + Next = url + query.ToQueryString(); } - query.Remove("afterID"); First = url + query.ToQueryString(); } diff --git a/back/src/Kyoo.Abstractions/Models/Utils/Sort.cs b/back/src/Kyoo.Abstractions/Models/Utils/Sort.cs index ddae48ed..e34efcd1 100644 --- a/back/src/Kyoo.Abstractions/Models/Utils/Sort.cs +++ b/back/src/Kyoo.Abstractions/Models/Utils/Sort.cs @@ -57,7 +57,7 @@ namespace Kyoo.Abstractions.Controllers public record Conglomerate(params Sort[] List) : Sort; /// Sort randomly items - public record Random() : Sort; + public record Random(int seed) : Sort; /// The default sort method for the given type. public record Default : Sort; @@ -73,10 +73,13 @@ namespace Kyoo.Abstractions.Controllers if (string.IsNullOrEmpty(sortBy) || sortBy == "default") return new Default(); if (sortBy == "random") - return new Random(); + return new Random(new System.Random().Next(int.MinValue, int.MaxValue)); if (sortBy.Contains(',')) return new Conglomerate(sortBy.Split(',').Select(From).ToArray()); + if (sortBy.StartsWith("random:")) + return new Random(int.Parse(sortBy["random:".Length..])); + string key = sortBy.Contains(':') ? sortBy[..sortBy.IndexOf(':')] : sortBy; string? order = sortBy.Contains(':') ? sortBy[(sortBy.IndexOf(':') + 1)..] : null; bool desendant = order switch diff --git a/back/src/Kyoo.Core/Controllers/Repositories/LocalRepository.cs b/back/src/Kyoo.Core/Controllers/Repositories/LocalRepository.cs index a8db7ce4..5c9ed0a3 100644 --- a/back/src/Kyoo.Core/Controllers/Repositories/LocalRepository.cs +++ b/back/src/Kyoo.Core/Controllers/Repositories/LocalRepository.cs @@ -28,6 +28,7 @@ using Kyoo.Abstractions.Models; using Kyoo.Abstractions.Models.Attributes; using Kyoo.Abstractions.Models.Exceptions; using Kyoo.Core.Api; +using Kyoo.Postgresql; using Kyoo.Utils; using Microsoft.EntityFrameworkCore; @@ -109,8 +110,9 @@ namespace Kyoo.Core.Controllers return _Sort(query, DefaultSort, then); case Sort.By(var key, var desc): return _SortBy(query, x => EF.Property(x, key), desc, then); - case Sort.Random: - return _SortBy(query, x => EF.Functions.Random(), false, then); + case Sort.Random(var seed): + // NOTE: To edit this, don't forget to edit the random handiling inside the KeysetPaginate function + return _SortBy(query, x => DatabaseContext.MD5(seed + x.Id.ToString()), false, then); case Sort.Conglomerate(var sorts): IOrderedQueryable nQuery = _Sort(query, sorts.First(), false); foreach (Sort sort in sorts.Skip(1)) @@ -136,6 +138,8 @@ namespace Kyoo.Core.Controllers : (greaterThan ? Expression.GreaterThan : Expression.LessThan); } + private record SortIndicator(string key, bool desc, string? seed); + /// /// Create a filter (where) expression on the query to skip everything before/after the referenceID. /// The generalized expression for this in pseudocode is: @@ -164,45 +168,66 @@ namespace Kyoo.Core.Controllers ParameterExpression x = Expression.Parameter(typeof(T), "x"); ConstantExpression referenceC = Expression.Constant(reference, typeof(T)); - IEnumerable.By> GetSortsBy(Sort sort) + void GetRandomSortKeys(string seed, out Expression left, out Expression right) + { + MethodInfo concat = typeof(string).GetMethod(nameof(string.Concat), new[] { typeof(string), typeof(string) }); + Expression id = Expression.Call(Expression.Property(x, "ID"), nameof(int.ToString), null); + Expression xrng = Expression.Call(concat, Expression.Constant(seed), id); + right = Expression.Call(typeof(DatabaseContext), nameof(DatabaseContext.MD5), null, Expression.Constant($"{seed}{reference.Id}")); + left = Expression.Call(typeof(DatabaseContext), nameof(DatabaseContext.MD5), null, xrng); + } + + IEnumerable GetSortsBy(Sort sort) { return sort switch { Sort.Default => GetSortsBy(DefaultSort), - Sort.By @sortBy => new[] { sortBy }, + Sort.By @sortBy => new[] { new SortIndicator(sortBy.Key, sortBy.Desendant, null) }, Sort.Conglomerate(var list) => list.SelectMany(GetSortsBy), - Sort.Random => throw new ArgumentException("Impossible to paginate randomly sorted items."), - _ => Array.Empty.By>(), + Sort.Random(var seed) => new[] { new SortIndicator("random", false, seed.ToString()) }, + _ => Array.Empty(), }; } // Don't forget that every sorts must end with a ID sort (to differentiate equalities). - Sort.By id = new(x => x.Id); - IEnumerable.By> sorts = GetSortsBy(sort).Append(id); + IEnumerable sorts = GetSortsBy(sort) + .Append(new SortIndicator("Id", false, null)); BinaryExpression filter = null; - List.By> previousSteps = new(); + List previousSteps = new(); // TODO: Add an outer query >= for perf // PERF: See https://use-the-index-luke.com/sql/partial-results/fetch-next-page#sb-equivalent-logic - foreach ((string key, bool desc) in sorts) + foreach ((string key, bool desc, string? seed) in sorts) { BinaryExpression compare = null; - PropertyInfo property = typeof(T).GetProperty(key); + PropertyInfo property = key != "random" + ? typeof(T).GetProperty(key) + : null; // Comparing a value with null always return false so we short opt < > comparisons with null. - if (property!.GetValue(reference) == null) + if (property != null && property.GetValue(reference) == null) { - previousSteps.Add(new Sort.By(key, desc)); + previousSteps.Add(new SortIndicator(key, desc, seed)); continue; } // Create all the equality statements for previous sorts. - foreach ((string pKey, bool pDesc) in previousSteps) + foreach ((string pKey, bool pDesc, string? pSeed) in previousSteps) { - BinaryExpression pcompare = Expression.Equal( - Expression.Property(x, pKey), - Expression.Property(referenceC, pKey) - ); + BinaryExpression pcompare; + + if (pSeed == null) + { + pcompare = Expression.Equal( + Expression.Property(x, pKey), + Expression.Property(referenceC, pKey) + ); + } + else + { + GetRandomSortKeys(pSeed, out Expression left, out Expression right); + pcompare = Expression.Equal(left, right); + } compare = compare != null ? Expression.AndAlso(compare, pcompare) : pcompare; @@ -210,14 +235,21 @@ namespace Kyoo.Core.Controllers // Create the last comparison of the statement. Func comparer = _GetComparisonExpression(desc, next, false); - MemberExpression xkey = Expression.Property(x, key); - MemberExpression rkey = Expression.Property(referenceC, key); + Expression xkey; + Expression rkey; + if (seed == null) + { + xkey = Expression.Property(x, key); + rkey = Expression.Property(referenceC, key); + } + else + GetRandomSortKeys(seed, out xkey, out rkey); BinaryExpression lastCompare = ApiHelper.StringCompatibleExpression(comparer, xkey, rkey); // Comparing a value with null always return false for nulls so we must add nulls to the results manually. // Postgres sorts them after values so we will do the same // We only add this condition if the column type is nullable - if (Nullable.GetUnderlyingType(property.PropertyType) != null) + if (property != null && Nullable.GetUnderlyingType(property.PropertyType) != null) { BinaryExpression equalNull = Expression.Equal(xkey, Expression.Constant(null)); lastCompare = Expression.OrElse(lastCompare, equalNull); @@ -231,7 +263,7 @@ namespace Kyoo.Core.Controllers ? Expression.OrElse(filter, compare) : compare; - previousSteps.Add(new(key, desc)); + previousSteps.Add(new SortIndicator(key, desc, seed)); } return Expression.Lambda>(filter!, x); } diff --git a/back/src/Kyoo.Core/Views/Helper/ApiHelper.cs b/back/src/Kyoo.Core/Views/Helper/ApiHelper.cs index 6d4b01d2..432a7a54 100644 --- a/back/src/Kyoo.Core/Views/Helper/ApiHelper.cs +++ b/back/src/Kyoo.Core/Views/Helper/ApiHelper.cs @@ -24,7 +24,6 @@ using System.Linq.Expressions; using System.Reflection; using JetBrains.Annotations; using Kyoo.Abstractions.Models; -using Kyoo.Utils; namespace Kyoo.Core.Api { @@ -53,7 +52,7 @@ namespace Kyoo.Core.Api [NotNull] Expression left, [NotNull] Expression right) { - if (left is not MemberExpression member || ((PropertyInfo)member.Member).PropertyType != typeof(string)) + if (left.Type != typeof(string)) return operand(left, right); MethodCallExpression call = Expression.Call(typeof(string), "Compare", null, left, right); return operand(call, Expression.Constant(0)); diff --git a/back/src/Kyoo.Postgresql/DatabaseContext.cs b/back/src/Kyoo.Postgresql/DatabaseContext.cs index 23437565..a4adb84b 100644 --- a/back/src/Kyoo.Postgresql/DatabaseContext.cs +++ b/back/src/Kyoo.Postgresql/DatabaseContext.cs @@ -41,6 +41,13 @@ namespace Kyoo.Postgresql /// public abstract class DatabaseContext : DbContext { + /// + /// Calculate the MD5 of a string, can only be used in database context. + /// + /// The string to hash + /// The hash + public static string MD5(string str) => throw new NotSupportedException(); + /// /// All collections of Kyoo. See . /// diff --git a/back/src/Kyoo.Postgresql/Kyoo.Postgresql.csproj b/back/src/Kyoo.Postgresql/Kyoo.Postgresql.csproj index 44ff1a9d..9c727fa3 100644 --- a/back/src/Kyoo.Postgresql/Kyoo.Postgresql.csproj +++ b/back/src/Kyoo.Postgresql/Kyoo.Postgresql.csproj @@ -2,6 +2,7 @@ Kyoo.Postgresql Kyoo.Postgresql + enable diff --git a/back/src/Kyoo.Postgresql/PostgresContext.cs b/back/src/Kyoo.Postgresql/PostgresContext.cs index 71ec5117..64fde1ed 100644 --- a/back/src/Kyoo.Postgresql/PostgresContext.cs +++ b/back/src/Kyoo.Postgresql/PostgresContext.cs @@ -18,12 +18,14 @@ using System; using System.Globalization; +using System.Linq; using System.Linq.Expressions; using System.Reflection; using EFCore.NamingConventions.Internal; using Kyoo.Abstractions.Models; using Kyoo.Utils; using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Query.SqlExpressions; using Npgsql; namespace Kyoo.Postgresql @@ -104,6 +106,18 @@ namespace Kyoo.Postgresql modelBuilder.HasPostgresEnum(); modelBuilder.HasPostgresEnum(); + modelBuilder.HasDbFunction(typeof(DatabaseContext).GetMethod(nameof(MD5))) + .HasTranslation(args => + new SqlFunctionExpression( + "md5", + args, + nullable: true, + argumentsPropagateNullability: new[] { false }, + type: args[0].Type, + typeMapping: args[0].TypeMapping + ) + ); + base.OnModelCreating(modelBuilder); }