From 253e256458ce9b1c2ddfec952fc9cbf653f6290f Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Fri, 24 Nov 2023 18:18:34 +0100 Subject: [PATCH] Add keyset pagination support via generic filters --- .../Kyoo.Abstractions/Models/Utils/Filter.cs | 49 ++++- .../Kyoo.Abstractions/Models/Utils/Sort.cs | 2 +- .../Repositories/LibraryItemRepository.cs | 16 +- .../Repositories/LocalRepository.cs | 188 ++++-------------- .../Repositories/RepositoryHelper.cs | 118 +++++++++++ 5 files changed, 211 insertions(+), 162 deletions(-) create mode 100644 back/src/Kyoo.Core/Controllers/Repositories/RepositoryHelper.cs diff --git a/back/src/Kyoo.Abstractions/Models/Utils/Filter.cs b/back/src/Kyoo.Abstractions/Models/Utils/Filter.cs index ed90f918..4de934a0 100644 --- a/back/src/Kyoo.Abstractions/Models/Utils/Filter.cs +++ b/back/src/Kyoo.Abstractions/Models/Utils/Filter.cs @@ -60,6 +60,18 @@ public abstract record Filter return new Filter.And(acc, filter!); }); } + + public static Filter? Or(params Filter?[] filters) + { + return filters + .Where(x => x != null) + .Aggregate((Filter?)null, (acc, filter) => + { + if (acc == null) + return filter; + return new Filter.Or(acc, filter!); + }); + } } public abstract record Filter : Filter @@ -70,9 +82,9 @@ public abstract record Filter : Filter public record Not(Filter Filter) : Filter; - public record Eq(string Property, object Value) : Filter; + public record Eq(string Property, object? Value) : Filter; - public record Ne(string Property, object Value) : Filter; + public record Ne(string Property, object? Value) : Filter; public record Gt(string Property, object Value) : Filter; @@ -84,6 +96,15 @@ public abstract record Filter : Filter public record Has(string Property, object Value) : Filter; + /// + /// Internal filter used for keyset paginations to resume random sorts. + /// The pseudo sql is md5(seed || table.id) = md5(seed || 'hardCodedId') + /// + public record EqRandom(string Seed, int ReferenceId) : Filter; + + /// + /// Internal filter used only in EF with hard coded lamdas (used for relations). + /// public record Lambda(Expression> Inner) : Filter; public static class FilterParsers @@ -181,7 +202,7 @@ public abstract record Filter : Filter private static Parser> _GetOperationParser( Parser op, Func> apply, - Func>? customTypeParser = null) + Func>? customTypeParser = null) { Parser property = Parse.LetterOrDigit.AtLeastOnce().Text(); @@ -194,7 +215,7 @@ public abstract record Filter : Filter if (propInfo == null) return ParseHelper.Error>($"The given filter '{prop}' is invalid."); - Parser value = customTypeParser != null + Parser value = customTypeParser != null ? customTypeParser(propInfo.PropertyType) : _GetValueParser(propInfo.PropertyType); @@ -207,12 +228,28 @@ public abstract record Filter : Filter public static readonly Parser> Eq = _GetOperationParser( Parse.IgnoreCase("eq").Or(Parse.String("=")).Token(), - (property, value) => new Eq(property, value) + (property, value) => new Eq(property, value), + (Type type) => + { + Type? inner = Nullable.GetUnderlyingType(type); + if (inner == null) + return _GetValueParser(type); + return Parse.String("null").Token().Return((object?)null) + .Or(_GetValueParser(inner)); + } ); public static readonly Parser> Ne = _GetOperationParser( Parse.IgnoreCase("ne").Or(Parse.String("!=")).Token(), - (property, value) => new Ne(property, value) + (property, value) => new Ne(property, value), + (Type type) => + { + Type? inner = Nullable.GetUnderlyingType(type); + if (inner == null) + return _GetValueParser(type); + return Parse.String("null").Token().Return((object?)null) + .Or(_GetValueParser(inner)); + } ); public static readonly Parser> Gt = _GetOperationParser( diff --git a/back/src/Kyoo.Abstractions/Models/Utils/Sort.cs b/back/src/Kyoo.Abstractions/Models/Utils/Sort.cs index 482be649..c8e26c6b 100644 --- a/back/src/Kyoo.Abstractions/Models/Utils/Sort.cs +++ b/back/src/Kyoo.Abstractions/Models/Utils/Sort.cs @@ -106,7 +106,7 @@ namespace Kyoo.Abstractions.Controllers Type[] types = typeof(T).GetCustomAttribute()?.Types ?? new[] { typeof(T) }; PropertyInfo? property = types .Select(x => x.GetProperty(key, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance)) - .FirstOrDefault(); + .FirstOrDefault(x => x != null); if (property == null) throw new ValidationException("The given sort key is not valid."); return new By(property.Name, desendant); diff --git a/back/src/Kyoo.Core/Controllers/Repositories/LibraryItemRepository.cs b/back/src/Kyoo.Core/Controllers/Repositories/LibraryItemRepository.cs index dd4b8cdd..6b22f928 100644 --- a/back/src/Kyoo.Core/Controllers/Repositories/LibraryItemRepository.cs +++ b/back/src/Kyoo.Core/Controllers/Repositories/LibraryItemRepository.cs @@ -114,7 +114,7 @@ namespace Kyoo.Core.Controllers string ret = sort switch { Sort.Default(var value) => ProcessSort(value, config, true), - Sort.By(string key, bool desc) => $"{_Property(key, config)} {(desc ? "desc nulls last" : "asc")}", + Sort.By(string key, bool desc) => $"{_Property(key, config)} {(desc ? "desc" : "asc")}", Sort.Random(var seed) => $"md5('{seed}' || {_Property("id", config)})", Sort.Conglomerate(var list) => string.Join(", ", list.Select(x => ProcessSort(x, config, true))), _ => throw new SwitchExpressionException(), @@ -216,14 +216,18 @@ namespace Kyoo.Core.Controllers Filter.And(var first, var second) => $"({Process(first)} and {Process(second)})", Filter.Or(var first, var second) => $"({Process(first)} or {Process(second)})", Filter.Not(var inner) => $"(not {Process(inner)})", - Filter.Eq(var property, var value) => Format(property, $"= {P(value)}"), - Filter.Ne(var property, var value) => Format(property, $"!= {P(value)}"), + Filter.Eq(var property, var value) when value is null => Format(property, $"is null"), + Filter.Ne(var property, var value) when value is null => Format(property, $"is not null"), + Filter.Eq(var property, var value) => Format(property, $"= {P(value!)}"), + Filter.Ne(var property, var value) => Format(property, $"!= {P(value!)}"), Filter.Gt(var property, var value) => Format(property, $"> {P(value)}"), Filter.Ge(var property, var value) => Format(property, $">= {P(value)}"), Filter.Lt(var property, var value) => Format(property, $"< {P(value)}"), Filter.Le(var property, var value) => Format(property, $"> {P(value)}"), Filter.Has(var property, var value) => $"{P(value)} = any({_Property(property, config):raw})", + Filter.EqRandom(var seed, var id) => $"md5({seed} || {config.Select(x => $"{x.Key}.id"):raw}) = md5({seed} || {id.ToString()})", Filter.Lambda(var lambda) => throw new NotSupportedException(), + _ => throw new NotImplementedException(), }; } return $"where {Process(filter)}"; @@ -266,6 +270,12 @@ namespace Kyoo.Core.Controllers {includeJoin:raw} """); + if (limit.AfterID != null) + { + ILibraryItem reference = await Get(limit.AfterID.Value); + Filter? keysetFilter = RepositoryHelper.KeysetPaginate(sort, reference, !limit.Reverse); + filter = Filter.And(filter, keysetFilter); + } if (filter != null) query += ProcessFilter(filter, config); query += $"order by {ProcessSort(sort, config):raw}"; diff --git a/back/src/Kyoo.Core/Controllers/Repositories/LocalRepository.cs b/back/src/Kyoo.Core/Controllers/Repositories/LocalRepository.cs index 135d6054..0d7a6759 100644 --- a/back/src/Kyoo.Core/Controllers/Repositories/LocalRepository.cs +++ b/back/src/Kyoo.Core/Controllers/Repositories/LocalRepository.cs @@ -119,6 +119,27 @@ namespace Kyoo.Core.Controllers ParameterExpression x = Expression.Parameter(typeof(T), "x"); + Expression EqRandomHandler(string seed, int refId) + { + MethodInfo concat = typeof(string).GetMethod(nameof(string.Concat), new[] { typeof(string), typeof(string) })!; + Expression id = Expression.Call(Expression.Property(x, "ID"), nameof(int.ToString), null); + Expression xrng = Expression.Call(concat, Expression.Constant(seed), id); + Expression left = Expression.Call(typeof(DatabaseContext), nameof(DatabaseContext.MD5), null, xrng); + Expression right = Expression.Call(typeof(DatabaseContext), nameof(DatabaseContext.MD5), null, Expression.Constant($"{seed}{refId}")); + return Expression.Equal(left, right); + } + + BinaryExpression StringCompatibleExpression( + Func operand, + Expression left, + Expression right) + { + if (left.Type != typeof(string)) + return operand(left, right); + MethodCallExpression call = Expression.Call(typeof(string), "Compare", null, left, right); + return operand(call, Expression.Constant(0)); + } + Expression Parse(Filter f) { return f switch @@ -128,12 +149,14 @@ namespace Kyoo.Core.Controllers Filter.Not(var inner) => Expression.Not(Parse(inner)), Filter.Eq(var property, var value) => Expression.Equal(Expression.Property(x, property), Expression.Constant(value)), Filter.Ne(var property, var value) => Expression.NotEqual(Expression.Property(x, property), Expression.Constant(value)), - Filter.Gt(var property, var value) => Expression.GreaterThan(Expression.Property(x, property), Expression.Constant(value)), - Filter.Ge(var property, var value) => Expression.GreaterThanOrEqual(Expression.Property(x, property), Expression.Constant(value)), - Filter.Lt(var property, var value) => Expression.LessThan(Expression.Property(x, property), Expression.Constant(value)), - Filter.Le(var property, var value) => Expression.LessThanOrEqual(Expression.Property(x, property), Expression.Constant(value)), + Filter.Gt(var property, var value) => StringCompatibleExpression(Expression.GreaterThan, Expression.Property(x, property), Expression.Constant(value)), + Filter.Ge(var property, var value) => StringCompatibleExpression(Expression.GreaterThanOrEqual, Expression.Property(x, property), Expression.Constant(value)), + Filter.Lt(var property, var value) => StringCompatibleExpression(Expression.LessThan, Expression.Property(x, property), Expression.Constant(value)), + Filter.Le(var property, var value) => StringCompatibleExpression(Expression.LessThanOrEqual, Expression.Property(x, property), Expression.Constant(value)), Filter.Has(var property, var value) => Expression.Call(typeof(Enumerable), "Contains", new[] { value.GetType() }, Expression.Property(x, property), Expression.Constant(value)), + Filter.EqRandom(var seed, var refId) => EqRandomHandler(seed, refId), Filter.Lambda(var lambda) => ExpressionArgumentReplacer.ReplaceParams(lambda.Body, lambda.Parameters, x), + _ => throw new NotImplementedException(), }; } @@ -141,148 +164,6 @@ namespace Kyoo.Core.Controllers return Expression.Lambda>(body, x); } - private static Func _GetComparisonExpression( - bool desc, - bool next, - bool orEqual) - { - bool greaterThan = desc ^ next; - - return orEqual - ? (greaterThan ? Expression.GreaterThanOrEqual : Expression.LessThanOrEqual) - : (greaterThan ? Expression.GreaterThan : Expression.LessThan); - } - - private record SortIndicator(string Key, bool Desc, string? Seed); - - /// - /// Create a filter (where) expression on the query to skip everything before/after the referenceID. - /// The generalized expression for this in pseudocode is: - /// (x > a) OR - /// (x = a AND y > b) OR - /// (x = a AND y = b AND z > c) OR... - /// - /// Of course, this will be a bit more complex when ASC and DESC are mixed. - /// Assume x is ASC, y is DESC, and z is ASC: - /// (x > a) OR - /// (x = a AND y < b) OR - /// (x = a AND y = b AND z > c) OR... - /// - /// How items are sorted in the query - /// The reference item (the AfterID query) - /// True if the following page should be returned, false for the previous. - /// An expression ready to be added to a Where close of a sorted query to handle the AfterID - protected Expression> KeysetPaginate( - Sort? sort, - T reference, - bool next = true) - { - sort ??= new Sort.Default(); - - // x => - ParameterExpression x = Expression.Parameter(typeof(T), "x"); - ConstantExpression referenceC = Expression.Constant(reference, typeof(T)); - - void GetRandomSortKeys(string seed, out Expression left, out Expression right) - { - MethodInfo concat = typeof(string).GetMethod(nameof(string.Concat), new[] { typeof(string), typeof(string) })!; - Expression id = Expression.Call(Expression.Property(x, "ID"), nameof(int.ToString), null); - Expression xrng = Expression.Call(concat, Expression.Constant(seed), id); - right = Expression.Call(typeof(DatabaseContext), nameof(DatabaseContext.MD5), null, Expression.Constant($"{seed}{reference.Id}")); - left = Expression.Call(typeof(DatabaseContext), nameof(DatabaseContext.MD5), null, xrng); - } - - IEnumerable GetSortsBy(Sort sort) - { - return sort switch - { - Sort.Default(var value) => GetSortsBy(value), - Sort.By @sortBy => new[] { new SortIndicator(sortBy.Key, sortBy.Desendant, null) }, - Sort.Conglomerate(var list) => list.SelectMany(GetSortsBy), - Sort.Random(var seed) => new[] { new SortIndicator("random", false, seed.ToString()) }, - _ => Array.Empty(), - }; - } - - // Don't forget that every sorts must end with a ID sort (to differentiate equalities). - IEnumerable sorts = GetSortsBy(sort) - .Append(new SortIndicator("Id", false, null)); - - BinaryExpression? filter = null; - List previousSteps = new(); - // TODO: Add an outer query >= for perf - // PERF: See https://use-the-index-luke.com/sql/partial-results/fetch-next-page#sb-equivalent-logic - foreach ((string key, bool desc, string? seed) in sorts) - { - BinaryExpression? compare = null; - PropertyInfo? property = key != "random" - ? typeof(T).GetProperty(key) - : null; - - // Comparing a value with null always return false so we short opt < > comparisons with null. - if (property != null && property.GetValue(reference) == null) - { - previousSteps.Add(new SortIndicator(key, desc, seed)); - continue; - } - - // Create all the equality statements for previous sorts. - foreach ((string pKey, bool pDesc, string? pSeed) in previousSteps) - { - BinaryExpression pcompare; - - if (pSeed == null) - { - pcompare = Expression.Equal( - Expression.Property(x, pKey), - Expression.Property(referenceC, pKey) - ); - } - else - { - GetRandomSortKeys(pSeed, out Expression left, out Expression right); - pcompare = Expression.Equal(left, right); - } - compare = compare != null - ? Expression.AndAlso(compare, pcompare) - : pcompare; - } - - // Create the last comparison of the statement. - Func comparer = _GetComparisonExpression(desc, next, false); - Expression xkey; - Expression rkey; - if (seed == null) - { - xkey = Expression.Property(x, key); - rkey = Expression.Property(referenceC, key); - } - else - GetRandomSortKeys(seed, out xkey, out rkey); - BinaryExpression lastCompare = null;//ApiHelper.StringCompatibleExpression(comparer, xkey, rkey); - - // Comparing a value with null always return false for nulls so we must add nulls to the results manually. - // Postgres sorts them after values so we will do the same - // We only add this condition if the column type is nullable - if (property != null && Nullable.GetUnderlyingType(property.PropertyType) != null) - { - BinaryExpression equalNull = Expression.Equal(xkey, Expression.Constant(null)); - lastCompare = Expression.OrElse(lastCompare, equalNull); - } - - compare = compare != null - ? Expression.AndAlso(compare, lastCompare) - : lastCompare; - - filter = filter != null - ? Expression.OrElse(filter, compare) - : compare; - - previousSteps.Add(new SortIndicator(key, desc, seed)); - } - return Expression.Lambda>(filter!, x); - } - protected IQueryable AddIncludes(IQueryable query, Include? include) { if (include == null) @@ -386,34 +267,37 @@ namespace Kyoo.Core.Controllers Include? include = default, Pagination limit = default) { - return ApplyFilters(Database.Set(), ParseFilter(filter), sort, limit, include); + return ApplyFilters(Database.Set(), filter, sort, limit, include); } /// /// Apply filters to a query to ease sort, pagination and where queries for resources of this repository /// /// The base query to filter. - /// An expression to filter based on arbitrary conditions + /// An expression to filter based on arbitrary conditions /// The sort settings (sort order and sort by) /// Pagination information (where to start and how many to get) /// Related fields to also load with this query. /// The filtered query protected async Task> ApplyFilters(IQueryable query, - Expression>? where = null, + Filter? filter = null, Sort? sort = default, Pagination limit = default, Include? include = default) { query = AddIncludes(query, include); query = Sort(query, sort); - if (where != null) - query = query.Where(where); if (limit.AfterID != null) { T reference = await Get(limit.AfterID.Value); - query = query.Where(KeysetPaginate(sort, reference, !limit.Reverse)); + Filter? keysetFilter = RepositoryHelper.KeysetPaginate(sort, reference, !limit.Reverse); + filter = Filter.And(filter, keysetFilter); + Console.WriteLine(filter); } + if (filter != null) + query = query.Where(ParseFilter(filter)); + if (limit.Reverse) query = query.Reverse(); if (limit.Limit > 0) diff --git a/back/src/Kyoo.Core/Controllers/Repositories/RepositoryHelper.cs b/back/src/Kyoo.Core/Controllers/Repositories/RepositoryHelper.cs new file mode 100644 index 00000000..e33af89e --- /dev/null +++ b/back/src/Kyoo.Core/Controllers/Repositories/RepositoryHelper.cs @@ -0,0 +1,118 @@ +// Kyoo - A portable and vast media library solution. +// Copyright (c) Kyoo. +// +// See AUTHORS.md and LICENSE file in the project root for full license information. +// +// Kyoo is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// any later version. +// +// Kyoo is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with Kyoo. If not, see . + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using Kyoo.Abstractions.Controllers; +using Kyoo.Abstractions.Models; +using Kyoo.Abstractions.Models.Attributes; +using Kyoo.Abstractions.Models.Utils; + +namespace Kyoo.Core; + +public class RepositoryHelper +{ + private record SortIndicator(string Key, bool Desc, string? Seed); + + /// + /// Create a filter (where) expression on the query to skip everything before/after the referenceID. + /// The generalized expression for this in pseudocode is: + /// (x > a) OR + /// (x = a AND y > b) OR + /// (x = a AND y = b AND z > c) OR... + /// + /// Of course, this will be a bit more complex when ASC and DESC are mixed. + /// Assume x is ASC, y is DESC, and z is ASC: + /// (x > a) OR + /// (x = a AND y < b) OR + /// (x = a AND y = b AND z > c) OR... + /// + /// How items are sorted in the query + /// The reference item (the AfterID query) + /// True if the following page should be returned, false for the previous. + /// The type to paginate for. + /// An expression ready to be added to a Where close of a sorted query to handle the AfterID + public static Filter? KeysetPaginate(Sort? sort, T reference, bool next = true) + where T : class, IResource, IQuery + { + sort ??= new Sort.Default(); + + IEnumerable GetSortsBy(Sort sort) + { + return sort switch + { + Sort.Default(var value) => GetSortsBy(value), + Sort.By @sortBy => new[] { new SortIndicator(sortBy.Key, sortBy.Desendant, null) }, + Sort.Conglomerate(var list) => list.SelectMany(GetSortsBy), + Sort.Random(var seed) => new[] { new SortIndicator("random", false, seed.ToString()) }, + _ => Array.Empty(), + }; + } + + // Don't forget that every sorts must end with a ID sort (to differentiate equalities). + IEnumerable sorts = GetSortsBy(sort) + .Append(new SortIndicator("Id", false, null)); + + Filter? ret = null; + List previousSteps = new(); + // TODO: Add an outer query >= for perf + // PERF: See https://use-the-index-luke.com/sql/partial-results/fetch-next-page#sb-equivalent-logic + foreach ((string key, bool desc, string? seed) in sorts) + { + object? value = reference.GetType().GetProperty(key)?.GetValue(reference); + // Comparing a value with null always return false so we short opt < > comparisons with null. + if (key != "random" && value == null) + { + previousSteps.Add(new SortIndicator(key, desc, seed)); + continue; + } + + // Create all the equality statements for previous sorts. + Filter? equals = null; + foreach ((string pKey, bool pDesc, string? pSeed) in previousSteps) + { + Filter pEquals = pSeed == null + ? new Filter.Eq(pKey, reference.GetType().GetProperty(pKey)?.GetValue(reference)) + : new Filter.EqRandom(pSeed, reference.Id); + equals = Filter.And(equals, pEquals); + } + + bool greaterThan = desc ^ next; + Func> comparer = greaterThan + ? (prop, val) => new Filter.Gt(prop, val) + : (prop, val) => new Filter.Lt(prop, val); + Filter last = seed == null + ? comparer(key, value!) + : new Filter.EqRandom(seed, reference.Id); + + if (key != "random") + { + Type[] types = typeof(T).GetCustomAttribute()?.Types ?? new[] { typeof(T) }; + PropertyInfo property = types.Select(x => x.GetProperty(key)!).First(x => x != null); + if (Nullable.GetUnderlyingType(property.PropertyType) != null) + last = new Filter.Or(last, new Filter.Eq(key, null)); + } + + ret = Filter.Or(ret, Filter.And(equals, last)); + previousSteps.Add(new SortIndicator(key, desc, seed)); + } + return ret; + } +}