Cutlery/Kyoo
1
0
Fork 0
mirror of https://github.com/zoriya/Kyoo.git synced 2025-05-24 02:02:36 -04:00
Code Issues Packages Projects Releases Wiki Activity

Add keyset pagination support via generic filters

Browse Source
This commit is contained in:
Zoe Roux 2023-11-24 18:18:34 +01:00
parent 238fdf5d40
commit 253e256458
5 changed files with 211 additions and 162 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
back/src/Kyoo.Abstractions/Models/Utils/Filter.cs
View File

@ -60,6 +60,18 @@ public abstract record Filter
return new Filter<T>.And(acc, filter!);
});
}
public static Filter<T>? Or<T>(params Filter<T>?[] filters)
{
return filters
.Where(x => x != null)
.Aggregate((Filter<T>?)null, (acc, filter) =>
{
if (acc == null)
return filter;
return new Filter<T>.Or(acc, filter!);
});
}
}
public abstract record Filter<T> : Filter
@ -70,9 +82,9 @@ public abstract record Filter<T> : Filter
public record Not(Filter<T> Filter) : Filter<T>;
public record Eq(string Property, object Value) : Filter<T>;
public record Eq(string Property, object? Value) : Filter<T>;
public record Ne(string Property, object Value) : Filter<T>;
public record Ne(string Property, object? Value) : Filter<T>;
public record Gt(string Property, object Value) : Filter<T>;
@ -84,6 +96,15 @@ public abstract record Filter<T> : Filter
public record Has(string Property, object Value) : Filter<T>;
/// <summary>
/// Internal filter used for keyset paginations to resume random sorts.
/// The pseudo sql is md5(seed || table.id) = md5(seed || 'hardCodedId')
/// </summary>
public record EqRandom(string Seed, int ReferenceId) : Filter<T>;
/// <summary>
/// Internal filter used only in EF with hard coded lamdas (used for relations).
/// </summary>
public record Lambda(Expression<Func<T, bool>> Inner) : Filter<T>;
public static class FilterParsers
@ -181,7 +202,7 @@ public abstract record Filter<T> : Filter
private static Parser<Filter<T>> _GetOperationParser(
Parser<object> op,
Func<string, object, Filter<T>> apply,
Func<Type, Parser<object>>? customTypeParser = null)
Func<Type, Parser<object?>>? customTypeParser = null)
{
Parser<string> property = Parse.LetterOrDigit.AtLeastOnce().Text();
@ -194,7 +215,7 @@ public abstract record Filter<T> : Filter
if (propInfo == null)
return ParseHelper.Error<Filter<T>>($"The given filter '{prop}' is invalid.");
Parser<object> value = customTypeParser != null
Parser<object?> value = customTypeParser != null
? customTypeParser(propInfo.PropertyType)
: _GetValueParser(propInfo.PropertyType);
@ -207,12 +228,28 @@ public abstract record Filter<T> : Filter
public static readonly Parser<Filter<T>> Eq = _GetOperationParser(
Parse.IgnoreCase("eq").Or(Parse.String("=")).Token(),
(property, value) => new Eq(property, value)
(property, value) => new Eq(property, value),
(Type type) =>
{
Type? inner = Nullable.GetUnderlyingType(type);
if (inner == null)
return _GetValueParser(type);
return Parse.String("null").Token().Return((object?)null)
.Or(_GetValueParser(inner));
}
);
public static readonly Parser<Filter<T>> Ne = _GetOperationParser(
Parse.IgnoreCase("ne").Or(Parse.String("!=")).Token(),
(property, value) => new Ne(property, value)
(property, value) => new Ne(property, value),
(Type type) =>
{
Type? inner = Nullable.GetUnderlyingType(type);
if (inner == null)
return _GetValueParser(type);
return Parse.String("null").Token().Return((object?)null)
.Or(_GetValueParser(inner));
}
);
public static readonly Parser<Filter<T>> Gt = _GetOperationParser(

2
back/src/Kyoo.Abstractions/Models/Utils/Sort.cs
View File

@ -106,7 +106,7 @@ namespace Kyoo.Abstractions.Controllers
Type[] types = typeof(T).GetCustomAttribute<OneOfAttribute>()?.Types ?? new[] { typeof(T) };
PropertyInfo? property = types
.Select(x => x.GetProperty(key, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance))
.FirstOrDefault();
.FirstOrDefault(x => x != null);
if (property == null)
throw new ValidationException("The given sort key is not valid.");
return new By(property.Name, desendant);

16
back/src/Kyoo.Core/Controllers/Repositories/LibraryItemRepository.cs
View File

@ -114,7 +114,7 @@ namespace Kyoo.Core.Controllers
string ret = sort switch
{
Sort<T>.Default(var value) => ProcessSort(value, config, true),
Sort<T>.By(string key, bool desc) => $"{_Property(key, config)} {(desc ? "desc nulls last" : "asc")}",
Sort<T>.By(string key, bool desc) => $"{_Property(key, config)} {(desc ? "desc" : "asc")}",
Sort<T>.Random(var seed) => $"md5('{seed}' || {_Property("id", config)})",
Sort<T>.Conglomerate(var list) => string.Join(", ", list.Select(x => ProcessSort(x, config, true))),
_ => throw new SwitchExpressionException(),
@ -216,14 +216,18 @@ namespace Kyoo.Core.Controllers
Filter<T>.And(var first, var second) => $"({Process(first)} and {Process(second)})",
Filter<T>.Or(var first, var second) => $"({Process(first)} or {Process(second)})",
Filter<T>.Not(var inner) => $"(not {Process(inner)})",
Filter<T>.Eq(var property, var value) => Format(property, $"= {P(value)}"),
Filter<T>.Ne(var property, var value) => Format(property, $"!= {P(value)}"),
Filter<T>.Eq(var property, var value) when value is null => Format(property, $"is null"),
Filter<T>.Ne(var property, var value) when value is null => Format(property, $"is not null"),
Filter<T>.Eq(var property, var value) => Format(property, $"= {P(value!)}"),
Filter<T>.Ne(var property, var value) => Format(property, $"!= {P(value!)}"),
Filter<T>.Gt(var property, var value) => Format(property, $"> {P(value)}"),
Filter<T>.Ge(var property, var value) => Format(property, $">= {P(value)}"),
Filter<T>.Lt(var property, var value) => Format(property, $"< {P(value)}"),
Filter<T>.Le(var property, var value) => Format(property, $"> {P(value)}"),
Filter<T>.Has(var property, var value) => $"{P(value)} = any({_Property(property, config):raw})",
Filter<T>.EqRandom(var seed, var id) => $"md5({seed} || {config.Select(x => $"{x.Key}.id"):raw}) = md5({seed} || {id.ToString()})",
Filter<T>.Lambda(var lambda) => throw new NotSupportedException(),
_ => throw new NotImplementedException(),
};
}
return $"where {Process(filter)}";
@ -266,6 +270,12 @@ namespace Kyoo.Core.Controllers
{includeJoin:raw}
""");
if (limit.AfterID != null)
{
ILibraryItem reference = await Get(limit.AfterID.Value);
Filter<ILibraryItem>? keysetFilter = RepositoryHelper.KeysetPaginate(sort, reference, !limit.Reverse);
filter = Filter.And(filter, keysetFilter);
}
if (filter != null)
query += ProcessFilter(filter, config);
query += $"order by {ProcessSort(sort, config):raw}";

188
back/src/Kyoo.Core/Controllers/Repositories/LocalRepository.cs
View File

@ -119,6 +119,27 @@ namespace Kyoo.Core.Controllers
ParameterExpression x = Expression.Parameter(typeof(T), "x");
Expression EqRandomHandler(string seed, int refId)
{
MethodInfo concat = typeof(string).GetMethod(nameof(string.Concat), new[] { typeof(string), typeof(string) })!;
Expression id = Expression.Call(Expression.Property(x, "ID"), nameof(int.ToString), null);
Expression xrng = Expression.Call(concat, Expression.Constant(seed), id);
Expression left = Expression.Call(typeof(DatabaseContext), nameof(DatabaseContext.MD5), null, xrng);
Expression right = Expression.Call(typeof(DatabaseContext), nameof(DatabaseContext.MD5), null, Expression.Constant($"{seed}{refId}"));
return Expression.Equal(left, right);
}
BinaryExpression StringCompatibleExpression(
Func<Expression, Expression, BinaryExpression> operand,
Expression left,
Expression right)
{
if (left.Type != typeof(string))
return operand(left, right);
MethodCallExpression call = Expression.Call(typeof(string), "Compare", null, left, right);
return operand(call, Expression.Constant(0));
}
Expression Parse(Filter<T> f)
{
return f switch
@ -128,12 +149,14 @@ namespace Kyoo.Core.Controllers
Filter<T>.Not(var inner) => Expression.Not(Parse(inner)),
Filter<T>.Eq(var property, var value) => Expression.Equal(Expression.Property(x, property), Expression.Constant(value)),
Filter<T>.Ne(var property, var value) => Expression.NotEqual(Expression.Property(x, property), Expression.Constant(value)),
Filter<T>.Gt(var property, var value) => Expression.GreaterThan(Expression.Property(x, property), Expression.Constant(value)),
Filter<T>.Ge(var property, var value) => Expression.GreaterThanOrEqual(Expression.Property(x, property), Expression.Constant(value)),
Filter<T>.Lt(var property, var value) => Expression.LessThan(Expression.Property(x, property), Expression.Constant(value)),
Filter<T>.Le(var property, var value) => Expression.LessThanOrEqual(Expression.Property(x, property), Expression.Constant(value)),
Filter<T>.Gt(var property, var value) => StringCompatibleExpression(Expression.GreaterThan, Expression.Property(x, property), Expression.Constant(value)),
Filter<T>.Ge(var property, var value) => StringCompatibleExpression(Expression.GreaterThanOrEqual, Expression.Property(x, property), Expression.Constant(value)),
Filter<T>.Lt(var property, var value) => StringCompatibleExpression(Expression.LessThan, Expression.Property(x, property), Expression.Constant(value)),
Filter<T>.Le(var property, var value) => StringCompatibleExpression(Expression.LessThanOrEqual, Expression.Property(x, property), Expression.Constant(value)),
Filter<T>.Has(var property, var value) => Expression.Call(typeof(Enumerable), "Contains", new[] { value.GetType() }, Expression.Property(x, property), Expression.Constant(value)),
Filter<T>.EqRandom(var seed, var refId) => EqRandomHandler(seed, refId),
Filter<T>.Lambda(var lambda) => ExpressionArgumentReplacer.ReplaceParams(lambda.Body, lambda.Parameters, x),
_ => throw new NotImplementedException(),
};
}
@ -141,148 +164,6 @@ namespace Kyoo.Core.Controllers
return Expression.Lambda<Func<T, bool>>(body, x);
}
private static Func<Expression, Expression, BinaryExpression> _GetComparisonExpression(
bool desc,
bool next,
bool orEqual)
{
bool greaterThan = desc ^ next;
return orEqual
? (greaterThan ? Expression.GreaterThanOrEqual : Expression.LessThanOrEqual)
: (greaterThan ? Expression.GreaterThan : Expression.LessThan);
}
private record SortIndicator(string Key, bool Desc, string? Seed);
/// <summary>
/// Create a filter (where) expression on the query to skip everything before/after the referenceID.
/// The generalized expression for this in pseudocode is:
/// (x > a) OR
/// (x = a AND y > b) OR
/// (x = a AND y = b AND z > c) OR...
///
/// Of course, this will be a bit more complex when ASC and DESC are mixed.
/// Assume x is ASC, y is DESC, and z is ASC:
/// (x > a) OR
/// (x = a AND y &lt; b) OR
/// (x = a AND y = b AND z > c) OR...
/// </summary>
/// <param name="sort">How items are sorted in the query</param>
/// <param name="reference">The reference item (the AfterID query)</param>
/// <param name="next">True if the following page should be returned, false for the previous.</param>
/// <returns>An expression ready to be added to a Where close of a sorted query to handle the AfterID</returns>
protected Expression<Func<T, bool>> KeysetPaginate(
Sort<T>? sort,
T reference,
bool next = true)
{
sort ??= new Sort<T>.Default();
// x =>
ParameterExpression x = Expression.Parameter(typeof(T), "x");
ConstantExpression referenceC = Expression.Constant(reference, typeof(T));
void GetRandomSortKeys(string seed, out Expression left, out Expression right)
{
MethodInfo concat = typeof(string).GetMethod(nameof(string.Concat), new[] { typeof(string), typeof(string) })!;
Expression id = Expression.Call(Expression.Property(x, "ID"), nameof(int.ToString), null);
Expression xrng = Expression.Call(concat, Expression.Constant(seed), id);
right = Expression.Call(typeof(DatabaseContext), nameof(DatabaseContext.MD5), null, Expression.Constant($"{seed}{reference.Id}"));
left = Expression.Call(typeof(DatabaseContext), nameof(DatabaseContext.MD5), null, xrng);
}
IEnumerable<SortIndicator> GetSortsBy(Sort<T> sort)
{
return sort switch
{
Sort<T>.Default(var value) => GetSortsBy(value),
Sort<T>.By @sortBy => new[] { new SortIndicator(sortBy.Key, sortBy.Desendant, null) },
Sort<T>.Conglomerate(var list) => list.SelectMany(GetSortsBy),
Sort<T>.Random(var seed) => new[] { new SortIndicator("random", false, seed.ToString()) },
_ => Array.Empty<SortIndicator>(),
};
}
// Don't forget that every sorts must end with a ID sort (to differentiate equalities).
IEnumerable<SortIndicator> sorts = GetSortsBy(sort)
.Append(new SortIndicator("Id", false, null));
BinaryExpression? filter = null;
List<SortIndicator> previousSteps = new();
// TODO: Add an outer query >= for perf
// PERF: See https://use-the-index-luke.com/sql/partial-results/fetch-next-page#sb-equivalent-logic
foreach ((string key, bool desc, string? seed) in sorts)
{
BinaryExpression? compare = null;
PropertyInfo? property = key != "random"
? typeof(T).GetProperty(key)
: null;
// Comparing a value with null always return false so we short opt < > comparisons with null.
if (property != null && property.GetValue(reference) == null)
{
previousSteps.Add(new SortIndicator(key, desc, seed));
continue;
}
// Create all the equality statements for previous sorts.
foreach ((string pKey, bool pDesc, string? pSeed) in previousSteps)
{
BinaryExpression pcompare;
if (pSeed == null)
{
pcompare = Expression.Equal(
Expression.Property(x, pKey),
Expression.Property(referenceC, pKey)
);
}
else
{
GetRandomSortKeys(pSeed, out Expression left, out Expression right);
pcompare = Expression.Equal(left, right);
}
compare = compare != null
? Expression.AndAlso(compare, pcompare)
: pcompare;
}
// Create the last comparison of the statement.
Func<Expression, Expression, BinaryExpression> comparer = _GetComparisonExpression(desc, next, false);
Expression xkey;
Expression rkey;
if (seed == null)
{
xkey = Expression.Property(x, key);
rkey = Expression.Property(referenceC, key);
}
else
GetRandomSortKeys(seed, out xkey, out rkey);
BinaryExpression lastCompare = null;//ApiHelper.StringCompatibleExpression(comparer, xkey, rkey);
// Comparing a value with null always return false for nulls so we must add nulls to the results manually.
// Postgres sorts them after values so we will do the same
// We only add this condition if the column type is nullable
if (property != null && Nullable.GetUnderlyingType(property.PropertyType) != null)
{
BinaryExpression equalNull = Expression.Equal(xkey, Expression.Constant(null));
lastCompare = Expression.OrElse(lastCompare, equalNull);
}
compare = compare != null
? Expression.AndAlso(compare, lastCompare)
: lastCompare;
filter = filter != null
? Expression.OrElse(filter, compare)
: compare;
previousSteps.Add(new SortIndicator(key, desc, seed));
}
return Expression.Lambda<Func<T, bool>>(filter!, x);
}
protected IQueryable<T> AddIncludes(IQueryable<T> query, Include<T>? include)
{
if (include == null)
@ -386,34 +267,37 @@ namespace Kyoo.Core.Controllers
Include<T>? include = default,
Pagination limit = default)
{
return ApplyFilters(Database.Set<T>(), ParseFilter(filter), sort, limit, include);
return ApplyFilters(Database.Set<T>(), filter, sort, limit, include);
}
/// <summary>
/// Apply filters to a query to ease sort, pagination and where queries for resources of this repository
/// </summary>
/// <param name="query">The base query to filter.</param>
/// <param name="where">An expression to filter based on arbitrary conditions</param>
/// <param name="filter">An expression to filter based on arbitrary conditions</param>
/// <param name="sort">The sort settings (sort order and sort by)</param>
/// <param name="limit">Pagination information (where to start and how many to get)</param>
/// <param name="include">Related fields to also load with this query.</param>
/// <returns>The filtered query</returns>
protected async Task<ICollection<T>> ApplyFilters(IQueryable<T> query,
Expression<Func<T, bool>>? where = null,
Filter<T>? filter = null,
Sort<T>? sort = default,
Pagination limit = default,
Include<T>? include = default)
{
query = AddIncludes(query, include);
query = Sort(query, sort);
if (where != null)
query = query.Where(where);
if (limit.AfterID != null)
{
T reference = await Get(limit.AfterID.Value);
query = query.Where(KeysetPaginate(sort, reference, !limit.Reverse));
Filter<T>? keysetFilter = RepositoryHelper.KeysetPaginate(sort, reference, !limit.Reverse);
filter = Filter.And(filter, keysetFilter);
Console.WriteLine(filter);
}
if (filter != null)
query = query.Where(ParseFilter(filter));
if (limit.Reverse)
query = query.Reverse();
if (limit.Limit > 0)

118
back/src/Kyoo.Core/Controllers/Repositories/RepositoryHelper.cs Normal file
View File

@ -0,0 +1,118 @@
// Kyoo - A portable and vast media library solution.
// Copyright (c) Kyoo.
//
// See AUTHORS.md and LICENSE file in the project root for full license information.
//
// Kyoo is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// any later version.
//
// Kyoo is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Kyoo. If not, see <https://www.gnu.org/licenses/>.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using Kyoo.Abstractions.Controllers;
using Kyoo.Abstractions.Models;
using Kyoo.Abstractions.Models.Attributes;
using Kyoo.Abstractions.Models.Utils;
namespace Kyoo.Core;
public class RepositoryHelper
{
private record SortIndicator(string Key, bool Desc, string? Seed);
/// <summary>
/// Create a filter (where) expression on the query to skip everything before/after the referenceID.
/// The generalized expression for this in pseudocode is:
/// (x > a) OR
/// (x = a AND y > b) OR
/// (x = a AND y = b AND z > c) OR...
///
/// Of course, this will be a bit more complex when ASC and DESC are mixed.
/// Assume x is ASC, y is DESC, and z is ASC:
/// (x > a) OR
/// (x = a AND y &lt; b) OR
/// (x = a AND y = b AND z > c) OR...
/// </summary>
/// <param name="sort">How items are sorted in the query</param>
/// <param name="reference">The reference item (the AfterID query)</param>
/// <param name="next">True if the following page should be returned, false for the previous.</param>
/// <typeparam name="T">The type to paginate for.</typeparam>
/// <returns>An expression ready to be added to a Where close of a sorted query to handle the AfterID</returns>
public static Filter<T>? KeysetPaginate<T>(Sort<T>? sort, T reference, bool next = true)
where T : class, IResource, IQuery
{
sort ??= new Sort<T>.Default();
IEnumerable<SortIndicator> GetSortsBy(Sort<T> sort)
{
return sort switch
{
Sort<T>.Default(var value) => GetSortsBy(value),
Sort<T>.By @sortBy => new[] { new SortIndicator(sortBy.Key, sortBy.Desendant, null) },
Sort<T>.Conglomerate(var list) => list.SelectMany(GetSortsBy),
Sort<T>.Random(var seed) => new[] { new SortIndicator("random", false, seed.ToString()) },
_ => Array.Empty<SortIndicator>(),
};
}
// Don't forget that every sorts must end with a ID sort (to differentiate equalities).
IEnumerable<SortIndicator> sorts = GetSortsBy(sort)
.Append(new SortIndicator("Id", false, null));
Filter<T>? ret = null;
List<SortIndicator> previousSteps = new();
// TODO: Add an outer query >= for perf
// PERF: See https://use-the-index-luke.com/sql/partial-results/fetch-next-page#sb-equivalent-logic
foreach ((string key, bool desc, string? seed) in sorts)
{
object? value = reference.GetType().GetProperty(key)?.GetValue(reference);
// Comparing a value with null always return false so we short opt < > comparisons with null.
if (key != "random" && value == null)
{
previousSteps.Add(new SortIndicator(key, desc, seed));
continue;
}
// Create all the equality statements for previous sorts.
Filter<T>? equals = null;
foreach ((string pKey, bool pDesc, string? pSeed) in previousSteps)
{
Filter<T> pEquals = pSeed == null
? new Filter<T>.Eq(pKey, reference.GetType().GetProperty(pKey)?.GetValue(reference))
: new Filter<T>.EqRandom(pSeed, reference.Id);
equals = Filter.And(equals, pEquals);
}
bool greaterThan = desc ^ next;
Func<string, object, Filter<T>> comparer = greaterThan
? (prop, val) => new Filter<T>.Gt(prop, val)
: (prop, val) => new Filter<T>.Lt(prop, val);
Filter<T> last = seed == null
? comparer(key, value!)
: new Filter<T>.EqRandom(seed, reference.Id);
if (key != "random")
{
Type[] types = typeof(T).GetCustomAttribute<OneOfAttribute>()?.Types ?? new[] { typeof(T) };
PropertyInfo property = types.Select(x => x.GetProperty(key)!).First(x => x != null);
if (Nullable.GetUnderlyingType(property.PropertyType) != null)
last = new Filter<T>.Or(last, new Filter<T>.Eq(key, null));
}
ret = Filter.Or(ret, Filter.And(equals, last));
previousSteps.Add(new SortIndicator(key, desc, seed));
}
return ret;
}
}