diff --git a/.pg_format b/.pg_format index 5d346b91..882260a2 100644 --- a/.pg_format +++ b/.pg_format @@ -2,3 +2,4 @@ tabs=1 function-case=1 #lowercase keyword-case=1 type-case=1 +no-space-function=1 diff --git a/back/src/Directory.Build.props b/back/src/Directory.Build.props index 53bd9b50..9934d2e0 100644 --- a/back/src/Directory.Build.props +++ b/back/src/Directory.Build.props @@ -46,7 +46,7 @@ $(MSBuildThisFileDirectory)../Kyoo.ruleset - 1591;1305;8618 + 1591;1305;8618;SYSLIB1045 diff --git a/back/src/Kyoo.Abstractions/Controllers/IRepository.cs b/back/src/Kyoo.Abstractions/Controllers/IRepository.cs index e3fea697..c6300930 100644 --- a/back/src/Kyoo.Abstractions/Controllers/IRepository.cs +++ b/back/src/Kyoo.Abstractions/Controllers/IRepository.cs @@ -113,7 +113,7 @@ namespace Kyoo.Abstractions.Controllers Task> GetAll(Filter? filter = null, Sort? sort = default, Include? include = default, - Pagination limit = default); + Pagination? limit = default); /// /// Get the number of resources that match the filter's predicate. diff --git a/back/src/Kyoo.Abstractions/Models/Resources/Episode.cs b/back/src/Kyoo.Abstractions/Models/Resources/Episode.cs index d46f32e5..de0deb0a 100644 --- a/back/src/Kyoo.Abstractions/Models/Resources/Episode.cs +++ b/back/src/Kyoo.Abstractions/Models/Resources/Episode.cs @@ -174,7 +174,7 @@ namespace Kyoo.Abstractions.Models // language=PostgreSQL Sql = """ select - "pe".*, + "pe".* -- Episode as pe from episodes as "pe" where @@ -210,7 +210,7 @@ namespace Kyoo.Abstractions.Models // language=PostgreSQL Sql = """ select - "ne".*, + "ne".* -- Episode as ne from episodes as "ne" where diff --git a/back/src/Kyoo.Abstractions/Models/Resources/Show.cs b/back/src/Kyoo.Abstractions/Models/Resources/Show.cs index d6dd84e2..4894731b 100644 --- a/back/src/Kyoo.Abstractions/Models/Resources/Show.cs +++ b/back/src/Kyoo.Abstractions/Models/Resources/Show.cs @@ -158,7 +158,7 @@ namespace Kyoo.Abstractions.Models // language=PostgreSQL Sql = """ select - "fe".* + "fe".* -- Episode as fe from ( select e.*, diff --git a/back/src/Kyoo.Core/Controllers/Repositories/DapperHelper.cs b/back/src/Kyoo.Core/Controllers/Repositories/DapperHelper.cs new file mode 100644 index 00000000..f1776144 --- /dev/null +++ b/back/src/Kyoo.Core/Controllers/Repositories/DapperHelper.cs @@ -0,0 +1,245 @@ +// Kyoo - A portable and vast media library solution. +// Copyright (c) Kyoo. +// +// See AUTHORS.md and LICENSE file in the project root for full license information. +// +// Kyoo is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// any later version. +// +// Kyoo is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with Kyoo. If not, see . + +using System; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations.Schema; +using System.Data; +using System.Linq; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using Dapper; +using InterpolatedSql.Dapper; +using Kyoo.Abstractions.Controllers; +using Kyoo.Abstractions.Models; +using Kyoo.Abstractions.Models.Utils; +using Kyoo.Utils; + +namespace Kyoo.Core.Controllers; + +public static class DapperHelper +{ + private static string _Property(string key, Dictionary config) + { + if (config.Count == 1) + return $"{config.First()}.{key.ToSnakeCase()}"; + + IEnumerable keys = config + .Where(x => key == "id" || x.Value.GetProperty(key) != null) + .Select(x => $"{x.Key}.{x.Value.GetProperty(key)?.GetCustomAttribute()?.Name ?? key.ToSnakeCase()}"); + return $"coalesce({string.Join(", ", keys)})"; + } + + public static string ProcessSort(Sort? sort, bool reverse, Dictionary config, bool recurse = false) + where T : IQuery + { + sort ??= new Sort.Default(); + + string ret = sort switch + { + Sort.Default(var value) => ProcessSort(value, reverse, config, true), + Sort.By(string key, bool desc) => $"{_Property(key, config)} {(desc ^ reverse ? "desc" : "asc")}", + Sort.Random(var seed) => $"md5('{seed}' || {_Property("id", config)}) {(reverse ? "desc" : "asc")}", + Sort.Conglomerate(var list) => string.Join(", ", list.Select(x => ProcessSort(x, reverse, config, true))), + _ => throw new SwitchExpressionException(), + }; + if (recurse) + return ret; + // always end query by an id sort. + return $"{ret}, {_Property("id", config)} {(reverse ? "desc" : "asc")}"; + } + + public static ( + Dictionary config, + string join, + Func, T> map + ) ProcessInclude(Include include, Dictionary config) + where T : class + { + int relation = 0; + Dictionary retConfig = new(); + StringBuilder join = new(); + + foreach (Include.Metadata metadata in include.Metadatas) + { + relation++; + switch (metadata) + { + case Include.SingleRelation(var name, var type, var rid): + string tableName = type.GetCustomAttribute()?.Name ?? $"{type.Name.ToSnakeCase()}s"; + retConfig.Add($"r{relation}", type); + join.Append($"\nleft join {tableName} as r{relation} on r{relation}.id = {_Property(rid, config)}"); + break; + case Include.CustomRelation(var name, var type, var sql, var on, var declaring): + string owner = config.First(x => x.Value == declaring).Key; + string lateral = sql.Contains("\"this\"") ? " lateral" : string.Empty; + sql = sql.Replace("\"this\"", owner); + on = on?.Replace("\"this\"", owner); + retConfig.Add($"r{relation}", type); + join.Append($"\nleft join{lateral} ({sql}) as r{relation} on r{relation}.{on}"); + break; + case Include.ProjectedRelation: + continue; + default: + throw new NotImplementedException(); + } + } + + T Map(T item, IEnumerable relations) + { + foreach ((string name, object? value) in include.Fields.Zip(relations)) + { + if (value == null) + continue; + PropertyInfo? prop = item.GetType().GetProperty(name); + if (prop != null) + prop.SetValue(item, value); + } + return item; + } + + return (retConfig, join.ToString(), Map); + } + + public static FormattableString ProcessFilter(Filter filter, Dictionary config) + { + FormattableString Format(string key, FormattableString op) + { + IEnumerable properties = config + .Where(x => key == "id" || x.Value.GetProperty(key) != null) + .Select(x => $"{x.Key}.{x.Value.GetProperty(key)?.GetCustomAttribute()?.Name ?? key.ToSnakeCase()}"); + + FormattableString ret = $"{properties.First():raw} {op}"; + foreach (string property in properties.Skip(1)) + ret = $"{ret} or {property:raw} {op}"; + return $"({ret})"; + } + + object P(object value) + { + if (value is Enum) + return new Wrapper(value); + return value; + } + + FormattableString Process(Filter fil) + { + return fil switch + { + Filter.And(var first, var second) => $"({Process(first)} and {Process(second)})", + Filter.Or(var first, var second) => $"({Process(first)} or {Process(second)})", + Filter.Not(var inner) => $"(not {Process(inner)})", + Filter.Eq(var property, var value) when value is null => Format(property, $"is null"), + Filter.Ne(var property, var value) when value is null => Format(property, $"is not null"), + Filter.Eq(var property, var value) => Format(property, $"= {P(value!)}"), + Filter.Ne(var property, var value) => Format(property, $"!= {P(value!)}"), + Filter.Gt(var property, var value) => Format(property, $"> {P(value)}"), + Filter.Ge(var property, var value) => Format(property, $">= {P(value)}"), + Filter.Lt(var property, var value) => Format(property, $"< {P(value)}"), + Filter.Le(var property, var value) => Format(property, $"> {P(value)}"), + Filter.Has(var property, var value) => $"{P(value)} = any({_Property(property, config):raw})", + Filter.EqRandom(var seed, var id) => $"md5({seed} || {config.Select(x => $"{x.Key}.id"):raw}) = md5({seed} || {id.ToString()})", + Filter.Lambda(var lambda) => throw new NotSupportedException(), + _ => throw new NotImplementedException(), + }; + } + return $"\nwhere {Process(filter)}"; + } + + public static string ExpendProjections(string type, string? prefix, Include include) + { + prefix = prefix != null ? $"{prefix}." : string.Empty; + IEnumerable projections = include.Metadatas + .Select(x => x is Include.ProjectedRelation(var name, var sql) ? sql : null!) + .Where(x => x != null) + .Select(x => x.Replace("\"this\".", prefix)); + return string.Join(string.Empty, projections.Select(x => $", {x}")); + } + + public static async Task> Query( + this IDbConnection db, + FormattableString command, + Dictionary config, + Func mapper, + Func> get, + Include? include, + Filter? filter, + Sort? sort, + Pagination limit) + where T : class, IResource, IQuery + { + InterpolatedSql.Dapper.SqlBuilders.SqlBuilder query = new(db, command); + + // Include handling + include ??= new(); + var (includeConfig, includeJoin, mapIncludes) = ProcessInclude(include, config); + query.AppendLiteral(includeJoin); + string includeProjection = string.Join(string.Empty, includeConfig.Select(x => $", {x.Key}.*")); + query.Replace("/* includes */", $"{includeProjection:raw}", out bool replaced); + if (!replaced) + throw new ArgumentException("Missing '/* includes */' placeholder in top level sql select to support includes."); + + // Handle pagination, orders and filter. + if (limit.AfterID != null) + { + T reference = await get(limit.AfterID.Value); + Filter? keysetFilter = RepositoryHelper.KeysetPaginate(sort, reference, !limit.Reverse); + filter = Filter.And(filter, keysetFilter); + } + if (filter != null) + query += ProcessFilter(filter, config); + query += $"\norder by {ProcessSort(sort, limit.Reverse, config):raw}"; + query += $"\nlimit {limit.Limit}"; + + // Build query and prepare to do the query/projections + IDapperSqlCommand cmd = query.Build(); + string sql = cmd.Sql; + Type[] types = config.Select(x => x.Value) + .Concat(includeConfig.Select(x => x.Value)) + .ToArray(); + + // Expand projections on every types received. + sql = Regex.Replace(sql, @"(,?) -- (\w+)( as (\w+))?", (match) => + { + string leadingComa = match.Groups[1].Value; + string type = match.Groups[2].Value; + string? prefix = match.Groups[3].Value; + + // Only project top level items with explicit includes. + string? projection = config.Any(x => x.Value.Name == type) + ? ExpendProjections(type, prefix, include) + : null; + if (string.IsNullOrEmpty(projection)) + return leadingComa; + return $", {projection}{leadingComa}"; + }); + + IEnumerable data = await db.QueryAsync( + sql, + types, + items => mapIncludes(mapper(items), items.Skip(config.Count)), + ParametersDictionary.LoadFrom(cmd) + ); + if (limit.Reverse) + data = data.Reverse(); + return data.ToList(); + } +} diff --git a/back/src/Kyoo.Core/Controllers/Repositories/DapperRepository.cs b/back/src/Kyoo.Core/Controllers/Repositories/DapperRepository.cs new file mode 100644 index 00000000..a9e4ec8b --- /dev/null +++ b/back/src/Kyoo.Core/Controllers/Repositories/DapperRepository.cs @@ -0,0 +1,104 @@ +// Kyoo - A portable and vast media library solution. +// Copyright (c) Kyoo. +// +// See AUTHORS.md and LICENSE file in the project root for full license information. +// +// Kyoo is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// any later version. +// +// Kyoo is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with Kyoo. If not, see . + +using System; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations.Schema; +using System.Linq; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading.Tasks; +using Kyoo.Abstractions.Controllers; +using Kyoo.Abstractions.Models; +using Kyoo.Abstractions.Models.Utils; +using Kyoo.Utils; + +namespace Kyoo.Core.Controllers; + +public class DapperRepository : IRepository + where T : class, IResource, IQuery +{ + public Type RepositoryType => typeof(T); + + public Task> FromIds(IList ids, Include? include = null) + { + throw new NotImplementedException(); + } + + public Task Get(int id, Include? include = null) + { + throw new NotImplementedException(); + } + + public Task Get(string slug, Include? include = null) + { + throw new NotImplementedException(); + } + + public Task Get(Filter filter, Include? include = null) + { + throw new NotImplementedException(); + } + + public Task> GetAll(Filter? filter = null, Sort? sort = null, Include? include = null, Pagination? limit = null) + { + throw new NotImplementedException(); + } + + public Task GetCount(Filter? filter = null) + { + throw new NotImplementedException(); + } + + public Task GetOrDefault(int id, Include? include = null) + { + throw new NotImplementedException(); + } + + public Task GetOrDefault(string slug, Include? include = null) + { + throw new NotImplementedException(); + } + + public Task GetOrDefault(Filter? filter, Include? include = null, Sort? sortBy = null) + { + throw new NotImplementedException(); + } + + public Task> Search(string query, Include? include = null) + { + throw new NotImplementedException(); + } + + public Task Create(T obj) => throw new NotImplementedException(); + + public Task CreateIfNotExists(T obj) => throw new NotImplementedException(); + + public Task Delete(int id) => throw new NotImplementedException(); + + public Task Delete(string slug) => throw new NotImplementedException(); + + public Task Delete(T obj) => throw new NotImplementedException(); + + public Task DeleteAll(Filter filter) => throw new NotImplementedException(); + + public Task Edit(T edited) => throw new NotImplementedException(); + + public Task Patch(int id, Func> patch) => throw new NotImplementedException(); +} diff --git a/back/src/Kyoo.Core/Controllers/Repositories/LibraryItemRepository.cs b/back/src/Kyoo.Core/Controllers/Repositories/LibraryItemRepository.cs index 55c18a02..6d2671ae 100644 --- a/back/src/Kyoo.Core/Controllers/Repositories/LibraryItemRepository.cs +++ b/back/src/Kyoo.Core/Controllers/Repositories/LibraryItemRepository.cs @@ -95,208 +95,52 @@ namespace Kyoo.Core.Controllers throw new NotImplementedException(); } - private static string _Property(string key, Dictionary config) - { - if (config.Count == 1) - return $"{config.First()}.{key.ToSnakeCase()}"; - - IEnumerable keys = config - .Where(x => key == "id" || x.Value.GetProperty(key) != null) - .Select(x => $"{x.Key}.{x.Value.GetProperty(key)?.GetCustomAttribute()?.Name ?? key.ToSnakeCase()}"); - return $"coalesce({string.Join(", ", keys)})"; - } - - public static string ProcessSort(Sort? sort, bool reverse, Dictionary config, bool recurse = false) - where T : IQuery - { - sort ??= new Sort.Default(); - - string ret = sort switch - { - Sort.Default(var value) => ProcessSort(value, reverse, config, true), - Sort.By(string key, bool desc) => $"{_Property(key, config)} {(desc ^ reverse ? "desc" : "asc")}", - Sort.Random(var seed) => $"md5('{seed}' || {_Property("id", config)}) {(reverse ? "desc" : "asc")}", - Sort.Conglomerate(var list) => string.Join(", ", list.Select(x => ProcessSort(x, reverse, config, true))), - _ => throw new SwitchExpressionException(), - }; - if (recurse) - return ret; - // always end query by an id sort. - return $"{ret}, {_Property("id", config)} {(reverse ? "desc" : "asc")}"; - } - - public static ( - Dictionary config, - string join, - Func, T> map - ) ProcessInclude(Include include, Dictionary config) - where T : class - { - int relation = 0; - Dictionary retConfig = new(); - StringBuilder join = new(); - - foreach (Include.Metadata metadata in include.Metadatas) - { - relation++; - switch (metadata) - { - case Include.SingleRelation(var name, var type, var rid): - string tableName = type.GetCustomAttribute()?.Name ?? $"{type.Name.ToSnakeCase()}s"; - retConfig.Add($"r{relation}", type); - join.AppendLine($"left join {tableName} as r{relation} on r{relation}.id = {_Property(rid, config)}"); - break; - case Include.CustomRelation(var name, var type, var sql, var on, var declaring): - string owner = config.First(x => x.Value == declaring).Key; - string lateral = sql.Contains("\"this\"") ? " lateral" : string.Empty; - sql = sql.Replace("\"this\"", owner); - on = on?.Replace("\"this\"", owner); - retConfig.Add($"r{relation}", type); - join.AppendLine($"left join{lateral} ({sql}) as r{relation} on r{relation}.{on}"); - break; - case Include.ProjectedRelation: - continue; - default: - throw new NotImplementedException(); - } - } - - T Map(T item, IEnumerable relations) - { - foreach ((string name, object? value) in include.Fields.Zip(relations)) - { - if (value == null) - continue; - PropertyInfo? prop = item.GetType().GetProperty(name); - if (prop != null) - prop.SetValue(item, value); - } - return item; - } - - return (retConfig, join.ToString(), Map); - } - - public static FormattableString ExpendProjections(string? prefix, Include include) - { - prefix = prefix != null ? $"{prefix}." : string.Empty; - IEnumerable projections = include.Metadatas - .Select(x => x is Include.ProjectedRelation(var name, var sql) ? sql : null!) - .Where(x => x != null) - .Select(x => x.Replace("\"this\".", prefix)); - string projStr = string.Join(string.Empty, projections.Select(x => $", {x}")); - return $"{prefix:raw}*{projStr:raw}"; - } - - public static FormattableString ProcessFilter(Filter filter, Dictionary config) - { - FormattableString Format(string key, FormattableString op) - { - IEnumerable properties = config - .Where(x => key == "id" || x.Value.GetProperty(key) != null) - .Select(x => $"{x.Key}.{x.Value.GetProperty(key)?.GetCustomAttribute()?.Name ?? key.ToSnakeCase()}"); - - FormattableString ret = $"{properties.First():raw} {op}"; - foreach (string property in properties.Skip(1)) - ret = $"{ret} or {property:raw} {op}"; - return $"({ret})"; - } - - object P(object value) - { - if (value is Enum) - return new Wrapper(value); - return value; - } - - FormattableString Process(Filter fil) - { - return fil switch - { - Filter.And(var first, var second) => $"({Process(first)} and {Process(second)})", - Filter.Or(var first, var second) => $"({Process(first)} or {Process(second)})", - Filter.Not(var inner) => $"(not {Process(inner)})", - Filter.Eq(var property, var value) when value is null => Format(property, $"is null"), - Filter.Ne(var property, var value) when value is null => Format(property, $"is not null"), - Filter.Eq(var property, var value) => Format(property, $"= {P(value!)}"), - Filter.Ne(var property, var value) => Format(property, $"!= {P(value!)}"), - Filter.Gt(var property, var value) => Format(property, $"> {P(value)}"), - Filter.Ge(var property, var value) => Format(property, $">= {P(value)}"), - Filter.Lt(var property, var value) => Format(property, $"< {P(value)}"), - Filter.Le(var property, var value) => Format(property, $"> {P(value)}"), - Filter.Has(var property, var value) => $"{P(value)} = any({_Property(property, config):raw})", - Filter.EqRandom(var seed, var id) => $"md5({seed} || {config.Select(x => $"{x.Key}.id"):raw}) = md5({seed} || {id.ToString()})", - Filter.Lambda(var lambda) => throw new NotSupportedException(), - _ => throw new NotImplementedException(), - }; - } - return $"where {Process(filter)}"; - } - - public async Task> GetAll(Filter? filter = null, + public Task> GetAll( + Filter? filter = null, Sort? sort = default, Include? include = default, - Pagination limit = default) + Pagination? limit = default) { - include ??= new(); - - Dictionary config = new() - { - { "s", typeof(Show) }, - { "m", typeof(Movie) }, - { "c", typeof(Collection) } - }; - var (includeConfig, includeJoin, mapIncludes) = ProcessInclude(include, config); - // language=PostgreSQL - var query = _database.SqlBuilder($""" + FormattableString sql = $""" select - {ExpendProjections("s", include)}, + s.*, -- Show as s m.*, c.* - {string.Join(string.Empty, includeConfig.Select(x => $", {x.Key}.*")):raw} + /* includes */ from shows as s full outer join ( select - {ExpendProjections(null, include)} + * -- Movie from movies) as m on false - full outer join ( + full outer join ( select - {ExpendProjections(null, include)} + * -- Collection from collections) as c on false - {includeJoin:raw} - """); + """; - if (limit.AfterID != null) - { - ILibraryItem reference = await Get(limit.AfterID.Value); - Filter? keysetFilter = RepositoryHelper.KeysetPaginate(sort, reference, !limit.Reverse); - filter = Filter.And(filter, keysetFilter); - } - if (filter != null) - query += ProcessFilter(filter, config); - query += $"order by {ProcessSort(sort, limit.Reverse, config):raw}"; - query += $"limit {limit.Limit}"; - - Type[] types = config.Select(x => x.Value) - .Concat(includeConfig.Select(x => x.Value)) - .ToArray(); - IEnumerable data = await query.QueryAsync(types, items => - { - if (items[0] is Show show && show.Id != 0) - return mapIncludes(show, items.Skip(3)); - if (items[1] is Movie movie && movie.Id != 0) - return mapIncludes(movie, items.Skip(3)); - if (items[2] is Collection collection && collection.Id != 0) - return mapIncludes(collection, items.Skip(3)); - throw new InvalidDataException(); - }); - if (limit.Reverse) - data = data.Reverse(); - return data.ToList(); + return _database.Query(sql, new() + { + { "s", typeof(Show) }, + { "m", typeof(Movie) }, + { "c", typeof(Collection) } + }, + items => + { + if (items[0] is Show show && show.Id != 0) + return show; + if (items[1] is Movie movie && movie.Id != 0) + return movie; + if (items[2] is Collection collection && collection.Id != 0) + return collection; + throw new InvalidDataException(); + }, + (id) => Get(id), + include, filter, sort, limit + ); } public Task GetCount(Filter? filter = null) diff --git a/back/src/Kyoo.Core/Controllers/Repositories/LocalRepository.cs b/back/src/Kyoo.Core/Controllers/Repositories/LocalRepository.cs index e77d0a78..3d445223 100644 --- a/back/src/Kyoo.Core/Controllers/Repositories/LocalRepository.cs +++ b/back/src/Kyoo.Core/Controllers/Repositories/LocalRepository.cs @@ -265,7 +265,7 @@ namespace Kyoo.Core.Controllers public virtual Task> GetAll(Filter? filter = null, Sort? sort = default, Include? include = default, - Pagination limit = default) + Pagination? limit = default) { return ApplyFilters(Database.Set(), filter, sort, limit, include); } @@ -282,11 +282,12 @@ namespace Kyoo.Core.Controllers protected async Task> ApplyFilters(IQueryable query, Filter? filter = null, Sort? sort = default, - Pagination limit = default, + Pagination? limit = default, Include? include = default) { query = AddIncludes(query, include); query = Sort(query, sort); + limit ??= new(); if (limit.AfterID != null) {