mirror of
https://github.com/zoriya/Kyoo.git
synced 2025-07-09 03:04:20 -04:00
Add filter to meilisearch in backend (#530)
This commit is contained in:
commit
12f3808579
@ -64,7 +64,7 @@ OIDC_SERVICE_PROFILE=https://url-of-the-profile-endpoint-of-the-oidc-service.com
|
||||
OIDC_SERVICE_SCOPE="the list of scopes space separeted like email identity"
|
||||
# Token authentication method as seen in https://openid.net/specs/openid-connect-core-1_0.html#ClientAuthentication
|
||||
# Supported values: ClientSecretBasic (default) or ClientSecretPost
|
||||
# If in doupt, leave this empty.
|
||||
# If in doubt, leave this empty.
|
||||
OIDC_SERVICE_AUTHMETHOD=ClientSecretBasic
|
||||
# on the previous list, service is the internal name of your service, you can add as many as you want.
|
||||
|
||||
@ -81,6 +81,11 @@ POSTGRES_DB=kyooDB
|
||||
POSTGRES_SERVER=postgres
|
||||
POSTGRES_PORT=5432
|
||||
|
||||
# Read by the api container to know if it should run meilisearch's migrations/sync
|
||||
# and download missing images. This is a good idea to only have one instance with this on
|
||||
# Note: it does not run postgres migrations, use the migration container for that.
|
||||
RUN_MIGRATIONS=true
|
||||
|
||||
MEILI_HOST="http://meilisearch:7700"
|
||||
MEILI_MASTER_KEY="ghvjkgisbgkbgskegblfqbgjkebbhgwkjfb"
|
||||
|
||||
|
@ -38,6 +38,7 @@ public interface ISearchManager
|
||||
public Task<SearchPage<ILibraryItem>.SearchResult> SearchItems(
|
||||
string? query,
|
||||
Sort<ILibraryItem> sortBy,
|
||||
Filter<ILibraryItem>? filter,
|
||||
SearchPagination pagination,
|
||||
Include<ILibraryItem>? include = default
|
||||
);
|
||||
@ -53,6 +54,7 @@ public interface ISearchManager
|
||||
public Task<SearchPage<Movie>.SearchResult> SearchMovies(
|
||||
string? query,
|
||||
Sort<Movie> sortBy,
|
||||
Filter<ILibraryItem>? filter,
|
||||
SearchPagination pagination,
|
||||
Include<Movie>? include = default
|
||||
);
|
||||
@ -68,6 +70,7 @@ public interface ISearchManager
|
||||
public Task<SearchPage<Show>.SearchResult> SearchShows(
|
||||
string? query,
|
||||
Sort<Show> sortBy,
|
||||
Filter<ILibraryItem>? filter,
|
||||
SearchPagination pagination,
|
||||
Include<Show>? include = default
|
||||
);
|
||||
@ -83,6 +86,7 @@ public interface ISearchManager
|
||||
public Task<SearchPage<Collection>.SearchResult> SearchCollections(
|
||||
string? query,
|
||||
Sort<Collection> sortBy,
|
||||
Filter<ILibraryItem>? filter,
|
||||
SearchPagination pagination,
|
||||
Include<Collection>? include = default
|
||||
);
|
||||
@ -98,6 +102,7 @@ public interface ISearchManager
|
||||
public Task<SearchPage<Episode>.SearchResult> SearchEpisodes(
|
||||
string? query,
|
||||
Sort<Episode> sortBy,
|
||||
Filter<Episode>? filter,
|
||||
SearchPagination pagination,
|
||||
Include<Episode>? include = default
|
||||
);
|
||||
@ -113,6 +118,7 @@ public interface ISearchManager
|
||||
public Task<SearchPage<Studio>.SearchResult> SearchStudios(
|
||||
string? query,
|
||||
Sort<Studio> sortBy,
|
||||
Filter<Studio>? filter,
|
||||
SearchPagination pagination,
|
||||
Include<Studio>? include = default
|
||||
);
|
||||
|
@ -215,14 +215,16 @@ public abstract record Filter<T> : Filter
|
||||
});
|
||||
}
|
||||
|
||||
if (type == typeof(DateTime))
|
||||
if (type == typeof(DateTime) || type == typeof(DateOnly))
|
||||
{
|
||||
return from year in Parse.Digit.Repeat(4).Text().Select(int.Parse)
|
||||
from yd in Parse.Char('-')
|
||||
from mouth in Parse.Digit.Repeat(2).Text().Select(int.Parse)
|
||||
from month in Parse.Digit.Repeat(2).Text().Select(int.Parse)
|
||||
from md in Parse.Char('-')
|
||||
from day in Parse.Digit.Repeat(2).Text().Select(int.Parse)
|
||||
select new DateTime(year, mouth, day) as object;
|
||||
select type == typeof(DateTime)
|
||||
? new DateTime(year, month, day) as object
|
||||
: new DateOnly(year, month, day) as object;
|
||||
}
|
||||
|
||||
if (typeof(IEnumerable).IsAssignableFrom(type))
|
||||
|
@ -17,6 +17,7 @@
|
||||
// along with Kyoo. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
using System;
|
||||
using Kyoo.Abstractions.Controllers;
|
||||
using Kyoo.Authentication;
|
||||
using Kyoo.Core;
|
||||
using Kyoo.Core.Controllers;
|
||||
@ -27,6 +28,7 @@ using Kyoo.RabbitMq;
|
||||
using Kyoo.Swagger;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Serilog;
|
||||
using Serilog.Events;
|
||||
@ -94,12 +96,17 @@ app.MapControllers();
|
||||
app.Services.GetRequiredService<MeiliSync>();
|
||||
app.Services.GetRequiredService<RabbitProducer>();
|
||||
|
||||
await using (AsyncServiceScope scope = app.Services.CreateAsyncScope())
|
||||
// Only run sync on the main instance
|
||||
if (app.Configuration.GetValue("RUN_MIGRATIONS", true))
|
||||
{
|
||||
await MeilisearchModule.Initialize(scope.ServiceProvider);
|
||||
await using (AsyncServiceScope scope = app.Services.CreateAsyncScope())
|
||||
{
|
||||
await MeilisearchModule.Initialize(app.Services);
|
||||
}
|
||||
|
||||
// The methods takes care of creating a scope and will download images on the background.
|
||||
_ = MeilisearchModule.SyncDatabase(app.Services);
|
||||
_ = MiscRepository.DownloadMissingImages(app.Services);
|
||||
}
|
||||
|
||||
// The methods takes care of creating a scope and will download images on the background.
|
||||
_ = MiscRepository.DownloadMissingImages(app.Services);
|
||||
|
||||
await app.RunAsync(Environment.GetEnvironmentVariable("KYOO_BIND_URL") ?? "http://*:5000");
|
||||
|
@ -44,7 +44,7 @@ public class SearchApi : BaseApi
|
||||
_searchManager = searchManager;
|
||||
}
|
||||
|
||||
// TODO: add filters and facets
|
||||
// TODO: add facets
|
||||
|
||||
/// <summary>
|
||||
/// Search collections
|
||||
@ -65,11 +65,14 @@ public class SearchApi : BaseApi
|
||||
public async Task<SearchPage<Collection>> SearchCollections(
|
||||
[FromQuery] string? q,
|
||||
[FromQuery] Sort<Collection> sortBy,
|
||||
[FromQuery] Filter<ILibraryItem>? filter,
|
||||
[FromQuery] SearchPagination pagination,
|
||||
[FromQuery] Include<Collection> fields
|
||||
)
|
||||
{
|
||||
return SearchPage(await _searchManager.SearchCollections(q, sortBy, pagination, fields));
|
||||
return SearchPage(
|
||||
await _searchManager.SearchCollections(q, sortBy, filter, pagination, fields)
|
||||
);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@ -91,11 +94,12 @@ public class SearchApi : BaseApi
|
||||
public async Task<SearchPage<Show>> SearchShows(
|
||||
[FromQuery] string? q,
|
||||
[FromQuery] Sort<Show> sortBy,
|
||||
[FromQuery] Filter<ILibraryItem>? filter,
|
||||
[FromQuery] SearchPagination pagination,
|
||||
[FromQuery] Include<Show> fields
|
||||
)
|
||||
{
|
||||
return SearchPage(await _searchManager.SearchShows(q, sortBy, pagination, fields));
|
||||
return SearchPage(await _searchManager.SearchShows(q, sortBy, filter, pagination, fields));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@ -117,11 +121,12 @@ public class SearchApi : BaseApi
|
||||
public async Task<SearchPage<Movie>> SearchMovies(
|
||||
[FromQuery] string? q,
|
||||
[FromQuery] Sort<Movie> sortBy,
|
||||
[FromQuery] Filter<ILibraryItem>? filter,
|
||||
[FromQuery] SearchPagination pagination,
|
||||
[FromQuery] Include<Movie> fields
|
||||
)
|
||||
{
|
||||
return SearchPage(await _searchManager.SearchMovies(q, sortBy, pagination, fields));
|
||||
return SearchPage(await _searchManager.SearchMovies(q, sortBy, filter, pagination, fields));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@ -143,11 +148,12 @@ public class SearchApi : BaseApi
|
||||
public async Task<SearchPage<ILibraryItem>> SearchItems(
|
||||
[FromQuery] string? q,
|
||||
[FromQuery] Sort<ILibraryItem> sortBy,
|
||||
[FromQuery] Filter<ILibraryItem>? filter,
|
||||
[FromQuery] SearchPagination pagination,
|
||||
[FromQuery] Include<ILibraryItem> fields
|
||||
)
|
||||
{
|
||||
return SearchPage(await _searchManager.SearchItems(q, sortBy, pagination, fields));
|
||||
return SearchPage(await _searchManager.SearchItems(q, sortBy, filter, pagination, fields));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@ -169,11 +175,14 @@ public class SearchApi : BaseApi
|
||||
public async Task<SearchPage<Episode>> SearchEpisodes(
|
||||
[FromQuery] string? q,
|
||||
[FromQuery] Sort<Episode> sortBy,
|
||||
[FromQuery] Filter<Episode>? filter,
|
||||
[FromQuery] SearchPagination pagination,
|
||||
[FromQuery] Include<Episode> fields
|
||||
)
|
||||
{
|
||||
return SearchPage(await _searchManager.SearchEpisodes(q, sortBy, pagination, fields));
|
||||
return SearchPage(
|
||||
await _searchManager.SearchEpisodes(q, sortBy, filter, pagination, fields)
|
||||
);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@ -195,10 +204,13 @@ public class SearchApi : BaseApi
|
||||
public async Task<SearchPage<Studio>> SearchStudios(
|
||||
[FromQuery] string? q,
|
||||
[FromQuery] Sort<Studio> sortBy,
|
||||
[FromQuery] Filter<Studio>? filter,
|
||||
[FromQuery] SearchPagination pagination,
|
||||
[FromQuery] Include<Studio> fields
|
||||
)
|
||||
{
|
||||
return SearchPage(await _searchManager.SearchStudios(q, sortBy, pagination, fields));
|
||||
return SearchPage(
|
||||
await _searchManager.SearchStudios(q, sortBy, filter, pagination, fields)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
52
back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs
Normal file
52
back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs
Normal file
@ -0,0 +1,52 @@
|
||||
using System.ComponentModel.DataAnnotations;
|
||||
using Kyoo.Abstractions.Models.Utils;
|
||||
using static System.Text.Json.JsonNamingPolicy;
|
||||
|
||||
namespace Kyoo.Meiliseach;
|
||||
|
||||
internal static class FilterExtensionMethods
|
||||
{
|
||||
public static string? CreateMeilisearchFilter<T>(this Filter<T>? filter)
|
||||
{
|
||||
return filter switch
|
||||
{
|
||||
Filter<T>.And and
|
||||
=> $"({and.First.CreateMeilisearchFilter()}) AND ({and.Second.CreateMeilisearchFilter()})",
|
||||
Filter<T>.Or or
|
||||
=> $"({or.First.CreateMeilisearchFilter()}) OR ({or.Second.CreateMeilisearchFilter()})",
|
||||
Filter<T>.Gt gt => CreateBasicFilterString(gt.Property, ">", gt.Value),
|
||||
Filter<T>.Lt lt => CreateBasicFilterString(lt.Property, "<", lt.Value),
|
||||
Filter<T>.Ge ge => CreateBasicFilterString(ge.Property, ">=", ge.Value),
|
||||
Filter<T>.Le le => CreateBasicFilterString(le.Property, "<=", le.Value),
|
||||
Filter<T>.Eq eq => CreateBasicFilterString(eq.Property, "=", eq.Value),
|
||||
Filter<T>.Has has => CreateBasicFilterString(has.Property, "=", has.Value),
|
||||
Filter<T>.Ne ne => CreateBasicFilterString(ne.Property, "!=", ne.Value),
|
||||
Filter<T>.Not not => $"NOT ({not.Filter.CreateMeilisearchFilter()})",
|
||||
Filter<T>.CmpRandom
|
||||
=> throw new ValidationException("Random comparison is not supported."),
|
||||
_ => null
|
||||
};
|
||||
}
|
||||
|
||||
private static string CreateBasicFilterString(string property, string @operator, object? value)
|
||||
{
|
||||
return $"{CamelCase.ConvertName(property)} {@operator} {value.InMeilsearchFilterFormat()}";
|
||||
}
|
||||
|
||||
private static object? InMeilsearchFilterFormat(this object? value)
|
||||
{
|
||||
return value switch
|
||||
{
|
||||
null => null,
|
||||
string s => s.Any(char.IsWhiteSpace) ? $"\"{s.Replace("\"", "\\\"")}\"" : s,
|
||||
DateTimeOffset dateTime => dateTime.ToUnixTimeSeconds(),
|
||||
DateOnly date => date.ToUnixTimeSeconds(),
|
||||
_ => value
|
||||
};
|
||||
}
|
||||
|
||||
public static long ToUnixTimeSeconds(this DateOnly date)
|
||||
{
|
||||
return new DateTimeOffset(date.ToDateTime(new TimeOnly())).ToUnixTimeSeconds();
|
||||
}
|
||||
}
|
@ -16,6 +16,7 @@
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with Kyoo. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
using System.Collections;
|
||||
using System.Dynamic;
|
||||
using System.Reflection;
|
||||
using Kyoo.Abstractions.Controllers;
|
||||
@ -60,7 +61,10 @@ public class MeiliSync
|
||||
var dictionary = (IDictionary<string, object?>)expando;
|
||||
|
||||
foreach (PropertyInfo property in item.GetType().GetProperties())
|
||||
dictionary.Add(CamelCase.ConvertName(property.Name), property.GetValue(item));
|
||||
dictionary.Add(
|
||||
CamelCase.ConvertName(property.Name),
|
||||
ConvertToMeilisearchFormat(property.GetValue(item))
|
||||
);
|
||||
dictionary.Add("ref", $"{kind}-{item.Id}");
|
||||
expando.kind = kind;
|
||||
return _client.Index(index).AddDocumentsAsync(new[] { expando });
|
||||
@ -76,4 +80,33 @@ public class MeiliSync
|
||||
}
|
||||
return _client.Index(index).DeleteOneDocumentAsync(id.ToString());
|
||||
}
|
||||
|
||||
private object? ConvertToMeilisearchFormat(object? value)
|
||||
{
|
||||
return value switch
|
||||
{
|
||||
null => null,
|
||||
string => value,
|
||||
Enum => value.ToString(),
|
||||
IEnumerable enumerable
|
||||
=> enumerable.Cast<object>().Select(ConvertToMeilisearchFormat).ToArray(),
|
||||
DateTimeOffset dateTime => dateTime.ToUnixTimeSeconds(),
|
||||
DateOnly date => date.ToUnixTimeSeconds(),
|
||||
_ => value
|
||||
};
|
||||
}
|
||||
|
||||
public async Task SyncEverything(ILibraryManager database)
|
||||
{
|
||||
foreach (Movie movie in await database.Movies.GetAll(limit: 0))
|
||||
await CreateOrUpdate("items", movie, nameof(Movie));
|
||||
foreach (Show show in await database.Shows.GetAll(limit: 0))
|
||||
await CreateOrUpdate("items", show, nameof(Show));
|
||||
foreach (Collection collection in await database.Collections.GetAll(limit: 0))
|
||||
await CreateOrUpdate("items", collection, nameof(Collection));
|
||||
foreach (Episode episode in await database.Episodes.GetAll(limit: 0))
|
||||
await CreateOrUpdate(nameof(Episode), episode);
|
||||
foreach (Studio studio in await database.Studios.GetAll(limit: 0))
|
||||
await CreateOrUpdate(nameof(Studio), studio);
|
||||
}
|
||||
}
|
||||
|
@ -49,6 +49,8 @@ public static class MeilisearchModule
|
||||
CamelCase.ConvertName(nameof(Movie.Genres)),
|
||||
CamelCase.ConvertName(nameof(Movie.Status)),
|
||||
CamelCase.ConvertName(nameof(Movie.AirDate)),
|
||||
CamelCase.ConvertName(nameof(Show.StartAir)),
|
||||
CamelCase.ConvertName(nameof(Show.EndAir)),
|
||||
CamelCase.ConvertName(nameof(Movie.StudioId)),
|
||||
"kind"
|
||||
},
|
||||
@ -117,11 +119,6 @@ public static class MeilisearchModule
|
||||
},
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Init meilisearch indexes.
|
||||
/// </summary>
|
||||
/// <param name="provider">The service list to retrieve the meilisearch client</param>
|
||||
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
|
||||
public static async Task Initialize(IServiceProvider provider)
|
||||
{
|
||||
MeilisearchClient client = provider.GetRequiredService<MeilisearchClient>();
|
||||
@ -131,6 +128,13 @@ public static class MeilisearchModule
|
||||
await _CreateIndex(client, nameof(Studio), false);
|
||||
}
|
||||
|
||||
public static async Task SyncDatabase(IServiceProvider provider)
|
||||
{
|
||||
await using AsyncServiceScope scope = provider.CreateAsyncScope();
|
||||
ILibraryManager database = scope.ServiceProvider.GetRequiredService<ILibraryManager>();
|
||||
await scope.ServiceProvider.GetRequiredService<MeiliSync>().SyncEverything(database);
|
||||
}
|
||||
|
||||
private static async Task _CreateIndex(MeilisearchClient client, string index, bool hasKind)
|
||||
{
|
||||
TaskInfo task = await client.CreateIndexAsync(
|
||||
|
@ -99,66 +99,125 @@ public class SearchManager : ISearchManager
|
||||
public Task<SearchPage<ILibraryItem>.SearchResult> SearchItems(
|
||||
string? query,
|
||||
Sort<ILibraryItem> sortBy,
|
||||
Filter<ILibraryItem>? filter,
|
||||
SearchPagination pagination,
|
||||
Include<ILibraryItem>? include = default
|
||||
)
|
||||
{
|
||||
return _Search("items", query, null, sortBy, pagination, include);
|
||||
return _Search(
|
||||
"items",
|
||||
query,
|
||||
filter.CreateMeilisearchFilter(),
|
||||
sortBy,
|
||||
pagination,
|
||||
include
|
||||
);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public Task<SearchPage<Movie>.SearchResult> SearchMovies(
|
||||
string? query,
|
||||
Sort<Movie> sortBy,
|
||||
Filter<ILibraryItem>? filter,
|
||||
SearchPagination pagination,
|
||||
Include<Movie>? include = default
|
||||
)
|
||||
{
|
||||
return _Search("items", query, $"kind = {nameof(Movie)}", sortBy, pagination, include);
|
||||
return _Search(
|
||||
"items",
|
||||
query,
|
||||
_CreateMediaTypeFilter<Movie>(filter),
|
||||
sortBy,
|
||||
pagination,
|
||||
include
|
||||
);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public Task<SearchPage<Show>.SearchResult> SearchShows(
|
||||
string? query,
|
||||
Sort<Show> sortBy,
|
||||
Filter<ILibraryItem>? filter,
|
||||
SearchPagination pagination,
|
||||
Include<Show>? include = default
|
||||
)
|
||||
{
|
||||
return _Search("items", query, $"kind = {nameof(Show)}", sortBy, pagination, include);
|
||||
return _Search(
|
||||
"items",
|
||||
query,
|
||||
_CreateMediaTypeFilter<Show>(filter),
|
||||
sortBy,
|
||||
pagination,
|
||||
include
|
||||
);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public Task<SearchPage<Collection>.SearchResult> SearchCollections(
|
||||
string? query,
|
||||
Sort<Collection> sortBy,
|
||||
Filter<ILibraryItem>? filter,
|
||||
SearchPagination pagination,
|
||||
Include<Collection>? include = default
|
||||
)
|
||||
{
|
||||
return _Search("items", query, $"kind = {nameof(Collection)}", sortBy, pagination, include);
|
||||
return _Search(
|
||||
"items",
|
||||
query,
|
||||
_CreateMediaTypeFilter<Collection>(filter),
|
||||
sortBy,
|
||||
pagination,
|
||||
include
|
||||
);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public Task<SearchPage<Episode>.SearchResult> SearchEpisodes(
|
||||
string? query,
|
||||
Sort<Episode> sortBy,
|
||||
Filter<Episode>? filter,
|
||||
SearchPagination pagination,
|
||||
Include<Episode>? include = default
|
||||
)
|
||||
{
|
||||
return _Search(nameof(Episode), query, null, sortBy, pagination, include);
|
||||
return _Search(
|
||||
nameof(Episode),
|
||||
query,
|
||||
filter.CreateMeilisearchFilter(),
|
||||
sortBy,
|
||||
pagination,
|
||||
include
|
||||
);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public Task<SearchPage<Studio>.SearchResult> SearchStudios(
|
||||
string? query,
|
||||
Sort<Studio> sortBy,
|
||||
Filter<Studio>? filter,
|
||||
SearchPagination pagination,
|
||||
Include<Studio>? include = default
|
||||
)
|
||||
{
|
||||
return _Search(nameof(Studio), query, null, sortBy, pagination, include);
|
||||
return _Search(
|
||||
nameof(Studio),
|
||||
query,
|
||||
filter.CreateMeilisearchFilter(),
|
||||
sortBy,
|
||||
pagination,
|
||||
include
|
||||
);
|
||||
}
|
||||
|
||||
private string _CreateMediaTypeFilter<T>(Filter<ILibraryItem>? filter)
|
||||
where T : ILibraryItem
|
||||
{
|
||||
string filterString = $"kind = {typeof(T).Name}";
|
||||
if (filter is not null)
|
||||
{
|
||||
filterString += $" AND ({filter.CreateMeilisearchFilter()})";
|
||||
}
|
||||
return filterString;
|
||||
}
|
||||
|
||||
private class IdResource
|
||||
|
Loading…
x
Reference in New Issue
Block a user