Add filter to meilisearch in backend (#530)

.env.example

@@ -64,7 +64,7 @@ OIDC_SERVICE_PROFILE=https://url-of-the-profile-endpoint-of-the-oidc-service.com
 OIDC_SERVICE_SCOPE="the list of scopes space separeted like email identity"
 # Token authentication method as seen in https://openid.net/specs/openid-connect-core-1_0.html#ClientAuthentication
 # Supported values: ClientSecretBasic (default) or ClientSecretPost
-# If in doupt, leave this empty.
+# If in doubt, leave this empty.
 OIDC_SERVICE_AUTHMETHOD=ClientSecretBasic
 # on the previous list, service is the internal name of your service, you can add as many as you want.
 
@@ -81,6 +81,11 @@ POSTGRES_DB=kyooDB
 POSTGRES_SERVER=postgres
 POSTGRES_PORT=5432
 
+# Read by the api container to know if it should run meilisearch's migrations/sync
+# and download missing images. This is a good idea to only have one instance with this on
+# Note: it does not run postgres migrations, use the migration container for that.
+RUN_MIGRATIONS=true
+
 MEILI_HOST="http://meilisearch:7700"
 MEILI_MASTER_KEY="ghvjkgisbgkbgskegblfqbgjkebbhgwkjfb"
 

back/src/Kyoo.Abstractions/Controllers/ISearchManager.cs

@@ -38,6 +38,7 @@ public interface ISearchManager
 	public Task<SearchPage<ILibraryItem>.SearchResult> SearchItems(
 		string? query,
 		Sort<ILibraryItem> sortBy,
+		Filter<ILibraryItem>? filter,
 		SearchPagination pagination,
 		Include<ILibraryItem>? include = default
 	);
@@ -53,6 +54,7 @@ public interface ISearchManager
 	public Task<SearchPage<Movie>.SearchResult> SearchMovies(
 		string? query,
 		Sort<Movie> sortBy,
+		Filter<ILibraryItem>? filter,
 		SearchPagination pagination,
 		Include<Movie>? include = default
 	);
@@ -68,6 +70,7 @@ public interface ISearchManager
 	public Task<SearchPage<Show>.SearchResult> SearchShows(
 		string? query,
 		Sort<Show> sortBy,
+		Filter<ILibraryItem>? filter,
 		SearchPagination pagination,
 		Include<Show>? include = default
 	);
@@ -83,6 +86,7 @@ public interface ISearchManager
 	public Task<SearchPage<Collection>.SearchResult> SearchCollections(
 		string? query,
 		Sort<Collection> sortBy,
+		Filter<ILibraryItem>? filter,
 		SearchPagination pagination,
 		Include<Collection>? include = default
 	);
@@ -98,6 +102,7 @@ public interface ISearchManager
 	public Task<SearchPage<Episode>.SearchResult> SearchEpisodes(
 		string? query,
 		Sort<Episode> sortBy,
+		Filter<Episode>? filter,
 		SearchPagination pagination,
 		Include<Episode>? include = default
 	);
@@ -113,6 +118,7 @@ public interface ISearchManager
 	public Task<SearchPage<Studio>.SearchResult> SearchStudios(
 		string? query,
 		Sort<Studio> sortBy,
+		Filter<Studio>? filter,
 		SearchPagination pagination,
 		Include<Studio>? include = default
 	);

back/src/Kyoo.Abstractions/Models/Utils/Filter.cs

@@ -215,14 +215,16 @@ public abstract record Filter<T> : Filter
 					});
 			}
 
-			if (type == typeof(DateTime))
+			if (type == typeof(DateTime) || type == typeof(DateOnly))
 			{
 				return from year in Parse.Digit.Repeat(4).Text().Select(int.Parse)
 					from yd in Parse.Char('-')
-					from mouth in Parse.Digit.Repeat(2).Text().Select(int.Parse)
+					from month in Parse.Digit.Repeat(2).Text().Select(int.Parse)
 					from md in Parse.Char('-')
 					from day in Parse.Digit.Repeat(2).Text().Select(int.Parse)
-					select new DateTime(year, mouth, day) as object;
+					select type == typeof(DateTime)
+						? new DateTime(year, month, day) as object
+						: new DateOnly(year, month, day) as object;
 			}
 
 			if (typeof(IEnumerable).IsAssignableFrom(type))

back/src/Kyoo.Core/Program.cs

@@ -17,6 +17,7 @@
 // along with Kyoo. If not, see <https://www.gnu.org/licenses/>.
 
 using System;
+using Kyoo.Abstractions.Controllers;
 using Kyoo.Authentication;
 using Kyoo.Core;
 using Kyoo.Core.Controllers;
@@ -27,6 +28,7 @@ using Kyoo.RabbitMq;
 using Kyoo.Swagger;
 using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Hosting;
+using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Serilog;
 using Serilog.Events;
@@ -94,12 +96,17 @@ app.MapControllers();
 app.Services.GetRequiredService<MeiliSync>();
 app.Services.GetRequiredService<RabbitProducer>();
 
+// Only run sync on the main instance
+if (app.Configuration.GetValue("RUN_MIGRATIONS", true))
+{
 	await using (AsyncServiceScope scope = app.Services.CreateAsyncScope())
 	{
-	await MeilisearchModule.Initialize(scope.ServiceProvider);
+		await MeilisearchModule.Initialize(app.Services);
 	}
 
 	// The methods takes care of creating a scope and will download images on the background.
+	_ = MeilisearchModule.SyncDatabase(app.Services);
 	_ = MiscRepository.DownloadMissingImages(app.Services);
+}
 
 await app.RunAsync(Environment.GetEnvironmentVariable("KYOO_BIND_URL") ?? "http://*:5000");

back/src/Kyoo.Core/Views/Resources/SearchApi.cs

@@ -44,7 +44,7 @@ public class SearchApi : BaseApi
 		_searchManager = searchManager;
 	}
 
-	// TODO: add filters and facets
+	// TODO: add facets
 
 	/// <summary>
 	/// Search collections
@@ -65,11 +65,14 @@ public class SearchApi : BaseApi
 	public async Task<SearchPage<Collection>> SearchCollections(
 		[FromQuery] string? q,
 		[FromQuery] Sort<Collection> sortBy,
+		[FromQuery] Filter<ILibraryItem>? filter,
 		[FromQuery] SearchPagination pagination,
 		[FromQuery] Include<Collection> fields
 	)
 	{
-		return SearchPage(await _searchManager.SearchCollections(q, sortBy, pagination, fields));
+		return SearchPage(
+			await _searchManager.SearchCollections(q, sortBy, filter, pagination, fields)
+		);
 	}
 
 	/// <summary>
@@ -91,11 +94,12 @@ public class SearchApi : BaseApi
 	public async Task<SearchPage<Show>> SearchShows(
 		[FromQuery] string? q,
 		[FromQuery] Sort<Show> sortBy,
+		[FromQuery] Filter<ILibraryItem>? filter,
 		[FromQuery] SearchPagination pagination,
 		[FromQuery] Include<Show> fields
 	)
 	{
-		return SearchPage(await _searchManager.SearchShows(q, sortBy, pagination, fields));
+		return SearchPage(await _searchManager.SearchShows(q, sortBy, filter, pagination, fields));
 	}
 
 	/// <summary>
@@ -117,11 +121,12 @@ public class SearchApi : BaseApi
 	public async Task<SearchPage<Movie>> SearchMovies(
 		[FromQuery] string? q,
 		[FromQuery] Sort<Movie> sortBy,
+		[FromQuery] Filter<ILibraryItem>? filter,
 		[FromQuery] SearchPagination pagination,
 		[FromQuery] Include<Movie> fields
 	)
 	{
-		return SearchPage(await _searchManager.SearchMovies(q, sortBy, pagination, fields));
+		return SearchPage(await _searchManager.SearchMovies(q, sortBy, filter, pagination, fields));
 	}
 
 	/// <summary>
@@ -143,11 +148,12 @@ public class SearchApi : BaseApi
 	public async Task<SearchPage<ILibraryItem>> SearchItems(
 		[FromQuery] string? q,
 		[FromQuery] Sort<ILibraryItem> sortBy,
+		[FromQuery] Filter<ILibraryItem>? filter,
 		[FromQuery] SearchPagination pagination,
 		[FromQuery] Include<ILibraryItem> fields
 	)
 	{
-		return SearchPage(await _searchManager.SearchItems(q, sortBy, pagination, fields));
+		return SearchPage(await _searchManager.SearchItems(q, sortBy, filter, pagination, fields));
 	}
 
 	/// <summary>
@@ -169,11 +175,14 @@ public class SearchApi : BaseApi
 	public async Task<SearchPage<Episode>> SearchEpisodes(
 		[FromQuery] string? q,
 		[FromQuery] Sort<Episode> sortBy,
+		[FromQuery] Filter<Episode>? filter,
 		[FromQuery] SearchPagination pagination,
 		[FromQuery] Include<Episode> fields
 	)
 	{
-		return SearchPage(await _searchManager.SearchEpisodes(q, sortBy, pagination, fields));
+		return SearchPage(
+			await _searchManager.SearchEpisodes(q, sortBy, filter, pagination, fields)
+		);
 	}
 
 	/// <summary>
@@ -195,10 +204,13 @@ public class SearchApi : BaseApi
 	public async Task<SearchPage<Studio>> SearchStudios(
 		[FromQuery] string? q,
 		[FromQuery] Sort<Studio> sortBy,
+		[FromQuery] Filter<Studio>? filter,
 		[FromQuery] SearchPagination pagination,
 		[FromQuery] Include<Studio> fields
 	)
 	{
-		return SearchPage(await _searchManager.SearchStudios(q, sortBy, pagination, fields));
+		return SearchPage(
+			await _searchManager.SearchStudios(q, sortBy, filter, pagination, fields)
+		);
 	}
 }

back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs

@@ -0,0 +1,52 @@
+using System.ComponentModel.DataAnnotations;
+using Kyoo.Abstractions.Models.Utils;
+using static System.Text.Json.JsonNamingPolicy;
+
+namespace Kyoo.Meiliseach;
+
+internal static class FilterExtensionMethods
+{
+	public static string? CreateMeilisearchFilter<T>(this Filter<T>? filter)
+	{
+		return filter switch
+		{
+			Filter<T>.And and
+				=> $"({and.First.CreateMeilisearchFilter()}) AND ({and.Second.CreateMeilisearchFilter()})",
+			Filter<T>.Or or
+				=> $"({or.First.CreateMeilisearchFilter()}) OR ({or.Second.CreateMeilisearchFilter()})",
+			Filter<T>.Gt gt => CreateBasicFilterString(gt.Property, ">", gt.Value),
+			Filter<T>.Lt lt => CreateBasicFilterString(lt.Property, "<", lt.Value),
+			Filter<T>.Ge ge => CreateBasicFilterString(ge.Property, ">=", ge.Value),
+			Filter<T>.Le le => CreateBasicFilterString(le.Property, "<=", le.Value),
+			Filter<T>.Eq eq => CreateBasicFilterString(eq.Property, "=", eq.Value),
+			Filter<T>.Has has => CreateBasicFilterString(has.Property, "=", has.Value),
+			Filter<T>.Ne ne => CreateBasicFilterString(ne.Property, "!=", ne.Value),
+			Filter<T>.Not not => $"NOT ({not.Filter.CreateMeilisearchFilter()})",
+			Filter<T>.CmpRandom
+				=> throw new ValidationException("Random comparison is not supported."),
+			_ => null
+		};
+	}
+
+	private static string CreateBasicFilterString(string property, string @operator, object? value)
+	{
+		return $"{CamelCase.ConvertName(property)} {@operator} {value.InMeilsearchFilterFormat()}";
+	}
+
+	private static object? InMeilsearchFilterFormat(this object? value)
+	{
+		return value switch
+		{
+			null => null,
+			string s => s.Any(char.IsWhiteSpace) ? $"\"{s.Replace("\"", "\\\"")}\"" : s,
+			DateTimeOffset dateTime => dateTime.ToUnixTimeSeconds(),
+			DateOnly date => date.ToUnixTimeSeconds(),
+			_ => value
+		};
+	}
+
+	public static long ToUnixTimeSeconds(this DateOnly date)
+	{
+		return new DateTimeOffset(date.ToDateTime(new TimeOnly())).ToUnixTimeSeconds();
+	}
+}

back/src/Kyoo.Meilisearch/MeiliSync.cs

@@ -16,6 +16,7 @@
 // You should have received a copy of the GNU General Public License
 // along with Kyoo. If not, see <https://www.gnu.org/licenses/>.
 
+using System.Collections;
 using System.Dynamic;
 using System.Reflection;
 using Kyoo.Abstractions.Controllers;
@@ -60,7 +61,10 @@ public class MeiliSync
 			var dictionary = (IDictionary<string, object?>)expando;
 
 			foreach (PropertyInfo property in item.GetType().GetProperties())
-				dictionary.Add(CamelCase.ConvertName(property.Name), property.GetValue(item));
+				dictionary.Add(
+					CamelCase.ConvertName(property.Name),
+					ConvertToMeilisearchFormat(property.GetValue(item))
+				);
 			dictionary.Add("ref", $"{kind}-{item.Id}");
 			expando.kind = kind;
 			return _client.Index(index).AddDocumentsAsync(new[] { expando });
@@ -76,4 +80,33 @@ public class MeiliSync
 		}
 		return _client.Index(index).DeleteOneDocumentAsync(id.ToString());
 	}
+
+	private object? ConvertToMeilisearchFormat(object? value)
+	{
+		return value switch
+		{
+			null => null,
+			string => value,
+			Enum => value.ToString(),
+			IEnumerable enumerable
+				=> enumerable.Cast<object>().Select(ConvertToMeilisearchFormat).ToArray(),
+			DateTimeOffset dateTime => dateTime.ToUnixTimeSeconds(),
+			DateOnly date => date.ToUnixTimeSeconds(),
+			_ => value
+		};
+	}
+
+	public async Task SyncEverything(ILibraryManager database)
+	{
+		foreach (Movie movie in await database.Movies.GetAll(limit: 0))
+			await CreateOrUpdate("items", movie, nameof(Movie));
+		foreach (Show show in await database.Shows.GetAll(limit: 0))
+			await CreateOrUpdate("items", show, nameof(Show));
+		foreach (Collection collection in await database.Collections.GetAll(limit: 0))
+			await CreateOrUpdate("items", collection, nameof(Collection));
+		foreach (Episode episode in await database.Episodes.GetAll(limit: 0))
+			await CreateOrUpdate(nameof(Episode), episode);
+		foreach (Studio studio in await database.Studios.GetAll(limit: 0))
+			await CreateOrUpdate(nameof(Studio), studio);
+	}
 }

back/src/Kyoo.Meilisearch/MeilisearchModule.cs

@@ -49,6 +49,8 @@ public static class MeilisearchModule
 						CamelCase.ConvertName(nameof(Movie.Genres)),
 						CamelCase.ConvertName(nameof(Movie.Status)),
 						CamelCase.ConvertName(nameof(Movie.AirDate)),
+						CamelCase.ConvertName(nameof(Show.StartAir)),
+						CamelCase.ConvertName(nameof(Show.EndAir)),
 						CamelCase.ConvertName(nameof(Movie.StudioId)),
 						"kind"
 					},
@@ -117,11 +119,6 @@ public static class MeilisearchModule
 			},
 		};
 
-	/// <summary>
-	/// Init meilisearch indexes.
-	/// </summary>
-	/// <param name="provider">The service list to retrieve the meilisearch client</param>
-	/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
 	public static async Task Initialize(IServiceProvider provider)
 	{
 		MeilisearchClient client = provider.GetRequiredService<MeilisearchClient>();
@@ -131,6 +128,13 @@ public static class MeilisearchModule
 		await _CreateIndex(client, nameof(Studio), false);
 	}
 
+	public static async Task SyncDatabase(IServiceProvider provider)
+	{
+		await using AsyncServiceScope scope = provider.CreateAsyncScope();
+		ILibraryManager database = scope.ServiceProvider.GetRequiredService<ILibraryManager>();
+		await scope.ServiceProvider.GetRequiredService<MeiliSync>().SyncEverything(database);
+	}
+
 	private static async Task _CreateIndex(MeilisearchClient client, string index, bool hasKind)
 	{
 		TaskInfo task = await client.CreateIndexAsync(

back/src/Kyoo.Meilisearch/SearchManager.cs

@@ -99,66 +99,125 @@ public class SearchManager : ISearchManager
 	public Task<SearchPage<ILibraryItem>.SearchResult> SearchItems(
 		string? query,
 		Sort<ILibraryItem> sortBy,
+		Filter<ILibraryItem>? filter,
 		SearchPagination pagination,
 		Include<ILibraryItem>? include = default
 	)
 	{
-		return _Search("items", query, null, sortBy, pagination, include);
+		return _Search(
+			"items",
+			query,
+			filter.CreateMeilisearchFilter(),
+			sortBy,
+			pagination,
+			include
+		);
 	}
 
 	/// <inheritdoc/>
 	public Task<SearchPage<Movie>.SearchResult> SearchMovies(
 		string? query,
 		Sort<Movie> sortBy,
+		Filter<ILibraryItem>? filter,
 		SearchPagination pagination,
 		Include<Movie>? include = default
 	)
 	{
-		return _Search("items", query, $"kind = {nameof(Movie)}", sortBy, pagination, include);
+		return _Search(
+			"items",
+			query,
+			_CreateMediaTypeFilter<Movie>(filter),
+			sortBy,
+			pagination,
+			include
+		);
 	}
 
 	/// <inheritdoc/>
 	public Task<SearchPage<Show>.SearchResult> SearchShows(
 		string? query,
 		Sort<Show> sortBy,
+		Filter<ILibraryItem>? filter,
 		SearchPagination pagination,
 		Include<Show>? include = default
 	)
 	{
-		return _Search("items", query, $"kind = {nameof(Show)}", sortBy, pagination, include);
+		return _Search(
+			"items",
+			query,
+			_CreateMediaTypeFilter<Show>(filter),
+			sortBy,
+			pagination,
+			include
+		);
 	}
 
 	/// <inheritdoc/>
 	public Task<SearchPage<Collection>.SearchResult> SearchCollections(
 		string? query,
 		Sort<Collection> sortBy,
+		Filter<ILibraryItem>? filter,
 		SearchPagination pagination,
 		Include<Collection>? include = default
 	)
 	{
-		return _Search("items", query, $"kind = {nameof(Collection)}", sortBy, pagination, include);
+		return _Search(
+			"items",
+			query,
+			_CreateMediaTypeFilter<Collection>(filter),
+			sortBy,
+			pagination,
+			include
+		);
 	}
 
 	/// <inheritdoc/>
 	public Task<SearchPage<Episode>.SearchResult> SearchEpisodes(
 		string? query,
 		Sort<Episode> sortBy,
+		Filter<Episode>? filter,
 		SearchPagination pagination,
 		Include<Episode>? include = default
 	)
 	{
-		return _Search(nameof(Episode), query, null, sortBy, pagination, include);
+		return _Search(
+			nameof(Episode),
+			query,
+			filter.CreateMeilisearchFilter(),
+			sortBy,
+			pagination,
+			include
+		);
 	}
 
 	/// <inheritdoc/>
 	public Task<SearchPage<Studio>.SearchResult> SearchStudios(
 		string? query,
 		Sort<Studio> sortBy,
+		Filter<Studio>? filter,
 		SearchPagination pagination,
 		Include<Studio>? include = default
 	)
 	{
-		return _Search(nameof(Studio), query, null, sortBy, pagination, include);
+		return _Search(
+			nameof(Studio),
+			query,
+			filter.CreateMeilisearchFilter(),
+			sortBy,
+			pagination,
+			include
+		);
+	}
+
+	private string _CreateMediaTypeFilter<T>(Filter<ILibraryItem>? filter)
+		where T : ILibraryItem
+	{
+		string filterString = $"kind = {typeof(T).Name}";
+		if (filter is not null)
+		{
+			filterString += $" AND ({filter.CreateMeilisearchFilter()})";
+		}
+		return filterString;
 	}
 
 	private class IdResource