From e7099c9c19ce256d2644b5c1670733dfb377b2ce Mon Sep 17 00:00:00 2001 From: Scott Merchant Date: Sun, 9 Jun 2024 17:08:10 +0930 Subject: [PATCH 01/13] Add filter to meilisearch in backend --- .../Controllers/ISearchManager.cs | 3 +++ .../Kyoo.Core/Views/Resources/SearchApi.cs | 11 +++++--- .../FilterExtensionMethods.cs | 26 +++++++++++++++++++ back/src/Kyoo.Meilisearch/SearchManager.cs | 9 ++++--- 4 files changed, 42 insertions(+), 7 deletions(-) create mode 100644 back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs diff --git a/back/src/Kyoo.Abstractions/Controllers/ISearchManager.cs b/back/src/Kyoo.Abstractions/Controllers/ISearchManager.cs index e6175018..ed210a52 100644 --- a/back/src/Kyoo.Abstractions/Controllers/ISearchManager.cs +++ b/back/src/Kyoo.Abstractions/Controllers/ISearchManager.cs @@ -38,6 +38,7 @@ public interface ISearchManager public Task.SearchResult> SearchItems( string? query, Sort sortBy, + Filter? filter, SearchPagination pagination, Include? include = default ); @@ -98,6 +99,7 @@ public interface ISearchManager public Task.SearchResult> SearchEpisodes( string? query, Sort sortBy, + Filter? filter, SearchPagination pagination, Include? include = default ); @@ -113,6 +115,7 @@ public interface ISearchManager public Task.SearchResult> SearchStudios( string? query, Sort sortBy, + Filter? filter, SearchPagination pagination, Include? include = default ); diff --git a/back/src/Kyoo.Core/Views/Resources/SearchApi.cs b/back/src/Kyoo.Core/Views/Resources/SearchApi.cs index b3754d3c..323efd07 100644 --- a/back/src/Kyoo.Core/Views/Resources/SearchApi.cs +++ b/back/src/Kyoo.Core/Views/Resources/SearchApi.cs @@ -44,7 +44,7 @@ public class SearchApi : BaseApi _searchManager = searchManager; } - // TODO: add filters and facets + // TODO: add facets /// /// Search collections @@ -143,11 +143,12 @@ public class SearchApi : BaseApi public async Task> SearchItems( [FromQuery] string? q, [FromQuery] Sort sortBy, + [FromQuery] Filter? filter, [FromQuery] SearchPagination pagination, [FromQuery] Include fields ) { - return SearchPage(await _searchManager.SearchItems(q, sortBy, pagination, fields)); + return SearchPage(await _searchManager.SearchItems(q, sortBy, filter, pagination, fields)); } /// @@ -169,11 +170,12 @@ public class SearchApi : BaseApi public async Task> SearchEpisodes( [FromQuery] string? q, [FromQuery] Sort sortBy, + [FromQuery] Filter? filter, [FromQuery] SearchPagination pagination, [FromQuery] Include fields ) { - return SearchPage(await _searchManager.SearchEpisodes(q, sortBy, pagination, fields)); + return SearchPage(await _searchManager.SearchEpisodes(q, sortBy, filter, pagination, fields)); } /// @@ -195,10 +197,11 @@ public class SearchApi : BaseApi public async Task> SearchStudios( [FromQuery] string? q, [FromQuery] Sort sortBy, + [FromQuery] Filter? filter, [FromQuery] SearchPagination pagination, [FromQuery] Include fields ) { - return SearchPage(await _searchManager.SearchStudios(q, sortBy, pagination, fields)); + return SearchPage(await _searchManager.SearchStudios(q, sortBy, filter, pagination, fields)); } } diff --git a/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs b/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs new file mode 100644 index 00000000..caa36f6a --- /dev/null +++ b/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs @@ -0,0 +1,26 @@ +using System.ComponentModel.DataAnnotations; +using Kyoo.Abstractions.Models.Utils; + +namespace Kyoo.Meiliseach; + +public static class FilterExtensionMethods +{ + public static string? CreateMeilisearchFilter(this Filter? filter) + { + return filter switch + { + Filter.And and => $"({and.First.CreateMeilisearchFilter()}) AND ({and.Second.CreateMeilisearchFilter()})", + Filter.Or or => $"({or.First.CreateMeilisearchFilter()}) OR ({or.Second.CreateMeilisearchFilter()})", + Filter.Gt gt => $"{gt.Property} > {gt.Value}", + Filter.Lt lt => $"{lt.Property} < {lt.Value}", + Filter.Ge ge => $"{ge.Property} >= {ge.Value}", + Filter.Le le => $"{le.Property} <= {le.Value}", + Filter.Eq eq => $"{eq.Property} = {eq.Value}", + Filter.Has has => $"{has.Property} = {has.Value}", + Filter.Ne ne => $"{ne.Property} != {ne.Value}", + Filter.Not not => $"NOT ({not.Filter.CreateMeilisearchFilter()})", + Filter.CmpRandom => throw new ValidationException("Random comparison is not supported."), + _ => null + }; + } +} diff --git a/back/src/Kyoo.Meilisearch/SearchManager.cs b/back/src/Kyoo.Meilisearch/SearchManager.cs index 07b5ed11..cb7c8235 100644 --- a/back/src/Kyoo.Meilisearch/SearchManager.cs +++ b/back/src/Kyoo.Meilisearch/SearchManager.cs @@ -99,11 +99,12 @@ public class SearchManager : ISearchManager public Task.SearchResult> SearchItems( string? query, Sort sortBy, + Filter? filter, SearchPagination pagination, Include? include = default ) { - return _Search("items", query, null, sortBy, pagination, include); + return _Search("items", query, filter.CreateMeilisearchFilter(), sortBy, pagination, include); } /// @@ -143,22 +144,24 @@ public class SearchManager : ISearchManager public Task.SearchResult> SearchEpisodes( string? query, Sort sortBy, + Filter? filter, SearchPagination pagination, Include? include = default ) { - return _Search(nameof(Episode), query, null, sortBy, pagination, include); + return _Search(nameof(Episode), query, filter.CreateMeilisearchFilter(), sortBy, pagination, include); } /// public Task.SearchResult> SearchStudios( string? query, Sort sortBy, + Filter? filter, SearchPagination pagination, Include? include = default ) { - return _Search(nameof(Studio), query, null, sortBy, pagination, include); + return _Search(nameof(Studio), query, filter.CreateMeilisearchFilter(), sortBy, pagination, include); } private class IdResource From ac38f2d6ebf276b4f644c895cd882abd58c07764 Mon Sep 17 00:00:00 2001 From: Scott Merchant Date: Sun, 9 Jun 2024 19:22:24 +0930 Subject: [PATCH 02/13] fix linting --- .../Kyoo.Core/Views/Resources/SearchApi.cs | 8 ++++-- .../FilterExtensionMethods.cs | 9 ++++--- back/src/Kyoo.Meilisearch/SearchManager.cs | 27 ++++++++++++++++--- 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/back/src/Kyoo.Core/Views/Resources/SearchApi.cs b/back/src/Kyoo.Core/Views/Resources/SearchApi.cs index 323efd07..d34af5ef 100644 --- a/back/src/Kyoo.Core/Views/Resources/SearchApi.cs +++ b/back/src/Kyoo.Core/Views/Resources/SearchApi.cs @@ -175,7 +175,9 @@ public class SearchApi : BaseApi [FromQuery] Include fields ) { - return SearchPage(await _searchManager.SearchEpisodes(q, sortBy, filter, pagination, fields)); + return SearchPage( + await _searchManager.SearchEpisodes(q, sortBy, filter, pagination, fields) + ); } /// @@ -202,6 +204,8 @@ public class SearchApi : BaseApi [FromQuery] Include fields ) { - return SearchPage(await _searchManager.SearchStudios(q, sortBy, filter, pagination, fields)); + return SearchPage( + await _searchManager.SearchStudios(q, sortBy, filter, pagination, fields) + ); } } diff --git a/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs b/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs index caa36f6a..50940668 100644 --- a/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs +++ b/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs @@ -9,8 +9,10 @@ public static class FilterExtensionMethods { return filter switch { - Filter.And and => $"({and.First.CreateMeilisearchFilter()}) AND ({and.Second.CreateMeilisearchFilter()})", - Filter.Or or => $"({or.First.CreateMeilisearchFilter()}) OR ({or.Second.CreateMeilisearchFilter()})", + Filter.And and + => $"({and.First.CreateMeilisearchFilter()}) AND ({and.Second.CreateMeilisearchFilter()})", + Filter.Or or + => $"({or.First.CreateMeilisearchFilter()}) OR ({or.Second.CreateMeilisearchFilter()})", Filter.Gt gt => $"{gt.Property} > {gt.Value}", Filter.Lt lt => $"{lt.Property} < {lt.Value}", Filter.Ge ge => $"{ge.Property} >= {ge.Value}", @@ -19,7 +21,8 @@ public static class FilterExtensionMethods Filter.Has has => $"{has.Property} = {has.Value}", Filter.Ne ne => $"{ne.Property} != {ne.Value}", Filter.Not not => $"NOT ({not.Filter.CreateMeilisearchFilter()})", - Filter.CmpRandom => throw new ValidationException("Random comparison is not supported."), + Filter.CmpRandom + => throw new ValidationException("Random comparison is not supported."), _ => null }; } diff --git a/back/src/Kyoo.Meilisearch/SearchManager.cs b/back/src/Kyoo.Meilisearch/SearchManager.cs index cb7c8235..ae7fc133 100644 --- a/back/src/Kyoo.Meilisearch/SearchManager.cs +++ b/back/src/Kyoo.Meilisearch/SearchManager.cs @@ -104,7 +104,14 @@ public class SearchManager : ISearchManager Include? include = default ) { - return _Search("items", query, filter.CreateMeilisearchFilter(), sortBy, pagination, include); + return _Search( + "items", + query, + filter.CreateMeilisearchFilter(), + sortBy, + pagination, + include + ); } /// @@ -149,7 +156,14 @@ public class SearchManager : ISearchManager Include? include = default ) { - return _Search(nameof(Episode), query, filter.CreateMeilisearchFilter(), sortBy, pagination, include); + return _Search( + nameof(Episode), + query, + filter.CreateMeilisearchFilter(), + sortBy, + pagination, + include + ); } /// @@ -161,7 +175,14 @@ public class SearchManager : ISearchManager Include? include = default ) { - return _Search(nameof(Studio), query, filter.CreateMeilisearchFilter(), sortBy, pagination, include); + return _Search( + nameof(Studio), + query, + filter.CreateMeilisearchFilter(), + sortBy, + pagination, + include + ); } private class IdResource From fec468401fe56bae963d83a534e1d2f45c3be3b3 Mon Sep 17 00:00:00 2001 From: Scott Merchant Date: Sun, 9 Jun 2024 22:00:44 +0930 Subject: [PATCH 03/13] fix casing for meilisearch filter string --- .../Kyoo.Meilisearch/FilterExtensionMethods.cs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs b/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs index 50940668..de68b10d 100644 --- a/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs +++ b/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs @@ -1,5 +1,6 @@ using System.ComponentModel.DataAnnotations; using Kyoo.Abstractions.Models.Utils; +using static System.Text.Json.JsonNamingPolicy; namespace Kyoo.Meiliseach; @@ -13,13 +14,13 @@ public static class FilterExtensionMethods => $"({and.First.CreateMeilisearchFilter()}) AND ({and.Second.CreateMeilisearchFilter()})", Filter.Or or => $"({or.First.CreateMeilisearchFilter()}) OR ({or.Second.CreateMeilisearchFilter()})", - Filter.Gt gt => $"{gt.Property} > {gt.Value}", - Filter.Lt lt => $"{lt.Property} < {lt.Value}", - Filter.Ge ge => $"{ge.Property} >= {ge.Value}", - Filter.Le le => $"{le.Property} <= {le.Value}", - Filter.Eq eq => $"{eq.Property} = {eq.Value}", - Filter.Has has => $"{has.Property} = {has.Value}", - Filter.Ne ne => $"{ne.Property} != {ne.Value}", + Filter.Gt gt => $"{CamelCase.ConvertName(gt.Property)} > '{gt.Value}'", + Filter.Lt lt => $"{CamelCase.ConvertName(lt.Property)} < '{lt.Value}'", + Filter.Ge ge => $"{CamelCase.ConvertName(ge.Property)} >= '{ge.Value}'", + Filter.Le le => $"{CamelCase.ConvertName(le.Property)} <= '{le.Value}'", + Filter.Eq eq => $"{CamelCase.ConvertName(eq.Property)} = '{eq.Value}'", + Filter.Has has => $"{CamelCase.ConvertName(has.Property)} = '{has.Value}'", + Filter.Ne ne => $"{CamelCase.ConvertName(ne.Property)} != '{ne.Value}'", Filter.Not not => $"NOT ({not.Filter.CreateMeilisearchFilter()})", Filter.CmpRandom => throw new ValidationException("Random comparison is not supported."), From abffdedf3eb1489067d84ba79284edd5b9e250fe Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 9 Jun 2024 16:23:30 +0200 Subject: [PATCH 04/13] Fix dateonly parsing of filters --- back/src/Kyoo.Abstractions/Models/Utils/Filter.cs | 8 +++++--- back/src/Kyoo.Meilisearch/MeilisearchModule.cs | 2 ++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/back/src/Kyoo.Abstractions/Models/Utils/Filter.cs b/back/src/Kyoo.Abstractions/Models/Utils/Filter.cs index 430c1218..4d832928 100644 --- a/back/src/Kyoo.Abstractions/Models/Utils/Filter.cs +++ b/back/src/Kyoo.Abstractions/Models/Utils/Filter.cs @@ -215,14 +215,16 @@ public abstract record Filter : Filter }); } - if (type == typeof(DateTime)) + if (type == typeof(DateTime) || type == typeof(DateOnly)) { return from year in Parse.Digit.Repeat(4).Text().Select(int.Parse) from yd in Parse.Char('-') - from mouth in Parse.Digit.Repeat(2).Text().Select(int.Parse) + from month in Parse.Digit.Repeat(2).Text().Select(int.Parse) from md in Parse.Char('-') from day in Parse.Digit.Repeat(2).Text().Select(int.Parse) - select new DateTime(year, mouth, day) as object; + select type == typeof(DateTime) + ? new DateTime(year, month, day) as object + : new DateOnly(year, month, day) as object; } if (typeof(IEnumerable).IsAssignableFrom(type)) diff --git a/back/src/Kyoo.Meilisearch/MeilisearchModule.cs b/back/src/Kyoo.Meilisearch/MeilisearchModule.cs index f2e971b4..a290a366 100644 --- a/back/src/Kyoo.Meilisearch/MeilisearchModule.cs +++ b/back/src/Kyoo.Meilisearch/MeilisearchModule.cs @@ -49,6 +49,8 @@ public static class MeilisearchModule CamelCase.ConvertName(nameof(Movie.Genres)), CamelCase.ConvertName(nameof(Movie.Status)), CamelCase.ConvertName(nameof(Movie.AirDate)), + CamelCase.ConvertName(nameof(Show.StartAir)), + CamelCase.ConvertName(nameof(Show.EndAir)), CamelCase.ConvertName(nameof(Movie.StudioId)), "kind" }, From 9b8fb7596fa01074e0a7031a448a59716f4b3baa Mon Sep 17 00:00:00 2001 From: Scott Merchant Date: Mon, 10 Jun 2024 15:29:46 +0930 Subject: [PATCH 05/13] Fix format issues --- .../FilterExtensionMethods.cs | 48 +++++++++++++++---- back/src/Kyoo.Meilisearch/MeiliSync.cs | 2 +- 2 files changed, 41 insertions(+), 9 deletions(-) diff --git a/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs b/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs index de68b10d..b9a55503 100644 --- a/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs +++ b/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs @@ -1,10 +1,11 @@ +using System.Collections; using System.ComponentModel.DataAnnotations; using Kyoo.Abstractions.Models.Utils; using static System.Text.Json.JsonNamingPolicy; namespace Kyoo.Meiliseach; -public static class FilterExtensionMethods +internal static class FilterExtensionMethods { public static string? CreateMeilisearchFilter(this Filter? filter) { @@ -14,17 +15,48 @@ public static class FilterExtensionMethods => $"({and.First.CreateMeilisearchFilter()}) AND ({and.Second.CreateMeilisearchFilter()})", Filter.Or or => $"({or.First.CreateMeilisearchFilter()}) OR ({or.Second.CreateMeilisearchFilter()})", - Filter.Gt gt => $"{CamelCase.ConvertName(gt.Property)} > '{gt.Value}'", - Filter.Lt lt => $"{CamelCase.ConvertName(lt.Property)} < '{lt.Value}'", - Filter.Ge ge => $"{CamelCase.ConvertName(ge.Property)} >= '{ge.Value}'", - Filter.Le le => $"{CamelCase.ConvertName(le.Property)} <= '{le.Value}'", - Filter.Eq eq => $"{CamelCase.ConvertName(eq.Property)} = '{eq.Value}'", - Filter.Has has => $"{CamelCase.ConvertName(has.Property)} = '{has.Value}'", - Filter.Ne ne => $"{CamelCase.ConvertName(ne.Property)} != '{ne.Value}'", + Filter.Gt gt => $"{CamelCase.ConvertName(gt.Property)} > {gt.Value.InMeilsearchFilterFormat()}", + Filter.Lt lt => $"{CamelCase.ConvertName(lt.Property)} < {lt.Value.InMeilsearchFilterFormat()}", + Filter.Ge ge => $"{CamelCase.ConvertName(ge.Property)} >= {ge.Value.InMeilsearchFilterFormat()}", + Filter.Le le => $"{CamelCase.ConvertName(le.Property)} <= {le.Value.InMeilsearchFilterFormat()}", + Filter.Eq eq => $"{CamelCase.ConvertName(eq.Property)} = {eq.Value.InMeilsearchFilterFormat()}", + Filter.Has has => $"{CamelCase.ConvertName(has.Property)} = {has.Value.InMeilsearchFilterFormat()}", + Filter.Ne ne => $"{CamelCase.ConvertName(ne.Property)} != {ne.Value.InMeilsearchFilterFormat()}", Filter.Not not => $"NOT ({not.Filter.CreateMeilisearchFilter()})", Filter.CmpRandom => throw new ValidationException("Random comparison is not supported."), _ => null }; } + + private static object? InMeilsearchFilterFormat(this object? value) + { + return value switch + { + null => null, + string s => s.Any(char.IsWhiteSpace) ? $"\"{s}\"" : s, + DateTimeOffset dateTime => dateTime.ToUnixTimeSeconds(), + DateOnly date => date.ToUnixTimeSeconds(), + _ => value + }; + } + + public static object? InMeilisearchFormat(this object? value) + { + return value switch + { + null => null, + string => value, + Enum => value.ToString(), + IEnumerable enumerable => enumerable.Cast().Select(InMeilisearchFormat).ToArray(), + DateTimeOffset dateTime => dateTime.ToUnixTimeSeconds(), + DateOnly date => date.ToUnixTimeSeconds(), + _ => value + }; + } + + private static long ToUnixTimeSeconds(this DateOnly date) + { + return new DateTimeOffset(date.ToDateTime(new TimeOnly())).ToUnixTimeSeconds(); + } } diff --git a/back/src/Kyoo.Meilisearch/MeiliSync.cs b/back/src/Kyoo.Meilisearch/MeiliSync.cs index 0005ef81..39e55c0f 100644 --- a/back/src/Kyoo.Meilisearch/MeiliSync.cs +++ b/back/src/Kyoo.Meilisearch/MeiliSync.cs @@ -60,7 +60,7 @@ public class MeiliSync var dictionary = (IDictionary)expando; foreach (PropertyInfo property in item.GetType().GetProperties()) - dictionary.Add(CamelCase.ConvertName(property.Name), property.GetValue(item)); + dictionary.Add(CamelCase.ConvertName(property.Name), property.GetValue(item).InMeilisearchFormat()); dictionary.Add("ref", $"{kind}-{item.Id}"); expando.kind = kind; return _client.Index(index).AddDocumentsAsync(new[] { expando }); From 7725d2534bed4e512e65836c0bc8fdf868e109cf Mon Sep 17 00:00:00 2001 From: Scott Merchant Date: Mon, 10 Jun 2024 16:16:00 +0930 Subject: [PATCH 06/13] Add filter support to collections, movies, and shows search endpoints --- .../Controllers/ISearchManager.cs | 3 +++ .../Kyoo.Core/Views/Resources/SearchApi.cs | 9 ++++++--- back/src/Kyoo.Meilisearch/SearchManager.cs | 19 ++++++++++++++++--- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/back/src/Kyoo.Abstractions/Controllers/ISearchManager.cs b/back/src/Kyoo.Abstractions/Controllers/ISearchManager.cs index ed210a52..d3985c6e 100644 --- a/back/src/Kyoo.Abstractions/Controllers/ISearchManager.cs +++ b/back/src/Kyoo.Abstractions/Controllers/ISearchManager.cs @@ -54,6 +54,7 @@ public interface ISearchManager public Task.SearchResult> SearchMovies( string? query, Sort sortBy, + Filter? filter, SearchPagination pagination, Include? include = default ); @@ -69,6 +70,7 @@ public interface ISearchManager public Task.SearchResult> SearchShows( string? query, Sort sortBy, + Filter? filter, SearchPagination pagination, Include? include = default ); @@ -84,6 +86,7 @@ public interface ISearchManager public Task.SearchResult> SearchCollections( string? query, Sort sortBy, + Filter? filter, SearchPagination pagination, Include? include = default ); diff --git a/back/src/Kyoo.Core/Views/Resources/SearchApi.cs b/back/src/Kyoo.Core/Views/Resources/SearchApi.cs index d34af5ef..f4eea079 100644 --- a/back/src/Kyoo.Core/Views/Resources/SearchApi.cs +++ b/back/src/Kyoo.Core/Views/Resources/SearchApi.cs @@ -65,11 +65,12 @@ public class SearchApi : BaseApi public async Task> SearchCollections( [FromQuery] string? q, [FromQuery] Sort sortBy, + [FromQuery] Filter? filter, [FromQuery] SearchPagination pagination, [FromQuery] Include fields ) { - return SearchPage(await _searchManager.SearchCollections(q, sortBy, pagination, fields)); + return SearchPage(await _searchManager.SearchCollections(q, sortBy, filter, pagination, fields)); } /// @@ -91,11 +92,12 @@ public class SearchApi : BaseApi public async Task> SearchShows( [FromQuery] string? q, [FromQuery] Sort sortBy, + [FromQuery] Filter? filter, [FromQuery] SearchPagination pagination, [FromQuery] Include fields ) { - return SearchPage(await _searchManager.SearchShows(q, sortBy, pagination, fields)); + return SearchPage(await _searchManager.SearchShows(q, sortBy, filter, pagination, fields)); } /// @@ -117,11 +119,12 @@ public class SearchApi : BaseApi public async Task> SearchMovies( [FromQuery] string? q, [FromQuery] Sort sortBy, + [FromQuery] Filter? filter, [FromQuery] SearchPagination pagination, [FromQuery] Include fields ) { - return SearchPage(await _searchManager.SearchMovies(q, sortBy, pagination, fields)); + return SearchPage(await _searchManager.SearchMovies(q, sortBy, filter, pagination, fields)); } /// diff --git a/back/src/Kyoo.Meilisearch/SearchManager.cs b/back/src/Kyoo.Meilisearch/SearchManager.cs index ae7fc133..058b949f 100644 --- a/back/src/Kyoo.Meilisearch/SearchManager.cs +++ b/back/src/Kyoo.Meilisearch/SearchManager.cs @@ -118,33 +118,36 @@ public class SearchManager : ISearchManager public Task.SearchResult> SearchMovies( string? query, Sort sortBy, + Filter? filter, SearchPagination pagination, Include? include = default ) { - return _Search("items", query, $"kind = {nameof(Movie)}", sortBy, pagination, include); + return _Search("items", query, _CreateMediaTypeFilter(filter), sortBy, pagination, include); } /// public Task.SearchResult> SearchShows( string? query, Sort sortBy, + Filter? filter, SearchPagination pagination, Include? include = default ) { - return _Search("items", query, $"kind = {nameof(Show)}", sortBy, pagination, include); + return _Search("items", query, _CreateMediaTypeFilter(filter), sortBy, pagination, include); } /// public Task.SearchResult> SearchCollections( string? query, Sort sortBy, + Filter? filter, SearchPagination pagination, Include? include = default ) { - return _Search("items", query, $"kind = {nameof(Collection)}", sortBy, pagination, include); + return _Search("items", query, _CreateMediaTypeFilter(filter), sortBy, pagination, include); } /// @@ -185,6 +188,16 @@ public class SearchManager : ISearchManager ); } + private string _CreateMediaTypeFilter(Filter? filter) where T : ILibraryItem + { + string filterString = $"kind = {typeof(T).Name}"; + if (filter is not null) + { + filterString += $" AND ({filter.CreateMeilisearchFilter()})"; + } + return filterString; + } + private class IdResource { public Guid Id { get; set; } From 34573359cb5f928807fa5bb37c896ed077b424c5 Mon Sep 17 00:00:00 2001 From: Scott Merchant Date: Mon, 10 Jun 2024 16:27:55 +0930 Subject: [PATCH 07/13] clean up CreateMeilisearchFilter --- .../FilterExtensionMethods.cs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs b/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs index b9a55503..c4f48b17 100644 --- a/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs +++ b/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs @@ -15,13 +15,13 @@ internal static class FilterExtensionMethods => $"({and.First.CreateMeilisearchFilter()}) AND ({and.Second.CreateMeilisearchFilter()})", Filter.Or or => $"({or.First.CreateMeilisearchFilter()}) OR ({or.Second.CreateMeilisearchFilter()})", - Filter.Gt gt => $"{CamelCase.ConvertName(gt.Property)} > {gt.Value.InMeilsearchFilterFormat()}", - Filter.Lt lt => $"{CamelCase.ConvertName(lt.Property)} < {lt.Value.InMeilsearchFilterFormat()}", - Filter.Ge ge => $"{CamelCase.ConvertName(ge.Property)} >= {ge.Value.InMeilsearchFilterFormat()}", - Filter.Le le => $"{CamelCase.ConvertName(le.Property)} <= {le.Value.InMeilsearchFilterFormat()}", - Filter.Eq eq => $"{CamelCase.ConvertName(eq.Property)} = {eq.Value.InMeilsearchFilterFormat()}", - Filter.Has has => $"{CamelCase.ConvertName(has.Property)} = {has.Value.InMeilsearchFilterFormat()}", - Filter.Ne ne => $"{CamelCase.ConvertName(ne.Property)} != {ne.Value.InMeilsearchFilterFormat()}", + Filter.Gt gt => CreateBasicFilterString(gt.Property, ">", gt.Value), + Filter.Lt lt => CreateBasicFilterString(lt.Property, "<", lt.Value), + Filter.Ge ge => CreateBasicFilterString(ge.Property, ">=", ge.Value), + Filter.Le le => CreateBasicFilterString(le.Property, "<=", le.Value), + Filter.Eq eq => CreateBasicFilterString(eq.Property, "=", eq.Value), + Filter.Has has => CreateBasicFilterString(has.Property, "=", has.Value), + Filter.Ne ne => CreateBasicFilterString(ne.Property, "!=", ne.Value), Filter.Not not => $"NOT ({not.Filter.CreateMeilisearchFilter()})", Filter.CmpRandom => throw new ValidationException("Random comparison is not supported."), @@ -29,6 +29,11 @@ internal static class FilterExtensionMethods }; } + private static string CreateBasicFilterString(string property, string @operator, object? value) + { + return $"{CamelCase.ConvertName(property)} {@operator} {value.InMeilisearchFormat()}"; + } + private static object? InMeilsearchFilterFormat(this object? value) { return value switch From 084ef0d68fb6d411bd894b5467e66dd67a79bbdb Mon Sep 17 00:00:00 2001 From: Scott Merchant Date: Mon, 10 Jun 2024 16:28:20 +0930 Subject: [PATCH 08/13] apply linting --- .../Kyoo.Core/Views/Resources/SearchApi.cs | 4 ++- .../FilterExtensionMethods.cs | 3 +- back/src/Kyoo.Meilisearch/MeiliSync.cs | 5 +++- back/src/Kyoo.Meilisearch/SearchManager.cs | 30 ++++++++++++++++--- 4 files changed, 35 insertions(+), 7 deletions(-) diff --git a/back/src/Kyoo.Core/Views/Resources/SearchApi.cs b/back/src/Kyoo.Core/Views/Resources/SearchApi.cs index f4eea079..14050382 100644 --- a/back/src/Kyoo.Core/Views/Resources/SearchApi.cs +++ b/back/src/Kyoo.Core/Views/Resources/SearchApi.cs @@ -70,7 +70,9 @@ public class SearchApi : BaseApi [FromQuery] Include fields ) { - return SearchPage(await _searchManager.SearchCollections(q, sortBy, filter, pagination, fields)); + return SearchPage( + await _searchManager.SearchCollections(q, sortBy, filter, pagination, fields) + ); } /// diff --git a/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs b/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs index c4f48b17..41b122f2 100644 --- a/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs +++ b/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs @@ -53,7 +53,8 @@ internal static class FilterExtensionMethods null => null, string => value, Enum => value.ToString(), - IEnumerable enumerable => enumerable.Cast().Select(InMeilisearchFormat).ToArray(), + IEnumerable enumerable + => enumerable.Cast().Select(InMeilisearchFormat).ToArray(), DateTimeOffset dateTime => dateTime.ToUnixTimeSeconds(), DateOnly date => date.ToUnixTimeSeconds(), _ => value diff --git a/back/src/Kyoo.Meilisearch/MeiliSync.cs b/back/src/Kyoo.Meilisearch/MeiliSync.cs index 39e55c0f..8930c0b8 100644 --- a/back/src/Kyoo.Meilisearch/MeiliSync.cs +++ b/back/src/Kyoo.Meilisearch/MeiliSync.cs @@ -60,7 +60,10 @@ public class MeiliSync var dictionary = (IDictionary)expando; foreach (PropertyInfo property in item.GetType().GetProperties()) - dictionary.Add(CamelCase.ConvertName(property.Name), property.GetValue(item).InMeilisearchFormat()); + dictionary.Add( + CamelCase.ConvertName(property.Name), + property.GetValue(item).InMeilisearchFormat() + ); dictionary.Add("ref", $"{kind}-{item.Id}"); expando.kind = kind; return _client.Index(index).AddDocumentsAsync(new[] { expando }); diff --git a/back/src/Kyoo.Meilisearch/SearchManager.cs b/back/src/Kyoo.Meilisearch/SearchManager.cs index 058b949f..7f7ccaf4 100644 --- a/back/src/Kyoo.Meilisearch/SearchManager.cs +++ b/back/src/Kyoo.Meilisearch/SearchManager.cs @@ -123,7 +123,14 @@ public class SearchManager : ISearchManager Include? include = default ) { - return _Search("items", query, _CreateMediaTypeFilter(filter), sortBy, pagination, include); + return _Search( + "items", + query, + _CreateMediaTypeFilter(filter), + sortBy, + pagination, + include + ); } /// @@ -135,7 +142,14 @@ public class SearchManager : ISearchManager Include? include = default ) { - return _Search("items", query, _CreateMediaTypeFilter(filter), sortBy, pagination, include); + return _Search( + "items", + query, + _CreateMediaTypeFilter(filter), + sortBy, + pagination, + include + ); } /// @@ -147,7 +161,14 @@ public class SearchManager : ISearchManager Include? include = default ) { - return _Search("items", query, _CreateMediaTypeFilter(filter), sortBy, pagination, include); + return _Search( + "items", + query, + _CreateMediaTypeFilter(filter), + sortBy, + pagination, + include + ); } /// @@ -188,7 +209,8 @@ public class SearchManager : ISearchManager ); } - private string _CreateMediaTypeFilter(Filter? filter) where T : ILibraryItem + private string _CreateMediaTypeFilter(Filter? filter) + where T : ILibraryItem { string filterString = $"kind = {typeof(T).Name}"; if (filter is not null) From 19871e28ba66ee8826d450010bf1a55e6bb1721b Mon Sep 17 00:00:00 2001 From: Scott Merchant Date: Mon, 10 Jun 2024 18:34:01 +0930 Subject: [PATCH 09/13] escape quotes for strings with spaces --- back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs b/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs index 41b122f2..8d6e5c60 100644 --- a/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs +++ b/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs @@ -39,7 +39,7 @@ internal static class FilterExtensionMethods return value switch { null => null, - string s => s.Any(char.IsWhiteSpace) ? $"\"{s}\"" : s, + string s => s.Any(char.IsWhiteSpace) ? $"\\\"{s}\\\"" : s, DateTimeOffset dateTime => dateTime.ToUnixTimeSeconds(), DateOnly date => date.ToUnixTimeSeconds(), _ => value From 505c51923fa32e5f96ea7525ddac3f7927a290ba Mon Sep 17 00:00:00 2001 From: Scott Merchant Date: Mon, 10 Jun 2024 18:34:13 +0930 Subject: [PATCH 10/13] fix typo in method name --- back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs b/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs index 8d6e5c60..56d68a41 100644 --- a/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs +++ b/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs @@ -31,7 +31,7 @@ internal static class FilterExtensionMethods private static string CreateBasicFilterString(string property, string @operator, object? value) { - return $"{CamelCase.ConvertName(property)} {@operator} {value.InMeilisearchFormat()}"; + return $"{CamelCase.ConvertName(property)} {@operator} {value.InMeilsearchFilterFormat()}"; } private static object? InMeilsearchFilterFormat(this object? value) From d683943eb3289f4744eadcb8783bb98733da6909 Mon Sep 17 00:00:00 2001 From: Scott Merchant Date: Mon, 10 Jun 2024 18:34:35 +0930 Subject: [PATCH 11/13] move InMeilisearchFormat to MeiliSync.cs --- .../Kyoo.Meilisearch/FilterExtensionMethods.cs | 18 +----------------- back/src/Kyoo.Meilisearch/MeiliSync.cs | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs b/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs index 56d68a41..fde9a00f 100644 --- a/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs +++ b/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs @@ -1,4 +1,3 @@ -using System.Collections; using System.ComponentModel.DataAnnotations; using Kyoo.Abstractions.Models.Utils; using static System.Text.Json.JsonNamingPolicy; @@ -46,22 +45,7 @@ internal static class FilterExtensionMethods }; } - public static object? InMeilisearchFormat(this object? value) - { - return value switch - { - null => null, - string => value, - Enum => value.ToString(), - IEnumerable enumerable - => enumerable.Cast().Select(InMeilisearchFormat).ToArray(), - DateTimeOffset dateTime => dateTime.ToUnixTimeSeconds(), - DateOnly date => date.ToUnixTimeSeconds(), - _ => value - }; - } - - private static long ToUnixTimeSeconds(this DateOnly date) + public static long ToUnixTimeSeconds(this DateOnly date) { return new DateTimeOffset(date.ToDateTime(new TimeOnly())).ToUnixTimeSeconds(); } diff --git a/back/src/Kyoo.Meilisearch/MeiliSync.cs b/back/src/Kyoo.Meilisearch/MeiliSync.cs index 8930c0b8..500fd974 100644 --- a/back/src/Kyoo.Meilisearch/MeiliSync.cs +++ b/back/src/Kyoo.Meilisearch/MeiliSync.cs @@ -16,6 +16,7 @@ // You should have received a copy of the GNU General Public License // along with Kyoo. If not, see . +using System.Collections; using System.Dynamic; using System.Reflection; using Kyoo.Abstractions.Controllers; @@ -62,7 +63,7 @@ public class MeiliSync foreach (PropertyInfo property in item.GetType().GetProperties()) dictionary.Add( CamelCase.ConvertName(property.Name), - property.GetValue(item).InMeilisearchFormat() + ConvertToMeilisearchFormat(property.GetValue(item)) ); dictionary.Add("ref", $"{kind}-{item.Id}"); expando.kind = kind; @@ -79,4 +80,19 @@ public class MeiliSync } return _client.Index(index).DeleteOneDocumentAsync(id.ToString()); } + + private object? ConvertToMeilisearchFormat(object? value) + { + return value switch + { + null => null, + string => value, + Enum => value.ToString(), + IEnumerable enumerable + => enumerable.Cast().Select(ConvertToMeilisearchFormat).ToArray(), + DateTimeOffset dateTime => dateTime.ToUnixTimeSeconds(), + DateOnly date => date.ToUnixTimeSeconds(), + _ => value + }; + } } From f22ba144c7ee9215bd91aacb6e941ae7b77dd6e4 Mon Sep 17 00:00:00 2001 From: Scott Merchant Date: Wed, 12 Jun 2024 19:12:30 +0930 Subject: [PATCH 12/13] Fix string escaping --- back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs b/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs index fde9a00f..d32d2a63 100644 --- a/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs +++ b/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs @@ -38,7 +38,7 @@ internal static class FilterExtensionMethods return value switch { null => null, - string s => s.Any(char.IsWhiteSpace) ? $"\\\"{s}\\\"" : s, + string s => s.Any(char.IsWhiteSpace) ? $"\"{s.Replace("\"", "\\\"")}\"" : s, DateTimeOffset dateTime => dateTime.ToUnixTimeSeconds(), DateOnly date => date.ToUnixTimeSeconds(), _ => value From 233a498f6b5a85c0d6627826fcca15a68bd01168 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 16 Jun 2024 15:17:37 +0000 Subject: [PATCH 13/13] Migrate meilisearch on startup --- .env.example | 7 ++++++- back/src/Kyoo.Core/Program.cs | 17 ++++++++++++----- back/src/Kyoo.Meilisearch/MeiliSync.cs | 14 ++++++++++++++ back/src/Kyoo.Meilisearch/MeilisearchModule.cs | 12 +++++++----- 4 files changed, 39 insertions(+), 11 deletions(-) diff --git a/.env.example b/.env.example index 2298b20a..f15219f1 100644 --- a/.env.example +++ b/.env.example @@ -64,7 +64,7 @@ OIDC_SERVICE_PROFILE=https://url-of-the-profile-endpoint-of-the-oidc-service.com OIDC_SERVICE_SCOPE="the list of scopes space separeted like email identity" # Token authentication method as seen in https://openid.net/specs/openid-connect-core-1_0.html#ClientAuthentication # Supported values: ClientSecretBasic (default) or ClientSecretPost -# If in doupt, leave this empty. +# If in doubt, leave this empty. OIDC_SERVICE_AUTHMETHOD=ClientSecretBasic # on the previous list, service is the internal name of your service, you can add as many as you want. @@ -81,6 +81,11 @@ POSTGRES_DB=kyooDB POSTGRES_SERVER=postgres POSTGRES_PORT=5432 +# Read by the api container to know if it should run meilisearch's migrations/sync +# and download missing images. This is a good idea to only have one instance with this on +# Note: it does not run postgres migrations, use the migration container for that. +RUN_MIGRATIONS=true + MEILI_HOST="http://meilisearch:7700" MEILI_MASTER_KEY="ghvjkgisbgkbgskegblfqbgjkebbhgwkjfb" diff --git a/back/src/Kyoo.Core/Program.cs b/back/src/Kyoo.Core/Program.cs index c95c9cc3..49afbd5c 100644 --- a/back/src/Kyoo.Core/Program.cs +++ b/back/src/Kyoo.Core/Program.cs @@ -17,6 +17,7 @@ // along with Kyoo. If not, see . using System; +using Kyoo.Abstractions.Controllers; using Kyoo.Authentication; using Kyoo.Core; using Kyoo.Core.Controllers; @@ -27,6 +28,7 @@ using Kyoo.RabbitMq; using Kyoo.Swagger; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Serilog; using Serilog.Events; @@ -94,12 +96,17 @@ app.MapControllers(); app.Services.GetRequiredService(); app.Services.GetRequiredService(); -await using (AsyncServiceScope scope = app.Services.CreateAsyncScope()) +// Only run sync on the main instance +if (app.Configuration.GetValue("RUN_MIGRATIONS", true)) { - await MeilisearchModule.Initialize(scope.ServiceProvider); + await using (AsyncServiceScope scope = app.Services.CreateAsyncScope()) + { + await MeilisearchModule.Initialize(app.Services); + } + + // The methods takes care of creating a scope and will download images on the background. + _ = MeilisearchModule.SyncDatabase(app.Services); + _ = MiscRepository.DownloadMissingImages(app.Services); } -// The methods takes care of creating a scope and will download images on the background. -_ = MiscRepository.DownloadMissingImages(app.Services); - await app.RunAsync(Environment.GetEnvironmentVariable("KYOO_BIND_URL") ?? "http://*:5000"); diff --git a/back/src/Kyoo.Meilisearch/MeiliSync.cs b/back/src/Kyoo.Meilisearch/MeiliSync.cs index 500fd974..e995ce6a 100644 --- a/back/src/Kyoo.Meilisearch/MeiliSync.cs +++ b/back/src/Kyoo.Meilisearch/MeiliSync.cs @@ -95,4 +95,18 @@ public class MeiliSync _ => value }; } + + public async Task SyncEverything(ILibraryManager database) + { + foreach (Movie movie in await database.Movies.GetAll(limit: 0)) + await CreateOrUpdate("items", movie, nameof(Movie)); + foreach (Show show in await database.Shows.GetAll(limit: 0)) + await CreateOrUpdate("items", show, nameof(Show)); + foreach (Collection collection in await database.Collections.GetAll(limit: 0)) + await CreateOrUpdate("items", collection, nameof(Collection)); + foreach (Episode episode in await database.Episodes.GetAll(limit: 0)) + await CreateOrUpdate(nameof(Episode), episode); + foreach (Studio studio in await database.Studios.GetAll(limit: 0)) + await CreateOrUpdate(nameof(Studio), studio); + } } diff --git a/back/src/Kyoo.Meilisearch/MeilisearchModule.cs b/back/src/Kyoo.Meilisearch/MeilisearchModule.cs index a290a366..d3e6f47b 100644 --- a/back/src/Kyoo.Meilisearch/MeilisearchModule.cs +++ b/back/src/Kyoo.Meilisearch/MeilisearchModule.cs @@ -119,11 +119,6 @@ public static class MeilisearchModule }, }; - /// - /// Init meilisearch indexes. - /// - /// The service list to retrieve the meilisearch client - /// A representing the asynchronous operation. public static async Task Initialize(IServiceProvider provider) { MeilisearchClient client = provider.GetRequiredService(); @@ -133,6 +128,13 @@ public static class MeilisearchModule await _CreateIndex(client, nameof(Studio), false); } + public static async Task SyncDatabase(IServiceProvider provider) + { + await using AsyncServiceScope scope = provider.CreateAsyncScope(); + ILibraryManager database = scope.ServiceProvider.GetRequiredService(); + await scope.ServiceProvider.GetRequiredService().SyncEverything(database); + } + private static async Task _CreateIndex(MeilisearchClient client, string index, bool hasKind) { TaskInfo task = await client.CreateIndexAsync(