diff --git a/.env.example b/.env.example index 2298b20a..f15219f1 100644 --- a/.env.example +++ b/.env.example @@ -64,7 +64,7 @@ OIDC_SERVICE_PROFILE=https://url-of-the-profile-endpoint-of-the-oidc-service.com OIDC_SERVICE_SCOPE="the list of scopes space separeted like email identity" # Token authentication method as seen in https://openid.net/specs/openid-connect-core-1_0.html#ClientAuthentication # Supported values: ClientSecretBasic (default) or ClientSecretPost -# If in doupt, leave this empty. +# If in doubt, leave this empty. OIDC_SERVICE_AUTHMETHOD=ClientSecretBasic # on the previous list, service is the internal name of your service, you can add as many as you want. @@ -81,6 +81,11 @@ POSTGRES_DB=kyooDB POSTGRES_SERVER=postgres POSTGRES_PORT=5432 +# Read by the api container to know if it should run meilisearch's migrations/sync +# and download missing images. This is a good idea to only have one instance with this on +# Note: it does not run postgres migrations, use the migration container for that. +RUN_MIGRATIONS=true + MEILI_HOST="http://meilisearch:7700" MEILI_MASTER_KEY="ghvjkgisbgkbgskegblfqbgjkebbhgwkjfb" diff --git a/back/src/Kyoo.Abstractions/Controllers/ISearchManager.cs b/back/src/Kyoo.Abstractions/Controllers/ISearchManager.cs index e6175018..d3985c6e 100644 --- a/back/src/Kyoo.Abstractions/Controllers/ISearchManager.cs +++ b/back/src/Kyoo.Abstractions/Controllers/ISearchManager.cs @@ -38,6 +38,7 @@ public interface ISearchManager public Task.SearchResult> SearchItems( string? query, Sort sortBy, + Filter? filter, SearchPagination pagination, Include? include = default ); @@ -53,6 +54,7 @@ public interface ISearchManager public Task.SearchResult> SearchMovies( string? query, Sort sortBy, + Filter? filter, SearchPagination pagination, Include? include = default ); @@ -68,6 +70,7 @@ public interface ISearchManager public Task.SearchResult> SearchShows( string? query, Sort sortBy, + Filter? filter, SearchPagination pagination, Include? include = default ); @@ -83,6 +86,7 @@ public interface ISearchManager public Task.SearchResult> SearchCollections( string? query, Sort sortBy, + Filter? filter, SearchPagination pagination, Include? include = default ); @@ -98,6 +102,7 @@ public interface ISearchManager public Task.SearchResult> SearchEpisodes( string? query, Sort sortBy, + Filter? filter, SearchPagination pagination, Include? include = default ); @@ -113,6 +118,7 @@ public interface ISearchManager public Task.SearchResult> SearchStudios( string? query, Sort sortBy, + Filter? filter, SearchPagination pagination, Include? include = default ); diff --git a/back/src/Kyoo.Abstractions/Models/Utils/Filter.cs b/back/src/Kyoo.Abstractions/Models/Utils/Filter.cs index 430c1218..4d832928 100644 --- a/back/src/Kyoo.Abstractions/Models/Utils/Filter.cs +++ b/back/src/Kyoo.Abstractions/Models/Utils/Filter.cs @@ -215,14 +215,16 @@ public abstract record Filter : Filter }); } - if (type == typeof(DateTime)) + if (type == typeof(DateTime) || type == typeof(DateOnly)) { return from year in Parse.Digit.Repeat(4).Text().Select(int.Parse) from yd in Parse.Char('-') - from mouth in Parse.Digit.Repeat(2).Text().Select(int.Parse) + from month in Parse.Digit.Repeat(2).Text().Select(int.Parse) from md in Parse.Char('-') from day in Parse.Digit.Repeat(2).Text().Select(int.Parse) - select new DateTime(year, mouth, day) as object; + select type == typeof(DateTime) + ? new DateTime(year, month, day) as object + : new DateOnly(year, month, day) as object; } if (typeof(IEnumerable).IsAssignableFrom(type)) diff --git a/back/src/Kyoo.Core/Program.cs b/back/src/Kyoo.Core/Program.cs index c95c9cc3..49afbd5c 100644 --- a/back/src/Kyoo.Core/Program.cs +++ b/back/src/Kyoo.Core/Program.cs @@ -17,6 +17,7 @@ // along with Kyoo. If not, see . using System; +using Kyoo.Abstractions.Controllers; using Kyoo.Authentication; using Kyoo.Core; using Kyoo.Core.Controllers; @@ -27,6 +28,7 @@ using Kyoo.RabbitMq; using Kyoo.Swagger; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Serilog; using Serilog.Events; @@ -94,12 +96,17 @@ app.MapControllers(); app.Services.GetRequiredService(); app.Services.GetRequiredService(); -await using (AsyncServiceScope scope = app.Services.CreateAsyncScope()) +// Only run sync on the main instance +if (app.Configuration.GetValue("RUN_MIGRATIONS", true)) { - await MeilisearchModule.Initialize(scope.ServiceProvider); + await using (AsyncServiceScope scope = app.Services.CreateAsyncScope()) + { + await MeilisearchModule.Initialize(app.Services); + } + + // The methods takes care of creating a scope and will download images on the background. + _ = MeilisearchModule.SyncDatabase(app.Services); + _ = MiscRepository.DownloadMissingImages(app.Services); } -// The methods takes care of creating a scope and will download images on the background. -_ = MiscRepository.DownloadMissingImages(app.Services); - await app.RunAsync(Environment.GetEnvironmentVariable("KYOO_BIND_URL") ?? "http://*:5000"); diff --git a/back/src/Kyoo.Core/Views/Resources/SearchApi.cs b/back/src/Kyoo.Core/Views/Resources/SearchApi.cs index b3754d3c..14050382 100644 --- a/back/src/Kyoo.Core/Views/Resources/SearchApi.cs +++ b/back/src/Kyoo.Core/Views/Resources/SearchApi.cs @@ -44,7 +44,7 @@ public class SearchApi : BaseApi _searchManager = searchManager; } - // TODO: add filters and facets + // TODO: add facets /// /// Search collections @@ -65,11 +65,14 @@ public class SearchApi : BaseApi public async Task> SearchCollections( [FromQuery] string? q, [FromQuery] Sort sortBy, + [FromQuery] Filter? filter, [FromQuery] SearchPagination pagination, [FromQuery] Include fields ) { - return SearchPage(await _searchManager.SearchCollections(q, sortBy, pagination, fields)); + return SearchPage( + await _searchManager.SearchCollections(q, sortBy, filter, pagination, fields) + ); } /// @@ -91,11 +94,12 @@ public class SearchApi : BaseApi public async Task> SearchShows( [FromQuery] string? q, [FromQuery] Sort sortBy, + [FromQuery] Filter? filter, [FromQuery] SearchPagination pagination, [FromQuery] Include fields ) { - return SearchPage(await _searchManager.SearchShows(q, sortBy, pagination, fields)); + return SearchPage(await _searchManager.SearchShows(q, sortBy, filter, pagination, fields)); } /// @@ -117,11 +121,12 @@ public class SearchApi : BaseApi public async Task> SearchMovies( [FromQuery] string? q, [FromQuery] Sort sortBy, + [FromQuery] Filter? filter, [FromQuery] SearchPagination pagination, [FromQuery] Include fields ) { - return SearchPage(await _searchManager.SearchMovies(q, sortBy, pagination, fields)); + return SearchPage(await _searchManager.SearchMovies(q, sortBy, filter, pagination, fields)); } /// @@ -143,11 +148,12 @@ public class SearchApi : BaseApi public async Task> SearchItems( [FromQuery] string? q, [FromQuery] Sort sortBy, + [FromQuery] Filter? filter, [FromQuery] SearchPagination pagination, [FromQuery] Include fields ) { - return SearchPage(await _searchManager.SearchItems(q, sortBy, pagination, fields)); + return SearchPage(await _searchManager.SearchItems(q, sortBy, filter, pagination, fields)); } /// @@ -169,11 +175,14 @@ public class SearchApi : BaseApi public async Task> SearchEpisodes( [FromQuery] string? q, [FromQuery] Sort sortBy, + [FromQuery] Filter? filter, [FromQuery] SearchPagination pagination, [FromQuery] Include fields ) { - return SearchPage(await _searchManager.SearchEpisodes(q, sortBy, pagination, fields)); + return SearchPage( + await _searchManager.SearchEpisodes(q, sortBy, filter, pagination, fields) + ); } /// @@ -195,10 +204,13 @@ public class SearchApi : BaseApi public async Task> SearchStudios( [FromQuery] string? q, [FromQuery] Sort sortBy, + [FromQuery] Filter? filter, [FromQuery] SearchPagination pagination, [FromQuery] Include fields ) { - return SearchPage(await _searchManager.SearchStudios(q, sortBy, pagination, fields)); + return SearchPage( + await _searchManager.SearchStudios(q, sortBy, filter, pagination, fields) + ); } } diff --git a/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs b/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs new file mode 100644 index 00000000..d32d2a63 --- /dev/null +++ b/back/src/Kyoo.Meilisearch/FilterExtensionMethods.cs @@ -0,0 +1,52 @@ +using System.ComponentModel.DataAnnotations; +using Kyoo.Abstractions.Models.Utils; +using static System.Text.Json.JsonNamingPolicy; + +namespace Kyoo.Meiliseach; + +internal static class FilterExtensionMethods +{ + public static string? CreateMeilisearchFilter(this Filter? filter) + { + return filter switch + { + Filter.And and + => $"({and.First.CreateMeilisearchFilter()}) AND ({and.Second.CreateMeilisearchFilter()})", + Filter.Or or + => $"({or.First.CreateMeilisearchFilter()}) OR ({or.Second.CreateMeilisearchFilter()})", + Filter.Gt gt => CreateBasicFilterString(gt.Property, ">", gt.Value), + Filter.Lt lt => CreateBasicFilterString(lt.Property, "<", lt.Value), + Filter.Ge ge => CreateBasicFilterString(ge.Property, ">=", ge.Value), + Filter.Le le => CreateBasicFilterString(le.Property, "<=", le.Value), + Filter.Eq eq => CreateBasicFilterString(eq.Property, "=", eq.Value), + Filter.Has has => CreateBasicFilterString(has.Property, "=", has.Value), + Filter.Ne ne => CreateBasicFilterString(ne.Property, "!=", ne.Value), + Filter.Not not => $"NOT ({not.Filter.CreateMeilisearchFilter()})", + Filter.CmpRandom + => throw new ValidationException("Random comparison is not supported."), + _ => null + }; + } + + private static string CreateBasicFilterString(string property, string @operator, object? value) + { + return $"{CamelCase.ConvertName(property)} {@operator} {value.InMeilsearchFilterFormat()}"; + } + + private static object? InMeilsearchFilterFormat(this object? value) + { + return value switch + { + null => null, + string s => s.Any(char.IsWhiteSpace) ? $"\"{s.Replace("\"", "\\\"")}\"" : s, + DateTimeOffset dateTime => dateTime.ToUnixTimeSeconds(), + DateOnly date => date.ToUnixTimeSeconds(), + _ => value + }; + } + + public static long ToUnixTimeSeconds(this DateOnly date) + { + return new DateTimeOffset(date.ToDateTime(new TimeOnly())).ToUnixTimeSeconds(); + } +} diff --git a/back/src/Kyoo.Meilisearch/MeiliSync.cs b/back/src/Kyoo.Meilisearch/MeiliSync.cs index 0005ef81..e995ce6a 100644 --- a/back/src/Kyoo.Meilisearch/MeiliSync.cs +++ b/back/src/Kyoo.Meilisearch/MeiliSync.cs @@ -16,6 +16,7 @@ // You should have received a copy of the GNU General Public License // along with Kyoo. If not, see . +using System.Collections; using System.Dynamic; using System.Reflection; using Kyoo.Abstractions.Controllers; @@ -60,7 +61,10 @@ public class MeiliSync var dictionary = (IDictionary)expando; foreach (PropertyInfo property in item.GetType().GetProperties()) - dictionary.Add(CamelCase.ConvertName(property.Name), property.GetValue(item)); + dictionary.Add( + CamelCase.ConvertName(property.Name), + ConvertToMeilisearchFormat(property.GetValue(item)) + ); dictionary.Add("ref", $"{kind}-{item.Id}"); expando.kind = kind; return _client.Index(index).AddDocumentsAsync(new[] { expando }); @@ -76,4 +80,33 @@ public class MeiliSync } return _client.Index(index).DeleteOneDocumentAsync(id.ToString()); } + + private object? ConvertToMeilisearchFormat(object? value) + { + return value switch + { + null => null, + string => value, + Enum => value.ToString(), + IEnumerable enumerable + => enumerable.Cast().Select(ConvertToMeilisearchFormat).ToArray(), + DateTimeOffset dateTime => dateTime.ToUnixTimeSeconds(), + DateOnly date => date.ToUnixTimeSeconds(), + _ => value + }; + } + + public async Task SyncEverything(ILibraryManager database) + { + foreach (Movie movie in await database.Movies.GetAll(limit: 0)) + await CreateOrUpdate("items", movie, nameof(Movie)); + foreach (Show show in await database.Shows.GetAll(limit: 0)) + await CreateOrUpdate("items", show, nameof(Show)); + foreach (Collection collection in await database.Collections.GetAll(limit: 0)) + await CreateOrUpdate("items", collection, nameof(Collection)); + foreach (Episode episode in await database.Episodes.GetAll(limit: 0)) + await CreateOrUpdate(nameof(Episode), episode); + foreach (Studio studio in await database.Studios.GetAll(limit: 0)) + await CreateOrUpdate(nameof(Studio), studio); + } } diff --git a/back/src/Kyoo.Meilisearch/MeilisearchModule.cs b/back/src/Kyoo.Meilisearch/MeilisearchModule.cs index f2e971b4..d3e6f47b 100644 --- a/back/src/Kyoo.Meilisearch/MeilisearchModule.cs +++ b/back/src/Kyoo.Meilisearch/MeilisearchModule.cs @@ -49,6 +49,8 @@ public static class MeilisearchModule CamelCase.ConvertName(nameof(Movie.Genres)), CamelCase.ConvertName(nameof(Movie.Status)), CamelCase.ConvertName(nameof(Movie.AirDate)), + CamelCase.ConvertName(nameof(Show.StartAir)), + CamelCase.ConvertName(nameof(Show.EndAir)), CamelCase.ConvertName(nameof(Movie.StudioId)), "kind" }, @@ -117,11 +119,6 @@ public static class MeilisearchModule }, }; - /// - /// Init meilisearch indexes. - /// - /// The service list to retrieve the meilisearch client - /// A representing the asynchronous operation. public static async Task Initialize(IServiceProvider provider) { MeilisearchClient client = provider.GetRequiredService(); @@ -131,6 +128,13 @@ public static class MeilisearchModule await _CreateIndex(client, nameof(Studio), false); } + public static async Task SyncDatabase(IServiceProvider provider) + { + await using AsyncServiceScope scope = provider.CreateAsyncScope(); + ILibraryManager database = scope.ServiceProvider.GetRequiredService(); + await scope.ServiceProvider.GetRequiredService().SyncEverything(database); + } + private static async Task _CreateIndex(MeilisearchClient client, string index, bool hasKind) { TaskInfo task = await client.CreateIndexAsync( diff --git a/back/src/Kyoo.Meilisearch/SearchManager.cs b/back/src/Kyoo.Meilisearch/SearchManager.cs index 07b5ed11..7f7ccaf4 100644 --- a/back/src/Kyoo.Meilisearch/SearchManager.cs +++ b/back/src/Kyoo.Meilisearch/SearchManager.cs @@ -99,66 +99,125 @@ public class SearchManager : ISearchManager public Task.SearchResult> SearchItems( string? query, Sort sortBy, + Filter? filter, SearchPagination pagination, Include? include = default ) { - return _Search("items", query, null, sortBy, pagination, include); + return _Search( + "items", + query, + filter.CreateMeilisearchFilter(), + sortBy, + pagination, + include + ); } /// public Task.SearchResult> SearchMovies( string? query, Sort sortBy, + Filter? filter, SearchPagination pagination, Include? include = default ) { - return _Search("items", query, $"kind = {nameof(Movie)}", sortBy, pagination, include); + return _Search( + "items", + query, + _CreateMediaTypeFilter(filter), + sortBy, + pagination, + include + ); } /// public Task.SearchResult> SearchShows( string? query, Sort sortBy, + Filter? filter, SearchPagination pagination, Include? include = default ) { - return _Search("items", query, $"kind = {nameof(Show)}", sortBy, pagination, include); + return _Search( + "items", + query, + _CreateMediaTypeFilter(filter), + sortBy, + pagination, + include + ); } /// public Task.SearchResult> SearchCollections( string? query, Sort sortBy, + Filter? filter, SearchPagination pagination, Include? include = default ) { - return _Search("items", query, $"kind = {nameof(Collection)}", sortBy, pagination, include); + return _Search( + "items", + query, + _CreateMediaTypeFilter(filter), + sortBy, + pagination, + include + ); } /// public Task.SearchResult> SearchEpisodes( string? query, Sort sortBy, + Filter? filter, SearchPagination pagination, Include? include = default ) { - return _Search(nameof(Episode), query, null, sortBy, pagination, include); + return _Search( + nameof(Episode), + query, + filter.CreateMeilisearchFilter(), + sortBy, + pagination, + include + ); } /// public Task.SearchResult> SearchStudios( string? query, Sort sortBy, + Filter? filter, SearchPagination pagination, Include? include = default ) { - return _Search(nameof(Studio), query, null, sortBy, pagination, include); + return _Search( + nameof(Studio), + query, + filter.CreateMeilisearchFilter(), + sortBy, + pagination, + include + ); + } + + private string _CreateMediaTypeFilter(Filter? filter) + where T : ILibraryItem + { + string filterString = $"kind = {typeof(T).Name}"; + if (filter is not null) + { + filterString += $" AND ({filter.CreateMeilisearchFilter()})"; + } + return filterString; } private class IdResource