Add items index on meilisearch

This commit is contained in:
Zoe Roux 2023-10-31 00:11:16 +01:00
parent 4368f0cbe5
commit d7dee62e97
15 changed files with 386 additions and 36 deletions

View File

@ -10,6 +10,7 @@ COPY src/Kyoo.Abstractions/Kyoo.Abstractions.csproj src/Kyoo.Abstractions/Kyoo.A
COPY src/Kyoo.Core/Kyoo.Core.csproj src/Kyoo.Core/Kyoo.Core.csproj
COPY src/Kyoo.Host/Kyoo.Host.csproj src/Kyoo.Host/Kyoo.Host.csproj
COPY src/Kyoo.Postgresql/Kyoo.Postgresql.csproj src/Kyoo.Postgresql/Kyoo.Postgresql.csproj
COPY src/Kyoo.Meilisearch/Kyoo.Meilisearch.csproj src/Kyoo.Meilisearch/Kyoo.Meilisearch.csproj
COPY src/Kyoo.Swagger/Kyoo.Swagger.csproj src/Kyoo.Swagger/Kyoo.Swagger.csproj
COPY tests/Kyoo.Tests/Kyoo.Tests.csproj tests/Kyoo.Tests/Kyoo.Tests.csproj
RUN dotnet restore -a $TARGETARCH

View File

@ -10,6 +10,7 @@ COPY src/Kyoo.Abstractions/Kyoo.Abstractions.csproj src/Kyoo.Abstractions/Kyoo.A
COPY src/Kyoo.Core/Kyoo.Core.csproj src/Kyoo.Core/Kyoo.Core.csproj
COPY src/Kyoo.Host/Kyoo.Host.csproj src/Kyoo.Host/Kyoo.Host.csproj
COPY src/Kyoo.Postgresql/Kyoo.Postgresql.csproj src/Kyoo.Postgresql/Kyoo.Postgresql.csproj
COPY src/Kyoo.Meilisearch/Kyoo.Meilisearch.csproj src/Kyoo.Meilisearch/Kyoo.Meilisearch.csproj
COPY src/Kyoo.Swagger/Kyoo.Swagger.csproj src/Kyoo.Swagger/Kyoo.Swagger.csproj
COPY tests/Kyoo.Tests/Kyoo.Tests.csproj tests/Kyoo.Tests/Kyoo.Tests.csproj
RUN dotnet restore

View File

@ -25,6 +25,9 @@ namespace Kyoo.Abstractions.Controllers
/// </summary>
public interface ILibraryManager
{
IRepository<T> Repository<T>()
where T : class, IResource;
/// <summary>
/// The repository that handle libraries items (a wrapper around shows and collections).
/// </summary>

View File

@ -0,0 +1,81 @@
// Kyoo - A portable and vast media library solution.
// Copyright (c) Kyoo.
//
// See AUTHORS.md and LICENSE file in the project root for full license information.
//
// Kyoo is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// any later version.
//
// Kyoo is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Kyoo. If not, see <https://www.gnu.org/licenses/>.
using System.Threading.Tasks;
using Kyoo.Abstractions.Models;
using Kyoo.Abstractions.Models.Utils;
namespace Kyoo.Abstractions.Controllers;
/// <summary>
/// The service to search items.
/// </summary>
public interface ISearchManager
{
/// <summary>
/// Search for items.
/// </summary>
/// <param name="query">The seach query.</param>
/// <param name="sortBy">Sort information about the query (sort by, sort order)</param>
/// <param name="pagination">How pagination should be done (where to start and how many to return)</param>
/// <param name="include">The related fields to include.</param>
/// <returns>A list of resources that match every filters</returns>
public Task<SearchPage<LibraryItem>.SearchResult> SearchItems(string? query,
Sort<LibraryItem> sortBy,
SearchPagination pagination,
Include<LibraryItem>? include = default);
/// <summary>
/// Search for movies.
/// </summary>
/// <param name="query">The seach query.</param>
/// <param name="sortBy">Sort information about the query (sort by, sort order)</param>
/// <param name="pagination">How pagination should be done (where to start and how many to return)</param>
/// <param name="include">The related fields to include.</param>
/// <returns>A list of resources that match every filters</returns>
public Task<SearchPage<Movie>.SearchResult> SearchMovies(string? query,
Sort<Movie> sortBy,
SearchPagination pagination,
Include<Movie>? include = default);
/// <summary>
/// Search for shows.
/// </summary>
/// <param name="query">The seach query.</param>
/// <param name="sortBy">Sort information about the query (sort by, sort order)</param>
/// <param name="pagination">How pagination should be done (where to start and how many to return)</param>
/// <param name="include">The related fields to include.</param>
/// <returns>A list of resources that match every filters</returns>
public Task<SearchPage<Show>.SearchResult> SearchShows(string? query,
Sort<Show> sortBy,
SearchPagination pagination,
Include<Show>? include = default);
/// <summary>
/// Search for collections.
/// </summary>
/// <param name="query">The seach query.</param>
/// <param name="sortBy">Sort information about the query (sort by, sort order)</param>
/// <param name="pagination">How pagination should be done (where to start and how many to return)</param>
/// <param name="include">The related fields to include.</param>
/// <returns>A list of resources that match every filters</returns>
public Task<SearchPage<Collection>.SearchResult> SearchCollections(string? query,
Sort<Collection> sortBy,
SearchPagination pagination,
Include<Collection>? include = default);
}

View File

@ -67,7 +67,7 @@ namespace Kyoo.Abstractions.Models
/// <param name="previous">The link of the previous page.</param>
/// <param name="next">The link of the next page.</param>
/// <param name="first">The link of the first page.</param>
public Page(ICollection<T> items, string @this, string previous, string next, string first)
public Page(ICollection<T> items, string @this, string? previous, string? next, string first)
{
Items = items;
This = @this;

View File

@ -0,0 +1,53 @@
// Kyoo - A portable and vast media library solution.
// Copyright (c) Kyoo.
//
// See AUTHORS.md and LICENSE file in the project root for full license information.
//
// Kyoo is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// any later version.
//
// Kyoo is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Kyoo. If not, see <https://www.gnu.org/licenses/>.
using System.Collections.Generic;
namespace Kyoo.Abstractions.Models
{
/// <summary>
/// Results of a search request.
/// </summary>
/// <typeparam name="T">The search item's type.</typeparam>
public class SearchPage<T> : Page<T>
where T : IResource
{
public SearchPage(
SearchResult result,
string @this,
string? previous,
string? next,
string first)
: base(result.Items, @this, previous, next, first)
{
Query = result.Query;
}
/// <summary>
/// The query of the search request.
/// </summary>
public string? Query { get; init; }
public class SearchResult
{
public string? Query { get; set; }
public ICollection<T> Items { get; set; }
}
}
}

View File

@ -16,19 +16,21 @@
// You should have received a copy of the GNU General Public License
// along with Kyoo. If not, see <https://www.gnu.org/licenses/>.
using System.Collections.Generic;
namespace Kyoo.Abstractions.Models
namespace Kyoo.Abstractions.Controllers
{
/// <summary>
/// Results of a search request.
/// Information about the pagination. How many items should be displayed and where to start.
/// </summary>
public class SearchPage<T> : Page<T>
where T : class, IResource
public class SearchPagination
{
/// <summary>
/// The query of the search request.
/// The count of items to return.
/// </summary>
public string Query { get; init; }
public int Limit { get; set; } = 50;
/// <summary>
/// Where to start? How many items to skip?
/// </summary>
public int? Skip { get; set; }
}
}

View File

@ -16,6 +16,7 @@
// You should have received a copy of the GNU General Public License
// along with Kyoo. If not, see <https://www.gnu.org/licenses/>.
using System.Linq;
using Kyoo.Abstractions.Controllers;
using Kyoo.Abstractions.Models;
@ -26,6 +27,8 @@ namespace Kyoo.Core.Controllers
/// </summary>
public class LibraryManager : ILibraryManager
{
private readonly IBaseRepository[] _repositories;
public LibraryManager(
IRepository<LibraryItem> libraryItemRepository,
IRepository<Collection> collectionRepository,
@ -46,6 +49,19 @@ namespace Kyoo.Core.Controllers
People = peopleRepository;
Studios = studioRepository;
Users = userRepository;
_repositories = new IBaseRepository[]
{
LibraryItems,
Collections,
Movies,
Shows,
Seasons,
Episodes,
People,
Studios,
Users
};
}
/// <inheritdoc />
@ -74,5 +90,11 @@ namespace Kyoo.Core.Controllers
/// <inheritdoc />
public IRepository<User> Users { get; }
public IRepository<T> Repository<T>()
where T : class, IResource
{
return (IRepository<T>)_repositories.First(x => x.RepositoryType == typeof(T));
}
}
}

View File

@ -20,7 +20,9 @@ using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using Kyoo.Abstractions.Controllers;
using Kyoo.Abstractions.Models;
using Kyoo.Utils;
using Microsoft.AspNetCore.Mvc;
namespace Kyoo.Core.Api
@ -63,5 +65,45 @@ namespace Kyoo.Core.Api
limit
);
}
protected SearchPage<TResult> SearchPage<TResult>(SearchPage<TResult>.SearchResult result)
where TResult : IResource
{
Dictionary<string, string> query = Request.Query.ToDictionary(
x => x.Key,
x => x.Value.ToString(),
StringComparer.InvariantCultureIgnoreCase
);
string self = Request.Path + query.ToQueryString();
string? previous = null;
string? next = null;
string first;
int limit = query.TryGetValue("limit", out string? limitStr) ? int.Parse(limitStr) : new SearchPagination().Limit;
int? skip = query.TryGetValue("skip", out string? skipStr) ? int.Parse(skipStr) : null;
if (skip != null)
{
query["skip"] = Math.Max(0, skip.Value - limit).ToString();
previous = Request.Path + query.ToQueryString();
}
if (result.Items.Count == limit && limit > 0)
{
int newSkip = skip.HasValue ? skip.Value + limit : limit;
query["skip"] = newSkip.ToString();
next = Request.Path + query.ToQueryString();
}
query.Remove("skip");
first = Request.Path + query.ToQueryString();
return new SearchPage<TResult>(
result,
self,
previous,
next,
first
);
}
}
}

View File

@ -33,26 +33,22 @@ namespace Kyoo.Core.Api
/// An endpoint to search for every resources of kyoo. Searching for only a specific type of resource
/// is available on the said endpoint.
/// </summary>
[Route("search/{query}")]
[Route("search/{query?}")]
[ApiController]
[ResourceView]
[ApiDefinition("Search", Group = ResourcesGroup)]
public class SearchApi : ControllerBase
public class SearchApi : BaseApi
{
/// <summary>
/// The library manager used to modify or retrieve information in the data store.
/// </summary>
private readonly ILibraryManager _libraryManager;
private readonly ISearchManager _searchManager;
/// <summary>
/// Create a new <see cref="SearchApi"/>.
/// </summary>
/// <param name="libraryManager">The library manager used to interact with the data store.</param>
public SearchApi(ILibraryManager libraryManager)
public SearchApi(ISearchManager searchManager)
{
_libraryManager = libraryManager;
_searchManager = searchManager;
}
// TODO: add filters and facets
/// <summary>
/// Search collections
/// </summary>
@ -91,6 +87,30 @@ namespace Kyoo.Core.Api
return _libraryManager.Shows.Search(query);
}
/// <summary>
/// Search movie
/// </summary>
/// <remarks>
/// Search for movie
/// </remarks>
/// <param name="query">The query to search for.</param>
/// <param name="sortBy">Sort information about the query (sort by, sort order).</param>
/// <param name="pagination">How many items per page should be returned, where should the page start...</param>
/// <param name="fields">The aditional fields to include in the result.</param>
/// <returns>A list of movies found for the specified query.</returns>
[HttpGet("movies")]
[HttpGet("movie", Order = AlternativeRoute)]
[Permission(nameof(Movie), Kind.Read)]
[ApiDefinition("Movie")]
[ProducesResponseType(StatusCodes.Status200OK)]
public async Task<SearchPage<Movie>> SearchMovies(string? query,
[FromQuery] Sort<Movie> sortBy,
[FromQuery] SearchPagination pagination,
[FromQuery] Include<Movie> fields)
{
return SearchPage(await _searchManager.SearchMovies(query, sortBy, pagination, fields));
}
/// <summary>
/// Search items
/// </summary>

View File

@ -23,6 +23,7 @@ using System.Threading;
using System.Threading.Tasks;
using Autofac;
using Autofac.Extensions.DependencyInjection;
using Kyoo.Meiliseach;
using Kyoo.Postgresql;
using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.Configuration;
@ -102,6 +103,7 @@ namespace Kyoo.Host
.Build();
PostgresModule.Initialize(host.Services);
await MeilisearchModule.Initialize(host.Services);
await _StartWithHost(host);
}

View File

@ -22,6 +22,7 @@
<ProjectReference Include="../Kyoo.Abstractions/Kyoo.Abstractions.csproj" />
<ProjectReference Include="../Kyoo.Core/Kyoo.Core.csproj" />
<ProjectReference Include="../Kyoo.Postgresql/Kyoo.Postgresql.csproj" />
<ProjectReference Include="../Kyoo.Meilisearch/Kyoo.Meilisearch.csproj" />
<ProjectReference Include="../Kyoo.Authentication/Kyoo.Authentication.csproj" />
<ProjectReference Include="../Kyoo.Swagger/Kyoo.Swagger.csproj" />
</ItemGroup>

View File

@ -25,6 +25,7 @@ using Kyoo.Abstractions.Controllers;
using Kyoo.Authentication;
using Kyoo.Core;
using Kyoo.Host.Controllers;
using Kyoo.Meiliseach;
using Kyoo.Postgresql;
using Kyoo.Swagger;
using Kyoo.Utils;
@ -64,6 +65,7 @@ namespace Kyoo.Host
typeof(CoreModule),
typeof(AuthenticationModule),
typeof(PostgresModule),
typeof(MeilisearchModule),
typeof(SwaggerModule)
);
}

View File

@ -41,11 +41,12 @@ namespace Kyoo.Meiliseach
/// Init meilisearch indexes.
/// </summary>
/// <param name="provider">The service list to retrieve the meilisearch client</param>
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
public static async Task Initialize(IServiceProvider provider)
{
MeilisearchClient client = provider.GetRequiredService<MeilisearchClient>();
await _CreateIndex(client, "items", new Settings()
await _CreateIndex(client, "items", true, new Settings()
{
SearchableAttributes = new[]
{
@ -61,7 +62,7 @@ namespace Kyoo.Meiliseach
nameof(LibraryItem.Genres),
nameof(LibraryItem.Status),
nameof(LibraryItem.AirDate),
nameof(LibraryItem.StudioID),
nameof(Movie.StudioID),
},
SortableAttributes = new[]
{
@ -69,15 +70,19 @@ namespace Kyoo.Meiliseach
nameof(LibraryItem.AddedDate),
nameof(LibraryItem.Kind),
},
DisplayedAttributes = new[] { nameof(LibraryItem.Id) },
DisplayedAttributes = new[]
{
nameof(LibraryItem.Id),
nameof(LibraryItem.Kind),
},
// TODO: Add stopwords
// TODO: Extend default ranking to add ratings.
});
}
private static async Task _CreateIndex(MeilisearchClient client, string index, Settings opts)
private static async Task _CreateIndex(MeilisearchClient client, string index, bool hasKind, Settings opts)
{
TaskInfo task = await client.CreateIndexAsync(index, "Id");
TaskInfo task = await client.CreateIndexAsync(index, hasKind ? "Ref" : nameof(IResource.Id));
await client.WaitForTaskAsync(task.TaskUid);
await client.Index(index).UpdateSettingsAsync(opts);
}
@ -88,8 +93,8 @@ namespace Kyoo.Meiliseach
builder.RegisterInstance(new MeilisearchClient(
_configuration.GetValue("MEILI_HOST", "http://meilisearch:7700"),
_configuration.GetValue<string?>("MEILI_MASTER_KEY")
)).InstancePerLifetimeScope();
builder.RegisterType<SearchManager>().InstancePerLifetimeScope();
)).SingleInstance();
builder.RegisterType<SearchManager>().SingleInstance();
}
}
}

View File

@ -16,6 +16,9 @@
// You should have received a copy of the GNU General Public License
// along with Kyoo. If not, see <https://www.gnu.org/licenses/>.
using System.ComponentModel.DataAnnotations;
using System.Dynamic;
using System.Reflection;
using Kyoo.Abstractions.Controllers;
using Kyoo.Abstractions.Models;
using Kyoo.Abstractions.Models.Utils;
@ -23,36 +26,148 @@ using Meilisearch;
namespace Kyoo.Meiliseach;
public class SearchManager
public class SearchManager : ISearchManager
{
private readonly MeilisearchClient _client;
private readonly ILibraryManager _libraryManager;
private static IEnumerable<string> _GetSortsBy<T>(Sort<T>? sort)
{
return sort switch
{
Sort<T>.Default => Array.Empty<string>(),
Sort<T>.By @sortBy => new[] { $"{sortBy.Key}:{(sortBy.Desendant ? "desc" : "asc")}" },
Sort<T>.Conglomerate(var list) => list.SelectMany(_GetSortsBy),
Sort<T>.Random => throw new ValidationException("Random sorting is not supported while searching."),
_ => Array.Empty<string>(),
};
}
public SearchManager(MeilisearchClient client, ILibraryManager libraryManager)
{
_client = client;
_libraryManager = libraryManager;
_libraryManager.Movies.OnCreated += (x) => _CreateOrUpdate("items", x);
_libraryManager.Movies.OnEdited += (x) => _CreateOrUpdate("items", x);
_libraryManager.Movies.OnDeleted += (x) => _Delete("items", x.Id);
IRepository<Movie>.OnCreated += (x) => _CreateOrUpdate("items", x, nameof(Movie));
IRepository<Movie>.OnEdited += (x) => _CreateOrUpdate("items", x, nameof(Movie));
IRepository<Movie>.OnDeleted += (x) => _Delete("items", x.Id, nameof(Movie));
IRepository<Show>.OnCreated += (x) => _CreateOrUpdate("items", x, nameof(Show));
IRepository<Show>.OnEdited += (x) => _CreateOrUpdate("items", x, nameof(Show));
IRepository<Show>.OnDeleted += (x) => _Delete("items", x.Id, nameof(Show));
IRepository<Collection>.OnCreated += (x) => _CreateOrUpdate("items", x, nameof(Collection));
IRepository<Collection>.OnEdited += (x) => _CreateOrUpdate("items", x, nameof(Collection));
IRepository<Collection>.OnDeleted += (x) => _Delete("items", x.Id, nameof(Collection));
}
private Task _CreateOrUpdate(string index, IResource item)
private Task _CreateOrUpdate(string index, IResource item, string? kind = null)
{
if (kind != null)
{
dynamic expando = new ExpandoObject();
foreach (PropertyInfo property in item.GetType().GetProperties())
expando.TryAdd(property.Name, property.GetValue(item));
expando.Ref = $"{kind}/{item.Id}";
expando.Kind = kind;
return _client.Index(index).AddDocumentsAsync(new[] { item });
}
return _client.Index(index).AddDocumentsAsync(new[] { item });
}
private Task _Delete(string index, int id)
private Task _Delete(string index, int id, string? kind = null)
{
if (kind != null)
{
return _client.Index(index).DeleteOneDocumentAsync($"{kind}/{id}");
}
return _client.Index(index).DeleteOneDocumentAsync(id);
}
private async Task<ICollection<T>> _Search<T>(string index, string? query, Include<T>? include = default)
private async Task<SearchPage<T>.SearchResult> _Search<T>(string index, string? query,
string? where = null,
Sort<T>? sortBy = default,
SearchPagination? pagination = default,
Include<T>? include = default)
where T : class, IResource
{
ISearchable<IResource> res = await _client.Index(index).SearchAsync<IResource>(query, new SearchQuery()
// TODO: add filters and facets
ISearchable<IdResource> res = await _client.Index(index).SearchAsync<IdResource>(query, new SearchQuery()
{
Filter = where,
Sort = _GetSortsBy(sortBy),
Limit = pagination?.Limit ?? 50,
Offset = pagination?.Skip ?? 0,
});
throw new NotImplementedException();
return new SearchPage<T>.SearchResult
{
Query = query,
Items = await _libraryManager.Repository<T>()
.FromIds(res.Hits.Select(x => x.Id).ToList(), include),
};
}
public async Task<SearchPage<LibraryItem>.SearchResult> SearchItems(string? query,
Sort<LibraryItem> sortBy,
SearchPagination pagination,
Include<LibraryItem>? include = default)
{
// TODO: add filters and facets
ISearchable<IdResource> res = await _client.Index("items").SearchAsync<IdResource>(query, new SearchQuery()
{
Sort = _GetSortsBy(sortBy),
Limit = pagination?.Limit ?? 50,
Offset = pagination?.Skip ?? 0,
});
// Since library items's ID are still ints mapped from real items ids, we must map it here to match the db's value.
// Look at the items Migration's sql to understand where magic numbers come from.
List<int> ids = res.Hits.Select(x => x.Kind switch
{
nameof(Show) => x.Id,
nameof(Movie) => -x.Id,
nameof(Collection) => x.Id + 10_000,
_ => throw new InvalidOperationException("An unknown item kind was found in meilisearch"),
}).ToList();
return new SearchPage<LibraryItem>.SearchResult
{
Query = query,
Items = await _libraryManager.LibraryItems
.FromIds(ids, include),
};
}
/// <inheritdoc/>
public Task<SearchPage<Movie>.SearchResult> SearchMovies(string? query,
Sort<Movie> sortBy,
SearchPagination pagination,
Include<Movie>? include = default)
{
return _Search("items", query, $"Kind = {nameof(Movie)}", sortBy, pagination, include);
}
/// <inheritdoc/>
public Task<SearchPage<Show>.SearchResult> SearchShows(string? query,
Sort<Show> sortBy,
SearchPagination pagination,
Include<Show>? include = default)
{
return _Search("items", query, $"Kind = {nameof(Show)}", sortBy, pagination, include);
}
/// <inheritdoc/>
public Task<SearchPage<Collection>.SearchResult> SearchCollections(string? query,
Sort<Collection> sortBy,
SearchPagination pagination,
Include<Collection>? include = default)
{
return _Search("items", query, $"Kind = {nameof(Collection)}", sortBy, pagination, include);
}
private class IdResource
{
public int Id { get; set; }
public string? Kind { get; set; }
}
}