Cutlery/Kyoo
1
0
Fork 0
mirror of https://github.com/zoriya/Kyoo.git synced 2025-05-24 02:02:36 -04:00
Code Issues Packages Projects Releases Wiki Activity

Fix meilisearch indexes setup

Browse Source
This commit is contained in:
Zoe Roux 2023-10-31 16:26:17 +01:00
parent 5b7bfa79f9
commit 68a3af0b52
6 changed files with 169 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
back/src/Kyoo.Abstractions/Controllers/ISearchManager.cs
View File

@ -78,4 +78,30 @@ public interface ISearchManager
Sort<Collection> sortBy,
SearchPagination pagination,
Include<Collection>? include = default);
/// <summary>
/// Search for episodes.
/// </summary>
/// <param name="query">The seach query.</param>
/// <param name="sortBy">Sort information about the query (sort by, sort order)</param>
/// <param name="pagination">How pagination should be done (where to start and how many to return)</param>
/// <param name="include">The related fields to include.</param>
/// <returns>A list of resources that match every filters</returns>
public Task<SearchPage<Episode>.SearchResult> SearchEpisodes(string? query,
Sort<Episode> sortBy,
SearchPagination pagination,
Include<Episode>? include = default);
/// <summary>
/// Search for studios.
/// </summary>
/// <param name="query">The seach query.</param>
/// <param name="sortBy">Sort information about the query (sort by, sort order)</param>
/// <param name="pagination">How pagination should be done (where to start and how many to return)</param>
/// <param name="include">The related fields to include.</param>
/// <returns>A list of resources that match every filters</returns>
public Task<SearchPage<Studio>.SearchResult> SearchStudios(string? query,
Sort<Studio> sortBy,
SearchPagination pagination,
Include<Studio>? include = default);
}

6
back/src/Kyoo.Core/Controllers/ThumbnailsManager.cs
View File

@ -75,6 +75,12 @@ namespace Kyoo.Core.Controllers
response.EnsureSuccessStatusCode();
await using Stream reader = await response.Content.ReadAsStreamAsync();
using SKCodec codec = SKCodec.Create(reader);
if (codec == null)
{
_logger.LogError("Unsupported codec for {What}", what);
return;
}
SKImageInfo info = codec.Info;
info.ColorType = SKColorType.Rgba8888;
using SKBitmap original = SKBitmap.Decode(codec, info);

72
back/src/Kyoo.Core/Views/Resources/SearchApi.cs
View File

@ -39,7 +39,6 @@ namespace Kyoo.Core.Api
[ApiDefinition("Search", Group = ResourcesGroup)]
public class SearchApi : BaseApi
{
private readonly ILibraryManager _libraryManager;
private readonly ISearchManager _searchManager;
public SearchApi(ISearchManager searchManager)
@ -56,6 +55,8 @@ namespace Kyoo.Core.Api
/// Search for collections
/// </remarks>
/// <param name="query">The query to search for.</param>
/// <param name="sortBy">Sort information about the query (sort by, sort order).</param>
/// <param name="pagination">How many items per page should be returned, where should the page start...</param>
/// <param name="fields">The aditional fields to include in the result.</param>
/// <returns>A list of collections found for the specified query.</returns>
[HttpGet("collections")]
@ -63,9 +64,12 @@ namespace Kyoo.Core.Api
[Permission(nameof(Collection), Kind.Read)]
[ApiDefinition("Collections")]
[ProducesResponseType(StatusCodes.Status200OK)]
public Task<ICollection<Collection>> SearchCollections(string query, [FromQuery] Include<Collection> fields)
public async Task<SearchPage<Collection>> SearchCollections(string? query,
[FromQuery] Sort<Collection> sortBy,
[FromQuery] SearchPagination pagination,
[FromQuery] Include<Collection> fields)
{
return _libraryManager.Collections.Search(query);
return SearchPage(await _searchManager.SearchCollections(query, sortBy, pagination, fields));
}
/// <summary>
@ -75,16 +79,21 @@ namespace Kyoo.Core.Api
/// Search for shows
/// </remarks>
/// <param name="query">The query to search for.</param>
/// <param name="sortBy">Sort information about the query (sort by, sort order).</param>
/// <param name="pagination">How many items per page should be returned, where should the page start...</param>
/// <param name="fields">The aditional fields to include in the result.</param>
/// <returns>A list of shows found for the specified query.</returns>
[HttpGet("shows")]
[HttpGet("show", Order = AlternativeRoute)]
[Permission(nameof(Show), Kind.Read)]
[ApiDefinition("Shows")]
[ApiDefinition("Show")]
[ProducesResponseType(StatusCodes.Status200OK)]
public Task<ICollection<Show>> SearchShows(string query, [FromQuery] Include<Show> fields)
public async Task<SearchPage<Show>> SearchShows(string? query,
[FromQuery] Sort<Show> sortBy,
[FromQuery] SearchPagination pagination,
[FromQuery] Include<Show> fields)
{
return _libraryManager.Shows.Search(query);
return SearchPage(await _searchManager.SearchShows(query, sortBy, pagination, fields));
}
/// <summary>
@ -118,16 +127,21 @@ namespace Kyoo.Core.Api
/// Search for items
/// </remarks>
/// <param name="query">The query to search for.</param>
/// <param name="sortBy">Sort information about the query (sort by, sort order).</param>
/// <param name="pagination">How many items per page should be returned, where should the page start...</param>
/// <param name="fields">The aditional fields to include in the result.</param>
/// <returns>A list of items found for the specified query.</returns>
[HttpGet("items")]
[HttpGet("item", Order = AlternativeRoute)]
[Permission(nameof(Show), Kind.Read)]
[ApiDefinition("Items")]
[Permission(nameof(LibraryItem), Kind.Read)]
[ApiDefinition("Item")]
[ProducesResponseType(StatusCodes.Status200OK)]
public Task<ICollection<LibraryItem>> SearchItems(string query, [FromQuery] Include<LibraryItem> fields)
public async Task<SearchPage<LibraryItem>> SearchItems(string? query,
[FromQuery] Sort<LibraryItem> sortBy,
[FromQuery] SearchPagination pagination,
[FromQuery] Include<LibraryItem> fields)
{
return _libraryManager.LibraryItems.Search(query);
return SearchPage(await _searchManager.SearchItems(query, sortBy, pagination, fields));
}
/// <summary>
@ -137,6 +151,8 @@ namespace Kyoo.Core.Api
/// Search for episodes
/// </remarks>
/// <param name="query">The query to search for.</param>
/// <param name="sortBy">Sort information about the query (sort by, sort order).</param>
/// <param name="pagination">How many items per page should be returned, where should the page start...</param>
/// <param name="fields">The aditional fields to include in the result.</param>
/// <returns>A list of episodes found for the specified query.</returns>
[HttpGet("episodes")]
@ -144,29 +160,12 @@ namespace Kyoo.Core.Api
[Permission(nameof(Episode), Kind.Read)]
[ApiDefinition("Episodes")]
[ProducesResponseType(StatusCodes.Status200OK)]
public Task<ICollection<Episode>> SearchEpisodes(string query, [FromQuery] Include<Episode> fields)
public async Task<SearchPage<Episode>> SearchEpisodes(string? query,
[FromQuery] Sort<Episode> sortBy,
[FromQuery] SearchPagination pagination,
[FromQuery] Include<Episode> fields)
{
return _libraryManager.Episodes.Search(query);
}
/// <summary>
/// Search staff
/// </summary>
/// <remarks>
/// Search for staff
/// </remarks>
/// <param name="query">The query to search for.</param>
/// <param name="fields">The aditional fields to include in the result.</param>
/// <returns>A list of staff members found for the specified query.</returns>
[HttpGet("staff")]
[HttpGet("person", Order = AlternativeRoute)]
[HttpGet("people", Order = AlternativeRoute)]
[Permission(nameof(People), Kind.Read)]
[ApiDefinition("Staff")]
[ProducesResponseType(StatusCodes.Status200OK)]
public Task<ICollection<People>> SearchPeople(string query, [FromQuery] Include<People> fields)
{
return _libraryManager.People.Search(query);
return SearchPage(await _searchManager.SearchEpisodes(query, sortBy, pagination, fields));
}
/// <summary>
@ -176,6 +175,8 @@ namespace Kyoo.Core.Api
/// Search for studios
/// </remarks>
/// <param name="query">The query to search for.</param>
/// <param name="sortBy">Sort information about the query (sort by, sort order).</param>
/// <param name="pagination">How many items per page should be returned, where should the page start...</param>
/// <param name="fields">The aditional fields to include in the result.</param>
/// <returns>A list of studios found for the specified query.</returns>
[HttpGet("studios")]
@ -183,9 +184,12 @@ namespace Kyoo.Core.Api
[Permission(nameof(Studio), Kind.Read)]
[ApiDefinition("Studios")]
[ProducesResponseType(StatusCodes.Status200OK)]
public Task<ICollection<Studio>> SearchStudios(string query, [FromQuery] Include<Studio> fields)
public async Task<SearchPage<Studio>> SearchStudios(string? query,
[FromQuery] Sort<Studio> sortBy,
[FromQuery] SearchPagination pagination,
[FromQuery] Include<Studio> fields)
{
return _libraryManager.Studios.Search(query);
return SearchPage(await _searchManager.SearchStudios(query, sortBy, pagination, fields));
}
}
}

75
back/src/Kyoo.Meilisearch/MeilisearchModule.cs
View File

@ -22,6 +22,7 @@ using Kyoo.Abstractions.Models;
using Meilisearch;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using static System.Text.Json.JsonNamingPolicy;
namespace Kyoo.Meiliseach
{
@ -50,39 +51,83 @@ namespace Kyoo.Meiliseach
{
SearchableAttributes = new[]
{
nameof(LibraryItem.Name),
nameof(LibraryItem.Slug),
nameof(LibraryItem.Aliases),
nameof(LibraryItem.Path),
nameof(LibraryItem.Tags),
CamelCase.ConvertName(nameof(LibraryItem.Name)),
CamelCase.ConvertName(nameof(LibraryItem.Slug)),
CamelCase.ConvertName(nameof(LibraryItem.Aliases)),
CamelCase.ConvertName(nameof(LibraryItem.Path)),
CamelCase.ConvertName(nameof(LibraryItem.Tags)),
// Overview could be included as well but I think it would be better without.
},
FilterableAttributes = new[]
{
nameof(LibraryItem.Genres),
nameof(LibraryItem.Status),
nameof(LibraryItem.AirDate),
nameof(Movie.StudioID),
nameof(LibraryItem.Kind),
CamelCase.ConvertName(nameof(LibraryItem.Genres)),
CamelCase.ConvertName(nameof(LibraryItem.Status)),
CamelCase.ConvertName(nameof(LibraryItem.AirDate)),
CamelCase.ConvertName(nameof(Movie.StudioID)),
CamelCase.ConvertName(nameof(LibraryItem.Kind)),
},
SortableAttributes = new[]
{
nameof(LibraryItem.AirDate),
nameof(LibraryItem.AddedDate),
CamelCase.ConvertName(nameof(LibraryItem.AirDate)),
CamelCase.ConvertName(nameof(LibraryItem.AddedDate)),
},
DisplayedAttributes = new[]
{
nameof(LibraryItem.Id),
nameof(LibraryItem.Kind),
CamelCase.ConvertName(nameof(LibraryItem.Id)),
CamelCase.ConvertName(nameof(LibraryItem.Kind)),
},
// TODO: Add stopwords
// TODO: Extend default ranking to add ratings.
});
await _CreateIndex(client, nameof(Episode), false, new Settings()
{
SearchableAttributes = new[]
{
CamelCase.ConvertName(nameof(Episode.Name)),
CamelCase.ConvertName(nameof(Episode.Overview)),
CamelCase.ConvertName(nameof(Episode.Slug)),
CamelCase.ConvertName(nameof(Episode.Path)),
},
FilterableAttributes = new[]
{
CamelCase.ConvertName(nameof(Episode.SeasonNumber)),
},
SortableAttributes = new[]
{
CamelCase.ConvertName(nameof(Episode.ReleaseDate)),
CamelCase.ConvertName(nameof(Episode.AddedDate)),
CamelCase.ConvertName(nameof(Episode.SeasonNumber)),
CamelCase.ConvertName(nameof(Episode.EpisodeNumber)),
CamelCase.ConvertName(nameof(Episode.AbsoluteNumber)),
},
DisplayedAttributes = new[]
{
CamelCase.ConvertName(nameof(Episode.Id)),
},
// TODO: Add stopwords
});
await _CreateIndex(client, nameof(Studio), false, new Settings()
{
SearchableAttributes = new[]
{
CamelCase.ConvertName(nameof(Studio.Name)),
CamelCase.ConvertName(nameof(Studio.Slug)),
},
FilterableAttributes = Array.Empty<string>(),
SortableAttributes = Array.Empty<string>(),
DisplayedAttributes = new[]
{
CamelCase.ConvertName(nameof(Studio.Id)),
},
// TODO: Add stopwords
});
}
private static async Task _CreateIndex(MeilisearchClient client, string index, bool hasKind, Settings opts)
{
TaskInfo task = await client.CreateIndexAsync(index, hasKind ? "Ref" : nameof(IResource.Id));
TaskInfo task = await client.CreateIndexAsync(index, hasKind ? "ref" : CamelCase.ConvertName(nameof(IResource.Id)));
await client.WaitForTaskAsync(task.TaskUid);
await client.Index(index).UpdateSettingsAsync(opts);
}

47
back/src/Kyoo.Meilisearch/SearchManager.cs
View File

@ -23,6 +23,7 @@ using Kyoo.Abstractions.Controllers;
using Kyoo.Abstractions.Models;
using Kyoo.Abstractions.Models.Utils;
using Meilisearch;
using static System.Text.Json.JsonNamingPolicy;
namespace Kyoo.Meiliseach;
@ -57,9 +58,17 @@ public class SearchManager : ISearchManager
IRepository<Collection>.OnCreated += (x) => _CreateOrUpdate("items", x, nameof(Collection));
IRepository<Collection>.OnEdited += (x) => _CreateOrUpdate("items", x, nameof(Collection));
IRepository<Collection>.OnDeleted += (x) => _Delete("items", x.Id, nameof(Collection));
IRepository<Episode>.OnCreated += (x) => _CreateOrUpdate(nameof(Episode), x);
IRepository<Episode>.OnEdited += (x) => _CreateOrUpdate(nameof(Episode), x);
IRepository<Episode>.OnDeleted += (x) => _Delete(nameof(Episode), x.Id);
IRepository<Studio>.OnCreated += (x) => _CreateOrUpdate(nameof(Studio), x);
IRepository<Studio>.OnEdited += (x) => _CreateOrUpdate(nameof(Studio), x);
IRepository<Studio>.OnDeleted += (x) => _Delete(nameof(Studio), x.Id);
}
private Task _CreateOrUpdate(string index, IResource item, string? kind = null)
private async Task _CreateOrUpdate(string index, IResource item, string? kind = null)
{
if (kind != null)
{
@ -67,12 +76,14 @@ public class SearchManager : ISearchManager
var dictionary = (IDictionary<string, object?>)expando;
foreach (PropertyInfo property in item.GetType().GetProperties())
dictionary.Add(property.Name, property.GetValue(item));
expando.Ref = $"{kind}/{item.Id}";
expando.Kind = kind;
return _client.Index(index).AddDocumentsAsync(new[] { item });
dictionary.Add(CamelCase.ConvertName(property.Name), property.GetValue(item));
dictionary.Add("ref", $"{kind}-{item.Id}");
expando.kind = kind;
var task = await _client.Index(index).AddDocumentsAsync(new[] { expando });
var ret = await _client.WaitForTaskAsync(task.TaskUid);
Console.WriteLine(ret.Error);
}
return _client.Index(index).AddDocumentsAsync(new[] { item });
await _client.Index(index).AddDocumentsAsync(new[] { item });
}
private Task _Delete(string index, int id, string? kind = null)
@ -144,7 +155,7 @@ public class SearchManager : ISearchManager
SearchPagination pagination,
Include<Movie>? include = default)
{
return _Search("items", query, $"Kind = {nameof(Movie)}", sortBy, pagination, include);
return _Search("items", query, $"kind = {nameof(Movie)}", sortBy, pagination, include);
}
/// <inheritdoc/>
@ -153,7 +164,7 @@ public class SearchManager : ISearchManager
SearchPagination pagination,
Include<Show>? include = default)
{
return _Search("items", query, $"Kind = {nameof(Show)}", sortBy, pagination, include);
return _Search("items", query, $"kind = {nameof(Show)}", sortBy, pagination, include);
}
/// <inheritdoc/>
@ -162,7 +173,25 @@ public class SearchManager : ISearchManager
SearchPagination pagination,
Include<Collection>? include = default)
{
return _Search("items", query, $"Kind = {nameof(Collection)}", sortBy, pagination, include);
return _Search("items", query, $"kind = {nameof(Collection)}", sortBy, pagination, include);
}
/// <inheritdoc/>
public Task<SearchPage<Episode>.SearchResult> SearchEpisodes(string? query,
Sort<Episode> sortBy,
SearchPagination pagination,
Include<Episode>? include = default)
{
return _Search(nameof(Episode), query, null, sortBy, pagination, include);
}
/// <inheritdoc/>
public Task<SearchPage<Studio>.SearchResult> SearchStudios(string? query,
Sort<Studio> sortBy,
SearchPagination pagination,
Include<Studio>? include = default)
{
return _Search(nameof(Studio), query, null, sortBy, pagination, include);
}
private class IdResource

2
scanner/scanner/scanner.py
View File

@ -40,7 +40,7 @@ class Scanner:
if len(deleted) != len(self.registered):
for x in deleted:
await self.delete(x)
else:
elif len(deleted) > 0:
logging.warning("All video files are unavailable. Check your disks.")
# We batch videos by 20 because too mutch at once kinda DDOS everything.