Add rescan button on admin interface (#462)

This commit is contained in:
Zoe Roux 2024-05-02 01:50:11 +02:00 committed by GitHub
commit 9d00aa6d2f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
15 changed files with 105 additions and 65 deletions

View File

@ -23,5 +23,6 @@ namespace Kyoo.Abstractions.Controllers;
public interface IScanner public interface IScanner
{ {
Task SendRescanRequest();
Task SendRefreshRequest(string kind, Guid id); Task SendRefreshRequest(string kind, Guid id);
} }

View File

@ -317,7 +317,7 @@ public abstract record Filter<T> : Filter
); );
public static readonly Parser<Filter<T>> Ge = _GetOperationParser( public static readonly Parser<Filter<T>> Ge = _GetOperationParser(
Parse.IgnoreCase("ge").Or(Parse.String(">=")).Token(), Parse.IgnoreCase("ge").Or(Parse.IgnoreCase("gte")).Or(Parse.String(">=")).Token(),
(property, value) => new Ge(property, value) (property, value) => new Ge(property, value)
); );
@ -327,7 +327,7 @@ public abstract record Filter<T> : Filter
); );
public static readonly Parser<Filter<T>> Le = _GetOperationParser( public static readonly Parser<Filter<T>> Le = _GetOperationParser(
Parse.IgnoreCase("le").Or(Parse.String("<=")).Token(), Parse.IgnoreCase("le").Or(Parse.IgnoreCase("lte")).Or(Parse.String("<=")).Token(),
(property, value) => new Le(property, value) (property, value) => new Le(property, value)
); );

View File

@ -19,6 +19,7 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Threading.Tasks; using System.Threading.Tasks;
using Kyoo.Abstractions.Controllers;
using Kyoo.Abstractions.Models.Permissions; using Kyoo.Abstractions.Models.Permissions;
using Kyoo.Core.Controllers; using Kyoo.Core.Controllers;
using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Http;
@ -65,6 +66,22 @@ public class Misc(MiscRepository repo) : BaseApi
return NoContent(); return NoContent();
} }
/// <summary>
/// Rescan library
/// </summary>
/// <remark>
/// Trigger a complete library rescan
/// </remark>
/// <returns>Nothing</returns>
[HttpPost("/rescan")]
[PartialPermission(Kind.Write)]
[ProducesResponseType(StatusCodes.Status204NoContent)]
public async Task<IActionResult> RescanLibrary([FromServices] IScanner scanner)
{
await scanner.SendRescanRequest();
return NoContent();
}
/// <summary> /// <summary>
/// List items to refresh. /// List items to refresh.
/// </summary> /// </summary>

View File

@ -34,7 +34,7 @@ namespace Kyoo.Core.Api;
/// </summary> /// </summary>
[Route("search")] [Route("search")]
[ApiController] [ApiController]
[ApiDefinition("Search", Group = ResourcesGroup)] [ApiDefinition("Search", Group = OtherGroup)]
public class SearchApi : BaseApi public class SearchApi : BaseApi
{ {
private readonly ISearchManager _searchManager; private readonly ISearchManager _searchManager;
@ -60,7 +60,7 @@ public class SearchApi : BaseApi
[HttpGet("collections")] [HttpGet("collections")]
[HttpGet("collection", Order = AlternativeRoute)] [HttpGet("collection", Order = AlternativeRoute)]
[Permission(nameof(Collection), Kind.Read)] [Permission(nameof(Collection), Kind.Read)]
[ApiDefinition("Collections")] [ApiDefinition("Collections", Group = ResourcesGroup)]
[ProducesResponseType(StatusCodes.Status200OK)] [ProducesResponseType(StatusCodes.Status200OK)]
public async Task<SearchPage<Collection>> SearchCollections( public async Task<SearchPage<Collection>> SearchCollections(
[FromQuery] string? q, [FromQuery] string? q,
@ -86,7 +86,7 @@ public class SearchApi : BaseApi
[HttpGet("shows")] [HttpGet("shows")]
[HttpGet("show", Order = AlternativeRoute)] [HttpGet("show", Order = AlternativeRoute)]
[Permission(nameof(Show), Kind.Read)] [Permission(nameof(Show), Kind.Read)]
[ApiDefinition("Show")] [ApiDefinition("Shows", Group = ResourcesGroup)]
[ProducesResponseType(StatusCodes.Status200OK)] [ProducesResponseType(StatusCodes.Status200OK)]
public async Task<SearchPage<Show>> SearchShows( public async Task<SearchPage<Show>> SearchShows(
[FromQuery] string? q, [FromQuery] string? q,
@ -112,7 +112,7 @@ public class SearchApi : BaseApi
[HttpGet("movies")] [HttpGet("movies")]
[HttpGet("movie", Order = AlternativeRoute)] [HttpGet("movie", Order = AlternativeRoute)]
[Permission(nameof(Movie), Kind.Read)] [Permission(nameof(Movie), Kind.Read)]
[ApiDefinition("Movie")] [ApiDefinition("Movies", Group = ResourcesGroup)]
[ProducesResponseType(StatusCodes.Status200OK)] [ProducesResponseType(StatusCodes.Status200OK)]
public async Task<SearchPage<Movie>> SearchMovies( public async Task<SearchPage<Movie>> SearchMovies(
[FromQuery] string? q, [FromQuery] string? q,
@ -138,7 +138,7 @@ public class SearchApi : BaseApi
[HttpGet("items")] [HttpGet("items")]
[HttpGet("item", Order = AlternativeRoute)] [HttpGet("item", Order = AlternativeRoute)]
[Permission(nameof(ILibraryItem), Kind.Read)] [Permission(nameof(ILibraryItem), Kind.Read)]
[ApiDefinition("Item")] [ApiDefinition("Items", Group = ResourcesGroup)]
[ProducesResponseType(StatusCodes.Status200OK)] [ProducesResponseType(StatusCodes.Status200OK)]
public async Task<SearchPage<ILibraryItem>> SearchItems( public async Task<SearchPage<ILibraryItem>> SearchItems(
[FromQuery] string? q, [FromQuery] string? q,
@ -164,7 +164,7 @@ public class SearchApi : BaseApi
[HttpGet("episodes")] [HttpGet("episodes")]
[HttpGet("episode", Order = AlternativeRoute)] [HttpGet("episode", Order = AlternativeRoute)]
[Permission(nameof(Episode), Kind.Read)] [Permission(nameof(Episode), Kind.Read)]
[ApiDefinition("Episodes")] [ApiDefinition("Episodes", Group = ResourcesGroup)]
[ProducesResponseType(StatusCodes.Status200OK)] [ProducesResponseType(StatusCodes.Status200OK)]
public async Task<SearchPage<Episode>> SearchEpisodes( public async Task<SearchPage<Episode>> SearchEpisodes(
[FromQuery] string? q, [FromQuery] string? q,
@ -190,7 +190,7 @@ public class SearchApi : BaseApi
[HttpGet("studios")] [HttpGet("studios")]
[HttpGet("studio", Order = AlternativeRoute)] [HttpGet("studio", Order = AlternativeRoute)]
[Permission(nameof(Studio), Kind.Read)] [Permission(nameof(Studio), Kind.Read)]
[ApiDefinition("Studios")] [ApiDefinition("Studios", Group = MetadataGroup)]
[ProducesResponseType(StatusCodes.Status200OK)] [ProducesResponseType(StatusCodes.Status200OK)]
public async Task<SearchPage<Studio>> SearchStudios( public async Task<SearchPage<Studio>> SearchStudios(
[FromQuery] string? q, [FromQuery] string? q,

View File

@ -50,7 +50,7 @@ public class ShowApi(ILibraryManager libraryManager) : CrudThumbsApi<Show>(libra
/// </remarks> /// </remarks>
/// <param name="identifier">The ID or slug of the <see cref="Show"/>.</param> /// <param name="identifier">The ID or slug of the <see cref="Show"/>.</param>
/// <returns>Nothing</returns> /// <returns>Nothing</returns>
/// <response code="404">No episode with the given ID or slug could be found.</response> /// <response code="404">No show with the given ID or slug could be found.</response>
[HttpPost("{identifier:id}/refresh")] [HttpPost("{identifier:id}/refresh")]
[PartialPermission(Kind.Write)] [PartialPermission(Kind.Write)]
[ProducesResponseType(StatusCodes.Status204NoContent)] [ProducesResponseType(StatusCodes.Status204NoContent)]

View File

@ -19,7 +19,6 @@
using System.Text; using System.Text;
using System.Text.Json; using System.Text.Json;
using Kyoo.Abstractions.Controllers; using Kyoo.Abstractions.Controllers;
using Kyoo.Abstractions.Models;
using Kyoo.Utils; using Kyoo.Utils;
using RabbitMQ.Client; using RabbitMQ.Client;
@ -35,29 +34,17 @@ public class ScannerProducer : IScanner
_channel.QueueDeclare("scanner", exclusive: false, autoDelete: false); _channel.QueueDeclare("scanner", exclusive: false, autoDelete: false);
} }
private IRepository<T>.ResourceEventHandler _Publish<T>( private Task _Publish<T>(T message)
string exchange,
string type,
string action
)
where T : IResource, IQuery
{ {
return (T resource) => var body = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(message, Utility.JsonOptions));
{ _channel.BasicPublish("", routingKey: "scanner", body: body);
Message<T> message = return Task.CompletedTask;
new() }
{
Action = action, public Task SendRescanRequest()
Type = type, {
Value = resource, var message = new { Action = "rescan", };
}; return _Publish(message);
_channel.BasicPublish(
exchange,
routingKey: message.AsRoutingKey(),
body: message.AsBytes()
);
return Task.CompletedTask;
};
} }
public Task SendRefreshRequest(string kind, Guid id) public Task SendRefreshRequest(string kind, Guid id)
@ -68,8 +55,6 @@ public class ScannerProducer : IScanner
Kind = kind.ToLowerInvariant(), Kind = kind.ToLowerInvariant(),
Id = id Id = id
}; };
var body = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(message, Utility.JsonOptions)); return _Publish(message);
_channel.BasicPublish("", routingKey: "scanner", body: body);
return Task.CompletedTask;
} }
} }

View File

@ -144,6 +144,9 @@ export const AccountProvider = ({
const oldSelected = useRef<{ id: string; token: string } | null>( const oldSelected = useRef<{ id: string; token: string } | null>(
selected ? { id: selected.id, token: selected.token.access_token } : null, selected ? { id: selected.id, token: selected.token.access_token } : null,
); );
const [permissionError, setPermissionError] = useState<KyooErrors | null>(null);
const userIsError = user.isError; const userIsError = user.isError;
useEffect(() => { useEffect(() => {
// if the user change account (or connect/disconnect), reset query cache. // if the user change account (or connect/disconnect), reset query cache.
@ -152,6 +155,7 @@ export const AccountProvider = ({
(userIsError && selected?.token.access_token !== oldSelected.current?.token) (userIsError && selected?.token.access_token !== oldSelected.current?.token)
) { ) {
initialSsrError.current = undefined; initialSsrError.current = undefined;
setPermissionError(null);
queryClient.resetQueries(); queryClient.resetQueries();
} }
oldSelected.current = selected ? { id: selected.id, token: selected.token.access_token } : null; oldSelected.current = selected ? { id: selected.id, token: selected.token.access_token } : null;
@ -164,8 +168,6 @@ export const AccountProvider = ({
} }
}, [selected, queryClient, userIsError]); }, [selected, queryClient, userIsError]);
const [permissionError, setPermissionError] = useState<KyooErrors | null>(null);
return ( return (
<AccountContext.Provider value={accounts}> <AccountContext.Provider value={accounts}>
<ConnectionErrorContext.Provider <ConnectionErrorContext.Provider

View File

@ -18,24 +18,44 @@
* along with Kyoo. If not, see <https://www.gnu.org/licenses/>. * along with Kyoo. If not, see <https://www.gnu.org/licenses/>.
*/ */
import { Issue, IssueP, QueryIdentifier, useFetch } from "@kyoo/models"; import { Issue, IssueP, QueryIdentifier, queryFn, useFetch } from "@kyoo/models";
import { useTranslation } from "react-i18next"; import { useTranslation } from "react-i18next";
import { SettingsContainer } from "../settings/base"; import { SettingsContainer } from "../settings/base";
import { Icon, P, Skeleton, tooltip, ts } from "@kyoo/primitives"; import { Button, Icon, P, Skeleton, tooltip, ts } from "@kyoo/primitives";
import { ErrorView } from "../errors"; import { ErrorView } from "../errors";
import { z } from "zod"; import { z } from "zod";
import { View } from "react-native"; import { View } from "react-native";
import { useYoshiki } from "yoshiki/native"; import { useYoshiki } from "yoshiki/native";
import Info from "@material-symbols/svg-400/outlined/info.svg"; import Info from "@material-symbols/svg-400/outlined/info.svg";
import Scan from "@material-symbols/svg-400/outlined/sensors.svg";
import { useMutation } from "@tanstack/react-query";
export const Scanner = () => { export const Scanner = () => {
const { css } = useYoshiki(); const { css } = useYoshiki();
const { t } = useTranslation(); const { t } = useTranslation();
const { data, error } = useFetch(Scanner.query()); const { data, error } = useFetch(Scanner.query());
const metadataRefreshMutation = useMutation({
mutationFn: () =>
queryFn({
path: ["rescan"],
method: "POST",
}),
});
return ( return (
<SettingsContainer title={t("admin.scanner.label")}> <SettingsContainer
title={t("admin.scanner.label")}
extraTop={
<Button
licon={<Icon icon={Scan} {...css({ marginX: ts(1) })} />}
text={t("admin.scanner.scan")}
onPress={() => metadataRefreshMutation.mutate()}
{...css({ marginBottom: ts(2) })}
/>
}
>
<> <>
{error != null ? ( {error != null ? (
<ErrorView error={error} /> <ErrorView error={error} />

View File

@ -166,7 +166,7 @@ EpisodeList.query = (
parser: EpisodeP, parser: EpisodeP,
path: ["show", slug, "episode"], path: ["show", slug, "episode"],
params: { params: {
seasonNumber: season ? `gte:${season}` : undefined, filter: season ? `seasonNumber gte ${season}` : undefined,
fields: ["watchStatus"], fields: ["watchStatus"],
}, },
infinite: { infinite: {

View File

@ -88,17 +88,20 @@ export const SettingsContainer = ({
children, children,
title, title,
extra, extra,
extraTop,
...props ...props
}: { }: {
children: ReactElement | (ReactElement | Falsy)[] | Falsy; children: ReactElement | (ReactElement | Falsy)[] | Falsy;
title: string; title: string;
extra?: ReactElement; extra?: ReactElement;
extraTop?: ReactElement;
}) => { }) => {
const { css } = useYoshiki(); const { css } = useYoshiki();
return ( return (
<Container {...props}> <Container {...props}>
<H1 {...css({ fontSize: rem(2) })}>{title}</H1> <H1 {...css({ fontSize: rem(2) })}>{title}</H1>
{extraTop}
<SwitchVariant> <SwitchVariant>
{({ css }) => ( {({ css }) => (
<View <View

View File

@ -237,6 +237,7 @@
}, },
"scanner": { "scanner": {
"label": "Scanner", "label": "Scanner",
"scan": "Trigger library scan",
"empty": "No issue found. All your items are registered." "empty": "No issue found. All your items are registered."
} }
} }

View File

@ -237,6 +237,7 @@
}, },
"scanner": { "scanner": {
"label": "Scanner", "label": "Scanner",
"scan": "Déclencher le scan de la bibliothèque",
"empty": "Aucun problème trouvé. Toutes vos vidéos sont enregistrés." "empty": "Aucun problème trouvé. Toutes vos vidéos sont enregistrés."
} }
} }

View File

@ -1,11 +1,12 @@
import asyncio import asyncio
from typing import Union, Literal from typing import Union, Literal
from msgspec import Struct, json from msgspec import Struct, json
import os
from logging import getLogger from logging import getLogger
from aio_pika import connect_robust
from aio_pika.abc import AbstractIncomingMessage from aio_pika.abc import AbstractIncomingMessage
from scanner.publisher import Publisher
from scanner.scanner import scan
from matcher.matcher import Matcher from matcher.matcher import Matcher
logger = getLogger(__name__) logger = getLogger(__name__)
@ -28,38 +29,29 @@ class Refresh(Message):
id: str id: str
decoder = json.Decoder(Union[Scan, Delete, Refresh]) class Rescan(Message):
pass
class Subscriber: decoder = json.Decoder(Union[Scan, Delete, Refresh, Rescan])
QUEUE = "scanner"
async def __aenter__(self):
self._con = await connect_robust(
host=os.environ.get("RABBITMQ_HOST", "rabbitmq"),
port=int(os.environ.get("RABBITMQ_PORT", "5672")),
login=os.environ.get("RABBITMQ_DEFAULT_USER", "guest"),
password=os.environ.get("RABBITMQ_DEFAULT_PASS", "guest"),
)
self._channel = await self._con.channel()
self._queue = await self._channel.declare_queue(self.QUEUE)
return self
async def __aexit__(self, exc_type, exc_value, exc_tb): class Subscriber(Publisher):
await self._con.close() async def listen(self, matcher: Matcher):
async def listen(self, scanner: Matcher):
async def on_message(message: AbstractIncomingMessage): async def on_message(message: AbstractIncomingMessage):
try: try:
msg = decoder.decode(message.body) msg = decoder.decode(message.body)
ack = False ack = False
match msg: match msg:
case Scan(path): case Scan(path):
ack = await scanner.identify(path) ack = await matcher.identify(path)
case Delete(path): case Delete(path):
ack = await scanner.delete(path) ack = await matcher.delete(path)
case Refresh(kind, id): case Refresh(kind, id):
ack = await scanner.refresh(kind, id) ack = await matcher.refresh(kind, id)
case Rescan():
await scan(None, self, matcher._client)
ack = True
case _: case _:
logger.error(f"Invalid action: {msg.action}") logger.error(f"Invalid action: {msg.action}")
if ack: if ack:

View File

@ -58,6 +58,16 @@ class KyooClient:
logger.error(f"Request error: {await r.text()}") logger.error(f"Request error: {await r.text()}")
r.raise_for_status() r.raise_for_status()
async def get_issues(self) -> List[str]:
async with self.client.get(
f"{self._url}/issues",
params={"limit": 0},
headers={"X-API-Key": self._api_key},
) as r:
r.raise_for_status()
ret = await r.json()
return [x["cause"] for x in ret if x["domain"] == "scanner"]
async def delete_issue(self, path: str): async def delete_issue(self, path: str):
async with self.client.delete( async with self.client.delete(
f'{self._url}/issues?filter=domain eq scanner and cause eq "{quote(path)}"', f'{self._url}/issues?filter=domain eq scanner and cause eq "{quote(path)}"',

View File

@ -1,6 +1,7 @@
import os import os
import re import re
import asyncio import asyncio
from typing import Optional
from logging import getLogger from logging import getLogger
from .publisher import Publisher from .publisher import Publisher
@ -10,8 +11,10 @@ logger = getLogger(__name__)
async def scan( async def scan(
path: str, publisher: Publisher, client: KyooClient, remove_deleted=False path_: Optional[str], publisher: Publisher, client: KyooClient, remove_deleted=False
): ):
path = path_ or os.environ.get("SCANNER_LIBRARY_ROOT", "/video")
logger.info("Starting the scan. It can take some times...") logger.info("Starting the scan. It can take some times...")
ignore_pattern = None ignore_pattern = None
try: try:
@ -35,5 +38,10 @@ async def scan(
elif len(deleted) > 0: elif len(deleted) > 0:
logger.warning("All video files are unavailable. Check your disks.") logger.warning("All video files are unavailable. Check your disks.")
issues = await client.get_issues()
for x in issues:
if x not in videos:
await client.delete_issue(x)
await asyncio.gather(*map(publisher.add, to_register)) await asyncio.gather(*map(publisher.add, to_register))
logger.info(f"Scan finished for {path}.") logger.info(f"Scan finished for {path}.")