From 6076e3af87010be1cf95fc560b62f3f07fe277ac Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Thu, 2 May 2024 00:40:03 +0200 Subject: [PATCH 01/10] Fix season selector query --- front/packages/ui/src/details/season.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/front/packages/ui/src/details/season.tsx b/front/packages/ui/src/details/season.tsx index cff1dc74..af6a83a8 100644 --- a/front/packages/ui/src/details/season.tsx +++ b/front/packages/ui/src/details/season.tsx @@ -166,7 +166,7 @@ EpisodeList.query = ( parser: EpisodeP, path: ["show", slug, "episode"], params: { - seasonNumber: season ? `gte:${season}` : undefined, + filter: season ? `seasonNumber gte ${season}` : undefined, fields: ["watchStatus"], }, infinite: { From 57842ea31ccfa1e1e6151e071479a088ff004b14 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Thu, 2 May 2024 00:40:13 +0200 Subject: [PATCH 02/10] Add gte and lte as valid filters --- back/src/Kyoo.Abstractions/Models/Utils/Filter.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/back/src/Kyoo.Abstractions/Models/Utils/Filter.cs b/back/src/Kyoo.Abstractions/Models/Utils/Filter.cs index 6c03dc08..430c1218 100644 --- a/back/src/Kyoo.Abstractions/Models/Utils/Filter.cs +++ b/back/src/Kyoo.Abstractions/Models/Utils/Filter.cs @@ -317,7 +317,7 @@ public abstract record Filter : Filter ); public static readonly Parser> Ge = _GetOperationParser( - Parse.IgnoreCase("ge").Or(Parse.String(">=")).Token(), + Parse.IgnoreCase("ge").Or(Parse.IgnoreCase("gte")).Or(Parse.String(">=")).Token(), (property, value) => new Ge(property, value) ); @@ -327,7 +327,7 @@ public abstract record Filter : Filter ); public static readonly Parser> Le = _GetOperationParser( - Parse.IgnoreCase("le").Or(Parse.String("<=")).Token(), + Parse.IgnoreCase("le").Or(Parse.IgnoreCase("lte")).Or(Parse.String("<=")).Token(), (property, value) => new Le(property, value) ); From 6925c6b2256b97b183fe1470e9ec557697025407 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Thu, 2 May 2024 00:51:56 +0200 Subject: [PATCH 03/10] Clean issues about removed items on scanner startup --- scanner/providers/kyoo_client.py | 10 ++++++++++ scanner/scanner/scanner.py | 5 +++++ 2 files changed, 15 insertions(+) diff --git a/scanner/providers/kyoo_client.py b/scanner/providers/kyoo_client.py index 19751d35..5920db4e 100644 --- a/scanner/providers/kyoo_client.py +++ b/scanner/providers/kyoo_client.py @@ -58,6 +58,16 @@ class KyooClient: logger.error(f"Request error: {await r.text()}") r.raise_for_status() + async def get_issues(self) -> List[str]: + async with self.client.get( + f"{self._url}/issues", + params={"limit": 0}, + headers={"X-API-Key": self._api_key}, + ) as r: + r.raise_for_status() + ret = await r.json() + return [x["cause"] for x in ret if x["domain"] == "scanner"] + async def delete_issue(self, path: str): async with self.client.delete( f'{self._url}/issues?filter=domain eq scanner and cause eq "{quote(path)}"', diff --git a/scanner/scanner/scanner.py b/scanner/scanner/scanner.py index 3275188a..0fa1d70a 100644 --- a/scanner/scanner/scanner.py +++ b/scanner/scanner/scanner.py @@ -35,5 +35,10 @@ async def scan( elif len(deleted) > 0: logger.warning("All video files are unavailable. Check your disks.") + issues = await client.get_issues() + for x in issues: + if x not in videos: + await client.delete_issue(x) + await asyncio.gather(*map(publisher.add, to_register)) logger.info(f"Scan finished for {path}.") From d63ad87971881de05497ff270ab8b1a3edf8775e Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Thu, 2 May 2024 00:52:16 +0200 Subject: [PATCH 04/10] Cleanup swagger groups definitions of search apis --- back/src/Kyoo.Core/Views/Resources/SearchApi.cs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/back/src/Kyoo.Core/Views/Resources/SearchApi.cs b/back/src/Kyoo.Core/Views/Resources/SearchApi.cs index c0043c15..b3754d3c 100644 --- a/back/src/Kyoo.Core/Views/Resources/SearchApi.cs +++ b/back/src/Kyoo.Core/Views/Resources/SearchApi.cs @@ -34,7 +34,7 @@ namespace Kyoo.Core.Api; /// [Route("search")] [ApiController] -[ApiDefinition("Search", Group = ResourcesGroup)] +[ApiDefinition("Search", Group = OtherGroup)] public class SearchApi : BaseApi { private readonly ISearchManager _searchManager; @@ -60,7 +60,7 @@ public class SearchApi : BaseApi [HttpGet("collections")] [HttpGet("collection", Order = AlternativeRoute)] [Permission(nameof(Collection), Kind.Read)] - [ApiDefinition("Collections")] + [ApiDefinition("Collections", Group = ResourcesGroup)] [ProducesResponseType(StatusCodes.Status200OK)] public async Task> SearchCollections( [FromQuery] string? q, @@ -86,7 +86,7 @@ public class SearchApi : BaseApi [HttpGet("shows")] [HttpGet("show", Order = AlternativeRoute)] [Permission(nameof(Show), Kind.Read)] - [ApiDefinition("Show")] + [ApiDefinition("Shows", Group = ResourcesGroup)] [ProducesResponseType(StatusCodes.Status200OK)] public async Task> SearchShows( [FromQuery] string? q, @@ -112,7 +112,7 @@ public class SearchApi : BaseApi [HttpGet("movies")] [HttpGet("movie", Order = AlternativeRoute)] [Permission(nameof(Movie), Kind.Read)] - [ApiDefinition("Movie")] + [ApiDefinition("Movies", Group = ResourcesGroup)] [ProducesResponseType(StatusCodes.Status200OK)] public async Task> SearchMovies( [FromQuery] string? q, @@ -138,7 +138,7 @@ public class SearchApi : BaseApi [HttpGet("items")] [HttpGet("item", Order = AlternativeRoute)] [Permission(nameof(ILibraryItem), Kind.Read)] - [ApiDefinition("Item")] + [ApiDefinition("Items", Group = ResourcesGroup)] [ProducesResponseType(StatusCodes.Status200OK)] public async Task> SearchItems( [FromQuery] string? q, @@ -164,7 +164,7 @@ public class SearchApi : BaseApi [HttpGet("episodes")] [HttpGet("episode", Order = AlternativeRoute)] [Permission(nameof(Episode), Kind.Read)] - [ApiDefinition("Episodes")] + [ApiDefinition("Episodes", Group = ResourcesGroup)] [ProducesResponseType(StatusCodes.Status200OK)] public async Task> SearchEpisodes( [FromQuery] string? q, @@ -190,7 +190,7 @@ public class SearchApi : BaseApi [HttpGet("studios")] [HttpGet("studio", Order = AlternativeRoute)] [Permission(nameof(Studio), Kind.Read)] - [ApiDefinition("Studios")] + [ApiDefinition("Studios", Group = MetadataGroup)] [ProducesResponseType(StatusCodes.Status200OK)] public async Task> SearchStudios( [FromQuery] string? q, From c1ed16b871c4bd2797b252110abeb6313d79f49e Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Thu, 2 May 2024 01:02:10 +0200 Subject: [PATCH 05/10] Add an api to publish a rescan request --- .../Kyoo.Abstractions/Controllers/IScanner.cs | 1 + back/src/Kyoo.Core/Views/Admin/Misc.cs | 17 +++++++++ back/src/Kyoo.Core/Views/Resources/ShowApi.cs | 2 +- back/src/Kyoo.RabbitMq/ScannerProducer.cs | 37 ++++++------------- 4 files changed, 30 insertions(+), 27 deletions(-) diff --git a/back/src/Kyoo.Abstractions/Controllers/IScanner.cs b/back/src/Kyoo.Abstractions/Controllers/IScanner.cs index 62f14f9f..776131b5 100644 --- a/back/src/Kyoo.Abstractions/Controllers/IScanner.cs +++ b/back/src/Kyoo.Abstractions/Controllers/IScanner.cs @@ -23,5 +23,6 @@ namespace Kyoo.Abstractions.Controllers; public interface IScanner { + Task SendRescanRequest(); Task SendRefreshRequest(string kind, Guid id); } diff --git a/back/src/Kyoo.Core/Views/Admin/Misc.cs b/back/src/Kyoo.Core/Views/Admin/Misc.cs index 3f0f9c96..edd53514 100644 --- a/back/src/Kyoo.Core/Views/Admin/Misc.cs +++ b/back/src/Kyoo.Core/Views/Admin/Misc.cs @@ -19,6 +19,7 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; +using Kyoo.Abstractions.Controllers; using Kyoo.Abstractions.Models.Permissions; using Kyoo.Core.Controllers; using Microsoft.AspNetCore.Http; @@ -65,6 +66,22 @@ public class Misc(MiscRepository repo) : BaseApi return NoContent(); } + /// + /// Rescan library + /// + /// + /// Trigger a complete library rescan + /// + /// Nothing + [HttpPost("/rescan")] + [PartialPermission(Kind.Write)] + [ProducesResponseType(StatusCodes.Status204NoContent)] + public async Task RescanLibrary([FromServices] IScanner scanner) + { + await scanner.SendRescanRequest(); + return NoContent(); + } + /// /// List items to refresh. /// diff --git a/back/src/Kyoo.Core/Views/Resources/ShowApi.cs b/back/src/Kyoo.Core/Views/Resources/ShowApi.cs index fb23e4f3..0517095a 100644 --- a/back/src/Kyoo.Core/Views/Resources/ShowApi.cs +++ b/back/src/Kyoo.Core/Views/Resources/ShowApi.cs @@ -50,7 +50,7 @@ public class ShowApi(ILibraryManager libraryManager) : CrudThumbsApi(libra /// /// The ID or slug of the . /// Nothing - /// No episode with the given ID or slug could be found. + /// No show with the given ID or slug could be found. [HttpPost("{identifier:id}/refresh")] [PartialPermission(Kind.Write)] [ProducesResponseType(StatusCodes.Status204NoContent)] diff --git a/back/src/Kyoo.RabbitMq/ScannerProducer.cs b/back/src/Kyoo.RabbitMq/ScannerProducer.cs index bc9db49d..75c575a0 100644 --- a/back/src/Kyoo.RabbitMq/ScannerProducer.cs +++ b/back/src/Kyoo.RabbitMq/ScannerProducer.cs @@ -19,7 +19,6 @@ using System.Text; using System.Text.Json; using Kyoo.Abstractions.Controllers; -using Kyoo.Abstractions.Models; using Kyoo.Utils; using RabbitMQ.Client; @@ -35,29 +34,17 @@ public class ScannerProducer : IScanner _channel.QueueDeclare("scanner", exclusive: false, autoDelete: false); } - private IRepository.ResourceEventHandler _Publish( - string exchange, - string type, - string action - ) - where T : IResource, IQuery + private Task _Publish(T message) { - return (T resource) => - { - Message message = - new() - { - Action = action, - Type = type, - Value = resource, - }; - _channel.BasicPublish( - exchange, - routingKey: message.AsRoutingKey(), - body: message.AsBytes() - ); - return Task.CompletedTask; - }; + var body = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(message, Utility.JsonOptions)); + _channel.BasicPublish("", routingKey: "scanner", body: body); + return Task.CompletedTask; + } + + public Task SendRescanRequest() + { + var message = new { Action = "rescan", }; + return _Publish(message); } public Task SendRefreshRequest(string kind, Guid id) @@ -68,8 +55,6 @@ public class ScannerProducer : IScanner Kind = kind.ToLowerInvariant(), Id = id }; - var body = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(message, Utility.JsonOptions)); - _channel.BasicPublish("", routingKey: "scanner", body: body); - return Task.CompletedTask; + return _Publish(message); } } From 21414d6c2f26038f83777436673240ed24c2a9ab Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Thu, 2 May 2024 01:07:46 +0200 Subject: [PATCH 06/10] Make the scanner's consumer also a publisher --- scanner/matcher/subscriber.py | 29 +++++++---------------------- 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/scanner/matcher/subscriber.py b/scanner/matcher/subscriber.py index acbe3e36..b2db3366 100644 --- a/scanner/matcher/subscriber.py +++ b/scanner/matcher/subscriber.py @@ -1,7 +1,6 @@ import asyncio from typing import Union, Literal from msgspec import Struct, json -import os from logging import getLogger from aio_pika import connect_robust from aio_pika.abc import AbstractIncomingMessage @@ -28,38 +27,24 @@ class Refresh(Message): id: str + + decoder = json.Decoder(Union[Scan, Delete, Refresh]) -class Subscriber: - QUEUE = "scanner" - - async def __aenter__(self): - self._con = await connect_robust( - host=os.environ.get("RABBITMQ_HOST", "rabbitmq"), - port=int(os.environ.get("RABBITMQ_PORT", "5672")), - login=os.environ.get("RABBITMQ_DEFAULT_USER", "guest"), - password=os.environ.get("RABBITMQ_DEFAULT_PASS", "guest"), - ) - self._channel = await self._con.channel() - self._queue = await self._channel.declare_queue(self.QUEUE) - return self - - async def __aexit__(self, exc_type, exc_value, exc_tb): - await self._con.close() - - async def listen(self, scanner: Matcher): +class Subscriber(Publisher): + async def listen(self, matcher: Matcher): async def on_message(message: AbstractIncomingMessage): try: msg = decoder.decode(message.body) ack = False match msg: case Scan(path): - ack = await scanner.identify(path) + ack = await matcher.identify(path) case Delete(path): - ack = await scanner.delete(path) + ack = await matcher.delete(path) case Refresh(kind, id): - ack = await scanner.refresh(kind, id) + ack = await matcher.refresh(kind, id) case _: logger.error(f"Invalid action: {msg.action}") if ack: From 3521d577c105fff617d5c6172bc379ddbd90f474 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Thu, 2 May 2024 01:22:53 +0200 Subject: [PATCH 07/10] Add rescan capabilities --- scanner/matcher/subscriber.py | 11 +++++++++-- scanner/scanner/scanner.py | 5 ++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/scanner/matcher/subscriber.py b/scanner/matcher/subscriber.py index b2db3366..fc7ec589 100644 --- a/scanner/matcher/subscriber.py +++ b/scanner/matcher/subscriber.py @@ -2,9 +2,11 @@ import asyncio from typing import Union, Literal from msgspec import Struct, json from logging import getLogger -from aio_pika import connect_robust from aio_pika.abc import AbstractIncomingMessage +from scanner.publisher import Publisher +from scanner.scanner import scan + from matcher.matcher import Matcher logger = getLogger(__name__) @@ -27,9 +29,11 @@ class Refresh(Message): id: str +class Rescan(Message): + pass -decoder = json.Decoder(Union[Scan, Delete, Refresh]) +decoder = json.Decoder(Union[Scan, Delete, Refresh, Rescan]) class Subscriber(Publisher): @@ -45,6 +49,9 @@ class Subscriber(Publisher): ack = await matcher.delete(path) case Refresh(kind, id): ack = await matcher.refresh(kind, id) + case Rescan(): + await scan(None, self, matcher._client) + ack = True case _: logger.error(f"Invalid action: {msg.action}") if ack: diff --git a/scanner/scanner/scanner.py b/scanner/scanner/scanner.py index 0fa1d70a..811dce2b 100644 --- a/scanner/scanner/scanner.py +++ b/scanner/scanner/scanner.py @@ -1,6 +1,7 @@ import os import re import asyncio +from typing import Optional from logging import getLogger from .publisher import Publisher @@ -10,8 +11,10 @@ logger = getLogger(__name__) async def scan( - path: str, publisher: Publisher, client: KyooClient, remove_deleted=False + path_: Optional[str], publisher: Publisher, client: KyooClient, remove_deleted=False ): + path = path_ or os.environ.get("SCANNER_LIBRARY_ROOT", "/video") + logger.info("Starting the scan. It can take some times...") ignore_pattern = None try: From efd2ac179d60f7f89551edb63ba10b50875f5adc Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Thu, 2 May 2024 01:23:06 +0200 Subject: [PATCH 08/10] Add rescan button on admin interface --- front/packages/ui/src/admin/scanner.tsx | 26 ++++++++++++++++++++++--- front/packages/ui/src/settings/base.tsx | 3 +++ front/translations/en.json | 1 + front/translations/fr.json | 1 + 4 files changed, 28 insertions(+), 3 deletions(-) diff --git a/front/packages/ui/src/admin/scanner.tsx b/front/packages/ui/src/admin/scanner.tsx index 59d0af30..a267a822 100644 --- a/front/packages/ui/src/admin/scanner.tsx +++ b/front/packages/ui/src/admin/scanner.tsx @@ -18,24 +18,44 @@ * along with Kyoo. If not, see . */ -import { Issue, IssueP, QueryIdentifier, useFetch } from "@kyoo/models"; +import { Issue, IssueP, QueryIdentifier, queryFn, useFetch } from "@kyoo/models"; import { useTranslation } from "react-i18next"; import { SettingsContainer } from "../settings/base"; -import { Icon, P, Skeleton, tooltip, ts } from "@kyoo/primitives"; +import { Button, Icon, P, Skeleton, tooltip, ts } from "@kyoo/primitives"; import { ErrorView } from "../errors"; import { z } from "zod"; import { View } from "react-native"; import { useYoshiki } from "yoshiki/native"; import Info from "@material-symbols/svg-400/outlined/info.svg"; +import Scan from "@material-symbols/svg-400/outlined/sensors.svg"; +import { useMutation } from "@tanstack/react-query"; export const Scanner = () => { const { css } = useYoshiki(); const { t } = useTranslation(); const { data, error } = useFetch(Scanner.query()); + const metadataRefreshMutation = useMutation({ + mutationFn: () => + queryFn({ + path: ["rescan"], + method: "POST", + }), + }); + return ( - + } + text={t("admin.scanner.scan")} + onPress={() => metadataRefreshMutation.mutate()} + {...css({ marginBottom: ts(2), })} + /> + } + > <> {error != null ? ( diff --git a/front/packages/ui/src/settings/base.tsx b/front/packages/ui/src/settings/base.tsx index e2bbec68..4c6cfbe0 100644 --- a/front/packages/ui/src/settings/base.tsx +++ b/front/packages/ui/src/settings/base.tsx @@ -88,17 +88,20 @@ export const SettingsContainer = ({ children, title, extra, + extraTop, ...props }: { children: ReactElement | (ReactElement | Falsy)[] | Falsy; title: string; extra?: ReactElement; + extraTop?: ReactElement; }) => { const { css } = useYoshiki(); return (

{title}

+ {extraTop} {({ css }) => ( Date: Thu, 2 May 2024 01:27:57 +0200 Subject: [PATCH 09/10] Format code --- front/packages/ui/src/admin/scanner.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/front/packages/ui/src/admin/scanner.tsx b/front/packages/ui/src/admin/scanner.tsx index a267a822..08c84910 100644 --- a/front/packages/ui/src/admin/scanner.tsx +++ b/front/packages/ui/src/admin/scanner.tsx @@ -52,7 +52,7 @@ export const Scanner = () => { licon={} text={t("admin.scanner.scan")} onPress={() => metadataRefreshMutation.mutate()} - {...css({ marginBottom: ts(2), })} + {...css({ marginBottom: ts(2) })} /> } > From 13ade4dc44571e10b11a125d1d137a37cd2c95d8 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Thu, 2 May 2024 01:35:21 +0200 Subject: [PATCH 10/10] Clear permissions errors on login --- front/packages/models/src/accounts.tsx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/front/packages/models/src/accounts.tsx b/front/packages/models/src/accounts.tsx index b92a9ac8..668710f9 100644 --- a/front/packages/models/src/accounts.tsx +++ b/front/packages/models/src/accounts.tsx @@ -144,6 +144,9 @@ export const AccountProvider = ({ const oldSelected = useRef<{ id: string; token: string } | null>( selected ? { id: selected.id, token: selected.token.access_token } : null, ); + + const [permissionError, setPermissionError] = useState(null); + const userIsError = user.isError; useEffect(() => { // if the user change account (or connect/disconnect), reset query cache. @@ -152,6 +155,7 @@ export const AccountProvider = ({ (userIsError && selected?.token.access_token !== oldSelected.current?.token) ) { initialSsrError.current = undefined; + setPermissionError(null); queryClient.resetQueries(); } oldSelected.current = selected ? { id: selected.id, token: selected.token.access_token } : null; @@ -164,8 +168,6 @@ export const AccountProvider = ({ } }, [selected, queryClient, userIsError]); - const [permissionError, setPermissionError] = useState(null); - return (