diff --git a/scanner/scanner/__init__.py b/scanner/scanner/__init__.py index b27457a1..5167937d 100644 --- a/scanner/scanner/__init__.py +++ b/scanner/scanner/__init__.py @@ -35,6 +35,7 @@ async def lifespan(_): RequestProcessor(db, client, CompositeProvider(tmdb)) as processor, get_db() as db, ): + _ = asyncio.create_task(processor.process_all()) scanner = FsScanner(client, RequestCreator(db)) if is_master: _ = asyncio.create_task(scanner.monitor()) diff --git a/scanner/scanner/requests.py b/scanner/scanner/requests.py index 2f6d7208..50f52ea3 100644 --- a/scanner/scanner/requests.py +++ b/scanner/scanner/requests.py @@ -62,7 +62,7 @@ class RequestProcessor: async def __aenter__(self): logger.info("Listening for requestes") - await self._database.add_listener("scanner_requests", self.process_request) + await self._database.add_listener("scanner_requests", self.process_all) return self async def __aexit__( @@ -71,7 +71,15 @@ class RequestProcessor: exc_value: BaseException | None, traceback: TracebackType | None, ): - await self._database.remove_listener("scanner_requests", self.process_request) + await self._database.remove_listener("scanner_requests", self.process_all) + + async def process_all(self): + found = True + while found: + try: + found = await self.process_request() + except Exception as e: + logger.error("Failed to process one of the metadata request", exc_info=e) async def process_request(self): cur = await self._database.fetchrow( @@ -80,11 +88,11 @@ class RequestProcessor: scanner.requests set status = 'running', - started_at = nom()::timestamptz + started_at = now()::timestamptz where pk in ( select - * + pk from scanner.requests where @@ -96,8 +104,9 @@ class RequestProcessor: * """ ) + logger.warning("toto %s", cur) if cur is None: - return + return False request = Request.model_validate(cur) logger.info(f"Starting to process {request.title}") @@ -127,6 +136,7 @@ class RequestProcessor: """, [request.pk], ) + return True async def _run_request(self, request: Request) -> Resource: if request.kind == "movie": diff --git a/scanner/scanner/utils.py b/scanner/scanner/utils.py index 9e71e83d..02c528f6 100644 --- a/scanner/scanner/utils.py +++ b/scanner/scanner/utils.py @@ -1,6 +1,8 @@ from abc import ABCMeta +from collections.abc import Mapping from typing import Annotated, Any, Callable, override +from asyncpg import Record from langcodes import Language as BaseLanguage from pydantic import BaseModel, ConfigDict, GetJsonSchemaHandler from pydantic.alias_generators import to_camel @@ -34,6 +36,9 @@ class Model(BaseModel): ) +Mapping.register(Record) # type: ignore + + class _LanguagePydanticAnnotation: @classmethod def __get_pydantic_core_schema__(