Cleanup startup events

This commit is contained in:
Zoe Roux 2025-05-11 16:40:31 +02:00
parent 6427aafc4d
commit 99b93f0f78
No known key found for this signature in database
7 changed files with 95 additions and 72 deletions

View File

@ -99,6 +99,6 @@ RABBITMQ_DEFAULT_PASS=aohohunuhouhuhhoahothonseuhaoensuthoaentsuhha
# v5 stuff, does absolutely nothing on master (aka: you can delete this) # v5 stuff, does absolutely nothing on master (aka: you can delete this)
EXTRA_CLAIMS='{"permissions": ["core.read"], "verified": false}' EXTRA_CLAIMS='{"permissions": ["core.read"], "verified": false}'
FIRST_USER_CLAIMS='{"permissions": ["users.read", "users.write", "apikeys.read", "apikeys.write", "users.delete", "core.read", "core.write"], "verified": true}' FIRST_USER_CLAIMS='{"permissions": ["users.read", "users.write", "apikeys.read", "apikeys.write", "users.delete", "core.read", "core.write", "scanner.trigger"], "verified": true}'
GUEST_CLAIMS='{"permissions": ["core.read"]}' GUEST_CLAIMS='{"permissions": ["core.read"]}'
PROTECTED_CLAIMS="permissions,verified" PROTECTED_CLAIMS="permissions,verified"

View File

@ -1,64 +1,43 @@
import asyncio import asyncio
import logging import logging
from contextlib import asynccontextmanager from contextlib import asynccontextmanager
from typing import Annotated
import asyncpg from fastapi import FastAPI
from fastapi import BackgroundTasks, FastAPI, Security
from .client import KyooClient from scanner.client import KyooClient
from .fsscan import Scanner from .database import get_db, init_pool
from .jwt import validate_bearer from scanner.fsscan import Scanner
from .providers.composite import CompositeProvider from scanner.providers.composite import CompositeProvider
from .providers.themoviedatabase import TheMovieDatabase from scanner.providers.themoviedatabase import TheMovieDatabase
from .requests import RequestCreator, RequestProcessor from scanner.requests import RequestCreator, RequestProcessor
logging.basicConfig(level=logging.INFO) logging.basicConfig(level=logging.INFO)
logging.getLogger("watchfiles").setLevel(logging.WARNING) logging.getLogger("watchfiles").setLevel(logging.WARNING)
logging.getLogger("rebulk").setLevel(logging.WARNING) logging.getLogger("rebulk").setLevel(logging.WARNING)
scanner: Scanner
@asynccontextmanager @asynccontextmanager
async def lifetime(): async def lifespan(_):
print("starting lifetime")
async with ( async with (
await asyncpg.create_pool() as pool, init_pool(),
create_request_processor(pool) as processor, get_db() as db,
create_scanner(pool) as (scan, is_master), KyooClient() as client,
TheMovieDatabase() as tmdb,
): ):
global scanner processor = RequestProcessor(db, client, CompositeProvider(tmdb))
scanner = scan
await processor.listen_for_requests() await processor.listen_for_requests()
if is_master:
_ = await asyncio.gather(
scanner.scan(remove_deleted=True),
scanner.monitor(),
)
yield
@asynccontextmanager
async def create_request_processor(pool: asyncpg.Pool):
async with ( async with (
pool.acquire() as db, get_db() as db,
KyooClient() as client,
TheMovieDatabase() as themoviedb,
):
yield RequestProcessor(db, client, CompositeProvider(themoviedb))
@asynccontextmanager
async def create_scanner(pool: asyncpg.Pool):
async with (
pool.acquire() as db,
KyooClient() as client, KyooClient() as client,
): ):
scanner = Scanner(client, RequestCreator(db))
# there's no way someone else used the same id, right? # there's no way someone else used the same id, right?
is_master: bool = await db.fetchval("select pg_try_advisory_lock(198347)") is_master = await db.fetchval("select pg_try_advisory_lock(198347)")
yield (Scanner(client, RequestCreator(db)), is_master) if is_master:
print("this is master")
_ = await asyncio.create_task(scanner.scan(remove_deleted=True))
_ = await asyncio.create_task(scanner.monitor())
yield
app = FastAPI( app = FastAPI(
@ -66,20 +45,5 @@ app = FastAPI(
description="API to control the long running scanner or interacting with external databases (themoviedb, tvdb...)\n\n" description="API to control the long running scanner or interacting with external databases (themoviedb, tvdb...)\n\n"
+ "Most of those APIs are for admins only.", + "Most of those APIs are for admins only.",
root_path="/scanner", root_path="/scanner",
lifetime=lifetime, lifespan=lifespan,
) )
@app.put(
"/scan",
status_code=204,
response_description="Scan started.",
)
async def trigger_scan(
tasks: BackgroundTasks,
_: Annotated[None, Security(validate_bearer, scopes=["scanner.trigger"])],
):
"""
Trigger a full scan of the filesystem, trying to find new videos & deleting old ones.
"""
tasks.add_task(scanner.scan)

View File

@ -0,0 +1,20 @@
from contextlib import asynccontextmanager
from typing import cast
from asyncpg import Connection, Pool, create_pool
pool: Pool
@asynccontextmanager
async def init_pool():
async with await create_pool() as p:
global pool
pool = p
yield
@asynccontextmanager
async def get_db():
async with pool.acquire() as db:
yield cast(Connection, db)

View File

@ -3,7 +3,9 @@ import re
from logging import getLogger from logging import getLogger
from mimetypes import guess_file_type from mimetypes import guess_file_type
from os.path import dirname, exists, isdir, join from os.path import dirname, exists, isdir, join
from typing import Annotated
from fastapi import Depends
from watchfiles import Change, awatch from watchfiles import Change, awatch
from .client import KyooClient from .client import KyooClient
@ -16,7 +18,11 @@ logger = getLogger(__name__)
class Scanner: class Scanner:
def __init__(self, client: KyooClient, requests: RequestCreator): def __init__(
self,
client: Annotated[KyooClient, Depends],
requests: Annotated[RequestCreator, Depends],
):
self._client = client self._client = client
self._requests = requests self._requests = requests
self._info: VideoInfo = None # type: ignore self._info: VideoInfo = None # type: ignore

View File

@ -1,14 +1,19 @@
from typing import override from typing import Annotated, override
from fastapi import Depends
from langcodes import Language from langcodes import Language
from ..models.movie import Movie, SearchMovie from ..models.movie import Movie, SearchMovie
from ..models.serie import SearchSerie, Serie from ..models.serie import SearchSerie, Serie
from .provider import Provider from .provider import Provider
from .themoviedatabase import TheMovieDatabase
class CompositeProvider(Provider): class CompositeProvider(Provider):
def __init__(self, themoviedb: Provider): def __init__(
self,
themoviedb: Annotated[TheMovieDatabase, Depends],
):
self._tvdb: Provider = None # type: ignore self._tvdb: Provider = None # type: ignore
self._themoviedb = themoviedb self._themoviedb = themoviedb

View File

@ -1,12 +1,14 @@
from __future__ import annotations from __future__ import annotations
from logging import getLogger from logging import getLogger
from typing import Literal from typing import Annotated, Literal
from asyncpg import Connection from asyncpg import Connection
from fastapi import Depends
from pydantic import Field from pydantic import Field
from .client import KyooClient from .client import KyooClient
from .database import get_db
from .models.videos import Guess, Resource from .models.videos import Guess, Resource
from .providers.composite import CompositeProvider from .providers.composite import CompositeProvider
from .utils import Model from .utils import Model
@ -28,7 +30,10 @@ class Request(Model, extra="allow"):
class RequestCreator: class RequestCreator:
def __init__(self, database: Connection): def __init__(
self,
database: Annotated[Connection, Depends(get_db)],
):
self._database = database self._database = database
async def enqueue(self, requests: list[Request]): async def enqueue(self, requests: list[Request]):
@ -48,9 +53,9 @@ class RequestCreator:
class RequestProcessor: class RequestProcessor:
def __init__( def __init__(
self, self,
database: Connection, database: Annotated[Connection, Depends(get_db)],
client: KyooClient, client: Annotated[KyooClient, Depends],
providers: CompositeProvider, providers: Annotated[CompositeProvider, Depends],
): ):
self._database = database self._database = database
self._client = client self._client = client

23
scanner/scanner/routes.py Normal file
View File

@ -0,0 +1,23 @@
from typing import Annotated
from fastapi import BackgroundTasks, Depends, Security
from scanner import app
from scanner.fsscan import Scanner
from scanner.jwt import validate_bearer
@app.put(
"/scan",
status_code=204,
response_description="Scan started.",
)
async def trigger_scan(
tasks: BackgroundTasks,
scanner: Annotated[Scanner, Depends],
_: Annotated[None, Security(validate_bearer, scopes=["scanner.trigger"])],
):
"""
Trigger a full scan of the filesystem, trying to find new videos & deleting old ones.
"""
tasks.add_task(scanner.scan)