From 8e9cd2d2f371881c39552d4052fb9aa27f9d8f53 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Mon, 31 Jul 2023 23:54:35 +0900 Subject: [PATCH] Delete files via the scanner/monitor. Add an ignore folder --- .env.example | 3 ++ .../Repositories/EpisodeRepository.cs | 6 ++++ scanner/scanner/__init__.py | 3 +- scanner/scanner/monitor.py | 5 ++- scanner/scanner/scanner.py | 33 ++++++++++++++++--- 5 files changed, 43 insertions(+), 7 deletions(-) diff --git a/.env.example b/.env.example index 3259c744..7b059ec9 100644 --- a/.env.example +++ b/.env.example @@ -3,6 +3,9 @@ LIBRARY_ROOT=./video CACHE_ROOT=/tmp/kyoo_cache LIBRARY_LANGUAGES=en +# A pattern (regex) to ignore video files. +LIBRARY_IGNORE_PATTERN=.*/[dD]ownloads?/.* + # The following two values should be set to a random sequence of characters. # You MUST change thoses when installing kyoo (for security) AUTHENTICATION_SECRET=4c@mraGB!KRfF@kpS8739y9FcHemKxBsqqxLbdR? diff --git a/back/src/Kyoo.Core/Controllers/Repositories/EpisodeRepository.cs b/back/src/Kyoo.Core/Controllers/Repositories/EpisodeRepository.cs index e5fefb8f..65451a5f 100644 --- a/back/src/Kyoo.Core/Controllers/Repositories/EpisodeRepository.cs +++ b/back/src/Kyoo.Core/Controllers/Repositories/EpisodeRepository.cs @@ -44,6 +44,8 @@ namespace Kyoo.Core.Controllers /// private readonly IProviderRepository _providers; + private readonly IShowRepository _shows; + /// // Use absolute numbers by default and fallback to season/episodes if it does not exists. protected override Sort DefaultSort => new Sort.Conglomerate( @@ -65,6 +67,7 @@ namespace Kyoo.Core.Controllers { _database = database; _providers = providers; + _shows = shows; // Edit episode slugs when the show's slug changes. shows.OnEdited += (show) => @@ -201,10 +204,13 @@ namespace Kyoo.Core.Controllers if (obj == null) throw new ArgumentNullException(nameof(obj)); + int epCount = await _database.Episodes.Where(x => x.ShowID == obj.ShowID).Take(2).CountAsync(); _database.Entry(obj).State = EntityState.Deleted; obj.ExternalIDs.ForEach(x => _database.Entry(x).State = EntityState.Deleted); await _database.SaveChangesAsync(); await base.Delete(obj); + if (epCount == 1) + await _shows.Delete(obj.ShowID); } } } diff --git a/scanner/scanner/__init__.py b/scanner/scanner/__init__.py index bbed00bc..6ced183d 100644 --- a/scanner/scanner/__init__.py +++ b/scanner/scanner/__init__.py @@ -30,8 +30,9 @@ async def main(): logging.basicConfig(level=logging.INFO) if len(sys.argv) > 1 and sys.argv[1] == "-vv": logging.basicConfig(level=logging.DEBUG) + logging.getLogger('watchfiles').setLevel(logging.WARNING) - jsons.set_serializer(lambda x, **_: format_date(x), Optional[date | int]) + jsons.set_serializer(lambda x, **_: format_date(x), Optional[date | int]) # type: ignore async with ClientSession( json_serialize=lambda *args, **kwargs: jsons.dumps( *args, key_transformer=jsons.KEY_TRANSFORMER_CAMELCASE, **kwargs diff --git a/scanner/scanner/monitor.py b/scanner/scanner/monitor.py index 3b83d7c7..b0485b9a 100644 --- a/scanner/scanner/monitor.py +++ b/scanner/scanner/monitor.py @@ -10,10 +10,13 @@ async def monitor(path: str, scanner: Scanner): try: if event == Change.added: await scanner.identify(file) + elif event == Change.deleted: + await scanner.delete(file); + elif event == Change.modified: + pass else: print(f"Change {event} occured for file {file}") except ProviderError as e: logging.error(str(e)) except Exception as e: logging.exception("Unhandled error", exc_info=e) - print("end", flush=True) diff --git a/scanner/scanner/scanner.py b/scanner/scanner/scanner.py index f62d6f04..8921cd87 100644 --- a/scanner/scanner/scanner.py +++ b/scanner/scanner/scanner.py @@ -2,6 +2,7 @@ import os import asyncio import logging import jsons +import re from aiohttp import ClientSession from pathlib import Path from guessit import guessit @@ -19,6 +20,11 @@ class Scanner: self._client = client self._api_key = api_key self._url = os.environ.get("KYOO_URL", "http://back:5000") + try: + self._ignore_pattern = re.compile(os.environ.get("LIBRARY_IGNORE_PATTERN", "")) + except Exception as e: + self._ignore_pattern = re.compile("") + logging.error(f"Invalid ignore pattern. Ignoring. Error: {e}") self.provider = Provider.get_all(client)[0] self.cache = {"shows": {}, "seasons": {}} self.languages = languages @@ -26,13 +32,20 @@ class Scanner: async def scan(self, path: str): logging.info("Starting the scan. It can take some times...") self.registered = await self.get_registered_paths() - videos = (str(p) for p in Path(path).rglob("*") if p.is_file()) + videos = [str(p) for p in Path(path).rglob("*") if p.is_file()] + deleted = [x for x in self.registered if x not in videos] + + if len(deleted) != len(self.registered): + for x in deleted: + await self.delete(x) + else: + logging.warning("All video files are unavailable. Check your disks.") + # We batch videos by 20 because too mutch at once kinda DDOS everything. - for group in batch(videos, 20): - logging.info("Batch finished. Starting a new one") + for group in batch(iter(videos), 20): await asyncio.gather(*map(self.identify, group)) - async def get_registered_paths(self) -> List[Path]: + async def get_registered_paths(self) -> List[str]: # TODO: Once movies are separated from the api, a new endpoint should be created to check for paths. async with self._client.get( f"{self._url}/episodes", @@ -45,7 +58,7 @@ class Scanner: @log_errors async def identify(self, path: str): - if path in self.registered: + if path in self.registered or self._ignore_pattern.match(path): return raw = guessit(path, "--episode-prefer-number") @@ -139,3 +152,13 @@ class Scanner: r.raise_for_status() ret = await r.json() return ret["id"] + + async def delete(self, path: str): + logging.info("Deleting %s", path) + # TODO: Adapt this for movies as well when they are split + async with self._client.delete( + f"{self._url}/episodes?path={path}", headers={"X-API-Key": self._api_key} + ) as r: + if not r.ok: + logging.error(f"Request error: {await r.text()}") + r.raise_for_status()