From d39694ae246a3d406732e2f5b95c3d7b325cbc6b Mon Sep 17 00:00:00 2001 From: Felipe Marinho Date: Thu, 14 Nov 2024 19:18:02 -0300 Subject: [PATCH 1/4] feat: add support for ".ignore" file --- scanner/scanner/monitor.py | 23 ++++++++++++++++++++--- scanner/scanner/scanner.py | 24 ++++++++++++++++-------- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/scanner/scanner/monitor.py b/scanner/scanner/monitor.py index e17ace46..1584622f 100644 --- a/scanner/scanner/monitor.py +++ b/scanner/scanner/monitor.py @@ -1,5 +1,5 @@ from logging import getLogger -from os.path import isdir +from os.path import isdir, dirname, exists, join from watchfiles import awatch, Change from .publisher import Publisher from .scanner import scan, get_ignore_pattern @@ -8,16 +8,33 @@ from providers.kyoo_client import KyooClient logger = getLogger(__name__) +def is_ignored_path(path: str) -> bool: + """Check if the path is within a directory that contains a `.ignore` file.""" + current_path = path + while current_path != "/": # Traverse up to the root directory + if exists(join(current_path, ".ignore")): + return True + current_path = dirname(current_path) + return False + + async def monitor(path: str, publisher: Publisher, client: KyooClient): ignore_pattern = get_ignore_pattern() async for changes in awatch(path, ignore_permission_denied=True): for event, file in changes: + # Check for ignore conditions + if is_ignored_path(file): + logger.info( + "Ignoring event %s for file %s (due to .ignore file)", event, file + ) + continue if ignore_pattern and ignore_pattern.match(file): logger.info( "Ignoring event %s for file %s (due to IGNORE_PATTERN)", event, file ) continue - logger.info("Change %s occured for file %s", event, file) + + logger.info("Change %s occurred for file %s", event, file) match event: case Change.added if isdir(file): await scan(file, publisher, client) @@ -28,4 +45,4 @@ async def monitor(path: str, publisher: Publisher, client: KyooClient): case Change.modified: pass case _: - logger.warn("Unknown file event %s (for file %s)", event, file) + logger.warning("Unknown file event %s (for file %s)", event, file) diff --git a/scanner/scanner/scanner.py b/scanner/scanner/scanner.py index 90f21daa..75cf420b 100644 --- a/scanner/scanner/scanner.py +++ b/scanner/scanner/scanner.py @@ -25,17 +25,25 @@ async def scan( path_: Optional[str], publisher: Publisher, client: KyooClient, remove_deleted=False ): path = path_ or os.environ.get("SCANNER_LIBRARY_ROOT", "/video") - - logger.info("Starting the scan. It can take some times...") + logger.info("Starting the scan. It can take some time...") ignore_pattern = get_ignore_pattern() registered = await client.get_registered_paths() - videos = [ - os.path.join(dir, file) for dir, _, files in os.walk(path) for file in files - ] - if ignore_pattern is not None: - logger.info(f"Ignoring with pattern {ignore_pattern}") - videos = [p for p in videos if not ignore_pattern.match(p)] + videos = [] + + for dirpath, dirnames, files in os.walk(path): + # Skip directories with a `.ignore` file + if ".ignore" in files: + dirnames.clear() # Prevents os.walk from descending into this directory + continue + + for file in files: + file_path = os.path.join(dirpath, file) + # Apply ignore pattern, if any + if ignore_pattern and ignore_pattern.match(file_path): + continue + videos.append(file_path) + to_register = [p for p in videos if p not in registered] if remove_deleted: From 8f4aecb2365fa95081b5c47c020222c8cd3b1357 Mon Sep 17 00:00:00 2001 From: Felipe Marinho Date: Thu, 14 Nov 2024 19:24:59 -0300 Subject: [PATCH 2/4] docs: add docs about ignoring dirs --- INSTALLING.md | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/INSTALLING.md b/INSTALLING.md index 77065924..4b219320 100644 --- a/INSTALLING.md +++ b/INSTALLING.md @@ -6,7 +6,7 @@ [`.env`](https://raw.githubusercontent.com/zoriya/Kyoo/master/.env.example) files 3. Fill the `.env` file with your configuration options 4. Look at [Hardware Acceleration section](#Hardware-Acceleration) if you need it -5. Look at [Custom Volumes](#Custom-Volumes) if you need it, +5. Look at [FAQ](#FAQ) if you need it, 6. Run `docker compose up -d` and see kyoo at `http://localhost:8901` # Installing @@ -91,7 +91,9 @@ You can also add `COMPOSE_PROFILES=nvidia` to your `.env` instead of adding the Note that most nvidia cards have an artificial limit on the number of encodes. You can confirm your card limit [here](https://developer.nvidia.com/video-encode-and-decode-gpu-support-matrix-new). This limit can also be removed by applying an [unofficial patch](https://github.com/keylase/nvidia-patch) to you driver. -# Custom volumes +# FAQ + +## Custom volumes To customize volumes, you can edit the `docker-compose.yml` manually. @@ -120,6 +122,18 @@ You can also edit the volume definition to use advanced volume drivers if you ne Don't forget to **also edit the scanner's volumes** if you edit the transcoder's volume. +## Ignoring Directories +Kyoo supports excluding specific directories from scanning and monitoring by detecting the presence of a `.ignore` file. When a directory contains a `.ignore` file, Kyoo will recursively exclude that directory and all its contents from processing. + +Example: +To exclude `/media/extras/**`, add a `.ignore` file: +```bash +touch /media/extras/.ignore +``` +Kyoo will skip `/media/extras` and its contents in all future scans and monitoring events. + # OpenID Connect Kyoo supports OpenID Connect (OIDC) for authentication. Please refer to the [OIDC.md](OIDC.md) file for more information. + + From 6992de5e2f9f4c33c83d25b1a006bce91e30cddd Mon Sep 17 00:00:00 2001 From: Felipe Marinho Date: Thu, 14 Nov 2024 19:41:50 -0300 Subject: [PATCH 3/4] refactor: use sets to avoid costly loops --- scanner/scanner/scanner.py | 51 +++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/scanner/scanner/scanner.py b/scanner/scanner/scanner.py index 75cf420b..02633c71 100644 --- a/scanner/scanner/scanner.py +++ b/scanner/scanner/scanner.py @@ -11,12 +11,11 @@ logger = getLogger(__name__) def get_ignore_pattern(): + """Compile ignore pattern from environment variable.""" try: pattern = os.environ.get("LIBRARY_IGNORE_PATTERN") - if pattern: - return re.compile(pattern) - return None - except Exception as e: + return re.compile(pattern) if pattern else None + except re.error as e: logger.error(f"Invalid ignore pattern. Ignoring. Error: {e}") return None @@ -25,11 +24,14 @@ async def scan( path_: Optional[str], publisher: Publisher, client: KyooClient, remove_deleted=False ): path = path_ or os.environ.get("SCANNER_LIBRARY_ROOT", "/video") - logger.info("Starting the scan. It can take some time...") - ignore_pattern = get_ignore_pattern() + logger.info("Starting scan at %s. This may take some time...", path) - registered = await client.get_registered_paths() - videos = [] + ignore_pattern = get_ignore_pattern() + if ignore_pattern: + logger.info(f"Applying ignore pattern: {ignore_pattern}") + + registered = set(await client.get_registered_paths()) + videos = set() for dirpath, dirnames, files in os.walk(path): # Skip directories with a `.ignore` file @@ -42,23 +44,26 @@ async def scan( # Apply ignore pattern, if any if ignore_pattern and ignore_pattern.match(file_path): continue - videos.append(file_path) + videos.add(file_path) - to_register = [p for p in videos if p not in registered] + to_register = videos - registered + to_delete = registered - videos if remove_deleted else set() + + if to_register: + logger.info("Found %d new files to register.", len(to_register)) + await asyncio.gather(*[publisher.add(path) for path in to_register]) + + if to_delete: + logger.info("Removing %d stale files.", len(to_delete)) + await asyncio.gather(*[publisher.delete(path) for path in to_delete]) if remove_deleted: - deleted = [x for x in registered if x not in videos] - logger.info("Found %d stale files to remove.", len(deleted)) - if len(deleted) != len(registered): - await asyncio.gather(*map(publisher.delete, deleted)) - elif len(deleted) > 0: - logger.warning("All video files are unavailable. Check your disks.") + issues = set(await client.get_issues()) + issues_to_delete = issues - videos + if issues_to_delete: + logger.info("Removing %d stale issues.", len(issues_to_delete)) + await asyncio.gather( + *[client.delete_issue(issue) for issue in issues_to_delete] + ) - issues = await client.get_issues() - for x in issues: - if x not in videos: - await client.delete_issue(x) - - logger.info("Found %d new files (counting non-video files)", len(to_register)) - await asyncio.gather(*map(publisher.add, to_register)) logger.info("Scan finished for %s.", path) From 46543d1f74936beb7fdb2c70cde765e7453a2445 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Wed, 1 Jan 2025 21:21:58 +0100 Subject: [PATCH 4/4] fix: reorder delete/register --- scanner/scanner/scanner.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/scanner/scanner/scanner.py b/scanner/scanner/scanner.py index 02633c71..a5209d93 100644 --- a/scanner/scanner/scanner.py +++ b/scanner/scanner/scanner.py @@ -49,14 +49,19 @@ async def scan( to_register = videos - registered to_delete = registered - videos if remove_deleted else set() - if to_register: - logger.info("Found %d new files to register.", len(to_register)) - await asyncio.gather(*[publisher.add(path) for path in to_register]) + if not any(to_register) and any(to_delete) and len(to_delete) == len(registered): + logger.warning("All video files are unavailable. Check your disks.") + return + # delete stale files before creating new ones to prevent potential conflicts if to_delete: logger.info("Removing %d stale files.", len(to_delete)) await asyncio.gather(*[publisher.delete(path) for path in to_delete]) + if to_register: + logger.info("Found %d new files to register.", len(to_register)) + await asyncio.gather(*[publisher.add(path) for path in to_register]) + if remove_deleted: issues = set(await client.get_issues()) issues_to_delete = issues - videos