diff --git a/INSTALLING.md b/INSTALLING.md index 77065924..4b219320 100644 --- a/INSTALLING.md +++ b/INSTALLING.md @@ -6,7 +6,7 @@ [`.env`](https://raw.githubusercontent.com/zoriya/Kyoo/master/.env.example) files 3. Fill the `.env` file with your configuration options 4. Look at [Hardware Acceleration section](#Hardware-Acceleration) if you need it -5. Look at [Custom Volumes](#Custom-Volumes) if you need it, +5. Look at [FAQ](#FAQ) if you need it, 6. Run `docker compose up -d` and see kyoo at `http://localhost:8901` # Installing @@ -91,7 +91,9 @@ You can also add `COMPOSE_PROFILES=nvidia` to your `.env` instead of adding the Note that most nvidia cards have an artificial limit on the number of encodes. You can confirm your card limit [here](https://developer.nvidia.com/video-encode-and-decode-gpu-support-matrix-new). This limit can also be removed by applying an [unofficial patch](https://github.com/keylase/nvidia-patch) to you driver. -# Custom volumes +# FAQ + +## Custom volumes To customize volumes, you can edit the `docker-compose.yml` manually. @@ -120,6 +122,18 @@ You can also edit the volume definition to use advanced volume drivers if you ne Don't forget to **also edit the scanner's volumes** if you edit the transcoder's volume. +## Ignoring Directories +Kyoo supports excluding specific directories from scanning and monitoring by detecting the presence of a `.ignore` file. When a directory contains a `.ignore` file, Kyoo will recursively exclude that directory and all its contents from processing. + +Example: +To exclude `/media/extras/**`, add a `.ignore` file: +```bash +touch /media/extras/.ignore +``` +Kyoo will skip `/media/extras` and its contents in all future scans and monitoring events. + # OpenID Connect Kyoo supports OpenID Connect (OIDC) for authentication. Please refer to the [OIDC.md](OIDC.md) file for more information. + + diff --git a/scanner/scanner/monitor.py b/scanner/scanner/monitor.py index e17ace46..1584622f 100644 --- a/scanner/scanner/monitor.py +++ b/scanner/scanner/monitor.py @@ -1,5 +1,5 @@ from logging import getLogger -from os.path import isdir +from os.path import isdir, dirname, exists, join from watchfiles import awatch, Change from .publisher import Publisher from .scanner import scan, get_ignore_pattern @@ -8,16 +8,33 @@ from providers.kyoo_client import KyooClient logger = getLogger(__name__) +def is_ignored_path(path: str) -> bool: + """Check if the path is within a directory that contains a `.ignore` file.""" + current_path = path + while current_path != "/": # Traverse up to the root directory + if exists(join(current_path, ".ignore")): + return True + current_path = dirname(current_path) + return False + + async def monitor(path: str, publisher: Publisher, client: KyooClient): ignore_pattern = get_ignore_pattern() async for changes in awatch(path, ignore_permission_denied=True): for event, file in changes: + # Check for ignore conditions + if is_ignored_path(file): + logger.info( + "Ignoring event %s for file %s (due to .ignore file)", event, file + ) + continue if ignore_pattern and ignore_pattern.match(file): logger.info( "Ignoring event %s for file %s (due to IGNORE_PATTERN)", event, file ) continue - logger.info("Change %s occured for file %s", event, file) + + logger.info("Change %s occurred for file %s", event, file) match event: case Change.added if isdir(file): await scan(file, publisher, client) @@ -28,4 +45,4 @@ async def monitor(path: str, publisher: Publisher, client: KyooClient): case Change.modified: pass case _: - logger.warn("Unknown file event %s (for file %s)", event, file) + logger.warning("Unknown file event %s (for file %s)", event, file) diff --git a/scanner/scanner/scanner.py b/scanner/scanner/scanner.py index 90f21daa..a5209d93 100644 --- a/scanner/scanner/scanner.py +++ b/scanner/scanner/scanner.py @@ -11,12 +11,11 @@ logger = getLogger(__name__) def get_ignore_pattern(): + """Compile ignore pattern from environment variable.""" try: pattern = os.environ.get("LIBRARY_IGNORE_PATTERN") - if pattern: - return re.compile(pattern) - return None - except Exception as e: + return re.compile(pattern) if pattern else None + except re.error as e: logger.error(f"Invalid ignore pattern. Ignoring. Error: {e}") return None @@ -25,32 +24,51 @@ async def scan( path_: Optional[str], publisher: Publisher, client: KyooClient, remove_deleted=False ): path = path_ or os.environ.get("SCANNER_LIBRARY_ROOT", "/video") + logger.info("Starting scan at %s. This may take some time...", path) - logger.info("Starting the scan. It can take some times...") ignore_pattern = get_ignore_pattern() + if ignore_pattern: + logger.info(f"Applying ignore pattern: {ignore_pattern}") - registered = await client.get_registered_paths() - videos = [ - os.path.join(dir, file) for dir, _, files in os.walk(path) for file in files - ] - if ignore_pattern is not None: - logger.info(f"Ignoring with pattern {ignore_pattern}") - videos = [p for p in videos if not ignore_pattern.match(p)] - to_register = [p for p in videos if p not in registered] + registered = set(await client.get_registered_paths()) + videos = set() + + for dirpath, dirnames, files in os.walk(path): + # Skip directories with a `.ignore` file + if ".ignore" in files: + dirnames.clear() # Prevents os.walk from descending into this directory + continue + + for file in files: + file_path = os.path.join(dirpath, file) + # Apply ignore pattern, if any + if ignore_pattern and ignore_pattern.match(file_path): + continue + videos.add(file_path) + + to_register = videos - registered + to_delete = registered - videos if remove_deleted else set() + + if not any(to_register) and any(to_delete) and len(to_delete) == len(registered): + logger.warning("All video files are unavailable. Check your disks.") + return + + # delete stale files before creating new ones to prevent potential conflicts + if to_delete: + logger.info("Removing %d stale files.", len(to_delete)) + await asyncio.gather(*[publisher.delete(path) for path in to_delete]) + + if to_register: + logger.info("Found %d new files to register.", len(to_register)) + await asyncio.gather(*[publisher.add(path) for path in to_register]) if remove_deleted: - deleted = [x for x in registered if x not in videos] - logger.info("Found %d stale files to remove.", len(deleted)) - if len(deleted) != len(registered): - await asyncio.gather(*map(publisher.delete, deleted)) - elif len(deleted) > 0: - logger.warning("All video files are unavailable. Check your disks.") + issues = set(await client.get_issues()) + issues_to_delete = issues - videos + if issues_to_delete: + logger.info("Removing %d stale issues.", len(issues_to_delete)) + await asyncio.gather( + *[client.delete_issue(issue) for issue in issues_to_delete] + ) - issues = await client.get_issues() - for x in issues: - if x not in videos: - await client.delete_issue(x) - - logger.info("Found %d new files (counting non-video files)", len(to_register)) - await asyncio.gather(*map(publisher.add, to_register)) logger.info("Scan finished for %s.", path)