mirror of
https://github.com/zoriya/Kyoo.git
synced 2025-07-09 03:04:20 -04:00
Add .ignore
support in the scanner (#679)
This commit is contained in:
commit
4db01dd910
@ -6,7 +6,7 @@
|
||||
[`.env`](https://raw.githubusercontent.com/zoriya/Kyoo/master/.env.example) files
|
||||
3. Fill the `.env` file with your configuration options
|
||||
4. Look at [Hardware Acceleration section](#Hardware-Acceleration) if you need it
|
||||
5. Look at [Custom Volumes](#Custom-Volumes) if you need it,
|
||||
5. Look at [FAQ](#FAQ) if you need it,
|
||||
6. Run `docker compose up -d` and see kyoo at `http://localhost:8901`
|
||||
|
||||
# Installing
|
||||
@ -91,7 +91,9 @@ You can also add `COMPOSE_PROFILES=nvidia` to your `.env` instead of adding the
|
||||
Note that most nvidia cards have an artificial limit on the number of encodes. You can confirm your card limit [here](https://developer.nvidia.com/video-encode-and-decode-gpu-support-matrix-new).
|
||||
This limit can also be removed by applying an [unofficial patch](https://github.com/keylase/nvidia-patch) to you driver.
|
||||
|
||||
# Custom volumes
|
||||
# FAQ
|
||||
|
||||
## Custom volumes
|
||||
|
||||
To customize volumes, you can edit the `docker-compose.yml` manually.
|
||||
|
||||
@ -120,6 +122,18 @@ You can also edit the volume definition to use advanced volume drivers if you ne
|
||||
|
||||
Don't forget to **also edit the scanner's volumes** if you edit the transcoder's volume.
|
||||
|
||||
## Ignoring Directories
|
||||
Kyoo supports excluding specific directories from scanning and monitoring by detecting the presence of a `.ignore` file. When a directory contains a `.ignore` file, Kyoo will recursively exclude that directory and all its contents from processing.
|
||||
|
||||
Example:
|
||||
To exclude `/media/extras/**`, add a `.ignore` file:
|
||||
```bash
|
||||
touch /media/extras/.ignore
|
||||
```
|
||||
Kyoo will skip `/media/extras` and its contents in all future scans and monitoring events.
|
||||
|
||||
# OpenID Connect
|
||||
|
||||
Kyoo supports OpenID Connect (OIDC) for authentication. Please refer to the [OIDC.md](OIDC.md) file for more information.
|
||||
|
||||
<!-- vim: set wrap: -->
|
||||
|
@ -1,5 +1,5 @@
|
||||
from logging import getLogger
|
||||
from os.path import isdir
|
||||
from os.path import isdir, dirname, exists, join
|
||||
from watchfiles import awatch, Change
|
||||
from .publisher import Publisher
|
||||
from .scanner import scan, get_ignore_pattern
|
||||
@ -8,16 +8,33 @@ from providers.kyoo_client import KyooClient
|
||||
logger = getLogger(__name__)
|
||||
|
||||
|
||||
def is_ignored_path(path: str) -> bool:
|
||||
"""Check if the path is within a directory that contains a `.ignore` file."""
|
||||
current_path = path
|
||||
while current_path != "/": # Traverse up to the root directory
|
||||
if exists(join(current_path, ".ignore")):
|
||||
return True
|
||||
current_path = dirname(current_path)
|
||||
return False
|
||||
|
||||
|
||||
async def monitor(path: str, publisher: Publisher, client: KyooClient):
|
||||
ignore_pattern = get_ignore_pattern()
|
||||
async for changes in awatch(path, ignore_permission_denied=True):
|
||||
for event, file in changes:
|
||||
# Check for ignore conditions
|
||||
if is_ignored_path(file):
|
||||
logger.info(
|
||||
"Ignoring event %s for file %s (due to .ignore file)", event, file
|
||||
)
|
||||
continue
|
||||
if ignore_pattern and ignore_pattern.match(file):
|
||||
logger.info(
|
||||
"Ignoring event %s for file %s (due to IGNORE_PATTERN)", event, file
|
||||
)
|
||||
continue
|
||||
logger.info("Change %s occured for file %s", event, file)
|
||||
|
||||
logger.info("Change %s occurred for file %s", event, file)
|
||||
match event:
|
||||
case Change.added if isdir(file):
|
||||
await scan(file, publisher, client)
|
||||
@ -28,4 +45,4 @@ async def monitor(path: str, publisher: Publisher, client: KyooClient):
|
||||
case Change.modified:
|
||||
pass
|
||||
case _:
|
||||
logger.warn("Unknown file event %s (for file %s)", event, file)
|
||||
logger.warning("Unknown file event %s (for file %s)", event, file)
|
||||
|
@ -11,12 +11,11 @@ logger = getLogger(__name__)
|
||||
|
||||
|
||||
def get_ignore_pattern():
|
||||
"""Compile ignore pattern from environment variable."""
|
||||
try:
|
||||
pattern = os.environ.get("LIBRARY_IGNORE_PATTERN")
|
||||
if pattern:
|
||||
return re.compile(pattern)
|
||||
return None
|
||||
except Exception as e:
|
||||
return re.compile(pattern) if pattern else None
|
||||
except re.error as e:
|
||||
logger.error(f"Invalid ignore pattern. Ignoring. Error: {e}")
|
||||
return None
|
||||
|
||||
@ -25,32 +24,51 @@ async def scan(
|
||||
path_: Optional[str], publisher: Publisher, client: KyooClient, remove_deleted=False
|
||||
):
|
||||
path = path_ or os.environ.get("SCANNER_LIBRARY_ROOT", "/video")
|
||||
logger.info("Starting scan at %s. This may take some time...", path)
|
||||
|
||||
logger.info("Starting the scan. It can take some times...")
|
||||
ignore_pattern = get_ignore_pattern()
|
||||
if ignore_pattern:
|
||||
logger.info(f"Applying ignore pattern: {ignore_pattern}")
|
||||
|
||||
registered = await client.get_registered_paths()
|
||||
videos = [
|
||||
os.path.join(dir, file) for dir, _, files in os.walk(path) for file in files
|
||||
]
|
||||
if ignore_pattern is not None:
|
||||
logger.info(f"Ignoring with pattern {ignore_pattern}")
|
||||
videos = [p for p in videos if not ignore_pattern.match(p)]
|
||||
to_register = [p for p in videos if p not in registered]
|
||||
registered = set(await client.get_registered_paths())
|
||||
videos = set()
|
||||
|
||||
for dirpath, dirnames, files in os.walk(path):
|
||||
# Skip directories with a `.ignore` file
|
||||
if ".ignore" in files:
|
||||
dirnames.clear() # Prevents os.walk from descending into this directory
|
||||
continue
|
||||
|
||||
for file in files:
|
||||
file_path = os.path.join(dirpath, file)
|
||||
# Apply ignore pattern, if any
|
||||
if ignore_pattern and ignore_pattern.match(file_path):
|
||||
continue
|
||||
videos.add(file_path)
|
||||
|
||||
to_register = videos - registered
|
||||
to_delete = registered - videos if remove_deleted else set()
|
||||
|
||||
if not any(to_register) and any(to_delete) and len(to_delete) == len(registered):
|
||||
logger.warning("All video files are unavailable. Check your disks.")
|
||||
return
|
||||
|
||||
# delete stale files before creating new ones to prevent potential conflicts
|
||||
if to_delete:
|
||||
logger.info("Removing %d stale files.", len(to_delete))
|
||||
await asyncio.gather(*[publisher.delete(path) for path in to_delete])
|
||||
|
||||
if to_register:
|
||||
logger.info("Found %d new files to register.", len(to_register))
|
||||
await asyncio.gather(*[publisher.add(path) for path in to_register])
|
||||
|
||||
if remove_deleted:
|
||||
deleted = [x for x in registered if x not in videos]
|
||||
logger.info("Found %d stale files to remove.", len(deleted))
|
||||
if len(deleted) != len(registered):
|
||||
await asyncio.gather(*map(publisher.delete, deleted))
|
||||
elif len(deleted) > 0:
|
||||
logger.warning("All video files are unavailable. Check your disks.")
|
||||
issues = set(await client.get_issues())
|
||||
issues_to_delete = issues - videos
|
||||
if issues_to_delete:
|
||||
logger.info("Removing %d stale issues.", len(issues_to_delete))
|
||||
await asyncio.gather(
|
||||
*[client.delete_issue(issue) for issue in issues_to_delete]
|
||||
)
|
||||
|
||||
issues = await client.get_issues()
|
||||
for x in issues:
|
||||
if x not in videos:
|
||||
await client.delete_issue(x)
|
||||
|
||||
logger.info("Found %d new files (counting non-video files)", len(to_register))
|
||||
await asyncio.gather(*map(publisher.add, to_register))
|
||||
logger.info("Scan finished for %s.", path)
|
||||
|
Loading…
x
Reference in New Issue
Block a user