mirror of
https://github.com/zoriya/Kyoo.git
synced 2025-05-31 20:24:27 -04:00
Cleanup scan + monitor with a class
This commit is contained in:
Zoe Roux
parent
07b39e0a97
commit
aeddf3366c
@ -16,88 +16,84 @@ from .queue import Request, enqueue
|
|||||||
logger = getLogger(__name__)
|
logger = getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def get_ignore_pattern():
|
class Scanner:
|
||||||
try:
|
def __init__(self, client: KyooClient):
|
||||||
pattern = os.environ.get("LIBRARY_IGNORE_PATTERN")
|
self._client = client
|
||||||
return re.compile(pattern) if pattern else None
|
self._info: VideoInfo = None # type: ignore
|
||||||
except re.error as e:
|
try:
|
||||||
logger.error(f"Invalid ignore pattern. Ignoring. Error: {e}")
|
pattern = os.environ.get("LIBRARY_IGNORE_PATTERN")
|
||||||
return None
|
self._ignore_pattern = re.compile(pattern) if pattern else None
|
||||||
|
except re.error as e:
|
||||||
|
logger.error(f"Invalid ignore pattern. Ignoring. Error: {e}")
|
||||||
|
|
||||||
|
async def scan(self, path: Optional[str], remove_deleted=False):
|
||||||
|
if path is None:
|
||||||
|
logger.info("Starting scan at %s. This may take some time...", path)
|
||||||
|
if self._ignore_pattern:
|
||||||
|
logger.info(f"Applying ignore pattern: {self._ignore_pattern}")
|
||||||
|
path = path or os.environ.get("SCANNER_LIBRARY_ROOT", "/video")
|
||||||
|
videos = self.walk_fs(path)
|
||||||
|
|
||||||
ignore_pattern = get_ignore_pattern()
|
self._info = await self._client.get_videos_info()
|
||||||
|
|
||||||
|
# TODO: handle unmatched
|
||||||
|
to_register = videos - self._info.paths
|
||||||
|
to_delete = self._info.paths - videos if remove_deleted else set()
|
||||||
|
|
||||||
def is_ignored_path(path: str) -> bool:
|
if (
|
||||||
current_path = path
|
not any(to_register)
|
||||||
# Traverse up to the root directory
|
and any(to_delete)
|
||||||
while current_path != "/":
|
and len(to_delete) == len(self._info.paths)
|
||||||
if exists(join(current_path, ".ignore")):
|
):
|
||||||
return True
|
logger.warning("All video files are unavailable. Check your disks.")
|
||||||
current_path = dirname(current_path)
|
return
|
||||||
return False
|
|
||||||
|
|
||||||
|
# delete stale files before creating new ones to prevent potential conflicts
|
||||||
|
if to_delete:
|
||||||
|
logger.info("Removing %d stale files.", len(to_delete))
|
||||||
|
await self._client.delete_videos(to_delete)
|
||||||
|
|
||||||
def walk_fs(root_path: str) -> set[str]:
|
if to_register:
|
||||||
videos: set[str] = set()
|
logger.info("Found %d new files to register.", len(to_register))
|
||||||
for dirpath, dirnames, files in os.walk(root_path):
|
await self._register(to_register)
|
||||||
# Skip directories with a `.ignore` file
|
|
||||||
if ".ignore" in files:
|
|
||||||
# Prevents os.walk from descending into this directory
|
|
||||||
dirnames.clear()
|
|
||||||
continue
|
|
||||||
|
|
||||||
for file in files:
|
logger.info("Scan finished for %s.", path)
|
||||||
file_path = os.path.join(dirpath, file)
|
|
||||||
# Apply ignore pattern, if any
|
|
||||||
if ignore_pattern and ignore_pattern.match(file_path):
|
|
||||||
continue
|
|
||||||
if is_video(file_path):
|
|
||||||
videos.add(file_path)
|
|
||||||
return videos
|
|
||||||
|
|
||||||
|
async def monitor(self, path: str, client: KyooClient):
|
||||||
|
async for changes in awatch(path, ignore_permission_denied=True):
|
||||||
|
for event, file in changes:
|
||||||
|
if not isdir(file) and not is_video(file):
|
||||||
|
continue
|
||||||
|
if (
|
||||||
|
self._ignore_pattern and self._ignore_pattern.match(file)
|
||||||
|
) or is_ignored_path(file):
|
||||||
|
logger.info("Ignoring event %s for file %s", event, file)
|
||||||
|
continue
|
||||||
|
|
||||||
def is_video(path: str) -> bool:
|
match event:
|
||||||
(mime, _) = guess_file_type(path, strict=False)
|
case Change.added if isdir(file):
|
||||||
return mime is not None and mime.startswith("video/")
|
logger.info("New dir found: %s", file)
|
||||||
|
await self.scan(file)
|
||||||
|
case Change.added:
|
||||||
async def scan(path: Optional[str], client: KyooClient, remove_deleted=False):
|
logger.info("New video found: %s", file)
|
||||||
if path is None:
|
await self._register([file])
|
||||||
logger.info("Starting scan at %s. This may take some time...", path)
|
case Change.deleted:
|
||||||
if ignore_pattern:
|
logger.info("Delete video at: %s", file)
|
||||||
logger.info(f"Applying ignore pattern: {ignore_pattern}")
|
await client.delete_videos([file])
|
||||||
path = path or os.environ.get("SCANNER_LIBRARY_ROOT", "/video")
|
case Change.modified:
|
||||||
videos = walk_fs(path)
|
pass
|
||||||
|
|
||||||
info = await client.get_videos_info()
|
|
||||||
|
|
||||||
# TODO: handle unmatched
|
|
||||||
to_register = videos - info.paths
|
|
||||||
to_delete = info.paths - videos if remove_deleted else set()
|
|
||||||
|
|
||||||
if not any(to_register) and any(to_delete) and len(to_delete) == len(info.paths):
|
|
||||||
logger.warning("All video files are unavailable. Check your disks.")
|
|
||||||
return
|
|
||||||
|
|
||||||
# delete stale files before creating new ones to prevent potential conflicts
|
|
||||||
if to_delete:
|
|
||||||
logger.info("Removing %d stale files.", len(to_delete))
|
|
||||||
await client.delete_videos(to_delete)
|
|
||||||
|
|
||||||
if to_register:
|
|
||||||
logger.info("Found %d new files to register.", len(to_register))
|
|
||||||
|
|
||||||
|
async def _register(self, videos: list[str] | set[str]):
|
||||||
# TODO: we should probably chunk those
|
# TODO: we should probably chunk those
|
||||||
vids: list[Video] = []
|
vids: list[Video] = []
|
||||||
for path in to_register:
|
for path in videos:
|
||||||
try:
|
try:
|
||||||
vid = await identify(path)
|
vid = await identify(path)
|
||||||
vid = match(info, vid)
|
vid = self._match(vid)
|
||||||
vids.append(vid)
|
vids.append(vid)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error("Couldn't identify %s.", path, exc_info=e)
|
logger.error("Couldn't identify %s.", path, exc_info=e)
|
||||||
created = await client.create_videos(vids)
|
created = await self._client.create_videos(vids)
|
||||||
|
|
||||||
await enqueue(
|
await enqueue(
|
||||||
[
|
[
|
||||||
@ -112,79 +108,87 @@ async def scan(path: Optional[str], client: KyooClient, remove_deleted=False):
|
|||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
logger.info("Scan finished for %s.", path)
|
def _match(self, video: Video) -> Video:
|
||||||
|
video.for_ = []
|
||||||
|
|
||||||
|
year_info = (
|
||||||
async def monitor(path: str, client: KyooClient):
|
self._info.guesses[video.guess.title]
|
||||||
async for changes in awatch(path, ignore_permission_denied=True):
|
if video.guess.title in self._info.guesses
|
||||||
for event, file in changes:
|
else {}
|
||||||
if not isdir(file) and not is_video(file):
|
)
|
||||||
continue
|
slugs = set(
|
||||||
if ignore_pattern and ignore_pattern.match(file) or is_ignored_path(file):
|
x
|
||||||
logger.info("Ignoring event %s for file %s", event, file)
|
for x in (
|
||||||
continue
|
[
|
||||||
|
year_info[str(y)].slug if str(y) in year_info else None
|
||||||
match event:
|
for y in video.guess.years
|
||||||
case Change.added if isdir(file):
|
]
|
||||||
logger.info("New dir found: %s", file)
|
+ ([year_info["unknown"].slug] if "unknown" in year_info else [])
|
||||||
await scan(file, client)
|
)
|
||||||
case Change.added:
|
if x is not None
|
||||||
logger.info("New video found: %s", file)
|
|
||||||
try:
|
|
||||||
vid = await identify(file)
|
|
||||||
vid = match(info, vid)
|
|
||||||
await client.create_videos([vid])
|
|
||||||
except Exception as e:
|
|
||||||
logger.error("Couldn't identify %s.", file, exc_info=e)
|
|
||||||
case Change.deleted:
|
|
||||||
logger.info("Delete video at: %s", file)
|
|
||||||
await client.delete_videos([file])
|
|
||||||
case Change.modified:
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def match(info: VideoInfo, video: Video) -> Video:
|
|
||||||
video.for_ = []
|
|
||||||
|
|
||||||
year_info = (
|
|
||||||
info.guesses[video.guess.title] if video.guess.title in info.guesses else {}
|
|
||||||
)
|
|
||||||
slugs = set(
|
|
||||||
x
|
|
||||||
for x in (
|
|
||||||
[
|
|
||||||
year_info[str(y)].slug if str(y) in year_info else None
|
|
||||||
for y in video.guess.years
|
|
||||||
]
|
|
||||||
+ ([year_info["unknown"].slug] if "unknown" in year_info else [])
|
|
||||||
)
|
)
|
||||||
if x is not None
|
|
||||||
)
|
|
||||||
|
|
||||||
if video.guess.kind == "movie":
|
if video.guess.kind == "movie":
|
||||||
for slug in slugs:
|
for slug in slugs:
|
||||||
video.for_.append(For.Movie(movie=slug))
|
video.for_.append(For.Movie(movie=slug))
|
||||||
|
|
||||||
for k, v in video.guess.external_id.items():
|
|
||||||
video.for_.append(For.ExternalId(external_id={k: MetadataId(data_id=v)}))
|
|
||||||
else:
|
|
||||||
for ep in video.guess.episodes:
|
|
||||||
if ep.season is not None:
|
|
||||||
for slug in slugs:
|
|
||||||
video.for_.append(
|
|
||||||
For.Episode(serie=slug, season=ep.season, episode=ep.episode)
|
|
||||||
)
|
|
||||||
|
|
||||||
for k, v in video.guess.external_id.items():
|
for k, v in video.guess.external_id.items():
|
||||||
video.for_.append(
|
video.for_.append(
|
||||||
For.ExternalId(
|
For.ExternalId(external_id={k: MetadataId(data_id=v)})
|
||||||
external_id={
|
|
||||||
k: EpisodeId(
|
|
||||||
serie_id=v, season=ep.season, episode=ep.episode
|
|
||||||
)
|
|
||||||
}
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
|
else:
|
||||||
|
for ep in video.guess.episodes:
|
||||||
|
if ep.season is not None:
|
||||||
|
for slug in slugs:
|
||||||
|
video.for_.append(
|
||||||
|
For.Episode(
|
||||||
|
serie=slug, season=ep.season, episode=ep.episode
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
# TODO: handle specials & movie as episodes (needs animelist or thexem)
|
for k, v in video.guess.external_id.items():
|
||||||
return video
|
video.for_.append(
|
||||||
|
For.ExternalId(
|
||||||
|
external_id={
|
||||||
|
k: EpisodeId(
|
||||||
|
serie_id=v, season=ep.season, episode=ep.episode
|
||||||
|
)
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
# TODO: handle specials & movie as episodes (needs animelist or thexem)
|
||||||
|
return video
|
||||||
|
|
||||||
|
def walk_fs(self, root_path: str) -> set[str]:
|
||||||
|
videos: set[str] = set()
|
||||||
|
for dirpath, dirnames, files in os.walk(root_path):
|
||||||
|
# Skip directories with a `.ignore` file
|
||||||
|
if ".ignore" in files:
|
||||||
|
# Prevents os.walk from descending into this directory
|
||||||
|
dirnames.clear()
|
||||||
|
continue
|
||||||
|
|
||||||
|
for file in files:
|
||||||
|
file_path = os.path.join(dirpath, file)
|
||||||
|
# Apply ignore pattern, if any
|
||||||
|
if self._ignore_pattern and self._ignore_pattern.match(file_path):
|
||||||
|
continue
|
||||||
|
if is_video(file_path):
|
||||||
|
videos.add(file_path)
|
||||||
|
return videos
|
||||||
|
|
||||||
|
|
||||||
|
def is_ignored_path(path: str) -> bool:
|
||||||
|
current_path = path
|
||||||
|
# Traverse up to the root directory
|
||||||
|
while current_path != "/":
|
||||||
|
if exists(join(current_path, ".ignore")):
|
||||||
|
return True
|
||||||
|
current_path = dirname(current_path)
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def is_video(path: str) -> bool:
|
||||||
|
(mime, _) = guess_file_type(path, strict=False)
|
||||||
|
return mime is not None and mime.startswith("video/")
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user