diff --git a/src/calibre/srv/auto_reload.py b/src/calibre/srv/auto_reload.py new file mode 100644 index 0000000000..661e885b33 --- /dev/null +++ b/src/calibre/srv/auto_reload.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python2 +# vim:fileencoding=utf-8 +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2015, Kovid Goyal ' + +import os, sys, subprocess, signal, time +from threading import Thread + +from calibre.constants import islinux + +class NoAutoReload(EnvironmentError): + pass + +EXTENSIONS_TO_WATCH = frozenset('py pyj css js xml'.split()) +BOUNCE_INTERVAL = 2 # seconds + +if islinux: + import select + from calibre.utils.inotify import INotifyTreeWatcher + + def ignore_event(path, name): + return name and name.rpartition('.')[-1] not in EXTENSIONS_TO_WATCH + + class Watcher(object): + + def __init__(self, root_dirs, server, log): + self.server, self.log = server, log + self.fd_map = {} + for d in frozenset(root_dirs): + w = INotifyTreeWatcher(d, ignore_event) + self.fd_map[w._inotify_fd] = w + self.last_restart_time = time.time() + fpath = os.path.abspath(__file__) + d = os.path.dirname + self.base = d(d(d(d(fpath)))) + + def loop(self): + while True: + r = select.select(list(self.fd_map.iterkeys()), [], [])[0] + modified = set() + for fd in r: + w = self.fd_map[fd] + modified |= w() + if modified: + if time.time() - self.last_restart_time > BOUNCE_INTERVAL: + modified = {os.path.relpath(x, self.base) if x.startswith(self.base) else x for x in modified if x} + changed = os.pathsep.join(sorted(modified)) + self.log('') + self.log('Restarting server because of changed files:', changed) + self.log('') + self.server.restart() + self.last_restart_time = time.time() +else: + Watcher = None + + +def find_dirs_to_watch(fpath, dirs, add_default_dirs): + dirs = {os.path.abspath(x) for x in dirs} + def add(x): + if os.path.isdir(x): + dirs.add(x) + if add_default_dirs: + d = os.path.dirname + srv = d(fpath) + add(srv) + base = d(d(d(srv))) + add(os.path.join(base, 'resources', 'server')) + add(os.path.join(base, 'src', 'calibre', 'db')) + return dirs + +def join_process(p, timeout=5): + t = Thread(target=p.wait, name='JoinProcess') + t.daemon = True + t.start() + t.join(timeout) + return p.poll() + +class Worker(object): + + def __init__(self, cmd, timeout=5): + self.cmd = cmd + self.p = None + self.timeout = timeout + + def __enter__(self): + self.restart() + return self + + def __exit__(self, *args): + if self.p and self.p.poll() is None: + # SIGINT will already have been sent to the child process + self.clean_kill(send_signal=False) + + def clean_kill(self, send_signal=True): + if self.p is not None: + if send_signal: + self.p.send_signal(getattr(signal, 'CTRL_BREAK_EVENT', signal.SIGINT)) + if join_process(self.p) is None: + self.p.kill() + self.p.wait() + self.p = None + + def restart(self): + self.clean_kill() + self.p = subprocess.Popen(self.cmd, creationflags=getattr(subprocess, 'CREATE_NEW_PROCESS_GROUP', 0)) + +def auto_reload(log, dirs=frozenset(), cmd=None, add_default_dirs=True): + if Watcher is None: + raise NoAutoReload('Auto-reload is not supported on this operating system') + fpath = os.path.abspath(__file__) + if not os.access(fpath, os.R_OK): + raise NoAutoReload('Auto-reload can only be used when running from source') + if cmd is None: + cmd = list(sys.argv) + cmd.remove('--auto-reload') + dirs = find_dirs_to_watch(fpath, dirs, add_default_dirs) + log('Auto-restarting server on changes press Ctrl-C to quit') + log('Watching %d directory trees for changes' % len(dirs)) + with Worker(cmd) as server: + w = Watcher(dirs, server, log) + try: + w.loop() + except KeyboardInterrupt: + pass diff --git a/src/calibre/utils/inotify.py b/src/calibre/utils/inotify.py new file mode 100644 index 0000000000..6149d6bef9 --- /dev/null +++ b/src/calibre/utils/inotify.py @@ -0,0 +1,326 @@ +#!/usr/bin/env python2 +# vim:fileencoding=utf-8 +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__copyright__ = '2013, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import sys, os, errno, select + +class INotifyError(Exception): + pass + +class NoSuchDir(ValueError): + pass + +class BaseDirChanged(ValueError): + pass + +class DirTooLarge(ValueError): + + def __init__(self, bdir): + ValueError.__init__(self, 'The directory {0} is too large to monitor. Try increasing the value in /proc/sys/fs/inotify/max_user_watches'.format(bdir)) + +_inotify = None + +def load_inotify(): # {{{ + ''' Initialize the inotify ctypes wrapper ''' + global _inotify + if _inotify is None: + if hasattr(sys, 'getwindowsversion'): + # On windows abort before loading the C library. Windows has + # multiple, incompatible C runtimes, and we have no way of knowing + # if the one chosen by ctypes is compatible with the currently + # loaded one. + raise INotifyError('INotify not available on windows') + if sys.platform == 'darwin': + raise INotifyError('INotify not available on OS X') + import ctypes + if not hasattr(ctypes, 'c_ssize_t'): + raise INotifyError('You need python >= 2.7 to use inotify') + libc = ctypes.CDLL(None, use_errno=True) + for function in ("inotify_add_watch", "inotify_init1", "inotify_rm_watch"): + if not hasattr(libc, function): + raise INotifyError('libc is too old') + # inotify_init1() + prototype = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int, use_errno=True) + init1 = prototype(('inotify_init1', libc), ((1, "flags", 0),)) + + # inotify_add_watch() + prototype = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint32, use_errno=True) + add_watch = prototype(('inotify_add_watch', libc), ( + (1, "fd"), (1, "pathname"), (1, "mask")), use_errno=True) + + # inotify_rm_watch() + prototype = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int, ctypes.c_int, use_errno=True) + rm_watch = prototype(('inotify_rm_watch', libc), ( + (1, "fd"), (1, "wd")), use_errno=True) + + # read() + prototype = ctypes.CFUNCTYPE(ctypes.c_ssize_t, ctypes.c_int, ctypes.c_void_p, ctypes.c_size_t, use_errno=True) + read = prototype(('read', libc), ( + (1, "fd"), (1, "buf"), (1, "count")), use_errno=True) + _inotify = (init1, add_watch, rm_watch, read) + return _inotify +# }}} + +class INotify(object): + + # See for the flags defined below + + # Supported events suitable for MASK parameter of INOTIFY_ADD_WATCH. + ACCESS = 0x00000001 # File was accessed. + MODIFY = 0x00000002 # File was modified. + ATTRIB = 0x00000004 # Metadata changed. + CLOSE_WRITE = 0x00000008 # Writtable file was closed. + CLOSE_NOWRITE = 0x00000010 # Unwrittable file closed. + OPEN = 0x00000020 # File was opened. + MOVED_FROM = 0x00000040 # File was moved from X. + MOVED_TO = 0x00000080 # File was moved to Y. + CREATE = 0x00000100 # Subfile was created. + DELETE = 0x00000200 # Subfile was deleted. + DELETE_SELF = 0x00000400 # Self was deleted. + MOVE_SELF = 0x00000800 # Self was moved. + + # Events sent by the kernel. + UNMOUNT = 0x00002000 # Backing fs was unmounted. + Q_OVERFLOW = 0x00004000 # Event queued overflowed. + IGNORED = 0x00008000 # File was ignored. + + # Helper events. + CLOSE = (CLOSE_WRITE | CLOSE_NOWRITE) # Close. + MOVE = (MOVED_FROM | MOVED_TO) # Moves. + + # Special flags. + ONLYDIR = 0x01000000 # Only watch the path if it is a directory. + DONT_FOLLOW = 0x02000000 # Do not follow a sym link. + EXCL_UNLINK = 0x04000000 # Exclude events on unlinked objects. + MASK_ADD = 0x20000000 # Add to the mask of an already existing watch. + ISDIR = 0x40000000 # Event occurred against dir. + ONESHOT = 0x80000000 # Only send event once. + + # All events which a program can wait on. + ALL_EVENTS = (ACCESS | MODIFY | ATTRIB | CLOSE_WRITE | CLOSE_NOWRITE | + OPEN | MOVED_FROM | MOVED_TO | CREATE | DELETE | + DELETE_SELF | MOVE_SELF) + + # See + CLOEXEC = 0x80000 + NONBLOCK = 0x800 + + def __init__(self, cloexec=True, nonblock=True): + import ctypes, struct + self._init1, self._add_watch, self._rm_watch, self._read = load_inotify() + flags = 0 + if cloexec: + flags |= self.CLOEXEC + if nonblock: + flags |= self.NONBLOCK + self._inotify_fd = self._init1(flags) + if self._inotify_fd == -1: + raise INotifyError(os.strerror(ctypes.get_errno())) + + self._buf = ctypes.create_string_buffer(5120) + self.fenc = sys.getfilesystemencoding() or 'utf-8' + self.hdr = struct.Struct(b'iIII') + if self.fenc == 'ascii': + self.fenc = 'utf-8' + # We keep a reference to os to prevent it from being deleted + # during interpreter shutdown, which would lead to errors in the + # __del__ method + self.os = os + + def handle_error(self): + import ctypes + eno = ctypes.get_errno() + extra = '' + if eno == errno.ENOSPC: + extra = 'You may need to increase the inotify limits on your system, via /proc/sys/inotify/max_user_*' + raise OSError(eno, self.os.strerror(eno) + extra) + + def __del__(self): + # This method can be called during interpreter shutdown, which means we + # must do the absolute minimum here. Note that there could be running + # daemon threads that are trying to call other methods on this object. + try: + self.os.close(self._inotify_fd) + except (AttributeError, TypeError): + pass + + def close(self): + if hasattr(self, '_inotify_fd'): + self.os.close(self._inotify_fd) + del self.os + del self._add_watch + del self._rm_watch + del self._inotify_fd + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + def read(self, get_name=True): + import ctypes + buf = [] + while True: + num = self._read(self._inotify_fd, self._buf, len(self._buf)) + if num == 0: + break + if num < 0: + en = ctypes.get_errno() + if en == errno.EAGAIN: + break # No more data + if en == errno.EINTR: + continue # Interrupted, try again + raise OSError(en, self.os.strerror(en)) + buf.append(self._buf.raw[:num]) + raw = b''.join(buf) + pos = 0 + lraw = len(raw) + while lraw - pos >= self.hdr.size: + wd, mask, cookie, name_len = self.hdr.unpack_from(raw, pos) + pos += self.hdr.size + name = None + if get_name: + name = raw[pos:pos+name_len].rstrip(b'\0').decode(self.fenc) + pos += name_len + self.process_event(wd, mask, cookie, name) + + def process_event(self, *args): + raise NotImplementedError() + + def wait(self, timeout=None): + 'Return True iff there are events waiting to be read. Blocks if timeout is None. Polls if timeout is 0.' + return len((select.select([self._inotify_fd], [], []) if timeout is None else select.select([self._inotify_fd], [], [], timeout))[0]) > 0 + +def realpath(path): + return os.path.abspath(os.path.realpath(path)) + +class INotifyTreeWatcher(INotify): + + is_dummy = False + + def __init__(self, basedir, ignore_event=None): + super(INotifyTreeWatcher, self).__init__() + self.basedir = realpath(basedir) + self.watch_tree() + self.modified = set() + self.ignore_event = (lambda path, name: False) if ignore_event is None else ignore_event + + def watch_tree(self): + self.watched_dirs = {} + self.watched_rmap = {} + try: + self.add_watches(self.basedir) + except OSError as e: + if e.errno == errno.ENOSPC: + raise DirTooLarge(self.basedir) + + def add_watches(self, base, top_level=True): + ''' Add watches for this directory and all its descendant directories, + recursively. ''' + base = realpath(base) + # There may exist a link which leads to an endless + # add_watches loop or to maximum recursion depth exceeded + if not top_level and base in self.watched_dirs: + return + try: + is_dir = self.add_watch(base) + except OSError as e: + if e.errno == errno.ENOENT: + # The entry could have been deleted between listdir() and + # add_watch(). + if top_level: + raise NoSuchDir('The dir {0} does not exist'.format(base)) + return + if e.errno == errno.EACCES: + # We silently ignore entries for which we dont have permission, + # unless they are the top level dir + if top_level: + raise NoSuchDir('You do not have permission to monitor {0}'.format(base)) + return + raise + else: + if is_dir: + try: + files = os.listdir(base) + except OSError as e: + if e.errno in (errno.ENOTDIR, errno.ENOENT): + # The dir was deleted/replaced between the add_watch() + # and listdir() + if top_level: + raise NoSuchDir('The dir {0} does not exist'.format(base)) + return + raise + for x in files: + self.add_watches(os.path.join(base, x), top_level=False) + elif top_level: + # The top level dir is a file, not good. + raise NoSuchDir('The dir {0} does not exist'.format(base)) + + def add_watch(self, path): + import ctypes + bpath = path if isinstance(path, bytes) else path.encode(self.fenc) + wd = self._add_watch(self._inotify_fd, ctypes.c_char_p(bpath), + # Ignore symlinks and watch only directories + self.DONT_FOLLOW | self.ONLYDIR | + + self.MODIFY | self.CREATE | self.DELETE | + self.MOVE_SELF | self.MOVED_FROM | self.MOVED_TO | + self.ATTRIB | self.DELETE_SELF) + if wd == -1: + eno = ctypes.get_errno() + if eno == errno.ENOTDIR: + return False + raise OSError(eno, 'Failed to add watch for: {0}: {1}'.format(path, self.os.strerror(eno))) + self.watched_dirs[path] = wd + self.watched_rmap[wd] = path + return True + + def process_event(self, wd, mask, cookie, name): + if wd == -1 and (mask & self.Q_OVERFLOW): + # We missed some INOTIFY events, so we dont + # know the state of any tracked dirs. + self.watch_tree() + self.modified.add(None) + return + path = self.watched_rmap.get(wd, None) + if path is not None: + if not self.ignore_event(path, name): + self.modified.add(os.path.join(path, name or '')) + if mask & self.CREATE: + # A new sub-directory might have been created, monitor it. + try: + self.add_watch(os.path.join(path, name)) + except OSError as e: + if e.errno == errno.ENOENT: + # Deleted before add_watch() + pass + elif e.errno == errno.ENOSPC: + raise DirTooLarge(self.basedir) + else: + raise + if (mask & self.DELETE_SELF or mask & self.MOVE_SELF) and path == self.basedir: + raise BaseDirChanged('The directory %s was moved/deleted' % path) + + def __call__(self): + self.read() + ret = self.modified + self.modified = set() + return ret + +if __name__ == '__main__': + w = INotifyTreeWatcher(sys.argv[-1]) + w() + print ('Monitoring', sys.argv[-1], 'press Ctrl-C to stop') + try: + while w.wait(): + modified = w() + for path in modified: + print (path or sys.argv[-1], 'changed') + raise SystemExit('inotify flaked out') + except KeyboardInterrupt: + pass