From 6e874b5377bffd9cb93935bd5450398f7943bcea Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 27 Oct 2015 20:33:56 +0530 Subject: [PATCH] --auto-reload now automatically refreshes the page in browsers as well Uses a WebSocket to send a signal to all browsers that have loaded the page. --- resources/content-server/autoreload.js | 45 ++++++++ resources/content-server/index.html | 34 +++--- src/calibre/srv/auto_reload.py | 143 +++++++++++++++++++++---- src/calibre/srv/code.py | 36 ++++++- src/calibre/srv/http_response.py | 5 +- src/calibre/srv/loop.py | 5 +- src/calibre/srv/standalone.py | 1 + src/pyj/srv.pyj | 7 +- 8 files changed, 234 insertions(+), 42 deletions(-) create mode 100644 resources/content-server/autoreload.js diff --git a/resources/content-server/autoreload.js b/resources/content-server/autoreload.js new file mode 100644 index 0000000000..32990d3c5d --- /dev/null +++ b/resources/content-server/autoreload.js @@ -0,0 +1,45 @@ +/* vim:fileencoding=utf-8 + * + * Copyright (C) 2015 Kovid Goyal + * + * Distributed under terms of the GPLv3 license + */ + +(function(autoreload_port) { + "use strict;"; + var url = 'ws://127.0.0.1:' + autoreload_port; + + function ReconnectingWebSocket() { + self = this; + self.retries = 0; + self.interval = 100; + + self.reconnect = function() { + self.ws = new WebSocket(url); + + self.ws.onopen = function(event) { + self.retries = 0; + self.interval = 100; + console.log('Connected to reloading WebSocket server at port: ' + autoreload_port); + }; + + self.ws.onmessage = function(event) { + console.log('Received mesasge from reload server: ' + event.data); + if (event.data === 'reload') window.location.reload(true); + }; + + self.ws.onclose = function(event) { + console.log('Connection to reload server closed with code: ' + event.code + ' and reason: ' + event.reason); + self.retries += 1; + if (self.retries < 60) { + self.interval *= 2; + setTimeout(self.reconnect, self.interval); + } + }; + }; + self.reconnect(); + } + + var sock = new ReconnectingWebSocket(); +})(AUTORELOAD_PORT); + diff --git a/resources/content-server/index.html b/resources/content-server/index.html index 8f9eff1599..5460c8cc33 100644 --- a/resources/content-server/index.html +++ b/resources/content-server/index.html @@ -3,27 +3,27 @@ calibre - - - - + + + + - - + + -
- - -
Loading library, please wait…
- -
+ +
diff --git a/src/calibre/srv/auto_reload.py b/src/calibre/srv/auto_reload.py index 67ccfa8f4b..67f013b700 100644 --- a/src/calibre/srv/auto_reload.py +++ b/src/calibre/srv/auto_reload.py @@ -6,37 +6,44 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import os, sys, subprocess, signal, time, errno -from threading import Thread +import os, sys, subprocess, signal, time, errno, socket +from threading import Thread, Lock from calibre.constants import islinux, iswindows, isosx +from calibre.srv.http_response import create_http_handler +from calibre.srv.loop import ServerLoop +from calibre.srv.opts import Options +from calibre.srv.standalone import create_option_parser +from calibre.srv.web_socket import DummyHandler +from calibre.utils.monotonic import monotonic class NoAutoReload(EnvironmentError): pass +# Filesystem watcher {{{ class WatcherBase(object): EXTENSIONS_TO_WATCH = frozenset('py pyj'.split()) BOUNCE_INTERVAL = 2 # seconds - def __init__(self, server, log): - self.server, self.log = server, log + def __init__(self, worker, log): + self.worker, self.log = worker, log fpath = os.path.abspath(__file__) d = os.path.dirname self.base = d(d(d(d(fpath)))) - self.last_restart_time = time.time() + self.last_restart_time = monotonic() def handle_modified(self, modified): if modified: - if time.time() - self.last_restart_time > self.BOUNCE_INTERVAL: + if monotonic() - self.last_restart_time > self.BOUNCE_INTERVAL: modified = {os.path.relpath(x, self.base) if x.startswith(self.base) else x for x in modified if x} changed = os.pathsep.join(sorted(modified)) self.log('') self.log.warn('Restarting server because of changed files:', changed) self.log('') - self.server.restart() - self.last_restart_time = time.time() + self.worker.restart() + self.last_restart_time = monotonic() def file_is_watched(self, fname): return fname and fname.rpartition('.')[-1] in self.EXTENSIONS_TO_WATCH @@ -47,8 +54,8 @@ if islinux: class Watcher(WatcherBase): - def __init__(self, root_dirs, server, log): - WatcherBase.__init__(self, server, log) + def __init__(self, root_dirs, worker, log): + WatcherBase.__init__(self, worker, log) self.fd_map = {} for d in frozenset(root_dirs): w = INotifyTreeWatcher(d, self.ignore_event) @@ -113,8 +120,8 @@ elif iswindows: class Watcher(WatcherBase): - def __init__(self, root_dirs, server, log): - WatcherBase.__init__(self, server, log) + def __init__(self, root_dirs, worker, log): + WatcherBase.__init__(self, worker, log) self.watchers = [] self.modified_queue = Queue() for d in frozenset(root_dirs): @@ -135,8 +142,8 @@ elif isosx: class Watcher(WatcherBase): - def __init__(self, root_dirs, server, log): - WatcherBase.__init__(self, server, log) + def __init__(self, root_dirs, worker, log): + WatcherBase.__init__(self, worker, log) self.stream = Stream(self.notify, *(x.encode('utf-8') for x in root_dirs), file_events=True) def loop(self): @@ -178,6 +185,7 @@ def find_dirs_to_watch(fpath, dirs, add_default_dirs): add(os.path.join(base, 'src', 'calibre', 'db')) add(os.path.join(base, 'src', 'pyj')) return dirs +# }}} def join_process(p, timeout=5): t = Thread(target=p.wait, name='JoinProcess') @@ -188,11 +196,32 @@ def join_process(p, timeout=5): class Worker(object): - def __init__(self, cmd, log, timeout=5): + def __init__(self, cmd, log, server, timeout=5): self.cmd = cmd self.log = log + self.server = server self.p = None self.timeout = timeout + cmd = self.cmd + if 'calibre-debug' in cmd[0].lower(): + try: + idx = cmd.index('--') + except ValueError: + cmd = ['srv'] + else: + cmd = ['srv'] + cmd[idx+1:] + + opts = create_option_parser().parse_args(cmd)[0] + self.port = opts.port + self.connection_timeout = opts.timeout + t = Thread(name='PingThread', target=self.ping_thread) + t.daemon = True + t.start() + + def ping_thread(self): + while True: + self.server.ping() + time.sleep(0.9 * self.connection_timeout) def __enter__(self): self.restart() @@ -222,8 +251,8 @@ class Worker(object): # Happens if the editor deletes and replaces a file being edited if e.errno != errno.ENOENT or not getattr(e, 'filename', False): raise - st = time.time() - while not os.path.exists(e.filename) and time.time() - st < 3: + st = monotonic() + while not os.path.exists(e.filename) and monotonic() - st < 3: time.sleep(0.01) compile_srv() except CompileFailure as e: @@ -234,6 +263,82 @@ class Worker(object): break self.p = subprocess.Popen(self.cmd, creationflags=getattr(subprocess, 'CREATE_NEW_PROCESS_GROUP', 0)) + self.wait_for_listen() + self.server.notify_reload() + + def wait_for_listen(self): + st = monotonic() + while monotonic() - st < 5: + try: + return socket.create_connection(('localhost', self.port), 5).close() + except socket.error: + time.sleep(0.01) + self.log.error('Restarted server did not start listening on:', self.port) + +# WebSocket reload notifier {{{ + +class ReloadHandler(DummyHandler): + + def __init__(self, *args, **kw): + DummyHandler.__init__(self, *args, **kw) + self.connections = {} + self.conn_lock = Lock() + + def handle_websocket_upgrade(self, connection_id, connection_ref, inheaders): + with self.conn_lock: + self.connections[connection_id] = connection_ref + + def handle_websocket_close(self, connection_id): + with self.conn_lock: + self.connections.pop(connection_id, None) + + def notify_reload(self): + with self.conn_lock: + for connref in self.connections.itervalues(): + conn = connref() + if conn is not None and conn.ready: + conn.send_websocket_message('reload') + + def ping(self): + with self.conn_lock: + for connref in self.connections.itervalues(): + conn = connref() + if conn is not None and conn.ready: + conn.send_websocket_ping() + + +class ReloadServer(Thread): + + daemon = True + + def __init__(self): + Thread.__init__(self, name='ReloadServer') + self.reload_handler = ReloadHandler() + self.loop = ServerLoop( + create_http_handler(websocket_handler=self.reload_handler), + opts=Options(shutdown_timeout=0.1, listen_on='127.0.0.1', port=0)) + self.loop.LISTENING_MSG = None + self.notify_reload = self.reload_handler.notify_reload + self.ping = self.reload_handler.ping + self.start() + + def run(self): + try: + self.loop.serve_forever() + except KeyboardInterrupt: + pass + + def __enter__(self): + while not self.loop.ready and self.is_alive(): + time.sleep(0.01) + self.address = self.loop.bound_address[:2] + os.environ['CALIBRE_AUTORELOAD_PORT'] = str(self.address[1]) + return self + + def __exit__(self, *args): + self.loop.stop() + self.join(self.loop.opts.shutdown_timeout) +# }}} def auto_reload(log, dirs=frozenset(), cmd=None, add_default_dirs=True): if Watcher is None: @@ -247,8 +352,8 @@ def auto_reload(log, dirs=frozenset(), cmd=None, add_default_dirs=True): dirs = find_dirs_to_watch(fpath, dirs, add_default_dirs) log('Auto-restarting server on changes press Ctrl-C to quit') log('Watching %d directory trees for changes' % len(dirs)) - with Worker(cmd, log) as server: - w = Watcher(dirs, server, log) + with ReloadServer() as server, Worker(cmd, log, server) as worker: + w = Watcher(dirs, worker, log) try: w.loop() except KeyboardInterrupt: diff --git a/src/calibre/srv/code.py b/src/calibre/srv/code.py index 2af99df41f..1a39d2fc9d 100644 --- a/src/calibre/srv/code.py +++ b/src/calibre/srv/code.py @@ -4,9 +4,43 @@ from __future__ import (unicode_literals, division, absolute_import, print_function) +import re +from functools import partial +from threading import Lock from calibre.srv.routes import endpoint +html_cache = {} +cache_lock = Lock() +autoreload_js = None + +def get_html(name, auto_reload_port, **replacements): + global autoreload_js + key = (name, auto_reload_port, tuple(replacements.iteritems())) + with cache_lock: + try: + return html_cache[key] + except KeyError: + with lopen(P(name), 'rb') as f: + raw = f.read() + for k, val in key[-1]: + if isinstance(val, type('')): + val = val.encode('utf-8') + if isinstance(k, type('')): + k = k.encode('utf-8') + raw = raw.replace(k, val) + if auto_reload_port > 0: + if autoreload_js is None: + autoreload_js = P('content-server/autoreload.js', data=True, allow_user_override=False).replace( + b'AUTORELOAD_PORT', bytes(str(auto_reload_port))) + raw = re.sub( + br'(<\s*/\s*head\s*>)', br'\1' % autoreload_js, + raw, flags=re.IGNORECASE) + html_cache[key] = raw + return raw + @endpoint('', auth_required=False) def index(ctx, rd): - return lopen(P('content-server/index.html'), 'rb') + return rd.generate_static_output('/', partial( + get_html, 'content-server/index.html', getattr(rd.opts, 'auto_reload_port', 0), + ENTRY_POINT='book list', LOADING_MSG=_('Loading library, please wait'))) diff --git a/src/calibre/srv/http_response.py b/src/calibre/srv/http_response.py index 926e483ef3..4bc0f4cec0 100644 --- a/src/calibre/srv/http_response.py +++ b/src/calibre/srv/http_response.py @@ -211,10 +211,13 @@ class RequestData(object): # {{{ self.tdir = tdir self.allowed_book_ids = {} - def generate_static_output(self, name, generator): + def generate_static_output(self, name, generator, content_type='text/html; charset=UTF-8'): ans = self.static_cache.get(name) if ans is None: ans = self.static_cache[name] = StaticOutput(generator()) + ct = self.outheaders.get('Content-Type') + if not ct: + self.outheaders.set('Content-Type', content_type, replace_all=True) return ans def filesystem_file_with_custom_etag(self, output, *etag_parts): diff --git a/src/calibre/srv/loop.py b/src/calibre/srv/loop.py index 9be9f7b6c3..93975fd8a1 100644 --- a/src/calibre/srv/loop.py +++ b/src/calibre/srv/loop.py @@ -272,6 +272,8 @@ class Connection(object): # {{{ class ServerLoop(object): + LISTENING_MSG = 'calibre server listening on' + def __init__( self, handler, @@ -383,7 +385,8 @@ class ServerLoop(object): with TemporaryDirectory(prefix='srv-') as tdir: self.tdir = tdir self.ready = True - self.log('calibre server listening on', ba) + if self.LISTENING_MSG: + self.log(self.LISTENING_MSG, ba) self.plugin_pool.start() while self.ready: diff --git a/src/calibre/srv/standalone.py b/src/calibre/srv/standalone.py index d19ed0a12e..3b2ea7c87b 100644 --- a/src/calibre/srv/standalone.py +++ b/src/calibre/srv/standalone.py @@ -121,6 +121,7 @@ def main(args=sys.argv): return auto_reload(default_log) except NoAutoReload as e: raise SystemExit(e.message) + opts.auto_reload_port = int(os.environ.get('CALIBRE_AUTORELOAD_PORT', 0)) try: server = Server(libraries, opts) except InvalidCredentials as e: diff --git a/src/pyj/srv.pyj b/src/pyj/srv.pyj index dec82912c5..d81764d019 100644 --- a/src/pyj/srv.pyj +++ b/src/pyj/srv.pyj @@ -23,10 +23,11 @@ def on_library_load_progress(loaded, total): p.max = total p.value = loaded -def on_document_loaded(): - ajax(window.interface_data_url, on_library_loaded, on_library_load_progress).send() +def on_load(): + if window.calibre_entry_point == 'book list': + ajax('ajax/interface-data', on_library_loaded, on_library_load_progress).send() # We wait for all page elements to load, since this is a single page app # with a largely empty starting document, we can use this to preload any resources # we know are going to be needed immediately. -window.addEventListener("load", on_document_loaded) +window.addEventListener("load", on_load)