Refactor console to run interpreter in separate process

src/calibre/utils/pyconsole/__init__.py

@@ -13,6 +13,9 @@ from calibre.utils.ipc.launch import Worker
 from calibre.constants import __appname__, __version__, iswindows
 from calibre.gui2 import error_dialog
 
+# Time to wait for communication to/from the interpreter process
+POLL_TIMEOUT = 0.01 # seconds
+
 preferred_encoding, isbytestring, __appname__, __version__, error_dialog, \
 iswindows
 

src/calibre/utils/pyconsole/console.py

@@ -9,13 +9,13 @@ import sys, textwrap, traceback, StringIO
 from functools import partial
 
 from PyQt4.Qt import QTextEdit, Qt, QTextFrameFormat, pyqtSignal, \
-    QApplication, QColor, QPalette, QMenu, QActionGroup
+    QApplication, QColor, QPalette, QMenu, QActionGroup, QTimer
 
 from pygments.lexers import PythonLexer, PythonTracebackLexer
 from pygments.styles import get_all_styles
 
 from calibre.utils.pyconsole.formatter import Formatter
-from calibre.utils.pyconsole.repl import Interpreter, DummyFile
+from calibre.utils.pyconsole.controller import Controller
 from calibre.utils.pyconsole import prints, prefs, __appname__, \
         __version__, error_dialog
 
@@ -113,7 +113,8 @@ class Console(QTextEdit):
             continuation='... ',
             parent=None):
         QTextEdit.__init__(self, parent)
-        self.buf = []
+        self.shutting_down = False
+        self.buf = self.old_buf = []
         self.prompt_frame = None
         self.allow_output = False
         self.prompt_frame_format = QTextFrameFormat()
@@ -152,20 +153,80 @@ class Console(QTextEdit):
         '''.format(sys.version.splitlines()[0], __appname__,
             __version__))
 
+        self.controllers = []
+        QTimer.singleShot(0, self.launch_controller)
+
+        sys.excepthook = self.unhandled_exception
+
         with EditBlock(self.cursor):
             self.render_block(motd)
 
-        sys.stdout = sys.stderr = DummyFile(parent=self)
-        sys.stdout.write_output.connect(self.show_output)
-        self.interpreter = Interpreter(parent=self)
-        self.interpreter.show_error.connect(self.show_error)
-
-        sys.excepthook = self.unhandled_exception
+    def shutdown(self):
+        self.shutton_down = True
+        for c in self.controllers:
+            c.kill()
 
     def contextMenuEvent(self, event):
         self.context_menu.popup(event.globalPos())
         event.accept()
 
+    # Controller management {{{
+    @property
+    def controller(self):
+        return self.controllers[-1]
+
+    def no_controller_error(self):
+        error_dialog(self, _('No interpreter'),
+                _('No active interpreter found. Try restarting the'
+                    ' console'), show=True)
+
+    def launch_controller(self, *args):
+        c = Controller(self)
+        c.write_output.connect(self.show_output, type=Qt.QueuedConnection)
+        c.show_error.connect(self.show_error, type=Qt.QueuedConnection)
+        c.interpreter_died.connect(self.interpreter_died,
+                type=Qt.QueuedConnection)
+        c.interpreter_done.connect(self.execution_done)
+        self.controllers.append(c)
+
+    def interpreter_died(self, controller, returncode):
+        if not self.shutting_down and controller.current_command is not None:
+            error_dialog(self, _('Interpreter died'),
+                    _('Interpreter dies while excuting a command. To see '
+                        'the command, click Show details'),
+                    det_msg=controller.current_command, show=True)
+
+    def execute(self, prompt_lines):
+        c = self.root_frame.lastCursorPosition()
+        self.setTextCursor(c)
+        self.old_prompt_frame = self.prompt_frame
+        self.prompt_frame = None
+        self.old_buf = self.buf
+        self.buf = []
+        self.running.emit()
+        self.controller.runsource('\n'.join(prompt_lines))
+
+    def execution_done(self, controller, ret):
+        if controller is self.controller:
+            self.running_done.emit()
+            if ret: # Incomplete command
+                self.buf = self.old_buf
+                self.prompt_frame = self.old_prompt_frame
+                c = self.prompt_frame.lastCursorPosition()
+                c.insertBlock()
+                self.setTextCursor(c)
+            else: # Command completed
+                try:
+                    self.old_prompt_frame.setFrameFormat(QTextFrameFormat())
+                except RuntimeError:
+                    # Happens if enough lines of output that the old
+                    # frame was deleted
+                    pass
+
+            self.render_current_prompt()
+
+    # }}}
+
     # Prompt management {{{
 
     @dynamic_property
@@ -264,7 +325,7 @@ class Console(QTextEdit):
         if restore_prompt:
             self.render_current_prompt()
 
-    def show_error(self, is_syntax_err, tb):
+    def show_error(self, is_syntax_err, tb, controller=None):
         if self.prompt_frame is not None:
             # At a prompt, so redirect output
             return prints(tb, end='')
@@ -279,7 +340,7 @@ class Console(QTextEdit):
         self.ensureCursorVisible()
         QApplication.processEvents()
 
-    def show_output(self, raw):
+    def show_output(self, raw, which='stdout', controller=None):
         def do_show():
             try:
                 self.buf.append(raw)
@@ -384,36 +445,11 @@ class Console(QTextEdit):
     def enter_pressed(self):
         if self.prompt_frame is None:
             return
+        if not self.controller.is_alive:
+            return self.no_controller_error()
         cp = list(self.prompt())
         if cp[0]:
-            c = self.root_frame.lastCursorPosition()
-            self.setTextCursor(c)
-            old_pf = self.prompt_frame
-            self.prompt_frame = None
-            oldbuf = self.buf
-            self.buf = []
-            self.running.emit()
-            try:
-                ret = self.interpreter.runsource('\n'.join(cp))
-            except SystemExit:
-                ret = False
-                self.show_output('Raising SystemExit not allowed\n')
-            self.running_done.emit()
-            if ret: # Incomplete command
-                self.buf = oldbuf
-                self.prompt_frame = old_pf
-                c = old_pf.lastCursorPosition()
-                c.insertBlock()
-                self.setTextCursor(c)
-            else: # Command completed
-                try:
-                    old_pf.setFrameFormat(QTextFrameFormat())
-                except RuntimeError:
-                    # Happens if enough lines of output that the old
-                    # frame was deleted
-                    pass
-
-            self.render_current_prompt()
+            self.execute(cp)
 
     def text_typed(self, text):
         if self.prompt_frame is not None:

src/calibre/utils/pyconsole/controller.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import os, cPickle, signal, time
+from Queue import Queue, Empty
+from multiprocessing.connection import Listener, arbitrary_address
+from binascii import hexlify
+
+from PyQt4.Qt import QThread, pyqtSignal
+
+from calibre.utils.pyconsole import Process, iswindows, POLL_TIMEOUT
+
+class Controller(QThread):
+
+    # show_error(is_syntax_error, traceback, self)
+    show_error = pyqtSignal(object, object, object)
+    # write_output(unicode_object, stdout or stderr, self)
+    write_output = pyqtSignal(object, object, object)
+    # Indicates interpreter has finished evaluating current command
+    interpreter_done = pyqtSignal(object, object)
+    # interpreter_died(self, returncode or None if no return code available)
+    interpreter_died = pyqtSignal(object, object)
+
+    def __init__(self, parent):
+        QThread.__init__(self, parent)
+        self.keep_going = True
+        self.current_command = None
+
+        self.out_queue = Queue()
+        self.address = arbitrary_address('AF_PIPE' if iswindows else 'AF_UNIX')
+        self.auth_key = os.urandom(32)
+        if iswindows and self.address[1] == ':':
+            self.address = self.address[2:]
+        self.listener = Listener(address=self.address,
+                authkey=self.auth_key, backlog=4)
+
+        self.env = {
+            'CALIBRE_LAUNCH_INTERPRETER': '1',
+            'CALIBRE_WORKER_ADDRESS':
+                    hexlify(cPickle.dumps(self.listener.address, -1)),
+            'CALIBRE_WORKER_KEY': hexlify(self.auth_key)
+        }
+        self.process = Process(self.env)
+        self.output_file_buf = self.process(redirect_output=False)
+        self.conn = self.listener.accept()
+        self.start()
+
+    def run(self):
+        while self.keep_going and self.is_alive:
+            try:
+                self.communicate()
+            except KeyboardInterrupt:
+                pass
+            except EOFError:
+                break
+        self.interpreter_died.emit(self, self.returncode)
+        try:
+            self.listener.close()
+        except:
+            pass
+
+    def communicate(self):
+        if self.conn.poll(POLL_TIMEOUT):
+            self.dispatch_incoming_message(self.conn.recv())
+        try:
+            obj = self.out_queue.get_nowait()
+        except Empty:
+            pass
+        else:
+            try:
+                self.conn.send(obj)
+            except:
+                raise EOFError('controller failed to send')
+
+    def dispatch_incoming_message(self, obj):
+        try:
+            cmd, data = obj
+        except:
+            print 'Controller received invalid message'
+            print repr(obj)
+            return
+        if cmd in ('stdout', 'stderr'):
+            self.write_output.emit(data, cmd, self)
+        elif cmd == 'syntaxerror':
+            self.show_error.emit(True, data, self)
+        elif cmd == 'traceback':
+            self.show_error(self, False, data)
+        elif cmd == 'done':
+            self.current_command = None
+            self.interpreter_done.emit(self, data)
+
+    def runsource(self, cmd):
+        self.current_command = cmd
+        self.out_queue.put(('run', cmd))
+
+    def __nonzero__(self):
+        return self.process.is_alive
+
+    @property
+    def returncode(self):
+        return self.process.returncode
+
+    @property
+    def interrupt(self):
+        if hasattr(signal, 'SIGINT'):
+            os.kill(self.process.pid, signal.SIGINT)
+        elif hasattr(signal, 'CTRL_C_EVENT'):
+            os.kill(self.process.pid, signal.CTRL_C_EVENT)
+
+    @property
+    def is_alive(self):
+        return self.process.is_alive
+
+    def kill(self):
+        self.out_queue.put(('quit', 0))
+        t = 0
+        while self.is_alive and t < 10:
+            time.sleep(0.1)
+        self.process.kill()
+        self.keep_going = False
+

src/calibre/utils/pyconsole/interpreter.py

@@ -0,0 +1,177 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import sys, cPickle, os
+from code import InteractiveInterpreter
+from Queue import Queue, Empty
+from threading import Thread
+from binascii import unhexlify
+from multiprocessing.connection import Client
+
+from calibre.utils.pyconsole import preferred_encoding, isbytestring, \
+        POLL_TIMEOUT
+
+'''
+Messages sent by client:
+
+    (stdout, unicode)
+    (stderr, unicode)
+    (syntaxerror, unicode)
+    (traceback, unicode)
+    (done, True iff incomplete command)
+
+Messages that can be received by client:
+    (quit, return code)
+    (run, unicode)
+
+'''
+
+def tounicode(raw): # {{{
+    if isbytestring(raw):
+        try:
+            raw = raw.decode(preferred_encoding, 'replace')
+        except:
+            raw = repr(raw)
+
+        if isbytestring(raw):
+            try:
+                raw.decode('utf-8', 'replace')
+            except:
+                raw = u'Undecodable bytestring'
+        return raw
+# }}}
+
+class DummyFile(object): # {{{
+
+    def __init__(self, what, out_queue):
+        self.closed = False
+        self.name = 'console'
+        self.softspace = 0
+        self.what = what
+        self.out_queue = out_queue
+
+    def flush(self):
+        pass
+
+    def close(self):
+        pass
+
+    def write(self, raw):
+        self.out_queue.put((self.what, tounicode(raw)))
+# }}}
+
+class Comm(Thread): # {{{
+
+    def __init__(self, conn, out_queue, in_queue):
+        Thread.__init__(self)
+        self.daemon = True
+        self.conn = conn
+        self.out_queue = out_queue
+        self.in_queue = in_queue
+        self.keep_going = True
+
+    def run(self):
+        while self.keep_going:
+            try:
+                self.communicate()
+            except KeyboardInterrupt:
+                pass
+            except EOFError:
+                pass
+
+    def communicate(self):
+        if self.conn.poll(POLL_TIMEOUT):
+            try:
+                obj = self.conn.recv()
+            except:
+                pass
+            else:
+                self.in_queue.put(obj)
+        try:
+            obj = self.out_queue.get_nowait()
+        except Empty:
+            pass
+        else:
+            try:
+                self.conn.send(obj)
+            except:
+                raise EOFError('interpreter failed to send')
+# }}}
+
+class Interpreter(InteractiveInterpreter): # {{{
+
+    def __init__(self, queue, local={}):
+        if '__name__' not in local:
+            local['__name__'] = '__console__'
+        if '__doc__' not in local:
+            local['__doc__'] = None
+        self.out_queue = queue
+        sys.stdout = DummyFile('stdout', queue)
+        sys.stderr = DummyFile('sdterr', queue)
+        InteractiveInterpreter.__init__(self, locals=local)
+
+    def showtraceback(self, *args, **kwargs):
+        self.is_syntax_error = False
+        InteractiveInterpreter.showtraceback(self, *args, **kwargs)
+
+    def showsyntaxerror(self, *args, **kwargs):
+        self.is_syntax_error = True
+        InteractiveInterpreter.showsyntaxerror(self, *args, **kwargs)
+
+    def write(self, raw):
+        what = 'syntaxerror' if self.is_syntax_error else 'traceback'
+        self.out_queue.put((what, tounicode(raw)))
+
+# }}}
+
+def connect():
+    os.chdir(os.environ['ORIGWD'])
+    address = cPickle.loads(unhexlify(os.environ['CALIBRE_WORKER_ADDRESS']))
+    key     = unhexlify(os.environ['CALIBRE_WORKER_KEY'])
+    return Client(address, authkey=key)
+
+def main():
+    out_queue = Queue()
+    in_queue = Queue()
+    conn = connect()
+    comm = Comm(conn, out_queue, in_queue)
+    comm.start()
+    interpreter = Interpreter(out_queue)
+
+    ret = 0
+
+    while True:
+        try:
+            try:
+                cmd, data = in_queue.get(1)
+            except Empty:
+                pass
+            else:
+                if cmd == 'quit':
+                    ret = data
+                    comm.keep_going = False
+                    comm.join()
+                    break
+                elif cmd == 'run':
+                    if not comm.is_alive():
+                        ret = 1
+                        break
+                    ret = False
+                    try:
+                        ret = interpreter.runsource(data)
+                    except KeyboardInterrupt:
+                        pass
+                    except SystemExit:
+                        out_queue.put(('stderr', 'SystemExit ignored\n'))
+                    out_queue.put(('done', ret))
+        except KeyboardInterrupt:
+            pass
+
+    return ret
+
+if __name__ == '__main__':
+    main()

src/calibre/utils/pyconsole/main.py

@@ -68,6 +68,7 @@ class MainWindow(QDialog):
     def closeEvent(self, *args):
         dynamic.set('console_window_geometry',
                 bytearray(self.saveGeometry()))
+        self.console.shutdown()
         return QDialog.closeEvent(self, *args)
 
 

src/calibre/utils/pyconsole/repl.py

@@ -1,67 +0,0 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
-
-__license__   = 'GPL v3'
-__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
-__docformat__ = 'restructuredtext en'
-
-from code import InteractiveInterpreter
-
-from PyQt4.Qt import QObject, pyqtSignal
-
-from calibre import isbytestring
-from calibre.constants import preferred_encoding
-
-class Interpreter(QObject, InteractiveInterpreter):
-
-    # show_error(is_syntax_error, traceback)
-    show_error = pyqtSignal(object, object)
-
-    def __init__(self, local={}, parent=None):
-        QObject.__init__(self, parent)
-        if '__name__' not in local:
-            local['__name__'] = '__console__'
-        if '__doc__' not in local:
-            local['__doc__'] = None
-        InteractiveInterpreter.__init__(self, locals=local)
-
-    def showtraceback(self, *args, **kwargs):
-        self.is_syntax_error = False
-        InteractiveInterpreter.showtraceback(self, *args, **kwargs)
-
-    def showsyntaxerror(self, *args, **kwargs):
-        self.is_syntax_error = True
-        InteractiveInterpreter.showsyntaxerror(self, *args, **kwargs)
-
-    def write(self, tb):
-        self.show_error.emit(self.is_syntax_error, tb)
-
-class DummyFile(QObject):
-
-    # write_output(unicode_object)
-    write_output = pyqtSignal(object)
-
-    def __init__(self, parent=None):
-        QObject.__init__(self, parent)
-        self.closed = False
-        self.name = 'console'
-        self.softspace = 0
-
-    def flush(self):
-        pass
-
-    def close(self):
-        pass
-
-    def write(self, raw):
-        #import sys, traceback
-        #print >> sys.__stdout__, 'file,write stack:\n', ''.join(traceback.format_stack())
-        if isbytestring(raw):
-            try:
-                raw = raw.decode(preferred_encoding, 'replace')
-            except:
-                raw = repr(raw)
-                if isbytestring(raw):
-                    raw = raw.decode(preferred_encoding, 'replace')
-        self.write_output.emit(raw)
-