Merge from trunk

src/calibre/library/database2.py

@@ -456,6 +456,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 parent = os.path.dirname(spath)
                 if len(os.listdir(parent)) == 0:
                     self.rmtree(parent, permanent=True)
+
         curpath = self.library_path
         c1, c2 = current_path.split('/'), path.split('/')
         if not self.is_case_sensitive and len(c1) == len(c2):
@@ -470,22 +471,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             # handles files in the directories, so no need to do them here.
             for oldseg, newseg in zip(c1, c2):
                 if oldseg.lower() == newseg.lower() and oldseg != newseg:
-                    while True:
-                        # need a temp name in the current segment for renames
-                        tempname = os.path.join(curpath, 'TEMP.%f'%time.time())
-                        if not os.path.exists(tempname):
-                            break
                     try:
-                        os.rename(os.path.join(curpath, oldseg), tempname)
+                        os.rename(os.path.join(curpath, oldseg),
-                    except (IOError, OSError):
+                                os.path.join(curpath, newseg))
-                        # Windows (at least) sometimes refuses to do the rename
+                    except:
-                        # probably because a file such a cover is open in the
+                        break # Fail silently since nothing catastrophic has happened
-                        # hierarchy. Just go on -- nothing is hurt beyond the
-                        # case of the filesystem not matching the case in
-                        # name stored by calibre
-                        print 'rename of library component failed'
-                    else:
-                        os.rename(tempname, os.path.join(curpath, newseg))
                 curpath = os.path.join(curpath, newseg)
 
     def add_listener(self, listener):

src/calibre/manual/tutorials.rst

@@ -11,6 +11,7 @@ Here you will find tutorials to get you started using |app|'s more advanced feat
 .. toctree::
    :maxdepth: 1
 
+   news
    xpath
    template_lang
    portable

src/calibre/utils/ipc/launch.py

@@ -21,7 +21,7 @@ class Worker(object):
     Platform independent object for launching child processes. All processes
     have the environment variable :envvar:`CALIBRE_WORKER` set.
 
-    Useful attributes: ``is_alive``, ``returncode``
+    Useful attributes: ``is_alive``, ``returncode``, ``pid``
     Useful methods: ``kill``
 
     To launch child simply call the Worker object. By default, the child's
@@ -94,6 +94,11 @@ class Worker(object):
         self.child.poll()
         return self.child.returncode
 
+    @property
+    def pid(self):
+        if not hasattr(self, 'child'): return None
+        return getattr(self.child, 'pid', None)
+
     def kill(self):
         try:
             if self.is_alive:

src/calibre/utils/pyconsole/__init__.py

@@ -5,11 +5,19 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import sys
+import sys, os
 
-from calibre import prints as prints_
+from calibre import prints as prints_, preferred_encoding, isbytestring
-from calibre.utils.config import Config, ConfigProxy
+from calibre.utils.config import Config, ConfigProxy, JSONConfig
+from calibre.utils.ipc.launch import Worker
+from calibre.constants import __appname__, __version__, iswindows
+from calibre.gui2 import error_dialog
 
+# Time to wait for communication to/from the interpreter process
+POLL_TIMEOUT = 0.01 # seconds
+
+preferred_encoding, isbytestring, __appname__, __version__, error_dialog, \
+iswindows
 
 def console_config():
     desc='Settings to control the calibre console'
@@ -20,10 +28,18 @@ def console_config():
     return c
 
 prefs = ConfigProxy(console_config())
-
+dynamic = JSONConfig('console')
 
 def prints(*args, **kwargs):
     kwargs['file'] = sys.__stdout__
     prints_(*args, **kwargs)
 
+class Process(Worker):
+
+    @property
+    def env(self):
+        env = dict(os.environ)
+        env.update(self._env)
+        return env
+
 

src/calibre/utils/pyconsole/console.py

@@ -9,16 +9,15 @@ import sys, textwrap, traceback, StringIO
 from functools import partial
 
 from PyQt4.Qt import QTextEdit, Qt, QTextFrameFormat, pyqtSignal, \
-    QApplication, QColor, QPalette, QMenu, QActionGroup
+    QApplication, QColor, QPalette, QMenu, QActionGroup, QTimer
 
 from pygments.lexers import PythonLexer, PythonTracebackLexer
 from pygments.styles import get_all_styles
 
-from calibre.constants import __appname__, __version__
 from calibre.utils.pyconsole.formatter import Formatter
-from calibre.utils.pyconsole.repl import Interpreter, DummyFile
+from calibre.utils.pyconsole.controller import Controller
-from calibre.utils.pyconsole import prints, prefs
+from calibre.utils.pyconsole import prints, prefs, __appname__, \
-from calibre.gui2 import error_dialog
+        __version__, error_dialog
 
 class EditBlock(object): # {{{
 
@@ -114,7 +113,8 @@ class Console(QTextEdit):
             continuation='... ',
             parent=None):
         QTextEdit.__init__(self, parent)
-        self.buf = []
+        self.shutting_down = False
+        self.buf = self.old_buf = []
         self.prompt_frame = None
         self.allow_output = False
         self.prompt_frame_format = QTextFrameFormat()
@@ -153,20 +153,80 @@ class Console(QTextEdit):
         '''.format(sys.version.splitlines()[0], __appname__,
             __version__))
 
+        self.controllers = []
+        QTimer.singleShot(0, self.launch_controller)
+
+        sys.excepthook = self.unhandled_exception
+
         with EditBlock(self.cursor):
             self.render_block(motd)
 
-        sys.stdout = sys.stderr = DummyFile(parent=self)
+    def shutdown(self):
-        sys.stdout.write_output.connect(self.show_output)
+        self.shutton_down = True
-        self.interpreter = Interpreter(parent=self)
+        for c in self.controllers:
-        self.interpreter.show_error.connect(self.show_error)
+            c.kill()
-
-        sys.excepthook = self.unhandled_exception
 
     def contextMenuEvent(self, event):
         self.context_menu.popup(event.globalPos())
         event.accept()
 
+    # Controller management {{{
+    @property
+    def controller(self):
+        return self.controllers[-1]
+
+    def no_controller_error(self):
+        error_dialog(self, _('No interpreter'),
+                _('No active interpreter found. Try restarting the'
+                    ' console'), show=True)
+
+    def launch_controller(self, *args):
+        c = Controller(self)
+        c.write_output.connect(self.show_output, type=Qt.QueuedConnection)
+        c.show_error.connect(self.show_error, type=Qt.QueuedConnection)
+        c.interpreter_died.connect(self.interpreter_died,
+                type=Qt.QueuedConnection)
+        c.interpreter_done.connect(self.execution_done)
+        self.controllers.append(c)
+
+    def interpreter_died(self, controller, returncode):
+        if not self.shutting_down and controller.current_command is not None:
+            error_dialog(self, _('Interpreter died'),
+                    _('Interpreter dies while excuting a command. To see '
+                        'the command, click Show details'),
+                    det_msg=controller.current_command, show=True)
+
+    def execute(self, prompt_lines):
+        c = self.root_frame.lastCursorPosition()
+        self.setTextCursor(c)
+        self.old_prompt_frame = self.prompt_frame
+        self.prompt_frame = None
+        self.old_buf = self.buf
+        self.buf = []
+        self.running.emit()
+        self.controller.runsource('\n'.join(prompt_lines))
+
+    def execution_done(self, controller, ret):
+        if controller is self.controller:
+            self.running_done.emit()
+            if ret: # Incomplete command
+                self.buf = self.old_buf
+                self.prompt_frame = self.old_prompt_frame
+                c = self.prompt_frame.lastCursorPosition()
+                c.insertBlock()
+                self.setTextCursor(c)
+            else: # Command completed
+                try:
+                    self.old_prompt_frame.setFrameFormat(QTextFrameFormat())
+                except RuntimeError:
+                    # Happens if enough lines of output that the old
+                    # frame was deleted
+                    pass
+
+            self.render_current_prompt()
+
+    # }}}
+
     # Prompt management {{{
 
     @dynamic_property
@@ -208,6 +268,11 @@ class Console(QTextEdit):
 
         return property(fget=fget, fset=fset, doc=doc)
 
+    def move_cursor_to_prompt(self):
+        if self.prompt_frame is not None and self.cursor_pos[0] < 0:
+            c = self.prompt_frame.lastCursorPosition()
+            self.setTextCursor(c)
+
     def prompt(self, strip_prompt_strings=True):
         if not self.prompt_frame:
             yield u'' if strip_prompt_strings else self.formatter.prompt
@@ -265,7 +330,7 @@ class Console(QTextEdit):
         if restore_prompt:
             self.render_current_prompt()
 
-    def show_error(self, is_syntax_err, tb):
+    def show_error(self, is_syntax_err, tb, controller=None):
         if self.prompt_frame is not None:
             # At a prompt, so redirect output
             return prints(tb, end='')
@@ -280,7 +345,7 @@ class Console(QTextEdit):
         self.ensureCursorVisible()
         QApplication.processEvents()
 
-    def show_output(self, raw):
+    def show_output(self, raw, which='stdout', controller=None):
         def do_show():
             try:
                 self.buf.append(raw)
@@ -385,39 +450,15 @@ class Console(QTextEdit):
     def enter_pressed(self):
         if self.prompt_frame is None:
             return
+        if not self.controller.is_alive:
+            return self.no_controller_error()
         cp = list(self.prompt())
         if cp[0]:
-            c = self.root_frame.lastCursorPosition()
+            self.execute(cp)
-            self.setTextCursor(c)
-            old_pf = self.prompt_frame
-            self.prompt_frame = None
-            oldbuf = self.buf
-            self.buf = []
-            self.running.emit()
-            try:
-                ret = self.interpreter.runsource('\n'.join(cp))
-            except SystemExit:
-                ret = False
-                self.show_output('Raising SystemExit not allowed\n')
-            self.running_done.emit()
-            if ret: # Incomplete command
-                self.buf = oldbuf
-                self.prompt_frame = old_pf
-                c = old_pf.lastCursorPosition()
-                c.insertBlock()
-                self.setTextCursor(c)
-            else: # Command completed
-                try:
-                    old_pf.setFrameFormat(QTextFrameFormat())
-                except RuntimeError:
-                    # Happens if enough lines of output that the old
-                    # frame was deleted
-                    pass
-
-            self.render_current_prompt()
 
     def text_typed(self, text):
         if self.prompt_frame is not None:
+            self.move_cursor_to_prompt()
             self.cursor.insertText(text)
             self.render_current_prompt(restore_cursor=True)
 

src/calibre/utils/pyconsole/controller.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import os, cPickle, signal, time
+from Queue import Queue, Empty
+from multiprocessing.connection import Listener, arbitrary_address
+from binascii import hexlify
+
+from PyQt4.Qt import QThread, pyqtSignal
+
+from calibre.utils.pyconsole import Process, iswindows, POLL_TIMEOUT
+
+class Controller(QThread):
+
+    # show_error(is_syntax_error, traceback, self)
+    show_error = pyqtSignal(object, object, object)
+    # write_output(unicode_object, stdout or stderr, self)
+    write_output = pyqtSignal(object, object, object)
+    # Indicates interpreter has finished evaluating current command
+    interpreter_done = pyqtSignal(object, object)
+    # interpreter_died(self, returncode or None if no return code available)
+    interpreter_died = pyqtSignal(object, object)
+
+    def __init__(self, parent):
+        QThread.__init__(self, parent)
+        self.keep_going = True
+        self.current_command = None
+
+        self.out_queue = Queue()
+        self.address = arbitrary_address('AF_PIPE' if iswindows else 'AF_UNIX')
+        self.auth_key = os.urandom(32)
+        if iswindows and self.address[1] == ':':
+            self.address = self.address[2:]
+        self.listener = Listener(address=self.address,
+                authkey=self.auth_key, backlog=4)
+
+        self.env = {
+            'CALIBRE_LAUNCH_INTERPRETER': '1',
+            'CALIBRE_WORKER_ADDRESS':
+                    hexlify(cPickle.dumps(self.listener.address, -1)),
+            'CALIBRE_WORKER_KEY': hexlify(self.auth_key)
+        }
+        self.process = Process(self.env)
+        self.output_file_buf = self.process(redirect_output=False)
+        self.conn = self.listener.accept()
+        self.start()
+
+    def run(self):
+        while self.keep_going and self.is_alive:
+            try:
+                self.communicate()
+            except KeyboardInterrupt:
+                pass
+            except EOFError:
+                break
+        self.interpreter_died.emit(self, self.returncode)
+        try:
+            self.listener.close()
+        except:
+            pass
+
+    def communicate(self):
+        if self.conn.poll(POLL_TIMEOUT):
+            self.dispatch_incoming_message(self.conn.recv())
+        try:
+            obj = self.out_queue.get_nowait()
+        except Empty:
+            pass
+        else:
+            try:
+                self.conn.send(obj)
+            except:
+                raise EOFError('controller failed to send')
+
+    def dispatch_incoming_message(self, obj):
+        try:
+            cmd, data = obj
+        except:
+            print 'Controller received invalid message'
+            print repr(obj)
+            return
+        if cmd in ('stdout', 'stderr'):
+            self.write_output.emit(data, cmd, self)
+        elif cmd == 'syntaxerror':
+            self.show_error.emit(True, data, self)
+        elif cmd == 'traceback':
+            self.show_error.emit(False, data, self)
+        elif cmd == 'done':
+            self.current_command = None
+            self.interpreter_done.emit(self, data)
+
+    def runsource(self, cmd):
+        self.current_command = cmd
+        self.out_queue.put(('run', cmd))
+
+    def __nonzero__(self):
+        return self.process.is_alive
+
+    @property
+    def returncode(self):
+        return self.process.returncode
+
+    @property
+    def interrupt(self):
+        if hasattr(signal, 'SIGINT'):
+            os.kill(self.process.pid, signal.SIGINT)
+        elif hasattr(signal, 'CTRL_C_EVENT'):
+            os.kill(self.process.pid, signal.CTRL_C_EVENT)
+
+    @property
+    def is_alive(self):
+        return self.process.is_alive
+
+    def kill(self):
+        self.out_queue.put(('quit', 0))
+        t = 0
+        while self.is_alive and t < 10:
+            time.sleep(0.1)
+        self.process.kill()
+        self.keep_going = False
+

src/calibre/utils/pyconsole/interpreter.py

@@ -0,0 +1,177 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import sys, cPickle, os
+from code import InteractiveInterpreter
+from Queue import Queue, Empty
+from threading import Thread
+from binascii import unhexlify
+from multiprocessing.connection import Client
+
+from calibre.utils.pyconsole import preferred_encoding, isbytestring, \
+        POLL_TIMEOUT
+
+'''
+Messages sent by client:
+
+    (stdout, unicode)
+    (stderr, unicode)
+    (syntaxerror, unicode)
+    (traceback, unicode)
+    (done, True iff incomplete command)
+
+Messages that can be received by client:
+    (quit, return code)
+    (run, unicode)
+
+'''
+
+def tounicode(raw): # {{{
+    if isbytestring(raw):
+        try:
+            raw = raw.decode(preferred_encoding, 'replace')
+        except:
+            raw = repr(raw)
+
+        if isbytestring(raw):
+            try:
+                raw.decode('utf-8', 'replace')
+            except:
+                raw = u'Undecodable bytestring'
+        return raw
+# }}}
+
+class DummyFile(object): # {{{
+
+    def __init__(self, what, out_queue):
+        self.closed = False
+        self.name = 'console'
+        self.softspace = 0
+        self.what = what
+        self.out_queue = out_queue
+
+    def flush(self):
+        pass
+
+    def close(self):
+        pass
+
+    def write(self, raw):
+        self.out_queue.put((self.what, tounicode(raw)))
+# }}}
+
+class Comm(Thread): # {{{
+
+    def __init__(self, conn, out_queue, in_queue):
+        Thread.__init__(self)
+        self.daemon = True
+        self.conn = conn
+        self.out_queue = out_queue
+        self.in_queue = in_queue
+        self.keep_going = True
+
+    def run(self):
+        while self.keep_going:
+            try:
+                self.communicate()
+            except KeyboardInterrupt:
+                pass
+            except EOFError:
+                pass
+
+    def communicate(self):
+        if self.conn.poll(POLL_TIMEOUT):
+            try:
+                obj = self.conn.recv()
+            except:
+                pass
+            else:
+                self.in_queue.put(obj)
+        try:
+            obj = self.out_queue.get_nowait()
+        except Empty:
+            pass
+        else:
+            try:
+                self.conn.send(obj)
+            except:
+                raise EOFError('interpreter failed to send')
+# }}}
+
+class Interpreter(InteractiveInterpreter): # {{{
+
+    def __init__(self, queue, local={}):
+        if '__name__' not in local:
+            local['__name__'] = '__console__'
+        if '__doc__' not in local:
+            local['__doc__'] = None
+        self.out_queue = queue
+        sys.stdout = DummyFile('stdout', queue)
+        sys.stderr = DummyFile('sdterr', queue)
+        InteractiveInterpreter.__init__(self, locals=local)
+
+    def showtraceback(self, *args, **kwargs):
+        self.is_syntax_error = False
+        InteractiveInterpreter.showtraceback(self, *args, **kwargs)
+
+    def showsyntaxerror(self, *args, **kwargs):
+        self.is_syntax_error = True
+        InteractiveInterpreter.showsyntaxerror(self, *args, **kwargs)
+
+    def write(self, raw):
+        what = 'syntaxerror' if self.is_syntax_error else 'traceback'
+        self.out_queue.put((what, tounicode(raw)))
+
+# }}}
+
+def connect():
+    os.chdir(os.environ['ORIGWD'])
+    address = cPickle.loads(unhexlify(os.environ['CALIBRE_WORKER_ADDRESS']))
+    key     = unhexlify(os.environ['CALIBRE_WORKER_KEY'])
+    return Client(address, authkey=key)
+
+def main():
+    out_queue = Queue()
+    in_queue = Queue()
+    conn = connect()
+    comm = Comm(conn, out_queue, in_queue)
+    comm.start()
+    interpreter = Interpreter(out_queue)
+
+    ret = 0
+
+    while True:
+        try:
+            try:
+                cmd, data = in_queue.get(1)
+            except Empty:
+                pass
+            else:
+                if cmd == 'quit':
+                    ret = data
+                    comm.keep_going = False
+                    comm.join()
+                    break
+                elif cmd == 'run':
+                    if not comm.is_alive():
+                        ret = 1
+                        break
+                    ret = False
+                    try:
+                        ret = interpreter.runsource(data)
+                    except KeyboardInterrupt:
+                        pass
+                    except SystemExit:
+                        out_queue.put(('stderr', 'SystemExit ignored\n'))
+                    out_queue.put(('done', ret))
+        except KeyboardInterrupt:
+            pass
+
+    return ret
+
+if __name__ == '__main__':
+    main()

src/calibre/utils/pyconsole/main.py

@@ -11,7 +11,7 @@ from functools import partial
 from PyQt4.Qt import QDialog, QToolBar, QStatusBar, QLabel, QFont, Qt, \
     QApplication, QIcon, QVBoxLayout, QAction
 
-from calibre.constants import __appname__, __version__
+from calibre.utils.pyconsole import dynamic, __appname__, __version__
 from calibre.utils.pyconsole.console import Console
 
 class MainWindow(QDialog):
@@ -26,6 +26,9 @@ class MainWindow(QDialog):
         self.setLayout(self.l)
 
         self.resize(800, 600)
+        geom = dynamic.get('console_window_geometry', None)
+        if geom is not None:
+            self.restoreGeometry(geom)
 
         # Setup tool bar {{{
         self.tool_bar = QToolBar(self)
@@ -62,17 +65,26 @@ class MainWindow(QDialog):
         self.restart_requested = True
         self.reject()
 
-def main():
+    def closeEvent(self, *args):
-    QApplication.setApplicationName(__appname__+' console')
+        dynamic.set('console_window_geometry',
-    QApplication.setOrganizationName('Kovid Goyal')
+                bytearray(self.saveGeometry()))
-    app = QApplication([])
+        self.console.shutdown()
-    app
+        return QDialog.closeEvent(self, *args)
+
+
+def show():
     while True:
         m = MainWindow()
         m.exec_()
         if not m.restart_requested:
             break
 
+def main():
+    QApplication.setApplicationName(__appname__+' console')
+    QApplication.setOrganizationName('Kovid Goyal')
+    app = QApplication([])
+    app
+    show()
 
 if __name__ == '__main__':
     main()

src/calibre/utils/pyconsole/repl.py

@@ -1,67 +0,0 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
-
-__license__   = 'GPL v3'
-__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
-__docformat__ = 'restructuredtext en'
-
-from code import InteractiveInterpreter
-
-from PyQt4.Qt import QObject, pyqtSignal
-
-from calibre import isbytestring
-from calibre.constants import preferred_encoding
-
-class Interpreter(QObject, InteractiveInterpreter):
-
-    # show_error(is_syntax_error, traceback)
-    show_error = pyqtSignal(object, object)
-
-    def __init__(self, local={}, parent=None):
-        QObject.__init__(self, parent)
-        if '__name__' not in local:
-            local['__name__'] = '__console__'
-        if '__doc__' not in local:
-            local['__doc__'] = None
-        InteractiveInterpreter.__init__(self, locals=local)
-
-    def showtraceback(self, *args, **kwargs):
-        self.is_syntax_error = False
-        InteractiveInterpreter.showtraceback(self, *args, **kwargs)
-
-    def showsyntaxerror(self, *args, **kwargs):
-        self.is_syntax_error = True
-        InteractiveInterpreter.showsyntaxerror(self, *args, **kwargs)
-
-    def write(self, tb):
-        self.show_error.emit(self.is_syntax_error, tb)
-
-class DummyFile(QObject):
-
-    # write_output(unicode_object)
-    write_output = pyqtSignal(object)
-
-    def __init__(self, parent=None):
-        QObject.__init__(self, parent)
-        self.closed = False
-        self.name = 'console'
-        self.softspace = 0
-
-    def flush(self):
-        pass
-
-    def close(self):
-        pass
-
-    def write(self, raw):
-        #import sys, traceback
-        #print >> sys.__stdout__, 'file,write stack:\n', ''.join(traceback.format_stack())
-        if isbytestring(raw):
-            try:
-                raw = raw.decode(preferred_encoding, 'replace')
-            except:
-                raw = repr(raw)
-                if isbytestring(raw):
-                    raw = raw.decode(preferred_encoding, 'replace')
-        self.write_output.emit(raw)
-