diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 14280d5209..fd5809f937 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -456,6 +456,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): parent = os.path.dirname(spath) if len(os.listdir(parent)) == 0: self.rmtree(parent, permanent=True) + curpath = self.library_path c1, c2 = current_path.split('/'), path.split('/') if not self.is_case_sensitive and len(c1) == len(c2): @@ -470,22 +471,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): # handles files in the directories, so no need to do them here. for oldseg, newseg in zip(c1, c2): if oldseg.lower() == newseg.lower() and oldseg != newseg: - while True: - # need a temp name in the current segment for renames - tempname = os.path.join(curpath, 'TEMP.%f'%time.time()) - if not os.path.exists(tempname): - break try: - os.rename(os.path.join(curpath, oldseg), tempname) - except (IOError, OSError): - # Windows (at least) sometimes refuses to do the rename - # probably because a file such a cover is open in the - # hierarchy. Just go on -- nothing is hurt beyond the - # case of the filesystem not matching the case in - # name stored by calibre - print 'rename of library component failed' - else: - os.rename(tempname, os.path.join(curpath, newseg)) + os.rename(os.path.join(curpath, oldseg), + os.path.join(curpath, newseg)) + except: + break # Fail silently since nothing catastrophic has happened curpath = os.path.join(curpath, newseg) def add_listener(self, listener): diff --git a/src/calibre/manual/tutorials.rst b/src/calibre/manual/tutorials.rst index d07316deb9..1e4cab8493 100644 --- a/src/calibre/manual/tutorials.rst +++ b/src/calibre/manual/tutorials.rst @@ -11,6 +11,7 @@ Here you will find tutorials to get you started using |app|'s more advanced feat .. toctree:: :maxdepth: 1 + news xpath template_lang portable diff --git a/src/calibre/utils/ipc/launch.py b/src/calibre/utils/ipc/launch.py index aa93469119..9c50208935 100644 --- a/src/calibre/utils/ipc/launch.py +++ b/src/calibre/utils/ipc/launch.py @@ -21,7 +21,7 @@ class Worker(object): Platform independent object for launching child processes. All processes have the environment variable :envvar:`CALIBRE_WORKER` set. - Useful attributes: ``is_alive``, ``returncode`` + Useful attributes: ``is_alive``, ``returncode``, ``pid`` Useful methods: ``kill`` To launch child simply call the Worker object. By default, the child's @@ -94,6 +94,11 @@ class Worker(object): self.child.poll() return self.child.returncode + @property + def pid(self): + if not hasattr(self, 'child'): return None + return getattr(self.child, 'pid', None) + def kill(self): try: if self.is_alive: diff --git a/src/calibre/utils/pyconsole/__init__.py b/src/calibre/utils/pyconsole/__init__.py index 32eb926143..3be9382413 100644 --- a/src/calibre/utils/pyconsole/__init__.py +++ b/src/calibre/utils/pyconsole/__init__.py @@ -5,11 +5,19 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import sys +import sys, os -from calibre import prints as prints_ -from calibre.utils.config import Config, ConfigProxy +from calibre import prints as prints_, preferred_encoding, isbytestring +from calibre.utils.config import Config, ConfigProxy, JSONConfig +from calibre.utils.ipc.launch import Worker +from calibre.constants import __appname__, __version__, iswindows +from calibre.gui2 import error_dialog +# Time to wait for communication to/from the interpreter process +POLL_TIMEOUT = 0.01 # seconds + +preferred_encoding, isbytestring, __appname__, __version__, error_dialog, \ +iswindows def console_config(): desc='Settings to control the calibre console' @@ -20,10 +28,18 @@ def console_config(): return c prefs = ConfigProxy(console_config()) - +dynamic = JSONConfig('console') def prints(*args, **kwargs): kwargs['file'] = sys.__stdout__ prints_(*args, **kwargs) +class Process(Worker): + + @property + def env(self): + env = dict(os.environ) + env.update(self._env) + return env + diff --git a/src/calibre/utils/pyconsole/console.py b/src/calibre/utils/pyconsole/console.py index 81169140cd..2611965345 100644 --- a/src/calibre/utils/pyconsole/console.py +++ b/src/calibre/utils/pyconsole/console.py @@ -9,16 +9,15 @@ import sys, textwrap, traceback, StringIO from functools import partial from PyQt4.Qt import QTextEdit, Qt, QTextFrameFormat, pyqtSignal, \ - QApplication, QColor, QPalette, QMenu, QActionGroup + QApplication, QColor, QPalette, QMenu, QActionGroup, QTimer from pygments.lexers import PythonLexer, PythonTracebackLexer from pygments.styles import get_all_styles -from calibre.constants import __appname__, __version__ from calibre.utils.pyconsole.formatter import Formatter -from calibre.utils.pyconsole.repl import Interpreter, DummyFile -from calibre.utils.pyconsole import prints, prefs -from calibre.gui2 import error_dialog +from calibre.utils.pyconsole.controller import Controller +from calibre.utils.pyconsole import prints, prefs, __appname__, \ + __version__, error_dialog class EditBlock(object): # {{{ @@ -114,7 +113,8 @@ class Console(QTextEdit): continuation='... ', parent=None): QTextEdit.__init__(self, parent) - self.buf = [] + self.shutting_down = False + self.buf = self.old_buf = [] self.prompt_frame = None self.allow_output = False self.prompt_frame_format = QTextFrameFormat() @@ -153,20 +153,80 @@ class Console(QTextEdit): '''.format(sys.version.splitlines()[0], __appname__, __version__)) + self.controllers = [] + QTimer.singleShot(0, self.launch_controller) + + sys.excepthook = self.unhandled_exception + with EditBlock(self.cursor): self.render_block(motd) - sys.stdout = sys.stderr = DummyFile(parent=self) - sys.stdout.write_output.connect(self.show_output) - self.interpreter = Interpreter(parent=self) - self.interpreter.show_error.connect(self.show_error) - - sys.excepthook = self.unhandled_exception + def shutdown(self): + self.shutton_down = True + for c in self.controllers: + c.kill() def contextMenuEvent(self, event): self.context_menu.popup(event.globalPos()) event.accept() + # Controller management {{{ + @property + def controller(self): + return self.controllers[-1] + + def no_controller_error(self): + error_dialog(self, _('No interpreter'), + _('No active interpreter found. Try restarting the' + ' console'), show=True) + + def launch_controller(self, *args): + c = Controller(self) + c.write_output.connect(self.show_output, type=Qt.QueuedConnection) + c.show_error.connect(self.show_error, type=Qt.QueuedConnection) + c.interpreter_died.connect(self.interpreter_died, + type=Qt.QueuedConnection) + c.interpreter_done.connect(self.execution_done) + self.controllers.append(c) + + def interpreter_died(self, controller, returncode): + if not self.shutting_down and controller.current_command is not None: + error_dialog(self, _('Interpreter died'), + _('Interpreter dies while excuting a command. To see ' + 'the command, click Show details'), + det_msg=controller.current_command, show=True) + + def execute(self, prompt_lines): + c = self.root_frame.lastCursorPosition() + self.setTextCursor(c) + self.old_prompt_frame = self.prompt_frame + self.prompt_frame = None + self.old_buf = self.buf + self.buf = [] + self.running.emit() + self.controller.runsource('\n'.join(prompt_lines)) + + def execution_done(self, controller, ret): + if controller is self.controller: + self.running_done.emit() + if ret: # Incomplete command + self.buf = self.old_buf + self.prompt_frame = self.old_prompt_frame + c = self.prompt_frame.lastCursorPosition() + c.insertBlock() + self.setTextCursor(c) + else: # Command completed + try: + self.old_prompt_frame.setFrameFormat(QTextFrameFormat()) + except RuntimeError: + # Happens if enough lines of output that the old + # frame was deleted + pass + + self.render_current_prompt() + + # }}} + # Prompt management {{{ @dynamic_property @@ -208,6 +268,11 @@ class Console(QTextEdit): return property(fget=fget, fset=fset, doc=doc) + def move_cursor_to_prompt(self): + if self.prompt_frame is not None and self.cursor_pos[0] < 0: + c = self.prompt_frame.lastCursorPosition() + self.setTextCursor(c) + def prompt(self, strip_prompt_strings=True): if not self.prompt_frame: yield u'' if strip_prompt_strings else self.formatter.prompt @@ -265,7 +330,7 @@ class Console(QTextEdit): if restore_prompt: self.render_current_prompt() - def show_error(self, is_syntax_err, tb): + def show_error(self, is_syntax_err, tb, controller=None): if self.prompt_frame is not None: # At a prompt, so redirect output return prints(tb, end='') @@ -280,7 +345,7 @@ class Console(QTextEdit): self.ensureCursorVisible() QApplication.processEvents() - def show_output(self, raw): + def show_output(self, raw, which='stdout', controller=None): def do_show(): try: self.buf.append(raw) @@ -385,39 +450,15 @@ class Console(QTextEdit): def enter_pressed(self): if self.prompt_frame is None: return + if not self.controller.is_alive: + return self.no_controller_error() cp = list(self.prompt()) if cp[0]: - c = self.root_frame.lastCursorPosition() - self.setTextCursor(c) - old_pf = self.prompt_frame - self.prompt_frame = None - oldbuf = self.buf - self.buf = [] - self.running.emit() - try: - ret = self.interpreter.runsource('\n'.join(cp)) - except SystemExit: - ret = False - self.show_output('Raising SystemExit not allowed\n') - self.running_done.emit() - if ret: # Incomplete command - self.buf = oldbuf - self.prompt_frame = old_pf - c = old_pf.lastCursorPosition() - c.insertBlock() - self.setTextCursor(c) - else: # Command completed - try: - old_pf.setFrameFormat(QTextFrameFormat()) - except RuntimeError: - # Happens if enough lines of output that the old - # frame was deleted - pass - - self.render_current_prompt() + self.execute(cp) def text_typed(self, text): if self.prompt_frame is not None: + self.move_cursor_to_prompt() self.cursor.insertText(text) self.render_current_prompt(restore_cursor=True) diff --git a/src/calibre/utils/pyconsole/controller.py b/src/calibre/utils/pyconsole/controller.py new file mode 100644 index 0000000000..368e665079 --- /dev/null +++ b/src/calibre/utils/pyconsole/controller.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai + +__license__ = 'GPL v3' +__copyright__ = '2010, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import os, cPickle, signal, time +from Queue import Queue, Empty +from multiprocessing.connection import Listener, arbitrary_address +from binascii import hexlify + +from PyQt4.Qt import QThread, pyqtSignal + +from calibre.utils.pyconsole import Process, iswindows, POLL_TIMEOUT + +class Controller(QThread): + + # show_error(is_syntax_error, traceback, self) + show_error = pyqtSignal(object, object, object) + # write_output(unicode_object, stdout or stderr, self) + write_output = pyqtSignal(object, object, object) + # Indicates interpreter has finished evaluating current command + interpreter_done = pyqtSignal(object, object) + # interpreter_died(self, returncode or None if no return code available) + interpreter_died = pyqtSignal(object, object) + + def __init__(self, parent): + QThread.__init__(self, parent) + self.keep_going = True + self.current_command = None + + self.out_queue = Queue() + self.address = arbitrary_address('AF_PIPE' if iswindows else 'AF_UNIX') + self.auth_key = os.urandom(32) + if iswindows and self.address[1] == ':': + self.address = self.address[2:] + self.listener = Listener(address=self.address, + authkey=self.auth_key, backlog=4) + + self.env = { + 'CALIBRE_LAUNCH_INTERPRETER': '1', + 'CALIBRE_WORKER_ADDRESS': + hexlify(cPickle.dumps(self.listener.address, -1)), + 'CALIBRE_WORKER_KEY': hexlify(self.auth_key) + } + self.process = Process(self.env) + self.output_file_buf = self.process(redirect_output=False) + self.conn = self.listener.accept() + self.start() + + def run(self): + while self.keep_going and self.is_alive: + try: + self.communicate() + except KeyboardInterrupt: + pass + except EOFError: + break + self.interpreter_died.emit(self, self.returncode) + try: + self.listener.close() + except: + pass + + def communicate(self): + if self.conn.poll(POLL_TIMEOUT): + self.dispatch_incoming_message(self.conn.recv()) + try: + obj = self.out_queue.get_nowait() + except Empty: + pass + else: + try: + self.conn.send(obj) + except: + raise EOFError('controller failed to send') + + def dispatch_incoming_message(self, obj): + try: + cmd, data = obj + except: + print 'Controller received invalid message' + print repr(obj) + return + if cmd in ('stdout', 'stderr'): + self.write_output.emit(data, cmd, self) + elif cmd == 'syntaxerror': + self.show_error.emit(True, data, self) + elif cmd == 'traceback': + self.show_error.emit(False, data, self) + elif cmd == 'done': + self.current_command = None + self.interpreter_done.emit(self, data) + + def runsource(self, cmd): + self.current_command = cmd + self.out_queue.put(('run', cmd)) + + def __nonzero__(self): + return self.process.is_alive + + @property + def returncode(self): + return self.process.returncode + + @property + def interrupt(self): + if hasattr(signal, 'SIGINT'): + os.kill(self.process.pid, signal.SIGINT) + elif hasattr(signal, 'CTRL_C_EVENT'): + os.kill(self.process.pid, signal.CTRL_C_EVENT) + + @property + def is_alive(self): + return self.process.is_alive + + def kill(self): + self.out_queue.put(('quit', 0)) + t = 0 + while self.is_alive and t < 10: + time.sleep(0.1) + self.process.kill() + self.keep_going = False + diff --git a/src/calibre/utils/pyconsole/interpreter.py b/src/calibre/utils/pyconsole/interpreter.py new file mode 100644 index 0000000000..6a1aff26c9 --- /dev/null +++ b/src/calibre/utils/pyconsole/interpreter.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai + +__license__ = 'GPL v3' +__copyright__ = '2010, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import sys, cPickle, os +from code import InteractiveInterpreter +from Queue import Queue, Empty +from threading import Thread +from binascii import unhexlify +from multiprocessing.connection import Client + +from calibre.utils.pyconsole import preferred_encoding, isbytestring, \ + POLL_TIMEOUT + +''' +Messages sent by client: + + (stdout, unicode) + (stderr, unicode) + (syntaxerror, unicode) + (traceback, unicode) + (done, True iff incomplete command) + +Messages that can be received by client: + (quit, return code) + (run, unicode) + +''' + +def tounicode(raw): # {{{ + if isbytestring(raw): + try: + raw = raw.decode(preferred_encoding, 'replace') + except: + raw = repr(raw) + + if isbytestring(raw): + try: + raw.decode('utf-8', 'replace') + except: + raw = u'Undecodable bytestring' + return raw +# }}} + +class DummyFile(object): # {{{ + + def __init__(self, what, out_queue): + self.closed = False + self.name = 'console' + self.softspace = 0 + self.what = what + self.out_queue = out_queue + + def flush(self): + pass + + def close(self): + pass + + def write(self, raw): + self.out_queue.put((self.what, tounicode(raw))) +# }}} + +class Comm(Thread): # {{{ + + def __init__(self, conn, out_queue, in_queue): + Thread.__init__(self) + self.daemon = True + self.conn = conn + self.out_queue = out_queue + self.in_queue = in_queue + self.keep_going = True + + def run(self): + while self.keep_going: + try: + self.communicate() + except KeyboardInterrupt: + pass + except EOFError: + pass + + def communicate(self): + if self.conn.poll(POLL_TIMEOUT): + try: + obj = self.conn.recv() + except: + pass + else: + self.in_queue.put(obj) + try: + obj = self.out_queue.get_nowait() + except Empty: + pass + else: + try: + self.conn.send(obj) + except: + raise EOFError('interpreter failed to send') +# }}} + +class Interpreter(InteractiveInterpreter): # {{{ + + def __init__(self, queue, local={}): + if '__name__' not in local: + local['__name__'] = '__console__' + if '__doc__' not in local: + local['__doc__'] = None + self.out_queue = queue + sys.stdout = DummyFile('stdout', queue) + sys.stderr = DummyFile('sdterr', queue) + InteractiveInterpreter.__init__(self, locals=local) + + def showtraceback(self, *args, **kwargs): + self.is_syntax_error = False + InteractiveInterpreter.showtraceback(self, *args, **kwargs) + + def showsyntaxerror(self, *args, **kwargs): + self.is_syntax_error = True + InteractiveInterpreter.showsyntaxerror(self, *args, **kwargs) + + def write(self, raw): + what = 'syntaxerror' if self.is_syntax_error else 'traceback' + self.out_queue.put((what, tounicode(raw))) + +# }}} + +def connect(): + os.chdir(os.environ['ORIGWD']) + address = cPickle.loads(unhexlify(os.environ['CALIBRE_WORKER_ADDRESS'])) + key = unhexlify(os.environ['CALIBRE_WORKER_KEY']) + return Client(address, authkey=key) + +def main(): + out_queue = Queue() + in_queue = Queue() + conn = connect() + comm = Comm(conn, out_queue, in_queue) + comm.start() + interpreter = Interpreter(out_queue) + + ret = 0 + + while True: + try: + try: + cmd, data = in_queue.get(1) + except Empty: + pass + else: + if cmd == 'quit': + ret = data + comm.keep_going = False + comm.join() + break + elif cmd == 'run': + if not comm.is_alive(): + ret = 1 + break + ret = False + try: + ret = interpreter.runsource(data) + except KeyboardInterrupt: + pass + except SystemExit: + out_queue.put(('stderr', 'SystemExit ignored\n')) + out_queue.put(('done', ret)) + except KeyboardInterrupt: + pass + + return ret + +if __name__ == '__main__': + main() diff --git a/src/calibre/utils/pyconsole/main.py b/src/calibre/utils/pyconsole/main.py index a708ca1652..664f41ef2e 100644 --- a/src/calibre/utils/pyconsole/main.py +++ b/src/calibre/utils/pyconsole/main.py @@ -11,7 +11,7 @@ from functools import partial from PyQt4.Qt import QDialog, QToolBar, QStatusBar, QLabel, QFont, Qt, \ QApplication, QIcon, QVBoxLayout, QAction -from calibre.constants import __appname__, __version__ +from calibre.utils.pyconsole import dynamic, __appname__, __version__ from calibre.utils.pyconsole.console import Console class MainWindow(QDialog): @@ -26,6 +26,9 @@ class MainWindow(QDialog): self.setLayout(self.l) self.resize(800, 600) + geom = dynamic.get('console_window_geometry', None) + if geom is not None: + self.restoreGeometry(geom) # Setup tool bar {{{ self.tool_bar = QToolBar(self) @@ -62,17 +65,26 @@ class MainWindow(QDialog): self.restart_requested = True self.reject() -def main(): - QApplication.setApplicationName(__appname__+' console') - QApplication.setOrganizationName('Kovid Goyal') - app = QApplication([]) - app + def closeEvent(self, *args): + dynamic.set('console_window_geometry', + bytearray(self.saveGeometry())) + self.console.shutdown() + return QDialog.closeEvent(self, *args) + + +def show(): while True: m = MainWindow() m.exec_() if not m.restart_requested: break +def main(): + QApplication.setApplicationName(__appname__+' console') + QApplication.setOrganizationName('Kovid Goyal') + app = QApplication([]) + app + show() if __name__ == '__main__': main() diff --git a/src/calibre/utils/pyconsole/repl.py b/src/calibre/utils/pyconsole/repl.py deleted file mode 100644 index de6262de14..0000000000 --- a/src/calibre/utils/pyconsole/repl.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python -# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai - -__license__ = 'GPL v3' -__copyright__ = '2010, Kovid Goyal ' -__docformat__ = 'restructuredtext en' - -from code import InteractiveInterpreter - -from PyQt4.Qt import QObject, pyqtSignal - -from calibre import isbytestring -from calibre.constants import preferred_encoding - -class Interpreter(QObject, InteractiveInterpreter): - - # show_error(is_syntax_error, traceback) - show_error = pyqtSignal(object, object) - - def __init__(self, local={}, parent=None): - QObject.__init__(self, parent) - if '__name__' not in local: - local['__name__'] = '__console__' - if '__doc__' not in local: - local['__doc__'] = None - InteractiveInterpreter.__init__(self, locals=local) - - def showtraceback(self, *args, **kwargs): - self.is_syntax_error = False - InteractiveInterpreter.showtraceback(self, *args, **kwargs) - - def showsyntaxerror(self, *args, **kwargs): - self.is_syntax_error = True - InteractiveInterpreter.showsyntaxerror(self, *args, **kwargs) - - def write(self, tb): - self.show_error.emit(self.is_syntax_error, tb) - -class DummyFile(QObject): - - # write_output(unicode_object) - write_output = pyqtSignal(object) - - def __init__(self, parent=None): - QObject.__init__(self, parent) - self.closed = False - self.name = 'console' - self.softspace = 0 - - def flush(self): - pass - - def close(self): - pass - - def write(self, raw): - #import sys, traceback - #print >> sys.__stdout__, 'file,write stack:\n', ''.join(traceback.format_stack()) - if isbytestring(raw): - try: - raw = raw.decode(preferred_encoding, 'replace') - except: - raw = repr(raw) - if isbytestring(raw): - raw = raw.decode(preferred_encoding, 'replace') - self.write_output.emit(raw) -