Beginnings of new IPC framework

src/calibre/devices/eb600/driver.py

@@ -43,7 +43,7 @@ class EB600(USBMS):
 
     def windows_sort_drives(self, drives):
         main = drives.get('main', None)
-        card = drives.get('card', None)
+        card = drives.get('carda', None)
         if card and main and card < main:
             drives['main'] = card
             drives['carda'] = main

src/calibre/devices/jetbook/driver.py

@@ -76,7 +76,7 @@ class JETBOOK(USBMS):
 
             if newpath == path:
                 newpath = os.path.join(newpath, author, title)
-                
+
             if not os.path.exists(newpath):
                 os.makedirs(newpath)
 
@@ -97,7 +97,7 @@ class JETBOOK(USBMS):
             self.report_progress((i+1) / float(len(files)), _('Transferring books to device...'))
 
         self.report_progress(1.0, _('Transferring books to device...'))
-        
+
         return zip(paths, cycle([on_card]))
 
     @classmethod
@@ -109,7 +109,7 @@ class JETBOOK(USBMS):
                 return txt.decode(sys.getfilesystemencoding(), 'replace')
 
             return txt
-            
+
         from calibre.devices.usbms.driver import metadata_from_formats
         mi = metadata_from_formats([path])
 
@@ -126,10 +126,10 @@ class JETBOOK(USBMS):
 
     def windows_sort_drives(self, drives):
         main = drives.get('main', None)
-        card = drives.get('card', None)
+        card = drives.get('carda', None)
         if card and main and card < main:
             drives['main'] = card
-            drives['card'] = main
+            drives['carda'] = main
 
         return drives
 

src/calibre/gui2/convert/gui_conversion.py

@@ -4,12 +4,15 @@ __license__ = 'GPL 3'
 __copyright__ = '2009, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
-from calibre.ebooks.conversion.plumber import Plumber
+from calibre.ebooks.conversion.plumber import Plumber, DummyReporter
 from calibre.utils.logging import Log
+from calibre.customize.conversion import OptionRecommendation
 
-def gui_convert(input, output, recommendations, notification):
+def gui_convert(input, output, recommendations, notification=DummyReporter()):
-    plumber = Plumber(input, output, Log(), notification)
+    recommendations = list(recommendations)
+    recommendations.append(('verbose', 2, OptionRecommendation.HIGH))
+    plumber = Plumber(input, output, Log(), report_progress=notification)
     plumber.merge_ui_recommendations(recommendations)
-    
+
     plumber.run()
 

src/calibre/gui2/tools.py

@@ -16,6 +16,7 @@ from calibre.gui2 import warning_dialog
 from calibre.gui2.convert.single import NoSupportedInputFormats
 from calibre.gui2.convert.single import Config as SingleConfig
 from calibre.gui2.convert.bulk import BulkConfig
+from calibre.customize.conversion import OptionRecommendation
 from calibre.utils.config import prefs
 
 def convert_single_ebook(parent, db, book_ids, auto_conversion=False, out_format=None):
@@ -131,13 +132,16 @@ def fetch_scheduled_recipe(recipe, script):
     fmt = prefs['output_format'].lower()
     pt = PersistentTemporaryFile(suffix='_recipe_out.%s'%fmt.lower())
     pt.close()
-    args = ['ebook-convert', script, pt.name, '-vv']
+    recs = []
+    args = [script, pt.name, recs]
     if recipe.needs_subscription:
         x = config.get('recipe_account_info_%s'%recipe.id, False)
         if not x:
             raise ValueError(_('You must set a username and password for %s')%recipe.title)
-        args.extend(['--username', x[0], '--password', x[1]])
+        recs.append(('username', x[0], OptionRecommendation.HIGH))
-
+        recs.append(('password', x[1], OptionRecommendation.HIGH))
-    return 'ebook-convert', [args], _('Fetch news from ')+recipe.title, fmt.upper(), [pt]
+
+
+    return 'gui_convert', args, _('Fetch news from ')+recipe.title, fmt.upper(), [pt]
 
 

src/calibre/ptempfile.py

@@ -31,6 +31,7 @@ class PersistentTemporaryFile(object):
                                     dir=dir)
         self._file = os.fdopen(fd, mode)
         self._name = name
+        self._fd = fd
         atexit.register(cleanup, name)
 
     def __getattr__(self, name):

src/calibre/utils/ipc/__init__.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import with_statement
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+

src/calibre/utils/ipc/launch.py

@@ -0,0 +1,150 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import with_statement
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import subprocess, os, sys, time
+
+from calibre.constants import iswindows, isosx, isfrozen
+from calibre.utils.config import prefs
+from calibre.ptempfile import PersistentTemporaryFile
+
+if iswindows:
+    import win32process
+
+class Worker(object):
+    '''
+    Platform independent object for launching child processes. All processes
+    have the environment variable :envvar:`CALIBRE_WORKER` set.
+
+    Useful attributes: ``is_alive``, ``returncode``
+    usefule methods: ``kill``
+
+    To launch child simply call the Worker object. By default, the child's
+    output is redirected to an on disk file, the path to which is returned by
+    the call.
+    '''
+
+    @property
+    def osx_interpreter(self):
+        exe = os.path.basename(sys.executable)
+        return exe if 'python' in exe else 'python'
+
+    @property
+    def osx_contents_dir(self):
+        fd = os.path.realpath(getattr(sys, 'frameworks_dir'))
+        return os.path.dirname(fd)
+
+    @property
+    def executable(self):
+        if iswindows:
+            return os.path.join(os.path.dirname(sys.executable),
+                   'calibre-parallel.exe' if isfrozen else \
+                           'Scripts\\calibre-parallel.exe')
+        if isosx:
+            if not isfrozen: return 'calibre-parallel'
+            contents = os.path.join(self.osx_contents_dir,
+                    'console.app', 'Contents')
+            return os.path.join(contents, 'MacOS', self.osx_interpreter)
+
+        return os.path.join(getattr(sys, 'frozen_path'), 'calibre-parallel') \
+                            if isfrozen else 'calibre-parallel'
+
+    @property
+    def gui_executable(self):
+        if isfrozen and isosx:
+            return os.path.join(self.osx_contents_dir,
+                    'MacOS', self.osx_interpreter)
+
+        return self.executable
+
+    @property
+    def env(self):
+        env = dict(os.environ)
+        env['CALIBRE_WORKER'] = '1'
+        env.update(self._env)
+        return env
+
+    @property
+    def is_alive(self):
+        return hasattr(self, 'child') and self.child.poll() is not None
+
+    @property
+    def returncode(self):
+        if not hasattr(self, 'child'): return None
+        self.child.poll()
+        return self.child.returncode
+
+    def kill(self):
+        try:
+            if self.is_alive:
+                if iswindows:
+                    return self.child.kill()
+                try:
+                    self.child.terminate()
+                    st = time.time()
+                    while self.is_alive and time.time()-st < 2:
+                        time.sleep(0.2)
+                finally:
+                    if self.is_alive:
+                        self.child.kill()
+        except:
+            pass
+
+    def __init__(self, env, gui=False):
+        self._env = {}
+        self.gui = gui
+        if isosx and isfrozen:
+            contents = os.path.join(self.osx_contents_dir, 'console.app', 'Contents')
+            resources = os.path.join(contents, 'Resources')
+            fd = os.path.join(contents, 'Frameworks')
+            self._env['PYTHONHOME']  = resources
+            self._env['MAGICK_HOME'] = os.path.join(fd, 'ImageMagick')
+            self._env['DYLD_LIBRARY_PATH'] = os.path.join(fd, 'ImageMagick', 'lib')
+        if isfrozen and not (iswindows or isosx):
+            self._env['LD_LIBRARY_PATH'] = getattr(sys, 'frozen_path') + ':'\
+                    + os.environ.get('LD_LIBRARY_PATH', '')
+        self._env.update(env)
+
+    def __call__(self, redirect_output=True, cwd=None, priority=None):
+        '''
+        If redirect_output is True, output from the child is redirected
+        to a file on disk and this method returns the path to that file.
+        '''
+        exe = self.gui_executable if self.gui else self.executable
+        env = self.env
+        env['ORIGWD'] = cwd or os.path.abspath(os.getcwd())
+        _cwd = cwd
+        if isfrozen and not iswindows and not isosx:
+            _cwd = getattr(sys, 'frozen_path', None)
+        if priority is None:
+            priority = prefs['worker_process_priority']
+        cmd = [exe]
+        if isosx:
+            cmd += ['-c', 'from calibre.utils.worker import main; main()']
+        args = {
+                'env' : env,
+                'cwd' : _cwd,
+                }
+        if iswindows:
+            priority = {
+                    'high'   : win32process.HIGH_PRIORITY_CLASS,
+                    'normal' : win32process.NORMAL_PRIORITY_CLASS,
+                    'low'    : win32process.IDLE_PRIORITY_CLASS}[priority]
+            args['creationflags'] = win32process.CREATE_NO_WINDOW|priority
+        ret = None
+        if redirect_output:
+            self._file = PersistentTemporaryFile('_worker_redirect.log')
+            args['stdout'] = self._file._fd
+            args['stderr'] = subprocess.STDOUT
+            ret = self._file.name
+
+        self.child = subprocess.Popen(cmd, **args)
+
+        return ret
+
+
+

src/calibre/utils/ipc/server.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import with_statement
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+

src/calibre/utils/ipc/worker.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import with_statement
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import os, cPickle
+from multiprocessing.connection import Client
+from threading import Thread
+from queue import Queue
+from contextlib import closing
+
+PARALLEL_FUNCS = {
+      'lrfviewer'    :
+        ('calibre.gui2.lrf_renderer.main', 'main', None),
+
+      'ebook-viewer'    :
+        ('calibre.gui2.viewer.main', 'main', None),
+
+      'render_pages' :
+        ('calibre.ebooks.comic.input', 'render_pages', 'notification'),
+
+      'gui_convert'     :
+        ('calibre.gui2.convert.gui_conversion', 'gui_convert', 'notification'),
+}
+
+class Progress(Thread):
+
+    def __init__(self, conn):
+        self.daemon = True
+        Thread.__init__(self)
+        self.conn = conn
+        self.queue = Queue()
+
+    def __call__(self, percent, msg=''):
+        self.queue.put((percent, msg))
+
+    def run(self):
+        while True:
+            x = self.queue.get()
+            if x is None:
+                break
+            try:
+                self.conn.send(x)
+            except:
+                break
+
+
+
+def get_func(name):
+    module, func, notification = PARALLEL_FUNCS[name]
+    module = __import__(module, fromlist=[1])
+    func = getattr(module, func)
+    return func, notification
+
+def main():
+    address = cPickle.loads(os.environ['CALIBRE_WORKER_ADDRESS'])
+    key     = os.environ['CALIBRE_WORKER_KEY']
+    with closing(Client(address, authkey=key)) as conn:
+        name, args, kwargs = conn.recv()
+        func, notification = get_func(name)
+        notifier = Progress(conn)
+        if notification:
+            kwargs[notification] = notifier
+            notifier.start()
+
+        func(*args, **kwargs)
+
+        notifier.queue.put(None)
+
+    return 0
+
+
+
+if __name__ == '__main__':
+    raise SystemExit(main())