Launch worker processes on demand

src/calibre/gui2/__init__.py

@@ -156,7 +156,9 @@ def _config():
     c.add_opt('plugin_search_history', default=[],
         help='Search history for the recipe scheduler')
     c.add_opt('worker_limit', default=6,
-            help=_('Maximum number of waiting worker processes'))
+            help=_(
+        'Maximum number of simultaneous conversion/news download jobs. '
+        'This number is twice the actual value for historical reasons.'))
     c.add_opt('get_social_metadata', default=True,
             help=_('Download social metadata (tags/rating/etc.)'))
     c.add_opt('overwrite_author_title_metadata', default=True,

src/calibre/gui2/preferences/misc.py

@@ -6,19 +6,27 @@ __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 
-from calibre.gui2.preferences import ConfigWidgetBase, test_widget
+from calibre.gui2.preferences import ConfigWidgetBase, test_widget, Setting
 from calibre.gui2.preferences.misc_ui import Ui_Form
 from calibre.gui2 import error_dialog, config, open_local_file, info_dialog
 from calibre.constants import isosx
 
-# Check Integrity {{{
+class WorkersSetting(Setting):
+
+    def set_gui_val(self, val):
+        val = val//2
+        Setting.set_gui_val(self, val)
+
+    def get_gui_val(self):
+        val = Setting.get_gui_val(self)
+        return val * 2
 
 class ConfigWidget(ConfigWidgetBase, Ui_Form):
 
     def genesis(self, gui):
         self.gui = gui
         r = self.register
-        r('worker_limit', config, restart_required=True)
+        r('worker_limit', config, restart_required=True, setting=WorkersSetting)
         r('enforce_cpu_limit', config, restart_required=True)
         self.device_detection_button.clicked.connect(self.debug_device_detection)
         self.button_open_config_dir.clicked.connect(self.open_config_dir)

src/calibre/gui2/preferences/misc.ui

@@ -17,7 +17,7 @@
    <item row="0" column="0">
     <widget class="QLabel" name="label_5">
      <property name="text">
-      <string>&amp;Maximum number of waiting worker processes (needs restart):</string>
+      <string>Max. simultaneous conversion/news download jobs:</string>
      </property>
      <property name="buddy">
       <cstring>opt_worker_limit</cstring>
@@ -27,13 +27,7 @@
    <item row="0" column="1">
     <widget class="QSpinBox" name="opt_worker_limit">
      <property name="minimum">
-      <number>2</number>
+      <number>1</number>
-     </property>
-     <property name="maximum">
-      <number>10000</number>
-     </property>
-     <property name="singleStep">
-      <number>2</number>
      </property>
     </widget>
    </item>

src/calibre/manual/faq.rst

@@ -549,17 +549,6 @@ How do I run calibre from my USB stick?
 
 A portable version of calibre is available at: `portableapps.com <http://portableapps.com/node/20518>`_. However, this is usually out of date. You can also setup your own portable calibre install by following :ref:`these instructions <portablecalibre>`.
 
-Why are there so many calibre-parallel processes on my system?
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-|app| maintains two separate worker process pools. One is used for adding books/saving to disk and the other for conversions. You can control the number of worker processes via :guilabel:`Preferences->Advanced->Miscellaneous`. So if you set it to 6 that means a maximum of 3 conversions will run simultaneously. And that is why you will see the number of worker processes changes by two when you use the up and down arrows. On windows, you can set the priority that these processes run with. This can be useful on older, single CPU machines, if you find them slowing down to a crawl when conversions are running.
-
-In addition to this some conversion plugins run tasks in their own pool of processes, so for example if you bulk convert comics, each comic conversion will use three separate processes to render the images. The job manager knows this so it will run only a single comic conversion simultaneously.
-
-And since I'm sure someone will ask: The reason adding/saving books are in separate processes is because of PDF. PDF processing libraries can crash on reading PDFs and I dont want the crash to take down all of calibre. Also when adding EPUB books, in order to extract the cover you have to sometimes render the HTML of the first page, which means that it either has to run in the GUI thread of the main process or in a separate process.
-
-Finally, the reason calibre keep workers alive and idle instead of launching on demand is to workaround the slow startup time of python processes.
-
 How do I run parts of |app| like news download and the content server on my own linux server?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

src/calibre/utils/ipc/server.py

@@ -17,7 +17,7 @@ from binascii import hexlify
 from calibre.utils.ipc.launch import Worker
 from calibre.utils.ipc.worker import PARALLEL_FUNCS
 from calibre import detect_ncpus as cpu_count
-from calibre.constants import iswindows
+from calibre.constants import iswindows, DEBUG
 from calibre.ptempfile import base_dir
 
 _counter = 0
@@ -106,14 +106,14 @@ class Server(Thread):
         self.add_jobs_queue, self.changed_jobs_queue = Queue(), Queue()
         self.kill_queue = Queue()
         self.waiting_jobs = []
-        self.pool, self.workers = deque(), deque()
+        self.workers = deque()
         self.launched_worker_count = 0
         self._worker_launch_lock = RLock()
 
         self.start()
 
     def launch_worker(self, gui=False, redirect_output=None):
-        #start = time.time()
+        start = time.time()
         with self._worker_launch_lock:
             self.launched_worker_count += 1
             id = self.launched_worker_count
@@ -137,7 +137,8 @@ class Server(Thread):
                 break
         if isinstance(cw, basestring):
             raise CriticalError('Failed to launch worker process:\n'+cw)
-        #print 'Launch took:', time.time() - start
+        if DEBUG:
+            print 'Worker Launch took:', time.time() - start
         return cw
 
     def do_launch(self, env, gui, redirect_output, rfile):
@@ -206,13 +207,6 @@ class Server(Thread):
                 job.duration = time.time() - job.start_time
                 self.changed_jobs_queue.put(job)
 
-            # Start new workers
-            if len(self.pool) + len(self.workers) < self.pool_size:
-                try:
-                    self.pool.append(self.launch_worker())
-                except Exception:
-                    pass
-
             # Start waiting jobs
             sj = self.suitable_waiting_job()
             if sj is not None:
@@ -224,7 +218,7 @@ class Server(Thread):
                     job.killed = job.failed = True
                     job.result = None
                 else:
-                    worker = self.pool.pop()
+                    worker = self.launch_worker()
                     worker.start_job(job)
                     self.workers.append(worker)
                     job.log_path = worker.log_path
@@ -238,7 +232,7 @@ class Server(Thread):
                     break
 
     def suitable_waiting_job(self):
-        available_workers = len(self.pool)
+        available_workers = self.pool_size - len(self.workers)
         for worker in self.workers:
             job = worker.job
             if job.core_usage == -1:
@@ -304,11 +298,6 @@ class Server(Thread):
                 worker.kill()
             except:
                 pass
-        for worker in list(self.pool):
-            try:
-                worker.kill()
-            except:
-                pass
 
     def __enter__(self):
         return self