Speed up adding/saving of small numbers of books by maintaining a spare job server

src/calibre/ebooks/metadata/worker.py

@@ -53,8 +53,9 @@ class Progress(object):
 
 class ReadMetadata(Thread):
 
-    def __init__(self, tasks, result_queue):
+    def __init__(self, tasks, result_queue, spare_server=None):
         self.tasks, self.result_queue = tasks, result_queue
+        self.spare_server = spare_server
         self.canceled = False
         Thread.__init__(self)
         self.daemon = True
@@ -67,7 +68,7 @@ class ReadMetadata(Thread):
             for b in t:
                 ids.add(b[0])
         progress = Progress(self.result_queue, self.tdir)
-        server = Server()
+        server = Server() if self.spare_server is None else self.spare_server
         for i, task in enumerate(self.tasks):
             job = ParallelJob('read_metadata',
                 'Read metadata (%d of %d)'%(i, len(self.tasks)),
@@ -110,20 +111,20 @@ class ReadMetadata(Thread):
                 os.remove(job.log_path)
 
 
-def read_metadata(paths, result_queue, chunk=50):
+def read_metadata(paths, result_queue, chunk=50, spare_server=None):
     tasks = []
     pos = 0
     while pos < len(paths):
         tasks.append(paths[pos:pos+chunk])
         pos += chunk
-    t = ReadMetadata(tasks, result_queue)
+    t = ReadMetadata(tasks, result_queue, spare_server=spare_server)
     t.start()
     return t
 
 class SaveWorker(Thread):
 
     def __init__(self, result_queue, db, ids, path, by_author=False,
-            single_dir=False,  single_format=None):
+            single_dir=False,  single_format=None, spare_server=None):
         Thread.__init__(self)
         self.daemon = True
         self.path, self.by_author = path, by_author
@@ -133,10 +134,11 @@ class SaveWorker(Thread):
         self.canceled = False
         self.result_queue = result_queue
         self.error = None
+        self.spare_server = spare_server
         self.start()
 
     def run(self):
-        server = Server()
+        server = Server() if self.spare_server is None else self.spare_server
         ids = set(self.ids)
         tasks = server.split(list(ids))
         jobs = set([])

src/calibre/gui2/add.py

@@ -39,9 +39,10 @@ class RecursiveFind(QThread):
 
 class Adder(QObject):
 
-    def __init__(self, parent, db, callback):
+    def __init__(self, parent, db, callback, spare_server=None):
         QObject.__init__(self, parent)
         self.pd = ProgressDialog(_('Adding...'), parent=parent)
+        self.spare_server = spare_server
         self.db = db
         self.pd.setModal(True)
         self.pd.show()
@@ -79,7 +80,8 @@ class Adder(QObject):
             tasks.append((i, b))
             self.ids[i] = b
             self.nmap[i] = os.path.basename(b[0])
-        self.worker = read_metadata(tasks, self.rq)
+        self.worker = read_metadata(tasks, self.rq,
+                spare_server=self.spare_server)
         self.pd.set_min(0)
         self.pd.set_max(len(self.ids))
         self.pd.value = 0
@@ -165,9 +167,11 @@ class Adder(QObject):
 class Saver(QObject):
 
     def __init__(self, parent, db, callback, rows, path,
-            by_author=False, single_dir=False, single_format=None):
+            by_author=False, single_dir=False, single_format=None,
+            spare_server=None):
         QObject.__init__(self, parent)
         self.pd = ProgressDialog(_('Saving...'), parent=parent)
+        self.spare_server = spare_server
         self.db = db
         self.pd.setModal(True)
         self.pd.show()
@@ -183,7 +187,7 @@ class Saver(QObject):
 
         from calibre.ebooks.metadata.worker import SaveWorker
         self.worker = SaveWorker(self.rq, db, self.ids, path, by_author,
-                single_dir, single_format)
+                single_dir, single_format, spare_server=self.spare_server)
         self.connect(self.pd, SIGNAL('canceled()'), self.canceled)
         self.timer = QTimer(self)
         self.connect(self.timer, SIGNAL('timeout()'), self.update)

src/calibre/gui2/main.py

@@ -18,6 +18,7 @@ from calibre import __version__, __appname__, sanitize_file_name, \
                     iswindows, isosx, prints, patheq
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.utils.config import prefs, dynamic
+from calibre.utils.ipc.server import Server
 from calibre.gui2 import APP_UID, warning_dialog, choose_files, error_dialog, \
                            initialize_file_icon_provider, question_dialog,\
                            pixmap_to_data, choose_dir, ORG_NAME, \
@@ -104,6 +105,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
 
     def __init__(self, listener, opts, actions, parent=None):
         self.preferences_action, self.quit_action = actions
+        self.spare_servers = []
         MainWindow.__init__(self, opts, parent)
         # Initialize fontconfig in a separate thread as this can be a lengthy
         # process if run for the first time on this machine
@@ -460,6 +462,8 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
 
 
         self.setMaximumHeight(max_available_height())
+        ####################### Start spare job server ########################
+        QTimer.singleShot(1000, self.add_spare_server)
 
         ####################### Setup device detection ########################
         self.device_manager = DeviceManager(Dispatcher(self.device_detected),
@@ -490,7 +494,16 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
         self.connect(self.action_sync, SIGNAL('triggered(bool)'),
                 self._sync_menu.trigger_default)
 
+    def add_spare_server(self, *args):
+        self.spare_servers.append(Server())
 
+    @property
+    def spare_server(self):
+        try:
+            QTimer.singleShot(1000, self.add_spare_server)
+            return self.spare_servers.pop()
+        except:
+            pass
 
     def no_op(self, *args):
         pass
@@ -730,7 +743,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
         from calibre.gui2.add import Adder
         self._adder = Adder(self,
                 self.library_view.model().db,
-                Dispatcher(self._files_added))
+                Dispatcher(self._files_added), spare_server=self.spare_server)
         self._adder.add_recursive(root, single)
 
     def add_recursive_single(self, checked):
@@ -793,7 +806,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
         self.__adder_func = partial(self._files_added, on_card=on_card)
         self._adder = Adder(self,
                 None if to_device else self.library_view.model().db,
-                Dispatcher(self.__adder_func))
+                Dispatcher(self.__adder_func), spare_server=self.spare_server)
         self._adder.add(paths)
 
     def _files_added(self, paths=[], names=[], infos=[], on_card=None):
@@ -986,7 +999,8 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
                     Dispatcher(self._books_saved), rows, path,
                     by_author=self.library_view.model().by_author,
                     single_dir=single_dir,
-                    single_format=single_format)
+                    single_format=single_format,
+                    spare_server=self.spare_server)
 
         else:
             paths = self.current_view().model().paths(rows)
@@ -1618,6 +1632,8 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
         self.check_messages_timer.stop()
         self.listener.close()
         self.job_manager.server.close()
+        while self.spare_servers:
+            self.spare_servers.pop().close()
         self.device_manager.keep_going = False
         self.cover_cache.stop()
         self.hide()