From e6728649be886a230ac0fe83c4884483fc8c9c5a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 2 Jun 2009 10:20:02 -0700 Subject: [PATCH] Speed up adding/saving of small numbers of books by maintaining a spare job server --- src/calibre/ebooks/metadata/worker.py | 14 ++++++++------ src/calibre/gui2/add.py | 12 ++++++++---- src/calibre/gui2/main.py | 22 +++++++++++++++++++--- 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/src/calibre/ebooks/metadata/worker.py b/src/calibre/ebooks/metadata/worker.py index 281039fb12..3c5ffd1be2 100644 --- a/src/calibre/ebooks/metadata/worker.py +++ b/src/calibre/ebooks/metadata/worker.py @@ -53,8 +53,9 @@ class Progress(object): class ReadMetadata(Thread): - def __init__(self, tasks, result_queue): + def __init__(self, tasks, result_queue, spare_server=None): self.tasks, self.result_queue = tasks, result_queue + self.spare_server = spare_server self.canceled = False Thread.__init__(self) self.daemon = True @@ -67,7 +68,7 @@ class ReadMetadata(Thread): for b in t: ids.add(b[0]) progress = Progress(self.result_queue, self.tdir) - server = Server() + server = Server() if self.spare_server is None else self.spare_server for i, task in enumerate(self.tasks): job = ParallelJob('read_metadata', 'Read metadata (%d of %d)'%(i, len(self.tasks)), @@ -110,20 +111,20 @@ class ReadMetadata(Thread): os.remove(job.log_path) -def read_metadata(paths, result_queue, chunk=50): +def read_metadata(paths, result_queue, chunk=50, spare_server=None): tasks = [] pos = 0 while pos < len(paths): tasks.append(paths[pos:pos+chunk]) pos += chunk - t = ReadMetadata(tasks, result_queue) + t = ReadMetadata(tasks, result_queue, spare_server=spare_server) t.start() return t class SaveWorker(Thread): def __init__(self, result_queue, db, ids, path, by_author=False, - single_dir=False, single_format=None): + single_dir=False, single_format=None, spare_server=None): Thread.__init__(self) self.daemon = True self.path, self.by_author = path, by_author @@ -133,10 +134,11 @@ class SaveWorker(Thread): self.canceled = False self.result_queue = result_queue self.error = None + self.spare_server = spare_server self.start() def run(self): - server = Server() + server = Server() if self.spare_server is None else self.spare_server ids = set(self.ids) tasks = server.split(list(ids)) jobs = set([]) diff --git a/src/calibre/gui2/add.py b/src/calibre/gui2/add.py index f5c8be4338..ec253e5ae0 100644 --- a/src/calibre/gui2/add.py +++ b/src/calibre/gui2/add.py @@ -39,9 +39,10 @@ class RecursiveFind(QThread): class Adder(QObject): - def __init__(self, parent, db, callback): + def __init__(self, parent, db, callback, spare_server=None): QObject.__init__(self, parent) self.pd = ProgressDialog(_('Adding...'), parent=parent) + self.spare_server = spare_server self.db = db self.pd.setModal(True) self.pd.show() @@ -79,7 +80,8 @@ class Adder(QObject): tasks.append((i, b)) self.ids[i] = b self.nmap[i] = os.path.basename(b[0]) - self.worker = read_metadata(tasks, self.rq) + self.worker = read_metadata(tasks, self.rq, + spare_server=self.spare_server) self.pd.set_min(0) self.pd.set_max(len(self.ids)) self.pd.value = 0 @@ -165,9 +167,11 @@ class Adder(QObject): class Saver(QObject): def __init__(self, parent, db, callback, rows, path, - by_author=False, single_dir=False, single_format=None): + by_author=False, single_dir=False, single_format=None, + spare_server=None): QObject.__init__(self, parent) self.pd = ProgressDialog(_('Saving...'), parent=parent) + self.spare_server = spare_server self.db = db self.pd.setModal(True) self.pd.show() @@ -183,7 +187,7 @@ class Saver(QObject): from calibre.ebooks.metadata.worker import SaveWorker self.worker = SaveWorker(self.rq, db, self.ids, path, by_author, - single_dir, single_format) + single_dir, single_format, spare_server=self.spare_server) self.connect(self.pd, SIGNAL('canceled()'), self.canceled) self.timer = QTimer(self) self.connect(self.timer, SIGNAL('timeout()'), self.update) diff --git a/src/calibre/gui2/main.py b/src/calibre/gui2/main.py index ad472ed4db..b2648409f2 100644 --- a/src/calibre/gui2/main.py +++ b/src/calibre/gui2/main.py @@ -18,6 +18,7 @@ from calibre import __version__, __appname__, sanitize_file_name, \ iswindows, isosx, prints, patheq from calibre.ptempfile import PersistentTemporaryFile from calibre.utils.config import prefs, dynamic +from calibre.utils.ipc.server import Server from calibre.gui2 import APP_UID, warning_dialog, choose_files, error_dialog, \ initialize_file_icon_provider, question_dialog,\ pixmap_to_data, choose_dir, ORG_NAME, \ @@ -104,6 +105,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): def __init__(self, listener, opts, actions, parent=None): self.preferences_action, self.quit_action = actions + self.spare_servers = [] MainWindow.__init__(self, opts, parent) # Initialize fontconfig in a separate thread as this can be a lengthy # process if run for the first time on this machine @@ -460,6 +462,8 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): self.setMaximumHeight(max_available_height()) + ####################### Start spare job server ######################## + QTimer.singleShot(1000, self.add_spare_server) ####################### Setup device detection ######################## self.device_manager = DeviceManager(Dispatcher(self.device_detected), @@ -490,7 +494,16 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): self.connect(self.action_sync, SIGNAL('triggered(bool)'), self._sync_menu.trigger_default) + def add_spare_server(self, *args): + self.spare_servers.append(Server()) + @property + def spare_server(self): + try: + QTimer.singleShot(1000, self.add_spare_server) + return self.spare_servers.pop() + except: + pass def no_op(self, *args): pass @@ -730,7 +743,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): from calibre.gui2.add import Adder self._adder = Adder(self, self.library_view.model().db, - Dispatcher(self._files_added)) + Dispatcher(self._files_added), spare_server=self.spare_server) self._adder.add_recursive(root, single) def add_recursive_single(self, checked): @@ -793,7 +806,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): self.__adder_func = partial(self._files_added, on_card=on_card) self._adder = Adder(self, None if to_device else self.library_view.model().db, - Dispatcher(self.__adder_func)) + Dispatcher(self.__adder_func), spare_server=self.spare_server) self._adder.add(paths) def _files_added(self, paths=[], names=[], infos=[], on_card=None): @@ -986,7 +999,8 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): Dispatcher(self._books_saved), rows, path, by_author=self.library_view.model().by_author, single_dir=single_dir, - single_format=single_format) + single_format=single_format, + spare_server=self.spare_server) else: paths = self.current_view().model().paths(rows) @@ -1618,6 +1632,8 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): self.check_messages_timer.stop() self.listener.close() self.job_manager.server.close() + while self.spare_servers: + self.spare_servers.pop().close() self.device_manager.keep_going = False self.cover_cache.stop() self.hide()