From db7c7931a34859578d25dc865692aef242117077 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 12 Nov 2014 13:00:46 +0530 Subject: [PATCH] Optimize the single book add/save case by using a spare pool --- src/calibre/gui2/actions/add.py | 8 +++--- src/calibre/gui2/actions/save_to_disk.py | 2 +- src/calibre/gui2/add2.py | 4 +-- src/calibre/gui2/save.py | 4 +-- src/calibre/gui2/ui.py | 34 ++++++++++-------------- src/calibre/utils/ipc/pool.py | 4 +-- 6 files changed, 26 insertions(+), 30 deletions(-) diff --git a/src/calibre/gui2/actions/add.py b/src/calibre/gui2/actions/add.py index 89e7ae6d0c..3a8bdcb5ae 100644 --- a/src/calibre/gui2/actions/add.py +++ b/src/calibre/gui2/actions/add.py @@ -159,7 +159,8 @@ class AddAction(InterfaceAction): def do_add_recursive(self, root, single): from calibre.gui2.add2 import Adder - Adder(root, single_book_per_directory=single, db=self.gui.current_db, callback=self._files_added, parent=self.gui) + Adder(root, single_book_per_directory=single, db=self.gui.current_db, + callback=self._files_added, parent=self.gui, pool=self.gui.spare_pool()) def add_recursive_single(self, *args): ''' @@ -362,7 +363,8 @@ class AddAction(InterfaceAction): if not paths: return from calibre.gui2.add2 import Adder - Adder(paths, db=None if to_device else self.gui.current_db, parent=self.gui, callback=partial(self._files_added, on_card=on_card)) + Adder(paths, db=None if to_device else self.gui.current_db, + parent=self.gui, callback=partial(self._files_added, on_card=on_card), pool=self.gui.spare_pool()) def _files_added(self, adder, on_card=None): if adder.items: @@ -485,4 +487,4 @@ class AddAction(InterfaceAction): if ok_paths: from calibre.gui2.add2 import Adder callback = partial(self._add_from_device_adder, on_card=None, model=view.model()) - Adder(ok_paths, db=None, parent=self.gui, callback=callback) + Adder(ok_paths, db=None, parent=self.gui, callback=callback, pool=self.gui.spare_pool()) diff --git a/src/calibre/gui2/actions/save_to_disk.py b/src/calibre/gui2/actions/save_to_disk.py index f568fa121a..7d1f7c454c 100644 --- a/src/calibre/gui2/actions/save_to_disk.py +++ b/src/calibre/gui2/actions/save_to_disk.py @@ -113,7 +113,7 @@ class SaveToDiskAction(InterfaceAction): if save_cover is not None: opts.save_cover = save_cover book_ids = set(map(self.gui.library_view.model().id, rows)) - Saver(book_ids, self.gui.current_db, opts, path, parent=self.gui) + Saver(book_ids, self.gui.current_db, opts, path, parent=self.gui, pool=self.gui.spare_pool()) else: paths = self.gui.current_view().model().paths(rows) self.gui.device_manager.save_books( diff --git a/src/calibre/gui2/add2.py b/src/calibre/gui2/add2.py index ef79a5cefe..ffeee6c261 100644 --- a/src/calibre/gui2/add2.py +++ b/src/calibre/gui2/add2.py @@ -55,7 +55,7 @@ class Adder(QObject): do_one_signal = pyqtSignal() - def __init__(self, source, single_book_per_directory=True, db=None, parent=None, callback=None): + def __init__(self, source, single_book_per_directory=True, db=None, parent=None, callback=None, pool=None): if not validate_source(source, parent): raise ValueError('Bad source') QObject.__init__(self, parent) @@ -64,7 +64,7 @@ class Adder(QObject): self.add_formats_to_existing = prefs['add_formats_to_existing'] self.do_one_signal.connect(self.tick, type=Qt.QueuedConnection) self.tdir = PersistentTemporaryDirectory('_add_books') - self.pool = None + self.pool = pool self.pd = ProgressDialog(_('Adding books...'), _('Scanning for files...'), min=0, max=0, parent=parent, icon='add_book.png') self.db = getattr(db, 'new_api', None) if self.db is not None: diff --git a/src/calibre/gui2/save.py b/src/calibre/gui2/save.py index 699794312e..1958a500ff 100644 --- a/src/calibre/gui2/save.py +++ b/src/calibre/gui2/save.py @@ -72,7 +72,7 @@ class Saver(QObject): do_one_signal = pyqtSignal() - def __init__(self, book_ids, db, opts, root, parent=None): + def __init__(self, book_ids, db, opts, root, parent=None, pool=None): QObject.__init__(self, parent) self.db = db.new_api self.plugboards = self.db.pref('plugboards', {}) @@ -87,7 +87,7 @@ class Saver(QObject): self.do_one = self.do_one_collect self.ids_to_collect = iter(self.all_book_ids) self.tdir = PersistentTemporaryDirectory('_save_to_disk') - self.pool = None + self.pool = pool self.pd.show() self.root, self.opts, self.path_length = sanitize_args(root, opts) diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index e0aa4eb898..8e1ac6cf53 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -20,10 +20,10 @@ from PyQt5.Qt import ( Qt, QTimer, QAction, QMenu, QIcon, pyqtSignal, QUrl, QFont, QDialog, QApplication, QSystemTrayIcon) -from calibre import prints, force_unicode +from calibre import prints, force_unicode, detect_ncpus from calibre.constants import __appname__, isosx, filesystem_encoding, DEBUG from calibre.utils.config import prefs, dynamic -from calibre.utils.ipc.server import Server +from calibre.utils.ipc.pool import Pool from calibre.db.legacy import LibraryDatabase from calibre.customize.ui import interface_actions, available_store_plugins from calibre.gui2 import (error_dialog, GetMetadata, open_url, @@ -213,7 +213,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ self.preferences_action, self.quit_action = actions self.library_path = library_path self.content_server = None - self.spare_servers = [] + self._spare_pool = None self.must_restart_before_config = False self.listener = Listener(listener) self.check_messages_timer = QTimer() @@ -312,7 +312,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ self.alt_esc_action.triggered.connect(self.clear_additional_restriction) # ###################### Start spare job server ######################## - QTimer.singleShot(1000, self.add_spare_server) + QTimer.singleShot(1000, self.create_spare_pool) # ###################### Location Manager ######################## self.location_manager.location_selected.connect(self.location_selected) @@ -472,21 +472,15 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ MainWindow.resizeEvent(self, ev) self.search.setMaximumWidth(self.width()-150) - def add_spare_server(self, *args): - self.spare_servers.append(Server(limit=int(config['worker_limit']/2.0))) + def create_spare_pool(self, *args): + if self._spare_pool is None: + num = min(detect_ncpus(), int(config['worker_limit']/2.0)) + self._spare_pool = Pool(max_workers=num, name='GUIPool') - @property - def spare_server(self): - # Because of the use of the property decorator, we're called one - # extra time. Ignore. - if not hasattr(self, '__spare_server_property_limiter'): - self.__spare_server_property_limiter = True - return None - try: - QTimer.singleShot(1000, self.add_spare_server) - return self.spare_servers.pop() - except: - pass + def spare_pool(self): + ans, self._spare_pool = self._spare_pool, None + QTimer.singleShot(1000, self.create_spare_pool) + return ans def do_proceed(self, func, payload): if callable(func): @@ -893,8 +887,6 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ self.listener.close() self.job_manager.server.close() self.job_manager.threaded_server.close() - while self.spare_servers: - self.spare_servers.pop().close() self.device_manager.keep_going = False self.auto_adder.stop() mb = self.library_view.model().metadata_backup @@ -912,6 +904,8 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ pass except KeyboardInterrupt: pass + if self._spare_pool is not None: + self._spare_pool.shutdown() from calibre.db.delete_service import shutdown shutdown() time.sleep(2) diff --git a/src/calibre/utils/ipc/pool.py b/src/calibre/utils/ipc/pool.py index f2ebf73364..794c8f582c 100644 --- a/src/calibre/utils/ipc/pool.py +++ b/src/calibre/utils/ipc/pool.py @@ -122,12 +122,12 @@ class Pool(Thread): else: join_with_timeout(self.tracker, timeout) - def shutdown(self): + def shutdown(self, wait_time=0.1): ''' Shutdown this pool, terminating all worker process. The pool cannot be used after a shutdown. ''' self.shutting_down = True self.events.put(None) - self.shutdown_workers() + self.shutdown_workers(wait_time=wait_time) def create_worker(self): from calibre.utils.ipc.simple_worker import start_pipe_worker