From 9ec829f46c20bf6ceef1b853daa846930780bdb7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 2 Sep 2013 15:36:21 +0530 Subject: [PATCH] Speed up sorting when book list is showing a subset Speed up sorting when the book list is showing a restricted set of books, such as when the results of a search are displayed or a virtual library is used. Fixes #1217622 [calculated column sorts update the entire database while inside a Virtual Library](https://bugs.launchpad.net/calibre/+bug/1217622) --- src/calibre/db/view.py | 57 ++++++++++++++++---- src/calibre/devices/kindle/driver.py | 2 +- src/calibre/gui2/library/models.py | 2 +- src/calibre/gui2/search_restriction_mixin.py | 2 +- src/calibre/gui2/tag_browser/model.py | 2 +- src/calibre/library/caches.py | 6 +-- src/calibre/library/catalogs/epub_mobi.py | 2 +- src/calibre/library/server/ajax.py | 2 +- src/calibre/library/server/browse.py | 2 +- src/calibre/library/server/cache.py | 2 +- src/calibre/library/server/mobile.py | 2 +- src/calibre/library/server/xml.py | 2 +- 12 files changed, 61 insertions(+), 22 deletions(-) diff --git a/src/calibre/db/view.py b/src/calibre/db/view.py index 17fd624057..80453e6878 100644 --- a/src/calibre/db/view.py +++ b/src/calibre/db/view.py @@ -7,13 +7,14 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import weakref +import weakref, operator from functools import partial from itertools import izip, imap from future_builtins import map from calibre.ebooks.metadata import title_sort from calibre.utils.config_base import tweaks +from calibre.db.write import uniq def sanitize_sort_field_name(field_metadata, field): field = field_metadata.search_term_to_field_key(field.lower().strip()) @@ -123,6 +124,12 @@ class View(object): self._map = tuple(sorted(self.cache.all_book_ids())) self._map_filtered = tuple(self._map) + self.full_map_is_sorted = True + self.sort_history = [('id', True)] + + def add_to_sort_history(self, items): + self.sort_history = uniq((list(items) + list(self.sort_history)), + operator.itemgetter(0))[:tweaks['maximum_resort_levels']] def count(self): return len(self._map) @@ -218,7 +225,7 @@ class View(object): ans = [':::'.join((adata[aid]['name'], adata[aid]['sort'], adata[aid]['link'])) for aid in ids if aid in adata] return ':#:'.join(ans) if ans else default_value - def multisort(self, fields=[], subsort=False, only_ids=None): + def _do_sort(self, ids_to_sort, fields=(), subsort=False): fields = [(sanitize_sort_field_name(self.field_metadata, x), bool(y)) for x, y in fields] keys = self.field_metadata.sortable_field_keys() fields = [x for x in fields if x[0] in keys] @@ -227,11 +234,16 @@ class View(object): if not fields: fields = [('timestamp', False)] - sorted_book_ids = self.cache.multisort( - fields, ids_to_sort=self._map if only_ids is None else only_ids, + return self.cache.multisort( + fields, ids_to_sort=ids_to_sort, virtual_fields={'marked':MarkedVirtualField(self.marked_ids)}) + + def multisort(self, fields=[], subsort=False, only_ids=None): + sorted_book_ids = self._do_sort(self._map if only_ids is None else only_ids, fields=fields, subsort=subsort) if only_ids is None: self._map = tuple(sorted_book_ids) + self.full_map_is_sorted = True + self.add_to_sort_history(fields) if len(self._map_filtered) == len(self._map): self._map_filtered = tuple(self._map) else: @@ -241,9 +253,16 @@ class View(object): smap = {book_id:i for i, book_id in enumerate(sorted_book_ids)} only_ids.sort(key=smap.get) - def search(self, query, return_matches=False): + def incremental_sort(self, fields=(), subsort=False): + if len(self._map) == len(self._map_filtered): + return self.multisort(fields=fields, subsort=subsort) + self._map_filtered = tuple(self._do_sort(self._map_filtered, fields=fields, subsort=subsort)) + self.full_map_is_sorted = False + self.add_to_sort_history(fields) + + def search(self, query, return_matches=False, sort_results=True): ans = self.search_getting_ids(query, self.search_restriction, - set_restriction_count=True) + set_restriction_count=True, sort_results=sort_results) if return_matches: return ans self._map_filtered = tuple(ans) @@ -258,7 +277,7 @@ class View(object): return restriction def search_getting_ids(self, query, search_restriction, - set_restriction_count=False, use_virtual_library=True): + set_restriction_count=False, use_virtual_library=True, sort_results=True): if use_virtual_library: search_restriction = self._build_restriction_string(search_restriction) q = '' @@ -271,10 +290,28 @@ class View(object): if not q: if set_restriction_count: self.search_restriction_book_count = len(self._map) - return list(self._map) + rv = list(self._map) + if sort_results and not self.full_map_is_sorted: + rv = self._do_sort(rv, fields=self.sort_history) + self._map = tuple(rv) + self.full_map_is_sorted = True + return rv matches = self.cache.search( query, search_restriction, virtual_fields={'marked':MarkedVirtualField(self.marked_ids)}) - rv = [x for x in self._map if x in matches] + if len(matches) == len(self._map): + rv = list(self._map) + else: + rv = [x for x in self._map if x in matches] + if sort_results and not self.full_map_is_sorted: + # We need to sort the search results + if matches.issubset(frozenset(self._map_filtered)): + rv = [x for x in self._map_filtered if x in matches] + else: + rv = self._do_sort(rv, fields=self.sort_history) + if len(matches) == len(self._map): + # We have sorted all ids, update self._map + self._map = tuple(rv) + self.full_map_is_sorted = True if set_restriction_count and q == search_restriction: self.search_restriction_book_count = len(rv) return rv @@ -338,6 +375,8 @@ class View(object): def refresh(self, field=None, ascending=True, clear_caches=True, do_search=True): self._map = tuple(sorted(self.cache.all_book_ids())) self._map_filtered = tuple(self._map) + self.full_map_is_sorted = True + self.sort_history = [('id', True)] if clear_caches: self.cache.clear_caches() if field is not None: diff --git a/src/calibre/devices/kindle/driver.py b/src/calibre/devices/kindle/driver.py index 4e083fbe04..214f8cc65c 100644 --- a/src/calibre/devices/kindle/driver.py +++ b/src/calibre/devices/kindle/driver.py @@ -270,7 +270,7 @@ class KINDLE(USBMS): elif bm.type == 'kindle_clippings': # Find 'My Clippings' author=Kindle in database, or add last_update = 'Last modified %s' % strftime(u'%x %X',bm.value['timestamp'].timetuple()) - mc_id = list(db.data.search_getting_ids('title:"My Clippings"', '')) + mc_id = list(db.data.search_getting_ids('title:"My Clippings"', '', sort_results=False)) if mc_id: db.add_format_with_hooks(mc_id[0], 'TXT', bm.value['path'], index_is_id=True) diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index ebf5e1d188..f3b8cbde56 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -407,7 +407,7 @@ class BooksModel(QAbstractTableModel): # {{{ def _sort(self, label, order, reset): self.about_to_be_sorted.emit(self.db.id) - self.db.sort(label, order) + self.db.data.incremental_sort([(label, order)]) if reset: self.reset() self.sorted_on = (label, order) diff --git a/src/calibre/gui2/search_restriction_mixin.py b/src/calibre/gui2/search_restriction_mixin.py index 52e456c4dc..2ab5959800 100644 --- a/src/calibre/gui2/search_restriction_mixin.py +++ b/src/calibre/gui2/search_restriction_mixin.py @@ -285,7 +285,7 @@ class CreateVirtualLibrary(QDialog): # {{{ try: db = self.gui.library_view.model().db - recs = db.data.search_getting_ids('', v, use_virtual_library=False) + recs = db.data.search_getting_ids('', v, use_virtual_library=False, sort_results=False) except ParseException as e: error_dialog(self.gui, _('Invalid search'), _('The search in the search box is not valid'), diff --git a/src/calibre/gui2/tag_browser/model.py b/src/calibre/gui2/tag_browser/model.py index 48ac9045dd..60081fd5b5 100644 --- a/src/calibre/gui2/tag_browser/model.py +++ b/src/calibre/gui2/tag_browser/model.py @@ -846,7 +846,7 @@ class TagsModel(QAbstractItemModel): # {{{ try: data = self.db.get_categories(sort=sort, icon_map=self.category_icon_map, - ids=self.db.search('', return_matches=True)) + ids=self.db.search('', return_matches=True, sort_results=False)) except: data = self.db.get_categories(sort=sort, icon_map=self.category_icon_map) self.restriction_error.emit() diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index a7ceb706f7..33c2f82707 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -816,9 +816,9 @@ class ResultCache(SearchQueryParser): # {{{ current_candidates -= matches return matches - def search(self, query, return_matches=False): + def search(self, query, return_matches=False, sort_results=True): ans = self.search_getting_ids(query, self.search_restriction, - set_restriction_count=True) + set_restriction_count=True, sort_results=sort_results) if return_matches: return ans self._map_filtered = ans @@ -833,7 +833,7 @@ class ResultCache(SearchQueryParser): # {{{ return restriction def search_getting_ids(self, query, search_restriction, - set_restriction_count=False, use_virtual_library=True): + set_restriction_count=False, use_virtual_library=True, sort_results=True): if use_virtual_library: search_restriction = self._build_restriction_string(search_restriction) q = '' diff --git a/src/calibre/library/catalogs/epub_mobi.py b/src/calibre/library/catalogs/epub_mobi.py index 673d764593..e2c80b8cdc 100644 --- a/src/calibre/library/catalogs/epub_mobi.py +++ b/src/calibre/library/catalogs/epub_mobi.py @@ -447,7 +447,7 @@ class EPUB_MOBI(CatalogPlugin): try: search_text = 'title:"%s" author:%s' % ( opts.catalog_title.replace('"', '\\"'), 'calibre') - matches = db.search(search_text, return_matches=True) + matches = db.search(search_text, return_matches=True, sort_results=False) if matches: cpath = db.cover(matches[0], index_is_id=True, as_path=True) if cpath and os.path.exists(cpath): diff --git a/src/calibre/library/server/ajax.py b/src/calibre/library/server/ajax.py index 05ddfc5015..80ead51d00 100644 --- a/src/calibre/library/server/ajax.py +++ b/src/calibre/library/server/ajax.py @@ -594,7 +594,7 @@ class AjaxServer(object): if isbytestring(query): query = query.decode('UTF-8') - ids = self.db.search_getting_ids(query.strip(), self.search_restriction) + ids = self.db.search_getting_ids(query.strip(), self.search_restriction, sort_results=False) ids = list(ids) self.db.data.multisort(fields=[(sfield, sort_order == 'asc')], subsort=True, only_ids=ids) diff --git a/src/calibre/library/server/browse.py b/src/calibre/library/server/browse.py index a770f19856..5e98d70c27 100644 --- a/src/calibre/library/server/browse.py +++ b/src/calibre/library/server/browse.py @@ -922,7 +922,7 @@ class BrowseServer(object): import random try: book_id = random.choice(self.db.search_getting_ids( - '', self.search_restriction)) + '', self.search_restriction, sort_results=False)) except IndexError: raise cherrypy.HTTPError(404, 'This library has no books') ans = self.browse_render_details(book_id, add_random_button=True) diff --git a/src/calibre/library/server/cache.py b/src/calibre/library/server/cache.py index 2ad7b543cb..2d27962dcc 100644 --- a/src/calibre/library/server/cache.py +++ b/src/calibre/library/server/cache.py @@ -21,7 +21,7 @@ class Cache(object): def search_cache(self, search): old = self._search_cache.pop(search, None) if old is None or old[0] <= self.db.last_modified(): - matches = self.db.data.search_getting_ids(search, self.search_restriction) + matches = self.db.data.search_getting_ids(search, self.search_restriction, sort_results=False) if not matches: matches = [] self._search_cache[search] = (utcnow(), frozenset(matches)) diff --git a/src/calibre/library/server/mobile.py b/src/calibre/library/server/mobile.py index 767a48a9d9..a18109c924 100644 --- a/src/calibre/library/server/mobile.py +++ b/src/calibre/library/server/mobile.py @@ -222,7 +222,7 @@ class MobileServer(object): search = '' if isbytestring(search): search = search.decode('UTF-8') - ids = self.db.search_getting_ids(search.strip(), self.search_restriction) + ids = self.db.search_getting_ids(search.strip(), self.search_restriction, sort_results=False) FM = self.db.FIELD_MAP items = [r for r in iter(self.db) if r[FM['id']] in ids] if sort is not None: diff --git a/src/calibre/library/server/xml.py b/src/calibre/library/server/xml.py index 18ddf6bb43..091ec43f36 100644 --- a/src/calibre/library/server/xml.py +++ b/src/calibre/library/server/xml.py @@ -53,7 +53,7 @@ class XMLServer(object): if isbytestring(search): search = search.decode('UTF-8') - ids = self.db.search_getting_ids(search.strip(), self.search_restriction) + ids = self.db.search_getting_ids(search.strip(), self.search_restriction, sort_results=False) FM = self.db.FIELD_MAP