Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-08 02:34:06 -04:00
Code Issues Packages Projects Releases Wiki Activity

Speed up sorting when book list is showing a subset

Browse Source 
Speed up sorting when the book list is showing a restricted set of
books, such as when the results of a search are displayed or a virtual
library is used. Fixes #1217622 [calculated column sorts update the entire database while inside a Virtual Library](https://bugs.launchpad.net/calibre/+bug/1217622)
This commit is contained in:
Kovid Goyal 2013-09-02 15:36:21 +05:30
parent 86564b27e9
commit 9ec829f46c
12 changed files with 61 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
src/calibre/db/view.py
View File

@ -7,13 +7,14 @@ __license__ = 'GPL v3'
__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import weakref
import weakref, operator
from functools import partial
from itertools import izip, imap
from future_builtins import map
from calibre.ebooks.metadata import title_sort
from calibre.utils.config_base import tweaks
from calibre.db.write import uniq
def sanitize_sort_field_name(field_metadata, field):
field = field_metadata.search_term_to_field_key(field.lower().strip())
@ -123,6 +124,12 @@ class View(object):
self._map = tuple(sorted(self.cache.all_book_ids()))
self._map_filtered = tuple(self._map)
self.full_map_is_sorted = True
self.sort_history = [('id', True)]
def add_to_sort_history(self, items):
self.sort_history = uniq((list(items) + list(self.sort_history)),
operator.itemgetter(0))[:tweaks['maximum_resort_levels']]
def count(self):
return len(self._map)
@ -218,7 +225,7 @@ class View(object):
ans = [':::'.join((adata[aid]['name'], adata[aid]['sort'], adata[aid]['link'])) for aid in ids if aid in adata]
return ':#:'.join(ans) if ans else default_value
def multisort(self, fields=[], subsort=False, only_ids=None):
def _do_sort(self, ids_to_sort, fields=(), subsort=False):
fields = [(sanitize_sort_field_name(self.field_metadata, x), bool(y)) for x, y in fields]
keys = self.field_metadata.sortable_field_keys()
fields = [x for x in fields if x[0] in keys]
@ -227,11 +234,16 @@ class View(object):
if not fields:
fields = [('timestamp', False)]
sorted_book_ids = self.cache.multisort(
fields, ids_to_sort=self._map if only_ids is None else only_ids,
return self.cache.multisort(
fields, ids_to_sort=ids_to_sort,
virtual_fields={'marked':MarkedVirtualField(self.marked_ids)})
def multisort(self, fields=[], subsort=False, only_ids=None):
sorted_book_ids = self._do_sort(self._map if only_ids is None else only_ids, fields=fields, subsort=subsort)
if only_ids is None:
self._map = tuple(sorted_book_ids)
self.full_map_is_sorted = True
self.add_to_sort_history(fields)
if len(self._map_filtered) == len(self._map):
self._map_filtered = tuple(self._map)
else:
@ -241,9 +253,16 @@ class View(object):
smap = {book_id:i for i, book_id in enumerate(sorted_book_ids)}
only_ids.sort(key=smap.get)
def search(self, query, return_matches=False):
def incremental_sort(self, fields=(), subsort=False):
if len(self._map) == len(self._map_filtered):
return self.multisort(fields=fields, subsort=subsort)
self._map_filtered = tuple(self._do_sort(self._map_filtered, fields=fields, subsort=subsort))
self.full_map_is_sorted = False
self.add_to_sort_history(fields)
def search(self, query, return_matches=False, sort_results=True):
ans = self.search_getting_ids(query, self.search_restriction,
set_restriction_count=True)
set_restriction_count=True, sort_results=sort_results)
if return_matches:
return ans
self._map_filtered = tuple(ans)
@ -258,7 +277,7 @@ class View(object):
return restriction
def search_getting_ids(self, query, search_restriction,
set_restriction_count=False, use_virtual_library=True):
set_restriction_count=False, use_virtual_library=True, sort_results=True):
if use_virtual_library:
search_restriction = self._build_restriction_string(search_restriction)
q = ''
@ -271,10 +290,28 @@ class View(object):
if not q:
if set_restriction_count:
self.search_restriction_book_count = len(self._map)
return list(self._map)
rv = list(self._map)
if sort_results and not self.full_map_is_sorted:
rv = self._do_sort(rv, fields=self.sort_history)
self._map = tuple(rv)
self.full_map_is_sorted = True
return rv
matches = self.cache.search(
query, search_restriction, virtual_fields={'marked':MarkedVirtualField(self.marked_ids)})
if len(matches) == len(self._map):
rv = list(self._map)
else:
rv = [x for x in self._map if x in matches]
if sort_results and not self.full_map_is_sorted:
# We need to sort the search results
if matches.issubset(frozenset(self._map_filtered)):
rv = [x for x in self._map_filtered if x in matches]
else:
rv = self._do_sort(rv, fields=self.sort_history)
if len(matches) == len(self._map):
# We have sorted all ids, update self._map
self._map = tuple(rv)
self.full_map_is_sorted = True
if set_restriction_count and q == search_restriction:
self.search_restriction_book_count = len(rv)
return rv
@ -338,6 +375,8 @@ class View(object):
def refresh(self, field=None, ascending=True, clear_caches=True, do_search=True):
self._map = tuple(sorted(self.cache.all_book_ids()))
self._map_filtered = tuple(self._map)
self.full_map_is_sorted = True
self.sort_history = [('id', True)]
if clear_caches:
self.cache.clear_caches()
if field is not None:

2
src/calibre/devices/kindle/driver.py
View File

@ -270,7 +270,7 @@ class KINDLE(USBMS):
elif bm.type == 'kindle_clippings':
# Find 'My Clippings' author=Kindle in database, or add
last_update = 'Last modified %s' % strftime(u'%x %X',bm.value['timestamp'].timetuple())
mc_id = list(db.data.search_getting_ids('title:"My Clippings"', ''))
mc_id = list(db.data.search_getting_ids('title:"My Clippings"', '', sort_results=False))
if mc_id:
db.add_format_with_hooks(mc_id[0], 'TXT', bm.value['path'],
index_is_id=True)

2
src/calibre/gui2/library/models.py
View File

@ -407,7 +407,7 @@ class BooksModel(QAbstractTableModel): # {{{
def _sort(self, label, order, reset):
self.about_to_be_sorted.emit(self.db.id)
self.db.sort(label, order)
self.db.data.incremental_sort([(label, order)])
if reset:
self.reset()
self.sorted_on = (label, order)

2
src/calibre/gui2/search_restriction_mixin.py
View File

@ -285,7 +285,7 @@ class CreateVirtualLibrary(QDialog): # {{{
try:
db = self.gui.library_view.model().db
recs = db.data.search_getting_ids('', v, use_virtual_library=False)
recs = db.data.search_getting_ids('', v, use_virtual_library=False, sort_results=False)
except ParseException as e:
error_dialog(self.gui, _('Invalid search'),
_('The search in the search box is not valid'),

2
src/calibre/gui2/tag_browser/model.py
View File

@ -846,7 +846,7 @@ class TagsModel(QAbstractItemModel): # {{{
try:
data = self.db.get_categories(sort=sort,
icon_map=self.category_icon_map,
ids=self.db.search('', return_matches=True))
ids=self.db.search('', return_matches=True, sort_results=False))
except:
data = self.db.get_categories(sort=sort, icon_map=self.category_icon_map)
self.restriction_error.emit()

6
src/calibre/library/caches.py
View File

@ -816,9 +816,9 @@ class ResultCache(SearchQueryParser): # {{{
current_candidates -= matches
return matches
def search(self, query, return_matches=False):
def search(self, query, return_matches=False, sort_results=True):
ans = self.search_getting_ids(query, self.search_restriction,
set_restriction_count=True)
set_restriction_count=True, sort_results=sort_results)
if return_matches:
return ans
self._map_filtered = ans
@ -833,7 +833,7 @@ class ResultCache(SearchQueryParser): # {{{
return restriction
def search_getting_ids(self, query, search_restriction,
set_restriction_count=False, use_virtual_library=True):
set_restriction_count=False, use_virtual_library=True, sort_results=True):
if use_virtual_library:
search_restriction = self._build_restriction_string(search_restriction)
q = ''

2
src/calibre/library/catalogs/epub_mobi.py
View File

@ -447,7 +447,7 @@ class EPUB_MOBI(CatalogPlugin):
try:
search_text = 'title:"%s" author:%s' % (
opts.catalog_title.replace('"', '\\"'), 'calibre')
matches = db.search(search_text, return_matches=True)
matches = db.search(search_text, return_matches=True, sort_results=False)
if matches:
cpath = db.cover(matches[0], index_is_id=True, as_path=True)
if cpath and os.path.exists(cpath):

2
src/calibre/library/server/ajax.py
View File

@ -594,7 +594,7 @@ class AjaxServer(object):
if isbytestring(query):
query = query.decode('UTF-8')
ids = self.db.search_getting_ids(query.strip(), self.search_restriction)
ids = self.db.search_getting_ids(query.strip(), self.search_restriction, sort_results=False)
ids = list(ids)
self.db.data.multisort(fields=[(sfield, sort_order == 'asc')], subsort=True,
only_ids=ids)

2
src/calibre/library/server/browse.py
View File

@ -922,7 +922,7 @@ class BrowseServer(object):
import random
try:
book_id = random.choice(self.db.search_getting_ids(
'', self.search_restriction))
'', self.search_restriction, sort_results=False))
except IndexError:
raise cherrypy.HTTPError(404, 'This library has no books')
ans = self.browse_render_details(book_id, add_random_button=True)

2
src/calibre/library/server/cache.py
View File

@ -21,7 +21,7 @@ class Cache(object):
def search_cache(self, search):
old = self._search_cache.pop(search, None)
if old is None or old[0] <= self.db.last_modified():
matches = self.db.data.search_getting_ids(search, self.search_restriction)
matches = self.db.data.search_getting_ids(search, self.search_restriction, sort_results=False)
if not matches:
matches = []
self._search_cache[search] = (utcnow(), frozenset(matches))

2
src/calibre/library/server/mobile.py
View File

@ -222,7 +222,7 @@ class MobileServer(object):
search = ''
if isbytestring(search):
search = search.decode('UTF-8')
ids = self.db.search_getting_ids(search.strip(), self.search_restriction)
ids = self.db.search_getting_ids(search.strip(), self.search_restriction, sort_results=False)
FM = self.db.FIELD_MAP
items = [r for r in iter(self.db) if r[FM['id']] in ids]
if sort is not None:

2
src/calibre/library/server/xml.py
View File

@ -53,7 +53,7 @@ class XMLServer(object):
if isbytestring(search):
search = search.decode('UTF-8')
ids = self.db.search_getting_ids(search.strip(), self.search_restriction)
ids = self.db.search_getting_ids(search.strip(), self.search_restriction, sort_results=False)
FM = self.db.FIELD_MAP