Speed up sorting when book list is showing a subset

Speed up sorting when the book list is showing a restricted set of
books, such as when the results of a search are displayed or a virtual
library is used. Fixes #1217622 [calculated column sorts update the entire database while inside a Virtual Library](https://bugs.launchpad.net/calibre/+bug/1217622)

src/calibre/db/view.py

@@ -7,13 +7,14 @@ __license__   = 'GPL v3'
 __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import weakref
+import weakref, operator
 from functools import partial
 from itertools import izip, imap
 from future_builtins import map
 
 from calibre.ebooks.metadata import title_sort
 from calibre.utils.config_base import tweaks
+from calibre.db.write import uniq
 
 def sanitize_sort_field_name(field_metadata, field):
     field = field_metadata.search_term_to_field_key(field.lower().strip())
@@ -123,6 +124,12 @@ class View(object):
 
         self._map = tuple(sorted(self.cache.all_book_ids()))
         self._map_filtered = tuple(self._map)
+        self.full_map_is_sorted = True
+        self.sort_history = [('id', True)]
+
+    def add_to_sort_history(self, items):
+        self.sort_history = uniq((list(items) + list(self.sort_history)),
+                                 operator.itemgetter(0))[:tweaks['maximum_resort_levels']]
 
     def count(self):
         return len(self._map)
@@ -218,7 +225,7 @@ class View(object):
             ans = [':::'.join((adata[aid]['name'], adata[aid]['sort'], adata[aid]['link'])) for aid in ids if aid in adata]
         return ':#:'.join(ans) if ans else default_value
 
-    def multisort(self, fields=[], subsort=False, only_ids=None):
+    def _do_sort(self, ids_to_sort, fields=(), subsort=False):
         fields = [(sanitize_sort_field_name(self.field_metadata, x), bool(y)) for x, y in fields]
         keys = self.field_metadata.sortable_field_keys()
         fields = [x for x in fields if x[0] in keys]
@@ -227,11 +234,16 @@ class View(object):
         if not fields:
             fields = [('timestamp', False)]
 
-        sorted_book_ids = self.cache.multisort(
+        return self.cache.multisort(
-            fields, ids_to_sort=self._map if only_ids is None else only_ids,
+            fields, ids_to_sort=ids_to_sort,
             virtual_fields={'marked':MarkedVirtualField(self.marked_ids)})
+
+    def multisort(self, fields=[], subsort=False, only_ids=None):
+        sorted_book_ids = self._do_sort(self._map if only_ids is None else only_ids, fields=fields, subsort=subsort)
         if only_ids is None:
             self._map = tuple(sorted_book_ids)
+            self.full_map_is_sorted = True
+            self.add_to_sort_history(fields)
             if len(self._map_filtered) == len(self._map):
                 self._map_filtered = tuple(self._map)
             else:
@@ -241,9 +253,16 @@ class View(object):
             smap = {book_id:i for i, book_id in enumerate(sorted_book_ids)}
             only_ids.sort(key=smap.get)
 
-    def search(self, query, return_matches=False):
+    def incremental_sort(self, fields=(), subsort=False):
+        if len(self._map) == len(self._map_filtered):
+            return self.multisort(fields=fields, subsort=subsort)
+        self._map_filtered = tuple(self._do_sort(self._map_filtered, fields=fields, subsort=subsort))
+        self.full_map_is_sorted = False
+        self.add_to_sort_history(fields)
+
+    def search(self, query, return_matches=False, sort_results=True):
         ans = self.search_getting_ids(query, self.search_restriction,
-                                      set_restriction_count=True)
+                                      set_restriction_count=True, sort_results=sort_results)
         if return_matches:
             return ans
         self._map_filtered = tuple(ans)
@@ -258,7 +277,7 @@ class View(object):
             return restriction
 
     def search_getting_ids(self, query, search_restriction,
-                           set_restriction_count=False, use_virtual_library=True):
+                           set_restriction_count=False, use_virtual_library=True, sort_results=True):
         if use_virtual_library:
             search_restriction = self._build_restriction_string(search_restriction)
         q = ''
@@ -271,10 +290,28 @@ class View(object):
         if not q:
             if set_restriction_count:
                 self.search_restriction_book_count = len(self._map)
-            return list(self._map)
+            rv = list(self._map)
+            if sort_results and not self.full_map_is_sorted:
+                rv = self._do_sort(rv, fields=self.sort_history)
+                self._map = tuple(rv)
+                self.full_map_is_sorted = True
+            return rv
         matches = self.cache.search(
             query, search_restriction, virtual_fields={'marked':MarkedVirtualField(self.marked_ids)})
-        rv = [x for x in self._map if x in matches]
+        if len(matches) == len(self._map):
+            rv = list(self._map)
+        else:
+            rv = [x for x in self._map if x in matches]
+        if sort_results and not self.full_map_is_sorted:
+            # We need to sort the search results
+            if matches.issubset(frozenset(self._map_filtered)):
+                rv = [x for x in self._map_filtered if x in matches]
+            else:
+                rv = self._do_sort(rv, fields=self.sort_history)
+            if len(matches) == len(self._map):
+                # We have sorted all ids, update self._map
+                self._map = tuple(rv)
+                self.full_map_is_sorted = True
         if set_restriction_count and q == search_restriction:
             self.search_restriction_book_count = len(rv)
         return rv
@@ -338,6 +375,8 @@ class View(object):
     def refresh(self, field=None, ascending=True, clear_caches=True, do_search=True):
         self._map = tuple(sorted(self.cache.all_book_ids()))
         self._map_filtered = tuple(self._map)
+        self.full_map_is_sorted = True
+        self.sort_history = [('id', True)]
         if clear_caches:
             self.cache.clear_caches()
         if field is not None:

src/calibre/devices/kindle/driver.py

@@ -270,7 +270,7 @@ class KINDLE(USBMS):
         elif bm.type == 'kindle_clippings':
             # Find 'My Clippings' author=Kindle in database, or add
             last_update = 'Last modified %s' % strftime(u'%x %X',bm.value['timestamp'].timetuple())
-            mc_id = list(db.data.search_getting_ids('title:"My Clippings"', ''))
+            mc_id = list(db.data.search_getting_ids('title:"My Clippings"', '', sort_results=False))
             if mc_id:
                 db.add_format_with_hooks(mc_id[0], 'TXT', bm.value['path'],
                         index_is_id=True)

src/calibre/gui2/library/models.py

@@ -407,7 +407,7 @@ class BooksModel(QAbstractTableModel):  # {{{
 
     def _sort(self, label, order, reset):
         self.about_to_be_sorted.emit(self.db.id)
-        self.db.sort(label, order)
+        self.db.data.incremental_sort([(label, order)])
         if reset:
             self.reset()
         self.sorted_on = (label, order)

src/calibre/gui2/search_restriction_mixin.py

@@ -285,7 +285,7 @@ class CreateVirtualLibrary(QDialog):  # {{{
 
         try:
             db = self.gui.library_view.model().db
-            recs = db.data.search_getting_ids('', v, use_virtual_library=False)
+            recs = db.data.search_getting_ids('', v, use_virtual_library=False, sort_results=False)
         except ParseException as e:
             error_dialog(self.gui, _('Invalid search'),
                          _('The search in the search box is not valid'),

src/calibre/gui2/tag_browser/model.py

@@ -846,7 +846,7 @@ class TagsModel(QAbstractItemModel):  # {{{
             try:
                 data = self.db.get_categories(sort=sort,
                         icon_map=self.category_icon_map,
-                        ids=self.db.search('', return_matches=True))
+                        ids=self.db.search('', return_matches=True, sort_results=False))
             except:
                 data = self.db.get_categories(sort=sort, icon_map=self.category_icon_map)
                 self.restriction_error.emit()

src/calibre/library/caches.py

@@ -816,9 +816,9 @@ class ResultCache(SearchQueryParser):  # {{{
                 current_candidates -= matches
         return matches
 
-    def search(self, query, return_matches=False):
+    def search(self, query, return_matches=False, sort_results=True):
         ans = self.search_getting_ids(query, self.search_restriction,
-                                      set_restriction_count=True)
+                                      set_restriction_count=True, sort_results=sort_results)
         if return_matches:
             return ans
         self._map_filtered = ans
@@ -833,7 +833,7 @@ class ResultCache(SearchQueryParser):  # {{{
             return restriction
 
     def search_getting_ids(self, query, search_restriction,
-                           set_restriction_count=False, use_virtual_library=True):
+                           set_restriction_count=False, use_virtual_library=True, sort_results=True):
         if use_virtual_library:
             search_restriction = self._build_restriction_string(search_restriction)
         q = ''

src/calibre/library/catalogs/epub_mobi.py

@@ -447,7 +447,7 @@ class EPUB_MOBI(CatalogPlugin):
             try:
                 search_text = 'title:"%s" author:%s' % (
                         opts.catalog_title.replace('"', '\\"'), 'calibre')
-                matches = db.search(search_text, return_matches=True)
+                matches = db.search(search_text, return_matches=True, sort_results=False)
                 if matches:
                     cpath = db.cover(matches[0], index_is_id=True, as_path=True)
                     if cpath and os.path.exists(cpath):

src/calibre/library/server/ajax.py

@@ -594,7 +594,7 @@ class AjaxServer(object):
 
         if isbytestring(query):
             query = query.decode('UTF-8')
-        ids = self.db.search_getting_ids(query.strip(), self.search_restriction)
+        ids = self.db.search_getting_ids(query.strip(), self.search_restriction, sort_results=False)
         ids = list(ids)
         self.db.data.multisort(fields=[(sfield, sort_order == 'asc')], subsort=True,
                 only_ids=ids)

src/calibre/library/server/browse.py

@@ -922,7 +922,7 @@ class BrowseServer(object):
         import random
         try:
             book_id = random.choice(self.db.search_getting_ids(
-                '', self.search_restriction))
+                '', self.search_restriction, sort_results=False))
         except IndexError:
             raise cherrypy.HTTPError(404, 'This library has no books')
         ans = self.browse_render_details(book_id, add_random_button=True)

src/calibre/library/server/cache.py

@@ -21,7 +21,7 @@ class Cache(object):
     def search_cache(self, search):
         old = self._search_cache.pop(search, None)
         if old is None or old[0] <= self.db.last_modified():
-            matches = self.db.data.search_getting_ids(search, self.search_restriction)
+            matches = self.db.data.search_getting_ids(search, self.search_restriction, sort_results=False)
             if not matches:
                 matches = []
             self._search_cache[search] = (utcnow(), frozenset(matches))

src/calibre/library/server/mobile.py

@@ -222,7 +222,7 @@ class MobileServer(object):
             search = ''
         if isbytestring(search):
             search = search.decode('UTF-8')
-        ids = self.db.search_getting_ids(search.strip(), self.search_restriction)
+        ids = self.db.search_getting_ids(search.strip(), self.search_restriction, sort_results=False)
         FM = self.db.FIELD_MAP
         items = [r for r in iter(self.db) if r[FM['id']] in ids]
         if sort is not None:

src/calibre/library/server/xml.py

@@ -53,7 +53,7 @@ class XMLServer(object):
         if isbytestring(search):
             search = search.decode('UTF-8')
 
-        ids = self.db.search_getting_ids(search.strip(), self.search_restriction)
+        ids = self.db.search_getting_ids(search.strip(), self.search_restriction, sort_results=False)
 
         FM = self.db.FIELD_MAP