From 6eaa75527b5754cfbb8df833ad3375b724d51cfd Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sat, 11 Sep 2010 21:01:26 +0100
Subject: [PATCH 1/8] resort maximum_resort_levels tweak implemented

---
 resources/default_tweaks.py        | 7 +++++++
 src/calibre/gui2/library/models.py | 4 ++--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/resources/default_tweaks.py b/resources/default_tweaks.py
index 66ee4d1471..9d9bc7651c 100644
--- a/resources/default_tweaks.py
+++ b/resources/default_tweaks.py
@@ -114,3 +114,10 @@ add_new_book_tags_when_importing_books = False
 # Set the maximum number of tags to show per book in the content server
 max_content_server_tags_shown=5
 
+
+# Set the maximum number of sort 'levels' that calibre will use to resort the
+# library after certain operations such as searches or device insertion. Each
+# sort level adds a performance penalty. If the database is large (thousands of
+# books) the penalty might be noticeable. If you are not concerned about multi-
+# level sorts, and if you are seeing a slowdown, reduce the value of this tweak.
+maximum_resort_levels = 5
\ No newline at end of file
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index 8ad0cd6818..d2f38cc0a1 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -266,8 +266,8 @@ class BooksModel(QAbstractTableModel): # {{{
         self.db.refresh(field=None)
         self.resort(reset=reset)
 
-    def resort(self, reset=True, history=5): # Bug report needed history=4 :)
-        for col,ord in reversed(self.sort_history[:history]):
+    def resort(self, reset=True):
+        for col,ord in reversed(self.sort_history[:tweaks['maximum_resort_levels']]):
             try:
                 col = self.column_map.index(col)
             except ValueError:

From 721e61ef2a1fd090566e232ff9ca65e37400fe44 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sat, 11 Sep 2010 21:05:05 +0100
Subject: [PATCH 2/8] Clean up tweaks.py formatting (add blank lines)

---
 resources/default_tweaks.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/resources/default_tweaks.py b/resources/default_tweaks.py
index 9d9bc7651c..71bf2c6c37 100644
--- a/resources/default_tweaks.py
+++ b/resources/default_tweaks.py
@@ -120,4 +120,5 @@ max_content_server_tags_shown=5
 # sort level adds a performance penalty. If the database is large (thousands of
 # books) the penalty might be noticeable. If you are not concerned about multi-
 # level sorts, and if you are seeing a slowdown, reduce the value of this tweak.
-maximum_resort_levels = 5
\ No newline at end of file
+maximum_resort_levels = 5
+

From 7382552d18d604dff3b5472195fa9f3c07b0186c Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Sat, 11 Sep 2010 19:11:30 -0600
Subject: [PATCH 3/8] Much faster sorting code

---
 src/calibre/library/caches.py | 178 ++++++++++++++++++++++++++++++++--
 1 file changed, 171 insertions(+), 7 deletions(-)

diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index eb0ceb3fe4..59d5b45d5f 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -607,16 +607,22 @@ class ResultCache(SearchQueryParser):
                 y = UNDEFINED_DATE
             return cmp(x, y)
         if subsort and ans == 0:
-            return cmp(self._data[x][11].lower(), self._data[y][11].lower())
+            idx = self.FIELD_MAP['sort']
+            return cmp(self._data[x][idx].lower(), self._data[y][idx].lower())
         return ans
 
-    def sort(self, field, ascending, subsort=False):
+    def sanitize_field_name(self, field):
         field = field.lower().strip()
-        if field in ('author', 'tag', 'comment'):
-            field += 's'
-        if   field == 'date': field = 'timestamp'
-        elif field == 'title': field = 'sort'
-        elif field == 'authors': field = 'author_sort'
+        if field not in self.field_metadata.iterkeys():
+            if field in ('author', 'tag', 'comment'):
+                field += 's'
+            if   field == 'date': field = 'timestamp'
+            elif field == 'title': field = 'sort'
+            elif field == 'authors': field = 'author_sort'
+        return field
+
+    def sort(self, field, ascending, subsort=False):
+        field = self.sanitize_field_name(field)
         as_string = field not in ('size', 'rating', 'timestamp')
 
         if self.first_sort:
@@ -643,6 +649,164 @@ class ResultCache(SearchQueryParser):
         self._map.sort(cmp=fcmp, reverse=not ascending)
         self._map_filtered = [id for id in self._map if id in self._map_filtered]
 
+    def multisort(self, fields=[], subsort=False):
+        fields = [(self.sanitize_field_name(x), bool(y)) for x, y in fields]
+        if subsort and 'sort' not in [x[0] for x in fields]:
+            fields += [('sort', True)]
+        if not fields:
+            fields = [('timestamp', False)]
+        keys = self.field_metadata.keys()
+        for f, order in fields:
+            if f not in keys:
+                raise ValueError(f + ' not an existing field name')
+
+        keyg = SortKeyGenerator(fields, self.field_metadata, self._data)
+        if len(fields) == 1:
+            self._map.sort(key=keyg, reverse=not fields[0][1])
+        else:
+            self._map.sort(key=keyg)
+        self._map_filtered = [id for id in self._map if id in self._map_filtered]
+
+
+class SortKey(object):
+
+    def __init__(self, orders, values):
+        self.orders, self.values = orders, values
+
+    def __cmp__(self, other):
+        for i, ascending in enumerate(self.orders):
+            ans = cmp(self.values[i], other.values[i])
+            if ans != 0:
+                if not ascending:
+                    ans *= -1
+                return ans
+        return 0
+
+class SortKeyGenerator(object):
+
+    def __init__(self, fields, field_metadata, data):
+        self.field_metadata = field_metadata
+        self.orders = [x[1] for x in fields]
+        self.entries = [(x[0], field_metadata[x[0]]) for x in fields]
+        self.library_order = tweaks['title_series_sorting'] == 'library_order'
+        self.data = data
+
+    def __call__(self, record):
+        values = tuple(self.itervals(self.data[record]))
+        if len(values) == 1:
+            return values[0]
+        return SortKey(self.orders, values)
+
+    def itervals(self, record):
+        for name, fm in self.entries:
+            dt = fm['datatype']
+            val = record[fm['rec_index']]
+
+            if dt == 'datetime':
+                if val is None:
+                    val = UNDEFINED_DATE
+
+            elif dt == 'series':
+                if val is None:
+                    val = ('', 1)
+                else:
+                    val = val.lower()
+                    if self.library_order:
+                        val = title_sort(val)
+                    sidx_fm = self.field_metadata[name + '_index']
+                    sidx = record[sidx_fm['rec_index']]
+                    val = (val, sidx)
+
+            elif dt in ('text', 'comments'):
+                if val is None:
+                    val = ''
+                val = val.lower()
+            yield val
+
     # }}}
 
 
+if __name__ == '__main__':
+    # Testing.timing for new multi-sort {{{
+    import time
+
+    from calibre.library import db
+    db = db()
+
+    db.refresh()
+
+    fields = db.field_metadata.keys()
+
+    print fields
+
+
+    def do_single_sort(meth, field, order):
+        if meth == 'old':
+            db.data.sort(field, order)
+        else:
+            db.data.multisort([(field, order)])
+
+    def test_single_sort(field):
+        for meth in ('old', 'new'):
+            ttime = 0
+            NUM = 10
+            asc = desc = None
+            for i in range(NUM):
+                db.data.sort('id', False)
+                st = time.time()
+                do_single_sort(meth, field, True)
+                asc = db.data._map
+                do_single_sort(meth, field, False)
+                desc = db.data._map
+                ttime += time.time() - st
+            yield (ttime/NUM, asc, desc)
+
+
+    print 'Running single sort differentials'
+    for field in fields:
+        if field in ('search', 'id', 'news', 'flags'): continue
+        print '\t', field
+        old, new = test_single_sort(field)
+        if old[1] != new[1] or old[2] != new[2]:
+            print '\t\t', 'Sort failure!'
+            raise SystemExit(1)
+        print '\t\t', 'Old:', old[0], 'New:', new[0], 'Ratio: %.2f'%(new[0]/old[0])
+
+    def do_multi_sort(meth, ms):
+        if meth == 'new':
+            db.data.multisort(ms)
+        else:
+            for s in reversed(ms):
+                db.data.sort(*s)
+
+    def test_multi_sort(ms):
+        for meth in ('old', 'new'):
+            ttime = 0
+            NUM = 10
+            for i in range(NUM):
+                db.data.sort('id', False)
+                st = time.time()
+                do_multi_sort(meth, ms)
+                ttime += time.time() - st
+            yield (ttime/NUM, db.data._map)
+
+    print 'Running multi-sort differentials'
+
+    for ms in [
+            [('timestamp', False), ('author', True), ('title', False)],
+            [('size', True), ('tags', True), ('author', False)],
+            [('series', False), ('title', True)],
+            [('size', True), ('tags', True), ('author', False), ('pubdate',
+                True), ('tags', False), ('formats', False), ('uuid', True)],
+
+            ]:
+        print '\t', ms
+        db.data.sort('id', False)
+        old, new = test_multi_sort(ms)
+        if old[1] != new[1]:
+            print '\t\t', 'Sort failure!'
+            raise SystemExit()
+        print '\t\t', 'Old:', old[0], 'New:', new[0], 'Ratio: %.2f'%(new[0]/old[0])
+
+    # }}}
+

From bcd0430791f44ec926910eeb8bb18d7cbbff5fc9 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 12 Sep 2010 13:37:28 +0100
Subject: [PATCH 4/8] Starting from Kovid's multisort: 1) change _map_filtered
 to an ordered dict to make 'in' operations much faster 2) add a method to
 field_metadata to return a dict of database fields. 3) fix a couple of places
 where field_metadata needed to be used. 4) make changes so
 gui2.library.models.resort uses multisort

---
 src/calibre/gui2/library/models.py    | 14 +++----
 src/calibre/library/caches.py         | 59 ++++++++++++++++-----------
 src/calibre/library/database2.py      |  1 +
 src/calibre/library/field_metadata.py |  3 ++
 4 files changed, 45 insertions(+), 32 deletions(-)

diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index d2f38cc0a1..d18516493a 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -247,7 +247,7 @@ class BooksModel(QAbstractTableModel): # {{{
             # the search and count records for restrictions
             self.searched.emit(True)
 
-    def sort(self, col, order, reset=True, update_history=True):
+    def sort(self, col, order, reset=True):
         if not self.db:
             return
         self.about_to_be_sorted.emit(self.db.id)
@@ -258,8 +258,7 @@ class BooksModel(QAbstractTableModel): # {{{
             self.clear_caches()
             self.reset()
         self.sorted_on = (label, order)
-        if update_history:
-            self.sort_history.insert(0, self.sorted_on)
+        self.sort_history.insert(0, self.sorted_on)
         self.sorting_done.emit(self.db.index)
 
     def refresh(self, reset=True):
@@ -267,12 +266,9 @@ class BooksModel(QAbstractTableModel): # {{{
         self.resort(reset=reset)
 
     def resort(self, reset=True):
-        for col,ord in reversed(self.sort_history[:tweaks['maximum_resort_levels']]):
-            try:
-                col = self.column_map.index(col)
-            except ValueError:
-                col = 0
-            self.sort(col, ord, reset=False, update_history=False)
+        if not self.db:
+            return
+        self.db.multisort(self.sort_history[:tweaks['maximum_resort_levels']])
         if reset:
             self.reset()
 
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index 59d5b45d5f..c342d5ff15 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -20,6 +20,7 @@ from calibre.utils.search_query_parser import SearchQueryParser
 from calibre.utils.pyparsing import ParseException
 from calibre.ebooks.metadata import title_sort
 from calibre import fit_image
+from calibre.utils.ordered_dict import OrderedDict
 
 class CoverCache(Thread):
 
@@ -112,7 +113,8 @@ class ResultCache(SearchQueryParser):
     '''
     def __init__(self, FIELD_MAP, field_metadata):
         self.FIELD_MAP = FIELD_MAP
-        self._map = self._map_filtered = self._data = []
+        self._map = self._data = []
+        self._map_filtered = OrderedDict()
         self.first_sort = True
         self.search_restriction = ''
         self.field_metadata = field_metadata
@@ -122,14 +124,14 @@ class ResultCache(SearchQueryParser):
         self.build_numeric_relop_dict()
 
     def __getitem__(self, row):
-        return self._data[self._map_filtered[row]]
+        return self._data[self._map_filtered.keys()[row]]
 
     def __len__(self):
         return len(self._map_filtered)
 
     def __iter__(self):
         for id in self._map_filtered:
-            yield self._data[id]
+            yield id
 
     def iterall(self):
         for x in self._data:
@@ -468,7 +470,7 @@ class ResultCache(SearchQueryParser):
         ans = self.search_getting_ids(query, self.search_restriction)
         if return_matches:
             return ans
-        self._map_filtered = ans
+        self._map_filtered = OrderedDict.fromkeys(ans, True)
 
     def search_getting_ids(self, query, search_restriction):
         q = ''
@@ -480,7 +482,7 @@ class ResultCache(SearchQueryParser):
                 q = u'%s (%s)' % (search_restriction, query)
         if not q:
             return list(self._map)
-        matches = sorted(self.parse(q))
+        matches = self.parse(q)
         return [id for id in self._map if id in matches]
 
     def set_search_restriction(self, s):
@@ -493,18 +495,18 @@ class ResultCache(SearchQueryParser):
         if id in self._map:
             self._map.remove(id)
         if id in self._map_filtered:
-            self._map_filtered.remove(id)
+            del self._map_filtered[id]
 
     def set(self, row, col, val, row_is_id=False):
-        id = row if row_is_id else self._map_filtered[row]
+        id = row if row_is_id else self._map_filtered.keys()[row]
         self._data[id][col] = val
 
     def get(self, row, col, row_is_id=False):
-        id = row if row_is_id else self._map_filtered[row]
+        id = row if row_is_id else self._map_filtered.keys()[row]
         return self._data[id][col]
 
     def index(self, id, cache=False):
-        x = self._map if cache else self._map_filtered
+        x = self._map if cache else self._map_filtered.keys()
         return x.index(id)
 
     def row(self, id):
@@ -544,13 +546,18 @@ class ResultCache(SearchQueryParser):
             self._data[id].append(db.has_cover(id, index_is_id=True))
             self._data[id].append(db.book_on_device_string(id))
         self._map[0:0] = ids
-        self._map_filtered[0:0] = ids
+        mf = OrderedDict()
+        for id in ids:
+            mf[id] = True
+        for id in self._map_filtered:
+            mf[id] = True
+        self._map_filtered = mf
 
     def books_deleted(self, ids):
         for id in ids:
             self._data[id] = None
             if id in self._map: self._map.remove(id)
-            if id in self._map_filtered: self._map_filtered.remove(id)
+            if id in self._map_filtered: del self._map_filtered[id]
 
     def count(self):
         return len(self._map)
@@ -573,7 +580,7 @@ class ResultCache(SearchQueryParser):
         self._map = [i[0] for i in self._data if i is not None]
         if field is not None:
             self.sort(field, ascending)
-        self._map_filtered = list(self._map)
+        self._map_filtered = OrderedDict.fromkeys(self._map, True)
         if self.search_restriction:
             self.search('', return_matches=False)
 
@@ -644,10 +651,14 @@ class ResultCache(SearchQueryParser):
                 self.FIELD_MAP['series_index'],
                 library_order=tweaks['title_series_sorting'] == 'library_order')
         else:
-            fcmp = functools.partial(self.cmp, self.FIELD_MAP[field],
+            fcmp = functools.partial(self.cmp, self.field_metadata[field]['rec_index'],
                                      subsort=subsort, asstr=as_string)
         self._map.sort(cmp=fcmp, reverse=not ascending)
-        self._map_filtered = [id for id in self._map if id in self._map_filtered]
+        mf = OrderedDict()
+        for id in self._map:
+            if id in self._map_filtered:
+                mf[id] = True
+        self._map_filtered = mf
 
     def multisort(self, fields=[], subsort=False):
         fields = [(self.sanitize_field_name(x), bool(y)) for x, y in fields]
@@ -655,7 +666,7 @@ class ResultCache(SearchQueryParser):
             fields += [('sort', True)]
         if not fields:
             fields = [('timestamp', False)]
-        keys = self.field_metadata.keys()
+        keys = self.field_metadata.field_keys()
         for f, order in fields:
             if f not in keys:
                 raise ValueError(f + ' not an existing field name')
@@ -665,7 +676,11 @@ class ResultCache(SearchQueryParser):
             self._map.sort(key=keyg, reverse=not fields[0][1])
         else:
             self._map.sort(key=keyg)
-        self._map_filtered = [id for id in self._map if id in self._map_filtered]
+        mf = OrderedDict()
+        for id in self._map:
+            if id in self._map_filtered:
+                mf[id] = id
+        self._map_filtered = mf
 
 
 class SortKey(object):
@@ -677,16 +692,14 @@ class SortKey(object):
         for i, ascending in enumerate(self.orders):
             ans = cmp(self.values[i], other.values[i])
             if ans != 0:
-                if not ascending:
-                    ans *= -1
-                return ans
+                return ans * ascending
         return 0
 
 class SortKeyGenerator(object):
 
     def __init__(self, fields, field_metadata, data):
         self.field_metadata = field_metadata
-        self.orders = [x[1] for x in fields]
+        self.orders = [-1 if x[1] else 1 for x in fields]
         self.entries = [(x[0], field_metadata[x[0]]) for x in fields]
         self.library_order = tweaks['title_series_sorting'] == 'library_order'
         self.data = data
@@ -735,7 +748,7 @@ if __name__ == '__main__':
 
     db.refresh()
 
-    fields = db.field_metadata.keys()
+    fields = db.field_metadata.field_keys()
 
     print fields
 
@@ -765,7 +778,7 @@ if __name__ == '__main__':
     print 'Running single sort differentials'
     for field in fields:
         if field in ('search', 'id', 'news', 'flags'): continue
-        print '\t', field
+        print '\t', field, db.field_metadata[field]['datatype']
         old, new = test_single_sort(field)
         if old[1] != new[1] or old[2] != new[2]:
             print '\t\t', 'Sort failure!'
@@ -797,7 +810,7 @@ if __name__ == '__main__':
             [('size', True), ('tags', True), ('author', False)],
             [('series', False), ('title', True)],
             [('size', True), ('tags', True), ('author', False), ('pubdate',
-                True), ('tags', False), ('formats', False), ('uuid', True)],
+                True), ('series', False), ('formats', False), ('uuid', True)],
 
             ]:
         print '\t', ms
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 4106f8c965..8a5ab75c3c 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -311,6 +311,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         self.search_getting_ids  = self.data.search_getting_ids
         self.refresh = functools.partial(self.data.refresh, self)
         self.sort    = self.data.sort
+        self.multisort = self.data.multisort
         self.index   = self.data.index
         self.refresh_ids = functools.partial(self.data.refresh_ids, self)
         self.row     = self.data.row
diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py
index 096dfa66fe..276a6ba971 100644
--- a/src/calibre/library/field_metadata.py
+++ b/src/calibre/library/field_metadata.py
@@ -335,6 +335,9 @@ class FieldMetadata(dict):
     def keys(self):
         return self._tb_cats.keys()
 
+    def field_keys(self):
+        return [k for k in self._tb_cats.keys() if self._tb_cats[k]['kind']=='field']
+
     def iterkeys(self):
         for key in self._tb_cats:
             yield key

From 8b09f4c293e82ff797635320c42487d9be190831 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 12 Sep 2010 13:42:37 +0100
Subject: [PATCH 5/8] Restore the second 'tags' to the tests

---
 src/calibre/library/caches.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index c342d5ff15..882de975db 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -810,7 +810,7 @@ if __name__ == '__main__':
             [('size', True), ('tags', True), ('author', False)],
             [('series', False), ('title', True)],
             [('size', True), ('tags', True), ('author', False), ('pubdate',
-                True), ('series', False), ('formats', False), ('uuid', True)],
+                True), ('tags', False), ('formats', False), ('uuid', True)],
 
             ]:
         print '\t', ms

From 5626418d1a6993b16f3d6a83c22a761a7490b7ee Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 12 Sep 2010 14:51:21 +0100
Subject: [PATCH 6/8] Correct regression in device handing -- sorting after
 sending a book.

---
 src/calibre/gui2/library/models.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index d18516493a..c746a5aa56 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -1024,6 +1024,11 @@ class DeviceBooksModel(BooksModel): # {{{
         if reset:
             self.reset()
 
+    def resort(self, reset=True):
+        if self.sorted_on:
+            self.sort(self.column_map.index(self.sorted_on[0]),
+                      self.sorted_on[1], reset=reset)
+
     def columnCount(self, parent):
         if parent and parent.isValid():
             return 0

From 6cc332089a421e6100fa4937c5126309c483e132 Mon Sep 17 00:00:00 2001
From: Starson17 <starson17@gmail.com>
Date: Sun, 12 Sep 2010 11:28:24 -0400
Subject: [PATCH 7/8] Change Merge and Safe Merge warnings re ISBN

---
 src/calibre/gui2/actions/edit_metadata.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/calibre/gui2/actions/edit_metadata.py b/src/calibre/gui2/actions/edit_metadata.py
index f0232d9859..878ba77a43 100644
--- a/src/calibre/gui2/actions/edit_metadata.py
+++ b/src/calibre/gui2/actions/edit_metadata.py
@@ -209,8 +209,9 @@ class EditMetadataAction(InterfaceAction):
         dest_id, src_books, src_ids = self.books_to_merge(rows)
         if safe_merge:
             if not confirm('<p>'+_(
-                'All book formats and metadata from the selected books '
-                'will be added to the <b>first selected book.</b><br><br> '
+                'Book formats and metadata from the selected books '
+                'will be added to the <b>first selected book.</b> '
+                'ISBN will <i>not</i> be merged.<br><br> '
                 'The second and subsequently selected books will not '
                 'be deleted or changed.<br><br>'
                 'Please confirm you want to proceed.')
@@ -220,8 +221,9 @@ class EditMetadataAction(InterfaceAction):
             self.merge_metadata(dest_id, src_ids)
         else:
             if not confirm('<p>'+_(
-                'All book formats and metadata from the selected books will be merged '
-                'into the <b>first selected book</b>.<br><br>'
+                'Book formats and metadata from the selected books will be merged '
+                'into the <b>first selected book</b>. '
+                'ISBN will <i>not</i> be merged.<br><br>'
                 'After merger the second and '
                 'subsequently selected books will be <b>deleted</b>. <br><br>'
                 'All book formats of the first selected book will be kept '

From 78874a9117941de749f3b09934be8588181dd4b7 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Sun, 12 Sep 2010 09:32:16 -0600
Subject: [PATCH 8/8] Use the new sorting code in the content server as well.

---
 src/calibre/library/caches.py         | 153 +-------------------------
 src/calibre/library/server/content.py |  38 +++----
 2 files changed, 18 insertions(+), 173 deletions(-)

diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index dfd7086076..4f795ab733 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -6,7 +6,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import re, itertools, functools
+import re, itertools
 from itertools import repeat
 from datetime import timedelta
 from threading import Thread, RLock
@@ -584,39 +584,7 @@ class ResultCache(SearchQueryParser):
 
     # Sorting functions {{{
 
-    def seriescmp(self, sidx, siidx, x, y, library_order=None):
-        try:
-            if library_order:
-                ans = cmp(title_sort(self._data[x][sidx].lower()),
-                                title_sort(self._data[y][sidx].lower()))
-            else:
-                ans = cmp(self._data[x][sidx].lower(),
-                                                self._data[y][sidx].lower())
-        except AttributeError: # Some entries may be None
-            ans = cmp(self._data[x][sidx], self._data[y][sidx])
-        if ans != 0: return ans
-        return cmp(self._data[x][siidx], self._data[y][siidx])
-
-    def cmp(self, loc, x, y, asstr=True, subsort=False):
-        try:
-            ans = cmp(self._data[x][loc].lower(), self._data[y][loc].lower()) if \
-                asstr else cmp(self._data[x][loc], self._data[y][loc])
-        except AttributeError: # Some entries may be None
-            ans = cmp(self._data[x][loc], self._data[y][loc])
-        except TypeError: ## raised when a datetime is None
-            x = self._data[x][loc]
-            if x is None:
-                x = UNDEFINED_DATE
-            y = self._data[y][loc]
-            if y is None:
-                y = UNDEFINED_DATE
-            return cmp(x, y)
-        if subsort and ans == 0:
-            idx = self.FIELD_MAP['sort']
-            return cmp(self._data[x][idx].lower(), self._data[y][idx].lower())
-        return ans
-
-    def sanitize_field_name(self, field):
+    def sanitize_sort_field_name(self, field):
         field = field.lower().strip()
         if field not in self.field_metadata.iterkeys():
             if field in ('author', 'tag', 'comment'):
@@ -627,38 +595,10 @@ class ResultCache(SearchQueryParser):
         return field
 
     def sort(self, field, ascending, subsort=False):
-        field = self.sanitize_field_name(field)
-        as_string = field not in ('size', 'rating', 'timestamp')
-
-        if self.first_sort:
-            subsort = True
-            self.first_sort = False
-        if self.field_metadata[field]['is_custom']:
-            if self.field_metadata[field]['datatype'] == 'series':
-                fcmp = functools.partial(self.seriescmp,
-                    self.field_metadata[field]['rec_index'],
-                    self.field_metadata.cc_series_index_column_for(field),
-                    library_order=tweaks['title_series_sorting'] == 'library_order')
-            else:
-                as_string = self.field_metadata[field]['datatype'] in ('comments', 'text')
-                field = self.field_metadata[field]['colnum']
-                fcmp = functools.partial(self.cmp, self.FIELD_MAP[field],
-                                     subsort=subsort, asstr=as_string)
-        elif field == 'series':
-            fcmp = functools.partial(self.seriescmp, self.FIELD_MAP['series'],
-                self.FIELD_MAP['series_index'],
-                library_order=tweaks['title_series_sorting'] == 'library_order')
-        else:
-            fcmp = functools.partial(self.cmp, self.field_metadata[field]['rec_index'],
-                                     subsort=subsort, asstr=as_string)
-        self._map.sort(cmp=fcmp, reverse=not ascending)
-        tmap = list(itertools.repeat(False, len(self._data)))
-        for x in self._map_filtered:
-            tmap[x] = True
-        self._map_filtered = [x for x in self._map if tmap[x]]
+        self.multisort([(field, ascending)])
 
     def multisort(self, fields=[], subsort=False):
-        fields = [(self.sanitize_field_name(x), bool(y)) for x, y in fields]
+        fields = [(self.sanitize_sort_field_name(x), bool(y)) for x, y in fields]
         keys = self.field_metadata.field_keys()
         fields = [x for x in fields if x[0] in keys]
         if subsort and 'sort' not in [x[0] for x in fields]:
@@ -671,6 +611,7 @@ class ResultCache(SearchQueryParser):
             self._map.sort(key=keyg, reverse=not fields[0][1])
         else:
             self._map.sort(key=keyg)
+
         tmap = list(itertools.repeat(False, len(self._data)))
         for x in self._map_filtered:
             tmap[x] = True
@@ -733,87 +674,3 @@ class SortKeyGenerator(object):
     # }}}
 
 
-if __name__ == '__main__':
-    # Testing.timing for new multi-sort {{{
-    import time
-
-    from calibre.library import db
-    db = db()
-
-    db.refresh()
-
-    fields = db.field_metadata.field_keys()
-
-    print fields
-
-
-    def do_single_sort(meth, field, order):
-        if meth == 'old':
-            db.data.sort(field, order)
-        else:
-            db.data.multisort([(field, order)])
-
-    def test_single_sort(field):
-        for meth in ('old', 'new'):
-            ttime = 0
-            NUM = 10
-            asc = desc = None
-            for i in range(NUM):
-                db.data.sort('id', False)
-                st = time.time()
-                do_single_sort(meth, field, True)
-                asc = db.data._map
-                do_single_sort(meth, field, False)
-                desc = db.data._map
-                ttime += time.time() - st
-            yield (ttime/NUM, asc, desc)
-
-
-    print 'Running single sort differentials'
-    for field in fields:
-        if field in ('search', 'id', 'news', 'flags'): continue
-        print '\t', field, db.field_metadata[field]['datatype']
-        old, new = test_single_sort(field)
-        if old[1] != new[1] or old[2] != new[2]:
-            print '\t\t', 'Sort failure!'
-            raise SystemExit(1)
-        print '\t\t', 'Old:', old[0], 'New:', new[0], 'Ratio: %.2f'%(new[0]/old[0])
-
-    def do_multi_sort(meth, ms):
-        if meth == 'new':
-            db.data.multisort(ms)
-        else:
-            for s in reversed(ms):
-                db.data.sort(*s)
-
-    def test_multi_sort(ms):
-        for meth in ('old', 'new'):
-            ttime = 0
-            NUM = 10
-            for i in range(NUM):
-                db.data.sort('id', False)
-                st = time.time()
-                do_multi_sort(meth, ms)
-                ttime += time.time() - st
-            yield (ttime/NUM, db.data._map)
-
-    print 'Running multi-sort differentials'
-
-    for ms in [
-            [('timestamp', False), ('author', True), ('title', False)],
-            [('size', True), ('tags', True), ('author', False)],
-            [('series', False), ('title', True)],
-            [('size', True), ('tags', True), ('author', False), ('pubdate',
-                True), ('tags', False), ('formats', False), ('uuid', True)],
-
-            ]:
-        print '\t', ms
-        db.data.sort('id', False)
-        old, new = test_multi_sort(ms)
-        if old[1] != new[1]:
-            print '\t\t', 'Sort failure!'
-            raise SystemExit()
-        print '\t\t', 'Old:', old[0], 'New:', new[0], 'Ratio: %.2f'%(new[0]/old[0])
-
-    # }}}
-
diff --git a/src/calibre/library/server/content.py b/src/calibre/library/server/content.py
index 6784abd8f4..ecb467b4c2 100644
--- a/src/calibre/library/server/content.py
+++ b/src/calibre/library/server/content.py
@@ -5,7 +5,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import re, os, cStringIO, operator
+import re, os, cStringIO
 
 import cherrypy
 try:
@@ -16,7 +16,15 @@ except ImportError:
 
 from calibre import fit_image, guess_type
 from calibre.utils.date import fromtimestamp
-from calibre.ebooks.metadata import title_sort
+from calibre.library.caches import SortKeyGenerator
+
+class CSSortKeyGenerator(SortKeyGenerator):
+
+    def __init__(self, fields, fm):
+        SortKeyGenerator.__init__(self, fields, fm, None)
+
+    def __call__(self, record):
+        return self.itervals(record).next()
 
 class ContentServer(object):
 
@@ -47,32 +55,12 @@ class ContentServer(object):
 
 
     def sort(self, items, field, order):
-        field = field.lower().strip()
-        if field == 'author':
-            field = 'authors'
-        if field == 'date':
-            field = 'timestamp'
+        field = self.db.data.sanitize_sort_field_name(field)
         if field not in ('title', 'authors', 'rating', 'timestamp', 'tags', 'size', 'series'):
             raise cherrypy.HTTPError(400, '%s is not a valid sort field'%field)
-        cmpf = cmp if field in ('rating', 'size', 'timestamp') else \
-                lambda x, y: cmp(x.lower() if x else '', y.lower() if y else '')
-        if field == 'series':
-            items.sort(cmp=self.seriescmp, reverse=not order)
-        else:
-            lookup = 'sort' if field == 'title' else field
-            lookup = 'author_sort' if field == 'authors' else field
-            field = self.db.FIELD_MAP[lookup]
-            getter = operator.itemgetter(field)
-            items.sort(cmp=lambda x, y: cmpf(getter(x), getter(y)), reverse=not order)
+        keyg = CSSortKeyGenerator([(field, order)], self.db.field_metadata)
+        items.sort(key=keyg, reverse=not order)
 
-    def seriescmp(self, x, y):
-        si = self.db.FIELD_MAP['series']
-        try:
-            ans = cmp(title_sort(x[si].lower()), title_sort(y[si].lower()))
-        except AttributeError: # Some entries may be None
-            ans = cmp(x[si], y[si])
-        if ans != 0: return ans
-        return cmp(x[self.db.FIELD_MAP['series_index']], y[self.db.FIELD_MAP['series_index']])
     # }}}