Merge from trunk

resources/default_tweaks.py

@@ -146,3 +146,10 @@ add_new_book_tags_when_importing_books = False
 max_content_server_tags_shown=5
 
 
+# Set the maximum number of sort 'levels' that calibre will use to resort the
+# library after certain operations such as searches or device insertion. Each
+# sort level adds a performance penalty. If the database is large (thousands of
+# books) the penalty might be noticeable. If you are not concerned about multi-
+# level sorts, and if you are seeing a slowdown, reduce the value of this tweak.
+maximum_resort_levels = 5
+

src/calibre/gui2/actions/edit_metadata.py

@@ -209,8 +209,9 @@ class EditMetadataAction(InterfaceAction):
         dest_id, src_books, src_ids = self.books_to_merge(rows)
         if safe_merge:
             if not confirm('<p>'+_(
-                'All book formats and metadata from the selected books '
+                'Book formats and metadata from the selected books '
-                'will be added to the <b>first selected book.</b><br><br> '
+                'will be added to the <b>first selected book.</b> '
+                'ISBN will <i>not</i> be merged.<br><br> '
                 'The second and subsequently selected books will not '
                 'be deleted or changed.<br><br>'
                 'Please confirm you want to proceed.')
@@ -220,8 +221,9 @@ class EditMetadataAction(InterfaceAction):
             self.merge_metadata(dest_id, src_ids)
         else:
             if not confirm('<p>'+_(
-                'All book formats and metadata from the selected books will be merged '
+                'Book formats and metadata from the selected books will be merged '
-                'into the <b>first selected book</b>.<br><br>'
+                'into the <b>first selected book</b>. '
+                'ISBN will <i>not</i> be merged.<br><br>'
                 'After merger the second and '
                 'subsequently selected books will be <b>deleted</b>. <br><br>'
                 'All book formats of the first selected book will be kept '

src/calibre/gui2/library/models.py

@@ -247,7 +247,7 @@ class BooksModel(QAbstractTableModel): # {{{
             # the search and count records for restrictions
             self.searched.emit(True)
 
-    def sort(self, col, order, reset=True, update_history=True):
+    def sort(self, col, order, reset=True):
         if not self.db:
             return
         self.about_to_be_sorted.emit(self.db.id)
@@ -258,7 +258,6 @@ class BooksModel(QAbstractTableModel): # {{{
             self.clear_caches()
             self.reset()
         self.sorted_on = (label, order)
-        if update_history:
         self.sort_history.insert(0, self.sorted_on)
         self.sorting_done.emit(self.db.index)
 
@@ -266,13 +265,10 @@ class BooksModel(QAbstractTableModel): # {{{
         self.db.refresh(field=None)
         self.resort(reset=reset)
 
-    def resort(self, reset=True, history=5): # Bug report needed history=4 :)
+    def resort(self, reset=True):
-        for col,ord in reversed(self.sort_history[:history]):
+        if not self.db:
-            try:
+            return
-                col = self.column_map.index(col)
+        self.db.multisort(self.sort_history[:tweaks['maximum_resort_levels']])
-            except ValueError:
-                col = 0
-            self.sort(col, ord, reset=False, update_history=False)
         if reset:
             self.reset()
 
@@ -1031,6 +1027,11 @@ class DeviceBooksModel(BooksModel): # {{{
         if reset:
             self.reset()
 
+    def resort(self, reset=True):
+        if self.sorted_on:
+            self.sort(self.column_map.index(self.sorted_on[0]),
+                      self.sorted_on[1], reset=reset)
+
     def columnCount(self, parent):
         if parent and parent.isValid():
             return 0

src/calibre/library/caches.py

@@ -6,7 +6,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import re, itertools, functools
+import re, itertools
 from itertools import repeat
 from datetime import timedelta
 from threading import Thread, RLock
@@ -112,7 +112,7 @@ class ResultCache(SearchQueryParser):
     '''
     def __init__(self, FIELD_MAP, field_metadata):
         self.FIELD_MAP = FIELD_MAP
-        self._map = self._map_filtered = self._data = []
+        self._map = self._data = self._map_filtered = []
         self.first_sort = True
         self.search_restriction = ''
         self.field_metadata = field_metadata
@@ -480,8 +480,11 @@ class ResultCache(SearchQueryParser):
                 q = u'%s (%s)' % (search_restriction, query)
         if not q:
             return list(self._map)
-        matches = sorted(self.parse(q))
+        matches = self.parse(q)
-        return [id for id in self._map if id in matches]
+        tmap = list(itertools.repeat(False, len(self._data)))
+        for x in matches:
+            tmap[x] = True
+        return [x for x in self._map if tmap[x]]
 
     def set_search_restriction(self, s):
         self.search_restriction = s
@@ -490,10 +493,14 @@ class ResultCache(SearchQueryParser):
 
     def remove(self, id):
         self._data[id] = None
-        if id in self._map:
+        try:
             self._map.remove(id)
-        if id in self._map_filtered:
+        except ValueError:
+            pass
+        try:
             self._map_filtered.remove(id)
+        except ValueError:
+            pass
 
     def set(self, row, col, val, row_is_id=False):
         id = row if row_is_id else self._map_filtered[row]
@@ -548,9 +555,7 @@ class ResultCache(SearchQueryParser):
 
     def books_deleted(self, ids):
         for id in ids:
-            self._data[id] = None
+            self.remove(id)
-            if id in self._map: self._map.remove(id)
-            if id in self._map_filtered: self._map_filtered.remove(id)
 
     def count(self):
         return len(self._map)
@@ -579,69 +584,92 @@ class ResultCache(SearchQueryParser):
 
     # Sorting functions {{{
 
-    def seriescmp(self, sidx, siidx, x, y, library_order=None):
+    def sanitize_sort_field_name(self, field):
-        try:
-            if library_order:
-                ans = cmp(title_sort(self._data[x][sidx].lower()),
-                                title_sort(self._data[y][sidx].lower()))
-            else:
-                ans = cmp(self._data[x][sidx].lower(),
-                                                self._data[y][sidx].lower())
-        except AttributeError: # Some entries may be None
-            ans = cmp(self._data[x][sidx], self._data[y][sidx])
-        if ans != 0: return ans
-        return cmp(self._data[x][siidx], self._data[y][siidx])
-
-    def cmp(self, loc, x, y, asstr=True, subsort=False):
-        try:
-            ans = cmp(self._data[x][loc].lower(), self._data[y][loc].lower()) if \
-                asstr else cmp(self._data[x][loc], self._data[y][loc])
-        except AttributeError: # Some entries may be None
-            ans = cmp(self._data[x][loc], self._data[y][loc])
-        except TypeError: ## raised when a datetime is None
-            x = self._data[x][loc]
-            if x is None:
-                x = UNDEFINED_DATE
-            y = self._data[y][loc]
-            if y is None:
-                y = UNDEFINED_DATE
-            return cmp(x, y)
-        if subsort and ans == 0:
-            return cmp(self._data[x][11].lower(), self._data[y][11].lower())
-        return ans
-
-    def sort(self, field, ascending, subsort=False):
         field = field.lower().strip()
+        if field not in self.field_metadata.iterkeys():
             if field in ('author', 'tag', 'comment'):
                 field += 's'
             if   field == 'date': field = 'timestamp'
             elif field == 'title': field = 'sort'
             elif field == 'authors': field = 'author_sort'
-        as_string = field not in ('size', 'rating', 'timestamp')
+        return field
 
-        if self.first_sort:
+    def sort(self, field, ascending, subsort=False):
-            subsort = True
+        self.multisort([(field, ascending)])
-            self.first_sort = False
+
-        if self.field_metadata[field]['is_custom']:
+    def multisort(self, fields=[], subsort=False):
-            if self.field_metadata[field]['datatype'] == 'series':
+        fields = [(self.sanitize_sort_field_name(x), bool(y)) for x, y in fields]
-                fcmp = functools.partial(self.seriescmp,
+        keys = self.field_metadata.field_keys()
-                    self.field_metadata[field]['rec_index'],
+        fields = [x for x in fields if x[0] in keys]
-                    self.field_metadata.cc_series_index_column_for(field),
+        if subsort and 'sort' not in [x[0] for x in fields]:
-                    library_order=tweaks['title_series_sorting'] == 'library_order')
+            fields += [('sort', True)]
+        if not fields:
+            fields = [('timestamp', False)]
+
+        keyg = SortKeyGenerator(fields, self.field_metadata, self._data)
+        if len(fields) == 1:
+            self._map.sort(key=keyg, reverse=not fields[0][1])
         else:
-                as_string = self.field_metadata[field]['datatype'] in ('comments', 'text')
+            self._map.sort(key=keyg)
-                field = self.field_metadata[field]['colnum']
+
-                fcmp = functools.partial(self.cmp, self.FIELD_MAP[field],
+        tmap = list(itertools.repeat(False, len(self._data)))
-                                     subsort=subsort, asstr=as_string)
+        for x in self._map_filtered:
-        elif field == 'series':
+            tmap[x] = True
-            fcmp = functools.partial(self.seriescmp, self.FIELD_MAP['series'],
+        self._map_filtered = [x for x in self._map if tmap[x]]
-                self.FIELD_MAP['series_index'],
+
-                library_order=tweaks['title_series_sorting'] == 'library_order')
+
+class SortKey(object):
+
+    def __init__(self, orders, values):
+        self.orders, self.values = orders, values
+
+    def __cmp__(self, other):
+        for i, ascending in enumerate(self.orders):
+            ans = cmp(self.values[i], other.values[i])
+            if ans != 0:
+                return ans * ascending
+        return 0
+
+class SortKeyGenerator(object):
+
+    def __init__(self, fields, field_metadata, data):
+        self.field_metadata = field_metadata
+        self.orders = [-1 if x[1] else 1 for x in fields]
+        self.entries = [(x[0], field_metadata[x[0]]) for x in fields]
+        self.library_order = tweaks['title_series_sorting'] == 'library_order'
+        self.data = data
+
+    def __call__(self, record):
+        values = tuple(self.itervals(self.data[record]))
+        if len(values) == 1:
+            return values[0]
+        return SortKey(self.orders, values)
+
+    def itervals(self, record):
+        for name, fm in self.entries:
+            dt = fm['datatype']
+            val = record[fm['rec_index']]
+
+            if dt == 'datetime':
+                if val is None:
+                    val = UNDEFINED_DATE
+
+            elif dt == 'series':
+                if val is None:
+                    val = ('', 1)
                 else:
-            fcmp = functools.partial(self.cmp, self.FIELD_MAP[field],
+                    val = val.lower()
-                                     subsort=subsort, asstr=as_string)
+                    if self.library_order:
-        self._map.sort(cmp=fcmp, reverse=not ascending)
+                        val = title_sort(val)
-        self._map_filtered = [id for id in self._map if id in self._map_filtered]
+                    sidx_fm = self.field_metadata[name + '_index']
+                    sidx = record[sidx_fm['rec_index']]
+                    val = (val, sidx)
+
+            elif dt in ('text', 'comments'):
+                if val is None:
+                    val = ''
+                val = val.lower()
+            yield val
 
     # }}}
 

src/calibre/library/database2.py

@@ -311,6 +311,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         self.search_getting_ids  = self.data.search_getting_ids
         self.refresh = functools.partial(self.data.refresh, self)
         self.sort    = self.data.sort
+        self.multisort = self.data.multisort
         self.index   = self.data.index
         self.refresh_ids = functools.partial(self.data.refresh_ids, self)
         self.row     = self.data.row

src/calibre/library/field_metadata.py

@@ -335,6 +335,9 @@ class FieldMetadata(dict):
     def keys(self):
         return self._tb_cats.keys()
 
+    def field_keys(self):
+        return [k for k in self._tb_cats.keys() if self._tb_cats[k]['kind']=='field']
+
     def iterkeys(self):
         for key in self._tb_cats:
             yield key

src/calibre/library/server/content.py

@@ -5,7 +5,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import re, os, cStringIO, operator
+import re, os, cStringIO
 
 import cherrypy
 try:
@@ -16,7 +16,15 @@ except ImportError:
 
 from calibre import fit_image, guess_type
 from calibre.utils.date import fromtimestamp
-from calibre.ebooks.metadata import title_sort
+from calibre.library.caches import SortKeyGenerator
+
+class CSSortKeyGenerator(SortKeyGenerator):
+
+    def __init__(self, fields, fm):
+        SortKeyGenerator.__init__(self, fields, fm, None)
+
+    def __call__(self, record):
+        return self.itervals(record).next()
 
 class ContentServer(object):
 
@@ -47,32 +55,12 @@ class ContentServer(object):
 
 
     def sort(self, items, field, order):
-        field = field.lower().strip()
+        field = self.db.data.sanitize_sort_field_name(field)
-        if field == 'author':
-            field = 'authors'
-        if field == 'date':
-            field = 'timestamp'
         if field not in ('title', 'authors', 'rating', 'timestamp', 'tags', 'size', 'series'):
             raise cherrypy.HTTPError(400, '%s is not a valid sort field'%field)
-        cmpf = cmp if field in ('rating', 'size', 'timestamp') else \
+        keyg = CSSortKeyGenerator([(field, order)], self.db.field_metadata)
-                lambda x, y: cmp(x.lower() if x else '', y.lower() if y else '')
+        items.sort(key=keyg, reverse=not order)
-        if field == 'series':
-            items.sort(cmp=self.seriescmp, reverse=not order)
-        else:
-            lookup = 'sort' if field == 'title' else field
-            lookup = 'author_sort' if field == 'authors' else field
-            field = self.db.FIELD_MAP[lookup]
-            getter = operator.itemgetter(field)
-            items.sort(cmp=lambda x, y: cmpf(getter(x), getter(y)), reverse=not order)
 
-    def seriescmp(self, x, y):
-        si = self.db.FIELD_MAP['series']
-        try:
-            ans = cmp(title_sort(x[si].lower()), title_sort(y[si].lower()))
-        except AttributeError: # Some entries may be None
-            ans = cmp(x[si], y[si])
-        if ans != 0: return ans
-        return cmp(x[self.db.FIELD_MAP['series_index']], y[self.db.FIELD_MAP['series_index']])
     # }}}