New db: Implement sorting for all field types. Also allow use of field_for() with composite columns

src/calibre/db/cache.py

@@ -9,7 +9,7 @@ __docformat__ = 'restructuredtext en'
 
 import os
 from collections import defaultdict
-from functools import wraps
+from functools import wraps, partial
 
 from calibre.db.locking import create_locks, RecordLock
 from calibre.db.fields import create_field
@@ -43,6 +43,7 @@ class Cache(object):
     def __init__(self, backend):
         self.backend = backend
         self.fields = {}
+        self.composites = set()
         self.read_lock, self.write_lock = create_locks()
         self.record_lock = RecordLock(self.read_lock)
         self.format_metadata_cache = defaultdict(dict)
@@ -82,7 +83,7 @@ class Cache(object):
         if name and path:
             return self.backend.format_abspath(book_id, fmt, name, path)
 
-    def _get_metadata(self, book_id, get_user_categories=True):
+    def _get_metadata(self, book_id, get_user_categories=True): # {{{
         mi = Metadata(None)
         author_ids = self._field_ids_for('authors', book_id)
         aut_list = [self._author_data(i) for i in author_ids]
@@ -162,6 +163,7 @@ class Cache(object):
         mi.user_categories = user_cat_vals
 
         return mi
+    # }}}
 
     # Cache Layer API {{{
 
@@ -175,6 +177,8 @@ class Cache(object):
 
             for field, table in self.backend.tables.iteritems():
                 self.fields[field] = create_field(field, table)
+                if table.metadata['datatype'] == 'composite':
+                    self.composites.add(field)
 
             self.fields['ondevice'] = create_field('ondevice', None)
 
@@ -187,19 +191,26 @@ class Cache(object):
 
         The returned value for is_multiple fields are always tuples.
         '''
+        if self.composites and name in self.composites:
+            return self.composite_for(name, book_id,
+                    default_value=default_value)
         try:
             return self.fields[name].for_book(book_id, default_value=default_value)
         except (KeyError, IndexError):
             return default_value
 
     @read_api
-    def composite_for(self, name, book_id, mi, default_value=''):
+    def composite_for(self, name, book_id, mi=None, default_value=''):
         try:
             f = self.fields[name]
         except KeyError:
             return default_value
 
-        f.render_composite(book_id, mi)
+        if mi is None:
+            return f.get_value_with_cache(book_id, partial(self._get_metadata,
+                get_user_categories=False))
+        else:
+            return f.render_composite(book_id, mi)
 
     @read_api
     def field_ids_for(self, name, book_id):

src/calibre/db/fields.py

@@ -2,12 +2,14 @@
 # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
 from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
+from future_builtins import map
 
 __license__   = 'GPL v3'
 __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 from calibre.db.tables import ONE_ONE, MANY_ONE, MANY_MANY
+from calibre.utils.icu import sort_key
 
 class Field(object):
 
@@ -16,6 +18,8 @@ class Field(object):
         self.has_text_data = self.metadata['datatype'] in ('text', 'comments',
                 'series', 'enumeration')
         self.table_type = self.table.table_type
+        dt = self.metadata['datatype']
+        self._sort_key = (sort_key if dt == 'text' else lambda x: x)
 
     @property
     def metadata(self):
@@ -49,6 +53,18 @@ class Field(object):
         '''
         raise NotImplementedError()
 
+    def sort_books(self, get_metadata, all_book_ids, ascending=True):
+        '''
+        Sort books by this field. Returns a sorted list of book_ids
+
+        :param _get_metadata: A callable which when called with the book_id
+        returns the Metadata object for that book. Needed for sorting composite
+        columns.
+
+        :param all_book_ids: The set of ids for all books.
+        '''
+        raise NotImplementedError()
+
 class OneToOneField(Field):
 
     def for_book(self, book_id, default_value=None):
@@ -66,6 +82,10 @@ class OneToOneField(Field):
     def iter_book_ids(self):
         return self.table.book_col_map.iterkeys()
 
+    def sort_books(self, get_metadata, all_book_ids, ascending=True):
+        return sorted(self.iter_book_ids(), reverse=not ascending,
+            key=lambda i: self._sort_key(self.book_col_map[i]))
+
 class CompositeField(OneToOneField):
 
     def __init__(self, *args, **kwargs):
@@ -86,6 +106,19 @@ class CompositeField(OneToOneField):
     def pop_cache(self, book_id):
         self._render_cache.pop(book_id, None)
 
+    def get_value_with_cache(self, book_id, get_metadata):
+        ans = self._render_cache.get(book_id, None)
+        if ans is None:
+            mi = get_metadata(book_id)
+            ans = mi.get(self.metadata['label'])
+            self._render_cache[book_id] = ans
+        return ans
+
+    def sort_books(self, get_metadata, all_book_ids, ascending=True):
+        return sorted(all_book_ids, reverse=not ascending,
+                key=lambda i: sort_key(self.get_value_with_cache(i,
+                    get_metadata)))
+
 class OnDeviceField(OneToOneField):
 
     def __init__(self, name, table):
@@ -120,6 +153,10 @@ class OnDeviceField(OneToOneField):
     def iter_book_ids(self):
         return iter(())
 
+    def sort_books(self, get_metadata, all_book_ids, ascending=True):
+        return sorted(all_book_ids, reverse=not ascending,
+                key=self.for_book)
+
 class ManyToOneField(Field):
 
     def for_book(self, book_id, default_value=None):
@@ -131,10 +168,10 @@ class ManyToOneField(Field):
         return ans
 
     def ids_for_book(self, book_id):
-        ids = self.table.book_col_map.get(book_id, None)
+        id_ = self.table.book_col_map.get(book_id, None)
-        if ids is None:
+        if id_ is None:
             return ()
-        return ids
+        return (id_,)
 
     def books_for(self, item_id):
         return self.table.col_book_map.get(item_id, ())
@@ -142,8 +179,21 @@ class ManyToOneField(Field):
     def __iter__(self):
         return self.table.id_map.iterkeys()
 
+    def sort_books(self, get_metadata, all_book_ids, ascending=True):
+        ids = sorted(self.id_map,
+                key=lambda i:self._sort_key(self.id_map[i]))
+        sm = {id_ : idx for idx, id_ in enumerate(ids)}
+        return sorted(all_book_ids, reverse=not ascending,
+                key=lambda book_id : sm.get(
+                    self.book_col_map.get(book_id, None),
+                    -1))
+
 class ManyToManyField(Field):
 
+    def __init__(self, *args, **kwargs):
+        Field.__init__(self, *args, **kwargs)
+        self.alphabetical_sort = self.name != 'authors'
+
     def for_book(self, book_id, default_value=None):
         ids = self.table.book_col_map.get(book_id, ())
         if ids:
@@ -161,6 +211,20 @@ class ManyToManyField(Field):
     def __iter__(self):
         return self.table.id_map.iterkeys()
 
+    def sort_books(self, get_metadata, all_book_ids, ascending=True):
+        ids = sorted(self.id_map,
+                key=lambda i:self._sort_key(self.id_map[i]))
+        sm = {id_ : idx for idx, id_ in enumerate(ids)}
+
+        def sort_key_for_book(book_id):
+            item_ids = self.table.book_col_map.get(book_id, ())
+            if self.alphabetical_sort:
+                item_ids = sorted(item_ids, key=sm.get)
+            return tuple(map(sm.get, item_ids))
+
+        return sorted(all_book_ids, reverse=not ascending,
+                key=sort_key_for_book)
+
 class AuthorsField(ManyToManyField):
 
     def author_data(self, author_id):