From 50082b9779932ac9189fc257a18265d90568bb49 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Thu, 24 Jan 2013 09:20:04 +0530
Subject: [PATCH 1/6] Driver for LG E400. Fixes #1103741 (Device not
 recognized)

---
 src/calibre/devices/android/driver.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py
index acee4938f5..d709ae91a9 100644
--- a/src/calibre/devices/android/driver.py
+++ b/src/calibre/devices/android/driver.py
@@ -141,7 +141,7 @@ class ANDROID(USBMS):
 
             # LG
             0x1004 : {
-                    0x61c5 : [0x100, 0x226, 0x227, 0x9999],
+                    0x61c5 : [0x100, 0x226, 0x227, 0x229, 0x9999],
                     0x61cc : [0x226, 0x227, 0x9999, 0x100],
                     0x61ce : [0x226, 0x227, 0x9999, 0x100],
                     0x618e : [0x226, 0x227, 0x9999, 0x100],
@@ -235,7 +235,7 @@ class ANDROID(USBMS):
             'ADVANCED', 'SGH-I727', 'USB_FLASH_DRIVER', 'ANDROID',
             'S5830I_CARD', 'MID7042', 'LINK-CREATE', '7035', 'VIEWPAD_7E',
             'NOVO7', 'MB526', '_USB#WYK7MSF8KE', 'TABLET_PC', 'F', 'MT65XX_MS',
-            'ICS']
+            'ICS', 'E400']
     WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897',
             'FILE-STOR_GADGET', 'SGH-T959_CARD', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD',
             'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB', 'SGH-T849_CARD',
@@ -246,7 +246,7 @@ class ANDROID(USBMS):
             'FILE-CD_GADGET', 'GT-I9001_CARD', 'USB_2.0', 'XT875',
             'UMS_COMPOSITE', 'PRO', '.KOBO_VOX', 'SGH-T989_CARD', 'SGH-I727',
             'USB_FLASH_DRIVER', 'ANDROID', 'MID7042', '7035', 'VIEWPAD_7E',
-            'NOVO7', 'ADVANCED', 'TABLET_PC', 'F']
+            'NOVO7', 'ADVANCED', 'TABLET_PC', 'F', 'E400_SD_CARD']
 
     OSX_MAIN_MEM = 'Android Device Main Memory'
 

From dd001ca8c1a8ac68dcf2002311c7a39a1fba9f90 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Thu, 24 Jan 2013 09:27:03 +0530
Subject: [PATCH 2/6] ...

---
 src/calibre/devices/apple/driver.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/calibre/devices/apple/driver.py b/src/calibre/devices/apple/driver.py
index 95dedf546a..f941ccd659 100644
--- a/src/calibre/devices/apple/driver.py
+++ b/src/calibre/devices/apple/driver.py
@@ -307,6 +307,14 @@ class ITUNES(DriverBase):
     update_msg = None
     update_needed = False
 
+    @property
+    def cache_dir(self):
+        return os.path.join(cache_dir(), 'itunes')
+
+    @property
+    def archive_path(self):
+        return os.path.join(self.cache_dir, "thumbs.zip")
+
     # Public methods
     def add_books_to_metadata(self, locations, metadata, booklists):
         '''
@@ -882,8 +890,6 @@ class ITUNES(DriverBase):
         logger().info(" BCD: %s" % ['0x%x' % x for x in sorted(self.BCD)])
         logger().info(" PRODUCT_ID: %s" % ['0x%x' % x for x in sorted(self.PRODUCT_ID)])
 
-        self.cache_dir = os.path.join(cache_dir(), 'itunes')
-        self.archive_path = os.path.join(self.cache_dir, "thumbs.zip")
         # Confirm/create thumbs archive
         if not os.path.exists(self.cache_dir):
             if DEBUG:

From 68f58f2c30b31accecd135391c2e2ff0183f83ce Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Thu, 24 Jan 2013 14:41:03 +0530
Subject: [PATCH 3/6] Start work on get_categories() for the new db backend

---
 src/calibre/db/cache.py          |   4 +-
 src/calibre/db/categories.py     | 105 +++++++++++++++++++++++++++++++
 src/calibre/db/fields.py         |  88 ++++++++++++++++++++++----
 src/calibre/db/tables.py         |  28 +++------
 src/calibre/library/database2.py |  36 +----------
 5 files changed, 191 insertions(+), 70 deletions(-)
 create mode 100644 src/calibre/db/categories.py

diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py
index 6803ab93ed..fec8d4920c 100644
--- a/src/calibre/db/cache.py
+++ b/src/calibre/db/cache.py
@@ -293,13 +293,13 @@ class Cache(object):
         Return all the books associated with the item identified by
         ``item_id``, where the item belongs to the field ``name``.
 
-        Returned value is a tuple of book ids, or the empty tuple if the item
+        Returned value is a set of book ids, or the empty set if the item
         or the field does not exist.
         '''
         try:
             return self.fields[name].books_for(item_id)
         except (KeyError, IndexError):
-            return ()
+            return set()
 
     @read_api
     def all_book_ids(self, type=frozenset):
diff --git a/src/calibre/db/categories.py b/src/calibre/db/categories.py
new file mode 100644
index 0000000000..5e8e5ae687
--- /dev/null
+++ b/src/calibre/db/categories.py
@@ -0,0 +1,105 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from functools import partial
+
+from calibre.library.field_metadata import TagsIcons
+from calibre.utils.config_base import tweaks
+
+CATEGORY_SORTS = { 'name', 'popularity', 'rating' }
+
+class Tag(object):
+
+    def __init__(self, name, id=None, count=0, state=0, avg=0, sort=None,
+                 tooltip=None, icon=None, category=None, id_set=None,
+                 is_editable=True, is_searchable=True, use_sort_as_name=False):
+        self.name = self.original_name = name
+        self.id = id
+        self.count = count
+        self.state = state
+        self.is_hierarchical = ''
+        self.is_editable = is_editable
+        self.is_searchable = is_searchable
+        self.id_set = id_set if id_set is not None else set([])
+        self.avg_rating = avg/2.0 if avg is not None else 0
+        self.sort = sort
+        self.use_sort_as_name = use_sort_as_name
+        if self.avg_rating > 0:
+            if tooltip:
+                tooltip = tooltip + ': '
+            tooltip = _('%(tt)sAverage rating is %(rating)3.1f')%dict(
+                    tt=tooltip, rating=self.avg_rating)
+        self.tooltip = tooltip
+        self.icon = icon
+        self.category = category
+
+    def __unicode__(self):
+        return u'%s:%s:%s:%s:%s:%s'%(self.name, self.count, self.id, self.state,
+                                  self.category, self.tooltip)
+
+    def __str__(self):
+        return unicode(self).encode('utf-8')
+
+    def __repr__(self):
+        return str(self)
+
+def find_categories(field_metadata):
+    for category, cat in field_metadata.iteritems():
+        if (cat['is_category'] and cat['kind'] not in { 'user', 'search' } and
+            category not in {'news', 'formats'} and not cat.get('is_csp', False)):
+            yield (category, cat['is_multiple'].get('cache_to_list', None), False)
+        elif (cat['datatype'] == 'composite' and
+              cat['display'].get('make_category', False)):
+            yield (category, cat['is_multiple'].get('cache_to_list', None), True)
+
+def create_tag_class(category, fm, icon_map):
+    cat = fm[category]
+    icon = None
+    tooltip = '(' + category + ')'
+    label = fm.key_to_label(category)
+    if icon_map:
+        if not fm.is_custom_field(category):
+            if category in icon_map:
+                icon = icon_map[label]
+        else:
+            icon = icon_map['custom:']
+            icon_map[category] = icon
+    is_editable = category not in { 'news', 'rating', 'languages' }
+
+    if (tweaks['categories_use_field_for_author_name'] == 'author_sort' and
+            (category == 'authors' or
+                (cat['display'].get('is_names', False) and
+                cat['is_custom'] and cat['is_multiple'] and
+                cat['datatype'] == 'text'))):
+        use_sort_as_name = True
+    else:
+        use_sort_as_name = False
+
+    return partial(Tag, use_sort_as_name=use_sort_as_name, icon=icon,
+                        tooltip=tooltip, is_editable=is_editable,
+                        category=category)
+
+
+def get_categories(dbcache, sort='name', book_ids=None, icon_map=None):
+    if icon_map is not None and type(icon_map) != TagsIcons:
+        raise TypeError('icon_map passed to get_categories must be of type TagIcons')
+    if sort not in CATEGORY_SORTS:
+        raise ValueError('sort ' + sort + ' not a valid value')
+
+    fm = dbcache.field_metadata
+    book_rating_map = dbcache.fields['rating'].book_value_map
+
+    categories = {}
+    book_ids = frozenset(book_ids)
+    for category, is_multiple, is_composite in find_categories(fm):
+        tag_class = create_tag_class(category, fm, icon_map)
+        categories[category] = dbcache.fields[category].get_categories(
+            tag_class, book_rating_map, sort, dbcache.fields['language'], book_ids)
+
+
diff --git a/src/calibre/db/fields.py b/src/calibre/db/fields.py
index e0950fff3b..f6baec4bdc 100644
--- a/src/calibre/db/fields.py
+++ b/src/calibre/db/fields.py
@@ -9,7 +9,8 @@ __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 from threading import Lock
-from collections import defaultdict
+from collections import defaultdict, Counter
+from operator import attrgetter
 
 from calibre.db.tables import ONE_ONE, MANY_ONE, MANY_MANY
 from calibre.ebooks.metadata import title_sort
@@ -24,22 +25,28 @@ class Field(object):
 
     def __init__(self, name, table):
         self.name, self.table = name, table
-        self.has_text_data = self.metadata['datatype'] in ('text', 'comments',
-                'series', 'enumeration')
-        self.table_type = self.table.table_type
         dt = self.metadata['datatype']
+        self.has_text_data = dt in {'text', 'comments', 'series', 'enumeration'}
+        self.table_type = self.table.table_type
         self._sort_key = (sort_key if dt in ('text', 'series', 'enumeration') else lambda x: x)
         self._default_sort_key = ''
-        if self.metadata['datatype'] in ('int', 'float', 'rating'):
+        if dt in { 'int', 'float', 'rating' }:
             self._default_sort_key = 0
-        elif self.metadata['datatype'] == 'bool':
+        elif dt == 'bool':
             self._default_sort_key = None
-        elif self.metadata['datatype'] == 'datetime':
+        elif dt == 'datetime':
             self._default_sort_key = UNDEFINED_DATE
         if self.name == 'languages':
             self._sort_key = lambda x:sort_key(calibre_langcode_to_name(x))
         self.is_multiple = (bool(self.metadata['is_multiple']) or self.name ==
                 'formats')
+        self.category_formatter = type(u'')
+        self.category_sort_reverse = False
+        if dt == 'rating':
+            self.category_formatter = lambda x:'\u2605'*int(x/2)
+            self.category_sort_reverse = True
+        elif name == 'languages':
+            self.category_formatter = calibre_langcode_to_name
 
     @property
     def metadata(self):
@@ -63,7 +70,7 @@ class Field(object):
     def books_for(self, item_id):
         '''
         Return the ids of all books associated with the item identified by
-        item_id as a tuple. An empty tuple is returned if no books are found.
+        item_id as a set. An empty set is returned if no books are found.
         '''
         raise NotImplementedError()
 
@@ -94,6 +101,34 @@ class Field(object):
         '''
         raise NotImplementedError()
 
+    def get_categories(self, tag_class, book_rating_map, sort, lang_field, book_ids=None):
+        ans = []
+        if not self.is_many:
+            return ans
+
+        special_sort = hasattr(self, 'category_sort_value')
+        for item_id, item_book_ids in self.table.col_book_map.iteritems():
+            if book_ids is not None:
+                item_book_ids = item_book_ids.intersection(book_ids)
+            if item_book_ids:
+                ratings = tuple(r for r in (book_rating_map.get(book_id, 0) for
+                                            book_id in item_book_ids) if r > 0)
+                avg = sum(ratings)/len(ratings)
+                name = self.category_formatter(self.table.id_map[item_id])
+                sval = (self.category_sort_value(item_id, item_book_ids, lang_field)
+                        if special_sort else name)
+                c = tag_class(name, id=item_id, sort=sval, avg=avg,
+                              id_set=item_book_ids, count=len(item_book_ids))
+                ans.append(c)
+        if sort == 'popularity':
+            key=attrgetter('count')
+        elif sort == 'rating':
+            key=attrgetter('avg_rating')
+        else:
+            key=lambda x:sort_key(x.sort or x.name)
+        ans.sort(key=key, reverse=self.category_sort_reverse)
+        return ans
+
 class OneToOneField(Field):
 
     def for_book(self, book_id, default_value=None):
@@ -103,7 +138,7 @@ class OneToOneField(Field):
         return (book_id,)
 
     def books_for(self, item_id):
-        return (item_id,)
+        return {item_id}
 
     def __iter__(self):
         return self.table.book_col_map.iterkeys()
@@ -223,7 +258,7 @@ class ManyToOneField(Field):
         return (id_,)
 
     def books_for(self, item_id):
-        return self.table.col_book_map.get(item_id, ())
+        return self.table.col_book_map.get(item_id, set())
 
     def __iter__(self):
         return self.table.id_map.iterkeys()
@@ -238,11 +273,17 @@ class ManyToOneField(Field):
 
     def iter_searchable_values(self, get_metadata, candidates, default_value=None):
         cbm = self.table.col_book_map
+        empty = set()
         for item_id, val in self.table.id_map.iteritems():
-            book_ids = set(cbm.get(item_id, ())).intersection(candidates)
+            book_ids = cbm.get(item_id, empty).intersection(candidates)
             if book_ids:
                 yield val, book_ids
 
+    @property
+    def book_value_map(self):
+        return {book_id:self.table.id_map[item_id] for book_id, item_id in
+                self.book_col_map.iteritems()}
+
 class ManyToManyField(Field):
 
     is_many = True
@@ -263,7 +304,7 @@ class ManyToManyField(Field):
         return self.table.book_col_map.get(book_id, ())
 
     def books_for(self, item_id):
-        return self.table.col_book_map.get(item_id, ())
+        return self.table.col_book_map.get(item_id, set())
 
     def __iter__(self):
         return self.table.id_map.iterkeys()
@@ -282,8 +323,9 @@ class ManyToManyField(Field):
 
     def iter_searchable_values(self, get_metadata, candidates, default_value=None):
         cbm = self.table.col_book_map
+        empty = set()
         for item_id, val in self.table.id_map.iteritems():
-            book_ids = set(cbm.get(item_id, ())).intersection(candidates)
+            book_ids = cbm.get(item_id, empty).intersection(candidates)
             if book_ids:
                 yield val, book_ids
 
@@ -327,6 +369,9 @@ class AuthorsField(ManyToManyField):
             'link' : self.table.alink_map[author_id],
         }
 
+    def category_sort_value(self, item_id, book_ids, language_field):
+        return self.table.asort_map[item_id]
+
 class FormatsField(ManyToManyField):
 
     def for_book(self, book_id, default_value=None):
@@ -361,6 +406,23 @@ class SeriesField(ManyToOneField):
         return {book_id:self.sort_key_for_series(book_id, get_lang, sso) for book_id
                 in all_book_ids}
 
+    def category_sort_value(self, item_id, book_ids, language_field):
+        lang = None
+        tss = tweaks['title_series_sorting']
+        if tss != 'strictly_alphabetic':
+            lang_map = language_field.book_col_map
+            c = Counter()
+
+            for book_id in book_ids:
+                l = lang_map.get(book_id, None)
+                if l:
+                    c[l[0]] += 1
+
+            if c:
+                lang = c.most_common(1)[0][0]
+        val = self.table.id_map[item_id]
+        return title_sort(val, order=tss, lang=lang)
+
 def create_field(name, table):
     cls = {
             ONE_ONE : OneToOneField,
diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py
index 234a7fe4a8..4a8b4492fd 100644
--- a/src/calibre/db/tables.py
+++ b/src/calibre/db/tables.py
@@ -132,13 +132,10 @@ class ManyToOneTable(Table):
                 'SELECT book, {0} FROM {1}'.format(
                     self.metadata['link_column'], self.link_table)):
             if row[1] not in self.col_book_map:
-                self.col_book_map[row[1]] = []
-            self.col_book_map[row[1]].append(row[0])
+                self.col_book_map[row[1]] = set()
+            self.col_book_map[row[1]].add(row[0])
             self.book_col_map[row[0]] = row[1]
 
-        for key in tuple(self.col_book_map.iterkeys()):
-            self.col_book_map[key] = tuple(self.col_book_map[key])
-
 class ManyToManyTable(ManyToOneTable):
 
     '''
@@ -154,15 +151,12 @@ class ManyToManyTable(ManyToOneTable):
         for row in db.conn.execute(
             self.selectq.format(self.metadata['link_column'], self.link_table)):
             if row[1] not in self.col_book_map:
-                self.col_book_map[row[1]] = []
-            self.col_book_map[row[1]].append(row[0])
+                self.col_book_map[row[1]] = set()
+            self.col_book_map[row[1]].add(row[0])
             if row[0] not in self.book_col_map:
                 self.book_col_map[row[0]] = []
             self.book_col_map[row[0]].append(row[1])
 
-        for key in tuple(self.col_book_map.iterkeys()):
-            self.col_book_map[key] = tuple(self.col_book_map[key])
-
         for key in tuple(self.book_col_map.iterkeys()):
             self.book_col_map[key] = tuple(self.book_col_map[key])
 
@@ -191,8 +185,8 @@ class FormatsTable(ManyToManyTable):
             if row[1] is not None:
                 fmt = row[1].upper()
                 if fmt not in self.col_book_map:
-                    self.col_book_map[fmt] = []
-                self.col_book_map[fmt].append(row[0])
+                    self.col_book_map[fmt] = set()
+                self.col_book_map[fmt].add(row[0])
                 if row[0] not in self.book_col_map:
                     self.book_col_map[row[0]] = []
                 self.book_col_map[row[0]].append(fmt)
@@ -200,9 +194,6 @@ class FormatsTable(ManyToManyTable):
                     self.fname_map[row[0]] = {}
                 self.fname_map[row[0]][fmt] = row[2]
 
-        for key in tuple(self.col_book_map.iterkeys()):
-            self.col_book_map[key] = tuple(self.col_book_map[key])
-
         for key in tuple(self.book_col_map.iterkeys()):
             self.book_col_map[key] = tuple(sorted(self.book_col_map[key]))
 
@@ -215,15 +206,12 @@ class IdentifiersTable(ManyToManyTable):
         for row in db.conn.execute('SELECT book, type, val FROM identifiers'):
             if row[1] is not None and row[2] is not None:
                 if row[1] not in self.col_book_map:
-                    self.col_book_map[row[1]] = []
-                self.col_book_map[row[1]].append(row[0])
+                    self.col_book_map[row[1]] = set()
+                self.col_book_map[row[1]].add(row[0])
                 if row[0] not in self.book_col_map:
                     self.book_col_map[row[0]] = {}
                 self.book_col_map[row[0]][row[1]] = row[2]
 
-        for key in tuple(self.col_book_map.iterkeys()):
-            self.col_book_map[key] = tuple(self.col_book_map[key])
-
 class LanguagesTable(ManyToManyTable):
 
     def read_id_maps(self, db):
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index ad15f1a022..5149b7a0de 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -44,47 +44,13 @@ from calibre.utils.recycle_bin import delete_file, delete_tree
 from calibre.utils.formatter_functions import load_user_template_functions
 from calibre.db.errors import NoSuchFormat
 from calibre.db.lazy import FormatMetadata, FormatsList
+from calibre.db.categories import Tag
 from calibre.utils.localization import (canonicalize_lang,
         calibre_langcode_to_name)
 
 copyfile = os.link if hasattr(os, 'link') else shutil.copyfile
 SPOOL_SIZE = 30*1024*1024
 
-class Tag(object):
-
-    def __init__(self, name, id=None, count=0, state=0, avg=0, sort=None,
-                 tooltip=None, icon=None, category=None, id_set=None,
-                 is_editable = True, is_searchable=True, use_sort_as_name=False):
-        self.name = self.original_name = name
-        self.id = id
-        self.count = count
-        self.state = state
-        self.is_hierarchical = ''
-        self.is_editable = is_editable
-        self.is_searchable = is_searchable
-        self.id_set = id_set if id_set is not None else set([])
-        self.avg_rating = avg/2.0 if avg is not None else 0
-        self.sort = sort
-        self.use_sort_as_name = use_sort_as_name
-        if self.avg_rating > 0:
-            if tooltip:
-                tooltip = tooltip + ': '
-            tooltip = _('%(tt)sAverage rating is %(rating)3.1f')%dict(
-                    tt=tooltip, rating=self.avg_rating)
-        self.tooltip = tooltip
-        self.icon = icon
-        self.category = category
-
-    def __unicode__(self):
-        return u'%s:%s:%s:%s:%s:%s'%(self.name, self.count, self.id, self.state,
-                                  self.category, self.tooltip)
-
-    def __str__(self):
-        return unicode(self).encode('utf-8')
-
-    def __repr__(self):
-        return str(self)
-
 class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
     '''
     An ebook metadata database that stores references to ebook files on disk.

From 6a9fb786d938ef0c4c5193ef064834d97673ed8e Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Thu, 24 Jan 2013 14:53:05 +0530
Subject: [PATCH 4/6] Avoid per book function call when sorting based on book
 language

---
 src/calibre/db/cache.py      |  8 +++-----
 src/calibre/db/categories.py |  3 ++-
 src/calibre/db/fields.py     | 40 +++++++++++++++++++++---------------
 3 files changed, 28 insertions(+), 23 deletions(-)

diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py
index fec8d4920c..65e2e614eb 100644
--- a/src/calibre/db/cache.py
+++ b/src/calibre/db/cache.py
@@ -415,9 +415,7 @@ class Cache(object):
         all_book_ids = frozenset(self._all_book_ids() if ids_to_sort is None
                 else ids_to_sort)
         get_metadata = partial(self._get_metadata, get_user_categories=False)
-        def get_lang(book_id):
-            ans = self._field_for('languages', book_id)
-            return ans[0] if ans else None
+        lang_map = self.fields['languages'].book_value_map
 
         fm = {'title':'sort', 'authors':'author_sort'}
 
@@ -426,10 +424,10 @@ class Cache(object):
             idx = field + '_index'
             is_series = idx in self.fields
             ans = self.fields[fm.get(field, field)].sort_keys_for_books(
-                get_metadata, get_lang, all_book_ids,)
+                get_metadata, lang_map, all_book_ids,)
             if is_series:
                 idx_ans = self.fields[idx].sort_keys_for_books(
-                    get_metadata, get_lang, all_book_ids)
+                    get_metadata, lang_map, all_book_ids)
                 ans = {k:(v, idx_ans[k]) for k, v in ans.iteritems()}
             return ans
 
diff --git a/src/calibre/db/categories.py b/src/calibre/db/categories.py
index 5e8e5ae687..0def3b144e 100644
--- a/src/calibre/db/categories.py
+++ b/src/calibre/db/categories.py
@@ -94,12 +94,13 @@ def get_categories(dbcache, sort='name', book_ids=None, icon_map=None):
 
     fm = dbcache.field_metadata
     book_rating_map = dbcache.fields['rating'].book_value_map
+    lang_map = dbcache.fileds['languages'].book_value_map
 
     categories = {}
     book_ids = frozenset(book_ids)
     for category, is_multiple, is_composite in find_categories(fm):
         tag_class = create_tag_class(category, fm, icon_map)
         categories[category] = dbcache.fields[category].get_categories(
-            tag_class, book_rating_map, sort, dbcache.fields['language'], book_ids)
+            tag_class, book_rating_map, sort, lang_map, book_ids)
 
 
diff --git a/src/calibre/db/fields.py b/src/calibre/db/fields.py
index f6baec4bdc..6a6deacf10 100644
--- a/src/calibre/db/fields.py
+++ b/src/calibre/db/fields.py
@@ -84,7 +84,7 @@ class Field(object):
         '''
         return iter(())
 
-    def sort_keys_for_books(self, get_metadata, get_lang, all_book_ids):
+    def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids):
         '''
         Return a mapping of book_id -> sort_key. The sort key is suitable for
         use in sorting the list of all books by this field, via the python cmp
@@ -101,7 +101,7 @@ class Field(object):
         '''
         raise NotImplementedError()
 
-    def get_categories(self, tag_class, book_rating_map, sort, lang_field, book_ids=None):
+    def get_categories(self, tag_class, book_rating_map, sort, lang_map, book_ids=None):
         ans = []
         if not self.is_many:
             return ans
@@ -115,7 +115,7 @@ class Field(object):
                                             book_id in item_book_ids) if r > 0)
                 avg = sum(ratings)/len(ratings)
                 name = self.category_formatter(self.table.id_map[item_id])
-                sval = (self.category_sort_value(item_id, item_book_ids, lang_field)
+                sval = (self.category_sort_value(item_id, item_book_ids, lang_map)
                         if special_sort else name)
                 c = tag_class(name, id=item_id, sort=sval, avg=avg,
                               id_set=item_book_ids, count=len(item_book_ids))
@@ -143,7 +143,7 @@ class OneToOneField(Field):
     def __iter__(self):
         return self.table.book_col_map.iterkeys()
 
-    def sort_keys_for_books(self, get_metadata, get_lang, all_book_ids):
+    def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids):
         return {id_ : self._sort_key(self.table.book_col_map.get(id_,
             self._default_sort_key)) for id_ in all_book_ids}
 
@@ -185,7 +185,7 @@ class CompositeField(OneToOneField):
             ans = mi.get('#'+self.metadata['label'])
         return ans
 
-    def sort_keys_for_books(self, get_metadata, get_lang, all_book_ids):
+    def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids):
         return {id_ : sort_key(self.get_value_with_cache(id_, get_metadata)) for id_ in
                 all_book_ids}
 
@@ -228,7 +228,7 @@ class OnDeviceField(OneToOneField):
     def __iter__(self):
         return iter(())
 
-    def sort_keys_for_books(self, get_metadata, get_lang, all_book_ids):
+    def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids):
         return {id_ : self.for_book(id_) for id_ in
                 all_book_ids}
 
@@ -263,7 +263,7 @@ class ManyToOneField(Field):
     def __iter__(self):
         return self.table.id_map.iterkeys()
 
-    def sort_keys_for_books(self, get_metadata, get_lang, all_book_ids):
+    def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids):
         ans = {id_ : self.table.book_col_map.get(id_, None)
                 for id_ in all_book_ids}
         sk_map = {cid : (self._default_sort_key if cid is None else
@@ -309,7 +309,7 @@ class ManyToManyField(Field):
     def __iter__(self):
         return self.table.id_map.iterkeys()
 
-    def sort_keys_for_books(self, get_metadata, get_lang, all_book_ids):
+    def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids):
         ans = {id_ : self.table.book_col_map.get(id_, ())
                 for id_ in all_book_ids}
         all_cids = set()
@@ -337,6 +337,11 @@ class ManyToManyField(Field):
         for count, book_ids in val_map.iteritems():
             yield count, book_ids
 
+    @property
+    def book_value_map(self):
+        return {book_id:tuple(self.table.id_map[item_id] for item_id in item_ids)
+                for book_id, item_ids in self.table.book_col_map.iteritems()}
+
 class IdentifiersField(ManyToManyField):
 
     def for_book(self, book_id, default_value=None):
@@ -345,7 +350,7 @@ class IdentifiersField(ManyToManyField):
             ids = default_value
         return ids
 
-    def sort_keys_for_books(self, get_metadata, get_lang, all_book_ids):
+    def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids):
         'Sort by identifier keys'
         ans = {id_ : self.table.book_col_map.get(id_, ())
                 for id_ in all_book_ids}
@@ -369,7 +374,7 @@ class AuthorsField(ManyToManyField):
             'link' : self.table.alink_map[author_id],
         }
 
-    def category_sort_value(self, item_id, book_ids, language_field):
+    def category_sort_value(self, item_id, book_ids, lang_map):
         return self.table.asort_map[item_id]
 
 class FormatsField(ManyToManyField):
@@ -393,24 +398,25 @@ class FormatsField(ManyToManyField):
 
 class SeriesField(ManyToOneField):
 
-    def sort_key_for_series(self, book_id, get_lang, series_sort_order):
+    def sort_key_for_series(self, book_id, lang_map, series_sort_order):
         sid = self.table.book_col_map.get(book_id, None)
         if sid is None:
             return self._default_sort_key
+        lang = lang_map.get(book_id, None) or None
+        if lang:
+            lang = lang[0]
         return self._sort_key(title_sort(self.table.id_map[sid],
-                                         order=series_sort_order,
-                                         lang=get_lang(book_id)))
+                                         order=series_sort_order, lang=lang))
 
-    def sort_keys_for_books(self, get_metadata, get_lang, all_book_ids):
+    def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids):
         sso = tweaks['title_series_sorting']
-        return {book_id:self.sort_key_for_series(book_id, get_lang, sso) for book_id
+        return {book_id:self.sort_key_for_series(book_id, lang_map, sso) for book_id
                 in all_book_ids}
 
-    def category_sort_value(self, item_id, book_ids, language_field):
+    def category_sort_value(self, item_id, book_ids, lang_map):
         lang = None
         tss = tweaks['title_series_sorting']
         if tss != 'strictly_alphabetic':
-            lang_map = language_field.book_col_map
             c = Counter()
 
             for book_id in book_ids:

From c7e8509e10b9a6d4ded0322b12ec3f903b136d14 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Thu, 24 Jan 2013 17:28:14 +0530
Subject: [PATCH 5/6] Categories for identifiers

---
 src/calibre/db/cache.py      |  6 ++++++
 src/calibre/db/categories.py | 24 ++++++++++++++++++------
 src/calibre/db/fields.py     | 28 +++++++++++++++-------------
 3 files changed, 39 insertions(+), 19 deletions(-)

diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py
index 65e2e614eb..47197aff3a 100644
--- a/src/calibre/db/cache.py
+++ b/src/calibre/db/cache.py
@@ -11,6 +11,7 @@ import os, traceback
 from collections import defaultdict
 from functools import wraps, partial
 
+from calibre.db.categories import get_categories
 from calibre.db.locking import create_locks, RecordLock
 from calibre.db.fields import create_field
 from calibre.db.search import Search
@@ -445,6 +446,11 @@ class Cache(object):
         return self._search_api(self, query, restriction,
                                 virtual_fields=virtual_fields)
 
+    @read_api
+    def get_categories(self, sort='name', book_ids=None, icon_map=None):
+        return get_categories(self, sort=sort, book_ids=book_ids,
+                              icon_map=icon_map)
+
     # }}}
 
 class SortKey(object):
diff --git a/src/calibre/db/categories.py b/src/calibre/db/categories.py
index 0def3b144e..f439cb5543 100644
--- a/src/calibre/db/categories.py
+++ b/src/calibre/db/categories.py
@@ -8,9 +8,11 @@ __copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 from functools import partial
+from operator import attrgetter
 
 from calibre.library.field_metadata import TagsIcons
 from calibre.utils.config_base import tweaks
+from calibre.utils.icu import sort_key
 
 CATEGORY_SORTS = { 'name', 'popularity', 'rating' }
 
@@ -52,7 +54,7 @@ class Tag(object):
 def find_categories(field_metadata):
     for category, cat in field_metadata.iteritems():
         if (cat['is_category'] and cat['kind'] not in { 'user', 'search' } and
-            category not in {'news', 'formats'} and not cat.get('is_csp', False)):
+            category not in {'news', 'formats'}):
             yield (category, cat['is_multiple'].get('cache_to_list', None), False)
         elif (cat['datatype'] == 'composite' and
               cat['display'].get('make_category', False)):
@@ -61,7 +63,7 @@ def find_categories(field_metadata):
 def create_tag_class(category, fm, icon_map):
     cat = fm[category]
     icon = None
-    tooltip = '(' + category + ')'
+    tooltip = None if category == 'identifiers' else ('(' + category + ')')
     label = fm.key_to_label(category)
     if icon_map:
         if not fm.is_custom_field(category):
@@ -94,13 +96,23 @@ def get_categories(dbcache, sort='name', book_ids=None, icon_map=None):
 
     fm = dbcache.field_metadata
     book_rating_map = dbcache.fields['rating'].book_value_map
-    lang_map = dbcache.fileds['languages'].book_value_map
+    lang_map = dbcache.fields['languages'].book_value_map
 
     categories = {}
-    book_ids = frozenset(book_ids)
+    book_ids = frozenset(book_ids) if book_ids else book_ids
     for category, is_multiple, is_composite in find_categories(fm):
         tag_class = create_tag_class(category, fm, icon_map)
-        categories[category] = dbcache.fields[category].get_categories(
-            tag_class, book_rating_map, sort, lang_map, book_ids)
+        cats = dbcache.fields[category].get_categories(
+            tag_class, book_rating_map, lang_map, book_ids)
+        if sort == 'popularity':
+            key=attrgetter('count')
+        elif sort == 'rating':
+            key=attrgetter('avg_rating')
+        else:
+            key=lambda x:sort_key(x.sort or x.name)
+        cats.sort(key=key)
+        categories[category] = cats
+
+    return categories
 
 
diff --git a/src/calibre/db/fields.py b/src/calibre/db/fields.py
index 6a6deacf10..13983ea5d5 100644
--- a/src/calibre/db/fields.py
+++ b/src/calibre/db/fields.py
@@ -10,7 +10,6 @@ __docformat__ = 'restructuredtext en'
 
 from threading import Lock
 from collections import defaultdict, Counter
-from operator import attrgetter
 
 from calibre.db.tables import ONE_ONE, MANY_ONE, MANY_MANY
 from calibre.ebooks.metadata import title_sort
@@ -41,10 +40,8 @@ class Field(object):
         self.is_multiple = (bool(self.metadata['is_multiple']) or self.name ==
                 'formats')
         self.category_formatter = type(u'')
-        self.category_sort_reverse = False
         if dt == 'rating':
             self.category_formatter = lambda x:'\u2605'*int(x/2)
-            self.category_sort_reverse = True
         elif name == 'languages':
             self.category_formatter = calibre_langcode_to_name
 
@@ -101,7 +98,7 @@ class Field(object):
         '''
         raise NotImplementedError()
 
-    def get_categories(self, tag_class, book_rating_map, sort, lang_map, book_ids=None):
+    def get_categories(self, tag_class, book_rating_map, lang_map, book_ids=None):
         ans = []
         if not self.is_many:
             return ans
@@ -113,20 +110,13 @@ class Field(object):
             if item_book_ids:
                 ratings = tuple(r for r in (book_rating_map.get(book_id, 0) for
                                             book_id in item_book_ids) if r > 0)
-                avg = sum(ratings)/len(ratings)
+                avg = sum(ratings)/len(ratings) if ratings else 0
                 name = self.category_formatter(self.table.id_map[item_id])
                 sval = (self.category_sort_value(item_id, item_book_ids, lang_map)
                         if special_sort else name)
                 c = tag_class(name, id=item_id, sort=sval, avg=avg,
                               id_set=item_book_ids, count=len(item_book_ids))
                 ans.append(c)
-        if sort == 'popularity':
-            key=attrgetter('count')
-        elif sort == 'rating':
-            key=attrgetter('avg_rating')
-        else:
-            key=lambda x:sort_key(x.sort or x.name)
-        ans.sort(key=key, reverse=self.category_sort_reverse)
         return ans
 
 class OneToOneField(Field):
@@ -282,7 +272,7 @@ class ManyToOneField(Field):
     @property
     def book_value_map(self):
         return {book_id:self.table.id_map[item_id] for book_id, item_id in
-                self.book_col_map.iteritems()}
+                self.table.book_col_map.iteritems()}
 
 class ManyToManyField(Field):
 
@@ -365,6 +355,18 @@ class IdentifiersField(ManyToManyField):
             if val:
                 yield val, {book_id}
 
+    def get_categories(self, tag_class, book_rating_map, lang_map, book_ids=None):
+        ans = []
+
+        for id_key, item_book_ids in self.table.col_book_map.iteritems():
+            if book_ids is not None:
+                item_book_ids = item_book_ids.intersection(book_ids)
+            if item_book_ids:
+                name = id_key
+                c = tag_class(name, id_set=item_book_ids, count=len(item_book_ids))
+                ans.append(c)
+        return ans
+
 class AuthorsField(ManyToManyField):
 
     def author_data(self, author_id):

From dee27d1a844d043d3bb9fdb2810c6fc382235b09 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Thu, 24 Jan 2013 17:39:49 +0530
Subject: [PATCH 6/6] Implement formats category

---
 src/calibre/db/categories.py    | 10 +++++-----
 src/calibre/db/fields.py        | 14 ++++++++++++--
 src/calibre/db/tests/base.py    |  1 +
 src/calibre/db/tests/reading.py | 12 ++++++++++++
 4 files changed, 30 insertions(+), 7 deletions(-)

diff --git a/src/calibre/db/categories.py b/src/calibre/db/categories.py
index f439cb5543..092b6da877 100644
--- a/src/calibre/db/categories.py
+++ b/src/calibre/db/categories.py
@@ -53,8 +53,8 @@ class Tag(object):
 
 def find_categories(field_metadata):
     for category, cat in field_metadata.iteritems():
-        if (cat['is_category'] and cat['kind'] not in { 'user', 'search' } and
-            category not in {'news', 'formats'}):
+        if (cat['is_category'] and cat['kind'] not in {'user', 'search'} and
+            category != 'news'):
             yield (category, cat['is_multiple'].get('cache_to_list', None), False)
         elif (cat['datatype'] == 'composite' and
               cat['display'].get('make_category', False)):
@@ -63,7 +63,7 @@ def find_categories(field_metadata):
 def create_tag_class(category, fm, icon_map):
     cat = fm[category]
     icon = None
-    tooltip = None if category == 'identifiers' else ('(' + category + ')')
+    tooltip = None if category in {'formats', 'identifiers'} else ('(' + category + ')')
     label = fm.key_to_label(category)
     if icon_map:
         if not fm.is_custom_field(category):
@@ -72,7 +72,8 @@ def create_tag_class(category, fm, icon_map):
         else:
             icon = icon_map['custom:']
             icon_map[category] = icon
-    is_editable = category not in { 'news', 'rating', 'languages' }
+    is_editable = category not in {'news', 'rating', 'languages', 'formats',
+                                   'identifiers'}
 
     if (tweaks['categories_use_field_for_author_name'] == 'author_sort' and
             (category == 'authors' or
@@ -87,7 +88,6 @@ def create_tag_class(category, fm, icon_map):
                         tooltip=tooltip, is_editable=is_editable,
                         category=category)
 
-
 def get_categories(dbcache, sort='name', book_ids=None, icon_map=None):
     if icon_map is not None and type(icon_map) != TagsIcons:
         raise TypeError('icon_map passed to get_categories must be of type TagIcons')
diff --git a/src/calibre/db/fields.py b/src/calibre/db/fields.py
index 13983ea5d5..4e73b5badf 100644
--- a/src/calibre/db/fields.py
+++ b/src/calibre/db/fields.py
@@ -362,8 +362,7 @@ class IdentifiersField(ManyToManyField):
             if book_ids is not None:
                 item_book_ids = item_book_ids.intersection(book_ids)
             if item_book_ids:
-                name = id_key
-                c = tag_class(name, id_set=item_book_ids, count=len(item_book_ids))
+                c = tag_class(id_key, id_set=item_book_ids, count=len(item_book_ids))
                 ans.append(c)
         return ans
 
@@ -398,6 +397,17 @@ class FormatsField(ManyToManyField):
         for val, book_ids in val_map.iteritems():
             yield val, book_ids
 
+    def get_categories(self, tag_class, book_rating_map, lang_map, book_ids=None):
+        ans = []
+
+        for fmt, item_book_ids in self.table.col_book_map.iteritems():
+            if book_ids is not None:
+                item_book_ids = item_book_ids.intersection(book_ids)
+            if item_book_ids:
+                c = tag_class(fmt, id_set=item_book_ids, count=len(item_book_ids))
+                ans.append(c)
+        return ans
+
 class SeriesField(ManyToOneField):
 
     def sort_key_for_series(self, book_id, lang_map, series_sort_order):
diff --git a/src/calibre/db/tests/base.py b/src/calibre/db/tests/base.py
index 8e72721c4e..b626551576 100644
--- a/src/calibre/db/tests/base.py
+++ b/src/calibre/db/tests/base.py
@@ -42,6 +42,7 @@ class BaseTest(unittest.TestCase):
             if attr == 'format_metadata': continue # TODO: Not implemented yet
             attr1, attr2 = getattr(mi1, attr), getattr(mi2, attr)
             if attr == 'formats':
+                continue # TODO: Not implemented yet
                 attr1, attr2 = map(lambda x:tuple(x) if x else (), (attr1, attr2))
             self.assertEqual(attr1, attr2,
                     '%s not the same: %r != %r'%(attr, attr1, attr2))
diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py
index 8183611f91..b1d4bd3142 100644
--- a/src/calibre/db/tests/reading.py
+++ b/src/calibre/db/tests/reading.py
@@ -241,6 +241,18 @@ class ReadingTest(BaseTest):
 
     # }}}
 
+    def test_get_categories(self): # {{{
+        'Check that get_categories() returns the same data for both backends'
+        from calibre.library.database2 import LibraryDatabase2
+        old = LibraryDatabase2(self.library_path)
+        old_categories = old.get_categories()
+        cache = self.init_cache(self.library_path)
+        import pprint
+        pprint.pprint(old_categories)
+        pprint.pprint(cache.get_categories())
+
+    # }}}
+
 def tests():
     return unittest.TestLoader().loadTestsFromTestCase(ReadingTest)