Fix bug #1422116: different characters that sort intermixed break tag browser first letter categorization.

Also make rating and count category sorts order the list by the tag's sort value if the numeric values are the same. This turns it into a semi-stable sort when switching from name to (for example) count.

src/calibre/db/cache.py

@@ -941,11 +941,13 @@ class Cache(object):
         return self._search_api(self, query, restriction, virtual_fields=virtual_fields, book_ids=book_ids)
 
     @api
-    def get_categories(self, sort='name', book_ids=None, icon_map=None, already_fixed=None):
+    def get_categories(self, sort='name', book_ids=None, icon_map=None, already_fixed=None,
+                       first_letter_sort=False):
         ' Used internally to implement the Tag Browser '
         try:
             with self.safe_read_lock:
-                return get_categories(self, sort=sort, book_ids=book_ids, icon_map=icon_map)
+                return get_categories(self, sort=sort, book_ids=book_ids, icon_map=icon_map,
+                                      first_letter_sort=first_letter_sort)
         except InvalidLinkTable as err:
             bad_field = err.field_name
             if bad_field == already_fixed:

src/calibre/db/categories.py

@@ -15,7 +15,7 @@ from future_builtins import map
 from calibre.ebooks.metadata import author_to_author_sort
 from calibre.library.field_metadata import TagsIcons
 from calibre.utils.config_base import tweaks
-from calibre.utils.icu import sort_key
+from calibre.utils.icu import sort_key, collation_order
 
 CATEGORY_SORTS = ('name', 'popularity', 'rating')  # This has to be a tuple not a set
 
@@ -116,19 +116,22 @@ def clean_user_categories(dbcache):
         pass
     return new_cats
 
-def sort_categories(items, sort):
+def sort_categories(items, sort, first_letter_sort=False):
-    reverse = True
     if sort == 'popularity':
-        key=attrgetter('count')
+        key=lambda x:(-getattr(x, 'count', 0), sort_key(x.sort or x.name))
     elif sort == 'rating':
-        key=attrgetter('avg_rating')
+        key=lambda x:(-getattr(x, 'avg_rating', 0.0), sort_key(x.sort or x.name))
+    else:
+        if first_letter_sort:
+            key=lambda x:(collation_order(icu_upper(x.sort or x.name or ' ')),
+                          sort_key(x.sort or x.name))
         else:
             key=lambda x:sort_key(x.sort or x.name)
-        reverse=False
+    items.sort(key=key)
-    items.sort(key=key, reverse=reverse)
     return items
 
-def get_categories(dbcache, sort='name', book_ids=None, icon_map=None):
+def get_categories(dbcache, sort='name', book_ids=None, icon_map=None,
+                   first_letter_sort=False):
     if icon_map is not None and type(icon_map) != TagsIcons:
         raise TypeError('icon_map passed to get_categories must be of type TagIcons')
     if sort not in CATEGORY_SORTS:
@@ -170,7 +173,7 @@ def get_categories(dbcache, sort='name', book_ids=None, icon_map=None):
                 cat['is_multiple'] and cat['display'].get('is_names', False)):
                 for item in cats:
                     item.sort = author_to_author_sort(item.sort)
-        sort_categories(cats, sort)
+        sort_categories(cats, sort, first_letter_sort=first_letter_sort)
         categories[category] = cats
 
     # Needed for legacy databases that have multiple ratings that

src/calibre/gui2/tag_browser/model.py

@@ -865,16 +865,19 @@ class TagsModel(QAbstractItemModel):  # {{{
         # Get the categories
         if self.db.data.get_base_restriction() or self.db.data.get_search_restriction():
             try:
-                data = self.db.get_categories(sort=sort,
+                data = self.db.new_api.get_categories(sort=sort,
                         icon_map=self.category_icon_map,
-                        ids=self.db.search('', return_matches=True, sort_results=False))
+                        book_ids=self.db.search('', return_matches=True, sort_results=False),
+                        first_letter_sort = self.collapse_model == 'first letter')
             except:
                 import traceback
                 traceback.print_exc()
-                data = self.db.get_categories(sort=sort, icon_map=self.category_icon_map)
+                data = self.db.new_api.get_categories(sort=sort, icon_map=self.category_icon_map,
+                        first_letter_sort = self.collapse_model == 'first letter')
                 self.restriction_error.emit()
         else:
-            data = self.db.get_categories(sort=sort, icon_map=self.category_icon_map)
+            data = self.db.new_api.get_categories(sort=sort, icon_map=self.category_icon_map,
+                        first_letter_sort = self.collapse_model == 'first letter')
 
         # Reconstruct the user categories, putting them into metadata
         self.db.field_metadata.remove_dynamic_categories()