Bug #1948560: Odd behaviour with digits in alphabetical partitioning

This is fixed for the tag browser in the gui and the content server. I couldn't find anywhere else that used first letter partitioning.

When using first letter partitioning get_categories() constructs the sort key as follows:
- If the first character of the string is a digit (I hope isdigit() is localized) then use that digit otherwise use a string that is larger than any digit
- The ICU collation order key
- The sort key.

As such, items that start with a digit will first sort by that digit, then if needed use the ICU key as the second level then if needed use the sort key as the third level. Items that do not start with a digit will sort by the ICU key then the sort key.

src/calibre/db/categories.py

@@ -115,13 +115,22 @@ def clean_user_categories(dbcache):
     return new_cats
 
 
+numeric_collation = tweaks['numeric_collation']
+def first_digit(x):
+    global numeric_collation
+    c = icu_upper(x.sort or x.name or ' ')[0]
+    # The idea is that '9999999999' is larger than any digit so all digits
+    # will sort in front. Non-digits will sort according to their ICU first letter
+    return c if numeric_collation and c.isdigit() else '9999999999'
+
+
 category_sort_keys = {True:{}, False: {}}
 category_sort_keys[True]['popularity'] = category_sort_keys[False]['popularity'] = \
     lambda x:(-getattr(x, 'count', 0), sort_key(x.sort or x.name))
 category_sort_keys[True]['rating'] = category_sort_keys[False]['rating'] = \
     lambda x:(-getattr(x, 'avg_rating', 0.0), sort_key(x.sort or x.name))
 category_sort_keys[True]['name'] = \
-    lambda x:(collation_order(icu_upper(x.sort or x.name or ' ')), sort_key(x.sort or x.name))
+    lambda x:(first_digit(x), collation_order(icu_upper(x.sort or x.name or ' ')), sort_key(x.sort or x.name))
 category_sort_keys[False]['name'] = \
     lambda x:sort_key(x.sort or x.name)
 

src/calibre/gui2/tag_browser/model.py

@@ -24,8 +24,8 @@ from calibre.library.field_metadata import category_icon_map
 from calibre.utils.config import prefs, tweaks
 from calibre.utils.formatter import EvalFormatter
 from calibre.utils.icu import (
-    collation_order, contains, lower, primary_contains, primary_strcmp, sort_key,
-    strcmp
+    contains, lower, primary_contains, primary_strcmp, sort_key,
+    strcmp, collation_order_for_partitioning
 )
 from calibre.utils.serialize import json_dumps, json_loads
 from polyglot.builtins import iteritems, itervalues
@@ -556,7 +556,7 @@ class TagsModel(QAbstractItemModel):  # {{{
                     # Deal with items that don't have sorts, such as formats
                     t = tag.sort if tag.sort else tag.name
                     c = icu_upper(t) if t else ' '
-                    ordnum, ordlen = collation_order(c)
+                    ordnum, ordlen = collation_order_for_partitioning(c)
                     if last_ordnum != ordnum:
                         if fl_collapse and idx > 0:
                             intervals.append(FL_Interval(last_c, last_c, idx-last_idx))

src/calibre/srv/metadata.py

@@ -17,7 +17,7 @@ from calibre.utils.date import isoformat, UNDEFINED_DATE, local_tz
 from calibre.utils.config import tweaks
 from calibre.utils.formatter import EvalFormatter
 from calibre.utils.file_type_icons import EXT_MAP
-from calibre.utils.icu import collation_order
+from calibre.utils.icu import collation_order_for_partitioning
 from calibre.utils.localization import calibre_langcode_to_name
 from calibre.library.comments import comments_to_html, markdown
 from calibre.library.field_metadata import category_icon_map
@@ -291,7 +291,7 @@ def build_first_letter_list(category_items):
             c = ' '
         else:
             c = icu_upper(tag.sort)
-        ordnum, ordlen = collation_order(c)
+        ordnum, ordlen = collation_order_for_partitioning(c)
         if last_ordnum != ordnum:
             last_c = c[0:ordlen]
             last_ordnum = ordnum