Fix #7787 (Saved search dropdown box looses selected search) and use sort_keys everywhere

resources/default_tweaks.py

@@ -217,3 +217,15 @@ generate_cover_foot_font = None
 # open_viewer, do_nothing, edit_cell. Default: open_viewer.
 # Example: doubleclick_on_library_view = 'do_nothing'
 doubleclick_on_library_view = 'open_viewer'
+
+
+# Language to use when sorting. Setting this tweak will force sorting to use the
+# collating order for the specified language. This might be useful if you run
+# calibre in English but want sorting to work in the language where you live.
+# Set the tweak to the desired ISO 639-1 language code, in lower case.
+# You can find the list of supported locales at
+# http://publib.boulder.ibm.com/infocenter/iseries/v5r3/topic/nls/rbagsicusortsequencetables.htm
+# Default: locale_for_sorting = '' -- use the language calibre displays in
+# Example: locale_for_sorting = 'fr' -- sort using French rules.
+# Example: locale_for_sorting = 'nb' -- sort using Norwegian rules.
+locale_for_sorting =  ''

src/calibre/devices/usbms/books.py

@@ -13,6 +13,7 @@ from calibre.devices.interface import BookList as _BookList
 from calibre.constants import preferred_encoding
 from calibre import isbytestring
 from calibre.utils.config import prefs, tweaks
+from calibre.utils.icu import sort_key
 
 class Book(Metadata):
     def __init__(self, prefix, lpath, size=None, other=None):
@@ -215,14 +216,17 @@ class CollectionsBookList(BookList):
                     elif is_series:
                         if doing_dc:
                             collections[cat_name][lpath] = \
-                                (book, book.get('series_index', sys.maxint), '')
+                                (book, book.get('series_index', sys.maxint),
+                                 book.get('title_sort', 'zzzz'))
                         else:
                             collections[cat_name][lpath] = \
-                                (book, book.get(attr+'_index', sys.maxint), '')
+                                (book, book.get(attr+'_index', sys.maxint),
+                                 book.get('title_sort', 'zzzz'))
                     else:
                         if lpath not in collections[cat_name]:
                             collections[cat_name][lpath] = \
-                                (book, book.get('title_sort', 'zzzz'), '')
+                                (book, book.get('title_sort', 'zzzz'),
+                                 book.get('title_sort', 'zzzz'))
         # Sort collections
         result = {}
 
@@ -230,14 +234,19 @@ class CollectionsBookList(BookList):
             x = xx[1]
             y = yy[1]
             if x is None and y is None:
+                # No sort_key needed here, because defaults are ascii
                 return cmp(xx[2], yy[2])
             if x is None:
                 return 1
             if y is None:
                 return -1
-            c = cmp(x, y)
+            if isinstance(x, (unicode, str)):
+                c = cmp(sort_key(x), sort_key(y))
+            else:
+                c = cmp(x, y)
             if c != 0:
                 return c
+            # same as above -- no sort_key needed here
             return cmp(xx[2], yy[2])
 
         for category, lpaths in collections.items():

src/calibre/gui2/actions/edit_metadata.py

@@ -16,6 +16,7 @@ from calibre.gui2.dialogs.metadata_bulk import MetadataBulkDialog
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2.dialogs.tag_list_editor import TagListEditor
 from calibre.gui2.actions import InterfaceAction
+from calibre.utils.icu import sort_key
 
 class EditMetadataAction(InterfaceAction):
 
@@ -363,8 +364,7 @@ class EditMetadataAction(InterfaceAction):
     def edit_device_collections(self, view, oncard=None):
         model = view.model()
         result = model.get_collections_with_ids()
-        compare = (lambda x,y:cmp(x.lower(), y.lower()))
+        d = TagListEditor(self.gui, tag_to_match=None, data=result, key=sort_key)
-        d = TagListEditor(self.gui, tag_to_match=None, data=result, compare=compare)
         d.exec_()
         if d.result() == d.Accepted:
             to_rename = d.to_rename # dict of new text to old ids

src/calibre/gui2/book_details.py

@@ -19,6 +19,7 @@ from calibre.ebooks import BOOK_EXTENSIONS
 from calibre.constants import preferred_encoding
 from calibre.library.comments import comments_to_html
 from calibre.gui2 import config, open_local_file
+from calibre.utils.icu import sort_key
 
 # render_rows(data) {{{
 WEIGHTS = collections.defaultdict(lambda : 100)
@@ -31,8 +32,8 @@ WEIGHTS[_('Tags')] = 4
 def render_rows(data):
     keys = data.keys()
     # First sort by name. The WEIGHTS sort will preserve this sub-order
-    keys.sort(cmp=lambda x, y: cmp(x.lower(), y.lower()))
+    keys.sort(key=sort_key)
-    keys.sort(cmp=lambda x, y: cmp(WEIGHTS[x], WEIGHTS[y]))
+    keys.sort(key=lambda x: WEIGHTS[x])
     rows = []
     for key in keys:
         txt = data[key]

src/calibre/gui2/convert/metadata.py

@@ -17,6 +17,7 @@ from calibre.ebooks.metadata import authors_to_string, string_to_authors, \
 from calibre.ebooks.metadata.opf2 import metadata_to_opf
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.gui2.convert import Widget
+from calibre.utils.icu import sort_key
 
 def create_opf_file(db, book_id):
     mi = db.get_metadata(book_id, index_is_id=True)
@@ -102,7 +103,7 @@ class MetadataWidget(Widget, Ui_Form):
 
     def initalize_authors(self):
         all_authors = self.db.all_authors()
-        all_authors.sort(cmp=lambda x, y : cmp(x[1], y[1]))
+        all_authors.sort(key=lambda x : sort_key(x[1]))
 
         for i in all_authors:
             id, name = i
@@ -117,7 +118,7 @@ class MetadataWidget(Widget, Ui_Form):
 
     def initialize_series(self):
         all_series = self.db.all_series()
-        all_series.sort(cmp=lambda x, y : cmp(x[1], y[1]))
+        all_series.sort(key=lambda x : sort_key(x[1]))
 
         for i in all_series:
             id, name = i
@@ -126,7 +127,7 @@ class MetadataWidget(Widget, Ui_Form):
 
     def initialize_publisher(self):
         all_publishers = self.db.all_publishers()
-        all_publishers.sort(cmp=lambda x, y : cmp(x[1], y[1]))
+        all_publishers.sort(key=lambda x : sort_key(x[1]))
 
         for i in all_publishers:
             id, name = i

src/calibre/gui2/custom_column_widgets.py

@@ -17,6 +17,7 @@ from calibre.utils.date import qt_to_dt, now
 from calibre.gui2.widgets import TagsLineEdit, EnComboBox
 from calibre.gui2 import UNDEFINED_QDATE, error_dialog
 from calibre.utils.config import tweaks
+from calibre.utils.icu import sort_key
 
 class Base(object):
 
@@ -207,7 +208,7 @@ class Text(Base):
 
     def setup_ui(self, parent):
         values = self.all_values = list(self.db.all_custom(num=self.col_id))
-        values.sort(cmp = lambda x,y: cmp(x.lower(), y.lower()))
+        values.sort(key=sort_key)
         if self.col_metadata['is_multiple']:
             w = TagsLineEdit(parent, values)
             w.setSizePolicy(QSizePolicy.Minimum, QSizePolicy.Preferred)
@@ -256,7 +257,7 @@ class Series(Base):
 
     def setup_ui(self, parent):
         values = self.all_values = list(self.db.all_custom(num=self.col_id))
-        values.sort(cmp = lambda x,y: cmp(x.lower(), y.lower()))
+        values.sort(key=sort_key)
         w = EnComboBox(parent)
         w.setSizeAdjustPolicy(w.AdjustToMinimumContentsLengthWithIcon)
         w.setMinimumContentsLength(25)
@@ -365,11 +366,10 @@ widgets = {
         'enumeration': Enumeration
 }
 
-def field_sort(y, z, x=None):
+def field_sort_key(y, x=None):
-    m1, m2 = x[y], x[z]
+    m1 = x[y]
     n1 = 'zzzzz' if m1['datatype'] == 'comments' else m1['name']
-    n2 = 'zzzzz' if m2['datatype'] == 'comments' else m2['name']
+    return sort_key(n1)
-    return cmp(n1.lower(), n2.lower())
 
 def populate_metadata_page(layout, db, book_id, bulk=False, two_column=False, parent=None):
     def widget_factory(type, col):
@@ -381,7 +381,7 @@ def populate_metadata_page(layout, db, book_id, bulk=False, two_column=False, pa
         return w
     x = db.custom_column_num_map
     cols = list(x)
-    cols.sort(cmp=partial(field_sort, x=x))
+    cols.sort(key=partial(field_sort_key, x=x))
     count_non_comment = len([c for c in cols if x[c]['datatype'] != 'comments'])
 
     layout.setColumnStretch(1, 10)
@@ -526,7 +526,7 @@ class BulkSeries(BulkBase):
 
     def setup_ui(self, parent):
         values = self.all_values = list(self.db.all_custom(num=self.col_id))
-        values.sort(cmp = lambda x,y: cmp(x.lower(), y.lower()))
+        values.sort(key=sort_key)
         w = EnComboBox(parent)
         w.setSizeAdjustPolicy(w.AdjustToMinimumContentsLengthWithIcon)
         w.setMinimumContentsLength(25)
@@ -678,7 +678,7 @@ class BulkText(BulkBase):
 
     def setup_ui(self, parent):
         values = self.all_values = list(self.db.all_custom(num=self.col_id))
-        values.sort(cmp = lambda x,y: cmp(x.lower(), y.lower()))
+        values.sort(key=sort_key)
         if self.col_metadata['is_multiple']:
             w = TagsLineEdit(parent, values)
             w.setSizePolicy(QSizePolicy.Minimum, QSizePolicy.Preferred)

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -17,6 +17,7 @@ from calibre.gui2 import error_dialog
 from calibre.gui2.progress_indicator import ProgressIndicator
 from calibre.utils.config import dynamic
 from calibre.utils.titlecase import titlecase
+from calibre.utils.icu import sort_key
 
 class MyBlockingBusy(QDialog):
 
@@ -594,7 +595,7 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
 
     def initalize_authors(self):
         all_authors = self.db.all_authors()
-        all_authors.sort(cmp=lambda x, y : cmp(x[1].lower(), y[1].lower()))
+        all_authors.sort(key=lambda x : sort_key(x[1]))
 
         for i in all_authors:
             id, name = i
@@ -604,7 +605,7 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
 
     def initialize_series(self):
         all_series = self.db.all_series()
-        all_series.sort(cmp=lambda x, y : cmp(x[1], y[1]))
+        all_series.sort(key=lambda x : sort_key(x[1]))
 
         for i in all_series:
             id, name = i
@@ -613,7 +614,7 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
 
     def initialize_publisher(self):
         all_publishers = self.db.all_publishers()
-        all_publishers.sort(cmp=lambda x, y : cmp(x[1], y[1]))
+        all_publishers.sort(key=lambda x : sort_key(x[1]))
 
         for i in all_publishers:
             id, name = i

src/calibre/gui2/dialogs/metadata_single.py

@@ -28,6 +28,7 @@ from calibre.ebooks.metadata.meta import get_metadata
 from calibre.ebooks.metadata import MetaInformation
 from calibre.utils.config import prefs, tweaks
 from calibre.utils.date import qt_to_dt, local_tz, utcfromtimestamp
+from calibre.utils.icu import sort_key
 from calibre.customize.ui import run_plugins_on_import, get_isbndb_key
 from calibre.gui2.preferences.social import SocialMetadata
 from calibre.gui2.custom_column_widgets import populate_metadata_page
@@ -660,7 +661,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
 
     def initalize_authors(self):
         all_authors = self.db.all_authors()
-        all_authors.sort(cmp=lambda x, y : cmp(x[1], y[1]))
+        all_authors.sort(key=lambda x : sort_key(x[1]))
         for i in all_authors:
             id, name = i
             name = [name.strip().replace('|', ',') for n in name.split(',')]
@@ -675,7 +676,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
     def initialize_series(self):
         self.series.setSizeAdjustPolicy(self.series.AdjustToContentsOnFirstShow)
         all_series = self.db.all_series()
-        all_series.sort(cmp=lambda x, y : cmp(x[1], y[1]))
+        all_series.sort(key=lambda x : sort_key(x[1]))
         series_id = self.db.series_id(self.row)
         idx, c = None, 0
         for i in all_series:
@@ -692,7 +693,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
 
     def initialize_publisher(self):
         all_publishers = self.db.all_publishers()
-        all_publishers.sort(cmp=lambda x, y : cmp(x[1], y[1]))
+        all_publishers.sort(key=lambda x : sort_key(x[1]))
         publisher_id = self.db.publisher_id(self.row)
         idx, c = None, 0
         for i in all_publishers:

src/calibre/gui2/dialogs/saved_search_editor.py

@@ -8,6 +8,7 @@ from PyQt4.QtGui import QDialog
 
 from calibre.gui2.dialogs.saved_search_editor_ui import Ui_SavedSearchEditor
 from calibre.utils.search_query_parser import saved_searches
+from calibre.utils.icu import sort_key
 from calibre.gui2.dialogs.confirm_delete import confirm
 
 class SavedSearchEditor(QDialog, Ui_SavedSearchEditor):
@@ -34,7 +35,7 @@ class SavedSearchEditor(QDialog, Ui_SavedSearchEditor):
 
     def populate_search_list(self):
         self.search_name_box.clear()
-        for name in sorted(self.searches.keys()):
+        for name in sorted(self.searches.keys(), key=sort_key):
             self.search_name_box.addItem(name)
 
     def add_search(self):

src/calibre/gui2/dialogs/search.py

@@ -8,6 +8,7 @@ from PyQt4.QtGui import QDialog, QDialogButtonBox
 from calibre.gui2.dialogs.search_ui import Ui_Dialog
 from calibre.library.caches import CONTAINS_MATCH, EQUALS_MATCH
 from calibre.gui2 import gprefs
+from calibre.utils.icu import sort_key
 
 box_values = {}
 
@@ -18,8 +19,7 @@ class SearchDialog(QDialog, Ui_Dialog):
         self.setupUi(self)
         self.mc = ''
         searchables = sorted(db.field_metadata.searchable_fields(),
-                             lambda x, y: cmp(x if x[0] != '#' else x[1:],
+                             key=lambda x: sort_key(x if x[0] != '#' else x[1:]))
-                                              y if y[0] != '#' else y[1:]))
         self.general_combo.addItems(searchables)
 
         self.box_last_values = copy.deepcopy(box_values)

src/calibre/gui2/dialogs/tag_categories.py

@@ -9,6 +9,7 @@ from PyQt4.QtGui import QDialog, QIcon, QListWidgetItem
 from calibre.gui2.dialogs.tag_categories_ui import Ui_TagCategories
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.constants import islinux
+from calibre.utils.icu import sort_key
 
 class Item:
     def __init__(self, name, label, index, icon, exists):
@@ -85,7 +86,7 @@ class TagCategories(QDialog, Ui_TagCategories):
                     # remove any references to a category that no longer exists
                     del self.categories[cat][item]
 
-        self.all_items_sorted = sorted(self.all_items, cmp=lambda x,y: cmp(x.name.lower(), y.name.lower()))
+        self.all_items_sorted = sorted(self.all_items, key=lambda x: sort_key(x.name))
         self.display_filtered_categories(0)
 
         for v in category_names:
@@ -135,7 +136,7 @@ class TagCategories(QDialog, Ui_TagCategories):
             index = self.all_items[node.data(Qt.UserRole).toPyObject()].index
             if index not in self.applied_items:
                 self.applied_items.append(index)
-        self.applied_items.sort(cmp=lambda x, y:cmp(self.all_items[x].name.lower(), self.all_items[y].name.lower()))
+        self.applied_items.sort(key=lambda x:sort_key(self.all_items[x]))
         self.display_filtered_categories(None)
 
     def unapply_tags(self, node=None):
@@ -198,5 +199,5 @@ class TagCategories(QDialog, Ui_TagCategories):
             self.categories[self.current_cat_name] = l
 
     def populate_category_list(self):
-        for n in sorted(self.categories.keys(), cmp=lambda x,y: cmp(x.lower(), y.lower())):
+        for n in sorted(self.categories.keys(), key=sort_key):
             self.category_box.addItem(n)

src/calibre/gui2/dialogs/tag_editor.py

@@ -6,12 +6,10 @@ from PyQt4.QtGui import QDialog
 from calibre.gui2.dialogs.tag_editor_ui import Ui_TagEditor
 from calibre.gui2 import question_dialog, error_dialog
 from calibre.constants import islinux
+from calibre.utils.icu import sort_key
 
 class TagEditor(QDialog, Ui_TagEditor):
 
-    def tag_cmp(self, x, y):
-        return cmp(x.lower(), y.lower())
-
     def __init__(self, window, db, index=None):
         QDialog.__init__(self, window)
         Ui_TagEditor.__init__(self)
@@ -25,7 +23,7 @@ class TagEditor(QDialog, Ui_TagEditor):
             tags = []
         if tags:
             tags = [tag.strip() for tag in tags.split(',') if tag.strip()]
-            tags.sort(cmp=self.tag_cmp)
+            tags.sort(key=sort_key)
             for tag in tags:
                 self.applied_tags.addItem(tag)
         else:
@@ -35,7 +33,7 @@ class TagEditor(QDialog, Ui_TagEditor):
 
         all_tags = [tag for tag in self.db.all_tags()]
         all_tags = list(set(all_tags))
-        all_tags.sort(cmp=self.tag_cmp)
+        all_tags.sort(key=sort_key)
         for tag in all_tags:
             if tag not in tags:
                 self.available_tags.addItem(tag)
@@ -82,7 +80,7 @@ class TagEditor(QDialog, Ui_TagEditor):
             self.tags.append(tag)
             self.available_tags.takeItem(self.available_tags.row(item))
 
-        self.tags.sort(cmp=self.tag_cmp)
+        self.tags.sort(key=sort_key)
         self.applied_tags.clear()
         for tag in self.tags:
             self.applied_tags.addItem(tag)
@@ -96,14 +94,14 @@ class TagEditor(QDialog, Ui_TagEditor):
             self.tags.remove(tag)
             self.available_tags.addItem(tag)
 
-        self.tags.sort(cmp=self.tag_cmp)
+        self.tags.sort(key=sort_key)
         self.applied_tags.clear()
         for tag in self.tags:
             self.applied_tags.addItem(tag)
 
         items = [unicode(self.available_tags.item(x).text()) for x in
                 range(self.available_tags.count())]
-        items.sort(cmp=self.tag_cmp)
+        items.sort(key=sort_key)
         self.available_tags.clear()
         for item in items:
             self.available_tags.addItem(item)
@@ -117,7 +115,7 @@ class TagEditor(QDialog, Ui_TagEditor):
             if tag not in self.tags:
                 self.tags.append(tag)
 
-        self.tags.sort(cmp=self.tag_cmp)
+        self.tags.sort(key=sort_key)
         self.applied_tags.clear()
         for tag in self.tags:
             self.applied_tags.addItem(tag)

src/calibre/gui2/dialogs/tag_list_editor.py

@@ -39,7 +39,7 @@ class ListWidgetItem(QListWidgetItem):
 
 class TagListEditor(QDialog, Ui_TagListEditor):
 
-    def __init__(self, window, tag_to_match, data, compare):
+    def __init__(self, window, tag_to_match, data, key):
         QDialog.__init__(self, window)
         Ui_TagListEditor.__init__(self)
         self.setupUi(self)
@@ -54,7 +54,7 @@ class TagListEditor(QDialog, Ui_TagListEditor):
 
         for k,v in data:
             self.all_tags[v] = k
-        for tag in sorted(self.all_tags.keys(), cmp=compare):
+        for tag in sorted(self.all_tags.keys(), key=key):
             item = ListWidgetItem(tag)
             item.setData(Qt.UserRole, self.all_tags[tag])
             self.available_tags.addItem(item)

src/calibre/gui2/dialogs/user_profiles.py

@@ -13,6 +13,7 @@ from calibre.gui2 import error_dialog, question_dialog, open_url, \
                          choose_files, ResizableDialog, NONE
 from calibre.gui2.widgets import PythonHighlighter
 from calibre.ptempfile import PersistentTemporaryFile
+from calibre.utils.icu import sort_key
 
 class CustomRecipeModel(QAbstractListModel):
 
@@ -256,7 +257,7 @@ class %(classname)s(%(base_class)s):
     def add_builtin_recipe(self):
         from calibre.web.feeds.recipes.collection import \
             get_builtin_recipe_by_title, get_builtin_recipe_titles
-        items = sorted(get_builtin_recipe_titles())
+        items = sorted(get_builtin_recipe_titles(), key=sort_key)
 
 
         title, ok = QInputDialog.getItem(self, _('Pick recipe'), _('Pick the recipe to customize'),

src/calibre/gui2/library/delegates.py

@@ -20,6 +20,7 @@ from calibre.gui2.widgets import EnLineEdit, TagsLineEdit
 from calibre.utils.date import now, format_date
 from calibre.utils.config import tweaks
 from calibre.utils.formatter import validation_formatter
+from calibre.utils.icu import sort_key
 from calibre.gui2.dialogs.comments_dialog import CommentsDialog
 
 class RatingDelegate(QStyledItemDelegate): # {{{
@@ -173,7 +174,8 @@ class TagsDelegate(QStyledItemDelegate): # {{{
                 editor = TagsLineEdit(parent, self.db.all_tags())
             else:
                 editor = TagsLineEdit(parent,
-                        sorted(list(self.db.all_custom(label=self.db.field_metadata.key_to_label(col)))))
+                        sorted(list(self.db.all_custom(label=self.db.field_metadata.key_to_label(col))),
+                               key=sort_key))
                 return editor
         else:
             editor = EnLineEdit(parent)
@@ -245,7 +247,8 @@ class CcTextDelegate(QStyledItemDelegate): # {{{
             editor.setDecimals(2)
         else:
             editor = EnLineEdit(parent)
-            complete_items = sorted(list(m.db.all_custom(label=m.db.field_metadata.key_to_label(col))))
+            complete_items = sorted(list(m.db.all_custom(label=m.db.field_metadata.key_to_label(col))),
+                                    key=sort_key)
             completer = QCompleter(complete_items, self)
             completer.setCaseSensitivity(Qt.CaseInsensitive)
             completer.setCompletionMode(QCompleter.PopupCompletion)

src/calibre/gui2/library/models.py

@@ -18,6 +18,7 @@ from calibre.ebooks.metadata import fmt_sidx, authors_to_string, string_to_autho
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.utils.config import tweaks, prefs
 from calibre.utils.date import dt_factory, qt_to_dt, isoformat
+from calibre.utils.icu import sort_key
 from calibre.ebooks.metadata.meta import set_metadata as _set_metadata
 from calibre.utils.search_query_parser import SearchQueryParser
 from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, \
@@ -305,9 +306,10 @@ class BooksModel(QAbstractTableModel): # {{{
         cdata = self.cover(idx)
         if cdata:
             data['cover'] = cdata
-        tags = self.db.tags(idx)
+        tags = list(self.db.get_tags(self.db.id(idx)))
         if tags:
-            tags = tags.replace(',', ', ')
+            tags.sort(key=sort_key)
+            tags = ', '.join(tags)
         else:
             tags = _('None')
         data[_('Tags')] = tags
@@ -544,7 +546,7 @@ class BooksModel(QAbstractTableModel): # {{{
         def tags(r, idx=-1):
             tags = self.db.data[r][idx]
             if tags:
-                return QVariant(', '.join(sorted(tags.split(','))))
+                return QVariant(', '.join(sorted(tags.split(','), key=sort_key)))
             return None
 
         def series_type(r, idx=-1, siix=-1):
@@ -595,7 +597,7 @@ class BooksModel(QAbstractTableModel): # {{{
         def text_type(r, mult=False, idx=-1):
             text = self.db.data[r][idx]
             if text and mult:
-                return QVariant(', '.join(sorted(text.split('|'))))
+                return QVariant(', '.join(sorted(text.split('|'),key=sort_key)))
             return QVariant(text)
 
         def number_type(r, idx=-1):
@@ -1033,8 +1035,8 @@ class DeviceBooksModel(BooksModel): # {{{
             x, y = int(self.db[x].size), int(self.db[y].size)
             return cmp(x, y)
         def tagscmp(x, y):
-            x = ','.join(sorted(getattr(self.db[x], 'device_collections', []))).lower()
+            x = ','.join(sorted(getattr(self.db[x], 'device_collections', []),key=sort_key))
-            y = ','.join(sorted(getattr(self.db[y], 'device_collections', []))).lower()
+            y = ','.join(sorted(getattr(self.db[y], 'device_collections', []),key=sort_key))
             return cmp(x, y)
         def libcmp(x, y):
             x, y = self.db[x].in_library, self.db[y].in_library
@@ -1211,7 +1213,7 @@ class DeviceBooksModel(BooksModel): # {{{
             elif cname == 'collections':
                 tags = self.db[self.map[row]].device_collections
                 if tags:
-                    tags.sort(cmp=lambda x,y: cmp(x.lower(), y.lower()))
+                    tags.sort(key=sort_key)
                     return QVariant(', '.join(tags))
             elif DEBUG and cname == 'inlibrary':
                 return QVariant(self.db[self.map[row]].in_library)

src/calibre/gui2/preferences/behavior.py

@@ -19,6 +19,7 @@ from calibre.utils.search_query_parser import saved_searches
 from calibre.ebooks import BOOK_EXTENSIONS
 from calibre.ebooks.oeb.iterator import is_supported
 from calibre.constants import iswindows
+from calibre.utils.icu import sort_key
 
 class ConfigWidget(ConfigWidgetBase, Ui_Form):
 
@@ -45,8 +46,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         choices = [(x.upper(), x) for x in output_formats]
         r('output_format', prefs, choices=choices)
 
-        restrictions = sorted(saved_searches().names(),
+        restrictions = sorted(saved_searches().names(), key=sort_key)
-                              cmp=lambda x,y: cmp(x.lower(), y.lower()))
         choices = [('', '')] + [(x, x) for x in restrictions]
         r('gui_restriction', db.prefs, choices=choices)
         r('new_book_tags', prefs, setting=CommaSeparatedList)

src/calibre/gui2/search_box.py

@@ -17,6 +17,7 @@ from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2.dialogs.saved_search_editor import SavedSearchEditor
 from calibre.gui2.dialogs.search import SearchDialog
 from calibre.utils.search_query_parser import saved_searches
+from calibre.utils.icu import sort_key
 
 class SearchLineEdit(QLineEdit): # {{{
     key_pressed = pyqtSignal(object)
@@ -204,7 +205,7 @@ class SearchBox2(QComboBox): # {{{
         self.blockSignals(yes)
         self.line_edit.blockSignals(yes)
 
-    def set_search_string(self, txt, store_in_history=False):
+    def set_search_string(self, txt, store_in_history=False, emit_changed=True):
         self.setFocus(Qt.OtherFocusReason)
         if not txt:
             self.clear()
@@ -212,7 +213,8 @@ class SearchBox2(QComboBox): # {{{
             self.normalize_state()
             self.setEditText(txt)
             self.line_edit.end(False)
-            self.changed.emit()
+            if emit_changed:
+                self.changed.emit()
             self._do_search(store_in_history=store_in_history)
         self.focus_to_library.emit()
 
@@ -292,7 +294,7 @@ class SavedSearchBox(QComboBox): # {{{
             self.search_box.clear()
             self.setEditText(qname)
             return
-        self.search_box.set_search_string(u'search:"%s"' % qname)
+        self.search_box.set_search_string(u'search:"%s"' % qname, emit_changed=False)
         self.setEditText(qname)
         self.setToolTip(saved_searches().lookup(qname))
 
@@ -417,7 +419,7 @@ class SavedSearchBoxMixin(object): # {{{
             b.setStatusTip(b.toolTip())
 
     def saved_searches_changed(self):
-        p = sorted(saved_searches().names(), cmp=lambda x,y: cmp(x.lower(), y.lower()))
+        p = sorted(saved_searches().names(), key=sort_key)
         t = unicode(self.search_restriction.currentText())
         # rebuild the restrictions combobox using current saved searches
         self.search_restriction.clear()

src/calibre/gui2/shortcuts.py

@@ -14,6 +14,7 @@ from PyQt4.Qt import QAbstractListModel, Qt, QKeySequence, QListView, \
 
 from calibre.gui2 import NONE, error_dialog
 from calibre.utils.config import XMLConfig
+from calibre.utils.icu import sort_key
 from calibre.gui2.shortcuts_ui import Ui_Frame
 
 DEFAULTS = Qt.UserRole
@@ -175,8 +176,7 @@ class Shortcuts(QAbstractListModel):
         for k, v in shortcuts.items():
             self.keys[k] = v[0]
         self.order = list(shortcuts)
-        self.order.sort(cmp=lambda x,y : cmp(self.descriptions[x],
+        self.order.sort(key=lambda x : sort_key(self.descriptions[x]))
-            self.descriptions[y]))
         self.sequences = {}
         for k, v in self.keys.items():
             self.sequences[k] = [QKeySequence(x) for x in v]

src/calibre/gui2/tag_view.py

@@ -18,6 +18,7 @@ from PyQt4.Qt import Qt, QTreeView, QApplication, pyqtSignal, \
 from calibre.ebooks.metadata import title_sort
 from calibre.gui2 import config, NONE
 from calibre.library.field_metadata import TagsIcons, category_icon_map
+from calibre.utils.icu import sort_key
 from calibre.utils.search_query_parser import saved_searches
 from calibre.gui2 import error_dialog
 from calibre.gui2.dialogs.confirm_delete import confirm
@@ -225,7 +226,7 @@ class TagsView(QTreeView): # {{{
                     partial(self.context_menu_handler, action='hide', category=category))
                 if self.hidden_categories:
                     m = self.context_menu.addMenu(_('Show category'))
-                    for col in sorted(self.hidden_categories, cmp=lambda x,y: cmp(x.lower(), y.lower())):
+                    for col in sorted(self.hidden_categories, key=sort_key):
                         m.addAction(col,
                             partial(self.context_menu_handler, action='show', category=col))
 
@@ -599,7 +600,8 @@ class TagsModel(QAbstractItemModel): # {{{
         # Reconstruct the user categories, putting them into metadata
         self.db.field_metadata.remove_dynamic_categories()
         tb_cats = self.db.field_metadata
-        for user_cat in sorted(self.db.prefs.get('user_categories', {}).keys()):
+        for user_cat in sorted(self.db.prefs.get('user_categories', {}).keys(),
+                               key=sort_key):
             cat_name = user_cat+':' # add the ':' to avoid name collision
             tb_cats.add_user_category(label=cat_name, name=user_cat)
         if len(saved_searches().names()):
@@ -878,13 +880,13 @@ class TagBrowserMixin(object): # {{{
         db=self.library_view.model().db
         if category == 'tags':
             result = db.get_tags_with_ids()
-            compare = (lambda x,y:cmp(x.lower(), y.lower()))
+            key = sort_key
         elif category == 'series':
             result = db.get_series_with_ids()
-            compare = (lambda x,y:cmp(title_sort(x).lower(), title_sort(y).lower()))
+            key = lambda x:sort_key(title_sort(x))
         elif category == 'publisher':
             result = db.get_publishers_with_ids()
-            compare = (lambda x,y:cmp(x.lower(), y.lower()))
+            key = sort_key
         else: # should be a custom field
             cc_label = None
             if category in db.field_metadata:
@@ -892,9 +894,9 @@ class TagBrowserMixin(object): # {{{
                 result = db.get_custom_items_with_ids(label=cc_label)
             else:
                 result = []
-            compare = (lambda x,y:cmp(x.lower(), y.lower()))
+            key = sort_key
 
-        d = TagListEditor(self, tag_to_match=tag, data=result, compare=compare)
+        d = TagListEditor(self, tag_to_match=tag, data=result, key=key)
         d.exec_()
         if d.result() == d.Accepted:
             to_rename = d.to_rename # dict of new text to old id

src/calibre/library/database2.py

@@ -14,6 +14,7 @@ from operator import itemgetter
 
 from PyQt4.QtGui import QImage
 
+
 from calibre.ebooks.metadata import title_sort, author_to_author_sort
 from calibre.ebooks.metadata.opf2 import metadata_to_opf
 from calibre.library.database import LibraryDatabase
@@ -33,6 +34,7 @@ from calibre import isbytestring
 from calibre.utils.filenames import ascii_filename
 from calibre.utils.date import utcnow, now as nowf, utcfromtimestamp
 from calibre.utils.config import prefs, tweaks
+from calibre.utils.icu import sort_key
 from calibre.utils.search_query_parser import saved_searches, set_saved_searches
 from calibre.ebooks import BOOK_EXTENSIONS, check_ebook_format
 from calibre.utils.magick.draw import save_cover_data_to
@@ -287,7 +289,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         # Assumption is that someone else will fix them if they change.
         self.field_metadata.remove_dynamic_categories()
         tb_cats = self.field_metadata
-        for user_cat in sorted(self.prefs.get('user_categories', {}).keys()):
+        for user_cat in sorted(self.prefs.get('user_categories', {}).keys(), key=sort_key):
             cat_name = user_cat+':' # add the ':' to avoid name collision
             tb_cats.add_user_category(label=cat_name, name=user_cat)
         if len(saved_searches().names()):
@@ -1065,7 +1067,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             if sort == 'popularity':
                 query += ' ORDER BY count DESC, sort ASC'
             elif sort == 'name':
-                query += ' ORDER BY sort ASC'
+                query += ' ORDER BY sort COLLATE icucollate'
             else:
                 query += ' ORDER BY avg_rating DESC, sort ASC'
             data = self.conn.get(query)
@@ -1137,6 +1139,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if sort == 'popularity':
             categories['formats'].sort(key=lambda x: x.count, reverse=True)
         else: # no ratings exist to sort on
+            # No need for ICU here.
             categories['formats'].sort(key = lambda x:x.name)
 
         #### Now do the user-defined categories. ####
@@ -1151,7 +1154,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         for c in categories.keys():
             taglist[c] = dict(map(lambda t:(t.name, t), categories[c]))
 
-        for user_cat in sorted(user_categories.keys()):
+        for user_cat in sorted(user_categories.keys(), key=sort_key):
             items = []
             for (name,label,ign) in user_categories[user_cat]:
                 if label in taglist and name in taglist[label]:
@@ -1167,7 +1170,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                         sorted(items, key=lambda x: x.count, reverse=True)
                 elif sort == 'name':
                     categories[cat_name] = \
-                        sorted(items, key=lambda x: x.sort.lower())
+                        sorted(items, key=lambda x: sort_key(x.sort))
                 else:
                     categories[cat_name] = \
                         sorted(items, key=lambda x:x.avg_rating, reverse=True)

src/calibre/library/server/browse.py

@@ -16,6 +16,7 @@ from calibre import isbytestring, force_unicode, fit_image, \
 from calibre.utils.ordered_dict import OrderedDict
 from calibre.utils.filenames import ascii_filename
 from calibre.utils.config import prefs
+from calibre.utils.icu import sort_key
 from calibre.utils.magick import Image
 from calibre.library.comments import comments_to_html
 from calibre.library.server import custom_fields_to_display
@@ -273,7 +274,7 @@ class BrowseServer(object):
         opts = ['<option %svalue="%s">%s</option>' % (
             'selected="selected" ' if k==sort else '',
             xml(k), xml(n), ) for k, n in
-                sorted(sort_opts, key=operator.itemgetter(1)) if k and n]
+                sorted(sort_opts, key=lambda x: sort_key(operator.itemgetter(1)(x))) if k and n]
         ans = ans.replace('{sort_select_options}', ('\n'+' '*20).join(opts))
         lp = self.db.library_path
         if isbytestring(lp):
@@ -337,8 +338,7 @@ class BrowseServer(object):
             return category_meta[x]['name'].lower()
 
         displayed_custom_fields = custom_fields_to_display(self.db)
-        for category in sorted(categories,
+        for category in sorted(categories, key=lambda x: sort_key(getter(x))):
-                    cmp=lambda x,y: cmp(getter(x), getter(y))):
             if len(categories[category]) == 0:
                 continue
             if category == 'formats':
@@ -375,12 +375,7 @@ class BrowseServer(object):
     def browse_sort_categories(self, items, sort):
         if sort not in ('rating', 'name', 'popularity'):
             sort = 'name'
-        def sorter(x):
+        items.sort(key=lambda x: sort_key(getattr(x, 'sort', x.name)))
-            ans = getattr(x, 'sort', x.name)
-            if hasattr(ans, 'upper'):
-                ans = ans.upper()
-            return ans
-        items.sort(key=sorter)
         if sort == 'popularity':
             items.sort(key=operator.attrgetter('count'), reverse=True)
         elif sort == 'rating':
@@ -703,7 +698,7 @@ class BrowseServer(object):
                                 args[field]
                 fields.append((m['name'], r))
 
-            fields.sort(key=lambda x: x[0].lower())
+            fields.sort(key=lambda x: sort_key(x[0]))
             fields = [u'<div class="field">{0}</div>'.format(f[1]) for f in
                     fields]
             fields = u'<div class="fields">%s</div>'%('\n\n'.join(fields))

src/calibre/library/server/mobile.py

@@ -21,6 +21,7 @@ from calibre.constants import __appname__
 from calibre import human_readable, isbytestring
 from calibre.utils.date import utcfromtimestamp
 from calibre.utils.filenames import ascii_filename
+from calibre.utils.icu import sort_key
 
 def CLASS(*args, **kwargs): # class is a reserved word in Python
     kwargs['class'] = ' '.join(args)
@@ -211,8 +212,7 @@ class MobileServer(object):
 
         CFM = self.db.field_metadata
         CKEYS = [key for key in sorted(custom_fields_to_display(self.db),
-             cmp=lambda x,y: cmp(CFM[x]['name'].lower(),
+                                       key=lambda x:sort_key(CFM[x]['name']))]
-                                 CFM[y]['name'].lower()))]
         # This method uses its own book dict, not the Metadata dict. The loop
         # below could be changed to use db.get_metadata instead of reading
         # info directly from the record made by the view, but it doesn't seem

src/calibre/library/server/opds.py

@@ -20,6 +20,7 @@ from calibre.library.comments import comments_to_html
 from calibre.library.server import custom_fields_to_display
 from calibre.library.server.utils import format_tag_string, Offsets
 from calibre import guess_type
+from calibre.utils.icu import sort_key
 from calibre.utils.ordered_dict import OrderedDict
 
 BASE_HREFS = {
@@ -279,8 +280,7 @@ class AcquisitionFeed(NavFeed):
         NavFeed.__init__(self, id_, updated, version, offsets, page_url, up_url)
         CFM = db.field_metadata
         CKEYS = [key for key in sorted(custom_fields_to_display(db),
-                 cmp=lambda x,y: cmp(CFM[x]['name'].lower(),
+                                       key=lambda x: sort_key(CFM[x]['name']))]
-                                     CFM[y]['name'].lower()))]
         for item in items:
             self.root.append(ACQUISITION_ENTRY(item, version, db, updated,
                                                CFM, CKEYS, prefix))
@@ -492,7 +492,7 @@ class OPDSServer(object):
                     val = 'A'
                 starts.add(val[0].upper())
             category_groups = OrderedDict()
-            for x in sorted(starts, cmp=lambda x,y:cmp(x.lower(), y.lower())):
+            for x in sorted(starts, key=sort_key):
                 category_groups[x] = len([y for y in items if
                     getattr(y, 'sort', y.name).startswith(x)])
             items = [Group(x, y) for x, y in category_groups.items()]
@@ -571,8 +571,7 @@ class OPDSServer(object):
                 ]
         def getter(x):
             return category_meta[x]['name'].lower()
-        for category in sorted(categories,
+        for category in sorted(categories, key=lambda x: sort_key(getter(x))):
-                               cmp=lambda x,y: cmp(getter(x), getter(y))):
             if len(categories[category]) == 0:
                 continue
             if category == 'formats':

src/calibre/library/server/utils.py

@@ -13,6 +13,7 @@ import cherrypy
 from calibre import strftime as _strftime, prints, isbytestring
 from calibre.utils.date import now as nowf
 from calibre.utils.config import tweaks
+from calibre.utils.icu import sort_key
 
 class Offsets(object):
     'Calculate offsets for a paginated view'
@@ -73,7 +74,7 @@ def format_tag_string(tags, sep, ignore_max=False, no_tag_count=False):
         tlist = [t.strip() for t in tags.split(sep)]
     else:
         tlist = []
-    tlist.sort(cmp=lambda x,y:cmp(x.lower(), y.lower()))
+    tlist.sort(key=sort_key)
     if len(tlist) > MAX:
         tlist = tlist[:MAX]+['...']
     if no_tag_count:

src/calibre/library/server/xml.py

@@ -17,6 +17,7 @@ from calibre.ebooks.metadata import fmt_sidx
 from calibre.constants import preferred_encoding
 from calibre import isbytestring
 from calibre.utils.filenames import ascii_filename
+from calibre.utils.icu import sort_key
 
 E = ElementMaker()
 
@@ -101,8 +102,7 @@ class XMLServer(object):
 
             CFM = self.db.field_metadata
             CKEYS = [key for key in sorted(custom_fields_to_display(self.db),
-                 cmp=lambda x,y: cmp(CFM[x]['name'].lower(),
+                                           key=lambda x: sort_key(CFM[x]['name']))]
-                                     CFM[y]['name'].lower()))]
             custcols = []
             for key in CKEYS:
                 def concat(name, val):

src/calibre/library/sqlite.py

@@ -115,6 +115,9 @@ def pynocase(one, two, encoding='utf-8'):
             pass
     return cmp(one.lower(), two.lower())
 
+def icu_collator(s1, s2, func=None):
+    return cmp(func(unicode(s1)), func(unicode(s2)))
+
 def load_c_extensions(conn, debug=DEBUG):
     try:
         conn.enable_load_extension(True)
@@ -166,6 +169,8 @@ class DBThread(Thread):
         self.conn.create_function('uuid4', 0, lambda : str(uuid.uuid4()))
         # Dummy functions for dynamically created filters
         self.conn.create_function('books_list_filter', 1, lambda x: 1)
+        from calibre.utils.icu import sort_key
+        self.conn.create_collation('icucollate', partial(icu_collator, func=sort_key))
 
     def run(self):
         try:

src/calibre/utils/icu.py

@@ -9,6 +9,7 @@ __docformat__ = 'restructuredtext en'
 from functools import partial
 
 from calibre.constants import plugins
+from calibre.utils.config import tweaks
 
 _icu = _collator = None
 _locale = None
@@ -20,7 +21,10 @@ def get_locale():
     global _locale
     if _locale is None:
         from calibre.utils.localization import get_lang
-        _locale = get_lang()
+        if tweaks['locale_for_sorting']:
+            _locale = tweaks['locale_for_sorting']
+        else:
+            _locale = get_lang()
     return _locale
 
 def load_icu():

src/calibre/utils/search_query_parser.py

@@ -22,6 +22,7 @@ from calibre.utils.pyparsing import CaselessKeyword, Group, Forward, \
         CharsNotIn, Suppress, OneOrMore, MatchFirst, CaselessLiteral, \
         Optional, NoMatch, ParseException, QuotedString
 from calibre.constants import preferred_encoding
+from calibre.utils.icu import sort_key
 
 
 
@@ -65,8 +66,7 @@ class SavedSearchQueries(object):
         self.db.prefs[self.opt_name] = self.queries
 
     def names(self):
-        return sorted(self.queries.keys(),
+        return sorted(self.queries.keys(),key=sort_key)
-                cmp=lambda x,y: cmp(x.lower(), y.lower()))
 
 '''
 Create a global instance of the saved searches. It is global so that the searches