Sync to trunk.

session.vim

@@ -1,5 +1,5 @@
 " Project wide builtins
-let g:pyflakes_builtins += ["dynamic_property", "__", "P", "I", "lopen"]
+let g:pyflakes_builtins += ["dynamic_property", "__", "P", "I", "lopen", "icu_lower", "icu_upper", "icu_title"]
 
 python << EOFPY
 import os

setup/check.py

@@ -63,7 +63,8 @@ class Check(Command):
 
     description = 'Check for errors in the calibre source code'
 
-    BUILTINS = ['_', '__', 'dynamic_property', 'I', 'P', 'lopen']
+    BUILTINS = ['_', '__', 'dynamic_property', 'I', 'P', 'lopen', 'icu_lower',
+            'icu_upper', 'icu_title']
     CACHE = '.check-cache.pickle'
 
     def get_files(self, cache):

src/calibre/devices/apple/driver.py

@@ -2637,7 +2637,7 @@ class ITUNES(DriverBase):
                 lb_added.composer.set(metadata_x.uuid)
                 lb_added.description.set("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S')))
                 lb_added.enabled.set(True)
-                lb_added.sort_artist.set(metadata_x.author_sort.title())
+                lb_added.sort_artist.set(icu_title(metadata_x.author_sort))
                 lb_added.sort_name.set(metadata.title_sort)
 
 
@@ -2648,7 +2648,7 @@ class ITUNES(DriverBase):
                 db_added.composer.set(metadata_x.uuid)
                 db_added.description.set("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S')))
                 db_added.enabled.set(True)
-                db_added.sort_artist.set(metadata_x.author_sort.title())
+                db_added.sort_artist.set(icu_title(metadata_x.author_sort))
                 db_added.sort_name.set(metadata.title_sort)
 
             if metadata_x.comments:
@@ -2729,7 +2729,7 @@ class ITUNES(DriverBase):
                 lb_added.Composer = metadata_x.uuid
                 lb_added.Description = ("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S')))
                 lb_added.Enabled = True
-                lb_added.SortArtist = metadata_x.author_sort.title()
+                lb_added.SortArtist = icu_title(metadata_x.author_sort)
                 lb_added.SortName = metadata.title_sort
 
             if db_added:
@@ -2739,7 +2739,7 @@ class ITUNES(DriverBase):
                 db_added.Composer = metadata_x.uuid
                 db_added.Description = ("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S')))
                 db_added.Enabled = True
-                db_added.SortArtist = metadata_x.author_sort.title()
+                db_added.SortArtist = icu_title(metadata_x.author_sort)
                 db_added.SortName = metadata.title_sort
 
             if metadata_x.comments:

src/calibre/ebooks/markdown/mdx_toc.py

@@ -18,9 +18,10 @@ def extract_alphanumeric(in_str=None):
     """
     # I'm sure this is really inefficient and
     # could be done with a lambda/map()
-    #x.strip().title().replace(' ', "")
+    #x.strip(). title().replace(' ', "")
     out_str=[]
-    for x in in_str.title():
+    for x in in_str:
+        x = icu_title(x)
         if x.isalnum(): out_str.append(x)
     return ''.join(out_str)
 

src/calibre/ebooks/oeb/base.py

@@ -607,7 +607,7 @@ class Metadata(object):
                     key = barename(key)
                 attrib[key] = prefixname(value, nsrmap)
             if namespace(self.term) == DC11_NS:
-                name = DC(barename(self.term).title())
+                name = DC(icu_title(barename(self.term)))
                 elem = element(dcmeta, name, attrib=attrib)
                 elem.text = self.value
             else:

src/calibre/ebooks/oeb/transforms/manglecase.py

@@ -50,11 +50,11 @@ class CaseMangler(object):
 
     def text_transform(self, transform, text):
         if transform == 'capitalize':
-            return text.title()
+            return icu_title(text)
         elif transform == 'uppercase':
-            return text.upper()
+            return icu_upper(text)
         elif transform == 'lowercase':
-            return text.lower()
+            return icu_lower(text)
         return text
 
     def split_text(self, text):

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -184,8 +184,8 @@ class MyBlockingBusy(QDialog):
 class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
 
     s_r_functions = {       ''              : lambda x: x,
-                            _('Lower Case') : lambda x: x.lower(),
+                            _('Lower Case') : lambda x: icu_lower(x),
-                            _('Upper Case') : lambda x: x.upper(),
+                            _('Upper Case') : lambda x: icu_upper(x),
                             _('Title Case') : lambda x: titlecase(x),
                     }
 

src/calibre/gui2/dialogs/tag_categories.py

@@ -22,6 +22,15 @@ class Item:
         return 'name=%s, label=%s, index=%s, exists='%(self.name, self.label, self.index, self.exists)
 
 class TagCategories(QDialog, Ui_TagCategories):
+    '''
+    The structure of user_categories stored in preferences is
+      {cat_name: [ [name, category, v], [], []}, cat_name [ [name, cat, v] ...}
+    where name is the item name, category is where it came from (series, etc),
+    and v is a scratch area that this editor uses to keep track of categories.
+
+    If you add a category, it is permissible to set v to zero. If you delete
+    a category, ensure that both the name and the category match.
+    '''
     category_labels_orig =   ['', 'authors', 'series', 'publisher', 'tags']
 
     def __init__(self, window, db, on_category=None):

src/calibre/library/caches.py

@@ -223,7 +223,7 @@ EQUALS_MATCH   = 1
 REGEXP_MATCH   = 2
 def _match(query, value, matchkind):
     for t in value:
-        t = t.lower()
+        t = icu_lower(t)
         try:     ### ignore regexp exceptions, required because search-ahead tries before typing is finished
             if ((matchkind == EQUALS_MATCH and query == t) or
                 (matchkind == REGEXP_MATCH and re.search(query, t, re.I)) or ### search unanchored
@@ -505,7 +505,7 @@ class ResultCache(SearchQueryParser): # {{{
                     query = query[1:]
             if matchkind != REGEXP_MATCH:
                 # leave case in regexps because it can be significant e.g. \S \W \D
-                query = query.lower()
+                query = icu_lower(query)
 
             if not isinstance(query, unicode):
                 query = query.decode('utf-8')

src/calibre/library/catalog.py

@@ -1476,20 +1476,20 @@ class EPUB_MOBI(CatalogPlugin):
                         self.opts.log.warn(" '%s' != '%s'" % (author[1], current_author[1]))
 
                     # New author, save the previous author/sort/count
-                    unique_authors.append((current_author[0], current_author[1].title(),
+                    unique_authors.append((current_author[0], icu_title(current_author[1]),
                                            books_by_current_author))
                     current_author = author
                     books_by_current_author = 1
                 elif i==0 and len(authors) == 1:
                     # Allow for single-book lists
-                    unique_authors.append((current_author[0], current_author[1].title(),
+                    unique_authors.append((current_author[0], icu_title(current_author[1]),
                                            books_by_current_author))
                 else:
                     books_by_current_author += 1
             else:
                 # Add final author to list or single-author dataset
                 if (current_author == author and len(authors) > 1) or not multiple_authors:
-                    unique_authors.append((current_author[0], current_author[1].title(),
+                    unique_authors.append((current_author[0], icu_title(current_author[1]),
                                            books_by_current_author))
 
             if False and self.verbose:

src/calibre/startup.py

@@ -199,6 +199,10 @@ if not _run_once:
 
     __builtin__.__dict__['lopen'] = local_open
 
+    from calibre.utils.icu import title_case, lower as icu_lower, upper as icu_upper
+    __builtin__.__dict__['icu_lower'] = icu_lower
+    __builtin__.__dict__['icu_upper'] = icu_upper
+    __builtin__.__dict__['icu_title'] = title_case
 
     import mimetypes
     mimetypes.init([P('mime.types')])

src/calibre/utils/titlecase.py

@@ -40,6 +40,11 @@ def titlecase(text):
 
     """
 
+    def capitalize(w):
+        w = icu_lower(w)
+        w = w.replace(w[0], icu_upper(w[0]))
+        return w
+
     all_caps = ALL_CAPS.match(text)
 
     words = re.split('\s', text)
@@ -50,29 +55,29 @@ def titlecase(text):
                 line.append(word)
                 continue
             else:
-                word = word.lower()
+                word = icu_lower(word)
 
         if APOS_SECOND.match(word):
-            word = word.replace(word[0], word[0].upper())
+            word = word.replace(word[0], icu_upper(word[0]))
-            word = word.replace(word[2], word[2].upper())
+            word = word.replace(word[2], icu_upper(word[2]))
             line.append(word)
             continue
         if INLINE_PERIOD.search(word) or UC_ELSEWHERE.match(word):
             line.append(word)
             continue
         if SMALL_WORDS.match(word):
-            line.append(word.lower())
+            line.append(icu_lower(word))
             continue
 
         match = MAC_MC.match(word)
         if match:
-            line.append("%s%s" % (match.group(1).capitalize(),
+            line.append("%s%s" % (capitalize(match.group(1)),
-                                  match.group(2).capitalize()))
+                                  capitalize(match.group(2))))
             continue
 
         hyphenated = []
         for item in word.split('-'):
-            hyphenated.append(CAPFIRST.sub(lambda m: m.group(0).upper(), item))
+            hyphenated.append(CAPFIRST.sub(lambda m: icu_upper(m.group(0)), item))
         line.append("-".join(hyphenated))
 
 
@@ -80,14 +85,14 @@ def titlecase(text):
 
     result = SMALL_FIRST.sub(lambda m: '%s%s' % (
         m.group(1),
-        m.group(2).capitalize()
+        capitalize(m.group(2))
     ), result)
 
-    result = SMALL_LAST.sub(lambda m: m.group(0).capitalize(), result)
+    result = SMALL_LAST.sub(lambda m: capitalize(m.group(0)), result)
 
     result = SUBPHRASE.sub(lambda m: '%s%s' % (
         m.group(1),
-        m.group(2).capitalize()
+        capitalize(m.group(2))
     ), result)
 
     return result