Merge from trunk

setup/build_environment.py

@@ -91,11 +91,15 @@ podofo_inc = '/usr/include/podofo'
 podofo_lib = '/usr/lib'
 chmlib_inc_dirs = chmlib_lib_dirs = []
 sqlite_inc_dirs = []
+icu_inc_dirs = []
+icu_lib_dirs = []
 
 if iswindows:
     prefix  = r'C:\cygwin\home\kovid\sw'
     sw_inc_dir  = os.path.join(prefix, 'include')
     sw_lib_dir  = os.path.join(prefix, 'lib')
+    icu_inc_dirs = [sw_inc_dir]
+    icu_lib_dirs = [sw_lib_dir]
     sqlite_inc_dirs = [sw_inc_dir]
     fc_inc = os.path.join(sw_inc_dir, 'fontconfig')
     fc_lib = sw_lib_dir

setup/extensions.py

@@ -18,7 +18,8 @@ from setup.build_environment import fc_inc, fc_lib, chmlib_inc_dirs, \
         QMAKE, msvc, MT, win_inc, win_lib, png_inc_dirs, win_ddk, \
         magick_inc_dirs, magick_lib_dirs, png_lib_dirs, png_libs, \
         magick_error, magick_libs, ft_lib_dirs, ft_libs, jpg_libs, \
-        jpg_lib_dirs, chmlib_lib_dirs, sqlite_inc_dirs
+        jpg_lib_dirs, chmlib_lib_dirs, sqlite_inc_dirs, icu_inc_dirs, \
+        icu_lib_dirs
 MT
 isunix = islinux or isosx or isfreebsd
 
@@ -56,8 +57,25 @@ pdfreflow_libs = []
 if iswindows:
     pdfreflow_libs = ['advapi32', 'User32', 'Gdi32', 'zlib']
 
+icu_libs = ['icudata', 'icui18n', 'icuuc', 'icuio']
+icu_cflags = []
+if iswindows:
+    icu_libs = ['icudt', 'icuin', 'icuuc', 'icuio']
+if isosx:
+    icu_libs = ['icucore']
+    icu_cflags = ['-DU_DISABLE_RENAMING'] # Needed to use system libicucore.dylib
+
+
 extensions = [
 
+    Extension('icu',
+        ['calibre/utils/icu.c'],
+        libraries=icu_libs,
+        lib_dirs=icu_lib_dirs,
+        inc_dirs=icu_inc_dirs,
+        cflags=icu_cflags
+        ),
+
     Extension('sqlite_custom',
         ['calibre/library/sqlite_custom.c'],
         inc_dirs=sqlite_inc_dirs

setup/installer/linux/freeze2.py

@@ -14,7 +14,8 @@ from setup import Command, modules, basenames, functions, __version__, \
 
 SITE_PACKAGES = ['IPython', 'PIL', 'dateutil', 'dns', 'PyQt4', 'mechanize',
         'sip.so', 'BeautifulSoup.py', 'cssutils', 'encutils', 'lxml',
-        'sipconfig.py', 'xdg']
+        'sipconfig.py', 'xdg', 'dbus', '_dbus_bindings.so', 'dbus_bindings.py',
+        '_dbus_glib_bindings.so']
 
 QTDIR          = '/usr/lib/qt4'
 QTDLLS         = ('QtCore', 'QtGui', 'QtNetwork', 'QtSvg', 'QtXml', 'QtWebKit', 'QtDBus')
@@ -49,6 +50,10 @@ binary_includes = [
                 '/lib/libreadline.so.6',
                 '/usr/lib/libchm.so.0',
                 '/usr/lib/liblcms2.so.2',
+                '/usr/lib/libicudata.so.46',
+                '/usr/lib/libicui18n.so.46',
+                '/usr/lib/libicuuc.so.46',
+                '/usr/lib/libicuio.so.46',
                 ]
 binary_includes += [os.path.join(QTDIR, 'lib%s.so.4'%x) for x in QTDLLS]
 

setup/installer/windows/freeze.py

@@ -199,7 +199,7 @@ class Win32Freeze(Command, WixMixIn):
         for pat in ('*.dll',):
             for f in glob.glob(os.path.join(bindir, pat)):
                 ok = True
-                for ex in ('expatw',):
+                for ex in ('expatw', 'testplug'):
                     if ex in f.lower():
                         ok = False
                 if not ok: continue

setup/installer/windows/notes.rst

@@ -77,6 +77,15 @@ Test it on the target system with
 
 calibre-debug -c "import _imaging, _imagingmath, _imagingft, _imagingcms"
 
+ICU
+-------
+
+Download the win32 msvc9 binary from http://www.icu-project.org/download/4.4.html
+
+Note that 4.4 is the last version of ICU that can be compiled (is precompiled) with msvc9
+
+Put the dlls into sw/bin and the unicode dir into sw/include and the contents of lib int sw/lib
+
 Libunrar
 ----------
 

src/calibre/constants.py

@@ -67,7 +67,8 @@ if plugins is None:
                 'pdfreflow',
                 'progress_indicator',
                 'chmlib',
-                'chm_extra'
+                'chm_extra',
+                'icu',
             ] + \
                     (['winutil'] if iswindows else []) + \
                     (['usbobserver'] if isosx else []):

src/calibre/ebooks/fb2/fb2ml.py

@@ -32,7 +32,6 @@ TAG_MAP = {
     'p' : 'p',
     'li' : 'p',
     'div': 'p',
-    'br' : 'p',
 }
 
 TAG_SPACE = []
@@ -42,7 +41,6 @@ TAG_IMAGES = [
 ]
 
 TAG_LINKS = [
-    'a',
 ]
 
 BLOCK = [
@@ -56,9 +54,8 @@ STYLES = [
 
 class FB2MLizer(object):
     '''
-    Todo: * Ensure all style tags are inside of the p tags.
+    Todo: * Include more FB2 specific tags in the conversion.
-          * Include more FB2 specific tags in the conversion.
+          * Handle a tags.
-          * Handle reopening of a tag properly.
           * Figure out some way to turn oeb_book.toc items into <section><title>
             <p> to allow for readers to generate toc from the document.
     '''
@@ -66,7 +63,8 @@ class FB2MLizer(object):
     def __init__(self, log):
         self.log = log
         self.image_hrefs = {}
-        self.link_hrefs = {}
+        # Used to ensure text and tags are always within <p> and </p>
+        self.in_p = False
 
     def extract_content(self, oeb_book, opts):
         self.log.info('Converting XHTML to FB2 markup...')
@@ -78,17 +76,15 @@ class FB2MLizer(object):
         self.image_hrefs = {}
         self.link_hrefs = {}
         output = [self.fb2_header()]
-        output.append(self.get_cover_page())
-        output.append(u'ghji87yhjko0Caliblre-toc-placeholder-for-insertion-later8ujko0987yjk')
         output.append(self.get_text())
         output.append(self.fb2_body_footer())
         output.append(self.fb2mlize_images())
         output.append(self.fb2_footer())
-        output = ''.join(output).replace(u'ghji87yhjko0Caliblre-toc-placeholder-for-insertion-later8ujko0987yjk', self.get_toc())
+        output = self.clean_text(u''.join(output))
-        output = self.clean_text(output)
+        if self.opts.pretty_print:
-        if self.opts.sectionize_chapters:
+            return u'<?xml version="1.0" encoding="UTF-8"?>\n%s' % etree.tostring(etree.fromstring(output), encoding=unicode, pretty_print=True)
-            output = self.sectionize_chapters(output)
+        else:
-        return u'<?xml version="1.0" encoding="UTF-8"?>\n%s' % etree.tostring(etree.fromstring(output), encoding=unicode, pretty_print=True)
+            return u'<?xml version="1.0" encoding="UTF-8"?>' + output
 
     def clean_text(self, text):
         text = re.sub(r'(?miu)<section>\s*</section>', '', text)
@@ -116,88 +112,40 @@ class FB2MLizer(object):
             author_middle = ' '.join(author_parts[1:-2])
             author_last = author_parts[-1]
 
-        return u'<FictionBook xmlns:xlink="http://www.w3.org/1999/xlink" ' \
+        return u'<FictionBook xmlns="http://www.gribuser.ru/xml/fictionbook/2.0" xmlns:xlink="http://www.w3.org/1999/xlink">' \
-        'xmlns="http://www.gribuser.ru/xml/fictionbook/2.0">\n' \
+                '<description>' \
-        '<description>\n<title-info>\n ' \
+                    '<title-info>' \
-        '<author>\n<first-name>%s</first-name>\n<middle-name>%s' \
+                        '<genre></genre>' \
-        '</middle-name>\n<last-name>%s</last-name>\n</author>\n' \
+                        '<author>' \
-        '<book-title>%s</book-title> ' \
+                            '<first-name>%s</first-name>' \
-        '</title-info><document-info> ' \
+                            '<middle-name>%s</middle-name>' \
-        '<program-used>%s - %s</program-used></document-info>\n' \
+                            '<last-name>%s</last-name>' \
-        '</description>\n<body>\n<section>' % tuple(map(prepare_string_for_xml,
+                        '</author>' \
-            (author_first, author_middle,
+                        '<book-title>%s</book-title>' \
-            author_last, self.oeb_book.metadata.title[0].value,
+                        '<annotation><p/></annotation>' \
-            __appname__, __version__)))
+                    '</title-info>' \
-
+                    '<document-info>' \
-    def get_cover_page(self):
+                        '<program-used>%s %s</program-used>' \
-        output = u''
+                    '</document-info>' \
-        if 'cover' in self.oeb_book.guide:
+                '</description><body>' % tuple(map(prepare_string_for_xml, (author_first, author_middle, author_last,
-            output += '<image xlink:href="#cover.jpg" />'
+                        self.oeb_book.metadata.title[0].value, __appname__, __version__)))
-            self.image_hrefs[self.oeb_book.guide['cover'].href] = 'cover.jpg'
-        if 'titlepage' in self.oeb_book.guide:
-            self.log.debug('Generating cover page...')
-            href = self.oeb_book.guide['titlepage'].href
-            item = self.oeb_book.manifest.hrefs[href]
-            if item.spine_position is None:
-                stylizer = Stylizer(item.data, item.href, self.oeb_book,
-                        self.opts, self.opts.output_profile)
-                output += ''.join(self.dump_text(item.data.find(XHTML('body')), stylizer, item))
-        return output
-
-    def get_toc(self):
-        toc = []
-        if self.opts.inline_toc:
-            self.log.debug('Generating table of contents...')
-            toc.append(u'<p>%s</p>' % _('Table of Contents:'))
-            for item in self.oeb_book.toc:
-                if item.href in self.link_hrefs.keys():
-                    toc.append('<p><a xlink:href="#%s">%s</a></p>\n' % (self.link_hrefs[item.href], item.title))
-                else:
-                    self.oeb.warn('Ignoring toc item: %s not found in document.' % item)
-        return ''.join(toc)
-
-    def sectionize_chapters(self, text):
-        def remove_p(t):
-            t = t.replace('<p>', '')
-            t = t.replace('</p>', '')
-            return t
-        text = re.sub(r'(?imsu)(<p>)\s*(?P<anchor><a\s+id="calibre_link-\d+"\s*/>)\s*(</p>)\s*(<p>)\s*(?P<strong><strong>.+?</strong>)\s*(</p>)', lambda mo: '</section><section>%s<title><p>%s</p></title>' % (mo.group('anchor'), remove_p(mo.group('strong'))), text)
-        text = re.sub(r'(?imsu)(<p>)\s*(?P<anchor><a\s+id="calibre_link-\d+"\s*/>)\s*(</p>)\s*(?P<strong><strong>.+?</strong>)', lambda mo: '</section><section>%s<title><p>%s</p></title>' % (mo.group('anchor'), remove_p(mo.group('strong'))), text)
-        text = re.sub(r'(?imsu)(?P<anchor><a\s+id="calibre_link-\d+"\s*/>)\s*(<p>)\s*(?P<strong><strong>.+?</strong>)\s*(</p>)', lambda mo: '</section><section>%s<title><p>%s</p></title>' % (mo.group('anchor'), remove_p(mo.group('strong'))), text)
-        text = re.sub(r'(?imsu)(<p>)\s*(?P<anchor><a\s+id="calibre_link-\d+"\s*/>)\s*(?P<strong><strong>.+?</strong>)\s*(</p>)', lambda mo: '</section><section>%s<title><p>%s</p></title>' % (mo.group('anchor'), remove_p(mo.group('strong'))), text)
-        text = re.sub(r'(?imsu)(?P<anchor><a\s+id="calibre_link-\d+"\s*/>)\s*(?P<strong><strong>.+?</strong>)', lambda mo: '</section><section>%s<title><p>%s</p></title>' % (mo.group('anchor'), remove_p(mo.group('strong'))), text)
-        return text
 
     def get_text(self):
         text = []
-        for i, item in enumerate(self.oeb_book.spine):
+        for item in self.oeb_book.spine:            
-            if self.opts.sectionize_chapters_using_file_structure and i is not 0:
-                text.append('<section>')
             self.log.debug('Converting %s to FictionBook2 XML' % item.href)
             stylizer = Stylizer(item.data, item.href, self.oeb_book, self.opts, self.opts.output_profile)
-            text.append(self.add_page_anchor(item))
+            text.append('<section>')
             text += self.dump_text(item.data.find(XHTML('body')), stylizer, item)
-            if self.opts.sectionize_chapters_using_file_structure and i is not len(self.oeb_book.spine) - 1:
+            text.append('</section>')
-                text.append('</section>')
         return ''.join(text)
 
     def fb2_body_footer(self):
-        return u'\n</section>\n</body>'
+        return u'</body>'
 
     def fb2_footer(self):
         return u'</FictionBook>'
 
-    def add_page_anchor(self, page):
-        return self.get_anchor(page, '')
-
-    def get_anchor(self, page, aid):
-        aid = prepare_string_for_xml(aid)
-        aid = '%s#%s' % (page.href, aid)
-        if aid not in self.link_hrefs.keys():
-            self.link_hrefs[aid] = 'calibre_link-%s' % len(self.link_hrefs.keys())
-        aid = self.link_hrefs[aid]
-        return '<a id="%s" />' % aid
-
     def fb2mlize_images(self):
         images = []
         for item in self.oeb_book.manifest:
@@ -218,12 +166,60 @@ class FB2MLizer(object):
                             col = 1
                         col += 1
                         data += char
-                    images.append('<binary id="%s" content-type="%s">%s\n</binary>' % (self.image_hrefs.get(item.href, '0000.JPEG'), item.media_type, data))
+                    images.append('<binary id="%s" content-type="%s">%s\n</binary>' % (self.image_hrefs.get(item.href, '_0000.JPEG'), item.media_type, data))
                 except Exception as e:
-                    self.log.error('Error: Could not include file %s becuase ' \
+                    self.log.error('Error: Could not include file %s because ' \
                         '%s.' % (item.href, e))
         return ''.join(images)
 
+    def ensure_p(self):
+        if self.in_p:
+            return [], []
+        else:
+            self.in_p = True
+            return ['<p>'], ['p']
+        
+    def insert_empty_line(self, tags):
+        if self.in_p:
+            text = ['']
+            closed_tags = []
+            tags.reverse()
+            for t in tags:
+                text.append('</%s>' % t)
+                closed_tags.append(t)
+                if t == 'p':
+                    break
+            text.append('<empty-line />')
+            closed_tags.reverse()
+            for t in closed_tags:
+                text.append('<%s>' % t)
+            return text
+        else:
+            return ['<empty-line />']
+
+    def close_open_p(self, tags):
+        text = ['']
+        added_p = False
+        
+        if self.in_p:
+            # Close all up to p. Close p. Reopen all closed tags including p.
+            closed_tags = []
+            tags.reverse()
+            for t in tags:
+                text.append('</%s>' % t)
+                closed_tags.append(t)
+                if t == 'p':
+                    break
+            closed_tags.reverse()
+            for t in closed_tags:
+                text.append('<%s>' % t)
+        else:
+            text.append('<p>')
+            added_p = True
+            self.in_p = True
+        
+        return text, added_p
+
     def dump_text(self, elem, stylizer, page, tag_stack=[]):
         if not isinstance(elem.tag, basestring) \
            or namespace(elem.tag) != XHTML_NS:
@@ -242,53 +238,28 @@ class FB2MLizer(object):
         if tag in TAG_IMAGES:
             if elem.attrib.get('src', None):
                 if page.abshref(elem.attrib['src']) not in self.image_hrefs.keys():
-                    self.image_hrefs[page.abshref(elem.attrib['src'])] = '%s.jpg' % len(self.image_hrefs.keys())
+                    self.image_hrefs[page.abshref(elem.attrib['src'])] = '_%s.jpg' % len(self.image_hrefs.keys())
+                p_txt, p_tag = self.ensure_p()
+                fb2_text += p_txt
+                tags += p_tag
                 fb2_text.append('<image xlink:href="#%s" />' % self.image_hrefs[page.abshref(elem.attrib['src'])])
 
-        if tag in TAG_LINKS:
-            href = elem.get('href')
-            if href:
-                href = prepare_string_for_xml(page.abshref(href))
-                href = href.replace('"', '&quot;')
-                if '://' in href:
-                    fb2_text.append('<a xlink:href="%s">' % href)
-                else:
-                    if href.startswith('#'):
-                        href = href[1:]
-                    if href not in self.link_hrefs.keys():
-                        self.link_hrefs[href] = 'calibre_link-%s' % len(self.link_hrefs.keys())
-                    href = self.link_hrefs[href]
-                    fb2_text.append('<a xlink:href="#%s">' % href)
-                tags.append('a')
-
-        # Anchor ids
-        id_name = elem.get('id')
-        if id_name:
-            fb2_text.append(self.get_anchor(page, id_name))
-
         if tag == 'h1' and self.opts.h1_to_title or tag == 'h2' and self.opts.h2_to_title or tag == 'h3' and self.opts.h3_to_title:
             fb2_text.append('<title>')
             tags.append('title')
+        if tag == 'br':
+            fb2_text += self.insert_empty_line(tag_stack+tags)
 
         fb2_tag = TAG_MAP.get(tag, None)
         if fb2_tag == 'p':
-            if 'p' in tag_stack+tags:
+            p_text, added_p = self.close_open_p(tag_stack+tags)
-                # Close all up to p. Close p. Reopen all closed tags including p.
+            fb2_text += p_text
-                all_tags = tag_stack+tags
+            if added_p:
-                closed_tags = []
-                all_tags.reverse()
-                for t in all_tags:
-                    fb2_text.append('</%s>' % t)
-                    closed_tags.append(t)
-                    if t == 'p':
-                        break
-                closed_tags.reverse()
-                for t in closed_tags:
-                    fb2_text.append('<%s>' % t)
-            else:
-                fb2_text.append('<p>')
                 tags.append('p')
         elif fb2_tag and fb2_tag not in tag_stack+tags:
+            p_text, p_tags = self.ensure_p()
+            fb2_text += p_text
+            tags += p_tags
             fb2_text.append('<%s>' % fb2_tag)
             tags.append(fb2_tag)
 
@@ -296,18 +267,21 @@ class FB2MLizer(object):
         for s in STYLES:
             style_tag = s[1].get(style[s[0]], None)
             if style_tag and style_tag not in tag_stack+tags:
+                p_text, p_tags = self.ensure_p()
+                fb2_text += p_text
+                tags += p_tags
                 fb2_text.append('<%s>' % style_tag)
                 tags.append(style_tag)
 
         if tag in TAG_SPACE:
-            if not fb2_text or fb2_text[-1] != ' ' or not fb2_text[-1].endswith(' '):
+            fb2_text.append(' ')
-                fb2_text.append(' ')
 
         if hasattr(elem, 'text') and elem.text:
-            if 'p' not in tag_stack+tags:
+            if not self.in_p:
-                fb2_text.append('<p>%s</p>' % prepare_string_for_xml(elem.text))
+                fb2_text.append('<p>')
-            else:
+            fb2_text.append(prepare_string_for_xml(elem.text))
-                fb2_text.append(prepare_string_for_xml(elem.text))
+            if not self.in_p:
+                fb2_text.append('</p>')
 
         for item in elem:
             fb2_text += self.dump_text(item, stylizer, page, tag_stack+tags)
@@ -316,10 +290,11 @@ class FB2MLizer(object):
         fb2_text += self.close_tags(tags)
 
         if hasattr(elem, 'tail') and elem.tail:
-            if 'p' not in tag_stack:
+            if not self.in_p:
-                fb2_text.append('<p>%s</p>' % prepare_string_for_xml(elem.tail))
+                fb2_text.append('<p>')
-            else:
+            fb2_text.append(prepare_string_for_xml(elem.tail))
-                fb2_text.append(prepare_string_for_xml(elem.tail))
+            if not self.in_p:
+                fb2_text.append('</p>')
 
         return fb2_text
 
@@ -327,5 +302,7 @@ class FB2MLizer(object):
         text = []
         for tag in tags:
             text.append('</%s>' % tag)
+            if tag == 'p':
+                self.in_p = False
 
         return text

src/calibre/ebooks/fb2/output.py

@@ -16,20 +16,6 @@ class FB2Output(OutputFormatPlugin):
     file_type = 'fb2'
 
     options = set([
-        OptionRecommendation(name='inline_toc',
-            recommended_value=False, level=OptionRecommendation.LOW,
-            help=_('Add Table of Contents to beginning of the book.')),
-        OptionRecommendation(name='sectionize_chapters',
-            recommended_value=False, level=OptionRecommendation.LOW,
-            help=_('Try to turn chapters into individual sections. ' \
-                   'WARNING: ' \
-                   'This option is experimental. It can cause conversion ' \
-                   'to fail. It can also produce unexpected output.')),
-        OptionRecommendation(name='sectionize_chapters_using_file_structure',
-            recommended_value=False, level=OptionRecommendation.LOW,
-            help=_('Try to turn chapters into individual sections using the ' \
-                   'internal structure of the ebook. This works well for EPUB ' \
-                   'books that have been internally split by chapter.')),
         OptionRecommendation(name='h1_to_title',
             recommended_value=False, level=OptionRecommendation.LOW,
             help=_('Wrap all h1 tags with fb2 title elements.')),

src/calibre/gui2/convert/fb2_output.py

@@ -17,8 +17,6 @@ class PluginWidget(Widget, Ui_Form):
     ICON = I('mimetypes/fb2.png')
 
     def __init__(self, parent, get_option, get_help, db=None, book_id=None):
-        Widget.__init__(self, parent, ['inline_toc', 'sectionize_chapters', 
+        Widget.__init__(self, parent, ['h1_to_title', 'h2_to_title', 'h3_to_title'])
-            'sectionize_chapters_using_file_structure', 'h1_to_title', 
-            'h2_to_title', 'h3_to_title'])
         self.db, self.book_id = db, book_id
         self.initialize_options(get_option, get_help, db, book_id)

src/calibre/gui2/convert/fb2_output.ui

@@ -14,7 +14,7 @@
    <string>Form</string>
   </property>
   <layout class="QGridLayout" name="gridLayout">
-   <item row="6" column="0">
+   <item row="3" column="0">
     <spacer name="verticalSpacer">
      <property name="orientation">
       <enum>Qt::Vertical</enum>
@@ -28,41 +28,20 @@
     </spacer>
    </item>
    <item row="0" column="0">
-    <widget class="QCheckBox" name="opt_inline_toc">
-     <property name="text">
-      <string>&amp;Inline TOC</string>
-     </property>
-    </widget>
-   </item>
-   <item row="1" column="0">
-    <widget class="QCheckBox" name="opt_sectionize_chapters">
-     <property name="text">
-      <string>Sectionize Chapters (Use with care!)</string>
-     </property>
-    </widget>
-   </item>
-   <item row="2" column="0">
-    <widget class="QCheckBox" name="opt_sectionize_chapters_using_file_structure">
-     <property name="text">
-      <string>Sectionize Chapters using file structure</string>
-     </property>
-    </widget>
-   </item>
-   <item row="3" column="0">
     <widget class="QCheckBox" name="opt_h1_to_title">
      <property name="text">
       <string>Wrap h1 tags with &lt;title&gt; elements</string>
      </property>
     </widget>
    </item>
-   <item row="4" column="0">
+   <item row="1" column="0">
     <widget class="QCheckBox" name="opt_h2_to_title">
      <property name="text">
       <string>Wrap h2 tags with &lt;title&gt; elements</string>
      </property>
     </widget>
    </item>
-   <item row="5" column="0">
+   <item row="2" column="0">
     <widget class="QCheckBox" name="opt_h3_to_title">
      <property name="text">
       <string>Wrap h3 tags with &lt;title&gt; elements</string>

src/calibre/library/caches.py

@@ -796,11 +796,13 @@ class SortKey(object):
 class SortKeyGenerator(object):
 
     def __init__(self, fields, field_metadata, data):
+        from calibre.utils.icu import sort_key
         self.field_metadata = field_metadata
         self.orders = [-1 if x[1] else 1 for x in fields]
         self.entries = [(x[0], field_metadata[x[0]]) for x in fields]
         self.library_order = tweaks['title_series_sorting'] == 'library_order'
         self.data = data
+        self.string_sort_key = sort_key
 
     def __call__(self, record):
         values = tuple(self.itervals(self.data[record]))
@@ -821,17 +823,14 @@ class SortKeyGenerator(object):
                 if val is None:
                     val = ('', 1)
                 else:
-                    val = val.lower()
                     if self.library_order:
                         val = title_sort(val)
                     sidx_fm = self.field_metadata[name + '_index']
                     sidx = record[sidx_fm['rec_index']]
-                    val = (val, sidx)
+                    val = (self.string_sort_key(val), sidx)
 
             elif dt in ('text', 'comments', 'composite', 'enumeration'):
-                if val is None:
+                val = self.string_sort_key(val)
-                    val = ''
-                val = val.lower()
 
             elif dt == 'bool':
                 val = {True: 1, False: 2, None: 3}.get(val, 3)

src/calibre/utils/icu.c

@@ -0,0 +1,223 @@
+#define UNICODE
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+#include <unicode/utypes.h>
+#include <unicode/uclean.h>
+#include <unicode/ucol.h>
+#include <unicode/ustring.h>
+
+
+// Collator object definition {{{
+typedef struct {
+    PyObject_HEAD
+    // Type-specific fields go here.
+    UCollator *collator;
+
+} icu_Collator;
+
+static void
+icu_Collator_dealloc(icu_Collator* self)
+{
+    if (self->collator != NULL) ucol_close(self->collator);
+    self->collator = NULL;
+    self->ob_type->tp_free((PyObject*)self);
+}
+
+static PyObject *
+icu_Collator_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
+{
+    icu_Collator *self;
+    const char *loc;
+    UErrorCode status = U_ZERO_ERROR;
+
+    if (!PyArg_ParseTuple(args, "s", &loc)) return NULL;
+
+    self = (icu_Collator *)type->tp_alloc(type, 0);
+    if (self != NULL) {
+        self->collator = ucol_open(loc, &status);
+        if (self->collator == NULL || U_FAILURE(status)) { 
+            PyErr_SetString(PyExc_Exception, "Failed to create collator.");
+            self->collator = NULL;
+            Py_DECREF(self);
+            return NULL;
+        }
+    }
+
+    return (PyObject *)self;
+}
+
+// Collator.display_name {{{
+static PyObject *
+icu_Collator_display_name(icu_Collator *self, void *closure) {
+    const char *loc = NULL;
+    UErrorCode status = U_ZERO_ERROR;
+    UChar dname[400];
+    char buf[100];
+
+    loc = ucol_getLocaleByType(self->collator, ULOC_ACTUAL_LOCALE, &status);
+    if (loc == NULL || U_FAILURE(status)) {
+        PyErr_SetString(PyExc_Exception, "Failed to get actual locale"); return NULL;
+    }
+    ucol_getDisplayName(loc, "en", dname, 100, &status);
+    if (U_FAILURE(status)) return PyErr_NoMemory();
+
+    u_strToUTF8(buf, 100, NULL, dname, -1, &status);
+    if (U_FAILURE(status)) {
+        PyErr_SetString(PyExc_Exception, "Failed ot convert dname to UTF-8"); return NULL;
+    }
+    return Py_BuildValue("s", buf);
+}
+
+// }}}
+
+// Collator.actual_locale {{{
+static PyObject *
+icu_Collator_actual_locale(icu_Collator *self, void *closure) {
+    const char *loc = NULL;
+    UErrorCode status = U_ZERO_ERROR;
+
+    loc = ucol_getLocaleByType(self->collator, ULOC_ACTUAL_LOCALE, &status);
+    if (loc == NULL || U_FAILURE(status)) {
+        PyErr_SetString(PyExc_Exception, "Failed to get actual locale"); return NULL;
+    }
+    return Py_BuildValue("s", loc);
+}
+
+// }}}
+
+// Collator.sort_key {{{
+static PyObject *
+icu_Collator_sort_key(icu_Collator *self, PyObject *args, PyObject *kwargs) {
+    char *input;
+    Py_ssize_t sz;
+    UChar *buf;
+    uint8_t *buf2;
+    PyObject *ans;
+    int32_t key_size;
+    UErrorCode status = U_ZERO_ERROR;
+  
+    if (!PyArg_ParseTuple(args, "es", "UTF-8", &input)) return NULL;
+
+    sz = strlen(input);
+
+    buf = (UChar*)calloc(sz*4 + 1, sizeof(UChar));
+
+    if (buf == NULL) return PyErr_NoMemory();
+
+    u_strFromUTF8(buf, sz*4 + 1, &key_size, input, sz, &status);
+
+    if (U_SUCCESS(status)) {
+        key_size = ucol_getSortKey(self->collator, buf, -1, NULL, 0);
+        buf2 = (uint8_t*)calloc(key_size + 1, sizeof(uint8_t));
+        if (buf2 == NULL) return PyErr_NoMemory();
+        ucol_getSortKey(self->collator, buf, -1, buf2, key_size+1);
+        ans = PyBytes_FromString((char *)buf2);
+        free(buf2);
+    } else ans = PyBytes_FromString("");
+
+    free(buf);
+    if (ans == NULL) return PyErr_NoMemory();
+
+    return ans;
+}
+
+static PyMethodDef icu_Collator_methods[] = {
+    {"sort_key", (PyCFunction)icu_Collator_sort_key, METH_VARARGS,
+     "sort_key(unicode object) -> Return a sort key for the given object as a bytestring. The idea is that these bytestring will sort using the builtin cmp function, just like the original unicode strings would sort in the current locale with ICU."
+    },
+
+    {NULL}  /* Sentinel */
+};
+
+static PyGetSetDef  icu_Collator_getsetters[] = {
+    {(char *)"actual_locale", 
+     (getter)icu_Collator_actual_locale, NULL,
+     (char *)"Actual locale used by this collator.",
+     NULL},
+
+    {(char *)"display_name", 
+     (getter)icu_Collator_display_name, NULL,
+     (char *)"Display name of this collator in English. The name reflects the actual data source used.",
+     NULL},
+
+    {NULL}  /* Sentinel */
+};
+
+static PyTypeObject icu_CollatorType = { // {{{
+    PyObject_HEAD_INIT(NULL)
+    0,                         /*ob_size*/
+    "icu.Collator",            /*tp_name*/
+    sizeof(icu_Collator),      /*tp_basicsize*/
+    0,                         /*tp_itemsize*/
+    (destructor)icu_Collator_dealloc, /*tp_dealloc*/
+    0,                         /*tp_print*/
+    0,                         /*tp_getattr*/
+    0,                         /*tp_setattr*/
+    0,                         /*tp_compare*/
+    0,                         /*tp_repr*/
+    0,                         /*tp_as_number*/
+    0,                         /*tp_as_sequence*/
+    0,                         /*tp_as_mapping*/
+    0,                         /*tp_hash */
+    0,                         /*tp_call*/
+    0,                         /*tp_str*/
+    0,                         /*tp_getattro*/
+    0,                         /*tp_setattro*/
+    0,                         /*tp_as_buffer*/
+    Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE,        /*tp_flags*/
+    "Collator",                  /* tp_doc */
+    0,		               /* tp_traverse */
+    0,		               /* tp_clear */
+    0,		               /* tp_richcompare */
+    0,		               /* tp_weaklistoffset */
+    0,		               /* tp_iter */
+    0,		               /* tp_iternext */
+    icu_Collator_methods,             /* tp_methods */
+    0,             /* tp_members */
+    icu_Collator_getsetters,                         /* tp_getset */
+    0,                         /* tp_base */
+    0,                         /* tp_dict */
+    0,                         /* tp_descr_get */
+    0,                         /* tp_descr_set */
+    0,                         /* tp_dictoffset */
+    0,      /* tp_init */
+    0,                         /* tp_alloc */
+    icu_Collator_new,                 /* tp_new */
+}; // }}}
+
+// }}
+
+
+// }}}
+
+// }}}
+
+// Module initialization {{{
+
+static PyMethodDef icu_methods[] = {
+    {NULL}  /* Sentinel */
+};
+
+
+PyMODINIT_FUNC
+initicu(void) 
+{
+    PyObject* m;
+    UErrorCode status = U_ZERO_ERROR;
+
+    u_init(&status);
+
+
+    if (PyType_Ready(&icu_CollatorType) < 0)
+        return;
+
+    m = Py_InitModule3("icu", icu_methods,
+                       "Wrapper for the ICU internationalization library");
+
+    Py_INCREF(&icu_CollatorType);
+    PyModule_AddObject(m, "Collator", (PyObject *)&icu_CollatorType);
+    // uint8_t must be the same size as char
+    PyModule_AddIntConstant(m, "ok", (U_SUCCESS(status) && sizeof(uint8_t) == sizeof(char)) ? 1 : 0);
+
+}
+// }}}

src/calibre/utils/icu.py

@@ -0,0 +1,160 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from functools import partial
+
+from calibre.constants import plugins
+
+_icu = _collator = None
+
+_none = u''
+_none2 = b''
+
+def load_icu():
+    global _icu
+    if _icu is None:
+        _icu = plugins['icu'][0]
+        if _icu is None:
+            print plugins['icu'][1]
+        else:
+            if not _icu.ok:
+                print 'icu not ok'
+                _icu = None
+    return _icu
+
+def load_collator():
+    global _collator
+    from calibre.utils.localization import get_lang
+    if _collator is None:
+        icu = load_icu()
+        if icu is not None:
+            _collator = icu.Collator(get_lang())
+    return _collator
+
+
+def py_sort_key(obj):
+    if not obj:
+        return _none
+    return obj.lower()
+
+def icu_sort_key(collator, obj):
+    if not obj:
+        return _none2
+    return collator.sort_key(obj.lower())
+
+load_icu()
+load_collator()
+sort_key = py_sort_key if _icu is None or _collator is None else \
+        partial(icu_sort_key, _collator)
+
+
+def test(): # {{{
+    # Data {{{
+    german = '''
+    Sonntag
+Montag
+Dienstag
+Januar
+Februar
+März
+Fuße
+Fluße
+Flusse
+flusse
+fluße
+flüße
+flüsse
+'''
+    german_good = '''
+    Dienstag
+Februar
+flusse
+Flusse
+fluße
+Fluße
+flüsse
+flüße
+Fuße
+Januar
+März
+Montag
+Sonntag'''
+    french = '''
+dimanche
+lundi
+mardi
+janvier
+février
+mars
+déjà
+Meme
+deja
+même
+dejà
+bpef
+bœg
+Boef
+Mémé
+bœf
+boef
+bnef
+pêche
+pèché
+pêché
+pêche
+pêché'''
+    french_good = '''
+            bnef
+        boef
+        Boef
+        bœf
+        bœg
+        bpef
+        deja
+        dejà
+        déjà
+        dimanche
+        février
+        janvier
+        lundi
+        mardi
+        mars
+        Meme
+        Mémé
+        même
+        pèché
+        pêche
+        pêche
+        pêché
+        pêché'''
+    # }}}
+
+    def create(l):
+        l = l.decode('utf-8').splitlines()
+        return [x.strip() for x in l if x.strip()]
+
+    german = create(german)
+    c = _icu.Collator('de')
+    print 'Sorted german:: (%s)'%c.actual_locale
+    gs = list(sorted(german, key=c.sort_key))
+    for x in gs:
+        print '\t', x.encode('utf-8')
+    if gs != create(german_good):
+        print 'German failed'
+        return
+    print
+    french = create(french)
+    c = _icu.Collator('fr')
+    print 'Sorted french:: (%s)'%c.actual_locale
+    fs = list(sorted(french, key=c.sort_key))
+    for x in fs:
+        print '\t', x.encode('utf-8')
+    if fs != create(french_good):
+        print 'French failed (note that French fails with icu < 4.6 i.e. on windows and OS X)'
+        return
+# }}}
+