0.9.30+

KG merged libiMobileDevice wrapper

recipes/weblogs_sl.recipe

@@ -3,7 +3,7 @@ __license__     = 'GPL v3'
 __copyright__   = '4 February 2011, desUBIKado'
 __author__      = 'desUBIKado'
 __version__     = 'v0.09'
-__date__        = '02, December 2012'
+__date__        = '14, May 2013'
 '''
 http://www.weblogssl.com/
 '''
@@ -56,15 +56,16 @@ class weblogssl(BasicNewsRecipe):
                           ,(u'Zona FandoM', u'http://feeds.weblogssl.com/zonafandom')
                           ,(u'Fandemia', u'http://feeds.weblogssl.com/fandemia')
                           ,(u'Tendencias', u'http://feeds.weblogssl.com/trendencias')
-                          ,(u'Beb\xe9s y m\xe1s', u'http://feeds.weblogssl.com/bebesymas')
+                          ,(u'Tendencias Belleza', u'http://feeds.weblogssl.com/trendenciasbelleza')
+                          ,(u'Tendencias Hombre', u'http://feeds.weblogssl.com/trendenciashombre')
+                          ,(u'Tendencias Shopping', u'http://feeds.weblogssl.com/trendenciasshopping')
                           ,(u'Directo al paladar', u'http://feeds.weblogssl.com/directoalpaladar')
                           ,(u'Compradicci\xf3n', u'http://feeds.weblogssl.com/compradiccion')
                           ,(u'Decoesfera', u'http://feeds.weblogssl.com/decoesfera')
                           ,(u'Embelezzia', u'http://feeds.weblogssl.com/embelezzia')
                           ,(u'Vit\xf3nica', u'http://feeds.weblogssl.com/vitonica')
                           ,(u'Ambiente G', u'http://feeds.weblogssl.com/ambienteg')
-                          ,(u'Tendencias Belleza', u'http://feeds.weblogssl.com/trendenciasbelleza')
+                          ,(u'Beb\xe9s y m\xe1s', u'http://feeds.weblogssl.com/bebesymas')
-                          ,(u'Tendencias Hombre', u'http://feeds.weblogssl.com/trendenciashombre')
                           ,(u'Peques y m\xe1s', u'http://feeds.weblogssl.com/pequesymas')
                           ,(u'Motorpasi\xf3n', u'http://feeds.weblogssl.com/motorpasion')
                           ,(u'Motorpasi\xf3n F1', u'http://feeds.weblogssl.com/motorpasionf1')
@@ -119,23 +120,6 @@ class weblogssl(BasicNewsRecipe):
 
         return soup
 
-    # Para obtener la url original del articulo a partir de la de "feedsportal"
-    # El siguiente código es gracias al usuario "bosplans" de www.mobileread.com
-    # http://www.mobileread.com/forums/showthread.php?t=130297
 
     def get_article_url(self, article):
-       link = article.get('link', None)
-       if link is None:
-           return article
-       # if link.split('/')[-4]=="xataka2":
-       #     return article.get('feedburner_origlink', article.get('link', article.get('guid')))
-       if link.split('/')[-4]=="xataka2":
            return article.get('guid', None)
-       if link.split('/')[-1]=="story01.htm":
-           link=link.split('/')[-2]
-           a=['0B','0C','0D','0E','0F','0G','0N'  ,'0L0S','0A']
-           b=['.' ,'/' ,'?' ,'-' ,'=' ,'&' ,'.com','www.','0']
-           for i in range(0,len(a)):
-              link=link.replace(a[i],b[i])
-           link="http://"+link
-       return link

setup/installer/linux/freeze2.py

@@ -38,7 +38,7 @@ binary_includes = [
                 '/lib/libz.so.1',
                 '/usr/lib/libtiff.so.5',
                 '/lib/libbz2.so.1',
-                '/usr/lib/libpoppler.so.28',
+                '/usr/lib/libpoppler.so.37',
                 '/usr/lib/libxml2.so.2',
                 '/usr/lib/libopenjpeg.so.2',
                 '/usr/lib/libxslt.so.1',

setup/installer/osx/app/main.py

@@ -378,7 +378,7 @@ class Py2App(object):
     @flush
     def add_poppler(self):
         info('\nAdding poppler')
-        for x in ('libpoppler.28.dylib',):
+        for x in ('libpoppler.37.dylib',):
             self.install_dylib(os.path.join(SW, 'lib', x))
         for x in ('pdftohtml', 'pdftoppm', 'pdfinfo'):
             self.install_dylib(os.path.join(SW, 'bin', x), False)

src/calibre/constants.py

@@ -66,10 +66,8 @@ else:
             filesystem_encoding = 'utf-8'
             # On linux, unicode arguments to os file functions are coerced to an ascii
             # bytestring if sys.getfilesystemencoding() == 'ascii', which is
-            # just plain dumb. So issue a warning.
+            # just plain dumb. This is fixed by the icu.py module which, when
-            print ('WARNING: You do not have the LANG environment variable set correctly. '
+            # imported changes ascii to utf-8
-                    'This will cause problems with non-ascii filenames. '
-                    'Set it to something like en_US.UTF-8.\n')
     except:
         filesystem_encoding = 'utf-8'
 

src/calibre/devices/android/driver.py

@@ -240,7 +240,8 @@ class ANDROID(USBMS):
             'ADVANCED', 'SGH-I727', 'USB_FLASH_DRIVER', 'ANDROID',
             'S5830I_CARD', 'MID7042', 'LINK-CREATE', '7035', 'VIEWPAD_7E',
             'NOVO7', 'MB526', '_USB#WYK7MSF8KE', 'TABLET_PC', 'F', 'MT65XX_MS',
-            'ICS', 'E400', '__FILE-STOR_GADG', 'ST80208-1', 'GT-S5660M_CARD', 'XT894']
+            'ICS', 'E400', '__FILE-STOR_GADG', 'ST80208-1', 'GT-S5660M_CARD', 'XT894', '_USB',
+    ]
     WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897',
             'FILE-STOR_GADGET', 'SGH-T959_CARD', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD',
             'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB', 'SGH-T849_CARD',
@@ -251,7 +252,9 @@ class ANDROID(USBMS):
             'FILE-CD_GADGET', 'GT-I9001_CARD', 'USB_2.0', 'XT875',
             'UMS_COMPOSITE', 'PRO', '.KOBO_VOX', 'SGH-T989_CARD', 'SGH-I727',
             'USB_FLASH_DRIVER', 'ANDROID', 'MID7042', '7035', 'VIEWPAD_7E',
-            'NOVO7', 'ADVANCED', 'TABLET_PC', 'F', 'E400_SD_CARD', 'ST80208-1', 'XT894']
+            'NOVO7', 'ADVANCED', 'TABLET_PC', 'F', 'E400_SD_CARD', 'ST80208-1', 'XT894',
+            '_USB',
+    ]
 
     OSX_MAIN_MEM = 'Android Device Main Memory'
 

src/calibre/devices/apple/driver.py

@@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en'
 
 import cStringIO, ctypes, datetime, os, re, shutil, sys, tempfile, time
 
-from calibre import fit_image, confirm_config_name, osx_version, strftime as _strftime
+from calibre import fit_image, confirm_config_name, strftime as _strftime
 from calibre.constants import (
     __appname__, __version__, isosx, iswindows, cache_dir as _cache_dir)
 from calibre.devices.errors import OpenFeedback, UserFeedback
@@ -1609,7 +1609,6 @@ class ITUNES(DriverBase):
                             if self.verbose:
                                 logger().info(" failed to write artwork")
 
-
             elif format == 'pdf':
                 if self.verbose:
                     logger().info("   unable to set PDF cover via automation interface")
@@ -1647,6 +1646,7 @@ class ITUNES(DriverBase):
     def _create_new_book(self, fpath, metadata, path, db_added, lb_added, thumb, format):
         '''
         '''
+        from calibre.utils.date import parse_date
         if self.verbose:
             logger().info(" %s._create_new_book()" % self.__class__.__name__)
 

src/calibre/devices/idevice/libimobiledevice.py

@@ -15,7 +15,9 @@ __copyright__ = '2013, Gregory Riker'
 import os, sys
 
 from collections import OrderedDict
-from ctypes import *
+from ctypes import (
+    c_int, c_long, c_void_p, c_char_p, Structure, POINTER, byref, cdll, c_char, c_ulonglong,
+    c_uint, c_ubyte, create_string_buffer, string_at)
 
 from calibre.constants import DEBUG, islinux, isosx, iswindows
 from calibre.devices.idevice.parse_xml import XmlPropertyListParser

src/calibre/devices/prst1/driver.py

@@ -50,10 +50,10 @@ class PRST1(USBMS):
 
     VENDOR_NAME        = 'SONY'
     WINDOWS_MAIN_MEM   = re.compile(
-            r'(PRS-T(1|2)&)'
+            r'(PRS-T(1|2|2N)&)'
             )
     WINDOWS_CARD_A_MEM = re.compile(
-            r'(PRS-T(1|2)__SD&)'
+            r'(PRS-T(1|2|2N)__SD&)'
             )
     MAIN_MEMORY_VOLUME_LABEL = 'SONY Reader Main Memory'
     STORAGE_CARD_VOLUME_LABEL = 'SONY Reader Storage Card'
@@ -66,7 +66,7 @@ class PRST1(USBMS):
 
     EXTRA_CUSTOMIZATION_MESSAGE = [
         _('Comma separated list of metadata fields '
-            'to turn into collections on the device. Possibilities include: ')+\
+            'to turn into collections on the device. Possibilities include: ')+
                     'series, tags, authors',
         _('Upload separate cover thumbnails for books') +
              ':::'+_('Normally, the SONY readers get the cover image from the'
@@ -194,11 +194,11 @@ class PRST1(USBMS):
                 time_offsets = {}
                 for i, row in enumerate(cursor):
                     try:
-                        comp_date = int(os.path.getmtime(self.normalize_path(prefix + row[0])) * 1000);
+                        comp_date = int(os.path.getmtime(self.normalize_path(prefix + row[0])) * 1000)
                     except (OSError, IOError, TypeError):
                         # In case the db has incorrect path info
                         continue
-                    device_date = int(row[1]);
+                    device_date = int(row[1])
                     offset = device_date - comp_date
                     time_offsets.setdefault(offset, 0)
                     time_offsets[offset] = time_offsets[offset] + 1
@@ -345,7 +345,7 @@ class PRST1(USBMS):
         # Insert the sequence Id if it doesn't
         query = ('INSERT INTO sqlite_sequence (name, seq) '
                 'SELECT ?, ? '
-                'WHERE NOT EXISTS (SELECT 1 FROM sqlite_sequence WHERE name = ?)');
+                'WHERE NOT EXISTS (SELECT 1 FROM sqlite_sequence WHERE name = ?)')
         cursor.execute(query, (table, sequence_id, table,))
 
         cursor.close()

src/calibre/devices/smart_device_app/driver.py

@@ -875,6 +875,9 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
             self.client_device_kind = result.get('deviceKind', '')
             self._debug('Client device kind', self.client_device_kind)
 
+            self.client_device_name = result.get('deviceName', self.client_device_kind)
+            self._debug('Client device name', self.client_device_name)
+
             self.max_book_packet_len = result.get('maxBookContentPacketLen',
                                                   self.BASE_PACKET_LEN)
             self._debug('max_book_packet_len', self.max_book_packet_len)
@@ -946,6 +949,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
         return False
 
     def get_gui_name(self):
+        if getattr(self, 'client_device_name', None):
+            return self.gui_name_template%(self.gui_name, self.client_device_name)
         if getattr(self, 'client_device_kind', None):
             return self.gui_name_template%(self.gui_name, self.client_device_kind)
         return self.gui_name

src/calibre/ebooks/docx/block_styles.py

@@ -208,7 +208,7 @@ class ParagraphStyle(object):
 
         # Misc.
         'text_indent', 'text_align', 'line_height', 'direction', 'background_color',
-        'numbering',
+        'numbering', 'font_family', 'font_size',
     )
 
     def __init__(self, pPr=None):
@@ -232,6 +232,8 @@ class ParagraphStyle(object):
             for s in XPath('./w:pStyle[@w:val]')(pPr):
                 self.linked_style = get(s, 'w:val')
 
+            self.font_family = self.font_size = inherit
+
         self._css = None
 
     def update(self, other):
@@ -271,10 +273,16 @@ class ParagraphStyle(object):
                 if val is not inherit:
                     c['margin-%s' % edge] = val
 
-            for x in ('text_indent', 'text_align', 'line_height', 'background_color'):
+            if self.line_height not in {inherit, '1'}:
+                c['line-height'] = self.line_height
+
+            for x in ('text_indent', 'text_align', 'background_color', 'font_family', 'font_size'):
                 val = getattr(self, x)
                 if val is not inherit:
+                    if x == 'font_size':
+                        val = '%.3gpt' % val
                     c[x.replace('_', '-')] = val
+
         return self._css
 
         # TODO: keepNext must be done at markup level

src/calibre/ebooks/docx/char_styles.py

@@ -113,6 +113,14 @@ def read_vert_align(parent, dest):
         if val and val in {'baseline', 'subscript', 'superscript'}:
             ans = val
     setattr(dest, 'vert_align', ans)
+
+def read_font_family(parent, dest):
+    ans = inherit
+    for col in XPath('./w:rFonts[@w:ascii]')(parent):
+        val = get(col, 'w:ascii')
+        if val:
+            ans = val
+    setattr(dest, 'font_family', ans)
 # }}}
 
 class RunStyle(object):
@@ -122,7 +130,7 @@ class RunStyle(object):
         'rtl', 'shadow', 'smallCaps', 'strike', 'vanish',
 
         'border_color', 'border_style', 'border_width', 'padding', 'color', 'highlight', 'background_color',
-        'letter_spacing', 'font_size', 'text_decoration', 'vert_align', 'lang',
+        'letter_spacing', 'font_size', 'text_decoration', 'vert_align', 'lang', 'font_family'
     }
 
     toggle_properties = {
@@ -141,7 +149,7 @@ class RunStyle(object):
             ):
                 setattr(self, p, binary_property(rPr, p))
 
-            for x in ('text_border', 'color', 'highlight', 'shd', 'letter_spacing', 'sz', 'underline', 'vert_align', 'lang'):
+            for x in ('text_border', 'color', 'highlight', 'shd', 'letter_spacing', 'sz', 'underline', 'vert_align', 'lang', 'font_family'):
                 f = globals()['read_%s' % x]
                 f(rPr, self)
 
@@ -164,6 +172,18 @@ class RunStyle(object):
             if val is inherit:
                 setattr(self, p, getattr(parent, p))
 
+    def get_border_css(self, ans):
+        for x in ('color', 'style', 'width'):
+            val = getattr(self, 'border_'+x)
+            if x == 'width' and val is not inherit:
+                val = '%.3gpt' % val
+            if val is not inherit:
+                ans['border-%s' % x] = val
+
+    def clear_border_css(self):
+        for x in ('color', 'style', 'width'):
+            setattr(self, 'border_'+x, inherit)
+
     @property
     def css(self):
         if self._css is None:
@@ -188,12 +208,7 @@ class RunStyle(object):
             if self.vanish is True:
                 c['display'] = 'none'
 
-            for x in ('color', 'style', 'width'):
+            self.get_border_css(c)
-                val = getattr(self, 'border_'+x)
-                if x == 'width' and val is not inherit:
-                    val = '%.3gpt' % val
-                if val is not inherit:
-                    c['border-%s' % x] = val
             if self.padding is not inherit:
                 c['padding'] = '%.3gpt' % self.padding
 
@@ -212,6 +227,10 @@ class RunStyle(object):
 
             if self.b:
                 c['font-weight'] = 'bold'
+
+            if self.font_family is not inherit:
+                c['font-family'] = self.font_family
+
         return self._css
 
     def same_border(self, other):

src/calibre/ebooks/docx/container.py

@@ -167,7 +167,9 @@ class DOCX(object):
 
     @property
     def document_relationships(self):
-        name = self.document_name
+        return self.get_relationships(self.document_name)
+
+    def get_relationships(self, name):
         base = '/'.join(name.split('/')[:-1])
         by_id, by_type = {}, {}
         parts = name.split('/')

src/calibre/ebooks/docx/dump.py

@@ -22,7 +22,7 @@ def dump(path):
         zf.extractall(dest)
 
     for f in walk(dest):
-        if f.endswith('.xml'):
+        if f.endswith('.xml') or f.endswith('.rels'):
             with open(f, 'r+b') as stream:
                 raw = stream.read()
                 root = etree.fromstring(raw)

src/calibre/ebooks/docx/fonts.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__ = 'GPL v3'
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+
+import os, re
+from collections import namedtuple
+
+from calibre.ebooks.docx.block_styles import binary_property, inherit
+from calibre.ebooks.docx.names import XPath, get
+from calibre.utils.filenames import ascii_filename
+from calibre.utils.fonts.scanner import font_scanner, NoFonts
+from calibre.utils.fonts.utils import panose_to_css_generic_family, is_truetype_font
+
+Embed = namedtuple('Embed', 'name key subsetted')
+
+def has_system_fonts(name):
+    try:
+        return bool(font_scanner.fonts_for_family(name))
+    except NoFonts:
+        return False
+
+def get_variant(bold=False, italic=False):
+    return {(False, False):'Regular', (False, True):'Italic',
+            (True, False):'Bold', (True, True):'BoldItalic'}[(bold, italic)]
+
+class Family(object):
+
+    def __init__(self, elem, embed_relationships):
+        self.name = self.family_name = get(elem, 'w:name')
+        self.alt_names = tuple(get(x, 'w:val') for x in XPath('./w:altName')(elem))
+        if self.alt_names and not has_system_fonts(self.name):
+            for x in self.alt_names:
+                if has_system_fonts(x):
+                    self.family_name = x
+                    break
+
+        self.embedded = {}
+        for x in ('Regular', 'Bold', 'Italic', 'BoldItalic'):
+            for y in XPath('./w:embed%s[@r:id]' % x)(elem):
+                rid = get(y, 'r:id')
+                key = get(y, 'w:fontKey')
+                subsetted = get(y, 'w:subsetted') in {'1', 'true', 'on'}
+                if rid in embed_relationships:
+                    self.embedded[x] = Embed(embed_relationships[rid], key, subsetted)
+
+        self.generic_family = 'auto'
+        for x in XPath('./w:family[@w:val]')(elem):
+            self.generic_family = get(x, 'w:val', 'auto')
+
+        ntt = binary_property(elem, 'notTrueType')
+        self.is_ttf = ntt is inherit or not ntt
+
+        self.panose1 = None
+        self.panose_name = None
+        for x in XPath('./w:panose1[@w:val]')(elem):
+            try:
+                v = get(x, 'w:val')
+                v = tuple(int(v[i:i+2], 16) for i in xrange(0, len(v), 2))
+            except (TypeError, ValueError, IndexError):
+                pass
+            else:
+                self.panose1 = v
+                self.panose_name = panose_to_css_generic_family(v)
+
+        self.css_generic_family = {'roman':'serif', 'swiss':'sans-serif', 'modern':'monospace',
+                                   'decorative':'fantasy', 'script':'cursive'}.get(self.generic_family, None)
+        self.css_generic_family = self.css_generic_family or self.panose_name or 'serif'
+
+
+class Fonts(object):
+
+    def __init__(self):
+        self.fonts = {}
+        self.used = set()
+
+    def __call__(self, root, embed_relationships, docx, dest_dir):
+        for elem in XPath('//w:font[@w:name]')(root):
+            self.fonts[get(elem, 'w:name')] = Family(elem, embed_relationships)
+
+    def family_for(self, name, bold=False, italic=False):
+        f = self.fonts.get(name, None)
+        if f is None:
+            return 'serif'
+        variant = get_variant(bold, italic)
+        self.used.add((name, variant))
+        name = f.name if variant in f.embedded else f.family_name
+        return '"%s", %s' % (name.replace('"', ''), f.css_generic_family)
+
+    def embed_fonts(self, dest_dir, docx):
+        defs = []
+        dest_dir = os.path.join(dest_dir, 'fonts')
+        for name, variant in self.used:
+            f = self.fonts[name]
+            if variant in f.embedded:
+                if not os.path.exists(dest_dir):
+                    os.mkdir(dest_dir)
+                fname = self.write(name, dest_dir, docx, variant)
+                if fname is not None:
+                    d = {'font-family':'"%s"' % name.replace('"', ''), 'src': 'url("fonts/%s")' % fname}
+                    if 'Bold' in variant:
+                        d['font-weight'] = 'bold'
+                    if 'Italic' in variant:
+                        d['font-style'] = 'italic'
+                    d = ['%s: %s' % (k, v) for k, v in d.iteritems()]
+                    d = ';\n\t'.join(d)
+                    defs.append('@font-face {\n\t%s\n}\n' % d)
+        return '\n'.join(defs)
+
+    def write(self, name, dest_dir, docx, variant):
+        f = self.fonts[name]
+        ef = f.embedded[variant]
+        raw = docx.read(ef.name)
+        prefix = raw[:32]
+        if ef.key:
+            key = re.sub(r'[^A-Fa-f0-9]', '', ef.key)
+            key = bytearray(reversed(tuple(int(key[i:i+2], 16) for i in xrange(0, len(key), 2))))
+            prefix = bytearray(prefix)
+            prefix = bytes(bytearray(prefix[i]^key[i % len(key)] for i in xrange(len(prefix))))
+        if not is_truetype_font(prefix):
+            return None
+        ext = 'otf' if prefix.startswith(b'OTTO') else 'ttf'
+        fname = ascii_filename('%s - %s.%s' % (name, variant, ext))
+        with open(os.path.join(dest_dir, fname), 'wb') as dest:
+            dest.write(prefix)
+            dest.write(raw[32:])
+
+        return fname
+

src/calibre/ebooks/docx/names.py

@@ -13,6 +13,7 @@ DOCPROPS  = 'http://schemas.openxmlformats.org/package/2006/relationships/metada
 APPPROPS  = 'http://schemas.openxmlformats.org/officeDocument/2006/relationships/extended-properties'
 STYLES    = 'http://schemas.openxmlformats.org/officeDocument/2006/relationships/styles'
 NUMBERING = 'http://schemas.openxmlformats.org/officeDocument/2006/relationships/numbering'
+FONTS     = 'http://schemas.openxmlformats.org/officeDocument/2006/relationships/fontTable'
 
 namespaces = {
     'mo': 'http://schemas.microsoft.com/office/mac/office/2008/main',

src/calibre/ebooks/docx/numbering.py

@@ -214,6 +214,8 @@ class Numbering(object):
                         p.set('list-template', val)
                     self.update_counter(counter, ilvl, d.levels)
 
+        templates = {}
+
         def commit(current_run):
             if not current_run:
                 return
@@ -244,6 +246,9 @@ class Numbering(object):
                         span.append(gc)
                     child.append(span)
                     span = SPAN(child.get('list-template'))
+                    last = templates.get(lvlid, '')
+                    if span.text and len(span.text) > len(last):
+                        templates[lvlid] = span.text
                     child.insert(0, span)
                 for attr in ('list-lvl', 'list-id', 'list-template'):
                     child.attrib.pop(attr, None)
@@ -272,8 +277,14 @@ class Numbering(object):
             commit(current_run)
 
         for wrap in body.xpath('//ol[@lvlid]'):
-            wrap.attrib.pop('lvlid')
+            lvlid = wrap.attrib.pop('lvlid')
             wrap.tag = 'div'
+            text = ''
+            maxtext = templates.get(lvlid, '').replace('.', '')[:-1]
+            for li in wrap.iterchildren('li'):
+                t = li[0].text
+                if t and len(t) > len(text):
+                    text = t
             for i, li in enumerate(wrap.iterchildren('li')):
                 li.tag = 'div'
                 li.attrib.pop('value', None)
@@ -281,7 +292,8 @@ class Numbering(object):
                 obj = object_map[li]
                 bs = styles.para_cache[obj]
                 if i == 0:
-                    wrap.set('style', 'display:table; margin-left: %s' % (bs.css.get('margin-left', 0)))
+                    m = len(maxtext)  # Move the table left to simulate the behavior of a list (number is to the left of text margin)
+                    wrap.set('style', 'display:table; margin-left: -%dem; padding-left: %s' % (m, bs.css.get('margin-left', 0)))
                 bs.css.pop('margin-left', None)
                 for child in li:
                     child.set('style', 'display:table-cell')

src/calibre/ebooks/docx/styles.py

@@ -97,7 +97,8 @@ class Styles(object):
     def get(self, key, default=None):
         return self.id_map.get(key, default)
 
-    def __call__(self, root):
+    def __call__(self, root, fonts):
+        self.fonts = fonts
         for s in XPath('//w:style')(root):
             s = Style(s)
             if s.style_id:
@@ -246,6 +247,9 @@ class Styles(object):
             for attr in ans.all_properties:
                 setattr(ans, attr, self.run_val(parent_styles, direct_formatting, attr))
 
+            if ans.font_family is not inherit:
+                ans.font_family = self.fonts.family_for(ans.font_family, ans.b, ans.i)
+
         return ans
 
     def resolve(self, obj):
@@ -254,6 +258,55 @@ class Styles(object):
         if obj.tag.endswith('}r'):
             return self.resolve_run(obj)
 
+    def cascade(self, layers):
+        self.body_font_family = 'serif'
+        self.body_font_size = '10pt'
+
+        for p, runs in layers.iteritems():
+            char_styles = [self.resolve_run(r) for r in runs]
+            block_style = self.resolve_paragraph(p)
+            c = Counter()
+            for s in char_styles:
+                if s.font_family is not inherit:
+                    c[s.font_family] += 1
+            if c:
+                family = c.most_common(1)[0][0]
+                block_style.font_family = family
+                for s in char_styles:
+                    if s.font_family == family:
+                        s.font_family = inherit
+
+            sizes = [s.font_size for s in char_styles if s.font_size is not inherit]
+            if sizes:
+                sz = block_style.font_size = sizes[0]
+                for s in char_styles:
+                    if s.font_size == sz:
+                        s.font_size = inherit
+
+        block_styles = [self.resolve_paragraph(p) for p in layers]
+        c = Counter()
+        for s in block_styles:
+            if s.font_family is not inherit:
+                c[s.font_family] += 1
+
+        if c:
+            self.body_font_family = family = c.most_common(1)[0][0]
+            for s in block_styles:
+                if s.font_family == family:
+                    s.font_family = inherit
+
+        c = Counter()
+        for s in block_styles:
+            if s.font_size is not inherit:
+                c[s.font_size] += 1
+
+        if c:
+            sz = c.most_common(1)[0][0]
+            for s in block_styles:
+                if s.font_size == sz:
+                    s.font_size = inherit
+            self.body_font_size = '%.3gpt' % sz
+
     def resolve_numbering(self, numbering):
         # When a numPr element appears inside a paragraph style, the lvl info
         # must be discarder and pStyle used instead.
@@ -290,13 +343,18 @@ class Styles(object):
         h = hash(frozenset(css.iteritems()))
         return self.classes.get(h, (None, None))[0]
 
-    def generate_css(self):
+    def generate_css(self, dest_dir, docx):
+        ef = self.fonts.embed_fonts(dest_dir, docx)
         prefix = textwrap.dedent(
             '''\
+            body { font-family: %s; font-size: %s }
+
             p { text-indent: 1.5em }
 
             ul, ol, p { margin: 0; padding: 0 }
-            ''')
+            ''') % (self.body_font_family, self.body_font_size)
+        if ef:
+            prefix = ef + '\n' + prefix
 
         ans = []
         for (cls, css) in sorted(self.classes.itervalues(), key=lambda x:x[0]):

src/calibre/ebooks/docx/to_html.py

@@ -14,9 +14,10 @@ from lxml.html.builder import (
     HTML, HEAD, TITLE, BODY, LINK, META, P, SPAN, BR)
 
 from calibre.ebooks.docx.container import DOCX, fromstring
-from calibre.ebooks.docx.names import XPath, is_tag, barename, XML, STYLES, NUMBERING
+from calibre.ebooks.docx.names import XPath, is_tag, XML, STYLES, NUMBERING, FONTS
 from calibre.ebooks.docx.styles import Styles, inherit
 from calibre.ebooks.docx.numbering import Numbering
+from calibre.ebooks.docx.fonts import Fonts
 from calibre.utils.localization import canonicalize_lang, lang_as_iso639_1
 
 class Text:
@@ -63,16 +64,14 @@ class Convert(object):
         doc = self.docx.document
         relationships_by_id, relationships_by_type = self.docx.document_relationships
         self.read_styles(relationships_by_type)
-        for top_level in XPath('/w:document/w:body/*')(doc):
+        self.layers = OrderedDict()
-            if is_tag(top_level, 'w:p'):
+        for wp in XPath('//w:p')(doc):
-                p = self.convert_p(top_level)
+            p = self.convert_p(wp)
             self.body.append(p)
-            elif is_tag(top_level, 'w:tbl'):
+        # TODO: tables <w:tbl> child of <w:body> (nested tables?)
-                pass  # TODO: tables
+        # TODO: Last section properties <w:sectPr> child of <w:body>
-            elif is_tag(top_level, 'w:sectPr'):
+
-                pass  # TODO: Last section properties
+        self.styles.cascade(self.layers)
-            else:
-                self.log.debug('Unknown top-level tag: %s, ignoring' % barename(top_level.tag))
 
         numbered = []
         for html_obj, obj in self.object_map.iteritems():
@@ -116,7 +115,18 @@ class Convert(object):
 
         nname = get_name(NUMBERING, 'numbering.xml')
         sname = get_name(STYLES, 'styles.xml')
+        fname = get_name(FONTS, 'fontTable.xml')
         numbering = self.numbering = Numbering()
+        fonts = self.fonts = Fonts()
+
+        if fname is not None:
+            embed_relationships = self.docx.get_relationships(fname)[0]
+            try:
+                raw = self.docx.read(fname)
+            except KeyError:
+                self.log.warn('Fonts table %s does not exist' % fname)
+            else:
+                fonts(fromstring(raw), embed_relationships, self.docx, self.dest_dir)
 
         if sname is not None:
             try:
@@ -124,7 +134,7 @@ class Convert(object):
             except KeyError:
                 self.log.warn('Styles %s do not exist' % sname)
             else:
-                self.styles(fromstring(raw))
+                self.styles(fromstring(raw), fonts)
 
         if nname is not None:
             try:
@@ -140,7 +150,7 @@ class Convert(object):
         raw = html.tostring(self.html, encoding='utf-8', doctype='<!DOCTYPE html>')
         with open(os.path.join(self.dest_dir, 'index.html'), 'wb') as f:
             f.write(raw)
-        css = self.styles.generate_css()
+        css = self.styles.generate_css(self.dest_dir, self.docx)
         if css:
             with open(os.path.join(self.dest_dir, 'docx.css'), 'wb') as f:
                 f.write(css.encode('utf-8'))
@@ -149,9 +159,11 @@ class Convert(object):
         dest = P()
         self.object_map[dest] = p
         style = self.styles.resolve_paragraph(p)
+        self.layers[p] = []
         for run in XPath('descendant::w:r')(p):
             span = self.convert_run(run)
             dest.append(span)
+            self.layers[p].append(run)
 
         m = re.match(r'heading\s+(\d+)$', style.style_name or '', re.IGNORECASE)
         if m is not None:
@@ -177,12 +189,9 @@ class Convert(object):
             spans = []
             bs = {}
             for span, style in border_run:
-                c = style.css
+                style.get_border_css(bs)
+                style.clear_border_css()
                 spans.append(span)
-                for x in ('width', 'color', 'style'):
-                    val = c.pop('border-%s' % x, None)
-                    if val is not None:
-                        bs['border-%s' % x] = val
             if bs:
                 cls = self.styles.register(bs, 'text_border')
                 wrapper = self.wrap_elems(spans, SPAN())

src/calibre/ebooks/oeb/polish/toc.py

@@ -9,7 +9,7 @@ __docformat__ = 'restructuredtext en'
 
 import re
 from urlparse import urlparse
-from collections import deque
+from collections import deque, Counter
 from functools import partial
 
 from lxml import etree
@@ -29,7 +29,8 @@ class TOC(object):
     def __init__(self, title=None, dest=None, frag=None):
         self.title, self.dest, self.frag = title, dest, frag
         self.dest_exists = self.dest_error = None
-        if self.title: self.title = self.title.strip()
+        if self.title:
+            self.title = self.title.strip()
         self.parent = None
         self.children = []
 
@@ -326,11 +327,13 @@ def create_ncx(toc, to_href, btitle, lang, uid):
     navmap = etree.SubElement(ncx, NCX('navMap'))
     spat = re.compile(r'\s+')
 
-    def process_node(xml_parent, toc_parent, play_order=0):
+    play_order = Counter()
+
+    def process_node(xml_parent, toc_parent):
         for child in toc_parent:
-            play_order += 1
+            play_order['c'] += 1
             point = etree.SubElement(xml_parent, NCX('navPoint'), id=uuid_id(),
-                            playOrder=str(play_order))
+                            playOrder=str(play_order['c']))
             label = etree.SubElement(point, NCX('navLabel'))
             title = child.title
             if title:
@@ -341,7 +344,7 @@ def create_ncx(toc, to_href, btitle, lang, uid):
                 if child.frag:
                     href += '#'+child.frag
                 etree.SubElement(point, NCX('content'), src=href)
-            process_node(point, child, play_order)
+            process_node(point, child)
 
     process_node(navmap, toc)
     return ncx

src/calibre/gui2/dialogs/template_dialog.py

@@ -3,17 +3,21 @@ __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __license__   = 'GPL v3'
 
-import json
+import json, os, traceback
 
 from PyQt4.Qt import (Qt, QDialog, QDialogButtonBox, QSyntaxHighlighter, QFont,
-                      QRegExp, QApplication, QTextCharFormat, QColor, QCursor)
+                      QRegExp, QApplication, QTextCharFormat, QColor, QCursor,
+                      QIcon, QSize)
 
-from calibre.gui2 import error_dialog
+from calibre import sanitize_file_name_unicode
+from calibre.constants import config_dir
 from calibre.gui2.dialogs.template_dialog_ui import Ui_TemplateDialog
 from calibre.utils.formatter_functions import formatter_functions
+from calibre.utils.icu import sort_key
 from calibre.ebooks.metadata.book.base import Metadata
 from calibre.ebooks.metadata.book.formatter import SafeFormat
-from calibre.library.coloring import (displayable_columns)
+from calibre.library.coloring import (displayable_columns, color_row_key)
+from calibre.gui2 import error_dialog, choose_files, pixmap_to_data
 
 
 class ParenPosition:
@@ -198,25 +202,56 @@ class TemplateHighlighter(QSyntaxHighlighter):
 
 class TemplateDialog(QDialog, Ui_TemplateDialog):
 
-    def __init__(self, parent, text, mi=None, fm=None, color_field=None):
+    def __init__(self, parent, text, mi=None, fm=None, color_field=None,
+                 icon_field_key=None, icon_rule_kind=None):
         QDialog.__init__(self, parent)
         Ui_TemplateDialog.__init__(self)
         self.setupUi(self)
 
         self.coloring = color_field is not None
+        self.iconing = icon_field_key is not None
+
+        cols = []
+        if fm is not None:
+            for key in sorted(displayable_columns(fm),
+                              key=lambda(k): sort_key(fm[k]['name']) if k != color_row_key else 0):
+                if key == color_row_key and not self.coloring:
+                    continue
+                from calibre.gui2.preferences.coloring import all_columns_string
+                name = all_columns_string if key == color_row_key else fm[key]['name']
+                if name:
+                    cols.append((name, key))
+
+        self.color_layout.setVisible(False)
+        self.icon_layout.setVisible(False)
+
         if self.coloring:
-            cols = sorted([k for k in displayable_columns(fm)])
+            self.color_layout.setVisible(True)
-            self.colored_field.addItems(cols)
+            for n1, k1 in cols:
-            self.colored_field.setCurrentIndex(self.colored_field.findText(color_field))
+                self.colored_field.addItem(n1, k1)
+            self.colored_field.setCurrentIndex(self.colored_field.findData(color_field))
             colors = QColor.colorNames()
             colors.sort()
             self.color_name.addItems(colors)
-        else:
+        elif self.iconing:
-            self.colored_field.setVisible(False)
+            self.icon_layout.setVisible(True)
-            self.colored_field_label.setVisible(False)
+            for n1, k1 in cols:
-            self.color_chooser_label.setVisible(False)
+                self.icon_field.addItem(n1, k1)
-            self.color_name.setVisible(False)
+            self.icon_file_names = []
-            self.color_copy_button.setVisible(False)
+            d = os.path.join(config_dir, 'cc_icons')
+            if os.path.exists(d):
+                for icon_file in os.listdir(d):
+                    icon_file = icu_lower(icon_file)
+                    if os.path.exists(os.path.join(d, icon_file)):
+                        if icon_file.endswith('.png'):
+                            self.icon_file_names.append(icon_file)
+            self.icon_file_names.sort(key=sort_key)
+            self.update_filename_box()
+            self.icon_with_text.setChecked(True)
+            if icon_rule_kind == 'icon_only':
+                self.icon_without_text.setChecked(True)
+            self.icon_field.setCurrentIndex(self.icon_field.findData(icon_field_key))
+
         if mi:
             self.mi = mi
         else:
@@ -248,6 +283,8 @@ class TemplateDialog(QDialog, Ui_TemplateDialog):
         self.buttonBox.button(QDialogButtonBox.Ok).setText(_('&OK'))
         self.buttonBox.button(QDialogButtonBox.Cancel).setText(_('&Cancel'))
         self.color_copy_button.clicked.connect(self.color_to_clipboard)
+        self.filename_button.clicked.connect(self.filename_button_clicked)
+        self.icon_copy_button.clicked.connect(self.icon_to_clipboard)
 
         try:
             with open(P('template-functions.json'), 'rb') as f:
@@ -276,11 +313,55 @@ class TemplateDialog(QDialog, Ui_TemplateDialog):
                 '<a href="http://manual.calibre-ebook.com/template_ref.html">'
                 '%s</a>'%tt)
 
+    def filename_button_clicked(self):
+        try:
+            path = choose_files(self, 'choose_category_icon',
+                        _('Select Icon'), filters=[
+                        ('Images', ['png', 'gif', 'jpg', 'jpeg'])],
+                    all_files=False, select_only_single_file=True)
+            if path:
+                icon_path = path[0]
+                icon_name = sanitize_file_name_unicode(
+                             os.path.splitext(
+                                   os.path.basename(icon_path))[0]+'.png')
+                if icon_name not in self.icon_file_names:
+                    self.icon_file_names.append(icon_name)
+                    self.update_filename_box()
+                    try:
+                        p = QIcon(icon_path).pixmap(QSize(128, 128))
+                        d = os.path.join(config_dir, 'cc_icons')
+                        if not os.path.exists(os.path.join(d, icon_name)):
+                            if not os.path.exists(d):
+                                os.makedirs(d)
+                            with open(os.path.join(d, icon_name), 'wb') as f:
+                                f.write(pixmap_to_data(p, format='PNG'))
+                    except:
+                        traceback.print_exc()
+                self.icon_files.setCurrentIndex(self.icon_files.findText(icon_name))
+                self.icon_files.adjustSize()
+        except:
+            traceback.print_exc()
+        return
+
+    def update_filename_box(self):
+        self.icon_files.clear()
+        self.icon_file_names.sort(key=sort_key)
+        self.icon_files.addItem('')
+        self.icon_files.addItems(self.icon_file_names)
+        for i,filename in enumerate(self.icon_file_names):
+            icon = QIcon(os.path.join(config_dir, 'cc_icons', filename))
+            self.icon_files.setItemIcon(i+1, icon)
+
     def color_to_clipboard(self):
         app = QApplication.instance()
         c = app.clipboard()
         c.setText(unicode(self.color_name.currentText()))
 
+    def icon_to_clipboard(self):
+        app = QApplication.instance()
+        c = app.clipboard()
+        c.setText(unicode(self.icon_files.currentText()))
+
     def textbox_changed(self):
         cur_text = unicode(self.textbox.toPlainText())
         if self.last_text != cur_text:
@@ -324,5 +405,14 @@ class TemplateDialog(QDialog, Ui_TemplateDialog):
                     _('The template box cannot be empty'), show=True)
                 return
 
-        self.rule = (unicode(self.colored_field.currentText()), txt)
+            self.rule = (unicode(self.colored_field.itemData(
+                                self.colored_field.currentIndex()).toString()), txt)
+        elif self.iconing:
+            rt = 'icon' if self.icon_with_text.isChecked() else 'icon_only'
+            self.rule = (rt,
+                         unicode(self.icon_field.itemData(
+                                self.icon_field.currentIndex()).toString()),
+                         txt)
+        else:
+            self.rule = ('', txt)
         QDialog.accept(self)

src/calibre/gui2/dialogs/template_dialog.ui

@@ -21,6 +21,7 @@
   </property>
   <layout class="QVBoxLayout" name="verticalLayout">
    <item>
+    <widget class="QWidget" name="color_layout">
      <layout class="QGridLayout">
       <item row="0" column="0">
        <widget class="QLabel" name="colored_field_label">
@@ -62,6 +63,97 @@
        </widget>
       </item>
      </layout>
+    </widget>
+   </item>
+   <item>
+    <widget class="QWidget" name="icon_layout">
+     <layout class="QGridLayout">
+      <item row="0" column="0" colspan="2">
+       <widget class="QGroupBox">
+        <property name="title">
+         <string>Kind</string>
+        </property>
+        <layout class="QHBoxLayout">
+         <item>
+          <widget class="QRadioButton" name="icon_without_text">
+           <property name="text">
+            <string>icon with no text</string>
+           </property>
+          </widget>
+         </item>
+         <item>
+          <widget class="QRadioButton" name="icon_with_text">
+           <property name="text">
+            <string>icon with text</string>
+           </property>
+          </widget>
+         </item>
+        </layout>
+        <property name="sizePolicy">
+         <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
+          <horstretch>100</horstretch>
+          <verstretch>0</verstretch>
+         </sizepolicy>
+        </property>
+       </widget>
+      </item>
+      <item row="1" column="0">
+       <widget class="QLabel" name="icon_chooser_label">
+        <property name="text">
+         <string>Apply the icon to column:</string>
+        </property>
+        <property name="buddy">
+         <cstring>icon_field</cstring>
+        </property>
+       </widget>
+      </item>
+      <item row="1" column="1">
+       <widget class="QComboBox" name="icon_field">
+       </widget>
+      </item>
+      <item row="2" column="0">
+       <widget class="QLabel" name="image_chooser_label">
+        <property name="text">
+         <string>Copy an icon file name to the clipboard:</string>
+        </property>
+        <property name="buddy">
+         <cstring>color_name</cstring>
+        </property>
+       </widget>
+      </item>
+      <item row="2" column="1">
+       <widget class="QWidget">
+        <layout class="QHBoxLayout">
+         <item>
+          <widget class="QComboBox" name="icon_files">
+          </widget>
+         </item>
+         <item>
+          <widget class="QToolButton" name="icon_copy_button">
+           <property name="icon">
+            <iconset resource="../../../../resources/images.qrc">
+             <normaloff>:/images/edit-copy.png</normaloff>:/images/edit-copy.png</iconset>
+           </property>
+           <property name="toolTip">
+            <string>Copy the selected icon file name to the clipboard</string>
+           </property>
+          </widget>
+         </item>
+         <item>
+          <widget class="QPushButton" name="filename_button">
+           <property name="text">
+            <string>Add icon</string>
+           </property>
+           <property name="toolTip">
+            <string>Add an icon file to the set of choices</string>
+           </property>
+          </widget>
+         </item>
+        </layout>
+       </widget>
+      </item>
+     </layout>
+    </widget>
    </item>
    <item>
     <widget class="QPlainTextEdit" name="textbox"/>

src/calibre/gui2/preferences/coloring.py

@@ -636,10 +636,20 @@ class RulesModel(QAbstractListModel): # {{{
 
     def rule_to_html(self, kind, col, rule):
         if not isinstance(rule, Rule):
+            if kind == 'color':
                 return _('''
                 <p>Advanced Rule for column <b>%(col)s</b>:
                 <pre>%(rule)s</pre>
                 ''')%dict(col=col, rule=prepare_string_for_xml(rule))
+            else:
+                return _('''
+                <p>Advanced Rule: set <b>%(typ)s</b> for column <b>%(col)s</b>:
+                <pre>%(rule)s</pre>
+                ''')%dict(col=col,
+                          typ=icon_rule_kinds[0][0]
+                            if kind == icon_rule_kinds[0][1] else icon_rule_kinds[1][0],
+                          rule=prepare_string_for_xml(rule))
+
         conditions = [self.condition_to_html(c) for c in rule.conditions]
 
         trans_kind = 'not found'
@@ -761,7 +771,7 @@ class EditRules(QWidget): # {{{
                 ' what icon to use. Click the Add Rule button below'
                 ' to get started.<p>You can <b>change an existing rule</b> by'
                 ' double clicking it.'))
-            self.add_advanced_button.setVisible(False)
+#             self.add_advanced_button.setVisible(False)
 
     def add_rule(self):
         d = RuleEditor(self.model.fm, self.pref_name)
@@ -774,6 +784,7 @@ class EditRules(QWidget): # {{{
                 self.changed.emit()
 
     def add_advanced(self):
+        if self.pref_name == 'column_color_rules':
             td = TemplateDialog(self, '', mi=self.mi, fm=self.fm, color_field='')
             if td.exec_() == td.Accepted:
                 col, r = td.rule
@@ -781,6 +792,15 @@ class EditRules(QWidget): # {{{
                     idx = self.model.add_rule('color', col, r)
                     self.rules_view.scrollTo(idx)
                     self.changed.emit()
+        else:
+            td = TemplateDialog(self, '', mi=self.mi, fm=self.fm, icon_field_key='')
+            if td.exec_() == td.Accepted:
+                print(td.rule)
+                typ, col, r = td.rule
+                if typ and r and col:
+                    idx = self.model.add_rule(typ, col, r)
+                    self.rules_view.scrollTo(idx)
+                    self.changed.emit()
 
     def edit_rule(self, index):
         try:
@@ -790,8 +810,12 @@ class EditRules(QWidget): # {{{
         if isinstance(rule, Rule):
             d = RuleEditor(self.model.fm, self.pref_name)
             d.apply_rule(kind, col, rule)
-        else:
+        elif self.pref_name == 'column_color_rules':
             d = TemplateDialog(self, rule, mi=self.mi, fm=self.fm, color_field=col)
+        else:
+            d = TemplateDialog(self, rule, mi=self.mi, fm=self.fm, icon_field_key=col,
+                               icon_rule_kind=kind)
+
         if d.exec_() == d.Accepted:
             if len(d.rule) == 2: # Convert template dialog rules to a triple
                 d.rule = ('color', d.rule[0], d.rule[1])

src/calibre/utils/fonts/scanner.py

@@ -13,13 +13,82 @@ from threading import Thread
 
 from calibre import walk, prints, as_unicode
 from calibre.constants import (config_dir, iswindows, isosx, plugins, DEBUG,
-        isworker)
+        isworker, filesystem_encoding)
 from calibre.utils.fonts.metadata import FontMetadata, UnsupportedFont
 from calibre.utils.icu import sort_key
 
 class NoFonts(ValueError):
     pass
 
+
+def default_font_dirs():
+    return [
+        '/opt/share/fonts',
+        '/usr/share/fonts',
+        '/usr/local/share/fonts',
+        os.path.expanduser('~/.local/share/fonts'),
+        os.path.expanduser('~/.fonts')
+    ]
+
+
+def fc_list():
+    import ctypes
+    from ctypes.util import find_library
+
+    lib = find_library('fontconfig')
+    if lib is None:
+        return default_font_dirs()
+    try:
+        lib = ctypes.CDLL(lib)
+    except:
+        return default_font_dirs()
+
+    prototype = ctypes.CFUNCTYPE(ctypes.c_void_p, ctypes.c_void_p)
+    try:
+        get_font_dirs = prototype(('FcConfigGetFontDirs', lib))
+    except (AttributeError):
+        return default_font_dirs()
+    prototype = ctypes.CFUNCTYPE(ctypes.c_char_p, ctypes.c_void_p)
+    try:
+        next_dir = prototype(('FcStrListNext', lib))
+    except (AttributeError):
+        return default_font_dirs()
+
+    prototype = ctypes.CFUNCTYPE(None, ctypes.c_void_p)
+    try:
+        end = prototype(('FcStrListDone', lib))
+    except (AttributeError):
+        return default_font_dirs()
+
+    str_list = get_font_dirs(ctypes.c_void_p())
+    if not str_list:
+        return default_font_dirs()
+
+    ans = []
+    while True:
+        d = next_dir(str_list)
+        if not d:
+            break
+        if d:
+            try:
+                ans.append(d.decode(filesystem_encoding))
+            except ValueError:
+                return default_font_dirs
+    end(str_list)
+    if len(ans) < 3:
+        return default_font_dirs()
+    parents = []
+    for f in ans:
+        found = False
+        for p in parents:
+            if f.startswith(p):
+                found = True
+                break
+        if not found:
+            parents.append(f)
+    return parents
+
+
 def font_dirs():
     if iswindows:
         winutil, err = plugins['winutil']
@@ -35,12 +104,7 @@ def font_dirs():
                 os.path.expanduser('~/.fonts'),
                 os.path.expanduser('~/Library/Fonts'),
                 ]
-    return [
+    return fc_list()
-            '/opt/share/fonts',
-            '/usr/share/fonts',
-            '/usr/local/share/fonts',
-            os.path.expanduser('~/.fonts')
-            ]
 
 class Scanner(Thread):
 
@@ -133,7 +197,8 @@ class Scanner(Thread):
 
         for family in self.find_font_families():
             faces = filter(filter_faces, self.fonts_for_family(family))
-            if not faces: continue
+            if not faces:
+                continue
             generic_family = panose_to_css_generic_family(faces[0]['panose'])
             if generic_family in allowed_families or generic_family == preferred_families[0]:
                 return (family, faces)
@@ -233,7 +298,8 @@ class Scanner(Thread):
     def build_families(self):
         families = defaultdict(list)
         for f in self.cached_fonts.itervalues():
-            if not f: continue
+            if not f:
+                continue
             lf = icu_lower(f['font-family'] or '')
             if lf:
                 families[lf].append(f)

src/calibre/utils/icu.c

@@ -661,6 +661,17 @@ icu_set_default_encoding(PyObject *self, PyObject *args) {
 }
 // }}}
 
+// set_default_encoding {{{
+static PyObject *
+icu_set_filesystem_encoding(PyObject *self, PyObject *args) {
+    char *encoding;
+    if (!PyArg_ParseTuple(args, "s:setfilesystemencoding", &encoding))
+        return NULL;
+    Py_FileSystemDefaultEncoding = strdup(encoding);
+    Py_RETURN_NONE;
+
+}
+// }}}
 // set_default_encoding {{{
 static PyObject *
 icu_get_available_transliterators(PyObject *self, PyObject *args) {
@@ -707,6 +718,10 @@ static PyMethodDef icu_methods[] = {
         "set_default_encoding(encoding) -> Set the default encoding for the python unicode implementation."
     },
 
+    {"set_filesystem_encoding", icu_set_filesystem_encoding, METH_VARARGS,
+        "set_filesystem_encoding(encoding) -> Set the filesystem encoding for python."
+    },
+
     {"get_available_transliterators", icu_get_available_transliterators, METH_VARARGS,
         "get_available_transliterators() -> Return list of available transliterators. This list is rather limited on OS X."
     },

src/calibre/utils/icu.py

@@ -163,11 +163,22 @@ load_collator()
 _icu_not_ok = _icu is None or _collator is None
 
 try:
-    if sys.getdefaultencoding().lower() == 'ascii':
+    senc = sys.getdefaultencoding()
+    if not senc or senc.lower() == 'ascii':
         _icu.set_default_encoding('utf-8')
+    del senc
 except:
     pass
 
+try:
+    fenc = sys.getfilesystemencoding()
+    if not fenc or fenc.lower() == 'ascii':
+        _icu.set_filesystem_encoding('utf-8')
+    del fenc
+except:
+    pass
+
+
 # }}}
 
 ################# The string functions ########################################