Pulled upstream changes

src/calibre/__init__.py

@@ -30,6 +30,11 @@ islinux   = not(iswindows or isosx)
 try:
     locale.setlocale(locale.LC_ALL, '')
 except:
+    dl = locale.getdefaultlocale()
+    try:
+        if dl:
+            locale.setlocale(dl[0])
+    except:
         pass
 
 try:

src/calibre/ebooks/lrf/fb2/convert_from.py

@@ -3,15 +3,13 @@ __copyright__ = '2008, Anatoly Shipitsin <norguhtar at gmail.com>'
 """
 Convert .fb2 files to .lrf
 """
-import os, sys, tempfile, subprocess, shutil, logging, glob
+import os, sys, tempfile, shutil, logging
+from base64 import b64decode
 
-from calibre.ptempfile import PersistentTemporaryFile
 from calibre.ebooks.lrf import option_parser as lrf_option_parser
 from calibre.ebooks.metadata.meta import get_metadata
-from calibre.ebooks import ConversionError
 from calibre.ebooks.lrf.html.convert_from import process_file as html_process_file
 from calibre import setup_cli_handlers, __appname__
-from calibre.ebooks.BeautifulSoup import BeautifulStoneSoup
 from calibre.resources import fb2_xsl
 
 def option_parser():
@@ -22,25 +20,27 @@ _('''%prog [options] mybook.fb2
 %prog converts mybook.fb2 to mybook.lrf'''))
     parser.add_option('--debug-html-generation', action='store_true', default=False,
                       dest='debug_html_generation', help=_('Print generated HTML to stdout and quit.'))
+    parser.add_option('--keep-intermediate-files', action='store_true', default=False,
+                      help=_('Keep generated HTML files after completing conversion to LRF.'))
     return parser
     
+def extract_embedded_content(doc):
+    for elem in doc.xpath('./*'):
+        if 'binary' in elem.tag and elem.attrib.has_key('id'):
+            fname = elem.attrib['id']
+            data = b64decode(elem.text.strip())
+            open(fname, 'wb').write(data)
 
 def generate_html(fb2file, encoding, logger):
     from lxml import etree
-    tdir = tempfile.mkdtemp(prefix=__appname__+'_')
+    tdir = tempfile.mkdtemp(prefix=__appname__+'_fb2_')
-    ofile = os.path.join(tdir, 'index.xml')
     cwd = os.getcwdu()
     os.chdir(tdir)
     try:
         logger.info('Parsing XML...')
         parser = etree.XMLParser(recover=True, no_network=True)
-        try:
         doc = etree.parse(fb2file, parser)
-        except:
+        extract_embedded_content(doc)
-            raise
-            logger.info('Parsing failed. Trying to clean up XML...')
-            soup = BeautifulStoneSoup(open(fb2file, 'rb').read())
-            doc = etree.fromstring(str(soup))
         logger.info('Converting XML to HTML...')
         styledoc = etree.fromstring(fb2_xsl)
 
@@ -72,7 +72,7 @@ def process_file(path, options, logger=None):
             options.output = os.path.abspath(os.path.basename(os.path.splitext(path)[0]) + ext)
         options.output = os.path.abspath(os.path.expanduser(options.output))
         if not mi.title:
-            mi.title = os.path.splitext(os.path.basename(rtf))[0]
+            mi.title = os.path.splitext(os.path.basename(fb2))[0]
         if (not options.title or options.title == 'Unknown'):
             options.title = mi.title
         if (not options.author or options.author == 'Unknown') and mi.authors:
@@ -85,7 +85,7 @@ def process_file(path, options, logger=None):
         html_process_file(htmlfile, options, logger)
     finally:
         os.chdir(cwd)
-        if hasattr(options, 'keep_intermediate_files') and options.keep_intermediate_files:
+        if getattr(options, 'keep_intermediate_files', False):
             logger.debug('Intermediate files in '+ tdir)
         else:
             shutil.rmtree(tdir)

src/calibre/ebooks/lrf/fb2/fb2.xsl

@@ -128,21 +128,40 @@
 	</xsl:template>
 
 	<xsl:template match="fb:section">
-		<a name="TOC_{generate-id()}"></a>
+        <xsl:variable name="section_has_title">
+            <xsl:choose>
+                <xsl:when test="./fb:title"><xsl:value-of select="generate-id()" /></xsl:when>
+                <xsl:otherwise>None</xsl:otherwise>
+            </xsl:choose>
+        </xsl:variable>
+        <xsl:if test="$section_has_title = 'None'">
+            <a name="TOC_{generate-id()}" />
             <xsl:if test="@id">
                 <xsl:element name="a">
                     <xsl:attribute name="name"><xsl:value-of select="@id"/></xsl:attribute>
                 </xsl:element>
             </xsl:if>
-		<xsl:apply-templates/>
+        </xsl:if>
+        <xsl:apply-templates>
+            <xsl:with-param name="section_toc_id" select="$section_has_title" />
+        </xsl:apply-templates>
 	</xsl:template>
 	
 	
 	<!-- section/title -->
 	<xsl:template match="fb:section/fb:title|fb:poem/fb:title">
+        <xsl:param name="section_toc_id" />
 		<xsl:choose>
 			<xsl:when test="count(ancestor::node()) &lt; 9">
 				<xsl:element name="{concat('h',count(ancestor::node())-3)}">
+                    <xsl:if test="../@id">
+                        <xsl:attribute name="id"><xsl:value-of select="../@id" /></xsl:attribute>
+                    </xsl:if>
+                    <xsl:if test="$section_toc_id != 'None'">
+                        <xsl:element name="a">
+                            <xsl:attribute name="name">TOC_<xsl:value-of select="$section_toc_id"/></xsl:attribute>
+                        </xsl:element>
+                    </xsl:if>
 					<a name="TOC_{generate-id()}"></a>
 					<xsl:if test="@id">
 						<xsl:element name="a">
@@ -166,7 +185,9 @@
 	</xsl:template>
 	<!-- section/title -->
 	<xsl:template match="fb:body/fb:title">
-		<h1><xsl:apply-templates mode="title"/></h1>
+        <xsl:element name="h1">
+            <xsl:apply-templates mode="title"/>
+        </xsl:element>
 	</xsl:template>
 
 	<xsl:template match="fb:title/fb:p">

src/calibre/ebooks/metadata/fb2.py

@@ -5,7 +5,8 @@ __copyright__ = '2008, Anatoly Shipitsin <norguhtar at gmail.com>'
 
 '''Read meta information from fb2 files'''
 
-import sys, os
+import sys, os, mimetypes
+from base64 import b64decode
 
 from calibre.ebooks.BeautifulSoup import BeautifulStoneSoup
 from calibre.ebooks.metadata import MetaInformation
@@ -18,15 +19,30 @@ def get_metadata(stream):
     author= [firstname+" "+lastname]
     title = soup.find("book-title").string
     comments = soup.find("annotation")
+    cp = soup.find('coverpage')
+    cdata = None
+    if cp:
+        cimage = cp.find('image', attrs={'l:href':True})
+        if cimage:
+            id = cimage['l:href'].replace('#', '')
+            binary = soup.find('binary', id=id, attrs={'content-type':True})
+            if binary:
+                mt = binary['content-type']
+                exts = mimetypes.guess_all_extensions(mt)
+                if not exts:
+                    exts = ['.jpg']
+                cdata = (exts[0][1:], b64decode(binary.string.strip()))
+                
     if comments and len(comments) > 1:
         comments = comments.p.contents[0]
     series = soup.find("sequence")
- #   series_index = series.index
     mi = MetaInformation(title, author)
     mi.comments = comments
+    mi.author_sort = lastname+'; '+firstname
     if series:
         mi.series = series.get('name', None)
- #   mi.series_index = series_index
+    if cdata:
+        mi.cover_data = cdata
     return mi
 
 def main(args=sys.argv):

src/calibre/ebooks/metadata/toc.py

@@ -70,7 +70,7 @@ class TOC(list):
                         break
         
         if toc is not None:
-            if toc.lower() != 'ncx':
+            if toc.lower() not in ('ncx', 'ncxtoc'):
                 toc = urlparse(unquote(toc))[2]
                 toc = toc.replace('/', os.sep)
                 if not os.path.isabs(toc):
@@ -88,6 +88,10 @@ class TOC(list):
                     traceback.print_exc(file=sys.stdout)
                     print 'Continuing anyway'
             else:
+                path = opfreader.manifest.item(toc.lower())
+                if path and os.access(path, os.R_OK):
+                    self.read_ncx_toc(path)
+                    return
                 cwd = os.path.abspath(self.base_path)
                 m = glob.glob(os.path.join(cwd, '*.ncx'))
                 if m:

src/calibre/gui2/library.py

@@ -691,7 +691,7 @@ class DeviceBooksModel(BooksModel):
         dt = item.datetime
         dt = datetime(*dt[0:6])
         dt = dt - timedelta(seconds=time.timezone) + timedelta(hours=time.daylight)
-        data[_('Timestamp')] = dt.ctime()
+        data[_('Timestamp')] = dt.strftime('%a %b %d %H:%M:%S %Y')
         data[_('Tags')] = ', '.join(item.tags)
         self.emit(SIGNAL('new_bookdisplay_data(PyQt_PyObject)'), data)
         

src/calibre/gui2/main.py

@@ -59,6 +59,9 @@ class Main(MainWindow, Ui_MainWindow):
     
     def __init__(self, single_instance, opts, parent=None):
         MainWindow.__init__(self, opts, parent)
+        # Initialize fontconfig in a separate thread as this can be a lengthy 
+        # process if run for the first time on this machine
+        self.fc = __import__('calibre.utils.fontconfig', fromlist=1)
         self.single_instance = single_instance
         if self.single_instance is not None:
             self.connect(self.single_instance, SIGNAL('message_received(PyQt_PyObject)'),

src/calibre/library/cli.py

@@ -170,6 +170,7 @@ def do_add(db, paths, one_book_per_directory, recurse, add_duplicates):
             for mi, formats in dir_dups:
                 db.import_book(mi, formats)
         else:
+            if dir_dups or file_duplicates:
                 print >>sys.stderr, _('The following books were not added as they already exist in the database (see --duplicates option):')
             for mi, formats in dir_dups:
                 title = mi.title

src/calibre/linux_installer.py

@@ -240,13 +240,21 @@ def do_postinstall(destdir):
         os.chdir(cwd)
 
 def download_tarball():
+    try:
         pb  = ProgressBar(TerminalController(sys.stdout), 'Downloading calibre...')
+    except ValueError:
+        print 'Downloading calibre...'
+        pb = None
     src = urllib2.urlopen(MOBILEREAD+'calibre-%version-i686.tar.bz2')
     size = int(src.info()['content-length'])
     f = tempfile.NamedTemporaryFile()
     while f.tell() < size:
         f.write(src.read(4*1024))
-        pb.update(f.tell()/float(size))
+        percent = f.tell()/float(size)
+        if pb is not None:
+            pb.update(percent)
+        else:
+            print '%d%%, '%int(percent*100),
     f.seek(0)
     return f
 

src/calibre/utils/fontconfig.py

@@ -128,30 +128,49 @@ lib.FcConfigParseAndLoad.restype = c_int
 lib.FcConfigBuildFonts.argtypes = [c_void_p]
 lib.FcConfigBuildFonts.restype  = c_int
 
-
+_init_error  = None 
-# Initialize the fontconfig library. This has to be done manually
+_initialized = False
-# for the OS X bundle as it may have its own private fontconfig.
+from threading import Timer
-if hasattr(sys, 'frameworks_dir'):
+def _do_init():
+    # Initialize the fontconfig library. This has to be done manually
+    # for the OS X bundle as it may have its own private fontconfig.
+    if hasattr(sys, 'frameworks_dir'):
         config_dir = os.path.join(os.path.dirname(getattr(sys, 'frameworks_dir')), 'Resources', 'fonts')
         if isinstance(config_dir, unicode):
             config_dir = config_dir.encode(sys.getfilesystemencoding())
         config = lib.FcConfigCreate()
         if not lib.FcConfigParseAndLoad(config, os.path.join(config_dir, 'fonts.conf'), 1):
-        raise RuntimeError('Could not parse the fontconfig configuration')
+            _init_error = 'Could not parse the fontconfig configuration'
+            return
         if not lib.FcConfigBuildFonts(config):
-        raise RuntimeError('Could not build fonts')
+            _init_error = 'Could not build fonts'
+            return
         if not lib.FcConfigSetCurrent(config):
-        raise RuntimeError('Could not set font config')
+            _init_error = 'Could not set font config'
-elif not lib.FcInit():
+            return
-    raise RuntimeError(_('Could not initialize the fontconfig library'))
+    elif not lib.FcInit():
+        _init_error = _('Could not initialize the fontconfig library')
+        return
+    global _initialized
+    _initialized = True
     
-def find_font_families(allowed_extensions=['ttf']):
+    
+_init_timer = Timer(0.1, _do_init)
+_init_timer.start()
+
+def join():
+    _init_timer.join()
+    if _init_error is not None:
+        raise RuntimeError(_init_error)
+
+def find_font_families(allowed_extensions=['ttf', 'otf']):
     '''
     Return an alphabetically sorted list of font families available on the system.
     
     `allowed_extensions`: A list of allowed extensions for font file types. Defaults to
-    `['ttf']`. If it is empty, it is ignored.
+    `['ttf', 'otf']`. If it is empty, it is ignored.
     '''
+    join()
     allowed_extensions = [i.lower() for i in allowed_extensions]
     
     empty_pattern = lib.FcPatternCreate()
@@ -193,6 +212,7 @@ def files_for_family(family, normalize=True):
     they are a tuple (slant, weight) otherwise they are strings from the set 
     `('normal', 'bold', 'italic', 'bi', 'light', 'li')`
     '''
+    join()
     if isinstance(family, unicode):
         family = family.encode(preferred_encoding)
     family_pattern = lib.FcPatternBuild(0, 'family', FcTypeString, family, 0)
@@ -268,6 +288,7 @@ def match(name, sort=False, verbose=False):
     decreasing closeness of matching.
     `verbose`: If `True` print debugging information to stdout
     '''
+    join()
     if isinstance(name, unicode):
         name = name.encode(preferred_encoding)
     pat = lib.FcNameParse(name)