Merge upstream changes.

installer/linux/freeze.py

@@ -122,6 +122,8 @@ def freeze():
     elif exe not in executables:
         print >>sys.stderr, 'Invalid invocation of calibre loader. CALIBRE_CX_EXE=%%s is unknown'%%exe
     else:
+        from PyQt4.QtCore import QCoreApplication
+        QCoreApplication.setLibraryPaths([sys.frozen_path, os.path.join(sys.frozen_path, "qtplugins")])
         sys.argv[0] = exe
         module, func = executables[exe]
         module = __import__(module, fromlist=[1])
@@ -179,7 +181,7 @@ def freeze():
             if not f.endswith('.so') or 'designer' in dirpath or 'codecs' in dirpath or 'sqldrivers' in dirpath: 
                 continue
             f = os.path.join(dirpath, f)
-            dest_dir = dirpath.replace(plugdir, os.path.join(FREEZE_DIR, 'qtlugins'))
+            dest_dir = dirpath.replace(plugdir, os.path.join(FREEZE_DIR, 'qtplugins'))
             copy_binary(f, dest_dir)
 
     print 'Creating launchers'

src/calibre/ebooks/epub/from_any.py

@@ -116,7 +116,8 @@ def unarchive(path, tdir):
                 return f, ext
     return find_html_index(files)
 
-def any2epub(opts, path, notification=None):
+def any2epub(opts, path, notification=None, create_epub=True, 
+             oeb_cover=False, extract_to=None):
     ext = os.path.splitext(path)[1]
     if not ext:
         raise ValueError('Unknown file type: '+path)
@@ -139,7 +140,9 @@ def any2epub(opts, path, notification=None):
             raise ValueError('Conversion from %s is not supported'%ext.upper())
         
         print 'Creating EPUB file...'
-        html2epub(path, opts, notification=notification)
+        html2epub(path, opts, notification=notification, 
+                  create_epub=create_epub, oeb_cover=oeb_cover,
+                  extract_to=extract_to)
 
 def config(defaults=None):
     return common_config(defaults=defaults)

src/calibre/ebooks/epub/from_html.py

@@ -32,14 +32,14 @@ Conversion of HTML/OPF files follows several stages:
     * The EPUB container is created.
 '''
 
-import os, sys, cStringIO, logging, re, functools
+import os, sys, cStringIO, logging, re, functools, shutil
 
 from lxml.etree import XPath
 from lxml import html
 from PyQt4.Qt import QApplication, QPixmap
 
 from calibre.ebooks.html import Processor, merge_metadata, get_filelist,\
-    opf_traverse, create_metadata, rebase_toc, Link
+    opf_traverse, create_metadata, rebase_toc, Link, parser
 from calibre.ebooks.epub import config as common_config, tostring
 from calibre.ptempfile import TemporaryDirectory
 from calibre.ebooks.metadata.toc import TOC
@@ -62,7 +62,7 @@ def remove_bad_link(element, attribute, link, pos):
 
 def check(opf_path, pretty_print):
     '''
-    Find a remove all invalid links in the HTML files 
+    Find and remove all invalid links in the HTML files 
     '''
     logger = logging.getLogger('html2epub')
     logger.info('\tChecking files for bad links...')
@@ -77,7 +77,7 @@ def check(opf_path, pretty_print):
         
         for path in html_files:
             base = os.path.dirname(path)
-            root = html.fromstring(open(content(path), 'rb').read())
+            root = html.fromstring(open(content(path), 'rb').read(), parser=parser)
             for element, attribute, link, pos in list(root.iterlinks()):
                 link = to_unicode(link)
                 plink = Link(link, base)
@@ -210,17 +210,16 @@ TITLEPAGE = '''\
 </html>
 '''
 
-def create_cover_image(src, dest, screen_size):
+def create_cover_image(src, dest, screen_size, rescale_cover=True):
-    from PyQt4.Qt import QApplication, QImage, Qt
-    if QApplication.instance() is None:
-        app = QApplication([])
-        app
-    im = QImage()
     try:
+        from PyQt4.Qt import QImage, Qt
+        if QApplication.instance() is None:
+            QApplication([])
+        im = QImage()
         im.load(src)
         if im.isNull():
-            raise ValueError
+            raise ValueError('Invalid cover image')
-        if screen_size is not None:
+        if rescale_cover and screen_size is not None:
             width, height = im.width(), im.height()
             dw, dh = (screen_size[0]-width)/float(width), (screen_size[1]-height)/float(height)
             delta = min(dw, dh)
@@ -241,7 +240,6 @@ def process_title_page(mi, filelist, htmlfilemap, opts, tdir):
     if mi.cover:
         if f(filelist[0].path) == f(mi.cover):
             old_title_page = htmlfilemap[filelist[0].path]
-            
     #logger = logging.getLogger('html2epub')
     metadata_cover = mi.cover
     if metadata_cover and not os.path.exists(metadata_cover):
@@ -250,14 +248,15 @@ def process_title_page(mi, filelist, htmlfilemap, opts, tdir):
     cpath = '/'.join(('resources', '_cover_.jpg'))
     cover_dest = os.path.join(tdir, 'content', *cpath.split('/'))
     if metadata_cover is not None:
-        if not create_cover_image(metadata_cover, cover_dest, opts.profile.screen_size):
+        if not create_cover_image(metadata_cover, cover_dest, 
+                                  opts.profile.screen_size):
             metadata_cover = None
-                
     specified_cover = opts.cover
     if specified_cover and not os.path.exists(specified_cover):
         specified_cover = None
     if specified_cover is not None:
-        if not create_cover_image(specified_cover, cover_dest, opts.profile.screen_size):
+        if not create_cover_image(specified_cover, cover_dest, 
+                                  opts.profile.screen_size):
             specified_cover = None
             
     cover = metadata_cover if specified_cover is None or (opts.prefer_metadata_cover and metadata_cover is not None) else specified_cover
@@ -273,8 +272,15 @@ def process_title_page(mi, filelist, htmlfilemap, opts, tdir):
         os.remove(cover_dest)
     return None, old_title_page is not None
 
+def find_oeb_cover(htmlfile):
+    if os.stat(htmlfile).st_size > 2048:
+        return None
+    match = re.search(r'(?i)<img[^<>]+src\s*=\s*[\'"](.+?)[\'"]', open(htmlfile, 'rb').read())
+    if match:
+        return match.group(1)
 
-def convert(htmlfile, opts, notification=None):
+def convert(htmlfile, opts, notification=None, create_epub=True, 
+            oeb_cover=False, extract_to=None):
     htmlfile = os.path.abspath(htmlfile)
     if opts.output is None:
         opts.output = os.path.splitext(os.path.basename(htmlfile))[0] + '.epub'
@@ -326,7 +332,7 @@ def convert(htmlfile, opts, notification=None):
         
         title_page, has_title_page = process_title_page(mi, filelist, htmlfile_map, opts, tdir)
         spine = [htmlfile_map[f.path] for f in filelist]
-        if title_page is not None:
+        if not oeb_cover and title_page is not None:
             spine = [title_page] + spine
         mi.cover = None
         mi.cover_data = (None, None)
@@ -358,24 +364,43 @@ def convert(htmlfile, opts, notification=None):
         check(opf_path, opts.pretty_print)
         opf = OPF(opf_path, tdir)
         opf.remove_guide()
-        if has_title_page:
+        oeb_cover_file = None
+        if oeb_cover and title_page is not None:
+            oeb_cover_file = find_oeb_cover(os.path.join(tdir, 'content', title_page))
+        if has_title_page or (oeb_cover and oeb_cover_file):
             opf.create_guide_element()
+            if has_title_page and not oeb_cover:
                 opf.add_guide_item('cover', 'Cover', 'content/'+spine[0])
+            if oeb_cover and oeb_cover_file:
+                opf.add_guide_item('cover', 'Cover', 'content/'+oeb_cover_file)
         
-        opf.add_path_to_manifest(os.path.join(tdir, 'content', 'resources', '_cover_.jpg'), 'image/jpeg')    
+        cpath = os.path.join(tdir, 'content', 'resources', '_cover_.jpg')
+        if os.path.exists(cpath):
+            opf.add_path_to_manifest(cpath, 'image/jpeg')    
         with open(opf_path, 'wb') as f:
             raw = opf.render()
             if not raw.startswith('<?xml '):
                 raw = '<?xml version="1.0"  encoding="UTF-8"?>\n'+raw
             f.write(raw)
+        if create_epub:
             epub = initialize_container(opts.output)
             epub.add_dir(tdir)
+            epub.close()
+            logger.info(_('Output written to ')+opts.output)
+        
         if opts.show_opf:
             print open(os.path.join(tdir, 'metadata.opf')).read()
-        logger.info('Output written to %s'%opts.output)
+        
         if opts.extract_to is not None:
-            epub.extractall(opts.extract_to)
+            if os.path.exists(opts.extract_to):
-        epub.close()
+                shutil.rmtree(opts.extract_to)
+            shutil.copytree(tdir, opts.extract_to)
+            
+        if extract_to is not None:
+            if os.path.exists(extract_to):
+                shutil.rmtree(extract_to)
+            shutil.copytree(tdir, extract_to)
+            
         
             
 def main(args=sys.argv):

src/calibre/ebooks/lit/from_any.py

@@ -37,8 +37,8 @@ def any2lit(opts, path):
         oebdir = os.path.join(tdir, 'oeb')
         os.mkdir(oebdir)
         opts.output = os.path.join(tdir, 'dummy.epub')
-        opts.extract_to = oebdir
+        opts.profile = 'None'
-        any2epub(opts, path)
+        any2epub(opts, path, create_epub=False, oeb_cover=True, extract_to=oebdir)
         opf = glob.glob(os.path.join(oebdir, '*.opf'))[0]
         opts.output = orig_output
         logging.getLogger('html2epub').info(_('Creating LIT file from EPUB...'))

src/calibre/ebooks/lit/writer.py

@@ -313,6 +313,9 @@ class LitWriter(object):
         elif MS_COVER_TYPE in oeb.guide:
             href = oeb.guide[MS_COVER_TYPE].href
             cover = oeb.manifest.hrefs[href]
+        elif 'cover' in oeb.guide:
+            href = oeb.guide['cover'].href
+            cover = oeb.manifest.hrefs[href]
         else:
             html = oeb.spine[0].data
             imgs = xpath(html, '//img[position()=1]')

src/calibre/gui2/dialogs/config.py

@@ -101,7 +101,6 @@ class ConfigDialog(QDialog, Ui_Dialog):
         for item in items:
             self.language.addItem(item[1], QVariant(item[0]))
             
-        self.output_format.setCurrentIndex(0 if prefs['output_format'] == 'LRF' else 1)
         self.pdf_metadata.setChecked(prefs['read_file_metadata'])
         
         added_html = False
@@ -255,16 +254,11 @@ class ConfigDialog(QDialog, Ui_Dialog):
         sc.set('max_cover', mcs)
         config['delete_news_from_library_on_upload'] = self.delete_news.isChecked()
         config['upload_news_to_device'] = self.sync_news.isChecked()
-        of = str(self.output_format.currentText())
         fmts = []
         for i in range(self.viewer.count()):
             if self.viewer.item(i).checkState() == Qt.Checked:
                 fmts.append(str(self.viewer.item(i).text()))
         config['internally_viewed_formats'] = fmts
-        if of != prefs['output_format'] and 'epub' in of.lower():
-            warning_dialog(self, 'Warning', 
-                '<p>EPUB support is still in beta. If you find bugs, please report them by opening a <a href="http://calibre.kovidgoyal.net">ticket</a>.').exec_()
-        prefs['output_format'] = of 
         
         if not path or not os.path.exists(path) or not os.path.isdir(path):
             d = error_dialog(self, _('Invalid database location'),

src/calibre/gui2/dialogs/config.ui

@@ -149,7 +149,7 @@
          </item>
          <item>
           <layout class="QGridLayout" name="gridLayout_2" >
-           <item row="1" column="0" >
+           <item row="0" column="0" >
             <widget class="QLabel" name="label_5" >
              <property name="text" >
               <string>Format for &amp;single file save:</string>
@@ -159,10 +159,10 @@
              </property>
             </widget>
            </item>
-           <item row="1" column="1" >
+           <item row="0" column="1" >
             <widget class="QComboBox" name="single_format" />
            </item>
-           <item row="2" column="0" >
+           <item row="1" column="0" >
             <widget class="QLabel" name="label_2" >
              <property name="text" >
               <string>Default network &amp;timeout:</string>
@@ -172,7 +172,7 @@
              </property>
             </widget>
            </item>
-           <item row="2" column="1" >
+           <item row="1" column="1" >
             <widget class="QSpinBox" name="timeout" >
              <property name="toolTip" >
               <string>Set the default timeout for network fetches (i.e. anytime we go out to the internet to get information)</string>
@@ -191,10 +191,10 @@
              </property>
             </widget>
            </item>
-           <item row="3" column="1" >
+           <item row="2" column="1" >
             <widget class="QComboBox" name="language" />
            </item>
-           <item row="3" column="0" >
+           <item row="2" column="0" >
             <widget class="QLabel" name="label_7" >
              <property name="text" >
               <string>Choose &amp;language (requires restart):</string>
@@ -204,34 +204,7 @@
              </property>
             </widget>
            </item>
-           <item row="0" column="1" >
+           <item row="3" column="1" >
-            <widget class="QComboBox" name="output_format" >
-             <property name="toolTip" >
-              <string>The default output format for ebook conversions.</string>
-             </property>
-             <item>
-              <property name="text" >
-               <string>LRF</string>
-              </property>
-             </item>
-             <item>
-              <property name="text" >
-               <string>EPUB</string>
-              </property>
-             </item>
-            </widget>
-           </item>
-           <item row="0" column="0" >
-            <widget class="QLabel" name="label_8" >
-             <property name="text" >
-              <string>&amp;Output format:</string>
-             </property>
-             <property name="buddy" >
-              <cstring>output_format</cstring>
-             </property>
-            </widget>
-           </item>
-           <item row="4" column="1" >
             <widget class="QComboBox" name="priority" >
              <item>
               <property name="text" >
@@ -250,7 +223,7 @@
              </item>
             </widget>
            </item>
-           <item row="4" column="0" >
+           <item row="3" column="0" >
             <widget class="QLabel" name="priority_label" >
              <property name="text" >
               <string>Job &amp;priority:</string>

src/calibre/gui2/jobs2.py

@@ -188,6 +188,6 @@ class DetailView(QDialog, Ui_Dialog):
         
             
     def update(self):
-        self.log.setPlainText(self.job.gui_text())
+        self.log.setPlainText(self.job.console_text())
         vbar = self.log.verticalScrollBar()
         vbar.setValue(vbar.maximum())

src/calibre/gui2/main.py

@@ -127,6 +127,21 @@ class Main(MainWindow, Ui_MainWindow):
         QObject.connect(self.stack, SIGNAL('currentChanged(int)'),
                         self.location_view.location_changed)
         
+        self.output_formats = sorted(['EPUB', 'LRF'])
+        for f in self.output_formats:
+            self.output_format.addItem(f)
+        self.output_format.setCurrentIndex(self.output_formats.index(prefs['output_format']))
+        def change_output_format(x):
+            of = unicode(x).strip()
+            if of != prefs['output_format']:
+                if of in ('EPUB', 'LIT'):
+                    warning_dialog(self, 'Warning', 
+                                   '<p>%s support is still in beta. If you find bugs, please report them by opening a <a href="http://calibre.kovidgoyal.net">ticket</a>.'%of).exec_()
+                prefs.set('output_format', of)
+        
+        self.connect(self.output_format, SIGNAL('currentIndexChanged(QString)'), 
+                     change_output_format)
+
         ####################### Vanity ########################
         self.vanity_template  = _('<p>For help visit <a href="http://%s.kovidgoyal.net/user_manual">%s.kovidgoyal.net</a><br>')%(__appname__, __appname__)
         self.vanity_template += _('<b>%s</b>: %s by <b>Kovid Goyal %%(version)s</b><br>%%(device)s</p>')%(__appname__, __version__)
@@ -489,7 +504,7 @@ class Main(MainWindow, Ui_MainWindow):
             return
         info, cp, fs = job.result
         self.location_view.model().update_devices(cp, fs)
-        self.device_info = _('Connected ')+' '.join(info[:-1])
+        self.device_info = _('Connected ')+info[0]
         self.vanity.setText(self.vanity_template%dict(version=self.latest_version, device=self.device_info))
 
         self.device_manager.books(Dispatcher(self.metadata_downloaded))

src/calibre/gui2/main.ui

@@ -27,15 +27,9 @@
     <normaloff>:/library</normaloff>:/library</iconset>
   </property>
   <widget class="QWidget" name="centralwidget" >
-   <layout class="QGridLayout" >
+   <layout class="QGridLayout" name="gridLayout" >
     <item row="0" column="0" >
-     <layout class="QHBoxLayout" >
+     <layout class="QHBoxLayout" name="horizontalLayout_3" >
-      <property name="spacing" >
-       <number>6</number>
-      </property>
-      <property name="margin" >
-       <number>0</number>
-      </property>
       <item>
        <widget class="LocationView" name="location_view" >
         <property name="sizePolicy" >
@@ -88,6 +82,8 @@
         </property>
        </widget>
       </item>
+      <item>
+       <layout class="QVBoxLayout" name="verticalLayout_3" >
         <item>
          <widget class="QLabel" name="vanity" >
           <property name="sizePolicy" >
@@ -113,6 +109,22 @@
           </property>
          </widget>
         </item>
+        <item>
+         <layout class="QHBoxLayout" name="horizontalLayout_2" >
+          <item>
+           <widget class="QLabel" name="label_2" >
+            <property name="text" >
+             <string>Output:</string>
+            </property>
+           </widget>
+          </item>
+          <item>
+           <widget class="QComboBox" name="output_format" />
+          </item>
+         </layout>
+        </item>
+       </layout>
+      </item>
      </layout>
     </item>
     <item row="1" column="0" >

src/calibre/parallel.py

@@ -567,15 +567,13 @@ class Job(object):
             return 'ERROR'
             
     def console_text(self):
-        ans = [u'Error in job: ']
+        ans = [u'Job: ']
         if self.description:
             ans[0] += self.description
-        if self.log:
+        if self.exception is not None:
-            if isinstance(self.log, str):
-                self.log = unicode(self.log, 'utf-8', 'replace')
-            ans.append(self.log)
             header = unicode(self.exception.__class__.__name__) if \
                     hasattr(self.exception, '__class__') else u'Error'
+            header = u'**%s**'%header
             header += u': '
             try:
                 header += unicode(self.exception)
@@ -583,7 +581,13 @@ class Job(object):
                 header += unicode(repr(self.exception))
             ans.append(header)
             if self.traceback:
-            ans.append(self.traceback)
+                ans.append(u'**Traceback**:')
+                ans.extend(self.traceback.split('\n'))
+        
+        if self.log:
+            if isinstance(self.log, str):
+                self.log = unicode(self.log, 'utf-8', 'replace')
+            ans.append(self.log)
         return (u'\n'.join(ans)).encode('utf-8')
     
     def gui_text(self):
@@ -611,7 +615,7 @@ class Job(object):
                 self.log = unicode(self.log, 'utf-8', 'replace')
             ans.extend(self.log.split('\n'))
             
-        return '\n'.join(ans)
+        return '<br>'.join(ans)
 
 
 class ParallelJob(Job):

src/calibre/web/feeds/news.py

@@ -789,6 +789,7 @@ class BasicNewsRecipe(object, LoggingInterface):
         html= u'''\
         <html>
             <head>
+                <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
                 <style type="text/css">
                     body {
                         background: white no-repeat fixed center center;
@@ -824,7 +825,7 @@ class BasicNewsRecipe(object, LoggingInterface):
                  app=__appname__ +' '+__version__,
                  img=img)
         f2 = tempfile.NamedTemporaryFile(suffix='cover.html')
-        f2.write(html)
+        f2.write(html.encode('utf-8'))
         f2.flush()
         page = QWebPage()
         pal = page.palette()

upload.py

@@ -213,7 +213,7 @@ def upload_src_tarball():
     check_call('scp dist/calibre-*.tar.gz divok:%s/'%DOWNLOADS)
 
 def stage_one():
-    check_call('sudo rm -rf build', shell=True)
+    check_call('sudo rm -rf build src/calibre/plugins/*', shell=True)
     os.mkdir('build')
     shutil.rmtree('docs')
     os.mkdir('docs')