Merge from trunk

src/calibre/customize/builtins.py

@@ -474,7 +474,7 @@ from calibre.devices.binatone.driver import README
 from calibre.devices.hanvon.driver import N516, EB511, ALEX, AZBOOKA, THEBOOK
 from calibre.devices.edge.driver import EDGE
 from calibre.devices.teclast.driver import TECLAST_K3, NEWSMY, IPAPYRUS, \
-        SOVOS, PICO, SUNSTECH_EB700
+        SOVOS, PICO, SUNSTECH_EB700, ARCHOS7O
 from calibre.devices.sne.driver import SNE
 from calibre.devices.misc import PALMPRE, AVANT, SWEEX, PDNOVEL, KOGAN, \
         GEMEI, VELOCITYMICRO, PDNOVEL_KOBO, Q600, LUMIREAD, ALURATEK_COLOR, \
@@ -581,7 +581,7 @@ plugins += [
     ELONEX,
     TECLAST_K3,
     NEWSMY,
-    PICO, SUNSTECH_EB700,
+    PICO, SUNSTECH_EB700, ARCHOS7O,
     IPAPYRUS,
     SOVOS,
     EDGE,

src/calibre/devices/eb600/driver.py

@@ -183,9 +183,8 @@ class BOOQ(EB600):
 
     FORMATS = ['epub', 'mobi', 'prc', 'fb2', 'pdf', 'doc', 'rtf', 'txt', 'html']
 
-    VENDOR_NAME = 'NETRONIX'
+    VENDOR_NAME = ['NETRONIX', '36LBOOKS']
-    WINDOWS_MAIN_MEM = 'EB600'
+    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['EB600', 'ELEQTOR']
-    WINDOWS_CARD_A_MEM = 'EB600'
 
 class MENTOR(EB600):
 

src/calibre/devices/teclast/driver.py

@@ -41,6 +41,16 @@ class NEWSMY(TECLAST_K3):
     WINDOWS_MAIN_MEM = 'NEWSMY'
     WINDOWS_CARD_A_MEM = 'USBDISK____SD'
 
+class ARCHOS7O(TECLAST_K3):
+    name = 'Archos 7O device interface'
+    gui_name = 'Archos'
+    description    = _('Communicate with the Archos reader.')
+
+    FORMATS = ['epub', 'mobi', 'fb2', 'rtf', 'ap', 'html', 'pdf', 'txt']
+
+    VENDOR_NAME      = 'ARCHOS'
+    WINDOWS_MAIN_MEM = 'USB-MSC'
+
 class PICO(NEWSMY):
     name = 'Pico device interface'
     gui_name = 'Pico'

src/calibre/ebooks/__init__.py

@@ -113,7 +113,7 @@ def render_html_svg_workaround(path_to_html, log, width=590, height=750):
 
 def render_html(path_to_html, width=590, height=750, as_xhtml=True):
     from PyQt4.QtWebKit import QWebPage
-    from PyQt4.Qt import QEventLoop, QPalette, Qt, SIGNAL, QUrl, QSize
+    from PyQt4.Qt import QEventLoop, QPalette, Qt, QUrl, QSize
     from calibre.gui2 import is_ok_to_use_qt
     if not is_ok_to_use_qt(): return None
     path_to_html = os.path.abspath(path_to_html)
@@ -127,8 +127,7 @@ def render_html(path_to_html, width=590, height=750, as_xhtml=True):
         page.mainFrame().setScrollBarPolicy(Qt.Horizontal, Qt.ScrollBarAlwaysOff)
         loop = QEventLoop()
         renderer = HTMLRenderer(page, loop)
-        page.connect(page, SIGNAL('loadFinished(bool)'), renderer,
+        page.loadFinished.connect(renderer, type=Qt.QueuedConnection)
-                Qt.QueuedConnection)
         if as_xhtml:
             page.mainFrame().setContent(open(path_to_html, 'rb').read(),
                     'application/xhtml+xml', QUrl.fromLocalFile(path_to_html))
@@ -136,6 +135,7 @@ def render_html(path_to_html, width=590, height=750, as_xhtml=True):
             page.mainFrame().load(QUrl.fromLocalFile(path_to_html))
         loop.exec_()
     renderer.loop = renderer.page = None
+    page.loadFinished.disconnect()
     del page
     del loop
     if isinstance(renderer.exception, ParserError) and as_xhtml:

src/calibre/ebooks/fb2/fb2ml.py

@@ -398,7 +398,7 @@ class FB2MLizer(object):
                     tags += p_tag
                     fb2_out.append('<image xlink:href="#%s" />' % self.image_hrefs[page.abshref(elem_tree.attrib['src'])])
         if tag in ('br', 'hr') or ems:
-            if not ems:
+            if ems < 1:
                 multiplier = 1
             else:
                 multiplier = ems

src/calibre/ebooks/metadata/epub.py

@@ -14,7 +14,8 @@ from calibre.ebooks.BeautifulSoup import BeautifulStoneSoup
 from calibre.ebooks.metadata import MetaInformation
 from calibre.ebooks.metadata.opf2 import OPF
 from calibre.ptempfile import TemporaryDirectory, PersistentTemporaryFile
-from calibre import CurrentDir
+from calibre import CurrentDir, walk
+from calibre.constants import isosx
 
 class EPubException(Exception):
     pass
@@ -159,6 +160,13 @@ def get_cover(opf, opf_path, stream, reader=None):
     with TemporaryDirectory('_epub_meta') as tdir:
         with CurrentDir(tdir):
             zf.extractall()
+            if isosx:
+                # On OS X trying to render an HTML cover which uses embedded
+                # fonts more than once in the same process causes a crash in Qt
+                # so be safe and remove the fonts.
+                for f in walk('.'):
+                    if os.path.splitext(f)[1].lower() in ('.ttf', '.otf'):
+                        os.remove(f)
             opf_path = opf_path.replace('/', os.sep)
             cpage = os.path.join(tdir, os.path.dirname(opf_path), cpage)
             if not os.path.exists(cpage):

src/calibre/ebooks/pml/pmlconverter.py

@@ -603,7 +603,7 @@ class PML_HTMLizer(object):
 
             if empty:
                 empty_count += 1
-                if empty_count == 3:
+                if empty_count == 2:
                     output.append('<p>&nbsp;</p>')
             else:
                 empty_count = 0

src/calibre/ebooks/pml/pmlml.py

@@ -10,6 +10,8 @@ Transform OEB content into PML markup
 
 import re
 
+from lxml import etree
+
 from calibre.ebooks.oeb.base import XHTML, XHTML_NS, barename, namespace
 from calibre.ebooks.oeb.stylizer import Stylizer
 from calibre.ebooks.pdb.ereader import image_name
@@ -64,8 +66,8 @@ SEPARATE_TAGS = [
     'h4',
     'h5',
     'h6',
-    'p',
+    'hr',
-    'div',
+    'img',
     'li',
     'tr',
 ]
@@ -122,9 +124,12 @@ class PMLMLizer(object):
         text = [u'']
         for item in self.oeb_book.spine:
             self.log.debug('Converting %s to PML markup...' % item.href)
-            stylizer = Stylizer(item.data, item.href, self.oeb_book, self.opts, self.opts.output_profile)
+            content = unicode(etree.tostring(item.data, encoding=unicode))
+            content = self.prepare_text(content)
+            content = etree.fromstring(content)
+            stylizer = Stylizer(content, item.href, self.oeb_book, self.opts, self.opts.output_profile)
             text.append(self.add_page_anchor(item))
-            text += self.dump_text(item.data.find(XHTML('body')), stylizer, item)
+            text += self.dump_text(content.find(XHTML('body')), stylizer, item)
         return ''.join(text)
 
     def add_page_anchor(self, page):
@@ -147,6 +152,21 @@ class PMLMLizer(object):
         text = text.replace('\r', ' ')
         return text
 
+    def prepare_string_for_pml(self, text):
+        text = self.remove_newlines(text)
+        # Replace \ with \\ so \ in the text is not interperted as
+        # a pml code.
+        text = text.replace('\\', '\\\\')
+        # Replace sequences of \\c \\c with pml sequences denoting
+        # empty lines.
+        text = text.replace('\\\\c \\\\c', '\\c \n\\c\n')
+        return text
+
+    def prepare_text(self, text):
+        # Replace empty paragraphs with \c pml codes used to denote emtpy lines.
+        text = re.sub(ur'(?<=</p>)\s*<p[^>]*>[\xc2\xa0\s]*</p>', '\\c\n\\c', text)
+        return text
+
     def clean_text(self, text):
         # Remove excessive \p tags
         text = re.sub(r'\\p\s*\\p', '', text)
@@ -172,15 +192,18 @@ class PMLMLizer(object):
         # Remove excessive spaces
         text = re.sub('[ ]{2,}', ' ', text)
 
+        # Condense excessive \c empty line sequences.
+        text = re.sub('(\\c\s*\\c\s*){2,}', '\\c \n\\c\n', text)
+
         # Remove excessive newlines.
         text = re.sub('\n[ ]+\n', '\n\n', text)
         if self.opts.remove_paragraph_spacing:
             text = re.sub('\n{2,}', '\n', text)
-            text = re.sub('(?imu)^(?P<text>.+)$', lambda mo: mo.group('text') if re.search(r'\\[XxCm]', mo.group('text')) else '    %s' % mo.group('text'), text)
+            # Only indent lines that don't have special formatting
+            text = re.sub('(?imu)^(?P<text>.+)$', lambda mo: mo.group('text') if re.search(r'\\[XxCmrctTp]', mo.group('text')) else '        %s' % mo.group('text'), text)
         else:
             text = re.sub('\n{3,}', '\n\n', text)
 
-
         return text
 
     def dump_text(self, elem, stylizer, page, tag_stack=[]):
@@ -203,7 +226,7 @@ class PMLMLizer(object):
             tags.append('block')
 
         # Process tags that need special processing and that do not have inner
-        # text. Usually these require an argument
+        # text. Usually these require an argument.
         if tag in IMAGE_TAGS:
             if elem.attrib.get('src', None):
                 if page.abshref(elem.attrib['src']) not in self.image_hrefs.keys():
@@ -212,7 +235,7 @@ class PMLMLizer(object):
                     else:
                         self.image_hrefs[page.abshref(elem.attrib['src'])] = image_name('%s.png' % len(self.image_hrefs.keys()), self.image_hrefs.keys()).strip('\x00')
                 text.append('\\m="%s"' % self.image_hrefs[page.abshref(elem.attrib['src'])])
-        if tag == 'hr':
+        elif tag == 'hr':
             w = '\\w'
             width = elem.get('width')
             if width:
@@ -222,6 +245,10 @@ class PMLMLizer(object):
             else:
                 w += '="50%"'
             text.append(w)
+        elif tag == 'br':
+            text.append('\n\\c \n\\c\n')
+
+        # TOC markers.
         toc_name = elem.attrib.get('name', None)
         toc_id = elem.attrib.get('id', None)
         if (toc_id or toc_name) and tag  not in ('h1', 'h2','h3','h4','h5','h6',):
@@ -234,9 +261,10 @@ class PMLMLizer(object):
 
         # Process style information that needs holds a single tag
         # Commented out because every page in an OEB book starts with this style
-        #if style['page-break-before'] == 'always':
+        if style['page-break-before'] == 'always':
-        #    text.append('\\p')
+            text.append('\\p')
 
+        # Process basic PML tags.
         pml_tag = TAG_MAP.get(tag, None)
         if pml_tag and pml_tag not in tag_stack+tags:
             text.append('\\%s' % pml_tag)
@@ -270,34 +298,60 @@ class PMLMLizer(object):
             if style_tag and style_tag not in tag_stack+tags:
                 text.append('\\%s' % style_tag)
                 tags.append(style_tag)
-        # margin
 
-        # Proccess tags that contain text.
+        # margin left
+        try:
+            mms = int(float(style['margin-left']) * 100 / style.height)
+            if mms:
+                text.append('\\T="%s%%"' % mms)
+        except:
+            pass
+
+        # Soft scene breaks.
+        try:
+            ems = int(round((float(style.marginTop) / style.fontSize) - 1))
+            if ems >= 1:
+                text.append('\n\\c \n\\c\n')
+        except:
+            pass
+
+        # Proccess text within this tag.
         if hasattr(elem, 'text') and elem.text:
-            text.append(self.remove_newlines(elem.text))
+            text.append(self.prepare_string_for_pml(elem.text))
 
+        # Process inner tags
         for item in elem:
             text += self.dump_text(item, stylizer, page, tag_stack+tags)
 
+        # Close opened tags.
         tags.reverse()
         text += self.close_tags(tags)
 
-        if tag in SEPARATE_TAGS:
+        #if tag in SEPARATE_TAGS:
-            text.append('\n\n')
+        #    text.append('\n\n')
 
-        #if style['page-break-after'] == 'always':
+        if style['page-break-after'] == 'always':
-        #    text.append('\\p')
+            text.append('\\p')
 
+        # Process text after this tag but not within another.
         if hasattr(elem, 'tail') and elem.tail:
-            text.append(self.remove_newlines(elem.tail))
+            text.append(self.prepare_string_for_pml(elem.tail))
 
         return text
 
     def close_tags(self, tags):
         text = []
         for tag in tags:
+            # block isn't a real tag we just use
+            # it to determine when we need to start
+            # a new text block.
             if tag == 'block':
                 text.append('\n\n')
+            else:
+                # closing \c and \r need to be placed
+                # on the next line per PML spec.
+                if tag in ('c', 'r'):
+                    text.append('\n\\%s' % tag)
                 else:
                     text.append('\\%s' % tag)
         return text

src/calibre/ebooks/txt/txtml.py

@@ -226,7 +226,7 @@ class TXTMLizer(object):
         # Soft scene breaks.
         try:
             ems = int(round((float(style.marginTop) / style.fontSize) - 1))
-            if ems:
+            if ems >= 1:
                 text.append('\n' * ems)
         except:
             pass

src/calibre/gui2/filename_pattern.ui

@@ -17,13 +17,10 @@
    <item>
     <widget class="QLabel" name="label">
      <property name="text">
-      <string>&lt;!DOCTYPE HTML PUBLIC &quot;-//W3C//DTD HTML 4.0//EN&quot; &quot;http://www.w3.org/TR/REC-html40/strict.dtd&quot;&gt;
+      <string>&lt;div style=&quot;font-size:10pt;&quot;&gt;
-&lt;html&gt;&lt;head&gt;&lt;meta name=&quot;qrichtext&quot; content=&quot;1&quot; /&gt;&lt;style type=&quot;text/css&quot;&gt;
+&lt;p&gt;Set a regular expression pattern to use when trying to guess ebook metadata from filenames. &lt;/p&gt;
-p, li { white-space: pre-wrap; }
+&lt;p&gt;A &lt;a href=&quot;http://calibre-ebook.com/user_manual/regexp.html&quot;&gt;tutorial&lt;/a&gt; on using regular expressions is available.&lt;/p&gt;
-&lt;/style&gt;&lt;/head&gt;&lt;body style=&quot; font-family:'Candara'; font-size:10pt; font-weight:400; font-style:normal;&quot;&gt;
+&lt;p&gt;Use the &lt;b&gt;Test&lt;/b&gt; functionality below to test your regular expression on a few sample filenames (remember to include the file extension). The group names for the various metadata entries are documented in tooltips.&lt;/p&gt;&lt;/div&gt;</string>
-&lt;p style=&quot; margin-top:12px; margin-bottom:12px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;Set a regular expression pattern to use when trying to guess ebook metadata from filenames. &lt;/p&gt;
-&lt;p style=&quot; margin-top:12px; margin-bottom:12px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;A &lt;a href=&quot;http://docs.python.org/lib/re-syntax.html&quot;&gt;&lt;span style=&quot; text-decoration: underline; color:#0000ff;&quot;&gt;reference&lt;/span&gt;&lt;/a&gt; on the syntax of regular expressions is available.&lt;/p&gt;
-&lt;p style=&quot; margin-top:12px; margin-bottom:12px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;Use the &lt;span style=&quot; font-weight:600;&quot;&gt;Test&lt;/span&gt; functionality below to test your regular expression on a few sample filenames. The group names for the various metadata entries are documented in tooltips.&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
      </property>
      <property name="textFormat">
       <enum>Qt::RichText</enum>
@@ -104,8 +101,8 @@ p, li { white-space: pre-wrap; }
             <rect>
              <x>0</x>
              <y>0</y>
-             <width>277</width>
+             <width>305</width>
-             <height>276</height>
+             <height>263</height>
             </rect>
            </property>
            <layout class="QGridLayout" name="gridLayout_2">

src/calibre/gui2/preferences/adding.py

@@ -7,7 +7,8 @@ __docformat__ = 'restructuredtext en'
 
 
 
-from calibre.gui2.preferences import ConfigWidgetBase, test_widget
+from calibre.gui2.preferences import ConfigWidgetBase, test_widget, \
+    CommaSeparatedList
 from calibre.gui2.preferences.adding_ui import Ui_Form
 from calibre.utils.config import prefs
 from calibre.gui2.widgets import FilenamePattern
@@ -22,6 +23,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         r('read_file_metadata', prefs)
         r('swap_author_names', prefs)
         r('add_formats_to_existing', prefs)
+        r('new_book_tags', prefs, setting=CommaSeparatedList)
 
         self.filename_pattern = FilenamePattern(self)
         self.metadata_box.layout().insertWidget(0, self.filename_pattern)

src/calibre/gui2/preferences/adding.ui

@@ -6,7 +6,7 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>1010</width>
+    <width>750</width>
     <height>339</height>
    </rect>
   </property>
@@ -27,10 +27,37 @@
    <item row="1" column="0">
     <widget class="QCheckBox" name="opt_read_file_metadata">
      <property name="text">
-      <string>Read metadata from &amp;file contents rather than file name</string>
+      <string>Read &amp;metadata from &amp;file contents rather than file name</string>
      </property>
     </widget>
    </item>
+   <item row="1" column="1">
+    <layout class="QHBoxLayout" name="horizontalLayout">
+     <item>
+      <spacer name="horizontalSpacer">
+       <property name="orientation">
+        <enum>Qt::Horizontal</enum>
+       </property>
+       <property name="sizeHint" stdset="0">
+        <size>
+         <width>40</width>
+         <height>20</height>
+        </size>
+       </property>
+      </spacer>
+     </item>
+     <item>
+      <widget class="QCheckBox" name="opt_swap_author_names">
+       <property name="toolTip">
+        <string>Swap the firstname and lastname of the author. This affects only metadata read from file names.</string>
+       </property>
+       <property name="text">
+        <string>&amp;Swap author firstname and lastname</string>
+       </property>
+      </widget>
+     </item>
+    </layout>
+   </item>
    <item row="2" column="0" colspan="2">
     <widget class="QCheckBox" name="opt_add_formats_to_existing">
      <property name="toolTip">
@@ -44,7 +71,24 @@ Title match ignores leading indefinite articles (&quot;the&quot;, &quot;a&quot;,
      </property>
     </widget>
    </item>
-   <item row="3" column="0" colspan="2">
+   <item row="3" column="0">
+    <widget class="QLabel" name="label_230">
+     <property name="text">
+      <string>&amp;Tags to apply when adding a book:</string>
+     </property>
+     <property name="buddy">
+      <cstring>opt_new_book_tags</cstring>
+     </property>
+    </widget>
+   </item>
+   <item row="3" column="1">
+    <widget class="QLineEdit" name="opt_new_book_tags">
+     <property name="toolTip">
+      <string>A comma-separated list of tags that will be applied to books added to the library</string>
+     </property>
+    </widget>
+   </item>
+   <item row="4" column="0" colspan="2">
     <widget class="QGroupBox" name="metadata_box">
      <property name="title">
       <string>&amp;Configure metadata from file name</string>
@@ -66,16 +110,6 @@ Title match ignores leading indefinite articles (&quot;the&quot;, &quot;a&quot;,
      </layout>
     </widget>
    </item>
-   <item row="1" column="1">
-    <widget class="QCheckBox" name="opt_swap_author_names">
-     <property name="toolTip">
-      <string>Swap the firstname and lastname of the author. This affects only metadata read from file names.</string>
-     </property>
-     <property name="text">
-      <string>&amp;Swap author firstname and lastname</string>
-     </property>
-    </widget>
-   </item>
   </layout>
  </widget>
  <resources/>

src/calibre/gui2/preferences/behavior.py

@@ -9,8 +9,7 @@ import re
 
 from PyQt4.Qt import Qt, QVariant, QListWidgetItem
 
-from calibre.gui2.preferences import ConfigWidgetBase, test_widget, \
+from calibre.gui2.preferences import ConfigWidgetBase, test_widget
-    CommaSeparatedList
 from calibre.gui2.preferences.behavior_ui import Ui_Form
 from calibre.gui2 import config, info_dialog, dynamic
 from calibre.utils.config import prefs
@@ -49,7 +48,6 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         restrictions = sorted(saved_searches().names(), key=sort_key)
         choices = [('', '')] + [(x, x) for x in restrictions]
         r('gui_restriction', db.prefs, choices=choices)
-        r('new_book_tags', prefs, setting=CommaSeparatedList)
         self.reset_confirmation_button.clicked.connect(self.reset_confirmation_dialogs)
 
         self.input_up_button.clicked.connect(self.up_input)

src/calibre/gui2/preferences/behavior.ui

@@ -164,20 +164,6 @@
        </property>
       </widget>
      </item>
-     <item row="4" column="1">
-      <widget class="QLineEdit" name="opt_new_book_tags">
-       <property name="toolTip">
-        <string>A comma-separated list of tags that will be applied to books added to the library</string>
-       </property>
-      </widget>
-     </item>
-     <item row="4" column="0">
-      <widget class="QLabel" name="label_230">
-       <property name="text">
-        <string>Tags to apply when adding a book:</string>
-       </property>
-      </widget>
-     </item>
     </layout>
    </item>
    <item row="6" column="0" colspan="2">