Merge from custcol trunk

src/calibre/constants.py

@@ -2,7 +2,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = 'calibre'
-__version__   = '0.6.92'
+__version__   = '0.6.94'
 __author__    = "Kovid Goyal <kovid@kovidgoyal.net>"
 
 import re

src/calibre/customize/builtins.py

@@ -451,7 +451,7 @@ from calibre.devices.iriver.driver import IRIVER_STORY
 from calibre.devices.binatone.driver import README
 from calibre.devices.hanvon.driver import N516, EB511, ALEX, AZBOOKA
 from calibre.devices.edge.driver import EDGE
-from calibre.devices.teclast.driver import TECLAST_K3
+from calibre.devices.teclast.driver import TECLAST_K3, NEWSMY, IPAPYRUS
 from calibre.devices.sne.driver import SNE
 from calibre.devices.misc import PALMPRE, KOBO, AVANT
 from calibre.devices.folder_device.driver import FOLDER_DEVICE_FOR_CONFIG
@@ -531,6 +531,8 @@ plugins += [
     EB511,
     ELONEX,
     TECLAST_K3,
+    NEWSMY,
+    IPAPYRUS,
     EDGE,
     SNE,
     ALEX,

src/calibre/devices/android/driver.py

@@ -24,6 +24,9 @@ class ANDROID(USBMS):
             # Motorola
             0x22b8 : { 0x41d9 : [0x216], 0x2d67 : [0x100], 0x41db : [0x216]},
 
+            # Sony Ericsson
+            0xfce : { 0xd12e : [0x0100]},
+
             0x18d1 : { 0x4e11 : [0x0100, 0x226], 0x4e12: [0x0100, 0x226]},
 
             # Samsung

src/calibre/devices/prs505/sony_cache.py

@@ -16,8 +16,7 @@ from calibre import prints, guess_type
 from calibre.devices.errors import DeviceError
 from calibre.constants import DEBUG
 from calibre.ebooks.chardet import xml_to_unicode
-from calibre.ebooks.metadata import string_to_authors, authors_to_string, \
+from calibre.ebooks.metadata import authors_to_string, title_sort
-    title_sort
 
 # Utility functions {{{
 EMPTY_CARD_CACHE = '''\

src/calibre/devices/teclast/driver.py

@@ -39,4 +39,28 @@ class TECLAST_K3(USBMS):
 
         return drives
 
+class NEWSMY(TECLAST_K3):
+    name = 'Newsmy device interface'
+    gui_name = 'Newsmy'
+    description    = _('Communicate with the Newsmy reader.')
+
+    FORMATS = ['epub', 'fb2', 'pdb', 'html', 'pdf', 'txt', 'skt']
+
+    VENDOR_NAME      = ''
+    WINDOWS_MAIN_MEM = 'NEWSMY'
+    WINDOWS_CARD_A_MEM = 'USBDISK____SD'
+
+    def windows_sort_drives(self, drives):
+        return drives
+
+class IPAPYRUS(TECLAST_K3):
+
+    name = 'iPapyrus device interface'
+    gui_name = 'iPapyrus'
+    description    = _('Communicate with the iPapyrus reader.')
+
+    FORMATS = ['epub', 'pdf', 'txt']
+
+    VENDOR_NAME      = 'E_READER'
+    WINDOWS_MAIN_MEM = ''
 

src/calibre/ebooks/chardet/__init__.py

@@ -32,7 +32,7 @@ def detect(aBuf):
 ENCODING_PATS = [
                  re.compile(r'<\?[^<>]+encoding=[\'"](.*?)[\'"][^<>]*>',
                             re.IGNORECASE),
-                 re.compile(r'<meta.*?content=[\'"].*?charset=([^\s\'"]+).*?[\'"].*?>',
+                 re.compile(r'''<meta\s+?[^<>]+?content=['"][^'"]*?charset=([-a-z0-9]+)[^'"]*?['"][^<>]*>''',
                             re.IGNORECASE)
                  ]
 ENTITY_PATTERN = re.compile(r'&(\S+?);')

src/calibre/ebooks/conversion/plumber.py

@@ -411,6 +411,18 @@ OptionRecommendation(name='asciiize',
 )
         ),
 
+OptionRecommendation(name='keep_ligatures',
+            recommended_value=False, level=OptionRecommendation.LOW,
+            help=_('Preserve ligatures present in the input document. '
+                'A ligature is a special rendering of a pair of '
+                'characters like ff, fi, fl et cetera. '
+                'Most readers do not have support for '
+                'ligatures in their default fonts, so they are '
+                'unlikely to render correctly. By default, calibre '
+                'will turn a ligature into the corresponding pair of normal '
+                'characters. This option will preserve them instead.')
+        ),
+
 OptionRecommendation(name='title',
     recommended_value=None, level=OptionRecommendation.LOW,
     help=_('Set the title.')),

src/calibre/ebooks/conversion/preprocess.py

@@ -18,6 +18,24 @@ convert_entities = functools.partial(entity_to_unicode, exceptions=['quot',
     'apos', 'lt', 'gt', 'amp', '#60', '#62'])
 _span_pat = re.compile('<span.*?</span>', re.DOTALL|re.IGNORECASE)
 
+LIGATURES = {
+        u'\u00c6': u'AE',
+        u'\u00e6': u'ae',
+        u'\u0152': u'OE',
+        u'\u0153': u'oe',
+        u'\u0132': u'IJ',
+        u'\u0133': u'ij',
+        u'\u1D6B': u'ue',
+        u'\uFB00': u'ff',
+        u'\uFB01': u'fi',
+        u'\uFB02': u'fl',
+        u'\uFB03': u'ffi',
+        u'\uFB04': u'ffl',
+        u'\uFB05': u'ft',
+        u'\uFB06': u'st',
+        }
+
+_ligpat = re.compile(u'|'.join(LIGATURES))
 
 def sanitize_head(match):
     x = match.group(1)
@@ -228,6 +246,9 @@ class HTMLPreProcessor(object):
         else:
             rules = []
 
+        if not self.extra_opts.keep_ligatures:
+            html = _ligpat.sub(lambda m:LIGATURES[m.group()], html)
+
         end_rules = []
         if getattr(self.extra_opts, 'remove_header', None):
             try:

src/calibre/ebooks/oeb/transforms/flatcss.py

@@ -164,7 +164,10 @@ class CSSFlattener(object):
             body = html.find(XHTML('body'))
             fsize = self.context.source.fbase
             self.baseline_node(body, stylizer, sizes, fsize)
-        sbase = max(sizes.items(), key=operator.itemgetter(1))[0]
+        try:
+            sbase = max(sizes.items(), key=operator.itemgetter(1))[0]
+        except:
+            sbase = 12.0
         self.oeb.logger.info(
             "Source base font size is %0.05fpt" % sbase)
         return sbase

src/calibre/gui2/convert/look_and_feel.py

@@ -6,7 +6,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-from PyQt4.Qt import SIGNAL
+from PyQt4.Qt import SIGNAL, QVariant
 
 from calibre.gui2.convert.look_and_feel_ui import Ui_Form
 from calibre.gui2.convert import Widget
@@ -24,8 +24,14 @@ class LookAndFeelWidget(Widget, Ui_Form):
                     'linearize_tables',
                     'disable_font_rescaling', 'insert_blank_line',
                     'remove_paragraph_spacing', 'remove_paragraph_spacing_indent_size','input_encoding',
-                    'asciiize']
+                    'asciiize', 'keep_ligatures']
                 )
+        for val, text in [
+                ('original', _('Original')),
+                ('left', _('Left align')),
+                ('justify', _('Justify text'))
+                ]:
+            self.opt_change_justification.addItem(text, QVariant(val))
         self.db, self.book_id = db, book_id
         self.initialize_options(get_option, get_help, db, book_id)
         self.opt_disable_font_rescaling.toggle()
@@ -35,6 +41,21 @@ class LookAndFeelWidget(Widget, Ui_Form):
         self.opt_remove_paragraph_spacing.toggle()
         self.opt_remove_paragraph_spacing.toggle()
 
+    def get_value_handler(self, g):
+        if g is self.opt_change_justification:
+            ans = unicode(g.itemData(g.currentIndex()).toString())
+            return ans
+        return Widget.get_value_handler(self, g)
+
+    def set_value_handler(self, g, val):
+        if g is self.opt_change_justification:
+            for i in range(g.count()):
+                c = unicode(g.itemData(i).toString())
+                if val == c:
+                    g.setCurrentIndex(i)
+                    break
+            return True
+
     def font_key_wizard(self):
         from calibre.gui2.convert.font_key import FontKeyChooser
         d = FontKeyChooser(self, self.opt_base_font_size.value(),

src/calibre/gui2/convert/look_and_feel.ui

@@ -31,7 +31,7 @@
      </property>
     </widget>
    </item>
-   <item row="1" column="2">
+   <item row="1" column="1" colspan="2">
     <widget class="QDoubleSpinBox" name="opt_base_font_size">
      <property name="suffix">
       <string> pt</string>
@@ -63,7 +63,7 @@
      </property>
     </widget>
    </item>
-   <item row="2" column="1" colspan="2">
+   <item row="2" column="1" colspan="3">
     <layout class="QHBoxLayout" name="horizontalLayout">
      <item>
       <widget class="QLineEdit" name="opt_font_size_mapping">
@@ -84,7 +84,7 @@
         <string>...</string>
        </property>
        <property name="icon">
-        <iconset>
+        <iconset resource="../../../../resources/images.qrc">
          <normaloff>:/images/wizard.svg</normaloff>:/images/wizard.svg</iconset>
        </property>
        <property name="iconSize">
@@ -107,7 +107,7 @@
      </property>
     </widget>
    </item>
-   <item row="3" column="2">
+   <item row="3" column="1" colspan="2">
     <widget class="QDoubleSpinBox" name="opt_line_height">
      <property name="suffix">
       <string> pt</string>
@@ -127,60 +127,50 @@
      </property>
     </widget>
    </item>
-   <item row="4" column="1" colspan="2">
+   <item row="4" column="1" colspan="3">
     <widget class="QLineEdit" name="opt_input_encoding"/>
    </item>
-   <item row="5" column="0" colspan="3">
+   <item row="5" column="0" colspan="2">
-    <layout class="QHBoxLayout" name="horizontalLayout_3">
+    <widget class="QCheckBox" name="opt_remove_paragraph_spacing">
+     <property name="text">
+      <string>Remove &amp;spacing between paragraphs</string>
+     </property>
+    </widget>
+   </item>
+   <item row="5" column="2" colspan="2">
+    <layout class="QHBoxLayout" name="horizontalLayout_2">
      <item>
-      <widget class="QCheckBox" name="opt_remove_paragraph_spacing">
+      <widget class="QLabel" name="label_4">
        <property name="text">
-        <string>Remove &amp;spacing between paragraphs</string>
+        <string>Indent size:</string>
+       </property>
+       <property name="alignment">
+        <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
        </property>
       </widget>
      </item>
      <item>
-      <spacer name="horizontalSpacer">
+      <widget class="QDoubleSpinBox" name="opt_remove_paragraph_spacing_indent_size">
-       <property name="orientation">
+       <property name="toolTip">
-        <enum>Qt::Horizontal</enum>
+        <string>&lt;p&gt;When calibre removes inter paragraph spacing, it automatically sets a paragraph indent, to ensure that paragraphs can be easily distinguished. This option controls the width of that indent.</string>
        </property>
-       <property name="sizeHint" stdset="0">
+       <property name="suffix">
-        <size>
+        <string> em</string>
-         <width>40</width>
-         <height>20</height>
-        </size>
        </property>
-      </spacer>
+       <property name="decimals">
-     </item>
+        <number>1</number>
-     <item>
+       </property>
-      <layout class="QHBoxLayout" name="horizontalLayout_2">
+      </widget>
-       <item>
-        <widget class="QLabel" name="label_4">
-         <property name="text">
-          <string>Indent size:</string>
-         </property>
-         <property name="alignment">
-          <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
-         </property>
-        </widget>
-       </item>
-       <item>
-        <widget class="QDoubleSpinBox" name="opt_remove_paragraph_spacing_indent_size">
-         <property name="toolTip">
-          <string>&lt;p&gt;When calibre removes inter paragraph spacing, it automatically sets a paragraph indent, to ensure that paragraphs can be easily distinguished. This option controls the width of that indent.</string>
-         </property>
-         <property name="suffix">
-          <string> em</string>
-         </property>
-         <property name="decimals">
-          <number>1</number>
-         </property>
-        </widget>
-       </item>
-      </layout>
      </item>
     </layout>
    </item>
+   <item row="6" column="0">
+    <widget class="QLabel" name="label_5">
+     <property name="text">
+      <string>Text justification:</string>
+     </property>
+    </widget>
+   </item>
    <item row="7" column="0">
     <widget class="QCheckBox" name="opt_linearize_tables">
      <property name="text">
@@ -188,14 +178,7 @@
      </property>
     </widget>
    </item>
-   <item row="9" column="0">
+   <item row="9" column="0" colspan="4">
-    <widget class="QCheckBox" name="opt_asciiize">
-     <property name="text">
-      <string>&amp;Transliterate unicode characters to ASCII.</string>
-     </property>
-    </widget>
-   </item>
-   <item row="10" column="0" colspan="3">
     <widget class="QGroupBox" name="groupBox">
      <property name="title">
       <string>Extra &amp;CSS</string>
@@ -207,6 +190,16 @@
      </layout>
     </widget>
    </item>
+   <item row="6" column="2" colspan="2">
+    <widget class="QComboBox" name="opt_change_justification"/>
+   </item>
+   <item row="7" column="1" colspan="3">
+    <widget class="QCheckBox" name="opt_asciiize">
+     <property name="text">
+      <string>&amp;Transliterate unicode characters to ASCII</string>
+     </property>
+    </widget>
+   </item>
    <item row="8" column="0">
     <widget class="QCheckBox" name="opt_insert_blank_line">
      <property name="text">
@@ -214,35 +207,13 @@
      </property>
     </widget>
    </item>
-   <item row="6" column="0">
+   <item row="8" column="1" colspan="2">
-    <widget class="QLabel" name="label_5">
+    <widget class="QCheckBox" name="opt_keep_ligatures">
      <property name="text">
-      <string>Text justification:</string>
+      <string>Keep &amp;ligatures</string>
      </property>
     </widget>
    </item>
-   <item row="6" column="2">
-    <widget class="QComboBox" name="opt_change_justification">
-     <property name="currentIndex">
-      <number>2</number>
-     </property>
-     <item>
-      <property name="text">
-       <string>justify</string>
-      </property>
-     </item>
-     <item>
-      <property name="text">
-       <string>left</string>
-      </property>
-     </item>
-     <item>
-      <property name="text">
-       <string>original</string>
-      </property>
-     </item>
-    </widget>
-   </item>
   </layout>
  </widget>
  <resources>

src/calibre/gui2/library/models.py

@@ -729,19 +729,18 @@ class BooksModel(QAbstractTableModel): # {{{
 
 class OnDeviceSearch(SearchQueryParser): # {{{
 
-    USABLE_LOCATIONS = set([
+    USABLE_LOCATIONS = [
         'collections',
         'title',
         'author',
         'format',
         'all',
-                 ])
+    ]
 
 
     def __init__(self, model):
-        SearchQueryParser.__init__(self)
+        SearchQueryParser.__init__(self, locations=self.USABLE_LOCATIONS)
         self.model = model
-        self.DEFAULT_LOCATIONS = set(self.DEFAULT_LOCATIONS) | self.USABLE_LOCATIONS
 
     def universal_set(self):
         return set(range(0, len(self.model.db)))
@@ -1054,6 +1053,8 @@ class DeviceBooksModel(BooksModel): # {{{
         return NONE
 
     def headerData(self, section, orientation, role):
+        if role == Qt.ToolTipRole:
+            return QVariant(_('The lookup/search name is "{0}"').format(self.column_map[section]))
         if role != Qt.DisplayRole:
             return NONE
         if orientation == Qt.Horizontal: