Sync to trunk.

resources/mime.types

@@ -585,7 +585,6 @@ application/vnd.osa.netdeploy
 application/vnd.osgi.bundle				
 application/vnd.osgi.dp				dp
 application/vnd.otps.ct-kip+xml				
-application/vnd.palm				oprc pdb pqa
 application/vnd.paos.xml				
 application/vnd.pg.format				str
 application/vnd.pg.osasli				ei6
@@ -1082,7 +1081,6 @@ chemical/x-ncbi-asn1				asn
 chemical/x-ncbi-asn1-ascii				ent prt
 chemical/x-ncbi-asn1-binary				aso val
 chemical/x-ncbi-asn1-spec				asn
-chemical/x-pdb				ent pdb
 chemical/x-rosdal				ros
 chemical/x-swissprot				sw
 chemical/x-vamas-iso14976				vms
@@ -1379,3 +1377,5 @@ application/x-cbr                          cbr
 application/x-cb7                          cb7
 application/x-koboreader-ebook             kobo
 image/wmf                                  wmf
+application/ereader                        pdb
+

resources/recipes/wsj.recipe

@@ -35,7 +35,7 @@ class WallStreetJournal(BasicNewsRecipe):
 
     remove_tags_before = dict(name='h1')
     remove_tags = [
-                    dict(id=["articleTabs_tab_article", "articleTabs_tab_comments", "articleTabs_tab_interactive","articleTabs_tab_video","articleTabs_tab_map","articleTabs_tab_slideshow"]),
+                    dict(id=["articleTabs_tab_article", "articleTabs_tab_comments", "articleTabs_tab_interactive","articleTabs_tab_video","articleTabs_tab_map","articleTabs_tab_slideshow","articleTabs_tab_quotes","articleTabs_tab_document"]),
                     {'class':['footer_columns','network','insetCol3wide','interactive','video','slideshow','map','insettip','insetClose','more_in', "insetContent", 'articleTools_bottom', 'aTools', "tooltip", "adSummary", "nav-inline"]},
                     dict(rel='shortcut icon'),
                     ]
@@ -101,7 +101,7 @@ class WallStreetJournal(BasicNewsRecipe):
                title = 'Front Section'
                url = 'http://online.wsj.com' + a['href']
                feeds = self.wsj_add_feed(feeds,title,url)
-               title = 'What''s News'
+               title = "What's News"
                url = url.replace('pageone','whatsnews')
                feeds = self.wsj_add_feed(feeds,title,url)
             else:

resources/recipes/wsj_free.recipe

@@ -10,7 +10,10 @@ class WallStreetJournal(BasicNewsRecipe):
 
     title = 'Wall Street Journal (free)'
     __author__ = 'Kovid Goyal, Sujata Raman, Joshua Oster-Morris, Starson17'
-    description = 'News and current affairs'
+    description = '''News and current affairs. This recipe only fetches complete
+    versions of the articles that are available free on the wsj.com website.
+    To get the rest of the articles, subscribe to the WSJ and use the other WSJ
+    recipe.'''
     language = 'en'
     cover_url           = 'http://dealbreaker.com/images/thumbs/Wall%20Street%20Journal%20A1.JPG'
     max_articles_per_feed = 1000
@@ -151,6 +154,4 @@ class WallStreetJournal(BasicNewsRecipe):
 
         return articles
 
-    def cleanup(self):
-        self.browser.open('http://online.wsj.com/logout?url=http://online.wsj.com')
 

src/calibre/debug.py

@@ -22,13 +22,15 @@ Run an embedded python interpreter.
     parser.add_option('-d', '--debug-device-driver', default=False, action='store_true',
                       help='Debug the specified device driver.')
     parser.add_option('-g', '--gui',  default=False, action='store_true',
-                      help='Run the GUI',)
+                      help='Run the GUI with debugging enabled. Debug output is '
+                      'printed to stdout and stderr.')
     parser.add_option('--gui-debug',  default=None,
                       help='Run the GUI with a debug console, logging to the'
-                      ' specified path',)
+                      ' specified path. For internal use only, use the -g'
+                      ' option to run the GUI in debug mode',)
     parser.add_option('--show-gui-debug',  default=None,
-                      help='Display the specified log file.',)
+                      help='Display the specified log file. For internal use'
-
+                      ' only.',)
     parser.add_option('-w', '--viewer',  default=False, action='store_true',
                       help='Run the ebook viewer',)
     parser.add_option('--paths', default=False, action='store_true',

src/calibre/devices/interface.py

@@ -35,6 +35,16 @@ class DevicePlugin(Plugin):
 
     #: Height for thumbnails on the device
     THUMBNAIL_HEIGHT = 68
+    #: Width for thumbnails on the device. Setting this will force thumbnails
+    #: to this size, not preserving aspect ratio. If it is not set, then
+    #: the aspect ratio will be preserved and the thumbnail will be no higher
+    #: than THUMBNAIL_HEIGHT
+    # THUMBNAIL_WIDTH = 68
+
+    #: Set this to True if the device supports updating cover thumbnails during
+    #: sync_booklists. Setting it to true will ask device.py to refresh the
+    #: cover thumbnails during book matching
+    WANTS_UPDATED_THUMBNAILS = False
 
     #: Whether the metadata on books can be set via the GUI.
     CAN_SET_METADATA = ['title', 'authors', 'collections']

src/calibre/devices/prs505/__init__.py

@@ -8,5 +8,5 @@ CACHE_XML = 'Sony Reader/database/cache.xml'
 CACHE_EXT = 'Sony Reader/database/cacheExt.xml'
 
 MEDIA_THUMBNAIL = 'database/thumbnail'
-CACHE_THUMBNAIL = 'Sony Reader/database/thumbnail'
+CACHE_THUMBNAIL = 'Sony Reader/thumbnail'
 

src/calibre/devices/prs505/driver.py

@@ -81,12 +81,19 @@ class PRS505(USBMS):
                 _('Set this option to have separate book covers uploaded '
                   'every time you connect your device. Unset this option if '
                   'you have so many books on the reader that performance is '
-                  'unacceptable.')
+                  'unacceptable.'),
+            _('Preserve cover aspect ratio when building thumbnails') +
+                ':::' +
+                _('Set this option if you want the cover thumbnails to have '
+                  'the same aspect ratio (width to height) as the cover. '
+                  'Unset it if you want the thumbnail to be the maximum size, '
+                  'ignoring aspect ratio.')
     ]
     EXTRA_CUSTOMIZATION_DEFAULT = [
                 ', '.join(['series', 'tags']),
                 False,
-                False
+                False,
+                True
     ]
 
     OPT_COLLECTIONS    = 0
@@ -96,7 +103,7 @@ class PRS505(USBMS):
     plugboard = None
     plugboard_func = None
 
-    THUMBNAIL_HEIGHT = 200
+    THUMBNAIL_HEIGHT = 217
 
     MAX_PATH_LEN = 201 # 250 - (max(len(CACHE_THUMBNAIL), len(MEDIA_THUMBNAIL)) +
                        # len('main_thumbnail.jpg') + 1)
@@ -138,6 +145,13 @@ class PRS505(USBMS):
             if not write_cache(self._card_b_prefix):
                 self._card_b_prefix = None
         self.booklist_class.rebuild_collections = self.rebuild_collections
+        # Set the thumbnail width to the theoretical max if the user has asked
+        # that we do not preserve aspect ratio
+        if not self.settings().extra_customization[3]:
+            self.THUMBNAIL_WIDTH = 168
+        # Set WANTS_UPDATED_THUMBNAILS if the user has asked that thumbnails be
+        # updated on every connect
+        self.WANTS_UPDATED_THUMBNAILS = self.settings().extra_customization[2]
 
     def get_device_information(self, end_session=True):
         return (self.gui_name, '', '', '')

src/calibre/ebooks/conversion/cli.py

@@ -46,7 +46,8 @@ HEURISTIC_OPTIONS = ['markup_chapter_headings',
                       'italicize_common_cases', 'fix_indents',
                       'html_unwrap_factor', 'unwrap_lines',
                       'delete_blank_paragraphs', 'format_scene_breaks',
-                      'dehyphenate', 'renumber_headings']
+                      'dehyphenate', 'renumber_headings',
+                      'replace_scene_breaks']
 
 def print_help(parser, log):
     help = parser.format_help().encode(preferred_encoding, 'replace')
@@ -143,7 +144,7 @@ def add_pipeline_options(parser, plumber):
                      ' patterns. Disabled by default. Use %s to enable. '
                      ' Individual actions can be disabled with the %s options.')
                   % ('--enable-heuristics', '--disable-*'),
-                  ['enable_heuristics', 'replace_scene_breaks'] + HEURISTIC_OPTIONS
+                  ['enable_heuristics'] + HEURISTIC_OPTIONS
                   ),
 
               'SEARCH AND REPLACE' : (

src/calibre/ebooks/conversion/plumber.py

@@ -530,10 +530,11 @@ OptionRecommendation(name='format_scene_breaks',
     help=_('Left aligned scene break markers are center aligned. '
            'Replace soft scene breaks that use multiple blank lines with'
            'horizontal rules.')),
-           
+
 OptionRecommendation(name='replace_scene_breaks',
     recommended_value='', level=OptionRecommendation.LOW,
-    help=_('Replace scene breaks with the specified text.')),
+    help=_('Replace scene breaks with the specified text. By default, the '
+        'text from the input document is used.')),
 
 OptionRecommendation(name='dehyphenate',
     recommended_value=True, level=OptionRecommendation.LOW,

src/calibre/ebooks/conversion/utils.py

@@ -423,11 +423,11 @@ class HeuristicProcessor(object):
             if getattr(self.extra_opts, option, False):
                 return True
         return False
-        
+
     def merge_blanks(self, html, blanks_count=None):
         base_em = .5 # Baseline is 1.5em per blank line, 1st line is .5 em css and 1em for the nbsp
         em_per_line = 1.5 # Add another 1.5 em for each additional blank
-        
+
         def merge_matches(match):
             to_merge = match.group(0)
             lines = float(len(self.single_blank.findall(to_merge))) - 1.
@@ -437,17 +437,17 @@ class HeuristicProcessor(object):
             else:
                 newline = self.any_multi_blank.sub('\n<p class="softbreak'+str(int(em * 10))+'" style="text-align:center; margin-top:'+str(em)+'em"> </p>', match.group(0))
             return newline
-            
+
         html = self.any_multi_blank.sub(merge_matches, html)
         return html
 
     def detect_whitespace(self, html):
-        blanks_around_headings = re.compile(r'(?P<initparas>(<p[^>]*>\s*</p>\s*){1,}\s*)?(?P<heading><h(?P<hnum>\d+)[^>]*>.*?</h(?P=hnum)>)(?P<endparas>\s*(<p[^>]*>\s*</p>\s*){1,})?', re.IGNORECASE)                                     
+        blanks_around_headings = re.compile(r'(?P<initparas>(<p[^>]*>\s*</p>\s*){1,}\s*)?(?P<heading><h(?P<hnum>\d+)[^>]*>.*?</h(?P=hnum)>)(?P<endparas>\s*(<p[^>]*>\s*</p>\s*){1,})?', re.IGNORECASE)
         blanks_n_nopunct = re.compile(r'(?P<initparas>(<p[^>]*>\s*</p>\s*){1,}\s*)?<p[^>]*>\s*(<(span|[ibu]|em|strong|font)[^>]*>\s*)*.{1,100}?[^\W](</(span|[ibu]|em|strong|font)>\s*)*</p>(?P<endparas>\s*(<p[^>]*>\s*</p>\s*){1,})?', re.IGNORECASE)
-        
+
         def merge_header_whitespace(match):
             initblanks = match.group('initparas')
-            endblanks = match.group('initparas') 
+            endblanks = match.group('initparas')
             heading = match.group('heading')
             top_margin = ''
             bottom_margin = ''
@@ -484,7 +484,7 @@ class HeuristicProcessor(object):
 
     def markup_user_break(self, replacement_break):
         '''
-        Takes string a user supplies and wraps it in markup that will be centered with 
+        Takes string a user supplies and wraps it in markup that will be centered with
         appropriate margins.  <hr> and <img> tags are allowed.  If the user specifies
         a style with width attributes in the <hr> tag then the appropriate margins are
         applied to wrapping divs.  This is because many ebook devices don't support margin:auto
@@ -499,10 +499,11 @@ class HeuristicProcessor(object):
                    hr_open = re.sub('45', str(divpercent), hr_open)
                    scene_break = hr_open+replacement_break+'</div>'
                 else:
-                    scene_break = hr_open+'<hr style="height: 3px; background:#505050" /></div>'
+                   scene_break = hr_open+'<hr style="height: 3px; background:#505050" /></div>'
             elif re.match('^<img', replacement_break):
                 scene_break = self.scene_break_open+replacement_break+'</p>'
             else:
+                from calibre.utils.html2text import html2text
                 replacement_break = html2text(replacement_break)
                 replacement_break = re.sub('\s', '&nbsp;', replacement_break)
                 scene_break = self.scene_break_open+replacement_break+'</p>'
@@ -646,11 +647,11 @@ class HeuristicProcessor(object):
                 if len(scene_break.findall(html)) >= 1:
                     html = scene_break.sub(replacement_break, html)
                 else:
-                    html = re.sub('<p\s+class="softbreak"[^>]*>\s*</p>', replacement_break, html) 
+                    html = re.sub('<p\s+class="softbreak"[^>]*>\s*</p>', replacement_break, html)
             else:
                 html = scene_break.sub(self.scene_break_open+'\g<break>'+'</p>', html)
 
         if self.deleted_nbsps:
             # put back non-breaking spaces in empty paragraphs so they render correctly
             html = self.anyblank.sub('\n'+r'\g<openline>'+u'\u00a0'+r'\g<closeline>', html)
-        return html
+        return html

src/calibre/ebooks/metadata/opf2.py

@@ -422,6 +422,33 @@ class MetadataField(object):
             elem = obj.create_metadata_element(self.name, is_dc=self.is_dc)
         obj.set_text(elem, self.renderer(val))
 
+class TitleSortField(MetadataField):
+
+    def __get__(self, obj, type=None):
+        c = self.__real_get__(obj, type)
+        if c is None:
+            matches = obj.title_path(obj.metadata)
+            if matches:
+                for match in matches:
+                    ans = match.get('{%s}file-as'%obj.NAMESPACES['opf'], None)
+                    if not ans:
+                        ans = match.get('file-as', None)
+                    if ans:
+                        c = ans
+        if not c:
+            c = self.none_is
+        else:
+            c = c.strip()
+        return c
+
+    def __set__(self, obj, val):
+        MetadataField.__set__(self, obj, val)
+        matches = obj.title_path(obj.metadata)
+        if matches:
+            for match in matches:
+                for attr in list(match.attrib):
+                    if attr.endswith('file-as'):
+                        del match.attrib[attr]
 
 def serialize_user_metadata(metadata_elem, all_user_metadata, tail='\n'+(' '*8)):
     from calibre.utils.config import to_json
@@ -490,6 +517,7 @@ class OPF(object): # {{{
     rights          = MetadataField('rights')
     series          = MetadataField('series', is_dc=False)
     series_index    = MetadataField('series_index', is_dc=False, formatter=float, none_is=1)
+    title_sort      = TitleSortField('title_sort', is_dc=False)
     rating          = MetadataField('rating', is_dc=False, formatter=int)
     pubdate         = MetadataField('date', formatter=parse_date,
                                     renderer=isoformat)
@@ -776,30 +804,6 @@ class OPF(object): # {{{
 
         return property(fget=fget, fset=fset)
 
-    @dynamic_property
-    def title_sort(self):
-
-        def fget(self):
-            matches = self.title_path(self.metadata)
-            if matches:
-                for match in matches:
-                    ans = match.get('{%s}file-as'%self.NAMESPACES['opf'], None)
-                    if not ans:
-                        ans = match.get('file-as', None)
-                    if ans:
-                        return ans
-
-        def fset(self, val):
-            matches = self.title_path(self.metadata)
-            if matches:
-                for key in matches[0].attrib:
-                    if key.endswith('file-as'):
-                        matches[0].attrib.pop(key)
-                matches[0].set('{%s}file-as'%self.NAMESPACES['opf'], unicode(val))
-
-        return property(fget=fget, fset=fset)
-
-
     @dynamic_property
     def tags(self):
 
@@ -1129,8 +1133,6 @@ class OPFCreator(Metadata):
         metadata = M.metadata()
         a = metadata.append
         role = {}
-        if self.title_sort:
-            role = {'file-as':self.title_sort}
         a(DC_ELEM('title', self.title if self.title else _('Unknown'),
             opf_attrs=role))
         for i, author in enumerate(self.authors):
@@ -1165,6 +1167,8 @@ class OPFCreator(Metadata):
             a(CAL_ELEM('calibre:series', self.series))
             if self.series_index is not None:
                 a(CAL_ELEM('calibre:series_index', self.format_series_index()))
+        if self.title_sort:
+            a(CAL_ELEM('calibre:title_sort', self.title_sort))
         if self.rating is not None:
             a(CAL_ELEM('calibre:rating', str(self.rating)))
         if self.timestamp is not None:
@@ -1320,7 +1324,6 @@ def test_m2o():
     mi.author_sort = 'author sort'
     mi.pubdate = nowf()
     mi.language = 'en'
-    mi.category = 'test'
     mi.comments = 'what a fun book\n\n'
     mi.publisher = 'publisher'
     mi.isbn = 'boooo'
@@ -1335,11 +1338,11 @@ def test_m2o():
     opf = metadata_to_opf(mi)
     print opf
     newmi = MetaInformation(OPF(StringIO(opf)))
-    for attr in ('author_sort', 'title_sort', 'comments', 'category',
+    for attr in ('author_sort', 'title_sort', 'comments',
                     'publisher', 'series', 'series_index', 'rating',
                     'isbn', 'tags', 'cover_data', 'application_id',
                     'language', 'cover',
-                    'book_producer', 'timestamp', 'lccn', 'lcc', 'ddc',
+                    'book_producer', 'timestamp',
                     'pubdate', 'rights', 'publication_type'):
         o, n = getattr(mi, attr), getattr(newmi, attr)
         if o != n and o.strip() != n.strip():
@@ -1441,4 +1444,6 @@ def test_user_metadata():
     print opf.render()
 
 if __name__ == '__main__':
-    test_user_metadata()
+    #test_user_metadata()
+    #test_m2o()
+    test()

src/calibre/ebooks/metadata/sources/google.py

@@ -65,7 +65,7 @@ def to_metadata(browser, log, entry_):
 
     mi = Metadata(title_, authors)
     try:
-        raw = browser.open(id_url).read()
+        raw = browser.open_novisit(id_url).read()
         feed = etree.fromstring(raw)
         extra = entry(feed)[0]
     except:
@@ -129,7 +129,7 @@ class Worker(Thread):
         for i in self.entries:
             try:
                 ans = to_metadata(self.browser, self.log, i)
-                if ans is not None:
+                if isinstance(ans, Metadata):
                     self.result_queue.put(ans)
             except:
                 self.log.exception(

src/calibre/ebooks/mobi/reader.py

@@ -103,6 +103,8 @@ class EXTHHeader(object):
                 pass
         elif id == 108:
             pass # Producer
+        elif id == 113:
+            pass # ASIN or UUID
         #else:
         #    print 'unhandled metadata record', id, repr(content)
 

src/calibre/ebooks/mobi/writer.py

@@ -1547,6 +1547,31 @@ class MobiWriter(object):
                     rights = 'Unknown'
                 exth.write(pack('>II', EXTH_CODES['rights'], len(rights) + 8))
                 exth.write(rights)
+                nrecs += 1
+
+        # Write UUID as ASIN
+        uuid = None
+        from calibre.ebooks.oeb.base import OPF
+        for x in oeb.metadata['identifier']:
+            if x.get(OPF('scheme'), None).lower() == 'uuid' or unicode(x).startswith('urn:uuid:'):
+                uuid = unicode(x).split(':')[-1]
+                break
+        if uuid is None:
+            from uuid import uuid4
+            uuid = str(uuid4())
+
+        if isinstance(uuid, unicode):
+            uuid = uuid.encode('utf-8')
+        exth.write(pack('>II', 113, len(uuid) + 8))
+        exth.write(uuid)
+        nrecs += 1
+
+        # Write cdetype
+        if not self.opts.mobi_periodical:
+            data = 'EBOK'
+            exth.write(pack('>II', 501, len(data)+8))
+            exth.write(data)
+            nrecs += 1
 
         # Add a publication date entry
         if oeb.metadata['date'] != [] :

src/calibre/gui2/complete.py

@@ -0,0 +1,357 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+from PyQt4.Qt import QLineEdit, QListView, QAbstractListModel, Qt, QTimer, \
+        QApplication, QPoint, QItemDelegate, QStyleOptionViewItem, \
+        QStyle, QEvent, pyqtSignal
+
+from calibre.utils.icu import sort_key, lower
+from calibre.gui2 import NONE
+from calibre.gui2.widgets import EnComboBox
+
+class CompleterItemDelegate(QItemDelegate): # {{{
+
+    ''' Renders the current item as thought it were selected '''
+
+    def __init__(self, view):
+        self.view = view
+        QItemDelegate.__init__(self, view)
+
+    def paint(self, p, opt, idx):
+        opt = QStyleOptionViewItem(opt)
+        opt.showDecorationSelected = True
+        if self.view.currentIndex() == idx:
+            opt.state |= QStyle.State_HasFocus
+        QItemDelegate.paint(self, p, opt, idx)
+
+# }}}
+
+class CompleteWindow(QListView): # {{{
+
+    '''
+    The completion popup. For keyboard and mouse handling see
+    :meth:`eventFilter`.
+    '''
+
+    #: This signal is emitted when the user selects one of the listed
+    #: completions, by mouse or keyboard
+    completion_selected = pyqtSignal(object)
+
+    def __init__(self, widget, model):
+        self.widget = widget
+        QListView.__init__(self)
+        self.setVisible(False)
+        self.setParent(None, Qt.Popup)
+        self.setAlternatingRowColors(True)
+        self.setFocusPolicy(Qt.NoFocus)
+        self._d = CompleterItemDelegate(self)
+        self.setItemDelegate(self._d)
+        self.setModel(model)
+        self.setFocusProxy(widget)
+        self.installEventFilter(self)
+        self.clicked.connect(self.do_selected)
+        self.entered.connect(self.do_entered)
+        self.setMouseTracking(True)
+
+    def do_entered(self, idx):
+        if idx.isValid():
+            self.setCurrentIndex(idx)
+
+    def do_selected(self, idx=None):
+        idx = self.currentIndex() if idx is None else idx
+        if not idx.isValid() and self.model().rowCount() > 0:
+            idx = self.model().index(0)
+        if idx.isValid():
+            data = unicode(self.model().data(idx, Qt.DisplayRole))
+            self.completion_selected.emit(data)
+        self.hide()
+
+    def eventFilter(self, o, e):
+        if o is not self:
+            return False
+        if e.type() == e.KeyPress:
+            key = e.key()
+            if key in (Qt.Key_Escape, Qt.Key_Backtab) or \
+                    (key == Qt.Key_F4 and (e.modifiers() & Qt.AltModifier)):
+                self.hide()
+                return True
+            elif key in (Qt.Key_Enter, Qt.Key_Return, Qt.Key_Tab):
+                self.do_selected()
+                return True
+            elif key in (Qt.Key_Up, Qt.Key_Down, Qt.Key_PageUp,
+                    Qt.Key_PageDown):
+                return False
+            # Send key event to associated line edit
+            self.widget.eat_focus_out = False
+            try:
+                self.widget.event(e)
+            finally:
+                self.widget.eat_focus_out = True
+            if not self.widget.hasFocus():
+                # Line edit lost focus
+                self.hide()
+            if e.isAccepted():
+                # Line edit consumed event
+                return True
+        elif e.type() == e.MouseButtonPress:
+            # Hide popup if user clicks outside it, otherwise
+            # pass event to popup
+            if not self.underMouse():
+                self.hide()
+                return True
+        elif e.type() in (e.InputMethod, e.ShortcutOverride):
+            QApplication.sendEvent(self.widget, e)
+
+        return False # Do not filter this event
+
+# }}}
+
+class CompleteModel(QAbstractListModel):
+
+    def __init__(self, parent=None):
+        QAbstractListModel.__init__(self, parent)
+        self.sep = ','
+        self.space_before_sep = False
+        self.items = []
+        self.lowered_items = []
+        self.matches = []
+
+    def set_items(self, items):
+        items = [unicode(x.strip()) for x in items]
+        self.items = list(sorted(items, key=lambda x: sort_key(x)))
+        self.lowered_items = [lower(x) for x in self.items]
+        self.matches = []
+        self.reset()
+
+    def rowCount(self, *args):
+        return len(self.matches)
+
+    def data(self, index, role):
+        if role == Qt.DisplayRole:
+            r = index.row()
+            try:
+                return self.matches[r]
+            except IndexError:
+                pass
+        return NONE
+
+    def get_matches(self, prefix):
+        '''
+        Return all matches that (case insensitively) start with prefix
+        '''
+        prefix = lower(prefix)
+        ans = []
+        if prefix:
+            for i, test in enumerate(self.lowered_items):
+                if test.startswith(prefix):
+                    ans.append(self.items[i])
+        return ans
+
+    def update_matches(self, matches):
+        self.matches = matches
+        self.reset()
+
+class MultiCompleteLineEdit(QLineEdit):
+    '''
+    A line edit that completes on multiple items separated by a
+    separator. Use the :meth:`update_items_cache` to set the list of
+    all possible completions. Separator can be controlled with the
+    :meth:`set_separator` and :meth:`set_space_before_sep` methods.
+
+    A call to self.set_separator(None) will allow this widget to be used
+    to complete non multiple fields as well.
+    '''
+
+    def __init__(self, parent=None):
+        self.eat_focus_out = True
+        self.max_visible_items = 7
+        self.current_prefix = None
+        QLineEdit.__init__(self, parent)
+
+        self._model = CompleteModel(parent=self)
+        self.complete_window = CompleteWindow(self, self._model)
+        self.textChanged.connect(self.text_changed)
+        self.cursorPositionChanged.connect(self.cursor_position_changed)
+        self.complete_window.completion_selected.connect(self.completion_selected)
+
+    # Interface {{{
+    def update_items_cache(self, complete_items):
+        self.all_items = complete_items
+
+    def set_separator(self, sep):
+        self.sep = sep
+
+    def set_space_before_sep(self, space_before):
+        self.space_before_sep = space_before
+
+    # }}}
+
+    def eventFilter(self, o, e):
+        if self.eat_focus_out and o is self and e.type() == QEvent.FocusOut:
+            if self.complete_window.isVisible():
+                return True # Filter this event since the cw is visible
+        return QLineEdit.eventFilter(self, o, e)
+
+
+    def text_changed(self, *args):
+        self.update_completions()
+
+    def cursor_position_changed(self, *args):
+        self.update_completions()
+
+    def update_completions(self):
+        ' Update the list of completions '
+        cpos = self.cursorPosition()
+        text = unicode(self.text())
+        prefix = text[:cpos]
+        self.current_prefix = prefix
+        complete_prefix = prefix.lstrip()
+        if self.sep:
+            complete_prefix = prefix = prefix.split(self.sep)[-1].lstrip()
+
+        matches = self._model.get_matches(complete_prefix)
+        self.update_complete_window(matches)
+
+    def get_completed_text(self, text):
+        '''
+        Get completed text from current cursor position and the completion
+        text
+        '''
+        if self.sep is None:
+            return text
+        else:
+            cursor_pos = self.cursorPosition()
+            before_text = unicode(self.text())[:cursor_pos]
+            after_text = unicode(self.text())[cursor_pos:]
+            after_parts = after_text.split(self.sep)
+            if len(after_parts) < 3 and not after_parts[-1].strip():
+                after_text = u''
+            prefix_len = len(before_text.split(self.sep)[-1].lstrip())
+            if self.space_before_sep:
+                complete_text_pat = '%s%s %s %s'
+                len_extra = 3
+            else:
+                complete_text_pat = '%s%s%s %s'
+                len_extra = 2
+            return prefix_len, len_extra, complete_text_pat % (
+                before_text[:cursor_pos - prefix_len], text, self.sep, after_text)
+
+    def completion_selected(self, text):
+        prefix_len, len_extra, ctext = self.get_completed_text(text)
+        if self.sep is None:
+            self.setText(ctext)
+            self.setCursorPosition(len(ctext))
+        else:
+            cursor_pos = self.cursorPosition()
+            self.setText(ctext)
+            self.setCursorPosition(cursor_pos - prefix_len + len(text) + len_extra)
+
+    def update_complete_window(self, matches):
+        self._model.update_matches(matches)
+        if matches:
+            self.show_complete_window()
+        else:
+            self.complete_window.hide()
+
+
+    def position_complete_window(self):
+        popup = self.complete_window
+        screen = QApplication.desktop().availableGeometry(self)
+        h = (popup.sizeHintForRow(0) * min(self.max_visible_items,
+            popup.model().rowCount()) + 3) + 3
+        hsb = popup.horizontalScrollBar()
+        if hsb and hsb.isVisible():
+            h += hsb.sizeHint().height()
+
+        rh = self.height()
+        pos = self.mapToGlobal(QPoint(0, self.height() - 2))
+        w = self.width()
+
+        if w > screen.width():
+            w = screen.width()
+        if (pos.x() + w) > (screen.x() + screen.width()):
+            pos.setX(screen.x() + screen.width() - w)
+        if (pos.x() < screen.x()):
+            pos.setX(screen.x())
+
+        top = pos.y() - rh - screen.top() + 2
+        bottom = screen.bottom() - pos.y()
+        h = max(h, popup.minimumHeight())
+        if h > bottom:
+            h = min(max(top, bottom), h)
+            if top > bottom:
+                pos.setY(pos.y() - h - rh + 2)
+
+        popup.setGeometry(pos.x(), pos.y(), w, h)
+
+
+    def show_complete_window(self):
+        self.position_complete_window()
+        self.complete_window.show()
+
+    def moveEvent(self, ev):
+        ret = QLineEdit.moveEvent(self, ev)
+        QTimer.singleShot(0, self.position_complete_window)
+        return ret
+
+    def resizeEvent(self, ev):
+        ret = QLineEdit.resizeEvent(self, ev)
+        QTimer.singleShot(0, self.position_complete_window)
+        return ret
+
+
+    @dynamic_property
+    def all_items(self):
+        def fget(self):
+            return self._model.items
+        def fset(self, items):
+            self._model.set_items(items)
+        return property(fget=fget, fset=fset)
+
+    @dynamic_property
+    def sep(self):
+        def fget(self):
+            return self._model.sep
+        def fset(self, val):
+            self._model.sep = val
+        return property(fget=fget, fset=fset)
+
+    @dynamic_property
+    def space_before_sep(self):
+        def fget(self):
+            return self._model.space_before_sep
+        def fset(self, val):
+            self._model.space_before_sep = val
+        return property(fget=fget, fset=fset)
+
+class MultiCompleteComboBox(EnComboBox):
+
+    def __init__(self, *args):
+        EnComboBox.__init__(self, *args)
+        self.setLineEdit(MultiCompleteLineEdit(self))
+
+    def update_items_cache(self, complete_items):
+        self.lineEdit().update_items_cache(complete_items)
+
+    def set_separator(self, sep):
+        self.lineEdit().set_separator(sep)
+
+    def set_space_before_sep(self, space_before):
+        self.lineEdit().set_space_before_sep(space_before)
+
+
+
+if __name__ == '__main__':
+    from PyQt4.Qt import QDialog, QVBoxLayout
+    app = QApplication([])
+    d = QDialog()
+    d.setLayout(QVBoxLayout())
+    le = MultiCompleteLineEdit(d)
+    d.layout().addWidget(le)
+    le.all_items = ['one', 'otwo', 'othree', 'ooone', 'ootwo', 'oothree']
+    d.exec_()

src/calibre/gui2/convert/heuristics.py

@@ -27,7 +27,8 @@ class HeuristicsWidget(Widget, Ui_Form):
                  'dehyphenate', 'renumber_headings']
                 )
         self.db, self.book_id = db, book_id
-        self.rssb_defaults = [u'', u'<hr />', u'* * *', u'• • •', u'✦ ✦ ✦', u'✮ ✮ ✮', 'u☆ ☆ ☆', u'❂ ❂ ❂', u'✣ ✣ ✣', u'❖ ❖ ❖', u'☼ ☼ ☼', u'✠ ✠ ✠']
+        self.rssb_defaults = [u'', u'<hr />', u'* * *', u'• • •', u'✦ ✦ ✦',
+                u'✮ ✮ ✮', u'☆ ☆ ☆', u'❂ ❂ ❂', u'✣ ✣ ✣', u'❖ ❖ ❖', u'☼ ☼ ☼', u'✠ ✠ ✠']
         self.initialize_options(get_option, get_help, db, book_id)
 
         self.load_histories()
@@ -39,7 +40,7 @@ class HeuristicsWidget(Widget, Ui_Form):
 
     def restore_defaults(self, get_option):
         Widget.restore_defaults(self, get_option)
-        
+
         self.save_histories()
         rssb_hist = gprefs['replace_scene_breaks_history']
         for x in self.rssb_defaults:
@@ -50,7 +51,7 @@ class HeuristicsWidget(Widget, Ui_Form):
 
     def commit_options(self, save_defaults=False):
         self.save_histories()
-        
+
         return Widget.commit_options(self, save_defaults)
 
     def break_cycles(self):
@@ -73,7 +74,7 @@ class HeuristicsWidget(Widget, Ui_Form):
     def load_histories(self):
         self.opt_replace_scene_breaks.clear()
         self.opt_replace_scene_breaks.lineEdit().setText('')
-        
+
         val = unicode(self.opt_replace_scene_breaks.currentText())
         rssb_hist = gprefs.get('replace_scene_breaks_history', self.rssb_defaults)
         if val in rssb_hist:

src/calibre/gui2/convert/heuristics.ui

@@ -164,7 +164,10 @@
            </sizepolicy>
           </property>
           <property name="text">
-           <string>Replace soft scene breaks:</string>
+           <string>Replace soft scene &amp;breaks:</string>
+          </property>
+          <property name="buddy">
+           <cstring>opt_replace_scene_breaks</cstring>
           </property>
          </widget>
         </item>

src/calibre/gui2/convert/structure_detection.ui

@@ -48,10 +48,10 @@
      </property>
     </widget>
    </item>
-   <item row="5" column="0" colspan="3">
+   <item row="6" column="0" colspan="3">
     <widget class="XPathEdit" name="opt_page_breaks_before" native="true"/>
    </item>
-   <item row="6" column="0" colspan="3">
+   <item row="7" column="0" colspan="3">
     <spacer name="verticalSpacer">
      <property name="orientation">
       <enum>Qt::Vertical</enum>
@@ -77,6 +77,16 @@
      </property>
     </spacer>
    </item>
+   <item row="4" column="0" colspan="3">
+    <widget class="QLabel" name="label_2">
+     <property name="text">
+      <string>The header and footer removal options have been replaced by the Search &amp; Replace options. Click the Search &amp; Replace category in the bar to the left to use these options. Leave the replace field blank and enter your header/footer removal regexps into the search field.</string>
+     </property>
+     <property name="wordWrap">
+      <bool>true</bool>
+     </property>
+    </widget>
+   </item>
   </layout>
  </widget>
  <customwidgets>

src/calibre/gui2/device.py

@@ -838,9 +838,10 @@ class DeviceMixin(object): # {{{
                             format_count[f] = 1
             for f in self.device_manager.device.settings().format_map:
                 if f in format_count.keys():
-                    formats.append((f, _('%i of %i Books') % (format_count[f], len(rows))), True if f in aval_out_formats else False)
+                    formats.append((f, _('%i of %i Books') % (format_count[f],
+                        len(rows)), True if f in aval_out_formats else False))
                 elif f in aval_out_formats:
-                    formats.append((f, _('0 of %i Books') % len(rows)), True)
+                    formats.append((f, _('0 of %i Books') % len(rows), True))
             d = ChooseFormatDeviceDialog(self, _('Choose format to send to device'), formats)
             if d.exec_() != QDialog.Accepted:
                 return
@@ -871,6 +872,16 @@ class DeviceMixin(object): # {{{
             self.send_by_mail(to, fmts, delete)
 
     def cover_to_thumbnail(self, data):
+        if self.device_manager.device and \
+                hasattr(self.device_manager.device, 'THUMBNAIL_WIDTH'):
+            try:
+                return thumbnail(data,
+                                 self.device_manager.device.THUMBNAIL_WIDTH,
+                                 self.device_manager.device.THUMBNAIL_HEIGHT,
+                                 preserve_aspect_ratio=False)
+            except:
+                pass
+            return
         ht = self.device_manager.device.THUMBNAIL_HEIGHT \
                 if self.device_manager else DevicePlugin.THUMBNAIL_HEIGHT
         try:
@@ -1272,6 +1283,8 @@ class DeviceMixin(object): # {{{
             x = x.lower() if x else ''
             return string_pat.sub('', x)
 
+        update_metadata = prefs['manage_device_metadata'] == 'on_connect'
+
         # Force a reset if the caches are not initialized
         if reset or not hasattr(self, 'db_book_title_cache'):
             # Build a cache (map) of the library, so the search isn't On**2
@@ -1284,8 +1297,13 @@ class DeviceMixin(object): # {{{
             except:
                 return False
 
+            get_covers = False
+            if update_metadata and self.device_manager.is_device_connected:
+                if self.device_manager.device.WANTS_UPDATED_THUMBNAILS:
+                    get_covers = True
+
             for id in db.data.iterallids():
-                mi = db.get_metadata(id, index_is_id=True)
+                mi = db.get_metadata(id, index_is_id=True, get_cover=get_covers)
                 title = clean_string(mi.title)
                 if title not in db_book_title_cache:
                     db_book_title_cache[title] = \
@@ -1311,7 +1329,6 @@ class DeviceMixin(object): # {{{
         # the application_id to the db_id of the matching book. This value
         # will be used by books_on_device to indicate matches.
 
-        update_metadata = prefs['manage_device_metadata'] == 'on_connect'
         for booklist in booklists:
             for book in booklist:
                 book.in_library = None
@@ -1382,6 +1399,12 @@ class DeviceMixin(object): # {{{
 
         if update_metadata:
             if self.device_manager.is_device_connected:
+                if self.device_manager.device.WANTS_UPDATED_THUMBNAILS:
+                    for blist in booklists:
+                        for book in blist:
+                            if book.cover and os.access(book.cover, os.R_OK):
+                                book.thumbnail = \
+                                    self.cover_to_thumbnail(open(book.cover, 'rb').read())
                 plugboards = self.library_view.model().db.prefs.get('plugboards', {})
                 self.device_manager.sync_booklists(
                                     Dispatcher(self.metadata_synced), booklists,

src/calibre/gui2/email.py

@@ -264,8 +264,9 @@ class EmailMixin(object): # {{{
         if _auto_ids != []:
             for id in _auto_ids:
                 if specific_format == None:
-                    formats = [f.lower() for f in self.library_view.model().db.formats(id, index_is_id=True).split(',')]
+                    dbfmts = self.library_view.model().db.formats(id, index_is_id=True)
-                    formats = formats if formats != None else []
+                    formats = [f.lower() for f in (dbfmts.split(',') if fmts else
+                        [])]
                     if list(set(formats).intersection(available_input_formats())) != [] and list(set(fmts).intersection(available_output_formats())) != []:
                         auto.append(id)
                     else:

src/calibre/gui2/metadata/basic_widgets.py

@@ -12,8 +12,8 @@ from PyQt4.Qt import Qt, QDateEdit, QDate, \
     QDoubleSpinBox, QListWidgetItem, QSize, QPixmap, \
     QPushButton, QSpinBox, QLineEdit
 
-from calibre.gui2.widgets import EnLineEdit, CompleteComboBox, \
+from calibre.gui2.widgets import EnLineEdit, EnComboBox, FormatList, ImageView
-        EnComboBox, FormatList, ImageView, CompleteLineEdit
+from calibre.gui2.complete import MultiCompleteLineEdit, MultiCompleteComboBox
 from calibre.utils.icu import sort_key
 from calibre.utils.config import tweaks, prefs
 from calibre.ebooks.metadata import title_sort, authors_to_string, \
@@ -149,14 +149,14 @@ class TitleSortEdit(TitleEdit):
 # }}}
 
 # Authors {{{
-class AuthorsEdit(CompleteComboBox):
+class AuthorsEdit(MultiCompleteComboBox):
 
     TOOLTIP = ''
     LABEL = _('&Author(s):')
 
     def __init__(self, parent):
         self.dialog = parent
-        CompleteComboBox.__init__(self, parent)
+        MultiCompleteComboBox.__init__(self, parent)
         self.setToolTip(self.TOOLTIP)
         self.setWhatsThis(self.TOOLTIP)
         self.setEditable(True)
@@ -814,14 +814,14 @@ class RatingEdit(QSpinBox): # {{{
 
 # }}}
 
-class TagsEdit(CompleteLineEdit): # {{{
+class TagsEdit(MultiCompleteLineEdit): # {{{
     LABEL = _('Ta&gs:')
     TOOLTIP = '<p>'+_('Tags categorize the book. This is particularly '
             'useful while searching. <br><br>They can be any words'
             'or phrases, separated by commas.')
 
     def __init__(self, parent):
-        CompleteLineEdit.__init__(self, parent)
+        MultiCompleteLineEdit.__init__(self, parent)
         self.setToolTip(self.TOOLTIP)
         self.setWhatsThis(self.TOOLTIP)
 
@@ -839,7 +839,7 @@ class TagsEdit(CompleteLineEdit): # {{{
         tags = db.tags(id_, index_is_id=True)
         tags = tags.split(',') if tags else []
         self.current_val = tags
-        self.update_items_cache(db.all_tags())
+        self.all_items = db.all_tags()
         self.original_val = self.current_val
 
     @property
@@ -860,7 +860,7 @@ class TagsEdit(CompleteLineEdit): # {{{
         d = TagEditor(self, db, id_)
         if d.exec_() == TagEditor.Accepted:
             self.current_val = d.tags
-            self.update_items_cache(db.all_tags())
+            self.all_items = db.all_tags()
 
 
     def commit(self, db, id_):

src/calibre/library/database2.py

@@ -430,8 +430,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         authors = self.authors(id, index_is_id=True)
         if not authors:
             authors = _('Unknown')
-        author = ascii_filename(authors.split(',')[0][:self.PATH_LIMIT]).decode(filesystem_encoding, 'ignore')
+        author = ascii_filename(authors.split(',')[0])[:self.PATH_LIMIT].decode(filesystem_encoding, 'replace')
-        title  = ascii_filename(self.title(id, index_is_id=True)[:self.PATH_LIMIT]).decode(filesystem_encoding, 'ignore')
+        title  = ascii_filename(self.title(id, index_is_id=True))[:self.PATH_LIMIT].decode(filesystem_encoding, 'replace')
         path   = author + '/' + title + ' (%d)'%id
         return path
 
@@ -442,8 +442,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         authors = self.authors(id, index_is_id=True)
         if not authors:
             authors = _('Unknown')
-        author = ascii_filename(authors.split(',')[0][:self.PATH_LIMIT]).decode(filesystem_encoding, 'replace')
+        author = ascii_filename(authors.split(',')[0])[:self.PATH_LIMIT].decode(filesystem_encoding, 'replace')
-        title  = ascii_filename(self.title(id, index_is_id=True)[:self.PATH_LIMIT]).decode(filesystem_encoding, 'replace')
+        title  = ascii_filename(self.title(id, index_is_id=True))[:self.PATH_LIMIT].decode(filesystem_encoding, 'replace')
         name   = title + ' - ' + author
         while name.endswith('.'):
             name = name[:-1]

src/calibre/manual/faq.rst

@@ -391,7 +391,7 @@ Take your pick:
   * A tribute to the SONY Librie which was the first e-ink based e-book reader
   * My wife chose it ;-)
 
-|app| is pronounced as cal-i-ber *not* ca-libre. If you're wondering, |app| is the British/commonwealth spelling for caliber. Being Indian, that's the natural spelling for me. 
+|app| is pronounced as cal-i-ber *not* ca-li-bre. If you're wondering, |app| is the British/commonwealth spelling for caliber. Being Indian, that's the natural spelling for me. 
 
 Why does |app| show only some of my fonts on OS X?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

src/calibre/utils/magick/draw.py

@@ -72,11 +72,17 @@ def save_cover_data_to(data, path, bgcolor='#ffffff', resize_to=None,
                 f.write(data)
     return ret
 
-def thumbnail(data, width=120, height=120, bgcolor='#ffffff', fmt='jpg'):
+def thumbnail(data, width=120, height=120, bgcolor='#ffffff', fmt='jpg',
+              preserve_aspect_ratio=True):
     img = Image()
     img.load(data)
     owidth, oheight = img.size
-    scaled, nwidth, nheight = fit_image(owidth, oheight, width, height)
+    if not preserve_aspect_ratio:
+        scaled = owidth > width or oheight > height
+        nwidth = width
+        nheight = height
+    else:
+        scaled, nwidth, nheight = fit_image(owidth, oheight, width, height)
     if scaled:
         img.size = (nwidth, nheight)
     canvas = create_canvas(img.size[0], img.size[1], bgcolor)