Merge from trunk

resources/recipes/lifehacker.recipe

@@ -8,7 +8,7 @@ from calibre.web.feeds.news import BasicNewsRecipe
 
 class Lifehacker(BasicNewsRecipe):
     title                 = 'Lifehacker'
-    __author__            = 'NA'
+    __author__            = 'Kovid Goyal'
     description           = "Computers make us more productive. Yeah, right. Lifehacker recommends the software downloads and web sites that actually save time. Don't live to geek; geek to live."
     publisher             = 'lifehacker.com'
     category              = 'news, IT, Internet, gadgets, tips and tricks, howto, diy'
@@ -32,14 +32,20 @@ class Lifehacker(BasicNewsRecipe):
                         , 'language'  : language
                         }
 
-    remove_attributes  = ['width','height']
+    remove_attributes  = ['width', 'height', 'style']
-    keep_only_tags     = [dict(attrs={'class':'content permalink'})]
     remove_tags_before = dict(name='h1')
-    remove_tags        = [dict(attrs={'class':'contactinfo'})]
+    keep_only_tags = [dict(id='container')]
-    remove_tags_after  = dict(attrs={'class':'contactinfo'})
+    remove_tags_after  = dict(attrs={'class':'post-body'})
+    remove_tags = [
+            dict(id="sharemenu"),
+            {'class': 'related'},
+    ]
 
     feeds = [(u'Articles', u'http://feeds.gawker.com/lifehacker/full')]
 
     def preprocess_html(self, soup):
         return self.adeify_images(soup)
 
+    def print_version(self, url):
+        return url.replace('#!', '?_escaped_fragment_=')
+

resources/recipes/nytimes_sub.recipe

@@ -668,7 +668,7 @@ class NYTimes(BasicNewsRecipe):
 
             try:
                 #remove "Related content" bar
-                runAroundsFound = soup.findAll('div',{'class':['articleInline runaroundLeft','articleInline doubleRule runaroundLeft','articleInline runaroundLeft firstArticleInline','articleInline runaroundLeft  ']})
+                runAroundsFound = soup.findAll('div',{'class':['articleInline runaroundLeft','articleInline doubleRule runaroundLeft','articleInline runaroundLeft firstArticleInline','articleInline runaroundLeft  ','articleInline runaroundLeft  lastArticleInline']})
                 if runAroundsFound:
                     for runAround in runAroundsFound:
                         #find all section headers

resources/recipes/workers_world.recipe

@@ -0,0 +1,26 @@
+from calibre.web.feeds.recipes import BasicNewsRecipe
+
+class WorkersWorld(BasicNewsRecipe):
+
+    title = u'Workers World'
+    description = u'Socialist news and analysis'
+    __author__ = u'urslnx'
+    no_stylesheets = True
+    use_embedded_content = False
+    remove_javascript = True
+    oldest_article = 7
+    max_articles_per_feed = 100
+    encoding = 'utf8'
+    publisher = 'workers.org'
+    category = 'news, politics, USA, world'
+    language = 'en'
+    publication_type = 'newsportal'
+    extra_css = ' body{ font-family: Verdana,Arial,Helvetica,sans-serif; } h1{ font-size: x-large; text-align: left; margin-top:0.5em; margin-bottom:0.25em; } h2{ font-size: large; }  p{ text-align: left; } .published{ font-size: small; } .byline{ font-size: small; } .copyright{ font-size: small; } '
+    remove_tags_before = dict(name='div', attrs={'id':'evernote'})
+    remove_tags_after = dict(name='div', attrs={'id':'footer'})
+
+    masthead_url='http://www.workers.org/graphics/wwlogo300.gif'
+    cover_url = 'http://www.workers.org/pdf/current.jpg'
+    feeds = [(u'Headlines', u'http://www.workers.org/rss/nonstandard_rss.xml'),
+]
+

src/calibre/customize/__init__.py

@@ -90,6 +90,11 @@ class Plugin(object): # {{{
         an optional method validate() that takes no arguments and is called
         immediately after the user clicks OK. Changes are applied if and only
         if the method returns True.
+
+        If for some reason you cannot perform the configuration at this time,
+        return a tuple of two strings (message, details), these will be
+        displayed as a warning dialog to the user and the process will be
+        aborted.
         '''
         raise NotImplementedError()
 
@@ -133,6 +138,12 @@ class Plugin(object): # {{{
         except NotImplementedError:
             config_widget = None
 
+        if isinstance(config_widget, tuple):
+            from calibre.gui2 import warning_dialog
+            warning_dialog(parent, _('Cannot configure'), config_widget[0],
+                    det_msg=config_widget[1], show=True)
+            return False
+
         if config_widget is not None:
             v.addWidget(config_widget)
             v.addWidget(button_box)

src/calibre/customize/builtins.py

@@ -511,14 +511,14 @@ from calibre.ebooks.metadata.fetch import GoogleBooks, ISBNDB, Amazon, \
 from calibre.ebooks.metadata.douban import DoubanBooks
 from calibre.ebooks.metadata.nicebooks import NiceBooks, NiceBooksCovers
 from calibre.ebooks.metadata.covers import OpenLibraryCovers, \
-        LibraryThingCovers, DoubanCovers
+        AmazonCovers, DoubanCovers
 from calibre.library.catalog import CSV_XML, EPUB_MOBI, BIBTEX
 from calibre.ebooks.epub.fix.unmanifested import Unmanifested
 from calibre.ebooks.epub.fix.epubcheck import Epubcheck
 
 plugins = [HTML2ZIP, PML2PMLZ, ArchiveExtract, GoogleBooks, ISBNDB, Amazon,
         KentDistrictLibrary, DoubanBooks, NiceBooks, CSV_XML, EPUB_MOBI, BIBTEX, Unmanifested,
-        Epubcheck, OpenLibraryCovers, LibraryThingCovers, DoubanCovers,
+        Epubcheck, OpenLibraryCovers, AmazonCovers, DoubanCovers,
         NiceBooksCovers]
 plugins += [
     ComicInput,

src/calibre/devices/android/driver.py

@@ -19,7 +19,7 @@ class ANDROID(USBMS):
 
     VENDOR_ID   = {
             # HTC
-            0x0bb4 : { 0x0c02 : [0x100, 0x0227, 0x0226],
+            0x0bb4 : { 0x0c02 : [0x100, 0x0227, 0x0226, 0x222],
                        0x0c01 : [0x100, 0x0227, 0x0226],
                        0x0ff9 : [0x0100, 0x0227, 0x0226],
                        0x0c87 : [0x0100, 0x0227, 0x0226],

src/calibre/devices/apple/driver.py

@@ -39,6 +39,7 @@ if iswindows:
 class DriverBase(DeviceConfig, DevicePlugin):
     # Needed for config_widget to work
     FORMATS = ['epub', 'pdf']
+    USER_CAN_ADD_NEW_FORMATS = False
     SUPPORTS_SUB_DIRS = True   # To enable second checkbox in customize widget
 
     @classmethod

src/calibre/devices/bambook/driver.py

@@ -32,6 +32,7 @@ class BAMBOOK(DeviceConfig, DevicePlugin):
     ip = None
 
     FORMATS = [ "snb" ]
+    USER_CAN_ADD_NEW_FORMATS = False
     VENDOR_ID = 0x230b
     PRODUCT_ID = 0x0001
     BCD = None
@@ -421,7 +422,7 @@ class BAMBOOK(DeviceConfig, DevicePlugin):
         from calibre.gui2.device_drivers.configwidget import ConfigWidget
         cw = ConfigWidget(cls.settings(), cls.FORMATS, cls.SUPPORTS_SUB_DIRS,
             cls.MUST_READ_METADATA, cls.SUPPORTS_USE_AUTHOR_SORT,
-            cls.EXTRA_CUSTOMIZATION_MESSAGE)
+            cls.EXTRA_CUSTOMIZATION_MESSAGE, cls)
         # Turn off the Save template
         cw.opt_save_template.setVisible(False)
         cw.label.setVisible(False)

src/calibre/devices/jetbook/driver.py

@@ -93,11 +93,11 @@ class MIBUK(USBMS):
 
     VENDOR_ID   = [0x0525]
     PRODUCT_ID  = [0xa4a5]
-    BCD         = [0x314]
+    BCD         = [0x314, 0x319]
     SUPPORTS_SUB_DIRS = True
 
-    VENDOR_NAME      = 'LINUX'
+    VENDOR_NAME      = ['LINUX', 'FILE_BAC']
-    WINDOWS_MAIN_MEM = 'WOLDERMIBUK'
+    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['WOLDERMIBUK', 'KED_STORAGE_GADG']
 
 class JETBOOK_MINI(USBMS):
 

src/calibre/devices/kindle/apnx.py

@@ -11,44 +11,42 @@ Generates and writes an APNX page mapping file.
 import struct
 import uuid
 
+from calibre.ebooks.mobi.reader import MobiReader
 from calibre.ebooks.pdb.header import PdbHeaderReader
+from calibre.utils.logging import default_log
 
 class APNXBuilder(object):
     '''
-    2300 characters of uncompressed text per page. This is
+    Create an APNX file using a pseudo page mapping.
-    not meant to map 1 to 1 to a print book but to be a
-    close enough measure.
-    
-    A test book was chosen and the characters were counted
-    on one page. This number was round to 2240 then 60
-    characters of markup were added to the total giving
-    2300.
-    
-    Uncompressed text length is used because it's easily
-    accessible in MOBI files (part of the header). Also,
-    It's faster to work off of the length then to
-    decompress and parse the actual text.
-    
-    A better but much more resource intensive and slower
-    method to calculate the page length would be to parse
-    the uncompressed text. For each paragraph we would
-    want to find how many lines it would occupy in a paper
-    back book. 70 characters per line and 32 lines per page.
-    So divide the number of characters (minus markup) in
-    each paragraph by 70. If there are less than 70
-    characters in the paragraph then it is 1 line. Then,
-    count every 32 lines and mark that location as a page.
     '''
 
-    def write_apnx(self, mobi_file_path, apnx_path):
+    def write_apnx(self, mobi_file_path, apnx_path, accurate=True):
+        # Check that this is really a MOBI file.
         with open(mobi_file_path, 'rb') as mf:
-            phead = PdbHeaderReader(mf)
+            ident = PdbHeaderReader(mf).identity()
-            r0 = phead.section_data(0)
+        if ident != 'BOOKMOBI':
-            text_length = struct.unpack('>I', r0[4:8])[0]
+            raise Exception(_('Not a valid MOBI file. Reports identity of %s' % ident))
 
-        pages = self.get_pages(text_length)
+        # Get the pages depending on the chosen parser
+        pages = []
+        if accurate:
+            try:
+                pages = self.get_pages_accurate(mobi_file_path)
+            except:
+                # Fall back to the fast parser if we can't
+                # use the accurate one. Typically this is
+                # due to the file having DRM.
+                pages = self.get_pages_fast(mobi_file_path)
+        else:
+            pages = self.get_pages_fast(mobi_file_path)
+
+        if not pages:
+            raise Exception(_('Could not generate page mapping.'))
+
+        # Generate the APNX file from the page mapping.
         apnx = self.generate_apnx(pages)
 
+        # Write the APNX.
         with open(apnx_path, 'wb') as apnxf:
             apnxf.write(apnx)
 
@@ -73,18 +71,126 @@ class APNXBuilder(object):
         apnx += struct.pack('>H', 32)
         apnx += page_header
 
-        # write page values to apnx
+        # Write page values to APNX.
         for page in pages:
-            apnx += struct.pack('>L', page)
+            apnx += struct.pack('>I', page)
 
         return apnx
 
-    def get_pages(self, text_length):
+    def get_pages_fast(self, mobi_file_path):
+        '''
+        2300 characters of uncompressed text per page. This is
+        not meant to map 1 to 1 to a print book but to be a
+        close enough measure.
+
+        A test book was chosen and the characters were counted
+        on one page. This number was round to 2240 then 60
+        characters of markup were added to the total giving
+        2300.
+
+        Uncompressed text length is used because it's easily
+        accessible in MOBI files (part of the header). Also,
+        It's faster to work off of the length then to
+        decompress and parse the actual text.
+        '''
+        text_length = 0
         pages = []
         count = 0
 
+        with open(mobi_file_path, 'rb') as mf:
+            phead = PdbHeaderReader(mf)
+            r0 = phead.section_data(0)
+            text_length = struct.unpack('>I', r0[4:8])[0]
+
         while count < text_length:
             pages.append(count)
             count += 2300
 
         return pages
+
+    def get_pages_accurate(self, mobi_file_path):
+        '''
+        A more accurate but much more resource intensive and slower
+        method to calculate the page length.
+
+        Parses the uncompressed text. In an average paper back book
+        There are 32 lines per page and a maximum of 70 characters
+        per line.
+
+        Each paragraph starts a new line and every 70 characters
+        (minus markup) in a paragraph starts a new line. The
+        position after every 30 lines will be marked as a new
+        page.
+
+        This can be make more accurate by accounting for
+        <div class="mbp_pagebreak" /> as a new page marker.
+        And <br> elements as an empty line.
+        '''
+        pages = []
+
+        # Get the MOBI html.
+        mr = MobiReader(mobi_file_path, default_log)
+        if mr.book_header.encryption_type != 0:
+            # DRMed book
+            return self.get_pages_fast(mobi_file_path)
+        mr.extract_text()
+
+        # States
+        in_tag = False
+        in_p = False
+        check_p = False
+        closing = False
+        p_char_count = 0
+
+        # Get positions of every line
+        # A line is either a paragraph starting
+        # or every 70 characters in a paragraph.
+        lines = []
+        pos = -1
+        # We want this to be as fast as possible so we
+        # are going to do one pass across the text. re
+        # and string functions will parse the text each
+        # time they are called.
+        #
+        # We can can use .lower() here because we are
+        # not modifying the text. In this case the case
+        # doesn't matter just the absolute character and
+        # the position within the stream.
+        for c in mr.mobi_html.lower():
+            pos += 1
+
+            # Check if we are starting or stopping a p tag.
+            if check_p:
+                if c == '/':
+                    closing = True
+                    continue
+                elif c == 'p':
+                    if closing:
+                        in_p = False
+                    else:
+                        in_p = True
+                        lines.append(pos - 2)
+                check_p = False
+                closing = False
+                continue
+
+            if c == '<':
+                in_tag = True
+                check_p = True
+                continue
+            elif c == '>':
+                in_tag = False
+                check_p = False
+                continue
+
+            if in_p and not in_tag:
+                p_char_count += 1
+                if p_char_count == 70:
+                    lines.append(pos)
+                    p_char_count = 0
+
+        # Every 30 lines is a new page
+        for i in xrange(0, len(lines), 32):
+            pages.append(lines[i])
+
+        return pages

src/calibre/devices/kindle/driver.py

@@ -176,6 +176,28 @@ class KINDLE2(KINDLE):
     PRODUCT_ID = [0x0002, 0x0004]
     BCD        = [0x0100]
 
+    EXTRA_CUSTOMIZATION_MESSAGE = [
+        _('Send page number information when sending books') +
+            ':::' +
+            _('The Kindle 3 and newer versions can use page number information '
+              'in MOBI files. With this option, calibre will calculate and send'
+              ' this information to the Kindle when uploading MOBI files by'
+              ' USB. Note that the page numbers do not correspond to any paper'
+              ' book.'),
+        _('Use slower but more accurate page number generation') +
+            ':::' +
+            _('There are two ways to generate the page number information. Using the more accurate '
+              'generator will produce pages that correspond better to a printed book. '
+              'However, this method is slower and will slow down sending files '
+              'to the Kindle.'),
+    ]
+    EXTRA_CUSTOMIZATION_DEFAULT = [
+        True,
+        False,
+    ]
+    OPT_APNX           = 0
+    OPT_APNX_ACCURATE  = 1
+
     def books(self, oncard=None, end_session=True):
         bl = USBMS.books(self, oncard=oncard, end_session=end_session)
         # Read collections information
@@ -212,13 +234,17 @@ class KINDLE2(KINDLE):
         '''
         Hijacking this function to write the apnx file.
         '''
-        if not filepath.lower().endswith('.mobi'):
+        opts = self.settings()
+        if not opts.extra_customization[self.OPT_APNX]:
+            return
+
+        if os.path.splitext(filepath.lower())[1] not in ('.azw', '.mobi', '.prc'):
             return
 
         apnx_path = '%s.apnx' % os.path.join(path, filename)
         apnx_builder = APNXBuilder()
         try:
-            apnx_builder.write_apnx(filepath, apnx_path)
+            apnx_builder.write_apnx(filepath, apnx_path, accurate=opts.extra_customization[self.OPT_APNX_ACCURATE])
         except:
             print 'Failed to generate APNX'
             import traceback

src/calibre/devices/kobo/driver.py

@@ -98,7 +98,6 @@ class KOBO(USBMS):
 
         def update_booklist(prefix, path, title, authors, mime, date, ContentType, ImageID, readstatus, MimeType):
             changed = False
-            # if path_to_ext(path) in self.FORMATS:
             try:
                 lpath = path.partition(self.normalize_path(prefix))[2]
                 if lpath.startswith(os.sep):
@@ -220,7 +219,7 @@ class KOBO(USBMS):
         #    2) volume_shorcover
         #    2) content
 
-        debug_print('delete_via_sql: ContentID: ', ContentID, 'ContentType: ', ContentType) 
+        debug_print('delete_via_sql: ContentID: ', ContentID, 'ContentType: ', ContentType)
         connection = sqlite.connect(self.normalize_path(self._main_prefix + '.kobo/KoboReader.sqlite'))
         cursor = connection.cursor()
         t = (ContentID,)
@@ -532,7 +531,7 @@ class KOBO(USBMS):
                         if result is None:
                             datelastread = '1970-01-01T00:00:00'
                         else:
-                            datelastread = result[0] if result[0] is not None else '1970-01-01T00:00:00' 
+                            datelastread = result[0] if result[0] is not None else '1970-01-01T00:00:00'
 
                         t = (datelastread,ContentID,)
 

src/calibre/devices/usbms/deviceconfig.py

@@ -34,6 +34,10 @@ class DeviceConfig(object):
     #: If None the default is used
     SAVE_TEMPLATE = None
 
+    #: If True the user can add new formats to the driver
+    USER_CAN_ADD_NEW_FORMATS = True
+
+
     @classmethod
     def _default_save_template(cls):
         from calibre.library.save_to_disk import config
@@ -73,7 +77,7 @@ class DeviceConfig(object):
         from calibre.gui2.device_drivers.configwidget import ConfigWidget
         cw = ConfigWidget(cls.settings(), cls.FORMATS, cls.SUPPORTS_SUB_DIRS,
             cls.MUST_READ_METADATA, cls.SUPPORTS_USE_AUTHOR_SORT,
-            cls.EXTRA_CUSTOMIZATION_MESSAGE)
+            cls.EXTRA_CUSTOMIZATION_MESSAGE, cls)
         return cw
 
     @classmethod

src/calibre/devices/usbms/driver.py

@@ -93,9 +93,11 @@ class USBMS(CLI, Device):
         for idx,b in enumerate(bl):
             bl_cache[b.lpath] = idx
 
+        all_formats = set(self.settings().format_map) | set(self.FORMATS)
+
         def update_booklist(filename, path, prefix):
             changed = False
-            if path_to_ext(filename) in self.FORMATS:
+            if path_to_ext(filename) in all_formats:
                 try:
                     lpath = os.path.join(path, filename).partition(self.normalize_path(prefix))[2]
                     if lpath.startswith(os.sep):

src/calibre/ebooks/conversion/utils.py

@@ -156,17 +156,17 @@ class HeuristicProcessor(object):
         ]
 
         ITALICIZE_STYLE_PATS = [
-            r'(?msu)(?<=[\s>])_(?P<words>[^_]+)?_',
+            r'(?msu)(?<=[\s>])_(?P<words>[^_]+)_',
-            r'(?msu)(?<=[\s>])/(?P<words>[^/]+)?/',
+            r'(?msu)(?<=[\s>])/(?P<words>[^/]+)/',
-            r'(?msu)(?<=[\s>])~~(?P<words>[^~]+)?~~',
+            r'(?msu)(?<=[\s>])~~(?P<words>[^~]+)~~',
-            r'(?msu)(?<=[\s>])\*(?P<words>[^\*]+)?\*',
+            r'(?msu)(?<=[\s>])\*(?P<words>[^\*]+)\*',
-            r'(?msu)(?<=[\s>])~(?P<words>[^~]+)?~',
+            r'(?msu)(?<=[\s>])~(?P<words>[^~]+)~',
-            r'(?msu)(?<=[\s>])_/(?P<words>[^/_]+)?/_',
+            r'(?msu)(?<=[\s>])_/(?P<words>[^/_]+)/_',
-            r'(?msu)(?<=[\s>])_\*(?P<words>[^\*_]+)?\*_',
+            r'(?msu)(?<=[\s>])_\*(?P<words>[^\*_]+)\*_',
-            r'(?msu)(?<=[\s>])\*/(?P<words>[^/\*]+)?/\*',
+            r'(?msu)(?<=[\s>])\*/(?P<words>[^/\*]+)/\*',
-            r'(?msu)(?<=[\s>])_\*/(?P<words>[^\*_]+)?/\*_',
+            r'(?msu)(?<=[\s>])_\*/(?P<words>[^\*_]+)/\*_',
-            r'(?msu)(?<=[\s>])/:(?P<words>[^:/]+)?:/',
+            r'(?msu)(?<=[\s>])/:(?P<words>[^:/]+):/',
-            r'(?msu)(?<=[\s>])\|:(?P<words>[^:\|]+)?:\|',
+            r'(?msu)(?<=[\s>])\|:(?P<words>[^:\|]+):\|',
         ]
 
         for word in ITALICIZE_WORDS:

src/calibre/ebooks/metadata/__init__.py

@@ -271,6 +271,8 @@ def check_isbn13(isbn):
     return None
 
 def check_isbn(isbn):
+    if not isbn:
+        return None
     isbn = re.sub(r'[^0-9X]', '', isbn.upper())
     if len(isbn) == 10:
         return check_isbn10(isbn)

src/calibre/ebooks/metadata/amazon.py

@@ -7,6 +7,7 @@ __docformat__ = 'restructuredtext en'
 Fetch metadata using Amazon AWS
 '''
 import sys, re
+from threading import RLock
 
 from lxml import html
 from lxml.html import soupparser
@@ -17,6 +18,10 @@ from calibre.ebooks.metadata.book.base import Metadata
 from calibre.ebooks.chardet import xml_to_unicode
 from calibre.library.comments import sanitize_comments_html
 
+asin_cache = {}
+cover_url_cache = {}
+cache_lock = RLock()
+
 def find_asin(br, isbn):
     q = 'http://www.amazon.com/s?field-keywords='+isbn
     raw = br.open_novisit(q).read()
@@ -29,6 +34,12 @@ def find_asin(br, isbn):
         return revs[0]
 
 def to_asin(br, isbn):
+    with cache_lock:
+        ans = asin_cache.get(isbn, None)
+    if ans:
+        return ans
+    if ans is False:
+        return None
     if len(isbn) == 13:
         try:
             asin = find_asin(br, isbn)
@@ -38,8 +49,11 @@ def to_asin(br, isbn):
             asin = None
     else:
         asin = isbn
+    with cache_lock:
+        asin_cache[isbn] = ans if ans else False
     return asin
 
+
 def get_social_metadata(title, authors, publisher, isbn):
     mi = Metadata(title, authors)
     if not isbn:
@@ -58,6 +72,68 @@ def get_social_metadata(title, authors, publisher, isbn):
             return mi
     return mi
 
+def get_cover_url(isbn, br):
+    isbn = check_isbn(isbn)
+    if not isbn:
+        return None
+    with cache_lock:
+        ans = cover_url_cache.get(isbn, None)
+    if ans:
+        return ans
+    if ans is False:
+        return None
+    asin = to_asin(br, isbn)
+    if asin:
+        ans = _get_cover_url(br, asin)
+        if ans:
+            with cache_lock:
+                cover_url_cache[isbn] = ans
+            return ans
+    from calibre.ebooks.metadata.xisbn import xisbn
+    for i in xisbn.get_associated_isbns(isbn):
+        asin = to_asin(br, i)
+        if asin:
+            ans = _get_cover_url(br, asin)
+            if ans:
+                with cache_lock:
+                    cover_url_cache[isbn] = ans
+                    cover_url_cache[i] = ans
+                return ans
+    with cache_lock:
+        cover_url_cache[isbn] = False
+    return None
+
+def _get_cover_url(br, asin):
+    q = 'http://amzn.com/'+asin
+    try:
+        raw = br.open_novisit(q).read()
+    except Exception, e:
+        if callable(getattr(e, 'getcode', None)) and \
+                e.getcode() == 404:
+            return None
+        raise
+    if '<title>404 - ' in raw:
+        return None
+    raw = xml_to_unicode(raw, strip_encoding_pats=True,
+            resolve_entities=True)[0]
+    try:
+        root = soupparser.fromstring(raw)
+    except:
+        return False
+
+    imgs = root.xpath('//img[@id="prodImage" and @src]')
+    if imgs:
+        src = imgs[0].get('src')
+        parts = src.split('/')
+        if len(parts) > 3:
+            bn = parts[-1]
+            sparts = bn.split('_')
+            if len(sparts) > 2:
+                bn = sparts[0] + sparts[-1]
+                return ('/'.join(parts[:-1]))+'/'+bn
+    return None
+
+
 def get_metadata(br, asin, mi):
     q = 'http://amzn.com/'+asin
     try:
@@ -111,18 +187,25 @@ def get_metadata(br, asin, mi):
 
 
 def main(args=sys.argv):
-    # Test xisbn
+    import tempfile, os
-    print get_social_metadata('Learning Python', None, None, '8324616489')
+    tdir = tempfile.gettempdir()
-    print
+    br = browser()
+    for title, isbn in [
+            ('Learning Python', '8324616489'), # Test xisbn
+            ('Angels & Demons', '9781416580829'), # Test sophisticated comment formatting
+            # Random tests
+            ('Star Trek: Destiny: Mere Mortals', '9781416551720'),
+            ('The Great Gatsby', '0743273567'),
+            ]:
+        cpath = os.path.join(tdir, title+'.jpg')
+        curl = get_cover_url(isbn, br)
+        if curl is None:
+            print 'No cover found for', title
+        else:
+            open(cpath, 'wb').write(br.open_novisit(curl).read())
+            print 'Cover for', title, 'saved to', cpath
 
-    # Test sophisticated comment formatting
+        print get_social_metadata(title, None, None, isbn)
-    print get_social_metadata('Angels & Demons', None, None, '9781416580829')
-    print
-
-    # Random tests
-    print get_social_metadata('Star Trek: Destiny: Mere Mortals', None, None, '9781416551720')
-    print
-    print get_social_metadata('The Great Gatsby', None, None, '0743273567')
 
     return 0
 

src/calibre/ebooks/metadata/covers.py

@@ -5,7 +5,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import traceback, socket, re, sys
+import traceback, socket, sys
 from functools import partial
 from threading import Thread, Event
 from Queue import Queue, Empty
@@ -15,7 +15,6 @@ import mechanize
 
 from calibre.customize import Plugin
 from calibre import browser, prints
-from calibre.ebooks.BeautifulSoup import BeautifulSoup
 from calibre.constants import preferred_encoding, DEBUG
 
 class CoverDownload(Plugin):
@@ -112,72 +111,38 @@ class OpenLibraryCovers(CoverDownload): # {{{
 
 # }}}
 
-class LibraryThingCovers(CoverDownload): # {{{
+class AmazonCovers(CoverDownload): # {{{
 
-    name = 'librarything.com covers'
+    name = 'amazon.com covers'
-    description = _('Download covers from librarything.com')
+    description = _('Download covers from amazon.com')
     author = 'Kovid Goyal'
 
-    LIBRARYTHING = 'http://www.librarything.com/isbn/'
-
-    def get_cover_url(self, isbn, br, timeout=5.):
-
-        try:
-            src = br.open_novisit('http://www.librarything.com/isbn/'+isbn,
-                    timeout=timeout).read().decode('utf-8', 'replace')
-        except Exception, err:
-            if isinstance(getattr(err, 'args', [None])[0], socket.timeout):
-                err = Exception(_('LibraryThing.com timed out. Try again later.'))
-            raise err
-        else:
-            if '/wiki/index.php/HelpThing:Verify' in src:
-                raise Exception('LibraryThing is blocking calibre.')
-            s = BeautifulSoup(src)
-            url = s.find('td', attrs={'class':'left'})
-            if url is None:
-                if s.find('div', attrs={'class':'highloadwarning'}) is not None:
-                    raise Exception(_('Could not fetch cover as server is experiencing high load. Please try again later.'))
-                raise Exception(_('ISBN: %s not found')%isbn)
-            url = url.find('img')
-            if url is None:
-                raise Exception(_('LibraryThing.com server error. Try again later.'))
-            url = re.sub(r'_S[XY]\d+', '', url['src'])
-            return url
 
     def has_cover(self, mi, ans, timeout=5.):
-        if not mi.isbn or not self.site_customization:
+        if not mi.isbn:
             return False
-        from calibre.ebooks.metadata.library_thing import get_browser, login
+        from calibre.ebooks.metadata.amazon import get_cover_url
-        br = get_browser()
+        br = browser()
-        un, _, pw = self.site_customization.partition(':')
-        login(br, un, pw)
         try:
-            self.get_cover_url(mi.isbn, br, timeout=timeout)
+            get_cover_url(mi.isbn, br)
             self.debug('cover for', mi.isbn, 'found')
             ans.set()
         except Exception, e:
             self.debug(e)
 
     def get_covers(self, mi, result_queue, abort, timeout=5.):
-        if not mi.isbn or not self.site_customization:
+        if not mi.isbn:
             return
-        from calibre.ebooks.metadata.library_thing import get_browser, login
+        from calibre.ebooks.metadata.amazon import get_cover_url
-        br = get_browser()
+        br = browser()
-        un, _, pw = self.site_customization.partition(':')
-        login(br, un, pw)
         try:
-            url = self.get_cover_url(mi.isbn, br, timeout=timeout)
+            url = get_cover_url(mi.isbn, br)
             cover_data = br.open_novisit(url).read()
             result_queue.put((True, cover_data, 'jpg', self.name))
         except Exception, e:
             result_queue.put((False, self.exception_to_string(e),
                 traceback.format_exc(), self.name))
 
-    def customization_help(self, gui=False):
-        ans = _('To use librarything.com you must sign up for a %sfree account%s '
-                'and enter your username and password separated by a : below.')
-        return '<p>'+ans%('<a href="http://www.librarything.com">', '</a>')
-
 # }}}
 
 def check_for_cover(mi, timeout=5.): # {{{

src/calibre/ebooks/mobi/mobiml.py

@@ -367,6 +367,9 @@ class MobiMLizer(object):
             istate.attrib['src'] = elem.attrib['src']
             istate.attrib['align'] = 'baseline'
             cssdict = style.cssdict()
+            valign = cssdict.get('vertical-align', None)
+            if valign in ('top', 'bottom', 'middle'):
+                istate.attrib['align'] = valign
             for prop in ('width', 'height'):
                 if cssdict[prop] != 'auto':
                     value = style[prop]
@@ -451,8 +454,11 @@ class MobiMLizer(object):
                 text = COLLAPSE.sub(' ', elem.text)
         valign = style['vertical-align']
         not_baseline = valign in ('super', 'sub', 'text-top',
-                'text-bottom')
+                'text-bottom') or (
-        vtag = 'sup' if valign in ('super', 'text-top') else 'sub'
+                isinstance(valign, (float, int)) and abs(valign) != 0)
+        issup = valign in ('super', 'text-top') or (
+            isinstance(valign, (float, int)) and valign > 0)
+        vtag = 'sup' if issup else 'sub'
         if not_baseline and not ignore_valign and tag not in NOT_VTAGS and not isblock:
             nroot = etree.Element(XHTML('html'), nsmap=MOBI_NSMAP)
             vbstate = BlockState(etree.SubElement(nroot, XHTML('body')))

src/calibre/ebooks/oeb/transforms/flatcss.py

@@ -207,7 +207,14 @@ class CSSFlattener(object):
             font_size = self.sbase if self.sbase is not None else \
                 self.context.source.fbase
         if 'align' in node.attrib:
-            cssdict['text-align'] = node.attrib['align']
+            if tag != 'img':
+                cssdict['text-align'] = node.attrib['align']
+            else:
+                val = node.attrib['align']
+                if val in ('middle', 'bottom', 'top'):
+                    cssdict['vertical-align'] = val
+                elif val in ('left', 'right'):
+                    cssdict['text-align'] = val
             del node.attrib['align']
         if node.tag == XHTML('font'):
             node.tag = XHTML('span')

src/calibre/ebooks/txt/input.py

@@ -4,10 +4,9 @@ __license__ = 'GPL 3'
 __copyright__ = '2009, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
-import glob
 import os
 
-from calibre import _ent_pat, xml_entity_to_unicode
+from calibre import _ent_pat, walk, xml_entity_to_unicode
 from calibre.customize.conversion import InputFormatPlugin, OptionRecommendation
 from calibre.ebooks.conversion.preprocess import DocAnalysis, Dehyphenator
 from calibre.ebooks.chardet import detect
@@ -16,7 +15,6 @@ from calibre.ebooks.txt.processor import convert_basic, convert_markdown, \
     preserve_spaces, detect_paragraph_type, detect_formatting_type, \
     normalize_line_endings, convert_textile, remove_indents, block_to_single_line, \
     separate_hard_scene_breaks
-from calibre.ptempfile import TemporaryDirectory
 from calibre.utils.zipfile import ZipFile
 
 class TXTInput(InputFormatPlugin):
@@ -28,20 +26,23 @@ class TXTInput(InputFormatPlugin):
 
     options = set([
         OptionRecommendation(name='paragraph_type', recommended_value='auto',
-            choices=['auto', 'block', 'single', 'print', 'unformatted'],
+            choices=['auto', 'block', 'single', 'print', 'unformatted', 'off'],
             help=_('Paragraph structure.\n'
-                   'choices are [\'auto\', \'block\', \'single\', \'print\', \'unformatted\']\n'
+                   'choices are [\'auto\', \'block\', \'single\', \'print\', \'unformatted\', \'off\']\n'
                    '* auto: Try to auto detect paragraph type.\n'
                    '* block: Treat a blank line as a paragraph break.\n'
                    '* single: Assume every line is a paragraph.\n'
                    '* print:  Assume every line starting with 2+ spaces or a tab '
-                   'starts a paragraph.'
+                   'starts a paragraph.\n'
-                   '* unformatted: Most lines have hard line breaks, few/no blank lines or indents.')),
+                   '* unformatted: Most lines have hard line breaks, few/no blank lines or indents. '
+                   'Tries to determine structure and reformat the differentiate elements.\n'
+                   '* off: Don\'t modify the paragraph structure. This is useful when combined with '
+                   'Markdown or Textile formatting to ensure no formatting is lost.')),
         OptionRecommendation(name='formatting_type', recommended_value='auto',
-            choices=['auto', 'none', 'heuristic', 'textile', 'markdown'],
+            choices=['auto', 'plain', 'heuristic', 'textile', 'markdown'],
             help=_('Formatting used within the document.'
                    '* auto: Automatically decide which formatting processor to use.\n'
-                   '* none: Do not process the document formatting. Everything is a '
+                   '* plain: Do not process the document formatting. Everything is a '
                    'paragraph and no styling is applied.\n'
                    '* heuristic: Process using heuristics to determine formatting such '
                    'as chapter headings and italic text.\n'
@@ -64,18 +65,17 @@ class TXTInput(InputFormatPlugin):
         txt = ''
         log.debug('Reading text from file...')
         length = 0
+        # [(u'path', mime),]
 
         # Extract content from zip archive.
         if file_ext == 'txtz':
-            log.debug('De-compressing content to temporary directory...')
+            zf = ZipFile(stream)
-            with TemporaryDirectory('_untxtz') as tdir:
+            zf.extractall('.')
-                zf = ZipFile(stream)
-                zf.extractall(tdir)
 
-                txts = glob.glob(os.path.join(tdir, '*.txt'))
+            for x in walk('.'):
-                for t in txts:
+                if os.path.splitext(x)[1].lower() == '.txt':
-                    with open(t, 'rb') as tf:
+                    with open(x, 'rb') as tf:
-                        txt += tf.read()
+                        txt += tf.read() + '\n\n'
         else:
             txt = stream.read()
 
@@ -134,7 +134,7 @@ class TXTInput(InputFormatPlugin):
             preprocessor = HeuristicProcessor(options, log=getattr(self, 'log', None))
             txt = preprocessor.punctuation_unwrap(length, txt, 'txt')
             txt = separate_paragraphs_single_line(txt)
-        else:
+        elif options.paragraph_type == 'block':
             txt = separate_hard_scene_breaks(txt)
             txt = block_to_single_line(txt)
 
@@ -178,7 +178,7 @@ class TXTInput(InputFormatPlugin):
             setattr(options, opt.option.name, opt.recommended_value)
         options.input_encoding = 'utf-8'
         base = os.getcwdu()
-        if hasattr(stream, 'name'):
+        if file_ext != 'txtz' and hasattr(stream, 'name'):
             base = os.path.dirname(stream.name)
         fname = os.path.join(base, 'index.html')
         c = 0
@@ -190,16 +190,16 @@ class TXTInput(InputFormatPlugin):
             htmlfile.write(html.encode('utf-8'))
         odi = options.debug_pipeline
         options.debug_pipeline = None
-        # Generate oeb from htl conversion.
+        # Generate oeb from html conversion.
         oeb = html_input.convert(open(htmlfile.name, 'rb'), options, 'html', log,
                 {})
         options.debug_pipeline = odi
         os.remove(htmlfile.name)
-        
+
         # Set metadata from file.
         from calibre.customize.ui import get_file_type_metadata
         from calibre.ebooks.oeb.transforms.metadata import meta_info_to_oeb_metadata
         mi = get_file_type_metadata(stream, file_ext)
         meta_info_to_oeb_metadata(mi, oeb.metadata, log)
-        
+
         return oeb

src/calibre/ebooks/txt/processor.py

@@ -126,7 +126,7 @@ def separate_hard_scene_breaks(txt):
             return '\n%s\n' % line
         else:
             return line
-    txt = re.sub(u'(?miu)^[ \t-=~\/]+$', lambda mo: sep_break(mo.group()), txt)
+    txt = re.sub(u'(?miu)^[ \t-=~\/_]+$', lambda mo: sep_break(mo.group()), txt)
     return txt
 
 def block_to_single_line(txt):

src/calibre/gui2/actions/add.py

@@ -204,7 +204,8 @@ class AddAction(InterfaceAction):
                         ]
         to_device = self.gui.stack.currentIndex() != 0
         if to_device:
-            filters = [(_('Supported books'), self.gui.device_manager.device.FORMATS)]
+            fmts = self.gui.device_manager.device.settings().format_map
+            filters = [(_('Supported books'), fmts)]
 
         books = choose_files(self.gui, 'add books dialog dir', 'Select books',
                              filters=filters)

src/calibre/gui2/complete.py

@@ -158,6 +158,8 @@ class MultiCompleteComboBox(EnComboBox):
         # item that matches case insensitively
         c = self.lineEdit().completer()
         c.setCaseSensitivity(Qt.CaseSensitive)
+        self.dummy_model = CompleteModel(self)
+        c.setModel(self.dummy_model)
 
     def update_items_cache(self, complete_items):
         self.lineEdit().update_items_cache(complete_items)

src/calibre/gui2/device_drivers/configwidget.py

@@ -9,15 +9,16 @@ import textwrap
 from PyQt4.Qt import QWidget, QListWidgetItem, Qt, QVariant, SIGNAL, \
                      QLabel, QLineEdit, QCheckBox
 
-from calibre.gui2 import error_dialog
+from calibre.gui2 import error_dialog, question_dialog
 from calibre.gui2.device_drivers.configwidget_ui import Ui_ConfigWidget
 from calibre.utils.formatter import validation_formatter
+from calibre.ebooks import BOOK_EXTENSIONS
 
 class ConfigWidget(QWidget, Ui_ConfigWidget):
 
     def __init__(self, settings, all_formats, supports_subdirs,
         must_read_metadata, supports_use_author_sort,
-        extra_customization_message):
+        extra_customization_message, device):
 
         QWidget.__init__(self)
         Ui_ConfigWidget.__init__(self)
@@ -25,9 +26,15 @@ class ConfigWidget(QWidget, Ui_ConfigWidget):
 
         self.settings = settings
 
+        all_formats = set(all_formats)
+        self.calibre_known_formats = device.FORMATS
+        self.device_name = device.get_gui_name()
+        if device.USER_CAN_ADD_NEW_FORMATS:
+            all_formats = set(all_formats) | set(BOOK_EXTENSIONS)
+
         format_map = settings.format_map
         disabled_formats = list(set(all_formats).difference(format_map))
-        for format in format_map + disabled_formats:
+        for format in format_map + list(sorted(disabled_formats)):
             item = QListWidgetItem(format, self.columns)
             item.setData(Qt.UserRole, QVariant(format))
             item.setFlags(Qt.ItemIsEnabled|Qt.ItemIsUserCheckable|Qt.ItemIsSelectable)
@@ -110,6 +117,18 @@ class ConfigWidget(QWidget, Ui_ConfigWidget):
         return self.opt_use_author_sort.isChecked()
 
     def validate(self):
+        formats = set(self.format_map())
+        extra = formats - set(self.calibre_known_formats)
+        if extra:
+            fmts = sorted([x.upper() for x in extra])
+            if not question_dialog(self, _('Unknown formats'),
+                    _('You have enabled the <b>{0}</b> formats for'
+                        ' your {1}. The {1} may not support them.'
+                        ' If you send these formats to your {1} they '
+                        'may not work. Are you sure?').format(
+                            (', '.join(fmts)), self.device_name)):
+                return False
+
         tmpl = unicode(self.opt_save_template.text())
         try:
             validation_formatter.validate(tmpl)

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -912,6 +912,7 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
 
     def series_changed(self, *args):
         self.write_series = True
+        self.autonumber_series.setEnabled(True)
 
     def s_r_remove_query(self, *args):
         if self.query_field.currentIndex() == 0:

src/calibre/gui2/dialogs/metadata_bulk.ui

@@ -303,6 +303,9 @@
             <layout class="QHBoxLayout" name="HLayout_3">
              <item>
               <widget class="QCheckBox" name="autonumber_series">
+               <property name="enabled">
+                <bool>false</bool>
+               </property>
                <property name="toolTip">
                 <string>If not checked, the series number for the books will be set to 1.
 If checked, selected books will be automatically numbered, in the order
@@ -1006,8 +1009,8 @@ not multiple and the destination field is multiple</string>
                <rect>
                 <x>0</x>
                 <y>0</y>
-                <width>938</width>
+                <width>197</width>
-                <height>268</height>
+                <height>60</height>
                </rect>
               </property>
               <layout class="QGridLayout" name="testgrid">

src/calibre/gui2/preferences/search.py

@@ -26,12 +26,19 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         r('limit_search_columns_to', prefs, setting=CommaSeparatedList)
         fl = gui.library_view.model().db.field_metadata.get_search_terms()
         self.opt_limit_search_columns_to.update_items_cache(fl)
+        self.clear_history_button.clicked.connect(self.clear_histories)
 
     def refresh_gui(self, gui):
         gui.search.search_as_you_type(config['search_as_you_type'])
         gui.library_view.model().set_highlight_only(config['highlight_search_matches'])
         gui.search.do_search()
 
+    def clear_histories(self, *args):
+        for key, val in config.defaults.iteritems():
+            if key.endswith('_search_history') and isinstance(val, list):
+                config[key] = []
+        self.gui.search.clear_history()
+
 if __name__ == '__main__':
     app = QApplication([])
     test_widget('Interface', 'Search')

src/calibre/gui2/preferences/search.ui

@@ -77,7 +77,7 @@
      </layout>
     </widget>
    </item>
-   <item row="3" column="0">
+   <item row="4" column="0">
     <spacer name="verticalSpacer">
      <property name="orientation">
       <enum>Qt::Vertical</enum>
@@ -90,13 +90,23 @@
      </property>
     </spacer>
    </item>
+   <item row="3" column="0">
+    <widget class="QPushButton" name="clear_history_button">
+     <property name="toolTip">
+      <string>Clear search histories from all over calibre. Including the book list, e-book viewer, fetch news dialog, etc.</string>
+     </property>
+     <property name="text">
+      <string>Clear search &amp;histories</string>
+     </property>
+    </widget>
+   </item>
   </layout>
  </widget>
  <customwidgets>
   <customwidget>
    <class>MultiCompleteLineEdit</class>
    <extends>QLineEdit</extends>
-   <header>calibre/gui2.complete.h</header>
+   <header>calibre/gui2/complete.h</header>
   </customwidget>
  </customwidgets>
  <resources/>

src/calibre/gui2/search_box.py

@@ -114,6 +114,9 @@ class SearchBox2(QComboBox): # {{{
     def text(self):
         return self.currentText()
 
+    def clear_history(self, *args):
+        QComboBox.clear(self)
+
     def clear(self, emit_search=True):
         self.normalize_state()
         self.setEditText('')

src/calibre/gui2/viewer/main.py

@@ -17,16 +17,16 @@ from calibre.gui2.viewer.bookmarkmanager import BookmarkManager
 from calibre.gui2.widgets import ProgressIndicator
 from calibre.gui2.main_window import MainWindow
 from calibre.gui2 import Application, ORG_NAME, APP_UID, choose_files, \
-                         info_dialog, error_dialog, open_url, available_height
+    info_dialog, error_dialog, open_url, available_height, gprefs
 from calibre.ebooks.oeb.iterator import EbookIterator
 from calibre.ebooks import DRMError
-from calibre.constants import islinux, isfreebsd, isosx
+from calibre.constants import islinux, isfreebsd, isosx, filesystem_encoding
 from calibre.utils.config import Config, StringConfig, dynamic
 from calibre.gui2.search_box import SearchBox2
 from calibre.ebooks.metadata import MetaInformation
 from calibre.customize.ui import available_input_formats
 from calibre.gui2.viewer.dictionary import Lookup
-from calibre import as_unicode
+from calibre import as_unicode, force_unicode, isbytestring
 
 class TOCItem(QStandardItem):
 
@@ -160,6 +160,12 @@ class HelpfulLineEdit(QLineEdit):
         self.setPalette(self.gray)
         self.setText(self.HELP_TEXT)
 
+class RecentAction(QAction):
+
+    def __init__(self, path, parent):
+        self.path = path
+        QAction.__init__(self, os.path.basename(path), parent)
+
 class EbookViewer(MainWindow, Ui_EbookViewer):
 
     STATE_VERSION = 1
@@ -284,8 +290,26 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
         ca = self.view.copy_action
         ca.setShortcut(QKeySequence.Copy)
         self.addAction(ca)
+        self.open_history_menu = QMenu()
+        self.build_recent_menu()
+        self.action_open_ebook.setMenu(self.open_history_menu)
+        self.open_history_menu.triggered[QAction].connect(self.open_recent)
+        w = self.tool_bar.widgetForAction(self.action_open_ebook)
+        w.setPopupMode(QToolButton.MenuButtonPopup)
+
         self.restore_state()
 
+    def build_recent_menu(self):
+        m = self.open_history_menu
+        m.clear()
+        count = 0
+        for path in gprefs.get('viewer_open_history', []):
+            if count > 9:
+                break
+            if os.path.exists(path):
+                m.addAction(RecentAction(path, m))
+                count += 1
+
     def closeEvent(self, e):
         self.save_state()
         return MainWindow.closeEvent(self, e)
@@ -425,6 +449,9 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
         if files:
             self.load_ebook(files[0])
 
+    def open_recent(self, action):
+        self.load_ebook(action.path)
+
     def font_size_larger(self, checked):
         frac = self.view.magnify_fonts()
         self.action_font_size_larger.setEnabled(self.view.multiplier() < 3)
@@ -647,6 +674,17 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
                     self.action_table_of_contents.setChecked(True)
             else:
                 self.action_table_of_contents.setChecked(False)
+            if isbytestring(pathtoebook):
+                pathtoebook = force_unicode(pathtoebook, filesystem_encoding)
+            vh = gprefs.get('viewer_open_history', [])
+            try:
+                vh.remove(pathtoebook)
+            except:
+                pass
+            vh.insert(0, pathtoebook)
+            gprefs.set('viewer_open_history', vh[:50])
+            self.build_recent_menu()
+
             self.action_table_of_contents.setDisabled(not self.iterator.toc)
             self.current_book_has_toc = bool(self.iterator.toc)
             self.current_title = title