Sync to trunk.

resources/recipes/apple_daily.recipe

@@ -0,0 +1,161 @@
+# -*- coding: utf-8 -*-
+import re
+from calibre.web.feeds.recipes import BasicNewsRecipe
+
+class AppleDaily(BasicNewsRecipe):
+
+    title       = u'蘋果日報'
+    __author__  = u'蘋果日報'
+    __publisher__  = u'蘋果日報'
+    description = u'蘋果日報'
+    masthead_url = 'http://hk.apple.nextmedia.com/template/common/header/2009/images/atnextheader_logo_appledaily.gif'
+    language = 'zh_TW'
+    encoding = 'UTF-8'
+    timefmt = ' [%a, %d %b, %Y]'
+    needs_subscription = False
+    remove_javascript = True
+    remove_tags_before = dict(name=['ul', 'h1'])
+    remove_tags_after  = dict(name='form')
+    remove_tags = [dict(attrs={'class':['articleTools', 'post-tools', 'side_tool', 'nextArticleLink clearfix']}),
+                dict(id=['footer', 'toolsRight', 'articleInline', 'navigation', 'archive', 'side_search', 'blog_sidebar', 'side_tool', 'side_index']),
+                dict(name=['script', 'noscript', 'style', 'form'])]
+    no_stylesheets = True
+    extra_css = '''
+    	@font-face {font-family: "uming", serif, sans-serif;  src: url(res:///usr/share/fonts/truetype/arphic/uming.ttc); }\n
+	    body {margin-right: 8pt; font-family: 'uming', serif;}
+        h1 {font-family: 'uming', serif, sans-serif}
+            '''
+    #extra_css = 'h1 {font: sans-serif large;}\n.byline {font:monospace;}'
+
+    preprocess_regexps = [
+       (re.compile(r'img.php?server=(?P<server>[^&]+)&path=(?P<path>[^&]+).*', re.DOTALL|re.IGNORECASE),
+        lambda match: 'http://' + match.group('server') + '/' + match.group('path')),
+    ]
+
+    def get_cover_url(self):
+        return 'http://hk.apple.nextmedia.com/template/common/header/2009/images/atnextheader_logo_appledaily.gif'
+
+
+    #def get_browser(self):
+        #br = BasicNewsRecipe.get_browser()
+        #if self.username is not None and self.password is not None:
+        #    br.open('http://www.nytimes.com/auth/login')
+        #    br.select_form(name='login')
+        #    br['USERID']   = self.username
+        #    br['PASSWORD'] = self.password
+        #    br.submit()
+        #return br
+
+    def preprocess_html(self, soup):
+        #process all the images
+        for tag in soup.findAll(lambda tag: tag.name.lower()=='img' and tag.has_key('src')):
+            iurl = tag['src']
+            #print 'checking image: ' + iurl
+
+            #img\.php?server\=(?P<server>[^&]+)&path=(?P<path>[^&]+)
+            p = re.compile(r'img\.php\?server=(?P<server>[^&]+)&path=(?P<path>[^&]+)', re.DOTALL|re.IGNORECASE)
+
+            m = p.search(iurl)
+
+            if m is not None:
+                iurl = 'http://' + m.group('server') + '/' + m.group('path')
+                #print 'working! new url: ' + iurl
+                tag['src'] = iurl
+            #else:
+                #print 'not good'
+
+        for tag in soup.findAll(lambda tag: tag.name.lower()=='a' and tag.has_key('href')):
+            iurl = tag['href']
+            #print 'checking image: ' + iurl
+
+            #img\.php?server\=(?P<server>[^&]+)&path=(?P<path>[^&]+)
+            p = re.compile(r'img\.php\?server=(?P<server>[^&]+)&path=(?P<path>[^&]+)', re.DOTALL|re.IGNORECASE)
+
+            m = p.search(iurl)
+
+            if m is not None:
+                iurl = 'http://' + m.group('server') + '/' + m.group('path')
+                #print 'working! new url: ' + iurl
+                tag['href'] = iurl
+            #else:
+                #print 'not good'
+
+        return soup
+
+
+    def parse_index(self):
+        base = 'http://news.hotpot.hk/fruit'
+        soup = self.index_to_soup('http://news.hotpot.hk/fruit/index.php')
+
+        #def feed_title(div):
+        #    return ''.join(div.findAll(text=True, recursive=False)).strip()
+
+        articles = {}
+        key = None
+        ans = []
+        for div in soup.findAll('li'):
+            key = div.find(text=True, recursive=True);
+            #if key == u'豪情':
+           #    continue;
+
+            print 'section=' + key
+
+            articles[key] = []
+
+            ans.append(key)
+
+            a = div.find('a', href=True)
+
+            if not a:
+                continue
+
+            url = base + '/' + a['href']
+            print 'url=' + url
+
+            if not articles.has_key(key):
+                articles[key] = []
+            else:
+                # sub page
+                subSoup = self.index_to_soup(url)
+
+                for subDiv in subSoup.findAll('li'):
+                    subA = subDiv.find('a', href=True)
+                    subTitle = subDiv.find(text=True, recursive=True)
+                    subUrl = base + '/' + subA['href']
+
+                    print 'subUrl' + subUrl
+
+                    articles[key].append(
+                        dict(title=subTitle,
+                         url=subUrl,
+                         date='',
+                         description='',
+                         content=''))
+
+
+#             elif div['class'] in ['story', 'story headline']:
+#                 a = div.find('a', href=True)
+#                 if not a:
+#                     continue
+#                 url = re.sub(r'\?.*', '', a['href'])
+#                 url += '?pagewanted=all'
+#                 title = self.tag_to_string(a, use_alt=True).strip()
+#                 description = ''
+#                 pubdate = strftime('%a, %d %b')
+#                 summary = div.find(True, attrs={'class':'summary'})
+#                 if summary:
+#                     description = self.tag_to_string(summary, use_alt=False)
+#
+#                 feed = key if key is not None else 'Uncategorized'
+#                 if not articles.has_key(feed):
+#                     articles[feed] = []
+#                 if not 'podcasts' in url:
+#                     articles[feed].append(
+#                               dict(title=title, url=url, date=pubdate,
+#                                    description=description,
+#                                    content=''))
+#        ans = self.sort_index_by(ans, {'The Front Page':-1, 'Dining In, Dining Out':1, 'Obituaries':2})
+        ans = [(unicode(key), articles[key]) for key in ans if articles.has_key(key)]
+        return ans
+
+

resources/recipes/nytimes_sub.recipe

@@ -668,7 +668,7 @@ class NYTimes(BasicNewsRecipe):
 
             try:
                 #remove "Related content" bar
-                runAroundsFound = soup.findAll('div',{'class':['articleInline runaroundLeft','articleInline doubleRule runaroundLeft','articleInline runaroundLeft firstArticleInline','articleInline runaroundLeft  ']})
+                runAroundsFound = soup.findAll('div',{'class':['articleInline runaroundLeft','articleInline doubleRule runaroundLeft','articleInline runaroundLeft firstArticleInline','articleInline runaroundLeft  ','articleInline runaroundLeft  lastArticleInline']})
                 if runAroundsFound:
                     for runAround in runAroundsFound:
                         #find all section headers

resources/recipes/workers_world.recipe

@@ -0,0 +1,26 @@
+from calibre.web.feeds.recipes import BasicNewsRecipe
+
+class WorkersWorld(BasicNewsRecipe):
+
+    title = u'Workers World'
+    description = u'Socialist news and analysis'
+    __author__ = u'urslnx'
+    no_stylesheets = True
+    use_embedded_content = False
+    remove_javascript = True
+    oldest_article = 7
+    max_articles_per_feed = 100
+    encoding = 'utf8'
+    publisher = 'workers.org'
+    category = 'news, politics, USA, world'
+    language = 'en'
+    publication_type = 'newsportal'
+    extra_css = ' body{ font-family: Verdana,Arial,Helvetica,sans-serif; } h1{ font-size: x-large; text-align: left; margin-top:0.5em; margin-bottom:0.25em; } h2{ font-size: large; }  p{ text-align: left; } .published{ font-size: small; } .byline{ font-size: small; } .copyright{ font-size: small; } '
+    remove_tags_before = dict(name='div', attrs={'id':'evernote'})
+    remove_tags_after = dict(name='div', attrs={'id':'footer'})
+
+    masthead_url='http://www.workers.org/graphics/wwlogo300.gif'
+    cover_url = 'http://www.workers.org/pdf/current.jpg'
+    feeds = [(u'Headlines', u'http://www.workers.org/rss/nonstandard_rss.xml'),
+]
+

src/calibre/customize/__init__.py

@@ -90,6 +90,11 @@ class Plugin(object): # {{{
         an optional method validate() that takes no arguments and is called
         immediately after the user clicks OK. Changes are applied if and only
         if the method returns True.
+
+        If for some reason you cannot perform the configuration at this time,
+        return a tuple of two strings (message, details), these will be
+        displayed as a warning dialog to the user and the process will be
+        aborted.
         '''
         raise NotImplementedError()
 
@@ -133,6 +138,12 @@ class Plugin(object): # {{{
         except NotImplementedError:
             config_widget = None
 
+        if isinstance(config_widget, tuple):
+            from calibre.gui2 import warning_dialog
+            warning_dialog(parent, _('Cannot configure'), config_widget[0],
+                    det_msg=config_widget[1], show=True)
+            return False
+
         if config_widget is not None:
             v.addWidget(config_widget)
             v.addWidget(button_box)

src/calibre/customize/builtins.py

@@ -511,14 +511,14 @@ from calibre.ebooks.metadata.fetch import GoogleBooks, ISBNDB, Amazon, \
 from calibre.ebooks.metadata.douban import DoubanBooks
 from calibre.ebooks.metadata.nicebooks import NiceBooks, NiceBooksCovers
 from calibre.ebooks.metadata.covers import OpenLibraryCovers, \
-        LibraryThingCovers, DoubanCovers
+        AmazonCovers, DoubanCovers
 from calibre.library.catalog import CSV_XML, EPUB_MOBI, BIBTEX
 from calibre.ebooks.epub.fix.unmanifested import Unmanifested
 from calibre.ebooks.epub.fix.epubcheck import Epubcheck
 
 plugins = [HTML2ZIP, PML2PMLZ, ArchiveExtract, GoogleBooks, ISBNDB, Amazon,
         KentDistrictLibrary, DoubanBooks, NiceBooks, CSV_XML, EPUB_MOBI, BIBTEX, Unmanifested,
-        Epubcheck, OpenLibraryCovers, LibraryThingCovers, DoubanCovers,
+        Epubcheck, OpenLibraryCovers, AmazonCovers, DoubanCovers,
         NiceBooksCovers]
 plugins += [
     ComicInput,

src/calibre/devices/android/driver.py

@@ -19,7 +19,7 @@ class ANDROID(USBMS):
 
     VENDOR_ID   = {
             # HTC
-            0x0bb4 : { 0x0c02 : [0x100, 0x0227, 0x0226],
+            0x0bb4 : { 0x0c02 : [0x100, 0x0227, 0x0226, 0x222],
                        0x0c01 : [0x100, 0x0227, 0x0226],
                        0x0ff9 : [0x0100, 0x0227, 0x0226],
                        0x0c87 : [0x0100, 0x0227, 0x0226],

src/calibre/devices/apple/driver.py

@@ -39,6 +39,7 @@ if iswindows:
 class DriverBase(DeviceConfig, DevicePlugin):
     # Needed for config_widget to work
     FORMATS = ['epub', 'pdf']
+    USER_CAN_ADD_NEW_FORMATS = False
     SUPPORTS_SUB_DIRS = True   # To enable second checkbox in customize widget
 
     @classmethod

src/calibre/devices/bambook/driver.py

@@ -32,6 +32,7 @@ class BAMBOOK(DeviceConfig, DevicePlugin):
     ip = None
 
     FORMATS = [ "snb" ]
+    USER_CAN_ADD_NEW_FORMATS = False
     VENDOR_ID = 0x230b
     PRODUCT_ID = 0x0001
     BCD = None
@@ -421,7 +422,7 @@ class BAMBOOK(DeviceConfig, DevicePlugin):
         from calibre.gui2.device_drivers.configwidget import ConfigWidget
         cw = ConfigWidget(cls.settings(), cls.FORMATS, cls.SUPPORTS_SUB_DIRS,
             cls.MUST_READ_METADATA, cls.SUPPORTS_USE_AUTHOR_SORT,
-            cls.EXTRA_CUSTOMIZATION_MESSAGE)
+            cls.EXTRA_CUSTOMIZATION_MESSAGE, cls)
         # Turn off the Save template
         cw.opt_save_template.setVisible(False)
         cw.label.setVisible(False)

src/calibre/devices/jetbook/driver.py

@@ -93,11 +93,11 @@ class MIBUK(USBMS):
 
     VENDOR_ID   = [0x0525]
     PRODUCT_ID  = [0xa4a5]
-    BCD         = [0x314]
+    BCD         = [0x314, 0x319]
     SUPPORTS_SUB_DIRS = True
 
-    VENDOR_NAME      = 'LINUX'
+    VENDOR_NAME      = ['LINUX', 'FILE_BAC']
-    WINDOWS_MAIN_MEM = 'WOLDERMIBUK'
+    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['WOLDERMIBUK', 'KED_STORAGE_GADG']
 
 class JETBOOK_MINI(USBMS):
 

src/calibre/devices/kindle/apnx.py

@@ -11,7 +11,6 @@ Generates and writes an APNX page mapping file.
 import struct
 import uuid
 
-from calibre.ebooks import DRMError
 from calibre.ebooks.mobi.reader import MobiReader
 from calibre.ebooks.pdb.header import PdbHeaderReader
 from calibre.utils.logging import default_log
@@ -40,10 +39,10 @@ class APNXBuilder(object):
                 pages = self.get_pages_fast(mobi_file_path)
         else:
             pages = self.get_pages_fast(mobi_file_path)
-        
+
         if not pages:
             raise Exception(_('Could not generate page mapping.'))
-        
+
         # Generate the APNX file from the page mapping.
         apnx = self.generate_apnx(pages)
 
@@ -83,12 +82,12 @@ class APNXBuilder(object):
         2300 characters of uncompressed text per page. This is
         not meant to map 1 to 1 to a print book but to be a
         close enough measure.
-        
+
         A test book was chosen and the characters were counted
         on one page. This number was round to 2240 then 60
         characters of markup were added to the total giving
         2300.
-        
+
         Uncompressed text length is used because it's easily
         accessible in MOBI files (part of the header). Also,
         It's faster to work off of the length then to
@@ -97,7 +96,7 @@ class APNXBuilder(object):
         text_length = 0
         pages = []
         count = 0
-        
+
         with open(mobi_file_path, 'rb') as mf:
             phead = PdbHeaderReader(mf)
             r0 = phead.section_data(0)
@@ -108,40 +107,41 @@ class APNXBuilder(object):
             count += 2300
 
         return pages
-    
+
     def get_pages_accurate(self, mobi_file_path):
         '''
         A more accurate but much more resource intensive and slower
         method to calculate the page length.
-        
+
         Parses the uncompressed text. In an average paper back book
         There are 32 lines per page and a maximum of 70 characters
         per line.
-        
+
         Each paragraph starts a new line and every 70 characters
         (minus markup) in a paragraph starts a new line. The
         position after every 30 lines will be marked as a new
         page.
-        
+
         This can be make more accurate by accounting for
         <div class="mbp_pagebreak" /> as a new page marker.
         And <br> elements as an empty line.
         '''
         pages = []
-        
+
         # Get the MOBI html.
         mr = MobiReader(mobi_file_path, default_log)
         if mr.book_header.encryption_type != 0:
-            raise DRMError()
+            # DRMed book
+            return self.get_pages_fast(mobi_file_path)
         mr.extract_text()
-        
+
         # States
         in_tag = False
         in_p = False
         check_p = False
         closing = False
         p_char_count = 0
-        
+
         # Get positions of every line
         # A line is either a paragraph starting
         # or every 70 characters in a paragraph.
@@ -158,7 +158,7 @@ class APNXBuilder(object):
         # the position within the stream.
         for c in mr.mobi_html.lower():
             pos += 1
-            
+
             # Check if we are starting or stopping a p tag.
             if check_p:
                 if c == '/':
@@ -173,7 +173,7 @@ class APNXBuilder(object):
                 check_p = False
                 closing = False
                 continue
-            
+
             if c == '<':
                 in_tag = True
                 check_p = True
@@ -188,7 +188,7 @@ class APNXBuilder(object):
                 if p_char_count == 70:
                     lines.append(pos)
                     p_char_count = 0
-        
+
         # Every 30 lines is a new page
         for i in xrange(0, len(lines), 32):
             pages.append(lines[i])

src/calibre/devices/kindle/driver.py

@@ -175,24 +175,25 @@ class KINDLE2(KINDLE):
 
     PRODUCT_ID = [0x0002, 0x0004]
     BCD        = [0x0100]
-    
+
     EXTRA_CUSTOMIZATION_MESSAGE = [
-        _('Write page mapping (APNX) file when sending books') +
+        _('Send page number information when sending books') +
             ':::' +
-            _('The APNX page mapping file is a new feature in the Kindle 3\'s '
+            _('The Kindle 3 and newer versions can use page number information '
-              '3.1 firmware. It allows for page numbers to that correspond to pages '
+              'in MOBI files. With this option, calibre will calculate and send'
-              'in a print book. This will write an APNX file that uses pseudo page '
+              ' this information to the Kindle when uploading MOBI files by'
-              'numbers based on the the average page length in a paper back book.'),
+              ' USB. Note that the page numbers do not correspond to any paper'
-        _('Use slower but more accurate APNX generation') +
+              ' book.'),
+        _('Use slower but more accurate page number generation') +
             ':::' +
-            _('There are two ways to generate the APNX file. Using the more accurate '
+            _('There are two ways to generate the page number information. Using the more accurate '
               'generator will produce pages that correspond better to a printed book. '
-              'However, this method is slower and more intensive. Unchecking this '
+              'However, this method is slower and will slow down sending files '
-              'option will default to using the faster but less accurate generator.'),
+              'to the Kindle.'),
     ]
     EXTRA_CUSTOMIZATION_DEFAULT = [
         True,
-        True,
+        False,
     ]
     OPT_APNX           = 0
     OPT_APNX_ACCURATE  = 1
@@ -236,7 +237,7 @@ class KINDLE2(KINDLE):
         opts = self.settings()
         if not opts.extra_customization[self.OPT_APNX]:
             return
-        
+
         if os.path.splitext(filepath.lower())[1] not in ('.azw', '.mobi', '.prc'):
             return
 

src/calibre/devices/kobo/driver.py

@@ -98,7 +98,6 @@ class KOBO(USBMS):
 
         def update_booklist(prefix, path, title, authors, mime, date, ContentType, ImageID, readstatus, MimeType):
             changed = False
-            # if path_to_ext(path) in self.FORMATS:
             try:
                 lpath = path.partition(self.normalize_path(prefix))[2]
                 if lpath.startswith(os.sep):
@@ -220,7 +219,7 @@ class KOBO(USBMS):
         #    2) volume_shorcover
         #    2) content
 
-        debug_print('delete_via_sql: ContentID: ', ContentID, 'ContentType: ', ContentType) 
+        debug_print('delete_via_sql: ContentID: ', ContentID, 'ContentType: ', ContentType)
         connection = sqlite.connect(self.normalize_path(self._main_prefix + '.kobo/KoboReader.sqlite'))
         cursor = connection.cursor()
         t = (ContentID,)
@@ -532,7 +531,7 @@ class KOBO(USBMS):
                         if result is None:
                             datelastread = '1970-01-01T00:00:00'
                         else:
-                            datelastread = result[0] if result[0] is not None else '1970-01-01T00:00:00' 
+                            datelastread = result[0] if result[0] is not None else '1970-01-01T00:00:00'
 
                         t = (datelastread,ContentID,)
 

src/calibre/devices/usbms/device.py

@@ -232,16 +232,37 @@ class Device(DeviceConfig, DevicePlugin):
 
         time.sleep(5)
         drives = {}
+        seen = set()
+        prod_pat = re.compile(r'PROD_(.+?)&')
+        dup_prod_id = False
+
+        def check_for_dups(pnp_id):
+            try:
+                match = prod_pat.search(pnp_id)
+                if match is not None:
+                    prodid = match.group(1)
+                    if prodid in seen:
+                        return True
+                    else:
+                        seen.add(prodid)
+            except:
+                pass
+            return False
+
+
         for drive, pnp_id in win_pnp_drives().items():
             if self.windows_match_device(pnp_id, 'WINDOWS_CARD_A_MEM') and \
                     not drives.get('carda', False):
                 drives['carda'] = drive
+                dup_prod_id |= check_for_dups(pnp_id)
             elif self.windows_match_device(pnp_id, 'WINDOWS_CARD_B_MEM') and \
                     not drives.get('cardb', False):
                 drives['cardb'] = drive
+                dup_prod_id |= check_for_dups(pnp_id)
             elif self.windows_match_device(pnp_id, 'WINDOWS_MAIN_MEM') and \
                     not drives.get('main', False):
                 drives['main'] = drive
+                dup_prod_id |= check_for_dups(pnp_id)
 
             if 'main' in drives.keys() and 'carda' in drives.keys() and \
                     'cardb' in drives.keys():
@@ -263,7 +284,8 @@ class Device(DeviceConfig, DevicePlugin):
 
         # Sort drives by their PNP drive numbers if the CARD and MAIN
         # MEM strings are identical
-        if self.WINDOWS_MAIN_MEM in (self.WINDOWS_CARD_A_MEM,
+        if dup_prod_id or \
+                self.WINDOWS_MAIN_MEM in (self.WINDOWS_CARD_A_MEM,
                 self.WINDOWS_CARD_B_MEM) or \
                 self.WINDOWS_CARD_A_MEM == self.WINDOWS_CARD_B_MEM:
             letters = sorted(drives.values(), cmp=drivecmp)

src/calibre/devices/usbms/deviceconfig.py

@@ -34,6 +34,10 @@ class DeviceConfig(object):
     #: If None the default is used
     SAVE_TEMPLATE = None
 
+    #: If True the user can add new formats to the driver
+    USER_CAN_ADD_NEW_FORMATS = True
+
+
     @classmethod
     def _default_save_template(cls):
         from calibre.library.save_to_disk import config
@@ -73,7 +77,7 @@ class DeviceConfig(object):
         from calibre.gui2.device_drivers.configwidget import ConfigWidget
         cw = ConfigWidget(cls.settings(), cls.FORMATS, cls.SUPPORTS_SUB_DIRS,
             cls.MUST_READ_METADATA, cls.SUPPORTS_USE_AUTHOR_SORT,
-            cls.EXTRA_CUSTOMIZATION_MESSAGE)
+            cls.EXTRA_CUSTOMIZATION_MESSAGE, cls)
         return cw
 
     @classmethod

src/calibre/devices/usbms/driver.py

@@ -93,9 +93,11 @@ class USBMS(CLI, Device):
         for idx,b in enumerate(bl):
             bl_cache[b.lpath] = idx
 
+        all_formats = set(self.settings().format_map) | set(self.FORMATS)
+
         def update_booklist(filename, path, prefix):
             changed = False
-            if path_to_ext(filename) in self.FORMATS:
+            if path_to_ext(filename) in all_formats:
                 try:
                     lpath = os.path.join(path, filename).partition(self.normalize_path(prefix))[2]
                     if lpath.startswith(os.sep):

src/calibre/ebooks/metadata/__init__.py

@@ -271,6 +271,8 @@ def check_isbn13(isbn):
     return None
 
 def check_isbn(isbn):
+    if not isbn:
+        return None
     isbn = re.sub(r'[^0-9X]', '', isbn.upper())
     if len(isbn) == 10:
         return check_isbn10(isbn)

src/calibre/ebooks/metadata/amazon.py

@@ -7,6 +7,7 @@ __docformat__ = 'restructuredtext en'
 Fetch metadata using Amazon AWS
 '''
 import sys, re
+from threading import RLock
 
 from lxml import html
 from lxml.html import soupparser
@@ -17,6 +18,10 @@ from calibre.ebooks.metadata.book.base import Metadata
 from calibre.ebooks.chardet import xml_to_unicode
 from calibre.library.comments import sanitize_comments_html
 
+asin_cache = {}
+cover_url_cache = {}
+cache_lock = RLock()
+
 def find_asin(br, isbn):
     q = 'http://www.amazon.com/s?field-keywords='+isbn
     raw = br.open_novisit(q).read()
@@ -29,6 +34,12 @@ def find_asin(br, isbn):
         return revs[0]
 
 def to_asin(br, isbn):
+    with cache_lock:
+        ans = asin_cache.get(isbn, None)
+    if ans:
+        return ans
+    if ans is False:
+        return None
     if len(isbn) == 13:
         try:
             asin = find_asin(br, isbn)
@@ -38,8 +49,11 @@ def to_asin(br, isbn):
             asin = None
     else:
         asin = isbn
+    with cache_lock:
+        asin_cache[isbn] = ans if ans else False
     return asin
 
+
 def get_social_metadata(title, authors, publisher, isbn):
     mi = Metadata(title, authors)
     if not isbn:
@@ -58,6 +72,68 @@ def get_social_metadata(title, authors, publisher, isbn):
             return mi
     return mi
 
+def get_cover_url(isbn, br):
+    isbn = check_isbn(isbn)
+    if not isbn:
+        return None
+    with cache_lock:
+        ans = cover_url_cache.get(isbn, None)
+    if ans:
+        return ans
+    if ans is False:
+        return None
+    asin = to_asin(br, isbn)
+    if asin:
+        ans = _get_cover_url(br, asin)
+        if ans:
+            with cache_lock:
+                cover_url_cache[isbn] = ans
+            return ans
+    from calibre.ebooks.metadata.xisbn import xisbn
+    for i in xisbn.get_associated_isbns(isbn):
+        asin = to_asin(br, i)
+        if asin:
+            ans = _get_cover_url(br, asin)
+            if ans:
+                with cache_lock:
+                    cover_url_cache[isbn] = ans
+                    cover_url_cache[i] = ans
+                return ans
+    with cache_lock:
+        cover_url_cache[isbn] = False
+    return None
+
+def _get_cover_url(br, asin):
+    q = 'http://amzn.com/'+asin
+    try:
+        raw = br.open_novisit(q).read()
+    except Exception, e:
+        if callable(getattr(e, 'getcode', None)) and \
+                e.getcode() == 404:
+            return None
+        raise
+    if '<title>404 - ' in raw:
+        return None
+    raw = xml_to_unicode(raw, strip_encoding_pats=True,
+            resolve_entities=True)[0]
+    try:
+        root = soupparser.fromstring(raw)
+    except:
+        return False
+
+    imgs = root.xpath('//img[@id="prodImage" and @src]')
+    if imgs:
+        src = imgs[0].get('src')
+        parts = src.split('/')
+        if len(parts) > 3:
+            bn = parts[-1]
+            sparts = bn.split('_')
+            if len(sparts) > 2:
+                bn = sparts[0] + sparts[-1]
+                return ('/'.join(parts[:-1]))+'/'+bn
+    return None
+
+
 def get_metadata(br, asin, mi):
     q = 'http://amzn.com/'+asin
     try:
@@ -111,18 +187,25 @@ def get_metadata(br, asin, mi):
 
 
 def main(args=sys.argv):
-    # Test xisbn
+    import tempfile, os
-    print get_social_metadata('Learning Python', None, None, '8324616489')
+    tdir = tempfile.gettempdir()
-    print
+    br = browser()
+    for title, isbn in [
+            ('Learning Python', '8324616489'), # Test xisbn
+            ('Angels & Demons', '9781416580829'), # Test sophisticated comment formatting
+            # Random tests
+            ('Star Trek: Destiny: Mere Mortals', '9781416551720'),
+            ('The Great Gatsby', '0743273567'),
+            ]:
+        cpath = os.path.join(tdir, title+'.jpg')
+        curl = get_cover_url(isbn, br)
+        if curl is None:
+            print 'No cover found for', title
+        else:
+            open(cpath, 'wb').write(br.open_novisit(curl).read())
+            print 'Cover for', title, 'saved to', cpath
 
-    # Test sophisticated comment formatting
+        print get_social_metadata(title, None, None, isbn)
-    print get_social_metadata('Angels & Demons', None, None, '9781416580829')
-    print
-
-    # Random tests
-    print get_social_metadata('Star Trek: Destiny: Mere Mortals', None, None, '9781416551720')
-    print
-    print get_social_metadata('The Great Gatsby', None, None, '0743273567')
 
     return 0
 

src/calibre/ebooks/metadata/covers.py

@@ -5,7 +5,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import traceback, socket, re, sys
+import traceback, socket, sys
 from functools import partial
 from threading import Thread, Event
 from Queue import Queue, Empty
@@ -15,7 +15,6 @@ import mechanize
 
 from calibre.customize import Plugin
 from calibre import browser, prints
-from calibre.ebooks.BeautifulSoup import BeautifulSoup
 from calibre.constants import preferred_encoding, DEBUG
 
 class CoverDownload(Plugin):
@@ -112,72 +111,38 @@ class OpenLibraryCovers(CoverDownload): # {{{
 
 # }}}
 
-class LibraryThingCovers(CoverDownload): # {{{
+class AmazonCovers(CoverDownload): # {{{
 
-    name = 'librarything.com covers'
+    name = 'amazon.com covers'
-    description = _('Download covers from librarything.com')
+    description = _('Download covers from amazon.com')
     author = 'Kovid Goyal'
 
-    LIBRARYTHING = 'http://www.librarything.com/isbn/'
-
-    def get_cover_url(self, isbn, br, timeout=5.):
-
-        try:
-            src = br.open_novisit('http://www.librarything.com/isbn/'+isbn,
-                    timeout=timeout).read().decode('utf-8', 'replace')
-        except Exception, err:
-            if isinstance(getattr(err, 'args', [None])[0], socket.timeout):
-                err = Exception(_('LibraryThing.com timed out. Try again later.'))
-            raise err
-        else:
-            if '/wiki/index.php/HelpThing:Verify' in src:
-                raise Exception('LibraryThing is blocking calibre.')
-            s = BeautifulSoup(src)
-            url = s.find('td', attrs={'class':'left'})
-            if url is None:
-                if s.find('div', attrs={'class':'highloadwarning'}) is not None:
-                    raise Exception(_('Could not fetch cover as server is experiencing high load. Please try again later.'))
-                raise Exception(_('ISBN: %s not found')%isbn)
-            url = url.find('img')
-            if url is None:
-                raise Exception(_('LibraryThing.com server error. Try again later.'))
-            url = re.sub(r'_S[XY]\d+', '', url['src'])
-            return url
 
     def has_cover(self, mi, ans, timeout=5.):
-        if not mi.isbn or not self.site_customization:
+        if not mi.isbn:
             return False
-        from calibre.ebooks.metadata.library_thing import get_browser, login
+        from calibre.ebooks.metadata.amazon import get_cover_url
-        br = get_browser()
+        br = browser()
-        un, _, pw = self.site_customization.partition(':')
-        login(br, un, pw)
         try:
-            self.get_cover_url(mi.isbn, br, timeout=timeout)
+            get_cover_url(mi.isbn, br)
             self.debug('cover for', mi.isbn, 'found')
             ans.set()
         except Exception, e:
             self.debug(e)
 
     def get_covers(self, mi, result_queue, abort, timeout=5.):
-        if not mi.isbn or not self.site_customization:
+        if not mi.isbn:
             return
-        from calibre.ebooks.metadata.library_thing import get_browser, login
+        from calibre.ebooks.metadata.amazon import get_cover_url
-        br = get_browser()
+        br = browser()
-        un, _, pw = self.site_customization.partition(':')
-        login(br, un, pw)
         try:
-            url = self.get_cover_url(mi.isbn, br, timeout=timeout)
+            url = get_cover_url(mi.isbn, br)
             cover_data = br.open_novisit(url).read()
             result_queue.put((True, cover_data, 'jpg', self.name))
         except Exception, e:
             result_queue.put((False, self.exception_to_string(e),
                 traceback.format_exc(), self.name))
 
-    def customization_help(self, gui=False):
-        ans = _('To use librarything.com you must sign up for a %sfree account%s '
-                'and enter your username and password separated by a : below.')
-        return '<p>'+ans%('<a href="http://www.librarything.com">', '</a>')
-
 # }}}
 
 def check_for_cover(mi, timeout=5.): # {{{

src/calibre/ebooks/mobi/mobiml.py

@@ -367,6 +367,9 @@ class MobiMLizer(object):
             istate.attrib['src'] = elem.attrib['src']
             istate.attrib['align'] = 'baseline'
             cssdict = style.cssdict()
+            valign = cssdict.get('vertical-align', None)
+            if valign in ('top', 'bottom', 'middle'):
+                istate.attrib['align'] = valign
             for prop in ('width', 'height'):
                 if cssdict[prop] != 'auto':
                     value = style[prop]

src/calibre/ebooks/oeb/transforms/flatcss.py

@@ -207,7 +207,14 @@ class CSSFlattener(object):
             font_size = self.sbase if self.sbase is not None else \
                 self.context.source.fbase
         if 'align' in node.attrib:
-            cssdict['text-align'] = node.attrib['align']
+            if tag != 'img':
+                cssdict['text-align'] = node.attrib['align']
+            else:
+                val = node.attrib['align']
+                if val in ('middle', 'bottom', 'top'):
+                    cssdict['vertical-align'] = val
+                elif val in ('left', 'right'):
+                    cssdict['text-align'] = val
             del node.attrib['align']
         if node.tag == XHTML('font'):
             node.tag = XHTML('span')

src/calibre/ebooks/txt/input.py

@@ -4,11 +4,10 @@ __license__ = 'GPL 3'
 __copyright__ = '2009, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
-import mimetypes
 import os
 import shutil
 
-from calibre import _ent_pat, walk, xml_entity_to_unicode
+from calibre import _ent_pat, walk, xml_entity_to_unicode, guess_type
 from calibre.customize.conversion import InputFormatPlugin, OptionRecommendation
 from calibre.ebooks.conversion.preprocess import DocAnalysis, Dehyphenator
 from calibre.ebooks.chardet import detect
@@ -85,13 +84,14 @@ class TXTInput(InputFormatPlugin):
                     if os.path.splitext(x)[1].lower() == '.txt':
                         with open(x, 'rb') as tf:
                             txt += tf.read() + '\n\n'
-                    if mimetypes.guess_type(x)[0] in OEB_IMAGES:
+                    mt = guess_type(x)[0]
+                    if mt in OEB_IMAGES:
                         path = os.path.relpath(x, tdir)
                         dir = os.path.join(os.getcwd(), os.path.dirname(path))
                         if not os.path.exists(dir):
                             os.makedirs(dir)
                         shutil.copy(x, os.path.join(os.getcwd(), path))
-                        images.append((path, mimetypes.guess_type(x)[0]))
+                        images.append((path, mt))
         else:
             txt = stream.read()
 
@@ -210,16 +210,18 @@ class TXTInput(InputFormatPlugin):
         oeb = html_input.convert(open(htmlfile.name, 'rb'), options, 'html', log,
                 {})
         # Add images from from txtz archive to oeb.
-        for image, mime in images:
+        # Disabled as the conversion pipeline adds unmanifested items that are
-            id, href = oeb.manifest.generate(id='image', href=image)
+        # referred to in the content automatically
-            oeb.manifest.add(id, href, mime)
+        #for image, mime in images:
+        #    id, href = oeb.manifest.generate(id='image', href=image)
+        #    oeb.manifest.add(id, href, mime)
         options.debug_pipeline = odi
         os.remove(htmlfile.name)
-        
+
         # Set metadata from file.
         from calibre.customize.ui import get_file_type_metadata
         from calibre.ebooks.oeb.transforms.metadata import meta_info_to_oeb_metadata
         mi = get_file_type_metadata(stream, file_ext)
         meta_info_to_oeb_metadata(mi, oeb.metadata, log)
-        
+
         return oeb

src/calibre/gui2/__init__.py

@@ -137,14 +137,18 @@ def _config():
             help=_('Automatically download the cover, if available'))
     c.add_opt('enforce_cpu_limit', default=True,
             help=_('Limit max simultaneous jobs to number of CPUs'))
-    c.add_opt('tag_browser_hidden_categories', default=set(),
-            help=_('tag browser categories not to display'))
     c.add_opt('gui_layout', choices=['wide', 'narrow'],
             help=_('The layout of the user interface'), default='wide')
     c.add_opt('show_avg_rating', default=True,
             help=_('Show the average rating per item indication in the tag browser'))
     c.add_opt('disable_animations', default=False,
             help=_('Disable UI animations'))
+
+    # This option is no longer used. It remains for compatibility with upgrades
+    # so the value can be migrated
+    c.add_opt('tag_browser_hidden_categories', default=set(),
+            help=_('tag browser categories not to display'))
+
     c.add_opt
     return ConfigProxy(c)
 

src/calibre/gui2/actions/add.py

@@ -204,7 +204,8 @@ class AddAction(InterfaceAction):
                         ]
         to_device = self.gui.stack.currentIndex() != 0
         if to_device:
-            filters = [(_('Supported books'), self.gui.device_manager.device.FORMATS)]
+            fmts = self.gui.device_manager.device.settings().format_map
+            filters = [(_('Supported books'), fmts)]
 
         books = choose_files(self.gui, 'add books dialog dir', 'Select books',
                              filters=filters)

src/calibre/gui2/complete.py

@@ -158,6 +158,8 @@ class MultiCompleteComboBox(EnComboBox):
         # item that matches case insensitively
         c = self.lineEdit().completer()
         c.setCaseSensitivity(Qt.CaseSensitive)
+        self.dummy_model = CompleteModel(self)
+        c.setModel(self.dummy_model)
 
     def update_items_cache(self, complete_items):
         self.lineEdit().update_items_cache(complete_items)

src/calibre/gui2/custom_column_widgets.py

@@ -551,7 +551,11 @@ class BulkBool(BulkBase, Bool):
 
     def setup_ui(self, parent):
         self.make_widgets(parent, QComboBox)
-        items = [_('Yes'), _('No'), _('Undefined')]
+        items = [_('Yes'), _('No')]
+        if tweaks['bool_custom_columns_are_tristate'] == 'no':
+            items.append('')
+        else:
+            items.append(_('Undefined'))
         icons = [I('ok.png'), I('list_remove.png'), I('blank.png')]
         self.main_widget.blockSignals(True)
         for icon, text in zip(icons, items):
@@ -560,7 +564,10 @@ class BulkBool(BulkBase, Bool):
 
     def getter(self):
         val = self.main_widget.currentIndex()
-        return {2: None, 1: False, 0: True}[val]
+        if tweaks['bool_custom_columns_are_tristate'] == 'no':
+            return {2: False, 1: False, 0: True}[val]
+        else:
+            return {2: None, 1: False, 0: True}[val]
 
     def setter(self, val):
         val = {None: 2, False: 1, True: 0}[val]
@@ -576,6 +583,14 @@ class BulkBool(BulkBase, Bool):
             val = False
         self.db.set_custom_bulk(book_ids, val, num=self.col_id, notify=notify)
 
+    def a_c_checkbox_changed(self):
+        if not self.ignore_change_signals:
+            if tweaks['bool_custom_columns_are_tristate'] == 'no' and \
+                                    self.main_widget.currentIndex() == 2:
+                self.a_c_checkbox.setChecked(False)
+            else:
+                self.a_c_checkbox.setChecked(True)
+
 class BulkInt(BulkBase):
 
     def setup_ui(self, parent):

src/calibre/gui2/device.py

@@ -1292,6 +1292,16 @@ class DeviceMixin(object): # {{{
         to both speed up matching and to count matches.
         '''
 
+        if not self.device_manager.is_device_connected:
+            return False
+
+        # It might be possible to get here without having initialized the
+        # library view. In this case, simply give up
+        try:
+            db = self.library_view.model().db
+        except:
+            return False
+
         string_pat = re.compile('(?u)\W|[_]')
         def clean_string(x):
             x = x.lower() if x else ''
@@ -1299,26 +1309,19 @@ class DeviceMixin(object): # {{{
 
         update_metadata = prefs['manage_device_metadata'] == 'on_connect'
 
+        get_covers = False
+        if update_metadata and self.device_manager.is_device_connected:
+            if self.device_manager.device.WANTS_UPDATED_THUMBNAILS:
+                get_covers = True
+
         # Force a reset if the caches are not initialized
         if reset or not hasattr(self, 'db_book_title_cache'):
             # Build a cache (map) of the library, so the search isn't On**2
             db_book_title_cache = {}
             db_book_uuid_cache = {}
-            # It might be possible to get here without having initialized the
-            # library view. In this case, simply give up
-            try:
-                db = self.library_view.model().db
-            except:
-                return False
 
-            get_covers = False
+            for id_ in db.data.iterallids():
-            if update_metadata and self.device_manager.is_device_connected:
+                title = clean_string(db.title(id_, index_is_id=True))
-                if self.device_manager.device.WANTS_UPDATED_THUMBNAILS:
-                    get_covers = True
-
-            for id in db.data.iterallids():
-                mi = db.get_metadata(id, index_is_id=True, get_cover=get_covers)
-                title = clean_string(mi.title)
                 if title not in db_book_title_cache:
                     db_book_title_cache[title] = \
                                 {'authors':{}, 'author_sort':{}, 'db_ids':{}}
@@ -1326,14 +1329,14 @@ class DeviceMixin(object): # {{{
                 # and author, then remember the last one. That is OK, because as
                 # we can't tell the difference between the books, one is as good
                 # as another.
-                if mi.authors:
+                authors = clean_string(db.authors(id_, index_is_id=True))
-                    authors = clean_string(authors_to_string(mi.authors))
+                if authors:
-                    db_book_title_cache[title]['authors'][authors] = mi
+                    db_book_title_cache[title]['authors'][authors] = id_
-                if mi.author_sort:
+                if db.author_sort(id_, index_is_id=True):
-                    aus = clean_string(mi.author_sort)
+                    aus = clean_string(db.author_sort(id_, index_is_id=True))
-                    db_book_title_cache[title]['author_sort'][aus] = mi
+                    db_book_title_cache[title]['author_sort'][aus] = id_
-                db_book_title_cache[title]['db_ids'][mi.application_id] = mi
+                db_book_title_cache[title]['db_ids'][id_] = id_
-                db_book_uuid_cache[mi.uuid] = mi
+                db_book_uuid_cache[db.uuid(id_, index_is_id=True)] = id_
             self.db_book_title_cache = db_book_title_cache
             self.db_book_uuid_cache = db_book_uuid_cache
 
@@ -1341,19 +1344,22 @@ class DeviceMixin(object): # {{{
         # in_library field. If the UUID matches a book in the library, then
         # do not consider that book for other matching. In all cases set
         # the application_id to the db_id of the matching book. This value
-        # will be used by books_on_device to indicate matches.
+        # will be used by books_on_device to indicate matches. While we are
+        # going by, update the metadata for a book if automatic management is on
 
         for booklist in booklists:
             for book in booklist:
                 book.in_library = None
                 if getattr(book, 'uuid', None) in self.db_book_uuid_cache:
+                    id_ = db_book_uuid_cache[book.uuid]
                     if update_metadata:
-                        book.smart_update(self.db_book_uuid_cache[book.uuid],
+                        book.smart_update(db.get_metadata(id_,
+                                                          index_is_id=True,
+                                                          get_cover=get_covers),
                                           replace_metadata=True)
                     book.in_library = 'UUID'
                     # ensure that the correct application_id is set
-                    book.application_id = \
+                    book.application_id = id_
-                        self.db_book_uuid_cache[book.uuid].application_id
                     continue
                 # No UUID exact match. Try metadata matching.
                 book_title = clean_string(book.title)
@@ -1363,21 +1369,25 @@ class DeviceMixin(object): # {{{
                     # will match if any of the db_id, author, or author_sort
                     # also match.
                     if getattr(book, 'application_id', None) in d['db_ids']:
-                        # app_id already matches a db_id. No need to set it.
                         if update_metadata:
-                            book.smart_update(d['db_ids'][book.application_id],
+                            id_ = getattr(book, 'application_id', None)
+                            book.smart_update(db.get_metadata(id_,
+                                                              index_is_id=True,
+                                                              get_cover=get_covers),
                                               replace_metadata=True)
                         book.in_library = 'APP_ID'
+                        # app_id already matches a db_id. No need to set it.
                         continue
                     # Sonys know their db_id independent of the application_id
                     # in the metadata cache. Check that as well.
                     if getattr(book, 'db_id', None) in d['db_ids']:
                         if update_metadata:
-                            book.smart_update(d['db_ids'][book.db_id],
+                            book.smart_update(db.get_metadata(book.db_id,
+                                                              index_is_id=True,
+                                                              get_cover=get_covers),
                                               replace_metadata=True)
                         book.in_library = 'DB_ID'
-                        book.application_id = \
+                        book.application_id = book.db_id
-                                    d['db_ids'][book.db_id].application_id
                         continue
                     # We now know that the application_id is not right. Set it
                     # to None to prevent book_on_device from accidentally
@@ -1389,19 +1399,23 @@ class DeviceMixin(object): # {{{
                         # either can appear as the author
                         book_authors = clean_string(authors_to_string(book.authors))
                         if book_authors in d['authors']:
+                            id_ = d['authors'][book_authors]
                             if update_metadata:
-                                book.smart_update(d['authors'][book_authors],
+                                book.smart_update(db.get_metadata(id_,
-                                                  replace_metadata=True)
+                                                              index_is_id=True,
+                                                              get_cover=get_covers),
+                                              replace_metadata=True)
                             book.in_library = 'AUTHOR'
-                            book.application_id = \
+                            book.application_id = id_
-                                    d['authors'][book_authors].application_id
                         elif book_authors in d['author_sort']:
+                            id_ = d['author_sort'][book_authors]
                             if update_metadata:
-                                book.smart_update(d['author_sort'][book_authors],
+                                book.smart_update(db.get_metadata(id_,
+                                                              index_is_id=True,
+                                                              get_cover=get_covers),
                                                   replace_metadata=True)
                             book.in_library = 'AUTH_SORT'
-                            book.application_id = \
+                            book.application_id = id_
-                                d['author_sort'][book_authors].application_id
                 else:
                     # Book definitely not matched. Clear its application ID
                     book.application_id = None

src/calibre/gui2/device_drivers/configwidget.py

@@ -9,15 +9,16 @@ import textwrap
 from PyQt4.Qt import QWidget, QListWidgetItem, Qt, QVariant, SIGNAL, \
                      QLabel, QLineEdit, QCheckBox
 
-from calibre.gui2 import error_dialog
+from calibre.gui2 import error_dialog, question_dialog
 from calibre.gui2.device_drivers.configwidget_ui import Ui_ConfigWidget
 from calibre.utils.formatter import validation_formatter
+from calibre.ebooks import BOOK_EXTENSIONS
 
 class ConfigWidget(QWidget, Ui_ConfigWidget):
 
     def __init__(self, settings, all_formats, supports_subdirs,
         must_read_metadata, supports_use_author_sort,
-        extra_customization_message):
+        extra_customization_message, device):
 
         QWidget.__init__(self)
         Ui_ConfigWidget.__init__(self)
@@ -25,9 +26,15 @@ class ConfigWidget(QWidget, Ui_ConfigWidget):
 
         self.settings = settings
 
+        all_formats = set(all_formats)
+        self.calibre_known_formats = device.FORMATS
+        self.device_name = device.get_gui_name()
+        if device.USER_CAN_ADD_NEW_FORMATS:
+            all_formats = set(all_formats) | set(BOOK_EXTENSIONS)
+
         format_map = settings.format_map
         disabled_formats = list(set(all_formats).difference(format_map))
-        for format in format_map + disabled_formats:
+        for format in format_map + list(sorted(disabled_formats)):
             item = QListWidgetItem(format, self.columns)
             item.setData(Qt.UserRole, QVariant(format))
             item.setFlags(Qt.ItemIsEnabled|Qt.ItemIsUserCheckable|Qt.ItemIsSelectable)
@@ -110,6 +117,18 @@ class ConfigWidget(QWidget, Ui_ConfigWidget):
         return self.opt_use_author_sort.isChecked()
 
     def validate(self):
+        formats = set(self.format_map())
+        extra = formats - set(self.calibre_known_formats)
+        if extra:
+            fmts = sorted([x.upper() for x in extra])
+            if not question_dialog(self, _('Unknown formats'),
+                    _('You have enabled the <b>{0}</b> formats for'
+                        ' your {1}. The {1} may not support them.'
+                        ' If you send these formats to your {1} they '
+                        'may not work. Are you sure?').format(
+                            (', '.join(fmts)), self.device_name)):
+                return False
+
         tmpl = unicode(self.opt_save_template.text())
         try:
             validation_formatter.validate(tmpl)

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -912,6 +912,7 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
 
     def series_changed(self, *args):
         self.write_series = True
+        self.autonumber_series.setEnabled(True)
 
     def s_r_remove_query(self, *args):
         if self.query_field.currentIndex() == 0:

src/calibre/gui2/dialogs/metadata_bulk.ui

@@ -303,6 +303,9 @@
             <layout class="QHBoxLayout" name="HLayout_3">
              <item>
               <widget class="QCheckBox" name="autonumber_series">
+               <property name="enabled">
+                <bool>false</bool>
+               </property>
                <property name="toolTip">
                 <string>If not checked, the series number for the books will be set to 1.
 If checked, selected books will be automatically numbered, in the order
@@ -1006,8 +1009,8 @@ not multiple and the destination field is multiple</string>
                <rect>
                 <x>0</x>
                 <y>0</y>
-                <width>938</width>
+                <width>197</width>
-                <height>268</height>
+                <height>60</height>
                </rect>
               </property>
               <layout class="QGridLayout" name="testgrid">

src/calibre/gui2/dialogs/tag_list_editor.py

@@ -99,8 +99,8 @@ class TagListEditor(QDialog, Ui_TagListEditor):
             return
         self.available_tags.editItem(item)
 
-    def delete_tags(self, item=None):
+    def delete_tags(self):
-        deletes = self.available_tags.selectedItems() if item is None else [item]
+        deletes = self.available_tags.selectedItems()
         if not deletes:
             error_dialog(self, _('No items selected'),
                          _('You must select at least one items from the list.')).exec_()

src/calibre/gui2/library/models.py

@@ -685,7 +685,7 @@ class BooksModel(QAbstractTableModel): # {{{
                 self.dc[col] = functools.partial(bool_type, idx=idx)
                 self.dc_decorator[col] = functools.partial(
                             bool_type_decorator, idx=idx,
-                            bool_cols_are_tristate=tweaks['bool_custom_columns_are_tristate'] == 'yes')
+                            bool_cols_are_tristate=tweaks['bool_custom_columns_are_tristate'] != 'no')
             elif datatype == 'rating':
                 self.dc[col] = functools.partial(rating_type, idx=idx)
             elif datatype == 'series':

src/calibre/gui2/preferences/search.py

@@ -26,12 +26,19 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         r('limit_search_columns_to', prefs, setting=CommaSeparatedList)
         fl = gui.library_view.model().db.field_metadata.get_search_terms()
         self.opt_limit_search_columns_to.update_items_cache(fl)
+        self.clear_history_button.clicked.connect(self.clear_histories)
 
     def refresh_gui(self, gui):
         gui.search.search_as_you_type(config['search_as_you_type'])
         gui.library_view.model().set_highlight_only(config['highlight_search_matches'])
         gui.search.do_search()
 
+    def clear_histories(self, *args):
+        for key, val in config.defaults.iteritems():
+            if key.endswith('_search_history') and isinstance(val, list):
+                config[key] = []
+        self.gui.search.clear_history()
+
 if __name__ == '__main__':
     app = QApplication([])
     test_widget('Interface', 'Search')

src/calibre/gui2/preferences/search.ui

@@ -77,7 +77,7 @@
      </layout>
     </widget>
    </item>
-   <item row="3" column="0">
+   <item row="4" column="0">
     <spacer name="verticalSpacer">
      <property name="orientation">
       <enum>Qt::Vertical</enum>
@@ -90,13 +90,23 @@
      </property>
     </spacer>
    </item>
+   <item row="3" column="0">
+    <widget class="QPushButton" name="clear_history_button">
+     <property name="toolTip">
+      <string>Clear search histories from all over calibre. Including the book list, e-book viewer, fetch news dialog, etc.</string>
+     </property>
+     <property name="text">
+      <string>Clear search &amp;histories</string>
+     </property>
+    </widget>
+   </item>
   </layout>
  </widget>
  <customwidgets>
   <customwidget>
    <class>MultiCompleteLineEdit</class>
    <extends>QLineEdit</extends>
-   <header>calibre/gui2.complete.h</header>
+   <header>calibre/gui2/complete.h</header>
   </customwidget>
  </customwidgets>
  <resources/>

src/calibre/gui2/search_box.py

@@ -114,6 +114,9 @@ class SearchBox2(QComboBox): # {{{
     def text(self):
         return self.currentText()
 
+    def clear_history(self, *args):
+        QComboBox.clear(self)
+
     def clear(self, emit_search=True):
         self.normalize_state()
         self.setEditText('')

src/calibre/gui2/tag_view.py

@@ -116,7 +116,14 @@ class TagsView(QTreeView): # {{{
         self.set_new_model(self._model.get_filter_categories_by())
 
     def set_database(self, db, tag_match, sort_by):
-        self.hidden_categories = config['tag_browser_hidden_categories']
+        self.hidden_categories = db.prefs.get('tag_browser_hidden_categories', None)
+        # migrate from config to db prefs
+        if self.hidden_categories is None:
+            self.hidden_categories = config['tag_browser_hidden_categories']
+            db.prefs.set('tag_browser_hidden_categories', list(self.hidden_categories))
+        else:
+            self.hidden_categories = set(self.hidden_categories)
+
         old = getattr(self, '_model', None)
         if old is not None:
             old.break_cycles()
@@ -234,7 +241,7 @@ class TagsView(QTreeView): # {{{
                     gprefs['tags_browser_partition_method'] = category
             elif action == 'defaults':
                 self.hidden_categories.clear()
-            config.set('tag_browser_hidden_categories', self.hidden_categories)
+            self.db.prefs.set('tag_browser_hidden_categories', list(self.hidden_categories))
             self.set_new_model()
         except:
             return

src/calibre/gui2/viewer/main.py

@@ -17,16 +17,16 @@ from calibre.gui2.viewer.bookmarkmanager import BookmarkManager
 from calibre.gui2.widgets import ProgressIndicator
 from calibre.gui2.main_window import MainWindow
 from calibre.gui2 import Application, ORG_NAME, APP_UID, choose_files, \
-                         info_dialog, error_dialog, open_url, available_height
+    info_dialog, error_dialog, open_url, available_height, gprefs
 from calibre.ebooks.oeb.iterator import EbookIterator
 from calibre.ebooks import DRMError
-from calibre.constants import islinux, isfreebsd, isosx
+from calibre.constants import islinux, isfreebsd, isosx, filesystem_encoding
 from calibre.utils.config import Config, StringConfig, dynamic
 from calibre.gui2.search_box import SearchBox2
 from calibre.ebooks.metadata import MetaInformation
 from calibre.customize.ui import available_input_formats
 from calibre.gui2.viewer.dictionary import Lookup
-from calibre import as_unicode
+from calibre import as_unicode, force_unicode, isbytestring
 
 class TOCItem(QStandardItem):
 
@@ -160,6 +160,12 @@ class HelpfulLineEdit(QLineEdit):
         self.setPalette(self.gray)
         self.setText(self.HELP_TEXT)
 
+class RecentAction(QAction):
+
+    def __init__(self, path, parent):
+        self.path = path
+        QAction.__init__(self, os.path.basename(path), parent)
+
 class EbookViewer(MainWindow, Ui_EbookViewer):
 
     STATE_VERSION = 1
@@ -284,8 +290,26 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
         ca = self.view.copy_action
         ca.setShortcut(QKeySequence.Copy)
         self.addAction(ca)
+        self.open_history_menu = QMenu()
+        self.build_recent_menu()
+        self.action_open_ebook.setMenu(self.open_history_menu)
+        self.open_history_menu.triggered[QAction].connect(self.open_recent)
+        w = self.tool_bar.widgetForAction(self.action_open_ebook)
+        w.setPopupMode(QToolButton.MenuButtonPopup)
+
         self.restore_state()
 
+    def build_recent_menu(self):
+        m = self.open_history_menu
+        m.clear()
+        count = 0
+        for path in gprefs.get('viewer_open_history', []):
+            if count > 9:
+                break
+            if os.path.exists(path):
+                m.addAction(RecentAction(path, m))
+                count += 1
+
     def closeEvent(self, e):
         self.save_state()
         return MainWindow.closeEvent(self, e)
@@ -425,6 +449,9 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
         if files:
             self.load_ebook(files[0])
 
+    def open_recent(self, action):
+        self.load_ebook(action.path)
+
     def font_size_larger(self, checked):
         frac = self.view.magnify_fonts()
         self.action_font_size_larger.setEnabled(self.view.multiplier() < 3)
@@ -647,6 +674,17 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
                     self.action_table_of_contents.setChecked(True)
             else:
                 self.action_table_of_contents.setChecked(False)
+            if isbytestring(pathtoebook):
+                pathtoebook = force_unicode(pathtoebook, filesystem_encoding)
+            vh = gprefs.get('viewer_open_history', [])
+            try:
+                vh.remove(pathtoebook)
+            except:
+                pass
+            vh.insert(0, pathtoebook)
+            gprefs.set('viewer_open_history', vh[:50])
+            self.build_recent_menu()
+
             self.action_table_of_contents.setDisabled(not self.iterator.toc)
             self.current_book_has_toc = bool(self.iterator.toc)
             self.current_title = title

src/calibre/library/caches.py

@@ -528,7 +528,7 @@ class ResultCache(SearchQueryParser): # {{{
                 location[i] = db_col[loc]
 
             # get the tweak here so that the string lookup and compare aren't in the loop
-            bools_are_tristate = tweaks['bool_custom_columns_are_tristate'] == 'yes'
+            bools_are_tristate = tweaks['bool_custom_columns_are_tristate'] != 'no'
 
             for loc in location: # location is now an array of field indices
                 if loc == db_col['authors']:
@@ -812,7 +812,10 @@ class SortKeyGenerator(object):
                 val = self.string_sort_key(val)
 
             elif dt == 'bool':
-                val = {True: 1, False: 2, None: 3}.get(val, 3)
+                if tweaks['bool_custom_columns_are_tristate'] == 'no':
+                    val = {True: 1, False: 2, None: 2}.get(val, 2)
+                else:
+                    val = {True: 1, False: 2, None: 3}.get(val, 3)
 
             yield val