merge from trunk

resources/recipes/apple_daily.recipe

@@ -0,0 +1,161 @@
+# -*- coding: utf-8 -*-
+import re
+from calibre.web.feeds.recipes import BasicNewsRecipe
+
+class AppleDaily(BasicNewsRecipe):
+
+    title       = u'蘋果日報'
+    __author__  = u'蘋果日報'
+    __publisher__  = u'蘋果日報'
+    description = u'蘋果日報'
+    masthead_url = 'http://hk.apple.nextmedia.com/template/common/header/2009/images/atnextheader_logo_appledaily.gif'
+    language = 'zh_TW'
+    encoding = 'UTF-8'
+    timefmt = ' [%a, %d %b, %Y]'
+    needs_subscription = False
+    remove_javascript = True
+    remove_tags_before = dict(name=['ul', 'h1'])
+    remove_tags_after  = dict(name='form')
+    remove_tags = [dict(attrs={'class':['articleTools', 'post-tools', 'side_tool', 'nextArticleLink clearfix']}),
+                dict(id=['footer', 'toolsRight', 'articleInline', 'navigation', 'archive', 'side_search', 'blog_sidebar', 'side_tool', 'side_index']),
+                dict(name=['script', 'noscript', 'style', 'form'])]
+    no_stylesheets = True
+    extra_css = '''
+    	@font-face {font-family: "uming", serif, sans-serif;  src: url(res:///usr/share/fonts/truetype/arphic/uming.ttc); }\n
+	    body {margin-right: 8pt; font-family: 'uming', serif;}
+        h1 {font-family: 'uming', serif, sans-serif}
+            '''
+    #extra_css = 'h1 {font: sans-serif large;}\n.byline {font:monospace;}'
+
+    preprocess_regexps = [
+       (re.compile(r'img.php?server=(?P<server>[^&]+)&path=(?P<path>[^&]+).*', re.DOTALL|re.IGNORECASE),
+        lambda match: 'http://' + match.group('server') + '/' + match.group('path')),
+    ]
+
+    def get_cover_url(self):
+        return 'http://hk.apple.nextmedia.com/template/common/header/2009/images/atnextheader_logo_appledaily.gif'
+
+
+    #def get_browser(self):
+        #br = BasicNewsRecipe.get_browser()
+        #if self.username is not None and self.password is not None:
+        #    br.open('http://www.nytimes.com/auth/login')
+        #    br.select_form(name='login')
+        #    br['USERID']   = self.username
+        #    br['PASSWORD'] = self.password
+        #    br.submit()
+        #return br
+
+    def preprocess_html(self, soup):
+        #process all the images
+        for tag in soup.findAll(lambda tag: tag.name.lower()=='img' and tag.has_key('src')):
+            iurl = tag['src']
+            #print 'checking image: ' + iurl
+
+            #img\.php?server\=(?P<server>[^&]+)&path=(?P<path>[^&]+)
+            p = re.compile(r'img\.php\?server=(?P<server>[^&]+)&path=(?P<path>[^&]+)', re.DOTALL|re.IGNORECASE)
+
+            m = p.search(iurl)
+
+            if m is not None:
+                iurl = 'http://' + m.group('server') + '/' + m.group('path')
+                #print 'working! new url: ' + iurl
+                tag['src'] = iurl
+            #else:
+                #print 'not good'
+
+        for tag in soup.findAll(lambda tag: tag.name.lower()=='a' and tag.has_key('href')):
+            iurl = tag['href']
+            #print 'checking image: ' + iurl
+
+            #img\.php?server\=(?P<server>[^&]+)&path=(?P<path>[^&]+)
+            p = re.compile(r'img\.php\?server=(?P<server>[^&]+)&path=(?P<path>[^&]+)', re.DOTALL|re.IGNORECASE)
+
+            m = p.search(iurl)
+
+            if m is not None:
+                iurl = 'http://' + m.group('server') + '/' + m.group('path')
+                #print 'working! new url: ' + iurl
+                tag['href'] = iurl
+            #else:
+                #print 'not good'
+
+        return soup
+
+
+    def parse_index(self):
+        base = 'http://news.hotpot.hk/fruit'
+        soup = self.index_to_soup('http://news.hotpot.hk/fruit/index.php')
+
+        #def feed_title(div):
+        #    return ''.join(div.findAll(text=True, recursive=False)).strip()
+
+        articles = {}
+        key = None
+        ans = []
+        for div in soup.findAll('li'):
+            key = div.find(text=True, recursive=True);
+            #if key == u'豪情':
+           #    continue;
+
+            print 'section=' + key
+
+            articles[key] = []
+
+            ans.append(key)
+
+            a = div.find('a', href=True)
+
+            if not a:
+                continue
+
+            url = base + '/' + a['href']
+            print 'url=' + url
+
+            if not articles.has_key(key):
+                articles[key] = []
+            else:
+                # sub page
+                subSoup = self.index_to_soup(url)
+
+                for subDiv in subSoup.findAll('li'):
+                    subA = subDiv.find('a', href=True)
+                    subTitle = subDiv.find(text=True, recursive=True)
+                    subUrl = base + '/' + subA['href']
+
+                    print 'subUrl' + subUrl
+
+                    articles[key].append(
+                        dict(title=subTitle,
+                         url=subUrl,
+                         date='',
+                         description='',
+                         content=''))
+
+
+#             elif div['class'] in ['story', 'story headline']:
+#                 a = div.find('a', href=True)
+#                 if not a:
+#                     continue
+#                 url = re.sub(r'\?.*', '', a['href'])
+#                 url += '?pagewanted=all'
+#                 title = self.tag_to_string(a, use_alt=True).strip()
+#                 description = ''
+#                 pubdate = strftime('%a, %d %b')
+#                 summary = div.find(True, attrs={'class':'summary'})
+#                 if summary:
+#                     description = self.tag_to_string(summary, use_alt=False)
+#
+#                 feed = key if key is not None else 'Uncategorized'
+#                 if not articles.has_key(feed):
+#                     articles[feed] = []
+#                 if not 'podcasts' in url:
+#                     articles[feed].append(
+#                               dict(title=title, url=url, date=pubdate,
+#                                    description=description,
+#                                    content=''))
+#        ans = self.sort_index_by(ans, {'The Front Page':-1, 'Dining In, Dining Out':1, 'Obituaries':2})
+        ans = [(unicode(key), articles[key]) for key in ans if articles.has_key(key)]
+        return ans
+
+

resources/recipes/workers_world.recipe

@@ -0,0 +1,26 @@
+from calibre.web.feeds.recipes import BasicNewsRecipe
+
+class WorkersWorld(BasicNewsRecipe):
+
+    title = u'Workers World'
+    description = u'Socialist news and analysis'
+    __author__ = u'urslnx'
+    no_stylesheets = True
+    use_embedded_content = False
+    remove_javascript = True
+    oldest_article = 7
+    max_articles_per_feed = 100
+    encoding = 'utf8'
+    publisher = 'workers.org'
+    category = 'news, politics, USA, world'
+    language = 'en'
+    publication_type = 'newsportal'
+    extra_css = ' body{ font-family: Verdana,Arial,Helvetica,sans-serif; } h1{ font-size: x-large; text-align: left; margin-top:0.5em; margin-bottom:0.25em; } h2{ font-size: large; }  p{ text-align: left; } .published{ font-size: small; } .byline{ font-size: small; } .copyright{ font-size: small; } '
+    remove_tags_before = dict(name='div', attrs={'id':'evernote'})
+    remove_tags_after = dict(name='div', attrs={'id':'footer'})
+
+    masthead_url='http://www.workers.org/graphics/wwlogo300.gif'
+    cover_url = 'http://www.workers.org/pdf/current.jpg'
+    feeds = [(u'Headlines', u'http://www.workers.org/rss/nonstandard_rss.xml'),
+]
+

src/calibre/customize/__init__.py

@@ -90,6 +90,11 @@ class Plugin(object): # {{{
         an optional method validate() that takes no arguments and is called
         immediately after the user clicks OK. Changes are applied if and only
         if the method returns True.
+
+        If for some reason you cannot perform the configuration at this time,
+        return a tuple of two strings (message, details), these will be
+        displayed as a warning dialog to the user and the process will be
+        aborted.
         '''
         raise NotImplementedError()
 
@@ -133,6 +138,12 @@ class Plugin(object): # {{{
         except NotImplementedError:
             config_widget = None
 
+        if isinstance(config_widget, tuple):
+            from calibre.gui2 import warning_dialog
+            warning_dialog(parent, _('Cannot configure'), config_widget[0],
+                    det_msg=config_widget[1], show=True)
+            return False
+
         if config_widget is not None:
             v.addWidget(config_widget)
             v.addWidget(button_box)

src/calibre/devices/android/driver.py

@@ -19,7 +19,7 @@ class ANDROID(USBMS):
 
     VENDOR_ID   = {
             # HTC
-            0x0bb4 : { 0x0c02 : [0x100, 0x0227, 0x0226],
+            0x0bb4 : { 0x0c02 : [0x100, 0x0227, 0x0226, 0x222],
                        0x0c01 : [0x100, 0x0227, 0x0226],
                        0x0ff9 : [0x0100, 0x0227, 0x0226],
                        0x0c87 : [0x0100, 0x0227, 0x0226],

src/calibre/devices/apple/driver.py

@@ -39,6 +39,7 @@ if iswindows:
 class DriverBase(DeviceConfig, DevicePlugin):
     # Needed for config_widget to work
     FORMATS = ['epub', 'pdf']
+    USER_CAN_ADD_NEW_FORMATS = False
     SUPPORTS_SUB_DIRS = True   # To enable second checkbox in customize widget
 
     @classmethod

src/calibre/devices/bambook/driver.py

@@ -32,6 +32,7 @@ class BAMBOOK(DeviceConfig, DevicePlugin):
     ip = None
 
     FORMATS = [ "snb" ]
+    USER_CAN_ADD_NEW_FORMATS = False
     VENDOR_ID = 0x230b
     PRODUCT_ID = 0x0001
     BCD = None
@@ -421,7 +422,7 @@ class BAMBOOK(DeviceConfig, DevicePlugin):
         from calibre.gui2.device_drivers.configwidget import ConfigWidget
         cw = ConfigWidget(cls.settings(), cls.FORMATS, cls.SUPPORTS_SUB_DIRS,
             cls.MUST_READ_METADATA, cls.SUPPORTS_USE_AUTHOR_SORT,
-            cls.EXTRA_CUSTOMIZATION_MESSAGE)
+            cls.EXTRA_CUSTOMIZATION_MESSAGE, cls)
         # Turn off the Save template
         cw.opt_save_template.setVisible(False)
         cw.label.setVisible(False)

src/calibre/devices/jetbook/driver.py

@@ -93,11 +93,11 @@ class MIBUK(USBMS):
 
     VENDOR_ID   = [0x0525]
     PRODUCT_ID  = [0xa4a5]
-    BCD         = [0x314]
+    BCD         = [0x314, 0x319]
     SUPPORTS_SUB_DIRS = True
 
-    VENDOR_NAME      = 'LINUX'
+    VENDOR_NAME      = ['LINUX', 'FILE_BAC']
-    WINDOWS_MAIN_MEM = 'WOLDERMIBUK'
+    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['WOLDERMIBUK', 'KED_STORAGE_GADG']
 
 class JETBOOK_MINI(USBMS):
 

src/calibre/devices/kobo/driver.py

@@ -98,7 +98,6 @@ class KOBO(USBMS):
 
         def update_booklist(prefix, path, title, authors, mime, date, ContentType, ImageID, readstatus, MimeType):
             changed = False
-            # if path_to_ext(path) in self.FORMATS:
             try:
                 lpath = path.partition(self.normalize_path(prefix))[2]
                 if lpath.startswith(os.sep):
@@ -220,7 +219,7 @@ class KOBO(USBMS):
         #    2) volume_shorcover
         #    2) content
 
-        debug_print('delete_via_sql: ContentID: ', ContentID, 'ContentType: ', ContentType) 
+        debug_print('delete_via_sql: ContentID: ', ContentID, 'ContentType: ', ContentType)
         connection = sqlite.connect(self.normalize_path(self._main_prefix + '.kobo/KoboReader.sqlite'))
         cursor = connection.cursor()
         t = (ContentID,)
@@ -532,7 +531,7 @@ class KOBO(USBMS):
                         if result is None:
                             datelastread = '1970-01-01T00:00:00'
                         else:
-                            datelastread = result[0] if result[0] is not None else '1970-01-01T00:00:00' 
+                            datelastread = result[0] if result[0] is not None else '1970-01-01T00:00:00'
 
                         t = (datelastread,ContentID,)
 

src/calibre/devices/usbms/deviceconfig.py

@@ -34,6 +34,10 @@ class DeviceConfig(object):
     #: If None the default is used
     SAVE_TEMPLATE = None
 
+    #: If True the user can add new formats to the driver
+    USER_CAN_ADD_NEW_FORMATS = True
+
+
     @classmethod
     def _default_save_template(cls):
         from calibre.library.save_to_disk import config
@@ -73,7 +77,7 @@ class DeviceConfig(object):
         from calibre.gui2.device_drivers.configwidget import ConfigWidget
         cw = ConfigWidget(cls.settings(), cls.FORMATS, cls.SUPPORTS_SUB_DIRS,
             cls.MUST_READ_METADATA, cls.SUPPORTS_USE_AUTHOR_SORT,
-            cls.EXTRA_CUSTOMIZATION_MESSAGE)
+            cls.EXTRA_CUSTOMIZATION_MESSAGE, cls)
         return cw
 
     @classmethod

src/calibre/devices/usbms/driver.py

@@ -93,9 +93,11 @@ class USBMS(CLI, Device):
         for idx,b in enumerate(bl):
             bl_cache[b.lpath] = idx
 
+        all_formats = set(self.settings().format_map) | set(self.FORMATS)
+
         def update_booklist(filename, path, prefix):
             changed = False
-            if path_to_ext(filename) in self.FORMATS:
+            if path_to_ext(filename) in all_formats:
                 try:
                     lpath = os.path.join(path, filename).partition(self.normalize_path(prefix))[2]
                     if lpath.startswith(os.sep):

src/calibre/ebooks/conversion/utils.py

@@ -156,17 +156,17 @@ class HeuristicProcessor(object):
         ]
 
         ITALICIZE_STYLE_PATS = [
-            r'(?msu)(?<=[\s>])_(?P<words>[^_]+)?_',
+            r'(?msu)(?<=[\s>])_(?P<words>[^_]+)_',
-            r'(?msu)(?<=[\s>])/(?P<words>[^/]+)?/',
+            r'(?msu)(?<=[\s>])/(?P<words>[^/]+)/',
-            r'(?msu)(?<=[\s>])~~(?P<words>[^~]+)?~~',
+            r'(?msu)(?<=[\s>])~~(?P<words>[^~]+)~~',
-            r'(?msu)(?<=[\s>])\*(?P<words>[^\*]+)?\*',
+            r'(?msu)(?<=[\s>])\*(?P<words>[^\*]+)\*',
-            r'(?msu)(?<=[\s>])~(?P<words>[^~]+)?~',
+            r'(?msu)(?<=[\s>])~(?P<words>[^~]+)~',
-            r'(?msu)(?<=[\s>])_/(?P<words>[^/_]+)?/_',
+            r'(?msu)(?<=[\s>])_/(?P<words>[^/_]+)/_',
-            r'(?msu)(?<=[\s>])_\*(?P<words>[^\*_]+)?\*_',
+            r'(?msu)(?<=[\s>])_\*(?P<words>[^\*_]+)\*_',
-            r'(?msu)(?<=[\s>])\*/(?P<words>[^/\*]+)?/\*',
+            r'(?msu)(?<=[\s>])\*/(?P<words>[^/\*]+)/\*',
-            r'(?msu)(?<=[\s>])_\*/(?P<words>[^\*_]+)?/\*_',
+            r'(?msu)(?<=[\s>])_\*/(?P<words>[^\*_]+)/\*_',
-            r'(?msu)(?<=[\s>])/:(?P<words>[^:/]+)?:/',
+            r'(?msu)(?<=[\s>])/:(?P<words>[^:/]+):/',
-            r'(?msu)(?<=[\s>])\|:(?P<words>[^:\|]+)?:\|',
+            r'(?msu)(?<=[\s>])\|:(?P<words>[^:\|]+):\|',
         ]
 
         for word in ITALICIZE_WORDS:

src/calibre/ebooks/metadata/covers.py

@@ -145,6 +145,7 @@ class LibraryThingCovers(CoverDownload): # {{{
             return url
 
     def has_cover(self, mi, ans, timeout=5.):
+        return False
         if not mi.isbn or not self.site_customization:
             return False
         from calibre.ebooks.metadata.library_thing import get_browser, login

src/calibre/gui2/actions/add.py

@@ -204,7 +204,8 @@ class AddAction(InterfaceAction):
                         ]
         to_device = self.gui.stack.currentIndex() != 0
         if to_device:
-            filters = [(_('Supported books'), self.gui.device_manager.device.FORMATS)]
+            fmts = self.gui.device_manager.device.settings().format_map
+            filters = [(_('Supported books'), fmts)]
 
         books = choose_files(self.gui, 'add books dialog dir', 'Select books',
                              filters=filters)

src/calibre/gui2/complete.py

@@ -158,6 +158,8 @@ class MultiCompleteComboBox(EnComboBox):
         # item that matches case insensitively
         c = self.lineEdit().completer()
         c.setCaseSensitivity(Qt.CaseSensitive)
+        self.dummy_model = CompleteModel(self)
+        c.setModel(self.dummy_model)
 
     def update_items_cache(self, complete_items):
         self.lineEdit().update_items_cache(complete_items)

src/calibre/gui2/custom_column_widgets.py

@@ -551,7 +551,11 @@ class BulkBool(BulkBase, Bool):
 
     def setup_ui(self, parent):
         self.make_widgets(parent, QComboBox)
-        items = [_('Yes'), _('No'), _('Undefined')]
+        items = [_('Yes'), _('No')]
+        if tweaks['bool_custom_columns_are_tristate'] == 'no':
+            items.append('')
+        else:
+            items.append(_('Undefined'))
         icons = [I('ok.png'), I('list_remove.png'), I('blank.png')]
         self.main_widget.blockSignals(True)
         for icon, text in zip(icons, items):
@@ -560,7 +564,10 @@ class BulkBool(BulkBase, Bool):
 
     def getter(self):
         val = self.main_widget.currentIndex()
-        return {2: None, 1: False, 0: True}[val]
+        if tweaks['bool_custom_columns_are_tristate'] == 'no':
+            return {2: False, 1: False, 0: True}[val]
+        else:
+            return {2: None, 1: False, 0: True}[val]
 
     def setter(self, val):
         val = {None: 2, False: 1, True: 0}[val]
@@ -576,6 +583,14 @@ class BulkBool(BulkBase, Bool):
             val = False
         self.db.set_custom_bulk(book_ids, val, num=self.col_id, notify=notify)
 
+    def a_c_checkbox_changed(self):
+        if not self.ignore_change_signals:
+            if tweaks['bool_custom_columns_are_tristate'] == 'no' and \
+                                    self.main_widget.currentIndex() == 2:
+                self.a_c_checkbox.setChecked(False)
+            else:
+                self.a_c_checkbox.setChecked(True)
+
 class BulkInt(BulkBase):
 
     def setup_ui(self, parent):

src/calibre/gui2/device.py

@@ -1292,6 +1292,16 @@ class DeviceMixin(object): # {{{
         to both speed up matching and to count matches.
         '''
 
+        if not self.device_manager.is_device_connected:
+            return False
+
+        # It might be possible to get here without having initialized the
+        # library view. In this case, simply give up
+        try:
+            db = self.library_view.model().db
+        except:
+            return False
+
         string_pat = re.compile('(?u)\W|[_]')
         def clean_string(x):
             x = x.lower() if x else ''
@@ -1299,26 +1309,19 @@ class DeviceMixin(object): # {{{
 
         update_metadata = prefs['manage_device_metadata'] == 'on_connect'
 
+        get_covers = False
+        if update_metadata and self.device_manager.is_device_connected:
+            if self.device_manager.device.WANTS_UPDATED_THUMBNAILS:
+                get_covers = True
+
         # Force a reset if the caches are not initialized
         if reset or not hasattr(self, 'db_book_title_cache'):
             # Build a cache (map) of the library, so the search isn't On**2
             db_book_title_cache = {}
             db_book_uuid_cache = {}
-            # It might be possible to get here without having initialized the
-            # library view. In this case, simply give up
-            try:
-                db = self.library_view.model().db
-            except:
-                return False
 
-            get_covers = False
+            for id_ in db.data.iterallids():
-            if update_metadata and self.device_manager.is_device_connected:
+                title = clean_string(db.title(id_, index_is_id=True))
-                if self.device_manager.device.WANTS_UPDATED_THUMBNAILS:
-                    get_covers = True
-
-            for id in db.data.iterallids():
-                mi = db.get_metadata(id, index_is_id=True, get_cover=get_covers)
-                title = clean_string(mi.title)
                 if title not in db_book_title_cache:
                     db_book_title_cache[title] = \
                                 {'authors':{}, 'author_sort':{}, 'db_ids':{}}
@@ -1326,14 +1329,14 @@ class DeviceMixin(object): # {{{
                 # and author, then remember the last one. That is OK, because as
                 # we can't tell the difference between the books, one is as good
                 # as another.
-                if mi.authors:
+                authors = clean_string(db.authors(id_, index_is_id=True))
-                    authors = clean_string(authors_to_string(mi.authors))
+                if authors:
-                    db_book_title_cache[title]['authors'][authors] = mi
+                    db_book_title_cache[title]['authors'][authors] = id_
-                if mi.author_sort:
+                if db.author_sort(id_, index_is_id=True):
-                    aus = clean_string(mi.author_sort)
+                    aus = clean_string(db.author_sort(id_, index_is_id=True))
-                    db_book_title_cache[title]['author_sort'][aus] = mi
+                    db_book_title_cache[title]['author_sort'][aus] = id_
-                db_book_title_cache[title]['db_ids'][mi.application_id] = mi
+                db_book_title_cache[title]['db_ids'][id_] = id_
-                db_book_uuid_cache[mi.uuid] = mi
+                db_book_uuid_cache[db.uuid(id_, index_is_id=True)] = id_
             self.db_book_title_cache = db_book_title_cache
             self.db_book_uuid_cache = db_book_uuid_cache
 
@@ -1341,19 +1344,22 @@ class DeviceMixin(object): # {{{
         # in_library field. If the UUID matches a book in the library, then
         # do not consider that book for other matching. In all cases set
         # the application_id to the db_id of the matching book. This value
-        # will be used by books_on_device to indicate matches.
+        # will be used by books_on_device to indicate matches. While we are
+        # going by, update the metadata for a book if automatic management is on
 
         for booklist in booklists:
             for book in booklist:
                 book.in_library = None
                 if getattr(book, 'uuid', None) in self.db_book_uuid_cache:
+                    id_ = db_book_uuid_cache[book.uuid]
                     if update_metadata:
-                        book.smart_update(self.db_book_uuid_cache[book.uuid],
+                        book.smart_update(db.get_metadata(id_,
+                                                          index_is_id=True,
+                                                          get_cover=get_covers),
                                           replace_metadata=True)
                     book.in_library = 'UUID'
                     # ensure that the correct application_id is set
-                    book.application_id = \
+                    book.application_id = id_
-                        self.db_book_uuid_cache[book.uuid].application_id
                     continue
                 # No UUID exact match. Try metadata matching.
                 book_title = clean_string(book.title)
@@ -1363,21 +1369,25 @@ class DeviceMixin(object): # {{{
                     # will match if any of the db_id, author, or author_sort
                     # also match.
                     if getattr(book, 'application_id', None) in d['db_ids']:
-                        # app_id already matches a db_id. No need to set it.
                         if update_metadata:
-                            book.smart_update(d['db_ids'][book.application_id],
+                            id_ = getattr(book, 'application_id', None)
+                            book.smart_update(db.get_metadata(id_,
+                                                              index_is_id=True,
+                                                              get_cover=get_covers),
                                               replace_metadata=True)
                         book.in_library = 'APP_ID'
+                        # app_id already matches a db_id. No need to set it.
                         continue
                     # Sonys know their db_id independent of the application_id
                     # in the metadata cache. Check that as well.
                     if getattr(book, 'db_id', None) in d['db_ids']:
                         if update_metadata:
-                            book.smart_update(d['db_ids'][book.db_id],
+                            book.smart_update(db.get_metadata(book.db_id,
+                                                              index_is_id=True,
+                                                              get_cover=get_covers),
                                               replace_metadata=True)
                         book.in_library = 'DB_ID'
-                        book.application_id = \
+                        book.application_id = book.db_id
-                                    d['db_ids'][book.db_id].application_id
                         continue
                     # We now know that the application_id is not right. Set it
                     # to None to prevent book_on_device from accidentally
@@ -1389,19 +1399,23 @@ class DeviceMixin(object): # {{{
                         # either can appear as the author
                         book_authors = clean_string(authors_to_string(book.authors))
                         if book_authors in d['authors']:
+                            id_ = d['authors'][book_authors]
                             if update_metadata:
-                                book.smart_update(d['authors'][book_authors],
+                                book.smart_update(db.get_metadata(id_,
-                                                  replace_metadata=True)
+                                                              index_is_id=True,
+                                                              get_cover=get_covers),
+                                              replace_metadata=True)
                             book.in_library = 'AUTHOR'
-                            book.application_id = \
+                            book.application_id = id_
-                                    d['authors'][book_authors].application_id
                         elif book_authors in d['author_sort']:
+                            id_ = d['author_sort'][book_authors]
                             if update_metadata:
-                                book.smart_update(d['author_sort'][book_authors],
+                                book.smart_update(db.get_metadata(id_,
+                                                              index_is_id=True,
+                                                              get_cover=get_covers),
                                                   replace_metadata=True)
                             book.in_library = 'AUTH_SORT'
-                            book.application_id = \
+                            book.application_id = id_
-                                d['author_sort'][book_authors].application_id
                 else:
                     # Book definitely not matched. Clear its application ID
                     book.application_id = None

src/calibre/gui2/device_drivers/configwidget.py

@@ -9,15 +9,16 @@ import textwrap
 from PyQt4.Qt import QWidget, QListWidgetItem, Qt, QVariant, SIGNAL, \
                      QLabel, QLineEdit, QCheckBox
 
-from calibre.gui2 import error_dialog
+from calibre.gui2 import error_dialog, question_dialog
 from calibre.gui2.device_drivers.configwidget_ui import Ui_ConfigWidget
 from calibre.utils.formatter import validation_formatter
+from calibre.ebooks import BOOK_EXTENSIONS
 
 class ConfigWidget(QWidget, Ui_ConfigWidget):
 
     def __init__(self, settings, all_formats, supports_subdirs,
         must_read_metadata, supports_use_author_sort,
-        extra_customization_message):
+        extra_customization_message, device):
 
         QWidget.__init__(self)
         Ui_ConfigWidget.__init__(self)
@@ -25,9 +26,15 @@ class ConfigWidget(QWidget, Ui_ConfigWidget):
 
         self.settings = settings
 
+        all_formats = set(all_formats)
+        self.calibre_known_formats = device.FORMATS
+        self.device_name = device.get_gui_name()
+        if device.USER_CAN_ADD_NEW_FORMATS:
+            all_formats = set(all_formats) | set(BOOK_EXTENSIONS)
+
         format_map = settings.format_map
         disabled_formats = list(set(all_formats).difference(format_map))
-        for format in format_map + disabled_formats:
+        for format in format_map + list(sorted(disabled_formats)):
             item = QListWidgetItem(format, self.columns)
             item.setData(Qt.UserRole, QVariant(format))
             item.setFlags(Qt.ItemIsEnabled|Qt.ItemIsUserCheckable|Qt.ItemIsSelectable)
@@ -110,6 +117,18 @@ class ConfigWidget(QWidget, Ui_ConfigWidget):
         return self.opt_use_author_sort.isChecked()
 
     def validate(self):
+        formats = set(self.format_map())
+        extra = formats - set(self.calibre_known_formats)
+        if extra:
+            fmts = sorted([x.upper() for x in extra])
+            if not question_dialog(self, _('Unknown formats'),
+                    _('You have enabled the <b>{0}</b> formats for'
+                        ' your {1}. The {1} may not support them.'
+                        ' If you send these formats to your {1} they '
+                        'may not work. Are you sure?').format(
+                            (', '.join(fmts)), self.device_name)):
+                return False
+
         tmpl = unicode(self.opt_save_template.text())
         try:
             validation_formatter.validate(tmpl)

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -912,6 +912,7 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
 
     def series_changed(self, *args):
         self.write_series = True
+        self.autonumber_series.setEnabled(True)
 
     def s_r_remove_query(self, *args):
         if self.query_field.currentIndex() == 0:

src/calibre/gui2/dialogs/metadata_bulk.ui

@@ -303,6 +303,9 @@
             <layout class="QHBoxLayout" name="HLayout_3">
              <item>
               <widget class="QCheckBox" name="autonumber_series">
+               <property name="enabled">
+                <bool>false</bool>
+               </property>
                <property name="toolTip">
                 <string>If not checked, the series number for the books will be set to 1.
 If checked, selected books will be automatically numbered, in the order
@@ -1006,8 +1009,8 @@ not multiple and the destination field is multiple</string>
                <rect>
                 <x>0</x>
                 <y>0</y>
-                <width>938</width>
+                <width>197</width>
-                <height>268</height>
+                <height>60</height>
                </rect>
               </property>
               <layout class="QGridLayout" name="testgrid">

src/calibre/gui2/library/models.py

@@ -685,7 +685,7 @@ class BooksModel(QAbstractTableModel): # {{{
                 self.dc[col] = functools.partial(bool_type, idx=idx)
                 self.dc_decorator[col] = functools.partial(
                             bool_type_decorator, idx=idx,
-                            bool_cols_are_tristate=tweaks['bool_custom_columns_are_tristate'] == 'yes')
+                            bool_cols_are_tristate=tweaks['bool_custom_columns_are_tristate'] != 'no')
             elif datatype == 'rating':
                 self.dc[col] = functools.partial(rating_type, idx=idx)
             elif datatype == 'series':

src/calibre/gui2/preferences/search.py

@@ -26,12 +26,19 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         r('limit_search_columns_to', prefs, setting=CommaSeparatedList)
         fl = gui.library_view.model().db.field_metadata.get_search_terms()
         self.opt_limit_search_columns_to.update_items_cache(fl)
+        self.clear_history_button.clicked.connect(self.clear_histories)
 
     def refresh_gui(self, gui):
         gui.search.search_as_you_type(config['search_as_you_type'])
         gui.library_view.model().set_highlight_only(config['highlight_search_matches'])
         gui.search.do_search()
 
+    def clear_histories(self, *args):
+        for key, val in config.defaults.iteritems():
+            if key.endswith('_search_history') and isinstance(val, list):
+                config[key] = []
+        self.gui.search.clear_history()
+
 if __name__ == '__main__':
     app = QApplication([])
     test_widget('Interface', 'Search')

src/calibre/gui2/preferences/search.ui

@@ -77,7 +77,7 @@
      </layout>
     </widget>
    </item>
-   <item row="3" column="0">
+   <item row="4" column="0">
     <spacer name="verticalSpacer">
      <property name="orientation">
       <enum>Qt::Vertical</enum>
@@ -90,13 +90,23 @@
      </property>
     </spacer>
    </item>
+   <item row="3" column="0">
+    <widget class="QPushButton" name="clear_history_button">
+     <property name="toolTip">
+      <string>Clear search histories from all over calibre. Including the book list, e-book viewer, fetch news dialog, etc.</string>
+     </property>
+     <property name="text">
+      <string>Clear search &amp;histories</string>
+     </property>
+    </widget>
+   </item>
   </layout>
  </widget>
  <customwidgets>
   <customwidget>
    <class>MultiCompleteLineEdit</class>
    <extends>QLineEdit</extends>
-   <header>calibre/gui2.complete.h</header>
+   <header>calibre/gui2/complete.h</header>
   </customwidget>
  </customwidgets>
  <resources/>

src/calibre/gui2/search_box.py

@@ -114,6 +114,9 @@ class SearchBox2(QComboBox): # {{{
     def text(self):
         return self.currentText()
 
+    def clear_history(self, *args):
+        QComboBox.clear(self)
+
     def clear(self, emit_search=True):
         self.normalize_state()
         self.setEditText('')

src/calibre/gui2/viewer/main.py

@@ -17,16 +17,16 @@ from calibre.gui2.viewer.bookmarkmanager import BookmarkManager
 from calibre.gui2.widgets import ProgressIndicator
 from calibre.gui2.main_window import MainWindow
 from calibre.gui2 import Application, ORG_NAME, APP_UID, choose_files, \
-                         info_dialog, error_dialog, open_url, available_height
+    info_dialog, error_dialog, open_url, available_height, gprefs
 from calibre.ebooks.oeb.iterator import EbookIterator
 from calibre.ebooks import DRMError
-from calibre.constants import islinux, isfreebsd, isosx
+from calibre.constants import islinux, isfreebsd, isosx, filesystem_encoding
 from calibre.utils.config import Config, StringConfig, dynamic
 from calibre.gui2.search_box import SearchBox2
 from calibre.ebooks.metadata import MetaInformation
 from calibre.customize.ui import available_input_formats
 from calibre.gui2.viewer.dictionary import Lookup
-from calibre import as_unicode
+from calibre import as_unicode, force_unicode, isbytestring
 
 class TOCItem(QStandardItem):
 
@@ -160,6 +160,12 @@ class HelpfulLineEdit(QLineEdit):
         self.setPalette(self.gray)
         self.setText(self.HELP_TEXT)
 
+class RecentAction(QAction):
+
+    def __init__(self, path, parent):
+        self.path = path
+        QAction.__init__(self, os.path.basename(path), parent)
+
 class EbookViewer(MainWindow, Ui_EbookViewer):
 
     STATE_VERSION = 1
@@ -284,8 +290,26 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
         ca = self.view.copy_action
         ca.setShortcut(QKeySequence.Copy)
         self.addAction(ca)
+        self.open_history_menu = QMenu()
+        self.build_recent_menu()
+        self.action_open_ebook.setMenu(self.open_history_menu)
+        self.open_history_menu.triggered[QAction].connect(self.open_recent)
+        w = self.tool_bar.widgetForAction(self.action_open_ebook)
+        w.setPopupMode(QToolButton.MenuButtonPopup)
+
         self.restore_state()
 
+    def build_recent_menu(self):
+        m = self.open_history_menu
+        m.clear()
+        count = 0
+        for path in gprefs.get('viewer_open_history', []):
+            if count > 9:
+                break
+            if os.path.exists(path):
+                m.addAction(RecentAction(path, m))
+                count += 1
+
     def closeEvent(self, e):
         self.save_state()
         return MainWindow.closeEvent(self, e)
@@ -425,6 +449,9 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
         if files:
             self.load_ebook(files[0])
 
+    def open_recent(self, action):
+        self.load_ebook(action.path)
+
     def font_size_larger(self, checked):
         frac = self.view.magnify_fonts()
         self.action_font_size_larger.setEnabled(self.view.multiplier() < 3)
@@ -647,6 +674,17 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
                     self.action_table_of_contents.setChecked(True)
             else:
                 self.action_table_of_contents.setChecked(False)
+            if isbytestring(pathtoebook):
+                pathtoebook = force_unicode(pathtoebook, filesystem_encoding)
+            vh = gprefs.get('viewer_open_history', [])
+            try:
+                vh.remove(pathtoebook)
+            except:
+                pass
+            vh.insert(0, pathtoebook)
+            gprefs.set('viewer_open_history', vh[:50])
+            self.build_recent_menu()
+
             self.action_table_of_contents.setDisabled(not self.iterator.toc)
             self.current_book_has_toc = bool(self.iterator.toc)
             self.current_title = title

src/calibre/library/caches.py

@@ -528,7 +528,7 @@ class ResultCache(SearchQueryParser): # {{{
                 location[i] = db_col[loc]
 
             # get the tweak here so that the string lookup and compare aren't in the loop
-            bools_are_tristate = tweaks['bool_custom_columns_are_tristate'] == 'yes'
+            bools_are_tristate = tweaks['bool_custom_columns_are_tristate'] != 'no'
 
             for loc in location: # location is now an array of field indices
                 if loc == db_col['authors']:
@@ -812,7 +812,10 @@ class SortKeyGenerator(object):
                 val = self.string_sort_key(val)
 
             elif dt == 'bool':
-                val = {True: 1, False: 2, None: 3}.get(val, 3)
+                if tweaks['bool_custom_columns_are_tristate'] == 'no':
+                    val = {True: 1, False: 2, None: 2}.get(val, 2)
+                else:
+                    val = {True: 1, False: 2, None: 3}.get(val, 3)
 
             yield val