Sync to trunk.

recipes/brand_eins.recipe

@@ -3,8 +3,7 @@
 
 __license__   = 'GPL v3'
 __copyright__ = '2010, Constantin Hofstetter <consti at consti.de>, Steffen Siebert <calibre at steffensiebert.de>'
-__version__   = '0.97'
+__version__   = '0.98' # 2011-04-10
-
 ''' http://brandeins.de - Wirtschaftsmagazin '''
 import re
 import string
@@ -14,8 +13,8 @@ from calibre.web.feeds.recipes import BasicNewsRecipe
 class BrandEins(BasicNewsRecipe):
 
   title = u'brand eins'
-  __author__ = 'Constantin Hofstetter'
+  __author__ = 'Constantin Hofstetter; Steffen Siebert'
-  description = u'Wirtschaftsmagazin'
+  description = u'Wirtschaftsmagazin: Gets the last full issue on default. Set a integer value for the username-field to get older issues: 1 -> the newest (but not complete) issue, 2 -> the last complete issue (default), 3 -> the issue before 2 etc.'
   publisher ='brandeins.de'
   category = 'politics, business, wirtschaft, Germany'
   use_embedded_content = False

recipes/dvhn.recipe

@@ -0,0 +1,32 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1302341394(BasicNewsRecipe):
+    title          = u'DvhN'
+    oldest_article = 1
+    max_articles_per_feed = 200
+
+    __author__ = 'Reijndert'
+    no_stylesheets = True
+    cover_url = 'http://www.dvhn.nl/template/Dagblad_v2.0/gfx/logo_DvhN.gif'
+    language = 'nl'
+    country = 'NL'
+    version = 1
+    publisher = u'Dagblad van het Noorden'
+    category = u'Nieuws'
+    description = u'Nieuws uit Noord Nederland'
+
+
+    keep_only_tags = [dict(name='div', attrs={'id':'fullPicture'})
+                               ,dict(name='div', attrs={'id':'articleText'})
+                  ]
+
+    remove_tags = [
+                    dict(name=['object','link','iframe','base'])
+                    ,dict(name='span',attrs={'class':'copyright'})
+                ]
+
+    feeds          = [(u'Drenthe', u'http://www.dvhn.nl/nieuws/drenthe/index.jsp?service=rss'), (u'Groningen', u'http://www.dvhn.nl/nieuws/groningen/index.jsp?service=rss'), (u'Nederland', u'http://www.dvhn.nl/nieuws/nederland/index.jsp?service=rss'), (u'Wereld', u'http://www.dvhn.nl/nieuws/wereld/index.jsp?service=rss'), (u'Economie', u'http://www.dvhn.nl/nieuws/economie/index.jsp?service=rss'), (u'Sport', u'http://www.dvhn.nl/nieuws/sport/index.jsp?service=rss'), (u'Cultuur', u'http://www.dvhn.nl/nieuws/kunst/index.jsp?service=rss'), (u'24 Uur', u'http://www.dvhn.nl/nieuws/24uurdvhn/index.jsp?service=rss&selectiontype=last24hours')]
+
+    extra_css = '''
+                body {font-family: verdana, arial, helvetica, geneva, sans-serif;}
+                '''

resources/default_tweaks.py

@@ -88,13 +88,6 @@ categories_collapsed_rating_template = r'{first.avg_rating:4.2f:ifempty(0)} - {l
 categories_collapsed_popularity_template = r'{first.count:d} - {last.count:d}'
 
 
-#: Set boolean custom columns to be tristate
-# Set whether boolean custom columns are two- or three-valued.
-#  Two-values for true booleans
-#  three-values for yes/no/unknown
-# Set to 'yes' for three-values, 'no' for two-values
-bool_custom_columns_are_tristate = 'yes'
-
 #: Specify columns to sort the booklist by on startup
 # Provide a set of columns to be sorted on when calibre starts
 #  The argument is None if saved sort history is to be used

setup/pygettext.py

@@ -170,8 +170,8 @@ from setup import __appname__, __version__ as version
 # there.
 pot_header = '''\
 # Translation template file..
-# Copyright (C) 2007 Kovid Goyal
+# Copyright (C) %(year)s Kovid Goyal
-# Kovid Goyal <kovid@kovidgoyal.net>, 2007.
+# Kovid Goyal <kovid@kovidgoyal.net>, %(year)s.
 #
 msgid ""
 msgstr ""
@@ -185,7 +185,7 @@ msgstr ""
 "Content-Transfer-Encoding: 8bit\\n"
 "Generated-By: pygettext.py %%(version)s\\n"
 
-'''%dict(appname=__appname__, version=version)
+'''%dict(appname=__appname__, version=version, year=time.strftime('%Y'))
 
 
 def usage(code, msg=''):

setup/translations.py

@@ -26,6 +26,38 @@ class POT(Command):
                     ans.append(os.path.abspath(os.path.join(root, name)))
         return ans
 
+    def get_tweaks_docs(self):
+        path = self.a(self.j(self.SRC, '..', 'resources', 'default_tweaks.py'))
+        with open(path, 'rb') as f:
+            raw = f.read().decode('utf-8')
+        msgs = []
+        lines = list(raw.splitlines())
+        for i, line in enumerate(lines):
+            if line.startswith('#:'):
+                msgs.append((i, line[2:].strip()))
+                j = i
+                block = []
+                while True:
+                    j += 1
+                    line = lines[j]
+                    if not line.startswith('#'):
+                        break
+                    block.append(line[1:].strip())
+                if block:
+                    msgs.append((i+1, '\n'.join(block)))
+
+        ans = []
+        for lineno, msg in msgs:
+            ans.append('#: %s:%d'%(path, lineno))
+            slash = unichr(92)
+            msg = msg.replace(slash, slash*2).replace('"', r'\"').replace('\n',
+                    r'\n').replace('\r', r'\r').replace('\t', r'\t')
+            ans.append('msgid "%s"'%msg)
+            ans.append('msgstr ""')
+            ans.append('')
+
+        return '\n'.join(ans)
+
 
     def run(self, opts):
         files = self.source_files()
@@ -35,10 +67,10 @@ class POT(Command):
         atexit.register(shutil.rmtree, tempdir)
         pygettext(buf, ['-k', '__', '-p', tempdir]+files)
         src = buf.getvalue()
+        src += '\n\n' + self.get_tweaks_docs()
         pot = os.path.join(self.PATH, __appname__+'.pot')
-        f = open(pot, 'wb')
+        with open(pot, 'wb') as f:
-        f.write(src)
+            f.write(src)
-        f.close()
         self.info('Translations template:', os.path.abspath(pot))
         return pot
 

src/calibre/customize/builtins.py

@@ -173,7 +173,7 @@ class ComicMetadataReader(MetadataReaderPlugin):
             stream.seek(pos)
             if id_ == b'Rar':
                 ftype = 'cbr'
-            elif id.startswith(b'PK'):
+            elif id_.startswith(b'PK'):
                 ftype = 'cbz'
         if ftype == 'cbr':
             from calibre.libunrar import extract_first_alphabetically as extract_first
@@ -1038,6 +1038,17 @@ class Server(PreferencesPlugin):
             'give you access to your calibre library from anywhere, '
             'on any device, over the internet')
 
+class MetadataSources(PreferencesPlugin):
+    name = 'Metadata download'
+    icon = I('metadata.png')
+    gui_name = _('Metadata download')
+    category = 'Sharing'
+    gui_category = _('Sharing')
+    category_order = 4
+    name_order = 3
+    config_widget = 'calibre.gui2.preferences.metadata_sources'
+    description = _('Control how calibre downloads ebook metadata from the net')
+
 class Plugins(PreferencesPlugin):
     name = 'Plugins'
     icon = I('plugins.png')
@@ -1076,6 +1087,9 @@ plugins += [LookAndFeel, Behavior, Columns, Toolbar, Search, InputOptions,
         CommonOptions, OutputOptions, Adding, Saving, Sending, Plugboard,
         Email, Server, Plugins, Tweaks, Misc, TemplateFunctions]
 
+if test_eight_code:
+    plugins.append(MetadataSources)
+
 #}}}
 
 

src/calibre/customize/ui.py

@@ -75,6 +75,17 @@ def enable_plugin(plugin_or_name):
     ep.add(x)
     config['enabled_plugins'] = ep
 
+def restore_plugin_state_to_default(plugin_or_name):
+    x = getattr(plugin_or_name, 'name', plugin_or_name)
+    dp = config['disabled_plugins']
+    if x in dp:
+        dp.remove(x)
+    config['disabled_plugins'] = dp
+    ep = config['enabled_plugins']
+    if x in ep:
+        ep.remove(x)
+    config['enabled_plugins'] = ep
+
 default_disabled_plugins = set([
     'Douban Books', 'Douban.com covers', 'Nicebooks', 'Nicebooks covers',
     'Kent District Library'
@@ -453,12 +464,15 @@ def epub_fixers():
 # Metadata sources2 {{{
 def metadata_plugins(capabilities):
     capabilities = frozenset(capabilities)
-    for plugin in _initialized_plugins:
+    for plugin in all_metadata_plugins():
-        if isinstance(plugin, Source) and \
+        if plugin.capabilities.intersection(capabilities) and \
-                plugin.capabilities.intersection(capabilities) and \
                 not is_disabled(plugin):
             yield plugin
 
+def all_metadata_plugins():
+    for plugin in _initialized_plugins:
+        if isinstance(plugin, Source):
+            yield plugin
 # }}}
 
 # Initialize plugins {{{

src/calibre/devices/android/driver.py

@@ -37,7 +37,7 @@ class ANDROID(USBMS):
             0x22b8 : { 0x41d9 : [0x216], 0x2d61 : [0x100], 0x2d67 : [0x100],
                        0x41db : [0x216], 0x4285 : [0x216], 0x42a3 : [0x216],
                        0x4286 : [0x216], 0x42b3 : [0x216], 0x42b4 : [0x216],
-                       0x7086 : [0x0226],
+                       0x7086 : [0x0226], 0x70a8: [0x9999],
                      },
 
             # Sony Ericsson
@@ -96,7 +96,8 @@ class ANDROID(USBMS):
 
     VENDOR_NAME      = ['HTC', 'MOTOROLA', 'GOOGLE_', 'ANDROID', 'ACER',
             'GT-I5700', 'SAMSUNG', 'DELL', 'LINUX', 'GOOGLE', 'ARCHOS',
-            'TELECHIP', 'HUAWEI', 'T-MOBILE', 'SEMC', 'LGE', 'NVIDIA']
+            'TELECHIP', 'HUAWEI', 'T-MOBILE', 'SEMC', 'LGE', 'NVIDIA',
+            'GENERIC-']
     WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE',
             '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897',
             'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID',
@@ -104,7 +105,7 @@ class ANDROID(USBMS):
             'SGH-T849', '_MB300', 'A70S', 'S_ANDROID', 'A101IT', 'A70H',
             'IDEOS_TABLET', 'MYTOUCH_4G', 'UMS_COMPOSITE', 'SCH-I800_CARD',
             '7', 'A956', 'A955', 'A43', 'ANDROID_PLATFORM', 'TEGRA_2',
-            'MB860']
+            'MB860', 'MULTI-CARD']
     WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897',
             'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD',
             'A70S', 'A101IT', '7']

src/calibre/devices/edge/driver.py

@@ -26,9 +26,9 @@ class EDGE(USBMS):
     PRODUCT_ID  = [0x0c02]
     BCD         = [0x0223]
 
-    VENDOR_NAME = 'ANDROID'
+    VENDOR_NAME = ['ANDROID', 'LINUX']
-    WINDOWS_MAIN_MEM = '__FILE-STOR_GADG'
+    WINDOWS_MAIN_MEM = ['__FILE-STOR_GADG', 'FILE-CD_GADGET']
-    WINDOWS_CARD_A_MEM = '__FILE-STOR_GADG'
+    WINDOWS_CARD_A_MEM = ['__FILE-STOR_GADG', 'FILE-CD_GADGET']
 
     MAIN_MEMORY_VOLUME_LABEL  = 'Edge Main Memory'
     STORAGE_CARD_VOLUME_LABEL = 'Edge Storage Card'

src/calibre/ebooks/fb2/output.py

@@ -28,7 +28,7 @@ class FB2Output(OutputFormatPlugin):
             'sf_horror', # Horror & mystic
             'sf_humor', # Humor
             'sf_fantasy', # Fantasy
-            'sf', # Science Fiction 
+            'sf', # Science Fiction
         # Detectives & Thrillers
             'det_classic', # Classical detectives
             'det_police', # Police Stories
@@ -41,20 +41,20 @@ class FB2Output(OutputFormatPlugin):
             'det_maniac', # Maniacs
             'det_hard', # Hard#boiled
             'thriller', # Thrillers
-            'detective', # Detectives 
+            'detective', # Detectives
         # Prose
             'prose_classic', # Classics prose
             'prose_history', # Historical prose
             'prose_contemporary', # Contemporary prose
             'prose_counter', # Counterculture
             'prose_rus_classic', # Russial classics prose
-            'prose_su_classics', # Soviet classics prose 
+            'prose_su_classics', # Soviet classics prose
         # Romance
             'love_contemporary', # Contemporary Romance
             'love_history', # Historical Romance
             'love_detective', # Detective Romance
             'love_short', # Short Romance
-            'love_erotica', # Erotica 
+            'love_erotica', # Erotica
         # Adventure
             'adv_western', # Western
             'adv_history', # History
@@ -62,7 +62,7 @@ class FB2Output(OutputFormatPlugin):
             'adv_maritime', # Maritime Fiction
             'adv_geo', # Travel & geography
             'adv_animal', # Nature & animals
-            'adventure', # Other 
+            'adventure', # Other
         # Children's
             'child_tale', # Fairy Tales
             'child_verse', # Verses
@@ -71,17 +71,17 @@ class FB2Output(OutputFormatPlugin):
             'child_det', # Detectives & Thrillers
             'child_adv', # Adventures
             'child_education', # Educational
-            'children', # Other 
+            'children', # Other
         # Poetry & Dramaturgy
             'poetry', # Poetry
-            'dramaturgy', # Dramaturgy 
+            'dramaturgy', # Dramaturgy
         # Antique literature
             'antique_ant', # Antique
             'antique_european', # European
             'antique_russian', # Old russian
             'antique_east', # Old east
             'antique_myths', # Myths. Legends. Epos
-            'antique', # Other 
+            'antique', # Other
         # Scientific#educational
             'sci_history', # History
             'sci_psychology', # Psychology
@@ -98,7 +98,7 @@ class FB2Output(OutputFormatPlugin):
             'sci_chem', # Chemistry
             'sci_biology', # Biology
             'sci_tech', # Technical
-            'science', # Other 
+            'science', # Other
         # Computers & Internet
             'comp_www', # Internet
             'comp_programming', # Programming
@@ -106,29 +106,29 @@ class FB2Output(OutputFormatPlugin):
             'comp_soft', # Software
             'comp_db', # Databases
             'comp_osnet', # OS & Networking
-            'computers', # Other 
+            'computers', # Other
         # Reference
             'ref_encyc', # Encyclopedias
             'ref_dict', # Dictionaries
             'ref_ref', # Reference
             'ref_guide', # Guidebooks
-            'reference', # Other 
+            'reference', # Other
         # Nonfiction
             'nonf_biography', # Biography & Memoirs
             'nonf_publicism', # Publicism
             'nonf_criticism', # Criticism
             'design', # Art & design
-            'nonfiction', # Other 
+            'nonfiction', # Other
         # Religion & Inspiration
             'religion_rel', # Religion
             'religion_esoterics', # Esoterics
             'religion_self', # Self#improvement
-            'religion', # Other 
+            'religion', # Other
         # Humor
             'humor_anecdote', # Anecdote (funny stories)
             'humor_prose', # Prose
             'humor_verse', # Verses
-            'humor', # Other 
+            'humor', # Other
         # Home & Family
             'home_cooking', # Cooking
             'home_pets', # Pets
@@ -155,14 +155,14 @@ class FB2Output(OutputFormatPlugin):
         OptionRecommendation(name='fb2_genre',
             recommended_value='antique', level=OptionRecommendation.LOW,
             choices=FB2_GENRES,
-            help=_('Genre for the book. Choices: %s\n\n See: ' % FB2_GENRES) + 'http://www.fictionbook.org/index.php/Eng:FictionBook_2.1_genres ' \
+            help=(_('Genre for the book. Choices: %s\n\n See: ') % FB2_GENRES) + 'http://www.fictionbook.org/index.php/Eng:FictionBook_2.1_genres ' \
                 + _('for a complete list with descriptions.')),
     ])
 
     def convert(self, oeb_book, output_path, input_plugin, opts, log):
         from calibre.ebooks.oeb.transforms.jacket import linearize_jacket
         from calibre.ebooks.oeb.transforms.rasterize import SVGRasterizer, Unavailable
-        
+
         try:
             rasterizer = SVGRasterizer()
             rasterizer(oeb_book, opts)

src/calibre/ebooks/metadata/sources/amazon.py

@@ -279,12 +279,13 @@ class Worker(Thread): # Get details {{{
 
 class Amazon(Source):
 
-    name = 'Amazon Store'
+    name = 'Amazon.com'
     description = _('Downloads metadata from Amazon')
 
     capabilities = frozenset(['identify', 'cover'])
     touched_fields = frozenset(['title', 'authors', 'identifier:amazon',
-        'identifier:isbn', 'rating', 'comments', 'publisher', 'pubdate'])
+        'identifier:isbn', 'rating', 'comments', 'publisher', 'pubdate',
+        'language'])
     has_html_comments = True
     supports_gzip_transfer_encoding = True
 
@@ -295,6 +296,14 @@ class Amazon(Source):
             'uk' : _('UK'),
     }
 
+    def get_book_url(self, identifiers): # {{{
+        asin = identifiers.get('amazon', None)
+        if asin is None:
+            asin = identifiers.get('asin', None)
+        if asin:
+            return 'http://amzn.com/%s'%asin
+    # }}}
+
     def create_query(self, log, title=None, authors=None, identifiers={}): # {{{
         domain = self.prefs.get('domain', 'com')
 
@@ -333,9 +342,10 @@ class Amazon(Source):
             # Insufficient metadata to make an identify query
             return None
 
-        utf8q = dict([(x.encode('utf-8'), y.encode('utf-8')) for x, y in
+        latin1q = dict([(x.encode('latin1', 'ignore'), y.encode('latin1',
+            'ignore')) for x, y in
             q.iteritems()])
-        url = 'http://www.amazon.%s/s/?'%domain + urlencode(utf8q)
+        url = 'http://www.amazon.%s/s/?'%domain + urlencode(latin1q)
         return url
 
     # }}}

src/calibre/ebooks/metadata/sources/base.py

@@ -78,8 +78,8 @@ class InternalMetadataCompareKeyGen(object):
         exact_title = 1 if title and \
                 cleanup_title(title) == cleanup_title(mi.title) else 2
 
-        has_cover = 2 if source_plugin.get_cached_cover_url(mi.identifiers)\
+        has_cover = 2 if (not source_plugin.cached_cover_url_is_reliable or
-                is None else 1
+                source_plugin.get_cached_cover_url(mi.identifiers) is None) else 1
 
         self.base = (isbn, has_cover, all_fields, exact_title)
         self.comments_len = len(mi.comments.strip() if mi.comments else '')
@@ -131,7 +131,22 @@ def fixcase(x):
         x = titlecase(x)
     return x
 
+class Option(object):
+    __slots__ = ['type', 'default', 'label', 'desc', 'name', 'choices']
 
+    def __init__(self, name, type_, default, label, desc, choices=None):
+        '''
+        :param name: The name of this option. Must be a valid python identifier
+        :param type_: The type of this option, one of ('number', 'string',
+                        'bool', 'choices')
+        :param default: The default value for this option
+        :param label: A short (few words) description of this option
+        :param desc: A longer description of this option
+        :param choices: A list of possible values, used only if type='choices'
+        '''
+        self.name, self.type, self.default, self.label, self.desc = (name,
+                type_, default, label, desc)
+        self.choices = choices
 
 class Source(Plugin):
 
@@ -157,6 +172,16 @@ class Source(Plugin):
     #: correctly first
     supports_gzip_transfer_encoding = False
 
+    #: Cached cover URLs can sometimes be unreliable (i.e. the download could
+    #: fail or the returned image could be bogus. If that is often the case
+    #: with this source set to False
+    cached_cover_url_is_reliable = True
+
+    #: A list of :class:`Option` objects. They will be used to automatically
+    #: construct the configuration widget for this plugin
+    options = ()
+
+
     def __init__(self, *args, **kwargs):
         Plugin.__init__(self, *args, **kwargs)
         self._isbn_to_identifier_cache = {}
@@ -164,6 +189,9 @@ class Source(Plugin):
         self.cache_lock = threading.RLock()
         self._config_obj = None
         self._browser = None
+        self.prefs.defaults['ignore_fields'] = []
+        for opt in self.options:
+            self.prefs.defaults[opt.name] = opt.default
 
     # Configuration {{{
 
@@ -174,6 +202,16 @@ class Source(Plugin):
         '''
         return True
 
+    def is_customizable(self):
+        return True
+
+    def config_widget(self):
+        from calibre.gui2.metadata.config import ConfigWidget
+        return ConfigWidget(self)
+
+    def save_settings(self, config_widget):
+        config_widget.commit()
+
     @property
     def prefs(self):
         if self._config_obj is None:
@@ -301,6 +339,13 @@ class Source(Plugin):
 
     # Metadata API {{{
 
+    def get_book_url(self, identifiers):
+        '''
+        Return the URL for the book identified by identifiers at this source.
+        If no URL is found, return None.
+        '''
+        return None
+
     def get_cached_cover_url(self, identifiers):
         '''
         Return cached cover URL for the book identified by

src/calibre/ebooks/metadata/sources/google.py

@@ -7,7 +7,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import time
+import time, hashlib
 from urllib import urlencode
 from functools import partial
 from Queue import Queue, Empty
@@ -133,7 +133,7 @@ def to_metadata(browser, log, entry_, timeout): # {{{
             default = utcnow().replace(day=15)
             mi.pubdate = parse_date(pubdate, assume_utc=True, default=default)
         except:
-            log.exception('Failed to parse pubdate')
+            log.error('Failed to parse pubdate %r'%pubdate)
 
     # Ratings
     for x in rating(extra):
@@ -164,9 +164,18 @@ class GoogleBooks(Source):
         'comments', 'publisher', 'identifier:isbn', 'rating',
         'identifier:google']) # language currently disabled
     supports_gzip_transfer_encoding = True
+    cached_cover_url_is_reliable = False
 
     GOOGLE_COVER = 'http://books.google.com/books?id=%s&printsec=frontcover&img=1'
 
+    DUMMY_IMAGE_MD5 = frozenset(['0de4383ebad0adad5eeb8975cd796657'])
+
+    def get_book_url(self, identifiers): # {{{
+        goog = identifiers.get('google', None)
+        if goog is not None:
+            return 'http://books.google.com/books?id=%s'%goog
+    # }}}
+
     def create_query(self, log, title=None, authors=None, identifiers={}): # {{{
         BASE_URL = 'http://books.google.com/books/feeds/volumes?'
         isbn = check_isbn(identifiers.get('isbn', None))
@@ -229,7 +238,11 @@ class GoogleBooks(Source):
         log('Downloading cover from:', cached_url)
         try:
             cdata = br.open_novisit(cached_url, timeout=timeout).read()
-            result_queue.put((self, cdata))
+            if cdata:
+                if hashlib.md5(cdata).hexdigest() in self.DUMMY_IMAGE_MD5:
+                    log.warning('Google returned a dummy image, ignoring')
+                else:
+                    result_queue.put((self, cdata))
         except:
             log.exception('Failed to download cover from:', cached_url)
 

src/calibre/ebooks/metadata/sources/identify.py

@@ -14,7 +14,7 @@ from threading import Thread
 from io import BytesIO
 from operator import attrgetter
 
-from calibre.customize.ui import metadata_plugins
+from calibre.customize.ui import metadata_plugins, all_metadata_plugins
 from calibre.ebooks.metadata.sources.base import create_log, msprefs
 from calibre.ebooks.metadata.xisbn import xisbn
 from calibre.ebooks.metadata.book.base import Metadata
@@ -338,8 +338,9 @@ def identify(log, abort, # {{{
 
         for i, result in enumerate(presults):
             result.relevance_in_source = i
-            result.has_cached_cover_url = \
+            result.has_cached_cover_url = (plugin.cached_cover_url_is_reliable
-                plugin.get_cached_cover_url(result.identifiers) is not None
+                    and plugin.get_cached_cover_url(result.identifiers) is not
+                    None)
             result.identify_plugin = plugin
 
     log('The identify phase took %.2f seconds'%(time.time() - start_time))
@@ -356,16 +357,29 @@ def identify(log, abort, # {{{
             if r.plugin.has_html_comments and r.comments:
                 r.comments = html2text(r.comments)
 
-    dummy = Metadata(_('Unknown'))
     max_tags = msprefs['max_tags']
     for r in results:
-        for f in msprefs['ignore_fields']:
-            setattr(r, f, getattr(dummy, f))
         r.tags = r.tags[:max_tags]
 
     return results
 # }}}
 
+def urls_from_identifiers(identifiers): # {{{
+    ans = []
+    for plugin in all_metadata_plugins():
+        try:
+            url = plugin.get_book_url(identifiers)
+            if url is not None:
+                ans.append((plugin.name, url))
+        except:
+            pass
+    isbn = identifiers.get('isbn', None)
+    if isbn:
+        ans.append(('ISBN',
+            'http://www.worldcat.org/search?q=bn%%3A%s&qt=advanced'%isbn))
+    return ans
+# }}}
+
 if __name__ == '__main__': # tests {{{
     # To run these test use: calibre-debug -e
     # src/calibre/ebooks/metadata/sources/identify.py

src/calibre/ebooks/metadata/sources/openlibrary.py

@@ -12,7 +12,7 @@ from calibre.ebooks.metadata.sources.base import Source
 class OpenLibrary(Source):
 
     name = 'Open Library'
-    description = _('Downloads metadata from The Open Library')
+    description = _('Downloads covers from The Open Library')
 
     capabilities = frozenset(['cover'])
 

src/calibre/gui2/__init__.py

@@ -81,6 +81,7 @@ gprefs.defaults['toolbar_text'] = 'auto'
 gprefs.defaults['font'] = None
 gprefs.defaults['tags_browser_partition_method'] = 'first letter'
 gprefs.defaults['tags_browser_collapse_at'] = 100
+gprefs.defaults['edit_metadata_single_layout'] = 'default'
 
 # }}}
 

src/calibre/gui2/actions/add.py

@@ -22,6 +22,8 @@ from calibre.constants import preferred_encoding, filesystem_encoding
 from calibre.gui2.actions import InterfaceAction
 from calibre.gui2 import config, question_dialog
 from calibre.ebooks.metadata import MetaInformation
+from calibre.utils.config import test_eight_code
+from calibre.ebooks.metadata.sources.base import msprefs
 
 def get_filters():
     return [
@@ -178,13 +180,26 @@ class AddAction(InterfaceAction):
             except IndexError:
                 self.gui.library_view.model().books_added(self.isbn_add_dialog.value)
                 self.isbn_add_dialog.accept()
-                orig = config['overwrite_author_title_metadata']
+                if test_eight_code:
-                config['overwrite_author_title_metadata'] = True
+                    orig = msprefs['ignore_fields']
-                try:
+                    new = list(orig)
-                    self.gui.iactions['Edit Metadata'].do_download_metadata(
+                    for x in ('title', 'authors'):
-                            self.add_by_isbn_ids)
+                        if x in new:
-                finally:
+                            new.remove(x)
-                    config['overwrite_author_title_metadata'] = orig
+                    msprefs['ignore_fields'] = new
+                    try:
+                        self.gui.iactions['Edit Metadata'].download_metadata(
+                            ids=self.add_by_isbn_ids)
+                    finally:
+                        msprefs['ignore_fields'] = orig
+                else:
+                    orig = config['overwrite_author_title_metadata']
+                    config['overwrite_author_title_metadata'] = True
+                    try:
+                        self.gui.iactions['Edit Metadata'].do_download_metadata(
+                                self.add_by_isbn_ids)
+                    finally:
+                        config['overwrite_author_title_metadata'] = orig
                 return
 
 

src/calibre/gui2/actions/device.py

@@ -165,6 +165,10 @@ class ConnectShareAction(InterfaceAction):
 
     def content_server_state_changed(self, running):
         self.share_conn_menu.server_state_changed(running)
+        if running:
+            self.qaction.setIcon(QIcon(I('connect_share_on.png')))
+        else:
+            self.qaction.setIcon(QIcon(I('connect_share.png')))
 
     def toggle_content_server(self):
         if self.gui.content_server is None:

src/calibre/gui2/actions/edit_metadata.py

@@ -35,16 +35,23 @@ class EditMetadataAction(InterfaceAction):
         md.addAction(_('Edit metadata in bulk'),
                 partial(self.edit_metadata, False, bulk=True))
         md.addSeparator()
-        md.addAction(_('Download metadata and covers'),
+        if test_eight_code:
-                partial(self.download_metadata, False, covers=True),
+            dall = self.download_metadata
+            dident = partial(self.download_metadata, covers=False)
+            dcovers = partial(self.download_metadata, identify=False)
+        else:
+            dall = partial(self.download_metadata_old, False, covers=True)
+            dident = partial(self.download_metadata_old, False, covers=False)
+            dcovers = partial(self.download_metadata_old, False, covers=True,
+                    set_metadata=False, set_social_metadata=False)
+
+        md.addAction(_('Download metadata and covers'), dall,
                 Qt.ControlModifier+Qt.Key_D)
-        md.addAction(_('Download only metadata'),
+        md.addAction(_('Download only metadata'), dident)
-                partial(self.download_metadata, False, covers=False))
+        md.addAction(_('Download only covers'), dcovers)
-        md.addAction(_('Download only covers'),
+        if not test_eight_code:
-                partial(self.download_metadata, False, covers=True,
+            md.addAction(_('Download only social metadata'),
-                    set_metadata=False, set_social_metadata=False))
+                partial(self.download_metadata_old, False, covers=False,
-        md.addAction(_('Download only social metadata'),
-                partial(self.download_metadata, False, covers=False,
                     set_metadata=False, set_social_metadata=True))
         self.metadata_menu = md
 
@@ -73,7 +80,16 @@ class EditMetadataAction(InterfaceAction):
         self.qaction.setEnabled(enabled)
         self.action_merge.setEnabled(enabled)
 
-    def download_metadata(self, checked, covers=True, set_metadata=True,
+    def download_metadata(self, identify=True, covers=True, ids=None):
+        if ids is None:
+            rows = self.gui.library_view.selectionModel().selectedRows()
+            if not rows or len(rows) == 0:
+                return error_dialog(self.gui, _('Cannot download metadata'),
+                            _('No books selected'), show=True)
+            db = self.gui.library_view.model().db
+            ids = [db.id(row.row()) for row in rows]
+
+    def download_metadata_old(self, checked, covers=True, set_metadata=True,
             set_social_metadata=None):
         rows = self.gui.library_view.selectionModel().selectedRows()
         if not rows or len(rows) == 0:

src/calibre/gui2/catalog/catalog_epub_mobi.py

@@ -193,7 +193,10 @@ class PluginWidget(QWidget,Ui_Form):
         opts_dict['header_note_source_field'] = self.header_note_source_field_name
 
         # Append the output profile
-        opts_dict['output_profile'] = [load_defaults('page_setup')['output_profile']]
+        try:
+            opts_dict['output_profile'] = [load_defaults('page_setup')['output_profile']]
+        except:
+            opts_dict['output_profile'] = ['default']
         if False:
             print "opts_dict"
             for opt in sorted(opts_dict.keys()):

src/calibre/gui2/dialogs/job_view.ui

@@ -1,7 +1,8 @@
-<ui version="4.0" >
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
  <class>Dialog</class>
- <widget class="QDialog" name="Dialog" >
+ <widget class="QDialog" name="Dialog">
-  <property name="geometry" >
+  <property name="geometry">
    <rect>
     <x>0</x>
     <y>0</y>
@@ -9,38 +10,41 @@
     <height>462</height>
    </rect>
   </property>
-  <property name="windowTitle" >
+  <property name="windowTitle">
    <string>Details of job</string>
   </property>
-  <property name="windowIcon" >
+  <property name="windowIcon">
-   <iconset resource="../../../../resources/images.qrc" >
+   <iconset resource="../../../../resources/images.qrc">
     <normaloff>:/images/view.png</normaloff>:/images/view.png</iconset>
   </property>
-  <layout class="QGridLayout" name="gridLayout" >
+  <layout class="QGridLayout" name="gridLayout">
-   <item row="0" column="0" >
+   <item row="0" column="0">
-    <widget class="QPlainTextEdit" name="log" >
+    <widget class="QPlainTextEdit" name="log">
-     <property name="undoRedoEnabled" >
+     <property name="undoRedoEnabled">
       <bool>false</bool>
      </property>
-     <property name="lineWrapMode" >
+     <property name="lineWrapMode">
       <enum>QPlainTextEdit::NoWrap</enum>
      </property>
-     <property name="readOnly" >
+     <property name="readOnly">
       <bool>true</bool>
      </property>
     </widget>
    </item>
-   <item row="1" column="0" >
+   <item row="2" column="0">
-    <widget class="QDialogButtonBox" name="buttonBox" >
+    <widget class="QDialogButtonBox" name="buttonBox">
-     <property name="standardButtons" >
+     <property name="standardButtons">
       <set>QDialogButtonBox::Ok</set>
      </property>
     </widget>
    </item>
+   <item row="1" column="0">
+    <widget class="QTextBrowser" name="tb"/>
+   </item>
   </layout>
  </widget>
  <resources>
-  <include location="../../../../resources/images.qrc" />
+  <include location="../../../../resources/images.qrc"/>
  </resources>
  <connections>
   <connection>
@@ -49,11 +53,11 @@
    <receiver>Dialog</receiver>
    <slot>accept()</slot>
    <hints>
-    <hint type="sourcelabel" >
+    <hint type="sourcelabel">
      <x>617</x>
      <y>442</y>
     </hint>
-    <hint type="destinationlabel" >
+    <hint type="destinationlabel">
      <x>206</x>
      <y>-5</y>
     </hint>

src/calibre/gui2/email.py

@@ -6,9 +6,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os, socket, time, cStringIO
+import os, socket, time
-from threading import Thread
-from Queue import Queue
 from binascii import unhexlify
 from functools import partial
 from itertools import repeat
@@ -16,67 +14,20 @@ from itertools import repeat
 from calibre.utils.smtp import compose_mail, sendmail, extract_email_address, \
         config as email_config
 from calibre.utils.filenames import ascii_filename
-from calibre.utils.ipc.job import BaseJob
-from calibre.ptempfile import PersistentTemporaryFile
 from calibre.customize.ui import available_input_formats, available_output_formats
 from calibre.ebooks.metadata import authors_to_string
 from calibre.constants import preferred_encoding
 from calibre.gui2 import config, Dispatcher, warning_dialog
 from calibre.library.save_to_disk import get_components
 from calibre.utils.config import tweaks
+from calibre.gui2.threaded_jobs import ThreadedJob
 
-class EmailJob(BaseJob): # {{{
+class Sendmail(object):
-
-    def __init__(self, callback, description, attachment, aname, to, subject, text, job_manager):
-        BaseJob.__init__(self, description)
-        self.exception = None
-        self.job_manager = job_manager
-        self.email_args = (attachment, aname, to, subject, text)
-        self.email_sent_callback = callback
-        self.log_path = None
-        self._log_file = cStringIO.StringIO()
-        self._log_file.write(self.description.encode('utf-8') + '\n')
-
-    @property
-    def log_file(self):
-        if self.log_path is not None:
-            return open(self.log_path, 'rb')
-        return cStringIO.StringIO(self._log_file.getvalue())
-
-    def start_work(self):
-        self.start_time = time.time()
-        self.job_manager.changed_queue.put(self)
-
-    def job_done(self):
-        self.duration = time.time() - self.start_time
-        self.percent = 1
-        # Dump log onto disk
-        lf = PersistentTemporaryFile('email_log')
-        lf.write(self._log_file.getvalue())
-        lf.close()
-        self.log_path = lf.name
-        self._log_file.close()
-        self._log_file = None
-
-        self.job_manager.changed_queue.put(self)
-
-    def log_write(self, what):
-        self._log_file.write(what)
-
-# }}}
-
-class Emailer(Thread): # {{{
 
     MAX_RETRIES = 1
 
-    def __init__(self, job_manager):
+    def __init__(self):
-        Thread.__init__(self)
-        self.daemon = True
-        self.jobs = Queue()
-        self.job_manager = job_manager
-        self._run = True
         self.calculate_rate_limit()
-
         self.last_send_time = time.time() - self.rate_limit
 
     def calculate_rate_limit(self):
@@ -87,70 +38,28 @@ class Emailer(Thread): # {{{
             'gmail.com' in rh or 'live.com' in rh):
             self.rate_limit = tweaks['public_smtp_relay_delay']
 
-    def stop(self):
+    def __call__(self, attachment, aname, to, subject, text, log=None,
-        self._run = False
+            abort=None, notifications=None):
-        self.jobs.put(None)
 
-    def run(self):
+        try_count = 0
-        while self._run:
+        while try_count <= self.MAX_RETRIES:
+            if try_count > 0:
+                log('\nRetrying in %d seconds...\n' %
+                        self.rate_limit)
             try:
-                job = self.jobs.get()
+                self.sendmail(attachment, aname, to, subject, text, log)
+                try_count = self.MAX_RETRIES
+                log('Email successfully sent')
             except:
-                break
+                if abort.is_set():
-            if job is None or not self._run:
+                    return
-                break
+                if try_count == self.MAX_RETRIES:
-            try_count = 0
+                    raise
-            failed, exc = False, None
+                log.exception('\nSending failed...\n')
-            job.start_work()
-            if job.kill_on_start:
-                job.log_write('Aborted\n')
-                job.failed = failed
-                job.killed = True
-                job.job_done()
-                continue
 
-            while try_count <= self.MAX_RETRIES:
+            try_count += 1
-                failed = False
-                if try_count > 0:
-                    job.log_write('\nRetrying in %d seconds...\n' %
-                            self.rate_limit)
-                try:
-                    self.sendmail(job)
-                    break
-                except Exception as e:
-                    if not self._run:
-                        return
-                    import traceback
-                    failed = True
-                    exc = e
-                    job.log_write('\nSending failed...\n')
-                    job.log_write(traceback.format_exc())
 
-                try_count += 1
+    def sendmail(self, attachment, aname, to, subject, text, log):
-
-            if not self._run:
-                break
-
-            job.failed = failed
-            job.exception = exc
-            job.job_done()
-            try:
-                job.email_sent_callback(job)
-            except:
-                import traceback
-                traceback.print_exc()
-
-    def send_mails(self, jobnames, callback, attachments, to_s, subjects,
-                  texts, attachment_names):
-        for name, attachment, to, subject, text, aname in zip(jobnames,
-                attachments, to_s, subjects, texts, attachment_names):
-            description = _('Email %s to %s') % (name, to)
-            job = EmailJob(callback, description, attachment, aname, to,
-                    subject, text, self.job_manager)
-            self.job_manager.add_job(job)
-            self.jobs.put(job)
-
-    def sendmail(self, job):
         while time.time() - self.last_send_time <= self.rate_limit:
             time.sleep(1)
         try:
@@ -158,7 +67,6 @@ class Emailer(Thread): # {{{
             from_ = opts.from_
             if not from_:
                 from_ = 'calibre <calibre@'+socket.getfqdn()+'>'
-            attachment, aname, to, subject, text = job.email_args
             msg = compose_mail(from_, to, text, subject, open(attachment, 'rb'),
                     aname)
             efrom, eto = map(extract_email_address, (from_, to))
@@ -169,48 +77,56 @@ class Emailer(Thread): # {{{
                         username=opts.relay_username,
                         password=unhexlify(opts.relay_password), port=opts.relay_port,
                         encryption=opts.encryption,
-                        debug_output=partial(print, file=job._log_file))
+                        debug_output=log.debug)
         finally:
             self.last_send_time = time.time()
 
-    def email_news(self, mi, remove, get_fmts, done):
+gui_sendmail = Sendmail()
-        opts = email_config().parse()
-        accounts = [(account, [x.strip().lower() for x in x[0].split(',')])
-                for account, x in opts.accounts.items() if x[1]]
-        sent_mails = []
-        for i, x in enumerate(accounts):
-            account, fmts = x
-            files = get_fmts(fmts)
-            files = [f for f in files if f is not None]
-            if not files:
-                continue
-            attachment = files[0]
-            to_s = [account]
-            subjects = [_('News:')+' '+mi.title]
-            texts    = [
-                    _('Attached is the %s periodical downloaded by calibre.')
-                     % (mi.title,)
-                    ]
-            attachment_names = [ascii_filename(mi.title)+os.path.splitext(attachment)[1]]
-            attachments = [attachment]
-            jobnames = [mi.title]
-            do_remove = []
-            if i == len(accounts) - 1:
-                do_remove = remove
-            self.send_mails(jobnames,
-                    Dispatcher(partial(done, remove=do_remove)),
-                    attachments, to_s, subjects, texts, attachment_names)
-            sent_mails.append(to_s[0])
-        return sent_mails
 
 
-# }}}
+def send_mails(jobnames, callback, attachments, to_s, subjects,
+                texts, attachment_names, job_manager):
+    for name, attachment, to, subject, text, aname in zip(jobnames,
+            attachments, to_s, subjects, texts, attachment_names):
+        description = _('Email %s to %s') % (name, to)
+        job = ThreadedJob('email', description, gui_sendmail, (attachment, aname, to,
+                subject, text), {}, callback, killable=False)
+        job_manager.run_threaded_job(job)
+
+
+def email_news(mi, remove, get_fmts, done, job_manager):
+    opts = email_config().parse()
+    accounts = [(account, [x.strip().lower() for x in x[0].split(',')])
+            for account, x in opts.accounts.items() if x[1]]
+    sent_mails = []
+    for i, x in enumerate(accounts):
+        account, fmts = x
+        files = get_fmts(fmts)
+        files = [f for f in files if f is not None]
+        if not files:
+            continue
+        attachment = files[0]
+        to_s = [account]
+        subjects = [_('News:')+' '+mi.title]
+        texts    = [
+                _('Attached is the %s periodical downloaded by calibre.')
+                    % (mi.title,)
+                ]
+        attachment_names = [ascii_filename(mi.title)+os.path.splitext(attachment)[1]]
+        attachments = [attachment]
+        jobnames = [mi.title]
+        do_remove = []
+        if i == len(accounts) - 1:
+            do_remove = remove
+        send_mails(jobnames,
+                Dispatcher(partial(done, remove=do_remove)),
+                attachments, to_s, subjects, texts, attachment_names,
+                job_manager)
+        sent_mails.append(to_s[0])
+    return sent_mails
 
 class EmailMixin(object): # {{{
 
-    def __init__(self):
-        self.emailer = Emailer(self.job_manager)
-
     def send_by_mail(self, to, fmts, delete_from_library, subject='', send_ids=None,
             do_auto_convert=True, specific_format=None):
         ids = [self.library_view.model().id(r) for r in self.library_view.selectionModel().selectedRows()] if send_ids is None else send_ids
@@ -246,8 +162,7 @@ class EmailMixin(object): # {{{
                     components = get_components(subject, mi, id)
                     if not components:
                         components = [mi.title]
-                    subject = os.path.join(*components)
+                    subjects.append(os.path.join(*components))
-                    subjects.append(subject)
                 a = authors_to_string(mi.authors if mi.authors else \
                         [_('Unknown')])
                 texts.append(_('Attached, you will find the e-book') + \
@@ -262,11 +177,10 @@ class EmailMixin(object): # {{{
 
         to_s = list(repeat(to, len(attachments)))
         if attachments:
-            if not self.emailer.is_alive():
+            send_mails(jobnames,
-                self.emailer.start()
-            self.emailer.send_mails(jobnames,
                     Dispatcher(partial(self.email_sent, remove=remove)),
-                    attachments, to_s, subjects, texts, attachment_names)
+                    attachments, to_s, subjects, texts, attachment_names,
+                    self.job_manager)
             self.status_bar.show_message(_('Sending email to')+' '+to, 3000)
 
         auto = []
@@ -334,10 +248,8 @@ class EmailMixin(object): # {{{
             files, auto = self.library_view.model().\
                     get_preferred_formats_from_ids([id_], fmts)
             return files
-        if not self.emailer.is_alive():
+        sent_mails = email_news(mi, remove,
-            self.emailer.start()
+                get_fmts, self.email_sent, self.job_manager)
-        sent_mails = self.emailer.email_news(mi, remove,
-                get_fmts, self.email_sent)
         if sent_mails:
             self.status_bar.show_message(_('Sent news to')+' '+\
                     ', '.join(sent_mails),  3000)

src/calibre/gui2/jobs.py

@@ -8,14 +8,13 @@ Job management.
 '''
 
 import re
-
 from Queue import Empty, Queue
 
-from PyQt4.Qt import QAbstractTableModel, QVariant, QModelIndex, Qt, \
+from PyQt4.Qt import (QAbstractTableModel, QVariant, QModelIndex, Qt,
-    QTimer, pyqtSignal, QIcon, QDialog, QAbstractItemDelegate, QApplication, \
+    QTimer, pyqtSignal, QIcon, QDialog, QAbstractItemDelegate, QApplication,
-    QSize, QStyleOptionProgressBarV2, QString, QStyle, QToolTip, QFrame, \
+    QSize, QStyleOptionProgressBarV2, QString, QStyle, QToolTip, QFrame,
-    QHBoxLayout, QVBoxLayout, QSizePolicy, QLabel, QCoreApplication, QAction, \
+    QHBoxLayout, QVBoxLayout, QSizePolicy, QLabel, QCoreApplication, QAction,
-    QByteArray
+    QByteArray)
 
 from calibre.utils.ipc.server import Server
 from calibre.utils.ipc.job import ParallelJob
@@ -25,8 +24,9 @@ from calibre.gui2.dialogs.jobs_ui import Ui_JobsDialog
 from calibre import __appname__
 from calibre.gui2.dialogs.job_view_ui import Ui_Dialog
 from calibre.gui2.progress_indicator import ProgressIndicator
+from calibre.gui2.threaded_jobs import ThreadedJobServer, ThreadedJob
 
-class JobManager(QAbstractTableModel):
+class JobManager(QAbstractTableModel): # {{{
 
     job_added = pyqtSignal(int)
     job_done  = pyqtSignal(int)
@@ -42,6 +42,7 @@ class JobManager(QAbstractTableModel):
         self.add_job       = Dispatcher(self._add_job)
         self.server        = Server(limit=int(config['worker_limit']/2.0),
                                 enforce_cpu_limit=config['enforce_cpu_limit'])
+        self.threaded_server = ThreadedJobServer()
         self.changed_queue = Queue()
 
         self.timer         = QTimer(self)
@@ -146,12 +147,21 @@ class JobManager(QAbstractTableModel):
                 jobs.add(self.server.changed_jobs_queue.get_nowait())
             except Empty:
                 break
+
+        # Update device jobs
         while True:
             try:
                 jobs.add(self.changed_queue.get_nowait())
             except Empty:
                 break
 
+        # Update threaded jobs
+        while True:
+            try:
+                jobs.add(self.threaded_server.changed_jobs.get_nowait())
+            except Empty:
+                break
+
         if jobs:
             needs_reset = False
             for job in jobs:
@@ -207,11 +217,22 @@ class JobManager(QAbstractTableModel):
         self.server.add_job(job)
         return job
 
+    def run_threaded_job(self, job):
+        self.add_job(job)
+        self.threaded_server.add_job(job)
+
     def launch_gui_app(self, name, args=[], kwargs={}, description=''):
         job = ParallelJob(name, description, lambda x: x,
                 args=args, kwargs=kwargs)
         self.server.run_job(job, gui=True, redirect_output=False)
 
+    def _kill_job(self, job):
+        if isinstance(job, ParallelJob):
+            self.server.kill_job(job)
+        elif isinstance(job, ThreadedJob):
+            self.threaded_server.kill_job(job)
+        else:
+            job.kill_on_start = True
 
     def kill_job(self, row, view):
         job = self.jobs[row]
@@ -221,29 +242,29 @@ class JobManager(QAbstractTableModel):
         if job.duration is not None:
             return error_dialog(view, _('Cannot kill job'),
                          _('Job has already run')).exec_()
-        if isinstance(job, ParallelJob):
+        if not getattr(job, 'killable', True):
-            self.server.kill_job(job)
+            return error_dialog(view, _('Cannot kill job'),
-        else:
+                    _('This job cannot be stopped'), show=True)
-            job.kill_on_start = True
+        self._kill_job(job)
 
     def kill_all_jobs(self):
         for job in self.jobs:
-            if isinstance(job, DeviceJob) or job.duration is not None:
+            if (isinstance(job, DeviceJob) or job.duration is not None or
+                    not getattr(job, 'killable', True)):
                 continue
-            if isinstance(job, ParallelJob):
+            self._kill_job(job)
-                self.server.kill_job(job)
-            else:
-                job.kill_on_start = True
 
     def terminate_all_jobs(self):
         self.server.killall()
         for job in self.jobs:
-            if isinstance(job, DeviceJob) or job.duration is not None:
+            if (isinstance(job, DeviceJob) or job.duration is not None or
+                    not getattr(job, 'killable', True)):
                 continue
             if not isinstance(job, ParallelJob):
-                job.kill_on_start = True
+                self._kill_job(job)
-
+# }}}
 
+# Jobs UI {{{
 class ProgressBarDelegate(QAbstractItemDelegate):
 
     def sizeHint(self, option, index):
@@ -269,6 +290,11 @@ class DetailView(QDialog, Ui_Dialog):
         self.setupUi(self)
         self.setWindowTitle(job.description)
         self.job = job
+        self.html_view = hasattr(job, 'html_details')
+        if self.html_view:
+            self.log.setVisible(False)
+        else:
+            self.tb.setVisible(False)
         self.next_pos = 0
         self.update()
         self.timer = QTimer(self)
@@ -277,12 +303,19 @@ class DetailView(QDialog, Ui_Dialog):
 
 
     def update(self):
-        f = self.job.log_file
+        if self.html_view:
-        f.seek(self.next_pos)
+            html = self.job.html_details
-        more = f.read()
+            if len(html) > self.next_pos:
-        self.next_pos = f.tell()
+                self.next_pos = len(html)
-        if more:
+                self.tb.setHtml(
-            self.log.appendPlainText(more.decode('utf-8', 'replace'))
+                    '<pre style="font-family:monospace">%s</pre>'%html)
+        else:
+            f = self.job.log_file
+            f.seek(self.next_pos)
+            more = f.read()
+            self.next_pos = f.tell()
+            if more:
+                self.log.appendPlainText(more.decode('utf-8', 'replace'))
 
 class JobsButton(QFrame):
 
@@ -441,3 +474,5 @@ class JobsDialog(QDialog, Ui_JobsDialog):
     def hide(self, *args):
         self.save_state()
         return QDialog.hide(self, *args)
+# }}}
+

src/calibre/gui2/layout.py

@@ -320,7 +320,8 @@ class BaseToolBar(QToolBar): # {{{
         self.setOrientation(Qt.Horizontal)
         self.setAllowedAreas(Qt.TopToolBarArea|Qt.BottomToolBarArea)
         self.setStyleSheet('QToolButton:checked { font-weight: bold }')
-    
+        self.preferred_width = self.sizeHint().width()
+
     def resizeEvent(self, ev):
         QToolBar.resizeEvent(self, ev)
         style = self.get_text_style()
@@ -333,10 +334,13 @@ class BaseToolBar(QToolBar): # {{{
             p = gprefs['toolbar_text']
             if p == 'never':
                 style = Qt.ToolButtonIconOnly
-            elif p == 'auto' and self.sizeHint().width() > self.width()+35:
+            elif p == 'auto' and self.preferred_width > self.width()+35:
                 style = Qt.ToolButtonIconOnly
         return style
 
+    def contextMenuEvent(self, *args):
+        pass
+
 # }}}
 
 class ToolBar(BaseToolBar): # {{{
@@ -368,9 +372,6 @@ class ToolBar(BaseToolBar): # {{{
         self.child_bar.setToolButtonStyle(style)
         self.donate_button.set_normal_icon_size(sz, sz)
 
-    def contextMenuEvent(self, *args):
-        pass
-
     def build_bar(self):
         self.showing_donate = False
         showing_device = self.location_manager.has_device
@@ -405,7 +406,8 @@ class ToolBar(BaseToolBar): # {{{
                     self.d_widget = QWidget()
                     self.d_widget.setLayout(QVBoxLayout())
                     self.d_widget.layout().addWidget(self.donate_button)
-                    self.d_widget.setStyleSheet('QWidget, QToolButton {background-color: none; border: none; }')
+                    if isosx:
+                        self.d_widget.setStyleSheet('QWidget, QToolButton {background-color: none; border: none; }')
                     bar.addWidget(self.d_widget)
                     self.showing_donate = True
                 elif what in self.gui.iactions:
@@ -413,6 +415,8 @@ class ToolBar(BaseToolBar): # {{{
                     bar.addAction(action.qaction)
                     self.added_actions.append(action.qaction)
                     self.setup_tool_button(bar, action.qaction, action.popup_type)
+        self.preferred_width = self.sizeHint().width()
+        self.child_bar.preferred_width = self.child_bar.sizeHint().width()
 
     def setup_tool_button(self, bar, ac, menu_mode=None):
         ch = bar.widgetForAction(ac)

src/calibre/gui2/library/models.py

@@ -604,7 +604,10 @@ class BooksModel(QAbstractTableModel): # {{{
         def size(r, idx=-1):
             size = self.db.data[r][idx]
             if size:
-                return QVariant('%.1f'%(float(size)/(1024*1024)))
+                ans = '%.1f'%(float(size)/(1024*1024))
+                if size > 0 and ans == '0.0':
+                    ans = '<0.1'
+                return QVariant(ans)
             return None
 
         def rating_type(r, idx=-1):

src/calibre/gui2/metadata/basic_widgets.py

@@ -9,7 +9,7 @@ __docformat__ = 'restructuredtext en'
 
 import textwrap, re, os
 
-from PyQt4.Qt import (Qt, QDateEdit, QDate,
+from PyQt4.Qt import (Qt, QDateEdit, QDate, pyqtSignal,
     QIcon, QToolButton, QWidget, QLabel, QGridLayout,
     QDoubleSpinBox, QListWidgetItem, QSize, QPixmap,
     QPushButton, QSpinBox, QLineEdit, QSizePolicy)
@@ -172,6 +172,7 @@ class AuthorsEdit(MultiCompleteComboBox):
         self.books_to_refresh = set([])
         all_authors = db.all_authors()
         all_authors.sort(key=lambda x : sort_key(x[1]))
+        self.clear()
         for i in all_authors:
             id, name = i
             name = [name.strip().replace('|', ',') for n in name.split(',')]
@@ -315,7 +316,7 @@ class SeriesEdit(MultiCompleteComboBox):
             if not val:
                 val = ''
             self.setEditText(val.strip())
-            self.setCursorPosition(0)
+            self.lineEdit().setCursorPosition(0)
 
         return property(fget=fget, fset=fset)
 
@@ -326,6 +327,7 @@ class SeriesEdit(MultiCompleteComboBox):
         self.update_items_cache([x[1] for x in all_series])
         series_id = db.series_id(id_, index_is_id=True)
         idx, c = None, 0
+        self.clear()
         for i in all_series:
             id, name = i
             if id == series_id:
@@ -613,6 +615,8 @@ class FormatsManager(QWidget): # {{{
 
 class Cover(ImageView): # {{{
 
+    download_cover = pyqtSignal()
+
     def __init__(self, parent):
         ImageView.__init__(self, parent)
         self.dialog = parent
@@ -703,9 +707,6 @@ class Cover(ImageView): # {{{
         cdata = im.export('png')
         self.current_val = cdata
 
-    def download_cover(self, *args):
-        pass # TODO: Implement this
-
     def generate_cover(self, *args):
         from calibre.ebooks import calibre_cover
         from calibre.ebooks.metadata import fmt_sidx
@@ -862,6 +863,7 @@ class TagsEdit(MultiCompleteLineEdit): # {{{
             if not val:
                 val = []
             self.setText(', '.join([x.strip() for x in val]))
+            self.setCursorPosition(0)
         return property(fget=fget, fset=fset)
 
     def initialize(self, db, id_):
@@ -928,6 +930,7 @@ class IdentifiersEdit(QLineEdit): # {{{
                 val = {}
             txt = ', '.join(['%s:%s'%(k, v) for k, v in val.iteritems()])
             self.setText(txt.strip())
+            self.setCursorPosition(0)
         return property(fget=fget, fset=fset)
 
     def initialize(self, db, id_):
@@ -977,7 +980,7 @@ class PublisherEdit(MultiCompleteComboBox): # {{{
             if not val:
                 val = ''
             self.setEditText(val.strip())
-            self.setCursorPosition(0)
+            self.lineEdit().setCursorPosition(0)
 
         return property(fget=fget, fset=fset)
 
@@ -987,13 +990,13 @@ class PublisherEdit(MultiCompleteComboBox): # {{{
         all_publishers.sort(key=lambda x : sort_key(x[1]))
         self.update_items_cache([x[1] for x in all_publishers])
         publisher_id = db.publisher_id(id_, index_is_id=True)
-        idx, c = None, 0
+        idx = None
-        for i in all_publishers:
+        self.clear()
-            id, name = i
+        for i, x in enumerate(all_publishers):
-            if id == publisher_id:
+            id_, name = x
-                idx = c
+            if id_ == publisher_id:
+                idx = i
             self.addItem(name)
-            c += 1
 
         self.setEditText('')
         if idx is not None:

src/calibre/gui2/metadata/bulk_download2.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+

src/calibre/gui2/metadata/config.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import textwrap
+
+from PyQt4.Qt import (QWidget, QGridLayout, QGroupBox, QListView, Qt, QSpinBox,
+        QDoubleSpinBox, QCheckBox, QLineEdit, QComboBox, QLabel)
+
+from calibre.gui2.preferences.metadata_sources import FieldsModel as FM
+
+class FieldsModel(FM): # {{{
+
+    def __init__(self, plugin):
+        FM.__init__(self)
+        self.plugin = plugin
+        self.exclude = frozenset(['title', 'authors']) | self.exclude
+        self.prefs = self.plugin.prefs
+
+    def initialize(self):
+        fields = self.plugin.touched_fields
+        self.fields = []
+        for x in fields:
+            if not x.startswith('identifier:') and x not in self.exclude:
+                self.fields.append(x)
+        self.fields.sort(key=lambda x:self.descs.get(x, x))
+        self.reset()
+
+    def state(self, field, defaults=False):
+        src = self.prefs.defaults if defaults else self.prefs
+        return (Qt.Unchecked if field in src['ignore_fields']
+                    else Qt.Checked)
+
+    def restore_defaults(self):
+        self.overrides = dict([(f, self.state(f, True)) for f in self.fields])
+        self.reset()
+
+    def commit(self):
+        val = [k for k, v in self.overrides.iteritems() if v == Qt.Unchecked]
+        self.prefs['ignore_fields'] = val
+
+# }}}
+
+class ConfigWidget(QWidget):
+
+    def __init__(self, plugin):
+        QWidget.__init__(self)
+        self.plugin = plugin
+
+        self.l = l = QGridLayout()
+        self.setLayout(l)
+
+        self.gb = QGroupBox(_('Downloaded metadata fields'), self)
+        l.addWidget(self.gb, 0, 0, 1, 2)
+        self.gb.l = QGridLayout()
+        self.gb.setLayout(self.gb.l)
+        self.fields_view = v = QListView(self)
+        self.gb.l.addWidget(v, 0, 0)
+        v.setFlow(v.LeftToRight)
+        v.setWrapping(True)
+        v.setResizeMode(v.Adjust)
+        self.fields_model = FieldsModel(self.plugin)
+        self.fields_model.initialize()
+        v.setModel(self.fields_model)
+
+        self.memory = []
+        self.widgets = []
+        for opt in plugin.options:
+            self.create_widgets(opt)
+
+    def create_widgets(self, opt):
+        val = self.plugin.prefs[opt.name]
+        if opt.type == 'number':
+            c = QSpinBox if isinstance(opt.default, int) else QDoubleSpinBox
+            widget = c(self)
+            widget.setValue(val)
+        elif opt.type == 'string':
+            widget = QLineEdit(self)
+            widget.setText(val)
+        elif opt.type == 'bool':
+            widget = QCheckBox(opt.label, self)
+            widget.setChecked(bool(val))
+        elif opt.type == 'choices':
+            widget = QComboBox(self)
+            for x in opt.choices:
+                widget.addItem(x)
+            idx = opt.choices.index(val)
+            widget.setCurrentIndex(idx)
+        widget.opt = opt
+        widget.setToolTip(textwrap.fill(opt.desc))
+        self.widgets.append(widget)
+        r = self.l.rowCount()
+        if opt.type == 'bool':
+            self.l.addWidget(widget, r, 0, 1, self.l.columnCount())
+        else:
+            l = QLabel(opt.label)
+            l.setToolTip(widget.toolTip())
+            self.memory.append(l)
+            l.setBuddy(widget)
+            self.l.addWidget(l, r, 0, 1, 1)
+            self.l.addWidget(widget, r, 1, 1, 1)
+
+
+    def commit(self):
+        self.fields_model.commit()
+        for w in self.widgets:
+            if isinstance(w, (QSpinBox, QDoubleSpinBox)):
+                val = w.value()
+            elif isinstance(w, QLineEdit):
+                val = unicode(w.text())
+            elif isinstance(w, QCheckBox):
+                val = w.isChecked()
+            elif isinstance(w, QComboBox):
+                val = unicode(w.currentText())
+            self.plugin.prefs[w.opt.name] = val
+
+

src/calibre/gui2/metadata/single.py

@@ -16,13 +16,15 @@ from PyQt4.Qt import (Qt, QVBoxLayout, QHBoxLayout, QWidget, QPushButton,
         QSizePolicy, QPalette, QFrame, QSize, QKeySequence)
 
 from calibre.ebooks.metadata import authors_to_string, string_to_authors
-from calibre.gui2 import ResizableDialog, error_dialog, gprefs
+from calibre.gui2 import ResizableDialog, error_dialog, gprefs, pixmap_to_data
 from calibre.gui2.metadata.basic_widgets import (TitleEdit, AuthorsEdit,
     AuthorSortEdit, TitleSortEdit, SeriesEdit, SeriesIndexEdit, IdentifiersEdit,
     RatingEdit, PublisherEdit, TagsEdit, FormatsManager, Cover, CommentsEdit,
     BuddyLabel, DateEdit, PubdateEdit)
+from calibre.gui2.metadata.single_download import FullFetch
 from calibre.gui2.custom_column_widgets import populate_metadata_page
 from calibre.utils.config import tweaks
+from calibre.ebooks.metadata.book.base import Metadata
 
 class MetadataSingleDialogBase(ResizableDialog):
 
@@ -132,6 +134,7 @@ class MetadataSingleDialogBase(ResizableDialog):
         self.formats_manager.cover_from_format_button.clicked.connect(
                 self.cover_from_format)
         self.cover = Cover(self)
+        self.cover.download_cover.connect(self.download_cover)
         self.basic_metadata_widgets.append(self.cover)
 
         self.comments = CommentsEdit(self, self.one_line_comments_toolbar)
@@ -158,12 +161,17 @@ class MetadataSingleDialogBase(ResizableDialog):
         self.basic_metadata_widgets.extend([self.timestamp, self.pubdate])
 
         self.fetch_metadata_button = QPushButton(
-                _('&Fetch metadata from server'), self)
+                _('&Download metadata'), self)
         self.fetch_metadata_button.clicked.connect(self.fetch_metadata)
         font = self.fmb_font = QFont()
         font.setBold(True)
         self.fetch_metadata_button.setFont(font)
 
+        self.config_metadata_button = QToolButton(self)
+        self.config_metadata_button.setIcon(QIcon(I('config.png')))
+        self.config_metadata_button.clicked.connect(self.configure_metadata)
+        self.config_metadata_button.setToolTip(
+            _('Change how calibre downloads metadata'))
 
     # }}}
 
@@ -303,7 +311,36 @@ class MetadataSingleDialogBase(ResizableDialog):
             self.comments.current_val = mi.comments
 
     def fetch_metadata(self, *args):
-        pass # TODO: fetch metadata
+        d = FullFetch(self.cover.pixmap(), self)
+        ret = d.start(title=self.title.current_val, authors=self.authors.current_val,
+                identifiers=self.identifiers.current_val)
+        if ret == d.Accepted:
+            from calibre.ebooks.metadata.sources.base import msprefs
+            mi = d.book
+            dummy = Metadata(_('Unknown'))
+            for f in msprefs['ignore_fields']:
+                setattr(mi, f, getattr(dummy, f))
+            if mi is not None:
+                self.update_from_mi(mi)
+            if d.cover_pixmap is not None:
+                self.cover.current_val = pixmap_to_data(d.cover_pixmap)
+
+    def configure_metadata(self):
+        from calibre.gui2.preferences import show_config_widget
+        gui = self.parent()
+        show_config_widget('Sharing', 'Metadata download', parent=self,
+                gui=gui, never_shutdown=True)
+
+    def download_cover(self, *args):
+        from calibre.gui2.metadata.single_download import CoverFetch
+        d = CoverFetch(self.cover.pixmap(), self)
+        ret = d.start(self.title.current_val, self.authors.current_val,
+                self.identifiers.current_val)
+        if ret == d.Accepted:
+            if d.cover_pixmap is not None:
+                self.cover.current_val = pixmap_to_data(d.cover_pixmap)
+
+
     # }}}
 
     def apply_changes(self):
@@ -430,7 +467,8 @@ class MetadataSingleDialog(MetadataSingleDialogBase): # {{{
 
         sto = QWidget.setTabOrder
         sto(self.button_box, self.fetch_metadata_button)
-        sto(self.fetch_metadata_button, self.title)
+        sto(self.fetch_metadata_button, self.config_metadata_button)
+        sto(self.config_metadata_button, self.title)
 
         def create_row(row, one, two, three, col=1, icon='forward.png'):
             ql = BuddyLabel(one)
@@ -509,7 +547,8 @@ class MetadataSingleDialog(MetadataSingleDialogBase): # {{{
         self.tabs[0].spc_two = QSpacerItem(10, 10, QSizePolicy.Expanding,
                 QSizePolicy.Expanding)
         l.addItem(self.tabs[0].spc_two, 8, 0, 1, 3)
-        l.addWidget(self.fetch_metadata_button, 9, 0, 1, 3)
+        l.addWidget(self.fetch_metadata_button, 9, 0, 1, 2)
+        l.addWidget(self.config_metadata_button, 9, 2, 1, 1)
 
         self.tabs[0].gb2 = gb = QGroupBox(_('Co&mments'), self)
         gb.l = l = QVBoxLayout()
@@ -521,18 +560,35 @@ class MetadataSingleDialog(MetadataSingleDialogBase): # {{{
 
 # }}}
 
+class DragTrackingWidget(QWidget): # {{{
+
+    def __init__(self, parent, on_drag_enter):
+        QWidget.__init__(self, parent)
+        self.on_drag_enter = on_drag_enter
+
+    def dragEnterEvent(self, ev):
+        self.on_drag_enter.emit()
+
+# }}}
+
 class MetadataSingleDialogAlt1(MetadataSingleDialogBase): # {{{
 
     cc_two_column = False
     one_line_comments_toolbar = True
 
+    on_drag_enter = pyqtSignal()
+
+    def handle_drag_enter(self):
+        self.central_widget.setCurrentIndex(1)
+
     def do_layout(self):
         self.central_widget.clear()
         self.tabs = []
         self.labels = []
         sto = QWidget.setTabOrder
 
-        self.tabs.append(QWidget(self))
+        self.on_drag_enter.connect(self.handle_drag_enter)
+        self.tabs.append(DragTrackingWidget(self, self.on_drag_enter))
         self.central_widget.addTab(self.tabs[0], _("&Metadata"))
         self.tabs[0].l = QGridLayout()
         self.tabs[0].setLayout(self.tabs[0].l)
@@ -542,6 +598,10 @@ class MetadataSingleDialogAlt1(MetadataSingleDialogBase): # {{{
         self.tabs[1].l = QGridLayout()
         self.tabs[1].setLayout(self.tabs[1].l)
 
+        # accept drop events so we can automatically switch to the second tab to
+        # drop covers and formats
+        self.tabs[0].setAcceptDrops(True)
+
         # Tab 0
         tab0 = self.tabs[0]
 
@@ -550,6 +610,12 @@ class MetadataSingleDialogAlt1(MetadataSingleDialogBase): # {{{
         self.tabs[0].l.addWidget(gb, 0, 0, 1, 1)
         gb.setLayout(tl)
 
+        self.button_box.addButton(self.fetch_metadata_button,
+                                  QDialogButtonBox.ActionRole)
+        self.config_metadata_button.setToolButtonStyle(Qt.ToolButtonTextOnly)
+        self.config_metadata_button.setText(_('Configure metadata downloading'))
+        self.button_box.addButton(self.config_metadata_button,
+                                  QDialogButtonBox.ActionRole)
         sto(self.button_box, self.title)
 
         def create_row(row, widget, tab_to, button=None, icon=None, span=1):
@@ -639,7 +705,6 @@ class MetadataSingleDialogAlt1(MetadataSingleDialogBase): # {{{
         wgl.addWidget(gb)
         wgl.addItem(QSpacerItem(10, 10, QSizePolicy.Expanding,
             QSizePolicy.Expanding))
-        wgl.addWidget(self.fetch_metadata_button)
         wgl.addItem(QSpacerItem(10, 10, QSizePolicy.Expanding,
             QSizePolicy.Expanding))
         wgl.addWidget(self.formats_manager)
@@ -658,7 +723,7 @@ editors = {'default': MetadataSingleDialog, 'alt1': MetadataSingleDialogAlt1}
 
 def edit_metadata(db, row_list, current_row, parent=None, view_slot=None,
         set_current_callback=None):
-    cls = db.prefs.get('edit_metadata_single_layout', '')
+    cls = gprefs.get('edit_metadata_single_layout', '')
     if cls not in editors:
         cls = 'default'
     d = editors[cls](db, parent)

src/calibre/gui2/metadata/single_download.py

@@ -7,23 +7,31 @@ __license__   = 'GPL v3'
 __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
+DEBUG_DIALOG = False
+
+# Imports {{{
 from threading import Thread, Event
 from operator import attrgetter
+from Queue import Queue, Empty
 
 from PyQt4.Qt import (QStyledItemDelegate, QTextDocument, QRectF, QIcon, Qt,
         QStyle, QApplication, QDialog, QVBoxLayout, QLabel, QDialogButtonBox,
         QStackedWidget, QWidget, QTableView, QGridLayout, QFontInfo, QPalette,
-        QTimer, pyqtSignal, QAbstractTableModel, QVariant, QSize)
+        QTimer, pyqtSignal, QAbstractTableModel, QVariant, QSize, QListView,
+        QPixmap, QAbstractListModel, QColor, QRect, QTextBrowser)
 from PyQt4.QtWebKit import QWebView
 
 from calibre.customize.ui import metadata_plugins
 from calibre.ebooks.metadata import authors_to_string
 from calibre.utils.logging import GUILog as Log
-from calibre.ebooks.metadata.sources.identify import identify
+from calibre.ebooks.metadata.sources.identify import (identify,
+        urls_from_identifiers)
 from calibre.ebooks.metadata.book.base import Metadata
 from calibre.gui2 import error_dialog, NONE
 from calibre.utils.date import utcnow, fromordinal, format_date
 from calibre.library.comments import comments_to_html
+from calibre import force_unicode
+# }}}
 
 class RichTextDelegate(QStyledItemDelegate): # {{{
 
@@ -36,7 +44,10 @@ class RichTextDelegate(QStyledItemDelegate): # {{{
         return doc
 
     def sizeHint(self, option, index):
-        ans = self.to_doc(index).size().toSize()
+        doc = self.to_doc(index)
+        ans = doc.size().toSize()
+        if ans.width() > 150:
+            ans.setWidth(160)
         ans.setHeight(ans.height()+10)
         return ans
 
@@ -52,6 +63,65 @@ class RichTextDelegate(QStyledItemDelegate): # {{{
         painter.restore()
 # }}}
 
+class CoverDelegate(QStyledItemDelegate): # {{{
+
+    needs_redraw = pyqtSignal()
+
+    def __init__(self, parent):
+        QStyledItemDelegate.__init__(self, parent)
+
+        self.angle = 0
+        self.timer = QTimer(self)
+        self.timer.timeout.connect(self.frame_changed)
+        self.color = parent.palette().color(QPalette.WindowText)
+        self.spinner_width = 64
+
+    def frame_changed(self, *args):
+        self.angle = (self.angle+30)%360
+        self.needs_redraw.emit()
+
+    def start_animation(self):
+        self.angle = 0
+        self.timer.start(200)
+
+    def stop_animation(self):
+        self.timer.stop()
+
+    def draw_spinner(self, painter, rect):
+        width = rect.width()
+
+        outer_radius = (width-1)*0.5
+        inner_radius = (width-1)*0.5*0.38
+
+        capsule_height = outer_radius - inner_radius
+        capsule_width  = int(capsule_height * (0.23 if width > 32 else 0.35))
+        capsule_radius = capsule_width//2
+
+        painter.save()
+        painter.setRenderHint(painter.Antialiasing)
+
+        for i in xrange(12):
+            color = QColor(self.color)
+            color.setAlphaF(1.0 - (i/12.0))
+            painter.setPen(Qt.NoPen)
+            painter.setBrush(color)
+            painter.save()
+            painter.translate(rect.center())
+            painter.rotate(self.angle - i*30.0)
+            painter.drawRoundedRect(-capsule_width*0.5,
+                    -(inner_radius+capsule_height), capsule_width,
+                    capsule_height, capsule_radius, capsule_radius)
+            painter.restore()
+        painter.restore()
+
+    def paint(self, painter, option, index):
+        QStyledItemDelegate.paint(self, painter, option, index)
+        if self.timer.isActive() and index.data(Qt.UserRole).toBool():
+            rect = QRect(0, 0, self.spinner_width, self.spinner_width)
+            rect.moveCenter(option.rect.center())
+            self.draw_spinner(painter, rect)
+# }}}
+
 class ResultsModel(QAbstractTableModel): # {{{
 
     COLUMNS = (
@@ -110,6 +180,13 @@ class ResultsModel(QAbstractTableModel): # {{{
                 return self.yes_icon
         elif role == Qt.UserRole:
             return book
+        elif role == Qt.ToolTipRole and col == 3:
+            return QVariant(
+                _('The has cover indication is not fully\n'
+                    'reliable. Sometimes results marked as not\n'
+                    'having a cover will find a cover in the download\n'
+                    'cover stage, and vice versa.'))
+
         return NONE
 
     def sort(self, col, order=Qt.AscendingOrder):
@@ -119,7 +196,7 @@ class ResultsModel(QAbstractTableModel): # {{{
         elif col == 1:
             key = attrgetter('title')
         elif col == 2:
-            key = attrgetter('authors')
+            key = attrgetter('pubdate')
         elif col == 3:
             key = attrgetter('has_cached_cover_url')
         elif key == 4:
@@ -170,6 +247,11 @@ class ResultsView(QTableView): # {{{
         if not book.is_null('rating'):
             parts.append('<div>%s</div>'%('\u2605'*int(book.rating)))
         parts.append('</center>')
+        if book.identifiers:
+            urls = urls_from_identifiers(book.identifiers)
+            ids = ['<a href="%s">%s</a>'%(url, name) for name, url in urls]
+            if ids:
+                parts.append('<div><b>%s:</b> %s</div><br>'%(_('See at'), ', '.join(ids)))
         if book.tags:
             parts.append('<div>%s</div><div>\u00a0</div>'%', '.join(book.tags))
         if book.comments:
@@ -201,6 +283,14 @@ class Comments(QWebView): # {{{
         self.page().setPalette(palette)
         self.setAttribute(Qt.WA_OpaquePaintEvent, False)
 
+        self.page().setLinkDelegationPolicy(self.page().DelegateAllLinks)
+        self.linkClicked.connect(self.link_clicked)
+
+    def link_clicked(self, url):
+        from calibre.gui2 import open_url
+        if unicode(url.toString()).startswith('http://'):
+            open_url(url)
+
     def turnoff_scrollbar(self, *args):
         self.page().mainFrame().setScrollBarPolicy(Qt.Horizontal, Qt.ScrollBarAlwaysOff)
 
@@ -268,7 +358,7 @@ class IdentifyWorker(Thread): # {{{
 
     def run(self):
         try:
-            if True:
+            if DEBUG_DIALOG:
                 self.results = self.sample_results()
             else:
                 self.results = identify(self.log, self.abort, title=self.title,
@@ -277,7 +367,7 @@ class IdentifyWorker(Thread): # {{{
                 result.gui_rank = i
         except:
             import traceback
-            self.error = traceback.format_exc()
+            self.error = force_unicode(traceback.format_exc())
 # }}}
 
 class IdentifyWidget(QWidget): # {{{
@@ -318,7 +408,7 @@ class IdentifyWidget(QWidget): # {{{
         self.query.setWordWrap(True)
         l.addWidget(self.query, 2, 0, 1, 2)
 
-        self.comments_view.show_data('<h2>'+_('Downloading')+
+        self.comments_view.show_data('<h2>'+_('Please wait')+
                 '<br><span id="dots">.</span></h2>'+
                 '''
                 <script type="text/javascript">
@@ -345,7 +435,7 @@ class IdentifyWidget(QWidget): # {{{
         if authors:
             parts.append('authors:'+authors_to_string(authors))
         if identifiers:
-            x = ', '.join('%s:%s'%(k, v) for k, v in identifiers)
+            x = ', '.join('%s:%s'%(k, v) for k, v in identifiers.iteritems())
             parts.append(x)
         self.query.setText(_('Query: ')+'; '.join(parts))
         self.log(unicode(self.query.text()))
@@ -398,23 +488,323 @@ class IdentifyWidget(QWidget): # {{{
         self.abort.set()
 # }}}
 
-class CoverWidget(QWidget): # {{{
+class CoverWorker(Thread): # {{{
 
-    def __init__(self, log, parent=None):
+    def __init__(self, log, abort, title, authors, identifiers):
+        Thread.__init__(self)
+        self.daemon = True
+
+        self.log, self.abort = log, abort
+        self.title, self.authors, self.identifiers = (title, authors,
+                identifiers)
+
+        self.rq = Queue()
+        self.error = None
+
+    def fake_run(self):
+        images = ['donate.png', 'config.png', 'column.png', 'eject.png', ]
+        import time
+        time.sleep(2)
+        for pl, im in zip(metadata_plugins(['cover']), images):
+            self.rq.put((pl, 1, 1, 'png', I(im, data=True)))
+
+    def run(self):
+        try:
+            if DEBUG_DIALOG:
+                self.fake_run()
+            else:
+                from calibre.ebooks.metadata.sources.covers import run_download
+                run_download(self.log, self.rq, self.abort, title=self.title,
+                        authors=self.authors, identifiers=self.identifiers)
+        except:
+            import traceback
+            self.error = force_unicode(traceback.format_exc())
+# }}}
+
+class CoversModel(QAbstractListModel): # {{{
+
+    def __init__(self, current_cover, parent=None):
+        QAbstractListModel.__init__(self, parent)
+
+        if current_cover is None:
+            current_cover = QPixmap(I('default_cover.png'))
+
+        self.blank = QPixmap(I('blank.png')).scaled(150, 200)
+
+        self.covers = [self.get_item(_('Current cover'), current_cover)]
+        self.plugin_map = {}
+        for i, plugin in enumerate(metadata_plugins(['cover'])):
+            self.covers.append((plugin.name+'\n'+_('Searching...'),
+                QVariant(self.blank), None, True))
+            self.plugin_map[plugin] = i+1
+
+    def get_item(self, src, pmap, waiting=False):
+        sz = '%dx%d'%(pmap.width(), pmap.height())
+        text = QVariant(src + '\n' + sz)
+        scaled = pmap.scaled(150, 200, Qt.IgnoreAspectRatio,
+                Qt.SmoothTransformation)
+        return (text, QVariant(scaled), pmap, waiting)
+
+    def rowCount(self, parent=None):
+        return len(self.covers)
+
+    def data(self, index, role):
+        try:
+            text, pmap, cover, waiting = self.covers[index.row()]
+        except:
+            return NONE
+        if role == Qt.DecorationRole:
+            return pmap
+        if role == Qt.DisplayRole:
+            return text
+        if role == Qt.UserRole:
+            return waiting
+        return NONE
+
+    def plugin_for_index(self, index):
+        row = index.row() if hasattr(index, 'row') else index
+        for k, v in self.plugin_map.iteritems():
+            if v == row:
+                return k
+
+    def cover_keygen(self, x):
+        pmap = x[2]
+        if pmap is None:
+            return 1
+        return pmap.width()*pmap.height()
+
+
+    def clear_failed(self):
+        good = []
+        pmap = {}
+        dcovers = sorted(self.covers[1:], key=self.cover_keygen, reverse=True)
+        for i, x in enumerate(self.covers[0:1] + dcovers):
+            if not x[-1]:
+                good.append(x)
+                if i > 0:
+                    plugin = self.plugin_for_index(i)
+                    pmap[plugin] = len(good) - 1
+        self.covers = good
+        self.plugin_map = pmap
+        self.reset()
+
+    def index_for_plugin(self, plugin):
+        idx = self.plugin_map.get(plugin, 0)
+        return self.index(idx)
+
+    def update_result(self, plugin, width, height, data):
+        try:
+            idx = self.plugin_map[plugin]
+        except:
+            return
+        pmap = QPixmap()
+        pmap.loadFromData(data)
+        if pmap.isNull():
+            return
+        self.covers[idx] = self.get_item(plugin.name, pmap, waiting=False)
+        self.dataChanged.emit(self.index(idx), self.index(idx))
+
+    def cover_pixmap(self, index):
+        row = index.row()
+        if row > 0 and row < len(self.covers):
+            pmap = self.covers[row][2]
+            if pmap is not None and not pmap.isNull():
+                return pmap
+
+# }}}
+
+class CoversView(QListView): # {{{
+
+    chosen = pyqtSignal()
+
+    def __init__(self, current_cover, parent=None):
+        QListView.__init__(self, parent)
+        self.m = CoversModel(current_cover, self)
+        self.setModel(self.m)
+
+        self.setFlow(self.LeftToRight)
+        self.setWrapping(True)
+        self.setResizeMode(self.Adjust)
+        self.setGridSize(QSize(190, 260))
+        self.setIconSize(QSize(150, 200))
+        self.setSelectionMode(self.SingleSelection)
+        self.setViewMode(self.IconMode)
+
+        self.delegate = CoverDelegate(self)
+        self.setItemDelegate(self.delegate)
+        self.delegate.needs_redraw.connect(self.viewport().update,
+                type=Qt.QueuedConnection)
+
+        self.doubleClicked.connect(self.chosen, type=Qt.QueuedConnection)
+
+    def select(self, num):
+        current = self.model().index(num)
+        sm = self.selectionModel()
+        sm.select(current, sm.SelectCurrent)
+
+    def start(self):
+        self.select(0)
+        self.delegate.start_animation()
+
+    def clear_failed(self):
+        plugin = self.m.plugin_for_index(self.currentIndex())
+        self.m.clear_failed()
+        self.select(self.m.index_for_plugin(plugin).row())
+
+# }}}
+
+class CoversWidget(QWidget): # {{{
+
+    chosen = pyqtSignal()
+    finished = pyqtSignal()
+
+    def __init__(self, log, current_cover, parent=None):
         QWidget.__init__(self, parent)
         self.log = log
+        self.abort = Event()
+
+        self.l = l = QGridLayout()
+        self.setLayout(l)
+
+        self.msg = QLabel()
+        self.msg.setWordWrap(True)
+        l.addWidget(self.msg, 0, 0)
+
+        self.covers_view = CoversView(current_cover, self)
+        self.covers_view.chosen.connect(self.chosen)
+        l.addWidget(self.covers_view, 1, 0)
+        self.continue_processing = True
 
     def start(self, book, current_cover, title, authors):
         self.book, self.current_cover = book, current_cover
         self.title, self.authors = title, authors
-        self.log('\n\nStarting cover download for:', book.title)
+        self.log('Starting cover download for:', book.title)
+        self.log('Query:', title, authors, self.book.identifiers)
+        self.msg.setText('<p>'+_('Downloading covers for <b>%s</b>, please wait...')%book.title)
+        self.covers_view.start()
+
+        self.worker = CoverWorker(self.log, self.abort, self.title,
+                self.authors, book.identifiers)
+        self.worker.start()
+        QTimer.singleShot(50, self.check)
+        self.covers_view.setFocus(Qt.OtherFocusReason)
+
+    def check(self):
+        if self.worker.is_alive() and not self.abort.is_set():
+            QTimer.singleShot(50, self.check)
+            try:
+                self.process_result(self.worker.rq.get_nowait())
+            except Empty:
+                pass
+        else:
+            self.process_results()
+
+    def process_results(self):
+        while self.continue_processing:
+            try:
+                self.process_result(self.worker.rq.get_nowait())
+            except Empty:
+                break
+
+        self.covers_view.clear_failed()
+
+        if self.worker.error is not None:
+            error_dialog(self, _('Download failed'),
+                    _('Failed to download any covers, click'
+                        ' "Show details" for details.'),
+                    det_msg=self.worker.error, show=True)
+
+        num = self.covers_view.model().rowCount()
+        if num < 2:
+            txt = _('Could not find any covers for <b>%s</b>')%self.book.title
+        else:
+            txt = _('Found <b>%d</b> covers of %s. Pick the one you like'
+                    ' best.')%(num-1, self.title)
+        self.msg.setText(txt)
+
+        self.finished.emit()
+
+    def process_result(self, result):
+        if not self.continue_processing:
+            return
+        plugin, width, height, fmt, data = result
+        self.covers_view.model().update_result(plugin, width, height, data)
+
+    def cleanup(self):
+        self.covers_view.delegate.stop_animation()
+        self.continue_processing = False
+
+    def cancel(self):
+        self.continue_processing = False
+        self.abort.set()
+
+    def cover_pixmap(self):
+        idx = None
+        for i in self.covers_view.selectionModel().selectedIndexes():
+            if i.isValid():
+                idx = i
+                break
+        if idx is None:
+            idx = self.covers_view.currentIndex()
+        return self.covers_view.model().cover_pixmap(idx)
+
+# }}}
+
+class LogViewer(QDialog): # {{{
+
+    def __init__(self, log, parent=None):
+        QDialog.__init__(self, parent)
+        self.log = log
+        self.l = l = QVBoxLayout()
+        self.setLayout(l)
+
+        self.tb = QTextBrowser(self)
+        l.addWidget(self.tb)
+
+        self.bb = QDialogButtonBox(QDialogButtonBox.Close)
+        l.addWidget(self.bb)
+        self.copy_button = self.bb.addButton(_('Copy to clipboard'),
+                self.bb.ActionRole)
+        self.copy_button.clicked.connect(self.copy_to_clipboard)
+        self.copy_button.setIcon(QIcon(I('edit-copy.png')))
+        self.bb.rejected.connect(self.reject)
+        self.bb.accepted.connect(self.accept)
+
+        self.setWindowTitle(_('Download log'))
+        self.setWindowIcon(QIcon(I('debug.png')))
+        self.resize(QSize(800, 400))
+
+        self.keep_updating = True
+        self.last_html = None
+        self.finished.connect(self.stop)
+        QTimer.singleShot(100, self.update_log)
+
+        self.show()
+
+    def copy_to_clipboard(self):
+        QApplication.clipboard().setText(''.join(self.log.plain_text))
+
+    def stop(self, *args):
+        self.keep_updating = False
+
+    def update_log(self):
+        if not self.keep_updating:
+            return
+        html = self.log.html
+        if html != self.last_html:
+            self.last_html = html
+            self.tb.setHtml('<pre style="font-family:monospace">%s</pre>'%html)
+        QTimer.singleShot(1000, self.update_log)
+
 # }}}
 
 class FullFetch(QDialog): # {{{
 
-    def __init__(self, log, current_cover=None, parent=None):
+    def __init__(self, current_cover=None, parent=None):
         QDialog.__init__(self, parent)
-        self.log, self.current_cover = log, current_cover
+        self.current_cover = current_cover
+        self.log = Log()
+        self.book = self.cover_pixmap = None
 
         self.setWindowTitle(_('Downloading metadata...'))
         self.setWindowIcon(QIcon(I('metadata.png')))
@@ -430,28 +820,39 @@ class FullFetch(QDialog): # {{{
         self.next_button = self.bb.addButton(_('Next'), self.bb.AcceptRole)
         self.next_button.setDefault(True)
         self.next_button.setEnabled(False)
+        self.next_button.setIcon(QIcon(I('ok.png')))
         self.next_button.clicked.connect(self.next_clicked)
         self.ok_button = self.bb.button(self.bb.Ok)
-        self.ok_button.setVisible(False)
         self.ok_button.clicked.connect(self.ok_clicked)
+        self.log_button = self.bb.addButton(_('View log'), self.bb.ActionRole)
+        self.log_button.clicked.connect(self.view_log)
+        self.log_button.setIcon(QIcon(I('debug.png')))
+        self.ok_button.setVisible(False)
 
-        self.identify_widget = IdentifyWidget(log, self)
+        self.identify_widget = IdentifyWidget(self.log, self)
         self.identify_widget.rejected.connect(self.reject)
         self.identify_widget.results_found.connect(self.identify_results_found)
         self.identify_widget.book_selected.connect(self.book_selected)
         self.stack.addWidget(self.identify_widget)
 
-        self.cover_widget = CoverWidget(self.log, parent=self)
+        self.covers_widget = CoversWidget(self.log, self.current_cover, parent=self)
-        self.stack.addWidget(self.cover_widget)
+        self.covers_widget.chosen.connect(self.ok_clicked)
+        self.stack.addWidget(self.covers_widget)
 
-        self.resize(850, 500)
+        self.resize(850, 550)
+
+        self.finished.connect(self.cleanup)
+
+    def view_log(self):
+        self._lv = LogViewer(self.log, self)
 
     def book_selected(self, book):
         self.next_button.setVisible(False)
         self.ok_button.setVisible(True)
         self.book = book
         self.stack.setCurrentIndex(1)
-        self.cover_widget.start(book, self.current_cover,
+        self.log('\n\n')
+        self.covers_widget.start(book, self.current_cover,
                 self.title, self.authors)
 
     def accept(self):
@@ -460,8 +861,12 @@ class FullFetch(QDialog): # {{{
 
     def reject(self):
         self.identify_widget.cancel()
+        self.covers_widget.cancel()
         return QDialog.reject(self)
 
+    def cleanup(self):
+        self.covers_widget.cleanup()
+
     def identify_results_found(self):
         self.next_button.setEnabled(True)
 
@@ -469,17 +874,79 @@ class FullFetch(QDialog): # {{{
         self.identify_widget.get_result()
 
     def ok_clicked(self, *args):
-        pass
+        self.cover_pixmap = self.covers_widget.cover_pixmap()
+        if DEBUG_DIALOG:
+            if self.cover_pixmap is not None:
+                self.w = QLabel()
+                self.w.setPixmap(self.cover_pixmap)
+                self.stack.addWidget(self.w)
+                self.stack.setCurrentIndex(2)
+        else:
+            QDialog.accept(self)
 
     def start(self, title=None, authors=None, identifiers={}):
         self.title, self.authors = title, authors
         self.identify_widget.start(title=title, authors=authors,
                 identifiers=identifiers)
-        self.exec_()
+        return self.exec_()
+# }}}
+
+class CoverFetch(QDialog): # {{{
+
+    def __init__(self, current_cover=None, parent=None):
+        QDialog.__init__(self, parent)
+        self.current_cover = current_cover
+        self.log = Log()
+        self.cover_pixmap = None
+
+        self.setWindowTitle(_('Downloading cover...'))
+        self.setWindowIcon(QIcon(I('book.png')))
+
+        self.l = l = QVBoxLayout()
+        self.setLayout(l)
+
+        self.covers_widget = CoversWidget(self.log, self.current_cover, parent=self)
+        self.covers_widget.chosen.connect(self.accept)
+        l.addWidget(self.covers_widget)
+
+        self.resize(850, 550)
+
+        self.finished.connect(self.cleanup)
+
+        self.bb = QDialogButtonBox(QDialogButtonBox.Cancel|QDialogButtonBox.Ok)
+        l.addWidget(self.bb)
+        self.log_button = self.bb.addButton(_('View log'), self.bb.ActionRole)
+        self.log_button.clicked.connect(self.view_log)
+        self.log_button.setIcon(QIcon(I('debug.png')))
+        self.bb.rejected.connect(self.reject)
+        self.bb.accepted.connect(self.accept)
+
+    def cleanup(self):
+        self.covers_widget.cleanup()
+
+    def reject(self):
+        self.covers_widget.cancel()
+        return QDialog.reject(self)
+
+    def accept(self, *args):
+        self.cover_pixmap = self.covers_widget.cover_pixmap()
+        QDialog.accept(self)
+
+    def start(self, title, authors, identifiers):
+        book = Metadata(title, authors)
+        book.identifiers = identifiers
+        self.covers_widget.start(book, self.current_cover,
+                title, authors)
+        return self.exec_()
+
+    def view_log(self):
+        self._lv = LogViewer(self.log, self)
+
 # }}}
 
 if __name__ == '__main__':
+    #DEBUG_DIALOG = True
     app = QApplication([])
-    d = FullFetch(Log())
+    d = FullFetch()
-    d.start(title='great gatsby', authors=['Fitzgerald'])
+    d.start(title='great gatsby', authors=['fitzgerald'])
 

src/calibre/gui2/preferences/__init__.py

@@ -7,8 +7,9 @@ __docformat__ = 'restructuredtext en'
 
 import textwrap
 
-from PyQt4.Qt import QWidget, pyqtSignal, QCheckBox, QAbstractSpinBox, \
+from PyQt4.Qt import (QWidget, pyqtSignal, QCheckBox, QAbstractSpinBox,
-    QLineEdit, QComboBox, QVariant, Qt
+    QLineEdit, QComboBox, QVariant, Qt, QIcon, QDialog, QVBoxLayout,
+    QDialogButtonBox)
 
 from calibre.customize.ui import preferences_plugins
 from calibre.utils.config import ConfigProxy
@@ -21,7 +22,7 @@ class ConfigWidgetInterface(object):
     '''
     This class defines the interface that all widgets displayed in the
     Preferences dialog must implement. See :class:`ConfigWidgetBase` for
-    a base class that implements this interface and defines various conveninece
+    a base class that implements this interface and defines various convenience
     methods as well.
     '''
 
@@ -284,7 +285,14 @@ def get_plugin(category, name):
             'No Preferences Plugin with category: %s and name: %s found' %
             (category, name))
 
-# Testing {{{
+class ConfigDialog(QDialog):
+    def set_widget(self, w): self.w = w
+    def accept(self):
+        try:
+            self.restart_required = self.w.commit()
+        except AbortCommit:
+            return
+        QDialog.accept(self)
 
 def init_gui():
     from calibre.gui2.ui import Main
@@ -298,21 +306,24 @@ def init_gui():
     gui.initialize(db.library_path, db, None, actions, show_gui=False)
     return gui
 
-def test_widget(category, name, gui=None):
+def show_config_widget(category, name, gui=None, show_restart_msg=False,
-    from PyQt4.Qt import QDialog, QVBoxLayout, QDialogButtonBox
+        parent=None, never_shutdown=False):
-    class Dialog(QDialog):
+    '''
-        def set_widget(self, w): self.w = w
+    Show the preferences plugin identified by category and name
-        def accept(self):
-            try:
-                self.restart_required = self.w.commit()
-            except AbortCommit:
-                return
-            QDialog.accept(self)
 
+    :param gui: gui instance, if None a hidden gui is created
+    :param show_restart_msg: If True and the preferences plugin indicates a
+    restart is required, show a message box telling the user to restart
+    :param parent: The parent of the displayed dialog
+
+    :return: True iff a restart is required for the changes made by the user to
+    take effect
+    '''
     pl = get_plugin(category, name)
-    d = Dialog()
+    d = ConfigDialog(parent)
     d.resize(750, 550)
-    d.setWindowTitle(category + " - " + name)
+    d.setWindowTitle(_('Configure ') + name)
+    d.setWindowIcon(QIcon(I('config.png')))
     bb = QDialogButtonBox(d)
     bb.setStandardButtons(bb.Apply|bb.Cancel|bb.RestoreDefaults)
     bb.accepted.connect(d.accept)
@@ -335,11 +346,18 @@ def test_widget(category, name, gui=None):
     w.genesis(gui)
     w.initialize()
     d.exec_()
-    if getattr(d, 'restart_required', False):
+    rr = getattr(d, 'restart_required', False)
+    if show_restart_msg and rr:
         from calibre.gui2 import warning_dialog
         warning_dialog(gui, 'Restart required', 'Restart required', show=True)
-    if mygui:
+    if mygui and not never_shutdown:
         gui.shutdown()
+    return rr
+
+# Testing {{{
+
+def test_widget(category, name, gui=None):
+    show_config_widget(category, name, gui=gui, show_restart_msg=True)
 
 def test_all():
     from PyQt4.Qt import QApplication

src/calibre/gui2/preferences/behavior.py

@@ -11,7 +11,7 @@ from PyQt4.Qt import Qt, QVariant, QListWidgetItem
 
 from calibre.gui2.preferences import ConfigWidgetBase, test_widget, Setting
 from calibre.gui2.preferences.behavior_ui import Ui_Form
-from calibre.gui2 import config, info_dialog, dynamic
+from calibre.gui2 import config, info_dialog, dynamic, gprefs
 from calibre.utils.config import prefs
 from calibre.customize.ui import available_output_formats, all_input_formats
 from calibre.utils.search_query_parser import saved_searches
@@ -19,6 +19,7 @@ from calibre.ebooks import BOOK_EXTENSIONS
 from calibre.ebooks.oeb.iterator import is_supported
 from calibre.constants import iswindows
 from calibre.utils.icu import sort_key
+from calibre.utils.config import test_eight_code
 
 class OutputFormatSetting(Setting):
 
@@ -42,6 +43,9 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
 
         r('overwrite_author_title_metadata', config)
         r('get_social_metadata', config)
+        if test_eight_code:
+            self.opt_overwrite_author_title_metadata.setVisible(False)
+            self.opt_get_social_metadata.setVisible(False)
         r('new_version_notification', config)
         r('upload_news_to_device', config)
         r('delete_news_from_library_on_upload', config)
@@ -62,6 +66,14 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
             signal = getattr(self.opt_internally_viewed_formats, 'item'+signal)
             signal.connect(self.internally_viewed_formats_changed)
 
+        r('bools_are_tristate', db.prefs, restart_required=True)
+        if test_eight_code:
+            r = self.register
+            choices = [(_('Default'), 'default'), (_('Compact Metadata'), 'alt1')]
+            r('edit_metadata_single_layout', gprefs, choices=choices)
+        else:
+            self.opt_edit_metadata_single_layout.setVisible(False)
+            self.edit_metadata_single_label.setVisible(False)
 
     def initialize(self):
         ConfigWidgetBase.initialize(self)

src/calibre/gui2/preferences/behavior.ui

@@ -14,44 +14,92 @@
    <string>Form</string>
   </property>
   <layout class="QGridLayout" name="gridLayout">
-   <item row="0" column="0" colspan="2">
+   <item row="0" column="1">
+    <spacer>
+     <property name="orientation">
+      <enum>Qt::Horizontal</enum>
+     </property>
+     <property name="sizeHint" stdset="0">
+      <size>
+       <width>10</width>
+       <height>00</height>
+      </size>
+     </property>
+    </spacer>
+   </item>
+   <item row="0" column="0">
     <widget class="QCheckBox" name="opt_overwrite_author_title_metadata">
      <property name="text">
       <string>&amp;Overwrite author and title by default when fetching metadata</string>
      </property>
     </widget>
    </item>
-   <item row="1" column="0" colspan="2">
+   <item row="0" column="2">
     <widget class="QCheckBox" name="opt_get_social_metadata">
      <property name="text">
       <string>Download &amp;social metadata (tags/ratings/etc.) by default</string>
      </property>
     </widget>
    </item>
-   <item row="2" column="0" colspan="2">
+   <item row="2" column="0">
     <widget class="QCheckBox" name="opt_new_version_notification">
      <property name="text">
       <string>Show notification when &amp;new version is available</string>
      </property>
     </widget>
    </item>
-   <item row="3" column="0" colspan="2">
+   <item row="2" column="2">
+    <widget class="QCheckBox" name="opt_bools_are_tristate">
+     <property name="text">
+      <string>Yes/No columns have three values (Requires restart)</string>
+     </property>
+     <property name="toolTip">
+      <string>If checked, Yes/No custom columns values can be Yes, No, or Unknown.
+If not checked, the values can be Yes or No.</string>
+     </property>
+    </widget>
+   </item>
+   <item row="4" column="0">
     <widget class="QCheckBox" name="opt_upload_news_to_device">
      <property name="text">
       <string>Automatically send downloaded &amp;news to ebook reader</string>
      </property>
     </widget>
    </item>
-   <item row="4" column="0" colspan="2">
+   <item row="4" column="2">
     <widget class="QCheckBox" name="opt_delete_news_from_library_on_upload">
      <property name="text">
       <string>&amp;Delete news from library when it is automatically sent to reader</string>
      </property>
     </widget>
    </item>
-   <item row="5" column="0" colspan="2">
+   <item row="6" column="0">
-    <layout class="QGridLayout" name="gridLayout_2">
+    <layout class="QHBoxLayout">
-     <item row="1" column="0">
+     <item>
+      <widget class="QLabel" name="label_23">
+       <property name="text">
+        <string>Preferred &amp;output format:</string>
+       </property>
+       <property name="buddy">
+        <cstring>opt_output_format</cstring>
+       </property>
+      </widget>
+     </item>
+     <item>
+      <widget class="QComboBox" name="opt_output_format">
+       <property name="sizeAdjustPolicy">
+        <enum>QComboBox::AdjustToMinimumContentsLengthWithIcon</enum>
+       </property>
+       <property name="minimumContentsLength">
+        <number>10</number>
+       </property>
+      </widget>
+     </item>
+    </layout>
+   </item>
+   <item row="6" column="2">
+    <layout class="QHBoxLayout">
+     <item>
       <widget class="QLabel" name="label_2">
        <property name="text">
         <string>Default network &amp;timeout:</string>
@@ -61,7 +109,7 @@
        </property>
       </widget>
      </item>
-     <item row="1" column="1">
+     <item>
       <widget class="QSpinBox" name="opt_network_timeout">
        <property name="toolTip">
         <string>Set the default timeout for network fetches (i.e. anytime we go out to the internet to get information)</string>
@@ -80,7 +128,21 @@
        </property>
       </widget>
      </item>
-     <item row="2" column="1">
+    </layout>
+   </item>
+   <item row="8" column="0">
+    <layout class="QHBoxLayout">
+     <item>
+      <widget class="QLabel" name="priority_label">
+       <property name="text">
+        <string>Job &amp;priority:</string>
+       </property>
+       <property name="buddy">
+        <cstring>opt_worker_process_priority</cstring>
+       </property>
+      </widget>
+     </item>
+     <item>
       <widget class="QComboBox" name="opt_worker_process_priority">
        <property name="sizeAdjustPolicy">
         <enum>QComboBox::AdjustToMinimumContentsLengthWithIcon</enum>
@@ -105,37 +167,11 @@
        </item>
       </widget>
      </item>
-     <item row="2" column="0">
+    </layout>
-      <widget class="QLabel" name="priority_label">
+   </item>
-       <property name="text">
+   <item row="8" column="2">
-        <string>Job &amp;priority:</string>
+    <layout class="QHBoxLayout">
-       </property>
+     <item>
-       <property name="buddy">
-        <cstring>opt_worker_process_priority</cstring>
-       </property>
-      </widget>
-     </item>
-     <item row="0" column="0">
-      <widget class="QLabel" name="label_23">
-       <property name="text">
-        <string>Preferred &amp;output format:</string>
-       </property>
-       <property name="buddy">
-        <cstring>opt_output_format</cstring>
-       </property>
-      </widget>
-     </item>
-     <item row="0" column="1">
-      <widget class="QComboBox" name="opt_output_format">
-       <property name="sizeAdjustPolicy">
-        <enum>QComboBox::AdjustToMinimumContentsLengthWithIcon</enum>
-       </property>
-       <property name="minimumContentsLength">
-        <number>10</number>
-       </property>
-      </widget>
-     </item>
-     <item row="3" column="0">
       <widget class="QLabel" name="label_170">
        <property name="text">
         <string>Restriction to apply when the current library is opened:</string>
@@ -145,7 +181,7 @@
        </property>
       </widget>
      </item>
-     <item row="3" column="1">
+     <item>
       <widget class="QComboBox" name="opt_gui_restriction">
        <property name="maximumSize">
         <size>
@@ -166,14 +202,28 @@
      </item>
     </layout>
    </item>
-   <item row="6" column="0" colspan="2">
+   <item row="9" column="0">
-    <widget class="QPushButton" name="reset_confirmation_button">
+    <layout class="QHBoxLayout">
-     <property name="text">
+     <item>
-      <string>Reset all disabled &amp;confirmation dialogs</string>
+      <widget class="QLabel" name="edit_metadata_single_label">
-     </property>
+       <property name="text">
-    </widget>
+        <string>Edit metadata (single) layout:</string>
+       </property>
+       <property name="buddy">
+        <cstring>opt_edit_metadata_single_layout</cstring>
+       </property>
+      </widget>
+     </item>
+     <item>
+      <widget class="QComboBox" name="opt_edit_metadata_single_layout">
+       <property name="toolTip">
+        <string>Choose a different layout for the Edit Metadata dialog. The compact metadata layout favors editing custom metadata over changing covers and formats.</string>
+       </property>
+      </widget>
+     </item>
+    </layout>
    </item>
-   <item row="7" column="0">
+   <item row="20" column="0">
     <widget class="QGroupBox" name="groupBox_5">
      <property name="title">
       <string>Preferred &amp;input format order:</string>
@@ -235,7 +285,7 @@
      </layout>
     </widget>
    </item>
-   <item row="7" column="1">
+   <item row="20" column="2">
     <widget class="QGroupBox" name="groupBox_3">
      <property name="title">
       <string>Use internal &amp;viewer for:</string>
@@ -254,6 +304,13 @@
      </layout>
     </widget>
    </item>
+   <item row="30" column="0" colspan="3">
+    <widget class="QPushButton" name="reset_confirmation_button">
+     <property name="text">
+      <string>Reset all disabled &amp;confirmation dialogs</string>
+     </property>
+    </widget>
+   </item>
   </layout>
  </widget>
  <resources>

src/calibre/gui2/preferences/columns.py

@@ -13,7 +13,6 @@ from calibre.gui2.preferences import ConfigWidgetBase, test_widget
 from calibre.gui2.preferences.columns_ui import Ui_Form
 from calibre.gui2.preferences.create_custom_column import CreateCustomColumn
 from calibre.gui2 import error_dialog, question_dialog, ALL_COLUMNS
-from calibre.utils.config import test_eight_code
 
 class ConfigWidget(ConfigWidgetBase, Ui_Form):
 
@@ -34,14 +33,6 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
             signal = getattr(self.opt_columns, 'item'+signal)
             signal.connect(self.columns_changed)
 
-        if test_eight_code:
-            r = self.register
-            choices = [(_('Default'), 'default'), (_('Compact Metadata'), 'alt1')]
-            r('edit_metadata_single_layout', db.prefs, choices=choices)
-            r('bools_are_tristate', db.prefs, restart_required=True)
-        else:
-            self.items_in_v_eight.setVisible(False)
-
     def initialize(self):
         ConfigWidgetBase.initialize(self)
         self.init_columns()
@@ -178,10 +169,6 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
                     must_restart = True
         return must_restart
 
-    def refresh_gui(self, gui):
-        gui.library_view.reset()
-
-
 
 if __name__ == '__main__':
     from PyQt4.Qt import QApplication

src/calibre/gui2/preferences/columns.ui

@@ -197,67 +197,6 @@
      </property>
     </widget>
    </item>
-   <item row="1" column="2">
-    <layout class="QVBoxLayout">
-     <item>
-      <widget class="QGroupBox" name="items_in_v_eight">
-       <property name="title">
-        <string>Related Options</string>
-       </property>
-       <layout class="QGridLayout" name="gridLayout">
-        <item row="0" column="0">
-         <widget class="QLabel">
-          <property name="text">
-           <string>Edit metadata layout:</string>
-          </property>
-          <property name="buddy">
-           <cstring>opt_edit_metadata_single_layout</cstring>
-          </property>
-         </widget>
-        </item>
-        <item row="0" column="1">
-         <widget class="QComboBox" name="opt_edit_metadata_single_layout">
-          <property name="toolTip">
-           <string>Choose a different layout for the Edit Metadata dialog. Alternate layouts make it easier to edit custom columns.</string>
-          </property>
-         </widget>
-        </item>
-        <item row="1" column="0">
-         <widget class="QLabel">
-          <property name="text">
-           <string>Boolean columns are tristate:</string>
-          </property>
-          <property name="buddy">
-           <cstring>opt_bools_are_tristate</cstring>
-          </property>
-         </widget>
-        </item>
-        <item row="1" column="1">
-        <widget class="QCheckBox" name="opt_bools_are_tristate">
-          <property name="toolTip">
-           <string>If checked, boolean columns values can be Yes, No, and Unknown.
-If not checked, the values can be Yes and No.</string>
-          </property>
-         </widget>
-        </item>
-       </layout>
-      </widget>
-     </item>
-     <item>
-      <spacer name="verticalSpacer_5">
-       <property name="orientation">
-        <enum>Qt::Vertical</enum>
-       </property>
-       <property name="sizeHint" stdset="0">
-        <size>
-         <width>20</width>
-         <height>40</height>
-        </size>
-       </property>
-      </spacer>
-     </item>
-    </layout>
-   </item>
   </layout>
  </widget>
  <resources>

src/calibre/gui2/preferences/emailp.py

@@ -84,11 +84,11 @@ class EmailAccounts(QAbstractTableModel): # {{{
         account = self.account_order[row]
         if col == 3:
             self.accounts[account][1] ^= True
-        if col == 2:
+        elif col == 2:
             self.subjects[account] = unicode(value.toString())
         elif col == 1:
             self.accounts[account][0] = unicode(value.toString()).upper()
-        else:
+        elif col == 0:
             na = unicode(value.toString())
             from email.utils import parseaddr
             addr = parseaddr(na)[-1]
@@ -100,7 +100,7 @@ class EmailAccounts(QAbstractTableModel): # {{{
                 self.accounts[na][0] = 'AZW, MOBI, TPZ, PRC, AZW1'
 
         self.dataChanged.emit(
-                self.index(index.row(), 0), self.index(index.row(), 2))
+                self.index(index.row(), 0), self.index(index.row(), 3))
         return True
 
     def make_default(self, index):

src/calibre/gui2/preferences/metadata_sources.py

@@ -0,0 +1,308 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from operator import attrgetter
+
+from PyQt4.Qt import (QAbstractTableModel, Qt, QAbstractListModel, QWidget,
+        pyqtSignal, QVBoxLayout, QDialogButtonBox, QFrame, QLabel)
+
+from calibre.gui2.preferences import ConfigWidgetBase, test_widget
+from calibre.gui2.preferences.metadata_sources_ui import Ui_Form
+from calibre.ebooks.metadata.sources.base import msprefs
+from calibre.customize.ui import (all_metadata_plugins, is_disabled,
+        enable_plugin, disable_plugin, default_disabled_plugins)
+from calibre.gui2 import NONE, error_dialog
+
+class SourcesModel(QAbstractTableModel): # {{{
+
+    def __init__(self, parent=None):
+        QAbstractTableModel.__init__(self, parent)
+
+        self.plugins = []
+        self.enabled_overrides = {}
+        self.cover_overrides = {}
+
+    def initialize(self):
+        self.plugins = list(all_metadata_plugins())
+        self.plugins.sort(key=attrgetter('name'))
+        self.enabled_overrides = {}
+        self.cover_overrides = {}
+        self.reset()
+
+    def rowCount(self, parent=None):
+        return len(self.plugins)
+
+    def columnCount(self, parent=None):
+        return 2
+
+    def headerData(self, section, orientation, role):
+        if orientation == Qt.Horizontal and role == Qt.DisplayRole:
+            if section == 0:
+                return _('Source')
+            if section == 1:
+                return _('Cover priority')
+        return NONE
+
+    def data(self, index, role):
+        try:
+            plugin = self.plugins[index.row()]
+        except:
+            return NONE
+        col = index.column()
+
+        if role == Qt.DisplayRole:
+            if col == 0:
+                return plugin.name
+            elif col == 1:
+                orig = msprefs['cover_priorities'].get(plugin.name, 1)
+                return self.cover_overrides.get(plugin, orig)
+        elif role == Qt.CheckStateRole and col == 0:
+            orig = Qt.Unchecked if is_disabled(plugin) else Qt.Checked
+            return self.enabled_overrides.get(plugin, orig)
+        elif role == Qt.UserRole:
+            return plugin
+        return NONE
+
+    def setData(self, index, val, role):
+        try:
+            plugin = self.plugins[index.row()]
+        except:
+            return False
+        col = index.column()
+        ret = False
+        if col == 0 and role == Qt.CheckStateRole:
+            val, ok = val.toInt()
+            if ok:
+                self.enabled_overrides[plugin] = val
+                ret = True
+        if col == 1 and role == Qt.EditRole:
+            val, ok = val.toInt()
+            if ok:
+                self.cover_overrides[plugin] = val
+                ret = True
+        if ret:
+            self.dataChanged.emit(index, index)
+        return ret
+
+
+    def flags(self, index):
+        col = index.column()
+        ans = QAbstractTableModel.flags(self, index)
+        if col == 0:
+            return ans | Qt.ItemIsUserCheckable
+        return Qt.ItemIsEditable | ans
+
+    def commit(self):
+        for plugin, val in self.enabled_overrides.iteritems():
+            if val == Qt.Checked:
+                enable_plugin(plugin)
+            elif val == Qt.Unchecked:
+                disable_plugin(plugin)
+
+        if self.cover_overrides:
+            cp = msprefs['cover_priorities']
+            for plugin, val in self.cover_overrides.iteritems():
+                if val == 1:
+                    cp.pop(plugin.name, None)
+                else:
+                    cp[plugin.name] = val
+            msprefs['cover_priorities'] = cp
+
+        self.enabled_overrides = {}
+        self.cover_overrides = {}
+
+    def restore_defaults(self):
+        self.enabled_overrides = dict([(p, (Qt.Unchecked if p.name in
+            default_disabled_plugins else Qt.Checked)) for p in self.plugins])
+        self.cover_overrides = dict([(p,
+            msprefs.defaults['cover_priorities'].get(p.name, 1))
+                for p in self.plugins])
+        self.reset()
+
+# }}}
+
+class FieldsModel(QAbstractListModel): # {{{
+
+
+    def __init__(self, parent=None):
+        QAbstractTableModel.__init__(self, parent)
+
+        self.fields = []
+        self.descs = {
+                'authors': _('Authors'),
+                'comments': _('Comments'),
+                'pubdate': _('Published date'),
+                'publisher': _('Publisher'),
+                'rating' : _('Rating'),
+                'tags' : _('Tags'),
+                'title': _('Title'),
+                'series': _('Series'),
+                'language': _('Language'),
+        }
+        self.overrides = {}
+        self.exclude = frozenset(['series_index'])
+
+    def rowCount(self, parent=None):
+        return len(self.fields)
+
+    def initialize(self):
+        fields = set()
+        for p in all_metadata_plugins():
+            fields |= p.touched_fields
+        self.fields = []
+        for x in fields:
+            if not x.startswith('identifier:') and x not in self.exclude:
+                self.fields.append(x)
+        self.fields.sort(key=lambda x:self.descs.get(x, x))
+        self.reset()
+
+    def state(self, field, defaults=False):
+        src = msprefs.defaults if defaults else msprefs
+        return (Qt.Unchecked if field in src['ignore_fields']
+                    else Qt.Checked)
+
+    def data(self, index, role):
+        try:
+            field = self.fields[index.row()]
+        except:
+            return None
+        if role == Qt.DisplayRole:
+            return self.descs.get(field, field)
+        if role == Qt.CheckStateRole:
+            return self.overrides.get(field, self.state(field))
+        return NONE
+
+    def flags(self, index):
+        ans = QAbstractTableModel.flags(self, index)
+        return ans | Qt.ItemIsUserCheckable
+
+    def restore_defaults(self):
+        self.overrides = dict([(f, self.state(f, True)) for f in self.fields])
+        self.reset()
+
+    def setData(self, index, val, role):
+        try:
+            field = self.fields[index.row()]
+        except:
+            return False
+        ret = False
+        if role == Qt.CheckStateRole:
+            val, ok = val.toInt()
+            if ok:
+                self.overrides[field] = val
+                ret = True
+        if ret:
+            self.dataChanged.emit(index, index)
+        return ret
+
+    def commit(self):
+        val = [k for k, v in self.overrides.iteritems() if v == Qt.Unchecked]
+        msprefs['ignore_fields'] = val
+
+
+# }}}
+
+class PluginConfig(QWidget): # {{{
+
+    finished = pyqtSignal()
+
+    def __init__(self, plugin, parent):
+        QWidget.__init__(self, parent)
+
+        self.plugin = plugin
+
+        self.l = l = QVBoxLayout()
+        self.setLayout(l)
+        self.c = c = QLabel(_('<b>Configure %s</b><br>%s') % (plugin.name,
+            plugin.description))
+        c.setAlignment(Qt.AlignHCenter)
+        l.addWidget(c)
+
+        self.config_widget = plugin.config_widget()
+        self.l.addWidget(self.config_widget)
+
+        self.bb = QDialogButtonBox(
+                QDialogButtonBox.Save|QDialogButtonBox.Cancel,
+                parent=self)
+        self.bb.accepted.connect(self.finished)
+        self.bb.rejected.connect(self.finished)
+        self.bb.accepted.connect(self.commit)
+        l.addWidget(self.bb)
+
+        self.f = QFrame(self)
+        self.f.setFrameShape(QFrame.HLine)
+        l.addWidget(self.f)
+
+    def commit(self):
+        self.plugin.save_settings(self.config_widget)
+# }}}
+
+class ConfigWidget(ConfigWidgetBase, Ui_Form):
+
+    def genesis(self, gui):
+        r = self.register
+        r('txt_comments', msprefs)
+        r('max_tags', msprefs)
+        r('wait_after_first_identify_result', msprefs)
+        r('wait_after_first_cover_result', msprefs)
+
+        self.configure_plugin_button.clicked.connect(self.configure_plugin)
+        self.sources_model = SourcesModel(self)
+        self.sources_view.setModel(self.sources_model)
+        self.sources_model.dataChanged.connect(self.changed_signal)
+
+        self.fields_model = FieldsModel(self)
+        self.fields_view.setModel(self.fields_model)
+        self.fields_model.dataChanged.connect(self.changed_signal)
+
+    def configure_plugin(self):
+        for index in self.sources_view.selectionModel().selectedRows():
+            plugin = self.sources_model.data(index, Qt.UserRole)
+            if plugin is not NONE:
+                return self.do_config(plugin)
+        error_dialog(self, _('No source selected'),
+                _('No source selected, cannot configure.'), show=True)
+
+    def do_config(self, plugin):
+        self.pc = PluginConfig(plugin, self)
+        self.stack.insertWidget(1, self.pc)
+        self.stack.setCurrentIndex(1)
+        self.pc.finished.connect(self.pc_finished)
+
+    def pc_finished(self):
+        try:
+            self.pc.finished.diconnect()
+        except:
+            pass
+        self.stack.setCurrentIndex(0)
+        self.stack.removeWidget(self.pc)
+        self.pc = None
+
+    def initialize(self):
+        ConfigWidgetBase.initialize(self)
+        self.sources_model.initialize()
+        self.sources_view.resizeColumnsToContents()
+        self.fields_model.initialize()
+
+    def restore_defaults(self):
+        ConfigWidgetBase.restore_defaults(self)
+        self.sources_model.restore_defaults()
+        self.fields_model.restore_defaults()
+        self.changed_signal.emit()
+
+    def commit(self):
+        self.sources_model.commit()
+        self.fields_model.commit()
+        return ConfigWidgetBase.commit(self)
+
+if __name__ == '__main__':
+    from PyQt4.Qt import QApplication
+    app = QApplication([])
+    test_widget('Sharing', 'Metadata download')
+

src/calibre/gui2/preferences/metadata_sources.ui

@@ -0,0 +1,149 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>Form</class>
+ <widget class="QWidget" name="Form">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>781</width>
+    <height>300</height>
+   </rect>
+  </property>
+  <property name="windowTitle">
+   <string>Form</string>
+  </property>
+  <layout class="QVBoxLayout" name="verticalLayout">
+   <property name="margin">
+    <number>0</number>
+   </property>
+   <item>
+    <widget class="QStackedWidget" name="stack">
+     <widget class="QWidget" name="page">
+      <layout class="QGridLayout" name="gridLayout">
+       <item row="0" column="0" rowspan="5">
+        <widget class="QGroupBox" name="groupBox">
+         <property name="title">
+          <string>Metadata sources</string>
+         </property>
+         <layout class="QVBoxLayout" name="verticalLayout_3">
+          <item>
+           <widget class="QLabel" name="label">
+            <property name="text">
+             <string>Disable any metadata sources you do not want by unchecking them. You can also set the cover priority. Covers from sources that have a higher (smaller) priority will be preferred when bulk downloading metadata.
+</string>
+            </property>
+            <property name="wordWrap">
+             <bool>true</bool>
+            </property>
+           </widget>
+          </item>
+          <item>
+           <widget class="QTableView" name="sources_view">
+            <property name="selectionMode">
+             <enum>QAbstractItemView::SingleSelection</enum>
+            </property>
+            <property name="selectionBehavior">
+             <enum>QAbstractItemView::SelectRows</enum>
+            </property>
+           </widget>
+          </item>
+          <item>
+           <widget class="QPushButton" name="configure_plugin_button">
+            <property name="text">
+             <string>Configure selected source</string>
+            </property>
+            <property name="icon">
+             <iconset resource="../../../../resources/images.qrc">
+              <normaloff>:/images/plugins.png</normaloff>:/images/plugins.png</iconset>
+            </property>
+           </widget>
+          </item>
+         </layout>
+        </widget>
+       </item>
+       <item row="0" column="1" colspan="2">
+        <widget class="QGroupBox" name="groupBox_2">
+         <property name="title">
+          <string>Downloaded metadata fields</string>
+         </property>
+         <layout class="QVBoxLayout" name="verticalLayout_2">
+          <item>
+           <widget class="QListView" name="fields_view">
+            <property name="toolTip">
+             <string>If you uncheck any fields, metadata for those fields will not be downloaded</string>
+            </property>
+            <property name="selectionMode">
+             <enum>QAbstractItemView::NoSelection</enum>
+            </property>
+           </widget>
+          </item>
+         </layout>
+        </widget>
+       </item>
+       <item row="1" column="1" colspan="2">
+        <widget class="QCheckBox" name="opt_txt_comments">
+         <property name="text">
+          <string>Convert all downloaded comments to plain &amp;text</string>
+         </property>
+        </widget>
+       </item>
+       <item row="2" column="1">
+        <widget class="QLabel" name="label_2">
+         <property name="text">
+          <string>Max. number of &amp;tags to download:</string>
+         </property>
+         <property name="buddy">
+          <cstring>opt_max_tags</cstring>
+         </property>
+        </widget>
+       </item>
+       <item row="2" column="2">
+        <widget class="QSpinBox" name="opt_max_tags"/>
+       </item>
+       <item row="3" column="1">
+        <widget class="QLabel" name="label_3">
+         <property name="text">
+          <string>Max. &amp;time to wait after first match is found:</string>
+         </property>
+         <property name="buddy">
+          <cstring>opt_wait_after_first_identify_result</cstring>
+         </property>
+        </widget>
+       </item>
+       <item row="3" column="2">
+        <widget class="QSpinBox" name="opt_wait_after_first_identify_result">
+         <property name="suffix">
+          <string> secs</string>
+         </property>
+        </widget>
+       </item>
+       <item row="4" column="1">
+        <widget class="QLabel" name="label_4">
+         <property name="text">
+          <string>Max. time to wait after first &amp;cover is found:</string>
+         </property>
+         <property name="buddy">
+          <cstring>opt_wait_after_first_cover_result</cstring>
+         </property>
+        </widget>
+       </item>
+       <item row="4" column="2">
+        <widget class="QSpinBox" name="opt_wait_after_first_cover_result">
+         <property name="suffix">
+          <string> secs</string>
+         </property>
+        </widget>
+       </item>
+      </layout>
+     </widget>
+     <widget class="QWidget" name="page_2"/>
+    </widget>
+   </item>
+  </layout>
+ </widget>
+ <resources>
+  <include location="../../../../resources/images.qrc"/>
+ </resources>
+ <connections/>
+</ui>

src/calibre/gui2/preferences/tweaks.py

@@ -14,9 +14,9 @@ from calibre.utils.config import read_raw_tweaks, write_tweaks
 from calibre.gui2.widgets import PythonHighlighter
 from calibre import isbytestring
 
-from PyQt4.Qt import QAbstractListModel, Qt, QStyledItemDelegate, QStyle, \
+from PyQt4.Qt import (QAbstractListModel, Qt, QStyledItemDelegate, QStyle,
-    QStyleOptionViewItem, QFont, QDialogButtonBox, QDialog, \
+    QStyleOptionViewItem, QFont, QDialogButtonBox, QDialog,
-    QVBoxLayout, QPlainTextEdit, QLabel
+    QVBoxLayout, QPlainTextEdit, QLabel)
 
 class Delegate(QStyledItemDelegate): # {{{
     def __init__(self, view):
@@ -35,8 +35,9 @@ class Delegate(QStyledItemDelegate): # {{{
 class Tweak(object): # {{{
 
     def __init__(self, name, doc, var_names, defaults, custom):
-        self.name = name
+        translate = __builtins__['_']
-        self.doc = doc.strip()
+        self.name = translate(name)
+        self.doc = translate(doc.strip())
         self.var_names = var_names
         self.default_values = {}
         for x in var_names:

src/calibre/gui2/search_box.py

@@ -436,14 +436,15 @@ class SavedSearchBoxMixin(object): # {{{
             b = getattr(self, x+'_search_button')
             b.setStatusTip(b.toolTip())
 
-    def saved_searches_changed(self, set_restriction=None):
+    def saved_searches_changed(self, set_restriction=None, recount=True):
         p = sorted(saved_searches().names(), key=sort_key)
         if set_restriction is None:
             set_restriction = unicode(self.search_restriction.currentText())
         # rebuild the restrictions combobox using current saved searches
         self.search_restriction.clear()
         self.search_restriction.addItem('')
-        self.tags_view.recount()
+        if recount:
+            self.tags_view.recount()
         for s in p:
             self.search_restriction.addItem(s)
         if set_restriction: # redo the search restriction if there was one

src/calibre/gui2/search_restriction_mixin.py

@@ -25,8 +25,9 @@ class SearchRestrictionMixin(object):
             r = self.search_restriction.findText(name)
             if r < 0:
                 r = 0
-        self.search_restriction.setCurrentIndex(r)
+        if r != self.search_restriction.currentIndex():
-        self.apply_search_restriction(r)
+            self.search_restriction.setCurrentIndex(r)
+            self.apply_search_restriction(r)
 
     def apply_search_restriction(self, i):
         r = unicode(self.search_restriction.currentText())

src/calibre/gui2/tag_view.py

@@ -1518,7 +1518,7 @@ class TagsModel(QAbstractItemModel): # {{{
                 if node.tag.category in \
                     ('tags', 'series', 'authors', 'rating', 'publisher') or \
                     (fm['is_custom'] and \
-                        fm['datatype'] in ['text', 'rating', 'series']):
+                        fm['datatype'] in ['text', 'rating', 'series', 'enumeration']):
                     ans |= Qt.ItemIsDropEnabled
             else:
                 ans |= Qt.ItemIsDropEnabled

src/calibre/gui2/threaded_jobs.py

@@ -0,0 +1,238 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import os, time, tempfile, json
+from threading import Thread, RLock, Event
+from Queue import Queue
+
+from calibre.utils.ipc.job import BaseJob
+from calibre.utils.logging import GUILog
+from calibre.ptempfile import base_dir
+
+class ThreadedJob(BaseJob):
+
+    def __init__(self,
+            type_, description,
+
+            func, args, kwargs,
+
+            callback,
+
+            max_concurrent_count=1,
+            killable=True,
+            log=None):
+        '''
+        A job that is run in its own thread in the calibre main process
+
+        :param type_: The type of this job (a string). The type is used in
+        conjunction with max_concurrent_count to prevent too many jobs of the
+        same type from running
+
+        :param description: A user viewable job description
+
+        :func: The function that actually does the work. This function *must*
+        accept at least three keyword arguments: abort, log and notifications. abort is
+        An Event object. func should periodically check abort.is_set(0 and if
+        it is True, it should stop processing as soon as possible. notifications
+        is a Queue. func should put progress notifications into it in the form
+        of a tuple (frac, msg). frac is a number between 0 and 1 indicating
+        progress and msg is a string describing the progress. log is a Log
+        object which func should use for all debugging output. func should
+        raise an Exception to indicate failure. This exception is stored in
+        job.exception and can thus be used to pass arbitrary information to
+        callback.
+
+        :param args,kwargs: These are passed to func when it is called
+
+        :param callback: A callable that is called on completion of this job.
+        Note that it is not called if the user kills the job. Check job.failed
+        to see if the job succeeded or not. And use job.log to get the job log.
+
+        :param killable: If False the GUI wont let the user kill this job
+
+        :param log: Must be a subclass of GUILog or None. If None a default
+        GUILog is created.
+        '''
+        BaseJob.__init__(self, description)
+
+        self.type = type_
+        self.max_concurrent_count = max_concurrent_count
+        self.killable = killable
+        self.callback = callback
+        self.abort = Event()
+        self.exception = None
+
+        kwargs['notifications'] = self.notifications
+        kwargs['abort'] = self.abort
+        self.log = GUILog() if log is None else log
+        kwargs['log'] = self.log
+
+        self.func, self.args, self.kwargs = func, args, kwargs
+        self.consolidated_log = None
+
+    def start_work(self):
+        self.start_time = time.time()
+        self.log('Starting job:', self.description)
+        try:
+            self.result = self.func(*self.args, **self.kwargs)
+        except Exception as e:
+            self.exception = e
+            self.failed = True
+            self.log.exception('Job: "%s" failed with error:'%self.description)
+            self.log.debug('Called with args:', self.args, self.kwargs)
+
+        self.duration = time.time() - self.start_time
+        try:
+            self.callback(self)
+        except:
+            pass
+        self._cleanup()
+
+    def _cleanup(self):
+
+        try:
+            self.consolidate_log()
+        except:
+            self.log.exception('Log consolidation failed')
+
+        # No need to keep references to these around anymore
+        self.func = self.args = self.kwargs = self.notifications = None
+
+    def kill(self):
+        if self.start_time is None:
+            self.start_time = time.time()
+            self.duration = 0.0001
+        else:
+            self.duration = time.time() - self.start_time()
+            self.abort.set()
+
+        self.log('Aborted job:', self.description)
+        self.killed = True
+        self.failed = True
+        self._cleanup()
+
+    def consolidate_log(self):
+        logs = [self.log.html, self.log.plain_text]
+        bdir = base_dir()
+        log_dir = os.path.join(bdir, 'threaded_job_logs')
+        if not os.path.exists(log_dir):
+            os.makedirs(log_dir)
+        fd, path = tempfile.mkstemp(suffix='.json', prefix='log-', dir=log_dir)
+        with os.fdopen(fd, 'wb') as f:
+            f.write(json.dumps(logs, ensure_ascii=False,
+                indent=2).encode('utf-8'))
+        self.consolidated_log = path
+        self.log = None
+
+    def read_consolidated_log(self):
+        with open(self.consolidated_log, 'rb') as f:
+            return json.loads(f.read().decode('utf-8'))
+
+    @property
+    def details(self):
+        if self.consolidated_log is None:
+            return self.log.plain_text
+        return self.read_consolidated_log()[1]
+
+    @property
+    def html_details(self):
+        if self.consolidated_log is None:
+            return self.log.html
+        return self.read_consolidated_log()[0]
+
+class ThreadedJobWorker(Thread):
+
+    def __init__(self, job):
+        Thread.__init__(self)
+        self.daemon = True
+        self.job = job
+
+    def run(self):
+        try:
+            self.job.start_work()
+        except:
+            import traceback
+            from calibre import prints
+            prints('Job had unhandled exception:', self.job.description)
+            traceback.print_exc()
+
+class ThreadedJobServer(Thread):
+
+    def __init__(self):
+        Thread.__init__(self)
+        self.daemon = True
+        self.lock = RLock()
+
+        self.queued_jobs = []
+        self.running_jobs = set()
+        self.changed_jobs = Queue()
+        self.keep_going = True
+
+    def close(self):
+        self.keep_going = False
+
+    def add_job(self, job):
+        with self.lock:
+            self.queued_jobs.append(job)
+
+        if not self.is_alive():
+            self.start()
+
+    def run(self):
+        while self.keep_going:
+            self.run_once()
+            time.sleep(0.1)
+
+    def run_once(self):
+        with self.lock:
+            remove = set()
+            for worker in self.running_jobs:
+                if worker.is_alive():
+                    # Get progress notifications
+                    if worker.job.consume_notifications():
+                        self.changed_jobs.put(worker.job)
+                else:
+                    remove.add(worker)
+                    self.changed_jobs.put(worker.job)
+
+            for worker in remove:
+                self.running_jobs.remove(worker)
+
+            jobs = self.get_startable_jobs()
+            for job in jobs:
+                w = ThreadedJobWorker(job)
+                w.start()
+                self.running_jobs.add(w)
+                self.changed_jobs.put(job)
+                self.queued_jobs.remove(job)
+
+    def kill_job(self, job):
+        with self.lock:
+            if job in self.queued_jobs:
+                self.queued_jobs.remove(job)
+            elif job in self.running_jobs:
+                self.running_jobs.remove(job)
+        job.kill()
+        self.changed_jobs.put(job)
+
+    def running_jobs_of_type(self, type_):
+        return len([w for w in self.running_jobs if w.job.type == type_])
+
+    def get_startable_jobs(self):
+        queued_types = []
+        ans = []
+        for job in self.queued_jobs:
+            num = self.running_jobs_of_type(job.type)
+            num += queued_types.count(job.type)
+            if num < job.max_concurrent_count:
+                queued_types.append(job.type)
+                ans.append(job)
+        return ans
+
+

src/calibre/gui2/ui.py

@@ -446,15 +446,16 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
         self.search.clear()
         self.saved_search.clear()
         self.book_details.reset_info()
-        self.library_view.model().count_changed()
         prefs['library_path'] = self.library_path
+        #self.library_view.model().count_changed()
         db = self.library_view.model().db
-        for action in self.iactions.values():
+        self.iactions['Choose Library'].count_changed(db.count())
-            action.library_changed(db)
         self.set_window_title()
         self.apply_named_search_restriction('') # reset restriction to null
-        self.saved_searches_changed() # reload the search restrictions combo box
+        self.saved_searches_changed(recount=False) # reload the search restrictions combo box
         self.apply_named_search_restriction(db.prefs['gui_restriction'])
+        for action in self.iactions.values():
+            action.library_changed(db)
         if olddb is not None:
             try:
                 if call_close:
@@ -607,6 +608,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
         self.update_checker.terminate()
         self.listener.close()
         self.job_manager.server.close()
+        self.job_manager.threaded_server.close()
         while self.spare_servers:
             self.spare_servers.pop().close()
         self.device_manager.keep_going = False
@@ -615,8 +617,6 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
             mb.stop()
 
         self.hide_windows()
-        if self.emailer.is_alive():
-            self.emailer.stop()
         try:
             try:
                 if self.content_server is not None:

src/calibre/library/caches.py

@@ -191,7 +191,8 @@ class CacheRow(list): # {{{
             if is_comp:
                 id = list.__getitem__(self, 0)
                 self._must_do = False
-                mi = self.db.get_metadata(id, index_is_id=True)
+                mi = self.db.get_metadata(id, index_is_id=True,
+                                          get_user_categories=False)
                 for c in self._composites:
                     self[c] =  mi.get(self._composites[c])
         return list.__getitem__(self, col)

src/calibre/library/database2.py

@@ -40,7 +40,6 @@ from calibre.ebooks import BOOK_EXTENSIONS, check_ebook_format
 from calibre.utils.magick.draw import save_cover_data_to
 from calibre.utils.recycle_bin import delete_file, delete_tree
 from calibre.utils.formatter_functions import load_user_template_functions
-from calibre.utils.config import test_eight_code
 
 copyfile = os.link if hasattr(os, 'link') else shutil.copyfile
 
@@ -213,11 +212,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         defs = self.prefs.defaults
         defs['gui_restriction'] = defs['cs_restriction'] = ''
         defs['categories_using_hierarchy'] = []
-        defs['edit_metadata_single_layout'] = 'default'
 
+        # Migrate the bool tristate tweak
         defs['bools_are_tristate'] = \
                 tweaks.get('bool_custom_columns_are_tristate', 'yes') == 'yes'
-        if self.prefs.get('bools_are_tristate') is None or not test_eight_code:
+        if self.prefs.get('bools_are_tristate') is None:
             self.prefs.set('bools_are_tristate', defs['bools_are_tristate'])
 
         # Migrate saved search and user categories to db preference scheme
@@ -824,7 +823,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             pass
         return (path, mi, sequence)
 
-    def get_metadata(self, idx, index_is_id=False, get_cover=False):
+    def get_metadata(self, idx, index_is_id=False, get_cover=False,
+                     get_user_categories=True):
         '''
         Convenience method to return metadata as a :class:`Metadata` object.
         Note that the list of formats is not verified.
@@ -883,16 +883,17 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
 
         user_cats = self.prefs['user_categories']
         user_cat_vals = {}
-        for ucat in user_cats:
+        if get_user_categories:
-            res = []
+            for ucat in user_cats:
-            for name,cat,ign in user_cats[ucat]:
+                res = []
-                v = mi.get(cat, None)
+                for name,cat,ign in user_cats[ucat]:
-                if isinstance(v, list):
+                    v = mi.get(cat, None)
-                    if name in v:
+                    if isinstance(v, list):
+                        if name in v:
+                            res.append([name,cat])
+                    elif name == v:
                         res.append([name,cat])
-                elif name == v:
+                user_cat_vals[ucat] = res
-                    res.append([name,cat])
-            user_cat_vals[ucat] = res
         mi.user_categories = user_cat_vals
 
         if get_cover:

src/calibre/manual/faq.rst

@@ -65,6 +65,22 @@ There are two aspects to this problem:
   2. When adding HTML files to |app|, you may need to tell |app| what encoding the files are in. To do this go to :guilabel:`Preferences->Advanced->Plugins->File Type plugins` and customize the HTML2Zip plugin, telling it what encoding your HTML files are in. Now when you add HTML files to |app| they will be correctly processed. HTML files from different sources often have different encodings, so you may have to change this setting repeatedly. A common encoding for many files from the web is ``cp1252`` and I would suggest you try that first. Note that when converting HTML files, leave the input encoding setting mentioned above blank. This is because the HTML2ZIP plugin automatically converts the HTML files to a standard encoding (utf-8).
   3. Embedding fonts: If you are generating an LRF file to read on your SONY Reader, you are limited by the fact that the Reader only supports a few non-English characters in the fonts it comes pre-loaded with. You can work around this problem by embedding a unicode-aware font that supports the character set your file uses into the LRF file. You should embed atleast a serif and a sans-serif font. Be aware that embedding fonts significantly slows down page-turn speed on the reader.
 
+What's the deal with Table of Contents in MOBI files?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The first thing to realize is that most ebooks have two tables of contents. One is the traditional Table of Contents, like the TOC you find in paper books. This Table of Contents is part of the main document flow and can be styled however you like. This TOC is called the *content TOC*.
+
+Then there is the *metadata TOC*. A metadata TOC is a TOC that is not part of the book text and is typically accessed by some special button on a reader. For example, in the calibre viewer, you use the Show Table of Contents button to see this TOC. This TOC cannot be styled by the book creator. How it is represented is up to the viewer program.
+
+In the MOBI format, the situation is a little confused. This is because the MOBI format, alone amongst mainstream ebook formats, *does not* have decent support for a metadata TOC. A MOBI book simulates the presence of a metadata TOC by putting an *extra* content TOC at the end of the book. When you click Goto Table of Contents on your Kindle, it is to this extra content TOC that the Kindle takes you. 
+
+Now it might well seem to you that the MOBI book has two identical TOCs. Remember that one is semantically a content TOC and the other is a metadata TOC, even though both might have exactly the same entries and look the same. One can be accessed directly from the Kindle's menus, the other cannot. 
+
+When converting to MOBI, calibre detects the *metadata TOC* in the input document and generates an end-of-file TOC in the output MOBI file. You can turn this off by an option in the MOBI Output settings. You cannot control where this generated TOC will go. Remember this TOC is semantically a *metadata TOC*, in any format other than MOBI it *cannot not be part of the text*. The fact that it is part of the text in MOBI is an accident caused by the limitations of MOBI. If you want a TOC at a particular location in your document text, create one by hand. 
+
+If you have a hand edited TOC in the input document, you can use the TOC detection options in calibre to automatically generate the metadata TOC from it. See the conversion section of the User Manual for more details on how to use these options.
+
+Finally, I encourage you to ditch the content TOC and only have a metadata TOC in your ebooks. Metadata TOCs will give the people reading your ebooks a much superior navigation experience (except on the Kindle, where they are essentially the same as a content TOC).
 
 How do I use some of the advanced features of the conversion tools?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

src/calibre/utils/config.py

@@ -785,9 +785,6 @@ def write_tweaks(raw):
 
 tweaks = read_tweaks()
 test_eight_code = tweaks.get('test_eight_code', False)
-# test_eight_code notes
-# Change documentation of bool columns are tristate to indicate that it can be
-# overridden on a per library basis via Preferences->Custom columns
 
 def migrate():
     if hasattr(os, 'geteuid') and os.geteuid() == 0:

src/calibre/utils/ipc/job.py

@@ -75,12 +75,20 @@ class BaseJob(object):
             self._run_state = self.RUNNING
             self._status_text = _('Working...')
 
-        while consume_notifications:
+        if consume_notifications:
+            return self.consume_notifications()
+        return False
+
+    def consume_notifications(self):
+        got_notification = False
+        while self.notifications is not None:
             try:
                 self.percent, self._message = self.notifications.get_nowait()
                 self.percent *= 100.
+                got_notification = True
             except Empty:
                 break
+        return got_notification
 
     @property
     def status_text(self):

src/calibre/utils/logging.py

@@ -66,7 +66,7 @@ class HTMLStream(Stream):
     color = {
             DEBUG: '<span style="color:green">',
             INFO:'<span>',
-            WARN: '<span style="color:yellow">',
+            WARN: '<span style="color:blue">',
             ERROR: '<span style="color:red">'
             }
     normal = '</span>'