Sync to trunk.

resources/recipes/newsweek.recipe

@@ -1,13 +1,13 @@
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
-    
+
 import re
 from calibre import strftime
 from calibre.ebooks.BeautifulSoup import BeautifulSoup
 from calibre.web.feeds.news import BasicNewsRecipe
-    
+
 class Newsweek(BasicNewsRecipe):
-    
+
 
     title          = 'Newsweek'
     __author__     = 'Kovid Goyal and Sujata Raman'
@@ -23,12 +23,12 @@ class Newsweek(BasicNewsRecipe):
                     .issueDate{font-family:arial,helvetica,sans-serif; color:#73726C; font-size:x-small; font-style:italic;}
                     h5{font-family:arial,helvetica,sans-serif; color:#73726C; font-size:x-small;}
                     h6{font-family:arial,helvetica,sans-serif; color:#73726C; font-size:x-small;}
-                    .story{font-family:georgia,sans-serif ; color:#363636;}
+                    .story{font-family:georgia,sans-serif ;color:black;}
                     .photoCredit{color:#999999; font-family:Arial,Helvetica,sans-serif;font-size:x-small;}
                     .photoCaption{color:#0A0A09;font-family:Arial,Helvetica,sans-serif;font-size:x-small;}
                     .fwArticle{font-family:Arial,Helvetica,sans-serif;font-size:x-small;font-weight:bold;}
                     '''
-   
+
     encoding       = 'utf-8'
     language = 'en'
 
@@ -44,7 +44,7 @@ class Newsweek(BasicNewsRecipe):
             dict(name='li', attrs={'id':['slug_bigbox']})
             ]
 
-  
+
     keep_only_tags = [{'class':['article HorizontalHeader', 'articlecontent','photoBox']}, ]
     recursions = 1
     match_regexps = [r'http://www.newsweek.com/id/\S+/page/\d+']

src/calibre/customize/builtins.py

@@ -374,7 +374,8 @@ from calibre.devices.eslick.driver import ESLICK
 from calibre.devices.nuut2.driver import NUUT2
 from calibre.devices.iriver.driver import IRIVER_STORY
 
-plugins = [HTML2ZIP]
+from calibre.ebooks.metadata.fetch import GoogleBooks, ISBNDB
+plugins = [HTML2ZIP, GoogleBooks, ISBNDB]
 plugins += [
     ComicInput,
     EPUBInput,

src/calibre/customize/ui.py

@@ -13,8 +13,9 @@ from calibre.customize.builtins import plugins as builtin_plugins
 from calibre.constants import numeric_version as version, iswindows, isosx
 from calibre.devices.interface import DevicePlugin
 from calibre.ebooks.metadata import MetaInformation
+from calibre.ebooks.metadata.fetch import MetadataSource
 from calibre.utils.config import make_config_dir, Config, ConfigProxy, \
-                                 plugin_dir, OptionParser
+                                 plugin_dir, OptionParser, prefs
 
 
 platform = 'linux'
@@ -89,6 +90,33 @@ def output_profiles():
         if isinstance(plugin, OutputProfile):
             yield plugin
 
+def metadata_sources(customize=True, isbndb_key=None):
+    for plugin in _initialized_plugins:
+        if isinstance(plugin, MetadataSource):
+            if is_disabled(plugin):
+                continue
+            if customize:
+                customization = config['plugin_customization']
+                plugin.site_customization = customization.get(plugin.name, None)
+            if plugin.name == 'IsbnDB' and isbndb_key is not None:
+                plugin.site_customization = isbndb_key
+            if not plugin.is_ok():
+                continue
+            yield plugin
+
+def get_isbndb_key():
+    return config['plugin_customization'].get('IsbnDB', None)
+
+def set_isbndb_key(key):
+    for plugin in _initialized_plugins:
+        if plugin.name == 'IsbnDB':
+            return customize_plugin(plugin, key)
+
+def migrate_isbndb_key():
+    key = prefs['isbndb_com_key']
+    if key:
+        prefs.set('isbndb_com_key', '')
+        set_isbndb_key(key)
 
 def reread_filetype_plugins():
     global _on_import

src/calibre/ebooks/metadata/fetch.py

@@ -8,46 +8,72 @@ from threading import Thread
 
 from calibre import preferred_encoding
 from calibre.utils.config import OptionParser
+from calibre.utils.logging import default_log
 
-class FetchGoogle(Thread):
+from calibre.customize import Plugin
-    name = 'Google Books'
 
-    def __init__(self, title, author, publisher, isbn, verbose):
+class MetadataSource(Plugin):
+
+    author = 'Kovid Goyal'
+    supported_platforms = ['windows', 'osx', 'linux']
+    type = _('Metadata download')
+
+    def __call__(self, title, author, publisher, isbn, verbose, log=None,
+            extra=None):
+        self.worker = Thread(target=self.fetch)
+        self.worker.daemon = True
         self.title = title
         self.verbose = verbose
         self.author = author
         self.publisher = publisher
         self.isbn = isbn
-        Thread.__init__(self, None)
+        self.log = log if log is not None else default_log
-        self.daemon = True
+        self.extra = extra
-        self.exception, self.tb = None, None
+        self.exception, self.tb, self.results = None, None, []
+        self.worker.start()
 
-    def run(self):
+    def fetch(self):
+        '''
+        All the actual work is done here.
+        '''
+        raise NotImplementedError
+
+    def is_ok(self):
+        '''
+        Used to check if the plugin has been correctly customized.
+        For example: The isbndb plugin checks to see if the site_customization
+        has been set with an isbndb.com access key.
+        '''
+        return True
+
+    def join(self):
+        return self.worker.join()
+
+class GoogleBooks(MetadataSource):
+
+    name = 'Google Books'
+
+    def is_ok(self):
+        return bool(self.site_customization)
+
+    def fetch(self):
         from calibre.ebooks.metadata.google_books import search
         try:
             self.results = search(self.title, self.author, self.publisher,
                                   self.isbn, max_results=10,
                                   verbose=self.verbose)
         except Exception, e:
-            self.results = []
             self.exception = e
             self.tb = traceback.format_exc()
 
 
-class FetchISBNDB(Thread):
+class ISBNDB(MetadataSource):
-    name = 'IsbnDB'
-    def __init__(self, title, author, publisher, isbn, verbose, key):
-        self.title = title
-        self.author = author
-        self.publisher = publisher
-        self.isbn = isbn
-        self.verbose = verbose
-        Thread.__init__(self, None)
-        self.daemon = True
-        self.exception, self.tb = None, None
-        self.key = key
 
-    def run(self):
+    name = 'IsbnDB'
+
+    def fetch(self):
+        if not self.site_customization:
+            return
         from calibre.ebooks.metadata.isbndb import option_parser, create_books
         args = ['isbndb']
         if self.isbn:
@@ -61,15 +87,23 @@ class FetchISBNDB(Thread):
                 args.extend(['--publisher', self.publisher])
         if self.verbose:
             args.extend(['--verbose'])
-        args.append(self.key)
+        args.append(self.site_customization) # IsbnDb key
         try:
             opts, args = option_parser().parse_args(args)
             self.results = create_books(opts, args)
         except Exception, e:
-            self.results = []
             self.exception = e
             self.tb = traceback.format_exc()
 
+    def customization_help(self, gui=False):
+        ans = _('To use isbndb.com you must sign up for a %sfree account%s '
+                'and enter your access key below.')
+        if gui:
+            ans = '<p>'+ans%('<a href="http://www.isbndb.com">', '</a>')
+        else:
+            ans = ans.replace('%s', '')
+        return ans
+
 def result_index(source, result):
     if not result.isbn:
         return -1
@@ -90,16 +124,14 @@ def search(title=None, author=None, publisher=None, isbn=None, isbndb_key=None,
            verbose=0):
     assert not(title is None and author is None and publisher is None and \
                    isbn is None)
+    from calibre.customize.ui import metadata_sources, migrate_isbndb_key
+    migrate_isbndb_key()
     if isbn is not None:
         isbn = re.sub(r'[^a-zA-Z0-9]', '', isbn).upper()
-    fetchers = [FetchGoogle(title, author, publisher, isbn, verbose)]
+    fetchers = list(metadata_sources(isbndb_key=isbndb_key))
-    if isbndb_key:
-        fetchers.append(FetchISBNDB(title, author, publisher, isbn, verbose,
-                                        isbndb_key))
-
 
     for fetcher in fetchers:
-        fetcher.start()
+        fetcher(title, author, publisher, isbn, verbose)
     for fetcher in fetchers:
         fetcher.join()
     for fetcher in fetchers[1:]:
@@ -131,7 +163,8 @@ def option_parser():
                       help='Maximum number of results to fetch')
     parser.add_option('-k', '--isbndb-key',
                       help=('The access key for your ISBNDB.com account. '
-                      'Only needed if you want to search isbndb.com'))
+                      'Only needed if you want to search isbndb.com '
+                      'and you haven\'t customized the IsbnDB plugin.'))
     parser.add_option('-v', '--verbose', default=0, action='count',
                       help='Be more verbose about errors')
     return parser

src/calibre/gui2/dialogs/config/__init__.py

@@ -6,7 +6,7 @@ from PyQt4.Qt import    QDialog, QListWidgetItem, QIcon, \
                         QDesktopServices, QVBoxLayout, QLabel, QPlainTextEdit, \
                         QStringListModel, QAbstractItemModel, QFont, \
                         SIGNAL, QThread, Qt, QSize, QVariant, QUrl, \
-                        QModelIndex, QInputDialog, QAbstractTableModel, \
+                        QModelIndex, QAbstractTableModel, \
                         QDialogButtonBox, QTabWidget, QBrush, QLineEdit, \
                         QProgressDialog
 
@@ -550,15 +550,16 @@ class ConfigDialog(ResizableDialog, Ui_Dialog):
                     info_dialog(self, _('Plugin not customizable'),
                         _('Plugin: %s does not need customization')%plugin.name).exec_()
                     return
+                config_dialog = QDialog(self)
+                button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+                v = QVBoxLayout(config_dialog)
+
+                config_dialog.connect(button_box, SIGNAL('accepted()'), config_dialog.accept)
+                config_dialog.connect(button_box, SIGNAL('rejected()'), config_dialog.reject)
+                config_dialog.setWindowTitle(_('Customize') + ' ' + plugin.name)
+
                 if hasattr(plugin, 'config_widget'):
-                    config_dialog = QDialog(self)
-                    button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
-
-                    config_dialog.connect(button_box, SIGNAL('accepted()'), config_dialog.accept)
-                    config_dialog.connect(button_box, SIGNAL('rejected()'), config_dialog.reject)
-
                     config_widget = plugin.config_widget()
-                    v = QVBoxLayout(config_dialog)
                     v.addWidget(config_widget)
                     v.addWidget(button_box)
                     config_dialog.exec_()
@@ -567,17 +568,28 @@ class ConfigDialog(ResizableDialog, Ui_Dialog):
                         plugin.save_settings(config_widget)
                         self._plugin_model.refresh_plugin(plugin)
                 else:
-                    help = plugin.customization_help()
+                    help_text = plugin.customization_help(gui=True)
+                    help_text = QLabel(help_text, config_dialog)
+                    help_text.setWordWrap(True)
+                    help_text.setTextInteractionFlags(Qt.LinksAccessibleByMouse
+                            | Qt.LinksAccessibleByKeyboard)
+                    help_text.setOpenExternalLinks(True)
+                    v.addWidget(help_text)
                     sc = plugin_customization(plugin)
                     if not sc:
                         sc = ''
                     sc = sc.strip()
-                    text, ok = QInputDialog.getText(self, _('Customize %s')%plugin.name,
+                    sc = QLineEdit(sc, config_dialog)
-                                                    help, QLineEdit.Normal, sc)
+                    v.addWidget(sc)
-                    if ok:
+                    v.addWidget(button_box)
-                        customize_plugin(plugin, unicode(text).strip())
+                    config_dialog.exec_()
+
+                    if config_dialog.result() == QDialog.Accepted:
+                        sc = unicode(sc.text()).strip()
+                        customize_plugin(plugin, sc)
+
                     self._plugin_model.refresh_plugin(plugin)
-            if op == 'remove':
+            elif op == 'remove':
                 if remove_plugin(plugin):
                     self._plugin_model.populate()
                     self._plugin_model.reset()

src/calibre/gui2/dialogs/fetch_metadata.py

@@ -13,8 +13,8 @@ from PyQt4.QtGui import QDialog, QItemSelectionModel
 from calibre.gui2.dialogs.fetch_metadata_ui import Ui_FetchMetadata
 from calibre.gui2 import error_dialog, NONE, info_dialog
 from calibre.gui2.widgets import ProgressIndicator
-from calibre.utils.config import prefs
 from calibre import strftime
+from calibre.customize.ui import get_isbndb_key, set_isbndb_key
 
 class Fetcher(QThread):
 
@@ -101,7 +101,7 @@ class FetchMetadata(QDialog, Ui_FetchMetadata):
         self.timeout = timeout
         QObject.connect(self.fetch, SIGNAL('clicked()'), self.fetch_metadata)
 
-        self.key.setText(prefs['isbndb_com_key'])
+        self.key.setText(get_isbndb_key())
 
         self.setWindowTitle(title if title else _('Unknown'))
         self.isbn = isbn
@@ -128,7 +128,7 @@ class FetchMetadata(QDialog, Ui_FetchMetadata):
         self.warning.setVisible(False)
         key = str(self.key.text())
         if key:
-            prefs['isbndb_com_key'] =  key
+            set_isbndb_key(key)
         else:
             key = None
         title = author = publisher = isbn = pubdate = None

src/calibre/gui2/dialogs/metadata_single.py

@@ -27,7 +27,7 @@ from calibre.ebooks.metadata.library_thing import cover_from_isbn
 from calibre import islinux
 from calibre.ebooks.metadata.meta import get_metadata
 from calibre.utils.config import prefs
-from calibre.customize.ui import run_plugins_on_import
+from calibre.customize.ui import run_plugins_on_import, get_isbndb_key
 
 class CoverFetcher(QThread):
 
@@ -50,7 +50,7 @@ class CoverFetcher(QThread):
                     self.needs_isbn = True
                     return
                 au = self.author if self.author else None
-                key = prefs['isbndb_com_key']
+                key = get_isbndb_key()
                 if not key:
                     key = None
                 results = search(title=self.title, author=au,

src/calibre/gui2/metadata.py

@@ -11,8 +11,8 @@ from Queue import Queue, Empty
 
 
 from calibre.ebooks.metadata.fetch import search
-from calibre.utils.config import prefs
 from calibre.ebooks.metadata.library_thing import cover_from_isbn
+from calibre.customize.ui import get_isbndb_key
 
 class Worker(Thread):
 
@@ -64,7 +64,7 @@ class DownloadMetadata(Thread):
             self.tb = traceback.format_exc()
 
     def _run(self):
-        self.key = prefs['isbndb_com_key']
+        self.key = get_isbndb_key()
         if not self.key:
             self.key = None
         self.fetched_metadata = {}

src/calibre/manual/customize.rst

@@ -117,3 +117,9 @@ Metadata plugins add the ability to read/write metadata from ebook files to |app
    :hidden:
    
    plugins
+
+Metadata download plugins
+----------------------------
+
+Metadata download plugins add various sources that |app| uses to download metadata based on title/author/isbn etc. See :ref:`pluginsMetadataSource`
+for details.

src/calibre/manual/plugins.rst

@@ -98,3 +98,32 @@ Metadata plugins
 .. automember:: MetadataWriterPlugin.file_types
 
 .. automethod:: MetadataWriterPlugin.set_metadata
+
+
+.. _pluginsMetadataSource:
+
+Metadata download plugins
+--------------------------
+
+.. class:: calibre.ebooks.metadata.fetch.MetadataSource
+
+    Represents a source to query for metadata. Subclasses must implement
+    at least the fetch method and optionally the is_ok method.
+
+    When :meth:`fetch` is called, the `self` object will have the following
+    useful attributes (each of which may be None)::
+
+        title, author, publisher, isbn, log, verbose and extra
+
+    Use these attributes to construct the search query. extra is reserved for
+    future use.
+
+    The fetch method must store the results in `self.results` as a list of
+    :class:`MetaInformation` objects. If there is an error, it should be stored
+    in `self.exception` and `self.tb` (for the traceback).
+
+.. automethod:: calibre.ebooks.metadata.fetch.MetadataSource.fetch
+
+.. automethod:: calibre.ebooks.metadata.fetch.MetadataSource.is_ok
+
+