From b31c9520079a756f9f6ca37d6418b212ff335260 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 10 Nov 2009 18:16:49 -0700 Subject: [PATCH] Convert metadata download system to use plugins and create builtin plugins to download metadata from Google Books and isbndb.com --- resources/recipes/newsweek.recipe | 12 +-- src/calibre/customize/builtins.py | 3 +- src/calibre/customize/ui.py | 30 ++++++- src/calibre/ebooks/metadata/fetch.py | 93 ++++++++++++++------- src/calibre/gui2/dialogs/config/__init__.py | 40 +++++---- src/calibre/gui2/dialogs/fetch_metadata.py | 6 +- src/calibre/gui2/dialogs/metadata_single.py | 4 +- src/calibre/gui2/metadata.py | 4 +- src/calibre/manual/customize.rst | 6 ++ src/calibre/manual/plugins.rst | 29 +++++++ 10 files changed, 168 insertions(+), 59 deletions(-) diff --git a/resources/recipes/newsweek.recipe b/resources/recipes/newsweek.recipe index c32c68d88a..ff408ca9a5 100644 --- a/resources/recipes/newsweek.recipe +++ b/resources/recipes/newsweek.recipe @@ -1,13 +1,13 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ' - + import re from calibre import strftime from calibre.ebooks.BeautifulSoup import BeautifulSoup from calibre.web.feeds.news import BasicNewsRecipe - + class Newsweek(BasicNewsRecipe): - + title = 'Newsweek' __author__ = 'Kovid Goyal and Sujata Raman' @@ -23,12 +23,12 @@ class Newsweek(BasicNewsRecipe): .issueDate{font-family:arial,helvetica,sans-serif; color:#73726C; font-size:x-small; font-style:italic;} h5{font-family:arial,helvetica,sans-serif; color:#73726C; font-size:x-small;} h6{font-family:arial,helvetica,sans-serif; color:#73726C; font-size:x-small;} - .story{font-family:georgia,sans-serif ; color:#363636;} + .story{font-family:georgia,sans-serif ;color:black;} .photoCredit{color:#999999; font-family:Arial,Helvetica,sans-serif;font-size:x-small;} .photoCaption{color:#0A0A09;font-family:Arial,Helvetica,sans-serif;font-size:x-small;} .fwArticle{font-family:Arial,Helvetica,sans-serif;font-size:x-small;font-weight:bold;} ''' - + encoding = 'utf-8' language = 'en' @@ -44,7 +44,7 @@ class Newsweek(BasicNewsRecipe): dict(name='li', attrs={'id':['slug_bigbox']}) ] - + keep_only_tags = [{'class':['article HorizontalHeader', 'articlecontent','photoBox']}, ] recursions = 1 match_regexps = [r'http://www.newsweek.com/id/\S+/page/\d+'] diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 99e2882b1c..22ae0d4b04 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -374,7 +374,8 @@ from calibre.devices.eslick.driver import ESLICK from calibre.devices.nuut2.driver import NUUT2 from calibre.devices.iriver.driver import IRIVER_STORY -plugins = [HTML2ZIP] +from calibre.ebooks.metadata.fetch import GoogleBooks, ISBNDB +plugins = [HTML2ZIP, GoogleBooks, ISBNDB] plugins += [ ComicInput, EPUBInput, diff --git a/src/calibre/customize/ui.py b/src/calibre/customize/ui.py index 87f853a990..66716fe4bb 100644 --- a/src/calibre/customize/ui.py +++ b/src/calibre/customize/ui.py @@ -13,8 +13,9 @@ from calibre.customize.builtins import plugins as builtin_plugins from calibre.constants import numeric_version as version, iswindows, isosx from calibre.devices.interface import DevicePlugin from calibre.ebooks.metadata import MetaInformation +from calibre.ebooks.metadata.fetch import MetadataSource from calibre.utils.config import make_config_dir, Config, ConfigProxy, \ - plugin_dir, OptionParser + plugin_dir, OptionParser, prefs platform = 'linux' @@ -89,6 +90,33 @@ def output_profiles(): if isinstance(plugin, OutputProfile): yield plugin +def metadata_sources(customize=True, isbndb_key=None): + for plugin in _initialized_plugins: + if isinstance(plugin, MetadataSource): + if is_disabled(plugin): + continue + if customize: + customization = config['plugin_customization'] + plugin.site_customization = customization.get(plugin.name, None) + if plugin.name == 'IsbnDB' and isbndb_key is not None: + plugin.site_customization = isbndb_key + if not plugin.is_ok(): + continue + yield plugin + +def get_isbndb_key(): + return config['plugin_customization'].get('IsbnDB', None) + +def set_isbndb_key(key): + for plugin in _initialized_plugins: + if plugin.name == 'IsbnDB': + return customize_plugin(plugin, key) + +def migrate_isbndb_key(): + key = prefs['isbndb_com_key'] + if key: + prefs.set('isbndb_com_key', '') + set_isbndb_key(key) def reread_filetype_plugins(): global _on_import diff --git a/src/calibre/ebooks/metadata/fetch.py b/src/calibre/ebooks/metadata/fetch.py index f7a6a559b7..5c90914bee 100644 --- a/src/calibre/ebooks/metadata/fetch.py +++ b/src/calibre/ebooks/metadata/fetch.py @@ -8,46 +8,72 @@ from threading import Thread from calibre import preferred_encoding from calibre.utils.config import OptionParser +from calibre.utils.logging import default_log -class FetchGoogle(Thread): - name = 'Google Books' +from calibre.customize import Plugin - def __init__(self, title, author, publisher, isbn, verbose): +class MetadataSource(Plugin): + + author = 'Kovid Goyal' + supported_platforms = ['windows', 'osx', 'linux'] + type = _('Metadata download') + + def __call__(self, title, author, publisher, isbn, verbose, log=None, + extra=None): + self.worker = Thread(target=self.fetch) + self.worker.daemon = True self.title = title self.verbose = verbose self.author = author self.publisher = publisher self.isbn = isbn - Thread.__init__(self, None) - self.daemon = True - self.exception, self.tb = None, None + self.log = log if log is not None else default_log + self.extra = extra + self.exception, self.tb, self.results = None, None, [] + self.worker.start() - def run(self): + def fetch(self): + ''' + All the actual work is done here. + ''' + raise NotImplementedError + + def is_ok(self): + ''' + Used to check if the plugin has been correctly customized. + For example: The isbndb plugin checks to see if the site_customization + has been set with an isbndb.com access key. + ''' + return True + + def join(self): + return self.worker.join() + +class GoogleBooks(MetadataSource): + + name = 'Google Books' + + def is_ok(self): + return bool(self.site_customization) + + def fetch(self): from calibre.ebooks.metadata.google_books import search try: self.results = search(self.title, self.author, self.publisher, self.isbn, max_results=10, verbose=self.verbose) except Exception, e: - self.results = [] self.exception = e self.tb = traceback.format_exc() -class FetchISBNDB(Thread): - name = 'IsbnDB' - def __init__(self, title, author, publisher, isbn, verbose, key): - self.title = title - self.author = author - self.publisher = publisher - self.isbn = isbn - self.verbose = verbose - Thread.__init__(self, None) - self.daemon = True - self.exception, self.tb = None, None - self.key = key +class ISBNDB(MetadataSource): - def run(self): + name = 'IsbnDB' + + def fetch(self): + if not self.site_customization: + return from calibre.ebooks.metadata.isbndb import option_parser, create_books args = ['isbndb'] if self.isbn: @@ -61,15 +87,23 @@ class FetchISBNDB(Thread): args.extend(['--publisher', self.publisher]) if self.verbose: args.extend(['--verbose']) - args.append(self.key) + args.append(self.site_customization) # IsbnDb key try: opts, args = option_parser().parse_args(args) self.results = create_books(opts, args) except Exception, e: - self.results = [] self.exception = e self.tb = traceback.format_exc() + def customization_help(self, gui=False): + ans = _('To use isbndb.com you must sign up for a %sfree account%s ' + 'and enter your access key below.') + if gui: + ans = '

'+ans%('', '') + else: + ans = ans.replace('%s', '') + return ans + def result_index(source, result): if not result.isbn: return -1 @@ -90,16 +124,14 @@ def search(title=None, author=None, publisher=None, isbn=None, isbndb_key=None, verbose=0): assert not(title is None and author is None and publisher is None and \ isbn is None) + from calibre.customize.ui import metadata_sources, migrate_isbndb_key + migrate_isbndb_key() if isbn is not None: isbn = re.sub(r'[^a-zA-Z0-9]', '', isbn).upper() - fetchers = [FetchGoogle(title, author, publisher, isbn, verbose)] - if isbndb_key: - fetchers.append(FetchISBNDB(title, author, publisher, isbn, verbose, - isbndb_key)) - + fetchers = list(metadata_sources(isbndb_key=isbndb_key)) for fetcher in fetchers: - fetcher.start() + fetcher(title, author, publisher, isbn, verbose) for fetcher in fetchers: fetcher.join() for fetcher in fetchers[1:]: @@ -131,7 +163,8 @@ def option_parser(): help='Maximum number of results to fetch') parser.add_option('-k', '--isbndb-key', help=('The access key for your ISBNDB.com account. ' - 'Only needed if you want to search isbndb.com')) + 'Only needed if you want to search isbndb.com ' + 'and you haven\'t customized the IsbnDB plugin.')) parser.add_option('-v', '--verbose', default=0, action='count', help='Be more verbose about errors') return parser diff --git a/src/calibre/gui2/dialogs/config/__init__.py b/src/calibre/gui2/dialogs/config/__init__.py index ad794e14e6..62f68054f1 100644 --- a/src/calibre/gui2/dialogs/config/__init__.py +++ b/src/calibre/gui2/dialogs/config/__init__.py @@ -6,7 +6,7 @@ from PyQt4.Qt import QDialog, QListWidgetItem, QIcon, \ QDesktopServices, QVBoxLayout, QLabel, QPlainTextEdit, \ QStringListModel, QAbstractItemModel, QFont, \ SIGNAL, QThread, Qt, QSize, QVariant, QUrl, \ - QModelIndex, QInputDialog, QAbstractTableModel, \ + QModelIndex, QAbstractTableModel, \ QDialogButtonBox, QTabWidget, QBrush, QLineEdit, \ QProgressDialog @@ -550,15 +550,16 @@ class ConfigDialog(ResizableDialog, Ui_Dialog): info_dialog(self, _('Plugin not customizable'), _('Plugin: %s does not need customization')%plugin.name).exec_() return + config_dialog = QDialog(self) + button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel) + v = QVBoxLayout(config_dialog) + + config_dialog.connect(button_box, SIGNAL('accepted()'), config_dialog.accept) + config_dialog.connect(button_box, SIGNAL('rejected()'), config_dialog.reject) + config_dialog.setWindowTitle(_('Customize') + ' ' + plugin.name) + if hasattr(plugin, 'config_widget'): - config_dialog = QDialog(self) - button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel) - - config_dialog.connect(button_box, SIGNAL('accepted()'), config_dialog.accept) - config_dialog.connect(button_box, SIGNAL('rejected()'), config_dialog.reject) - config_widget = plugin.config_widget() - v = QVBoxLayout(config_dialog) v.addWidget(config_widget) v.addWidget(button_box) config_dialog.exec_() @@ -567,17 +568,28 @@ class ConfigDialog(ResizableDialog, Ui_Dialog): plugin.save_settings(config_widget) self._plugin_model.refresh_plugin(plugin) else: - help = plugin.customization_help() + help_text = plugin.customization_help(gui=True) + help_text = QLabel(help_text, config_dialog) + help_text.setWordWrap(True) + help_text.setTextInteractionFlags(Qt.LinksAccessibleByMouse + | Qt.LinksAccessibleByKeyboard) + help_text.setOpenExternalLinks(True) + v.addWidget(help_text) sc = plugin_customization(plugin) if not sc: sc = '' sc = sc.strip() - text, ok = QInputDialog.getText(self, _('Customize %s')%plugin.name, - help, QLineEdit.Normal, sc) - if ok: - customize_plugin(plugin, unicode(text).strip()) + sc = QLineEdit(sc, config_dialog) + v.addWidget(sc) + v.addWidget(button_box) + config_dialog.exec_() + + if config_dialog.result() == QDialog.Accepted: + sc = unicode(sc.text()).strip() + customize_plugin(plugin, sc) + self._plugin_model.refresh_plugin(plugin) - if op == 'remove': + elif op == 'remove': if remove_plugin(plugin): self._plugin_model.populate() self._plugin_model.reset() diff --git a/src/calibre/gui2/dialogs/fetch_metadata.py b/src/calibre/gui2/dialogs/fetch_metadata.py index ea076b42a1..d092c1f5c4 100644 --- a/src/calibre/gui2/dialogs/fetch_metadata.py +++ b/src/calibre/gui2/dialogs/fetch_metadata.py @@ -13,8 +13,8 @@ from PyQt4.QtGui import QDialog, QItemSelectionModel from calibre.gui2.dialogs.fetch_metadata_ui import Ui_FetchMetadata from calibre.gui2 import error_dialog, NONE, info_dialog from calibre.gui2.widgets import ProgressIndicator -from calibre.utils.config import prefs from calibre import strftime +from calibre.customize.ui import get_isbndb_key, set_isbndb_key class Fetcher(QThread): @@ -101,7 +101,7 @@ class FetchMetadata(QDialog, Ui_FetchMetadata): self.timeout = timeout QObject.connect(self.fetch, SIGNAL('clicked()'), self.fetch_metadata) - self.key.setText(prefs['isbndb_com_key']) + self.key.setText(get_isbndb_key()) self.setWindowTitle(title if title else _('Unknown')) self.isbn = isbn @@ -128,7 +128,7 @@ class FetchMetadata(QDialog, Ui_FetchMetadata): self.warning.setVisible(False) key = str(self.key.text()) if key: - prefs['isbndb_com_key'] = key + set_isbndb_key(key) else: key = None title = author = publisher = isbn = pubdate = None diff --git a/src/calibre/gui2/dialogs/metadata_single.py b/src/calibre/gui2/dialogs/metadata_single.py index 76e0a02210..77f214a793 100644 --- a/src/calibre/gui2/dialogs/metadata_single.py +++ b/src/calibre/gui2/dialogs/metadata_single.py @@ -27,7 +27,7 @@ from calibre.ebooks.metadata.library_thing import cover_from_isbn from calibre import islinux from calibre.ebooks.metadata.meta import get_metadata from calibre.utils.config import prefs -from calibre.customize.ui import run_plugins_on_import +from calibre.customize.ui import run_plugins_on_import, get_isbndb_key class CoverFetcher(QThread): @@ -50,7 +50,7 @@ class CoverFetcher(QThread): self.needs_isbn = True return au = self.author if self.author else None - key = prefs['isbndb_com_key'] + key = get_isbndb_key() if not key: key = None results = search(title=self.title, author=au, diff --git a/src/calibre/gui2/metadata.py b/src/calibre/gui2/metadata.py index 247880d7b0..adbc2bfb9a 100644 --- a/src/calibre/gui2/metadata.py +++ b/src/calibre/gui2/metadata.py @@ -11,8 +11,8 @@ from Queue import Queue, Empty from calibre.ebooks.metadata.fetch import search -from calibre.utils.config import prefs from calibre.ebooks.metadata.library_thing import cover_from_isbn +from calibre.customize.ui import get_isbndb_key class Worker(Thread): @@ -64,7 +64,7 @@ class DownloadMetadata(Thread): self.tb = traceback.format_exc() def _run(self): - self.key = prefs['isbndb_com_key'] + self.key = get_isbndb_key() if not self.key: self.key = None self.fetched_metadata = {} diff --git a/src/calibre/manual/customize.rst b/src/calibre/manual/customize.rst index 21a65ef1fa..149a139216 100644 --- a/src/calibre/manual/customize.rst +++ b/src/calibre/manual/customize.rst @@ -117,3 +117,9 @@ Metadata plugins add the ability to read/write metadata from ebook files to |app :hidden: plugins + +Metadata download plugins +---------------------------- + +Metadata download plugins add various sources that |app| uses to download metadata based on title/author/isbn etc. See :ref:`pluginsMetadataSource` +for details. diff --git a/src/calibre/manual/plugins.rst b/src/calibre/manual/plugins.rst index 706dc281d3..8ba33e036d 100644 --- a/src/calibre/manual/plugins.rst +++ b/src/calibre/manual/plugins.rst @@ -98,3 +98,32 @@ Metadata plugins .. automember:: MetadataWriterPlugin.file_types .. automethod:: MetadataWriterPlugin.set_metadata + + +.. _pluginsMetadataSource: + +Metadata download plugins +-------------------------- + +.. class:: calibre.ebooks.metadata.fetch.MetadataSource + + Represents a source to query for metadata. Subclasses must implement + at least the fetch method and optionally the is_ok method. + + When :meth:`fetch` is called, the `self` object will have the following + useful attributes (each of which may be None):: + + title, author, publisher, isbn, log, verbose and extra + + Use these attributes to construct the search query. extra is reserved for + future use. + + The fetch method must store the results in `self.results` as a list of + :class:`MetaInformation` objects. If there is an error, it should be stored + in `self.exception` and `self.tb` (for the traceback). + +.. automethod:: calibre.ebooks.metadata.fetch.MetadataSource.fetch + +.. automethod:: calibre.ebooks.metadata.fetch.MetadataSource.is_ok + +