From e5da008e24bb90d257767098dac84ebfdf0a1558 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 10 Apr 2011 17:12:24 -0600 Subject: [PATCH 01/14] Add button to edit metadata single to configure metadata downloading --- src/calibre/ebooks/metadata/sources/amazon.py | 3 +- src/calibre/gui2/metadata/single.py | 17 ++++- src/calibre/gui2/preferences/__init__.py | 52 ++++++++----- .../gui2/preferences/metadata_sources.py | 73 +++++++++++++++++-- .../gui2/preferences/metadata_sources.ui | 3 + 5 files changed, 121 insertions(+), 27 deletions(-) diff --git a/src/calibre/ebooks/metadata/sources/amazon.py b/src/calibre/ebooks/metadata/sources/amazon.py index 2ad5bfbacc..24df68e51d 100644 --- a/src/calibre/ebooks/metadata/sources/amazon.py +++ b/src/calibre/ebooks/metadata/sources/amazon.py @@ -284,7 +284,8 @@ class Amazon(Source): capabilities = frozenset(['identify', 'cover']) touched_fields = frozenset(['title', 'authors', 'identifier:amazon', - 'identifier:isbn', 'rating', 'comments', 'publisher', 'pubdate']) + 'identifier:isbn', 'rating', 'comments', 'publisher', 'pubdate', + 'language']) has_html_comments = True supports_gzip_transfer_encoding = True diff --git a/src/calibre/gui2/metadata/single.py b/src/calibre/gui2/metadata/single.py index 175af291f1..7c9dd00734 100644 --- a/src/calibre/gui2/metadata/single.py +++ b/src/calibre/gui2/metadata/single.py @@ -166,6 +166,11 @@ class MetadataSingleDialogBase(ResizableDialog): font.setBold(True) self.fetch_metadata_button.setFont(font) + self.config_metadata_button = QToolButton(self) + self.config_metadata_button.setIcon(QIcon(I('config.png'))) + self.config_metadata_button.clicked.connect(self.configure_metadata) + self.config_metadata_button.setToolTip( + _('Change how calibre downloads metadata')) # }}} @@ -315,6 +320,12 @@ class MetadataSingleDialogBase(ResizableDialog): if d.cover_pixmap is not None: self.cover.current_val = pixmap_to_data(d.cover_pixmap) + def configure_metadata(self): + from calibre.gui2.preferences import show_config_widget + gui = self.parent() + show_config_widget('Sharing', 'Metadata download', parent=self, + gui=gui, never_shutdown=True) + def download_cover(self, *args): from calibre.gui2.metadata.single_download import CoverFetch d = CoverFetch(self.cover.pixmap(), self) @@ -451,7 +462,8 @@ class MetadataSingleDialog(MetadataSingleDialogBase): # {{{ sto = QWidget.setTabOrder sto(self.button_box, self.fetch_metadata_button) - sto(self.fetch_metadata_button, self.title) + sto(self.fetch_metadata_button, self.config_metadata_button) + sto(self.config_metadata_button, self.title) def create_row(row, one, two, three, col=1, icon='forward.png'): ql = BuddyLabel(one) @@ -530,7 +542,8 @@ class MetadataSingleDialog(MetadataSingleDialogBase): # {{{ self.tabs[0].spc_two = QSpacerItem(10, 10, QSizePolicy.Expanding, QSizePolicy.Expanding) l.addItem(self.tabs[0].spc_two, 8, 0, 1, 3) - l.addWidget(self.fetch_metadata_button, 9, 0, 1, 3) + l.addWidget(self.fetch_metadata_button, 9, 0, 1, 2) + l.addWidget(self.config_metadata_button, 9, 2, 1, 1) self.tabs[0].gb2 = gb = QGroupBox(_('Co&mments'), self) gb.l = l = QVBoxLayout() diff --git a/src/calibre/gui2/preferences/__init__.py b/src/calibre/gui2/preferences/__init__.py index b482429504..1669e24059 100644 --- a/src/calibre/gui2/preferences/__init__.py +++ b/src/calibre/gui2/preferences/__init__.py @@ -7,8 +7,9 @@ __docformat__ = 'restructuredtext en' import textwrap -from PyQt4.Qt import QWidget, pyqtSignal, QCheckBox, QAbstractSpinBox, \ - QLineEdit, QComboBox, QVariant, Qt +from PyQt4.Qt import (QWidget, pyqtSignal, QCheckBox, QAbstractSpinBox, + QLineEdit, QComboBox, QVariant, Qt, QIcon, QDialog, QVBoxLayout, + QDialogButtonBox) from calibre.customize.ui import preferences_plugins from calibre.utils.config import ConfigProxy @@ -284,7 +285,14 @@ def get_plugin(category, name): 'No Preferences Plugin with category: %s and name: %s found' % (category, name)) -# Testing {{{ +class ConfigDialog(QDialog): + def set_widget(self, w): self.w = w + def accept(self): + try: + self.restart_required = self.w.commit() + except AbortCommit: + return + QDialog.accept(self) def init_gui(): from calibre.gui2.ui import Main @@ -298,21 +306,24 @@ def init_gui(): gui.initialize(db.library_path, db, None, actions, show_gui=False) return gui -def test_widget(category, name, gui=None): - from PyQt4.Qt import QDialog, QVBoxLayout, QDialogButtonBox - class Dialog(QDialog): - def set_widget(self, w): self.w = w - def accept(self): - try: - self.restart_required = self.w.commit() - except AbortCommit: - return - QDialog.accept(self) +def show_config_widget(category, name, gui=None, show_restart_msg=False, + parent=None, never_shutdown=False): + ''' + Show the preferences plugin identified by category and name + :param gui: gui instance, if None a hidden gui is created + :param show_restart_msg: If True and the preferences plugin indicates a + restart is required, show a message box telling the user to restart + :param parent: The parent of the displayed dialog + + :return: True iff a restart is required for the changes made by the user to + take effect + ''' pl = get_plugin(category, name) - d = Dialog() + d = ConfigDialog(parent) d.resize(750, 550) - d.setWindowTitle(category + " - " + name) + d.setWindowTitle(_('Configure ') + name) + d.setWindowIcon(QIcon(I('config.png'))) bb = QDialogButtonBox(d) bb.setStandardButtons(bb.Apply|bb.Cancel|bb.RestoreDefaults) bb.accepted.connect(d.accept) @@ -335,11 +346,18 @@ def test_widget(category, name, gui=None): w.genesis(gui) w.initialize() d.exec_() - if getattr(d, 'restart_required', False): + rr = getattr(d, 'restart_required', False) + if show_restart_msg and rr: from calibre.gui2 import warning_dialog warning_dialog(gui, 'Restart required', 'Restart required', show=True) - if mygui: + if mygui and not never_shutdown: gui.shutdown() + return rr + +# Testing {{{ + +def test_widget(category, name, gui=None): + show_config_widget(category, name, gui=gui, show_restart_msg=True) def test_all(): from PyQt4.Qt import QApplication diff --git a/src/calibre/gui2/preferences/metadata_sources.py b/src/calibre/gui2/preferences/metadata_sources.py index 66fd35e038..8324684bc8 100644 --- a/src/calibre/gui2/preferences/metadata_sources.py +++ b/src/calibre/gui2/preferences/metadata_sources.py @@ -15,7 +15,7 @@ from calibre.gui2.preferences import ConfigWidgetBase, test_widget from calibre.gui2.preferences.metadata_sources_ui import Ui_Form from calibre.ebooks.metadata.sources.base import msprefs from calibre.customize.ui import (all_metadata_plugins, is_disabled, - enable_plugin, disable_plugin, restore_plugin_state_to_default) + enable_plugin, disable_plugin, default_disabled_plugins) from calibre.gui2 import NONE class SourcesModel(QAbstractTableModel): # {{{ @@ -116,11 +116,11 @@ class SourcesModel(QAbstractTableModel): # {{{ self.cover_overrides = {} def restore_defaults(self): - del msprefs['cover_priorities'] - self.enabled_overrides = {} - self.cover_overrides = {} - for plugin in self.plugins: - restore_plugin_state_to_default(plugin) + self.enabled_overrides = dict([(p, (Qt.Unchecked if p.name in + default_disabled_plugins else Qt.Checked)) for p in self.plugins]) + self.cover_overrides = dict([(p, + msprefs.defaults['cover_priorities'].get(p.name, 1)) + for p in self.plugins]) self.reset() # }}} @@ -131,6 +131,18 @@ class FieldsModel(QAbstractListModel): # {{{ QAbstractTableModel.__init__(self, parent) self.fields = [] + self.descs = { + 'authors': _('Authors'), + 'comments': _('Comments'), + 'pubdate': _('Published date'), + 'publisher': _('Publisher'), + 'rating' : _('Rating'), + 'tags' : _('Tags'), + 'title': _('Title'), + 'series': _('Series'), + 'language': _('Language'), + } + self.overrides = {} def rowCount(self, parent=None): return len(self.fields) @@ -141,10 +153,54 @@ class FieldsModel(QAbstractListModel): # {{{ fields |= p.touched_fields self.fields = [] for x in fields: - if not x.startswith('identifiers:'): + if not x.startswith('identifier:') and x not in ('series_index',): self.fields.append(x) + self.fields.sort(key=lambda x:self.descs.get(x, x)) self.reset() + def state(self, field, defaults=False): + src = msprefs.defaults if defaults else msprefs + return (Qt.Unchecked if field in src['ignore_fields'] + else Qt.Checked) + + def data(self, index, role): + try: + field = self.fields[index.row()] + except: + return None + if role == Qt.DisplayRole: + return self.descs.get(field, field) + if role == Qt.CheckStateRole: + return self.overrides.get(field, self.state(field)) + return NONE + + def flags(self, index): + ans = QAbstractTableModel.flags(self, index) + return ans | Qt.ItemIsUserCheckable + + def restore_defaults(self): + self.overrides = dict([(f, self.state(f, True)) for f in self.fields]) + self.reset() + + def setData(self, index, val, role): + try: + field = self.fields[index.row()] + except: + return False + ret = False + if role == Qt.CheckStateRole: + val, ok = val.toInt() + if ok: + self.overrides[field] = val + ret = True + if ret: + self.dataChanged.emit(index, index) + return ret + + def commit(self): + val = [k for k, v in self.overrides.iteritems() if v == Qt.Unchecked] + msprefs['ignore_fields'] = val + # }}} @@ -173,14 +229,17 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): ConfigWidgetBase.initialize(self) self.sources_model.initialize() self.sources_view.resizeColumnsToContents() + self.fields_model.initialize() def restore_defaults(self): ConfigWidgetBase.restore_defaults(self) self.sources_model.restore_defaults() + self.fields_model.restore_defaults() self.changed_signal.emit() def commit(self): self.sources_model.commit() + self.fields_model.commit() return ConfigWidgetBase.commit(self) if __name__ == '__main__': diff --git a/src/calibre/gui2/preferences/metadata_sources.ui b/src/calibre/gui2/preferences/metadata_sources.ui index 68ac6352cf..546120f628 100644 --- a/src/calibre/gui2/preferences/metadata_sources.ui +++ b/src/calibre/gui2/preferences/metadata_sources.ui @@ -73,6 +73,9 @@ If you uncheck any fields, metadata for those fields will not be downloaded + + QAbstractItemView::NoSelection + From 9084f0be4529b1cfe88cc46ae845a4c365ccc69d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 10 Apr 2011 17:23:12 -0600 Subject: [PATCH 02/14] For kiwidude --- src/calibre/gui2/ui.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 8b2beedfd4..4a9d76418c 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -449,12 +449,12 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ self.library_view.model().count_changed() prefs['library_path'] = self.library_path db = self.library_view.model().db - for action in self.iactions.values(): - action.library_changed(db) self.set_window_title() self.apply_named_search_restriction('') # reset restriction to null self.saved_searches_changed() # reload the search restrictions combo box self.apply_named_search_restriction(db.prefs['gui_restriction']) + for action in self.iactions.values(): + action.library_changed(db) if olddb is not None: try: if call_close: From 385986fe4b0f284e50e19436b5a13612a1267fdb Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 10 Apr 2011 17:28:38 -0600 Subject: [PATCH 03/14] ... --- src/calibre/ebooks/metadata/sources/identify.py | 3 --- src/calibre/gui2/metadata/single.py | 5 +++++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/calibre/ebooks/metadata/sources/identify.py b/src/calibre/ebooks/metadata/sources/identify.py index cd658a0daf..fad810c26e 100644 --- a/src/calibre/ebooks/metadata/sources/identify.py +++ b/src/calibre/ebooks/metadata/sources/identify.py @@ -357,11 +357,8 @@ def identify(log, abort, # {{{ if r.plugin.has_html_comments and r.comments: r.comments = html2text(r.comments) - dummy = Metadata(_('Unknown')) max_tags = msprefs['max_tags'] for r in results: - for f in msprefs['ignore_fields']: - setattr(r, f, getattr(dummy, f)) r.tags = r.tags[:max_tags] return results diff --git a/src/calibre/gui2/metadata/single.py b/src/calibre/gui2/metadata/single.py index 7c9dd00734..4ce76d8cc8 100644 --- a/src/calibre/gui2/metadata/single.py +++ b/src/calibre/gui2/metadata/single.py @@ -24,6 +24,7 @@ from calibre.gui2.metadata.basic_widgets import (TitleEdit, AuthorsEdit, from calibre.gui2.metadata.single_download import FullFetch from calibre.gui2.custom_column_widgets import populate_metadata_page from calibre.utils.config import tweaks +from calibre.ebooks.metadata.book.base import Metadata class MetadataSingleDialogBase(ResizableDialog): @@ -314,7 +315,11 @@ class MetadataSingleDialogBase(ResizableDialog): ret = d.start(title=self.title.current_val, authors=self.authors.current_val, identifiers=self.identifiers.current_val) if ret == d.Accepted: + from calibre.ebooks.metadata.sources.base import msprefs mi = d.book + dummy = Metadata(_('Unknown')) + for f in msprefs['ignore_fields']: + setattr(mi, f, getattr(dummy, f)) if mi is not None: self.update_from_mi(mi) if d.cover_pixmap is not None: From f628964592de086e984f6793e9b9cc84e35f7263 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 10 Apr 2011 21:15:20 -0600 Subject: [PATCH 04/14] ... --- src/calibre/ebooks/metadata/sources/base.py | 36 +++++++++- .../ebooks/metadata/sources/openlibrary.py | 2 +- .../gui2/preferences/metadata_sources.py | 69 +++++++++++++++++-- 3 files changed, 99 insertions(+), 8 deletions(-) diff --git a/src/calibre/ebooks/metadata/sources/base.py b/src/calibre/ebooks/metadata/sources/base.py index 67a80d5785..5089d8951b 100644 --- a/src/calibre/ebooks/metadata/sources/base.py +++ b/src/calibre/ebooks/metadata/sources/base.py @@ -131,7 +131,22 @@ def fixcase(x): x = titlecase(x) return x +class Option(object): + __slots__ = ['type', 'default', 'label', 'desc', 'name', 'choices'] + def __init__(self, name, type_, default, label, desc, choices=None): + ''' + :param name: The name of this option. Must be a valid python identifier + :param type_: The type of this option, one of ('number', 'string', + 'bool', 'choices') + :param default: The default value for this option + :param label: A short (few words) description of this option + :param desc: A longer description of this option + :param choices: A list of possible values, used only if type='choices' + ''' + self.name, self.type, self.default, self.label, self.desc = (name, + type_, default, label, desc) + self.choices = choices class Source(Plugin): @@ -158,10 +173,14 @@ class Source(Plugin): supports_gzip_transfer_encoding = False #: Cached cover URLs can sometimes be unreliable (i.e. the download could - #: fail or the returned image could be bogus. If that is the case set this to - #: False + #: fail or the returned image could be bogus. If that is often the case + #: with this source set to False cached_cover_url_is_reliable = True + #: A list of :class:`Option` objects. They will be used to automatically + #: construct the configuration widget for this plugin + options = () + def __init__(self, *args, **kwargs): Plugin.__init__(self, *args, **kwargs) @@ -170,6 +189,9 @@ class Source(Plugin): self.cache_lock = threading.RLock() self._config_obj = None self._browser = None + self.prefs.defaults['ignore_fields'] = [] + for opt in self.options: + self.prefs.defaults[opt.name] = opt.default # Configuration {{{ @@ -180,6 +202,16 @@ class Source(Plugin): ''' return True + def is_customizable(self): + return True + + def config_widget(self): + from calibre.gui2.metadata.config import ConfigWidget + return ConfigWidget(self) + + def save_settings(self, config_widget): + config_widget.commit() + @property def prefs(self): if self._config_obj is None: diff --git a/src/calibre/ebooks/metadata/sources/openlibrary.py b/src/calibre/ebooks/metadata/sources/openlibrary.py index 19b8747265..4645d2a18a 100644 --- a/src/calibre/ebooks/metadata/sources/openlibrary.py +++ b/src/calibre/ebooks/metadata/sources/openlibrary.py @@ -12,7 +12,7 @@ from calibre.ebooks.metadata.sources.base import Source class OpenLibrary(Source): name = 'Open Library' - description = _('Downloads metadata from The Open Library') + description = _('Downloads covers from The Open Library') capabilities = frozenset(['cover']) diff --git a/src/calibre/gui2/preferences/metadata_sources.py b/src/calibre/gui2/preferences/metadata_sources.py index 8324684bc8..4500a03b30 100644 --- a/src/calibre/gui2/preferences/metadata_sources.py +++ b/src/calibre/gui2/preferences/metadata_sources.py @@ -9,14 +9,15 @@ __docformat__ = 'restructuredtext en' from operator import attrgetter -from PyQt4.Qt import (QAbstractTableModel, Qt, QAbstractListModel) +from PyQt4.Qt import (QAbstractTableModel, Qt, QAbstractListModel, QWidget, + pyqtSignal, QVBoxLayout, QDialogButtonBox, QFrame, QLabel) from calibre.gui2.preferences import ConfigWidgetBase, test_widget from calibre.gui2.preferences.metadata_sources_ui import Ui_Form from calibre.ebooks.metadata.sources.base import msprefs from calibre.customize.ui import (all_metadata_plugins, is_disabled, enable_plugin, disable_plugin, default_disabled_plugins) -from calibre.gui2 import NONE +from calibre.gui2 import NONE, error_dialog class SourcesModel(QAbstractTableModel): # {{{ @@ -64,7 +65,8 @@ class SourcesModel(QAbstractTableModel): # {{{ elif role == Qt.CheckStateRole and col == 0: orig = Qt.Unchecked if is_disabled(plugin) else Qt.Checked return self.enabled_overrides.get(plugin, orig) - + elif role == Qt.UserRole: + return plugin return NONE def setData(self, index, val, role): @@ -127,6 +129,7 @@ class SourcesModel(QAbstractTableModel): # {{{ class FieldsModel(QAbstractListModel): # {{{ + def __init__(self, parent=None): QAbstractTableModel.__init__(self, parent) @@ -143,6 +146,7 @@ class FieldsModel(QAbstractListModel): # {{{ 'language': _('Language'), } self.overrides = {} + self.exclude = frozenset(['series_index']) def rowCount(self, parent=None): return len(self.fields) @@ -153,7 +157,7 @@ class FieldsModel(QAbstractListModel): # {{{ fields |= p.touched_fields self.fields = [] for x in fields: - if not x.startswith('identifier:') and x not in ('series_index',): + if not x.startswith('identifier:') and x not in self.exclude: self.fields.append(x) self.fields.sort(key=lambda x:self.descs.get(x, x)) self.reset() @@ -204,6 +208,41 @@ class FieldsModel(QAbstractListModel): # {{{ # }}} +class PluginConfig(QWidget): # {{{ + + finished = pyqtSignal() + + def __init__(self, plugin, parent): + QWidget.__init__(self, parent) + + self.plugin = plugin + + self.l = l = QVBoxLayout() + self.setLayout(l) + self.c = c = QLabel(_('Configure %s
%s') % (plugin.name, + plugin.description)) + c.setAlignment(Qt.AlignHCenter) + l.addWidget(c) + + self.config_widget = plugin.config_widget() + self.l.addWidget(self.config_widget) + + self.bb = QDialogButtonBox( + QDialogButtonBox.Save|QDialogButtonBox.Cancel, + parent=self) + self.bb.accepted.connect(self.finished) + self.bb.rejected.connect(self.finished) + self.bb.accepted.connect(self.commit) + l.addWidget(self.bb) + + self.f = QFrame(self) + self.f.setFrameShape(QFrame.HLine) + l.addWidget(self.f) + + def commit(self): + self.plugin.save_settings(self.config_widget) +# }}} + class ConfigWidget(ConfigWidgetBase, Ui_Form): def genesis(self, gui): @@ -223,7 +262,27 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): self.fields_model.dataChanged.connect(self.changed_signal) def configure_plugin(self): - pass + for index in self.sources_view.selectionModel().selectedRows(): + plugin = self.sources_model.data(index, Qt.UserRole) + if plugin is not NONE: + return self.do_config(plugin) + error_dialog(self, _('No source selected'), + _('No source selected, cannot configure.'), show=True) + + def do_config(self, plugin): + self.pc = PluginConfig(plugin, self) + self.stack.insertWidget(1, self.pc) + self.stack.setCurrentIndex(1) + self.pc.finished.connect(self.pc_finished) + + def pc_finished(self): + try: + self.pc.finished.diconnect() + except: + pass + self.stack.setCurrentIndex(0) + self.stack.removeWidget(self.pc) + self.pc = None def initialize(self): ConfigWidgetBase.initialize(self) From 5289f7edc81cba424321def02bd32685c610c4cd Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 10 Apr 2011 21:21:33 -0600 Subject: [PATCH 05/14] ... --- src/calibre/gui2/metadata/config.py | 70 +++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 src/calibre/gui2/metadata/config.py diff --git a/src/calibre/gui2/metadata/config.py b/src/calibre/gui2/metadata/config.py new file mode 100644 index 0000000000..83d51c2e28 --- /dev/null +++ b/src/calibre/gui2/metadata/config.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + + +from PyQt4.Qt import (QWidget, QGridLayout, QGroupBox, QListView, Qt) +from calibre.gui2.preferences.metadata_sources import FieldsModel as FM + +class FieldsModel(FM): # {{{ + + def __init__(self, plugin): + FM.__init__(self) + self.plugin = plugin + self.exclude = frozenset(['title', 'authors']) | self.exclude + self.prefs = self.plugin.prefs + + def initialize(self): + fields = self.plugin.touched_fields + self.fields = [] + for x in fields: + if not x.startswith('identifier:') and x not in self.exclude: + self.fields.append(x) + self.fields.sort(key=lambda x:self.descs.get(x, x)) + self.reset() + + def state(self, field, defaults=False): + src = self.prefs.defaults if defaults else self.prefs + return (Qt.Unchecked if field in src['ignore_fields'] + else Qt.Checked) + + def restore_defaults(self): + self.overrides = dict([(f, self.state(f, True)) for f in self.fields]) + self.reset() + + def commit(self): + val = [k for k, v in self.overrides.iteritems() if v == Qt.Unchecked] + self.prefs['ignore_fields'] = val + +# }}} + +class ConfigWidget(QWidget): + + def __init__(self, plugin): + QWidget.__init__(self) + self.plugin = plugin + + self.l = l = QGridLayout() + self.setLayout(l) + + self.gb = QGroupBox(_('Downloaded metadata fields'), self) + l.addWidget(self.gb, 0, 0) + self.gb.l = QGridLayout() + self.gb.setLayout(self.gb.l) + self.fields_view = v = QListView(self) + self.gb.l.addWidget(v, 0, 0) + v.setFlow(v.LeftToRight) + v.setWrapping(True) + v.setResizeMode(v.Adjust) + self.fields_model = FieldsModel(self.plugin) + self.fields_model.initialize() + v.setModel(self.fields_model) + + def commit(self): + self.fields_model.commit() + From b9b5ea7e6254ef5751cec07001b21f0a0f3ffdaf Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 10 Apr 2011 21:46:02 -0600 Subject: [PATCH 06/14] Configuration for metadata download sources completed --- src/calibre/gui2/metadata/config.py | 56 +++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/metadata/config.py b/src/calibre/gui2/metadata/config.py index 83d51c2e28..68c935061d 100644 --- a/src/calibre/gui2/metadata/config.py +++ b/src/calibre/gui2/metadata/config.py @@ -7,8 +7,11 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' +import textwrap + +from PyQt4.Qt import (QWidget, QGridLayout, QGroupBox, QListView, Qt, QSpinBox, + QDoubleSpinBox, QCheckBox, QLineEdit, QComboBox, QLabel) -from PyQt4.Qt import (QWidget, QGridLayout, QGroupBox, QListView, Qt) from calibre.gui2.preferences.metadata_sources import FieldsModel as FM class FieldsModel(FM): # {{{ @@ -53,7 +56,7 @@ class ConfigWidget(QWidget): self.setLayout(l) self.gb = QGroupBox(_('Downloaded metadata fields'), self) - l.addWidget(self.gb, 0, 0) + l.addWidget(self.gb, 0, 0, 1, 2) self.gb.l = QGridLayout() self.gb.setLayout(self.gb.l) self.fields_view = v = QListView(self) @@ -65,6 +68,55 @@ class ConfigWidget(QWidget): self.fields_model.initialize() v.setModel(self.fields_model) + self.memory = [] + self.widgets = [] + for opt in plugin.options: + self.create_widgets(opt) + + def create_widgets(self, opt): + val = self.plugin.prefs[opt.name] + if opt.type == 'number': + c = QSpinBox if isinstance(opt.default, int) else QDoubleSpinBox + widget = c(self) + widget.setValue(val) + elif opt.type == 'string': + widget = QLineEdit(self) + widget.setText(val) + elif opt.type == 'bool': + widget = QCheckBox(opt.label, self) + widget.setChecked(bool(val)) + elif opt.type == 'choices': + widget = QComboBox(self) + for x in opt.choices: + widget.addItem(x) + idx = opt.choices.index(val) + widget.setCurrentIndex(idx) + widget.opt = opt + widget.setToolTip(textwrap.fill(opt.desc)) + self.widgets.append(widget) + r = self.l.rowCount() + if opt.type == 'bool': + self.l.addWidget(widget, r, 0, 1, self.l.columnCount()) + else: + l = QLabel(opt.label) + l.setToolTip(widget.toolTip()) + self.memory.append(l) + l.setBuddy(widget) + self.l.addWidget(l, r, 0, 1, 1) + self.l.addWidget(widget, r, 1, 1, 1) + + def commit(self): self.fields_model.commit() + for w in self.widgets: + if isinstance(w, (QSpinBox, QDoubleSpinBox)): + val = w.value() + elif isinstance(w, QLineEdit): + val = unicode(w.text()) + elif isinstance(w, QCheckBox): + val = w.isChecked() + elif isinstance(w, QComboBox): + val = unicode(w.currentText()) + self.plugin.prefs[w.opt.name] = val + From 4c3256457febe6fa805d7834e8a25b3954ee30bc Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 10 Apr 2011 22:38:31 -0600 Subject: [PATCH 07/14] ... --- src/calibre/manual/faq.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/calibre/manual/faq.rst b/src/calibre/manual/faq.rst index f8b257fd75..44e3665131 100644 --- a/src/calibre/manual/faq.rst +++ b/src/calibre/manual/faq.rst @@ -65,6 +65,22 @@ There are two aspects to this problem: 2. When adding HTML files to |app|, you may need to tell |app| what encoding the files are in. To do this go to :guilabel:`Preferences->Advanced->Plugins->File Type plugins` and customize the HTML2Zip plugin, telling it what encoding your HTML files are in. Now when you add HTML files to |app| they will be correctly processed. HTML files from different sources often have different encodings, so you may have to change this setting repeatedly. A common encoding for many files from the web is ``cp1252`` and I would suggest you try that first. Note that when converting HTML files, leave the input encoding setting mentioned above blank. This is because the HTML2ZIP plugin automatically converts the HTML files to a standard encoding (utf-8). 3. Embedding fonts: If you are generating an LRF file to read on your SONY Reader, you are limited by the fact that the Reader only supports a few non-English characters in the fonts it comes pre-loaded with. You can work around this problem by embedding a unicode-aware font that supports the character set your file uses into the LRF file. You should embed atleast a serif and a sans-serif font. Be aware that embedding fonts significantly slows down page-turn speed on the reader. +What's the deal with Table of Contents in MOBI files? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The first thing to realize is that most ebooks have two tables of contents. One is the traditional Table of Contents, like the TOC you find in paper books. This Table of Contents is part of the main document flow and can be styled however you like. This TOC is called the *content TOC*. + +Then there is the *metadata TOC*. A metadata TOC is a TOC that is not part of the book text and is typically accessed by some special button on a reader. For example, in the calibre viewer, you use the Show Table of Contents button to see this TOC. This TOC cannot be styled by the book creator. How it is represented is up to the viewer program. + +In the MOBI format, the situation is a little confused. This is because the MOBI format, alone amongst mainstream ebook formats, *does not* have decent support for a metadata TOC. A MOBI book simulates the presence of a metadata TOC by putting an *extra* content TOC at the end of the book. When you click Goto Table of Contents on your Kindle, it is to this extra content TOC that the Kindle takes you. + +Now it might well seem to you that the MOBI book has two identical TOCs. Remember that one is semantically a content TOC and the other is a metadata TOC, even though both might have exactly the same entries and look the same. One can be accessed directly from the Kindle's menus, the other cannot. + +When converting to MOBI, calibre detects the *metadata TOC* in the input document and generates an end-of-file TOC in the output MOBI file. You can turn this off by an option in the MOBI Output settings. You cannot control where this generated TOC will go. Remember this TOC is semantically a *metadata TOC*, in any format other than MOBI it *cannot not be part of the text*. The fact that it is part of the text in MOBI is an accident caused by the limitations of MOBI. If you want a TOC at a particular location in your document text, create one by hand. + +If you have a hand edited TOC in the input document, you can use the TOC detection options in calibre to automatically generate the metadata TOC from it. See the conversion section of the User Manual for more details on how to use these options. + +Finally, I encourage you to ditch the content TOC and only have a metadata TOC in your ebooks. Metadata TOCs will give the people reading your ebooks a much superior navigation experience (except on the Kindle, where they are essentially the same as a content TOC). How do I use some of the advanced features of the conversion tools? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From f979f0d357e4fcf86ff49e98462242f6a959df5a Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 11 Apr 2011 14:49:54 +0100 Subject: [PATCH 08/14] Improve performance of ui.library_moved(). Changes remove multiple calls to tags_view.recount(). --- src/calibre/gui2/search_box.py | 5 +++-- src/calibre/gui2/search_restriction_mixin.py | 5 +++-- src/calibre/gui2/ui.py | 5 +++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/calibre/gui2/search_box.py b/src/calibre/gui2/search_box.py index 8936493290..ea7cab95d0 100644 --- a/src/calibre/gui2/search_box.py +++ b/src/calibre/gui2/search_box.py @@ -436,14 +436,15 @@ class SavedSearchBoxMixin(object): # {{{ b = getattr(self, x+'_search_button') b.setStatusTip(b.toolTip()) - def saved_searches_changed(self, set_restriction=None): + def saved_searches_changed(self, set_restriction=None, recount=True): p = sorted(saved_searches().names(), key=sort_key) if set_restriction is None: set_restriction = unicode(self.search_restriction.currentText()) # rebuild the restrictions combobox using current saved searches self.search_restriction.clear() self.search_restriction.addItem('') - self.tags_view.recount() + if recount: + self.tags_view.recount() for s in p: self.search_restriction.addItem(s) if set_restriction: # redo the search restriction if there was one diff --git a/src/calibre/gui2/search_restriction_mixin.py b/src/calibre/gui2/search_restriction_mixin.py index 73c191101c..74e448da6e 100644 --- a/src/calibre/gui2/search_restriction_mixin.py +++ b/src/calibre/gui2/search_restriction_mixin.py @@ -25,8 +25,9 @@ class SearchRestrictionMixin(object): r = self.search_restriction.findText(name) if r < 0: r = 0 - self.search_restriction.setCurrentIndex(r) - self.apply_search_restriction(r) + if r != self.search_restriction.currentIndex(): + self.search_restriction.setCurrentIndex(r) + self.apply_search_restriction(r) def apply_search_restriction(self, i): r = unicode(self.search_restriction.currentText()) diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 4a9d76418c..4d363c283a 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -446,12 +446,13 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ self.search.clear() self.saved_search.clear() self.book_details.reset_info() - self.library_view.model().count_changed() prefs['library_path'] = self.library_path + #self.library_view.model().count_changed() db = self.library_view.model().db + self.iactions['Choose Library'].count_changed(db.count()) self.set_window_title() self.apply_named_search_restriction('') # reset restriction to null - self.saved_searches_changed() # reload the search restrictions combo box + self.saved_searches_changed(recount=False) # reload the search restrictions combo box self.apply_named_search_restriction(db.prefs['gui_restriction']) for action in self.iactions.values(): action.library_changed(db) From 3a2a3f1b4f2c7e0e6225910a6b26c83c1625dd2c Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 11 Apr 2011 16:28:50 +0100 Subject: [PATCH 09/14] Optimization of composite column evaluation: don't compute the user_categories, as they cannot appear in a composite column. --- src/calibre/library/caches.py | 3 ++- src/calibre/library/database2.py | 22 ++++++++++++---------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 74ddb2bc41..a108feb388 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -191,7 +191,8 @@ class CacheRow(list): # {{{ if is_comp: id = list.__getitem__(self, 0) self._must_do = False - mi = self.db.get_metadata(id, index_is_id=True) + mi = self.db.get_metadata(id, index_is_id=True, + get_user_categories=False) for c in self._composites: self[c] = mi.get(self._composites[c]) return list.__getitem__(self, col) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 50ecc4f1e5..0b1182c0bf 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -823,7 +823,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): pass return (path, mi, sequence) - def get_metadata(self, idx, index_is_id=False, get_cover=False): + def get_metadata(self, idx, index_is_id=False, get_cover=False, + get_user_categories=True): ''' Convenience method to return metadata as a :class:`Metadata` object. Note that the list of formats is not verified. @@ -882,16 +883,17 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): user_cats = self.prefs['user_categories'] user_cat_vals = {} - for ucat in user_cats: - res = [] - for name,cat,ign in user_cats[ucat]: - v = mi.get(cat, None) - if isinstance(v, list): - if name in v: + if get_user_categories: + for ucat in user_cats: + res = [] + for name,cat,ign in user_cats[ucat]: + v = mi.get(cat, None) + if isinstance(v, list): + if name in v: + res.append([name,cat]) + elif name == v: res.append([name,cat]) - elif name == v: - res.append([name,cat]) - user_cat_vals[ucat] = res + user_cat_vals[ucat] = res mi.user_categories = user_cat_vals if get_cover: From ea42c67e05c51e4faa7c5e7de4487ed81b1973e0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 11 Apr 2011 09:47:20 -0600 Subject: [PATCH 10/14] ... --- src/calibre/gui2/actions/add.py | 29 +++++++++++++----- src/calibre/gui2/actions/edit_metadata.py | 36 ++++++++++++++++------- 2 files changed, 48 insertions(+), 17 deletions(-) diff --git a/src/calibre/gui2/actions/add.py b/src/calibre/gui2/actions/add.py index cf67cd6cfa..f8dd0693ea 100644 --- a/src/calibre/gui2/actions/add.py +++ b/src/calibre/gui2/actions/add.py @@ -22,6 +22,8 @@ from calibre.constants import preferred_encoding, filesystem_encoding from calibre.gui2.actions import InterfaceAction from calibre.gui2 import config, question_dialog from calibre.ebooks.metadata import MetaInformation +from calibre.utils.config import test_eight_code +from calibre.ebooks.metadata.sources.base import msprefs def get_filters(): return [ @@ -178,13 +180,26 @@ class AddAction(InterfaceAction): except IndexError: self.gui.library_view.model().books_added(self.isbn_add_dialog.value) self.isbn_add_dialog.accept() - orig = config['overwrite_author_title_metadata'] - config['overwrite_author_title_metadata'] = True - try: - self.gui.iactions['Edit Metadata'].do_download_metadata( - self.add_by_isbn_ids) - finally: - config['overwrite_author_title_metadata'] = orig + if test_eight_code: + orig = msprefs['ignore_fields'] + new = list(orig) + for x in ('title', 'authors'): + if x in new: + new.remove(x) + msprefs['ignore_fields'] = new + try: + self.gui.iactions['Edit Metadata'].download_metadata( + ids=self.add_by_isbn_ids) + finally: + msprefs['ignore_fields'] = orig + else: + orig = config['overwrite_author_title_metadata'] + config['overwrite_author_title_metadata'] = True + try: + self.gui.iactions['Edit Metadata'].do_download_metadata( + self.add_by_isbn_ids) + finally: + config['overwrite_author_title_metadata'] = orig return diff --git a/src/calibre/gui2/actions/edit_metadata.py b/src/calibre/gui2/actions/edit_metadata.py index e29e6c344d..fc663d268a 100644 --- a/src/calibre/gui2/actions/edit_metadata.py +++ b/src/calibre/gui2/actions/edit_metadata.py @@ -35,16 +35,23 @@ class EditMetadataAction(InterfaceAction): md.addAction(_('Edit metadata in bulk'), partial(self.edit_metadata, False, bulk=True)) md.addSeparator() - md.addAction(_('Download metadata and covers'), - partial(self.download_metadata, False, covers=True), + if test_eight_code: + dall = self.download_metadata + dident = partial(self.download_metadata, covers=False) + dcovers = partial(self.download_metadata, identify=False) + else: + dall = partial(self.download_metadata_old, False, covers=True) + dident = partial(self.download_metadata_old, False, covers=False) + dcovers = partial(self.download_metadata_old, False, covers=True, + set_metadata=False, set_social_metadata=False) + + md.addAction(_('Download metadata and covers'), dall, Qt.ControlModifier+Qt.Key_D) - md.addAction(_('Download only metadata'), - partial(self.download_metadata, False, covers=False)) - md.addAction(_('Download only covers'), - partial(self.download_metadata, False, covers=True, - set_metadata=False, set_social_metadata=False)) - md.addAction(_('Download only social metadata'), - partial(self.download_metadata, False, covers=False, + md.addAction(_('Download only metadata'), dident) + md.addAction(_('Download only covers'), dcovers) + if not test_eight_code: + md.addAction(_('Download only social metadata'), + partial(self.download_metadata_old, False, covers=False, set_metadata=False, set_social_metadata=True)) self.metadata_menu = md @@ -73,7 +80,16 @@ class EditMetadataAction(InterfaceAction): self.qaction.setEnabled(enabled) self.action_merge.setEnabled(enabled) - def download_metadata(self, checked, covers=True, set_metadata=True, + def download_metadata(self, identify=True, covers=True, ids=None): + if ids is None: + rows = self.gui.library_view.selectionModel().selectedRows() + if not rows or len(rows) == 0: + return error_dialog(self.gui, _('Cannot download metadata'), + _('No books selected'), show=True) + db = self.gui.library_view.model().db + ids = [db.id(row.row()) for row in rows] + + def download_metadata_old(self, checked, covers=True, set_metadata=True, set_social_metadata=None): rows = self.gui.library_view.selectionModel().selectedRows() if not rows or len(rows) == 0: From 72c9625094afc23c03db8d6767e2fde2e0303dc4 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 11 Apr 2011 17:17:44 +0100 Subject: [PATCH 11/14] Add the new configure metadata download tool button to the alternate metadata single dialog --- src/calibre/gui2/metadata/single.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/calibre/gui2/metadata/single.py b/src/calibre/gui2/metadata/single.py index 4ce76d8cc8..2e5b43ceba 100644 --- a/src/calibre/gui2/metadata/single.py +++ b/src/calibre/gui2/metadata/single.py @@ -612,6 +612,10 @@ class MetadataSingleDialogAlt1(MetadataSingleDialogBase): # {{{ self.button_box.addButton(self.fetch_metadata_button, QDialogButtonBox.ActionRole) + self.config_metadata_button.setToolButtonStyle(Qt.ToolButtonTextOnly) + self.config_metadata_button.setText(_('Configure metadata downloading')) + self.button_box.addButton(self.config_metadata_button, + QDialogButtonBox.ActionRole) sto(self.button_box, self.title) def create_row(row, widget, tab_to, button=None, icon=None, span=1): From aac23508dff35df7ccec7c319d4586f5b7e58f23 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 11 Apr 2011 15:43:34 -0600 Subject: [PATCH 12/14] Fix regression that caused clicking auto send to also change the email address in Preferences->Email --- src/calibre/gui2/preferences/emailp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/preferences/emailp.py b/src/calibre/gui2/preferences/emailp.py index ded6891387..f5c7b5a3a7 100644 --- a/src/calibre/gui2/preferences/emailp.py +++ b/src/calibre/gui2/preferences/emailp.py @@ -88,7 +88,7 @@ class EmailAccounts(QAbstractTableModel): # {{{ self.subjects[account] = unicode(value.toString()) elif col == 1: self.accounts[account][0] = unicode(value.toString()).upper() - else: + elif col == 0: na = unicode(value.toString()) from email.utils import parseaddr addr = parseaddr(na)[-1] @@ -100,7 +100,7 @@ class EmailAccounts(QAbstractTableModel): # {{{ self.accounts[na][0] = 'AZW, MOBI, TPZ, PRC, AZW1' self.dataChanged.emit( - self.index(index.row(), 0), self.index(index.row(), 2)) + self.index(index.row(), 0), self.index(index.row(), 3)) return True def make_default(self, index): From 64dd32eaf5629f997b4f683c054061e66dff1b28 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 11 Apr 2011 15:43:58 -0600 Subject: [PATCH 13/14] ... --- src/calibre/gui2/preferences/emailp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/preferences/emailp.py b/src/calibre/gui2/preferences/emailp.py index f5c7b5a3a7..1644dc6b73 100644 --- a/src/calibre/gui2/preferences/emailp.py +++ b/src/calibre/gui2/preferences/emailp.py @@ -84,7 +84,7 @@ class EmailAccounts(QAbstractTableModel): # {{{ account = self.account_order[row] if col == 3: self.accounts[account][1] ^= True - if col == 2: + elif col == 2: self.subjects[account] = unicode(value.toString()) elif col == 1: self.accounts[account][0] = unicode(value.toString()).upper() From 71967fd4ba173c5ad9c82f60a166b881fe29809d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 11 Apr 2011 15:53:34 -0600 Subject: [PATCH 14/14] New framework for running I/O bound jobs in threads inthe calibre main process. Migrate email sending to the new framework. --- src/calibre/gui2/dialogs/job_view.ui | 40 ++-- src/calibre/gui2/email.py | 226 ++++++------------- src/calibre/gui2/jobs.py | 85 +++++-- src/calibre/gui2/metadata/bulk_download2.py | 11 + src/calibre/gui2/threaded_jobs.py | 238 ++++++++++++++++++++ src/calibre/gui2/ui.py | 3 +- src/calibre/utils/ipc/job.py | 10 +- 7 files changed, 410 insertions(+), 203 deletions(-) create mode 100644 src/calibre/gui2/metadata/bulk_download2.py create mode 100644 src/calibre/gui2/threaded_jobs.py diff --git a/src/calibre/gui2/dialogs/job_view.ui b/src/calibre/gui2/dialogs/job_view.ui index 8b54c23573..1e854c0f29 100644 --- a/src/calibre/gui2/dialogs/job_view.ui +++ b/src/calibre/gui2/dialogs/job_view.ui @@ -1,7 +1,8 @@ - + + Dialog - - + + 0 0 @@ -9,38 +10,41 @@ 462 - + Details of job - - + + :/images/view.png:/images/view.png - - - - + + + + false - + QPlainTextEdit::NoWrap - + true - - - + + + QDialogButtonBox::Ok + + + - + @@ -49,11 +53,11 @@ Dialog accept() - + 617 442 - + 206 -5 diff --git a/src/calibre/gui2/email.py b/src/calibre/gui2/email.py index c6d58fa340..c8adeb7d31 100644 --- a/src/calibre/gui2/email.py +++ b/src/calibre/gui2/email.py @@ -6,9 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, socket, time, cStringIO -from threading import Thread -from Queue import Queue +import os, socket, time from binascii import unhexlify from functools import partial from itertools import repeat @@ -16,67 +14,20 @@ from itertools import repeat from calibre.utils.smtp import compose_mail, sendmail, extract_email_address, \ config as email_config from calibre.utils.filenames import ascii_filename -from calibre.utils.ipc.job import BaseJob -from calibre.ptempfile import PersistentTemporaryFile from calibre.customize.ui import available_input_formats, available_output_formats from calibre.ebooks.metadata import authors_to_string from calibre.constants import preferred_encoding from calibre.gui2 import config, Dispatcher, warning_dialog from calibre.library.save_to_disk import get_components from calibre.utils.config import tweaks +from calibre.gui2.threaded_jobs import ThreadedJob -class EmailJob(BaseJob): # {{{ - - def __init__(self, callback, description, attachment, aname, to, subject, text, job_manager): - BaseJob.__init__(self, description) - self.exception = None - self.job_manager = job_manager - self.email_args = (attachment, aname, to, subject, text) - self.email_sent_callback = callback - self.log_path = None - self._log_file = cStringIO.StringIO() - self._log_file.write(self.description.encode('utf-8') + '\n') - - @property - def log_file(self): - if self.log_path is not None: - return open(self.log_path, 'rb') - return cStringIO.StringIO(self._log_file.getvalue()) - - def start_work(self): - self.start_time = time.time() - self.job_manager.changed_queue.put(self) - - def job_done(self): - self.duration = time.time() - self.start_time - self.percent = 1 - # Dump log onto disk - lf = PersistentTemporaryFile('email_log') - lf.write(self._log_file.getvalue()) - lf.close() - self.log_path = lf.name - self._log_file.close() - self._log_file = None - - self.job_manager.changed_queue.put(self) - - def log_write(self, what): - self._log_file.write(what) - -# }}} - -class Emailer(Thread): # {{{ +class Sendmail(object): MAX_RETRIES = 1 - def __init__(self, job_manager): - Thread.__init__(self) - self.daemon = True - self.jobs = Queue() - self.job_manager = job_manager - self._run = True + def __init__(self): self.calculate_rate_limit() - self.last_send_time = time.time() - self.rate_limit def calculate_rate_limit(self): @@ -87,70 +38,28 @@ class Emailer(Thread): # {{{ 'gmail.com' in rh or 'live.com' in rh): self.rate_limit = tweaks['public_smtp_relay_delay'] - def stop(self): - self._run = False - self.jobs.put(None) + def __call__(self, attachment, aname, to, subject, text, log=None, + abort=None, notifications=None): - def run(self): - while self._run: + try_count = 0 + while try_count <= self.MAX_RETRIES: + if try_count > 0: + log('\nRetrying in %d seconds...\n' % + self.rate_limit) try: - job = self.jobs.get() + self.sendmail(attachment, aname, to, subject, text, log) + try_count = self.MAX_RETRIES + log('Email successfully sent') except: - break - if job is None or not self._run: - break - try_count = 0 - failed, exc = False, None - job.start_work() - if job.kill_on_start: - job.log_write('Aborted\n') - job.failed = failed - job.killed = True - job.job_done() - continue + if abort.is_set(): + return + if try_count == self.MAX_RETRIES: + raise + log.exception('\nSending failed...\n') - while try_count <= self.MAX_RETRIES: - failed = False - if try_count > 0: - job.log_write('\nRetrying in %d seconds...\n' % - self.rate_limit) - try: - self.sendmail(job) - break - except Exception as e: - if not self._run: - return - import traceback - failed = True - exc = e - job.log_write('\nSending failed...\n') - job.log_write(traceback.format_exc()) + try_count += 1 - try_count += 1 - - if not self._run: - break - - job.failed = failed - job.exception = exc - job.job_done() - try: - job.email_sent_callback(job) - except: - import traceback - traceback.print_exc() - - def send_mails(self, jobnames, callback, attachments, to_s, subjects, - texts, attachment_names): - for name, attachment, to, subject, text, aname in zip(jobnames, - attachments, to_s, subjects, texts, attachment_names): - description = _('Email %s to %s') % (name, to) - job = EmailJob(callback, description, attachment, aname, to, - subject, text, self.job_manager) - self.job_manager.add_job(job) - self.jobs.put(job) - - def sendmail(self, job): + def sendmail(self, attachment, aname, to, subject, text, log): while time.time() - self.last_send_time <= self.rate_limit: time.sleep(1) try: @@ -158,7 +67,6 @@ class Emailer(Thread): # {{{ from_ = opts.from_ if not from_: from_ = 'calibre ' - attachment, aname, to, subject, text = job.email_args msg = compose_mail(from_, to, text, subject, open(attachment, 'rb'), aname) efrom, eto = map(extract_email_address, (from_, to)) @@ -169,48 +77,56 @@ class Emailer(Thread): # {{{ username=opts.relay_username, password=unhexlify(opts.relay_password), port=opts.relay_port, encryption=opts.encryption, - debug_output=partial(print, file=job._log_file)) + debug_output=log.debug) finally: self.last_send_time = time.time() - def email_news(self, mi, remove, get_fmts, done): - opts = email_config().parse() - accounts = [(account, [x.strip().lower() for x in x[0].split(',')]) - for account, x in opts.accounts.items() if x[1]] - sent_mails = [] - for i, x in enumerate(accounts): - account, fmts = x - files = get_fmts(fmts) - files = [f for f in files if f is not None] - if not files: - continue - attachment = files[0] - to_s = [account] - subjects = [_('News:')+' '+mi.title] - texts = [ - _('Attached is the %s periodical downloaded by calibre.') - % (mi.title,) - ] - attachment_names = [ascii_filename(mi.title)+os.path.splitext(attachment)[1]] - attachments = [attachment] - jobnames = [mi.title] - do_remove = [] - if i == len(accounts) - 1: - do_remove = remove - self.send_mails(jobnames, - Dispatcher(partial(done, remove=do_remove)), - attachments, to_s, subjects, texts, attachment_names) - sent_mails.append(to_s[0]) - return sent_mails +gui_sendmail = Sendmail() -# }}} +def send_mails(jobnames, callback, attachments, to_s, subjects, + texts, attachment_names, job_manager): + for name, attachment, to, subject, text, aname in zip(jobnames, + attachments, to_s, subjects, texts, attachment_names): + description = _('Email %s to %s') % (name, to) + job = ThreadedJob('email', description, gui_sendmail, (attachment, aname, to, + subject, text), {}, callback, killable=False) + job_manager.run_threaded_job(job) + + +def email_news(mi, remove, get_fmts, done, job_manager): + opts = email_config().parse() + accounts = [(account, [x.strip().lower() for x in x[0].split(',')]) + for account, x in opts.accounts.items() if x[1]] + sent_mails = [] + for i, x in enumerate(accounts): + account, fmts = x + files = get_fmts(fmts) + files = [f for f in files if f is not None] + if not files: + continue + attachment = files[0] + to_s = [account] + subjects = [_('News:')+' '+mi.title] + texts = [ + _('Attached is the %s periodical downloaded by calibre.') + % (mi.title,) + ] + attachment_names = [ascii_filename(mi.title)+os.path.splitext(attachment)[1]] + attachments = [attachment] + jobnames = [mi.title] + do_remove = [] + if i == len(accounts) - 1: + do_remove = remove + send_mails(jobnames, + Dispatcher(partial(done, remove=do_remove)), + attachments, to_s, subjects, texts, attachment_names, + job_manager) + sent_mails.append(to_s[0]) + return sent_mails class EmailMixin(object): # {{{ - def __init__(self): - self.emailer = Emailer(self.job_manager) - def send_by_mail(self, to, fmts, delete_from_library, subject='', send_ids=None, do_auto_convert=True, specific_format=None): ids = [self.library_view.model().id(r) for r in self.library_view.selectionModel().selectedRows()] if send_ids is None else send_ids @@ -246,8 +162,7 @@ class EmailMixin(object): # {{{ components = get_components(subject, mi, id) if not components: components = [mi.title] - subject = os.path.join(*components) - subjects.append(subject) + subjects.append(os.path.join(*components)) a = authors_to_string(mi.authors if mi.authors else \ [_('Unknown')]) texts.append(_('Attached, you will find the e-book') + \ @@ -262,11 +177,10 @@ class EmailMixin(object): # {{{ to_s = list(repeat(to, len(attachments))) if attachments: - if not self.emailer.is_alive(): - self.emailer.start() - self.emailer.send_mails(jobnames, + send_mails(jobnames, Dispatcher(partial(self.email_sent, remove=remove)), - attachments, to_s, subjects, texts, attachment_names) + attachments, to_s, subjects, texts, attachment_names, + self.job_manager) self.status_bar.show_message(_('Sending email to')+' '+to, 3000) auto = [] @@ -334,10 +248,8 @@ class EmailMixin(object): # {{{ files, auto = self.library_view.model().\ get_preferred_formats_from_ids([id_], fmts) return files - if not self.emailer.is_alive(): - self.emailer.start() - sent_mails = self.emailer.email_news(mi, remove, - get_fmts, self.email_sent) + sent_mails = email_news(mi, remove, + get_fmts, self.email_sent, self.job_manager) if sent_mails: self.status_bar.show_message(_('Sent news to')+' '+\ ', '.join(sent_mails), 3000) diff --git a/src/calibre/gui2/jobs.py b/src/calibre/gui2/jobs.py index dbde030e81..34eef4406a 100644 --- a/src/calibre/gui2/jobs.py +++ b/src/calibre/gui2/jobs.py @@ -8,14 +8,13 @@ Job management. ''' import re - from Queue import Empty, Queue -from PyQt4.Qt import QAbstractTableModel, QVariant, QModelIndex, Qt, \ - QTimer, pyqtSignal, QIcon, QDialog, QAbstractItemDelegate, QApplication, \ - QSize, QStyleOptionProgressBarV2, QString, QStyle, QToolTip, QFrame, \ - QHBoxLayout, QVBoxLayout, QSizePolicy, QLabel, QCoreApplication, QAction, \ - QByteArray +from PyQt4.Qt import (QAbstractTableModel, QVariant, QModelIndex, Qt, + QTimer, pyqtSignal, QIcon, QDialog, QAbstractItemDelegate, QApplication, + QSize, QStyleOptionProgressBarV2, QString, QStyle, QToolTip, QFrame, + QHBoxLayout, QVBoxLayout, QSizePolicy, QLabel, QCoreApplication, QAction, + QByteArray) from calibre.utils.ipc.server import Server from calibre.utils.ipc.job import ParallelJob @@ -25,8 +24,9 @@ from calibre.gui2.dialogs.jobs_ui import Ui_JobsDialog from calibre import __appname__ from calibre.gui2.dialogs.job_view_ui import Ui_Dialog from calibre.gui2.progress_indicator import ProgressIndicator +from calibre.gui2.threaded_jobs import ThreadedJobServer, ThreadedJob -class JobManager(QAbstractTableModel): +class JobManager(QAbstractTableModel): # {{{ job_added = pyqtSignal(int) job_done = pyqtSignal(int) @@ -42,6 +42,7 @@ class JobManager(QAbstractTableModel): self.add_job = Dispatcher(self._add_job) self.server = Server(limit=int(config['worker_limit']/2.0), enforce_cpu_limit=config['enforce_cpu_limit']) + self.threaded_server = ThreadedJobServer() self.changed_queue = Queue() self.timer = QTimer(self) @@ -146,12 +147,21 @@ class JobManager(QAbstractTableModel): jobs.add(self.server.changed_jobs_queue.get_nowait()) except Empty: break + + # Update device jobs while True: try: jobs.add(self.changed_queue.get_nowait()) except Empty: break + # Update threaded jobs + while True: + try: + jobs.add(self.threaded_server.changed_jobs.get_nowait()) + except Empty: + break + if jobs: needs_reset = False for job in jobs: @@ -207,11 +217,22 @@ class JobManager(QAbstractTableModel): self.server.add_job(job) return job + def run_threaded_job(self, job): + self.add_job(job) + self.threaded_server.add_job(job) + def launch_gui_app(self, name, args=[], kwargs={}, description=''): job = ParallelJob(name, description, lambda x: x, args=args, kwargs=kwargs) self.server.run_job(job, gui=True, redirect_output=False) + def _kill_job(self, job): + if isinstance(job, ParallelJob): + self.server.kill_job(job) + elif isinstance(job, ThreadedJob): + self.threaded_server.kill_job(job) + else: + job.kill_on_start = True def kill_job(self, row, view): job = self.jobs[row] @@ -221,29 +242,29 @@ class JobManager(QAbstractTableModel): if job.duration is not None: return error_dialog(view, _('Cannot kill job'), _('Job has already run')).exec_() - if isinstance(job, ParallelJob): - self.server.kill_job(job) - else: - job.kill_on_start = True + if not getattr(job, 'killable', True): + return error_dialog(view, _('Cannot kill job'), + _('This job cannot be stopped'), show=True) + self._kill_job(job) def kill_all_jobs(self): for job in self.jobs: - if isinstance(job, DeviceJob) or job.duration is not None: + if (isinstance(job, DeviceJob) or job.duration is not None or + not getattr(job, 'killable', True)): continue - if isinstance(job, ParallelJob): - self.server.kill_job(job) - else: - job.kill_on_start = True + self._kill_job(job) def terminate_all_jobs(self): self.server.killall() for job in self.jobs: - if isinstance(job, DeviceJob) or job.duration is not None: + if (isinstance(job, DeviceJob) or job.duration is not None or + not getattr(job, 'killable', True)): continue if not isinstance(job, ParallelJob): - job.kill_on_start = True - + self._kill_job(job) +# }}} +# Jobs UI {{{ class ProgressBarDelegate(QAbstractItemDelegate): def sizeHint(self, option, index): @@ -269,6 +290,11 @@ class DetailView(QDialog, Ui_Dialog): self.setupUi(self) self.setWindowTitle(job.description) self.job = job + self.html_view = hasattr(job, 'html_details') + if self.html_view: + self.log.setVisible(False) + else: + self.tb.setVisible(False) self.next_pos = 0 self.update() self.timer = QTimer(self) @@ -277,12 +303,19 @@ class DetailView(QDialog, Ui_Dialog): def update(self): - f = self.job.log_file - f.seek(self.next_pos) - more = f.read() - self.next_pos = f.tell() - if more: - self.log.appendPlainText(more.decode('utf-8', 'replace')) + if self.html_view: + html = self.job.html_details + if len(html) > self.next_pos: + self.next_pos = len(html) + self.tb.setHtml( + '
%s
'%html) + else: + f = self.job.log_file + f.seek(self.next_pos) + more = f.read() + self.next_pos = f.tell() + if more: + self.log.appendPlainText(more.decode('utf-8', 'replace')) class JobsButton(QFrame): @@ -441,3 +474,5 @@ class JobsDialog(QDialog, Ui_JobsDialog): def hide(self, *args): self.save_state() return QDialog.hide(self, *args) +# }}} + diff --git a/src/calibre/gui2/metadata/bulk_download2.py b/src/calibre/gui2/metadata/bulk_download2.py new file mode 100644 index 0000000000..cc6da1e995 --- /dev/null +++ b/src/calibre/gui2/metadata/bulk_download2.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + + + diff --git a/src/calibre/gui2/threaded_jobs.py b/src/calibre/gui2/threaded_jobs.py new file mode 100644 index 0000000000..f29baf4134 --- /dev/null +++ b/src/calibre/gui2/threaded_jobs.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import os, time, tempfile, json +from threading import Thread, RLock, Event +from Queue import Queue + +from calibre.utils.ipc.job import BaseJob +from calibre.utils.logging import GUILog +from calibre.ptempfile import base_dir + +class ThreadedJob(BaseJob): + + def __init__(self, + type_, description, + + func, args, kwargs, + + callback, + + max_concurrent_count=1, + killable=True, + log=None): + ''' + A job that is run in its own thread in the calibre main process + + :param type_: The type of this job (a string). The type is used in + conjunction with max_concurrent_count to prevent too many jobs of the + same type from running + + :param description: A user viewable job description + + :func: The function that actually does the work. This function *must* + accept at least three keyword arguments: abort, log and notifications. abort is + An Event object. func should periodically check abort.is_set(0 and if + it is True, it should stop processing as soon as possible. notifications + is a Queue. func should put progress notifications into it in the form + of a tuple (frac, msg). frac is a number between 0 and 1 indicating + progress and msg is a string describing the progress. log is a Log + object which func should use for all debugging output. func should + raise an Exception to indicate failure. This exception is stored in + job.exception and can thus be used to pass arbitrary information to + callback. + + :param args,kwargs: These are passed to func when it is called + + :param callback: A callable that is called on completion of this job. + Note that it is not called if the user kills the job. Check job.failed + to see if the job succeeded or not. And use job.log to get the job log. + + :param killable: If False the GUI wont let the user kill this job + + :param log: Must be a subclass of GUILog or None. If None a default + GUILog is created. + ''' + BaseJob.__init__(self, description) + + self.type = type_ + self.max_concurrent_count = max_concurrent_count + self.killable = killable + self.callback = callback + self.abort = Event() + self.exception = None + + kwargs['notifications'] = self.notifications + kwargs['abort'] = self.abort + self.log = GUILog() if log is None else log + kwargs['log'] = self.log + + self.func, self.args, self.kwargs = func, args, kwargs + self.consolidated_log = None + + def start_work(self): + self.start_time = time.time() + self.log('Starting job:', self.description) + try: + self.result = self.func(*self.args, **self.kwargs) + except Exception as e: + self.exception = e + self.failed = True + self.log.exception('Job: "%s" failed with error:'%self.description) + self.log.debug('Called with args:', self.args, self.kwargs) + + self.duration = time.time() - self.start_time + try: + self.callback(self) + except: + pass + self._cleanup() + + def _cleanup(self): + + try: + self.consolidate_log() + except: + self.log.exception('Log consolidation failed') + + # No need to keep references to these around anymore + self.func = self.args = self.kwargs = self.notifications = None + + def kill(self): + if self.start_time is None: + self.start_time = time.time() + self.duration = 0.0001 + else: + self.duration = time.time() - self.start_time() + self.abort.set() + + self.log('Aborted job:', self.description) + self.killed = True + self.failed = True + self._cleanup() + + def consolidate_log(self): + logs = [self.log.html, self.log.plain_text] + bdir = base_dir() + log_dir = os.path.join(bdir, 'threaded_job_logs') + if not os.path.exists(log_dir): + os.makedirs(log_dir) + fd, path = tempfile.mkstemp(suffix='.json', prefix='log-', dir=log_dir) + with os.fdopen(fd, 'wb') as f: + f.write(json.dumps(logs, ensure_ascii=False, + indent=2).encode('utf-8')) + self.consolidated_log = path + self.log = None + + def read_consolidated_log(self): + with open(self.consolidated_log, 'rb') as f: + return json.loads(f.read().decode('utf-8')) + + @property + def details(self): + if self.consolidated_log is None: + return self.log.plain_text + return self.read_consolidated_log()[1] + + @property + def html_details(self): + if self.consolidated_log is None: + return self.log.html + return self.read_consolidated_log()[0] + +class ThreadedJobWorker(Thread): + + def __init__(self, job): + Thread.__init__(self) + self.daemon = True + self.job = job + + def run(self): + try: + self.job.start_work() + except: + import traceback + from calibre import prints + prints('Job had unhandled exception:', self.job.description) + traceback.print_exc() + +class ThreadedJobServer(Thread): + + def __init__(self): + Thread.__init__(self) + self.daemon = True + self.lock = RLock() + + self.queued_jobs = [] + self.running_jobs = set() + self.changed_jobs = Queue() + self.keep_going = True + + def close(self): + self.keep_going = False + + def add_job(self, job): + with self.lock: + self.queued_jobs.append(job) + + if not self.is_alive(): + self.start() + + def run(self): + while self.keep_going: + self.run_once() + time.sleep(0.1) + + def run_once(self): + with self.lock: + remove = set() + for worker in self.running_jobs: + if worker.is_alive(): + # Get progress notifications + if worker.job.consume_notifications(): + self.changed_jobs.put(worker.job) + else: + remove.add(worker) + self.changed_jobs.put(worker.job) + + for worker in remove: + self.running_jobs.remove(worker) + + jobs = self.get_startable_jobs() + for job in jobs: + w = ThreadedJobWorker(job) + w.start() + self.running_jobs.add(w) + self.changed_jobs.put(job) + self.queued_jobs.remove(job) + + def kill_job(self, job): + with self.lock: + if job in self.queued_jobs: + self.queued_jobs.remove(job) + elif job in self.running_jobs: + self.running_jobs.remove(job) + job.kill() + self.changed_jobs.put(job) + + def running_jobs_of_type(self, type_): + return len([w for w in self.running_jobs if w.job.type == type_]) + + def get_startable_jobs(self): + queued_types = [] + ans = [] + for job in self.queued_jobs: + num = self.running_jobs_of_type(job.type) + num += queued_types.count(job.type) + if num < job.max_concurrent_count: + queued_types.append(job.type) + ans.append(job) + return ans + + diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 4d363c283a..e7853b9491 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -608,6 +608,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ self.update_checker.terminate() self.listener.close() self.job_manager.server.close() + self.job_manager.threaded_server.close() while self.spare_servers: self.spare_servers.pop().close() self.device_manager.keep_going = False @@ -616,8 +617,6 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ mb.stop() self.hide_windows() - if self.emailer.is_alive(): - self.emailer.stop() try: try: if self.content_server is not None: diff --git a/src/calibre/utils/ipc/job.py b/src/calibre/utils/ipc/job.py index 91db333791..f4b54aee95 100644 --- a/src/calibre/utils/ipc/job.py +++ b/src/calibre/utils/ipc/job.py @@ -75,12 +75,20 @@ class BaseJob(object): self._run_state = self.RUNNING self._status_text = _('Working...') - while consume_notifications: + if consume_notifications: + return self.consume_notifications() + return False + + def consume_notifications(self): + got_notification = False + while self.notifications is not None: try: self.percent, self._message = self.notifications.get_nowait() self.percent *= 100. + got_notification = True except Empty: break + return got_notification @property def status_text(self):