diff --git a/recipes/hbr_blogs.recipe b/recipes/hbr_blogs.recipe
index bd72a95ebf..acee567d8d 100644
--- a/recipes/hbr_blogs.recipe
+++ b/recipes/hbr_blogs.recipe
@@ -6,7 +6,7 @@ class HBR(BasicNewsRecipe):
title = 'Harvard Business Review Blogs'
description = 'To subscribe go to http://hbr.harvardbusiness.org'
needs_subscription = True
- __author__ = 'Kovid Goyal and Sujata Raman, enhanced by BrianG'
+ __author__ = 'Kovid Goyal, enhanced by BrianG'
language = 'en'
no_stylesheets = True
diff --git a/src/calibre/ebooks/mobi/reader.py b/src/calibre/ebooks/mobi/reader.py
index 934e8476d2..46505de4bd 100644
--- a/src/calibre/ebooks/mobi/reader.py
+++ b/src/calibre/ebooks/mobi/reader.py
@@ -348,7 +348,6 @@ class MobiReader(object):
self.processed_html = self.remove_random_bytes(self.processed_html)
root = soupparser.fromstring(self.processed_html)
-
if root.tag != 'html':
self.log.warn('File does not have opening tag')
nroot = html.fromstring('')
diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py
index 8499e304c3..f69788e849 100644
--- a/src/calibre/gui2/__init__.py
+++ b/src/calibre/gui2/__init__.py
@@ -271,6 +271,9 @@ class Dispatcher(QObject):
Convenience class to use Qt signals with arbitrary python callables.
By default, ensures that a function call always happens in the
thread this Dispatcher was created in.
+
+ Note that if you create the Dispatcher in a thread without an event loop of
+ its own, the function call will happen in the GUI thread (I think).
'''
dispatch_signal = pyqtSignal(object, object)
@@ -292,11 +295,20 @@ class FunctionDispatcher(QObject):
'''
Convenience class to use Qt signals with arbitrary python functions.
By default, ensures that a function call always happens in the
- thread this Dispatcher was created in.
+ thread this FunctionDispatcher was created in.
+
+ Note that you must create FunctionDispatcher objects in the GUI thread.
'''
dispatch_signal = pyqtSignal(object, object, object)
def __init__(self, func, queued=True, parent=None):
+ global gui_thread
+ if gui_thread is None:
+ gui_thread = QThread.currentThread()
+ if not is_gui_thread():
+ raise ValueError(
+ 'You can only create a FunctionDispatcher in the GUI thread')
+
QObject.__init__(self, parent)
self.func = func
typ = Qt.QueuedConnection
@@ -307,6 +319,8 @@ class FunctionDispatcher(QObject):
self.lock = threading.Lock()
def __call__(self, *args, **kwargs):
+ if is_gui_thread():
+ return self.func(*args, **kwargs)
with self.lock:
self.dispatch_signal.emit(self.q, args, kwargs)
res = self.q.get()
diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py
index 97cb725f7e..c9c79095f3 100644
--- a/src/calibre/gui2/device.py
+++ b/src/calibre/gui2/device.py
@@ -17,7 +17,7 @@ from calibre.gui2.dialogs.choose_format_device import ChooseFormatDeviceDialog
from calibre.utils.ipc.job import BaseJob
from calibre.devices.scanner import DeviceScanner
from calibre.gui2 import (config, error_dialog, Dispatcher, dynamic,
- warning_dialog, info_dialog, choose_dir)
+ warning_dialog, info_dialog, choose_dir, FunctionDispatcher)
from calibre.ebooks.metadata import authors_to_string
from calibre import preferred_encoding, prints, force_unicode, as_unicode
from calibre.utils.filenames import ascii_filename
@@ -35,8 +35,13 @@ class DeviceJob(BaseJob): # {{{
def __init__(self, func, done, job_manager, args=[], kwargs={},
description=''):
- BaseJob.__init__(self, description, done=done)
+ BaseJob.__init__(self, description)
self.func = func
+ self.callback_on_done = done
+ if not isinstance(self.callback_on_done, (Dispatcher,
+ FunctionDispatcher)):
+ self.callback_on_done = FunctionDispatcher(self.callback_on_done)
+
self.args, self.kwargs = args, kwargs
self.exception = None
self.job_manager = job_manager
@@ -50,6 +55,10 @@ class DeviceJob(BaseJob): # {{{
def job_done(self):
self.duration = time.time() - self.start_time
self.percent = 1
+ try:
+ self.callback_on_done(self)
+ except:
+ pass
self.job_manager.changed_queue.put(self)
def report_progress(self, percent, msg=''):
@@ -254,7 +263,8 @@ class DeviceManager(Thread): # {{{
job = self.next()
if job is not None:
self.current_job = job
- self.device.set_progress_reporter(job.report_progress)
+ if self.device is not None:
+ self.device.set_progress_reporter(job.report_progress)
self.current_job.run()
self.current_job = None
else:
@@ -587,7 +597,7 @@ class DeviceMenu(QMenu): # {{{
# }}}
-class DeviceSignals(QObject):
+class DeviceSignals(QObject): # {{{
#: This signal is emitted once, after metadata is downloaded from the
#: connected device.
#: The sequence: gui.device_manager.is_device_connected will become True,
@@ -604,6 +614,7 @@ class DeviceSignals(QObject):
device_connection_changed = pyqtSignal(object)
device_signals = DeviceSignals()
+# }}}
class DeviceMixin(object): # {{{
@@ -611,7 +622,7 @@ class DeviceMixin(object): # {{{
self.device_error_dialog = error_dialog(self, _('Error'),
_('Error communicating with device'), ' ')
self.device_error_dialog.setModal(Qt.NonModal)
- self.device_manager = DeviceManager(Dispatcher(self.device_detected),
+ self.device_manager = DeviceManager(FunctionDispatcher(self.device_detected),
self.job_manager, Dispatcher(self.status_bar.show_message),
Dispatcher(self.show_open_feedback))
self.device_manager.start()
@@ -736,7 +747,7 @@ class DeviceMixin(object): # {{{
self.set_device_menu_items_state(connected)
if connected:
self.device_manager.get_device_information(\
- Dispatcher(self.info_read))
+ FunctionDispatcher(self.info_read))
self.set_default_thumbnail(\
self.device_manager.device.THUMBNAIL_HEIGHT)
self.status_bar.show_message(_('Device: ')+\
@@ -767,7 +778,7 @@ class DeviceMixin(object): # {{{
self.device_manager.device.icon)
self.bars_manager.update_bars()
self.status_bar.device_connected(info[0])
- self.device_manager.books(Dispatcher(self.metadata_downloaded))
+ self.device_manager.books(FunctionDispatcher(self.metadata_downloaded))
def metadata_downloaded(self, job):
'''
@@ -810,7 +821,7 @@ class DeviceMixin(object): # {{{
def remove_paths(self, paths):
return self.device_manager.delete_books(
- Dispatcher(self.books_deleted), paths)
+ FunctionDispatcher(self.books_deleted), paths)
def books_deleted(self, job):
'''
@@ -1187,7 +1198,7 @@ class DeviceMixin(object): # {{{
Upload metadata to device.
'''
plugboards = self.library_view.model().db.prefs.get('plugboards', {})
- self.device_manager.sync_booklists(Dispatcher(self.metadata_synced),
+ self.device_manager.sync_booklists(FunctionDispatcher(self.metadata_synced),
self.booklists(), plugboards)
def metadata_synced(self, job):
@@ -1222,7 +1233,7 @@ class DeviceMixin(object): # {{{
titles = [i.title for i in metadata]
plugboards = self.library_view.model().db.prefs.get('plugboards', {})
job = self.device_manager.upload_books(
- Dispatcher(self.books_uploaded),
+ FunctionDispatcher(self.books_uploaded),
files, names, on_card=on_card,
metadata=metadata, titles=titles, plugboards=plugboards
)
@@ -1475,7 +1486,7 @@ class DeviceMixin(object): # {{{
self.cover_to_thumbnail(open(book.cover, 'rb').read())
plugboards = self.library_view.model().db.prefs.get('plugboards', {})
self.device_manager.sync_booklists(
- Dispatcher(self.metadata_synced), booklists,
+ FunctionDispatcher(self.metadata_synced), booklists,
plugboards)
return update_metadata
# }}}
diff --git a/src/calibre/gui2/email.py b/src/calibre/gui2/email.py
index 4b4c920a7e..b82f421e1e 100644
--- a/src/calibre/gui2/email.py
+++ b/src/calibre/gui2/email.py
@@ -9,6 +9,7 @@ __docformat__ = 'restructuredtext en'
import os, socket, time
from binascii import unhexlify
from functools import partial
+from threading import Thread
from itertools import repeat
from calibre.utils.smtp import (compose_mail, sendmail, extract_email_address,
@@ -22,9 +23,30 @@ from calibre.library.save_to_disk import get_components
from calibre.utils.config import tweaks
from calibre.gui2.threaded_jobs import ThreadedJob
+class Worker(Thread):
+
+ def __init__(self, func, args):
+ Thread.__init__(self)
+ self.daemon = True
+ self.exception = self.tb = None
+ self.func, self.args = func, args
+
+ def run(self):
+ #time.sleep(1000)
+ try:
+ self.func(*self.args)
+ except Exception as e:
+ import traceback
+ self.exception = e
+ self.tb = traceback.format_exc()
+ finally:
+ self.func = self.args = None
+
+
class Sendmail(object):
MAX_RETRIES = 1
+ TIMEOUT = 15 * 60 # seconds
def __init__(self):
self.calculate_rate_limit()
@@ -42,22 +64,32 @@ class Sendmail(object):
abort=None, notifications=None):
try_count = 0
- while try_count <= self.MAX_RETRIES:
+ while True:
if try_count > 0:
log('\nRetrying in %d seconds...\n' %
self.rate_limit)
- try:
- self.sendmail(attachment, aname, to, subject, text, log)
- try_count = self.MAX_RETRIES
- log('Email successfully sent')
- except:
+ worker = Worker(self.sendmail,
+ (attachment, aname, to, subject, text, log))
+ worker.start()
+ start_time = time.time()
+ while worker.is_alive():
+ worker.join(0.2)
if abort.is_set():
+ log('Sending aborted by user')
return
- if try_count == self.MAX_RETRIES:
- raise
- log.exception('\nSending failed...\n')
-
+ if time.time() - start_time > self.TIMEOUT:
+ log('Sending timed out')
+ raise Exception(
+ 'Sending email %r to %r timed out, aborting'% (subject,
+ to))
+ if worker.exception is None:
+ log('Email successfully sent')
+ return
+ log.error('\nSending failed...\n')
+ log.debug(worker.tb)
try_count += 1
+ if try_count > self.MAX_RETRIES:
+ raise worker.exception
def sendmail(self, attachment, aname, to, subject, text, log):
while time.time() - self.last_send_time <= self.rate_limit:
@@ -90,7 +122,7 @@ def send_mails(jobnames, callback, attachments, to_s, subjects,
attachments, to_s, subjects, texts, attachment_names):
description = _('Email %s to %s') % (name, to)
job = ThreadedJob('email', description, gui_sendmail, (attachment, aname, to,
- subject, text), {}, callback, killable=False)
+ subject, text), {}, callback)
job_manager.run_threaded_job(job)
diff --git a/src/calibre/gui2/jobs.py b/src/calibre/gui2/jobs.py
index 6aae892d61..20154a298b 100644
--- a/src/calibre/gui2/jobs.py
+++ b/src/calibre/gui2/jobs.py
@@ -457,7 +457,7 @@ class JobsDialog(QDialog, Ui_JobsDialog):
def kill_job(self, *args):
if question_dialog(self, _('Are you sure?'), _('Do you really want to stop the selected job?')):
- for index in self.jobs_view.selectedIndexes():
+ for index in self.jobs_view.selectionModel().selectedRows():
row = index.row()
self.model.kill_job(row, self)
diff --git a/src/calibre/gui2/preferences/coloring.py b/src/calibre/gui2/preferences/coloring.py
index f8376e9b84..b3c7873b45 100644
--- a/src/calibre/gui2/preferences/coloring.py
+++ b/src/calibre/gui2/preferences/coloring.py
@@ -462,11 +462,11 @@ class EditRules(QWidget): # {{{
self.l = l = QGridLayout(self)
self.setLayout(l)
- self.l1 = l1 = QLabel(_(
+ self.l1 = l1 = QLabel(''+_(
'You can control the color of columns in the'
' book list by creating "rules" that tell calibre'
' what color to use. Click the Add Rule button below'
- ' to get started. You can change an existing rule by double'
+ ' to get started.
You can change an existing rule by double'
' clicking it.'))
l1.setWordWrap(True)
l.addWidget(l1, 0, 0, 1, 2)
diff --git a/src/calibre/gui2/store/amazon_plugin.py b/src/calibre/gui2/store/amazon_plugin.py
index a9ec17e694..693ef883fb 100644
--- a/src/calibre/gui2/store/amazon_plugin.py
+++ b/src/calibre/gui2/store/amazon_plugin.py
@@ -139,14 +139,26 @@ class AmazonKindleStore(StorePlugin):
cover_xpath = './/div[@class="productTitle"]//img/@src'
title_xpath = './/div[@class="productTitle"]/a//text()'
price_xpath = './/div[@class="newPrice"]/span/text()'
- # Vertical list of books. Search "martin"
+ # Vertical list of books.
else:
- data_xpath = '//div[contains(@class, "results")]//div[contains(@class, "result")]'
- format_xpath = './/span[@class="binding"]//text()'
- asin_xpath = './/div[@class="image"]/a[1]'
- cover_xpath = './/img[@class="productImage"]/@src'
- title_xpath = './/a[@class="title"]/text()'
- price_xpath = './/span[@class="price"]/text()'
+ # New style list. Search "Paolo Bacigalupi"
+ if doc.xpath('boolean(//div[@class="image"])'):
+ data_xpath = '//div[contains(@class, "results")]//div[contains(@class, "result")]'
+ format_xpath = './/span[@class="binding"]//text()'
+ asin_xpath = './/div[@class="image"]/a[1]'
+ cover_xpath = './/img[@class="productImage"]/@src'
+ title_xpath = './/a[@class="title"]/text()'
+ price_xpath = './/span[@class="price"]/text()'
+ # Old style list. Search "martin"
+ else:
+ data_xpath = '//div[contains(@class, "result")]'
+ format_xpath = './/span[@class="format"]//text()'
+ asin_xpath = './/div[@class="productImage"]/a[1]'
+ cover_xpath = './/div[@class="productImage"]//img/@src'
+ title_xpath = './/div[@class="productTitle"]/a/text()'
+ price_xpath = './/div[@class="newPrice"]//span//text()'
+
+
for data in doc.xpath(data_xpath):
if counter <= 0:
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 03c0712daa..67c67b1ff7 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -26,7 +26,7 @@ from calibre.library.custom_columns import CustomColumns
from calibre.library.sqlite import connect, IntegrityError
from calibre.library.prefs import DBPrefs
from calibre.ebooks.metadata.book.base import Metadata
-from calibre.constants import preferred_encoding, iswindows, isosx, filesystem_encoding
+from calibre.constants import preferred_encoding, iswindows, filesystem_encoding
from calibre.ptempfile import PersistentTemporaryFile
from calibre.customize.ui import run_plugins_on_import
from calibre import isbytestring
@@ -188,8 +188,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
apply_default_prefs = not os.path.exists(self.dbpath)
self.connect()
- self.is_case_sensitive = not iswindows and not isosx and \
- not os.path.exists(self.dbpath.replace('metadata.db', 'MeTAdAtA.dB'))
+ self.is_case_sensitive = (not iswindows and
+ not os.path.exists(self.dbpath.replace('metadata.db',
+ 'MeTAdAtA.dB')))
SchemaUpgrade.__init__(self)
# Guarantee that the library_id is set
self.library_id
@@ -606,7 +607,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
f = self.format(id, format, index_is_id=True, as_file=True)
if f is None:
continue
- with tempfile.SpooledTemporaryFile(max_size=100*(1024**2)) as stream:
+ with tempfile.SpooledTemporaryFile(max_size=30*(1024**2)) as stream:
with f:
shutil.copyfileobj(f, stream)
stream.seek(0)
diff --git a/src/calibre/manual/conversion.rst b/src/calibre/manual/conversion.rst
index 540da0fc9a..fea828f706 100644
--- a/src/calibre/manual/conversion.rst
+++ b/src/calibre/manual/conversion.rst
@@ -189,7 +189,7 @@ Extra CSS
This option allows you to specify arbitrary CSS that will be applied to all HTML files in the
input. This CSS is applied with very high priority and so should override most CSS present in
-the input document itself. You can use this setting to fine tune the presentation/layout of your
+the **input document** itself. You can use this setting to fine tune the presentation/layout of your
document. For example, if you want all paragraphs of class `endnote` to be right aligned, just
add::
@@ -200,7 +200,8 @@ or if you want to change the indentation of all paragraphs::
p { text-indent: 5mm; }
:guilabel:`Extra CSS` is a very powerful option, but you do need an understanding of how CSS works
-to use it to its full potential.
+to use it to its full potential. You can use the debug pipeline option described above to see what
+CSS is present in your input document.
Miscellaneous
~~~~~~~~~~~~~~
diff --git a/src/calibre/web/feeds/__init__.py b/src/calibre/web/feeds/__init__.py
index a10fb03f91..dbd1f74f82 100644
--- a/src/calibre/web/feeds/__init__.py
+++ b/src/calibre/web/feeds/__init__.py
@@ -317,6 +317,9 @@ def feed_from_xml(raw_xml, title=None, oldest_article=7,
max_articles_per_feed=100,
get_article_url=lambda item: item.get('link', None),
log=default_log):
+ # Handle unclosed escaped entities. They trip up feedparser and HBR for one
+ # generates them
+ raw_xml = re.sub(r'(&#\d+)([^0-9;])', r'\1;\2', raw_xml)
feed = parse(raw_xml)
pfeed = Feed(get_article_url=get_article_url, log=log)
pfeed.populate_from_feed(feed, title=title,
diff --git a/src/calibre/web/feeds/news.py b/src/calibre/web/feeds/news.py
index 325fcf5209..c74a9b662c 100644
--- a/src/calibre/web/feeds/news.py
+++ b/src/calibre/web/feeds/news.py
@@ -13,8 +13,8 @@ from functools import partial
from contextlib import nested, closing
-from calibre import browser, __appname__, iswindows, \
- strftime, preferred_encoding, as_unicode
+from calibre import (browser, __appname__, iswindows,
+ strftime, preferred_encoding, as_unicode)
from calibre.ebooks.BeautifulSoup import BeautifulSoup, NavigableString, CData, Tag
from calibre.ebooks.metadata.opf2 import OPFCreator
from calibre import entity_to_unicode