From ee04420a2e030cb4b5ac58471bed56ecc69d5ceb Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 12 Jun 2011 08:22:52 -0600 Subject: [PATCH 01/10] ... --- src/calibre/gui2/device.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index ab02a183c6..9f71c3088d 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -38,6 +38,10 @@ class DeviceJob(BaseJob): # {{{ BaseJob.__init__(self, description) self.func = func self.callback_on_done = done + if not isinstance(self.callback_on_done, (Dispatcher, + FunctionDispatcher)): + self.callback_on_done = FunctionDispatcher(self.callback_on_done) + self.args, self.kwargs = args, kwargs self.exception = None self.job_manager = job_manager @@ -259,7 +263,8 @@ class DeviceManager(Thread): # {{{ job = self.next() if job is not None: self.current_job = job - self.device.set_progress_reporter(job.report_progress) + if self.device is not None: + self.device.set_progress_reporter(job.report_progress) self.current_job.run() self.current_job = None else: From 33b8feb4bf55f3c79679251731691d151a292db6 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 12 Jun 2011 11:53:35 -0400 Subject: [PATCH 02/10] Store: Fix Amazon plugin, listing of books uses old style vertical list for some search terms. --- src/calibre/gui2/store/amazon_plugin.py | 26 ++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/calibre/gui2/store/amazon_plugin.py b/src/calibre/gui2/store/amazon_plugin.py index a9ec17e694..693ef883fb 100644 --- a/src/calibre/gui2/store/amazon_plugin.py +++ b/src/calibre/gui2/store/amazon_plugin.py @@ -139,14 +139,26 @@ class AmazonKindleStore(StorePlugin): cover_xpath = './/div[@class="productTitle"]//img/@src' title_xpath = './/div[@class="productTitle"]/a//text()' price_xpath = './/div[@class="newPrice"]/span/text()' - # Vertical list of books. Search "martin" + # Vertical list of books. else: - data_xpath = '//div[contains(@class, "results")]//div[contains(@class, "result")]' - format_xpath = './/span[@class="binding"]//text()' - asin_xpath = './/div[@class="image"]/a[1]' - cover_xpath = './/img[@class="productImage"]/@src' - title_xpath = './/a[@class="title"]/text()' - price_xpath = './/span[@class="price"]/text()' + # New style list. Search "Paolo Bacigalupi" + if doc.xpath('boolean(//div[@class="image"])'): + data_xpath = '//div[contains(@class, "results")]//div[contains(@class, "result")]' + format_xpath = './/span[@class="binding"]//text()' + asin_xpath = './/div[@class="image"]/a[1]' + cover_xpath = './/img[@class="productImage"]/@src' + title_xpath = './/a[@class="title"]/text()' + price_xpath = './/span[@class="price"]/text()' + # Old style list. Search "martin" + else: + data_xpath = '//div[contains(@class, "result")]' + format_xpath = './/span[@class="format"]//text()' + asin_xpath = './/div[@class="productImage"]/a[1]' + cover_xpath = './/div[@class="productImage"]//img/@src' + title_xpath = './/div[@class="productTitle"]/a/text()' + price_xpath = './/div[@class="newPrice"]//span//text()' + + for data in doc.xpath(data_xpath): if counter <= 0: From 7d44fed9e47b4e17ecfa84211e76ebf875d20911 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 12 Jun 2011 10:03:00 -0600 Subject: [PATCH 03/10] ... --- src/calibre/manual/conversion.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/calibre/manual/conversion.rst b/src/calibre/manual/conversion.rst index 540da0fc9a..fea828f706 100644 --- a/src/calibre/manual/conversion.rst +++ b/src/calibre/manual/conversion.rst @@ -189,7 +189,7 @@ Extra CSS This option allows you to specify arbitrary CSS that will be applied to all HTML files in the input. This CSS is applied with very high priority and so should override most CSS present in -the input document itself. You can use this setting to fine tune the presentation/layout of your +the **input document** itself. You can use this setting to fine tune the presentation/layout of your document. For example, if you want all paragraphs of class `endnote` to be right aligned, just add:: @@ -200,7 +200,8 @@ or if you want to change the indentation of all paragraphs:: p { text-indent: 5mm; } :guilabel:`Extra CSS` is a very powerful option, but you do need an understanding of how CSS works -to use it to its full potential. +to use it to its full potential. You can use the debug pipeline option described above to see what +CSS is present in your input document. Miscellaneous ~~~~~~~~~~~~~~ From c1c5c8e676055ba8a91fdcd346d530e750d67635 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 12 Jun 2011 10:12:12 -0600 Subject: [PATCH 04/10] ... --- src/calibre/gui2/device.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 9f71c3088d..c9c79095f3 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -597,7 +597,7 @@ class DeviceMenu(QMenu): # {{{ # }}} -class DeviceSignals(QObject): +class DeviceSignals(QObject): # {{{ #: This signal is emitted once, after metadata is downloaded from the #: connected device. #: The sequence: gui.device_manager.is_device_connected will become True, @@ -614,6 +614,7 @@ class DeviceSignals(QObject): device_connection_changed = pyqtSignal(object) device_signals = DeviceSignals() +# }}} class DeviceMixin(object): # {{{ From ba2b3f05bd58447e4a122d4cd52b15ca8bfef3e5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 12 Jun 2011 10:24:54 -0600 Subject: [PATCH 05/10] Database: Explicitly test for case sensitivity on OS X instead of assuming a case insensitive filesystem. Fixes #796258 (Various file management functions/integrity checks broken on case sensitive OSX) --- src/calibre/library/database2.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 03c0712daa..926ac5527c 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -26,7 +26,7 @@ from calibre.library.custom_columns import CustomColumns from calibre.library.sqlite import connect, IntegrityError from calibre.library.prefs import DBPrefs from calibre.ebooks.metadata.book.base import Metadata -from calibre.constants import preferred_encoding, iswindows, isosx, filesystem_encoding +from calibre.constants import preferred_encoding, iswindows, filesystem_encoding from calibre.ptempfile import PersistentTemporaryFile from calibre.customize.ui import run_plugins_on_import from calibre import isbytestring @@ -188,8 +188,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): apply_default_prefs = not os.path.exists(self.dbpath) self.connect() - self.is_case_sensitive = not iswindows and not isosx and \ - not os.path.exists(self.dbpath.replace('metadata.db', 'MeTAdAtA.dB')) + self.is_case_sensitive = (not iswindows and + not os.path.exists(self.dbpath.replace('metadata.db', + 'MeTAdAtA.dB'))) SchemaUpgrade.__init__(self) # Guarantee that the library_id is set self.library_id From dea910599f0017b4fe44b0896c7f4f26d498bc92 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 12 Jun 2011 11:24:32 -0600 Subject: [PATCH 06/10] Fix "stop selected Jobs" button trying to stop the same job multiple times --- src/calibre/gui2/jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/jobs.py b/src/calibre/gui2/jobs.py index 6aae892d61..20154a298b 100644 --- a/src/calibre/gui2/jobs.py +++ b/src/calibre/gui2/jobs.py @@ -457,7 +457,7 @@ class JobsDialog(QDialog, Ui_JobsDialog): def kill_job(self, *args): if question_dialog(self, _('Are you sure?'), _('Do you really want to stop the selected job?')): - for index in self.jobs_view.selectedIndexes(): + for index in self.jobs_view.selectionModel().selectedRows(): row = index.row() self.model.kill_job(row, self) From 685ef02000573dc46012fd80960d8e87e1700ff4 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 12 Jun 2011 13:16:31 -0600 Subject: [PATCH 07/10] Smaller memory limit for spooled file when moving files in db2 --- src/calibre/library/database2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 926ac5527c..67c67b1ff7 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -607,7 +607,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): f = self.format(id, format, index_is_id=True, as_file=True) if f is None: continue - with tempfile.SpooledTemporaryFile(max_size=100*(1024**2)) as stream: + with tempfile.SpooledTemporaryFile(max_size=30*(1024**2)) as stream: with f: shutil.copyfileobj(f, stream) stream.seek(0) From db595796fc417e254632dc1377acc223babc9751 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 12 Jun 2011 15:16:56 -0600 Subject: [PATCH 08/10] Email sending: Allow user to stop email jobs (note that stopping may not actually prevent the email from being sent, depending on when the stop happens). Also automatically abort email sending if it takes longer than 15mins. Fixes #795960 (E-mail task hangs) --- src/calibre/gui2/email.py | 54 +++++++++++++++++++++++++++++++-------- 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/src/calibre/gui2/email.py b/src/calibre/gui2/email.py index 4b4c920a7e..b82f421e1e 100644 --- a/src/calibre/gui2/email.py +++ b/src/calibre/gui2/email.py @@ -9,6 +9,7 @@ __docformat__ = 'restructuredtext en' import os, socket, time from binascii import unhexlify from functools import partial +from threading import Thread from itertools import repeat from calibre.utils.smtp import (compose_mail, sendmail, extract_email_address, @@ -22,9 +23,30 @@ from calibre.library.save_to_disk import get_components from calibre.utils.config import tweaks from calibre.gui2.threaded_jobs import ThreadedJob +class Worker(Thread): + + def __init__(self, func, args): + Thread.__init__(self) + self.daemon = True + self.exception = self.tb = None + self.func, self.args = func, args + + def run(self): + #time.sleep(1000) + try: + self.func(*self.args) + except Exception as e: + import traceback + self.exception = e + self.tb = traceback.format_exc() + finally: + self.func = self.args = None + + class Sendmail(object): MAX_RETRIES = 1 + TIMEOUT = 15 * 60 # seconds def __init__(self): self.calculate_rate_limit() @@ -42,22 +64,32 @@ class Sendmail(object): abort=None, notifications=None): try_count = 0 - while try_count <= self.MAX_RETRIES: + while True: if try_count > 0: log('\nRetrying in %d seconds...\n' % self.rate_limit) - try: - self.sendmail(attachment, aname, to, subject, text, log) - try_count = self.MAX_RETRIES - log('Email successfully sent') - except: + worker = Worker(self.sendmail, + (attachment, aname, to, subject, text, log)) + worker.start() + start_time = time.time() + while worker.is_alive(): + worker.join(0.2) if abort.is_set(): + log('Sending aborted by user') return - if try_count == self.MAX_RETRIES: - raise - log.exception('\nSending failed...\n') - + if time.time() - start_time > self.TIMEOUT: + log('Sending timed out') + raise Exception( + 'Sending email %r to %r timed out, aborting'% (subject, + to)) + if worker.exception is None: + log('Email successfully sent') + return + log.error('\nSending failed...\n') + log.debug(worker.tb) try_count += 1 + if try_count > self.MAX_RETRIES: + raise worker.exception def sendmail(self, attachment, aname, to, subject, text, log): while time.time() - self.last_send_time <= self.rate_limit: @@ -90,7 +122,7 @@ def send_mails(jobnames, callback, attachments, to_s, subjects, attachments, to_s, subjects, texts, attachment_names): description = _('Email %s to %s') % (name, to) job = ThreadedJob('email', description, gui_sendmail, (attachment, aname, to, - subject, text), {}, callback, killable=False) + subject, text), {}, callback) job_manager.run_threaded_job(job) From 0d5c964ea72635b0f1cdec0357abb61838d7f70d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 12 Jun 2011 15:36:49 -0600 Subject: [PATCH 09/10] ... --- recipes/hbr_blogs.recipe | 2 +- src/calibre/web/feeds/__init__.py | 3 +++ src/calibre/web/feeds/news.py | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/recipes/hbr_blogs.recipe b/recipes/hbr_blogs.recipe index bd72a95ebf..acee567d8d 100644 --- a/recipes/hbr_blogs.recipe +++ b/recipes/hbr_blogs.recipe @@ -6,7 +6,7 @@ class HBR(BasicNewsRecipe): title = 'Harvard Business Review Blogs' description = 'To subscribe go to http://hbr.harvardbusiness.org' needs_subscription = True - __author__ = 'Kovid Goyal and Sujata Raman, enhanced by BrianG' + __author__ = 'Kovid Goyal, enhanced by BrianG' language = 'en' no_stylesheets = True diff --git a/src/calibre/web/feeds/__init__.py b/src/calibre/web/feeds/__init__.py index a10fb03f91..dbd1f74f82 100644 --- a/src/calibre/web/feeds/__init__.py +++ b/src/calibre/web/feeds/__init__.py @@ -317,6 +317,9 @@ def feed_from_xml(raw_xml, title=None, oldest_article=7, max_articles_per_feed=100, get_article_url=lambda item: item.get('link', None), log=default_log): + # Handle unclosed escaped entities. They trip up feedparser and HBR for one + # generates them + raw_xml = re.sub(r'(&#\d+)([^0-9;])', r'\1;\2', raw_xml) feed = parse(raw_xml) pfeed = Feed(get_article_url=get_article_url, log=log) pfeed.populate_from_feed(feed, title=title, diff --git a/src/calibre/web/feeds/news.py b/src/calibre/web/feeds/news.py index 325fcf5209..c74a9b662c 100644 --- a/src/calibre/web/feeds/news.py +++ b/src/calibre/web/feeds/news.py @@ -13,8 +13,8 @@ from functools import partial from contextlib import nested, closing -from calibre import browser, __appname__, iswindows, \ - strftime, preferred_encoding, as_unicode +from calibre import (browser, __appname__, iswindows, + strftime, preferred_encoding, as_unicode) from calibre.ebooks.BeautifulSoup import BeautifulSoup, NavigableString, CData, Tag from calibre.ebooks.metadata.opf2 import OPFCreator from calibre import entity_to_unicode From 67964530424b3644bb14a57cc2d43f159fffc987 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 12 Jun 2011 17:05:55 -0600 Subject: [PATCH 10/10] ... --- src/calibre/ebooks/mobi/reader.py | 1 - src/calibre/gui2/preferences/coloring.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/calibre/ebooks/mobi/reader.py b/src/calibre/ebooks/mobi/reader.py index 934e8476d2..46505de4bd 100644 --- a/src/calibre/ebooks/mobi/reader.py +++ b/src/calibre/ebooks/mobi/reader.py @@ -348,7 +348,6 @@ class MobiReader(object): self.processed_html = self.remove_random_bytes(self.processed_html) root = soupparser.fromstring(self.processed_html) - if root.tag != 'html': self.log.warn('File does not have opening tag') nroot = html.fromstring('') diff --git a/src/calibre/gui2/preferences/coloring.py b/src/calibre/gui2/preferences/coloring.py index f8376e9b84..b3c7873b45 100644 --- a/src/calibre/gui2/preferences/coloring.py +++ b/src/calibre/gui2/preferences/coloring.py @@ -462,11 +462,11 @@ class EditRules(QWidget): # {{{ self.l = l = QGridLayout(self) self.setLayout(l) - self.l1 = l1 = QLabel(_( + self.l1 = l1 = QLabel('

'+_( 'You can control the color of columns in the' ' book list by creating "rules" that tell calibre' ' what color to use. Click the Add Rule button below' - ' to get started. You can change an existing rule by double' + ' to get started.

You can change an existing rule by double' ' clicking it.')) l1.setWordWrap(True) l.addWidget(l1, 0, 0, 1, 2)