From f979f0d357e4fcf86ff49e98462242f6a959df5a Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 11 Apr 2011 14:49:54 +0100 Subject: [PATCH 01/36] Improve performance of ui.library_moved(). Changes remove multiple calls to tags_view.recount(). --- src/calibre/gui2/search_box.py | 5 +++-- src/calibre/gui2/search_restriction_mixin.py | 5 +++-- src/calibre/gui2/ui.py | 5 +++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/calibre/gui2/search_box.py b/src/calibre/gui2/search_box.py index 8936493290..ea7cab95d0 100644 --- a/src/calibre/gui2/search_box.py +++ b/src/calibre/gui2/search_box.py @@ -436,14 +436,15 @@ class SavedSearchBoxMixin(object): # {{{ b = getattr(self, x+'_search_button') b.setStatusTip(b.toolTip()) - def saved_searches_changed(self, set_restriction=None): + def saved_searches_changed(self, set_restriction=None, recount=True): p = sorted(saved_searches().names(), key=sort_key) if set_restriction is None: set_restriction = unicode(self.search_restriction.currentText()) # rebuild the restrictions combobox using current saved searches self.search_restriction.clear() self.search_restriction.addItem('') - self.tags_view.recount() + if recount: + self.tags_view.recount() for s in p: self.search_restriction.addItem(s) if set_restriction: # redo the search restriction if there was one diff --git a/src/calibre/gui2/search_restriction_mixin.py b/src/calibre/gui2/search_restriction_mixin.py index 73c191101c..74e448da6e 100644 --- a/src/calibre/gui2/search_restriction_mixin.py +++ b/src/calibre/gui2/search_restriction_mixin.py @@ -25,8 +25,9 @@ class SearchRestrictionMixin(object): r = self.search_restriction.findText(name) if r < 0: r = 0 - self.search_restriction.setCurrentIndex(r) - self.apply_search_restriction(r) + if r != self.search_restriction.currentIndex(): + self.search_restriction.setCurrentIndex(r) + self.apply_search_restriction(r) def apply_search_restriction(self, i): r = unicode(self.search_restriction.currentText()) diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 4a9d76418c..4d363c283a 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -446,12 +446,13 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ self.search.clear() self.saved_search.clear() self.book_details.reset_info() - self.library_view.model().count_changed() prefs['library_path'] = self.library_path + #self.library_view.model().count_changed() db = self.library_view.model().db + self.iactions['Choose Library'].count_changed(db.count()) self.set_window_title() self.apply_named_search_restriction('') # reset restriction to null - self.saved_searches_changed() # reload the search restrictions combo box + self.saved_searches_changed(recount=False) # reload the search restrictions combo box self.apply_named_search_restriction(db.prefs['gui_restriction']) for action in self.iactions.values(): action.library_changed(db) From 3a2a3f1b4f2c7e0e6225910a6b26c83c1625dd2c Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 11 Apr 2011 16:28:50 +0100 Subject: [PATCH 02/36] Optimization of composite column evaluation: don't compute the user_categories, as they cannot appear in a composite column. --- src/calibre/library/caches.py | 3 ++- src/calibre/library/database2.py | 22 ++++++++++++---------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 74ddb2bc41..a108feb388 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -191,7 +191,8 @@ class CacheRow(list): # {{{ if is_comp: id = list.__getitem__(self, 0) self._must_do = False - mi = self.db.get_metadata(id, index_is_id=True) + mi = self.db.get_metadata(id, index_is_id=True, + get_user_categories=False) for c in self._composites: self[c] = mi.get(self._composites[c]) return list.__getitem__(self, col) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 50ecc4f1e5..0b1182c0bf 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -823,7 +823,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): pass return (path, mi, sequence) - def get_metadata(self, idx, index_is_id=False, get_cover=False): + def get_metadata(self, idx, index_is_id=False, get_cover=False, + get_user_categories=True): ''' Convenience method to return metadata as a :class:`Metadata` object. Note that the list of formats is not verified. @@ -882,16 +883,17 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): user_cats = self.prefs['user_categories'] user_cat_vals = {} - for ucat in user_cats: - res = [] - for name,cat,ign in user_cats[ucat]: - v = mi.get(cat, None) - if isinstance(v, list): - if name in v: + if get_user_categories: + for ucat in user_cats: + res = [] + for name,cat,ign in user_cats[ucat]: + v = mi.get(cat, None) + if isinstance(v, list): + if name in v: + res.append([name,cat]) + elif name == v: res.append([name,cat]) - elif name == v: - res.append([name,cat]) - user_cat_vals[ucat] = res + user_cat_vals[ucat] = res mi.user_categories = user_cat_vals if get_cover: From ea42c67e05c51e4faa7c5e7de4487ed81b1973e0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 11 Apr 2011 09:47:20 -0600 Subject: [PATCH 03/36] ... --- src/calibre/gui2/actions/add.py | 29 +++++++++++++----- src/calibre/gui2/actions/edit_metadata.py | 36 ++++++++++++++++------- 2 files changed, 48 insertions(+), 17 deletions(-) diff --git a/src/calibre/gui2/actions/add.py b/src/calibre/gui2/actions/add.py index cf67cd6cfa..f8dd0693ea 100644 --- a/src/calibre/gui2/actions/add.py +++ b/src/calibre/gui2/actions/add.py @@ -22,6 +22,8 @@ from calibre.constants import preferred_encoding, filesystem_encoding from calibre.gui2.actions import InterfaceAction from calibre.gui2 import config, question_dialog from calibre.ebooks.metadata import MetaInformation +from calibre.utils.config import test_eight_code +from calibre.ebooks.metadata.sources.base import msprefs def get_filters(): return [ @@ -178,13 +180,26 @@ class AddAction(InterfaceAction): except IndexError: self.gui.library_view.model().books_added(self.isbn_add_dialog.value) self.isbn_add_dialog.accept() - orig = config['overwrite_author_title_metadata'] - config['overwrite_author_title_metadata'] = True - try: - self.gui.iactions['Edit Metadata'].do_download_metadata( - self.add_by_isbn_ids) - finally: - config['overwrite_author_title_metadata'] = orig + if test_eight_code: + orig = msprefs['ignore_fields'] + new = list(orig) + for x in ('title', 'authors'): + if x in new: + new.remove(x) + msprefs['ignore_fields'] = new + try: + self.gui.iactions['Edit Metadata'].download_metadata( + ids=self.add_by_isbn_ids) + finally: + msprefs['ignore_fields'] = orig + else: + orig = config['overwrite_author_title_metadata'] + config['overwrite_author_title_metadata'] = True + try: + self.gui.iactions['Edit Metadata'].do_download_metadata( + self.add_by_isbn_ids) + finally: + config['overwrite_author_title_metadata'] = orig return diff --git a/src/calibre/gui2/actions/edit_metadata.py b/src/calibre/gui2/actions/edit_metadata.py index e29e6c344d..fc663d268a 100644 --- a/src/calibre/gui2/actions/edit_metadata.py +++ b/src/calibre/gui2/actions/edit_metadata.py @@ -35,16 +35,23 @@ class EditMetadataAction(InterfaceAction): md.addAction(_('Edit metadata in bulk'), partial(self.edit_metadata, False, bulk=True)) md.addSeparator() - md.addAction(_('Download metadata and covers'), - partial(self.download_metadata, False, covers=True), + if test_eight_code: + dall = self.download_metadata + dident = partial(self.download_metadata, covers=False) + dcovers = partial(self.download_metadata, identify=False) + else: + dall = partial(self.download_metadata_old, False, covers=True) + dident = partial(self.download_metadata_old, False, covers=False) + dcovers = partial(self.download_metadata_old, False, covers=True, + set_metadata=False, set_social_metadata=False) + + md.addAction(_('Download metadata and covers'), dall, Qt.ControlModifier+Qt.Key_D) - md.addAction(_('Download only metadata'), - partial(self.download_metadata, False, covers=False)) - md.addAction(_('Download only covers'), - partial(self.download_metadata, False, covers=True, - set_metadata=False, set_social_metadata=False)) - md.addAction(_('Download only social metadata'), - partial(self.download_metadata, False, covers=False, + md.addAction(_('Download only metadata'), dident) + md.addAction(_('Download only covers'), dcovers) + if not test_eight_code: + md.addAction(_('Download only social metadata'), + partial(self.download_metadata_old, False, covers=False, set_metadata=False, set_social_metadata=True)) self.metadata_menu = md @@ -73,7 +80,16 @@ class EditMetadataAction(InterfaceAction): self.qaction.setEnabled(enabled) self.action_merge.setEnabled(enabled) - def download_metadata(self, checked, covers=True, set_metadata=True, + def download_metadata(self, identify=True, covers=True, ids=None): + if ids is None: + rows = self.gui.library_view.selectionModel().selectedRows() + if not rows or len(rows) == 0: + return error_dialog(self.gui, _('Cannot download metadata'), + _('No books selected'), show=True) + db = self.gui.library_view.model().db + ids = [db.id(row.row()) for row in rows] + + def download_metadata_old(self, checked, covers=True, set_metadata=True, set_social_metadata=None): rows = self.gui.library_view.selectionModel().selectedRows() if not rows or len(rows) == 0: From 72c9625094afc23c03db8d6767e2fde2e0303dc4 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 11 Apr 2011 17:17:44 +0100 Subject: [PATCH 04/36] Add the new configure metadata download tool button to the alternate metadata single dialog --- src/calibre/gui2/metadata/single.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/calibre/gui2/metadata/single.py b/src/calibre/gui2/metadata/single.py index 4ce76d8cc8..2e5b43ceba 100644 --- a/src/calibre/gui2/metadata/single.py +++ b/src/calibre/gui2/metadata/single.py @@ -612,6 +612,10 @@ class MetadataSingleDialogAlt1(MetadataSingleDialogBase): # {{{ self.button_box.addButton(self.fetch_metadata_button, QDialogButtonBox.ActionRole) + self.config_metadata_button.setToolButtonStyle(Qt.ToolButtonTextOnly) + self.config_metadata_button.setText(_('Configure metadata downloading')) + self.button_box.addButton(self.config_metadata_button, + QDialogButtonBox.ActionRole) sto(self.button_box, self.title) def create_row(row, widget, tab_to, button=None, icon=None, span=1): From aac23508dff35df7ccec7c319d4586f5b7e58f23 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 11 Apr 2011 15:43:34 -0600 Subject: [PATCH 05/36] Fix regression that caused clicking auto send to also change the email address in Preferences->Email --- src/calibre/gui2/preferences/emailp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/preferences/emailp.py b/src/calibre/gui2/preferences/emailp.py index ded6891387..f5c7b5a3a7 100644 --- a/src/calibre/gui2/preferences/emailp.py +++ b/src/calibre/gui2/preferences/emailp.py @@ -88,7 +88,7 @@ class EmailAccounts(QAbstractTableModel): # {{{ self.subjects[account] = unicode(value.toString()) elif col == 1: self.accounts[account][0] = unicode(value.toString()).upper() - else: + elif col == 0: na = unicode(value.toString()) from email.utils import parseaddr addr = parseaddr(na)[-1] @@ -100,7 +100,7 @@ class EmailAccounts(QAbstractTableModel): # {{{ self.accounts[na][0] = 'AZW, MOBI, TPZ, PRC, AZW1' self.dataChanged.emit( - self.index(index.row(), 0), self.index(index.row(), 2)) + self.index(index.row(), 0), self.index(index.row(), 3)) return True def make_default(self, index): From 64dd32eaf5629f997b4f683c054061e66dff1b28 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 11 Apr 2011 15:43:58 -0600 Subject: [PATCH 06/36] ... --- src/calibre/gui2/preferences/emailp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/preferences/emailp.py b/src/calibre/gui2/preferences/emailp.py index f5c7b5a3a7..1644dc6b73 100644 --- a/src/calibre/gui2/preferences/emailp.py +++ b/src/calibre/gui2/preferences/emailp.py @@ -84,7 +84,7 @@ class EmailAccounts(QAbstractTableModel): # {{{ account = self.account_order[row] if col == 3: self.accounts[account][1] ^= True - if col == 2: + elif col == 2: self.subjects[account] = unicode(value.toString()) elif col == 1: self.accounts[account][0] = unicode(value.toString()).upper() From 71967fd4ba173c5ad9c82f60a166b881fe29809d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 11 Apr 2011 15:53:34 -0600 Subject: [PATCH 07/36] New framework for running I/O bound jobs in threads inthe calibre main process. Migrate email sending to the new framework. --- src/calibre/gui2/dialogs/job_view.ui | 40 ++-- src/calibre/gui2/email.py | 226 ++++++------------- src/calibre/gui2/jobs.py | 85 +++++-- src/calibre/gui2/metadata/bulk_download2.py | 11 + src/calibre/gui2/threaded_jobs.py | 238 ++++++++++++++++++++ src/calibre/gui2/ui.py | 3 +- src/calibre/utils/ipc/job.py | 10 +- 7 files changed, 410 insertions(+), 203 deletions(-) create mode 100644 src/calibre/gui2/metadata/bulk_download2.py create mode 100644 src/calibre/gui2/threaded_jobs.py diff --git a/src/calibre/gui2/dialogs/job_view.ui b/src/calibre/gui2/dialogs/job_view.ui index 8b54c23573..1e854c0f29 100644 --- a/src/calibre/gui2/dialogs/job_view.ui +++ b/src/calibre/gui2/dialogs/job_view.ui @@ -1,7 +1,8 @@ - + + Dialog - - + + 0 0 @@ -9,38 +10,41 @@ 462 - + Details of job - - + + :/images/view.png:/images/view.png - - - - + + + + false - + QPlainTextEdit::NoWrap - + true - - - + + + QDialogButtonBox::Ok + + + - + @@ -49,11 +53,11 @@ Dialog accept() - + 617 442 - + 206 -5 diff --git a/src/calibre/gui2/email.py b/src/calibre/gui2/email.py index c6d58fa340..c8adeb7d31 100644 --- a/src/calibre/gui2/email.py +++ b/src/calibre/gui2/email.py @@ -6,9 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, socket, time, cStringIO -from threading import Thread -from Queue import Queue +import os, socket, time from binascii import unhexlify from functools import partial from itertools import repeat @@ -16,67 +14,20 @@ from itertools import repeat from calibre.utils.smtp import compose_mail, sendmail, extract_email_address, \ config as email_config from calibre.utils.filenames import ascii_filename -from calibre.utils.ipc.job import BaseJob -from calibre.ptempfile import PersistentTemporaryFile from calibre.customize.ui import available_input_formats, available_output_formats from calibre.ebooks.metadata import authors_to_string from calibre.constants import preferred_encoding from calibre.gui2 import config, Dispatcher, warning_dialog from calibre.library.save_to_disk import get_components from calibre.utils.config import tweaks +from calibre.gui2.threaded_jobs import ThreadedJob -class EmailJob(BaseJob): # {{{ - - def __init__(self, callback, description, attachment, aname, to, subject, text, job_manager): - BaseJob.__init__(self, description) - self.exception = None - self.job_manager = job_manager - self.email_args = (attachment, aname, to, subject, text) - self.email_sent_callback = callback - self.log_path = None - self._log_file = cStringIO.StringIO() - self._log_file.write(self.description.encode('utf-8') + '\n') - - @property - def log_file(self): - if self.log_path is not None: - return open(self.log_path, 'rb') - return cStringIO.StringIO(self._log_file.getvalue()) - - def start_work(self): - self.start_time = time.time() - self.job_manager.changed_queue.put(self) - - def job_done(self): - self.duration = time.time() - self.start_time - self.percent = 1 - # Dump log onto disk - lf = PersistentTemporaryFile('email_log') - lf.write(self._log_file.getvalue()) - lf.close() - self.log_path = lf.name - self._log_file.close() - self._log_file = None - - self.job_manager.changed_queue.put(self) - - def log_write(self, what): - self._log_file.write(what) - -# }}} - -class Emailer(Thread): # {{{ +class Sendmail(object): MAX_RETRIES = 1 - def __init__(self, job_manager): - Thread.__init__(self) - self.daemon = True - self.jobs = Queue() - self.job_manager = job_manager - self._run = True + def __init__(self): self.calculate_rate_limit() - self.last_send_time = time.time() - self.rate_limit def calculate_rate_limit(self): @@ -87,70 +38,28 @@ class Emailer(Thread): # {{{ 'gmail.com' in rh or 'live.com' in rh): self.rate_limit = tweaks['public_smtp_relay_delay'] - def stop(self): - self._run = False - self.jobs.put(None) + def __call__(self, attachment, aname, to, subject, text, log=None, + abort=None, notifications=None): - def run(self): - while self._run: + try_count = 0 + while try_count <= self.MAX_RETRIES: + if try_count > 0: + log('\nRetrying in %d seconds...\n' % + self.rate_limit) try: - job = self.jobs.get() + self.sendmail(attachment, aname, to, subject, text, log) + try_count = self.MAX_RETRIES + log('Email successfully sent') except: - break - if job is None or not self._run: - break - try_count = 0 - failed, exc = False, None - job.start_work() - if job.kill_on_start: - job.log_write('Aborted\n') - job.failed = failed - job.killed = True - job.job_done() - continue + if abort.is_set(): + return + if try_count == self.MAX_RETRIES: + raise + log.exception('\nSending failed...\n') - while try_count <= self.MAX_RETRIES: - failed = False - if try_count > 0: - job.log_write('\nRetrying in %d seconds...\n' % - self.rate_limit) - try: - self.sendmail(job) - break - except Exception as e: - if not self._run: - return - import traceback - failed = True - exc = e - job.log_write('\nSending failed...\n') - job.log_write(traceback.format_exc()) + try_count += 1 - try_count += 1 - - if not self._run: - break - - job.failed = failed - job.exception = exc - job.job_done() - try: - job.email_sent_callback(job) - except: - import traceback - traceback.print_exc() - - def send_mails(self, jobnames, callback, attachments, to_s, subjects, - texts, attachment_names): - for name, attachment, to, subject, text, aname in zip(jobnames, - attachments, to_s, subjects, texts, attachment_names): - description = _('Email %s to %s') % (name, to) - job = EmailJob(callback, description, attachment, aname, to, - subject, text, self.job_manager) - self.job_manager.add_job(job) - self.jobs.put(job) - - def sendmail(self, job): + def sendmail(self, attachment, aname, to, subject, text, log): while time.time() - self.last_send_time <= self.rate_limit: time.sleep(1) try: @@ -158,7 +67,6 @@ class Emailer(Thread): # {{{ from_ = opts.from_ if not from_: from_ = 'calibre ' - attachment, aname, to, subject, text = job.email_args msg = compose_mail(from_, to, text, subject, open(attachment, 'rb'), aname) efrom, eto = map(extract_email_address, (from_, to)) @@ -169,48 +77,56 @@ class Emailer(Thread): # {{{ username=opts.relay_username, password=unhexlify(opts.relay_password), port=opts.relay_port, encryption=opts.encryption, - debug_output=partial(print, file=job._log_file)) + debug_output=log.debug) finally: self.last_send_time = time.time() - def email_news(self, mi, remove, get_fmts, done): - opts = email_config().parse() - accounts = [(account, [x.strip().lower() for x in x[0].split(',')]) - for account, x in opts.accounts.items() if x[1]] - sent_mails = [] - for i, x in enumerate(accounts): - account, fmts = x - files = get_fmts(fmts) - files = [f for f in files if f is not None] - if not files: - continue - attachment = files[0] - to_s = [account] - subjects = [_('News:')+' '+mi.title] - texts = [ - _('Attached is the %s periodical downloaded by calibre.') - % (mi.title,) - ] - attachment_names = [ascii_filename(mi.title)+os.path.splitext(attachment)[1]] - attachments = [attachment] - jobnames = [mi.title] - do_remove = [] - if i == len(accounts) - 1: - do_remove = remove - self.send_mails(jobnames, - Dispatcher(partial(done, remove=do_remove)), - attachments, to_s, subjects, texts, attachment_names) - sent_mails.append(to_s[0]) - return sent_mails +gui_sendmail = Sendmail() -# }}} +def send_mails(jobnames, callback, attachments, to_s, subjects, + texts, attachment_names, job_manager): + for name, attachment, to, subject, text, aname in zip(jobnames, + attachments, to_s, subjects, texts, attachment_names): + description = _('Email %s to %s') % (name, to) + job = ThreadedJob('email', description, gui_sendmail, (attachment, aname, to, + subject, text), {}, callback, killable=False) + job_manager.run_threaded_job(job) + + +def email_news(mi, remove, get_fmts, done, job_manager): + opts = email_config().parse() + accounts = [(account, [x.strip().lower() for x in x[0].split(',')]) + for account, x in opts.accounts.items() if x[1]] + sent_mails = [] + for i, x in enumerate(accounts): + account, fmts = x + files = get_fmts(fmts) + files = [f for f in files if f is not None] + if not files: + continue + attachment = files[0] + to_s = [account] + subjects = [_('News:')+' '+mi.title] + texts = [ + _('Attached is the %s periodical downloaded by calibre.') + % (mi.title,) + ] + attachment_names = [ascii_filename(mi.title)+os.path.splitext(attachment)[1]] + attachments = [attachment] + jobnames = [mi.title] + do_remove = [] + if i == len(accounts) - 1: + do_remove = remove + send_mails(jobnames, + Dispatcher(partial(done, remove=do_remove)), + attachments, to_s, subjects, texts, attachment_names, + job_manager) + sent_mails.append(to_s[0]) + return sent_mails class EmailMixin(object): # {{{ - def __init__(self): - self.emailer = Emailer(self.job_manager) - def send_by_mail(self, to, fmts, delete_from_library, subject='', send_ids=None, do_auto_convert=True, specific_format=None): ids = [self.library_view.model().id(r) for r in self.library_view.selectionModel().selectedRows()] if send_ids is None else send_ids @@ -246,8 +162,7 @@ class EmailMixin(object): # {{{ components = get_components(subject, mi, id) if not components: components = [mi.title] - subject = os.path.join(*components) - subjects.append(subject) + subjects.append(os.path.join(*components)) a = authors_to_string(mi.authors if mi.authors else \ [_('Unknown')]) texts.append(_('Attached, you will find the e-book') + \ @@ -262,11 +177,10 @@ class EmailMixin(object): # {{{ to_s = list(repeat(to, len(attachments))) if attachments: - if not self.emailer.is_alive(): - self.emailer.start() - self.emailer.send_mails(jobnames, + send_mails(jobnames, Dispatcher(partial(self.email_sent, remove=remove)), - attachments, to_s, subjects, texts, attachment_names) + attachments, to_s, subjects, texts, attachment_names, + self.job_manager) self.status_bar.show_message(_('Sending email to')+' '+to, 3000) auto = [] @@ -334,10 +248,8 @@ class EmailMixin(object): # {{{ files, auto = self.library_view.model().\ get_preferred_formats_from_ids([id_], fmts) return files - if not self.emailer.is_alive(): - self.emailer.start() - sent_mails = self.emailer.email_news(mi, remove, - get_fmts, self.email_sent) + sent_mails = email_news(mi, remove, + get_fmts, self.email_sent, self.job_manager) if sent_mails: self.status_bar.show_message(_('Sent news to')+' '+\ ', '.join(sent_mails), 3000) diff --git a/src/calibre/gui2/jobs.py b/src/calibre/gui2/jobs.py index dbde030e81..34eef4406a 100644 --- a/src/calibre/gui2/jobs.py +++ b/src/calibre/gui2/jobs.py @@ -8,14 +8,13 @@ Job management. ''' import re - from Queue import Empty, Queue -from PyQt4.Qt import QAbstractTableModel, QVariant, QModelIndex, Qt, \ - QTimer, pyqtSignal, QIcon, QDialog, QAbstractItemDelegate, QApplication, \ - QSize, QStyleOptionProgressBarV2, QString, QStyle, QToolTip, QFrame, \ - QHBoxLayout, QVBoxLayout, QSizePolicy, QLabel, QCoreApplication, QAction, \ - QByteArray +from PyQt4.Qt import (QAbstractTableModel, QVariant, QModelIndex, Qt, + QTimer, pyqtSignal, QIcon, QDialog, QAbstractItemDelegate, QApplication, + QSize, QStyleOptionProgressBarV2, QString, QStyle, QToolTip, QFrame, + QHBoxLayout, QVBoxLayout, QSizePolicy, QLabel, QCoreApplication, QAction, + QByteArray) from calibre.utils.ipc.server import Server from calibre.utils.ipc.job import ParallelJob @@ -25,8 +24,9 @@ from calibre.gui2.dialogs.jobs_ui import Ui_JobsDialog from calibre import __appname__ from calibre.gui2.dialogs.job_view_ui import Ui_Dialog from calibre.gui2.progress_indicator import ProgressIndicator +from calibre.gui2.threaded_jobs import ThreadedJobServer, ThreadedJob -class JobManager(QAbstractTableModel): +class JobManager(QAbstractTableModel): # {{{ job_added = pyqtSignal(int) job_done = pyqtSignal(int) @@ -42,6 +42,7 @@ class JobManager(QAbstractTableModel): self.add_job = Dispatcher(self._add_job) self.server = Server(limit=int(config['worker_limit']/2.0), enforce_cpu_limit=config['enforce_cpu_limit']) + self.threaded_server = ThreadedJobServer() self.changed_queue = Queue() self.timer = QTimer(self) @@ -146,12 +147,21 @@ class JobManager(QAbstractTableModel): jobs.add(self.server.changed_jobs_queue.get_nowait()) except Empty: break + + # Update device jobs while True: try: jobs.add(self.changed_queue.get_nowait()) except Empty: break + # Update threaded jobs + while True: + try: + jobs.add(self.threaded_server.changed_jobs.get_nowait()) + except Empty: + break + if jobs: needs_reset = False for job in jobs: @@ -207,11 +217,22 @@ class JobManager(QAbstractTableModel): self.server.add_job(job) return job + def run_threaded_job(self, job): + self.add_job(job) + self.threaded_server.add_job(job) + def launch_gui_app(self, name, args=[], kwargs={}, description=''): job = ParallelJob(name, description, lambda x: x, args=args, kwargs=kwargs) self.server.run_job(job, gui=True, redirect_output=False) + def _kill_job(self, job): + if isinstance(job, ParallelJob): + self.server.kill_job(job) + elif isinstance(job, ThreadedJob): + self.threaded_server.kill_job(job) + else: + job.kill_on_start = True def kill_job(self, row, view): job = self.jobs[row] @@ -221,29 +242,29 @@ class JobManager(QAbstractTableModel): if job.duration is not None: return error_dialog(view, _('Cannot kill job'), _('Job has already run')).exec_() - if isinstance(job, ParallelJob): - self.server.kill_job(job) - else: - job.kill_on_start = True + if not getattr(job, 'killable', True): + return error_dialog(view, _('Cannot kill job'), + _('This job cannot be stopped'), show=True) + self._kill_job(job) def kill_all_jobs(self): for job in self.jobs: - if isinstance(job, DeviceJob) or job.duration is not None: + if (isinstance(job, DeviceJob) or job.duration is not None or + not getattr(job, 'killable', True)): continue - if isinstance(job, ParallelJob): - self.server.kill_job(job) - else: - job.kill_on_start = True + self._kill_job(job) def terminate_all_jobs(self): self.server.killall() for job in self.jobs: - if isinstance(job, DeviceJob) or job.duration is not None: + if (isinstance(job, DeviceJob) or job.duration is not None or + not getattr(job, 'killable', True)): continue if not isinstance(job, ParallelJob): - job.kill_on_start = True - + self._kill_job(job) +# }}} +# Jobs UI {{{ class ProgressBarDelegate(QAbstractItemDelegate): def sizeHint(self, option, index): @@ -269,6 +290,11 @@ class DetailView(QDialog, Ui_Dialog): self.setupUi(self) self.setWindowTitle(job.description) self.job = job + self.html_view = hasattr(job, 'html_details') + if self.html_view: + self.log.setVisible(False) + else: + self.tb.setVisible(False) self.next_pos = 0 self.update() self.timer = QTimer(self) @@ -277,12 +303,19 @@ class DetailView(QDialog, Ui_Dialog): def update(self): - f = self.job.log_file - f.seek(self.next_pos) - more = f.read() - self.next_pos = f.tell() - if more: - self.log.appendPlainText(more.decode('utf-8', 'replace')) + if self.html_view: + html = self.job.html_details + if len(html) > self.next_pos: + self.next_pos = len(html) + self.tb.setHtml( + '
%s
'%html) + else: + f = self.job.log_file + f.seek(self.next_pos) + more = f.read() + self.next_pos = f.tell() + if more: + self.log.appendPlainText(more.decode('utf-8', 'replace')) class JobsButton(QFrame): @@ -441,3 +474,5 @@ class JobsDialog(QDialog, Ui_JobsDialog): def hide(self, *args): self.save_state() return QDialog.hide(self, *args) +# }}} + diff --git a/src/calibre/gui2/metadata/bulk_download2.py b/src/calibre/gui2/metadata/bulk_download2.py new file mode 100644 index 0000000000..cc6da1e995 --- /dev/null +++ b/src/calibre/gui2/metadata/bulk_download2.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + + + diff --git a/src/calibre/gui2/threaded_jobs.py b/src/calibre/gui2/threaded_jobs.py new file mode 100644 index 0000000000..f29baf4134 --- /dev/null +++ b/src/calibre/gui2/threaded_jobs.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import os, time, tempfile, json +from threading import Thread, RLock, Event +from Queue import Queue + +from calibre.utils.ipc.job import BaseJob +from calibre.utils.logging import GUILog +from calibre.ptempfile import base_dir + +class ThreadedJob(BaseJob): + + def __init__(self, + type_, description, + + func, args, kwargs, + + callback, + + max_concurrent_count=1, + killable=True, + log=None): + ''' + A job that is run in its own thread in the calibre main process + + :param type_: The type of this job (a string). The type is used in + conjunction with max_concurrent_count to prevent too many jobs of the + same type from running + + :param description: A user viewable job description + + :func: The function that actually does the work. This function *must* + accept at least three keyword arguments: abort, log and notifications. abort is + An Event object. func should periodically check abort.is_set(0 and if + it is True, it should stop processing as soon as possible. notifications + is a Queue. func should put progress notifications into it in the form + of a tuple (frac, msg). frac is a number between 0 and 1 indicating + progress and msg is a string describing the progress. log is a Log + object which func should use for all debugging output. func should + raise an Exception to indicate failure. This exception is stored in + job.exception and can thus be used to pass arbitrary information to + callback. + + :param args,kwargs: These are passed to func when it is called + + :param callback: A callable that is called on completion of this job. + Note that it is not called if the user kills the job. Check job.failed + to see if the job succeeded or not. And use job.log to get the job log. + + :param killable: If False the GUI wont let the user kill this job + + :param log: Must be a subclass of GUILog or None. If None a default + GUILog is created. + ''' + BaseJob.__init__(self, description) + + self.type = type_ + self.max_concurrent_count = max_concurrent_count + self.killable = killable + self.callback = callback + self.abort = Event() + self.exception = None + + kwargs['notifications'] = self.notifications + kwargs['abort'] = self.abort + self.log = GUILog() if log is None else log + kwargs['log'] = self.log + + self.func, self.args, self.kwargs = func, args, kwargs + self.consolidated_log = None + + def start_work(self): + self.start_time = time.time() + self.log('Starting job:', self.description) + try: + self.result = self.func(*self.args, **self.kwargs) + except Exception as e: + self.exception = e + self.failed = True + self.log.exception('Job: "%s" failed with error:'%self.description) + self.log.debug('Called with args:', self.args, self.kwargs) + + self.duration = time.time() - self.start_time + try: + self.callback(self) + except: + pass + self._cleanup() + + def _cleanup(self): + + try: + self.consolidate_log() + except: + self.log.exception('Log consolidation failed') + + # No need to keep references to these around anymore + self.func = self.args = self.kwargs = self.notifications = None + + def kill(self): + if self.start_time is None: + self.start_time = time.time() + self.duration = 0.0001 + else: + self.duration = time.time() - self.start_time() + self.abort.set() + + self.log('Aborted job:', self.description) + self.killed = True + self.failed = True + self._cleanup() + + def consolidate_log(self): + logs = [self.log.html, self.log.plain_text] + bdir = base_dir() + log_dir = os.path.join(bdir, 'threaded_job_logs') + if not os.path.exists(log_dir): + os.makedirs(log_dir) + fd, path = tempfile.mkstemp(suffix='.json', prefix='log-', dir=log_dir) + with os.fdopen(fd, 'wb') as f: + f.write(json.dumps(logs, ensure_ascii=False, + indent=2).encode('utf-8')) + self.consolidated_log = path + self.log = None + + def read_consolidated_log(self): + with open(self.consolidated_log, 'rb') as f: + return json.loads(f.read().decode('utf-8')) + + @property + def details(self): + if self.consolidated_log is None: + return self.log.plain_text + return self.read_consolidated_log()[1] + + @property + def html_details(self): + if self.consolidated_log is None: + return self.log.html + return self.read_consolidated_log()[0] + +class ThreadedJobWorker(Thread): + + def __init__(self, job): + Thread.__init__(self) + self.daemon = True + self.job = job + + def run(self): + try: + self.job.start_work() + except: + import traceback + from calibre import prints + prints('Job had unhandled exception:', self.job.description) + traceback.print_exc() + +class ThreadedJobServer(Thread): + + def __init__(self): + Thread.__init__(self) + self.daemon = True + self.lock = RLock() + + self.queued_jobs = [] + self.running_jobs = set() + self.changed_jobs = Queue() + self.keep_going = True + + def close(self): + self.keep_going = False + + def add_job(self, job): + with self.lock: + self.queued_jobs.append(job) + + if not self.is_alive(): + self.start() + + def run(self): + while self.keep_going: + self.run_once() + time.sleep(0.1) + + def run_once(self): + with self.lock: + remove = set() + for worker in self.running_jobs: + if worker.is_alive(): + # Get progress notifications + if worker.job.consume_notifications(): + self.changed_jobs.put(worker.job) + else: + remove.add(worker) + self.changed_jobs.put(worker.job) + + for worker in remove: + self.running_jobs.remove(worker) + + jobs = self.get_startable_jobs() + for job in jobs: + w = ThreadedJobWorker(job) + w.start() + self.running_jobs.add(w) + self.changed_jobs.put(job) + self.queued_jobs.remove(job) + + def kill_job(self, job): + with self.lock: + if job in self.queued_jobs: + self.queued_jobs.remove(job) + elif job in self.running_jobs: + self.running_jobs.remove(job) + job.kill() + self.changed_jobs.put(job) + + def running_jobs_of_type(self, type_): + return len([w for w in self.running_jobs if w.job.type == type_]) + + def get_startable_jobs(self): + queued_types = [] + ans = [] + for job in self.queued_jobs: + num = self.running_jobs_of_type(job.type) + num += queued_types.count(job.type) + if num < job.max_concurrent_count: + queued_types.append(job.type) + ans.append(job) + return ans + + diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 4d363c283a..e7853b9491 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -608,6 +608,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ self.update_checker.terminate() self.listener.close() self.job_manager.server.close() + self.job_manager.threaded_server.close() while self.spare_servers: self.spare_servers.pop().close() self.device_manager.keep_going = False @@ -616,8 +617,6 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ mb.stop() self.hide_windows() - if self.emailer.is_alive(): - self.emailer.stop() try: try: if self.content_server is not None: diff --git a/src/calibre/utils/ipc/job.py b/src/calibre/utils/ipc/job.py index 91db333791..f4b54aee95 100644 --- a/src/calibre/utils/ipc/job.py +++ b/src/calibre/utils/ipc/job.py @@ -75,12 +75,20 @@ class BaseJob(object): self._run_state = self.RUNNING self._status_text = _('Working...') - while consume_notifications: + if consume_notifications: + return self.consume_notifications() + return False + + def consume_notifications(self): + got_notification = False + while self.notifications is not None: try: self.percent, self._message = self.notifications.get_nowait() self.percent *= 100. + got_notification = True except Empty: break + return got_notification @property def status_text(self): From d7438cbc4922469888cb2579c2dd2cac2adad9a9 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 11 Apr 2011 17:19:38 -0600 Subject: [PATCH 08/36] ... --- src/calibre/gui2/actions/edit_metadata.py | 7 +++- src/calibre/gui2/metadata/bulk_download2.py | 39 +++++++++++++++++++++ src/calibre/gui2/threaded_jobs.py | 5 ++- 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/actions/edit_metadata.py b/src/calibre/gui2/actions/edit_metadata.py index fc663d268a..09040bcafc 100644 --- a/src/calibre/gui2/actions/edit_metadata.py +++ b/src/calibre/gui2/actions/edit_metadata.py @@ -10,7 +10,7 @@ from functools import partial from PyQt4.Qt import Qt, QMenu, QModelIndex -from calibre.gui2 import error_dialog, config +from calibre.gui2 import error_dialog, config, Dispatcher from calibre.gui2.dialogs.metadata_single import MetadataSingleDialog from calibre.gui2.dialogs.metadata_bulk import MetadataBulkDialog from calibre.gui2.dialogs.confirm_delete import confirm @@ -88,6 +88,11 @@ class EditMetadataAction(InterfaceAction): _('No books selected'), show=True) db = self.gui.library_view.model().db ids = [db.id(row.row()) for row in rows] + from calibre.gui2.metadata.bulk_download2 import start_download + start_download(self.gui, ids, Dispatcher(self.bulk_metadata_downloaded)) + + def bulk_metadata_downloaded(self, job): + print repr(job.result) def download_metadata_old(self, checked, covers=True, set_metadata=True, set_social_metadata=None): diff --git a/src/calibre/gui2/metadata/bulk_download2.py b/src/calibre/gui2/metadata/bulk_download2.py index cc6da1e995..d691c651d9 100644 --- a/src/calibre/gui2/metadata/bulk_download2.py +++ b/src/calibre/gui2/metadata/bulk_download2.py @@ -7,5 +7,44 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' +from functools import partial + +from PyQt4.Qt import QIcon + +from calibre.gui2.dialogs.message_box import MessageBox +from calibre.gui2.threaded_jobs import ThreadedJob + +def show_config(gui, parent): + from calibre.gui2.preferences import show_config_widget + show_config_widget('Sharing', 'Metadata download', parent=parent, + gui=gui, never_shutdown=True) + +def start_download(gui, ids, callback): + q = MessageBox(MessageBox.QUESTION, _('Schedule download?'), + _('The download of metadata for %d book(s) will' + ' run in the background. Proceed?')%len(ids), + show_copy_button=False, parent=gui) + b = q.bb.addButton(_('Configure download'), q.bb.ActionRole) + b.setIcon(QIcon(I('config.png'))) + b.clicked.connect(partial(show_config, gui, q)) + q.det_msg_toggle.setVisible(False) + + ret = q.exec_() + b.clicked.disconnect() + if ret != q.Accepted: + return + + job = ThreadedJob('metadata bulk download', + _('Download metadata for %d books')%len(ids), + download, (ids, gui.current_db), {}, callback) + gui.job_manager.run_threaded_job(job) + + +def download(ids, db, log=None, abort=None, notifications=None): + ids = list(ids) + metadata = [db.get_metadata(i, index_is_id=True, get_user_categories=False) + for i in ids] + return (ids, [mi.last_modified for mi in metadata]) + diff --git a/src/calibre/gui2/threaded_jobs.py b/src/calibre/gui2/threaded_jobs.py index f29baf4134..f98488da79 100644 --- a/src/calibre/gui2/threaded_jobs.py +++ b/src/calibre/gui2/threaded_jobs.py @@ -91,7 +91,8 @@ class ThreadedJob(BaseJob): try: self.callback(self) except: - pass + import traceback + traceback.print_exc() self._cleanup() def _cleanup(self): @@ -103,6 +104,8 @@ class ThreadedJob(BaseJob): # No need to keep references to these around anymore self.func = self.args = self.kwargs = self.notifications = None + # We can't delete self.callback as it might be a Dispatch object and if + # it is garbage collected it won't work def kill(self): if self.start_time is None: From c21095c457bdc30bc98c18aa410daeb4ef3bee03 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 11 Apr 2011 21:42:25 -0600 Subject: [PATCH 09/36] End of day --- src/calibre/gui2/actions/edit_metadata.py | 6 +- src/calibre/gui2/metadata/bulk_download2.py | 78 +++++++++++++++++++-- 2 files changed, 79 insertions(+), 5 deletions(-) diff --git a/src/calibre/gui2/actions/edit_metadata.py b/src/calibre/gui2/actions/edit_metadata.py index 09040bcafc..f5e9e8c4a0 100644 --- a/src/calibre/gui2/actions/edit_metadata.py +++ b/src/calibre/gui2/actions/edit_metadata.py @@ -92,7 +92,11 @@ class EditMetadataAction(InterfaceAction): start_download(self.gui, ids, Dispatcher(self.bulk_metadata_downloaded)) def bulk_metadata_downloaded(self, job): - print repr(job.result) + if job.failed: + self.job_exception(job, dialog_title=_('Failed to download metadata')) + return + from calibre.gui2.metadata.bulk_download2 import proceed + proceed(self.gui, job) def download_metadata_old(self, checked, covers=True, set_metadata=True, set_social_metadata=None): diff --git a/src/calibre/gui2/metadata/bulk_download2.py b/src/calibre/gui2/metadata/bulk_download2.py index d691c651d9..cb7f1686f6 100644 --- a/src/calibre/gui2/metadata/bulk_download2.py +++ b/src/calibre/gui2/metadata/bulk_download2.py @@ -8,8 +8,10 @@ __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' from functools import partial +from itertools import izip -from PyQt4.Qt import QIcon +from PyQt4.Qt import (QIcon, QDialog, QVBoxLayout, QTextBrowser, + QDialogButtonBox, QApplication) from calibre.gui2.dialogs.message_box import MessageBox from calibre.gui2.threaded_jobs import ThreadedJob @@ -21,8 +23,13 @@ def show_config(gui, parent): def start_download(gui, ids, callback): q = MessageBox(MessageBox.QUESTION, _('Schedule download?'), - _('The download of metadata for %d book(s) will' - ' run in the background. Proceed?')%len(ids), + '

'+_('The download of metadata for the %d selected book(s) will' + ' run in the background. Proceed?')%len(ids) + + '

'+_('You can monitor the progress of the download ' + 'by clicking the rotating spinner in the bottom right ' + 'corner.') + + '

'+_('When the download completes you will be asked for' + ' confirmation before calibre applies the downloaded metadata.'), show_copy_button=False, parent=gui) b = q.bb.addButton(_('Configure download'), q.bb.ActionRole) b.setIcon(QIcon(I('config.png'))) @@ -39,12 +46,75 @@ def start_download(gui, ids, callback): download, (ids, gui.current_db), {}, callback) gui.job_manager.run_threaded_job(job) +class ViewLog(QDialog): + + def __init__(self, html, parent=None): + QDialog.__init__(self, parent) + self.l = l = QVBoxLayout() + self.setLayout(l) + + self.tb = QTextBrowser(self) + self.tb.setHtml('

%s
' % html) + l.addWidget(self.tb) + + self.bb = QDialogButtonBox(QDialogButtonBox.Ok) + self.bb.accepted.connect(self.accept) + self.bb.rejected.connect(self.reject) + self.copy_button = self.bb.addButton(_('Copy to clipboard'), + self.bb.ActionRole) + self.copy_button.setIcon(QIcon(I('edit-copy.png'))) + self.copy_button.clicked.connect(self.copy_to_clipboard) + self.setModal(False) + self.resize(self.sizeHint()) + self.show() + + def copy_to_clipboard(self): + txt = self.tb.toPlainText() + QApplication.clipboard().setText(txt) + +_vl = None +def view_log(job, parent): + global _vl + _vl = ViewLog(job.html_details, parent) + +def apply(job, gui, q): + q.vlb.clicked.disconnect() + q.finished.diconnect() + id_map, failed_ids = job.result + print (id_map) + +def proceed(gui, job): + id_map, failed_ids = job.result + fmsg = det_msg = '' + if failed_ids: + fmsg = _('Could not download metadata for %d of the books. Click' + ' "Show details" to see which books.')%len(failed_ids) + det_msg = '\n'.join([id_map[i].title for i in failed_ids]) + msg = '

' + _('Finished downloading metadata for %d books. ' + 'Proceed with updating the metadata in your library?')%len(id_map) + q = MessageBox(MessageBox.QUESTION, _('Download complete'), + msg + fmsg, det_msg=det_msg, show_copy_button=bool(failed_ids), + parent=gui) + q.vlb = q.bb.addButton(_('View log'), q.bb.ActionRole) + q.vlb.setIcon(QIcon(I('debug.png'))) + q.vlb.clicked.connect(partial(view_log, job, q)) + q.det_msg_toggle.setVisible(bool(failed_ids)) + q.setModal(False) + q.show() + q.finished.connect(partial(job, gui, q)) + def download(ids, db, log=None, abort=None, notifications=None): + log('Starting metadata download for %d books'%len(ids)) ids = list(ids) metadata = [db.get_metadata(i, index_is_id=True, get_user_categories=False) for i in ids] - return (ids, [mi.last_modified for mi in metadata]) + failed_ids = set() + ans = {} + for i, mi in izip(ids, metadata): + ans[i] = mi + log('Download complete, with %d failures'%len(failed_ids)) + return (ans, failed_ids) From f1ad415fac721d16255c24e763492e8f713ac59d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 12 Apr 2011 08:12:45 -0600 Subject: [PATCH 10/36] MOBI Output: The Ignore margins setting no longer ignores blockquotes, only margins set via CSS on other elements. Fixes #758675 (Conversion to mobi with the 'ignore margins' option deletes existing blockquotes) --- src/calibre/ebooks/mobi/mobiml.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/calibre/ebooks/mobi/mobiml.py b/src/calibre/ebooks/mobi/mobiml.py index 40ad5e9e78..1e626cf916 100644 --- a/src/calibre/ebooks/mobi/mobiml.py +++ b/src/calibre/ebooks/mobi/mobiml.py @@ -495,6 +495,10 @@ class MobiMLizer(object): vtag.append(child) return + if tag == 'blockquote': + old_mim = self.opts.mobi_ignore_margins + self.opts.mobi_ignore_margins = False + if text or tag in CONTENT_TAGS or tag in NESTABLE_TAGS: self.mobimlize_content(tag, text, bstate, istates) for child in elem: @@ -510,6 +514,8 @@ class MobiMLizer(object): if tail: self.mobimlize_content(tag, tail, bstate, istates) + if tag == 'blockquote': + self.opts.mobi_ignore_margins = old_mim if bstate.content and style['page-break-after'] in PAGE_BREAKS: bstate.pbreak = True From 2d9625f5b2d2e9117554cd0b0fbc8e099afb6875 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 12 Apr 2011 09:48:31 -0600 Subject: [PATCH 11/36] ... --- src/calibre/library/database2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 0b1182c0bf..50b404b4be 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -1781,7 +1781,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): path_changed = True if set_authors: if not mi.authors: - mi.authors = [_('Unknown')] + mi.authors = [_('Unknown')] authors = [] for a in mi.authors: authors += string_to_authors(a) From a404e9827d9efdd7d34a6a11f070458845fcb5b6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 12 Apr 2011 11:04:01 -0600 Subject: [PATCH 12/36] Bulk download UI --- src/calibre/gui2/actions/edit_metadata.py | 3 +- src/calibre/gui2/metadata/bulk_download2.py | 147 ++++++++++++++++++-- 2 files changed, 136 insertions(+), 14 deletions(-) diff --git a/src/calibre/gui2/actions/edit_metadata.py b/src/calibre/gui2/actions/edit_metadata.py index f5e9e8c4a0..9f2cacb177 100644 --- a/src/calibre/gui2/actions/edit_metadata.py +++ b/src/calibre/gui2/actions/edit_metadata.py @@ -89,7 +89,8 @@ class EditMetadataAction(InterfaceAction): db = self.gui.library_view.model().db ids = [db.id(row.row()) for row in rows] from calibre.gui2.metadata.bulk_download2 import start_download - start_download(self.gui, ids, Dispatcher(self.bulk_metadata_downloaded)) + start_download(self.gui, ids, + Dispatcher(self.bulk_metadata_downloaded), identify, covers) def bulk_metadata_downloaded(self, job): if job.failed: diff --git a/src/calibre/gui2/metadata/bulk_download2.py b/src/calibre/gui2/metadata/bulk_download2.py index cb7f1686f6..add689e616 100644 --- a/src/calibre/gui2/metadata/bulk_download2.py +++ b/src/calibre/gui2/metadata/bulk_download2.py @@ -10,18 +10,21 @@ __docformat__ = 'restructuredtext en' from functools import partial from itertools import izip -from PyQt4.Qt import (QIcon, QDialog, QVBoxLayout, QTextBrowser, - QDialogButtonBox, QApplication) +from PyQt4.Qt import (QIcon, QDialog, QVBoxLayout, QTextBrowser, QSize, + QDialogButtonBox, QApplication, QTimer, QLabel, QProgressBar) from calibre.gui2.dialogs.message_box import MessageBox from calibre.gui2.threaded_jobs import ThreadedJob +from calibre.utils.icu import lower +from calibre.ebooks.metadata import authors_to_string +from calibre.gui2 import question_dialog, error_dialog def show_config(gui, parent): from calibre.gui2.preferences import show_config_widget show_config_widget('Sharing', 'Metadata download', parent=parent, gui=gui, never_shutdown=True) -def start_download(gui, ids, callback): +def start_download(gui, ids, callback, identify, covers): q = MessageBox(MessageBox.QUESTION, _('Schedule download?'), '

'+_('The download of metadata for the %d selected book(s) will' ' run in the background. Proceed?')%len(ids) + @@ -43,10 +46,10 @@ def start_download(gui, ids, callback): job = ThreadedJob('metadata bulk download', _('Download metadata for %d books')%len(ids), - download, (ids, gui.current_db), {}, callback) + download, (ids, gui.current_db, identify, covers), {}, callback) gui.job_manager.run_threaded_job(job) -class ViewLog(QDialog): +class ViewLog(QDialog): # {{{ def __init__(self, html, parent=None): QDialog.__init__(self, parent) @@ -64,8 +67,11 @@ class ViewLog(QDialog): self.bb.ActionRole) self.copy_button.setIcon(QIcon(I('edit-copy.png'))) self.copy_button.clicked.connect(self.copy_to_clipboard) + l.addWidget(self.bb) self.setModal(False) - self.resize(self.sizeHint()) + self.resize(QSize(500, 400)) + self.setWindowTitle(_('Download log')) + self.setWindowIcon(QIcon(I('debug.png'))) self.show() def copy_to_clipboard(self): @@ -77,11 +83,118 @@ def view_log(job, parent): global _vl _vl = ViewLog(job.html_details, parent) -def apply(job, gui, q): +# }}} + +class ApplyDialog(QDialog): + + def __init__(self, id_map, gui): + QDialog.__init__(self, gui) + + self.l = l = QVBoxLayout() + self.setLayout(l) + l.addWidget(QLabel(_('Applying downloaded metadata to your library'))) + + self.pb = QProgressBar(self) + l.addWidget(self.pb) + self.pb.setMinimum(0) + self.pb.setMaximum(len(id_map)) + + self.bb = QDialogButtonBox(QDialogButtonBox.Cancel) + self.bb.rejected.connect(self.reject) + self.bb.accepted.connect(self.accept) + l.addWidget(self.bb) + + self.db = gui.current_db + self.id_map = list(id_map.iteritems()) + self.current_idx = 0 + + self.failures = [] + self.canceled = False + + QTimer.singleShot(20, self.do_one) + self.exec_() + + def do_one(self): + if self.canceled: + return + i, mi = self.id_map[self.current_idx] + try: + set_title = not mi.is_null('title') + set_authors = not mi.is_null('authors') + self.db.set_metadata(i, mi, commit=False, set_title=set_title, + set_authors=set_authors) + except: + import traceback + self.failures.append((i, traceback.format_exc())) + + self.pb.setValue(self.pb.value()+1) + + if self.current_idx >= len(self.id_map) - 1: + self.finalize() + else: + self.current_idx += 1 + QTimer.singleShot(20, self.do_one) + + def reject(self): + self.canceled = True + QDialog.reject(self) + + def finalize(self): + if self.canceled: + return + if self.failures: + msg = [] + for i, tb in self.failures: + title = self.db.title(i, index_is_id=True) + authors = self.db.authors(i, index_is_id=True) + if authors: + authors = [x.replace('|', ',') for x in authors.split(',')] + title += ' - ' + authors_to_string(authors) + msg.append(title+'\n\n'+tb+'\n'+('*'*80)) + + error_dialog(self, _('Some failures'), + _('Failed to apply updated metadata for some books' + ' in your library. Click "Show Details" to see ' + 'details.'), det_msg='\n\n'.join(msg), show=True) + self.accept() + +_amd = None +def apply_metadata(job, gui, q, result): + global _amd q.vlb.clicked.disconnect() - q.finished.diconnect() + q.finished.disconnect() + if result != q.Accepted: + return id_map, failed_ids = job.result - print (id_map) + id_map = dict([(k, v) for k, v in id_map.iteritems() if k not in + failed_ids]) + if not id_map: + return + + modified = set() + db = gui.current_db + + for i, mi in id_map.iteritems(): + lm = db.metadata_last_modified(i, index_is_id=True) + if lm > mi.last_modified: + title = db.title(i, index_is_id=True) + authors = db.authors(i, index_is_id=True) + if authors: + authors = [x.replace('|', ',') for x in authors.split(',')] + title += ' - ' + authors_to_string(authors) + modified.add(title) + + if modified: + modified = sorted(modified, key=lower) + if not question_dialog(gui, _('Some books changed'), '

'+ + _('The metadata for some books in your library has' + ' changed since you started the download. If you' + ' proceed, some of those changes may be overwritten. ' + 'Click "Show details" to see the list of changed books. ' + 'Do you want to proceed?'), det_msg='\n'.join(modified)): + return + + _amd = ApplyDialog(id_map, gui) def proceed(gui, job): id_map, failed_ids = job.result @@ -90,7 +203,7 @@ def proceed(gui, job): fmsg = _('Could not download metadata for %d of the books. Click' ' "Show details" to see which books.')%len(failed_ids) det_msg = '\n'.join([id_map[i].title for i in failed_ids]) - msg = '

' + _('Finished downloading metadata for %d books. ' + msg = '

' + _('Finished downloading metadata for %d book(s). ' 'Proceed with updating the metadata in your library?')%len(id_map) q = MessageBox(MessageBox.QUESTION, _('Download complete'), msg + fmsg, det_msg=det_msg, show_copy_button=bool(failed_ids), @@ -101,18 +214,26 @@ def proceed(gui, job): q.det_msg_toggle.setVisible(bool(failed_ids)) q.setModal(False) q.show() - q.finished.connect(partial(job, gui, q)) + q.finished.connect(partial(apply_metadata, job, gui, q)) -def download(ids, db, log=None, abort=None, notifications=None): - log('Starting metadata download for %d books'%len(ids)) +def download(ids, db, identify, covers, + log=None, abort=None, notifications=None): ids = list(ids) metadata = [db.get_metadata(i, index_is_id=True, get_user_categories=False) for i in ids] failed_ids = set() ans = {} + count = 0 for i, mi in izip(ids, metadata): + if abort.is_set(): + log.error('Aborting...') + break + # TODO: Apply ignore_fields and set unchanged values to null values ans[i] = mi + count += 1 + notifications.put((count/len(ids), + _('Downloaded %d of %d')%(count, len(ids)))) log('Download complete, with %d failures'%len(failed_ids)) return (ans, failed_ids) From 9dc0e77a2556d0830f02b47e67e1f285641272aa Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 12 Apr 2011 14:20:05 -0600 Subject: [PATCH 13/36] Updated Vecernje Novosti. Fixes #759058 (Updated recipe for site Vecernje Novosti) --- recipes/novosti.recipe | 59 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 48 insertions(+), 11 deletions(-) diff --git a/recipes/novosti.recipe b/recipes/novosti.recipe index d66e7d28b7..a0a573d7ba 100644 --- a/recipes/novosti.recipe +++ b/recipes/novosti.recipe @@ -1,6 +1,6 @@ __license__ = 'GPL v3' -__copyright__ = '2008-2010, Darko Miletic ' +__copyright__ = '2008-2011, Darko Miletic ' ''' novosti.rs ''' @@ -21,34 +21,71 @@ class Novosti(BasicNewsRecipe): encoding = 'utf-8' language = 'sr' publication_type = 'newspaper' + masthead_url = 'http://www.novosti.rs/images/basic/logo-print.png' extra_css = """ @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} .article_description,body{font-family: Arial,Helvetica,sans1,sans-serif} .author{font-size: small} .articleLead{font-size: large; font-weight: bold} + img{display: block; margin-bottom: 1em; margin-top: 1em} """ conversion_options = { - 'comment' : description - , 'tags' : category - , 'publisher' : publisher - , 'language' : language + 'comment' : description + , 'tags' : category + , 'publisher' : publisher + , 'language' : language + , 'pretty_print' : True } preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')] - keep_only_tags = [dict(attrs={'class':['articleTitle','author','articleLead','articleBody']})] - remove_tags = [dict(name=['embed','object','iframe','base','link','meta'])] - feeds = [(u'Vesti', u'http://www.novosti.rs/rss/rss-vesti')] + keep_only_tags = [dict(attrs={'class':['articleTitle','articleInfo','articleLead','singlePhoto fl','articleBody']})] + remove_tags = [ + dict(name=['embed','object','iframe','base','link','meta']) + ,dict(name='a', attrs={'class':'loadComments topCommentsLink'}) + ] + remove_attributes = ['lang','xmlns:fb'] + + feeds = [ + (u'Politika' , u'http://www.novosti.rs/rss/2-Sve%20vesti') + ,(u'Drustvo' , u'http://www.novosti.rs/rss/1-Sve%20vesti') + ,(u'Ekonomija' , u'http://www.novosti.rs/rss/3-Sve%20vesti') + ,(u'Hronika' , u'http://www.novosti.rs/rss/4-Sve%20vesti') + ,(u'Dosije' , u'http://www.novosti.rs/rss/5-Sve%20vesti') + ,(u'Reportaze' , u'http://www.novosti.rs/rss/6-Sve%20vesti') + ,(u'Tehnologije' , u'http://www.novosti.rs/rss/35-Sve%20vesti') + ,(u'Zanimljivosti', u'http://www.novosti.rs/rss/26-Sve%20vesti') + ,(u'Auto' , u'http://www.novosti.rs/rss/50-Sve%20vesti') + ,(u'Sport' , u'http://www.novosti.rs/rss/11|47|12|14|13-Sve%20vesti') + ,(u'Svet' , u'http://www.novosti.rs/rss/7-Sve%20vesti') + ,(u'Region' , u'http://www.novosti.rs/rss/8-Sve%20vesti') + ,(u'Dijaspora' , u'http://www.novosti.rs/rss/9-Sve%20vesti') + ,(u'Spektakl' , u'http://www.novosti.rs/rss/10-Sve%20vesti') + ,(u'Kultura' , u'http://www.novosti.rs/rss/31-Sve%20vesti') + ,(u'Srbija' , u'http://www.novosti.rs/rss/15-Sve%20vesti') + ,(u'Beograd' , u'http://www.novosti.rs/rss/16-Sve%20vesti') + ,(u'Zivot+' , u'http://www.novosti.rs/rss/24|33|34|25|20|18|32|19-Sve%20vesti') + ,(u'Turizam' , u'http://www.novosti.rs/rss/36-Sve%20vesti') + ] def preprocess_html(self, soup): for item in soup.findAll(style=True): del item['style'] - for item in soup.findAll('span', attrs={'class':'author'}): - item.name='p' + for item in soup.findAll('a'): + limg = item.find('img') + if item.string is not None: + str = item.string + item.replaceWith(str) + else: + if limg: + item.name = 'div' + item.attrs = [] + else: + str = self.tag_to_string(item) + item.replaceWith(str) for item in soup.findAll('img'): if not item.has_key('alt'): item['alt'] = 'image' return soup - From 933f81b65f05b152787e2f66ef4a4a657cc0f3be Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 12 Apr 2011 14:23:17 -0600 Subject: [PATCH 14/36] Fix #759073 (Asus EeeNote) --- src/calibre/devices/misc.py | 2 +- src/calibre/gui2/metadata/bulk_download2.py | 49 ++++++++++++++++++++- 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/src/calibre/devices/misc.py b/src/calibre/devices/misc.py index 01eba48a30..b9710d1958 100644 --- a/src/calibre/devices/misc.py +++ b/src/calibre/devices/misc.py @@ -244,7 +244,7 @@ class EEEREADER(USBMS): FORMATS = ['epub', 'fb2', 'txt', 'pdf'] VENDOR_ID = [0x0b05] - PRODUCT_ID = [0x178f] + PRODUCT_ID = [0x178f, 0x17a1] BCD = [0x0319] EBOOK_DIR_MAIN = EBOOK_DIR_CARD_A = 'Book' diff --git a/src/calibre/gui2/metadata/bulk_download2.py b/src/calibre/gui2/metadata/bulk_download2.py index add689e616..19cd3df9d4 100644 --- a/src/calibre/gui2/metadata/bulk_download2.py +++ b/src/calibre/gui2/metadata/bulk_download2.py @@ -7,6 +7,7 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' +import os from functools import partial from itertools import izip @@ -18,6 +19,11 @@ from calibre.gui2.threaded_jobs import ThreadedJob from calibre.utils.icu import lower from calibre.ebooks.metadata import authors_to_string from calibre.gui2 import question_dialog, error_dialog +from calibre.ebooks.metadata.sources.identify import identify, msprefs +from calibre.ebooks.metadata.sources.covers import download_cover +from calibre.ebooks.metadata.book.base import Metadata +from calibre.customize.ui import metadata_plugins +from calibre.ptempfile import PersistentTemporaryFile def show_config(gui, parent): from calibre.gui2.preferences import show_config_widget @@ -127,6 +133,12 @@ class ApplyDialog(QDialog): import traceback self.failures.append((i, traceback.format_exc())) + try: + if mi.cover: + os.remove(mi.cover) + except: + pass + self.pb.setValue(self.pb.value()+1) if self.current_idx >= len(self.id_map) - 1: @@ -216,8 +228,21 @@ def proceed(gui, job): q.show() q.finished.connect(partial(apply_metadata, job, gui, q)) +def merge_result(oldmi, newmi): + dummy = Metadata(_('Unknown')) + for f in msprefs['ignore_fields']: + setattr(newmi, f, getattr(dummy, f)) + fields = set() + for plugin in metadata_plugins(['identify']): + fields |= plugin.touched_fields -def download(ids, db, identify, covers, + for f in fields: + # Optimize so that set_metadata does not have to do extra work later + if not f.startswith('identifier:'): + if (not newmi.is_null(f) and getattr(newmi, f) == getattr(oldmi, f)): + setattr(newmi, f, getattr(dummy, f)) + +def download(ids, db, do_identify, covers, log=None, abort=None, notifications=None): ids = list(ids) metadata = [db.get_metadata(i, index_is_id=True, get_user_categories=False) @@ -229,7 +254,27 @@ def download(ids, db, identify, covers, if abort.is_set(): log.error('Aborting...') break - # TODO: Apply ignore_fields and set unchanged values to null values + title, authors, identifiers = mi.title, mi.authors, mi.identifiers + if do_identify: + results = [] + try: + results = identify(log, abort, title=title, authors=authors, + identifiers=identifiers) + except: + pass + if results: + mi = merge_result(mi, results[0]) + identifiers = mi.identifiers + else: + log.error('Failed to download metadata for', title) + failed_ids.add(mi) + if covers: + cdata = download_cover(log, title=title, authors=authors, + identifiers=identifiers) + if cdata: + with PersistentTemporaryFile('.jpg', 'downloaded-cover-') as f: + f.write(cdata) + mi.cover = f.name ans[i] = mi count += 1 notifications.put((count/len(ids), From 566fa8d80f9ecb3c58dc749d53c309d168438a5b Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 12 Apr 2011 22:11:43 +0100 Subject: [PATCH 15/36] Make true and false searches work correctly for numeric fields. --- src/calibre/library/caches.py | 82 +++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 38 deletions(-) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index a108feb388..01b7335bf4 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -402,47 +402,55 @@ class ResultCache(SearchQueryParser): # {{{ matches = set([]) if len(query) == 0: return matches - if query == 'false': - query = '0' - elif query == 'true': - query = '!=0' - relop = None - for k in self.numeric_search_relops.keys(): - if query.startswith(k): - (p, relop) = self.numeric_search_relops[k] - query = query[p:] - if relop is None: - (p, relop) = self.numeric_search_relops['='] if val_func is None: loc = self.field_metadata[location]['rec_index'] val_func = lambda item, loc=loc: item[loc] - dt = self.field_metadata[location]['datatype'] - if dt == 'int': - cast = (lambda x: int (x)) - adjust = lambda x: x - elif dt == 'rating': - cast = (lambda x: int (x)) - adjust = lambda x: x/2 - elif dt in ('float', 'composite'): - cast = lambda x : float (x) - adjust = lambda x: x - else: # count operation - cast = (lambda x: int (x)) - adjust = lambda x: x - - if len(query) > 1: - mult = query[-1:].lower() - mult = {'k':1024.,'m': 1024.**2, 'g': 1024.**3}.get(mult, 1.0) - if mult != 1.0: - query = query[:-1] + if query == 'false': + q = '' + relop = lambda x,y: x is None + val_func = lambda item, loc=loc: item[loc] + cast = adjust = lambda x: x + elif query == 'true': + q = '' + relop = lambda x,y: x is not None + val_func = lambda item, loc=loc: item[loc] + cast = adjust = lambda x: x else: - mult = 1.0 - try: - q = cast(query) * mult - except: - return matches + relop = None + for k in self.numeric_search_relops.keys(): + if query.startswith(k): + (p, relop) = self.numeric_search_relops[k] + query = query[p:] + if relop is None: + (p, relop) = self.numeric_search_relops['='] + + dt = self.field_metadata[location]['datatype'] + if dt == 'int': + cast = lambda x: int (x) if x is not None else None + adjust = lambda x: x + elif dt == 'rating': + cast = lambda x: int (x) + adjust = lambda x: x/2 + elif dt in ('float', 'composite'): + cast = lambda x : float (x) if x is not None else None + adjust = lambda x: x + else: # count operation + cast = (lambda x: int (x)) + adjust = lambda x: x + + if len(query) > 1: + mult = query[-1:].lower() + mult = {'k':1024.,'m': 1024.**2, 'g': 1024.**3}.get(mult, 1.0) + if mult != 1.0: + query = query[:-1] + else: + mult = 1.0 + try: + q = cast(query) * mult + except: + return matches for id_ in candidates: item = self._data[id_] @@ -452,9 +460,7 @@ class ResultCache(SearchQueryParser): # {{{ v = cast(val_func(item)) except: v = 0 - if not v: - v = 0 - else: + if v: v = adjust(v) if relop(v, q): matches.add(item[0]) From 18d0f6a6ef79bc80b23a3a58a1c7c607169fe662 Mon Sep 17 00:00:00 2001 From: John Schember Date: Tue, 12 Apr 2011 18:58:54 -0400 Subject: [PATCH 16/36] Add HTMLZ as a book extension. Use HTML icon for HTMLZ. --- src/calibre/ebooks/__init__.py | 2 +- src/calibre/gui2/__init__.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/__init__.py b/src/calibre/ebooks/__init__.py index 7776be5e28..a56abb907e 100644 --- a/src/calibre/ebooks/__init__.py +++ b/src/calibre/ebooks/__init__.py @@ -26,7 +26,7 @@ class ParserError(ValueError): pass BOOK_EXTENSIONS = ['lrf', 'rar', 'zip', 'rtf', 'lit', 'txt', 'txtz', 'text', 'htm', 'xhtm', - 'html', 'xhtml', 'pdf', 'pdb', 'pdr', 'prc', 'mobi', 'azw', 'doc', + 'html', 'htmlz', 'xhtml', 'pdf', 'pdb', 'pdr', 'prc', 'mobi', 'azw', 'doc', 'epub', 'fb2', 'djvu', 'lrx', 'cbr', 'cbz', 'cbc', 'oebzip', 'rb', 'imp', 'odt', 'chm', 'tpz', 'azw1', 'pml', 'pmlz', 'mbp', 'tan', 'snb'] diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index 22aaabf592..e39427021e 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -357,6 +357,7 @@ class FileIconProvider(QFileIconProvider): 'bmp' : 'bmp', 'svg' : 'svg', 'html' : 'html', + 'htmlz' : 'html', 'htm' : 'html', 'xhtml' : 'html', 'xhtm' : 'html', From d5119f0c2f0bad0220122e7771cbb6388d22a21a Mon Sep 17 00:00:00 2001 From: John Schember Date: Tue, 12 Apr 2011 19:11:52 -0400 Subject: [PATCH 17/36] HTMLZ Output: Handle SVG data returned as lxml.etree._Element properly. --- src/calibre/ebooks/htmlz/output.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/htmlz/output.py b/src/calibre/ebooks/htmlz/output.py index 7cdf04bcdb..03fe12c89e 100644 --- a/src/calibre/ebooks/htmlz/output.py +++ b/src/calibre/ebooks/htmlz/output.py @@ -12,7 +12,7 @@ from lxml import etree from calibre.customize.conversion import OutputFormatPlugin, \ OptionRecommendation -from calibre.ebooks.oeb.base import OEB_IMAGES +from calibre.ebooks.oeb.base import OEB_IMAGES, SVG_MIME from calibre.ptempfile import TemporaryDirectory from calibre.utils.zipfile import ZipFile @@ -71,9 +71,13 @@ class HTMLZOutput(OutputFormatPlugin): os.makedirs(os.path.join(tdir, 'images')) for item in oeb_book.manifest: if item.media_type in OEB_IMAGES and item.href in images: + if item.media_type == SVG_MIME: + data = unicode(etree.tostring(item.data, encoding=unicode)) + else: + data = item.data fname = os.path.join(tdir, 'images', images[item.href]) with open(fname, 'wb') as img: - img.write(item.data) + img.write(data) # Metadata with open(os.path.join(tdir, 'metadata.opf'), 'wb') as mdataf: From 1d6521aa5e34fc04902130680b0e73a1979ae0c7 Mon Sep 17 00:00:00 2001 From: John Schember Date: Tue, 12 Apr 2011 19:53:04 -0400 Subject: [PATCH 18/36] extZ metadata: Read and write first opf file found in archive. --- src/calibre/ebooks/metadata/extz.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/src/calibre/ebooks/metadata/extz.py b/src/calibre/ebooks/metadata/extz.py index 0ecdbe9ea6..b49f3f6ddd 100644 --- a/src/calibre/ebooks/metadata/extz.py +++ b/src/calibre/ebooks/metadata/extz.py @@ -25,14 +25,30 @@ def get_metadata(stream, extract_cover=True): with TemporaryDirectory('_untxtz_mdata') as tdir: try: - zf = ZipFile(stream) - zf.extract('metadata.opf', tdir) - with open(os.path.join(tdir, 'metadata.opf'), 'rb') as opff: - mi = OPF(opff).to_book_metadata() + with ZipFile(stream) as zf: + opf_name = get_first_opf_name(stream) + opf_stream = StringIO(zf.read(opf_name)) + mi = OPF(opf_stream).to_book_metadata() except: return mi return mi def set_metadata(stream, mi): opf = StringIO(metadata_to_opf(mi)) - safe_replace(stream, 'metadata.opf', opf) + try: + opf_name = get_first_opf_name(stream) + except: + opf_name = 'metadata.opf' + safe_replace(stream, opf_name, opf) + +def get_first_opf_name(stream): + with ZipFile(stream) as zf: + names = zf.namelist() + opfs = [] + for n in names: + if n.endswith('.opf') and '/' not in n: + opfs.append(n) + if not opfs: + raise Exception('No OPF found') + opfs.sort() + return opfs[0] From f3beb13b6221aebb0366dfc1044fdfd959646f2f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 12 Apr 2011 18:06:18 -0600 Subject: [PATCH 19/36] Bulk metadata download works again. More testing of corner cases needed --- src/calibre/ebooks/metadata/sources/covers.py | 2 +- src/calibre/gui2/actions/edit_metadata.py | 2 +- src/calibre/gui2/metadata/bulk_download2.py | 29 +++++++++++++++---- src/calibre/gui2/threaded_jobs.py | 6 +++- 4 files changed, 30 insertions(+), 9 deletions(-) diff --git a/src/calibre/ebooks/metadata/sources/covers.py b/src/calibre/ebooks/metadata/sources/covers.py index cf6ec90c54..44f902eeee 100644 --- a/src/calibre/ebooks/metadata/sources/covers.py +++ b/src/calibre/ebooks/metadata/sources/covers.py @@ -145,7 +145,7 @@ def download_cover(log, Synchronous cover download. Returns the "best" cover as per user prefs/cover resolution. - Return cover is a tuple: (plugin, width, height, fmt, data) + Returned cover is a tuple: (plugin, width, height, fmt, data) Returns None if no cover is found. ''' diff --git a/src/calibre/gui2/actions/edit_metadata.py b/src/calibre/gui2/actions/edit_metadata.py index 9f2cacb177..18a73fb282 100644 --- a/src/calibre/gui2/actions/edit_metadata.py +++ b/src/calibre/gui2/actions/edit_metadata.py @@ -94,7 +94,7 @@ class EditMetadataAction(InterfaceAction): def bulk_metadata_downloaded(self, job): if job.failed: - self.job_exception(job, dialog_title=_('Failed to download metadata')) + self.gui.job_exception(job, dialog_title=_('Failed to download metadata')) return from calibre.gui2.metadata.bulk_download2 import proceed proceed(self.gui, job) diff --git a/src/calibre/gui2/metadata/bulk_download2.py b/src/calibre/gui2/metadata/bulk_download2.py index 19cd3df9d4..05c61f6037 100644 --- a/src/calibre/gui2/metadata/bulk_download2.py +++ b/src/calibre/gui2/metadata/bulk_download2.py @@ -54,6 +54,8 @@ def start_download(gui, ids, callback, identify, covers): _('Download metadata for %d books')%len(ids), download, (ids, gui.current_db, identify, covers), {}, callback) gui.job_manager.run_threaded_job(job) + gui.status_bar.show_message(_('Metadata download started'), 3000) + class ViewLog(QDialog): # {{{ @@ -110,11 +112,12 @@ class ApplyDialog(QDialog): self.bb.accepted.connect(self.accept) l.addWidget(self.bb) - self.db = gui.current_db + self.gui = gui self.id_map = list(id_map.iteritems()) self.current_idx = 0 self.failures = [] + self.ids = [] self.canceled = False QTimer.singleShot(20, self.do_one) @@ -124,11 +127,13 @@ class ApplyDialog(QDialog): if self.canceled: return i, mi = self.id_map[self.current_idx] + db = self.gui.current_db try: set_title = not mi.is_null('title') set_authors = not mi.is_null('authors') - self.db.set_metadata(i, mi, commit=False, set_title=set_title, + db.set_metadata(i, mi, commit=False, set_title=set_title, set_authors=set_authors) + self.ids.append(i) except: import traceback self.failures.append((i, traceback.format_exc())) @@ -156,9 +161,10 @@ class ApplyDialog(QDialog): return if self.failures: msg = [] + db = self.gui.current_db for i, tb in self.failures: - title = self.db.title(i, index_is_id=True) - authors = self.db.authors(i, index_is_id=True) + title = db.title(i, index_is_id=True) + authors = db.authors(i, index_is_id=True) if authors: authors = [x.replace('|', ',') for x in authors.split(',')] title += ' - ' + authors_to_string(authors) @@ -169,6 +175,12 @@ class ApplyDialog(QDialog): ' in your library. Click "Show Details" to see ' 'details.'), det_msg='\n\n'.join(msg), show=True) self.accept() + if self.ids: + cr = self.gui.library_view.currentIndex().row() + self.gui.library_view.model().refresh_ids( + self.ids, cr) + if self.gui.cover_flow: + self.gui.cover_flow.dataChanged() _amd = None def apply_metadata(job, gui, q, result): @@ -209,6 +221,7 @@ def apply_metadata(job, gui, q, result): _amd = ApplyDialog(id_map, gui) def proceed(gui, job): + gui.status_bar.show_message(_('Metadata download completed'), 3000) id_map, failed_ids = job.result fmsg = det_msg = '' if failed_ids: @@ -242,6 +255,10 @@ def merge_result(oldmi, newmi): if (not newmi.is_null(f) and getattr(newmi, f) == getattr(oldmi, f)): setattr(newmi, f, getattr(dummy, f)) + newmi.last_modified = oldmi.last_modified + + return newmi + def download(ids, db, do_identify, covers, log=None, abort=None, notifications=None): ids = list(ids) @@ -271,9 +288,9 @@ def download(ids, db, do_identify, covers, if covers: cdata = download_cover(log, title=title, authors=authors, identifiers=identifiers) - if cdata: + if cdata is not None: with PersistentTemporaryFile('.jpg', 'downloaded-cover-') as f: - f.write(cdata) + f.write(cdata[-1]) mi.cover = f.name ans[i] = mi count += 1 diff --git a/src/calibre/gui2/threaded_jobs.py b/src/calibre/gui2/threaded_jobs.py index f98488da79..9c791c5b0d 100644 --- a/src/calibre/gui2/threaded_jobs.py +++ b/src/calibre/gui2/threaded_jobs.py @@ -189,7 +189,11 @@ class ThreadedJobServer(Thread): def run(self): while self.keep_going: - self.run_once() + try: + self.run_once() + except: + import traceback + traceback.print_exc() time.sleep(0.1) def run_once(self): From 184692b587e67d79ef35edc04ff5b97c0c27654d Mon Sep 17 00:00:00 2001 From: John Schember Date: Tue, 12 Apr 2011 20:39:01 -0400 Subject: [PATCH 20/36] extZ metadata: Get cover, update OPF without losing other data such as spine, and guide. --- src/calibre/ebooks/metadata/extz.py | 34 +++++++++++++++++------------ 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/src/calibre/ebooks/metadata/extz.py b/src/calibre/ebooks/metadata/extz.py index b49f3f6ddd..338c4dd91d 100644 --- a/src/calibre/ebooks/metadata/extz.py +++ b/src/calibre/ebooks/metadata/extz.py @@ -7,13 +7,10 @@ __copyright__ = '2011, John Schember ' Read meta information from extZ (TXTZ, HTMLZ...) files. ''' -import os - from cStringIO import StringIO from calibre.ebooks.metadata import MetaInformation -from calibre.ebooks.metadata.opf2 import OPF, metadata_to_opf -from calibre.ptempfile import TemporaryDirectory +from calibre.ebooks.metadata.opf2 import OPF from calibre.utils.zipfile import ZipFile, safe_replace def get_metadata(stream, extract_cover=True): @@ -23,23 +20,32 @@ def get_metadata(stream, extract_cover=True): mi = MetaInformation(_('Unknown'), [_('Unknown')]) stream.seek(0) - with TemporaryDirectory('_untxtz_mdata') as tdir: - try: - with ZipFile(stream) as zf: - opf_name = get_first_opf_name(stream) - opf_stream = StringIO(zf.read(opf_name)) - mi = OPF(opf_stream).to_book_metadata() - except: - return mi + try: + with ZipFile(stream) as zf: + opf_name = get_first_opf_name(stream) + opf_stream = StringIO(zf.read(opf_name)) + opf = OPF(opf_stream) + mi = opf.to_book_metadata() + if extract_cover: + cover_name = opf.raster_cover + if cover_name: + mi.cover_data = ('jpg', zf.read(cover_name)) + except: + return mi return mi def set_metadata(stream, mi): - opf = StringIO(metadata_to_opf(mi)) try: opf_name = get_first_opf_name(stream) + with ZipFile(stream) as zf: + opf_stream = StringIO(zf.read(opf_name)) + opf = OPF(opf_stream) except: opf_name = 'metadata.opf' - safe_replace(stream, opf_name, opf) + opf = OPF(StringIO()) + opf.smart_update(mi, replace_metadata=True) + newopf = StringIO(opf.render()) + safe_replace(stream, opf_name, newopf) def get_first_opf_name(stream): with ZipFile(stream) as zf: From 15f638784d2829d6b96e55a475a36cfdcacfba97 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 12 Apr 2011 18:39:12 -0600 Subject: [PATCH 21/36] ... --- src/calibre/gui2/metadata/bulk_download2.py | 39 +++++++++++++++------ 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/src/calibre/gui2/metadata/bulk_download2.py b/src/calibre/gui2/metadata/bulk_download2.py index 05c61f6037..5f0af1b316 100644 --- a/src/calibre/gui2/metadata/bulk_download2.py +++ b/src/calibre/gui2/metadata/bulk_download2.py @@ -77,7 +77,7 @@ class ViewLog(QDialog): # {{{ self.copy_button.clicked.connect(self.copy_to_clipboard) l.addWidget(self.bb) self.setModal(False) - self.resize(QSize(500, 400)) + self.resize(QSize(700, 500)) self.setWindowTitle(_('Download log')) self.setWindowIcon(QIcon(I('debug.png'))) self.show() @@ -121,7 +121,6 @@ class ApplyDialog(QDialog): self.canceled = False QTimer.singleShot(20, self.do_one) - self.exec_() def do_one(self): if self.canceled: @@ -189,7 +188,7 @@ def apply_metadata(job, gui, q, result): q.finished.disconnect() if result != q.Accepted: return - id_map, failed_ids = job.result + id_map, failed_ids, failed_covers, title_map = job.result id_map = dict([(k, v) for k, v in id_map.iteritems() if k not in failed_ids]) if not id_map: @@ -219,24 +218,32 @@ def apply_metadata(job, gui, q, result): return _amd = ApplyDialog(id_map, gui) + _amd.exec_() def proceed(gui, job): gui.status_bar.show_message(_('Metadata download completed'), 3000) - id_map, failed_ids = job.result + id_map, failed_ids, failed_covers, title_map = job.result fmsg = det_msg = '' - if failed_ids: - fmsg = _('Could not download metadata for %d of the books. Click' + if failed_ids or failed_covers: + fmsg = '

'+_('Could not download metadata and/or covers for %d of the books. Click' ' "Show details" to see which books.')%len(failed_ids) - det_msg = '\n'.join([id_map[i].title for i in failed_ids]) + det_msg = [] + for i in failed_ids | failed_covers: + title = title_map[i] + if i in failed_ids: + title += (' ' + _('(Failed metadata)')) + if i in failed_covers: + title += (' ' + _('(Failed cover)')) + det_msg.append(title) msg = '

' + _('Finished downloading metadata for %d book(s). ' 'Proceed with updating the metadata in your library?')%len(id_map) q = MessageBox(MessageBox.QUESTION, _('Download complete'), - msg + fmsg, det_msg=det_msg, show_copy_button=bool(failed_ids), + msg + fmsg, det_msg='\n'.join(det_msg), show_copy_button=bool(failed_ids), parent=gui) q.vlb = q.bb.addButton(_('View log'), q.bb.ActionRole) q.vlb.setIcon(QIcon(I('debug.png'))) q.vlb.clicked.connect(partial(view_log, job, q)) - q.det_msg_toggle.setVisible(bool(failed_ids)) + q.det_msg_toggle.setVisible(bool(failed_ids | failed_covers)) q.setModal(False) q.show() q.finished.connect(partial(apply_metadata, job, gui, q)) @@ -265,6 +272,8 @@ def download(ids, db, do_identify, covers, metadata = [db.get_metadata(i, index_is_id=True, get_user_categories=False) for i in ids] failed_ids = set() + failed_covers = set() + title_map = {} ans = {} count = 0 for i, mi in izip(ids, metadata): @@ -272,6 +281,7 @@ def download(ids, db, do_identify, covers, log.error('Aborting...') break title, authors, identifiers = mi.title, mi.authors, mi.identifiers + title_map[i] = title if do_identify: results = [] try: @@ -282,9 +292,14 @@ def download(ids, db, do_identify, covers, if results: mi = merge_result(mi, results[0]) identifiers = mi.identifiers + if not mi.is_null('rating'): + # set_metadata expects a rating out of 10 + mi.rating *= 2 else: log.error('Failed to download metadata for', title) - failed_ids.add(mi) + failed_ids.add(i) + # We don't want set_metadata operating on anything but covers + mi = merge_result(mi, mi) if covers: cdata = download_cover(log, title=title, authors=authors, identifiers=identifiers) @@ -292,12 +307,14 @@ def download(ids, db, do_identify, covers, with PersistentTemporaryFile('.jpg', 'downloaded-cover-') as f: f.write(cdata[-1]) mi.cover = f.name + else: + failed_covers.add(i) ans[i] = mi count += 1 notifications.put((count/len(ids), _('Downloaded %d of %d')%(count, len(ids)))) log('Download complete, with %d failures'%len(failed_ids)) - return (ans, failed_ids) + return (ans, failed_ids, failed_covers, title_map) From 40d01c5aeace074eb1700fc4a3b9a62a120d9beb Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 12 Apr 2011 18:45:16 -0600 Subject: [PATCH 22/36] MOBI Output: Fix bug that would cause conversion to unneccessarily abort when malformed hyperlinks are present in the input document. Fixes #759313 (Private bug) --- src/calibre/ebooks/mobi/writer.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/calibre/ebooks/mobi/writer.py b/src/calibre/ebooks/mobi/writer.py index 5f4c47cdf3..89ef9fcd82 100644 --- a/src/calibre/ebooks/mobi/writer.py +++ b/src/calibre/ebooks/mobi/writer.py @@ -310,10 +310,11 @@ class Serializer(object): if href not in id_offsets: self.logger.warn('Hyperlink target %r not found' % href) href, _ = urldefrag(href) - ioff = self.id_offsets[href] - for hoff in hoffs: - buffer.seek(hoff) - buffer.write('%010d' % ioff) + else: + ioff = self.id_offsets[href] + for hoff in hoffs: + buffer.seek(hoff) + buffer.write('%010d' % ioff) class MobiWriter(object): COLLAPSE_RE = re.compile(r'[ \t\r\n\v]+') From 5e75259355e47a19c45554d12711af9df907e727 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 12 Apr 2011 18:54:06 -0600 Subject: [PATCH 23/36] ... --- src/calibre/ebooks/mobi/writer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/mobi/writer.py b/src/calibre/ebooks/mobi/writer.py index 89ef9fcd82..fc47b26c02 100644 --- a/src/calibre/ebooks/mobi/writer.py +++ b/src/calibre/ebooks/mobi/writer.py @@ -310,7 +310,7 @@ class Serializer(object): if href not in id_offsets: self.logger.warn('Hyperlink target %r not found' % href) href, _ = urldefrag(href) - else: + if href in self.id_offsets: ioff = self.id_offsets[href] for hoff in hoffs: buffer.seek(hoff) From fbde96b7a1b947f349fa3c71d1c5b6e090418fd9 Mon Sep 17 00:00:00 2001 From: John Schember Date: Tue, 12 Apr 2011 20:54:14 -0400 Subject: [PATCH 24/36] extZ metadata: Set cover. --- src/calibre/ebooks/metadata/extz.py | 52 ++++++++++++++++++++++++++--- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/src/calibre/ebooks/metadata/extz.py b/src/calibre/ebooks/metadata/extz.py index 338c4dd91d..18c5a25671 100644 --- a/src/calibre/ebooks/metadata/extz.py +++ b/src/calibre/ebooks/metadata/extz.py @@ -7,10 +7,14 @@ __copyright__ = '2011, John Schember ' Read meta information from extZ (TXTZ, HTMLZ...) files. ''' +import os +import posixpath + from cStringIO import StringIO from calibre.ebooks.metadata import MetaInformation from calibre.ebooks.metadata.opf2 import OPF +from calibre.ptempfile import PersistentTemporaryFile from calibre.utils.zipfile import ZipFile, safe_replace def get_metadata(stream, extract_cover=True): @@ -35,17 +39,50 @@ def get_metadata(stream, extract_cover=True): return mi def set_metadata(stream, mi): + replacements = {} + + # Get the OPF in the archive. try: - opf_name = get_first_opf_name(stream) + opf_path = get_first_opf_name(stream) with ZipFile(stream) as zf: - opf_stream = StringIO(zf.read(opf_name)) + opf_stream = StringIO(zf.read(opf_path)) opf = OPF(opf_stream) except: - opf_name = 'metadata.opf' + opf_path = 'metadata.opf' opf = OPF(StringIO()) + + # Cover. + new_cdata = None + try: + new_cdata = mi.cover_data[1] + if not new_cdata: + raise Exception('no cover') + except: + try: + new_cdata = open(mi.cover, 'rb').read() + except: + pass + if new_cdata: + raster_cover = opf.raster_cover + if not raster_cover: + raster_cover = 'cover.jpg' + cpath = posixpath.join(posixpath.dirname(opf_path), raster_cover) + new_cover = _write_new_cover(new_cdata, cpath) + replacements[cpath] = open(new_cover.name, 'rb') + + # Update the metadata. opf.smart_update(mi, replace_metadata=True) newopf = StringIO(opf.render()) - safe_replace(stream, opf_name, newopf) + safe_replace(stream, opf_path, newopf, extra_replacements=replacements) + + # Cleanup temporary files. + try: + if cpath is not None: + replacements[cpath].close() + os.remove(replacements[cpath].name) + except: + pass + def get_first_opf_name(stream): with ZipFile(stream) as zf: @@ -58,3 +95,10 @@ def get_first_opf_name(stream): raise Exception('No OPF found') opfs.sort() return opfs[0] + +def _write_new_cover(new_cdata, cpath): + from calibre.utils.magick.draw import save_cover_data_to + new_cover = PersistentTemporaryFile(suffix=os.path.splitext(cpath)[1]) + new_cover.close() + save_cover_data_to(new_cdata, new_cover.name) + return new_cover From 5b82c42e4bc5b96ee242f61bc30d0be3d8ecf703 Mon Sep 17 00:00:00 2001 From: John Schember Date: Tue, 12 Apr 2011 20:55:06 -0400 Subject: [PATCH 25/36] ... --- src/calibre/ebooks/metadata/extz.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/calibre/ebooks/metadata/extz.py b/src/calibre/ebooks/metadata/extz.py index 18c5a25671..6d41f7819d 100644 --- a/src/calibre/ebooks/metadata/extz.py +++ b/src/calibre/ebooks/metadata/extz.py @@ -83,7 +83,6 @@ def set_metadata(stream, mi): except: pass - def get_first_opf_name(stream): with ZipFile(stream) as zf: names = zf.namelist() From eecf3ec73e8e4a33600d67c4c3d9b8235e2ec6b5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 12 Apr 2011 19:38:31 -0600 Subject: [PATCH 26/36] ... --- src/calibre/ebooks/metadata/sources/covers.py | 5 +++ .../ebooks/metadata/sources/identify.py | 4 ++ src/calibre/ebooks/metadata/sources/isbndb.py | 42 ++++++++++++++++++- 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/metadata/sources/covers.py b/src/calibre/ebooks/metadata/sources/covers.py index 44f902eeee..d28ce146c6 100644 --- a/src/calibre/ebooks/metadata/sources/covers.py +++ b/src/calibre/ebooks/metadata/sources/covers.py @@ -76,6 +76,11 @@ def run_download(log, results, abort, (plugin, width, height, fmt, bytes) ''' + if title == _('Unknown'): + title = None + if authors == [_('Unknown')]: + authors = None + plugins = [p for p in metadata_plugins(['cover']) if p.is_configured()] rq = Queue() diff --git a/src/calibre/ebooks/metadata/sources/identify.py b/src/calibre/ebooks/metadata/sources/identify.py index fad810c26e..b494e05e1a 100644 --- a/src/calibre/ebooks/metadata/sources/identify.py +++ b/src/calibre/ebooks/metadata/sources/identify.py @@ -253,6 +253,10 @@ def merge_identify_results(result_map, log): def identify(log, abort, # {{{ title=None, authors=None, identifiers={}, timeout=30): + if title == _('Unknown'): + title = None + if authors == [_('Unknown')]: + authors = None start_time = time.time() plugins = [p for p in metadata_plugins(['identify']) if p.is_configured()] diff --git a/src/calibre/ebooks/metadata/sources/isbndb.py b/src/calibre/ebooks/metadata/sources/isbndb.py index ab9342c6cb..af192227c1 100644 --- a/src/calibre/ebooks/metadata/sources/isbndb.py +++ b/src/calibre/ebooks/metadata/sources/isbndb.py @@ -7,7 +7,13 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' -from calibre.ebooks.metadata.sources.base import Source +from urllib import quote + +from calibre.ebooks.metadata import check_isbn +from calibre.ebooks.metadata.sources.base import Source, Option + +BASE_URL = 'http://isbndb.com/api/books.xml?access_key=%s&page_number=1&results=subjects,authors,texts&' + class ISBNDB(Source): @@ -18,6 +24,14 @@ class ISBNDB(Source): touched_fields = frozenset(['title', 'authors', 'identifier:isbn', 'comments', 'publisher']) supports_gzip_transfer_encoding = True + # Shortcut, since we have no cached cover URLS + cached_cover_url_is_reliable = False + + options = ( + Option('isbndb_key', 'string', None, _('IsbnDB key:'), + _('To use isbndb.com you have to sign up for a free account' + 'at isbndb.com and get an access key.')), + ) def __init__(self, *args, **kwargs): Source.__init__(self, *args, **kwargs) @@ -35,9 +49,33 @@ class ISBNDB(Source): except: pass - self.isbndb_key = prefs['isbndb_key'] + @property + def isbndb_key(self): + return self.prefs['isbndb_key'] def is_configured(self): return self.isbndb_key is not None + def create_query(self, log, title=None, authors=None, identifiers={}): # {{{ + base_url = BASE_URL%self.isbndb_key + isbn = check_isbn(identifiers.get('isbn', None)) + q = '' + if isbn is not None: + q = 'index1=isbn&value1='+isbn + elif title or authors: + tokens = [] + title_tokens = list(self.get_title_tokens(title)) + tokens += title_tokens + author_tokens = self.get_author_tokens(authors, + only_first_author=True) + tokens += author_tokens + tokens = [quote(t) for t in tokens] + q = '+'.join(tokens) + q = 'index1=combined&value1='+q + + if not q: + return None + if isinstance(q, unicode): + q = q.encode('utf-8') + return base_url + q From 2bdc0c48a48125db99b1a76c853fe94f2fd48f13 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 12 Apr 2011 22:39:38 -0600 Subject: [PATCH 27/36] Complete migration of ISBNDB plugin. However, I'm not enabling it, as it seems to provide largely useless results anyway. --- src/calibre/ebooks/metadata/sources/isbndb.py | 140 +++++++++++++++++- src/calibre/manual/server.rst | 2 + 2 files changed, 141 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/metadata/sources/isbndb.py b/src/calibre/ebooks/metadata/sources/isbndb.py index af192227c1..361554ad9c 100644 --- a/src/calibre/ebooks/metadata/sources/isbndb.py +++ b/src/calibre/ebooks/metadata/sources/isbndb.py @@ -9,8 +9,14 @@ __docformat__ = 'restructuredtext en' from urllib import quote +from lxml import etree + from calibre.ebooks.metadata import check_isbn from calibre.ebooks.metadata.sources.base import Source, Option +from calibre.ebooks.chardet import xml_to_unicode +from calibre.utils.cleantext import clean_ascii_chars +from calibre.utils.icu import lower +from calibre.ebooks.metadata.book.base import Metadata BASE_URL = 'http://isbndb.com/api/books.xml?access_key=%s&page_number=1&results=subjects,authors,texts&' @@ -56,7 +62,7 @@ class ISBNDB(Source): def is_configured(self): return self.isbndb_key is not None - def create_query(self, log, title=None, authors=None, identifiers={}): # {{{ + def create_query(self, title=None, authors=None, identifiers={}): # {{{ base_url = BASE_URL%self.isbndb_key isbn = check_isbn(identifiers.get('isbn', None)) q = '' @@ -78,4 +84,136 @@ class ISBNDB(Source): if isinstance(q, unicode): q = q.encode('utf-8') return base_url + q + # }}} + def identify(self, log, result_queue, abort, title=None, authors=None, # {{{ + identifiers={}, timeout=30): + if not self.is_configured(): + return + query = self.create_query(title=title, authors=authors, + identifiers=identifiers) + if not query: + err = 'Insufficient metadata to construct query' + log.error(err) + return err + + results = [] + try: + results = self.make_query(query, abort, title=title, authors=authors, + identifiers=identifiers, timeout=timeout) + except: + err = 'Failed to make query to ISBNDb, aborting.' + log.exception(err) + return err + + if not results and identifiers.get('isbn', False) and title and authors and \ + not abort.is_set(): + return self.identify(log, result_queue, abort, title=title, + authors=authors, timeout=timeout) + + for result in results: + self.clean_downloaded_metadata(result) + result_queue.put(result) + + def parse_feed(self, feed, seen, orig_title, orig_authors, identifiers): + + def tostring(x): + if x is None: + return '' + return etree.tostring(x, method='text', encoding=unicode).strip() + + orig_isbn = identifiers.get('isbn', None) + title_tokens = self.get_title_tokens(orig_title) + author_tokens = self.get_author_tokens(orig_authors) + results = [] + + def ismatch(title, authors): + authors = lower(' '.join(authors)) + title = lower(title) + match = False + for t in title_tokens: + if lower(t) in title: + match = True + break + if not title_tokens: match = True + amatch = False + for a in author_tokens: + if a in authors: + amatch = True + break + if not author_tokens: amatch = True + return match and amatch + + bl = feed.find('BookList') + if bl is None: + err = tostring(etree.find('errormessage')) + raise ValueError('ISBNDb query failed:' + err) + total_results = int(bl.get('total_results')) + shown_results = int(bl.get('shown_results')) + for bd in bl.xpath('.//BookData'): + isbn = check_isbn(bd.get('isbn13', bd.get('isbn', None))) + if not isbn: + continue + if orig_isbn and isbn != orig_isbn: + continue + title = tostring(bd.find('Title')) + if not title: + continue + authors = [] + for au in bd.xpath('.//Authors/Person'): + au = tostring(au) + if au: + if ',' in au: + ln, _, fn = au.partition(',') + au = fn.strip() + ' ' + ln.strip() + authors.append(au) + if not authors: + continue + id_ = (title, tuple(authors)) + if id_ in seen: + continue + seen.add(id_) + if not ismatch(title, authors): + continue + publisher = tostring(bd.find('PublisherText')) + if not publisher: publisher = None + comments = tostring(bd.find('Summary')) + if not comments: comments = None + mi = Metadata(title, authors) + mi.isbn = isbn + mi.publisher = publisher + mi.comments = comments + results.append(mi) + return total_results, shown_results, results + + def make_query(self, q, abort, title=None, authors=None, identifiers={}, + max_pages=10, timeout=30): + page_num = 1 + parser = etree.XMLParser(recover=True, no_network=True) + br = self.browser + + seen = set() + + candidates = [] + total_found = 0 + while page_num <= max_pages and not abort.is_set(): + url = q.replace('&page_number=1&', '&page_number=%d&'%page_num) + page_num += 1 + raw = br.open_novisit(url, timeout=timeout).read() + feed = etree.fromstring(xml_to_unicode(clean_ascii_chars(raw), + strip_encoding_pats=True)[0], parser=parser) + total, found, results = self.parse_feed( + feed, seen, title, authors, identifiers) + total_found += found + if results or total_found >= total: + candidates += results + break + + return candidates + # }}} + +if __name__ == '__main__': + s = ISBNDB(None) + t, a = 'great gatsby', ['fitzgerald'] + q = s.create_query(title=t, authors=a) + s.make_query(q, title=t, authors=a) diff --git a/src/calibre/manual/server.rst b/src/calibre/manual/server.rst index 82ec5c2927..aa98ba57df 100644 --- a/src/calibre/manual/server.rst +++ b/src/calibre/manual/server.rst @@ -22,6 +22,8 @@ First start the |app| content server as shown below:: calibre-server --url-prefix /calibre --port 8080 +The key parameter here is ``--url-prefix /calibre``. This causes the content server to serve all URLs prefixed by calibre. To see this in action, visit ``http://localhost:8080/calibre`` in your browser. You should see the normal content server website, but now it will run under /calibre. + Now suppose you are using Apache as your main server. First enable the proxy modules in apache, by adding the following to :file:`httpd.conf`:: LoadModule proxy_module modules/mod_proxy.so From cf675d79d862b26928083f6dd622064baddc7692 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 12 Apr 2011 22:42:47 -0600 Subject: [PATCH 28/36] ... --- src/calibre/ebooks/metadata/sources/isbndb.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/calibre/ebooks/metadata/sources/isbndb.py b/src/calibre/ebooks/metadata/sources/isbndb.py index 361554ad9c..18d797ba71 100644 --- a/src/calibre/ebooks/metadata/sources/isbndb.py +++ b/src/calibre/ebooks/metadata/sources/isbndb.py @@ -169,6 +169,11 @@ class ISBNDB(Source): authors.append(au) if not authors: continue + comments = tostring(bd.find('Summary')) + if not comments: + # Require comments, since without them the result is useless + # anyway + continue id_ = (title, tuple(authors)) if id_ in seen: continue @@ -177,8 +182,6 @@ class ISBNDB(Source): continue publisher = tostring(bd.find('PublisherText')) if not publisher: publisher = None - comments = tostring(bd.find('Summary')) - if not comments: comments = None mi = Metadata(title, authors) mi.isbn = isbn mi.publisher = publisher @@ -213,7 +216,8 @@ class ISBNDB(Source): # }}} if __name__ == '__main__': + from threading import Event s = ISBNDB(None) t, a = 'great gatsby', ['fitzgerald'] q = s.create_query(title=t, authors=a) - s.make_query(q, title=t, authors=a) + s.make_query(q, Event(), title=t, authors=a) From ec583f232d0611f9ba48e7f3e4f61f71717390e5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 12 Apr 2011 23:01:55 -0600 Subject: [PATCH 29/36] On second thoughts enable the ISBNDB plugin by default --- src/calibre/customize/builtins.py | 3 +- src/calibre/ebooks/metadata/sources/isbndb.py | 43 +++++++++++++------ 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 8f6c597ee5..d5957eb70a 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -625,8 +625,9 @@ if test_eight_code: from calibre.ebooks.metadata.sources.google import GoogleBooks from calibre.ebooks.metadata.sources.amazon import Amazon from calibre.ebooks.metadata.sources.openlibrary import OpenLibrary + from calibre.ebooks.metadata.sources.isbndb import ISBNDB - plugins += [GoogleBooks, Amazon, OpenLibrary] + plugins += [GoogleBooks, Amazon, OpenLibrary, ISBNDB] # }}} else: diff --git a/src/calibre/ebooks/metadata/sources/isbndb.py b/src/calibre/ebooks/metadata/sources/isbndb.py index 18d797ba71..a2a10708fb 100644 --- a/src/calibre/ebooks/metadata/sources/isbndb.py +++ b/src/calibre/ebooks/metadata/sources/isbndb.py @@ -123,22 +123,21 @@ class ISBNDB(Source): return etree.tostring(x, method='text', encoding=unicode).strip() orig_isbn = identifiers.get('isbn', None) - title_tokens = self.get_title_tokens(orig_title) - author_tokens = self.get_author_tokens(orig_authors) + title_tokens = list(self.get_title_tokens(orig_title)) + author_tokens = list(self.get_author_tokens(orig_authors)) results = [] def ismatch(title, authors): authors = lower(' '.join(authors)) title = lower(title) - match = False + match = not title_tokens for t in title_tokens: if lower(t) in title: match = True break - if not title_tokens: match = True - amatch = False + amatch = not author_tokens for a in author_tokens: - if a in authors: + if lower(a) in authors: amatch = True break if not author_tokens: amatch = True @@ -182,6 +181,8 @@ class ISBNDB(Source): continue publisher = tostring(bd.find('PublisherText')) if not publisher: publisher = None + if publisher and 'audio' in publisher.lower(): + continue mi = Metadata(title, authors) mi.isbn = isbn mi.publisher = publisher @@ -208,16 +209,32 @@ class ISBNDB(Source): total, found, results = self.parse_feed( feed, seen, title, authors, identifiers) total_found += found - if results or total_found >= total: - candidates += results + candidates += results + if total_found >= total or len(candidates) > 9: break return candidates # }}} if __name__ == '__main__': - from threading import Event - s = ISBNDB(None) - t, a = 'great gatsby', ['fitzgerald'] - q = s.create_query(title=t, authors=a) - s.make_query(q, Event(), title=t, authors=a) + # To run these test use: + # calibre-debug -e src/calibre/ebooks/metadata/sources/isbndb.py + from calibre.ebooks.metadata.sources.test import (test_identify_plugin, + title_test, authors_test) + test_identify_plugin(ISBNDB.name, + [ + + + ( + {'title':'Great Gatsby', + 'authors':['Fitzgerald']}, + [title_test('The great gatsby', exact=True), + authors_test(['F. Scott Fitzgerald'])] + ), + + ( + {'title': 'Flatland', 'authors':['Abbott']}, + [title_test('Flatland', exact=False)] + ), + ]) + From 9bd44ed078134b034da2d82cbe52dbbd7e04622f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 12 Apr 2011 23:24:55 -0600 Subject: [PATCH 30/36] ... --- src/calibre/ebooks/metadata/sources/base.py | 4 ++++ src/calibre/ebooks/metadata/sources/isbndb.py | 6 ++++++ src/calibre/gui2/metadata/config.py | 9 +++++++-- src/calibre/gui2/preferences/metadata_sources.py | 9 ++++++++- src/calibre/gui2/preferences/metadata_sources.ui | 10 ++++++++++ 5 files changed, 35 insertions(+), 3 deletions(-) diff --git a/src/calibre/ebooks/metadata/sources/base.py b/src/calibre/ebooks/metadata/sources/base.py index 5089d8951b..d9144fdf34 100644 --- a/src/calibre/ebooks/metadata/sources/base.py +++ b/src/calibre/ebooks/metadata/sources/base.py @@ -181,6 +181,10 @@ class Source(Plugin): #: construct the configuration widget for this plugin options = () + #: A string that is displayed at the top of the config widget for this + #: plugin + config_help_message = None + def __init__(self, *args, **kwargs): Plugin.__init__(self, *args, **kwargs) diff --git a/src/calibre/ebooks/metadata/sources/isbndb.py b/src/calibre/ebooks/metadata/sources/isbndb.py index a2a10708fb..b8deea56df 100644 --- a/src/calibre/ebooks/metadata/sources/isbndb.py +++ b/src/calibre/ebooks/metadata/sources/isbndb.py @@ -39,6 +39,12 @@ class ISBNDB(Source): 'at isbndb.com and get an access key.')), ) + config_help_message = '

'+_('To use metadata from isbndb.com you must sign' + ' up for a free account and get an isbndb key and enter it below.' + ' Instructions to get the key are ' + 'here.') + + def __init__(self, *args, **kwargs): Source.__init__(self, *args, **kwargs) diff --git a/src/calibre/gui2/metadata/config.py b/src/calibre/gui2/metadata/config.py index 68c935061d..abb45faa46 100644 --- a/src/calibre/gui2/metadata/config.py +++ b/src/calibre/gui2/metadata/config.py @@ -56,7 +56,12 @@ class ConfigWidget(QWidget): self.setLayout(l) self.gb = QGroupBox(_('Downloaded metadata fields'), self) - l.addWidget(self.gb, 0, 0, 1, 2) + if plugin.config_help_message: + self.pchm = QLabel(plugin.config_help_message) + self.pchm.setWordWrap(True) + self.pchm.setOpenExternalLinks(True) + l.addWidget(self.pchm, 0, 0, 1, 2) + l.addWidget(self.gb, l.rowCount(), 0, 1, 2) self.gb.l = QGridLayout() self.gb.setLayout(self.gb.l) self.fields_view = v = QListView(self) @@ -81,7 +86,7 @@ class ConfigWidget(QWidget): widget.setValue(val) elif opt.type == 'string': widget = QLineEdit(self) - widget.setText(val) + widget.setText(val if val else '') elif opt.type == 'bool': widget = QCheckBox(opt.label, self) widget.setChecked(bool(val)) diff --git a/src/calibre/gui2/preferences/metadata_sources.py b/src/calibre/gui2/preferences/metadata_sources.py index 4500a03b30..17a70bcc33 100644 --- a/src/calibre/gui2/preferences/metadata_sources.py +++ b/src/calibre/gui2/preferences/metadata_sources.py @@ -10,7 +10,7 @@ __docformat__ = 'restructuredtext en' from operator import attrgetter from PyQt4.Qt import (QAbstractTableModel, Qt, QAbstractListModel, QWidget, - pyqtSignal, QVBoxLayout, QDialogButtonBox, QFrame, QLabel) + pyqtSignal, QVBoxLayout, QDialogButtonBox, QFrame, QLabel, QIcon) from calibre.gui2.preferences import ConfigWidgetBase, test_widget from calibre.gui2.preferences.metadata_sources_ui import Ui_Form @@ -67,6 +67,13 @@ class SourcesModel(QAbstractTableModel): # {{{ return self.enabled_overrides.get(plugin, orig) elif role == Qt.UserRole: return plugin + elif (role == Qt.DecorationRole and col == 0 and not + plugin.is_configured()): + return QIcon(I('list_remove.png')) + elif role == Qt.ToolTipRole: + if plugin.is_configured(): + return _('This source is configured and ready to go') + return _('This source needs configuration') return NONE def setData(self, index, val, role): diff --git a/src/calibre/gui2/preferences/metadata_sources.ui b/src/calibre/gui2/preferences/metadata_sources.ui index 546120f628..b515f13ba1 100644 --- a/src/calibre/gui2/preferences/metadata_sources.ui +++ b/src/calibre/gui2/preferences/metadata_sources.ui @@ -48,6 +48,16 @@ + + + + Sources with a red X next to their names must be configured before they will be used. + + + true + + + From db2cadd070e4c682e3e0da9f4b4c2747574b7283 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Wed, 13 Apr 2011 10:54:21 +0100 Subject: [PATCH 31/36] Fixes for search: 1) raise an exception if a non-existent field is used. 2) make numeric searches correctly respect None values (python treats None as less than anything) 3) make rating-type columns treat None as zero and zero as False --- src/calibre/library/caches.py | 50 +++++++++++++----------- src/calibre/utils/search_query_parser.py | 17 ++++---- 2 files changed, 37 insertions(+), 30 deletions(-) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 01b7335bf4..af9a766174 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -391,11 +391,11 @@ class ResultCache(SearchQueryParser): # {{{ def build_numeric_relop_dict(self): self.numeric_search_relops = { '=':[1, lambda r, q: r == q], - '>':[1, lambda r, q: r > q], - '<':[1, lambda r, q: r < q], + '>':[1, lambda r, q: r is not None and r > q], + '<':[1, lambda r, q: r is not None and r < q], '!=':[2, lambda r, q: r != q], - '>=':[2, lambda r, q: r >= q], - '<=':[2, lambda r, q: r <= q] + '>=':[2, lambda r, q: r is not None and r >= q], + '<=':[2, lambda r, q: r is not None and r <= q] } def get_numeric_matches(self, location, query, candidates, val_func = None): @@ -406,17 +406,22 @@ class ResultCache(SearchQueryParser): # {{{ if val_func is None: loc = self.field_metadata[location]['rec_index'] val_func = lambda item, loc=loc: item[loc] + dt = self.field_metadata[location]['datatype'] + + q = '' + val_func = lambda item, loc=loc: item[loc] + cast = adjust = lambda x: x if query == 'false': - q = '' - relop = lambda x,y: x is None - val_func = lambda item, loc=loc: item[loc] - cast = adjust = lambda x: x + if dt == 'rating': + relop = lambda x,y: not bool(x) + else: + relop = lambda x,y: x is None elif query == 'true': - q = '' - relop = lambda x,y: x is not None - val_func = lambda item, loc=loc: item[loc] - cast = adjust = lambda x: x + if dt == 'rating': + relop = lambda x,y: bool(x) + else: + relop = lambda x,y: x is not None else: relop = None for k in self.numeric_search_relops.keys(): @@ -426,19 +431,15 @@ class ResultCache(SearchQueryParser): # {{{ if relop is None: (p, relop) = self.numeric_search_relops['='] - dt = self.field_metadata[location]['datatype'] if dt == 'int': - cast = lambda x: int (x) if x is not None else None - adjust = lambda x: x - elif dt == 'rating': cast = lambda x: int (x) + elif dt == 'rating': + cast = lambda x: 0 if x is None else int (x) adjust = lambda x: x/2 elif dt in ('float', 'composite'): - cast = lambda x : float (x) if x is not None else None - adjust = lambda x: x + cast = lambda x : float (x) else: # count operation cast = (lambda x: int (x)) - adjust = lambda x: x if len(query) > 1: mult = query[-1:].lower() @@ -450,7 +451,8 @@ class ResultCache(SearchQueryParser): # {{{ try: q = cast(query) * mult except: - return matches + raise ParseException(query, len(query), + 'Non-numeric value in query', self) for id_ in candidates: item = self._data[id_] @@ -459,11 +461,14 @@ class ResultCache(SearchQueryParser): # {{{ try: v = cast(val_func(item)) except: - v = 0 + v = None if v: v = adjust(v) if relop(v, q): matches.add(item[0]) + print v, q, 'YES' + else: + print v, q, 'NO' return matches def get_user_category_matches(self, location, query, candidates): @@ -590,8 +595,7 @@ class ResultCache(SearchQueryParser): # {{{ candidates = self.universal_set() if len(candidates) == 0: return matches - if location not in self.all_search_locations: - return matches + self.test_location_is_valid(location, query) if len(location) > 2 and location.startswith('@') and \ location[1:] in self.db_prefs['grouped_search_terms']: diff --git a/src/calibre/utils/search_query_parser.py b/src/calibre/utils/search_query_parser.py index a50ca20fc1..387ad1487e 100644 --- a/src/calibre/utils/search_query_parser.py +++ b/src/calibre/utils/search_query_parser.py @@ -20,7 +20,7 @@ import sys, string, operator from calibre.utils.pyparsing import CaselessKeyword, Group, Forward, \ CharsNotIn, Suppress, OneOrMore, MatchFirst, CaselessLiteral, \ - Optional, NoMatch, ParseException, QuotedString + Optional, NoMatch, ParseException, QuotedString, Word from calibre.constants import preferred_encoding from calibre.utils.icu import sort_key @@ -128,12 +128,8 @@ class SearchQueryParser(object): self._tests_failed = False self.optimize = optimize # Define a token - standard_locations = map(lambda x : CaselessLiteral(x)+Suppress(':'), - locations) - location = NoMatch() - for l in standard_locations: - location |= l - location = Optional(location, default='all') + self.standard_locations = locations + location = Optional(Word(string.ascii_letters+'#')+Suppress(':'), default='all') word_query = CharsNotIn(string.whitespace + '()') #quoted_query = Suppress('"')+CharsNotIn('"')+Suppress('"') quoted_query = QuotedString('"', escChar='\\') @@ -250,7 +246,14 @@ class SearchQueryParser(object): raise ParseException(query, len(query), 'undefined saved search', self) return self._get_matches(location, query, candidates) + def test_location_is_valid(self, location, query): + if location not in self.standard_locations: + raise ParseException(query, len(query), + _('No column exists with lookup name ') + location, self) + def _get_matches(self, location, query, candidates): + location = location.lower() + self.test_location_is_valid(location, query) if self.optimize: return self.get_matches(location, query, candidates=candidates) else: From ae3e40eccb5e28641275d22773e714f3a6a46dd4 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Wed, 13 Apr 2011 11:39:51 +0100 Subject: [PATCH 32/36] ... --- src/calibre/library/caches.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index af9a766174..5f4cfcba07 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -466,9 +466,6 @@ class ResultCache(SearchQueryParser): # {{{ v = adjust(v) if relop(v, q): matches.add(item[0]) - print v, q, 'YES' - else: - print v, q, 'NO' return matches def get_user_category_matches(self, location, query, candidates): From e753b0fe2a39a139ea4221a4904624dd1b063bef Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Wed, 13 Apr 2011 11:47:09 +0100 Subject: [PATCH 33/36] Add 'size' to mi produced by get_metadata. --- src/calibre/library/database2.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 50b404b4be..7b4d52dbcd 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -854,6 +854,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): mi.uuid = row[fm['uuid']] mi.title_sort = row[fm['sort']] mi.last_modified = row[fm['last_modified']] + mi.size = row[fm['size']] formats = row[fm['formats']] if not formats: formats = None From d73afc11da4336fde2a8aaa0f2bf8c883d8be406 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 13 Apr 2011 08:53:23 -0600 Subject: [PATCH 34/36] Update Tabu.ro --- recipes/tabu.recipe | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/recipes/tabu.recipe b/recipes/tabu.recipe index d0ede613fd..f98ed8a155 100644 --- a/recipes/tabu.recipe +++ b/recipes/tabu.recipe @@ -24,30 +24,29 @@ class TabuRo(BasicNewsRecipe): cover_url = 'http://www.tabu.ro/img/tabu-logo2.png' conversion_options = { - 'comments' : description - ,'tags' : category - ,'language' : language - ,'publisher' : publisher - } + 'comments' : description + ,'tags' : category + ,'language' : language + ,'publisher' : publisher + } keep_only_tags = [ - dict(name='div', attrs={'id':'Article'}), - ] + dict(name='h2', attrs={'class':'articol_titlu'}), + dict(name='div', attrs={'class':'poza_articol_featured'}), + dict(name='div', attrs={'class':'articol_text'}) + ] remove_tags = [ - dict(name='div', attrs={'id':['advertisementArticle']}), - dict(name='div', attrs={'class':'voting_number'}), - dict(name='div', attrs={'id':'number_votes'}), - dict(name='div', attrs={'id':'rating_one'}), - dict(name='div', attrs={'class':'float: right;'}) + dict(name='div', attrs={'class':'asemanatoare'}) ] remove_tags_after = [ dict(name='div', attrs={'id':'comments'}), - ] + dict(name='div', attrs={'class':'asemanatoare'}) + ] feeds = [ - (u'Feeds', u'http://www.tabu.ro/rss_all.xml') + (u'Feeds', u'http://www.tabu.ro/feed/') ] def preprocess_html(self, soup): From 1c38548fd769d55d9598c2c6c6fc2113e5e33a0f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 13 Apr 2011 09:12:20 -0600 Subject: [PATCH 35/36] Make the icons in the status bar look a little nicer on OS X --- src/calibre/gui2/init.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/calibre/gui2/init.py b/src/calibre/gui2/init.py index 80f1f1c2cf..a75ff01b21 100644 --- a/src/calibre/gui2/init.py +++ b/src/calibre/gui2/init.py @@ -247,6 +247,11 @@ class LayoutMixin(object): # {{{ for x in ('cb', 'tb', 'bd'): button = getattr(self, x+'_splitter').button button.setIconSize(QSize(24, 24)) + if isosx: + button.setStyleSheet(''' + QToolButton { background: none; border:none; padding: 0px; } + QToolButton:checked { background: rgba(0, 0, 0, 25%); } + ''') self.status_bar.addPermanentWidget(button) self.status_bar.addPermanentWidget(self.jobs_button) self.setStatusBar(self.status_bar) From 50acfc314d2b510b92107bee4ffdcb41d10bf08b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 13 Apr 2011 09:15:42 -0600 Subject: [PATCH 36/36] ... --- src/calibre/gui2/layout.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/gui2/layout.py b/src/calibre/gui2/layout.py index e98817a02f..c01e708933 100644 --- a/src/calibre/gui2/layout.py +++ b/src/calibre/gui2/layout.py @@ -408,6 +408,7 @@ class ToolBar(BaseToolBar): # {{{ self.d_widget.layout().addWidget(self.donate_button) if isosx: self.d_widget.setStyleSheet('QWidget, QToolButton {background-color: none; border: none; }') + self.d_widget.layout().addWidget(QLabel(u'\u00a0')) bar.addWidget(self.d_widget) self.showing_donate = True elif what in self.gui.iactions: