From aac23508dff35df7ccec7c319d4586f5b7e58f23 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 11 Apr 2011 15:43:34 -0600 Subject: [PATCH 01/10] Fix regression that caused clicking auto send to also change the email address in Preferences->Email --- src/calibre/gui2/preferences/emailp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/preferences/emailp.py b/src/calibre/gui2/preferences/emailp.py index ded6891387..f5c7b5a3a7 100644 --- a/src/calibre/gui2/preferences/emailp.py +++ b/src/calibre/gui2/preferences/emailp.py @@ -88,7 +88,7 @@ class EmailAccounts(QAbstractTableModel): # {{{ self.subjects[account] = unicode(value.toString()) elif col == 1: self.accounts[account][0] = unicode(value.toString()).upper() - else: + elif col == 0: na = unicode(value.toString()) from email.utils import parseaddr addr = parseaddr(na)[-1] @@ -100,7 +100,7 @@ class EmailAccounts(QAbstractTableModel): # {{{ self.accounts[na][0] = 'AZW, MOBI, TPZ, PRC, AZW1' self.dataChanged.emit( - self.index(index.row(), 0), self.index(index.row(), 2)) + self.index(index.row(), 0), self.index(index.row(), 3)) return True def make_default(self, index): From 64dd32eaf5629f997b4f683c054061e66dff1b28 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 11 Apr 2011 15:43:58 -0600 Subject: [PATCH 02/10] ... --- src/calibre/gui2/preferences/emailp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/preferences/emailp.py b/src/calibre/gui2/preferences/emailp.py index f5c7b5a3a7..1644dc6b73 100644 --- a/src/calibre/gui2/preferences/emailp.py +++ b/src/calibre/gui2/preferences/emailp.py @@ -84,7 +84,7 @@ class EmailAccounts(QAbstractTableModel): # {{{ account = self.account_order[row] if col == 3: self.accounts[account][1] ^= True - if col == 2: + elif col == 2: self.subjects[account] = unicode(value.toString()) elif col == 1: self.accounts[account][0] = unicode(value.toString()).upper() From 71967fd4ba173c5ad9c82f60a166b881fe29809d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 11 Apr 2011 15:53:34 -0600 Subject: [PATCH 03/10] New framework for running I/O bound jobs in threads inthe calibre main process. Migrate email sending to the new framework. --- src/calibre/gui2/dialogs/job_view.ui | 40 ++-- src/calibre/gui2/email.py | 226 ++++++------------- src/calibre/gui2/jobs.py | 85 +++++-- src/calibre/gui2/metadata/bulk_download2.py | 11 + src/calibre/gui2/threaded_jobs.py | 238 ++++++++++++++++++++ src/calibre/gui2/ui.py | 3 +- src/calibre/utils/ipc/job.py | 10 +- 7 files changed, 410 insertions(+), 203 deletions(-) create mode 100644 src/calibre/gui2/metadata/bulk_download2.py create mode 100644 src/calibre/gui2/threaded_jobs.py diff --git a/src/calibre/gui2/dialogs/job_view.ui b/src/calibre/gui2/dialogs/job_view.ui index 8b54c23573..1e854c0f29 100644 --- a/src/calibre/gui2/dialogs/job_view.ui +++ b/src/calibre/gui2/dialogs/job_view.ui @@ -1,7 +1,8 @@ - + + Dialog - - + + 0 0 @@ -9,38 +10,41 @@ 462 - + Details of job - - + + :/images/view.png:/images/view.png - - - - + + + + false - + QPlainTextEdit::NoWrap - + true - - - + + + QDialogButtonBox::Ok + + + - + @@ -49,11 +53,11 @@ Dialog accept() - + 617 442 - + 206 -5 diff --git a/src/calibre/gui2/email.py b/src/calibre/gui2/email.py index c6d58fa340..c8adeb7d31 100644 --- a/src/calibre/gui2/email.py +++ b/src/calibre/gui2/email.py @@ -6,9 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, socket, time, cStringIO -from threading import Thread -from Queue import Queue +import os, socket, time from binascii import unhexlify from functools import partial from itertools import repeat @@ -16,67 +14,20 @@ from itertools import repeat from calibre.utils.smtp import compose_mail, sendmail, extract_email_address, \ config as email_config from calibre.utils.filenames import ascii_filename -from calibre.utils.ipc.job import BaseJob -from calibre.ptempfile import PersistentTemporaryFile from calibre.customize.ui import available_input_formats, available_output_formats from calibre.ebooks.metadata import authors_to_string from calibre.constants import preferred_encoding from calibre.gui2 import config, Dispatcher, warning_dialog from calibre.library.save_to_disk import get_components from calibre.utils.config import tweaks +from calibre.gui2.threaded_jobs import ThreadedJob -class EmailJob(BaseJob): # {{{ - - def __init__(self, callback, description, attachment, aname, to, subject, text, job_manager): - BaseJob.__init__(self, description) - self.exception = None - self.job_manager = job_manager - self.email_args = (attachment, aname, to, subject, text) - self.email_sent_callback = callback - self.log_path = None - self._log_file = cStringIO.StringIO() - self._log_file.write(self.description.encode('utf-8') + '\n') - - @property - def log_file(self): - if self.log_path is not None: - return open(self.log_path, 'rb') - return cStringIO.StringIO(self._log_file.getvalue()) - - def start_work(self): - self.start_time = time.time() - self.job_manager.changed_queue.put(self) - - def job_done(self): - self.duration = time.time() - self.start_time - self.percent = 1 - # Dump log onto disk - lf = PersistentTemporaryFile('email_log') - lf.write(self._log_file.getvalue()) - lf.close() - self.log_path = lf.name - self._log_file.close() - self._log_file = None - - self.job_manager.changed_queue.put(self) - - def log_write(self, what): - self._log_file.write(what) - -# }}} - -class Emailer(Thread): # {{{ +class Sendmail(object): MAX_RETRIES = 1 - def __init__(self, job_manager): - Thread.__init__(self) - self.daemon = True - self.jobs = Queue() - self.job_manager = job_manager - self._run = True + def __init__(self): self.calculate_rate_limit() - self.last_send_time = time.time() - self.rate_limit def calculate_rate_limit(self): @@ -87,70 +38,28 @@ class Emailer(Thread): # {{{ 'gmail.com' in rh or 'live.com' in rh): self.rate_limit = tweaks['public_smtp_relay_delay'] - def stop(self): - self._run = False - self.jobs.put(None) + def __call__(self, attachment, aname, to, subject, text, log=None, + abort=None, notifications=None): - def run(self): - while self._run: + try_count = 0 + while try_count <= self.MAX_RETRIES: + if try_count > 0: + log('\nRetrying in %d seconds...\n' % + self.rate_limit) try: - job = self.jobs.get() + self.sendmail(attachment, aname, to, subject, text, log) + try_count = self.MAX_RETRIES + log('Email successfully sent') except: - break - if job is None or not self._run: - break - try_count = 0 - failed, exc = False, None - job.start_work() - if job.kill_on_start: - job.log_write('Aborted\n') - job.failed = failed - job.killed = True - job.job_done() - continue + if abort.is_set(): + return + if try_count == self.MAX_RETRIES: + raise + log.exception('\nSending failed...\n') - while try_count <= self.MAX_RETRIES: - failed = False - if try_count > 0: - job.log_write('\nRetrying in %d seconds...\n' % - self.rate_limit) - try: - self.sendmail(job) - break - except Exception as e: - if not self._run: - return - import traceback - failed = True - exc = e - job.log_write('\nSending failed...\n') - job.log_write(traceback.format_exc()) + try_count += 1 - try_count += 1 - - if not self._run: - break - - job.failed = failed - job.exception = exc - job.job_done() - try: - job.email_sent_callback(job) - except: - import traceback - traceback.print_exc() - - def send_mails(self, jobnames, callback, attachments, to_s, subjects, - texts, attachment_names): - for name, attachment, to, subject, text, aname in zip(jobnames, - attachments, to_s, subjects, texts, attachment_names): - description = _('Email %s to %s') % (name, to) - job = EmailJob(callback, description, attachment, aname, to, - subject, text, self.job_manager) - self.job_manager.add_job(job) - self.jobs.put(job) - - def sendmail(self, job): + def sendmail(self, attachment, aname, to, subject, text, log): while time.time() - self.last_send_time <= self.rate_limit: time.sleep(1) try: @@ -158,7 +67,6 @@ class Emailer(Thread): # {{{ from_ = opts.from_ if not from_: from_ = 'calibre ' - attachment, aname, to, subject, text = job.email_args msg = compose_mail(from_, to, text, subject, open(attachment, 'rb'), aname) efrom, eto = map(extract_email_address, (from_, to)) @@ -169,48 +77,56 @@ class Emailer(Thread): # {{{ username=opts.relay_username, password=unhexlify(opts.relay_password), port=opts.relay_port, encryption=opts.encryption, - debug_output=partial(print, file=job._log_file)) + debug_output=log.debug) finally: self.last_send_time = time.time() - def email_news(self, mi, remove, get_fmts, done): - opts = email_config().parse() - accounts = [(account, [x.strip().lower() for x in x[0].split(',')]) - for account, x in opts.accounts.items() if x[1]] - sent_mails = [] - for i, x in enumerate(accounts): - account, fmts = x - files = get_fmts(fmts) - files = [f for f in files if f is not None] - if not files: - continue - attachment = files[0] - to_s = [account] - subjects = [_('News:')+' '+mi.title] - texts = [ - _('Attached is the %s periodical downloaded by calibre.') - % (mi.title,) - ] - attachment_names = [ascii_filename(mi.title)+os.path.splitext(attachment)[1]] - attachments = [attachment] - jobnames = [mi.title] - do_remove = [] - if i == len(accounts) - 1: - do_remove = remove - self.send_mails(jobnames, - Dispatcher(partial(done, remove=do_remove)), - attachments, to_s, subjects, texts, attachment_names) - sent_mails.append(to_s[0]) - return sent_mails +gui_sendmail = Sendmail() -# }}} +def send_mails(jobnames, callback, attachments, to_s, subjects, + texts, attachment_names, job_manager): + for name, attachment, to, subject, text, aname in zip(jobnames, + attachments, to_s, subjects, texts, attachment_names): + description = _('Email %s to %s') % (name, to) + job = ThreadedJob('email', description, gui_sendmail, (attachment, aname, to, + subject, text), {}, callback, killable=False) + job_manager.run_threaded_job(job) + + +def email_news(mi, remove, get_fmts, done, job_manager): + opts = email_config().parse() + accounts = [(account, [x.strip().lower() for x in x[0].split(',')]) + for account, x in opts.accounts.items() if x[1]] + sent_mails = [] + for i, x in enumerate(accounts): + account, fmts = x + files = get_fmts(fmts) + files = [f for f in files if f is not None] + if not files: + continue + attachment = files[0] + to_s = [account] + subjects = [_('News:')+' '+mi.title] + texts = [ + _('Attached is the %s periodical downloaded by calibre.') + % (mi.title,) + ] + attachment_names = [ascii_filename(mi.title)+os.path.splitext(attachment)[1]] + attachments = [attachment] + jobnames = [mi.title] + do_remove = [] + if i == len(accounts) - 1: + do_remove = remove + send_mails(jobnames, + Dispatcher(partial(done, remove=do_remove)), + attachments, to_s, subjects, texts, attachment_names, + job_manager) + sent_mails.append(to_s[0]) + return sent_mails class EmailMixin(object): # {{{ - def __init__(self): - self.emailer = Emailer(self.job_manager) - def send_by_mail(self, to, fmts, delete_from_library, subject='', send_ids=None, do_auto_convert=True, specific_format=None): ids = [self.library_view.model().id(r) for r in self.library_view.selectionModel().selectedRows()] if send_ids is None else send_ids @@ -246,8 +162,7 @@ class EmailMixin(object): # {{{ components = get_components(subject, mi, id) if not components: components = [mi.title] - subject = os.path.join(*components) - subjects.append(subject) + subjects.append(os.path.join(*components)) a = authors_to_string(mi.authors if mi.authors else \ [_('Unknown')]) texts.append(_('Attached, you will find the e-book') + \ @@ -262,11 +177,10 @@ class EmailMixin(object): # {{{ to_s = list(repeat(to, len(attachments))) if attachments: - if not self.emailer.is_alive(): - self.emailer.start() - self.emailer.send_mails(jobnames, + send_mails(jobnames, Dispatcher(partial(self.email_sent, remove=remove)), - attachments, to_s, subjects, texts, attachment_names) + attachments, to_s, subjects, texts, attachment_names, + self.job_manager) self.status_bar.show_message(_('Sending email to')+' '+to, 3000) auto = [] @@ -334,10 +248,8 @@ class EmailMixin(object): # {{{ files, auto = self.library_view.model().\ get_preferred_formats_from_ids([id_], fmts) return files - if not self.emailer.is_alive(): - self.emailer.start() - sent_mails = self.emailer.email_news(mi, remove, - get_fmts, self.email_sent) + sent_mails = email_news(mi, remove, + get_fmts, self.email_sent, self.job_manager) if sent_mails: self.status_bar.show_message(_('Sent news to')+' '+\ ', '.join(sent_mails), 3000) diff --git a/src/calibre/gui2/jobs.py b/src/calibre/gui2/jobs.py index dbde030e81..34eef4406a 100644 --- a/src/calibre/gui2/jobs.py +++ b/src/calibre/gui2/jobs.py @@ -8,14 +8,13 @@ Job management. ''' import re - from Queue import Empty, Queue -from PyQt4.Qt import QAbstractTableModel, QVariant, QModelIndex, Qt, \ - QTimer, pyqtSignal, QIcon, QDialog, QAbstractItemDelegate, QApplication, \ - QSize, QStyleOptionProgressBarV2, QString, QStyle, QToolTip, QFrame, \ - QHBoxLayout, QVBoxLayout, QSizePolicy, QLabel, QCoreApplication, QAction, \ - QByteArray +from PyQt4.Qt import (QAbstractTableModel, QVariant, QModelIndex, Qt, + QTimer, pyqtSignal, QIcon, QDialog, QAbstractItemDelegate, QApplication, + QSize, QStyleOptionProgressBarV2, QString, QStyle, QToolTip, QFrame, + QHBoxLayout, QVBoxLayout, QSizePolicy, QLabel, QCoreApplication, QAction, + QByteArray) from calibre.utils.ipc.server import Server from calibre.utils.ipc.job import ParallelJob @@ -25,8 +24,9 @@ from calibre.gui2.dialogs.jobs_ui import Ui_JobsDialog from calibre import __appname__ from calibre.gui2.dialogs.job_view_ui import Ui_Dialog from calibre.gui2.progress_indicator import ProgressIndicator +from calibre.gui2.threaded_jobs import ThreadedJobServer, ThreadedJob -class JobManager(QAbstractTableModel): +class JobManager(QAbstractTableModel): # {{{ job_added = pyqtSignal(int) job_done = pyqtSignal(int) @@ -42,6 +42,7 @@ class JobManager(QAbstractTableModel): self.add_job = Dispatcher(self._add_job) self.server = Server(limit=int(config['worker_limit']/2.0), enforce_cpu_limit=config['enforce_cpu_limit']) + self.threaded_server = ThreadedJobServer() self.changed_queue = Queue() self.timer = QTimer(self) @@ -146,12 +147,21 @@ class JobManager(QAbstractTableModel): jobs.add(self.server.changed_jobs_queue.get_nowait()) except Empty: break + + # Update device jobs while True: try: jobs.add(self.changed_queue.get_nowait()) except Empty: break + # Update threaded jobs + while True: + try: + jobs.add(self.threaded_server.changed_jobs.get_nowait()) + except Empty: + break + if jobs: needs_reset = False for job in jobs: @@ -207,11 +217,22 @@ class JobManager(QAbstractTableModel): self.server.add_job(job) return job + def run_threaded_job(self, job): + self.add_job(job) + self.threaded_server.add_job(job) + def launch_gui_app(self, name, args=[], kwargs={}, description=''): job = ParallelJob(name, description, lambda x: x, args=args, kwargs=kwargs) self.server.run_job(job, gui=True, redirect_output=False) + def _kill_job(self, job): + if isinstance(job, ParallelJob): + self.server.kill_job(job) + elif isinstance(job, ThreadedJob): + self.threaded_server.kill_job(job) + else: + job.kill_on_start = True def kill_job(self, row, view): job = self.jobs[row] @@ -221,29 +242,29 @@ class JobManager(QAbstractTableModel): if job.duration is not None: return error_dialog(view, _('Cannot kill job'), _('Job has already run')).exec_() - if isinstance(job, ParallelJob): - self.server.kill_job(job) - else: - job.kill_on_start = True + if not getattr(job, 'killable', True): + return error_dialog(view, _('Cannot kill job'), + _('This job cannot be stopped'), show=True) + self._kill_job(job) def kill_all_jobs(self): for job in self.jobs: - if isinstance(job, DeviceJob) or job.duration is not None: + if (isinstance(job, DeviceJob) or job.duration is not None or + not getattr(job, 'killable', True)): continue - if isinstance(job, ParallelJob): - self.server.kill_job(job) - else: - job.kill_on_start = True + self._kill_job(job) def terminate_all_jobs(self): self.server.killall() for job in self.jobs: - if isinstance(job, DeviceJob) or job.duration is not None: + if (isinstance(job, DeviceJob) or job.duration is not None or + not getattr(job, 'killable', True)): continue if not isinstance(job, ParallelJob): - job.kill_on_start = True - + self._kill_job(job) +# }}} +# Jobs UI {{{ class ProgressBarDelegate(QAbstractItemDelegate): def sizeHint(self, option, index): @@ -269,6 +290,11 @@ class DetailView(QDialog, Ui_Dialog): self.setupUi(self) self.setWindowTitle(job.description) self.job = job + self.html_view = hasattr(job, 'html_details') + if self.html_view: + self.log.setVisible(False) + else: + self.tb.setVisible(False) self.next_pos = 0 self.update() self.timer = QTimer(self) @@ -277,12 +303,19 @@ class DetailView(QDialog, Ui_Dialog): def update(self): - f = self.job.log_file - f.seek(self.next_pos) - more = f.read() - self.next_pos = f.tell() - if more: - self.log.appendPlainText(more.decode('utf-8', 'replace')) + if self.html_view: + html = self.job.html_details + if len(html) > self.next_pos: + self.next_pos = len(html) + self.tb.setHtml( + '
%s
'%html) + else: + f = self.job.log_file + f.seek(self.next_pos) + more = f.read() + self.next_pos = f.tell() + if more: + self.log.appendPlainText(more.decode('utf-8', 'replace')) class JobsButton(QFrame): @@ -441,3 +474,5 @@ class JobsDialog(QDialog, Ui_JobsDialog): def hide(self, *args): self.save_state() return QDialog.hide(self, *args) +# }}} + diff --git a/src/calibre/gui2/metadata/bulk_download2.py b/src/calibre/gui2/metadata/bulk_download2.py new file mode 100644 index 0000000000..cc6da1e995 --- /dev/null +++ b/src/calibre/gui2/metadata/bulk_download2.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + + + diff --git a/src/calibre/gui2/threaded_jobs.py b/src/calibre/gui2/threaded_jobs.py new file mode 100644 index 0000000000..f29baf4134 --- /dev/null +++ b/src/calibre/gui2/threaded_jobs.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import os, time, tempfile, json +from threading import Thread, RLock, Event +from Queue import Queue + +from calibre.utils.ipc.job import BaseJob +from calibre.utils.logging import GUILog +from calibre.ptempfile import base_dir + +class ThreadedJob(BaseJob): + + def __init__(self, + type_, description, + + func, args, kwargs, + + callback, + + max_concurrent_count=1, + killable=True, + log=None): + ''' + A job that is run in its own thread in the calibre main process + + :param type_: The type of this job (a string). The type is used in + conjunction with max_concurrent_count to prevent too many jobs of the + same type from running + + :param description: A user viewable job description + + :func: The function that actually does the work. This function *must* + accept at least three keyword arguments: abort, log and notifications. abort is + An Event object. func should periodically check abort.is_set(0 and if + it is True, it should stop processing as soon as possible. notifications + is a Queue. func should put progress notifications into it in the form + of a tuple (frac, msg). frac is a number between 0 and 1 indicating + progress and msg is a string describing the progress. log is a Log + object which func should use for all debugging output. func should + raise an Exception to indicate failure. This exception is stored in + job.exception and can thus be used to pass arbitrary information to + callback. + + :param args,kwargs: These are passed to func when it is called + + :param callback: A callable that is called on completion of this job. + Note that it is not called if the user kills the job. Check job.failed + to see if the job succeeded or not. And use job.log to get the job log. + + :param killable: If False the GUI wont let the user kill this job + + :param log: Must be a subclass of GUILog or None. If None a default + GUILog is created. + ''' + BaseJob.__init__(self, description) + + self.type = type_ + self.max_concurrent_count = max_concurrent_count + self.killable = killable + self.callback = callback + self.abort = Event() + self.exception = None + + kwargs['notifications'] = self.notifications + kwargs['abort'] = self.abort + self.log = GUILog() if log is None else log + kwargs['log'] = self.log + + self.func, self.args, self.kwargs = func, args, kwargs + self.consolidated_log = None + + def start_work(self): + self.start_time = time.time() + self.log('Starting job:', self.description) + try: + self.result = self.func(*self.args, **self.kwargs) + except Exception as e: + self.exception = e + self.failed = True + self.log.exception('Job: "%s" failed with error:'%self.description) + self.log.debug('Called with args:', self.args, self.kwargs) + + self.duration = time.time() - self.start_time + try: + self.callback(self) + except: + pass + self._cleanup() + + def _cleanup(self): + + try: + self.consolidate_log() + except: + self.log.exception('Log consolidation failed') + + # No need to keep references to these around anymore + self.func = self.args = self.kwargs = self.notifications = None + + def kill(self): + if self.start_time is None: + self.start_time = time.time() + self.duration = 0.0001 + else: + self.duration = time.time() - self.start_time() + self.abort.set() + + self.log('Aborted job:', self.description) + self.killed = True + self.failed = True + self._cleanup() + + def consolidate_log(self): + logs = [self.log.html, self.log.plain_text] + bdir = base_dir() + log_dir = os.path.join(bdir, 'threaded_job_logs') + if not os.path.exists(log_dir): + os.makedirs(log_dir) + fd, path = tempfile.mkstemp(suffix='.json', prefix='log-', dir=log_dir) + with os.fdopen(fd, 'wb') as f: + f.write(json.dumps(logs, ensure_ascii=False, + indent=2).encode('utf-8')) + self.consolidated_log = path + self.log = None + + def read_consolidated_log(self): + with open(self.consolidated_log, 'rb') as f: + return json.loads(f.read().decode('utf-8')) + + @property + def details(self): + if self.consolidated_log is None: + return self.log.plain_text + return self.read_consolidated_log()[1] + + @property + def html_details(self): + if self.consolidated_log is None: + return self.log.html + return self.read_consolidated_log()[0] + +class ThreadedJobWorker(Thread): + + def __init__(self, job): + Thread.__init__(self) + self.daemon = True + self.job = job + + def run(self): + try: + self.job.start_work() + except: + import traceback + from calibre import prints + prints('Job had unhandled exception:', self.job.description) + traceback.print_exc() + +class ThreadedJobServer(Thread): + + def __init__(self): + Thread.__init__(self) + self.daemon = True + self.lock = RLock() + + self.queued_jobs = [] + self.running_jobs = set() + self.changed_jobs = Queue() + self.keep_going = True + + def close(self): + self.keep_going = False + + def add_job(self, job): + with self.lock: + self.queued_jobs.append(job) + + if not self.is_alive(): + self.start() + + def run(self): + while self.keep_going: + self.run_once() + time.sleep(0.1) + + def run_once(self): + with self.lock: + remove = set() + for worker in self.running_jobs: + if worker.is_alive(): + # Get progress notifications + if worker.job.consume_notifications(): + self.changed_jobs.put(worker.job) + else: + remove.add(worker) + self.changed_jobs.put(worker.job) + + for worker in remove: + self.running_jobs.remove(worker) + + jobs = self.get_startable_jobs() + for job in jobs: + w = ThreadedJobWorker(job) + w.start() + self.running_jobs.add(w) + self.changed_jobs.put(job) + self.queued_jobs.remove(job) + + def kill_job(self, job): + with self.lock: + if job in self.queued_jobs: + self.queued_jobs.remove(job) + elif job in self.running_jobs: + self.running_jobs.remove(job) + job.kill() + self.changed_jobs.put(job) + + def running_jobs_of_type(self, type_): + return len([w for w in self.running_jobs if w.job.type == type_]) + + def get_startable_jobs(self): + queued_types = [] + ans = [] + for job in self.queued_jobs: + num = self.running_jobs_of_type(job.type) + num += queued_types.count(job.type) + if num < job.max_concurrent_count: + queued_types.append(job.type) + ans.append(job) + return ans + + diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 4d363c283a..e7853b9491 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -608,6 +608,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ self.update_checker.terminate() self.listener.close() self.job_manager.server.close() + self.job_manager.threaded_server.close() while self.spare_servers: self.spare_servers.pop().close() self.device_manager.keep_going = False @@ -616,8 +617,6 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ mb.stop() self.hide_windows() - if self.emailer.is_alive(): - self.emailer.stop() try: try: if self.content_server is not None: diff --git a/src/calibre/utils/ipc/job.py b/src/calibre/utils/ipc/job.py index 91db333791..f4b54aee95 100644 --- a/src/calibre/utils/ipc/job.py +++ b/src/calibre/utils/ipc/job.py @@ -75,12 +75,20 @@ class BaseJob(object): self._run_state = self.RUNNING self._status_text = _('Working...') - while consume_notifications: + if consume_notifications: + return self.consume_notifications() + return False + + def consume_notifications(self): + got_notification = False + while self.notifications is not None: try: self.percent, self._message = self.notifications.get_nowait() self.percent *= 100. + got_notification = True except Empty: break + return got_notification @property def status_text(self): From d7438cbc4922469888cb2579c2dd2cac2adad9a9 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 11 Apr 2011 17:19:38 -0600 Subject: [PATCH 04/10] ... --- src/calibre/gui2/actions/edit_metadata.py | 7 +++- src/calibre/gui2/metadata/bulk_download2.py | 39 +++++++++++++++++++++ src/calibre/gui2/threaded_jobs.py | 5 ++- 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/actions/edit_metadata.py b/src/calibre/gui2/actions/edit_metadata.py index fc663d268a..09040bcafc 100644 --- a/src/calibre/gui2/actions/edit_metadata.py +++ b/src/calibre/gui2/actions/edit_metadata.py @@ -10,7 +10,7 @@ from functools import partial from PyQt4.Qt import Qt, QMenu, QModelIndex -from calibre.gui2 import error_dialog, config +from calibre.gui2 import error_dialog, config, Dispatcher from calibre.gui2.dialogs.metadata_single import MetadataSingleDialog from calibre.gui2.dialogs.metadata_bulk import MetadataBulkDialog from calibre.gui2.dialogs.confirm_delete import confirm @@ -88,6 +88,11 @@ class EditMetadataAction(InterfaceAction): _('No books selected'), show=True) db = self.gui.library_view.model().db ids = [db.id(row.row()) for row in rows] + from calibre.gui2.metadata.bulk_download2 import start_download + start_download(self.gui, ids, Dispatcher(self.bulk_metadata_downloaded)) + + def bulk_metadata_downloaded(self, job): + print repr(job.result) def download_metadata_old(self, checked, covers=True, set_metadata=True, set_social_metadata=None): diff --git a/src/calibre/gui2/metadata/bulk_download2.py b/src/calibre/gui2/metadata/bulk_download2.py index cc6da1e995..d691c651d9 100644 --- a/src/calibre/gui2/metadata/bulk_download2.py +++ b/src/calibre/gui2/metadata/bulk_download2.py @@ -7,5 +7,44 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' +from functools import partial + +from PyQt4.Qt import QIcon + +from calibre.gui2.dialogs.message_box import MessageBox +from calibre.gui2.threaded_jobs import ThreadedJob + +def show_config(gui, parent): + from calibre.gui2.preferences import show_config_widget + show_config_widget('Sharing', 'Metadata download', parent=parent, + gui=gui, never_shutdown=True) + +def start_download(gui, ids, callback): + q = MessageBox(MessageBox.QUESTION, _('Schedule download?'), + _('The download of metadata for %d book(s) will' + ' run in the background. Proceed?')%len(ids), + show_copy_button=False, parent=gui) + b = q.bb.addButton(_('Configure download'), q.bb.ActionRole) + b.setIcon(QIcon(I('config.png'))) + b.clicked.connect(partial(show_config, gui, q)) + q.det_msg_toggle.setVisible(False) + + ret = q.exec_() + b.clicked.disconnect() + if ret != q.Accepted: + return + + job = ThreadedJob('metadata bulk download', + _('Download metadata for %d books')%len(ids), + download, (ids, gui.current_db), {}, callback) + gui.job_manager.run_threaded_job(job) + + +def download(ids, db, log=None, abort=None, notifications=None): + ids = list(ids) + metadata = [db.get_metadata(i, index_is_id=True, get_user_categories=False) + for i in ids] + return (ids, [mi.last_modified for mi in metadata]) + diff --git a/src/calibre/gui2/threaded_jobs.py b/src/calibre/gui2/threaded_jobs.py index f29baf4134..f98488da79 100644 --- a/src/calibre/gui2/threaded_jobs.py +++ b/src/calibre/gui2/threaded_jobs.py @@ -91,7 +91,8 @@ class ThreadedJob(BaseJob): try: self.callback(self) except: - pass + import traceback + traceback.print_exc() self._cleanup() def _cleanup(self): @@ -103,6 +104,8 @@ class ThreadedJob(BaseJob): # No need to keep references to these around anymore self.func = self.args = self.kwargs = self.notifications = None + # We can't delete self.callback as it might be a Dispatch object and if + # it is garbage collected it won't work def kill(self): if self.start_time is None: From c21095c457bdc30bc98c18aa410daeb4ef3bee03 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 11 Apr 2011 21:42:25 -0600 Subject: [PATCH 05/10] End of day --- src/calibre/gui2/actions/edit_metadata.py | 6 +- src/calibre/gui2/metadata/bulk_download2.py | 78 +++++++++++++++++++-- 2 files changed, 79 insertions(+), 5 deletions(-) diff --git a/src/calibre/gui2/actions/edit_metadata.py b/src/calibre/gui2/actions/edit_metadata.py index 09040bcafc..f5e9e8c4a0 100644 --- a/src/calibre/gui2/actions/edit_metadata.py +++ b/src/calibre/gui2/actions/edit_metadata.py @@ -92,7 +92,11 @@ class EditMetadataAction(InterfaceAction): start_download(self.gui, ids, Dispatcher(self.bulk_metadata_downloaded)) def bulk_metadata_downloaded(self, job): - print repr(job.result) + if job.failed: + self.job_exception(job, dialog_title=_('Failed to download metadata')) + return + from calibre.gui2.metadata.bulk_download2 import proceed + proceed(self.gui, job) def download_metadata_old(self, checked, covers=True, set_metadata=True, set_social_metadata=None): diff --git a/src/calibre/gui2/metadata/bulk_download2.py b/src/calibre/gui2/metadata/bulk_download2.py index d691c651d9..cb7f1686f6 100644 --- a/src/calibre/gui2/metadata/bulk_download2.py +++ b/src/calibre/gui2/metadata/bulk_download2.py @@ -8,8 +8,10 @@ __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' from functools import partial +from itertools import izip -from PyQt4.Qt import QIcon +from PyQt4.Qt import (QIcon, QDialog, QVBoxLayout, QTextBrowser, + QDialogButtonBox, QApplication) from calibre.gui2.dialogs.message_box import MessageBox from calibre.gui2.threaded_jobs import ThreadedJob @@ -21,8 +23,13 @@ def show_config(gui, parent): def start_download(gui, ids, callback): q = MessageBox(MessageBox.QUESTION, _('Schedule download?'), - _('The download of metadata for %d book(s) will' - ' run in the background. Proceed?')%len(ids), + '

'+_('The download of metadata for the %d selected book(s) will' + ' run in the background. Proceed?')%len(ids) + + '

'+_('You can monitor the progress of the download ' + 'by clicking the rotating spinner in the bottom right ' + 'corner.') + + '

'+_('When the download completes you will be asked for' + ' confirmation before calibre applies the downloaded metadata.'), show_copy_button=False, parent=gui) b = q.bb.addButton(_('Configure download'), q.bb.ActionRole) b.setIcon(QIcon(I('config.png'))) @@ -39,12 +46,75 @@ def start_download(gui, ids, callback): download, (ids, gui.current_db), {}, callback) gui.job_manager.run_threaded_job(job) +class ViewLog(QDialog): + + def __init__(self, html, parent=None): + QDialog.__init__(self, parent) + self.l = l = QVBoxLayout() + self.setLayout(l) + + self.tb = QTextBrowser(self) + self.tb.setHtml('

%s
' % html) + l.addWidget(self.tb) + + self.bb = QDialogButtonBox(QDialogButtonBox.Ok) + self.bb.accepted.connect(self.accept) + self.bb.rejected.connect(self.reject) + self.copy_button = self.bb.addButton(_('Copy to clipboard'), + self.bb.ActionRole) + self.copy_button.setIcon(QIcon(I('edit-copy.png'))) + self.copy_button.clicked.connect(self.copy_to_clipboard) + self.setModal(False) + self.resize(self.sizeHint()) + self.show() + + def copy_to_clipboard(self): + txt = self.tb.toPlainText() + QApplication.clipboard().setText(txt) + +_vl = None +def view_log(job, parent): + global _vl + _vl = ViewLog(job.html_details, parent) + +def apply(job, gui, q): + q.vlb.clicked.disconnect() + q.finished.diconnect() + id_map, failed_ids = job.result + print (id_map) + +def proceed(gui, job): + id_map, failed_ids = job.result + fmsg = det_msg = '' + if failed_ids: + fmsg = _('Could not download metadata for %d of the books. Click' + ' "Show details" to see which books.')%len(failed_ids) + det_msg = '\n'.join([id_map[i].title for i in failed_ids]) + msg = '

' + _('Finished downloading metadata for %d books. ' + 'Proceed with updating the metadata in your library?')%len(id_map) + q = MessageBox(MessageBox.QUESTION, _('Download complete'), + msg + fmsg, det_msg=det_msg, show_copy_button=bool(failed_ids), + parent=gui) + q.vlb = q.bb.addButton(_('View log'), q.bb.ActionRole) + q.vlb.setIcon(QIcon(I('debug.png'))) + q.vlb.clicked.connect(partial(view_log, job, q)) + q.det_msg_toggle.setVisible(bool(failed_ids)) + q.setModal(False) + q.show() + q.finished.connect(partial(job, gui, q)) + def download(ids, db, log=None, abort=None, notifications=None): + log('Starting metadata download for %d books'%len(ids)) ids = list(ids) metadata = [db.get_metadata(i, index_is_id=True, get_user_categories=False) for i in ids] - return (ids, [mi.last_modified for mi in metadata]) + failed_ids = set() + ans = {} + for i, mi in izip(ids, metadata): + ans[i] = mi + log('Download complete, with %d failures'%len(failed_ids)) + return (ans, failed_ids) From f1ad415fac721d16255c24e763492e8f713ac59d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 12 Apr 2011 08:12:45 -0600 Subject: [PATCH 06/10] MOBI Output: The Ignore margins setting no longer ignores blockquotes, only margins set via CSS on other elements. Fixes #758675 (Conversion to mobi with the 'ignore margins' option deletes existing blockquotes) --- src/calibre/ebooks/mobi/mobiml.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/calibre/ebooks/mobi/mobiml.py b/src/calibre/ebooks/mobi/mobiml.py index 40ad5e9e78..1e626cf916 100644 --- a/src/calibre/ebooks/mobi/mobiml.py +++ b/src/calibre/ebooks/mobi/mobiml.py @@ -495,6 +495,10 @@ class MobiMLizer(object): vtag.append(child) return + if tag == 'blockquote': + old_mim = self.opts.mobi_ignore_margins + self.opts.mobi_ignore_margins = False + if text or tag in CONTENT_TAGS or tag in NESTABLE_TAGS: self.mobimlize_content(tag, text, bstate, istates) for child in elem: @@ -510,6 +514,8 @@ class MobiMLizer(object): if tail: self.mobimlize_content(tag, tail, bstate, istates) + if tag == 'blockquote': + self.opts.mobi_ignore_margins = old_mim if bstate.content and style['page-break-after'] in PAGE_BREAKS: bstate.pbreak = True From 2d9625f5b2d2e9117554cd0b0fbc8e099afb6875 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 12 Apr 2011 09:48:31 -0600 Subject: [PATCH 07/10] ... --- src/calibre/library/database2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 0b1182c0bf..50b404b4be 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -1781,7 +1781,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): path_changed = True if set_authors: if not mi.authors: - mi.authors = [_('Unknown')] + mi.authors = [_('Unknown')] authors = [] for a in mi.authors: authors += string_to_authors(a) From a404e9827d9efdd7d34a6a11f070458845fcb5b6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 12 Apr 2011 11:04:01 -0600 Subject: [PATCH 08/10] Bulk download UI --- src/calibre/gui2/actions/edit_metadata.py | 3 +- src/calibre/gui2/metadata/bulk_download2.py | 147 ++++++++++++++++++-- 2 files changed, 136 insertions(+), 14 deletions(-) diff --git a/src/calibre/gui2/actions/edit_metadata.py b/src/calibre/gui2/actions/edit_metadata.py index f5e9e8c4a0..9f2cacb177 100644 --- a/src/calibre/gui2/actions/edit_metadata.py +++ b/src/calibre/gui2/actions/edit_metadata.py @@ -89,7 +89,8 @@ class EditMetadataAction(InterfaceAction): db = self.gui.library_view.model().db ids = [db.id(row.row()) for row in rows] from calibre.gui2.metadata.bulk_download2 import start_download - start_download(self.gui, ids, Dispatcher(self.bulk_metadata_downloaded)) + start_download(self.gui, ids, + Dispatcher(self.bulk_metadata_downloaded), identify, covers) def bulk_metadata_downloaded(self, job): if job.failed: diff --git a/src/calibre/gui2/metadata/bulk_download2.py b/src/calibre/gui2/metadata/bulk_download2.py index cb7f1686f6..add689e616 100644 --- a/src/calibre/gui2/metadata/bulk_download2.py +++ b/src/calibre/gui2/metadata/bulk_download2.py @@ -10,18 +10,21 @@ __docformat__ = 'restructuredtext en' from functools import partial from itertools import izip -from PyQt4.Qt import (QIcon, QDialog, QVBoxLayout, QTextBrowser, - QDialogButtonBox, QApplication) +from PyQt4.Qt import (QIcon, QDialog, QVBoxLayout, QTextBrowser, QSize, + QDialogButtonBox, QApplication, QTimer, QLabel, QProgressBar) from calibre.gui2.dialogs.message_box import MessageBox from calibre.gui2.threaded_jobs import ThreadedJob +from calibre.utils.icu import lower +from calibre.ebooks.metadata import authors_to_string +from calibre.gui2 import question_dialog, error_dialog def show_config(gui, parent): from calibre.gui2.preferences import show_config_widget show_config_widget('Sharing', 'Metadata download', parent=parent, gui=gui, never_shutdown=True) -def start_download(gui, ids, callback): +def start_download(gui, ids, callback, identify, covers): q = MessageBox(MessageBox.QUESTION, _('Schedule download?'), '

'+_('The download of metadata for the %d selected book(s) will' ' run in the background. Proceed?')%len(ids) + @@ -43,10 +46,10 @@ def start_download(gui, ids, callback): job = ThreadedJob('metadata bulk download', _('Download metadata for %d books')%len(ids), - download, (ids, gui.current_db), {}, callback) + download, (ids, gui.current_db, identify, covers), {}, callback) gui.job_manager.run_threaded_job(job) -class ViewLog(QDialog): +class ViewLog(QDialog): # {{{ def __init__(self, html, parent=None): QDialog.__init__(self, parent) @@ -64,8 +67,11 @@ class ViewLog(QDialog): self.bb.ActionRole) self.copy_button.setIcon(QIcon(I('edit-copy.png'))) self.copy_button.clicked.connect(self.copy_to_clipboard) + l.addWidget(self.bb) self.setModal(False) - self.resize(self.sizeHint()) + self.resize(QSize(500, 400)) + self.setWindowTitle(_('Download log')) + self.setWindowIcon(QIcon(I('debug.png'))) self.show() def copy_to_clipboard(self): @@ -77,11 +83,118 @@ def view_log(job, parent): global _vl _vl = ViewLog(job.html_details, parent) -def apply(job, gui, q): +# }}} + +class ApplyDialog(QDialog): + + def __init__(self, id_map, gui): + QDialog.__init__(self, gui) + + self.l = l = QVBoxLayout() + self.setLayout(l) + l.addWidget(QLabel(_('Applying downloaded metadata to your library'))) + + self.pb = QProgressBar(self) + l.addWidget(self.pb) + self.pb.setMinimum(0) + self.pb.setMaximum(len(id_map)) + + self.bb = QDialogButtonBox(QDialogButtonBox.Cancel) + self.bb.rejected.connect(self.reject) + self.bb.accepted.connect(self.accept) + l.addWidget(self.bb) + + self.db = gui.current_db + self.id_map = list(id_map.iteritems()) + self.current_idx = 0 + + self.failures = [] + self.canceled = False + + QTimer.singleShot(20, self.do_one) + self.exec_() + + def do_one(self): + if self.canceled: + return + i, mi = self.id_map[self.current_idx] + try: + set_title = not mi.is_null('title') + set_authors = not mi.is_null('authors') + self.db.set_metadata(i, mi, commit=False, set_title=set_title, + set_authors=set_authors) + except: + import traceback + self.failures.append((i, traceback.format_exc())) + + self.pb.setValue(self.pb.value()+1) + + if self.current_idx >= len(self.id_map) - 1: + self.finalize() + else: + self.current_idx += 1 + QTimer.singleShot(20, self.do_one) + + def reject(self): + self.canceled = True + QDialog.reject(self) + + def finalize(self): + if self.canceled: + return + if self.failures: + msg = [] + for i, tb in self.failures: + title = self.db.title(i, index_is_id=True) + authors = self.db.authors(i, index_is_id=True) + if authors: + authors = [x.replace('|', ',') for x in authors.split(',')] + title += ' - ' + authors_to_string(authors) + msg.append(title+'\n\n'+tb+'\n'+('*'*80)) + + error_dialog(self, _('Some failures'), + _('Failed to apply updated metadata for some books' + ' in your library. Click "Show Details" to see ' + 'details.'), det_msg='\n\n'.join(msg), show=True) + self.accept() + +_amd = None +def apply_metadata(job, gui, q, result): + global _amd q.vlb.clicked.disconnect() - q.finished.diconnect() + q.finished.disconnect() + if result != q.Accepted: + return id_map, failed_ids = job.result - print (id_map) + id_map = dict([(k, v) for k, v in id_map.iteritems() if k not in + failed_ids]) + if not id_map: + return + + modified = set() + db = gui.current_db + + for i, mi in id_map.iteritems(): + lm = db.metadata_last_modified(i, index_is_id=True) + if lm > mi.last_modified: + title = db.title(i, index_is_id=True) + authors = db.authors(i, index_is_id=True) + if authors: + authors = [x.replace('|', ',') for x in authors.split(',')] + title += ' - ' + authors_to_string(authors) + modified.add(title) + + if modified: + modified = sorted(modified, key=lower) + if not question_dialog(gui, _('Some books changed'), '

'+ + _('The metadata for some books in your library has' + ' changed since you started the download. If you' + ' proceed, some of those changes may be overwritten. ' + 'Click "Show details" to see the list of changed books. ' + 'Do you want to proceed?'), det_msg='\n'.join(modified)): + return + + _amd = ApplyDialog(id_map, gui) def proceed(gui, job): id_map, failed_ids = job.result @@ -90,7 +203,7 @@ def proceed(gui, job): fmsg = _('Could not download metadata for %d of the books. Click' ' "Show details" to see which books.')%len(failed_ids) det_msg = '\n'.join([id_map[i].title for i in failed_ids]) - msg = '

' + _('Finished downloading metadata for %d books. ' + msg = '

' + _('Finished downloading metadata for %d book(s). ' 'Proceed with updating the metadata in your library?')%len(id_map) q = MessageBox(MessageBox.QUESTION, _('Download complete'), msg + fmsg, det_msg=det_msg, show_copy_button=bool(failed_ids), @@ -101,18 +214,26 @@ def proceed(gui, job): q.det_msg_toggle.setVisible(bool(failed_ids)) q.setModal(False) q.show() - q.finished.connect(partial(job, gui, q)) + q.finished.connect(partial(apply_metadata, job, gui, q)) -def download(ids, db, log=None, abort=None, notifications=None): - log('Starting metadata download for %d books'%len(ids)) +def download(ids, db, identify, covers, + log=None, abort=None, notifications=None): ids = list(ids) metadata = [db.get_metadata(i, index_is_id=True, get_user_categories=False) for i in ids] failed_ids = set() ans = {} + count = 0 for i, mi in izip(ids, metadata): + if abort.is_set(): + log.error('Aborting...') + break + # TODO: Apply ignore_fields and set unchanged values to null values ans[i] = mi + count += 1 + notifications.put((count/len(ids), + _('Downloaded %d of %d')%(count, len(ids)))) log('Download complete, with %d failures'%len(failed_ids)) return (ans, failed_ids) From 9dc0e77a2556d0830f02b47e67e1f285641272aa Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 12 Apr 2011 14:20:05 -0600 Subject: [PATCH 09/10] Updated Vecernje Novosti. Fixes #759058 (Updated recipe for site Vecernje Novosti) --- recipes/novosti.recipe | 59 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 48 insertions(+), 11 deletions(-) diff --git a/recipes/novosti.recipe b/recipes/novosti.recipe index d66e7d28b7..a0a573d7ba 100644 --- a/recipes/novosti.recipe +++ b/recipes/novosti.recipe @@ -1,6 +1,6 @@ __license__ = 'GPL v3' -__copyright__ = '2008-2010, Darko Miletic ' +__copyright__ = '2008-2011, Darko Miletic ' ''' novosti.rs ''' @@ -21,34 +21,71 @@ class Novosti(BasicNewsRecipe): encoding = 'utf-8' language = 'sr' publication_type = 'newspaper' + masthead_url = 'http://www.novosti.rs/images/basic/logo-print.png' extra_css = """ @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} .article_description,body{font-family: Arial,Helvetica,sans1,sans-serif} .author{font-size: small} .articleLead{font-size: large; font-weight: bold} + img{display: block; margin-bottom: 1em; margin-top: 1em} """ conversion_options = { - 'comment' : description - , 'tags' : category - , 'publisher' : publisher - , 'language' : language + 'comment' : description + , 'tags' : category + , 'publisher' : publisher + , 'language' : language + , 'pretty_print' : True } preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')] - keep_only_tags = [dict(attrs={'class':['articleTitle','author','articleLead','articleBody']})] - remove_tags = [dict(name=['embed','object','iframe','base','link','meta'])] - feeds = [(u'Vesti', u'http://www.novosti.rs/rss/rss-vesti')] + keep_only_tags = [dict(attrs={'class':['articleTitle','articleInfo','articleLead','singlePhoto fl','articleBody']})] + remove_tags = [ + dict(name=['embed','object','iframe','base','link','meta']) + ,dict(name='a', attrs={'class':'loadComments topCommentsLink'}) + ] + remove_attributes = ['lang','xmlns:fb'] + + feeds = [ + (u'Politika' , u'http://www.novosti.rs/rss/2-Sve%20vesti') + ,(u'Drustvo' , u'http://www.novosti.rs/rss/1-Sve%20vesti') + ,(u'Ekonomija' , u'http://www.novosti.rs/rss/3-Sve%20vesti') + ,(u'Hronika' , u'http://www.novosti.rs/rss/4-Sve%20vesti') + ,(u'Dosije' , u'http://www.novosti.rs/rss/5-Sve%20vesti') + ,(u'Reportaze' , u'http://www.novosti.rs/rss/6-Sve%20vesti') + ,(u'Tehnologije' , u'http://www.novosti.rs/rss/35-Sve%20vesti') + ,(u'Zanimljivosti', u'http://www.novosti.rs/rss/26-Sve%20vesti') + ,(u'Auto' , u'http://www.novosti.rs/rss/50-Sve%20vesti') + ,(u'Sport' , u'http://www.novosti.rs/rss/11|47|12|14|13-Sve%20vesti') + ,(u'Svet' , u'http://www.novosti.rs/rss/7-Sve%20vesti') + ,(u'Region' , u'http://www.novosti.rs/rss/8-Sve%20vesti') + ,(u'Dijaspora' , u'http://www.novosti.rs/rss/9-Sve%20vesti') + ,(u'Spektakl' , u'http://www.novosti.rs/rss/10-Sve%20vesti') + ,(u'Kultura' , u'http://www.novosti.rs/rss/31-Sve%20vesti') + ,(u'Srbija' , u'http://www.novosti.rs/rss/15-Sve%20vesti') + ,(u'Beograd' , u'http://www.novosti.rs/rss/16-Sve%20vesti') + ,(u'Zivot+' , u'http://www.novosti.rs/rss/24|33|34|25|20|18|32|19-Sve%20vesti') + ,(u'Turizam' , u'http://www.novosti.rs/rss/36-Sve%20vesti') + ] def preprocess_html(self, soup): for item in soup.findAll(style=True): del item['style'] - for item in soup.findAll('span', attrs={'class':'author'}): - item.name='p' + for item in soup.findAll('a'): + limg = item.find('img') + if item.string is not None: + str = item.string + item.replaceWith(str) + else: + if limg: + item.name = 'div' + item.attrs = [] + else: + str = self.tag_to_string(item) + item.replaceWith(str) for item in soup.findAll('img'): if not item.has_key('alt'): item['alt'] = 'image' return soup - From 933f81b65f05b152787e2f66ef4a4a657cc0f3be Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 12 Apr 2011 14:23:17 -0600 Subject: [PATCH 10/10] Fix #759073 (Asus EeeNote) --- src/calibre/devices/misc.py | 2 +- src/calibre/gui2/metadata/bulk_download2.py | 49 ++++++++++++++++++++- 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/src/calibre/devices/misc.py b/src/calibre/devices/misc.py index 01eba48a30..b9710d1958 100644 --- a/src/calibre/devices/misc.py +++ b/src/calibre/devices/misc.py @@ -244,7 +244,7 @@ class EEEREADER(USBMS): FORMATS = ['epub', 'fb2', 'txt', 'pdf'] VENDOR_ID = [0x0b05] - PRODUCT_ID = [0x178f] + PRODUCT_ID = [0x178f, 0x17a1] BCD = [0x0319] EBOOK_DIR_MAIN = EBOOK_DIR_CARD_A = 'Book' diff --git a/src/calibre/gui2/metadata/bulk_download2.py b/src/calibre/gui2/metadata/bulk_download2.py index add689e616..19cd3df9d4 100644 --- a/src/calibre/gui2/metadata/bulk_download2.py +++ b/src/calibre/gui2/metadata/bulk_download2.py @@ -7,6 +7,7 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' +import os from functools import partial from itertools import izip @@ -18,6 +19,11 @@ from calibre.gui2.threaded_jobs import ThreadedJob from calibre.utils.icu import lower from calibre.ebooks.metadata import authors_to_string from calibre.gui2 import question_dialog, error_dialog +from calibre.ebooks.metadata.sources.identify import identify, msprefs +from calibre.ebooks.metadata.sources.covers import download_cover +from calibre.ebooks.metadata.book.base import Metadata +from calibre.customize.ui import metadata_plugins +from calibre.ptempfile import PersistentTemporaryFile def show_config(gui, parent): from calibre.gui2.preferences import show_config_widget @@ -127,6 +133,12 @@ class ApplyDialog(QDialog): import traceback self.failures.append((i, traceback.format_exc())) + try: + if mi.cover: + os.remove(mi.cover) + except: + pass + self.pb.setValue(self.pb.value()+1) if self.current_idx >= len(self.id_map) - 1: @@ -216,8 +228,21 @@ def proceed(gui, job): q.show() q.finished.connect(partial(apply_metadata, job, gui, q)) +def merge_result(oldmi, newmi): + dummy = Metadata(_('Unknown')) + for f in msprefs['ignore_fields']: + setattr(newmi, f, getattr(dummy, f)) + fields = set() + for plugin in metadata_plugins(['identify']): + fields |= plugin.touched_fields -def download(ids, db, identify, covers, + for f in fields: + # Optimize so that set_metadata does not have to do extra work later + if not f.startswith('identifier:'): + if (not newmi.is_null(f) and getattr(newmi, f) == getattr(oldmi, f)): + setattr(newmi, f, getattr(dummy, f)) + +def download(ids, db, do_identify, covers, log=None, abort=None, notifications=None): ids = list(ids) metadata = [db.get_metadata(i, index_is_id=True, get_user_categories=False) @@ -229,7 +254,27 @@ def download(ids, db, identify, covers, if abort.is_set(): log.error('Aborting...') break - # TODO: Apply ignore_fields and set unchanged values to null values + title, authors, identifiers = mi.title, mi.authors, mi.identifiers + if do_identify: + results = [] + try: + results = identify(log, abort, title=title, authors=authors, + identifiers=identifiers) + except: + pass + if results: + mi = merge_result(mi, results[0]) + identifiers = mi.identifiers + else: + log.error('Failed to download metadata for', title) + failed_ids.add(mi) + if covers: + cdata = download_cover(log, title=title, authors=authors, + identifiers=identifiers) + if cdata: + with PersistentTemporaryFile('.jpg', 'downloaded-cover-') as f: + f.write(cdata) + mi.cover = f.name ans[i] = mi count += 1 notifications.put((count/len(ids),