Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge from trunk

Browse Source
This commit is contained in:
Charles Haley 2011-04-12 22:12:44 +01:00
commit a1141767e5
13 changed files with 758 additions and 220 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
recipes/novosti.recipe
View File

@ -1,6 +1,6 @@
__license__ = 'GPL v3'
__copyright__ = '2008-2010, Darko Miletic <darko.miletic at gmail.com>'
__copyright__ = '2008-2011, Darko Miletic <darko.miletic at gmail.com>'
'''
novosti.rs
'''
@ -21,34 +21,71 @@ class Novosti(BasicNewsRecipe):
encoding = 'utf-8'
language = 'sr'
publication_type = 'newspaper'
masthead_url = 'http://www.novosti.rs/images/basic/logo-print.png'
extra_css = """ @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)}
.article_description,body{font-family: Arial,Helvetica,sans1,sans-serif}
.author{font-size: small}
.articleLead{font-size: large; font-weight: bold}
img{display: block; margin-bottom: 1em; margin-top: 1em}
"""
conversion_options = {
'comment' : description
, 'tags' : category
, 'publisher' : publisher
, 'language' : language
'comment' : description
, 'tags' : category
, 'publisher' : publisher
, 'language' : language
, 'pretty_print' : True
}
preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
keep_only_tags = [dict(attrs={'class':['articleTitle','author','articleLead','articleBody']})]
remove_tags = [dict(name=['embed','object','iframe','base','link','meta'])]
feeds = [(u'Vesti', u'http://www.novosti.rs/rss/rss-vesti')]
keep_only_tags = [dict(attrs={'class':['articleTitle','articleInfo','articleLead','singlePhoto fl','articleBody']})]
remove_tags = [
dict(name=['embed','object','iframe','base','link','meta'])
,dict(name='a', attrs={'class':'loadComments topCommentsLink'})
]
remove_attributes = ['lang','xmlns:fb']
feeds = [
(u'Politika' , u'http://www.novosti.rs/rss/2-Sve%20vesti')
,(u'Drustvo' , u'http://www.novosti.rs/rss/1-Sve%20vesti')
,(u'Ekonomija' , u'http://www.novosti.rs/rss/3-Sve%20vesti')
,(u'Hronika' , u'http://www.novosti.rs/rss/4-Sve%20vesti')
,(u'Dosije' , u'http://www.novosti.rs/rss/5-Sve%20vesti')
,(u'Reportaze' , u'http://www.novosti.rs/rss/6-Sve%20vesti')
,(u'Tehnologije' , u'http://www.novosti.rs/rss/35-Sve%20vesti')
,(u'Zanimljivosti', u'http://www.novosti.rs/rss/26-Sve%20vesti')
,(u'Auto' , u'http://www.novosti.rs/rss/50-Sve%20vesti')
,(u'Sport' , u'http://www.novosti.rs/rss/11|47|12|14|13-Sve%20vesti')
,(u'Svet' , u'http://www.novosti.rs/rss/7-Sve%20vesti')
,(u'Region' , u'http://www.novosti.rs/rss/8-Sve%20vesti')
,(u'Dijaspora' , u'http://www.novosti.rs/rss/9-Sve%20vesti')
,(u'Spektakl' , u'http://www.novosti.rs/rss/10-Sve%20vesti')
,(u'Kultura' , u'http://www.novosti.rs/rss/31-Sve%20vesti')
,(u'Srbija' , u'http://www.novosti.rs/rss/15-Sve%20vesti')
,(u'Beograd' , u'http://www.novosti.rs/rss/16-Sve%20vesti')
,(u'Zivot+' , u'http://www.novosti.rs/rss/24|33|34|25|20|18|32|19-Sve%20vesti')
,(u'Turizam' , u'http://www.novosti.rs/rss/36-Sve%20vesti')
]
def preprocess_html(self, soup):
for item in soup.findAll(style=True):
del item['style']
for item in soup.findAll('span', attrs={'class':'author'}):
item.name='p'
for item in soup.findAll('a'):
limg = item.find('img')
if item.string is not None:
str = item.string
item.replaceWith(str)
else:
if limg:
item.name = 'div'
item.attrs = []
else:
str = self.tag_to_string(item)
item.replaceWith(str)
for item in soup.findAll('img'):
if not item.has_key('alt'):
item['alt'] = 'image'
return soup

2
src/calibre/devices/misc.py
View File

@ -244,7 +244,7 @@ class EEEREADER(USBMS):
FORMATS = ['epub', 'fb2', 'txt', 'pdf']
VENDOR_ID = [0x0b05]
PRODUCT_ID = [0x178f]
PRODUCT_ID = [0x178f, 0x17a1]
BCD = [0x0319]
EBOOK_DIR_MAIN = EBOOK_DIR_CARD_A = 'Book'

6
src/calibre/ebooks/mobi/mobiml.py
View File

@ -495,6 +495,10 @@ class MobiMLizer(object):
vtag.append(child)
return
if tag == 'blockquote':
old_mim = self.opts.mobi_ignore_margins
self.opts.mobi_ignore_margins = False
if text or tag in CONTENT_TAGS or tag in NESTABLE_TAGS:
self.mobimlize_content(tag, text, bstate, istates)
for child in elem:
@ -510,6 +514,8 @@ class MobiMLizer(object):
if tail:
self.mobimlize_content(tag, tail, bstate, istates)
if tag == 'blockquote':
self.opts.mobi_ignore_margins = old_mim
if bstate.content and style['page-break-after'] in PAGE_BREAKS:
bstate.pbreak = True

12
src/calibre/gui2/actions/edit_metadata.py
View File

@ -10,7 +10,7 @@ from functools import partial
from PyQt4.Qt import Qt, QMenu, QModelIndex
from calibre.gui2 import error_dialog, config
from calibre.gui2 import error_dialog, config, Dispatcher
from calibre.gui2.dialogs.metadata_single import MetadataSingleDialog
from calibre.gui2.dialogs.metadata_bulk import MetadataBulkDialog
from calibre.gui2.dialogs.confirm_delete import confirm
@ -88,6 +88,16 @@ class EditMetadataAction(InterfaceAction):
_('No books selected'), show=True)
db = self.gui.library_view.model().db
ids = [db.id(row.row()) for row in rows]
from calibre.gui2.metadata.bulk_download2 import start_download
start_download(self.gui, ids,
Dispatcher(self.bulk_metadata_downloaded), identify, covers)
def bulk_metadata_downloaded(self, job):
if job.failed:
self.job_exception(job, dialog_title=_('Failed to download metadata'))
return
from calibre.gui2.metadata.bulk_download2 import proceed
proceed(self.gui, job)
def download_metadata_old(self, checked, covers=True, set_metadata=True,
set_social_metadata=None):

40
src/calibre/gui2/dialogs/job_view.ui
View File

@ -1,7 +1,8 @@
<ui version="4.0" >
<?xml version="1.0" encoding="UTF-8"?>
<ui version="4.0">
<class>Dialog</class>
<widget class="QDialog" name="Dialog" >
<property name="geometry" >
<widget class="QDialog" name="Dialog">
<property name="geometry">
<rect>
<x>0</x>
<y>0</y>
@ -9,38 +10,41 @@
<height>462</height>
</rect>
</property>
<property name="windowTitle" >
<property name="windowTitle">
<string>Details of job</string>
</property>
<property name="windowIcon" >
<iconset resource="../../../../resources/images.qrc" >
<property name="windowIcon">
<iconset resource="../../../../resources/images.qrc">
<normaloff>:/images/view.png</normaloff>:/images/view.png</iconset>
</property>
<layout class="QGridLayout" name="gridLayout" >
<item row="0" column="0" >
<widget class="QPlainTextEdit" name="log" >
<property name="undoRedoEnabled" >
<layout class="QGridLayout" name="gridLayout">
<item row="0" column="0">
<widget class="QPlainTextEdit" name="log">
<property name="undoRedoEnabled">
<bool>false</bool>
</property>
<property name="lineWrapMode" >
<property name="lineWrapMode">
<enum>QPlainTextEdit::NoWrap</enum>
</property>
<property name="readOnly" >
<property name="readOnly">
<bool>true</bool>
</property>
</widget>
</item>
<item row="1" column="0" >
<widget class="QDialogButtonBox" name="buttonBox" >
<property name="standardButtons" >
<item row="2" column="0">
<widget class="QDialogButtonBox" name="buttonBox">
<property name="standardButtons">
<set>QDialogButtonBox::Ok</set>
</property>
</widget>
</item>
<item row="1" column="0">
<widget class="QTextBrowser" name="tb"/>
</item>
</layout>
</widget>
<resources>
<include location="../../../../resources/images.qrc" />
<include location="../../../../resources/images.qrc"/>
</resources>
<connections>
<connection>
@ -49,11 +53,11 @@
<receiver>Dialog</receiver>
<slot>accept()</slot>
<hints>
<hint type="sourcelabel" >
<hint type="sourcelabel">
<x>617</x>
<y>442</y>
</hint>
<hint type="destinationlabel" >
<hint type="destinationlabel">
<x>206</x>
<y>-5</y>
</hint>

226
src/calibre/gui2/email.py
View File

@ -6,9 +6,7 @@ __license__ = 'GPL v3'
__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import os, socket, time, cStringIO
from threading import Thread
from Queue import Queue
import os, socket, time
from binascii import unhexlify
from functools import partial
from itertools import repeat
@ -16,67 +14,20 @@ from itertools import repeat
from calibre.utils.smtp import compose_mail, sendmail, extract_email_address, \
config as email_config
from calibre.utils.filenames import ascii_filename
from calibre.utils.ipc.job import BaseJob
from calibre.ptempfile import PersistentTemporaryFile
from calibre.customize.ui import available_input_formats, available_output_formats
from calibre.ebooks.metadata import authors_to_string
from calibre.constants import preferred_encoding
from calibre.gui2 import config, Dispatcher, warning_dialog
from calibre.library.save_to_disk import get_components
from calibre.utils.config import tweaks
from calibre.gui2.threaded_jobs import ThreadedJob
class EmailJob(BaseJob): # {{{
def __init__(self, callback, description, attachment, aname, to, subject, text, job_manager):
BaseJob.__init__(self, description)
self.exception = None
self.job_manager = job_manager
self.email_args = (attachment, aname, to, subject, text)
self.email_sent_callback = callback
self.log_path = None
self._log_file = cStringIO.StringIO()
self._log_file.write(self.description.encode('utf-8') + '\n')
@property
def log_file(self):
if self.log_path is not None:
return open(self.log_path, 'rb')
return cStringIO.StringIO(self._log_file.getvalue())
def start_work(self):
self.start_time = time.time()
self.job_manager.changed_queue.put(self)
def job_done(self):
self.duration = time.time() - self.start_time
self.percent = 1
# Dump log onto disk
lf = PersistentTemporaryFile('email_log')
lf.write(self._log_file.getvalue())
lf.close()
self.log_path = lf.name
self._log_file.close()
self._log_file = None
self.job_manager.changed_queue.put(self)
def log_write(self, what):
self._log_file.write(what)
# }}}
class Emailer(Thread): # {{{
class Sendmail(object):
MAX_RETRIES = 1
def __init__(self, job_manager):
Thread.__init__(self)
self.daemon = True
self.jobs = Queue()
self.job_manager = job_manager
self._run = True
def __init__(self):
self.calculate_rate_limit()
self.last_send_time = time.time() - self.rate_limit
def calculate_rate_limit(self):
@ -87,70 +38,28 @@ class Emailer(Thread): # {{{
'gmail.com' in rh or 'live.com' in rh):
self.rate_limit = tweaks['public_smtp_relay_delay']
def stop(self):
self._run = False
self.jobs.put(None)
def __call__(self, attachment, aname, to, subject, text, log=None,
abort=None, notifications=None):
def run(self):
while self._run:
try_count = 0
while try_count <= self.MAX_RETRIES:
if try_count > 0:
log('\nRetrying in %d seconds...\n' %
self.rate_limit)
try:
job = self.jobs.get()
self.sendmail(attachment, aname, to, subject, text, log)
try_count = self.MAX_RETRIES
log('Email successfully sent')
except:
break
if job is None or not self._run:
break
try_count = 0
failed, exc = False, None
job.start_work()
if job.kill_on_start:
job.log_write('Aborted\n')
job.failed = failed
job.killed = True
job.job_done()
continue
if abort.is_set():
return
if try_count == self.MAX_RETRIES:
raise
log.exception('\nSending failed...\n')
while try_count <= self.MAX_RETRIES:
failed = False
if try_count > 0:
job.log_write('\nRetrying in %d seconds...\n' %
self.rate_limit)
try:
self.sendmail(job)
break
except Exception as e:
if not self._run:
return
import traceback
failed = True
exc = e
job.log_write('\nSending failed...\n')
job.log_write(traceback.format_exc())
try_count += 1
try_count += 1
if not self._run:
break
job.failed = failed
job.exception = exc
job.job_done()
try:
job.email_sent_callback(job)
except:
import traceback
traceback.print_exc()
def send_mails(self, jobnames, callback, attachments, to_s, subjects,
texts, attachment_names):
for name, attachment, to, subject, text, aname in zip(jobnames,
attachments, to_s, subjects, texts, attachment_names):
description = _('Email %s to %s') % (name, to)
job = EmailJob(callback, description, attachment, aname, to,
subject, text, self.job_manager)
self.job_manager.add_job(job)
self.jobs.put(job)
def sendmail(self, job):
def sendmail(self, attachment, aname, to, subject, text, log):
while time.time() - self.last_send_time <= self.rate_limit:
time.sleep(1)
try:
@ -158,7 +67,6 @@ class Emailer(Thread): # {{{
from_ = opts.from_
if not from_:
from_ = 'calibre <calibre@'+socket.getfqdn()+'>'
attachment, aname, to, subject, text = job.email_args
msg = compose_mail(from_, to, text, subject, open(attachment, 'rb'),
aname)
efrom, eto = map(extract_email_address, (from_, to))
@ -169,48 +77,56 @@ class Emailer(Thread): # {{{
username=opts.relay_username,
password=unhexlify(opts.relay_password), port=opts.relay_port,
encryption=opts.encryption,
debug_output=partial(print, file=job._log_file))
debug_output=log.debug)
finally:
self.last_send_time = time.time()
def email_news(self, mi, remove, get_fmts, done):
opts = email_config().parse()
accounts = [(account, [x.strip().lower() for x in x[0].split(',')])
for account, x in opts.accounts.items() if x[1]]
sent_mails = []
for i, x in enumerate(accounts):
account, fmts = x
files = get_fmts(fmts)
files = [f for f in files if f is not None]
if not files:
continue
attachment = files[0]
to_s = [account]
subjects = [_('News:')+' '+mi.title]
texts = [
_('Attached is the %s periodical downloaded by calibre.')
% (mi.title,)
]
attachment_names = [ascii_filename(mi.title)+os.path.splitext(attachment)[1]]
attachments = [attachment]
jobnames = [mi.title]
do_remove = []
if i == len(accounts) - 1:
do_remove = remove
self.send_mails(jobnames,
Dispatcher(partial(done, remove=do_remove)),
attachments, to_s, subjects, texts, attachment_names)
sent_mails.append(to_s[0])
return sent_mails
gui_sendmail = Sendmail()
# }}}
def send_mails(jobnames, callback, attachments, to_s, subjects,
texts, attachment_names, job_manager):
for name, attachment, to, subject, text, aname in zip(jobnames,
attachments, to_s, subjects, texts, attachment_names):
description = _('Email %s to %s') % (name, to)
job = ThreadedJob('email', description, gui_sendmail, (attachment, aname, to,
subject, text), {}, callback, killable=False)
job_manager.run_threaded_job(job)
def email_news(mi, remove, get_fmts, done, job_manager):
opts = email_config().parse()
accounts = [(account, [x.strip().lower() for x in x[0].split(',')])
for account, x in opts.accounts.items() if x[1]]
sent_mails = []
for i, x in enumerate(accounts):
account, fmts = x
files = get_fmts(fmts)
files = [f for f in files if f is not None]
if not files:
continue
attachment = files[0]
to_s = [account]
subjects = [_('News:')+' '+mi.title]
texts = [
_('Attached is the %s periodical downloaded by calibre.')
% (mi.title,)
]
attachment_names = [ascii_filename(mi.title)+os.path.splitext(attachment)[1]]
attachments = [attachment]
jobnames = [mi.title]
do_remove = []
if i == len(accounts) - 1:
do_remove = remove
send_mails(jobnames,
Dispatcher(partial(done, remove=do_remove)),
attachments, to_s, subjects, texts, attachment_names,
job_manager)
sent_mails.append(to_s[0])
return sent_mails
class EmailMixin(object): # {{{
def __init__(self):
self.emailer = Emailer(self.job_manager)
def send_by_mail(self, to, fmts, delete_from_library, subject='', send_ids=None,
do_auto_convert=True, specific_format=None):
ids = [self.library_view.model().id(r) for r in self.library_view.selectionModel().selectedRows()] if send_ids is None else send_ids
@ -246,8 +162,7 @@ class EmailMixin(object): # {{{
components = get_components(subject, mi, id)
if not components:
components = [mi.title]
subject = os.path.join(*components)
subjects.append(subject)
subjects.append(os.path.join(*components))
a = authors_to_string(mi.authors if mi.authors else \
[_('Unknown')])
texts.append(_('Attached, you will find the e-book') + \
@ -262,11 +177,10 @@ class EmailMixin(object): # {{{
to_s = list(repeat(to, len(attachments)))
if attachments:
if not self.emailer.is_alive():
self.emailer.start()
self.emailer.send_mails(jobnames,
send_mails(jobnames,
Dispatcher(partial(self.email_sent, remove=remove)),
attachments, to_s, subjects, texts, attachment_names)
attachments, to_s, subjects, texts, attachment_names,
self.job_manager)
self.status_bar.show_message(_('Sending email to')+' '+to, 3000)
auto = []
@ -334,10 +248,8 @@ class EmailMixin(object): # {{{
files, auto = self.library_view.model().\
get_preferred_formats_from_ids([id_], fmts)
return files
if not self.emailer.is_alive():
self.emailer.start()
sent_mails = self.emailer.email_news(mi, remove,
get_fmts, self.email_sent)
sent_mails = email_news(mi, remove,
get_fmts, self.email_sent, self.job_manager)
if sent_mails:
self.status_bar.show_message(_('Sent news to')+' '+\
', '.join(sent_mails), 3000)

85
src/calibre/gui2/jobs.py
View File

@ -8,14 +8,13 @@ Job management.
'''
import re
from Queue import Empty, Queue
from PyQt4.Qt import QAbstractTableModel, QVariant, QModelIndex, Qt, \
QTimer, pyqtSignal, QIcon, QDialog, QAbstractItemDelegate, QApplication, \
QSize, QStyleOptionProgressBarV2, QString, QStyle, QToolTip, QFrame, \
QHBoxLayout, QVBoxLayout, QSizePolicy, QLabel, QCoreApplication, QAction, \
QByteArray
from PyQt4.Qt import (QAbstractTableModel, QVariant, QModelIndex, Qt,
QTimer, pyqtSignal, QIcon, QDialog, QAbstractItemDelegate, QApplication,
QSize, QStyleOptionProgressBarV2, QString, QStyle, QToolTip, QFrame,
QHBoxLayout, QVBoxLayout, QSizePolicy, QLabel, QCoreApplication, QAction,
QByteArray)
from calibre.utils.ipc.server import Server
from calibre.utils.ipc.job import ParallelJob
@ -25,8 +24,9 @@ from calibre.gui2.dialogs.jobs_ui import Ui_JobsDialog
from calibre import __appname__
from calibre.gui2.dialogs.job_view_ui import Ui_Dialog
from calibre.gui2.progress_indicator import ProgressIndicator
from calibre.gui2.threaded_jobs import ThreadedJobServer, ThreadedJob
class JobManager(QAbstractTableModel):
class JobManager(QAbstractTableModel): # {{{
job_added = pyqtSignal(int)
job_done = pyqtSignal(int)
@ -42,6 +42,7 @@ class JobManager(QAbstractTableModel):
self.add_job = Dispatcher(self._add_job)
self.server = Server(limit=int(config['worker_limit']/2.0),
enforce_cpu_limit=config['enforce_cpu_limit'])
self.threaded_server = ThreadedJobServer()
self.changed_queue = Queue()
self.timer = QTimer(self)
@ -146,12 +147,21 @@ class JobManager(QAbstractTableModel):
jobs.add(self.server.changed_jobs_queue.get_nowait())
except Empty:
break
# Update device jobs
while True:
try:
jobs.add(self.changed_queue.get_nowait())
except Empty:
break
# Update threaded jobs
while True:
try:
jobs.add(self.threaded_server.changed_jobs.get_nowait())
except Empty:
break
if jobs:
needs_reset = False
for job in jobs:
@ -207,11 +217,22 @@ class JobManager(QAbstractTableModel):
self.server.add_job(job)
return job
def run_threaded_job(self, job):
self.add_job(job)
self.threaded_server.add_job(job)
def launch_gui_app(self, name, args=[], kwargs={}, description=''):
job = ParallelJob(name, description, lambda x: x,
args=args, kwargs=kwargs)
self.server.run_job(job, gui=True, redirect_output=False)
def _kill_job(self, job):
if isinstance(job, ParallelJob):
self.server.kill_job(job)
elif isinstance(job, ThreadedJob):
self.threaded_server.kill_job(job)
else:
job.kill_on_start = True
def kill_job(self, row, view):
job = self.jobs[row]
@ -221,29 +242,29 @@ class JobManager(QAbstractTableModel):
if job.duration is not None:
return error_dialog(view, _('Cannot kill job'),
_('Job has already run')).exec_()
if isinstance(job, ParallelJob):
self.server.kill_job(job)
else:
job.kill_on_start = True
if not getattr(job, 'killable', True):
return error_dialog(view, _('Cannot kill job'),
_('This job cannot be stopped'), show=True)
self._kill_job(job)
def kill_all_jobs(self):
for job in self.jobs:
if isinstance(job, DeviceJob) or job.duration is not None:
if (isinstance(job, DeviceJob) or job.duration is not None or
not getattr(job, 'killable', True)):
continue
if isinstance(job, ParallelJob):
self.server.kill_job(job)
else:
job.kill_on_start = True
self._kill_job(job)
def terminate_all_jobs(self):
self.server.killall()
for job in self.jobs:
if isinstance(job, DeviceJob) or job.duration is not None:
if (isinstance(job, DeviceJob) or job.duration is not None or
not getattr(job, 'killable', True)):
continue
if not isinstance(job, ParallelJob):
job.kill_on_start = True
self._kill_job(job)
# }}}
# Jobs UI {{{
class ProgressBarDelegate(QAbstractItemDelegate):
def sizeHint(self, option, index):
@ -269,6 +290,11 @@ class DetailView(QDialog, Ui_Dialog):
self.setupUi(self)
self.setWindowTitle(job.description)
self.job = job
self.html_view = hasattr(job, 'html_details')
if self.html_view:
self.log.setVisible(False)
else:
self.tb.setVisible(False)
self.next_pos = 0
self.update()
self.timer = QTimer(self)
@ -277,12 +303,19 @@ class DetailView(QDialog, Ui_Dialog):
def update(self):
f = self.job.log_file
f.seek(self.next_pos)
more = f.read()
self.next_pos = f.tell()
if more:
self.log.appendPlainText(more.decode('utf-8', 'replace'))
if self.html_view:
html = self.job.html_details
if len(html) > self.next_pos:
self.next_pos = len(html)
self.tb.setHtml(
'<pre style="font-family:monospace">%s</pre>'%html)
else:
f = self.job.log_file
f.seek(self.next_pos)
more = f.read()
self.next_pos = f.tell()
if more:
self.log.appendPlainText(more.decode('utf-8', 'replace'))
class JobsButton(QFrame):
@ -441,3 +474,5 @@ class JobsDialog(QDialog, Ui_JobsDialog):
def hide(self, *args):
self.save_state()
return QDialog.hide(self, *args)
# }}}

286
src/calibre/gui2/metadata/bulk_download2.py Normal file
View File

@ -0,0 +1,286 @@
#!/usr/bin/env python
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
from __future__ import (unicode_literals, division, absolute_import,
print_function)
__license__ = 'GPL v3'
__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import os
from functools import partial
from itertools import izip
from PyQt4.Qt import (QIcon, QDialog, QVBoxLayout, QTextBrowser, QSize,
QDialogButtonBox, QApplication, QTimer, QLabel, QProgressBar)
from calibre.gui2.dialogs.message_box import MessageBox
from calibre.gui2.threaded_jobs import ThreadedJob
from calibre.utils.icu import lower
from calibre.ebooks.metadata import authors_to_string
from calibre.gui2 import question_dialog, error_dialog
from calibre.ebooks.metadata.sources.identify import identify, msprefs
from calibre.ebooks.metadata.sources.covers import download_cover
from calibre.ebooks.metadata.book.base import Metadata
from calibre.customize.ui import metadata_plugins
from calibre.ptempfile import PersistentTemporaryFile
def show_config(gui, parent):
from calibre.gui2.preferences import show_config_widget
show_config_widget('Sharing', 'Metadata download', parent=parent,
gui=gui, never_shutdown=True)
def start_download(gui, ids, callback, identify, covers):
q = MessageBox(MessageBox.QUESTION, _('Schedule download?'),
'<p>'+_('The download of metadata for the <b>%d selected book(s)</b> will'
' run in the background. Proceed?')%len(ids) +
'<p>'+_('You can monitor the progress of the download '
'by clicking the rotating spinner in the bottom right '
'corner.') +
'<p>'+_('When the download completes you will be asked for'
' confirmation before calibre applies the downloaded metadata.'),
show_copy_button=False, parent=gui)
b = q.bb.addButton(_('Configure download'), q.bb.ActionRole)
b.setIcon(QIcon(I('config.png')))
b.clicked.connect(partial(show_config, gui, q))
q.det_msg_toggle.setVisible(False)
ret = q.exec_()
b.clicked.disconnect()
if ret != q.Accepted:
return
job = ThreadedJob('metadata bulk download',
_('Download metadata for %d books')%len(ids),
download, (ids, gui.current_db, identify, covers), {}, callback)
gui.job_manager.run_threaded_job(job)
class ViewLog(QDialog): # {{{
def __init__(self, html, parent=None):
QDialog.__init__(self, parent)
self.l = l = QVBoxLayout()
self.setLayout(l)
self.tb = QTextBrowser(self)
self.tb.setHtml('<pre style="font-family: monospace">%s</pre>' % html)
l.addWidget(self.tb)
self.bb = QDialogButtonBox(QDialogButtonBox.Ok)
self.bb.accepted.connect(self.accept)
self.bb.rejected.connect(self.reject)
self.copy_button = self.bb.addButton(_('Copy to clipboard'),
self.bb.ActionRole)
self.copy_button.setIcon(QIcon(I('edit-copy.png')))
self.copy_button.clicked.connect(self.copy_to_clipboard)
l.addWidget(self.bb)
self.setModal(False)
self.resize(QSize(500, 400))
self.setWindowTitle(_('Download log'))
self.setWindowIcon(QIcon(I('debug.png')))
self.show()
def copy_to_clipboard(self):
txt = self.tb.toPlainText()
QApplication.clipboard().setText(txt)
_vl = None
def view_log(job, parent):
global _vl
_vl = ViewLog(job.html_details, parent)
# }}}
class ApplyDialog(QDialog):
def __init__(self, id_map, gui):
QDialog.__init__(self, gui)
self.l = l = QVBoxLayout()
self.setLayout(l)
l.addWidget(QLabel(_('Applying downloaded metadata to your library')))
self.pb = QProgressBar(self)
l.addWidget(self.pb)
self.pb.setMinimum(0)
self.pb.setMaximum(len(id_map))
self.bb = QDialogButtonBox(QDialogButtonBox.Cancel)
self.bb.rejected.connect(self.reject)
self.bb.accepted.connect(self.accept)
l.addWidget(self.bb)
self.db = gui.current_db
self.id_map = list(id_map.iteritems())
self.current_idx = 0
self.failures = []
self.canceled = False
QTimer.singleShot(20, self.do_one)
self.exec_()
def do_one(self):
if self.canceled:
return
i, mi = self.id_map[self.current_idx]
try:
set_title = not mi.is_null('title')
set_authors = not mi.is_null('authors')
self.db.set_metadata(i, mi, commit=False, set_title=set_title,
set_authors=set_authors)
except:
import traceback
self.failures.append((i, traceback.format_exc()))
try:
if mi.cover:
os.remove(mi.cover)
except:
pass
self.pb.setValue(self.pb.value()+1)
if self.current_idx >= len(self.id_map) - 1:
self.finalize()
else:
self.current_idx += 1
QTimer.singleShot(20, self.do_one)
def reject(self):
self.canceled = True
QDialog.reject(self)
def finalize(self):
if self.canceled:
return
if self.failures:
msg = []
for i, tb in self.failures:
title = self.db.title(i, index_is_id=True)
authors = self.db.authors(i, index_is_id=True)
if authors:
authors = [x.replace('|', ',') for x in authors.split(',')]
title += ' - ' + authors_to_string(authors)
msg.append(title+'\n\n'+tb+'\n'+('*'*80))
error_dialog(self, _('Some failures'),
_('Failed to apply updated metadata for some books'
' in your library. Click "Show Details" to see '
'details.'), det_msg='\n\n'.join(msg), show=True)
self.accept()
_amd = None
def apply_metadata(job, gui, q, result):
global _amd
q.vlb.clicked.disconnect()
q.finished.disconnect()
if result != q.Accepted:
return
id_map, failed_ids = job.result
id_map = dict([(k, v) for k, v in id_map.iteritems() if k not in
failed_ids])
if not id_map:
return
modified = set()
db = gui.current_db
for i, mi in id_map.iteritems():
lm = db.metadata_last_modified(i, index_is_id=True)
if lm > mi.last_modified:
title = db.title(i, index_is_id=True)
authors = db.authors(i, index_is_id=True)
if authors:
authors = [x.replace('|', ',') for x in authors.split(',')]
title += ' - ' + authors_to_string(authors)
modified.add(title)
if modified:
modified = sorted(modified, key=lower)
if not question_dialog(gui, _('Some books changed'), '<p>'+
_('The metadata for some books in your library has'
' changed since you started the download. If you'
' proceed, some of those changes may be overwritten. '
'Click "Show details" to see the list of changed books. '
'Do you want to proceed?'), det_msg='\n'.join(modified)):
return
_amd = ApplyDialog(id_map, gui)
def proceed(gui, job):
id_map, failed_ids = job.result
fmsg = det_msg = ''
if failed_ids:
fmsg = _('Could not download metadata for %d of the books. Click'
' "Show details" to see which books.')%len(failed_ids)
det_msg = '\n'.join([id_map[i].title for i in failed_ids])
msg = '<p>' + _('Finished downloading metadata for <b>%d book(s)</b>. '
'Proceed with updating the metadata in your library?')%len(id_map)
q = MessageBox(MessageBox.QUESTION, _('Download complete'),
msg + fmsg, det_msg=det_msg, show_copy_button=bool(failed_ids),
parent=gui)
q.vlb = q.bb.addButton(_('View log'), q.bb.ActionRole)
q.vlb.setIcon(QIcon(I('debug.png')))
q.vlb.clicked.connect(partial(view_log, job, q))
q.det_msg_toggle.setVisible(bool(failed_ids))
q.setModal(False)
q.show()
q.finished.connect(partial(apply_metadata, job, gui, q))
def merge_result(oldmi, newmi):
dummy = Metadata(_('Unknown'))
for f in msprefs['ignore_fields']:
setattr(newmi, f, getattr(dummy, f))
fields = set()
for plugin in metadata_plugins(['identify']):
fields |= plugin.touched_fields
for f in fields:
# Optimize so that set_metadata does not have to do extra work later
if not f.startswith('identifier:'):
if (not newmi.is_null(f) and getattr(newmi, f) == getattr(oldmi, f)):
setattr(newmi, f, getattr(dummy, f))
def download(ids, db, do_identify, covers,
log=None, abort=None, notifications=None):
ids = list(ids)
metadata = [db.get_metadata(i, index_is_id=True, get_user_categories=False)
for i in ids]
failed_ids = set()
ans = {}
count = 0
for i, mi in izip(ids, metadata):
if abort.is_set():
log.error('Aborting...')
break
title, authors, identifiers = mi.title, mi.authors, mi.identifiers
if do_identify:
results = []
try:
results = identify(log, abort, title=title, authors=authors,
identifiers=identifiers)
except:
pass
if results:
mi = merge_result(mi, results[0])
identifiers = mi.identifiers
else:
log.error('Failed to download metadata for', title)
failed_ids.add(mi)
if covers:
cdata = download_cover(log, title=title, authors=authors,
identifiers=identifiers)
if cdata:
with PersistentTemporaryFile('.jpg', 'downloaded-cover-') as f:
f.write(cdata)
mi.cover = f.name
ans[i] = mi
count += 1
notifications.put((count/len(ids),
_('Downloaded %d of %d')%(count, len(ids))))
log('Download complete, with %d failures'%len(failed_ids))
return (ans, failed_ids)

6
src/calibre/gui2/preferences/emailp.py
View File

@ -84,11 +84,11 @@ class EmailAccounts(QAbstractTableModel): # {{{
account = self.account_order[row]
if col == 3:
self.accounts[account][1] ^= True
if col == 2:
elif col == 2:
self.subjects[account] = unicode(value.toString())
elif col == 1:
self.accounts[account][0] = unicode(value.toString()).upper()
else:
elif col == 0:
na = unicode(value.toString())
from email.utils import parseaddr
addr = parseaddr(na)[-1]
@ -100,7 +100,7 @@ class EmailAccounts(QAbstractTableModel): # {{{
self.accounts[na][0] = 'AZW, MOBI, TPZ, PRC, AZW1'
self.dataChanged.emit(
self.index(index.row(), 0), self.index(index.row(), 2))
self.index(index.row(), 0), self.index(index.row(), 3))
return True
def make_default(self, index):

241
src/calibre/gui2/threaded_jobs.py Normal file
View File

@ -0,0 +1,241 @@
#!/usr/bin/env python
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
from __future__ import (unicode_literals, division, absolute_import,
print_function)
__license__ = 'GPL v3'
__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import os, time, tempfile, json
from threading import Thread, RLock, Event
from Queue import Queue
from calibre.utils.ipc.job import BaseJob
from calibre.utils.logging import GUILog
from calibre.ptempfile import base_dir
class ThreadedJob(BaseJob):
def __init__(self,
type_, description,
func, args, kwargs,
callback,
max_concurrent_count=1,
killable=True,
log=None):
'''
A job that is run in its own thread in the calibre main process
:param type_: The type of this job (a string). The type is used in
conjunction with max_concurrent_count to prevent too many jobs of the
same type from running
:param description: A user viewable job description
:func: The function that actually does the work. This function *must*
accept at least three keyword arguments: abort, log and notifications. abort is
An Event object. func should periodically check abort.is_set(0 and if
it is True, it should stop processing as soon as possible. notifications
is a Queue. func should put progress notifications into it in the form
of a tuple (frac, msg). frac is a number between 0 and 1 indicating
progress and msg is a string describing the progress. log is a Log
object which func should use for all debugging output. func should
raise an Exception to indicate failure. This exception is stored in
job.exception and can thus be used to pass arbitrary information to
callback.
:param args,kwargs: These are passed to func when it is called
:param callback: A callable that is called on completion of this job.
Note that it is not called if the user kills the job. Check job.failed
to see if the job succeeded or not. And use job.log to get the job log.
:param killable: If False the GUI wont let the user kill this job
:param log: Must be a subclass of GUILog or None. If None a default
GUILog is created.
'''
BaseJob.__init__(self, description)
self.type = type_
self.max_concurrent_count = max_concurrent_count
self.killable = killable
self.callback = callback
self.abort = Event()
self.exception = None
kwargs['notifications'] = self.notifications
kwargs['abort'] = self.abort
self.log = GUILog() if log is None else log
kwargs['log'] = self.log
self.func, self.args, self.kwargs = func, args, kwargs
self.consolidated_log = None
def start_work(self):
self.start_time = time.time()
self.log('Starting job:', self.description)
try:
self.result = self.func(*self.args, **self.kwargs)
except Exception as e:
self.exception = e
self.failed = True
self.log.exception('Job: "%s" failed with error:'%self.description)
self.log.debug('Called with args:', self.args, self.kwargs)
self.duration = time.time() - self.start_time
try:
self.callback(self)
except:
import traceback
traceback.print_exc()
self._cleanup()
def _cleanup(self):
try:
self.consolidate_log()
except:
self.log.exception('Log consolidation failed')
# No need to keep references to these around anymore
self.func = self.args = self.kwargs = self.notifications = None
# We can't delete self.callback as it might be a Dispatch object and if
# it is garbage collected it won't work
def kill(self):
if self.start_time is None:
self.start_time = time.time()
self.duration = 0.0001
else:
self.duration = time.time() - self.start_time()
self.abort.set()
self.log('Aborted job:', self.description)
self.killed = True
self.failed = True
self._cleanup()
def consolidate_log(self):
logs = [self.log.html, self.log.plain_text]
bdir = base_dir()
log_dir = os.path.join(bdir, 'threaded_job_logs')
if not os.path.exists(log_dir):
os.makedirs(log_dir)
fd, path = tempfile.mkstemp(suffix='.json', prefix='log-', dir=log_dir)
with os.fdopen(fd, 'wb') as f:
f.write(json.dumps(logs, ensure_ascii=False,
indent=2).encode('utf-8'))
self.consolidated_log = path
self.log = None
def read_consolidated_log(self):
with open(self.consolidated_log, 'rb') as f:
return json.loads(f.read().decode('utf-8'))
@property
def details(self):
if self.consolidated_log is None:
return self.log.plain_text
return self.read_consolidated_log()[1]
@property
def html_details(self):
if self.consolidated_log is None:
return self.log.html
return self.read_consolidated_log()[0]
class ThreadedJobWorker(Thread):
def __init__(self, job):
Thread.__init__(self)
self.daemon = True
self.job = job
def run(self):
try:
self.job.start_work()
except:
import traceback
from calibre import prints
prints('Job had unhandled exception:', self.job.description)
traceback.print_exc()
class ThreadedJobServer(Thread):
def __init__(self):
Thread.__init__(self)
self.daemon = True
self.lock = RLock()
self.queued_jobs = []
self.running_jobs = set()
self.changed_jobs = Queue()
self.keep_going = True
def close(self):
self.keep_going = False
def add_job(self, job):
with self.lock:
self.queued_jobs.append(job)
if not self.is_alive():
self.start()
def run(self):
while self.keep_going:
self.run_once()
time.sleep(0.1)
def run_once(self):
with self.lock:
remove = set()
for worker in self.running_jobs:
if worker.is_alive():
# Get progress notifications
if worker.job.consume_notifications():
self.changed_jobs.put(worker.job)
else:
remove.add(worker)
self.changed_jobs.put(worker.job)
for worker in remove:
self.running_jobs.remove(worker)
jobs = self.get_startable_jobs()
for job in jobs:
w = ThreadedJobWorker(job)
w.start()
self.running_jobs.add(w)
self.changed_jobs.put(job)
self.queued_jobs.remove(job)
def kill_job(self, job):
with self.lock:
if job in self.queued_jobs:
self.queued_jobs.remove(job)
elif job in self.running_jobs:
self.running_jobs.remove(job)
job.kill()
self.changed_jobs.put(job)
def running_jobs_of_type(self, type_):
return len([w for w in self.running_jobs if w.job.type == type_])
def get_startable_jobs(self):
queued_types = []
ans = []
for job in self.queued_jobs:
num = self.running_jobs_of_type(job.type)
num += queued_types.count(job.type)
if num < job.max_concurrent_count:
queued_types.append(job.type)
ans.append(job)
return ans

3
src/calibre/gui2/ui.py
View File

@ -608,6 +608,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
self.update_checker.terminate()
self.listener.close()
self.job_manager.server.close()
self.job_manager.threaded_server.close()
while self.spare_servers:
self.spare_servers.pop().close()
self.device_manager.keep_going = False
@ -616,8 +617,6 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
mb.stop()
self.hide_windows()
if self.emailer.is_alive():
self.emailer.stop()
try:
try:
if self.content_server is not None:

2
src/calibre/library/database2.py
View File

@ -1781,7 +1781,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
path_changed = True
if set_authors:
if not mi.authors:
mi.authors = [_('Unknown')]
mi.authors = [_('Unknown')]
authors = []
for a in mi.authors:
authors += string_to_authors(a)

10
src/calibre/utils/ipc/job.py
View File

@ -75,12 +75,20 @@ class BaseJob(object):
self._run_state = self.RUNNING
self._status_text = _('Working...')
while consume_notifications:
if consume_notifications:
return self.consume_notifications()
return False
def consume_notifications(self):
got_notification = False
while self.notifications is not None:
try:
self.percent, self._message = self.notifications.get_nowait()
self.percent *= 100.
got_notification = True
except Empty:
break
return got_notification
@property
def status_text(self):