mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-07-09 03:04:10 -04:00
Bulk downloading of metadata/covers now shows progress and can be canceled
This commit is contained in:
Kovid Goyal
parent
f90e1a7bd8
commit
028dbbcf9b
@ -8,15 +8,14 @@ __docformat__ = 'restructuredtext en'
|
||||
import os
|
||||
from functools import partial
|
||||
|
||||
from PyQt4.Qt import Qt, QTimer, QMenu
|
||||
from PyQt4.Qt import Qt, QMenu
|
||||
|
||||
from calibre.gui2 import error_dialog, config, warning_dialog
|
||||
from calibre.gui2 import error_dialog, config
|
||||
from calibre.gui2.dialogs.metadata_single import MetadataSingleDialog
|
||||
from calibre.gui2.dialogs.metadata_bulk import MetadataBulkDialog
|
||||
from calibre.gui2.dialogs.confirm_delete import confirm
|
||||
from calibre.gui2.dialogs.tag_list_editor import TagListEditor
|
||||
from calibre.gui2.actions import InterfaceAction
|
||||
from calibre.gui2.dialogs.progress import BlockingBusy
|
||||
|
||||
class EditMetadataAction(InterfaceAction):
|
||||
|
||||
@ -90,51 +89,27 @@ class EditMetadataAction(InterfaceAction):
|
||||
get_social_metadata = config['get_social_metadata']
|
||||
else:
|
||||
get_social_metadata = set_social_metadata
|
||||
from calibre.gui2.metadata import DownloadMetadata
|
||||
self._download_book_metadata = DownloadMetadata(db, ids,
|
||||
from calibre.gui2.metadata import DoDownload
|
||||
if set_social_metadata is not None and set_social_metadata:
|
||||
x = _('social metadata')
|
||||
else:
|
||||
x = _('covers') if covers and not set_metadata else _('metadata')
|
||||
title = _('Downloading %s for %d book(s)')%(x, len(ids))
|
||||
self._download_book_metadata = DoDownload(self.gui, title, db, ids,
|
||||
get_covers=covers, set_metadata=set_metadata,
|
||||
get_social_metadata=get_social_metadata)
|
||||
m.stop_metadata_backup()
|
||||
try:
|
||||
self._download_book_metadata.start()
|
||||
if set_social_metadata is not None and set_social_metadata:
|
||||
x = _('social metadata')
|
||||
else:
|
||||
x = _('covers') if covers and not set_metadata else _('metadata')
|
||||
self._book_metadata_download_check = QTimer(self.gui)
|
||||
self._book_metadata_download_check.timeout.connect(self.book_metadata_download_check,
|
||||
type=Qt.QueuedConnection)
|
||||
self._book_metadata_download_check.start(100)
|
||||
self._bb_dialog = BlockingBusy(_('Downloading %s for %d book(s)')%(x,
|
||||
len(ids)), parent=self.gui)
|
||||
self._bb_dialog.exec_()
|
||||
self._download_book_metadata.exec_()
|
||||
finally:
|
||||
m.start_metadata_backup()
|
||||
|
||||
def book_metadata_download_check(self):
|
||||
if self._download_book_metadata.is_alive():
|
||||
return
|
||||
self._book_metadata_download_check.stop()
|
||||
self._bb_dialog.accept()
|
||||
cr = self.gui.library_view.currentIndex().row()
|
||||
x = self._download_book_metadata
|
||||
self._download_book_metadata = None
|
||||
if x.exception is None:
|
||||
if x.updated:
|
||||
self.gui.library_view.model().refresh_ids(
|
||||
x.updated, cr)
|
||||
if self.gui.cover_flow:
|
||||
self.gui.cover_flow.dataChanged()
|
||||
if x.failures:
|
||||
details = ['%s: %s'%(title, reason) for title,
|
||||
reason in x.failures.values()]
|
||||
details = '%s\n'%('\n'.join(details))
|
||||
warning_dialog(self.gui, _('Failed to download some metadata'),
|
||||
_('Failed to download metadata for the following:'),
|
||||
det_msg=details).exec_()
|
||||
else:
|
||||
err = _('Failed to download metadata:')
|
||||
error_dialog(self.gui, _('Error'), err, det_msg=x.tb).exec_()
|
||||
|
||||
|
||||
def edit_metadata(self, checked, bulk=None):
|
||||
'''
|
||||
|
||||
@ -9,52 +9,59 @@ __docformat__ = 'restructuredtext en'
|
||||
import traceback
|
||||
from threading import Thread
|
||||
from Queue import Queue, Empty
|
||||
from functools import partial
|
||||
|
||||
from PyQt4.Qt import QObject, Qt, pyqtSignal, QTimer, QDialog, \
|
||||
QVBoxLayout, QTextBrowser, QLabel, QGroupBox, QDialogButtonBox
|
||||
|
||||
from calibre.ebooks.metadata.fetch import search, get_social_metadata
|
||||
from calibre.gui2 import config
|
||||
from calibre.gui2 import config, error_dialog
|
||||
from calibre.gui2.dialogs.progress import ProgressDialog
|
||||
from calibre.ebooks.metadata.covers import download_cover
|
||||
from calibre.customize.ui import get_isbndb_key
|
||||
from calibre import prints
|
||||
from calibre.constants import DEBUG
|
||||
|
||||
class Worker(Thread):
|
||||
'Cover downloader'
|
||||
|
||||
def __init__(self):
|
||||
Thread.__init__(self)
|
||||
self.setDaemon(True)
|
||||
self.daemon = True
|
||||
self.jobs = Queue()
|
||||
self.results = Queue()
|
||||
|
||||
def run(self):
|
||||
while True:
|
||||
mi = self.jobs.get()
|
||||
id, mi = self.jobs.get()
|
||||
if not getattr(mi, 'isbn', False):
|
||||
break
|
||||
try:
|
||||
cdata, errors = download_cover(mi)
|
||||
if cdata:
|
||||
self.results.put((mi.isbn, cdata))
|
||||
elif DEBUG:
|
||||
prints('Cover download failed:', errors)
|
||||
self.results.put((id, mi, True, cdata))
|
||||
else:
|
||||
msg = []
|
||||
for e in errors:
|
||||
if not e[0]:
|
||||
msg.append(e[-1] + ' - ' + e[1])
|
||||
self.results.put((id, mi, False, '\n'.join(msg)))
|
||||
except:
|
||||
traceback.print_exc()
|
||||
self.results.put((id, mi, False, traceback.format_exc()))
|
||||
|
||||
def __enter__(self):
|
||||
self.start()
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
self.jobs.put(False)
|
||||
self.jobs.put((False, False))
|
||||
|
||||
|
||||
class DownloadMetadata(Thread):
|
||||
'Metadata downloader'
|
||||
|
||||
def __init__(self, db, ids, get_covers, set_metadata=True,
|
||||
get_social_metadata=True):
|
||||
Thread.__init__(self)
|
||||
self.setDaemon(True)
|
||||
self.daemon = True
|
||||
self.metadata = {}
|
||||
self.covers = {}
|
||||
self.set_metadata = set_metadata
|
||||
@ -64,34 +71,42 @@ class DownloadMetadata(Thread):
|
||||
self.updated = set([])
|
||||
self.get_covers = get_covers
|
||||
self.worker = Worker()
|
||||
self.results = Queue()
|
||||
self.keep_going = True
|
||||
for id in ids:
|
||||
self.metadata[id] = db.get_metadata(id, index_is_id=True)
|
||||
self.metadata[id].rating = None
|
||||
self.total = len(ids)
|
||||
if self.get_covers:
|
||||
self.total += len(ids)
|
||||
self.fetched_metadata = {}
|
||||
self.fetched_covers = {}
|
||||
self.failures = {}
|
||||
self.cover_failures = {}
|
||||
self.exception = self.tb = None
|
||||
|
||||
def run(self):
|
||||
self.exception = self.tb = None
|
||||
try:
|
||||
self._run()
|
||||
except Exception, e:
|
||||
self.exception = e
|
||||
import traceback
|
||||
self.tb = traceback.format_exc()
|
||||
|
||||
def _run(self):
|
||||
self.key = get_isbndb_key()
|
||||
if not self.key:
|
||||
self.key = None
|
||||
self.fetched_metadata = {}
|
||||
self.failures = {}
|
||||
with self.worker:
|
||||
for id, mi in self.metadata.items():
|
||||
if not self.keep_going:
|
||||
break
|
||||
args = {}
|
||||
if mi.isbn:
|
||||
args['isbn'] = mi.isbn
|
||||
else:
|
||||
if mi.is_null('title'):
|
||||
self.failures[id] = \
|
||||
(str(id), _('Book has neither title nor ISBN'))
|
||||
_('Book has neither title nor ISBN')
|
||||
continue
|
||||
args['title'] = mi.title
|
||||
if mi.authors and mi.authors[0] != _('Unknown'):
|
||||
@ -102,8 +117,11 @@ class DownloadMetadata(Thread):
|
||||
if results:
|
||||
fmi = results[0]
|
||||
self.fetched_metadata[id] = fmi
|
||||
if fmi.isbn and self.get_covers:
|
||||
self.worker.jobs.put(fmi)
|
||||
if self.get_covers:
|
||||
if fmi.isbn:
|
||||
self.worker.jobs.put((id, fmi))
|
||||
else:
|
||||
self.results.put((id, 'cover', False, mi.title))
|
||||
if (not config['overwrite_author_title_metadata']):
|
||||
fmi.authors = mi.authors
|
||||
fmi.author_sort = mi.author_sort
|
||||
@ -115,42 +133,163 @@ class DownloadMetadata(Thread):
|
||||
mi.rating *= 2
|
||||
if not self.get_social_metadata:
|
||||
mi.tags = []
|
||||
self.results.put((id, 'metadata', True, mi.title))
|
||||
else:
|
||||
self.failures[id] = (mi.title,
|
||||
_('No matches found for this book'))
|
||||
self.failures[id] = _('No matches found for this book')
|
||||
self.results.put((id, 'metadata', False, mi.title))
|
||||
self.results.put((id, 'cover', False, mi.title))
|
||||
self.commit_covers()
|
||||
|
||||
self.commit_covers(True)
|
||||
for id in self.fetched_metadata:
|
||||
mi = self.metadata[id]
|
||||
if self.set_metadata:
|
||||
self.db.set_metadata(id, mi)
|
||||
if not self.set_metadata and self.get_social_metadata:
|
||||
if mi.rating:
|
||||
self.db.set_rating(id, mi.rating)
|
||||
if mi.tags:
|
||||
self.db.set_tags(id, mi.tags)
|
||||
if mi.comments:
|
||||
self.db.set_comment(id, mi.comments)
|
||||
if mi.series:
|
||||
self.db.set_series(id, mi.series)
|
||||
if mi.series_index is not None:
|
||||
self.db.set_series_index(id, mi.series_index)
|
||||
|
||||
self.updated = set(self.fetched_metadata)
|
||||
|
||||
|
||||
def commit_covers(self, all=False):
|
||||
if all:
|
||||
self.worker.jobs.put(False)
|
||||
while True:
|
||||
try:
|
||||
isbn, cdata = self.worker.results.get(False)
|
||||
for id, mi in self.metadata.items():
|
||||
if mi.isbn == isbn:
|
||||
self.db.set_cover(id, cdata)
|
||||
id, fmi, ok, cdata = self.worker.results.get(False)
|
||||
if ok:
|
||||
self.fetched_covers[id] = cdata
|
||||
self.results.put((id, 'cover', ok, fmi.title))
|
||||
else:
|
||||
self.results.put((id, 'cover', ok, fmi.title))
|
||||
try:
|
||||
self.cover_failures[id] = unicode(cdata)
|
||||
except:
|
||||
self.cover_failures[id] = repr(cdata)
|
||||
except Empty:
|
||||
if not all or not self.worker.is_alive():
|
||||
return
|
||||
|
||||
class DoDownload(QObject):
|
||||
|
||||
idle_process = pyqtSignal()
|
||||
|
||||
def __init__(self, parent, title, db, ids, get_covers, set_metadata=True,
|
||||
get_social_metadata=True):
|
||||
QObject.__init__(self, parent)
|
||||
self.pd = ProgressDialog(title, min=0, max=0, parent=parent)
|
||||
self.pd.canceled_signal.connect(self.cancel)
|
||||
self.idle_process.connect(self.do_one, type=Qt.QueuedConnection)
|
||||
self.downloader = None
|
||||
self.create = partial(DownloadMetadata, db, ids, get_covers,
|
||||
set_metadata=set_metadata,
|
||||
get_social_metadata=get_social_metadata)
|
||||
self.timer = QTimer(self)
|
||||
self.timer.timeout.connect(self.do_one, type=Qt.QueuedConnection)
|
||||
self.db = db
|
||||
self.updated = set([])
|
||||
self.total = len(ids)
|
||||
|
||||
def exec_(self):
|
||||
self.timer.start(50)
|
||||
ret = self.pd.exec_()
|
||||
if getattr(self.downloader, 'exception', None) is not None and \
|
||||
ret == self.pd.Accepted:
|
||||
error_dialog(self.parent(), _('Failed'),
|
||||
_('Failed to download metadata'), show=True)
|
||||
else:
|
||||
self.show_report()
|
||||
return ret
|
||||
|
||||
def cancel(self, *args):
|
||||
self.timer.stop()
|
||||
self.downloader.keep_going = False
|
||||
self.pd.reject()
|
||||
|
||||
def do_one(self):
|
||||
if self.downloader is None:
|
||||
self.downloader = self.create()
|
||||
self.downloader.start()
|
||||
self.pd.set_min(0)
|
||||
self.pd.set_max(self.downloader.total)
|
||||
try:
|
||||
r = self.downloader.results.get_nowait()
|
||||
self.handle_result(r)
|
||||
except Empty:
|
||||
pass
|
||||
if not self.downloader.is_alive():
|
||||
self.timer.stop()
|
||||
self.pd.accept()
|
||||
|
||||
def handle_result(self, r):
|
||||
id_, typ, ok, title = r
|
||||
what = _('cover') if typ == 'cover' else _('metadata')
|
||||
which = _('Downloaded') if ok else _('Failed to get')
|
||||
self.pd.set_msg(_('%s %s for: %s') % (which, what, title))
|
||||
self.pd.value += 1
|
||||
if ok:
|
||||
self.updated.add(id_)
|
||||
if typ == 'cover':
|
||||
try:
|
||||
self.db.set_cover(id_,
|
||||
self.downloader.fetched_covers.pop(id_))
|
||||
except:
|
||||
self.downloader.cover_failures[id_] = \
|
||||
traceback.format_exc()
|
||||
else:
|
||||
try:
|
||||
self.set_metadata(id_)
|
||||
except:
|
||||
self.downloader.failures[id_] = \
|
||||
traceback.format_exc()
|
||||
|
||||
def set_metadata(self, id_):
|
||||
mi = self.downloader.metadata[id_]
|
||||
if self.downloader.set_metadata:
|
||||
self.db.set_metadata(id_, mi)
|
||||
if not self.downloader.set_metadata and self.downloader.get_social_metadata:
|
||||
if mi.rating:
|
||||
self.db.set_rating(id_, mi.rating)
|
||||
if mi.tags:
|
||||
self.db.set_tags(id_, mi.tags)
|
||||
if mi.comments:
|
||||
self.db.set_comment(id_, mi.comments)
|
||||
if mi.series:
|
||||
self.db.set_series(id_, mi.series)
|
||||
if mi.series_index is not None:
|
||||
self.db.set_series_index(id_, mi.series_index)
|
||||
|
||||
def show_report(self):
|
||||
f, cf = self.downloader.failures, self.downloader.cover_failures
|
||||
report = []
|
||||
if f:
|
||||
report.append(
|
||||
'<h3>Failed to download metadata for the following:</h3><ol>')
|
||||
for id_, err in f.items():
|
||||
mi = self.downloader.metadata[id_]
|
||||
report.append('<li><b>%s</b><pre>%s</pre></li>' % (mi.title,
|
||||
unicode(err)))
|
||||
report.append('</ol>')
|
||||
if cf:
|
||||
report.append(
|
||||
'<h3>Failed to download cover for the following:</h3><ol>')
|
||||
for id_, err in cf.items():
|
||||
mi = self.downloader.metadata[id_]
|
||||
report.append('<li><b>%s</b><pre>%s</pre></li>' % (mi.title,
|
||||
unicode(err)))
|
||||
report.append('</ol>')
|
||||
|
||||
if len(self.updated) != self.total or report:
|
||||
d = QDialog(self.parent())
|
||||
bb = QDialogButtonBox(QDialogButtonBox.Ok, parent=d)
|
||||
v1 = QVBoxLayout()
|
||||
d.setLayout(v1)
|
||||
d.setWindowTitle(_('Done'))
|
||||
v1.addWidget(QLabel(_('Successfully downloaded metadata for %d out of %d books') %
|
||||
(len(self.updated), self.total)))
|
||||
gb = QGroupBox(_('Details'), self.parent())
|
||||
v2 = QVBoxLayout()
|
||||
gb.setLayout(v2)
|
||||
b = QTextBrowser(self.parent())
|
||||
v2.addWidget(b)
|
||||
b.setHtml('\n'.join(report))
|
||||
v1.addWidget(gb)
|
||||
v1.addWidget(bb)
|
||||
bb.accepted.connect(d.accept)
|
||||
d.resize(800, 600)
|
||||
d.exec_()
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user