Bulk downloading of metadata/covers now shows progress and can be canceled

src/calibre/gui2/actions/edit_metadata.py

@@ -8,15 +8,14 @@ __docformat__ = 'restructuredtext en'
 import os
 from functools import partial
 
-from PyQt4.Qt import Qt, QTimer, QMenu
+from PyQt4.Qt import Qt, QMenu
 
-from calibre.gui2 import error_dialog, config, warning_dialog
+from calibre.gui2 import error_dialog, config
 from calibre.gui2.dialogs.metadata_single import MetadataSingleDialog
 from calibre.gui2.dialogs.metadata_bulk import MetadataBulkDialog
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2.dialogs.tag_list_editor import TagListEditor
 from calibre.gui2.actions import InterfaceAction
-from calibre.gui2.dialogs.progress import BlockingBusy
 
 class EditMetadataAction(InterfaceAction):
 
@@ -90,51 +89,27 @@ class EditMetadataAction(InterfaceAction):
             get_social_metadata = config['get_social_metadata']
         else:
             get_social_metadata = set_social_metadata
-        from calibre.gui2.metadata import DownloadMetadata
+        from calibre.gui2.metadata import DoDownload
-        self._download_book_metadata = DownloadMetadata(db, ids,
+        if set_social_metadata is not None and set_social_metadata:
+            x = _('social metadata')
+        else:
+            x = _('covers') if covers and not set_metadata else _('metadata')
+        title = _('Downloading %s for %d book(s)')%(x, len(ids))
+        self._download_book_metadata = DoDownload(self.gui, title, db, ids,
                 get_covers=covers, set_metadata=set_metadata,
                 get_social_metadata=get_social_metadata)
         m.stop_metadata_backup()
         try:
-            self._download_book_metadata.start()
+            self._download_book_metadata.exec_()
-            if set_social_metadata is not None and set_social_metadata:
-                x = _('social metadata')
-            else:
-                x = _('covers') if covers and not set_metadata else _('metadata')
-            self._book_metadata_download_check = QTimer(self.gui)
-            self._book_metadata_download_check.timeout.connect(self.book_metadata_download_check,
-                    type=Qt.QueuedConnection)
-            self._book_metadata_download_check.start(100)
-            self._bb_dialog = BlockingBusy(_('Downloading %s for %d book(s)')%(x,
-                len(ids)), parent=self.gui)
-            self._bb_dialog.exec_()
         finally:
             m.start_metadata_backup()
-
-    def book_metadata_download_check(self):
-        if self._download_book_metadata.is_alive():
-            return
-        self._book_metadata_download_check.stop()
-        self._bb_dialog.accept()
         cr = self.gui.library_view.currentIndex().row()
         x = self._download_book_metadata
-        self._download_book_metadata = None
+        if x.updated:
-        if x.exception is None:
             self.gui.library_view.model().refresh_ids(
                 x.updated, cr)
             if self.gui.cover_flow:
                 self.gui.cover_flow.dataChanged()
-            if x.failures:
-                details = ['%s: %s'%(title, reason) for title,
-                        reason in x.failures.values()]
-                details = '%s\n'%('\n'.join(details))
-                warning_dialog(self.gui, _('Failed to download some metadata'),
-                    _('Failed to download metadata for the following:'),
-                    det_msg=details).exec_()
-        else:
-            err = _('Failed to download metadata:')
-            error_dialog(self.gui, _('Error'), err, det_msg=x.tb).exec_()
-
 
     def edit_metadata(self, checked, bulk=None):
         '''

src/calibre/gui2/metadata.py

@@ -9,52 +9,59 @@ __docformat__ = 'restructuredtext en'
 import traceback
 from threading import Thread
 from Queue import Queue, Empty
+from functools import partial
 
+from PyQt4.Qt import QObject, Qt, pyqtSignal, QTimer, QDialog, \
+        QVBoxLayout, QTextBrowser, QLabel, QGroupBox, QDialogButtonBox
 
 from calibre.ebooks.metadata.fetch import search, get_social_metadata
-from calibre.gui2 import config
+from calibre.gui2 import config, error_dialog
+from calibre.gui2.dialogs.progress import ProgressDialog
 from calibre.ebooks.metadata.covers import download_cover
 from calibre.customize.ui import get_isbndb_key
-from calibre import prints
-from calibre.constants import DEBUG
 
 class Worker(Thread):
     'Cover downloader'
 
     def __init__(self):
         Thread.__init__(self)
-        self.setDaemon(True)
+        self.daemon = True
         self.jobs = Queue()
         self.results = Queue()
 
     def run(self):
         while True:
-            mi = self.jobs.get()
+            id, mi = self.jobs.get()
             if not getattr(mi, 'isbn', False):
                 break
             try:
                 cdata, errors = download_cover(mi)
                 if cdata:
-                    self.results.put((mi.isbn, cdata))
+                    self.results.put((id, mi, True, cdata))
-                elif DEBUG:
+                else:
-                    prints('Cover download failed:', errors)
+                    msg = []
+                    for e in errors:
+                        if not e[0]:
+                            msg.append(e[-1] + ' - ' + e[1])
+                    self.results.put((id, mi, False, '\n'.join(msg)))
             except:
-                traceback.print_exc()
+                self.results.put((id, mi, False, traceback.format_exc()))
 
     def __enter__(self):
         self.start()
         return self
 
     def __exit__(self, *args):
-        self.jobs.put(False)
+        self.jobs.put((False, False))
 
 
 class DownloadMetadata(Thread):
+    'Metadata downloader'
 
     def __init__(self, db, ids, get_covers, set_metadata=True,
             get_social_metadata=True):
         Thread.__init__(self)
-        self.setDaemon(True)
+        self.daemon = True
         self.metadata = {}
         self.covers   = {}
         self.set_metadata = set_metadata
@@ -64,34 +71,42 @@ class DownloadMetadata(Thread):
         self.updated = set([])
         self.get_covers = get_covers
         self.worker = Worker()
+        self.results = Queue()
+        self.keep_going = True
         for id in ids:
             self.metadata[id] = db.get_metadata(id, index_is_id=True)
             self.metadata[id].rating = None
+        self.total = len(ids)
+        if self.get_covers:
+            self.total += len(ids)
+        self.fetched_metadata = {}
+        self.fetched_covers = {}
+        self.failures = {}
+        self.cover_failures = {}
+        self.exception = self.tb = None
 
     def run(self):
-        self.exception = self.tb = None
         try:
             self._run()
         except Exception, e:
             self.exception = e
-            import traceback
             self.tb = traceback.format_exc()
 
     def _run(self):
         self.key = get_isbndb_key()
         if not self.key:
             self.key = None
-        self.fetched_metadata = {}
-        self.failures = {}
         with self.worker:
             for id, mi in self.metadata.items():
+                if not self.keep_going:
+                    break
                 args = {}
                 if mi.isbn:
                     args['isbn'] = mi.isbn
                 else:
                     if mi.is_null('title'):
                         self.failures[id] = \
-                                (str(id), _('Book has neither title nor ISBN'))
+                                _('Book has neither title nor ISBN')
                         continue
                     args['title'] = mi.title
                     if mi.authors and mi.authors[0] != _('Unknown'):
@@ -102,8 +117,11 @@ class DownloadMetadata(Thread):
                 if results:
                     fmi = results[0]
                     self.fetched_metadata[id] = fmi
-                    if fmi.isbn and self.get_covers:
+                    if self.get_covers:
-                        self.worker.jobs.put(fmi)
+                        if fmi.isbn:
+                            self.worker.jobs.put((id, fmi))
+                        else:
+                            self.results.put((id, 'cover', False, mi.title))
                     if (not config['overwrite_author_title_metadata']):
                         fmi.authors = mi.authors
                         fmi.author_sort = mi.author_sort
@@ -115,42 +133,163 @@ class DownloadMetadata(Thread):
                             mi.rating *= 2
                     if not self.get_social_metadata:
                         mi.tags = []
+                    self.results.put((id, 'metadata', True, mi.title))
                 else:
-                    self.failures[id] = (mi.title,
+                    self.failures[id] = _('No matches found for this book')
-                        _('No matches found for this book'))
+                    self.results.put((id, 'metadata', False, mi.title))
+                    self.results.put((id, 'cover', False, mi.title))
                 self.commit_covers()
 
         self.commit_covers(True)
-        for id in self.fetched_metadata:
-            mi = self.metadata[id]
-            if self.set_metadata:
-                self.db.set_metadata(id, mi)
-            if not self.set_metadata and self.get_social_metadata:
-                if mi.rating:
-                    self.db.set_rating(id, mi.rating)
-                if mi.tags:
-                    self.db.set_tags(id, mi.tags)
-                if mi.comments:
-                    self.db.set_comment(id, mi.comments)
-                if mi.series:
-                    self.db.set_series(id, mi.series)
-                    if mi.series_index is not None:
-                        self.db.set_series_index(id, mi.series_index)
-
-        self.updated = set(self.fetched_metadata)
-
 
     def commit_covers(self, all=False):
         if all:
             self.worker.jobs.put(False)
         while True:
             try:
-                isbn, cdata = self.worker.results.get(False)
+                id, fmi, ok, cdata = self.worker.results.get(False)
-                for id, mi in self.metadata.items():
+                if ok:
-                    if mi.isbn == isbn:
+                    self.fetched_covers[id] = cdata
-                        self.db.set_cover(id, cdata)
+                    self.results.put((id, 'cover', ok, fmi.title))
+                else:
+                    self.results.put((id, 'cover', ok, fmi.title))
+                    try:
+                        self.cover_failures[id] = unicode(cdata)
+                    except:
+                        self.cover_failures[id] = repr(cdata)
             except Empty:
                 if not all or not self.worker.is_alive():
                     return
 
+class DoDownload(QObject):
+
+    idle_process = pyqtSignal()
+
+    def __init__(self, parent, title, db, ids, get_covers, set_metadata=True,
+            get_social_metadata=True):
+        QObject.__init__(self, parent)
+        self.pd = ProgressDialog(title, min=0, max=0, parent=parent)
+        self.pd.canceled_signal.connect(self.cancel)
+        self.idle_process.connect(self.do_one, type=Qt.QueuedConnection)
+        self.downloader = None
+        self.create = partial(DownloadMetadata, db, ids, get_covers,
+                set_metadata=set_metadata,
+                get_social_metadata=get_social_metadata)
+        self.timer = QTimer(self)
+        self.timer.timeout.connect(self.do_one, type=Qt.QueuedConnection)
+        self.db = db
+        self.updated = set([])
+        self.total = len(ids)
+
+    def exec_(self):
+        self.timer.start(50)
+        ret = self.pd.exec_()
+        if getattr(self.downloader, 'exception', None) is not None and \
+                ret == self.pd.Accepted:
+            error_dialog(self.parent(), _('Failed'),
+                    _('Failed to download metadata'), show=True)
+        else:
+            self.show_report()
+        return ret
+
+    def cancel(self, *args):
+        self.timer.stop()
+        self.downloader.keep_going = False
+        self.pd.reject()
+
+    def do_one(self):
+        if self.downloader is None:
+            self.downloader = self.create()
+            self.downloader.start()
+            self.pd.set_min(0)
+            self.pd.set_max(self.downloader.total)
+        try:
+            r = self.downloader.results.get_nowait()
+            self.handle_result(r)
+        except Empty:
+            pass
+        if not self.downloader.is_alive():
+            self.timer.stop()
+            self.pd.accept()
+
+    def handle_result(self, r):
+        id_, typ, ok, title = r
+        what = _('cover') if typ == 'cover' else _('metadata')
+        which = _('Downloaded') if ok else _('Failed to get')
+        self.pd.set_msg(_('%s %s for: %s') % (which, what, title))
+        self.pd.value += 1
+        if ok:
+            self.updated.add(id_)
+            if typ == 'cover':
+                try:
+                    self.db.set_cover(id_,
+                            self.downloader.fetched_covers.pop(id_))
+                except:
+                    self.downloader.cover_failures[id_] = \
+                            traceback.format_exc()
+            else:
+                try:
+                    self.set_metadata(id_)
+                except:
+                    self.downloader.failures[id_] = \
+                            traceback.format_exc()
+
+    def set_metadata(self, id_):
+        mi = self.downloader.metadata[id_]
+        if self.downloader.set_metadata:
+            self.db.set_metadata(id_, mi)
+        if not self.downloader.set_metadata and self.downloader.get_social_metadata:
+            if mi.rating:
+                self.db.set_rating(id_, mi.rating)
+            if mi.tags:
+                self.db.set_tags(id_, mi.tags)
+            if mi.comments:
+                self.db.set_comment(id_, mi.comments)
+            if mi.series:
+                self.db.set_series(id_, mi.series)
+                if mi.series_index is not None:
+                    self.db.set_series_index(id_, mi.series_index)
+
+    def show_report(self):
+        f, cf = self.downloader.failures, self.downloader.cover_failures
+        report = []
+        if f:
+            report.append(
+                '<h3>Failed to download metadata for the following:</h3><ol>')
+            for id_, err in f.items():
+                mi = self.downloader.metadata[id_]
+                report.append('<li><b>%s</b><pre>%s</pre></li>' % (mi.title,
+                    unicode(err)))
+            report.append('</ol>')
+        if cf:
+            report.append(
+                '<h3>Failed to download cover for the following:</h3><ol>')
+            for id_, err in cf.items():
+                mi = self.downloader.metadata[id_]
+                report.append('<li><b>%s</b><pre>%s</pre></li>' % (mi.title,
+                    unicode(err)))
+            report.append('</ol>')
+
+        if len(self.updated) != self.total or report:
+            d = QDialog(self.parent())
+            bb = QDialogButtonBox(QDialogButtonBox.Ok, parent=d)
+            v1 = QVBoxLayout()
+            d.setLayout(v1)
+            d.setWindowTitle(_('Done'))
+            v1.addWidget(QLabel(_('Successfully downloaded metadata for %d out of %d books') %
+                (len(self.updated), self.total)))
+            gb = QGroupBox(_('Details'), self.parent())
+            v2 = QVBoxLayout()
+            gb.setLayout(v2)
+            b = QTextBrowser(self.parent())
+            v2.addWidget(b)
+            b.setHtml('\n'.join(report))
+            v1.addWidget(gb)
+            v1.addWidget(bb)
+            bb.accepted.connect(d.accept)
+            d.resize(800, 600)
+            d.exec_()
+
+
+