Fix logging for the new bulk metadata download path

src/calibre/ebooks/metadata/sources/worker.py

@@ -89,7 +89,7 @@ def main(do_identify, covers, metadata, ensure_fields):
                 all_failed = False
 
         with open('%d.log'%book_id, 'wb') as f:
-            f.write(log.html.encode('utf-8'))
+            f.write(log.plain_text.encode('utf-8'))
 
     return failed_ids, failed_covers, all_failed
 

src/calibre/gui2/actions/edit_metadata.py

@@ -80,15 +80,11 @@ class EditMetadataAction(InterfaceAction):
                 Dispatcher(self.metadata_downloaded),
                 ensure_fields=ensure_fields)
 
-    def cleanup_bulk_download(self, tdir, log_file):
+    def cleanup_bulk_download(self, tdir):
         try:
             shutil.rmtree(tdir, ignore_errors=True)
         except:
             pass
-        try:
-            os.remove(log_file)
-        except:
-            pass
 
     def metadata_downloaded(self, job):
         if job.failed:
@@ -98,9 +94,9 @@ class EditMetadataAction(InterfaceAction):
         (aborted, id_map, tdir, log_file, failed_ids, failed_covers, all_failed,
                 det_msg, lm_map) = get_job_details(job)
         if aborted:
-            return self.cleanup_bulk_download(tdir, log_file)
+            return self.cleanup_bulk_download(tdir)
         if all_failed:
-            self.cleanup_bulk_download(tdir, log_file)
+            self.cleanup_bulk_download(tdir)
             return error_dialog(self.gui, _('Download failed'),
             _('Failed to download metadata or covers for any of the %d'
                ' book(s).') % len(id_map), det_msg=det_msg, show=True)
@@ -120,10 +116,10 @@ class EditMetadataAction(InterfaceAction):
         payload = (id_map, tdir, log_file, lm_map)
         from calibre.gui2.dialogs.message_box import ProceedNotification
         p = ProceedNotification(self.apply_downloaded_metadata,
-                payload, open(log_file, 'rb').read().decode('utf-8'),
+                payload, log_file,
                 _('Download log'), _('Download complete'), msg,
                 det_msg=det_msg, show_copy_button=show_copy_button,
-                parent=self.gui)
+                parent=self.gui, log_is_file=True)
         p.show()
 
     def apply_downloaded_metadata(self, payload):
@@ -167,7 +163,7 @@ class EditMetadataAction(InterfaceAction):
             id_map[bid] = (opf, cov)
 
         self.apply_metadata_changes(id_map, callback=lambda x:
-                self.cleanup_bulk_download(tdir, log_file))
+                self.cleanup_bulk_download(tdir))
 
     # }}}
 

src/calibre/gui2/dialogs/message_box.py

@@ -160,7 +160,7 @@ class ProceedNotification(MessageBox): # {{{
 
     def __init__(self, callback, payload, html_log, log_viewer_title, title, msg,
             det_msg='', show_copy_button=False, parent=None,
-            cancel_callback=None):
+            cancel_callback=None, log_is_file=False):
         '''
         A non modal popup that notifies the user that a background task has
         been completed.
@@ -175,12 +175,15 @@ class ProceedNotification(MessageBox): # {{{
         :param title: The title for this popup
         :param msg: The msg to display
         :param det_msg: Detailed message
+        :param log_is_file: If True the html_log parameter is interpreted as
+        the path to a file on disk containing the log encoded with utf-8
         '''
         MessageBox.__init__(self, MessageBox.QUESTION, title, msg,
                 det_msg=det_msg, show_copy_button=show_copy_button,
                 parent=parent)
         self.payload = payload
         self.html_log = html_log
+        self.log_is_file = log_is_file
         self.log_viewer_title = log_viewer_title
 
         self.vlb = self.bb.addButton(_('View log'), self.bb.ActionRole)
@@ -192,7 +195,11 @@ class ProceedNotification(MessageBox): # {{{
         _proceed_memory.append(self)
 
     def show_log(self):
-        self.log_viewer = ViewLog(self.log_viewer_title, self.html_log,
+        log = self.html_log
+        if self.log_is_file:
+            with open(log, 'rb') as f:
+                log = f.read().decode('utf-8')
+        self.log_viewer = ViewLog(self.log_viewer_title, log,
                 parent=self)
 
     def do_proceed(self, result):

src/calibre/gui2/jobs.py

@@ -402,7 +402,8 @@ class DetailView(QDialog, Ui_Dialog): # {{{
         self.setupUi(self)
         self.setWindowTitle(job.description)
         self.job = job
-        self.html_view = hasattr(job, 'html_details')
+        self.html_view = (hasattr(job, 'html_details') and not getattr(job,
+            'ignore_html_details', False))
         if self.html_view:
             self.log.setVisible(False)
         else:

src/calibre/gui2/metadata/bulk_download.py

@@ -20,6 +20,28 @@ from calibre.ptempfile import (PersistentTemporaryDirectory,
         PersistentTemporaryFile)
 
 # Start download {{{
+
+class Job(ThreadedJob):
+
+    ignore_html_details = True
+
+    def consolidate_log(self):
+        self.consolidated_log = self.log.plain_text
+        self.log = None
+
+    def read_consolidated_log(self):
+        return self.consolidated_log
+
+    @property
+    def details(self):
+        if self.consolidated_log is None:
+            return self.log.plain_text
+        return self.read_consolidated_log()
+
+    @property
+    def log_file(self):
+        return open(self.download_debug_log, 'rb')
+
 def show_config(gui, parent):
     from calibre.gui2.preferences import show_config_widget
     show_config_widget('Sharing', 'Metadata download', parent=parent,
@@ -101,11 +123,14 @@ def start_download(gui, ids, callback, ensure_fields=None):
     d.b.clicked.disconnect()
     if ret != d.Accepted:
         return
+    tf = PersistentTemporaryFile('_metadata_bulk_log_')
+    tf.close()
 
-    job = ThreadedJob('metadata bulk download',
+    job = Job('metadata bulk download',
         _('Download metadata for %d books')%len(ids),
-        download, (ids, gui.current_db, d.identify, d.covers,
+        download, (ids, tf.name, gui.current_db, d.identify, d.covers,
             ensure_fields), {}, callback)
+    job.download_debug_log = tf.name
     gui.job_manager.run_threaded_job(job)
     gui.status_bar.show_message(_('Metadata download started'), 3000)
 
@@ -144,18 +169,15 @@ class HeartBeat(object):
             self.last_time = time.time()
         return True
 
-# Fix log viewer, get_job_details, database update code
+# Fix log viewer, ratings
 # Test: abort, covers only, metadata only, both, 200 entry download, memory
 # consumption, all errors and on and on
 
-def download(all_ids, db, do_identify, covers, ensure_fields,
+def download(all_ids, tf, db, do_identify, covers, ensure_fields,
         log=None, abort=None, notifications=None):
     batch_size = 10
     batches = split_jobs(all_ids, batch_size=batch_size)
     tdir = PersistentTemporaryDirectory('_metadata_bulk_')
-    tf = PersistentTemporaryFile('_metadata_bulk_log_')
-    tf.close()
-    tf = tf.name
     heartbeat = HeartBeat(tdir)
 
     failed_ids = set()
@@ -201,8 +223,10 @@ def download(all_ids, db, do_identify, covers, ensure_fields,
         for book_id in ids:
             lp = os.path.join(tdir, '%d.log'%book_id)
             if os.path.exists(lp):
-                with open(lp, 'rb') as f, open(tf, 'ab') as d:
-                    shutil.copyfileobj(f, d)
+                with open(tf, 'ab') as dest, open(lp, 'rb') as src:
+                    dest.write(('\n'+'#'*20 + ' Log for %s '%title_map[book_id] +
+                        '#'*20+'\n').encode('utf-8'))
+                    shutil.copyfileobj(src, dest)
 
     if abort.is_set():
         aborted = True