From d24e8e842e39def140f0e5f770e58444f86b8741 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 2 Apr 2012 15:25:46 +0530 Subject: [PATCH] Fix logging for the new bulk metadata download path --- src/calibre/ebooks/metadata/sources/worker.py | 2 +- src/calibre/gui2/actions/edit_metadata.py | 16 +++---- src/calibre/gui2/dialogs/message_box.py | 11 ++++- src/calibre/gui2/jobs.py | 3 +- src/calibre/gui2/metadata/bulk_download.py | 42 +++++++++++++++---- 5 files changed, 51 insertions(+), 23 deletions(-) diff --git a/src/calibre/ebooks/metadata/sources/worker.py b/src/calibre/ebooks/metadata/sources/worker.py index 91ca31d2b8..f2db60e01f 100644 --- a/src/calibre/ebooks/metadata/sources/worker.py +++ b/src/calibre/ebooks/metadata/sources/worker.py @@ -89,7 +89,7 @@ def main(do_identify, covers, metadata, ensure_fields): all_failed = False with open('%d.log'%book_id, 'wb') as f: - f.write(log.html.encode('utf-8')) + f.write(log.plain_text.encode('utf-8')) return failed_ids, failed_covers, all_failed diff --git a/src/calibre/gui2/actions/edit_metadata.py b/src/calibre/gui2/actions/edit_metadata.py index 4e6a4723f7..4a0d12e3d3 100644 --- a/src/calibre/gui2/actions/edit_metadata.py +++ b/src/calibre/gui2/actions/edit_metadata.py @@ -80,15 +80,11 @@ class EditMetadataAction(InterfaceAction): Dispatcher(self.metadata_downloaded), ensure_fields=ensure_fields) - def cleanup_bulk_download(self, tdir, log_file): + def cleanup_bulk_download(self, tdir): try: shutil.rmtree(tdir, ignore_errors=True) except: pass - try: - os.remove(log_file) - except: - pass def metadata_downloaded(self, job): if job.failed: @@ -98,9 +94,9 @@ class EditMetadataAction(InterfaceAction): (aborted, id_map, tdir, log_file, failed_ids, failed_covers, all_failed, det_msg, lm_map) = get_job_details(job) if aborted: - return self.cleanup_bulk_download(tdir, log_file) + return self.cleanup_bulk_download(tdir) if all_failed: - self.cleanup_bulk_download(tdir, log_file) + self.cleanup_bulk_download(tdir) return error_dialog(self.gui, _('Download failed'), _('Failed to download metadata or covers for any of the %d' ' book(s).') % len(id_map), det_msg=det_msg, show=True) @@ -120,10 +116,10 @@ class EditMetadataAction(InterfaceAction): payload = (id_map, tdir, log_file, lm_map) from calibre.gui2.dialogs.message_box import ProceedNotification p = ProceedNotification(self.apply_downloaded_metadata, - payload, open(log_file, 'rb').read().decode('utf-8'), + payload, log_file, _('Download log'), _('Download complete'), msg, det_msg=det_msg, show_copy_button=show_copy_button, - parent=self.gui) + parent=self.gui, log_is_file=True) p.show() def apply_downloaded_metadata(self, payload): @@ -167,7 +163,7 @@ class EditMetadataAction(InterfaceAction): id_map[bid] = (opf, cov) self.apply_metadata_changes(id_map, callback=lambda x: - self.cleanup_bulk_download(tdir, log_file)) + self.cleanup_bulk_download(tdir)) # }}} diff --git a/src/calibre/gui2/dialogs/message_box.py b/src/calibre/gui2/dialogs/message_box.py index fb0725651b..64c8bf75ba 100644 --- a/src/calibre/gui2/dialogs/message_box.py +++ b/src/calibre/gui2/dialogs/message_box.py @@ -160,7 +160,7 @@ class ProceedNotification(MessageBox): # {{{ def __init__(self, callback, payload, html_log, log_viewer_title, title, msg, det_msg='', show_copy_button=False, parent=None, - cancel_callback=None): + cancel_callback=None, log_is_file=False): ''' A non modal popup that notifies the user that a background task has been completed. @@ -175,12 +175,15 @@ class ProceedNotification(MessageBox): # {{{ :param title: The title for this popup :param msg: The msg to display :param det_msg: Detailed message + :param log_is_file: If True the html_log parameter is interpreted as + the path to a file on disk containing the log encoded with utf-8 ''' MessageBox.__init__(self, MessageBox.QUESTION, title, msg, det_msg=det_msg, show_copy_button=show_copy_button, parent=parent) self.payload = payload self.html_log = html_log + self.log_is_file = log_is_file self.log_viewer_title = log_viewer_title self.vlb = self.bb.addButton(_('View log'), self.bb.ActionRole) @@ -192,7 +195,11 @@ class ProceedNotification(MessageBox): # {{{ _proceed_memory.append(self) def show_log(self): - self.log_viewer = ViewLog(self.log_viewer_title, self.html_log, + log = self.html_log + if self.log_is_file: + with open(log, 'rb') as f: + log = f.read().decode('utf-8') + self.log_viewer = ViewLog(self.log_viewer_title, log, parent=self) def do_proceed(self, result): diff --git a/src/calibre/gui2/jobs.py b/src/calibre/gui2/jobs.py index 8c1b5388d7..c0d61332ab 100644 --- a/src/calibre/gui2/jobs.py +++ b/src/calibre/gui2/jobs.py @@ -402,7 +402,8 @@ class DetailView(QDialog, Ui_Dialog): # {{{ self.setupUi(self) self.setWindowTitle(job.description) self.job = job - self.html_view = hasattr(job, 'html_details') + self.html_view = (hasattr(job, 'html_details') and not getattr(job, + 'ignore_html_details', False)) if self.html_view: self.log.setVisible(False) else: diff --git a/src/calibre/gui2/metadata/bulk_download.py b/src/calibre/gui2/metadata/bulk_download.py index 24d081dc77..3487ffd8f2 100644 --- a/src/calibre/gui2/metadata/bulk_download.py +++ b/src/calibre/gui2/metadata/bulk_download.py @@ -20,6 +20,28 @@ from calibre.ptempfile import (PersistentTemporaryDirectory, PersistentTemporaryFile) # Start download {{{ + +class Job(ThreadedJob): + + ignore_html_details = True + + def consolidate_log(self): + self.consolidated_log = self.log.plain_text + self.log = None + + def read_consolidated_log(self): + return self.consolidated_log + + @property + def details(self): + if self.consolidated_log is None: + return self.log.plain_text + return self.read_consolidated_log() + + @property + def log_file(self): + return open(self.download_debug_log, 'rb') + def show_config(gui, parent): from calibre.gui2.preferences import show_config_widget show_config_widget('Sharing', 'Metadata download', parent=parent, @@ -101,11 +123,14 @@ def start_download(gui, ids, callback, ensure_fields=None): d.b.clicked.disconnect() if ret != d.Accepted: return + tf = PersistentTemporaryFile('_metadata_bulk_log_') + tf.close() - job = ThreadedJob('metadata bulk download', + job = Job('metadata bulk download', _('Download metadata for %d books')%len(ids), - download, (ids, gui.current_db, d.identify, d.covers, + download, (ids, tf.name, gui.current_db, d.identify, d.covers, ensure_fields), {}, callback) + job.download_debug_log = tf.name gui.job_manager.run_threaded_job(job) gui.status_bar.show_message(_('Metadata download started'), 3000) @@ -144,18 +169,15 @@ class HeartBeat(object): self.last_time = time.time() return True -# Fix log viewer, get_job_details, database update code +# Fix log viewer, ratings # Test: abort, covers only, metadata only, both, 200 entry download, memory # consumption, all errors and on and on -def download(all_ids, db, do_identify, covers, ensure_fields, +def download(all_ids, tf, db, do_identify, covers, ensure_fields, log=None, abort=None, notifications=None): batch_size = 10 batches = split_jobs(all_ids, batch_size=batch_size) tdir = PersistentTemporaryDirectory('_metadata_bulk_') - tf = PersistentTemporaryFile('_metadata_bulk_log_') - tf.close() - tf = tf.name heartbeat = HeartBeat(tdir) failed_ids = set() @@ -201,8 +223,10 @@ def download(all_ids, db, do_identify, covers, ensure_fields, for book_id in ids: lp = os.path.join(tdir, '%d.log'%book_id) if os.path.exists(lp): - with open(lp, 'rb') as f, open(tf, 'ab') as d: - shutil.copyfileobj(f, d) + with open(tf, 'ab') as dest, open(lp, 'rb') as src: + dest.write(('\n'+'#'*20 + ' Log for %s '%title_map[book_id] + + '#'*20+'\n').encode('utf-8')) + shutil.copyfileobj(src, dest) if abort.is_set(): aborted = True