Save to disk: When multiple books end up with the same file name because they have the same metadata, automatically adjust the filename to make it unique. Fixes #863472 [SaveToDisk loses one of identical Authors/Title books](https://bugs.launchpad.net/calibre/+bug/863472)

src/calibre/customize/ui.py

@@ -370,6 +370,13 @@ def set_file_type_metadata(stream, mi, ftype, report_error=None):
                         else:
                             report_error(mi, ftype, traceback.format_exc())
 
+def can_set_metadata(ftype):
+    ftype = ftype.lower().strip()
+    for plugin in _metadata_writers.get(ftype, ()):
+        if not is_disabled(plugin):
+            return True
+    return False
+
 # }}}
 
 # Add/remove plugins {{{

src/calibre/ebooks/metadata/worker.py

@@ -8,14 +8,13 @@ __docformat__ = 'restructuredtext en'
 
 from threading import Thread
 from Queue import Empty
-import os, time, sys, shutil, json
+import os, time, sys, shutil
 
 from calibre.utils.ipc.job import ParallelJob
 from calibre.utils.ipc.server import Server
 from calibre.ptempfile import PersistentTemporaryDirectory, TemporaryDirectory
-from calibre import prints, isbytestring
+from calibre import prints
 from calibre.constants import filesystem_encoding
-from calibre.db.errors import NoSuchFormat
 
 def debug(*args):
     prints(*args)
@@ -50,7 +49,8 @@ def do_read_metadata(task, tdir, mdir, notification):
         except:
             continue
         try:
-            if isinstance(formats, basestring): formats = [formats]
+            if isinstance(formats, basestring):
+                formats = [formats]
             import_map = {}
             fmts, metadata_fmts = [], []
             for format in formats:
@@ -101,7 +101,8 @@ class Progress(object):
 
     def __call__(self, id):
         cover = os.path.join(self.tdir, str(id))
-        if not os.path.exists(cover): cover = None
+        if not os.path.exists(cover):
+            cover = None
         res = os.path.join(self.tdir, '%s.error'%id)
         if not os.path.exists(res):
             res = res.replace('.error', '.opf')
@@ -118,7 +119,6 @@ class ReadMetadata(Thread):
         self.failure_details = {}
         self.tdir = PersistentTemporaryDirectory('_rm_worker')
 
-
     def run(self):
         jobs, ids = set([]), set([])
         for t in self.tasks:
@@ -181,162 +181,3 @@ def read_metadata(paths, result_queue, chunk=50, spare_server=None):
     t = ReadMetadata(tasks, result_queue, spare_server=spare_server)
     t.start()
     return t
-
-
-###########################################################################
-############ Saving #####################
-###########################################################################
-
-class SaveWorker(Thread):
-
-    def __init__(self, result_queue, db, ids, path, opts, spare_server=None):
-        Thread.__init__(self)
-        self.daemon = True
-        self.path, self.opts = path, opts
-        self.ids = ids
-        self.db = db
-        self.canceled = False
-        self.result_queue = result_queue
-        self.error = None
-        self.spare_server = spare_server
-        self.start()
-
-    def collect_data(self, ids, tdir):
-        from calibre.ebooks.metadata.opf2 import metadata_to_opf
-        data = {}
-        for i in set(ids):
-            mi = self.db.get_metadata(i, index_is_id=True, get_cover=True,
-                    cover_as_data=True)
-            opf = metadata_to_opf(mi)
-            if isbytestring(opf):
-                opf = opf.decode('utf-8')
-            cpath = None
-            if mi.cover_data and mi.cover_data[1]:
-                cpath = os.path.join(tdir, 'cover_%s.jpg'%i)
-                with lopen(cpath, 'wb') as f:
-                    f.write(mi.cover_data[1])
-                if isbytestring(cpath):
-                    cpath = cpath.decode(filesystem_encoding)
-            formats = {}
-            if mi.formats:
-                for fmt in mi.formats:
-                    fpath = os.path.join(tdir, 'fmt_%s.%s'%(i, fmt.lower()))
-                    with lopen(fpath, 'wb') as f:
-                        try:
-                            self.db.copy_format_to(i, fmt, f, index_is_id=True)
-                        except NoSuchFormat:
-                            continue
-                        else:
-                            if isbytestring(fpath):
-                                fpath = fpath.decode(filesystem_encoding)
-                            formats[fmt.lower()] = fpath
-            data[i] = [opf, cpath, formats, mi.last_modified.isoformat()]
-        return data
-
-    def run(self):
-        with TemporaryDirectory('save_to_disk_data') as tdir:
-            self._run(tdir)
-
-    def _run(self, tdir):
-        from calibre.library.save_to_disk import config
-        server = Server() if self.spare_server is None else self.spare_server
-        ids = set(self.ids)
-        tasks = server.split(list(ids))
-        jobs = set([])
-        c = config()
-        recs = {}
-        for pref in c.preferences:
-            recs[pref.name] = getattr(self.opts, pref.name)
-
-        plugboards = self.db.prefs.get('plugboards', {})
-        template_functions = self.db.prefs.get('user_template_functions', [])
-
-        for i, task in enumerate(tasks):
-            tids = [x[-1] for x in task]
-            data = self.collect_data(tids, tdir)
-            dpath = os.path.join(tdir, '%d.json'%i)
-            with open(dpath, 'wb') as f:
-                f.write(json.dumps(data, ensure_ascii=False).encode('utf-8'))
-
-            job = ParallelJob('save_book',
-                    'Save books (%d of %d)'%(i, len(tasks)),
-                    lambda x,y:x,
-                    args=[tids, dpath, plugboards, template_functions, self.path, recs])
-            jobs.add(job)
-            server.add_job(job)
-
-
-        while not self.canceled:
-            time.sleep(0.2)
-            running = False
-            for job in jobs:
-                self.get_notifications(job, ids)
-                if not job.is_finished:
-                    running = True
-
-            if not running:
-                break
-
-        for job in jobs:
-            if not job.result:
-                continue
-            for id_, title, ok, tb in job.result:
-                if id_ in ids:
-                    self.result_queue.put((id_, title, ok, tb))
-                    ids.remove(id_)
-
-        server.close()
-        time.sleep(1)
-
-        if self.canceled:
-            return
-
-        for job in jobs:
-            if job.failed:
-                prints(job.details)
-                self.error = job.details
-            if os.path.exists(job.log_path):
-                try:
-                    os.remove(job.log_path)
-                except:
-                    pass
-
-    def get_notifications(self, job, ids):
-        job.update(consume_notifications=False)
-        while True:
-            try:
-                id, title, ok, tb = job.notifications.get_nowait()[0]
-                if id in ids:
-                    self.result_queue.put((id, title, ok, tb))
-                    ids.remove(id)
-            except Empty:
-                break
-
-
-def save_book(ids, dpath, plugboards, template_functions, path, recs,
-              notification=lambda x,y:x):
-    from calibre.library.save_to_disk import config, save_serialized_to_disk
-    from calibre.customize.ui import apply_null_metadata
-    from calibre.utils.formatter_functions import load_user_template_functions
-    load_user_template_functions('', template_functions)
-    opts = config().parse()
-    for name in recs:
-        setattr(opts, name, recs[name])
-
-    results = []
-
-    def callback(id, title, failed, tb):
-        results.append((id, title, not failed, tb))
-        notification((id, title, not failed, tb))
-        return True
-
-    data_ = json.loads(open(dpath, 'rb').read().decode('utf-8'))
-    data = {}
-    for k, v in data_.iteritems():
-        data[int(k)] = v
-
-    with apply_null_metadata:
-        save_serialized_to_disk(ids, data, plugboards, path, opts, callback)
-
-    return results
-

src/calibre/gui2/actions/save_to_disk.py

@@ -7,11 +7,11 @@ __docformat__ = 'restructuredtext en'
 
 import os
 from functools import partial
+from future_builtins import map
 
 
 from calibre.utils.config import prefs
-from calibre.gui2 import (error_dialog, Dispatcher, gprefs,
+from calibre.gui2 import error_dialog, Dispatcher, choose_dir
-    choose_dir, warning_dialog, open_local_file)
 from calibre.gui2.actions import InterfaceAction
 
 class SaveToDiskAction(InterfaceAction):
@@ -57,9 +57,9 @@ class SaveToDiskAction(InterfaceAction):
 
     def save_specific_format_disk(self):
         rb = self.gui.iactions['Remove Books']
-        ids = rb._get_selected_ids(err_title=
+        ids = rb._get_selected_ids(err_title=_('Cannot save to disk'))
-            _('Cannot save to disk'))
+        if not ids:
-        if not ids: return
+            return
         fmts = rb._get_selected_formats(
                 _('Choose format to save to disk'), ids,
                 single=True)
@@ -96,7 +96,7 @@ class SaveToDiskAction(InterfaceAction):
                       'files from your calibre library elsewhere.'), show=True)
 
         if self.gui.current_view() is self.gui.library_view:
-            from calibre.gui2.add import Saver
+            from calibre.gui2.save import Saver
             from calibre.library.save_to_disk import config
             opts = config().parse()
             if single_format is not None:
@@ -112,10 +112,8 @@ class SaveToDiskAction(InterfaceAction):
                 opts.write_opf = write_opf
             if save_cover is not None:
                 opts.save_cover = save_cover
-            self._saver = Saver(self.gui, self.gui.library_view.model().db,
+            book_ids = set(map(self.gui.library_view.model().id, rows))
-                    Dispatcher(self._books_saved), rows, path, opts,
+            Saver(book_ids, self.gui.current_db, opts, path, parent=self.gui, spare_server=self.gui.spare_server)
-                    spare_server=self.gui.spare_server)
-
         else:
             paths = self.gui.current_view().model().paths(rows)
             self.gui.device_manager.save_books(
@@ -129,26 +127,6 @@ class SaveToDiskAction(InterfaceAction):
         self.save_to_disk(True, single_dir=single_dir, single_format=fmt,
                 rows=rows, write_opf=False, save_cover=False)
 
-    def _books_saved(self, path, failures, error):
-        self._saver = None
-        if error:
-            return error_dialog(self.gui, _('Error while saving'),
-                    _('There was an error while saving.'),
-                    error, show=True)
-        if failures:
-            failures = [u'%s\n\t%s'%
-                    (title, '\n\t'.join(err.splitlines())) for title, err in
-                    failures]
-
-            warning_dialog(self.gui, _('Could not save some books'),
-            _('Could not save some books') + ', ' +
-            _('Click the show details button to see which ones.'),
-            u'\n\n'.join(failures), show=True)
-        if gprefs['show_files_after_save']:
-            open_local_file(path)
-
     def books_saved(self, job):
         if job.failed:
             return self.gui.device_job_exception(job)
-
-

src/calibre/gui2/add.py

@@ -494,91 +494,3 @@ class Adder(QObject):  # {{{
 
 # }}}
 
-class Saver(QObject):  # {{{
-
-    def __init__(self, parent, db, callback, rows, path, opts,
-            spare_server=None):
-        QObject.__init__(self, parent)
-        self.pd = ProgressDialog(_('Saving...'), parent=parent)
-        self.spare_server = spare_server
-        self.db = db
-        self.opts = opts
-        self.pd.setModal(True)
-        self.pd.show()
-        self.pd.set_min(0)
-        self.pd.set_msg(_('Collecting data, please wait...'))
-        self._parent = parent
-        self.callback = callback
-        self.callback_called = False
-        self.rq = Queue()
-        self.ids = [x for x in map(db.id, [r.row() for r in rows]) if x is not None]
-        self.pd_max = len(self.ids)
-        self.pd.set_max(0)
-        self.pd.value = 0
-        self.failures = set([])
-
-        from calibre.ebooks.metadata.worker import SaveWorker
-        self.worker = SaveWorker(self.rq, db, self.ids, path, self.opts,
-                spare_server=self.spare_server)
-        self.pd.canceled_signal.connect(self.canceled)
-        self.continue_updating = True
-        single_shot(self.update)
-
-    def canceled(self):
-        self.continue_updating = False
-        if self.worker is not None:
-            self.worker.canceled = True
-        self.pd.hide()
-        if not self.callback_called:
-            self.callback(self.worker.path, self.failures, self.worker.error)
-            self.callback_called = True
-
-    def update(self):
-        if not self.continue_updating:
-            return
-        if not self.worker.is_alive():
-            # Check that all ids were processed
-            while self.ids:
-                # Get all queued results since worker is dead
-                before = len(self.ids)
-                self.get_result()
-                if before == len(self.ids):
-                    # No results available => worker died unexpectedly
-                    for i in list(self.ids):
-                        self.failures.add(('id:%d'%i, 'Unknown error'))
-                        self.ids.remove(i)
-
-        if not self.ids:
-            self.continue_updating = False
-            self.pd.hide()
-            if not self.callback_called:
-                try:
-                    # Give the worker time to clean up and set worker.error
-                    self.worker.join(2)
-                except:
-                    pass  # The worker was not yet started
-                self.callback_called = True
-                self.callback(self.worker.path, self.failures, self.worker.error)
-
-        if self.continue_updating:
-            self.get_result()
-            single_shot(self.update)
-
-    def get_result(self):
-        try:
-            id, title, ok, tb = self.rq.get_nowait()
-        except Empty:
-            return
-        if self.pd.max != self.pd_max:
-            self.pd.max = self.pd_max
-        self.pd.value += 1
-        self.ids.remove(id)
-        if not isinstance(title, unicode):
-            title = str(title).decode(preferred_encoding, 'replace')
-        self.pd.set_msg(_('Saved')+' '+title)
-
-        if not ok:
-            self.failures.add((title, tb))
-# }}}
-
-

src/calibre/gui2/save.py

@@ -6,21 +6,26 @@ from __future__ import (unicode_literals, division, absolute_import,
 __license__ = 'GPL v3'
 __copyright__ = '2014, Kovid Goyal <kovid at kovidgoyal.net>'
 
-import traceback, errno, os
+import traceback, errno, os, time, shutil
 from collections import namedtuple, defaultdict
 from tempfile import SpooledTemporaryFile
-from functools import partial
+from Queue import Empty
 
 from PyQt5.Qt import QObject, Qt, pyqtSignal
 
-from calibre.customize.ui import apply_null_metadata
+from calibre import prints
+from calibre.constants import DEBUG
+from calibre.customize.ui import can_set_metadata
 from calibre.db.errors import NoSuchFormat
 from calibre.ebooks.metadata import authors_to_string
 from calibre.ebooks.metadata.opf2 import metadata_to_opf
+from calibre.ptempfile import PersistentTemporaryDirectory
 from calibre.gui2 import error_dialog, warning_dialog, gprefs, open_local_file
 from calibre.gui2.dialogs.progress import ProgressDialog
 from calibre.utils.formatter_functions import load_user_template_functions
-from calibre.library.save_to_disk import sanitize_args, get_path_components, update_metadata
+from calibre.utils.ipc.job import ParallelJob
+from calibre.utils.ipc.server import Server
+from calibre.library.save_to_disk import sanitize_args, get_path_components, find_plugboard, plugboard_save_to_disk_value
 
 BookId = namedtuple('BookId', 'title authors')
 
@@ -68,7 +73,7 @@ class Saver(QObject):
 
     do_one_signal = pyqtSignal()
 
-    def __init__(self, book_ids, db, opts, root, parent=None):
+    def __init__(self, book_ids, db, opts, root, parent=None, spare_server=None):
         QObject.__init__(self, parent)
         if parent is not None:
             setattr(parent, 'no_gc_%s' % id(self), self)
@@ -77,30 +82,40 @@ class Saver(QObject):
         self.template_functions = self.db.pref('user_template_functions', [])
         load_user_template_functions('', self.template_functions)
         self.collected_data = {}
+        self.metadata_data = {}
         self.errors = defaultdict(list)
         self._book_id_data = {}
         self.all_book_ids = frozenset(book_ids)
-        self.pd = ProgressDialog(_('Saving...'), _('Collecting metadata...'), min=0, max=0, parent=parent, icon='save.png')
+        self.pd = ProgressDialog(_('Saving %d books...') % len(self.all_book_ids), _('Collecting metadata...'), min=0, max=0, parent=parent, icon='save.png')
         self.do_one_signal.connect(self.tick, type=Qt.QueuedConnection)
         self.do_one = self.do_one_collect
         self.ids_to_collect = iter(self.all_book_ids)
+        self.plugboard_cache = {}
+        self.tdir = PersistentTemporaryDirectory('_save_to_disk')
+        self.server = spare_server
 
         self.pd.show()
         self.root, self.opts, self.path_length = sanitize_args(root, opts)
         self.do_one_signal.emit()
+        if DEBUG:
+            self.start_time = time.time()
 
     def tick(self):
         if self.pd.canceled:
             self.pd.close()
+            self.pd.deleteLater()
             self.break_cycles()
             return
         self.do_one()
 
     def break_cycles(self):
+        shutil.rmtree(self.tdir, ignore_errors=True)
         p = self.parent()
         if p is not None:
             setattr(p, 'no_gc_%s' % id(self), None)
-        self.plugboards = self.template_functions = self.collected_data = self.all_book_ids = self.pd = self.db = None
+        if self.server is not None:
+            self.server.close()
+        self.jobs = self.server = self.metadata_data = self.plugboard_cache = self.plugboards = self.template_functions = self.collected_data = self.all_book_ids = self.pd = self.db = None  # noqa
 
     def book_id_data(self, book_id):
         ans = self._book_id_data.get(book_id)
@@ -126,6 +141,7 @@ class Saver(QObject):
 
     def collect_data(self, book_id):
         mi = self.db.get_metadata(book_id)
+        self._book_id_data[book_id] = BookId(mi.title, mi.authors)
         components = get_path_components(self.opts, mi, book_id, self.path_length)
         self.collected_data[book_id] = (mi, components)
 
@@ -133,6 +149,7 @@ class Saver(QObject):
         self.do_one = self.do_one_write
         ensure_unique_components(self.collected_data)
         self.ids_to_write = iter(self.collected_data)
+        self.pd.title = _('Copying files from calibre library to disk...')
         self.pd.max = len(self.collected_data)
         self.pd.value = 0
         self.do_one_signal.emit()
@@ -146,90 +163,143 @@ class Saver(QObject):
         self.pd.msg = self.book_id_data(book_id).title
         self.pd.value += 1
         try:
-            self.fmts_to_write = self.write_book(book_id)
+            self.write_book(book_id, *self.collected_data[book_id])
         except Exception:
             self.errors[book_id].append(('critical', traceback.format_exc()))
-            self.fmts_to_write = iter(())
-        self.do_one = self.do_one_fmt
         self.do_one_signal.emit()
 
-    def writing_finished(self):
+    def write_book(self, book_id, mi, components):
-        self.pd.close()
-        self.report()
-        self.break_cycles()
-        if gprefs['show_files_after_save']:
-            open_local_file(self.root)
-
-    def write_book(self, book_id):
-        mi, components = self.collected_data[book_id]
         base_path = os.path.join(self.root, *components)
         base_dir = os.path.dirname(base_path)
         fmts = {f.lower() for f in self.db.formats(book_id)}
         if self.opts.formats != 'all':
             asked_formats = {x.lower().strip() for x in self.opts.formats.split(',')}
             fmts = asked_formats.intersection(fmts)
+            if not fmts:
+                self.errors[book_id] = ('critical', _('Requested formats not available'))
+                return
 
         if not fmts and not self.opts.write_opf and not self.opts.save_cover:
             return
+
         try:
             os.makedirs(base_dir)
         except EnvironmentError as err:
             if err.errno != errno.EEXIST:
                 raise
 
+        if self.opts.update_metadata:
+            self.metadata_data[book_id] = d = {}
+            d['last_modified'] = mi.last_modified.isoformat()
+
         cdata = self.db.cover(book_id)
         mi.cover, mi.cover_data = None, (None, None)
 
-        if self.opts.save_cover and cdata:
+        if cdata:
-            with lopen(base_path + os.extsep + 'jpg', 'wb') as f:
+            fname = None
-                f.write(cdata)
+            if self.opts.save_cover:
-            mi.cover = os.path.basename(f.name)
+                fname = base_path + os.extsep + 'jpg'
+                mi.cover = os.path.basename(fname)
+            elif self.opts.update_metadata:
+                fname = os.path.join(self.tdir, '%d.jpg' % book_id)
 
+            if fname:
+                with lopen(fname, 'wb') as f:
+                    f.write(cdata)
+                if self.opts.update_metadata:
+                    d['cover'] = fname
+
+        fname = None
         if self.opts.write_opf:
+            fname = base_path + os.extsep + 'opf'
+        elif self.opts.update_metadata:
+            fname = os.path.join(self.tdir, '%d.opf' % book_id)
+        if fname:
             opf = metadata_to_opf(mi)
-            with lopen(base_path + os.extsep + 'opf', 'wb') as f:
+            with lopen(fname, 'wb') as f:
                 f.write(opf)
-
+            if self.opts.update_metadata:
+                d['opf'] = fname
         mi.cover, mi.cover_data = None, (None, None)
+        if self.opts.update_metadata:
+            d['fmts'] = []
+        for fmt in fmts:
+            try:
+                fmtpath = self.write_fmt(book_id, fmt, base_path)
+                if fmtpath and self.opts.update_metadata and can_set_metadata(fmt):
+                    d['fmts'].append(fmtpath)
+            except Exception:
+                self.errors[book_id].append(('fmt', (fmt, traceback.format_exc())))
+        if self.opts.update_metadata and not d['fmts']:
+            self.metadata_data.pop(book_id, None)
 
-        return ((book_id, f, base_path, mi, cdata) for f in fmts)
+    def write_fmt(self, book_id, fmt, base_path):
-
-    def do_one_fmt(self):
-        try:
-            args = next(self.fmts_to_write)
-        except StopIteration:
-            del self.fmts_to_write
-            self.do_one = self.do_one_write
-            self.do_one_signal.emit()
-            return
-        try:
-            self.write_fmt(*args)
-        except Exception:
-            self.errors[args[0]].append(('fmt', (args[1], traceback.format_exc())))
-        self.do_one_signal.emit()
-
-    def report_update_metadata_error(self, book_id, fmt, tb):
-        self.errors[book_id].append(('metadata', (fmt, tb)))
-
-    def write_fmt(self, book_id, fmt, base_path, mi, cdata):
         fmtpath = base_path + os.extsep + fmt
         written = False
         with lopen(fmtpath, 'w+b') as f:
-            sf = SpooledFile(f)
             try:
-                self.db.copy_format_to(book_id, fmt, sf)
+                self.db.copy_format_to(book_id, fmt, f)
-            except NoSuchFormat:
-                pass
-            else:
-                if self.opts.update_metadata:
-                    sf.seek(0)
-                    with apply_null_metadata:
-                        update_metadata(mi, fmt, sf, self.plugboards, cdata,
-                                        error_report=partial(self.report_update_metadata_error, book_id))
-                sf.rollover()
                 written = True
+            except NoSuchFormat:
+                self.errors[book_id].append(('fmt', (fmt, _('No %s format file present') % fmt.upper())))
         if not written:
             os.remove(fmtpath)
+        if written:
+            return fmtpath
+
+    def writing_finished(self):
+        if not self.opts.update_metadata:
+            self.metadata_data = {}
+        if not self.metadata_data:
+            self.updating_metadata_finished()
+            return
+        self.pd.title = _('Updating metadata...')
+        self.pd.max = len(self.metadata_data)
+        self.pd.value = 0
+
+        all_fmts = {path.rpartition(os.extsep)[-1] for d in self.metadata_data.itervalues() for path in d['fmts']}
+        plugboards_cache = {fmt:find_plugboard(plugboard_save_to_disk_value, fmt, self.plugboards) for fmt in all_fmts}
+        self.server = server = Server() if self.server is None else self.server
+        tasks = server.split(list(self.metadata_data))
+        self.jobs = set()
+        for i, book_ids in enumerate(tasks):
+            data = {book_id:self.metadata_data[book_id] for j, book_id in book_ids}
+            job = ParallelJob('save_book', 'Save books (job %d of %d)' % (i+1, len(tasks)), lambda x, y:x, args=(data, plugboards_cache))
+            self.jobs.add(job)
+            server.add_job(job)
+        self.do_one = self.do_one_update
+        self.do_one_signal.emit()
+
+    def do_one_update(self):
+        running = False
+        for job in self.jobs:
+            if not job.is_finished:
+                running = True
+                job.update(consume_notifications=False)
+                while True:
+                    try:
+                        book_id = job.notifications.get_nowait()[0]
+                        self.pd.value += 1
+                        self.pd.msg = self.book_id_data(book_id).title
+                    except Empty:
+                        break
+        if running:
+            self.do_one_signal.emit()
+        else:
+            for job in self.jobs:
+                for book_id, fmt, tb in (job.result or ()):
+                    self.errors[book_id].append(('metadata', (fmt, tb)))
+            self.updating_metadata_finished()
+
+    def updating_metadata_finished(self):
+        if DEBUG:
+            prints('Saved %d books in %.1f seconds' % (len(self.all_book_ids), time.time() - self.start_time))
+        self.pd.close()
+        self.pd.deleteLater()
+        self.report()
+        self.break_cycles()
+        if gprefs['show_files_after_save']:
+            open_local_file(self.root)
 
     def format_report(self):
         report = []

src/calibre/library/save_to_disk.py

@@ -6,18 +6,16 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os, traceback, cStringIO, re, shutil
+import os, traceback, re, shutil
 
 from calibre.constants import DEBUG
 from calibre.utils.config import Config, StringConfig, tweaks
 from calibre.utils.formatter import TemplateFormatter
-from calibre.utils.filenames import shorten_components_to, supports_long_names, \
+from calibre.utils.filenames import shorten_components_to, supports_long_names, ascii_filename
-                                    ascii_filename
-from calibre.ebooks.metadata.opf2 import metadata_to_opf
 from calibre.constants import preferred_encoding
 from calibre.ebooks.metadata import fmt_sidx
 from calibre.ebooks.metadata import title_sort
-from calibre.utils.date import parse_date, as_local_time
+from calibre.utils.date import as_local_time
 from calibre import strftime, prints, sanitize_file_name_unicode
 from calibre.ptempfile import SpooledTemporaryFile
 from calibre.db.lazy import FormatsList
@@ -301,11 +299,17 @@ def get_path_components(opts, mi, book_id, path_length):
     return components
 
 
-def update_metadata(mi, fmt, stream, plugboards, cdata, error_report=None):
+def update_metadata(mi, fmt, stream, plugboards, cdata, error_report=None, plugboard_cache=None):
-    global plugboard_save_to_disk_value, plugboard_any_format_value
     from calibre.ebooks.metadata.meta import set_metadata
+    if error_report is not None:
+        def report_error(mi, fmt, tb):
+            error_report(fmt, tb)
+
     try:
-        cpb = find_plugboard(plugboard_save_to_disk_value, fmt, plugboards)
+        if plugboard_cache is not None:
+            cpb = plugboard_cache[fmt]
+        else:
+            cpb = find_plugboard(plugboard_save_to_disk_value, fmt, plugboards)
         if cpb:
             newmi = mi.deepcopy_metadata()
             newmi.template_to_attribute(mi, cpb)
@@ -313,7 +317,7 @@ def update_metadata(mi, fmt, stream, plugboards, cdata, error_report=None):
             newmi = mi
         if cdata:
             newmi.cover_data = ('jpg', cdata)
-        set_metadata(stream, newmi, fmt)
+        set_metadata(stream, newmi, fmt, report_error=None if error_report is None else report_error)
     except:
         if error_report is None:
             prints('Failed to set metadata for the', fmt, 'format of', mi.title)
@@ -359,6 +363,7 @@ def do_save_book_to_disk(id_, mi, cover, plugboards,
         mi.cover = None
 
     if opts.write_opf:
+        from calibre.ebooks.metadata.opf2 import metadata_to_opf
         opf = metadata_to_opf(mi)
         with open(base_path+'.opf', 'wb') as f:
             f.write(opf)
@@ -426,37 +431,44 @@ def save_to_disk(db, ids, root, opts=None, callback=None):
                 break
     return failures
 
-def save_serialized_to_disk(ids, data, plugboards, root, opts, callback):
+def read_serialized_metadata(book_id, data):
     from calibre.ebooks.metadata.opf2 import OPF
-    root, opts, length = sanitize_args(root, opts)
+    from calibre.utils.date import parse_date
-    failures = []
+    mi = OPF(data['opf'], try_to_guess_cover=False, populate_spine=False, basedir=os.path.dirname(data['opf'])).to_book_metadata()
-    for x in ids:
+    try:
-        opf, cover, format_map, last_modified = data[x]
+        mi.last_modified = parse_date(data['last_modified'])
-        if isinstance(opf, unicode):
+    except:
-            opf = opf.encode('utf-8')
+        pass
-        mi = OPF(cStringIO.StringIO(opf)).to_book_metadata()
+    mi.cover, mi.cover_data = None, (None, None)
-        try:
+    cdata = None
-            mi.last_modified = parse_date(last_modified)
+    if 'cover' in data:
-        except:
+        with lopen(data['cover'], 'rb') as f:
-            pass
+            cdata = f.read()
-        tb = ''
+    return mi, cdata
-        try:
-            with open(cover, 'rb') as f:
-                cover = f.read()
-        except:
-            cover = None
-        try:
-            failed, id, title = do_save_book_to_disk(x, mi, cover,
-                plugboards, format_map, root, opts, length)
-            tb = _('Requested formats not available')
-        except:
-            failed, id, title = True, x, mi.title
-            tb = traceback.format_exc()
-        if failed:
-            failures.append((id, title, tb))
-        if callable(callback):
-            if not callback(int(id), title, failed, tb):
-                break
 
-    return failures
+def update_serialized_metadata(books, plugboard_cache, notification=lambda x,y:x):
+    result = []
+    from calibre.customize.ui import apply_null_metadata
+    with apply_null_metadata:
 
+        for book_id, data in books.iteritems():
+            fmts = [fp.rpartition(os.extsep)[-1] for fp in data['fmts']]
+            try:
+                mi, cdata = read_serialized_metadata(book_id, data)
+            except Exception:
+                tb = traceback.format_exc()
+                for fmt in fmts:
+                    result.append((book_id, fmt, tb))
+            else:
+                def report_error(fmt, tb):
+                    result.append((book_id, fmt, tb))
+
+                for fmt, fmtpath in zip(fmts, data['fmts']):
+                    try:
+                        with lopen(fmtpath, 'r+b') as stream:
+                            update_metadata(mi, fmt, stream, (), cdata, error_report=report_error, plugboard_cache=plugboard_cache)
+                    except Exception:
+                        report_error(fmt, traceback.format_exc())
+            notification(book_id)
+
+    return result

src/calibre/utils/ipc/worker.py

@@ -50,7 +50,7 @@ PARALLEL_FUNCS = {
       ('calibre.ebooks.metadata.worker', 'read_metadata_', 'notification'),
 
       'save_book' :
-      ('calibre.ebooks.metadata.worker', 'save_book', 'notification'),
+      ('calibre.library.save_to_disk', 'update_serialized_metadata', 'notification'),
 
       'arbitrary' :
       ('calibre.utils.ipc.worker', 'arbitrary', None),