From e6f4c169d9bf6517e6ad99a5a050636d838e7264 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 8 Nov 2014 16:48:47 +0530 Subject: [PATCH] Save to disk: When multiple books end up with the same file name because they have the same metadata, automatically adjust the filename to make it unique. Fixes #863472 [SaveToDisk loses one of identical Authors/Title books](https://bugs.launchpad.net/calibre/+bug/863472) --- src/calibre/customize/ui.py | 7 + src/calibre/ebooks/metadata/worker.py | 171 +-------------------- src/calibre/gui2/actions/save_to_disk.py | 38 +---- src/calibre/gui2/add.py | 88 ----------- src/calibre/gui2/save.py | 182 ++++++++++++++++------- src/calibre/library/save_to_disk.py | 92 +++++++----- src/calibre/utils/ipc/worker.py | 2 +- 7 files changed, 200 insertions(+), 380 deletions(-) diff --git a/src/calibre/customize/ui.py b/src/calibre/customize/ui.py index 346b60c4b7..ca44fa4691 100644 --- a/src/calibre/customize/ui.py +++ b/src/calibre/customize/ui.py @@ -370,6 +370,13 @@ def set_file_type_metadata(stream, mi, ftype, report_error=None): else: report_error(mi, ftype, traceback.format_exc()) +def can_set_metadata(ftype): + ftype = ftype.lower().strip() + for plugin in _metadata_writers.get(ftype, ()): + if not is_disabled(plugin): + return True + return False + # }}} # Add/remove plugins {{{ diff --git a/src/calibre/ebooks/metadata/worker.py b/src/calibre/ebooks/metadata/worker.py index e8147958d8..0614496f58 100644 --- a/src/calibre/ebooks/metadata/worker.py +++ b/src/calibre/ebooks/metadata/worker.py @@ -8,14 +8,13 @@ __docformat__ = 'restructuredtext en' from threading import Thread from Queue import Empty -import os, time, sys, shutil, json +import os, time, sys, shutil from calibre.utils.ipc.job import ParallelJob from calibre.utils.ipc.server import Server from calibre.ptempfile import PersistentTemporaryDirectory, TemporaryDirectory -from calibre import prints, isbytestring +from calibre import prints from calibre.constants import filesystem_encoding -from calibre.db.errors import NoSuchFormat def debug(*args): prints(*args) @@ -50,7 +49,8 @@ def do_read_metadata(task, tdir, mdir, notification): except: continue try: - if isinstance(formats, basestring): formats = [formats] + if isinstance(formats, basestring): + formats = [formats] import_map = {} fmts, metadata_fmts = [], [] for format in formats: @@ -101,7 +101,8 @@ class Progress(object): def __call__(self, id): cover = os.path.join(self.tdir, str(id)) - if not os.path.exists(cover): cover = None + if not os.path.exists(cover): + cover = None res = os.path.join(self.tdir, '%s.error'%id) if not os.path.exists(res): res = res.replace('.error', '.opf') @@ -118,7 +119,6 @@ class ReadMetadata(Thread): self.failure_details = {} self.tdir = PersistentTemporaryDirectory('_rm_worker') - def run(self): jobs, ids = set([]), set([]) for t in self.tasks: @@ -181,162 +181,3 @@ def read_metadata(paths, result_queue, chunk=50, spare_server=None): t = ReadMetadata(tasks, result_queue, spare_server=spare_server) t.start() return t - - -########################################################################### -############ Saving ##################### -########################################################################### - -class SaveWorker(Thread): - - def __init__(self, result_queue, db, ids, path, opts, spare_server=None): - Thread.__init__(self) - self.daemon = True - self.path, self.opts = path, opts - self.ids = ids - self.db = db - self.canceled = False - self.result_queue = result_queue - self.error = None - self.spare_server = spare_server - self.start() - - def collect_data(self, ids, tdir): - from calibre.ebooks.metadata.opf2 import metadata_to_opf - data = {} - for i in set(ids): - mi = self.db.get_metadata(i, index_is_id=True, get_cover=True, - cover_as_data=True) - opf = metadata_to_opf(mi) - if isbytestring(opf): - opf = opf.decode('utf-8') - cpath = None - if mi.cover_data and mi.cover_data[1]: - cpath = os.path.join(tdir, 'cover_%s.jpg'%i) - with lopen(cpath, 'wb') as f: - f.write(mi.cover_data[1]) - if isbytestring(cpath): - cpath = cpath.decode(filesystem_encoding) - formats = {} - if mi.formats: - for fmt in mi.formats: - fpath = os.path.join(tdir, 'fmt_%s.%s'%(i, fmt.lower())) - with lopen(fpath, 'wb') as f: - try: - self.db.copy_format_to(i, fmt, f, index_is_id=True) - except NoSuchFormat: - continue - else: - if isbytestring(fpath): - fpath = fpath.decode(filesystem_encoding) - formats[fmt.lower()] = fpath - data[i] = [opf, cpath, formats, mi.last_modified.isoformat()] - return data - - def run(self): - with TemporaryDirectory('save_to_disk_data') as tdir: - self._run(tdir) - - def _run(self, tdir): - from calibre.library.save_to_disk import config - server = Server() if self.spare_server is None else self.spare_server - ids = set(self.ids) - tasks = server.split(list(ids)) - jobs = set([]) - c = config() - recs = {} - for pref in c.preferences: - recs[pref.name] = getattr(self.opts, pref.name) - - plugboards = self.db.prefs.get('plugboards', {}) - template_functions = self.db.prefs.get('user_template_functions', []) - - for i, task in enumerate(tasks): - tids = [x[-1] for x in task] - data = self.collect_data(tids, tdir) - dpath = os.path.join(tdir, '%d.json'%i) - with open(dpath, 'wb') as f: - f.write(json.dumps(data, ensure_ascii=False).encode('utf-8')) - - job = ParallelJob('save_book', - 'Save books (%d of %d)'%(i, len(tasks)), - lambda x,y:x, - args=[tids, dpath, plugboards, template_functions, self.path, recs]) - jobs.add(job) - server.add_job(job) - - - while not self.canceled: - time.sleep(0.2) - running = False - for job in jobs: - self.get_notifications(job, ids) - if not job.is_finished: - running = True - - if not running: - break - - for job in jobs: - if not job.result: - continue - for id_, title, ok, tb in job.result: - if id_ in ids: - self.result_queue.put((id_, title, ok, tb)) - ids.remove(id_) - - server.close() - time.sleep(1) - - if self.canceled: - return - - for job in jobs: - if job.failed: - prints(job.details) - self.error = job.details - if os.path.exists(job.log_path): - try: - os.remove(job.log_path) - except: - pass - - def get_notifications(self, job, ids): - job.update(consume_notifications=False) - while True: - try: - id, title, ok, tb = job.notifications.get_nowait()[0] - if id in ids: - self.result_queue.put((id, title, ok, tb)) - ids.remove(id) - except Empty: - break - - -def save_book(ids, dpath, plugboards, template_functions, path, recs, - notification=lambda x,y:x): - from calibre.library.save_to_disk import config, save_serialized_to_disk - from calibre.customize.ui import apply_null_metadata - from calibre.utils.formatter_functions import load_user_template_functions - load_user_template_functions('', template_functions) - opts = config().parse() - for name in recs: - setattr(opts, name, recs[name]) - - results = [] - - def callback(id, title, failed, tb): - results.append((id, title, not failed, tb)) - notification((id, title, not failed, tb)) - return True - - data_ = json.loads(open(dpath, 'rb').read().decode('utf-8')) - data = {} - for k, v in data_.iteritems(): - data[int(k)] = v - - with apply_null_metadata: - save_serialized_to_disk(ids, data, plugboards, path, opts, callback) - - return results - diff --git a/src/calibre/gui2/actions/save_to_disk.py b/src/calibre/gui2/actions/save_to_disk.py index 10574aea29..83ced9f158 100644 --- a/src/calibre/gui2/actions/save_to_disk.py +++ b/src/calibre/gui2/actions/save_to_disk.py @@ -7,11 +7,11 @@ __docformat__ = 'restructuredtext en' import os from functools import partial +from future_builtins import map from calibre.utils.config import prefs -from calibre.gui2 import (error_dialog, Dispatcher, gprefs, - choose_dir, warning_dialog, open_local_file) +from calibre.gui2 import error_dialog, Dispatcher, choose_dir from calibre.gui2.actions import InterfaceAction class SaveToDiskAction(InterfaceAction): @@ -57,9 +57,9 @@ class SaveToDiskAction(InterfaceAction): def save_specific_format_disk(self): rb = self.gui.iactions['Remove Books'] - ids = rb._get_selected_ids(err_title= - _('Cannot save to disk')) - if not ids: return + ids = rb._get_selected_ids(err_title=_('Cannot save to disk')) + if not ids: + return fmts = rb._get_selected_formats( _('Choose format to save to disk'), ids, single=True) @@ -96,7 +96,7 @@ class SaveToDiskAction(InterfaceAction): 'files from your calibre library elsewhere.'), show=True) if self.gui.current_view() is self.gui.library_view: - from calibre.gui2.add import Saver + from calibre.gui2.save import Saver from calibre.library.save_to_disk import config opts = config().parse() if single_format is not None: @@ -112,10 +112,8 @@ class SaveToDiskAction(InterfaceAction): opts.write_opf = write_opf if save_cover is not None: opts.save_cover = save_cover - self._saver = Saver(self.gui, self.gui.library_view.model().db, - Dispatcher(self._books_saved), rows, path, opts, - spare_server=self.gui.spare_server) - + book_ids = set(map(self.gui.library_view.model().id, rows)) + Saver(book_ids, self.gui.current_db, opts, path, parent=self.gui, spare_server=self.gui.spare_server) else: paths = self.gui.current_view().model().paths(rows) self.gui.device_manager.save_books( @@ -129,26 +127,6 @@ class SaveToDiskAction(InterfaceAction): self.save_to_disk(True, single_dir=single_dir, single_format=fmt, rows=rows, write_opf=False, save_cover=False) - def _books_saved(self, path, failures, error): - self._saver = None - if error: - return error_dialog(self.gui, _('Error while saving'), - _('There was an error while saving.'), - error, show=True) - if failures: - failures = [u'%s\n\t%s'% - (title, '\n\t'.join(err.splitlines())) for title, err in - failures] - - warning_dialog(self.gui, _('Could not save some books'), - _('Could not save some books') + ', ' + - _('Click the show details button to see which ones.'), - u'\n\n'.join(failures), show=True) - if gprefs['show_files_after_save']: - open_local_file(path) - def books_saved(self, job): if job.failed: return self.gui.device_job_exception(job) - - diff --git a/src/calibre/gui2/add.py b/src/calibre/gui2/add.py index 92028345e9..804d13c89d 100644 --- a/src/calibre/gui2/add.py +++ b/src/calibre/gui2/add.py @@ -494,91 +494,3 @@ class Adder(QObject): # {{{ # }}} -class Saver(QObject): # {{{ - - def __init__(self, parent, db, callback, rows, path, opts, - spare_server=None): - QObject.__init__(self, parent) - self.pd = ProgressDialog(_('Saving...'), parent=parent) - self.spare_server = spare_server - self.db = db - self.opts = opts - self.pd.setModal(True) - self.pd.show() - self.pd.set_min(0) - self.pd.set_msg(_('Collecting data, please wait...')) - self._parent = parent - self.callback = callback - self.callback_called = False - self.rq = Queue() - self.ids = [x for x in map(db.id, [r.row() for r in rows]) if x is not None] - self.pd_max = len(self.ids) - self.pd.set_max(0) - self.pd.value = 0 - self.failures = set([]) - - from calibre.ebooks.metadata.worker import SaveWorker - self.worker = SaveWorker(self.rq, db, self.ids, path, self.opts, - spare_server=self.spare_server) - self.pd.canceled_signal.connect(self.canceled) - self.continue_updating = True - single_shot(self.update) - - def canceled(self): - self.continue_updating = False - if self.worker is not None: - self.worker.canceled = True - self.pd.hide() - if not self.callback_called: - self.callback(self.worker.path, self.failures, self.worker.error) - self.callback_called = True - - def update(self): - if not self.continue_updating: - return - if not self.worker.is_alive(): - # Check that all ids were processed - while self.ids: - # Get all queued results since worker is dead - before = len(self.ids) - self.get_result() - if before == len(self.ids): - # No results available => worker died unexpectedly - for i in list(self.ids): - self.failures.add(('id:%d'%i, 'Unknown error')) - self.ids.remove(i) - - if not self.ids: - self.continue_updating = False - self.pd.hide() - if not self.callback_called: - try: - # Give the worker time to clean up and set worker.error - self.worker.join(2) - except: - pass # The worker was not yet started - self.callback_called = True - self.callback(self.worker.path, self.failures, self.worker.error) - - if self.continue_updating: - self.get_result() - single_shot(self.update) - - def get_result(self): - try: - id, title, ok, tb = self.rq.get_nowait() - except Empty: - return - if self.pd.max != self.pd_max: - self.pd.max = self.pd_max - self.pd.value += 1 - self.ids.remove(id) - if not isinstance(title, unicode): - title = str(title).decode(preferred_encoding, 'replace') - self.pd.set_msg(_('Saved')+' '+title) - - if not ok: - self.failures.add((title, tb)) -# }}} - - diff --git a/src/calibre/gui2/save.py b/src/calibre/gui2/save.py index 278bbe8406..8201f1cbfc 100644 --- a/src/calibre/gui2/save.py +++ b/src/calibre/gui2/save.py @@ -6,21 +6,26 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2014, Kovid Goyal ' -import traceback, errno, os +import traceback, errno, os, time, shutil from collections import namedtuple, defaultdict from tempfile import SpooledTemporaryFile -from functools import partial +from Queue import Empty from PyQt5.Qt import QObject, Qt, pyqtSignal -from calibre.customize.ui import apply_null_metadata +from calibre import prints +from calibre.constants import DEBUG +from calibre.customize.ui import can_set_metadata from calibre.db.errors import NoSuchFormat from calibre.ebooks.metadata import authors_to_string from calibre.ebooks.metadata.opf2 import metadata_to_opf +from calibre.ptempfile import PersistentTemporaryDirectory from calibre.gui2 import error_dialog, warning_dialog, gprefs, open_local_file from calibre.gui2.dialogs.progress import ProgressDialog from calibre.utils.formatter_functions import load_user_template_functions -from calibre.library.save_to_disk import sanitize_args, get_path_components, update_metadata +from calibre.utils.ipc.job import ParallelJob +from calibre.utils.ipc.server import Server +from calibre.library.save_to_disk import sanitize_args, get_path_components, find_plugboard, plugboard_save_to_disk_value BookId = namedtuple('BookId', 'title authors') @@ -68,7 +73,7 @@ class Saver(QObject): do_one_signal = pyqtSignal() - def __init__(self, book_ids, db, opts, root, parent=None): + def __init__(self, book_ids, db, opts, root, parent=None, spare_server=None): QObject.__init__(self, parent) if parent is not None: setattr(parent, 'no_gc_%s' % id(self), self) @@ -77,30 +82,40 @@ class Saver(QObject): self.template_functions = self.db.pref('user_template_functions', []) load_user_template_functions('', self.template_functions) self.collected_data = {} + self.metadata_data = {} self.errors = defaultdict(list) self._book_id_data = {} self.all_book_ids = frozenset(book_ids) - self.pd = ProgressDialog(_('Saving...'), _('Collecting metadata...'), min=0, max=0, parent=parent, icon='save.png') + self.pd = ProgressDialog(_('Saving %d books...') % len(self.all_book_ids), _('Collecting metadata...'), min=0, max=0, parent=parent, icon='save.png') self.do_one_signal.connect(self.tick, type=Qt.QueuedConnection) self.do_one = self.do_one_collect self.ids_to_collect = iter(self.all_book_ids) + self.plugboard_cache = {} + self.tdir = PersistentTemporaryDirectory('_save_to_disk') + self.server = spare_server self.pd.show() self.root, self.opts, self.path_length = sanitize_args(root, opts) self.do_one_signal.emit() + if DEBUG: + self.start_time = time.time() def tick(self): if self.pd.canceled: self.pd.close() + self.pd.deleteLater() self.break_cycles() return self.do_one() def break_cycles(self): + shutil.rmtree(self.tdir, ignore_errors=True) p = self.parent() if p is not None: setattr(p, 'no_gc_%s' % id(self), None) - self.plugboards = self.template_functions = self.collected_data = self.all_book_ids = self.pd = self.db = None + if self.server is not None: + self.server.close() + self.jobs = self.server = self.metadata_data = self.plugboard_cache = self.plugboards = self.template_functions = self.collected_data = self.all_book_ids = self.pd = self.db = None # noqa def book_id_data(self, book_id): ans = self._book_id_data.get(book_id) @@ -126,6 +141,7 @@ class Saver(QObject): def collect_data(self, book_id): mi = self.db.get_metadata(book_id) + self._book_id_data[book_id] = BookId(mi.title, mi.authors) components = get_path_components(self.opts, mi, book_id, self.path_length) self.collected_data[book_id] = (mi, components) @@ -133,6 +149,7 @@ class Saver(QObject): self.do_one = self.do_one_write ensure_unique_components(self.collected_data) self.ids_to_write = iter(self.collected_data) + self.pd.title = _('Copying files from calibre library to disk...') self.pd.max = len(self.collected_data) self.pd.value = 0 self.do_one_signal.emit() @@ -146,90 +163,143 @@ class Saver(QObject): self.pd.msg = self.book_id_data(book_id).title self.pd.value += 1 try: - self.fmts_to_write = self.write_book(book_id) + self.write_book(book_id, *self.collected_data[book_id]) except Exception: self.errors[book_id].append(('critical', traceback.format_exc())) - self.fmts_to_write = iter(()) - self.do_one = self.do_one_fmt self.do_one_signal.emit() - def writing_finished(self): - self.pd.close() - self.report() - self.break_cycles() - if gprefs['show_files_after_save']: - open_local_file(self.root) - - def write_book(self, book_id): - mi, components = self.collected_data[book_id] + def write_book(self, book_id, mi, components): base_path = os.path.join(self.root, *components) base_dir = os.path.dirname(base_path) fmts = {f.lower() for f in self.db.formats(book_id)} if self.opts.formats != 'all': asked_formats = {x.lower().strip() for x in self.opts.formats.split(',')} fmts = asked_formats.intersection(fmts) + if not fmts: + self.errors[book_id] = ('critical', _('Requested formats not available')) + return if not fmts and not self.opts.write_opf and not self.opts.save_cover: return + try: os.makedirs(base_dir) except EnvironmentError as err: if err.errno != errno.EEXIST: raise + if self.opts.update_metadata: + self.metadata_data[book_id] = d = {} + d['last_modified'] = mi.last_modified.isoformat() + cdata = self.db.cover(book_id) mi.cover, mi.cover_data = None, (None, None) - if self.opts.save_cover and cdata: - with lopen(base_path + os.extsep + 'jpg', 'wb') as f: - f.write(cdata) - mi.cover = os.path.basename(f.name) + if cdata: + fname = None + if self.opts.save_cover: + fname = base_path + os.extsep + 'jpg' + mi.cover = os.path.basename(fname) + elif self.opts.update_metadata: + fname = os.path.join(self.tdir, '%d.jpg' % book_id) + if fname: + with lopen(fname, 'wb') as f: + f.write(cdata) + if self.opts.update_metadata: + d['cover'] = fname + + fname = None if self.opts.write_opf: + fname = base_path + os.extsep + 'opf' + elif self.opts.update_metadata: + fname = os.path.join(self.tdir, '%d.opf' % book_id) + if fname: opf = metadata_to_opf(mi) - with lopen(base_path + os.extsep + 'opf', 'wb') as f: + with lopen(fname, 'wb') as f: f.write(opf) - + if self.opts.update_metadata: + d['opf'] = fname mi.cover, mi.cover_data = None, (None, None) + if self.opts.update_metadata: + d['fmts'] = [] + for fmt in fmts: + try: + fmtpath = self.write_fmt(book_id, fmt, base_path) + if fmtpath and self.opts.update_metadata and can_set_metadata(fmt): + d['fmts'].append(fmtpath) + except Exception: + self.errors[book_id].append(('fmt', (fmt, traceback.format_exc()))) + if self.opts.update_metadata and not d['fmts']: + self.metadata_data.pop(book_id, None) - return ((book_id, f, base_path, mi, cdata) for f in fmts) - - def do_one_fmt(self): - try: - args = next(self.fmts_to_write) - except StopIteration: - del self.fmts_to_write - self.do_one = self.do_one_write - self.do_one_signal.emit() - return - try: - self.write_fmt(*args) - except Exception: - self.errors[args[0]].append(('fmt', (args[1], traceback.format_exc()))) - self.do_one_signal.emit() - - def report_update_metadata_error(self, book_id, fmt, tb): - self.errors[book_id].append(('metadata', (fmt, tb))) - - def write_fmt(self, book_id, fmt, base_path, mi, cdata): + def write_fmt(self, book_id, fmt, base_path): fmtpath = base_path + os.extsep + fmt written = False with lopen(fmtpath, 'w+b') as f: - sf = SpooledFile(f) try: - self.db.copy_format_to(book_id, fmt, sf) - except NoSuchFormat: - pass - else: - if self.opts.update_metadata: - sf.seek(0) - with apply_null_metadata: - update_metadata(mi, fmt, sf, self.plugboards, cdata, - error_report=partial(self.report_update_metadata_error, book_id)) - sf.rollover() + self.db.copy_format_to(book_id, fmt, f) written = True + except NoSuchFormat: + self.errors[book_id].append(('fmt', (fmt, _('No %s format file present') % fmt.upper()))) if not written: os.remove(fmtpath) + if written: + return fmtpath + + def writing_finished(self): + if not self.opts.update_metadata: + self.metadata_data = {} + if not self.metadata_data: + self.updating_metadata_finished() + return + self.pd.title = _('Updating metadata...') + self.pd.max = len(self.metadata_data) + self.pd.value = 0 + + all_fmts = {path.rpartition(os.extsep)[-1] for d in self.metadata_data.itervalues() for path in d['fmts']} + plugboards_cache = {fmt:find_plugboard(plugboard_save_to_disk_value, fmt, self.plugboards) for fmt in all_fmts} + self.server = server = Server() if self.server is None else self.server + tasks = server.split(list(self.metadata_data)) + self.jobs = set() + for i, book_ids in enumerate(tasks): + data = {book_id:self.metadata_data[book_id] for j, book_id in book_ids} + job = ParallelJob('save_book', 'Save books (job %d of %d)' % (i+1, len(tasks)), lambda x, y:x, args=(data, plugboards_cache)) + self.jobs.add(job) + server.add_job(job) + self.do_one = self.do_one_update + self.do_one_signal.emit() + + def do_one_update(self): + running = False + for job in self.jobs: + if not job.is_finished: + running = True + job.update(consume_notifications=False) + while True: + try: + book_id = job.notifications.get_nowait()[0] + self.pd.value += 1 + self.pd.msg = self.book_id_data(book_id).title + except Empty: + break + if running: + self.do_one_signal.emit() + else: + for job in self.jobs: + for book_id, fmt, tb in (job.result or ()): + self.errors[book_id].append(('metadata', (fmt, tb))) + self.updating_metadata_finished() + + def updating_metadata_finished(self): + if DEBUG: + prints('Saved %d books in %.1f seconds' % (len(self.all_book_ids), time.time() - self.start_time)) + self.pd.close() + self.pd.deleteLater() + self.report() + self.break_cycles() + if gprefs['show_files_after_save']: + open_local_file(self.root) def format_report(self): report = [] diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py index fadffdbe19..90520b11c0 100644 --- a/src/calibre/library/save_to_disk.py +++ b/src/calibre/library/save_to_disk.py @@ -6,18 +6,16 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, traceback, cStringIO, re, shutil +import os, traceback, re, shutil from calibre.constants import DEBUG from calibre.utils.config import Config, StringConfig, tweaks from calibre.utils.formatter import TemplateFormatter -from calibre.utils.filenames import shorten_components_to, supports_long_names, \ - ascii_filename -from calibre.ebooks.metadata.opf2 import metadata_to_opf +from calibre.utils.filenames import shorten_components_to, supports_long_names, ascii_filename from calibre.constants import preferred_encoding from calibre.ebooks.metadata import fmt_sidx from calibre.ebooks.metadata import title_sort -from calibre.utils.date import parse_date, as_local_time +from calibre.utils.date import as_local_time from calibre import strftime, prints, sanitize_file_name_unicode from calibre.ptempfile import SpooledTemporaryFile from calibre.db.lazy import FormatsList @@ -301,11 +299,17 @@ def get_path_components(opts, mi, book_id, path_length): return components -def update_metadata(mi, fmt, stream, plugboards, cdata, error_report=None): - global plugboard_save_to_disk_value, plugboard_any_format_value +def update_metadata(mi, fmt, stream, plugboards, cdata, error_report=None, plugboard_cache=None): from calibre.ebooks.metadata.meta import set_metadata + if error_report is not None: + def report_error(mi, fmt, tb): + error_report(fmt, tb) + try: - cpb = find_plugboard(plugboard_save_to_disk_value, fmt, plugboards) + if plugboard_cache is not None: + cpb = plugboard_cache[fmt] + else: + cpb = find_plugboard(plugboard_save_to_disk_value, fmt, plugboards) if cpb: newmi = mi.deepcopy_metadata() newmi.template_to_attribute(mi, cpb) @@ -313,7 +317,7 @@ def update_metadata(mi, fmt, stream, plugboards, cdata, error_report=None): newmi = mi if cdata: newmi.cover_data = ('jpg', cdata) - set_metadata(stream, newmi, fmt) + set_metadata(stream, newmi, fmt, report_error=None if error_report is None else report_error) except: if error_report is None: prints('Failed to set metadata for the', fmt, 'format of', mi.title) @@ -359,6 +363,7 @@ def do_save_book_to_disk(id_, mi, cover, plugboards, mi.cover = None if opts.write_opf: + from calibre.ebooks.metadata.opf2 import metadata_to_opf opf = metadata_to_opf(mi) with open(base_path+'.opf', 'wb') as f: f.write(opf) @@ -426,37 +431,44 @@ def save_to_disk(db, ids, root, opts=None, callback=None): break return failures -def save_serialized_to_disk(ids, data, plugboards, root, opts, callback): +def read_serialized_metadata(book_id, data): from calibre.ebooks.metadata.opf2 import OPF - root, opts, length = sanitize_args(root, opts) - failures = [] - for x in ids: - opf, cover, format_map, last_modified = data[x] - if isinstance(opf, unicode): - opf = opf.encode('utf-8') - mi = OPF(cStringIO.StringIO(opf)).to_book_metadata() - try: - mi.last_modified = parse_date(last_modified) - except: - pass - tb = '' - try: - with open(cover, 'rb') as f: - cover = f.read() - except: - cover = None - try: - failed, id, title = do_save_book_to_disk(x, mi, cover, - plugboards, format_map, root, opts, length) - tb = _('Requested formats not available') - except: - failed, id, title = True, x, mi.title - tb = traceback.format_exc() - if failed: - failures.append((id, title, tb)) - if callable(callback): - if not callback(int(id), title, failed, tb): - break + from calibre.utils.date import parse_date + mi = OPF(data['opf'], try_to_guess_cover=False, populate_spine=False, basedir=os.path.dirname(data['opf'])).to_book_metadata() + try: + mi.last_modified = parse_date(data['last_modified']) + except: + pass + mi.cover, mi.cover_data = None, (None, None) + cdata = None + if 'cover' in data: + with lopen(data['cover'], 'rb') as f: + cdata = f.read() + return mi, cdata - return failures +def update_serialized_metadata(books, plugboard_cache, notification=lambda x,y:x): + result = [] + from calibre.customize.ui import apply_null_metadata + with apply_null_metadata: + for book_id, data in books.iteritems(): + fmts = [fp.rpartition(os.extsep)[-1] for fp in data['fmts']] + try: + mi, cdata = read_serialized_metadata(book_id, data) + except Exception: + tb = traceback.format_exc() + for fmt in fmts: + result.append((book_id, fmt, tb)) + else: + def report_error(fmt, tb): + result.append((book_id, fmt, tb)) + + for fmt, fmtpath in zip(fmts, data['fmts']): + try: + with lopen(fmtpath, 'r+b') as stream: + update_metadata(mi, fmt, stream, (), cdata, error_report=report_error, plugboard_cache=plugboard_cache) + except Exception: + report_error(fmt, traceback.format_exc()) + notification(book_id) + + return result diff --git a/src/calibre/utils/ipc/worker.py b/src/calibre/utils/ipc/worker.py index 28371d8000..ac190b7b7d 100644 --- a/src/calibre/utils/ipc/worker.py +++ b/src/calibre/utils/ipc/worker.py @@ -50,7 +50,7 @@ PARALLEL_FUNCS = { ('calibre.ebooks.metadata.worker', 'read_metadata_', 'notification'), 'save_book' : - ('calibre.ebooks.metadata.worker', 'save_book', 'notification'), + ('calibre.library.save_to_disk', 'update_serialized_metadata', 'notification'), 'arbitrary' : ('calibre.utils.ipc.worker', 'arbitrary', None),