mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-07-09 03:04:10 -04:00
Save to disk: When multiple books end up with the same file name because they have the same metadata, automatically adjust the filename to make it unique. Fixes #863472 [SaveToDisk loses one of identical Authors/Title books](https://bugs.launchpad.net/calibre/+bug/863472)
This commit is contained in:
Kovid Goyal
parent
fab081b392
commit
e6f4c169d9
@ -370,6 +370,13 @@ def set_file_type_metadata(stream, mi, ftype, report_error=None):
|
||||
else:
|
||||
report_error(mi, ftype, traceback.format_exc())
|
||||
|
||||
def can_set_metadata(ftype):
|
||||
ftype = ftype.lower().strip()
|
||||
for plugin in _metadata_writers.get(ftype, ()):
|
||||
if not is_disabled(plugin):
|
||||
return True
|
||||
return False
|
||||
|
||||
# }}}
|
||||
|
||||
# Add/remove plugins {{{
|
||||
|
||||
@ -8,14 +8,13 @@ __docformat__ = 'restructuredtext en'
|
||||
|
||||
from threading import Thread
|
||||
from Queue import Empty
|
||||
import os, time, sys, shutil, json
|
||||
import os, time, sys, shutil
|
||||
|
||||
from calibre.utils.ipc.job import ParallelJob
|
||||
from calibre.utils.ipc.server import Server
|
||||
from calibre.ptempfile import PersistentTemporaryDirectory, TemporaryDirectory
|
||||
from calibre import prints, isbytestring
|
||||
from calibre import prints
|
||||
from calibre.constants import filesystem_encoding
|
||||
from calibre.db.errors import NoSuchFormat
|
||||
|
||||
def debug(*args):
|
||||
prints(*args)
|
||||
@ -50,7 +49,8 @@ def do_read_metadata(task, tdir, mdir, notification):
|
||||
except:
|
||||
continue
|
||||
try:
|
||||
if isinstance(formats, basestring): formats = [formats]
|
||||
if isinstance(formats, basestring):
|
||||
formats = [formats]
|
||||
import_map = {}
|
||||
fmts, metadata_fmts = [], []
|
||||
for format in formats:
|
||||
@ -101,7 +101,8 @@ class Progress(object):
|
||||
|
||||
def __call__(self, id):
|
||||
cover = os.path.join(self.tdir, str(id))
|
||||
if not os.path.exists(cover): cover = None
|
||||
if not os.path.exists(cover):
|
||||
cover = None
|
||||
res = os.path.join(self.tdir, '%s.error'%id)
|
||||
if not os.path.exists(res):
|
||||
res = res.replace('.error', '.opf')
|
||||
@ -118,7 +119,6 @@ class ReadMetadata(Thread):
|
||||
self.failure_details = {}
|
||||
self.tdir = PersistentTemporaryDirectory('_rm_worker')
|
||||
|
||||
|
||||
def run(self):
|
||||
jobs, ids = set([]), set([])
|
||||
for t in self.tasks:
|
||||
@ -181,162 +181,3 @@ def read_metadata(paths, result_queue, chunk=50, spare_server=None):
|
||||
t = ReadMetadata(tasks, result_queue, spare_server=spare_server)
|
||||
t.start()
|
||||
return t
|
||||
|
||||
|
||||
###########################################################################
|
||||
############ Saving #####################
|
||||
###########################################################################
|
||||
|
||||
class SaveWorker(Thread):
|
||||
|
||||
def __init__(self, result_queue, db, ids, path, opts, spare_server=None):
|
||||
Thread.__init__(self)
|
||||
self.daemon = True
|
||||
self.path, self.opts = path, opts
|
||||
self.ids = ids
|
||||
self.db = db
|
||||
self.canceled = False
|
||||
self.result_queue = result_queue
|
||||
self.error = None
|
||||
self.spare_server = spare_server
|
||||
self.start()
|
||||
|
||||
def collect_data(self, ids, tdir):
|
||||
from calibre.ebooks.metadata.opf2 import metadata_to_opf
|
||||
data = {}
|
||||
for i in set(ids):
|
||||
mi = self.db.get_metadata(i, index_is_id=True, get_cover=True,
|
||||
cover_as_data=True)
|
||||
opf = metadata_to_opf(mi)
|
||||
if isbytestring(opf):
|
||||
opf = opf.decode('utf-8')
|
||||
cpath = None
|
||||
if mi.cover_data and mi.cover_data[1]:
|
||||
cpath = os.path.join(tdir, 'cover_%s.jpg'%i)
|
||||
with lopen(cpath, 'wb') as f:
|
||||
f.write(mi.cover_data[1])
|
||||
if isbytestring(cpath):
|
||||
cpath = cpath.decode(filesystem_encoding)
|
||||
formats = {}
|
||||
if mi.formats:
|
||||
for fmt in mi.formats:
|
||||
fpath = os.path.join(tdir, 'fmt_%s.%s'%(i, fmt.lower()))
|
||||
with lopen(fpath, 'wb') as f:
|
||||
try:
|
||||
self.db.copy_format_to(i, fmt, f, index_is_id=True)
|
||||
except NoSuchFormat:
|
||||
continue
|
||||
else:
|
||||
if isbytestring(fpath):
|
||||
fpath = fpath.decode(filesystem_encoding)
|
||||
formats[fmt.lower()] = fpath
|
||||
data[i] = [opf, cpath, formats, mi.last_modified.isoformat()]
|
||||
return data
|
||||
|
||||
def run(self):
|
||||
with TemporaryDirectory('save_to_disk_data') as tdir:
|
||||
self._run(tdir)
|
||||
|
||||
def _run(self, tdir):
|
||||
from calibre.library.save_to_disk import config
|
||||
server = Server() if self.spare_server is None else self.spare_server
|
||||
ids = set(self.ids)
|
||||
tasks = server.split(list(ids))
|
||||
jobs = set([])
|
||||
c = config()
|
||||
recs = {}
|
||||
for pref in c.preferences:
|
||||
recs[pref.name] = getattr(self.opts, pref.name)
|
||||
|
||||
plugboards = self.db.prefs.get('plugboards', {})
|
||||
template_functions = self.db.prefs.get('user_template_functions', [])
|
||||
|
||||
for i, task in enumerate(tasks):
|
||||
tids = [x[-1] for x in task]
|
||||
data = self.collect_data(tids, tdir)
|
||||
dpath = os.path.join(tdir, '%d.json'%i)
|
||||
with open(dpath, 'wb') as f:
|
||||
f.write(json.dumps(data, ensure_ascii=False).encode('utf-8'))
|
||||
|
||||
job = ParallelJob('save_book',
|
||||
'Save books (%d of %d)'%(i, len(tasks)),
|
||||
lambda x,y:x,
|
||||
args=[tids, dpath, plugboards, template_functions, self.path, recs])
|
||||
jobs.add(job)
|
||||
server.add_job(job)
|
||||
|
||||
|
||||
while not self.canceled:
|
||||
time.sleep(0.2)
|
||||
running = False
|
||||
for job in jobs:
|
||||
self.get_notifications(job, ids)
|
||||
if not job.is_finished:
|
||||
running = True
|
||||
|
||||
if not running:
|
||||
break
|
||||
|
||||
for job in jobs:
|
||||
if not job.result:
|
||||
continue
|
||||
for id_, title, ok, tb in job.result:
|
||||
if id_ in ids:
|
||||
self.result_queue.put((id_, title, ok, tb))
|
||||
ids.remove(id_)
|
||||
|
||||
server.close()
|
||||
time.sleep(1)
|
||||
|
||||
if self.canceled:
|
||||
return
|
||||
|
||||
for job in jobs:
|
||||
if job.failed:
|
||||
prints(job.details)
|
||||
self.error = job.details
|
||||
if os.path.exists(job.log_path):
|
||||
try:
|
||||
os.remove(job.log_path)
|
||||
except:
|
||||
pass
|
||||
|
||||
def get_notifications(self, job, ids):
|
||||
job.update(consume_notifications=False)
|
||||
while True:
|
||||
try:
|
||||
id, title, ok, tb = job.notifications.get_nowait()[0]
|
||||
if id in ids:
|
||||
self.result_queue.put((id, title, ok, tb))
|
||||
ids.remove(id)
|
||||
except Empty:
|
||||
break
|
||||
|
||||
|
||||
def save_book(ids, dpath, plugboards, template_functions, path, recs,
|
||||
notification=lambda x,y:x):
|
||||
from calibre.library.save_to_disk import config, save_serialized_to_disk
|
||||
from calibre.customize.ui import apply_null_metadata
|
||||
from calibre.utils.formatter_functions import load_user_template_functions
|
||||
load_user_template_functions('', template_functions)
|
||||
opts = config().parse()
|
||||
for name in recs:
|
||||
setattr(opts, name, recs[name])
|
||||
|
||||
results = []
|
||||
|
||||
def callback(id, title, failed, tb):
|
||||
results.append((id, title, not failed, tb))
|
||||
notification((id, title, not failed, tb))
|
||||
return True
|
||||
|
||||
data_ = json.loads(open(dpath, 'rb').read().decode('utf-8'))
|
||||
data = {}
|
||||
for k, v in data_.iteritems():
|
||||
data[int(k)] = v
|
||||
|
||||
with apply_null_metadata:
|
||||
save_serialized_to_disk(ids, data, plugboards, path, opts, callback)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
@ -7,11 +7,11 @@ __docformat__ = 'restructuredtext en'
|
||||
|
||||
import os
|
||||
from functools import partial
|
||||
from future_builtins import map
|
||||
|
||||
|
||||
from calibre.utils.config import prefs
|
||||
from calibre.gui2 import (error_dialog, Dispatcher, gprefs,
|
||||
choose_dir, warning_dialog, open_local_file)
|
||||
from calibre.gui2 import error_dialog, Dispatcher, choose_dir
|
||||
from calibre.gui2.actions import InterfaceAction
|
||||
|
||||
class SaveToDiskAction(InterfaceAction):
|
||||
@ -57,9 +57,9 @@ class SaveToDiskAction(InterfaceAction):
|
||||
|
||||
def save_specific_format_disk(self):
|
||||
rb = self.gui.iactions['Remove Books']
|
||||
ids = rb._get_selected_ids(err_title=
|
||||
_('Cannot save to disk'))
|
||||
if not ids: return
|
||||
ids = rb._get_selected_ids(err_title=_('Cannot save to disk'))
|
||||
if not ids:
|
||||
return
|
||||
fmts = rb._get_selected_formats(
|
||||
_('Choose format to save to disk'), ids,
|
||||
single=True)
|
||||
@ -96,7 +96,7 @@ class SaveToDiskAction(InterfaceAction):
|
||||
'files from your calibre library elsewhere.'), show=True)
|
||||
|
||||
if self.gui.current_view() is self.gui.library_view:
|
||||
from calibre.gui2.add import Saver
|
||||
from calibre.gui2.save import Saver
|
||||
from calibre.library.save_to_disk import config
|
||||
opts = config().parse()
|
||||
if single_format is not None:
|
||||
@ -112,10 +112,8 @@ class SaveToDiskAction(InterfaceAction):
|
||||
opts.write_opf = write_opf
|
||||
if save_cover is not None:
|
||||
opts.save_cover = save_cover
|
||||
self._saver = Saver(self.gui, self.gui.library_view.model().db,
|
||||
Dispatcher(self._books_saved), rows, path, opts,
|
||||
spare_server=self.gui.spare_server)
|
||||
|
||||
book_ids = set(map(self.gui.library_view.model().id, rows))
|
||||
Saver(book_ids, self.gui.current_db, opts, path, parent=self.gui, spare_server=self.gui.spare_server)
|
||||
else:
|
||||
paths = self.gui.current_view().model().paths(rows)
|
||||
self.gui.device_manager.save_books(
|
||||
@ -129,26 +127,6 @@ class SaveToDiskAction(InterfaceAction):
|
||||
self.save_to_disk(True, single_dir=single_dir, single_format=fmt,
|
||||
rows=rows, write_opf=False, save_cover=False)
|
||||
|
||||
def _books_saved(self, path, failures, error):
|
||||
self._saver = None
|
||||
if error:
|
||||
return error_dialog(self.gui, _('Error while saving'),
|
||||
_('There was an error while saving.'),
|
||||
error, show=True)
|
||||
if failures:
|
||||
failures = [u'%s\n\t%s'%
|
||||
(title, '\n\t'.join(err.splitlines())) for title, err in
|
||||
failures]
|
||||
|
||||
warning_dialog(self.gui, _('Could not save some books'),
|
||||
_('Could not save some books') + ', ' +
|
||||
_('Click the show details button to see which ones.'),
|
||||
u'\n\n'.join(failures), show=True)
|
||||
if gprefs['show_files_after_save']:
|
||||
open_local_file(path)
|
||||
|
||||
def books_saved(self, job):
|
||||
if job.failed:
|
||||
return self.gui.device_job_exception(job)
|
||||
|
||||
|
||||
|
||||
@ -494,91 +494,3 @@ class Adder(QObject): # {{{
|
||||
|
||||
# }}}
|
||||
|
||||
class Saver(QObject): # {{{
|
||||
|
||||
def __init__(self, parent, db, callback, rows, path, opts,
|
||||
spare_server=None):
|
||||
QObject.__init__(self, parent)
|
||||
self.pd = ProgressDialog(_('Saving...'), parent=parent)
|
||||
self.spare_server = spare_server
|
||||
self.db = db
|
||||
self.opts = opts
|
||||
self.pd.setModal(True)
|
||||
self.pd.show()
|
||||
self.pd.set_min(0)
|
||||
self.pd.set_msg(_('Collecting data, please wait...'))
|
||||
self._parent = parent
|
||||
self.callback = callback
|
||||
self.callback_called = False
|
||||
self.rq = Queue()
|
||||
self.ids = [x for x in map(db.id, [r.row() for r in rows]) if x is not None]
|
||||
self.pd_max = len(self.ids)
|
||||
self.pd.set_max(0)
|
||||
self.pd.value = 0
|
||||
self.failures = set([])
|
||||
|
||||
from calibre.ebooks.metadata.worker import SaveWorker
|
||||
self.worker = SaveWorker(self.rq, db, self.ids, path, self.opts,
|
||||
spare_server=self.spare_server)
|
||||
self.pd.canceled_signal.connect(self.canceled)
|
||||
self.continue_updating = True
|
||||
single_shot(self.update)
|
||||
|
||||
def canceled(self):
|
||||
self.continue_updating = False
|
||||
if self.worker is not None:
|
||||
self.worker.canceled = True
|
||||
self.pd.hide()
|
||||
if not self.callback_called:
|
||||
self.callback(self.worker.path, self.failures, self.worker.error)
|
||||
self.callback_called = True
|
||||
|
||||
def update(self):
|
||||
if not self.continue_updating:
|
||||
return
|
||||
if not self.worker.is_alive():
|
||||
# Check that all ids were processed
|
||||
while self.ids:
|
||||
# Get all queued results since worker is dead
|
||||
before = len(self.ids)
|
||||
self.get_result()
|
||||
if before == len(self.ids):
|
||||
# No results available => worker died unexpectedly
|
||||
for i in list(self.ids):
|
||||
self.failures.add(('id:%d'%i, 'Unknown error'))
|
||||
self.ids.remove(i)
|
||||
|
||||
if not self.ids:
|
||||
self.continue_updating = False
|
||||
self.pd.hide()
|
||||
if not self.callback_called:
|
||||
try:
|
||||
# Give the worker time to clean up and set worker.error
|
||||
self.worker.join(2)
|
||||
except:
|
||||
pass # The worker was not yet started
|
||||
self.callback_called = True
|
||||
self.callback(self.worker.path, self.failures, self.worker.error)
|
||||
|
||||
if self.continue_updating:
|
||||
self.get_result()
|
||||
single_shot(self.update)
|
||||
|
||||
def get_result(self):
|
||||
try:
|
||||
id, title, ok, tb = self.rq.get_nowait()
|
||||
except Empty:
|
||||
return
|
||||
if self.pd.max != self.pd_max:
|
||||
self.pd.max = self.pd_max
|
||||
self.pd.value += 1
|
||||
self.ids.remove(id)
|
||||
if not isinstance(title, unicode):
|
||||
title = str(title).decode(preferred_encoding, 'replace')
|
||||
self.pd.set_msg(_('Saved')+' '+title)
|
||||
|
||||
if not ok:
|
||||
self.failures.add((title, tb))
|
||||
# }}}
|
||||
|
||||
|
||||
|
||||
@ -6,21 +6,26 @@ from __future__ import (unicode_literals, division, absolute_import,
|
||||
__license__ = 'GPL v3'
|
||||
__copyright__ = '2014, Kovid Goyal <kovid at kovidgoyal.net>'
|
||||
|
||||
import traceback, errno, os
|
||||
import traceback, errno, os, time, shutil
|
||||
from collections import namedtuple, defaultdict
|
||||
from tempfile import SpooledTemporaryFile
|
||||
from functools import partial
|
||||
from Queue import Empty
|
||||
|
||||
from PyQt5.Qt import QObject, Qt, pyqtSignal
|
||||
|
||||
from calibre.customize.ui import apply_null_metadata
|
||||
from calibre import prints
|
||||
from calibre.constants import DEBUG
|
||||
from calibre.customize.ui import can_set_metadata
|
||||
from calibre.db.errors import NoSuchFormat
|
||||
from calibre.ebooks.metadata import authors_to_string
|
||||
from calibre.ebooks.metadata.opf2 import metadata_to_opf
|
||||
from calibre.ptempfile import PersistentTemporaryDirectory
|
||||
from calibre.gui2 import error_dialog, warning_dialog, gprefs, open_local_file
|
||||
from calibre.gui2.dialogs.progress import ProgressDialog
|
||||
from calibre.utils.formatter_functions import load_user_template_functions
|
||||
from calibre.library.save_to_disk import sanitize_args, get_path_components, update_metadata
|
||||
from calibre.utils.ipc.job import ParallelJob
|
||||
from calibre.utils.ipc.server import Server
|
||||
from calibre.library.save_to_disk import sanitize_args, get_path_components, find_plugboard, plugboard_save_to_disk_value
|
||||
|
||||
BookId = namedtuple('BookId', 'title authors')
|
||||
|
||||
@ -68,7 +73,7 @@ class Saver(QObject):
|
||||
|
||||
do_one_signal = pyqtSignal()
|
||||
|
||||
def __init__(self, book_ids, db, opts, root, parent=None):
|
||||
def __init__(self, book_ids, db, opts, root, parent=None, spare_server=None):
|
||||
QObject.__init__(self, parent)
|
||||
if parent is not None:
|
||||
setattr(parent, 'no_gc_%s' % id(self), self)
|
||||
@ -77,30 +82,40 @@ class Saver(QObject):
|
||||
self.template_functions = self.db.pref('user_template_functions', [])
|
||||
load_user_template_functions('', self.template_functions)
|
||||
self.collected_data = {}
|
||||
self.metadata_data = {}
|
||||
self.errors = defaultdict(list)
|
||||
self._book_id_data = {}
|
||||
self.all_book_ids = frozenset(book_ids)
|
||||
self.pd = ProgressDialog(_('Saving...'), _('Collecting metadata...'), min=0, max=0, parent=parent, icon='save.png')
|
||||
self.pd = ProgressDialog(_('Saving %d books...') % len(self.all_book_ids), _('Collecting metadata...'), min=0, max=0, parent=parent, icon='save.png')
|
||||
self.do_one_signal.connect(self.tick, type=Qt.QueuedConnection)
|
||||
self.do_one = self.do_one_collect
|
||||
self.ids_to_collect = iter(self.all_book_ids)
|
||||
self.plugboard_cache = {}
|
||||
self.tdir = PersistentTemporaryDirectory('_save_to_disk')
|
||||
self.server = spare_server
|
||||
|
||||
self.pd.show()
|
||||
self.root, self.opts, self.path_length = sanitize_args(root, opts)
|
||||
self.do_one_signal.emit()
|
||||
if DEBUG:
|
||||
self.start_time = time.time()
|
||||
|
||||
def tick(self):
|
||||
if self.pd.canceled:
|
||||
self.pd.close()
|
||||
self.pd.deleteLater()
|
||||
self.break_cycles()
|
||||
return
|
||||
self.do_one()
|
||||
|
||||
def break_cycles(self):
|
||||
shutil.rmtree(self.tdir, ignore_errors=True)
|
||||
p = self.parent()
|
||||
if p is not None:
|
||||
setattr(p, 'no_gc_%s' % id(self), None)
|
||||
self.plugboards = self.template_functions = self.collected_data = self.all_book_ids = self.pd = self.db = None
|
||||
if self.server is not None:
|
||||
self.server.close()
|
||||
self.jobs = self.server = self.metadata_data = self.plugboard_cache = self.plugboards = self.template_functions = self.collected_data = self.all_book_ids = self.pd = self.db = None # noqa
|
||||
|
||||
def book_id_data(self, book_id):
|
||||
ans = self._book_id_data.get(book_id)
|
||||
@ -126,6 +141,7 @@ class Saver(QObject):
|
||||
|
||||
def collect_data(self, book_id):
|
||||
mi = self.db.get_metadata(book_id)
|
||||
self._book_id_data[book_id] = BookId(mi.title, mi.authors)
|
||||
components = get_path_components(self.opts, mi, book_id, self.path_length)
|
||||
self.collected_data[book_id] = (mi, components)
|
||||
|
||||
@ -133,6 +149,7 @@ class Saver(QObject):
|
||||
self.do_one = self.do_one_write
|
||||
ensure_unique_components(self.collected_data)
|
||||
self.ids_to_write = iter(self.collected_data)
|
||||
self.pd.title = _('Copying files from calibre library to disk...')
|
||||
self.pd.max = len(self.collected_data)
|
||||
self.pd.value = 0
|
||||
self.do_one_signal.emit()
|
||||
@ -146,90 +163,143 @@ class Saver(QObject):
|
||||
self.pd.msg = self.book_id_data(book_id).title
|
||||
self.pd.value += 1
|
||||
try:
|
||||
self.fmts_to_write = self.write_book(book_id)
|
||||
self.write_book(book_id, *self.collected_data[book_id])
|
||||
except Exception:
|
||||
self.errors[book_id].append(('critical', traceback.format_exc()))
|
||||
self.fmts_to_write = iter(())
|
||||
self.do_one = self.do_one_fmt
|
||||
self.do_one_signal.emit()
|
||||
|
||||
def writing_finished(self):
|
||||
self.pd.close()
|
||||
self.report()
|
||||
self.break_cycles()
|
||||
if gprefs['show_files_after_save']:
|
||||
open_local_file(self.root)
|
||||
|
||||
def write_book(self, book_id):
|
||||
mi, components = self.collected_data[book_id]
|
||||
def write_book(self, book_id, mi, components):
|
||||
base_path = os.path.join(self.root, *components)
|
||||
base_dir = os.path.dirname(base_path)
|
||||
fmts = {f.lower() for f in self.db.formats(book_id)}
|
||||
if self.opts.formats != 'all':
|
||||
asked_formats = {x.lower().strip() for x in self.opts.formats.split(',')}
|
||||
fmts = asked_formats.intersection(fmts)
|
||||
if not fmts:
|
||||
self.errors[book_id] = ('critical', _('Requested formats not available'))
|
||||
return
|
||||
|
||||
if not fmts and not self.opts.write_opf and not self.opts.save_cover:
|
||||
return
|
||||
|
||||
try:
|
||||
os.makedirs(base_dir)
|
||||
except EnvironmentError as err:
|
||||
if err.errno != errno.EEXIST:
|
||||
raise
|
||||
|
||||
if self.opts.update_metadata:
|
||||
self.metadata_data[book_id] = d = {}
|
||||
d['last_modified'] = mi.last_modified.isoformat()
|
||||
|
||||
cdata = self.db.cover(book_id)
|
||||
mi.cover, mi.cover_data = None, (None, None)
|
||||
|
||||
if self.opts.save_cover and cdata:
|
||||
with lopen(base_path + os.extsep + 'jpg', 'wb') as f:
|
||||
f.write(cdata)
|
||||
mi.cover = os.path.basename(f.name)
|
||||
if cdata:
|
||||
fname = None
|
||||
if self.opts.save_cover:
|
||||
fname = base_path + os.extsep + 'jpg'
|
||||
mi.cover = os.path.basename(fname)
|
||||
elif self.opts.update_metadata:
|
||||
fname = os.path.join(self.tdir, '%d.jpg' % book_id)
|
||||
|
||||
if fname:
|
||||
with lopen(fname, 'wb') as f:
|
||||
f.write(cdata)
|
||||
if self.opts.update_metadata:
|
||||
d['cover'] = fname
|
||||
|
||||
fname = None
|
||||
if self.opts.write_opf:
|
||||
fname = base_path + os.extsep + 'opf'
|
||||
elif self.opts.update_metadata:
|
||||
fname = os.path.join(self.tdir, '%d.opf' % book_id)
|
||||
if fname:
|
||||
opf = metadata_to_opf(mi)
|
||||
with lopen(base_path + os.extsep + 'opf', 'wb') as f:
|
||||
with lopen(fname, 'wb') as f:
|
||||
f.write(opf)
|
||||
|
||||
if self.opts.update_metadata:
|
||||
d['opf'] = fname
|
||||
mi.cover, mi.cover_data = None, (None, None)
|
||||
if self.opts.update_metadata:
|
||||
d['fmts'] = []
|
||||
for fmt in fmts:
|
||||
try:
|
||||
fmtpath = self.write_fmt(book_id, fmt, base_path)
|
||||
if fmtpath and self.opts.update_metadata and can_set_metadata(fmt):
|
||||
d['fmts'].append(fmtpath)
|
||||
except Exception:
|
||||
self.errors[book_id].append(('fmt', (fmt, traceback.format_exc())))
|
||||
if self.opts.update_metadata and not d['fmts']:
|
||||
self.metadata_data.pop(book_id, None)
|
||||
|
||||
return ((book_id, f, base_path, mi, cdata) for f in fmts)
|
||||
|
||||
def do_one_fmt(self):
|
||||
try:
|
||||
args = next(self.fmts_to_write)
|
||||
except StopIteration:
|
||||
del self.fmts_to_write
|
||||
self.do_one = self.do_one_write
|
||||
self.do_one_signal.emit()
|
||||
return
|
||||
try:
|
||||
self.write_fmt(*args)
|
||||
except Exception:
|
||||
self.errors[args[0]].append(('fmt', (args[1], traceback.format_exc())))
|
||||
self.do_one_signal.emit()
|
||||
|
||||
def report_update_metadata_error(self, book_id, fmt, tb):
|
||||
self.errors[book_id].append(('metadata', (fmt, tb)))
|
||||
|
||||
def write_fmt(self, book_id, fmt, base_path, mi, cdata):
|
||||
def write_fmt(self, book_id, fmt, base_path):
|
||||
fmtpath = base_path + os.extsep + fmt
|
||||
written = False
|
||||
with lopen(fmtpath, 'w+b') as f:
|
||||
sf = SpooledFile(f)
|
||||
try:
|
||||
self.db.copy_format_to(book_id, fmt, sf)
|
||||
except NoSuchFormat:
|
||||
pass
|
||||
else:
|
||||
if self.opts.update_metadata:
|
||||
sf.seek(0)
|
||||
with apply_null_metadata:
|
||||
update_metadata(mi, fmt, sf, self.plugboards, cdata,
|
||||
error_report=partial(self.report_update_metadata_error, book_id))
|
||||
sf.rollover()
|
||||
self.db.copy_format_to(book_id, fmt, f)
|
||||
written = True
|
||||
except NoSuchFormat:
|
||||
self.errors[book_id].append(('fmt', (fmt, _('No %s format file present') % fmt.upper())))
|
||||
if not written:
|
||||
os.remove(fmtpath)
|
||||
if written:
|
||||
return fmtpath
|
||||
|
||||
def writing_finished(self):
|
||||
if not self.opts.update_metadata:
|
||||
self.metadata_data = {}
|
||||
if not self.metadata_data:
|
||||
self.updating_metadata_finished()
|
||||
return
|
||||
self.pd.title = _('Updating metadata...')
|
||||
self.pd.max = len(self.metadata_data)
|
||||
self.pd.value = 0
|
||||
|
||||
all_fmts = {path.rpartition(os.extsep)[-1] for d in self.metadata_data.itervalues() for path in d['fmts']}
|
||||
plugboards_cache = {fmt:find_plugboard(plugboard_save_to_disk_value, fmt, self.plugboards) for fmt in all_fmts}
|
||||
self.server = server = Server() if self.server is None else self.server
|
||||
tasks = server.split(list(self.metadata_data))
|
||||
self.jobs = set()
|
||||
for i, book_ids in enumerate(tasks):
|
||||
data = {book_id:self.metadata_data[book_id] for j, book_id in book_ids}
|
||||
job = ParallelJob('save_book', 'Save books (job %d of %d)' % (i+1, len(tasks)), lambda x, y:x, args=(data, plugboards_cache))
|
||||
self.jobs.add(job)
|
||||
server.add_job(job)
|
||||
self.do_one = self.do_one_update
|
||||
self.do_one_signal.emit()
|
||||
|
||||
def do_one_update(self):
|
||||
running = False
|
||||
for job in self.jobs:
|
||||
if not job.is_finished:
|
||||
running = True
|
||||
job.update(consume_notifications=False)
|
||||
while True:
|
||||
try:
|
||||
book_id = job.notifications.get_nowait()[0]
|
||||
self.pd.value += 1
|
||||
self.pd.msg = self.book_id_data(book_id).title
|
||||
except Empty:
|
||||
break
|
||||
if running:
|
||||
self.do_one_signal.emit()
|
||||
else:
|
||||
for job in self.jobs:
|
||||
for book_id, fmt, tb in (job.result or ()):
|
||||
self.errors[book_id].append(('metadata', (fmt, tb)))
|
||||
self.updating_metadata_finished()
|
||||
|
||||
def updating_metadata_finished(self):
|
||||
if DEBUG:
|
||||
prints('Saved %d books in %.1f seconds' % (len(self.all_book_ids), time.time() - self.start_time))
|
||||
self.pd.close()
|
||||
self.pd.deleteLater()
|
||||
self.report()
|
||||
self.break_cycles()
|
||||
if gprefs['show_files_after_save']:
|
||||
open_local_file(self.root)
|
||||
|
||||
def format_report(self):
|
||||
report = []
|
||||
|
||||
@ -6,18 +6,16 @@ __license__ = 'GPL v3'
|
||||
__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
|
||||
__docformat__ = 'restructuredtext en'
|
||||
|
||||
import os, traceback, cStringIO, re, shutil
|
||||
import os, traceback, re, shutil
|
||||
|
||||
from calibre.constants import DEBUG
|
||||
from calibre.utils.config import Config, StringConfig, tweaks
|
||||
from calibre.utils.formatter import TemplateFormatter
|
||||
from calibre.utils.filenames import shorten_components_to, supports_long_names, \
|
||||
ascii_filename
|
||||
from calibre.ebooks.metadata.opf2 import metadata_to_opf
|
||||
from calibre.utils.filenames import shorten_components_to, supports_long_names, ascii_filename
|
||||
from calibre.constants import preferred_encoding
|
||||
from calibre.ebooks.metadata import fmt_sidx
|
||||
from calibre.ebooks.metadata import title_sort
|
||||
from calibre.utils.date import parse_date, as_local_time
|
||||
from calibre.utils.date import as_local_time
|
||||
from calibre import strftime, prints, sanitize_file_name_unicode
|
||||
from calibre.ptempfile import SpooledTemporaryFile
|
||||
from calibre.db.lazy import FormatsList
|
||||
@ -301,11 +299,17 @@ def get_path_components(opts, mi, book_id, path_length):
|
||||
return components
|
||||
|
||||
|
||||
def update_metadata(mi, fmt, stream, plugboards, cdata, error_report=None):
|
||||
global plugboard_save_to_disk_value, plugboard_any_format_value
|
||||
def update_metadata(mi, fmt, stream, plugboards, cdata, error_report=None, plugboard_cache=None):
|
||||
from calibre.ebooks.metadata.meta import set_metadata
|
||||
if error_report is not None:
|
||||
def report_error(mi, fmt, tb):
|
||||
error_report(fmt, tb)
|
||||
|
||||
try:
|
||||
cpb = find_plugboard(plugboard_save_to_disk_value, fmt, plugboards)
|
||||
if plugboard_cache is not None:
|
||||
cpb = plugboard_cache[fmt]
|
||||
else:
|
||||
cpb = find_plugboard(plugboard_save_to_disk_value, fmt, plugboards)
|
||||
if cpb:
|
||||
newmi = mi.deepcopy_metadata()
|
||||
newmi.template_to_attribute(mi, cpb)
|
||||
@ -313,7 +317,7 @@ def update_metadata(mi, fmt, stream, plugboards, cdata, error_report=None):
|
||||
newmi = mi
|
||||
if cdata:
|
||||
newmi.cover_data = ('jpg', cdata)
|
||||
set_metadata(stream, newmi, fmt)
|
||||
set_metadata(stream, newmi, fmt, report_error=None if error_report is None else report_error)
|
||||
except:
|
||||
if error_report is None:
|
||||
prints('Failed to set metadata for the', fmt, 'format of', mi.title)
|
||||
@ -359,6 +363,7 @@ def do_save_book_to_disk(id_, mi, cover, plugboards,
|
||||
mi.cover = None
|
||||
|
||||
if opts.write_opf:
|
||||
from calibre.ebooks.metadata.opf2 import metadata_to_opf
|
||||
opf = metadata_to_opf(mi)
|
||||
with open(base_path+'.opf', 'wb') as f:
|
||||
f.write(opf)
|
||||
@ -426,37 +431,44 @@ def save_to_disk(db, ids, root, opts=None, callback=None):
|
||||
break
|
||||
return failures
|
||||
|
||||
def save_serialized_to_disk(ids, data, plugboards, root, opts, callback):
|
||||
def read_serialized_metadata(book_id, data):
|
||||
from calibre.ebooks.metadata.opf2 import OPF
|
||||
root, opts, length = sanitize_args(root, opts)
|
||||
failures = []
|
||||
for x in ids:
|
||||
opf, cover, format_map, last_modified = data[x]
|
||||
if isinstance(opf, unicode):
|
||||
opf = opf.encode('utf-8')
|
||||
mi = OPF(cStringIO.StringIO(opf)).to_book_metadata()
|
||||
try:
|
||||
mi.last_modified = parse_date(last_modified)
|
||||
except:
|
||||
pass
|
||||
tb = ''
|
||||
try:
|
||||
with open(cover, 'rb') as f:
|
||||
cover = f.read()
|
||||
except:
|
||||
cover = None
|
||||
try:
|
||||
failed, id, title = do_save_book_to_disk(x, mi, cover,
|
||||
plugboards, format_map, root, opts, length)
|
||||
tb = _('Requested formats not available')
|
||||
except:
|
||||
failed, id, title = True, x, mi.title
|
||||
tb = traceback.format_exc()
|
||||
if failed:
|
||||
failures.append((id, title, tb))
|
||||
if callable(callback):
|
||||
if not callback(int(id), title, failed, tb):
|
||||
break
|
||||
from calibre.utils.date import parse_date
|
||||
mi = OPF(data['opf'], try_to_guess_cover=False, populate_spine=False, basedir=os.path.dirname(data['opf'])).to_book_metadata()
|
||||
try:
|
||||
mi.last_modified = parse_date(data['last_modified'])
|
||||
except:
|
||||
pass
|
||||
mi.cover, mi.cover_data = None, (None, None)
|
||||
cdata = None
|
||||
if 'cover' in data:
|
||||
with lopen(data['cover'], 'rb') as f:
|
||||
cdata = f.read()
|
||||
return mi, cdata
|
||||
|
||||
return failures
|
||||
def update_serialized_metadata(books, plugboard_cache, notification=lambda x,y:x):
|
||||
result = []
|
||||
from calibre.customize.ui import apply_null_metadata
|
||||
with apply_null_metadata:
|
||||
|
||||
for book_id, data in books.iteritems():
|
||||
fmts = [fp.rpartition(os.extsep)[-1] for fp in data['fmts']]
|
||||
try:
|
||||
mi, cdata = read_serialized_metadata(book_id, data)
|
||||
except Exception:
|
||||
tb = traceback.format_exc()
|
||||
for fmt in fmts:
|
||||
result.append((book_id, fmt, tb))
|
||||
else:
|
||||
def report_error(fmt, tb):
|
||||
result.append((book_id, fmt, tb))
|
||||
|
||||
for fmt, fmtpath in zip(fmts, data['fmts']):
|
||||
try:
|
||||
with lopen(fmtpath, 'r+b') as stream:
|
||||
update_metadata(mi, fmt, stream, (), cdata, error_report=report_error, plugboard_cache=plugboard_cache)
|
||||
except Exception:
|
||||
report_error(fmt, traceback.format_exc())
|
||||
notification(book_id)
|
||||
|
||||
return result
|
||||
|
||||
@ -50,7 +50,7 @@ PARALLEL_FUNCS = {
|
||||
('calibre.ebooks.metadata.worker', 'read_metadata_', 'notification'),
|
||||
|
||||
'save_book' :
|
||||
('calibre.ebooks.metadata.worker', 'save_book', 'notification'),
|
||||
('calibre.library.save_to_disk', 'update_serialized_metadata', 'notification'),
|
||||
|
||||
'arbitrary' :
|
||||
('calibre.utils.ipc.worker', 'arbitrary', None),
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user