diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index e46cdbccb8..de976e24e9 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -1446,7 +1446,7 @@ class DB(object): if wam is not None: wam.close_handles() - def add_format(self, book_id, fmt, stream, title, author, path, current_name): + def add_format(self, book_id, fmt, stream, title, author, path, current_name, mtime=None): fmt = ('.' + fmt.lower()) if fmt else '' fname = self.construct_file_name(book_id, title, author, len(fmt)) path = os.path.join(self.library_path, path) @@ -1475,8 +1475,12 @@ class DB(object): with lopen(dest, 'wb') as f: shutil.copyfileobj(stream, f) size = f.tell() + if mtime is not None: + os.utime(dest, (mtime, mtime)) elif os.path.exists(dest): size = os.path.getsize(dest) + if mtime is not None: + os.utime(dest, (mtime, mtime)) return size, fname diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 1b0514f8ee..ee43d3cd7f 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -1320,7 +1320,7 @@ class Cache(object): self._reload_from_db() raise - def _do_add_format(self, book_id, fmt, stream, name=None): + def _do_add_format(self, book_id, fmt, stream, name=None, mtime=None): path = self._field_for('path', book_id) if path is None: # Theoretically, this should never happen, but apparently it @@ -1335,7 +1335,7 @@ class Cache(object): except IndexError: author = _('Unknown') - size, fname = self.backend.add_format(book_id, fmt, stream, title, author, path, name) + size, fname = self.backend.add_format(book_id, fmt, stream, title, author, path, name, mtime=mtime) return size, fname @api @@ -2122,9 +2122,10 @@ class Cache(object): progress(self._field_for('title', book_id), i + 1, total) format_metadata[book_id] = {} for fmt in self._formats(book_id): + mdata = self.format_metadata(book_id, fmt) key = '%s:%s:%s' % (key_prefix, book_id, fmt) format_metadata[book_id][fmt] = key - with exporter.start_file(key) as dest: + with exporter.start_file(key, mtime=mdata.get('mtime')) as dest: self._copy_format_to(book_id, fmt, dest, report_file_size=dest.ensure_space) cover_key = '%s:%s:%s' % (key_prefix, book_id, '.cover') with exporter.start_file(cover_key) as dest: @@ -2133,7 +2134,6 @@ class Cache(object): else: format_metadata[book_id]['.cover'] = cover_key exporter.set_metadata(library_key, metadata) - exporter.commit() if progress is not None: progress(_('Completed'), total, total) @@ -2162,7 +2162,7 @@ def import_library(library_key, importer, library_path, progress=None): cache.backend.set_cover(book_id, path, stream, no_processing=True) else: stream = importer.start_file(fmtkey, _('{0} format for {1}').format(fmt.upper(), title)) - size, fname = cache._do_add_format(book_id, fmt, stream) + size, fname = cache._do_add_format(book_id, fmt, stream, mtime=stream.mtime) cache.fields['formats'].table.update_fmt(book_id, fmt, fname, size, cache.backend) stream.close() cache.dump_metadata({book_id}) diff --git a/src/calibre/db/tests/filesystem.py b/src/calibre/db/tests/filesystem.py index da4910063f..a4def468d2 100644 --- a/src/calibre/db/tests/filesystem.py +++ b/src/calibre/db/tests/filesystem.py @@ -152,6 +152,7 @@ class FilesystemTest(BaseTest): with TemporaryDirectory('export_lib') as tdir, TemporaryDirectory('import_lib') as idir: exporter = Exporter(tdir, part_size=part_size) cache.export_library('l', exporter) + exporter.commit() importer = Importer(tdir) ic = import_library('l', importer, idir) self.assertEqual(cache.all_book_ids(), ic.all_book_ids()) @@ -159,3 +160,4 @@ class FilesystemTest(BaseTest): self.assertEqual(cache.cover(book_id), ic.cover(book_id), 'Covers not identical for book: %d' % book_id) for fmt in cache.formats(book_id): self.assertEqual(cache.format(book_id, fmt), ic.format(book_id, fmt)) + self.assertEqual(cache.format_metadata(book_id, fmt)['mtime'], cache.format_metadata(book_id, fmt)['mtime']) diff --git a/src/calibre/utils/exim.py b/src/calibre/utils/exim.py index 4b51416ce0..d5b69c41d8 100644 --- a/src/calibre/utils/exim.py +++ b/src/calibre/utils/exim.py @@ -4,7 +4,15 @@ from __future__ import (unicode_literals, division, absolute_import, print_function) -import os, json, struct, hashlib +import os, json, struct, hashlib, sys +from binascii import hexlify + +from calibre.constants import config_dir +from calibre.utils.config import prefs +from calibre.utils.filenames import samefile + + +# Export {{{ def send_file(from_obj, to_obj, chunksize=1<<20): m = hashlib.sha1() @@ -18,11 +26,12 @@ def send_file(from_obj, to_obj, chunksize=1<<20): class FileDest(object): - def __init__(self, key, exporter): + def __init__(self, key, exporter, mtime=None): self.exporter, self.key = exporter, key self.hasher = hashlib.sha1() self.start_pos = exporter.f.tell() self._discard = False + self.mtime = None def discard(self): self._discard = True @@ -43,7 +52,7 @@ class FileDest(object): if not self._discard: size = self.exporter.f.tell() - self.start_pos digest = type('')(self.hasher.hexdigest()) - self.exporter.file_metadata[self.key] = (len(self.exporter.parts), self.start_pos, size, digest) + self.exporter.file_metadata[self.key] = (len(self.exporter.parts), self.start_pos, size, digest, self.mtime) del self.exporter, self.hasher def __enter__(self): @@ -87,8 +96,11 @@ class Exporter(object): self.parts[-1] = self.f.name def ensure_space(self, size): - if size + self.f.tell() < self.part_size: - return + try: + if size + self.f.tell() < self.part_size: + return + except AttributeError: + raise RuntimeError('This exporter has already been commited, cannot add to it') self.commit_part() self.new_part() @@ -109,15 +121,82 @@ class Exporter(object): pos = self.f.tell() digest = send_file(fileobj, self.f) size = self.f.tell() - pos - self.file_metadata[key] = (len(self.parts), pos, size, digest) + mtime = os.fstat(fileobj.fileno()).st_mtime + self.file_metadata[key] = (len(self.parts), pos, size, digest, mtime) - def start_file(self, key): - return FileDest(key, self) + def start_file(self, key, mtime=None): + return FileDest(key, self, mtime=mtime) + + def export_dir(self, path, dir_key): + pkey = hexlify(dir_key) + self.metadata[dir_key] = files = [] + for dirpath, dirnames, filenames in os.walk(path): + for fname in filenames: + fpath = os.path.join(dirpath, fname) + rpath = os.path.relpath(fpath, path).replace(os.sep, '/') + key = '%s:%s' % (pkey, rpath) + with lopen(fpath, 'rb') as f: + self.add_file(f, key) + files.append((key, rpath)) + +def all_known_libraries(): + from calibre.gui2 import gprefs + paths = set(gprefs.get('library_usage_stats', ())) + if prefs['library_path']: + paths.add(prefs['library_path']) + added = set() + for path in paths: + mdb = os.path.join(path) + if os.path.isdir(path) and os.path.exists(mdb): + seen = False + for c in added: + if samefile(mdb, c): + seen = True + break + if not seen: + added.add(path) + return added + +def export(destdir, library_paths=None, dbmap=None, progress1=None, progress2=None): + from calibre.db.cache import Cache + from calibre.db.backend import DB + if library_paths is None: + library_paths = all_known_libraries() + dbmap = dbmap or {} + dbmap = {os.path.normace(os.path.abspath(k)):v for k, v in dbmap.iteritems()} + exporter = Exporter(destdir) + exporter.metadata['libraries'] = libraries = [] + total = len(library_paths) + 2 + for i, lpath in enumerate(library_paths): + if progress1 is not None: + progress1(i + 1, total, lpath) + key = os.path.normcase(os.path.abspath(lpath)) + db, closedb = dbmap.get(lpath), False + if db is None: + db = Cache(DB(lpath, load_user_formatter_functions=False)) + db.init() + closedb = True + else: + db = db.new_api + db.export_library(key, exporter, progress=progress2) + if closedb: + db.close() + libraries.append(key) + if progress1 is not None: + progress1(total - 1, total, _('Settings and plugins')) + exporter.export_dir(config_dir, 'config_dir') + exporter.commit() + if progress1 is not None: + progress1(total, total, _('Completed')) +# }}} + +# Import {{{ class FileSource(object): - def __init__(self, f, size, digest, description, importer): + def __init__(self, f, size, digest, description, mtime, importer): self.f, self.size, self.digest, self.description = f, size, digest, description + self.mtime = mtime self.end = f.tell() + size self.hasher = hashlib.sha1() self.importer = importer @@ -180,7 +259,11 @@ class Importer(object): return lopen(self.part_map[num], 'rb') def start_file(self, key, description): - partnum, pos, size, digest = self.file_metadata[key] + partnum, pos, size, digest, mtime = self.file_metadata[key] f = self.part(partnum) f.seek(pos) - return FileSource(f, size, digest, description, self) + return FileSource(f, size, digest, description, mtime, self) +# }}} + +if __name__ == '__main__': + export(sys.argv[-1], progress1=print, progress2=print)