diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index 034f81eb65..b1bbd77680 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -57,6 +57,7 @@ from polyglot.builtins import ( # }}} COVER_FILE_NAME = 'cover.jpg' +METADATA_FILE_NAME = 'metadata.opf' DEFAULT_TRASH_EXPIRY_TIME_SECONDS = 14 * 86400 TRASH_DIR_NAME = '.caltrash' BOOK_ID_PATH_TEMPLATE = ' ({})' @@ -1584,7 +1585,8 @@ class DB: try: f = open(path, 'rb') except OSError: - time.sleep(0.2) + if iswindows: + time.sleep(0.2) try: f = open(path, 'rb') except OSError as e: @@ -1624,7 +1626,8 @@ class DB: try: f = open(path, 'rb') except OSError: - time.sleep(0.2) + if iswindows: + time.sleep(0.2) f = open(path, 'rb') with f: return True, f.read(), stat.st_mtime @@ -1660,7 +1663,8 @@ class DB: try: os.remove(path) except OSError: - time.sleep(0.2) + if iswindows: + time.sleep(0.2) os.remove(path) else: if no_processing: @@ -1671,7 +1675,8 @@ class DB: try: save_cover_data_to(data, path) except OSError: - time.sleep(0.2) + if iswindows: + time.sleep(0.2) save_cover_data_to(data, path) def copy_format_to(self, book_id, fmt, fname, path, dest, @@ -1884,8 +1889,42 @@ class DB: os.makedirs(tpath) update_paths_in_db() + def iter_extra_files(self, book_id, book_path, formats_field): + known_files = {COVER_FILE_NAME, METADATA_FILE_NAME} + for fmt in formats_field.for_book(book_id, default_value=()): + fname = formats_field.format_fname(book_id, fmt) + fpath = self.format_abspath(book_id, fmt, fname, book_path, do_file_rename=False) + if fpath: + known_files.add(os.path.basename(fpath)) + full_book_path = os.path.abspath(os.path.join(self.library_path, book_path)) + for dirpath, dirnames, filenames in os.walk(full_book_path): + for fname in filenames: + path = os.path.join(dirpath, fname) + if os.access(path, os.R_OK): + relpath = os.path.relpath(path, full_book_path) + relpath = relpath.replace(os.sep, '/') + if relpath not in known_files: + try: + src = open(path, 'rb') + except OSError: + if iswindows: + time.sleep(1) + src = open(path, 'rb') + with src: + yield relpath, src, os.path.getmtime(path) + + def add_extra_file(self, relpath, stream, book_path): + dest = os.path.abspath(os.path.join(self.library_path, book_path, relpath)) + try: + d = open(dest, 'wb') + except OSError: + os.makedirs(os.path.dirname(dest), exist_ok=True) + d = open(dest, 'wb') + with d: + shutil.copyfileobj(stream, d) + def write_backup(self, path, raw): - path = os.path.abspath(os.path.join(self.library_path, path, 'metadata.opf')) + path = os.path.abspath(os.path.join(self.library_path, path, METADATA_FILE_NAME)) try: with open(path, 'wb') as f: f.write(raw) @@ -1904,7 +1943,7 @@ class DB: f.write(raw) def read_backup(self, path): - path = os.path.abspath(os.path.join(self.library_path, path, 'metadata.opf')) + path = os.path.abspath(os.path.join(self.library_path, path, METADATA_FILE_NAME)) with open(path, 'rb') as f: return f.read() @@ -1972,7 +2011,7 @@ class DB: mi, _, annotations = read_opf(bdir, read_annotations=read_annotations) formats = [] for x in os.scandir(bdir): - if x.is_file() and x.name not in (COVER_FILE_NAME, 'metadata.opf') and '.' in x.name: + if x.is_file() and x.name not in (COVER_FILE_NAME, METADATA_FILE_NAME) and '.' in x.name: try: size = x.stat(follow_symlinks=False).st_size except OSError: @@ -2016,7 +2055,7 @@ class DB: mtime = x.stat(follow_symlinks=False).st_mtime except Exception: continue - opf = OPF(os.path.join(x.path, 'metadata.opf'), basedir=x.path) + opf = OPF(os.path.join(x.path, METADATA_FILE_NAME), basedir=x.path) books.append(TrashEntry(book_id, opf.title or unknown, (opf.authors or au)[0], os.path.join(x.path, COVER_FILE_NAME), mtime)) base = os.path.join(self.trash_dir, 'f') um = {'title': unknown, 'authors': au} diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 1b67a028b9..ea9c7211cc 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -2927,7 +2927,8 @@ class Cache: os.remove(pt.name) format_metadata = {} - metadata = {'format_data':format_metadata, 'metadata.db':dbkey, 'total':total} + extra_files = {} + metadata = {'format_data':format_metadata, 'metadata.db':dbkey, 'total':total, 'extra_files': extra_files} if has_fts: metadata['full-text-search.db'] = ftsdbkey for i, book_id in enumerate(book_ids): @@ -2935,11 +2936,11 @@ class Cache: return if progress is not None: progress(self._field_for('title', book_id), i + poff, total) - format_metadata[book_id] = {} + format_metadata[book_id] = fm = {} for fmt in self._formats(book_id): mdata = self.format_metadata(book_id, fmt) key = f'{key_prefix}:{book_id}:{fmt}' - format_metadata[book_id][fmt] = key + fm[fmt] = key with exporter.start_file(key, mtime=mdata.get('mtime')) as dest: self._copy_format_to(book_id, fmt, dest, report_file_size=dest.ensure_space) cover_key = '{}:{}:{}'.format(key_prefix, book_id, '.cover') @@ -2947,7 +2948,15 @@ class Cache: if not self.copy_cover_to(book_id, dest, report_file_size=dest.ensure_space): dest.discard() else: - format_metadata[book_id]['.cover'] = cover_key + fm['.cover'] = cover_key + bp = self.field_for('path', book_id) + extra_files[book_id] = ef = {} + if bp: + for (relpath, fobj, mtime) in self.backend.iter_extra_files(book_id, bp, self.fields['formats']): + key = f'{key_prefix}:{book_id}:.|{relpath}' + with exporter.start_file(key, mtime=mtime) as dest: + shutil.copyfileobj(fobj, dest) + ef[relpath] = key exporter.set_metadata(library_key, metadata) if progress is not None: progress(_('Completed'), total, total) @@ -3067,6 +3076,7 @@ def import_library(library_key, importer, library_path, progress=None, abort=Non cache = Cache(DB(library_path, load_user_formatter_functions=False)) cache.init() format_data = {int(book_id):data for book_id, data in iteritems(metadata['format_data'])} + extra_files = {int(book_id):data for book_id, data in metadata.get('extra_files', {}).items()} for i, (book_id, fmt_key_map) in enumerate(iteritems(format_data)): if abort is not None and abort.is_set(): return @@ -3084,6 +3094,10 @@ def import_library(library_key, importer, library_path, progress=None, abort=Non size, fname = cache._do_add_format(book_id, fmt, stream, mtime=stream.mtime) cache.fields['formats'].table.update_fmt(book_id, fmt, fname, size, cache.backend) stream.close() + for relpath, efkey in extra_files.get(book_id, {}).items(): + stream = importer.start_file(efkey, _('{0} for {1}').format(relpath, title)) + path = cache._field_for('path', book_id).replace('/', os.sep) + cache.backend.add_extra_file(relpath, stream, path) cache.dump_metadata({book_id}) if progress is not None: progress(_('Completed'), total, total) diff --git a/src/calibre/db/tests/filesystem.py b/src/calibre/db/tests/filesystem.py index 0aee02d959..545c068e26 100644 --- a/src/calibre/db/tests/filesystem.py +++ b/src/calibre/db/tests/filesystem.py @@ -215,6 +215,12 @@ class FilesystemTest(BaseTest): from calibre.db.cache import import_library from calibre.utils.exim import Exporter, Importer cache = self.init_cache() + bookdir = os.path.dirname(cache.format_abspath(1, '__COVER_INTERNAL__')) + with open(os.path.join(bookdir, 'exf'), 'w') as f: + f.write('exf') + os.mkdir(os.path.join(bookdir, 'sub')) + with open(os.path.join(bookdir, 'sub', 'recurse'), 'w') as f: + f.write('recurse') for part_size in (1 << 30, 100, 1): with TemporaryDirectory('export_lib') as tdir, TemporaryDirectory('import_lib') as idir: exporter = Exporter(tdir, part_size=part_size) @@ -228,6 +234,9 @@ class FilesystemTest(BaseTest): for fmt in cache.formats(book_id): self.assertEqual(cache.format(book_id, fmt), ic.format(book_id, fmt)) self.assertEqual(cache.format_metadata(book_id, fmt)['mtime'], cache.format_metadata(book_id, fmt)['mtime']) + bookdir = os.path.dirname(ic.format_abspath(1, '__COVER_INTERNAL__')) + self.assertEqual('exf', open(os.path.join(bookdir, 'exf')).read()) + self.assertEqual('recurse', open(os.path.join(bookdir, 'sub', 'recurse')).read()) cache.add_format(1, 'TXT', BytesIO(b'testing exim')) cache.fts_indexing_sleep_time = 0.001 cache.enable_fts()