From b45ca52f0f99aaa6da040e594ce7ba3cc7145e02 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 19 Apr 2023 17:19:33 +0530 Subject: [PATCH] calibredb export: Support exporting extra data files --- src/calibre/db/backend.py | 11 ++++++++++- src/calibre/db/cache.py | 16 +++++++++++----- src/calibre/db/cli/cmd_export.py | 28 +++++++++++++++++++++++++--- src/calibre/library/save_to_disk.py | 22 +++++++++++++--------- 4 files changed, 59 insertions(+), 18 deletions(-) diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index 56ef791c74..535027cc8f 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -42,7 +42,7 @@ from calibre.utils.date import EPOCH, parse_date, utcfromtimestamp, utcnow from calibre.utils.filenames import ( WindowsAtomicFolderMove, ascii_filename, atomic_rename, copyfile_using_links, copytree_using_links, hardlink_file, is_case_sensitive, is_fat_filesystem, - remove_dir_if_empty, samefile, + make_long_path_useable, remove_dir_if_empty, samefile, ) from calibre.utils.formatter_functions import ( compile_user_template_functions, formatter_functions, load_user_template_functions, @@ -1889,6 +1889,15 @@ class DB: os.makedirs(tpath) update_paths_in_db() + def copy_extra_file_to(self, book_id, book_path, relpath, stream_or_path): + full_book_path = os.path.abspath(os.path.join(self.library_path, book_path)) + src_path = make_long_path_useable(os.path.join(full_book_path, relpath)) + if isinstance(stream_or_path, str): + shutil.copy2(src_path, make_long_path_useable(stream_or_path)) + else: + with open(src_path, 'rb') as src: + shutil.copyfileobj(src, stream_or_path) + def iter_extra_files(self, book_id, book_path, formats_field, yield_paths=False, pattern=''): known_files = {COVER_FILE_NAME, METADATA_FILE_NAME} for fmt in formats_field.for_book(book_id, default_value=()): diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 02ccb7a61d..6081fcd5ca 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -3058,15 +3058,21 @@ class Cache: @read_api def list_extra_files_matching(self, book_id, pattern=''): - ' List extra data files matching the specified patter. Empty pattern matches all. Recursive globbing with ** is supported. ' - path = self._field_for('path', book_id).replace('/', os.sep) + ' List extra data files matching the specified pattern. Empty pattern matches all. Recursive globbing with ** is supported. ' + path = self._field_for('path', book_id) ans = {} if path: - for (relpath, path, mtime) in self.backend.iter_extra_files( - book_id, path, self.fields['formats'], yield_paths=True, pattern=pattern): - ans[relpath] = path + book_path = path.replace('/', os.sep) + for (relpath, file_path, mtime) in self.backend.iter_extra_files( + book_id, book_path, self.fields['formats'], yield_paths=True, pattern=pattern): + ans[relpath] = file_path return ans + @read_api + def copy_extra_file_to(self, book_id, relpath, stream_or_path): + path = self._field_for('path', book_id).replace('/', os.sep) + self.backend.copy_extra_file_to(book_id, path, relpath, stream_or_path) + def import_library(library_key, importer, library_path, progress=None, abort=None): from calibre.db.backend import DB diff --git a/src/calibre/db/cli/cmd_export.py b/src/calibre/db/cli/cmd_export.py index 38eeaf8f94..20032dd5e4 100644 --- a/src/calibre/db/cli/cmd_export.py +++ b/src/calibre/db/cli/cmd_export.py @@ -21,9 +21,13 @@ def implementation(db, notify_changes, action, *args): return db.all_book_ids() if action == 'setup': book_id, formats = args + if not db.has_id(book_id): + raise KeyError(f'No book with id {book_id} present') mi = db.get_metadata(book_id) plugboards = db.pref('plugboards', {}) formats = get_formats(db.formats(book_id), formats) + extra_files_for_export = tuple(db.list_extra_files_matching(book_id, 'data/**/*')) + plugboards['extra_files_for_export'] = extra_files_for_export return mi, plugboards, formats, db.library_id, db.pref( 'user_template_functions', [] ) @@ -34,6 +38,14 @@ def implementation(db, notify_changes, action, *args): if is_remote: return db.format(book_id, fmt) db.copy_format_to(book_id, fmt, dest) + if action == 'extra_file': + book_id, relpath, dest = args + if is_remote: + from io import BytesIO + output = BytesIO() + db.copy_extra_file_to(book_id, relpath, output) + return output.getvalue() + db.copy_extra_file_to(book_id, relpath, dest) def option_parser(get_parser, args): @@ -44,7 +56,8 @@ def option_parser(get_parser, args): Export the books specified by ids (a comma separated list) to the filesystem. The export operation saves all formats of the book, its cover and metadata (in -an opf file). You can get id numbers from the search command. +an OPF file). Any extra data files associated with the book are also saved. +You can get id numbers from the search command. ''' ) ) @@ -74,7 +87,7 @@ an opf file). You can get id numbers from the search command. help=_('Report progress') ) c = config() - for pref in ['asciiize', 'update_metadata', 'write_opf', 'save_cover']: + for pref in ['asciiize', 'update_metadata', 'write_opf', 'save_cover', 'save_extra_files']: opt = c.get_option(pref) switch = '--dont-' + pref.replace('_', '-') parser.add_option( @@ -118,15 +131,24 @@ class DBProxy: with open(path, 'wb') as f: f.write(fdata) + def copy_extra_file_to(self, book_id, relpath, path): + fdata = self.dbctx.run('export', 'extra_file', book_id, relpath, path) + if self.dbctx.is_remote: + if fdata is None: + raise FileNotFoundError(relpath) + with open(path, 'wb') as f: + f.write(fdata) + def export(opts, dbctx, book_id, dest, dbproxy, length, first): mi, plugboards, formats, library_id, template_funcs = dbctx.run( 'export', 'setup', book_id, opts.formats ) + extra_files = plugboards.pop('extra_files_for_export', ()) if dbctx.is_remote and first: load_user_template_functions(library_id, template_funcs) return do_save_book_to_disk( - dbproxy, book_id, mi, plugboards, formats, dest, opts, length + dbproxy, book_id, mi, plugboards, formats, dest, opts, length, extra_files ) diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py index ccf88d5e8b..8c6c67cf14 100644 --- a/src/calibre/library/save_to_disk.py +++ b/src/calibre/library/save_to_disk.py @@ -5,7 +5,6 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import errno import os import re import traceback @@ -17,7 +16,9 @@ from calibre.db.lazy import FormatsList from calibre.ebooks.metadata import fmt_sidx, title_sort from calibre.utils.config import Config, StringConfig, tweaks from calibre.utils.date import as_local_time, is_date_undefined -from calibre.utils.filenames import ascii_filename, shorten_components_to +from calibre.utils.filenames import ( + ascii_filename, make_long_path_useable, shorten_components_to, +) from calibre.utils.formatter import TemplateFormatter from calibre.utils.localization import _ @@ -322,7 +323,7 @@ def update_metadata(mi, fmt, stream, plugboards, cdata, error_report=None, plugb def do_save_book_to_disk(db, book_id, mi, plugboards, - formats, root, opts, length): + formats, root, opts, length, extra_files=()): originals = mi.cover, mi.pubdate, mi.timestamp formats_written = False try: @@ -335,12 +336,7 @@ def do_save_book_to_disk(db, book_id, mi, plugboards, base_path = os.path.join(root, *components) base_name = os.path.basename(base_path) dirpath = os.path.dirname(base_path) - try: - os.makedirs(dirpath) - except OSError as err: - if err.errno != errno.EEXIST: - raise - + os.makedirs(dirpath, exist_ok=True) cdata = None if opts.save_cover: cdata = db.cover(book_id) @@ -357,6 +353,14 @@ def do_save_book_to_disk(db, book_id, mi, plugboards, finally: mi.cover, mi.pubdate, mi.timestamp = originals + if extra_files and opts.save_extra_files: + for relpath in extra_files: + data_dest_path = os.path.abspath(os.path.join(dirpath, relpath)) + try: + db.copy_extra_file_to(book_id, relpath, data_dest_path) + except FileNotFoundError: + os.makedirs(make_long_path_useable(os.path.dirname(data_dest_path))) + db.copy_extra_file_to(book_id, relpath, data_dest_path) if not formats: return not formats_written, book_id, mi.title