calibredb export: Support exporting extra data files

src/calibre/db/backend.py

@@ -42,7 +42,7 @@ from calibre.utils.date import EPOCH, parse_date, utcfromtimestamp, utcnow
 from calibre.utils.filenames import (
     WindowsAtomicFolderMove, ascii_filename, atomic_rename, copyfile_using_links,
     copytree_using_links, hardlink_file, is_case_sensitive, is_fat_filesystem,
-    remove_dir_if_empty, samefile,
+    make_long_path_useable, remove_dir_if_empty, samefile,
 )
 from calibre.utils.formatter_functions import (
     compile_user_template_functions, formatter_functions, load_user_template_functions,
@@ -1889,6 +1889,15 @@ class DB:
             os.makedirs(tpath)
         update_paths_in_db()
 
+    def copy_extra_file_to(self, book_id, book_path, relpath, stream_or_path):
+        full_book_path = os.path.abspath(os.path.join(self.library_path, book_path))
+        src_path = make_long_path_useable(os.path.join(full_book_path, relpath))
+        if isinstance(stream_or_path, str):
+            shutil.copy2(src_path, make_long_path_useable(stream_or_path))
+        else:
+            with open(src_path, 'rb') as src:
+                shutil.copyfileobj(src, stream_or_path)
+
     def iter_extra_files(self, book_id, book_path, formats_field, yield_paths=False, pattern=''):
         known_files = {COVER_FILE_NAME, METADATA_FILE_NAME}
         for fmt in formats_field.for_book(book_id, default_value=()):

src/calibre/db/cache.py

@@ -3058,15 +3058,21 @@ class Cache:
 
     @read_api
     def list_extra_files_matching(self, book_id, pattern=''):
-        ' List extra data files matching the specified patter. Empty pattern matches all. Recursive globbing with ** is supported. '
-        path = self._field_for('path', book_id).replace('/', os.sep)
+        ' List extra data files matching the specified pattern. Empty pattern matches all. Recursive globbing with ** is supported. '
+        path = self._field_for('path', book_id)
         ans = {}
         if path:
-            for (relpath, path, mtime) in self.backend.iter_extra_files(
-                    book_id, path, self.fields['formats'], yield_paths=True, pattern=pattern):
-                ans[relpath] = path
+            book_path = path.replace('/', os.sep)
+            for (relpath, file_path, mtime) in self.backend.iter_extra_files(
+                    book_id, book_path, self.fields['formats'], yield_paths=True, pattern=pattern):
+                ans[relpath] = file_path
         return ans
 
+    @read_api
+    def copy_extra_file_to(self, book_id, relpath, stream_or_path):
+        path = self._field_for('path', book_id).replace('/', os.sep)
+        self.backend.copy_extra_file_to(book_id, path, relpath, stream_or_path)
+
 
 def import_library(library_key, importer, library_path, progress=None, abort=None):
     from calibre.db.backend import DB

src/calibre/db/cli/cmd_export.py

@@ -21,9 +21,13 @@ def implementation(db, notify_changes, action, *args):
         return db.all_book_ids()
     if action == 'setup':
         book_id, formats = args
+        if not db.has_id(book_id):
+            raise KeyError(f'No book with id {book_id} present')
         mi = db.get_metadata(book_id)
         plugboards = db.pref('plugboards', {})
         formats = get_formats(db.formats(book_id), formats)
+        extra_files_for_export = tuple(db.list_extra_files_matching(book_id, 'data/**/*'))
+        plugboards['extra_files_for_export'] = extra_files_for_export
         return mi, plugboards, formats, db.library_id, db.pref(
             'user_template_functions', []
         )
@@ -34,6 +38,14 @@ def implementation(db, notify_changes, action, *args):
         if is_remote:
             return db.format(book_id, fmt)
         db.copy_format_to(book_id, fmt, dest)
+    if action == 'extra_file':
+        book_id, relpath, dest = args
+        if is_remote:
+            from io import BytesIO
+            output = BytesIO()
+            db.copy_extra_file_to(book_id, relpath, output)
+            return output.getvalue()
+        db.copy_extra_file_to(book_id, relpath, dest)
 
 
 def option_parser(get_parser, args):
@@ -44,7 +56,8 @@ def option_parser(get_parser, args):
 
 Export the books specified by ids (a comma separated list) to the filesystem.
 The export operation saves all formats of the book, its cover and metadata (in
-an opf file). You can get id numbers from the search command.
+an OPF file). Any extra data files associated with the book are also saved.
+You can get id numbers from the search command.
 '''
         )
     )
@@ -74,7 +87,7 @@ an opf file). You can get id numbers from the search command.
         help=_('Report progress')
     )
     c = config()
-    for pref in ['asciiize', 'update_metadata', 'write_opf', 'save_cover']:
+    for pref in ['asciiize', 'update_metadata', 'write_opf', 'save_cover', 'save_extra_files']:
         opt = c.get_option(pref)
         switch = '--dont-' + pref.replace('_', '-')
         parser.add_option(
@@ -118,15 +131,24 @@ class DBProxy:
             with open(path, 'wb') as f:
                 f.write(fdata)
 
+    def copy_extra_file_to(self, book_id, relpath, path):
+        fdata = self.dbctx.run('export', 'extra_file', book_id, relpath, path)
+        if self.dbctx.is_remote:
+            if fdata is None:
+                raise FileNotFoundError(relpath)
+            with open(path, 'wb') as f:
+                f.write(fdata)
+
 
 def export(opts, dbctx, book_id, dest, dbproxy, length, first):
     mi, plugboards, formats, library_id, template_funcs = dbctx.run(
         'export', 'setup', book_id, opts.formats
     )
+    extra_files = plugboards.pop('extra_files_for_export', ())
     if dbctx.is_remote and first:
         load_user_template_functions(library_id, template_funcs)
     return do_save_book_to_disk(
-        dbproxy, book_id, mi, plugboards, formats, dest, opts, length
+        dbproxy, book_id, mi, plugboards, formats, dest, opts, length, extra_files
     )
 
 

src/calibre/library/save_to_disk.py

@@ -5,7 +5,6 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import errno
 import os
 import re
 import traceback
@@ -17,7 +16,9 @@ from calibre.db.lazy import FormatsList
 from calibre.ebooks.metadata import fmt_sidx, title_sort
 from calibre.utils.config import Config, StringConfig, tweaks
 from calibre.utils.date import as_local_time, is_date_undefined
-from calibre.utils.filenames import ascii_filename, shorten_components_to
+from calibre.utils.filenames import (
+    ascii_filename, make_long_path_useable, shorten_components_to,
+)
 from calibre.utils.formatter import TemplateFormatter
 from calibre.utils.localization import _
 
@@ -322,7 +323,7 @@ def update_metadata(mi, fmt, stream, plugboards, cdata, error_report=None, plugb
 
 
 def do_save_book_to_disk(db, book_id, mi, plugboards,
-        formats, root, opts, length):
+        formats, root, opts, length, extra_files=()):
     originals = mi.cover, mi.pubdate, mi.timestamp
     formats_written = False
     try:
@@ -335,12 +336,7 @@ def do_save_book_to_disk(db, book_id, mi, plugboards,
         base_path = os.path.join(root, *components)
         base_name = os.path.basename(base_path)
         dirpath = os.path.dirname(base_path)
-        try:
-            os.makedirs(dirpath)
-        except OSError as err:
-            if err.errno != errno.EEXIST:
-                raise
-
+        os.makedirs(dirpath, exist_ok=True)
         cdata = None
         if opts.save_cover:
             cdata = db.cover(book_id)
@@ -357,6 +353,14 @@ def do_save_book_to_disk(db, book_id, mi, plugboards,
     finally:
         mi.cover, mi.pubdate, mi.timestamp = originals
 
+    if extra_files and opts.save_extra_files:
+        for relpath in extra_files:
+            data_dest_path = os.path.abspath(os.path.join(dirpath, relpath))
+            try:
+                db.copy_extra_file_to(book_id, relpath, data_dest_path)
+            except FileNotFoundError:
+                os.makedirs(make_long_path_useable(os.path.dirname(data_dest_path)))
+                db.copy_extra_file_to(book_id, relpath, data_dest_path)
     if not formats:
         return not formats_written, book_id, mi.title