Save to disk now exports data files associated with the book

src/calibre/db/backend.py

@@ -1889,7 +1889,7 @@ class DB:
             os.makedirs(tpath)
         update_paths_in_db()
 
-    def iter_extra_files(self, book_id, book_path, formats_field, yield_paths=False):
+    def iter_extra_files(self, book_id, book_path, formats_field, yield_paths=False, pattern=''):
         known_files = {COVER_FILE_NAME, METADATA_FILE_NAME}
         for fmt in formats_field.for_book(book_id, default_value=()):
             fname = formats_field.format_fname(book_id, fmt)
@@ -1897,25 +1897,35 @@ class DB:
             if fpath:
                 known_files.add(os.path.basename(fpath))
         full_book_path = os.path.abspath(os.path.join(self.library_path, book_path))
-        for dirpath, dirnames, filenames in os.walk(full_book_path):
+        if pattern:
-            for fname in filenames:
+            from pathlib import Path
-                path = os.path.join(dirpath, fname)
+            def iterator():
-                if os.access(path, os.R_OK):
+                p = Path(full_book_path)
-                    relpath = os.path.relpath(path, full_book_path)
+                for x in p.glob(pattern):
-                    relpath = relpath.replace(os.sep, '/')
+                    yield str(x)
-                    if relpath not in known_files:
+        else:
-                        mtime = os.path.getmtime(path)
+            def iterator():
-                        if yield_paths:
+                for dirpath, dirnames, filenames in os.walk(full_book_path):
-                            yield relpath, path, mtime
+                    for fname in filenames:
-                        else:
+                        path = os.path.join(dirpath, fname)
-                            try:
+                        yield path
-                                src = open(path, 'rb')
+        for path in iterator():
-                            except OSError:
+            if os.access(path, os.R_OK):
-                                if iswindows:
+                relpath = os.path.relpath(path, full_book_path)
-                                    time.sleep(1)
+                relpath = relpath.replace(os.sep, '/')
-                                src = open(path, 'rb')
+                if relpath not in known_files:
-                            with src:
+                    mtime = os.path.getmtime(path)
-                                yield relpath, src, mtime
+                    if yield_paths:
+                        yield relpath, path, mtime
+                    else:
+                        try:
+                            src = open(path, 'rb')
+                        except OSError:
+                            if iswindows:
+                                time.sleep(1)
+                            src = open(path, 'rb')
+                        with src:
+                            yield relpath, src, mtime
 
     def add_extra_file(self, relpath, stream, book_path):
         dest = os.path.abspath(os.path.join(self.library_path, book_path, relpath))

src/calibre/db/cache.py

@@ -3051,10 +3051,22 @@ class Cache:
 
     @write_api
     def add_extra_files(self, book_id, map_of_relpath_to_stream_or_path):
+        ' Add extra data files '
         path = self._field_for('path', book_id).replace('/', os.sep)
         for relpath, stream_or_path in map_of_relpath_to_stream_or_path.items():
             self.backend.add_extra_file(relpath, stream_or_path, path)
 
+    @read_api
+    def list_extra_files_matching(self, book_id, pattern=''):
+        ' List extra data files matching the specified patter. Empty pattern matches all. Recursive globbing with ** is supported. '
+        path = self._field_for('path', book_id).replace('/', os.sep)
+        ans = {}
+        if path:
+            for (relpath, path, mtime) in self.backend.iter_extra_files(
+                    book_id, path, self.fields['formats'], yield_paths=True, pattern=pattern):
+                ans[relpath] = path
+        return ans
+
 
 def import_library(library_key, importer, library_path, progress=None, abort=None):
     from calibre.db.backend import DB

src/calibre/db/tests/filesystem.py

@@ -221,6 +221,8 @@ class FilesystemTest(BaseTest):
         os.mkdir(os.path.join(bookdir, 'sub'))
         with open(os.path.join(bookdir, 'sub', 'recurse'), 'w') as f:
             f.write('recurse')
+        self.assertEqual(set(cache.list_extra_files_matching(1, 'sub/**/*')), {'sub/recurse'})
+        self.assertEqual(set(cache.list_extra_files_matching(1, '')), {'exf', 'sub/recurse'})
         for part_size in (1 << 30, 100, 1):
             with TemporaryDirectory('export_lib') as tdir, TemporaryDirectory('import_lib') as idir:
                 exporter = Exporter(tdir, part_size=part_size)

src/calibre/gui2/preferences/save_template.py

@@ -18,8 +18,8 @@ class SaveTemplate(QWidget, Ui_Form):
 
     changed_signal = pyqtSignal()
 
-    def __init__(self, *args):
+    def __init__(self, parent=None):
-        QWidget.__init__(self, *args)
+        QWidget.__init__(self, parent)
         Ui_Form.__init__(self)
         self.setupUi(self)
         self.orig_help_text = self.help_label.text()

src/calibre/gui2/preferences/saving.py

@@ -23,7 +23,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         r = self.register
 
         for x in ('asciiize', 'update_metadata', 'save_cover', 'write_opf',
-                'replace_whitespace', 'to_lowercase', 'formats', 'timefmt'):
+                'replace_whitespace', 'to_lowercase', 'formats', 'timefmt', 'save_extra_files'):
             r(x, self.proxy)
         r('show_files_after_save', gprefs)
 

src/calibre/gui2/preferences/saving.ui

@@ -14,20 +14,40 @@
    <string>Form</string>
   </property>
   <layout class="QGridLayout" name="gridLayout">
+   <item row="6" column="1">
+    <widget class="QLineEdit" name="opt_timefmt"/>
+   </item>
+   <item row="8" column="0" colspan="2">
+    <widget class="SaveTemplate" name="save_template" native="true"/>
+   </item>
+   <item row="3" column="0">
+    <widget class="QCheckBox" name="opt_write_opf">
+     <property name="text">
+      <string>Save metadata in a separate &amp;OPF file</string>
+     </property>
+    </widget>
+   </item>
+   <item row="5" column="0">
+    <widget class="QCheckBox" name="opt_save_extra_files">
+     <property name="text">
+      <string>Save &amp;data files as well</string>
+     </property>
+    </widget>
+   </item>
    <item row="0" column="0" colspan="2">
     <widget class="QLabel" name="label">
      <property name="text">
-      <string>Here you can control how calibre will save your books when you click the "Save to disk" button:</string>
+      <string>Here you can control how calibre will save your books when you click the &quot;Save to disk&quot; button:</string>
      </property>
      <property name="wordWrap">
       <bool>true</bool>
      </property>
     </widget>
    </item>
-   <item row="1" column="0">
+   <item row="2" column="1">
-    <widget class="QCheckBox" name="opt_save_cover">
+    <widget class="QCheckBox" name="opt_to_lowercase">
      <property name="text">
-      <string>Save &amp;cover separately</string>
+      <string>Change paths to &amp;lowercase</string>
      </property>
     </widget>
    </item>
@@ -38,19 +58,8 @@
      </property>
     </widget>
    </item>
-   <item row="2" column="0">
+   <item row="7" column="1">
-    <widget class="QCheckBox" name="opt_update_metadata">
+    <widget class="QLineEdit" name="opt_formats"/>
-     <property name="text">
-      <string>Update &amp;metadata in saved copies</string>
-     </property>
-    </widget>
-   </item>
-   <item row="2" column="1">
-    <widget class="QCheckBox" name="opt_to_lowercase">
-     <property name="text">
-      <string>Change paths to &amp;lowercase</string>
-     </property>
-    </widget>
    </item>
    <item row="6" column="0">
     <widget class="QLabel" name="label_2">
@@ -62,8 +71,12 @@
      </property>
     </widget>
    </item>
-   <item row="6" column="1">
+   <item row="2" column="0">
-    <widget class="QLineEdit" name="opt_timefmt"/>
+    <widget class="QCheckBox" name="opt_update_metadata">
+     <property name="text">
+      <string>Update &amp;metadata in saved copies</string>
+     </property>
+    </widget>
    </item>
    <item row="7" column="0">
     <widget class="QLabel" name="label_3">
@@ -75,11 +88,12 @@
      </property>
     </widget>
    </item>
-   <item row="7" column="1">
+   <item row="1" column="0">
-    <widget class="QLineEdit" name="opt_formats"/>
+    <widget class="QCheckBox" name="opt_save_cover">
-   </item>
+     <property name="text">
-   <item row="8" column="0" colspan="2">
+      <string>Save &amp;cover separately</string>
-    <widget class="SaveTemplate" name="save_template" native="true"/>
+     </property>
+    </widget>
    </item>
    <item row="3" column="1">
     <widget class="QCheckBox" name="opt_asciiize">
@@ -88,14 +102,7 @@
      </property>
     </widget>
    </item>
-   <item row="3" column="0">
+   <item row="5" column="1">
-    <widget class="QCheckBox" name="opt_write_opf">
-     <property name="text">
-      <string>Save metadata in a separate &amp;OPF file</string>
-     </property>
-    </widget>
-   </item>
-   <item row="4" column="0" colspan="2">
     <widget class="QCheckBox" name="opt_show_files_after_save">
      <property name="text">
       <string>&amp;Show files in the file browser after saving to disk</string>

src/calibre/gui2/save.py

@@ -4,23 +4,29 @@
 __license__ = 'GPL v3'
 __copyright__ = '2014, Kovid Goyal <kovid at kovidgoyal.net>'
 
-import traceback, errno, os, time, shutil
+import errno
-from collections import namedtuple, defaultdict
+import os
-
+import shutil
+import time
+import traceback
+from collections import defaultdict, namedtuple
 from qt.core import QObject, Qt, pyqtSignal
 
-from calibre import prints, force_unicode
+from calibre import force_unicode, prints
 from calibre.constants import DEBUG
 from calibre.customize.ui import can_set_metadata
 from calibre.db.errors import NoSuchFormat
 from calibre.ebooks.metadata import authors_to_string
 from calibre.ebooks.metadata.opf2 import metadata_to_opf
-from calibre.ptempfile import PersistentTemporaryDirectory, SpooledTemporaryFile
+from calibre.gui2 import error_dialog, gprefs, open_local_file, warning_dialog
-from calibre.gui2 import error_dialog, warning_dialog, gprefs, open_local_file
 from calibre.gui2.dialogs.progress import ProgressDialog
+from calibre.library.save_to_disk import (
+    find_plugboard, get_path_components, plugboard_save_to_disk_value, sanitize_args,
+)
+from calibre.ptempfile import PersistentTemporaryDirectory, SpooledTemporaryFile
+from calibre.utils.filenames import make_long_path_useable
 from calibre.utils.formatter_functions import load_user_template_functions
-from calibre.utils.ipc.pool import Pool, Failure
+from calibre.utils.ipc.pool import Failure, Pool
-from calibre.library.save_to_disk import sanitize_args, get_path_components, find_plugboard, plugboard_save_to_disk_value
 from polyglot.builtins import iteritems, itervalues
 from polyglot.queue import Empty
 
@@ -205,7 +211,10 @@ class Saver(QObject):
                 self.errors[book_id].append(('critical', _('Requested formats not available')))
                 return
 
-        if not fmts and not self.opts.write_opf and not self.opts.save_cover:
+        extra_files = {}
+        if self.opts.save_extra_files:
+            extra_files = self.db.new_api.list_extra_files_matching(int(book_id), 'data/**/*')
+        if not fmts and not self.opts.write_opf and not self.opts.save_cover and not extra_files:
             return
 
         # On windows python incorrectly raises an access denied exception
@@ -252,6 +261,16 @@ class Saver(QObject):
         mi.cover, mi.cover_data = None, (None, None)
         if self.opts.update_metadata:
             d['fmts'] = []
+        if extra_files:
+            for relpath, src_path in extra_files.items():
+                src_path = make_long_path_useable(src_path)
+                if os.access(src_path, os.R_OK):
+                    dest = make_long_path_useable(os.path.abspath(os.path.join(base_dir, relpath)))
+                    try:
+                        shutil.copy2(src_path, dest)
+                    except FileNotFoundError:
+                        os.makedirs(os.path.dirname(dest), exist_ok=True)
+                        shutil.copy2(src_path, dest)
         for fmt in fmts:
             try:
                 fmtpath = self.write_fmt(book_id, fmt, base_path)

src/calibre/library/save_to_disk.py

@@ -125,6 +125,8 @@ def config(defaults=None):
     x('single_dir', default=False,
             help=_('Save into a single folder, ignoring the template'
                 ' folder structure'))
+    x('save_extra_files', default=True, help=_(
+        'Save any data files associated with the book when saving the book'))
     return c