From abde717e29d7469f53837bde19d7f82af3e63316 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 19 Apr 2023 12:09:12 +0530 Subject: [PATCH] Save to disk now exports data files associated with the book --- src/calibre/db/backend.py | 50 +++++++------ src/calibre/db/cache.py | 12 ++++ src/calibre/db/tests/filesystem.py | 2 + src/calibre/gui2/preferences/save_template.py | 4 +- src/calibre/gui2/preferences/saving.py | 2 +- src/calibre/gui2/preferences/saving.ui | 71 ++++++++++--------- src/calibre/gui2/save.py | 37 +++++++--- src/calibre/library/save_to_disk.py | 2 + 8 files changed, 116 insertions(+), 64 deletions(-) diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index 96557722ef..56ef791c74 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -1889,7 +1889,7 @@ class DB: os.makedirs(tpath) update_paths_in_db() - def iter_extra_files(self, book_id, book_path, formats_field, yield_paths=False): + def iter_extra_files(self, book_id, book_path, formats_field, yield_paths=False, pattern=''): known_files = {COVER_FILE_NAME, METADATA_FILE_NAME} for fmt in formats_field.for_book(book_id, default_value=()): fname = formats_field.format_fname(book_id, fmt) @@ -1897,25 +1897,35 @@ class DB: if fpath: known_files.add(os.path.basename(fpath)) full_book_path = os.path.abspath(os.path.join(self.library_path, book_path)) - for dirpath, dirnames, filenames in os.walk(full_book_path): - for fname in filenames: - path = os.path.join(dirpath, fname) - if os.access(path, os.R_OK): - relpath = os.path.relpath(path, full_book_path) - relpath = relpath.replace(os.sep, '/') - if relpath not in known_files: - mtime = os.path.getmtime(path) - if yield_paths: - yield relpath, path, mtime - else: - try: - src = open(path, 'rb') - except OSError: - if iswindows: - time.sleep(1) - src = open(path, 'rb') - with src: - yield relpath, src, mtime + if pattern: + from pathlib import Path + def iterator(): + p = Path(full_book_path) + for x in p.glob(pattern): + yield str(x) + else: + def iterator(): + for dirpath, dirnames, filenames in os.walk(full_book_path): + for fname in filenames: + path = os.path.join(dirpath, fname) + yield path + for path in iterator(): + if os.access(path, os.R_OK): + relpath = os.path.relpath(path, full_book_path) + relpath = relpath.replace(os.sep, '/') + if relpath not in known_files: + mtime = os.path.getmtime(path) + if yield_paths: + yield relpath, path, mtime + else: + try: + src = open(path, 'rb') + except OSError: + if iswindows: + time.sleep(1) + src = open(path, 'rb') + with src: + yield relpath, src, mtime def add_extra_file(self, relpath, stream, book_path): dest = os.path.abspath(os.path.join(self.library_path, book_path, relpath)) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index f7776cd830..02ccb7a61d 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -3051,10 +3051,22 @@ class Cache: @write_api def add_extra_files(self, book_id, map_of_relpath_to_stream_or_path): + ' Add extra data files ' path = self._field_for('path', book_id).replace('/', os.sep) for relpath, stream_or_path in map_of_relpath_to_stream_or_path.items(): self.backend.add_extra_file(relpath, stream_or_path, path) + @read_api + def list_extra_files_matching(self, book_id, pattern=''): + ' List extra data files matching the specified patter. Empty pattern matches all. Recursive globbing with ** is supported. ' + path = self._field_for('path', book_id).replace('/', os.sep) + ans = {} + if path: + for (relpath, path, mtime) in self.backend.iter_extra_files( + book_id, path, self.fields['formats'], yield_paths=True, pattern=pattern): + ans[relpath] = path + return ans + def import_library(library_key, importer, library_path, progress=None, abort=None): from calibre.db.backend import DB diff --git a/src/calibre/db/tests/filesystem.py b/src/calibre/db/tests/filesystem.py index 545c068e26..724994c053 100644 --- a/src/calibre/db/tests/filesystem.py +++ b/src/calibre/db/tests/filesystem.py @@ -221,6 +221,8 @@ class FilesystemTest(BaseTest): os.mkdir(os.path.join(bookdir, 'sub')) with open(os.path.join(bookdir, 'sub', 'recurse'), 'w') as f: f.write('recurse') + self.assertEqual(set(cache.list_extra_files_matching(1, 'sub/**/*')), {'sub/recurse'}) + self.assertEqual(set(cache.list_extra_files_matching(1, '')), {'exf', 'sub/recurse'}) for part_size in (1 << 30, 100, 1): with TemporaryDirectory('export_lib') as tdir, TemporaryDirectory('import_lib') as idir: exporter = Exporter(tdir, part_size=part_size) diff --git a/src/calibre/gui2/preferences/save_template.py b/src/calibre/gui2/preferences/save_template.py index 82cf951261..47d4510d53 100644 --- a/src/calibre/gui2/preferences/save_template.py +++ b/src/calibre/gui2/preferences/save_template.py @@ -18,8 +18,8 @@ class SaveTemplate(QWidget, Ui_Form): changed_signal = pyqtSignal() - def __init__(self, *args): - QWidget.__init__(self, *args) + def __init__(self, parent=None): + QWidget.__init__(self, parent) Ui_Form.__init__(self) self.setupUi(self) self.orig_help_text = self.help_label.text() diff --git a/src/calibre/gui2/preferences/saving.py b/src/calibre/gui2/preferences/saving.py index 647787ddeb..7325dafd68 100644 --- a/src/calibre/gui2/preferences/saving.py +++ b/src/calibre/gui2/preferences/saving.py @@ -23,7 +23,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): r = self.register for x in ('asciiize', 'update_metadata', 'save_cover', 'write_opf', - 'replace_whitespace', 'to_lowercase', 'formats', 'timefmt'): + 'replace_whitespace', 'to_lowercase', 'formats', 'timefmt', 'save_extra_files'): r(x, self.proxy) r('show_files_after_save', gprefs) diff --git a/src/calibre/gui2/preferences/saving.ui b/src/calibre/gui2/preferences/saving.ui index a1e59629d9..4668072a4a 100644 --- a/src/calibre/gui2/preferences/saving.ui +++ b/src/calibre/gui2/preferences/saving.ui @@ -14,20 +14,40 @@ Form + + + + + + + + + + Save metadata in a separate &OPF file + + + + + + + Save &data files as well + + + - Here you can control how calibre will save your books when you click the "Save to disk" button: + Here you can control how calibre will save your books when you click the "Save to disk" button: true - - + + - Save &cover separately + Change paths to &lowercase @@ -38,19 +58,8 @@ - - - - Update &metadata in saved copies - - - - - - - Change paths to &lowercase - - + + @@ -62,8 +71,12 @@ - - + + + + Update &metadata in saved copies + + @@ -75,11 +88,12 @@ - - - - - + + + + Save &cover separately + + @@ -88,14 +102,7 @@ - - - - Save metadata in a separate &OPF file - - - - + &Show files in the file browser after saving to disk diff --git a/src/calibre/gui2/save.py b/src/calibre/gui2/save.py index 9c49a897df..9976a42a5e 100644 --- a/src/calibre/gui2/save.py +++ b/src/calibre/gui2/save.py @@ -4,23 +4,29 @@ __license__ = 'GPL v3' __copyright__ = '2014, Kovid Goyal ' -import traceback, errno, os, time, shutil -from collections import namedtuple, defaultdict - +import errno +import os +import shutil +import time +import traceback +from collections import defaultdict, namedtuple from qt.core import QObject, Qt, pyqtSignal -from calibre import prints, force_unicode +from calibre import force_unicode, prints from calibre.constants import DEBUG from calibre.customize.ui import can_set_metadata from calibre.db.errors import NoSuchFormat from calibre.ebooks.metadata import authors_to_string from calibre.ebooks.metadata.opf2 import metadata_to_opf -from calibre.ptempfile import PersistentTemporaryDirectory, SpooledTemporaryFile -from calibre.gui2 import error_dialog, warning_dialog, gprefs, open_local_file +from calibre.gui2 import error_dialog, gprefs, open_local_file, warning_dialog from calibre.gui2.dialogs.progress import ProgressDialog +from calibre.library.save_to_disk import ( + find_plugboard, get_path_components, plugboard_save_to_disk_value, sanitize_args, +) +from calibre.ptempfile import PersistentTemporaryDirectory, SpooledTemporaryFile +from calibre.utils.filenames import make_long_path_useable from calibre.utils.formatter_functions import load_user_template_functions -from calibre.utils.ipc.pool import Pool, Failure -from calibre.library.save_to_disk import sanitize_args, get_path_components, find_plugboard, plugboard_save_to_disk_value +from calibre.utils.ipc.pool import Failure, Pool from polyglot.builtins import iteritems, itervalues from polyglot.queue import Empty @@ -205,7 +211,10 @@ class Saver(QObject): self.errors[book_id].append(('critical', _('Requested formats not available'))) return - if not fmts and not self.opts.write_opf and not self.opts.save_cover: + extra_files = {} + if self.opts.save_extra_files: + extra_files = self.db.new_api.list_extra_files_matching(int(book_id), 'data/**/*') + if not fmts and not self.opts.write_opf and not self.opts.save_cover and not extra_files: return # On windows python incorrectly raises an access denied exception @@ -252,6 +261,16 @@ class Saver(QObject): mi.cover, mi.cover_data = None, (None, None) if self.opts.update_metadata: d['fmts'] = [] + if extra_files: + for relpath, src_path in extra_files.items(): + src_path = make_long_path_useable(src_path) + if os.access(src_path, os.R_OK): + dest = make_long_path_useable(os.path.abspath(os.path.join(base_dir, relpath))) + try: + shutil.copy2(src_path, dest) + except FileNotFoundError: + os.makedirs(os.path.dirname(dest), exist_ok=True) + shutil.copy2(src_path, dest) for fmt in fmts: try: fmtpath = self.write_fmt(book_id, fmt, base_path) diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py index 28bd91440c..ccf88d5e8b 100644 --- a/src/calibre/library/save_to_disk.py +++ b/src/calibre/library/save_to_disk.py @@ -125,6 +125,8 @@ def config(defaults=None): x('single_dir', default=False, help=_('Save into a single folder, ignoring the template' ' folder structure')) + x('save_extra_files', default=True, help=_( + 'Save any data files associated with the book when saving the book')) return c