calibredb export: Support exporting extra data files

This commit is contained in:
Kovid Goyal 2023-04-19 17:19:33 +05:30
parent 4982620da4
commit b45ca52f0f
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
4 changed files with 59 additions and 18 deletions

View File

@ -42,7 +42,7 @@ from calibre.utils.date import EPOCH, parse_date, utcfromtimestamp, utcnow
from calibre.utils.filenames import ( from calibre.utils.filenames import (
WindowsAtomicFolderMove, ascii_filename, atomic_rename, copyfile_using_links, WindowsAtomicFolderMove, ascii_filename, atomic_rename, copyfile_using_links,
copytree_using_links, hardlink_file, is_case_sensitive, is_fat_filesystem, copytree_using_links, hardlink_file, is_case_sensitive, is_fat_filesystem,
remove_dir_if_empty, samefile, make_long_path_useable, remove_dir_if_empty, samefile,
) )
from calibre.utils.formatter_functions import ( from calibre.utils.formatter_functions import (
compile_user_template_functions, formatter_functions, load_user_template_functions, compile_user_template_functions, formatter_functions, load_user_template_functions,
@ -1889,6 +1889,15 @@ class DB:
os.makedirs(tpath) os.makedirs(tpath)
update_paths_in_db() update_paths_in_db()
def copy_extra_file_to(self, book_id, book_path, relpath, stream_or_path):
full_book_path = os.path.abspath(os.path.join(self.library_path, book_path))
src_path = make_long_path_useable(os.path.join(full_book_path, relpath))
if isinstance(stream_or_path, str):
shutil.copy2(src_path, make_long_path_useable(stream_or_path))
else:
with open(src_path, 'rb') as src:
shutil.copyfileobj(src, stream_or_path)
def iter_extra_files(self, book_id, book_path, formats_field, yield_paths=False, pattern=''): def iter_extra_files(self, book_id, book_path, formats_field, yield_paths=False, pattern=''):
known_files = {COVER_FILE_NAME, METADATA_FILE_NAME} known_files = {COVER_FILE_NAME, METADATA_FILE_NAME}
for fmt in formats_field.for_book(book_id, default_value=()): for fmt in formats_field.for_book(book_id, default_value=()):

View File

@ -3058,15 +3058,21 @@ class Cache:
@read_api @read_api
def list_extra_files_matching(self, book_id, pattern=''): def list_extra_files_matching(self, book_id, pattern=''):
' List extra data files matching the specified patter. Empty pattern matches all. Recursive globbing with ** is supported. ' ' List extra data files matching the specified pattern. Empty pattern matches all. Recursive globbing with ** is supported. '
path = self._field_for('path', book_id).replace('/', os.sep) path = self._field_for('path', book_id)
ans = {} ans = {}
if path: if path:
for (relpath, path, mtime) in self.backend.iter_extra_files( book_path = path.replace('/', os.sep)
book_id, path, self.fields['formats'], yield_paths=True, pattern=pattern): for (relpath, file_path, mtime) in self.backend.iter_extra_files(
ans[relpath] = path book_id, book_path, self.fields['formats'], yield_paths=True, pattern=pattern):
ans[relpath] = file_path
return ans return ans
@read_api
def copy_extra_file_to(self, book_id, relpath, stream_or_path):
path = self._field_for('path', book_id).replace('/', os.sep)
self.backend.copy_extra_file_to(book_id, path, relpath, stream_or_path)
def import_library(library_key, importer, library_path, progress=None, abort=None): def import_library(library_key, importer, library_path, progress=None, abort=None):
from calibre.db.backend import DB from calibre.db.backend import DB

View File

@ -21,9 +21,13 @@ def implementation(db, notify_changes, action, *args):
return db.all_book_ids() return db.all_book_ids()
if action == 'setup': if action == 'setup':
book_id, formats = args book_id, formats = args
if not db.has_id(book_id):
raise KeyError(f'No book with id {book_id} present')
mi = db.get_metadata(book_id) mi = db.get_metadata(book_id)
plugboards = db.pref('plugboards', {}) plugboards = db.pref('plugboards', {})
formats = get_formats(db.formats(book_id), formats) formats = get_formats(db.formats(book_id), formats)
extra_files_for_export = tuple(db.list_extra_files_matching(book_id, 'data/**/*'))
plugboards['extra_files_for_export'] = extra_files_for_export
return mi, plugboards, formats, db.library_id, db.pref( return mi, plugboards, formats, db.library_id, db.pref(
'user_template_functions', [] 'user_template_functions', []
) )
@ -34,6 +38,14 @@ def implementation(db, notify_changes, action, *args):
if is_remote: if is_remote:
return db.format(book_id, fmt) return db.format(book_id, fmt)
db.copy_format_to(book_id, fmt, dest) db.copy_format_to(book_id, fmt, dest)
if action == 'extra_file':
book_id, relpath, dest = args
if is_remote:
from io import BytesIO
output = BytesIO()
db.copy_extra_file_to(book_id, relpath, output)
return output.getvalue()
db.copy_extra_file_to(book_id, relpath, dest)
def option_parser(get_parser, args): def option_parser(get_parser, args):
@ -44,7 +56,8 @@ def option_parser(get_parser, args):
Export the books specified by ids (a comma separated list) to the filesystem. Export the books specified by ids (a comma separated list) to the filesystem.
The export operation saves all formats of the book, its cover and metadata (in The export operation saves all formats of the book, its cover and metadata (in
an opf file). You can get id numbers from the search command. an OPF file). Any extra data files associated with the book are also saved.
You can get id numbers from the search command.
''' '''
) )
) )
@ -74,7 +87,7 @@ an opf file). You can get id numbers from the search command.
help=_('Report progress') help=_('Report progress')
) )
c = config() c = config()
for pref in ['asciiize', 'update_metadata', 'write_opf', 'save_cover']: for pref in ['asciiize', 'update_metadata', 'write_opf', 'save_cover', 'save_extra_files']:
opt = c.get_option(pref) opt = c.get_option(pref)
switch = '--dont-' + pref.replace('_', '-') switch = '--dont-' + pref.replace('_', '-')
parser.add_option( parser.add_option(
@ -118,15 +131,24 @@ class DBProxy:
with open(path, 'wb') as f: with open(path, 'wb') as f:
f.write(fdata) f.write(fdata)
def copy_extra_file_to(self, book_id, relpath, path):
fdata = self.dbctx.run('export', 'extra_file', book_id, relpath, path)
if self.dbctx.is_remote:
if fdata is None:
raise FileNotFoundError(relpath)
with open(path, 'wb') as f:
f.write(fdata)
def export(opts, dbctx, book_id, dest, dbproxy, length, first): def export(opts, dbctx, book_id, dest, dbproxy, length, first):
mi, plugboards, formats, library_id, template_funcs = dbctx.run( mi, plugboards, formats, library_id, template_funcs = dbctx.run(
'export', 'setup', book_id, opts.formats 'export', 'setup', book_id, opts.formats
) )
extra_files = plugboards.pop('extra_files_for_export', ())
if dbctx.is_remote and first: if dbctx.is_remote and first:
load_user_template_functions(library_id, template_funcs) load_user_template_functions(library_id, template_funcs)
return do_save_book_to_disk( return do_save_book_to_disk(
dbproxy, book_id, mi, plugboards, formats, dest, opts, length dbproxy, book_id, mi, plugboards, formats, dest, opts, length, extra_files
) )

View File

@ -5,7 +5,6 @@ __license__ = 'GPL v3'
__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>' __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en' __docformat__ = 'restructuredtext en'
import errno
import os import os
import re import re
import traceback import traceback
@ -17,7 +16,9 @@ from calibre.db.lazy import FormatsList
from calibre.ebooks.metadata import fmt_sidx, title_sort from calibre.ebooks.metadata import fmt_sidx, title_sort
from calibre.utils.config import Config, StringConfig, tweaks from calibre.utils.config import Config, StringConfig, tweaks
from calibre.utils.date import as_local_time, is_date_undefined from calibre.utils.date import as_local_time, is_date_undefined
from calibre.utils.filenames import ascii_filename, shorten_components_to from calibre.utils.filenames import (
ascii_filename, make_long_path_useable, shorten_components_to,
)
from calibre.utils.formatter import TemplateFormatter from calibre.utils.formatter import TemplateFormatter
from calibre.utils.localization import _ from calibre.utils.localization import _
@ -322,7 +323,7 @@ def update_metadata(mi, fmt, stream, plugboards, cdata, error_report=None, plugb
def do_save_book_to_disk(db, book_id, mi, plugboards, def do_save_book_to_disk(db, book_id, mi, plugboards,
formats, root, opts, length): formats, root, opts, length, extra_files=()):
originals = mi.cover, mi.pubdate, mi.timestamp originals = mi.cover, mi.pubdate, mi.timestamp
formats_written = False formats_written = False
try: try:
@ -335,12 +336,7 @@ def do_save_book_to_disk(db, book_id, mi, plugboards,
base_path = os.path.join(root, *components) base_path = os.path.join(root, *components)
base_name = os.path.basename(base_path) base_name = os.path.basename(base_path)
dirpath = os.path.dirname(base_path) dirpath = os.path.dirname(base_path)
try: os.makedirs(dirpath, exist_ok=True)
os.makedirs(dirpath)
except OSError as err:
if err.errno != errno.EEXIST:
raise
cdata = None cdata = None
if opts.save_cover: if opts.save_cover:
cdata = db.cover(book_id) cdata = db.cover(book_id)
@ -357,6 +353,14 @@ def do_save_book_to_disk(db, book_id, mi, plugboards,
finally: finally:
mi.cover, mi.pubdate, mi.timestamp = originals mi.cover, mi.pubdate, mi.timestamp = originals
if extra_files and opts.save_extra_files:
for relpath in extra_files:
data_dest_path = os.path.abspath(os.path.join(dirpath, relpath))
try:
db.copy_extra_file_to(book_id, relpath, data_dest_path)
except FileNotFoundError:
os.makedirs(make_long_path_useable(os.path.dirname(data_dest_path)))
db.copy_extra_file_to(book_id, relpath, data_dest_path)
if not formats: if not formats:
return not formats_written, book_id, mi.title return not formats_written, book_id, mi.title