diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 08bce569f0..fd996c4b29 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -20,6 +20,7 @@ from io import DEFAULT_BUFFER_SIZE, BytesIO from queue import Queue from threading import Lock from time import monotonic, sleep, time +from typing import NamedTuple, Tuple from calibre import as_unicode, detect_ncpus, isbytestring from calibre.constants import iswindows, preferred_encoding @@ -53,6 +54,12 @@ from calibre.utils.localization import canonicalize_lang from polyglot.builtins import cmp, iteritems, itervalues, string_or_bytes +class ExtraFile(NamedTuple): + relpath: str + file_path: str + stat_result: os.stat_result + + def api(f): f.is_cache_api = True return f @@ -3096,7 +3103,7 @@ class Cache: return added @read_api - def list_extra_files(self, book_id, use_cache=False, pattern=''): + def list_extra_files(self, book_id, use_cache=False, pattern='') -> Tuple[ExtraFile, ...]: ''' Get information about extra files in the book's directory. @@ -3104,8 +3111,9 @@ class Cache: :param pattern: the pattern of filenames to search for. Empty pattern matches all extra files. Patterns must use / as separator. Use the DATA_FILE_PATTERN constant to match files inside the data directory. - :return: A tuple of all extra files matching the specified pattern. Each element of the tuple is (relpath, file_path, stat_result) - where relpath is the relative path of the file to the book directory using / as a separator. + :return: A tuple of all extra files matching the specified pattern. Each element of the tuple is + ExtraFile(relpath, file_path, stat_result). Where relpath is the relative path of the file + to the book directory using / as a separator. stat_result is the result of calling os.stat() on the file. ''' ans = self.extra_files_cache.setdefault(book_id, {}).get(pattern) @@ -3116,7 +3124,7 @@ class Cache: for (relpath, file_path, stat_result) in self.backend.iter_extra_files( book_id, path, self.fields['formats'], yield_paths=True, pattern=pattern ): - ans.append((relpath, file_path, stat_result)) + ans.append(ExtraFile(relpath, file_path, stat_result)) self.extra_files_cache[book_id][pattern] = ans = tuple(ans) return ans diff --git a/src/calibre/db/cli/cmd_export.py b/src/calibre/db/cli/cmd_export.py index 66e463d290..e2ce552daf 100644 --- a/src/calibre/db/cli/cmd_export.py +++ b/src/calibre/db/cli/cmd_export.py @@ -27,7 +27,7 @@ def implementation(db, notify_changes, action, *args): mi = db.get_metadata(book_id) plugboards = db.pref('plugboards', {}) formats = get_formats(db.formats(book_id), formats) - extra_files_for_export = tuple(relpath for (relpath, file_path, stat_result) in db.list_extra_files(book_id, pattern=DATA_FILE_PATTERN)) + extra_files_for_export = tuple(ef.relpath for ef in db.list_extra_files(book_id, pattern=DATA_FILE_PATTERN)) plugboards['extra_files_for_export'] = extra_files_for_export return mi, plugboards, formats, db.library_id, db.pref( 'user_template_functions', [] diff --git a/src/calibre/db/copy_to_library.py b/src/calibre/db/copy_to_library.py index 98dbe81d83..f470a805be 100644 --- a/src/calibre/db/copy_to_library.py +++ b/src/calibre/db/copy_to_library.py @@ -80,8 +80,8 @@ def copy_one_book( format_map = {} fmts = list(db.formats(book_id, verify_formats=False)) extra_file_map = {} - for (relpath, file_path, stat_result) in db.list_extra_files(book_id): - extra_file_map[relpath] = file_path + for ef in db.list_extra_files(book_id): + extra_file_map[ef.relpath] = ef.file_path for fmt in fmts: path = db.format_abspath(book_id, fmt) if path: diff --git a/src/calibre/db/tests/add_remove.py b/src/calibre/db/tests/add_remove.py index 636ad8f330..4c3e8edafe 100644 --- a/src/calibre/db/tests/add_remove.py +++ b/src/calibre/db/tests/add_remove.py @@ -410,8 +410,8 @@ class AddRemoveTest(BaseTest): self.assertFalse(os.path.exists(os.path.join(bookdir, 'sub', 'recurse'))) def clear_extra_files(book_id): - for (relpath, file_path, stat_result) in dest_db.list_extra_files(book_id): - os.remove(file_path) + for ef in dest_db.list_extra_files(book_id): + os.remove(ef.file_path) assert_does_not_have_extra_files(1) @@ -468,9 +468,9 @@ class AddRemoveTest(BaseTest): def extra_files_for(book_id): ans = {} - for relpath, file_path, stat_result in db.list_extra_files(book_id): - with open(file_path) as f: - ans[relpath] = f.read() + for ef in db.list_extra_files(book_id): + with open(ef.file_path) as f: + ans[ef.relpath] = f.read() return ans add_extra(1, 'one'), add_extra(1, 'sub/one') diff --git a/src/calibre/db/tests/filesystem.py b/src/calibre/db/tests/filesystem.py index 83bf8917cc..e7e851da50 100644 --- a/src/calibre/db/tests/filesystem.py +++ b/src/calibre/db/tests/filesystem.py @@ -104,9 +104,9 @@ class FilesystemTest(BaseTest): # test only formats being changed init_cache() ef = set() - for (relpath, file_path, stat_result) in cache.list_extra_files(1): - ef.add(relpath) - self.assertTrue(os.path.exists(file_path)) + for efx in cache.list_extra_files(1): + ef.add(efx.relpath) + self.assertTrue(os.path.exists(efx.file_path)) self.assertEqual(ef, {'a.side', 'subdir/a.fmt1'}) fname = cache.fields['formats'].table.fname_map[1]['FMT1'] cache.fields['formats'].table.fname_map[1]['FMT1'] = 'some thing else' @@ -229,8 +229,8 @@ class FilesystemTest(BaseTest): os.mkdir(os.path.join(bookdir, 'sub')) with open(os.path.join(bookdir, 'sub', 'recurse'), 'w') as f: f.write('recurse') - self.assertEqual({relpath for (relpath, _, _) in cache.list_extra_files(1, pattern='sub/**/*')}, {'sub/recurse'}) - self.assertEqual({relpath for (relpath, _, _) in cache.list_extra_files(1)}, {'exf', 'sub/recurse'}) + self.assertEqual({ef.relpath for ef in cache.list_extra_files(1, pattern='sub/**/*')}, {'sub/recurse'}) + self.assertEqual({ef.relpath for ef in cache.list_extra_files(1)}, {'exf', 'sub/recurse'}) for part_size in (1 << 30, 100, 1): with TemporaryDirectory('export_lib') as tdir, TemporaryDirectory('import_lib') as idir: exporter = Exporter(tdir, part_size=part_size) diff --git a/src/calibre/gui2/extra_files_watcher.py b/src/calibre/gui2/extra_files_watcher.py index f5b94c01f4..2bd39361c1 100644 --- a/src/calibre/gui2/extra_files_watcher.py +++ b/src/calibre/gui2/extra_files_watcher.py @@ -56,8 +56,8 @@ class ExtraFilesWatcher(QObject): def get_extra_files(self, book_id): db = self.gui.current_db.new_api - return tuple(ExtraFile(relpath, stat_result.st_mtime, stat_result.st_size) for - relpath, file_path, stat_result in db.list_extra_files(book_id, pattern=DATA_FILE_PATTERN)) + return tuple(ExtraFile(ef.relpath, ef.stat_result.st_mtime, ef.stat_result.st_size) for + ef in db.list_extra_files(book_id, pattern=DATA_FILE_PATTERN)) def check_registered_books(self): changed = {} diff --git a/src/calibre/gui2/save.py b/src/calibre/gui2/save.py index 83c512f764..584c64d353 100644 --- a/src/calibre/gui2/save.py +++ b/src/calibre/gui2/save.py @@ -215,8 +215,8 @@ class Saver(QObject): extra_files = {} if self.opts.save_extra_files: extra_files = {} - for (relpath, file_path, stat_result) in self.db.new_api.list_extra_files(int(book_id), pattern=DATA_FILE_PATTERN): - extra_files[relpath] = file_path + for efx in self.db.new_api.list_extra_files(int(book_id), pattern=DATA_FILE_PATTERN): + extra_files[efx.relpath] = efx.file_path if not fmts and not self.opts.write_opf and not self.opts.save_cover and not extra_files: return diff --git a/src/calibre/utils/formatter_functions.py b/src/calibre/utils/formatter_functions.py index b30fe22693..a9a567a958 100644 --- a/src/calibre/utils/formatter_functions.py +++ b/src/calibre/utils/formatter_functions.py @@ -2400,8 +2400,7 @@ class BuiltinHasExtraFiles(BuiltinFormatterFunction): def evaluate(self, formatter, kwargs, mi, locals): db = self.get_database(mi).new_api try: - files = db.list_extra_files(mi.id, use_cache=True, pattern=DATA_FILE_PATTERN) - return 'Yes' if files else '' + return 'Yes' if db.list_extra_files(mi.id, use_cache=True, pattern=DATA_FILE_PATTERN) else '' except Exception as e: traceback.print_exc() raise ValueError(e) @@ -2419,7 +2418,7 @@ class BuiltinExtraFileNames(BuiltinFormatterFunction): db = self.get_database(mi).new_api try: files = db.list_extra_files(mi.id, use_cache=True, pattern=DATA_FILE_PATTERN) - return sep.join(file[0].partition('/')[-1] for file in files) + return sep.join(file.relpath.partition('/')[-1] for file in files) except Exception as e: traceback.print_exc() raise ValueError(e) @@ -2438,10 +2437,9 @@ class BuiltinExtraFileSize(BuiltinFormatterFunction): db = self.get_database(mi).new_api try: q = posixpath.join(DATA_DIR_NAME, file_name) - files = db.list_extra_files(mi.id, use_cache=True, pattern=DATA_FILE_PATTERN) - for f in files: - if f[0] == q: - return str(f[2].st_size) + for f in db.list_extra_files(mi.id, use_cache=True, pattern=DATA_FILE_PATTERN): + if f.relpath == q: + return str(f.stat_result.st_size) return str(-1) except Exception as e: traceback.print_exc() @@ -2465,10 +2463,9 @@ class BuiltinExtraFileModtime(BuiltinFormatterFunction): db = self.get_database(mi).new_api try: q = posixpath.join(DATA_DIR_NAME, file_name) - files = db.list_extra_files(mi.id, use_cache=True, pattern=DATA_FILE_PATTERN) - for f in files: - if f[0] == q: - val = f[2].st_mtime + for f in db.list_extra_files(mi.id, use_cache=True, pattern=DATA_FILE_PATTERN): + if f.relpath == q: + val = f.stat_result.st_mtime if format_string: return format_date(datetime.fromtimestamp(val), format_string) return str(val)