This PR implements what we were talking about by email -- creating a folder containing metadata.db and .calnotes while in the GUI then passing that via the subprocess to the backend.

src/calibre/db/backend.py

@@ -444,7 +444,7 @@ class DB:
 
     def __init__(self, library_path, default_prefs=None, read_only=False,
                  restore_all_prefs=False, progress_callback=lambda x, y:True,
-                 load_user_formatter_functions=True):
+                 load_user_formatter_functions=True, temp_db_path=None):
         self.is_closed = False
         if isbytestring(library_path):
             library_path = library_path.decode(filesystem_encoding)
@@ -452,8 +452,7 @@ class DB:
 
         self.library_path = os.path.abspath(library_path)
         self.dbpath = os.path.join(library_path, 'metadata.db')
-        self.dbpath = os.environ.get('CALIBRE_OVERRIDE_DATABASE_PATH',
+        self.dbpath = os.environ.get('CALIBRE_OVERRIDE_DATABASE_PATH', self.dbpath)
-                self.dbpath)
 
         if iswindows and len(self.library_path) + 4*self.PATH_LIMIT + 10 > 259:
             raise ValueError(_(
@@ -468,7 +467,14 @@ class DB:
                     'Path to library too long. It must be less than'
                     ' %d characters.')%self.WINDOWS_LIBRARY_PATH_LIMIT)
 
-        if read_only and os.path.exists(self.dbpath):
+        if temp_db_path is not None:
+            if not os.path.exists(temp_db_path):
+                raise FileNotFoundError(f"temp_db_path '{temp_db_path} doesn't refer to a file")
+            # temp_db_path specifies a path to the database to use for this
+            # library. It should be in its own folder along with .calnotes.
+            # It overrides the environment variable CALIBRE_OVERRIDE_DATABASE_PATH.
+            self.dbpath = temp_db_path
+        elif read_only and os.path.exists(self.dbpath):
             # Work on only a copy of metadata.db to ensure that
             # metadata.db is not changed
             pt = PersistentTemporaryFile('_metadata_ro.db')

src/calibre/db/legacy.py

@@ -42,11 +42,12 @@ def cleanup_tags(tags):
 def create_backend(
         library_path, default_prefs=None, read_only=False,
         progress_callback=lambda x, y:True, restore_all_prefs=False,
-        load_user_formatter_functions=True):
+        load_user_formatter_functions=True, temp_db_path=None):
     return DB(library_path, default_prefs=default_prefs,
                      read_only=read_only, restore_all_prefs=restore_all_prefs,
                      progress_callback=progress_callback,
-                     load_user_formatter_functions=load_user_formatter_functions)
+                     load_user_formatter_functions=load_user_formatter_functions,
+                     temp_db_path=temp_db_path)
 
 
 def set_global_state(db):
@@ -179,7 +180,8 @@ class LibraryDatabase:
 
     def __init__(self, library_path,
             default_prefs=None, read_only=False, is_second_db=False,
-            progress_callback=None, restore_all_prefs=False, row_factory=False):
+            progress_callback=None, restore_all_prefs=False, row_factory=False,
+            temp_db_path=None):
 
         self.is_second_db = is_second_db
         if progress_callback is None:
@@ -190,7 +192,8 @@ class LibraryDatabase:
         backend = self.backend = create_backend(library_path, default_prefs=default_prefs,
                     read_only=read_only, restore_all_prefs=restore_all_prefs,
                     progress_callback=progress_callback,
-                    load_user_formatter_functions=not is_second_db)
+                    load_user_formatter_functions=not is_second_db,
+                    temp_db_path=temp_db_path)
         cache = self.new_api = Cache(backend, library_database_instance=self)
         cache.init()
         self.data = View(cache)

src/calibre/gui2/convert/gui_conversion.py

@@ -40,14 +40,16 @@ def gui_convert_override(input, output, recommendations, notification=DummyRepor
             override_input_metadata=True)
 
 
-def gui_catalog(fmt, title, dbspec, ids, out_file_name, sync, fmt_options, connected_device,
+def gui_catalog(library_path, temp_db_path, fmt, title, dbspec, ids, out_file_name, sync, fmt_options, connected_device,
         notification=DummyReporter(), log=None):
     if log is None:
         log = Log()
-    from calibre.library import db
     from calibre.utils.config import prefs
     prefs.refresh()
-    db = db(read_only=True)
+
+    # Open the temp database created while still in the GUI thread
+    from calibre.db.legacy import LibraryDatabase
+    db = LibraryDatabase(library_path, temp_db_path=temp_db_path)
     db.catalog_plugin_on_device_temp_mapping = dbspec
 
     # Create a minimal OptionParser that we can append to

src/calibre/gui2/tools.py

@@ -10,6 +10,7 @@ Logic for setting up conversion jobs
 '''
 
 import os
+import shutil
 
 from qt.core import QDialog, QProgressDialog, QTimer
 
@@ -27,7 +28,7 @@ from calibre.gui2.convert import bulk_defaults_for_input_format
 from calibre.gui2.convert.bulk import BulkConfig
 from calibre.gui2.convert.metadata import create_cover_file, create_opf_file
 from calibre.gui2.convert.single import Config as SingleConfig
-from calibre.ptempfile import PersistentTemporaryFile
+from calibre.ptempfile import PersistentTemporaryDirectory, PersistentTemporaryFile
 from calibre.utils.config import prefs
 from polyglot.builtins import as_bytes
 
@@ -371,8 +372,23 @@ def generate_catalog(parent, dbspec, ids, device_manager, db):  # {{{
         except:
             pass
 
+    # Create a temporary copy of the databases to pass into the generation
+    # process. The backend looks for the notes directory (.calnotes) in the
+    # directory containing the metadata.db file. Copy the one from the current
+    # library.
+    from calibre.gui2.ui import get_gui
+    library_path = get_gui().current_db.library_path
+    temp_db_directory = PersistentTemporaryDirectory('_callib')
+    temp_db_path = os.path.join(temp_db_directory, 'metadata.db')
+    shutil.copy(os.path.join(library_path, 'metadata.db'), temp_db_directory)
+    notes_dir = os.path.join(library_path, '.calnotes')
+    if os.path.exists(notes_dir):
+        shutil.copytree(notes_dir, os.path.join(temp_db_directory, '.calnotes'))
+
     # These args are passed inline to gui2.convert.gui_conversion:gui_catalog
     args = [
+        library_path,
+        temp_db_path,
         d.catalog_format,
         d.catalog_title,
         dbspec,