Use calibre-parallel rather than calibre-debug for multiprocessing

src/calibre/debug.py

@@ -125,9 +125,6 @@ as a shebang in scripts, like this:
         'Convert the specified EPUB file to KEPUB without doing a full conversion. This is what the Kobo driver does when sending files to the device.'))
     parser.add_option('--un-kepubify', default=False, action='store_true', help=_(
         'Convert the specified KEPUB file to EPUB without doing a full conversion. This is what the Kobo driver does when importing files from the device.'))
-    parser.add_option('--fix-multiprocessing', default=False, action='store_true',
-        help=_('For internal use'))
-
     return parser
 
 
@@ -230,10 +227,6 @@ def main(args=sys.argv):
     from calibre.constants import debug
 
     opts, args = option_parser().parse_args(args)
-    if opts.fix_multiprocessing:
-        sys.argv = [sys.argv[0], '--multiprocessing-fork']
-        exec(args[-1])
-        return
     if not opts.run_without_debug:
         debug()
     if opts.gui:

src/calibre/startup.py

@@ -25,15 +25,13 @@ builtins.__dict__['dynamic_property'] = lambda func: func(None)
 from calibre.constants import DEBUG, isfreebsd, islinux, ismacos, iswindows
 
 
-def get_debug_executable(headless=False):
+def get_debug_executable(headless=False, exe_name='calibre-debug'):
-    exe_name = 'calibre-debug' + ('.exe' if iswindows else '')
+    exe_name = exe_name + ('.exe' if iswindows else '')
     if hasattr(sys, 'frameworks_dir'):
         base = os.path.dirname(sys.frameworks_dir)
         if headless:
-            from calibre.utils.ipc.launch import Worker
+            from calibre.utils.ipc.launch import headless_exe_path
-            class W(Worker):
+            return [headless_exe_path(exe_name)]
-                exe_name = 'calibre-debug'
-            return [W().executable]
         return [os.path.join(base, 'MacOS', exe_name)]
     if getattr(sys, 'run_local', None):
         return [sys.run_local, exe_name]
@@ -96,11 +94,19 @@ def initialize_calibre():
     # Fix multiprocessing
     from multiprocessing import spawn, util
 
+    def get_executable() -> list[str]:
+        return get_debug_executable(headless=True, exe_name='calibre-parallel')
+
     def get_command_line(**kwds):
-        prog = 'from multiprocessing.spawn import spawn_main; spawn_main(%s)'
+        prog = ', '.join('{}={!r}'.format(*item) for item in kwds.items())
-        prog %= ', '.join('{}={!r}'.format(*item) for item in kwds.items())
+        prog = f'from multiprocessing.spawn import spawn_main; spawn_main({prog})'
-        return get_debug_executable() + ['--fix-multiprocessing', '--', prog]
+        return get_executable() + ['__multiprocessing__', prog]
     spawn.get_command_line = get_command_line
+    spawn._fixup_main_from_path = lambda *a: None
+    if iswindows:
+        # On windows multiprocessing does not run the result of
+        # get_command_line directly, see popen_spawn_win32.py
+        spawn.set_executable(get_executable()[-1])
     orig_spawn_passfds = util.spawnv_passfds
     orig_remove_temp_dir = util._remove_temp_dir
 
@@ -124,7 +130,7 @@ def initialize_calibre():
             idx = args.index('-c')
         except ValueError:
             return wrapped_orig_spawn_fds(args, passfds)
-        patched_args = get_debug_executable() + ['--fix-multiprocessing', '--'] + args[idx + 1:]
+        patched_args = get_executable() + ['__multiprocessing__'] + args[idx + 1:]
         return wrapped_orig_spawn_fds(patched_args, passfds)
     util.spawnv_passfds = spawnv_passfds
     util._remove_temp_dir = safe_remove_temp_dir

src/calibre/utils/ipc/launch.py

@@ -69,6 +69,12 @@ def exe_path(exe_name):
     return e
 
 
+def headless_exe_path(exe_name='calibre-parallel'):
+    if ismacos and not hasattr(sys, 'running_from_setup'):
+        return os.path.join(macos_headless_bundle_path(), exe_name)
+    return exe_path(exe_name)
+
+
 class Worker:
     '''
     Platform independent object for launching child processes. All processes
@@ -86,9 +92,7 @@ class Worker:
 
     @property
     def executable(self):
-        if ismacos and not hasattr(sys, 'running_from_setup'):
+        return headless_exe_path(self.exe_name)
-            return os.path.join(macos_headless_bundle_path(), self.exe_name)
-        return exe_path(self.exe_name)
 
     @property
     def gui_executable(self):

src/calibre/utils/ipc/worker.py

@@ -165,13 +165,11 @@ def get_func(name):
 
 
 def main():
-    if iswindows:
+    if '__multiprocessing__' in sys.argv:
-        if '--multiprocessing-fork' in sys.argv:
+        payload = sys.argv[-1]
-            # We are using the multiprocessing module on windows to launch a
+        sys.argv = [sys.argv[0], '--multiprocessing-fork']
-            # worker process
+        exec(payload)
-            from multiprocessing import freeze_support
+        return 0
-            freeze_support()
-            return 0
     if ismacos and 'CALIBRE_WORKER_FD' not in os.environ and 'CALIBRE_SIMPLE_WORKER' not in os.environ and '--pipe-worker' not in sys.argv:
         # On some OS X computers launchd apparently tries to
         # launch the last run process from the bundle