From 2c3e858d9c2c6f1f2099e65e1021d3794023d26d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 7 Apr 2025 07:07:23 +0530 Subject: [PATCH] Use calibre-parallel rather than calibre-debug for multiprocessing --- src/calibre/debug.py | 7 ------- src/calibre/startup.py | 26 ++++++++++++++++---------- src/calibre/utils/ipc/launch.py | 10 +++++++--- src/calibre/utils/ipc/worker.py | 12 +++++------- 4 files changed, 28 insertions(+), 27 deletions(-) diff --git a/src/calibre/debug.py b/src/calibre/debug.py index aacd6aa331..4453916de5 100644 --- a/src/calibre/debug.py +++ b/src/calibre/debug.py @@ -125,9 +125,6 @@ as a shebang in scripts, like this: 'Convert the specified EPUB file to KEPUB without doing a full conversion. This is what the Kobo driver does when sending files to the device.')) parser.add_option('--un-kepubify', default=False, action='store_true', help=_( 'Convert the specified KEPUB file to EPUB without doing a full conversion. This is what the Kobo driver does when importing files from the device.')) - parser.add_option('--fix-multiprocessing', default=False, action='store_true', - help=_('For internal use')) - return parser @@ -230,10 +227,6 @@ def main(args=sys.argv): from calibre.constants import debug opts, args = option_parser().parse_args(args) - if opts.fix_multiprocessing: - sys.argv = [sys.argv[0], '--multiprocessing-fork'] - exec(args[-1]) - return if not opts.run_without_debug: debug() if opts.gui: diff --git a/src/calibre/startup.py b/src/calibre/startup.py index 566f03f322..7328b438f4 100644 --- a/src/calibre/startup.py +++ b/src/calibre/startup.py @@ -25,15 +25,13 @@ builtins.__dict__['dynamic_property'] = lambda func: func(None) from calibre.constants import DEBUG, isfreebsd, islinux, ismacos, iswindows -def get_debug_executable(headless=False): - exe_name = 'calibre-debug' + ('.exe' if iswindows else '') +def get_debug_executable(headless=False, exe_name='calibre-debug'): + exe_name = exe_name + ('.exe' if iswindows else '') if hasattr(sys, 'frameworks_dir'): base = os.path.dirname(sys.frameworks_dir) if headless: - from calibre.utils.ipc.launch import Worker - class W(Worker): - exe_name = 'calibre-debug' - return [W().executable] + from calibre.utils.ipc.launch import headless_exe_path + return [headless_exe_path(exe_name)] return [os.path.join(base, 'MacOS', exe_name)] if getattr(sys, 'run_local', None): return [sys.run_local, exe_name] @@ -96,11 +94,19 @@ def initialize_calibre(): # Fix multiprocessing from multiprocessing import spawn, util + def get_executable() -> list[str]: + return get_debug_executable(headless=True, exe_name='calibre-parallel') + def get_command_line(**kwds): - prog = 'from multiprocessing.spawn import spawn_main; spawn_main(%s)' - prog %= ', '.join('{}={!r}'.format(*item) for item in kwds.items()) - return get_debug_executable() + ['--fix-multiprocessing', '--', prog] + prog = ', '.join('{}={!r}'.format(*item) for item in kwds.items()) + prog = f'from multiprocessing.spawn import spawn_main; spawn_main({prog})' + return get_executable() + ['__multiprocessing__', prog] spawn.get_command_line = get_command_line + spawn._fixup_main_from_path = lambda *a: None + if iswindows: + # On windows multiprocessing does not run the result of + # get_command_line directly, see popen_spawn_win32.py + spawn.set_executable(get_executable()[-1]) orig_spawn_passfds = util.spawnv_passfds orig_remove_temp_dir = util._remove_temp_dir @@ -124,7 +130,7 @@ def initialize_calibre(): idx = args.index('-c') except ValueError: return wrapped_orig_spawn_fds(args, passfds) - patched_args = get_debug_executable() + ['--fix-multiprocessing', '--'] + args[idx + 1:] + patched_args = get_executable() + ['__multiprocessing__'] + args[idx + 1:] return wrapped_orig_spawn_fds(patched_args, passfds) util.spawnv_passfds = spawnv_passfds util._remove_temp_dir = safe_remove_temp_dir diff --git a/src/calibre/utils/ipc/launch.py b/src/calibre/utils/ipc/launch.py index 29e0dccef9..1070917915 100644 --- a/src/calibre/utils/ipc/launch.py +++ b/src/calibre/utils/ipc/launch.py @@ -69,6 +69,12 @@ def exe_path(exe_name): return e +def headless_exe_path(exe_name='calibre-parallel'): + if ismacos and not hasattr(sys, 'running_from_setup'): + return os.path.join(macos_headless_bundle_path(), exe_name) + return exe_path(exe_name) + + class Worker: ''' Platform independent object for launching child processes. All processes @@ -86,9 +92,7 @@ class Worker: @property def executable(self): - if ismacos and not hasattr(sys, 'running_from_setup'): - return os.path.join(macos_headless_bundle_path(), self.exe_name) - return exe_path(self.exe_name) + return headless_exe_path(self.exe_name) @property def gui_executable(self): diff --git a/src/calibre/utils/ipc/worker.py b/src/calibre/utils/ipc/worker.py index 4aa9190c3a..1dce85410b 100644 --- a/src/calibre/utils/ipc/worker.py +++ b/src/calibre/utils/ipc/worker.py @@ -165,13 +165,11 @@ def get_func(name): def main(): - if iswindows: - if '--multiprocessing-fork' in sys.argv: - # We are using the multiprocessing module on windows to launch a - # worker process - from multiprocessing import freeze_support - freeze_support() - return 0 + if '__multiprocessing__' in sys.argv: + payload = sys.argv[-1] + sys.argv = [sys.argv[0], '--multiprocessing-fork'] + exec(payload) + return 0 if ismacos and 'CALIBRE_WORKER_FD' not in os.environ and 'CALIBRE_SIMPLE_WORKER' not in os.environ and '--pipe-worker' not in sys.argv: # On some OS X computers launchd apparently tries to # launch the last run process from the bundle