From 2938e2c2037e1a47a4536bf373f9e72e7e5a8037 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 28 Sep 2020 16:28:51 +0530 Subject: [PATCH] New bypy freezing framework for Linux builds --- bypy/linux/__main__.py | 25 +++-- bypy/run-python.h | 207 ++--------------------------------------- 2 files changed, 27 insertions(+), 205 deletions(-) diff --git a/bypy/linux/__main__.py b/bypy/linux/__main__.py index 01adf0e253..f37c83d512 100644 --- a/bypy/linux/__main__.py +++ b/bypy/linux/__main__.py @@ -15,6 +15,10 @@ from functools import partial from bypy.constants import ( OUTPUT_DIR, PREFIX, SRC as CALIBRE_DIR, is64bit, python_major_minor_version ) +from bypy.freeze import ( + extract_extension_modules, freeze_python, path_to_freeze_dir, + save_importer_src_to_header +) from bypy.utils import ( create_job, get_dll_path, mkdtemp, parallel_build, py_compile, run, walk ) @@ -146,12 +150,6 @@ def copy_python(env, ext_dir): shutil.copytree(c, j(dest, x), ignore=partial(ignore_in_lib, ignored_dirs={})) elif os.path.isfile(c): shutil.copy2(c, j(dest, x)) - pdir = j(env.lib_dir, 'calibre-extensions') - if not os.path.exists(pdir): - os.mkdir(pdir) - for x in glob.glob(j(ext_dir, '*.so')): - shutil.copy2(x, j(pdir, os.path.basename(x))) - shutil.copytree(j(env.src_root, 'resources'), j(env.base, 'resources')) for pak in glob.glob(j(QT_PREFIX, 'resources', '*.pak')): shutil.copy2(pak, j(env.base, 'resources')) @@ -160,7 +158,21 @@ def copy_python(env, ext_dir): sitepy = j(self_dir, 'site.py') shutil.copy2(sitepy, j(env.py_dir, 'site.py')) + pdir = j(env.lib_dir, 'calibre-extensions') + if not os.path.exists(pdir): + os.mkdir(pdir) + for x in os.listdir(j(env.py_dir, 'site-packages')): + os.rename(j(env.py_dir, 'site-packages', x), j(env.py_dir, x)) + os.rmdir(j(env.py_dir, 'site-packages')) + print('Extracting extension modules from', ext_dir, 'to', pdir) + ext_map = extract_extension_modules(ext_dir, pdir) + shutil.rmtree(j(env.py_dir, 'calibre', 'plugins')) + print('Extracting extension modules from', env.py_dir, 'to', pdir) + ext_map.update(extract_extension_modules(env.py_dir, pdir)) py_compile(env.py_dir) + freeze_python(env.py_dir, pdir, env.obj_dir) + shutil.rmtree(env.py_dir) + save_importer_src_to_header(env.obj_dir, ext_map, develop_mode_env_var='CALIBRE_DEVELOP_FROM') def build_launchers(env): @@ -169,6 +181,7 @@ def build_launchers(env): objects = [j(env.obj_dir, os.path.basename(x) + '.o') for x in sources] cflags = '-fno-strict-aliasing -W -Wall -c -O2 -pipe -DPY_VERSION_MAJOR={} -DPY_VERSION_MINOR={}'.format(*py_ver.split('.')) cflags = cflags.split() + ['-I%s/include/python%s' % (PREFIX, py_ver)] + cflags += [f'-I{path_to_freeze_dir()}', f'-I{env.obj_dir}'] for src, obj in zip(sources, objects): cmd = ['gcc'] + cflags + ['-fPIC', '-o', obj, src] run(*cmd) diff --git a/bypy/run-python.h b/bypy/run-python.h index a1f5bc21ef..cae241b539 100644 --- a/bypy/run-python.h +++ b/bypy/run-python.h @@ -22,127 +22,12 @@ #ifdef __APPLE__ #include #endif +#include -#define arraysz(x) (sizeof(x)/sizeof(x[0])) - -static bool use_os_log = false; - -#ifdef _WIN32 -static void -log_error(const char *fmt, ...) { - va_list ar; - va_start(ar, fmt); - vfprintf(stderr, fmt, ar); - va_end(ar); - fprintf(stderr, "\n"); -} - -static bool stdout_is_a_tty = false, stderr_is_a_tty = false; -static DWORD console_old_mode = 0; -static UINT code_page = CP_UTF8; -static bool console_mode_changed = false; - -static void -detect_tty(void) { - stdout_is_a_tty = _isatty(_fileno(stdout)); - stderr_is_a_tty = _isatty(_fileno(stderr)); -} - -static void -setup_vt_terminal_mode(void) { - if (stdout_is_a_tty || stderr_is_a_tty) { - HANDLE h = GetStdHandle(stdout_is_a_tty ? STD_OUTPUT_HANDLE : STD_ERROR_HANDLE); - if (h != INVALID_HANDLE_VALUE) { - if (GetConsoleMode(h, &console_old_mode)) { - console_mode_changed = true; - SetConsoleMode(h, console_old_mode | ENABLE_VIRTUAL_TERMINAL_PROCESSING); - } - } - } -} - -static void -restore_vt_terminal_mode(void) { - if (console_mode_changed) SetConsoleMode(GetStdHandle(stdout_is_a_tty ? STD_OUTPUT_HANDLE : STD_ERROR_HANDLE), console_old_mode); -} - -static void -cleanup_console_state() { - if (code_page != CP_UTF8) SetConsoleOutputCP(CP_UTF8); - restore_vt_terminal_mode(); -} -#else -static void -log_error(const char *fmt, ...) __attribute__ ((format (printf, 1, 2))); - - -static void -log_error(const char *fmt, ...) { - va_list ar; - struct timeval tv; -#ifdef __APPLE__ - // Apple does not provide a varargs style os_logv - char logbuf[16 * 1024] = {0}; -#else - char logbuf[4]; -#endif - char *p = logbuf; -#define bufprint(func, ...) { if ((size_t)(p - logbuf) < sizeof(logbuf) - 2) { p += func(p, sizeof(logbuf) - (p - logbuf), __VA_ARGS__); } } - if (!use_os_log) { // Apple's os_log already records timestamps - gettimeofday(&tv, NULL); - struct tm *tmp = localtime(&tv.tv_sec); - if (tmp) { - char tbuf[256] = {0}, buf[256] = {0}; - if (strftime(buf, sizeof(buf), "%j %H:%M:%S.%%06u", tmp) != 0) { - snprintf(tbuf, sizeof(tbuf), buf, tv.tv_usec); - fprintf(stderr, "[%s] ", tbuf); - } - } - } - va_start(ar, fmt); - if (use_os_log) { bufprint(vsnprintf, fmt, ar); } - else vfprintf(stderr, fmt, ar); - va_end(ar); -#ifdef __APPLE__ - if (use_os_log) os_log(OS_LOG_DEFAULT, "%{public}s", logbuf); -#endif - if (!use_os_log) fprintf(stderr, "\n"); -} -#endif - - -#define fatal(...) { log_error(__VA_ARGS__); exit(EXIT_FAILURE); } - -static void -set_sys_string(const char* key, const wchar_t* val) { - PyObject *temp = PyUnicode_FromWideChar(val, -1); - if (temp) { - if (PySys_SetObject(key, temp) != 0) fatal("Failed to set attribute on sys: %s", key); - Py_DECREF(temp); - } else { - fatal("Failed to set attribute on sys, PyUnicode_FromWideChar failed"); - } -} - -static void -set_sys_bool(const char* key, const bool val) { - PyObject *pyval = PyBool_FromLong(val); - if (PySys_SetObject(key, pyval) != 0) fatal("Failed to set attribute on sys: %s", key); - Py_DECREF(pyval); -} static void pre_initialize_interpreter(bool is_gui_app) { - PyStatus status; - use_os_log = is_gui_app; - PyPreConfig preconfig; - PyPreConfig_InitIsolatedConfig(&preconfig); - preconfig.utf8_mode = 1; - preconfig.coerce_c_locale = 1; - preconfig.isolated = 1; - - status = Py_PreInitialize(&preconfig); - if (PyStatus_Exception(status)) Py_ExitStatusException(status); + bypy_pre_initialize_interpreter(is_gui_app); } #define decode_char_buf(src, dest) { \ @@ -158,9 +43,7 @@ pre_initialize_interpreter(bool is_gui_app) { #define MAX_SYS_PATHS 3 typedef struct { - wchar_t* sys_paths[MAX_SYS_PATHS]; - wchar_t sys_path_buf[MAX_SYS_PATHS * PATH_MAX]; - size_t sys_paths_count; + int argc; wchar_t exe_path[PATH_MAX], python_home_path[PATH_MAX], python_lib_path[PATH_MAX]; wchar_t extensions_path[PATH_MAX], resources_path[PATH_MAX], executables_path[PATH_MAX]; #ifdef __APPLE__ @@ -169,8 +52,6 @@ typedef struct { wchar_t app_dir[PATH_MAX]; #endif const wchar_t *basename, *module, *function; - int argc; - PyObject* (*calibre_os_module)(void); #ifdef _WIN32 wchar_t* const *argv; #else @@ -178,74 +59,13 @@ typedef struct { #endif } InterpreterData; -static InterpreterData interpreter_data = {{0}}; - -static wchar_t* -add_sys_path() { - if (interpreter_data.sys_paths_count >= MAX_SYS_PATHS) fatal("Trying to add too many entries to sys.path"); - wchar_t *ans = interpreter_data.sys_path_buf + PATH_MAX * interpreter_data.sys_paths_count; - interpreter_data.sys_paths[interpreter_data.sys_paths_count] = ans; - interpreter_data.sys_paths_count++; - return ans; -} - -static void -add_sys_paths() { -#ifdef _WIN32 - swprintf(add_sys_path(), PATH_MAX, L"%ls\\app\\pylib.zip", interpreter_data.app_dir); - swprintf(add_sys_path(), PATH_MAX, L"%ls\\app\\bin", interpreter_data.app_dir); -#else - swprintf(add_sys_path(), PATH_MAX, L"%ls", interpreter_data.python_lib_path); - swprintf(add_sys_path(), PATH_MAX, L"%ls/lib-dynload", interpreter_data.python_lib_path); -#ifdef __APPLE__ - swprintf(add_sys_path(), PATH_MAX, L"%ls/Python/site-packages", interpreter_data.bundle_resource_path); -#else - swprintf(add_sys_path(), PATH_MAX, L"%ls/site-packages", interpreter_data.python_lib_path); -#endif -#endif -} +static InterpreterData interpreter_data = {0}; static void run_interpreter() { -#define CHECK_STATUS if (PyStatus_Exception(status)) { PyConfig_Clear(&config); Py_ExitStatusException(status); } - PyStatus status; - PyConfig config; - - PyConfig_InitIsolatedConfig(&config); - add_sys_paths(); - status = PyConfig_SetWideStringList(&config, &config.module_search_paths, interpreter_data.sys_paths_count, interpreter_data.sys_paths); - CHECK_STATUS; - - config.module_search_paths_set = 1; - config.optimization_level = 2; - config.write_bytecode = 0; - config.use_environment = 0; - config.user_site_directory = 0; - config.configure_c_stdio = 1; - config.isolated = 1; - - status = PyConfig_SetString(&config, &config.program_name, interpreter_data.exe_path); - CHECK_STATUS; -#ifndef _WIN32 - status = PyConfig_SetString(&config, &config.home, interpreter_data.python_home_path); - CHECK_STATUS; -#endif - status = PyConfig_SetString(&config, &config.run_module, L"site"); - CHECK_STATUS; -#ifdef _WIN32 - status = PyConfig_SetArgv(&config, interpreter_data.argc, interpreter_data.argv); -#else - status = PyConfig_SetBytesArgv(&config, interpreter_data.argc, interpreter_data.argv); -#endif - CHECK_STATUS; - if (interpreter_data.calibre_os_module) { - if (PyImport_AppendInittab("calibre_os_module", interpreter_data.calibre_os_module) == -1) { - fatal("Failed to add calibre_os_module to the init table"); - } - } - status = Py_InitializeFromConfig(&config); - CHECK_STATUS; - + bypy_initialize_interpreter( + interpreter_data.exe_path, interpreter_data.python_home_path, L"site", interpreter_data.extensions_path, + interpreter_data.argc, interpreter_data.argv); set_sys_bool("gui_app", use_os_log); set_sys_bool("frozen", true); set_sys_string("calibre_basename", interpreter_data.basename); @@ -265,17 +85,6 @@ run_interpreter() { set_sys_string("frozen_path", interpreter_data.executables_path); #endif -#ifdef _WIN32 - code_page = GetConsoleOutputCP(); - if (code_page != CP_UTF8) SetConsoleOutputCP(CP_UTF8); - setup_vt_terminal_mode(); -#endif - - int ret = Py_RunMain(); - PyConfig_Clear(&config); -#ifdef _WIN32 - cleanup_console_state(); -#endif + int ret = bypy_run_interpreter(); exit(ret); -#undef CHECK_STATUS }