Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-06-10 00:54:32 -04:00
Code Issues Packages Projects Releases Wiki Activity

New bypy freezing framework for Linux builds

Browse Source
This commit is contained in:
Kovid Goyal 2020-09-28 16:28:51 +05:30
parent f42b0b9f59
commit 2938e2c203
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
2 changed files with 27 additions and 205 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
bypy/linux/__main__.py
View File

@ -15,6 +15,10 @@ from functools import partial
from bypy.constants import ( from bypy.constants import (
OUTPUT_DIR, PREFIX, SRC as CALIBRE_DIR, is64bit, python_major_minor_version OUTPUT_DIR, PREFIX, SRC as CALIBRE_DIR, is64bit, python_major_minor_version
) )
from bypy.freeze import (
extract_extension_modules, freeze_python, path_to_freeze_dir,
save_importer_src_to_header
)
from bypy.utils import ( from bypy.utils import (
create_job, get_dll_path, mkdtemp, parallel_build, py_compile, run, walk create_job, get_dll_path, mkdtemp, parallel_build, py_compile, run, walk
) )
@ -146,12 +150,6 @@ def copy_python(env, ext_dir):
shutil.copytree(c, j(dest, x), ignore=partial(ignore_in_lib, ignored_dirs={})) shutil.copytree(c, j(dest, x), ignore=partial(ignore_in_lib, ignored_dirs={}))
elif os.path.isfile(c): elif os.path.isfile(c):
shutil.copy2(c, j(dest, x)) shutil.copy2(c, j(dest, x))
pdir = j(env.lib_dir, 'calibre-extensions')
if not os.path.exists(pdir):
os.mkdir(pdir)
for x in glob.glob(j(ext_dir, '*.so')):
shutil.copy2(x, j(pdir, os.path.basename(x)))
shutil.copytree(j(env.src_root, 'resources'), j(env.base, 'resources')) shutil.copytree(j(env.src_root, 'resources'), j(env.base, 'resources'))
for pak in glob.glob(j(QT_PREFIX, 'resources', '*.pak')): for pak in glob.glob(j(QT_PREFIX, 'resources', '*.pak')):
shutil.copy2(pak, j(env.base, 'resources')) shutil.copy2(pak, j(env.base, 'resources'))
@ -160,7 +158,21 @@ def copy_python(env, ext_dir):
sitepy = j(self_dir, 'site.py') sitepy = j(self_dir, 'site.py')
shutil.copy2(sitepy, j(env.py_dir, 'site.py')) shutil.copy2(sitepy, j(env.py_dir, 'site.py'))
pdir = j(env.lib_dir, 'calibre-extensions')
if not os.path.exists(pdir):
os.mkdir(pdir)
for x in os.listdir(j(env.py_dir, 'site-packages')):
os.rename(j(env.py_dir, 'site-packages', x), j(env.py_dir, x))
os.rmdir(j(env.py_dir, 'site-packages'))
print('Extracting extension modules from', ext_dir, 'to', pdir)
ext_map = extract_extension_modules(ext_dir, pdir)
shutil.rmtree(j(env.py_dir, 'calibre', 'plugins'))
print('Extracting extension modules from', env.py_dir, 'to', pdir)
ext_map.update(extract_extension_modules(env.py_dir, pdir))
py_compile(env.py_dir) py_compile(env.py_dir)
freeze_python(env.py_dir, pdir, env.obj_dir)
shutil.rmtree(env.py_dir)
save_importer_src_to_header(env.obj_dir, ext_map, develop_mode_env_var='CALIBRE_DEVELOP_FROM')
def build_launchers(env): def build_launchers(env):
@ -169,6 +181,7 @@ def build_launchers(env):
objects = [j(env.obj_dir, os.path.basename(x) + '.o') for x in sources] objects = [j(env.obj_dir, os.path.basename(x) + '.o') for x in sources]
cflags = '-fno-strict-aliasing -W -Wall -c -O2 -pipe -DPY_VERSION_MAJOR={} -DPY_VERSION_MINOR={}'.format(*py_ver.split('.')) cflags = '-fno-strict-aliasing -W -Wall -c -O2 -pipe -DPY_VERSION_MAJOR={} -DPY_VERSION_MINOR={}'.format(*py_ver.split('.'))
cflags = cflags.split() + ['-I%s/include/python%s' % (PREFIX, py_ver)] cflags = cflags.split() + ['-I%s/include/python%s' % (PREFIX, py_ver)]
cflags += [f'-I{path_to_freeze_dir()}', f'-I{env.obj_dir}']
for src, obj in zip(sources, objects): for src, obj in zip(sources, objects):
cmd = ['gcc'] + cflags + ['-fPIC', '-o', obj, src] cmd = ['gcc'] + cflags + ['-fPIC', '-o', obj, src]
run(*cmd) run(*cmd)

207
bypy/run-python.h
View File

@ -22,127 +22,12 @@
#ifdef __APPLE__ #ifdef __APPLE__
#include <os/log.h> #include <os/log.h>
#endif #endif
#include <bypy-freeze.h>
#define arraysz(x) (sizeof(x)/sizeof(x[0]))
static bool use_os_log = false;
#ifdef _WIN32
static void
log_error(const char *fmt, ...) {
va_list ar;
va_start(ar, fmt);
vfprintf(stderr, fmt, ar);
va_end(ar);
fprintf(stderr, "\n");
}
static bool stdout_is_a_tty = false, stderr_is_a_tty = false;
static DWORD console_old_mode = 0;
static UINT code_page = CP_UTF8;
static bool console_mode_changed = false;
static void
detect_tty(void) {
stdout_is_a_tty = _isatty(_fileno(stdout));
stderr_is_a_tty = _isatty(_fileno(stderr));
}
static void
setup_vt_terminal_mode(void) {
if (stdout_is_a_tty || stderr_is_a_tty) {
HANDLE h = GetStdHandle(stdout_is_a_tty ? STD_OUTPUT_HANDLE : STD_ERROR_HANDLE);
if (h != INVALID_HANDLE_VALUE) {
if (GetConsoleMode(h, &console_old_mode)) {
console_mode_changed = true;
SetConsoleMode(h, console_old_mode | ENABLE_VIRTUAL_TERMINAL_PROCESSING);
}
}
}
}
static void
restore_vt_terminal_mode(void) {
if (console_mode_changed) SetConsoleMode(GetStdHandle(stdout_is_a_tty ? STD_OUTPUT_HANDLE : STD_ERROR_HANDLE), console_old_mode);
}
static void
cleanup_console_state() {
if (code_page != CP_UTF8) SetConsoleOutputCP(CP_UTF8);
restore_vt_terminal_mode();
}
#else
static void
log_error(const char *fmt, ...) __attribute__ ((format (printf, 1, 2)));
static void
log_error(const char *fmt, ...) {
va_list ar;
struct timeval tv;
#ifdef __APPLE__
// Apple does not provide a varargs style os_logv
char logbuf[16 * 1024] = {0};
#else
char logbuf[4];
#endif
char *p = logbuf;
#define bufprint(func, ...) { if ((size_t)(p - logbuf) < sizeof(logbuf) - 2) { p += func(p, sizeof(logbuf) - (p - logbuf), __VA_ARGS__); } }
if (!use_os_log) { // Apple's os_log already records timestamps
gettimeofday(&tv, NULL);
struct tm *tmp = localtime(&tv.tv_sec);
if (tmp) {
char tbuf[256] = {0}, buf[256] = {0};
if (strftime(buf, sizeof(buf), "%j %H:%M:%S.%%06u", tmp) != 0) {
snprintf(tbuf, sizeof(tbuf), buf, tv.tv_usec);
fprintf(stderr, "[%s] ", tbuf);
}
}
}
va_start(ar, fmt);
if (use_os_log) { bufprint(vsnprintf, fmt, ar); }
else vfprintf(stderr, fmt, ar);
va_end(ar);
#ifdef __APPLE__
if (use_os_log) os_log(OS_LOG_DEFAULT, "%{public}s", logbuf);
#endif
if (!use_os_log) fprintf(stderr, "\n");
}
#endif
#define fatal(...) { log_error(__VA_ARGS__); exit(EXIT_FAILURE); }
static void
set_sys_string(const char* key, const wchar_t* val) {
PyObject *temp = PyUnicode_FromWideChar(val, -1);
if (temp) {
if (PySys_SetObject(key, temp) != 0) fatal("Failed to set attribute on sys: %s", key);
Py_DECREF(temp);
} else {
fatal("Failed to set attribute on sys, PyUnicode_FromWideChar failed");
}
}
static void
set_sys_bool(const char* key, const bool val) {
PyObject *pyval = PyBool_FromLong(val);
if (PySys_SetObject(key, pyval) != 0) fatal("Failed to set attribute on sys: %s", key);
Py_DECREF(pyval);
}
static void static void
pre_initialize_interpreter(bool is_gui_app) { pre_initialize_interpreter(bool is_gui_app) {
PyStatus status; bypy_pre_initialize_interpreter(is_gui_app);
use_os_log = is_gui_app;
PyPreConfig preconfig;
PyPreConfig_InitIsolatedConfig(&preconfig);
preconfig.utf8_mode = 1;
preconfig.coerce_c_locale = 1;
preconfig.isolated = 1;
status = Py_PreInitialize(&preconfig);
if (PyStatus_Exception(status)) Py_ExitStatusException(status);
} }
#define decode_char_buf(src, dest) { \ #define decode_char_buf(src, dest) { \
@ -158,9 +43,7 @@ pre_initialize_interpreter(bool is_gui_app) {
#define MAX_SYS_PATHS 3 #define MAX_SYS_PATHS 3
typedef struct { typedef struct {
wchar_t* sys_paths[MAX_SYS_PATHS]; int argc;
wchar_t sys_path_buf[MAX_SYS_PATHS * PATH_MAX];
size_t sys_paths_count;
wchar_t exe_path[PATH_MAX], python_home_path[PATH_MAX], python_lib_path[PATH_MAX]; wchar_t exe_path[PATH_MAX], python_home_path[PATH_MAX], python_lib_path[PATH_MAX];
wchar_t extensions_path[PATH_MAX], resources_path[PATH_MAX], executables_path[PATH_MAX]; wchar_t extensions_path[PATH_MAX], resources_path[PATH_MAX], executables_path[PATH_MAX];
#ifdef __APPLE__ #ifdef __APPLE__
@ -169,8 +52,6 @@ typedef struct {
wchar_t app_dir[PATH_MAX]; wchar_t app_dir[PATH_MAX];
#endif #endif
const wchar_t *basename, *module, *function; const wchar_t *basename, *module, *function;
int argc;
PyObject* (*calibre_os_module)(void);
#ifdef _WIN32 #ifdef _WIN32
wchar_t* const *argv; wchar_t* const *argv;
#else #else
@ -178,74 +59,13 @@ typedef struct {
#endif #endif
} InterpreterData; } InterpreterData;
static InterpreterData interpreter_data = {{0}}; static InterpreterData interpreter_data = {0};
static wchar_t*
add_sys_path() {
if (interpreter_data.sys_paths_count >= MAX_SYS_PATHS) fatal("Trying to add too many entries to sys.path");
wchar_t *ans = interpreter_data.sys_path_buf + PATH_MAX * interpreter_data.sys_paths_count;
interpreter_data.sys_paths[interpreter_data.sys_paths_count] = ans;
interpreter_data.sys_paths_count++;
return ans;
}
static void
add_sys_paths() {
#ifdef _WIN32
swprintf(add_sys_path(), PATH_MAX, L"%ls\\app\\pylib.zip", interpreter_data.app_dir);
swprintf(add_sys_path(), PATH_MAX, L"%ls\\app\\bin", interpreter_data.app_dir);
#else
swprintf(add_sys_path(), PATH_MAX, L"%ls", interpreter_data.python_lib_path);
swprintf(add_sys_path(), PATH_MAX, L"%ls/lib-dynload", interpreter_data.python_lib_path);
#ifdef __APPLE__
swprintf(add_sys_path(), PATH_MAX, L"%ls/Python/site-packages", interpreter_data.bundle_resource_path);
#else
swprintf(add_sys_path(), PATH_MAX, L"%ls/site-packages", interpreter_data.python_lib_path);
#endif
#endif
}
static void static void
run_interpreter() { run_interpreter() {
#define CHECK_STATUS if (PyStatus_Exception(status)) { PyConfig_Clear(&config); Py_ExitStatusException(status); } bypy_initialize_interpreter(
PyStatus status; interpreter_data.exe_path, interpreter_data.python_home_path, L"site", interpreter_data.extensions_path,
PyConfig config; interpreter_data.argc, interpreter_data.argv);
PyConfig_InitIsolatedConfig(&config);
add_sys_paths();
status = PyConfig_SetWideStringList(&config, &config.module_search_paths, interpreter_data.sys_paths_count, interpreter_data.sys_paths);
CHECK_STATUS;
config.module_search_paths_set = 1;
config.optimization_level = 2;
config.write_bytecode = 0;
config.use_environment = 0;
config.user_site_directory = 0;
config.configure_c_stdio = 1;
config.isolated = 1;
status = PyConfig_SetString(&config, &config.program_name, interpreter_data.exe_path);
CHECK_STATUS;
#ifndef _WIN32
status = PyConfig_SetString(&config, &config.home, interpreter_data.python_home_path);
CHECK_STATUS;
#endif
status = PyConfig_SetString(&config, &config.run_module, L"site");
CHECK_STATUS;
#ifdef _WIN32
status = PyConfig_SetArgv(&config, interpreter_data.argc, interpreter_data.argv);
#else
status = PyConfig_SetBytesArgv(&config, interpreter_data.argc, interpreter_data.argv);
#endif
CHECK_STATUS;
if (interpreter_data.calibre_os_module) {
if (PyImport_AppendInittab("calibre_os_module", interpreter_data.calibre_os_module) == -1) {
fatal("Failed to add calibre_os_module to the init table");
}
}
status = Py_InitializeFromConfig(&config);
CHECK_STATUS;
set_sys_bool("gui_app", use_os_log); set_sys_bool("gui_app", use_os_log);
set_sys_bool("frozen", true); set_sys_bool("frozen", true);
set_sys_string("calibre_basename", interpreter_data.basename); set_sys_string("calibre_basename", interpreter_data.basename);
@ -265,17 +85,6 @@ run_interpreter() {
set_sys_string("frozen_path", interpreter_data.executables_path); set_sys_string("frozen_path", interpreter_data.executables_path);
#endif #endif
#ifdef _WIN32 int ret = bypy_run_interpreter();
code_page = GetConsoleOutputCP();
if (code_page != CP_UTF8) SetConsoleOutputCP(CP_UTF8);
setup_vt_terminal_mode();
#endif
int ret = Py_RunMain();
PyConfig_Clear(&config);
#ifdef _WIN32
cleanup_console_state();
#endif
exit(ret); exit(ret);
#undef CHECK_STATUS
} }