Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-06-23 15:30:45 -04:00
Code Issues Packages Projects Releases Wiki Activity

Get the linux frozen build working with py3

Browse Source
This commit is contained in:
Kovid Goyal 2019-11-18 18:31:54 +05:30
parent 520c46dfb9
commit ec167b7ac7
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
8 changed files with 147 additions and 289 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bypy/init_env.py
View File

@ -2,8 +2,6 @@
# vim:fileencoding=utf-8 # vim:fileencoding=utf-8
# License: GPL v3 Copyright: 2019, Kovid Goyal <kovid at kovidgoyal.net> # License: GPL v3 Copyright: 2019, Kovid Goyal <kovid at kovidgoyal.net>
import json import json
import os import os
import re import re

9
bypy/linux/__main__.py
View File

@ -2,8 +2,6 @@
# vim:fileencoding=utf-8 # vim:fileencoding=utf-8
# License: GPLv3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net> # License: GPLv3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
import errno import errno
import glob import glob
import os import os
@ -141,7 +139,6 @@ def copy_python(env, ext_dir):
srcdir = j(srcdir, 'site-packages') srcdir = j(srcdir, 'site-packages')
dest = j(env.py_dir, 'site-packages') dest = j(env.py_dir, 'site-packages')
import_site_packages(srcdir, dest) import_site_packages(srcdir, dest)
shutil.rmtree(j(dest, 'PyQt5/uic/port_v3'))
filter_pyqt = {x + '.so' for x in PYQT_MODULES} | {'sip.so'} filter_pyqt = {x + '.so' for x in PYQT_MODULES} | {'sip.so'}
pyqt = j(dest, 'PyQt5') pyqt = j(dest, 'PyQt5')
@ -176,7 +173,7 @@ def build_launchers(env):
base = self_dir base = self_dir
sources = [j(base, x) for x in ['util.c']] sources = [j(base, x) for x in ['util.c']]
objects = [j(env.obj_dir, os.path.basename(x) + '.o') for x in sources] objects = [j(env.obj_dir, os.path.basename(x) + '.o') for x in sources]
cflags = '-fno-strict-aliasing -W -Wall -c -O2 -pipe -DPYTHON_VER="python%s"' % py_ver cflags = '-fno-strict-aliasing -W -Wall -c -O2 -pipe -DPYTHON_VER=L"python%s"' % py_ver
cflags = cflags.split() + ['-I%s/include/python%s' % (PREFIX, py_ver)] cflags = cflags.split() + ['-I%s/include/python%s' % (PREFIX, py_ver)]
for src, obj in zip(sources, objects): for src, obj in zip(sources, objects):
cmd = ['gcc'] + cflags + ['-fPIC', '-o', obj, src] cmd = ['gcc'] + cflags + ['-fPIC', '-o', obj, src]
@ -204,8 +201,8 @@ def build_launchers(env):
xflags = list(cflags) xflags = list(cflags)
xflags.remove('-c') xflags.remove('-c')
xflags += ['-DGUI_APP=' + ('1' if typ == 'gui' else '0')] xflags += ['-DGUI_APP=' + ('1' if typ == 'gui' else '0')]
xflags += ['-DMODULE="%s"' % mod, '-DBASENAME="%s"' % bname, xflags += ['-DMODULE=L"%s"' % mod, '-DBASENAME=L"%s"' % bname,
'-DFUNCTION="%s"' % func] '-DFUNCTION=L"%s"' % func]
exe = j(env.bin_dir, bname) exe = j(env.bin_dir, bname)
cmd = ['gcc'] + xflags + [src, '-o', exe, '-L' + env.lib_dir, '-lcalibre-launcher'] cmd = ['gcc'] + xflags + [src, '-o', exe, '-L' + env.lib_dir, '-lcalibre-launcher']

4
bypy/linux/main.c
View File

@ -5,9 +5,7 @@
int main(int argc, char **argv) { int main(int argc, char **argv) {
int ret = 0; int ret = 0;
set_gui_app(GUI_APP); set_gui_app(GUI_APP);
ret = execute_python_entrypoint(argc, argv, BASENAME, MODULE, FUNCTION, NULL, NULL); ret = execute_python_entrypoint(argc, argv, BASENAME, MODULE, FUNCTION);
return ret; return ret;
} }

80
bypy/linux/site.py
View File

@ -2,53 +2,27 @@
# vim:fileencoding=utf-8 # vim:fileencoding=utf-8
# License: GPLv3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net> # License: GPLv3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
import sys import builtins
import encodings # noqa
import __builtin__
import locale
import os import os
import codecs import sys
import _sitebuiltins
def set_default_encoding(): def set_quit():
try: """Define new builtins 'quit' and 'exit'.
locale.setlocale(locale.LC_ALL, '')
except:
print ('WARNING: Failed to set default libc locale, using en_US.UTF-8')
locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')
try:
enc = locale.getdefaultlocale()[1]
except Exception:
enc = None
if not enc:
enc = locale.nl_langinfo(locale.CODESET)
if not enc or enc.lower() == 'ascii':
enc = 'UTF-8'
try:
enc = codecs.lookup(enc).name
except LookupError:
enc = 'UTF-8'
sys.setdefaultencoding(enc)
del sys.setdefaultencoding
These are objects which make the interpreter exit when called.
class _Helper(object): The repr of each object contains a hint at how it works.
"""Define the builtin 'help'.
This is a wrapper around pydoc.help (with a twist).
""" """
if os.sep == '\\':
eof = 'Ctrl-Z plus Return'
else:
eof = 'Ctrl-D (i.e. EOF)'
def __repr__(self): builtins.quit = _sitebuiltins.Quitter('quit', eof)
return "Type help() for interactive help, " \ builtins.exit = _sitebuiltins.Quitter('exit', eof)
"or help(object) for help about object."
def __call__(self, *args, **kwds):
import pydoc
return pydoc.help(*args, **kwds)
def set_helper():
__builtin__.help = _Helper()
def setup_openssl_environment(): def setup_openssl_environment():
@ -56,33 +30,29 @@ def setup_openssl_environment():
# out of their asses and implement a common location for SSL certificates. # out of their asses and implement a common location for SSL certificates.
# It's not that hard people, there exists a wonderful tool called the symlink # It's not that hard people, there exists a wonderful tool called the symlink
# See http://www.mobileread.com/forums/showthread.php?t=256095 # See http://www.mobileread.com/forums/showthread.php?t=256095
if b'SSL_CERT_FILE' not in os.environ and b'SSL_CERT_DIR' not in os.environ: if 'SSL_CERT_FILE' not in os.environ and 'SSL_CERT_DIR' not in os.environ:
if os.access('/etc/pki/tls/certs/ca-bundle.crt', os.R_OK): if os.access('/etc/pki/tls/certs/ca-bundle.crt', os.R_OK):
os.environ['SSL_CERT_FILE'] = '/etc/pki/tls/certs/ca-bundle.crt' os.environ['SSL_CERT_FILE'] = '/etc/pki/tls/certs/ca-bundle.crt'
elif os.path.isdir('/etc/ssl/certs'): elif os.path.isdir('/etc/ssl/certs'):
os.environ['SSL_CERT_DIR'] = '/etc/ssl/certs' os.environ['SSL_CERT_DIR'] = '/etc/ssl/certs'
def set_helper():
builtins.help = _sitebuiltins._Helper()
def main(): def main():
try:
sys.argv[0] = sys.calibre_basename sys.argv[0] = sys.calibre_basename
dfv = os.environ.get('CALIBRE_DEVELOP_FROM', None) dfv = os.environ.get('CALIBRE_DEVELOP_FROM')
if dfv and os.path.exists(dfv): if dfv and os.path.exists(dfv):
sys.path.insert(0, os.path.abspath(dfv)) sys.path.insert(0, os.path.abspath(dfv))
set_default_encoding()
set_helper() set_helper()
setup_openssl_environment() setup_openssl_environment()
set_quit()
mod = __import__(sys.calibre_module, fromlist=[1]) mod = __import__(sys.calibre_module, fromlist=[1])
func = getattr(mod, sys.calibre_function) func = getattr(mod, sys.calibre_function)
return func() return func()
except SystemExit as err:
if err.code is None:
return 0 if __name__ == '__main__':
if isinstance(err.code, int): main()
return err.code
print (err.code)
return 1
except:
import traceback
traceback.print_exc()
return 1

305
bypy/linux/util.c
View File

@ -5,97 +5,35 @@
#include <stdio.h> #include <stdio.h>
#include <errno.h> #include <errno.h>
static bool GUI_APP = False; #define arraysz(x) (sizeof(x)/sizeof(x[0]))
static char exe_path[PATH_MAX]; static bool GUI_APP = false;
static char base_dir[PATH_MAX]; static char exe_path_char[PATH_MAX];
static char bin_dir[PATH_MAX]; static wchar_t exe_path[PATH_MAX];
static char lib_dir[PATH_MAX]; static wchar_t base_dir[PATH_MAX];
static char extensions_dir[PATH_MAX]; static wchar_t bin_dir[PATH_MAX];
static char resources_dir[PATH_MAX]; static wchar_t lib_dir[PATH_MAX];
static wchar_t extensions_dir[PATH_MAX];
static wchar_t resources_dir[PATH_MAX];
void set_gui_app(bool yes) { GUI_APP = yes; } void set_gui_app(bool yes) { GUI_APP = yes; }
int report_error(const char *msg, int code) { static int
report_error(const char *msg, int code) {
fprintf(stderr, "%s\n", msg); fprintf(stderr, "%s\n", msg);
return code; return code;
} }
int report_libc_error(const char *msg) { static void
char buf[2000]; get_paths() {
int err = errno;
snprintf(buf, 2000, "%s::%s", msg, strerror(err));
return report_error(buf, err);
}
int pyobject_to_int(PyObject *res) {
int ret = 0; PyObject *tmp;
if (res != NULL) {
tmp = PyNumber_Int(res);
if (tmp == NULL) ret = (PyObject_IsTrue(res)) ? 1 : 0;
else ret = (int)PyInt_AS_LONG(tmp);
}
return ret;
}
int handle_sysexit(PyObject *e) {
PyObject *code;
code = PyObject_GetAttrString(e, "code");
if (!code) return 0;
return pyobject_to_int(code);
}
int report_python_error(const char *preamble, int code) {
PyObject *exc, *val, *tb, *str;
int ret, issysexit = 0; char *i, *buf;
if (!PyErr_Occurred()) return code;
issysexit = PyErr_ExceptionMatches(PyExc_SystemExit);
PyErr_Fetch(&exc, &val, &tb);
if (exc != NULL) {
PyErr_NormalizeException(&exc, &val, &tb);
if (issysexit) {
return (val) ? handle_sysexit(val) : 0;
}
if (val != NULL) {
str = PyObject_Unicode(val);
if (str == NULL) {
PyErr_Clear();
str = PyObject_Str(val);
}
i = PyString_AsString(str);
if (i == NULL) OOM;
buf = (char*)calloc(strlen(i)+strlen(preamble)+5, sizeof(char));
if (buf == NULL) OOM;
sprintf(buf, "%s::%s", preamble, i);
ret = report_error(buf, code);
if (buf) free(buf);
if (tb != NULL) {
PyErr_Restore(exc, val, tb);
PyErr_Print();
}
return ret;
}
}
return report_error(preamble, code);
}
static void get_paths()
{
char linkname[256]; /* /proc/<pid>/exe */ char linkname[256]; /* /proc/<pid>/exe */
char *p; wchar_t *p;
pid_t pid; pid_t pid;
int ret; int ret;
pid = getpid(); pid = getpid();
if (snprintf(linkname, sizeof(linkname), "/proc/%i/exe", pid) < 0) if (snprintf(linkname, sizeof(linkname), "/proc/%i/exe", pid) < 0) {
{
/* This should only happen on large word systems. I'm not sure /* This should only happen on large word systems. I'm not sure
what the proper response is here. what the proper response is here.
Since it really is an assert-like condition, aborting the Since it really is an assert-like condition, aborting the
@ -103,160 +41,117 @@ static void get_paths()
exit(report_error("PID too large", EXIT_FAILURE)); exit(report_error("PID too large", EXIT_FAILURE));
} }
ret = readlink(linkname, exe_path_char, sizeof(exe_path_char));
ret = readlink(linkname, exe_path, sizeof(exe_path));
if (ret == -1) { if (ret == -1) {
exit(report_error("Failed to read exe path.", EXIT_FAILURE)); exit(report_error("Failed to read exe path.", EXIT_FAILURE));
} }
if ((size_t)ret >= sizeof(exe_path_char)) {
if ((size_t)ret >= sizeof(exe_path)) {
exit(report_error("exe path buffer too small.", EXIT_FAILURE)); exit(report_error("exe path buffer too small.", EXIT_FAILURE));
} }
exe_path_char[ret] = 0;
size_t tsz;
wchar_t* temp = Py_DecodeLocale(exe_path_char, &tsz);
if (!temp) {
exit(report_error("Failed to decode exe path", EXIT_FAILURE));
}
memcpy(exe_path, temp, tsz * sizeof(wchar_t));
exe_path[tsz] = 0;
PyMem_RawFree(temp);
exe_path[ret] = 0; p = wcsrchr(exe_path, '/');
p = rindex(exe_path, '/');
if (p == NULL) { if (p == NULL) {
exit(report_error("No path separators in executable path", EXIT_FAILURE)); exit(report_error("No path separators in executable path", EXIT_FAILURE));
} }
strncat(base_dir, exe_path, p - exe_path); wcsncat(base_dir, exe_path, p - exe_path);
p = rindex(base_dir, '/'); p = wcsrchr(base_dir, '/');
if (p == NULL) { if (p == NULL) {
exit(report_error("Only one path separator in executable path", EXIT_FAILURE)); exit(report_error("Only one path separator in executable path", EXIT_FAILURE));
} }
*p = 0; *p = 0;
if (strlen(base_dir) == 0) { if (wcslen(base_dir) == 0) {
exit(report_error("base directory empty", EXIT_FAILURE)); exit(report_error("base directory empty", EXIT_FAILURE));
} }
snprintf(bin_dir, sizeof(bin_dir), "%s/bin", base_dir); swprintf(bin_dir, arraysz(bin_dir), L"%ls/bin", base_dir);
snprintf(lib_dir, sizeof(lib_dir), "%s/lib", base_dir); swprintf(lib_dir, arraysz(lib_dir), L"%ls/lib", base_dir);
snprintf(resources_dir, sizeof(resources_dir), "%s/resources", base_dir); swprintf(resources_dir, arraysz(resources_dir), L"%ls/resources", base_dir);
snprintf(extensions_dir, sizeof(extensions_dir), "%s/%s/site-packages/calibre/plugins", lib_dir, PYTHON_VER); swprintf(extensions_dir, arraysz(extensions_dir), L"%ls/%ls/site-packages/calibre/plugins", lib_dir, PYTHON_VER);
} }
static void
void setup_stream(const char *name, const char *errors) { set_sys_string(const char* key, const wchar_t* val) {
PyObject *stream; PyObject *temp = PyUnicode_FromWideChar(val, -1);
char buf[100]; if (temp) {
if (PySys_SetObject(key, temp) != 0) {
snprintf(buf, 20, "%s", name); exit(report_error("Failed to set attribute on sys", EXIT_FAILURE));
stream = PySys_GetObject(buf); }
Py_DECREF(temp);
snprintf(buf, 20, "%s", "utf-8"); } else {
snprintf(buf+21, 30, "%s", errors); exit(report_error("Failed to set attribute on sys, decode failed", EXIT_FAILURE));
if (!PyFile_SetEncodingAndErrors(stream, buf, buf+21))
exit(report_python_error("Failed to set stream encoding", 1));
}
void setup_streams() {
if (!GUI_APP) { // Remove buffering
setvbuf(stdin, NULL, _IONBF, 2);
setvbuf(stdout, NULL, _IONBF, 2);
setvbuf(stderr, NULL, _IONBF, 2);
} }
/*setup_stream("stdin", "strict");
setup_stream("stdout", "strict");
setup_stream("stderr", "strict");*/
} }
void initialize_interpreter(int argc, char **argv, char *outr, char *errr, static int
const char *basename, const char *module, const char *function) { initialize_interpreter(int argc, char * const *argv, const wchar_t *basename, const wchar_t *module, const wchar_t *function) {
char *encoding, *p; PyStatus status;
PyPreConfig preconfig;
PyConfig config;
PyPreConfig_InitIsolatedConfig(&preconfig);
preconfig.utf8_mode = 1;
preconfig.coerce_c_locale = 1;
preconfig.isolated = 1;
#define CHECK_STATUS if (PyStatus_Exception(status)) { PyConfig_Clear(&config); Py_ExitStatusException(status); return 1; }
status = Py_PreInitialize(&preconfig);
CHECK_STATUS;
PyConfig_InitIsolatedConfig(&config);
get_paths(); get_paths();
char path[3*PATH_MAX]; static wchar_t* items[3];
static wchar_t path[arraysz(items)*PATH_MAX];
for (size_t i = 0; i < arraysz(items); i++) items[i] = path + i * PATH_MAX;
swprintf(items[0], PATH_MAX, L"%ls/%ls", lib_dir, PYTHON_VER);
swprintf(items[1], PATH_MAX, L"%ls/%ls/lib-dynload", lib_dir, PYTHON_VER);
swprintf(items[2], PATH_MAX, L"%ls/%ls/site-packages", lib_dir, PYTHON_VER);
status = PyConfig_SetWideStringList(&config, &config.module_search_paths, arraysz(items), items);
CHECK_STATUS;
config.module_search_paths_set = 1;
config.optimization_level = 2;
config.write_bytecode = 0;
config.use_environment = 0;
config.user_site_directory = 0;
config.configure_c_stdio = 1;
config.isolated = 1;
snprintf(path, sizeof(path), status = PyConfig_SetString(&config, &config.program_name, exe_path);
"%s/%s:%s/%s/plat-linux2:%s/%s/lib-dynload:%s/%s/site-packages", CHECK_STATUS;
lib_dir, PYTHON_VER, lib_dir, PYTHON_VER, lib_dir, PYTHON_VER, status = PyConfig_SetString(&config, &config.home, base_dir);
lib_dir, PYTHON_VER); CHECK_STATUS;
status = PyConfig_SetString(&config, &config.run_module, L"site");
CHECK_STATUS;
status = PyConfig_SetBytesArgv(&config, argc, argv);
CHECK_STATUS;
status = Py_InitializeFromConfig(&config);
CHECK_STATUS;
#undef CHECK_STATUS
Py_OptimizeFlag = 2; PySys_SetObject("gui_app", GUI_APP ? Py_True : Py_False);
Py_NoSiteFlag = 1;
Py_DontWriteBytecodeFlag = 1;
Py_IgnoreEnvironmentFlag = 1;
Py_NoUserSiteDirectory = 1;
Py_VerboseFlag = 0;
Py_DebugFlag = 0;
Py_HashRandomizationFlag = 1;
Py_SetProgramName(exe_path);
Py_SetPythonHome(base_dir);
//printf("Path before Py_Initialize(): %s\r\n\n", Py_GetPath());
Py_Initialize();
if (!Py_FileSystemDefaultEncoding) {
encoding = getenv("PYTHONIOENCODING");
if (encoding != NULL) {
Py_FileSystemDefaultEncoding = strndup(encoding, 20);
p = index(Py_FileSystemDefaultEncoding, ':');
if (p != NULL) *p = 0;
} else
Py_FileSystemDefaultEncoding = strndup("UTF-8", 10);
}
setup_streams();
PySys_SetArgv(argc, argv);
//printf("Path after Py_Initialize(): %s\r\n\n", Py_GetPath());
PySys_SetPath(path);
//printf("Path set by me: %s\r\n\n", path);
PySys_SetObject("gui_app", PyBool_FromLong((long)GUI_APP));
PySys_SetObject("calibre_basename", PyBytes_FromString(basename));
PySys_SetObject("calibre_module", PyBytes_FromString(module));
PySys_SetObject("calibre_function", PyBytes_FromString(function));
PySys_SetObject("extensions_location", PyBytes_FromString(extensions_dir));
PySys_SetObject("resources_location", PyBytes_FromString(resources_dir));
PySys_SetObject("executables_location", PyBytes_FromString(base_dir));
PySys_SetObject("frozen_path", PyBytes_FromString(base_dir));
PySys_SetObject("frozen", Py_True); PySys_SetObject("frozen", Py_True);
Py_INCREF(Py_True); set_sys_string("calibre_basename", basename);
set_sys_string("calibre_module", module);
set_sys_string("calibre_function", function);
set_sys_string("extensions_location", extensions_dir);
set_sys_string("resources_location", resources_dir);
set_sys_string("executables_location", base_dir);
set_sys_string("frozen_path", base_dir);
int ret = Py_RunMain();
if (GUI_APP && outr && errr) { PyConfig_Clear(&config);
// PySys_SetObject("stdout_redirect", PyUnicode_FromWideChar(outr, wcslen(outr)));
// PySys_SetObject("stderr_redirect", PyUnicode_FromWideChar(errr, wcslen(outr)));
}
}
int execute_python_entrypoint(int argc, char **argv, const char *basename, const char *module, const char *function,
char *outr, char *errr) {
PyObject *site, *pmain, *res;
int ret = 0;
initialize_interpreter(argc, argv, outr, errr, basename, module, function);
site = PyImport_ImportModule("site");
if (site == NULL)
ret = report_python_error("Failed to import site module", 1);
else {
Py_XINCREF(site);
pmain = PyObject_GetAttrString(site, "main");
if (pmain == NULL || !PyCallable_Check(pmain))
ret = report_python_error("site module has no main function", 1);
else {
Py_XINCREF(pmain);
res = PyObject_CallObject(pmain, NULL);
if (res == NULL)
ret = report_python_error("Python function terminated unexpectedly", 1);
ret = pyobject_to_int(res);
}
}
PyErr_Clear();
Py_Finalize();
//printf("11111 Returning: %d\r\n", ret);
return ret; return ret;
} }
int
execute_python_entrypoint(int argc, char * const *argv, const wchar_t *basename, const wchar_t *module, const wchar_t *function) {
return initialize_interpreter(argc, argv, basename, module, function);
}

8
bypy/linux/util.h
View File

@ -9,10 +9,10 @@
#define OOM exit(report_error("Out of memory", EXIT_FAILURE)) #define OOM exit(report_error("Out of memory", EXIT_FAILURE))
#define True 1 #define True 1
#define False 0 #define False 0
typedef int bool; #include <wchar.h>
#include <stdbool.h>
void set_gui_app(bool yes); void set_gui_app(bool yes);
int execute_python_entrypoint(int argc, char **argv, const char *basename, int execute_python_entrypoint(int argc, char * const *argv, const wchar_t *basename,
const char *module, const char *function, const wchar_t *module, const wchar_t *function);
char *outr, char *errr);

4
setup/build.py
View File

@ -218,8 +218,8 @@ def init_env():
class Build(Command): class Build(Command):
short_description = 'Build calibre C/C++ extension modules' short_description = 'Build calibre C/C++ extension modules'
DEFAULT_OUTPUTDIR = os.path.abspath(os.path.join(SRC, 'calibre', 'plugins', str(sys.version_info.major))) DEFAULT_OUTPUTDIR = os.path.abspath(os.path.join(SRC, 'calibre', 'plugins'))
DEFAULT_BUILDDIR = os.path.abspath(os.path.join(os.path.dirname(SRC), 'build', str(sys.version_info.major))) DEFAULT_BUILDDIR = os.path.abspath(os.path.join(os.path.dirname(SRC), 'build'))
description = textwrap.dedent('''\ description = textwrap.dedent('''\
calibre depends on several python extensions written in C/C++. calibre depends on several python extensions written in C/C++.

2
src/calibre/constants.py
View File

@ -150,7 +150,7 @@ def cache_dir():
return ans return ans
plugins_loc = os.path.join(sys.extensions_location, str(sys.version_info.major)) plugins_loc = sys.extensions_location
# plugins {{{ # plugins {{{