diff --git a/bypy/linux/__main__.py b/bypy/linux/__main__.py index ccc04771aa..4459625a4e 100644 --- a/bypy/linux/__main__.py +++ b/bypy/linux/__main__.py @@ -2,18 +2,22 @@ # vim:fileencoding=utf-8 # License: GPLv3 Copyright: 2016, Kovid Goyal +from __future__ import print_function + import errno import glob import os import shutil import stat import subprocess +import sys import tarfile import time from functools import partial from bypy.constants import ( - PREFIX, SRC as CALIBRE_DIR, SW, is64bit, python_major_minor_version + LIBDIR, PREFIX, PYTHON, SRC as CALIBRE_DIR, SW, build_dir, is64bit, + python_major_minor_version, worker_env ) from bypy.pkgs.qt import PYQT_MODULES, QT_DLLS, QT_PLUGINS from bypy.utils import ( @@ -28,6 +32,7 @@ arch = 'x86_64' if is64bit else 'i686' py_ver = '.'.join(map(str, python_major_minor_version())) QT_PREFIX = os.path.join(PREFIX, 'qt') calibre_constants = globals()['init_env']['calibre_constants'] +qt_get_dll_path = partial(get_dll_path, loc=os.path.join(QT_PREFIX, 'lib')) def binary_includes(): @@ -50,8 +55,7 @@ def binary_includes(): # distros do not have libstdc++.so.6, so it should be safe to leave it out. # https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html (The current # debian stable libstdc++ is libstdc++.so.6.0.17) - ] + [ - j(QT_PREFIX, 'lib', 'lib%s.so.5' % x) for x in QT_DLLS] + ] + list(map(qt_get_dll_path, QT_DLLS)) class Env(object): @@ -276,17 +280,35 @@ def create_tarfile(env, compression_level='9'): def run_tests(path_to_calibre_debug, cwd_on_failure): + env = os.environ.copy() + env['LD_LIBRARY_PATH'] = LIBDIR p = subprocess.Popen([path_to_calibre_debug, '--test-build']) if p.wait() != 0: os.chdir(cwd_on_failure) + print('running calibre build tests failed', file=sys.stderr) run_shell() raise SystemExit(p.wait()) +def build_extensions(env, ext_dir): + wenv = os.environ.copy() + wenv.update(worker_env) + wenv['LD_LIBRARY_PATH'] = LIBDIR + wenv['QMAKE'] = os.path.join(QT_PREFIX, 'bin', 'qmake') + wenv['SW'] = PREFIX + p = subprocess.Popen([PYTHON, 'setup.py', 'build', '--build-dir=' + build_dir(), '--output-dir=' + ext_dir], env=wenv, cwd=CALIBRE_DIR) + if p.wait() != 0: + os.chdir(CALIBRE_DIR) + print('building calibre extensions failed', file=sys.stderr) + run_shell() + raise SystemExit(p.returncode) + + def main(): args = globals()['args'] ext_dir = globals()['ext_dir'] env = Env() + build_extensions(env, ext_dir) copy_libs(env) copy_python(env, ext_dir) build_launchers(env) diff --git a/bypy/linux/launcher.c b/bypy/linux/launcher.c new file mode 100644 index 0000000000..f25abbcfb6 --- /dev/null +++ b/bypy/linux/launcher.c @@ -0,0 +1,53 @@ +/* + * launcher.c + * Copyright (C) 2014 Kovid Goyal + * + * Distributed under terms of the GPL3 license. + */ + +#include +#include +#include +#include +#include +#include + +#define PATHLEN 1023 + +#define SET(x, y) if (setenv(x, y, 1) != 0) { fprintf(stderr, "Failed to set environment variable with error: %s\n", strerror(errno)); return 1; } + +int main(int argc, char **argv) { + static char buf[PATHLEN+1] = {0}, lib[PATHLEN+1] = {0}, base[PATHLEN+1] = {0}, exe[PATHLEN+1] = {0}, *ldp = NULL; + + if (readlink("/proc/self/exe", buf, PATHLEN) == -1) { + fprintf(stderr, "Failed to read path of executable with error: %s\n", strerror(errno)); + return 1; + } + strncpy(lib, buf, PATHLEN); + strncpy(base, dirname(lib), PATHLEN); + snprintf(exe, PATHLEN, "%s/bin/%s", base, basename(buf)); + memset(lib, 0, PATHLEN); + snprintf(lib, PATHLEN, "%s/lib", base); + + /* qt-at-spi causes crashes and performance issues in various distros, so disable it */ + SET("QT_ACCESSIBILITY", "0") + memset(buf, 0, PATHLEN); + ldp = getenv("QT_PLUGIN_PATH"); + if (ldp == NULL) snprintf(buf, PATHLEN, "%s/qt_plugins", lib); + else snprintf(buf, PATHLEN, "%s/qt_plugins:%s", lib, ldp); + SET("QT_PLUGIN_PATH", buf); + + memset(buf, 0, PATHLEN); + ldp = getenv("LD_LIBRARY_PATH"); + if (ldp == NULL) strncpy(buf, lib, PATHLEN); + else snprintf(buf, PATHLEN, "%s:%s", lib, ldp); + SET("LD_LIBRARY_PATH", buf) + + argv[0] = exe; + if (execv(exe, argv) == -1) { + fprintf(stderr, "Failed to execute binary: %s with error: %s\n", exe, strerror(errno)); + return 1; + } + + return 0; +} diff --git a/bypy/linux/main.c b/bypy/linux/main.c new file mode 100644 index 0000000000..7bb32db5df --- /dev/null +++ b/bypy/linux/main.c @@ -0,0 +1,13 @@ +#include "util.h" + +#include + +int main(int argc, char **argv) { + int ret = 0; + set_gui_app(GUI_APP); + ret = execute_python_entrypoint(argc, argv, BASENAME, MODULE, FUNCTION, NULL, NULL); + + return ret; +} + + diff --git a/bypy/linux/site.py b/bypy/linux/site.py new file mode 100644 index 0000000000..bf8f5a3e69 --- /dev/null +++ b/bypy/linux/site.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python2 +# vim:fileencoding=utf-8 +# License: GPLv3 Copyright: 2016, Kovid Goyal + +import sys +import encodings # noqa +import __builtin__ +import locale +import os +import codecs + + +def set_default_encoding(): + try: + locale.setlocale(locale.LC_ALL, '') + except: + print ('WARNING: Failed to set default libc locale, using en_US.UTF-8') + locale.setlocale(locale.LC_ALL, 'en_US.UTF-8') + try: + enc = locale.getdefaultlocale()[1] + except Exception: + enc = None + if not enc: + enc = locale.nl_langinfo(locale.CODESET) + if not enc or enc.lower() == 'ascii': + enc = 'UTF-8' + try: + enc = codecs.lookup(enc).name + except LookupError: + enc = 'UTF-8' + sys.setdefaultencoding(enc) + del sys.setdefaultencoding + + +class _Helper(object): + """Define the builtin 'help'. + This is a wrapper around pydoc.help (with a twist). + + """ + + def __repr__(self): + return "Type help() for interactive help, " \ + "or help(object) for help about object." + + def __call__(self, *args, **kwds): + import pydoc + return pydoc.help(*args, **kwds) + + +def set_helper(): + __builtin__.help = _Helper() + + +def setup_openssl_environment(): + # Workaround for Linux distros that have still failed to get their heads + # out of their asses and implement a common location for SSL certificates. + # It's not that hard people, there exists a wonderful tool called the symlink + # See http://www.mobileread.com/forums/showthread.php?t=256095 + if b'SSL_CERT_FILE' not in os.environ and b'SSL_CERT_DIR' not in os.environ: + if os.access('/etc/pki/tls/certs/ca-bundle.crt', os.R_OK): + os.environ['SSL_CERT_FILE'] = '/etc/pki/tls/certs/ca-bundle.crt' + elif os.path.isdir('/etc/ssl/certs'): + os.environ['SSL_CERT_DIR'] = '/etc/ssl/certs' + + +def main(): + try: + sys.argv[0] = sys.calibre_basename + dfv = os.environ.get('CALIBRE_DEVELOP_FROM', None) + if dfv and os.path.exists(dfv): + sys.path.insert(0, os.path.abspath(dfv)) + set_default_encoding() + set_helper() + setup_openssl_environment() + mod = __import__(sys.calibre_module, fromlist=[1]) + func = getattr(mod, sys.calibre_function) + return func() + except SystemExit as err: + if err.code is None: + return 0 + if isinstance(err.code, int): + return err.code + print (err.code) + return 1 + except: + import traceback + traceback.print_exc() + return 1 diff --git a/bypy/linux/util.c b/bypy/linux/util.c new file mode 100644 index 0000000000..fcb4004a63 --- /dev/null +++ b/bypy/linux/util.c @@ -0,0 +1,262 @@ +#include "util.h" +#include +#include +#include +#include +#include + +static bool GUI_APP = False; + +static char exe_path[PATH_MAX]; +static char base_dir[PATH_MAX]; +static char bin_dir[PATH_MAX]; +static char lib_dir[PATH_MAX]; +static char extensions_dir[PATH_MAX]; +static char resources_dir[PATH_MAX]; + +void set_gui_app(bool yes) { GUI_APP = yes; } + +int report_error(const char *msg, int code) { + fprintf(stderr, "%s\n", msg); + return code; +} + +int report_libc_error(const char *msg) { + char buf[2000]; + int err = errno; + + snprintf(buf, 2000, "%s::%s", msg, strerror(err)); + return report_error(buf, err); +} + +int pyobject_to_int(PyObject *res) { + int ret = 0; PyObject *tmp; + if (res != NULL) { + tmp = PyNumber_Int(res); + if (tmp == NULL) ret = (PyObject_IsTrue(res)) ? 1 : 0; + else ret = (int)PyInt_AS_LONG(tmp); + } + return ret; +} + +int handle_sysexit(PyObject *e) { + PyObject *code; + + code = PyObject_GetAttrString(e, "code"); + if (!code) return 0; + return pyobject_to_int(code); +} + +int report_python_error(const char *preamble, int code) { + PyObject *exc, *val, *tb, *str; + int ret, issysexit = 0; char *i, *buf; + + if (!PyErr_Occurred()) return code; + issysexit = PyErr_ExceptionMatches(PyExc_SystemExit); + + PyErr_Fetch(&exc, &val, &tb); + + if (exc != NULL) { + PyErr_NormalizeException(&exc, &val, &tb); + + if (issysexit) { + return (val) ? handle_sysexit(val) : 0; + } + if (val != NULL) { + str = PyObject_Unicode(val); + if (str == NULL) { + PyErr_Clear(); + str = PyObject_Str(val); + } + i = PyString_AsString(str); + if (i == NULL) OOM; + buf = (char*)calloc(strlen(i)+strlen(preamble)+5, sizeof(char)); + if (buf == NULL) OOM; + sprintf(buf, "%s::%s", preamble, i); + ret = report_error(buf, code); + if (buf) free(buf); + if (tb != NULL) { + PyErr_Restore(exc, val, tb); + PyErr_Print(); + } + return ret; + } + } + return report_error(preamble, code); +} + +static void get_paths() +{ + char linkname[256]; /* /proc//exe */ + char *p; + pid_t pid; + int ret; + + pid = getpid(); + + if (snprintf(linkname, sizeof(linkname), "/proc/%i/exe", pid) < 0) + { + /* This should only happen on large word systems. I'm not sure + what the proper response is here. + Since it really is an assert-like condition, aborting the + program seems to be in order. */ + exit(report_error("PID too large", EXIT_FAILURE)); + } + + + ret = readlink(linkname, exe_path, sizeof(exe_path)); + + if (ret == -1) { + exit(report_error("Failed to read exe path.", EXIT_FAILURE)); + } + + if ((size_t)ret >= sizeof(exe_path)) { + exit(report_error("exe path buffer too small.", EXIT_FAILURE)); + } + + exe_path[ret] = 0; + + p = rindex(exe_path, '/'); + + if (p == NULL) { + exit(report_error("No path separators in executable path", EXIT_FAILURE)); + } + strncat(base_dir, exe_path, p - exe_path); + p = rindex(base_dir, '/'); + if (p == NULL) { + exit(report_error("Only one path separator in executable path", EXIT_FAILURE)); + } + *p = 0; + if (strlen(base_dir) == 0) { + exit(report_error("base directory empty", EXIT_FAILURE)); + } + + snprintf(bin_dir, sizeof(bin_dir), "%s/bin", base_dir); + snprintf(lib_dir, sizeof(lib_dir), "%s/lib", base_dir); + snprintf(resources_dir, sizeof(resources_dir), "%s/resources", base_dir); + snprintf(extensions_dir, sizeof(extensions_dir), "%s/%s/site-packages/calibre/plugins", lib_dir, PYTHON_VER); +} + + +void setup_stream(const char *name, const char *errors) { + PyObject *stream; + char buf[100]; + + snprintf(buf, 20, "%s", name); + stream = PySys_GetObject(buf); + + snprintf(buf, 20, "%s", "utf-8"); + snprintf(buf+21, 30, "%s", errors); + + if (!PyFile_SetEncodingAndErrors(stream, buf, buf+21)) + exit(report_python_error("Failed to set stream encoding", 1)); + +} + +void setup_streams() { + if (!GUI_APP) { // Remove buffering + setvbuf(stdin, NULL, _IONBF, 2); + setvbuf(stdout, NULL, _IONBF, 2); + setvbuf(stderr, NULL, _IONBF, 2); + } + + /*setup_stream("stdin", "strict"); + setup_stream("stdout", "strict"); + setup_stream("stderr", "strict");*/ +} + +void initialize_interpreter(int argc, char **argv, char *outr, char *errr, + const char *basename, const char *module, const char *function) { + char *encoding, *p; + + get_paths(); + char path[3*PATH_MAX]; + + snprintf(path, sizeof(path), + "%s/%s:%s/%s/plat-linux2:%s/%s/lib-dynload:%s/%s/site-packages", + lib_dir, PYTHON_VER, lib_dir, PYTHON_VER, lib_dir, PYTHON_VER, + lib_dir, PYTHON_VER); + + Py_OptimizeFlag = 2; + Py_NoSiteFlag = 1; + Py_DontWriteBytecodeFlag = 1; + Py_IgnoreEnvironmentFlag = 1; + Py_NoUserSiteDirectory = 1; + Py_VerboseFlag = 0; + Py_DebugFlag = 0; + Py_HashRandomizationFlag = 1; + + Py_SetProgramName(exe_path); + Py_SetPythonHome(base_dir); + + //printf("Path before Py_Initialize(): %s\r\n\n", Py_GetPath()); + Py_Initialize(); + if (!Py_FileSystemDefaultEncoding) { + encoding = getenv("PYTHONIOENCODING"); + if (encoding != NULL) { + Py_FileSystemDefaultEncoding = strndup(encoding, 20); + p = index(Py_FileSystemDefaultEncoding, ':'); + if (p != NULL) *p = 0; + } else + Py_FileSystemDefaultEncoding = strndup("UTF-8", 10); + } + + + setup_streams(); + + PySys_SetArgv(argc, argv); + //printf("Path after Py_Initialize(): %s\r\n\n", Py_GetPath()); + PySys_SetPath(path); + //printf("Path set by me: %s\r\n\n", path); + PySys_SetObject("gui_app", PyBool_FromLong((long)GUI_APP)); + PySys_SetObject("calibre_basename", PyBytes_FromString(basename)); + PySys_SetObject("calibre_module", PyBytes_FromString(module)); + PySys_SetObject("calibre_function", PyBytes_FromString(function)); + PySys_SetObject("extensions_location", PyBytes_FromString(extensions_dir)); + PySys_SetObject("resources_location", PyBytes_FromString(resources_dir)); + PySys_SetObject("executables_location", PyBytes_FromString(base_dir)); + PySys_SetObject("frozen_path", PyBytes_FromString(base_dir)); + PySys_SetObject("frozen", Py_True); + Py_INCREF(Py_True); + + + if (GUI_APP && outr && errr) { + // PySys_SetObject("stdout_redirect", PyUnicode_FromWideChar(outr, wcslen(outr))); + // PySys_SetObject("stderr_redirect", PyUnicode_FromWideChar(errr, wcslen(outr))); + } + +} + +int execute_python_entrypoint(int argc, char **argv, const char *basename, const char *module, const char *function, + char *outr, char *errr) { + PyObject *site, *pmain, *res; + int ret = 0; + + initialize_interpreter(argc, argv, outr, errr, basename, module, function); + + site = PyImport_ImportModule("site"); + + if (site == NULL) + ret = report_python_error("Failed to import site module", 1); + else { + Py_XINCREF(site); + + pmain = PyObject_GetAttrString(site, "main"); + if (pmain == NULL || !PyCallable_Check(pmain)) + ret = report_python_error("site module has no main function", 1); + else { + Py_XINCREF(pmain); + res = PyObject_CallObject(pmain, NULL); + + if (res == NULL) + ret = report_python_error("Python function terminated unexpectedly", 1); + + ret = pyobject_to_int(res); + } + } + PyErr_Clear(); + Py_Finalize(); + + //printf("11111 Returning: %d\r\n", ret); + return ret; +} diff --git a/bypy/linux/util.h b/bypy/linux/util.h new file mode 100644 index 0000000000..de4c97870d --- /dev/null +++ b/bypy/linux/util.h @@ -0,0 +1,18 @@ +#pragma once + +#define UNICODE + +#ifndef PATH_MAX +#define PATH_MAX 4096 +#endif + +#define OOM exit(report_error("Out of memory", EXIT_FAILURE)) +#define True 1 +#define False 0 +typedef int bool; + +void set_gui_app(bool yes); + +int execute_python_entrypoint(int argc, char **argv, const char *basename, + const char *module, const char *function, + char *outr, char *errr);