More work on porting Linux freeze

bypy/linux/__main__.py

@@ -2,18 +2,22 @@
 # vim:fileencoding=utf-8
 # License: GPLv3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
 
+from __future__ import print_function
+
 import errno
 import glob
 import os
 import shutil
 import stat
 import subprocess
+import sys
 import tarfile
 import time
 from functools import partial
 
 from bypy.constants import (
-    PREFIX, SRC as CALIBRE_DIR, SW, is64bit, python_major_minor_version
+    LIBDIR, PREFIX, PYTHON, SRC as CALIBRE_DIR, SW, build_dir, is64bit,
+    python_major_minor_version, worker_env
 )
 from bypy.pkgs.qt import PYQT_MODULES, QT_DLLS, QT_PLUGINS
 from bypy.utils import (
@@ -28,6 +32,7 @@ arch = 'x86_64' if is64bit else 'i686'
 py_ver = '.'.join(map(str, python_major_minor_version()))
 QT_PREFIX = os.path.join(PREFIX, 'qt')
 calibre_constants = globals()['init_env']['calibre_constants']
+qt_get_dll_path = partial(get_dll_path, loc=os.path.join(QT_PREFIX, 'lib'))
 
 
 def binary_includes():
@@ -50,8 +55,7 @@ def binary_includes():
         # distros do not have libstdc++.so.6, so it should be safe to leave it out.
         # https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html (The current
         # debian stable libstdc++ is  libstdc++.so.6.0.17)
-    ] + [
+    ] + list(map(qt_get_dll_path, QT_DLLS))
-        j(QT_PREFIX, 'lib', 'lib%s.so.5' % x) for x in QT_DLLS]
 
 
 class Env(object):
@@ -276,17 +280,35 @@ def create_tarfile(env, compression_level='9'):
 
 
 def run_tests(path_to_calibre_debug, cwd_on_failure):
+    env = os.environ.copy()
+    env['LD_LIBRARY_PATH'] = LIBDIR
     p = subprocess.Popen([path_to_calibre_debug, '--test-build'])
     if p.wait() != 0:
         os.chdir(cwd_on_failure)
+        print('running calibre build tests failed', file=sys.stderr)
         run_shell()
         raise SystemExit(p.wait())
 
 
+def build_extensions(env, ext_dir):
+    wenv = os.environ.copy()
+    wenv.update(worker_env)
+    wenv['LD_LIBRARY_PATH'] = LIBDIR
+    wenv['QMAKE'] = os.path.join(QT_PREFIX, 'bin', 'qmake')
+    wenv['SW'] = PREFIX
+    p = subprocess.Popen([PYTHON, 'setup.py', 'build', '--build-dir=' + build_dir(), '--output-dir=' + ext_dir], env=wenv, cwd=CALIBRE_DIR)
+    if p.wait() != 0:
+        os.chdir(CALIBRE_DIR)
+        print('building calibre extensions failed', file=sys.stderr)
+        run_shell()
+        raise SystemExit(p.returncode)
+
+
 def main():
     args = globals()['args']
     ext_dir = globals()['ext_dir']
     env = Env()
+    build_extensions(env, ext_dir)
     copy_libs(env)
     copy_python(env, ext_dir)
     build_launchers(env)

bypy/linux/launcher.c

@@ -0,0 +1,53 @@
+/*
+ * launcher.c
+ * Copyright (C) 2014 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the GPL3 license.
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include <libgen.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#define PATHLEN 1023
+
+#define SET(x, y) if (setenv(x, y, 1) != 0) { fprintf(stderr, "Failed to set environment variable with error: %s\n", strerror(errno)); return 1; }
+
+int main(int argc, char **argv) {
+    static char buf[PATHLEN+1] = {0}, lib[PATHLEN+1] = {0}, base[PATHLEN+1] = {0}, exe[PATHLEN+1] = {0}, *ldp = NULL;
+
+    if (readlink("/proc/self/exe", buf, PATHLEN) == -1) {
+        fprintf(stderr, "Failed to read path of executable with error: %s\n", strerror(errno));
+        return 1;
+    }
+    strncpy(lib, buf, PATHLEN);
+    strncpy(base, dirname(lib), PATHLEN);
+    snprintf(exe, PATHLEN, "%s/bin/%s", base, basename(buf));
+    memset(lib, 0, PATHLEN);
+    snprintf(lib, PATHLEN, "%s/lib", base);
+
+    /* qt-at-spi causes crashes and performance issues in various distros, so disable it */
+    SET("QT_ACCESSIBILITY", "0")
+    memset(buf, 0, PATHLEN);
+    ldp = getenv("QT_PLUGIN_PATH");
+    if (ldp == NULL) snprintf(buf, PATHLEN, "%s/qt_plugins", lib);
+    else snprintf(buf, PATHLEN, "%s/qt_plugins:%s", lib, ldp);
+    SET("QT_PLUGIN_PATH", buf);
+
+    memset(buf, 0, PATHLEN);
+    ldp = getenv("LD_LIBRARY_PATH");
+    if (ldp == NULL) strncpy(buf, lib, PATHLEN);
+    else snprintf(buf, PATHLEN, "%s:%s", lib, ldp);
+    SET("LD_LIBRARY_PATH", buf)
+
+    argv[0] = exe;
+    if (execv(exe, argv) == -1) {
+        fprintf(stderr, "Failed to execute binary: %s with error: %s\n", exe, strerror(errno));
+        return 1;
+    }
+
+    return 0;
+}

bypy/linux/main.c

@@ -0,0 +1,13 @@
+#include "util.h"
+
+#include <stdlib.h>
+
+int main(int argc, char **argv) {
+    int ret = 0;
+    set_gui_app(GUI_APP);
+	ret = execute_python_entrypoint(argc, argv, BASENAME, MODULE, FUNCTION, NULL, NULL);
+
+    return ret;
+}
+
+

bypy/linux/site.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python2
+# vim:fileencoding=utf-8
+# License: GPLv3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
+
+import sys
+import encodings  # noqa
+import __builtin__
+import locale
+import os
+import codecs
+
+
+def set_default_encoding():
+    try:
+        locale.setlocale(locale.LC_ALL, '')
+    except:
+        print ('WARNING: Failed to set default libc locale, using en_US.UTF-8')
+        locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')
+    try:
+        enc = locale.getdefaultlocale()[1]
+    except Exception:
+        enc = None
+    if not enc:
+        enc = locale.nl_langinfo(locale.CODESET)
+    if not enc or enc.lower() == 'ascii':
+        enc = 'UTF-8'
+    try:
+        enc = codecs.lookup(enc).name
+    except LookupError:
+        enc = 'UTF-8'
+    sys.setdefaultencoding(enc)
+    del sys.setdefaultencoding
+
+
+class _Helper(object):
+    """Define the builtin 'help'.
+    This is a wrapper around pydoc.help (with a twist).
+
+    """
+
+    def __repr__(self):
+        return "Type help() for interactive help, " \
+            "or help(object) for help about object."
+
+    def __call__(self, *args, **kwds):
+        import pydoc
+        return pydoc.help(*args, **kwds)
+
+
+def set_helper():
+    __builtin__.help = _Helper()
+
+
+def setup_openssl_environment():
+    # Workaround for Linux distros that have still failed to get their heads
+    # out of their asses and implement a common location for SSL certificates.
+    # It's not that hard people, there exists a wonderful tool called the symlink
+    # See http://www.mobileread.com/forums/showthread.php?t=256095
+    if b'SSL_CERT_FILE' not in os.environ and b'SSL_CERT_DIR' not in os.environ:
+        if os.access('/etc/pki/tls/certs/ca-bundle.crt', os.R_OK):
+            os.environ['SSL_CERT_FILE'] = '/etc/pki/tls/certs/ca-bundle.crt'
+        elif os.path.isdir('/etc/ssl/certs'):
+            os.environ['SSL_CERT_DIR'] = '/etc/ssl/certs'
+
+
+def main():
+    try:
+        sys.argv[0] = sys.calibre_basename
+        dfv = os.environ.get('CALIBRE_DEVELOP_FROM', None)
+        if dfv and os.path.exists(dfv):
+            sys.path.insert(0, os.path.abspath(dfv))
+        set_default_encoding()
+        set_helper()
+        setup_openssl_environment()
+        mod = __import__(sys.calibre_module, fromlist=[1])
+        func = getattr(mod, sys.calibre_function)
+        return func()
+    except SystemExit as err:
+        if err.code is None:
+            return 0
+        if isinstance(err.code, int):
+            return err.code
+        print (err.code)
+        return 1
+    except:
+        import traceback
+        traceback.print_exc()
+    return 1

bypy/linux/util.c

@@ -0,0 +1,262 @@
+#include "util.h"
+#include <Python.h>
+#include <stdlib.h>
+#include <strings.h>
+#include <stdio.h>
+#include <errno.h>
+
+static bool GUI_APP = False;
+
+static char exe_path[PATH_MAX];
+static char base_dir[PATH_MAX];
+static char bin_dir[PATH_MAX];
+static char lib_dir[PATH_MAX];
+static char extensions_dir[PATH_MAX];
+static char resources_dir[PATH_MAX];
+
+void set_gui_app(bool yes) { GUI_APP = yes; }
+
+int report_error(const char *msg, int code) {
+    fprintf(stderr, "%s\n", msg);
+    return code;
+}
+
+int report_libc_error(const char *msg) {
+    char buf[2000];
+    int err = errno;
+
+    snprintf(buf, 2000, "%s::%s", msg, strerror(err));
+    return report_error(buf, err);
+}
+
+int pyobject_to_int(PyObject *res) {
+    int ret = 0; PyObject *tmp;
+    if (res != NULL) {
+        tmp = PyNumber_Int(res);
+        if (tmp == NULL) ret = (PyObject_IsTrue(res)) ? 1 : 0;
+        else ret = (int)PyInt_AS_LONG(tmp);
+    }
+    return ret;
+}
+
+int handle_sysexit(PyObject *e) {
+    PyObject *code;
+
+    code = PyObject_GetAttrString(e, "code");
+    if (!code) return 0;
+    return pyobject_to_int(code);
+}
+
+int report_python_error(const char *preamble, int code) {
+    PyObject *exc, *val, *tb, *str;
+    int ret, issysexit = 0; char *i, *buf;
+
+    if (!PyErr_Occurred()) return code;
+    issysexit = PyErr_ExceptionMatches(PyExc_SystemExit);
+
+    PyErr_Fetch(&exc, &val, &tb);
+
+    if (exc != NULL) {
+        PyErr_NormalizeException(&exc, &val, &tb);
+
+        if (issysexit) {
+            return (val) ? handle_sysexit(val) : 0;
+        }
+        if (val != NULL) {
+            str = PyObject_Unicode(val);
+            if (str == NULL) {
+                PyErr_Clear();
+                str = PyObject_Str(val);
+            }
+            i = PyString_AsString(str);
+            if (i == NULL) OOM;
+            buf = (char*)calloc(strlen(i)+strlen(preamble)+5, sizeof(char));
+            if (buf == NULL) OOM;
+            sprintf(buf, "%s::%s", preamble, i);
+            ret = report_error(buf, code);
+            if (buf) free(buf);
+            if (tb != NULL) {
+                PyErr_Restore(exc, val, tb);
+                PyErr_Print();
+            }
+            return ret;
+        }
+    }
+    return report_error(preamble, code);
+}
+
+static void get_paths()
+{
+	char linkname[256]; /* /proc/<pid>/exe */
+    char *p;
+	pid_t pid;
+	int ret;
+
+	pid = getpid();
+
+	if (snprintf(linkname, sizeof(linkname), "/proc/%i/exe", pid) < 0)
+		{
+		/* This should only happen on large word systems. I'm not sure
+		   what the proper response is here.
+		   Since it really is an assert-like condition, aborting the
+		   program seems to be in order. */
+        exit(report_error("PID too large", EXIT_FAILURE));
+		}
+
+
+	ret = readlink(linkname, exe_path, sizeof(exe_path));
+
+	if (ret == -1) {
+        exit(report_error("Failed to read exe path.", EXIT_FAILURE));
+    }
+
+	if ((size_t)ret >= sizeof(exe_path)) {
+        exit(report_error("exe path buffer too small.", EXIT_FAILURE));
+    }
+
+	exe_path[ret] = 0;
+
+    p = rindex(exe_path, '/');
+
+    if (p ==  NULL) {
+        exit(report_error("No path separators in executable path", EXIT_FAILURE));
+    }
+    strncat(base_dir, exe_path, p - exe_path);
+    p = rindex(base_dir, '/');
+    if (p ==  NULL) {
+        exit(report_error("Only one path separator in executable path", EXIT_FAILURE));
+    }
+    *p = 0;
+    if (strlen(base_dir) == 0) {
+        exit(report_error("base directory empty", EXIT_FAILURE));
+    }
+
+    snprintf(bin_dir,        sizeof(bin_dir), "%s/bin", base_dir);
+    snprintf(lib_dir,        sizeof(lib_dir), "%s/lib", base_dir);
+    snprintf(resources_dir,  sizeof(resources_dir), "%s/resources", base_dir);
+    snprintf(extensions_dir, sizeof(extensions_dir), "%s/%s/site-packages/calibre/plugins", lib_dir, PYTHON_VER);
+}
+
+
+void setup_stream(const char *name, const char *errors) {
+    PyObject *stream;
+    char buf[100];
+
+    snprintf(buf, 20, "%s", name);
+    stream = PySys_GetObject(buf);
+
+    snprintf(buf, 20, "%s", "utf-8");
+    snprintf(buf+21, 30, "%s", errors);
+
+    if (!PyFile_SetEncodingAndErrors(stream, buf, buf+21))
+        exit(report_python_error("Failed to set stream encoding", 1));
+
+}
+
+void setup_streams() {
+    if (!GUI_APP) { // Remove buffering
+        setvbuf(stdin,  NULL, _IONBF, 2);
+        setvbuf(stdout, NULL, _IONBF, 2);
+        setvbuf(stderr, NULL, _IONBF, 2);
+    }
+
+    /*setup_stream("stdin", "strict");
+    setup_stream("stdout", "strict");
+    setup_stream("stderr", "strict");*/
+}
+
+void initialize_interpreter(int argc, char **argv, char *outr, char *errr,
+        const char *basename, const char *module, const char *function) {
+    char *encoding, *p;
+
+    get_paths();
+    char path[3*PATH_MAX];
+
+    snprintf(path, sizeof(path),
+            "%s/%s:%s/%s/plat-linux2:%s/%s/lib-dynload:%s/%s/site-packages",
+            lib_dir, PYTHON_VER, lib_dir, PYTHON_VER, lib_dir, PYTHON_VER,
+            lib_dir, PYTHON_VER);
+
+    Py_OptimizeFlag = 2;
+    Py_NoSiteFlag = 1;
+    Py_DontWriteBytecodeFlag = 1;
+    Py_IgnoreEnvironmentFlag = 1;
+    Py_NoUserSiteDirectory = 1;
+    Py_VerboseFlag = 0;
+    Py_DebugFlag = 0;
+    Py_HashRandomizationFlag = 1;
+
+    Py_SetProgramName(exe_path);
+    Py_SetPythonHome(base_dir);
+
+    //printf("Path before Py_Initialize(): %s\r\n\n", Py_GetPath());
+    Py_Initialize();
+    if (!Py_FileSystemDefaultEncoding) {
+        encoding = getenv("PYTHONIOENCODING");
+        if (encoding != NULL) {
+            Py_FileSystemDefaultEncoding = strndup(encoding, 20);
+            p = index(Py_FileSystemDefaultEncoding, ':');
+            if (p != NULL) *p = 0;
+        } else
+            Py_FileSystemDefaultEncoding = strndup("UTF-8", 10);
+    }
+
+
+    setup_streams();
+
+    PySys_SetArgv(argc, argv);
+    //printf("Path after Py_Initialize(): %s\r\n\n", Py_GetPath());
+    PySys_SetPath(path);
+    //printf("Path set by me: %s\r\n\n", path);
+    PySys_SetObject("gui_app", PyBool_FromLong((long)GUI_APP));
+    PySys_SetObject("calibre_basename", PyBytes_FromString(basename));
+    PySys_SetObject("calibre_module",   PyBytes_FromString(module));
+    PySys_SetObject("calibre_function", PyBytes_FromString(function));
+    PySys_SetObject("extensions_location", PyBytes_FromString(extensions_dir));
+    PySys_SetObject("resources_location", PyBytes_FromString(resources_dir));
+    PySys_SetObject("executables_location", PyBytes_FromString(base_dir));
+    PySys_SetObject("frozen_path", PyBytes_FromString(base_dir));
+    PySys_SetObject("frozen", Py_True);
+    Py_INCREF(Py_True);
+
+
+    if (GUI_APP && outr && errr) {
+    //    PySys_SetObject("stdout_redirect", PyUnicode_FromWideChar(outr, wcslen(outr)));
+    //    PySys_SetObject("stderr_redirect", PyUnicode_FromWideChar(errr, wcslen(outr)));
+    }
+
+}
+
+int execute_python_entrypoint(int argc, char **argv, const char *basename, const char *module, const char *function,
+        char *outr, char *errr) {
+    PyObject *site, *pmain, *res;
+    int ret = 0;
+
+    initialize_interpreter(argc, argv, outr, errr, basename, module, function);
+
+    site = PyImport_ImportModule("site");
+
+    if (site == NULL)
+        ret = report_python_error("Failed to import site module",  1);
+    else {
+        Py_XINCREF(site);
+
+        pmain = PyObject_GetAttrString(site, "main");
+        if (pmain == NULL || !PyCallable_Check(pmain))
+            ret = report_python_error("site module has no main function", 1);
+        else {
+            Py_XINCREF(pmain);
+            res = PyObject_CallObject(pmain, NULL);
+
+            if (res == NULL)
+                ret = report_python_error("Python function terminated unexpectedly", 1);
+
+            ret = pyobject_to_int(res);
+        }
+    }
+    PyErr_Clear();
+    Py_Finalize();
+
+    //printf("11111 Returning: %d\r\n", ret);
+    return ret;
+}

bypy/linux/util.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#define UNICODE
+
+#ifndef PATH_MAX
+#define PATH_MAX 4096
+#endif
+
+#define OOM exit(report_error("Out of memory", EXIT_FAILURE))
+#define True 1
+#define False 0
+typedef int bool;
+
+void set_gui_app(bool yes);
+
+int execute_python_entrypoint(int argc, char **argv, const char *basename,
+        const char *module, const char *function,
+        char *outr, char *errr);