diff --git a/setup/commands.py b/setup/commands.py index 0a09efa935..e13fe6a448 100644 --- a/setup/commands.py +++ b/setup/commands.py @@ -18,7 +18,7 @@ __all__ = [ 'pypi_register', 'pypi_upload', 'upload_to_server', 'upload_user_manual', 'upload_to_mobileread', 'upload_demo', 'upload_to_sourceforge', 'upload_to_google_code', - 'linux32', 'linux64', 'linux', 'linux_freeze', + 'linux32', 'linux64', 'linux', 'linux_freeze', 'linux_freeze2', 'osx32_freeze', 'osx32', 'osx', 'rsync', 'win32_freeze', 'win32', 'win', 'stage1', 'stage2', 'stage3', 'publish' @@ -77,6 +77,8 @@ linux32 = Linux32() linux64 = Linux64() from setup.installer.linux.freeze import LinuxFreeze linux_freeze = LinuxFreeze() +from setup.installer.linux.freeze2 import LinuxFreeze2 +linux_freeze2 = LinuxFreeze2() from setup.installer.osx import OSX, OSX32 osx = OSX() diff --git a/setup/installer/linux/freeze2.py b/setup/installer/linux/freeze2.py new file mode 100644 index 0000000000..cd8443e11c --- /dev/null +++ b/setup/installer/linux/freeze2.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import with_statement + +__license__ = 'GPL v3' +__copyright__ = '2009, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import sys, os, shutil, platform, subprocess, stat, py_compile, glob + +from setup import Command, modules, basenames, functions + +is64bit = platform.architecture()[0] == '64bit' +arch = 'x86_64' if is64bit else 'i686' +ffi = '/usr/lib/libffi.so.5' if is64bit else '/usr/lib/gcc/i686-pc-linux-gnu/4.4.1/libffi.so.4' + + +QTDIR = '/usr/lib/qt4' +QTDLLS = ('QtCore', 'QtGui', 'QtNetwork', 'QtSvg', 'QtXml', 'QtWebKit', 'QtDBus') + + +binary_includes = [ + '/usr/bin/pdftohtml', + '/usr/lib/libwmflite-0.2.so.7', + '/usr/lib/liblcms.so.1', + '/usr/lib/libunrar.so', + '/usr/lib/libsqlite3.so.0', + '/usr/lib/libsqlite3.so.0', + '/usr/lib/libmng.so.1', + '/usr/lib/libpodofo.so.0.6.99', + '/lib/libz.so.1', + '/usr/lib/libtiff.so.3', + '/lib/libbz2.so.1', + '/usr/lib/libpoppler.so.5', + '/usr/lib/libxml2.so.2', + '/usr/lib/libopenjpeg.so.2', + '/usr/lib/libxslt.so.1', + '/usr/lib/libjpeg.so.7', + '/usr/lib/libxslt.so.1', + '/usr/lib/libgthread-2.0.so.0', + '/usr/lib/gcc/***-pc-linux-gnu/4.4.1/libstdc++.so.6'.replace('***', + arch), + ffi, + '/usr/lib/libpng12.so.0', + '/usr/lib/libexslt.so.0', + '/usr/lib/libMagickWand.so.2', + '/usr/lib/libMagickCore.so.2', + '/usr/lib/libgcrypt.so.11', + '/usr/lib/libgpg-error.so.0', + '/usr/lib/libphonon.so.4', + '/usr/lib/libssl.so.0.9.8', + '/usr/lib/libcrypto.so.0.9.8', + '/lib/libreadline.so.6', + ] +binary_includes += [os.path.join(QTDIR, 'lib%s.so.4'%x) for x in QTDLLS] + +SITE_PACKAGES = ['IPython', 'PIL', 'dateutil', 'dns', 'PyQt4', 'mechanize', + 'sip.so', 'BeautifulSoup.py', 'ClientForm.py', 'lxml'] + +class LinuxFreeze2(Command): + + def run(self, opts): + self.drop_privileges() + self.opts = opts + self.src_root = self.d(self.SRC) + self.base = self.j(self.src_root, 'build', 'linfrozen') + self.py_ver = '.'.join(map(str, sys.version_info[:2])) + self.lib_dir = self.j(self.base, 'lib') + self.bin_dir = self.j(self.base, 'bin') + + #self.initbase() + #self.copy_libs() + #self.copy_python() + #self.compile_mount_helper() + self.build_launchers() + + def initbase(self): + if os.path.exists(self.base): + shutil.rmtree(self.base) + os.makedirs(self.base) + + def copy_libs(self): + self.info('Copying libs...') + os.mkdir(self.lib_dir) + os.mkdir(self.bin_dir) + for x in binary_includes: + dest = self.bin_dir if '/bin/' in x else self.lib_dir + shutil.copy2(x, dest) + shutil.copy2('/usr/lib/libpython%s.so.1.0'%self.py_ver, dest) + + base = self.j(QTDIR, 'plugins') + dest = self.j(self.lib_dir, 'qt_plugins') + os.mkdir(dest) + for x in os.listdir(base): + y = self.j(base, x) + if x not in ('designer', 'sqldrivers', 'codecs'): + shutil.copytree(y, self.j(dest, x)) + + im = glob.glob('/usr/lib/ImageMagick-*')[0] + dest = self.j(self.lib_dir, 'ImageMagick') + shutil.copytree(im, dest, ignore=shutil.ignore_patterns('*.a')) + + def compile_mount_helper(self): + self.info('Compiling mount helper...') + self.regain_privileges() + dest = self.j(self.bin_dir, 'calibre-mount-helper') + subprocess.check_call(['gcc', '-Wall', '-pedantic', + self.j(self.SRC, 'calibre', 'devices', + 'linux_mount_helper.c'), '-o', dest]) + os.chown(dest, 0, 0) + os.chmod(dest, stat.S_ISUID|stat.S_ISGID|stat.S_IRUSR|stat.S_IWUSR|\ + stat.S_IXUSR|stat.S_IXGRP|stat.S_IXOTH) + self.drop_privileges() + + def copy_python(self): + self.info('Copying python...') + + def ignore_in_lib(base, items): + ans = [] + for x in items: + x = os.path.join(base, x) + if (os.path.isfile(x) and os.path.splitext(x)[1] in ('.so', + '.py')) or \ + (os.path.isdir(x) and x in ('.svn', '.bzr', 'test')): + continue + ans.append(x) + return ans + + srcdir = self.j('/usr/lib/python'+self.py_ver) + self.py_dir = self.j(self.lib_dir, self.b(srcdir)) + os.mkdir(self.py_dir) + + for x in os.listdir(srcdir): + y = self.j(srcdir, x) + ext = os.path.splitext(x)[1] + if os.path.isdir(y) and x not in ('test', 'hotshot', 'distutils', + 'site-packages', 'idlelib', 'test', 'lib2to3'): + shutil.copytree(y, self.j(self.py_dir, x), + ignore=ignore_in_lib) + if os.path.isfile(y) and ext in ('.py', '.so'): + shutil.copy2(y, self.py_dir) + + srcdir = self.j(srcdir, 'site-packages') + dest = self.j(self.py_dir, 'site-packages') + os.mkdir(dest) + for x in SITE_PACKAGES: + x = self.j(srcdir, x) + ext = os.path.splitext(x)[1] + if os.path.isdir(x): + shutil.copytree(x, self.j(dest, self.b(x)), + ignore=ignore_in_lib) + if os.path.isfile(x) and ext in ('.py', '.so'): + shutil.copy2(x, dest) + + for x in os.listdir(self.SRC): + shutil.copytree(self.j(self.SRC, x), self.j(dest, x), + ignore=ignore_in_lib) + for x in ('translations', 'manual'): + x = self.j(dest, 'calibre', x) + shutil.rmtree(x) + + shutil.copytree(self.j(self.src_root, 'resources'), self.j(self.base, + 'resources')) + + for x in os.walk(self.py_dir): + for f in x[-1]: + if f.endswith('.py'): + y = self.j(x[0], f) + rel = os.path.relpath(y, self.py_dir) + try: + py_compile.compile(y, dfile=rel, doraise=True) + os.remove(y) + z = y+'c' + if os.path.exists(z): + os.remove(z) + except: + self.warn('Failed to byte-compile', y) + + def run_builder(self, cmd): + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.info(*cmd) + self.info(p.stdout.read()) + self.info(p.stderr.read()) + + if p.wait() != 0: + self.info('Failed to run builder') + sys.exit(1) + + def build_launchers(self): + self.obj_dir = self.j(self.src_root, 'build', 'launcher') + if not os.path.exists(self.obj_dir): + os.makedirs(self.obj_dir) + base = self.j(self.src_root, 'setup', 'installer', 'linux') + sources = [self.j(base, x) for x in ['util.c']] + headers = [self.j(base, x) for x in ['util.h']] + objects = [self.j(self.obj_dir, self.b(x)+'.o') for x in sources] + cflags = '-W -Wall -c -O2 -pipe -DPYTHON_VER="python%s"'%self.py_ver + cflags = cflags.split() + ['-I/usr/include/python'+self.py_ver] + for src, obj in zip(sources, objects): + if not self.newer(obj, headers+[src, __file__]): continue + cmd = ['gcc'] + cflags + ['-fPIC', '-o', obj, src] + self.run_builder(cmd) + + dll = self.j(self.lib_dir, 'libcalibre-launcher.so') + if self.newer(dll, objects): + cmd = ['gcc', '-O2', '-Wl,--rpath=$ORIGIN/../lib', '-fPIC', '-o', dll, '-shared'] + objects + \ + ['-lpython'+self.py_ver] + self.info('Linking libcalibre-launcher.so') + self.run_builder(cmd) + + src = self.j(base, 'main.c') + for typ in ('console', 'gui', ): + self.info('Processing %s launchers'%typ) + for mod, bname, func in zip(modules[typ], basenames[typ], + functions[typ]): + xflags = list(cflags) + xflags += ['-DGUI_APP='+('1' if type == 'gui' else '0')] + xflags += ['-DMODULE="%s"'%mod, '-DBASENAME="%s"'%bname, + '-DFUNCTION="%s"'%func] + + dest = self.j(self.obj_dir, bname+'.o') + if self.newer(dest, [src, __file__]+headers): + self.info('Compiling', bname) + cmd = ['gcc'] + xflags + [src, '-o', dest] + self.run_builder(cmd) + exe = self.j(self.bin_dir, bname) + if self.newer(exe, [dest, __file__]): + self.info('Linking', bname) + cmd = ['gcc', '-O2', '-Wl,--rpath=$ORIGIN/../lib', + '-o', exe, + dest, + '-L'+self.lib_dir, + '-lcalibre-launcher', + ] + + self.run_builder(cmd) + + + + + + diff --git a/setup/installer/linux/main.c b/setup/installer/linux/main.c new file mode 100644 index 0000000000..7bb32db5df --- /dev/null +++ b/setup/installer/linux/main.c @@ -0,0 +1,13 @@ +#include "util.h" + +#include + +int main(int argc, char **argv) { + int ret = 0; + set_gui_app(GUI_APP); + ret = execute_python_entrypoint(argc, argv, BASENAME, MODULE, FUNCTION, NULL, NULL); + + return ret; +} + + diff --git a/setup/installer/linux/util.c b/setup/installer/linux/util.c new file mode 100644 index 0000000000..f7482d2501 --- /dev/null +++ b/setup/installer/linux/util.c @@ -0,0 +1,268 @@ +#include "util.h" +#include +#include +#include +#include +#include + +static bool GUI_APP = False; + +static char exe_path[PATH_MAX]; +static char base_dir[PATH_MAX]; +static char bin_dir[PATH_MAX]; +static char lib_dir[PATH_MAX]; +static char qt_dir[PATH_MAX]; +static char extensions_dir[PATH_MAX]; +static char resources_dir[PATH_MAX]; +static char magick_dir[PATH_MAX]; + +void set_gui_app(bool yes) { GUI_APP = yes; } + +int report_error(const char *msg, int code) { + fprintf(stderr, "%s\n", msg); + return code; +} + +int report_libc_error(const char *msg) { + char buf[2000]; + int err = errno; + snprintf("%s::%s", 2000, msg, strerror(err)); + return report_error(buf, err); +} + +int pyobject_to_int(PyObject *res) { + int ret; PyObject *tmp; + tmp = PyNumber_Int(res); + if (tmp == NULL) ret = (PyObject_IsTrue(res)) ? 1 : 0; + else ret = (int)PyInt_AS_LONG(tmp); + + return ret; +} + +int handle_sysexit(PyObject *e) { + PyObject *code; + + code = PyObject_GetAttrString(e, "code"); + if (!code) return 0; + return pyobject_to_int(code); +} + +int report_python_error(const char *preamble, int code) { + PyObject *exc, *val, *tb, *str; + int ret, issysexit = 0; char *i, *buf; + + if (!PyErr_Occurred()) return code; + issysexit = PyErr_ExceptionMatches(PyExc_SystemExit); + + PyErr_Fetch(&exc, &val, &tb); + + if (exc != NULL) { + PyErr_NormalizeException(&exc, &val, &tb); + + if (issysexit) { + return (val) ? handle_sysexit(val) : 0; + } + if (val != NULL) { + str = PyObject_Unicode(val); + if (str == NULL) { + PyErr_Clear(); + str = PyObject_Str(val); + } + i = PyString_AsString(str); + if (i == NULL) OOM; + buf = (char*)calloc(strlen(i)+strlen(preamble)+5, sizeof(char)); + if (buf == NULL) OOM; + sprintf(buf, "%s::%s", preamble, i); + ret = report_error(buf, code); + if (buf) free(buf); + if (tb != NULL) { + PyErr_Restore(exc, val, tb); + PyErr_Print(); + } + return ret; + } + } + return report_error(preamble, code); +} + +static void get_paths() +{ + char linkname[256]; /* /proc//exe */ + char *p; + pid_t pid; + int ret; + + pid = getpid(); + + if (snprintf(linkname, sizeof(linkname), "/proc/%i/exe", pid) < 0) + { + /* This should only happen on large word systems. I'm not sure + what the proper response is here. + Since it really is an assert-like condition, aborting the + program seems to be in order. */ + exit(report_error("PID too large", EXIT_FAILURE)); + } + + + ret = readlink(linkname, exe_path, PATH_MAX); + + if (ret == -1) { + exit(report_error("Failed to read exe path.", EXIT_FAILURE)); + } + + if (ret >= PATH_MAX) { + exit(report_error("exe path buffer too small.", EXIT_FAILURE)); + } + + exe_path[ret] = 0; + + p = rindex(exe_path, '/'); + + if (p == NULL) { + exit(report_error("No path separators in executable path", EXIT_FAILURE)); + } + strncat(base_dir, exe_path, p - exe_path); + strcat(base_dir, "\0"); + snprintf(bin_dir, PATH_MAX, "%s/bin", base_dir); + snprintf(lib_dir, PATH_MAX, "%s/lib", base_dir); + snprintf(magick_dir, PATH_MAX, "%s/ImageMagick", lib_dir); + snprintf(qt_dir, PATH_MAX, "%s/qt_plugins", lib_dir); + snprintf(resources_dir, PATH_MAX, "%s/resources", base_dir); + snprintf(extensions_dir, PATH_MAX, "%s/%s/site-packages/calibre/plugins", lib_dir, PYTHON_VER); +} + +void init_env() { + char buf[PATH_MAX]; + + if (setenv("QT_PLUGIN_PATH", qt_dir, 1) == -1) + exit(report_libc_error("Failed to set environment variable")); + + snprintf(buf, PATH_MAX, "%s/config", magick_dir); + if (setenv("MAGICK_CONFIGURE_PATH", buf, 1) == -1) + exit(report_libc_error("Failed to set environment variable")); + + snprintf(buf, PATH_MAX, "%s/modules-Q16/coders", magick_dir); + if (setenv("MAGICK_CODER_MODULE_PATH", buf, 1) == -1) + exit(report_libc_error("Failed to set environment variable")); + + snprintf(buf, PATH_MAX, "%s/modules-Q16/filters", magick_dir); + if (setenv("MAGICK_CODER_FILTER_PATH", buf, 1) == -1) + exit(report_libc_error("Failed to set environment variable")); + + if (setenv("PYTHONIOENCODING", "utf-8", 1) == -1) + exit(report_libc_error("Failed to set environment variable")); + + if (setenv("PYTHONHOME", base_dir, 1) == -1) + exit(report_libc_error("Failed to set environment variable")); + +} + +void setup_stream(const char *name, const char *errors) { + PyObject *stream; + char *buf = (char *)calloc(100, sizeof(char)); + if (!buf) OOM; + + snprintf(buf, 100, "%s", "utf-8"); + + stream = PySys_GetObject(name); + + if (!PyFile_SetEncodingAndErrors(stream, buf, errors)) + exit(report_python_error("Failed to set stream encoding", 1)); + + free(buf); + +} + +void setup_streams() { + if (!GUI_APP) { // Remove buffering + setvbuf(stdin, NULL, _IONBF, 2); + setvbuf(stdout, NULL, _IONBF, 2); + setvbuf(stderr, NULL, _IONBF, 2); + } + + /*setup_stream("stdin", "strict"); + setup_stream("stdout", "strict"); + setup_stream("stderr", "strict");*/ +} + +void initialize_interpreter(int argc, char **argv, char *outr, char *errr, + const char *basename, const char *module, const char *function) { + char *path; + + get_paths(); + init_env(); + + path = (char*)calloc(3*PATH_MAX, sizeof(char)); + if (!path) OOM; + + snprintf(path, 3*PATH_MAX, + "%s/%s:%s/%s/plat-linux2:%s/%s/lib-dynload:%s/%s/site-packages", + lib_dir, PYTHON_VER, lib_dir, PYTHON_VER, lib_dir, PYTHON_VER, + lib_dir, PYTHON_VER); + + Py_OptimizeFlag = 2; + Py_NoSiteFlag = 1; + Py_DontWriteBytecodeFlag = 1; + Py_IgnoreEnvironmentFlag = 1; + Py_NoUserSiteDirectory = 1; + Py_VerboseFlag = 0; + Py_DebugFlag = 0; + + Py_SetProgramName(exe_path); + Py_SetPythonHome(base_dir); + + //printf("Path before Py_Initialize(): %s\r\n\n", Py_GetPath()); + Py_Initialize(); + setup_streams(); + + PySys_SetArgv(argc, argv); + //printf("Path after Py_Initialize(): %s\r\n\n", Py_GetPath()); + PySys_SetPath(path); + //printf("Path set by me: %s\r\n\n", path); + PySys_SetObject("gui_app", PyBool_FromLong((long)GUI_APP)); + PySys_SetObject("app_dir", PyString_FromString(base_dir)); + + PySys_SetObject("calibre_basename", PyBytes_FromString(basename)); + PySys_SetObject("calibre_module", PyBytes_FromString(module)); + PySys_SetObject("calibre_function", PyBytes_FromString(function)); + + //if (GUI_APP && outr && errr) { + // PySys_SetObject("stdout_redirect", PyUnicode_FromWideChar(outr, wcslen(outr))); + // PySys_SetObject("stderr_redirect", PyUnicode_FromWideChar(errr, wcslen(outr))); + //} + +} + +int execute_python_entrypoint(int argc, char **argv, const char *basename, const char *module, const char *function, + char *outr, char *errr) { + PyObject *site, *pmain, *res; + int ret = 0; + + initialize_interpreter(argc, argv, outr, errr, basename, module, function); + + site = PyImport_ImportModule("site"); + + if (site == NULL) + ret = report_python_error("Failed to import site module", 1); + else { + Py_XINCREF(site); + + pmain = PyObject_GetAttrString(site, "main"); + if (pmain == NULL || !PyCallable_Check(pmain)) + ret = report_python_error("site module has no main function", 1); + else { + Py_XINCREF(pmain); + res = PyObject_CallObject(pmain, NULL); + + if (res == NULL) + ret = report_python_error("Python function terminated unexpectedly", 1); + } + } + PyErr_Clear(); + Py_Finalize(); + + //printf("11111 Returning: %d\r\n", ret); + return ret; +} + + diff --git a/setup/installer/linux/util.h b/setup/installer/linux/util.h new file mode 100644 index 0000000000..de4c97870d --- /dev/null +++ b/setup/installer/linux/util.h @@ -0,0 +1,18 @@ +#pragma once + +#define UNICODE + +#ifndef PATH_MAX +#define PATH_MAX 4096 +#endif + +#define OOM exit(report_error("Out of memory", EXIT_FAILURE)) +#define True 1 +#define False 0 +typedef int bool; + +void set_gui_app(bool yes); + +int execute_python_entrypoint(int argc, char **argv, const char *basename, + const char *module, const char *function, + char *outr, char *errr);