From 01198cf010a8531b53f5d798da6ea9305230f6bf Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 5 May 2019 07:16:35 +0530 Subject: [PATCH] Work on porting the calibre linux freeze script --- bypy/init_env.py | 65 +++++++++ bypy/linux/__main__.py | 301 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 366 insertions(+) create mode 100644 bypy/init_env.py create mode 100644 bypy/linux/__main__.py diff --git a/bypy/init_env.py b/bypy/init_env.py new file mode 100644 index 0000000000..96238c58df --- /dev/null +++ b/bypy/init_env.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# vim:fileencoding=utf-8 +# License: GPL v3 Copyright: 2019, Kovid Goyal + +import json +import os +import re + +from bypy.constants import SRC as CALIBRE_DIR + + +def read_cal_file(name): + with open(os.path.join(CALIBRE_DIR, 'src', 'calibre', name), 'rb') as f: + return f.read().decode('utf-8') + + +def initialize_constants(): + calibre_constants = {} + src = read_cal_file('constants.py') + nv = re.search(r'numeric_version\s+=\s+\((\d+), (\d+), (\d+)\)', src) + calibre_constants['version' + ] = '%s.%s.%s' % (nv.group(1), nv.group(2), nv.group(3)) + calibre_constants['appname'] = re.search( + r'__appname__\s+=\s+(u{0,1})[\'"]([^\'"]+)[\'"]', src + ).group(2) + epsrc = re.compile(r'entry_points = (\{.*?\})', + re.DOTALL).search(read_cal_file('linux.py')).group(1) + entry_points = eval(epsrc, {'__appname__': calibre_constants['appname']}) + + def e2b(ep): + return re.search(r'\s*(.*?)\s*=', ep).group(1).strip() + + def e2s(ep, base='src'): + return ( + base + os.path.sep + + re.search(r'.*=\s*(.*?):', ep).group(1).replace('.', '/') + '.py' + ).strip() + + def e2m(ep): + return re.search(r'.*=\s*(.*?)\s*:', ep).group(1).strip() + + def e2f(ep): + return ep[ep.rindex(':') + 1:].strip() + + calibre_constants['basenames'] = basenames = {} + calibre_constants['functions'] = functions = {} + calibre_constants['modules'] = modules = {} + calibre_constants['scripts'] = scripts = {} + for x in ('console', 'gui'): + y = x + '_scripts' + basenames[x] = list(map(e2b, entry_points[y])) + functions[x] = list(map(e2f, entry_points[y])) + modules[x] = list(map(e2m, entry_points[y])) + scripts[x] = list(map(e2s, entry_points[y])) + + src = read_cal_file('ebooks/__init__.py') + be = re.search( + r'^BOOK_EXTENSIONS\s*=\s*(\[.+?\])', src, flags=re.DOTALL | re.MULTILINE + ).group(1) + calibre_constants['book_extensions'] = json.loads(be.replace("'", '"')) + return calibre_constants + + +if __name__ == 'program': + calibre_constants = initialize_constants() diff --git a/bypy/linux/__main__.py b/bypy/linux/__main__.py new file mode 100644 index 0000000000..ccc04771aa --- /dev/null +++ b/bypy/linux/__main__.py @@ -0,0 +1,301 @@ +#!/usr/bin/env python +# vim:fileencoding=utf-8 +# License: GPLv3 Copyright: 2016, Kovid Goyal + +import errno +import glob +import os +import shutil +import stat +import subprocess +import tarfile +import time +from functools import partial + +from bypy.constants import ( + PREFIX, SRC as CALIBRE_DIR, SW, is64bit, python_major_minor_version +) +from bypy.pkgs.qt import PYQT_MODULES, QT_DLLS, QT_PLUGINS +from bypy.utils import ( + create_job, get_dll_path, mkdtemp, parallel_build, py_compile, run, run_shell, + walk +) + +j = os.path.join +self_dir = os.path.dirname(os.path.abspath(__file__)) +arch = 'x86_64' if is64bit else 'i686' + +py_ver = '.'.join(map(str, python_major_minor_version())) +QT_PREFIX = os.path.join(PREFIX, 'qt') +calibre_constants = globals()['init_env']['calibre_constants'] + + +def binary_includes(): + return [ + j(PREFIX, 'bin', x) for x in ('pdftohtml', 'pdfinfo', 'pdftoppm', 'optipng', 'JxrDecApp')] + [ + + j(PREFIX, 'private', 'mozjpeg', 'bin', x) for x in ('jpegtran', 'cjpeg')] + [ + j(PREFIX, 'lib', 'lib' + x) for x in ( + 'unrar.so', 'ssl.so.1.0.0', 'crypto.so.1.0.0', 'python{}.so.1.0'.format(py_ver)) + ] + list(map( + get_dll_path, + ('usb-1.0 mtp expat sqlite3 ffi podofo z bz2 poppler dbus-1 iconv xml2 xslt jpeg png16' + ' webp exslt ncursesw readline chm icudata icui18n icuuc icuio gcrypt gpg-error' + ' gobject-2.0 glib-2.0 gthread-2.0 gmodule-2.0 gio-2.0 dbus-glib-1').split() + )) + [ + # We dont include libstdc++.so as the OpenGL dlls on the target + # computer fail to load in the QPA xcb plugin if they were compiled + # with a newer version of gcc than the one on the build computer. + # libstdc++, like glibc is forward compatible and I dont think any + # distros do not have libstdc++.so.6, so it should be safe to leave it out. + # https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html (The current + # debian stable libstdc++ is libstdc++.so.6.0.17) + ] + [ + j(QT_PREFIX, 'lib', 'lib%s.so.5' % x) for x in QT_DLLS] + + +class Env(object): + + def __init__(self): + self.src_root = CALIBRE_DIR + self.base = mkdtemp('frozen-') + self.lib_dir = j(self.base, 'lib') + self.py_dir = j(self.lib_dir, 'python' + py_ver) + os.makedirs(self.py_dir) + self.bin_dir = j(self.base, 'bin') + os.mkdir(self.bin_dir) + self.SRC = j(self.src_root, 'src') + self.obj_dir = mkdtemp('launchers-') + + +def ignore_in_lib(base, items, ignored_dirs=None): + ans = [] + if ignored_dirs is None: + ignored_dirs = {'.svn', '.bzr', '.git', 'test', 'tests', 'testing'} + for name in items: + path = j(base, name) + if os.path.isdir(path): + if name in ignored_dirs or not os.path.exists(j(path, '__init__.py')): + if name != 'plugins': + ans.append(name) + else: + if name.rpartition('.')[-1] not in ('so', 'py'): + ans.append(name) + return ans + + +def import_site_packages(srcdir, dest): + if not os.path.exists(dest): + os.mkdir(dest) + for x in os.listdir(srcdir): + ext = x.rpartition('.')[-1] + f = j(srcdir, x) + if ext in ('py', 'so'): + shutil.copy2(f, dest) + elif ext == 'pth' and x != 'setuptools.pth': + for line in open(f, 'rb').read().splitlines(): + src = os.path.abspath(j(srcdir, line)) + if os.path.exists(src) and os.path.isdir(src): + import_site_packages(src, dest) + elif os.path.exists(j(f, '__init__.py')): + shutil.copytree(f, j(dest, x), ignore=ignore_in_lib) + + +def copy_libs(env): + print('Copying libs...') + + for x in binary_includes(): + dest = env.bin_dir if '/bin/' in x else env.lib_dir + shutil.copy2(x, dest) + + base = j(QT_PREFIX, 'plugins') + dest = j(env.lib_dir, 'qt_plugins') + os.mkdir(dest) + for x in QT_PLUGINS: + if x not in ('audio', 'printsupport'): + shutil.copytree(j(base, x), j(dest, x)) + + +def copy_python(env, ext_dir): + print('Copying python...') + srcdir = j(PREFIX, 'lib/python' + py_ver) + + for x in os.listdir(srcdir): + y = j(srcdir, x) + ext = os.path.splitext(x)[1] + if os.path.isdir(y) and x not in ('test', 'hotshot', 'distutils', + 'site-packages', 'idlelib', 'lib2to3', 'dist-packages'): + shutil.copytree(y, j(env.py_dir, x), ignore=ignore_in_lib) + if os.path.isfile(y) and ext in ('.py', '.so'): + shutil.copy2(y, env.py_dir) + + srcdir = j(srcdir, 'site-packages') + dest = j(env.py_dir, 'site-packages') + import_site_packages(srcdir, dest) + shutil.rmtree(j(dest, 'PyQt5/uic/port_v3')) + + filter_pyqt = {x + '.so' for x in PYQT_MODULES} + pyqt = j(dest, 'PyQt5') + for x in os.listdir(pyqt): + if x.endswith('.so') and x not in filter_pyqt: + os.remove(j(pyqt, x)) + + for x in os.listdir(env.SRC): + c = j(env.SRC, x) + if os.path.exists(j(c, '__init__.py')): + shutil.copytree(c, j(dest, x), ignore=partial(ignore_in_lib, ignored_dirs={})) + elif os.path.isfile(c): + shutil.copy2(c, j(dest, x)) + pdir = j(dest, 'calibre', 'plugins') + if not os.path.exists(pdir): + os.mkdir(pdir) + for x in glob.glob(j(ext_dir, '*.so')): + shutil.copy2(x, j(pdir, os.path.basename(x))) + + shutil.copytree(j(env.src_root, 'resources'), j(env.base, 'resources')) + sitepy = j(self_dir, 'site.py') + shutil.copy2(sitepy, j(env.py_dir, 'site.py')) + + py_compile(env.py_dir) + + +def build_launchers(env): + base = self_dir + sources = [j(base, x) for x in ['util.c']] + objects = [j(env.obj_dir, os.path.basename(x) + '.o') for x in sources] + cflags = '-fno-strict-aliasing -W -Wall -c -O2 -pipe -DPYTHON_VER="python%s"' % py_ver + cflags = cflags.split() + ['-I%s/include/python%s' % (PREFIX, py_ver)] + for src, obj in zip(sources, objects): + cmd = ['gcc'] + cflags + ['-fPIC', '-o', obj, src] + run(*cmd) + + dll = j(env.lib_dir, 'libcalibre-launcher.so') + cmd = ['gcc', '-O2', '-Wl,--rpath=$ORIGIN/../lib', '-fPIC', '-o', dll, '-shared'] + objects + \ + ['-L%s/lib' % PREFIX, '-lpython' + py_ver] + run(*cmd) + + src = j(base, 'main.c') + + modules, basenames, functions = calibre_constants['modules'].copy(), calibre_constants['basenames'].copy(), calibre_constants['functions'].copy() + modules['console'].append('calibre.linux') + basenames['console'].append('calibre_postinstall') + functions['console'].append('main') + c_launcher = '/tmp/calibre-c-launcher' + lsrc = os.path.join(base, 'launcher.c') + cmd = ['gcc', '-O2', '-o', c_launcher, lsrc, ] + run(*cmd) + + jobs = [] + for typ in ('console', 'gui', ): + for mod, bname, func in zip(modules[typ], basenames[typ], functions[typ]): + xflags = list(cflags) + xflags.remove('-c') + xflags += ['-DGUI_APP=' + ('1' if typ == 'gui' else '0')] + xflags += ['-DMODULE="%s"' % mod, '-DBASENAME="%s"' % bname, + '-DFUNCTION="%s"' % func] + + exe = j(env.bin_dir, bname) + cmd = ['gcc'] + xflags + [src, '-o', exe, '-L' + env.lib_dir, '-lcalibre-launcher'] + jobs.append(create_job(cmd)) + sh = j(env.base, bname) + shutil.copy2(c_launcher, sh) + os.chmod(sh, + stat.S_IREAD | stat.S_IEXEC | stat.S_IWRITE | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) + if jobs: + if not parallel_build(jobs, verbose=False): + raise SystemExit(1) + + +def is_elf(path): + with open(path, 'rb') as f: + return f.read(4) == b'\x7fELF' + + +STRIPCMD = ['strip'] + + +def strip_files(files, argv_max=(256 * 1024)): + """ Strip a list of files """ + while files: + cmd = list(STRIPCMD) + pathlen = sum(len(s) + 1 for s in cmd) + while pathlen < argv_max and files: + f = files.pop() + cmd.append(f) + pathlen += len(f) + 1 + if len(cmd) > len(STRIPCMD): + all_files = cmd[len(STRIPCMD):] + unwritable_files = tuple(filter(None, (None if os.access(x, os.W_OK) else (x, os.stat(x).st_mode) for x in all_files))) + [os.chmod(x, stat.S_IWRITE | old_mode) for x, old_mode in unwritable_files] + subprocess.check_call(cmd) + [os.chmod(x, old_mode) for x, old_mode in unwritable_files] + + +def strip_binaries(env): + files = {j(env.bin_dir, x) for x in os.listdir(env.bin_dir)} | { + x for x in { + j(os.path.dirname(env.bin_dir), x) for x in os.listdir(env.bin_dir)} if os.path.exists(x)} + for x in walk(env.lib_dir): + x = os.path.realpath(x) + if x not in files and is_elf(x): + files.add(x) + print('Stripping %d files...' % len(files)) + before = sum(os.path.getsize(x) for x in files) + strip_files(files) + after = sum(os.path.getsize(x) for x in files) + print('Stripped %.1f MB' % ((before - after) / (1024 * 1024.))) + + +def create_tarfile(env, compression_level='9'): + print('Creating archive...') + base = os.path.join(SW, 'dist') + try: + shutil.rmtree(base) + except EnvironmentError as err: + if err.errno != errno.ENOENT: + raise + os.mkdir(base) + dist = os.path.join(base, '%s-%s-%s.tar' % (calibre_constants['appname'], calibre_constants['version'], arch)) + with tarfile.open(dist, mode='w', format=tarfile.PAX_FORMAT) as tf: + cwd = os.getcwd() + os.chdir(env.base) + try: + for x in os.listdir('.'): + tf.add(x) + finally: + os.chdir(cwd) + print('Compressing archive...') + ans = dist.rpartition('.')[0] + '.txz' + start_time = time.time() + subprocess.check_call(['xz', '--threads=0', '-f', '-' + compression_level, dist]) + secs = time.time() - start_time + print('Compressed in %d minutes %d seconds' % (secs // 60, secs % 60)) + os.rename(dist + '.xz', ans) + print('Archive %s created: %.2f MB' % ( + os.path.basename(ans), os.stat(ans).st_size / (1024.**2))) + + +def run_tests(path_to_calibre_debug, cwd_on_failure): + p = subprocess.Popen([path_to_calibre_debug, '--test-build']) + if p.wait() != 0: + os.chdir(cwd_on_failure) + run_shell() + raise SystemExit(p.wait()) + + +def main(): + args = globals()['args'] + ext_dir = globals()['ext_dir'] + env = Env() + copy_libs(env) + copy_python(env, ext_dir) + build_launchers(env) + if not args.skip_tests: + run_tests(j(env.base, 'calibre-debug'), env.base) + if not args.dont_strip: + strip_binaries(env) + create_tarfile(env, args.compression_level) + + +if __name__ == '__main__': + main()