calibre/bypy/linux/__main__.py

#!/usr/bin/env python
# vim:fileencoding=utf-8
# License: GPLv3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>

import errno
import glob
import os
import shutil
import stat
import subprocess
import tarfile
import time
from functools import partial

from bypy.constants import LIBDIR, OUTPUT_DIR, PREFIX, python_major_minor_version
from bypy.constants import SRC as CALIBRE_DIR
from bypy.freeze import extract_extension_modules, fix_pycryptodome, freeze_python, is_package_dir, path_to_freeze_dir
from bypy.pkgs.piper import copy_piper_dir
from bypy.utils import create_job, get_dll_path, mkdtemp, parallel_build, py_compile, run, walk

j = os.path.join
self_dir = os.path.dirname(os.path.abspath(__file__))
machine = (os.uname()[4] or '').lower()
py_ver = '.'.join(map(str, python_major_minor_version()))
QT_PREFIX = os.path.join(PREFIX, 'qt')
FFMPEG_PREFIX = os.path.join(PREFIX, 'ffmpeg', 'lib')
iv = globals()['init_env']
calibre_constants = iv['calibre_constants']
QT_DLLS, QT_PLUGINS, PYQT_MODULES = iv['QT_DLLS'], iv['QT_PLUGINS'], iv['PYQT_MODULES']
qt_get_dll_path = partial(get_dll_path, loc=os.path.join(QT_PREFIX, 'lib'))
ffmpeg_get_dll_path = partial(get_dll_path, loc=FFMPEG_PREFIX)


def binary_includes():
    ffmpeg_dlls = tuple(os.path.basename(x).partition('.')[0][3:] for x in glob.glob(os.path.join(FFMPEG_PREFIX, '*.so')))
    return [
        j(PREFIX, 'bin', x) for x in ('pdftohtml', 'pdfinfo', 'pdftoppm', 'pdftotext', 'optipng', 'cwebp', 'JxrDecApp')] + [

        j(PREFIX, 'private', 'mozjpeg', 'bin', x) for x in ('jpegtran', 'cjpeg')] + [
        ] + list(map(
            get_dll_path,
            ('usb-1.0 mtp expat sqlite3 ffi z lzma openjp2 poppler dbus-1 iconv xml2 xslt jpeg png16'
             ' webp webpmux webpdemux sharpyuv exslt ncursesw readline chm hunspell-1.7 hyphen'
             ' icudata icui18n icuuc icuio stemmer gcrypt gpg-error uchardet graphite2'
             ' brotlicommon brotlidec brotlienc zstd podofo ssl crypto deflate tiff'
             ' gobject-2.0 glib-2.0 gthread-2.0 gmodule-2.0 gio-2.0 dbus-glib-1').split()
        )) + [
            # debian/ubuntu for for some typical stupid reason use libpcre.so.3
            # instead of libpcre.so.0 like other distros. And Qt's idiotic build
            # system links against this pcre library despite being told to use
            # the bundled pcre. Since libpcre doesn't depend on anything other
            # than libc and libpthread we bundle the Ubuntu one here
            glob.glob('/usr/lib/*/libpcre.so.3')[0],

            get_dll_path('bz2', 2), j(PREFIX, 'lib', 'libunrar.so'),
            get_dll_path('python' + py_ver, 2), get_dll_path('jbig', 2),

            # We don't include libstdc++.so as the OpenGL dlls on the target
            # computer fail to load in the QPA xcb plugin if they were compiled
            # with a newer version of gcc than the one on the build computer.
            # libstdc++, like glibc is forward compatible and I don't think any
            # distros do not have libstdc++.so.6, so it should be safe to leave it out.
            # https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html (The current
            # debian stable libstdc++ is  libstdc++.so.6.0.17)
    ] + list(map(qt_get_dll_path, QT_DLLS)) + list(map(ffmpeg_get_dll_path, ffmpeg_dlls))


class Env:

    def __init__(self):
        self.src_root = CALIBRE_DIR
        self.base = mkdtemp('frozen-')
        self.lib_dir = j(self.base, 'lib')
        self.py_dir = j(self.lib_dir, 'python' + py_ver)
        os.makedirs(self.py_dir)
        self.bin_dir = j(self.base, 'bin')
        os.mkdir(self.bin_dir)
        self.SRC = j(self.src_root, 'src')
        self.obj_dir = mkdtemp('launchers-')


def ignore_in_lib(base, items, ignored_dirs=None):
    ans = []
    if ignored_dirs is None:
        ignored_dirs = {'.svn', '.bzr', '.git', 'test', 'tests', 'testing'}
    for name in items:
        path = j(base, name)
        is_kakasi = 'pykakasi' in path
        if os.path.isdir(path):
            if name != 'plugins' and (name in ignored_dirs or not is_package_dir(path)) and not (is_kakasi and name == 'data'):
                ans.append(name)
        else:
            if name.rpartition('.')[-1] not in ('so', 'py') and not (is_kakasi and name.endswith('.db')):
                ans.append(name)
    return ans


def import_site_packages(srcdir, dest):
    if not os.path.exists(dest):
        os.mkdir(dest)
    for x in os.listdir(srcdir):
        ext = x.rpartition('.')[-1]
        f = j(srcdir, x)
        if ext in ('py', 'so'):
            shutil.copy2(f, dest)
        elif ext == 'pth' and x != 'setuptools.pth':
            for line in open(f, 'rb').read().decode('utf-8').splitlines():
                src = os.path.abspath(j(srcdir, line))
                if os.path.exists(src) and os.path.isdir(src):
                    import_site_packages(src, dest)
        elif is_package_dir(f):
            shutil.copytree(f, j(dest, x), ignore=ignore_in_lib)


def copy_piper(env):
    print('Copying piper...')
    copy_piper_dir(PREFIX, env.bin_dir)


def copy_libs(env):
    print('Copying libs...')

    for x in binary_includes():
        dest = env.bin_dir if '/bin/' in x else env.lib_dir
        shutil.copy2(x, dest)
        os.chmod(j(
            dest, os.path.basename(x)),
            stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
    for x in ('ossl-modules',):
        shutil.copytree(os.path.join(LIBDIR, x), os.path.join(env.lib_dir, x))

    base = j(QT_PREFIX, 'plugins')
    dest = j(env.lib_dir, '..', 'plugins')
    os.mkdir(dest)
    for x in QT_PLUGINS:
        if x not in ('audio', 'printsupport'):
            shutil.copytree(j(base, x), j(dest, x))
    dest = j(env.lib_dir, '..', 'libexec')
    os.mkdir(dest)
    shutil.copy2(os.path.join(QT_PREFIX, 'libexec', 'QtWebEngineProcess'), dest)


def copy_python(env, ext_dir):
    print('Copying python...')
    srcdir = j(PREFIX, 'lib/python' + py_ver)

    for x in os.listdir(srcdir):
        y = j(srcdir, x)
        ext = os.path.splitext(x)[1]
        if os.path.isdir(y) and x not in ('test', 'hotshot',
                                          'site-packages', 'idlelib', 'dist-packages'):
            shutil.copytree(y, j(env.py_dir, x), ignore=ignore_in_lib)
        if os.path.isfile(y) and ext in ('.py', '.so'):
            shutil.copy2(y, env.py_dir)

    srcdir = j(srcdir, 'site-packages')
    dest = j(env.py_dir, 'site-packages')
    import_site_packages(srcdir, dest)

    for x in os.listdir(env.SRC):
        c = j(env.SRC, x)
        if os.path.exists(j(c, '__init__.py')):
            shutil.copytree(c, j(dest, x), ignore=partial(ignore_in_lib, ignored_dirs={}))
        elif os.path.isfile(c):
            shutil.copy2(c, j(dest, x))
    shutil.copytree(j(env.src_root, 'resources'), j(env.base, 'resources'))
    for pak in glob.glob(j(QT_PREFIX, 'resources', '*')):
        shutil.copy2(pak, j(env.base, 'resources'))
    os.mkdir(j(env.base, 'translations'))
    shutil.copytree(j(QT_PREFIX, 'translations', 'qtwebengine_locales'), j(env.base, 'translations', 'qtwebengine_locales'))
    sitepy = j(self_dir, 'site.py')
    shutil.copy2(sitepy, j(env.py_dir, 'site.py'))

    pdir = j(env.lib_dir, 'calibre-extensions')
    if not os.path.exists(pdir):
        os.mkdir(pdir)
    fix_pycryptodome(j(env.py_dir, 'site-packages'))
    for x in os.listdir(j(env.py_dir, 'site-packages')):
        os.rename(j(env.py_dir, 'site-packages', x), j(env.py_dir, x))
    os.rmdir(j(env.py_dir, 'site-packages'))
    print('Extracting extension modules from', ext_dir, 'to', pdir)
    ext_map = extract_extension_modules(ext_dir, pdir)
    shutil.rmtree(j(env.py_dir, 'calibre', 'plugins'))
    print('Extracting extension modules from', env.py_dir, 'to', pdir)
    ext_map.update(extract_extension_modules(env.py_dir, pdir))
    py_compile(env.py_dir)
    freeze_python(env.py_dir, pdir, env.obj_dir, ext_map, develop_mode_env_var='CALIBRE_DEVELOP_FROM')
    shutil.rmtree(env.py_dir)


def build_launchers(env):
    base = self_dir
    sources = [j(base, x) for x in ['util.c']]
    objects = [j(env.obj_dir, os.path.basename(x) + '.o') for x in sources]
    cflags = '-fno-strict-aliasing -W -Wall -c -O2 -pipe -DPY_VERSION_MAJOR={} -DPY_VERSION_MINOR={}'.format(*py_ver.split('.'))
    cflags = cflags.split() + ['-I%s/include/python%s' % (PREFIX, py_ver)]
    cflags += [f'-I{path_to_freeze_dir()}', f'-I{env.obj_dir}']
    for src, obj in zip(sources, objects):
        cmd = ['gcc'] + cflags + ['-fPIC', '-o', obj, src]
        run(*cmd)

    dll = j(env.lib_dir, 'libcalibre-launcher.so')
    cmd = ['gcc', '-O2', '-Wl,--rpath=$ORIGIN/../lib', '-fPIC', '-o', dll, '-shared'] + objects + \
        ['-L%s/lib' % PREFIX, '-lpython' + py_ver]
    run(*cmd)

    src = j(base, 'main.c')

    modules, basenames, functions = calibre_constants['modules'].copy(), calibre_constants['basenames'].copy(), calibre_constants['functions'].copy()
    modules['console'].append('calibre.linux')
    basenames['console'].append('calibre_postinstall')
    functions['console'].append('main')
    c_launcher = '/tmp/calibre-c-launcher'
    lsrc = os.path.join(base, 'launcher.c')
    cmd = ['gcc', '-O2', '-o', c_launcher, lsrc, ]
    run(*cmd)

    jobs = []
    for typ in ('console', 'gui', ):
        for mod, bname, func in zip(modules[typ], basenames[typ], functions[typ]):
            xflags = list(cflags)
            xflags.remove('-c')
            xflags += ['-DGUI_APP=' + ('1' if typ == 'gui' else '0')]
            xflags += ['-DMODULE=L"%s"' % mod, '-DBASENAME=L"%s"' % bname,
                       '-DFUNCTION=L"%s"' % func]

            exe = j(env.bin_dir, bname)
            cmd = ['gcc'] + xflags + [src, '-o', exe, '-L' + env.lib_dir, '-lcalibre-launcher']
            jobs.append(create_job(cmd))
            sh = j(env.base, bname)
            shutil.copy2(c_launcher, sh)
            os.chmod(sh,
                     stat.S_IREAD | stat.S_IEXEC | stat.S_IWRITE | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
    if jobs:
        if not parallel_build(jobs, verbose=False):
            raise SystemExit(1)


def is_elf(path):
    with open(path, 'rb') as f:
        return f.read(4) == b'\x7fELF'


STRIPCMD = ['strip']


def strip_files(files, argv_max=(256 * 1024)):
    """ Strip a list of files """
    while files:
        cmd = list(STRIPCMD)
        pathlen = sum(len(s) + 1 for s in cmd)
        while pathlen < argv_max and files:
            f = files.pop()
            cmd.append(f)
            pathlen += len(f) + 1
        if len(cmd) > len(STRIPCMD):
            all_files = cmd[len(STRIPCMD):]
            unwritable_files = tuple(filter(None, (None if os.access(x, os.W_OK) else (x, os.stat(x).st_mode) for x in all_files)))
            [os.chmod(x, stat.S_IWRITE | old_mode) for x, old_mode in unwritable_files]
            subprocess.check_call(cmd)
            [os.chmod(x, old_mode) for x, old_mode in unwritable_files]


def strip_binaries(env):
    files = {j(env.bin_dir, x) for x in os.listdir(env.bin_dir) if x != 'piper'} | {
        x for x in {
            j(os.path.dirname(env.bin_dir), x) for x in os.listdir(env.bin_dir)} if os.path.exists(x)}
    for x in walk(env.lib_dir):
        x = os.path.realpath(x)
        if x not in files and is_elf(x):
            files.add(x)
    files.add(j(env.lib_dir, '..', 'libexec', 'QtWebEngineProcess'))
    print('Stripping %d files...' % len(files))
    before = sum(os.path.getsize(x) for x in files)
    strip_files(files)
    after = sum(os.path.getsize(x) for x in files)
    print('Stripped %.1f MB' % ((before - after) / (1024 * 1024.)))


def create_tarfile(env, compression_level='9'):
    print('Creating archive...')
    base = OUTPUT_DIR
    arch = 'arm64' if 'arm64' in os.environ['BYPY_ARCH'] else ('i686' if 'i386' in os.environ['BYPY_ARCH'] else 'x86_64')
    try:
        shutil.rmtree(base)
    except EnvironmentError as err:
        if err.errno not in (errno.ENOENT, errno.EBUSY):
            raise
    os.makedirs(base, exist_ok=True)  # when base is a mount point deleting it fails with EBUSY
    dist = os.path.join(base, '%s-%s-%s.tar' % (calibre_constants['appname'], calibre_constants['version'], arch))
    with tarfile.open(dist, mode='w', format=tarfile.PAX_FORMAT) as tf:
        cwd = os.getcwd()
        os.chdir(env.base)
        try:
            for x in os.listdir('.'):
                tf.add(x)
        finally:
            os.chdir(cwd)
    print('Compressing archive...')
    ans = dist.rpartition('.')[0] + '.txz'
    start_time = time.time()
    subprocess.check_call(['xz', '--verbose', '--threads=0', '-f', '-' + compression_level, dist])
    secs = time.time() - start_time
    print('Compressed in %d minutes %d seconds' % (secs // 60, secs % 60))
    os.rename(dist + '.xz', ans)
    print('Archive %s created: %.2f MB' % (
        os.path.basename(ans), os.stat(ans).st_size / (1024.**2)))


def main():
    args = globals()['args']
    ext_dir = globals()['ext_dir']
    run_tests = iv['run_tests']
    env = Env()
    copy_libs(env)
    copy_python(env, ext_dir)
    copy_piper(env)
    build_launchers(env)
    if not args.skip_tests:
        run_tests(j(env.base, 'calibre-debug'), env.base)
    if not args.dont_strip:
        strip_binaries(env)
    create_tarfile(env, args.compression_level)


if __name__ == '__main__':
    main()