Work on new calibre macOS build

bypy/init_env.py

@@ -2,11 +2,18 @@
 # vim:fileencoding=utf-8
 # License: GPL v3 Copyright: 2019, Kovid Goyal <kovid at kovidgoyal.net>
 
+from __future__ import print_function
+
 import json
 import os
 import re
+import subprocess
+import sys
 
-from bypy.constants import SRC as CALIBRE_DIR
+from bypy.constants import (
+    LIBDIR, PREFIX, PYTHON, SRC as CALIBRE_DIR, build_dir, worker_env
+)
+from bypy.utils import run_shell
 
 
 def read_cal_file(name):
@@ -61,5 +68,35 @@ def initialize_constants():
     return calibre_constants
 
 
+def run(*args):
+    env = os.environ.copy()
+    env.update(worker_env)
+    env['SW'] = PREFIX
+    env['LD_LIBRARY_PATH'] = LIBDIR
+    env['SIP_BIN'] = os.path.join(PREFIX, 'bin', 'sip')
+    env['QMAKE'] = os.path.join(PREFIX, 'qt', 'bin', 'qmake')
+    return subprocess.call(list(args), env=env, cwd=CALIBRE_DIR)
+
+
+def build_c_extensions(ext_dir):
+    bdir = os.path.join(build_dir(), 'calibre-extension-objects')
+    if run(
+        PYTHON, 'setup.py', 'build',
+        '--output-dir', ext_dir, '--build-dir', bdir
+    ) != 0:
+        print('Building of calibre C extensions failed', file=sys.stderr)
+        os.chdir(CALIBRE_DIR)
+        run_shell()
+        raise SystemExit('Building of calibre C extensions failed')
+
+
+def run_tests(path_to_calibre_debug, cwd_on_failure):
+    if run(path_to_calibre_debug, '--test-build') != 0:
+        os.chdir(cwd_on_failure)
+        print('running calibre build tests failed', file=sys.stderr)
+        run_shell()
+        raise SystemExit('running calibre build tests failed')
+
+
 if __name__ == 'program':
     calibre_constants = initialize_constants()

bypy/linux/__main__.py

@@ -10,19 +10,16 @@ import os
 import shutil
 import stat
 import subprocess
-import sys
 import tarfile
 import time
 from functools import partial
 
 from bypy.constants import (
-    LIBDIR, PREFIX, PYTHON, SRC as CALIBRE_DIR, SW, build_dir, is64bit,
-    python_major_minor_version, worker_env
+    PREFIX, SRC as CALIBRE_DIR, SW, is64bit, python_major_minor_version
 )
 from bypy.pkgs.qt import PYQT_MODULES, QT_DLLS, QT_PLUGINS
 from bypy.utils import (
-    create_job, get_dll_path, mkdtemp, parallel_build, py_compile, run, run_shell,
-    walk
+    create_job, get_dll_path, mkdtemp, parallel_build, py_compile, run, walk
 )
 
 j = os.path.join
@@ -282,36 +279,12 @@ def create_tarfile(env, compression_level='9'):
         os.path.basename(ans), os.stat(ans).st_size / (1024.**2)))
 
 
-def run_tests(path_to_calibre_debug, cwd_on_failure):
-    p = subprocess.Popen([path_to_calibre_debug, '--test-build'])
-    if p.wait() != 0:
-        os.chdir(cwd_on_failure)
-        print('running calibre build tests failed', file=sys.stderr)
-        run_shell()
-        raise SystemExit(p.wait())
-
-
-def build_extensions(env, ext_dir):
-    wenv = os.environ.copy()
-    wenv.update(worker_env)
-    wenv['LD_LIBRARY_PATH'] = LIBDIR
-    wenv['QMAKE'] = os.path.join(QT_PREFIX, 'bin', 'qmake')
-    wenv['SW'] = PREFIX
-    wenv['SIP_BIN'] = os.path.join(PREFIX, 'bin', 'sip')
-    p = subprocess.Popen([PYTHON, 'setup.py', 'build', '--build-dir=' + build_dir(), '--output-dir=' + ext_dir], env=wenv, cwd=CALIBRE_DIR)
-    if p.wait() != 0:
-        os.chdir(CALIBRE_DIR)
-        print('building calibre extensions failed', file=sys.stderr)
-        run_shell()
-        raise SystemExit(p.returncode)
-
-
 def main():
     args = globals()['args']
     ext_dir = globals()['ext_dir']
+    run_tests = globals()['init_env']['run_tests']
     env = Env()
     copy_libs(env)
-    build_extensions(env, ext_dir)
     copy_python(env, ext_dir)
     build_launchers(env)
     if not args.skip_tests:

bypy/macos/__main__.py

@@ -0,0 +1,724 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+# License: GPLv3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import errno
+import glob
+import json
+import operator
+import os
+import plistlib
+import runpy
+import shutil
+import stat
+import subprocess
+import sys
+import tempfile
+import time
+import zipfile
+from functools import partial, reduce
+from itertools import repeat
+
+from bypy.constants import (
+    PREFIX, PYTHON, SRC as CALIBRE_DIR, SW, python_major_minor_version
+)
+from bypy.pkgs.qt import PYQT_MODULES, QT_DLLS, QT_PLUGINS
+from bypy.utils import current_dir, mkdtemp, py_compile, timeit, walk
+
+
+abspath, join, basename, dirname = os.path.abspath, os.path.join, os.path.basename, os.path.dirname
+calibre_constants = globals()['init_env']['calibre_constants']
+py_ver = '.'.join(map(str, python_major_minor_version()))
+sign_app = runpy.run_path(join(dirname(abspath(__file__)), 'sign.py'))['sign_app']
+
+QT_PREFIX = os.path.join(PREFIX, 'qt')
+QT_FRAMEWORKS = [x.replace('5', '') for x in QT_DLLS]
+
+ENV = dict(
+    FONTCONFIG_PATH='@executable_path/../Resources/fonts',
+    FONTCONFIG_FILE='@executable_path/../Resources/fonts/fonts.conf',
+    QT_PLUGIN_PATH='@executable_path/../MacOS/qt-plugins',
+    PYTHONIOENCODING='UTF-8',
+    SSL_CERT_FILE='@executable_path/../Resources/resources/mozilla-ca-certs.pem',
+)
+APPNAME, VERSION = calibre_constants['appname'], calibre_constants['version']
+basenames, main_modules, main_functions = calibre_constants['basenames'], calibre_constants['modules'], calibre_constants['functions']
+
+
+def compile_launcher_lib(contents_dir, gcc, base):
+    print('\tCompiling calibre_launcher.dylib')
+    fd = join(contents_dir, 'Frameworks')
+    dest = join(fd, 'calibre-launcher.dylib')
+    src = join(base, 'util.c')
+    cmd = [gcc] + '-Wall -dynamiclib -std=gnu99'.split() + [src] + \
+        ['-I' + base] + \
+        ['-I%s/python/Python.framework/Versions/Current/Headers' % PREFIX] + \
+        '-current_version 1.0 -compatibility_version 1.0'.split() + \
+        '-fvisibility=hidden -o'.split() + [dest] + \
+        ['-install_name',
+         '@executable_path/../Frameworks/' + os.path.basename(dest)] + \
+        [('-F%s/python' % PREFIX), '-framework', 'Python', '-framework', 'CoreFoundation', '-headerpad_max_install_names']
+    # print('\t'+' '.join(cmd))
+    sys.stdout.flush()
+    subprocess.check_call(cmd)
+    return dest
+
+
+gcc = os.environ.get('CC', 'clang')
+
+
+def compile_launchers(contents_dir, xprograms, pyver):
+    base = dirname(abspath(__file__))
+    lib = compile_launcher_lib(contents_dir, gcc, base)
+    src = open(join(base, 'launcher.c'), 'rb').read().decode('utf-8')
+    env, env_vals = [], []
+    for key, val in ENV.items():
+        env.append('"%s"' % key)
+        env_vals.append('"%s"' % val)
+    env = ', '.join(env) + ', '
+    env_vals = ', '.join(env_vals) + ', '
+    src = src.replace('/*ENV_VARS*/', env)
+    src = src.replace('/*ENV_VAR_VALS*/', env_vals)
+    programs = [lib]
+    for program, x in xprograms.items():
+        module, func, ptype = x
+        print('\tCompiling', program)
+        out = join(contents_dir, 'MacOS', program)
+        programs.append(out)
+        psrc = src.replace('**PROGRAM**', program)
+        psrc = psrc.replace('**MODULE**', module)
+        psrc = psrc.replace('**FUNCTION**', func)
+        psrc = psrc.replace('**PYVER**', pyver)
+        psrc = psrc.replace('**IS_GUI**', ('1' if ptype == 'gui' else '0'))
+        fsrc = '/tmp/%s.c' % program
+        with open(fsrc, 'wb') as f:
+            f.write(psrc.encode('utf-8'))
+        cmd = [gcc, '-Wall', '-I' + base, fsrc, lib, '-o', out,
+               '-headerpad_max_install_names']
+        # print('\t'+' '.join(cmd))
+        sys.stdout.flush()
+        subprocess.check_call(cmd)
+    return programs
+
+
+def flipwritable(fn, mode=None):
+    """
+    Flip the writability of a file and return the old mode. Returns None
+    if the file is already writable.
+    """
+    if os.access(fn, os.W_OK):
+        return None
+    old_mode = os.stat(fn).st_mode
+    os.chmod(fn, stat.S_IWRITE | old_mode)
+    return old_mode
+
+
+STRIPCMD = ['/usr/bin/strip', '-x', '-S', '-']
+
+
+def strip_files(files, argv_max=(256 * 1024)):
+    """
+    Strip a list of files
+    """
+    tostrip = [(fn, flipwritable(fn)) for fn in files if os.path.exists(fn)]
+    while tostrip:
+        cmd = list(STRIPCMD)
+        flips = []
+        pathlen = reduce(operator.add, [len(s) + 1 for s in cmd])
+        while pathlen < argv_max:
+            if not tostrip:
+                break
+            added, flip = tostrip.pop()
+            pathlen += len(added) + 1
+            cmd.append(added)
+            flips.append((added, flip))
+        else:
+            cmd.pop()
+            tostrip.append(flips.pop())
+        os.spawnv(os.P_WAIT, cmd[0], cmd)
+        for args in flips:
+            flipwritable(*args)
+
+
+def flush(func):
+    def ff(*args, **kwargs):
+        sys.stdout.flush()
+        sys.stderr.flush()
+        ret = func(*args, **kwargs)
+        sys.stdout.flush()
+        sys.stderr.flush()
+        return ret
+    return ff
+
+
+class Freeze(object):
+
+    FID = '@executable_path/../Frameworks'
+
+    def __init__(self, build_dir, ext_dir, test_runner, test_launchers=False, dont_strip=False, sign_installers=False):
+        self.build_dir = build_dir
+        self.sign_installers = sign_installers
+        self.ext_dir = ext_dir
+        self.test_runner = test_runner
+        self.dont_strip = dont_strip
+        self.contents_dir = join(self.build_dir, 'Contents')
+        self.resources_dir = join(self.contents_dir, 'Resources')
+        self.frameworks_dir = join(self.contents_dir, 'Frameworks')
+        self.site_packages = join(self.resources_dir, 'Python', 'site-packages')
+        self.to_strip = []
+        self.warnings = []
+
+        self.run(test_launchers)
+
+    def run(self, test_launchers):
+        ret = 0
+        if not test_launchers:
+            if os.path.exists(self.build_dir):
+                shutil.rmtree(self.build_dir)
+            os.makedirs(self.build_dir)
+            self.create_skeleton()
+            self.create_plist()
+
+            self.add_python_framework()
+            self.add_site_packages()
+            self.add_stdlib()
+            self.add_qt_frameworks()
+            self.add_calibre_plugins()
+            self.add_podofo()
+            self.add_poppler()
+            self.add_imaging_libs()
+            self.add_fontconfig()
+            self.add_misc_libraries()
+
+            self.add_resources()
+            self.compile_py_modules()
+
+        self.copy_site()
+        self.create_exe()
+        if not test_launchers and not self.dont_strip:
+            self.strip_files()
+        if not test_launchers:
+            self.create_console_app()
+            self.create_gui_apps()
+
+        self.run_tests()
+        ret = self.makedmg(self.build_dir, APPNAME + '-' + VERSION)
+
+        return ret
+
+    @flush
+    def run_tests(self):
+        cc_dir = os.path.join(self.contents_dir, 'calibre-debug.app', 'Contents')
+        self.test_runner(join(cc_dir, 'MacOS', 'calibre-debug'), self.contents_dir)
+
+    @flush
+    def add_resources(self):
+        shutil.copytree('resources', os.path.join(self.resources_dir,
+                                                  'resources'))
+
+    @flush
+    def strip_files(self):
+        print('\nStripping files...')
+        strip_files(self.to_strip)
+
+    @flush
+    def create_exe(self):
+        print('\nCreating launchers')
+        programs = {}
+        progs = []
+        for x in ('console', 'gui'):
+            progs += list(zip(basenames[x], main_modules[x], main_functions[x], repeat(x)))
+        for program, module, func, ptype in progs:
+            programs[program] = (module, func, ptype)
+        programs = compile_launchers(self.contents_dir, programs, py_ver)
+        for out in programs:
+            self.fix_dependencies_in_lib(out)
+
+    @flush
+    def set_id(self, path_to_lib, new_id):
+        old_mode = flipwritable(path_to_lib)
+        subprocess.check_call(['install_name_tool', '-id', new_id, path_to_lib])
+        if old_mode is not None:
+            flipwritable(path_to_lib, old_mode)
+
+    @flush
+    def get_dependencies(self, path_to_lib):
+        install_name = subprocess.check_output(['otool', '-D', path_to_lib]).splitlines()[-1].strip()
+        raw = subprocess.check_output(['otool', '-L', path_to_lib]).decode('utf-8')
+        for line in raw.splitlines():
+            if 'compatibility' not in line or line.strip().endswith(':'):
+                continue
+            idx = line.find('(')
+            path = line[:idx].strip()
+            yield path, path == install_name
+
+    @flush
+    def get_local_dependencies(self, path_to_lib):
+        for x, is_id in self.get_dependencies(path_to_lib):
+            if x.startswith('@rpath/Qt'):
+                yield x, x[len('@rpath/'):], is_id
+            else:
+                for y in (PREFIX + '/lib/', PREFIX + '/python/Python.framework/'):
+                    if x.startswith(y):
+                        if y == PREFIX + '/python/Python.framework/':
+                            y = PREFIX + '/python/'
+                        yield x, x[len(y):], is_id
+                        break
+
+    @flush
+    def change_dep(self, old_dep, new_dep, is_id, path_to_lib):
+        cmd = ['-id', new_dep] if is_id else ['-change', old_dep, new_dep]
+        subprocess.check_call(['install_name_tool'] + cmd + [path_to_lib])
+
+    @flush
+    def fix_dependencies_in_lib(self, path_to_lib):
+        self.to_strip.append(path_to_lib)
+        old_mode = flipwritable(path_to_lib)
+        for dep, bname, is_id in self.get_local_dependencies(path_to_lib):
+            ndep = self.FID + '/' + bname
+            self.change_dep(dep, ndep, is_id, path_to_lib)
+        ldeps = list(self.get_local_dependencies(path_to_lib))
+        if ldeps:
+            print('\nFailed to fix dependencies in', path_to_lib)
+            print('Remaining local dependencies:', ldeps)
+            raise SystemExit(1)
+        if old_mode is not None:
+            flipwritable(path_to_lib, old_mode)
+
+    @flush
+    def add_python_framework(self):
+        print('\nAdding Python framework')
+        src = join(PREFIX + '/python', 'Python.framework')
+        x = join(self.frameworks_dir, 'Python.framework')
+        curr = os.path.realpath(join(src, 'Versions', 'Current'))
+        currd = join(x, 'Versions', basename(curr))
+        rd = join(currd, 'Resources')
+        os.makedirs(rd)
+        shutil.copy2(join(curr, 'Resources', 'Info.plist'), rd)
+        shutil.copy2(join(curr, 'Python'), currd)
+        self.set_id(join(currd, 'Python'),
+                    self.FID + '/Python.framework/Versions/%s/Python' % basename(curr))
+        # The following is needed for codesign in OS X >= 10.9.5
+        with current_dir(x):
+            os.symlink(basename(curr), 'Versions/Current')
+            for y in ('Python', 'Resources'):
+                os.symlink('Versions/Current/%s' % y, y)
+
+    @flush
+    def add_qt_frameworks(self):
+        print('\nAdding Qt Frameworks')
+        for f in QT_FRAMEWORKS:
+            self.add_qt_framework(f)
+        pdir = join(QT_PREFIX, 'plugins')
+        ddir = join(self.contents_dir, 'MacOS', 'qt-plugins')
+        os.mkdir(ddir)
+        for x in QT_PLUGINS:
+            shutil.copytree(join(pdir, x), join(ddir, x))
+        for l in glob.glob(join(ddir, '*/*.dylib')):
+            self.fix_dependencies_in_lib(l)
+            x = os.path.relpath(l, ddir)
+            self.set_id(l, '@executable_path/' + x)
+
+    def add_qt_framework(self, f):
+        libname = f
+        f = f + '.framework'
+        src = join(PREFIX, 'qt', 'lib', f)
+        ignore = shutil.ignore_patterns('Headers', '*.h', 'Headers/*')
+        dest = join(self.frameworks_dir, f)
+        shutil.copytree(src, dest, symlinks=True,
+                        ignore=ignore)
+        lib = os.path.realpath(join(dest, libname))
+        rpath = os.path.relpath(lib, self.frameworks_dir)
+        self.set_id(lib, self.FID + '/' + rpath)
+        self.fix_dependencies_in_lib(lib)
+        # The following is needed for codesign in OS X >= 10.9.5
+        # The presence of the .prl file in the root of the framework causes
+        # codesign to fail.
+        with current_dir(dest):
+            for x in os.listdir('.'):
+                if x != 'Versions' and not os.path.islink(x):
+                    os.remove(x)
+
+    @flush
+    def create_skeleton(self):
+        c = join(self.build_dir, 'Contents')
+        for x in ('Frameworks', 'MacOS', 'Resources'):
+            os.makedirs(join(c, x))
+        icons = glob.glob(join(CALIBRE_DIR, 'icons', 'icns', '*.iconset'))
+        if not icons:
+            raise SystemExit('Failed to find icns format icons')
+        for x in icons:
+            subprocess.check_call([
+                'iconutil', '-c', 'icns', x, '-o', join(
+                    self.resources_dir, basename(x).partition('.')[0] + '.icns')])
+
+    @flush
+    def add_calibre_plugins(self):
+        dest = join(self.frameworks_dir, 'plugins')
+        os.mkdir(dest)
+        plugins = glob.glob(self.ext_dir + '/*.so')
+        if not plugins:
+            raise SystemExit('No calibre plugins found in: ' + self.ext_dir)
+        for f in plugins:
+            shutil.copy2(f, dest)
+            self.fix_dependencies_in_lib(join(dest, basename(f)))
+
+    @flush
+    def create_plist(self):
+        BOOK_EXTENSIONS = calibre_constants['book_extensions']
+        env = dict(**ENV)
+        env['CALIBRE_LAUNCHED_FROM_BUNDLE'] = '1'
+        docs = [{'CFBundleTypeName': 'E-book',
+                 'CFBundleTypeExtensions': list(BOOK_EXTENSIONS),
+                 'CFBundleTypeIconFile': 'book.icns',
+                 'CFBundleTypeRole': 'Viewer',
+                 }]
+
+        pl = dict(
+            CFBundleDevelopmentRegion='English',
+            CFBundleDisplayName=APPNAME,
+            CFBundleName=APPNAME,
+            CFBundleIdentifier='net.kovidgoyal.calibre',
+            CFBundleVersion=VERSION,
+            CFBundleShortVersionString=VERSION,
+            CFBundlePackageType='APPL',
+            CFBundleSignature='????',
+            CFBundleExecutable='calibre',
+            CFBundleDocumentTypes=docs,
+            LSMinimumSystemVersion='10.9.5',
+            LSRequiresNativeExecution=True,
+            NSAppleScriptEnabled=False,
+            NSHumanReadableCopyright=time.strftime('Copyright %Y, Kovid Goyal'),
+            CFBundleGetInfoString=('calibre, an E-book management '
+                                   'application. Visit https://calibre-ebook.com for details.'),
+            CFBundleIconFile='calibre.icns',
+            NSHighResolutionCapable=True,
+            LSApplicationCategoryType='public.app-category.productivity',
+            LSEnvironment=env
+        )
+        with open(join(self.contents_dir, 'Info.plist'), 'wb') as p:
+            plistlib.dump(pl, p)
+
+    @flush
+    def install_dylib(self, path, set_id=True):
+        shutil.copy2(path, self.frameworks_dir)
+        if set_id:
+            self.set_id(join(self.frameworks_dir, basename(path)),
+                        self.FID + '/' + basename(path))
+        self.fix_dependencies_in_lib(join(self.frameworks_dir, basename(path)))
+
+    @flush
+    def add_podofo(self):
+        print('\nAdding PoDoFo')
+        pdf = join(PREFIX, 'lib', 'libpodofo.0.9.6.dylib')
+        self.install_dylib(pdf)
+
+    @flush
+    def add_poppler(self):
+        print('\nAdding poppler')
+        for x in ('libpoppler.87.dylib',):
+            self.install_dylib(os.path.join(PREFIX, 'lib', x))
+        for x in ('pdftohtml', 'pdftoppm', 'pdfinfo'):
+            self.install_dylib(os.path.join(PREFIX, 'bin', x), False)
+
+    @flush
+    def add_imaging_libs(self):
+        print('\nAdding libjpeg, libpng, libwebp, optipng and mozjpeg')
+        for x in ('jpeg.8', 'png16.16', 'webp.7'):
+            self.install_dylib(os.path.join(PREFIX, 'lib', 'lib%s.dylib' % x))
+        for x in 'optipng', 'JxrDecApp':
+            self.install_dylib(os.path.join(PREFIX, 'bin', x), False)
+        for x in ('jpegtran', 'cjpeg'):
+            self.install_dylib(os.path.join(PREFIX, 'private', 'mozjpeg', 'bin', x), False)
+
+    @flush
+    def add_fontconfig(self):
+        print('\nAdding fontconfig')
+        for x in ('fontconfig.1', 'freetype.6', 'expat.1'):
+            src = os.path.join(PREFIX, 'lib', 'lib' + x + '.dylib')
+            self.install_dylib(src)
+        dst = os.path.join(self.resources_dir, 'fonts')
+        if os.path.exists(dst):
+            shutil.rmtree(dst)
+        src = os.path.join(PREFIX, 'etc', 'fonts')
+        shutil.copytree(src, dst, symlinks=False)
+        fc = os.path.join(dst, 'fonts.conf')
+        raw = open(fc, 'rb').read().decode('utf-8')
+        raw = raw.replace('<dir>/usr/share/fonts</dir>', '''\
+        <dir>/Library/Fonts</dir>
+        <dir>/System/Library/Fonts</dir>
+        <dir>/usr/X11R6/lib/X11/fonts</dir>
+        <dir>/usr/share/fonts</dir>
+        <dir>/var/root/Library/Fonts</dir>
+        <dir>/usr/share/fonts</dir>
+        ''')
+        open(fc, 'wb').write(raw.encode('utf-8'))
+
+    @flush
+    def add_misc_libraries(self):
+        for x in (
+                'usb-1.0.0', 'mtp.9', 'chm.0', 'sqlite3.0',
+                'icudata.64', 'icui18n.64', 'icuio.64', 'icuuc.64',
+                'xslt.1', 'exslt.0', 'xml2.2', 'z.1', 'unrar',
+                'crypto.1.0.0', 'ssl.1.0.0', 'iconv.2',  # 'ltdl.7'
+        ):
+            print('\nAdding', x)
+            x = 'lib%s.dylib' % x
+            src = join(PREFIX, 'lib', x)
+            shutil.copy2(src, self.frameworks_dir)
+            dest = join(self.frameworks_dir, x)
+            self.set_id(dest, self.FID + '/' + x)
+            self.fix_dependencies_in_lib(dest)
+
+    @flush
+    def add_site_packages(self):
+        print('\nAdding site-packages')
+        os.makedirs(self.site_packages)
+        sys_path = json.loads(subprocess.check_output([
+            PYTHON, '-c', 'import sys, json; json.dump(sys.path, sys.stdout)']))
+        paths = reversed(tuple(map(abspath, [x for x in sys_path if x.startswith('/') and not x.startswith('/Library/')])))
+        upaths = []
+        for x in paths:
+            if x not in upaths and (x.endswith('.egg') or x.endswith('/site-packages')):
+                upaths.append(x)
+        upaths.append(join(CALIBRE_DIR, 'src'))
+        for x in upaths:
+            print('\t', x)
+            tdir = None
+            try:
+                if not os.path.isdir(x):
+                    zf = zipfile.ZipFile(x)
+                    tdir = tempfile.mkdtemp()
+                    zf.extractall(tdir)
+                    x = tdir
+                self.add_modules_from_dir(x)
+                self.add_packages_from_dir(x)
+            finally:
+                if tdir is not None:
+                    shutil.rmtree(tdir)
+        try:
+            shutil.rmtree(os.path.join(self.site_packages, 'calibre', 'plugins'))
+        except OSError as err:
+            if err.errno != errno.ENOENT:
+                raise
+        sp = join(self.resources_dir, 'Python', 'site-packages')
+        for x in os.listdir(join(sp, 'PyQt5')):
+            if x.endswith('.so') and x.rpartition('.')[0] not in PYQT_MODULES:
+                os.remove(join(sp, 'PyQt5', x))
+        os.remove(join(sp, 'PyQt5', 'uic/port_v3/proxy_base.py'))
+        self.remove_bytecode(sp)
+
+    @flush
+    def add_modules_from_dir(self, src):
+        for x in glob.glob(join(src, '*.py')) + glob.glob(join(src, '*.so')):
+            dest = os.path.join(self.site_packages, os.path.basename(x))
+            shutil.copy2(x, dest)
+            if x.endswith('.so'):
+                self.fix_dependencies_in_lib(dest)
+
+    @flush
+    def add_packages_from_dir(self, src):
+        for x in os.listdir(src):
+            x = join(src, x)
+            if os.path.isdir(x) and os.path.exists(join(x, '__init__.py')):
+                if self.filter_package(basename(x)):
+                    continue
+                self.add_package_dir(x)
+
+    @flush
+    def add_package_dir(self, x, dest=None):
+        def ignore(root, files):
+            ans = []
+            for y in files:
+                ext = os.path.splitext(y)[1]
+                if ext not in ('', '.py', '.so') or \
+                        (not ext and not os.path.isdir(join(root, y))):
+                    ans.append(y)
+
+            return ans
+        if dest is None:
+            dest = self.site_packages
+        dest = join(dest, basename(x))
+        shutil.copytree(x, dest, symlinks=True, ignore=ignore)
+        self.postprocess_package(x, dest)
+        for x in os.walk(dest):
+            for f in x[-1]:
+                if f.endswith('.so'):
+                    f = join(x[0], f)
+                    self.fix_dependencies_in_lib(f)
+
+    @flush
+    def filter_package(self, name):
+        return name in ('Cython', 'modulegraph', 'macholib', 'py2app',
+                        'bdist_mpkg', 'altgraph')
+
+    @flush
+    def postprocess_package(self, src_path, dest_path):
+        pass
+
+    @flush
+    def add_stdlib(self):
+        print('\nAdding python stdlib')
+        src = PREFIX + '/python/Python.framework/Versions/Current/lib/python'
+        src += py_ver
+        dest = join(self.resources_dir, 'Python', 'lib', 'python')
+        dest += py_ver
+        os.makedirs(dest)
+        for x in os.listdir(src):
+            if x in ('site-packages', 'config', 'test', 'lib2to3', 'lib-tk',
+                     'lib-old', 'idlelib', 'plat-mac', 'plat-darwin', 'site.py'):
+                continue
+            x = join(src, x)
+            if os.path.isdir(x):
+                self.add_package_dir(x, dest)
+            elif os.path.splitext(x)[1] in ('.so', '.py'):
+                shutil.copy2(x, dest)
+                dest2 = join(dest, basename(x))
+                if dest2.endswith('.so'):
+                    self.fix_dependencies_in_lib(dest2)
+
+        target = join(self.resources_dir, 'Python', 'lib')
+        self.remove_bytecode(target)
+        for path in walk(target):
+            if path.endswith('.so'):
+                self.fix_dependencies_in_lib(path)
+
+    @flush
+    def remove_bytecode(self, dest):
+        for x in os.walk(dest):
+            root = x[0]
+            for f in x[-1]:
+                if os.path.splitext(f) in ('.pyc', '.pyo'):
+                    os.remove(join(root, f))
+
+    @flush
+    def compile_py_modules(self):
+        print('\nCompiling Python modules')
+        base = join(self.resources_dir, 'Python')
+        py_compile(base)
+
+    def create_app_clone(self, name, specialise_plist, remove_doc_types=True):
+        print('\nCreating ' + name)
+        cc_dir = os.path.join(self.contents_dir, name, 'Contents')
+        exe_dir = join(cc_dir, 'MacOS')
+        os.makedirs(exe_dir)
+        for x in os.listdir(self.contents_dir):
+            if x.endswith('.app'):
+                continue
+            if x == 'Info.plist':
+                with open(join(self.contents_dir, x), 'rb') as r:
+                    plist = plistlib.load(r)
+                specialise_plist(plist)
+                if remove_doc_types:
+                    plist.pop('CFBundleDocumentTypes')
+                exe = plist['CFBundleExecutable']
+                # We cannot symlink the bundle executable as if we do,
+                # codesigning fails
+                plist['CFBundleExecutable'] = exe + '-placeholder-for-codesigning'
+                nexe = join(exe_dir, plist['CFBundleExecutable'])
+                base = os.path.dirname(abspath(__file__))
+                cmd = [gcc, '-Wall', '-Werror', '-DEXE_NAME="%s"' % exe, join(base, 'placeholder.c'), '-o', nexe, '-headerpad_max_install_names']
+                subprocess.check_call(cmd)
+                with open(join(cc_dir, x), 'wb') as p:
+                    plistlib.dump(plist, p)
+            elif x == 'MacOS':
+                for item in os.listdir(join(self.contents_dir, 'MacOS')):
+                    os.symlink('../../../MacOS/' + item, join(exe_dir, item))
+            else:
+                os.symlink(join('../..', x), join(cc_dir, x))
+
+    @flush
+    def create_console_app(self):
+        def specialise_plist(plist):
+            plist['LSBackgroundOnly'] = '1'
+            plist['CFBundleIdentifier'] = 'com.calibre-ebook.console'
+            plist['CFBundleExecutable'] = 'calibre-parallel'
+        self.create_app_clone('console.app', specialise_plist)
+
+    @flush
+    def create_gui_apps(self):
+        input_formats = sorted(json.loads(
+            subprocess.check_output([
+                join(self.contents_dir, 'MacOS', 'calibre-debug'), '-c',
+                'from calibre.customize.ui import all_input_formats; import sys, json; sys.stdout.write(json.dumps(set(all_input_formats())))'
+            ])
+        ))
+
+        def specialise_plist(launcher, remove_types, plist):
+            plist['CFBundleDisplayName'] = plist['CFBundleName'] = {
+                'ebook-viewer': 'E-book Viewer', 'ebook-edit': 'Edit Book', 'calibre-debug': 'calibre (debug)',
+            }[launcher]
+            plist['CFBundleExecutable'] = launcher
+            if launcher != 'calibre-debug':
+                plist['CFBundleIconFile'] = launcher + '.icns'
+            plist['CFBundleIdentifier'] = 'com.calibre-ebook.' + launcher
+            if not remove_types:
+                e = plist['CFBundleDocumentTypes'][0]
+                exts = 'epub azw3'.split() if launcher == 'ebook-edit' else input_formats
+                e['CFBundleTypeExtensions'] = exts
+        for launcher in ('ebook-viewer', 'ebook-edit', 'calibre-debug'):
+            remove_types = launcher == 'calibre-debug'
+            self.create_app_clone(launcher + '.app', partial(specialise_plist, launcher, remove_types), remove_doc_types=remove_types)
+
+    @flush
+    def copy_site(self):
+        base = os.path.dirname(abspath(__file__))
+        shutil.copy2(join(base, 'site.py'), join(self.resources_dir, 'Python',
+                                                 'lib', 'python' + py_ver))
+
+    @flush
+    def makedmg(self, d, volname, internet_enable=True, format='UDBZ'):
+        ''' Copy a directory d into a dmg named volname '''
+        print('\nSigning...')
+        sys.stdout.flush()
+        destdir = os.path.join(SW, 'dist')
+        try:
+            shutil.rmtree(destdir)
+        except EnvironmentError as err:
+            if err.errno != errno.ENOENT:
+                raise
+        os.mkdir(destdir)
+        dmg = os.path.join(destdir, volname + '.dmg')
+        if os.path.exists(dmg):
+            os.unlink(dmg)
+        tdir = tempfile.mkdtemp()
+        appdir = os.path.join(tdir, os.path.basename(d))
+        shutil.copytree(d, appdir, symlinks=True)
+        if self.sign_installers:
+            with timeit() as times:
+                sign_app(appdir)
+            print('Signing completed in %d minutes %d seconds' % tuple(times))
+        os.symlink('/Applications', os.path.join(tdir, 'Applications'))
+        size_in_mb = int(subprocess.check_output(['du', '-s', '-k', tdir]).decode('utf-8').split()[0]) / 1024.
+        cmd = ['/usr/bin/hdiutil', 'create', '-srcfolder', tdir, '-volname', volname, '-format', format]
+        if 190 < size_in_mb < 250:
+            # We need -size 255m because of a bug in hdiutil. When the size of
+            # srcfolder is close to 200MB hdiutil fails with
+            # diskimages-helper: resize request is above maximum size allowed.
+            cmd += ['-size', '255m']
+        print('\nCreating dmg...')
+        with timeit() as times:
+            subprocess.check_call(cmd + [dmg])
+            if internet_enable:
+                subprocess.check_call(['/usr/bin/hdiutil', 'internet-enable', '-yes', dmg])
+        print('dmg created in %d minutes and %d seconds' % tuple(times))
+        shutil.rmtree(tdir)
+        size = os.stat(dmg).st_size / (1024 * 1024.)
+        print('\nInstaller size: %.2fMB\n' % size)
+        return dmg
+
+
+def main(args, ext_dir, test_runner):
+    build_dir = abspath(join(mkdtemp('frozen-'), APPNAME + '.app'))
+    if args.skip_tests:
+        test_runner = lambda *a: None
+    Freeze(build_dir, ext_dir, test_runner, dont_strip=args.dont_strip, sign_installers=args.sign_installers)
+
+
+if __name__ == '__main__':
+    args = globals()['args']
+    ext_dir = globals()['ext_dir']
+    run_tests = globals()['init_env']['run_tests']
+    main(args, ext_dir, run_tests)

bypy/macos/launcher.c

@@ -0,0 +1,17 @@
+#include "util.h"
+#include <stdlib.h>
+
+// These variables must be filled in before compiling
+static const char *ENV_VARS[] = { /*ENV_VARS*/ NULL };
+static const char *ENV_VAR_VALS[] = { /*ENV_VAR_VALS*/ NULL};
+static char PROGRAM[] = "**PROGRAM**";
+static const char MODULE[] = "**MODULE**";
+static const char FUNCTION[] = "**FUNCTION**";
+static const char PYVER[] = "**PYVER**";
+
+
+int 
+main(int argc, const char **argv, const char **envp) {
+    return run(ENV_VARS, ENV_VAR_VALS, PROGRAM, MODULE, FUNCTION, PYVER, **IS_GUI**, argc, argv, envp);
+}
+

bypy/macos/placeholder.c

@@ -0,0 +1,41 @@
+/*
+ * placeholder.c
+ * Copyright (C) 2018 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the GPL3 license.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <libproc.h>
+#include <unistd.h>
+
+
+int
+main(int argc, char * const *argv, const char **envp) {
+    int ret;
+    pid_t pid;
+    char pathbuf[PROC_PIDPATHINFO_MAXSIZE], realpath_buf[PROC_PIDPATHINFO_MAXSIZE * 5];
+
+    pid = getpid();
+    ret = proc_pidpath(pid, pathbuf, sizeof(pathbuf));
+    if (ret <= 0) {
+        perror("failed to get executable path for current pid with error");
+        return 1;
+    }
+    char *path = realpath(pathbuf, realpath_buf);
+    if (path == NULL) {
+        perror("failed to get realpath for executable path with error");
+        return 1;
+    }
+    char *t = rindex(path, '/');
+    if (t == NULL) {
+        fprintf(stderr, "No / in executable path: %s\n", path);
+        return 1;
+    }
+    *(t + 1) = 0;
+    snprintf(t + 1, sizeof(realpath_buf) - strlen(path), "%s", EXE_NAME);
+    execv(path, argv);
+}

bypy/macos/sign.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python2
+# vim:fileencoding=utf-8
+# License: GPLv3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
+
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+import subprocess
+import os
+import plistlib
+from glob import glob
+
+
+from bypy.utils import current_dir
+
+CODESIGN_KEYCHAIN = '/Users/kovid/codesign.keychain'
+
+
+def codesign(items):
+    if isinstance(items, str):
+        items = [items]
+    # If you get errors while codesigning that look like "A timestamp was
+    # expected but not found" it means that codesign  failed to contact Apple's time
+    # servers, probably due to network congestion, so add --timestamp=none to
+    # this command line. That means the signature will fail once your code
+    # signing key expires and key revocation wont work, but...
+    subprocess.check_call(['codesign', '-s', 'Kovid Goyal', '--keychain', CODESIGN_KEYCHAIN] + list(items))
+
+
+def files_in(folder):
+    for record in os.walk(folder):
+        for f in record[-1]:
+            yield os.path.join(record[0], f)
+
+
+def expand_dirs(items):
+    items = set(items)
+    dirs = set(x for x in items if os.path.isdir(x))
+    items.difference_update(dirs)
+    for x in dirs:
+        items.update(set(files_in(x)))
+    return items
+
+
+def get_executable(info_path):
+    return plistlib.readPlist(info_path)['CFBundleExecutable']
+
+
+def sign_app(appdir):
+    appdir = os.path.abspath(appdir)
+    subprocess.check_call(['security', 'unlock-keychain', '-p', 'keychains are stupid', CODESIGN_KEYCHAIN])
+    with current_dir(os.path.join(appdir, 'Contents')):
+        executables = {get_executable('Info.plist')}
+
+        # Sign the sub application bundles
+        sub_apps = glob('*.app')
+        for sa in sub_apps:
+            exe = get_executable(sa + '/Contents/Info.plist')
+            if exe in executables:
+                raise ValueError('Multiple app bundles share the same executable: %s' % exe)
+            executables.add(exe)
+        codesign(sub_apps)
+
+        # Sign everything in MacOS except the main executables of the various
+        # app bundles which will be signed automatically by codesign when
+        # signing the app bundles
+        with current_dir('MacOS'):
+            items = set(os.listdir('.')) - executables
+            codesign(expand_dirs(items))
+
+        # Sign everything in Frameworks
+        with current_dir('Frameworks'):
+            fw = set(glob('*.framework'))
+            codesign(fw)
+            items = set(os.listdir('.')) - fw
+            codesign(expand_dirs(items))
+
+    # Now sign the main app
+    codesign(appdir)
+    # Verify the signature
+    subprocess.check_call(['codesign', '--deep', '--verify', '-v', appdir])
+    subprocess.check_call('spctl --verbose=4 --assess --type execute'.split() + [appdir])
+
+    return 0

bypy/macos/site.py

@@ -0,0 +1,154 @@
+"""
+Append module search paths for third-party packages to sys.path.
+
+This is stripped down and customized for use in py2app applications
+"""
+
+import sys
+import os
+
+def makepath(*paths):
+    dir = os.path.abspath(os.path.join(*paths))
+    return dir, os.path.normcase(dir)
+
+def abs__file__():
+    """Set all module __file__ attribute to an absolute path"""
+    for m in sys.modules.values():
+        if hasattr(m, '__loader__'):
+            continue   # don't mess with a PEP 302-supplied __file__
+        try:
+            m.__file__ = os.path.abspath(m.__file__)
+        except AttributeError:
+            continue
+
+# This ensures that the initial path provided by the interpreter contains
+# only absolute pathnames, even if we're running from the build directory.
+L = []
+_dirs_in_sys_path = {}
+dir = dircase = None  # sys.path may be empty at this point
+for dir in sys.path:
+    # Filter out duplicate paths (on case-insensitive file systems also
+    # if they only differ in case); turn relative paths into absolute
+    # paths.
+    dir, dircase = makepath(dir)
+    if dircase not in _dirs_in_sys_path:
+        L.append(dir)
+        _dirs_in_sys_path[dircase] = 1
+sys.path[:] = L
+del dir, dircase, L
+_dirs_in_sys_path = None
+
+def _init_pathinfo():
+    global _dirs_in_sys_path
+    _dirs_in_sys_path = d = {}
+    for dir in sys.path:
+        if dir and not os.path.isdir(dir):
+            continue
+        dir, dircase = makepath(dir)
+        d[dircase] = 1
+
+def addsitedir(sitedir):
+    global _dirs_in_sys_path
+    if _dirs_in_sys_path is None:
+        _init_pathinfo()
+        reset = 1
+    else:
+        reset = 0
+    sitedir, sitedircase = makepath(sitedir)
+    if sitedircase not in _dirs_in_sys_path:
+        sys.path.append(sitedir)        # Add path component
+    try:
+        names = os.listdir(sitedir)
+    except os.error:
+        return
+    names.sort()
+    for name in names:
+        if name[-4:] == os.extsep + "pth":
+            addpackage(sitedir, name)
+    if reset:
+        _dirs_in_sys_path = None
+
+def addpackage(sitedir, name):
+    global _dirs_in_sys_path
+    if _dirs_in_sys_path is None:
+        _init_pathinfo()
+        reset = 1
+    else:
+        reset = 0
+    fullname = os.path.join(sitedir, name)
+    try:
+        f = open(fullname)
+    except IOError:
+        return
+    while True:
+        dir = f.readline()
+        if not dir:
+            break
+        if dir[0] == '#':
+            continue
+        if dir.startswith("import"):
+            exec dir
+            continue
+        if dir[-1] == '\n':
+            dir = dir[:-1]
+        dir, dircase = makepath(sitedir, dir)
+        if dircase not in _dirs_in_sys_path and os.path.exists(dir):
+            sys.path.append(dir)
+            _dirs_in_sys_path[dircase] = 1
+    if reset:
+        _dirs_in_sys_path = None
+
+
+# Remove sys.setdefaultencoding() so that users cannot change the
+# encoding after initialization.  The test for presence is needed when
+# this module is run as a script, because this code is executed twice.
+#
+if hasattr(sys, "setdefaultencoding"):
+    sys.setdefaultencoding('utf-8')
+    del sys.setdefaultencoding
+
+def run_entry_point():
+    bname, mod, func = sys.calibre_basename, sys.calibre_module, sys.calibre_function
+    sys.argv[0] = bname
+    pmod = __import__(mod, fromlist=[1], level=0)
+    return getattr(pmod, func)()
+
+def add_calibre_vars(base):
+    sys.frameworks_dir = os.path.join(os.path.dirname(base), 'Frameworks')
+    sys.resources_location = os.path.abspath(os.path.join(base, 'resources'))
+    sys.extensions_location = os.path.join(sys.frameworks_dir, 'plugins')
+    sys.binaries_path = os.path.join(os.path.dirname(base), 'MacOS')
+    sys.console_binaries_path = os.path.join(os.path.dirname(base),
+        'console.app', 'Contents', 'MacOS')
+
+    dv = os.environ.get('CALIBRE_DEVELOP_FROM', None)
+    if dv and os.path.exists(dv):
+        sys.path.insert(0, os.path.abspath(dv))
+
+def nuke_stdout():
+    # Redirect stdout, stdin and stderr to /dev/null
+    from calibre.constants import plugins
+    plugins['speedup'][0].detach(os.devnull)
+
+def main():
+    global __file__
+
+    # Needed on OS X <= 10.8, which passes -psn_... as a command line arg when
+    # starting via launch services
+    for arg in tuple(sys.argv[1:]):
+        if arg.startswith('-psn_'):
+            sys.argv.remove(arg)
+
+    base = sys.resourcepath
+    sys.frozen = 'macosx_app'
+    sys.new_app_bundle = True
+    abs__file__()
+
+    add_calibre_vars(base)
+    addsitedir(sys.site_packages)
+
+    if sys.calibre_is_gui_app and not (
+            sys.stdout.isatty() or sys.stderr.isatty() or sys.stdin.isatty()):
+        nuke_stdout()
+
+    return run_entry_point()

bypy/macos/util.c

@@ -0,0 +1,229 @@
+#include "util.h"
+#include <stdlib.h>
+#include <strings.h>
+#include <CoreFoundation/CoreFoundation.h>
+#include <mach-o/dyld.h>
+#include <Python.h>
+
+#define EXPORT __attribute__((visibility("default")))
+
+static const char *ERR_OOM = "Out of memory";
+
+static int
+report_error(const char *msg) {
+    fprintf(stderr, "%s\n", msg);
+    fflush(stderr);
+    return -1;
+}
+
+static int
+report_code(const char *preamble, const char* msg, int code) {
+    fprintf(stderr, "%s: %s\n", preamble, msg);
+    fflush(stderr);
+    return code;
+}
+
+#define EXE "@executable_path/.."
+
+static void
+set_env_vars(const char **ENV_VARS, const char **ENV_VAR_VALS, const char* exe_path) {
+    int i = 0;
+    char buf[3*PATH_MAX];
+    const char *env_var, *val;
+
+    while(1) {
+        env_var = ENV_VARS[i];
+        if (env_var == NULL) break;
+        val = ENV_VAR_VALS[i++];
+        if (strstr(val, EXE) == val && strlen(val) >= strlen(EXE)+1) {
+            strncpy(buf, exe_path, 3*PATH_MAX-150);
+            strncpy(buf+strlen(exe_path), val+strlen(EXE), 150);
+            setenv(env_var, buf, 1);
+        } else
+            setenv(env_var, val, 1);
+    }
+    return;
+}
+
+void initialize_interpreter(const char **ENV_VARS, const char **ENV_VAR_VALS,
+        char *PROGRAM, const char *MODULE, const char *FUNCTION, const char *PYVER, int IS_GUI,
+        const char* exe_path, const char *rpath, int argc, const char **argv) {
+    PyObject *pargv, *v;
+    int i;
+    Py_OptimizeFlag = 2;
+    Py_NoSiteFlag = 1;
+    Py_DontWriteBytecodeFlag = 1;
+    Py_IgnoreEnvironmentFlag = 1;
+    Py_NoUserSiteDirectory = 1;
+    Py_HashRandomizationFlag = 1;
+
+    //Py_VerboseFlag = 1;
+    //Py_DebugFlag = 1;
+    
+    Py_SetProgramName(PROGRAM);
+
+    char pyhome[1000];
+    snprintf(pyhome, 1000, "%s/Python", rpath);
+    Py_SetPythonHome(pyhome);
+
+    set_env_vars(ENV_VARS, ENV_VAR_VALS, exe_path);
+
+    //printf("Path before Py_Initialize(): %s\r\n\n", Py_GetPath());
+    Py_Initialize();
+
+    char *dummy_argv[1] = {""};
+    PySys_SetArgv(1, dummy_argv);
+    //printf("Path after Py_Initialize(): %s\r\n\n", Py_GetPath());
+    char path[3000];
+    snprintf(path, 3000, "%s/lib/python%s:%s/lib/python%s/lib-dynload:%s/site-packages", pyhome, PYVER, pyhome, PYVER, pyhome);
+
+    PySys_SetPath(path);
+    //printf("Path set by me: %s\r\n\n", path);
+
+    PySys_SetObject("calibre_basename", PyBytes_FromString(PROGRAM));
+    PySys_SetObject("calibre_module", PyBytes_FromString(MODULE));
+    PySys_SetObject("calibre_function", PyBytes_FromString(FUNCTION));
+    PySys_SetObject("calibre_is_gui_app", ((IS_GUI) ? Py_True : Py_False));
+    PySys_SetObject("resourcepath", PyBytes_FromString(rpath));
+    snprintf(path, 3000, "%s/site-packages", pyhome);
+    PySys_SetObject("site_packages", PyBytes_FromString(pyhome));
+
+
+    pargv = PyList_New(argc);
+    if (pargv == NULL) exit(report_error(ERR_OOM));
+    for (i = 0; i < argc; i++) {
+        v = PyBytes_FromString(argv[i]);
+        if (v == NULL) exit(report_error(ERR_OOM));
+        PyList_SetItem(pargv, i, v);
+    }
+    PySys_SetObject("argv", pargv);
+
+}
+
+
+int pyobject_to_int(PyObject *res) {
+    int ret; PyObject *tmp;
+    tmp = PyNumber_Int(res);
+    if (tmp == NULL) ret = (PyObject_IsTrue(res)) ? 1 : 0;
+    else ret = (int)PyInt_AS_LONG(tmp);
+
+    return ret;
+}
+
+int handle_sysexit(PyObject *e) {
+    PyObject *code;
+
+    code = PyObject_GetAttrString(e, "code");
+    if (!code) return 0;
+    if (!PyInt_Check(code)) {
+        PyObject_Print(code, stderr, Py_PRINT_RAW);
+        fflush(stderr);
+    }
+    return pyobject_to_int(code);
+}
+
+int calibre_show_python_error(const char *preamble, int code) {
+    PyObject *exc, *val, *tb, *str;
+    int ret, issysexit = 0; char *i; 
+
+    if (!PyErr_Occurred()) return code;
+    issysexit = PyErr_ExceptionMatches(PyExc_SystemExit);
+
+    PyErr_Fetch(&exc, &val, &tb);
+
+    if (exc != NULL) {
+        PyErr_NormalizeException(&exc, &val, &tb);
+
+        if (issysexit) {
+            return (val) ? handle_sysexit(val) : 0;
+        }
+        if (val != NULL) {
+            str = PyObject_Unicode(val);
+            if (str == NULL) {
+                PyErr_Clear();
+                str = PyObject_Str(val);
+            }
+            i = PyString_AsString(str);
+            ret = report_code(preamble, (i==NULL)?ERR_OOM:i, code);
+            if (tb != NULL) {
+                PyErr_Restore(exc, val, tb);
+                PyErr_Print();
+            }
+            return ret;
+        }
+    }
+    return report_code(preamble, "", code);
+}
+
+EXPORT
+int
+run(const char **ENV_VARS, const char **ENV_VAR_VALS, char *PROGRAM,
+        const char *MODULE, const char *FUNCTION, const char *PYVER,
+        int IS_GUI, int argc, const char **argv, const char **envp) {
+    char *pathPtr = NULL, *t = NULL;
+    char buf[3*PATH_MAX];
+    int ret = 0, i;
+    PyObject *site, *mainf, *res;
+    uint32_t buf_size = PATH_MAX+1;
+    char *ebuf = calloc(buf_size, sizeof(char));
+
+    ret = _NSGetExecutablePath(ebuf, &buf_size);
+    if (ret == -1) {
+        free(ebuf);
+        ebuf = calloc(buf_size, sizeof(char));
+        if (_NSGetExecutablePath(ebuf, &buf_size) != 0)
+            return report_error("Failed to find real path of executable.");
+    }
+    pathPtr = realpath(ebuf, buf);
+    if (pathPtr == NULL) {
+        return report_error(strerror(errno));
+    }
+    for (i = 0; i < 3; i++) {
+        t = rindex(pathPtr, '/');
+        if (t == NULL) return report_error("Failed to determine bundle path.");
+        *t = '\0';
+    }
+    if (strstr(pathPtr, "/calibre.app/Contents/") != NULL) {
+        // We are one of the duplicate executables created to workaround codesign's limitations
+        for (i = 0; i < 2; i++) {
+            t = rindex(pathPtr, '/');
+            if (t == NULL) return report_error("Failed to resolve bundle path in dummy executable");
+            *t = '\0';
+        }
+    }
+
+    char rpath[PATH_MAX+1], exe_path[PATH_MAX+1];
+    snprintf(exe_path, PATH_MAX+1, "%s/Contents", pathPtr);
+    snprintf(rpath, PATH_MAX+1, "%s/Resources", exe_path);
+    initialize_interpreter(ENV_VARS, ENV_VAR_VALS, PROGRAM, MODULE, FUNCTION, PYVER, IS_GUI,
+            exe_path, rpath, argc, argv);
+
+    site = PyImport_ImportModule("site");
+
+    if (site == NULL)
+        ret = calibre_show_python_error("Failed to import site module",  -1);
+    else {
+        Py_XINCREF(site);
+
+        mainf = PyObject_GetAttrString(site, "main");
+        if (mainf == NULL || !PyCallable_Check(mainf)) 
+            ret = calibre_show_python_error("site module has no main function", -1);
+        else {
+            Py_XINCREF(mainf);
+            res = PyObject_CallObject(mainf, NULL);
+
+            if (res == NULL) 
+                ret = calibre_show_python_error("Python function terminated unexpectedly", -1);
+            else {
+            }
+        }
+    }
+    PyErr_Clear();
+    Py_Finalize();
+
+    //printf("11111 Returning: %d\r\n", ret);
+    return ret;
+}
+
+
+

bypy/macos/util.h

@@ -0,0 +1,5 @@
+#pragma once
+
+int run(const char **ENV_VARS, const char **ENV_VAR_VALS, char *PROGRAM,
+        const char *MODULE, const char *FUNCTION, const char *PYVER, int IS_GUI,
+        int argc, const char **argv, const char **envp);

setup/build_environment.py

@@ -149,12 +149,12 @@ if iswindows:
     podofo_lib = sw_lib_dir
 elif isosx:
     sw = os.environ.get('SW', os.path.expanduser('~/sw'))
-    podofo_inc = os.path.join(sw, 'include', 'podofo')
-    podofo_lib = os.path.join(sw, 'lib')
+    sw_inc_dir  = os.path.join(sw, 'include')
+    sw_lib_dir  = os.path.join(sw, 'lib')
+    podofo_inc = os.path.join(sw_inc_dir, 'podofo')
+    podofo_lib = sw_lib_dir
     ft_libs = ['freetype']
     ft_inc_dirs = [sw + '/include/freetype2']
-    icu_inc_dirs = [sw + '/include']
-    icu_lib_dirs = [sw + '/lib']
     SSL = os.environ.get('OPENSSL_DIR', os.path.join(sw, 'private', 'ssl'))
     openssl_inc_dirs = [os.path.join(SSL, 'include')]
     openssl_lib_dirs = [os.path.join(SSL, 'lib')]