IGN:calibre-x86_64 now build and runs on OS X Leopard

installer/osx/py2app/launcher.c

@@ -0,0 +1,138 @@
+#include <stdlib.h>
+#include <strings.h>
+#include <CoreFoundation/CoreFoundation.h>
+#include <mach-o/dyld.h>
+#include <Python.h>
+
+static const char *ERR_UNKNOWNPYTHONEXCEPTION = "An uncaught exception was raised during execution of the main script, but its class or name could not be determined";
+
+static int
+report_error(const char *msg) {
+    fprintf(stderr, msg);
+    fprintf(stderr, "\n");
+    fflush(stderr);
+    return -1;
+}
+
+// These variable must be filled in before compiling
+static const char *ENV_VARS[] = { /*ENV_VARS*/ NULL };
+static const char *ENV_VAR_VALS[] = { /*ENV_VAR_VALS*/ NULL};
+static char PROGRAM[] = "**PROGRAM**";
+static const char MODULE[] = "**MODULE**";
+
+#define EXE "@executable_path/.."
+
+static void
+set_env_vars(const char* exe_path, const char* rpath) {
+    int i = 0;
+    char buf[3*PATH_MAX];
+    const char *env_var, *val;
+
+    while(1) {
+        env_var = ENV_VARS[i];
+        if (env_var == NULL) break;
+        val = ENV_VAR_VALS[i++];
+        if (strstr(val, EXE) == val && strlen(val) >= strlen(EXE)+1) {
+            strncpy(buf, exe_path, 3*PATH_MAX-150);
+            strncpy(buf+strlen(exe_path), val+strlen(EXE), 150);
+            setenv(env_var, buf, 1);
+        } else
+            setenv(env_var, val, 1);
+    }
+    setenv("CALIBRE_LAUNCH_MODULE", MODULE, 1);
+    setenv("RESOURCEPATH", rpath, 1);
+    return;
+}
+
+int 
+main(int argc, char * const *argv, char * const *envp) {
+    char *pathPtr = NULL;
+    char buf[3*PATH_MAX];
+    int ret, i;
+
+    
+    uint32_t buf_size = PATH_MAX+1;
+    char *ebuf = calloc(buf_size, sizeof(char));
+    ret = _NSGetExecutablePath(ebuf, &buf_size);
+    if (ret == -1) {
+        free(ebuf);
+        ebuf = calloc(buf_size, sizeof(char));
+        if (_NSGetExecutablePath(ebuf, &buf_size) != 0)
+            return report_error("Failed to find real path of executable.");
+    }
+    pathPtr = realpath(ebuf, buf);
+    if (pathPtr == NULL) {
+        return report_error(strerror(errno));
+    }
+    char *t;
+    for (i = 0; i < 3; i++) {
+        t = rindex(pathPtr, '/');
+        if (t == NULL) return report_error("Failed to determine bundle path.");
+        *t = '\0';
+    }
+
+        
+
+    char rpath[PATH_MAX+1];
+    strncpy(rpath, pathPtr, strlen(pathPtr));
+    strncat(rpath, "/Contents/Resources", 50);
+    char exe_path[PATH_MAX+1];
+    strncpy(exe_path, pathPtr, strlen(pathPtr));
+    strncat(exe_path, "/Contents", 50);
+    
+    set_env_vars(exe_path, rpath);
+
+    char main_script[PATH_MAX+1];
+    strncpy(main_script, rpath, strlen(rpath));
+    strncat(main_script, "/launcher.py", 20);
+
+    Py_SetProgramName(PROGRAM);
+
+    Py_Initialize();
+    
+    char **argv_new = calloc(argc+1, sizeof(char *));
+    argv_new[argc] = NULL;
+    argv_new[0] = main_script;
+    memcpy(&argv_new[1], &argv[1], (argc - 1) * sizeof(char *));
+    PySys_SetArgv(argc, argv_new);
+
+    FILE *main_script_file = fopen(main_script, "r");
+    int rval = PyRun_SimpleFileEx(main_script_file, main_script, 1);
+ 
+    while (rval != 0) {
+        PyObject *exc, *exceptionClassName, *v, *exceptionName;
+        exc = PySys_GetObject("last_type");
+
+        if ( !exc ) {
+            rval = report_error(ERR_UNKNOWNPYTHONEXCEPTION);
+            break;
+        }
+
+        exceptionClassName = PyObject_GetAttrString(exc, "__name__");
+        if (!exceptionClassName) {
+            rval = report_error(ERR_UNKNOWNPYTHONEXCEPTION);
+            break;
+        }
+
+        v = PySys_GetObject("last_value");
+        exceptionName = (v ? PyObject_Str(v) : NULL);
+
+        char *class = PyString_AsString(exceptionClassName);
+        char *exception = "";
+        Py_DecRef(exceptionClassName);
+        if (exceptionName) {
+            exception = PyString_AsString(exceptionName);
+            Py_DecRef(exceptionName);
+        }
+        char msg[2000];
+        strncpy(msg, "An unexpected error occurred: ", 100);
+        strncpy(msg, class, 500);
+        strncpy(msg, " : ", 3);
+        strncpy(msg, exception, 500);
+        rval = report_error(msg);
+        break;
+
+    }
+    Py_Finalize();
+    return rval;
+}

installer/osx/py2app/launcher.py

@@ -35,22 +35,24 @@ _recipes_pil_prescript(['Hdf5StubImagePlugin', 'FitsStubImagePlugin', 'SunImageP
 
 def _run():
     global __file__
-    import os, sys, site
+    import os, sys
-    sys.frozen = 'macosx_app'
     base = os.environ['RESOURCEPATH']
+
+    sys.frozen = 'macosx_app'
     sys.frameworks_dir = os.path.join(os.path.dirname(base), 'Frameworks')
     sys.new_app_bundle = True
-    site.addsitedir(base)
+    sys.site_packages = os.path.join(base, 'Python', 'site-packages')
-    site.addsitedir(os.path.join(base, 'Python', 'site-packages'))
+    sys.binaries_path = os.path.join(os.path.dirname(base), 'MacOS')
+    sys.console_binaries_path = os.path.join(os.path.dirname(base),
+        'console.app', 'Contents', 'MacOS')
+
     exe = os.environ.get('CALIBRE_LAUNCH_MODULE', 'calibre.gui2.main')
     exe = os.path.join(base, 'Python', 'site-packages', *exe.split('.'))
     exe += '.py'
     sys.argv[0] = __file__ = exe
-    argv = os.environ.get('CALIBRE_LAUNCH_ARGV', None)
+    for arg in list(sys.argv[1:]):
-    if argv is not None:
+        if arg.startswith('-psn'):
-        import cPickle
+            sys.argv.remove(arg)
-        argv = cPickle.loads(argv)
-        sys.argv[1:] = argv
     execfile(exe, globals(), globals())
 
 _run()

installer/osx/py2app/loader.py

@@ -1,33 +0,0 @@
-#!/usr/bin/env python
-
-__license__   = 'GPL v3'
-__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
-__docformat__ = 'restructuredtext en'
-
-
-import os, sys, cPickle
-
-ENV = {}##ENV##
-MODULE = ''##MODULE##
-
-path = os.path.abspath(os.path.realpath(__file__))
-dirpath = os.path.dirname(path)
-name = os.path.basename(path)
-base_dir = os.path.dirname(os.path.dirname(dirpath))
-resources_dir = os.path.join(base_dir, 'Resources')
-frameworks_dir = os.path.join(base_dir, 'Frameworks')
-exe_dir = os.path.join(base_dir, 'MacOS')
-base_name = os.path.splitext(name)[0]
-python = os.path.join(base_dir, 'MacOS', 'Python')
-
-for key, val in ENV.items():
-    if val.startswith('@exec'):
-        ENV[key] = os.path.normpath(val.replace('@executable_path', exe_dir))
-ENV['CALIBRE_LAUNCH_MODULE'] = MODULE
-ENV['CALIBRE_LAUNCH_ARGV'] = cPickle.dumps(sys.argv[1:], -1)
-ENV['RESOURCEPATH'] = resources_dir
-os.environ.update(ENV)
-launcher = os.path.join(resources_dir, 'launcher.py')
-args = ['-OO', launcher]
-os.execv(python, args)
-

installer/osx/py2app/main.py

@@ -20,17 +20,50 @@ main_functions = l['main_functions']
 main_modules = l['main_modules']
 LICENSE = open('LICENSE', 'rb').read()
 ENV = dict(
-        FC_CONFIG_DIR='@executable_path/../Resources/fonts',
-        MAGICK_HOME='@executable_path/../Frameworks/ImageMagick',
-        PYTHONDONTWRITEBYTECODE='1',
-        PYTHONIOENCODING='utf-8:replace',
         PYTHONPATH='@executable_path/../Resources/Python/site-packages',
         PYTHONHOME='@executable_path/../Resources/Python',
+        FC_CONFIG_DIR='@executable_path/../Resources/fonts',
+        MAGICK_HOME='@executable_path/../Frameworks/ImageMagick',
+        QT_PLUGIN_PATH='@executable_path/../MacOS',
+        PYTHONDONTWRITEBYTECODE='1',
+        PYTHONIOENCODING='utf-8:replace',
         PYTHONOPTIMIZE='2',
-        QT_PLUGIN_PATH='@executable_path'
         )
 
-SW = os.environ.get('SW')
+SW = os.environ.get('SW', '/sw')
+
+def compile_launchers(contents_dir, xprograms):
+    gcc = os.environ.get('CC', 'gcc')
+    base = os.path.dirname(__file__)
+    src = open(join(base, 'launcher.c'), 'rb').read()
+    env, env_vals = [], []
+    for key, val in ENV.items():
+        env.append('"%s"'% key)
+        env_vals.append('"%s"'% val)
+    env = ', '.join(env)+', '
+    env_vals = ', '.join(env_vals)+', '
+    src = src.replace('/*ENV_VARS*/', env)
+    src = src.replace('/*ENV_VAR_VALS*/', env_vals)
+    programs = []
+    for program, module in xprograms.items():
+        print '\tCompiling', program
+        out = join(contents_dir, 'MacOS', program)
+        programs.append(out)
+        psrc = src.replace('**PROGRAM**', program)
+        psrc = psrc.replace('**MODULE**', module)
+        fsrc = '/tmp/%s.c'%program
+        with open(fsrc, 'wb') as f:
+            f.write(psrc)
+        cmd = [gcc, '-Wall', '-arch', 'x86_64',
+            '-I%s/python/Python.framework/Headers'%SW,
+            fsrc, '-o', out, '-F%s/python'%SW,
+            '-framework', 'Python', '-framework', 'CoreFoundation',
+            '-headerpad_max_install_names']
+        print ' '.join(cmd)
+        sys.stdout.flush()
+        subprocess.check_call(cmd)
+    return programs
+
 
 def flipwritable(fn, mode=None):
     """
@@ -43,6 +76,15 @@ def flipwritable(fn, mode=None):
     os.chmod(fn, stat.S_IWRITE | old_mode)
     return old_mode
 
+def thin(path):
+    try:
+        subprocess.check_call(['lipo', path, '-verify_arch', 'ppc64'])
+        print '\tThinning', path
+    except:
+        return
+    else:
+        subprocess.check_call(['lipo', path, '-thin', 'x86_64', '-output', path])
+
 STRIPCMD = ['/usr/bin/strip', '-x', '-S', '-']
 def strip_files(files, argv_max=(256 * 1024)):
     """
@@ -120,8 +162,8 @@ class Py2App(object):
 
         self.copy_launcher_and_site()
         self.create_exe()
+        self.thin_to_x86_64()
         self.strip_files()
-        self.create_launchers()
 
         ret = self.makedmg(self.build_dir, APPNAME+'-'+VERSION+'-x86_64')
         sys.stdout.flush()
@@ -134,23 +176,17 @@ class Py2App(object):
         return ret
 
     @flush
-    def create_launchers(self):
+    def thin_to_x86_64(self):
-        print '\nCreating launchers'
+        print '\nThinning to x86_64'
-        all_names   = basenames['console'] + basenames['gui']
+        for y in (self.frameworks_dir, join(self.resources_dir, 'Python')):
-        all_modules   = main_modules['console'] + main_modules['gui']
+            for x in os.walk(y):
-        launcher = join(os.path.dirname(__file__), 'loader.py')
+                for f in x[-1]:
-        launcher = open(launcher, 'rb').read()
+                    f = join(x[0], f)
-        launcher = launcher.replace('{}##ENV##', repr(ENV))
+                    if not os.path.isfile(f): continue
-        os.mkdir(join(self.resources_dir, 'loaders'))
+                    for t in ('.so', '.dylib', '/Python'):
-        for basename, module in zip(all_names, all_modules):
+                        if f.endswith(t):
-            py_file = join('src', *module.split('.'))+'.py'
+                            thin(f)
-            shutil.copy2(py_file, join(self.resources_dir, 'Python',
+                            break
-            'site-packages', *module.split('.'))+'.py')
-            raw = launcher.replace("''##MODULE##", repr(module))
-            path = join(self.resources_dir, 'loaders', basename)
-            open(path, 'wb').write(raw)
-            os.chmod(path, stat.S_IXUSR|stat.S_IXGRP|stat.S_IXOTH|stat.S_IREAD\
-                     |stat.S_IWUSR|stat.S_IROTH|stat.S_IRGRP)
 
     @flush
     def strip_files(self):
@@ -159,13 +195,19 @@ class Py2App(object):
 
     @flush
     def create_exe(self):
-        print '\nCreating executable'
+        print '\nCreating launchers'
-        gcc = os.environ.get('CC', 'gcc')
+        programs = {}
-        base = os.path.dirname(__file__)
+        for program, module in zip(basenames['console'],
-        out = join(self.contents_dir, 'MacOS', 'calibre')
+                main_modules['console'])+zip(basenames['gui'],
-        subprocess.check_call([gcc, '-Wall', '-arch', 'x86_64', join(base,
+                main_modules['gui']):
-            'main.c'), '-o', out])
+            programs[program] = module
-        self.to_strip.append(out)
+        programs = compile_launchers(self.contents_dir, programs)
+        for out in programs:
+            self.fix_dependencies_in_lib(out)
+        for module in main_modules['console'] + main_modules['gui']:
+            base = join(*module.split('.'))+'.py'
+            shutil.copy2(join('src', base),
+                    join(self.resources_dir, 'Python', 'site-packages', base))
 
     @flush
     def set_id(self, path_to_lib, new_id):
@@ -226,10 +268,6 @@ class Py2App(object):
         shutil.copy2(join(curr, 'Python'), currd)
         self.set_id(join(currd, 'Python'),
             self.FID+'/Python.framework/Versions/%s/Python'%basename(curr))
-        python = '%s/python/Python.framework/Versions/%s/Resources/Python.app/Contents/MacOS/Python'\
-                % (SW, self.version_info)
-        shutil.copy2(python, join(self.contents_dir, 'MacOS'))
-        self.fix_dependencies_in_lib(join(self.contents_dir, 'MacOS', 'Python'))
 
     @flush
     def add_qt_frameworks(self):
@@ -279,6 +317,9 @@ class Py2App(object):
 
     @flush
     def create_plist(self):
+        env = dict(**ENV)
+        env['CALIBRE_LAUNCHED_FROM_BUNDLE']='1';
+
         pl = dict(
                 CFBundleDevelopmentRegion='English',
                 CFBundleDisplayName=APPNAME,
@@ -289,7 +330,6 @@ class Py2App(object):
                 CFBundleSignature='????',
                 CFBundleExecutable='calibre',
                 LSMinimumSystemVersion='10.5.2',
-                PyRuntimeLocations=[self.FID+'/Python.framework/Versions/%s/Python'%self.version_info],
                 LSRequiresNativeExecution=True,
                 NSAppleScriptEnabled=False,
                 NSHumanReadableCopyright='Copyright 2008, Kovid Goyal',
@@ -297,7 +337,7 @@ class Py2App(object):
                 'application. Visit http://calibre.kovidgoyal.net for details.'),
                 CFBundleIconFile='library.icns',
                 LSMultipleInstancesProhibited=True,
-                LSEnvironment=ENV
+                LSEnvironment=env
         )
         plistlib.writePlist(pl, join(self.contents_dir, 'Info.plist'))
 
@@ -554,9 +594,16 @@ class Py2App(object):
         print '\nInstaller size: %.2fMB\n'%size
         return dmg
 
+def test_exe():
+    build_dir = abspath(join('build', APPNAME+'.app'))
+    py2app = Py2App(build_dir)
+    py2app.create_exe()
+    return 0
 
 
 def main():
+    if 'test_exe' in sys.argv:
+        return test_exe()
     build_dir = abspath(join('build', APPNAME+'.app'))
     if os.path.exists(build_dir):
         shutil.rmtree(build_dir)

setup.py

@@ -73,7 +73,12 @@ if __name__ == '__main__':
                         upload_to_pypi, stage3, stage2, stage1, upload, \
                         upload_rss, betas, build_linux32, build_linux64, \
                         build_osx64
-    resources.SCRIPTS = list(basenames['console']+basenames['gui'])
+    resources.SCRIPTS = {}
+    for x in ('console', 'gui'):
+        for name in basenames[x]:
+            resources.SCRIPTS[name] = x
+
+    list(basenames['console']+basenames['gui'])
 
     entry_points['console_scripts'].append(
                             'calibre_postinstall = calibre.linux:post_install')

src/calibre/debug.py

@@ -6,7 +6,7 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 Embedded console for debugging.
 '''
 
-import sys, os, re
+import sys, os, re, shutil
 from calibre.utils.config import OptionParser
 from calibre.constants import iswindows, isosx
 from calibre.libunzip import update
@@ -45,6 +45,9 @@ def update_zipfile(zipfile, mod, path):
     name = mod.replace('.', '/') + os.path.splitext(path)[-1]
     update(zipfile, [pat], [path], [name])
 
+def update_site_packages(sp, mod, path):
+    dest = os.path.join(sp, *mod.split('.'))+'.py'
+    shutil.copy2(path, dest)
 
 def update_module(mod, path):
     if not hasattr(sys, 'frozen'):
@@ -52,6 +55,8 @@ def update_module(mod, path):
     zp = None
     if iswindows:
         zp = os.path.join(os.path.dirname(sys.executable), 'library.zip')
+    elif getattr(sys, 'new_app_bundle', False):
+        update_site_packages(sys.site_packages, mod, path)
     elif isosx:
         zp = os.path.join(os.path.dirname(getattr(sys, 'frameworks_dir')),
                             'Resources', 'lib',

src/calibre/gui2/dialogs/config/__init__.py

@@ -465,8 +465,10 @@ class ConfigDialog(QDialog, Ui_Dialog):
         from calibre.utils.osx_symlinks import create_symlinks
         loc, paths = create_symlinks()
         info_dialog(self, _('Command line tools installed'),
-                _('Command line tools installed in')+' '+loc,
+            '<p>'+_('Command line tools installed in')+' '+loc+
-                det_msg=paths, show=True)
+            '<br>'+ _('If you move calibre.app, you have to re-install '
+                    'the command line tools.'),
+                det_msg='\n'.join(paths), show=True)
 
     def setup_conversion_options(self):
         self.conversion_options = ConfigTabs(self)

src/calibre/gui2/dialogs/config/config.ui

@@ -699,10 +699,7 @@
          <item>
           <layout class="QVBoxLayout" name="verticalLayout_12">
            <item>
-            <widget class="QToolButton" name="compact_button">
+            <widget class="QPushButton" name="compact_button">
-             <property name="toolTip">
-              <string>Free unused diskspace from the database</string>
-             </property>
              <property name="text">
               <string>&amp;Check database integrity</string>
              </property>

src/calibre/utils/ipc/launch.py

@@ -16,6 +16,8 @@ if iswindows:
     import win32process
     _windows_null_file = open(os.devnull, 'wb')
 
+isnewosx = isosx and getattr(sys, 'new_app_bundle')
+
 class Worker(object):
     '''
     Platform independent object for launching child processes. All processes
@@ -45,6 +47,9 @@ class Worker(object):
             return os.path.join(os.path.dirname(sys.executable),
                    'calibre-parallel.exe' if isfrozen else \
                            'Scripts\\calibre-parallel.exe')
+        if isnewosx:
+            return os.path.join(sys.console_binaries_path, 'calibre-parallel')
+
         if isosx:
             if not isfrozen: return 'calibre-parallel'
             contents = os.path.join(self.osx_contents_dir,
@@ -56,6 +61,9 @@ class Worker(object):
 
     @property
     def gui_executable(self):
+        if isnewosx:
+           return os.path.join(sys.binaries_path, 'calibre-parallel')
+
         if isfrozen and isosx:
             return os.path.join(self.osx_contents_dir,
                     'MacOS', self.osx_interpreter)
@@ -98,7 +106,7 @@ class Worker(object):
     def __init__(self, env, gui=False):
         self._env = {}
         self.gui = gui
-        if isosx and isfrozen:
+        if isosx and isfrozen and not isnewosx:
             contents = os.path.join(self.osx_contents_dir, 'console.app', 'Contents')
             resources = os.path.join(contents, 'Resources')
             fd = os.path.join(contents, 'Frameworks')
@@ -133,7 +141,7 @@ class Worker(object):
         if priority is None:
             priority = prefs['worker_process_priority']
         cmd = [exe]
-        if isosx:
+        if isosx and not isnewosx:
             cmd += ['-c', self.osx_prefix + 'from calibre.utils.ipc.worker import main; main()']
         args = {
                 'env' : env,
@@ -155,7 +163,7 @@ class Worker(object):
             ret = self._file.name
 
         if iswindows and 'stdin' not in args:
-            # On windows when usingthepythonw interpreter,
+            # On windows when using the pythonw interpreter,
             # stdout, stderr and stdin may not be valid
             args['stdin'] = subprocess.PIPE
             args['stdout'] = _windows_null_file

src/calibre/utils/osx_symlinks.py

@@ -6,7 +6,9 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-AUTHTOOL="""#!%s
+import sys, os
+
+AUTHTOOL="""#!/usr/bin/python
 import os
 scripts = %s
 links = %s
@@ -23,15 +25,31 @@ for s, l in zip(scripts, links):
 DEST_PATH = '/usr/bin'
 
 def create_symlinks():
-    import os, tempfile, traceback, sys
+    return create_symlinks_new() if getattr(sys, 'new_app_bundle', False) else create_symlinks_old()
-    from Authorization import Authorization, kAuthorizationFlagDestroyRights
+
+def create_symlinks_new():
     from calibre.resources import scripts
 
+    links   = [os.path.join(DEST_PATH, i) for i in scripts]
+    scripts = [os.path.join(
+        sys.binaries_path if scripts[i] == 'gui' else sys.console_binaries_path, i) for i in scripts]
+
+    return do_it(scripts, links)
+
+
+def create_symlinks_old():
+    from calibre.resources import scripts
 
     resources_path = os.environ['RESOURCEPATH']
     links   = [os.path.join(DEST_PATH, i) for i in scripts]
     scripts = [os.path.join(resources_path, 'loaders', i) for i in scripts]
 
+    return do_it(scripts, links)
+
+def do_it(scripts, links):
+    import os, tempfile, traceback
+    from Authorization import Authorization, kAuthorizationFlagDestroyRights
+
     bad = False
     for s, l in zip(scripts, links):
         if os.path.exists(l) and os.path.exists(os.path.realpath(l)):
@@ -39,19 +57,28 @@ def create_symlinks():
         bad = True
         break
     if bad:
+        ph, pp = os.environ.get('PYTHONHOME', None), os.environ.get('PYTHONPATH', None)
         auth = Authorization(destroyflags=(kAuthorizationFlagDestroyRights,))
         fd, name = tempfile.mkstemp('.py')
-        os.write(fd, AUTHTOOL % (sys.executable, repr(scripts), repr(links)))
+        os.write(fd, AUTHTOOL % (repr(scripts), repr(links)))
         os.close(fd)
         os.chmod(name, 0700)
         try:
-            pipe = auth.executeWithPrivileges(sys.executable, name)
+            if pp:
+                del os.environ['PYTHONPATH']
+            if ph:
+                del os.environ['PYTHONHOME']
+            pipe = auth.executeWithPrivileges(name)
             sys.stdout.write(pipe.read())
             pipe.close()
         except:
             traceback.print_exc()
         finally:
             os.unlink(name)
+            if pp:
+                os.environ['PYTHONPATH'] = pp
+            if ph:
+                os.environ['PYTHONHOME'] = ph
 
     return DEST_PATH, links