Installer building ported to new setup framework

.bzrignore

@@ -11,7 +11,7 @@ resources/localization
 resources/images.qrc
 resources/recipes.pickle
 resources/scripts.pickle
-installer/windows/calibre/build.log
+setup/installer/windows/calibre/build.log
 src/calibre/translations/.errors
 src/cssutils/.svn/
 src/cssutils/_todo/

installer/linux/freeze.py

@@ -1,253 +0,0 @@
-#!/usr/bin/env  python
-from __future__ import with_statement
-__license__   = 'GPL v3'
-__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
-__docformat__ = 'restructuredtext en'
-
-'''
-Create linux binary.
-'''
-
-def freeze():
-    import glob, sys, tarfile, os, textwrap, shutil, platform
-    from contextlib import closing
-    from cx_Freeze import Executable, setup
-    from calibre.constants import __version__, __appname__
-    from calibre.linux import entry_points
-    from calibre import walk
-    from calibre.web.feeds.recipes import recipe_modules
-    from calibre.ebooks.lrf.fonts import FONT_MAP
-    import calibre
-
-    is64bit = platform.architecture()[0] == '64bit'
-    arch = 'x86_64' if is64bit else 'i686'
-
-
-    QTDIR          = '/usr/lib/qt4'
-    QTDLLS         = ('QtCore', 'QtGui', 'QtNetwork', 'QtSvg', 'QtXml',
-            'QtWebKit', 'QtDBus')
-
-    binary_excludes = ['libGLcore*', 'libGL*', 'libnvidia*']
-
-    os.system('sudo cp /usr/bin/calibre-mount-helper /tmp/calibre-mount-helper')
-    os.system('sudo chown kovid:users /tmp/calibre-mount-helper')
-
-    binary_includes = [
-                       '/usr/bin/pdftohtml',
-                       '/tmp/calibre-mount-helper',
-                       '/usr/lib/libunrar.so',
-                       '/usr/lib/libsqlite3.so.0',
-                       '/usr/lib/libsqlite3.so.0',
-                       '/usr/lib/libmng.so.1',
-                       '/usr/lib/libpodofo.so.0.6.99',
-                       '/lib/libz.so.1',
-                       '/lib/libbz2.so.1',
-                       '/usr/lib/libpoppler.so.4',
-                       '/usr/lib/libpoppler-qt4.so.3',
-                       '/usr/lib/libxml2.so.2',
-                       '/usr/lib/libopenjpeg.so.2',
-                       '/usr/lib/libxslt.so.1',
-                       '/usr/lib64/libjpeg.so.7'.replace('64', '64' if is64bit
-                           else ''),
-                       '/usr/lib/libxslt.so.1',
-                       '/usr/lib/libgthread-2.0.so.0',
-                       '/usr/lib/gcc/***-pc-linux-gnu/4.4.1/libstdc++.so.6'.replace('***',
-                           arch),
-                       '/usr/lib/libpng12.so.0',
-                       '/usr/lib/libexslt.so.0',
-                       '/usr/lib/libMagickWand.so',
-                       '/usr/lib/libMagickCore.so',
-                       '/usr/lib/libgcrypt.so.11',
-                       '/usr/lib/libgpg-error.so.0',
-                       '/usr/lib/libphonon.so.4',
-                       '/usr/lib/libssl.so.0.9.8',
-                       '/usr/lib/libcrypto.so.0.9.8',
-                       '/lib/libreadline.so.6',
-                       ]
-
-    binary_includes += [os.path.join(QTDIR, 'lib%s.so.4'%x) for x in QTDLLS]
-
-
-    d = os.path.dirname
-    CALIBRESRC = d(d(d(os.path.abspath(calibre.__file__))))
-    CALIBREPLUGINS = os.path.join(CALIBRESRC, 'src', 'calibre', 'plugins')
-    FREEZE_DIR = os.path.join(CALIBRESRC, 'build', 'cx_freeze')
-    DIST_DIR   = os.path.join(CALIBRESRC, 'dist')
-
-    os.chdir(CALIBRESRC)
-
-    print 'Freezing calibre located at', CALIBRESRC
-
-    sys.path.insert(0, os.path.join(CALIBRESRC, 'src'))
-
-    entry_points = entry_points['console_scripts'] + entry_points['gui_scripts']
-    entry_points = ['calibre_postinstall=calibre.linux:binary_install',
-                    'calibre-parallel=calibre.parallel:main'] + entry_points
-    executables = {}
-    for ep in entry_points:
-        executables[ep.split('=')[0].strip()] = (ep.split('=')[1].split(':')[0].strip(),
-                                                 ep.split(':')[-1].strip())
-
-    if os.path.exists(FREEZE_DIR):
-        shutil.rmtree(FREEZE_DIR)
-    os.makedirs(FREEZE_DIR)
-
-    if not os.path.exists(DIST_DIR):
-        os.makedirs(DIST_DIR)
-
-    includes = [x[0] for x in executables.values()]
-    includes += ['calibre.ebooks.lrf.fonts.prs500.'+x for x in FONT_MAP.values()]
-    includes += ['email.iterators', 'email.generator', 'sqlite3.dump']
-
-
-    excludes = ['matplotlib', "Tkconstants", "Tkinter", "tcl", "_imagingtk",
-                "ImageTk", "FixTk", 'wx', 'PyQt4.QtAssistant', 'PyQt4.QtOpenGL.so',
-                'PyQt4.QtScript.so', 'PyQt4.QtSql.so', 'PyQt4.QtTest.so', 'qt',
-                'glib', 'gobject']
-
-    packages = ['calibre', 'encodings', 'cherrypy', 'cssutils', 'xdg',
-                'dateutil', 'dns', 'email']
-
-    includes += ['calibre.web.feeds.recipes.'+r for r in recipe_modules]
-    includes += ['calibre.gui2.convert.'+x.split('/')[-1].rpartition('.')[0] for x in \
-            glob.glob('src/calibre/gui2/convert/*.py')]
-
-    LOADER = '/tmp/loader.py'
-    open(LOADER, 'wb').write('# This script is never actually used.\nimport sys')
-
-    INIT_SCRIPT = '/tmp/init.py'
-    open(INIT_SCRIPT, 'wb').write(textwrap.dedent('''
-    ## Load calibre module specified in the environment variable CALIBRE_CX_EXE
-    ## Also restrict sys.path to the executables' directory and add the
-    ## executables directory to LD_LIBRARY_PATH
-    import encodings
-    import os
-    import sys
-    import warnings
-    import zipimport
-    import locale
-    import codecs
-
-    enc = locale.getdefaultlocale()[1]
-    if not enc:
-        enc = locale.nl_langinfo(locale.CODESET)
-    enc = codecs.lookup(enc if enc else 'UTF-8').name
-    sys.setdefaultencoding(enc)
-
-    paths = os.environ.get('LD_LIBRARY_PATH', '').split(os.pathsep)
-    if DIR_NAME not in paths or not sys.getfilesystemencoding():
-        paths.insert(0, DIR_NAME)
-        os.environ['LD_LIBRARY_PATH'] = os.pathsep.join(paths)
-        os.environ['PYTHONIOENCODING'] = enc
-        os.execv(sys.executable, sys.argv)
-
-    sys.path = sys.path[:3]
-    sys.frozen = True
-    sys.frozen_path = DIR_NAME
-
-    executables = %(executables)s
-
-    exe = os.environ.get('CALIBRE_CX_EXE', False)
-    ret = 1
-    if not exe:
-        print >>sys.stderr, 'Invalid invocation of calibre loader. CALIBRE_CX_EXE not set'
-    elif exe not in executables:
-        print >>sys.stderr, 'Invalid invocation of calibre loader. CALIBRE_CX_EXE=%%s is unknown'%%exe
-    else:
-        from PyQt4.QtCore import QCoreApplication
-        QCoreApplication.setLibraryPaths([sys.frozen_path, os.path.join(sys.frozen_path, "qtplugins")])
-        sys.argv[0] = exe
-        module, func = executables[exe]
-        module = __import__(module, fromlist=[1])
-        func = getattr(module, func)
-        ret = func()
-
-    module = sys.modules.get("threading")
-    if module is not None:
-        module._shutdown()
-    sys.exit(ret)
-    ''')%dict(executables=repr(executables)))
-    sys.argv = ['freeze', 'build_exe']
-    setup(
-          name        = __appname__,
-          version     = __version__,
-          executables = [Executable(script=LOADER, targetName='loader', compress=False)],
-          options     = { 'build_exe' :
-                         {
-                          'build_exe'       : os.path.join(CALIBRESRC, 'build/cx_freeze'),
-                          'optimize'        : 2,
-                          'excludes'        : excludes,
-                          'includes'        : includes,
-                          'packages'        : packages,
-                          'init_script'     : INIT_SCRIPT,
-                          'copy_dependent_files' : True,
-                          'create_shared_zip'    : False,
-                          }
-                         }
-          )
-
-    def copy_binary(src, dest_dir):
-        dest = os.path.join(dest_dir, os.path.basename(src))
-        if not os.path.exists(dest_dir):
-            os.makedirs(dest_dir)
-        shutil.copyfile(os.path.realpath(src), dest)
-        shutil.copymode(os.path.realpath(src), dest)
-
-    for f in binary_includes:
-        copy_binary(f, FREEZE_DIR)
-
-    for pat in binary_excludes:
-        matches = glob.glob(os.path.join(FREEZE_DIR, pat))
-        for f in matches:
-            os.remove(f)
-
-    print 'Adding calibre plugins...'
-    os.makedirs(os.path.join(FREEZE_DIR, 'plugins'))
-    for f in glob.glob(os.path.join(CALIBREPLUGINS, '*.so')):
-        copy_binary(f, os.path.join(FREEZE_DIR, 'plugins'))
-
-    print 'Adding Qt plugins...'
-    plugdir = os.path.join(QTDIR, 'plugins')
-    for dirpath, dirnames, filenames in os.walk(plugdir):
-        for f in filenames:
-            if not f.endswith('.so') or 'designer' in dirpath or 'codecs' in dirpath or 'sqldrivers' in dirpath:
-                continue
-            f = os.path.join(dirpath, f)
-            dest_dir = dirpath.replace(plugdir, os.path.join(FREEZE_DIR, 'qtplugins'))
-            copy_binary(f, dest_dir)
-
-    print 'Creating launchers'
-    for exe in executables:
-        path = os.path.join(FREEZE_DIR, exe)
-        open(path, 'wb').write(textwrap.dedent('''\
-        #!/bin/sh
-        export CALIBRE_CX_EXE=%s
-        path=`readlink -e $0`
-        base=`dirname $path`
-        loader=$base/loader
-        export LD_LIBRARY_PATH=$base:$LD_LIBRARY_PATH
-        $loader "$@"
-        ''')%exe)
-        os.chmod(path, 0755)
-
-    exes = list(executables.keys())
-    exes.remove('calibre_postinstall')
-    exes.remove('calibre-parallel')
-    open(os.path.join(FREEZE_DIR, 'manifest'), 'wb').write('\n'.join(exes))
-
-    print 'Creating archive...'
-    dist = open(os.path.join(DIST_DIR, 'calibre-%s-%s.tar.bz2'%(__version__,
-        arch)), 'wb')
-    with closing(tarfile.open(fileobj=dist, mode='w:bz2',
-                              format=tarfile.PAX_FORMAT)) as tf:
-        for f in walk(FREEZE_DIR):
-            name = f.replace(FREEZE_DIR, '')[1:]
-            if name:
-                tf.add(f, name)
-    dist.flush()
-    dist.seek(0, 2)
-    print 'Archive %s created: %.2f MB'%(dist.name, dist.tell()/(1024.**2))
-    return 0
-
-if __name__ == '__main__':
-    freeze()

setup/__init__.py

@@ -6,12 +6,16 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import sys, re, os
+import sys, re, os, platform
 
+is64bit = platform.architecture()[0] == '64bit'
 iswindows = re.search('win(32|64)', sys.platform)
 isosx = 'darwin' in sys.platform
 islinux = not isosx and not iswindows
 SRC = os.path.abspath('src')
+sys.path.insert(0, SRC)
+sys.resources_location = os.path.join(os.path.dirname(SRC), 'resources')
+sys.extensions_location = os.path.join(SRC, 'calibre', 'plugins')
 
 __version__ = __appname__ = modules = functions = basenames = scripts = None
 
@@ -197,3 +201,17 @@ class Command(object):
         warnings.append((args, kwargs))
         sys.stdout.flush()
 
+def installer_name(ext, is64bit=False):
+    if ext == 'exe':
+        return 'dist/%s-%s.%s'%(__appname__, __version__, ext)
+    if ext == 'dmg':
+        if is64bit:
+            return 'dist/%s-%s-x86_64.%s'%(__appname__, __version__, ext)
+        return 'dist/%s-%s.%s'%(__appname__, __version__, ext)
+
+    ans = 'dist/%s-%s-i686.%s'%(__appname__, __version__, ext)
+    if is64bit:
+        ans = ans.replace('i686', 'x86_64')
+    return ans
+
+

setup/build_environment.py

@@ -6,7 +6,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os
+import os, socket, struct
 from distutils.spawn import find_executable
 
 from PyQt4 import pyqtconfig
@@ -98,4 +98,21 @@ podofo_error = None if os.path.exists(os.path.join(podofo_inc, 'podofo.h')) else
     ' functionality will not work. Use the PODOFO_INC_DIR and',
     ' PODOFO_LIB_DIR environment variables.')
 
+def get_ip_address(ifname):
+    import fcntl
+    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    return socket.inet_ntoa(fcntl.ioctl(
+        s.fileno(),
+        0x8915,  # SIOCGIFADDR
+        struct.pack('256s', ifname[:15])
+    )[20:24])
 
+try:
+    HOST=get_ip_address('eth0')
+except:
+    try:
+        HOST=get_ip_address('wlan0')
+    except:
+        HOST='unknown'
+
+PROJECT=os.path.basename(os.path.abspath('.'))

setup/commands.py

@@ -14,7 +14,11 @@ __all__ = [
         'resources',
         'check',
         'sdist',
-        'manual',
+        'manual', 'tag_release', 'upload_rss',
+        'upload_user_manual', 'upload_installers', 'upload_demo',
+        'linux32', 'linux64', 'linux', 'linux_freeze',
+        'osx32_freeze', 'osx32', 'osx',
+        'win32_freeze', 'win32', 'win',
         ]
 
 
@@ -41,8 +45,35 @@ check = Check()
 from setup.resources import Resources
 resources = Resources()
 
-from setup.publish import Manual
+from setup.publish import Manual, TagRelease, UploadRss
 manual = Manual()
+tag_release = TagRelease()
+upload_rss = UploadRss()
+
+from setup.upload import UploadUserManual, UploadInstallers, UploadDemo
+upload_user_manual = UploadUserManual()
+upload_installers = UploadInstallers()
+upload_demo = UploadDemo()
+
+from setup.installer.linux import Linux, Linux32, Linux64
+linux = Linux()
+linux32 = Linux32()
+linux64 = Linux64()
+from setup.installer.linux.freeze import LinuxFreeze
+linux_freeze = LinuxFreeze()
+
+from setup.installer.osx import OSX, OSX32
+osx = OSX()
+osx32 = OSX32()
+from setup.installer.osx.freeze import OSX32_Freeze
+osx32_freeze = OSX32_Freeze()
+
+from setup.installer.windows import Win, Win32
+win = Win()
+win32 = Win32()
+from setup.installer.windows.freeze import Win32Freeze
+win32_freeze = Win32Freeze()
+
 
 commands = {}
 for x in __all__:

setup/extensions.py

@@ -201,6 +201,8 @@ class Build(Command):
                 else:
                     raise Exception(ext.error)
             dest = self.dest(ext)
+            if not os.path.exists(self.d(dest)):
+                os.makedirs(self.d(dest))
             self.info('\n####### Building extension', ext.name, '#'*7)
             self.build(ext, dest)
 

setup/install.py

@@ -62,10 +62,31 @@ class Develop(Command):
         self.regain_privileges()
         self.find_locations(opts)
         self.write_templates(opts)
+        self.setup_mount_helper()
         self.install_files(opts)
         self.run_postinstall()
         self.success()
 
+    def setup_mount_helper(self):
+        def warn():
+            self.warn('Failed to compile mount helper. Auto mounting of',
+                'devices will not work')
+
+        if os.geteuid() != 0:
+            return warn()
+        import stat
+        src = os.path.join(self.SRC, 'calibre', 'devices', 'linux_mount_helper.c')
+        dest = os.path.join(self.bindir, 'calibre-mount-helper')
+        self.info('Installing mount helper to '+ dest)
+        p = subprocess.Popen(['gcc', '-Wall', src, '-o', dest])
+        ret = p.wait()
+        if ret != 0:
+            return warn()
+        os.chown(dest, 0, 0)
+        os.chmod(dest,
+        stat.S_ISUID|stat.S_ISGID|stat.S_IRUSR|stat.S_IWUSR|stat.S_IXUSR|stat.S_IXGRP|stat.S_IXOTH)
+        return dest
+
     def install_files(self, opts):
         pass
 
@@ -76,9 +97,13 @@ class Develop(Command):
         self.info('\nDevelopment environment successfully setup')
 
     def find_locations(self, opts):
+        self.prefix = opts.prefix
+        if self.prefix is None:
+            self.prefix = sys.prefix
         self.path = self.SRC
         self.resources = self.j(self.d(self.SRC), 'resources')
         self.extensions = self.j(self.SRC, 'calibre', 'plugins')
+        self.bindir = self.j(self.prefix, 'bin')
 
     def write_templates(self, opts):
         for typ in ('console', 'gui'):
@@ -93,10 +118,7 @@ class Develop(Command):
                 module=mod, func=func,
                 path=self.path, resources=self.resources,
                 extensions=self.extensions)
-        prefix = opts.prefix
+        path = self.j(self.bindir, name)
-        if prefix is None:
-            prefix = sys.prefix
-        path = self.j(prefix, 'bin', name)
         self.info('Installing binary:', path)
         open(path, 'wb').write(script)
         os.chmod(path, self.MODE)
@@ -129,6 +151,8 @@ class Install(Develop):
             opts.bindir = self.j(opts.prefix, 'bin')
         if opts.sharedir is None:
             opts.sharedir = self.j(opts.prefix, 'share', 'calibre')
+        self.prefix = opts.prefix
+        self.bindir = opts.bindir
         self.path = opts.libdir
         self.resources = opts.sharedir
         self.extensions = self.j(self.path, 'calibre', 'plugins')

setup/installer/__init__.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import with_statement
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import subprocess, tempfile, os, time
+
+from setup import Command, installer_name
+from setup.build_environment import HOST, PROJECT
+
+class VMInstaller(Command):
+
+    EXTRA_SLEEP = 5
+
+    INSTALLER_EXT = None
+    VM = None
+    VM_NAME = None
+    FREEZE_COMMAND = None
+    FREEZE_TEMPLATE = 'python setup.py {freeze_command}'
+    SHUTDOWN_CMD = ['sudo', 'shutdown', '-h', 'now']
+    IS_64_BIT = False
+
+    BUILD_CMD = 'ssh -t %s bash build-calibre'
+    BUILD_PREFIX = ['#!/bin/bash', 'export CALIBRE_BUILDBOT=1']
+    BUILD_RSYNC  = [r'cd ~/build', (
+        'rsync -avz --exclude src/calibre/plugins '
+               '--exclude calibre/src/calibre.egg-info --exclude docs '
+               '--exclude .bzr --exclude .build --exclude build --exclude dist '
+               '--exclude "*.pyc" --exclude "*.pyo" --exclude "*.swp" --exclude "*.swo" '
+               'rsync://{host}/work/{project} . ')]
+    BUILD_CLEAN = ['cd {project} ',
+                   'rm -rf dist/* build/* src/calibre/plugins/*']
+    BUILD_BUILD = ['python setup.py build',]
+
+    def add_options(self, parser):
+        parser.add_option('-s', '--dont-shutdown', default=False,
+            action='store_true', help='Dont shutdown the VM after building')
+        parser.add_option('--vm', help='Path to VM launcher script')
+
+
+    def get_build_script(self):
+        ans = '\n'.join(self.BUILD_PREFIX)+'\n\n'
+        ans += ' && \\\n'.join(self.BUILD_RSYNC)+ ' && \\\n'
+        ans += ' && \\\n'.join(self.BUILD_CLEAN) + ' && \\\n'
+        ans += ' && \\\n'.join(self.BUILD_BUILD) + ' && \\\n'
+        ans += self.FREEZE_TEMPLATE.format(freeze_command=self.FREEZE_COMMAND) + '\n'
+        ans = ans.format(project=PROJECT, host=HOST)
+        return ans
+
+    def vmware_started(self):
+        return 'started' in subprocess.Popen('/etc/init.d/vmware status', shell=True, stdout=subprocess.PIPE).stdout.read()
+
+    def start_vmware(self):
+        if not self.vmware_started():
+            if os.path.exists('/dev/kvm'):
+                subprocess.check_call('sudo rmmod -w kvm-intel kvm', shell=True)
+            subprocess.Popen('sudo /etc/init.d/vmware start', shell=True)
+
+    def stop_vmware(self):
+            while True:
+                try:
+                    subprocess.check_call('sudo /etc/init.d/vmware stop', shell=True)
+                    break
+                except:
+                    pass
+            while 'vmblock' in open('/proc/modules').read():
+                subprocess.check_call('sudo rmmod -f vmblock')
+
+
+    def run_vm(self):
+        self.__p = subprocess.Popen([self.vm])
+
+    def start_vm(self, sleep=75):
+        ssh_host = self.VM_NAME
+        self.run_vm()
+        build_script = self.get_build_script()
+        t = tempfile.NamedTemporaryFile(suffix='.sh')
+        t.write(build_script)
+        t.flush()
+        print 'Waiting for VM to startup'
+        while subprocess.call('ping -q -c1 '+ssh_host, shell=True,
+                   stdout=open('/dev/null', 'w')) != 0:
+            time.sleep(5)
+        time.sleep(self.EXTRA_SLEEP)
+        print 'Trying to SSH into VM'
+        subprocess.check_call(('scp', t.name, ssh_host+':build-calibre'))
+        subprocess.check_call(self.BUILD_CMD%ssh_host, shell=True)
+
+    def installer(self):
+        return installer_name(self.INSTALLER_EXT, self.IS_64_BIT)
+
+    def run(self, opts):
+        for x in ('dont_shutdown', 'vm'):
+            setattr(self, x, getattr(opts, x))
+        if self.vm is None:
+            self.vm = self.VM
+        if not self.vmware_started():
+            self.start_vmware()
+        self.start_vm()
+        self.download_installer()
+        if not self.dont_shutdown:
+            subprocess.call(['ssh', self.VM_NAME]+self.SHUTDOWN_CMD)
+
+    def download_installer(self):
+        installer = self.installer()
+        subprocess.check_call(['scp',
+            self.VM_NAME+':build/calibre/'+installer, 'dist'])
+        if not os.path.exists(installer):
+            self.warn('Failed to download installer')
+            raise SystemExit(1)

setup/installer/linux/__init__.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import with_statement
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import os
+
+from setup.installer import VMInstaller
+from setup import Command, installer_name
+
+class Linux32(VMInstaller):
+
+    INSTALLER_EXT = 'tar.bz2'
+    VM_NAME = 'gentoo32_build'
+    VM = '/vmware/bin/gentoo32_build'
+    FREEZE_COMMAND = 'linux_freeze'
+
+
+class Linux64(Command):
+
+    sub_commands = ['linux_freeze']
+
+    def run(self, opts):
+        installer = installer_name('tar.bz2', True)
+        if not os.path.exists(installer):
+            raise Exception('Failed to build installer '+installer)
+        return os.path.basename(installer)
+
+class Linux(Command):
+
+    sub_commands = ['linux64', 'linux32']

setup/installer/linux/freeze.py

@@ -0,0 +1,256 @@
+#!/usr/bin/env  python
+from __future__ import with_statement
+__license__   = 'GPL v3'
+__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
+__docformat__ = 'restructuredtext en'
+
+'''
+Create linux binary.
+'''
+
+from setup import Command, __version__, __appname__
+
+class LinuxFreeze(Command):
+
+    description = 'Create frozen linux binary'
+
+    def run(self, opts):
+        import glob, sys, tarfile, os, textwrap, shutil, platform
+        from contextlib import closing
+        from cx_Freeze import Executable, setup
+        from calibre.linux import entry_points
+        from calibre import walk
+        from calibre.web.feeds.recipes import recipe_modules
+
+        is64bit = platform.architecture()[0] == '64bit'
+        arch = 'x86_64' if is64bit else 'i686'
+
+
+        QTDIR          = '/usr/lib/qt4'
+        QTDLLS         = ('QtCore', 'QtGui', 'QtNetwork', 'QtSvg', 'QtXml',
+                'QtWebKit', 'QtDBus')
+
+        binary_excludes = ['libGLcore*', 'libGL*', 'libnvidia*']
+
+        os.system('sudo cp /usr/bin/calibre-mount-helper /tmp/calibre-mount-helper')
+        os.system('sudo chown kovid:users /tmp/calibre-mount-helper')
+
+        binary_includes = [
+                        '/usr/bin/pdftohtml',
+                        '/tmp/calibre-mount-helper',
+                        '/usr/lib/libunrar.so',
+                        '/usr/lib/libsqlite3.so.0',
+                        '/usr/lib/libsqlite3.so.0',
+                        '/usr/lib/libmng.so.1',
+                        '/usr/lib/libpodofo.so.0.6.99',
+                        '/lib/libz.so.1',
+                        '/usr/lib/libtiff.so.3',
+                        '/lib/libbz2.so.1',
+                        '/usr/lib/libpoppler.so.4',
+                        '/usr/lib/libpoppler-qt4.so.3',
+                        '/usr/lib/libxml2.so.2',
+                        '/usr/lib/libopenjpeg.so.2',
+                        '/usr/lib/libxslt.so.1',
+                        '/usr/lib64/libjpeg.so.7'.replace('64', '64' if is64bit
+                            else ''),
+                        '/usr/lib/libxslt.so.1',
+                        '/usr/lib/libgthread-2.0.so.0',
+                        '/usr/lib/gcc/***-pc-linux-gnu/4.4.1/libstdc++.so.6'.replace('***',
+                            arch),
+                        '/usr/lib/libpng12.so.0',
+                        '/usr/lib/libexslt.so.0',
+                        '/usr/lib/libMagickWand.so',
+                        '/usr/lib/libMagickCore.so',
+                        '/usr/lib/libgcrypt.so.11',
+                        '/usr/lib/libgpg-error.so.0',
+                        '/usr/lib/libphonon.so.4',
+                        '/usr/lib/libssl.so.0.9.8',
+                        '/usr/lib/libcrypto.so.0.9.8',
+                        '/lib/libreadline.so.6',
+                        ]
+
+        binary_includes += [os.path.join(QTDIR, 'lib%s.so.4'%x) for x in QTDLLS]
+
+
+        CALIBRESRC = self.d(self.SRC)
+        CALIBREPLUGINS = os.path.join(CALIBRESRC, 'src', 'calibre', 'plugins')
+        FREEZE_DIR = os.path.join(CALIBRESRC, 'build', 'cx_freeze')
+        DIST_DIR   = os.path.join(CALIBRESRC, 'dist')
+
+        os.chdir(CALIBRESRC)
+
+        self.info('Freezing calibre located at', CALIBRESRC)
+
+        entry_points = entry_points['console_scripts'] + entry_points['gui_scripts']
+        entry_points = ['calibre_postinstall=calibre.linux:binary_install',
+                        'calibre-parallel=calibre.parallel:main'] + entry_points
+        executables = {}
+        for ep in entry_points:
+            executables[ep.split('=')[0].strip()] = (ep.split('=')[1].split(':')[0].strip(),
+                                                    ep.split(':')[-1].strip())
+
+        if os.path.exists(FREEZE_DIR):
+            shutil.rmtree(FREEZE_DIR)
+        os.makedirs(FREEZE_DIR)
+
+        if not os.path.exists(DIST_DIR):
+            os.makedirs(DIST_DIR)
+
+        includes = [x[0] for x in executables.values()]
+        includes += ['email.iterators', 'email.generator', 'sqlite3.dump']
+
+
+        excludes = ['matplotlib', "Tkconstants", "Tkinter", "tcl", "_imagingtk",
+                    "ImageTk", "FixTk", 'wx', 'PyQt4.QtAssistant', 'PyQt4.QtOpenGL.so',
+                    'PyQt4.QtScript.so', 'PyQt4.QtSql.so', 'PyQt4.QtTest.so', 'qt',
+                    'glib', 'gobject']
+
+        packages = ['calibre', 'encodings', 'cherrypy', 'cssutils', 'xdg',
+                    'dateutil', 'dns', 'email']
+
+        includes += ['calibre.web.feeds.recipes.'+r for r in recipe_modules]
+        includes += ['calibre.gui2.convert.'+x.split('/')[-1].rpartition('.')[0] for x in \
+                glob.glob('src/calibre/gui2/convert/*.py')]
+
+        LOADER = '/tmp/loader.py'
+        open(LOADER, 'wb').write('# This script is never actually used.\nimport sys')
+
+        INIT_SCRIPT = '/tmp/init.py'
+        open(INIT_SCRIPT, 'wb').write(textwrap.dedent('''
+        ## Load calibre module specified in the environment variable CALIBRE_CX_EXE
+        ## Also restrict sys.path to the executables' directory and add the
+        ## executables directory to LD_LIBRARY_PATH
+        import encodings
+        import os
+        import sys
+        import warnings
+        import zipimport
+        import locale
+        import codecs
+
+        enc = locale.getdefaultlocale()[1]
+        if not enc:
+            enc = locale.nl_langinfo(locale.CODESET)
+        enc = codecs.lookup(enc if enc else 'UTF-8').name
+        sys.setdefaultencoding(enc)
+
+        paths = os.environ.get('LD_LIBRARY_PATH', '').split(os.pathsep)
+        if DIR_NAME not in paths or not sys.getfilesystemencoding():
+            paths.insert(0, DIR_NAME)
+            os.environ['LD_LIBRARY_PATH'] = os.pathsep.join(paths)
+            os.environ['PYTHONIOENCODING'] = enc
+            os.execv(sys.executable, sys.argv)
+
+        sys.path = sys.path[:3]
+        sys.frozen = True
+        sys.frozen_path = DIR_NAME
+        sys.extensions_location = os.path.join(DIR_NAME, 'plugins')
+        sys.resources_location = os.path.join(DIR_NAME, 'resources')
+
+        executables = %(executables)s
+
+        exe = os.environ.get('CALIBRE_CX_EXE', False)
+        ret = 1
+        if not exe:
+            print >>sys.stderr, 'Invalid invocation of calibre loader. CALIBRE_CX_EXE not set'
+        elif exe not in executables:
+            print >>sys.stderr, 'Invalid invocation of calibre loader. CALIBRE_CX_EXE=%%s is unknown'%%exe
+        else:
+            from PyQt4.QtCore import QCoreApplication
+            QCoreApplication.setLibraryPaths([sys.frozen_path, os.path.join(sys.frozen_path, "qtplugins")])
+            sys.argv[0] = exe
+            module, func = executables[exe]
+            module = __import__(module, fromlist=[1])
+            func = getattr(module, func)
+            ret = func()
+
+        module = sys.modules.get("threading")
+        if module is not None:
+            module._shutdown()
+        sys.exit(ret)
+        ''')%dict(executables=repr(executables)))
+        sys.argv = ['freeze', 'build_exe']
+        setup(
+            name        = __appname__,
+            version     = __version__,
+            executables = [Executable(script=LOADER, targetName='loader', compress=False)],
+            options     = { 'build_exe' :
+                            {
+                            'build_exe'       : os.path.join(CALIBRESRC, 'build/cx_freeze'),
+                            'optimize'        : 2,
+                            'excludes'        : excludes,
+                            'includes'        : includes,
+                            'packages'        : packages,
+                            'init_script'     : INIT_SCRIPT,
+                            'copy_dependent_files' : True,
+                            'create_shared_zip'    : False,
+                            }
+                            }
+            )
+
+        def copy_binary(src, dest_dir):
+            dest = os.path.join(dest_dir, os.path.basename(src))
+            if not os.path.exists(dest_dir):
+                os.makedirs(dest_dir)
+            shutil.copyfile(os.path.realpath(src), dest)
+            shutil.copymode(os.path.realpath(src), dest)
+
+        for f in binary_includes:
+            copy_binary(f, FREEZE_DIR)
+
+        for pat in binary_excludes:
+            matches = glob.glob(os.path.join(FREEZE_DIR, pat))
+            for f in matches:
+                os.remove(f)
+
+        self.info('Adding calibre plugins...')
+        os.makedirs(os.path.join(FREEZE_DIR, 'plugins'))
+        for f in glob.glob(os.path.join(CALIBREPLUGINS, '*.so')):
+            copy_binary(f, os.path.join(FREEZE_DIR, 'plugins'))
+
+        self.info('Adding calibre resources...')
+        shutil.copytree('resources', os.path.join(FREEZE_DIR, 'resources'))
+
+        self.info('Adding Qt plugins...')
+        plugdir = os.path.join(QTDIR, 'plugins')
+        for dirpath, dirnames, filenames in os.walk(plugdir):
+            for f in filenames:
+                if not f.endswith('.so') or 'designer' in dirpath or 'codecs' in dirpath or 'sqldrivers' in dirpath:
+                    continue
+                f = os.path.join(dirpath, f)
+                dest_dir = dirpath.replace(plugdir, os.path.join(FREEZE_DIR, 'qtplugins'))
+                copy_binary(f, dest_dir)
+
+        self.info('Creating launchers')
+        for exe in executables:
+            path = os.path.join(FREEZE_DIR, exe)
+            open(path, 'wb').write(textwrap.dedent('''\
+            #!/bin/sh
+            export CALIBRE_CX_EXE=%s
+            path=`readlink -e $0`
+            base=`dirname $path`
+            loader=$base/loader
+            export LD_LIBRARY_PATH=$base:$LD_LIBRARY_PATH
+            $loader "$@"
+            ''')%exe)
+            os.chmod(path, 0755)
+
+        exes = list(executables.keys())
+        exes.remove('calibre_postinstall')
+        exes.remove('calibre-parallel')
+        open(os.path.join(FREEZE_DIR, 'manifest'), 'wb').write('\n'.join(exes))
+
+        self.info('Creating archive...')
+        dist = open(os.path.join(DIST_DIR, 'calibre-%s-%s.tar.bz2'%(__version__,
+            arch)), 'wb')
+        with closing(tarfile.open(fileobj=dist, mode='w:bz2',
+                                format=tarfile.PAX_FORMAT)) as tf:
+            for f in walk(FREEZE_DIR):
+                name = f.replace(FREEZE_DIR, '')[1:]
+                if name:
+                    tf.add(f, name)
+        dist.flush()
+        dist.seek(0, 2)
+        self.info('Archive %s created: %.2f MB'%(dist.name,
+            dist.tell()/(1024.**2)))
+

setup/installer/osx/__init__.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import with_statement
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+from setup import Command
+from setup.installer import VMInstaller
+
+class OSX(Command):
+
+    sub_commands = ['osx32']
+
+    def run(self, opts):
+        pass
+
+
+class OSX32(VMInstaller):
+
+    INSTALLER_EXT = 'dmg'
+    VM_NAME = 'tiger_build'
+    VM = '/vmware/bin/%s'%VM_NAME
+    FREEZE_COMMAND = 'osx32_freeze'
+    BUILD_PREFIX = VMInstaller.BUILD_PREFIX + ['source ~/.profile']

setup/installer/osx/freeze.py

@@ -4,21 +4,29 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 ''' Create an OSX installer '''
 
 import sys, re, os, shutil, subprocess, stat, glob, zipfile, plistlib
-sys.path = sys.path[1:]
+from setup import __version__ as VERSION, __appname__ as APPNAME, SRC, Command, \
-l = {}
+        scripts, basenames, functions as main_functions, modules as main_modules
-exec open('setup.py').read() in l
-VERSION = l['VERSION']
-APPNAME = l['APPNAME']
-scripts = l['scripts']
-basenames = l['basenames']
-main_functions = l['main_functions']
-main_modules = l['main_modules']
 from setuptools import setup
-from py2app.build_app import py2app
+
-from modulegraph.find_modules import find_modules
+try:
+    from py2app.build_app import py2app
+    from modulegraph.find_modules import find_modules
+    py2app
+except ImportError:
+    py2app = object
 
 PYTHON = '/Library/Frameworks/Python.framework/Versions/Current/bin/python'
 
+info = warn = None
+
+class OSX32_Freeze(Command):
+
+    def run(self, opts):
+        global info, warn
+        info, warn = self.info, self.warn
+        main()
+
+
 class BuildAPP(py2app):
     QT_PREFIX = '/Volumes/sw/qt'
     LOADER_TEMPLATE = \
@@ -30,6 +38,8 @@ name = os.path.basename(path)
 base_dir = os.path.dirname(os.path.dirname(dirpath))
 resources_dir = os.path.join(base_dir, 'Resources')
 frameworks_dir = os.path.join(base_dir, 'Frameworks')
+extensions_dir = os.path.join(frameworks_dir, 'plugins')
+r_dir = os.path.join(resources_dir, 'resources')
 base_name = os.path.splitext(name)[0]
 python = os.path.join(base_dir, 'MacOS', 'python')
 qt_plugins = os.path.join(os.path.realpath(base_dir), 'MacOS')
@@ -42,6 +52,8 @@ print >>loader, 'if', repr(dirpath), 'in sys.path: sys.path.remove(', repr(dirpa
 print >>loader, 'sys.path.append(', repr(site_packages), ')'
 print >>loader, 'sys.frozen = "macosx_app"'
 print >>loader, 'sys.frameworks_dir =', repr(frameworks_dir)
+print >>loader, 'sys.extensions_location =', repr(extensions_dir)
+print >>loader, 'sys.resources_location =', repr(r_dir)
 print >>loader, 'import os'
 print >>loader, 'from %(module)s import %(function)s'
 print >>loader, '%(function)s()'
@@ -74,6 +86,8 @@ os.execv(python, args)
                 internet_enable=True,
                 format='UDBZ'):
         ''' Copy a directory d into a dmg named volname '''
+        if not os.path.exists(destdir):
+            os.makedirs(destdir)
         dmg = os.path.join(destdir, volname+'.dmg')
         if os.path.exists(dmg):
             os.unlink(dmg)
@@ -99,13 +113,13 @@ os.execv(python, args)
     @classmethod
     def fix_qt_dependencies(cls, path, deps):
         fp = '@executable_path/../Frameworks/'
-        print 'Fixing qt dependencies for:', os.path.basename(path)
+        info('Fixing qt dependencies for:', os.path.basename(path))
         for dep in deps:
             match = re.search(r'(Qt\w+?)\.framework', dep)
             if not match:
                 match = re.search(r'(phonon)\.framework', dep)
                 if not match:
-                    print dep
+                    warn(dep)
                     raise Exception('Unknown Qt dependency')
             module = match.group(1)
             newpath = fp + '%s.framework/Versions/Current/%s'%(module, module)
@@ -184,12 +198,18 @@ os.execv(python, args)
         all_names   = basenames['console'] + basenames['gui']
         all_modules   = main_modules['console'] + main_modules['gui']
         all_functions = main_functions['console'] + main_functions['gui']
-        print
+
-        print 'Adding PoDoFo'
+        info('\nAdding resources')
+        dest = os.path.join(resource_dir, 'resources')
+        if os.path.exists(dest):
+            shutil.rmtree(dest)
+        shutil.copytree(os.path.join(os.path.dirname(SRC), 'resources'), dest)
+
+        info('\nAdding PoDoFo')
         pdf = glob.glob(os.path.expanduser('/Volumes/sw/podofo/libpodofo*.dylib'))[0]
         shutil.copyfile(pdf, os.path.join(frameworks_dir, os.path.basename(pdf)))
-        print
+
-        print 'Adding poppler'
+        info('\nAdding poppler')
         for x in ('pdftohtml', 'libpoppler.4.dylib', 'libpoppler-qt4.3.dylib'):
             tgt = os.path.join(frameworks_dir, x)
             os.link(os.path.join(os.path.expanduser('~/poppler'), x), tgt)
@@ -202,7 +222,7 @@ os.execv(python, args)
             os.mkdir(loader_path)
         for name, module, function in zip(all_names, all_modules, all_functions):
             path = os.path.join(loader_path, name)
-            print 'Creating loader:', path
+            info('Creating loader:', path)
             f = open(path, 'w')
             f.write(BuildAPP.LOADER_TEMPLATE % dict(module=module,
                                                         function=function))
@@ -211,7 +231,7 @@ os.execv(python, args)
                      |stat.S_IWUSR|stat.S_IROTH|stat.S_IRGRP)
 
 
-        print 'Adding fontconfig'
+        info('Adding fontconfig')
         for f in glob.glob(os.path.expanduser('~/fontconfig-bundled/*')):
             dest = os.path.join(frameworks_dir, os.path.basename(f))
             if os.path.exists(dest):
@@ -224,22 +244,22 @@ os.execv(python, args)
 
         self.add_plugins()
 
-        print
+
-        print 'Adding IPython'
+        info('Adding IPython')
         dst = os.path.join(resource_dir, 'lib', 'python2.6', 'IPython')
         if os.path.exists(dst): shutil.rmtree(dst)
         shutil.copytree(os.path.expanduser('~/build/ipython/IPython'), dst)
 
-        print
+
-        print 'Adding ImageMagick'
+        info('Adding ImageMagick')
         dest = os.path.join(frameworks_dir, 'ImageMagick')
         if os.path.exists(dest):
             shutil.rmtree(dest)
         shutil.copytree(os.path.expanduser('~/ImageMagick'), dest, True)
         shutil.copyfile('/usr/local/lib/libpng12.0.dylib', os.path.join(dest, 'lib', 'libpng12.0.dylib'))
 
-        print
+
-        print 'Installing prescipt'
+        info('Installing prescipt')
         sf = [os.path.basename(s) for s in all_names]
         launcher_path = os.path.join(resource_dir, '__boot__.py')
         f = open(launcher_path, 'r')
@@ -249,19 +269,21 @@ os.execv(python, args)
         src = re.sub('(_run\s*\(.*?.py.*?\))', '%s'%(
 '''
 sys.frameworks_dir = os.path.join(os.path.dirname(os.environ['RESOURCEPATH']), 'Frameworks')
+sys.resources_location = os.path.join(os.environ['RESOURCEPATH'], 'resources')
+sys.extensions_location = os.path.join(sys.frameworks_dir, 'plugins')
 ''') + r'\n\1', src)
         f = open(launcher_path, 'w')
         print >>f, 'import sys, os'
         f.write(src)
         f.close()
-        print
+
-        print 'Adding main scripts to site-packages'
+        info('\nAdding main scripts to site-packages')
         f = zipfile.ZipFile(os.path.join(self.dist_dir, APPNAME+'.app', 'Contents', 'Resources', 'lib', 'python'+sys.version[:3], 'site-packages.zip'), 'a', zipfile.ZIP_DEFLATED)
         for script in scripts['gui']+scripts['console']:
             f.write(script, script.partition('/')[-1])
         f.close()
-        print
+
-        print 'Creating console.app'
+        info('\nCreating console.app')
         contents_dir = os.path.dirname(resource_dir)
         cc_dir = os.path.join(contents_dir, 'console.app', 'Contents')
         os.makedirs(cc_dir)
@@ -275,12 +297,11 @@ sys.frameworks_dir = os.path.join(os.path.dirname(os.environ['RESOURCEPATH']), '
             else:
                 os.symlink(os.path.join('../..', x),
                            os.path.join(cc_dir, x))
-        print
+
-        print 'Building disk image'
+        info('\nBuilding disk image')
         BuildAPP.makedmg(os.path.join(self.dist_dir, APPNAME+'.app'), APPNAME+'-'+VERSION)
 
 def main():
-    sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src'))
     sys.argv[1:2] = ['py2app']
     d = os.path.dirname
     icon = os.path.abspath('icons/library.icns')
@@ -302,8 +323,9 @@ def main():
                                        'mechanize', 'ClientForm', 'usbobserver',
                                        'genshi', 'calibre.web.feeds.recipes.*',
                                        'calibre.gui2.convert.*',
+                                       'PyQt4.QtNetwork',
                                        'keyword', 'codeop', 'pydoc', 'readline',
-                                       'BeautifulSoup', 'calibre.ebooks.lrf.fonts.prs500.*',
+                                       'BeautifulSoup',
                                        'dateutil', 'email.iterators',
                                        'email.generator', 'sqlite3.dump',
                                        'calibre.ebooks.metadata.amazon',
@@ -330,5 +352,3 @@ def main():
         )
     return 0
 
-if __name__ == '__main__':
-    sys.exit(main())

setup/installer/windows/__init__.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import with_statement
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import os, shutil, subprocess
+
+from setup import Command, __appname__
+from setup.installer import VMInstaller
+from setup.installer.windows import build_installer
+
+class Win(Command):
+
+    sub_commands = ['win32']
+
+    def run(self, opts):
+        pass
+
+
+class Win32(VMInstaller):
+
+    INSTALLER_EXT = 'exe'
+    VM_NAME = 'xp_build'
+    VM = '/vmware/bin/%s'%VM_NAME
+    FREEZE_COMMAND = 'win32_freeze'
+
+    def download_installer(self):
+        installer = self.installer()
+        if os.path.exists('build/py2exe'):
+            shutil.rmtree('build/py2exe')
+        subprocess.check_call(('scp', '-rp', 'xp_build:build/%s/build/py2exe'%__appname__,
+                     'build'))
+        if not os.path.exists('build/py2exe'):
+            self.warn('Failed to run py2exe')
+            raise SystemExit(1)
+        self.run_windows_install_jammer(installer)
+
+    def run_windows_install_jammer(self, installer):
+        build_installer.run_install_jammer(
+                                    installer_name=os.path.basename(installer))
+        if not os.path.exists(installer):
+            self.warn('Failed to run installjammer')
+            raise SystemExit(1)
+
+

setup/installer/windows/build_installer.py

@@ -6,7 +6,7 @@ __docformat__ = 'restructuredtext en'
 '''
 '''
 import sys, time, subprocess, os, re
-from calibre import __appname__, __version__
+from setup import SRC, __appname__, __version__
 
 INSTALLJAMMER = '/usr/local/installjammer/installjammer'
 
@@ -23,8 +23,8 @@ cmdline = [
     '-DPackageSummary', '%s: E-book library management'%__appname__,
     '-DVersion', __version__,
     '-DInstallVersion', sv + '.0',
-    '-DLicense', open(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'LICENSE')).read().replace('\n', '\r\n'),
+    '-DLicense', open(os.path.join(os.path.dirname(SRC), 'LICENSE'), 'rb').read().replace('\n', '\r\n'),
-    '--output-dir', os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'dist'),
+    '--output-dir', os.path.join(os.path.dirname(SRC), 'dist'),
     '--platform', 'Windows',
     '--verbose'
 ]

setup/installer/windows/calibre/calibre.mpi

@@ -291,16 +291,13 @@ File ::8147A9D4-D17C-55EB-CAA5-CC0C04EDC0D2 -name prs500.cat -parent 48EA1D8C-F4
 File ::1085BEED-C76A-5FF4-209A-E03CFB697462 -name prs500.inf -parent 48EA1D8C-F4C8-3D34-229D-B501057802F3
 File ::0AC00D67-8452-CABB-6843-FE6A464E9AE9 -type dir -name plugins -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::540FEFD3-9B1B-00E6-B8A5-EA8EB959428E -name podofo.pyd -parent 0AC00D67-8452-CABB-6843-FE6A464E9AE9
-File ::87BCFAF5-FAEB-219C-BA28-1DFD1B2A7743 -name podofo.pyd.manifest -parent 0AC00D67-8452-CABB-6843-FE6A464E9AE9
 File ::0C77BC18-CE53-35EF-A667-C25B81C43E64 -name pictureflow.pyd -parent 0AC00D67-8452-CABB-6843-FE6A464E9AE9
-File ::9F15B633-412B-51ED-2B7D-4EB0A0847740 -name winutil.pyd.manifest -parent 0AC00D67-8452-CABB-6843-FE6A464E9AE9
-File ::9E3D908E-B119-30AE-391D-D16CA6BEE7AC -name msdes.pyd.manifest -parent 0AC00D67-8452-CABB-6843-FE6A464E9AE9
 File ::F1D1D581-9194-D12F-6139-F920EEC1B14E -name winutil.pyd -parent 0AC00D67-8452-CABB-6843-FE6A464E9AE9
 File ::A94E4D31-5F8C-A363-AE61-A8F2E3A7B756 -name lzx.pyd -parent 0AC00D67-8452-CABB-6843-FE6A464E9AE9
-File ::34C430AE-BC1B-8352-1713-50C89C176C77 -name lzx.pyd.manifest -parent 0AC00D67-8452-CABB-6843-FE6A464E9AE9
 File ::2C88788E-B9FB-729D-442B-600ED97096A0 -name cPalmdoc.pyd -parent 0AC00D67-8452-CABB-6843-FE6A464E9AE9
-File ::8A15A76F-105D-3B6C-6E54-A01F8EA16CE3 -name cPalmdoc.pyd.manifest -parent 0AC00D67-8452-CABB-6843-FE6A464E9AE9
 File ::26503DDE-8C5F-102F-B689-C6A17B2D6C93 -name msdes.pyd -parent 0AC00D67-8452-CABB-6843-FE6A464E9AE9
+File ::0FE771BC-EBF3-E4E0-DDE0-2D69F2BD99C6 -name calibre_poppler.pyd -parent 0AC00D67-8452-CABB-6843-FE6A464E9AE9
+File ::FCB0D098-8F90-1C97-9E20-65546F08A203 -name fontconfig.pyd -parent 0AC00D67-8452-CABB-6843-FE6A464E9AE9
 File ::01034EB7-C79C-42B9-6FF0-E06C72EF2623 -type dir -name iconengines -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::8ADB07A0-6B9E-8F53-34DF-2035C7C343F1 -name qsvgicon4.dll -parent 01034EB7-C79C-42B9-6FF0-E06C72EF2623
 File ::41F512E3-9F39-68C6-BB7F-58F572755D35 -name qsvgicon4.exp -parent 01034EB7-C79C-42B9-6FF0-E06C72EF2623
@@ -567,6 +564,9 @@ File ::9E4E5E8F-30C0-E631-9516-2AE01A5CA0E9 -name ebook-device.exe.local -parent
 File ::7BE6B538-70D5-A7EB-5F91-E14CE57B394B -name calibre-complete.exe.local -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::C4E40030-3EE0-8B05-E6B9-89E81433EE1F -name phonon4.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::9E84342F-36ED-7ED3-8F90-1EC55267BCFC -name poppler-qt4.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
+File ::C9967023-A4C2-856C-1D90-DC710105EBCD -name jpeg62.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
+File ::12FA46DA-F25E-0D26-E23C-006013332FED -name freetype.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
+File ::B1560042-C99B-9803-552E-21C15F0DFD85 -type dir -name resources -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 Component ::F6829AB7-9F66-4CEE-CA0E-21F54C6D3609 -setup Install -active Yes -platforms {AIX-ppc FreeBSD-4-x86 FreeBSD-x86 HPUX-hppa Linux-x86 Solaris-sparc Windows FreeBSD-5-x86 FreeBSD-6-x86 FreeBSD-7-x86 Linux-x86_64 Solaris-x86} -name Main -parent Components
 SetupType ::D9ADE41C-B744-690C-2CED-CF826BF03D2E -setup Install -active Yes -platforms {AIX-ppc FreeBSD-4-x86 FreeBSD-x86 HPUX-hppa Linux-x86 Solaris-sparc Windows FreeBSD-5-x86 FreeBSD-6-x86 FreeBSD-7-x86 Linux-x86_64 Solaris-x86} -name Typical -parent SetupTypes
 

setup/installer/windows/freeze.py

@@ -17,26 +17,27 @@ PODOFO           = 'C:\\podofo'
 FONTCONFIG_DIR   = 'C:\\fontconfig'
 VC90             = r'C:\VC90.CRT'
 
-# ModuleFinder can't handle runtime changes to __path__, but win32com uses them
 import sys
-import py2exe.mf as modulefinder
+
-import win32com
+def fix_module_finder():
-for p in win32com.__path__[1:]:
+    # ModuleFinder can't handle runtime changes to __path__, but win32com uses them
-    modulefinder.AddPackagePath("win32com", p)
+    import py2exe.mf as modulefinder
-for extra in ["win32com.shell"]: #,"win32com.mapi"
+    import win32com
-    __import__(extra)
+    for p in win32com.__path__[1:]:
-    m = sys.modules[extra]
+        modulefinder.AddPackagePath("win32com", p)
-    for p in m.__path__[1:]:
+    for extra in ["win32com.shell"]: #,"win32com.mapi"
-        modulefinder.AddPackagePath(extra, p)
+        __import__(extra)
+        m = sys.modules[extra]
+        for p in m.__path__[1:]:
+            modulefinder.AddPackagePath(extra, p)
 
 
-import os, py2exe, shutil, zipfile, glob, re
+import os, shutil, zipfile, glob, re
 from distutils.core import setup
-BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
+from setup import __version__ as VERSION, __appname__ as APPNAME, scripts, \
-sys.path.insert(0, BASE_DIR)
+    basenames, SRC, Command
-from setup import VERSION, APPNAME, scripts, basenames
-sys.path.remove(BASE_DIR)
 
+BASE_DIR = os.path.dirname(SRC)
 ICONS = [os.path.abspath(os.path.join(BASE_DIR, 'icons', i)) for i in ('library.ico', 'viewer.ico')]
 for icon in ICONS:
     if not os.access(icon, os.R_OK):
@@ -47,11 +48,95 @@ WINVER = VERSION+'.0'
 
 PY2EXE_DIR = os.path.join(BASE_DIR, 'build','py2exe')
 
-class BuildEXE(py2exe.build_exe.py2exe):
+info = warn = None
+
+class Win32Freeze(Command):
+
+    def run(self, opts):
+        global info, warn
+        info, warn = self.info, self.warn
+        main()
+
+BOOT_COMMON = '''\
+import sys, os
+if sys.frozen == "windows_exe":
+    class Stderr(object):
+        softspace = 0
+        _file = None
+        _error = None
+        def write(self, text, alert=sys._MessageBox, fname=os.path.expanduser('~\calibre.log')):
+            if self._file is None and self._error is None:
+                try:
+                    self._file = open(fname, 'wb')
+                except Exception, details:
+                    self._error = details
+                    import atexit
+                    atexit.register(alert, 0,
+                                    ("The logfile %s could not be opened: "
+                                    "\\n%s\\n\\nTry setting the HOME environment "
+                                    "variable to a directory for which you "
+                                    "have write permission.") % (fname, details),
+                                    "Errors occurred")
+                else:
+                    import atexit
+                    #atexit.register(alert, 0,
+                    #                "See the logfile '%s' for details" % fname,
+                    #                "Errors occurred")
+            if self._file is not None:
+                self._file.write(text)
+                self._file.flush()
+        def flush(self):
+            if self._file is not None:
+                self._file.flush()
+
+    #del sys._MessageBox
+    #del Stderr
+
+    class Blackhole(object):
+        softspace = 0
+        def write(self, text):
+            pass
+        def flush(self):
+            pass
+    sys.stdout = Stderr()
+    sys.stderr = Stderr()
+    del Blackhole
+
+# Disable linecache.getline() which is called by
+# traceback.extract_stack() when an exception occurs to try and read
+# the filenames embedded in the packaged python code.  This is really
+# annoying on windows when the d: or e: on our build box refers to
+# someone elses removable or network drive so the getline() call
+# causes it to ask them to insert a disk in that drive.
+import linecache
+def fake_getline(filename, lineno, module_globals=None):
+    return ''
+linecache.orig_getline = linecache.getline
+linecache.getline = fake_getline
+
+del linecache, fake_getline
+
+fenc = sys.getfilesystemencoding( )
+base = os.path.dirname(sys.executable.decode(fenc))
+sys.resources_location = os.path.join(base, 'resources')
+sys.extensions_location = os.path.join(base, 'plugins')
+
+
+del sys
+'''
+
+try:
+    import py2exe
+    bc = py2exe.build_exe.py2exe
+except ImportError:
+    py2exe = object
+    bc = object
+
+class BuildEXE(bc):
 
     def run(self):
         py2exe.build_exe.py2exe.run(self)
-        print 'Adding plugins...'
+        info('\nAdding plugins...')
         tgt = os.path.join(self.dist_dir, 'plugins')
         if not os.path.exists(tgt):
             os.mkdir(tgt)
@@ -62,30 +147,35 @@ class BuildEXE(py2exe.build_exe.py2exe):
         for f in glob.glob(os.path.join(BASE_DIR, 'src', 'calibre', 'plugins', '*.manifest')):
             shutil.copyfile(f, os.path.join(tgt, os.path.basename(f)))
         shutil.copyfile('LICENSE', os.path.join(self.dist_dir, 'LICENSE'))
-        print
+
-        print 'Adding QtXml4.dll'
+
+        info('\nAdding resources...')
+        tgt = os.path.join(self.dist_dir, 'resources')
+        if os.path.exists(tgt):
+            shutil.rmtree(tgt)
+        shutil.copytree(os.path.join(BASE_DIR, 'resources'), tgt)
+
+        info('\nAdding QtXml4.dll')
         shutil.copyfile(os.path.join(QT_DIR, 'bin', 'QtXml4.dll'),
                             os.path.join(self.dist_dir, 'QtXml4.dll'))
-        print 'Adding Qt plugins...',
+        info('\nAdding Qt plugins...')
         qt_prefix = QT_DIR
         plugdir = os.path.join(qt_prefix, 'plugins')
         for d in ('imageformats', 'codecs', 'iconengines'):
-            print d,
+            info(d)
             imfd = os.path.join(plugdir, d)
             tg = os.path.join(self.dist_dir, d)
             if os.path.exists(tg):
                 shutil.rmtree(tg)
             shutil.copytree(imfd, tg)
 
-        print
+        info('Adding main scripts')
-        print 'Adding main scripts'
         f = zipfile.ZipFile(os.path.join(PY2EXE_DIR, 'library.zip'), 'a', zipfile.ZIP_DEFLATED)
         for i in scripts['console'] + scripts['gui']:
             f.write(i, i.partition('\\')[-1])
         f.close()
 
-        print
+        info('Copying icons')
-        print 'Copying icons'
         for icon in ICONS:
             shutil.copyfile(icon, os.path.join(PY2EXE_DIR, os.path.basename(icon)))
 
@@ -149,6 +239,12 @@ def main(args=sys.argv):
     if os.path.exists(PY2EXE_DIR):
         shutil.rmtree(PY2EXE_DIR)
 
+    fix_module_finder()
+
+    boot_common = os.path.join(sys.prefix, 'Lib', 'site-packages', 'py2exe',
+    'boot_common.py')
+    open(boot_common, 'wb').write(BOOT_COMMON)
+
     console = [exe_factory(basenames['console'][i], scripts['console'][i])
                for i in range(len(scripts['console']))]
     setup(
@@ -190,5 +286,5 @@ def main(args=sys.argv):
           )
     return 0
 
-if __name__ == '__main__':
+
-    sys.exit(main())
+

setup/publish.py

@@ -6,7 +6,8 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os, shutil, subprocess
+import sys, os, shutil, subprocess, re, time
+from datetime import datetime
 
 from setup import Command, __appname__, __version__
 
@@ -40,4 +41,84 @@ class Manual(Command):
         if os.path.exists(path):
             shutil.rmtree(path)
 
+class TagRelease(Command):
+
+    description = 'Tag a new release in bzr'
+
+    def run(self, opts):
+        self.info('Tagging release')
+        subprocess.check_call(('bzr tag '+__version__).split())
+        subprocess.check_call('bzr commit --unchanged -m'.split() + ['IGN:Tag release'])
+
+if os.environ.get('CALIBRE_BUILDBOT', None) == '1':
+    class UploadRss(Command):
+        pass
+else:
+    class UploadRss(Command):
+
+        description = 'Generate and uplaod a RSS feed of calibre releases'
+
+        from bzrlib import log as blog
+
+        class ChangelogFormatter(blog.LogFormatter):
+            supports_tags = True
+            supports_merge_revisions = False
+            _show_advice = False
+
+            def __init__(self, num_of_versions=20):
+                sys.path.insert(0, os.path.join(Command.SRC, 'calibre', 'utils'))
+                from rss_gen import RSS2
+                self.num_of_versions = num_of_versions
+                self.rss = RSS2(
+                                title = 'calibre releases',
+                                link  = 'http://calibre.kovidgoyal.net/wiki/Changelog',
+                                description = 'Latest release of calibre',
+                                lastBuildDate = datetime.utcnow()
+                                )
+                self.current_entry = None
+
+            def log_revision(self, r):
+                from rss_gen import RSSItem, Guid
+                if len(self.rss.items) > self.num_of_versions-1:
+                    return
+                msg = r.rev.message
+                match = re.match(r'version\s+(\d+\.\d+.\d+)', msg)
+
+                if match:
+                    if self.current_entry is not None:
+                        mkup = '<div><ul>%s</ul></div>'
+                        self.current_entry.description = mkup%(''.join(
+                                    self.current_entry.description))
+                        if match.group(1) == '0.5.14':
+                            self.current_entry.description = \
+                            '''<div>See <a href="http://calibre.kovidgoyal.net/new_in_6">New in
+                            6</a></div>'''
+                        self.rss.items.append(self.current_entry)
+                    timestamp = r.rev.timezone + r.rev.timestamp
+                    self.current_entry = RSSItem(
+                            title = 'calibre %s released'%match.group(1),
+                            link  = 'http://calibre.kovidgoyal.net/download',
+                            guid = Guid(match.group(), False),
+                            pubDate = datetime(*time.gmtime(timestamp)[:6]),
+                            description = []
+                    )
+                elif self.current_entry is not None:
+                    if re.search(r'[a-zA-Z]', msg) and len(msg.strip()) > 5:
+                        if 'translation' not in msg and not msg.startswith('IGN'):
+                            msg = msg.replace('<', '&lt;').replace('>', '&gt;')
+                            msg = re.sub('#(\d+)', r'<a href="http://calibre.kovidgoyal.net/ticket/\1">#\1</a>',
+                                            msg)
+
+                            self.current_entry.description.append(
+                                            '<li>%s</li>'%msg.strip())
+
+
+        def run(self, opts):
+            from bzrlib import log, branch
+            bzr_path = os.path.expanduser('~/work/calibre')
+            b = branch.Branch.open(bzr_path)
+            lf = UploadRss.ChangelogFormatter()
+            log.show_log(b, lf)
+            lf.rss.write_xml(open('/tmp/releases.xml', 'wb'))
+            #subprocess.check_call('scp /tmp/releases.xml divok:/var/www/calibre.kovidgoyal.net/htdocs/downloads'.split())
 

setup/upload.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import with_statement
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import os, re, cStringIO
+from subprocess import check_call
+
+from setup import Command, __version__, installer_name
+
+PREFIX = "/var/www/calibre.kovidgoyal.net"
+DOWNLOADS = PREFIX+"/htdocs/downloads"
+BETAS = DOWNLOADS +'/betas'
+DOCS = PREFIX+"/htdocs/apidocs"
+USER_MANUAL = PREFIX+'/htdocs/user_manual'
+HTML2LRF = "calibre/ebooks/lrf/html/demo"
+TXT2LRF  = "src/calibre/ebooks/lrf/txt/demo"
+MOBILEREAD = 'ftp://dev.mobileread.com/calibre/'
+
+
+
+class UploadInstallers(Command):
+    description = 'Upload any installers present in dist/'
+    def curl_list_dir(self, url=MOBILEREAD, listonly=1):
+        import pycurl
+        c = pycurl.Curl()
+        c.setopt(pycurl.URL, url)
+        c.setopt(c.FTP_USE_EPSV, 1)
+        c.setopt(c.NETRC, c.NETRC_REQUIRED)
+        c.setopt(c.FTPLISTONLY, listonly)
+        c.setopt(c.FTP_CREATE_MISSING_DIRS, 1)
+        b = cStringIO.StringIO()
+        c.setopt(c.WRITEFUNCTION, b.write)
+        c.perform()
+        c.close()
+        return b.getvalue().split() if listonly else b.getvalue().splitlines()
+
+    def curl_delete_file(self, path, url=MOBILEREAD):
+        import pycurl
+        c = pycurl.Curl()
+        c.setopt(pycurl.URL, url)
+        c.setopt(c.FTP_USE_EPSV, 1)
+        c.setopt(c.NETRC, c.NETRC_REQUIRED)
+        self.info('Deleting file %s on %s'%(path, url))
+        c.setopt(c.QUOTE, ['dele '+ path])
+        c.perform()
+        c.close()
+
+
+    def curl_upload_file(self, stream, url):
+        import pycurl
+        c = pycurl.Curl()
+        c.setopt(pycurl.URL, url)
+        c.setopt(pycurl.UPLOAD, 1)
+        c.setopt(c.NETRC, c.NETRC_REQUIRED)
+        c.setopt(pycurl.READFUNCTION, stream.read)
+        stream.seek(0, 2)
+        c.setopt(pycurl.INFILESIZE_LARGE, stream.tell())
+        stream.seek(0)
+        c.setopt(c.NOPROGRESS, 0)
+        c.setopt(c.FTP_CREATE_MISSING_DIRS, 1)
+        self.info('Uploading file %s to url %s' % (getattr(stream, 'name', ''),
+            url))
+        try:
+            c.perform()
+            c.close()
+        except:
+            pass
+        files = self.curl_list_dir(listonly=0)
+        for line in files:
+            line = line.split()
+            if url.endswith(line[-1]):
+                size = long(line[4])
+                stream.seek(0,2)
+                if size != stream.tell():
+                    raise RuntimeError('curl failed to upload %s correctly'%getattr(stream, 'name', ''))
+
+    def upload_installer(self, name):
+        if not os.path.exists(name):
+            return
+        bname = os.path.basename(name)
+        pat = re.compile(bname.replace(__version__, r'\d+\.\d+\.\d+'))
+        for f in self.curl_list_dir():
+            if pat.search(f):
+                self.curl_delete_file('/calibre/'+f)
+        self.curl_upload_file(open(name, 'rb'), MOBILEREAD+os.path.basename(name))
+
+    def run(self, opts):
+        self.info('Uploading installers...')
+        installers = list(map(installer_name, ('dmg', 'exe', 'tar.bz2')))
+        installers.append(installer_name('tar.bz2', is64bit=True))
+        map(self.upload_installer, installers)
+
+        check_call('''ssh divok echo %s \\> %s/latest_version'''\
+                   %(__version__, DOWNLOADS), shell=True)
+
+class UploadUserManual(Command):
+    description = 'Build and upload the User Manual'
+    sub_commands = ['manual']
+
+    def run(self, opts):
+        check_call(' '.join(['scp', '-r', 'src/calibre/manual/.build/html/*',
+                    'divok:%s'%USER_MANUAL]), shell=True)
+
+
+class UploadDemo(Command):
+
+    description = 'Rebuild and upload various demos'
+
+    def run(self, opts):
+        check_call(
+           '''ebook-convert %s/demo.html /tmp/html2lrf.lrf '''
+           '''--title='Demonstration of html2lrf' --authors='Kovid Goyal' '''
+           '''--header '''
+           '''--serif-family "/usr/share/fonts/corefonts, Times New Roman" '''
+           '''--mono-family  "/usr/share/fonts/corefonts, Andale Mono" '''
+           ''''''%self.j(self.SRC, HTML2LRF), shell=True)
+
+        check_call(
+            'cd src/calibre/ebooks/lrf/html/demo/ && '
+            'zip -j /tmp/html-demo.zip * /tmp/html2lrf.lrf', shell=True)
+
+        check_call('scp /tmp/html-demo.zip divok:%s/'%(DOWNLOADS,), shell=True)
+
+
+

src/calibre/utils/ipc/launch.py

@@ -109,8 +109,10 @@ class Worker(object):
             resources = os.path.join(contents, 'Resources')
             fd = os.path.join(contents, 'Frameworks')
             sp = os.path.join(resources, 'lib', 'python'+sys.version[:3], 'site-packages.zip')
-            self.osx_prefix = 'import sys; sys.frameworks_dir = "%s"; sys.frozen = "macosx_app"; '%fd
+            self.osx_prefix = 'import sys, os; sys.frameworks_dir = "%s"; sys.frozen = "macosx_app"; '%fd
             self.osx_prefix += 'sys.path.insert(0, %s); '%repr(sp)
+            self.osx_prefix += 'sys.extensions_location = os.path.join(sys.frameworks_dir, "plugins");'
+            self.osx_prefix += 'sys.resources_location = os.path.join(os.path.dirname(sys.frameworks_dir), "Resources", "resources"); '
 
             self._env['PYTHONHOME']  = resources
             self._env['MAGICK_HOME'] = os.path.join(fd, 'ImageMagick')