IGN:Use distutils to build PyQt extensions

pyqtdistutils.py

@@ -0,0 +1,195 @@
+#!/usr/bin/env  python
+__license__   = 'GPL v3'
+__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
+__docformat__ = 'restructuredtext en'
+
+'''
+Build PyQt extensions. Integrates with distutils (but uses the PyQt build system).
+'''
+from distutils.core import Extension
+from distutils.command.build_ext import build_ext as _build_ext
+from distutils.dep_util import newer_group
+from distutils import log
+    
+import sipconfig, os, sys, string, glob, shutil
+from PyQt4 import pyqtconfig
+iswindows = 'win32' in sys.platform
+WINDOWS_PYTHON = ['C:/Python25/libs']
+OSX_SDK = 'TODO:'
+
+def replace_suffix(path, new_suffix):
+    return os.path.splitext(path)[0] + new_suffix
+
+class PyQtExtension(Extension):
+    
+    def __init__(self, name, sources, sip_sources, **kw):
+        '''
+        :param sources: Qt .cpp and .h files needed for this extension
+        :param sip_sources: List of .sip files this extension depends on. The
+                            first .sip file will be used toactually build the extension.
+        '''
+        self.module_makefile = pyqtconfig.QtGuiModuleMakefile
+        self.sip_sources = map(lambda x: x.replace('/', os.sep), sip_sources)
+        Extension.__init__(self, name, sources, **kw)
+        
+
+class build_ext(_build_ext):
+    
+    def make(self, makefile):
+        make = 'make'
+        if iswindows:
+            make = 'mingw32-make'
+        self.spawn([make, '-f', makefile])
+    
+    def build_qt_objects(self, ext, bdir):
+        if not iswindows:
+            bdir = os.path.join(bdir, 'qt')
+        if not os.path.exists(bdir):
+            os.makedirs(bdir)
+        cwd = os.getcwd()
+        sources = map(os.path.abspath, ext.sources)
+        os.chdir(bdir)
+        try:
+            headers = set([f for f in sources if f.endswith('.h')])
+            sources = set(sources) - headers
+            name = ext.name.rpartition('.')[-1] 
+            pro = '''\
+TARGET   = %s
+TEMPLATE = lib
+HEADERS  = %s
+SOURCES  = %s
+VERSION  = 1.0.0
+CONFIG   += x86 ppc
+'''%(name, ' '.join(headers), ' '.join(sources))
+            open(name+'.pro', 'wb').write(pro)
+            self.spawn(['qmake', '-o', 'Makefile.qt', name+'.pro'])
+            self.make('Makefile.qt')
+            pat = 'release\\*.o' if iswindows else '*.o'
+            return glob.glob(pat)
+        finally:
+            os.chdir(cwd)
+    
+    def build_sbf(self, sip, sbf, bdir):
+        sip_bin = self.sipcfg.sip_bin
+        self.spawn([sip_bin,
+                    "-c", bdir,
+                    "-b", sbf,
+                    '-I', self.pyqtcfg.pyqt_sip_dir,
+                    ] + self.pyqtcfg.pyqt_sip_flags.split()+
+                    [sip])
+    
+    def build_pyqt(self, bdir, sbf, ext, qtobjs, headers):
+        makefile = ext.module_makefile(configuration=self.pyqtcfg,
+                                       build_file=sbf, dir=bdir, 
+                                       makefile='Makefile.pyqt',
+                                       universal=OSX_SDK, qt=1)
+        if 'win32' in sys.platform:
+            makefile.extra_lib_dirs += WINDOWS_PYTHON
+        makefile.extra_include_dirs = list(set(map(os.path.dirname, headers)))
+        makefile.extra_lflags += qtobjs
+        makefile.generate()
+        cwd = os.getcwd()
+        os.chdir(bdir)
+        try:
+            self.make('Makefile.pyqt')
+        finally:
+            os.chdir(cwd)
+        
+        
+    
+    def build_extension(self, ext):
+        self.inplace = True
+        if not isinstance(ext, PyQtExtension):
+            return _build_ext.build_extension(self, ext)
+        
+        fullname = self.get_ext_fullname(ext.name)
+        if self.inplace:
+            # ignore build-lib -- put the compiled extension into
+            # the source tree along with pure Python modules
+
+            modpath = string.split(fullname, '.')
+            package = string.join(modpath[0:-1], '.')
+            base = modpath[-1]
+
+            build_py = self.get_finalized_command('build_py')
+            package_dir = build_py.get_package_dir(package)
+            ext_filename = os.path.join(package_dir,
+                                        self.get_ext_filename(base))
+        else:
+            ext_filename = os.path.join(self.build_lib,
+                                        self.get_ext_filename(fullname))
+        bdir = os.path.abspath(os.path.join(self.build_temp, fullname))
+        if not os.path.exists(bdir):
+            os.makedirs(bdir)
+        ext.sources = map(os.path.abspath, ext.sources)
+        qt_dir = 'qt\\release' if iswindows else 'qt'
+        objects = set(map(lambda x: os.path.join(bdir, qt_dir, replace_suffix(os.path.basename(x), '.o')),
+                      [s for s in ext.sources if not s.endswith('.h')]))
+        newer = False
+        for object in objects:
+            if newer_group(ext.sources, object, missing='newer'):
+                newer = True
+                break
+        headers = [f for f in ext.sources if f.endswith('.h')]
+        if self.force or newer:
+            log.info('building \'%s\' extension', ext.name)
+            objects = self.build_qt_objects(ext, bdir)
+        
+        self.sipcfg  = sipconfig.Configuration()
+        self.pyqtcfg = pyqtconfig.Configuration()
+        sbf_sources = []
+        for sip in ext.sip_sources:
+            sipbasename = os.path.basename(sip)
+            sbf = os.path.join(bdir, replace_suffix(sipbasename, ".sbf"))
+            sbf_sources.append(sbf)
+            if self.force or newer_group(ext.sip_sources, sbf, 'newer'):
+                self.build_sbf(sip, sbf, bdir)
+        generated_sources = []
+        for sbf in sbf_sources:
+            generated_sources += self.get_sip_output_list(sbf, bdir)
+        
+        depends = generated_sources + list(objects)
+        mod = os.path.join(bdir, os.path.basename(ext_filename))
+        
+        if self.force or newer_group(depends, mod, 'newer'):
+            self.build_pyqt(bdir, sbf_sources[0], ext, list(objects), headers)
+        
+        if self.force or newer_group([mod], ext_filename, 'newer'):
+            if os.path.exists(ext_filename):
+                os.unlink(ext_filename)
+            shutil.copyfile(mod, ext_filename)
+            shutil.copymode(mod, ext_filename)
+        
+    def get_sip_output_list(self, sbf, bdir):
+        """
+        Parse the sbf file specified to extract the name of the generated source
+        files. Make them absolute assuming they reside in the temp directory.
+        """
+        for L in file(sbf):
+            key, value = L.split("=", 1)
+            if key.strip() == "sources":
+                out = []
+                for o in value.split():
+                    out.append(os.path.join(bdir, o))
+                return out
+
+        raise RuntimeError, "cannot parse SIP-generated '%s'" % sbf
+    
+    def run_sip(self, sip_files):
+        sip_bin = self.sipcfg.sip_bin
+        sip_sources = [i[0] for i in sip_files]
+        generated_sources = []
+        for sip, sbf in sip_files:
+            if not (self.force or newer_group(sip_sources, sbf, 'newer')):
+                log.info(sbf + ' is up to date')
+                continue
+            self.spawn([sip_bin,
+                    "-c", self.build_temp,
+                    "-b", sbf,
+                    '-I', self.pyqtcfg.pyqt_sip_dir,
+                    ] + self.pyqtcfg.pyqt_sip_flags.split()+
+                    [sip])
+            generated_sources += self.get_sip_output_list(sbf)
+        return generated_sources 
+            
+        

setup.py

@@ -47,45 +47,25 @@ main_functions = {
 
 if __name__ == '__main__':
     from setuptools import setup, find_packages, Extension
+    from pyqtdistutils import PyQtExtension, build_ext
     import subprocess, glob
-    if 'mydevelop' in sys.argv:
-        LAUNCHER = '''\
-#!%(exe)s
-import sys
-sys.path.insert(0, %(ppath)s)
-sys.argv[0] = %(basename)s
-from %(module)s import %(func)s
-%(func)s()
-
-        '''
-        bindir = os.path.expanduser('~/bin')
-        def create_launcher(basename, module, func):
-            args = dict(exe=os.path.realpath(sys.executable),
-                            ppath=repr(os.path.abspath('src')),
-                            basename=repr(basename),
-                            module=module,
-                            func=func)
-            script = LAUNCHER%args
-            p = os.path.join(bindir, basename)
-            open(p, 'wb').write(script)
-            subprocess.check_call('chmod +x '+p, shell=True)
-        for x in ('console', 'gui'):
-            for i in range(len(basenames[x])):
-                create_launcher(basenames[x][i], main_modules[x][i], main_functions[x][i])
-        create_launcher('calibre_postinstall', 'calibre.linux', 'post_install')
-        subprocess.check_call('python setup.py build', shell=True)
-        subprocess.check_call('sudo %s/calibre_postinstall'%bindir, shell=True)
-    else:
     
     entry_points['console_scripts'].append('calibre_postinstall = calibre.linux:post_install')
-        ext_modules = [Extension('calibre.plugins.lzx',
+    ext_modules = [
+                   Extension('calibre.plugins.lzx',
                              sources=['src/calibre/utils/lzx/lzxmodule.c',
                                       'src/calibre/utils/lzx/lzxd.c'],
                              include_dirs=['src/calibre/utils/lzx']),
                    Extension('calibre.plugins.msdes',
                              sources=['src/calibre/utils/msdes/msdesmodule.c',
                                       'src/calibre/utils/msdes/des.c'],
-                                 include_dirs=['src/calibre/utils/msdes'])]
+                             include_dirs=['src/calibre/utils/msdes']),
+                    PyQtExtension('calibre.plugins.pictureflow',
+                                  ['src/calibre/gui2/pictureflow/pictureflow.cpp',
+                                   'src/calibre/gui2/pictureflow/pictureflow.h'],
+                                   ['src/calibre/gui2/pictureflow/pictureflow.sip']
+                                  )
+                 ]
     if iswindows:
         ext_modules.append(Extension('calibre.plugins.winutil',
                 sources=['src/calibre/utils/windows/winutil.c'],
@@ -97,42 +77,6 @@ from %(module)s import %(func)s
                 sources=['src/calibre/devices/usbobserver/usbobserver.c'])
                            )
     
-        def build_PyQt_extension(path):
-            pro      = glob.glob(os.path.join(path, '*.pro'))[0]
-            raw = open(pro).read()
-            base = qtplugin = re.search(r'TARGET\s*=\s*(.*)', raw).group(1)
-            ver  = re.search(r'VERSION\s*=\s*(\d+)', raw).group(1)
-            cwd = os.getcwd()
-            os.chdir(os.path.dirname(pro))
-            try:
-                if not os.path.exists('.build'):
-                    os.mkdir('.build')
-                os.chdir('.build')
-                subprocess.check_call(( (os.path.expanduser('~/qt/bin/qmake') if isosx else 'qmake'), '..'+os.sep+os.path.basename(pro)))
-                subprocess.check_call(['mingw32-make' if iswindows else 'make'])
-                os.chdir(os.path.join('..', 'PyQt'))
-                if not os.path.exists('.build'):
-                    os.mkdir('.build')
-                os.chdir('.build')
-                python = '/Library/Frameworks/Python.framework/Versions/Current/bin/python' if isosx else 'python'
-                subprocess.check_call([python, '..'+os.sep+'configure.py'])
-                subprocess.check_call(['mingw32-make' if iswindows else 'make'])
-                ext = '.pyd' if iswindows else '.so'
-                plugin = glob.glob(base+ext)[0]
-                shutil.copyfile(plugin, os.path.join(cwd, 'src', 'calibre', 'plugins', plugin))
-            finally:
-                os.chdir(cwd)
-                if islinux or isosx:
-                    for f in glob.glob(os.path.join('src', 'calibre', 'plugins', '*')):
-                        try:
-                            os.readlink(f)
-                            os.unlink(f)
-                        except:
-                            continue
-
-        for path in [(os.path.join('src', 'calibre', 'gui2', 'pictureflow'))]:
-            build_PyQt_extension(path)
-
     setup(
           name=APPNAME,
           packages = find_packages('src'),
@@ -180,7 +124,8 @@ from %(module)s import %(func)s
             'Programming Language :: Python',
             'Topic :: Software Development :: Libraries :: Python Modules',
             'Topic :: System :: Hardware :: Hardware Drivers'
-                ]
+            ],
+          cmdclass = {'build_ext': build_ext},
          )
 
     if 'develop' in ' '.join(sys.argv) and islinux:

src/calibre/gui2/pictureflow/PyQt/configure.py

@@ -1,52 +0,0 @@
-import os, sys, glob
-if os.environ.get('PYQT4PATH', None):
-    print os.environ['PYQT4PATH']
-    sys.path.insert(0, os.environ['PYQT4PATH'])
-from PyQt4 import pyqtconfig
-
-# The name of the SIP build file generated by SIP and used by the build
-# system.
-build_file = "pictureflow.sbf"
-
-# Get the PyQt configuration information.
-config = pyqtconfig.Configuration()
-
-# Run SIP to generate the code.  Note that we tell SIP where to find the qt
-# module's specification files using the -I flag.
-sip = [config.sip_bin, "-c", ".", "-b", build_file, "-I",
-       config.pyqt_sip_dir, config.pyqt_sip_flags, "../pictureflow.sip"]
-os.system(" ".join(sip))
-
-
-
-installs=[]
-
-# Create the Makefile.  The QtModuleMakefile class provided by the
-# pyqtconfig module takes care of all the extra preprocessor, compiler and
-# linker flags needed by the Qt library.
-makefile = pyqtconfig.QtGuiModuleMakefile (
-    configuration=config,
-    build_file=build_file,
-    installs=installs,
-    qt=1,
-)
-
-# Setup the platform dependent Makefile parameters
-d = os.path.dirname
-if 'darwin' in sys.platform:
-    makefile.extra_cflags += ['-arch i386', '-arch ppc']
-    makefile.extra_lflags += ['-arch i386', '-arch ppc']
-qtdir = os.path.join(d(d(os.getcwd())), '.build')
-if 'win32' in sys.platform:
-    qtdir = os.path.join(qtdir, 'release')
-    makefile.extra_lib_dirs += ['C:/Python25/libs']
-
-# Add the compiled Qt objects
-qtobjs = map(lambda x:'"'+x+'"', glob.glob(os.path.join(qtdir, '*.o')))
-makefile.extra_lflags += qtobjs
-makefile.extra_cxxflags = makefile.extra_cflags
-
-# Generate the Makefile itself.
-makefile.generate()
-
-

src/calibre/gui2/pictureflow/pictureflow.sip

@@ -9,7 +9,7 @@
 class FlowImages : QObject {
 
 %TypeHeaderCode
-#include "../../pictureflow.h"
+#include <pictureflow.h>
 %End
 
 public:
@@ -22,7 +22,7 @@ public:
 class PictureFlow : QWidget  {
 
 %TypeHeaderCode
-#include "../../pictureflow.h"
+#include <pictureflow.h>
 %End
 
 

src/calibre/utils/windows/winutil.c

@@ -49,7 +49,7 @@ wherever possible in this module.
     Get command line arguments as unicode objects. Note that the
     first argument will be the path to the interpreter, *not* the
     script being run. So to replace sys.argv, you should use
-    sys.argv[1:] = winutil.argv()[1:].
+    `if len(sys.argv) > 1: sys.argv[1:] = winutil.argv()[1-len(sys.argv):]`
 
 */
 

upload.py

@@ -28,9 +28,8 @@ MOBILEREAD = 'ftp://dev.mobileread.com/calibre/'
 BUILD_SCRIPT ='''\
 #!/bin/bash
 cd ~/build && \
-rsync -avz --exclude src/calibre/plugins --exclude docs --exclude .bzr --exclude .build --exclude build --exclude dist --exclude "*.pyc" --exclude "*.pyo" rsync://%(host)s/work/%(project)s . && \
+rsync -avz --exclude src/calibre/plugins --exclude calibre/src/calibre.egg-info --exclude docs --exclude .bzr --exclude .build --exclude build --exclude dist --exclude "*.pyc" --exclude "*.pyo" rsync://%(host)s/work/%(project)s . && \
-cd %(project)s && rm -rf build dist src/calibre/plugins && \
+cd %(project)s && \
-mkdir -p build dist src/calibre/plugins && \
 %%s && \
 rm -rf build/* && \
 %%s %%s
@@ -194,14 +193,12 @@ def upload_installers():
 
 
 def upload_docs():
-    os.environ['PYTHONPATH'] = os.path.abspath('src')
     check_call('''epydoc --config epydoc.conf''')
     check_call('''scp -r docs/html divok:%s/'''%(DOCS,))
     check_call('''epydoc -v --config epydoc-pdf.conf''')
     check_call('''scp docs/pdf/api.pdf divok:%s/'''%(DOCS,))
 
 def upload_user_manual():
-    os.environ['PYTHONPATH'] = os.path.abspath('src')
     cwd = os.getcwdu()
     os.chdir('src/calibre/manual')
     try:
@@ -223,7 +220,8 @@ def stage_one():
     os.mkdir('build')
     shutil.rmtree('docs')
     os.mkdir('docs')
-    check_call('python setup.py mydevelop', shell=True)
+    subprocess.call('python setup.py develop', shell=True)
+    check_call('sudo python setup.py develop', shell=True)
     check_call('make', shell=True)
     tag_release()
     upload_demo()