#!/usr/bin/env python # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai from __future__ import with_statement __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' import textwrap, os, shlex, subprocess, glob, shutil from distutils import sysconfig from multiprocessing import cpu_count from PyQt4.pyqtconfig import QtGuiModuleMakefile from setup import Command, islinux, isbsd, isosx, SRC, iswindows from setup.build_environment import (chmlib_inc_dirs, podofo_inc, podofo_lib, podofo_error, pyqt, OSX_SDK, NMAKE, QMAKE, msvc, MT, win_inc, win_lib, win_ddk, magick_inc_dirs, magick_lib_dirs, magick_libs, chmlib_lib_dirs, sqlite_inc_dirs, icu_inc_dirs, icu_lib_dirs, win_ddk_lib_dirs, ft_libs, ft_lib_dirs, ft_inc_dirs, zlib_libs, zlib_lib_dirs, zlib_inc_dirs, is64bit, qt_private_inc) MT isunix = islinux or isosx or isbsd make = 'make' if isunix else NMAKE class Extension(object): def absolutize(self, paths): return list(set([x if os.path.isabs(x) else os.path.join(SRC, x.replace('/', os.sep)) for x in paths])) def __init__(self, name, sources, **kwargs): self.name = name self.needs_cxx = bool([1 for x in sources if os.path.splitext(x)[1] in ('.cpp', '.c++', '.cxx')]) self.sources = self.absolutize(sources) self.headers = self.absolutize(kwargs.get('headers', [])) self.sip_files = self.absolutize(kwargs.get('sip_files', [])) self.inc_dirs = self.absolutize(kwargs.get('inc_dirs', [])) self.lib_dirs = self.absolutize(kwargs.get('lib_dirs', [])) self.extra_objs = self.absolutize(kwargs.get('extra_objs', [])) self.error = kwargs.get('error', None) self.libraries = kwargs.get('libraries', []) self.cflags = kwargs.get('cflags', []) self.ldflags = kwargs.get('ldflags', []) self.optional = kwargs.get('optional', False) self.needs_ddk = kwargs.get('needs_ddk', False) of = kwargs.get('optimize_level', None) if of is None: of = '/Ox' if iswindows else '-O3' else: flag = '/O%d' if iswindows else '-O%d' of = flag % of self.cflags.insert(0, of) def preflight(self, obj_dir, compiler, linker, builder, cflags, ldflags): pass reflow_sources = glob.glob(os.path.join(SRC, 'calibre', 'ebooks', 'pdf', '*.cpp')) reflow_headers = glob.glob(os.path.join(SRC, 'calibre', 'ebooks', 'pdf', '*.h')) icu_libs = ['icudata', 'icui18n', 'icuuc', 'icuio'] icu_cflags = [] if iswindows: icu_libs = ['icudt', 'icuin', 'icuuc', 'icuio'] if isosx: icu_libs = ['icucore'] icu_cflags = ['-DU_DISABLE_RENAMING'] # Needed to use system libicucore.dylib extensions = [ Extension('hunspell', ['hunspell/'+x for x in 'affentry.cxx affixmgr.cxx csutil.cxx dictmgr.cxx filemgr.cxx hashmgr.cxx hunspell.cxx phonet.cxx replist.cxx suggestmgr.cxx'.split() ] + ['calibre/utils/spell/hunspell_wrapper.cpp',], inc_dirs=['hunspell'], cflags='/DHUNSPELL_STATIC /D_CRT_SECURE_NO_WARNINGS /DUNICODE /D_UNICODE'.split() if iswindows else ['-DHUNSPELL_STATIC'], optimize_level=2, ), Extension('_regex', ['regex/_regex.c', 'regex/_regex_unicode.c'], headers=['regex/_regex.h'], optimize_level=2, ), Extension('speedup', ['calibre/utils/speedup.c'], ), Extension('_patiencediff_c', ['calibre/gui2/tweak_book/diff/_patiencediff_c.c'], ), Extension('icu', ['calibre/utils/icu.c'], headers=['calibre/utils/icu_calibre_utils.h'], libraries=icu_libs, lib_dirs=icu_lib_dirs, inc_dirs=icu_inc_dirs, cflags=icu_cflags ), Extension('sqlite_custom', ['calibre/library/sqlite_custom.c'], inc_dirs=sqlite_inc_dirs ), Extension('chmlib', ['calibre/utils/chm/swig_chm.c'], libraries=['ChmLib' if iswindows else 'chm'], inc_dirs=chmlib_inc_dirs, lib_dirs=chmlib_lib_dirs, cflags=["-DSWIG_COBJECT_TYPES"]), Extension('chm_extra', ['calibre/utils/chm/extra.c'], libraries=['ChmLib' if iswindows else 'chm'], inc_dirs=chmlib_inc_dirs, lib_dirs=chmlib_lib_dirs, cflags=["-D__PYTHON__"]), Extension('magick', ['calibre/utils/magick/magick.c'], headers=['calibre/utils/magick/magick_constants.h'], libraries=magick_libs, lib_dirs=magick_lib_dirs, inc_dirs=magick_inc_dirs, cflags=['-DMAGICKCORE_QUANTUM_DEPTH=16', '-DMAGICKCORE_HDRI_ENABLE=0'] ), Extension('lzx', ['calibre/utils/lzx/lzxmodule.c', 'calibre/utils/lzx/compressor.c', 'calibre/utils/lzx/lzxd.c', 'calibre/utils/lzx/lzc.c', 'calibre/utils/lzx/lzxc.c'], headers=['calibre/utils/lzx/msstdint.h', 'calibre/utils/lzx/lzc.h', 'calibre/utils/lzx/lzxmodule.h', 'calibre/utils/lzx/system.h', 'calibre/utils/lzx/lzxc.h', 'calibre/utils/lzx/lzxd.h', 'calibre/utils/lzx/mspack.h'], inc_dirs=['calibre/utils/lzx']), Extension('freetype', ['calibre/utils/fonts/freetype.cpp'], inc_dirs=ft_inc_dirs, libraries=ft_libs, lib_dirs=ft_lib_dirs), Extension('woff', ['calibre/utils/fonts/woff/main.c', 'calibre/utils/fonts/woff/woff.c'], headers=[ 'calibre/utils/fonts/woff/woff.h', 'calibre/utils/fonts/woff/woff-private.h'], libraries=zlib_libs, lib_dirs=zlib_lib_dirs, inc_dirs=zlib_inc_dirs, ), Extension('msdes', ['calibre/utils/msdes/msdesmodule.c', 'calibre/utils/msdes/des.c'], headers=['calibre/utils/msdes/spr.h', 'calibre/utils/msdes/d3des.h'], inc_dirs=['calibre/utils/msdes']), Extension('cPalmdoc', ['calibre/ebooks/compression/palmdoc.c']), Extension('bzzdec', ['calibre/ebooks/djvu/bzzdecoder.c'], inc_dirs=(['calibre/utils/chm'] if iswindows else []) # For stdint.h ), Extension('matcher', ['calibre/utils/matcher.c'], headers=['calibre/utils/icu_calibre_utils.h'], libraries=icu_libs, lib_dirs=icu_lib_dirs, cflags=icu_cflags, inc_dirs=icu_inc_dirs ), Extension('podofo', [ 'calibre/utils/podofo/utils.cpp', 'calibre/utils/podofo/output.cpp', 'calibre/utils/podofo/doc.cpp', 'calibre/utils/podofo/outline.cpp', 'calibre/utils/podofo/podofo.cpp', ], headers=[ 'calibre/utils/podofo/global.h', ], libraries=['podofo'], lib_dirs=[podofo_lib], inc_dirs=[podofo_inc, os.path.dirname(podofo_inc)], error=podofo_error), Extension('pictureflow', ['calibre/gui2/pictureflow/pictureflow.cpp'], inc_dirs=['calibre/gui2/pictureflow'], headers=['calibre/gui2/pictureflow/pictureflow.h'], sip_files=['calibre/gui2/pictureflow/pictureflow.sip'] ), Extension('progress_indicator', ['calibre/gui2/progress_indicator/QProgressIndicator.cpp'], inc_dirs=['calibre/gui2/progress_indicator'], headers=['calibre/gui2/progress_indicator/QProgressIndicator.h'], sip_files=['calibre/gui2/progress_indicator/QProgressIndicator.sip'] ), Extension('qt_hack', ['calibre/ebooks/pdf/render/qt_hack.cpp'], inc_dirs=qt_private_inc + ['calibre/ebooks/pdf/render', 'qt-harfbuzz/src'], headers=['calibre/ebooks/pdf/render/qt_hack.h'], sip_files=['calibre/ebooks/pdf/render/qt_hack.sip'] ), Extension('unrar', ['unrar/%s.cpp'%(x.partition('.')[0]) for x in ''' rar.o strlist.o strfn.o pathfn.o savepos.o smallfn.o global.o file.o filefn.o filcreat.o archive.o arcread.o unicode.o system.o isnt.o crypt.o crc.o rawread.o encname.o resource.o match.o timefn.o rdwrfn.o consio.o options.o ulinks.o errhnd.o rarvm.o secpassword.o rijndael.o getbits.o sha1.o extinfo.o extract.o volume.o list.o find.o unpack.o cmddata.o filestr.o scantree.o '''.split()] + ['calibre/utils/unrar.cpp'], inc_dirs=['unrar'], cflags=[('/' if iswindows else '-') + x for x in ( 'DSILENT', 'DRARDLL', 'DUNRAR')] + ( [] if iswindows else ['-D_FILE_OFFSET_BITS=64', '-D_LARGEFILE_SOURCE']), optimize_level=2, libraries=['User32', 'Advapi32', 'kernel32', 'Shell32'] if iswindows else [] ), ] if iswindows: extensions.extend([ Extension('winutil', ['calibre/utils/windows/winutil.c'], libraries=['shell32', 'setupapi', 'wininet'], cflags=['/X'] ), Extension('wpd', [ 'calibre/devices/mtp/windows/utils.cpp', 'calibre/devices/mtp/windows/device_enumeration.cpp', 'calibre/devices/mtp/windows/content_enumeration.cpp', 'calibre/devices/mtp/windows/device.cpp', 'calibre/devices/mtp/windows/wpd.cpp', ], headers=[ 'calibre/devices/mtp/windows/global.h', ], libraries=['ole32', 'oleaut32', 'portabledeviceguids', 'user32'], # needs_ddk=True, cflags=['/X'] ), Extension('winfonts', ['calibre/utils/fonts/winfonts.cpp'], libraries=['Gdi32', 'User32'], cflags=['/X'] ), ]) if isosx: extensions.append(Extension('usbobserver', ['calibre/devices/usbobserver/usbobserver.c'], ldflags=['-framework', 'CoreServices', '-framework', 'IOKit']) ) if islinux or isosx: extensions.append(Extension('libusb', ['calibre/devices/libusb/libusb.c'], libraries=['usb-1.0'] )) extensions.append(Extension('libmtp', [ 'calibre/devices/mtp/unix/devices.c', 'calibre/devices/mtp/unix/libmtp.c' ], headers=[ 'calibre/devices/mtp/unix/devices.h', 'calibre/devices/mtp/unix/upstream/music-players.h', 'calibre/devices/mtp/unix/upstream/device-flags.h', ], libraries=['mtp'] )) if isunix: cc = os.environ.get('CC', 'gcc') cxx = os.environ.get('CXX', 'g++') debug = '' # debug = '-ggdb' cflags = os.environ.get('OVERRIDE_CFLAGS', '-Wall -DNDEBUG %s -fno-strict-aliasing -pipe' % debug) cflags = shlex.split(cflags) + ['-fPIC'] ldflags = os.environ.get('OVERRIDE_LDFLAGS', '-Wall') ldflags = shlex.split(ldflags) cflags += shlex.split(os.environ.get('CFLAGS', '')) ldflags += shlex.split(os.environ.get('LDFLAGS', '')) if islinux: cflags.append('-pthread') ldflags.append('-shared') cflags.append('-I'+sysconfig.get_python_inc()) ldflags.append('-lpython'+sysconfig.get_python_version()) if isbsd: cflags.append('-pthread') ldflags.append('-shared') cflags.append('-I'+sysconfig.get_python_inc()) ldflags.append('-lpython'+sysconfig.get_python_version()) if isosx: x, p = ('i386', 'x86_64') archs = ['-arch', x, '-arch', p, '-isysroot', OSX_SDK] cflags.append('-D_OSX') cflags.extend(archs) ldflags.extend(archs) ldflags.extend('-bundle -undefined dynamic_lookup'.split()) cflags.extend(['-fno-common', '-dynamic']) cflags.append('-I'+sysconfig.get_python_inc()) if iswindows: cc = cxx = msvc.cc cflags = '/c /nologo /MD /W3 /EHsc /DNDEBUG'.split() ldflags = '/DLL /nologo /INCREMENTAL:NO /NODEFAULTLIB:libcmt.lib'.split() #cflags = '/c /nologo /Ox /MD /W3 /EHsc /Zi'.split() #ldflags = '/DLL /nologo /INCREMENTAL:NO /DEBUG'.split() if is64bit: cflags.append('/GS-') for p in win_inc: cflags.append('-I'+p) for p in win_lib: ldflags.append('/LIBPATH:'+p) cflags.append('-I%s'%sysconfig.get_python_inc()) ldflags.append('/LIBPATH:'+os.path.join(sysconfig.PREFIX, 'libs')) class Build(Command): short_description = 'Build calibre C/C++ extension modules' description = textwrap.dedent('''\ calibre depends on several python extensions written in C/C++. This command will compile them. You can influence the compile process by several environment variables, listed below: CC - C Compiler defaults to gcc CXX - C++ Compiler, defaults to g++ CFLAGS - Extra compiler flags LDFLAGS - Extra linker flags POPPLER_INC_DIR - poppler header files POPPLER_LIB_DIR - poppler-qt4 library PODOFO_INC_DIR - podofo header files PODOFO_LIB_DIR - podofo library files QMAKE - Path to qmake VS90COMNTOOLS - Location of Microsoft Visual Studio 9 Tools (windows only) ''') def add_options(self, parser): choices = [e.name for e in extensions]+['all', 'style'] parser.add_option('-1', '--only', choices=choices, default='all', help=('Build only the named extension. Available: '+ ', '.join(choices)+'. Default:%default')) parser.add_option('--no-compile', default=False, action='store_true', help='Skip compiling all C/C++ extensions.') def run(self, opts): if opts.no_compile: self.info('--no-compile specified, skipping compilation') return self.obj_dir = os.path.join(os.path.dirname(SRC), 'build', 'objects') if not os.path.exists(self.obj_dir): os.makedirs(self.obj_dir) if opts.only in {'all', 'style'}: self.build_style(self.j(self.SRC, 'calibre', 'plugins')) for ext in extensions: if opts.only != 'all' and opts.only != ext.name: continue if ext.error is not None: if ext.optional: self.warn(ext.error) continue else: raise Exception(ext.error) dest = self.dest(ext) if not os.path.exists(self.d(dest)): os.makedirs(self.d(dest)) self.info('\n####### Building extension', ext.name, '#'*7) self.build(ext, dest) def dest(self, ext): ex = '.pyd' if iswindows else '.so' return os.path.join(SRC, 'calibre', 'plugins', ext.name)+ex def inc_dirs_to_cflags(self, dirs): return ['-I'+x for x in dirs] def lib_dirs_to_ldflags(self, dirs): pref = '/LIBPATH:' if iswindows else '-L' return [pref+x for x in dirs] def libraries_to_ldflags(self, dirs): pref = '' if iswindows else '-l' suff = '.lib' if iswindows else '' return [pref+x+suff for x in dirs] def build(self, ext, dest): if ext.sip_files: return self.build_pyqt_extension(ext, dest) compiler = cxx if ext.needs_cxx else cc linker = msvc.linker if iswindows else compiler objects = [] obj_dir = self.j(self.obj_dir, ext.name) ext.preflight(obj_dir, compiler, linker, self, cflags, ldflags) einc = self.inc_dirs_to_cflags(ext.inc_dirs) if ext.needs_ddk: ddk_flags = ['-I'+x for x in win_ddk] cflags.extend(ddk_flags) ldflags.extend(['/LIBPATH:'+x for x in win_ddk_lib_dirs]) if not os.path.exists(obj_dir): os.makedirs(obj_dir) for src in ext.sources: obj = self.j(obj_dir, os.path.splitext(self.b(src))[0]+'.o') objects.append(obj) if self.newer(obj, [src]+ext.headers): inf = '/Tp' if src.endswith('.cpp') or src.endswith('.cxx') else '/Tc' sinc = [inf+src] if iswindows else ['-c', src] oinc = ['/Fo'+obj] if iswindows else ['-o', obj] cmd = [compiler] + cflags + ext.cflags + einc + sinc + oinc self.info(' '.join(cmd)) self.check_call(cmd) dest = self.dest(ext) elib = self.lib_dirs_to_ldflags(ext.lib_dirs) xlib = self.libraries_to_ldflags(ext.libraries) if self.newer(dest, objects+ext.extra_objs): print 'Linking', ext.name cmd = [linker] if iswindows: cmd += ldflags + ext.ldflags + elib + xlib + \ ['/EXPORT:init'+ext.name] + objects + ext.extra_objs + ['/OUT:'+dest] else: cmd += objects + ext.extra_objs + ['-o', dest] + ldflags + ext.ldflags + elib + xlib self.info('\n\n', ' '.join(cmd), '\n\n') self.check_call(cmd) if iswindows: #manifest = dest+'.manifest' #cmd = [MT, '-manifest', manifest, '-outputresource:%s;2'%dest] # self.info(*cmd) # self.check_call(cmd) # os.remove(manifest) for x in ('.exp', '.lib'): x = os.path.splitext(dest)[0]+x if os.path.exists(x): os.remove(x) def check_call(self, *args, **kwargs): """print cmdline if an error occured If something is missing (qmake e.g.) you get a non-informative error self.check_call(qmc + [ext.name+'.pro']) so you would have to look a the source to see the actual command. """ try: subprocess.check_call(*args, **kwargs) except: cmdline = ' '.join(['"%s"' % (arg) if ' ' in arg else arg for arg in args[0]]) print "Error while executing: %s\n" % (cmdline) raise def build_style(self, dest): self.info('\n####### Building calibre style', '#'*7) sdir = self.j(self.SRC, 'qtcurve') def path(x): x=self.j(sdir, x) return ('"%s"'%x).replace(os.sep, '/') headers = [ "common/colorutils.h", "common/common.h", "common/config_file.h", "style/blurhelper.h", "style/fixx11h.h", "style/pixmaps.h", "style/qtcurve.h", "style/shortcuthandler.h", "style/utils.h", "style/windowmanager.h", ] sources = [ "common/colorutils.c", "common/common.c", "common/config_file.c", "style/blurhelper.cpp", "style/qtcurve.cpp", "style/shortcuthandler.cpp", "style/utils.cpp", "style/windowmanager.cpp", ] if not iswindows and not isosx: headers.append("style/shadowhelper.h") sources.append('style/shadowhelper.cpp') pro = textwrap.dedent(''' TEMPLATE = lib CONFIG += qt plugin release CONFIG -= embed_manifest_dll VERSION = 1.0.0 DESTDIR = . TARGET = calibre QT *= svg INCLUDEPATH *= {conf} {inc} win32-msvc*:DEFINES *= _CRT_SECURE_NO_WARNINGS # Force C++ language *g++*:QMAKE_CFLAGS *= -x c++ *msvc*:QMAKE_CFLAGS *= -TP *msvc*:QMAKE_CXXFLAGS += /MP ''').format(conf=path(''), inc=path('common')) if isosx: pro += '\nCONFIG += x86 x86_64\n' else: pro += '\nunix:QT *= dbus\n' for x in headers: pro += 'HEADERS += %s\n'%path(x) for x in sources: pro += 'SOURCES += %s\n'%path(x) odir = self.j(self.d(self.SRC), 'build', 'qtcurve') if not os.path.exists(odir): os.makedirs(odir) ocwd = os.getcwdu() os.chdir(odir) try: if not os.path.exists('qtcurve.pro') or (open('qtcurve.pro', 'rb').read() != pro): with open('qtcurve.pro', 'wb') as f: f.write(pro) qmc = [QMAKE, '-o', 'Makefile'] if iswindows: qmc += ['-spec', 'win32-msvc2008'] self.check_call(qmc + ['qtcurve.pro']) self.check_call([make]+([] if iswindows else ['-j%d'%(cpu_count() or 1)])) src = (glob.glob('*.so') + glob.glob('release/*.dll') + glob.glob('*.dylib')) ext = 'pyd' if iswindows else 'so' if not os.path.exists(dest): os.makedirs(dest) shutil.copy2(src[0], self.j(dest, 'calibre_style.'+ext)) finally: os.chdir(ocwd) def build_qt_objects(self, ext): obj_pat = 'release\\*.obj' if iswindows else '*.o' objects = glob.glob(obj_pat) if not objects or self.newer(objects, ext.sources+ext.headers): archs = 'x86 x86_64' pro = textwrap.dedent('''\ TARGET = %s TEMPLATE = lib HEADERS = %s SOURCES = %s VERSION = 1.0.0 CONFIG += %s ''')%(ext.name, ' '.join(ext.headers), ' '.join(ext.sources), archs) if ext.inc_dirs: idir = ' '.join(ext.inc_dirs) pro += 'INCLUDEPATH = %s\n'%idir pro = pro.replace('\\', '\\\\') open(ext.name+'.pro', 'wb').write(pro) qmc = [QMAKE, '-o', 'Makefile'] if iswindows: qmc += ['-spec', 'win32-msvc2008'] self.check_call(qmc + [ext.name+'.pro']) self.check_call([make, '-f', 'Makefile']) objects = glob.glob(obj_pat) return list(map(self.a, objects)) def build_pyqt_extension(self, ext, dest): pyqt_dir = self.j(self.d(self.SRC), 'build', 'pyqt') src_dir = self.j(pyqt_dir, ext.name) qt_dir = self.j(src_dir, 'qt') if not self.e(qt_dir): os.makedirs(qt_dir) cwd = os.getcwd() try: os.chdir(qt_dir) qt_objects = self.build_qt_objects(ext) finally: os.chdir(cwd) sip_files = ext.sip_files ext.sip_files = [] sipf = sip_files[0] sbf = self.j(src_dir, self.b(sipf)+'.sbf') if self.newer(sbf, [sipf]+ext.headers): exe = '.exe' if iswindows else '' cmd = [pyqt.sip_bin+exe, '-w', '-c', src_dir, '-b', sbf, '-I'+ pyqt.pyqt_sip_dir] + shlex.split(pyqt.pyqt_sip_flags) + [sipf] self.info(' '.join(cmd)) self.check_call(cmd) module = self.j(src_dir, self.b(dest)) if self.newer(dest, [sbf]+qt_objects): mf = self.j(src_dir, 'Makefile') makefile = QtGuiModuleMakefile(configuration=pyqt, build_file=sbf, makefile=mf, universal=OSX_SDK, qt=1) makefile.extra_lflags = qt_objects makefile.extra_include_dirs = ext.inc_dirs makefile.generate() self.check_call([make, '-f', mf], cwd=src_dir) shutil.copy2(module, dest) def clean(self): for ext in extensions: dest = self.dest(ext) for x in (dest, dest+'.manifest'): if os.path.exists(x): os.remove(x) build_dir = self.j(self.d(self.SRC), 'build') if os.path.exists(build_dir): shutil.rmtree(build_dir)