mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-11-03 19:17:02 -05:00
483 lines
19 KiB
Python
483 lines
19 KiB
Python
#!/usr/bin/env python2
|
|
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
|
|
from __future__ import with_statement
|
|
|
|
__license__ = 'GPL v3'
|
|
__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
|
|
__docformat__ = 'restructuredtext en'
|
|
|
|
import textwrap, os, shlex, subprocess, glob, shutil, re, sys, json
|
|
from collections import namedtuple
|
|
|
|
from setup import Command, islinux, isbsd, isosx, SRC, iswindows, __version__
|
|
isunix = islinux or isosx or isbsd
|
|
|
|
py_lib = os.path.join(sys.prefix, 'libs', 'python%d%d.lib' % sys.version_info[:2])
|
|
|
|
def absolutize(paths):
|
|
return list(set([x if os.path.isabs(x) else os.path.join(SRC, x.replace('/', os.sep)) for x in paths]))
|
|
|
|
class Extension(object):
|
|
|
|
def __init__(self, name, sources, **kwargs):
|
|
self.data = d = {}
|
|
self.name = d['name'] = name
|
|
self.sources = d['sources'] = absolutize(sources)
|
|
self.needs_cxx = d['needs_cxx'] = bool([1 for x in self.sources if os.path.splitext(x)[1] in ('.cpp', '.c++', '.cxx')])
|
|
self.headers = d['headers'] = absolutize(kwargs.get('headers', []))
|
|
self.sip_files = d['sip_files'] = absolutize(kwargs.get('sip_files', []))
|
|
self.inc_dirs = d['inc_dirs'] = absolutize(kwargs.get('inc_dirs', []))
|
|
self.lib_dirs = d['lib_dirs'] = absolutize(kwargs.get('lib_dirs', []))
|
|
self.extra_objs = d['extra_objs'] = absolutize(kwargs.get('extra_objs', []))
|
|
self.error = d['error'] = kwargs.get('error', None)
|
|
self.libraries = d['libraries'] = kwargs.get('libraries', [])
|
|
self.cflags = d['cflags'] = kwargs.get('cflags', [])
|
|
self.ldflags = d['ldflags'] = kwargs.get('ldflags', [])
|
|
self.optional = d['options'] = kwargs.get('optional', False)
|
|
of = kwargs.get('optimize_level', None)
|
|
if of is None:
|
|
of = '/Ox' if iswindows else '-O3'
|
|
else:
|
|
flag = '/O%d' if iswindows else '-O%d'
|
|
of = flag % of
|
|
self.cflags.insert(0, of)
|
|
self.qt_private_headers = d['qt_private_headers'] = kwargs.get('qt_private', [])
|
|
|
|
|
|
def lazy_load(name):
|
|
if name.startswith('!'):
|
|
name = name[1:]
|
|
from setup import build_environment
|
|
try:
|
|
return getattr(build_environment, name)
|
|
except AttributeError:
|
|
raise ImportError('The setup.build_environment module has no symbol named: %s' % name)
|
|
|
|
|
|
def expand_file_list(items, is_paths=True):
|
|
if not items:
|
|
return []
|
|
ans = []
|
|
for item in items:
|
|
if item.startswith('!'):
|
|
item = lazy_load(item)
|
|
if isinstance(item, basestring):
|
|
item = [item]
|
|
ans.extend(expand_file_list(item, is_paths=is_paths))
|
|
else:
|
|
if '*' in item:
|
|
ans.extend(expand_file_list(glob.glob(os.path.join(SRC, item)), is_paths=is_paths))
|
|
else:
|
|
item = [item]
|
|
if is_paths:
|
|
item = absolutize(item)
|
|
ans.extend(item)
|
|
return ans
|
|
|
|
|
|
def is_ext_allowed(ext):
|
|
only = ext.get('only', '')
|
|
if only:
|
|
only = only.split()
|
|
q = 'windows' if iswindows else 'osx' if isosx else 'bsd' if isbsd else 'linux'
|
|
return q in only
|
|
return True
|
|
|
|
def parse_extension(ext):
|
|
ext = ext.copy()
|
|
ext.pop('only', None)
|
|
kw = {}
|
|
name = ext.pop('name')
|
|
def get(k, default=''):
|
|
ans = ext.pop(k, default)
|
|
if iswindows:
|
|
ans = ext.pop('windows_' + k, ans)
|
|
elif isosx:
|
|
ans = ext.pop('osx_' + k, ans)
|
|
elif isbsd:
|
|
ans = ext.pop('bsd_' + k, ans)
|
|
else:
|
|
ans = ext.pop('linux_' + k, ans)
|
|
return ans
|
|
for k in 'libraries qt_private ldflags cflags error'.split():
|
|
kw[k] = expand_file_list(get(k).split(), is_paths=False)
|
|
defines = get('defines')
|
|
if defines:
|
|
if 'cflags' not in kw:
|
|
kw['cflags'] = []
|
|
cflags = kw['cflags']
|
|
prefix = '/D' if iswindows else '-D'
|
|
cflags.extend(prefix + x for x in defines.split())
|
|
for k in 'inc_dirs lib_dirs sources headers sip_files'.split():
|
|
v = get(k)
|
|
if v:
|
|
kw[k] = expand_file_list(v.split())
|
|
kw.update(ext)
|
|
return Extension(name, **kw)
|
|
|
|
|
|
def read_extensions():
|
|
if hasattr(read_extensions, 'extensions'):
|
|
return read_extensions.extensions
|
|
ans = read_extensions.extensions = json.load(open(os.path.dirname(os.path.abspath(__file__)) + '/extensions.json', 'rb'))
|
|
return ans
|
|
|
|
|
|
def init_env():
|
|
from setup.build_environment import msvc, is64bit, win_inc, win_lib, NMAKE
|
|
from distutils import sysconfig
|
|
linker = None
|
|
if isunix:
|
|
cc = os.environ.get('CC', 'gcc')
|
|
cxx = os.environ.get('CXX', 'g++')
|
|
debug = ''
|
|
# debug = '-ggdb'
|
|
cflags = os.environ.get('OVERRIDE_CFLAGS',
|
|
'-Wall -DNDEBUG %s -fno-strict-aliasing -pipe' % debug)
|
|
cflags = shlex.split(cflags) + ['-fPIC']
|
|
ldflags = os.environ.get('OVERRIDE_LDFLAGS', '-Wall')
|
|
ldflags = shlex.split(ldflags)
|
|
cflags += shlex.split(os.environ.get('CFLAGS', ''))
|
|
ldflags += shlex.split(os.environ.get('LDFLAGS', ''))
|
|
|
|
if islinux:
|
|
cflags.append('-pthread')
|
|
ldflags.append('-shared')
|
|
cflags.append('-I'+sysconfig.get_python_inc())
|
|
ldflags.append('-lpython'+sysconfig.get_python_version())
|
|
|
|
if isbsd:
|
|
cflags.append('-pthread')
|
|
ldflags.append('-shared')
|
|
cflags.append('-I'+sysconfig.get_python_inc())
|
|
ldflags.append('-lpython'+sysconfig.get_python_version())
|
|
|
|
if isosx:
|
|
cflags.append('-D_OSX')
|
|
ldflags.extend('-bundle -undefined dynamic_lookup'.split())
|
|
cflags.extend(['-fno-common', '-dynamic'])
|
|
cflags.append('-I'+sysconfig.get_python_inc())
|
|
|
|
if iswindows:
|
|
cc = cxx = msvc.cc
|
|
cflags = '/c /nologo /MD /W3 /EHsc /DNDEBUG'.split()
|
|
ldflags = '/DLL /nologo /INCREMENTAL:NO /NODEFAULTLIB:libcmt.lib'.split()
|
|
# cflags = '/c /nologo /Ox /MD /W3 /EHsc /Zi'.split()
|
|
# ldflags = '/DLL /nologo /INCREMENTAL:NO /DEBUG'.split()
|
|
if is64bit:
|
|
cflags.append('/GS-')
|
|
|
|
for p in win_inc:
|
|
cflags.append('-I'+p)
|
|
for p in win_lib:
|
|
ldflags.append('/LIBPATH:'+p)
|
|
cflags.append('-I%s'%sysconfig.get_python_inc())
|
|
ldflags.append('/LIBPATH:'+os.path.join(sysconfig.PREFIX, 'libs'))
|
|
linker = msvc.linker
|
|
return namedtuple('Environment', 'cc cxx cflags ldflags linker make')(
|
|
cc=cc, cxx=cxx, cflags=cflags, ldflags=ldflags, linker=linker, make=NMAKE if iswindows else 'make')
|
|
|
|
|
|
class Build(Command):
|
|
|
|
short_description = 'Build calibre C/C++ extension modules'
|
|
DEFAULT_OUTPUTDIR = os.path.abspath(os.path.join(SRC, 'calibre', 'plugins'))
|
|
DEFAULT_BUILDDIR = os.path.abspath(os.path.join(os.path.dirname(SRC), 'build'))
|
|
|
|
description = textwrap.dedent('''\
|
|
calibre depends on several python extensions written in C/C++.
|
|
This command will compile them. You can influence the compile
|
|
process by several environment variables, listed below:
|
|
|
|
CC - C Compiler defaults to gcc
|
|
CXX - C++ Compiler, defaults to g++
|
|
CFLAGS - Extra compiler flags
|
|
LDFLAGS - Extra linker flags
|
|
|
|
POPPLER_INC_DIR - poppler header files
|
|
POPPLER_LIB_DIR - poppler-qt4 library
|
|
|
|
PODOFO_INC_DIR - podofo header files
|
|
PODOFO_LIB_DIR - podofo library files
|
|
|
|
QMAKE - Path to qmake
|
|
SIP_BIN - Path to the sip binary
|
|
VS90COMNTOOLS - Location of Microsoft Visual Studio 9 Tools (windows only)
|
|
|
|
''')
|
|
|
|
def add_options(self, parser):
|
|
choices = [e['name'] for e in read_extensions() if is_ext_allowed(e)]+['all', 'headless']
|
|
parser.add_option('-1', '--only', choices=choices, default='all',
|
|
help=('Build only the named extension. Available: '+
|
|
', '.join(choices)+'. Default:%default'))
|
|
parser.add_option('--no-compile', default=False, action='store_true',
|
|
help='Skip compiling all C/C++ extensions.')
|
|
parser.add_option('--build-dir', default=None,
|
|
help='Path to directory in which to place object files during the build process, defaults to "build"')
|
|
parser.add_option('--output-dir', default=None,
|
|
help='Path to directory in which to place the built extensions. Defaults to src/calibre/plugins')
|
|
|
|
def run(self, opts):
|
|
if opts.no_compile:
|
|
self.info('--no-compile specified, skipping compilation')
|
|
return
|
|
self.env = init_env()
|
|
extensions = map(parse_extension, filter(is_ext_allowed, read_extensions()))
|
|
self.build_dir = os.path.abspath(opts.build_dir or self.DEFAULT_BUILDDIR)
|
|
self.output_dir = os.path.abspath(opts.output_dir or self.DEFAULT_OUTPUTDIR)
|
|
self.obj_dir = os.path.join(self.build_dir, 'objects')
|
|
for x in (self.output_dir, self.obj_dir):
|
|
if not os.path.exists(x):
|
|
os.makedirs(x)
|
|
for ext in extensions:
|
|
if opts.only != 'all' and opts.only != ext.name:
|
|
continue
|
|
if ext.error:
|
|
if ext.optional:
|
|
self.warn(ext.error)
|
|
continue
|
|
else:
|
|
raise Exception(ext.error)
|
|
dest = self.dest(ext)
|
|
if not os.path.exists(self.d(dest)):
|
|
os.makedirs(self.d(dest))
|
|
self.info('\n####### Building extension', ext.name, '#'*7)
|
|
self.build(ext, dest)
|
|
if opts.only in {'all', 'headless'}:
|
|
self.build_headless()
|
|
|
|
def dest(self, ext):
|
|
ex = '.pyd' if iswindows else '.so'
|
|
return os.path.join(self.output_dir, getattr(ext, 'name', ext))+ex
|
|
|
|
def inc_dirs_to_cflags(self, dirs):
|
|
return ['-I'+x for x in dirs]
|
|
|
|
def lib_dirs_to_ldflags(self, dirs):
|
|
pref = '/LIBPATH:' if iswindows else '-L'
|
|
return [pref+x for x in dirs]
|
|
|
|
def libraries_to_ldflags(self, dirs):
|
|
pref = '' if iswindows else '-l'
|
|
suff = '.lib' if iswindows else ''
|
|
return [pref+x+suff for x in dirs]
|
|
|
|
def build(self, ext, dest):
|
|
from setup.parallel_build import create_job, parallel_build
|
|
if ext.sip_files:
|
|
return self.build_pyqt_extension(ext, dest)
|
|
compiler = self.env.cxx if ext.needs_cxx else self.env.cc
|
|
linker = self.env.linker if iswindows else compiler
|
|
objects = []
|
|
obj_dir = self.j(self.obj_dir, ext.name)
|
|
einc = self.inc_dirs_to_cflags(ext.inc_dirs)
|
|
if not os.path.exists(obj_dir):
|
|
os.makedirs(obj_dir)
|
|
|
|
jobs = []
|
|
for src in ext.sources:
|
|
obj = self.j(obj_dir, os.path.splitext(self.b(src))[0]+'.o')
|
|
objects.append(obj)
|
|
if self.newer(obj, [src]+ext.headers):
|
|
inf = '/Tp' if src.endswith('.cpp') or src.endswith('.cxx') else '/Tc'
|
|
sinc = [inf+src] if iswindows else ['-c', src]
|
|
oinc = ['/Fo'+obj] if iswindows else ['-o', obj]
|
|
cmd = [compiler] + self.env.cflags + ext.cflags + einc + sinc + oinc
|
|
jobs.append(create_job(cmd))
|
|
if jobs:
|
|
self.info('Compiling', ext.name)
|
|
if not parallel_build(jobs, self.info):
|
|
raise SystemExit(1)
|
|
|
|
dest = self.dest(ext)
|
|
elib = self.lib_dirs_to_ldflags(ext.lib_dirs)
|
|
xlib = self.libraries_to_ldflags(ext.libraries)
|
|
if self.newer(dest, objects+ext.extra_objs):
|
|
self.info('Linking', ext.name)
|
|
cmd = [linker]
|
|
if iswindows:
|
|
cmd += self.env.ldflags + ext.ldflags + elib + xlib + \
|
|
['/EXPORT:init'+ext.name] + objects + ext.extra_objs + ['/OUT:'+dest]
|
|
else:
|
|
cmd += objects + ext.extra_objs + ['-o', dest] + self.env.ldflags + ext.ldflags + elib + xlib
|
|
self.info('\n\n', ' '.join(cmd), '\n\n')
|
|
self.check_call(cmd)
|
|
if iswindows:
|
|
for x in ('.exp', '.lib'):
|
|
x = os.path.splitext(dest)[0]+x
|
|
if os.path.exists(x):
|
|
os.remove(x)
|
|
|
|
def check_call(self, *args, **kwargs):
|
|
"""print cmdline if an error occured
|
|
|
|
If something is missing (qmake e.g.) you get a non-informative error
|
|
self.check_call(qmc + [ext.name+'.pro'])
|
|
so you would have to look a the source to see the actual command.
|
|
"""
|
|
try:
|
|
subprocess.check_call(*args, **kwargs)
|
|
except:
|
|
cmdline = ' '.join(['"%s"' % (arg) if ' ' in arg else arg for arg in args[0]])
|
|
print "Error while executing: %s\n" % (cmdline)
|
|
raise
|
|
|
|
def build_headless(self):
|
|
from setup.parallel_build import cpu_count
|
|
if iswindows or isosx:
|
|
return # Dont have headless operation on these platforms
|
|
from setup.build_environment import glib_flags, fontconfig_flags, ft_inc_dirs, QMAKE
|
|
from PyQt5.QtCore import QT_VERSION
|
|
self.info('\n####### Building headless QPA plugin', '#'*7)
|
|
a = absolutize
|
|
headers = a([
|
|
'calibre/headless/headless_backingstore.h',
|
|
'calibre/headless/headless_integration.h',
|
|
])
|
|
sources = a([
|
|
'calibre/headless/main.cpp',
|
|
'calibre/headless/headless_backingstore.cpp',
|
|
'calibre/headless/headless_integration.cpp',
|
|
])
|
|
if QT_VERSION >= 0x50401:
|
|
headers.extend(a(['calibre/headless/fontconfig_database.h']))
|
|
sources.extend(a(['calibre/headless/fontconfig_database.cpp']))
|
|
others = a(['calibre/headless/headless.json'])
|
|
target = self.dest('headless')
|
|
if not self.newer(target, headers + sources + others):
|
|
return
|
|
# Arch and possibly other distros (fedora?) monkey patches qmake as a
|
|
# result of which it fails to add glib-2.0 and freetype2 to the list of
|
|
# library dependencies. Compiling QPA plugins uses the static
|
|
# libQt5PlatformSupport.a which needs glib to be specified after it for
|
|
# linking to succeed, so we add it to QMAKE_LIBS_PRIVATE (we cannot use
|
|
# LIBS as that would put -lglib-2.0 before libQt5PlatformSupport. See
|
|
# https://bugs.archlinux.org/task/38819
|
|
|
|
pro = textwrap.dedent(
|
|
'''\
|
|
TARGET = headless
|
|
PLUGIN_TYPE = platforms
|
|
PLUGIN_CLASS_NAME = HeadlessIntegrationPlugin
|
|
load(qt_plugin)
|
|
QT += core-private gui-private platformsupport-private
|
|
HEADERS = {headers}
|
|
SOURCES = {sources}
|
|
OTHER_FILES = {others}
|
|
INCLUDEPATH += {freetype}
|
|
DESTDIR = {destdir}
|
|
CONFIG -= create_cmake # Prevent qmake from generating a cmake build file which it puts in the calibre src directory
|
|
QMAKE_LIBS_PRIVATE += {glib} {fontconfig}
|
|
''').format(
|
|
headers=' '.join(headers), sources=' '.join(sources), others=' '.join(others), destdir=self.d(
|
|
target), glib=glib_flags, fontconfig=fontconfig_flags, freetype=' '.join(ft_inc_dirs))
|
|
bdir = self.j(self.build_dir, 'headless')
|
|
if not os.path.exists(bdir):
|
|
os.makedirs(bdir)
|
|
pf = self.j(bdir, 'headless.pro')
|
|
open(self.j(bdir, '.qmake.conf'), 'wb').close()
|
|
with open(pf, 'wb') as f:
|
|
f.write(pro.encode('utf-8'))
|
|
cwd = os.getcwd()
|
|
os.chdir(bdir)
|
|
try:
|
|
self.check_call([QMAKE] + [self.b(pf)])
|
|
self.check_call([self.env.make] + ['-j%d'%(cpu_count or 1)])
|
|
finally:
|
|
os.chdir(cwd)
|
|
|
|
def build_sip_files(self, ext, src_dir):
|
|
from setup.build_environment import pyqt
|
|
sip_files = ext.sip_files
|
|
ext.sip_files = []
|
|
sipf = sip_files[0]
|
|
sbf = self.j(src_dir, self.b(sipf)+'.sbf')
|
|
if self.newer(sbf, [sipf]+ext.headers):
|
|
cmd = [pyqt['sip_bin'], '-w', '-c', src_dir, '-b', sbf, '-I'+
|
|
pyqt['pyqt_sip_dir']] + shlex.split(pyqt['sip_flags']) + [sipf]
|
|
self.info(' '.join(cmd))
|
|
self.check_call(cmd)
|
|
self.info('')
|
|
raw = open(sbf, 'rb').read().decode('utf-8')
|
|
def read(x):
|
|
ans = re.search('^%s\s*=\s*(.+)$' % x, raw, flags=re.M).group(1).strip()
|
|
if x != 'target':
|
|
ans = ans.split()
|
|
return ans
|
|
return {x:read(x) for x in ('target', 'sources', 'headers')}
|
|
|
|
def build_pyqt_extension(self, ext, dest):
|
|
from setup.build_environment import pyqt, qmakespec, QMAKE
|
|
from setup.parallel_build import cpu_count
|
|
from distutils import sysconfig
|
|
pyqt_dir = self.j(self.build_dir, 'pyqt')
|
|
src_dir = self.j(pyqt_dir, ext.name)
|
|
if not os.path.exists(src_dir):
|
|
os.makedirs(src_dir)
|
|
sip = self.build_sip_files(ext, src_dir)
|
|
pro = textwrap.dedent(
|
|
'''\
|
|
TEMPLATE = lib
|
|
CONFIG += release plugin
|
|
QT += widgets
|
|
TARGET = {target}
|
|
HEADERS = {headers}
|
|
SOURCES = {sources}
|
|
INCLUDEPATH += {sipinc} {pyinc}
|
|
VERSION = {ver}
|
|
win32 {{
|
|
LIBS += {py_lib}
|
|
TARGET_EXT = .dll
|
|
}}
|
|
macx {{
|
|
QMAKE_LFLAGS += "-undefined dynamic_lookup"
|
|
}}
|
|
''').format(
|
|
target=sip['target'], headers=' '.join(sip['headers'] + ext.headers), sources=' '.join(ext.sources + sip['sources']),
|
|
sipinc=pyqt['sip_inc_dir'], pyinc=sysconfig.get_python_inc(), py_lib=py_lib,
|
|
ver=__version__
|
|
)
|
|
for incdir in ext.inc_dirs:
|
|
pro += '\nINCLUDEPATH += ' + incdir
|
|
if not iswindows and not isosx:
|
|
# Ensure that only the init symbol is exported
|
|
pro += '\nQMAKE_LFLAGS += -Wl,--version-script=%s.exp' % sip['target']
|
|
with open(os.path.join(src_dir, sip['target'] + '.exp'), 'wb') as f:
|
|
f.write(('{ global: init%s; local: *; };' % sip['target']).encode('utf-8'))
|
|
if ext.qt_private_headers:
|
|
qph = ' '.join(x + '-private' for x in ext.qt_private_headers)
|
|
pro += '\nQT += ' + qph
|
|
proname = '%s.pro' % sip['target']
|
|
with open(os.path.join(src_dir, proname), 'wb') as f:
|
|
f.write(pro.encode('utf-8'))
|
|
cwd = os.getcwdu()
|
|
qmc = []
|
|
if iswindows:
|
|
qmc += ['-spec', qmakespec]
|
|
fext = 'dll' if iswindows else 'dylib' if isosx else 'so'
|
|
name = '%s%s.%s' % ('release/' if iswindows else 'lib', sip['target'], fext)
|
|
try:
|
|
os.chdir(src_dir)
|
|
if self.newer(dest, sip['headers'] + sip['sources'] + ext.sources + ext.headers):
|
|
self.check_call([QMAKE] + qmc + [proname])
|
|
self.check_call([self.env.make]+([] if iswindows else ['-j%d'%(cpu_count or 1)]))
|
|
shutil.copy2(os.path.realpath(name), dest)
|
|
if iswindows and os.path.exists(name + '.manifest'):
|
|
shutil.copy2(name + '.manifest', dest + '.manifest')
|
|
|
|
finally:
|
|
os.chdir(cwd)
|
|
|
|
def clean(self):
|
|
self.output_dir = self.DEFAULT_OUTPUTDIR
|
|
extensions = map(parse_extension, filter(is_ext_allowed, read_extensions()))
|
|
for ext in extensions:
|
|
dest = self.dest(ext)
|
|
for x in (dest, dest+'.manifest'):
|
|
if os.path.exists(x):
|
|
os.remove(x)
|
|
build_dir = self.DEFAULT_BUILDDIR
|
|
if os.path.exists(build_dir):
|
|
shutil.rmtree(build_dir)
|