Fully parallelize building of non-pyqt extensions

This commit is contained in:
Kovid Goyal 2019-12-06 12:02:44 +05:30
parent 2331baf157
commit dc4e482799
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C

View File

@ -12,6 +12,8 @@ from setup import Command, islinux, isbsd, isfreebsd, isosx, ishaiku, SRC, iswin
isunix = islinux or isosx or isbsd or ishaiku isunix = islinux or isosx or isbsd or ishaiku
py_lib = os.path.join(sys.prefix, 'libs', 'python%d%d.lib' % sys.version_info[:2]) py_lib = os.path.join(sys.prefix, 'libs', 'python%d%d.lib' % sys.version_info[:2])
CompileCommand = namedtuple('CompileCommand', 'cmd src dest')
LinkCommand = namedtuple('LinkCommand', 'cmd objects dest')
def init_symbol_name(name): def init_symbol_name(name):
@ -255,18 +257,19 @@ class Build(Command):
help='Path to directory in which to place the built extensions. Defaults to src/calibre/plugins') help='Path to directory in which to place the built extensions. Defaults to src/calibre/plugins')
def run(self, opts): def run(self, opts):
from setup.parallel_build import parallel_build, create_job
if opts.no_compile: if opts.no_compile:
self.info('--no-compile specified, skipping compilation') self.info('--no-compile specified, skipping compilation')
return return
self.env = init_env() self.env = init_env()
extensions = map(parse_extension, filter(is_ext_allowed, read_extensions())) all_extensions = map(parse_extension, filter(is_ext_allowed, read_extensions()))
self.build_dir = os.path.abspath(opts.build_dir or self.DEFAULT_BUILDDIR) self.build_dir = os.path.abspath(opts.build_dir or self.DEFAULT_BUILDDIR)
self.output_dir = os.path.abspath(opts.output_dir or self.DEFAULT_OUTPUTDIR) self.output_dir = os.path.abspath(opts.output_dir or self.DEFAULT_OUTPUTDIR)
self.obj_dir = os.path.join(self.build_dir, 'objects') self.obj_dir = os.path.join(self.build_dir, 'objects')
for x in (self.output_dir, self.obj_dir): for x in (self.output_dir, self.obj_dir):
if not os.path.exists(x): os.makedirs(x, exist_ok=True)
os.makedirs(x) pyqt_extensions, extensions = [], []
for ext in extensions: for ext in all_extensions:
if opts.only != 'all' and opts.only != ext.name: if opts.only != 'all' and opts.only != ext.name:
continue continue
if ext.error: if ext.error:
@ -276,10 +279,39 @@ class Build(Command):
else: else:
raise Exception(ext.error) raise Exception(ext.error)
dest = self.dest(ext) dest = self.dest(ext)
if not os.path.exists(self.d(dest)): os.makedirs(self.d(dest), exist_ok=True)
os.makedirs(self.d(dest)) (pyqt_extensions if ext.sip_files else extensions).append((ext, dest))
jobs = []
objects_map = {}
self.info(f'Building {len(extensions)+len(pyqt_extensions)} extensions')
for (ext, dest) in extensions:
cmds, objects = self.get_compile_commands(ext, dest)
objects_map[id(ext)] = objects
for cmd in cmds:
jobs.append(create_job(cmd.cmd))
if jobs:
self.info(f'Compiling {len(jobs)} files...')
if not parallel_build(jobs, self.info):
raise SystemExit(1)
jobs, link_commands = [], []
for (ext, dest) in extensions:
objects = objects_map[id(ext)]
cmd = self.get_link_command(ext, dest, objects)
if cmd is not None:
link_commands.append(cmd)
jobs.append(create_job(cmd.cmd))
if jobs:
self.info(f'Linking {len(jobs)} files...')
if not parallel_build(jobs, self.info):
raise SystemExit(1)
for cmd in link_commands:
self.post_link_cleanup(cmd)
for (ext, dest) in pyqt_extensions:
self.info('\n####### Building extension', ext.name, '#'*7) self.info('\n####### Building extension', ext.name, '#'*7)
self.build(ext, dest) self.build_pyqt_extension(ext, dest)
if opts.only in {'all', 'headless'}: if opts.only in {'all', 'headless'}:
self.build_headless() self.build_headless()
@ -299,19 +331,14 @@ class Build(Command):
suff = '.lib' if iswindows else '' suff = '.lib' if iswindows else ''
return [pref+x+suff for x in dirs] return [pref+x+suff for x in dirs]
def build(self, ext, dest): def get_compile_commands(self, ext, dest):
from setup.parallel_build import create_job, parallel_build
if ext.sip_files:
return self.build_pyqt_extension(ext, dest)
compiler = self.env.cxx if ext.needs_cxx else self.env.cc compiler = self.env.cxx if ext.needs_cxx else self.env.cc
linker = self.env.linker if iswindows else compiler
objects = [] objects = []
ans = []
obj_dir = self.j(self.obj_dir, ext.name) obj_dir = self.j(self.obj_dir, ext.name)
einc = self.inc_dirs_to_cflags(ext.inc_dirs) einc = self.inc_dirs_to_cflags(ext.inc_dirs)
if not os.path.exists(obj_dir): os.makedirs(obj_dir, exist_ok=True)
os.makedirs(obj_dir)
jobs = []
for src in ext.sources: for src in ext.sources:
obj = self.j(obj_dir, os.path.splitext(self.b(src))[0]+'.o') obj = self.j(obj_dir, os.path.splitext(self.b(src))[0]+'.o')
objects.append(obj) objects.append(obj)
@ -320,17 +347,16 @@ class Build(Command):
sinc = [inf+src] if iswindows else ['-c', src] sinc = [inf+src] if iswindows else ['-c', src]
oinc = ['/Fo'+obj] if iswindows else ['-o', obj] oinc = ['/Fo'+obj] if iswindows else ['-o', obj]
cmd = [compiler] + self.env.cflags + ext.cflags + einc + sinc + oinc cmd = [compiler] + self.env.cflags + ext.cflags + einc + sinc + oinc
jobs.append(create_job(cmd)) ans.append(CompileCommand(cmd, src, obj))
if jobs: return ans, objects
self.info('Compiling', ext.name)
if not parallel_build(jobs, self.info):
raise SystemExit(1)
def get_link_command(self, ext, dest, objects):
compiler = self.env.cxx if ext.needs_cxx else self.env.cc
linker = self.env.linker if iswindows else compiler
dest = self.dest(ext) dest = self.dest(ext)
elib = self.lib_dirs_to_ldflags(ext.lib_dirs) elib = self.lib_dirs_to_ldflags(ext.lib_dirs)
xlib = self.libraries_to_ldflags(ext.libraries) xlib = self.libraries_to_ldflags(ext.libraries)
if self.newer(dest, objects+ext.extra_objs): if self.newer(dest, objects+ext.extra_objs):
self.info('Linking', ext.name)
cmd = [linker] cmd = [linker]
if iswindows: if iswindows:
pre_ld_flags = [] pre_ld_flags = []
@ -341,13 +367,15 @@ class Build(Command):
['/EXPORT:' + init_symbol_name(ext.name)] + objects + ext.extra_objs + ['/OUT:'+dest] ['/EXPORT:' + init_symbol_name(ext.name)] + objects + ext.extra_objs + ['/OUT:'+dest]
else: else:
cmd += objects + ext.extra_objs + ['-o', dest] + self.env.ldflags + ext.ldflags + elib + xlib cmd += objects + ext.extra_objs + ['-o', dest] + self.env.ldflags + ext.ldflags + elib + xlib
self.info('\n\n', ' '.join(cmd), '\n\n') return LinkCommand(cmd, objects, dest)
self.check_call(cmd)
if iswindows: def post_link_cleanup(self, link_command):
for x in ('.exp', '.lib'): if iswindows:
x = os.path.splitext(dest)[0]+x dest = link_command.dest
if os.path.exists(x): for x in ('.exp', '.lib'):
os.remove(x) x = os.path.splitext(dest)[0]+x
if os.path.exists(x):
os.remove(x)
def check_call(self, *args, **kwargs): def check_call(self, *args, **kwargs):
"""print cmdline if an error occured """print cmdline if an error occured