mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-07-09 03:04:10 -04:00
Fully parallelize building of non-pyqt extensions
This commit is contained in:
parent
2331baf157
commit
dc4e482799
@ -12,6 +12,8 @@ from setup import Command, islinux, isbsd, isfreebsd, isosx, ishaiku, SRC, iswin
|
|||||||
isunix = islinux or isosx or isbsd or ishaiku
|
isunix = islinux or isosx or isbsd or ishaiku
|
||||||
|
|
||||||
py_lib = os.path.join(sys.prefix, 'libs', 'python%d%d.lib' % sys.version_info[:2])
|
py_lib = os.path.join(sys.prefix, 'libs', 'python%d%d.lib' % sys.version_info[:2])
|
||||||
|
CompileCommand = namedtuple('CompileCommand', 'cmd src dest')
|
||||||
|
LinkCommand = namedtuple('LinkCommand', 'cmd objects dest')
|
||||||
|
|
||||||
|
|
||||||
def init_symbol_name(name):
|
def init_symbol_name(name):
|
||||||
@ -255,18 +257,19 @@ class Build(Command):
|
|||||||
help='Path to directory in which to place the built extensions. Defaults to src/calibre/plugins')
|
help='Path to directory in which to place the built extensions. Defaults to src/calibre/plugins')
|
||||||
|
|
||||||
def run(self, opts):
|
def run(self, opts):
|
||||||
|
from setup.parallel_build import parallel_build, create_job
|
||||||
if opts.no_compile:
|
if opts.no_compile:
|
||||||
self.info('--no-compile specified, skipping compilation')
|
self.info('--no-compile specified, skipping compilation')
|
||||||
return
|
return
|
||||||
self.env = init_env()
|
self.env = init_env()
|
||||||
extensions = map(parse_extension, filter(is_ext_allowed, read_extensions()))
|
all_extensions = map(parse_extension, filter(is_ext_allowed, read_extensions()))
|
||||||
self.build_dir = os.path.abspath(opts.build_dir or self.DEFAULT_BUILDDIR)
|
self.build_dir = os.path.abspath(opts.build_dir or self.DEFAULT_BUILDDIR)
|
||||||
self.output_dir = os.path.abspath(opts.output_dir or self.DEFAULT_OUTPUTDIR)
|
self.output_dir = os.path.abspath(opts.output_dir or self.DEFAULT_OUTPUTDIR)
|
||||||
self.obj_dir = os.path.join(self.build_dir, 'objects')
|
self.obj_dir = os.path.join(self.build_dir, 'objects')
|
||||||
for x in (self.output_dir, self.obj_dir):
|
for x in (self.output_dir, self.obj_dir):
|
||||||
if not os.path.exists(x):
|
os.makedirs(x, exist_ok=True)
|
||||||
os.makedirs(x)
|
pyqt_extensions, extensions = [], []
|
||||||
for ext in extensions:
|
for ext in all_extensions:
|
||||||
if opts.only != 'all' and opts.only != ext.name:
|
if opts.only != 'all' and opts.only != ext.name:
|
||||||
continue
|
continue
|
||||||
if ext.error:
|
if ext.error:
|
||||||
@ -276,10 +279,39 @@ class Build(Command):
|
|||||||
else:
|
else:
|
||||||
raise Exception(ext.error)
|
raise Exception(ext.error)
|
||||||
dest = self.dest(ext)
|
dest = self.dest(ext)
|
||||||
if not os.path.exists(self.d(dest)):
|
os.makedirs(self.d(dest), exist_ok=True)
|
||||||
os.makedirs(self.d(dest))
|
(pyqt_extensions if ext.sip_files else extensions).append((ext, dest))
|
||||||
|
|
||||||
|
jobs = []
|
||||||
|
objects_map = {}
|
||||||
|
self.info(f'Building {len(extensions)+len(pyqt_extensions)} extensions')
|
||||||
|
for (ext, dest) in extensions:
|
||||||
|
cmds, objects = self.get_compile_commands(ext, dest)
|
||||||
|
objects_map[id(ext)] = objects
|
||||||
|
for cmd in cmds:
|
||||||
|
jobs.append(create_job(cmd.cmd))
|
||||||
|
if jobs:
|
||||||
|
self.info(f'Compiling {len(jobs)} files...')
|
||||||
|
if not parallel_build(jobs, self.info):
|
||||||
|
raise SystemExit(1)
|
||||||
|
jobs, link_commands = [], []
|
||||||
|
for (ext, dest) in extensions:
|
||||||
|
objects = objects_map[id(ext)]
|
||||||
|
cmd = self.get_link_command(ext, dest, objects)
|
||||||
|
if cmd is not None:
|
||||||
|
link_commands.append(cmd)
|
||||||
|
jobs.append(create_job(cmd.cmd))
|
||||||
|
if jobs:
|
||||||
|
self.info(f'Linking {len(jobs)} files...')
|
||||||
|
if not parallel_build(jobs, self.info):
|
||||||
|
raise SystemExit(1)
|
||||||
|
for cmd in link_commands:
|
||||||
|
self.post_link_cleanup(cmd)
|
||||||
|
|
||||||
|
for (ext, dest) in pyqt_extensions:
|
||||||
self.info('\n####### Building extension', ext.name, '#'*7)
|
self.info('\n####### Building extension', ext.name, '#'*7)
|
||||||
self.build(ext, dest)
|
self.build_pyqt_extension(ext, dest)
|
||||||
|
|
||||||
if opts.only in {'all', 'headless'}:
|
if opts.only in {'all', 'headless'}:
|
||||||
self.build_headless()
|
self.build_headless()
|
||||||
|
|
||||||
@ -299,19 +331,14 @@ class Build(Command):
|
|||||||
suff = '.lib' if iswindows else ''
|
suff = '.lib' if iswindows else ''
|
||||||
return [pref+x+suff for x in dirs]
|
return [pref+x+suff for x in dirs]
|
||||||
|
|
||||||
def build(self, ext, dest):
|
def get_compile_commands(self, ext, dest):
|
||||||
from setup.parallel_build import create_job, parallel_build
|
|
||||||
if ext.sip_files:
|
|
||||||
return self.build_pyqt_extension(ext, dest)
|
|
||||||
compiler = self.env.cxx if ext.needs_cxx else self.env.cc
|
compiler = self.env.cxx if ext.needs_cxx else self.env.cc
|
||||||
linker = self.env.linker if iswindows else compiler
|
|
||||||
objects = []
|
objects = []
|
||||||
|
ans = []
|
||||||
obj_dir = self.j(self.obj_dir, ext.name)
|
obj_dir = self.j(self.obj_dir, ext.name)
|
||||||
einc = self.inc_dirs_to_cflags(ext.inc_dirs)
|
einc = self.inc_dirs_to_cflags(ext.inc_dirs)
|
||||||
if not os.path.exists(obj_dir):
|
os.makedirs(obj_dir, exist_ok=True)
|
||||||
os.makedirs(obj_dir)
|
|
||||||
|
|
||||||
jobs = []
|
|
||||||
for src in ext.sources:
|
for src in ext.sources:
|
||||||
obj = self.j(obj_dir, os.path.splitext(self.b(src))[0]+'.o')
|
obj = self.j(obj_dir, os.path.splitext(self.b(src))[0]+'.o')
|
||||||
objects.append(obj)
|
objects.append(obj)
|
||||||
@ -320,17 +347,16 @@ class Build(Command):
|
|||||||
sinc = [inf+src] if iswindows else ['-c', src]
|
sinc = [inf+src] if iswindows else ['-c', src]
|
||||||
oinc = ['/Fo'+obj] if iswindows else ['-o', obj]
|
oinc = ['/Fo'+obj] if iswindows else ['-o', obj]
|
||||||
cmd = [compiler] + self.env.cflags + ext.cflags + einc + sinc + oinc
|
cmd = [compiler] + self.env.cflags + ext.cflags + einc + sinc + oinc
|
||||||
jobs.append(create_job(cmd))
|
ans.append(CompileCommand(cmd, src, obj))
|
||||||
if jobs:
|
return ans, objects
|
||||||
self.info('Compiling', ext.name)
|
|
||||||
if not parallel_build(jobs, self.info):
|
|
||||||
raise SystemExit(1)
|
|
||||||
|
|
||||||
|
def get_link_command(self, ext, dest, objects):
|
||||||
|
compiler = self.env.cxx if ext.needs_cxx else self.env.cc
|
||||||
|
linker = self.env.linker if iswindows else compiler
|
||||||
dest = self.dest(ext)
|
dest = self.dest(ext)
|
||||||
elib = self.lib_dirs_to_ldflags(ext.lib_dirs)
|
elib = self.lib_dirs_to_ldflags(ext.lib_dirs)
|
||||||
xlib = self.libraries_to_ldflags(ext.libraries)
|
xlib = self.libraries_to_ldflags(ext.libraries)
|
||||||
if self.newer(dest, objects+ext.extra_objs):
|
if self.newer(dest, objects+ext.extra_objs):
|
||||||
self.info('Linking', ext.name)
|
|
||||||
cmd = [linker]
|
cmd = [linker]
|
||||||
if iswindows:
|
if iswindows:
|
||||||
pre_ld_flags = []
|
pre_ld_flags = []
|
||||||
@ -341,13 +367,15 @@ class Build(Command):
|
|||||||
['/EXPORT:' + init_symbol_name(ext.name)] + objects + ext.extra_objs + ['/OUT:'+dest]
|
['/EXPORT:' + init_symbol_name(ext.name)] + objects + ext.extra_objs + ['/OUT:'+dest]
|
||||||
else:
|
else:
|
||||||
cmd += objects + ext.extra_objs + ['-o', dest] + self.env.ldflags + ext.ldflags + elib + xlib
|
cmd += objects + ext.extra_objs + ['-o', dest] + self.env.ldflags + ext.ldflags + elib + xlib
|
||||||
self.info('\n\n', ' '.join(cmd), '\n\n')
|
return LinkCommand(cmd, objects, dest)
|
||||||
self.check_call(cmd)
|
|
||||||
if iswindows:
|
def post_link_cleanup(self, link_command):
|
||||||
for x in ('.exp', '.lib'):
|
if iswindows:
|
||||||
x = os.path.splitext(dest)[0]+x
|
dest = link_command.dest
|
||||||
if os.path.exists(x):
|
for x in ('.exp', '.lib'):
|
||||||
os.remove(x)
|
x = os.path.splitext(dest)[0]+x
|
||||||
|
if os.path.exists(x):
|
||||||
|
os.remove(x)
|
||||||
|
|
||||||
def check_call(self, *args, **kwargs):
|
def check_call(self, *args, **kwargs):
|
||||||
"""print cmdline if an error occured
|
"""print cmdline if an error occured
|
||||||
|
Loading…
x
Reference in New Issue
Block a user