Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Parallelize building of linux launchers

Browse Source
This commit is contained in:
Kovid Goyal 2014-07-12 18:35:21 +05:30
parent f49cd34288
commit cf5b0f68d2
3 changed files with 18 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
setup/installer/__init__.py
View File

@ -163,7 +163,8 @@ class VMInstaller(Command):
start_time = time.time() start_time = time.time()
self.run_vm_builder() self.run_vm_builder()
print ('Startup completed in %d seconds' % round(startup_time)) print ('Startup completed in %d seconds' % round(startup_time))
print ('Build completed in %d seconds' % round(time.time() - start_time)) secs = time.time() - start_time
print ('Build completed in %d minutes %d seconds' % (secs // 60, secs % 60))
if not opts.dont_shutdown: if not opts.dont_shutdown:
print ('Shutting down', self.VM_NAME) print ('Shutting down', self.VM_NAME)
subprocess.call(['ssh', self.VM_NAME]+self.SHUTDOWN_CMD) subprocess.call(['ssh', self.VM_NAME]+self.SHUTDOWN_CMD)

27
setup/installer/linux/freeze2.py
View File

@ -57,6 +57,7 @@ from functools import partial
from setup import Command, modules, basenames, functions, __version__, __appname__ from setup import Command, modules, basenames, functions, __version__, __appname__
from setup.build_environment import QT_DLLS, QT_PLUGINS, qt, PYQT_MODULES, sw as SW from setup.build_environment import QT_DLLS, QT_PLUGINS, qt, PYQT_MODULES, sw as SW
from setup.parallel_build import create_job, parallel_build
j = os.path.join j = os.path.join
is64bit = platform.architecture()[0] == '64bit' is64bit = platform.architecture()[0] == '64bit'
@ -283,7 +284,8 @@ class LinuxFreeze(Command):
else: else:
start_time = time.time() start_time = time.time()
subprocess.check_call(['xz', '--threads=0', '-f', '-9', dist]) subprocess.check_call(['xz', '--threads=0', '-f', '-9', dist])
self.info('Compressed in %d seconds' % round(time.time() - start_time)) secs = time.time() - start_time
self.info('Compressed in %d minutes %d seconds' % (secs // 60, secs % 60))
os.rename(dist + '.xz', ans) os.rename(dist + '.xz', ans)
self.info('Archive %s created: %.2f MB'%( self.info('Archive %s created: %.2f MB'%(
os.path.basename(ans), os.stat(ans).st_size/(1024.**2))) os.path.basename(ans), os.stat(ans).st_size/(1024.**2)))
@ -322,8 +324,9 @@ class LinuxFreeze(Command):
self.info('Compiling launcher') self.info('Compiling launcher')
self.run_builder(cmd, verbose=False) self.run_builder(cmd, verbose=False)
jobs = []
self.info('Processing launchers')
for typ in ('console', 'gui', ): for typ in ('console', 'gui', ):
self.info('Processing %s launchers'%typ)
for mod, bname, func in zip(modules[typ], basenames[typ], for mod, bname, func in zip(modules[typ], basenames[typ],
functions[typ]): functions[typ]):
xflags = list(cflags) xflags = list(cflags)
@ -331,25 +334,17 @@ class LinuxFreeze(Command):
xflags += ['-DMODULE="%s"'%mod, '-DBASENAME="%s"'%bname, xflags += ['-DMODULE="%s"'%mod, '-DBASENAME="%s"'%bname,
'-DFUNCTION="%s"'%func] '-DFUNCTION="%s"'%func]
dest = self.j(self.obj_dir, bname+'.o')
if self.newer(dest, [src, __file__]+headers):
cmd = ['gcc'] + xflags + [src, '-o', dest]
self.run_builder(cmd, verbose=False)
exe = self.j(self.bin_dir, bname) exe = self.j(self.bin_dir, bname)
if self.newer(exe, [src, __file__]+headers):
cmd = ['gcc'] + xflags + [src, '-o', exe, '-L' + self.lib_dir, '-lcalibre-launcher']
jobs.append(create_job(cmd))
sh = self.j(self.base, bname) sh = self.j(self.base, bname)
shutil.copy2(c_launcher, sh) shutil.copy2(c_launcher, sh)
os.chmod(sh, os.chmod(sh,
stat.S_IREAD|stat.S_IEXEC|stat.S_IWRITE|stat.S_IRGRP|stat.S_IXGRP|stat.S_IROTH|stat.S_IXOTH) stat.S_IREAD|stat.S_IEXEC|stat.S_IWRITE|stat.S_IRGRP|stat.S_IXGRP|stat.S_IROTH|stat.S_IXOTH)
if jobs:
if self.newer(exe, [dest, __file__]): if not parallel_build(jobs, self.info, verbose=False):
cmd = ['gcc', '-O2', raise SystemExit(1)
'-o', exe,
dest,
'-L'+self.lib_dir,
'-lcalibre-launcher',
]
self.run_builder(cmd, verbose=False)
def strip_files(self): def strip_files(self):
from calibre import walk from calibre import walk

3
setup/parallel_build.py
View File

@ -26,9 +26,10 @@ def run_worker(job):
def create_job(cmd, human_text=None): def create_job(cmd, human_text=None):
return (cmd, human_text) return (cmd, human_text)
def parallel_build(jobs, log): def parallel_build(jobs, log, verbose=True):
p = Pool(cpu_count) p = Pool(cpu_count)
for ok, stdout, stderr in p.imap(run_worker, jobs): for ok, stdout, stderr in p.imap(run_worker, jobs):
if verbose or not ok:
log(stdout) log(stdout)
if stderr: if stderr:
log(stderr) log(stderr)