From 496a0789fd9eb9e520eff165e5a215cdfb5ee35e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 7 Jun 2014 13:13:32 +0530 Subject: [PATCH] Switch to using VirtualBox vms for building calibre --- setup/installer/__init__.py | 101 +++++++++++----------------- setup/installer/linux/__init__.py | 6 +- setup/installer/osx/__init__.py | 6 +- setup/installer/windows/__init__.py | 22 +----- 4 files changed, 46 insertions(+), 89 deletions(-) diff --git a/setup/installer/__init__.py b/setup/installer/__init__.py index 1667fc8209..04cb117bbf 100644 --- a/setup/installer/__init__.py +++ b/setup/installer/__init__.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai -from __future__ import with_statement +from __future__ import (unicode_literals, division, absolute_import, + print_function) __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' @@ -9,36 +10,31 @@ __docformat__ = 'restructuredtext en' import subprocess, tempfile, os, time, socket from setup import Command, installer_name -from setup.build_environment import HOST, PROJECT +from setup.build_environment import BUILD_HOST, PROJECT BASE_RSYNC = ['rsync', '-av', '--delete', '--force'] EXCLUDES = [] for x in [ - 'src/calibre/plugins', 'manual', + 'src/calibre/plugins', 'manual', 'translations', '.bzr', '.git', '.build', '.svn', 'build', 'dist', 'imgsrc', '*.pyc', '*.pyo', '*.swp', '*.swo', 'format_docs', 'translations']: EXCLUDES.extend(['--exclude', x]) SAFE_EXCLUDES = ['"%s"'%x if '*' in x else x for x in EXCLUDES] def get_rsync_pw(): - return open('/home/kovid/work/kde/conf/buildbot').read().partition( + return open('/home/kovid/work/env/private/buildbot').read().decode('utf-8').partition( ':')[-1].strip() def is_vm_running(name): - pat = '/%s/'%name - pids= [pid for pid in os.listdir('/proc') if pid.isdigit()] - for pid in pids: - try: - cmdline = open(os.path.join('/proc', pid, 'cmdline'), 'rb').read() - except IOError: - continue # file went away - if 'vmware-vmx' in cmdline and pat in cmdline: + qname = '"%s"' % name + for line in subprocess.check_output('VBoxManage list runningvms'.split()).decode('utf-8').splitlines(): + if line.startswith(qname): return True return False -def is_host_reachable(name): +def is_host_reachable(name, timeout=1): try: - socket.create_connection((name, 22), 5).close() + socket.create_connection((name, 22), timeout).close() return True except: return False @@ -51,7 +47,7 @@ class Rsync(Command): ['rsync://buildbot@{host}/work/{project}', '..']) def run(self, opts): - cmd = self.SYNC_CMD.format(host=HOST, project=PROJECT) + cmd = self.SYNC_CMD.format(host=BUILD_HOST, project=PROJECT) env = dict(os.environ) env['RSYNC_PASSWORD'] = get_rsync_pw() self.info(cmd) @@ -66,7 +62,7 @@ def push(host, vmname, available): if ok: available[vmname or host] = True rcmd = BASE_RSYNC + EXCLUDES + ['.', host] - print '\n\nPushing to:', vmname or host, '\n' + print ('\n\nPushing to:', vmname or host, '\n') subprocess.check_call(rcmd, stdout=open(os.devnull, 'wb')) class Push(Command): @@ -91,21 +87,17 @@ class Push(Command): thread.join(0.01) if not thread.is_alive(): if available.get(name, False): - print '\n\n', name, 'done' + print ('\n\n', name, 'done') threads.pop(name) class VMInstaller(Command): - EXTRA_SLEEP = 5 - INSTALLER_EXT = None - VM = None VM_NAME = None - VM_CHECK = None FREEZE_COMMAND = None FREEZE_TEMPLATE = 'python setup.py {freeze_command}' - SHUTDOWN_CMD = ['sudo', 'poweroff'] + SHUTDOWN_CMD = ['sudo', 'shutdown', '-h', 'now'] IS_64_BIT = False BUILD_CMD = 'ssh -t %s bash build-calibre' @@ -113,13 +105,12 @@ class VMInstaller(Command): BUILD_RSYNC = [r'cd ~/build/{project}', Rsync.SYNC_CMD] BUILD_CLEAN = ['rm -rf dist/* build/* src/calibre/plugins/*'] BUILD_BUILD = ['python setup.py build',] + FORCE_SHUTDOWN = 0 # number of seconds to wait before doing a forced power off (0 means disabled) def add_options(self, parser): if not parser.has_option('--dont-shutdown'): parser.add_option('-s', '--dont-shutdown', default=False, action='store_true', help='Dont shutdown the VM after building') - if not parser.has_option('--vm'): - parser.add_option('--vm', help='Path to VM launcher script') def get_build_script(self): rs = ['export RSYNC_PASSWORD=%s'%get_rsync_pw()] @@ -128,64 +119,50 @@ class VMInstaller(Command): ans += ' && \\\n'.join(self.BUILD_CLEAN) + ' && \\\n' ans += ' && \\\n'.join(self.BUILD_BUILD) + ' && \\\n' ans += self.FREEZE_TEMPLATE.format(freeze_command=self.FREEZE_COMMAND) + '\n' - ans = ans.format(project=PROJECT, host=HOST) + ans = ans.format(project=PROJECT, host=BUILD_HOST) return ans - def vmware_started(self): - return 'started' in subprocess.Popen('/etc/init.d/vmware status', shell=True, stdout=subprocess.PIPE).stdout.read() - - def start_vmware(self): - if not self.vmware_started(): - if os.path.exists('/dev/kvm'): - subprocess.check_call('sudo rmmod -w kvm-intel kvm', shell=True) - subprocess.Popen('sudo /etc/init.d/vmware start', shell=True) - - def stop_vmware(self): - while True: - try: - subprocess.check_call('sudo /etc/init.d/vmware stop', shell=True) - break - except: - pass - while 'vmblock' in open('/proc/modules').read(): - subprocess.check_call('sudo rmmod -f vmblock') - def run_vm(self): - if is_vm_running(self.VM_CHECK or self.VM_NAME): - return - self.__p = subprocess.Popen([self.vm]) + if is_vm_running(self.VM_NAME): + return True + self.__p = subprocess.Popen(("VBoxManage startvm %s --type gui" % self.VM_NAME).split()) + return False def start_vm(self, sleep=75): ssh_host = self.VM_NAME - self.run_vm() + already_running = self.run_vm() + if not already_running: + time.sleep(2) + print ('Waiting for SSH server to start') + while not is_host_reachable(ssh_host, timeout=1): + time.sleep(0.1) + + def run_vm_builder(self): + ssh_host = self.VM_NAME build_script = self.get_build_script() t = tempfile.NamedTemporaryFile(suffix='.sh') t.write(build_script) t.flush() - print 'Waiting for VM to startup' - while subprocess.call('ping -q -c1 '+ssh_host, shell=True, - stdout=open('/dev/null', 'w')) != 0: - time.sleep(5) - time.sleep(self.EXTRA_SLEEP) - print 'Trying to SSH into VM' + print ('Running VM builder') subprocess.check_call(('scp', t.name, ssh_host+':build-calibre')) subprocess.check_call(self.BUILD_CMD%ssh_host, shell=True) + self.download_installer() def installer(self): return installer_name(self.INSTALLER_EXT, self.IS_64_BIT) def run(self, opts): - for x in ('dont_shutdown', 'vm'): - setattr(self, x, getattr(opts, x)) - if self.vm is None: - self.vm = self.VM - if not self.vmware_started(): - self.start_vmware() subprocess.call(['chmod', '-R', '+r', 'recipes']) self.start_vm() - self.download_installer() - if not self.dont_shutdown: + self.run_vm_builder() + if not opts.dont_shutdown: + print ('Shutting down', self.VM_NAME) subprocess.call(['ssh', self.VM_NAME]+self.SHUTDOWN_CMD) + if self.FORCE_SHUTDOWN: + while is_host_reachable(self.VM_NAME): + time.sleep(0.1) # wait for SSH server to shutdown + time.sleep(self.FORCE_SHUTDOWN) + subprocess.check_call(('VBoxManage controlvm %s poweroff' % self.VM_NAME).split()) def download_installer(self): installer = self.installer() diff --git a/setup/installer/linux/__init__.py b/setup/installer/linux/__init__.py index 3d759cff9a..e65e0a7063 100644 --- a/setup/installer/linux/__init__.py +++ b/setup/installer/linux/__init__.py @@ -15,8 +15,7 @@ class Linux32(VMInstaller): description = 'Build 32bit linux binary installer' INSTALLER_EXT = 'tar.bz2' - VM_NAME = 'gentoo32_build' - VM = '/vmware/bin/gentoo32_build' + VM_NAME = 'linux32-build' FREEZE_COMMAND = 'linux_freeze' FREEZE_TEMPLATE = 'sudo python -OO setup.py {freeze_command}' @@ -24,8 +23,7 @@ class Linux32(VMInstaller): class Linux64(Linux32): description = 'Build 64bit linux binary installer' - VM_NAME = 'gentoo64_build' - VM = '/vmware/bin/gentoo64_build' + VM_NAME = 'linux64-build' IS_64_BIT = True class Linux(Command): diff --git a/setup/installer/osx/__init__.py b/setup/installer/osx/__init__.py index dfc129eab6..e2167a2393 100644 --- a/setup/installer/osx/__init__.py +++ b/setup/installer/osx/__init__.py @@ -14,9 +14,7 @@ class OSX(VMInstaller): description = 'Build OS X binary installer' INSTALLER_EXT = 'dmg' - VM_NAME = 'osx_build' - VM = '/vmware/bin/%s'%VM_NAME + VM_NAME = 'osx-build' FREEZE_TEMPLATE = 'python -OO setup.py {freeze_command}' FREEZE_COMMAND = 'osx32_freeze' - BUILD_PREFIX = VMInstaller.BUILD_PREFIX + ['source ~/.profile'] - SHUTDOWN_CMD = ['sudo', 'halt'] + FORCE_SHUTDOWN = 10 # number of seconds to wait before doing a forced power off diff --git a/setup/installer/windows/__init__.py b/setup/installer/windows/__init__.py index 5e1ad4a7c0..481038c514 100644 --- a/setup/installer/windows/__init__.py +++ b/setup/installer/windows/__init__.py @@ -8,7 +8,7 @@ __docformat__ = 'restructuredtext en' import os, shutil, subprocess -from setup import Command, __appname__, __version__, installer_name +from setup import Command, __appname__, __version__ from setup.installer import VMInstaller class Win(Command): @@ -30,14 +30,7 @@ class WinBase(VMInstaller): class Win32(WinBase): description = 'Build 32bit windows binary installer' - - VM_NAME = 'xp_build' - VM = '/vmware/bin/%s'%VM_NAME - VM_CHECK = 'calibre_windows_xp_home' - - @property - def msi64(self): - return installer_name('msi', is64bit=True) + VM_NAME = 'win32-build' def do_dl(self, installer, errmsg): subprocess.check_call(('scp', @@ -59,15 +52,6 @@ class Win64(WinBase): description = 'Build 64bit windows binary installer' - VM_NAME = 'win64' - VM = '/vmware/bin/%s'%VM_NAME - VM_CHECK = 'win64' + VM_NAME = 'win64-build' IS_64_BIT = True - BUILD_PREFIX = WinBase.BUILD_PREFIX + [ - 'if [ -f "$HOME/.bash_profile" ] ; then', - ' source "$HOME/.bash_profile"', - 'fi', - ] - -