mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-07-07 10:14:46 -04:00
Switch to using VirtualBox vms for building calibre
This commit is contained in:
Kovid Goyal
parent
57a1e1ee8e
commit
496a0789fd
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env python
|
||||
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
|
||||
from __future__ import with_statement
|
||||
from __future__ import (unicode_literals, division, absolute_import,
|
||||
print_function)
|
||||
|
||||
__license__ = 'GPL v3'
|
||||
__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
|
||||
@ -9,36 +10,31 @@ __docformat__ = 'restructuredtext en'
|
||||
import subprocess, tempfile, os, time, socket
|
||||
|
||||
from setup import Command, installer_name
|
||||
from setup.build_environment import HOST, PROJECT
|
||||
from setup.build_environment import BUILD_HOST, PROJECT
|
||||
|
||||
BASE_RSYNC = ['rsync', '-av', '--delete', '--force']
|
||||
EXCLUDES = []
|
||||
for x in [
|
||||
'src/calibre/plugins', 'manual',
|
||||
'src/calibre/plugins', 'manual', 'translations',
|
||||
'.bzr', '.git', '.build', '.svn', 'build', 'dist', 'imgsrc', '*.pyc', '*.pyo', '*.swp',
|
||||
'*.swo', 'format_docs', 'translations']:
|
||||
EXCLUDES.extend(['--exclude', x])
|
||||
SAFE_EXCLUDES = ['"%s"'%x if '*' in x else x for x in EXCLUDES]
|
||||
|
||||
def get_rsync_pw():
|
||||
return open('/home/kovid/work/kde/conf/buildbot').read().partition(
|
||||
return open('/home/kovid/work/env/private/buildbot').read().decode('utf-8').partition(
|
||||
':')[-1].strip()
|
||||
|
||||
def is_vm_running(name):
|
||||
pat = '/%s/'%name
|
||||
pids= [pid for pid in os.listdir('/proc') if pid.isdigit()]
|
||||
for pid in pids:
|
||||
try:
|
||||
cmdline = open(os.path.join('/proc', pid, 'cmdline'), 'rb').read()
|
||||
except IOError:
|
||||
continue # file went away
|
||||
if 'vmware-vmx' in cmdline and pat in cmdline:
|
||||
qname = '"%s"' % name
|
||||
for line in subprocess.check_output('VBoxManage list runningvms'.split()).decode('utf-8').splitlines():
|
||||
if line.startswith(qname):
|
||||
return True
|
||||
return False
|
||||
|
||||
def is_host_reachable(name):
|
||||
def is_host_reachable(name, timeout=1):
|
||||
try:
|
||||
socket.create_connection((name, 22), 5).close()
|
||||
socket.create_connection((name, 22), timeout).close()
|
||||
return True
|
||||
except:
|
||||
return False
|
||||
@ -51,7 +47,7 @@ class Rsync(Command):
|
||||
['rsync://buildbot@{host}/work/{project}', '..'])
|
||||
|
||||
def run(self, opts):
|
||||
cmd = self.SYNC_CMD.format(host=HOST, project=PROJECT)
|
||||
cmd = self.SYNC_CMD.format(host=BUILD_HOST, project=PROJECT)
|
||||
env = dict(os.environ)
|
||||
env['RSYNC_PASSWORD'] = get_rsync_pw()
|
||||
self.info(cmd)
|
||||
@ -66,7 +62,7 @@ def push(host, vmname, available):
|
||||
if ok:
|
||||
available[vmname or host] = True
|
||||
rcmd = BASE_RSYNC + EXCLUDES + ['.', host]
|
||||
print '\n\nPushing to:', vmname or host, '\n'
|
||||
print ('\n\nPushing to:', vmname or host, '\n')
|
||||
subprocess.check_call(rcmd, stdout=open(os.devnull, 'wb'))
|
||||
|
||||
class Push(Command):
|
||||
@ -91,21 +87,17 @@ class Push(Command):
|
||||
thread.join(0.01)
|
||||
if not thread.is_alive():
|
||||
if available.get(name, False):
|
||||
print '\n\n', name, 'done'
|
||||
print ('\n\n', name, 'done')
|
||||
threads.pop(name)
|
||||
|
||||
|
||||
class VMInstaller(Command):
|
||||
|
||||
EXTRA_SLEEP = 5
|
||||
|
||||
INSTALLER_EXT = None
|
||||
VM = None
|
||||
VM_NAME = None
|
||||
VM_CHECK = None
|
||||
FREEZE_COMMAND = None
|
||||
FREEZE_TEMPLATE = 'python setup.py {freeze_command}'
|
||||
SHUTDOWN_CMD = ['sudo', 'poweroff']
|
||||
SHUTDOWN_CMD = ['sudo', 'shutdown', '-h', 'now']
|
||||
IS_64_BIT = False
|
||||
|
||||
BUILD_CMD = 'ssh -t %s bash build-calibre'
|
||||
@ -113,13 +105,12 @@ class VMInstaller(Command):
|
||||
BUILD_RSYNC = [r'cd ~/build/{project}', Rsync.SYNC_CMD]
|
||||
BUILD_CLEAN = ['rm -rf dist/* build/* src/calibre/plugins/*']
|
||||
BUILD_BUILD = ['python setup.py build',]
|
||||
FORCE_SHUTDOWN = 0 # number of seconds to wait before doing a forced power off (0 means disabled)
|
||||
|
||||
def add_options(self, parser):
|
||||
if not parser.has_option('--dont-shutdown'):
|
||||
parser.add_option('-s', '--dont-shutdown', default=False,
|
||||
action='store_true', help='Dont shutdown the VM after building')
|
||||
if not parser.has_option('--vm'):
|
||||
parser.add_option('--vm', help='Path to VM launcher script')
|
||||
|
||||
def get_build_script(self):
|
||||
rs = ['export RSYNC_PASSWORD=%s'%get_rsync_pw()]
|
||||
@ -128,64 +119,50 @@ class VMInstaller(Command):
|
||||
ans += ' && \\\n'.join(self.BUILD_CLEAN) + ' && \\\n'
|
||||
ans += ' && \\\n'.join(self.BUILD_BUILD) + ' && \\\n'
|
||||
ans += self.FREEZE_TEMPLATE.format(freeze_command=self.FREEZE_COMMAND) + '\n'
|
||||
ans = ans.format(project=PROJECT, host=HOST)
|
||||
ans = ans.format(project=PROJECT, host=BUILD_HOST)
|
||||
return ans
|
||||
|
||||
def vmware_started(self):
|
||||
return 'started' in subprocess.Popen('/etc/init.d/vmware status', shell=True, stdout=subprocess.PIPE).stdout.read()
|
||||
|
||||
def start_vmware(self):
|
||||
if not self.vmware_started():
|
||||
if os.path.exists('/dev/kvm'):
|
||||
subprocess.check_call('sudo rmmod -w kvm-intel kvm', shell=True)
|
||||
subprocess.Popen('sudo /etc/init.d/vmware start', shell=True)
|
||||
|
||||
def stop_vmware(self):
|
||||
while True:
|
||||
try:
|
||||
subprocess.check_call('sudo /etc/init.d/vmware stop', shell=True)
|
||||
break
|
||||
except:
|
||||
pass
|
||||
while 'vmblock' in open('/proc/modules').read():
|
||||
subprocess.check_call('sudo rmmod -f vmblock')
|
||||
|
||||
def run_vm(self):
|
||||
if is_vm_running(self.VM_CHECK or self.VM_NAME):
|
||||
return
|
||||
self.__p = subprocess.Popen([self.vm])
|
||||
if is_vm_running(self.VM_NAME):
|
||||
return True
|
||||
self.__p = subprocess.Popen(("VBoxManage startvm %s --type gui" % self.VM_NAME).split())
|
||||
return False
|
||||
|
||||
def start_vm(self, sleep=75):
|
||||
ssh_host = self.VM_NAME
|
||||
self.run_vm()
|
||||
already_running = self.run_vm()
|
||||
if not already_running:
|
||||
time.sleep(2)
|
||||
print ('Waiting for SSH server to start')
|
||||
while not is_host_reachable(ssh_host, timeout=1):
|
||||
time.sleep(0.1)
|
||||
|
||||
def run_vm_builder(self):
|
||||
ssh_host = self.VM_NAME
|
||||
build_script = self.get_build_script()
|
||||
t = tempfile.NamedTemporaryFile(suffix='.sh')
|
||||
t.write(build_script)
|
||||
t.flush()
|
||||
print 'Waiting for VM to startup'
|
||||
while subprocess.call('ping -q -c1 '+ssh_host, shell=True,
|
||||
stdout=open('/dev/null', 'w')) != 0:
|
||||
time.sleep(5)
|
||||
time.sleep(self.EXTRA_SLEEP)
|
||||
print 'Trying to SSH into VM'
|
||||
print ('Running VM builder')
|
||||
subprocess.check_call(('scp', t.name, ssh_host+':build-calibre'))
|
||||
subprocess.check_call(self.BUILD_CMD%ssh_host, shell=True)
|
||||
self.download_installer()
|
||||
|
||||
def installer(self):
|
||||
return installer_name(self.INSTALLER_EXT, self.IS_64_BIT)
|
||||
|
||||
def run(self, opts):
|
||||
for x in ('dont_shutdown', 'vm'):
|
||||
setattr(self, x, getattr(opts, x))
|
||||
if self.vm is None:
|
||||
self.vm = self.VM
|
||||
if not self.vmware_started():
|
||||
self.start_vmware()
|
||||
subprocess.call(['chmod', '-R', '+r', 'recipes'])
|
||||
self.start_vm()
|
||||
self.download_installer()
|
||||
if not self.dont_shutdown:
|
||||
self.run_vm_builder()
|
||||
if not opts.dont_shutdown:
|
||||
print ('Shutting down', self.VM_NAME)
|
||||
subprocess.call(['ssh', self.VM_NAME]+self.SHUTDOWN_CMD)
|
||||
if self.FORCE_SHUTDOWN:
|
||||
while is_host_reachable(self.VM_NAME):
|
||||
time.sleep(0.1) # wait for SSH server to shutdown
|
||||
time.sleep(self.FORCE_SHUTDOWN)
|
||||
subprocess.check_call(('VBoxManage controlvm %s poweroff' % self.VM_NAME).split())
|
||||
|
||||
def download_installer(self):
|
||||
installer = self.installer()
|
||||
|
||||
@ -15,8 +15,7 @@ class Linux32(VMInstaller):
|
||||
description = 'Build 32bit linux binary installer'
|
||||
|
||||
INSTALLER_EXT = 'tar.bz2'
|
||||
VM_NAME = 'gentoo32_build'
|
||||
VM = '/vmware/bin/gentoo32_build'
|
||||
VM_NAME = 'linux32-build'
|
||||
FREEZE_COMMAND = 'linux_freeze'
|
||||
FREEZE_TEMPLATE = 'sudo python -OO setup.py {freeze_command}'
|
||||
|
||||
@ -24,8 +23,7 @@ class Linux32(VMInstaller):
|
||||
class Linux64(Linux32):
|
||||
|
||||
description = 'Build 64bit linux binary installer'
|
||||
VM_NAME = 'gentoo64_build'
|
||||
VM = '/vmware/bin/gentoo64_build'
|
||||
VM_NAME = 'linux64-build'
|
||||
IS_64_BIT = True
|
||||
|
||||
class Linux(Command):
|
||||
|
||||
@ -14,9 +14,7 @@ class OSX(VMInstaller):
|
||||
description = 'Build OS X binary installer'
|
||||
|
||||
INSTALLER_EXT = 'dmg'
|
||||
VM_NAME = 'osx_build'
|
||||
VM = '/vmware/bin/%s'%VM_NAME
|
||||
VM_NAME = 'osx-build'
|
||||
FREEZE_TEMPLATE = 'python -OO setup.py {freeze_command}'
|
||||
FREEZE_COMMAND = 'osx32_freeze'
|
||||
BUILD_PREFIX = VMInstaller.BUILD_PREFIX + ['source ~/.profile']
|
||||
SHUTDOWN_CMD = ['sudo', 'halt']
|
||||
FORCE_SHUTDOWN = 10 # number of seconds to wait before doing a forced power off
|
||||
|
||||
@ -8,7 +8,7 @@ __docformat__ = 'restructuredtext en'
|
||||
|
||||
import os, shutil, subprocess
|
||||
|
||||
from setup import Command, __appname__, __version__, installer_name
|
||||
from setup import Command, __appname__, __version__
|
||||
from setup.installer import VMInstaller
|
||||
|
||||
class Win(Command):
|
||||
@ -30,14 +30,7 @@ class WinBase(VMInstaller):
|
||||
class Win32(WinBase):
|
||||
|
||||
description = 'Build 32bit windows binary installer'
|
||||
|
||||
VM_NAME = 'xp_build'
|
||||
VM = '/vmware/bin/%s'%VM_NAME
|
||||
VM_CHECK = 'calibre_windows_xp_home'
|
||||
|
||||
@property
|
||||
def msi64(self):
|
||||
return installer_name('msi', is64bit=True)
|
||||
VM_NAME = 'win32-build'
|
||||
|
||||
def do_dl(self, installer, errmsg):
|
||||
subprocess.check_call(('scp',
|
||||
@ -59,15 +52,6 @@ class Win64(WinBase):
|
||||
|
||||
description = 'Build 64bit windows binary installer'
|
||||
|
||||
VM_NAME = 'win64'
|
||||
VM = '/vmware/bin/%s'%VM_NAME
|
||||
VM_CHECK = 'win64'
|
||||
VM_NAME = 'win64-build'
|
||||
IS_64_BIT = True
|
||||
BUILD_PREFIX = WinBase.BUILD_PREFIX + [
|
||||
'if [ -f "$HOME/.bash_profile" ] ; then',
|
||||
' source "$HOME/.bash_profile"',
|
||||
'fi',
|
||||
]
|
||||
|
||||
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user