Switch to using VirtualBox vms for building calibre

setup/installer/__init__.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
-from __future__ import with_statement
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
 
 __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
@@ -9,36 +10,31 @@ __docformat__ = 'restructuredtext en'
 import subprocess, tempfile, os, time, socket
 
 from setup import Command, installer_name
-from setup.build_environment import HOST, PROJECT
+from setup.build_environment import BUILD_HOST, PROJECT
 
 BASE_RSYNC = ['rsync', '-av', '--delete', '--force']
 EXCLUDES = []
 for x in [
-    'src/calibre/plugins', 'manual',
+    'src/calibre/plugins', 'manual', 'translations',
     '.bzr', '.git', '.build', '.svn', 'build', 'dist', 'imgsrc', '*.pyc', '*.pyo', '*.swp',
     '*.swo', 'format_docs', 'translations']:
     EXCLUDES.extend(['--exclude', x])
 SAFE_EXCLUDES = ['"%s"'%x if '*' in x else x for x in EXCLUDES]
 
 def get_rsync_pw():
-    return open('/home/kovid/work/kde/conf/buildbot').read().partition(
+    return open('/home/kovid/work/env/private/buildbot').read().decode('utf-8').partition(
                 ':')[-1].strip()
 
 def is_vm_running(name):
-    pat = '/%s/'%name
+    qname = '"%s"' % name
-    pids= [pid for pid in os.listdir('/proc') if pid.isdigit()]
+    for line in subprocess.check_output('VBoxManage list runningvms'.split()).decode('utf-8').splitlines():
-    for pid in pids:
+        if line.startswith(qname):
-        try:
-            cmdline = open(os.path.join('/proc', pid, 'cmdline'), 'rb').read()
-        except IOError:
-            continue  # file went away
-        if 'vmware-vmx' in cmdline and pat in cmdline:
             return True
     return False
 
-def is_host_reachable(name):
+def is_host_reachable(name, timeout=1):
     try:
-        socket.create_connection((name, 22), 5).close()
+        socket.create_connection((name, 22), timeout).close()
         return True
     except:
         return False
@@ -51,7 +47,7 @@ class Rsync(Command):
             ['rsync://buildbot@{host}/work/{project}', '..'])
 
     def run(self, opts):
-        cmd = self.SYNC_CMD.format(host=HOST, project=PROJECT)
+        cmd = self.SYNC_CMD.format(host=BUILD_HOST, project=PROJECT)
         env = dict(os.environ)
         env['RSYNC_PASSWORD'] = get_rsync_pw()
         self.info(cmd)
@@ -66,7 +62,7 @@ def push(host, vmname, available):
     if ok:
         available[vmname or host] = True
         rcmd = BASE_RSYNC + EXCLUDES + ['.', host]
-        print '\n\nPushing to:', vmname or host, '\n'
+        print ('\n\nPushing to:', vmname or host, '\n')
         subprocess.check_call(rcmd, stdout=open(os.devnull, 'wb'))
 
 class Push(Command):
@@ -91,21 +87,17 @@ class Push(Command):
                 thread.join(0.01)
                 if not thread.is_alive():
                     if available.get(name, False):
-                        print '\n\n', name, 'done'
+                        print ('\n\n', name, 'done')
                     threads.pop(name)
 
 
 class VMInstaller(Command):
 
-    EXTRA_SLEEP = 5
-
     INSTALLER_EXT = None
-    VM = None
     VM_NAME = None
-    VM_CHECK = None
     FREEZE_COMMAND = None
     FREEZE_TEMPLATE = 'python setup.py {freeze_command}'
-    SHUTDOWN_CMD = ['sudo', 'poweroff']
+    SHUTDOWN_CMD = ['sudo', 'shutdown', '-h', 'now']
     IS_64_BIT = False
 
     BUILD_CMD = 'ssh -t %s bash build-calibre'
@@ -113,13 +105,12 @@ class VMInstaller(Command):
     BUILD_RSYNC  = [r'cd ~/build/{project}', Rsync.SYNC_CMD]
     BUILD_CLEAN  = ['rm -rf dist/* build/* src/calibre/plugins/*']
     BUILD_BUILD  = ['python setup.py build',]
+    FORCE_SHUTDOWN = 0  # number of seconds to wait before doing a forced power off (0 means disabled)
 
     def add_options(self, parser):
         if not parser.has_option('--dont-shutdown'):
             parser.add_option('-s', '--dont-shutdown', default=False,
                 action='store_true', help='Dont shutdown the VM after building')
-        if not parser.has_option('--vm'):
-            parser.add_option('--vm', help='Path to VM launcher script')
 
     def get_build_script(self):
         rs = ['export RSYNC_PASSWORD=%s'%get_rsync_pw()]
@@ -128,64 +119,50 @@ class VMInstaller(Command):
         ans += ' && \\\n'.join(self.BUILD_CLEAN) + ' && \\\n'
         ans += ' && \\\n'.join(self.BUILD_BUILD) + ' && \\\n'
         ans += self.FREEZE_TEMPLATE.format(freeze_command=self.FREEZE_COMMAND) + '\n'
-        ans = ans.format(project=PROJECT, host=HOST)
+        ans = ans.format(project=PROJECT, host=BUILD_HOST)
         return ans
 
-    def vmware_started(self):
-        return 'started' in subprocess.Popen('/etc/init.d/vmware status', shell=True, stdout=subprocess.PIPE).stdout.read()
-
-    def start_vmware(self):
-        if not self.vmware_started():
-            if os.path.exists('/dev/kvm'):
-                subprocess.check_call('sudo rmmod -w kvm-intel kvm', shell=True)
-            subprocess.Popen('sudo /etc/init.d/vmware start', shell=True)
-
-    def stop_vmware(self):
-            while True:
-                try:
-                    subprocess.check_call('sudo /etc/init.d/vmware stop', shell=True)
-                    break
-                except:
-                    pass
-            while 'vmblock' in open('/proc/modules').read():
-                subprocess.check_call('sudo rmmod -f vmblock')
-
     def run_vm(self):
-        if is_vm_running(self.VM_CHECK or self.VM_NAME):
+        if is_vm_running(self.VM_NAME):
-            return
+            return True
-        self.__p = subprocess.Popen([self.vm])
+        self.__p = subprocess.Popen(("VBoxManage startvm %s --type gui" % self.VM_NAME).split())
+        return False
 
     def start_vm(self, sleep=75):
         ssh_host = self.VM_NAME
-        self.run_vm()
+        already_running = self.run_vm()
+        if not already_running:
+            time.sleep(2)
+        print ('Waiting for SSH server to start')
+        while not is_host_reachable(ssh_host, timeout=1):
+            time.sleep(0.1)
+
+    def run_vm_builder(self):
+        ssh_host = self.VM_NAME
         build_script = self.get_build_script()
         t = tempfile.NamedTemporaryFile(suffix='.sh')
         t.write(build_script)
         t.flush()
-        print 'Waiting for VM to startup'
+        print ('Running VM builder')
-        while subprocess.call('ping -q -c1 '+ssh_host, shell=True,
-                   stdout=open('/dev/null', 'w')) != 0:
-            time.sleep(5)
-        time.sleep(self.EXTRA_SLEEP)
-        print 'Trying to SSH into VM'
         subprocess.check_call(('scp', t.name, ssh_host+':build-calibre'))
         subprocess.check_call(self.BUILD_CMD%ssh_host, shell=True)
+        self.download_installer()
 
     def installer(self):
         return installer_name(self.INSTALLER_EXT, self.IS_64_BIT)
 
     def run(self, opts):
-        for x in ('dont_shutdown', 'vm'):
-            setattr(self, x, getattr(opts, x))
-        if self.vm is None:
-            self.vm = self.VM
-        if not self.vmware_started():
-            self.start_vmware()
         subprocess.call(['chmod', '-R', '+r', 'recipes'])
         self.start_vm()
-        self.download_installer()
+        self.run_vm_builder()
-        if not self.dont_shutdown:
+        if not opts.dont_shutdown:
+            print ('Shutting down', self.VM_NAME)
             subprocess.call(['ssh', self.VM_NAME]+self.SHUTDOWN_CMD)
+            if self.FORCE_SHUTDOWN:
+                while is_host_reachable(self.VM_NAME):
+                    time.sleep(0.1)  # wait for SSH server to shutdown
+                time.sleep(self.FORCE_SHUTDOWN)
+                subprocess.check_call(('VBoxManage controlvm %s poweroff' % self.VM_NAME).split())
 
     def download_installer(self):
         installer = self.installer()

setup/installer/linux/__init__.py

@@ -15,8 +15,7 @@ class Linux32(VMInstaller):
     description = 'Build 32bit linux binary installer'
 
     INSTALLER_EXT = 'tar.bz2'
-    VM_NAME = 'gentoo32_build'
+    VM_NAME = 'linux32-build'
-    VM = '/vmware/bin/gentoo32_build'
     FREEZE_COMMAND = 'linux_freeze'
     FREEZE_TEMPLATE = 'sudo python -OO setup.py {freeze_command}'
 
@@ -24,8 +23,7 @@ class Linux32(VMInstaller):
 class Linux64(Linux32):
 
     description = 'Build 64bit linux binary installer'
-    VM_NAME = 'gentoo64_build'
+    VM_NAME = 'linux64-build'
-    VM = '/vmware/bin/gentoo64_build'
     IS_64_BIT = True
 
 class Linux(Command):

setup/installer/osx/__init__.py

@@ -14,9 +14,7 @@ class OSX(VMInstaller):
     description = 'Build OS X binary installer'
 
     INSTALLER_EXT = 'dmg'
-    VM_NAME = 'osx_build'
+    VM_NAME = 'osx-build'
-    VM = '/vmware/bin/%s'%VM_NAME
     FREEZE_TEMPLATE = 'python -OO setup.py {freeze_command}'
     FREEZE_COMMAND = 'osx32_freeze'
-    BUILD_PREFIX = VMInstaller.BUILD_PREFIX + ['source ~/.profile']
+    FORCE_SHUTDOWN = 10  # number of seconds to wait before doing a forced power off
-    SHUTDOWN_CMD = ['sudo', 'halt']

setup/installer/windows/__init__.py

@@ -8,7 +8,7 @@ __docformat__ = 'restructuredtext en'
 
 import os, shutil, subprocess
 
-from setup import Command, __appname__, __version__, installer_name
+from setup import Command, __appname__, __version__
 from setup.installer import VMInstaller
 
 class Win(Command):
@@ -30,14 +30,7 @@ class WinBase(VMInstaller):
 class Win32(WinBase):
 
     description = 'Build 32bit windows binary installer'
-
+    VM_NAME = 'win32-build'
-    VM_NAME = 'xp_build'
-    VM = '/vmware/bin/%s'%VM_NAME
-    VM_CHECK = 'calibre_windows_xp_home'
-
-    @property
-    def msi64(self):
-        return installer_name('msi', is64bit=True)
 
     def do_dl(self, installer, errmsg):
         subprocess.check_call(('scp',
@@ -59,15 +52,6 @@ class Win64(WinBase):
 
     description = 'Build 64bit windows binary installer'
 
-    VM_NAME = 'win64'
+    VM_NAME = 'win64-build'
-    VM = '/vmware/bin/%s'%VM_NAME
-    VM_CHECK = 'win64'
     IS_64_BIT = True
-    BUILD_PREFIX = WinBase.BUILD_PREFIX + [
-        'if [ -f "$HOME/.bash_profile" ] ; then',
-        '    source "$HOME/.bash_profile"',
-        'fi',
-    ]
-
-