calibre/setup/installer/__init__.py

#!/usr/bin/env python2
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
from __future__ import (unicode_literals, division, absolute_import,
                        print_function)

__license__   = 'GPL v3'
__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'

import subprocess, os, time, socket

from setup import Command, installer_name
from setup.build_environment import BUILD_HOST, PROJECT

BASE_RSYNC = ['rsync', '-av', '--delete', '--force']
EXCLUDES = []
for x in [
    '/src/calibre/plugins', '/manual', '/translations', '/build', '/dist', '/imgsrc', '/format_docs'
    '.bzr', '.git', '.build', '.svn',  '*.pyc', '*.pyo', '*.swp', '*.swo',]:
    EXCLUDES.extend(['--exclude', ('/calibre' + x) if x.startswith('/') else x])
SAFE_EXCLUDES = ['"%s"'%x if '*' in x else x for x in EXCLUDES]

def get_rsync_pw():
    return open('/home/kovid/work/env/private/buildbot').read().decode('utf-8').partition(
                ':')[-1].strip()

def is_vm_running(name):
    qname = '"%s"' % name
    try:
        lines = subprocess.check_output('VBoxManage list runningvms'.split()).decode('utf-8').splitlines()
    except Exception:
        time.sleep(1)
        lines = subprocess.check_output('VBoxManage list runningvms'.split()).decode('utf-8').splitlines()
    for line in lines:
        if line.startswith(qname):
            return True
    return False

def is_host_reachable(name, timeout=1):
    try:
        socket.create_connection((name, 22), timeout).close()
        return True
    except:
        return False

class Rsync(Command):

    description = 'Sync source tree from development machine'

    SYNC_CMD = ' '.join(BASE_RSYNC+SAFE_EXCLUDES+
            ['rsync://buildbot@{host}/work/{project}', '..'])

    def run(self, opts):
        cmd = self.SYNC_CMD.format(host=BUILD_HOST, project=PROJECT)
        env = dict(os.environ)
        env['RSYNC_PASSWORD'] = get_rsync_pw()
        self.info(cmd)
        subprocess.check_call(cmd, shell=True, env=env)

def push(host, vmname, available):
    if vmname is None:
        hostname = host.partition(':')[0].partition('@')[-1]
        ok = is_host_reachable(hostname)
    else:
        ok = is_vm_running(vmname)
    if ok:
        available[vmname or host] = True
        rcmd = BASE_RSYNC + EXCLUDES + ['.', host]
        print ('\n\nPushing to:', vmname or host, '\n')
        subprocess.check_call(rcmd, stdout=open(os.devnull, 'wb'))

class Push(Command):

    description = 'Push code to another host'

    def run(self, opts):
        from threading import Thread
        threads, available = {}, {}
        for host, vmname in {
                r'Owner@winxp:/cygdrive/c/Documents\ and\ Settings/Owner/calibre':'winxp',
                'kovid@ox:calibre':None,
                r'kovid@win7:/cygdrive/c/Users/kovid/calibre':'Windows 7',
                'kovid@win7-x64:calibre':'win7-x64',
                'kovid@tiny:calibre':None,
                'kovid@getafix:calibre-src':None,
                }.iteritems():
                threads[vmname or host] = thread = Thread(target=push, args=(host, vmname, available))
                thread.start()
        while threads:
            for name, thread in tuple(threads.iteritems()):
                thread.join(0.01)
                if not thread.is_alive():
                    if available.get(name, False):
                        print ('\n\n', name, 'done')
                    threads.pop(name)


class VMInstaller(Command):

    INSTALLER_EXT = None
    VM_NAME = None
    FREEZE_COMMAND = None
    FREEZE_TEMPLATE = 'python setup.py {freeze_command}'
    SHUTDOWN_CMD = ['sudo', 'shutdown', '-h', 'now']
    IS_64_BIT = False

    BUILD_PREFIX = ['#!/bin/sh', 'export CALIBRE_BUILDBOT=1']
    BUILD_RSYNC  = ['mkdir -p ~/build/{project}', r'cd ~/build/{project}', Rsync.SYNC_CMD]
    BUILD_CLEAN  = ['rm -rf dist/* build/* src/calibre/plugins/*']
    BUILD_BUILD  = ['python setup.py build',]
    FORCE_SHUTDOWN = 0  # number of seconds to wait before doing a forced power off (0 means disabled)

    def add_options(self, parser):
        if not parser.has_option('--dont-shutdown'):
            parser.add_option('-s', '--dont-shutdown', default=False,
                action='store_true', help='Dont shutdown the VM after building')
        if not parser.has_option('--dont-strip'):
            parser.add_option('-x', '--dont-strip', default=False,
                action='store_true', help='Dont strip the generated binaries')

    def get_build_script(self):
        rs = ['export RSYNC_PASSWORD=%s'%get_rsync_pw()]
        ans = '\n'.join(self.BUILD_PREFIX + rs)+'\n\n'
        ans += ' && \\\n'.join(self.BUILD_RSYNC) + ' && \\\n'
        ans += ' && \\\n'.join(self.BUILD_CLEAN) + ' && \\\n'
        ans += ' && \\\n'.join(self.BUILD_BUILD) + ' && \\\n'
        ans += self.FREEZE_TEMPLATE.format(freeze_command=self.FREEZE_COMMAND) + '\n'
        ans = ans.format(project=PROJECT, host=BUILD_HOST)
        return ans

    def run_vm(self):
        if is_vm_running(self.VM_NAME):
            return True
        self.__p = subprocess.Popen(("VBoxManage startvm %s --type gui" % self.VM_NAME).split())
        return False

    def start_vm(self, sleep=75):
        ssh_host = self.VM_NAME
        already_running = self.run_vm()
        if not already_running:
            time.sleep(2)
        print ('Waiting for SSH server to start')
        while not is_host_reachable(ssh_host, timeout=1):
            time.sleep(0.1)

    def run_vm_builder(self):
        ssh_host = self.VM_NAME
        build_script = self.get_build_script()
        build_script = build_script.encode('utf-8') if isinstance(build_script, unicode) else build_script
        print ('Running VM builder')
        p = subprocess.Popen(['ssh', ssh_host, 'bash -s'], stdin=subprocess.PIPE)
        p.stdin.write(build_script), p.stdin.flush(), p.stdin.close()
        # Save the build script on the build machine for convenient manual
        # invocation, if needed
        p2 = subprocess.Popen(['ssh', ssh_host, 'cat > build-calibre'], stdin=subprocess.PIPE)
        p2.stdin.write(build_script), p2.stdin.flush(), p2.stdin.close()
        p2.wait()
        rc = p.wait()
        if rc != 0:
            raise SystemExit('VM builder failed with error code: %s' % rc)
        self.download_installer()

    def installer(self):
        return installer_name(self.INSTALLER_EXT, self.IS_64_BIT)

    def run(self, opts):
        if opts.dont_strip:
            self.FREEZE_COMMAND += ' --dont-strip'
        subprocess.call(['chmod', '-R', '+r', 'recipes'])
        start_time = time.time()
        self.start_vm()
        startup_time = time.time() - start_time
        start_time = time.time()
        self.run_vm_builder()
        print ('Startup completed in %d seconds' % round(startup_time))
        secs = time.time() - start_time
        print ('Build completed in %d minutes %d seconds' % (secs // 60, secs % 60))
        if not opts.dont_shutdown:
            print ('Shutting down', self.VM_NAME)
            subprocess.call(['ssh', self.VM_NAME]+self.SHUTDOWN_CMD)
            if self.FORCE_SHUTDOWN:
                while is_host_reachable(self.VM_NAME):
                    time.sleep(0.1)  # wait for SSH server to shutdown
                time.sleep(self.FORCE_SHUTDOWN)
                subprocess.check_call(('VBoxManage controlvm %s poweroff' % self.VM_NAME).split())

    def download_installer(self):
        installer = self.installer()
        subprocess.check_call(['scp',
            self.VM_NAME+':build/calibre/'+installer, 'dist'])
        if not os.path.exists(installer):
            raise SystemExit('Failed to download installer: '+installer)

    def clean(self):
        installer = self.installer()
        if os.path.exists(installer):
            os.remove(installer)