calibre/setup/upload.py

#!/usr/bin/env python2
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai

__license__   = 'GPL v3'
__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'

import os, subprocess, hashlib, shutil, glob, stat, sys, time, shlex
from subprocess import check_call
from tempfile import NamedTemporaryFile, mkdtemp, gettempdir
from zipfile import ZipFile

if __name__ == '__main__':
    d = os.path.dirname
    sys.path.insert(0, d(d(os.path.abspath(__file__))))

from setup import Command, __version__, installer_name, __appname__

DOWNLOADS = '/srv/main/downloads'
HTML2LRF = "calibre/ebooks/lrf/html/demo"
TXT2LRF  = "src/calibre/ebooks/lrf/txt/demo"
STAGING_HOST = 'download.calibre-ebook.com'
STAGING_USER = 'root'
STAGING_DIR = '/root/staging'

def installers(include_source=True):
    installers = list(map(installer_name, ('dmg', 'msi', 'txz')))
    installers.append(installer_name('txz', is64bit=True))
    installers.append(installer_name('msi', is64bit=True))
    if include_source:
        installers.insert(0, 'dist/%s-%s.tar.xz'%(__appname__, __version__))
    installers.append('dist/%s-portable-installer-%s.exe'%(__appname__, __version__))
    return installers

def installer_description(fname):
    if fname.endswith('.tar.xz'):
        return 'Source code'
    if fname.endswith('.txz'):
        bits = '32' if 'i686' in fname else '64'
        return bits + 'bit Linux binary'
    if fname.endswith('.msi'):
        return 'Windows %sinstaller'%('64bit ' if '64bit' in fname else '')
    if fname.endswith('.dmg'):
        return 'OS X dmg'
    if fname.endswith('.exe'):
        return 'Calibre Portable'
    return 'Unknown file'

def upload_signatures():
    tdir = mkdtemp()
    for installer in installers():
        if not os.path.exists(installer):
            continue
        with open(installer, 'rb') as f:
            raw = f.read()
        fingerprint = hashlib.sha512(raw).hexdigest()
        fname = os.path.basename(installer+'.sha512')
        with open(os.path.join(tdir, fname), 'wb') as f:
            f.write(fingerprint)
    check_call('scp %s/*.sha512 code:/srv/code/signatures/' % tdir, shell=True)
    check_call(shlex.split('ssh code chown -R http:http /srv/code/signatures'))
    shutil.rmtree(tdir)

class ReUpload(Command):  # {{{

    description = 'Re-upload any installers present in dist/'

    sub_commands = ['upload_installers']

    def pre_sub_commands(self, opts):
        opts.replace = True
        exists = {x for x in installers() if os.path.exists(x)}
        if not exists:
            print ('There appear to be no installers!')
            raise SystemExit(1)

    def run(self, opts):
        for x in installers():
            if os.path.exists(x):
                os.remove(x)
# }}}

# Data {{{
def get_github_data():
    with open(os.path.expanduser('~/work/env/private/github'), 'rb') as f:
        un, pw = f.read().strip().split(':')
    return {
        'username':un, 'password':pw
    }

def get_sourceforge_data():
    return {'username':'kovidgoyal', 'project':'calibre'}

def send_data(loc):
    subprocess.check_call(['rsync', '--inplace', '--delete', '-r', '-z', '-h', '--progress', '-e', 'ssh -x',
        loc+'/', '%s@%s:%s'%(STAGING_USER, STAGING_HOST, STAGING_DIR)])

def gh_cmdline(ver, data):
    return [__appname__, ver, 'fmap', 'github', __appname__, data['username'], data['password']]

def sf_cmdline(ver, sdata):
    return [__appname__, ver, 'fmap', 'sourceforge', sdata['project'],
            sdata['username']]

def calibre_cmdline(ver):
    return [__appname__, ver, 'fmap', 'calibre']

def dbs_cmdline(ver):
    return [__appname__, ver, 'fmap', 'dbs']

def run_remote_upload(args):
    print 'Running remotely:', ' '.join(args)
    subprocess.check_call(['ssh', '-x', '%s@%s'%(STAGING_USER, STAGING_HOST),
        'cd', STAGING_DIR, '&&', 'python', 'hosting.py']+args)

# }}}

class UploadInstallers(Command):  # {{{

    def add_options(self, parser):
        parser.add_option('--replace', default=False, action='store_true', help='Replace existing installers')

    def run(self, opts):
        all_possible = set(installers())
        available = set(glob.glob('dist/*'))
        files = {x:installer_description(x) for x in
                all_possible.intersection(available)}
        for x in files:
            os.chmod(x, stat.S_IRUSR|stat.S_IWUSR|stat.S_IRGRP|stat.S_IROTH)
        sizes = {os.path.basename(x):os.path.getsize(x) for x in files}
        self.record_sizes(sizes)
        tdir = mkdtemp()
        backup = os.path.join('/mnt/external/calibre/%s' % __version__)
        if not os.path.exists(backup):
            os.mkdir(backup)
        try:
            self.upload_to_staging(tdir, backup, files)
            self.upload_to_calibre()
            if opts.replace:
                upload_signatures()
                check_call('ssh code /apps/update-calibre-version.py'.split())
            # self.upload_to_sourceforge()
            self.upload_to_dbs()
            self.upload_to_github(opts.replace)
        finally:
            shutil.rmtree(tdir, ignore_errors=True)

    def record_sizes(self, sizes):
        print ('\nRecording dist sizes')
        args = ['%s:%s:%s' % (__version__, fname, size) for fname, size in sizes.iteritems()]
        check_call(['ssh', 'code', '/usr/local/bin/dist_sizes'] + args)

    def upload_to_staging(self, tdir, backup, files):
        os.mkdir(tdir+'/dist')
        hosting = os.path.join(os.path.dirname(os.path.abspath(__file__)),
            'hosting.py')
        shutil.copyfile(hosting, os.path.join(tdir, 'hosting.py'))

        for f in files:
            for x in (tdir+'/dist', backup):
                dest = os.path.join(x, os.path.basename(f))
                shutil.copy2(f, x)
                os.chmod(dest, stat.S_IREAD|stat.S_IWRITE|stat.S_IRGRP|stat.S_IROTH)

        with open(os.path.join(tdir, 'fmap'), 'wb') as fo:
            for f, desc in files.iteritems():
                fo.write('%s: %s\n'%(f, desc))

        while True:
            try:
                send_data(tdir)
            except:
                print('\nUpload to staging failed, retrying in a minute')
                time.sleep(60)
            else:
                break

    def upload_to_github(self, replace):
        data = get_github_data()
        args = gh_cmdline(__version__, data)
        if replace:
            args = ['--replace'] + args
        run_remote_upload(args)

    def upload_to_sourceforge(self):
        sdata = get_sourceforge_data()
        args = sf_cmdline(__version__, sdata)
        run_remote_upload(args)

    def upload_to_calibre(self):
        run_remote_upload(calibre_cmdline(__version__))

    def upload_to_dbs(self):
        run_remote_upload(dbs_cmdline(__version__))
# }}}

class UploadUserManual(Command):  # {{{
    description = 'Build and upload the User Manual'
    sub_commands = ['manual']

    def build_plugin_example(self, path):
        from calibre import CurrentDir
        with NamedTemporaryFile(suffix='.zip') as f:
            os.fchmod(f.fileno(),
                stat.S_IRUSR|stat.S_IRGRP|stat.S_IROTH|stat.S_IWRITE)
            with CurrentDir(path):
                with ZipFile(f, 'w') as zf:
                    for x in os.listdir('.'):
                        if x.endswith('.swp'):
                            continue
                        zf.write(x)
                        if os.path.isdir(x):
                            for y in os.listdir(x):
                                zf.write(os.path.join(x, y))
            bname = self.b(path) + '_plugin.zip'
            dest = '%s/%s'%(DOWNLOADS, bname)
            subprocess.check_call(['scp', f.name, 'main:'+dest])

    def run(self, opts):
        path = self.j(self.SRC, '..', 'manual', 'plugin_examples')
        for x in glob.glob(self.j(path, '*')):
            self.build_plugin_example(x)

        srcdir = self.j(gettempdir(), 'user-manual-build', 'en', 'html') + '/'
        for host in ('download', 'files'):
            check_call(' '.join(['rsync', '-zrl', '--progress',
                srcdir, '%s:/srv/manual/' % host]), shell=True)
# }}}

class UploadDemo(Command):  # {{{

    description = 'Rebuild and upload various demos'

    def run(self, opts):
        check_call(
           '''ebook-convert %s/demo.html /tmp/html2lrf.lrf '''
           '''--title='Demonstration of html2lrf' --authors='Kovid Goyal' '''
           '''--header '''
           '''--serif-family "/usr/share/fonts/corefonts, Times New Roman" '''
           '''--mono-family  "/usr/share/fonts/corefonts, Andale Mono" '''
           ''''''%self.j(self.SRC, HTML2LRF), shell=True)

        lrf = self.j(self.SRC, 'calibre', 'ebooks', 'lrf', 'html', 'demo')
        check_call(
            'cd %s && zip -j /tmp/html-demo.zip * /tmp/html2lrf.lrf' % lrf, shell=True)

        check_call('scp /tmp/html-demo.zip main:%s/'%(DOWNLOADS,), shell=True)
# }}}

class UploadToServer(Command):  # {{{

    description = 'Upload miscellaneous data to calibre server'

    def run(self, opts):
        src_file = glob.glob('dist/calibre-*.tar.xz')[0]
        upload_signatures()
        check_call(['git', 'push'])
        check_call(['/home/kovid/work/env/private/gpg-as-kovid', '--armor', '--yes', '--detach-sign', src_file])
        check_call(['scp', src_file + '.asc', 'code:/srv/code/signatures/'])
        check_call('ssh code /usr/local/bin/update-calibre-code.py'.split())
        check_call(('ssh code /apps/update-calibre-version.py ' + __version__).split())
        check_call('ssh main /usr/local/bin/update-calibre-code.py && /apps/static/generate.py'.split())
# }}}

# Testing {{{

def write_files(fmap):
    for f in fmap:
        with open(f, 'wb') as f:
            f.write(os.urandom(100))
            f.write(b'a'*1000000)
    with open('fmap', 'wb') as fo:
        for f, desc in fmap.iteritems():
            fo.write('%s: %s\n'%(f, desc))

def setup_installers():
    ver = '0.0.1'
    files = {x.replace(__version__, ver):installer_description(x) for x in installers()}
    tdir = mkdtemp()
    os.chdir(tdir)
    return tdir, files, ver

# }}}