diff --git a/setup/hosting.py b/setup/hosting.py index 8707388181..9853f181a4 100644 --- a/setup/hosting.py +++ b/setup/hosting.py @@ -10,13 +10,13 @@ __docformat__ = 'restructuredtext en' import os, time, sys, traceback, subprocess, urllib2, re, base64, httplib from argparse import ArgumentParser, FileType from subprocess import check_call -from tempfile import NamedTemporaryFile#, mkdtemp +from tempfile import NamedTemporaryFile from collections import OrderedDict import mechanize from lxml import html -def login_to_google(username, password): +def login_to_google(username, password): # {{{ br = mechanize.Browser() br.addheaders = [('User-agent', 'Mozilla/5.0 (X11; Linux x86_64; rv:9.0) Gecko/20100101 Firefox/9.0')] @@ -30,15 +30,16 @@ def login_to_google(username, password): x = re.search(br'(?is).*?', raw) if x is not None: print ('Title of post login page: %s'%x.group()) - #open('/tmp/goog.html', 'wb').write(raw) + # open('/tmp/goog.html', 'wb').write(raw) raise ValueError(('Failed to login to google with credentials: %s %s' '\nGoogle sometimes requires verification when logging in from a ' 'new IP address. Use lynx to login and supply the verification, ' 'at: lynx -accept_all_cookies https://accounts.google.com/ServiceLogin?service=code') %(username, password)) return br +# }}} -class ReadFileWithProgressReporting(file): # {{{ +class ReadFileWithProgressReporting(file): # {{{ def __init__(self, path, mode='rb'): file.__init__(self, path, mode) @@ -101,7 +102,7 @@ class Base(object): # {{{ #}}} -class GoogleCode(Base):# {{{ +class GoogleCode(Base): # {{{ def __init__(self, # A mapping of filenames to file descriptions. The descriptions are @@ -141,7 +142,7 @@ class GoogleCode(Base):# {{{ # The pattern to match filenames for the files being uploaded and # extract version information from them. Must have a named group # named version - filename_pattern=r'{appname}-(?:portable-installer-)?(?P.+?)(?:-(?:i686|x86_64|32bit|64bit))?\.(?:zip|exe|msi|dmg|tar\.bz2|tar\.xz|txz|tbz2)' + filename_pattern=r'{appname}-(?:portable-installer-)?(?P.+?)(?:-(?:i686|x86_64|32bit|64bit))?\.(?:zip|exe|msi|dmg|tar\.bz2|tar\.xz|txz|tbz2)' # noqa ): self.username, self.password, = username, password @@ -227,7 +228,8 @@ class GoogleCode(Base):# {{{ paths = eval(raw) if raw else {} paths.update(self.paths) rem = [x for x in paths if self.version not in x] - for x in rem: paths.pop(x) + for x in rem: + paths.pop(x) raw = ['%r : %r,'%(k, v) for k, v in paths.items()] raw = '{\n\n%s\n\n}\n'%('\n'.join(raw)) with NamedTemporaryFile() as t: @@ -347,7 +349,7 @@ class GoogleCode(Base):# {{{ # }}} -class SourceForge(Base): # {{{ +class SourceForge(Base): # {{{ # Note that you should manually ssh once to username,project@frs.sourceforge.net # on the staging server so that the host key is setup @@ -378,6 +380,28 @@ class SourceForge(Base): # {{{ # }}} +def upload_to_servers(files, version): # {{{ + for server, rdir in {'files':'/usr/share/nginx/html'}.iteritems(): + print('Uploading to server:', server) + server = '%s.calibre-ebook.com' % server + rdir = '%s/%s/' % (rdir, version) + for x in files: + start = time.time() + print ('Uploading', x) + for i in range(5): + try: + check_call(['rsync', '-h', '-z', '--progress', '-e', 'ssh -x', x, + 'root@%s:%s'%(server, rdir)]) + except KeyboardInterrupt: + raise SystemExit(1) + except: + print ('\nUpload failed, trying again in 30 seconds') + time.sleep(30) + else: + break + print ('Uploaded in', int(time.time() - start), 'seconds\n\n') +# }}} + # CLI {{{ def cli_parser(): epilog='Copyright Kovid Goyal 2012' @@ -409,6 +433,7 @@ def cli_parser(): sf = subparsers.add_parser('sourceforge', help='Upload to sourceforge', epilog=epilog) cron = subparsers.add_parser('cron', help='Call script from cron') + subparsers.add_parser('calibre', help='Upload to calibre file servers') a = gc.add_argument @@ -471,8 +496,11 @@ def main(args=None): sf() elif args.service == 'cron': login_to_google(args.username, args.password) + elif args.service == 'calibre': + upload_to_servers(ofiles, args.version) if __name__ == '__main__': main() # }}} + diff --git a/setup/upload.py b/setup/upload.py index 673f9f4679..1c7348bfe9 100644 --- a/setup/upload.py +++ b/setup/upload.py @@ -81,7 +81,7 @@ class ReUpload(Command): # {{{ # Data {{{ def get_google_data(): - with open(os.path.expanduser('~/work/kde/conf/googlecodecalibre'), 'rb') as f: + with open(os.path.expanduser('~/work/env/private/googlecodecalibre'), 'rb') as f: gc_password, ga_un, pw = f.read().strip().split('|') return { @@ -111,6 +111,9 @@ def sf_cmdline(ver, sdata): return [__appname__, ver, 'fmap', 'sourceforge', sdata['project'], sdata['username']] +def calibre_cmdline(ver): + return [__appname__, ver, 'fmap', 'calibre'] + def run_remote_upload(args): print 'Running remotely:', ' '.join(args) subprocess.check_call(['ssh', '-x', '%s@%s'%(STAGING_USER, STAGING_HOST), @@ -133,7 +136,8 @@ class UploadInstallers(Command): # {{{ try: self.upload_to_staging(tdir, files) self.upload_to_sourceforge() - self.upload_to_google(opts.replace) + self.upload_to_calibre() + # self.upload_to_google(opts.replace) finally: shutil.rmtree(tdir, ignore_errors=True) @@ -170,6 +174,10 @@ class UploadInstallers(Command): # {{{ sdata = get_sourceforge_data() args = sf_cmdline(__version__, sdata) run_remote_upload(args) + + def upload_to_calibre(self): + run_remote_upload(calibre_cmdline(__version__)) + # }}} class UploadUserManual(Command): # {{{