From a3f353853a0b243be8ede49de9ef6275c3bed694 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 3 Jan 2012 15:42:57 +0530 Subject: [PATCH] Change upload installers code to work via a staging server with more upload bandwidth than my current DSL connection --- setup/commands.py | 12 +- setup/hosting.py | 459 ++++++++++++++++++++++++++++++++++++++++++ setup/publish.py | 2 +- setup/upload.py | 493 +++++++++++++--------------------------------- 4 files changed, 602 insertions(+), 364 deletions(-) create mode 100644 setup/hosting.py diff --git a/setup/commands.py b/setup/commands.py index e8ac6d8e42..9fbc048254 100644 --- a/setup/commands.py +++ b/setup/commands.py @@ -16,8 +16,8 @@ __all__ = [ 'sdist', 'manual', 'tag_release', 'pypi_register', 'pypi_upload', 'upload_to_server', - 'upload_user_manual', 'upload_to_mobileread', 'upload_demo', - 'upload_to_sourceforge', 'upload_to_google_code', 'reupload', + 'upload_installers', + 'upload_user_manual', 'upload_demo', 'reupload', 'linux32', 'linux64', 'linux', 'linux_freeze', 'osx32_freeze', 'osx', 'rsync', 'push', 'win32_freeze', 'win32', 'win', @@ -65,14 +65,12 @@ stage4 = Stage4() stage5 = Stage5() publish = Publish() -from setup.upload import UploadUserManual, UploadInstallers, UploadDemo, \ - UploadToServer, UploadToSourceForge, UploadToGoogleCode, ReUpload +from setup.upload import (UploadUserManual, UploadDemo, UploadInstallers, + UploadToServer, ReUpload) upload_user_manual = UploadUserManual() -upload_to_mobileread = UploadInstallers() upload_demo = UploadDemo() upload_to_server = UploadToServer() -upload_to_sourceforge = UploadToSourceForge() -upload_to_google_code = UploadToGoogleCode() +upload_installers = UploadInstallers() reupload = ReUpload() from setup.installer import Rsync, Push diff --git a/setup/hosting.py b/setup/hosting.py new file mode 100644 index 0000000000..d86cef2d31 --- /dev/null +++ b/setup/hosting.py @@ -0,0 +1,459 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import os, time, sys, traceback, subprocess, urllib2, re, base64, httplib +from argparse import ArgumentParser, FileType +from subprocess import check_call +from tempfile import NamedTemporaryFile#, mkdtemp +from collections import OrderedDict + +import mechanize +from lxml import html + +def login_to_google(username, password): + br = mechanize.Browser() + br.addheaders = [('User-agent', + 'Mozilla/5.0 (X11; Linux x86_64; rv:9.0) Gecko/20100101 Firefox/9.0')] + br.set_handle_robots(False) + br.open('https://accounts.google.com/ServiceLogin?service=code') + br.select_form(nr=0) + br.form['Email'] = username + br.form['Passwd'] = password + raw = br.submit().read() + if b'Account overview - Account Settings' not in raw: + raise ValueError(('Failed to login to google with credentials: %s %s' + '\nGoogle sometimes requires verification when logging in from a ' + 'new IP address. Use lynx to login and supply the verification.') + %(username, password)) + return br + +class ReadFileWithProgressReporting(file): # {{{ + + def __init__(self, path, mode='rb'): + file.__init__(self, path, mode) + self.seek(0, os.SEEK_END) + self._total = self.tell() + self.seek(0) + self.start_time = time.time() + + def __len__(self): + return self._total + + def read(self, size): + data = file.read(self, size) + if data: + self.report_progress(len(data)) + return data + + def report_progress(self, size): + sys.stdout.write(b'\x1b[s') + sys.stdout.write(b'\x1b[K') + frac = float(self.tell())/self._total + mb_pos = self.tell()/float(1024**2) + mb_tot = self._total/float(1024**2) + kb_pos = self.tell()/1024.0 + kb_rate = kb_pos/(time.time()-self.start_time) + bit_rate = kb_rate * 1024 + eta = int((self._total - self.tell())/bit_rate) + 1 + eta_m, eta_s = eta / 60, eta % 60 + sys.stdout.write( + ' %.1f%% %.1f/%.1fMB %.1f KB/sec %d minutes, %d seconds left'%( + frac*100, mb_pos, mb_tot, kb_rate, eta_m, eta_s)) + sys.stdout.write(b'\x1b[u') + if self.tell() >= self._total: + sys.stdout.write('\n') + t = int(time.time() - self.start_time) + 1 + print ('Upload took %d minutes and %d seconds at %.1f KB/sec' % ( + t/60, t%60, kb_rate)) + sys.stdout.flush() +# }}} + +class Base(object): # {{{ + + def __init__(self): + self.d = os.path.dirname + self.j = os.path.join + self.a = os.path.abspath + self.b = os.path.basename + self.s = os.path.splitext + self.e = os.path.exists + + def info(self, *args, **kwargs): + print(*args, **kwargs) + sys.stdout.flush() + + def warn(self, *args, **kwargs): + print('\n'+'_'*20, 'WARNING','_'*20) + print(*args, **kwargs) + print('_'*50) + sys.stdout.flush() + +#}}} + +class GoogleCode(Base):# {{{ + + def __init__(self, + # A mapping of filenames to file descriptions. The descriptions are + # used to populate the description field for the upload on google + # code + files, + + # The unix name for the application. + appname, + + # The version being uploaded + version, + + # Google account username + username, + + # Googlecode.com password + password, + + # Google account password + gmail_password, + + # The name of the google code project we are uploading to + gc_project, + + # Server to which to upload the mapping of file names to google + # code URLs. If not None, upload is performed via shelling out to + # ssh, so you must have ssh-agent setup with the authenticated key + # and ssh agent forwarding enabled + gpaths_server=None, + # The path on gpaths_server to which to upload the mapping data + gpaths=None, + + # If True, files are replaced, otherwise existing files are skipped + reupload=False, + + # The pattern to match filenames for the files being uploaded and + # extract version information from them. Must have a named group + # named version + filename_pattern=r'{appname}-(?:portable-)?(?P.+?)(?:-(?:i686|x86_64|32bit|64bit))?\.(?:zip|exe|msi|dmg|tar\.bz2|tar\.xz|txz|tbz2)' + + ): + self.username, self.password, = username, password + self.gmail_password, self.gc_project = gmail_password, gc_project + self.reupload, self.files, self.version = reupload, files, version + self.gpaths, self.gpaths_server = gpaths, gpaths_server + + self.upload_host = '%s.googlecode.com'%gc_project + self.files_list = 'http://code.google.com/p/%s/downloads/list'%gc_project + self.delete_url = 'http://code.google.com/p/%s/downloads/delete?name=%%s'%gc_project + + self.filename_pat = re.compile(filename_pattern.format(appname=appname)) + for x in self.files: + if self.filename_pat.match(os.path.basename(x)) is None: + raise ValueError(('The filename %s does not match the ' + 'filename pattern')%os.path.basename(x)) + + def upload_one(self, fname, retries=2): + self.info('\nUploading', fname) + typ = 'Type-' + ('Source' if fname.endswith('.xz') else 'Archive' if + fname.endswith('.zip') else 'Installer') + ext = os.path.splitext(fname)[1][1:] + op = 'OpSys-'+{'msi':'Windows','zip':'Windows', + 'dmg':'OSX','bz2':'Linux','xz':'All'}[ext] + desc = self.files[fname] + start = time.time() + for i in range(retries): + try: + path = self.upload(os.path.abspath(fname), desc, + labels=[typ, op, 'Featured'], retry=100) + except KeyboardInterrupt: + raise SystemExit(1) + except: + traceback.print_exc() + print ('\nUpload failed, trying again in 30 secs.', + '%d retries left.'%(retries-1)) + time.sleep(30) + else: + break + self.info('Uploaded to:', path, 'in', int(time.time() - start), + 'seconds') + return path + + def re_upload(self): + fnames = {os.path.basename(x):x for x in self.files} + existing = self.old_files.intersection(set(fnames)) + br = self.login_to_google() + for x, src in fnames.iteritems(): + if not os.access(src, os.R_OK): + continue + if x in existing: + self.info('Deleting', x) + br.open(self.delete_url%x) + br.select_form(predicate=lambda y: 'delete.do' in y.action) + br.form.find_control(name='delete') + br.submit(name='delete') + self.upload_one(src) + + def __call__(self): + self.paths = {} + self.old_files = self.get_old_files() + if self.reupload: + return self.re_upload() + + for fname in self.files: + bname = os.path.basename(fname) + if bname in self.old_files: + path = 'http://%s.googlecode.com/files/%s'%(self.gc_project, + bname) + self.info( + '%s already uploaded, skipping. Assuming URL is: %s'%( + bname, path)) + self.old_files.remove(bname) + else: + path = self.upload_one(fname) + self.paths[bname] = path + self.info('Updating path map') + for k, v in self.paths.iteritems(): + self.info('\t%s => %s'%(k, v)) + if self.gpaths and self.gpaths_server: + raw = subprocess.Popen(['ssh', self.gpaths_server, 'cat', self.gpaths], + stdout=subprocess.PIPE).stdout.read() + paths = eval(raw) if raw else {} + paths.update(self.paths) + rem = [x for x in paths if self.version not in x] + for x in rem: paths.pop(x) + raw = ['%r : %r,'%(k, v) for k, v in paths.items()] + raw = '{\n\n%s\n\n}\n'%('\n'.join(raw)) + with NamedTemporaryFile() as t: + t.write(raw) + t.flush() + check_call(['scp', t.name, '%s:%s'%(self.gpaths_server, + self.gpaths)]) + if self.old_files: + self.br = self.login_to_google() + self.delete_old_files() + + def login_to_google(self): + self.info('Logging into Google') + return login_to_google(self.username, self.gmail_password) + + def get_files_hosted_by_google_code(self): + self.info('Getting existing files in google code:', self.gc_project) + raw = urllib2.urlopen(self.files_list).read() + root = html.fromstring(raw) + ans = {} + for a in root.xpath('//td[@class="vt id col_0"]/a[@href]'): + ans[a.text.strip()] = a.get('href') + return ans + + def get_old_files(self): + ans = set() + for fname in self.get_files_hosted_by_google_code(): + m = self.filename_pat.match(fname) + if m is not None: + ans.add(fname) + return ans + + def delete_old_files(self): + if not self.old_files: + return + self.info('Deleting old files from Google Code...') + for fname in self.old_files: + self.info('\tDeleting', fname) + self.br.open(self.delete_url%fname) + self.br.select_form(predicate=lambda x: 'delete.do' in x.action) + self.br.form.find_control(name='delete') + self.br.submit(name='delete') + + def encode_upload_request(self, fields, file_path): + BOUNDARY = '----------Googlecode_boundary_reindeer_flotilla' + + body = [] + + # Add the metadata about the upload first + for key, value in fields: + body.extend( + ['--' + BOUNDARY, + 'Content-Disposition: form-data; name="%s"' % key, + '', + value, + ]) + + # Now add the file itself + file_name = os.path.basename(file_path) + with open(file_path, 'rb') as f: + file_content = f.read() + + body.extend( + ['--' + BOUNDARY, + 'Content-Disposition: form-data; name="filename"; filename="%s"' + % file_name, + # The upload server determines the mime-type, no need to set it. + 'Content-Type: application/octet-stream', + '', + file_content, + ]) + + # Finalize the form body + body.extend(['--' + BOUNDARY + '--', '']) + body = [x.encode('ascii') if isinstance(x, unicode) else x for x in + body] + + return ('multipart/form-data; boundary=%s' % BOUNDARY, + b'\r\n'.join(body)) + + def upload(self, fname, desc, labels=[], retry=0): + form_fields = [('summary', desc)] + form_fields.extend([('label', l.strip()) for l in labels]) + + content_type, body = self.encode_upload_request(form_fields, fname) + upload_uri = '/files' + auth_token = base64.b64encode('%s:%s'% (self.username, self.password)) + headers = { + 'Authorization': 'Basic %s' % auth_token, + 'User-Agent': 'googlecode.com uploader v1', + 'Content-Type': content_type, + } + + with NamedTemporaryFile(delete=False) as f: + f.write(body) + + try: + body = ReadFileWithProgressReporting(f.name) + server = httplib.HTTPSConnection(self.upload_host) + server.request('POST', upload_uri, body, headers) + resp = server.getresponse() + server.close() + finally: + os.remove(f.name) + + if resp.status == 201: + return resp.getheader('Location') + + print ('Failed to upload with code %d and reason: %s'%(resp.status, + resp.reason)) + if retry < 1: + print ('Retrying in 5 seconds....') + time.sleep(5) + return self.upload(fname, desc, labels=labels, retry=retry+1) + raise Exception('Failed to upload '+fname) + + +# }}} + +class SourceForge(Base): # {{{ + + def __init__(self, files, project, version, username, replace=False): + self.username, self.project, self.version = username, project, version + self.base = '/home/frs/project/c/ca/'+project + self.rdir = self.base + '/' + version + self.files = files + + def __call__(self): + for x in self.files: + start = time.time() + self.info('Uploading', x) + for i in range(5): + try: + check_call(['rsync', '-h', '-z', '--progress', '-e', 'ssh -x', x, + '%s,%s@frs.sourceforge.net:%s'%(self.username, self.project, + self.rdir+'/')]) + except KeyboardInterrupt: + raise SystemExit(1) + except: + print ('\nUpload failed, trying again in 30 seconds') + time.sleep(30) + else: + break + print ('Uploaded in', int(time.time() - start), 'seconds\n\n') + +# }}} + +# CLI {{{ +def cli_parser(): + epilog='Copyright Kovid Goyal 2012' + + p = ArgumentParser( + description='Upload project files to a hosting service automatically', + epilog=epilog + ) + a = p.add_argument + a('appname', help='The name of the application, all files to' + ' upload should begin with this name') + a('version', help='The version of the application, all files to' + ' upload should contain this version') + a('file_map', type=FileType('rb'), + help='A file containing a mapping of files to be uploaded to ' + 'descriptions of the files. The descriptions will be visible ' + 'to users trying to get the file from the hosting service. ' + 'The format of the file is filename: description, with one per ' + 'line. filename can be a path to the file relative to the current ' + 'directory.') + a('--replace', action='store_true', default=False, + help='If specified, existing files are replaced, otherwise ' + 'they are skipped.') + + subparsers = p.add_subparsers(help='Where to upload to', dest='service', + title='Service', description='Hosting service to upload to') + gc = subparsers.add_parser('googlecode', help='Upload to googlecode', + epilog=epilog) + sf = subparsers.add_parser('sourceforge', help='Upload to sourceforge', + epilog=epilog) + a = gc.add_argument + + a('project', + help='The name of the project on google code we are uploading to') + a('username', + help='Username to log into your google account') + a('password', + help='Password to log into your google account') + a('gc_password', + help='Password for google code hosting.' + ' Get it from http://code.google.com/hosting/settings') + + a('--path-map-server', + help='A server to which the mapping of filenames to googlecode ' + 'URLs will be uploaded. The upload happens via ssh, so you must ' + 'have a working ssh agent') + a('--path-map-location', + help='Path on the server where the path map is placed.') + + a = sf.add_argument + a('project', + help='The name of the project on sourceforge we are uploading to') + a('username', + help='Sourceforge username') + + return p + +def main(args=None): + cli = cli_parser() + args = cli.parse_args(args) + files = {} + with args.file_map as f: + for line in f: + fname, _, desc = line.partition(':') + fname, desc = fname.strip(), desc.strip() + if fname and desc: + files[fname] = desc + + ofiles = OrderedDict() + for x in sorted(files, key=lambda x:os.stat(x).st_size, reverse=True): + ofiles[x] = files[x] + + if args.service == 'googlecode': + gc = GoogleCode(ofiles, args.appname, args.version, args.username, + args.gc_password, args.password, args.project, + gpaths_server=args.path_map_server, + gpaths=args.path_map_location, reupload=args.replace) + gc() + elif args.service == 'sourceforge': + sf = SourceForge(ofiles, args.project, args.version, args.username, + replace=args.replace) + sf() + +if __name__ == '__main__': + main() +# }}} + diff --git a/setup/publish.py b/setup/publish.py index 26769f271e..72bf6c9cbb 100644 --- a/setup/publish.py +++ b/setup/publish.py @@ -45,7 +45,7 @@ class Stage3(Command): class Stage4(Command): description = 'Stage 4 of the publish process' - sub_commands = ['upload_to_sourceforge', 'upload_to_google_code'] + sub_commands = ['upload_installers'] class Stage5(Command): diff --git a/setup/upload.py b/setup/upload.py index fae0f1a3cc..d1ab1f3c85 100644 --- a/setup/upload.py +++ b/setup/upload.py @@ -5,12 +5,15 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, re, cStringIO, base64, httplib, subprocess, hashlib, shutil, time, \ - glob, stat, sys +import os, re, subprocess, hashlib, shutil, glob, stat, sys from subprocess import check_call from tempfile import NamedTemporaryFile, mkdtemp from zipfile import ZipFile +if __name__ == '__main__': + d = os.path.dirname + sys.path.insert(0, d(d(os.path.abspath(__file__)))) + from setup import Command, __version__, installer_name, __appname__ PREFIX = "/var/www/calibre-ebook.com" @@ -19,8 +22,9 @@ BETAS = DOWNLOADS +'/betas' USER_MANUAL = '/var/www/localhost/htdocs/' HTML2LRF = "calibre/ebooks/lrf/html/demo" TXT2LRF = "src/calibre/ebooks/lrf/txt/demo" -MOBILEREAD = 'ftp://dev.mobileread.com/calibre/' - +STAGING_HOST = '67.207.135.179' +STAGING_USER = 'root' +STAGING_DIR = '/root/staging' def installers(): installers = list(map(installer_name, ('dmg', 'msi', 'tar.bz2'))) @@ -47,10 +51,10 @@ class ReUpload(Command): # {{{ description = 'Re-uplaod any installers present in dist/' - sub_commands = ['upload_to_google_code', 'upload_to_sourceforge'] + sub_commands = ['upload_installers'] def pre_sub_commands(self, opts): - opts.re_upload = True + opts.replace = True def run(self, opts): for x in installers(): @@ -58,371 +62,91 @@ class ReUpload(Command): # {{{ os.remove(x) # }}} -class ReadFileWithProgressReporting(file): # {{{ - - def __init__(self, path, mode='rb'): - file.__init__(self, path, mode) - self.seek(0, os.SEEK_END) - self._total = self.tell() - self.seek(0) - self.start_time = time.time() - - def __len__(self): - return self._total - - def read(self, size): - data = file.read(self, size) - if data: - self.report_progress(len(data)) - return data - - def report_progress(self, size): - sys.stdout.write(b'\x1b[s') - sys.stdout.write(b'\x1b[K') - frac = float(self.tell())/self._total - mb_pos = self.tell()/float(1024**2) - mb_tot = self._total/float(1024**2) - kb_pos = self.tell()/1024.0 - kb_rate = kb_pos/(time.time()-self.start_time) - bit_rate = kb_rate * 1024 - eta = int((self._total - self.tell())/bit_rate) + 1 - eta_m, eta_s = eta / 60, eta % 60 - sys.stdout.write( - ' %.1f%% %.1f/%.1fMB %.1f KB/sec %d minutes, %d seconds left'%( - frac*100, mb_pos, mb_tot, kb_rate, eta_m, eta_s)) - sys.stdout.write(b'\x1b[u') - if self.tell() >= self._total: - sys.stdout.write('\n') - t = int(time.time() - self.start_time) + 1 - print ('Upload took %d minutes and %d seconds at %.1f KB/sec' % ( - t/60, t%60, kb_rate)) - sys.stdout.flush() -# }}} - -class UploadToGoogleCode(Command): # {{{ - - USERNAME = 'kovidgoyal' - # Password can be gotten by going to - # http://code.google.com/hosting/settings - # while logged into gmail +# Data {{{ +def get_google_data(): PASSWORD_FILE = os.path.expanduser('~/.googlecodecalibre') OFFLINEIMAP = os.path.expanduser('~/work/kde/conf/offlineimap/rc') - GPATHS = '/var/www/status.calibre-ebook.com/googlepaths' - # If you change this, remember to change the default URL used by - # http://calibre-ebook.com as well - GC_PROJECT = 'calibre-ebook-ii' - UPLOAD_HOST = '%s.googlecode.com'%GC_PROJECT - FILES_LIST = 'http://code.google.com/p/%s/downloads/list'%GC_PROJECT - DELETE_URL = 'http://code.google.com/p/%s/downloads/delete?name=%%s'%GC_PROJECT + gc_password = open(PASSWORD_FILE).read().strip() + raw = open(OFFLINEIMAP).read() + pw = re.search(r'(?s)remoteuser = .*@gmail.com.*?remotepass = (\S+)', + raw).group(1).strip() + return { + 'username':'kovidgoyal@gmail.com', 'password':pw, 'gc_password':gc_password, + 'path_map_server':'root@kovidgoyal.net', + 'path_map_location':'/var/www/status.calibre-ebook.com/googlepaths', + 'project':'calibre-ebook-ii' + } - def add_options(self, parser): - parser.add_option('--re-upload', default=False, action='store_true', - help='Re-upload all installers currently in dist/') +def get_sourceforge_data(): + return {'username':'kovidgoyal', 'project':'calibre'} - def re_upload(self): - fnames = set([os.path.basename(x) for x in installers() if not - x.endswith('.tar.xz') and os.path.exists(x)]) - existing = set(self.old_files.keys()).intersection(fnames) - br = self.login_to_gmail() - for x in fnames: - src = os.path.join('dist', x) - if not os.access(src, os.R_OK): - continue - if x in existing: - self.info('Deleting', x) - br.open(self.DELETE_URL%x) - br.select_form(predicate=lambda y: 'delete.do' in y.action) - br.form.find_control(name='delete') - br.submit(name='delete') - self.upload_one(src) +def send_data(loc): + subprocess.check_call(['rsync', '-r', '-z', '-h', '--progress', '-e', 'ssh -x', + loc+'/', '%s@%s:%s'%(STAGING_USER, STAGING_HOST, STAGING_DIR)]) - def upload_one(self, fname): - self.info('\nUploading', fname) - typ = 'Type-' + ('Source' if fname.endswith('.xz') else 'Archive' if - fname.endswith('.zip') else 'Installer') - ext = os.path.splitext(fname)[1][1:] - op = 'OpSys-'+{'msi':'Windows','zip':'Windows', - 'dmg':'OSX','bz2':'Linux','xz':'All'}[ext] - desc = installer_description(fname) - start = time.time() - for i in range(5): - try: - path = self.upload(os.path.abspath(fname), desc, - labels=[typ, op, 'Featured']) - except KeyboardInterrupt: - raise SystemExit(1) - except: - import traceback - traceback.print_exc() - print ('\nUpload failed, trying again in 30 secs') - time.sleep(30) - else: - break - self.info('Uploaded to:', path, 'in', int(time.time() - start), - 'seconds') - return path +def gc_cmdline(ver, gdata): + return [__appname__, ver, 'fmap', 'googlecode', + gdata['project'], gdata['username'], gdata['password'], + gdata['gc_password'], '--path-map-server', + gdata['path_map_server'], '--path-map-location', + gdata['path_map_location']] - def run(self, opts): - self.opts = opts - self.password = open(self.PASSWORD_FILE).read().strip() - self.paths = {} - self.old_files = self.get_files_hosted_by_google_code() +def sf_cmdline(ver, sdata): + return [__appname__, ver, 'fmap', 'sourceforge', sdata['project'], + sdata['username']] - if opts.re_upload: - return self.re_upload() - - for fname in installers(): - bname = os.path.basename(fname) - if bname in self.old_files: - path = 'http://%s.googlecode.com/files/%s'%(self.GC_PROJECT, - bname) - self.info( - '%s already uploaded, skipping. Assuming URL is: %s'%( - bname, path)) - self.old_files.pop(bname) - else: - path = self.upload_one(fname) - self.paths[bname] = path - self.info('Updating path map') - self.info(repr(self.paths)) - raw = subprocess.Popen(['ssh', 'divok', 'cat', self.GPATHS], - stdout=subprocess.PIPE).stdout.read() - paths = eval(raw) - paths.update(self.paths) - rem = [x for x in paths if __version__ not in x] - for x in rem: paths.pop(x) - raw = ['%r : %r,'%(k, v) for k, v in paths.items()] - raw = '{\n\n%s\n\n}\n'%('\n'.join(raw)) - t = NamedTemporaryFile() - t.write(raw) - t.flush() - check_call(['scp', t.name, 'divok:'+self.GPATHS]) - self.br = self.login_to_gmail() - self.delete_old_files() - #if len(self.get_files_hosted_by_google_code()) > len(installers()): - # self.warn('Some old files were not deleted from Google Code') - - def login_to_gmail(self): - import mechanize - self.info('Logging into Gmail') - raw = open(self.OFFLINEIMAP).read() - pw = re.search(r'(?s)remoteuser = .*@gmail.com.*?remotepass = (\S+)', - raw).group(1).strip() - br = mechanize.Browser() - br.set_handle_robots(False) - br.open('http://gmail.com') - br.select_form(nr=0) - br.form['Email'] = self.USERNAME - br.form['Passwd'] = pw - br.submit() - return br - - def get_files_hosted_by_google_code(self): - import urllib2 - from lxml import html - self.info('Getting existing files in google code') - raw = urllib2.urlopen(self.FILES_LIST).read() - root = html.fromstring(raw) - ans = {} - for a in root.xpath('//td[@class="vt id col_0"]/a[@href]'): - ans[a.text.strip()] = a.get('href') - return ans - - def delete_old_files(self): - self.info('Deleting old files from Google Code...') - for fname in self.old_files: - ext = fname.rpartition('.')[-1] - if ext in ('flv', 'mp4', 'ogg', 'avi'): - continue - self.info('\tDeleting', fname) - self.br.open(self.DELETE_URL%fname) - self.br.select_form(predicate=lambda x: 'delete.do' in x.action) - self.br.form.find_control(name='delete') - self.br.submit(name='delete') - - def encode_upload_request(self, fields, file_path): - BOUNDARY = '----------Googlecode_boundary_reindeer_flotilla' - CRLF = '\r\n' - - body = [] - - # Add the metadata about the upload first - for key, value in fields: - body.extend( - ['--' + BOUNDARY, - 'Content-Disposition: form-data; name="%s"' % key, - '', - value, - ]) - - # Now add the file itself - file_name = os.path.basename(file_path) - with open(file_path, 'rb') as f: - file_content = f.read() - - body.extend( - ['--' + BOUNDARY, - 'Content-Disposition: form-data; name="filename"; filename="%s"' - % file_name, - # The upload server determines the mime-type, no need to set it. - 'Content-Type: application/octet-stream', - '', - file_content, - ]) - - # Finalize the form body - body.extend(['--' + BOUNDARY + '--', '']) - - return 'multipart/form-data; boundary=%s' % BOUNDARY, CRLF.join(body) - - def upload(self, fname, desc, labels=[], retry=0): - form_fields = [('summary', desc)] - form_fields.extend([('label', l.strip()) for l in labels]) - - content_type, body = self.encode_upload_request(form_fields, fname) - upload_uri = '/files' - auth_token = base64.b64encode('%s:%s'% (self.USERNAME, self.password)) - headers = { - 'Authorization': 'Basic %s' % auth_token, - 'User-Agent': 'Calibre googlecode.com uploader v0.1.0', - 'Content-Type': content_type, - } - - with NamedTemporaryFile(delete=False) as f: - f.write(body) - - try: - body = ReadFileWithProgressReporting(f.name) - server = httplib.HTTPSConnection(self.UPLOAD_HOST) - server.request('POST', upload_uri, body, headers) - resp = server.getresponse() - server.close() - finally: - os.remove(f.name) - - if resp.status == 201: - return resp.getheader('Location') - - print 'Failed to upload with code %d and reason: %s'%(resp.status, - resp.reason) - if retry < 1: - print 'Retrying in 5 seconds....' - time.sleep(5) - return self.upload(fname, desc, labels=labels, retry=retry+1) - raise Exception('Failed to upload '+fname) - -# }}} - -class UploadToSourceForge(Command): # {{{ - - description = 'Upload release files to sourceforge' - - USERNAME = 'kovidgoyal' - PROJECT = 'calibre' - BASE = '/home/frs/project/c/ca/'+PROJECT - - @property - def rdir(self): - return self.BASE+'/'+__version__ - - def upload_installers(self): - for x in installers(): - if not os.path.exists(x): continue - start = time.time() - self.info('Uploading', x) - for i in range(5): - try: - check_call(['rsync', '-z', '--progress', '-e', 'ssh -x', x, - '%s,%s@frs.sourceforge.net:%s'%(self.USERNAME, self.PROJECT, - self.rdir+'/')]) - except KeyboardInterrupt: - raise SystemExit(1) - except: - print ('\nUpload failed, trying again in 30 seconds') - time.sleep(30) - else: - break - print 'Uploaded in', int(time.time() - start), 'seconds' - print ('\n') - - def run(self, opts): - self.opts = opts - self.upload_installers() +def run_remote_upload(args): + print 'Running remotely:', ' '.join(args) + subprocess.check_call(['ssh', '-x', '%s@%s'%(STAGING_USER, STAGING_HOST), + 'cd', STAGING_DIR, '&&', 'python', 'hosting.py']+args) # }}} class UploadInstallers(Command): # {{{ - description = 'Upload any installers present in dist/ to mobileread' - def curl_list_dir(self, url=MOBILEREAD, listonly=1): - import pycurl - c = pycurl.Curl() - c.setopt(pycurl.URL, url) - c.setopt(c.FTP_USE_EPSV, 1) - c.setopt(c.NETRC, c.NETRC_REQUIRED) - c.setopt(c.FTPLISTONLY, listonly) - c.setopt(c.FTP_CREATE_MISSING_DIRS, 1) - b = cStringIO.StringIO() - c.setopt(c.WRITEFUNCTION, b.write) - c.perform() - c.close() - return b.getvalue().split() if listonly else b.getvalue().splitlines() - def curl_delete_file(self, path, url=MOBILEREAD): - import pycurl - c = pycurl.Curl() - c.setopt(pycurl.URL, url) - c.setopt(c.FTP_USE_EPSV, 1) - c.setopt(c.NETRC, c.NETRC_REQUIRED) - self.info('Deleting file %s on %s'%(path, url)) - c.setopt(c.QUOTE, ['dele '+ path]) - c.perform() - c.close() - - - def curl_upload_file(self, stream, url): - import pycurl - c = pycurl.Curl() - c.setopt(pycurl.URL, url) - c.setopt(pycurl.UPLOAD, 1) - c.setopt(c.NETRC, c.NETRC_REQUIRED) - c.setopt(pycurl.READFUNCTION, stream.read) - stream.seek(0, 2) - c.setopt(pycurl.INFILESIZE_LARGE, stream.tell()) - stream.seek(0) - c.setopt(c.NOPROGRESS, 0) - c.setopt(c.FTP_CREATE_MISSING_DIRS, 1) - self.info('Uploading file %s to url %s' % (getattr(stream, 'name', ''), - url)) - try: - c.perform() - c.close() - except: - pass - files = self.curl_list_dir(listonly=0) - for line in files: - line = line.split() - if url.endswith(line[-1]): - size = long(line[4]) - stream.seek(0,2) - if size != stream.tell(): - raise RuntimeError('curl failed to upload %s correctly'%getattr(stream, 'name', '')) - - def upload_installer(self, name): - if not os.path.exists(name): - return - bname = os.path.basename(name) - pat = re.compile(bname.replace(__version__, r'\d+\.\d+\.\d+')) - for f in self.curl_list_dir(): - if pat.search(f): - self.curl_delete_file('/calibre/'+f) - self.curl_upload_file(open(name, 'rb'), MOBILEREAD+os.path.basename(name)) + def add_option(self, parser): + parser.add_option('--replace', help= + 'Replace existing installers, when uploading to google') def run(self, opts): - self.info('Uploading installers...') - installers = list(map(installer_name, ('dmg', 'msi', 'tar.bz2'))) - installers.append(installer_name('tar.bz2', is64bit=True)) - map(self.upload_installer, installers) + all_possible = set(installers()) + available = set(glob.glob('dist/*')) + files = {x:installer_description(x) for x in + all_possible.intersection(available)} + tdir = mkdtemp() + try: + self.upload_to_staging(tdir, files) + self.upload_to_sourceforge() + self.upload_to_google(opts.replace) + finally: + shutil.rmtree(tdir, ignore_errors=True) + + def upload_to_staging(self, tdir, files): + os.mkdir(tdir+'/dist') + hosting = os.path.join(os.path.dirname(os.path.abspath(__file__)), + 'hosting.py') + shutil.copyfile(hosting, os.path.join(tdir, 'hosting.py')) + + for f in files: + shutil.copyfile(f, os.path.join(tdir, f)) + + with open(os.path.join(tdir, 'fmap'), 'wb') as fo: + for f, desc in files.iteritems(): + fo.write('%s: %s\n'%(f, desc)) + send_data(tdir) + + def upload_to_google(self, replace): + gdata = get_google_data() + args = gc_cmdline(__version__, gdata) + if replace: + args = ['--replace'] + args + run_remote_upload(args) + + def upload_to_sourceforge(self): + sdata = get_sourceforge_data() + args = sf_cmdline(__version__, sdata) + run_remote_upload(args) # }}} class UploadUserManual(Command): # {{{ @@ -508,4 +232,61 @@ class UploadToServer(Command): # {{{ shutil.rmtree(tdir) # }}} +# Testing {{{ + +def write_files(fmap): + for f in fmap: + with open(f, 'wb') as f: + f.write(os.urandom(100)) + f.write(b'a'*1000000) + with open('fmap', 'wb') as fo: + for f, desc in fmap.iteritems(): + fo.write('%s: %s\n'%(f, desc)) + +def setup_installers(): + ver = '0.0.1' + files = {x.replace(__version__, ver):installer_description(x) for x in installers()} + tdir = mkdtemp() + os.chdir(tdir) + return tdir, files, ver + +def test_google_uploader(): + gdata = get_google_data() + gdata['project'] = 'calibre-hosting-uploader' + gdata['path_map_location'] += '-test' + hosting = os.path.join(os.path.dirname(os.path.abspath(__file__)), + 'hosting.py') + + tdir, files, ver = setup_installers() + try: + os.mkdir('dist') + write_files(files) + shutil.copyfile(hosting, 'hosting.py') + send_data(tdir) + args = gc_cmdline(ver, gdata) + + print ('Doing initial upload') + run_remote_upload(args) + raw_input('Press Enter to proceed:') + + print ('\nDoing re-upload') + run_remote_upload(['--replace']+args) + raw_input('Press Enter to proceed:') + + nv = ver + '.1' + files = {x.replace(__version__, nv):installer_description(x) for x in installers()} + write_files(files) + send_data(tdir) + args[1] = nv + print ('\nDoing update upload') + run_remote_upload(args) + print ('\nDont forget to delete any remaining files in the %s project'% + gdata['project']) + + finally: + shutil.rmtree(tdir) +# }}} + +if __name__ == '__main__': + test_google_uploader()