diff --git a/setup/hosting.py b/setup/hosting.py index 3f556ce5b5..139528e10c 100644 --- a/setup/hosting.py +++ b/setup/hosting.py @@ -7,37 +7,12 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, time, sys, traceback, subprocess, urllib2, re, base64, httplib, shutil, glob, json, mimetypes +import os, time, sys, shutil, glob, json, mimetypes from pprint import pprint from argparse import ArgumentParser, FileType from subprocess import check_call, CalledProcessError, check_output -from tempfile import NamedTemporaryFile from collections import OrderedDict -def login_to_google(username, password): # {{{ - import mechanize - br = mechanize.Browser() - br.addheaders = [('User-agent', - 'Mozilla/5.0 (X11; Linux x86_64; rv:9.0) Gecko/20100101 Firefox/9.0')] - br.set_handle_robots(False) - br.open('https://accounts.google.com/ServiceLogin?service=code') - br.select_form(nr=0) - br.form['Email'] = username - br.form['Passwd'] = password - raw = br.submit().read() - if re.search(br'(?i).*?Account Settings', raw) is None: - x = re.search(br'(?is).*?', raw) - if x is not None: - print ('Title of post login page: %s'%x.group()) - # open('/tmp/goog.html', 'wb').write(raw) - raise ValueError(('Failed to login to google with credentials: %s %s' - '\nGoogle sometimes requires verification when logging in from a ' - 'new IP address. Use lynx to login and supply the verification, ' - 'at: lynx -accept_all_cookies https://accounts.google.com/ServiceLogin?service=code') - %(username, password)) - return br -# }}} - class ReadFileWithProgressReporting(file): # {{{ def __init__(self, path, mode='rb'): @@ -101,254 +76,6 @@ class Base(object): # {{{ # }}} -class GoogleCode(Base): # {{{ - - def __init__(self, - # A mapping of filenames to file descriptions. The descriptions are - # used to populate the description field for the upload on google - # code - files, - - # The unix name for the application. - appname, - - # The version being uploaded - version, - - # Google account username - username, - - # Googlecode.com password - password, - - # Google account password - gmail_password, - - # The name of the google code project we are uploading to - gc_project, - - # Server to which to upload the mapping of file names to google - # code URLs. If not None, upload is performed via shelling out to - # ssh, so you must have ssh-agent setup with the authenticated key - # and ssh agent forwarding enabled - gpaths_server=None, - # The path on gpaths_server to which to upload the mapping data - gpaths=None, - - # If True, files are replaced, otherwise existing files are skipped - reupload=False, - - # The pattern to match filenames for the files being uploaded and - # extract version information from them. Must have a named group - # named version - filename_pattern=r'{appname}-(?:portable-installer-)?(?P.+?)(?:-(?:i686|x86_64|32bit|64bit))?\.(?:zip|exe|msi|dmg|tar\.bz2|tar\.xz|txz|tbz2)' # noqa - - ): - self.username, self.password, = username, password - self.gmail_password, self.gc_project = gmail_password, gc_project - self.reupload, self.files, self.version = reupload, files, version - self.gpaths, self.gpaths_server = gpaths, gpaths_server - - self.upload_host = '%s.googlecode.com'%gc_project - self.files_list = 'http://code.google.com/p/%s/downloads/list'%gc_project - self.delete_url = 'http://code.google.com/p/%s/downloads/delete?name=%%s'%gc_project - - self.filename_pat = re.compile(filename_pattern.format(appname=appname)) - for x in self.files: - if self.filename_pat.match(os.path.basename(x)) is None: - raise ValueError(('The filename %s does not match the ' - 'filename pattern')%os.path.basename(x)) - - def upload_one(self, fname, retries=2): - self.info('\nUploading', fname) - typ = 'Type-' + ('Source' if fname.endswith('.xz') else 'Archive' if - fname.endswith('.zip') else 'Installer') - ext = os.path.splitext(fname)[1][1:] - op = 'OpSys-'+{'msi':'Windows','exe':'Windows', - 'dmg':'OSX','txz':'Linux','xz':'All'}[ext] - desc = self.files[fname] - start = time.time() - for i in range(retries): - try: - path = self.upload(os.path.abspath(fname), desc, - labels=[typ, op, 'Featured'], retry=100) - except KeyboardInterrupt: - raise SystemExit(1) - except: - traceback.print_exc() - print ('\nUpload failed, trying again in 30 secs.', - '%d retries left.'%(retries-1)) - time.sleep(30) - else: - break - self.info('Uploaded to:', path, 'in', int(time.time() - start), - 'seconds') - return path - - def re_upload(self): - fnames = {os.path.basename(x):x for x in self.files} - existing = self.old_files.intersection(set(fnames)) - br = self.login_to_google() - for x, src in fnames.iteritems(): - if not os.access(src, os.R_OK): - continue - if x in existing: - self.info('Deleting', x) - br.open(self.delete_url%x) - br.select_form(predicate=lambda y: 'delete.do' in y.action) - br.form.find_control(name='delete') - br.submit(name='delete') - self.upload_one(src) - - def __call__(self): - self.paths = {} - self.old_files = self.get_old_files() - if self.reupload: - return self.re_upload() - - for fname in self.files: - bname = os.path.basename(fname) - if bname in self.old_files: - path = 'http://%s.googlecode.com/files/%s'%(self.gc_project, - bname) - self.info( - '%s already uploaded, skipping. Assuming URL is: %s'%( - bname, path)) - self.old_files.remove(bname) - else: - path = self.upload_one(fname) - self.paths[bname] = path - self.info('Updating path map') - for k, v in self.paths.iteritems(): - self.info('\t%s => %s'%(k, v)) - if self.gpaths and self.gpaths_server: - raw = subprocess.Popen(['ssh', self.gpaths_server, 'cat', self.gpaths], - stdout=subprocess.PIPE).stdout.read() - paths = eval(raw) if raw else {} - paths.update(self.paths) - rem = [x for x in paths if self.version not in x] - for x in rem: - paths.pop(x) - raw = ['%r : %r,'%(k, v) for k, v in paths.items()] - raw = '{\n\n%s\n\n}\n'%('\n'.join(raw)) - with NamedTemporaryFile() as t: - t.write(raw) - t.flush() - check_call(['scp', t.name, '%s:%s'%(self.gpaths_server, - self.gpaths)]) - if self.old_files: - self.br = self.login_to_google() - self.delete_old_files() - - def login_to_google(self): - self.info('Logging into Google') - return login_to_google(self.username, self.gmail_password) - - def get_files_hosted_by_google_code(self): - from lxml import html - self.info('Getting existing files in google code:', self.gc_project) - raw = urllib2.urlopen(self.files_list).read() - root = html.fromstring(raw) - ans = {} - for a in root.xpath('//td[@class="vt id col_0"]/a[@href]'): - ans[a.text.strip()] = a.get('href') - return ans - - def get_old_files(self): - ans = set() - for fname in self.get_files_hosted_by_google_code(): - m = self.filename_pat.match(fname) - if m is not None: - ans.add(fname) - return ans - - def delete_old_files(self): - if not self.old_files: - return - self.info('Deleting old files from Google Code...') - for fname in self.old_files: - self.info('\tDeleting', fname) - self.br.open(self.delete_url%fname) - self.br.select_form(predicate=lambda x: 'delete.do' in x.action) - self.br.form.find_control(name='delete') - self.br.submit(name='delete') - - def encode_upload_request(self, fields, file_path): - BOUNDARY = '----------Googlecode_boundary_reindeer_flotilla' - - body = [] - - # Add the metadata about the upload first - for key, value in fields: - body.extend( - ['--' + BOUNDARY, - 'Content-Disposition: form-data; name="%s"' % key, - '', - value, - ]) - - # Now add the file itself - file_name = os.path.basename(file_path) - with open(file_path, 'rb') as f: - file_content = f.read() - - body.extend( - ['--' + BOUNDARY, - 'Content-Disposition: form-data; name="filename"; filename="%s"' - % file_name, - # The upload server determines the mime-type, no need to set it. - 'Content-Type: application/octet-stream', - '', - file_content, - ]) - - # Finalize the form body - body.extend(['--' + BOUNDARY + '--', '']) - body = [x.encode('ascii') if isinstance(x, unicode) else x for x in - body] - - return ('multipart/form-data; boundary=%s' % BOUNDARY, - b'\r\n'.join(body)) - - def upload(self, fname, desc, labels=[], retry=0): - form_fields = [('summary', desc)] - form_fields.extend([('label', l.strip()) for l in labels]) - - content_type, body = self.encode_upload_request(form_fields, fname) - upload_uri = '/files' - auth_token = base64.b64encode('%s:%s'% (self.username, self.password)) - headers = { - 'Authorization': 'Basic %s' % auth_token, - 'User-Agent': 'googlecode.com uploader v1', - 'Content-Type': content_type, - } - - with NamedTemporaryFile(delete=False) as f: - f.write(body) - - try: - body = ReadFileWithProgressReporting(f.name) - server = httplib.HTTPSConnection(self.upload_host) - server.request('POST', upload_uri, body, headers) - resp = server.getresponse() - server.close() - finally: - os.remove(f.name) - - if resp.status == 201: - return resp.getheader('Location') - - print ('Failed to upload with code %d and reason: %s'%(resp.status, - resp.reason)) - if retry < 1: - print ('Retrying in 5 seconds....') - time.sleep(5) - return self.upload(fname, desc, labels=labels, retry=retry+1) - raise Exception('Failed to upload '+fname) - - -# }}} - class SourceForge(Base): # {{{ # Note that you should manually ssh once to username,project@frs.sourceforge.net @@ -687,47 +414,19 @@ def cli_parser(): subparsers = p.add_subparsers(help='Where to upload to', dest='service', title='Service', description='Hosting service to upload to') - gc = subparsers.add_parser('googlecode', help='Upload to googlecode', - epilog=epilog) sf = subparsers.add_parser('sourceforge', help='Upload to sourceforge', epilog=epilog) gh = subparsers.add_parser('github', help='Upload to GitHub', epilog=epilog) - cron = subparsers.add_parser('cron', help='Call script from cron') subparsers.add_parser('calibre', help='Upload to calibre file servers') subparsers.add_parser('dbs', help='Upload to fosshub.com') - a = gc.add_argument - - a('project', - help='The name of the project on google code we are uploading to') - a('username', - help='Username to log into your google account') - a('password', - help='Password to log into your google account') - a('gc_password', - help='Password for google code hosting.' - ' Get it from http://code.google.com/hosting/settings') - - a('--path-map-server', - help='A server to which the mapping of filenames to googlecode ' - 'URLs will be uploaded. The upload happens via ssh, so you must ' - 'have a working ssh agent') - a('--path-map-location', - help='Path on the server where the path map is placed.') - a = sf.add_argument a('project', help='The name of the project on sourceforge we are uploading to') a('username', help='Sourceforge username') - a = cron.add_argument - a('username', - help='Username to log into your google account') - a('password', - help='Password to log into your google account') - a = gh.add_argument a('project', help='The name of the repository on GitHub we are uploading to') @@ -742,25 +441,18 @@ def main(args=None): cli = cli_parser() args = cli.parse_args(args) files = {} - if args.service != 'cron': - with args.file_map as f: - for line in f: - fname, _, desc = line.partition(':') - fname, desc = fname.strip(), desc.strip() - if fname and desc: - files[fname] = desc + with args.file_map as f: + for line in f: + fname, _, desc = line.partition(':') + fname, desc = fname.strip(), desc.strip() + if fname and desc: + files[fname] = desc ofiles = OrderedDict() for x in sorted(files, key=lambda x:os.stat(x).st_size, reverse=True): ofiles[x] = files[x] - if args.service == 'googlecode': - gc = GoogleCode(ofiles, args.appname, args.version, args.username, - args.gc_password, args.password, args.project, - gpaths_server=args.path_map_server, - gpaths=args.path_map_location, reupload=args.replace) - gc() - elif args.service == 'sourceforge': + if args.service == 'sourceforge': sf = SourceForge(ofiles, args.project, args.version, args.username, replace=args.replace) sf() @@ -768,8 +460,6 @@ def main(args=None): gh = GitHub(ofiles, args.project, args.version, args.username, args.password, replace=args.replace) gh() - elif args.service == 'cron': - login_to_google(args.username, args.password) elif args.service == 'calibre': upload_to_servers(ofiles, args.version) elif args.service == 'dbs': diff --git a/setup/upload.py b/setup/upload.py index 44b855271f..8806b741b3 100644 --- a/setup/upload.py +++ b/setup/upload.py @@ -83,19 +83,6 @@ class ReUpload(Command): # {{{ # }}} # Data {{{ -def get_google_data(): - with open(os.path.expanduser('~/work/env/private/googlecodecalibre'), 'rb') as f: - gc_password, ga_un, pw = f.read().strip().split('|') - - return { - 'username':ga_un, 'password':pw, 'gc_password':gc_password, - 'path_map_server':'root@kovidgoyal.net', - 'path_map_location':'/var/www/status.calibre-ebook.com/googlepaths', - # If you change this remember to change it in the - # status.calibre-ebook.com server as well - 'project':'calibre-ebook' - } - def get_github_data(): with open(os.path.expanduser('~/work/env/private/github'), 'rb') as f: un, pw = f.read().strip().split(':') @@ -110,13 +97,6 @@ def send_data(loc): subprocess.check_call(['rsync', '--inplace', '--delete', '-r', '-z', '-h', '--progress', '-e', 'ssh -x', loc+'/', '%s@%s:%s'%(STAGING_USER, STAGING_HOST, STAGING_DIR)]) -def gc_cmdline(ver, gdata): - return [__appname__, ver, 'fmap', 'googlecode', - gdata['project'], gdata['username'], gdata['password'], - gdata['gc_password'], '--path-map-server', - gdata['path_map_server'], '--path-map-location', - gdata['path_map_location']] - def gh_cmdline(ver, data): return [__appname__, ver, 'fmap', 'github', __appname__, data['username'], data['password']] @@ -164,7 +144,6 @@ class UploadInstallers(Command): # {{{ self.upload_to_sourceforge() self.upload_to_dbs() self.upload_to_github(opts.replace) - # self.upload_to_google(opts.replace) finally: shutil.rmtree(tdir, ignore_errors=True) @@ -198,13 +177,6 @@ class UploadInstallers(Command): # {{{ else: break - def upload_to_google(self, replace): - gdata = get_google_data() - args = gc_cmdline(__version__, gdata) - if replace: - args = ['--replace'] + args - run_remote_upload(args) - def upload_to_github(self, replace): data = get_github_data() args = gh_cmdline(__version__, data) @@ -307,42 +279,5 @@ def setup_installers(): os.chdir(tdir) return tdir, files, ver -def test_google_uploader(): - gdata = get_google_data() - gdata['project'] = 'calibre-hosting-uploader' - gdata['path_map_location'] += '-test' - hosting = os.path.join(os.path.dirname(os.path.abspath(__file__)), - 'hosting.py') - - tdir, files, ver = setup_installers() - try: - os.mkdir('dist') - write_files(files) - shutil.copyfile(hosting, 'hosting.py') - send_data(tdir) - args = gc_cmdline(ver, gdata) - - print ('Doing initial upload') - run_remote_upload(args) - raw_input('Press Enter to proceed:') - - print ('\nDoing re-upload') - run_remote_upload(['--replace']+args) - raw_input('Press Enter to proceed:') - - nv = ver + '.1' - files = {x.replace(__version__, nv):installer_description(x) for x in installers()} - write_files(files) - send_data(tdir) - args[1] = nv - print ('\nDoing update upload') - run_remote_upload(args) - print ('\nDont forget to delete any remaining files in the %s project'% - gdata['project']) - - finally: - shutil.rmtree(tdir) # }}} -if __name__ == '__main__': - test_google_uploader()