diff --git a/setup/hosting.py b/setup/hosting.py index ef83e3b162..57c2315716 100644 --- a/setup/hosting.py +++ b/setup/hosting.py @@ -7,7 +7,8 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, time, sys, traceback, subprocess, urllib2, re, base64, httplib, shutil, glob +import os, time, sys, traceback, subprocess, urllib2, re, base64, httplib, shutil, glob, json, mimetypes +from pprint import pprint from argparse import ArgumentParser, FileType from subprocess import check_call, CalledProcessError, check_output from tempfile import NamedTemporaryFile @@ -379,6 +380,88 @@ class SourceForge(Base): # {{{ # }}} +class GitHub(Base): # {{{ + + API = 'https://api.github.com/' + + def __init__(self, files, reponame, version, username, password, replace=False): + self.files, self.reponame, self.version, self.username, self.password, self.replace = ( + files, reponame, version, username, password, replace) + import requests + self.requests = s = requests.Session() + s.auth = (self.username, self.password) + s.headers.update({'Accept': 'application/vnd.github.v3+json'}) + + def __call__(self): + release = self.create_release() + upload_url = release['upload_url'].partition('{')[0] + existing_assets = self.existing_assets(release['id']) + for path, desc in self.files.iteritems(): + self.info('') + url = self.API + 'repos/%s/%s/releases/assets/{}' % (self.username, self.reponame) + fname = os.path.basename(path) + if fname in existing_assets: + self.info('Deleting %s from GitHub with id: %s' % (fname, existing_assets[fname])) + r = self.requests.delete(url.format(existing_assets[fname])) + if r.status_code != 204: + self.fail(r, 'Failed to delete %s from GitHub' % fname) + r = self.do_upload(upload_url, path, desc, fname) + if r.status_code != 201: + self.fail(r, 'Failed to upload file: %s' % fname) + r = self.requests.patch(url.format(r.json()['id']), + data=json.dumps({'name':fname, 'label':desc})) + if r.status_code != 200: + self.fail(r, 'Failed to set label for %s' % fname) + + def do_upload(self, url, path, desc, fname): + mime_type = mimetypes.guess_type(fname)[0] + self.info('Uploading to GitHub: %s (%s)' % (fname, mime_type)) + with ReadFileWithProgressReporting(path) as f: + return self.requests.post( + url, headers={'Content-Type': mime_type, 'Content-Length':str(f._total)}, params={'name':fname}, + data=f) + + def fail(self, r, msg): + print (msg, ' Status Code: %s' % r.status_code, file=sys.stderr) + print ("JSON from response:", file=sys.stderr) + pprint(dict(r.json()), stream=sys.stderr) + raise SystemExit(1) + + def already_exists(self, r): + error_code = r.json().get('errors', [{}])[0].get('code', None) + return error_code == 'already_exists' + + def existing_assets(self, release_id): + url = self.API + 'repos/%s/%s/releases/%s/assets' % (self.username, self.reponame, release_id) + r = self.requests.get(url) + if r.status_code != 200: + self.fail('Failed to get assets for release') + return {asset['name']:asset['id'] for asset in r.json()} + + def create_release(self): + ' Create a release on GitHub or if it already exists, return the existing release ' + url = self.API + 'repos/%s/%s/releases' % (self.username, self.reponame) + r = self.requests.post(url, data=json.dumps({ + 'tag_name':'v%s' % self.version, + 'target_commitish': 'master', + 'name': 'version %s' % self.version, + 'body': 'Release version %s' % self.version, + 'draft': False, 'prerelease':False + })) + if r.status_code != 201: + if not self.already_exists(r): + self.fail(r, 'Failed to create release for version: %s' % self.version) + # Find existing release + r = self.requests.get(url) + if r.status_code != 200: + self.fail(r, 'Failed to list releases') + for release in reversed(r.json()): + if release.get('tag_name', None) == 'v' + self.version: + return release + return r.json() + +# }}} + def generate_index(): # {{{ os.chdir('/srv/download') releases = set() @@ -587,6 +670,8 @@ def cli_parser(): epilog=epilog) sf = subparsers.add_parser('sourceforge', help='Upload to sourceforge', epilog=epilog) + gh = subparsers.add_parser('github', help='Upload to GitHub', + epilog=epilog) cron = subparsers.add_parser('cron', help='Call script from cron') subparsers.add_parser('calibre', help='Upload to calibre file servers') subparsers.add_parser('dbs', help='Upload to fosshub.com') @@ -622,6 +707,14 @@ def cli_parser(): a('password', help='Password to log into your google account') + a = gh.add_argument + a('project', + help='The name of the repository on GitHub we are uploading to') + a('username', + help='Username to log into your GitHub account') + a('password', + help='Password to log into your GitHub account') + return p def main(args=None): @@ -650,6 +743,10 @@ def main(args=None): sf = SourceForge(ofiles, args.project, args.version, args.username, replace=args.replace) sf() + elif args.service == 'github': + gh = GitHub(ofiles, args.project, args.version, args.username, args.password, + replace=args.replace) + gh() elif args.service == 'cron': login_to_google(args.username, args.password) elif args.service == 'calibre': diff --git a/setup/upload.py b/setup/upload.py index 81407e42d0..28692213de 100644 --- a/setup/upload.py +++ b/setup/upload.py @@ -95,6 +95,13 @@ def get_google_data(): 'project':'calibre-ebook' } +def get_github_data(): + with open(os.path.expanduser('~/work/env/private/github'), 'rb') as f: + un, pw = f.read().strip().split(':') + return { + 'username':un, 'password':pw + } + def get_sourceforge_data(): return {'username':'kovidgoyal', 'project':'calibre'} @@ -109,6 +116,9 @@ def gc_cmdline(ver, gdata): gdata['path_map_server'], '--path-map-location', gdata['path_map_location']] +def gh_cmdline(ver, data): + return [__appname__, ver, 'fmap', 'github', __appname__, data['username'], data['password']] + def sf_cmdline(ver, sdata): return [__appname__, ver, 'fmap', 'sourceforge', sdata['project'], sdata['username']] @@ -151,6 +161,7 @@ class UploadInstallers(Command): # {{{ upload_signatures() self.upload_to_sourceforge() self.upload_to_dbs() + self.upload_to_github(opts.replace) # self.upload_to_google(opts.replace) finally: shutil.rmtree(tdir, ignore_errors=True) @@ -192,6 +203,13 @@ class UploadInstallers(Command): # {{{ args = ['--replace'] + args run_remote_upload(args) + def upload_to_github(self, replace): + data = get_github_data() + args = gh_cmdline(__version__, data) + if replace: + args = ['--replace'] + args + run_remote_upload(args) + def upload_to_sourceforge(self): sdata = get_sourceforge_data() args = sf_cmdline(__version__, sdata)