From 1ead1415e34513f2a952825eee159fa34120ef64 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 17 Aug 2016 10:00:55 +0530 Subject: [PATCH] New fosshub upload API --- setup/hosting.py | 56 ++---------------------------------------------- setup/upload.py | 31 +++++++++++++++++++++------ 2 files changed, 26 insertions(+), 61 deletions(-) diff --git a/setup/hosting.py b/setup/hosting.py index 8009056de4..13a8e23d01 100644 --- a/setup/hosting.py +++ b/setup/hosting.py @@ -7,10 +7,10 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, time, sys, shutil, glob, json, mimetypes +import os, time, sys, shutil, json, mimetypes from pprint import pprint from argparse import ArgumentParser, FileType -from subprocess import check_call, CalledProcessError, check_output +from subprocess import check_call from collections import OrderedDict class ReadFileWithProgressReporting(file): # {{{ @@ -339,55 +339,6 @@ def upload_to_servers(files, version): # {{{ # # }}} -def upload_to_dbs(files, version): # {{{ - print('Uploading to fosshub.com') - sys.stdout.flush() - server = 'mirror10.fosshub.com' - rdir = 'release/' - def run_ssh(command, func=check_call): - cmd = ['ssh', '-x', 'kovid@%s' % server, command] - try: - return func(cmd) - except CalledProcessError as err: - # fosshub is being a little flaky sshing into it is failing the first - # time, needing a retry - if err.returncode != 255: - raise - return func(cmd) - - old_files = set(run_ssh('ls ' + rdir, func=check_output).decode('utf-8').split()) - if len(files) < 7: - existing = set(map(os.path.basename, files)) - # fosshub does not support partial re-uploads - for f in glob.glob('%s/%s/calibre-*' % (SERVER_BASE, version)): - if os.path.basename(f) not in existing: - files[f] = None - - for x in files: - start = time.time() - print ('Uploading', x) - sys.stdout.flush() - old_files.discard(os.path.basename(x)) - for i in range(5): - try: - check_call(['rsync', '-h', '-z', '--progress', '-e', 'ssh -x', x, - 'kovid@%s:%s'%(server, rdir)]) - except KeyboardInterrupt: - raise SystemExit(1) - except: - print ('\nUpload failed, trying again in 30 seconds') - sys.stdout.flush() - time.sleep(30) - else: - break - print ('Uploaded in', int(time.time() - start), 'seconds\n\n') - sys.stdout.flush() - - if old_files: - run_ssh('rm -f %s' % (' '.join(rdir + x for x in old_files))) - run_ssh('/home/kovid/uploadFiles') -# }}} - # CLI {{{ def cli_parser(): epilog='Copyright Kovid Goyal 2012' @@ -419,7 +370,6 @@ def cli_parser(): gh = subparsers.add_parser('github', help='Upload to GitHub', epilog=epilog) subparsers.add_parser('calibre', help='Upload to calibre file servers') - subparsers.add_parser('dbs', help='Upload to fosshub.com') a = sf.add_argument a('project', @@ -462,8 +412,6 @@ def main(args=None): gh() elif args.service == 'calibre': upload_to_servers(ofiles, args.version) - elif args.service == 'dbs': - upload_to_dbs(ofiles, args.version) if __name__ == '__main__': main() diff --git a/setup/upload.py b/setup/upload.py index 96cc0c0b48..b2c2444220 100644 --- a/setup/upload.py +++ b/setup/upload.py @@ -5,7 +5,7 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, subprocess, hashlib, shutil, glob, stat, sys, time +import os, subprocess, hashlib, shutil, glob, stat, sys, time, urllib2, urllib from subprocess import check_call from tempfile import NamedTemporaryFile, mkdtemp, gettempdir from zipfile import ZipFile @@ -99,6 +99,10 @@ def get_github_data(): def get_sourceforge_data(): return {'username':'kovidgoyal', 'project':'calibre'} +def get_fosshub_data(): + with open(os.path.expanduser('~/work/env/private/fosshub'), 'rb') as f: + return f.read().decode('utf-8') + def send_data(loc): subprocess.check_call(['rsync', '--inplace', '--delete', '-r', '-z', '-h', '--progress', '-e', 'ssh -x', loc+'/', '%s@%s:%s'%(STAGING_USER, STAGING_HOST, STAGING_DIR)]) @@ -113,9 +117,6 @@ def sf_cmdline(ver, sdata): def calibre_cmdline(ver): return [__appname__, ver, 'fmap', 'calibre'] -def dbs_cmdline(ver): - return [__appname__, ver, 'fmap', 'dbs'] - def run_remote_upload(args): print 'Running remotely:', ' '.join(args) subprocess.check_call(['ssh', '-x', '%s@%s'%(STAGING_USER, STAGING_HOST), @@ -123,6 +124,24 @@ def run_remote_upload(args): # }}} +def upload_to_fosshub(files=None): + if files is None: + files = set(installers()) + entries = [] + for fname in files: + desc = installer_description(fname) + url = 'https://download.calibre-ebook.com/%s/%s' % (__version__, os.path.basename(fname)) + entries.append({ + 'url':url, + 'type': desc, + 'version': __version__, + }) + jq = {'software': 'Calibre', 'apiKey':get_fosshub_data(), 'upload':entries} + rq = urllib2.urlopen('https://www.fosshub.com/JSTools/uploadJson', urllib.urlencode(jq)) + if rq.getcode() != 200: + raise SystemExit('Failed to upload to fosshub, with HTTP error code: %d' % rq.getcode()) + + class UploadInstallers(Command): # {{{ def add_options(self, parser): @@ -148,7 +167,7 @@ class UploadInstallers(Command): # {{{ upload_signatures() check_call('ssh code /apps/update-calibre-version.py'.split()) # self.upload_to_sourceforge() - # self.upload_to_dbs() + upload_to_fosshub(files) self.upload_to_github(opts.replace) finally: shutil.rmtree(tdir, ignore_errors=True) @@ -198,8 +217,6 @@ class UploadInstallers(Command): # {{{ def upload_to_calibre(self): run_remote_upload(calibre_cmdline(__version__)) - def upload_to_dbs(self): - run_remote_upload(dbs_cmdline(__version__)) # }}} class UploadUserManual(Command): # {{{