New fosshub upload API

setup/hosting.py

@@ -7,10 +7,10 @@ __license__   = 'GPL v3'
 __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os, time, sys, shutil, glob, json, mimetypes
+import os, time, sys, shutil, json, mimetypes
 from pprint import pprint
 from argparse import ArgumentParser, FileType
-from subprocess import check_call, CalledProcessError, check_output
+from subprocess import check_call
 from collections import OrderedDict
 
 class ReadFileWithProgressReporting(file):  # {{{
@@ -339,55 +339,6 @@ def upload_to_servers(files, version):  # {{{
     #
 # }}}
 
-def upload_to_dbs(files, version):  # {{{
-    print('Uploading to fosshub.com')
-    sys.stdout.flush()
-    server = 'mirror10.fosshub.com'
-    rdir = 'release/'
-    def run_ssh(command, func=check_call):
-        cmd = ['ssh', '-x', 'kovid@%s' % server, command]
-        try:
-            return func(cmd)
-        except CalledProcessError as err:
-            # fosshub is being a little flaky sshing into it is failing the first
-            # time, needing a retry
-            if err.returncode != 255:
-                raise
-            return func(cmd)
-
-    old_files = set(run_ssh('ls ' + rdir, func=check_output).decode('utf-8').split())
-    if len(files) < 7:
-        existing = set(map(os.path.basename, files))
-        # fosshub does not support partial re-uploads
-        for f in glob.glob('%s/%s/calibre-*' % (SERVER_BASE, version)):
-            if os.path.basename(f) not in existing:
-                files[f] = None
-
-    for x in files:
-        start = time.time()
-        print ('Uploading', x)
-        sys.stdout.flush()
-        old_files.discard(os.path.basename(x))
-        for i in range(5):
-            try:
-                check_call(['rsync', '-h', '-z', '--progress', '-e', 'ssh -x', x,
-                'kovid@%s:%s'%(server, rdir)])
-            except KeyboardInterrupt:
-                raise SystemExit(1)
-            except:
-                print ('\nUpload failed, trying again in 30 seconds')
-                sys.stdout.flush()
-                time.sleep(30)
-            else:
-                break
-        print ('Uploaded in', int(time.time() - start), 'seconds\n\n')
-        sys.stdout.flush()
-
-    if old_files:
-        run_ssh('rm -f %s' % (' '.join(rdir + x for x in old_files)))
-    run_ssh('/home/kovid/uploadFiles')
-# }}}
-
 # CLI {{{
 def cli_parser():
     epilog='Copyright Kovid Goyal 2012'
@@ -419,7 +370,6 @@ def cli_parser():
     gh = subparsers.add_parser('github', help='Upload to GitHub',
             epilog=epilog)
     subparsers.add_parser('calibre', help='Upload to calibre file servers')
-    subparsers.add_parser('dbs', help='Upload to fosshub.com')
 
     a = sf.add_argument
     a('project',
@@ -462,8 +412,6 @@ def main(args=None):
         gh()
     elif args.service == 'calibre':
         upload_to_servers(ofiles, args.version)
-    elif args.service == 'dbs':
-        upload_to_dbs(ofiles, args.version)
 
 if __name__ == '__main__':
     main()

setup/upload.py

@@ -5,7 +5,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os, subprocess, hashlib, shutil, glob, stat, sys, time
+import os, subprocess, hashlib, shutil, glob, stat, sys, time, urllib2, urllib
 from subprocess import check_call
 from tempfile import NamedTemporaryFile, mkdtemp, gettempdir
 from zipfile import ZipFile
@@ -99,6 +99,10 @@ def get_github_data():
 def get_sourceforge_data():
     return {'username':'kovidgoyal', 'project':'calibre'}
 
+def get_fosshub_data():
+    with open(os.path.expanduser('~/work/env/private/fosshub'), 'rb') as f:
+        return f.read().decode('utf-8')
+
 def send_data(loc):
     subprocess.check_call(['rsync', '--inplace', '--delete', '-r', '-z', '-h', '--progress', '-e', 'ssh -x',
         loc+'/', '%s@%s:%s'%(STAGING_USER, STAGING_HOST, STAGING_DIR)])
@@ -113,9 +117,6 @@ def sf_cmdline(ver, sdata):
 def calibre_cmdline(ver):
     return [__appname__, ver, 'fmap', 'calibre']
 
-def dbs_cmdline(ver):
-    return [__appname__, ver, 'fmap', 'dbs']
-
 def run_remote_upload(args):
     print 'Running remotely:', ' '.join(args)
     subprocess.check_call(['ssh', '-x', '%s@%s'%(STAGING_USER, STAGING_HOST),
@@ -123,6 +124,24 @@ def run_remote_upload(args):
 
 # }}}
 
+def upload_to_fosshub(files=None):
+    if files is None:
+        files = set(installers())
+    entries = []
+    for fname in files:
+        desc = installer_description(fname)
+        url = 'https://download.calibre-ebook.com/%s/%s' % (__version__, os.path.basename(fname))
+        entries.append({
+            'url':url,
+            'type': desc,
+            'version': __version__,
+        })
+    jq = {'software': 'Calibre', 'apiKey':get_fosshub_data(), 'upload':entries}
+    rq = urllib2.urlopen('https://www.fosshub.com/JSTools/uploadJson', urllib.urlencode(jq))
+    if rq.getcode() != 200:
+        raise SystemExit('Failed to upload to fosshub, with HTTP error code: %d' % rq.getcode())
+
+
 class UploadInstallers(Command):  # {{{
 
     def add_options(self, parser):
@@ -148,7 +167,7 @@ class UploadInstallers(Command):  # {{{
                 upload_signatures()
                 check_call('ssh code /apps/update-calibre-version.py'.split())
             # self.upload_to_sourceforge()
-            # self.upload_to_dbs()
+            upload_to_fosshub(files)
             self.upload_to_github(opts.replace)
         finally:
             shutil.rmtree(tdir, ignore_errors=True)
@@ -198,8 +217,6 @@ class UploadInstallers(Command):  # {{{
     def upload_to_calibre(self):
         run_remote_upload(calibre_cmdline(__version__))
 
-    def upload_to_dbs(self):
-        run_remote_upload(dbs_cmdline(__version__))
 # }}}
 
 class UploadUserManual(Command):  # {{{