Change upload installers code to work via a staging server with more upload bandwidth than my current DSL connection

setup/commands.py

@@ -16,8 +16,8 @@ __all__ = [
         'sdist',
         'manual', 'tag_release',
         'pypi_register', 'pypi_upload', 'upload_to_server',
-        'upload_user_manual', 'upload_to_mobileread', 'upload_demo',
+        'upload_installers',
-        'upload_to_sourceforge', 'upload_to_google_code', 'reupload',
+        'upload_user_manual', 'upload_demo', 'reupload',
         'linux32', 'linux64', 'linux', 'linux_freeze',
         'osx32_freeze', 'osx', 'rsync', 'push',
         'win32_freeze', 'win32', 'win',
@@ -65,14 +65,12 @@ stage4 = Stage4()
 stage5 = Stage5()
 publish = Publish()
 
-from setup.upload import UploadUserManual, UploadInstallers, UploadDemo, \
+from setup.upload import (UploadUserManual, UploadDemo, UploadInstallers,
-        UploadToServer, UploadToSourceForge, UploadToGoogleCode, ReUpload
+        UploadToServer, ReUpload)
 upload_user_manual = UploadUserManual()
-upload_to_mobileread = UploadInstallers()
 upload_demo = UploadDemo()
 upload_to_server = UploadToServer()
-upload_to_sourceforge = UploadToSourceForge()
+upload_installers = UploadInstallers()
-upload_to_google_code = UploadToGoogleCode()
 reupload = ReUpload()
 
 from setup.installer import Rsync, Push

setup/hosting.py

@@ -0,0 +1,459 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import os, time, sys, traceback, subprocess, urllib2, re, base64, httplib
+from argparse import ArgumentParser, FileType
+from subprocess import check_call
+from tempfile import NamedTemporaryFile#, mkdtemp
+from collections import OrderedDict
+
+import mechanize
+from lxml import html
+
+def login_to_google(username, password):
+    br = mechanize.Browser()
+    br.addheaders = [('User-agent',
+        'Mozilla/5.0 (X11; Linux x86_64; rv:9.0) Gecko/20100101 Firefox/9.0')]
+    br.set_handle_robots(False)
+    br.open('https://accounts.google.com/ServiceLogin?service=code')
+    br.select_form(nr=0)
+    br.form['Email'] = username
+    br.form['Passwd'] = password
+    raw = br.submit().read()
+    if b'<title>Account overview - Account Settings</title>' not in raw:
+        raise ValueError(('Failed to login to google with credentials: %s %s'
+            '\nGoogle sometimes requires verification when logging in from a '
+            'new IP address. Use lynx to login and supply the verification.')
+                %(username, password))
+    return br
+
+class ReadFileWithProgressReporting(file): # {{{
+
+    def __init__(self, path, mode='rb'):
+        file.__init__(self, path, mode)
+        self.seek(0, os.SEEK_END)
+        self._total = self.tell()
+        self.seek(0)
+        self.start_time = time.time()
+
+    def __len__(self):
+        return self._total
+
+    def read(self, size):
+        data = file.read(self, size)
+        if data:
+            self.report_progress(len(data))
+        return data
+
+    def report_progress(self, size):
+        sys.stdout.write(b'\x1b[s')
+        sys.stdout.write(b'\x1b[K')
+        frac = float(self.tell())/self._total
+        mb_pos = self.tell()/float(1024**2)
+        mb_tot = self._total/float(1024**2)
+        kb_pos = self.tell()/1024.0
+        kb_rate = kb_pos/(time.time()-self.start_time)
+        bit_rate = kb_rate * 1024
+        eta = int((self._total - self.tell())/bit_rate) + 1
+        eta_m, eta_s = eta / 60, eta % 60
+        sys.stdout.write(
+            '  %.1f%%   %.1f/%.1fMB %.1f KB/sec    %d minutes, %d seconds left'%(
+                frac*100, mb_pos, mb_tot, kb_rate, eta_m, eta_s))
+        sys.stdout.write(b'\x1b[u')
+        if self.tell() >= self._total:
+            sys.stdout.write('\n')
+            t = int(time.time() - self.start_time) + 1
+            print ('Upload took %d minutes and %d seconds at %.1f KB/sec' % (
+                t/60, t%60, kb_rate))
+        sys.stdout.flush()
+# }}}
+
+class Base(object):  # {{{
+
+    def __init__(self):
+        self.d = os.path.dirname
+        self.j = os.path.join
+        self.a = os.path.abspath
+        self.b = os.path.basename
+        self.s = os.path.splitext
+        self.e = os.path.exists
+
+    def info(self, *args, **kwargs):
+        print(*args, **kwargs)
+        sys.stdout.flush()
+
+    def warn(self, *args, **kwargs):
+        print('\n'+'_'*20, 'WARNING','_'*20)
+        print(*args, **kwargs)
+        print('_'*50)
+        sys.stdout.flush()
+
+#}}}
+
+class GoogleCode(Base):# {{{
+
+    def __init__(self,
+            # A mapping of filenames to file descriptions. The descriptions are
+            # used to populate the description field for the upload on google
+            # code
+            files,
+
+            # The unix name for the application.
+            appname,
+
+            # The version being uploaded
+            version,
+
+            # Google account username
+            username,
+
+            # Googlecode.com password
+            password,
+
+            # Google account password
+            gmail_password,
+
+            # The name of the google code project we are uploading to
+            gc_project,
+
+            # Server to which to upload the mapping of file names to google
+            # code URLs. If not None, upload is performed via shelling out to
+            # ssh, so you must have ssh-agent setup with the authenticated key
+            # and ssh agent forwarding enabled
+            gpaths_server=None,
+            # The path on gpaths_server to which to upload the mapping data
+            gpaths=None,
+
+            # If True, files are replaced, otherwise existing files are skipped
+            reupload=False,
+
+            # The pattern to match filenames for the files being uploaded and
+            # extract version information from them. Must have a named group
+            # named version
+            filename_pattern=r'{appname}-(?:portable-)?(?P<version>.+?)(?:-(?:i686|x86_64|32bit|64bit))?\.(?:zip|exe|msi|dmg|tar\.bz2|tar\.xz|txz|tbz2)'
+
+            ):
+        self.username, self.password, = username, password
+        self.gmail_password, self.gc_project = gmail_password, gc_project
+        self.reupload, self.files, self.version = reupload, files, version
+        self.gpaths, self.gpaths_server = gpaths, gpaths_server
+
+        self.upload_host = '%s.googlecode.com'%gc_project
+        self.files_list = 'http://code.google.com/p/%s/downloads/list'%gc_project
+        self.delete_url = 'http://code.google.com/p/%s/downloads/delete?name=%%s'%gc_project
+
+        self.filename_pat = re.compile(filename_pattern.format(appname=appname))
+        for x in self.files:
+            if self.filename_pat.match(os.path.basename(x)) is None:
+                raise ValueError(('The filename %s does not match the '
+                        'filename pattern')%os.path.basename(x))
+
+    def upload_one(self, fname, retries=2):
+        self.info('\nUploading', fname)
+        typ = 'Type-' + ('Source' if fname.endswith('.xz') else 'Archive' if
+                fname.endswith('.zip') else 'Installer')
+        ext = os.path.splitext(fname)[1][1:]
+        op  = 'OpSys-'+{'msi':'Windows','zip':'Windows',
+                'dmg':'OSX','bz2':'Linux','xz':'All'}[ext]
+        desc = self.files[fname]
+        start = time.time()
+        for i in range(retries):
+            try:
+                path = self.upload(os.path.abspath(fname), desc,
+                    labels=[typ, op, 'Featured'], retry=100)
+            except KeyboardInterrupt:
+                raise SystemExit(1)
+            except:
+                traceback.print_exc()
+                print ('\nUpload failed, trying again in 30 secs.',
+                        '%d retries left.'%(retries-1))
+                time.sleep(30)
+            else:
+                break
+        self.info('Uploaded to:', path, 'in', int(time.time() - start),
+                'seconds')
+        return path
+
+    def re_upload(self):
+        fnames = {os.path.basename(x):x for x in self.files}
+        existing = self.old_files.intersection(set(fnames))
+        br = self.login_to_google()
+        for x, src in fnames.iteritems():
+            if not os.access(src, os.R_OK):
+                continue
+            if x in existing:
+                self.info('Deleting', x)
+                br.open(self.delete_url%x)
+                br.select_form(predicate=lambda y: 'delete.do' in y.action)
+                br.form.find_control(name='delete')
+                br.submit(name='delete')
+            self.upload_one(src)
+
+    def __call__(self):
+        self.paths = {}
+        self.old_files = self.get_old_files()
+        if self.reupload:
+            return self.re_upload()
+
+        for fname in self.files:
+            bname = os.path.basename(fname)
+            if bname in self.old_files:
+                path = 'http://%s.googlecode.com/files/%s'%(self.gc_project,
+                        bname)
+                self.info(
+                    '%s already uploaded, skipping. Assuming URL is: %s'%(
+                        bname, path))
+                self.old_files.remove(bname)
+            else:
+                path = self.upload_one(fname)
+            self.paths[bname] = path
+        self.info('Updating path map')
+        for k, v in self.paths.iteritems():
+            self.info('\t%s => %s'%(k, v))
+        if self.gpaths and self.gpaths_server:
+            raw = subprocess.Popen(['ssh', self.gpaths_server, 'cat', self.gpaths],
+                    stdout=subprocess.PIPE).stdout.read()
+            paths = eval(raw) if raw else {}
+            paths.update(self.paths)
+            rem = [x for x in paths if self.version not in x]
+            for x in rem: paths.pop(x)
+            raw = ['%r : %r,'%(k, v) for k, v in paths.items()]
+            raw = '{\n\n%s\n\n}\n'%('\n'.join(raw))
+            with NamedTemporaryFile() as t:
+                t.write(raw)
+                t.flush()
+                check_call(['scp', t.name, '%s:%s'%(self.gpaths_server,
+                    self.gpaths)])
+        if self.old_files:
+            self.br = self.login_to_google()
+            self.delete_old_files()
+
+    def login_to_google(self):
+        self.info('Logging into Google')
+        return login_to_google(self.username, self.gmail_password)
+
+    def get_files_hosted_by_google_code(self):
+        self.info('Getting existing files in google code:', self.gc_project)
+        raw = urllib2.urlopen(self.files_list).read()
+        root = html.fromstring(raw)
+        ans = {}
+        for a in root.xpath('//td[@class="vt id col_0"]/a[@href]'):
+            ans[a.text.strip()] = a.get('href')
+        return ans
+
+    def get_old_files(self):
+        ans = set()
+        for fname in self.get_files_hosted_by_google_code():
+            m = self.filename_pat.match(fname)
+            if m is not None:
+                ans.add(fname)
+        return ans
+
+    def delete_old_files(self):
+        if not self.old_files:
+            return
+        self.info('Deleting old files from Google Code...')
+        for fname in self.old_files:
+            self.info('\tDeleting', fname)
+            self.br.open(self.delete_url%fname)
+            self.br.select_form(predicate=lambda x: 'delete.do' in x.action)
+            self.br.form.find_control(name='delete')
+            self.br.submit(name='delete')
+
+    def encode_upload_request(self, fields, file_path):
+        BOUNDARY = '----------Googlecode_boundary_reindeer_flotilla'
+
+        body = []
+
+        # Add the metadata about the upload first
+        for key, value in fields:
+            body.extend(
+            ['--' + BOUNDARY,
+            'Content-Disposition: form-data; name="%s"' % key,
+            '',
+            value,
+            ])
+
+        # Now add the file itself
+        file_name = os.path.basename(file_path)
+        with open(file_path, 'rb') as f:
+            file_content = f.read()
+
+        body.extend(
+            ['--' + BOUNDARY,
+            'Content-Disposition: form-data; name="filename"; filename="%s"'
+            % file_name,
+            # The upload server determines the mime-type, no need to set it.
+            'Content-Type: application/octet-stream',
+            '',
+            file_content,
+            ])
+
+        # Finalize the form body
+        body.extend(['--' + BOUNDARY + '--', ''])
+        body = [x.encode('ascii') if isinstance(x, unicode) else x for x in
+                body]
+
+        return ('multipart/form-data; boundary=%s' % BOUNDARY,
+                b'\r\n'.join(body))
+
+    def upload(self, fname, desc, labels=[], retry=0):
+        form_fields = [('summary', desc)]
+        form_fields.extend([('label', l.strip()) for l in labels])
+
+        content_type, body = self.encode_upload_request(form_fields, fname)
+        upload_uri = '/files'
+        auth_token = base64.b64encode('%s:%s'% (self.username, self.password))
+        headers = {
+            'Authorization': 'Basic %s' % auth_token,
+            'User-Agent': 'googlecode.com uploader v1',
+            'Content-Type': content_type,
+            }
+
+        with NamedTemporaryFile(delete=False) as f:
+            f.write(body)
+
+        try:
+            body = ReadFileWithProgressReporting(f.name)
+            server = httplib.HTTPSConnection(self.upload_host)
+            server.request('POST', upload_uri, body, headers)
+            resp = server.getresponse()
+            server.close()
+        finally:
+            os.remove(f.name)
+
+        if resp.status == 201:
+            return resp.getheader('Location')
+
+        print ('Failed to upload with code %d and reason: %s'%(resp.status,
+                resp.reason))
+        if retry < 1:
+            print ('Retrying in 5 seconds....')
+            time.sleep(5)
+            return self.upload(fname, desc, labels=labels, retry=retry+1)
+        raise Exception('Failed to upload '+fname)
+
+
+# }}}
+
+class SourceForge(Base): # {{{
+
+    def __init__(self, files, project, version, username, replace=False):
+        self.username, self.project, self.version = username, project, version
+        self.base = '/home/frs/project/c/ca/'+project
+        self.rdir = self.base + '/' + version
+        self.files = files
+
+    def __call__(self):
+        for x in self.files:
+            start = time.time()
+            self.info('Uploading', x)
+            for i in range(5):
+                try:
+                    check_call(['rsync', '-h', '-z', '--progress', '-e', 'ssh -x', x,
+                    '%s,%s@frs.sourceforge.net:%s'%(self.username, self.project,
+                        self.rdir+'/')])
+                except KeyboardInterrupt:
+                    raise SystemExit(1)
+                except:
+                    print ('\nUpload failed, trying again in 30 seconds')
+                    time.sleep(30)
+                else:
+                    break
+            print ('Uploaded in', int(time.time() - start), 'seconds\n\n')
+
+# }}}
+
+# CLI {{{
+def cli_parser():
+    epilog='Copyright Kovid Goyal 2012'
+
+    p = ArgumentParser(
+            description='Upload project files to a hosting service automatically',
+            epilog=epilog
+            )
+    a = p.add_argument
+    a('appname', help='The name of the application, all files to'
+            ' upload should begin with this name')
+    a('version', help='The version of the application, all files to'
+            ' upload should contain this version')
+    a('file_map', type=FileType('rb'),
+            help='A file containing a mapping of files to be uploaded to '
+            'descriptions of the files. The descriptions will be visible '
+            'to users trying to get the file from the hosting service. '
+            'The format of the file is filename: description, with one per '
+            'line. filename can be a path to the file relative to the current '
+            'directory.')
+    a('--replace', action='store_true', default=False,
+            help='If specified, existing files are replaced, otherwise '
+            'they are skipped.')
+
+    subparsers = p.add_subparsers(help='Where to upload to', dest='service',
+            title='Service', description='Hosting service to upload to')
+    gc = subparsers.add_parser('googlecode', help='Upload to googlecode',
+            epilog=epilog)
+    sf = subparsers.add_parser('sourceforge', help='Upload to sourceforge',
+            epilog=epilog)
+    a = gc.add_argument
+
+    a('project',
+            help='The name of the project on google code we are uploading to')
+    a('username',
+            help='Username to log into your google account')
+    a('password',
+            help='Password to log into your google account')
+    a('gc_password',
+            help='Password for google code hosting.'
+            ' Get it from http://code.google.com/hosting/settings')
+
+    a('--path-map-server',
+            help='A server to which the mapping of filenames to googlecode '
+            'URLs will be uploaded. The upload happens via ssh, so you must '
+            'have a working ssh agent')
+    a('--path-map-location',
+            help='Path on the server where the path map is placed.')
+
+    a = sf.add_argument
+    a('project',
+            help='The name of the project on sourceforge we are uploading to')
+    a('username',
+            help='Sourceforge username')
+
+    return p
+
+def main(args=None):
+    cli = cli_parser()
+    args = cli.parse_args(args)
+    files = {}
+    with args.file_map as f:
+        for line in f:
+            fname, _, desc = line.partition(':')
+            fname, desc = fname.strip(), desc.strip()
+            if fname and desc:
+                files[fname] = desc
+
+    ofiles = OrderedDict()
+    for x in sorted(files, key=lambda x:os.stat(x).st_size, reverse=True):
+        ofiles[x] = files[x]
+
+    if args.service == 'googlecode':
+        gc = GoogleCode(ofiles, args.appname, args.version, args.username,
+                args.gc_password, args.password, args.project,
+                gpaths_server=args.path_map_server,
+                gpaths=args.path_map_location, reupload=args.replace)
+        gc()
+    elif args.service == 'sourceforge':
+        sf = SourceForge(ofiles, args.project, args.version, args.username,
+                replace=args.replace)
+        sf()
+
+if __name__ == '__main__':
+    main()
+# }}}
+

setup/publish.py

@@ -45,7 +45,7 @@ class Stage3(Command):
 class Stage4(Command):
 
     description = 'Stage 4 of the publish process'
-    sub_commands = ['upload_to_sourceforge', 'upload_to_google_code']
+    sub_commands = ['upload_installers']
 
 class Stage5(Command):
 

setup/upload.py

@@ -5,12 +5,15 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os, re, cStringIO, base64, httplib, subprocess, hashlib, shutil, time, \
+import os, re, subprocess, hashlib, shutil, glob, stat, sys
-    glob, stat, sys
 from subprocess import check_call
 from tempfile import NamedTemporaryFile, mkdtemp
 from zipfile import ZipFile
 
+if __name__ == '__main__':
+    d = os.path.dirname
+    sys.path.insert(0, d(d(os.path.abspath(__file__))))
+
 from setup import Command, __version__, installer_name, __appname__
 
 PREFIX = "/var/www/calibre-ebook.com"
@@ -19,8 +22,9 @@ BETAS = DOWNLOADS +'/betas'
 USER_MANUAL = '/var/www/localhost/htdocs/'
 HTML2LRF = "calibre/ebooks/lrf/html/demo"
 TXT2LRF  = "src/calibre/ebooks/lrf/txt/demo"
-MOBILEREAD = 'ftp://dev.mobileread.com/calibre/'
+STAGING_HOST = '67.207.135.179'
-
+STAGING_USER = 'root'
+STAGING_DIR = '/root/staging'
 
 def installers():
     installers = list(map(installer_name, ('dmg', 'msi', 'tar.bz2')))
@@ -47,10 +51,10 @@ class ReUpload(Command): # {{{
 
     description = 'Re-uplaod any installers present in dist/'
 
-    sub_commands = ['upload_to_google_code', 'upload_to_sourceforge']
+    sub_commands = ['upload_installers']
 
     def pre_sub_commands(self, opts):
-        opts.re_upload = True
+        opts.replace = True
 
     def run(self, opts):
         for x in installers():
@@ -58,371 +62,91 @@ class ReUpload(Command): # {{{
                 os.remove(x)
 # }}}
 
-class ReadFileWithProgressReporting(file): # {{{
+# Data {{{
-
+def get_google_data():
-    def __init__(self, path, mode='rb'):
-        file.__init__(self, path, mode)
-        self.seek(0, os.SEEK_END)
-        self._total = self.tell()
-        self.seek(0)
-        self.start_time = time.time()
-
-    def __len__(self):
-        return self._total
-
-    def read(self, size):
-        data = file.read(self, size)
-        if data:
-            self.report_progress(len(data))
-        return data
-
-    def report_progress(self, size):
-        sys.stdout.write(b'\x1b[s')
-        sys.stdout.write(b'\x1b[K')
-        frac = float(self.tell())/self._total
-        mb_pos = self.tell()/float(1024**2)
-        mb_tot = self._total/float(1024**2)
-        kb_pos = self.tell()/1024.0
-        kb_rate = kb_pos/(time.time()-self.start_time)
-        bit_rate = kb_rate * 1024
-        eta = int((self._total - self.tell())/bit_rate) + 1
-        eta_m, eta_s = eta / 60, eta % 60
-        sys.stdout.write(
-            '  %.1f%%   %.1f/%.1fMB %.1f KB/sec    %d minutes, %d seconds left'%(
-                frac*100, mb_pos, mb_tot, kb_rate, eta_m, eta_s))
-        sys.stdout.write(b'\x1b[u')
-        if self.tell() >= self._total:
-            sys.stdout.write('\n')
-            t = int(time.time() - self.start_time) + 1
-            print ('Upload took %d minutes and %d seconds at %.1f KB/sec' % (
-                t/60, t%60, kb_rate))
-        sys.stdout.flush()
-# }}}
-
-class UploadToGoogleCode(Command): # {{{
-
-    USERNAME = 'kovidgoyal'
-    # Password can be gotten by going to
-    # http://code.google.com/hosting/settings
-    # while logged into gmail
     PASSWORD_FILE = os.path.expanduser('~/.googlecodecalibre')
     OFFLINEIMAP   = os.path.expanduser('~/work/kde/conf/offlineimap/rc')
-    GPATHS = '/var/www/status.calibre-ebook.com/googlepaths'
-    # If you change this, remember to change the default URL used by
-    # http://calibre-ebook.com as well
-    GC_PROJECT = 'calibre-ebook-ii'
 
-    UPLOAD_HOST = '%s.googlecode.com'%GC_PROJECT
+    gc_password = open(PASSWORD_FILE).read().strip()
-    FILES_LIST = 'http://code.google.com/p/%s/downloads/list'%GC_PROJECT
+    raw = open(OFFLINEIMAP).read()
-    DELETE_URL = 'http://code.google.com/p/%s/downloads/delete?name=%%s'%GC_PROJECT
+    pw = re.search(r'(?s)remoteuser = .*@gmail.com.*?remotepass = (\S+)',
+            raw).group(1).strip()
+    return {
+        'username':'kovidgoyal@gmail.com', 'password':pw, 'gc_password':gc_password,
+        'path_map_server':'root@kovidgoyal.net',
+        'path_map_location':'/var/www/status.calibre-ebook.com/googlepaths',
+        'project':'calibre-ebook-ii'
+    }
 
-    def add_options(self, parser):
+def get_sourceforge_data():
-        parser.add_option('--re-upload', default=False, action='store_true',
+    return {'username':'kovidgoyal', 'project':'calibre'}
-                help='Re-upload all installers currently in dist/')
 
-    def re_upload(self):
+def send_data(loc):
-        fnames = set([os.path.basename(x) for x in installers() if not
+    subprocess.check_call(['rsync', '-r', '-z', '-h', '--progress', '-e', 'ssh -x',
-                x.endswith('.tar.xz') and os.path.exists(x)])
+        loc+'/', '%s@%s:%s'%(STAGING_USER, STAGING_HOST, STAGING_DIR)])
-        existing = set(self.old_files.keys()).intersection(fnames)
-        br = self.login_to_gmail()
-        for x in fnames:
-            src = os.path.join('dist', x)
-            if not os.access(src, os.R_OK):
-                continue
-            if x in existing:
-                self.info('Deleting', x)
-                br.open(self.DELETE_URL%x)
-                br.select_form(predicate=lambda y: 'delete.do' in y.action)
-                br.form.find_control(name='delete')
-                br.submit(name='delete')
-            self.upload_one(src)
 
-    def upload_one(self, fname):
+def gc_cmdline(ver, gdata):
-        self.info('\nUploading', fname)
+    return [__appname__, ver, 'fmap', 'googlecode',
-        typ = 'Type-' + ('Source' if fname.endswith('.xz') else 'Archive' if
+                gdata['project'], gdata['username'], gdata['password'],
-                fname.endswith('.zip') else 'Installer')
+                gdata['gc_password'], '--path-map-server',
-        ext = os.path.splitext(fname)[1][1:]
+                gdata['path_map_server'], '--path-map-location',
-        op  = 'OpSys-'+{'msi':'Windows','zip':'Windows',
+                gdata['path_map_location']]
-                'dmg':'OSX','bz2':'Linux','xz':'All'}[ext]
-        desc = installer_description(fname)
-        start = time.time()
-        for i in range(5):
-            try:
-                path = self.upload(os.path.abspath(fname), desc,
-                    labels=[typ, op, 'Featured'])
-            except KeyboardInterrupt:
-                raise SystemExit(1)
-            except:
-                import traceback
-                traceback.print_exc()
-                print ('\nUpload failed, trying again in 30 secs')
-                time.sleep(30)
-            else:
-                break
-        self.info('Uploaded to:', path, 'in', int(time.time() - start),
-                'seconds')
-        return path
 
-    def run(self, opts):
+def sf_cmdline(ver, sdata):
-        self.opts = opts
+    return [__appname__, ver, 'fmap', 'sourceforge', sdata['project'],
-        self.password = open(self.PASSWORD_FILE).read().strip()
+            sdata['username']]
-        self.paths = {}
-        self.old_files = self.get_files_hosted_by_google_code()
 
-        if opts.re_upload:
+def run_remote_upload(args):
-            return self.re_upload()
+    print 'Running remotely:', ' '.join(args)
-
+    subprocess.check_call(['ssh', '-x', '%s@%s'%(STAGING_USER, STAGING_HOST),
-        for fname in installers():
+        'cd', STAGING_DIR, '&&', 'python', 'hosting.py']+args)
-            bname = os.path.basename(fname)
-            if bname in self.old_files:
-                path = 'http://%s.googlecode.com/files/%s'%(self.GC_PROJECT,
-                        bname)
-                self.info(
-                    '%s already uploaded, skipping. Assuming URL is: %s'%(
-                        bname, path))
-                self.old_files.pop(bname)
-            else:
-                path = self.upload_one(fname)
-            self.paths[bname] = path
-        self.info('Updating path map')
-        self.info(repr(self.paths))
-        raw = subprocess.Popen(['ssh', 'divok', 'cat', self.GPATHS],
-                stdout=subprocess.PIPE).stdout.read()
-        paths = eval(raw)
-        paths.update(self.paths)
-        rem = [x for x in paths if __version__ not in x]
-        for x in rem: paths.pop(x)
-        raw = ['%r : %r,'%(k, v) for k, v in paths.items()]
-        raw = '{\n\n%s\n\n}\n'%('\n'.join(raw))
-        t = NamedTemporaryFile()
-        t.write(raw)
-        t.flush()
-        check_call(['scp', t.name, 'divok:'+self.GPATHS])
-        self.br = self.login_to_gmail()
-        self.delete_old_files()
-        #if len(self.get_files_hosted_by_google_code()) > len(installers()):
-        #    self.warn('Some old files were not deleted from Google Code')
-
-    def login_to_gmail(self):
-        import mechanize
-        self.info('Logging into Gmail')
-        raw = open(self.OFFLINEIMAP).read()
-        pw = re.search(r'(?s)remoteuser = .*@gmail.com.*?remotepass = (\S+)',
-                raw).group(1).strip()
-        br = mechanize.Browser()
-        br.set_handle_robots(False)
-        br.open('http://gmail.com')
-        br.select_form(nr=0)
-        br.form['Email'] = self.USERNAME
-        br.form['Passwd'] = pw
-        br.submit()
-        return br
-
-    def get_files_hosted_by_google_code(self):
-        import urllib2
-        from lxml import html
-        self.info('Getting existing files in google code')
-        raw = urllib2.urlopen(self.FILES_LIST).read()
-        root = html.fromstring(raw)
-        ans = {}
-        for a in root.xpath('//td[@class="vt id col_0"]/a[@href]'):
-            ans[a.text.strip()] = a.get('href')
-        return ans
-
-    def delete_old_files(self):
-        self.info('Deleting old files from Google Code...')
-        for fname in self.old_files:
-            ext = fname.rpartition('.')[-1]
-            if ext in ('flv', 'mp4', 'ogg', 'avi'):
-                continue
-            self.info('\tDeleting', fname)
-            self.br.open(self.DELETE_URL%fname)
-            self.br.select_form(predicate=lambda x: 'delete.do' in x.action)
-            self.br.form.find_control(name='delete')
-            self.br.submit(name='delete')
-
-    def encode_upload_request(self, fields, file_path):
-        BOUNDARY = '----------Googlecode_boundary_reindeer_flotilla'
-        CRLF = '\r\n'
-
-        body = []
-
-        # Add the metadata about the upload first
-        for key, value in fields:
-            body.extend(
-            ['--' + BOUNDARY,
-            'Content-Disposition: form-data; name="%s"' % key,
-            '',
-            value,
-            ])
-
-        # Now add the file itself
-        file_name = os.path.basename(file_path)
-        with open(file_path, 'rb') as f:
-            file_content = f.read()
-
-        body.extend(
-            ['--' + BOUNDARY,
-            'Content-Disposition: form-data; name="filename"; filename="%s"'
-            % file_name,
-            # The upload server determines the mime-type, no need to set it.
-            'Content-Type: application/octet-stream',
-            '',
-            file_content,
-            ])
-
-        # Finalize the form body
-        body.extend(['--' + BOUNDARY + '--', ''])
-
-        return 'multipart/form-data; boundary=%s' % BOUNDARY, CRLF.join(body)
-
-    def upload(self, fname, desc, labels=[], retry=0):
-        form_fields = [('summary', desc)]
-        form_fields.extend([('label', l.strip()) for l in labels])
-
-        content_type, body = self.encode_upload_request(form_fields, fname)
-        upload_uri = '/files'
-        auth_token = base64.b64encode('%s:%s'% (self.USERNAME, self.password))
-        headers = {
-            'Authorization': 'Basic %s' % auth_token,
-            'User-Agent': 'Calibre googlecode.com uploader v0.1.0',
-            'Content-Type': content_type,
-            }
-
-        with NamedTemporaryFile(delete=False) as f:
-            f.write(body)
-
-        try:
-            body = ReadFileWithProgressReporting(f.name)
-            server = httplib.HTTPSConnection(self.UPLOAD_HOST)
-            server.request('POST', upload_uri, body, headers)
-            resp = server.getresponse()
-            server.close()
-        finally:
-            os.remove(f.name)
-
-        if resp.status == 201:
-            return resp.getheader('Location')
-
-        print 'Failed to upload with code %d and reason: %s'%(resp.status,
-                resp.reason)
-        if retry < 1:
-            print 'Retrying in 5 seconds....'
-            time.sleep(5)
-            return self.upload(fname, desc, labels=labels, retry=retry+1)
-        raise Exception('Failed to upload '+fname)
-
-# }}}
-
-class UploadToSourceForge(Command): # {{{
-
-    description = 'Upload release files to sourceforge'
-
-    USERNAME = 'kovidgoyal'
-    PROJECT  = 'calibre'
-    BASE     = '/home/frs/project/c/ca/'+PROJECT
-
-    @property
-    def rdir(self):
-        return self.BASE+'/'+__version__
-
-    def upload_installers(self):
-        for x in installers():
-            if not os.path.exists(x): continue
-            start = time.time()
-            self.info('Uploading', x)
-            for i in range(5):
-                try:
-                    check_call(['rsync', '-z', '--progress', '-e', 'ssh -x', x,
-                    '%s,%s@frs.sourceforge.net:%s'%(self.USERNAME, self.PROJECT,
-                        self.rdir+'/')])
-                except KeyboardInterrupt:
-                    raise SystemExit(1)
-                except:
-                    print ('\nUpload failed, trying again in 30 seconds')
-                    time.sleep(30)
-                else:
-                    break
-            print 'Uploaded in', int(time.time() - start), 'seconds'
-            print ('\n')
-
-    def run(self, opts):
-        self.opts = opts
-        self.upload_installers()
 
 # }}}
 
 class UploadInstallers(Command): # {{{
-    description = 'Upload any installers present in dist/ to mobileread'
-    def curl_list_dir(self, url=MOBILEREAD, listonly=1):
-        import pycurl
-        c = pycurl.Curl()
-        c.setopt(pycurl.URL, url)
-        c.setopt(c.FTP_USE_EPSV, 1)
-        c.setopt(c.NETRC, c.NETRC_REQUIRED)
-        c.setopt(c.FTPLISTONLY, listonly)
-        c.setopt(c.FTP_CREATE_MISSING_DIRS, 1)
-        b = cStringIO.StringIO()
-        c.setopt(c.WRITEFUNCTION, b.write)
-        c.perform()
-        c.close()
-        return b.getvalue().split() if listonly else b.getvalue().splitlines()
 
-    def curl_delete_file(self, path, url=MOBILEREAD):
+    def add_option(self, parser):
-        import pycurl
+        parser.add_option('--replace', help=
-        c = pycurl.Curl()
+                'Replace existing installers, when uploading to google')
-        c.setopt(pycurl.URL, url)
-        c.setopt(c.FTP_USE_EPSV, 1)
-        c.setopt(c.NETRC, c.NETRC_REQUIRED)
-        self.info('Deleting file %s on %s'%(path, url))
-        c.setopt(c.QUOTE, ['dele '+ path])
-        c.perform()
-        c.close()
-
-
-    def curl_upload_file(self, stream, url):
-        import pycurl
-        c = pycurl.Curl()
-        c.setopt(pycurl.URL, url)
-        c.setopt(pycurl.UPLOAD, 1)
-        c.setopt(c.NETRC, c.NETRC_REQUIRED)
-        c.setopt(pycurl.READFUNCTION, stream.read)
-        stream.seek(0, 2)
-        c.setopt(pycurl.INFILESIZE_LARGE, stream.tell())
-        stream.seek(0)
-        c.setopt(c.NOPROGRESS, 0)
-        c.setopt(c.FTP_CREATE_MISSING_DIRS, 1)
-        self.info('Uploading file %s to url %s' % (getattr(stream, 'name', ''),
-            url))
-        try:
-            c.perform()
-            c.close()
-        except:
-            pass
-        files = self.curl_list_dir(listonly=0)
-        for line in files:
-            line = line.split()
-            if url.endswith(line[-1]):
-                size = long(line[4])
-                stream.seek(0,2)
-                if size != stream.tell():
-                    raise RuntimeError('curl failed to upload %s correctly'%getattr(stream, 'name', ''))
-
-    def upload_installer(self, name):
-        if not os.path.exists(name):
-            return
-        bname = os.path.basename(name)
-        pat = re.compile(bname.replace(__version__, r'\d+\.\d+\.\d+'))
-        for f in self.curl_list_dir():
-            if pat.search(f):
-                self.curl_delete_file('/calibre/'+f)
-        self.curl_upload_file(open(name, 'rb'), MOBILEREAD+os.path.basename(name))
 
     def run(self, opts):
-        self.info('Uploading installers...')
+        all_possible = set(installers())
-        installers = list(map(installer_name, ('dmg', 'msi', 'tar.bz2')))
+        available = set(glob.glob('dist/*'))
-        installers.append(installer_name('tar.bz2', is64bit=True))
+        files = {x:installer_description(x) for x in
-        map(self.upload_installer, installers)
+                all_possible.intersection(available)}
+        tdir = mkdtemp()
+        try:
+            self.upload_to_staging(tdir, files)
+            self.upload_to_sourceforge()
+            self.upload_to_google(opts.replace)
+        finally:
+            shutil.rmtree(tdir, ignore_errors=True)
+
+    def upload_to_staging(self, tdir, files):
+        os.mkdir(tdir+'/dist')
+        hosting = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+            'hosting.py')
+        shutil.copyfile(hosting, os.path.join(tdir, 'hosting.py'))
+
+        for f in files:
+            shutil.copyfile(f, os.path.join(tdir, f))
+
+        with open(os.path.join(tdir, 'fmap'), 'wb') as fo:
+            for f, desc in files.iteritems():
+                fo.write('%s: %s\n'%(f, desc))
+        send_data(tdir)
+
+    def upload_to_google(self, replace):
+        gdata = get_google_data()
+        args = gc_cmdline(__version__, gdata)
+        if replace:
+            args = ['--replace'] + args
+        run_remote_upload(args)
+
+    def upload_to_sourceforge(self):
+        sdata = get_sourceforge_data()
+        args = sf_cmdline(__version__, sdata)
+        run_remote_upload(args)
 # }}}
 
 class UploadUserManual(Command): # {{{
@@ -508,4 +232,61 @@ class UploadToServer(Command): # {{{
         shutil.rmtree(tdir)
 # }}}
 
+# Testing {{{
+
+def write_files(fmap):
+    for f in fmap:
+        with open(f, 'wb') as f:
+            f.write(os.urandom(100))
+            f.write(b'a'*1000000)
+    with open('fmap', 'wb') as fo:
+        for f, desc in fmap.iteritems():
+            fo.write('%s: %s\n'%(f, desc))
+
+def setup_installers():
+    ver = '0.0.1'
+    files = {x.replace(__version__, ver):installer_description(x) for x in installers()}
+    tdir = mkdtemp()
+    os.chdir(tdir)
+    return tdir, files, ver
+
+def test_google_uploader():
+    gdata = get_google_data()
+    gdata['project'] = 'calibre-hosting-uploader'
+    gdata['path_map_location'] += '-test'
+    hosting = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+        'hosting.py')
+
+    tdir, files, ver = setup_installers()
+    try:
+        os.mkdir('dist')
+        write_files(files)
+        shutil.copyfile(hosting, 'hosting.py')
+        send_data(tdir)
+        args = gc_cmdline(ver, gdata)
+
+        print ('Doing initial upload')
+        run_remote_upload(args)
+        raw_input('Press Enter to proceed:')
+
+        print ('\nDoing re-upload')
+        run_remote_upload(['--replace']+args)
+        raw_input('Press Enter to proceed:')
+
+        nv = ver + '.1'
+        files = {x.replace(__version__, nv):installer_description(x) for x in installers()}
+        write_files(files)
+        send_data(tdir)
+        args[1] = nv
+        print ('\nDoing update upload')
+        run_remote_upload(args)
+        print ('\nDont forget to delete any remaining files in the %s project'%
+                gdata['project'])
+
+    finally:
+        shutil.rmtree(tdir)
+# }}}
+
+if __name__ == '__main__':
+    test_google_uploader()