diff --git a/setup/hosting.py b/setup/hosting.py
index 8707388181..9853f181a4 100644
--- a/setup/hosting.py
+++ b/setup/hosting.py
@@ -10,13 +10,13 @@ __docformat__ = 'restructuredtext en'
import os, time, sys, traceback, subprocess, urllib2, re, base64, httplib
from argparse import ArgumentParser, FileType
from subprocess import check_call
-from tempfile import NamedTemporaryFile#, mkdtemp
+from tempfile import NamedTemporaryFile
from collections import OrderedDict
import mechanize
from lxml import html
-def login_to_google(username, password):
+def login_to_google(username, password): # {{{
br = mechanize.Browser()
br.addheaders = [('User-agent',
'Mozilla/5.0 (X11; Linux x86_64; rv:9.0) Gecko/20100101 Firefox/9.0')]
@@ -30,15 +30,16 @@ def login_to_google(username, password):
x = re.search(br'(?is)
.*?', raw)
if x is not None:
print ('Title of post login page: %s'%x.group())
- #open('/tmp/goog.html', 'wb').write(raw)
+ # open('/tmp/goog.html', 'wb').write(raw)
raise ValueError(('Failed to login to google with credentials: %s %s'
'\nGoogle sometimes requires verification when logging in from a '
'new IP address. Use lynx to login and supply the verification, '
'at: lynx -accept_all_cookies https://accounts.google.com/ServiceLogin?service=code')
%(username, password))
return br
+# }}}
-class ReadFileWithProgressReporting(file): # {{{
+class ReadFileWithProgressReporting(file): # {{{
def __init__(self, path, mode='rb'):
file.__init__(self, path, mode)
@@ -101,7 +102,7 @@ class Base(object): # {{{
#}}}
-class GoogleCode(Base):# {{{
+class GoogleCode(Base): # {{{
def __init__(self,
# A mapping of filenames to file descriptions. The descriptions are
@@ -141,7 +142,7 @@ class GoogleCode(Base):# {{{
# The pattern to match filenames for the files being uploaded and
# extract version information from them. Must have a named group
# named version
- filename_pattern=r'{appname}-(?:portable-installer-)?(?P.+?)(?:-(?:i686|x86_64|32bit|64bit))?\.(?:zip|exe|msi|dmg|tar\.bz2|tar\.xz|txz|tbz2)'
+ filename_pattern=r'{appname}-(?:portable-installer-)?(?P.+?)(?:-(?:i686|x86_64|32bit|64bit))?\.(?:zip|exe|msi|dmg|tar\.bz2|tar\.xz|txz|tbz2)' # noqa
):
self.username, self.password, = username, password
@@ -227,7 +228,8 @@ class GoogleCode(Base):# {{{
paths = eval(raw) if raw else {}
paths.update(self.paths)
rem = [x for x in paths if self.version not in x]
- for x in rem: paths.pop(x)
+ for x in rem:
+ paths.pop(x)
raw = ['%r : %r,'%(k, v) for k, v in paths.items()]
raw = '{\n\n%s\n\n}\n'%('\n'.join(raw))
with NamedTemporaryFile() as t:
@@ -347,7 +349,7 @@ class GoogleCode(Base):# {{{
# }}}
-class SourceForge(Base): # {{{
+class SourceForge(Base): # {{{
# Note that you should manually ssh once to username,project@frs.sourceforge.net
# on the staging server so that the host key is setup
@@ -378,6 +380,28 @@ class SourceForge(Base): # {{{
# }}}
+def upload_to_servers(files, version): # {{{
+ for server, rdir in {'files':'/usr/share/nginx/html'}.iteritems():
+ print('Uploading to server:', server)
+ server = '%s.calibre-ebook.com' % server
+ rdir = '%s/%s/' % (rdir, version)
+ for x in files:
+ start = time.time()
+ print ('Uploading', x)
+ for i in range(5):
+ try:
+ check_call(['rsync', '-h', '-z', '--progress', '-e', 'ssh -x', x,
+ 'root@%s:%s'%(server, rdir)])
+ except KeyboardInterrupt:
+ raise SystemExit(1)
+ except:
+ print ('\nUpload failed, trying again in 30 seconds')
+ time.sleep(30)
+ else:
+ break
+ print ('Uploaded in', int(time.time() - start), 'seconds\n\n')
+# }}}
+
# CLI {{{
def cli_parser():
epilog='Copyright Kovid Goyal 2012'
@@ -409,6 +433,7 @@ def cli_parser():
sf = subparsers.add_parser('sourceforge', help='Upload to sourceforge',
epilog=epilog)
cron = subparsers.add_parser('cron', help='Call script from cron')
+ subparsers.add_parser('calibre', help='Upload to calibre file servers')
a = gc.add_argument
@@ -471,8 +496,11 @@ def main(args=None):
sf()
elif args.service == 'cron':
login_to_google(args.username, args.password)
+ elif args.service == 'calibre':
+ upload_to_servers(ofiles, args.version)
if __name__ == '__main__':
main()
# }}}
+
diff --git a/setup/upload.py b/setup/upload.py
index 673f9f4679..1c7348bfe9 100644
--- a/setup/upload.py
+++ b/setup/upload.py
@@ -81,7 +81,7 @@ class ReUpload(Command): # {{{
# Data {{{
def get_google_data():
- with open(os.path.expanduser('~/work/kde/conf/googlecodecalibre'), 'rb') as f:
+ with open(os.path.expanduser('~/work/env/private/googlecodecalibre'), 'rb') as f:
gc_password, ga_un, pw = f.read().strip().split('|')
return {
@@ -111,6 +111,9 @@ def sf_cmdline(ver, sdata):
return [__appname__, ver, 'fmap', 'sourceforge', sdata['project'],
sdata['username']]
+def calibre_cmdline(ver):
+ return [__appname__, ver, 'fmap', 'calibre']
+
def run_remote_upload(args):
print 'Running remotely:', ' '.join(args)
subprocess.check_call(['ssh', '-x', '%s@%s'%(STAGING_USER, STAGING_HOST),
@@ -133,7 +136,8 @@ class UploadInstallers(Command): # {{{
try:
self.upload_to_staging(tdir, files)
self.upload_to_sourceforge()
- self.upload_to_google(opts.replace)
+ self.upload_to_calibre()
+ # self.upload_to_google(opts.replace)
finally:
shutil.rmtree(tdir, ignore_errors=True)
@@ -170,6 +174,10 @@ class UploadInstallers(Command): # {{{
sdata = get_sourceforge_data()
args = sf_cmdline(__version__, sdata)
run_remote_upload(args)
+
+ def upload_to_calibre(self):
+ run_remote_upload(calibre_cmdline(__version__))
+
# }}}
class UploadUserManual(Command): # {{{