From 2096dce1cdcea621f314145f71d5ad5a7390a13e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 25 Jun 2013 13:09:51 +0530 Subject: [PATCH] Move User Manual and staging to the download server --- setup/hosting.py | 113 ++++++++++++++++++++++++++++++++++++++++++++--- setup/upload.py | 11 +++-- 2 files changed, 112 insertions(+), 12 deletions(-) diff --git a/setup/hosting.py b/setup/hosting.py index 76ab3992a0..1e78f4694d 100644 --- a/setup/hosting.py +++ b/setup/hosting.py @@ -7,16 +7,14 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, time, sys, traceback, subprocess, urllib2, re, base64, httplib +import os, time, sys, traceback, subprocess, urllib2, re, base64, httplib, shutil from argparse import ArgumentParser, FileType from subprocess import check_call from tempfile import NamedTemporaryFile from collections import OrderedDict -import mechanize -from lxml import html - def login_to_google(username, password): # {{{ + import mechanize br = mechanize.Browser() br.addheaders = [('User-agent', 'Mozilla/5.0 (X11; Linux x86_64; rv:9.0) Gecko/20100101 Firefox/9.0')] @@ -246,6 +244,7 @@ class GoogleCode(Base): # {{{ return login_to_google(self.username, self.gmail_password) def get_files_hosted_by_google_code(self): + from lxml import html self.info('Getting existing files in google code:', self.gc_project) raw = urllib2.urlopen(self.files_list).read() root = html.fromstring(raw) @@ -380,11 +379,111 @@ class SourceForge(Base): # {{{ # }}} +def generate_index(): # {{{ + os.chdir('/srv/download') + releases = set() + for x in os.listdir('.'): + if os.path.isdir(x) and '.' in x: + releases.add(tuple((int(y) for y in x.split('.')))) + rmap = OrderedDict() + for rnum in sorted(releases, reverse=True): + series = rnum[:2] if rnum[0] == 0 else rnum[:1] + if series not in rmap: + rmap[series] = [] + rmap[series].append(rnum) + + template = '''\n {title}

{title}

{msg}

{body} ''' # noqa + style = ''' + body { font-family: sans-serif; background-color: #eee; } + a { text-decoration: none; } + a:visited { color: blue } + a:hover { color: red } + ul { list-style-type: none } + li { padding-bottom: 1ex } + dd li { text-indent: 0; margin: 0 } + dd ul { padding: 0; margin: 0 } + dt { font-weight: bold } + dd { margin-bottom: 2ex } + ''' + body = [] + for series in rmap: + body.append('
  • {0}.x\xa0\xa0\xa0[{1} releases]
    • '.format( # noqa + '.'.join(map(type(''), series)), len(rmap[series]))) + body = ''.format(' '.join(body)) + index = template.format(title='Previous calibre releases', style=style, msg='Choose a series of calibre releases', body=body) + with open('index.html', 'wb') as f: + f.write(index.encode('utf-8')) + + for series, releases in rmap.iteritems(): + sname = '.'.join(map(type(''), series)) + body = [ + '
  • {0}
    • '.format('.'.join(map(type(''), r))) + for r in releases] + body = ''.format(' '.join(body)) + index = template.format(title='Previous calibre releases (%s.x)' % sname, style=style, + msg='Choose a calibre release', body=body) + with open('%s.html' % sname, 'wb') as f: + f.write(index.encode('utf-8')) + + for r in releases: + rname = '.'.join(map(type(''), r)) + os.chdir(rname) + try: + body = [] + files = os.listdir('.') + windows = [x for x in files if x.endswith('.msi')] + if windows: + windows = ['
  • {1}
    • '.format( + x, 'Windows 64-bit Installer' if '64bit' in x else 'Windows 32-bit Installer') + for x in windows] + body.append('
    Windows
      {0}
    '.format(' '.join(windows))) + portable = [x for x in files if '-portable-' in x] + if portable: + body.append('
    Calibre Portable
    {1}
    '.format( + portable[0], 'Calibre Portable Installer')) + osx = [x for x in files if x.endswith('.dmg')] + if osx: + body.append('
    Apple Mac
    {1}
    '.format( + osx[0], 'OS X Disk Image (.dmg)')) + linux = [x for x in files if x.endswith('.bz2')] + if linux: + linux = ['
  • {1}
    • '.format( + x, 'Linux 64-bit binary' if 'x86_64' in x else 'Linux 32-bit binary') + for x in linux] + body.append('
    Linux
      {0}
    '.format(' '.join(linux))) + source = [x for x in files if x.endswith('.xz') or x.endswith('.gz')] + if source: + body.append('
    Source Code
    {1}
    '.format( + source[0], 'Source code (all platforms)')) + + body = '
    {0}
    '.format(''.join(body)) + index = template.format(title='calibre release (%s)' % rname, style=style, + msg='', body=body) + with open('index.html', 'wb') as f: + f.write(index.encode('utf-8')) + finally: + os.chdir('..') + +# }}} + def upload_to_servers(files, version): # {{{ - for server, rdir in {'files':'/usr/share/nginx/html'}.iteritems(): + base = '/srv/download/' + dest = os.path.join(base, version) + if not os.path.exists(dest): + os.mkdir(dest) + for src in files: + shutil.copyfile(src, os.path.join(dest, os.path.basename(src))) + generate_index() + + for server, rdir in {'files':'/srv/download/'}.iteritems(): print('Uploading to server:', server) server = '%s.calibre-ebook.com' % server - rdir = '%s/%s/' % (rdir, version) + # Copy the generated index files + print ('Copying generated index') + check_call(['rsync', '-hzr', '-e', 'ssh -x', '--include', '*.html', + '--filter', '-! */', base, 'root@%s:%s' % (server, rdir)]) + # Copy the release files + rdir = '%s%s/' % (rdir, version) for x in files: start = time.time() print ('Uploading', x) @@ -400,6 +499,7 @@ def upload_to_servers(files, version): # {{{ else: break print ('Uploaded in', int(time.time() - start), 'seconds\n\n') + # }}} def upload_to_dbs(files, version): # {{{ @@ -530,3 +630,4 @@ if __name__ == '__main__': # }}} + diff --git a/setup/upload.py b/setup/upload.py index 8a4e467dd0..639a2e98d5 100644 --- a/setup/upload.py +++ b/setup/upload.py @@ -19,10 +19,9 @@ from setup import Command, __version__, installer_name, __appname__ PREFIX = "/var/www/calibre-ebook.com" DOWNLOADS = PREFIX+"/htdocs/downloads" BETAS = DOWNLOADS +'/betas' -USER_MANUAL = '/var/www/localhost/htdocs/' HTML2LRF = "calibre/ebooks/lrf/html/demo" TXT2LRF = "src/calibre/ebooks/lrf/txt/demo" -STAGING_HOST = '67.207.135.179' +STAGING_HOST = 'download.calibre-ebook.com' STAGING_USER = 'root' STAGING_DIR = '/root/staging' @@ -141,8 +140,8 @@ class UploadInstallers(Command): # {{{ os.mkdir(backup) try: self.upload_to_staging(tdir, backup, files) - self.upload_to_sourceforge() self.upload_to_calibre() + self.upload_to_sourceforge() self.upload_to_dbs() # self.upload_to_google(opts.replace) finally: @@ -219,9 +218,9 @@ class UploadUserManual(Command): # {{{ for x in glob.glob(self.j(path, '*')): self.build_plugin_example(x) - check_call(' '.join(['rsync', '-z', '-r', '--progress', - 'manual/.build/html/', - 'bugs:%s'%USER_MANUAL]), shell=True) + for host in ('download', 'files'): + check_call(' '.join(['rsync', '-z', '-r', '--progress', + 'manual/.build/html/', '%s:/srv/manual/' % host]), shell=True) # }}} class UploadDemo(Command): # {{{