Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Move User Manual and staging to the download server

Browse Source
This commit is contained in:
Kovid Goyal 2013-06-25 13:09:51 +05:30
parent 5b07091d59
commit 2096dce1cd
2 changed files with 112 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

113
setup/hosting.py
View File

@ -7,16 +7,14 @@ __license__ = 'GPL v3'
__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>' __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en' __docformat__ = 'restructuredtext en'
import os, time, sys, traceback, subprocess, urllib2, re, base64, httplib import os, time, sys, traceback, subprocess, urllib2, re, base64, httplib, shutil
from argparse import ArgumentParser, FileType from argparse import ArgumentParser, FileType
from subprocess import check_call from subprocess import check_call
from tempfile import NamedTemporaryFile from tempfile import NamedTemporaryFile
from collections import OrderedDict from collections import OrderedDict
import mechanize
from lxml import html
def login_to_google(username, password): # {{{ def login_to_google(username, password): # {{{
import mechanize
br = mechanize.Browser() br = mechanize.Browser()
br.addheaders = [('User-agent', br.addheaders = [('User-agent',
'Mozilla/5.0 (X11; Linux x86_64; rv:9.0) Gecko/20100101 Firefox/9.0')] 'Mozilla/5.0 (X11; Linux x86_64; rv:9.0) Gecko/20100101 Firefox/9.0')]
@ -246,6 +244,7 @@ class GoogleCode(Base): # {{{
return login_to_google(self.username, self.gmail_password) return login_to_google(self.username, self.gmail_password)
def get_files_hosted_by_google_code(self): def get_files_hosted_by_google_code(self):
from lxml import html
self.info('Getting existing files in google code:', self.gc_project) self.info('Getting existing files in google code:', self.gc_project)
raw = urllib2.urlopen(self.files_list).read() raw = urllib2.urlopen(self.files_list).read()
root = html.fromstring(raw) root = html.fromstring(raw)
@ -380,11 +379,111 @@ class SourceForge(Base): # {{{
# }}} # }}}
def generate_index(): # {{{
os.chdir('/srv/download')
releases = set()
for x in os.listdir('.'):
if os.path.isdir(x) and '.' in x:
releases.add(tuple((int(y) for y in x.split('.'))))
rmap = OrderedDict()
for rnum in sorted(releases, reverse=True):
series = rnum[:2] if rnum[0] == 0 else rnum[:1]
if series not in rmap:
rmap[series] = []
rmap[series].append(rnum)
template = '''<!DOCTYPE html>\n<html lang="en"> <head> <meta charset="utf-8"> <title>{title}</title> <style type="text/css"> {style} </style> </head> <body> <h1>{title}</h1> <p>{msg}</p> {body} </body> </html> ''' # noqa
style = '''
body { font-family: sans-serif; background-color: #eee; }
a { text-decoration: none; }
a:visited { color: blue }
a:hover { color: red }
ul { list-style-type: none }
li { padding-bottom: 1ex }
dd li { text-indent: 0; margin: 0 }
dd ul { padding: 0; margin: 0 }
dt { font-weight: bold }
dd { margin-bottom: 2ex }
'''
body = []
for series in rmap:
body.append('<li><a href="{0}.html" title="Releases in the {0}.x series">{0}.x</a>\xa0\xa0\xa0<span style="font-size:smaller">[{1} releases]</span></li>'.format( # noqa
'.'.join(map(type(''), series)), len(rmap[series])))
body = '<ul>{0}</ul>'.format(' '.join(body))
index = template.format(title='Previous calibre releases', style=style, msg='Choose a series of calibre releases', body=body)
with open('index.html', 'wb') as f:
f.write(index.encode('utf-8'))
for series, releases in rmap.iteritems():
sname = '.'.join(map(type(''), series))
body = [
'<li><a href="{0}/" title="Release {0}">{0}</a></li>'.format('.'.join(map(type(''), r)))
for r in releases]
body = '<ul class="release-list">{0}</ul>'.format(' '.join(body))
index = template.format(title='Previous calibre releases (%s.x)' % sname, style=style,
msg='Choose a calibre release', body=body)
with open('%s.html' % sname, 'wb') as f:
f.write(index.encode('utf-8'))
for r in releases:
rname = '.'.join(map(type(''), r))
os.chdir(rname)
try:
body = []
files = os.listdir('.')
windows = [x for x in files if x.endswith('.msi')]
if windows:
windows = ['<li><a href="{0}" title="{1}">{1}</a></li>'.format(
x, 'Windows 64-bit Installer' if '64bit' in x else 'Windows 32-bit Installer')
for x in windows]
body.append('<dt>Windows</dt><dd><ul>{0}</ul></dd>'.format(' '.join(windows)))
portable = [x for x in files if '-portable-' in x]
if portable:
body.append('<dt>Calibre Portable</dt><dd><a href="{0}" title="{1}">{1}</a></dd>'.format(
portable[0], 'Calibre Portable Installer'))
osx = [x for x in files if x.endswith('.dmg')]
if osx:
body.append('<dt>Apple Mac</dt><dd><a href="{0}" title="{1}">{1}</a></dd>'.format(
osx[0], 'OS X Disk Image (.dmg)'))
linux = [x for x in files if x.endswith('.bz2')]
if linux:
linux = ['<li><a href="{0}" title="{1}">{1}</a></li>'.format(
x, 'Linux 64-bit binary' if 'x86_64' in x else 'Linux 32-bit binary')
for x in linux]
body.append('<dt>Linux</dt><dd><ul>{0}</ul></dd>'.format(' '.join(linux)))
source = [x for x in files if x.endswith('.xz') or x.endswith('.gz')]
if source:
body.append('<dt>Source Code</dt><dd><a href="{0}" title="{1}">{1}</a></dd>'.format(
source[0], 'Source code (all platforms)'))
body = '<dl>{0}</dl>'.format(''.join(body))
index = template.format(title='calibre release (%s)' % rname, style=style,
msg='', body=body)
with open('index.html', 'wb') as f:
f.write(index.encode('utf-8'))
finally:
os.chdir('..')
# }}}
def upload_to_servers(files, version): # {{{ def upload_to_servers(files, version): # {{{
for server, rdir in {'files':'/usr/share/nginx/html'}.iteritems(): base = '/srv/download/'
dest = os.path.join(base, version)
if not os.path.exists(dest):
os.mkdir(dest)
for src in files:
shutil.copyfile(src, os.path.join(dest, os.path.basename(src)))
generate_index()
for server, rdir in {'files':'/srv/download/'}.iteritems():
print('Uploading to server:', server) print('Uploading to server:', server)
server = '%s.calibre-ebook.com' % server server = '%s.calibre-ebook.com' % server
rdir = '%s/%s/' % (rdir, version) # Copy the generated index files
print ('Copying generated index')
check_call(['rsync', '-hzr', '-e', 'ssh -x', '--include', '*.html',
'--filter', '-! */', base, 'root@%s:%s' % (server, rdir)])
# Copy the release files
rdir = '%s%s/' % (rdir, version)
for x in files: for x in files:
start = time.time() start = time.time()
print ('Uploading', x) print ('Uploading', x)
@ -400,6 +499,7 @@ def upload_to_servers(files, version): # {{{
else: else:
break break
print ('Uploaded in', int(time.time() - start), 'seconds\n\n') print ('Uploaded in', int(time.time() - start), 'seconds\n\n')
# }}} # }}}
def upload_to_dbs(files, version): # {{{ def upload_to_dbs(files, version): # {{{
@ -530,3 +630,4 @@ if __name__ == '__main__':
# }}} # }}}

9
setup/upload.py
View File

@ -19,10 +19,9 @@ from setup import Command, __version__, installer_name, __appname__
PREFIX = "/var/www/calibre-ebook.com" PREFIX = "/var/www/calibre-ebook.com"
DOWNLOADS = PREFIX+"/htdocs/downloads" DOWNLOADS = PREFIX+"/htdocs/downloads"
BETAS = DOWNLOADS +'/betas' BETAS = DOWNLOADS +'/betas'
USER_MANUAL = '/var/www/localhost/htdocs/'
HTML2LRF = "calibre/ebooks/lrf/html/demo" HTML2LRF = "calibre/ebooks/lrf/html/demo"
TXT2LRF = "src/calibre/ebooks/lrf/txt/demo" TXT2LRF = "src/calibre/ebooks/lrf/txt/demo"
STAGING_HOST = '67.207.135.179' STAGING_HOST = 'download.calibre-ebook.com'
STAGING_USER = 'root' STAGING_USER = 'root'
STAGING_DIR = '/root/staging' STAGING_DIR = '/root/staging'
@ -141,8 +140,8 @@ class UploadInstallers(Command): # {{{
os.mkdir(backup) os.mkdir(backup)
try: try:
self.upload_to_staging(tdir, backup, files) self.upload_to_staging(tdir, backup, files)
self.upload_to_sourceforge()
self.upload_to_calibre() self.upload_to_calibre()
self.upload_to_sourceforge()
self.upload_to_dbs() self.upload_to_dbs()
# self.upload_to_google(opts.replace) # self.upload_to_google(opts.replace)
finally: finally:
@ -219,9 +218,9 @@ class UploadUserManual(Command): # {{{
for x in glob.glob(self.j(path, '*')): for x in glob.glob(self.j(path, '*')):
self.build_plugin_example(x) self.build_plugin_example(x)
for host in ('download', 'files'):
check_call(' '.join(['rsync', '-z', '-r', '--progress', check_call(' '.join(['rsync', '-z', '-r', '--progress',
'manual/.build/html/', 'manual/.build/html/', '%s:/srv/manual/' % host]), shell=True)
'bugs:%s'%USER_MANUAL]), shell=True)
# }}} # }}}
class UploadDemo(Command): # {{{ class UploadDemo(Command): # {{{