Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Change upload installers code to work via a staging server with more upload bandwidth than my current DSL connection

Browse Source
This commit is contained in:
Kovid Goyal 2012-01-03 15:42:57 +05:30
parent 1b6e034512
commit a3f353853a
4 changed files with 602 additions and 364 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
setup/commands.py
View File

@ -16,8 +16,8 @@ __all__ = [
'sdist',
'manual', 'tag_release',
'pypi_register', 'pypi_upload', 'upload_to_server',
'upload_user_manual', 'upload_to_mobileread', 'upload_demo',
'upload_to_sourceforge', 'upload_to_google_code', 'reupload',
'upload_installers',
'upload_user_manual', 'upload_demo', 'reupload',
'linux32', 'linux64', 'linux', 'linux_freeze',
'osx32_freeze', 'osx', 'rsync', 'push',
'win32_freeze', 'win32', 'win',
@ -65,14 +65,12 @@ stage4 = Stage4()
stage5 = Stage5()
publish = Publish()
from setup.upload import UploadUserManual, UploadInstallers, UploadDemo, \
UploadToServer, UploadToSourceForge, UploadToGoogleCode, ReUpload
from setup.upload import (UploadUserManual, UploadDemo, UploadInstallers,
UploadToServer, ReUpload)
upload_user_manual = UploadUserManual()
upload_to_mobileread = UploadInstallers()
upload_demo = UploadDemo()
upload_to_server = UploadToServer()
upload_to_sourceforge = UploadToSourceForge()
upload_to_google_code = UploadToGoogleCode()
upload_installers = UploadInstallers()
reupload = ReUpload()
from setup.installer import Rsync, Push

459
setup/hosting.py Normal file
View File

@ -0,0 +1,459 @@
#!/usr/bin/env python
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
from __future__ import (unicode_literals, division, absolute_import,
print_function)
__license__ = 'GPL v3'
__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import os, time, sys, traceback, subprocess, urllib2, re, base64, httplib
from argparse import ArgumentParser, FileType
from subprocess import check_call
from tempfile import NamedTemporaryFile#, mkdtemp
from collections import OrderedDict
import mechanize
from lxml import html
def login_to_google(username, password):
br = mechanize.Browser()
br.addheaders = [('User-agent',
'Mozilla/5.0 (X11; Linux x86_64; rv:9.0) Gecko/20100101 Firefox/9.0')]
br.set_handle_robots(False)
br.open('https://accounts.google.com/ServiceLogin?service=code')
br.select_form(nr=0)
br.form['Email'] = username
br.form['Passwd'] = password
raw = br.submit().read()
if b'<title>Account overview - Account Settings</title>' not in raw:
raise ValueError(('Failed to login to google with credentials: %s %s'
'\nGoogle sometimes requires verification when logging in from a '
'new IP address. Use lynx to login and supply the verification.')
%(username, password))
return br
class ReadFileWithProgressReporting(file): # {{{
def __init__(self, path, mode='rb'):
file.__init__(self, path, mode)
self.seek(0, os.SEEK_END)
self._total = self.tell()
self.seek(0)
self.start_time = time.time()
def __len__(self):
return self._total
def read(self, size):
data = file.read(self, size)
if data:
self.report_progress(len(data))
return data
def report_progress(self, size):
sys.stdout.write(b'\x1b[s')
sys.stdout.write(b'\x1b[K')
frac = float(self.tell())/self._total
mb_pos = self.tell()/float(1024**2)
mb_tot = self._total/float(1024**2)
kb_pos = self.tell()/1024.0
kb_rate = kb_pos/(time.time()-self.start_time)
bit_rate = kb_rate * 1024
eta = int((self._total - self.tell())/bit_rate) + 1
eta_m, eta_s = eta / 60, eta % 60
sys.stdout.write(
' %.1f%% %.1f/%.1fMB %.1f KB/sec %d minutes, %d seconds left'%(
frac*100, mb_pos, mb_tot, kb_rate, eta_m, eta_s))
sys.stdout.write(b'\x1b[u')
if self.tell() >= self._total:
sys.stdout.write('\n')
t = int(time.time() - self.start_time) + 1
print ('Upload took %d minutes and %d seconds at %.1f KB/sec' % (
t/60, t%60, kb_rate))
sys.stdout.flush()
# }}}
class Base(object): # {{{
def __init__(self):
self.d = os.path.dirname
self.j = os.path.join
self.a = os.path.abspath
self.b = os.path.basename
self.s = os.path.splitext
self.e = os.path.exists
def info(self, *args, **kwargs):
print(*args, **kwargs)
sys.stdout.flush()
def warn(self, *args, **kwargs):
print('\n'+'_'*20, 'WARNING','_'*20)
print(*args, **kwargs)
print('_'*50)
sys.stdout.flush()
#}}}
class GoogleCode(Base):# {{{
def __init__(self,
# A mapping of filenames to file descriptions. The descriptions are
# used to populate the description field for the upload on google
# code
files,
# The unix name for the application.
appname,
# The version being uploaded
version,
# Google account username
username,
# Googlecode.com password
password,
# Google account password
gmail_password,
# The name of the google code project we are uploading to
gc_project,
# Server to which to upload the mapping of file names to google
# code URLs. If not None, upload is performed via shelling out to
# ssh, so you must have ssh-agent setup with the authenticated key
# and ssh agent forwarding enabled
gpaths_server=None,
# The path on gpaths_server to which to upload the mapping data
gpaths=None,
# If True, files are replaced, otherwise existing files are skipped
reupload=False,
# The pattern to match filenames for the files being uploaded and
# extract version information from them. Must have a named group
# named version
filename_pattern=r'{appname}-(?:portable-)?(?P<version>.+?)(?:-(?:i686|x86_64|32bit|64bit))?\.(?:zip|exe|msi|dmg|tar\.bz2|tar\.xz|txz|tbz2)'
):
self.username, self.password, = username, password
self.gmail_password, self.gc_project = gmail_password, gc_project
self.reupload, self.files, self.version = reupload, files, version
self.gpaths, self.gpaths_server = gpaths, gpaths_server
self.upload_host = '%s.googlecode.com'%gc_project
self.files_list = 'http://code.google.com/p/%s/downloads/list'%gc_project
self.delete_url = 'http://code.google.com/p/%s/downloads/delete?name=%%s'%gc_project
self.filename_pat = re.compile(filename_pattern.format(appname=appname))
for x in self.files:
if self.filename_pat.match(os.path.basename(x)) is None:
raise ValueError(('The filename %s does not match the '
'filename pattern')%os.path.basename(x))
def upload_one(self, fname, retries=2):
self.info('\nUploading', fname)
typ = 'Type-' + ('Source' if fname.endswith('.xz') else 'Archive' if
fname.endswith('.zip') else 'Installer')
ext = os.path.splitext(fname)[1][1:]
op = 'OpSys-'+{'msi':'Windows','zip':'Windows',
'dmg':'OSX','bz2':'Linux','xz':'All'}[ext]
desc = self.files[fname]
start = time.time()
for i in range(retries):
try:
path = self.upload(os.path.abspath(fname), desc,
labels=[typ, op, 'Featured'], retry=100)
except KeyboardInterrupt:
raise SystemExit(1)
except:
traceback.print_exc()
print ('\nUpload failed, trying again in 30 secs.',
'%d retries left.'%(retries-1))
time.sleep(30)
else:
break
self.info('Uploaded to:', path, 'in', int(time.time() - start),
'seconds')
return path
def re_upload(self):
fnames = {os.path.basename(x):x for x in self.files}
existing = self.old_files.intersection(set(fnames))
br = self.login_to_google()
for x, src in fnames.iteritems():
if not os.access(src, os.R_OK):
continue
if x in existing:
self.info('Deleting', x)
br.open(self.delete_url%x)
br.select_form(predicate=lambda y: 'delete.do' in y.action)
br.form.find_control(name='delete')
br.submit(name='delete')
self.upload_one(src)
def __call__(self):
self.paths = {}
self.old_files = self.get_old_files()
if self.reupload:
return self.re_upload()
for fname in self.files:
bname = os.path.basename(fname)
if bname in self.old_files:
path = 'http://%s.googlecode.com/files/%s'%(self.gc_project,
bname)
self.info(
'%s already uploaded, skipping. Assuming URL is: %s'%(
bname, path))
self.old_files.remove(bname)
else:
path = self.upload_one(fname)
self.paths[bname] = path
self.info('Updating path map')
for k, v in self.paths.iteritems():
self.info('\t%s => %s'%(k, v))
if self.gpaths and self.gpaths_server:
raw = subprocess.Popen(['ssh', self.gpaths_server, 'cat', self.gpaths],
stdout=subprocess.PIPE).stdout.read()
paths = eval(raw) if raw else {}
paths.update(self.paths)
rem = [x for x in paths if self.version not in x]
for x in rem: paths.pop(x)
raw = ['%r : %r,'%(k, v) for k, v in paths.items()]
raw = '{\n\n%s\n\n}\n'%('\n'.join(raw))
with NamedTemporaryFile() as t:
t.write(raw)
t.flush()
check_call(['scp', t.name, '%s:%s'%(self.gpaths_server,
self.gpaths)])
if self.old_files:
self.br = self.login_to_google()
self.delete_old_files()
def login_to_google(self):
self.info('Logging into Google')
return login_to_google(self.username, self.gmail_password)
def get_files_hosted_by_google_code(self):
self.info('Getting existing files in google code:', self.gc_project)
raw = urllib2.urlopen(self.files_list).read()
root = html.fromstring(raw)
ans = {}
for a in root.xpath('//td[@class="vt id col_0"]/a[@href]'):
ans[a.text.strip()] = a.get('href')
return ans
def get_old_files(self):
ans = set()
for fname in self.get_files_hosted_by_google_code():
m = self.filename_pat.match(fname)
if m is not None:
ans.add(fname)
return ans
def delete_old_files(self):
if not self.old_files:
return
self.info('Deleting old files from Google Code...')
for fname in self.old_files:
self.info('\tDeleting', fname)
self.br.open(self.delete_url%fname)
self.br.select_form(predicate=lambda x: 'delete.do' in x.action)
self.br.form.find_control(name='delete')
self.br.submit(name='delete')
def encode_upload_request(self, fields, file_path):
BOUNDARY = '----------Googlecode_boundary_reindeer_flotilla'
body = []
# Add the metadata about the upload first
for key, value in fields:
body.extend(
['--' + BOUNDARY,
'Content-Disposition: form-data; name="%s"' % key,
'',
value,
])
# Now add the file itself
file_name = os.path.basename(file_path)
with open(file_path, 'rb') as f:
file_content = f.read()
body.extend(
['--' + BOUNDARY,
'Content-Disposition: form-data; name="filename"; filename="%s"'
% file_name,
# The upload server determines the mime-type, no need to set it.
'Content-Type: application/octet-stream',
'',
file_content,
])
# Finalize the form body
body.extend(['--' + BOUNDARY + '--', ''])
body = [x.encode('ascii') if isinstance(x, unicode) else x for x in
body]
return ('multipart/form-data; boundary=%s' % BOUNDARY,
b'\r\n'.join(body))
def upload(self, fname, desc, labels=[], retry=0):
form_fields = [('summary', desc)]
form_fields.extend([('label', l.strip()) for l in labels])
content_type, body = self.encode_upload_request(form_fields, fname)
upload_uri = '/files'
auth_token = base64.b64encode('%s:%s'% (self.username, self.password))
headers = {
'Authorization': 'Basic %s' % auth_token,
'User-Agent': 'googlecode.com uploader v1',
'Content-Type': content_type,
}
with NamedTemporaryFile(delete=False) as f:
f.write(body)
try:
body = ReadFileWithProgressReporting(f.name)
server = httplib.HTTPSConnection(self.upload_host)
server.request('POST', upload_uri, body, headers)
resp = server.getresponse()
server.close()
finally:
os.remove(f.name)
if resp.status == 201:
return resp.getheader('Location')
print ('Failed to upload with code %d and reason: %s'%(resp.status,
resp.reason))
if retry < 1:
print ('Retrying in 5 seconds....')
time.sleep(5)
return self.upload(fname, desc, labels=labels, retry=retry+1)
raise Exception('Failed to upload '+fname)
# }}}
class SourceForge(Base): # {{{
def __init__(self, files, project, version, username, replace=False):
self.username, self.project, self.version = username, project, version
self.base = '/home/frs/project/c/ca/'+project
self.rdir = self.base + '/' + version
self.files = files
def __call__(self):
for x in self.files:
start = time.time()
self.info('Uploading', x)
for i in range(5):
try:
check_call(['rsync', '-h', '-z', '--progress', '-e', 'ssh -x', x,
'%s,%s@frs.sourceforge.net:%s'%(self.username, self.project,
self.rdir+'/')])
except KeyboardInterrupt:
raise SystemExit(1)
except:
print ('\nUpload failed, trying again in 30 seconds')
time.sleep(30)
else:
break
print ('Uploaded in', int(time.time() - start), 'seconds\n\n')
# }}}
# CLI {{{
def cli_parser():
epilog='Copyright Kovid Goyal 2012'
p = ArgumentParser(
description='Upload project files to a hosting service automatically',
epilog=epilog
)
a = p.add_argument
a('appname', help='The name of the application, all files to'
' upload should begin with this name')
a('version', help='The version of the application, all files to'
' upload should contain this version')
a('file_map', type=FileType('rb'),
help='A file containing a mapping of files to be uploaded to '
'descriptions of the files. The descriptions will be visible '
'to users trying to get the file from the hosting service. '
'The format of the file is filename: description, with one per '
'line. filename can be a path to the file relative to the current '
'directory.')
a('--replace', action='store_true', default=False,
help='If specified, existing files are replaced, otherwise '
'they are skipped.')
subparsers = p.add_subparsers(help='Where to upload to', dest='service',
title='Service', description='Hosting service to upload to')
gc = subparsers.add_parser('googlecode', help='Upload to googlecode',
epilog=epilog)
sf = subparsers.add_parser('sourceforge', help='Upload to sourceforge',
epilog=epilog)
a = gc.add_argument
a('project',
help='The name of the project on google code we are uploading to')
a('username',
help='Username to log into your google account')
a('password',
help='Password to log into your google account')
a('gc_password',
help='Password for google code hosting.'
' Get it from http://code.google.com/hosting/settings')
a('--path-map-server',
help='A server to which the mapping of filenames to googlecode '
'URLs will be uploaded. The upload happens via ssh, so you must '
'have a working ssh agent')
a('--path-map-location',
help='Path on the server where the path map is placed.')
a = sf.add_argument
a('project',
help='The name of the project on sourceforge we are uploading to')
a('username',
help='Sourceforge username')
return p
def main(args=None):
cli = cli_parser()
args = cli.parse_args(args)
files = {}
with args.file_map as f:
for line in f:
fname, _, desc = line.partition(':')
fname, desc = fname.strip(), desc.strip()
if fname and desc:
files[fname] = desc
ofiles = OrderedDict()
for x in sorted(files, key=lambda x:os.stat(x).st_size, reverse=True):
ofiles[x] = files[x]
if args.service == 'googlecode':
gc = GoogleCode(ofiles, args.appname, args.version, args.username,
args.gc_password, args.password, args.project,
gpaths_server=args.path_map_server,
gpaths=args.path_map_location, reupload=args.replace)
gc()
elif args.service == 'sourceforge':
sf = SourceForge(ofiles, args.project, args.version, args.username,
replace=args.replace)
sf()
if __name__ == '__main__':
main()
# }}}

2
setup/publish.py
View File

@ -45,7 +45,7 @@ class Stage3(Command):
class Stage4(Command):
description = 'Stage 4 of the publish process'
sub_commands = ['upload_to_sourceforge', 'upload_to_google_code']
sub_commands = ['upload_installers']
class Stage5(Command):

487
setup/upload.py
View File

@ -5,12 +5,15 @@ __license__ = 'GPL v3'
__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import os, re, cStringIO, base64, httplib, subprocess, hashlib, shutil, time, \
glob, stat, sys
import os, re, subprocess, hashlib, shutil, glob, stat, sys
from subprocess import check_call
from tempfile import NamedTemporaryFile, mkdtemp
from zipfile import ZipFile
if __name__ == '__main__':
d = os.path.dirname
sys.path.insert(0, d(d(os.path.abspath(__file__))))
from setup import Command, __version__, installer_name, __appname__
PREFIX = "/var/www/calibre-ebook.com"
@ -19,8 +22,9 @@ BETAS = DOWNLOADS +'/betas'
USER_MANUAL = '/var/www/localhost/htdocs/'
HTML2LRF = "calibre/ebooks/lrf/html/demo"
TXT2LRF = "src/calibre/ebooks/lrf/txt/demo"
MOBILEREAD = 'ftp://dev.mobileread.com/calibre/'
STAGING_HOST = '67.207.135.179'
STAGING_USER = 'root'
STAGING_DIR = '/root/staging'
def installers():
installers = list(map(installer_name, ('dmg', 'msi', 'tar.bz2')))
@ -47,10 +51,10 @@ class ReUpload(Command): # {{{
description = 'Re-uplaod any installers present in dist/'
sub_commands = ['upload_to_google_code', 'upload_to_sourceforge']
sub_commands = ['upload_installers']
def pre_sub_commands(self, opts):
opts.re_upload = True
opts.replace = True
def run(self, opts):
for x in installers():
@ -58,371 +62,91 @@ class ReUpload(Command): # {{{
os.remove(x)
# }}}
class ReadFileWithProgressReporting(file): # {{{
def __init__(self, path, mode='rb'):
file.__init__(self, path, mode)
self.seek(0, os.SEEK_END)
self._total = self.tell()
self.seek(0)
self.start_time = time.time()
def __len__(self):
return self._total
def read(self, size):
data = file.read(self, size)
if data:
self.report_progress(len(data))
return data
def report_progress(self, size):
sys.stdout.write(b'\x1b[s')
sys.stdout.write(b'\x1b[K')
frac = float(self.tell())/self._total
mb_pos = self.tell()/float(1024**2)
mb_tot = self._total/float(1024**2)
kb_pos = self.tell()/1024.0
kb_rate = kb_pos/(time.time()-self.start_time)
bit_rate = kb_rate * 1024
eta = int((self._total - self.tell())/bit_rate) + 1
eta_m, eta_s = eta / 60, eta % 60
sys.stdout.write(
' %.1f%% %.1f/%.1fMB %.1f KB/sec %d minutes, %d seconds left'%(
frac*100, mb_pos, mb_tot, kb_rate, eta_m, eta_s))
sys.stdout.write(b'\x1b[u')
if self.tell() >= self._total:
sys.stdout.write('\n')
t = int(time.time() - self.start_time) + 1
print ('Upload took %d minutes and %d seconds at %.1f KB/sec' % (
t/60, t%60, kb_rate))
sys.stdout.flush()
# }}}
class UploadToGoogleCode(Command): # {{{
USERNAME = 'kovidgoyal'
# Password can be gotten by going to
# http://code.google.com/hosting/settings
# while logged into gmail
# Data {{{
def get_google_data():
PASSWORD_FILE = os.path.expanduser('~/.googlecodecalibre')
OFFLINEIMAP = os.path.expanduser('~/work/kde/conf/offlineimap/rc')
GPATHS = '/var/www/status.calibre-ebook.com/googlepaths'
# If you change this, remember to change the default URL used by
# http://calibre-ebook.com as well
GC_PROJECT = 'calibre-ebook-ii'
UPLOAD_HOST = '%s.googlecode.com'%GC_PROJECT
FILES_LIST = 'http://code.google.com/p/%s/downloads/list'%GC_PROJECT
DELETE_URL = 'http://code.google.com/p/%s/downloads/delete?name=%%s'%GC_PROJECT
def add_options(self, parser):
parser.add_option('--re-upload', default=False, action='store_true',
help='Re-upload all installers currently in dist/')
def re_upload(self):
fnames = set([os.path.basename(x) for x in installers() if not
x.endswith('.tar.xz') and os.path.exists(x)])
existing = set(self.old_files.keys()).intersection(fnames)
br = self.login_to_gmail()
for x in fnames:
src = os.path.join('dist', x)
if not os.access(src, os.R_OK):
continue
if x in existing:
self.info('Deleting', x)
br.open(self.DELETE_URL%x)
br.select_form(predicate=lambda y: 'delete.do' in y.action)
br.form.find_control(name='delete')
br.submit(name='delete')
self.upload_one(src)
def upload_one(self, fname):
self.info('\nUploading', fname)
typ = 'Type-' + ('Source' if fname.endswith('.xz') else 'Archive' if
fname.endswith('.zip') else 'Installer')
ext = os.path.splitext(fname)[1][1:]
op = 'OpSys-'+{'msi':'Windows','zip':'Windows',
'dmg':'OSX','bz2':'Linux','xz':'All'}[ext]
desc = installer_description(fname)
start = time.time()
for i in range(5):
try:
path = self.upload(os.path.abspath(fname), desc,
labels=[typ, op, 'Featured'])
except KeyboardInterrupt:
raise SystemExit(1)
except:
import traceback
traceback.print_exc()
print ('\nUpload failed, trying again in 30 secs')
time.sleep(30)
else:
break
self.info('Uploaded to:', path, 'in', int(time.time() - start),
'seconds')
return path
def run(self, opts):
self.opts = opts
self.password = open(self.PASSWORD_FILE).read().strip()
self.paths = {}
self.old_files = self.get_files_hosted_by_google_code()
if opts.re_upload:
return self.re_upload()
for fname in installers():
bname = os.path.basename(fname)
if bname in self.old_files:
path = 'http://%s.googlecode.com/files/%s'%(self.GC_PROJECT,
bname)
self.info(
'%s already uploaded, skipping. Assuming URL is: %s'%(
bname, path))
self.old_files.pop(bname)
else:
path = self.upload_one(fname)
self.paths[bname] = path
self.info('Updating path map')
self.info(repr(self.paths))
raw = subprocess.Popen(['ssh', 'divok', 'cat', self.GPATHS],
stdout=subprocess.PIPE).stdout.read()
paths = eval(raw)
paths.update(self.paths)
rem = [x for x in paths if __version__ not in x]
for x in rem: paths.pop(x)
raw = ['%r : %r,'%(k, v) for k, v in paths.items()]
raw = '{\n\n%s\n\n}\n'%('\n'.join(raw))
t = NamedTemporaryFile()
t.write(raw)
t.flush()
check_call(['scp', t.name, 'divok:'+self.GPATHS])
self.br = self.login_to_gmail()
self.delete_old_files()
#if len(self.get_files_hosted_by_google_code()) > len(installers()):
# self.warn('Some old files were not deleted from Google Code')
def login_to_gmail(self):
import mechanize
self.info('Logging into Gmail')
raw = open(self.OFFLINEIMAP).read()
gc_password = open(PASSWORD_FILE).read().strip()
raw = open(OFFLINEIMAP).read()
pw = re.search(r'(?s)remoteuser = .*@gmail.com.*?remotepass = (\S+)',
raw).group(1).strip()
br = mechanize.Browser()
br.set_handle_robots(False)
br.open('http://gmail.com')
br.select_form(nr=0)
br.form['Email'] = self.USERNAME
br.form['Passwd'] = pw
br.submit()
return br
def get_files_hosted_by_google_code(self):
import urllib2
from lxml import html
self.info('Getting existing files in google code')
raw = urllib2.urlopen(self.FILES_LIST).read()
root = html.fromstring(raw)
ans = {}
for a in root.xpath('//td[@class="vt id col_0"]/a[@href]'):
ans[a.text.strip()] = a.get('href')
return ans
def delete_old_files(self):
self.info('Deleting old files from Google Code...')
for fname in self.old_files:
ext = fname.rpartition('.')[-1]
if ext in ('flv', 'mp4', 'ogg', 'avi'):
continue
self.info('\tDeleting', fname)
self.br.open(self.DELETE_URL%fname)
self.br.select_form(predicate=lambda x: 'delete.do' in x.action)
self.br.form.find_control(name='delete')
self.br.submit(name='delete')
def encode_upload_request(self, fields, file_path):
BOUNDARY = '----------Googlecode_boundary_reindeer_flotilla'
CRLF = '\r\n'
body = []
# Add the metadata about the upload first
for key, value in fields:
body.extend(
['--' + BOUNDARY,
'Content-Disposition: form-data; name="%s"' % key,
'',
value,
])
# Now add the file itself
file_name = os.path.basename(file_path)
with open(file_path, 'rb') as f:
file_content = f.read()
body.extend(
['--' + BOUNDARY,
'Content-Disposition: form-data; name="filename"; filename="%s"'
% file_name,
# The upload server determines the mime-type, no need to set it.
'Content-Type: application/octet-stream',
'',
file_content,
])
# Finalize the form body
body.extend(['--' + BOUNDARY + '--', ''])
return 'multipart/form-data; boundary=%s' % BOUNDARY, CRLF.join(body)
def upload(self, fname, desc, labels=[], retry=0):
form_fields = [('summary', desc)]
form_fields.extend([('label', l.strip()) for l in labels])
content_type, body = self.encode_upload_request(form_fields, fname)
upload_uri = '/files'
auth_token = base64.b64encode('%s:%s'% (self.USERNAME, self.password))
headers = {
'Authorization': 'Basic %s' % auth_token,
'User-Agent': 'Calibre googlecode.com uploader v0.1.0',
'Content-Type': content_type,
return {
'username':'kovidgoyal@gmail.com', 'password':pw, 'gc_password':gc_password,
'path_map_server':'root@kovidgoyal.net',
'path_map_location':'/var/www/status.calibre-ebook.com/googlepaths',
'project':'calibre-ebook-ii'
}
with NamedTemporaryFile(delete=False) as f:
f.write(body)
def get_sourceforge_data():
return {'username':'kovidgoyal', 'project':'calibre'}
try:
body = ReadFileWithProgressReporting(f.name)
server = httplib.HTTPSConnection(self.UPLOAD_HOST)
server.request('POST', upload_uri, body, headers)
resp = server.getresponse()
server.close()
finally:
os.remove(f.name)
def send_data(loc):
subprocess.check_call(['rsync', '-r', '-z', '-h', '--progress', '-e', 'ssh -x',
loc+'/', '%s@%s:%s'%(STAGING_USER, STAGING_HOST, STAGING_DIR)])
if resp.status == 201:
return resp.getheader('Location')
def gc_cmdline(ver, gdata):
return [__appname__, ver, 'fmap', 'googlecode',
gdata['project'], gdata['username'], gdata['password'],
gdata['gc_password'], '--path-map-server',
gdata['path_map_server'], '--path-map-location',
gdata['path_map_location']]
print 'Failed to upload with code %d and reason: %s'%(resp.status,
resp.reason)
if retry < 1:
print 'Retrying in 5 seconds....'
time.sleep(5)
return self.upload(fname, desc, labels=labels, retry=retry+1)
raise Exception('Failed to upload '+fname)
def sf_cmdline(ver, sdata):
return [__appname__, ver, 'fmap', 'sourceforge', sdata['project'],
sdata['username']]
# }}}
class UploadToSourceForge(Command): # {{{
description = 'Upload release files to sourceforge'
USERNAME = 'kovidgoyal'
PROJECT = 'calibre'
BASE = '/home/frs/project/c/ca/'+PROJECT
@property
def rdir(self):
return self.BASE+'/'+__version__
def upload_installers(self):
for x in installers():
if not os.path.exists(x): continue
start = time.time()
self.info('Uploading', x)
for i in range(5):
try:
check_call(['rsync', '-z', '--progress', '-e', 'ssh -x', x,
'%s,%s@frs.sourceforge.net:%s'%(self.USERNAME, self.PROJECT,
self.rdir+'/')])
except KeyboardInterrupt:
raise SystemExit(1)
except:
print ('\nUpload failed, trying again in 30 seconds')
time.sleep(30)
else:
break
print 'Uploaded in', int(time.time() - start), 'seconds'
print ('\n')
def run(self, opts):
self.opts = opts
self.upload_installers()
def run_remote_upload(args):
print 'Running remotely:', ' '.join(args)
subprocess.check_call(['ssh', '-x', '%s@%s'%(STAGING_USER, STAGING_HOST),
'cd', STAGING_DIR, '&&', 'python', 'hosting.py']+args)
# }}}
class UploadInstallers(Command): # {{{
description = 'Upload any installers present in dist/ to mobileread'
def curl_list_dir(self, url=MOBILEREAD, listonly=1):
import pycurl
c = pycurl.Curl()
c.setopt(pycurl.URL, url)
c.setopt(c.FTP_USE_EPSV, 1)
c.setopt(c.NETRC, c.NETRC_REQUIRED)
c.setopt(c.FTPLISTONLY, listonly)
c.setopt(c.FTP_CREATE_MISSING_DIRS, 1)
b = cStringIO.StringIO()
c.setopt(c.WRITEFUNCTION, b.write)
c.perform()
c.close()
return b.getvalue().split() if listonly else b.getvalue().splitlines()
def curl_delete_file(self, path, url=MOBILEREAD):
import pycurl
c = pycurl.Curl()
c.setopt(pycurl.URL, url)
c.setopt(c.FTP_USE_EPSV, 1)
c.setopt(c.NETRC, c.NETRC_REQUIRED)
self.info('Deleting file %s on %s'%(path, url))
c.setopt(c.QUOTE, ['dele '+ path])
c.perform()
c.close()
def curl_upload_file(self, stream, url):
import pycurl
c = pycurl.Curl()
c.setopt(pycurl.URL, url)
c.setopt(pycurl.UPLOAD, 1)
c.setopt(c.NETRC, c.NETRC_REQUIRED)
c.setopt(pycurl.READFUNCTION, stream.read)
stream.seek(0, 2)
c.setopt(pycurl.INFILESIZE_LARGE, stream.tell())
stream.seek(0)
c.setopt(c.NOPROGRESS, 0)
c.setopt(c.FTP_CREATE_MISSING_DIRS, 1)
self.info('Uploading file %s to url %s' % (getattr(stream, 'name', ''),
url))
try:
c.perform()
c.close()
except:
pass
files = self.curl_list_dir(listonly=0)
for line in files:
line = line.split()
if url.endswith(line[-1]):
size = long(line[4])
stream.seek(0,2)
if size != stream.tell():
raise RuntimeError('curl failed to upload %s correctly'%getattr(stream, 'name', ''))
def upload_installer(self, name):
if not os.path.exists(name):
return
bname = os.path.basename(name)
pat = re.compile(bname.replace(__version__, r'\d+\.\d+\.\d+'))
for f in self.curl_list_dir():
if pat.search(f):
self.curl_delete_file('/calibre/'+f)
self.curl_upload_file(open(name, 'rb'), MOBILEREAD+os.path.basename(name))
def add_option(self, parser):
parser.add_option('--replace', help=
'Replace existing installers, when uploading to google')
def run(self, opts):
self.info('Uploading installers...')
installers = list(map(installer_name, ('dmg', 'msi', 'tar.bz2')))
installers.append(installer_name('tar.bz2', is64bit=True))
map(self.upload_installer, installers)
all_possible = set(installers())
available = set(glob.glob('dist/*'))
files = {x:installer_description(x) for x in
all_possible.intersection(available)}
tdir = mkdtemp()
try:
self.upload_to_staging(tdir, files)
self.upload_to_sourceforge()
self.upload_to_google(opts.replace)
finally:
shutil.rmtree(tdir, ignore_errors=True)
def upload_to_staging(self, tdir, files):
os.mkdir(tdir+'/dist')
hosting = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'hosting.py')
shutil.copyfile(hosting, os.path.join(tdir, 'hosting.py'))
for f in files:
shutil.copyfile(f, os.path.join(tdir, f))
with open(os.path.join(tdir, 'fmap'), 'wb') as fo:
for f, desc in files.iteritems():
fo.write('%s: %s\n'%(f, desc))
send_data(tdir)
def upload_to_google(self, replace):
gdata = get_google_data()
args = gc_cmdline(__version__, gdata)
if replace:
args = ['--replace'] + args
run_remote_upload(args)
def upload_to_sourceforge(self):
sdata = get_sourceforge_data()
args = sf_cmdline(__version__, sdata)
run_remote_upload(args)
# }}}
class UploadUserManual(Command): # {{{
@ -508,4 +232,61 @@ class UploadToServer(Command): # {{{
shutil.rmtree(tdir)
# }}}
# Testing {{{
def write_files(fmap):
for f in fmap:
with open(f, 'wb') as f:
f.write(os.urandom(100))
f.write(b'a'*1000000)
with open('fmap', 'wb') as fo:
for f, desc in fmap.iteritems():
fo.write('%s: %s\n'%(f, desc))
def setup_installers():
ver = '0.0.1'
files = {x.replace(__version__, ver):installer_description(x) for x in installers()}
tdir = mkdtemp()
os.chdir(tdir)
return tdir, files, ver
def test_google_uploader():
gdata = get_google_data()
gdata['project'] = 'calibre-hosting-uploader'
gdata['path_map_location'] += '-test'
hosting = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'hosting.py')
tdir, files, ver = setup_installers()
try:
os.mkdir('dist')
write_files(files)
shutil.copyfile(hosting, 'hosting.py')
send_data(tdir)
args = gc_cmdline(ver, gdata)
print ('Doing initial upload')
run_remote_upload(args)
raw_input('Press Enter to proceed:')
print ('\nDoing re-upload')
run_remote_upload(['--replace']+args)
raw_input('Press Enter to proceed:')
nv = ver + '.1'
files = {x.replace(__version__, nv):installer_description(x) for x in installers()}
write_files(files)
send_data(tdir)
args[1] = nv
print ('\nDoing update upload')
run_remote_upload(args)
print ('\nDont forget to delete any remaining files in the %s project'%
gdata['project'])
finally:
shutil.rmtree(tdir)
# }}}
if __name__ == '__main__':
test_google_uploader()