mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-07-09 03:04:10 -04:00
Purge dead code for uploading to the now defunct googlecode
This commit is contained in:
Kovid Goyal
parent
89cf80ab74
commit
d8ee6308d6
326
setup/hosting.py
326
setup/hosting.py
@ -7,37 +7,12 @@ __license__ = 'GPL v3'
|
||||
__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
|
||||
__docformat__ = 'restructuredtext en'
|
||||
|
||||
import os, time, sys, traceback, subprocess, urllib2, re, base64, httplib, shutil, glob, json, mimetypes
|
||||
import os, time, sys, shutil, glob, json, mimetypes
|
||||
from pprint import pprint
|
||||
from argparse import ArgumentParser, FileType
|
||||
from subprocess import check_call, CalledProcessError, check_output
|
||||
from tempfile import NamedTemporaryFile
|
||||
from collections import OrderedDict
|
||||
|
||||
def login_to_google(username, password): # {{{
|
||||
import mechanize
|
||||
br = mechanize.Browser()
|
||||
br.addheaders = [('User-agent',
|
||||
'Mozilla/5.0 (X11; Linux x86_64; rv:9.0) Gecko/20100101 Firefox/9.0')]
|
||||
br.set_handle_robots(False)
|
||||
br.open('https://accounts.google.com/ServiceLogin?service=code')
|
||||
br.select_form(nr=0)
|
||||
br.form['Email'] = username
|
||||
br.form['Passwd'] = password
|
||||
raw = br.submit().read()
|
||||
if re.search(br'(?i)<title>.*?Account Settings</title>', raw) is None:
|
||||
x = re.search(br'(?is)<title>.*?</title>', raw)
|
||||
if x is not None:
|
||||
print ('Title of post login page: %s'%x.group())
|
||||
# open('/tmp/goog.html', 'wb').write(raw)
|
||||
raise ValueError(('Failed to login to google with credentials: %s %s'
|
||||
'\nGoogle sometimes requires verification when logging in from a '
|
||||
'new IP address. Use lynx to login and supply the verification, '
|
||||
'at: lynx -accept_all_cookies https://accounts.google.com/ServiceLogin?service=code')
|
||||
%(username, password))
|
||||
return br
|
||||
# }}}
|
||||
|
||||
class ReadFileWithProgressReporting(file): # {{{
|
||||
|
||||
def __init__(self, path, mode='rb'):
|
||||
@ -101,254 +76,6 @@ class Base(object): # {{{
|
||||
|
||||
# }}}
|
||||
|
||||
class GoogleCode(Base): # {{{
|
||||
|
||||
def __init__(self,
|
||||
# A mapping of filenames to file descriptions. The descriptions are
|
||||
# used to populate the description field for the upload on google
|
||||
# code
|
||||
files,
|
||||
|
||||
# The unix name for the application.
|
||||
appname,
|
||||
|
||||
# The version being uploaded
|
||||
version,
|
||||
|
||||
# Google account username
|
||||
username,
|
||||
|
||||
# Googlecode.com password
|
||||
password,
|
||||
|
||||
# Google account password
|
||||
gmail_password,
|
||||
|
||||
# The name of the google code project we are uploading to
|
||||
gc_project,
|
||||
|
||||
# Server to which to upload the mapping of file names to google
|
||||
# code URLs. If not None, upload is performed via shelling out to
|
||||
# ssh, so you must have ssh-agent setup with the authenticated key
|
||||
# and ssh agent forwarding enabled
|
||||
gpaths_server=None,
|
||||
# The path on gpaths_server to which to upload the mapping data
|
||||
gpaths=None,
|
||||
|
||||
# If True, files are replaced, otherwise existing files are skipped
|
||||
reupload=False,
|
||||
|
||||
# The pattern to match filenames for the files being uploaded and
|
||||
# extract version information from them. Must have a named group
|
||||
# named version
|
||||
filename_pattern=r'{appname}-(?:portable-installer-)?(?P<version>.+?)(?:-(?:i686|x86_64|32bit|64bit))?\.(?:zip|exe|msi|dmg|tar\.bz2|tar\.xz|txz|tbz2)' # noqa
|
||||
|
||||
):
|
||||
self.username, self.password, = username, password
|
||||
self.gmail_password, self.gc_project = gmail_password, gc_project
|
||||
self.reupload, self.files, self.version = reupload, files, version
|
||||
self.gpaths, self.gpaths_server = gpaths, gpaths_server
|
||||
|
||||
self.upload_host = '%s.googlecode.com'%gc_project
|
||||
self.files_list = 'http://code.google.com/p/%s/downloads/list'%gc_project
|
||||
self.delete_url = 'http://code.google.com/p/%s/downloads/delete?name=%%s'%gc_project
|
||||
|
||||
self.filename_pat = re.compile(filename_pattern.format(appname=appname))
|
||||
for x in self.files:
|
||||
if self.filename_pat.match(os.path.basename(x)) is None:
|
||||
raise ValueError(('The filename %s does not match the '
|
||||
'filename pattern')%os.path.basename(x))
|
||||
|
||||
def upload_one(self, fname, retries=2):
|
||||
self.info('\nUploading', fname)
|
||||
typ = 'Type-' + ('Source' if fname.endswith('.xz') else 'Archive' if
|
||||
fname.endswith('.zip') else 'Installer')
|
||||
ext = os.path.splitext(fname)[1][1:]
|
||||
op = 'OpSys-'+{'msi':'Windows','exe':'Windows',
|
||||
'dmg':'OSX','txz':'Linux','xz':'All'}[ext]
|
||||
desc = self.files[fname]
|
||||
start = time.time()
|
||||
for i in range(retries):
|
||||
try:
|
||||
path = self.upload(os.path.abspath(fname), desc,
|
||||
labels=[typ, op, 'Featured'], retry=100)
|
||||
except KeyboardInterrupt:
|
||||
raise SystemExit(1)
|
||||
except:
|
||||
traceback.print_exc()
|
||||
print ('\nUpload failed, trying again in 30 secs.',
|
||||
'%d retries left.'%(retries-1))
|
||||
time.sleep(30)
|
||||
else:
|
||||
break
|
||||
self.info('Uploaded to:', path, 'in', int(time.time() - start),
|
||||
'seconds')
|
||||
return path
|
||||
|
||||
def re_upload(self):
|
||||
fnames = {os.path.basename(x):x for x in self.files}
|
||||
existing = self.old_files.intersection(set(fnames))
|
||||
br = self.login_to_google()
|
||||
for x, src in fnames.iteritems():
|
||||
if not os.access(src, os.R_OK):
|
||||
continue
|
||||
if x in existing:
|
||||
self.info('Deleting', x)
|
||||
br.open(self.delete_url%x)
|
||||
br.select_form(predicate=lambda y: 'delete.do' in y.action)
|
||||
br.form.find_control(name='delete')
|
||||
br.submit(name='delete')
|
||||
self.upload_one(src)
|
||||
|
||||
def __call__(self):
|
||||
self.paths = {}
|
||||
self.old_files = self.get_old_files()
|
||||
if self.reupload:
|
||||
return self.re_upload()
|
||||
|
||||
for fname in self.files:
|
||||
bname = os.path.basename(fname)
|
||||
if bname in self.old_files:
|
||||
path = 'http://%s.googlecode.com/files/%s'%(self.gc_project,
|
||||
bname)
|
||||
self.info(
|
||||
'%s already uploaded, skipping. Assuming URL is: %s'%(
|
||||
bname, path))
|
||||
self.old_files.remove(bname)
|
||||
else:
|
||||
path = self.upload_one(fname)
|
||||
self.paths[bname] = path
|
||||
self.info('Updating path map')
|
||||
for k, v in self.paths.iteritems():
|
||||
self.info('\t%s => %s'%(k, v))
|
||||
if self.gpaths and self.gpaths_server:
|
||||
raw = subprocess.Popen(['ssh', self.gpaths_server, 'cat', self.gpaths],
|
||||
stdout=subprocess.PIPE).stdout.read()
|
||||
paths = eval(raw) if raw else {}
|
||||
paths.update(self.paths)
|
||||
rem = [x for x in paths if self.version not in x]
|
||||
for x in rem:
|
||||
paths.pop(x)
|
||||
raw = ['%r : %r,'%(k, v) for k, v in paths.items()]
|
||||
raw = '{\n\n%s\n\n}\n'%('\n'.join(raw))
|
||||
with NamedTemporaryFile() as t:
|
||||
t.write(raw)
|
||||
t.flush()
|
||||
check_call(['scp', t.name, '%s:%s'%(self.gpaths_server,
|
||||
self.gpaths)])
|
||||
if self.old_files:
|
||||
self.br = self.login_to_google()
|
||||
self.delete_old_files()
|
||||
|
||||
def login_to_google(self):
|
||||
self.info('Logging into Google')
|
||||
return login_to_google(self.username, self.gmail_password)
|
||||
|
||||
def get_files_hosted_by_google_code(self):
|
||||
from lxml import html
|
||||
self.info('Getting existing files in google code:', self.gc_project)
|
||||
raw = urllib2.urlopen(self.files_list).read()
|
||||
root = html.fromstring(raw)
|
||||
ans = {}
|
||||
for a in root.xpath('//td[@class="vt id col_0"]/a[@href]'):
|
||||
ans[a.text.strip()] = a.get('href')
|
||||
return ans
|
||||
|
||||
def get_old_files(self):
|
||||
ans = set()
|
||||
for fname in self.get_files_hosted_by_google_code():
|
||||
m = self.filename_pat.match(fname)
|
||||
if m is not None:
|
||||
ans.add(fname)
|
||||
return ans
|
||||
|
||||
def delete_old_files(self):
|
||||
if not self.old_files:
|
||||
return
|
||||
self.info('Deleting old files from Google Code...')
|
||||
for fname in self.old_files:
|
||||
self.info('\tDeleting', fname)
|
||||
self.br.open(self.delete_url%fname)
|
||||
self.br.select_form(predicate=lambda x: 'delete.do' in x.action)
|
||||
self.br.form.find_control(name='delete')
|
||||
self.br.submit(name='delete')
|
||||
|
||||
def encode_upload_request(self, fields, file_path):
|
||||
BOUNDARY = '----------Googlecode_boundary_reindeer_flotilla'
|
||||
|
||||
body = []
|
||||
|
||||
# Add the metadata about the upload first
|
||||
for key, value in fields:
|
||||
body.extend(
|
||||
['--' + BOUNDARY,
|
||||
'Content-Disposition: form-data; name="%s"' % key,
|
||||
'',
|
||||
value,
|
||||
])
|
||||
|
||||
# Now add the file itself
|
||||
file_name = os.path.basename(file_path)
|
||||
with open(file_path, 'rb') as f:
|
||||
file_content = f.read()
|
||||
|
||||
body.extend(
|
||||
['--' + BOUNDARY,
|
||||
'Content-Disposition: form-data; name="filename"; filename="%s"'
|
||||
% file_name,
|
||||
# The upload server determines the mime-type, no need to set it.
|
||||
'Content-Type: application/octet-stream',
|
||||
'',
|
||||
file_content,
|
||||
])
|
||||
|
||||
# Finalize the form body
|
||||
body.extend(['--' + BOUNDARY + '--', ''])
|
||||
body = [x.encode('ascii') if isinstance(x, unicode) else x for x in
|
||||
body]
|
||||
|
||||
return ('multipart/form-data; boundary=%s' % BOUNDARY,
|
||||
b'\r\n'.join(body))
|
||||
|
||||
def upload(self, fname, desc, labels=[], retry=0):
|
||||
form_fields = [('summary', desc)]
|
||||
form_fields.extend([('label', l.strip()) for l in labels])
|
||||
|
||||
content_type, body = self.encode_upload_request(form_fields, fname)
|
||||
upload_uri = '/files'
|
||||
auth_token = base64.b64encode('%s:%s'% (self.username, self.password))
|
||||
headers = {
|
||||
'Authorization': 'Basic %s' % auth_token,
|
||||
'User-Agent': 'googlecode.com uploader v1',
|
||||
'Content-Type': content_type,
|
||||
}
|
||||
|
||||
with NamedTemporaryFile(delete=False) as f:
|
||||
f.write(body)
|
||||
|
||||
try:
|
||||
body = ReadFileWithProgressReporting(f.name)
|
||||
server = httplib.HTTPSConnection(self.upload_host)
|
||||
server.request('POST', upload_uri, body, headers)
|
||||
resp = server.getresponse()
|
||||
server.close()
|
||||
finally:
|
||||
os.remove(f.name)
|
||||
|
||||
if resp.status == 201:
|
||||
return resp.getheader('Location')
|
||||
|
||||
print ('Failed to upload with code %d and reason: %s'%(resp.status,
|
||||
resp.reason))
|
||||
if retry < 1:
|
||||
print ('Retrying in 5 seconds....')
|
||||
time.sleep(5)
|
||||
return self.upload(fname, desc, labels=labels, retry=retry+1)
|
||||
raise Exception('Failed to upload '+fname)
|
||||
|
||||
|
||||
# }}}
|
||||
|
||||
class SourceForge(Base): # {{{
|
||||
|
||||
# Note that you should manually ssh once to username,project@frs.sourceforge.net
|
||||
@ -687,47 +414,19 @@ def cli_parser():
|
||||
|
||||
subparsers = p.add_subparsers(help='Where to upload to', dest='service',
|
||||
title='Service', description='Hosting service to upload to')
|
||||
gc = subparsers.add_parser('googlecode', help='Upload to googlecode',
|
||||
epilog=epilog)
|
||||
sf = subparsers.add_parser('sourceforge', help='Upload to sourceforge',
|
||||
epilog=epilog)
|
||||
gh = subparsers.add_parser('github', help='Upload to GitHub',
|
||||
epilog=epilog)
|
||||
cron = subparsers.add_parser('cron', help='Call script from cron')
|
||||
subparsers.add_parser('calibre', help='Upload to calibre file servers')
|
||||
subparsers.add_parser('dbs', help='Upload to fosshub.com')
|
||||
|
||||
a = gc.add_argument
|
||||
|
||||
a('project',
|
||||
help='The name of the project on google code we are uploading to')
|
||||
a('username',
|
||||
help='Username to log into your google account')
|
||||
a('password',
|
||||
help='Password to log into your google account')
|
||||
a('gc_password',
|
||||
help='Password for google code hosting.'
|
||||
' Get it from http://code.google.com/hosting/settings')
|
||||
|
||||
a('--path-map-server',
|
||||
help='A server to which the mapping of filenames to googlecode '
|
||||
'URLs will be uploaded. The upload happens via ssh, so you must '
|
||||
'have a working ssh agent')
|
||||
a('--path-map-location',
|
||||
help='Path on the server where the path map is placed.')
|
||||
|
||||
a = sf.add_argument
|
||||
a('project',
|
||||
help='The name of the project on sourceforge we are uploading to')
|
||||
a('username',
|
||||
help='Sourceforge username')
|
||||
|
||||
a = cron.add_argument
|
||||
a('username',
|
||||
help='Username to log into your google account')
|
||||
a('password',
|
||||
help='Password to log into your google account')
|
||||
|
||||
a = gh.add_argument
|
||||
a('project',
|
||||
help='The name of the repository on GitHub we are uploading to')
|
||||
@ -742,25 +441,18 @@ def main(args=None):
|
||||
cli = cli_parser()
|
||||
args = cli.parse_args(args)
|
||||
files = {}
|
||||
if args.service != 'cron':
|
||||
with args.file_map as f:
|
||||
for line in f:
|
||||
fname, _, desc = line.partition(':')
|
||||
fname, desc = fname.strip(), desc.strip()
|
||||
if fname and desc:
|
||||
files[fname] = desc
|
||||
with args.file_map as f:
|
||||
for line in f:
|
||||
fname, _, desc = line.partition(':')
|
||||
fname, desc = fname.strip(), desc.strip()
|
||||
if fname and desc:
|
||||
files[fname] = desc
|
||||
|
||||
ofiles = OrderedDict()
|
||||
for x in sorted(files, key=lambda x:os.stat(x).st_size, reverse=True):
|
||||
ofiles[x] = files[x]
|
||||
|
||||
if args.service == 'googlecode':
|
||||
gc = GoogleCode(ofiles, args.appname, args.version, args.username,
|
||||
args.gc_password, args.password, args.project,
|
||||
gpaths_server=args.path_map_server,
|
||||
gpaths=args.path_map_location, reupload=args.replace)
|
||||
gc()
|
||||
elif args.service == 'sourceforge':
|
||||
if args.service == 'sourceforge':
|
||||
sf = SourceForge(ofiles, args.project, args.version, args.username,
|
||||
replace=args.replace)
|
||||
sf()
|
||||
@ -768,8 +460,6 @@ def main(args=None):
|
||||
gh = GitHub(ofiles, args.project, args.version, args.username, args.password,
|
||||
replace=args.replace)
|
||||
gh()
|
||||
elif args.service == 'cron':
|
||||
login_to_google(args.username, args.password)
|
||||
elif args.service == 'calibre':
|
||||
upload_to_servers(ofiles, args.version)
|
||||
elif args.service == 'dbs':
|
||||
|
||||
@ -83,19 +83,6 @@ class ReUpload(Command): # {{{
|
||||
# }}}
|
||||
|
||||
# Data {{{
|
||||
def get_google_data():
|
||||
with open(os.path.expanduser('~/work/env/private/googlecodecalibre'), 'rb') as f:
|
||||
gc_password, ga_un, pw = f.read().strip().split('|')
|
||||
|
||||
return {
|
||||
'username':ga_un, 'password':pw, 'gc_password':gc_password,
|
||||
'path_map_server':'root@kovidgoyal.net',
|
||||
'path_map_location':'/var/www/status.calibre-ebook.com/googlepaths',
|
||||
# If you change this remember to change it in the
|
||||
# status.calibre-ebook.com server as well
|
||||
'project':'calibre-ebook'
|
||||
}
|
||||
|
||||
def get_github_data():
|
||||
with open(os.path.expanduser('~/work/env/private/github'), 'rb') as f:
|
||||
un, pw = f.read().strip().split(':')
|
||||
@ -110,13 +97,6 @@ def send_data(loc):
|
||||
subprocess.check_call(['rsync', '--inplace', '--delete', '-r', '-z', '-h', '--progress', '-e', 'ssh -x',
|
||||
loc+'/', '%s@%s:%s'%(STAGING_USER, STAGING_HOST, STAGING_DIR)])
|
||||
|
||||
def gc_cmdline(ver, gdata):
|
||||
return [__appname__, ver, 'fmap', 'googlecode',
|
||||
gdata['project'], gdata['username'], gdata['password'],
|
||||
gdata['gc_password'], '--path-map-server',
|
||||
gdata['path_map_server'], '--path-map-location',
|
||||
gdata['path_map_location']]
|
||||
|
||||
def gh_cmdline(ver, data):
|
||||
return [__appname__, ver, 'fmap', 'github', __appname__, data['username'], data['password']]
|
||||
|
||||
@ -164,7 +144,6 @@ class UploadInstallers(Command): # {{{
|
||||
self.upload_to_sourceforge()
|
||||
self.upload_to_dbs()
|
||||
self.upload_to_github(opts.replace)
|
||||
# self.upload_to_google(opts.replace)
|
||||
finally:
|
||||
shutil.rmtree(tdir, ignore_errors=True)
|
||||
|
||||
@ -198,13 +177,6 @@ class UploadInstallers(Command): # {{{
|
||||
else:
|
||||
break
|
||||
|
||||
def upload_to_google(self, replace):
|
||||
gdata = get_google_data()
|
||||
args = gc_cmdline(__version__, gdata)
|
||||
if replace:
|
||||
args = ['--replace'] + args
|
||||
run_remote_upload(args)
|
||||
|
||||
def upload_to_github(self, replace):
|
||||
data = get_github_data()
|
||||
args = gh_cmdline(__version__, data)
|
||||
@ -307,42 +279,5 @@ def setup_installers():
|
||||
os.chdir(tdir)
|
||||
return tdir, files, ver
|
||||
|
||||
def test_google_uploader():
|
||||
gdata = get_google_data()
|
||||
gdata['project'] = 'calibre-hosting-uploader'
|
||||
gdata['path_map_location'] += '-test'
|
||||
hosting = os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
||||
'hosting.py')
|
||||
|
||||
tdir, files, ver = setup_installers()
|
||||
try:
|
||||
os.mkdir('dist')
|
||||
write_files(files)
|
||||
shutil.copyfile(hosting, 'hosting.py')
|
||||
send_data(tdir)
|
||||
args = gc_cmdline(ver, gdata)
|
||||
|
||||
print ('Doing initial upload')
|
||||
run_remote_upload(args)
|
||||
raw_input('Press Enter to proceed:')
|
||||
|
||||
print ('\nDoing re-upload')
|
||||
run_remote_upload(['--replace']+args)
|
||||
raw_input('Press Enter to proceed:')
|
||||
|
||||
nv = ver + '.1'
|
||||
files = {x.replace(__version__, nv):installer_description(x) for x in installers()}
|
||||
write_files(files)
|
||||
send_data(tdir)
|
||||
args[1] = nv
|
||||
print ('\nDoing update upload')
|
||||
run_remote_upload(args)
|
||||
print ('\nDont forget to delete any remaining files in the %s project'%
|
||||
gdata['project'])
|
||||
|
||||
finally:
|
||||
shutil.rmtree(tdir)
|
||||
# }}}
|
||||
|
||||
if __name__ == '__main__':
|
||||
test_google_uploader()
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user