Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

pep8

Browse Source
This commit is contained in:
Kovid Goyal 2017-02-06 22:12:59 +05:30
parent 6b76d3e17a
commit 66e3d8740e
2 changed files with 235 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
setup/installer/__init__.py
View File

@ -1,7 +1,6 @@
#!/usr/bin/env python2
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
from __future__ import (unicode_literals, division, absolute_import,
print_function)
from __future__ import (unicode_literals, division, absolute_import, print_function)
__license__ = 'GPL v3'
__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
@ -15,27 +14,34 @@ from setup.build_environment import BUILD_HOST, PROJECT
BASE_RSYNC = ['rsync', '-av', '--delete', '--force']
EXCLUDES = []
for x in [
'/src/calibre/plugins', '/manual', '/translations', '/build', '/dist', '/imgsrc', '/format_docs', '/.build-cache',
'.bzr', '.git', '.build', '.svn', '*.pyc', '*.pyo', '*.swp', '*.swo', '*.pyj-cached']:
'/src/calibre/plugins', '/manual', '/translations', '/build', '/dist', '/imgsrc',
'/format_docs', '/.build-cache', '.bzr', '.git', '.build', '.svn', '*.pyc',
'*.pyo', '*.swp', '*.swo', '*.pyj-cached'
]:
EXCLUDES.extend(['--exclude', ('/calibre' + x) if x.startswith('/') else x])
SAFE_EXCLUDES = ['"%s"' % x if '*' in x else x for x in EXCLUDES]
def get_rsync_pw():
return open('/home/kovid/work/env/private/buildbot').read().decode('utf-8').partition(
':')[-1].strip()
return open(os.environ['PENV'] + '/buildbot'
).read().decode('utf-8').partition(':')[-1].strip()
def is_vm_running(name):
qname = '"%s"' % name
try:
lines = subprocess.check_output('VBoxManage list runningvms'.split()).decode('utf-8').splitlines()
lines = subprocess.check_output('VBoxManage list runningvms'.split()
).decode('utf-8').splitlines()
except Exception:
time.sleep(1)
lines = subprocess.check_output('VBoxManage list runningvms'.split()).decode('utf-8').splitlines()
lines = subprocess.check_output('VBoxManage list runningvms'.split()
).decode('utf-8').splitlines()
for line in lines:
if line.startswith(qname):
return True
return False
def is_host_reachable(name, timeout=1):
try:
socket.create_connection((name, 22), timeout).close()
@ -43,12 +49,15 @@ def is_host_reachable(name, timeout=1):
except:
return False
class Rsync(Command):
description = 'Sync source tree from development machine'
SYNC_CMD = ' '.join(BASE_RSYNC+SAFE_EXCLUDES+
['rsync://buildbot@{host}/work/{project}', '..'])
SYNC_CMD = ' '.join(
BASE_RSYNC + SAFE_EXCLUDES +
['rsync://buildbot@{host}/work/{project}', '..']
)
def run(self, opts):
cmd = self.SYNC_CMD.format(host=BUILD_HOST, project=PROJECT)
@ -57,6 +66,7 @@ class Rsync(Command):
self.info(cmd)
subprocess.check_call(cmd, shell=True, env=env)
def push(host, vmname, available):
if vmname is None:
hostname = host.partition(':')[0].partition('@')[-1]
@ -69,6 +79,7 @@ def push(host, vmname, available):
print('\n\nPushing to:', vmname or host, '\n')
subprocess.check_call(rcmd, stdout=open(os.devnull, 'wb'))
class Push(Command):
description = 'Push code to another host'
@ -77,14 +88,17 @@ class Push(Command):
from threading import Thread
threads, available = {}, {}
for host, vmname in {
r'Owner@winxp:/cygdrive/c/Documents\ and\ Settings/Owner/calibre':'winxp',
r'Owner@winxp:/cygdrive/c/Documents\ and\ Settings/Owner/calibre':
'winxp',
'kovid@ox:calibre': None,
r'kovid@win7:/cygdrive/c/Users/kovid/calibre': 'Windows 7',
'kovid@win7-x64:calibre': 'win7-x64',
'kovid@tiny:calibre': None,
'kovid@getafix:calibre-src': None,
}.iteritems():
threads[vmname or host] = thread = Thread(target=push, args=(host, vmname, available))
threads[vmname or host] = thread = Thread(
target=push, args=(host, vmname, available)
)
thread.start()
while threads:
for name, thread in tuple(threads.iteritems()):
@ -105,18 +119,32 @@ class VMInstaller(Command):
IS_64_BIT = False
BUILD_PREFIX = ['#!/bin/sh', 'export CALIBRE_BUILDBOT=1']
BUILD_RSYNC = ['mkdir -p ~/build/{project}', r'cd ~/build/{project}', Rsync.SYNC_CMD]
BUILD_RSYNC = [
'mkdir -p ~/build/{project}', r'cd ~/build/{project}', Rsync.SYNC_CMD
]
BUILD_CLEAN = ['rm -rf dist/* build/* src/calibre/plugins/*']
BUILD_BUILD = ['python setup.py build',]
BUILD_BUILD = [
'python setup.py build',
]
FORCE_SHUTDOWN = 0 # number of seconds to wait before doing a forced power off (0 means disabled)
def add_options(self, parser):
if not parser.has_option('--dont-shutdown'):
parser.add_option('-s', '--dont-shutdown', default=False,
action='store_true', help='Dont shutdown the VM after building')
parser.add_option(
'-s',
'--dont-shutdown',
default=False,
action='store_true',
help='Dont shutdown the VM after building'
)
if not parser.has_option('--dont-strip'):
parser.add_option('-x', '--dont-strip', default=False,
action='store_true', help='Dont strip the generated binaries')
parser.add_option(
'-x',
'--dont-strip',
default=False,
action='store_true',
help='Dont strip the generated binaries'
)
def get_build_script(self):
rs = ['export RSYNC_PASSWORD=%s' % get_rsync_pw()]
@ -131,7 +159,9 @@ class VMInstaller(Command):
def run_vm(self):
if is_vm_running(self.VM_NAME):
return True
self.__p = subprocess.Popen(("VBoxManage startvm %s --type gui" % self.VM_NAME).split())
self.__p = subprocess.Popen(
("VBoxManage startvm %s --type gui" % self.VM_NAME).split()
)
return False
def start_vm(self, sleep=75):
@ -146,13 +176,16 @@ class VMInstaller(Command):
def run_vm_builder(self):
ssh_host = self.VM_NAME
build_script = self.get_build_script()
build_script = build_script.encode('utf-8') if isinstance(build_script, unicode) else build_script
build_script = build_script.encode('utf-8') if isinstance(
build_script, unicode
) else build_script
print('Running VM builder')
p = subprocess.Popen(['ssh', ssh_host, 'bash -s'], stdin=subprocess.PIPE)
p.stdin.write(build_script), p.stdin.flush(), p.stdin.close()
# Save the build script on the build machine for convenient manual
# invocation, if needed
p2 = subprocess.Popen(['ssh', ssh_host, 'cat > build-calibre'], stdin=subprocess.PIPE)
p2 = subprocess.Popen(['ssh', ssh_host, 'cat > build-calibre'],
stdin=subprocess.PIPE)
p2.stdin.write(build_script), p2.stdin.flush(), p2.stdin.close()
p2.wait()
rc = p.wait()
@ -182,12 +215,15 @@ class VMInstaller(Command):
while is_host_reachable(self.VM_NAME):
time.sleep(0.1) # wait for SSH server to shutdown
time.sleep(self.FORCE_SHUTDOWN)
subprocess.check_call(('VBoxManage controlvm %s poweroff' % self.VM_NAME).split())
subprocess.check_call(
('VBoxManage controlvm %s poweroff' % self.VM_NAME).split()
)
def download_installer(self):
installer = self.installer()
subprocess.check_call(['scp',
self.VM_NAME+':build/calibre/'+installer, 'dist'])
subprocess.check_call([
'scp', self.VM_NAME + ':build/calibre/' + installer, 'dist'
])
if not os.path.exists(installer):
raise SystemExit('Failed to download installer: ' + installer)
@ -195,4 +231,3 @@ class VMInstaller(Command):
installer = self.installer()
if os.path.exists(installer):
os.remove(installer)

165
setup/upload.py
View File

@ -23,15 +23,19 @@ STAGING_HOST = 'download.calibre-ebook.com'
STAGING_USER = 'root'
STAGING_DIR = '/root/staging'
def installers(include_source=True):
installers = list(map(installer_name, ('dmg', 'msi', 'txz')))
installers.append(installer_name('txz', is64bit=True))
installers.append(installer_name('msi', is64bit=True))
if include_source:
installers.insert(0, 'dist/%s-%s.tar.xz' % (__appname__, __version__))
installers.append('dist/%s-portable-installer-%s.exe'%(__appname__, __version__))
installers.append(
'dist/%s-portable-installer-%s.exe' % (__appname__, __version__)
)
return installers
def installer_description(fname):
if fname.endswith('.tar.xz'):
return 'Source code'
@ -46,6 +50,7 @@ def installer_description(fname):
return 'Calibre Portable'
return 'Unknown file'
def upload_signatures():
tdir = mkdtemp()
scp = ['scp']
@ -55,7 +60,10 @@ def upload_signatures():
continue
sig = os.path.join(tdir, os.path.basename(installer + '.sig'))
scp.append(sig)
check_call([os.path.expanduser('~/work/env/private/gpg-as-kovid'), '--output', sig, '--detach-sig', installer])
check_call([
os.environ['PENV'] + '/gpg-as-kovid', '--output', sig,
'--detach-sig', installer
])
with open(installer, 'rb') as f:
raw = f.read()
fingerprint = hashlib.sha512(raw).hexdigest()
@ -65,10 +73,13 @@ def upload_signatures():
scp.append(sha512)
for srv in 'code main'.split():
check_call(scp + ['{0}:/srv/{0}/signatures/'.format(srv)])
check_call(['ssh', srv, 'chown', '-R', 'http:http', '/srv/%s/signatures' % srv])
check_call(
['ssh', srv, 'chown', '-R', 'http:http', '/srv/%s/signatures' % srv]
)
finally:
shutil.rmtree(tdir)
class ReUpload(Command): # {{{
description = 'Re-upload any installers present in dist/'
@ -86,44 +97,62 @@ class ReUpload(Command): # {{{
for x in installers():
if os.path.exists(x):
os.remove(x)
# }}}
# Data {{{
def get_github_data():
with open(os.path.expanduser('~/work/env/private/github'), 'rb') as f:
with open(os.environ['PENV'] + '/github', 'rb') as f:
un, pw = f.read().strip().split(':')
return {
'username':un, 'password':pw
}
return {'username': un, 'password': pw}
def get_sourceforge_data():
return {'username': 'kovidgoyal', 'project': 'calibre'}
def get_fosshub_data():
with open(os.path.expanduser('~/work/env/private/fosshub'), 'rb') as f:
with open(os.environ['PENV'] + '/fosshub', 'rb') as f:
return f.read().decode('utf-8')
def send_data(loc):
subprocess.check_call(['rsync', '--inplace', '--delete', '-r', '-z', '-h', '--progress', '-e', 'ssh -x',
loc+'/', '%s@%s:%s'%(STAGING_USER, STAGING_HOST, STAGING_DIR)])
subprocess.check_call([
'rsync', '--inplace', '--delete', '-r', '-z', '-h', '--progress', '-e',
'ssh -x', loc + '/', '%s@%s:%s' % (STAGING_USER, STAGING_HOST, STAGING_DIR)
])
def gh_cmdline(ver, data):
return [__appname__, ver, 'fmap', 'github', __appname__, data['username'], data['password']]
return [
__appname__, ver, 'fmap', 'github', __appname__, data['username'],
data['password']
]
def sf_cmdline(ver, sdata):
return [__appname__, ver, 'fmap', 'sourceforge', sdata['project'],
sdata['username']]
return [
__appname__, ver, 'fmap', 'sourceforge', sdata['project'], sdata['username']
]
def calibre_cmdline(ver):
return [__appname__, ver, 'fmap', 'calibre']
def run_remote_upload(args):
print 'Running remotely:', ' '.join(args)
subprocess.check_call(['ssh', '-x', '%s@%s'%(STAGING_USER, STAGING_HOST),
'cd', STAGING_DIR, '&&', 'python2', 'hosting.py']+args)
subprocess.check_call([
'ssh', '-x', '%s@%s' % (STAGING_USER, STAGING_HOST), 'cd', STAGING_DIR, '&&',
'python2', 'hosting.py'
] + args)
# }}}
def upload_to_fosshub():
# fosshub has no API to do partial uploads, so we always upload all files.
print('Sending upload request to fosshub...')
@ -131,31 +160,57 @@ def upload_to_fosshub():
entries = []
for fname in files:
desc = installer_description(fname)
url = 'https://download.calibre-ebook.com/%s/%s' % (__version__, os.path.basename(fname))
url = 'https://download.calibre-ebook.com/%s/%s' % (
__version__, os.path.basename(fname)
)
entries.append({
'url': url,
'type': desc,
'version': __version__,
})
jq = {'software': 'Calibre', 'apiKey':get_fosshub_data(), 'upload':entries, 'delete':[{'type':'*', 'version':'*', 'name':'*'}]}
jq = {
'software': 'Calibre',
'apiKey': get_fosshub_data(),
'upload': entries,
'delete': [{
'type': '*',
'version': '*',
'name': '*'
}]
}
# print(json.dumps(jq, indent=2))
rq = urllib2.urlopen('https://www.fosshub.com/JSTools/uploadJson', urllib.urlencode({'content':json.dumps(jq)}))
rq = urllib2.urlopen(
'https://www.fosshub.com/JSTools/uploadJson',
urllib.urlencode({
'content': json.dumps(jq)
})
)
resp = rq.read()
if rq.getcode() != httplib.OK:
raise SystemExit('Failed to upload to fosshub, with HTTP error code: %d and response: %s' % (rq.getcode(), resp))
raise SystemExit(
'Failed to upload to fosshub, with HTTP error code: %d and response: %s'
% (rq.getcode(), resp)
)
class UploadInstallers(Command): # {{{
def add_options(self, parser):
parser.add_option('--replace', default=False, action='store_true', help='Replace existing installers')
parser.add_option(
'--replace',
default=False,
action='store_true',
help='Replace existing installers'
)
def run(self, opts):
# return upload_to_fosshub()
all_possible = set(installers())
available = set(glob.glob('dist/*'))
files = {x:installer_description(x) for x in
all_possible.intersection(available)}
files = {
x: installer_description(x)
for x in all_possible.intersection(available)
}
for x in files:
os.chmod(x, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
sizes = {os.path.basename(x): os.path.getsize(x) for x in files}
@ -178,20 +233,26 @@ class UploadInstallers(Command): # {{{
def record_sizes(self, sizes):
print('\nRecording dist sizes')
args = ['%s:%s:%s' % (__version__, fname, size) for fname, size in sizes.iteritems()]
args = [
'%s:%s:%s' % (__version__, fname, size)
for fname, size in sizes.iteritems()
]
check_call(['ssh', 'code', '/usr/local/bin/dist_sizes'] + args)
def upload_to_staging(self, tdir, backup, files):
os.mkdir(tdir + '/dist')
hosting = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'hosting.py')
hosting = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'hosting.py'
)
shutil.copyfile(hosting, os.path.join(tdir, 'hosting.py'))
for f in files:
for x in (tdir + '/dist', backup):
dest = os.path.join(x, os.path.basename(f))
shutil.copy2(f, x)
os.chmod(dest, stat.S_IREAD|stat.S_IWRITE|stat.S_IRGRP|stat.S_IROTH)
os.chmod(
dest, stat.S_IREAD | stat.S_IWRITE | stat.S_IRGRP | stat.S_IROTH
)
with open(os.path.join(tdir, 'fmap'), 'wb') as fo:
for f, desc in files.iteritems():
@ -221,8 +282,10 @@ class UploadInstallers(Command): # {{{
def upload_to_calibre(self):
run_remote_upload(calibre_cmdline(__version__))
# }}}
class UploadUserManual(Command): # {{{
description = 'Build and upload the User Manual'
sub_commands = ['manual']
@ -230,8 +293,10 @@ class UploadUserManual(Command): # {{{
def build_plugin_example(self, path):
from calibre import CurrentDir
with NamedTemporaryFile(suffix='.zip') as f:
os.fchmod(f.fileno(),
stat.S_IRUSR|stat.S_IRGRP|stat.S_IROTH|stat.S_IWRITE)
os.fchmod(
f.fileno(), stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH |
stat.S_IWRITE
)
with CurrentDir(path):
with ZipFile(f, 'w') as zf:
for x in os.listdir('.'):
@ -251,10 +316,18 @@ class UploadUserManual(Command): # {{{
self.build_plugin_example(x)
srcdir = self.j(gettempdir(), 'user-manual-build', 'en', 'html') + '/'
check_call(' '.join(['rsync', '-zrl', '--info=progress2', srcdir, 'main:/srv/manual/']), shell=True)
check_call(
' '.join(
['rsync', '-zrl', '--info=progress2', srcdir, 'main:/srv/manual/']
),
shell=True
)
check_call('ssh main chown -R http:http /srv/manual'.split())
# }}}
class UploadDemo(Command): # {{{
description = 'Rebuild and upload various demos'
@ -266,15 +339,22 @@ class UploadDemo(Command): # {{{
'''--header '''
'''--serif-family "/usr/share/fonts/corefonts, Times New Roman" '''
'''--mono-family "/usr/share/fonts/corefonts, Andale Mono" '''
''''''%self.j(self.SRC, HTML2LRF), shell=True)
'''''' % self.j(self.SRC, HTML2LRF),
shell=True
)
lrf = self.j(self.SRC, 'calibre', 'ebooks', 'lrf', 'html', 'demo')
check_call(
'cd %s && zip -j /tmp/html-demo.zip * /tmp/html2lrf.lrf' % lrf, shell=True)
'cd %s && zip -j /tmp/html-demo.zip * /tmp/html2lrf.lrf' % lrf,
shell=True
)
check_call('scp /tmp/html-demo.zip main:%s/' % (DOWNLOADS, ), shell=True)
# }}}
class UploadToServer(Command): # {{{
description = 'Upload miscellaneous data to calibre server'
@ -283,17 +363,26 @@ class UploadToServer(Command): # {{{
src_file = glob.glob('dist/calibre-*.tar.xz')[0]
upload_signatures()
check_call(['git', 'push'])
check_call(['/home/kovid/work/env/private/gpg-as-kovid', '--armor', '--yes', '--detach-sign', src_file])
check_call([
os['PENV'] + '/gpg-as-kovid', '--armor', '--yes',
'--detach-sign', src_file
])
check_call(['scp', src_file + '.asc', 'code:/srv/code/signatures/'])
check_call('ssh code /usr/local/bin/update-calibre-code.py'.split())
check_call(('ssh code /apps/update-calibre-version.py ' + __version__).split())
check_call(
('ssh code /apps/update-calibre-version.py ' + __version__).split()
)
check_call((
'ssh main /usr/local/bin/update-calibre-version.py %s && /usr/local/bin/update-calibre-code.py && /apps/static/generate.py' % __version__
'ssh main /usr/local/bin/update-calibre-version.py %s && /usr/local/bin/update-calibre-code.py && /apps/static/generate.py'
% __version__
).split())
# }}}
# Testing {{{
def write_files(fmap):
for f in fmap:
with open(f, 'wb') as f:
@ -303,12 +392,16 @@ def write_files(fmap):
for f, desc in fmap.iteritems():
fo.write('%s: %s\n' % (f, desc))
def setup_installers():
ver = '0.0.1'
files = {x.replace(__version__, ver):installer_description(x) for x in installers()}
files = {
x.replace(__version__, ver): installer_description(x)
for x in installers()
}
tdir = mkdtemp()
os.chdir(tdir)
return tdir, files, ver
# }}}
# }}}