Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

pep8

Browse Source
This commit is contained in:
Kovid Goyal 2017-02-06 22:12:59 +05:30
parent 6b76d3e17a
commit 66e3d8740e
2 changed files with 235 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

127
setup/installer/__init__.py
View File

@ -1,9 +1,8 @@
#!/usr/bin/env python2 #!/usr/bin/env python2
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
from __future__ import (unicode_literals, division, absolute_import, from __future__ import (unicode_literals, division, absolute_import, print_function)
print_function)
__license__ = 'GPL v3' __license__ = 'GPL v3'
__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>' __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en' __docformat__ = 'restructuredtext en'
@ -15,27 +14,34 @@ from setup.build_environment import BUILD_HOST, PROJECT
BASE_RSYNC = ['rsync', '-av', '--delete', '--force'] BASE_RSYNC = ['rsync', '-av', '--delete', '--force']
EXCLUDES = [] EXCLUDES = []
for x in [ for x in [
'/src/calibre/plugins', '/manual', '/translations', '/build', '/dist', '/imgsrc', '/format_docs', '/.build-cache', '/src/calibre/plugins', '/manual', '/translations', '/build', '/dist', '/imgsrc',
'.bzr', '.git', '.build', '.svn', '*.pyc', '*.pyo', '*.swp', '*.swo', '*.pyj-cached']: '/format_docs', '/.build-cache', '.bzr', '.git', '.build', '.svn', '*.pyc',
'*.pyo', '*.swp', '*.swo', '*.pyj-cached'
]:
EXCLUDES.extend(['--exclude', ('/calibre' + x) if x.startswith('/') else x]) EXCLUDES.extend(['--exclude', ('/calibre' + x) if x.startswith('/') else x])
SAFE_EXCLUDES = ['"%s"'%x if '*' in x else x for x in EXCLUDES] SAFE_EXCLUDES = ['"%s"' % x if '*' in x else x for x in EXCLUDES]
def get_rsync_pw(): def get_rsync_pw():
return open('/home/kovid/work/env/private/buildbot').read().decode('utf-8').partition( return open(os.environ['PENV'] + '/buildbot'
':')[-1].strip() ).read().decode('utf-8').partition(':')[-1].strip()
def is_vm_running(name): def is_vm_running(name):
qname = '"%s"' % name qname = '"%s"' % name
try: try:
lines = subprocess.check_output('VBoxManage list runningvms'.split()).decode('utf-8').splitlines() lines = subprocess.check_output('VBoxManage list runningvms'.split()
).decode('utf-8').splitlines()
except Exception: except Exception:
time.sleep(1) time.sleep(1)
lines = subprocess.check_output('VBoxManage list runningvms'.split()).decode('utf-8').splitlines() lines = subprocess.check_output('VBoxManage list runningvms'.split()
).decode('utf-8').splitlines()
for line in lines: for line in lines:
if line.startswith(qname): if line.startswith(qname):
return True return True
return False return False
def is_host_reachable(name, timeout=1): def is_host_reachable(name, timeout=1):
try: try:
socket.create_connection((name, 22), timeout).close() socket.create_connection((name, 22), timeout).close()
@ -43,12 +49,15 @@ def is_host_reachable(name, timeout=1):
except: except:
return False return False
class Rsync(Command): class Rsync(Command):
description = 'Sync source tree from development machine' description = 'Sync source tree from development machine'
SYNC_CMD = ' '.join(BASE_RSYNC+SAFE_EXCLUDES+ SYNC_CMD = ' '.join(
['rsync://buildbot@{host}/work/{project}', '..']) BASE_RSYNC + SAFE_EXCLUDES +
['rsync://buildbot@{host}/work/{project}', '..']
)
def run(self, opts): def run(self, opts):
cmd = self.SYNC_CMD.format(host=BUILD_HOST, project=PROJECT) cmd = self.SYNC_CMD.format(host=BUILD_HOST, project=PROJECT)
@ -57,6 +66,7 @@ class Rsync(Command):
self.info(cmd) self.info(cmd)
subprocess.check_call(cmd, shell=True, env=env) subprocess.check_call(cmd, shell=True, env=env)
def push(host, vmname, available): def push(host, vmname, available):
if vmname is None: if vmname is None:
hostname = host.partition(':')[0].partition('@')[-1] hostname = host.partition(':')[0].partition('@')[-1]
@ -66,9 +76,10 @@ def push(host, vmname, available):
if ok: if ok:
available[vmname or host] = True available[vmname or host] = True
rcmd = BASE_RSYNC + EXCLUDES + ['.', host] rcmd = BASE_RSYNC + EXCLUDES + ['.', host]
print ('\n\nPushing to:', vmname or host, '\n') print('\n\nPushing to:', vmname or host, '\n')
subprocess.check_call(rcmd, stdout=open(os.devnull, 'wb')) subprocess.check_call(rcmd, stdout=open(os.devnull, 'wb'))
class Push(Command): class Push(Command):
description = 'Push code to another host' description = 'Push code to another host'
@ -77,21 +88,24 @@ class Push(Command):
from threading import Thread from threading import Thread
threads, available = {}, {} threads, available = {}, {}
for host, vmname in { for host, vmname in {
r'Owner@winxp:/cygdrive/c/Documents\ and\ Settings/Owner/calibre':'winxp', r'Owner@winxp:/cygdrive/c/Documents\ and\ Settings/Owner/calibre':
'kovid@ox:calibre':None, 'winxp',
r'kovid@win7:/cygdrive/c/Users/kovid/calibre':'Windows 7', 'kovid@ox:calibre': None,
'kovid@win7-x64:calibre':'win7-x64', r'kovid@win7:/cygdrive/c/Users/kovid/calibre': 'Windows 7',
'kovid@tiny:calibre':None, 'kovid@win7-x64:calibre': 'win7-x64',
'kovid@getafix:calibre-src':None, 'kovid@tiny:calibre': None,
}.iteritems(): 'kovid@getafix:calibre-src': None,
threads[vmname or host] = thread = Thread(target=push, args=(host, vmname, available)) }.iteritems():
thread.start() threads[vmname or host] = thread = Thread(
target=push, args=(host, vmname, available)
)
thread.start()
while threads: while threads:
for name, thread in tuple(threads.iteritems()): for name, thread in tuple(threads.iteritems()):
thread.join(0.01) thread.join(0.01)
if not thread.is_alive(): if not thread.is_alive():
if available.get(name, False): if available.get(name, False):
print ('\n\n', name, 'done') print('\n\n', name, 'done')
threads.pop(name) threads.pop(name)
@ -105,22 +119,36 @@ class VMInstaller(Command):
IS_64_BIT = False IS_64_BIT = False
BUILD_PREFIX = ['#!/bin/sh', 'export CALIBRE_BUILDBOT=1'] BUILD_PREFIX = ['#!/bin/sh', 'export CALIBRE_BUILDBOT=1']
BUILD_RSYNC = ['mkdir -p ~/build/{project}', r'cd ~/build/{project}', Rsync.SYNC_CMD] BUILD_RSYNC = [
BUILD_CLEAN = ['rm -rf dist/* build/* src/calibre/plugins/*'] 'mkdir -p ~/build/{project}', r'cd ~/build/{project}', Rsync.SYNC_CMD
BUILD_BUILD = ['python setup.py build',] ]
BUILD_CLEAN = ['rm -rf dist/* build/* src/calibre/plugins/*']
BUILD_BUILD = [
'python setup.py build',
]
FORCE_SHUTDOWN = 0 # number of seconds to wait before doing a forced power off (0 means disabled) FORCE_SHUTDOWN = 0 # number of seconds to wait before doing a forced power off (0 means disabled)
def add_options(self, parser): def add_options(self, parser):
if not parser.has_option('--dont-shutdown'): if not parser.has_option('--dont-shutdown'):
parser.add_option('-s', '--dont-shutdown', default=False, parser.add_option(
action='store_true', help='Dont shutdown the VM after building') '-s',
'--dont-shutdown',
default=False,
action='store_true',
help='Dont shutdown the VM after building'
)
if not parser.has_option('--dont-strip'): if not parser.has_option('--dont-strip'):
parser.add_option('-x', '--dont-strip', default=False, parser.add_option(
action='store_true', help='Dont strip the generated binaries') '-x',
'--dont-strip',
default=False,
action='store_true',
help='Dont strip the generated binaries'
)
def get_build_script(self): def get_build_script(self):
rs = ['export RSYNC_PASSWORD=%s'%get_rsync_pw()] rs = ['export RSYNC_PASSWORD=%s' % get_rsync_pw()]
ans = '\n'.join(self.BUILD_PREFIX + rs)+'\n\n' ans = '\n'.join(self.BUILD_PREFIX + rs) + '\n\n'
ans += ' && \\\n'.join(self.BUILD_RSYNC) + ' && \\\n' ans += ' && \\\n'.join(self.BUILD_RSYNC) + ' && \\\n'
ans += ' && \\\n'.join(self.BUILD_CLEAN) + ' && \\\n' ans += ' && \\\n'.join(self.BUILD_CLEAN) + ' && \\\n'
ans += ' && \\\n'.join(self.BUILD_BUILD) + ' && \\\n' ans += ' && \\\n'.join(self.BUILD_BUILD) + ' && \\\n'
@ -131,7 +159,9 @@ class VMInstaller(Command):
def run_vm(self): def run_vm(self):
if is_vm_running(self.VM_NAME): if is_vm_running(self.VM_NAME):
return True return True
self.__p = subprocess.Popen(("VBoxManage startvm %s --type gui" % self.VM_NAME).split()) self.__p = subprocess.Popen(
("VBoxManage startvm %s --type gui" % self.VM_NAME).split()
)
return False return False
def start_vm(self, sleep=75): def start_vm(self, sleep=75):
@ -139,20 +169,23 @@ class VMInstaller(Command):
already_running = self.run_vm() already_running = self.run_vm()
if not already_running: if not already_running:
time.sleep(2) time.sleep(2)
print ('Waiting for SSH server to start') print('Waiting for SSH server to start')
while not is_host_reachable(ssh_host, timeout=1): while not is_host_reachable(ssh_host, timeout=1):
time.sleep(0.1) time.sleep(0.1)
def run_vm_builder(self): def run_vm_builder(self):
ssh_host = self.VM_NAME ssh_host = self.VM_NAME
build_script = self.get_build_script() build_script = self.get_build_script()
build_script = build_script.encode('utf-8') if isinstance(build_script, unicode) else build_script build_script = build_script.encode('utf-8') if isinstance(
print ('Running VM builder') build_script, unicode
) else build_script
print('Running VM builder')
p = subprocess.Popen(['ssh', ssh_host, 'bash -s'], stdin=subprocess.PIPE) p = subprocess.Popen(['ssh', ssh_host, 'bash -s'], stdin=subprocess.PIPE)
p.stdin.write(build_script), p.stdin.flush(), p.stdin.close() p.stdin.write(build_script), p.stdin.flush(), p.stdin.close()
# Save the build script on the build machine for convenient manual # Save the build script on the build machine for convenient manual
# invocation, if needed # invocation, if needed
p2 = subprocess.Popen(['ssh', ssh_host, 'cat > build-calibre'], stdin=subprocess.PIPE) p2 = subprocess.Popen(['ssh', ssh_host, 'cat > build-calibre'],
stdin=subprocess.PIPE)
p2.stdin.write(build_script), p2.stdin.flush(), p2.stdin.close() p2.stdin.write(build_script), p2.stdin.flush(), p2.stdin.close()
p2.wait() p2.wait()
rc = p.wait() rc = p.wait()
@ -172,27 +205,29 @@ class VMInstaller(Command):
startup_time = time.time() - start_time startup_time = time.time() - start_time
start_time = time.time() start_time = time.time()
self.run_vm_builder() self.run_vm_builder()
print ('Startup completed in %d seconds' % round(startup_time)) print('Startup completed in %d seconds' % round(startup_time))
secs = time.time() - start_time secs = time.time() - start_time
print ('Build completed in %d minutes %d seconds' % (secs // 60, secs % 60)) print('Build completed in %d minutes %d seconds' % (secs // 60, secs % 60))
if not opts.dont_shutdown: if not opts.dont_shutdown:
print ('Shutting down', self.VM_NAME) print('Shutting down', self.VM_NAME)
subprocess.call(['ssh', self.VM_NAME]+self.SHUTDOWN_CMD) subprocess.call(['ssh', self.VM_NAME] + self.SHUTDOWN_CMD)
if self.FORCE_SHUTDOWN: if self.FORCE_SHUTDOWN:
while is_host_reachable(self.VM_NAME): while is_host_reachable(self.VM_NAME):
time.sleep(0.1) # wait for SSH server to shutdown time.sleep(0.1) # wait for SSH server to shutdown
time.sleep(self.FORCE_SHUTDOWN) time.sleep(self.FORCE_SHUTDOWN)
subprocess.check_call(('VBoxManage controlvm %s poweroff' % self.VM_NAME).split()) subprocess.check_call(
('VBoxManage controlvm %s poweroff' % self.VM_NAME).split()
)
def download_installer(self): def download_installer(self):
installer = self.installer() installer = self.installer()
subprocess.check_call(['scp', subprocess.check_call([
self.VM_NAME+':build/calibre/'+installer, 'dist']) 'scp', self.VM_NAME + ':build/calibre/' + installer, 'dist'
])
if not os.path.exists(installer): if not os.path.exists(installer):
raise SystemExit('Failed to download installer: '+installer) raise SystemExit('Failed to download installer: ' + installer)
def clean(self): def clean(self):
installer = self.installer() installer = self.installer()
if os.path.exists(installer): if os.path.exists(installer):
os.remove(installer) os.remove(installer)

215
setup/upload.py
View File

@ -1,7 +1,7 @@
#!/usr/bin/env python2 #!/usr/bin/env python2
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
__license__ = 'GPL v3' __license__ = 'GPL v3'
__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>' __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en' __docformat__ = 'restructuredtext en'
@ -18,20 +18,24 @@ from setup import Command, __version__, installer_name, __appname__
DOWNLOADS = '/srv/main/downloads' DOWNLOADS = '/srv/main/downloads'
HTML2LRF = "calibre/ebooks/lrf/html/demo" HTML2LRF = "calibre/ebooks/lrf/html/demo"
TXT2LRF = "src/calibre/ebooks/lrf/txt/demo" TXT2LRF = "src/calibre/ebooks/lrf/txt/demo"
STAGING_HOST = 'download.calibre-ebook.com' STAGING_HOST = 'download.calibre-ebook.com'
STAGING_USER = 'root' STAGING_USER = 'root'
STAGING_DIR = '/root/staging' STAGING_DIR = '/root/staging'
def installers(include_source=True): def installers(include_source=True):
installers = list(map(installer_name, ('dmg', 'msi', 'txz'))) installers = list(map(installer_name, ('dmg', 'msi', 'txz')))
installers.append(installer_name('txz', is64bit=True)) installers.append(installer_name('txz', is64bit=True))
installers.append(installer_name('msi', is64bit=True)) installers.append(installer_name('msi', is64bit=True))
if include_source: if include_source:
installers.insert(0, 'dist/%s-%s.tar.xz'%(__appname__, __version__)) installers.insert(0, 'dist/%s-%s.tar.xz' % (__appname__, __version__))
installers.append('dist/%s-portable-installer-%s.exe'%(__appname__, __version__)) installers.append(
'dist/%s-portable-installer-%s.exe' % (__appname__, __version__)
)
return installers return installers
def installer_description(fname): def installer_description(fname):
if fname.endswith('.tar.xz'): if fname.endswith('.tar.xz'):
return 'Source code' return 'Source code'
@ -39,13 +43,14 @@ def installer_description(fname):
bits = '32' if 'i686' in fname else '64' bits = '32' if 'i686' in fname else '64'
return bits + 'bit Linux binary' return bits + 'bit Linux binary'
if fname.endswith('.msi'): if fname.endswith('.msi'):
return 'Windows %sinstaller'%('64bit ' if '64bit' in fname else '') return 'Windows %sinstaller' % ('64bit ' if '64bit' in fname else '')
if fname.endswith('.dmg'): if fname.endswith('.dmg'):
return 'OS X dmg' return 'OS X dmg'
if fname.endswith('.exe'): if fname.endswith('.exe'):
return 'Calibre Portable' return 'Calibre Portable'
return 'Unknown file' return 'Unknown file'
def upload_signatures(): def upload_signatures():
tdir = mkdtemp() tdir = mkdtemp()
scp = ['scp'] scp = ['scp']
@ -53,22 +58,28 @@ def upload_signatures():
for installer in installers(): for installer in installers():
if not os.path.exists(installer): if not os.path.exists(installer):
continue continue
sig = os.path.join(tdir, os.path.basename(installer+'.sig')) sig = os.path.join(tdir, os.path.basename(installer + '.sig'))
scp.append(sig) scp.append(sig)
check_call([os.path.expanduser('~/work/env/private/gpg-as-kovid'), '--output', sig, '--detach-sig', installer]) check_call([
os.environ['PENV'] + '/gpg-as-kovid', '--output', sig,
'--detach-sig', installer
])
with open(installer, 'rb') as f: with open(installer, 'rb') as f:
raw = f.read() raw = f.read()
fingerprint = hashlib.sha512(raw).hexdigest() fingerprint = hashlib.sha512(raw).hexdigest()
sha512 = os.path.join(tdir, os.path.basename(installer+'.sha512')) sha512 = os.path.join(tdir, os.path.basename(installer + '.sha512'))
with open(sha512, 'wb') as f: with open(sha512, 'wb') as f:
f.write(fingerprint) f.write(fingerprint)
scp.append(sha512) scp.append(sha512)
for srv in 'code main'.split(): for srv in 'code main'.split():
check_call(scp + ['{0}:/srv/{0}/signatures/'.format(srv)]) check_call(scp + ['{0}:/srv/{0}/signatures/'.format(srv)])
check_call(['ssh', srv, 'chown', '-R', 'http:http', '/srv/%s/signatures' % srv]) check_call(
['ssh', srv, 'chown', '-R', 'http:http', '/srv/%s/signatures' % srv]
)
finally: finally:
shutil.rmtree(tdir) shutil.rmtree(tdir)
class ReUpload(Command): # {{{ class ReUpload(Command): # {{{
description = 'Re-upload any installers present in dist/' description = 'Re-upload any installers present in dist/'
@ -79,51 +90,69 @@ class ReUpload(Command): # {{{
opts.replace = True opts.replace = True
exists = {x for x in installers() if os.path.exists(x)} exists = {x for x in installers() if os.path.exists(x)}
if not exists: if not exists:
print ('There appear to be no installers!') print('There appear to be no installers!')
raise SystemExit(1) raise SystemExit(1)
def run(self, opts): def run(self, opts):
for x in installers(): for x in installers():
if os.path.exists(x): if os.path.exists(x):
os.remove(x) os.remove(x)
# }}} # }}}
# Data {{{ # Data {{{
def get_github_data(): def get_github_data():
with open(os.path.expanduser('~/work/env/private/github'), 'rb') as f: with open(os.environ['PENV'] + '/github', 'rb') as f:
un, pw = f.read().strip().split(':') un, pw = f.read().strip().split(':')
return { return {'username': un, 'password': pw}
'username':un, 'password':pw
}
def get_sourceforge_data(): def get_sourceforge_data():
return {'username':'kovidgoyal', 'project':'calibre'} return {'username': 'kovidgoyal', 'project': 'calibre'}
def get_fosshub_data(): def get_fosshub_data():
with open(os.path.expanduser('~/work/env/private/fosshub'), 'rb') as f: with open(os.environ['PENV'] + '/fosshub', 'rb') as f:
return f.read().decode('utf-8') return f.read().decode('utf-8')
def send_data(loc): def send_data(loc):
subprocess.check_call(['rsync', '--inplace', '--delete', '-r', '-z', '-h', '--progress', '-e', 'ssh -x', subprocess.check_call([
loc+'/', '%s@%s:%s'%(STAGING_USER, STAGING_HOST, STAGING_DIR)]) 'rsync', '--inplace', '--delete', '-r', '-z', '-h', '--progress', '-e',
'ssh -x', loc + '/', '%s@%s:%s' % (STAGING_USER, STAGING_HOST, STAGING_DIR)
])
def gh_cmdline(ver, data): def gh_cmdline(ver, data):
return [__appname__, ver, 'fmap', 'github', __appname__, data['username'], data['password']] return [
__appname__, ver, 'fmap', 'github', __appname__, data['username'],
data['password']
]
def sf_cmdline(ver, sdata): def sf_cmdline(ver, sdata):
return [__appname__, ver, 'fmap', 'sourceforge', sdata['project'], return [
sdata['username']] __appname__, ver, 'fmap', 'sourceforge', sdata['project'], sdata['username']
]
def calibre_cmdline(ver): def calibre_cmdline(ver):
return [__appname__, ver, 'fmap', 'calibre'] return [__appname__, ver, 'fmap', 'calibre']
def run_remote_upload(args): def run_remote_upload(args):
print 'Running remotely:', ' '.join(args) print 'Running remotely:', ' '.join(args)
subprocess.check_call(['ssh', '-x', '%s@%s'%(STAGING_USER, STAGING_HOST), subprocess.check_call([
'cd', STAGING_DIR, '&&', 'python2', 'hosting.py']+args) 'ssh', '-x', '%s@%s' % (STAGING_USER, STAGING_HOST), 'cd', STAGING_DIR, '&&',
'python2', 'hosting.py'
] + args)
# }}} # }}}
def upload_to_fosshub(): def upload_to_fosshub():
# fosshub has no API to do partial uploads, so we always upload all files. # fosshub has no API to do partial uploads, so we always upload all files.
print('Sending upload request to fosshub...') print('Sending upload request to fosshub...')
@ -131,34 +160,60 @@ def upload_to_fosshub():
entries = [] entries = []
for fname in files: for fname in files:
desc = installer_description(fname) desc = installer_description(fname)
url = 'https://download.calibre-ebook.com/%s/%s' % (__version__, os.path.basename(fname)) url = 'https://download.calibre-ebook.com/%s/%s' % (
__version__, os.path.basename(fname)
)
entries.append({ entries.append({
'url':url, 'url': url,
'type': desc, 'type': desc,
'version': __version__, 'version': __version__,
}) })
jq = {'software': 'Calibre', 'apiKey':get_fosshub_data(), 'upload':entries, 'delete':[{'type':'*', 'version':'*', 'name':'*'}]} jq = {
'software': 'Calibre',
'apiKey': get_fosshub_data(),
'upload': entries,
'delete': [{
'type': '*',
'version': '*',
'name': '*'
}]
}
# print(json.dumps(jq, indent=2)) # print(json.dumps(jq, indent=2))
rq = urllib2.urlopen('https://www.fosshub.com/JSTools/uploadJson', urllib.urlencode({'content':json.dumps(jq)})) rq = urllib2.urlopen(
'https://www.fosshub.com/JSTools/uploadJson',
urllib.urlencode({
'content': json.dumps(jq)
})
)
resp = rq.read() resp = rq.read()
if rq.getcode() != httplib.OK: if rq.getcode() != httplib.OK:
raise SystemExit('Failed to upload to fosshub, with HTTP error code: %d and response: %s' % (rq.getcode(), resp)) raise SystemExit(
'Failed to upload to fosshub, with HTTP error code: %d and response: %s'
% (rq.getcode(), resp)
)
class UploadInstallers(Command): # {{{ class UploadInstallers(Command): # {{{
def add_options(self, parser): def add_options(self, parser):
parser.add_option('--replace', default=False, action='store_true', help='Replace existing installers') parser.add_option(
'--replace',
default=False,
action='store_true',
help='Replace existing installers'
)
def run(self, opts): def run(self, opts):
# return upload_to_fosshub() # return upload_to_fosshub()
all_possible = set(installers()) all_possible = set(installers())
available = set(glob.glob('dist/*')) available = set(glob.glob('dist/*'))
files = {x:installer_description(x) for x in files = {
all_possible.intersection(available)} x: installer_description(x)
for x in all_possible.intersection(available)
}
for x in files: for x in files:
os.chmod(x, stat.S_IRUSR|stat.S_IWUSR|stat.S_IRGRP|stat.S_IROTH) os.chmod(x, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
sizes = {os.path.basename(x):os.path.getsize(x) for x in files} sizes = {os.path.basename(x): os.path.getsize(x) for x in files}
self.record_sizes(sizes) self.record_sizes(sizes)
tdir = mkdtemp() tdir = mkdtemp()
backup = os.path.join('/mnt/external/calibre/%s' % __version__) backup = os.path.join('/mnt/external/calibre/%s' % __version__)
@ -177,25 +232,31 @@ class UploadInstallers(Command): # {{{
shutil.rmtree(tdir, ignore_errors=True) shutil.rmtree(tdir, ignore_errors=True)
def record_sizes(self, sizes): def record_sizes(self, sizes):
print ('\nRecording dist sizes') print('\nRecording dist sizes')
args = ['%s:%s:%s' % (__version__, fname, size) for fname, size in sizes.iteritems()] args = [
'%s:%s:%s' % (__version__, fname, size)
for fname, size in sizes.iteritems()
]
check_call(['ssh', 'code', '/usr/local/bin/dist_sizes'] + args) check_call(['ssh', 'code', '/usr/local/bin/dist_sizes'] + args)
def upload_to_staging(self, tdir, backup, files): def upload_to_staging(self, tdir, backup, files):
os.mkdir(tdir+'/dist') os.mkdir(tdir + '/dist')
hosting = os.path.join(os.path.dirname(os.path.abspath(__file__)), hosting = os.path.join(
'hosting.py') os.path.dirname(os.path.abspath(__file__)), 'hosting.py'
)
shutil.copyfile(hosting, os.path.join(tdir, 'hosting.py')) shutil.copyfile(hosting, os.path.join(tdir, 'hosting.py'))
for f in files: for f in files:
for x in (tdir+'/dist', backup): for x in (tdir + '/dist', backup):
dest = os.path.join(x, os.path.basename(f)) dest = os.path.join(x, os.path.basename(f))
shutil.copy2(f, x) shutil.copy2(f, x)
os.chmod(dest, stat.S_IREAD|stat.S_IWRITE|stat.S_IRGRP|stat.S_IROTH) os.chmod(
dest, stat.S_IREAD | stat.S_IWRITE | stat.S_IRGRP | stat.S_IROTH
)
with open(os.path.join(tdir, 'fmap'), 'wb') as fo: with open(os.path.join(tdir, 'fmap'), 'wb') as fo:
for f, desc in files.iteritems(): for f, desc in files.iteritems():
fo.write('%s: %s\n'%(f, desc)) fo.write('%s: %s\n' % (f, desc))
while True: while True:
try: try:
@ -221,8 +282,10 @@ class UploadInstallers(Command): # {{{
def upload_to_calibre(self): def upload_to_calibre(self):
run_remote_upload(calibre_cmdline(__version__)) run_remote_upload(calibre_cmdline(__version__))
# }}} # }}}
class UploadUserManual(Command): # {{{ class UploadUserManual(Command): # {{{
description = 'Build and upload the User Manual' description = 'Build and upload the User Manual'
sub_commands = ['manual'] sub_commands = ['manual']
@ -230,8 +293,10 @@ class UploadUserManual(Command): # {{{
def build_plugin_example(self, path): def build_plugin_example(self, path):
from calibre import CurrentDir from calibre import CurrentDir
with NamedTemporaryFile(suffix='.zip') as f: with NamedTemporaryFile(suffix='.zip') as f:
os.fchmod(f.fileno(), os.fchmod(
stat.S_IRUSR|stat.S_IRGRP|stat.S_IROTH|stat.S_IWRITE) f.fileno(), stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH |
stat.S_IWRITE
)
with CurrentDir(path): with CurrentDir(path):
with ZipFile(f, 'w') as zf: with ZipFile(f, 'w') as zf:
for x in os.listdir('.'): for x in os.listdir('.'):
@ -242,8 +307,8 @@ class UploadUserManual(Command): # {{{
for y in os.listdir(x): for y in os.listdir(x):
zf.write(os.path.join(x, y)) zf.write(os.path.join(x, y))
bname = self.b(path) + '_plugin.zip' bname = self.b(path) + '_plugin.zip'
dest = '%s/%s'%(DOWNLOADS, bname) dest = '%s/%s' % (DOWNLOADS, bname)
subprocess.check_call(['scp', f.name, 'main:'+dest]) subprocess.check_call(['scp', f.name, 'main:' + dest])
def run(self, opts): def run(self, opts):
path = self.j(self.SRC, '..', 'manual', 'plugin_examples') path = self.j(self.SRC, '..', 'manual', 'plugin_examples')
@ -251,30 +316,45 @@ class UploadUserManual(Command): # {{{
self.build_plugin_example(x) self.build_plugin_example(x)
srcdir = self.j(gettempdir(), 'user-manual-build', 'en', 'html') + '/' srcdir = self.j(gettempdir(), 'user-manual-build', 'en', 'html') + '/'
check_call(' '.join(['rsync', '-zrl', '--info=progress2', srcdir, 'main:/srv/manual/']), shell=True) check_call(
' '.join(
['rsync', '-zrl', '--info=progress2', srcdir, 'main:/srv/manual/']
),
shell=True
)
check_call('ssh main chown -R http:http /srv/manual'.split()) check_call('ssh main chown -R http:http /srv/manual'.split())
# }}} # }}}
class UploadDemo(Command): # {{{ class UploadDemo(Command): # {{{
description = 'Rebuild and upload various demos' description = 'Rebuild and upload various demos'
def run(self, opts): def run(self, opts):
check_call( check_call(
'''ebook-convert %s/demo.html /tmp/html2lrf.lrf ''' '''ebook-convert %s/demo.html /tmp/html2lrf.lrf '''
'''--title='Demonstration of html2lrf' --authors='Kovid Goyal' ''' '''--title='Demonstration of html2lrf' --authors='Kovid Goyal' '''
'''--header ''' '''--header '''
'''--serif-family "/usr/share/fonts/corefonts, Times New Roman" ''' '''--serif-family "/usr/share/fonts/corefonts, Times New Roman" '''
'''--mono-family "/usr/share/fonts/corefonts, Andale Mono" ''' '''--mono-family "/usr/share/fonts/corefonts, Andale Mono" '''
''''''%self.j(self.SRC, HTML2LRF), shell=True) '''''' % self.j(self.SRC, HTML2LRF),
shell=True
)
lrf = self.j(self.SRC, 'calibre', 'ebooks', 'lrf', 'html', 'demo') lrf = self.j(self.SRC, 'calibre', 'ebooks', 'lrf', 'html', 'demo')
check_call( check_call(
'cd %s && zip -j /tmp/html-demo.zip * /tmp/html2lrf.lrf' % lrf, shell=True) 'cd %s && zip -j /tmp/html-demo.zip * /tmp/html2lrf.lrf' % lrf,
shell=True
)
check_call('scp /tmp/html-demo.zip main:%s/' % (DOWNLOADS, ), shell=True)
check_call('scp /tmp/html-demo.zip main:%s/'%(DOWNLOADS,), shell=True)
# }}} # }}}
class UploadToServer(Command): # {{{ class UploadToServer(Command): # {{{
description = 'Upload miscellaneous data to calibre server' description = 'Upload miscellaneous data to calibre server'
@ -283,32 +363,45 @@ class UploadToServer(Command): # {{{
src_file = glob.glob('dist/calibre-*.tar.xz')[0] src_file = glob.glob('dist/calibre-*.tar.xz')[0]
upload_signatures() upload_signatures()
check_call(['git', 'push']) check_call(['git', 'push'])
check_call(['/home/kovid/work/env/private/gpg-as-kovid', '--armor', '--yes', '--detach-sign', src_file]) check_call([
os['PENV'] + '/gpg-as-kovid', '--armor', '--yes',
'--detach-sign', src_file
])
check_call(['scp', src_file + '.asc', 'code:/srv/code/signatures/']) check_call(['scp', src_file + '.asc', 'code:/srv/code/signatures/'])
check_call('ssh code /usr/local/bin/update-calibre-code.py'.split()) check_call('ssh code /usr/local/bin/update-calibre-code.py'.split())
check_call(('ssh code /apps/update-calibre-version.py ' + __version__).split()) check_call(
('ssh code /apps/update-calibre-version.py ' + __version__).split()
)
check_call(( check_call((
'ssh main /usr/local/bin/update-calibre-version.py %s && /usr/local/bin/update-calibre-code.py && /apps/static/generate.py' % __version__ 'ssh main /usr/local/bin/update-calibre-version.py %s && /usr/local/bin/update-calibre-code.py && /apps/static/generate.py'
% __version__
).split()) ).split())
# }}} # }}}
# Testing {{{ # Testing {{{
def write_files(fmap): def write_files(fmap):
for f in fmap: for f in fmap:
with open(f, 'wb') as f: with open(f, 'wb') as f:
f.write(os.urandom(100)) f.write(os.urandom(100))
f.write(b'a'*1000000) f.write(b'a' * 1000000)
with open('fmap', 'wb') as fo: with open('fmap', 'wb') as fo:
for f, desc in fmap.iteritems(): for f, desc in fmap.iteritems():
fo.write('%s: %s\n'%(f, desc)) fo.write('%s: %s\n' % (f, desc))
def setup_installers(): def setup_installers():
ver = '0.0.1' ver = '0.0.1'
files = {x.replace(__version__, ver):installer_description(x) for x in installers()} files = {
x.replace(__version__, ver): installer_description(x)
for x in installers()
}
tdir = mkdtemp() tdir = mkdtemp()
os.chdir(tdir) os.chdir(tdir)
return tdir, files, ver return tdir, files, ver
# }}}
# }}}