mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-06-01 04:34:16 -04:00
403 lines
13 KiB
Python
403 lines
13 KiB
Python
#!/usr/bin/env python2
|
|
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
|
|
|
|
from __future__ import print_function
|
|
__license__ = 'GPL v3'
|
|
__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
|
|
__docformat__ = 'restructuredtext en'
|
|
|
|
import os, subprocess, hashlib, shutil, glob, stat, sys, time, json
|
|
from subprocess import check_call
|
|
from tempfile import NamedTemporaryFile, mkdtemp, gettempdir
|
|
from zipfile import ZipFile
|
|
from polyglot.builtins import iteritems
|
|
from polyglot.urllib import urlopen, Request
|
|
|
|
if __name__ == '__main__':
|
|
d = os.path.dirname
|
|
sys.path.insert(0, d(d(os.path.abspath(__file__))))
|
|
|
|
from setup import Command, __version__, installer_name, __appname__
|
|
|
|
DOWNLOADS = '/srv/main/downloads'
|
|
HTML2LRF = "calibre/ebooks/lrf/html/demo"
|
|
TXT2LRF = "src/calibre/ebooks/lrf/txt/demo"
|
|
STAGING_HOST = 'download.calibre-ebook.com'
|
|
STAGING_USER = 'root'
|
|
STAGING_DIR = '/root/staging'
|
|
|
|
|
|
def installers(include_source=True):
|
|
installers = list(map(installer_name, ('dmg', 'msi', 'txz')))
|
|
installers.append(installer_name('txz', is64bit=True))
|
|
installers.append(installer_name('msi', is64bit=True))
|
|
if include_source:
|
|
installers.insert(0, 'dist/%s-%s.tar.xz' % (__appname__, __version__))
|
|
installers.append(
|
|
'dist/%s-portable-installer-%s.exe' % (__appname__, __version__)
|
|
)
|
|
return installers
|
|
|
|
|
|
def installer_description(fname):
|
|
if fname.endswith('.tar.xz'):
|
|
return 'Source code'
|
|
if fname.endswith('.txz'):
|
|
bits = '32' if 'i686' in fname else '64'
|
|
return bits + 'bit Linux binary'
|
|
if fname.endswith('.msi'):
|
|
return 'Windows %sinstaller' % ('64bit ' if '64bit' in fname else '')
|
|
if fname.endswith('.dmg'):
|
|
return 'OS X dmg'
|
|
if fname.endswith('.exe'):
|
|
return 'Calibre Portable'
|
|
return 'Unknown file'
|
|
|
|
|
|
def upload_signatures():
|
|
tdir = mkdtemp()
|
|
scp = ['scp']
|
|
try:
|
|
for installer in installers():
|
|
if not os.path.exists(installer):
|
|
continue
|
|
sig = os.path.join(tdir, os.path.basename(installer + '.sig'))
|
|
scp.append(sig)
|
|
check_call([
|
|
os.environ['PENV'] + '/gpg-as-kovid', '--output', sig,
|
|
'--detach-sig', installer
|
|
])
|
|
with open(installer, 'rb') as f:
|
|
raw = f.read()
|
|
fingerprint = hashlib.sha512(raw).hexdigest()
|
|
sha512 = os.path.join(tdir, os.path.basename(installer + '.sha512'))
|
|
with open(sha512, 'wb') as f:
|
|
f.write(fingerprint)
|
|
scp.append(sha512)
|
|
for srv in 'code main'.split():
|
|
check_call(scp + ['{0}:/srv/{0}/signatures/'.format(srv)])
|
|
check_call(
|
|
['ssh', srv, 'chown', '-R', 'http:http', '/srv/%s/signatures' % srv]
|
|
)
|
|
finally:
|
|
shutil.rmtree(tdir)
|
|
|
|
|
|
class ReUpload(Command): # {{{
|
|
|
|
description = 'Re-upload any installers present in dist/'
|
|
|
|
sub_commands = ['upload_installers']
|
|
|
|
def pre_sub_commands(self, opts):
|
|
opts.replace = True
|
|
exists = {x for x in installers() if os.path.exists(x)}
|
|
if not exists:
|
|
print('There appear to be no installers!')
|
|
raise SystemExit(1)
|
|
|
|
def run(self, opts):
|
|
for x in installers():
|
|
if os.path.exists(x):
|
|
os.remove(x)
|
|
|
|
|
|
# }}}
|
|
|
|
|
|
# Data {{{
|
|
def get_github_data():
|
|
with open(os.environ['PENV'] + '/github-token', 'rb') as f:
|
|
un, pw = f.read().strip().split(':')
|
|
return {'username': un, 'password': pw}
|
|
|
|
|
|
def get_sourceforge_data():
|
|
return {'username': 'kovidgoyal', 'project': 'calibre'}
|
|
|
|
|
|
def get_fosshub_data():
|
|
with open(os.environ['PENV'] + '/fosshub', 'rb') as f:
|
|
return f.read().decode('utf-8').strip()
|
|
|
|
|
|
def send_data(loc):
|
|
subprocess.check_call([
|
|
'rsync', '--inplace', '--delete', '-r', '-zz', '-h', '--progress', '-e',
|
|
'ssh -x', loc + '/', '%s@%s:%s' % (STAGING_USER, STAGING_HOST, STAGING_DIR)
|
|
])
|
|
|
|
|
|
def gh_cmdline(ver, data):
|
|
return [
|
|
__appname__, ver, 'fmap', 'github', __appname__, data['username'],
|
|
data['password']
|
|
]
|
|
|
|
|
|
def sf_cmdline(ver, sdata):
|
|
return [
|
|
__appname__, ver, 'fmap', 'sourceforge', sdata['project'], sdata['username']
|
|
]
|
|
|
|
|
|
def calibre_cmdline(ver):
|
|
return [__appname__, ver, 'fmap', 'calibre']
|
|
|
|
|
|
def run_remote_upload(args):
|
|
print('Running remotely:', ' '.join(args))
|
|
subprocess.check_call([
|
|
'ssh', '-x', '%s@%s' % (STAGING_USER, STAGING_HOST), 'cd', STAGING_DIR, '&&',
|
|
'python2', 'hosting.py'
|
|
] + args)
|
|
|
|
|
|
# }}}
|
|
|
|
|
|
def upload_to_fosshub():
|
|
# https://devzone.fosshub.com/dashboard/restApi
|
|
# fosshub has no API to do partial uploads, so we always upload all files.
|
|
api_key = get_fosshub_data()
|
|
|
|
def request(path, data=None):
|
|
r = Request('https://api.fosshub.com/rest/' + path.lstrip('/'),
|
|
headers={
|
|
'Content-Type': 'application/json',
|
|
'X-auth-key': api_key,
|
|
'User-Agent': 'calibre'
|
|
})
|
|
res = urlopen(r, data=data)
|
|
ans = json.loads(res.read())
|
|
if ans.get('error'):
|
|
raise SystemExit(ans['error'])
|
|
if res.getcode() != 200:
|
|
raise SystemExit('Request to {} failed with response code: {}'.format(path, res.getcode()))
|
|
# from pprint import pprint
|
|
# pprint(ans)
|
|
return ans['status'] if 'status' in ans else ans['data']
|
|
|
|
print('Sending upload request to fosshub...')
|
|
project_id = None
|
|
|
|
for project in request('projects'):
|
|
if project['name'].lower() == 'calibre':
|
|
project_id = project['id']
|
|
break
|
|
else:
|
|
raise SystemExit('No calibre project found')
|
|
|
|
files = set(installers())
|
|
entries = []
|
|
for fname in files:
|
|
desc = installer_description(fname)
|
|
url = 'https://download.calibre-ebook.com/%s/%s' % (
|
|
__version__, os.path.basename(fname)
|
|
)
|
|
entries.append({
|
|
'fileUrl': url,
|
|
'type': desc,
|
|
'version': __version__,
|
|
})
|
|
jq = {
|
|
'version': __version__,
|
|
'files': entries,
|
|
'publish': True,
|
|
'isOldRelease': False,
|
|
}
|
|
# print(json.dumps(jq, indent=2))
|
|
if not request('projects/{}/releases/'.format(project_id), data=json.dumps(jq)):
|
|
raise SystemExit('Failed to queue publish job with fosshub')
|
|
|
|
|
|
class UploadInstallers(Command): # {{{
|
|
|
|
def add_options(self, parser):
|
|
parser.add_option(
|
|
'--replace',
|
|
default=False,
|
|
action='store_true',
|
|
help='Replace existing installers'
|
|
)
|
|
|
|
def run(self, opts):
|
|
# return upload_to_fosshub()
|
|
all_possible = set(installers())
|
|
available = set(glob.glob('dist/*'))
|
|
files = {
|
|
x: installer_description(x)
|
|
for x in all_possible.intersection(available)
|
|
}
|
|
for x in files:
|
|
os.chmod(x, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
|
|
sizes = {os.path.basename(x): os.path.getsize(x) for x in files}
|
|
self.record_sizes(sizes)
|
|
tdir = mkdtemp()
|
|
backup = os.path.join('/mnt/external/calibre/%s' % __version__)
|
|
if not os.path.exists(backup):
|
|
os.mkdir(backup)
|
|
try:
|
|
self.upload_to_staging(tdir, backup, files)
|
|
self.upload_to_calibre()
|
|
if opts.replace:
|
|
upload_signatures()
|
|
check_call('ssh code /apps/update-calibre-version.py'.split())
|
|
# self.upload_to_sourceforge()
|
|
upload_to_fosshub()
|
|
self.upload_to_github(opts.replace)
|
|
finally:
|
|
shutil.rmtree(tdir, ignore_errors=True)
|
|
|
|
def record_sizes(self, sizes):
|
|
print('\nRecording dist sizes')
|
|
args = [
|
|
'%s:%s:%s' % (__version__, fname, size)
|
|
for fname, size in iteritems(sizes)
|
|
]
|
|
check_call(['ssh', 'code', '/usr/local/bin/dist_sizes'] + args)
|
|
|
|
def upload_to_staging(self, tdir, backup, files):
|
|
os.mkdir(tdir + '/dist')
|
|
hosting = os.path.join(
|
|
os.path.dirname(os.path.abspath(__file__)), 'hosting.py'
|
|
)
|
|
shutil.copyfile(hosting, os.path.join(tdir, 'hosting.py'))
|
|
|
|
for f in files:
|
|
for x in (tdir + '/dist', backup):
|
|
dest = os.path.join(x, os.path.basename(f))
|
|
shutil.copy2(f, x)
|
|
os.chmod(
|
|
dest, stat.S_IREAD | stat.S_IWRITE | stat.S_IRGRP | stat.S_IROTH
|
|
)
|
|
|
|
with open(os.path.join(tdir, 'fmap'), 'wb') as fo:
|
|
for f, desc in iteritems(files):
|
|
fo.write('%s: %s\n' % (f, desc))
|
|
|
|
while True:
|
|
try:
|
|
send_data(tdir)
|
|
except:
|
|
print('\nUpload to staging failed, retrying in a minute')
|
|
time.sleep(60)
|
|
else:
|
|
break
|
|
|
|
def upload_to_github(self, replace):
|
|
data = get_github_data()
|
|
args = gh_cmdline(__version__, data)
|
|
if replace:
|
|
args = ['--replace'] + args
|
|
run_remote_upload(args)
|
|
|
|
def upload_to_sourceforge(self):
|
|
sdata = get_sourceforge_data()
|
|
args = sf_cmdline(__version__, sdata)
|
|
run_remote_upload(args)
|
|
|
|
def upload_to_calibre(self):
|
|
run_remote_upload(calibre_cmdline(__version__))
|
|
|
|
|
|
# }}}
|
|
|
|
|
|
class UploadUserManual(Command): # {{{
|
|
description = 'Build and upload the User Manual'
|
|
sub_commands = ['manual']
|
|
|
|
def build_plugin_example(self, path):
|
|
from calibre import CurrentDir
|
|
with NamedTemporaryFile(suffix='.zip') as f:
|
|
os.fchmod(
|
|
f.fileno(), stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH |
|
|
stat.S_IWRITE
|
|
)
|
|
with CurrentDir(path):
|
|
with ZipFile(f, 'w') as zf:
|
|
for x in os.listdir('.'):
|
|
if x.endswith('.swp'):
|
|
continue
|
|
zf.write(x)
|
|
if os.path.isdir(x):
|
|
for y in os.listdir(x):
|
|
zf.write(os.path.join(x, y))
|
|
bname = self.b(path) + '_plugin.zip'
|
|
dest = '%s/%s' % (DOWNLOADS, bname)
|
|
subprocess.check_call(['scp', f.name, 'main:' + dest])
|
|
|
|
def run(self, opts):
|
|
path = self.j(self.SRC, '..', 'manual', 'plugin_examples')
|
|
for x in glob.glob(self.j(path, '*')):
|
|
self.build_plugin_example(x)
|
|
|
|
srcdir = self.j(gettempdir(), 'user-manual-build', 'en', 'html') + '/'
|
|
check_call(
|
|
' '.join(
|
|
['rsync', '-zz' '-rl', '--info=progress2', srcdir, 'main:/srv/manual/']
|
|
),
|
|
shell=True
|
|
)
|
|
check_call('ssh main chown -R http:http /srv/manual'.split())
|
|
|
|
|
|
# }}}
|
|
|
|
|
|
class UploadDemo(Command): # {{{
|
|
|
|
description = 'Rebuild and upload various demos'
|
|
|
|
def run(self, opts):
|
|
check_call(
|
|
'''ebook-convert %s/demo.html /tmp/html2lrf.lrf '''
|
|
'''--title='Demonstration of html2lrf' --authors='Kovid Goyal' '''
|
|
'''--header '''
|
|
'''--serif-family "/usr/share/fonts/corefonts, Times New Roman" '''
|
|
'''--mono-family "/usr/share/fonts/corefonts, Andale Mono" '''
|
|
'''''' % self.j(self.SRC, HTML2LRF),
|
|
shell=True
|
|
)
|
|
|
|
lrf = self.j(self.SRC, 'calibre', 'ebooks', 'lrf', 'html', 'demo')
|
|
check_call(
|
|
'cd %s && zip -j /tmp/html-demo.zip * /tmp/html2lrf.lrf' % lrf,
|
|
shell=True
|
|
)
|
|
|
|
check_call('scp /tmp/html-demo.zip main:%s/' % (DOWNLOADS, ), shell=True)
|
|
|
|
|
|
# }}}
|
|
|
|
|
|
class UploadToServer(Command): # {{{
|
|
|
|
description = 'Upload miscellaneous data to calibre server'
|
|
|
|
def run(self, opts):
|
|
check_call('scp translations/website/locales.zip main:/srv/main/'.split())
|
|
check_call('scp translations/changelog/locales.zip main:/srv/main/changelog-locales.zip'.split())
|
|
check_call('ssh main /apps/static/generate.py'.split())
|
|
src_file = glob.glob('dist/calibre-*.tar.xz')[0]
|
|
upload_signatures()
|
|
check_call(['git', 'push'])
|
|
check_call([
|
|
os.environ['PENV'] + '/gpg-as-kovid', '--armor', '--yes',
|
|
'--detach-sign', src_file
|
|
])
|
|
check_call(['scp', src_file + '.asc', 'code:/srv/code/signatures/'])
|
|
check_call('ssh code /usr/local/bin/update-calibre-code.py'.split())
|
|
check_call(
|
|
('ssh code /apps/update-calibre-version.py ' + __version__).split()
|
|
)
|
|
check_call((
|
|
'ssh main /usr/local/bin/update-calibre-version.py %s && /usr/local/bin/update-calibre-code.py && /apps/static/generate.py'
|
|
% __version__
|
|
).split())
|
|
|
|
|
|
# }}}
|