diff --git a/setup/hyphenation.py b/setup/hyphenation.py index b61b7cb128..86b5ebd83d 100644 --- a/setup/hyphenation.py +++ b/setup/hyphenation.py @@ -10,11 +10,8 @@ import os import shutil import tarfile from io import BytesIO -from zipfile import ZipFile -from setup import Command, download_securely - -URL = 'https://github.com/LibreOffice/dictionaries/archive/master.zip' +from setup.revendor import ReVendor def locales_from_dicts(dicts): @@ -83,30 +80,19 @@ def compress_tar(buf, outf): compress(buf, outf) -class Hyphenation(Command): +class Hyphenation(ReVendor): description = 'Download the hyphenation dictionaries' - - def add_options(self, parser): - pass - # parser.add_option('--path-to-mathjax', help='Path to the MathJax source code') - - @property - def hyphenation_dir(self): - return self.j(self.RESOURCES, 'hyphenation') - - def clean(self): - if os.path.exists(self.hyphenation_dir): - shutil.rmtree(self.hyphenation_dir) + NAME = 'hyphenation' + TAR_NAME = 'hyphenation dictionaries' + VERSION = 'master' + DOWNLOAD_URL = 'https://github.com/LibreOffice/dictionaries/archive/%s.tar.gz' % VERSION def run(self, opts): self.clean() - os.makedirs(self.hyphenation_dir) - self.info('Downloading hyphenation dictionaries...') - with self.temp_dir() as src, ZipFile(BytesIO(download_securely(URL))) as zf, self.temp_dir() as output_dir: - zf.extractall(src) - if len(os.listdir(src)) == 1: - src = os.path.join(src, os.listdir(src)[0]) + os.makedirs(self.vendored_dir) + with self.temp_dir() as dl_src, self.temp_dir() as output_dir: + src = opts.path_to_hyphenation or self.download_vendor_release(dl_src, opts.hyphenation_url) process_dictionaries(src, output_dir) dics = sorted(x for x in os.listdir(output_dir) if x.endswith('.dic')) m = hashlib.sha1() @@ -123,8 +109,8 @@ class Hyphenation(Command): tinfo.uid = tinfo.gid = 1000 tinfo.uname = tinfo.gname = 'kovid' tf.addfile(tinfo, df) - with open(os.path.join(self.hyphenation_dir, 'dictionaries.tar.xz'), 'wb') as f: + with open(os.path.join(self.vendored_dir, 'dictionaries.tar.xz'), 'wb') as f: compress_tar(buf, f) - with open(os.path.join(self.hyphenation_dir, 'sha1sum'), 'w') as f: + with open(os.path.join(self.vendored_dir, 'sha1sum'), 'w') as f: f.write(hsh) - shutil.copy(os.path.join(output_dir, 'locales.json'), self.hyphenation_dir) + shutil.copy(os.path.join(output_dir, 'locales.json'), self.vendored_dir) diff --git a/setup/mathjax.py b/setup/mathjax.py index 999cb3cbea..15f927010b 100644 --- a/setup/mathjax.py +++ b/setup/mathjax.py @@ -6,74 +6,34 @@ __license__ = 'GPL v3' __copyright__ = '2012, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, shutil, json +import os, json from io import BytesIO -from zipfile import ZipFile from hashlib import sha1 -from tempfile import mkdtemp -from setup import Command, download_securely +from setup.revendor import ReVendor -class MathJax(Command): +class MathJax(ReVendor): description = 'Create the MathJax bundle' - MATH_JAX_VERSION = '2.7.6' - MATH_JAX_URL = 'https://github.com/mathjax/MathJax/archive/%s.zip' % MATH_JAX_VERSION + NAME = 'mathjax' + TAR_NAME = 'MathJax' + VERSION = '2.7.6' + DOWNLOAD_URL = 'https://github.com/mathjax/MathJax/archive/%s.tar.gz' % VERSION FONT_FAMILY = 'TeX' - def add_options(self, parser): - parser.add_option('--path-to-mathjax', help='Path to the MathJax source code') - parser.add_option('--mathjax-url', default=self.MATH_JAX_URL, help='URL to MathJax source archive in zip format') - parser.add_option('--system-mathjax', default=False, action='store_true', - help='Treat MathJax as system copy and symlink instead of copy') - - def download_mathjax_release(self, tdir, url): - self.info('Downloading MathJax:', url) - raw = download_securely(url) - with ZipFile(BytesIO(raw)) as zf: - zf.extractall(tdir) - return os.path.join(tdir, 'MathJax-' + self.MATH_JAX_VERSION) - - def add_file(self, path, name): - with open(path, 'rb') as f: - raw = f.read() + def add_file_pre(self, name, raw): self.h.update(raw) self.mathjax_files[name] = len(raw) - dest = self.j(self.mathjax_dir, *name.split('/')) - base = os.path.dirname(dest) - if not os.path.exists(base): - os.makedirs(base) - if self.use_symlinks: - os.symlink(path, dest) - else: - with open(dest, 'wb') as f: - f.write(raw) - - def add_tree(self, base, prefix, ignore=lambda n:False): - for dirpath, dirnames, filenames in os.walk(base): - for fname in filenames: - f = os.path.join(dirpath, fname) - name = prefix + '/' + os.path.relpath(f, base).replace(os.sep, '/') - if not ignore(name): - self.add_file(f, name) - - @property - def mathjax_dir(self): - return self.j(self.RESOURCES, 'mathjax') def already_present(self): - manifest = self.j(self.mathjax_dir, 'manifest.json') + manifest = self.j(self.vendored_dir, 'manifest.json') if os.path.exists(manifest): with open(manifest, 'rb') as f: - return json.load(f).get('version') == self.MATH_JAX_VERSION + return json.load(f).get('version') == self.VERSION return False - def clean(self): - if os.path.exists(self.mathjax_dir): - shutil.rmtree(self.mathjax_dir) - def run(self, opts): if not opts.system_mathjax and self.already_present(): self.info('MathJax already present in the resources directory, not downloading') @@ -82,10 +42,9 @@ class MathJax(Command): self.h = sha1() self.mathjax_files = {} self.clean() - os.mkdir(self.mathjax_dir) - tdir = mkdtemp('calibre-mathjax-build') - try: - src = opts.path_to_mathjax or self.download_mathjax_release(tdir, opts.mathjax_url) + os.mkdir(self.vendored_dir) + with self.temp_dir(suffix='-calibre-mathjax-build') as tdir: + src = opts.path_to_mathjax or self.download_vendor_release(tdir, opts.mathjax_url) self.info('Adding MathJax...') unpacked = 'unpacked' if self.e(self.j(src, 'unpacked')) else '' self.add_file(self.j(src, unpacked, 'MathJax.js'), 'MathJax.js') @@ -97,6 +56,4 @@ class MathJax(Command): self.add_tree(self.j(src, unpacked, *d.split('/')), d) etag = self.h.hexdigest() with open(self.j(self.RESOURCES, 'mathjax', 'manifest.json'), 'wb') as f: - f.write(json.dumps({'etag': etag, 'files': self.mathjax_files, 'version': self.MATH_JAX_VERSION}, indent=2).encode('utf-8')) - finally: - shutil.rmtree(tdir) + f.write(json.dumps({'etag': etag, 'files': self.mathjax_files, 'version': self.VERSION}, indent=2).encode('utf-8')) diff --git a/setup/revendor.py b/setup/revendor.py new file mode 100755 index 0000000000..4475439061 --- /dev/null +++ b/setup/revendor.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python2 +# vim:fileencoding=utf-8 +# License: GPL v3 Copyright: 2019, Eli Schwartz + +import os, shutil +from io import BytesIO +import tarfile + + +from setup import Command, download_securely + + +class ReVendor(Command): + + # NAME = TAR_NAME = VERSION = DOWNLOAD_URL = '' + + def add_options(self, parser): + parser.add_option('--path-to-%s' % self.NAME, help='Path to the extracted %s source' % self.TAR_NAME) + parser.add_option('--%s-url' % self.NAME, default=self.DOWNLOAD_URL, + help='URL to %s source archive in tar.gz format' % self.TAR_NAME) + parser.add_option('--system-%s' % self.NAME, default=False, action='store_true', + help='Treat %s as system copy and symlink instead of copy' % self.TAR_NAME) + + def download_vendor_release(self, tdir, url): + self.info('Downloading %s:' % self.TAR_NAME, url) + raw = download_securely(url) + with tarfile.open(fileobj=BytesIO(raw)) as tf: + tf.extractall(tdir) + if len(os.listdir(tdir)) == 1: + return self.j(tdir, os.listdir(tdir)[0]) + else: + return tdir + + def add_file_pre(self, name, raw): + pass + + def add_file(self, path, name): + with open(path, 'rb') as f: + raw = f.read() + self.add_file_pre(name, raw) + dest = self.j(self.vendored_dir, *name.split('/')) + base = os.path.dirname(dest) + if not os.path.exists(base): + os.makedirs(base) + if self.use_symlinks: + os.symlink(path, dest) + else: + with open(dest, 'wb') as f: + f.write(raw) + + def add_tree(self, base, prefix, ignore=lambda n:False): + for dirpath, dirnames, filenames in os.walk(base): + for fname in filenames: + f = os.path.join(dirpath, fname) + name = prefix + '/' + os.path.relpath(f, base).replace(os.sep, '/') + if not ignore(name): + self.add_file(f, name) + + @property + def vendored_dir(self): + return self.j(self.RESOURCES, self.NAME) + + def clean(self): + if os.path.exists(self.vendored_dir): + shutil.rmtree(self.vendored_dir)