Merge branch 'revendor' of https://github.com/eli-schwartz/calibre

setup/hyphenation.py

@@ -10,11 +10,8 @@ import os
 import shutil
 import tarfile
 from io import BytesIO
-from zipfile import ZipFile
 
-from setup import Command, download_securely
+from setup.revendor import ReVendor
-
-URL = 'https://github.com/LibreOffice/dictionaries/archive/master.zip'
 
 
 def locales_from_dicts(dicts):
@@ -83,30 +80,19 @@ def compress_tar(buf, outf):
         compress(buf, outf)
 
 
-class Hyphenation(Command):
+class Hyphenation(ReVendor):
 
     description = 'Download the hyphenation dictionaries'
-
+    NAME = 'hyphenation'
-    def add_options(self, parser):
+    TAR_NAME = 'hyphenation dictionaries'
-        pass
+    VERSION = 'master'
-        # parser.add_option('--path-to-mathjax', help='Path to the MathJax source code')
+    DOWNLOAD_URL = 'https://github.com/LibreOffice/dictionaries/archive/%s.tar.gz' % VERSION
-
-    @property
-    def hyphenation_dir(self):
-        return self.j(self.RESOURCES, 'hyphenation')
-
-    def clean(self):
-        if os.path.exists(self.hyphenation_dir):
-            shutil.rmtree(self.hyphenation_dir)
 
     def run(self, opts):
         self.clean()
-        os.makedirs(self.hyphenation_dir)
+        os.makedirs(self.vendored_dir)
-        self.info('Downloading hyphenation dictionaries...')
+        with self.temp_dir() as dl_src, self.temp_dir() as output_dir:
-        with self.temp_dir() as src, ZipFile(BytesIO(download_securely(URL))) as zf, self.temp_dir() as output_dir:
+            src = opts.path_to_hyphenation or self.download_vendor_release(dl_src, opts.hyphenation_url)
-            zf.extractall(src)
-            if len(os.listdir(src)) == 1:
-                src = os.path.join(src, os.listdir(src)[0])
             process_dictionaries(src, output_dir)
             dics = sorted(x for x in os.listdir(output_dir) if x.endswith('.dic'))
             m = hashlib.sha1()
@@ -123,8 +109,8 @@ class Hyphenation(Command):
                         tinfo.uid = tinfo.gid = 1000
                         tinfo.uname = tinfo.gname = 'kovid'
                         tf.addfile(tinfo, df)
-            with open(os.path.join(self.hyphenation_dir, 'dictionaries.tar.xz'), 'wb') as f:
+            with open(os.path.join(self.vendored_dir, 'dictionaries.tar.xz'), 'wb') as f:
                 compress_tar(buf, f)
-            with open(os.path.join(self.hyphenation_dir, 'sha1sum'), 'w') as f:
+            with open(os.path.join(self.vendored_dir, 'sha1sum'), 'w') as f:
                 f.write(hsh)
-            shutil.copy(os.path.join(output_dir, 'locales.json'), self.hyphenation_dir)
+            shutil.copy(os.path.join(output_dir, 'locales.json'), self.vendored_dir)

setup/mathjax.py

@@ -6,74 +6,34 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os, shutil, json
+import os, json
 from io import BytesIO
-from zipfile import ZipFile
 from hashlib import sha1
-from tempfile import mkdtemp
 
 
-from setup import Command, download_securely
+from setup.revendor import ReVendor
 
 
-class MathJax(Command):
+class MathJax(ReVendor):
 
     description = 'Create the MathJax bundle'
-    MATH_JAX_VERSION = '2.7.6'
+    NAME = 'mathjax'
-    MATH_JAX_URL = 'https://github.com/mathjax/MathJax/archive/%s.zip' % MATH_JAX_VERSION
+    TAR_NAME = 'MathJax'
+    VERSION = '2.7.6'
+    DOWNLOAD_URL = 'https://github.com/mathjax/MathJax/archive/%s.tar.gz' % VERSION
     FONT_FAMILY = 'TeX'
 
-    def add_options(self, parser):
+    def add_file_pre(self, name, raw):
-        parser.add_option('--path-to-mathjax', help='Path to the MathJax source code')
-        parser.add_option('--mathjax-url', default=self.MATH_JAX_URL, help='URL to MathJax source archive in zip format')
-        parser.add_option('--system-mathjax', default=False, action='store_true',
-                help='Treat MathJax as system copy and symlink instead of copy')
-
-    def download_mathjax_release(self, tdir, url):
-        self.info('Downloading MathJax:', url)
-        raw = download_securely(url)
-        with ZipFile(BytesIO(raw)) as zf:
-            zf.extractall(tdir)
-            return os.path.join(tdir, 'MathJax-' + self.MATH_JAX_VERSION)
-
-    def add_file(self, path, name):
-        with open(path, 'rb') as f:
-            raw = f.read()
         self.h.update(raw)
         self.mathjax_files[name] = len(raw)
-        dest = self.j(self.mathjax_dir, *name.split('/'))
-        base = os.path.dirname(dest)
-        if not os.path.exists(base):
-            os.makedirs(base)
-        if self.use_symlinks:
-            os.symlink(path, dest)
-        else:
-            with open(dest, 'wb') as f:
-                f.write(raw)
-
-    def add_tree(self, base, prefix, ignore=lambda n:False):
-        for dirpath, dirnames, filenames in os.walk(base):
-            for fname in filenames:
-                f = os.path.join(dirpath, fname)
-                name = prefix + '/' + os.path.relpath(f, base).replace(os.sep, '/')
-                if not ignore(name):
-                    self.add_file(f, name)
-
-    @property
-    def mathjax_dir(self):
-        return self.j(self.RESOURCES, 'mathjax')
 
     def already_present(self):
-        manifest = self.j(self.mathjax_dir, 'manifest.json')
+        manifest = self.j(self.vendored_dir, 'manifest.json')
         if os.path.exists(manifest):
             with open(manifest, 'rb') as f:
-                return json.load(f).get('version') == self.MATH_JAX_VERSION
+                return json.load(f).get('version') == self.VERSION
         return False
 
-    def clean(self):
-        if os.path.exists(self.mathjax_dir):
-            shutil.rmtree(self.mathjax_dir)
-
     def run(self, opts):
         if not opts.system_mathjax and self.already_present():
             self.info('MathJax already present in the resources directory, not downloading')
@@ -82,10 +42,9 @@ class MathJax(Command):
         self.h = sha1()
         self.mathjax_files = {}
         self.clean()
-        os.mkdir(self.mathjax_dir)
+        os.mkdir(self.vendored_dir)
-        tdir = mkdtemp('calibre-mathjax-build')
+        with self.temp_dir(suffix='-calibre-mathjax-build') as tdir:
-        try:
+            src = opts.path_to_mathjax or self.download_vendor_release(tdir, opts.mathjax_url)
-            src = opts.path_to_mathjax or self.download_mathjax_release(tdir, opts.mathjax_url)
             self.info('Adding MathJax...')
             unpacked = 'unpacked' if self.e(self.j(src, 'unpacked')) else ''
             self.add_file(self.j(src, unpacked, 'MathJax.js'), 'MathJax.js')
@@ -97,6 +56,4 @@ class MathJax(Command):
                 self.add_tree(self.j(src, unpacked, *d.split('/')), d)
             etag = self.h.hexdigest()
             with open(self.j(self.RESOURCES, 'mathjax', 'manifest.json'), 'wb') as f:
-                f.write(json.dumps({'etag': etag, 'files': self.mathjax_files, 'version': self.MATH_JAX_VERSION}, indent=2).encode('utf-8'))
+                f.write(json.dumps({'etag': etag, 'files': self.mathjax_files, 'version': self.VERSION}, indent=2).encode('utf-8'))
-        finally:
-            shutil.rmtree(tdir)

setup/revendor.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python2
+# vim:fileencoding=utf-8
+# License: GPL v3 Copyright: 2019, Eli Schwartz <eschwartz@archlinux.org>
+
+import os, shutil
+from io import BytesIO
+import tarfile
+
+
+from setup import Command, download_securely
+
+
+class ReVendor(Command):
+
+    # NAME = TAR_NAME = VERSION = DOWNLOAD_URL = ''
+
+    def add_options(self, parser):
+        parser.add_option('--path-to-%s' % self.NAME, help='Path to the extracted %s source' % self.TAR_NAME)
+        parser.add_option('--%s-url' % self.NAME, default=self.DOWNLOAD_URL,
+                help='URL to %s source archive in tar.gz format' % self.TAR_NAME)
+        parser.add_option('--system-%s' % self.NAME, default=False, action='store_true',
+                help='Treat %s as system copy and symlink instead of copy' % self.TAR_NAME)
+
+    def download_vendor_release(self, tdir, url):
+        self.info('Downloading %s:' % self.TAR_NAME, url)
+        raw = download_securely(url)
+        with tarfile.open(fileobj=BytesIO(raw)) as tf:
+            tf.extractall(tdir)
+            if len(os.listdir(tdir)) == 1:
+                return self.j(tdir, os.listdir(tdir)[0])
+            else:
+                return tdir
+
+    def add_file_pre(self, name, raw):
+        pass
+
+    def add_file(self, path, name):
+        with open(path, 'rb') as f:
+            raw = f.read()
+        self.add_file_pre(name, raw)
+        dest = self.j(self.vendored_dir, *name.split('/'))
+        base = os.path.dirname(dest)
+        if not os.path.exists(base):
+            os.makedirs(base)
+        if self.use_symlinks:
+            os.symlink(path, dest)
+        else:
+            with open(dest, 'wb') as f:
+                f.write(raw)
+
+    def add_tree(self, base, prefix, ignore=lambda n:False):
+        for dirpath, dirnames, filenames in os.walk(base):
+            for fname in filenames:
+                f = os.path.join(dirpath, fname)
+                name = prefix + '/' + os.path.relpath(f, base).replace(os.sep, '/')
+                if not ignore(name):
+                    self.add_file(f, name)
+
+    @property
+    def vendored_dir(self):
+        return self.j(self.RESOURCES, self.NAME)
+
+    def clean(self):
+        if os.path.exists(self.vendored_dir):
+            shutil.rmtree(self.vendored_dir)