Do not use a compressed archive for mathjax

Improves performance for local clients such as PDF output and the
viewer. Since we have now removed the old unbundled mathjax, the file
count in the resources directory does not go up too much.

.gitignore

@@ -28,8 +28,7 @@ resources/viewer.html
 resources/content-server/index-generated.html
 resources/content-server/calibre.appcache
 resources/content-server/locales.zip
-resources/content-server/mathjax.zip.xz
+resources/mathjax
-resources/content-server/mathjax.version
 resources/mozilla-ca-certs.pem
 resources/user-agent-data.json
 icons/icns/*.iconset

session.vim

@@ -16,7 +16,7 @@ let g:syntastic_python_flake8_exec = 'flake8-python2'
 let g:syntastic_python_flake8_args = '--filename='. shellescape('*.py,*.recipe')
 let g:python_version_2 = 1
 
-set wildignore+=resources/viewer/mathjax/*
+set wildignore+=resources/mathjax/*
 set wildignore+=resources/rapydscript/lib/*
 set wildignore+=build/*
 set wildignore+=dist/*

setup/mathjax.py

@@ -7,7 +7,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os, shutil
+import os, shutil, json
 from io import BytesIO
 from zipfile import ZipFile, ZIP_STORED, ZipInfo
 from hashlib import sha1
@@ -36,44 +36,42 @@ class MathJax(Command):
             zf.extractall(tdir)
             return os.path.join(tdir, 'MathJax-' + self.MATH_JAX_VERSION)
 
-    def add_file(self, zf, path, name):
+    def add_file(self, path, name):
         with open(path, 'rb') as f:
             raw = f.read()
         self.h.update(raw)
-        zi = ZipInfo(name)
+        self.mathjax_files[name] = len(raw)
-        zi.external_attr = 0o444 << 16
+        dest = self.j(self.mathjax_dir, *name.split('/'))
-        zf.writestr(zi, raw)
+        base = os.path.dirname(dest)
+        if not os.path.exists(base):
+            os.makedirs(base)
+        with open(dest, 'wb') as f:
+            f.write(raw)
 
-    def add_tree(self, zf, base, prefix, ignore=lambda n:False):
+    def add_tree(self, base, prefix):
         for dirpath, dirnames, filenames in os.walk(base):
             for fname in filenames:
                 f = os.path.join(dirpath, fname)
                 name = prefix + '/' + os.path.relpath(f, base).replace(os.sep, '/')
-                if not ignore(name):
+                self.add_file(f, name)
-                    self.add_file(zf, f, name)
-
-    def ignore_fonts(self, name):
-        return '/fonts/' in name and self.FONT_FAMILY not in name
 
     def run(self, opts):
-        from lzma.xz import compress
         self.h = sha1()
+        self.mathjax_dir = self.j(self.RESOURCES, 'mathjax')
+        self.mathjax_files = {}
+        if os.path.exists(self.mathjax_dir):
+            shutil.rmtree(self.mathjax_dir)
+        os.mkdir(self.mathjax_dir)
         tdir = mkdtemp('calibre-mathjax-build')
         try:
             src = opts.path_to_mathjax or self.download_mathjax_release(tdir, opts.mathjax_url)
-            self.info('Compressing MathJax...')
+            self.info('Adding MathJax...')
-            t = SpooledTemporaryFile()
+            self.add_file(self.j(src, 'unpacked', 'MathJax.js'), 'MathJax.js')
-            with ZipFile(t, 'w', ZIP_STORED) as zf:
+            self.add_tree(self.j(src, 'fonts', 'HTML-CSS', self.FONT_FAMILY, 'woff'), 'fonts/HTML-CSS/%s/woff' % self.FONT_FAMILY)
-                self.add_file(zf, self.j(src, 'unpacked', 'MathJax.js'), 'MathJax.js')
+            for d in 'extensions jax/element jax/input jax/output/CommonHTML'.split():
-                self.add_tree(zf, self.j(src, 'fonts', 'HTML-CSS', self.FONT_FAMILY, 'woff'), 'fonts/HTML-CSS/%s/woff' % self.FONT_FAMILY)
+                self.add_tree(self.j(src, 'unpacked', *d.split('/')), d)
-                for d in 'extensions jax/element jax/input jax/output/CommonHTML'.split():
+            etag = self.h.hexdigest()
-                    self.add_tree(zf, self.j(src, 'unpacked', *d.split('/')), d)
+            with open(self.j(self.RESOURCES, 'mathjax', 'manifest.json'), 'wb') as f:
-
+                f.write(json.dumps({'etag': etag, 'files': self.mathjax_files}, indent=2).encode('utf-8'))
-                zf.comment = self.h.hexdigest()
-            t.seek(0)
-            with open(self.j(self.RESOURCES, 'content-server', 'mathjax.zip.xz'), 'wb') as f:
-                compress(t, f, level=4 if is_ci else 9)
-            with open(self.j(self.RESOURCES, 'content-server', 'mathjax.version'), 'wb') as f:
-                f.write(zf.comment)
         finally:
             shutil.rmtree(tdir)

setup/resources.py

@@ -287,7 +287,7 @@ class RapydScript(Command):  # {{{
 class Resources(Command):  # {{{
 
     description = 'Compile various needed calibre resources'
-    sub_commands = ['kakasi', 'coffee', 'rapydscript']
+    sub_commands = ['kakasi', 'coffee', 'rapydscript', 'mathjax']
 
     def run(self, opts):
         from calibre.utils.serialize import msgpack_dumps

src/calibre/ebooks/pdf/render/from_html.py

@@ -150,7 +150,7 @@ class PDFWriter(QObject):
         QObject.__init__(self)
 
         self.logger = self.log = log
-        self.mathjax_tdir = None
+        self.mathjax_dir = P('mathjax', allow_user_override=False)
         current_log(log)
         self.opts = opts
         self.cover_data = cover_data
@@ -310,12 +310,7 @@ class PDFWriter(QObject):
 
     def load_mathjax(self):
         evaljs = self.view.page().mainFrame().evaluateJavaScript
-        if self.mathjax_tdir is None:
+        mjpath = self.mathjax_dir.replace(os.sep, '/')
-            self.mathjax_tdir = PersistentTemporaryDirectory('jax')
-            from calibre.srv.books import get_mathjax_manifest
-            get_mathjax_manifest(self.mathjax_tdir)
-
-        mjpath = os.path.join(self.mathjax_tdir, 'mathjax').replace(os.sep, '/')
         if iswindows:
             mjpath = u'/' + mjpath
         if bool(evaljs('''

src/calibre/gui2/viewer/fake_net.py

@@ -107,7 +107,7 @@ class NetworkAccessManager(QNetworkAccessManager):
         self.mathjax_base = '%s://%s/%s/' % (FAKE_PROTOCOL, FAKE_HOST, self.mathjax_prefix)
         self.root = self.orig_root = os.path.dirname(P('viewer/blank.html', allow_user_override=False))
         self.mime_map, self.single_pages, self.codec_map = {}, set(), {}
-        self.mathjax_tdir = None
+        self.mathjax_dir = P('mathjax', allow_user_override=False)
 
     def set_book_data(self, root, spine):
         self.orig_root = root
@@ -168,11 +168,7 @@ class NetworkAccessManager(QNetworkAccessManager):
         if operation == QNetworkAccessManager.GetOperation and qurl.host() == FAKE_HOST:
             name = qurl.path()[1:]
             if name.startswith(self.mathjax_prefix):
-                if self.mathjax_tdir is None:
+                base = normpath(self.mathjax_dir)
-                    self.mathjax_tdir = PersistentTemporaryDirectory('ev-jax')
-                    from calibre.srv.books import get_mathjax_manifest
-                    get_mathjax_manifest(self.mathjax_tdir)
-                base = normpath(os.path.join(self.mathjax_tdir, 'mathjax'))
                 path = normpath(os.path.join(base, name.partition('/')[2]))
             else:
                 base = self.root

src/calibre/srv/books.py

@@ -227,33 +227,21 @@ mathjax_lock = Lock()
 mathjax_manifest = None
 
 
-def get_mathjax_manifest(tdir=None):
+def manifest_as_json():
+    return P('mathjax/manifest.json', data=True, allow_user_override=False)
+
+
+def get_mathjax_manifest():
     global mathjax_manifest
     with mathjax_lock:
         if mathjax_manifest is None:
-            mathjax_manifest = {}
+            mathjax_manifest = jsonlib.loads(manifest_as_json())
-            f = decompress(P('content-server/mathjax.zip.xz', data=True, allow_user_override=False))
+    return mathjax_manifest
-            f.seek(0)
-            tdir = os.path.join(tdir, 'mathjax')
-            os.mkdir(tdir)
-            zf = ZipFile(f)
-            zf.extractall(tdir)
-            mathjax_manifest['etag'] = type('')(zf.comment)
-            mathjax_manifest['files'] = {type('')(zi.filename):zi.file_size for zi in zf.infolist()}
-            zf.close(), f.close()
-        return mathjax_manifest
-
-
-def manifest_as_json():
-    ans = jsonlib.dumps(get_mathjax_manifest(), ensure_ascii=False)
-    if not isinstance(ans, bytes):
-        ans = ans.encode('utf-8')
-    return ans
 
 
 @endpoint('/mathjax/{+which=""}', auth_required=False)
 def mathjax(ctx, rd, which):
-    manifest = get_mathjax_manifest(rd.tdir)
+    manifest = get_mathjax_manifest()
     if not which:
         return rd.etagged_dynamic_response(manifest['etag'], manifest_as_json, content_type='application/json; charset=UTF-8')
     if which not in manifest['files']: