Allow the server to serve up MathJax

.gitignore

@@ -22,6 +22,7 @@ resources/editor-functions.json
 resources/user-manual-translation-stats.json
 resources/content-server/index-generated.html
 resources/content-server/locales.zip
+resources/content-server/mathjax.zip.xz
 resources/mozilla-ca-certs.pem
 icons/icns/*.iconset
 setup/installer/windows/calibre/build.log

setup/install.py

@@ -327,7 +327,7 @@ class Bootstrap(Command):
 
     description = 'Bootstrap a fresh checkout of calibre from git to a state where it can be installed. Requires various development tools/libraries/headers'
     TRANSLATIONS_REPO = 'https://github.com/kovidgoyal/calibre-translations.git'
-    sub_commands = 'cacerts build iso639 iso3166 translations gui resources'.split()
+    sub_commands = 'cacerts build iso639 iso3166 translations gui resources mathjax'.split()
 
     def pre_sub_commands(self, opts):
         tdir = self.j(self.d(self.SRC), 'translations')

setup/mathjax.py

@@ -7,44 +7,78 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os, shutil
+import os
+from urllib import urlretrieve
+from zipfile import ZipFile, ZIP_STORED, ZipInfo
+from hashlib import sha1
+
+from lzma.xz import compress
 
 from setup import Command
 
-def size_dir(d):
-    file_walker = (
-        os.path.join(root, f)
-        for root, _, files in os.walk(d)
-        for f in files
-    )
-    return sum(os.path.getsize(f) for f in file_walker)
-
 class MathJax(Command):
 
-    description = 'Rebuild the bundled copy of mathjax'
+    description = 'Create the MathJax bundle'
+    MATH_JAX_VERSION = '2.6.1'
+    MATH_JAX_URL = 'https://github.com/mathjax/MathJax/archive/%s.zip' % MATH_JAX_VERSION
+    FONT_FAMILY = 'STIX-Web'
 
-    MATHJAX_PATH = '../mathjax'
+    def add_options(self, parser):
+        parser.add_option('--path-to-mathjax', help='Path to the MathJax source code')
+
+    def download_mathjax_release(self):
+        from calibre.ptempfile import TemporaryDirectory
+        self.info('Downloading MathJax:', self.MATH_JAX_URL)
+        filename = urlretrieve(self.MATH_JAX_URL)[0]
+        with ZipFile(filename) as zf, TemporaryDirectory() as tdir:
+            zf.extractall(tdir)
+            for d in os.listdir(tdir):
+                q = os.path.join(tdir, d)
+                if os.path.isdir(q):
+                    return q
+
+    def patch_jax(self, raw):
+        self.info('Patching HTML-CSS jax web font loader...')
+        nraw = raw.replace(b'dir+"/"+fullname', b'AJAX.fileURL(dir+"/"+fullname)')
+        if nraw == raw:
+            raise SystemExit('Failed to path the HTML-CSS jax font loading code')
+        return nraw
+
+    def add_file(self, zf, path, name):
+        with open(path, 'rb') as f:
+            raw = f.read()
+        if name == 'jax/output/HTML-CSS/jax.js':
+            raw = self.patch_jax(raw)
+        self.h.update(raw)
+        zi = ZipInfo(name)
+        zi.external_attr = 0o444 << 16L
+        zf.writestr(zi, raw)
+
+    def add_tree(self, zf, base, prefix, ignore=lambda n:False):
+        from calibre import walk
+        for f in walk(base):
+            name = prefix + '/' + os.path.relpath(f, base).replace(os.sep, '/')
+            if not ignore(name):
+                self.add_file(zf, f, name)
+
+    def ignore_fonts(self, name):
+        return '/fonts/' in name and self.FONT_FAMILY not in name
 
     def run(self, opts):
-        base = self.a(self.j(self.d(self.SRC), self.MATHJAX_PATH))
+        self.h = sha1()
-        dest = self.j(self.RESOURCES, 'viewer', 'mathjax')
+        src = opts.path_to_mathjax or self.download_mathjax_release()
-        if os.path.exists(dest):
+        self.info('Compressing MathJax...')
-            shutil.rmtree(dest)
+        from calibre.ptempfile import SpooledTemporaryFile
-        os.mkdir(dest)
+        t = SpooledTemporaryFile()
-        up = self.j(base, 'unpacked')
+        with ZipFile(t, 'w', ZIP_STORED) as zf:
-        for x in os.listdir(up):
+            self.add_tree(zf, self.j(src, 'fonts', 'HTML-CSS', self.FONT_FAMILY, 'woff'), 'fonts/HTML-CSS/STIX-Web/woff')
-            if x == 'config': continue
+            self.add_tree(zf, self.j(src, 'unpacked', 'extensions'), 'extensions')
-            if os.path.isdir(self.j(up, x)):
+            self.add_tree(zf, self.j(src, 'unpacked', 'jax', 'element'), 'jax/element')
-                shutil.copytree(self.j(up, x), self.j(dest, x))
+            self.add_tree(zf, self.j(src, 'unpacked', 'jax', 'input'), 'jax/input')
-            else:
+            self.add_tree(zf, self.j(src, 'unpacked', 'jax', 'output', 'HTML-CSS'), 'jax/output/HTML-CSS', ignore=self.ignore_fonts)
-                shutil.copy(self.j(up, x), dest)
+            self.add_file(zf, self.j(src, 'unpacked', 'MathJax.js'), 'MathJax.js')
-
-        op = self.j(dest, 'jax', 'output')
-        for x in os.listdir(op):
-            if x != 'SVG':
-                shutil.rmtree(self.j(op, x))
-
-        shutil.rmtree(self.j(dest, 'extensions', 'HTML-CSS'))
-
-        print ('MathJax bundle updated. Size: %g MB'%(size_dir(dest)/(1024**2)))
 
+            zf.comment = self.h.hexdigest()
+        t.seek(0)
+        with open(self.j(self.RESOURCES, 'content-server', 'mathjax.zip.xz'), 'wb') as f:
+            compress(t, f, level=9)

src/calibre/srv/books.py

@@ -6,10 +6,12 @@ from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 from hashlib import sha1
 from functools import partial
-from threading import RLock
+from threading import RLock, Lock
 from cPickle import dumps
+from zipfile import ZipFile
 import errno, os, tempfile, shutil, time, json as jsonlib
 
+from lzma.xz import decompress
 from calibre.constants import cache_dir, iswindows
 from calibre.customize.ui import plugin_for_input_format
 from calibre.srv.metadata import book_as_json
@@ -161,3 +163,40 @@ def book_file(ctx, rd, book_id, fmt, size, mtime, name):
         if e.errno != errno.ENOENT:
             raise
         raise HTTPNotFound('No book file with hash: %s and name: %s' % (bhash, name))
+
+mathjax_lock = Lock()
+mathjax_manifest = None
+
+def get_mathjax_manifest(tdir=None):
+    global mathjax_manifest
+    with mathjax_lock:
+        if mathjax_manifest is None:
+            mathjax_manifest = {}
+            f = decompress(P('content-server/mathjax.zip.xz', data=True, allow_user_override=False))
+            f.seek(0)
+            tdir = os.path.join(tdir, 'mathjax')
+            os.mkdir(tdir)
+            zf = ZipFile(f)
+            zf.extractall(tdir)
+            mathjax_manifest['etag'] = type('')(zf.comment)
+            mathjax_manifest['files'] = {type('')(zi.filename):zi.file_size for zi in zf.infolist()}
+            zf.close(), f.close()
+        return mathjax_manifest
+
+def manifest_as_json():
+    ans = jsonlib.dumps(get_mathjax_manifest(), ensure_ascii=False)
+    if not isinstance(ans, bytes):
+        ans = ans.encode('utf-8')
+    return ans
+
+@endpoint('/mathjax/{+which=""}')
+def mathjax(ctx, rd, which):
+    manifest = get_mathjax_manifest(rd.tdir)
+    if not which:
+        return rd.etagged_dynamic_response(manifest['etag'], manifest_as_json, content_type='application/json; charset=UTF-8')
+    if which not in manifest['files']:
+        raise HTTPNotFound('No MathJax file named: %s' % which)
+    path = os.path.abspath(os.path.join(rd.tdir, 'mathjax', which))
+    if not path.startswith(rd.tdir):
+        raise HTTPNotFound('No MathJax file named: %s' % which)
+    return rd.filesystem_file_with_constant_etag(lopen(path, 'rb'), manifest['etag'])

src/calibre/srv/http_response.py

@@ -228,6 +228,9 @@ class RequestData(object):  # {{{
         tuple(map(lambda x:etag.update(string(x)), etag_parts))
         return ETaggedFile(output, etag.hexdigest())
 
+    def filesystem_file_with_constant_etag(self, output, etag_as_hexencoded_string):
+        return ETaggedFile(output, etag_as_hexencoded_string)
+
     def etagged_dynamic_response(self, etag, func, content_type='text/html; charset=UTF-8'):
         ' A response that is generated only if the etag does not match '
         ct = self.outheaders.get('Content-Type')