From 6531811efa9bc1ee298c8d82fec30b0fbdfd44e2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 1 May 2016 16:50:33 +0530 Subject: [PATCH] Allow the server to serve up MathJax --- .gitignore | 1 + setup/install.py | 2 +- setup/mathjax.py | 98 +++++++++++++++++++++----------- src/calibre/srv/books.py | 41 ++++++++++++- src/calibre/srv/http_response.py | 3 + 5 files changed, 111 insertions(+), 34 deletions(-) diff --git a/.gitignore b/.gitignore index 563089287d..4f363562c9 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ resources/editor-functions.json resources/user-manual-translation-stats.json resources/content-server/index-generated.html resources/content-server/locales.zip +resources/content-server/mathjax.zip.xz resources/mozilla-ca-certs.pem icons/icns/*.iconset setup/installer/windows/calibre/build.log diff --git a/setup/install.py b/setup/install.py index 351d791485..6f197bcc76 100644 --- a/setup/install.py +++ b/setup/install.py @@ -327,7 +327,7 @@ class Bootstrap(Command): description = 'Bootstrap a fresh checkout of calibre from git to a state where it can be installed. Requires various development tools/libraries/headers' TRANSLATIONS_REPO = 'https://github.com/kovidgoyal/calibre-translations.git' - sub_commands = 'cacerts build iso639 iso3166 translations gui resources'.split() + sub_commands = 'cacerts build iso639 iso3166 translations gui resources mathjax'.split() def pre_sub_commands(self, opts): tdir = self.j(self.d(self.SRC), 'translations') diff --git a/setup/mathjax.py b/setup/mathjax.py index ca88612929..3c2350d169 100644 --- a/setup/mathjax.py +++ b/setup/mathjax.py @@ -7,44 +7,78 @@ __license__ = 'GPL v3' __copyright__ = '2012, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, shutil +import os +from urllib import urlretrieve +from zipfile import ZipFile, ZIP_STORED, ZipInfo +from hashlib import sha1 + +from lzma.xz import compress from setup import Command -def size_dir(d): - file_walker = ( - os.path.join(root, f) - for root, _, files in os.walk(d) - for f in files - ) - return sum(os.path.getsize(f) for f in file_walker) - class MathJax(Command): - description = 'Rebuild the bundled copy of mathjax' + description = 'Create the MathJax bundle' + MATH_JAX_VERSION = '2.6.1' + MATH_JAX_URL = 'https://github.com/mathjax/MathJax/archive/%s.zip' % MATH_JAX_VERSION + FONT_FAMILY = 'STIX-Web' - MATHJAX_PATH = '../mathjax' + def add_options(self, parser): + parser.add_option('--path-to-mathjax', help='Path to the MathJax source code') + + def download_mathjax_release(self): + from calibre.ptempfile import TemporaryDirectory + self.info('Downloading MathJax:', self.MATH_JAX_URL) + filename = urlretrieve(self.MATH_JAX_URL)[0] + with ZipFile(filename) as zf, TemporaryDirectory() as tdir: + zf.extractall(tdir) + for d in os.listdir(tdir): + q = os.path.join(tdir, d) + if os.path.isdir(q): + return q + + def patch_jax(self, raw): + self.info('Patching HTML-CSS jax web font loader...') + nraw = raw.replace(b'dir+"/"+fullname', b'AJAX.fileURL(dir+"/"+fullname)') + if nraw == raw: + raise SystemExit('Failed to path the HTML-CSS jax font loading code') + return nraw + + def add_file(self, zf, path, name): + with open(path, 'rb') as f: + raw = f.read() + if name == 'jax/output/HTML-CSS/jax.js': + raw = self.patch_jax(raw) + self.h.update(raw) + zi = ZipInfo(name) + zi.external_attr = 0o444 << 16L + zf.writestr(zi, raw) + + def add_tree(self, zf, base, prefix, ignore=lambda n:False): + from calibre import walk + for f in walk(base): + name = prefix + '/' + os.path.relpath(f, base).replace(os.sep, '/') + if not ignore(name): + self.add_file(zf, f, name) + + def ignore_fonts(self, name): + return '/fonts/' in name and self.FONT_FAMILY not in name def run(self, opts): - base = self.a(self.j(self.d(self.SRC), self.MATHJAX_PATH)) - dest = self.j(self.RESOURCES, 'viewer', 'mathjax') - if os.path.exists(dest): - shutil.rmtree(dest) - os.mkdir(dest) - up = self.j(base, 'unpacked') - for x in os.listdir(up): - if x == 'config': continue - if os.path.isdir(self.j(up, x)): - shutil.copytree(self.j(up, x), self.j(dest, x)) - else: - shutil.copy(self.j(up, x), dest) - - op = self.j(dest, 'jax', 'output') - for x in os.listdir(op): - if x != 'SVG': - shutil.rmtree(self.j(op, x)) - - shutil.rmtree(self.j(dest, 'extensions', 'HTML-CSS')) - - print ('MathJax bundle updated. Size: %g MB'%(size_dir(dest)/(1024**2))) + self.h = sha1() + src = opts.path_to_mathjax or self.download_mathjax_release() + self.info('Compressing MathJax...') + from calibre.ptempfile import SpooledTemporaryFile + t = SpooledTemporaryFile() + with ZipFile(t, 'w', ZIP_STORED) as zf: + self.add_tree(zf, self.j(src, 'fonts', 'HTML-CSS', self.FONT_FAMILY, 'woff'), 'fonts/HTML-CSS/STIX-Web/woff') + self.add_tree(zf, self.j(src, 'unpacked', 'extensions'), 'extensions') + self.add_tree(zf, self.j(src, 'unpacked', 'jax', 'element'), 'jax/element') + self.add_tree(zf, self.j(src, 'unpacked', 'jax', 'input'), 'jax/input') + self.add_tree(zf, self.j(src, 'unpacked', 'jax', 'output', 'HTML-CSS'), 'jax/output/HTML-CSS', ignore=self.ignore_fonts) + self.add_file(zf, self.j(src, 'unpacked', 'MathJax.js'), 'MathJax.js') + zf.comment = self.h.hexdigest() + t.seek(0) + with open(self.j(self.RESOURCES, 'content-server', 'mathjax.zip.xz'), 'wb') as f: + compress(t, f, level=9) diff --git a/src/calibre/srv/books.py b/src/calibre/srv/books.py index 02c87a7edc..8a3fe1df6b 100644 --- a/src/calibre/srv/books.py +++ b/src/calibre/srv/books.py @@ -6,10 +6,12 @@ from __future__ import (unicode_literals, division, absolute_import, print_function) from hashlib import sha1 from functools import partial -from threading import RLock +from threading import RLock, Lock from cPickle import dumps +from zipfile import ZipFile import errno, os, tempfile, shutil, time, json as jsonlib +from lzma.xz import decompress from calibre.constants import cache_dir, iswindows from calibre.customize.ui import plugin_for_input_format from calibre.srv.metadata import book_as_json @@ -161,3 +163,40 @@ def book_file(ctx, rd, book_id, fmt, size, mtime, name): if e.errno != errno.ENOENT: raise raise HTTPNotFound('No book file with hash: %s and name: %s' % (bhash, name)) + +mathjax_lock = Lock() +mathjax_manifest = None + +def get_mathjax_manifest(tdir=None): + global mathjax_manifest + with mathjax_lock: + if mathjax_manifest is None: + mathjax_manifest = {} + f = decompress(P('content-server/mathjax.zip.xz', data=True, allow_user_override=False)) + f.seek(0) + tdir = os.path.join(tdir, 'mathjax') + os.mkdir(tdir) + zf = ZipFile(f) + zf.extractall(tdir) + mathjax_manifest['etag'] = type('')(zf.comment) + mathjax_manifest['files'] = {type('')(zi.filename):zi.file_size for zi in zf.infolist()} + zf.close(), f.close() + return mathjax_manifest + +def manifest_as_json(): + ans = jsonlib.dumps(get_mathjax_manifest(), ensure_ascii=False) + if not isinstance(ans, bytes): + ans = ans.encode('utf-8') + return ans + +@endpoint('/mathjax/{+which=""}') +def mathjax(ctx, rd, which): + manifest = get_mathjax_manifest(rd.tdir) + if not which: + return rd.etagged_dynamic_response(manifest['etag'], manifest_as_json, content_type='application/json; charset=UTF-8') + if which not in manifest['files']: + raise HTTPNotFound('No MathJax file named: %s' % which) + path = os.path.abspath(os.path.join(rd.tdir, 'mathjax', which)) + if not path.startswith(rd.tdir): + raise HTTPNotFound('No MathJax file named: %s' % which) + return rd.filesystem_file_with_constant_etag(lopen(path, 'rb'), manifest['etag']) diff --git a/src/calibre/srv/http_response.py b/src/calibre/srv/http_response.py index 093c218ee7..dad2464260 100644 --- a/src/calibre/srv/http_response.py +++ b/src/calibre/srv/http_response.py @@ -228,6 +228,9 @@ class RequestData(object): # {{{ tuple(map(lambda x:etag.update(string(x)), etag_parts)) return ETaggedFile(output, etag.hexdigest()) + def filesystem_file_with_constant_etag(self, output, etag_as_hexencoded_string): + return ETaggedFile(output, etag_as_hexencoded_string) + def etagged_dynamic_response(self, etag, func, content_type='text/html; charset=UTF-8'): ' A response that is generated only if the etag does not match ' ct = self.outheaders.get('Content-Type')