Do not use a compressed archive for mathjax

Improves performance for local clients such as PDF output and the
viewer. Since we have now removed the old unbundled mathjax, the file
count in the resources directory does not go up too much.
This commit is contained in:
Kovid Goyal 2019-01-04 21:53:06 +05:30
parent a15857ba14
commit f78a7dad58
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
7 changed files with 39 additions and 63 deletions

3
.gitignore vendored
View File

@ -28,8 +28,7 @@ resources/viewer.html
resources/content-server/index-generated.html resources/content-server/index-generated.html
resources/content-server/calibre.appcache resources/content-server/calibre.appcache
resources/content-server/locales.zip resources/content-server/locales.zip
resources/content-server/mathjax.zip.xz resources/mathjax
resources/content-server/mathjax.version
resources/mozilla-ca-certs.pem resources/mozilla-ca-certs.pem
resources/user-agent-data.json resources/user-agent-data.json
icons/icns/*.iconset icons/icns/*.iconset

View File

@ -16,7 +16,7 @@ let g:syntastic_python_flake8_exec = 'flake8-python2'
let g:syntastic_python_flake8_args = '--filename='. shellescape('*.py,*.recipe') let g:syntastic_python_flake8_args = '--filename='. shellescape('*.py,*.recipe')
let g:python_version_2 = 1 let g:python_version_2 = 1
set wildignore+=resources/viewer/mathjax/* set wildignore+=resources/mathjax/*
set wildignore+=resources/rapydscript/lib/* set wildignore+=resources/rapydscript/lib/*
set wildignore+=build/* set wildignore+=build/*
set wildignore+=dist/* set wildignore+=dist/*

View File

@ -7,7 +7,7 @@ __license__ = 'GPL v3'
__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>' __copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
__docformat__ = 'restructuredtext en' __docformat__ = 'restructuredtext en'
import os, shutil import os, shutil, json
from io import BytesIO from io import BytesIO
from zipfile import ZipFile, ZIP_STORED, ZipInfo from zipfile import ZipFile, ZIP_STORED, ZipInfo
from hashlib import sha1 from hashlib import sha1
@ -36,44 +36,42 @@ class MathJax(Command):
zf.extractall(tdir) zf.extractall(tdir)
return os.path.join(tdir, 'MathJax-' + self.MATH_JAX_VERSION) return os.path.join(tdir, 'MathJax-' + self.MATH_JAX_VERSION)
def add_file(self, zf, path, name): def add_file(self, path, name):
with open(path, 'rb') as f: with open(path, 'rb') as f:
raw = f.read() raw = f.read()
self.h.update(raw) self.h.update(raw)
zi = ZipInfo(name) self.mathjax_files[name] = len(raw)
zi.external_attr = 0o444 << 16 dest = self.j(self.mathjax_dir, *name.split('/'))
zf.writestr(zi, raw) base = os.path.dirname(dest)
if not os.path.exists(base):
os.makedirs(base)
with open(dest, 'wb') as f:
f.write(raw)
def add_tree(self, zf, base, prefix, ignore=lambda n:False): def add_tree(self, base, prefix):
for dirpath, dirnames, filenames in os.walk(base): for dirpath, dirnames, filenames in os.walk(base):
for fname in filenames: for fname in filenames:
f = os.path.join(dirpath, fname) f = os.path.join(dirpath, fname)
name = prefix + '/' + os.path.relpath(f, base).replace(os.sep, '/') name = prefix + '/' + os.path.relpath(f, base).replace(os.sep, '/')
if not ignore(name): self.add_file(f, name)
self.add_file(zf, f, name)
def ignore_fonts(self, name):
return '/fonts/' in name and self.FONT_FAMILY not in name
def run(self, opts): def run(self, opts):
from lzma.xz import compress
self.h = sha1() self.h = sha1()
self.mathjax_dir = self.j(self.RESOURCES, 'mathjax')
self.mathjax_files = {}
if os.path.exists(self.mathjax_dir):
shutil.rmtree(self.mathjax_dir)
os.mkdir(self.mathjax_dir)
tdir = mkdtemp('calibre-mathjax-build') tdir = mkdtemp('calibre-mathjax-build')
try: try:
src = opts.path_to_mathjax or self.download_mathjax_release(tdir, opts.mathjax_url) src = opts.path_to_mathjax or self.download_mathjax_release(tdir, opts.mathjax_url)
self.info('Compressing MathJax...') self.info('Adding MathJax...')
t = SpooledTemporaryFile() self.add_file(self.j(src, 'unpacked', 'MathJax.js'), 'MathJax.js')
with ZipFile(t, 'w', ZIP_STORED) as zf: self.add_tree(self.j(src, 'fonts', 'HTML-CSS', self.FONT_FAMILY, 'woff'), 'fonts/HTML-CSS/%s/woff' % self.FONT_FAMILY)
self.add_file(zf, self.j(src, 'unpacked', 'MathJax.js'), 'MathJax.js')
self.add_tree(zf, self.j(src, 'fonts', 'HTML-CSS', self.FONT_FAMILY, 'woff'), 'fonts/HTML-CSS/%s/woff' % self.FONT_FAMILY)
for d in 'extensions jax/element jax/input jax/output/CommonHTML'.split(): for d in 'extensions jax/element jax/input jax/output/CommonHTML'.split():
self.add_tree(zf, self.j(src, 'unpacked', *d.split('/')), d) self.add_tree(self.j(src, 'unpacked', *d.split('/')), d)
etag = self.h.hexdigest()
zf.comment = self.h.hexdigest() with open(self.j(self.RESOURCES, 'mathjax', 'manifest.json'), 'wb') as f:
t.seek(0) f.write(json.dumps({'etag': etag, 'files': self.mathjax_files}, indent=2).encode('utf-8'))
with open(self.j(self.RESOURCES, 'content-server', 'mathjax.zip.xz'), 'wb') as f:
compress(t, f, level=4 if is_ci else 9)
with open(self.j(self.RESOURCES, 'content-server', 'mathjax.version'), 'wb') as f:
f.write(zf.comment)
finally: finally:
shutil.rmtree(tdir) shutil.rmtree(tdir)

View File

@ -287,7 +287,7 @@ class RapydScript(Command): # {{{
class Resources(Command): # {{{ class Resources(Command): # {{{
description = 'Compile various needed calibre resources' description = 'Compile various needed calibre resources'
sub_commands = ['kakasi', 'coffee', 'rapydscript'] sub_commands = ['kakasi', 'coffee', 'rapydscript', 'mathjax']
def run(self, opts): def run(self, opts):
from calibre.utils.serialize import msgpack_dumps from calibre.utils.serialize import msgpack_dumps

View File

@ -150,7 +150,7 @@ class PDFWriter(QObject):
QObject.__init__(self) QObject.__init__(self)
self.logger = self.log = log self.logger = self.log = log
self.mathjax_tdir = None self.mathjax_dir = P('mathjax', allow_user_override=False)
current_log(log) current_log(log)
self.opts = opts self.opts = opts
self.cover_data = cover_data self.cover_data = cover_data
@ -310,12 +310,7 @@ class PDFWriter(QObject):
def load_mathjax(self): def load_mathjax(self):
evaljs = self.view.page().mainFrame().evaluateJavaScript evaljs = self.view.page().mainFrame().evaluateJavaScript
if self.mathjax_tdir is None: mjpath = self.mathjax_dir.replace(os.sep, '/')
self.mathjax_tdir = PersistentTemporaryDirectory('jax')
from calibre.srv.books import get_mathjax_manifest
get_mathjax_manifest(self.mathjax_tdir)
mjpath = os.path.join(self.mathjax_tdir, 'mathjax').replace(os.sep, '/')
if iswindows: if iswindows:
mjpath = u'/' + mjpath mjpath = u'/' + mjpath
if bool(evaljs(''' if bool(evaljs('''

View File

@ -107,7 +107,7 @@ class NetworkAccessManager(QNetworkAccessManager):
self.mathjax_base = '%s://%s/%s/' % (FAKE_PROTOCOL, FAKE_HOST, self.mathjax_prefix) self.mathjax_base = '%s://%s/%s/' % (FAKE_PROTOCOL, FAKE_HOST, self.mathjax_prefix)
self.root = self.orig_root = os.path.dirname(P('viewer/blank.html', allow_user_override=False)) self.root = self.orig_root = os.path.dirname(P('viewer/blank.html', allow_user_override=False))
self.mime_map, self.single_pages, self.codec_map = {}, set(), {} self.mime_map, self.single_pages, self.codec_map = {}, set(), {}
self.mathjax_tdir = None self.mathjax_dir = P('mathjax', allow_user_override=False)
def set_book_data(self, root, spine): def set_book_data(self, root, spine):
self.orig_root = root self.orig_root = root
@ -168,11 +168,7 @@ class NetworkAccessManager(QNetworkAccessManager):
if operation == QNetworkAccessManager.GetOperation and qurl.host() == FAKE_HOST: if operation == QNetworkAccessManager.GetOperation and qurl.host() == FAKE_HOST:
name = qurl.path()[1:] name = qurl.path()[1:]
if name.startswith(self.mathjax_prefix): if name.startswith(self.mathjax_prefix):
if self.mathjax_tdir is None: base = normpath(self.mathjax_dir)
self.mathjax_tdir = PersistentTemporaryDirectory('ev-jax')
from calibre.srv.books import get_mathjax_manifest
get_mathjax_manifest(self.mathjax_tdir)
base = normpath(os.path.join(self.mathjax_tdir, 'mathjax'))
path = normpath(os.path.join(base, name.partition('/')[2])) path = normpath(os.path.join(base, name.partition('/')[2]))
else: else:
base = self.root base = self.root

View File

@ -227,33 +227,21 @@ mathjax_lock = Lock()
mathjax_manifest = None mathjax_manifest = None
def get_mathjax_manifest(tdir=None): def manifest_as_json():
return P('mathjax/manifest.json', data=True, allow_user_override=False)
def get_mathjax_manifest():
global mathjax_manifest global mathjax_manifest
with mathjax_lock: with mathjax_lock:
if mathjax_manifest is None: if mathjax_manifest is None:
mathjax_manifest = {} mathjax_manifest = jsonlib.loads(manifest_as_json())
f = decompress(P('content-server/mathjax.zip.xz', data=True, allow_user_override=False))
f.seek(0)
tdir = os.path.join(tdir, 'mathjax')
os.mkdir(tdir)
zf = ZipFile(f)
zf.extractall(tdir)
mathjax_manifest['etag'] = type('')(zf.comment)
mathjax_manifest['files'] = {type('')(zi.filename):zi.file_size for zi in zf.infolist()}
zf.close(), f.close()
return mathjax_manifest return mathjax_manifest
def manifest_as_json():
ans = jsonlib.dumps(get_mathjax_manifest(), ensure_ascii=False)
if not isinstance(ans, bytes):
ans = ans.encode('utf-8')
return ans
@endpoint('/mathjax/{+which=""}', auth_required=False) @endpoint('/mathjax/{+which=""}', auth_required=False)
def mathjax(ctx, rd, which): def mathjax(ctx, rd, which):
manifest = get_mathjax_manifest(rd.tdir) manifest = get_mathjax_manifest()
if not which: if not which:
return rd.etagged_dynamic_response(manifest['etag'], manifest_as_json, content_type='application/json; charset=UTF-8') return rd.etagged_dynamic_response(manifest['etag'], manifest_as_json, content_type='application/json; charset=UTF-8')
if which not in manifest['files']: if which not in manifest['files']: