From f78a7dad581aa2c29800598e371a42889d6e0eb7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 4 Jan 2019 21:53:06 +0530 Subject: [PATCH] Do not use a compressed archive for mathjax Improves performance for local clients such as PDF output and the viewer. Since we have now removed the old unbundled mathjax, the file count in the resources directory does not go up too much. --- .gitignore | 3 +- session.vim | 2 +- setup/mathjax.py | 50 +++++++++++----------- setup/resources.py | 2 +- src/calibre/ebooks/pdf/render/from_html.py | 9 +--- src/calibre/gui2/viewer/fake_net.py | 8 +--- src/calibre/srv/books.py | 28 ++++-------- 7 files changed, 39 insertions(+), 63 deletions(-) diff --git a/.gitignore b/.gitignore index 6122bbe6d7..ccbeab143b 100644 --- a/.gitignore +++ b/.gitignore @@ -28,8 +28,7 @@ resources/viewer.html resources/content-server/index-generated.html resources/content-server/calibre.appcache resources/content-server/locales.zip -resources/content-server/mathjax.zip.xz -resources/content-server/mathjax.version +resources/mathjax resources/mozilla-ca-certs.pem resources/user-agent-data.json icons/icns/*.iconset diff --git a/session.vim b/session.vim index 0251e760db..a1918c47e4 100644 --- a/session.vim +++ b/session.vim @@ -16,7 +16,7 @@ let g:syntastic_python_flake8_exec = 'flake8-python2' let g:syntastic_python_flake8_args = '--filename='. shellescape('*.py,*.recipe') let g:python_version_2 = 1 -set wildignore+=resources/viewer/mathjax/* +set wildignore+=resources/mathjax/* set wildignore+=resources/rapydscript/lib/* set wildignore+=build/* set wildignore+=dist/* diff --git a/setup/mathjax.py b/setup/mathjax.py index 72d7b53dff..b65bba6935 100644 --- a/setup/mathjax.py +++ b/setup/mathjax.py @@ -7,7 +7,7 @@ __license__ = 'GPL v3' __copyright__ = '2012, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, shutil +import os, shutil, json from io import BytesIO from zipfile import ZipFile, ZIP_STORED, ZipInfo from hashlib import sha1 @@ -36,44 +36,42 @@ class MathJax(Command): zf.extractall(tdir) return os.path.join(tdir, 'MathJax-' + self.MATH_JAX_VERSION) - def add_file(self, zf, path, name): + def add_file(self, path, name): with open(path, 'rb') as f: raw = f.read() self.h.update(raw) - zi = ZipInfo(name) - zi.external_attr = 0o444 << 16 - zf.writestr(zi, raw) + self.mathjax_files[name] = len(raw) + dest = self.j(self.mathjax_dir, *name.split('/')) + base = os.path.dirname(dest) + if not os.path.exists(base): + os.makedirs(base) + with open(dest, 'wb') as f: + f.write(raw) - def add_tree(self, zf, base, prefix, ignore=lambda n:False): + def add_tree(self, base, prefix): for dirpath, dirnames, filenames in os.walk(base): for fname in filenames: f = os.path.join(dirpath, fname) name = prefix + '/' + os.path.relpath(f, base).replace(os.sep, '/') - if not ignore(name): - self.add_file(zf, f, name) - - def ignore_fonts(self, name): - return '/fonts/' in name and self.FONT_FAMILY not in name + self.add_file(f, name) def run(self, opts): - from lzma.xz import compress self.h = sha1() + self.mathjax_dir = self.j(self.RESOURCES, 'mathjax') + self.mathjax_files = {} + if os.path.exists(self.mathjax_dir): + shutil.rmtree(self.mathjax_dir) + os.mkdir(self.mathjax_dir) tdir = mkdtemp('calibre-mathjax-build') try: src = opts.path_to_mathjax or self.download_mathjax_release(tdir, opts.mathjax_url) - self.info('Compressing MathJax...') - t = SpooledTemporaryFile() - with ZipFile(t, 'w', ZIP_STORED) as zf: - self.add_file(zf, self.j(src, 'unpacked', 'MathJax.js'), 'MathJax.js') - self.add_tree(zf, self.j(src, 'fonts', 'HTML-CSS', self.FONT_FAMILY, 'woff'), 'fonts/HTML-CSS/%s/woff' % self.FONT_FAMILY) - for d in 'extensions jax/element jax/input jax/output/CommonHTML'.split(): - self.add_tree(zf, self.j(src, 'unpacked', *d.split('/')), d) - - zf.comment = self.h.hexdigest() - t.seek(0) - with open(self.j(self.RESOURCES, 'content-server', 'mathjax.zip.xz'), 'wb') as f: - compress(t, f, level=4 if is_ci else 9) - with open(self.j(self.RESOURCES, 'content-server', 'mathjax.version'), 'wb') as f: - f.write(zf.comment) + self.info('Adding MathJax...') + self.add_file(self.j(src, 'unpacked', 'MathJax.js'), 'MathJax.js') + self.add_tree(self.j(src, 'fonts', 'HTML-CSS', self.FONT_FAMILY, 'woff'), 'fonts/HTML-CSS/%s/woff' % self.FONT_FAMILY) + for d in 'extensions jax/element jax/input jax/output/CommonHTML'.split(): + self.add_tree(self.j(src, 'unpacked', *d.split('/')), d) + etag = self.h.hexdigest() + with open(self.j(self.RESOURCES, 'mathjax', 'manifest.json'), 'wb') as f: + f.write(json.dumps({'etag': etag, 'files': self.mathjax_files}, indent=2).encode('utf-8')) finally: shutil.rmtree(tdir) diff --git a/setup/resources.py b/setup/resources.py index 2dd246d8af..8fea7c2512 100644 --- a/setup/resources.py +++ b/setup/resources.py @@ -287,7 +287,7 @@ class RapydScript(Command): # {{{ class Resources(Command): # {{{ description = 'Compile various needed calibre resources' - sub_commands = ['kakasi', 'coffee', 'rapydscript'] + sub_commands = ['kakasi', 'coffee', 'rapydscript', 'mathjax'] def run(self, opts): from calibre.utils.serialize import msgpack_dumps diff --git a/src/calibre/ebooks/pdf/render/from_html.py b/src/calibre/ebooks/pdf/render/from_html.py index 94a9907b2a..962917966c 100644 --- a/src/calibre/ebooks/pdf/render/from_html.py +++ b/src/calibre/ebooks/pdf/render/from_html.py @@ -150,7 +150,7 @@ class PDFWriter(QObject): QObject.__init__(self) self.logger = self.log = log - self.mathjax_tdir = None + self.mathjax_dir = P('mathjax', allow_user_override=False) current_log(log) self.opts = opts self.cover_data = cover_data @@ -310,12 +310,7 @@ class PDFWriter(QObject): def load_mathjax(self): evaljs = self.view.page().mainFrame().evaluateJavaScript - if self.mathjax_tdir is None: - self.mathjax_tdir = PersistentTemporaryDirectory('jax') - from calibre.srv.books import get_mathjax_manifest - get_mathjax_manifest(self.mathjax_tdir) - - mjpath = os.path.join(self.mathjax_tdir, 'mathjax').replace(os.sep, '/') + mjpath = self.mathjax_dir.replace(os.sep, '/') if iswindows: mjpath = u'/' + mjpath if bool(evaljs(''' diff --git a/src/calibre/gui2/viewer/fake_net.py b/src/calibre/gui2/viewer/fake_net.py index c8e1a7f5db..12c7f28752 100644 --- a/src/calibre/gui2/viewer/fake_net.py +++ b/src/calibre/gui2/viewer/fake_net.py @@ -107,7 +107,7 @@ class NetworkAccessManager(QNetworkAccessManager): self.mathjax_base = '%s://%s/%s/' % (FAKE_PROTOCOL, FAKE_HOST, self.mathjax_prefix) self.root = self.orig_root = os.path.dirname(P('viewer/blank.html', allow_user_override=False)) self.mime_map, self.single_pages, self.codec_map = {}, set(), {} - self.mathjax_tdir = None + self.mathjax_dir = P('mathjax', allow_user_override=False) def set_book_data(self, root, spine): self.orig_root = root @@ -168,11 +168,7 @@ class NetworkAccessManager(QNetworkAccessManager): if operation == QNetworkAccessManager.GetOperation and qurl.host() == FAKE_HOST: name = qurl.path()[1:] if name.startswith(self.mathjax_prefix): - if self.mathjax_tdir is None: - self.mathjax_tdir = PersistentTemporaryDirectory('ev-jax') - from calibre.srv.books import get_mathjax_manifest - get_mathjax_manifest(self.mathjax_tdir) - base = normpath(os.path.join(self.mathjax_tdir, 'mathjax')) + base = normpath(self.mathjax_dir) path = normpath(os.path.join(base, name.partition('/')[2])) else: base = self.root diff --git a/src/calibre/srv/books.py b/src/calibre/srv/books.py index 565b42308b..641cd80305 100644 --- a/src/calibre/srv/books.py +++ b/src/calibre/srv/books.py @@ -227,33 +227,21 @@ mathjax_lock = Lock() mathjax_manifest = None -def get_mathjax_manifest(tdir=None): +def manifest_as_json(): + return P('mathjax/manifest.json', data=True, allow_user_override=False) + + +def get_mathjax_manifest(): global mathjax_manifest with mathjax_lock: if mathjax_manifest is None: - mathjax_manifest = {} - f = decompress(P('content-server/mathjax.zip.xz', data=True, allow_user_override=False)) - f.seek(0) - tdir = os.path.join(tdir, 'mathjax') - os.mkdir(tdir) - zf = ZipFile(f) - zf.extractall(tdir) - mathjax_manifest['etag'] = type('')(zf.comment) - mathjax_manifest['files'] = {type('')(zi.filename):zi.file_size for zi in zf.infolist()} - zf.close(), f.close() - return mathjax_manifest - - -def manifest_as_json(): - ans = jsonlib.dumps(get_mathjax_manifest(), ensure_ascii=False) - if not isinstance(ans, bytes): - ans = ans.encode('utf-8') - return ans + mathjax_manifest = jsonlib.loads(manifest_as_json()) + return mathjax_manifest @endpoint('/mathjax/{+which=""}', auth_required=False) def mathjax(ctx, rd, which): - manifest = get_mathjax_manifest(rd.tdir) + manifest = get_mathjax_manifest() if not which: return rd.etagged_dynamic_response(manifest['etag'], manifest_as_json, content_type='application/json; charset=UTF-8') if which not in manifest['files']: