Establish communication between parent and sanboxed render iframe

.gitignore

@@ -21,6 +21,7 @@ resources/template-functions.json
 resources/editor-functions.json
 resources/user-manual-translation-stats.json
 resources/content-server/main.js
+resources/content-server/iframe.js
 resources/content-server/locales.zip
 resources/mozilla-ca-certs.pem
 icons/icns/*.iconset

src/calibre/srv/render_book.py

@@ -22,7 +22,8 @@ from calibre.ebooks.oeb.polish.utils import guess_type
 from calibre.utils.short_uuid import uuid4
 from calibre.utils.logging import default_log
 
-RENDER_VERSION = 1  # Also change this in read_book.ui.pyj
+RENDER_VERSION = 1
+
 BLANK_JPEG = b'\xff\xd8\xff\xdb\x00C\x00\x03\x02\x02\x02\x02\x02\x03\x02\x02\x02\x03\x03\x03\x03\x04\x06\x04\x04\x04\x04\x04\x08\x06\x06\x05\x06\t\x08\n\n\t\x08\t\t\n\x0c\x0f\x0c\n\x0b\x0e\x0b\t\t\r\x11\r\x0e\x0f\x10\x10\x11\x10\n\x0c\x12\x13\x12\x10\x13\x0f\x10\x10\x10\xff\xc9\x00\x0b\x08\x00\x01\x00\x01\x01\x01\x11\x00\xff\xcc\x00\x06\x00\x10\x10\x05\xff\xda\x00\x08\x01\x01\x00\x00?\x00\xd2\xcf \xff\xd9'  # noqa
 
 def encode_component(x):

src/calibre/utils/rapydscript.py

@@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
 __license__ = 'GPL v3'
 __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
 
-import os, sys, atexit, errno, subprocess, glob, shutil, json
+import os, sys, atexit, errno, subprocess, glob, shutil, json, hashlib, re
 from io import BytesIO
 from threading import local
 from functools import partial
@@ -101,11 +101,20 @@ def compile_srv():
     d = os.path.dirname
     base = d(d(d(d(os.path.abspath(__file__)))))
     rapydscript_dir = os.path.join(base, 'src', 'pyj')
-    fname = os.path.join(rapydscript_dir, 'srv.pyj')
-    with open(fname, 'rb') as f:
-        raw = compile_pyj(f.read(), fname)
+    rb = os.path.join(base, 'src', 'calibre', 'srv', 'render_book.py')
+    with lopen(rb, 'rb') as f:
+        rv = str(int(re.search(br'^RENDER_VERSION\s+=\s+(\d+)', f.read(), re.M).group(1)))
     base = P('content-server', allow_user_override=False)
-    with open(os.path.join(base, 'main.js'), 'wb') as f:
+    fname = os.path.join(rapydscript_dir, 'reader.pyj')
+    with lopen(fname, 'rb') as f:
+        reader = compile_pyj(f.read(), fname)
+    sha = hashlib.sha1(reader).hexdigest()
+    with lopen(os.path.join(base, 'iframe.js'), 'wb') as f:
+        f.write(reader.encode('utf-8'))
+    fname = os.path.join(rapydscript_dir, 'srv.pyj')
+    with lopen(fname, 'rb') as f:
+        raw = compile_pyj(f.read(), fname).replace("__IFRAME_SCRIPT_HASH__", sha).replace('__RENDER_VERSION__', rv)
+    with lopen(os.path.join(base, 'main.js'), 'wb') as f:
         f.write(raw.encode('utf-8'))
 
 # }}}

src/pyj/read_book/db.pyj

@@ -1,8 +1,9 @@
 # vim:fileencoding=utf-8
 # License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
 
+from ajax import ajax
 from gettext import gettext as _
-from utils import base64encode
+from utils import base64encode, base64decode
 
 def upgrade_schema(idb, old_version, new_version):
     print('upgrade_schema:', old_version, new_version)
@@ -10,6 +11,8 @@ def upgrade_schema(idb, old_version, new_version):
         idb.createObjectStore('books', {'keyPath':'key'})
     if not idb.objectStoreNames.contains('files'):
         idb.createObjectStore('files')
+    if not idb.objectStoreNames.contains('objects'):
+        idb.createObjectStore('objects', {'keyPath':'key'})
 
 def file_store_name(book, name):
     return book.book_hash + ' ' + name
@@ -21,13 +24,17 @@ def get_error_details(event):
     elif desc.errorCode:
         desc = desc.errorCode
 
-DB_NAME = 'calibre-books-db-test'  # TODO: Remove test suffix
+IFRAME_SCRIPT_HASH = "__IFRAME_SCRIPT_HASH__"
+
+DB_NAME = 'calibre-books-db-testing'  # TODO: Remove test suffix and change version back to 1
+DB_VERSION = 1
 
 class DB:
 
-    def __init__(self, idb, ui, supports_blobs):
+    def __init__(self, idb, ui, supports_blobs, iframe_script):
         self.interface_data = ui.interface_data
         self.idb = idb
+        self.iframe_script = iframe_script
         self.supports_blobs = supports_blobs
         if not supports_blobs:
             print('IndexedDB does not support Blob storage, using base64 encoding instead')
@@ -83,7 +90,8 @@ class DB:
                 'metadata': metadata,
                 'manifest': None,
                 'cover_width': None,
-                'cover_height': None
+                'cover_height': None,
+                'last_read_position': None,
             })
         )
 
@@ -143,11 +151,30 @@ class DB:
         book.is_complete = True
         self.do_op(['books'], book, _('Failed to write to the books database'), proceed, op='put')
 
+    def update_last_read_time(self, book):
+        book.last_read = Date()
+        self.do_op(['books'], book, _('Failed to write to the books database'), op='put')
+
+    def get_file(self, book, name, proceed):
+        key = file_store_name(book, name)
+        err = str.format(_(
+            'Failed to read the file {0} for the book {1} from the database'), name, book.metadata.title)
+        self.do_op(['files'], key, err, def (result):
+            if not result:
+                self.show_error(_('Cannot read book'), err)
+                return
+            fdata = book.stored_files[key]
+            mt = fdata.mimetype or 'application/octet-stream'
+            if fdata.encoded:
+                result = Blob([base64decode(fdata)], {'type':mt})
+            proceed(result, name, mt, book)
+        )
+
 def create_db(ui, interface_data):
     if not window.indexedDB:
-        ui.db = _('Your browser does not support IndexedDB. Cannot read books. Consider using a modern browser, such as Firefox, Chrome or Edge.')
-        return
-    request = window.indexedDB.open(DB_NAME, 1)
+        return ui.db_initialized(_('Your browser does not support IndexedDB. Cannot read books. Consider using a modern browser, such as Firefox, Chrome or Edge.'))
+
+    request = window.indexedDB.open(DB_NAME, DB_VERSION)
 
     request.onupgradeneeded = def(event):
         upgrade_schema(event.target.result, event.oldVersion, event.newVersion)
@@ -156,7 +183,7 @@ def create_db(ui, interface_data):
         alert(_('Please close all other tabs with a calibre book open'))
 
     request.onerror = def(event):
-        ui.db = _('You must allow calibre to use IndexedDB storage in your browser to read books')
+        ui.db_initialized(_('You must allow calibre to use IndexedDB storage in your browser to read books'))
 
     request.onsuccess = def(event):
         blob = Blob(['test'], {'type':"text/plain"})
@@ -164,9 +191,30 @@ def create_db(ui, interface_data):
         try:
             req = idb.transaction(['files'], 'readwrite').objectStore('files').put(blob, ':-test-blob-:')
         except Exception:
-            ui.db_initialized(DB(idb, ui, False))
+            print('WARNING: browser does not support blob storage, calibre falling back to base64 encoding')
+            create_db_stage2(idb, ui, interface_data, False)
             return
         req.onsuccess = def(event):
-            ui.db_initialized(DB(idb, ui, True))
+            create_db_stage2(idb, ui, interface_data, True)
         req.onerror = def(event):
-            ui.db_initialized(DB(idb, ui, False))
+            print('WARNING: browser does not support blob storage, calibre falling back to base64 encoding')
+            create_db_stage2(idb, ui, interface_data, False)
+
+def create_db_stage2(idb, ui, interface_data, supports_blobs):
+    req = idb.transaction(['objects']).objectStore('objects').get('iframe.js')
+    req.onerror = def(event):
+        ui.db_initialized(_('Failed to initialize books database: ') + get_error_details(event))
+    req.onsuccess = def(event):
+        s = event.result
+        if s and s.script_hash is IFRAME_SCRIPT_HASH:
+            return ui.db_initialized(DB(idb, ui, supports_blobs, s.src))
+        ajax('static/iframe.js', def(end_type, xhr, event):
+            if end_type != 'load':
+                return ui.db_initialized('<div>' + _('Failed to load book reader script') + '</div>' + xhr.error_html)
+            obj = {'key':'iframe.js', 'script_hash': IFRAME_SCRIPT_HASH, 'src':xhr.responseText}
+            req = idb.transaction(['objects'], 'readwrite').objectStore('objects').put(obj)
+            req.onerror = def(event):
+                ui.db_initialized(_('Failed to store book reader script in database: ') + get_error_details(event))
+            req.onsuccess = def(event):
+                ui.db_initialized(DB(idb, ui, supports_blobs, obj.src))
+        ).send()

src/pyj/read_book/globals.pyj

@@ -0,0 +1,12 @@
+# vim:fileencoding=utf-8
+# License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
+
+_boss = None
+
+def set_boss(b):
+    nonlocal _boss
+    _boss = b
+
+def get_boss():
+    return _boss
+

src/pyj/read_book/iframe.pyj

@@ -0,0 +1,42 @@
+# vim:fileencoding=utf-8
+# License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
+
+from aes import GCM
+from read_book.globals import set_boss
+
+class Boss:
+
+    def __init__(self, gcm):
+        self.gcm = gcm
+        self.ready_sent = False
+        window.addEventListener('message', self.handle_message.bind(self), False)
+        window.addEventListener('load', def():
+            if not self.ready_sent:
+                self.send_message({'action':'ready'})
+                self.ready_sent = True
+        )
+        set_boss(self)
+
+    def handle_message(self, event):
+        if event.source is not window.parent:
+            return
+        try:
+            data = JSON.parse(self.gcm.decrypt(event.data))
+        except Exception as e:
+            print('Could not process message from parent:')
+            console.log(e)
+        if data.action is 'load':
+            pass
+
+    def send_message(self, data):
+        data = self.gcm.encrypt(JSON.stringify(data))
+        window.parent.postMessage(data, '*')
+
+
+def init():
+    script = document.getElementById('bootstrap')
+    gcm = GCM(eval(script.getAttribute('data-key')))
+    script.removeAttribute('data-key')
+    script.parentNode.removeChild(script)
+    script = None
+    Boss(gcm)

src/pyj/read_book/resources.pyj

@@ -0,0 +1,139 @@
+# vim:fileencoding=utf-8
+# License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
+
+from aes import GCM
+
+def decode_component(x):
+    x = str.replace(x,',p', '|')
+    return str.replace(x, ',c', ',')
+
+def decode_url(x):
+    parts = x.split(',,')
+    return decode_component(parts[0]), decode_component(parts[1] or '')
+
+secret_key = Uint8Array(32)
+window.crypto.getRandomValues(secret_key)
+secret_key_as_js = repr(secret_key)
+gcm = GCM(secret_key)
+iframe_id = 'read-book-iframe'
+
+def send_message(data):
+    data = gcm.encrypt(JSON.stringify(data))
+    document.getElementById(iframe_id).contentWindow.postMessage(data, '*')
+
+def decrypt_message(data):
+    return JSON.parse(gcm.decrypt(data))
+
+class Resource:
+
+    def __init__(self, name, mimetype, data, placeholder, parent):
+        self.name = name
+        self.placeholder = placeholder
+        if type(data) is 'string':
+            self.text = data
+            self.mimetype = mimetype
+        else:
+            if data:
+                self.url = window.URL.createObjectURL(data)
+        self.dependencies = []
+        self.append = self.dependencies.append.bind(self.dependencies)
+        self.remove = self.dependencies.remove.bind(self.dependencies)
+        self.parent = parent
+        if parent:
+            parent.append(self)
+
+    def transfer(self, parent):
+        self.parent.remove(self)
+        self.parent = parent
+        parent.append(self)
+
+    def free(self):
+        if self.url:
+            window.URL.revokeObjectURL(self.url)
+            self.url = None
+            for child in self.dependencies:
+                child.free()
+
+    def finalize(self):
+        if not self.text:
+            return
+        for child in self.dependencies:
+            child.finalize()
+            if child.placeholder and child.url:
+                self.text = str.replace(self.text, child.placeholder, child.url)
+        self.url = window.createObjectURL(Blob([self.text], {'type':self.mimetype}))
+        self.text = None
+
+    def find_match(self, name):
+        if self.name is name:
+            return self
+        for child in self.dependencies:
+            x = child.find_match(name)
+            if x:
+                return x
+
+class ResourceManager:
+
+    def __init__(self):
+        self.root_resource = Resource()
+        self.pending_resources = []
+
+    def new_root(self, db, book, root_name, proceed):
+        self.db = db
+        self.book = book
+        self.root_name = root_name
+        self.proceed = proceed
+        self.old_root_resource = self.root_resource
+        self.root_resource = Resource()
+        self.pending_resources = [{'name':root_name, 'parent':self.root_resource, 'placeholder':None}]
+        self.link_pat = RegExp(book.manifest.link_uid + r'\|([^|]+)\|', 'g')
+        self.do_one()
+
+    def do_one(self):
+        if not self.pending_resources.length:
+            self.root_resource.finalize()
+            self.old_root_resource.free()
+            self.old_root_resource = None
+            self.proceed(self.root_resource.dependencies[0].url)
+
+        r = self.pending_resources.pypop(0)
+        if self.root_resource.find_match(r.name):
+            return self.do_one()
+        oldr = self.old_root_resource.find_match(r.name)
+        if oldr:
+            oldr.transfer(r.parent)
+            return self.do_one()
+
+        self.db.get_file(self.book, r.name, self.got_one.bind(self, r))
+
+    def got_one(self, pending_resource, data, name, mimetype):
+        if name is self.root_name:
+            data = self.process_spine_item(data)
+            mimetype = 'application/xhtml+xml'
+        r = Resource(name, mimetype, data, pending_resource.placeholder, pending_resource.parent)
+        if type(data) is 'string':
+            self.find_virtualized_resources(data, r)
+        self.do_one()
+
+    def find_virtualized_resources(self, text, parent):
+        seen = set()
+        while True:
+            m = self.link_pat.exec(text)
+            if not m:
+                break
+            name = decode_url(m[1])[0]
+            if name in seen:
+                continue
+            seen.add(name)
+            self.pending_resources.push({'name':name, 'parent':parent, 'placeholder':m[0]})
+
+    def process_spine_item(self, text):
+        if self.root_name is self.book.manifest.title_page_name:
+            w = self.book.manifest.cover_width or 600
+            h = self.book.manifest.cover_height or 800
+            ar = 'xMidYMid meet' # or 'none'
+            text = str.replace(text, '__ar__', ar)
+            text = str.replace(text, '__viewbox__', '0 0 ' + w + ' ' + h)
+            text = str.replace(text, '__width__', w + '')
+            text = str.replace(text, '__height__', h + '')
+        return text

src/pyj/read_book/ui.pyj

@@ -1,5 +1,6 @@
 # vim:fileencoding=utf-8
 # License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
+# globals: __RENDER_VERSION__
 
 from ajax import ajax, encode_query
 from elementmaker import E
@@ -10,7 +11,7 @@ from utils import human_readable
 from read_book.db import create_db
 from read_book.view import View
 
-RENDER_VERSION = 1  # Also change this in render_book.py
+RENDER_VERSION = __RENDER_VERSION__
 
 class ReadUI:
 
@@ -44,7 +45,7 @@ class ReadUI:
         container.appendChild(E.div(
             id=self.display_id, style='display:none',
         ))
-        self.view = View(container.lastChild)
+        self.view = View(container.lastChild, self)
 
     def show_stack(self, name):
         ans = None
@@ -92,6 +93,8 @@ class ReadUI:
 
     def db_initialized(self, db):
         self.db = db
+        if type(self.db) is not 'string':
+            self.view.create_src_doc(self.db.iframe_script)
         if self.pending_load is not None:
             pl, self.pending_load = self.pending_load, None
             self.start_load(*pl)
@@ -225,3 +228,4 @@ class ReadUI:
 
     def display_book(self, book):
         self.show_stack(self.display_id)
+        self.view.display_book(book)

src/pyj/read_book/view.pyj

@@ -3,28 +3,82 @@
 
 from elementmaker import E
 from gettext import gettext as _
+from read_book.resources import ResourceManager, secret_key_as_js, iframe_id, decrypt_message
 
 LOADING_DOC = '''
-<p>{}</p>
+<!DOCTYPE html>
+<html>
+<head>
+<script type="text/javascript" id="bootstrap" data-key="__KEY__">
+__SCRIPT__
+</script>
+</head>
+<body>
+__BS__
+</body>
+</html>
 '''
+
 class View:
 
-    def __init__(self, container):
-        self.iframe_id = 'read-book-iframe'
+    def __init__(self, container, ui):
+        self.ui = ui
+        self.resource_manager = ResourceManager()
+        self.virtualized_resources = {}
         container.appendChild(
             E.iframe(
-                id=self.iframe_id,
+                id=iframe_id,
                 seamless=True,
                 sandbox='allow-popups allow-scripts',
             )
         )
-        self.show_loading()
+        self.src_doc = None
+        self.iframe_ready = False
+        self.pending_spine_load = None
+        window.addEventListener('message', self.handle_message.bind(self), False)
 
     @property
     def iframe(self):
-        return document.getElementById(self.iframe_id)
+        return document.getElementById(iframe_id)
 
-    def show_loading(self):
-        iframe = self.iframe
-        iframe.setAttribute('srcdoc', str.format(LOADING_DOC, _(
-            'Loading, please wait...')))
+    def create_src_doc(self, iframe_script):
+        self.src_doc = self.iframe.srcdoc = LOADING_DOC.replace(
+            '__SCRIPT__', iframe_script).replace(
+            '__BS__', _('Bootstrapping book reader...')).replace(
+            '__KEY__', 'new ' + secret_key_as_js)
+
+    def init_iframe(self, iframe_script):
+        self.iframe.srcdoc = self.src_doc
+
+    def handle_message(self, event):
+        if event.source is not self.iframe.contentWindow:
+            return
+        try:
+            data = decrypt_message(event.data)
+        except Exception as e:
+            print('Could not process message from iframe:')
+            console.log(e)
+        if data.action is 'ready':
+            self.iframe_ready = True
+            if self.pending_spine_load:
+                self.show_spine_item_stage2()
+
+    def show_loading(self, title):
+        return  # TODO: Implement this
+
+    def display_book(self, book):
+        self.book = book
+        self.show_loading(book.metadata.title)
+        self.ui.db.update_last_read_time(book)
+        # TODO: Check for last open position of book
+        name = book.manifest.spine[0]
+        self.resource_manager.new_root(self.ui.db, book, name, self.show_spine_item.bind(self))
+
+    def show_spine_item(self, resource_data):
+        # Re-init the iframe to ensure any changes made to the environment by the last spine item are lost
+        self.init_iframe()
+        # Now wait for frame to message that it is ready
+        self.pending_spine_load = resource_data
+
+    def show_spine_item_stage2(self):
+        pass

src/pyj/reader.pyj

@@ -0,0 +1,5 @@
+# vim:fileencoding=utf-8
+# License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
+
+from read_book.iframe import init
+init()

src/pyj/utils.pyj

@@ -52,6 +52,17 @@ def base64encode(bytes):
         ans.push(encodings[(chunk & 64512) >> 10], encodings[(chunk & 1008) >> 4], encodings[(chunk & 15) << 2], '=')
     return ans.join('')
 
+def base64decode(string):
+    # convert the output of base64encode back into an array of bytes (Uint8Array)
+    if type(window) is not 'undefined':
+        chars = window.atob(string)
+    else:
+        chars = new Buffer(string, 'base64').toString('binary')  # noqa: undef
+    ans = Uint8Array(chars.length)
+    for i in range(ans.length):
+        ans[i] = chars.charCodeAt(i)
+    return ans
+
 def parse_url_params(url=None, allow_multiple=False):
     url = url or window.location.href
     qs = url.indexOf('?')
@@ -114,4 +125,3 @@ def human_readable(size, sep=' '):
 if __name__ is '__main__':
     print(fmt_sidx(10), fmt_sidx(1.2))
     print(list(map(human_readable, [1, 1024.0, 1025, 1024*1024*2.3])))
-    print(base64encode(list(range(256))))