diff --git a/.gitignore b/.gitignore index a50921714d..5aaeaa8b9c 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ resources/template-functions.json resources/editor-functions.json resources/user-manual-translation-stats.json resources/content-server/main.js +resources/content-server/iframe.js resources/content-server/locales.zip resources/mozilla-ca-certs.pem icons/icns/*.iconset diff --git a/src/calibre/srv/render_book.py b/src/calibre/srv/render_book.py index 1feb993a83..4817654372 100644 --- a/src/calibre/srv/render_book.py +++ b/src/calibre/srv/render_book.py @@ -22,7 +22,8 @@ from calibre.ebooks.oeb.polish.utils import guess_type from calibre.utils.short_uuid import uuid4 from calibre.utils.logging import default_log -RENDER_VERSION = 1 # Also change this in read_book.ui.pyj +RENDER_VERSION = 1 + BLANK_JPEG = b'\xff\xd8\xff\xdb\x00C\x00\x03\x02\x02\x02\x02\x02\x03\x02\x02\x02\x03\x03\x03\x03\x04\x06\x04\x04\x04\x04\x04\x08\x06\x06\x05\x06\t\x08\n\n\t\x08\t\t\n\x0c\x0f\x0c\n\x0b\x0e\x0b\t\t\r\x11\r\x0e\x0f\x10\x10\x11\x10\n\x0c\x12\x13\x12\x10\x13\x0f\x10\x10\x10\xff\xc9\x00\x0b\x08\x00\x01\x00\x01\x01\x01\x11\x00\xff\xcc\x00\x06\x00\x10\x10\x05\xff\xda\x00\x08\x01\x01\x00\x00?\x00\xd2\xcf \xff\xd9' # noqa def encode_component(x): diff --git a/src/calibre/utils/rapydscript.py b/src/calibre/utils/rapydscript.py index d898ff7809..3fc89b1e64 100644 --- a/src/calibre/utils/rapydscript.py +++ b/src/calibre/utils/rapydscript.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import os, sys, atexit, errno, subprocess, glob, shutil, json +import os, sys, atexit, errno, subprocess, glob, shutil, json, hashlib, re from io import BytesIO from threading import local from functools import partial @@ -101,11 +101,20 @@ def compile_srv(): d = os.path.dirname base = d(d(d(d(os.path.abspath(__file__))))) rapydscript_dir = os.path.join(base, 'src', 'pyj') - fname = os.path.join(rapydscript_dir, 'srv.pyj') - with open(fname, 'rb') as f: - raw = compile_pyj(f.read(), fname) + rb = os.path.join(base, 'src', 'calibre', 'srv', 'render_book.py') + with lopen(rb, 'rb') as f: + rv = str(int(re.search(br'^RENDER_VERSION\s+=\s+(\d+)', f.read(), re.M).group(1))) base = P('content-server', allow_user_override=False) - with open(os.path.join(base, 'main.js'), 'wb') as f: + fname = os.path.join(rapydscript_dir, 'reader.pyj') + with lopen(fname, 'rb') as f: + reader = compile_pyj(f.read(), fname) + sha = hashlib.sha1(reader).hexdigest() + with lopen(os.path.join(base, 'iframe.js'), 'wb') as f: + f.write(reader.encode('utf-8')) + fname = os.path.join(rapydscript_dir, 'srv.pyj') + with lopen(fname, 'rb') as f: + raw = compile_pyj(f.read(), fname).replace("__IFRAME_SCRIPT_HASH__", sha).replace('__RENDER_VERSION__', rv) + with lopen(os.path.join(base, 'main.js'), 'wb') as f: f.write(raw.encode('utf-8')) # }}} diff --git a/src/pyj/read_book/db.pyj b/src/pyj/read_book/db.pyj index f82e9419da..d33ea0a836 100644 --- a/src/pyj/read_book/db.pyj +++ b/src/pyj/read_book/db.pyj @@ -1,8 +1,9 @@ # vim:fileencoding=utf-8 # License: GPL v3 Copyright: 2016, Kovid Goyal +from ajax import ajax from gettext import gettext as _ -from utils import base64encode +from utils import base64encode, base64decode def upgrade_schema(idb, old_version, new_version): print('upgrade_schema:', old_version, new_version) @@ -10,6 +11,8 @@ def upgrade_schema(idb, old_version, new_version): idb.createObjectStore('books', {'keyPath':'key'}) if not idb.objectStoreNames.contains('files'): idb.createObjectStore('files') + if not idb.objectStoreNames.contains('objects'): + idb.createObjectStore('objects', {'keyPath':'key'}) def file_store_name(book, name): return book.book_hash + ' ' + name @@ -21,13 +24,17 @@ def get_error_details(event): elif desc.errorCode: desc = desc.errorCode -DB_NAME = 'calibre-books-db-test' # TODO: Remove test suffix +IFRAME_SCRIPT_HASH = "__IFRAME_SCRIPT_HASH__" + +DB_NAME = 'calibre-books-db-testing' # TODO: Remove test suffix and change version back to 1 +DB_VERSION = 1 class DB: - def __init__(self, idb, ui, supports_blobs): + def __init__(self, idb, ui, supports_blobs, iframe_script): self.interface_data = ui.interface_data self.idb = idb + self.iframe_script = iframe_script self.supports_blobs = supports_blobs if not supports_blobs: print('IndexedDB does not support Blob storage, using base64 encoding instead') @@ -83,7 +90,8 @@ class DB: 'metadata': metadata, 'manifest': None, 'cover_width': None, - 'cover_height': None + 'cover_height': None, + 'last_read_position': None, }) ) @@ -143,11 +151,30 @@ class DB: book.is_complete = True self.do_op(['books'], book, _('Failed to write to the books database'), proceed, op='put') + def update_last_read_time(self, book): + book.last_read = Date() + self.do_op(['books'], book, _('Failed to write to the books database'), op='put') + + def get_file(self, book, name, proceed): + key = file_store_name(book, name) + err = str.format(_( + 'Failed to read the file {0} for the book {1} from the database'), name, book.metadata.title) + self.do_op(['files'], key, err, def (result): + if not result: + self.show_error(_('Cannot read book'), err) + return + fdata = book.stored_files[key] + mt = fdata.mimetype or 'application/octet-stream' + if fdata.encoded: + result = Blob([base64decode(fdata)], {'type':mt}) + proceed(result, name, mt, book) + ) + def create_db(ui, interface_data): if not window.indexedDB: - ui.db = _('Your browser does not support IndexedDB. Cannot read books. Consider using a modern browser, such as Firefox, Chrome or Edge.') - return - request = window.indexedDB.open(DB_NAME, 1) + return ui.db_initialized(_('Your browser does not support IndexedDB. Cannot read books. Consider using a modern browser, such as Firefox, Chrome or Edge.')) + + request = window.indexedDB.open(DB_NAME, DB_VERSION) request.onupgradeneeded = def(event): upgrade_schema(event.target.result, event.oldVersion, event.newVersion) @@ -156,7 +183,7 @@ def create_db(ui, interface_data): alert(_('Please close all other tabs with a calibre book open')) request.onerror = def(event): - ui.db = _('You must allow calibre to use IndexedDB storage in your browser to read books') + ui.db_initialized(_('You must allow calibre to use IndexedDB storage in your browser to read books')) request.onsuccess = def(event): blob = Blob(['test'], {'type':"text/plain"}) @@ -164,9 +191,30 @@ def create_db(ui, interface_data): try: req = idb.transaction(['files'], 'readwrite').objectStore('files').put(blob, ':-test-blob-:') except Exception: - ui.db_initialized(DB(idb, ui, False)) + print('WARNING: browser does not support blob storage, calibre falling back to base64 encoding') + create_db_stage2(idb, ui, interface_data, False) return req.onsuccess = def(event): - ui.db_initialized(DB(idb, ui, True)) + create_db_stage2(idb, ui, interface_data, True) req.onerror = def(event): - ui.db_initialized(DB(idb, ui, False)) + print('WARNING: browser does not support blob storage, calibre falling back to base64 encoding') + create_db_stage2(idb, ui, interface_data, False) + +def create_db_stage2(idb, ui, interface_data, supports_blobs): + req = idb.transaction(['objects']).objectStore('objects').get('iframe.js') + req.onerror = def(event): + ui.db_initialized(_('Failed to initialize books database: ') + get_error_details(event)) + req.onsuccess = def(event): + s = event.result + if s and s.script_hash is IFRAME_SCRIPT_HASH: + return ui.db_initialized(DB(idb, ui, supports_blobs, s.src)) + ajax('static/iframe.js', def(end_type, xhr, event): + if end_type != 'load': + return ui.db_initialized('
' + _('Failed to load book reader script') + '
' + xhr.error_html) + obj = {'key':'iframe.js', 'script_hash': IFRAME_SCRIPT_HASH, 'src':xhr.responseText} + req = idb.transaction(['objects'], 'readwrite').objectStore('objects').put(obj) + req.onerror = def(event): + ui.db_initialized(_('Failed to store book reader script in database: ') + get_error_details(event)) + req.onsuccess = def(event): + ui.db_initialized(DB(idb, ui, supports_blobs, obj.src)) + ).send() diff --git a/src/pyj/read_book/globals.pyj b/src/pyj/read_book/globals.pyj new file mode 100644 index 0000000000..3a70c17906 --- /dev/null +++ b/src/pyj/read_book/globals.pyj @@ -0,0 +1,12 @@ +# vim:fileencoding=utf-8 +# License: GPL v3 Copyright: 2016, Kovid Goyal + +_boss = None + +def set_boss(b): + nonlocal _boss + _boss = b + +def get_boss(): + return _boss + diff --git a/src/pyj/read_book/iframe.pyj b/src/pyj/read_book/iframe.pyj new file mode 100644 index 0000000000..edc6d5a448 --- /dev/null +++ b/src/pyj/read_book/iframe.pyj @@ -0,0 +1,42 @@ +# vim:fileencoding=utf-8 +# License: GPL v3 Copyright: 2016, Kovid Goyal + +from aes import GCM +from read_book.globals import set_boss + +class Boss: + + def __init__(self, gcm): + self.gcm = gcm + self.ready_sent = False + window.addEventListener('message', self.handle_message.bind(self), False) + window.addEventListener('load', def(): + if not self.ready_sent: + self.send_message({'action':'ready'}) + self.ready_sent = True + ) + set_boss(self) + + def handle_message(self, event): + if event.source is not window.parent: + return + try: + data = JSON.parse(self.gcm.decrypt(event.data)) + except Exception as e: + print('Could not process message from parent:') + console.log(e) + if data.action is 'load': + pass + + def send_message(self, data): + data = self.gcm.encrypt(JSON.stringify(data)) + window.parent.postMessage(data, '*') + + +def init(): + script = document.getElementById('bootstrap') + gcm = GCM(eval(script.getAttribute('data-key'))) + script.removeAttribute('data-key') + script.parentNode.removeChild(script) + script = None + Boss(gcm) diff --git a/src/pyj/read_book/resources.pyj b/src/pyj/read_book/resources.pyj new file mode 100644 index 0000000000..1e7797257e --- /dev/null +++ b/src/pyj/read_book/resources.pyj @@ -0,0 +1,139 @@ +# vim:fileencoding=utf-8 +# License: GPL v3 Copyright: 2016, Kovid Goyal + +from aes import GCM + +def decode_component(x): + x = str.replace(x,',p', '|') + return str.replace(x, ',c', ',') + +def decode_url(x): + parts = x.split(',,') + return decode_component(parts[0]), decode_component(parts[1] or '') + +secret_key = Uint8Array(32) +window.crypto.getRandomValues(secret_key) +secret_key_as_js = repr(secret_key) +gcm = GCM(secret_key) +iframe_id = 'read-book-iframe' + +def send_message(data): + data = gcm.encrypt(JSON.stringify(data)) + document.getElementById(iframe_id).contentWindow.postMessage(data, '*') + +def decrypt_message(data): + return JSON.parse(gcm.decrypt(data)) + +class Resource: + + def __init__(self, name, mimetype, data, placeholder, parent): + self.name = name + self.placeholder = placeholder + if type(data) is 'string': + self.text = data + self.mimetype = mimetype + else: + if data: + self.url = window.URL.createObjectURL(data) + self.dependencies = [] + self.append = self.dependencies.append.bind(self.dependencies) + self.remove = self.dependencies.remove.bind(self.dependencies) + self.parent = parent + if parent: + parent.append(self) + + def transfer(self, parent): + self.parent.remove(self) + self.parent = parent + parent.append(self) + + def free(self): + if self.url: + window.URL.revokeObjectURL(self.url) + self.url = None + for child in self.dependencies: + child.free() + + def finalize(self): + if not self.text: + return + for child in self.dependencies: + child.finalize() + if child.placeholder and child.url: + self.text = str.replace(self.text, child.placeholder, child.url) + self.url = window.createObjectURL(Blob([self.text], {'type':self.mimetype})) + self.text = None + + def find_match(self, name): + if self.name is name: + return self + for child in self.dependencies: + x = child.find_match(name) + if x: + return x + +class ResourceManager: + + def __init__(self): + self.root_resource = Resource() + self.pending_resources = [] + + def new_root(self, db, book, root_name, proceed): + self.db = db + self.book = book + self.root_name = root_name + self.proceed = proceed + self.old_root_resource = self.root_resource + self.root_resource = Resource() + self.pending_resources = [{'name':root_name, 'parent':self.root_resource, 'placeholder':None}] + self.link_pat = RegExp(book.manifest.link_uid + r'\|([^|]+)\|', 'g') + self.do_one() + + def do_one(self): + if not self.pending_resources.length: + self.root_resource.finalize() + self.old_root_resource.free() + self.old_root_resource = None + self.proceed(self.root_resource.dependencies[0].url) + + r = self.pending_resources.pypop(0) + if self.root_resource.find_match(r.name): + return self.do_one() + oldr = self.old_root_resource.find_match(r.name) + if oldr: + oldr.transfer(r.parent) + return self.do_one() + + self.db.get_file(self.book, r.name, self.got_one.bind(self, r)) + + def got_one(self, pending_resource, data, name, mimetype): + if name is self.root_name: + data = self.process_spine_item(data) + mimetype = 'application/xhtml+xml' + r = Resource(name, mimetype, data, pending_resource.placeholder, pending_resource.parent) + if type(data) is 'string': + self.find_virtualized_resources(data, r) + self.do_one() + + def find_virtualized_resources(self, text, parent): + seen = set() + while True: + m = self.link_pat.exec(text) + if not m: + break + name = decode_url(m[1])[0] + if name in seen: + continue + seen.add(name) + self.pending_resources.push({'name':name, 'parent':parent, 'placeholder':m[0]}) + + def process_spine_item(self, text): + if self.root_name is self.book.manifest.title_page_name: + w = self.book.manifest.cover_width or 600 + h = self.book.manifest.cover_height or 800 + ar = 'xMidYMid meet' # or 'none' + text = str.replace(text, '__ar__', ar) + text = str.replace(text, '__viewbox__', '0 0 ' + w + ' ' + h) + text = str.replace(text, '__width__', w + '') + text = str.replace(text, '__height__', h + '') + return text diff --git a/src/pyj/read_book/ui.pyj b/src/pyj/read_book/ui.pyj index ee0f901440..bf2b26f674 100644 --- a/src/pyj/read_book/ui.pyj +++ b/src/pyj/read_book/ui.pyj @@ -1,5 +1,6 @@ # vim:fileencoding=utf-8 # License: GPL v3 Copyright: 2016, Kovid Goyal +# globals: __RENDER_VERSION__ from ajax import ajax, encode_query from elementmaker import E @@ -10,7 +11,7 @@ from utils import human_readable from read_book.db import create_db from read_book.view import View -RENDER_VERSION = 1 # Also change this in render_book.py +RENDER_VERSION = __RENDER_VERSION__ class ReadUI: @@ -44,7 +45,7 @@ class ReadUI: container.appendChild(E.div( id=self.display_id, style='display:none', )) - self.view = View(container.lastChild) + self.view = View(container.lastChild, self) def show_stack(self, name): ans = None @@ -92,6 +93,8 @@ class ReadUI: def db_initialized(self, db): self.db = db + if type(self.db) is not 'string': + self.view.create_src_doc(self.db.iframe_script) if self.pending_load is not None: pl, self.pending_load = self.pending_load, None self.start_load(*pl) @@ -225,3 +228,4 @@ class ReadUI: def display_book(self, book): self.show_stack(self.display_id) + self.view.display_book(book) diff --git a/src/pyj/read_book/view.pyj b/src/pyj/read_book/view.pyj index 96d08bc79b..451f3ffa3d 100644 --- a/src/pyj/read_book/view.pyj +++ b/src/pyj/read_book/view.pyj @@ -3,28 +3,82 @@ from elementmaker import E from gettext import gettext as _ +from read_book.resources import ResourceManager, secret_key_as_js, iframe_id, decrypt_message LOADING_DOC = ''' -

{}

+ + + + + + +__BS__ + + ''' + class View: - def __init__(self, container): - self.iframe_id = 'read-book-iframe' + def __init__(self, container, ui): + self.ui = ui + self.resource_manager = ResourceManager() + self.virtualized_resources = {} container.appendChild( E.iframe( - id=self.iframe_id, + id=iframe_id, seamless=True, sandbox='allow-popups allow-scripts', ) ) - self.show_loading() + self.src_doc = None + self.iframe_ready = False + self.pending_spine_load = None + window.addEventListener('message', self.handle_message.bind(self), False) @property def iframe(self): - return document.getElementById(self.iframe_id) + return document.getElementById(iframe_id) - def show_loading(self): - iframe = self.iframe - iframe.setAttribute('srcdoc', str.format(LOADING_DOC, _( - 'Loading, please wait...'))) + def create_src_doc(self, iframe_script): + self.src_doc = self.iframe.srcdoc = LOADING_DOC.replace( + '__SCRIPT__', iframe_script).replace( + '__BS__', _('Bootstrapping book reader...')).replace( + '__KEY__', 'new ' + secret_key_as_js) + + def init_iframe(self, iframe_script): + self.iframe.srcdoc = self.src_doc + + def handle_message(self, event): + if event.source is not self.iframe.contentWindow: + return + try: + data = decrypt_message(event.data) + except Exception as e: + print('Could not process message from iframe:') + console.log(e) + if data.action is 'ready': + self.iframe_ready = True + if self.pending_spine_load: + self.show_spine_item_stage2() + + def show_loading(self, title): + return # TODO: Implement this + + def display_book(self, book): + self.book = book + self.show_loading(book.metadata.title) + self.ui.db.update_last_read_time(book) + # TODO: Check for last open position of book + name = book.manifest.spine[0] + self.resource_manager.new_root(self.ui.db, book, name, self.show_spine_item.bind(self)) + + def show_spine_item(self, resource_data): + # Re-init the iframe to ensure any changes made to the environment by the last spine item are lost + self.init_iframe() + # Now wait for frame to message that it is ready + self.pending_spine_load = resource_data + + def show_spine_item_stage2(self): + pass diff --git a/src/pyj/reader.pyj b/src/pyj/reader.pyj new file mode 100644 index 0000000000..52f95608b1 --- /dev/null +++ b/src/pyj/reader.pyj @@ -0,0 +1,5 @@ +# vim:fileencoding=utf-8 +# License: GPL v3 Copyright: 2016, Kovid Goyal + +from read_book.iframe import init +init() diff --git a/src/pyj/utils.pyj b/src/pyj/utils.pyj index 2d2da81df7..cf333b13a5 100644 --- a/src/pyj/utils.pyj +++ b/src/pyj/utils.pyj @@ -52,6 +52,17 @@ def base64encode(bytes): ans.push(encodings[(chunk & 64512) >> 10], encodings[(chunk & 1008) >> 4], encodings[(chunk & 15) << 2], '=') return ans.join('') +def base64decode(string): + # convert the output of base64encode back into an array of bytes (Uint8Array) + if type(window) is not 'undefined': + chars = window.atob(string) + else: + chars = new Buffer(string, 'base64').toString('binary') # noqa: undef + ans = Uint8Array(chars.length) + for i in range(ans.length): + ans[i] = chars.charCodeAt(i) + return ans + def parse_url_params(url=None, allow_multiple=False): url = url or window.location.href qs = url.indexOf('?') @@ -114,4 +125,3 @@ def human_readable(size, sep=' '): if __name__ is '__main__': print(fmt_sidx(10), fmt_sidx(1.2)) print(list(map(human_readable, [1, 1024.0, 1025, 1024*1024*2.3]))) - print(base64encode(list(range(256))))