Finish up code to transfer resources into render iframe

src/calibre/srv/render_book.py

@@ -37,14 +37,12 @@ def decode_component(x):
 def encode_url(name, frag=''):
     name = encode_component(name)
     if frag:
-        name += '#' + encode_component(frag)
+        name += '#' + frag
     return name
 
 def decode_url(x):
-    parts = list(map(decode_component, x.split('#', 1)))
+    parts = x.split('#', 1)
-    if len(parts) == 1:
+    return decode_component(parts[0]), parts[1] or ''
-        parts.append('')
-    return parts
 
 class Container(ContainerBase):
 

src/pyj/read_book/iframe.pyj

@@ -2,7 +2,9 @@
 # License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
 
 from aes import GCM
+from gettext import install
 from read_book.globals import set_boss
+from read_book.resources import finalize_resources
 
 class Boss:
 
@@ -17,7 +19,8 @@ class Boss:
         )
         set_boss(self)
         self.handlers = {
-            'keys':self.create_gcm.bind(self),
+            'initialize':self.initialize.bind(self),
+            'display': self.display.bind(self),
         }
 
     def handle_message(self, event):
@@ -33,12 +36,24 @@ class Boss:
                 return
         func = self.handlers[data.action]
         if func:
-            func(data)
+            try:
+                func(data)
+            except Exception as e:
+                console.log('Error in iframe message handler:')
+                console.log(e)
+                self.send_message({'action':'error', 'details':e.stack, 'msg':e.toString()})
         else:
             print('Unknown action in message to iframe from parent: ' + data.action)
 
-    def create_gcm(self, data):
+    def initialize(self, data):
         self.gcm_from_parent, self.gcm_to_parent = GCM(data.secret.subarray(0, 32)), GCM(data.secret.subarray(32))
+        install(data.translations)
+
+    def display(self, data):
+        self.encrypted_communications = True
+        self.book = data.book
+        root_data = finalize_resources(self.book, data.name, data.resource_data)
+        root_data
 
     def send_message(self, data):
         if self.encrypted_communications:

src/pyj/read_book/resources.pyj

@@ -1,125 +1,144 @@
 # vim:fileencoding=utf-8
 # License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
 
+from encodings import base64decode, utf8_decode
+
+JSON_XHTML_MIMETYPE = 'application/calibre+xhtml+json'
 
 def decode_component(x):
-    x = str.replace(x,',p', '|')
+    return utf8_decode(base64decode(x))
-    return str.replace(x, ',c', ',')
 
 def decode_url(x):
-    parts = x.split(',,')
+    parts = x.split('#', 1)
-    return decode_component(parts[0]), decode_component(parts[1] or '')
+    return decode_component(parts[0]), parts[1] or ''
 
-class Resource:
+def create_link_pat(book):
+    return RegExp(book.manifest.link_uid + r'\|([^|]+)\|', 'g')
 
-    def __init__(self, name, mimetype, data, placeholder, parent):
+def load_resources(db, book, root_name, previous_resources, proceed):
-        self.name = name
+    ans = {}
-        self.placeholder = placeholder
+    pending_resources = v'[root_name]'
-        if type(data) is 'string':
+    link_pat = create_link_pat(book)
-            self.text = data
-            self.mimetype = mimetype
-        else:
-            if data:
-                self.url = window.URL.createObjectURL(data)
-        self.dependencies = []
-        self.append = self.dependencies.append.bind(self.dependencies)
-        self.remove = self.dependencies.remove.bind(self.dependencies)
-        self.parent = parent
-        if parent:
-            parent.append(self)
 
-    def transfer(self, parent):
+    def do_one():
-        self.parent.remove(self)
+        if not pending_resources.length:
-        self.parent = parent
+            for k in previous_resources:
-        parent.append(self)
+                v'delete previous_resources[k]'
-
+            proceed(ans)
-    def free(self):
-        if self.url:
-            window.URL.revokeObjectURL(self.url)
-            self.url = None
-            for child in self.dependencies:
-                child.free()
-
-    def finalize(self):
-        if not self.text:
             return
-        for child in self.dependencies:
+        name = pending_resources.shift()
-            child.finalize()
+        if name in ans:
-            if child.placeholder and child.url:
+            return setTimeout(do_one, 0)
-                self.text = str.replace(self.text, child.placeholder, child.url)
+        if name in previous_resources:
-        self.url = window.createObjectURL(Blob([self.text], {'type':self.mimetype}))
+            ans[name] = data = previous_resources[name]
-        self.text = None
+            if type(data) is 'string':
+                find_virtualized_resources(data)
+            return setTimeout(do_one, 0)
+        db.get_file(book, name, got_one)
 
-    def find_match(self, name):
+    def got_one(data, name, mimetype):
-        if self.name is name:
+        if name is book.manifest.title_page_name:
-            return self
+            w = book.manifest.cover_width or 600
-        for child in self.dependencies:
+            h = book.manifest.cover_height or 800
-            x = child.find_match(name)
+            ar = 'xMidYMid meet' # or 'none'
-            if x:
+            data = str.replace(data, '__ar__', ar)
-                return x
+            data = str.replace(data, '__viewbox__', '0 0 ' + w + ' ' + h)
-
+            data = str.replace(data, '__width__', w + '')
-class ResourceManager:
+            data = str.replace(data, '__height__', h + '')
-
+        ans[name] = v'[data, mimetype]'
-    def __init__(self):
-        self.root_resource = Resource()
-        self.pending_resources = []
-
-    def new_root(self, db, book, root_name, proceed):
-        self.db = db
-        self.book = book
-        self.root_name = root_name
-        self.proceed = proceed
-        self.old_root_resource = self.root_resource
-        self.root_resource = Resource()
-        self.pending_resources = [{'name':root_name, 'parent':self.root_resource, 'placeholder':None}]
-        self.link_pat = RegExp(book.manifest.link_uid + r'\|([^|]+)\|', 'g')
-        self.do_one()
-
-    def do_one(self):
-        if not self.pending_resources.length:
-            self.root_resource.finalize()
-            self.old_root_resource.free()
-            self.old_root_resource = None
-            self.proceed(self.root_resource.dependencies[0].url)
-
-        r = self.pending_resources.pypop(0)
-        if self.root_resource.find_match(r.name):
-            return self.do_one()
-        oldr = self.old_root_resource.find_match(r.name)
-        if oldr:
-            oldr.transfer(r.parent)
-            return self.do_one()
-
-        self.db.get_file(self.book, r.name, self.got_one.bind(self, r))
-
-    def got_one(self, pending_resource, data, name, mimetype):
-        if name is self.root_name:
-            data = self.process_spine_item(data)
-            mimetype = 'application/xhtml+xml'
-        r = Resource(name, mimetype, data, pending_resource.placeholder, pending_resource.parent)
         if type(data) is 'string':
-            self.find_virtualized_resources(data, r)
+            find_virtualized_resources(data)
-        self.do_one()
+        return setTimeout(do_one, 0)
 
-    def find_virtualized_resources(self, text, parent):
+    def find_virtualized_resources(text):
         seen = set()
+        already_pending = {x.name for x in pending_resources}
         while True:
-            m = self.link_pat.exec(text)
+            m = link_pat.exec(text)
             if not m:
                 break
             name = decode_url(m[1])[0]
-            if name in seen:
+            if name in seen or name in already_pending:
                 continue
             seen.add(name)
-            self.pending_resources.push({'name':name, 'parent':parent, 'placeholder':m[0]})
+            pending_resources.push(name)
 
-    def process_spine_item(self, text):
+    do_one()
-        if self.root_name is self.book.manifest.title_page_name:
+
-            w = self.book.manifest.cover_width or 600
+def finalize_resources(book, root_name, resource_data):
-            h = self.book.manifest.cover_height or 800
+    blob_url_map = {}
-            ar = 'xMidYMid meet' # or 'none'
+    root_data = None
-            text = str.replace(text, '__ar__', ar)
+    link_pat = create_link_pat(book)
-            text = str.replace(text, '__viewbox__', '0 0 ' + w + ' ' + h)
+
-            text = str.replace(text, '__width__', w + '')
+    # Resolve the non virtualized resources immediately
-            text = str.replace(text, '__height__', h + '')
+    for name in resource_data:
-        return text
+        data, mimetype = resource_data[name]
+        if type(data) is not 'string':
+            blob_url_map[name] = window.URL.createObjectURL(data)
+    for name in blob_url_map:
+        v'delete resource_data[name]'
+
+    def add_virtualized_resource(name, text, mimetype):
+        nonlocal root_data
+        if name is root_name:
+            root_data = JSON.parse(text)
+        else:
+            blob_url_map[name] = window.URL.createObjectURL(Blob([text], {'type': mimetype}))
+
+    def replace_deps(text):
+        replacements = v'[]'
+        unresolved_deps = set()
+        while True:
+            m = link_pat.exec(text)
+            if not m:
+                break
+            dname, frag = decode_url(m[1])
+            if dname in blob_url_map:
+                rtext = blob_url_map[dname]
+                if frag:
+                    rtext += '#' + frag
+                replacements.push(v'[m.index, m[0].length, rtext]')
+            elif unresolved_deps:
+                unresolved_deps.add(dname)
+        for index, sz, repl in reversed(replacements):
+            text = text[:index] + repl + text[index:]
+        return unresolved_deps, text
+
+    unresolved_deps_map = {}
+
+    def has_unresolvable_deps(name):
+        deps = unresolved_deps_map[name]
+        if not deps or not deps.length:
+            return False
+        deps = [x for x in deps if x not in blob_url_map]
+        return deps.length > 0
+
+    while True:
+        resolved = v'[]'
+        num = 0
+        for name in resource_data:
+            if name not in blob_url_map:
+                num += 1
+                text, mimetype = resource_data[name]
+                if not has_unresolvable_deps(name):
+                    unresolved_deps, text = replace_deps(text)
+                    unresolved_deps_map[name] = unresolved_deps
+                    if not unresolved_deps.length:
+                        add_virtualized_resource(name, text, mimetype)
+                        resolved.push(name)
+        if not num:
+            break
+        if not resolved.length:
+            unresolved = [name for name in resource_data if name not in blob_url_map]
+            print(str.format('ERROR: Could not resolve all dependencies of {} because of a cyclic dependency. Remaining deps: {}', root_name, unresolved))
+            # Add the items anyway, without resolving remaining deps
+            for name in resource_data:
+                if name not in blob_url_map:
+                    text, mimetype = resource_data[name]
+                    text = replace_deps(text)[1]
+                    add_virtualized_resource(name, text, mimetype)
+            break
+        for name in resolved:
+            v'delete resource_data[name]'
+
+    return root_data

src/pyj/read_book/view.pyj

@@ -4,7 +4,7 @@
 from elementmaker import E
 from gettext import gettext as _
 from read_book.globals import messenger, iframe_id
-from read_book.resources import ResourceManager
+from read_book.resources import load_resources
 
 LOADING_DOC = '''
 <!DOCTYPE html>
@@ -15,7 +15,9 @@ __SCRIPT__
 end_script
 </head>
 <body>
+<div style="font-family: sans-serif; font-size:larger; font-weight: bold; margin-top:48vh; text-align:center">
 __BS__
+</div>
 </body>
 </html>
 '''.replace('end_script', '<' + '/script>')  # cannot have a closing script tag as this is embedded inside a script tag in index.html
@@ -24,8 +26,7 @@ class View:
 
     def __init__(self, container, ui):
         self.ui = ui
-        self.resource_manager = ResourceManager()
+        self.loaded_resources = {}
-        self.virtualized_resources = {}
         container.appendChild(
             E.iframe(
                 id=iframe_id,
@@ -41,6 +42,7 @@ class View:
         window.addEventListener('message', self.handle_message.bind(self), False)
         self.handlers = {
             'ready': self.on_iframe_ready.bind(self),
+            'error': self.on_iframe_error.bind(self),
         }
 
     @property
@@ -51,8 +53,8 @@ class View:
         iframe_script = self.ui.interface_data.main_js.replace(/is_running_in_iframe\s*=\s*false/, 'is_running_in_iframe = true')
         self.ui.interface_data.main_js = None
         self.src_doc = self.iframe.srcdoc = LOADING_DOC.replace(
-            '__SCRIPT__', iframe_script).replace(
+            '__BS__', _('Bootstrapping book reader...')).replace(
-            '__BS__', _('Bootstrapping book reader...'))
+            '__SCRIPT__', iframe_script)
 
     def init_iframe(self, iframe_script):
         self.encrypted_communications = False
@@ -82,10 +84,15 @@ class View:
 
     def on_iframe_ready(self, data):
         messenger.reset()
-        self.send_message({'action':'keys', 'secret':messenger.secret})
+        self.send_message({'action':'initialize', 'secret':messenger.secret, 'translations':self.ui.interface_data.translations})
         self.iframe_ready = True
         if self.pending_spine_load:
-            self.show_spine_item_stage2()
+            data = self.pending_spine_load
+            self.pending_spine_load = None
+            self.show_spine_item_stage2(data)
+
+    def on_iframe_error(self, data):
+        self.ui.show_error((data.title or _('There was an error processing the book')), data.msg, data.details)
 
     def show_loading(self, title):
         return  # TODO: Implement this
@@ -96,13 +103,16 @@ class View:
         self.ui.db.update_last_read_time(book)
         # TODO: Check for last open position of book
         name = book.manifest.spine[0]
-        self.resource_manager.new_root(self.ui.db, book, name, self.show_spine_item.bind(self))
+        load_resources(self.ui.db, book, name, self.loaded_resources, self.show_spine_item.bind(self, name))
 
-    def show_spine_item(self, resource_data):
+    def show_spine_item(self, name, resource_data):
+        self.loaded_resources = resource_data
         # Re-init the iframe to ensure any changes made to the environment by the last spine item are lost
         self.init_iframe()
         # Now wait for frame to message that it is ready
-        self.pending_spine_load = resource_data
+        self.pending_spine_load = [name, resource_data]
 
-    def show_spine_item_stage2(self):
+    def show_spine_item_stage2(self, x):
-        pass
+        name, resource_data = x
+        self.send_message({'action':'display', 'resource_data':resource_data, 'book':self.book, 'name':name})
+        self.encrypted_communications = True