Viewer: Dont virtualise resources

We dont need to virtualize since we can override network access. This
should improve individual HTML file loading performance a bit. Since we
dont need to replace virtualised links and load content as blobs

src/calibre/gui2/viewer/convert_book.py

@@ -120,7 +120,7 @@ def prepare_convert(temp_path, key, st):
 def do_convert(path, temp_path, key, instance):
     tdir = os.path.join(temp_path, instance['path'])
     fork_job('calibre.srv.render_book', 'render', args=(
-        path, tdir, {'size': instance['file_size'], 'mtime': instance['file_mtime'], 'hash': key}, True, True,
+        path, tdir, {'size': instance['file_size'], 'mtime': instance['file_mtime'], 'hash': key}, True, True, False,
         ), timeout=3000, no_output=True
     )
     size = 0

src/calibre/gui2/viewer/web_view.py

@@ -115,25 +115,31 @@ class UrlSchemeHandler(QWebEngineUrlSchemeHandler):
         QWebEngineUrlSchemeHandler.__init__(self, parent)
         self.mathjax_dir = P('mathjax', allow_user_override=False)
         self.mathjax_manifest = None
+        self.allowed_hosts = (FAKE_HOST, FAKE_HOST.rpartition('.')[0] + '.sandbox')
 
     def requestStarted(self, rq):
         if bytes(rq.requestMethod()) != b'GET':
             rq.fail(rq.RequestDenied)
             return
         url = rq.requestUrl()
-        if url.host() != FAKE_HOST or url.scheme() != FAKE_PROTOCOL:
+        if url.host() not in self.allowed_hosts or url.scheme() != FAKE_PROTOCOL:
             rq.fail(rq.UrlNotFound)
             return
         name = url.path()[1:]
         if name.startswith('book/'):
             name = name.partition('/')[2]
+            if name == '__index__':
+                send_reply(rq, 'text/html', b'<div>\xa0</div>')
+                return
+            elif name == '__popup__':
+                send_reply(rq, 'text/html', b'<div id="calibre-viewer-footnote-iframe">\xa0</div>')
+                return
             try:
                 data, mime_type = get_data(name)
                 if data is None:
                     rq.fail(rq.UrlNotFound)
                     return
-                if isinstance(data, type('')):
+                data = as_bytes(data)
-                    data = data.encode('utf-8')
                 mime_type = {
                     # Prevent warning in console about mimetype of fonts
                     'application/vnd.ms-opentype':'application/x-font-ttf',

src/calibre/srv/render_book.py

@@ -85,6 +85,39 @@ def convert_fontsize(length, unit, base_font_size=16.0, dpi=96.0):
     return length * length_factors.get(unit, 1) * pt_to_rem
 
 
+def create_link_replacer(container, link_uid, changed):
+    resource_template = link_uid + '|{}|'
+
+    def link_replacer(base, url):
+        if url.startswith('#'):
+            frag = urlunquote(url[1:])
+            if not frag:
+                return url
+            changed.add(base)
+            return resource_template.format(encode_url(base, frag))
+        purl = urlparse(url)
+        if purl.netloc or purl.query:
+            return url
+        if purl.scheme and purl.scheme != 'file':
+            return url
+        if not purl.path or purl.path.startswith('/'):
+            return url
+        url, frag = purl.path, purl.fragment
+        name = container.href_to_name(url, base)
+        if name:
+            if container.has_name_and_is_not_empty(name):
+                frag = urlunquote(frag)
+                url = resource_template.format(encode_url(name, frag))
+            else:
+                if isinstance(name, unicode_type):
+                    name = name.encode('utf-8')
+                url = 'missing:' + force_unicode(quote(name), 'utf-8')
+            changed.add(base)
+        return url
+
+    return link_replacer
+
+
 page_break_properties = ('page-break-before', 'page-break-after', 'page-break-inside')
 
 
@@ -217,7 +250,10 @@ class Container(ContainerBase):
 
     tweak_mode = True
 
-    def __init__(self, path_to_ebook, tdir, log=None, book_hash=None, save_bookmark_data=False, book_metadata=None, allow_no_cover=True):
+    def __init__(
+        self, path_to_ebook, tdir, log=None, book_hash=None, save_bookmark_data=False,
+        book_metadata=None, allow_no_cover=True, virtualize_resources=True
+    ):
         log = log or default_log
         self.allow_no_cover = allow_no_cover
         book_fmt, opfpath, input_fmt = extract_book(path_to_ebook, tdir, log=log)
@@ -265,9 +301,8 @@ class Container(ContainerBase):
         # Mark the spine as dirty since we have to ensure it is normalized
         for name in data['spine']:
             self.parsed(name), self.dirty(name)
-        self.transform_all()
         self.virtualized_names = set()
-        self.virtualize_resources()
+        self.transform_all(virtualize_resources)
 
         def manifest_data(name):
             mt = (self.mime_map.get(name) or 'application/octet-stream').lower()
@@ -363,8 +398,9 @@ class Container(ContainerBase):
             self.dirty(self.opf_name)
         return raster_cover_name, titlepage_name
 
-    def transform_html(self, name):
+    def transform_html(self, name, virtualize_resources):
         style_xpath = XPath('//h:style')
+        link_xpath = XPath('//h:a[@href]')
         img_xpath = XPath('//h:img[@src]')
         res_link_xpath = XPath('//h:link[@href]')
         head = ensure_head(self.parsed(name))
@@ -407,6 +443,20 @@ class Container(ContainerBase):
         if transform_inline_styles(self, name, transform_sheet=transform_sheet, transform_style=transform_declaration):
             changed = True
 
+        if not virtualize_resources:
+            link_uid = self.book_render_data['link_uid']
+            link_replacer = create_link_replacer(self, link_uid, set())
+            ltm = self.book_render_data['link_to_map']
+            for a in link_xpath(root):
+                href = link_replacer(name, a.get('href'))
+                if href and href.startswith(link_uid):
+                    a.set('href', 'javascript:void(0)')
+                    parts = decode_url(href.split('|')[1])
+                    lname, lfrag = parts[0], parts[1]
+                    ltm.setdefault(lname, {}).setdefault(lfrag or '', set()).add(name)
+                    a.set('data-' + link_uid, json.dumps({'name':lname, 'frag':lfrag}, ensure_ascii=False))
+                    changed = True
+
         if changed:
             self.dirty(name)
 
@@ -415,48 +465,30 @@ class Container(ContainerBase):
         if transform_sheet(sheet):
             self.dirty(name)
 
-    def transform_all(self):
+    def transform_all(self, virtualize_resources):
         for name, mt in tuple(iteritems(self.mime_map)):
             mt = mt.lower()
             if mt in OEB_DOCS:
-                self.transform_html(name)
+                self.transform_html(name, virtualize_resources)
-            elif mt in OEB_STYLES:
+        for name, mt in tuple(iteritems(self.mime_map)):
+            mt = mt.lower()
+            if mt in OEB_STYLES:
                 self.transform_css(name)
+        if virtualize_resources:
+            self.virtualize_resources()
+
+        ltm = self.book_render_data['link_to_map']
+        for name, amap in iteritems(ltm):
+            for k, v in tuple(iteritems(amap)):
+                amap[k] = tuple(v)  # needed for JSON serialization
 
     def virtualize_resources(self):
 
         changed = set()
         link_uid = self.book_render_data['link_uid']
-        resource_template = link_uid + '|{}|'
         xlink_xpath = XPath('//*[@xl:href]')
         link_xpath = XPath('//h:a[@href]')
-
+        link_replacer = create_link_replacer(self, link_uid, changed)
-        def link_replacer(base, url):
-            if url.startswith('#'):
-                frag = urlunquote(url[1:])
-                if not frag:
-                    return url
-                changed.add(base)
-                return resource_template.format(encode_url(base, frag))
-            purl = urlparse(url)
-            if purl.netloc or purl.query:
-                return url
-            if purl.scheme and purl.scheme != 'file':
-                return url
-            if not purl.path or purl.path.startswith('/'):
-                return url
-            url, frag = purl.path, purl.fragment
-            name = self.href_to_name(url, base)
-            if name:
-                if self.has_name_and_is_not_empty(name):
-                    frag = urlunquote(frag)
-                    url = resource_template.format(encode_url(name, frag))
-                else:
-                    if isinstance(name, unicode_type):
-                        name = name.encode('utf-8')
-                    url = 'missing:' + force_unicode(quote(name), 'utf-8')
-                changed.add(base)
-            return url
 
         ltm = self.book_render_data['link_to_map']
 
@@ -492,10 +524,6 @@ class Container(ContainerBase):
                 if altered:
                     changed.add(name)
 
-        for name, amap in iteritems(ltm):
-            for k, v in tuple(iteritems(amap)):
-                amap[k] = tuple(v)  # needed for JSON serialization
-
         tuple(map(self.dirty, changed))
 
     def serialize_item(self, name):
@@ -712,14 +740,17 @@ def get_stored_annotations(container):
                 yield {'type': 'last-read', 'pos': epubcfi, 'pos_type': 'epubcfi', 'timestamp': EPOCH}
 
 
-def render(pathtoebook, output_dir, book_hash=None, serialize_metadata=False, extract_annotations=False):
+def render(pathtoebook, output_dir, book_hash=None, serialize_metadata=False, extract_annotations=False, virtualize_resources=True):
     mi = None
     if serialize_metadata:
         from calibre.ebooks.metadata.meta import get_metadata
         from calibre.customize.ui import quick_metadata
         with lopen(pathtoebook, 'rb') as f, quick_metadata:
             mi = get_metadata(f, os.path.splitext(pathtoebook)[1][1:].lower())
-    container = Container(pathtoebook, output_dir, book_hash=book_hash, save_bookmark_data=extract_annotations, book_metadata=mi)
+    container = Container(
+        pathtoebook, output_dir, book_hash=book_hash, save_bookmark_data=extract_annotations,
+        book_metadata=mi, virtualize_resources=virtualize_resources
+    )
     if serialize_metadata:
         from calibre.utils.serialize import json_dumps
         from calibre.ebooks.metadata.book.serialize import metadata_as_dict

src/pyj/iframe_comm.pyj

@@ -58,14 +58,14 @@ class Messenger:
 
 class IframeWrapper:
 
-    def __init__(self, handlers, iframe, entry_point, bootstrap_text, srcdoc):
+    def __init__(self, handlers, iframe, entry_point, bootstrap_text, url):
         self.messenger = Messenger()
         self.iframe_id = ensure_id(iframe, 'content-iframe')
         self.needs_init = True
         self.ready = False
         self.encrypted_communications = False
         self.srcdoc_created = False
-        self.constructor_srcdoc = srcdoc
+        self.constructor_url = url
         self.entry_point = entry_point
         self.bootstrap_text = bootstrap_text
         self.handlers = {k: handlers[k] for k in handlers}
@@ -91,7 +91,7 @@ class IframeWrapper:
             }
             self.iframe.srcdoc = LOADING_DOC.replace(r, def(match, field): return data[field];)
         else:
-            self.iframe.srcdoc = self.constructor_srcdoc or '<div>\xa0</div>'
+            self.iframe.src = self.constructor_url
         self.srcdoc_created = True
 
     def init(self):
@@ -100,9 +100,13 @@ class IframeWrapper:
         self.needs_init = False
         iframe = self.iframe
         if self.srcdoc_created:
+            if self.entry_point:
                 sdoc = iframe.srcdoc
                 iframe.srcdoc = '<p>&nbsp;</p>'
                 iframe.srcdoc = sdoc
+            else:
+                iframe.src = 'about:blank'
+                iframe.src = self.constructor_url
         else:
             self.create_srcdoc()
 
@@ -129,11 +133,15 @@ class IframeWrapper:
             return
         data = event.data
         if self.encrypted_communications:
+            if data.tag is undefined:
+                print('Ignoring unencrypted message from iframe:', data)
+                return
             try:
                 data = self.messenger.decrypt(data)
             except Exception as e:
-                print('Could not process message from iframe:')
+                print('Could not decrypt message from iframe:')
                 console.log(e)
+                traceback.print_exc()
                 return
         if not data.action:
             return
@@ -174,10 +182,10 @@ class IframeClient:
     def initialize(self, data):
         nonlocal print
         self.gcm_from_parent, self.gcm_to_parent = GCM(data.secret.subarray(0, 32)), GCM(data.secret.subarray(32))
+        self.encrypted_communications = True
         if data.translations:
             install(data.translations)
         print = self.print_to_parent
-        self.encrypted_communications = True
         if self.initialize_handler:
             self.initialize_handler(data)
 
@@ -203,9 +211,11 @@ class IframeClient:
             try:
                 func(data)
             except Exception as e:
-                console.log('Error in iframe message handler:')
+                console.log('Error in iframe message handler {}:'.format(data.action))
                 console.log(e)
-                self.send_message('error', title=_('Error in message handler'), details=traceback.format_exc(), msg=e.toString())
+                details = traceback.format_exc()
+                console.log(details)
+                self.send_message('error', title=_('Error in message handler'), details=details, msg=e.toString())
         else:
             print('Unknown action in message to iframe from parent: ' + data.action)
 

src/pyj/read_book/content_popup.pyj

@@ -42,7 +42,10 @@ class ContentPopupOverlay:
         self.loaded_resources = {}
         c = self.container
         c.classList.add(CLASS_NAME)
-        iframe = E.iframe(seamless=True, sandbox='allow-scripts', style='width: 100%; max-height: 70vh')
+        sandbox = 'allow-scripts'
+        if runtime.is_standalone_viewer:
+            sandbox += ' allow-same-origin'
+        iframe = E.iframe(seamless=True, sandbox=sandbox, style='width: 100%; max-height: 70vh')
 
         c.appendChild(E.div(
             E.div(),
@@ -60,7 +63,7 @@ class ContentPopupOverlay:
         entry_point = None if runtime.is_standalone_viewer else 'read_book.footnotes'
         self.iframe_wrapper = IframeWrapper(
             handlers, iframe, entry_point, _('Loading data, please wait...'),
-            '<div id="calibre-viewer-footnote-iframe">\xa0</div>')
+            f'{runtime.FAKE_PROTOCOL}://{runtime.SANDBOX_HOST}/book/__popup__')
         self.pending_load = None
 
     @property
@@ -83,7 +86,7 @@ class ContentPopupOverlay:
         c.style.display = TOP_LEVEL_DISPLAY
 
     def on_iframe_ready(self, msg):
-        return self.do_pending_load
+        return self.do_pending_load()
 
     def apply_color_scheme(self, bg, fg):
         c = self.container.firstChild
@@ -139,7 +142,7 @@ class ContentPopupOverlay:
     def show_footnote_item_stage2(self, resource_data):
         self.iframe_wrapper.send_unencrypted_message('display',
             resource_data=resource_data, book=self.view.book, name=self.current_footnote_data.name,
-            frag=self.current_footnote_data.frag, settings=self.view.iframe_settings())
+            frag=self.current_footnote_data.frag, settings=self.view.currently_showing.settings)
 
     def on_content_loaded(self, data):
         self.iframe.style.height = f'{data.height}px'

src/pyj/read_book/footnotes.pyj

@@ -196,12 +196,12 @@ class PopupIframeBoss:
         self.book = data.book
         self.name = data.name
         self.frag = data.frag
+        update_settings(data.settings)
         for name in self.blob_url_map:
             window.URL.revokeObjectURL(self.blob_url_map[name])
         document.body.style.removeProperty('font-family')
         root_data, self.mathjax, self.blob_url_map = finalize_resources(self.book, data.name, data.resource_data)
         self.resource_urls = unserialize_html(root_data, self.content_loaded, self.show_only_footnote)
-        update_settings(data.settings)
 
     def on_clear(self, data):
         clear(document.head)
@@ -218,6 +218,9 @@ class PopupIframeBoss:
         show_footnote(self.frag, known_anchors)
 
     def content_loaded(self):
+        if not self.comm.encrypted_communications:
+            window.setTimeout(self.content_loaded, 2)
+            return
         apply_settings()
         self.comm.send_message('content_loaded', height=document.documentElement.scrollHeight + 25)
 

src/pyj/read_book/view.pyj

@@ -170,6 +170,9 @@ class View:
             oncontextmenu=self.margin_context_menu.bind(None, 'right'))
         set_right_margin_handler(right_margin)
         iframe_id = unique_id('read-book-iframe')
+        sandbox = 'allow-popups allow-scripts allow-popups-to-escape-sandbox'
+        if runtime.is_standalone_viewer:
+            sandbox += '  allow-same-origin'
         container.appendChild(
             E.div(style='max-height: 100vh; width: 100vw; height: 100vh; overflow: hidden; display: flex; align-items: stretch',  # container for horizontally aligned panels
                   E.div(style='max-height: 100vh; display: flex; flex-direction: column; align-items: stretch; flex-grow:2',  # container for iframe and any other panels in the same column
@@ -177,7 +180,7 @@ class View:
                         left_margin,
                         E.div(style='flex-grow:2; display:flex; align-items:stretch; flex-direction: column', # container for top and bottom margins
                             margin_elem(sd, 'margin_top', 'book-top-margin', self.top_margin_clicked, self.margin_context_menu.bind(None, 'top')),
-                            E.iframe(id=iframe_id, seamless=True, sandbox='allow-popups allow-scripts allow-popups-to-escape-sandbox', style='flex-grow: 2', allowfullscreen='true'),
+                            E.iframe(id=iframe_id, seamless=True, sandbox=sandbox, style='flex-grow: 2', allowfullscreen='true'),
                             margin_elem(sd, 'margin_bottom', 'book-bottom-margin', self.bottom_margin_clicked, self.margin_context_menu.bind(None, 'bottom')),
                         ),
                         right_margin,
@@ -218,7 +221,9 @@ class View:
         if runtime.is_standalone_viewer:
             document.documentElement.addEventListener('keydown', self.handle_keypress, {'passive': False})
         self.current_color_scheme = resolve_color_scheme()
-        self.iframe_wrapper = IframeWrapper(handlers, document.getElementById(iframe_id), entry_point, _('Bootstrapping book reader...'), runtime.FAKE_PROTOCOL, runtime.FAKE_HOST)
+        self.iframe_wrapper = IframeWrapper(
+            handlers, document.getElementById(iframe_id), entry_point, _('Bootstrapping book reader...'),
+            f'{runtime.FAKE_PROTOCOL}://{runtime.SANDBOX_HOST}/book/__index__')
         self.search_overlay = SearchOverlay(self)
         self.content_popup_overlay = ContentPopupOverlay(self)
         self.overlay = Overlay(self)

src/pyj/viewer-main.pyj

@@ -26,6 +26,7 @@ from viewer.constants import FAKE_HOST, FAKE_PROTOCOL, READER_BACKGROUND_URL
 
 runtime.is_standalone_viewer = True
 runtime.FAKE_HOST = FAKE_HOST
+runtime.SANDBOX_HOST = FAKE_HOST.rpartition('.')[0] + '.sandbox'
 runtime.FAKE_PROTOCOL = FAKE_PROTOCOL
 add_standalone_viewer_shortcuts()
 book = None