E-book viewer: Fix regression in previous release that broke text layout for some books. Fixes #1652408 [Last update does not always center the pages in the reader.](https://bugs.launchpad.net/calibre/+bug/1652408)

src/calibre/ebooks/oeb/display/extract.coffee

@@ -50,8 +50,8 @@ get_containing_block = (node) ->
 trim = (str) ->
     return str.replace(/^\s\s*/, '').replace(/\s\s*$/, '')
 
-is_footnote_link = (node, url, linked_to_anchors) ->
+is_footnote_link = (node, url, linked_to_anchors, prefix) ->
-    if not url or url.substr(0, 'file://'.length).toLowerCase() != 'file://'
+    if not url or url.substr(0, prefix.length) != prefix
         return false  # Ignore non-local links
     epub_type = get_epub_type(node, ['noteref'])
     if epub_type and epub_type.toLowerCase() == 'noteref'
@@ -163,8 +163,8 @@ class CalibreExtract
         cnode = inline_styles(node)
         return cnode.outerHTML
 
-    is_footnote_link: (a) ->
+    is_footnote_link: (a, prefix) ->
-        return is_footnote_link(a, a.href, py_bridge.value)
+        return is_footnote_link(a, a.href, py_bridge.value, prefix)
 
     show_footnote: (target, known_targets) ->
         if not target

src/calibre/ebooks/oeb/display/mathjax.coffee

@@ -32,7 +32,7 @@ class MathJax
         scale = if is_windows then 160 else 100
 
         script.type = 'text/javascript'
-        script.src = 'file://' + this.base + '/MathJax.js'
+        script.src = this.base + 'MathJax.js'
         script.text = user_config + ('''
         MathJax.Hub.signal.Interest(function (message) {if (String(message).match(/error/i)) {console.log(message)}});
         MathJax.Hub.Config({
@@ -111,5 +111,3 @@ class MathJax
 
 if window?
     window.mathjax = new MathJax()
-
-

src/calibre/ebooks/oeb/display/webview.py

@@ -33,9 +33,20 @@ def self_closing_sub(match):
     return '<%s%s></%s>'%(match.group(1), match.group(2), match.group(1))
 
 
+def cleanup_html(html):
+    html = EntityDeclarationProcessor(html).processed_html
+    self_closing_pat = re.compile(r'<\s*([:A-Za-z0-9-]+)([^>]*)/\s*>')
+    html = self_closing_pat.sub(self_closing_sub, html)
+    return html
+
+
+def load_as_html(html):
+    return re.search(r'<[a-zA-Z0-9-]+:svg', html) is None and '<![CDATA[' not in html
+
+
 def load_html(path, view, codec='utf-8', mime_type=None,
               pre_load_callback=lambda x:None, path_is_html=False,
-              force_as_html=False):
+              force_as_html=False, loading_url=None):
     from PyQt5.Qt import QUrl, QByteArray
     if mime_type is None:
         mime_type = guess_type(path)[0]
@@ -47,14 +58,11 @@ def load_html(path, view, codec='utf-8', mime_type=None,
         with open(path, 'rb') as f:
             html = f.read().decode(codec, 'replace')
 
-    html = EntityDeclarationProcessor(html).processed_html
+    html = cleanup_html(html)
-    self_closing_pat = re.compile(r'<\s*([:A-Za-z0-9-]+)([^>]*)/\s*>')
+    loading_url = loading_url or QUrl.fromLocalFile(path)
-    html = self_closing_pat.sub(self_closing_sub, html)
-
-    loading_url = QUrl.fromLocalFile(path)
     pre_load_callback(loading_url)
 
-    if force_as_html or re.search(r'<[a-zA-Z0-9-]+:svg', html) is None and '<![CDATA[' not in html:
+    if force_as_html or load_as_html(html):
         view.setHtml(html, loading_url)
     else:
         view.setContent(QByteArray(html.encode(codec)), mime_type,

src/calibre/gui2/viewer/documentview.py

@@ -4,7 +4,7 @@ __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 
 # Imports {{{
-import os, math, json
+import math, json
 from base64 import b64encode
 from functools import partial
 from future_builtins import map
@@ -18,7 +18,7 @@ from PyQt5.QtWebKit import QWebSettings, QWebElement
 
 from calibre.gui2.viewer.flip import SlideFlip
 from calibre.gui2.shortcuts import Shortcuts
-from calibre.gui2 import open_url, secure_web_page
+from calibre.gui2 import open_url, secure_web_page, error_dialog
 from calibre import prints
 from calibre.customize.ui import all_viewer_plugins
 from calibre.gui2.viewer.keys import SHORTCUTS
@@ -30,6 +30,7 @@ from calibre.gui2.viewer.table_popup import TablePopup
 from calibre.gui2.viewer.inspector import WebInspector
 from calibre.gui2.viewer.gestures import GestureHandler
 from calibre.gui2.viewer.footnote import Footnotes
+from calibre.gui2.viewer.fake_net import NetworkAccessManager
 from calibre.ebooks.oeb.display.webview import load_html
 from calibre.constants import isxp, iswindows, DEBUG, __version__
 # }}}
@@ -84,6 +85,8 @@ class Document(QWebPage):  # {{{
 
     def __init__(self, shortcuts, parent=None, debug_javascript=False):
         QWebPage.__init__(self, parent)
+        self.nam = NetworkAccessManager(self)
+        self.setNetworkAccessManager(self.nam)
         self.setObjectName("py_bridge")
         self.in_paged_mode = False
         # Use this to pass arbitrary JSON encodable objects between python and
@@ -212,11 +215,7 @@ class Document(QWebPage):  # {{{
         evaljs('window.calibre_utils.setup_epub_reading_system(%s, %s, %s, %s)' % tuple(map(json.dumps, (
             'calibre-desktop', __version__, 'paginated' if self.in_paged_mode else 'scrolling',
             'dom-manipulation layout-changes mouse-events keyboard-events'.split()))))
-        mjpath = P(u'viewer/mathjax').replace(os.sep, '/')
+        self.javascript(u'window.mathjax.base = %s'%(json.dumps(self.nam.mathjax_base, ensure_ascii=False)))
-        if iswindows:
-            mjpath = u'/' + mjpath
-        self.javascript(u'window.mathjax.base = %s'%(json.dumps(mjpath,
-            ensure_ascii=False)))
         for pl in self.all_viewer_plugins:
             pl.load_javascript(evaljs)
         evaljs('py_bridge.mark_element.connect(window.calibre_extract.mark)')
@@ -299,8 +298,7 @@ class Document(QWebPage):  # {{{
             cols_per_screen, self.top_margin, self.side_margin,
             self.bottom_margin
             ))
-        force_fullscreen_layout = bool(getattr(last_loaded_path,
+        force_fullscreen_layout = self.nam.is_single_page(last_loaded_path)
-                                               'is_single_page', False))
         self.update_contents_size_for_paged_mode(force_fullscreen_layout)
 
     def update_contents_size_for_paged_mode(self, force_fullscreen_layout=None):
@@ -564,6 +562,7 @@ class DocumentView(QWebView):  # {{{
         self.to_bottom = False
         self.document = Document(self.shortcuts, parent=self,
                 debug_javascript=debug_javascript)
+        self.document.nam.load_error.connect(self.on_unhandled_load_error)
         self.footnotes = Footnotes(self)
         self.document.settings_changed.connect(self.footnotes.clone_settings)
         self.setPage(self.document)
@@ -719,7 +718,7 @@ class DocumentView(QWebView):  # {{{
 
     def popup_table(self):
         html = self.document.extract_node()
-        self.table_popup(html, QUrl.fromLocalFile(self.last_loaded_path),
+        self.table_popup(html, self.as_url(self.last_loaded_path),
                          self.document.font_magnification_step)
 
     def contextMenuEvent(self, ev):
@@ -914,8 +913,12 @@ class DocumentView(QWebView):  # {{{
             self.document.javascript('paged_display.snap_to_selection()')
         return found
 
-    def path(self):
+    def path(self, url=None):
-        return os.path.abspath(unicode(self.url().toLocalFile()))
+        url = url or self.url()
+        return self.document.nam.as_abspath(url)
+
+    def as_url(self, path):
+        return self.document.nam.as_url(path)
 
     def load_path(self, path, pos=0.0):
         self.initial_pos = pos
@@ -924,13 +927,7 @@ class DocumentView(QWebView):  # {{{
         # evaluated in read_document_margins() in paged mode.
         self.document.setPreferredContentsSize(QSize())
 
-        def callback(lu):
+        url = self.as_url(path)
-            self.loading_url = lu
-            if self.manager is not None:
-                self.manager.load_started()
-
-        load_html(path, self, codec=getattr(path, 'encoding', 'utf-8'), mime_type=getattr(path,
-            'mime_type', 'text/html'), pre_load_callback=callback)
         entries = set()
         for ie in getattr(path, 'index_entries', []):
             if ie.start_anchor:
@@ -939,6 +936,18 @@ class DocumentView(QWebView):  # {{{
                 entries.add(ie.end_anchor)
         self.document.index_anchors = entries
 
+        def callback(lu):
+            self.loading_url = lu
+            if self.manager is not None:
+                self.manager.load_started()
+
+        load_html(path, self, codec=getattr(path, 'encoding', 'utf-8'), mime_type=getattr(path,
+            'mime_type', 'text/html'), loading_url=url, pre_load_callback=callback)
+
+    def on_unhandled_load_error(self, name, tb):
+        error_dialog(self, _('Failed to load file'), _(
+            'Failed to load the file: {}. Click "Show details" for more information').format(name), det_msg=tb, show=True)
+
     def initialize_scrollbar(self):
         if getattr(self, 'scrollbar', None) is not None:
             if self.document.in_paged_mode:
@@ -1428,4 +1437,7 @@ class DocumentView(QWebView):  # {{{
         if qurl and qurl.isValid():
             self.link_clicked(qurl)
 
+    def set_book_data(self, iterator):
+        self.document.nam.set_book_data(iterator.base, iterator.spine)
+
 # }}}

src/calibre/gui2/viewer/fake_net.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python2
+# vim:fileencoding=utf-8
+# License: GPLv3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
+
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+import os
+
+from PyQt5.Qt import QNetworkReply, QNetworkAccessManager, QUrl, QNetworkRequest, QTimer, pyqtSignal
+
+from calibre import guess_type as _guess_type, prints
+from calibre.constants import FAKE_HOST, FAKE_PROTOCOL, DEBUG
+from calibre.ebooks.oeb.base import OEB_DOCS
+from calibre.ebooks.oeb.display.webview import cleanup_html, load_as_html
+from calibre.utils.short_uuid import uuid4
+
+
+def guess_type(x):
+    return _guess_type(x)[0] or 'application/octet-stream'
+
+
+class NetworkReply(QNetworkReply):
+
+    def __init__(self, parent, request, mime_type, data):
+        QNetworkReply.__init__(self, parent)
+        self.setOpenMode(QNetworkReply.ReadOnly | QNetworkReply.Unbuffered)
+        self.setRequest(request)
+        self.setUrl(request.url())
+        self._aborted = False
+        self.__data = data
+        self.setHeader(QNetworkRequest.ContentTypeHeader, mime_type)
+        self.setHeader(QNetworkRequest.ContentLengthHeader, len(self.__data))
+        QTimer.singleShot(0, self.finalize_reply)
+
+    def bytesAvailable(self):
+        return len(self.__data)
+
+    def isSequential(self):
+        return True
+
+    def abort(self):
+        pass
+
+    def readData(self, maxlen):
+        if maxlen >= len(self.__data):
+            ans, self.__data = self.__data, b''
+            return ans
+        ans, self.__data = self.__data[:maxlen], self.__data[maxlen:]
+        return ans
+    read = readData
+
+    def finalize_reply(self):
+        self.setFinished(True)
+        self.setAttribute(QNetworkRequest.HttpStatusCodeAttribute, 200)
+        self.setAttribute(QNetworkRequest.HttpReasonPhraseAttribute, "Ok")
+        self.metaDataChanged.emit()
+        self.downloadProgress.emit(len(self.__data), len(self.__data))
+        self.readyRead.emit()
+        self.finished.emit()
+
+
+def normpath(p):
+    return os.path.normcase(os.path.abspath(p))
+
+
+class NetworkAccessManager(QNetworkAccessManager):
+
+    load_error = pyqtSignal(object, object)
+
+    def __init__(self, parent=None):
+        QNetworkAccessManager.__init__(self, parent)
+        self.mathjax_prefix = str(uuid4())
+        self.mathjax_base = '%s://%s/%s/' % (FAKE_PROTOCOL, FAKE_HOST, self.mathjax_prefix)
+        self.root = self.orig_root = os.path.dirname(P('viewer/blank.html', allow_user_override=False))
+        self.mime_map, self.single_pages, self.codec_map = {}, set(), {}
+
+    def set_book_data(self, root, spine):
+        self.orig_root = root
+        self.root = os.path.normcase(os.path.abspath(root))
+        self.mime_map, self.single_pages, self.codec_map = {}, set(), {}
+        for p in spine:
+            mt = getattr(p, 'mime_type', None)
+            key = normpath(p)
+            if mt is not None:
+                self.mime_map[key] = mt
+            self.codec_map[key] = getattr(p, 'encoding', 'utf-8')
+            if getattr(p, 'is_single_page', False):
+                self.single_pages.add(key)
+
+    def is_single_page(self, path):
+        if not path:
+            return False
+        key = normpath(path)
+        return key in self.single_pages
+
+    def as_abspath(self, qurl):
+        name = qurl.path()[1:]
+        return os.path.join(self.orig_root, *name.split('/'))
+
+    def as_url(self, abspath):
+        name = os.path.relpath(abspath, self.root).replace('\\', '/')
+        ans = QUrl()
+        ans.setScheme(FAKE_PROTOCOL), ans.setAuthority(FAKE_HOST), ans.setPath('/' + name)
+        return ans
+
+    def guess_type(self, name):
+        mime_type = guess_type(name)
+        mime_type = {
+            # Prevent warning in console about mimetype of fonts
+            'application/vnd.ms-opentype':'application/x-font-ttf',
+            'application/x-font-truetype':'application/x-font-ttf',
+            'application/x-font-opentype':'application/x-font-ttf',
+            'application/x-font-otf':'application/x-font-ttf',
+            'application/font-sfnt': 'application/x-font-ttf',
+        }.get(mime_type, mime_type)
+        return mime_type
+
+    def preprocess_data(self, data, path):
+        mt = self.mime_map.get(path, self.guess_type(path))
+        if mt.lower() in OEB_DOCS:
+            enc = self.codec_map.get(path, 'utf-8')
+            html = data.decode(enc, 'replace')
+            html = cleanup_html(html)
+            data = html.encode('utf-8')
+            if load_as_html(html):
+                mt = 'text/html; charset=utf-8'
+            else:
+                mt = 'application/xhtml+xml; charset=utf-8'
+        return data, mt
+
+    def createRequest(self, operation, request, data):
+        qurl = request.url()
+        if operation == QNetworkAccessManager.GetOperation and qurl.host() == FAKE_HOST:
+            name = qurl.path()[1:]
+            if name.startswith(self.mathjax_prefix):
+                base = normpath(P('viewer/mathjax'))
+                path = normpath(os.path.join(base, name.partition('/')[2]))
+            else:
+                base = self.root
+                path = normpath(os.path.join(self.root, name))
+            if path.startswith(base) and os.path.exists(path):
+                try:
+                    with lopen(path, 'rb') as f:
+                        data = f.read()
+                    data, mime_type = self.preprocess_data(data, path)
+                    return NetworkReply(self, request, mime_type, data)
+                except Exception:
+                    import traceback
+                    self.load_error.emit(name, traceback.format_exc())
+        if DEBUG:
+            prints('URL not found in book: %r' % qurl.toString())
+        return QNetworkAccessManager.createRequest(self, operation, request)

src/calibre/gui2/viewer/footnote.py

@@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
 __license__ = 'GPL v3'
 __copyright__ = '2014, Kovid Goyal <kovid at kovidgoyal.net>'
 
-import json, os
+import json
 from collections import defaultdict
 
 from PyQt5.Qt import (
@@ -16,7 +16,7 @@ from PyQt5.QtWebKitWidgets import QWebView, QWebPage
 from PyQt5.QtWebKit import QWebSettings
 
 from calibre import prints
-from calibre.constants import DEBUG
+from calibre.constants import DEBUG, FAKE_PROTOCOL, FAKE_HOST
 from calibre.ebooks.oeb.display.webview import load_html
 
 
@@ -124,10 +124,10 @@ class Footnotes(object):
             pass
 
     def get_footnote_data(self, a, qurl):
-        current_path = os.path.abspath(unicode(self.view.document.mainFrame().baseUrl().toLocalFile()))
+        current_path = self.view.path()
         if not current_path:
             return  # Not viewing a local file
-        dest_path = self.spine_path(os.path.abspath(unicode(qurl.toLocalFile())))
+        dest_path = self.spine_path(self.view.path(qurl))
         if dest_path is not None:
             if dest_path == current_path:
                 # We deliberately ignore linked to anchors if the destination is
@@ -138,7 +138,7 @@ class Footnotes(object):
             else:
                 linked_to_anchors = {anchor:0 for path, anchor in dest_path.verified_links if path == current_path}
             self.view.document.bridge_value = linked_to_anchors
-            if a.evaluateJavaScript('calibre_extract.is_footnote_link(this)'):
+            if a.evaluateJavaScript('calibre_extract.is_footnote_link(this, "%s://%s")' % (FAKE_PROTOCOL, FAKE_HOST)):
                 if dest_path not in self.known_footnote_targets:
                     self.known_footnote_targets[dest_path] = s = set()
                     for item in self.view.manager.iterator.spine:

src/calibre/gui2/viewer/main.py

@@ -7,7 +7,7 @@ from threading import Thread
 
 from PyQt5.Qt import (
     QApplication, Qt, QIcon, QTimer, QByteArray, QSize, QTime, QObject,
-    QPropertyAnimation, QUrl, QInputDialog, QAction, QModelIndex, pyqtSignal)
+    QPropertyAnimation, QInputDialog, QAction, QModelIndex, pyqtSignal)
 
 from calibre.gui2.viewer.ui import Main as MainWindow
 from calibre.gui2.viewer.toc import TOC
@@ -549,7 +549,7 @@ class EbookViewer(MainWindow):
                     return error_dialog(self, _('No such location'),
                             _('The location pointed to by this item'
                                 ' does not exist.'), det_msg=item.abspath, show=True)
-                url = QUrl.fromLocalFile(item.abspath)
+                url = self.view.as_url(item.abspath)
                 if item.fragment:
                     url.setFragment(item.fragment)
                 self.link_clicked(url)
@@ -664,7 +664,7 @@ class EbookViewer(MainWindow):
         self.history.add(prev_pos)
 
     def link_clicked(self, url):
-        path = os.path.abspath(unicode(url.toLocalFile()))
+        path = self.view.path(url)
         frag = None
         if path in self.iterator.spine:
             self.update_page_number()  # Ensure page number is accurate as it is used for history
@@ -986,6 +986,7 @@ class EbookViewer(MainWindow):
             vh.insert(0, pathtoebook)
             vprefs.set('viewer_open_history', vh[:50])
             self.build_recent_menu()
+            self.view.set_book_data(self.iterator)
 
             self.footnotes_dock.close()
             self.action_table_of_contents.setDisabled(not self.iterator.toc)
@@ -1266,5 +1267,6 @@ def main(args=sys.argv):
         return app.exec_()
     return 0
 
+
 if __name__ == '__main__':
     sys.exit(main())