Sync to trunk.

src/calibre/ebooks/mobi/writer.py

@@ -325,6 +325,8 @@ class MobiWriter(object):
         self._imagemax = imagemax or OTHER_MAX_IMAGE_SIZE
         self._prefer_author_sort = prefer_author_sort
         self._primary_index_record = None
+        self._hasValideNCXEntries = False
+        self._ctoc = ""
         self._HTMLRecords = []
         self._tbSequence = ""
         self._initialIndexRecordFound = False
@@ -364,7 +366,7 @@ class MobiWriter(object):
         self._map_image_names()
         self._generate_text()
         #if INDEXING and not self.opts.no_mobi_index:
-        if INDEXING :
+        if INDEXING and self._hasValidNCXEntries :
             try:
                 self._generate_index()
             except:
@@ -411,6 +413,8 @@ class MobiWriter(object):
 
     def _build_HTMLRecords_Data_List(self):
         # Assemble a HTMLRecordData instance for each HTML record
+        # Return True if valid, False if invalid
+        self._oeb.logger.info('Indexing navPoints ...')
 
         numberOfHTMLRecords = ( self._content_length // RECORD_SIZE ) + 1
 
@@ -423,17 +427,28 @@ class MobiWriter(object):
         toc = self._oeb.toc
         myIndex = 0
         myEndingRecord = 0
+        previousOffset = 0
+        previousLength = 0
+        offset = 0
+        length = 0
         entries = list(toc.iter())[1:]
-        # borrowed from _generate_indxt
+
+
+        # Get offset, length per entry
         for i, child in enumerate(entries):
+
+            '''
             if not child.title or not child.title.strip():
-                child.title = _('Unnamed')
+                child.title = "(none)"
+            '''
             h = child.href
             if h not in self._id_offsets:
                 self._oeb.log.warning('Could not find TOC entry:', child.title)
                 continue
             offset = self._id_offsets[h]
+
             length = None
+
             for sibling in entries[i+1:]:
                 h2 = sibling.href
                 if h2 in self._id_offsets:
@@ -441,9 +456,25 @@ class MobiWriter(object):
                     if offset2 > offset:
                         length = offset2 - offset
                         break
+
             if length is None:
                 length = self._content_length - offset
 
+            # Look a gap between nodes
+            if (i) :
+                if offset != previousOffset + previousLength :
+                    self._oeb.log.warning("\tnodes %d and %d have a gap:" % (i-1, i))
+                    self._oeb.log.warning("\tnode %d offset: 0x%X \t node %d: offset: 0x%X length: 0x%X" % \
+                        (i, offset, i-1, previousOffset, previousLength) )
+                    self._oeb.log.warning('Failed to generate index')
+                    # Zero out self._HTMLRecords, return False
+                    self._HTMLRecords = []
+                    last_name = None
+                    return False
+
+            previousOffset = offset
+            previousLength = length
+
             # Calculate the HTML record for this entry
             myStartingRecord = offset // RECORD_SIZE
 
@@ -461,19 +492,30 @@ class MobiWriter(object):
             # Calculate the ending HTMLRecord of this entry
             myEndingRecord = (offset + length) // RECORD_SIZE
 
-            # Tell the future HTML records about us
             if myEndingRecord > myStartingRecord :
                 interimSpanRecord = myStartingRecord + 1
                 while interimSpanRecord <= myEndingRecord :
                     self._HTMLRecords[interimSpanRecord].continuingNode = myIndex
                     self._HTMLRecords[interimSpanRecord].currentSectionNodeCount = 1
                     interimSpanRecord += 1
+                if self.opts.verbose > 3 :self._oeb.logger.info("\tnode %03d %-15.15s... spans HTML records %03d - %03d \t offset: 0x%06X length: 0x%06X" % \
+                    (myIndex, child.title if child.title.strip() > "" else "(missing)", myStartingRecord, interimSpanRecord, offset, length) )
+            else :
+                if self.opts.verbose > 3 : self._oeb.logger.info("\tnode %03d %-15.15s... spans HTML records %03d - %03d \t offset: 0x%06X length: 0x%06X" % \
+                    (myIndex, child.title if child.title.strip() > "" else "(missing)", myStartingRecord, myStartingRecord, offset, length) )
 
             ctoc_offset = self._ctoc_map[child]
-            last_name = "%04d" % myIndex
+            last_name = "%04X" % myIndex
             myIndex += 1
 
+        # Successfully parsed the entries
+        return True
+
+
     def _build_TBS_Book(self, nrecords, lastrecord):
+        if self.opts.verbose > 3 and False :
+            self._oeb.logger.info("_build_TBS_Book: HTML record %d of %d" % (nrecords, lastrecord) )
+            self._HTMLRecords[nrecords].dumpData(nrecords,self._oeb)
 
         # Variables for trailing byte sequence
         tbsType = 0x00
@@ -533,7 +575,9 @@ class MobiWriter(object):
                 tbSequence += chr(self._HTMLRecords[nrecords].currentSectionNodeCount)
             tbSequence += decint(len(tbSequence) + 1, DECINT_FORWARD)
 
-        # print "record %d: tbsType %d" % (nrecords, tbsType)
+        if self.opts.verbose > 3  and False:
+            self._oeb.logger.info("record %d: tbsType %d" % (nrecords, tbsType) )
+
         self._tbSequence = tbSequence
 
 
@@ -556,16 +600,16 @@ class MobiWriter(object):
             self._oeb.logger.info('Compressing markup content...')
         data, overlap = self._read_text_record(text)
 
-        # GR borrowed this from generate_index
+        # We need entries[] before calling self._build_HTMLRecords_Data_List()
-        # We seem to need it before calling self._build_HTMLRecords_Data_List()
+        self._ctoc = self._generate_ctoc()
-        ctoc = self._generate_ctoc()
 
         # Build the HTMLRecords list so we can assemble the trailing bytes sequences in the following while loop
         toc = self._oeb.toc
         entries = list(toc.iter())[1:]
-        hasNCXEntries = True if len(entries) else False
+        if len(entries) :
-        if hasNCXEntries :
+            self._hasValidNCXEntries = self._build_HTMLRecords_Data_List()
-            self._build_HTMLRecords_Data_List()
+        else :
+            self._hasValidNCXEntries = False
 
         while len(data) > 0:
             if self._compression == PALMDOC:
@@ -579,7 +623,7 @@ class MobiWriter(object):
             running = offset
 
             # Write Trailing Byte Sequence
-            if INDEXING and hasNCXEntries :
+            if INDEXING and self._hasValidNCXEntries:
                 # Dispatch to different TBS generators based upon publication type
                 booktype = 0x101 if self.opts.mobi_periodical else 0x002
                 if booktype == 0x002 :
@@ -639,6 +683,7 @@ class MobiWriter(object):
         self._text_nrecords = nrecords
 
     def _generate_indxt(self, ctoc):
+
         if self.opts.mobi_periodical:
             raise NotImplementedError('Indexing for periodicals not implemented')
         toc = self._oeb.toc
@@ -655,7 +700,7 @@ class MobiWriter(object):
 
             pos = 0xc0 + indxt.tell()
             indices.write(pack('>H', pos))
-            name = "%04d"%count
+            name = "%04X"%count
             indxt.write(chr(len(name)) + name)
             indxt.write(INDXT['chapter'])
             indxt.write(decint(offset, DECINT_FORWARD))
@@ -666,8 +711,6 @@ class MobiWriter(object):
 
         entries = list(toc.iter())[1:]
         for i, child in enumerate(entries):
-            if not child.title or not child.title.strip():
-                continue
             h = child.href
             if h not in self._id_offsets:
                 self._oeb.log.warning('Could not find TOC entry:', child.title)
@@ -686,7 +729,7 @@ class MobiWriter(object):
 
             add_node(child, offset, length, c)
             ctoc_offset = self._ctoc_map[child]
-            last_name = "%04d"%c
+            last_name = "%04X"%c
             c += 1
 
         return align_block(indxt.getvalue()), c, \
@@ -696,9 +739,9 @@ class MobiWriter(object):
     def _generate_index(self):
         self._oeb.log('Generating primary index...')
         self._primary_index_record = None
-        ctoc = self._generate_ctoc()
+
         indxt, indxt_count, indices, last_name = \
-                self._generate_indxt(ctoc)
+                self._generate_indxt(self._ctoc)
         if last_name is None:
             self._oeb.log.warn('Input document has no TOC. No index generated.')
             return
@@ -801,7 +844,7 @@ class MobiWriter(object):
         indx0 = indx0.getvalue()
 
         self._primary_index_record = len(self._records)
-        self._records.extend([indx0, indx1, ctoc])
+        self._records.extend([indx0, indx1, self._ctoc])
 
         # Turn this off for now
         if False:
@@ -862,30 +905,36 @@ class MobiWriter(object):
     def _generate_ctoc(self):
         if self.opts.mobi_periodical:
             raise NotImplementedError('Indexing for periodicals not implemented')
+        self._oeb.logger.info('Generating CTOC ...')
+
         toc = self._oeb.toc
         self._ctoc_map = {}
         self._ctoc_name_map = {}
         self._last_toc_entry = None
         ctoc = StringIO()
 
-        def add_node(node, cls):
+        def add_node(node, cls, title=None):
-            t = node.title
+            t = node.title if title is None else title
-            if not t:
+
-                t = _('Unnamed')
+            if t and t.strip():
-            t = t.strip()
+                t = t.strip()
-            if not isinstance(t, unicode):
+                if not isinstance(t, unicode):
-                t = t.decode('utf-8', 'replace')
+                    t = t.decode('utf-8', 'replace')
-            t = t.encode('utf-8')
+                t = t.encode('utf-8')
-            self._last_toc_entry = t
+                self._last_toc_entry = t
-            self._ctoc_map[node] = ctoc.tell()
+                self._ctoc_map[node] = ctoc.tell()
-            self._ctoc_name_map[node] = t
+                self._ctoc_name_map[node] = t
-            ctoc.write(decint(len(t), DECINT_FORWARD)+t)
+                ctoc.write(decint(len(t), DECINT_FORWARD)+t)
+            else :
+                t = "(none)".encode('utf-8')
+                self._last_toc_entry = t
+                self._ctoc_map[node] = ctoc.tell()
+                self._ctoc_name_map[node] = t
+                ctoc.write(decint(len(t), DECINT_FORWARD)+t)
 
         first = True
         for child in toc.iter():
-            if not child.title:
+            add_node(child, 'chapter')#, title='Title Page' if first else None)
-                child.title = _('Unnamed')
-            add_node(child, 'chapter')
             first = False
 
         return align_block(ctoc.getvalue())

src/calibre/gui2/viewer/documentview.py

@@ -16,6 +16,7 @@ from calibre.gui2.viewer.config_ui import Ui_Dialog
 from calibre.gui2.viewer.js import bookmarks, referencing
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.constants import iswindows
+from calibre import prints
 
 def load_builtin_fonts():
     from calibre.ebooks.lrf.fonts.liberation import LiberationMono_BoldItalic
@@ -124,6 +125,7 @@ class Document(QWebPage):
 
     def __init__(self, *args):
         QWebPage.__init__(self, *args)
+        self.debug_javascript = False
         self.setLinkDelegationPolicy(self.DelegateAllLinks)
         self.scroll_marks = []
         pal = self.palette()
@@ -187,6 +189,20 @@ class Document(QWebPage):
             return unicode(ans.toString())
         return ans
 
+    def javaScriptConsoleMessage(self, msg, lineno, msgid):
+        if self.debug_javascript:
+            prints( 'JS:', msgid, lineno)
+            prints(msg)
+            prints(' ')
+        else:
+            return QWebPage.javaScriptConsoleMessage(self, msg, lineno, msgid)
+
+    def javaScriptAlert(self, frame, msg):
+        if self.debug_javascript:
+            prints(msg)
+        else:
+            return QWebPage.javaScriptAlert(self, frame, msg)
+
     def scroll_by(self, dx=0, dy=0):
         self.mainFrame().scroll(dx, dy)
 
@@ -272,6 +288,7 @@ class DocumentView(QWebView):
 
     def __init__(self, *args):
         QWidget.__init__(self, *args)
+        self.debug_javascript = False
         self.setSizePolicy(QSizePolicy(QSizePolicy.Expanding, QSizePolicy.Expanding))
         self._size_hint = QSize(510, 680)
         self.initial_pos = 0.0

src/calibre/gui2/viewer/js.py

@@ -1,27 +1,5 @@
 bookmarks = '''
 
-function find_enclosing_block(y) {
-    var elements = $("*", document.body);
-    var min = 0;
-    var temp, left, top, elem, width, height, ratio;
-    for (i = 0; i < elements.length; i++) {
-        elem = $(elements[i]);
-        temp = elem.offset();
-        left = temp.left; top = temp.top;
-        width = elem.width(); height = elem.height();
-        if (top > y+50) break;
-        for ( x = 40; x < window.innerWidth; x += 20) {
-            if (x >= left && x <= left+width && y >= top && y <= top+height) {
-                if (min == 0 || min.height() > height) { min = elem; break; }
-            }
-        }
-        if (min != 0 && min.height() < 200) break;
-    }
-    if (y <= 0) return document.body;
-    if (min == 0) { return find_enclosing_block(y-20); }
-    return min;
-}
-
 function selector_in_parent(elem) {
     var num = elem.prevAll().length;
     var sel = " > *:eq("+num+") ";
@@ -38,8 +16,37 @@ function selector(elem) {
     return sel;
 }
 
+function find_closest_enclosing_block(top) {
+    var START = top-1000;
+    var STOP = top;
+    var matches = [];
+    var elem, temp;
+    var width = 1000;
+
+    for (y = START; y < STOP; y += 20) {
+        for ( x = 0; x < width; x += 20) {
+            elem = document.elementFromPoint(x, y);
+            try {
+                elem = $(elem);
+                temp = elem.offset().top
+                matches.push(elem);
+                if (Math.abs(temp - START) < 25) { y = STOP; break}
+            } catch(error) {}
+        }
+    }
+
+    var miny = Math.abs(matches[0].offset().top - START), min_elem = matches[0];
+
+    for (i = 1; i < matches.length; i++) {
+        elem = matches[i];
+        temp = Math.abs(elem.offset().top - START);
+        if ( temp < miny ) { miny = temp; min_elem = elem; }
+    }
+    return min_elem;
+}
+
 function calculate_bookmark(y) {
-    var elem = find_enclosing_block(y);
+    var elem = find_closest_enclosing_block(y);
     var sel = selector(elem);
     var ratio = (y - elem.offset().top)/elem.height();
     if (ratio > 1) { ratio = 1; }

src/calibre/gui2/viewer/main.py

@@ -72,8 +72,6 @@ class ProgressIndicator(QWidget):
         self.status = QLabel(self)
         self.status.setWordWrap(True)
         self.status.setAlignment(Qt.AlignHCenter|Qt.AlignTop)
-        self.status.font().setBold(True)
-        self.status.font().setPointSize(self.font().pointSize()+6)
         self.setVisible(False)
 
     def start(self, msg=''):
@@ -85,7 +83,7 @@ class ProgressIndicator(QWidget):
         self.ml.move(int((self.size().width()-self.ml.size().width())/2.), 0)
         self.status.resize(self.size().width(), self.size().height()-self.ml.size().height()-10)
         self.status.move(0, self.ml.size().height()+10)
-        self.status.setText(msg)
+        self.status.setText('<h1>'+msg+'</h1>')
         self.setVisible(True)
         self.movie.setPaused(False)
 
@@ -140,7 +138,9 @@ class Metadata(QLabel):
 
     def show_opf(self, opf, ext=''):
         mi = MetaInformation(opf)
-        html = '<h2 align="center">%s</h2>%s\n<b>Source Type</b> %s'%(_('Metadata'), u''.join(mi.to_html()), ext)
+        html = '<h2 align="center">%s</h2>%s\n<b>%s:</b> %s'\
+                %(_('Metadata'), u''.join(mi.to_html()),
+                        _('Book format'), ext.upper())
         self.setText(html)
 
     def setVisible(self, x):
@@ -189,7 +189,7 @@ class HelpfulLineEdit(QLineEdit):
 
 class EbookViewer(MainWindow, Ui_EbookViewer):
 
-    def __init__(self, pathtoebook=None):
+    def __init__(self, pathtoebook=None, debug_javascript=False):
         MainWindow.__init__(self, None)
         self.setupUi(self)
         self.iterator          = None
@@ -219,6 +219,7 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
         self.search.setToolTip(_('Search for text in book'))
         self.tool_bar2.insertWidget(self.action_find_next, self.search)
         self.view.set_manager(self)
+        self.view.document.debug_javascript = debug_javascript
         self.pi = ProgressIndicator(self)
         self.toc.setVisible(False)
         self.action_quit = QAction(self)
@@ -630,6 +631,8 @@ def config(defaults=None):
                      'front when started.'))
     c.add_opt('remember_window_size', default=False,
         help=_('Remember last used window size'))
+    c.add_opt('debug_javascript', ['--debug-javascript'], default=False,
+        help=_('Print javascript alert and console messages to the console'))
 
     return c
 
@@ -651,7 +654,8 @@ def main(args=sys.argv):
         app.setWindowIcon(QIcon(':/images/viewer.svg'))
         QApplication.setOrganizationName(ORG_NAME)
         QApplication.setApplicationName(APP_UID)
-        main = EbookViewer(args[1] if len(args) > 1 else None)
+        main = EbookViewer(args[1] if len(args) > 1 else None,
+                debug_javascript=opts.debug_javascript)
         sys.excepthook = main.unhandled_exception
         main.show()
         if opts.raise_window:

src/calibre/gui2/widgets.py

@@ -34,8 +34,6 @@ class ProgressIndicator(QWidget):
         self.status = QLabel(self)
         self.status.setWordWrap(True)
         self.status.setAlignment(Qt.AlignHCenter|Qt.AlignTop)
-        self.status.font().setBold(True)
-        self.status.font().setPointSize(self.font().pointSize()+6)
         self.setVisible(False)
 
     def start(self, msg=''):
@@ -47,7 +45,7 @@ class ProgressIndicator(QWidget):
         self.ml.move(int((self.size().width()-self.ml.size().width())/2.), 0)
         self.status.resize(self.size().width(), self.size().height()-self.ml.size().height()-10)
         self.status.move(0, self.ml.size().height()+10)
-        self.status.setText(msg)
+        self.status.setText('<h1>'+msg+'</h1>')
         self.setVisible(True)
         self.movie.setPaused(False)