Edit Book: Run syntax highlighting in the background so that it does not affect responsiveness of the user interface

Technically, the syntax highlighting now does idle processing, that is
it yields after every line of text is highlighted to allow the rest of
the UI to respond to user interaction. This gives an excellent
illusion of running int he background, except for pathological files
with very long lines (such as those that have all their markup on a
single line).

src/calibre/gui2/tweak_book/editor/smart/html.py

@@ -238,6 +238,7 @@ class HTMLSmarts(NullSmarts):
         return ans
 
     def rename_block_tag(self, editor, new_name):
+        editor.highlighter.join()
         c = editor.textCursor()
         block, offset = c.block(), c.positionInBlock()
         tag = None
@@ -268,6 +269,7 @@ class HTMLSmarts(NullSmarts):
                 'No suitable block level tag was found to rename'), show=True)
 
     def get_smart_selection(self, editor, update=True):
+        editor.highlighter.join()
         cursor = editor.textCursor()
         if not cursor.hasSelection():
             return ''
@@ -288,6 +290,7 @@ class HTMLSmarts(NullSmarts):
         return editor.selected_text_from_cursor(cursor)
 
     def insert_hyperlink(self, editor, target, text):
+        editor.highlighter.join()
         c = editor.textCursor()
         if c.hasSelection():
             c.insertText('')  # delete any existing selected text
@@ -301,6 +304,7 @@ class HTMLSmarts(NullSmarts):
         editor.setTextCursor(c)
 
     def insert_tag(self, editor, name):
+        editor.highlighter.join()
         name = name.lstrip()
         text = self.get_smart_selection(editor, update=True)
         c = editor.textCursor()
@@ -314,6 +318,7 @@ class HTMLSmarts(NullSmarts):
     def verify_for_spellcheck(self, cursor, highlighter):
         # Return True iff the cursor is in a location where spelling is
         # checked (inside a tag or inside a checked attribute)
+        highlighter.join()
         block = cursor.block()
         start_pos = cursor.anchor() - block.position()
         end_pos = cursor.position() - block.position()
@@ -393,6 +398,7 @@ class HTMLSmarts(NullSmarts):
     def get_inner_HTML(self, editor):
         ''' Select the inner HTML of the current tag. Return a cursor with the
         inner HTML selected or None. '''
+        editor.highlighter.join()
         c = editor.textCursor()
         block = c.block()
         offset = c.position() - block.position()

src/calibre/gui2/tweak_book/editor/syntax/base.py

@@ -7,10 +7,10 @@ __license__ = 'GPL v3'
 __copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
 
 import sys
-from collections import defaultdict
+from collections import defaultdict, deque
 
 from PyQt4.Qt import (
-    QTextCursor, pyqtSlot, QTextBlockUserData, QTextLayout)
+    QTextCursor, pyqtSlot, QTextBlockUserData, QTextLayout, QTimer)
 
 from ..themes import highlight_to_char_format
 from calibre.gui2.tweak_book.widgets import BusyCursor
@@ -71,6 +71,12 @@ class SyntaxHighlighter(object):
 
     def __init__(self):
         self.doc = None
+        self.requests = deque()
+        self.ignore_requests = False
+
+    @property
+    def has_requests(self):
+        return bool(self.requests)
 
     def apply_theme(self, theme):
         self.theme = {k:highlight_to_char_format(v) for k, v in theme.iteritems()}
@@ -117,24 +123,77 @@ class SyntaxHighlighter(object):
     @pyqtSlot(int, int, int)
     def reformat_blocks(self, position, removed, added):
         doc = self.doc
-        if doc is None or not hasattr(self, 'state_map'):
+        if doc is None or self.ignore_requests or not hasattr(self, 'state_map'):
             return
+
+        block = doc.findBlock(position)
+        if not block.isValid():
+            return
+        start_cursor = QTextCursor(block)
         last_block = doc.findBlock(position + added + (1 if removed > 0 else 0))
         if not last_block.isValid():
             last_block = doc.lastBlock()
-        end_pos = last_block.position() + last_block.length()
+        end_cursor = QTextCursor(last_block)
-        force_next_highlight = False
+        end_cursor.movePosition(end_cursor.EndOfBlock)
+        self.requests.append((start_cursor, end_cursor))
+        QTimer.singleShot(0, self.do_one_block)
 
-        doc.contentsChange.disconnect(self.reformat_blocks)
+    def do_one_block(self):
         try:
-            block = doc.findBlock(position)
+            start_cursor, end_cursor = self.requests[0]
-            while block.isValid() and (block.position() < end_pos or force_next_highlight):
+        except IndexError:
-                formats, force_next_highlight = self.parse_single_block(block)
+            return
-                self.apply_format_changes(block, formats)
+        self.ignore_requests = True
-                doc.markContentsDirty(block.position(), block.length())
+        try:
-                block = block.next()
+            block = start_cursor.block()
+            if not block.isValid():
+                self.requests.popleft()
+                return
+            formats, force_next_highlight = self.parse_single_block(block)
+            self.apply_format_changes(block, formats)
+            try:
+                self.doc.markContentsDirty(block.position(), block.length())
+            except AttributeError:
+                self.requests.clear()
+                return
+            ok = start_cursor.movePosition(start_cursor.NextBlock)
+            if not ok:
+                self.requests.popleft()
+                return
+            next_block = start_cursor.block()
+            if next_block.position() > end_cursor.position():
+                if force_next_highlight:
+                    end_cursor.setPosition(next_block.position() + 1)
+                else:
+                    self.requests.popleft()
+                return
         finally:
-            doc.contentsChange.connect(self.reformat_blocks)
+            self.ignore_requests = False
+            QTimer.singleShot(0, self.do_one_block)
+
+    def join(self):
+        ''' Blocks until all pending highlighting requests are handled '''
+        doc = self.doc
+        if doc is None:
+            self.requests.clear()
+            return
+        self.ignore_requests = True
+        try:
+            while self.requests:
+                start_cursor, end_cursor = self.requests.popleft()
+                block = start_cursor.block()
+                last_block = end_cursor.block()
+                if not last_block.isValid():
+                    last_block = doc.lastBlock()
+                end_pos = last_block.position() + last_block.length()
+                force_next_highlight = False
+                while block.isValid() and (force_next_highlight or block.position() < end_pos):
+                    formats, force_next_highlight = self.parse_single_block(block)
+                    self.apply_format_changes(block, formats)
+                    doc.markContentsDirty(block.position(), block.length())
+                    block = block.next()
+        finally:
+            self.ignore_requests = False
 
     def parse_single_block(self, block):
         ud, is_new_ud = self.get_user_data(block)

src/calibre/gui2/tweak_book/editor/syntax/html.py

@@ -554,9 +554,11 @@ def profile():
     theme = get_theme(tprefs['editor_theme'])
     h.apply_theme(theme)
     h.set_document(doc)
+    h.join()
     import cProfile
     print ('Running profile on', sys.argv[-2])
-    cProfile.runctx('h.rehighlight()', {}, {'h':h}, sys.argv[-1])
+    h.rehighlight()
+    cProfile.runctx('h.join()', {}, {'h':h}, sys.argv[-1])
     print ('Stats saved to:', sys.argv[-1])
     del h
     del doc

src/calibre/gui2/tweak_book/editor/text.py

@@ -265,7 +265,7 @@ class TextEdit(PlainTextEdit):
             sel.append(self.current_cursor_line)
         if self.current_search_mark is not None:
             sel.append(self.current_search_mark)
-        if instant:
+        if instant and not self.highlighter.has_requests:
             sel.extend(self.smarts.get_extra_selections(self))
         else:
             self.smarts_highlight_timer.start()