Diff tool: Add syntax highlighting for many other filetypes

Now I can use the diff tool as my primary diff tool in all contexts.
Syntax highlighting for non-core filetypes is courtesy of pygments.

src/calibre/gui2/tweak_book/diff/highlight.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__ = 'GPL v3'
+__copyright__ = '2014, Kovid Goyal <kovid at kovidgoyal.net>'
+
+import os
+
+from PyQt4.Qt import QTextDocument, QTextCursor, QTextCharFormat, QPlainTextDocumentLayout
+
+from calibre.gui2.tweak_book import tprefs
+from calibre.gui2.tweak_book.editor.text import get_highlighter as calibre_highlighter, SyntaxHighlighter
+from calibre.gui2.tweak_book.editor.themes import THEMES, default_theme, highlight_to_char_format
+
+def get_theme():
+    theme = THEMES.get(tprefs['editor_theme'], None)
+    if theme is None:
+        theme = THEMES[default_theme()]
+    return theme
+
+NULL_FMT = QTextCharFormat()
+
+class QtHighlighter(QTextDocument):
+
+    def __init__(self, parent, text, hlclass):
+        QTextDocument.__init__(self, parent)
+        self.l = QPlainTextDocumentLayout(self)
+        self.setDocumentLayout(self.l)
+        self.highlighter = hlclass(self)
+        self.highlighter.apply_theme(get_theme())
+        self.highlighter.setDocument(self)
+        self.setPlainText(text)
+
+    def copy_lines(self, lo, hi, cursor):
+        ''' Copy specified lines from the syntax highlighted buffer into the
+        destination cursor, preserving all formatting created by the syntax
+        highlighter. '''
+        num = hi - lo
+        if num > 0:
+            block = self.findBlockByNumber(lo)
+            while num > 0:
+                num -= 1
+                cursor.insertText(block.text())
+                dest_block = cursor.block()
+                c = QTextCursor(dest_block)
+                for af in block.layout().additionalFormats():
+                    start = dest_block.position() + af.start
+                    c.setPosition(start), c.setPosition(start + af.length, c.KeepAnchor)
+                    c.setCharFormat(af.format)
+                cursor.insertBlock()
+                cursor.setCharFormat(NULL_FMT)
+                block = block.next()
+
+class NullHighlighter(object):
+
+    def __init__(self, text):
+        self.lines = text.splitlines()
+
+    def copy_lines(self, lo, hi, cursor):
+        for i in xrange(lo, hi):
+            cursor.insertText(self.lines[i])
+            cursor.insertBlock()
+
+def pygments_lexer(filename):
+    try:
+        from pygments.lexers import get_lexer_for_filename
+        from pygments.util import ClassNotFound
+    except ImportError:
+        return None
+    try:
+        return get_lexer_for_filename(filename)
+    except ClassNotFound:
+        if filename.lower().endswith('.recipe'):
+            return get_lexer_for_filename('a.py')
+        return None
+
+_pyg_map = None
+def pygments_map():
+    global _pyg_map
+    if _pyg_map is None:
+        from pygments.token import Token
+        _pyg_map = {
+            Token: None,
+            Token.Comment: 'Comment',
+            Token.Comment.Preproc: 'PreProc',
+            Token.String: 'String',
+            Token.Number: 'Number',
+            Token.Keyword.Type: 'Type',
+            Token.Keyword: 'Keyword',
+            Token.Name.Builtin: 'Identifier',
+            Token.Operator: 'Statement',
+            Token.Name.Function: 'Function',
+            Token.Literal: 'Constant',
+            Token.Error: 'Error',
+        }
+    return _pyg_map
+
+def format_for_token(theme, cache, token):
+    try:
+        return cache[token]
+    except KeyError:
+        pass
+    pmap = pygments_map()
+    while token is not None:
+        try:
+            name = pmap[token]
+        except KeyError:
+            token = token.parent
+            continue
+        cache[token] = ans = theme[name]
+        return ans
+    cache[token] = ans = NULL_FMT
+    return ans
+
+class PygmentsHighlighter(object):
+
+    def __init__(self, text, lexer):
+        theme, cache = get_theme(), {}
+        theme = {k:highlight_to_char_format(v) for k, v in theme.iteritems()}
+        theme[None] = NULL_FMT
+        def fmt(token):
+            return format_for_token(theme, cache, token)
+
+        from pygments import lex
+        lines = self.lines = [[]]
+        current_line = lines[0]
+        for token, val in lex(text, lexer):
+            for v in val.splitlines(True):
+                current_line.append((fmt(token), v))
+                if v[-1] in '\n\r':
+                    lines.append([])
+                    current_line = lines[-1]
+                    continue
+
+    def copy_lines(self, lo, hi, cursor):
+        for i in xrange(lo, hi):
+            for fmt, text in self.lines[i]:
+                cursor.insertText(text, fmt)
+            cursor.setCharFormat(NULL_FMT)
+
+def get_highlighter(parent, text, syntax):
+    hlclass = calibre_highlighter(syntax)
+    if hlclass is SyntaxHighlighter:
+        filename = os.path.basename(parent.headers[-1][1])
+        lexer = pygments_lexer(filename)
+        if lexer is None:
+            return NullHighlighter(text)
+        return PygmentsHighlighter(text, lexer)
+    return QtHighlighter(parent, text, hlclass)

src/calibre/gui2/tweak_book/diff/view.py

@@ -16,7 +16,7 @@ from future_builtins import zip
 
 import regex
 from PyQt4.Qt import (
-    QSplitter, QApplication, QPlainTextDocumentLayout, QTextDocument, QTimer,
+    QSplitter, QApplication, QTimer,
     QTextCursor, QTextCharFormat, Qt, QRect, QPainter, QPalette, QPen, QBrush,
     QColor, QTextLayout, QCursor, QFont, QSplitterHandle, QPainterPath,
     QHBoxLayout, QWidget, QScrollBar, QEventLoop, pyqtSignal, QImage, QPixmap,
@@ -25,9 +25,10 @@ from PyQt4.Qt import (
 from calibre import human_readable, fit_image
 from calibre.gui2 import info_dialog
 from calibre.gui2.tweak_book import tprefs
-from calibre.gui2.tweak_book.editor.text import PlainTextEdit, get_highlighter, default_font_family, LineNumbers
+from calibre.gui2.tweak_book.editor.text import PlainTextEdit, default_font_family, LineNumbers
-from calibre.gui2.tweak_book.editor.themes import THEMES, default_theme, theme_color
+from calibre.gui2.tweak_book.editor.themes import theme_color
 from calibre.gui2.tweak_book.diff import get_sequence_matcher
+from calibre.gui2.tweak_book.diff.highlight import get_theme, get_highlighter
 
 Change = namedtuple('Change', 'ltop lbot rtop rbot kind')
 
@@ -39,12 +40,6 @@ class BusyCursor(object):
     def __exit__(self, *args):
         QApplication.restoreOverrideCursor()
 
-def get_theme():
-    theme = THEMES.get(tprefs['editor_theme'], None)
-    if theme is None:
-        theme = THEMES[default_theme()]
-    return theme
-
 def beautify_text(raw, syntax):
     from lxml import etree
     from calibre.ebooks.oeb.polish.parsing import parse
@@ -392,38 +387,6 @@ class TextBrowser(PlainTextEdit):  # {{{
 
 # }}}
 
-class Highlight(QTextDocument):  # {{{
-
-    def __init__(self, parent, text, syntax):
-        QTextDocument.__init__(self, parent)
-        self.l = QPlainTextDocumentLayout(self)
-        self.setDocumentLayout(self.l)
-        self.highlighter = get_highlighter(syntax)(self)
-        self.highlighter.apply_theme(get_theme())
-        self.highlighter.setDocument(self)
-        self.setPlainText(text)
-
-    def copy_lines(self, lo, hi, cursor):
-        ''' Copy specified lines from the syntax highlighted buffer into the
-        destination cursor, preserving all formatting created by the syntax
-        highlighter. '''
-        num = hi - lo
-        if num > 0:
-            block = self.findBlockByNumber(lo)
-            while num > 0:
-                num -= 1
-                cursor.insertText(block.text())
-                dest_block = cursor.block()
-                c = QTextCursor(dest_block)
-                for af in block.layout().additionalFormats():
-                    start = dest_block.position() + af.start
-                    c.setPosition(start), c.setPosition(start + af.length, c.KeepAnchor)
-                    c.setCharFormat(af.format)
-                cursor.insertBlock()
-                cursor.setCharFormat(QTextCharFormat())
-                block = block.next()
-# }}}
-
 class DiffSplitHandle(QSplitterHandle):  # {{{
 
     WIDTH = 30  # px
@@ -697,7 +660,7 @@ class DiffSplit(QSplitter):  # {{{
 
         cruncher = get_sequence_matcher()(None, left_lines, right_lines)
 
-        left_highlight, right_highlight = Highlight(self, left_text, syntax), Highlight(self, right_text, syntax)
+        left_highlight, right_highlight = get_highlighter(self.left, left_text, syntax), get_highlighter(self.right, right_text, syntax)
         cl, cr = self.left_cursor, self.right_cursor = self.left.textCursor(), self.right.textCursor()
         cl.beginEditBlock(), cr.beginEditBlock()
         cl.movePosition(cl.End), cr.movePosition(cr.End)