Refactor the pygments util function into their own module

Ensures that cdiff does not fail if pygments is unavailable

src/calibre/gui2/tweak_book/diff/highlight.py

@@ -13,7 +13,7 @@ from PyQt5.Qt import QTextDocument, QTextCursor, QPlainTextDocumentLayout
 from calibre.gui2.tweak_book import tprefs
 from calibre.gui2.tweak_book.editor.text import get_highlighter as calibre_highlighter, SyntaxHighlighter
 from calibre.gui2.tweak_book.editor.themes import get_theme, highlight_to_char_format
-from calibre.gui2.tweak_book.editor.syntax.python import format_for_token, NULL_FMT
+from calibre.gui2.tweak_book.editor.syntax.utils import format_for_pygments_token, NULL_FMT
 
 class QtHighlighter(QTextDocument):
 
@@ -78,7 +78,7 @@ class PygmentsHighlighter(object):
         theme = {k:highlight_to_char_format(v) for k, v in theme.iteritems()}
         theme[None] = NULL_FMT
         def fmt(token):
-            return format_for_token(theme, cache, token)
+            return format_for_pygments_token(theme, cache, token)
 
         from pygments import lex
         lines = self.lines = [[]]

src/calibre/gui2/tweak_book/editor/syntax/python.py

@@ -8,16 +8,16 @@ __copyright__ = '2014, Kovid Goyal <kovid at kovidgoyal.net>'
 
 from functools import partial
 
-from PyQt5.Qt import QTextCharFormat, QTextBlockUserData
+from PyQt5.Qt import QTextBlockUserData
 
 from calibre.gui2.tweak_book.editor.syntax.base import SyntaxHighlighter
+from calibre.gui2.tweak_book.editor.syntax.utils import format_for_pygments_token, NULL_FMT
 
 from pygments.lexer import _TokenType, Text, Error
 from pygments.lexers import PythonLexer
 
 NORMAL = 0
 
-NULL_FMT = QTextCharFormat()
 
 class QtLexer(PythonLexer):
 
@@ -82,43 +82,6 @@ class QtLexer(PythonLexer):
 
 lexer = QtLexer()
 
-_pyg_map = None
-def pygments_map():
-    global _pyg_map
-    if _pyg_map is None:
-        from pygments.token import Token
-        _pyg_map = {
-            Token: None,
-            Token.Comment: 'Comment', Token.Comment.Preproc: 'PreProc',
-            Token.String: 'String',
-            Token.Number: 'Number',
-            Token.Keyword.Type: 'Type',
-            Token.Keyword: 'Keyword',
-            Token.Name.Builtin: 'Identifier',
-            Token.Operator: 'Statement',
-            Token.Name.Function: 'Function',
-            Token.Literal: 'Constant',
-            Token.Error: 'Error',
-        }
-    return _pyg_map
-
-def format_for_token(theme, cache, token):
-    try:
-        return cache[token]
-    except KeyError:
-        pass
-    pmap = pygments_map()
-    while token is not None:
-        try:
-            name = pmap[token]
-        except KeyError:
-            token = token.parent
-            continue
-        cache[token] = ans = theme[name]
-        return ans
-    cache[token] = ans = NULL_FMT
-    return ans
-
 class State(object):
 
     __slots__ = ('parse', 'pygments_stack')
@@ -177,7 +140,7 @@ def create_formats(highlighter):
     cache = {}
     theme = highlighter.theme.copy()
     theme[None] = NULL_FMT
-    return partial(format_for_token, theme, cache)
+    return partial(format_for_pygments_token, theme, cache)
 
 class PythonHighlighter(SyntaxHighlighter):
 

src/calibre/gui2/tweak_book/editor/syntax/utils.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__ = 'GPL v3'
+__copyright__ = '2014, Kovid Goyal <kovid at kovidgoyal.net>'
+
+from PyQt5.Qt import QTextCharFormat
+
+NULL_FMT = QTextCharFormat()
+
+_pyg_map = None
+
+def pygments_map():
+    global _pyg_map
+    if _pyg_map is None:
+        from pygments.token import Token
+        _pyg_map = {
+            Token: None,
+            Token.Comment: 'Comment', Token.Comment.Preproc: 'PreProc',
+            Token.String: 'String',
+            Token.Number: 'Number',
+            Token.Keyword.Type: 'Type',
+            Token.Keyword: 'Keyword',
+            Token.Name.Builtin: 'Identifier',
+            Token.Operator: 'Statement',
+            Token.Name.Function: 'Function',
+            Token.Literal: 'Constant',
+            Token.Error: 'Error',
+        }
+    return _pyg_map
+
+def format_for_pygments_token(theme, cache, token):
+    try:
+        return cache[token]
+    except KeyError:
+        pass
+    pmap = pygments_map()
+    while token is not None:
+        try:
+            name = pmap[token]
+        except KeyError:
+            token = token.parent
+            continue
+        cache[token] = ans = theme[name]
+        return ans
+    cache[token] = ans = NULL_FMT
+    return ans
+
+
+