From a66904da25d2821de76852ca521949a4a1e28d08 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Tue, 29 Oct 2013 22:10:52 +0530
Subject: [PATCH] Start work on the editor component for Tweak Book

---
 src/calibre/gui2/tweak_book/__init__.py       |   4 +
 .../gui2/tweak_book/editor/__init__.py        |  10 +
 .../gui2/tweak_book/editor/syntax/__init__.py |  10 +
 .../gui2/tweak_book/editor/syntax/base.py     |  28 +++
 .../gui2/tweak_book/editor/syntax/html.py     | 238 ++++++++++++++++++
 src/calibre/gui2/tweak_book/editor/text.py    | 131 ++++++++++
 src/calibre/gui2/tweak_book/editor/themes.py  | 131 ++++++++++
 7 files changed, 552 insertions(+)
 create mode 100644 src/calibre/gui2/tweak_book/editor/__init__.py
 create mode 100644 src/calibre/gui2/tweak_book/editor/syntax/__init__.py
 create mode 100644 src/calibre/gui2/tweak_book/editor/syntax/base.py
 create mode 100644 src/calibre/gui2/tweak_book/editor/syntax/html.py
 create mode 100644 src/calibre/gui2/tweak_book/editor/text.py
 create mode 100644 src/calibre/gui2/tweak_book/editor/themes.py

diff --git a/src/calibre/gui2/tweak_book/__init__.py b/src/calibre/gui2/tweak_book/__init__.py
index 63519c9bea..0af48bb02e 100644
--- a/src/calibre/gui2/tweak_book/__init__.py
+++ b/src/calibre/gui2/tweak_book/__init__.py
@@ -9,6 +9,10 @@ __copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
 from calibre.utils.config import JSONConfig
 tprefs = JSONConfig('tweak_book_gui')
 
+tprefs.defaults['editor_theme'] = None
+tprefs.defaults['editor_font_family'] = None
+tprefs.defaults['editor_font_size'] = 12
+
 _current_container = None
 
 def current_container():
diff --git a/src/calibre/gui2/tweak_book/editor/__init__.py b/src/calibre/gui2/tweak_book/editor/__init__.py
new file mode 100644
index 0000000000..2dfbaa2fb8
--- /dev/null
+++ b/src/calibre/gui2/tweak_book/editor/__init__.py
@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__ = 'GPL v3'
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+
+
+
diff --git a/src/calibre/gui2/tweak_book/editor/syntax/__init__.py b/src/calibre/gui2/tweak_book/editor/syntax/__init__.py
new file mode 100644
index 0000000000..2dfbaa2fb8
--- /dev/null
+++ b/src/calibre/gui2/tweak_book/editor/syntax/__init__.py
@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__ = 'GPL v3'
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+
+
+
diff --git a/src/calibre/gui2/tweak_book/editor/syntax/base.py b/src/calibre/gui2/tweak_book/editor/syntax/base.py
new file mode 100644
index 0000000000..c0d3ca5541
--- /dev/null
+++ b/src/calibre/gui2/tweak_book/editor/syntax/base.py
@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__ = 'GPL v3'
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+
+from PyQt4.Qt import (QSyntaxHighlighter, QApplication, QCursor, Qt)
+
+from ..themes import highlight_to_char_format
+
+class SyntaxHighlighter(QSyntaxHighlighter):
+
+    def rehighlight(self):
+        self.outlineexplorer_data = {}
+        QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
+        QSyntaxHighlighter.rehighlight(self)
+        QApplication.restoreOverrideCursor()
+
+    def apply_theme(self, theme):
+        self.theme = {k:highlight_to_char_format(v) for k, v in theme.iteritems()}
+        self.create_formats()
+        self.rehighlight()
+
+    def create_formats(self):
+        pass
+
diff --git a/src/calibre/gui2/tweak_book/editor/syntax/html.py b/src/calibre/gui2/tweak_book/editor/syntax/html.py
new file mode 100644
index 0000000000..cbae374885
--- /dev/null
+++ b/src/calibre/gui2/tweak_book/editor/syntax/html.py
@@ -0,0 +1,238 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__ = 'GPL v3'
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+
+import re
+
+from PyQt4.Qt import (QTextCharFormat)
+
+from .base import SyntaxHighlighter
+from html5lib.constants import cdataElements, rcdataElements
+
+entity_pat = re.compile(r'&#{0,1}[a-zA-Z0-9]{1,8};')
+tag_name_pat = re.compile(r'/{0,1}[a-zA-Z0-9:]+')
+space_chars = ' \t\r\n\u000c'
+attribute_name_pat = re.compile(r'''[^%s"'/>=]+''' % space_chars)
+self_closing_pat = re.compile(r'/\s*>')
+unquoted_val_pat = re.compile(r'''[^%s'"=<>`]+''' % space_chars)
+
+class State(object):
+
+    ''' Store the parsing state, a stack of bold and italic formatting and the
+    last seen open tag, all in a single integer, so that it can be used with.
+    This assumes an int is at least 32 bits.'''
+
+    NORMAL = 0
+    IN_OPENING_TAG = 1
+    IN_CLOSING_TAG = 2
+    IN_COMMENT = 3
+    IN_PI = 4
+    IN_DOCTYPE = 5
+    ATTRIBUTE_NAME = 6
+    ATTRIBUTE_VALUE = 7
+    SQ_VAL = 8
+    DQ_VAL = 9
+
+    TAGS = {x:i+1 for i, x in enumerate(cdataElements | rcdataElements | {'b', 'em', 'i', 'string', 'a'} | {'h%d' % d for d in range(1, 7)})}
+    TAGS_RMAP = {v:k for k, v in TAGS.iteritems()}
+    UNKNOWN_TAG = '___'
+
+    def __init__(self, num):
+        self.parse  = num & 0b1111
+        self.bold   = (num >> 4) & 0b11111111
+        self.italic = (num >> 12) & 0b11111111
+        self.tag    = self.TAGS_RMAP.get(num >> 20, self.UNKNOWN_TAG)
+
+    @property
+    def value(self):
+        tag = self.TAGS.get(self.tag.lower(), 0)
+        return (self.parse & 0b1111) | ((self.bold & 0b11111111) << 4) | ((self.italic & 0b11111111) << 12) | (tag << 20)
+
+def err(formats, msg):
+    ans = QTextCharFormat(formats['error'])
+    ans.setToolTip(msg)
+    return ans
+
+def normal(state, text, i, formats):
+    ' The normal state in between tags '
+    ch = text[i]
+    if ch == '<':
+        if text[i:i+4] == '<!--':
+            state.parse, fmt = state.IN_COMMENT, formats['comment']
+            return [(4, fmt)]
+
+        if text[i:i+2] == '<?':
+            state.parse, fmt = state.IN_PI, formats['special']
+            return [(2, fmt)]
+
+        if text[i:i+2] == '<!' and text[i+2:].lstrip().lower().startswith('doctype'):
+            state.parse, fmt = state.IN_DOCTYPE, formats['special']
+            return [(2, fmt)]
+
+        m = tag_name_pat.match(text, i + 1)
+        if m is None:
+            return [(1, err(formats, _('An unescaped < is not allowed. Replace it with &lt;')))]
+
+        name = m.group()
+        closing = name.startswith('/')
+        state.parse = state.IN_CLOSING_TAG if closing else state.IN_OPENING_TAG
+        state.tag = name[1:] if closing else name
+        num = 2 if closing else 1
+        return [(num, formats['end_tag' if closing else 'tag']), (len(state.tag), formats['tag_name'])]
+
+    if ch == '&':
+        m = entity_pat.match(text, i)
+        if m is None:
+            return [(1, err(formats, _('An unescaped ampersand is not allowed. Replace it with &amp;')))]
+        return [(len(m.group()), formats['entity'])]
+
+    if ch == '>':
+        return [(1, err(formats, _('An unescaped > is not allowed. Replace it with &gt;')))]
+
+    return [(1, None)]
+
+def opening_tag(state, text, i, formats):
+    'An opening tag, like <a>'
+    ch = text[i]
+    if ch in space_chars:
+        return [(1, None)]
+    if ch == '/':
+        m = self_closing_pat.match(text, i)
+        if m is None:
+            return [(1, err(formats, _('/ not allowed except at the end of the tag')))]
+        state.parse = state.NORMAL
+        state.tag = State.UNKNOWN_TAG
+        return [(len(m.group()), formats['tag'])]
+    if ch == '>':
+        state.parse = state.NORMAL
+        state.tag = State.UNKNOWN_TAG
+        return [(1, formats['tag'])]
+    m = attribute_name_pat.match(text, i)
+    if m is None:
+        return [(1, err(formats, _('Unknown character')))]
+    state.parse = state.ATTRIBUTE_NAME
+    num = len(m.group())
+    return [(num, formats['attr'])]
+
+def attribute_name(state, text, i, formats):
+    ' After attribute name '
+    ch = text[i]
+    if ch in space_chars:
+        return [(1, None)]
+    if ch == '=':
+        state.parse = State.ATTRIBUTE_VALUE
+        return [(1, formats['attr'])]
+    state.parse = State.IN_OPENING_TAG
+    return [(-1, None)]
+
+def attribute_value(state, text, i, formats):
+    ' After attribute = '
+    ch = text[i]
+    if ch in space_chars:
+        return [(1, None)]
+    if ch in {'"', "'"}:
+        state.parse = State.SQ_VAL if ch == "'" else State.DQ_VAL
+        return [(1, formats['string'])]
+    m = unquoted_val_pat.match(text, i)
+    state.parse = State.IN_OPENING_TAG
+    return [(len(m.group()), formats['string'])]
+
+def quoted_val(state, text, i, formats):
+    ' A quoted attribute value '
+    quote = '"' if state.parse == State.DQ_VAL else "'"
+    pos = text.find(quote, i)
+    if pos == -1:
+        num = len(text) - i
+    else:
+        num = pos - i + 1
+        state.parse = State.IN_OPENING_TAG
+    return [(num, formats['string'])]
+
+def closing_tag(state, text, i, formats):
+    ' A closing tag like </a> '
+    ch = text[i]
+    if ch in space_chars:
+        return [(1, None)]
+    pos = text.find('>', i)
+    if pos == -1:
+        return [(len(text) - i, err(formats, _('A closing tag must contain only the tag name and nothing else')))]
+    state.parse = state.NORMAL
+    num = pos - i + 1
+    ans = [(1, formats['end_tag'])]
+    if num > 1:
+        ans.insert(0, (num - 1, err(formats, _('A closing tag must contain only the tag name and nothing else'))))
+    return ans
+
+def in_comment(state, text, i, formats):
+    ' Comment, processing instruction or doctype '
+    end = {state.IN_COMMENT:'-->', state.IN_PI:'?>'}.get(state.parse, '>')
+    pos = text.find(end, i+1)
+    fmt = formats['comment' if state.parse == state.IN_COMMENT else 'special']
+    if pos == -1:
+        num = len(text) - i
+    else:
+        num = pos - i + len(end)
+        state.parse = state.NORMAL
+    return [(num, fmt)]
+
+state_map = {
+    State.NORMAL:normal,
+    State.IN_OPENING_TAG: opening_tag,
+    State.IN_CLOSING_TAG: closing_tag,
+    State.ATTRIBUTE_NAME: attribute_name,
+    State.ATTRIBUTE_VALUE: attribute_value,
+}
+
+for x in (State.IN_COMMENT, State.IN_PI, State.IN_DOCTYPE):
+    state_map[x] = in_comment
+
+for x in (State.SQ_VAL, State.DQ_VAL):
+    state_map[x] = quoted_val
+
+class HTMLHighlighter(SyntaxHighlighter):
+
+    def __init__(self, parent):
+        SyntaxHighlighter.__init__(self, parent)
+
+    def create_formats(self):
+        t = self.theme
+        self.formats = {
+            'normal': QTextCharFormat(),
+            'tag': t['Function'],
+            'end_tag': t['Identifier'],
+            'attr': t['Type'],
+            'tag_name' : t['Statement'],
+            'entity': t['Special'],
+            'error': t['Error'],
+            'comment': t['Comment'],
+            'special': t['Special'],
+            'string': t['String'],
+        }
+
+    def highlightBlock(self, text):
+        try:
+            self.do_highlight(unicode(text))
+        except:
+            import traceback
+            traceback.print_exc()
+
+    def do_highlight(self, text):
+        state = self.previousBlockState()
+        if state == -1:
+            state = State.NORMAL
+        state = State(state)
+
+        i = 0
+        while i < len(text):
+            fmt = state_map[state.parse](state, text, i, self.formats)
+            for num, f in fmt:
+                if f is not None:
+                    self.setFormat(i, num, f)
+                i += num
+
+        self.setCurrentBlockState(state.value)
+
diff --git a/src/calibre/gui2/tweak_book/editor/text.py b/src/calibre/gui2/tweak_book/editor/text.py
new file mode 100644
index 0000000000..0a40b9da92
--- /dev/null
+++ b/src/calibre/gui2/tweak_book/editor/text.py
@@ -0,0 +1,131 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__ = 'GPL v3'
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+
+import textwrap
+
+from PyQt4.Qt import (
+    QPlainTextEdit, QApplication, QFontDatabase, QToolTip, QPalette, QFont)
+
+from calibre.gui2.tweak_book import tprefs
+from calibre.gui2.tweak_book.editor.themes import THEMES, DEFAULT_THEME, theme_color
+from calibre.gui2.tweak_book.editor.syntax.base import SyntaxHighlighter
+from calibre.gui2.tweak_book.editor.syntax.html import HTMLHighlighter
+
+_dff = None
+def default_font_family():
+    global _dff
+    if _dff is None:
+        families = set(map(unicode, QFontDatabase().families()))
+        for x in ('Ubuntu Mono', 'Consolas', 'Liberation Mono'):
+            if x in families:
+                _dff = x
+                break
+        if _dff is None:
+            _dff = 'Courier New'
+    return _dff
+
+class TextEdit(QPlainTextEdit):
+
+    def __init__(self, parent=None):
+        QPlainTextEdit.__init__(self, parent)
+        self.highlighter = SyntaxHighlighter(self)
+        self.apply_theme()
+        self.setMouseTracking(True)
+
+    def apply_theme(self):
+        theme = THEMES.get(tprefs['editor_theme'], None)
+        if theme is None:
+            theme = THEMES[DEFAULT_THEME]
+        self.theme = theme
+        pal = self.palette()
+        pal.setColor(pal.Base, theme_color(theme, 'Normal', 'bg'))
+        pal.setColor(pal.Text, theme_color(theme, 'Normal', 'fg'))
+        pal.setColor(pal.Highlight, theme_color(theme, 'Visual', 'bg'))
+        pal.setColor(pal.HighlightedText, theme_color(theme, 'Visual', 'fg'))
+        self.setPalette(pal)
+        self.tooltip_palette = pal = QPalette()
+        pal.setColor(pal.ToolTipBase, theme_color(theme, 'Tooltip', 'bg'))
+        pal.setColor(pal.ToolTipText, theme_color(theme, 'Tooltip', 'fg'))
+        font = self.font()
+        ff = tprefs['editor_font_family']
+        if ff is None:
+            ff = default_font_family()
+        font.setFamily(ff)
+        font.setPointSize(tprefs['editor_font_size'])
+        self.tooltip_font = QFont(font)
+        self.tooltip_font.setPointSize(font.pointSize() - 1)
+        self.setFont(font)
+        self.highlighter.apply_theme(theme)
+
+    def load_text(self, text, syntax='html'):
+        self.highlighter = {'html':HTMLHighlighter}.get(syntax, SyntaxHighlighter)(self)
+        self.highlighter.apply_theme(self.theme)
+        self.highlighter.setDocument(self.document())
+        self.setPlainText(text)
+
+    def event(self, ev):
+        if ev.type() == ev.ToolTip:
+            self.show_tooltip(ev)
+            return True
+        return QPlainTextEdit.event(self, ev)
+
+    def syntax_format_for_cursor(self, cursor):
+        if cursor.isNull():
+            return
+        pos = cursor.positionInBlock()
+        for r in cursor.block().layout().additionalFormats():
+            if r.start <= pos < r.start + r.length:
+                return r.format
+
+    def show_tooltip(self, ev):
+        c = self.cursorForPosition(ev.pos())
+        fmt = self.syntax_format_for_cursor(c)
+        if fmt is not None:
+            tt = unicode(fmt.toolTip())
+            if tt:
+                QToolTip.setFont(self.tooltip_font)
+                QToolTip.setPalette(self.tooltip_palette)
+                QToolTip.showText(ev.globalPos(), textwrap.fill(tt))
+        QToolTip.hideText()
+        ev.ignore()
+
+if __name__ == '__main__':
+    app = QApplication([])
+    t = TextEdit()
+    t.show()
+    t.load_text(textwrap.dedent('''\
+                <html>
+                <head>
+                    <title>Page title</title>
+                    <style type="text/css">
+                        body { color: green; }
+                    </style>
+                </head id="1">
+                <body>
+                <!-- The start of the document -->a
+                <h1 class="head" id="one" >A heading</h1>
+                <p> A single &. An proper entity &amp;.
+                A single < and a single >.
+                These cases are perfectly simple and easy to
+                distinguish. In a free hour, when our power of choice is
+                untrammelled and when nothing prevents our being able to do
+                what we like best, every pleasure is to be welcomed and every
+                pain avoided.</p>
+
+                <p>
+                But in certain circumstances and owing to the claims of duty or the obligations
+                of business it will frequently occur that pleasures have to be
+                repudiated and annoyances accepted. The wise man therefore
+                always holds in these matters to this principle of selection:
+                he rejects pleasures to secure other greater pleasures, or else
+                he endures pains.</p>
+                </body>
+                </html>
+                '''))
+    app.exec_()
+
diff --git a/src/calibre/gui2/tweak_book/editor/themes.py b/src/calibre/gui2/tweak_book/editor/themes.py
new file mode 100644
index 0000000000..f068be64de
--- /dev/null
+++ b/src/calibre/gui2/tweak_book/editor/themes.py
@@ -0,0 +1,131 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__ = 'GPL v3'
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+
+from collections import namedtuple
+
+from PyQt4.Qt import (QColor, QTextCharFormat, QBrush, QFont)
+
+underline_styles = {'single', 'dash', 'dot', 'dash_dot', 'dash_dot_dot', 'wave', 'spell'}
+
+DEFAULT_THEME = 'calibre-dark'
+
+THEMES = {
+    'calibre-dark':  # {{{  Based on the wombat color scheme for vim
+    '''
+    CursorLine   bg=2d2d2d
+    CursorColumn bg=2d2d2d
+    ColorColumn  bg=2d2d2d
+    MatchParen   fg=f6f3e8 bg=857b6f bold
+    Pmenu        fg=f6f3e8 bg=444444
+    PmenuSel     fg=yellow bg=cae682
+    Tooltip      fg=black bg=ffffed
+
+    Cursor       bg=656565
+    Normal       fg=f6f3e8 bg=242424
+    NonText      fg=808080 bg=303030
+    LineNr       fg=857b6f bg=000000
+    StatusLine   fg=f6f3e8 bg=444444 italic
+    StatusLineNC fg=857b6f bg=444444
+    VertSplit    fg=444444 bg=444444
+    Folded       bg=384048 fg=a0a8b0
+    Title        fg=f6f3e8 bold
+    Visual       fg=f6f3e8 bg=444444
+    SpecialKey   fg=808080 bg=343434
+
+    Comment      fg=99968b
+    Todo         fg=8f8f8f
+    Constant     fg=e5786d
+    String       fg=95e454
+    Identifier   fg=cae682
+    Function     fg=cae682
+    Type         fg=cae682
+    Statement    fg=8ac6f2
+    Keyword      fg=8ac6f2
+    PreProc      fg=e5786d
+    Number       fg=e5786d
+    Special      fg=e7f6da
+    Error        us=wave uc=red
+
+    ''',  # }}}
+
+}
+
+def read_color(col):
+    if QColor.isValidColor(col):
+        return QBrush(QColor(col))
+    try:
+        r, g, b = col[0:2], col[2:4], col[4:6]
+        r, g, b = int(r, 16), int(g, 16), int(b, 16)
+        return QBrush(QColor(r, g, b))
+    except Exception:
+        pass
+
+Highlight = namedtuple('Highlight', 'fg bg bold italic underline underline_color')
+
+def read_theme(raw):
+    ans = {}
+    for line in raw.splitlines():
+        line = line.strip()
+        if not line or line.startswith('#'):
+            continue
+        bold = italic = False
+        fg = bg = name = underline = underline_color = None
+        for i, token in enumerate(line.split()):
+            if i == 0:
+                name = token
+            else:
+                if token == 'bold':
+                    bold = True
+                elif token == 'italic':
+                    italic = True
+                elif '=' in token:
+                    prefix, val = token.partition('=')[0::2]
+                    if prefix == 'us':
+                        underline = val if val in underline_styles else None
+                    elif prefix == 'uc':
+                        underline_color = read_color(val)
+                    elif prefix == 'fg':
+                        fg = read_color(val)
+                    elif prefix == 'bg':
+                        bg = read_color(val)
+        if name is not None:
+            ans[name] = Highlight(fg, bg, bold, italic, underline, underline_color)
+    return ans
+
+
+THEMES = {k:read_theme(raw) for k, raw in THEMES.iteritems()}
+
+def u(x):
+    x = {'spell':'SpellCheck', 'dash_dot':'DashDot', 'dash_dot_dot':'DashDotDot'}.get(x, x.capitalize())
+    if 'Dot' in x:
+        return x + 'Line'
+    return x + 'Underline'
+underline_styles = {x:getattr(QTextCharFormat, u(x)) for x in underline_styles}
+
+def highlight_to_char_format(h):
+    ans = QTextCharFormat()
+    if h.bold:
+        ans.setFontWeight(QFont.Bold)
+    if h.italic:
+        ans.setFontItalic(True)
+    if h.fg is not None:
+        ans.setForeground(h.fg)
+    if h.bg is not None:
+        ans.setBackground(h.bg)
+    if h.underline is not None:
+        ans.setUnderlineStyle(underline_styles[h.underline])
+        if h.underline_color is not None:
+            ans.setUnderlineColor(h.underline_color.color())
+    return ans
+
+def theme_color(theme, name, attr):
+    try:
+        return getattr(theme[name], attr).color()
+    except (KeyError, AttributeError):
+        return getattr(THEMES[DEFAULT_THEME], attr).color()
+