From c66a7e7332bac74c640a66945a9579f4e47b5570 Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Mon, 29 Mar 2021 11:32:59 +0100 Subject: [PATCH] Enhancement: add 'breakpoints' to the template tester. Includes adding line numbers. Also improved syntax highlighting. This will undoubtably change again, but I want to get it out for testing. --- src/calibre/gui2/dialogs/template_dialog.py | 212 ++++++++++-- src/calibre/gui2/dialogs/template_dialog.ui | 167 +++++++++- .../dialogs/template_dialog_code_widget.py | 132 ++++++++ src/calibre/utils/formatter.py | 304 ++++++++++++------ 4 files changed, 690 insertions(+), 125 deletions(-) create mode 100644 src/calibre/gui2/dialogs/template_dialog_code_widget.py diff --git a/src/calibre/gui2/dialogs/template_dialog.py b/src/calibre/gui2/dialogs/template_dialog.py index 7f7c6749e3..93a60ace82 100644 --- a/src/calibre/gui2/dialogs/template_dialog.py +++ b/src/calibre/gui2/dialogs/template_dialog.py @@ -9,20 +9,20 @@ import json, os, traceback from qt.core import (Qt, QDialog, QDialogButtonBox, QSyntaxHighlighter, QFont, QRegExp, QApplication, QTextCharFormat, QColor, QCursor, - QIcon, QSize, QPalette, QLineEdit, QByteArray, - QFontInfo, QFontDatabase) + QIcon, QSize, QPalette, QLineEdit, QByteArray, QFontInfo, + QFontDatabase, QVBoxLayout, QTableWidget, QTableWidgetItem, + QFontComboBox, QComboBox) from calibre import sanitize_file_name from calibre.constants import config_dir -from calibre.gui2 import gprefs +from calibre.gui2 import gprefs, error_dialog, choose_files, pixmap_to_data from calibre.gui2.dialogs.template_dialog_ui import Ui_TemplateDialog from calibre.utils.formatter_functions import formatter_functions from calibre.utils.icu import sort_key +from calibre.utils.localization import localize_user_manual_link from calibre.ebooks.metadata.book.base import Metadata from calibre.ebooks.metadata.book.formatter import SafeFormat from calibre.library.coloring import (displayable_columns, color_row_key) -from calibre.gui2 import error_dialog, choose_files, pixmap_to_data -from calibre.utils.localization import localize_user_manual_link from polyglot.builtins import unicode_type @@ -45,16 +45,23 @@ class TemplateHighlighter(QSyntaxHighlighter): Formats = {} BN_FACTOR = 1000 - KEYWORDS = ["program", 'if', 'then', 'else', 'elif', 'fi'] + KEYWORDS = ["program", 'if', 'then', 'else', 'elif', 'fi', 'for', 'in', + 'separator', 'rof'] def __init__(self, parent=None, builtin_functions=None): super(TemplateHighlighter, self).__init__(parent) self.initializeFormats() + TemplateHighlighter.Rules.append((QRegExp( + r"\b[a-zA-Z]\w*\b(?!\(|\s+\()" + r"|\$+#?[a-zA-Z]\w*"), + "identifier")) + TemplateHighlighter.Rules.append((QRegExp( "|".join([r"\b%s\b" % keyword for keyword in self.KEYWORDS])), "keyword")) + TemplateHighlighter.Rules.append((QRegExp( "|".join([r"\b%s\b" % builtin for builtin in (builtin_functions if builtin_functions else @@ -96,6 +103,7 @@ class TemplateHighlighter(QSyntaxHighlighter): ("normal", None, False, False), ("keyword", pal.color(QPalette.ColorRole.Link).name(), True, False), ("builtin", pal.color(QPalette.ColorRole.Link).name(), False, False), + ("identifier", None, False, True), ("comment", "#007F00", False, True), ("string", "#808000", False, False), ("number", "#924900", False, False), @@ -109,16 +117,16 @@ class TemplateHighlighter(QSyntaxHighlighter): Config["fontsize"] = size baseFormat.setFontPointSize(Config["fontsize"]) - for name in ("normal", "keyword", "builtin", "comment", + for name in ("normal", "keyword", "builtin", "comment", "identifier", "string", "number", "lparen", "rparen"): - format = QTextCharFormat(baseFormat) + format_ = QTextCharFormat(baseFormat) col = Config["%sfontcolor" % name] if col: - format.setForeground(QColor(col)) + format_.setForeground(QColor(col)) if Config["%sfontbold" % name]: - format.setFontWeight(QFont.Weight.Bold) - format.setFontItalic(Config["%sfontitalic" % name]) - self.Formats[name] = format + format_.setFontWeight(QFont.Weight.Bold) + format_.setFontItalic(Config["%sfontitalic" % name]) + self.Formats[name] = format_ def find_paren(self, bn, pos): dex = bn * self.BN_FACTOR + pos @@ -136,16 +144,16 @@ class TemplateHighlighter(QSyntaxHighlighter): self.setFormat(0, textLength, self.Formats["comment"]) return - for regex, format in TemplateHighlighter.Rules: + for regex, format_ in TemplateHighlighter.Rules: i = regex.indexIn(text) while i >= 0: length = regex.matchedLength() - if format in ['lparen', 'rparen']: + if format_ in ['lparen', 'rparen']: pp = self.find_paren(bn, i) if pp and pp.highlight: - self.setFormat(i, length, self.Formats[format]) + self.setFormat(i, length, self.Formats[format_]) else: - self.setFormat(i, length, self.Formats[format]) + self.setFormat(i, length, self.Formats[format_]) i = regex.indexIn(text, i + length) if self.generate_paren_positions: @@ -369,6 +377,7 @@ class TemplateDialog(QDialog, Ui_TemplateDialog): self.highlighter = TemplateHighlighter(self.textbox.document(), builtin_functions=self.builtins) self.textbox.cursorPositionChanged.connect(self.text_cursor_changed) self.textbox.textChanged.connect(self.textbox_changed) + self.textbox.setFont(self.get_current_font()) self.textbox.setTabStopWidth(10) self.source_code.setTabStopWidth(10) @@ -416,8 +425,19 @@ class TemplateDialog(QDialog, Ui_TemplateDialog): '%s' % ( localize_user_manual_link('https://manual.calibre-ebook.com/generated/en/template_ref.html'), tt)) - self.set_up_font_boxes() + s = gprefs.get('template_editor_break_on_print', False) + self.go_button.setEnabled(s) + self.remove_all_button.setEnabled(s) + self.toggle_button.setEnabled(s) + self.breakpoint_line_box.setEnabled(s) + self.breakpoint_line_box_label.setEnabled(s) + self.break_box.setChecked(s) + self.break_box.stateChanged.connect(self.break_box_changed) + self.go_button.clicked.connect(self.go_button_pressed) self.textbox.setFocus() + self.set_up_font_boxes() + self.toggle_button.clicked.connect(self.toggle_button_pressed) + self.remove_all_button.clicked.connect(self.remove_all_button_pressed) # Now geometry try: geom = gprefs.get('template_editor_dialog_geometry', None) @@ -426,7 +446,7 @@ class TemplateDialog(QDialog, Ui_TemplateDialog): except Exception: pass - def set_up_font_boxes(self): + def get_current_font(self): font_name = gprefs.get('gpm_template_editor_font', None) size = gprefs['gpm_template_editor_font_size'] if font_name is None: @@ -435,11 +455,15 @@ class TemplateDialog(QDialog, Ui_TemplateDialog): font.setPointSize(size) else: font = QFont(font_name, pointSize=size) + return font + + def set_up_font_boxes(self): + font = self.get_current_font() self.font_box.setWritingSystem(QFontDatabase.Latin) self.font_box.setCurrentFont(font) self.font_box.setEditable(False) gprefs['gpm_template_editor_font'] = unicode_type(font.family()) - self.font_size_box.setValue(size) + self.font_size_box.setValue(font.pointSize()) self.font_box.currentFontChanged.connect(self.font_changed) self.font_size_box.valueChanged.connect(self.font_size_changed) self.highlighter.initializeFormats() @@ -448,14 +472,51 @@ class TemplateDialog(QDialog, Ui_TemplateDialog): def font_changed(self, font): fi = QFontInfo(font) gprefs['gpm_template_editor_font'] = unicode_type(fi.family()) + self.textbox.setFont(self.get_current_font()) self.highlighter.initializeFormats() self.highlighter.rehighlight() def font_size_changed(self, toWhat): gprefs['gpm_template_editor_font_size'] = toWhat + self.textbox.setFont(self.get_current_font()) self.highlighter.initializeFormats() self.highlighter.rehighlight() + def break_box_changed(self, new_state): + gprefs['template_editor_break_on_print'] = new_state != 0 + self.go_button.setEnabled(new_state != 0) + self.remove_all_button.setEnabled(new_state != 0) + self.toggle_button.setEnabled(new_state != 0) + self.breakpoint_line_box.setEnabled(new_state != 0) + self.breakpoint_line_box_label.setEnabled(new_state != 0) + + def go_button_pressed(self): + self.display_values(unicode_type(self.textbox.toPlainText())) + + def remove_all_button_pressed(self): + self.textbox.set_clicked_line_numbers(set()) + + def toggle_button_pressed(self): + ln = self.breakpoint_line_box.value() + if ln > self.textbox.blockCount(): + return + cln = self.textbox.clicked_line_numbers + if ln: + if ln in self.textbox.clicked_line_numbers: + cln.discard(ln) + else: + cln.add(ln) + self.textbox.set_clicked_line_numbers(cln) + + def break_reporter(self, txt, val, locals_={}, line_number=0): + if self.break_box.isChecked(): + if line_number not in self.textbox.clicked_line_numbers: + return + self.break_reporter_dialog = BreakReporter(self, self.mi[0], + txt, val, locals_, line_number) + if not self.break_reporter_dialog.exec_(): + raise ValueError(_('Stop requested')) + def filename_button_clicked(self): try: path = choose_files(self, 'choose_category_icon', @@ -511,7 +572,8 @@ class TemplateDialog(QDialog, Ui_TemplateDialog): self.last_text = cur_text self.highlighter.regenerate_paren_positions() self.text_cursor_changed() - self.display_values(cur_text) + if self.break_box.checkState() == 0: + self.display_values(cur_text) def display_values(self, txt): tv = self.template_value @@ -520,8 +582,9 @@ class TemplateDialog(QDialog, Ui_TemplateDialog): w.setText(mi.title) w.setCursorPosition(0) v = SafeFormat().safe_format(txt, mi, _('EXCEPTION: '), - mi, global_vars=self.global_vars, - template_functions=self.all_functions) + mi, global_vars=self.global_vars, + template_functions=self.all_functions, + break_reporter=self.break_reporter if r == 0 else None) w = tv.cellWidget(r, 1) w.setText(v) w.setCursorPosition(0) @@ -608,6 +671,111 @@ class TemplateDialog(QDialog, Ui_TemplateDialog): break +class BreakReporterItem(QTableWidgetItem): + + def __init__(self, txt): + super().__init__(txt) + self.setFlags(self.flags() & ~(Qt.ItemFlag.ItemIsEditable)) + + +class BreakReporter(QDialog): + + def __init__(self, parent, mi, op_label, op_value, locals_, line_number): + super().__init__(parent) + self.mi = mi + self.setModal(True) + l = QVBoxLayout(self) + t = self.table = QTableWidget(self) + t.setColumnCount(2) + t.setHorizontalHeaderLabels((_('Name'), _('Value'))) + t.setRowCount(2) + l.addWidget(t) + + self.table_column_widths = None + try: + self.table_column_widths = \ + gprefs.get('template_editor_break_table_widths', None) + t.setColumnWidth(0, self.table_column_widths[0]) + except: + t.setColumnWidth(0, t.fontMetrics().averageCharWidth() * 20) + t.horizontalHeader().sectionResized.connect(self.table_column_resized) + t.horizontalHeader().setStretchLastSection(True); + + bb = QDialogButtonBox() + b = bb.addButton(_('&Continue'), QDialogButtonBox.ButtonRole.AcceptRole) + b.setIcon(QIcon(I('sync-right.png'))) + b.setToolTip(_('Continue running the template')) + b.setDefault(True) + l.addWidget(bb) + b = bb.addButton(_('&Stop'), QDialogButtonBox.ButtonRole.RejectRole) + b.setIcon(QIcon(I('list_remove.png'))) + b.setToolTip(_('Stop running the template')) + l.addWidget(bb) + bb.accepted.connect(self.accept) + bb.rejected.connect(self.reject) + self.setLayout(l) + + self.setWindowTitle(_('Book "%s": break on line number %d') % (self.mi.title,line_number)) + + local_names = sorted(locals_.keys()) + rows = len(local_names) + self.table.setRowCount(rows+2) + self.table.setItem(0, 0, BreakReporterItem(op_label)) + self.table.item(0,0).setToolTip(_('The name of the template language operation')) + self.table.setItem(0, 1, BreakReporterItem(op_value)) + + self.mi_combo = QComboBox() + t.setCellWidget(1, 0, self.mi_combo) + self.mi_combo.addItems(self.get_field_keys()) + self.mi_combo.setToolTip('Choose a book metadata field to display') + self.mi_combo.setCurrentIndex(-1) + self.mi_combo.currentTextChanged.connect(self.get_field_value) + for i,k in enumerate(local_names): + itm = BreakReporterItem(k) + itm.setToolTip(_('A variable in the template')) + self.table.setItem(i+2, 0, itm) + itm = BreakReporterItem(locals_[k]) + itm.setToolTip(_('The value of the variable')) + self.table.setItem(i+2, 1, itm) + + try: + geom = gprefs.get('template_editor_break_geometry', None) + if geom is not None: + QApplication.instance().safe_restore_geometry(self, QByteArray(geom)) + except Exception: + pass + + def get_field_value(self, field): + val = self.mi.format_field('timestamp' if field == 'date' else field)[1] + self.table.setItem(1, 1, BreakReporterItem(val)) + + def table_column_resized(self, col, old, new): + self.table_column_widths = [] + for c in range(0, self.table.columnCount()): + self.table_column_widths.append(self.table.columnWidth(c)) + + def get_field_keys(self): + from calibre.gui2.ui import get_gui + keys = set(get_gui().current_db.new_api.field_metadata.displayable_field_keys()) + keys.discard('sort') + keys.discard('timestamp') + keys.add('title_sort') + keys.add('date') + return sorted(keys) + + def save_geometry(self): + gprefs['template_editor_break_geometry'] = bytearray(self.saveGeometry()) + gprefs['template_editor_break_table_widths'] = self.table_column_widths + + def reject(self): + self.save_geometry() + QDialog.reject(self) + + def accept(self): + self.save_geometry() + QDialog.accept(self) + + class EmbeddedTemplateDialog(TemplateDialog): def __init__(self, parent): diff --git a/src/calibre/gui2/dialogs/template_dialog.ui b/src/calibre/gui2/dialogs/template_dialog.ui index 709bf30039..937c603edb 100644 --- a/src/calibre/gui2/dialogs/template_dialog.ui +++ b/src/calibre/gui2/dialogs/template_dialog.ui @@ -179,12 +179,154 @@ textbox - Qt::AlignLeading|Qt::AlignLeft|Qt::AlignTop + Qt::AlignLeading|Qt::AlignLeft|Qt::AlignVCenter + + + + + + Qt::Horizontal + + + + 1 + 0 + + + + + + + + Enable &breakpoints + + + <p>If checked, the template evaluator will stop when it +evaluates an expression on a double-clicked line number, opening a dialog showing +you the value as well as all the local variables</p> + + + + + + + QFrame::VLine + + + QFrame::Raised + + + 3 + + + + + + + &Go + + + + :/images/sync-right.png:/images/sync-right.png + + + If 'Enable breakpoints' is checked then click this button to run your template + + + + + + + QFrame::VLine + + + QFrame::Raised + + + 3 + + + + + + + &Line: + + + breakpoint_line_box + + + Line number to toggle + + + + + + + Line number to toggle + + + 1 + + + 999 + + + 1 + + + Qt::AlignRight + + + + + + + &Toggle + + + + :/images/swap.png:/images/swap.png + + + Toggle the breakpoint on the line number in the box + + + + + + + QFrame::VLine + + + QFrame::Raised + + + 3 + + + + + + + &Remove all + + + + :/images/list_remove.png:/images/list_remove.png + + + Remove all breakpoints + + + + + - + The template program text @@ -210,7 +352,7 @@ - + Template value: @@ -223,7 +365,7 @@ - + @@ -376,11 +518,7 @@ - - - - - + @@ -418,7 +556,7 @@ - + Qt::Horizontal @@ -428,14 +566,14 @@ - + QFrame::HLine - + @@ -558,6 +696,11 @@ QBoxLayout
calibre/gui2/dialogs/template_dialog_box_layout.h
+ + CodeEditor + QPlainTextEdit +
calibre/gui2/dialogs/template_dialog_code_widget.h
+ diff --git a/src/calibre/gui2/dialogs/template_dialog_code_widget.py b/src/calibre/gui2/dialogs/template_dialog_code_widget.py new file mode 100644 index 0000000000..f48a7ef3de --- /dev/null +++ b/src/calibre/gui2/dialogs/template_dialog_code_widget.py @@ -0,0 +1,132 @@ +''' +Created on 26 Mar 2021 + +@author: Charles Haley +Based on classes in calibre.gui2.tweak_book.editor + +License: GPLv3 Copyright: 2021, Kovid Goyal +''' + +from qt.core import (Qt, QWidget, QSize, QPlainTextEdit, QPainter, + QRect, QFont, QPalette, QTextEdit, QTextFormat) + +from calibre.gui2.tweak_book.editor.themes import (get_theme, theme_color) +from calibre.gui2.tweak_book.editor.text import LineNumbers +from polyglot.builtins import unicode_type + +class LineNumberArea(LineNumbers): + + def mouseDoubleClickEvent(self, event): + super().mousePressEvent(event) + self.parent().line_area_doubleclick_event(event) + + +class CodeEditor(QPlainTextEdit): + + def __init__(self, parent): + QPlainTextEdit.__init__(self, parent) + + + # Use the default theme from the book editor + theme = get_theme(None) + self.line_number_palette = pal = QPalette() + pal.setColor(QPalette.ColorRole.Base, theme_color(theme, 'LineNr', 'bg')) + pal.setColor(QPalette.ColorRole.Text, theme_color(theme, 'LineNr', 'fg')) + pal.setColor(QPalette.ColorRole.BrightText, theme_color(theme, 'LineNrC', 'fg')) + + self.line_number_area = LineNumberArea(self) + + self.blockCountChanged.connect(self.update_line_number_area_width) + self.updateRequest.connect(self.update_line_number_area) + self.cursorPositionChanged.connect(self.highlight_cursor_line) + + self.update_line_number_area_width(0) + self.highlight_cursor_line() + self.clicked_line_numbers = set() + + def highlight_cursor_line(self): + sel = QTextEdit.ExtraSelection() + sel.format.setBackground(self.palette().alternateBase()) + sel.format.setProperty(QTextFormat.Property.FullWidthSelection, True) + sel.cursor = self.textCursor() + sel.cursor.clearSelection() + self.setExtraSelections([sel,]) + + def update_line_number_area_width(self, block_count=0): + self.gutter_width = self.line_number_area_width() + self.setViewportMargins(self.gutter_width, 0, 0, 0) + + def line_number_area_width(self): + # get largest width of digits + w = self.fontMetrics() + self.number_width = max(map(lambda x:w.width(unicode_type(x)), range(10))) + digits = 1 + limit = max(1, self.blockCount()) + while limit >= 10: + limit /= 10 + digits += 1 + return self.number_width * (digits+1) + + def update_line_number_area(self, rect, dy): + if dy: + self.line_number_area.scroll(0, dy) + else: + self.line_number_area.update(0, rect.y(), self.line_number_area.width(), rect.height()) + if rect.contains(self.viewport().rect()): + self.update_line_number_area_width() + + def resizeEvent(self, ev): + QPlainTextEdit.resizeEvent(self, ev) + cr = self.contentsRect() + self.line_number_area.setGeometry(QRect(cr.left(), cr.top(), + self.line_number_area_width(), cr.height())) + + def line_area_doubleclick_event(self, event): + # remember that the result of the divide will be zero-based + line = event.y()//self.fontMetrics().height() + 1 + self.firstVisibleBlock().blockNumber() + if line in self.clicked_line_numbers: + self.clicked_line_numbers.discard(line) + else: + self.clicked_line_numbers.add(line) + self.update(self.line_number_area.geometry()) + + def set_clicked_line_numbers(self, new_set): + self.clicked_line_numbers = new_set + self.update(self.line_number_area.geometry()) + + def paint_line_numbers(self, ev): + painter = QPainter(self.line_number_area) + painter.fillRect(ev.rect(), self.line_number_palette.color(QPalette.ColorRole.Base)) + + block = self.firstVisibleBlock() + num = block.blockNumber() + top = int(self.blockBoundingGeometry(block).translated(self.contentOffset()).top()) + bottom = top + int(self.blockBoundingRect(block).height()) + current = self.textCursor().block().blockNumber() + painter.setPen(self.line_number_palette.color(QPalette.ColorRole.Text)) + + while block.isValid() and top <= ev.rect().bottom(): + if block.isVisible() and bottom >= ev.rect().top(): + set_bold = False + set_italic = False + if current == num: + set_bold = True + if num+1 in self.clicked_line_numbers: + set_italic = True + painter.save() + if set_bold or set_italic: + f = QFont(self.font()) + if set_bold: + f.setBold(set_bold) + painter.setPen(self.line_number_palette.color(QPalette.ColorRole.BrightText)) + f.setItalic(set_italic) + painter.setFont(f) + else: + painter.setFont(self.font()) + painter.drawText(0, top, self.line_number_area.width() - 5, self.fontMetrics().height(), + Qt.AlignmentFlag.AlignRight, unicode_type(num + 1)) + painter.restore() + block = block.next() + top = bottom + bottom = top + int(self.blockBoundingRect(block).height()) + num += 1 diff --git a/src/calibre/utils/formatter.py b/src/calibre/utils/formatter.py index 65f15cea9e..76066a05f3 100644 --- a/src/calibre/utils/formatter.py +++ b/src/calibre/utils/formatter.py @@ -11,6 +11,7 @@ __docformat__ = 'restructuredtext en' import re, string, traceback, numbers from math import modf +from functools import partial from calibre import prints from calibre.constants import DEBUG @@ -40,11 +41,20 @@ class Node(object): NODE_UNARY_LOGOP = 18 NODE_BINARY_ARITHOP = 19 NODE_UNARY_ARITHOP = 20 + NODE_PRINT = 21 + NODE_LINE_NUMBER = 22 + + def __init__(self, line_number, name): + self.line_number = line_number + self.my_node_name = name + + def node_name(self): + return self.my_node_name class IfNode(Node): - def __init__(self, condition, then_part, else_part): - Node.__init__(self) + def __init__(self, line_number, condition, then_part, else_part): + Node.__init__(self, line_number, 'IF') self.node_type = self.NODE_IF self.condition = condition self.then_part = then_part @@ -52,8 +62,8 @@ class IfNode(Node): class ForNode(Node): - def __init__(self, variable, list_field_expr, separator, block): - Node.__init__(self) + def __init__(self, line_number, variable, list_field_expr, separator, block): + Node.__init__(self, line_number, 'FOR') self.node_type = self.NODE_FOR self.variable = variable self.list_field_expr = list_field_expr @@ -62,53 +72,53 @@ class ForNode(Node): class AssignNode(Node): - def __init__(self, left, right): - Node.__init__(self) + def __init__(self, line_number, left, right): + Node.__init__(self, line_number, 'ASSIGN') self.node_type = self.NODE_ASSIGN self.left = left self.right = right class FunctionNode(Node): - def __init__(self, function_name, expression_list): - Node.__init__(self) + def __init__(self, line_number, function_name, expression_list): + Node.__init__(self, line_number, 'FUNCTION CALL') self.node_type = self.NODE_FUNC self.name = function_name self.expression_list = expression_list class CallNode(Node): - def __init__(self, function, expression_list): - Node.__init__(self) + def __init__(self, line_number, function, expression_list): + Node.__init__(self, line_number, 'TEMPLATE CALL') self.node_type = self.NODE_CALL self.function = function self.expression_list = expression_list class ArgumentsNode(Node): - def __init__(self, expression_list): - Node.__init__(self) + def __init__(self, line_number, expression_list): + Node.__init__(self, line_number, 'ARGUMENTS') self.node_type = self.NODE_ARGUMENTS self.expression_list = expression_list class GlobalsNode(Node): - def __init__(self, expression_list): - Node.__init__(self) + def __init__(self, line_number, expression_list): + Node.__init__(self, line_number, 'GLOBALS') self.node_type = self.NODE_GLOBALS self.expression_list = expression_list class SetGlobalsNode(Node): - def __init__(self, expression_list): - Node.__init__(self) + def __init__(self, line_number, expression_list): + Node.__init__(self, line_number, 'SET_GLOBALS') self.node_type = self.NODE_SET_GLOBALS self.expression_list = expression_list class StringCompareNode(Node): - def __init__(self, operator, left, right): - Node.__init__(self) + def __init__(self, line_number, operator, left, right): + Node.__init__(self, line_number, 'COMPARE STRING') self.node_type = self.NODE_COMPARE_STRING self.operator = operator self.left = left @@ -116,8 +126,8 @@ class StringCompareNode(Node): class NumericCompareNode(Node): - def __init__(self, operator, left, right): - Node.__init__(self) + def __init__(self, line_number, operator, left, right): + Node.__init__(self, line_number, 'COMPARE NUMBERS') self.node_type = self.NODE_COMPARE_NUMERIC self.operator = operator self.left = left @@ -125,8 +135,8 @@ class NumericCompareNode(Node): class LogopBinaryNode(Node): - def __init__(self, operator, left, right): - Node.__init__(self) + def __init__(self, line_number, operator, left, right): + Node.__init__(self, line_number, 'BINARY LOGICAL OP') self.node_type = self.NODE_BINARY_LOGOP self.operator = operator self.left = left @@ -134,16 +144,16 @@ class LogopBinaryNode(Node): class LogopUnaryNode(Node): - def __init__(self, operator, expr): - Node.__init__(self) + def __init__(self, line_number, operator, expr): + Node.__init__(self, line_number, 'UNARY LOGICAL OP') self.node_type = self.NODE_UNARY_LOGOP self.operator = operator self.expr = expr class NumericBinaryNode(Node): - def __init__(self, operator, left, right): - Node.__init__(self) + def __init__(self, line_number, operator, left, right): + Node.__init__(self, line_number, 'BINARY ARITHMETIC OP') self.node_type = self.NODE_BINARY_ARITHOP self.operator = operator self.left = left @@ -151,52 +161,52 @@ class NumericBinaryNode(Node): class NumericUnaryNode(Node): - def __init__(self, operator, expr): - Node.__init__(self) + def __init__(self, line_number, operator, expr): + Node.__init__(self, line_number, 'UNARY ARITHMETIC OP') self.node_type = self.NODE_UNARY_ARITHOP self.operator = operator self.expr = expr class ConstantNode(Node): - def __init__(self, value): - Node.__init__(self) + def __init__(self, line_number, value): + Node.__init__(self, line_number, 'CONSTANT') self.node_type = self.NODE_CONSTANT self.value = value class VariableNode(Node): - def __init__(self, name): - Node.__init__(self) + def __init__(self, line_number, name): + Node.__init__(self, line_number, 'VARIABLE') self.node_type = self.NODE_RVALUE self.name = name class FieldNode(Node): - def __init__(self, expression): - Node.__init__(self) + def __init__(self, line_number, expression): + Node.__init__(self, line_number, 'FIELD FUNCTION') self.node_type = self.NODE_FIELD self.expression = expression class RawFieldNode(Node): - def __init__(self, expression, default=None): - Node.__init__(self) + def __init__(self, line_number, expression, default=None): + Node.__init__(self, line_number, 'RAW_FIELD FUNCTION') self.node_type = self.NODE_RAW_FIELD self.expression = expression self.default = default class FirstNonEmptyNode(Node): - def __init__(self, expression_list): - Node.__init__(self) + def __init__(self, line_number, expression_list): + Node.__init__(self, line_number, 'FIRST_NON_EMPTY FUNCTION') self.node_type = self.NODE_FIRST_NON_EMPTY self.expression_list = expression_list class ContainsNode(Node): - def __init__(self, arguments): - Node.__init__(self) + def __init__(self, line_number, arguments): + Node.__init__(self, line_number, 'CONTAINS FUNCTION') self.node_type = self.NODE_CONTAINS self.value_expression = arguments[0] self.test_expression = arguments[1] @@ -204,6 +214,19 @@ class ContainsNode(Node): self.not_match_expression = arguments[3] +class PrintNode(Node): + def __init__(self, line_number, arguments): + Node.__init__(self, line_number, 'PRINT') + self.node_type = self.NODE_PRINT + self.arguments = arguments + + +class LineNumberNode(Node): + def __init__(self, line_number): + Node.__init__(self, line_number, 'LINE NUMBER') + self.node_type = self.NODE_LINE_NUMBER + + class _Parser(object): LEX_OP = 1 LEX_ID = 2 @@ -212,21 +235,33 @@ class _Parser(object): LEX_STRING_INFIX = 5 LEX_NUMERIC_INFIX = 6 LEX_KEYWORD = 7 + LEX_NEWLINE = 8 def error(self, message): + ln = None try: tval = "'" + self.prog[self.lex_pos-1][1] + "'" except Exception: tval = _('Unknown') - if self.lex_pos > 0: - location = tval - elif self.lex_pos < self.prog_len: + if self.lex_pos > 0 and self.lex_pos < self.prog_len: location = tval + ln = self.line_number else: location = _('the end of the program') - raise ValueError(_('{0}: {1} near {2}').format('Formatter', message, location)) + if ln: + raise ValueError(_('{0}: {1} near {2} on line {3}').format( + 'Formatter', message, location, ln)) + else: + raise ValueError(_('{0}: {1} near {2}').format( + 'Formatter', message, location)) + + def check_eol(self): + while self.lex_pos < len(self.prog) and self.prog[self.lex_pos] == self.LEX_NEWLINE: + self.line_number += 1 + self.consume() def token(self): + self.check_eol() try: token = self.prog[self.lex_pos][1] self.lex_pos += 1 @@ -238,6 +273,7 @@ class _Parser(object): self.lex_pos += 1 def token_op_is_equals(self): + self.check_eol() try: token = self.prog[self.lex_pos] return token[1] == '=' and token[0] == self.LEX_OP @@ -245,18 +281,21 @@ class _Parser(object): return False def token_op_is_string_infix_compare(self): + self.check_eol() try: return self.prog[self.lex_pos][0] == self.LEX_STRING_INFIX except: return False def token_op_is_numeric_infix_compare(self): + self.check_eol() try: return self.prog[self.lex_pos][0] == self.LEX_NUMERIC_INFIX except: return False def token_op_is_lparen(self): + self.check_eol() try: token = self.prog[self.lex_pos] return token[1] == '(' and token[0] == self.LEX_OP @@ -264,6 +303,7 @@ class _Parser(object): return False def token_op_is_rparen(self): + self.check_eol() try: token = self.prog[self.lex_pos] return token[1] == ')' and token[0] == self.LEX_OP @@ -271,6 +311,7 @@ class _Parser(object): return False def token_op_is_comma(self): + self.check_eol() try: token = self.prog[self.lex_pos] return token[1] == ',' and token[0] == self.LEX_OP @@ -278,6 +319,7 @@ class _Parser(object): return False def token_op_is_semicolon(self): + self.check_eol() try: token = self.prog[self.lex_pos] return token[1] == ';' and token[0] == self.LEX_OP @@ -285,6 +327,7 @@ class _Parser(object): return False def token_op_is_colon(self): + self.check_eol() try: token = self.prog[self.lex_pos] return token[1] == ':' and token[0] == self.LEX_OP @@ -292,6 +335,7 @@ class _Parser(object): return False def token_op_is_plus(self): + self.check_eol() try: token = self.prog[self.lex_pos] return token[1] == '+' and token[0] == self.LEX_OP @@ -299,6 +343,7 @@ class _Parser(object): return False def token_op_is_minus(self): + self.check_eol() try: token = self.prog[self.lex_pos] return token[1] == '-' and token[0] == self.LEX_OP @@ -306,6 +351,7 @@ class _Parser(object): return False def token_op_is_times(self): + self.check_eol() try: token = self.prog[self.lex_pos] return token[1] == '*' and token[0] == self.LEX_OP @@ -313,6 +359,7 @@ class _Parser(object): return False def token_op_is_divide(self): + self.check_eol() try: token = self.prog[self.lex_pos] return token[1] == '/' and token[0] == self.LEX_OP @@ -320,6 +367,7 @@ class _Parser(object): return False def token_op_is_and(self): + self.check_eol() try: token = self.prog[self.lex_pos] return token[1] == '&&' and token[0] == self.LEX_OP @@ -327,6 +375,7 @@ class _Parser(object): return False def token_op_is_or(self): + self.check_eol() try: token = self.prog[self.lex_pos] return token[1] == '||' and token[0] == self.LEX_OP @@ -334,19 +383,25 @@ class _Parser(object): return False def token_op_is_not(self): + self.check_eol() try: token = self.prog[self.lex_pos] return token[1] == '!' and token[0] == self.LEX_OP except: return False + def token_is_newline(self): + return self.lex_pos < len(self.prog) and self.prog[self.lex_pos] == self.LEX_NEWLINE + def token_is_id(self): + self.check_eol() try: return self.prog[self.lex_pos][0] == self.LEX_ID except: return False def token_is_call(self): + self.check_eol() try: token = self.prog[self.lex_pos] return token[1] == 'call' and token[0] == self.LEX_KEYWORD @@ -354,6 +409,7 @@ class _Parser(object): return False def token_is_if(self): + self.check_eol() try: token = self.prog[self.lex_pos] return token[1] == 'if' and token[0] == self.LEX_KEYWORD @@ -361,6 +417,7 @@ class _Parser(object): return False def token_is_then(self): + self.check_eol() try: token = self.prog[self.lex_pos] return token[1] == 'then' and token[0] == self.LEX_KEYWORD @@ -368,6 +425,7 @@ class _Parser(object): return False def token_is_else(self): + self.check_eol() try: token = self.prog[self.lex_pos] return token[1] == 'else' and token[0] == self.LEX_KEYWORD @@ -375,6 +433,7 @@ class _Parser(object): return False def token_is_elif(self): + self.check_eol() try: token = self.prog[self.lex_pos] return token[1] == 'elif' and token[0] == self.LEX_KEYWORD @@ -382,6 +441,7 @@ class _Parser(object): return False def token_is_fi(self): + self.check_eol() try: token = self.prog[self.lex_pos] return token[1] == 'fi' and token[0] == self.LEX_KEYWORD @@ -389,6 +449,7 @@ class _Parser(object): return False def token_is_for(self): + self.check_eol() try: token = self.prog[self.lex_pos] return token[1] == 'for' and token[0] == self.LEX_KEYWORD @@ -396,6 +457,7 @@ class _Parser(object): return False def token_is_in(self): + self.check_eol() try: token = self.prog[self.lex_pos] return token[1] == 'in' and token[0] == self.LEX_KEYWORD @@ -403,6 +465,7 @@ class _Parser(object): return False def token_is_rof(self): + self.check_eol() try: token = self.prog[self.lex_pos] return token[1] == 'rof' and token[0] == self.LEX_KEYWORD @@ -410,6 +473,7 @@ class _Parser(object): return False def token_is_separator(self): + self.check_eol() try: token = self.prog[self.lex_pos] return token[1] == 'separator' and token[0] == self.LEX_ID @@ -417,18 +481,21 @@ class _Parser(object): return False def token_is_constant(self): + self.check_eol() try: return self.prog[self.lex_pos][0] == self.LEX_CONST except: return False def token_is_eof(self): + self.check_eol() try: return self.prog[self.lex_pos][0] == self.LEX_EOF except: return True def program(self, parent, funcs, prog): + self.line_number = 1 self.lex_pos = 0 self.parent = parent self.funcs = funcs @@ -444,22 +511,30 @@ class _Parser(object): def expression_list(self): expr_list = [] - while not self.token_is_eof(): - expr_list.append(self.top_expr()) - if not self.token_op_is_semicolon(): + while True: + while self.token_is_newline(): + self.line_number += 1 + expr_list.append(LineNumberNode(self.line_number)) + self.consume() + if self.token_is_eof(): + break + expr_list.append(self.top_expr()) + if self.token_op_is_semicolon(): + self.consume() + else: break - self.consume() return expr_list def if_expression(self): self.consume() + line_number = self.line_number condition = self.top_expr() if not self.token_is_then(): self.error(_("Missing 'then' in if statement")) self.consume() then_part = self.expression_list() if self.token_is_elif(): - return IfNode(condition, then_part, [self.if_expression(),]) + return IfNode(line_number, condition, then_part, [self.if_expression(),]) if self.token_is_else(): self.consume() else_part = self.expression_list() @@ -468,7 +543,7 @@ class _Parser(object): if not self.token_is_fi(): self.error(_("Missing 'fi' in if statement")) self.consume() - return IfNode(condition, then_part, else_part) + return IfNode(line_number, condition, then_part, else_part) def for_expression(self): self.consume() @@ -491,7 +566,7 @@ class _Parser(object): if not self.token_is_rof(): self.error(_("Missing 'rof' in for statement")) self.consume() - return ForNode(variable, list_expr, separator, block) + return ForNode(self.line_number, variable, list_expr, separator, block) def top_expr(self): return self.or_expr() @@ -501,7 +576,7 @@ class _Parser(object): while self.token_op_is_or(): self.consume() right = self.and_expr() - left = LogopBinaryNode('or', left, right) + left = LogopBinaryNode(self.line_number, 'or', left, right) return left def and_expr(self): @@ -509,23 +584,23 @@ class _Parser(object): while self.token_op_is_and(): self.consume() right = self.not_expr() - left = LogopBinaryNode('and', left, right) + left = LogopBinaryNode(self.line_number, 'and', left, right) return left def not_expr(self): if self.token_op_is_not(): self.consume() - return LogopUnaryNode('not', self.not_expr()) + return LogopUnaryNode(self.line_number, 'not', self.not_expr()) return self.compare_expr() def compare_expr(self): left = self.add_subtract_expr() if self.token_op_is_string_infix_compare() or self.token_is_in(): operator = self.token() - return StringCompareNode(operator, left, self.add_subtract_expr()) + return StringCompareNode(self.line_number, operator, left, self.add_subtract_expr()) if self.token_op_is_numeric_infix_compare(): operator = self.token() - return NumericCompareNode(operator, left, self.add_subtract_expr()) + return NumericCompareNode(self.line_number, operator, left, self.add_subtract_expr()) return left def add_subtract_expr(self): @@ -533,7 +608,7 @@ class _Parser(object): while self.token_op_is_plus() or self.token_op_is_minus(): operator = self.token() right = self.times_divide_expr() - left = NumericBinaryNode(operator, left, right) + left = NumericBinaryNode(self.line_number, operator, left, right) return left def times_divide_expr(self): @@ -541,16 +616,16 @@ class _Parser(object): while self.token_op_is_times() or self.token_op_is_divide(): operator = self.token() right = self.unary_plus_minus_expr() - left = NumericBinaryNode(operator, left, right) + left = NumericBinaryNode(self.line_number, operator, left, right) return left def unary_plus_minus_expr(self): if self.token_op_is_plus(): self.consume() - return NumericUnaryNode('+', self.unary_plus_minus_expr()) + return NumericUnaryNode(self.line_number, '+', self.unary_plus_minus_expr()) if self.token_op_is_minus(): self.consume() - return NumericUnaryNode('-', self.unary_plus_minus_expr()) + return NumericUnaryNode(self.line_number, '-', self.unary_plus_minus_expr()) return self.expr() def call_expression(self, name, arguments): @@ -563,7 +638,7 @@ class _Parser(object): subprog = _Parser().program(self, self.funcs, self.parent.lex_scanner.scan(text)) self.funcs[name].cached_parse_tree = subprog - return CallNode(subprog, arguments) + return CallNode(self.line_number, subprog, arguments) def expr(self): if self.token_op_is_lparen(): @@ -582,15 +657,15 @@ class _Parser(object): # We have an identifier. Check if it is a field reference if len(id_) > 1 and id_[0] == '$': if id_[1] == '$': - return RawFieldNode(ConstantNode(id_[2:])) - return FieldNode(ConstantNode(id_[1:])) + return RawFieldNode(self.line_number, ConstantNode(self.line_number, id_[2:])) + return FieldNode(self.line_number, ConstantNode(self.line_number, id_[1:])) # Determine if it is a function if not self.token_op_is_lparen(): if self.token_op_is_equals(): # classic assignment statement self.consume() - return AssignNode(id_, self.top_expr()) - return VariableNode(id_) + return AssignNode(self.line_number, id_, self.top_expr()) + return VariableNode(self.line_number, id_) # We have a function. # Check if it is a known one. We do this here so error reporting is @@ -610,15 +685,15 @@ class _Parser(object): if self.token() != ')': self.error(_('Missing closing parenthesis')) if id_ == 'field' and len(arguments) == 1: - return FieldNode(arguments[0]) + return FieldNode(self.line_number, arguments[0]) if id_ == 'raw_field' and (len(arguments) in (1, 2)): - return RawFieldNode(*arguments) + return RawFieldNode(self.line_number, *arguments) if id_ == 'test' and len(arguments) == 3: - return IfNode(arguments[0], (arguments[1],), (arguments[2],)) + return IfNode(self.line_number, arguments[0], (arguments[1],), (arguments[2],)) if id_ == 'first_non_empty' and len(arguments) > 0: - return FirstNonEmptyNode(arguments) + return FirstNonEmptyNode(self.line_number, arguments) if (id_ == 'assign' and len(arguments) == 2 and arguments[0].node_type == Node.NODE_RVALUE): - return AssignNode(arguments[0].name, arguments[1]) + return AssignNode(self.line_number, arguments[0].name, arguments[1]) if id_ == 'arguments' or id_ == 'globals' or id_ == 'set_globals': new_args = [] for arg_list in arguments: @@ -627,48 +702,66 @@ class _Parser(object): self.error(_("Parameters to '{}' must be " "variables or assignments").format(id_)) if arg.node_type == Node.NODE_RVALUE: - arg = AssignNode(arg.name, ConstantNode('')) + arg = AssignNode(self.line_number, arg.name, ConstantNode(self.line_number, '')) new_args.append(arg) if id_ == 'arguments': - return ArgumentsNode(new_args) + return ArgumentsNode(self.line_number, new_args) if id_ == 'set_globals': - return SetGlobalsNode(new_args) - return GlobalsNode(new_args) + return SetGlobalsNode(self.line_number, new_args) + return GlobalsNode(self.line_number, new_args) if id_ == 'contains' and len(arguments) == 4: - return ContainsNode(arguments) + return ContainsNode(self.line_number, arguments) + if id_ == 'print': + return PrintNode(self.line_number, arguments) if id_ in self.func_names and not self.funcs[id_].is_python: return self.call_expression(id_, arguments) cls = self.funcs[id_] if cls.arg_count != -1 and len(arguments) != cls.arg_count: self.error(_('Incorrect number of arguments for function {0}').format(id_)) - return FunctionNode(id_, arguments) + return FunctionNode(self.line_number, id_, arguments) elif self.token_is_constant(): # String or number - return ConstantNode(self.token()) + return ConstantNode(self.line_number, self.token()) else: self.error(_('Expression is not function or constant')) class _Interpreter(object): def error(self, message): - m = 'Interpreter: ' + message + m = _('Interpreter: {0} - line number {1}').format(message, self.line_number) raise ValueError(m) - def program(self, funcs, parent, prog, val, is_call=False, args=None, global_vars=None): + def program(self, funcs, parent, prog, val, is_call=False, args=None, + global_vars=None, break_reporter=None): self.parent = parent self.parent_kwargs = parent.kwargs self.parent_book = parent.book + self.line_number = 1 self.funcs = funcs self.locals = {'$':val} self.global_vars = global_vars if isinstance(global_vars, dict) else {} + if break_reporter: + self.break_reporter = self.call_break_reporter + self.real_break_reporter = break_reporter + else: + self.break_reporter = None + if is_call: - return self.do_node_call(CallNode(prog, None), args=args) + return self.do_node_call(CallNode(self.line_number, prog, None), args=args) return self.expression_list(prog) + def call_break_reporter(self, txt, val, line_number=None): + self.real_break_reporter(txt, val, self.locals, + line_number if line_number else self.line_number) + def expression_list(self, prog): val = '' for p in prog: val = self.expr(p) + if (self.break_reporter and + p.node_type != Node.NODE_LINE_NUMBER and + p.node_type != Node.NODE_IF): + self.break_reporter(p.node_name(), val) return val INFIX_STRING_COMPARE_OPS = { @@ -712,11 +805,20 @@ class _Interpreter(object): self.error(_('Value used in comparison is not a number. Operator {0}').format(prog.operator)) def do_node_if(self, prog): + line_number = prog.line_number test_part = self.expr(prog.condition) + if self.break_reporter: + self.break_reporter('if: condition', test_part, line_number=line_number) if test_part: - return self.expression_list(prog.then_part) + v = self.expression_list(prog.then_part) + if self.break_reporter: + self.break_reporter('if: then part', v, line_number=line_number) + return v elif prog.else_part: - return self.expression_list(prog.else_part) + v = self.expression_list(prog.else_part) + if self.break_reporter: + self.break_reporter('if: else part', v, line_number=line_number) + return v return '' def do_node_rvalue(self, prog): @@ -821,10 +923,15 @@ class _Interpreter(object): if not isinstance(res, list): res = [r.strip() for r in res.split(separator) if r.strip()] ret = '' + if self.break_reporter: + self.break_reporter(_("'for' value list"), separator.join(res)) for x in res: self.locals[v] = x ret = self.expression_list(prog.block) return ret + elif self.break_reporter: + self.break_reporter(_("'for' value list"), '') + self.error(_('The field {0} is not a list').format(f)) except ValueError as e: raise e @@ -887,6 +994,17 @@ class _Interpreter(object): except: self.error(_('Error during arithmetic operator evaluation. Operator {0}').format(prog.operator)) + def do_node_print(self, prog): + res = [] + for arg in prog.arguments: + res.append(self.expr(arg)) + print(res) + return res[0] if res else '' + + def do_node_line_number(self, prog): + self.line_number = prog.line_number + return '' + NODE_OPS = { Node.NODE_IF: do_node_if, Node.NODE_ASSIGN: do_node_assign, @@ -908,6 +1026,8 @@ class _Interpreter(object): Node.NODE_UNARY_LOGOP: do_node_logop_unary, Node.NODE_BINARY_ARITHOP: do_node_binary_arithop, Node.NODE_UNARY_ARITHOP: do_node_unary_arithop, + Node.NODE_PRINT: do_node_print, + Node.NODE_LINE_NUMBER: do_node_line_number, } def expr(self, prog): @@ -1002,11 +1122,11 @@ class TemplateFormatter(string.Formatter): (r'\w+', lambda x,t: (_Parser.LEX_ID, t)), # noqa (r'".*?((?= 0: p += 1 if p >= 0 and fmt[-1] == '\'': - val = self._eval_program(val, fmt[p+1:-1], None, self.global_vars) + val = self._eval_program(val, fmt[p+1:-1], None, self.global_vars, None) colon = fmt[0:p].find(':') if colon < 0: dispfmt = '' @@ -1103,10 +1224,10 @@ class TemplateFormatter(string.Formatter): return '' return prefix + val + suffix - def evaluate(self, fmt, args, kwargs, global_vars): + def evaluate(self, fmt, args, kwargs, global_vars, break_reporter=None): if fmt.startswith('program:'): ans = self._eval_program(kwargs.get('$', None), fmt[8:], - self.column_name, global_vars) + self.column_name, global_vars, break_reporter) else: ans = self.vformat(fmt, args, kwargs) if self.strip_results: @@ -1130,7 +1251,7 @@ class TemplateFormatter(string.Formatter): def safe_format(self, fmt, kwargs, error_value, book, column_name=None, template_cache=None, strip_results=True, template_functions=None, - global_vars=None): + global_vars=None, break_reporter=None): self.strip_results = strip_results self.column_name = column_name self.template_cache = template_cache @@ -1144,7 +1265,8 @@ class TemplateFormatter(string.Formatter): self.composite_values = {} self.locals = {} try: - ans = self.evaluate(fmt, [], kwargs, self.global_vars) + ans = self.evaluate(fmt, [], kwargs, self.global_vars, + break_reporter=break_reporter) except Exception as e: if DEBUG: # and getattr(e, 'is_locking_error', False): traceback.print_exc()