More template stuff:

- Replace the newline() function with the character('name') function
- Document the new function
- In the template editor/debugger, display special characters like newline as escaped characters

manual/template_lang.rst

@@ -383,6 +383,7 @@ In `GPM` the functions described in `Single Function Mode` all require an additi
 
   More than one of ``is_undefined``, ``is_false``, or ``is_true`` can be set to 1.
 * ``ceiling(x)`` -- returns the smallest integer greater than or equal to ``x``. Throws an exception if ``x`` is not a number.
+* ``character(character_name)`` -- returns the character named by character_name. For example, ``character('newline')`` returns a newline character (``'\n'``). The supported character names are ``newline``, ``return``, ``tab``, and ``backslash``.
 * ``cmp(x, y, lt, eq, gt)`` -- compares ``x`` and ``y`` after converting both to numbers. Returns ``lt`` if ``x <# y``, ``eq`` if ``x ==# y``, otherwise ``gt``. This function can usually be replaced with one of the numeric compare operators (``==#``, ``<#``, ``>#``, etc).
 * ``connected_device_name(storage_location_key)`` -- if a device is connected then return the device name, otherwise return the empty string. Each storage location on a device has its own device name. The ``storage_location_key`` names are ``'main'``, ``'carda'`` and ``'cardb'``. This function works only in the GUI.
 * ``connected_device_uuid(storage_location_key)`` -- if a device is connected then return the device uuid (unique id), otherwise return the empty string. Each storage location on a device has a different uuid. The ``storage_location_key`` location names are ``'main'``, ``'carda'`` and ``'cardb'``. This function works only in the GUI.

src/calibre/gui2/dialogs/template_dialog.py

@@ -228,6 +228,13 @@ class TemplateHighlighter(QSyntaxHighlighter):
         self.generate_paren_positions = False
 
 
+translate_table = str.maketrans({
+    '\n': '\\n',
+    '\r': '\\r',
+    '\t': '\\t',
+    '\\': '\\\\',
+})
+
 class TemplateDialog(QDialog, Ui_TemplateDialog):
 
     def __init__(self, parent, text, mi=None, fm=None, color_field=None,
@@ -665,10 +672,9 @@ class TemplateDialog(QDialog, Ui_TemplateDialog):
             v = SafeFormat().safe_format(txt, mi, _('EXCEPTION: '),
                              mi, global_vars=self.global_vars,
                              template_functions=self.all_functions,
-                             strip_results=False,
                              break_reporter=self.break_reporter if r == break_on_mi else None)
             w = tv.cellWidget(r, 1)
-            w.setText(v.replace('\n', '\\n'))
+            w.setText(v.translate(translate_table))
             w.setCursorPosition(0)
 
     def text_cursor_changed(self):
@@ -759,7 +765,7 @@ class TemplateDialog(QDialog, Ui_TemplateDialog):
 class BreakReporterItem(QTableWidgetItem):
 
     def __init__(self, txt):
-        super().__init__(txt.replace('\n', '\\n') if txt else txt)
+        super().__init__(txt.translate(translate_table) if txt else txt)
         self.setFlags(self.flags() & ~(Qt.ItemFlag.ItemIsEditable|Qt.ItemFlag.ItemIsSelectable))
 
 

src/calibre/utils/formatter.py

@@ -45,6 +45,7 @@ class Node(object):
     NODE_BREAK = 22
     NODE_CONTINUE = 23
     NODE_RETURN = 24
+    NODE_CHARACTER = 25
 
     def __init__(self, line_number, name):
         self.my_line_number = line_number
@@ -247,6 +248,13 @@ class PrintNode(Node):
         self.arguments = arguments
 
 
+class CharacterNode(Node):
+    def __init__(self, line_number, expression):
+        Node.__init__(self, line_number, 'character')
+        self.node_type = self.NODE_CHARACTER
+        self.expression = expression
+
+
 class _Parser(object):
     LEX_OP = 1
     LEX_ID = 2
@@ -332,6 +340,13 @@ class _Parser(object):
         except:
             return False
 
+    def token_is_keyword(self):
+        self.check_eol()
+        try:
+            return self.prog[self.lex_pos][0] == self.LEX_KEYWORD
+        except:
+            return False
+
     def token_is_constant(self):
         self.check_eol()
         try:
@@ -358,7 +373,7 @@ class _Parser(object):
         self.lex_pos = 0
         self.parent = parent
         self.funcs = funcs
-        self.func_names = frozenset(set(self.funcs.keys()) | set(('newline',)))
+        self.func_names = frozenset(set(self.funcs.keys()))
         self.prog = prog[0]
         self.prog_len = len(self.prog)
         if prog[1] != '':
@@ -516,8 +531,6 @@ class _Parser(object):
 
     # {inlined_function_name: tuple(constraint on number of length, node builder) }
     inlined_function_nodes = {
-        'newline':          (lambda args: len(args) == 0,
-                             lambda ln, _: ConstantNode(ln, '\n')),
         'field':            (lambda args: len(args) == 1,
                              lambda ln, args: FieldNode(ln, args[0])),
         'raw_field':        (lambda args: len(args) == 1,
@@ -530,6 +543,8 @@ class _Parser(object):
                              lambda ln, args: AssignNode(ln, args[0].name, args[1])),
         'contains':         (lambda args: len(args) == 4,
                              lambda ln, args: ContainsNode(ln, args)),
+        'character':        (lambda args: len(args) == 1,
+                             lambda ln, args: CharacterNode(ln, args[0])),
         'print':            (lambda _: True,
                              lambda ln, args: PrintNode(ln, args)),
     }
@@ -544,13 +559,14 @@ class _Parser(object):
             return rv
 
         # Check if we have a keyword-type expression
-        t = self.token_text()
+        if self.token_is_keyword():
-        kw_tuple = self.keyword_nodes.get(t, None)
+            t = self.token_text()
-        if kw_tuple:
+            kw_tuple = self.keyword_nodes.get(t, None)
-            # These are keywords, so there can't be ambiguity between these, ids,
+            if kw_tuple:
-            # and functions.
+                # These are keywords, so there can't be ambiguity between these,
-            kw_tuple[0](self)
+                # ids, and functions.
-            return kw_tuple[1](self)
+                kw_tuple[0](self)
+                return kw_tuple[1](self)
 
         # Not a keyword. Check if we have an id reference or a function call
         if self.token_is_id():
@@ -1053,6 +1069,20 @@ class _Interpreter(object):
             self.error(_("Error during operator evaluation: "
                          "operator '{0}'").format(prog.operator), prog.line_number)
 
+    characters = {
+        'return':    '\r',
+        'newline':   '\n',
+        'tab':       '\t',
+        'backslash': '\\',
+    }
+    def do_node_character(self, prog):
+        key = self.expr(prog.expression)
+        ret = self.characters.get(key, None)
+        if ret is None:
+            self.error(_("Function {0}: invalid character name '{1}")
+                       .format('character', key), prog.line_number)
+        return ret
+
     def do_node_print(self, prog):
         res = []
         for arg in prog.arguments:
@@ -1085,6 +1115,7 @@ class _Interpreter(object):
         Node.NODE_BREAK:          do_node_break,
         Node.NODE_CONTINUE:       do_node_continue,
         Node.NODE_RETURN:         do_node_return,
+        Node.NODE_CHARACTER:      do_node_character,
         }
 
     def expr(self, prog):
@@ -1290,8 +1321,10 @@ class TemplateFormatter(string.Formatter):
                                      self.column_name, global_vars, break_reporter)
         else:
             ans = self.vformat(fmt, args, kwargs)
+            if self.strip_results:
+                ans = self.compress_spaces.sub(' ', ans)
         if self.strip_results:
-            return self.compress_spaces.sub(' ', ans).strip()
+            ans = ans.strip(' ')
         return ans
 
     # ######### a formatter that throws exceptions ############

src/calibre/utils/formatter_functions.py

@@ -2022,11 +2022,26 @@ class BuiltinFieldExists(BuiltinFormatterFunction):
         return ''
 
 
+class BuiltinCharacter(BuiltinFormatterFunction):
+    name = 'character'
+    arg_count = 1
+    category = 'String manipulation'
+    __doc__ = doc = _('character(character_name) -- returns the '
+                      'character named by character_name. For example, '
+                      "character('newline') returns a newline character ('\n'). "
+                      "The supported character names are 'newline', 'return', "
+                      "'tab', and 'backslash'.")
+
+    def evaluate(self, formatter, kwargs, mi, locals, character_name):
+        # The globals function is implemented in-line in the formatter
+        raise NotImplementedError()
+
+
 _formatter_builtins = [
     BuiltinAdd(), BuiltinAnd(), BuiltinApproximateFormats(), BuiltinArguments(),
     BuiltinAssign(),
     BuiltinAuthorLinks(), BuiltinAuthorSorts(), BuiltinBooksize(),
-    BuiltinCapitalize(), BuiltinCheckYesNo(), BuiltinCeiling(),
+    BuiltinCapitalize(), BuiltinCharacter(), BuiltinCheckYesNo(), BuiltinCeiling(),
     BuiltinCmp(), BuiltinConnectedDeviceName(), BuiltinConnectedDeviceUUID(), BuiltinContains(),
     BuiltinCount(), BuiltinCurrentLibraryName(), BuiltinCurrentLibraryPath(),
     BuiltinDaysBetween(), BuiltinDivide(), BuiltinEval(), BuiltinFirstNonEmpty(),