Various improvements:

1) Changes to the template program language discussed in https://www.mobileread.com/forums/showthread.php?t=337668
2) General improvement of the template documentation, including documentation of the above changes. I looked at the changes using a markdown interpreter, but there might be problems exposed by generation of the web page.
3) Focus the program text box when opening the template dialog
4) Small changes to non-built-in template functions to improve performance

src/calibre/gui2/dialogs/template_dialog.py

@@ -372,6 +372,7 @@ class TemplateDialog(QDialog, Ui_TemplateDialog):
 
         self.font_size_box.setValue(gprefs['gpm_template_editor_font_size'])
         self.font_size_box.valueChanged.connect(self.font_size_changed)
+        self.textbox.setFocus()
 
     def font_size_changed(self, toWhat):
         gprefs['gpm_template_editor_font_size'] = toWhat

src/calibre/utils/formatter.py

@@ -10,6 +10,7 @@ __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 import re, string, traceback, numbers
+from math import modf
 
 from calibre import prints
 from calibre.constants import DEBUG
@@ -23,8 +24,8 @@ class Node(object):
     NODE_IF = 2
     NODE_ASSIGN = 3
     NODE_FUNC = 4
-    NODE_STRING_INFIX = 5
+    NODE_COMPARE_STRING = 5
-    NODE_NUMERIC_INFIX = 6
+    NODE_COMPARE_NUMERIC = 6
     NODE_CONSTANT = 7
     NODE_FIELD = 8
     NODE_RAW_FIELD = 9
@@ -35,6 +36,10 @@ class Node(object):
     NODE_GLOBALS = 14
     NODE_SET_GLOBALS = 15
     NODE_CONTAINS = 16
+    NODE_BINARY_LOGOP = 17
+    NODE_UNARY_LOGOP = 18
+    NODE_BINARY_ARITHOP = 19
+    NODE_UNARY_ARITHOP = 20
 
 
 class IfNode(Node):
@@ -101,24 +106,58 @@ class SetGlobalsNode(Node):
         self.expression_list = expression_list
 
 
-class StringInfixNode(Node):
+class StringCompareNode(Node):
     def __init__(self, operator, left, right):
         Node.__init__(self)
-        self.node_type = self.NODE_STRING_INFIX
+        self.node_type = self.NODE_COMPARE_STRING
         self.operator = operator
         self.left = left
         self.right = right
 
 
-class NumericInfixNode(Node):
+class NumericCompareNode(Node):
     def __init__(self, operator, left, right):
         Node.__init__(self)
-        self.node_type = self.NODE_NUMERIC_INFIX
+        self.node_type = self.NODE_COMPARE_NUMERIC
         self.operator = operator
         self.left = left
         self.right = right
 
 
+class LogopBinaryNode(Node):
+    def __init__(self, operator, left, right):
+        Node.__init__(self)
+        self.node_type = self.NODE_BINARY_LOGOP
+        self.operator = operator
+        self.left = left
+        self.right = right
+
+
+class LogopUnaryNode(Node):
+    def __init__(self, operator, expr):
+        Node.__init__(self)
+        self.node_type = self.NODE_UNARY_LOGOP
+        self.operator = operator
+        self.expr = expr
+
+
+class NumericBinaryNode(Node):
+    def __init__(self, operator, left, right):
+        Node.__init__(self)
+        self.node_type = self.NODE_BINARY_ARITHOP
+        self.operator = operator
+        self.left = left
+        self.right = right
+
+
+class NumericUnaryNode(Node):
+    def __init__(self, operator, expr):
+        Node.__init__(self)
+        self.node_type = self.NODE_UNARY_ARITHOP
+        self.operator = operator
+        self.expr = expr
+
+
 class ConstantNode(Node):
     def __init__(self, value):
         Node.__init__(self)
@@ -252,6 +291,55 @@ class _Parser(object):
         except:
             return False
 
+    def token_op_is_plus(self):
+        try:
+            token = self.prog[self.lex_pos]
+            return token[1] == '+' and token[0] == self.LEX_OP
+        except:
+            return False
+
+    def token_op_is_minus(self):
+        try:
+            token = self.prog[self.lex_pos]
+            return token[1] == '-' and token[0] == self.LEX_OP
+        except:
+            return False
+
+    def token_op_is_times(self):
+        try:
+            token = self.prog[self.lex_pos]
+            return token[1] == '*' and token[0] == self.LEX_OP
+        except:
+            return False
+
+    def token_op_is_divide(self):
+        try:
+            token = self.prog[self.lex_pos]
+            return token[1] == '/' and token[0] == self.LEX_OP
+        except:
+            return False
+
+    def token_op_is_and(self):
+        try:
+            token = self.prog[self.lex_pos]
+            return token[1] == '&&' and token[0] == self.LEX_OP
+        except:
+            return False
+
+    def token_op_is_or(self):
+        try:
+            token = self.prog[self.lex_pos]
+            return token[1] == '||' and token[0] == self.LEX_OP
+        except:
+            return False
+
+    def token_op_is_not(self):
+        try:
+            token = self.prog[self.lex_pos]
+            return token[1] == '!' and token[0] == self.LEX_OP
+        except:
+            return False
+
     def token_is_id(self):
         try:
             return self.prog[self.lex_pos][0] == self.LEX_ID
@@ -357,7 +445,7 @@ class _Parser(object):
     def expression_list(self):
         expr_list = []
         while not self.token_is_eof():
-            expr_list.append(self.infix_expr())
+            expr_list.append(self.top_expr())
             if not self.token_op_is_semicolon():
                 break
             self.consume()
@@ -365,7 +453,7 @@ class _Parser(object):
 
     def if_expression(self):
         self.consume()
-        condition = self.infix_expr()
+        condition = self.top_expr()
         if not self.token_is_then():
             self.error(_("Missing 'then' in if statement"))
         self.consume()
@@ -390,7 +478,7 @@ class _Parser(object):
         if not self.token_is_in():
             self.error(_("Missing 'in' in for statement"))
         self.consume()
-        list_expr = self.infix_expr()
+        list_expr = self.top_expr()
         if self.token_is_separator():
             self.consume()
             separator = self.expr()
@@ -405,16 +493,66 @@ class _Parser(object):
         self.consume()
         return ForNode(variable, list_expr, separator, block)
 
-    def infix_expr(self):
+    def top_expr(self):
-        left = self.expr()
+        return self.or_expr()
+
+    def or_expr(self):
+        left = self.and_expr()
+        while self.token_op_is_or():
+            self.consume()
+            right = self.and_expr()
+            left = LogopBinaryNode('or', left, right)
+        return left
+
+    def and_expr(self):
+        left = self.not_expr()
+        while self.token_op_is_and():
+            self.consume()
+            right = self.not_expr()
+            left = LogopBinaryNode('and', left, right)
+        return left
+
+    def not_expr(self):
+        if self.token_op_is_not():
+            self.consume()
+            return LogopUnaryNode('not', self.not_expr())
+        return self.compare_expr()
+
+    def compare_expr(self):
+        left = self.add_subtract_expr()
         if self.token_op_is_string_infix_compare() or self.token_is_in():
             operator = self.token()
-            return StringInfixNode(operator, left, self.expr())
+            return StringCompareNode(operator, left, self.add_subtract_expr())
         if self.token_op_is_numeric_infix_compare():
             operator = self.token()
-            return NumericInfixNode(operator, left, self.expr())
+            return NumericCompareNode(operator, left, self.add_subtract_expr())
         return left
 
+    def add_subtract_expr(self):
+        left = self.times_divide_expr()
+        while self.token_op_is_plus() or self.token_op_is_minus():
+            operator = self.token()
+            right = self.times_divide_expr()
+            left = NumericBinaryNode(operator, left, right)
+        return left
+
+    def times_divide_expr(self):
+        left = self.unary_plus_minus_expr()
+        while self.token_op_is_times() or self.token_op_is_divide():
+            operator = self.token()
+            right = self.unary_plus_minus_expr()
+            left = NumericBinaryNode(operator, left, right)
+        return left
+
+    def unary_plus_minus_expr(self):
+        if self.token_op_is_plus():
+            self.consume()
+            return NumericUnaryNode('+', self.unary_plus_minus_expr())
+        if self.token_op_is_minus():
+            self.consume()
+            return NumericUnaryNode('-', self.unary_plus_minus_expr())
+        return self.expr()
+
     def call_expression(self, name, arguments):
         subprog = self.funcs[name].cached_parse_tree
         if subprog is None:
@@ -428,6 +566,13 @@ class _Parser(object):
         return CallNode(subprog, arguments)
 
     def expr(self):
+        if self.token_op_is_lparen():
+            self.consume()
+            rv = self.top_expr()
+            if not self.token_op_is_rparen():
+                self.error(_('Missing )'))
+            self.consume()
+            return rv
         if self.token_is_if():
             return self.if_expression()
         if self.token_is_for():
@@ -439,7 +584,7 @@ class _Parser(object):
                 if self.token_op_is_equals():
                     # classic assignment statement
                     self.consume()
-                    return AssignNode(id_, self.infix_expr())
+                    return AssignNode(id_, self.top_expr())
                 return VariableNode(id_)
 
             # We have a function.
@@ -453,7 +598,7 @@ class _Parser(object):
             arguments = list()
             while not self.token_op_is_rparen():
                 # evaluate the expression (recursive call)
-                arguments.append(self.infix_expr())
+                arguments.append(self.top_expr())
                 if not self.token_op_is_comma():
                     break
                 self.consume()
@@ -520,7 +665,7 @@ class _Interpreter(object):
             val = self.expr(p)
         return val
 
-    INFIX_STRING_OPS = {
+    INFIX_STRING_COMPARE_OPS = {
         "==": lambda x, y: strcmp(x, y) == 0,
         "!=": lambda x, y: strcmp(x, y) != 0,
         "<":  lambda x, y: strcmp(x, y) < 0,
@@ -534,11 +679,11 @@ class _Interpreter(object):
         try:
             left = self.expr(prog.left)
             right = self.expr(prog.right)
-            return ('1' if self.INFIX_STRING_OPS[prog.operator](left, right) else '')
+            return ('1' if self.INFIX_STRING_COMPARE_OPS[prog.operator](left, right) else '')
         except:
             self.error(_('Error during string comparison. Operator {0}').format(prog.operator))
 
-    INFIX_NUMERIC_OPS = {
+    INFIX_NUMERIC_COMPARE_OPS = {
         "==#": lambda x, y: x == y,
         "!=#": lambda x, y: x != y,
         "<#":  lambda x, y: x < y,
@@ -556,7 +701,7 @@ class _Interpreter(object):
         try:
             left = self.float_deal_with_none(self.expr(prog.left))
             right = self.float_deal_with_none(self.expr(prog.right))
-            return '1' if self.INFIX_NUMERIC_OPS[prog.operator](left, right) else ''
+            return '1' if self.INFIX_NUMERIC_COMPARE_OPS[prog.operator](left, right) else ''
         except:
             self.error(_('Value used in comparison is not a number. Operator {0}').format(prog.operator))
 
@@ -687,6 +832,55 @@ class _Interpreter(object):
             return self.expr(prog.match_expression)
         return self.expr(prog.not_match_expression)
 
+    LOGICAL_BINARY_OPS = {
+        'and':      lambda self, x, y: self.expr(x) and self.expr(y),
+        'or':       lambda self, x, y: self.expr(x) or self.expr(y),
+    }
+
+    def do_node_logop(self, prog):
+        try:
+            return ('1' if self.LOGICAL_BINARY_OPS[prog.operator](self, prog.left, prog.right) else '')
+        except:
+            self.error(_('Error during operator evaluation. Operator {0}').format(prog.operator))
+
+    LOGICAL_UNARY_OPS = {
+        'not':       lambda x: not x,
+    }
+
+    def do_node_logop_unary(self, prog):
+        try:
+            expr = self.expr(prog.expr)
+            return ('1' if self.LOGICAL_UNARY_OPS[prog.operator](expr) else '')
+        except:
+            self.error(_('Error during operator evaluation. Operator {0}').format(prog.operator))
+
+    ARITHMETIC_BINARY_OPS = {
+        '+': lambda x, y: x + y,
+        '-': lambda x, y: x - y,
+        '*': lambda x, y: x * y,
+        '/': lambda x, y: x / y,
+    }
+
+    def do_node_binary_arithop(self, prog):
+        try:
+            answer = self.ARITHMETIC_BINARY_OPS[prog.operator](float(self.expr(prog.left)),
+                                                               float(self.expr(prog.right)))
+            return unicode_type(answer if modf(answer)[0] != 0 else int(answer))
+        except:
+            self.error(_('Error during arithmetic operator evaluation. Operator {0}').format(prog.operator))
+
+    ARITHMETIC_UNARY_OPS = {
+        '+': lambda x: x,
+        '-': lambda x: -x,
+    }
+
+    def do_node_unary_arithop(self, prog):
+        try:
+            expr = self.ARITHMETIC_UNARY_OPS[prog.operator](float(self.expr(prog.expr)))
+            return unicode_type(expr if modf(expr)[0] != 0 else int(expr))
+        except:
+            self.error(_('Error during arithmetic operator evaluation. Operator {0}').format(prog.operator))
+
     NODE_OPS = {
         Node.NODE_IF:             do_node_if,
         Node.NODE_ASSIGN:         do_node_assign,
@@ -695,8 +889,8 @@ class _Interpreter(object):
         Node.NODE_FUNC:           do_node_func,
         Node.NODE_FIELD:          do_node_field,
         Node.NODE_RAW_FIELD:      do_node_raw_field,
-        Node.NODE_STRING_INFIX:   do_node_string_infix,
+        Node.NODE_COMPARE_STRING: do_node_string_infix,
-        Node.NODE_NUMERIC_INFIX:  do_node_numeric_infix,
+        Node.NODE_COMPARE_NUMERIC:do_node_numeric_infix,
         Node.NODE_ARGUMENTS:      do_node_arguments,
         Node.NODE_CALL:           do_node_call,
         Node.NODE_FIRST_NON_EMPTY:do_node_first_non_empty,
@@ -704,6 +898,10 @@ class _Interpreter(object):
         Node.NODE_GLOBALS:        do_node_globals,
         Node.NODE_SET_GLOBALS:    do_node_set_globals,
         Node.NODE_CONTAINS:       do_node_contains,
+        Node.NODE_BINARY_LOGOP:   do_node_logop,
+        Node.NODE_UNARY_LOGOP:    do_node_logop_unary,
+        Node.NODE_BINARY_ARITHOP: do_node_binary_arithop,
+        Node.NODE_UNARY_ARITHOP:  do_node_unary_arithop,
         }
 
     def expr(self, prog):
@@ -781,14 +979,15 @@ class TemplateFormatter(string.Formatter):
                 (r'.*?\)', lambda x,t: t[:-1]),
         ])
 
-    # ################# 'Functional' template language ######################
+    # ################# Template language lexical analyzer ######################
 
     lex_scanner = re.Scanner([
             (r'(==#|!=#|<=#|<#|>=#|>#)', lambda x,t: (_Parser.LEX_NUMERIC_INFIX, t)),
             (r'(==|!=|<=|<|>=|>)',       lambda x,t: (_Parser.LEX_STRING_INFIX, t)),  # noqa
             (r'(if|then|else|elif|fi)\b',lambda x,t: (_Parser.LEX_KEYWORD, t)),  # noqa
             (r'(for|in|rof)\b',          lambda x,t: (_Parser.LEX_KEYWORD, t)),  # noqa
-            (r'[(),=;:]',                lambda x,t: (_Parser.LEX_OP, t)),  # noqa
+            (r'(\|\||&&|!)',             lambda x,t: (_Parser.LEX_OP, t)),  # noqa
+            (r'[(),=;:\+\-*/]',          lambda x,t: (_Parser.LEX_OP, t)),  # noqa
             (r'-?[\d\.]+',               lambda x,t: (_Parser.LEX_CONST, t)),  # noqa
             (r'\$',                      lambda x,t: (_Parser.LEX_ID, t)),  # noqa
             (r'\w+',                     lambda x,t: (_Parser.LEX_ID, t)),  # noqa

src/calibre/utils/formatter_functions.py

@@ -884,9 +884,10 @@ class BuiltinSelect(BuiltinFormatterFunction):
         if not val:
             return ''
         vals = [v.strip() for v in val.split(',')]
+        tkey = key+':'
         for v in vals:
-            if v.startswith(key+':'):
+            if v.startswith(tkey):
-                return v[len(key)+1:]
+                return v[len(tkey):]
         return ''
 
 
@@ -1096,7 +1097,7 @@ class BuiltinSubitems(BuiltinFormatterFunction):
         si = int(start_index)
         ei = int(end_index)
         has_periods = '.' in val
-        items = [v.strip() for v in val.split(',')]
+        items = [v.strip() for v in val.split(',') if v.strip()]
         rv = set()
         for item in items:
             if has_periods and '.' in item: