Merge branch 'master' of https://github.com/cbhaley/calibre

manual/template_lang.rst

@@ -226,7 +226,8 @@ General Program Mode
     times_div_op    ::= '*' | '/'
     unary_op_expr   ::= [ add_sub_op unary_op_expr ]* | expression
     expression      ::= identifier | constant | function | assignment | field_reference |
-                        if_expression | for_expression | '(' expression_list ')'
+                        if_expr | for_expr | break_expr | continue_expr |
+                        '(' expression_list ')'
     field_reference ::= '$' [ '$' ] [ '#' ] identifier
     identifier      ::= id_start [ id_rest ]*
     id_start        ::= letter | underscore
@@ -234,13 +235,15 @@ General Program Mode
     constant        ::= " string " | ' string ' | number
     function        ::= identifier '(' expression_list [ ',' expression_list ]* ')'
     assignment      ::= identifier '=' top_expression
-    if_expression   ::= 'if' condition 'then' expression_list
+    if_expr         ::= 'if' condition 'then' expression_list
-                        [ elif_expression ] [ 'else' expression_list ] 'fi'
+                        [ elif_expr ] [ 'else' expression_list ] 'fi'
     condition       ::= top_expression
-    elif_expression ::= 'elif' condition 'then' expression_list elif_expression | ''
+    elif_expr       ::= 'elif' condition 'then' expression_list elif_expr | ''
-    for_expression  ::= 'for' identifier 'in' list_expression
+    for_expr        ::= 'for' identifier 'in' list_expr
                         [ 'separator' separator_expr ] ':' expression_list 'rof'
-    list_expression ::= top_expression
+    list_expr       ::= top_expression
+    break_expr      ::= 'break'
+    continue_expr   ::= 'continue'
     separator_expr  ::= top_expression
 
 Notes:
@@ -317,7 +320,7 @@ As a last example, this program returns the value of the ``series`` column if th
 
 **For Expressions**
 
-The ``for`` expression iterates over a list of values, processing them one at a time. The ``list_expression`` must evaluate to either a metadata field ``lookup name``, for example ``tags`` or ``#genre``, or a list of values. If the result is a valid ``lookup name`` then the field's value is fetched and the separator specified for that field type is used. If the result isn't a valid lookup name then it is assumed to be a list of values. The list is assumed to be separated by commas unless the optional keyword ``separator`` is supplied, in which case the list values must be separated by the result of evaluating the ``separator_expr``. Each value in the list is assigned to the specified variable then the ``expression_list`` is evaluated.
+The ``for`` expression iterates over a list of values, processing them one at a time. The ``list_expression`` must evaluate to either a metadata field ``lookup name``, for example ``tags`` or ``#genre``, or a list of values. If the result is a valid ``lookup name`` then the field's value is fetched and the separator specified for that field type is used. If the result isn't a valid lookup name then it is assumed to be a list of values. The list is assumed to be separated by commas unless the optional keyword ``separator`` is supplied, in which case the list values must be separated by the result of evaluating the ``separator_expr``. Each value in the list is assigned to the specified variable then the ``expression_list`` is evaluated. You can use ``break`` to jump out of the loop, and ``continue`` to jump to the beginning of the loop for the next iteration.
 
 Example: This template removes the first hierarchical name for each value in Genre (``#genre``), constructing a list with the new names::
 

src/calibre/gui2/dialogs/template_dialog.py

@@ -6,6 +6,7 @@ __docformat__ = 'restructuredtext en'
 __license__   = 'GPL v3'
 
 import json, os, traceback
+from polyglot.builtins import unicode_type
 
 from qt.core import (Qt, QDialog, QDialogButtonBox, QSyntaxHighlighter, QFont,
                       QRegExp, QApplication, QTextCharFormat, QColor, QCursor,
@@ -15,15 +16,15 @@ from qt.core import (Qt, QDialog, QDialogButtonBox, QSyntaxHighlighter, QFont,
 
 from calibre import sanitize_file_name
 from calibre.constants import config_dir
-from calibre.gui2 import gprefs, error_dialog, choose_files, choose_save_file, pixmap_to_data
-from calibre.gui2.dialogs.template_dialog_ui import Ui_TemplateDialog
-from calibre.utils.formatter_functions import formatter_functions
-from calibre.utils.icu import sort_key
-from calibre.utils.localization import localize_user_manual_link
 from calibre.ebooks.metadata.book.base import Metadata
 from calibre.ebooks.metadata.book.formatter import SafeFormat
+from calibre.gui2 import gprefs, error_dialog, choose_files, choose_save_file, pixmap_to_data
+from calibre.gui2.dialogs.template_dialog_ui import Ui_TemplateDialog
 from calibre.library.coloring import (displayable_columns, color_row_key)
-from polyglot.builtins import unicode_type
+from calibre.utils.formatter_functions import formatter_functions
+from calibre.utils.formatter import StopException
+from calibre.utils.icu import sort_key
+from calibre.utils.localization import localize_user_manual_link
 
 
 class ParenPosition:
@@ -46,7 +47,7 @@ class TemplateHighlighter(QSyntaxHighlighter):
     BN_FACTOR = 1000
 
     KEYWORDS = ["program", 'if', 'then', 'else', 'elif', 'fi', 'for', 'in',
-                'separator', 'rof']
+                'separator', 'rof', 'break', 'continue']
 
     def __init__(self, parent=None, builtin_functions=None):
         super(TemplateHighlighter, self).__init__(parent)
@@ -577,7 +578,7 @@ class TemplateDialog(QDialog, Ui_TemplateDialog):
             self.break_reporter_dialog = BreakReporter(self, mi_to_use,
                                                        txt, val, locals_, line_number)
             if not self.break_reporter_dialog.exec_():
-                raise ValueError(_('Stop requested'))
+                raise StopException()
 
     def filename_button_clicked(self):
         try:

src/calibre/gui2/preferences/template_functions.py

@@ -75,16 +75,16 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         ''')
         self.textBrowser.setHtml(help_text)
         help_text = '<p>' + _('''
-        Here you can add and remove stored templates used in template processing.
+        Here you can create, edit (replace), and delete stored templates used
-        You use a stored template in another template with the '{0}' template
+        in template processing. You use a stored template in another template as
-        function, as in '{0}(some_name, arguments...)'. Stored templates must use
+        if it were a template function, for example 'some_name(arg1, arg2...)'.
-        General Program Mode -- they must begin with the text '{1}'.
+        Stored templates must use General Program Mode -- they must begin with
-        In the stored template you retrieve the arguments using the '{2}()'
+        the text '{0}'. You retrieve arguments passed to a stored template using
-        template function, as in '{2}(var1, var2, ...)'. The calling arguments
+        the '{1}()' template function, as in '{1}(var1, var2, ...)'. The passed
-        are copied to the named variables. See the template language tutorial
+        arguments are copied to the named variables. See the template language
-        for more information.
+        tutorial for more information.
         ''') + '</p>'
-        self.st_textBrowser.setHtml(help_text.format('call', 'program:', 'arguments'))
+        self.st_textBrowser.setHtml(help_text.format('program:', 'arguments'))
         self.st_textBrowser.adjustSize()
 
     def initialize(self):

src/calibre/utils/formatter.py

@@ -41,6 +41,8 @@ class Node(object):
     NODE_BINARY_ARITHOP = 19
     NODE_UNARY_ARITHOP = 20
     NODE_PRINT = 21
+    NODE_BREAK = 22
+    NODE_CONTINUE = 23
 
     def __init__(self, line_number, name):
         self.my_line_number = line_number
@@ -74,6 +76,18 @@ class ForNode(Node):
         self.block = block
 
 
+class BreakNode(Node):
+    def __init__(self, line_number):
+        Node.__init__(self, line_number, 'break')
+        self.node_type = self.NODE_BREAK
+
+
+class ContinueNode(Node):
+    def __init__(self, line_number):
+        Node.__init__(self, line_number, 'continue')
+        self.node_type = self.NODE_CONTINUE
+
+
 class AssignNode(Node):
     def __init__(self, line_number, left, right):
         Node.__init__(self, line_number, 'assign to ' + left)
@@ -477,6 +491,22 @@ class _Parser(object):
         except:
             return False
 
+    def token_is_break(self):
+        self.check_eol()
+        try:
+            token = self.prog[self.lex_pos]
+            return token[1] == 'break' and token[0] == self.LEX_KEYWORD
+        except:
+            return False
+
+    def token_is_continue(self):
+        self.check_eol()
+        try:
+            token = self.prog[self.lex_pos]
+            return token[1] == 'continue' and token[0] == self.LEX_KEYWORD
+        except:
+            return False
+
     def token_is_constant(self):
         self.check_eol()
         try:
@@ -649,6 +679,12 @@ class _Parser(object):
             return self.if_expression()
         if self.token_is_for():
             return self.for_expression()
+        if self.token_is_break():
+            self.consume()
+            return BreakNode(self.line_number)
+        if self.token_is_continue():
+            self.consume()
+            return ContinueNode(self.line_number)
         if self.token_is_id():
             line_number = self.line_number
             id_ = self.token()
@@ -724,6 +760,33 @@ class _Parser(object):
             self.error(_('Expression is not function or constant'))
 
 
+class ExecutionBase(Exception):
+    def __init__(self, name):
+        super().__init__(_('{0} outside of for loop').format(name))
+        self.value = ''
+
+    def set_value(self, v):
+        self.value = v
+
+    def get_value(self):
+        return self.value
+
+
+class ContinueExecuted(ExecutionBase):
+    def __init__(self):
+        super().__init__('continue')
+
+
+class BreakExecuted(ExecutionBase):
+    def __init__(self):
+        super().__init__('break')
+
+
+class StopException(Exception):
+    def __init__(self):
+        super().__init__('Template evaluation stopped')
+
+
 class _Interpreter(object):
     def error(self, message, line_number):
         m = _('Interpreter: {0} - line number {1}').format(message, line_number)
@@ -752,8 +815,12 @@ class _Interpreter(object):
 
     def expression_list(self, prog):
         val = ''
-        for p in prog:
+        try:
-            val = self.expr(p)
+            for p in prog:
+                val = self.expr(p)
+        except (BreakExecuted, ContinueExecuted) as e:
+            e.set_value(val)
+            raise e
         return val
 
     INFIX_STRING_COMPARE_OPS = {
@@ -774,6 +841,8 @@ class _Interpreter(object):
             if (self.break_reporter):
                 self.break_reporter(prog.node_name, res, prog.line_number)
             return res
+        except (StopException, ValueError) as e:
+            raise e
         except:
             self.error(
                _('Error during string comparison. Operator {0}').format(prog.operator),
@@ -801,6 +870,8 @@ class _Interpreter(object):
             if (self.break_reporter):
                 self.break_reporter(prog.node_name, res, prog.line_number)
             return res
+        except (StopException, ValueError) as e:
+            raise e
         except:
             self.error(
                _('Value used in comparison is not a number. Operator {0}').format(prog.operator),
@@ -901,7 +972,7 @@ class _Interpreter(object):
             except:
                 self.error(_('Unknown field {0}').format(name),
                            prog.line_number)
-        except ValueError as e:
+        except (StopException, ValueError) as e:
             raise e
         except:
             self.error(_('Unknown field {0}').format('internal parse error'),
@@ -930,7 +1001,7 @@ class _Interpreter(object):
             if (self.break_reporter):
                 self.break_reporter(prog.node_name, res, prog.line_number)
             return res
-        except ValueError as e:
+        except (StopException, ValueError) as e:
             raise e
         except:
             self.error(_('Unknown field {0}').format('internal parse error'))
@@ -965,9 +1036,15 @@ class _Interpreter(object):
                 ret = ''
                 if self.break_reporter:
                     self.break_reporter("'for' list value", separator.join(res), line_number)
-                for x in res:
+                try:
-                    self.locals[v] = x
+                    for x in res:
-                    ret = self.expression_list(prog.block)
+                        try:
+                            self.locals[v] = x
+                            ret = self.expression_list(prog.block)
+                        except ContinueExecuted as e:
+                            ret = e.get_value()
+                except BreakExecuted as e:
+                    ret = e.get_value()
                 if (self.break_reporter):
                     self.break_reporter("'for' block value", ret, line_number)
             elif self.break_reporter:
@@ -975,11 +1052,21 @@ class _Interpreter(object):
                 self.break_reporter("'for' list value", '', line_number)
                 ret = ''
             return ret
-        except ValueError as e:
+        except (StopException, ValueError) as e:
             raise e
         except Exception as e:
             self.error(_('Unhandled exception {0}').format(e), line_number)
 
+    def do_node_break(self, prog):
+        if (self.break_reporter):
+            self.break_reporter(prog.node_name, '', prog.line_number)
+        raise BreakExecuted()
+
+    def do_node_continue(self, prog):
+        if (self.break_reporter):
+            self.break_reporter(prog.node_name, '', prog.line_number)
+        raise ContinueExecuted()
+
     def do_node_contains(self, prog):
         v = self.expr(prog.value_expression)
         t = self.expr(prog.test_expression)
@@ -1002,6 +1089,8 @@ class _Interpreter(object):
             if (self.break_reporter):
                 self.break_reporter(prog.node_name, res, prog.line_number)
             return res
+        except (StopException, ValueError) as e:
+            raise e
         except:
             self.error(
                _('Error during operator evaluation. Operator {0}').format(prog.operator),
@@ -1018,6 +1107,8 @@ class _Interpreter(object):
             if (self.break_reporter):
                 self.break_reporter(prog.node_name, res, prog.line_number)
             return res
+        except (StopException, ValueError) as e:
+            raise e
         except:
             self.error(
                _('Error during operator evaluation. Operator {0}').format(prog.operator),
@@ -1038,6 +1129,8 @@ class _Interpreter(object):
             if (self.break_reporter):
                 self.break_reporter(prog.node_name, res, prog.line_number)
             return res
+        except (StopException, ValueError) as e:
+            raise e
         except:
             self.error(
                _('Error during operator evaluation. Operator {0}').format(prog.operator),
@@ -1055,6 +1148,8 @@ class _Interpreter(object):
             if (self.break_reporter):
                 self.break_reporter(prog.node_name, res, prog.line_number)
             return res
+        except (StopException, ValueError) as e:
+            raise e
         except:
             self.error(
                _('Error during operator evaluation. Operator {0}').format(prog.operator),
@@ -1089,6 +1184,8 @@ class _Interpreter(object):
         Node.NODE_BINARY_ARITHOP: do_node_binary_arithop,
         Node.NODE_UNARY_ARITHOP:  do_node_unary_arithop,
         Node.NODE_PRINT:          do_node_print,
+        Node.NODE_BREAK:          do_node_break,
+        Node.NODE_CONTINUE:       do_node_continue,
         }
 
     def expr(self, prog):
@@ -1096,7 +1193,7 @@ class _Interpreter(object):
             if isinstance(prog, list):
                 return self.expression_list(prog)
             return self.NODE_OPS[prog.node_type](self, prog)
-        except ValueError as e:
+        except (ValueError, ContinueExecuted, BreakExecuted, StopException) as e:
             raise e
         except:
             if (DEBUG):
@@ -1175,6 +1272,7 @@ class TemplateFormatter(string.Formatter):
             (r'(==|!=|<=|<|>=|>)',       lambda x,t: (_Parser.LEX_STRING_INFIX, t)),  # noqa
             (r'(if|then|else|elif|fi)\b',lambda x,t: (_Parser.LEX_KEYWORD, t)),  # noqa
             (r'(for|in|rof)\b',          lambda x,t: (_Parser.LEX_KEYWORD, t)),  # noqa
+            (r'(break|continue)\b',      lambda x,t: (_Parser.LEX_KEYWORD, t)),  # noqa
             (r'(\|\||&&|!)',             lambda x,t: (_Parser.LEX_OP, t)),  # noqa
             (r'[(),=;:\+\-*/]',          lambda x,t: (_Parser.LEX_OP, t)),  # noqa
             (r'-?[\d\.]+',               lambda x,t: (_Parser.LEX_CONST, t)),  # noqa
@@ -1328,6 +1426,8 @@ class TemplateFormatter(string.Formatter):
         try:
             ans = self.evaluate(fmt, [], kwargs, self.global_vars,
                                 break_reporter=break_reporter)
+        except StopException as e:
+            ans = error_message(e)
         except Exception as e:
             if DEBUG:  # and getattr(e, 'is_locking_error', False):
                 traceback.print_exc()