Merge branch 'master' of https://github.com/cbhaley/calibre

manual/template_lang.rst

@@ -481,6 +481,9 @@ parameters can be statements (sequences of expressions). Note that the definitiv
       If `opt_replace` is not the empty string, then apply the replacement before adding the item to the returned list.
     * ``list_re_group(src_list, separator, include_re, search_re, template_for_group_1, for_group_2, ...)`` -- Like list_re except
       replacements are not optional. It uses re_group(item, search_re, template ...) when doing the replacements.
+    * ``list_remove_duplicates(list, separator)`` -- return a list made by removing duplicate items in the source list. If items
+      differ only in case, the last of them is returned. The items in source list are separated by separator, as are
+      the items in the returned list.
     * ``list_sort(list, direction, separator)`` -- return list sorted using a case-insensitive sort. If ``direction`` is zero, ``list`` is
       sorted ascending, otherwise descending. The list items are separated by separator, as are the items in the returned list.
     * ``list_union(list1, list2, separator)`` -- return a list made by merging the items in ``list1`` and ``list2``, removing
@@ -544,9 +547,9 @@ parameters can be statements (sequences of expressions). Note that the definitiv
 Using General Program Mode
 -----------------------------------
 
-For more complicated template programs it is often easier to avoid template syntax (all the `{` and `}` characters), instead writing a more
+For more complicated template programs it is often easier to avoid template syntax (all the `{` and `}` characters), instead writing
-classic-looking program. You can do this by beginning the template with `program:`. The template program is compiled and executed. No template
+a more classic-looking program. You can do this by beginning the template with `program:`. The template program is compiled
-processing (e.g., formatting, prefixes, suffixes) is done. The special variable `$` is not set.
+and executed. No template processing (e.g., formatting, prefixes, suffixes) is done. The special variable `$` is not set.
 
 One advantage of `program:` mode is that braces are no longer special. For example, it is not necessary to use `[[` and `]]` when using the
 `template()` function. Another advantage is readability.
@@ -601,14 +604,15 @@ An ``if`` produces a value like any other language expression. This means that a
 
 The template language supports **``for`` expressions** with the following syntax::
 
-    for <<id>> in <<expression>>:
+    for <<id>> in <<expression>> [separator <<expression>>]:
         <<expression_list>>
     rof
 
-The expression must evaluate to either a metadata field lookup key, for example ``tags`` or ``#genre``, or a comma-separated list of
+The expression must evaluate to either a metadata field lookup key, for example ``tags`` or ``#genre``, or a list of
-values. If the result is a valid lookup name then the field's value is fetched, otherwise the list is broken into its
+values. If the result is a valid lookup name then the field's value is fetched and the separator specified for that field type
-individual values. Each resulting value in the list is assigned to the variable ``id`` then the ``expression_list``
+is used. If the result isn't a valid lookup name then it is assumed to be a list of values. If the optional keyword ``separator``
-is evaluated.
+is supplied then the list values must be separated by the result of evaluating the second ``expression``. If the separator is not specified then the list values must be separated by commas. Each resulting value in the list is assigned to the
+variable ``id`` then the ``expression_list`` is evaluated.
 
 Example: This template removes the first hierarchical name for each value in Genre (``#genre``), constructing a list with
 the new names::

src/calibre/utils/formatter.py

@@ -47,11 +47,12 @@ class IfNode(Node):
 
 
 class ForNode(Node):
-    def __init__(self, variable, list_field_expr, block):
+    def __init__(self, variable, list_field_expr, separator, block):
         Node.__init__(self)
         self.node_type = self.NODE_FOR
         self.variable = variable
         self.list_field_expr = list_field_expr
+        self.separator = separator
         self.block = block
 
 
@@ -319,6 +320,13 @@ class _Parser(object):
         except:
             return False
 
+    def token_is_separator(self):
+        try:
+            token = self.prog[self.lex_pos]
+            return token[1] == 'separator' and token[0] == self.LEX_ID
+        except:
+            return False
+
     def token_is_constant(self):
         try:
             return self.prog[self.lex_pos][0] == self.LEX_CONST
@@ -382,6 +390,11 @@ class _Parser(object):
             self.error(_("Missing 'in' in for statement"))
         self.consume()
         list_expr = self.infix_expr()
+        if self.token_is_separator():
+            self.consume()
+            separator = self.expr()
+        else:
+            separator = None
         if not self.token_op_is_colon():
             self.error(_("Missing colon (':') in for statement"))
         self.consume()
@@ -389,7 +402,7 @@ class _Parser(object):
         if not self.token_is_rof():
             self.error(_("Missing 'rof' in for statement"))
         self.consume()
-        return ForNode(variable, list_expr, block)
+        return ForNode(variable, list_expr, separator, block)
 
     def infix_expr(self):
         left = self.expr()
@@ -645,12 +658,13 @@ class _Interpreter(object):
 
     def do_node_for(self, prog):
         try:
+            separator = ',' if prog.separator is None else self.expr(prog.separator)
             v = prog.variable
             f = self.expr(prog.list_field_expr)
             res = getattr(self.parent_book, f, f)
             if res is not None:
                 if not isinstance(res, list):
-                    res = [r.strip() for r in res.split(',') if r.strip()]
+                    res = [r.strip() for r in res.split(separator) if r.strip()]
                 ret = ''
                 for x in res:
                     self.locals[v] = x

src/calibre/utils/formatter_functions.py

@@ -1361,6 +1361,25 @@ class BuiltinListUnion(BuiltinFormatterFunction):
     def evaluate(self, formatter, kwargs, mi, locals, list1, list2, separator):
         res = {icu_lower(l.strip()): l.strip() for l in list2.split(separator) if l.strip()}
         res.update({icu_lower(l.strip()): l.strip() for l in list1.split(separator) if l.strip()})
+        if separator == ',':
+            separator = ', '
+        return separator.join(res.values())
+
+
+class BuiltinListRemoveDuplicates(BuiltinFormatterFunction):
+    name = 'list_remove_duplicates'
+    arg_count = 2
+    category = 'List manipulation'
+    __doc__ = doc = _('list_remove_duplicates(list, separator) -- '
+            'return a list made by removing duplicate items in the source list. '
+            'If items differ only in case, the last of them is returned. '
+            'The items in source list are separated by separator, as are '
+            'the items in the returned list.')
+
+    def evaluate(self, formatter, kwargs, mi, locals, list_, separator):
+        res = {icu_lower(l.strip()): l.strip() for l in list_.split(separator) if l.strip()}
+        if separator == ',':
+            separator = ', '
         return separator.join(res.values())
 
 
@@ -1971,8 +1990,8 @@ _formatter_builtins = [
     BuiltinIfempty(), BuiltinLanguageCodes(), BuiltinLanguageStrings(),
     BuiltinInList(), BuiltinIsMarked(), BuiltinListDifference(), BuiltinListEquals(),
     BuiltinListIntersection(), BuiltinListitem(), BuiltinListRe(),
-    BuiltinListReGroup(), BuiltinListSort(), BuiltinListSplit(), BuiltinListUnion(),
+    BuiltinListReGroup(), BuiltinListRemoveDuplicates(), BuiltinListSort(),
-    BuiltinLookup(),
+    BuiltinListSplit(), BuiltinListUnion(),BuiltinLookup(),
     BuiltinLowercase(), BuiltinMod(), BuiltinMultiply(), BuiltinNot(), BuiltinOndevice(),
     BuiltinOr(), BuiltinPrint(), BuiltinRatingToStars(), BuiltinRawField(), BuiltinRawList(),
     BuiltinRe(), BuiltinReGroup(), BuiltinRound(), BuiltinSelect(), BuiltinSeriesSort(),