Add a tweak to control which custom columns are displayed in the Book details panel. Implement a more sophisticated 'functional programming' template language. See the User Manual for details.

resources/default_tweaks.py

@@ -181,19 +181,25 @@ max_content_server_tags_shown=5
 # content_server_will_display is a list of custom fields to be displayed.
 # content_server_wont_display is a list of custom fields not to be displayed.
 # wont_display has priority over will_display.
-# The special value '*' means all custom fields.
+# The special value '*' means all custom fields. The value [] means no entries.
 # Defaults:
 #    content_server_will_display = ['*']
-#    content_server_wont_display = ['']
+#    content_server_wont_display = []
 # Examples:
 # To display only the custom fields #mytags and #genre:
 #   content_server_will_display = ['#mytags', '#genre']
-#   content_server_wont_display = ['']
+#   content_server_wont_display = []
 # To display all fields except #mycomments:
 #   content_server_will_display = ['*']
 #   content_server_wont_display['#mycomments']
 content_server_will_display = ['*']
-content_server_wont_display = ['']
+content_server_wont_display = []
+
+# Same as above (content server) but for the book details pane. Same syntax.
+# As above, this tweak affects only display of custom fields. The standard
+# fields are not affected
+book_details_will_display = ['*']
+book_details_wont_display = []
 
 
 # Set the maximum number of sort 'levels' that calibre will use to resort the

resources/recipes/toyokeizai.recipe

@@ -64,5 +64,5 @@ class Toyokeizai(BasicNewsRecipe):
             br.select_form(nr=0)
             br['kaiin_id']   = self.username
             br['password'] = self.password
-            res = br.submit()
+            br.submit()
         return br

src/calibre/gui2/dialogs/comments_dialog.py

@@ -3,7 +3,7 @@ __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __license__   = 'GPL v3'
 
-from PyQt4.Qt import Qt, QDialog
+from PyQt4.Qt import Qt, QDialog, QDialogButtonBox
 from calibre.gui2.dialogs.comments_dialog_ui import Ui_CommentsDialog
 
 class CommentsDialog(QDialog, Ui_CommentsDialog):
@@ -20,3 +20,6 @@ class CommentsDialog(QDialog, Ui_CommentsDialog):
         if text is not None:
             self.textbox.setPlainText(text)
         self.textbox.setTabChangesFocus(True)
+        self.buttonBox.button(QDialogButtonBox.Ok).setText(_('&OK'))
+        self.buttonBox.button(QDialogButtonBox.Cancel).setText(_('&Cancel'))
+

src/calibre/gui2/dialogs/comments_dialog.ui

@@ -19,15 +19,6 @@
   <property name="windowTitle">
    <string>Edit Comments</string>
   </property>
-  <widget class="QWidget" name="verticalLayoutWidget">
-   <property name="geometry">
-    <rect>
-     <x>10</x>
-     <y>10</y>
-     <width>311</width>
-     <height>211</height>
-    </rect>
-   </property>
    <layout class="QVBoxLayout" name="verticalLayout">
     <item>
      <widget class="QPlainTextEdit" name="textbox"/>
@@ -43,7 +34,6 @@
      </widget>
     </item>
    </layout>
-  </widget>
  </widget>
  <resources/>
  <connections>

src/calibre/gui2/library/delegates.py

@@ -349,10 +349,19 @@ class CcTemplateDelegate(QStyledItemDelegate): # {{{
         QStyledItemDelegate.__init__(self, parent)
 
     def createEditor(self, parent, option, index):
-        return EnLineEdit(parent)
+        m = index.model()
+        text = m.custom_columns[m.column_map[index.column()]]['display']['composite_template']
+        editor = CommentsDialog(parent, text)
+        editor.setWindowTitle(_("Edit template"))
+        editor.textbox.setTabChangesFocus(False)
+        editor.textbox.setTabStopWidth(20)
+        d = editor.exec_()
+        if d:
+            m.setData(index, QVariant(editor.textbox.toPlainText()), Qt.EditRole)
+        return None
 
     def setModelData(self, editor, model, index):
-        val = unicode(editor.text())
+        val = unicode(editor.textbox.toPlainText())
         try:
             validation_formatter.validate(val)
         except Exception, err:
@@ -364,7 +373,7 @@ class CcTemplateDelegate(QStyledItemDelegate): # {{{
     def setEditorData(self, editor, index):
         m = index.model()
         val = m.custom_columns[m.column_map[index.column()]]['display']['composite_template']
-        editor.setText(val)
+        editor.textbox.setPlainText(val)
 
 
 # }}}

src/calibre/gui2/library/models.py

@@ -303,6 +303,20 @@ class BooksModel(QAbstractTableModel): # {{{
         return self.rowCount(None)
 
     def get_book_display_info(self, idx):
+        def custom_keys_to_display():
+            ans = getattr(self, '_custom_fields_in_book_info', None)
+            if ans is None:
+                cfkeys = set(self.db.custom_field_keys())
+                yes_fields = set(tweaks['book_details_will_display'])
+                no_fields = set(tweaks['book_details_wont_display'])
+                if '*' in yes_fields:
+                    yes_fields = cfkeys
+                if '*' in no_fields:
+                    no_fields = cfkeys
+                ans = frozenset(yes_fields - no_fields)
+                setattr(self, '_custom_fields_in_book_info', ans)
+            return ans
+
         data = {}
         cdata = self.cover(idx)
         if cdata:
@@ -334,7 +348,10 @@ class BooksModel(QAbstractTableModel): # {{{
                 _('Book %s of %s.')%\
                     (sidx, prepare_string_for_xml(series))
         mi = self.db.get_metadata(idx)
+        cf_to_display = custom_keys_to_display()
         for key in mi.custom_field_keys():
+            if key not in cf_to_display:
+                continue
             name, val = mi.format_field(key)
             if val:
                 data[name] = val

src/calibre/manual/template_lang.rst

@@ -101,8 +101,8 @@ Composite columns can use any template option, including formatting.
 
 You cannot change the data contained in a composite column. If you edit a composite column by double-clicking on any item, you will open the template for editing, not the underlying data. Editing the template on the GUI is a quick way of testing and changing composite columns.
 
-Using functions in templates
+Using functions in templates - single-function mode
------------------------------
+---------------------------------------------------
 
 Suppose you want to display the value of a field in upper case, when that field is normally in title case. You can do this (and many more things) using the functions available for templates. For example, to display the title in upper case, use ``{title:uppercase()}``. To display it in title case, use ``{title:titlecase()}``.
 
@@ -137,6 +137,82 @@ Note that you can use the prefix and suffix as well. If you want the number to a
     {#myint:0>3s:ifempty(0)|[|]}
     
 
+Using functions in templates - program mode
+-------------------------------------------
+
+The template language program mode differs from single-function mode in that it permits you to write template expressions that refer to other metadata fields, modify values, and do arithmetic. It is a reasonably complete programming language.
+
+Beginning with an example, assume that you want your template to show the series for a book if it has one, otherwise show the value of a custom field #genre. You cannot do this in the basic language because you cannot make reference to another metadata field within a template expression. In program mode, you can. The following expression works::
+
+    {#series:'ifempty($, field('#genre'))'}
+
+The example shows several things:
+
+    * program mode is used if the expression begins with ``:'`` and ends with ``'``. Anything else is assumed to be single-function.
+    * the variable ``$`` stands for the field the expression is operating upon, ``#series`` in this case.
+    * functions must be given all their arguments. There is no default value. This is true for the standard builtin functions, and is a significant difference from single-function mode.
+    * white space is ignored and can be used anywhere within the expression.
+    * constant strings are enclosed in matching quotes, either ``'`` or ``"``.
+    
+The language is similar to ``functional`` languages in that it is built almost entirely from functions. A statement is a function. An expression is a function. Constants and identifiers can be thought of as functions returning the value indicated by the constant or stored in the identifier.
+
+The syntax of the language is shown by the following grammar::
+
+    constant   ::= " string " | ' string ' | number
+    identifier ::= sequence of letters or ``_`` characters
+    function   ::= identifier ( statement [ , statement ]* )
+    expression ::= identifier | constant | function
+    statement  ::= expression [ ; expression ]*
+    program    ::= statement
+
+An ``expression`` always has a value, either the value of the constant, the value contained in the identifier, or the value returned by a function. The value of a ``statement`` is the value of the last expression in the sequence of statements. As such, the value of the program (statement)::
+
+    1; 2; 'foobar'; 3
+    
+is 3.
+
+Another example of a complex but rather silly program might help make things clearer::
+
+    {series_index:'
+        substr(
+            strcat($, '->', 
+                cmp(divide($, 2), 1, 
+                    assign(c, 1); substr('lt123', c, 0), 
+                    'eq', 'gt')),
+            0, 6)
+       '| prefix | suffix}
+    
+This program does the following: 
+
+    * specify that the field being looked at is series_index. This sets the value of the variable ``$``.
+    * calls the ``substr`` function, which takes 3 parameters ``(str, start, end)``. It returns a string formed by extracting the start through end characters from string, zero-based (the first character is character zero). In this case the string will be computed by the ``strcat`` function, the start is 0, and the end is 6. In this case it will return the first 6 characters of the string returned by ``strcat``, which must be evaluated before substr can return.
+    * calls the ``strcat`` (string concatenation) function. Strcat accepts 1 or more arguments, and returns a string formed by concatenating all the values. In this case there are three arguments. The first parameter is the value in ``$``, which here is the value of ``series_index``. The second paremeter is the constant string ``'->'``. The third parameter is the value returned by the ``cmp`` function, which must be fully evaluated before ``strcat`` can return.
+    * The ``cmp`` function takes 5 arguments ``(x, y, lt, eq, gt)``. It compares x and y and returns the third argument ``lt`` if x < y, the fourth argument ``eq`` if x == y, and the fifth argument ``gt`` if x > y. As with all functions, all of the parameters can be statements. In this case the first parameter (the value for ``x``) is the result of dividing the series_index by 2. The second parameter ``y`` is the constant ``1``. The third parameter ``lt`` is a statement (more later). The fourth parameter ``eq`` is the constant string ``'eq'``. The fifth parameter is the constant string ``'gt'``.
+    * The third parameter (the one for ``lt``) is a statement, or a sequence of expressions. Remember that a statement (a sequence of semicolon-separated expressions) is also an expression, returning the value of the last expression in the list. In this case, the program first assigns the value ``1`` to a local variable ``c``, then returns a substring made by extracting the c'th character to the end. Since c always contains the constant ``1``, the substring will return the second through end'th characters, or ``'t123'``.
+    * Once the statement providing the value to the third parameter is executed, ``cmp`` can return a value. At that point, ``strcat` can return a value, then ``substr`` can return a value. The program then terminates.
+
+For various values of series_index, the program returns:
+
+    * series_index == undefined, result = ``prefix ->t123 suffix``
+    * series_index == 0.5, result = ``prefix 0.50-> suffix``
+    * series_index == 1, result = ``prefix 1->t12 suffix``
+    * series_index == 2, result = ``prefix 2->eq suffix``
+    * series_index == 3, result = ``prefix 3->gt suffix``
+
+All the functions listed under single-function mode can be used in program mode, noting that unlike the functions described below you must supply a first parameter providing the value the function is to act upon. 
+
+The following functions are available in addition to those described in single-function mode. With the exception of the ``id`` parameter of assign, all parameters can be statements (sequences of expressions):
+
+    * ``add(x, y)``	-- returns x + y. Throws an exception if either x or y are not numbers.
+    * ``assign(id, val)`` -- assigns val to id, then returns val. id must be an identifier, not an expression
+    * ``cmp(x, y, lt, eq, gt)`` -- compares x and y after converting both to numbers. Returns ``lt`` if x < y. Returns ``eq`` if x == y. Otherwise returns ``gt``.
+    * ``divide(x, y)``	-- returns x / y. Throws an exception if either x or y are not numbers.
+    * ``field(name)`` -- returns the metadata field named by ``name``.
+    * ``multiply``	-- returns x * y. Throws an exception if either x or y are not numbers.
+    * ``strcat(a, b, ...)`` -- can take any number of arguments. Returns a string formed by concatenating all the arguments.
+    * ``strcmp(x, y, lt, eq, gt)`` -- does a case-insensitive comparison x and y as strings. Returns ``lt`` if x < y. Returns ``eq`` if x == y. Otherwise returns ``gt``.
+    * ``substr(str, start, end)`` -- returns the ``start``'th through the ``end``'th characters of ``str``. The first character in ``str`` is the zero'th character. If end is negative, then it indicates that many characters counting from the right. If end is zero, then it indicates the last character. For example, ``substr('12345', 1, 0)`` returns ``'2345'``, and ``substr('12345', 1, -1)`` returns ``'234'``.
+    * ``subtract``	-- returns x - y. Throws an exception if either x or y are not numbers.
     
 Special notes for save/send templates
 -------------------------------------

src/calibre/utils/formatter.py

@@ -5,10 +5,183 @@ Created on 23 Sep 2010
 '''
 
 import re, string, traceback
+from functools import partial
 
 from calibre.constants import DEBUG
 from calibre.utils.titlecase import titlecase
-from calibre.utils.icu import capitalize
+from calibre.utils.icu import capitalize, strcmp
+
+class _Parser(object):
+    LEX_OP  = 1
+    LEX_ID  = 2
+    LEX_STR = 3
+    LEX_NUM = 4
+    LEX_EOF = 5
+
+    def _strcmp(self, x, y, lt, eq, gt):
+        v = strcmp(x, y)
+        if v < 0:
+            return lt
+        if v == 0:
+            return eq
+        return gt
+
+    def _cmp(self, x, y, lt, eq, gt):
+        x = float(x if x else 0)
+        y = float(y if y else 0)
+        if x < y:
+            return lt
+        if x == y:
+            return eq
+        return gt
+
+    def _assign(self, target, value):
+        setattr(self, target, value)
+        return value
+
+    def _concat(self, *args):
+        i = 0
+        res = ''
+        for i in range(0, len(args)):
+            res += args[i]
+        return res
+
+    def _math(self, x, y, op=None):
+        ops = {
+               '+': lambda x, y: x + y,
+               '-': lambda x, y: x - y,
+               '*': lambda x, y: x * y,
+               '/': lambda x, y: x / y,
+            }
+        x = float(x if x else 0)
+        y = float(y if y else 0)
+        return ops[op](x, y)
+
+    local_functions = {
+            'add'      : (2, partial(_math, op='+')),
+            'assign'   : (2, _assign),
+            'cmp'      : (5, _cmp),
+            'divide'   : (2, partial(_math, op='/')),
+            'field'    : (1, lambda s, x: s.parent.get_value(x, [], s.parent.kwargs)),
+            'multiply' : (2, partial(_math, op='*')),
+            'strcat'   : (-1, _concat),
+            'strcmp'   : (5, _strcmp),
+            'substr'   : (3, lambda s, x, y, z: x[int(y): len(x) if int(z) == 0 else int(z)]),
+            'subtract' : (2, partial(_math, op='-')),
+    }
+
+    def __init__(self, val, prog, parent):
+        self.lex_pos = 0
+        self.prog = prog[0]
+        if prog[1] != '':
+            self.error(_('failed to scan program. Invalid input {0}').format(prog[1]))
+        self.parent = parent
+        setattr(self, '$', val)
+
+    def error(self, message):
+        m = 'Formatter: ' + message + _(' near ')
+        if self.lex_pos > 0:
+            m = '{0} {1}'.format(m, self.prog[self.lex_pos-1][1])
+        m = '{0} {1}'.format(m, self.prog[self.lex_pos][1])
+        if self.lex_pos < len(self.prog):
+            m = '{0} {1}'.format(m, self.prog[self.lex_pos+1][1])
+        raise ValueError(m)
+
+    def token(self):
+        if self.lex_pos >= len(self.prog):
+            return None
+        token = self.prog[self.lex_pos]
+        self.lex_pos += 1
+        return token[1]
+
+    def lookahead(self):
+        if self.lex_pos >= len(self.prog):
+            return (self.LEX_EOF, '')
+        return self.prog[self.lex_pos]
+
+    def consume(self):
+        self.lex_pos += 1
+
+    def token_op_is_a(self, val):
+        token = self.lookahead()
+        return token[0] == self.LEX_OP and token[1] == val
+
+    def token_is_id(self):
+        token = self.lookahead()
+        return token[0] == self.LEX_ID
+
+    def token_is_constant(self):
+        token = self.lookahead()
+        return token[0] == self.LEX_STR or token[0] == self.LEX_NUM
+
+    def token_is_eof(self):
+        token = self.lookahead()
+        return token[0] == self.LEX_EOF
+
+    def program(self):
+        val = self.statement()
+        if not self.token_is_eof():
+            self.error(_('syntax error - program ends before EOF'))
+        return val
+
+    def statement(self):
+        while True:
+            val = self.expr()
+            if self.token_is_eof():
+                return val
+            if not self.token_op_is_a(';'):
+                return val
+            self.consume()
+
+    def expr(self):
+        if self.token_is_id():
+            # We have an identifier. Determine if it is a function
+            id = self.token()
+            if not self.token_op_is_a('('):
+                return getattr(self, id, _('unknown id ') + id)
+            # We have a function.
+            # Check if it is a known one. We do this here so error reporting is
+            # better, as it can identify the tokens near the problem.
+            if id not in self.parent.functions and id not in self.local_functions:
+                self.error(_('unknown function {0}').format(id))
+            # Eat the paren
+            self.consume()
+            args = list()
+            while not self.token_op_is_a(')'):
+                if id == 'assign' and len(args) == 0:
+                    # Must handle the lvalue semantics of the assign function.
+                    # The first argument is the name of the destination, not
+                    # the value.
+                    if not self.token_is_id():
+                        self.error('assign requires the first parameter be an id')
+                    args.append(self.token())
+                else:
+                    # evaluate the argument (recursive call)
+                    args.append(self.statement())
+                if not self.token_op_is_a(','):
+                    break
+                self.consume()
+            if self.token() != ')':
+                self.error(_('missing closing parenthesis'))
+
+            # Evaluate the function
+            if id in self.local_functions:
+                f = self.local_functions[id]
+                if f[0] != -1 and len(args) != f[0]:
+                    self.error('incorrect number of arguments for function {}'.format(id))
+                return f[1](self, *args)
+            else:
+                f = self.parent.functions[id]
+                if f[0] != -1 and len(args) != f[0]+1:
+                    self.error('incorrect number of arguments for function {}'.format(id))
+                return f[1](self.parent, *args)
+            # can't get here
+        elif self.token_is_constant():
+            # String or number
+            return self.token()
+        else:
+            self.error(_('expression is not function or constant'))
+
 
 class TemplateFormatter(string.Formatter):
     '''
@@ -25,6 +198,7 @@ class TemplateFormatter(string.Formatter):
         string.Formatter.__init__(self)
         self.book = None
         self.kwargs = None
+        self.program_cache = {}
 
     def _lookup(self, val, *args):
         if len(args) == 2: # here for backwards compatibility
@@ -135,7 +309,7 @@ class TemplateFormatter(string.Formatter):
                 traceback.print_exc()
             return fmt, '', ''
 
-    format_string_re = re.compile(r'^(.*)\|(.*)\|(.*)$')
+    format_string_re = re.compile(r'^(.*)\|([^\|]*)\|(.*)$', re.DOTALL)
     compress_spaces = re.compile(r'\s+')
     backslash_comma_to_comma = re.compile(r'\\,')
 
@@ -145,6 +319,29 @@ class TemplateFormatter(string.Formatter):
                 (r'.*?\)', lambda x,t: t[:-1]),
         ])
 
+    ################## 'Functional' template language ######################
+
+    lex_scanner = re.Scanner([
+                (r'[(),=;]',            lambda x,t: (1, t)),
+                (r'-?[\d\.]+',          lambda x,t: (3, t)),
+                (r'\$',                 lambda x,t: (2, t)),
+                (r'\w+',                lambda x,t: (2, t)),
+                (r'".*?((?<!\\)")',     lambda x,t: (3, t[1:-1])),
+                (r'\'.*?((?<!\\)\')',   lambda x,t: (3, t[1:-1])),
+                (r'\s',                 None)
+        ])
+
+    def _eval_program(self, val, prog):
+        # keep a cache of the lex'ed program under the theory that re-lexing
+        # is much more expensive than the cache lookup. This is certainly true
+        # for more than a few tokens, but it isn't clear for simple programs.
+        lprog = self.program_cache.get(prog, None)
+        if not lprog:
+            lprog = self.lex_scanner.scan(prog)
+            self.program_cache[prog] = lprog
+        parser = _Parser(val, lprog, self)
+        return parser.program()
+
     ################## Override parent classes methods #####################
 
     def get_value(self, key, args, kwargs):
@@ -155,33 +352,49 @@ class TemplateFormatter(string.Formatter):
         fmt, prefix, suffix = self._explode_format_string(fmt)
 
         # Handle functions
-        p = fmt.find('(')
+        # First see if we have a functional-style expression
-        dispfmt = fmt
+        if fmt.startswith('\''):
-        if p >= 0 and fmt[-1] == ')':
+            p = 0
+        else:
+            p = fmt.find(':\'')
+            if p >= 0:
+                p += 1
+        if p >= 0 and fmt[-1] == '\'':
+            val = self._eval_program(val, fmt[p+1:-1])
             colon = fmt[0:p].find(':')
             if colon < 0:
                 dispfmt = ''
-                colon = 0
             else:
                 dispfmt = fmt[0:colon]
-                colon += 1
+        else:
-            if fmt[colon:p] in self.functions:
+            # check for old-style function references
-                field = fmt[colon:p]
+            p = fmt.find('(')
-                func = self.functions[field]
+            dispfmt = fmt
-                if func[0] == 1:
+            if p >= 0 and fmt[-1] == ')':
-                    # only one arg expected. Don't bother to scan. Avoids need
+                colon = fmt[0:p].find(':')
-                    # for escaping characters
+                if colon < 0:
-                    args = [fmt[p+1:-1]]
+                    dispfmt = ''
+                    colon = 0
                 else:
-                    args = self.arg_parser.scan(fmt[p+1:])[0]
+                    dispfmt = fmt[0:colon]
-                    args = [self.backslash_comma_to_comma.sub(',', a) for a in args]
+                    colon += 1
-                if (func[0] == 0 and (len(args) != 1 or args[0])) or \
+                if fmt[colon:p] in self.functions:
-                        (func[0] > 0 and func[0] != len(args)):
+                    field = fmt[colon:p]
-                    raise ValueError('Incorrect number of arguments for function '+ fmt[0:p])
+                    func = self.functions[field]
-                if func[0] == 0:
+                    if func[0] == 1:
-                    val = func[1](self, val).strip()
+                        # only one arg expected. Don't bother to scan. Avoids need
-                else:
+                        # for escaping characters
-                    val = func[1](self, val, *args).strip()
+                        args = [fmt[p+1:-1]]
+                    else:
+                        args = self.arg_parser.scan(fmt[p+1:])[0]
+                        args = [self.backslash_comma_to_comma.sub(',', a) for a in args]
+                    if (func[0] == 0 and (len(args) != 1 or args[0])) or \
+                            (func[0] > 0 and func[0] != len(args)):
+                        raise ValueError('Incorrect number of arguments for function '+ fmt[0:p])
+                    if func[0] == 0:
+                        val = func[1](self, val).strip()
+                    else:
+                        val = func[1](self, val, *args).strip()
         if val:
             val = self._do_format(val, dispfmt)
         if not val:
@@ -200,10 +413,10 @@ class TemplateFormatter(string.Formatter):
         self.composite_values = {}
         try:
             ans = self.vformat(fmt, [], kwargs).strip()
-        except:
+        except Exception, e:
             if DEBUG:
                 traceback.print_exc()
-            ans = error_value
+            ans = error_value + ' ' + e.message
         return ans
 
 class ValidateFormat(TemplateFormatter):