Template language: Allow you to create your own formatting functions. Accessible via Preferences->Advanced->Template functions

src/calibre/customize/builtins.py

@@ -847,6 +847,17 @@ class Plugboard(PreferencesPlugin):
     config_widget = 'calibre.gui2.preferences.plugboard'
     description = _('Change metadata fields before saving/sending')
 
+class TemplateFunctions(PreferencesPlugin):
+    name = 'TemplateFunctions'
+    icon = I('template_funcs.png')
+    gui_name = _('Template Functions')
+    category = 'Advanced'
+    gui_category = _('Advanced')
+    category_order = 5
+    name_order = 4
+    config_widget = 'calibre.gui2.preferences.template_functions'
+    description = _('Create your own template functions')
+
 class Email(PreferencesPlugin):
     name = 'Email'
     icon = I('mail.png')
@@ -908,6 +919,6 @@ class Misc(PreferencesPlugin):
 
 plugins += [LookAndFeel, Behavior, Columns, Toolbar, InputOptions,
         CommonOptions, OutputOptions, Adding, Saving, Sending, Plugboard,
-        Email, Server, Plugins, Tweaks, Misc]
+        Email, Server, Plugins, Tweaks, Misc, TemplateFunctions]
 
 #}}}

src/calibre/gui2/preferences/template_functions.py

@@ -0,0 +1,184 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import traceback
+
+from calibre.gui2 import error_dialog
+from calibre.gui2.preferences import ConfigWidgetBase, test_widget
+from calibre.gui2.preferences.template_functions_ui import Ui_Form
+from calibre.utils.formatter_functions import formatter_functions, compile_user_function
+
+
+class ConfigWidget(ConfigWidgetBase, Ui_Form):
+
+    def genesis(self, gui):
+        self.gui = gui
+        self.db = gui.library_view.model().db
+        self.current_plugboards = self.db.prefs.get('plugboards',{})
+        help_text = _('''
+        <p>Here you can add and remove functions used in template processing. A
+        template function is written in python. It takes information from the
+        book, processes it in some way, then returns a string result. Functions
+        defined here are usable in templates in the same way that builtin
+        functions are usable. The function must be named evaluate, and must
+        have the signature shown below.</p>
+        <p><code>evaluate(self, formatter, kwargs, mi, locals, your_arguments)
+        &rarr; returning a unicode string</code></p>
+        <p>The arguments to evaluate are:
+        <ul>
+        <li><b>formatter:</b> the instance of the formatter being used to
+        evaluate the current template. You can use this to do recursive
+        template evaluation.</li>
+        <li><b>kwargs:</b> a dictionary of metadata. Field values are in this
+        dictionary. mi: a Metadata instance. Used to get field information.
+        This parameter can be None in some cases, such as when evaluating
+        non-book templates.</li>
+        <li><b>locals:</b> the local variables assigned to by the current
+        template program. Your_arguments must be one or more parameter (number
+        matching the arg count box), or the value *args for a variable number
+        of arguments. These are values passed into the function. One argument
+        is required, and is usually the value of the field being operated upon.
+        Note that when writing in basic template mode, the user does not
+        provide this first argument. Instead it is the value of the field the
+        function is operating upon.</li>
+        </ul></p>
+        <p>
+        The following example function looks for various values in the tags
+        metadata field, returning those values that appear in tags.
+        <pre>
+        def evaluate(self, formatter, kwargs, mi, locals, val):
+            awards=['allbooks', 'PBook', 'ggff']
+            return ', '.join([t for t in kwargs.get('tags') if t in awards])
+        </pre>
+        </p>
+        ''')
+        self.textBrowser.setHtml(help_text)
+
+    def initialize(self):
+        self.funcs = formatter_functions.get_functions()
+        self.builtins = formatter_functions.get_builtins()
+
+        self.build_function_names_box()
+        self.function_name.currentIndexChanged[str].connect(self.function_index_changed)
+        self.function_name.editTextChanged.connect(self.function_name_edited)
+        self.create_button.clicked.connect(self.create_button_clicked)
+        self.delete_button.clicked.connect(self.delete_button_clicked)
+        self.create_button.setEnabled(False)
+        self.delete_button.setEnabled(False)
+        self.clear_button.clicked.connect(self.clear_button_clicked)
+        self.program.setTabStopWidth(20)
+
+    def clear_button_clicked(self):
+        self.build_function_names_box()
+        self.program.clear()
+        self.documentation.clear()
+        self.argument_count.clear()
+        self.create_button.setEnabled(False)
+        self.delete_button.setEnabled(False)
+
+    def build_function_names_box(self, scroll_to='', set_to=''):
+        self.function_name.blockSignals(True)
+        func_names = sorted(self.funcs)
+        self.function_name.clear()
+        self.function_name.addItem('')
+        self.function_name.addItems(func_names)
+        self.function_name.setCurrentIndex(0)
+        if set_to:
+            self.function_name.setEditText(set_to)
+            self.create_button.setEnabled(True)
+        self.function_name.blockSignals(False)
+        if scroll_to:
+            idx = self.function_name.findText(scroll_to)
+            if idx >= 0:
+                self.function_name.setCurrentIndex(idx)
+                if scroll_to not in self.builtins:
+                    self.delete_button.setEnabled(True)
+
+    def delete_button_clicked(self):
+        name = unicode(self.function_name.currentText())
+        if name in self.builtins:
+            error_dialog(self.gui, _('Template functions'),
+                         _('You cannot delete a built-in function'), show=True)
+        if name in self.funcs:
+            del self.funcs[name]
+            self.changed_signal.emit()
+            self.create_button.setEnabled(True)
+            self.delete_button.setEnabled(False)
+            self.build_function_names_box(set_to=name)
+        else:
+            error_dialog(self.gui, _('Template functions'),
+                         _('Function not defined'), show=True)
+
+    def create_button_clicked(self):
+        self.changed_signal.emit()
+        name = unicode(self.function_name.currentText())
+        if name in self.funcs:
+            error_dialog(self.gui, _('Template functions'),
+                         _('Name already used'), show=True)
+            return
+        if self.argument_count.value() == 0:
+            error_dialog(self.gui, _('Template functions'),
+                         _('Argument count must be -1 or greater than zero'),
+                         show=True)
+            return
+        try:
+            prog = unicode(self.program.toPlainText())
+            cls = compile_user_function(name, unicode(self.documentation.toPlainText()),
+                                        self.argument_count.value(), prog)
+            self.funcs[name] = cls
+            self.build_function_names_box(scroll_to=name)
+        except:
+            error_dialog(self.gui, _('Template functions'),
+                         _('Exception while compiling function'), show=True,
+                         det_msg=traceback.format_exc())
+
+    def function_name_edited(self, txt):
+        self.documentation.setReadOnly(False)
+        self.argument_count.setReadOnly(False)
+        self.create_button.setEnabled(True)
+
+    def function_index_changed(self, txt):
+        txt = unicode(txt)
+        self.create_button.setEnabled(False)
+        if not txt:
+            self.argument_count.clear()
+            self.documentation.clear()
+            self.documentation.setReadOnly(False)
+            self.argument_count.setReadOnly(False)
+            return
+        func = self.funcs[txt]
+        self.argument_count.setValue(func.arg_count)
+        self.documentation.setText(func.doc)
+        if txt in self.builtins:
+            self.documentation.setReadOnly(True)
+            self.argument_count.setReadOnly(True)
+            self.program.clear()
+            self.delete_button.setEnabled(False)
+        else:
+            self.program.setPlainText(func.program_text)
+            self.delete_button.setEnabled(True)
+
+    def refresh_gui(self, gui):
+        pass
+
+    def commit(self):
+        formatter_functions.reset_to_builtins()
+        pref_value = []
+        for f in self.funcs:
+            if f in self.builtins:
+                continue
+            func = self.funcs[f]
+            formatter_functions.register_function(func)
+            pref_value.append((func.name, func.doc, func.arg_count, func.program_text))
+        self.db.prefs.set('user_template_functions', pref_value)
+
+
+if __name__ == '__main__':
+    from PyQt4.Qt import QApplication
+    app = QApplication([])
+    test_widget('Advanced', 'TemplateFunctions')
+

src/calibre/gui2/preferences/template_functions.ui

@@ -0,0 +1,154 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>Form</class>
+ <widget class="QWidget" name="Form">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>798</width>
+    <height>672</height>
+   </rect>
+  </property>
+  <property name="windowTitle">
+   <string>Form</string>
+  </property>
+  <layout class="QGridLayout" name="gridLayout">
+   <item row="1" column="0" colspan="2">
+    <widget class="Line" name="line">
+     <property name="orientation">
+      <enum>Qt::Horizontal</enum>
+     </property>
+    </widget>
+   </item>
+   <item row="2" column="0">
+    <layout class="QHBoxLayout" name="horizontalLayout">
+     <item>
+      <layout class="QGridLayout" name="gridLayout_3">
+       <item row="0" column="0">
+        <widget class="QLabel" name="label_2">
+         <property name="text">
+          <string>&amp;Function:</string>
+         </property>
+         <property name="buddy">
+          <cstring>function_name</cstring>
+         </property>
+        </widget>
+       </item>
+       <item row="0" column="1">
+        <widget class="QComboBox" name="function_name">
+         <property name="toolTip">
+          <string>Enter the name of the function to create</string>
+         </property>
+         <property name="editable">
+          <bool>true</bool>
+         </property>
+        </widget>
+       </item>
+       <item row="1" column="0">
+        <widget class="QLabel" name="label_3">
+         <property name="toolTip">
+          <string></string>
+         </property>
+         <property name="text">
+          <string>Arg &amp;count:</string>
+         </property>
+         <property name="buddy">
+          <cstring>argument_count</cstring>
+         </property>
+        </widget>
+       </item>
+       <item row="1" column="1">
+        <widget class="QSpinBox" name="argument_count">
+         <property name="toolTip">
+          <string>Set this to -1 if the function takes a variable number of arguments</string>
+         </property>
+         <property name="minimum">
+          <number>-1</number>
+         </property>
+        </widget>
+       </item>
+       <item row="2" column="1">
+        <widget class="QTextEdit" name="documentation"/>
+       </item>
+       <item row="2" column="0">
+        <widget class="QLabel" name="label_4">
+         <property name="text">
+          <string>&amp;Documentation:</string>
+         </property>
+         <property name="alignment">
+          <set>Qt::AlignLeading|Qt::AlignLeft|Qt::AlignTop</set>
+         </property>
+         <property name="buddy">
+          <cstring>documentation</cstring>
+         </property>
+        </widget>
+       </item>
+       <item row="3" column="1">
+        <layout class="QHBoxLayout" name="horizontalLayout_2">
+         <item>
+          <widget class="QPushButton" name="clear_button">
+           <property name="text">
+            <string>&amp;Clear</string>
+           </property>
+          </widget>
+         </item>
+         <item>
+          <widget class="QPushButton" name="delete_button">
+           <property name="text">
+            <string>&amp;Delete</string>
+           </property>
+          </widget>
+         </item>
+         <item>
+          <widget class="QPushButton" name="create_button">
+           <property name="text">
+            <string>C&amp;reate</string>
+           </property>
+          </widget>
+         </item>
+        </layout>
+       </item>
+      </layout>
+     </item>
+     <item>
+      <layout class="QVBoxLayout" name="horizontalLayout1">
+       <item>
+        <widget class="QLabel" name="label_4">
+         <property name="text">
+          <string>&amp;Program Code: (be sure to follow python indenting rules)</string>
+         </property>
+         <property name="buddy">
+          <cstring>program</cstring>
+         </property>
+        </widget>
+       </item>
+       <item>
+        <widget class="QPlainTextEdit" name="program">
+         <property name="minimumSize">
+          <size>
+           <width>400</width>
+           <height>0</height>
+          </size>
+         </property>
+         <property name="documentTitle">
+          <string notr="true"/>
+         </property>
+         <property name="tabStopWidth">
+          <number>30</number>
+         </property>
+        </widget>
+       </item>
+      </layout>
+     </item>
+    </layout>
+   </item>
+   <item row="0" column="0">
+    <widget class="QTextBrowser" name="textBrowser">
+    </widget>
+   </item>
+  </layout>
+ </widget>
+ <resources/>
+ <connections/>
+</ui>

src/calibre/library/database2.py

@@ -38,6 +38,7 @@ from calibre.utils.search_query_parser import saved_searches, set_saved_searches
 from calibre.ebooks import BOOK_EXTENSIONS, check_ebook_format
 from calibre.utils.magick.draw import save_cover_data_to
 from calibre.utils.recycle_bin import delete_file, delete_tree
+from calibre.utils.formatter_functions import load_user_template_functions
 
 
 copyfile = os.link if hasattr(os, 'link') else shutil.copyfile
@@ -185,6 +186,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         migrate_preference('saved_searches', {})
         set_saved_searches(self, 'saved_searches')
 
+        load_user_template_functions(self.prefs.get('user_template_functions', []))
+
         self.conn.executescript('''
         DROP TRIGGER IF EXISTS author_insert_trg;
         CREATE TEMP TRIGGER author_insert_trg

src/calibre/utils/formatter.py

@@ -4,12 +4,14 @@ Created on 23 Sep 2010
 @author: charles
 '''
 
+__license__   = 'GPL v3'
+__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
 import re, string, traceback
-from functools import partial
 
 from calibre.constants import DEBUG
-from calibre.utils.titlecase import titlecase
+from calibre.utils.formatter_functions import formatter_functions
-from calibre.utils.icu import capitalize, strcmp
 
 class _Parser(object):
     LEX_OP  = 1
@@ -18,93 +20,6 @@ class _Parser(object):
     LEX_NUM = 4
     LEX_EOF = 5
 
-    def _python(self, func):
-        locals = {}
-        exec func in locals
-        if 'evaluate' not in locals:
-            self.error('no evaluate function in python')
-        try:
-            result = locals['evaluate'](self.parent.kwargs)
-            if isinstance(result, (float, int)):
-                result = unicode(result)
-            elif isinstance(result, list):
-                result = ','.join(result)
-            elif isinstance(result, str):
-                result = unicode(result)
-            return result
-        except Exception as e:
-            self.error('python function threw exception: ' + e.msg)
-
-
-    def _strcmp(self, x, y, lt, eq, gt):
-        v = strcmp(x, y)
-        if v < 0:
-            return lt
-        if v == 0:
-            return eq
-        return gt
-
-    def _cmp(self, x, y, lt, eq, gt):
-        x = float(x if x else 0)
-        y = float(y if y else 0)
-        if x < y:
-            return lt
-        if x == y:
-            return eq
-        return gt
-
-    def _assign(self, target, value):
-        self.variables[target] = value
-        return value
-
-    def _concat(self, *args):
-        i = 0
-        res = ''
-        for i in range(0, len(args)):
-            res += args[i]
-        return res
-
-    def _math(self, x, y, op=None):
-        ops = {
-               '+': lambda x, y: x + y,
-               '-': lambda x, y: x - y,
-               '*': lambda x, y: x * y,
-               '/': lambda x, y: x / y,
-            }
-        x = float(x if x else 0)
-        y = float(y if y else 0)
-        return unicode(ops[op](x, y))
-
-    def _template(self, template):
-        template = template.replace('[[', '{').replace(']]', '}')
-        return self.parent.safe_format(template, self.parent.kwargs, 'TEMPLATE',
-                                       self.parent.book)
-
-    def _eval(self, template):
-        template = template.replace('[[', '{').replace(']]', '}')
-        return eval_formatter.safe_format(template, self.variables, 'EVAL', None)
-
-    def _print(self, *args):
-        print args
-        return None
-
-    local_functions = {
-            'add'      : (2, partial(_math, op='+')),
-            'assign'   : (2, _assign),
-            'cmp'      : (5, _cmp),
-            'divide'   : (2, partial(_math, op='/')),
-            'eval'     : (1, _eval),
-            'field'    : (1, lambda s, x: s.parent.get_value(x, [], s.parent.kwargs)),
-            'multiply' : (2, partial(_math, op='*')),
-            'print'    : (-1, _print),
-            'python'   : (1, _python),
-            'strcat'   : (-1, _concat),
-            'strcmp'   : (5, _strcmp),
-            'substr'   : (3, lambda s, x, y, z: x[int(y): len(x) if int(z) == 0 else int(z)]),
-            'subtract' : (2, partial(_math, op='-')),
-            'template' : (1, _template)
-    }
-
     def __init__(self, val, prog, parent):
         self.lex_pos = 0
         self.prog = prog[0]
@@ -184,7 +99,9 @@ class _Parser(object):
             # We have a function.
             # Check if it is a known one. We do this here so error reporting is
             # better, as it can identify the tokens near the problem.
-            if id not in self.parent.functions and id not in self.local_functions:
+            funcs = formatter_functions.get_functions()
+
+            if id not in funcs:
                 self.error(_('unknown function {0}').format(id))
             # Eat the paren
             self.consume()
@@ -207,11 +124,12 @@ class _Parser(object):
                 self.error(_('missing closing parenthesis'))
 
             # Evaluate the function
-            if id in self.local_functions:
+            if id in funcs:
-                f = self.local_functions[id]
+                cls = funcs[id]
-                if f[0] != -1 and len(args) != f[0]:
+                if cls.arg_count != -1 and len(args) != cls.arg_count:
                     self.error('incorrect number of arguments for function {}'.format(id))
-                return f[1](self, *args)
+                return cls.eval(self.parent, self.parent.kwargs,
+                                self.parent.book, locals, *args)
             else:
                 f = self.parent.functions[id]
                 if f[0] != -1 and len(args) != f[0]+1:
@@ -242,91 +160,6 @@ class TemplateFormatter(string.Formatter):
         self.kwargs = None
         self.program_cache = {}
 
-    def _lookup(self, val, *args):
-        if len(args) == 2: # here for backwards compatibility
-            if val:
-                return self.vformat('{'+args[0].strip()+'}', [], self.kwargs)
-            else:
-                return self.vformat('{'+args[1].strip()+'}', [], self.kwargs)
-        if (len(args) % 2) != 1:
-            raise ValueError(_('lookup requires either 2 or an odd number of arguments'))
-        i = 0
-        while i < len(args):
-            if i + 1 >= len(args):
-                return self.vformat('{' + args[i].strip() + '}', [], self.kwargs)
-            if re.search(args[i], val):
-                return self.vformat('{'+args[i+1].strip() + '}', [], self.kwargs)
-            i += 2
-
-    def _test(self, val, value_if_set, value_not_set):
-        if val:
-            return value_if_set
-        else:
-            return value_not_set
-
-    def _contains(self, val, test, value_if_present, value_if_not):
-        if re.search(test, val):
-            return value_if_present
-        else:
-            return value_if_not
-
-    def _switch(self, val, *args):
-        if (len(args) % 2) != 1:
-            raise ValueError(_('switch requires an odd number of arguments'))
-        i = 0
-        while i < len(args):
-            if i + 1 >= len(args):
-                return args[i]
-            if re.search(args[i], val):
-                return args[i+1]
-            i += 2
-
-    def _re(self, val, pattern, replacement):
-        return re.sub(pattern, replacement, val)
-
-    def _ifempty(self, val, value_if_empty):
-        if val:
-            return val
-        else:
-            return value_if_empty
-
-    def _shorten(self, val, leading, center_string, trailing):
-        l = max(0, int(leading))
-        t = max(0, int(trailing))
-        if len(val) > l + len(center_string) + t:
-            return val[0:l] + center_string + ('' if t == 0 else val[-t:])
-        else:
-            return val
-
-    def _count(self, val, sep):
-        return unicode(len(val.split(sep)))
-
-    def _list_item(self, val, index, sep):
-        if not val:
-            return ''
-        index = int(index)
-        val = val.split(sep)
-        try:
-            return val[index]
-        except:
-            return ''
-
-    functions = {
-                    'uppercase'     : (0, lambda s,x: x.upper()),
-                    'lowercase'     : (0, lambda s,x: x.lower()),
-                    'titlecase'     : (0, lambda s,x: titlecase(x)),
-                    'capitalize'    : (0, lambda s,x: capitalize(x)),
-                    'contains'      : (3, _contains),
-                    'count'         : (1, _count),
-                    'ifempty'       : (1, _ifempty),
-                    'list_item'     : (2, _list_item),
-                    'lookup'        : (-1, _lookup),
-                    're'            : (2, _re),
-                    'shorten'       : (3, _shorten),
-                    'switch'        : (-1, _switch),
-                    'test'          : (2, _test)
-        }
-
     def _do_format(self, val, fmt):
         if not fmt or not val:
             return val
@@ -436,23 +269,27 @@ class TemplateFormatter(string.Formatter):
                 else:
                     dispfmt = fmt[0:colon]
                     colon += 1
-                if fmt[colon:p] in self.functions:
+
+                funcs = formatter_functions.get_functions()
+                if fmt[colon:p] in funcs:
                     field = fmt[colon:p]
-                    func = self.functions[field]
+                    func = funcs[field]
-                    if func[0] == 1:
+                    if func.arg_count == 2:
                         # only one arg expected. Don't bother to scan. Avoids need
                         # for escaping characters
                         args = [fmt[p+1:-1]]
                     else:
                         args = self.arg_parser.scan(fmt[p+1:])[0]
                         args = [self.backslash_comma_to_comma.sub(',', a) for a in args]
-                    if (func[0] == 0 and (len(args) != 1 or args[0])) or \
+                    if (func.arg_count == 1 and (len(args) != 0)) or \
-                            (func[0] > 0 and func[0] != len(args)):
+                            (func.arg_count > 1 and func.arg_count != len(args)+1):
+                        print args
                         raise ValueError('Incorrect number of arguments for function '+ fmt[0:p])
-                    if func[0] == 0:
+                    if func.arg_count == 1:
-                        val = func[1](self, val).strip()
+                        val = func.eval(self, self.kwargs, self.book, locals, val).strip()
                     else:
-                        val = func[1](self, val, *args).strip()
+                        val = func.eval(self, self.kwargs, self.book, locals,
+                                        val, *args).strip()
         if val:
             val = self._do_format(val, dispfmt)
         if not val:

src/calibre/utils/formatter_functions.py

@@ -0,0 +1,469 @@
+'''
+Created on 13 Jan 2011
+
+@author: charles
+'''
+
+__license__   = 'GPL v3'
+__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import re, traceback
+
+from calibre.utils.titlecase import titlecase
+from calibre.utils.icu import capitalize, strcmp
+
+
+class FormatterFunctions(object):
+
+    def __init__(self):
+        self.builtins = {}
+        self.functions = {}
+
+    def register_builtin(self, func_class):
+        if not isinstance(func_class, FormatterFunction):
+            raise ValueError('Class %s is not an instance of FormatterFunction'%(
+                                    func_class.__class__.__name__))
+        name = func_class.name
+        if name in self.functions:
+            raise ValueError('Name %s already used'%name)
+        self.builtins[name] = func_class
+        self.functions[name] = func_class
+
+    def register_function(self, func_class):
+        if not isinstance(func_class, FormatterFunction):
+            raise ValueError('Class %s is not an instance of FormatterFunction'%(
+                                    func_class.__class__.__name__))
+        name = func_class.name
+        if name in self.functions:
+            raise ValueError('Name %s already used'%name)
+        self.functions[name] = func_class
+
+    def get_builtins(self):
+        return self.builtins
+
+    def get_functions(self):
+        return self.functions
+
+    def reset_to_builtins(self):
+        self.functions = dict([t for t in self.builtins.items()])
+
+formatter_functions = FormatterFunctions()
+
+
+
+class FormatterFunction(object):
+
+    doc = _('No documentation provided')
+    name = 'no name provided'
+    arg_count = 0
+
+    def __init__(self):
+        formatter_functions.register_builtin(self)
+
+    def evaluate(self, formatter, kwargs, mi, locals, *args):
+        raise NotImplementedError()
+
+    def eval(self, formatter, kwargs, mi, locals, *args):
+        try:
+            ret = self.evaluate(formatter, kwargs, mi, locals, *args)
+            if isinstance(ret, (str, unicode)):
+                return ret
+            if isinstance(ret, (int, float, bool)):
+                return unicode(ret)
+            if isinstance(ret, list):
+                return ','.join(list)
+        except:
+            return _('Function threw exception' + traceback.format_exc())
+
+class BuiltinStrcmp(FormatterFunction):
+    name = 'strcmp'
+    arg_count = 5
+    doc = _('strcmp(x, y, lt, eq, gt) -- does a case-insensitive comparison of x '
+            'and y as strings. Returns lt if x < y. Returns eq if x == y. '
+            'Otherwise returns gt.')
+
+    def evaluate(self, formatter, kwargs, mi, locals, x, y, lt, eq, gt):
+        v = strcmp(x, y)
+        if v < 0:
+            return lt
+        if v == 0:
+            return eq
+        return gt
+
+class BuiltinCmp(FormatterFunction):
+    name = 'cmp'
+    arg_count = 5
+    doc =   _('cmp(x, y, lt, eq, gt) -- compares x and y after converting both to '
+            'numbers. Returns lt if x < y. Returns eq if x == y. Otherwise returns gt.')
+
+    def evaluate(self, formatter, kwargs, mi, locals, x, y, lt, eq, gt):
+        x = float(x if x else 0)
+        y = float(y if y else 0)
+        if x < y:
+            return lt
+        if x == y:
+            return eq
+        return gt
+
+class BuiltinStrcat(FormatterFunction):
+    name = 'strcat'
+    arg_count = -1
+    doc = _('strcat(a, b, ...) -- can take any number of arguments. Returns a '
+            'string formed by concatenating all the arguments')
+
+    def evaluate(self, formatter, kwargs, mi, locals, *args):
+        i = 0
+        res = ''
+        for i in range(0, len(args)):
+            res += args[i]
+        return res
+
+class BuiltinAdd(FormatterFunction):
+    name = 'add'
+    arg_count = 2
+    doc = _('add(x, y) -- returns x + y. Throws an exception if either x or y are not numbers.')
+
+    def evaluate(self, formatter, kwargs, mi, locals, x, y):
+        x = float(x if x else 0)
+        y = float(y if y else 0)
+        return unicode(x + y)
+
+class BuiltinSubtract(FormatterFunction):
+    name = 'subtract'
+    arg_count = 2
+    doc = _('subtract(x, y) -- returns x - y. Throws an exception if either x or y are not numbers.')
+
+    def evaluate(self, formatter, kwargs, mi, locals, x, y):
+        x = float(x if x else 0)
+        y = float(y if y else 0)
+        return unicode(x - y)
+
+class BuiltinMultiply(FormatterFunction):
+    name = 'multiply'
+    arg_count = 2
+    doc = _('multiply(x, y) -- returns x * y. Throws an exception if either x or y are not numbers.')
+
+    def evaluate(self, formatter, kwargs, mi, locals, x, y):
+        x = float(x if x else 0)
+        y = float(y if y else 0)
+        return unicode(x * y)
+
+class BuiltinDivide(FormatterFunction):
+    name = 'divide'
+    arg_count = 2
+    doc = _('divide(x, y) -- returns x / y. Throws an exception if either x or y are not numbers.')
+
+    def evaluate(self, formatter, kwargs, mi, locals, x, y):
+        x = float(x if x else 0)
+        y = float(y if y else 0)
+        return unicode(x / y)
+
+class BuiltinTemplate(FormatterFunction):
+    name = 'template'
+    arg_count = 1
+    doc = _('template(x) -- evaluates x as a template. The evaluation is done '
+            'in its own context, meaning that variables are not shared between '
+            'the caller and the template evaluation. Because the { and } '
+            'characters are special, you must use [[ for the { character and '
+            ']] for the } character; they are converted automatically. '
+            'For example, ``template(\'[[title_sort]]\') will evaluate the '
+            'template {title_sort} and return its value.')
+
+    def evaluate(self, formatter, kwargs, mi, locals, template):
+        template = template.replace('[[', '{').replace(']]', '}')
+        return formatter.safe_format(template, kwargs, 'TEMPLATE', mi)
+
+class BuiltinEval(FormatterFunction):
+    name = 'eval'
+    arg_count = 1
+    doc = _('eval(template)`` -- evaluates the template, passing the local '
+            'variables (those \'assign\'ed to) instead of the book metadata. '
+            ' This permits using the template processor to construct complex '
+            'results from local variables.')
+
+    def evaluate(self, formatter, kwargs, mi, locals, template):
+        from formatter import eval_formatter
+        template = template.replace('[[', '{').replace(']]', '}')
+        return eval_formatter.safe_format(template, locals, 'EVAL', None)
+
+class BuiltinAssign(FormatterFunction):
+    name = 'assign'
+    arg_count = 2
+    doc = _('assign(id, val) -- assigns val to id, then returns val. '
+            'id must be an identifier, not an expression')
+
+    def evaluate(self, formatter, kwargs, mi, locals, target, value):
+        locals[target] = value
+        return value
+
+class BuiltinPrint(FormatterFunction):
+    name = 'print'
+    arg_count = -1
+    doc = _('print(a, b, ...) -- prints the arguments to standard output. '
+            'Unless you start calibre from the command line (calibre-debug -g), '
+            'the output will go to a black hole.')
+
+    def evaluate(self, formatter, kwargs, mi, locals, *args):
+        print args
+        return None
+
+class BuiltinField(FormatterFunction):
+    name = 'field'
+    arg_count = 1
+    doc = _('field(name) -- returns the metadata field named by name')
+
+    def evaluate(self, formatter, kwargs, mi, locals, name):
+        return formatter.get_value(name, [], kwargs)
+
+class BuiltinSubstr(FormatterFunction):
+    name = 'substr'
+    arg_count = 3
+    doc = _('substr(str, start, end) -- returns the start\'th through the end\'th '
+            'characters of str. The first character in str is the zero\'th '
+            'character. If end is negative, then it indicates that many '
+            'characters counting from the right. If end is zero, then it '
+            'indicates the last character. For example, substr(\'12345\', 1, 0) '
+            'returns \'2345\', and substr(\'12345\', 1, -1) returns \'234\'.')
+
+    def evaluate(self, formatter, kwargs, mi, locals, str_, start_, end_):
+        return str_[int(start_): len(str_) if int(end_) == 0 else int(end_)]
+
+class BuiltinLookup(FormatterFunction):
+    name = 'lookup'
+    arg_count = -1
+    doc = _('lookup(val, pattern, field, pattern, field, ..., else_field) -- '
+            'like switch, except the arguments are field (metadata) names, not '
+            'text. The value of the appropriate field will be fetched and used. '
+            'Note that because composite columns are fields, you can use this '
+            'function in one composite field to use the value of some other '
+            'composite field. This is extremely useful when constructing '
+            'variable save paths')
+
+    def evaluate(self, formatter, kwargs, mi, locals, val, *args):
+        if len(args) == 2: # here for backwards compatibility
+            if val:
+                return formatter.vformat('{'+args[0].strip()+'}', [], kwargs)
+            else:
+                return formatter.vformat('{'+args[1].strip()+'}', [], kwargs)
+        if (len(args) % 2) != 1:
+            raise ValueError(_('lookup requires either 2 or an odd number of arguments'))
+        i = 0
+        while i < len(args):
+            if i + 1 >= len(args):
+                return formatter.vformat('{' + args[i].strip() + '}', [], kwargs)
+            if re.search(args[i], val):
+                return formatter.vformat('{'+args[i+1].strip() + '}', [], kwargs)
+            i += 2
+
+class BuiltinTest(FormatterFunction):
+    name = 'test'
+    arg_count = 3
+    doc = _('test(val, text if not empty, text if empty) -- return `text if not '
+            'empty` if the field is not empty, otherwise return `text if empty`')
+
+    def evaluate(self, formatter, kwargs, mi, locals, val, value_if_set, value_not_set):
+        if val:
+            return value_if_set
+        else:
+            return value_not_set
+
+class BuiltinContains(FormatterFunction):
+    name = 'contains'
+    arg_count = 4
+    doc = _('contains(val, pattern, text if match, text if not match) -- checks '
+            'if field contains matches for the regular expression `pattern`. '
+            'Returns `text if match` if matches are found, otherwise it returns '
+            '`text if no match`')
+
+    def evaluate(self, formatter, kwargs, mi, locals,
+                 val, test, value_if_present, value_if_not):
+        if re.search(test, val):
+            return value_if_present
+        else:
+            return value_if_not
+
+class BuiltinSwitch(FormatterFunction):
+    name = 'switch'
+    arg_count = -1
+    doc = _('switch(val, pattern, value, pattern, value, ..., else_value) -- '
+            'for each ``pattern, value`` pair, checks if the field matches '
+            'the regular expression ``pattern`` and if so, returns that '
+            'value. If no pattern matches, then else_value is returned. '
+            'You can have as many `pattern, value` pairs as you want')
+
+    def evaluate(self, formatter, kwargs, mi, locals, val, *args):
+        if (len(args) % 2) != 1:
+            raise ValueError(_('switch requires an odd number of arguments'))
+        i = 0
+        while i < len(args):
+            if i + 1 >= len(args):
+                return args[i]
+            if re.search(args[i], val):
+                return args[i+1]
+            i += 2
+
+class BuiltinRe(FormatterFunction):
+    name = 're'
+    arg_count = 3
+    doc = _('re(val, pattern, replacement) -- return the field after applying '
+            'the regular expression. All instances of `pattern` are replaced '
+            'with `replacement`. As in all of calibre, these are '
+            'python-compatible regular expressions')
+
+    def evaluate(self, formatter, kwargs, mi, locals, val, pattern, replacement):
+        return re.sub(pattern, replacement, val)
+
+class BuiltinEvaluate(FormatterFunction):
+    name = 'evaluate'
+    arg_count = 2
+    doc = _('evaluate(val, text if empty) -- return val if val is not empty, '
+            'otherwise return `text if empty`')
+
+    def evaluate(self, formatter, kwargs, mi, locals, val, value_if_empty):
+        if val:
+            return val
+        else:
+            return value_if_empty
+
+class BuiltinShorten(FormatterFunction):
+    name = 'shorten    '
+    arg_count = 4
+    doc = _('shorten(val, left chars, middle text, right chars) -- Return a '
+            'shortened version of the field, consisting of `left chars` '
+            'characters from the beginning of the field, followed by '
+            '`middle text`, followed by `right chars` characters from '
+            'the end of the string. `Left chars` and `right chars` must be '
+            'integers. For example, assume the title of the book is '
+            '`Ancient English Laws in the Times of Ivanhoe`, and you want '
+            'it to fit in a space of at most 15 characters. If you use '
+            '{title:shorten(9,-,5)}, the result will be `Ancient E-nhoe`. '
+            'If the field\'s length is less than left chars + right chars + '
+            'the length of `middle text`, then the field will be used '
+            'intact. For example, the title `The Dome` would not be changed.')
+
+    def evaluate(self, formatter, kwargs, mi, locals,
+                 val, leading, center_string, trailing):
+        l = max(0, int(leading))
+        t = max(0, int(trailing))
+        if len(val) > l + len(center_string) + t:
+            return val[0:l] + center_string + ('' if t == 0 else val[-t:])
+        else:
+            return val
+
+class BuiltinCount(FormatterFunction):
+    name = 'count'
+    arg_count = 2
+    doc = _('count(val, separator) -- interprets the value as a list of items '
+            'separated by `separator`, returning the number of items in the '
+            'list. Most lists use a comma as the separator, but authors '
+            'uses an ampersand. Examples: {tags:count(,)}, {authors:count(&)}')
+
+    def evaluate(self, formatter, kwargs, mi, locals, val, sep):
+        return unicode(len(val.split(sep)))
+
+class BuiltinListitem(FormatterFunction):
+    name = 'list_item'
+    arg_count = 3
+    doc = _('list_item(val, index, separator) -- interpret the value as a list of '
+            'items separated by `separator`, returning the `index`th item. '
+            'The first item is number zero. The last item can be returned '
+            'using `list_item(-1,separator)`. If the item is not in the list, '
+            'then the empty value is returned. The separator has the same '
+            'meaning as in the count function.')
+
+    def evaluate(self, formatter, kwargs, mi, locals, val, index, sep):
+        if not val:
+            return ''
+        index = int(index)
+        val = val.split(sep)
+        try:
+            return val[index]
+        except:
+            return ''
+
+class BuiltinUppercase(FormatterFunction):
+    name = 'uppercase'
+    arg_count = 1
+    doc = _('uppercase(val) -- return value of the field in upper case')
+
+    def evaluate(self, formatter, kwargs, mi, locals, val):
+        return val.upper()
+
+class BuiltinLowercase(FormatterFunction):
+    name = 'lowercase'
+    arg_count = 1
+    doc = _('lowercase(val) -- return value of the field in lower case')
+
+    def evaluate(self, formatter, kwargs, mi, locals, val):
+        return val.lower()
+
+class BuiltinTitlecase(FormatterFunction):
+    name = 'titlecase'
+    arg_count = 1
+    doc = _('titlecase(val) -- return value of the field in title case')
+
+    def evaluate(self, formatter, kwargs, mi, locals, val):
+        return titlecase(val)
+
+class BuiltinCapitalize(FormatterFunction):
+    name = 'capitalize'
+    arg_count = 1
+    doc = _('capitalize(val) -- return value of the field capitalized')
+
+    def evaluate(self, formatter, kwargs, mi, locals, val):
+        return capitalize(val)
+
+builtin_add         = BuiltinAdd()
+builtin_assign      = BuiltinAssign()
+builtin_capitalize  = BuiltinCapitalize()
+builtin_cmp         = BuiltinCmp()
+builtin_contains    = BuiltinContains()
+builtin_count       = BuiltinCount()
+builtin_divide      = BuiltinDivide()
+builtin_eval        = BuiltinEval()
+builtin_evaluate    = BuiltinEvaluate()
+builtin_field       = BuiltinField()
+builtin_list_item   = BuiltinListitem()
+builtin_lookup      = BuiltinLookup()
+builtin_lowercase   = BuiltinLowercase()
+builtin_multiply    = BuiltinMultiply()
+builtin_print       = BuiltinPrint()
+builtin_re          = BuiltinRe()
+builtin_shorten     = BuiltinShorten()
+builtin_strcat      = BuiltinStrcat()
+builtin_strcmp      = BuiltinStrcmp()
+builtin_substr      = BuiltinSubstr()
+builtin_subtract    = BuiltinSubtract()
+builtin_switch      = BuiltinSwitch()
+builtin_template    = BuiltinTemplate()
+builtin_test        = BuiltinTest()
+builtin_titlecase   = BuiltinTitlecase()
+builtin_uppercase   = BuiltinUppercase()
+
+class FormatterUserFunction(FormatterFunction):
+    def __init__(self, name, doc, arg_count, program_text):
+        self.name = name
+        self.doc = doc
+        self.arg_count = arg_count
+        self.program_text = program_text
+
+def compile_user_function(name, doc, arg_count, eval_func):
+    func = '\t' + eval_func.replace('\n', '\n\t')
+    prog = '''
+from calibre.utils.formatter_functions import FormatterUserFunction
+class UserFunction(FormatterUserFunction):
+''' + func
+    locals = {}
+    exec prog in locals
+    cls = locals['UserFunction'](name, doc, arg_count, eval_func)
+    return cls
+
+def load_user_template_functions(funcs):
+    formatter_functions.reset_to_builtins()
+    for func in funcs:
+        try:
+            cls = compile_user_function(*func)
+            formatter_functions.register_function(cls)
+        except:
+            traceback.print_exc()