Store and access user template functions on a per-library basis. This is used when the DB evaluates composites, avoiding using the global copy.

The ability to pre-compile templates was removed because the one-true list of available functions is known only at execution time. The compiled copy depended on the global state being accurate. And in any event, I was never totally convinced that the compiled version was significantly faster than the pre-lexed version.

Includes a bug fix: the booklist must be refreshed if a user-defined function changed.

resources/default_tweaks.py

@@ -485,16 +485,6 @@ gui_view_history_size = 15
 # negative number to increase or decrease the font size.
 change_book_details_font_size_by = 0
 
-#: Compile general program mode templates to Python
-# Compiled general program mode templates are significantly faster than
-# interpreted templates. Setting this tweak to True causes calibre to compile
-# (in most cases) general program mode templates. Setting it to False causes
-# calibre to use the old behavior -- interpreting the templates. Set the tweak
-# to False if some compiled templates produce incorrect values.
-# Default:    compile_gpm_templates = True
-# No compile: compile_gpm_templates = False
-compile_gpm_templates = True
-
 #: What format to default to when using the "Unpack book" feature
 # The "Unpack book" feature of calibre allows direct editing of a book format.
 # If multiple formats are available, calibre will offer you a choice

src/calibre/db/backend.py

@@ -31,7 +31,9 @@ from calibre.utils.filenames import (
     WindowsAtomicFolderMove, atomic_rename, remove_dir_if_empty,
     copytree_using_links, copyfile_using_links)
 from calibre.utils.img import save_cover_data_to
-from calibre.utils.formatter_functions import load_user_template_functions, unload_user_template_functions
+from calibre.utils.formatter_functions import (load_user_template_functions,
+            unload_user_template_functions,
+            compile_user_template_functions)
 from calibre.db.tables import (OneToOneTable, ManyToOneTable, ManyToManyTable,
         SizeTable, FormatsTable, AuthorsTable, IdentifiersTable, PathTable,
         CompositeTable, UUIDTable, RatingTable)
@@ -316,9 +318,14 @@ class Connection(apsw.Connection):  # {{{
 # }}}
 
 
-def set_global_state(backend):
+def set_global_state(backend, precompiled_user_functions=None):
-    load_user_template_functions(backend.library_id,
+    if precompiled_user_functions:
-                                 backend.prefs.get('user_template_functions', []))
+        load_user_template_functions(backend.library_id,
+                             [],
+                             precompiled_user_functions=precompiled_user_functions)
+    else:
+        load_user_template_functions(backend.library_id,
+                             backend.prefs.get('user_template_functions', []))
 
 
 class DB(object):
@@ -406,8 +413,16 @@ class DB(object):
         self.initialize_prefs(default_prefs, restore_all_prefs, progress_callback)
         self.initialize_custom_columns()
         self.initialize_tables()
+        self.set_user_template_functions(compile_user_template_functions(
+                                 self.prefs.get('user_template_functions', [])))
         if load_user_formatter_functions:
-            set_global_state(self)
+            set_global_state(self, precompiled_user_functions = self.get_user_template_functions())
+
+    def get_user_template_functions(self):
+        return self._user_template_functions
+
+    def set_user_template_functions(self, user_formatter_functions):
+        self._user_template_functions = user_formatter_functions
 
     def initialize_prefs(self, default_prefs, restore_all_prefs, progress_callback):  # {{{
         self.prefs = DBPrefs(self)

src/calibre/db/cache.py

@@ -203,6 +203,10 @@ class Cache(object):
     def initialize_template_cache(self):
         self.formatter_template_cache = {}
 
+    @write_api
+    def set_user_template_functions(self, user_template_functions):
+        self.backend.set_user_template_functions(user_template_functions)
+
     @write_api
     def clear_composite_caches(self, book_ids=None):
         for field in self.composites.itervalues():
@@ -351,12 +355,14 @@ class Cache(object):
             bools_are_tristate = self.backend.prefs['bools_are_tristate']
 
             for field, table in self.backend.tables.iteritems():
-                self.fields[field] = create_field(field, table, bools_are_tristate)
+                self.fields[field] = create_field(field, table, bools_are_tristate,
+                                          self.backend.get_user_template_functions)
                 if table.metadata['datatype'] == 'composite':
                     self.composites[field] = self.fields[field]
 
             self.fields['ondevice'] = create_field('ondevice',
-                    VirtualTable('ondevice'), bools_are_tristate)
+                    VirtualTable('ondevice'), bools_are_tristate,
+                    self.backend.get_user_template_functions)
 
             for name, field in self.fields.iteritems():
                 if name[0] == '#' and name.endswith('_index'):

src/calibre/db/fields.py

@@ -41,7 +41,7 @@ class Field(object):
     is_many_many = False
     is_composite = False
 
-    def __init__(self, name, table, bools_are_tristate):
+    def __init__(self, name, table, bools_are_tristate, get_user_formatter_functions):
         self.name, self.table = name, table
         dt = self.metadata['datatype']
         self.has_text_data = dt in {'text', 'comments', 'series', 'enumeration'}
@@ -88,6 +88,7 @@ class Field(object):
             self.category_formatter = calibre_langcode_to_name
         self.writer = Writer(self)
         self.series_field = None
+        self.get_user_formatter_functions = get_user_formatter_functions
 
     @property
     def metadata(self):
@@ -203,8 +204,8 @@ class CompositeField(OneToOneField):
     is_composite = True
     SIZE_SUFFIX_MAP = {suffix:i for i, suffix in enumerate(('', 'K', 'M', 'G', 'T', 'P', 'E'))}
 
-    def __init__(self, name, table, bools_are_tristate):
+    def __init__(self, name, table, bools_are_tristate, get_user_formatter_functions):
-        OneToOneField.__init__(self, name, table, bools_are_tristate)
+        OneToOneField.__init__(self, name, table, bools_are_tristate, get_user_formatter_functions)
 
         self._render_cache = {}
         self._lock = Lock()
@@ -264,7 +265,8 @@ class CompositeField(OneToOneField):
         ' INTERNAL USE ONLY. DO NOT USE THIS OUTSIDE THIS CLASS! '
         ans = formatter.safe_format(
             self.metadata['display']['composite_template'], mi, _('TEMPLATE ERROR'),
-            mi, column_name=self._composite_name, template_cache=template_cache).strip()
+            mi, column_name=self._composite_name, template_cache=template_cache,
+            user_functions=self.get_user_formatter_functions()).strip()
         with self._lock:
             self._render_cache[book_id] = ans
         return ans
@@ -347,7 +349,7 @@ class CompositeField(OneToOneField):
 
 class OnDeviceField(OneToOneField):
 
-    def __init__(self, name, table, bools_are_tristate):
+    def __init__(self, name, table, bools_are_tristate, get_user_formatter_functions):
         self.name = name
         self.book_on_device_func = None
         self.is_multiple = False
@@ -733,7 +735,7 @@ class TagsField(ManyToManyField):
         return ans
 
 
-def create_field(name, table, bools_are_tristate):
+def create_field(name, table, bools_are_tristate, get_user_formatter_functions):
     cls = {
             ONE_ONE: OneToOneField,
             MANY_ONE: ManyToOneField,
@@ -753,5 +755,5 @@ def create_field(name, table, bools_are_tristate):
         cls = CompositeField
     elif table.metadata['datatype'] == 'series':
         cls = SeriesField
-    return cls(name, table, bools_are_tristate)
+    return cls(name, table, bools_are_tristate, get_user_formatter_functions)
 

src/calibre/gui2/preferences/template_functions.py

@@ -14,7 +14,8 @@ from calibre.gui2.preferences import ConfigWidgetBase, test_widget
 from calibre.gui2.preferences.template_functions_ui import Ui_Form
 from calibre.gui2.widgets import PythonHighlighter
 from calibre.utils.formatter_functions import (formatter_functions,
-                        compile_user_function, load_user_template_functions)
+                        compile_user_function, compile_user_template_functions,
+                        load_user_template_functions)
 
 
 class ConfigWidget(ConfigWidgetBase, Ui_Form):
@@ -225,7 +226,10 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
             if name not in self.builtins:
                 pref_value.append((cls.name, cls.doc, cls.arg_count, cls.program_text))
         self.db.new_api.set_pref('user_template_functions', pref_value)
-        load_user_template_functions(self.db.library_id, pref_value)
+        funcs = compile_user_template_functions(pref_value)
+        self.db.new_api.set_user_template_functions(funcs)
+        self.gui.library_view.model().refresh()
+        load_user_template_functions(self.db.library_id, [], funcs)
         return False
 
 

src/calibre/utils/formatter.py

@@ -25,7 +25,7 @@ class _Parser(object):
 
     LEX_CONSTANTS = frozenset([LEX_STR, LEX_NUM])
 
-    def __init__(self, val, prog, parent):
+    def __init__(self, val, prog, funcs, parent):
         self.lex_pos = 0
         self.prog = prog[0]
         self.prog_len = len(self.prog)
@@ -35,6 +35,7 @@ class _Parser(object):
         self.parent_kwargs = parent.kwargs
         self.parent_book = parent.book
         self.locals = {'$':val}
+        self.funcs = funcs
 
     def error(self, message):
         m = 'Formatter: ' + message + _(' near ')
@@ -121,14 +122,13 @@ class _Parser(object):
 
     def expr(self):
         if self.token_is_id():
-            funcs = formatter_functions().get_functions()
             # We have an identifier. Determine if it is a function
             id = self.token()
             if not self.token_op_is_a_lparen():
                 if self.token_op_is_a_equals():
                     # classic assignment statement
                     self.consume()
-                    cls = funcs['assign']
+                    cls = self.funcs['assign']
                     return cls.eval_(self.parent, self.parent_kwargs,
                                     self.parent_book, self.locals, id, self.expr())
                 val = self.locals.get(id, None)
@@ -139,7 +139,7 @@ class _Parser(object):
             # Check if it is a known one. We do this here so error reporting is
             # better, as it can identify the tokens near the problem.
             id = id.strip()
-            if id not in funcs:
+            if id not in self.funcs:
                 self.error(_('unknown function {0}').format(id))
 
             # Eat the paren
@@ -163,7 +163,7 @@ class _Parser(object):
                 self.error(_('missing closing parenthesis'))
 
             # Evaluate the function
-            cls = funcs[id]
+            cls = self.funcs[id]
             if cls.arg_count != -1 and len(args) != cls.arg_count:
                 self.error('incorrect number of arguments for function {}'.format(id))
             return cls.eval_(self.parent, self.parent_kwargs,
@@ -174,142 +174,6 @@ class _Parser(object):
         else:
             self.error(_('expression is not function or constant'))
 
-
-class _CompileParser(_Parser):
-
-    def __init__(self, val, prog, parent, compile_text):
-        self.lex_pos = 0
-        self.prog = prog[0]
-        self.prog_len = len(self.prog)
-        if prog[1] != '':
-            self.error(_('failed to scan program. Invalid input {0}').format(prog[1]))
-        self.parent = parent
-        self.parent_kwargs = parent.kwargs
-        self.parent_book = parent.book
-        self.locals = {'$':val}
-        self.compile_text = compile_text
-
-    def program(self):
-        if self.compile_text:
-            t = self.compile_text
-            self.compile_text = '\n'
-        self.max_level = 0
-        val = self.statement()
-        if not self.token_is_eof():
-            self.error(_('syntax error - program ends before EOF'))
-        if self.compile_text:
-            t += "\targs=[[]"
-            for i in range(0, self.max_level):
-                t += ", None"
-            t += ']'
-            self.compile_text = t + self.compile_text + "\treturn args[0][0]\n"
-        return val
-
-    def statement(self, level=0):
-        while True:
-            val = self.expr(level)
-            if self.token_is_eof():
-                return val
-            if not self.token_op_is_a_semicolon():
-                return val
-            self.consume()
-            if self.token_is_eof():
-                return val
-            if self.compile_text:
-                self.compile_text += "\targs[%d] = list()\n"%(level,)
-
-    def expr(self, level):
-        if self.compile_text:
-            self.max_level = max(level+1, self.max_level)
-
-        if self.token_is_id():
-            funcs = formatter_functions().get_functions()
-            # We have an identifier. Determine if it is a function
-            id = self.token()
-            if not self.token_op_is_a_lparen():
-                if self.token_op_is_a_equals():
-                    # classic assignment statement
-                    self.consume()
-                    cls = funcs['assign']
-                    if self.compile_text:
-                        self.compile_text += '\targs[%d] = list()\n'%(level+1,)
-                    val = cls.eval_(self.parent, self.parent_kwargs,
-                                    self.parent_book, self.locals, id, self.expr(level+1))
-                    if self.compile_text:
-                        self.compile_text += "\tlocals['%s'] = args[%d][0]\n"%(id, level+1)
-                        self.compile_text += "\targs[%d].append(args[%d][0])\n"%(level, level+1)
-                    return val
-                val = self.locals.get(id, None)
-                if val is None:
-                    self.error(_('Unknown identifier ') + id)
-                if self.compile_text:
-                    self.compile_text += "\targs[%d].append(locals.get('%s'))\n"%(level, id)
-                return val
-            # We have a function.
-            # Check if it is a known one. We do this here so error reporting is
-            # better, as it can identify the tokens near the problem.
-            id = id.strip()
-            if id not in funcs:
-                self.error(_('unknown function {0}').format(id))
-
-            # Eat the paren
-            self.consume()
-            args = list()
-            if self.compile_text:
-                self.compile_text += '\targs[%d] = list()\n'%(level+1, )
-            if id == 'field':
-                val = self.expr(level+1)
-                val = self.parent.get_value(val, [], self.parent_kwargs)
-                if self.compile_text:
-                    self.compile_text += "\targs[%d].append(formatter.get_value(args[%d][0], [], kwargs))\n"%(level, level+1)
-                if self.token() != ')':
-                    self.error(_('missing closing parenthesis'))
-                return val
-            while not self.token_op_is_a_rparen():
-                if id == 'assign' and len(args) == 0:
-                    # Must handle the lvalue semantics of the assign function.
-                    # The first argument is the name of the destination, not
-                    # the value.
-                    if not self.token_is_id():
-                        self.error('assign requires the first parameter be an id')
-                    t = self.token()
-                    args.append(t)
-                    if self.compile_text:
-                        self.compile_text += "\targs[%d].append('%s')\n"%(level+1, t)
-                else:
-                    # evaluate the argument (recursive call)
-                    args.append(self.statement(level=level+1))
-                if not self.token_op_is_a_comma():
-                    break
-                self.consume()
-            if self.token() != ')':
-                self.error(_('missing closing parenthesis'))
-
-            # Evaluate the function
-            cls = funcs[id]
-            if cls.arg_count != -1 and len(args) != cls.arg_count:
-                self.error('incorrect number of arguments for function {}'.format(id))
-            if self.compile_text:
-                self.compile_text += (
-                    "\targs[%d].append(self.__funcs__['%s']"
-                    ".eval_(formatter, kwargs, book, locals, *args[%d]))\n")%(level, id, level+1)
-            return cls.eval_(self.parent, self.parent_kwargs,
-                            self.parent_book, self.locals, *args)
-        elif self.token_is_constant():
-            # String or number
-            v = unicode(self.token())
-            if self.compile_text:
-                tv = v.replace("\\", "\\\\")
-                tv = tv.replace("'", "\\'")
-                self.compile_text += "\targs[%d].append(unicode('%s'))\n"%(level, tv)
-            return v
-        else:
-            self.error(_('expression is not function or constant'))
-
-
-compile_counter = 0
-
-
 class TemplateFormatter(string.Formatter):
     '''
     Provides a format function that substitutes '' for any missing value
@@ -327,6 +191,7 @@ class TemplateFormatter(string.Formatter):
         self.kwargs = None
         self.strip_results = True
         self.locals = {}
+        self.funcs = formatter_functions().get_functions()
 
     def _do_format(self, val, fmt):
         if not fmt or not val:
@@ -388,36 +253,15 @@ class TemplateFormatter(string.Formatter):
         # keep a cache of the lex'ed program under the theory that re-lexing
         # is much more expensive than the cache lookup. This is certainly true
         # for more than a few tokens, but it isn't clear for simple programs.
-        if tweaks['compile_gpm_templates']:
+        if column_name is not None and self.template_cache is not None:
-            if column_name is not None and self.template_cache is not None:
+            lprog = self.template_cache.get(column_name, None)
-                lprog = self.template_cache.get(column_name, None)
+            if not lprog:
-                if lprog:
-                    return lprog.evaluate(self, self.kwargs, self.book, self.locals)
                 lprog = self.lex_scanner.scan(prog)
-                compile_text = ('__funcs__ = formatter_functions().get_functions()\n'
+                self.template_cache[column_name] = lprog
-                                'def evaluate(self, formatter, kwargs, book, locals):\n'
-                                )
-            else:
-                lprog = self.lex_scanner.scan(prog)
-                compile_text = None
-            parser = _CompileParser(val, lprog, self, compile_text)
-            val = parser.program()
-            if parser.compile_text:
-                global compile_counter
-                compile_counter += 1
-                f = compile_user_function("__A" + str(compile_counter), 'doc', -1, parser.compile_text)
-                self.template_cache[column_name] = f
         else:
-            if column_name is not None and self.template_cache is not None:
+            lprog = self.lex_scanner.scan(prog)
-                lprog = self.template_cache.get(column_name, None)
+        parser = _Parser(val, lprog, self.funcs, self)
-                if not lprog:
+        return parser.program()
-                    lprog = self.lex_scanner.scan(prog)
-                    self.template_cache[column_name] = lprog
-            else:
-                lprog = self.lex_scanner.scan(prog)
-            parser = _Parser(val, lprog, self)
-            val = parser.program()
-        return val
 
     # ################# Override parent classes methods #####################
 
@@ -462,10 +306,9 @@ class TemplateFormatter(string.Formatter):
                     dispfmt = fmt[0:colon]
                     colon += 1
 
-                funcs = formatter_functions().get_functions()
                 fname = fmt[colon:p].strip()
-                if fname in funcs:
+                if fname in self.funcs:
-                    func = funcs[fname]
+                    func = self.funcs[fname]
                     if func.arg_count == 2:
                         # only one arg expected. Don't bother to scan. Avoids need
                         # for escaping characters
@@ -516,12 +359,15 @@ class TemplateFormatter(string.Formatter):
 
     def safe_format(self, fmt, kwargs, error_value, book,
                     column_name=None, template_cache=None,
-                    strip_results=True):
+                    strip_results=True, user_functions=None):
         self.strip_results = strip_results
         self.column_name = column_name
         self.template_cache = template_cache
         self.kwargs = kwargs
         self.book = book
+        if user_functions:
+            self.funcs = formatter_functions().get_builtins().copy()
+            self.funcs.update(user_functions)
         self.composite_values = {}
         self.locals = {}
         try:

src/calibre/utils/formatter_functions.py

@@ -1622,11 +1622,8 @@ class UserFunction(FormatterUserFunction):
     cls = locals_['UserFunction'](name, doc, arg_count, eval_func)
     return cls
 
-
+def compile_user_template_functions(funcs):
-def load_user_template_functions(library_uuid, funcs):
+    compiled_funcs = {}
-    unload_user_template_functions(library_uuid)
-
-    compiled_funcs = []
     for func in funcs:
         try:
             # Force a name conflict to test the logic
@@ -1637,11 +1634,19 @@ def load_user_template_functions(library_uuid, funcs):
             # source. This helps ensure that if the function already is defined
             # then white space differences don't cause them to compare differently
 
-            compiled_funcs.append(compile_user_function(*func))
+            cls = compile_user_function(*func)
+            compiled_funcs[cls.name] = cls
         except:
             traceback.print_exc()
-    formatter_functions().register_functions(library_uuid, compiled_funcs)
+    return compiled_funcs
 
+def load_user_template_functions(library_uuid, funcs, precompiled_user_functions=None):
+    unload_user_template_functions(library_uuid)
+    if precompiled_user_functions:
+        compiled_funcs = precompiled_user_functions
+    else:
+        compiled_funcs = compile_user_template_functions(funcs)
+    formatter_functions().register_functions(library_uuid, compiled_funcs.values())
 
 def unload_user_template_functions(library_uuid):
     formatter_functions().unregister_functions(library_uuid)