diff --git a/resources/default_tweaks.py b/resources/default_tweaks.py index ab078629ed..81e471598b 100644 --- a/resources/default_tweaks.py +++ b/resources/default_tweaks.py @@ -485,16 +485,6 @@ gui_view_history_size = 15 # negative number to increase or decrease the font size. change_book_details_font_size_by = 0 -#: Compile general program mode templates to Python -# Compiled general program mode templates are significantly faster than -# interpreted templates. Setting this tweak to True causes calibre to compile -# (in most cases) general program mode templates. Setting it to False causes -# calibre to use the old behavior -- interpreting the templates. Set the tweak -# to False if some compiled templates produce incorrect values. -# Default: compile_gpm_templates = True -# No compile: compile_gpm_templates = False -compile_gpm_templates = True - #: What format to default to when using the "Unpack book" feature # The "Unpack book" feature of calibre allows direct editing of a book format. # If multiple formats are available, calibre will offer you a choice diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index a41080be5b..b96855047a 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -31,7 +31,9 @@ from calibre.utils.filenames import ( WindowsAtomicFolderMove, atomic_rename, remove_dir_if_empty, copytree_using_links, copyfile_using_links) from calibre.utils.img import save_cover_data_to -from calibre.utils.formatter_functions import load_user_template_functions, unload_user_template_functions +from calibre.utils.formatter_functions import (load_user_template_functions, + unload_user_template_functions, + compile_user_template_functions) from calibre.db.tables import (OneToOneTable, ManyToOneTable, ManyToManyTable, SizeTable, FormatsTable, AuthorsTable, IdentifiersTable, PathTable, CompositeTable, UUIDTable, RatingTable) @@ -316,9 +318,14 @@ class Connection(apsw.Connection): # {{{ # }}} -def set_global_state(backend): - load_user_template_functions(backend.library_id, - backend.prefs.get('user_template_functions', [])) +def set_global_state(backend, precompiled_user_functions=None): + if precompiled_user_functions: + load_user_template_functions(backend.library_id, + [], + precompiled_user_functions=precompiled_user_functions) + else: + load_user_template_functions(backend.library_id, + backend.prefs.get('user_template_functions', [])) class DB(object): @@ -406,8 +413,16 @@ class DB(object): self.initialize_prefs(default_prefs, restore_all_prefs, progress_callback) self.initialize_custom_columns() self.initialize_tables() + self.set_user_template_functions(compile_user_template_functions( + self.prefs.get('user_template_functions', []))) if load_user_formatter_functions: - set_global_state(self) + set_global_state(self, precompiled_user_functions = self.get_user_template_functions()) + + def get_user_template_functions(self): + return self._user_template_functions + + def set_user_template_functions(self, user_formatter_functions): + self._user_template_functions = user_formatter_functions def initialize_prefs(self, default_prefs, restore_all_prefs, progress_callback): # {{{ self.prefs = DBPrefs(self) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 39f2acf341..17450cbae0 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -203,6 +203,10 @@ class Cache(object): def initialize_template_cache(self): self.formatter_template_cache = {} + @write_api + def set_user_template_functions(self, user_template_functions): + self.backend.set_user_template_functions(user_template_functions) + @write_api def clear_composite_caches(self, book_ids=None): for field in self.composites.itervalues(): @@ -351,12 +355,14 @@ class Cache(object): bools_are_tristate = self.backend.prefs['bools_are_tristate'] for field, table in self.backend.tables.iteritems(): - self.fields[field] = create_field(field, table, bools_are_tristate) + self.fields[field] = create_field(field, table, bools_are_tristate, + self.backend.get_user_template_functions) if table.metadata['datatype'] == 'composite': self.composites[field] = self.fields[field] self.fields['ondevice'] = create_field('ondevice', - VirtualTable('ondevice'), bools_are_tristate) + VirtualTable('ondevice'), bools_are_tristate, + self.backend.get_user_template_functions) for name, field in self.fields.iteritems(): if name[0] == '#' and name.endswith('_index'): diff --git a/src/calibre/db/fields.py b/src/calibre/db/fields.py index 1fa013d103..21611d4258 100644 --- a/src/calibre/db/fields.py +++ b/src/calibre/db/fields.py @@ -41,7 +41,7 @@ class Field(object): is_many_many = False is_composite = False - def __init__(self, name, table, bools_are_tristate): + def __init__(self, name, table, bools_are_tristate, get_user_formatter_functions): self.name, self.table = name, table dt = self.metadata['datatype'] self.has_text_data = dt in {'text', 'comments', 'series', 'enumeration'} @@ -88,6 +88,7 @@ class Field(object): self.category_formatter = calibre_langcode_to_name self.writer = Writer(self) self.series_field = None + self.get_user_formatter_functions = get_user_formatter_functions @property def metadata(self): @@ -203,8 +204,8 @@ class CompositeField(OneToOneField): is_composite = True SIZE_SUFFIX_MAP = {suffix:i for i, suffix in enumerate(('', 'K', 'M', 'G', 'T', 'P', 'E'))} - def __init__(self, name, table, bools_are_tristate): - OneToOneField.__init__(self, name, table, bools_are_tristate) + def __init__(self, name, table, bools_are_tristate, get_user_formatter_functions): + OneToOneField.__init__(self, name, table, bools_are_tristate, get_user_formatter_functions) self._render_cache = {} self._lock = Lock() @@ -264,7 +265,8 @@ class CompositeField(OneToOneField): ' INTERNAL USE ONLY. DO NOT USE THIS OUTSIDE THIS CLASS! ' ans = formatter.safe_format( self.metadata['display']['composite_template'], mi, _('TEMPLATE ERROR'), - mi, column_name=self._composite_name, template_cache=template_cache).strip() + mi, column_name=self._composite_name, template_cache=template_cache, + user_functions=self.get_user_formatter_functions()).strip() with self._lock: self._render_cache[book_id] = ans return ans @@ -347,7 +349,7 @@ class CompositeField(OneToOneField): class OnDeviceField(OneToOneField): - def __init__(self, name, table, bools_are_tristate): + def __init__(self, name, table, bools_are_tristate, get_user_formatter_functions): self.name = name self.book_on_device_func = None self.is_multiple = False @@ -733,7 +735,7 @@ class TagsField(ManyToManyField): return ans -def create_field(name, table, bools_are_tristate): +def create_field(name, table, bools_are_tristate, get_user_formatter_functions): cls = { ONE_ONE: OneToOneField, MANY_ONE: ManyToOneField, @@ -753,5 +755,5 @@ def create_field(name, table, bools_are_tristate): cls = CompositeField elif table.metadata['datatype'] == 'series': cls = SeriesField - return cls(name, table, bools_are_tristate) + return cls(name, table, bools_are_tristate, get_user_formatter_functions) diff --git a/src/calibre/gui2/preferences/template_functions.py b/src/calibre/gui2/preferences/template_functions.py index 3ebfe54cc4..9b67465ab5 100644 --- a/src/calibre/gui2/preferences/template_functions.py +++ b/src/calibre/gui2/preferences/template_functions.py @@ -14,7 +14,8 @@ from calibre.gui2.preferences import ConfigWidgetBase, test_widget from calibre.gui2.preferences.template_functions_ui import Ui_Form from calibre.gui2.widgets import PythonHighlighter from calibre.utils.formatter_functions import (formatter_functions, - compile_user_function, load_user_template_functions) + compile_user_function, compile_user_template_functions, + load_user_template_functions) class ConfigWidget(ConfigWidgetBase, Ui_Form): @@ -225,7 +226,10 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): if name not in self.builtins: pref_value.append((cls.name, cls.doc, cls.arg_count, cls.program_text)) self.db.new_api.set_pref('user_template_functions', pref_value) - load_user_template_functions(self.db.library_id, pref_value) + funcs = compile_user_template_functions(pref_value) + self.db.new_api.set_user_template_functions(funcs) + self.gui.library_view.model().refresh() + load_user_template_functions(self.db.library_id, [], funcs) return False diff --git a/src/calibre/utils/formatter.py b/src/calibre/utils/formatter.py index 177ead9eb5..3c93663360 100644 --- a/src/calibre/utils/formatter.py +++ b/src/calibre/utils/formatter.py @@ -25,7 +25,7 @@ class _Parser(object): LEX_CONSTANTS = frozenset([LEX_STR, LEX_NUM]) - def __init__(self, val, prog, parent): + def __init__(self, val, prog, funcs, parent): self.lex_pos = 0 self.prog = prog[0] self.prog_len = len(self.prog) @@ -35,6 +35,7 @@ class _Parser(object): self.parent_kwargs = parent.kwargs self.parent_book = parent.book self.locals = {'$':val} + self.funcs = funcs def error(self, message): m = 'Formatter: ' + message + _(' near ') @@ -121,14 +122,13 @@ class _Parser(object): def expr(self): if self.token_is_id(): - funcs = formatter_functions().get_functions() # We have an identifier. Determine if it is a function id = self.token() if not self.token_op_is_a_lparen(): if self.token_op_is_a_equals(): # classic assignment statement self.consume() - cls = funcs['assign'] + cls = self.funcs['assign'] return cls.eval_(self.parent, self.parent_kwargs, self.parent_book, self.locals, id, self.expr()) val = self.locals.get(id, None) @@ -139,7 +139,7 @@ class _Parser(object): # Check if it is a known one. We do this here so error reporting is # better, as it can identify the tokens near the problem. id = id.strip() - if id not in funcs: + if id not in self.funcs: self.error(_('unknown function {0}').format(id)) # Eat the paren @@ -163,7 +163,7 @@ class _Parser(object): self.error(_('missing closing parenthesis')) # Evaluate the function - cls = funcs[id] + cls = self.funcs[id] if cls.arg_count != -1 and len(args) != cls.arg_count: self.error('incorrect number of arguments for function {}'.format(id)) return cls.eval_(self.parent, self.parent_kwargs, @@ -174,142 +174,6 @@ class _Parser(object): else: self.error(_('expression is not function or constant')) - -class _CompileParser(_Parser): - - def __init__(self, val, prog, parent, compile_text): - self.lex_pos = 0 - self.prog = prog[0] - self.prog_len = len(self.prog) - if prog[1] != '': - self.error(_('failed to scan program. Invalid input {0}').format(prog[1])) - self.parent = parent - self.parent_kwargs = parent.kwargs - self.parent_book = parent.book - self.locals = {'$':val} - self.compile_text = compile_text - - def program(self): - if self.compile_text: - t = self.compile_text - self.compile_text = '\n' - self.max_level = 0 - val = self.statement() - if not self.token_is_eof(): - self.error(_('syntax error - program ends before EOF')) - if self.compile_text: - t += "\targs=[[]" - for i in range(0, self.max_level): - t += ", None" - t += ']' - self.compile_text = t + self.compile_text + "\treturn args[0][0]\n" - return val - - def statement(self, level=0): - while True: - val = self.expr(level) - if self.token_is_eof(): - return val - if not self.token_op_is_a_semicolon(): - return val - self.consume() - if self.token_is_eof(): - return val - if self.compile_text: - self.compile_text += "\targs[%d] = list()\n"%(level,) - - def expr(self, level): - if self.compile_text: - self.max_level = max(level+1, self.max_level) - - if self.token_is_id(): - funcs = formatter_functions().get_functions() - # We have an identifier. Determine if it is a function - id = self.token() - if not self.token_op_is_a_lparen(): - if self.token_op_is_a_equals(): - # classic assignment statement - self.consume() - cls = funcs['assign'] - if self.compile_text: - self.compile_text += '\targs[%d] = list()\n'%(level+1,) - val = cls.eval_(self.parent, self.parent_kwargs, - self.parent_book, self.locals, id, self.expr(level+1)) - if self.compile_text: - self.compile_text += "\tlocals['%s'] = args[%d][0]\n"%(id, level+1) - self.compile_text += "\targs[%d].append(args[%d][0])\n"%(level, level+1) - return val - val = self.locals.get(id, None) - if val is None: - self.error(_('Unknown identifier ') + id) - if self.compile_text: - self.compile_text += "\targs[%d].append(locals.get('%s'))\n"%(level, id) - return val - # We have a function. - # Check if it is a known one. We do this here so error reporting is - # better, as it can identify the tokens near the problem. - id = id.strip() - if id not in funcs: - self.error(_('unknown function {0}').format(id)) - - # Eat the paren - self.consume() - args = list() - if self.compile_text: - self.compile_text += '\targs[%d] = list()\n'%(level+1, ) - if id == 'field': - val = self.expr(level+1) - val = self.parent.get_value(val, [], self.parent_kwargs) - if self.compile_text: - self.compile_text += "\targs[%d].append(formatter.get_value(args[%d][0], [], kwargs))\n"%(level, level+1) - if self.token() != ')': - self.error(_('missing closing parenthesis')) - return val - while not self.token_op_is_a_rparen(): - if id == 'assign' and len(args) == 0: - # Must handle the lvalue semantics of the assign function. - # The first argument is the name of the destination, not - # the value. - if not self.token_is_id(): - self.error('assign requires the first parameter be an id') - t = self.token() - args.append(t) - if self.compile_text: - self.compile_text += "\targs[%d].append('%s')\n"%(level+1, t) - else: - # evaluate the argument (recursive call) - args.append(self.statement(level=level+1)) - if not self.token_op_is_a_comma(): - break - self.consume() - if self.token() != ')': - self.error(_('missing closing parenthesis')) - - # Evaluate the function - cls = funcs[id] - if cls.arg_count != -1 and len(args) != cls.arg_count: - self.error('incorrect number of arguments for function {}'.format(id)) - if self.compile_text: - self.compile_text += ( - "\targs[%d].append(self.__funcs__['%s']" - ".eval_(formatter, kwargs, book, locals, *args[%d]))\n")%(level, id, level+1) - return cls.eval_(self.parent, self.parent_kwargs, - self.parent_book, self.locals, *args) - elif self.token_is_constant(): - # String or number - v = unicode(self.token()) - if self.compile_text: - tv = v.replace("\\", "\\\\") - tv = tv.replace("'", "\\'") - self.compile_text += "\targs[%d].append(unicode('%s'))\n"%(level, tv) - return v - else: - self.error(_('expression is not function or constant')) - - -compile_counter = 0 - - class TemplateFormatter(string.Formatter): ''' Provides a format function that substitutes '' for any missing value @@ -327,6 +191,7 @@ class TemplateFormatter(string.Formatter): self.kwargs = None self.strip_results = True self.locals = {} + self.funcs = formatter_functions().get_functions() def _do_format(self, val, fmt): if not fmt or not val: @@ -388,36 +253,15 @@ class TemplateFormatter(string.Formatter): # keep a cache of the lex'ed program under the theory that re-lexing # is much more expensive than the cache lookup. This is certainly true # for more than a few tokens, but it isn't clear for simple programs. - if tweaks['compile_gpm_templates']: - if column_name is not None and self.template_cache is not None: - lprog = self.template_cache.get(column_name, None) - if lprog: - return lprog.evaluate(self, self.kwargs, self.book, self.locals) + if column_name is not None and self.template_cache is not None: + lprog = self.template_cache.get(column_name, None) + if not lprog: lprog = self.lex_scanner.scan(prog) - compile_text = ('__funcs__ = formatter_functions().get_functions()\n' - 'def evaluate(self, formatter, kwargs, book, locals):\n' - ) - else: - lprog = self.lex_scanner.scan(prog) - compile_text = None - parser = _CompileParser(val, lprog, self, compile_text) - val = parser.program() - if parser.compile_text: - global compile_counter - compile_counter += 1 - f = compile_user_function("__A" + str(compile_counter), 'doc', -1, parser.compile_text) - self.template_cache[column_name] = f + self.template_cache[column_name] = lprog else: - if column_name is not None and self.template_cache is not None: - lprog = self.template_cache.get(column_name, None) - if not lprog: - lprog = self.lex_scanner.scan(prog) - self.template_cache[column_name] = lprog - else: - lprog = self.lex_scanner.scan(prog) - parser = _Parser(val, lprog, self) - val = parser.program() - return val + lprog = self.lex_scanner.scan(prog) + parser = _Parser(val, lprog, self.funcs, self) + return parser.program() # ################# Override parent classes methods ##################### @@ -462,10 +306,9 @@ class TemplateFormatter(string.Formatter): dispfmt = fmt[0:colon] colon += 1 - funcs = formatter_functions().get_functions() fname = fmt[colon:p].strip() - if fname in funcs: - func = funcs[fname] + if fname in self.funcs: + func = self.funcs[fname] if func.arg_count == 2: # only one arg expected. Don't bother to scan. Avoids need # for escaping characters @@ -516,12 +359,15 @@ class TemplateFormatter(string.Formatter): def safe_format(self, fmt, kwargs, error_value, book, column_name=None, template_cache=None, - strip_results=True): + strip_results=True, user_functions=None): self.strip_results = strip_results self.column_name = column_name self.template_cache = template_cache self.kwargs = kwargs self.book = book + if user_functions: + self.funcs = formatter_functions().get_builtins().copy() + self.funcs.update(user_functions) self.composite_values = {} self.locals = {} try: diff --git a/src/calibre/utils/formatter_functions.py b/src/calibre/utils/formatter_functions.py index 211e2e89ac..79b20da9cc 100644 --- a/src/calibre/utils/formatter_functions.py +++ b/src/calibre/utils/formatter_functions.py @@ -1622,11 +1622,8 @@ class UserFunction(FormatterUserFunction): cls = locals_['UserFunction'](name, doc, arg_count, eval_func) return cls - -def load_user_template_functions(library_uuid, funcs): - unload_user_template_functions(library_uuid) - - compiled_funcs = [] +def compile_user_template_functions(funcs): + compiled_funcs = {} for func in funcs: try: # Force a name conflict to test the logic @@ -1637,11 +1634,19 @@ def load_user_template_functions(library_uuid, funcs): # source. This helps ensure that if the function already is defined # then white space differences don't cause them to compare differently - compiled_funcs.append(compile_user_function(*func)) + cls = compile_user_function(*func) + compiled_funcs[cls.name] = cls except: traceback.print_exc() - formatter_functions().register_functions(library_uuid, compiled_funcs) + return compiled_funcs +def load_user_template_functions(library_uuid, funcs, precompiled_user_functions=None): + unload_user_template_functions(library_uuid) + if precompiled_user_functions: + compiled_funcs = precompiled_user_functions + else: + compiled_funcs = compile_user_template_functions(funcs) + formatter_functions().register_functions(library_uuid, compiled_funcs.values()) def unload_user_template_functions(library_uuid): formatter_functions().unregister_functions(library_uuid)