mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-07-09 03:04:10 -04:00
Store and access user template functions on a per-library basis. This is used when the DB evaluates composites, avoiding using the global copy.
The ability to pre-compile templates was removed because the one-true list of available functions is known only at execution time. The compiled copy depended on the global state being accurate. And in any event, I was never totally convinced that the compiled version was significantly faster than the pre-lexed version. Includes a bug fix: the booklist must be refreshed if a user-defined function changed.
This commit is contained in:
Charles Haley
Kovid Goyal
parent
c77ffc6075
commit
1aa59276bd
@ -485,16 +485,6 @@ gui_view_history_size = 15
|
||||
# negative number to increase or decrease the font size.
|
||||
change_book_details_font_size_by = 0
|
||||
|
||||
#: Compile general program mode templates to Python
|
||||
# Compiled general program mode templates are significantly faster than
|
||||
# interpreted templates. Setting this tweak to True causes calibre to compile
|
||||
# (in most cases) general program mode templates. Setting it to False causes
|
||||
# calibre to use the old behavior -- interpreting the templates. Set the tweak
|
||||
# to False if some compiled templates produce incorrect values.
|
||||
# Default: compile_gpm_templates = True
|
||||
# No compile: compile_gpm_templates = False
|
||||
compile_gpm_templates = True
|
||||
|
||||
#: What format to default to when using the "Unpack book" feature
|
||||
# The "Unpack book" feature of calibre allows direct editing of a book format.
|
||||
# If multiple formats are available, calibre will offer you a choice
|
||||
|
||||
@ -31,7 +31,9 @@ from calibre.utils.filenames import (
|
||||
WindowsAtomicFolderMove, atomic_rename, remove_dir_if_empty,
|
||||
copytree_using_links, copyfile_using_links)
|
||||
from calibre.utils.img import save_cover_data_to
|
||||
from calibre.utils.formatter_functions import load_user_template_functions, unload_user_template_functions
|
||||
from calibre.utils.formatter_functions import (load_user_template_functions,
|
||||
unload_user_template_functions,
|
||||
compile_user_template_functions)
|
||||
from calibre.db.tables import (OneToOneTable, ManyToOneTable, ManyToManyTable,
|
||||
SizeTable, FormatsTable, AuthorsTable, IdentifiersTable, PathTable,
|
||||
CompositeTable, UUIDTable, RatingTable)
|
||||
@ -316,7 +318,12 @@ class Connection(apsw.Connection): # {{{
|
||||
# }}}
|
||||
|
||||
|
||||
def set_global_state(backend):
|
||||
def set_global_state(backend, precompiled_user_functions=None):
|
||||
if precompiled_user_functions:
|
||||
load_user_template_functions(backend.library_id,
|
||||
[],
|
||||
precompiled_user_functions=precompiled_user_functions)
|
||||
else:
|
||||
load_user_template_functions(backend.library_id,
|
||||
backend.prefs.get('user_template_functions', []))
|
||||
|
||||
@ -406,8 +413,16 @@ class DB(object):
|
||||
self.initialize_prefs(default_prefs, restore_all_prefs, progress_callback)
|
||||
self.initialize_custom_columns()
|
||||
self.initialize_tables()
|
||||
self.set_user_template_functions(compile_user_template_functions(
|
||||
self.prefs.get('user_template_functions', [])))
|
||||
if load_user_formatter_functions:
|
||||
set_global_state(self)
|
||||
set_global_state(self, precompiled_user_functions = self.get_user_template_functions())
|
||||
|
||||
def get_user_template_functions(self):
|
||||
return self._user_template_functions
|
||||
|
||||
def set_user_template_functions(self, user_formatter_functions):
|
||||
self._user_template_functions = user_formatter_functions
|
||||
|
||||
def initialize_prefs(self, default_prefs, restore_all_prefs, progress_callback): # {{{
|
||||
self.prefs = DBPrefs(self)
|
||||
|
||||
@ -203,6 +203,10 @@ class Cache(object):
|
||||
def initialize_template_cache(self):
|
||||
self.formatter_template_cache = {}
|
||||
|
||||
@write_api
|
||||
def set_user_template_functions(self, user_template_functions):
|
||||
self.backend.set_user_template_functions(user_template_functions)
|
||||
|
||||
@write_api
|
||||
def clear_composite_caches(self, book_ids=None):
|
||||
for field in self.composites.itervalues():
|
||||
@ -351,12 +355,14 @@ class Cache(object):
|
||||
bools_are_tristate = self.backend.prefs['bools_are_tristate']
|
||||
|
||||
for field, table in self.backend.tables.iteritems():
|
||||
self.fields[field] = create_field(field, table, bools_are_tristate)
|
||||
self.fields[field] = create_field(field, table, bools_are_tristate,
|
||||
self.backend.get_user_template_functions)
|
||||
if table.metadata['datatype'] == 'composite':
|
||||
self.composites[field] = self.fields[field]
|
||||
|
||||
self.fields['ondevice'] = create_field('ondevice',
|
||||
VirtualTable('ondevice'), bools_are_tristate)
|
||||
VirtualTable('ondevice'), bools_are_tristate,
|
||||
self.backend.get_user_template_functions)
|
||||
|
||||
for name, field in self.fields.iteritems():
|
||||
if name[0] == '#' and name.endswith('_index'):
|
||||
|
||||
@ -41,7 +41,7 @@ class Field(object):
|
||||
is_many_many = False
|
||||
is_composite = False
|
||||
|
||||
def __init__(self, name, table, bools_are_tristate):
|
||||
def __init__(self, name, table, bools_are_tristate, get_user_formatter_functions):
|
||||
self.name, self.table = name, table
|
||||
dt = self.metadata['datatype']
|
||||
self.has_text_data = dt in {'text', 'comments', 'series', 'enumeration'}
|
||||
@ -88,6 +88,7 @@ class Field(object):
|
||||
self.category_formatter = calibre_langcode_to_name
|
||||
self.writer = Writer(self)
|
||||
self.series_field = None
|
||||
self.get_user_formatter_functions = get_user_formatter_functions
|
||||
|
||||
@property
|
||||
def metadata(self):
|
||||
@ -203,8 +204,8 @@ class CompositeField(OneToOneField):
|
||||
is_composite = True
|
||||
SIZE_SUFFIX_MAP = {suffix:i for i, suffix in enumerate(('', 'K', 'M', 'G', 'T', 'P', 'E'))}
|
||||
|
||||
def __init__(self, name, table, bools_are_tristate):
|
||||
OneToOneField.__init__(self, name, table, bools_are_tristate)
|
||||
def __init__(self, name, table, bools_are_tristate, get_user_formatter_functions):
|
||||
OneToOneField.__init__(self, name, table, bools_are_tristate, get_user_formatter_functions)
|
||||
|
||||
self._render_cache = {}
|
||||
self._lock = Lock()
|
||||
@ -264,7 +265,8 @@ class CompositeField(OneToOneField):
|
||||
' INTERNAL USE ONLY. DO NOT USE THIS OUTSIDE THIS CLASS! '
|
||||
ans = formatter.safe_format(
|
||||
self.metadata['display']['composite_template'], mi, _('TEMPLATE ERROR'),
|
||||
mi, column_name=self._composite_name, template_cache=template_cache).strip()
|
||||
mi, column_name=self._composite_name, template_cache=template_cache,
|
||||
user_functions=self.get_user_formatter_functions()).strip()
|
||||
with self._lock:
|
||||
self._render_cache[book_id] = ans
|
||||
return ans
|
||||
@ -347,7 +349,7 @@ class CompositeField(OneToOneField):
|
||||
|
||||
class OnDeviceField(OneToOneField):
|
||||
|
||||
def __init__(self, name, table, bools_are_tristate):
|
||||
def __init__(self, name, table, bools_are_tristate, get_user_formatter_functions):
|
||||
self.name = name
|
||||
self.book_on_device_func = None
|
||||
self.is_multiple = False
|
||||
@ -733,7 +735,7 @@ class TagsField(ManyToManyField):
|
||||
return ans
|
||||
|
||||
|
||||
def create_field(name, table, bools_are_tristate):
|
||||
def create_field(name, table, bools_are_tristate, get_user_formatter_functions):
|
||||
cls = {
|
||||
ONE_ONE: OneToOneField,
|
||||
MANY_ONE: ManyToOneField,
|
||||
@ -753,5 +755,5 @@ def create_field(name, table, bools_are_tristate):
|
||||
cls = CompositeField
|
||||
elif table.metadata['datatype'] == 'series':
|
||||
cls = SeriesField
|
||||
return cls(name, table, bools_are_tristate)
|
||||
return cls(name, table, bools_are_tristate, get_user_formatter_functions)
|
||||
|
||||
|
||||
@ -14,7 +14,8 @@ from calibre.gui2.preferences import ConfigWidgetBase, test_widget
|
||||
from calibre.gui2.preferences.template_functions_ui import Ui_Form
|
||||
from calibre.gui2.widgets import PythonHighlighter
|
||||
from calibre.utils.formatter_functions import (formatter_functions,
|
||||
compile_user_function, load_user_template_functions)
|
||||
compile_user_function, compile_user_template_functions,
|
||||
load_user_template_functions)
|
||||
|
||||
|
||||
class ConfigWidget(ConfigWidgetBase, Ui_Form):
|
||||
@ -225,7 +226,10 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
|
||||
if name not in self.builtins:
|
||||
pref_value.append((cls.name, cls.doc, cls.arg_count, cls.program_text))
|
||||
self.db.new_api.set_pref('user_template_functions', pref_value)
|
||||
load_user_template_functions(self.db.library_id, pref_value)
|
||||
funcs = compile_user_template_functions(pref_value)
|
||||
self.db.new_api.set_user_template_functions(funcs)
|
||||
self.gui.library_view.model().refresh()
|
||||
load_user_template_functions(self.db.library_id, [], funcs)
|
||||
return False
|
||||
|
||||
|
||||
|
||||
@ -25,7 +25,7 @@ class _Parser(object):
|
||||
|
||||
LEX_CONSTANTS = frozenset([LEX_STR, LEX_NUM])
|
||||
|
||||
def __init__(self, val, prog, parent):
|
||||
def __init__(self, val, prog, funcs, parent):
|
||||
self.lex_pos = 0
|
||||
self.prog = prog[0]
|
||||
self.prog_len = len(self.prog)
|
||||
@ -35,6 +35,7 @@ class _Parser(object):
|
||||
self.parent_kwargs = parent.kwargs
|
||||
self.parent_book = parent.book
|
||||
self.locals = {'$':val}
|
||||
self.funcs = funcs
|
||||
|
||||
def error(self, message):
|
||||
m = 'Formatter: ' + message + _(' near ')
|
||||
@ -121,14 +122,13 @@ class _Parser(object):
|
||||
|
||||
def expr(self):
|
||||
if self.token_is_id():
|
||||
funcs = formatter_functions().get_functions()
|
||||
# We have an identifier. Determine if it is a function
|
||||
id = self.token()
|
||||
if not self.token_op_is_a_lparen():
|
||||
if self.token_op_is_a_equals():
|
||||
# classic assignment statement
|
||||
self.consume()
|
||||
cls = funcs['assign']
|
||||
cls = self.funcs['assign']
|
||||
return cls.eval_(self.parent, self.parent_kwargs,
|
||||
self.parent_book, self.locals, id, self.expr())
|
||||
val = self.locals.get(id, None)
|
||||
@ -139,7 +139,7 @@ class _Parser(object):
|
||||
# Check if it is a known one. We do this here so error reporting is
|
||||
# better, as it can identify the tokens near the problem.
|
||||
id = id.strip()
|
||||
if id not in funcs:
|
||||
if id not in self.funcs:
|
||||
self.error(_('unknown function {0}').format(id))
|
||||
|
||||
# Eat the paren
|
||||
@ -163,7 +163,7 @@ class _Parser(object):
|
||||
self.error(_('missing closing parenthesis'))
|
||||
|
||||
# Evaluate the function
|
||||
cls = funcs[id]
|
||||
cls = self.funcs[id]
|
||||
if cls.arg_count != -1 and len(args) != cls.arg_count:
|
||||
self.error('incorrect number of arguments for function {}'.format(id))
|
||||
return cls.eval_(self.parent, self.parent_kwargs,
|
||||
@ -174,142 +174,6 @@ class _Parser(object):
|
||||
else:
|
||||
self.error(_('expression is not function or constant'))
|
||||
|
||||
|
||||
class _CompileParser(_Parser):
|
||||
|
||||
def __init__(self, val, prog, parent, compile_text):
|
||||
self.lex_pos = 0
|
||||
self.prog = prog[0]
|
||||
self.prog_len = len(self.prog)
|
||||
if prog[1] != '':
|
||||
self.error(_('failed to scan program. Invalid input {0}').format(prog[1]))
|
||||
self.parent = parent
|
||||
self.parent_kwargs = parent.kwargs
|
||||
self.parent_book = parent.book
|
||||
self.locals = {'$':val}
|
||||
self.compile_text = compile_text
|
||||
|
||||
def program(self):
|
||||
if self.compile_text:
|
||||
t = self.compile_text
|
||||
self.compile_text = '\n'
|
||||
self.max_level = 0
|
||||
val = self.statement()
|
||||
if not self.token_is_eof():
|
||||
self.error(_('syntax error - program ends before EOF'))
|
||||
if self.compile_text:
|
||||
t += "\targs=[[]"
|
||||
for i in range(0, self.max_level):
|
||||
t += ", None"
|
||||
t += ']'
|
||||
self.compile_text = t + self.compile_text + "\treturn args[0][0]\n"
|
||||
return val
|
||||
|
||||
def statement(self, level=0):
|
||||
while True:
|
||||
val = self.expr(level)
|
||||
if self.token_is_eof():
|
||||
return val
|
||||
if not self.token_op_is_a_semicolon():
|
||||
return val
|
||||
self.consume()
|
||||
if self.token_is_eof():
|
||||
return val
|
||||
if self.compile_text:
|
||||
self.compile_text += "\targs[%d] = list()\n"%(level,)
|
||||
|
||||
def expr(self, level):
|
||||
if self.compile_text:
|
||||
self.max_level = max(level+1, self.max_level)
|
||||
|
||||
if self.token_is_id():
|
||||
funcs = formatter_functions().get_functions()
|
||||
# We have an identifier. Determine if it is a function
|
||||
id = self.token()
|
||||
if not self.token_op_is_a_lparen():
|
||||
if self.token_op_is_a_equals():
|
||||
# classic assignment statement
|
||||
self.consume()
|
||||
cls = funcs['assign']
|
||||
if self.compile_text:
|
||||
self.compile_text += '\targs[%d] = list()\n'%(level+1,)
|
||||
val = cls.eval_(self.parent, self.parent_kwargs,
|
||||
self.parent_book, self.locals, id, self.expr(level+1))
|
||||
if self.compile_text:
|
||||
self.compile_text += "\tlocals['%s'] = args[%d][0]\n"%(id, level+1)
|
||||
self.compile_text += "\targs[%d].append(args[%d][0])\n"%(level, level+1)
|
||||
return val
|
||||
val = self.locals.get(id, None)
|
||||
if val is None:
|
||||
self.error(_('Unknown identifier ') + id)
|
||||
if self.compile_text:
|
||||
self.compile_text += "\targs[%d].append(locals.get('%s'))\n"%(level, id)
|
||||
return val
|
||||
# We have a function.
|
||||
# Check if it is a known one. We do this here so error reporting is
|
||||
# better, as it can identify the tokens near the problem.
|
||||
id = id.strip()
|
||||
if id not in funcs:
|
||||
self.error(_('unknown function {0}').format(id))
|
||||
|
||||
# Eat the paren
|
||||
self.consume()
|
||||
args = list()
|
||||
if self.compile_text:
|
||||
self.compile_text += '\targs[%d] = list()\n'%(level+1, )
|
||||
if id == 'field':
|
||||
val = self.expr(level+1)
|
||||
val = self.parent.get_value(val, [], self.parent_kwargs)
|
||||
if self.compile_text:
|
||||
self.compile_text += "\targs[%d].append(formatter.get_value(args[%d][0], [], kwargs))\n"%(level, level+1)
|
||||
if self.token() != ')':
|
||||
self.error(_('missing closing parenthesis'))
|
||||
return val
|
||||
while not self.token_op_is_a_rparen():
|
||||
if id == 'assign' and len(args) == 0:
|
||||
# Must handle the lvalue semantics of the assign function.
|
||||
# The first argument is the name of the destination, not
|
||||
# the value.
|
||||
if not self.token_is_id():
|
||||
self.error('assign requires the first parameter be an id')
|
||||
t = self.token()
|
||||
args.append(t)
|
||||
if self.compile_text:
|
||||
self.compile_text += "\targs[%d].append('%s')\n"%(level+1, t)
|
||||
else:
|
||||
# evaluate the argument (recursive call)
|
||||
args.append(self.statement(level=level+1))
|
||||
if not self.token_op_is_a_comma():
|
||||
break
|
||||
self.consume()
|
||||
if self.token() != ')':
|
||||
self.error(_('missing closing parenthesis'))
|
||||
|
||||
# Evaluate the function
|
||||
cls = funcs[id]
|
||||
if cls.arg_count != -1 and len(args) != cls.arg_count:
|
||||
self.error('incorrect number of arguments for function {}'.format(id))
|
||||
if self.compile_text:
|
||||
self.compile_text += (
|
||||
"\targs[%d].append(self.__funcs__['%s']"
|
||||
".eval_(formatter, kwargs, book, locals, *args[%d]))\n")%(level, id, level+1)
|
||||
return cls.eval_(self.parent, self.parent_kwargs,
|
||||
self.parent_book, self.locals, *args)
|
||||
elif self.token_is_constant():
|
||||
# String or number
|
||||
v = unicode(self.token())
|
||||
if self.compile_text:
|
||||
tv = v.replace("\\", "\\\\")
|
||||
tv = tv.replace("'", "\\'")
|
||||
self.compile_text += "\targs[%d].append(unicode('%s'))\n"%(level, tv)
|
||||
return v
|
||||
else:
|
||||
self.error(_('expression is not function or constant'))
|
||||
|
||||
|
||||
compile_counter = 0
|
||||
|
||||
|
||||
class TemplateFormatter(string.Formatter):
|
||||
'''
|
||||
Provides a format function that substitutes '' for any missing value
|
||||
@ -327,6 +191,7 @@ class TemplateFormatter(string.Formatter):
|
||||
self.kwargs = None
|
||||
self.strip_results = True
|
||||
self.locals = {}
|
||||
self.funcs = formatter_functions().get_functions()
|
||||
|
||||
def _do_format(self, val, fmt):
|
||||
if not fmt or not val:
|
||||
@ -388,26 +253,6 @@ class TemplateFormatter(string.Formatter):
|
||||
# keep a cache of the lex'ed program under the theory that re-lexing
|
||||
# is much more expensive than the cache lookup. This is certainly true
|
||||
# for more than a few tokens, but it isn't clear for simple programs.
|
||||
if tweaks['compile_gpm_templates']:
|
||||
if column_name is not None and self.template_cache is not None:
|
||||
lprog = self.template_cache.get(column_name, None)
|
||||
if lprog:
|
||||
return lprog.evaluate(self, self.kwargs, self.book, self.locals)
|
||||
lprog = self.lex_scanner.scan(prog)
|
||||
compile_text = ('__funcs__ = formatter_functions().get_functions()\n'
|
||||
'def evaluate(self, formatter, kwargs, book, locals):\n'
|
||||
)
|
||||
else:
|
||||
lprog = self.lex_scanner.scan(prog)
|
||||
compile_text = None
|
||||
parser = _CompileParser(val, lprog, self, compile_text)
|
||||
val = parser.program()
|
||||
if parser.compile_text:
|
||||
global compile_counter
|
||||
compile_counter += 1
|
||||
f = compile_user_function("__A" + str(compile_counter), 'doc', -1, parser.compile_text)
|
||||
self.template_cache[column_name] = f
|
||||
else:
|
||||
if column_name is not None and self.template_cache is not None:
|
||||
lprog = self.template_cache.get(column_name, None)
|
||||
if not lprog:
|
||||
@ -415,9 +260,8 @@ class TemplateFormatter(string.Formatter):
|
||||
self.template_cache[column_name] = lprog
|
||||
else:
|
||||
lprog = self.lex_scanner.scan(prog)
|
||||
parser = _Parser(val, lprog, self)
|
||||
val = parser.program()
|
||||
return val
|
||||
parser = _Parser(val, lprog, self.funcs, self)
|
||||
return parser.program()
|
||||
|
||||
# ################# Override parent classes methods #####################
|
||||
|
||||
@ -462,10 +306,9 @@ class TemplateFormatter(string.Formatter):
|
||||
dispfmt = fmt[0:colon]
|
||||
colon += 1
|
||||
|
||||
funcs = formatter_functions().get_functions()
|
||||
fname = fmt[colon:p].strip()
|
||||
if fname in funcs:
|
||||
func = funcs[fname]
|
||||
if fname in self.funcs:
|
||||
func = self.funcs[fname]
|
||||
if func.arg_count == 2:
|
||||
# only one arg expected. Don't bother to scan. Avoids need
|
||||
# for escaping characters
|
||||
@ -516,12 +359,15 @@ class TemplateFormatter(string.Formatter):
|
||||
|
||||
def safe_format(self, fmt, kwargs, error_value, book,
|
||||
column_name=None, template_cache=None,
|
||||
strip_results=True):
|
||||
strip_results=True, user_functions=None):
|
||||
self.strip_results = strip_results
|
||||
self.column_name = column_name
|
||||
self.template_cache = template_cache
|
||||
self.kwargs = kwargs
|
||||
self.book = book
|
||||
if user_functions:
|
||||
self.funcs = formatter_functions().get_builtins().copy()
|
||||
self.funcs.update(user_functions)
|
||||
self.composite_values = {}
|
||||
self.locals = {}
|
||||
try:
|
||||
|
||||
@ -1622,11 +1622,8 @@ class UserFunction(FormatterUserFunction):
|
||||
cls = locals_['UserFunction'](name, doc, arg_count, eval_func)
|
||||
return cls
|
||||
|
||||
|
||||
def load_user_template_functions(library_uuid, funcs):
|
||||
unload_user_template_functions(library_uuid)
|
||||
|
||||
compiled_funcs = []
|
||||
def compile_user_template_functions(funcs):
|
||||
compiled_funcs = {}
|
||||
for func in funcs:
|
||||
try:
|
||||
# Force a name conflict to test the logic
|
||||
@ -1637,11 +1634,19 @@ def load_user_template_functions(library_uuid, funcs):
|
||||
# source. This helps ensure that if the function already is defined
|
||||
# then white space differences don't cause them to compare differently
|
||||
|
||||
compiled_funcs.append(compile_user_function(*func))
|
||||
cls = compile_user_function(*func)
|
||||
compiled_funcs[cls.name] = cls
|
||||
except:
|
||||
traceback.print_exc()
|
||||
formatter_functions().register_functions(library_uuid, compiled_funcs)
|
||||
return compiled_funcs
|
||||
|
||||
def load_user_template_functions(library_uuid, funcs, precompiled_user_functions=None):
|
||||
unload_user_template_functions(library_uuid)
|
||||
if precompiled_user_functions:
|
||||
compiled_funcs = precompiled_user_functions
|
||||
else:
|
||||
compiled_funcs = compile_user_template_functions(funcs)
|
||||
formatter_functions().register_functions(library_uuid, compiled_funcs.values())
|
||||
|
||||
def unload_user_template_functions(library_uuid):
|
||||
formatter_functions().unregister_functions(library_uuid)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user