From 6acfebf8c16750e981587639ab3b71bec513aa9e Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Sun, 3 Jul 2022 19:03:55 +0100 Subject: [PATCH] The template database functions we discussed. --- manual/template_lang.rst | 27 +++++++++ resources/default_tweaks.py | 7 +++ src/calibre/db/fields.py | 5 +- src/calibre/db/search.py | 4 +- src/calibre/utils/formatter_functions.py | 70 ++++++++++++++++++++---- 5 files changed, 100 insertions(+), 13 deletions(-) diff --git a/manual/template_lang.rst b/manual/template_lang.rst index 3211aad499..018794f2b4 100644 --- a/manual/template_lang.rst +++ b/manual/template_lang.rst @@ -413,6 +413,33 @@ In `GPM` the functions described in `Single Function Mode` all require an additi An author is separated from its link value by the ``val_separator`` string with no added spaces. ``author:linkvalue`` pairs are separated by the ``pair_separator`` string argument with no added spaces. It is up to you to choose separator strings that do not occur in author names or links. An author is included even if the author link is empty. * ``author_sorts(val_separator)`` -- returns a string containing a list of author's sort values for the authors of the book. The sort is the one in the author metadata information (different from the author_sort in books). The returned list has the form ``author sort 1`` ``val_separator`` ``author sort 2`` etc. with no added spaces. The author sort values in this list are in the same order as the authors of the book. If you want spaces around ``val_separator`` then include them in the ``val_separator`` string. +* ``book_count(query, use_vl)`` -- returns the count of books found by searching for ``query``. If ``use_vl`` is ``0`` (zero) then virtual libraries are ignored. This function and its companion `book_values()`` are particularly useful in template searches, supporting searches that combine information from many books such as looking for series with only one book. It cannot be used in composite columns unless the tweak ``allow_template_database_functions_in_composites`` is set to True. It can be used only in the GUI. + + For example this tempate search uses this function and its companion to find all series with only one book: + + 1) Define a stored template (using :guilabel:`Preferences->Advanced->Template functions`) named ``series_only_one_book`` (the name is arbitrary). The template is:: + + program: + vals = globals(vals=''); + if !vals then + all_series = book_values('series', 'series:true', ',', 0); + for series in all_series: + if book_count('series:="' & series & '"', 0) == 1 then + vals = list_join(',', vals, ',', series, ',') + fi + rof; + set_globals(vals) + fi; + str_in_list(vals, ',', $series, 1, '') + + The first time the template runs (the first book checked) it stores the results of the database lookups in a ``global`` template variable named ``vals``. These results are used to check subsequent books without redoing the lookups. + + 2) Use the stored template in a template search:: + + template:"program: series_only_one_book()#@#:n:1" + + Using a stored template instead of putting the template into the search eliminates problems caused by the requirement to escape quotes in search expressions. +* ``book_values(column, query, sep, use_vl)`` -- returns a list of the unique values contained in the column ``column`` (a lookup name), separated by ``sep``, in the books found by searching for ``query``. If ``use_vl`` is ``0`` (zero) then virtual libraries are ignored. This function and its companion `book_count()`` are particularly useful in template searches, supporting searches that combine information from many books such as looking for series with only one book. It cannot be used in composite columns unless the tweak ``allow_template_database_functions_in_composites`` is set to True. It can be used only in the GUI. * ``booksize()`` -- returns the value of the calibre 'size' field. Returns '' if there are no formats. * ``check_yes_no(field_name, is_undefined, is_false, is_true)`` -- checks if the value of the yes/no field named by the lookup name ``field_name`` is one of the values specified by the parameters, returning ``'yes'`` if a match is found otherwise returning the empty string. Set the parameter ``is_undefined``, ``is_false``, or ``is_true`` to 1 (the number) to check that condition, otherwise set it to 0. Example: diff --git a/resources/default_tweaks.py b/resources/default_tweaks.py index 9b921e4ffa..f18863b472 100644 --- a/resources/default_tweaks.py +++ b/resources/default_tweaks.py @@ -585,3 +585,10 @@ template_editor_tab_stop_width = 4 # value_for_undefined_numbers_when_sorting = 'minimum' # value_for_undefined_numbers_when_sorting = 'maximum' value_for_undefined_numbers_when_sorting = 0 + +#: Allow template database functions in composite columns +# If True then the template database functions book_values() and book_count() +# can be used in composite custom columns. Note: setting this tweak to True and +# using these functions in composites can be very slow. +# Default: False +allow_template_database_functions_in_composites = False diff --git a/src/calibre/db/fields.py b/src/calibre/db/fields.py index 557f9e740f..c4fce8cd2d 100644 --- a/src/calibre/db/fields.py +++ b/src/calibre/db/fields.py @@ -21,6 +21,8 @@ from calibre.utils.localization import calibre_langcode_to_name from polyglot.builtins import iteritems +rendering_composite_name = '__rendering_composite__' + def bool_sort_key(bools_are_tristate): return (lambda x:{True: 1, False: 2, None: 3}.get(x, 3)) if bools_are_tristate else lambda x:{True: 1, False: 2, None: 2}.get(x, 2) @@ -296,7 +298,8 @@ class CompositeField(OneToOneField): ans = formatter.safe_format( self.metadata['display']['composite_template'], mi, _('TEMPLATE ERROR'), mi, column_name=self._composite_name, template_cache=template_cache, - template_functions=self.get_template_functions()).strip() + template_functions=self.get_template_functions(), + global_vars={rendering_composite_name:'1'}).strip() with self._lock: self._render_cache[book_id] = ans return ans diff --git a/src/calibre/db/search.py b/src/calibre/db/search.py index cc81e8472e..03c86b36c9 100644 --- a/src/calibre/db/search.py +++ b/src/calibre/db/search.py @@ -654,11 +654,13 @@ class Parser(SearchQueryParser): # {{{ matches = set() error_string = '*@*TEMPLATE_ERROR*@*' template_cache = {} + global_vars = {} for book_id in candidates: mi = self.dbcache.get_proxy_metadata(book_id) val = mi.formatter.safe_format(template, {}, error_string, mi, column_name='search template', - template_cache=template_cache) + template_cache=template_cache, + global_vars=global_vars) if val.startswith(error_string): raise ParseException(val[len(error_string):]) if sep == 't': diff --git a/src/calibre/utils/formatter_functions.py b/src/calibre/utils/formatter_functions.py index 0615b4c642..e061398405 100644 --- a/src/calibre/utils/formatter_functions.py +++ b/src/calibre/utils/formatter_functions.py @@ -2111,7 +2111,7 @@ class BuiltinSwapAroundArticles(BuiltinFormatterFunction): class BuiltinArguments(BuiltinFormatterFunction): name = 'arguments' arg_count = -1 - category = 'other' + category = 'Other' __doc__ = doc = _('arguments(id[=expression] [, id[=expression]]*) ' '-- Used in a stored template to retrieve the arguments ' 'passed in the call. It both declares and initializes ' @@ -2131,7 +2131,7 @@ class BuiltinArguments(BuiltinFormatterFunction): class BuiltinGlobals(BuiltinFormatterFunction): name = 'globals' arg_count = -1 - category = 'other' + category = 'Other' __doc__ = doc = _('globals(id[=expression] [, id[=expression]]*) ' '-- Retrieves "global variables" that can be passed into ' 'the formatter. It both declares and initializes local ' @@ -2150,14 +2150,11 @@ class BuiltinSetGlobals(BuiltinFormatterFunction): name = 'set_globals' arg_count = -1 category = 'other' - __doc__ = doc = _('globals(id[=expression] [, id[=expression]]*) ' - '-- Retrieves "global variables" that can be passed into ' - 'the formatter. It both declares and initializes local ' - 'variables with the names of the global variables passed ' - 'in. If the corresponding variable is not provided in ' - 'the passed-in globals then it assigns that variable the ' - 'provided default value. If there is no default value ' - 'then the variable is set to the empty string.') + __doc__ = doc = _('set_globals(id[=expression] [, id[=expression]]*) ' + '-- Sets "global variables" that can be passed into ' + 'the formatter. The globals are given the name of the id ' + 'passed in. The value of the id is used unless an ' + 'expression is provided.') def evaluate(self, formatter, kwargs, mi, locals, *args): # The globals function is implemented in-line in the formatter @@ -2228,10 +2225,61 @@ class BuiltinUrlsFromIdentifiers(BuiltinFormatterFunction): return str(e) +class BuiltinBookCount(BuiltinFormatterFunction): + name = 'book_count' + arg_count = 2 + category = 'Template database functions' + __doc__ = doc = _('book_count(query, use_vl) -- returns the count of ' + 'books found by searching for query. If use_vl is ' + '0 (zero) then virtual libraries are ignored. This ' + 'function can be used only in the GUI.') + + def evaluate(self, formatter, kwargs, mi, locals, query, use_vl): + from calibre.db.fields import rendering_composite_name + if (not tweaks.get('allow_template_database_functions_in_composites', False) and + formatter.global_vars.get(rendering_composite_name, None)): + raise ValueError(_('The book_count() function cannot be used in a composite column')) + with suppress(Exception): + try: + from calibre.gui2.ui import get_gui + ids = get_gui().current_db.search_getting_ids(query, None, + use_virtual_library=use_vl != '0') + return len(ids) + except Exception as e: + traceback.print_exc() + return _('This function can be used only in the GUI') + + +class BuiltinBookValues(BuiltinFormatterFunction): + name = 'book_values' + arg_count = 4 + category = 'Template database functions' + __doc__ = doc = _('book_values(column, query, sep, use_vl) -- returns a list ' + 'of the values contained in the column "column", separated ' + 'by "sep", in the books found by searching for "query". ' + 'If use_vl is 0 (zero) then virtual libraries are ignored. ' + 'This function can be used only in the GUI.') + + def evaluate(self, formatter, kwargs, mi, locals, column, query, sep, use_vl): + from calibre.db.fields import rendering_composite_name + if (not tweaks.get('allow_template_database_functions_in_composites', False) and + formatter.global_vars.get(rendering_composite_name, None)): + raise ValueError(_('The book_values() function cannot be used in a composite column')) + with suppress(Exception): + from calibre.gui2.ui import get_gui + db = get_gui().current_db + ids = db.search_getting_ids(query, None, use_virtual_library=use_vl != '0') + ff = db.new_api.field_for + s = {ff(column, id_) for id_ in ids if ff(column, id_)} + return sep.join(s) + return _('This function can be used only in the GUI') + + _formatter_builtins = [ BuiltinAdd(), BuiltinAnd(), BuiltinApproximateFormats(), BuiltinArguments(), BuiltinAssign(), - BuiltinAuthorLinks(), BuiltinAuthorSorts(), BuiltinBooksize(), + BuiltinAuthorLinks(), BuiltinAuthorSorts(), BuiltinBookCount(), + BuiltinBookValues(), BuiltinBooksize(), BuiltinCapitalize(), BuiltinCharacter(), BuiltinCheckYesNo(), BuiltinCeiling(), BuiltinCmp(), BuiltinConnectedDeviceName(), BuiltinConnectedDeviceUUID(), BuiltinContains(), BuiltinCount(), BuiltinCurrentLibraryName(), BuiltinCurrentLibraryPath(),