Merge branch 'master' of https://github.com/cbhaley/calibre

manual/gui.rst

@@ -739,8 +739,10 @@ To choose icons for values in categories, right-click on a value then choose `Ma
 
   * ``category``: the lookup name of the category, for example ``authors``, ``series``, ``#mycolumn``.
   * ``value``: the value of the item within the category.
+  * ``count``: the number of books with this value. If the value is part of a hierarchy then the count includes the children.
+  * ``avg_rating``: the average rating for books with this value. If the value is part of a hierarchy then the average includes the children.
 
-  Book metadata such as title is not available.
+  Book metadata such as title is not available. Template database functions such as book_count() and book_values() will work, but the performance might not be acceptable. Python templates have full access to the calibre database API.
 
   For example, this template specifies that any value in the clicked-on category beginning with `History` will have an icon named ``flower.png``::
 

src/calibre/gui2/dialogs/template_dialog.py

@@ -1083,6 +1083,8 @@ def evaluate(book, context):
         tv = self.template_value
         l = self.template_value.selectionModel().selectedRows()
         break_on_mi = 0 if len(l) == 0 else l[0].row()
+        from calibre.gui2.ui import get_gui
+        db = get_gui().current_db
         for r,mi in enumerate(self.mi):
             w = tv.cellWidget(r, 0)
             w.setText(mi.get('title', _('No title provided')))
@@ -1096,7 +1098,8 @@ def evaluate(book, context):
                                  mi, global_vars=self.global_vars,
                                  template_functions=self.all_functions,
                                  break_reporter=self.break_reporter if r == break_on_mi else None,
-                                 python_context_object=self.python_context_object)
+                                 python_context_object=self.python_context_object,
+                                 database=db)
                 w = tv.cellWidget(r, 2)
                 w.setText(v.translate(translate_table))
                 w.setCursorPosition(0)

src/calibre/gui2/tag_browser/model.py

@@ -55,6 +55,7 @@ class TagTreeItem:  # {{{
     icon_config_dir = {}
     file_icon_provider = None
     eval_formatter = EvalFormatter()
+    database = None
 
     def __init__(self, data=None, is_category=False, icon_map=None,
                  parent=None, tooltip=None, category_key=None, temporary=False,
@@ -143,9 +144,12 @@ class TagTreeItem:  # {{{
                                 if node.type != self.TAG or node.type == self.ROOT:
                                     break
                             if val_icon is None and TEMPLATE_ICON_INDICATOR in self.value_icons[category]:
+                                v = {'category': category, 'value': self.tag.original_name,
+                                     'count': getattr(self.tag, 'count', ''),
+                                     'avg_rating': getattr(self.tag, 'avg_rating', '')}
                                 t = self.eval_formatter.safe_format(self.value_icons[category][TEMPLATE_ICON_INDICATOR][0],
-                                                                    {'category': category, 'value': self.tag.original_name},
+                                                                    v, 'VALUE_ICON_TEMPLATE_ERROR', {},
-                                                                    'VALUE_ICON_TEMPLATE_ERROR', {})
+                                                                    database=self.database)
                                 if t:
                                     val_icon = (os.path.join('template_icons', t), False)
                                 else:
@@ -406,8 +410,8 @@ class TagsModel(QAbstractItemModel):  # {{{
         self.filter_categories_by = None
         self.collapse_model = 'disable'
         self.row_map = []
-        self.root_item = self.create_node(icon_map=self.icon_state_map)
         self.db = None
+        self.root_item = self.create_node(icon_map=self.icon_state_map)
         self._build_in_progress = False
         self.reread_collapse_model({}, rebuild=False)
         self.show_error_after_event_loop_tick_signal.connect(self.on_show_error_after_event_loop_tick, type=Qt.ConnectionType.QueuedConnection)
@@ -1483,6 +1487,7 @@ class TagsModel(QAbstractItemModel):  # {{{
         node.value_icons = self.value_icons
         node.value_icon_cache = self.value_icon_cache
         node.icon_config_dir = self.icon_config_dir
+        node.database = self.db
         return node
 
     def get_node(self, idx):

src/calibre/gui2/tag_browser/view.py

@@ -686,14 +686,21 @@ class TagsView(QTreeView):  # {{{
             if action == 'set_icon':
                 if category is None:
                     if index is not None:
-                        current_item = self._model.get_node(index).tag.original_name
+                        tag = self._model.get_node(index).tag
+                        current_item = tag.original_name
+                        count = tag.count
+                        avg_rating = tag.avg_rating
                     else:
                         current_item = _('No value available')
+                        count = ''
+                        avg_rating = ''
                     template = self._model.value_icons.get(key, {}).get(TEMPLATE_ICON_INDICATOR, ('', False))[0]
                     from calibre.gui2.dialogs.template_dialog import TemplateDialog
                     from calibre.utils.formatter import EvalFormatter
+                    v = {'title': key, 'category': key, 'value': current_item,
+                         'count': count, 'avg_rating': avg_rating}
                     d = TemplateDialog(parent=self, text=template,
-                                       mi={'title': key, 'category': key, 'value': current_item},
+                                       mi=v,
                                        doing_emblem=True,
                                        # fm=None, color_field=None, icon_field_key=None,
                                        # icon_rule_kind=None,  text_is_placeholder=False,

src/calibre/srv/metadata.py

@@ -166,7 +166,8 @@ def get_gpref(name: str, defval = None):
     return gprefs.get(name, defval)
 
 
-def get_icon_for_node(node, parent, node_to_tag_map, tag_map, eval_formatter):
+def get_icon_for_node(node, parent, node_to_tag_map, tag_map, eval_formatter, db):
+    # This needs a legacy database so legacy formatter functions work
     category = node['category']
     if category in ('search', 'formats') or category.startswith('@'):
         return
@@ -190,10 +191,13 @@ def get_icon_for_node(node, parent, node_to_tag_map, tag_map, eval_formatter):
             if val_icon is not None and for_children:
                 break
             par = pd
+            val_icon = None
     if val_icon is None and TEMPLATE_ICON_INDICATOR in value_icons.get(category, {}):
+        v = {'category': category, 'value': name_for_icon(node),
+            'count': node.get('count', ''), 'avg_rating': node.get('avg_rating', '')}
         t = eval_formatter.safe_format(
-            value_icons[category][TEMPLATE_ICON_INDICATOR][0], {'category': category, 'value': name_for_icon(node)},
+            value_icons[category][TEMPLATE_ICON_INDICATOR][0], v,
-            'VALUE_ICON_TEMPLATE_ERROR', {})
+            'VALUE_ICON_TEMPLATE_ERROR', {}, database=db)
         if t:
             # Use POSIX path separator
             val_icon = 'template_icons/' + t
@@ -428,7 +432,7 @@ def collapse_first_letter(collapse_nodes, items, category_node, cl_list, idx, is
 def process_category_node(
         category_node, items, category_data, eval_formatter, field_metadata,
         opts, tag_map, hierarchical_tags, node_to_tag_map, collapse_nodes,
-        intermediate_nodes, hierarchical_items):
+        intermediate_nodes, hierarchical_items, db):
     category = items[category_node['id']]['category']
     if category not in category_data:
         # This can happen for user categories that are hierarchical and missing their parent.
@@ -469,7 +473,7 @@ def process_category_node(
         node = {'id':node_id, 'children':[]}
         parent['children'].append(node)
         try:
-            get_icon_for_node(node_data, parent, node_to_tag_map, tag_map, eval_formatter)
+            get_icon_for_node(node_data, parent, node_to_tag_map, tag_map, eval_formatter, db)
         except Exception:
             import traceback
             traceback.print_exc()
@@ -555,7 +559,7 @@ def iternode_descendants(node):
         yield from iternode_descendants(child)
 
 
-def fillout_tree(root, items, node_id_map, category_nodes, category_data, field_metadata, opts, book_rating_map):
+def fillout_tree(root, items, node_id_map, category_nodes, category_data, field_metadata, opts, book_rating_map, db):
     eval_formatter = EvalFormatter()
     tag_map, hierarchical_tags, node_to_tag_map = {}, set(), {}
     first, later, collapse_nodes, intermediate_nodes, hierarchical_items = [], [], [], {}, set()
@@ -572,7 +576,7 @@ def fillout_tree(root, items, node_id_map, category_nodes, category_data, field_
             process_category_node(
                 cnode, items, category_data, eval_formatter, field_metadata,
                 opts, tag_map, hierarchical_tags, node_to_tag_map,
-                collapse_nodes, intermediate_nodes, hierarchical_items)
+                collapse_nodes, intermediate_nodes, hierarchical_items, db)
 
     # Do not store id_set in the tag items as it is a lot of data, with not
     # much use. Instead only update the ratings and counts based on id_set
@@ -600,7 +604,7 @@ def render_categories(opts, db, category_data):
     items = {}
     with db.safe_read_lock:
         root, node_id_map, category_nodes, recount_nodes = create_toplevel_tree(category_data, items, db.field_metadata, opts, db)
-        fillout_tree(root, items, node_id_map, category_nodes, category_data, db.field_metadata, opts, db.fields['rating'].book_value_map)
+        fillout_tree(root, items, node_id_map, category_nodes, category_data, db.field_metadata, opts, db.fields['rating'].book_value_map, db)
     for node in recount_nodes:
         item = items[node['id']]
         item['count'] = sum(1 for x in iternode_descendants(node) if not items[x['id']].get('is_category', False))

src/calibre/utils/formatter.py

@@ -1651,6 +1651,7 @@ class TemplateFormatter(string.Formatter):
         self.recursion_level = -1
         self._caller = None
         self.python_context_object = None
+        self.database = None
 
     def _do_format(self, val, fmt):
         if not fmt or not val:
@@ -1759,7 +1760,8 @@ class TemplateFormatter(string.Formatter):
     def _run_python_template(self, compiled_template, arguments):
         try:
             self.python_context_object.set_values(
-                         db=get_database(self.book, get_database(self.book, None)),
+                         db=(self.database if self.database is not None
+                             else get_database(self.book, get_database(self.book, None))),
                          globals=self.global_vars,
                          arguments=arguments,
                          formatter=self,
@@ -1914,7 +1916,8 @@ class TemplateFormatter(string.Formatter):
              self.funcs,
              self.locals,
              self._caller,
-             self.python_context_object))
+             self.python_context_object,
+             self.database))
 
     def restore_state(self, state):
         self.recursion_level -= 1
@@ -1929,7 +1932,8 @@ class TemplateFormatter(string.Formatter):
          self.funcs,
          self.locals,
          self._caller,
-         self.python_context_object) = state
+         self.python_context_object,
+         self.database) = state
 
     # Allocate an interpreter if the formatter encounters a GPM or TPM template.
     # We need to allocate additional interpreters if there is composite recursion
@@ -1980,12 +1984,13 @@ class TemplateFormatter(string.Formatter):
                     column_name=None, template_cache=None,
                     strip_results=True, template_functions=None,
                     global_vars=None, break_reporter=None,
-                    python_context_object=None):
+                    python_context_object=None, database=None):
         state = self.save_state()
         if self.recursion_level == 0:
-            # Initialize the composite values dict if this is the base-level
+            # Initialize the composite values dict and database if this is the
-            # call. Recursive calls will use the same dict.
+            # base-level call. Recursive calls will use the same dict.
             self.composite_values = {}
+            self.database = database
         try:
             self._caller = FormatterFuncsCaller(self)
             self.strip_results = strip_results

src/calibre/utils/formatter_functions.py

@@ -214,7 +214,7 @@ def get_database(mi, name):
     wr = getattr(cache, 'library_database_instance', None)
     if wr is None:
         if name is not None:
-            only_in_gui_error()
+            only_in_gui_error(name)
         return None
     db = wr()
     if db is None:
@@ -248,7 +248,12 @@ class FormatterFunction:
     def only_in_gui_error(self):
         only_in_gui_error(self.name)
 
-    def get_database(self, mi):
+    def get_database(self, mi, formatter=None):
+        if formatter is not None:
+            if hasattr(formatter, 'database'):
+                db = formatter.database
+                if db is not None:
+                    return db
         return get_database(mi, self.name)
 
 
@@ -1680,7 +1685,7 @@ attached to the current book.[/] This function works only in the GUI.
 ''')
 
     def evaluate(self, formatter, kwargs, mi, locals):
-        c = self.get_database(mi).new_api.annotation_count_for_book(mi.id)
+        c = self.get_database(mi, formatter=formatter).new_api.annotation_count_for_book(mi.id)
         return '' if c == 0 else str(c)
 
 
@@ -1697,7 +1702,7 @@ not marked. This function works only in the GUI.
 ''')
 
     def evaluate(self, formatter, kwargs, mi, locals):
-        c = self.get_database(mi).data.get_marked(mi.id)
+        c = self.get_database(mi, formatter=formatter).data.get_marked(mi.id)
         return c if c else ''
 
 
@@ -2342,11 +2347,12 @@ r'''
 contain this book.[/] This function works only in the GUI. If you want to use these
 values in save-to-disk or send-to-device templates then you must make a custom
 "Column built from other columns", use the function in that column's template,
-and use that column's value in your save/send templates.
+and use that column's value in your save/send templates. This function works
+only in the GUI.
 ''')
 
     def evaluate(self, formatter, kwargs, mi, locals_):
-        db = self.get_database(mi)
+        db = self.get_database(mi, formatter=formatter)
         try:
             a = db.data.get_virtual_libraries_for_books((mi.id,))
             return ', '.join(a[mi.id])
@@ -2370,7 +2376,7 @@ This function works only in the GUI.
 ''')
 
     def evaluate(self, formatter, kwargs, mi, locals):
-        return self.get_database(mi).data.get_base_restriction_name()
+        return self.get_database(mi, formatter=formatter).data.get_base_restriction_name()
 
 
 class BuiltinUserCategories(BuiltinFormatterFunction):
@@ -2438,10 +2444,11 @@ program:
 ans
 [/CODE]
 [/LIST]
+This function works only in the GUI.
 ''')
 
     def evaluate(self, formatter, kwargs, mi, locals, field_name, field_value):
-        db = self.get_database(mi).new_api
+        db = self.get_database(mi, formatter=formatter).new_api
         try:
             link = None
             item_id = db.get_item_id(field_name, field_value, case_sensitive=True)
@@ -2602,7 +2609,7 @@ More than one of ``is_undefined``, ``is_false``, or ``is_true`` can be set to 1.
 
     def evaluate(self, formatter, kwargs, mi, locals, field, is_undefined, is_false, is_true):
         # 'field' is a lookup name, not a value
-        if field not in self.get_database(mi).field_metadata:
+        if field not in self.get_database(mi, formatter=formatter).field_metadata:
             raise ValueError(_("The column {} doesn't exist").format(field))
         res = getattr(mi, field, None)
         if res is None:
@@ -2858,6 +2865,7 @@ Using a stored template instead of putting the template into the search
 eliminates problems caused by the requirement to escape quotes in search
 expressions.
 [/LIST]
+This function can be used only in the GUI.
 ''')
 
     def evaluate(self, formatter, kwargs, mi, locals, query, use_vl):
@@ -2865,10 +2873,15 @@ expressions.
         if (not tweaks.get('allow_template_database_functions_in_composites', False) and
                 formatter.global_vars.get(rendering_composite_name, None)):
             raise ValueError(_('The book_count() function cannot be used in a composite column'))
-        db = self.get_database(mi)
+        db = self.get_database(mi, formatter=formatter)
         try:
-            ids = db.search_getting_ids(query, None, use_virtual_library=use_vl != '0')
+            if use_vl == '0':
-            return len(ids)
+                # use the new_api search that doesn't use virtual libraries to let
+                # the function work in content server icon rules.
+                ids = db.new_api.search(query, None)
+            else:
+                ids = db.search_getting_ids(query, None, use_virtual_library=True)
+            return str(len(ids))
         except Exception:
             traceback.print_exc()
 
@@ -2886,8 +2899,8 @@ then virtual libraries are ignored. This function and its companion
 ``book_count()`` are particularly useful in template searches, supporting
 searches that combine information from many books such as looking for series
 with only one book. It cannot be used in composite columns unless the tweak
-``allow_template_database_functions_in_composites`` is set to True. It can be
+``allow_template_database_functions_in_composites`` is set to True. This function
-used only in the GUI.
+can be used only in the GUI.
 ''')
 
     def evaluate(self, formatter, kwargs, mi, locals, column, query, sep, use_vl):
@@ -2895,11 +2908,14 @@ used only in the GUI.
         if (not tweaks.get('allow_template_database_functions_in_composites', False) and
                 formatter.global_vars.get(rendering_composite_name, None)):
             raise ValueError(_('The book_values() function cannot be used in a composite column'))
-        db = self.get_database(mi)
+        db = self.get_database(mi, formatter=formatter)
         if column not in db.field_metadata:
             raise ValueError(_("The column {} doesn't exist").format(column))
         try:
-            ids = db.search_getting_ids(query, None, use_virtual_library=use_vl != '0')
+            if use_vl == '0':
+                ids = db.new_api.search(query, None)
+            else:
+                ids = db.search_getting_ids(query, None, use_virtual_library=True)
             s = set()
             for id_ in ids:
                 f = db.new_api.get_proxy_metadata(id_).get(column, None)
@@ -2930,7 +2946,7 @@ This function can be used only in the GUI.
         if len(args) > 1:
             raise ValueError(_('Incorrect number of arguments for function {0}').format('has_extra_files'))
         pattern = args[0] if len(args) == 1 else None
-        db = self.get_database(mi).new_api
+        db = self.get_database(mi, formatter=formatter).new_api
         try:
             files = tuple(f.relpath.partition('/')[-1] for f in
                           db.list_extra_files(mi.id, use_cache=True, pattern=DATA_FILE_PATTERN))
@@ -2961,7 +2977,7 @@ the functions :ref:`has_extra_files`, :ref:`extra_file_modtime` and
         if len(args) > 1:
             raise ValueError(_('Incorrect number of arguments for function {0}').format('has_extra_files'))
         pattern = args[0] if len(args) == 1 else None
-        db = self.get_database(mi).new_api
+        db = self.get_database(mi, formatter=formatter).new_api
         try:
             files = tuple(f.relpath.partition('/')[-1] for f in
                           db.list_extra_files(mi.id, use_cache=True, pattern=DATA_FILE_PATTERN))
@@ -2987,7 +3003,7 @@ also the functions :ref:`has_extra_files`, :ref:`extra_file_names` and
 ''')
 
     def evaluate(self, formatter, kwargs, mi, locals, file_name):
-        db = self.get_database(mi).new_api
+        db = self.get_database(mi, formatter=formatter).new_api
         try:
             q = posixpath.join(DATA_DIR_NAME, file_name)
             for f in db.list_extra_files(mi.id, use_cache=True, pattern=DATA_FILE_PATTERN):
@@ -3016,7 +3032,7 @@ This function can be used only in the GUI.
 ''')
 
     def evaluate(self, formatter, kwargs, mi, locals, file_name, format_string):
-        db = self.get_database(mi).new_api
+        db = self.get_database(mi, formatter=formatter).new_api
         try:
             q = posixpath.join(DATA_DIR_NAME, file_name)
             for f in db.list_extra_files(mi.id, use_cache=True, pattern=DATA_FILE_PATTERN):
@@ -3057,7 +3073,7 @@ program:
 ''')
 
     def evaluate(self, formatter, kwargs, mi, locals, field_name, field_value, plain_text):
-        db = self.get_database(mi).new_api
+        db = self.get_database(mi, formatter=formatter).new_api
         try:
             note = None
             item_id = db.get_item_id(field_name, field_value, case_sensitive=True)
@@ -3125,7 +3141,7 @@ values in ``field_name``. Example:
 ''')
 
     def evaluate(self, formatter, kwargs, mi, locals, field_name, field_value):
-        db = self.get_database(mi).new_api
+        db = self.get_database(mi, formatter=formatter).new_api
         if field_value:
             note = None
             try: