Start implementation of hierarchical grouping in Tag Browser

src/calibre/ebooks/metadata/book/__init__.py

@@ -83,6 +83,10 @@ CALIBRE_METADATA_FIELDS = frozenset([
     'application_id',   # An application id, currently set to the db_id.
     'db_id',            # the calibre primary key of the item.
     'formats',          # list of formats (extensions) for this book
+    # a dict of user category names, where the value is a list of item names
+    # from the book that are in that category
+    'user_categories',
+
     ]
 )
 

src/calibre/ebooks/metadata/book/base.py

@@ -30,6 +30,7 @@ NULL_VALUES = {
                 'author_sort_map': {},
                 'authors'      : [_('Unknown')],
                 'title'        : _('Unknown'),
+                'user_categories' : {},
                 'language'     : 'und'
 }
 

src/calibre/ebooks/metadata/opf2.py

@@ -470,6 +470,13 @@ def serialize_user_metadata(metadata_elem, all_user_metadata, tail='\n'+(' '*8))
         metadata_elem.append(meta)
 
 
+def dump_user_categories(cats):
+    if not cats:
+        cats = {}
+    from calibre.ebooks.metadata.book.json_codec import object_to_unicode
+    return json.dumps(object_to_unicode(cats), ensure_ascii=False,
+            skipkeys=True)
+
 class OPF(object): # {{{
 
     MIMETYPE         = 'application/oebps-package+xml'
@@ -524,6 +531,9 @@ class OPF(object): # {{{
     publication_type = MetadataField('publication_type', is_dc=False)
     timestamp       = MetadataField('timestamp', is_dc=False,
                                     formatter=parse_date, renderer=isoformat)
+    user_categories = MetadataField('user_categories', is_dc=False,
+                                    formatter=json.loads,
+                                    renderer=dump_user_categories)
 
 
     def __init__(self, stream, basedir=os.getcwdu(), unquote_urls=True,
@@ -994,7 +1004,7 @@ class OPF(object): # {{{
         for attr in ('title', 'authors', 'author_sort', 'title_sort',
                      'publisher', 'series', 'series_index', 'rating',
                      'isbn', 'tags', 'category', 'comments',
-                     'pubdate'):
+                     'pubdate', 'user_categories'):
             val = getattr(mi, attr, None)
             if val is not None and val != [] and val != (None, None):
                 setattr(self, attr, val)
@@ -1175,6 +1185,10 @@ class OPFCreator(Metadata):
             a(CAL_ELEM('calibre:timestamp', self.timestamp.isoformat()))
         if self.publication_type is not None:
             a(CAL_ELEM('calibre:publication_type', self.publication_type))
+        if self.user_categories:
+            from calibre.ebooks.metadata.book.json_codec import object_to_unicode
+            a(CAL_ELEM('calibre:user_categories',
+                       json.dumps(object_to_unicode(self.user_categories))))
         manifest = E.manifest()
         if self.manifest is not None:
             for ref in self.manifest:
@@ -1299,6 +1313,8 @@ def metadata_to_opf(mi, as_string=True):
         meta('publication_type', mi.publication_type)
     if mi.title_sort:
         meta('title_sort', mi.title_sort)
+    if mi.user_categories:
+        meta('user_categories', dump_user_categories(mi.user_categories))
 
     serialize_user_metadata(metadata, mi.get_all_user_metadata(False))
 

src/calibre/gui2/dialogs/tag_categories.py

@@ -73,16 +73,17 @@ class TagCategories(QDialog, Ui_TagCategories):
             if idx == 0:
                 continue
             for n in category_values[idx]():
-                t = Item(name=n, label=label, index=len(self.all_items),icon=category_icons[idx], exists=True)
+                t = Item(name=n, label=label, index=len(self.all_items),
+                         icon=category_icons[idx], exists=True)
                 self.all_items.append(t)
-                self.all_items_dict[label+':'+n] = t
+                self.all_items_dict[icu_lower(label+':'+n)] = t
 
         self.categories = dict.copy(db.prefs.get('user_categories', {}))
         if self.categories is None:
             self.categories = {}
         for cat in self.categories:
             for item,l in enumerate(self.categories[cat]):
-                key = ':'.join([l[1], l[0]])
+                key = icu_lower(':'.join([l[1], l[0]]))
                 t = self.all_items_dict.get(key, None)
                 if l[1] in self.category_labels:
                     if t is None:

src/calibre/gui2/preferences/look_feel.py

@@ -7,17 +7,19 @@ __docformat__ = 'restructuredtext en'
 
 from PyQt4.Qt import QApplication, QFont, QFontInfo, QFontDialog
 
-from calibre.gui2.preferences import ConfigWidgetBase, test_widget
+from calibre.gui2.preferences import ConfigWidgetBase, test_widget, CommaSeparatedList
 from calibre.gui2.preferences.look_feel_ui import Ui_Form
 from calibre.gui2 import config, gprefs, qt_app
 from calibre.utils.localization import available_translations, \
     get_language, get_lang
 from calibre.utils.config import prefs
+from calibre.utils.icu import sort_key
 
 class ConfigWidget(ConfigWidgetBase, Ui_Form):
 
     def genesis(self, gui):
         self.gui = gui
+        db = gui.library_view.model().db
 
         r = self.register
 
@@ -61,6 +63,15 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         r('tags_browser_partition_method', gprefs, choices=choices)
         r('tags_browser_collapse_at', gprefs)
 
+        choices = set([k for k in db.field_metadata.all_field_keys()
+                    if db.field_metadata[k]['is_category'] and
+                       db.field_metadata[k]['datatype'] in ['text', 'series', 'enumeration']])
+        choices -= set(['authors', 'publisher', 'formats', 'news'])
+        self.opt_categories_using_hierarchy.update_items_cache(choices)
+        r('categories_using_hierarchy', db.prefs, setting=CommaSeparatedList,
+          choices=sorted(list(choices), key=sort_key))
+
+
         self.current_font = None
         self.change_font_button.clicked.connect(self.change_font)
 

src/calibre/gui2/preferences/look_feel.ui

@@ -7,7 +7,7 @@
     <x>0</x>
     <y>0</y>
     <width>670</width>
-    <height>392</height>
+    <height>422</height>
    </rect>
   </property>
   <property name="windowTitle">
@@ -136,7 +136,7 @@
      <item>
       <widget class="QLabel" name="label_6">
        <property name="text">
-        <string>Tags browser category partitioning method:</string>
+        <string>Tags browser category &amp;partitioning method:</string>
        </property>
        <property name="buddy">
         <cstring>opt_tags_browser_partition_method</cstring>
@@ -157,7 +157,7 @@ if you never want subcategories</string>
      <item>
       <widget class="QLabel" name="label_6">
        <property name="text">
-        <string>Collapse when more items than:</string>
+        <string>&amp;Collapse when more items than:</string>
        </property>
        <property name="buddy">
         <cstring>opt_tags_browser_collapse_at</cstring>
@@ -190,6 +190,28 @@ up into sub-categories. If the partition method is set to disable, this value is
      </item>
     </layout>
    </item>
+   <item row="8" column="0">
+    <widget class="QLabel" name="label_81">
+     <property name="text">
+      <string>Categories with &amp;hierarchical items:</string>
+     </property>
+     <property name="buddy">
+      <cstring>opt_categories_using_hierarchy</cstring>
+     </property>
+    </widget>
+   </item>
+   <item row="8" column="1">
+    <widget class="MultiCompleteLineEdit" name="opt_categories_using_hierarchy">
+     <property name="toolTip">
+      <string>A comma-separated list of columns in which items containing
+periods are displayed in the tag browser trees. For example, if
+this box contains 'tags' then tags of the form 'Mystery.English'
+and 'Mystery.Thriller' will be displayed with English and Thriller
+both under 'Mystery'. If 'tags' is not in this box,
+then the tags will be displayed each on their own line.</string>
+     </property>
+    </widget>
+   </item>
    <item row="15" column="0" colspan="2">
     <widget class="QGroupBox" name="groupBox_2">
      <property name="title">
@@ -275,6 +297,13 @@ up into sub-categories. If the partition method is set to disable, this value is
    </item>
   </layout>
  </widget>
+ <customwidgets>
+  <customwidget>
+   <class>MultiCompleteLineEdit</class>
+   <extends>QLineEdit</extends>
+   <header>calibre/gui2/complete.h</header>
+  </customwidget>
+ </customwidgets>
  <resources/>
  <connections/>
 </ui>

src/calibre/gui2/tag_view.py

@@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en'
 Browsing book collection by tags.
 '''
 
-import traceback
+import traceback, copy
 
 from itertools import izip
 from functools import partial
@@ -258,11 +258,15 @@ class TagsView(QTreeView): # {{{
 
             if item.type == TagTreeItem.TAG:
                 tag_item = item
-                tag_name = item.tag.name
+                t = item.tag
-                tag_id = item.tag.id
+                tag_name = t.name
+                tag_id = t.id
+                can_edit = getattr(t, 'can_edit', True)
+                while item.type != TagTreeItem.CATEGORY:
                     item = item.parent
 
             if item.type == TagTreeItem.CATEGORY:
+                if not item.category_key.startswith('@'):
                     while item.parent != self._model.root_item:
                         item = item.parent
                 category = unicode(item.name.toString())
@@ -275,7 +279,8 @@ class TagsView(QTreeView): # {{{
                 if tag_name:
                     # If the user right-clicked on an editable item, then offer
                     # the possibility of renaming that item.
-                    if key in ['authors', 'tags', 'series', 'publisher', 'search'] or \
+                    if can_edit and \
+                            key in ['authors', 'tags', 'series', 'publisher', 'search'] or \
                             (self.db.field_metadata[key]['is_custom'] and \
                              self.db.field_metadata[key]['datatype'] != 'rating'):
                         # Add the 'rename' items
@@ -514,6 +519,12 @@ class TagTreeItem(object): # {{{
                 tweaks['categories_use_field_for_author_name'] == 'author_sort':
             name = tag.sort
             tt_author = True
+        else:
+            p = self
+            while p.parent.type != self.ROOT:
+                p = p.parent
+            if p.category_key.startswith('@'):
+                name = getattr(tag, 'original_name', tag.name)
             else:
                 name = tag.name
             tt_author = False
@@ -523,7 +534,7 @@ class TagTreeItem(object): # {{{
             else:
                 return QVariant('[%d] %s'%(tag.count, name))
         if role == Qt.EditRole:
-            return QVariant(tag.name)
+            return QVariant(getattr(tag, 'original_name', tag.name))
         if role == Qt.DecorationRole:
             return self.icon_state_map[tag.state]
         if role == Qt.ToolTipRole:
@@ -550,12 +561,12 @@ class TagTreeItem(object): # {{{
 
     def child_tags(self):
         res = []
-        for t in self.children:
+        def recurse(nodes, res):
-            if t.type == TagTreeItem.CATEGORY:
+            for t in nodes:
-                for c in t.children:
+                if t.type != TagTreeItem.CATEGORY:
-                    res.append(c)
-            else:
                     res.append(t)
+                recurse(t.children, res)
+        recurse(self.children, res)
         return res
     # }}}
 
@@ -590,6 +601,10 @@ class TagsModel(QAbstractItemModel): # {{{
         data = self.get_node_tree(config['sort_tags_by'])
         gst = db.prefs.get('grouped_search_terms', {})
         self.root_item = TagTreeItem()
+        self.category_nodes = []
+
+        last_category_node = None
+        category_node_map = {}
         for i, r in enumerate(self.row_map):
             if self.hidden_categories and self.categories[i] in self.hidden_categories:
                 continue
@@ -599,10 +614,32 @@ class TagsModel(QAbstractItemModel): # {{{
                 tt = ''
             else:
                 tt = _(u'The lookup/search name is "{0}"').format(r)
-            TagTreeItem(parent=self.root_item,
+
+            if r.startswith('@') and r.find('.') >= 0:
+                path_parts = [p.strip() for p in r.split('.') if p.strip()]
+                path = ''
+                for i,p in enumerate(path_parts):
+                    path += p
+                    if path not in category_node_map:
+                        node = TagTreeItem(parent=last_category_node,
+                                           data=p[1:] if i == 0 else p,
+                                           category_icon=self.category_icon_map[r],
+                                           tooltip=tt if path == r else path,
+                                           category_key=path)
+                        last_category_node = node
+                        category_node_map[path] = node
+                        self.category_nodes.append(node)
+                    else:
+                        last_category_node = category_node_map[path]
+                    path += '.'
+            else:
+                node = TagTreeItem(parent=self.root_item,
                                    data=self.categories[i],
                                    category_icon=self.category_icon_map[r],
                                    tooltip=tt, category_key=r)
+                category_node_map[r] = node
+                last_category_node = node
+                self.category_nodes.append(node)
         self.refresh(data=data)
 
     def break_cycles(self):
@@ -754,10 +791,15 @@ class TagsModel(QAbstractItemModel): # {{{
         for user_cat in sorted(self.db.prefs.get('user_categories', {}).keys(),
                                key=sort_key):
             cat_name = '@' + user_cat # add the '@' to avoid name collision
+            while True:
                 try:
                     tb_cats.add_user_category(label=cat_name, name=user_cat)
+                    dot = cat_name.rfind('.')
+                    if dot < 0:
+                        break
+                    cat_name = cat_name[:dot]
                 except ValueError:
-                traceback.print_exc()
+                    break
 
         for cat in sorted(self.db.prefs.get('grouped_search_terms', {}).keys(),
                           key=sort_key):
@@ -794,7 +836,7 @@ class TagsModel(QAbstractItemModel): # {{{
             data = self.get_node_tree(sort_by) # get category data
         if data is None:
             return False
-        row_index = -1
+
         collapse = gprefs['tags_browser_collapse_at']
         collapse_model = self.collapse_model
         if collapse == 0:
@@ -810,53 +852,42 @@ class TagsModel(QAbstractItemModel): # {{{
                 collapse_template = tweaks['categories_collapsed_popularity_template']
         collapse_letter = collapse_letter_sk = None
 
-        for i, r in enumerate(self.row_map):
+        def process_one_node(category, state_map, collapse_letter, collapse_letter_sk):
-            if self.hidden_categories and self.categories[i] in self.hidden_categories:
+            category_index = self.createIndex(category.row(), 0, category)
-                continue
-            row_index += 1
-            category = self.root_item.children[row_index]
-            names = []
-            states = []
-            children = category.child_tags()
-            states = [t.tag.state for t in children]
-            names = [t.tag.name for names in children]
-            state_map = dict(izip(names, states))
-            category_index = self.index(row_index, 0, QModelIndex())
             category_node = category_index.internalPointer()
-            if len(category.children) > 0:
+            key = category_node.category_key
-                self.beginRemoveRows(category_index, 0,
+            if key not in data:
-                        len(category.children)-1)
+                return ((collapse_letter, collapse_letter_sk))
-                category.children = []
+            cat_len = len(data[key])
-                self.endRemoveRows()
-            cat_len = len(data[r])
             if cat_len <= 0:
-                continue
+                return ((collapse_letter, collapse_letter_sk))
 
-            self.beginInsertRows(category_index, 0, len(data[r])-1)
+            clear_rating = True if key not in self.categories_with_ratings and \
-            clear_rating = True if r not in self.categories_with_ratings and \
+                                not self.db.field_metadata[key]['is_custom'] and \
-                                not self.db.field_metadata[r]['is_custom'] and \
+                                not self.db.field_metadata[key]['kind'] == 'user' \
-                                not self.db.field_metadata[r]['kind'] == 'user' \
                             else False
-            tt = r if self.db.field_metadata[r]['kind'] == 'user' else None
+            tt = key if self.db.field_metadata[key]['kind'] == 'user' else None
-            for idx,tag in enumerate(data[r]):
+            for idx,tag in enumerate(data[key]):
                 if clear_rating:
                     tag.avg_rating = None
-                tag.state = state_map.get(tag.name, 0)
+                tag.state = state_map.get((tag.name, tag.category), 0)
 
                 if collapse_model != 'disable' and cat_len > collapse:
                     if collapse_model == 'partition':
                         if (idx % collapse) == 0:
                             d = {'first': tag}
                             if cat_len > idx + collapse:
-                                d['last'] = data[r][idx+collapse-1]
+                                d['last'] = data[key][idx+collapse-1]
                             else:
-                                d['last'] = data[r][cat_len-1]
+                                d['last'] = data[key][cat_len-1]
                             name = eval_formatter.safe_format(collapse_template,
                                                               d, 'TAG_VIEW', None)
+                            self.beginInsertRows(category_index, 999999, 1) #len(data[key])-1)
                             sub_cat = TagTreeItem(parent=category,
                                      data = name, tooltip = None,
                                      category_icon = category_node.icon,
                                      category_key=category_node.category_key)
+                            self.endInsertRows()
                     else:
                         ts = tag.sort
                         if not ts:
@@ -877,12 +908,60 @@ class TagsModel(QAbstractItemModel): # {{{
                                      category_icon = category_node.icon,
                                      tooltip = None,
                                      category_key=category_node.category_key)
-                    t = TagTreeItem(parent=sub_cat, data=tag, tooltip=tt,
+                    node_parent = sub_cat
-                                        icon_map=self.icon_state_map)
                 else:
-                    t = TagTreeItem(parent=category, data=tag, tooltip=tt,
+                    node_parent = category
+
+                components = [t for t in tag.name.split('.')]
+                if key not in self.db.prefs.get('categories_using_hierarchy', []) \
+                        or len(components) == 1 or \
+                        self.db.field_metadata[key]['kind'] == 'user':
+                    self.beginInsertRows(category_index, 999999, 1)
+                    TagTreeItem(parent=node_parent, data=tag, tooltip=tt,
                                     icon_map=self.icon_state_map)
                     self.endInsertRows()
+                else:
+                    for i,comp in enumerate(components):
+                        child_map = dict([(t.tag.name, t) for t in node_parent.children
+                                            if t.type != TagTreeItem.CATEGORY])
+                        if comp in child_map:
+                            node_parent = child_map[comp]
+                            node_parent.tag.count += tag.count
+                        else:
+                            if i < len(components)-1:
+                                t = copy.copy(tag)
+                                t.original_name = '.'.join(components[:i+1])
+                                t.can_edit = False
+                            else:
+                                t = tag
+                                t.original_name = t.name
+                                t.can_edit = True
+                            t.use_prefix = True
+                            t.name = comp
+                            self.beginInsertRows(category_index, 999999, 1)
+                            node_parent = TagTreeItem(parent=node_parent, data=t,
+                                            tooltip=tt, icon_map=self.icon_state_map)
+                            self.endInsertRows()
+
+            return ((collapse_letter, collapse_letter_sk))
+
+        for category in self.category_nodes:
+            if len(category.children) > 0:
+                child_map = category.children
+                states = [c.tag.state for c in category.child_tags()]
+                names = [(c.tag.name, c.tag.category) for c in category.child_tags()]
+                state_map = dict(izip(names, states))
+                ctags = [c for c in child_map if c.type == TagTreeItem.CATEGORY]
+                start = len(ctags)
+                self.beginRemoveRows(self.createIndex(category.row(), 0, category),
+                                     start, len(child_map)-1)
+                category.children = ctags
+                self.endRemoveRows()
+            else:
+                state_map = {}
+
+            collapse_letter, collapse_letter_sk = process_one_node(category,
+                                state_map, collapse_letter, collapse_letter_sk)
         return True
 
     def columnCount(self, parent):
@@ -907,7 +986,10 @@ class TagsModel(QAbstractItemModel): # {{{
                         _('An item cannot be set to nothing. Delete it instead.')).exec_()
             return False
         item = index.internalPointer()
-        key = item.parent.category_key
+        itm = item.parent
+        while itm.type != TagTreeItem.CATEGORY:
+            itm = itm.parent
+        key = itm.category_key
         # make certain we know about the item's category
         if key not in self.db.field_metadata:
             return False
@@ -1022,27 +1104,31 @@ class TagsModel(QAbstractItemModel): # {{{
 
     def reset_all_states(self, except_=None):
         update_list = []
-        def process_tag(tag_index, tag_item):
+        def process_tag(tag_item):
+            if tag_item.type != TagTreeItem.CATEGORY:
                 tag = tag_item.tag
                 if tag is except_:
+                    tag_index = self.createIndex(tag_item.row(), 0, tag_item)
                     self.dataChanged.emit(tag_index, tag_index)
-                return
+                elif tag.state != 0 or tag in update_list:
-            if tag.state != 0 or tag in update_list:
+                    tag_index = self.createIndex(tag_item.row(), 0, tag_item)
                     tag.state = 0
                     update_list.append(tag)
                     self.dataChanged.emit(tag_index, tag_index)
+            for t in tag_item.children:
+                process_tag(t)
 
-        def process_level(category_index):
+#        def process_level(category_index):
-            for j in xrange(self.rowCount(category_index)):
+#            for j in xrange(self.rowCount(category_index)):
-                tag_index = self.index(j, 0, category_index)
+#                tag_index = self.index(j, 0, category_index)
-                tag_item = tag_index.internalPointer()
+#                tag_item = tag_index.internalPointer()
-                if tag_item.type == TagTreeItem.CATEGORY:
+#                if tag_item.type == TagTreeItem.CATEGORY:
-                    process_level(tag_index)
+#                    process_level(tag_index)
-                else:
+#                else:
-                    process_tag(tag_index, tag_item)
+#                    process_tag(tag_index, tag_item)
 
-        for i in xrange(self.rowCount(QModelIndex())):
+        for t in self.root_item.children:
-            process_level(self.index(i, 0, QModelIndex()))
+            process_tag(t)
 
     def clear_state(self):
         self.reset_all_states()
@@ -1073,14 +1159,10 @@ class TagsModel(QAbstractItemModel): # {{{
         # They will be 'checked' in both places, but we want to put the node
         # into the search string only once. The nodes_seen set helps us do that
         nodes_seen = set()
-        row_index = -1
 
-        for i, key in enumerate(self.row_map):
+        for node in self.category_nodes:
-            if self.hidden_categories and self.categories[i] in self.hidden_categories:
+            key = node.category_key
-                continue
+            for tag_item in node.child_tags():
-            row_index += 1
-            category_item = self.root_item.children[row_index]
-            for tag_item in category_item.child_tags():
                 tag = tag_item.tag
                 if tag.state != TAG_SEARCH_STATES['clear']:
                     prefix = ' not ' if tag.state == TAG_SEARCH_STATES['mark_minus'] \
@@ -1089,15 +1171,18 @@ class TagsModel(QAbstractItemModel): # {{{
                     if tag.name and tag.name[0] == u'\u2605': # char is a star. Assume rating
                         ans.append('%s%s:%s'%(prefix, category, len(tag.name)))
                     else:
+                        name = getattr(tag, 'original_name', tag.name)
+                        use_prefix = getattr(tag, 'use_prefix', False)
                         if category == 'tags':
-                            if tag.name in tags_seen:
+                            if name in tags_seen:
                                 continue
-                            tags_seen.add(tag.name)
+                            tags_seen.add(name)
                         if tag in nodes_seen:
                             continue
                         nodes_seen.add(tag)
-                        ans.append('%s%s:"=%s"'%(prefix, category,
+                        ans.append('%s%s:"=%s%s"'%(prefix, category,
-                                                 tag.name.replace(r'"', r'\"')))
+                                                '.' if use_prefix else '',
+                                                 name.replace(r'"', r'\"')))
         return ans
 
     def find_item_node(self, key, txt, start_path):

src/calibre/library/caches.py

@@ -124,8 +124,14 @@ def _match(query, value, matchkind):
     for t in value:
         t = icu_lower(t)
         try:     ### ignore regexp exceptions, required because search-ahead tries before typing is finished
-            if ((matchkind == EQUALS_MATCH and query == t) or
+            if (matchkind == EQUALS_MATCH):
-                (matchkind == REGEXP_MATCH and re.search(query, t, re.I)) or ### search unanchored
+                if query[0] == '.':
+                    if t.startswith(query[1:]):
+                        ql = len(query) - 1
+                        return (len(t) == ql) or (t[ql:ql+1] == '.')
+                elif query == t:
+                    return True
+            elif ((matchkind == REGEXP_MATCH and re.search(query, t, re.I)) or ### search unanchored
                   (matchkind == CONTAINS_MATCH and query in t)):
                     return True
         except re.error:
@@ -415,10 +421,20 @@ class ResultCache(SearchQueryParser): # {{{
         if self.db_prefs is None:
             return  res
         user_cats = self.db_prefs.get('user_categories', [])
-        if location not in user_cats:
-            return res
         c = set(candidates)
-        for (item, category, ign) in user_cats[location]:
+        l = location.rfind('.')
+        if l > 0:
+            alt_loc = location[0:l]
+            alt_item = location[l+1:]
+        for key in user_cats:
+            if key == location or key.startswith(location + '.'):
+                for (item, category, ign) in user_cats[key]:
+                    s = self.get_matches(category, '=' + item, candidates=c)
+                    c -= s
+                    res |= s
+            elif key == alt_loc:
+                for (item, category, ign) in user_cats[key]:
+                    if item == alt_item:
                         s = self.get_matches(category, '=' + item, candidates=c)
                         c -= s
                         res |= s

src/calibre/library/database2.py

@@ -174,6 +174,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         self.prefs = DBPrefs(self)
         defs = self.prefs.defaults
         defs['gui_restriction'] = defs['cs_restriction'] = ''
+        defs['categories_using_hierarchy'] = []
 
         # Migrate saved search and user categories to db preference scheme
         def migrate_preference(key, default):
@@ -812,6 +813,21 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                                             index_is_id=index_is_id),
                         extra=self.get_custom_extra(idx, label=meta['label'],
                                                     index_is_id=index_is_id))
+
+        user_cats = self.prefs['user_categories']
+        user_cat_vals = {}
+        for ucat in user_cats:
+            res = []
+            for name,cat,ign in user_cats[ucat]:
+                v = mi.get(cat, None)
+                if isinstance(v, list):
+                    if name in v:
+                        res.append([name,cat])
+                elif name == v:
+                    res.append([name,cat])
+            user_cat_vals[ucat] = res
+        mi.user_categories = user_cat_vals
+
         if get_cover:
             mi.cover = self.cover(id, index_is_id=True, as_path=True)
         return mi
@@ -1406,7 +1422,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         # temporarily duplicating the categories lists.
         taglist = {}
         for c in categories.keys():
-            taglist[c] = dict(map(lambda t:(t.name, t), categories[c]))
+            taglist[c] = dict(map(lambda t:(icu_lower(t.name), t), categories[c]))
 
         muc = self.prefs.get('grouped_search_make_user_categories', [])
         gst = self.prefs.get('grouped_search_terms', {})
@@ -1422,8 +1438,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         for user_cat in sorted(user_categories.keys(), key=sort_key):
             items = []
             for (name,label,ign) in user_categories[user_cat]:
-                if label in taglist and name in taglist[label]:
+                n = icu_lower(name)
-                    items.append(taglist[label][name])
+                if label in taglist and n in taglist[label]:
+                    items.append(taglist[label][n])
                 # else: do nothing, to not include nodes w zero counts
             cat_name = '@' + user_cat # add the '@' to avoid name collision
             # Not a problem if we accumulate entries in the icon map

src/calibre/library/field_metadata.py

@@ -32,7 +32,7 @@ category_icon_map = {
                     'news'      : 'news.png',
                     'tags'      : 'tags.png',
                     'custom:'   : 'column.png',
-                    'user:'     : 'drawer.png',
+                    'user:'     : 'tb_folder.png',
                     'search'    : 'search.png'
             }
 

src/calibre/utils/formatter_functions.py

@@ -396,6 +396,34 @@ class BuiltinListitem(BuiltinFormatterFunction):
         except:
             return ''
 
+class BuiltinSublist(BuiltinFormatterFunction):
+    name = 'sublist'
+    arg_count = 4
+    doc = _('sublist(val, start_index, end_index, separator) -- interpret the '
+            ' value as a list of items separated by `separator`, returning a '
+            ' new list made from the `start_index`th to the `end_index`th item. '
+            'The first item is number zero. If an index is negative, then it '
+            'counts from the end of the list. As a special case, an end_index '
+            'of zero is assumed to be the length of the list. Examples using '
+            'basic template mode and assuming a #genre value if A.B.C: '
+            '{#genre:sublist(-1,0,.)} returns C<br/>'
+            '{#genre:sublist(0,1,.)} returns A<br/>'
+            '{#genre:sublist(0,-1,.)} returns A.B')
+
+    def evaluate(self, formatter, kwargs, mi, locals, val, start_index, end_index, sep):
+        if not val:
+            return ''
+        si = int(start_index)
+        ei = int(end_index)
+        val = val.split(sep)
+        try:
+            if ei == 0:
+                return sep.join(val[si:])
+            else:
+                return sep.join(val[si:ei])
+        except:
+            return ''
+
 class BuiltinUppercase(BuiltinFormatterFunction):
     name = 'uppercase'
     arg_count = 1
@@ -447,6 +475,7 @@ builtin_re          = BuiltinRe()
 builtin_shorten     = BuiltinShorten()
 builtin_strcat      = BuiltinStrcat()
 builtin_strcmp      = BuiltinStrcmp()
+builtin_sublist     = BuiltinSublist()
 builtin_substr      = BuiltinSubstr()
 builtin_subtract    = BuiltinSubtract()
 builtin_switch      = BuiltinSwitch()