Changes to improve robustness:

1) Ensure that the server displays columns added using the command line.
2) Make "official" the fact that get_categories returns an ordered dict.

While in the process, move the general methods category_display_order and is_standard_category from the gui to db.get_categories().

Finally, improve the tooltip for L&F / Tag browser / hierarchical items

src/calibre/db/categories.py

@@ -6,6 +6,7 @@ __copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 import copy
+from collections import OrderedDict
 from functools import partial
 from polyglot.builtins import iteritems, native_string_type
 
@@ -115,6 +116,32 @@ def clean_user_categories(dbcache):
     return new_cats
 
 
+def is_standard_category(key):
+    return not (key.startswith('@') or key == 'search')
+
+
+def category_display_order(ordered_cats, all_cats):
+    # ordered_cats is the desired order. all_cats is the list of keys returned
+    # by get_categories, which is in the default order
+    cat_ord = []
+    all_cat_set = frozenset(all_cats)
+    # Do the standard categories first
+    # Verify all the columns in ordered_cats are actually in all_cats
+    for key in ordered_cats:
+        if is_standard_category(key) and key in all_cat_set:
+            cat_ord.append(key)
+    # Add any new standard cats at the end of the list
+    for key in all_cats:
+        if key not in cat_ord and is_standard_category(key):
+            cat_ord.append(key)
+    # Now add the non-standard cats (user cats and search)
+    for key in all_cats:
+        if not is_standard_category(key):
+            cat_ord.append(key)
+    return cat_ord
+
+
+
 numeric_collation = prefs['numeric_collation']
 
 
@@ -139,6 +166,10 @@ category_sort_keys[False]['name'] = \
     lambda x:sort_key(x.sort or x.name)
 
 
+# Various parts of calibre depend on the the order of fields in the returned
+# dict being in the default display order: standard fields, custom in alpha order,
+# user categories, then saved searches. This works because the backend adds
+# custom columns to field metadata in the right order.
 def get_categories(dbcache, sort='name', book_ids=None, first_letter_sort=False):
     if sort not in CATEGORY_SORTS:
         raise ValueError('sort ' + sort + ' not a valid value')
@@ -147,7 +178,7 @@ def get_categories(dbcache, sort='name', book_ids=None, first_letter_sort=False)
     book_rating_map = dbcache.fields['rating'].book_value_map
     lang_map = dbcache.fields['languages'].book_value_map
 
-    categories = {}
+    categories = OrderedDict()
     book_ids = frozenset(book_ids) if book_ids else book_ids
     pm_cache = {}
 

src/calibre/gui2/preferences/look_feel.py

@@ -18,8 +18,9 @@ from qt.core import (
 )
 
 from calibre import human_readable
-from calibre.ebooks.metadata.book.render import DEFAULT_AUTHOR_LINK
 from calibre.constants import ismacos, iswindows
+from calibre.db.categories import is_standard_category
+from calibre.ebooks.metadata.book.render import DEFAULT_AUTHOR_LINK
 from calibre.ebooks.metadata.sources.prefs import msprefs
 from calibre.gui2.custom_column_widgets import get_field_list as em_get_field_list
 from calibre.gui2 import default_author_link, icon_resource_manager, choose_save_file, choose_files
@@ -389,9 +390,6 @@ class TBDisplayedFields(DisplayedFields):  # {{{
         self.fields = [[x, x not in hc] for x in cat_ord]
         self.endResetModel()
 
-    def is_standard_category(self, key):
-        return self.gui.tags_view.model().is_standard_category(key)
-
     def commit(self):
         if self.changed:
             self.db.prefs.set('tag_browser_hidden_categories', [k for k,v in self.fields if not v])
@@ -825,9 +823,9 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
             model = self.tb_display_model
             fields = model.fields
             key = fields[row][0]
-            if not model.is_standard_category(key):
+            if not is_standard_category(key):
                 return
-            if row < len(fields) and model.is_standard_category(fields[row+1][0]):
+            if row < len(fields) and is_standard_category(fields[row+1][0]):
                 move_field_down(self.tb_display_order, model)
 
     def tb_up_button_clicked(self):
@@ -837,7 +835,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
             model = self.tb_display_model
             fields = model.fields
             key = fields[row][0]
-            if not model.is_standard_category(key):
+            if not is_standard_category(key):
                 return
             move_field_up(self.tb_display_order, model)
 

src/calibre/gui2/preferences/look_feel.ui

@@ -1192,7 +1192,7 @@ using the Tab key. The F2 (Edit) key will still open the template editor.</p&
           </property>
           <property name="sizeHint" stdset="0">
            <size>
-            <width>20</width>
+            <width>50</width>
             <height>40</height>
            </size>
           </property>
@@ -1278,7 +1278,8 @@ hierarchical tree in the Tag browser. For example, if you check
 and 'Mystery.Thriller' will be displayed with English and Thriller
 both under 'Mystery'. If 'tags' is not checked
 then the tags will be displayed each on their own line.&lt;/p&gt;
-&lt;p&gt;Some categories such as authors cannot be hierarchical.&lt;/p&gt;</string>
+&lt;p&gt;The categories 'authors', 'publisher', 'news', 'formats', and 'rating'
+cannot be hierarchical.&lt;/p&gt;</string>
           </property>
          </widget>
         </item>

src/calibre/gui2/tag_browser/model.py

@@ -15,7 +15,7 @@ from qt.core import (
 )
 
 from calibre.constants import config_dir
-from calibre.db.categories import Tag
+from calibre.db.categories import Tag, category_display_order
 from calibre.ebooks.metadata import rating_to_stars
 from calibre.gui2 import config, error_dialog, file_icon_provider, gprefs, question_dialog
 from calibre.gui2.dialogs.confirm_delete import confirm
@@ -1131,9 +1131,6 @@ class TagsModel(QAbstractItemModel):  # {{{
             return self.db.search('', return_matches=True, sort_results=False)
         return None
 
-    def is_standard_category(self, key):
-        return not (key.startswith('@') or key == 'search')
-
     def get_ordered_categories(self, use_defaults=False, pref_data_override=None):
         if use_defaults:
             tbo = []
@@ -1141,22 +1138,7 @@ class TagsModel(QAbstractItemModel):  # {{{
             tbo = [k for k,_ in pref_data_override]
         else:
             tbo = self.db.new_api.pref('tag_browser_category_order', [])
-        disp_cats = self.categories.keys()
+        return category_display_order(tbo, list(self.categories.keys()))
-        cat_ord = []
-        # Do the standard categories first
-        # Verify all the columns in the pref are actually in the tag browser
-        for key in tbo:
-            if self.is_standard_category(key) and key in disp_cats:
-                cat_ord.append(key)
-        # Add any new standard cats to the order pref at the end of the list
-        for key in disp_cats:
-            if key not in cat_ord and self.is_standard_category(key):
-                cat_ord.append(key)
-        # Now add the non-standard cats (user cats and search)
-        for key in disp_cats:
-            if not self.is_standard_category(key):
-                cat_ord.append(key)
-        return cat_ord
 
     def _get_category_nodes(self, sort):
         '''

src/calibre/srv/metadata.py

@@ -10,7 +10,7 @@ from functools import partial
 from threading import Lock
 
 from calibre.constants import config_dir
-from calibre.db.categories import Tag
+from calibre.db.categories import Tag, category_display_order
 from calibre.ebooks.metadata.sources.identify import urls_from_identifiers
 from calibre.utils.date import isoformat, UNDEFINED_DATE, local_tz
 from calibre.utils.config import tweaks
@@ -222,8 +222,7 @@ def create_toplevel_tree(category_data, items, field_metadata, opts, db):
     last_category_node, category_node_map, root = None, {}, {'id':None, 'children':[]}
     node_id_map = {}
     category_nodes, recount_nodes = [], []
-    scats = db.pref('tag_browser_category_order', [k for k in category_data])
+    scats = category_display_order(db.pref('tag_browser_category_order', []), list(category_data.keys()))
-    scats = [k for k in scats if k in field_metadata]
 
     for category in scats:
         is_user_category = category.startswith('@')