diff --git a/src/calibre/gui2/book_details.py b/src/calibre/gui2/book_details.py
index 8a9d40d73b..5214f1a1d5 100644
--- a/src/calibre/gui2/book_details.py
+++ b/src/calibre/gui2/book_details.py
@@ -20,7 +20,6 @@ from calibre.constants import preferred_encoding
 from calibre.library.comments import comments_to_html
 from calibre.gui2 import config, open_local_file
 from calibre.utils.icu import sort_key
-from calibre.utils.config import tweaks
 
 # render_rows(data) {{{
 WEIGHTS = collections.defaultdict(lambda : 100)
@@ -30,25 +29,8 @@ WEIGHTS[_('Collections')] = 2
 WEIGHTS[_('Series')] = 3
 WEIGHTS[_('Tags')] = 4
 
-def keys_to_display(data):
-    kt = data.get('__cf_kt__', None)
-    if kt is None:
-        return data.keys()
-    cfkeys = frozenset([k for k in kt.values()])
-    yes_fields = set(tweaks['book_details_will_display'])
-    no_fields = set(tweaks['book_details_wont_display'])
-    if '*' in yes_fields:
-        yes_fields = cfkeys
-    if '*' in no_fields:
-        no_fields = cfkeys
-    todisplay = frozenset(yes_fields - no_fields)
-    res = [k for k in data.keys()
-              if k not in kt or (kt[k] not in cfkeys or kt[k] in todisplay)]
-    res.remove('__cf_kt__')
-    return res
-
 def render_rows(data):
-    keys = keys_to_display(data)
+    keys = data.keys()
     # First sort by name. The WEIGHTS sort will preserve this sub-order
     keys.sort(key=sort_key)
     keys.sort(key=lambda x: WEIGHTS[x])
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index a2e7bdfa33..37d7d56ce1 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -303,6 +303,20 @@ class BooksModel(QAbstractTableModel): # {{{
         return self.rowCount(None)
 
     def get_book_display_info(self, idx):
+        def custom_keys_to_display():
+            ans = getattr(self, '_custom_fields_in_book_info', None)
+            if ans is None:
+                cfkeys = set(self.db.custom_field_keys())
+                yes_fields = set(tweaks['book_details_will_display'])
+                no_fields = set(tweaks['book_details_wont_display'])
+                if '*' in yes_fields:
+                    yes_fields = cfkeys
+                if '*' in no_fields:
+                    no_fields = cfkeys
+                ans = frozenset(yes_fields - no_fields)
+                setattr(self, '_custom_fields_in_book_info', ans)
+            return ans
+
         data = {}
         cdata = self.cover(idx)
         if cdata:
@@ -334,15 +348,13 @@ class BooksModel(QAbstractTableModel): # {{{
                 _('Book %s of %s.')%\
                     (sidx, prepare_string_for_xml(series))
         mi = self.db.get_metadata(idx)
+        cf_to_display = custom_keys_to_display()
         for key in mi.custom_field_keys():
+            if key not in cf_to_display:
+                continue
             name, val = mi.format_field(key)
             if val:
                 data[name] = val
-        cf_kt = {}
-        for key,mi in self.db.all_metadata().items():
-            if mi['is_custom']:
-                cf_kt[mi['name']] = key
-        data['__cf_kt__'] = cf_kt
         return data
 
     def set_cache(self, idx):