diff --git a/src/calibre/library/server/__init__.py b/src/calibre/library/server/__init__.py
index 2a08c0b9b7..4a7fdddc52 100644
--- a/src/calibre/library/server/__init__.py
+++ b/src/calibre/library/server/__init__.py
@@ -51,14 +51,14 @@ def server_config(defaults=None):
     return c
 
 def custom_fields_to_display(db):
-    ckeys = db.field_metadata.ignorable_field_keys()
+    ckeys = set(db.field_metadata.ignorable_field_keys())
     yes_fields = set(tweaks['content_server_will_display'])
     no_fields = set(tweaks['content_server_wont_display'])
     if '*' in yes_fields:
-        yes_fields = set(ckeys)
+        yes_fields = ckeys
     if '*' in no_fields:
-        no_fields = set(ckeys)
-    return frozenset(yes_fields - no_fields)
+        no_fields = ckeys
+    return frozenset(ckeys & (yes_fields - no_fields))
 
 def main():
     from calibre.library.server.main import main