diff --git a/src/calibre/gui2/tag_browser/model.py b/src/calibre/gui2/tag_browser/model.py
index ee27b46080..35045be8b6 100644
--- a/src/calibre/gui2/tag_browser/model.py
+++ b/src/calibre/gui2/tag_browser/model.py
@@ -685,44 +685,37 @@ class TagsModel(QAbstractItemModel): # {{{
 
     def handle_user_category_drop(self, on_node, ids, column):
         categories = self.db.prefs.get('user_categories', {})
-        category = categories.get(on_node.category_key[1:], None)
-        if category is None:
+        cat_contents = categories.get(on_node.category_key[1:], None)
+        if cat_contents is None:
             return
+        cat_contents = set([(v, c) for v,c,ign in cat_contents])
+
         fm_src = self.db.metadata_for_field(column)
+        label = fm_src['label']
+
         for id in ids:
-            label = fm_src['label']
             if not fm_src['is_custom']:
                 if label == 'authors':
-                    items = self.db.get_authors_with_ids()
-                    items = [(i[0], i[1].replace('|', ',')) for i in items]
                     value = self.db.authors(id, index_is_id=True)
                     value = [v.replace('|', ',') for v in value.split(',')]
                 elif label == 'publisher':
-                    items = self.db.get_publishers_with_ids()
                     value = self.db.publisher(id, index_is_id=True)
                 elif label == 'series':
-                    items = self.db.get_series_with_ids()
                     value = self.db.series(id, index_is_id=True)
             else:
-                items = self.db.get_custom_items_with_ids(label=label)
                 if fm_src['datatype'] != 'composite':
                     value = self.db.get_custom(id, label=label, index_is_id=True)
                 else:
                     value = self.db.get_property(id, loc=fm_src['rec_index'],
                                                  index_is_id=True)
-            if value is None:
-                return
-            if not isinstance(value, list):
-                value = [value]
-            for val in value:
-                for (v, c, id) in category:
-                    if v == val and c == column:
-                        break
-                else:
-                    category.append([val, column, 0])
-            categories[on_node.category_key[1:]] = category
-            self.db.prefs.set('user_categories', categories)
-            self.refresh_required.emit()
+            if value:
+                if not isinstance(value, list):
+                    value = [value]
+                cat_contents |= set([(v, column) for v in value])
+
+        categories[on_node.category_key[1:]] = [[v, c, 0] for v,c in cat_contents]
+        self.db.prefs.set('user_categories', categories)
+        self.refresh_required.emit()
 
     def handle_drop(self, on_node, ids):
         #print 'Dropped ids:', ids, on_node.tag