diff --git a/src/calibre/gui2/viewer/highlights.py b/src/calibre/gui2/viewer/highlights.py
index 336442d35b..98f1beea30 100644
--- a/src/calibre/gui2/viewer/highlights.py
+++ b/src/calibre/gui2/viewer/highlights.py
@@ -184,12 +184,10 @@ class Highlights(QTreeWidget):
         self.context_menu = m = QMenu(self)
         if h is not None:
             m.addAction(QIcon(I('edit_input.png')), _('Modify this highlight'), self.edit_requested.emit)
-            m.addAction(QIcon(I('edit_input.png')), _('Edit notes for this highlight'), self.edit_notes_requested.emit)
+            m.addAction(QIcon(I('modified.png')), _('Edit notes for this highlight'), self.edit_notes_requested.emit)
             m.addAction(QIcon(I('trash.png')), ngettext(
                 'Delete this highlight', 'Delete selected highlights', len(self.selectedItems())
             ), self.delete_requested.emit)
-            if h.get('notes'):
-                m.addAction(QIcon(I('modified.png')), _('Edit notes for this highlight'), self.edit_notes_requested.emit)
         m.addSeparator()
         m.addAction(_('Expand all'), self.expandAll)
         m.addAction(_('Collapse all'), self.collapseAll)