diff --git a/src/calibre/gui2/viewer/highlights.py b/src/calibre/gui2/viewer/highlights.py
index e7caedadcc..4f6204b666 100644
--- a/src/calibre/gui2/viewer/highlights.py
+++ b/src/calibre/gui2/viewer/highlights.py
@@ -484,7 +484,7 @@ class HighlightsPanel(QWidget):
         self.setFocusPolicy(Qt.FocusPolicy.NoFocus)
         self.l = l = QVBoxLayout(self)
         l.setContentsMargins(0, 0, 0, 0)
-        self.search_input = si = SearchInput(self, 'highlights-search')
+        self.search_input = si = SearchInput(self, 'highlights-search', show_return_button=False)
         si.do_search.connect(self.search_requested)
         l.addWidget(si)
 
diff --git a/src/calibre/gui2/viewer/search.py b/src/calibre/gui2/viewer/search.py
index c92d541b91..a6bc791c69 100644
--- a/src/calibre/gui2/viewer/search.py
+++ b/src/calibre/gui2/viewer/search.py
@@ -407,7 +407,7 @@ class SearchInput(QWidget):  # {{{
     cleared = pyqtSignal()
     go_back = pyqtSignal()
 
-    def __init__(self, parent=None, panel_name='search'):
+    def __init__(self, parent=None, panel_name='search', show_return_button=True):
         QWidget.__init__(self, parent)
         self.ignore_search_type_changes = False
         self.l = l = QVBoxLayout(self)
@@ -477,6 +477,7 @@ class SearchInput(QWidget):  # {{{
         rb.setToolTip(_('Go back to where you were before searching'))
         rb.clicked.connect(self.go_back)
         h.addWidget(rb)
+        rb.setVisible(show_return_button)
 
     def history_saved(self, new_text, history):
         if new_text: