diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index 6fa23c2813..31b8cf46bf 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -93,9 +93,9 @@ class BooksModel(QAbstractTableModel): # {{{
         self.bool_no_icon = QIcon(I('list_remove.png'))
         self.bool_blank_icon = QIcon(I('blank.png'))
         self.device_connected = False
-        self.rows_to_highlight = []
-        self.rows_to_highlight_set = set()
-        self.current_highlighted_row = None
+        self.ids_to_highlight = []
+        self.ids_to_highlight_set = set()
+        self.current_highlighted_idx = None
         self.highlight_only = False
         self.read_config()
 
@@ -131,9 +131,9 @@ class BooksModel(QAbstractTableModel): # {{{
         self.book_on_device = func
 
     def set_database(self, db):
-        self.rows_to_highlight = []
-        self.rows_to_highlight_set = set()
-        self.current_highlighted_row = None
+        self.ids_to_highlight = []
+        self.ids_to_highlight_set = set()
+        self.current_highlighted_idx = None
         self.db = db
         self.custom_columns = self.db.field_metadata.custom_field_metadata()
         self.column_map = list(self.orig_headers.keys()) + \
@@ -241,43 +241,55 @@ class BooksModel(QAbstractTableModel): # {{{
         if self.last_search:
             self.research()
 
-    def get_current_highlighted_row(self):
-        if len(self.rows_to_highlight) == 0 or self.current_highlighted_row is None:
+    def get_current_highlighted_id(self):
+        if len(self.ids_to_highlight) == 0 or self.current_highlighted_idx is None:
             return None
         try:
-            return self.rows_to_highlight[self.current_highlighted_row]
+            return self.ids_to_highlight[self.current_highlighted_idx]
         except:
             return None
 
-    def get_next_highlighted_row(self, forward):
-        if len(self.rows_to_highlight) == 0 or self.current_highlighted_row is None:
+    def get_next_highlighted_id(self, current_row, forward):
+        if len(self.ids_to_highlight) == 0 or self.current_highlighted_idx is None:
             return None
-        self.current_highlighted_row += 1 if forward else -1
-        if self.current_highlighted_row < 0:
-            self.current_highlighted_row = len(self.rows_to_highlight) - 1;
-        elif self.current_highlighted_row >= len(self.rows_to_highlight):
-            self.current_highlighted_row = 0
-        return self.get_current_highlighted_row()
+        if current_row is None:
+            row_ = self.current_highlighted_idx
+        else:
+            row_ = current_row
+        while True:
+            row_ += 1 if forward else -1
+            if row_ < 0:
+                row_ = self.count() - 1;
+            elif row_ >= self.count():
+                row_ = 0
+            if self.id(row_) in self.ids_to_highlight_set:
+                break
+        try:
+            self.current_highlighted_idx = self.ids_to_highlight.index(self.id(row_))
+        except:
+            # This shouldn't happen ...
+            return None
+        return self.get_current_highlighted_id()
 
     def search(self, text, reset=True):
         try:
             if self.highlight_only:
                 self.db.search('')
                 if not text:
-                    self.rows_to_highlight = []
-                    self.rows_to_highlight_set = set()
-                    self.current_highlighted_row = None
+                    self.ids_to_highlight = []
+                    self.ids_to_highlight_set = set()
+                    self.current_highlighted_idx = None
                 else:
-                    self.rows_to_highlight = self.db.search(text, return_matches=True)
-                    self.rows_to_highlight_set = set(self.rows_to_highlight)
-                    if self.rows_to_highlight:
-                        self.current_highlighted_row = 0
+                    self.ids_to_highlight = self.db.search(text, return_matches=True)
+                    self.ids_to_highlight_set = set(self.ids_to_highlight)
+                    if self.ids_to_highlight:
+                        self.current_highlighted_idx = 0
                     else:
-                        self.current_highlighted_row = None
+                        self.current_highlighted_idx = None
             else:
-                self.rows_to_highlight = []
-                self.rows_to_highlight_set = set()
-                self.current_highlighted_row = None
+                self.ids_to_highlight = []
+                self.ids_to_highlight_set = set()
+                self.current_highlighted_idx = None
                 self.db.search(text)
         except ParseException as e:
             self.searched.emit(e.msg)
@@ -700,7 +712,7 @@ class BooksModel(QAbstractTableModel): # {{{
         if role in (Qt.DisplayRole, Qt.EditRole):
             return self.column_to_dc_map[col](index.row())
         elif role == Qt.BackgroundColorRole:
-            if self.id(index) in self.rows_to_highlight_set:
+            if self.id(index) in self.ids_to_highlight_set:
                 return QColor('lightgreen')
         elif role == Qt.DecorationRole:
             if self.column_to_dc_decorator_map[col] is not None:
diff --git a/src/calibre/gui2/library/views.py b/src/calibre/gui2/library/views.py
index e5cc259244..3ff0fc3cd7 100644
--- a/src/calibre/gui2/library/views.py
+++ b/src/calibre/gui2/library/views.py
@@ -681,13 +681,18 @@ class BooksView(QTableView): # {{{
         self._model.set_editable(editable)
 
     def move_highlighted_row(self, forward):
-        id_to_select = self._model.get_next_highlighted_row(forward)
+        rows = self.selectionModel().selectedRows()
+        if len(rows) > 0:
+            current_row = rows[0].row()
+        else:
+            current_row = None
+        id_to_select = self._model.get_next_highlighted_id(current_row, forward)
         if id_to_select is not None:
             self.select_rows([id_to_select], using_ids=True)
 
     def search_proxy(self, txt):
         self._model.search(txt)
-        id_to_select = self._model.get_current_highlighted_row()
+        id_to_select = self._model.get_current_highlighted_id()
         if id_to_select is not None:
             self.select_rows([id_to_select], using_ids=True)
         self.setFocus(Qt.OtherFocusReason)