diff --git a/src/calibre/gui2/library/views.py b/src/calibre/gui2/library/views.py
index 61161cd5e6..c62936a46f 100644
--- a/src/calibre/gui2/library/views.py
+++ b/src/calibre/gui2/library/views.py
@@ -551,8 +551,10 @@ class BooksView(QTableView): # {{{
         return mods & Qt.ControlModifier or mods & Qt.ShiftModifier
 
     def mousePressEvent(self, event):
-        if event.button() == Qt.LeftButton and not self.event_has_mods():
-            self.drag_start_pos = event.pos()
+        ep = event.pos()
+        if self.indexAt(ep) in self.selectionModel().selectedIndexes() and \
+                event.button() == Qt.LeftButton and not self.event_has_mods():
+            self.drag_start_pos = ep
         return QTableView.mousePressEvent(self, event)
 
     def mouseMoveEvent(self, event):