Database integrity check now dumps and reloads SQL. This should allow recovery from some database corruption.

src/calibre/gui2/dialogs/config/__init__.py

@@ -2,12 +2,13 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 import os, re, time, textwrap
 
-from PyQt4.Qt import    QDialog, QMessageBox, QListWidgetItem, QIcon, \
+from PyQt4.Qt import    QDialog, QListWidgetItem, QIcon, \
                         QDesktopServices, QVBoxLayout, QLabel, QPlainTextEdit, \
                         QStringListModel, QAbstractItemModel, QFont, \
                         SIGNAL, QThread, Qt, QSize, QVariant, QUrl, \
                         QModelIndex, QInputDialog, QAbstractTableModel, \
-                        QDialogButtonBox, QTabWidget, QBrush, QLineEdit
+                        QDialogButtonBox, QTabWidget, QBrush, QLineEdit, \
+                        QProgressDialog
 
 from calibre.constants import islinux, iswindows
 from calibre.gui2.dialogs.config.config_ui import Ui_Dialog
@@ -648,7 +649,7 @@ class ConfigDialog(QDialog, Ui_Dialog):
         QDesktopServices.openUrl(QUrl('http://127.0.0.1:'+str(self.port.value())))
 
     def compact(self, toggled):
-        d = Vacuum(self, self.db)
+        d = CheckIntegrity(self.db, self)
         d.exec_()
 
     def browse(self):
@@ -739,25 +740,48 @@ class VacThread(QThread):
         self._parent = parent
 
     def run(self):
-        bad = self.db.check_integrity()
+        err = bad = None
-        self.emit(SIGNAL('check_done(PyQt_PyObject)'), bad)
+        try:
+            bad = self.db.check_integrity(self.callback)
+        except:
+            import traceback
+            err = traceback.format_exc()
+        self.emit(SIGNAL('check_done(PyQt_PyObject, PyQt_PyObject)'), bad, err)
 
-class Vacuum(QMessageBox):
+    def callback(self, progress, msg):
+        self.emit(SIGNAL('callback(PyQt_PyObject,PyQt_PyObject)'), progress,
+                msg)
+
+class CheckIntegrity(QProgressDialog):
+
+    def __init__(self, db, parent=None):
+        QProgressDialog.__init__(self, parent)
+        self.setCancelButtonText('')
+        self.setMinimum(0)
+        self.setMaximum(100)
+        self.setWindowTitle(_('Checking database integrity'))
+        self.setAutoReset(False)
+        self.setValue(0)
 
-    def __init__(self, parent, db):
-        self.db = db
-        QMessageBox.__init__(self, QMessageBox.Information, _('Checking...'),
-                             _('Checking database integrity. This may take a while.'),
-                             QMessageBox.NoButton, parent)
         self.vthread = VacThread(self, db)
-        self.connect(self.vthread, SIGNAL('check_done(PyQt_PyObject)'),
+        self.connect(self.vthread, SIGNAL('check_done(PyQt_PyObject,PyQt_PyObject)'),
                 self.check_done,
                 Qt.QueuedConnection)
+        self.connect(self.vthread,
+                SIGNAL('callback(PyQt_PyObject,PyQt_PyObject)'),
+                self.callback, Qt.QueuedConnection)
         self.vthread.start()
 
+    def callback(self, progress, msg):
+        self.setLabelText(msg)
+        self.setValue(int(100*progress))
 
-    def check_done(self, bad):
+    def check_done(self, bad, err):
-        if bad:
+        if err:
+            error_dialog(self, _('Error'),
+                    _('Failed to check database integrity'),
+                    det_msg=err, show=True)
+        elif bad:
             titles = [self.db.title(x, index_is_id=True) for x in bad]
             det_msg = '\n'.join(titles)
             warning_dialog(self, _('Some inconsistencies found'),
@@ -767,7 +791,7 @@ class Vacuum(QMessageBox):
                         'You should check them manually. This can '
                         'happen if you manipulate the files in the '
                         'library folder directly.'), det_msg=det_msg, show=True)
-        self.accept()
+        self.reset()
 
 
 

src/calibre/library/database2.py

@@ -23,7 +23,7 @@ from PyQt4.QtGui import QImage
 
 from calibre.ebooks.metadata import title_sort
 from calibre.library.database import LibraryDatabase
-from calibre.library.sqlite import connect, IntegrityError
+from calibre.library.sqlite import connect, IntegrityError, DBThread
 from calibre.utils.search_query_parser import SearchQueryParser
 from calibre.ebooks.metadata import string_to_authors, authors_to_string, \
                                     MetaInformation, authors_to_sort_string
@@ -1670,9 +1670,40 @@ books_series_link      feeds
 
         return duplicates
 
-    def check_integrity(self):
+    def check_integrity(self, callback):
+        callback(0., _('Checking SQL integrity...'))
+        user_version = self.user_version
+        sql = self.conn.dump()
+        self.conn.close()
+        dest = self.dbpath+'.old'
+        if os.path.exists(dest):
+            os.remove(dest)
+        shutil.copyfile(self.dbpath, dest)
+        try:
+            os.remove(self.dbpath)
+            ndb = DBThread(self.dbpath, None)
+            ndb.connect()
+            conn = ndb.conn
+            conn.executescript(sql)
+            conn.commit()
+            conn.execute('pragma user_version=%d'%user_version)
+            conn.commit()
+            conn.close()
+        except:
+            if os.path.exists(self.dbpath):
+                os.remove(self.dbpath)
+            shutil.copyfile(dest, self.dbpath)
+            os.remove(dest)
+            raise
+        else:
+            os.remove(dest)
+            self.connect()
+            self.refresh()
+        callback(0.1, _('Checking for missing files.'))
         bad = {}
-        for id in self.data.universal_set():
+        us = self.data.universal_set()
+        total = float(len(us))
+        for i, id in enumerate(us):
             formats = self.data.get(id, FIELD_MAP['formats'], row_is_id=True)
             if not formats:
                 formats = []
@@ -1692,6 +1723,7 @@ books_series_link      feeds
                 if id not in bad:
                     bad[id] = []
                 bad[id].append(fmt)
+            callback(0.1+0.9*(1+i)/total, _('Checked id') + ' %d'%id)
 
         for id in bad:
             for fmt in bad[id]:
@@ -1699,8 +1731,6 @@ books_series_link      feeds
         self.conn.commit()
         self.refresh_ids(list(bad.keys()))
 
-        self.vacuum()
-
         return bad
 
 

src/calibre/library/sqlite.py

@@ -130,11 +130,17 @@ class DBThread(Thread):
                 if func == self.CLOSE:
                     self.conn.close()
                     break
-                func = getattr(self.conn, func)
+                if func == 'dump':
-                try:
+                    try:
-                    ok, res = True, func(*args, **kwargs)
+                        ok, res = True, '\n'.join(self.conn.iterdump())
-                except Exception, err:
+                    except Exception, err:
-                    ok, res = False, (err, traceback.format_exc())
+                        ok, res = False, (err, traceback.format_exc())
+                else:
+                    func = getattr(self.conn, func)
+                    try:
+                        ok, res = True, func(*args, **kwargs)
+                    except Exception, err:
+                        ok, res = False, (err, traceback.format_exc())
                 self.results.put((ok, res))
         except Exception, err:
             self.unhandled_error = (err, traceback.format_exc())
@@ -197,6 +203,9 @@ class ConnectionProxy(object):
     @proxy
     def cursor(self): pass
 
+    @proxy
+    def dump(self): pass
+
 def connect(dbpath, row_factory=None):
     conn = ConnectionProxy(DBThread(dbpath, row_factory))
     conn.proxy.start()