From 967829c664e883ac1d2b05538d4b652e655888a0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 22 Aug 2009 12:23:33 -0600 Subject: [PATCH] Database integrity check now dumps and reloads SQL. This should allow recovery from some database corruption. --- src/calibre/gui2/dialogs/config/__init__.py | 54 +++++++++++++++------ src/calibre/library/database2.py | 40 +++++++++++++-- src/calibre/library/sqlite.py | 19 ++++++-- 3 files changed, 88 insertions(+), 25 deletions(-) diff --git a/src/calibre/gui2/dialogs/config/__init__.py b/src/calibre/gui2/dialogs/config/__init__.py index feab592467..d719f24d66 100644 --- a/src/calibre/gui2/dialogs/config/__init__.py +++ b/src/calibre/gui2/dialogs/config/__init__.py @@ -2,12 +2,13 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ' import os, re, time, textwrap -from PyQt4.Qt import QDialog, QMessageBox, QListWidgetItem, QIcon, \ +from PyQt4.Qt import QDialog, QListWidgetItem, QIcon, \ QDesktopServices, QVBoxLayout, QLabel, QPlainTextEdit, \ QStringListModel, QAbstractItemModel, QFont, \ SIGNAL, QThread, Qt, QSize, QVariant, QUrl, \ QModelIndex, QInputDialog, QAbstractTableModel, \ - QDialogButtonBox, QTabWidget, QBrush, QLineEdit + QDialogButtonBox, QTabWidget, QBrush, QLineEdit, \ + QProgressDialog from calibre.constants import islinux, iswindows from calibre.gui2.dialogs.config.config_ui import Ui_Dialog @@ -648,7 +649,7 @@ class ConfigDialog(QDialog, Ui_Dialog): QDesktopServices.openUrl(QUrl('http://127.0.0.1:'+str(self.port.value()))) def compact(self, toggled): - d = Vacuum(self, self.db) + d = CheckIntegrity(self.db, self) d.exec_() def browse(self): @@ -739,25 +740,48 @@ class VacThread(QThread): self._parent = parent def run(self): - bad = self.db.check_integrity() - self.emit(SIGNAL('check_done(PyQt_PyObject)'), bad) + err = bad = None + try: + bad = self.db.check_integrity(self.callback) + except: + import traceback + err = traceback.format_exc() + self.emit(SIGNAL('check_done(PyQt_PyObject, PyQt_PyObject)'), bad, err) -class Vacuum(QMessageBox): + def callback(self, progress, msg): + self.emit(SIGNAL('callback(PyQt_PyObject,PyQt_PyObject)'), progress, + msg) + +class CheckIntegrity(QProgressDialog): + + def __init__(self, db, parent=None): + QProgressDialog.__init__(self, parent) + self.setCancelButtonText('') + self.setMinimum(0) + self.setMaximum(100) + self.setWindowTitle(_('Checking database integrity')) + self.setAutoReset(False) + self.setValue(0) - def __init__(self, parent, db): - self.db = db - QMessageBox.__init__(self, QMessageBox.Information, _('Checking...'), - _('Checking database integrity. This may take a while.'), - QMessageBox.NoButton, parent) self.vthread = VacThread(self, db) - self.connect(self.vthread, SIGNAL('check_done(PyQt_PyObject)'), + self.connect(self.vthread, SIGNAL('check_done(PyQt_PyObject,PyQt_PyObject)'), self.check_done, Qt.QueuedConnection) + self.connect(self.vthread, + SIGNAL('callback(PyQt_PyObject,PyQt_PyObject)'), + self.callback, Qt.QueuedConnection) self.vthread.start() + def callback(self, progress, msg): + self.setLabelText(msg) + self.setValue(int(100*progress)) - def check_done(self, bad): - if bad: + def check_done(self, bad, err): + if err: + error_dialog(self, _('Error'), + _('Failed to check database integrity'), + det_msg=err, show=True) + elif bad: titles = [self.db.title(x, index_is_id=True) for x in bad] det_msg = '\n'.join(titles) warning_dialog(self, _('Some inconsistencies found'), @@ -767,7 +791,7 @@ class Vacuum(QMessageBox): 'You should check them manually. This can ' 'happen if you manipulate the files in the ' 'library folder directly.'), det_msg=det_msg, show=True) - self.accept() + self.reset() diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index daec400101..c605be0879 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -23,7 +23,7 @@ from PyQt4.QtGui import QImage from calibre.ebooks.metadata import title_sort from calibre.library.database import LibraryDatabase -from calibre.library.sqlite import connect, IntegrityError +from calibre.library.sqlite import connect, IntegrityError, DBThread from calibre.utils.search_query_parser import SearchQueryParser from calibre.ebooks.metadata import string_to_authors, authors_to_string, \ MetaInformation, authors_to_sort_string @@ -1670,9 +1670,40 @@ books_series_link feeds return duplicates - def check_integrity(self): + def check_integrity(self, callback): + callback(0., _('Checking SQL integrity...')) + user_version = self.user_version + sql = self.conn.dump() + self.conn.close() + dest = self.dbpath+'.old' + if os.path.exists(dest): + os.remove(dest) + shutil.copyfile(self.dbpath, dest) + try: + os.remove(self.dbpath) + ndb = DBThread(self.dbpath, None) + ndb.connect() + conn = ndb.conn + conn.executescript(sql) + conn.commit() + conn.execute('pragma user_version=%d'%user_version) + conn.commit() + conn.close() + except: + if os.path.exists(self.dbpath): + os.remove(self.dbpath) + shutil.copyfile(dest, self.dbpath) + os.remove(dest) + raise + else: + os.remove(dest) + self.connect() + self.refresh() + callback(0.1, _('Checking for missing files.')) bad = {} - for id in self.data.universal_set(): + us = self.data.universal_set() + total = float(len(us)) + for i, id in enumerate(us): formats = self.data.get(id, FIELD_MAP['formats'], row_is_id=True) if not formats: formats = [] @@ -1692,6 +1723,7 @@ books_series_link feeds if id not in bad: bad[id] = [] bad[id].append(fmt) + callback(0.1+0.9*(1+i)/total, _('Checked id') + ' %d'%id) for id in bad: for fmt in bad[id]: @@ -1699,8 +1731,6 @@ books_series_link feeds self.conn.commit() self.refresh_ids(list(bad.keys())) - self.vacuum() - return bad diff --git a/src/calibre/library/sqlite.py b/src/calibre/library/sqlite.py index f874796ade..a1caf506d9 100644 --- a/src/calibre/library/sqlite.py +++ b/src/calibre/library/sqlite.py @@ -130,11 +130,17 @@ class DBThread(Thread): if func == self.CLOSE: self.conn.close() break - func = getattr(self.conn, func) - try: - ok, res = True, func(*args, **kwargs) - except Exception, err: - ok, res = False, (err, traceback.format_exc()) + if func == 'dump': + try: + ok, res = True, '\n'.join(self.conn.iterdump()) + except Exception, err: + ok, res = False, (err, traceback.format_exc()) + else: + func = getattr(self.conn, func) + try: + ok, res = True, func(*args, **kwargs) + except Exception, err: + ok, res = False, (err, traceback.format_exc()) self.results.put((ok, res)) except Exception, err: self.unhandled_error = (err, traceback.format_exc()) @@ -197,6 +203,9 @@ class ConnectionProxy(object): @proxy def cursor(self): pass + @proxy + def dump(self): pass + def connect(dbpath, row_factory=None): conn = ConnectionProxy(DBThread(dbpath, row_factory)) conn.proxy.start()