Combine the database integrity check and library check into a single menu item. Also nicer implementation of the db integrity check.

src/calibre/gui2/actions/choose_library.py

@@ -8,7 +8,7 @@ __docformat__ = 'restructuredtext en'
 import os, shutil
 from functools import partial
 
-from PyQt4.Qt import QMenu, Qt, QInputDialog, QThread, pyqtSignal, QProgressDialog
+from PyQt4.Qt import QMenu, Qt, QInputDialog
 
 from calibre import isbytestring
 from calibre.constants import filesystem_encoding
@@ -16,7 +16,7 @@ from calibre.utils.config import prefs
 from calibre.gui2 import gprefs, warning_dialog, Dispatcher, error_dialog, \
     question_dialog, info_dialog
 from calibre.gui2.actions import InterfaceAction
-from calibre.gui2.dialogs.check_library import CheckLibraryDialog
+from calibre.gui2.dialogs.check_library import CheckLibraryDialog, DBCheck
 
 class LibraryUsageStats(object): # {{{
 
@@ -76,76 +76,6 @@ class LibraryUsageStats(object): # {{{
         self.write_stats()
 # }}}
 
-# Check Integrity {{{
-
-class VacThread(QThread):
-
-    check_done = pyqtSignal(object, object)
-    callback   = pyqtSignal(object, object)
-
-    def __init__(self, parent, db):
-        QThread.__init__(self, parent)
-        self.db = db
-        self._parent = parent
-
-    def run(self):
-        err = bad = None
-        try:
-            bad = self.db.check_integrity(self.callbackf)
-        except:
-            import traceback
-            err = traceback.format_exc()
-        self.check_done.emit(bad, err)
-
-    def callbackf(self, progress, msg):
-        self.callback.emit(progress, msg)
-
-
-class CheckIntegrity(QProgressDialog):
-
-    def __init__(self, db, parent=None):
-        QProgressDialog.__init__(self, parent)
-        self.db = db
-        self.setCancelButton(None)
-        self.setMinimum(0)
-        self.setMaximum(100)
-        self.setWindowTitle(_('Checking database integrity'))
-        self.setAutoReset(False)
-        self.setValue(0)
-
-        self.vthread = VacThread(self, db)
-        self.vthread.check_done.connect(self.check_done,
-                type=Qt.QueuedConnection)
-        self.vthread.callback.connect(self.callback, type=Qt.QueuedConnection)
-        self.vthread.start()
-
-    def callback(self, progress, msg):
-        self.setLabelText(msg)
-        self.setValue(int(100*progress))
-
-    def check_done(self, bad, err):
-        if err:
-            error_dialog(self, _('Error'),
-                    _('Failed to check database integrity'),
-                    det_msg=err, show=True)
-        elif bad:
-            titles = [self.db.title(x, index_is_id=True) for x in bad]
-            det_msg = '\n'.join(titles)
-            warning_dialog(self, _('Some inconsistencies found'),
-                    _('The following books had formats or covers listed in the '
-                        'database that are not actually available. '
-                        'The entries for the formats/covers have been removed. '
-                        'You should check them manually. This can '
-                        'happen if you manipulate the files in the '
-                        'library folder directly.'), det_msg=det_msg, show=True)
-        else:
-            info_dialog(self, _('No errors found'),
-                    _('The integrity check completed with no uncorrectable errors found.'),
-                    show=True)
-        self.reset()
-
-# }}}
-
 class ChooseLibraryAction(InterfaceAction):
 
     name = 'Choose Library'
@@ -209,14 +139,6 @@ class ChooseLibraryAction(InterfaceAction):
                                       None, None), attr='action_check_library')
         ac.triggered.connect(self.check_library, type=Qt.QueuedConnection)
         self.maintenance_menu.addAction(ac)
-        ac = self.create_action(spec=(_('Check database integrity'), 'lt.png',
-                                      None, None), attr='action_check_database')
-        ac.triggered.connect(self.check_database, type=Qt.QueuedConnection)
-        self.maintenance_menu.addAction(ac)
-        ac = self.create_action(spec=(_('Recover database'), 'lt.png',
-                                    None, None), attr='action_restore_database')
-        ac.triggered.connect(self.restore_database, type=Qt.QueuedConnection)
-        self.maintenance_menu.addAction(ac)
         self.choose_menu.addMenu(self.maintenance_menu)
 
     def pick_random(self, *args):
@@ -346,28 +268,35 @@ class ChooseLibraryAction(InterfaceAction):
               'rate of approximately 1 book every three seconds.'), show=True)
 
     def check_library(self):
-        db = self.gui.library_view.model().db
+        self.gui.library_view.save_state()
-        d = CheckLibraryDialog(self.gui.parent(), db)
-        d.exec_()
-
-    def check_database(self, *args):
         m = self.gui.library_view.model()
         m.stop_metadata_backup()
-        try:
+        db = m.db
-            d = CheckIntegrity(m.db, self.gui)
+        db.prefs.disable_setting = True
-            d.exec_()
-        finally:
-            m.start_metadata_backup()
 
-    def restore_database(self):
+        d = DBCheck(self.gui, db)
-        info_dialog(self.gui, _('Recover database'), '<p>'+
+        d.start()
-            _(
+        try:
-              'This command rebuilds your calibre database from the information '
+            d.conn.close()
-              'stored by calibre in the OPF files.<p>'
+        except:
-              'This function is not currently available in the GUI. You can '
+            pass
-              'recover your database using the \'calibredb restore_database\' '
+        d.break_cycles()
-              'command line function.'
+        self.gui.library_moved(db.library_path, call_close=not
-              ), show=True)
+                d.closed_orig_conn)
+        if d.rejected:
+            return
+        if d.error is None:
+            if not question_dialog(self.gui, _('Success'),
+                    _('Found no errors in your calibre library database.'
+                        ' Do you want calibre to check if the files in your '
+                        ' library match the information in the database?')):
+                return
+        else:
+            return error_dialog(self.gui, _('Failed'),
+                    _('Database integrity check failed, click Show details'
+                        ' for details.'), show=True, det_msg=d.error[1])
+        d = CheckLibraryDialog(self.gui, m.db)
+        d.exec_()
 
     def switch_requested(self, location):
         if not self.change_library_allowed():

src/calibre/gui2/dialogs/check_library.py

@@ -3,16 +3,132 @@ __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __license__   = 'GPL v3'
 
-import os
+import os, shutil
 
 from PyQt4.Qt import QDialog, QVBoxLayout, QHBoxLayout, QTreeWidget, QLabel, \
             QPushButton, QDialogButtonBox, QApplication, QTreeWidgetItem, \
-            QLineEdit, Qt
+            QLineEdit, Qt, QProgressBar, QSize, QTimer
 
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.library.check_library import CheckLibrary, CHECKS
 from calibre.library.database2 import delete_file, delete_tree
-from calibre import prints
+from calibre import prints, as_unicode
+from calibre.ptempfile import PersistentTemporaryFile
+from calibre.library.sqlite import DBThread, OperationalError
+
+class DBCheck(QDialog):
+
+    def __init__(self, parent, db):
+        QDialog.__init__(self, parent)
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+        self.l1 = QLabel(_('Checking database integrity')+'...')
+        self.setWindowTitle(_('Checking database integrity'))
+        self.l.addWidget(self.l1)
+        self.pb = QProgressBar(self)
+        self.l.addWidget(self.pb)
+        self.pb.setMaximum(0)
+        self.pb.setMinimum(0)
+        self.msg = QLabel('')
+        self.l.addWidget(self.msg)
+        self.msg.setWordWrap(True)
+        self.bb = QDialogButtonBox(QDialogButtonBox.Cancel)
+        self.l.addWidget(self.bb)
+        self.bb.rejected.connect(self.reject)
+        self.resize(self.sizeHint() + QSize(100, 50))
+        self.error = None
+        self.db = db
+        self.closed_orig_conn = False
+
+    def start(self):
+        self.user_version = self.db.user_version
+        self.rejected = False
+        self.db.clean()
+        self.db.conn.close()
+        self.closed_orig_conn = True
+        t = DBThread(self.db.dbpath, False)
+        t.connect()
+        self.conn = t.conn
+        self.dump = self.conn.iterdump()
+        self.statements = []
+        self.count = 0
+        self.msg.setText(_('Dumping database to SQL'))
+        # Give the backup thread time to stop
+        QTimer.singleShot(2000, self.do_one_dump)
+        self.exec_()
+
+    def do_one_dump(self):
+        if self.rejected:
+            return
+        try:
+            try:
+                self.statements.append(self.dump.next())
+                self.count += 1
+            except StopIteration:
+                self.start_load()
+                return
+            QTimer.singleShot(0, self.do_one_dump)
+        except Exception, e:
+            import traceback
+            self.error = (as_unicode(e), traceback.format_exc())
+            self.reject()
+
+    def start_load(self):
+        self.conn.close()
+        self.pb.setMaximum(self.count)
+        self.pb.setValue(0)
+        self.msg.setText(_('Loading database from SQL'))
+        self.db.conn.close()
+        self.ndbpath = PersistentTemporaryFile('.db')
+        self.ndbpath.close()
+        self.ndbpath = self.ndbpath.name
+        t = DBThread(self.ndbpath, False)
+        t.connect()
+        self.conn = t.conn
+        self.conn.execute('create temporary table temp_sequence(id INTEGER PRIMARY KEY AUTOINCREMENT)')
+        self.conn.commit()
+
+        QTimer.singleShot(0, self.do_one_load)
+
+    def do_one_load(self):
+        if self.rejected:
+            return
+        if self.count > 0:
+            try:
+                try:
+                    self.conn.execute(self.statements.pop(0))
+                except OperationalError:
+                    if self.count > 1:
+                        # The last statement in the dump could be an extra
+                        # commit, so ignore it.
+                        raise
+                self.pb.setValue(self.pb.value() + 1)
+                self.count -= 1
+                QTimer.singleShot(0, self.do_one_load)
+            except Exception, e:
+                import traceback
+                self.error = (as_unicode(e), traceback.format_exc())
+                self.reject()
+
+        else:
+            self.replace_db()
+
+    def replace_db(self):
+        self.conn.commit()
+        self.conn.execute('pragma user_version=%d'%int(self.user_version))
+        self.conn.commit()
+        self.conn.close()
+        shutil.copyfile(self.ndbpath, self.db.dbpath)
+        self.db = None
+        self.accept()
+
+    def break_cycles(self):
+        self.statements = self.unpickler = self.db = self.conn = None
+
+    def reject(self):
+        self.rejected = True
+        QDialog.reject(self)
+
 
 class Item(QTreeWidgetItem):
     pass

src/calibre/gui2/ui.py

@@ -408,7 +408,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
     def booklists(self):
         return self.memory_view.model().db, self.card_a_view.model().db, self.card_b_view.model().db
 
-    def library_moved(self, newloc, copy_structure=False):
+    def library_moved(self, newloc, copy_structure=False, call_close=True):
         if newloc is None: return
         default_prefs = None
         try:
@@ -441,7 +441,8 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
         self.apply_named_search_restriction(db.prefs['gui_restriction'])
         if olddb is not None:
             try:
-                olddb.conn.close()
+                if call_close:
+                    olddb.conn.close()
             except:
                 import traceback
                 traceback.print_exc()

src/calibre/library/database2.py

@@ -2795,86 +2795,4 @@ books_series_link      feeds
         for id, title, script in self.conn.get('SELECT id,title,script FROM feeds'):
             yield id, title, script
 
-    def reconnect(self):
-        'Used to reconnect after calling self.conn.close()'
-        self.connect()
-        self.initialize_dynamic()
-        self.refresh()
 
-    def check_integrity(self, callback):
-        callback(0., _('Checking SQL integrity...'))
-        self.clean()
-        user_version = self.user_version
-        sql = '\n'.join(self.conn.dump())
-        self.conn.close()
-        dest = self.dbpath+'.tmp'
-        if os.path.exists(dest):
-            os.remove(dest)
-        conn = None
-        try:
-            ndb = DBThread(dest, None)
-            ndb.connect()
-            conn = ndb.conn
-            conn.execute('create table temp_sequence(id INTEGER PRIMARY KEY AUTOINCREMENT)')
-            conn.commit()
-            conn.executescript(sql)
-            conn.commit()
-            conn.execute('pragma user_version=%d'%user_version)
-            conn.commit()
-            conn.execute('drop table temp_sequence')
-            conn.commit()
-            conn.close()
-        except:
-            if conn is not None:
-                try:
-                    conn.close()
-                except:
-                    pass
-            if os.path.exists(dest):
-                os.remove(dest)
-            raise
-        else:
-            shutil.copyfile(dest, self.dbpath)
-            self.reconnect()
-        if os.path.exists(dest):
-            os.remove(dest)
-        callback(0.1, _('Checking for missing files.'))
-        bad = {}
-        us = self.data.universal_set()
-        total = float(len(us))
-        for i, id in enumerate(us):
-            formats = self.data.get(id, self.FIELD_MAP['formats'], row_is_id=True)
-            if not formats:
-                formats = []
-            else:
-                formats = [x.lower() for x in formats.split(',')]
-            actual_formats = self.formats(id, index_is_id=True)
-            if not actual_formats:
-                actual_formats = []
-            else:
-                actual_formats = [x.lower() for x in actual_formats.split(',')]
-
-            for fmt in formats:
-                if fmt in actual_formats:
-                    continue
-                if id not in bad:
-                    bad[id] = []
-                bad[id].append(fmt)
-            has_cover = self.data.get(id, self.FIELD_MAP['cover'],
-                    row_is_id=True)
-            if has_cover and self.cover(id, index_is_id=True, as_path=True) is None:
-                if id not in bad:
-                    bad[id] = []
-                bad[id].append('COVER')
-            callback(0.1+0.9*(1+i)/total, _('Checked id') + ' %d'%id)
-
-        for id in bad:
-            for fmt in bad[id]:
-                if fmt != 'COVER':
-                    self.conn.execute('DELETE FROM data WHERE book=? AND format=?', (id, fmt.upper()))
-                else:
-                    self.conn.execute('UPDATE books SET has_cover=0 WHERE id=?', (id,))
-        self.conn.commit()
-        self.refresh_ids(list(bad.keys()))
-
-        return bad

src/calibre/library/prefs.py

@@ -17,6 +17,7 @@ class DBPrefs(dict):
         dict.__init__(self)
         self.db = db
         self.defaults = {}
+        self.disable_setting = False
         for key, val in self.db.conn.get('SELECT key,val FROM preferences'):
             try:
                 val = self.raw_to_object(val)
@@ -45,6 +46,8 @@ class DBPrefs(dict):
         self.db.conn.commit()
 
     def __setitem__(self, key, val):
+        if self.disable_setting:
+            return
         raw = self.to_raw(val)
         self.db.conn.execute('DELETE FROM preferences WHERE key=?', (key,))
         self.db.conn.execute('INSERT INTO preferences (key,val) VALUES (?,?)', (key,