New API version of library check

src/calibre/db/__init__.py

@@ -132,7 +132,7 @@ def get_db_loader():
 
 '''
 Various things that require other things before they can be migrated:
-    1. Port library/restore.py, check_library.py
+    1. Port library/restore.py
     2. Check that content server reloading on metadata,db change, metadata
     backup, refresh gui on calibredb add and moving libraries all work (check
     them on windows as well for file locking issues)

src/calibre/db/backend.py

@@ -16,7 +16,7 @@ import apsw
 from calibre import isbytestring, force_unicode, prints
 from calibre.constants import (iswindows, filesystem_encoding,
         preferred_encoding)
-from calibre.ptempfile import PersistentTemporaryFile
+from calibre.ptempfile import PersistentTemporaryFile, TemporaryFile
 from calibre.db import SPOOL_SIZE
 from calibre.db.schema_upgrades import SchemaUpgrade
 from calibre.db.errors import NoSuchFormat
@@ -25,8 +25,8 @@ from calibre.ebooks.metadata import title_sort, author_to_author_sort
 from calibre.utils.icu import sort_key
 from calibre.utils.config import to_json, from_json, prefs, tweaks
 from calibre.utils.date import utcfromtimestamp, parse_date
-from calibre.utils.filenames import (is_case_sensitive, samefile, hardlink_file, ascii_filename,
+from calibre.utils.filenames import (
-                                     WindowsAtomicFolderMove)
+    is_case_sensitive, samefile, hardlink_file, ascii_filename, WindowsAtomicFolderMove, atomic_rename)
 from calibre.utils.magick.draw import save_cover_data_to
 from calibre.utils.recycle_bin import delete_tree, delete_file
 from calibre.utils.formatter_functions import load_user_template_functions
@@ -967,10 +967,41 @@ class DB(object):
         self.conn.execute('UPDATE custom_columns SET mark_for_delete=1 WHERE id=?', (data['num'],))
 
     def close(self):
-        if self._conn is not None:
+        if getattr(self, '_conn', None) is not None:
             self._conn.close()
             del self._conn
 
+    def reopen(self):
+        self.close()
+        self._conn = None
+        self.conn
+
+    def dump_and_restore(self, callback=None, sql=None):
+        from io import StringIO
+        from contextlib import closing
+        if callback is None:
+            callback = lambda x: x
+        uv = int(self.user_version)
+
+        if sql is None:
+            callback(_('Dumping database to SQL') + '...')
+            buf = StringIO()
+            shell = apsw.Shell(db=self.conn, stdout=buf)
+            shell.process_command('.dump')
+            sql = buf.getvalue()
+
+        with TemporaryFile(suffix='_tmpdb.db', dir=os.path.dirname(self.dbpath)) as tmpdb:
+            callback(_('Restoring database from SQL') + '...')
+            with closing(Connection(tmpdb)) as conn:
+                conn.execute(sql)
+                conn.execute('PRAGMA user_version=%d;'%uv)
+
+            self.close()
+            try:
+                atomic_rename(tmpdb, self.dbpath)
+            finally:
+                self.reopen()
+
     @dynamic_property
     def user_version(self):
         doc = 'The user version of this database'

src/calibre/db/cache.py

@@ -1564,6 +1564,10 @@ class Cache(object):
     def change_search_locations(self, newlocs):
         self._search_api.change_locations(newlocs)
 
+    @write_api
+    def dump_and_restore(self, callback=None, sql=None):
+        return self.backend.dump_and_restore(callback=callback, sql=sql)
+
     # }}}
 
 class SortKey(object):  # {{{

src/calibre/gui2/actions/choose_library.py

@@ -413,14 +413,17 @@ class ChooseLibraryAction(InterfaceAction):
             self.gui.library_moved(db.library_path, call_close=False)
 
     def check_library(self):
-        from calibre.gui2.dialogs.check_library import CheckLibraryDialog, DBCheck
+        from calibre.gui2.dialogs.check_library import CheckLibraryDialog, DBCheck, DBCheckNew
         self.gui.library_view.save_state()
         m = self.gui.library_view.model()
         m.stop_metadata_backup()
         db = m.db
         db.prefs.disable_setting = True
 
-        d = DBCheck(self.gui, db)
+        if hasattr(db, 'new_api'):
+            d = DBCheckNew(self.gui, db)
+        else:
+            d = DBCheck(self.gui, db)
         d.start()
         try:
             d.conn.close()

src/calibre/gui2/dialogs/check_library.py

@@ -4,11 +4,12 @@ __docformat__ = 'restructuredtext en'
 __license__   = 'GPL v3'
 
 import os, shutil
+from threading import Thread
 
 from PyQt4.Qt import (QDialog, QVBoxLayout, QHBoxLayout, QTreeWidget, QLabel,
             QPushButton, QDialogButtonBox, QApplication, QTreeWidgetItem,
             QLineEdit, Qt, QProgressBar, QSize, QTimer, QIcon, QTextEdit,
-            QSplitter, QWidget)
+            QSplitter, QWidget, pyqtSignal)
 
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.library.check_library import CheckLibrary, CHECKS
@@ -17,6 +18,62 @@ from calibre import prints, as_unicode
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.library.sqlite import DBThread, OperationalError
 
+class DBCheckNew(QDialog):  # {{{
+
+    update_msg = pyqtSignal(object)
+
+    def __init__(self, parent, db):
+        QDialog.__init__(self, parent)
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+        self.l1 = QLabel(_('Checking database integrity') + ' ' +
+                         _('This will take a while, please wait...'))
+        self.setWindowTitle(_('Checking database integrity'))
+        self.l1.setWordWrap(True)
+        self.l.addWidget(self.l1)
+        self.msg = QLabel('')
+        self.update_msg.connect(self.msg.setText, type=Qt.QueuedConnection)
+        self.l.addWidget(self.msg)
+        self.msg.setWordWrap(True)
+        self.bb = QDialogButtonBox(QDialogButtonBox.Cancel)
+        self.l.addWidget(self.bb)
+        self.bb.rejected.connect(self.reject)
+        self.resize(self.sizeHint() + QSize(100, 50))
+        self.error = None
+        self.db = db.new_api
+        self.closed_orig_conn = False
+        self.rejected = False
+
+    def start(self):
+        t = self.thread = Thread(target=self.dump_and_restore)
+        t.daemon = True
+        t.start()
+        QTimer.singleShot(100, self.check)
+        self.exec_()
+
+    def dump_and_restore(self):
+        try:
+            self.db.dump_and_restore(self.update_msg.emit)
+        except Exception as e:
+            import traceback
+            self.error = (as_unicode(e), traceback.format_exc())
+
+    def reject(self):
+        self.rejected = True
+        return QDialog.reject(self)
+
+    def check(self):
+        if self.rejected:
+            return
+        if self.thread.is_alive():
+            QTimer.singleShot(100, self.check)
+        else:
+            self.accept()
+
+    def break_cycles(self):
+        self.db = self.thread = None
+
+# }}}
 
 class DBCheck(QDialog):  # {{{
 

src/calibre/utils/filenames.py

@@ -383,4 +383,12 @@ def hardlink_file(src, dest):
         return
     os.link(src, dest)
 
-
+def atomic_rename(oldpath, newpath):
+    '''Replace the file newpath with the file oldpath. Can fail if the files
+    are on different volumes. If succeeds, guaranteed to be atomic. newpath may
+    or may not exist. If it exists, it is replaced. '''
+    if iswindows:
+        import win32file
+        win32file.MoveFileEx(oldpath, newpath, win32file.MOVEFILE_REPLACE_EXISTING|win32file.MOVEFILE_WRITE_THROUGH)
+    else:
+        os.rename(oldpath, newpath)