From 39425a15a33f3bdbdee15494b36bb41d0d9cf296 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 20 Jul 2013 15:46:34 +0530 Subject: [PATCH] New API version of library check --- src/calibre/db/__init__.py | 2 +- src/calibre/db/backend.py | 39 ++++++++++++-- src/calibre/db/cache.py | 4 ++ src/calibre/gui2/actions/choose_library.py | 7 ++- src/calibre/gui2/dialogs/check_library.py | 59 +++++++++++++++++++++- src/calibre/utils/filenames.py | 10 +++- 6 files changed, 112 insertions(+), 9 deletions(-) diff --git a/src/calibre/db/__init__.py b/src/calibre/db/__init__.py index 0628e7b51b..1fe7da1e04 100644 --- a/src/calibre/db/__init__.py +++ b/src/calibre/db/__init__.py @@ -132,7 +132,7 @@ def get_db_loader(): ''' Various things that require other things before they can be migrated: - 1. Port library/restore.py, check_library.py + 1. Port library/restore.py 2. Check that content server reloading on metadata,db change, metadata backup, refresh gui on calibredb add and moving libraries all work (check them on windows as well for file locking issues) diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index 914248fedc..f203f5ed0c 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -16,7 +16,7 @@ import apsw from calibre import isbytestring, force_unicode, prints from calibre.constants import (iswindows, filesystem_encoding, preferred_encoding) -from calibre.ptempfile import PersistentTemporaryFile +from calibre.ptempfile import PersistentTemporaryFile, TemporaryFile from calibre.db import SPOOL_SIZE from calibre.db.schema_upgrades import SchemaUpgrade from calibre.db.errors import NoSuchFormat @@ -25,8 +25,8 @@ from calibre.ebooks.metadata import title_sort, author_to_author_sort from calibre.utils.icu import sort_key from calibre.utils.config import to_json, from_json, prefs, tweaks from calibre.utils.date import utcfromtimestamp, parse_date -from calibre.utils.filenames import (is_case_sensitive, samefile, hardlink_file, ascii_filename, - WindowsAtomicFolderMove) +from calibre.utils.filenames import ( + is_case_sensitive, samefile, hardlink_file, ascii_filename, WindowsAtomicFolderMove, atomic_rename) from calibre.utils.magick.draw import save_cover_data_to from calibre.utils.recycle_bin import delete_tree, delete_file from calibre.utils.formatter_functions import load_user_template_functions @@ -967,10 +967,41 @@ class DB(object): self.conn.execute('UPDATE custom_columns SET mark_for_delete=1 WHERE id=?', (data['num'],)) def close(self): - if self._conn is not None: + if getattr(self, '_conn', None) is not None: self._conn.close() del self._conn + def reopen(self): + self.close() + self._conn = None + self.conn + + def dump_and_restore(self, callback=None, sql=None): + from io import StringIO + from contextlib import closing + if callback is None: + callback = lambda x: x + uv = int(self.user_version) + + if sql is None: + callback(_('Dumping database to SQL') + '...') + buf = StringIO() + shell = apsw.Shell(db=self.conn, stdout=buf) + shell.process_command('.dump') + sql = buf.getvalue() + + with TemporaryFile(suffix='_tmpdb.db', dir=os.path.dirname(self.dbpath)) as tmpdb: + callback(_('Restoring database from SQL') + '...') + with closing(Connection(tmpdb)) as conn: + conn.execute(sql) + conn.execute('PRAGMA user_version=%d;'%uv) + + self.close() + try: + atomic_rename(tmpdb, self.dbpath) + finally: + self.reopen() + @dynamic_property def user_version(self): doc = 'The user version of this database' diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 9a19f4bb73..37d0428046 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -1564,6 +1564,10 @@ class Cache(object): def change_search_locations(self, newlocs): self._search_api.change_locations(newlocs) + @write_api + def dump_and_restore(self, callback=None, sql=None): + return self.backend.dump_and_restore(callback=callback, sql=sql) + # }}} class SortKey(object): # {{{ diff --git a/src/calibre/gui2/actions/choose_library.py b/src/calibre/gui2/actions/choose_library.py index 6857454ae4..f85eb09f37 100644 --- a/src/calibre/gui2/actions/choose_library.py +++ b/src/calibre/gui2/actions/choose_library.py @@ -413,14 +413,17 @@ class ChooseLibraryAction(InterfaceAction): self.gui.library_moved(db.library_path, call_close=False) def check_library(self): - from calibre.gui2.dialogs.check_library import CheckLibraryDialog, DBCheck + from calibre.gui2.dialogs.check_library import CheckLibraryDialog, DBCheck, DBCheckNew self.gui.library_view.save_state() m = self.gui.library_view.model() m.stop_metadata_backup() db = m.db db.prefs.disable_setting = True - d = DBCheck(self.gui, db) + if hasattr(db, 'new_api'): + d = DBCheckNew(self.gui, db) + else: + d = DBCheck(self.gui, db) d.start() try: d.conn.close() diff --git a/src/calibre/gui2/dialogs/check_library.py b/src/calibre/gui2/dialogs/check_library.py index 2769a422ac..c1f8ed4f18 100644 --- a/src/calibre/gui2/dialogs/check_library.py +++ b/src/calibre/gui2/dialogs/check_library.py @@ -4,11 +4,12 @@ __docformat__ = 'restructuredtext en' __license__ = 'GPL v3' import os, shutil +from threading import Thread from PyQt4.Qt import (QDialog, QVBoxLayout, QHBoxLayout, QTreeWidget, QLabel, QPushButton, QDialogButtonBox, QApplication, QTreeWidgetItem, QLineEdit, Qt, QProgressBar, QSize, QTimer, QIcon, QTextEdit, - QSplitter, QWidget) + QSplitter, QWidget, pyqtSignal) from calibre.gui2.dialogs.confirm_delete import confirm from calibre.library.check_library import CheckLibrary, CHECKS @@ -17,6 +18,62 @@ from calibre import prints, as_unicode from calibre.ptempfile import PersistentTemporaryFile from calibre.library.sqlite import DBThread, OperationalError +class DBCheckNew(QDialog): # {{{ + + update_msg = pyqtSignal(object) + + def __init__(self, parent, db): + QDialog.__init__(self, parent) + self.l = QVBoxLayout() + self.setLayout(self.l) + self.l1 = QLabel(_('Checking database integrity') + ' ' + + _('This will take a while, please wait...')) + self.setWindowTitle(_('Checking database integrity')) + self.l1.setWordWrap(True) + self.l.addWidget(self.l1) + self.msg = QLabel('') + self.update_msg.connect(self.msg.setText, type=Qt.QueuedConnection) + self.l.addWidget(self.msg) + self.msg.setWordWrap(True) + self.bb = QDialogButtonBox(QDialogButtonBox.Cancel) + self.l.addWidget(self.bb) + self.bb.rejected.connect(self.reject) + self.resize(self.sizeHint() + QSize(100, 50)) + self.error = None + self.db = db.new_api + self.closed_orig_conn = False + self.rejected = False + + def start(self): + t = self.thread = Thread(target=self.dump_and_restore) + t.daemon = True + t.start() + QTimer.singleShot(100, self.check) + self.exec_() + + def dump_and_restore(self): + try: + self.db.dump_and_restore(self.update_msg.emit) + except Exception as e: + import traceback + self.error = (as_unicode(e), traceback.format_exc()) + + def reject(self): + self.rejected = True + return QDialog.reject(self) + + def check(self): + if self.rejected: + return + if self.thread.is_alive(): + QTimer.singleShot(100, self.check) + else: + self.accept() + + def break_cycles(self): + self.db = self.thread = None + +# }}} class DBCheck(QDialog): # {{{ diff --git a/src/calibre/utils/filenames.py b/src/calibre/utils/filenames.py index d756978040..23ac8fd43a 100644 --- a/src/calibre/utils/filenames.py +++ b/src/calibre/utils/filenames.py @@ -383,4 +383,12 @@ def hardlink_file(src, dest): return os.link(src, dest) - +def atomic_rename(oldpath, newpath): + '''Replace the file newpath with the file oldpath. Can fail if the files + are on different volumes. If succeeds, guaranteed to be atomic. newpath may + or may not exist. If it exists, it is replaced. ''' + if iswindows: + import win32file + win32file.MoveFileEx(oldpath, newpath, win32file.MOVEFILE_REPLACE_EXISTING|win32file.MOVEFILE_WRITE_THROUGH) + else: + os.rename(oldpath, newpath)