diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index ed1b53218b..2d6f88b405 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -1003,23 +1003,7 @@ class DB(object): self.close() try: - try: - atomic_rename(tmpdb, self.dbpath) - except: - import gc - for i in xrange(3): - gc.collect() - # Try the rename repeatedly in case something like a virus - # scanner has opened one of the files (I love windows) - for i in xrange(10): - time.sleep(1) - try: - atomic_rename(tmpdb, self.dbpath) - break - except: - if i > 8: - raise - + atomic_rename(tmpdb, self.dbpath) finally: self.reopen() diff --git a/src/calibre/debug.py b/src/calibre/debug.py index 904ee11be8..043d425c14 100644 --- a/src/calibre/debug.py +++ b/src/calibre/debug.py @@ -54,11 +54,7 @@ Everything after the -- is passed to the script. 'plugin code.') parser.add_option('--reinitialize-db', default=None, help='Re-initialize the sqlite calibre database at the ' - 'specified path. Useful to recover from db corruption.' - ' You can also specify the path to an SQL dump which ' - 'will be used instead of trying to dump the database.' - ' This can be useful when dumping fails, but dumping ' - 'with sqlite3 works.') + 'specified path. Useful to recover from db corruption.') parser.add_option('-p', '--py-console', help='Run python console', default=False, action='store_true') parser.add_option('-m', '--inspect-mobi', action='store_true', @@ -84,39 +80,42 @@ Everything after the -- is passed to the script. return parser -def reinit_db(dbpath, callback=None, sql_dump=None): - from calibre.db.backend import Connection - import apsw - import shutil - from io import StringIO +def reinit_db(dbpath): from contextlib import closing - if callback is None: - callback = lambda x, y: None - if not os.path.exists(dbpath): - raise ValueError(dbpath + ' does not exist') - - with closing(Connection(dbpath)) as conn: - uv = int(conn.get('PRAGMA user_version;', all=False)) - if sql_dump is None: - buf = StringIO() - shell = apsw.Shell(db=conn, stdout=buf) - shell.process_command('.dump') - sql = buf.getvalue() - else: - sql = open(sql_dump, 'rb').read().decode('utf-8') - - dest = dbpath + '.tmp' - callback(1, True) - try: - with closing(Connection(dest)) as conn: - conn.execute(sql) - conn.execute('PRAGMA user_version=%d;'%int(uv)) - os.remove(dbpath) - shutil.copyfile(dest, dbpath) - finally: - callback(1, False) - if os.path.exists(dest): - os.remove(dest) + from calibre import as_unicode + from calibre.ptempfile import TemporaryFile + from calibre.utils.filenames import atomic_rename + # We have to use sqlite3 instead of apsw as apsw has no way to discard + # problematic statements + import sqlite3 + from calibre.library.sqlite import do_connect + with TemporaryFile(suffix='_tmpdb.db', dir=os.path.dirname(dbpath)) as tmpdb: + with closing(do_connect(dbpath)) as src, closing(do_connect(tmpdb)) as dest: + dest.execute('create temporary table temp_sequence(id INTEGER PRIMARY KEY AUTOINCREMENT)') + dest.commit() + uv = int(src.execute('PRAGMA user_version;').fetchone()[0]) + dump = src.iterdump() + last_restore_error = None + while True: + try: + statement = next(dump) + except StopIteration: + break + except sqlite3.OperationalError as e: + prints('Failed to dump a line:', as_unicode(e)) + if last_restore_error: + prints('Failed to restore a line:', last_restore_error) + last_restore_error = None + try: + dest.execute(statement) + except sqlite3.OperationalError as e: + last_restore_error = as_unicode(e) + # The dump produces an extra commit at the end, so + # only print this error if there are more + # statements to be restored + dest.execute('PRAGMA user_version=%d;'%uv) + dest.commit() + atomic_rename(tmpdb, dbpath) prints('Database successfully re-initialized') def debug_device_driver(): @@ -238,10 +237,7 @@ def main(args=sys.argv): prints('CALIBRE_EXTENSIONS_PATH='+sys.extensions_location) prints('CALIBRE_PYTHON_PATH='+os.pathsep.join(sys.path)) elif opts.reinitialize_db is not None: - sql_dump = None - if len(args) > 1 and os.access(args[-1], os.R_OK): - sql_dump = args[-1] - reinit_db(opts.reinitialize_db, sql_dump=sql_dump) + reinit_db(opts.reinitialize_db) elif opts.inspect_mobi: for path in args[1:]: inspect_mobi(path) diff --git a/src/calibre/library/sqlite.py b/src/calibre/library/sqlite.py index 2e438ceb55..2ffc38b50f 100644 --- a/src/calibre/library/sqlite.py +++ b/src/calibre/library/sqlite.py @@ -206,6 +206,29 @@ def load_c_extensions(conn, debug=DEBUG): print e return False +def do_connect(path, row_factory=None): + conn = sqlite.connect(path, factory=Connection, + detect_types=sqlite.PARSE_DECLTYPES|sqlite.PARSE_COLNAMES) + conn.execute('pragma cache_size=5000') + encoding = conn.execute('pragma encoding').fetchone()[0] + conn.create_aggregate('sortconcat', 2, SortedConcatenate) + conn.create_aggregate('sortconcat_bar', 2, SortedConcatenateBar) + conn.create_aggregate('sortconcat_amper', 2, SortedConcatenateAmper) + conn.create_aggregate('identifiers_concat', 2, IdentifiersConcat) + load_c_extensions(conn) + conn.row_factory = sqlite.Row if row_factory else (lambda cursor, row : list(row)) + conn.create_aggregate('concat', 1, Concatenate) + conn.create_aggregate('aum_sortconcat', 4, AumSortedConcatenate) + conn.create_collation('PYNOCASE', partial(pynocase, + encoding=encoding)) + conn.create_function('title_sort', 1, title_sort) + conn.create_function('author_to_author_sort', 1, + _author_to_author_sort) + conn.create_function('uuid4', 0, lambda : str(uuid.uuid4())) + # Dummy functions for dynamically created filters + conn.create_function('books_list_filter', 1, lambda x: 1) + conn.create_collation('icucollate', icu_collator) + return conn class DBThread(Thread): @@ -222,27 +245,7 @@ class DBThread(Thread): self.conn = None def connect(self): - self.conn = sqlite.connect(self.path, factory=Connection, - detect_types=sqlite.PARSE_DECLTYPES|sqlite.PARSE_COLNAMES) - self.conn.execute('pragma cache_size=5000') - encoding = self.conn.execute('pragma encoding').fetchone()[0] - self.conn.create_aggregate('sortconcat', 2, SortedConcatenate) - self.conn.create_aggregate('sortconcat_bar', 2, SortedConcatenateBar) - self.conn.create_aggregate('sortconcat_amper', 2, SortedConcatenateAmper) - self.conn.create_aggregate('identifiers_concat', 2, IdentifiersConcat) - load_c_extensions(self.conn) - self.conn.row_factory = sqlite.Row if self.row_factory else lambda cursor, row : list(row) - self.conn.create_aggregate('concat', 1, Concatenate) - self.conn.create_aggregate('aum_sortconcat', 4, AumSortedConcatenate) - self.conn.create_collation('PYNOCASE', partial(pynocase, - encoding=encoding)) - self.conn.create_function('title_sort', 1, title_sort) - self.conn.create_function('author_to_author_sort', 1, - _author_to_author_sort) - self.conn.create_function('uuid4', 0, lambda : str(uuid.uuid4())) - # Dummy functions for dynamically created filters - self.conn.create_function('books_list_filter', 1, lambda x: 1) - self.conn.create_collation('icucollate', icu_collator) + self.conn = do_connect(self.path, self.row_factory) def run(self): try: diff --git a/src/calibre/utils/filenames.py b/src/calibre/utils/filenames.py index 30645e7380..a701c44947 100644 --- a/src/calibre/utils/filenames.py +++ b/src/calibre/utils/filenames.py @@ -406,6 +406,15 @@ def atomic_rename(oldpath, newpath): or may not exist. If it exists, it is replaced. ''' if iswindows: import win32file - win32file.MoveFileEx(oldpath, newpath, win32file.MOVEFILE_REPLACE_EXISTING|win32file.MOVEFILE_WRITE_THROUGH) + for i in xrange(10): + try: + win32file.MoveFileEx(oldpath, newpath, win32file.MOVEFILE_REPLACE_EXISTING|win32file.MOVEFILE_WRITE_THROUGH) + break + except: + if i > 8: + raise + # Try the rename repeatedly in case something like a virus + # scanner has opened one of the files (I love windows) + time.sleep(1) else: os.rename(oldpath, newpath)