Use pysqlite for db re-init as apsw cannot discard problem statements

Also move the atomic_rename retry logic into atomic_rename itself,
making it simpler.

src/calibre/db/backend.py

@@ -1003,23 +1003,7 @@ class DB(object):
 
             self.close()
             try:
-                try:
+                atomic_rename(tmpdb, self.dbpath)
-                    atomic_rename(tmpdb, self.dbpath)
-                except:
-                    import gc
-                    for i in xrange(3):
-                        gc.collect()
-                    # Try the rename repeatedly in case something like a virus
-                    # scanner has opened one of the files (I love windows)
-                    for i in xrange(10):
-                        time.sleep(1)
-                        try:
-                            atomic_rename(tmpdb, self.dbpath)
-                            break
-                        except:
-                            if i > 8:
-                                raise
-
             finally:
                 self.reopen()
 

src/calibre/debug.py

@@ -54,11 +54,7 @@ Everything after the -- is passed to the script.
             'plugin code.')
     parser.add_option('--reinitialize-db', default=None,
             help='Re-initialize the sqlite calibre database at the '
-            'specified path. Useful to recover from db corruption.'
+            'specified path. Useful to recover from db corruption.')
-            ' You can also specify the path to an SQL dump which '
-            'will be used instead of trying to dump the database.'
-            ' This can be useful when dumping fails, but dumping '
-            'with sqlite3 works.')
     parser.add_option('-p', '--py-console', help='Run python console',
             default=False, action='store_true')
     parser.add_option('-m', '--inspect-mobi', action='store_true',
@@ -84,39 +80,42 @@ Everything after the -- is passed to the script.
 
     return parser
 
-def reinit_db(dbpath, callback=None, sql_dump=None):
+def reinit_db(dbpath):
-    from calibre.db.backend import Connection
-    import apsw
-    import shutil
-    from io import StringIO
     from contextlib import closing
-    if callback is None:
+    from calibre import as_unicode
-        callback = lambda x, y: None
+    from calibre.ptempfile import TemporaryFile
-    if not os.path.exists(dbpath):
+    from calibre.utils.filenames import atomic_rename
-        raise ValueError(dbpath + ' does not exist')
+    # We have to use sqlite3 instead of apsw as apsw has no way to discard
-
+    # problematic statements
-    with closing(Connection(dbpath)) as conn:
+    import sqlite3
-        uv = int(conn.get('PRAGMA user_version;', all=False))
+    from calibre.library.sqlite import do_connect
-        if sql_dump is None:
+    with TemporaryFile(suffix='_tmpdb.db', dir=os.path.dirname(dbpath)) as tmpdb:
-            buf = StringIO()
+        with closing(do_connect(dbpath)) as src, closing(do_connect(tmpdb)) as dest:
-            shell = apsw.Shell(db=conn, stdout=buf)
+            dest.execute('create temporary table temp_sequence(id INTEGER PRIMARY KEY AUTOINCREMENT)')
-            shell.process_command('.dump')
+            dest.commit()
-            sql = buf.getvalue()
+            uv = int(src.execute('PRAGMA user_version;').fetchone()[0])
-        else:
+            dump = src.iterdump()
-            sql = open(sql_dump, 'rb').read().decode('utf-8')
+            last_restore_error = None
-
+            while True:
-    dest = dbpath + '.tmp'
+                try:
-    callback(1, True)
+                    statement = next(dump)
-    try:
+                except StopIteration:
-        with closing(Connection(dest)) as conn:
+                    break
-            conn.execute(sql)
+                except sqlite3.OperationalError as e:
-            conn.execute('PRAGMA user_version=%d;'%int(uv))
+                    prints('Failed to dump a line:', as_unicode(e))
-        os.remove(dbpath)
+                if last_restore_error:
-        shutil.copyfile(dest, dbpath)
+                    prints('Failed to restore a line:', last_restore_error)
-    finally:
+                    last_restore_error = None
-        callback(1, False)
+                try:
-        if os.path.exists(dest):
+                    dest.execute(statement)
-            os.remove(dest)
+                except sqlite3.OperationalError as e:
+                    last_restore_error = as_unicode(e)
+                    # The dump produces an extra commit at the end, so
+                    # only print this error if there are more
+                    # statements to be restored
+            dest.execute('PRAGMA user_version=%d;'%uv)
+            dest.commit()
+        atomic_rename(tmpdb, dbpath)
     prints('Database successfully re-initialized')
 
 def debug_device_driver():
@@ -238,10 +237,7 @@ def main(args=sys.argv):
         prints('CALIBRE_EXTENSIONS_PATH='+sys.extensions_location)
         prints('CALIBRE_PYTHON_PATH='+os.pathsep.join(sys.path))
     elif opts.reinitialize_db is not None:
-        sql_dump = None
+        reinit_db(opts.reinitialize_db)
-        if len(args) > 1 and os.access(args[-1], os.R_OK):
-            sql_dump = args[-1]
-        reinit_db(opts.reinitialize_db, sql_dump=sql_dump)
     elif opts.inspect_mobi:
         for path in args[1:]:
             inspect_mobi(path)

src/calibre/library/sqlite.py

@@ -206,6 +206,29 @@ def load_c_extensions(conn, debug=DEBUG):
             print e
     return False
 
+def do_connect(path, row_factory=None):
+    conn = sqlite.connect(path, factory=Connection,
+                                detect_types=sqlite.PARSE_DECLTYPES|sqlite.PARSE_COLNAMES)
+    conn.execute('pragma cache_size=5000')
+    encoding = conn.execute('pragma encoding').fetchone()[0]
+    conn.create_aggregate('sortconcat', 2, SortedConcatenate)
+    conn.create_aggregate('sortconcat_bar', 2, SortedConcatenateBar)
+    conn.create_aggregate('sortconcat_amper', 2, SortedConcatenateAmper)
+    conn.create_aggregate('identifiers_concat', 2, IdentifiersConcat)
+    load_c_extensions(conn)
+    conn.row_factory = sqlite.Row if row_factory else (lambda cursor, row : list(row))
+    conn.create_aggregate('concat', 1, Concatenate)
+    conn.create_aggregate('aum_sortconcat', 4, AumSortedConcatenate)
+    conn.create_collation('PYNOCASE', partial(pynocase,
+        encoding=encoding))
+    conn.create_function('title_sort', 1, title_sort)
+    conn.create_function('author_to_author_sort', 1,
+            _author_to_author_sort)
+    conn.create_function('uuid4', 0, lambda : str(uuid.uuid4()))
+    # Dummy functions for dynamically created filters
+    conn.create_function('books_list_filter', 1, lambda x: 1)
+    conn.create_collation('icucollate', icu_collator)
+    return conn
 
 class DBThread(Thread):
 
@@ -222,27 +245,7 @@ class DBThread(Thread):
         self.conn = None
 
     def connect(self):
-        self.conn = sqlite.connect(self.path, factory=Connection,
+        self.conn = do_connect(self.path, self.row_factory)
-                                   detect_types=sqlite.PARSE_DECLTYPES|sqlite.PARSE_COLNAMES)
-        self.conn.execute('pragma cache_size=5000')
-        encoding = self.conn.execute('pragma encoding').fetchone()[0]
-        self.conn.create_aggregate('sortconcat', 2, SortedConcatenate)
-        self.conn.create_aggregate('sortconcat_bar', 2, SortedConcatenateBar)
-        self.conn.create_aggregate('sortconcat_amper', 2, SortedConcatenateAmper)
-        self.conn.create_aggregate('identifiers_concat', 2, IdentifiersConcat)
-        load_c_extensions(self.conn)
-        self.conn.row_factory = sqlite.Row if self.row_factory else  lambda cursor, row : list(row)
-        self.conn.create_aggregate('concat', 1, Concatenate)
-        self.conn.create_aggregate('aum_sortconcat', 4, AumSortedConcatenate)
-        self.conn.create_collation('PYNOCASE', partial(pynocase,
-            encoding=encoding))
-        self.conn.create_function('title_sort', 1, title_sort)
-        self.conn.create_function('author_to_author_sort', 1,
-                _author_to_author_sort)
-        self.conn.create_function('uuid4', 0, lambda : str(uuid.uuid4()))
-        # Dummy functions for dynamically created filters
-        self.conn.create_function('books_list_filter', 1, lambda x: 1)
-        self.conn.create_collation('icucollate', icu_collator)
 
     def run(self):
         try:

src/calibre/utils/filenames.py

@@ -406,6 +406,15 @@ def atomic_rename(oldpath, newpath):
     or may not exist. If it exists, it is replaced. '''
     if iswindows:
         import win32file
-        win32file.MoveFileEx(oldpath, newpath, win32file.MOVEFILE_REPLACE_EXISTING|win32file.MOVEFILE_WRITE_THROUGH)
+        for i in xrange(10):
+            try:
+                win32file.MoveFileEx(oldpath, newpath, win32file.MOVEFILE_REPLACE_EXISTING|win32file.MOVEFILE_WRITE_THROUGH)
+                break
+            except:
+                if i > 8:
+                    raise
+                # Try the rename repeatedly in case something like a virus
+                # scanner has opened one of the files (I love windows)
+                time.sleep(1)
     else:
         os.rename(oldpath, newpath)