Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-06-23 15:30:45 -04:00
Code Issues Packages Projects Releases Wiki Activity

Use pysqlite for db re-init as apsw cannot discard problem statements

Browse Source 
Also move the atomic_rename retry logic into atomic_rename itself,
making it simpler.
This commit is contained in:
Kovid Goyal 2013-08-24 13:08:21 +05:30
parent 403865c914
commit e90ba09426
4 changed files with 72 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/calibre/db/backend.py
View File

@ -1003,23 +1003,7 @@ class DB(object):
self.close()
try:
try:
atomic_rename(tmpdb, self.dbpath)
except:
import gc
for i in xrange(3):
gc.collect()
# Try the rename repeatedly in case something like a virus
# scanner has opened one of the files (I love windows)
for i in xrange(10):
time.sleep(1)
try:
atomic_rename(tmpdb, self.dbpath)
break
except:
if i > 8:
raise
atomic_rename(tmpdb, self.dbpath)
finally:
self.reopen()

78
src/calibre/debug.py
View File

@ -54,11 +54,7 @@ Everything after the -- is passed to the script.
'plugin code.')
parser.add_option('--reinitialize-db', default=None,
help='Re-initialize the sqlite calibre database at the '
'specified path. Useful to recover from db corruption.'
' You can also specify the path to an SQL dump which '
'will be used instead of trying to dump the database.'
' This can be useful when dumping fails, but dumping '
'with sqlite3 works.')
'specified path. Useful to recover from db corruption.')
parser.add_option('-p', '--py-console', help='Run python console',
default=False, action='store_true')
parser.add_option('-m', '--inspect-mobi', action='store_true',
@ -84,39 +80,42 @@ Everything after the -- is passed to the script.
return parser
def reinit_db(dbpath, callback=None, sql_dump=None):
from calibre.db.backend import Connection
import apsw
import shutil
from io import StringIO
def reinit_db(dbpath):
from contextlib import closing
if callback is None:
callback = lambda x, y: None
if not os.path.exists(dbpath):
raise ValueError(dbpath + ' does not exist')
with closing(Connection(dbpath)) as conn:
uv = int(conn.get('PRAGMA user_version;', all=False))
if sql_dump is None:
buf = StringIO()
shell = apsw.Shell(db=conn, stdout=buf)
shell.process_command('.dump')
sql = buf.getvalue()
else:
sql = open(sql_dump, 'rb').read().decode('utf-8')
dest = dbpath + '.tmp'
callback(1, True)
try:
with closing(Connection(dest)) as conn:
conn.execute(sql)
conn.execute('PRAGMA user_version=%d;'%int(uv))
os.remove(dbpath)
shutil.copyfile(dest, dbpath)
finally:
callback(1, False)
if os.path.exists(dest):
os.remove(dest)
from calibre import as_unicode
from calibre.ptempfile import TemporaryFile
from calibre.utils.filenames import atomic_rename
# We have to use sqlite3 instead of apsw as apsw has no way to discard
# problematic statements
import sqlite3
from calibre.library.sqlite import do_connect
with TemporaryFile(suffix='_tmpdb.db', dir=os.path.dirname(dbpath)) as tmpdb:
with closing(do_connect(dbpath)) as src, closing(do_connect(tmpdb)) as dest:
dest.execute('create temporary table temp_sequence(id INTEGER PRIMARY KEY AUTOINCREMENT)')
dest.commit()
uv = int(src.execute('PRAGMA user_version;').fetchone()[0])
dump = src.iterdump()
last_restore_error = None
while True:
try:
statement = next(dump)
except StopIteration:
break
except sqlite3.OperationalError as e:
prints('Failed to dump a line:', as_unicode(e))
if last_restore_error:
prints('Failed to restore a line:', last_restore_error)
last_restore_error = None
try:
dest.execute(statement)
except sqlite3.OperationalError as e:
last_restore_error = as_unicode(e)
# The dump produces an extra commit at the end, so
# only print this error if there are more
# statements to be restored
dest.execute('PRAGMA user_version=%d;'%uv)
dest.commit()
atomic_rename(tmpdb, dbpath)
prints('Database successfully re-initialized')
def debug_device_driver():
@ -238,10 +237,7 @@ def main(args=sys.argv):
prints('CALIBRE_EXTENSIONS_PATH='+sys.extensions_location)
prints('CALIBRE_PYTHON_PATH='+os.pathsep.join(sys.path))
elif opts.reinitialize_db is not None:
sql_dump = None
if len(args) > 1 and os.access(args[-1], os.R_OK):
sql_dump = args[-1]
reinit_db(opts.reinitialize_db, sql_dump=sql_dump)
reinit_db(opts.reinitialize_db)
elif opts.inspect_mobi:
for path in args[1:]:
inspect_mobi(path)

45
src/calibre/library/sqlite.py
View File

@ -206,6 +206,29 @@ def load_c_extensions(conn, debug=DEBUG):
print e
return False
def do_connect(path, row_factory=None):
conn = sqlite.connect(path, factory=Connection,
detect_types=sqlite.PARSE_DECLTYPES|sqlite.PARSE_COLNAMES)
conn.execute('pragma cache_size=5000')
encoding = conn.execute('pragma encoding').fetchone()[0]
conn.create_aggregate('sortconcat', 2, SortedConcatenate)
conn.create_aggregate('sortconcat_bar', 2, SortedConcatenateBar)
conn.create_aggregate('sortconcat_amper', 2, SortedConcatenateAmper)
conn.create_aggregate('identifiers_concat', 2, IdentifiersConcat)
load_c_extensions(conn)
conn.row_factory = sqlite.Row if row_factory else (lambda cursor, row : list(row))
conn.create_aggregate('concat', 1, Concatenate)
conn.create_aggregate('aum_sortconcat', 4, AumSortedConcatenate)
conn.create_collation('PYNOCASE', partial(pynocase,
encoding=encoding))
conn.create_function('title_sort', 1, title_sort)
conn.create_function('author_to_author_sort', 1,
_author_to_author_sort)
conn.create_function('uuid4', 0, lambda : str(uuid.uuid4()))
# Dummy functions for dynamically created filters
conn.create_function('books_list_filter', 1, lambda x: 1)
conn.create_collation('icucollate', icu_collator)
return conn
class DBThread(Thread):
@ -222,27 +245,7 @@ class DBThread(Thread):
self.conn = None
def connect(self):
self.conn = sqlite.connect(self.path, factory=Connection,
detect_types=sqlite.PARSE_DECLTYPES|sqlite.PARSE_COLNAMES)
self.conn.execute('pragma cache_size=5000')
encoding = self.conn.execute('pragma encoding').fetchone()[0]
self.conn.create_aggregate('sortconcat', 2, SortedConcatenate)
self.conn.create_aggregate('sortconcat_bar', 2, SortedConcatenateBar)
self.conn.create_aggregate('sortconcat_amper', 2, SortedConcatenateAmper)
self.conn.create_aggregate('identifiers_concat', 2, IdentifiersConcat)
load_c_extensions(self.conn)
self.conn.row_factory = sqlite.Row if self.row_factory else lambda cursor, row : list(row)
self.conn.create_aggregate('concat', 1, Concatenate)
self.conn.create_aggregate('aum_sortconcat', 4, AumSortedConcatenate)
self.conn.create_collation('PYNOCASE', partial(pynocase,
encoding=encoding))
self.conn.create_function('title_sort', 1, title_sort)
self.conn.create_function('author_to_author_sort', 1,
_author_to_author_sort)
self.conn.create_function('uuid4', 0, lambda : str(uuid.uuid4()))
# Dummy functions for dynamically created filters
self.conn.create_function('books_list_filter', 1, lambda x: 1)
self.conn.create_collation('icucollate', icu_collator)
self.conn = do_connect(self.path, self.row_factory)
def run(self):
try:

11
src/calibre/utils/filenames.py
View File

@ -406,6 +406,15 @@ def atomic_rename(oldpath, newpath):
or may not exist. If it exists, it is replaced. '''
if iswindows:
import win32file
win32file.MoveFileEx(oldpath, newpath, win32file.MOVEFILE_REPLACE_EXISTING|win32file.MOVEFILE_WRITE_THROUGH)
for i in xrange(10):
try:
win32file.MoveFileEx(oldpath, newpath, win32file.MOVEFILE_REPLACE_EXISTING|win32file.MOVEFILE_WRITE_THROUGH)
break
except:
if i > 8:
raise
# Try the rename repeatedly in case something like a virus
# scanner has opened one of the files (I love windows)
time.sleep(1)
else:
os.rename(oldpath, newpath)