From 48f8bf55d880592cf4eb75a4eae3aef3a798650a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 8 Mar 2025 13:27:58 +0530 Subject: [PATCH] Kobo driver: Use sqlite savepoints for greater robustness Also, if sqlite reports an IOError when reading from the database, copy the db file into /tmp and operate on it there. Should allow for working with the device when its FS is mounted using a filesystem sqlite doesnt support. --- src/calibre/devices/kobo/db.py | 73 ++++++++++++++++++++++++++++++ src/calibre/devices/kobo/driver.py | 70 ++++++++-------------------- 2 files changed, 93 insertions(+), 50 deletions(-) create mode 100644 src/calibre/devices/kobo/db.py diff --git a/src/calibre/devices/kobo/db.py b/src/calibre/devices/kobo/db.py new file mode 100644 index 0000000000..d73882e6fc --- /dev/null +++ b/src/calibre/devices/kobo/db.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +# License: GPLv3 Copyright: 2025, Kovid Goyal + +import os +import shutil +from contextlib import closing, suppress + +import apsw + +from calibre.prints import debug_print +from calibre.ptempfile import PersistentTemporaryFile + + +def row_factory(cursor, row): + return {k[0]: row[i] for i, k in enumerate(cursor.getdescription())} + + +class Database: + + def __init__(self, path_on_device: str): + self.path_on_device = self.dbpath = path_on_device + self.dbversion = 0 + def connect(path: str = path_on_device) -> None: + with closing(apsw.Connection(path)) as conn: + conn.setrowtrace(row_factory) + cursor = conn.cursor() + cursor.execute('SELECT version FROM dbversion') + with suppress(StopIteration): + result = next(cursor) + self.dbversion = result['version'] + debug_print('Database Version: ', self.dbversion) + self.dbpath = path + self.needs_copy = True + self.use_row_factory = True + try: + connect() + self.needs_copy = False + except apsw.IOError: + debug_print(f'Kobo: I/O error connecting to {self.device_database_path} copying it into temporary storage and connecting there') + with open(self.path_on_device, 'rb') as src, PersistentTemporaryFile(suffix='-kobo-db.sqlite') as dest: + shutil.copyfileobj(src, dest) + try: + connect(dest.name) + except Exception: + os.remove(dest.name) + raise + + def __enter__(self) -> apsw.Connection: + self.conn = apsw.Connection(self.dbpath) + if self.use_row_factory: + self.conn.setrowtrace(row_factory) + return self.conn.__enter__() + + def __exit__(self, exc_type, exc_value, tb) -> bool | None: + suppress_exception = self.conn.__exit__(exc_type, exc_value, tb) + if self.needs_copy and (suppress_exception or (exc_type is None and exc_value is None and tb is None)): + self.copy_db() + return suppress_exception + + def copy_db(self): + self.conn.cache_flush() + with PersistentTemporaryFile() as f: + needs_remove = True + try: + with closing(apsw.Connection(f.name)) as dest, self.conn.backup('main', dest, 'main') as b: + while not b.done: + b.step() + shutil.move(f.name, self.path_on_device) + needs_remove = False + finally: + if needs_remove: + with suppress(OSError): + os.remove(f.name) diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index 59a4f5de58..b52b1733f4 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -17,7 +17,7 @@ import os import re import shutil import time -from contextlib import closing, suppress +from contextlib import suppress from datetime import datetime from calibre import fsync, prints, strftime @@ -73,7 +73,6 @@ def any_in(haystack, *needles): class DummyCSSPreProcessor: def __call__(self, data, add_namespace=False): - return data @@ -180,32 +179,6 @@ class KOBO(USBMS): self._device_version_info = None super().eject() - def device_database_path(self): - return os.path.join(self._main_prefix, KOBO_ROOT_DIR_NAME, 'KoboReader.sqlite') - - def device_database_connection(self, use_row_factory=False): - import apsw - db_connection = apsw.Connection(self.device_database_path()) - - if use_row_factory: - db_connection.setrowtrace(self.row_factory) - - return db_connection - - def row_factory(self, cursor, row): - return {k[0]: row[i] for i, k in enumerate(cursor.getdescription())} - - def get_database_version(self, connection): - cursor = connection.cursor() - cursor.execute('SELECT version FROM dbversion') - try: - result = next(cursor) - dbversion = result['version'] - except StopIteration: - dbversion = 0 - - return dbversion - def device_version_info(self): debug_print('device_version_info - start') if not self._device_version_info: @@ -375,11 +348,7 @@ class KOBO(USBMS): traceback.print_exc() return changed - with closing(self.device_database_connection(use_row_factory=True)) as connection: - - self.dbversion = self.get_database_version(connection) - debug_print('Database Version: ', self.dbversion) - + with self.database_transaction(use_row_factory=True) as connection: cursor = connection.cursor() opts = self.settings() if self.dbversion >= 33: @@ -485,7 +454,7 @@ class KOBO(USBMS): # 2) content debug_print('delete_via_sql: ContentID: ', ContentID, 'ContentType: ', ContentType) - with closing(self.device_database_connection()) as connection: + with self.database_transaction() as connection: cursor = connection.cursor() t = (ContentID,) @@ -946,7 +915,7 @@ class KOBO(USBMS): # the last book from the collection the list of books is empty # and the removal of the last book would not occur - with closing(self.device_database_connection()) as connection: + with self.database_transaction() as connection: if collections: @@ -1083,7 +1052,7 @@ class KOBO(USBMS): ContentType = self.get_content_type_from_extension(extension) if extension != '' else self.get_content_type_from_path(filepath) ContentID = self.contentid_from_path(filepath, ContentType) - with closing(self.device_database_connection()) as connection: + with self.database_transaction() as connection: cursor = connection.cursor() t = (ContentID,) @@ -1252,7 +1221,7 @@ class KOBO(USBMS): path_map, book_ext = resolve_bookmark_paths(storage, path_map) bookmarked_books = {} - with closing(self.device_database_connection(use_row_factory=True)) as connection: + with self.database_transaction(use_row_factory=True) as connection: for book_id in path_map: extension = os.path.splitext(path_map[book_id])[1] ContentType = self.get_content_type_from_extension(extension) if extension else self.get_content_type_from_path(path_map[book_id]) @@ -1632,6 +1601,7 @@ class KOBOTOUCH(KOBO): return super().get_device_information(end_session) def post_open_callback(self): + from calibre.devices.kobo.db import Database # delete empty directories in root they get left behind when deleting # books on device. for prefix in (self._main_prefix, self._card_a_prefix, self._card_b_prefix): @@ -1641,10 +1611,16 @@ class KOBOTOUCH(KOBO): if not de.name.startswith('.') and de.is_dir(): with suppress(OSError): os.rmdir(de.path) + self.device_database_path = os.path.join(self._main_prefix, KOBO_ROOT_DIR_NAME, 'KoboReader.sqlite') + self.db_manager = Database(self.device_database_path) + self.dbversion = self.db_manager.dbversion + + def database_transaction(self, use_row_factory=False): + self.db_manager.use_row_factory = use_row_factory + return self.db_manager def open_linux(self): super().open_linux() - self.swap_drives_if_needed() def open_osx(self): @@ -1652,7 +1628,7 @@ class KOBOTOUCH(KOBO): super().open_osx() # Wrap some debugging output in a try/except so that it is unlikely to break things completely. - try: + with suppress(Exception): if DEBUG: from calibre_extensions.usbobserver import get_mounted_filesystems mount_map = get_mounted_filesystems() @@ -1660,9 +1636,6 @@ class KOBOTOUCH(KOBO): debug_print('KoboTouch::open_osx - self._main_prefix=', self._main_prefix) debug_print('KoboTouch::open_osx - self._card_a_prefix=', self._card_a_prefix) debug_print('KoboTouch::open_osx - self._card_b_prefix=', self._card_b_prefix) - except: - pass - self.swap_drives_if_needed() def swap_drives_if_needed(self): @@ -2017,12 +1990,9 @@ class KOBOTOUCH(KOBO): self.debug_index = 0 - with closing(self.device_database_connection(use_row_factory=True)) as connection: + with self.database_transaction(use_row_factory=True) as connection: debug_print('KoboTouch:books - reading device database') - self.dbversion = self.get_database_version(connection) - debug_print('Database Version: ', self.dbversion) - self.bookshelvelist = self.get_bookshelflist(connection) debug_print('KoboTouch:books - shelf list:', self.bookshelvelist) @@ -2345,7 +2315,7 @@ class KOBOTOUCH(KOBO): if self.dbversion >= 53: try: - with closing(self.device_database_connection()) as connection: + with self.database_transaction() as connection: cursor = connection.cursor() cleanup_query = f'DELETE FROM content WHERE ContentID = ? AND Accessibility = 1 AND IsDownloaded = {self.bool_for_query(False)}' for fname, cycle in result: @@ -2498,7 +2468,7 @@ class KOBOTOUCH(KOBO): if self.dbversion >= 53: debug_print(f'KoboTouch:delete_via_sql: ContentID="{ContentID}"', f'ContentType="{ContentType}"') try: - with closing(self.device_database_connection()) as connection: + with self.database_transaction() as connection: debug_print('KoboTouch:delete_via_sql: have database connection') cursor = connection.cursor() @@ -2672,7 +2642,7 @@ class KOBOTOUCH(KOBO): # the last book from the collection the list of books is empty # and the removal of the last book would not occur - with closing(self.device_database_connection(use_row_factory=True)) as connection: + with self.database_transaction(use_row_factory=True) as connection: if self.manage_collections: if collections is not None: @@ -2967,7 +2937,7 @@ class KOBOTOUCH(KOBO): ContentID = self.contentid_from_path(filepath, ContentType) try: - with closing(self.device_database_connection()) as connection: + with self.database_transaction() as connection: cursor = connection.cursor() t = (ContentID,)