From e26a125daa25743c21fe82ab8d1fd450f6c94825 Mon Sep 17 00:00:00 2001 From: David Date: Sat, 2 Jul 2016 21:39:52 +1000 Subject: [PATCH 1/9] Fix #1598017 - KoboTouch configuration migration not working for older settings If the KoboTouch configuration was last saved using a very old version of calibre, the migration to the new style will fail. This will increases the likelyhood that the migration will work. --- src/calibre/devices/kobo/driver.py | 42 +++++++++----------- src/calibre/devices/kobo/kobotouch_config.py | 1 + 2 files changed, 19 insertions(+), 24 deletions(-) diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index 40f1fdf2b4..683f9b2067 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -64,7 +64,7 @@ class KOBO(USBMS): gui_name = 'Kobo Reader' description = _('Communicate with the Kobo Reader') author = 'Timothy Legge and David Forrester' - version = (2, 2, 0) + version = (2, 2, 1) dbversion = 0 fwversion = 0 @@ -2801,9 +2801,10 @@ class KOBOTOUCH(KOBO): If that is not found looks for a device default and if that is not found uses the global default.''' # debug_print("KoboTouch::get_prefs - key=", key, "cls=", cls) - opts = cls.settings() + if not cls.opts: + cls.opts = cls.settings() try: - return getattr(opts, key) + return getattr(cls.opts, key) except: debug_print("KoboTouch::get_prefs - probably an extra_customization:", key) return None @@ -2841,6 +2842,7 @@ class KOBOTOUCH(KOBO): c.add_opt('support_newer_firmware', default=False) c.add_opt('debugging_title', default='') + c.add_opt('driver_version', default='') # Mainly for debugging purposes, but might use if need to migrate between versions. return c @@ -2848,23 +2850,6 @@ class KOBOTOUCH(KOBO): @classmethod def settings(cls): opts = cls._config().parse() - if isinstance(cls.EXTRA_CUSTOMIZATION_DEFAULT, list) and len(cls.EXTRA_CUSTOMIZATION_DEFAULT) > 0: - if opts.extra_customization is None: - opts.extra_customization = [] - if not isinstance(opts.extra_customization, list): - opts.extra_customization = [opts.extra_customization] - if len(cls.EXTRA_CUSTOMIZATION_DEFAULT) > len(opts.extra_customization): - extra_options_offset = 0 - extra_customization = [] - for i,d in enumerate(cls.EXTRA_CUSTOMIZATION_DEFAULT): - if i >= len(opts.extra_customization) + extra_options_offset: - extra_customization.append(d) - elif d.__class__ != opts.extra_customization[i - extra_options_offset].__class__: - extra_options_offset += 1 - extra_customization.append(d) - else: - extra_customization.append(opts.extra_customization[i - extra_options_offset]) - opts.extra_customization = extra_customization if opts.extra_customization: opts = cls.migrate_old_settings(opts) @@ -3038,6 +3023,7 @@ class KOBOTOUCH(KOBO): @classmethod def migrate_old_settings(cls, settings): debug_print("KoboTouch::migrate_old_settings - start") + debug_print("KoboTouch::migrate_old_settings - settings.extra_customization=", settings.extra_customization) count_options = 0 OPT_COLLECTIONS = count_options @@ -3067,6 +3053,8 @@ class KOBOTOUCH(KOBO): OPT_DEBUGGING_TITLE = count_options if len(settings.extra_customization) >= count_options: + config = cls._config() + debug_print("KoboTouch::migrate_old_settings - config.preferences=", config.preferences) debug_print("KoboTouch::migrate_old_settings - settings need to be migrated") settings.manage_collections = True settings.collections_columns = settings.extra_customization[OPT_COLLECTIONS] @@ -3085,10 +3073,16 @@ class KOBOTOUCH(KOBO): settings.update_series = settings.extra_customization[OPT_UPDATE_SERIES_DETAILS] settings.update_metadata = settings.update_series - settings.modify_css = settings.extra_customization[OPT_MODIFY_CSS] - - settings.support_newer_firmware = settings.extra_customization[OPT_SUPPORT_NEWER_FIRMWARE] - settings.debugging_title = settings.extra_customization[OPT_DEBUGGING_TITLE] + # Check if these are very old settings. + if len(settings.extra_customization) == count_options: + config = cls._config() + settings.modify_css = config.get_option('modify_css') + settings.support_newer_firmware = settings.extra_customization[OPT_SUPPORT_NEWER_FIRMWARE - 1] + settings.debugging_title = settings.extra_customization[OPT_DEBUGGING_TITLE - 1] + else: + settings.modify_css = settings.extra_customization[OPT_MODIFY_CSS] + settings.support_newer_firmware = settings.extra_customization[OPT_SUPPORT_NEWER_FIRMWARE] + settings.debugging_title = settings.extra_customization[OPT_DEBUGGING_TITLE] settings.extra_customization = settings.extra_customization[count_options + 1:] return settings diff --git a/src/calibre/devices/kobo/kobotouch_config.py b/src/calibre/devices/kobo/kobotouch_config.py index 56dd6fde94..f87c6c16a4 100644 --- a/src/calibre/devices/kobo/kobotouch_config.py +++ b/src/calibre/devices/kobo/kobotouch_config.py @@ -113,6 +113,7 @@ class KOBOTOUCHConfig(TabbedDeviceConfig): p['support_newer_firmware'] = self.support_newer_firmware p['debugging_title'] = self.debugging_title + p['driver_version'] = '.'.join([unicode(i) for i in self.device.version]) return p From 475a8375e80c4dae271e455db65c5be406c7dbce Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 3 Jul 2016 09:29:39 +0530 Subject: [PATCH 2/9] Edit Book: Reports: Characters: Fix sorting by count and name not working. Fixes #1598518 [Editor Report Characters](https://bugs.launchpad.net/calibre/+bug/1598518) --- src/calibre/gui2/tweak_book/reports.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/tweak_book/reports.py b/src/calibre/gui2/tweak_book/reports.py index 1bc58b022a..eb2ee41eb7 100644 --- a/src/calibre/gui2/tweak_book/reports.py +++ b/src/calibre/gui2/tweak_book/reports.py @@ -745,11 +745,13 @@ class CharsModel(FileCollection): self.files = data['chars'] self.all_chars = tuple(entry.char for entry in self.files) psk = numeric_sort_key - self.sort_keys = tuple((psk(entry.char), None, entry.codepoint, len(entry.usage)) for entry in self.files) + self.sort_keys = tuple((psk(entry.char), None, entry.codepoint, entry.count) for entry in self.files) self.endResetModel() def data(self, index, role=Qt.DisplayRole): if role == SORT_ROLE: + if index.column() == 1: + return self.data(index) try: return self.sort_keys[index.row()][index.column()] except IndexError: From d2e42dd4a64be5f8f4b41d923ae449281de006b9 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 3 Jul 2016 10:10:28 +0530 Subject: [PATCH 3/9] Better exclusion rule for libusb test --- src/calibre/test_build.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/calibre/test_build.py b/src/calibre/test_build.py index 1474f926e6..3298ac55a2 100644 --- a/src/calibre/test_build.py +++ b/src/calibre/test_build.py @@ -68,14 +68,13 @@ class BuildTest(unittest.TestCase): # C++ name mangling incompatibilities preventing some modules # from loading exclusions.update(set('podofo'.split())) - else: - # libusb fails to initialize in the travis container - exclusions.update(set('libusb libmtp'.split())) + if islinux and (not os.path.exists('/dev/bus/usb') and not os.path.exists('/proc/bus/usb')): + # libusb fails to initialize in containers without USB subsystems + exclusions.update(set('libusb libmtp'.split())) for name in plugins: if name in exclusions: - if not isosx: - # libusb fails to initialize on travis, so just check that the - # DLL can be loaded + if name in ('libusb', 'libmtp'): + # Just check that the DLL can be loaded ctypes.CDLL(os.path.join(sys.extensions_location, name + ('.dylib' if isosx else '.so'))) continue mod, err = plugins[name] @@ -244,7 +243,7 @@ class TestRunner(unittest.main): self.test = find_tests() def test(): - result = TestRunner(verbosity=2, buffer=True, catchbreak=True, failfast=True, argv=sys.argv[:1], exit=False).result + result = TestRunner(verbosity=2, buffer=True, catchbreak=True, failfast=False, argv=sys.argv[:1], exit=False).result if not result.wasSuccessful(): raise SystemExit(1) From 0cc5371090b95491c5afa07add5e17e1ad314e47 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 3 Jul 2016 10:20:06 +0530 Subject: [PATCH 4/9] Update dukpy --- src/duktape/tests.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/duktape/tests.py b/src/duktape/tests.py index c981025ab3..30ae305a60 100644 --- a/src/duktape/tests.py +++ b/src/duktape/tests.py @@ -1,5 +1,6 @@ import os import sys +import tempfile import unittest from threading import Thread, Event from duktape import dukpy @@ -128,9 +129,10 @@ class EvalTests(unittest.TestCase): self.ctx = Context() self.g = self.ctx.g - self.testfile = 'dukpy_test.js' - with open(self.testfile, 'w') as fobj: - fobj.write('1+1') + with tempfile.NamedTemporaryFile( + prefix='dukpy-test-', suffix='.js', delete=False) as fobj: + fobj.write(b'1+1') + self.testfile = fobj.name def tearDown(self): os.remove(self.testfile) From c7365b93333d35367aca12756f1e5d14f490b503 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 3 Jul 2016 11:30:39 +0530 Subject: [PATCH 5/9] Re-use run_cli for running the build tests --- src/calibre/test_build.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/calibre/test_build.py b/src/calibre/test_build.py index 3298ac55a2..df87cef400 100644 --- a/src/calibre/test_build.py +++ b/src/calibre/test_build.py @@ -237,15 +237,9 @@ def find_tests(): ans.addTests(find_tests()) return ans -class TestRunner(unittest.main): - - def createTests(self): - self.test = find_tests() - def test(): - result = TestRunner(verbosity=2, buffer=True, catchbreak=True, failfast=False, argv=sys.argv[:1], exit=False).result - if not result.wasSuccessful(): - raise SystemExit(1) + from calibre.utils.run_tests import run_cli + run_cli(find_tests()) if __name__ == '__main__': test() From 95a42eb768e235b2657d8fb8f3f979e02477b883 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 3 Jul 2016 15:13:07 +0530 Subject: [PATCH 6/9] Run the session bus test if the session bus env var is set --- src/calibre/test_build.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/calibre/test_build.py b/src/calibre/test_build.py index df87cef400..143f24389e 100644 --- a/src/calibre/test_build.py +++ b/src/calibre/test_build.py @@ -32,12 +32,14 @@ class BuildTest(unittest.TestCase): @unittest.skipUnless(islinux, 'DBUS only used on linux') def test_dbus(self): import dbus + bus = None if 'DISPLAY' in os.environ: bus = dbus.SystemBus() self.assertTrue(bus.list_names(), 'Failed to list names on the system bus') + if 'DBUS_SESSION_BUS_ADDRESS' in os.environ: bus = dbus.SessionBus() self.assertTrue(bus.list_names(), 'Failed to list names on the session bus') - del bus + del bus def test_regex(self): import regex From 83ba85ef1b110045954e105f02fb587416ac4b39 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 3 Jul 2016 16:04:37 +0530 Subject: [PATCH 7/9] Condition DBUS connectivity test on a single envvar That way it will still run in the future if X goes away --- src/calibre/test_build.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/calibre/test_build.py b/src/calibre/test_build.py index 143f24389e..245d695864 100644 --- a/src/calibre/test_build.py +++ b/src/calibre/test_build.py @@ -32,14 +32,12 @@ class BuildTest(unittest.TestCase): @unittest.skipUnless(islinux, 'DBUS only used on linux') def test_dbus(self): import dbus - bus = None - if 'DISPLAY' in os.environ: + if 'DBUS_SESSION_BUS_ADDRESS' in os.environ: bus = dbus.SystemBus() self.assertTrue(bus.list_names(), 'Failed to list names on the system bus') - if 'DBUS_SESSION_BUS_ADDRESS' in os.environ: bus = dbus.SessionBus() self.assertTrue(bus.list_names(), 'Failed to list names on the session bus') - del bus + del bus def test_regex(self): import regex From a001db9ef1146593820e4dd364856f08e68ac16e Mon Sep 17 00:00:00 2001 From: David Date: Sun, 3 Jul 2016 23:20:58 +1000 Subject: [PATCH 8/9] Change Kobo drivers to use apsw This reworks both the Kobo and KoboTouch drivers to use apsw instead of sqlite3. While doing this, I have refactored the code for the database connections and getting the versions from the device. --- src/calibre/devices/kobo/bookmark.py | 154 ++++++++-------- src/calibre/devices/kobo/driver.py | 257 ++++++++++++--------------- 2 files changed, 194 insertions(+), 217 deletions(-) diff --git a/src/calibre/devices/kobo/bookmark.py b/src/calibre/devices/kobo/bookmark.py index 71d096d95e..3a74c53061 100644 --- a/src/calibre/devices/kobo/bookmark.py +++ b/src/calibre/devices/kobo/bookmark.py @@ -13,7 +13,7 @@ class Bookmark(): # {{{ A simple class fetching bookmark data kobo-specific ''' - def __init__(self, db_path, contentid, path, id, book_format, bookmark_extension): + def __init__(self, db_connection, contentid, path, id, book_format, bookmark_extension): self.book_format = book_format self.bookmark_extension = bookmark_extension self.book_length = 0 # Not Used @@ -23,7 +23,7 @@ class Bookmark(): # {{{ self.path = path self.timestamp = 0 self.user_notes = None - self.db_path = db_path + self.db_connection = db_connection self.contentid = contentid self.percent_read = 0 self.get_bookmark_data() @@ -31,91 +31,93 @@ class Bookmark(): # {{{ def get_bookmark_data(self): ''' Return the timestamp and last_read_location ''' - import sqlite3 as sqlite user_notes = {} self.timestamp = os.path.getmtime(self.path) - with closing(sqlite.connect(self.db_path)) as connection: - # return bytestrings if the content cannot the decoded as unicode - connection.text_factory = lambda x: unicode(x, "utf-8", "ignore") - cursor = connection.cursor() - t = (self.contentid,) + cursor = self.db_connection.cursor() + t = (self.contentid,) - kepub_chapter_query = ( - 'SELECT Title, volumeIndex ' - 'FROM content ' - 'WHERE ContentID LIKE ? ' - ) - bookmark_query = ('SELECT bm.bookmarkid, bm.ContentID, bm.text, bm.annotation, ' - 'bm.ChapterProgress, bm.StartContainerChildIndex, bm.StartOffset, ' - 'c.BookTitle, c.TITLE, c.volumeIndex, c.MimeType ' - 'FROM Bookmark bm LEFT OUTER JOIN Content c ON ' - 'c.ContentID = bm.ContentID ' - 'WHERE bm.Hidden = "false" ' - 'AND bm.volumeid = ? ' - 'ORDER BY bm.ContentID, bm.chapterprogress') - cursor.execute(bookmark_query, t) + kepub_chapter_query = ( + 'SELECT Title, volumeIndex ' + 'FROM content ' + 'WHERE ContentID LIKE ? ' + ) + bookmark_query = ('SELECT bm.bookmarkid, bm.ContentID, bm.text, bm.annotation, ' + 'bm.ChapterProgress, bm.StartContainerChildIndex, bm.StartOffset, ' + 'c.BookTitle, c.TITLE, c.volumeIndex, c.MimeType ' + 'FROM Bookmark bm LEFT OUTER JOIN Content c ON ' + 'c.ContentID = bm.ContentID ' + 'WHERE bm.Hidden = "false" ' + 'AND bm.volumeid = ? ' + 'ORDER BY bm.ContentID, bm.chapterprogress') + cursor.execute(bookmark_query, t) - previous_chapter = 0 - bm_count = 0 - for row in cursor: - current_chapter = row[9] - chapter_title = row[8] - # For kepubs on newer firmware, the title needs to come from an 899 row. - if not row[10] or row[10] == 'application/xhtml+xml' or row[10] == 'application/x-kobo-epub+zip': - cursor2 = connection.cursor() - kepub_chapter_data = ('{0}-%'.format(row[1]), ) - cursor2.execute(kepub_chapter_query, kepub_chapter_data) - kepub_chapter = cursor2.fetchone() - if kepub_chapter: - chapter_title = kepub_chapter[0] - current_chapter = kepub_chapter[1] + previous_chapter = 0 + bm_count = 0 + for row in cursor: + current_chapter = row[9] + chapter_title = row[8] + # For kepubs on newer firmware, the title needs to come from an 899 row. + if not row[10] or row[10] == 'application/xhtml+xml' or row[10] == 'application/x-kobo-epub+zip': + cursor2 = self.db_connection.cursor() + kepub_chapter_data = ('{0}-%'.format(row[1]), ) + cursor2.execute(kepub_chapter_query, kepub_chapter_data) + try: + kepub_chapter = cursor2.next() + chapter_title = kepub_chapter[0] + current_chapter = kepub_chapter[1] + except StopIteration: + pass + finally: cursor2.close - if previous_chapter == current_chapter: - bm_count = bm_count + 1 - else: - bm_count = 0 + if previous_chapter == current_chapter: + bm_count = bm_count + 1 + else: + bm_count = 0 - text = row[2] - annotation = row[3] + text = row[2] + annotation = row[3] - # A dog ear (bent upper right corner) is a bookmark - if row[5] == row[6] == 0: # StartContainerChildIndex = StartOffset = 0 - e_type = 'Bookmark' - text = row[8] - # highlight is text with no annotation - elif text is not None and (annotation is None or annotation == ""): - e_type = 'Highlight' - elif text and annotation: - e_type = 'Annotation' - else: - e_type = 'Unknown annotation type' + # A dog ear (bent upper right corner) is a bookmark + if row[5] == row[6] == 0: # StartContainerChildIndex = StartOffset = 0 + e_type = 'Bookmark' + text = row[8] + # highlight is text with no annotation + elif text is not None and (annotation is None or annotation == ""): + e_type = 'Highlight' + elif text and annotation: + e_type = 'Annotation' + else: + e_type = 'Unknown annotation type' - note_id = current_chapter * 1000 + bm_count + note_id = current_chapter * 1000 + bm_count - # book_title = row[8] - chapter_progress = min(round(float(100*row[4]),2),100) - user_notes[note_id] = dict(id=self.id, - displayed_location=note_id, - type=e_type, - text=text, - annotation=annotation, - chapter=current_chapter, - chapter_title=chapter_title, - chapter_progress=chapter_progress) - previous_chapter = current_chapter - # debug_print("e_type:" , e_type, '\t', 'loc: ', note_id, 'text: ', text, - # 'annotation: ', annotation, 'chapter_title: ', chapter_title, - # 'chapter_progress: ', chapter_progress, 'date: ') + # book_title = row[8] + chapter_progress = min(round(float(100*row[4]),2),100) + user_notes[note_id] = dict(id=self.id, + displayed_location=note_id, + type=e_type, + text=text, + annotation=annotation, + chapter=current_chapter, + chapter_title=chapter_title, + chapter_progress=chapter_progress) + previous_chapter = current_chapter + # debug_print("e_type:" , e_type, '\t', 'loc: ', note_id, 'text: ', text, + # 'annotation: ', annotation, 'chapter_title: ', chapter_title, + # 'chapter_progress: ', chapter_progress, 'date: ') - cursor.execute('select datelastread, ___PercentRead from content ' - 'where bookid is Null and ' - 'contentid = ?', t) - for row in cursor: - self.last_read = row[0] - self.percent_read = row[1] - # print row[1] - cursor.close() + cursor.execute('SELECT datelastread, ___PercentRead ' + 'FROM content ' + 'WHERE bookid IS NULL ' + 'AND ReadStatus > 0 ' + 'AND contentid = ?', + t) + for row in cursor: + self.last_read = row[0] + self.percent_read = row[1] + # print row[1] + cursor.close() # self.last_read_location = self.last_read - self.pdf_page_offset self.user_notes = user_notes diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index 683f9b2067..3e87f12255 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -31,6 +31,7 @@ from calibre.utils.config_base import prefs EPUB_EXT = '.epub' KEPUB_EXT = '.kepub' +USE_SQLITE3 = False # Implementation of QtQHash for strings. This doesn't seem to be in the Python implementation. def qhash(inputstr): @@ -64,10 +65,10 @@ class KOBO(USBMS): gui_name = 'Kobo Reader' description = _('Communicate with the Kobo Reader') author = 'Timothy Legge and David Forrester' - version = (2, 2, 1) + version = (2, 3, 0) dbversion = 0 - fwversion = 0 + fwversion = (0,0,0) supported_dbversion = 125 has_kepubs = False @@ -157,10 +158,49 @@ class KOBO(USBMS): def device_database_path(self): return self.normalize_path(self._main_prefix + '.kobo/KoboReader.sqlite') + def device_database_connection(self): + if USE_SQLITE3: + import sqlite3 as sqlite + db_connection = sqlite.connect(self.device_database_path()) + + # return bytestrings if the content cannot the decoded as unicode + db_connection.text_factory = lambda x: unicode(x, "utf-8", "ignore") + else: + import apsw + db_connection = apsw.Connection(self.device_database_path()) + + return db_connection + + def get_database_version(self, connection): + cursor = connection.cursor() + cursor.execute('SELECT version FROM dbversion') + try: + result = cursor.next() + dbversion = result[0] + except StopIteration: + dbversion = 0 + + return dbversion + + + def get_firmware_version(self): + # Determine the firmware version + try: + with lopen(self.normalize_path(self._main_prefix + '.kobo/version'), 'rb') as f: + fwversion = f.readline().split(',')[2] + fwversion = tuple((int(x) for x in fwversion.split('.'))) + except: + debug_print("Kobo::get_firmware_version - didn't get firmware version from file'") + fwversion = (0,0,0) + + return fwversion + + def sanitize_path_components(self, components): invalid_filename_chars_re = re.compile(r'[\/\\\?%\*:;\|\"\'><\$!]', re.IGNORECASE | re.UNICODE) return [invalid_filename_chars_re.sub('_', x) for x in components] + def books(self, oncard=None, end_session=True): from calibre.ebooks.metadata.meta import path_to_ext @@ -180,15 +220,9 @@ class KOBO(USBMS): self._card_b_prefix if oncard == 'cardb' \ else self._main_prefix - # Determine the firmware version - try: - with lopen(self.normalize_path(self._main_prefix + '.kobo/version'), - 'rb') as f: - self.fwversion = f.readline().split(',')[2] - except: - self.fwversion = 'unknown' + self.fwversion = self.get_firmware_version() - if self.fwversion != '1.0' and self.fwversion != '1.4': + if not (self.fwversion == (1,0) or self.fwversion == (1,4)): self.has_kepubs = True debug_print('Version of driver: ', self.version, 'Has kepubs:', self.has_kepubs) debug_print('Version of firmware: ', self.fwversion, 'Has kepubs:', self.has_kepubs) @@ -293,22 +327,12 @@ class KOBO(USBMS): traceback.print_exc() return changed - import sqlite3 as sqlite - with closing(sqlite.connect( - self.normalize_path(self._main_prefix + - '.kobo/KoboReader.sqlite'))) as connection: - - # return bytestrings if the content cannot the decoded as unicode - connection.text_factory = lambda x: unicode(x, "utf-8", "ignore") - - cursor = connection.cursor() - - cursor.execute('select version from dbversion') - result = cursor.fetchone() - self.dbversion = result[0] + with closing(self.device_database_connection()) as connection: + self.dbversion = self.get_database_version(connection) debug_print("Database Version: ", self.dbversion) + cursor = connection.cursor() opts = self.settings() if self.dbversion >= 33: query= ('select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' @@ -410,13 +434,8 @@ class KOBO(USBMS): # 2) volume_shorcover # 2) content - import sqlite3 as sqlite debug_print('delete_via_sql: ContentID: ', ContentID, 'ContentType: ', ContentType) - with closing(sqlite.connect(self.normalize_path(self._main_prefix + - '.kobo/KoboReader.sqlite'))) as connection: - - # return bytestrings if the content cannot the decoded as unicode - connection.text_factory = lambda x: unicode(x, "utf-8", "ignore") + with closing(self.device_database_connection()) as connection: cursor = connection.cursor() t = (ContentID,) @@ -463,8 +482,6 @@ class KOBO(USBMS): else: cursor.execute('delete from content where BookID is Null and ContentID =?',t) - connection.commit() - cursor.close() if ImageID == None: print "Error condition ImageID was not found" @@ -624,7 +641,7 @@ class KOBO(USBMS): ContentType = 16 elif extension == '.rtf' or extension == '.txt' or extension == '.htm' or extension == '.html': # print "txt" - if self.fwversion == '1.0' or self.fwversion == '1.4' or self.fwversion == '1.7.4': + if self.fwversion == (1,0) or self.fwversion == (1,4) or self.fwversion == (1,7,4): ContentType = 999 else: ContentType = 901 @@ -754,21 +771,18 @@ class KOBO(USBMS): except: debug_print(' Database Exception: Unable to reset ReadStatus list') raise - else: - connection.commit() - # debug_print(' Commit: Reset ReadStatus list') - - cursor.close() + finally: + cursor.close() def set_readstatus(self, connection, ContentID, ReadStatus): cursor = connection.cursor() t = (ContentID,) cursor.execute('select DateLastRead from Content where BookID is Null and ContentID = ?', t) - result = cursor.fetchone() - if result is None: - datelastread = '1970-01-01T00:00:00' - else: + try: + result = cursor.next() datelastread = result[0] if result[0] is not None else '1970-01-01T00:00:00' + except StopIteration: + datelastread = '1970-01-01T00:00:00' t = (ReadStatus,datelastread,ContentID,) @@ -777,9 +791,7 @@ class KOBO(USBMS): except: debug_print(' Database Exception: Unable update ReadStatus') raise - else: - connection.commit() - # debug_print(' Commit: Setting ReadStatus List') + cursor.close() def reset_favouritesindex(self, connection, oncard): @@ -796,9 +808,8 @@ class KOBO(USBMS): debug_print(' Database Exception: Unable to reset Shortlist list') if 'no such column' not in str(e): raise - else: - connection.commit() - # debug_print(' Commit: Reset FavouritesIndex list') + finally: + cursor.close() def set_favouritesindex(self, connection, ContentID): cursor = connection.cursor() @@ -811,9 +822,8 @@ class KOBO(USBMS): debug_print(' Database Exception: Unable set book as Shortlist') if 'no such column' not in str(e): raise - else: - connection.commit() - # debug_print(' Commit: Set FavouritesIndex') + finally: + cursor.close() def update_device_database_collections(self, booklists, collections_attributes, oncard): debug_print("Kobo:update_device_database_collections - oncard='%s'"%oncard) @@ -854,12 +864,7 @@ class KOBO(USBMS): # the last book from the collection the list of books is empty # and the removal of the last book would not occur - import sqlite3 as sqlite - with closing(sqlite.connect(self.normalize_path(self._main_prefix + - '.kobo/KoboReader.sqlite'))) as connection: - - # return bytestrings if the content cannot the decoded as unicode - connection.text_factory = lambda x: unicode(x, "utf-8", "ignore") + with closing(self.device_database_connection()) as connection: if collections: @@ -988,25 +993,20 @@ class KOBO(USBMS): ContentType = self.get_content_type_from_extension(extension) if extension != '' else self.get_content_type_from_path(filepath) ContentID = self.contentid_from_path(filepath, ContentType) - import sqlite3 as sqlite - with closing(sqlite.connect(self.normalize_path(self._main_prefix + - '.kobo/KoboReader.sqlite'))) as connection: - - # return bytestrings if the content cannot the decoded as unicode - connection.text_factory = lambda x: unicode(x, "utf-8", "ignore") + with closing(self.device_database_connection()) as connection: cursor = connection.cursor() t = (ContentID,) cursor.execute('select ImageId from Content where BookID is Null and ContentID = ?', t) - result = cursor.fetchone() - if result is None: + try: + result = cursor.next() +# debug_print("ImageId: ", result[0]) + ImageID = result[0] + except StopIteration: debug_print("No rows exist in the database - cannot upload") return - else: - ImageID = result[0] -# debug_print("ImageId: ", result[0]) - - cursor.close() + finally: + cursor.close() if ImageID != None: path_prefix = '.kobo/images/' @@ -1158,17 +1158,17 @@ class KOBO(USBMS): path_map, book_ext = resolve_bookmark_paths(storage, path_map) bookmarked_books = {} - for id in path_map: - extension = os.path.splitext(path_map[id])[1] - ContentType = self.get_content_type_from_extension(extension) if extension != '' else self.get_content_type_from_path(path_map[id]) - ContentID = self.contentid_from_path(path_map[id], ContentType) - debug_print("get_annotations - ContentID: ", ContentID, "ContentType: ", ContentType) + with closing(self.device_database_connection()) as connection: + for id in path_map: + extension = os.path.splitext(path_map[id])[1] + ContentType = self.get_content_type_from_extension(extension) if extension != '' else self.get_content_type_from_path(path_map[id]) + ContentID = self.contentid_from_path(path_map[id], ContentType) + debug_print("get_annotations - ContentID: ", ContentID, "ContentType: ", ContentType) - bookmark_ext = extension + bookmark_ext = extension - db_path = self.normalize_path(self._main_prefix + '.kobo/KoboReader.sqlite') - myBookmark = Bookmark(db_path, ContentID, path_map[id], id, book_ext[id], bookmark_ext) - bookmarked_books[id] = self.UserAnnotation(type='kobo_bookmark', value=myBookmark) + myBookmark = Bookmark(connection, ContentID, path_map[id], id, book_ext[id], bookmark_ext) + bookmarked_books[id] = self.UserAnnotation(type='kobo_bookmark', value=myBookmark) # This returns as job.result in gui2.ui.annotations_fetched(self,job) return bookmarked_books @@ -1180,7 +1180,7 @@ class KOBO(USBMS): #last_read_location = bookmark.last_read_location #timestamp = bookmark.timestamp percent_read = bookmark.percent_read - debug_print("Date: ", bookmark.last_read) + debug_print("Kobo::generate_annotation_html - last_read: ", bookmark.last_read) if bookmark.last_read is not None: try: last_read = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(calendar.timegm(time.strptime(bookmark.last_read, "%Y-%m-%dT%H:%M:%S")))) @@ -1188,7 +1188,10 @@ class KOBO(USBMS): try: last_read = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(calendar.timegm(time.strptime(bookmark.last_read, "%Y-%m-%dT%H:%M:%S.%f")))) except: - last_read = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(calendar.timegm(time.strptime(bookmark.last_read, "%Y-%m-%dT%H:%M:%SZ")))) + try: + last_read = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(calendar.timegm(time.strptime(bookmark.last_read, "%Y-%m-%dT%H:%M:%SZ")))) + except: + last_read = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()) else: #self.datetime = time.gmtime() last_read = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()) @@ -1277,7 +1280,7 @@ class KOBO(USBMS): bm = annotation ignore_tags = set(['Catalog', 'Clippings']) - if bm.type == 'kobo_bookmark': + if bm.type == 'kobo_bookmark' and bm.value.last_read: mi = db.get_metadata(db_id, index_is_id=True) debug_print("KOBO:add_annotation_to_library - Title: ", mi.title) user_notes_soup = self.generate_annotation_html(bm.value) @@ -1312,9 +1315,9 @@ class KOBO(USBMS): class KOBOTOUCH(KOBO): name = 'KoboTouch' - gui_name = 'Kobo Touch/Glo/Mini/Aura HD' + gui_name = 'Kobo Touch/Glo/Mini/Aura HD/Aura H2O/Glo HD/Touch 2' author = 'David Forrester' - description = 'Communicate with the Kobo Touch, Glo, Mini and Aura HD ereaders. Based on the existing Kobo driver by %s.' % (KOBO.author) + description = 'Communicate with the Kobo Touch, Glo, Mini, Aura HD, Aura H2O, Glo HD and Touch 2ereaders. Based on the existing Kobo driver by %s.' % (KOBO.author) # icon = I('devices/kobotouch.jpg') supported_dbversion = 125 @@ -1437,13 +1440,7 @@ class KOBOTOUCH(KOBO): else self._main_prefix debug_print("KoboTouch:books - oncard='%s', prefix='%s'"%(oncard, prefix)) - # Determine the firmware version - try: - with lopen(self.normalize_path(self._main_prefix + '.kobo/version'), 'rb') as f: - self.fwversion = f.readline().split(',')[2] - self.fwversion = tuple((int(x) for x in self.fwversion.split('.'))) - except: - self.fwversion = (0,0,0) + self.fwversion = self.get_firmware_version() debug_print('Kobo device: %s' % self.gui_name) debug_print('Version of driver:', self.version, 'Has kepubs:', self.has_kepubs) @@ -1667,25 +1664,18 @@ class KOBOTOUCH(KOBO): return bookshelves self.debug_index = 0 - import sqlite3 as sqlite - with closing(sqlite.connect(self.device_database_path())) as connection: + + with closing(self.device_database_connection()) as connection: debug_print("KoboTouch:books - reading device database") - # return bytestrings if the content cannot the decoded as unicode - connection.text_factory = lambda x: unicode(x, "utf-8", "ignore") + self.dbversion = self.get_database_version(connection) + debug_print("Database Version: ", self.dbversion) cursor = connection.cursor() - cursor.execute('select version from dbversion') - result = cursor.fetchone() - self.dbversion = result[0] - debug_print("Database Version=%d"%self.dbversion) - self.bookshelvelist = self.get_bookshelflist(connection) debug_print("KoboTouch:books - shelf list:", self.bookshelvelist) - opts = self.settings() - columns = 'Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ImageID, ReadStatus' if self.dbversion >= 16: columns += ', ___ExpirationStatus, FavouritesIndex, Accessibility' @@ -1930,11 +1920,8 @@ class KOBOTOUCH(KOBO): # debug_print('KoboTouch:upload_books - result=', result) if self.dbversion >= 53: - import sqlite3 as sqlite try: - with closing(sqlite.connect(self.normalize_path(self._main_prefix + - '.kobo/KoboReader.sqlite'))) as connection: - connection.text_factory = lambda x: unicode(x, "utf-8", "ignore") + with closing(self.device_database_connection()) as connection: cursor = connection.cursor() cleanup_query = "DELETE FROM content WHERE ContentID = ? AND Accessibility = 1 AND IsDownloaded = 'false'" @@ -1954,7 +1941,6 @@ class KOBOTOUCH(KOBO): if not self.upload_covers: imageID = self.imageid_from_contentid(contentID) self.delete_images(imageID, fname) - connection.commit() cursor.close() except Exception as e: @@ -2064,13 +2050,10 @@ class KOBOTOUCH(KOBO): imageId = super(KOBOTOUCH, self).delete_via_sql(ContentID, ContentType) if self.dbversion >= 53: - import sqlite3 as sqlite debug_print('KoboTouch:delete_via_sql: ContentID="%s"'%ContentID, 'ContentType="%s"'%ContentType) try: - with closing(sqlite.connect(self.device_database_path())) as connection: + with closing(self.device_database_connection()) as connection: debug_print('KoboTouch:delete_via_sql: have database connection') - # return bytestrings if the content cannot the decoded as unicode - connection.text_factory = lambda x: unicode(x, "utf-8", "ignore") cursor = connection.cursor() debug_print('KoboTouch:delete_via_sql: have cursor') @@ -2101,8 +2084,6 @@ class KOBOTOUCH(KOBO): debug_print('KoboTouch:delete_via_sql: delete from Activity') cursor.execute('delete from Activity where Id =?', t) - connection.commit() - cursor.close() debug_print('KoboTouch:delete_via_sql: finished SQL') debug_print('KoboTouch:delete_via_sql: After SQL, no exception') @@ -2227,11 +2208,7 @@ class KOBOTOUCH(KOBO): # the last book from the collection the list of books is empty # and the removal of the last book would not occur - import sqlite3 as sqlite - with closing(sqlite.connect(self.device_database_path())) as connection: - - # return bytestrings if the content cannot the decoded as unicode - connection.text_factory = lambda x: unicode(x, "utf-8", "ignore") + with closing(self.device_database_connection()) as connection: if self.manage_collections and collections: # debug_print("KoboTouch:update_device_database_collections - length collections=" + unicode(len(collections))) @@ -2427,23 +2404,17 @@ class KOBOTOUCH(KOBO): ContentID = self.contentid_from_path(filepath, ContentType) try: - import sqlite3 as sqlite - with closing(sqlite.connect(self.device_database_path())) as connection: - - # return bytestrings if the content cannot the decoded as unicode - connection.text_factory = lambda x: unicode(x, "utf-8", "ignore") + with closing(self.device_database_connection()) as connection: cursor = connection.cursor() t = (ContentID,) cursor.execute('select ImageId from Content where BookID is Null and ContentID = ?', t) - result = cursor.fetchone() - - if result is None: + try: + result = cursor.next() + ImageID = result[0] + except StopIteration: ImageID = self.imageid_from_contentid(ContentID) debug_print("KoboTouch:_upload_cover - No rows exist in the database - generated ImageID='%s'" % ImageID) - else: - ImageID = result[0] - # debug_print("ImageId: ", result[0]) cursor.close() @@ -2527,7 +2498,6 @@ class KOBOTOUCH(KOBO): cursor = connection.cursor() cursor.execute(query, values) - connection.commit() cursor.close() def set_filesize_in_device_database(self, connection, contentID, fpath): @@ -2548,7 +2518,11 @@ class KOBOTOUCH(KOBO): cursor = connection.cursor() cursor.execute(test_query, test_values) - result = cursor.fetchone() + try: + result = cursor.next() + except StopIteration: + result = None + if result is None: if show_debug: debug_print(' Did not find a record - new book on device') @@ -2562,7 +2536,6 @@ class KOBOTOUCH(KOBO): if show_debug: debug_print(' Size updated.') - connection.commit() cursor.close() # debug_print("KoboTouch:set_filesize_in_device_database - end") @@ -2600,7 +2573,6 @@ class KOBOTOUCH(KOBO): cursor.execute(update_query) if self.has_activity_table(): cursor.execute(delete_activity_query) - connection.commit() cursor.close() debug_print("KoboTouch:delete_empty_bookshelves - end") @@ -2647,7 +2619,11 @@ class KOBOTOUCH(KOBO): cursor = connection.cursor() cursor.execute(test_query, test_values) - result = cursor.fetchone() + try: + result = cursor.next() + except StopIteration: + result = None + if result is None: if show_debug: debug_print(' Did not find a record - adding') @@ -2657,8 +2633,6 @@ class KOBOTOUCH(KOBO): debug_print(' Found a record - updating - result=', result) cursor.execute(updatequery, update_values) - connection.commit() - cursor.close() # debug_print("KoboTouch:set_bookshelf - end") @@ -2693,7 +2667,11 @@ class KOBOTOUCH(KOBO): cursor = connection.cursor() cursor.execute(test_query, test_values) - result = cursor.fetchone() + try: + result = cursor.next() + except StopIteration: + result = None + if result is None: if show_debug: debug_print(' Did not find a record - adding shelf "%s"' % bookshelf_name) @@ -2702,7 +2680,6 @@ class KOBOTOUCH(KOBO): debug_print('KoboTouch:check_for_bookshelf - Shelf "%s" is deleted - undeleting. result[2]="%s"' % (bookshelf_name, unicode(result[2]))) cursor.execute(updatequery, test_values) - connection.commit() cursor.close() # Update the bookshelf list. @@ -2735,7 +2712,6 @@ class KOBOTOUCH(KOBO): debug_print('KoboTouch:remove_from_bookshelf values=', values) cursor = connection.cursor() cursor.execute(query, values) - connection.commit() cursor.close() debug_print("KoboTouch:remove_from_bookshelf - end") @@ -2776,9 +2752,8 @@ class KOBOTOUCH(KOBO): except: debug_print(' Database Exception: Unable to set series info') raise - else: - connection.commit() - cursor.close() + finally: + cursor.close() if show_debug: debug_print("KoboTouch:set_series - end") From f508fa69f32100016e0752ecdd26b2df7116e90c Mon Sep 17 00:00:00 2001 From: David Date: Fri, 8 Jul 2016 20:01:07 +1000 Subject: [PATCH 9/9] Remove the last sqlite3 reference and fix some cover issues Had left an easy out to go back to sqlite3, but everything is working. For the covers, shouldn't be sending one version with current firmware, and fixed a check that was happening. --- src/calibre/devices/kobo/driver.py | 43 ++++++++++++------------------ 1 file changed, 17 insertions(+), 26 deletions(-) diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index 3e87f12255..835ab0219e 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -31,8 +31,6 @@ from calibre.utils.config_base import prefs EPUB_EXT = '.epub' KEPUB_EXT = '.kepub' -USE_SQLITE3 = False - # Implementation of QtQHash for strings. This doesn't seem to be in the Python implementation. def qhash(inputstr): instr = b"" @@ -159,15 +157,8 @@ class KOBO(USBMS): return self.normalize_path(self._main_prefix + '.kobo/KoboReader.sqlite') def device_database_connection(self): - if USE_SQLITE3: - import sqlite3 as sqlite - db_connection = sqlite.connect(self.device_database_path()) - - # return bytestrings if the content cannot the decoded as unicode - db_connection.text_factory = lambda x: unicode(x, "utf-8", "ignore") - else: - import apsw - db_connection = apsw.Connection(self.device_database_path()) + import apsw + db_connection = apsw.Connection(self.device_database_path()) return db_connection @@ -1378,20 +1369,20 @@ class KOBOTOUCH(KOBO): ' - N3_LIBRARY_FULL.parsed':[(355,473),0, 200,False,], # Used for Details screen before FW2.8.1, then for current book tile on home screen ' - N3_LIBRARY_GRID.parsed':[(149,198),0, 200,False,], # Used for library lists ' - N3_LIBRARY_LIST.parsed':[(60,90),0, 53,False,], - ' - AndroidBookLoadTablet_Aspect.parsed':[(355,473), 82, 200,False,], # Used for Details screen from FW2.8.1 + ' - AndroidBookLoadTablet_Aspect.parsed':[(355,473), 82, 100,False,], # Used for Details screen from FW2.8.1 # ' - N3_LIBRARY_SHELF.parsed': [(40,60),0, 52,], } GLO_COVER_FILE_ENDINGS = { # Glo and Aura share resolution, so the image sizes should be the same. ' - N3_FULL.parsed':[(758,1024),0, 200,True,], # Used for screensaver, home screen ' - N3_LIBRARY_FULL.parsed':[(355,479),0, 200,False,], # Used for Details screen before FW2.8.1, then for current book tile on home screen ' - N3_LIBRARY_GRID.parsed':[(149,201),0, 200,False,], # Used for library lists - ' - AndroidBookLoadTablet_Aspect.parsed':[(355,479), 88, 200,False,], # Used for Details screen from FW2.8.1 + ' - AndroidBookLoadTablet_Aspect.parsed':[(355,479), 88, 100,False,], # Used for Details screen from FW2.8.1 } AURA_HD_COVER_FILE_ENDINGS = { ' - N3_FULL.parsed': [(1080,1440), 0, 200,True,], # Used for screensaver, home screen ' - N3_LIBRARY_FULL.parsed':[(355, 471), 0, 200,False,], # Used for Details screen before FW2.8.1, then for current book tile on home screen ' - N3_LIBRARY_GRID.parsed':[(149, 198), 0, 200,False,], # Used for library lists - ' - AndroidBookLoadTablet_Aspect.parsed':[(355, 471), 88, 200,False,], # Used for Details screen from FW2.8.1 + ' - AndroidBookLoadTablet_Aspect.parsed':[(355, 471), 88, 100,False,], # Used for Details screen from FW2.8.1 } # Following are the sizes used with pre2.1.4 firmware # COVER_FILE_ENDINGS = { @@ -1846,18 +1837,18 @@ class KOBOTOUCH(KOBO): def imagefilename_from_imageID(self, prefix, ImageID): show_debug = self.is_debugging_title(ImageID) - path = self.images_path(prefix, ImageID) -# path = self.normalize_path(path.replace('/', os.sep)) - - for ending, cover_options in self.cover_file_endings().items(): - fpath = path + ending - if os.path.exists(fpath): - if show_debug: - debug_print("KoboTouch:imagefilename_from_imageID - have cover image fpath=%s" % (fpath)) - return fpath - - if show_debug: - debug_print("KoboTouch:imagefilename_from_imageID - no cover image found - ImageID=%s" % (ImageID)) + if len(ImageID) > 0: + path = self.images_path(prefix, ImageID) + + for ending in self.cover_file_endings().keys(): + fpath = path + ending + if os.path.exists(fpath): + if show_debug: + debug_print("KoboTouch:imagefilename_from_imageID - have cover image fpath=%s" % (fpath)) + return fpath + + if show_debug: + debug_print("KoboTouch:imagefilename_from_imageID - no cover image found - ImageID=%s" % (ImageID)) return None def get_extra_css(self):