From 96c71a0b50e3498953b1fe3ba729fac01a6b0f47 Mon Sep 17 00:00:00 2001 From: "david.hobley" Date: Sat, 7 May 2016 14:27:57 +1000 Subject: [PATCH] Include the Paladin driver --- src/calibre/devices/paladin/__init__.py | 7 + src/calibre/devices/paladin/driver.py | 572 ++++++++++++++++++++++++ 2 files changed, 579 insertions(+) create mode 100644 src/calibre/devices/paladin/__init__.py create mode 100644 src/calibre/devices/paladin/driver.py diff --git a/src/calibre/devices/paladin/__init__.py b/src/calibre/devices/paladin/__init__.py new file mode 100644 index 0000000000..4ed1c1cbbe --- /dev/null +++ b/src/calibre/devices/paladin/__init__.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + diff --git a/src/calibre/devices/paladin/driver.py b/src/calibre/devices/paladin/driver.py new file mode 100644 index 0000000000..6b49069e08 --- /dev/null +++ b/src/calibre/devices/paladin/driver.py @@ -0,0 +1,572 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +''' +Device driver for the Paladin devices +''' + +import os, time, re, sys +import apsw +from contextlib import closing +from datetime import date + +from calibre import fsync +from calibre.devices.mime import mime_type_ext +from calibre.devices.errors import DeviceError +from calibre.devices.usbms.driver import USBMS, debug_print +from calibre.devices.usbms.device import USBDevice +from calibre.devices.usbms.books import CollectionsBookList +from calibre.devices.usbms.books import BookList +from calibre.ebooks.metadata import authors_to_sort_string, authors_to_string +from calibre.constants import islinux + +DBPATH = 'paladin/database/books.db' + +class ImageWrapper(object): + def __init__(self, image_path): + self.image_path = image_path + +class PALADIN(USBMS): + name = 'Paladin Device Interface' + gui_name = 'Paladin eLibrary' + description = _('Communicate with the Paladin range of readers') + author = 'Kovid Goyal & David Hobley' + supported_platforms = ['windows', 'osx', 'linux'] + path_sep = '/' + booklist_class = CollectionsBookList + + FORMATS = ['epub', 'pdf'] + CAN_SET_METADATA = ['collections'] + CAN_DO_DEVICE_DB_PLUGBOARD = True + + VENDOR_ID = [0x2207] #: Onyx Vendor Id (currently) + PRODUCT_ID = [0x0010] + BCD = None + + VENDOR_NAME = 'ONYX' + + SUPPORTS_SUB_DIRS = True + SUPPORTS_USE_AUTHOR_SORT = True + MUST_READ_METADATA = True + EBOOK_DIR_MAIN = 'paladin/books' + + OPT_COLLECTIONS = 0 + + plugboards = None + plugboard_func = None + + device_offset = None + + def can_handle(self, devinfo, debug=False): + if islinux: + dev = USBDevice(devinfo) + main, carda, cardb = self.find_device_nodes(detected_device=dev) + if main is None and carda is None and cardb is None: + if debug: + print ('\tPALADIN: Appears to be in non data mode' + ' or was ejected, ignoring') + return False + return True + + def books(self, oncard=None, end_session=True): + dummy_bl = BookList(None, None, None) + + if ( + (oncard == 'carda' and not self._card_a_prefix) or + (oncard and oncard != 'carda') + ): + self.report_progress(1.0, _('Getting list of books on device...')) + return dummy_bl + + prefix = self._card_a_prefix if oncard == 'carda' else self._main_prefix + + # Let parent driver get the books + self.booklist_class.rebuild_collections = self.rebuild_collections + bl = USBMS.books(self, oncard=oncard, end_session=end_session) + + dbpath = self.normalize_path(prefix + DBPATH) + debug_print("SQLite DB Path: " + dbpath) + + with closing(apsw.Connection(dbpath)) as connection: + cursor = connection.cursor() + # Query collections + query = ''' + SELECT books._id, tags.tagname + FROM booktags + LEFT OUTER JOIN books + LEFT OUTER JOIN tags + WHERE booktags.book_id = books._id AND + booktags.tag_id = tags._id + ''' + cursor.execute(query) + + bl_collections = {} + for i, row in enumerate(cursor): + bl_collections.setdefault(row[0], []) + bl_collections[row[0]].append(row[1]) + + # collect information on offsets, but assume any + # offset we already calculated is correct + if self.device_offset is None: + query = 'SELECT filename, addeddate FROM books' + cursor.execute(query) + + time_offsets = {} + for i, row in enumerate(cursor): + try: + comp_date = int(os.path.getmtime(self.normalize_path(prefix + row[0])) * 1000) + except (OSError, IOError, TypeError): + # In case the db has incorrect path info + continue + device_date = int(row[1]) + offset = device_date - comp_date + time_offsets.setdefault(offset, 0) + time_offsets[offset] = time_offsets[offset] + 1 + + try: + device_offset = max(time_offsets, key=lambda a: time_offsets.get(a)) + debug_print("Device Offset: %d ms"%device_offset) + self.device_offset = device_offset + except ValueError: + debug_print("No Books To Detect Device Offset.") + + for idx, book in enumerate(bl): + query = 'SELECT _id, thumbnail FROM books WHERE filename = ?' + t = (book.lpath,) + cursor.execute(query, t) + + for i, row in enumerate(cursor): + book.device_collections = bl_collections.get(row[0], None) + thumbnail = row[1] + if thumbnail is not None: + thumbnail = self.normalize_path(prefix + thumbnail) + book.thumbnail = ImageWrapper(thumbnail) + + cursor.close() + + return bl + + def set_plugboards(self, plugboards, pb_func): + self.plugboards = plugboards + self.plugboard_func = pb_func + + def sync_booklists(self, booklists, end_session=True): + debug_print('PALADIN: starting sync_booklists') + + opts = self.settings() + if opts.extra_customization: + collections = [x.strip() for x in + opts.extra_customization[self.OPT_COLLECTIONS].split(',')] + else: + collections = [] + debug_print('PALADIN: collection fields:', collections) + + if booklists[0] is not None: + self.update_device_database(booklists[0], collections, None) + if len(booklists) > 1 and booklists[1] is not None: + self.update_device_database(booklists[1], collections, 'carda') + + USBMS.sync_booklists(self, booklists, end_session=end_session) + debug_print('PALADIN: finished sync_booklists') + + def update_device_database(self, booklist, collections_attributes, oncard): + debug_print('PALADIN: starting update_device_database') + + plugboard = None + if self.plugboard_func: + plugboard = self.plugboard_func(self.__class__.__name__, + 'device_db', self.plugboards) + debug_print("PALADIN: Using Plugboard", plugboard) + + prefix = self._card_a_prefix if oncard == 'carda' else self._main_prefix + if prefix is None: + # Reader has no sd card inserted + return + source_id = 1 if oncard == 'carda' else 0 + + dbpath = self.normalize_path(prefix + DBPATH) + debug_print("SQLite DB Path: " + dbpath) + + collections = booklist.get_collections(collections_attributes) + + with closing(apsw.Connection(dbpath)) as connection: + self.remove_orphaned_records(connection, dbpath) + self.update_device_books(connection, booklist, source_id, + plugboard, dbpath) + self.update_device_collections(connection, booklist, collections, source_id, dbpath) + + debug_print('PALADIN: finished update_device_database') + + def remove_orphaned_records(self, connection, dbpath): + try: + cursor = connection.cursor() + + debug_print("Removing Orphaned Collection Records") + + # Purge any collections references that point into the abyss + query = 'DELETE FROM booktags WHERE book_id NOT IN (SELECT _id FROM books)' + cursor.execute(query) + query = 'DELETE FROM booktags WHERE tag_id NOT IN (SELECT _id FROM tags)' + cursor.execute(query) + + debug_print("Removing Orphaned Book Records") + + cursor.close() + except: + import traceback + tb = traceback.format_exc() + raise DeviceError((('The Paladin database is corrupted. ' + ' Delete the file %s on your reader and then disconnect ' + ' reconnect it. If you are using an SD card, you ' + ' should delete the file on the card as well. Note that ' + ' deleting this file will cause your reader to forget ' + ' any notes/highlights, etc.')%dbpath)+' Underlying error:' + '\n'+tb) + + def get_lastrowid(self, cursor): + # SQLite3 + Python has a fun issue on 32-bit systems with integer overflows. + # Issue a SQL query instead, getting the value as a string, and then converting to a long python int manually. + query = 'SELECT last_insert_rowid()' + cursor.execute(query) + row = cursor.next() + + return long(row[0]) + + def get_database_min_id(self, source_id): + sequence_min = 0L + if source_id == 1: + sequence_min = 4294967296L + + return sequence_min + + def set_database_sequence_id(self, connection, table, sequence_id): + cursor = connection.cursor() + + # Update the sequence Id if it exists + query = 'UPDATE sqlite_sequence SET seq = ? WHERE name = ?' + t = (sequence_id, table,) + cursor.execute(query, t) + + # Insert the sequence Id if it doesn't + query = ('INSERT INTO sqlite_sequence (name, seq) ' + 'SELECT ?, ? ' + 'WHERE NOT EXISTS (SELECT 1 FROM sqlite_sequence WHERE name = ?)') + cursor.execute(query, (table, sequence_id, table,)) + + cursor.close() + + def read_device_books(self, connection, source_id, dbpath): + sequence_min = self.get_database_min_id(source_id) + sequence_max = sequence_min + sequence_dirty = 0 + + debug_print("Book Sequence Min: %d, Source Id: %d"%(sequence_min,source_id)) + + try: + cursor = connection.cursor() + + # Get existing books + query = 'SELECT filename, _id FROM books' + cursor.execute(query) + except: + import traceback + tb = traceback.format_exc() + raise DeviceError((('The Paladin database is corrupted. ' + ' Delete the file %s on your reader and then disconnect ' + ' reconnect it. If you are using an SD card, you ' + ' should delete the file on the card as well. Note that ' + ' deleting this file will cause your reader to forget ' + ' any notes/highlights, etc.')%dbpath)+' Underlying error:' + '\n'+tb) + + # Get the books themselves, but keep track of any that are less than the minimum. + # Record what the max id being used is as well. + db_books = {} + for i, row in enumerate(cursor): + if not hasattr(row[0], 'replace'): + continue + lpath = row[0].replace('\\', '/') + db_books[lpath] = row[1] + if row[1] < sequence_min: + sequence_dirty = 1 + else: + sequence_max = max(sequence_max, row[1]) + + # If the database is 'dirty', then we should fix up the Ids and the sequence number + if sequence_dirty == 1: + debug_print("Book Sequence Dirty for Source Id: %d"%source_id) + sequence_max = sequence_max + 1 + for book, bookId in db_books.items(): + if bookId < sequence_min: + # Record the new Id and write it to the DB + db_books[book] = sequence_max + sequence_max = sequence_max + 1 + + # Fix the Books DB + query = 'UPDATE books SET _id = ? WHERE filename = ?' + t = (db_books[book], book,) + cursor.execute(query, t) + + # Fix any references so that they point back to the right book + t = (db_books[book], bookId,) + query = 'UPDATE booktags SET tag_id = ? WHERE tag_id = ?' + cursor.execute(query, t) + + self.set_database_sequence_id(connection, 'books', sequence_max) + debug_print("Book Sequence Max: %d, Source Id: %d"%(sequence_max,source_id)) + + cursor.close() + return db_books + + def update_device_books(self, connection, booklist, source_id, plugboard, + dbpath): + from calibre.ebooks.metadata.meta import path_to_ext + opts = self.settings() + + db_books = self.read_device_books(connection, source_id, dbpath) + cursor = connection.cursor() + + for book in booklist: + # Run through plugboard if needed + if plugboard is not None: + newmi = book.deepcopy_metadata() + newmi.template_to_attribute(book, plugboard) + else: + newmi = book + + # Get Metadata We Want + lpath = book.lpath + try: + if opts.use_author_sort: + if newmi.author_sort: + author = newmi.author_sort + else: + author = authors_to_sort_string(newmi.authors) + else: + author = authors_to_string(newmi.authors) + except: + author = _('Unknown') + title = newmi.title or _('Unknown') + + # Get modified date + # If there was a detected offset, use that. Otherwise use UTC (same as Sony software) + modified_date = os.path.getmtime(book.path) * 1000 + if self.device_offset is not None: + modified_date = modified_date + self.device_offset + + if lpath not in db_books: + query = ''' + INSERT INTO books + (bookname, authorname, description, addeddate, seriesname, seriesorder, filename, mimetype) + values (?,?,?,?,?,?,?,?) + ''' + t = (title, author, book.get('comments', None), int(time.time() * 1000), + book.get('series', None), book.get('series_index', sys.maxint), lpath, + book.mime or mime_type_ext(path_to_ext(lpath))) + cursor.execute(query, t) + book.bookId = self.get_lastrowid(cursor) + debug_print('Inserted New Book: (%u) '%book.bookId + book.title) + else: + query = ''' + UPDATE books + SET bookname = ?, authorname = ?, addeddate = ? + WHERE filename = ? + ''' + t = (title, author, modified_date, lpath) + cursor.execute(query, t) + book.bookId = db_books[lpath] + db_books[lpath] = None + + for book, bookId in db_books.items(): + if bookId is not None: + # Remove From Collections + query = 'DELETE FROM tags WHERE _id in (select tag_id from booktags where book_id = ?)' + t = (bookId,) + cursor.execute(query, t) + # Remove from Books + query = 'DELETE FROM books where _id = ?' + t = (bookId,) + cursor.execute(query, t) + debug_print('Deleted Book:' + book) + + cursor.close() + + def read_device_collections(self, connection, source_id, dbpath): + sequence_min = self.get_database_min_id(source_id) + sequence_max = sequence_min + sequence_dirty = 0 + + debug_print("Collection Sequence Min: %d, Source Id: %d"%(sequence_min,source_id)) + + try: + cursor = connection.cursor() + + # Get existing collections + query = 'SELECT _id, tagname FROM tags' + cursor.execute(query) + except: + import traceback + tb = traceback.format_exc() + raise DeviceError((('The Paladin database is corrupted. ' + ' Delete the file %s on your reader and then disconnect ' + ' reconnect it. If you are using an SD card, you ' + ' should delete the file on the card as well. Note that ' + ' deleting this file will cause your reader to forget ' + ' any notes/highlights, etc.')%dbpath)+' Underlying error:' + '\n'+tb) + + db_collections = {} + for i, row in enumerate(cursor): + db_collections[row[1]] = row[0] + if row[0] < sequence_min: + sequence_dirty = 1 + else: + sequence_max = max(sequence_max, row[0]) + + # If the database is 'dirty', then we should fix up the Ids and the sequence number + if sequence_dirty == 1: + debug_print("Collection Sequence Dirty for Source Id: %d"%source_id) + sequence_max = sequence_max + 1 + for collection, collectionId in db_collections.items(): + if collectionId < sequence_min: + # Record the new Id and write it to the DB + db_collections[collection] = sequence_max + sequence_max = sequence_max + 1 + + # Fix the collection DB + query = 'UPDATE tags SET _id = ? WHERE tagname = ?' + t = (db_collections[collection], collection, ) + cursor.execute(query, t) + + # Fix any references in existing collections + query = 'UPDATE booktags SET tag_id = ? WHERE tag_id = ?' + t = (db_collections[collection], collectionId,) + cursor.execute(query, t) + + self.set_database_sequence_id(connection, 'tags', sequence_max) + debug_print("Collection Sequence Max: %d, Source Id: %d"%(sequence_max,source_id)) + + # Fix up the collections table now... + sequence_dirty = 0 + sequence_max = sequence_min + + debug_print("Collections Sequence Min: %d, Source Id: %d"%(sequence_min,source_id)) + + query = 'SELECT _id FROM booktags' + cursor.execute(query) + + db_collection_pairs = [] + for i, row in enumerate(cursor): + db_collection_pairs.append(row[0]) + if row[0] < sequence_min: + sequence_dirty = 1 + else: + sequence_max = max(sequence_max, row[0]) + + if sequence_dirty == 1: + debug_print("Collections Sequence Dirty for Source Id: %d"%source_id) + sequence_max = sequence_max + 1 + for pairId in db_collection_pairs: + if pairId < sequence_min: + # Record the new Id and write it to the DB + query = 'UPDATE booktags SET _id = ? WHERE _id = ?' + t = (sequence_max, pairId,) + cursor.execute(query, t) + sequence_max = sequence_max + 1 + + self.set_database_sequence_id(connection, 'booktags', sequence_max) + debug_print("Collections Sequence Max: %d, Source Id: %d"%(sequence_max,source_id)) + + cursor.close() + return db_collections + + def update_device_collections(self, connection, booklist, collections, + source_id, dbpath): + + if collections: + db_collections = self.read_device_collections(connection, source_id, dbpath) + cursor = connection.cursor() + + for collection, books in collections.items(): + if collection not in db_collections: + query = 'INSERT INTO tags (tagname) VALUES (?)' + t = (collection,) + cursor.execute(query, t) + db_collections[collection] = self.get_lastrowid(cursor) + debug_print('Inserted New Collection: (%u) '%db_collections[collection] + collection) + + # Get existing books in collection + query = ''' + SELECT books.filename, book_id + FROM booktags + LEFT OUTER JOIN books + WHERE tag_id = ? AND books._id = booktags.book_id + ''' + t = (db_collections[collection],) + cursor.execute(query, t) + + db_books = {} + for i, row in enumerate(cursor): + db_books[row[0]] = row[1] + + for idx, book in enumerate(books): + if collection not in book.device_collections: + book.device_collections.append(collection) + if db_books.get(book.lpath, None) is None: + query = ''' + INSERT INTO booktags (tag_id, book_id) values (?,?) + ''' + t = (db_collections[collection], book.bookId) + cursor.execute(query, t) + debug_print('Inserted Book Into Collection: ' + + book.title + ' -> ' + collection) + + db_books[book.lpath] = None + + for bookPath, bookId in db_books.items(): + if bookId is not None: + query = ('DELETE FROM booktags ' + 'WHERE book_id = ? AND tag_id = ? ') + t = (bookId, db_collections[collection],) + cursor.execute(query, t) + debug_print('Deleted Book From Collection: ' + bookPath + + ' -> ' + collection) + + db_collections[collection] = None + + for collection, collectionId in db_collections.items(): + if collectionId is not None: + # Remove Books from Collection + query = ('DELETE FROM booktags ' + 'WHERE tag_id = ?') + t = (collectionId,) + cursor.execute(query, t) + # Remove Collection + query = ('DELETE FROM tags ' + 'WHERE _id = ?') + t = (collectionId,) + cursor.execute(query, t) + debug_print('Deleted Collection: ' + repr(collection)) + + cursor.close() + + def rebuild_collections(self, booklist, oncard): + debug_print('PALADIN: starting rebuild_collections') + + opts = self.settings() + if opts.extra_customization: + collections = [x.strip() for x in + opts.extra_customization[self.OPT_COLLECTIONS].split(',')] + else: + collections = [] + debug_print('PALADIN: collection fields:', collections) + + self.update_device_database(booklist, collections, oncard) + + debug_print('PALADIN: finished rebuild_collections')