From 1ed44f06585159d03053e18b94513cd27a35dd67 Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Thu, 4 Sep 2014 10:49:53 +0200 Subject: [PATCH 1/2] Add ability for devices to request that book format files be sent to the device. The wireless device driver uses this to ensure (optionally) that the books on the device are up-to-date with the books in calibre. --- src/calibre/devices/interface.py | 23 ++++++++--- .../devices/smart_device_app/driver.py | 41 ++++++++++++++++--- src/calibre/gui2/device.py | 41 ++++++++++++++++++- 3 files changed, 92 insertions(+), 13 deletions(-) diff --git a/src/calibre/devices/interface.py b/src/calibre/devices/interface.py index 610af7a687..9550394404 100644 --- a/src/calibre/devices/interface.py +++ b/src/calibre/devices/interface.py @@ -732,12 +732,23 @@ class DevicePlugin(Plugin): a book in calibre's db. The method is responsible for syncronizing data from the device to calibre's db (if needed). - The method must return a set of calibre book ids changed if calibre's - database was changed, None if the database was not changed. If the - method returns an empty set then the metadata for the book on the - device is updated with calibre's metadata and given back to the device, - but no GUI refresh of that book is done. This is useful when the calire - data is correct but must be sent to the device. + The method must return a two-value tuple. The first value is a set of + calibre book ids changed if calibre's database was changed or None if the + database was not changed. If the first value is an empty set then the + metadata for the book on the device is updated with calibre's metadata + and given back to the device, but no GUI refresh of that book is done. + This is useful when the calire data is correct but must be sent to the + device. + + The second value in the tuple specifies whether a book format should be + sent to the device. The intent is to permit verifying that the book on + the device is the same as the book in calibre. Return None if no book is + to be sent, otherwise return the base file name on the device (a string + like foobar.epub). Be sure to include the extension in the name. The + device subsystem will construct a send_books job for all books with not- + None returned values. Note: other than to later retrieve the extension, + the name is ignored in cases where the device uses a template to + generate the file name, which most do. Extremely important: this method is called on the GUI thread. It must be threadsafe with respect to the device manager's thread. diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py index 6e10266992..d9f0446f3b 100644 --- a/src/calibre/devices/smart_device_app/driver.py +++ b/src/calibre/devices/smart_device_app/driver.py @@ -999,7 +999,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): 'pubdateFormat': tweaks['gui_pubdate_display_format'], 'timestampFormat': tweaks['gui_timestamp_display_format'], 'lastModifiedFormat': tweaks['gui_last_modified_display_format'], - 'calibre_version': numeric_version}) + 'calibre_version': numeric_version, + 'canSupportUpdateBooks': True}) if opcode != 'OK': # Something wrong with the return. Close the socket # and continue. @@ -1043,6 +1044,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): self._debug('Can send OK to sendbook', self.can_send_ok_to_sendbook) self.can_accept_library_info = result.get('canAcceptLibraryInfo', False) self._debug('Can accept library info', self.can_accept_library_info) + self.will_ask_for_update_books = result.get('willAskForUpdateBooks', False) + self._debug('Will ask for update books', self.will_ask_for_update_books) if not self.settings().extra_customization[self.OPT_USE_METADATA_CACHE]: self.client_can_use_metadata_cache = False @@ -1223,7 +1226,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): 'canScan':True, 'willUseCachedMetadata': self.client_can_use_metadata_cache, 'supportsSync': (bool(self.is_read_sync_col) or - bool(self.is_read_date_sync_col))}) + bool(self.is_read_date_sync_col)), + 'canSupportBookFormatSync': True}) bl = CollectionsBookList(None, self.PREFIX, self.settings) if opcode == 'OK': count = result['count'] @@ -1252,6 +1256,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): book.set('_is_read_', r.get('_is_read_', None)) book.set('_sync_type_', r.get('_sync_type_', None)) book.set('_last_read_date_', r.get('_last_read_date_', None)) + book.set('_format_mtime_', r.get('_format_mtime_', None)) else: books_to_send.append(r['priKey']) @@ -1529,6 +1534,30 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): def specialize_global_preferences(self, device_prefs): device_prefs.set_overrides(manage_device_metadata='on_connect') + def _check_if_format_send_needed(self, db, id_, book): + if not self.will_ask_for_update_books: + return None + + from calibre.utils.date import parse_date, now, isoformat + try: + if not hasattr(book, '_format_mtime_'): + return None + + cc_mtime = parse_date(book.get('_format_mtime_'), as_utc=False) + ext = posixpath.splitext(book.lpath)[1][1:] + fmt_metadata = db.new_api.format_metadata(id_, ext) + if fmt_metadata: + calibre_mtime = fmt_metadata['mtime'] + self._debug(book.title, 'cal_mtime', calibre_mtime, 'cc_mtime', cc_mtime) + if cc_mtime < calibre_mtime: + book.set('_format_mtime_', isoformat(now())) + return posixpath.basename(book.lpath) + except: + self._debug('exception checking if must send format', book.title) + traceback.print_exc() + return None + + @synchronous('sync_lock') def synchronize_with_db(self, db, id_, book): from calibre.utils.date import parse_date, is_date_undefined @@ -1540,7 +1569,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): if self.have_bad_sync_columns or not (self.is_read_sync_col or self.is_read_date_sync_col): # Not syncing or sync columns are invalid - return None + return (None, self._check_if_format_send_needed(db, id_, book)) # Check the validity of the columns once per connection. We do it # here because we have access to the db to get field_metadata @@ -1572,7 +1601,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): self.have_checked_sync_columns = True if self.have_bad_sync_columns: - return None + return (None, self._check_if_format_send_needed(db, id_, book)) sync_type = book.get('_sync_type_', None) # We need to check if our attributes are in the book. If they are not @@ -1697,14 +1726,14 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): if changed_books or force_return_changed_books: # One of the two values was synced, giving a (perhaps empty) list of # changed books. Return that. - return changed_books + return (changed_books, self._check_if_format_send_needed(db, id_, book)) # Nothing was synced. The user might have changed the value in calibre. # If so, that value will be sent to the device in the normal way. Note # that because any updated value has already been synced and so will # also be sent, the device should put the calibre value into its # checkbox (or whatever it uses) - return None + return (None, self._check_if_format_send_needed(db, id_, book)) @synchronous('sync_lock') def startup(self): diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index e5e8baef19..0bebd83a75 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -11,6 +11,8 @@ from PyQt5.Qt import ( QObject, QVBoxLayout, QDialogButtonBox, QCursor, QCoreApplication, QApplication, QEventLoop) +from calibre import isbytestring +from calibre.constants import filesystem_encoding from calibre.customize.ui import (available_input_formats, available_output_formats, device_plugins, disabled_device_plugins) from calibre.devices.interface import DevicePlugin @@ -34,6 +36,7 @@ from calibre.constants import DEBUG from calibre.utils.config import tweaks, device_prefs from calibre.utils.magick.draw import thumbnail from calibre.library.save_to_disk import find_plugboard +from calibre.ptempfile import PersistentTemporaryFile # }}} class DeviceJob(BaseJob): # {{{ @@ -1774,6 +1777,7 @@ class DeviceMixin(object): # {{{ self.db_book_uuid_cache = db_book_uuid_cache book_ids_to_refresh = set() + book_formats_to_send = list() def update_book(id_, book) : if not update_metadata: @@ -1789,7 +1793,10 @@ class DeviceMixin(object): # {{{ return False if self.device_manager.device is not None: - set_of_ids = self.device_manager.device.synchronize_with_db(db, id_, book) + set_of_ids, fmt_name = \ + self.device_manager.device.synchronize_with_db(db, id_, book) + if fmt_name is not None: + book_formats_to_send.append((id_, fmt_name)) if set_of_ids is not None: book_ids_to_refresh.update(set_of_ids) return True @@ -1913,6 +1920,38 @@ class DeviceMixin(object): # {{{ # This shouldn't ever happen, but just in case ... traceback.print_exc() + # Sync books if necessary + try: + files, names, metadata = [], [], [] + for id_, fmt_name in book_formats_to_send: + if DEBUG: + prints('DeviceJob: Syncing book. id:', id_, 'name from device', fmt_name) + ext = os.path.splitext(fmt_name)[1][1:] + fmt_info = db.new_api.format_metadata(id_, ext) + if fmt_info: + try: + pt = PersistentTemporaryFile(suffix='caltmpfmt.'+ext) + db.new_api.copy_format_to(id_, ext, pt) + pt.close() + def to_uni(x): + if isbytestring(x): + x = x.decode(filesystem_encoding) + return x + files.append(to_uni(os.path.abspath(pt.name))) + names.append(fmt_name) + metadata.append(db.new_api.get_metadata(id_, get_cover=True)) + except: + prints('Problem creating temporary file for', fmt_name) + traceback.print_exc() + else: + if DEBUG: + prints("DeviceJob: book doesn't have that format") + if files: + self.upload_books(files, names, metadata) + except: + # Shouldn't ever happen, but just in case + traceback.print_exc() + if DEBUG: prints('DeviceJob: set_books_in_library finished: time=', time.time() - start_time) From 741bfed8f8bf6335821e615f822e60abd953c291 Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Thu, 4 Sep 2014 11:43:14 +0200 Subject: [PATCH 2/2] Support a device option to temp-mark books that have had their is-read information changed to what is on the device. --- .../devices/smart_device_app/driver.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py index d9f0446f3b..2ca7df6926 100644 --- a/src/calibre/devices/smart_device_app/driver.py +++ b/src/calibre/devices/smart_device_app/driver.py @@ -1018,6 +1018,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): self._close_device_socket() return False + # Set up to recheck the sync columns + self.have_checked_sync_columns = False client_can_stream_books = result.get('canStreamBooks', False) self._debug('Device can stream books', client_can_stream_books) client_can_stream_metadata = result.get('canStreamMetadata', False) @@ -1046,6 +1048,10 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): self._debug('Can accept library info', self.can_accept_library_info) self.will_ask_for_update_books = result.get('willAskForUpdateBooks', False) self._debug('Will ask for update books', self.will_ask_for_update_books) + self.set_temp_mark_when_syncing_read = \ + result.get('setTempMarkWhenReadInfoSynced', False) + self._debug('Will set temp mark when syncing read', + self.set_temp_mark_when_syncing_read) if not self.settings().extra_customization[self.OPT_USE_METADATA_CACHE]: self.client_can_use_metadata_cache = False @@ -1603,6 +1609,11 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): if self.have_bad_sync_columns: return (None, self._check_if_format_send_needed(db, id_, book)) + # if we are marking synced books, clear all the current marks + if self.set_temp_mark_when_syncing_read: + self._debug('clearing temp marks') + db.set_marked_ids(()) + sync_type = book.get('_sync_type_', None) # We need to check if our attributes are in the book. If they are not # then this is metadata coming from calibre to the device for the first @@ -1653,6 +1664,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): book.get('title', 'huh?'), 'to', is_read, calibre_val) changed_books = db.new_api.set_field(self.is_read_sync_col, {id_: is_read}) + if self.set_temp_mark_when_syncing_read: + db.data.toggle_marked_ids({id_}) elif calibre_val is not None: # Calibre value wins. Force the metadata for the # book to be sent to the device even if the mod @@ -1677,6 +1690,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): book.get('title', 'huh?'), 'to', is_read_date, calibre_val) changed_books |= db.new_api.set_field(self.is_read_date_sync_col, {id_: is_read_date}) + if self.set_temp_mark_when_syncing_read: + db.data.toggle_marked_ids({id_}) elif calibre_val is not None: self._debug('special update is_read_date to calibre value', book.get('title', 'huh?'), 'to', calibre_val) @@ -1703,6 +1718,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): 'to', is_read, 'was', orig_is_read) changed_books = db.new_api.set_field(self.is_read_sync_col, {id_: is_read}) + if self.set_temp_mark_when_syncing_read: + db.data.toggle_marked_ids({id_}) except: self._debug('exception standard syncing is_read', self.is_read_sync_col) traceback.print_exc() @@ -1718,6 +1735,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): 'to', is_read_date, 'was', orig_is_read_date) changed_books |= db.new_api.set_field(self.is_read_date_sync_col, {id_: is_read_date}) + if self.set_temp_mark_when_syncing_read: + db.data.toggle_marked_ids({id_}) except: self._debug('Exception standard syncing is_read_date', self.is_read_date_sync_col)