From 28b998bb1907f38d92f1bd0f7358510e9b3a6264 Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Sat, 1 Mar 2014 16:31:37 +0100 Subject: [PATCH 1/4] 1) Undo setting _debug as synchronized. It was already synchronized inside itself, and adding the wrapper broke getting the correct method name. 2) Import the date methods once. 3) Make date_read sync'able in both directions --- .../devices/smart_device_app/driver.py | 57 ++++++++++--------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py index c981e83c0a..392c86209b 100644 --- a/src/calibre/devices/smart_device_app/driver.py +++ b/src/calibre/devices/smart_device_app/driver.py @@ -33,6 +33,7 @@ from calibre.ebooks.metadata.book.json_codec import JsonCodec from calibre.library import current_library_name from calibre.library.server import server_config as content_server_config from calibre.ptempfile import PersistentTemporaryFile +from calibre.utils.date import isoformat, now, parse_date from calibre.utils.ipc import eintr_retry_call from calibre.utils.config_base import tweaks from calibre.utils.filenames import ascii_filename as sanitize, shorten_components_to @@ -355,8 +356,6 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): self.debug_start_time = time.time() self.debug_time = time.time() - # This must be protected by a lock because it is called from three threads - @synchronous('sync_lock') def _debug(self, *args): # manual synchronization so we don't lose the calling method name import inspect @@ -388,7 +387,6 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): # copied from USBMS. Perhaps this could be a classmethod in usbms? def _update_driveinfo_record(self, dinfo, prefix, location_code, name=None): - from calibre.utils.date import isoformat, now import uuid if not isinstance(dinfo, dict): dinfo = {} @@ -695,7 +693,6 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): def _metadata_in_cache(self, uuid, ext_or_lpath, lastmod): try: - from calibre.utils.date import parse_date, now key = self._make_metadata_cache_key(uuid, ext_or_lpath) if isinstance(lastmod, unicode): if lastmod == 'None': @@ -796,7 +793,6 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): def _write_metadata_cache(self): self._debug() - from calibre.utils.date import now from calibre.utils.config import to_json cache_file_name = os.path.join(cache_dir(), 'wireless_device_' + self.device_uuid + @@ -833,7 +829,6 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): return key def _set_known_metadata(self, book, remove=False): - from calibre.utils.date import now lpath = book.lpath ext = os.path.splitext(lpath)[1] uuid = book.get('uuid', None) @@ -963,7 +958,6 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): @synchronous('sync_lock') def open(self, connected_device, library_uuid): - from calibre.utils.date import isoformat, now self._debug() if not self.is_connected: # We have been called to retry the connection. Give up immediately @@ -1522,30 +1516,37 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): # Both values are None. Do nothing return None - orig_is_read = book.get(self.is_read_sync_col, None) - if is_read != orig_is_read: - # The value in the device's is_read checkbox is not the same as the - # last one that came to the device from calibre during the last - # connect, meaning that the user changed it. Write the one from the - # checkbox to calibre's db. - changed_books = set() - is_read_date = book.get('_last_read_date_', None); - self._debug('standard update book', book.get('title', 'huh?'), 'to', - is_read, is_read_date) - if self.is_read_sync_col: - try: + changed_books = set() + try: + orig_is_read = book.get(self.is_read_sync_col, None) + if is_read != orig_is_read: + # The value in the device's is_read checkbox is not the same as the + # last one that came to the device from calibre during the last + # connect, meaning that the user changed it. Write the one from the + # checkbox to calibre's db. + self._debug('standard update book is_read', book.get('title', 'huh?'), + 'to', is_read) + if self.is_read_sync_col: changed_books = db.new_api.set_field(self.is_read_sync_col, {id_: is_read}) - except: - self._debug('setting read sync col tossed exception', - self.is_read_sync_col) - if self.is_read_date_sync_col: - try: + except: + self._debug('exception syncing is_read col', self.is_read_sync_col) + traceback.print_exc() + + try: + is_read_date = parse_date(book.get('_last_read_date_', None)); + orig_is_read_date = book.get(self.is_read_date_sync_col, None) + if is_read_date != orig_is_read_date: + self._debug('standard update book is_read_date', book.get('title', 'huh?'), + 'to', is_read_date) + if self.is_read_date_sync_col: changed_books |= db.new_api.set_field(self.is_read_date_sync_col, - {id_: is_read_date}) - except: - self._debug('setting read date sync col tossed exception', - self.is_read_date_sync_col) + {id_: is_read_date}) + except: + self._debug('Exception while syncing is_read_date', self.is_read_date_sync_col) + traceback.print_exc() + + if changed_books: return changed_books # The user might have changed the value in calibre. If so, that value From 130e6d964997baf8a81179ab8ac4509b727b6fc1 Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Sat, 1 Mar 2014 20:34:36 +0100 Subject: [PATCH 2/4] 1) Fix crash that is triggered by apps not using caching. 2) Don't refresh the GUI when requesting the transfer of unchanged metadata to the device. --- src/calibre/devices/smart_device_app/driver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py index 392c86209b..1bc025b3cb 100644 --- a/src/calibre/devices/smart_device_app/driver.py +++ b/src/calibre/devices/smart_device_app/driver.py @@ -1251,7 +1251,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): book = self.json_codec.raw_to_book(result, SDBook, self.PREFIX) book.set('_is_read_', result.get('_is_read_', None)) book.set('_is_read_changed_', result.get('_is_read_changed_', None)) - book.set('_last_read_date_', r.get('_last_read_date_', None)) + book.set('_last_read_date_', result.get('_last_read_date_', None)) bl.add_book(book, replace_metadata=True) if '_new_book_' in result: book.set('_new_book_', True) @@ -1512,7 +1512,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): book.set('_force_send_metadata_', True) self._debug('special update book', book.get('title', 'huh?'), 'to', calibre_val) - return set(id_) + return set() # Both values are None. Do nothing return None From 91adef995fbe87676e0f1c4a5ec68ebb348fab7d Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Sun, 2 Mar 2014 09:33:54 +0100 Subject: [PATCH 3/4] 1) Put the embedded import statements back 2) Fix syncing not to attempt to sync if either column is bad 3) Make independent syncing of the date read work --- .../devices/smart_device_app/driver.py | 83 +++++++++++++------ 1 file changed, 59 insertions(+), 24 deletions(-) diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py index 1bc025b3cb..f76dcd1daf 100644 --- a/src/calibre/devices/smart_device_app/driver.py +++ b/src/calibre/devices/smart_device_app/driver.py @@ -33,7 +33,6 @@ from calibre.ebooks.metadata.book.json_codec import JsonCodec from calibre.library import current_library_name from calibre.library.server import server_config as content_server_config from calibre.ptempfile import PersistentTemporaryFile -from calibre.utils.date import isoformat, now, parse_date from calibre.utils.ipc import eintr_retry_call from calibre.utils.config_base import tweaks from calibre.utils.filenames import ascii_filename as sanitize, shorten_components_to @@ -387,6 +386,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): # copied from USBMS. Perhaps this could be a classmethod in usbms? def _update_driveinfo_record(self, dinfo, prefix, location_code, name=None): + from calibre.utils.date import isoformat, now import uuid if not isinstance(dinfo, dict): dinfo = {} @@ -692,6 +692,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): return None def _metadata_in_cache(self, uuid, ext_or_lpath, lastmod): + from calibre.utils.date import now, parse_date try: key = self._make_metadata_cache_key(uuid, ext_or_lpath) if isinstance(lastmod, unicode): @@ -792,6 +793,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): traceback.print_exc() def _write_metadata_cache(self): + from calibre.utils.date import now self._debug() from calibre.utils.config import to_json cache_file_name = os.path.join(cache_dir(), @@ -829,6 +831,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): return key def _set_known_metadata(self, book, remove=False): + from calibre.utils.date import now lpath = book.lpath ext = os.path.splitext(lpath)[1] uuid = book.get('uuid', None) @@ -958,6 +961,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): @synchronous('sync_lock') def open(self, connected_device, library_uuid): + from calibre.utils.date import isoformat, now self._debug() if not self.is_connected: # We have been called to retry the connection. Give up immediately @@ -1460,70 +1464,100 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): @synchronous('sync_lock') def synchronize_with_db(self, db, id_, book): + from calibre.utils.date import parse_date, UNDEFINED_DATE def show_message(message): self._call_client("DISPLAY_MESSAGE", {'messageKind': self.MESSAGE_SHOW_TOAST, 'message': message}) - if not (self.is_read_sync_col or self.is_read_date_sync_col): - # Not syncing + if self.have_bad_sync_columns or not (self.is_read_sync_col or + self.is_read_date_sync_col): + # Not syncing or sync columns are invalid return None + # Check the validity of the columns once per connection. We do it + # here because we have access to the db to get field_metadata if not self.have_checked_sync_columns: - # Check the validity of the columns once per connection. We do it - # here because we have access to the db to get field_metadata fm = db.field_metadata.custom_field_metadata() if self.is_read_sync_col: if self.is_read_sync_col not in fm: self._debug('is_read_sync_col not in field_metadata') show_message(_("The read sync column %s is " "not in calibre's library")%self.is_read_sync_col) + self.have_bad_sync_columns = True elif fm[self.is_read_sync_col]['datatype'] != 'bool': self._debug('is_read_sync_col not bool type') show_message(_("The read sync column %s is " "not a Yes/No column")%self.is_read_sync_col) + self.have_bad_sync_columns = True if self.is_read_date_sync_col: if self.is_read_date_sync_col not in fm: self._debug('is_read_date_sync_col not in field_metadata') show_message(_("The read date sync column %s is " "not in calibre's library")%self.is_read_date_sync_col) + self.have_bad_sync_columns = True elif fm[self.is_read_date_sync_col]['datatype'] != 'datetime': self._debug('is_read_date_sync_col not date type') show_message(_("The read date sync column %s is " "not a Date column")%self.is_read_date_sync_col) + self.have_bad_sync_columns = True self.have_checked_sync_columns = True + if self.have_bad_sync_columns: + return None is_changed = book.get('_is_read_changed_', None); is_read = book.get('_is_read_', None) + # This returns UNDEFINED_DATE if the value is None + is_read_date = parse_date(book.get('_last_read_date_', None)); + value_to_return = None - if is_changed == 2 and is_read is None: + if is_changed == 2: # This is a special case where the user just set the sync column. In # this case the device value wins if it is not None by falling # through to the normal sync situation below, otherwise the calibre - # value wins. - calibre_val = db.new_api.field_for(self.is_read_sync_col, - id_, default_value=None) - if calibre_val is not None: - # This will force the metadata for the book to be sent . Note - # that because the devices last_read date is one-way sync, this - # could leave an empty date in the device. - book.set('_force_send_metadata_', True) - self._debug('special update book', book.get('title', 'huh?'), - 'to', calibre_val) - return set() - # Both values are None. Do nothing - return None + # value wins. The orig_* values are set to None to force the normal + # sync code to actually sync because the values are different + orig_is_read_date = UNDEFINED_DATE + orig_is_read = None + if is_read is None: + calibre_val = db.new_api.field_for(self.is_read_sync_col, + id_, default_value=None) + if calibre_val is not None: + # This forces the metadata for the book to be sent to the + # device even if the mod dates haven't changed. + book.set('_force_send_metadata_', True) + self._debug('special update is_read', book.get('title', 'huh?'), + 'to', calibre_val) + value_to_return = set() + + if is_read_date == UNDEFINED_DATE: + calibre_val = db.new_api.field_for(self.is_read_date_sync_col, + id_, default_value=None) + if calibre_val != UNDEFINED_DATE: + book.set('_force_send_metadata_', True) + self._debug('special update is_read_date', book.get('title', 'huh?'), + 'to', calibre_val) + value_to_return = set() + # Fall through to the normal sync. At this point either the is_read* + # values are different from the orig_is_read* which will cause a + # sync below, or they are both None which will cause the code below + # to do nothing. If either of the calibre data fields were set, the + # method will return set(), which will force updated metadata to be + # given back to the device, effectively forcing the sync of the + # calibre values back to the device. + else: + orig_is_read = book.get(self.is_read_sync_col, None) + orig_is_read_date = book.get(self.is_read_date_sync_col, None) changed_books = set() try: - orig_is_read = book.get(self.is_read_sync_col, None) if is_read != orig_is_read: # The value in the device's is_read checkbox is not the same as the # last one that came to the device from calibre during the last # connect, meaning that the user changed it. Write the one from the - # checkbox to calibre's db. + # device to calibre's db. self._debug('standard update book is_read', book.get('title', 'huh?'), 'to', is_read) if self.is_read_sync_col: @@ -1534,8 +1568,6 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): traceback.print_exc() try: - is_read_date = parse_date(book.get('_last_read_date_', None)); - orig_is_read_date = book.get(self.is_read_date_sync_col, None) if is_read_date != orig_is_read_date: self._debug('standard update book is_read_date', book.get('title', 'huh?'), 'to', is_read_date) @@ -1547,6 +1579,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): traceback.print_exc() if changed_books: + # One of the two values was synced, giving a list of changed books. + # Return that. return changed_books # The user might have changed the value in calibre. If so, that value @@ -1554,7 +1588,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): # updated value has already been synced and so will also be sent, the # device should put the calibre value into its checkbox (or whatever it # uses) - return None + return value_to_return @synchronous('sync_lock') def startup(self): @@ -1582,6 +1616,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): self.is_read_sync_col = None self.is_read_date_sync_col = None self.have_checked_sync_columns = False + self.have_bad_sync_columns = False message = None compression_quality_ok = True From 011a71efe9fa07b0b69fc5b4c98d5d5de1393845 Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Sun, 2 Mar 2014 17:41:39 +0100 Subject: [PATCH 4/4] Add ability for device to do a version check for syncing. --- src/calibre/devices/smart_device_app/driver.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py index f76dcd1daf..2bf368e3f7 100644 --- a/src/calibre/devices/smart_device_app/driver.py +++ b/src/calibre/devices/smart_device_app/driver.py @@ -988,7 +988,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): 'currentLibraryUUID': library_uuid, 'pubdateFormat': tweaks['gui_pubdate_display_format'], 'timestampFormat': tweaks['gui_timestamp_display_format'], - 'lastModifiedFormat': tweaks['gui_last_modified_display_format']}) + 'lastModifiedFormat': tweaks['gui_last_modified_display_format'], + 'calibre_version': numeric_version}) if opcode != 'OK': # Something wrong with the return. Close the socket # and continue. @@ -1087,7 +1088,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): self.is_read_sync_col = result.get('isReadSyncCol', None) self._debug('Device is_read sync col', self.is_read_sync_col) - self.is_read_date_sync_col = result.get('isReadDateSyncCol', False) + self.is_read_date_sync_col = result.get('isReadDateSyncCol', None) self._debug('Device is_read_date sync col', self.is_read_date_sync_col) if password: @@ -1207,7 +1208,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): {'canStream':True, 'canScan':True, 'willUseCachedMetadata': self.client_can_use_metadata_cache, - 'supportsSync': True}) + 'supportsSync': (self.is_read_sync_col or + self.is_read_date_sync_col)}) bl = CollectionsBookList(None, self.PREFIX, self.settings) if opcode == 'OK': count = result['count'] @@ -1307,7 +1309,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): self._call_client('SEND_BOOKLISTS', {'count': count, 'collections': coldict, 'willStreamMetadata': True, - 'supportsSync': True}, + 'supportsSync': (self.is_read_sync_col or + self.is_read_date_sync_col)}, wait_for_response=False) if count: @@ -1316,7 +1319,9 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): self._set_known_metadata(book) opcode, result = self._call_client( 'SEND_BOOK_METADATA', - {'index': i, 'count': count, 'data': book, 'supportsSync': True}, + {'index': i, 'count': count, 'data': book, + 'supportsSync': (self.is_read_sync_col or + self.is_read_date_sync_col)}, print_debug_info=False, wait_for_response=False)