Infrastructure for allowing a device to "synchronize" data with calibre's database. The exact meaning of "synchronize" is left to the device. This implementation adds almost zero overhead (nanoseconds per book) if the device driver does not do any synchronization.

During book matching in gui2.device, whenever a book on the device is matched to a book in the DB, the (new) device method synchronize_with_db is called. It can copy information from the device to calibre's db or vice versa. If it returns True then the book's metadata is updated from calibre's DB even if it otherwise would not be. This permits the synchronization process to request that data in calibre be unconditionally sent to the device.

Note that the synchronize_with_db method is called on the GUI thread. It is the device driver's responsibility to be threadsafe with respect to the device manager thread.

A lightly-tested implementation in the wireless driver is included to show how the new method might be used. It implements syncing a yes/no "is read" column and a "read date" column from the device to calibre.

src/calibre/devices/interface.py

@@ -718,6 +718,23 @@ class DevicePlugin(Plugin):
         '''
         return False
 
+    def synchronize_with_db(self, db, book_id, book_metadata):
+        '''
+        Called during book matching when a book on the device is matched with
+        a book in calibre's db. The method is responsible for syncronizing
+        data from the device to calibre's db (if needed).
+
+        The method must return True if either calibre's database or the device
+        book's metadata were changed, False otherwise.
+
+        Extremely important: this method is called on the GUI thread. It must
+        be threadsafe with respect to the device manager's thread.
+
+        book_id: the calibre id for the book in the database.
+        book_metadata: the Metadata object for the book coming from the device.
+        '''
+        return False
+
 class BookList(list):
     '''
     A list of books. Each Book object must have the fields

src/calibre/devices/smart_device_app/driver.py

@@ -1080,6 +1080,12 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
             elif hasattr(self, 'THUMBNAIL_WIDTH'):
                     delattr(self, 'THUMBNAIL_WIDTH')
 
+            self.is_read_sync_col = result.get('isReadSyncCol', None)
+            self._debug('Device is_read sync col', self.is_read_sync_col)
+
+            self.is_read_date_sync_col = result.get('isReadDateSyncCol', False)
+            self._debug('Device is_read_date sync col', self.is_read_date_sync_col)
+
             if password:
                 returned_hash = result.get('passwordHash', None)
                 if result.get('passwordHash', None) is None:
@@ -1196,7 +1202,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
         opcode, result = self._call_client('GET_BOOK_COUNT',
                             {'canStream':True,
                              'canScan':True,
-                             'willUseCachedMetadata': self.client_can_use_metadata_cache})
+                             'willUseCachedMetadata': self.client_can_use_metadata_cache,
+                             'supportsSync': True})
         bl = CollectionsBookList(None, self.PREFIX, self.settings)
         if opcode == 'OK':
             count = result['count']
@@ -1219,6 +1226,9 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                                                        r['last_modified'])
                     if book:
                         bl.add_book(book, replace_metadata=True)
+                        book.set('_is_read_', r.get('_is_read_', None))
+                        book.set('_is_read_changed_', r.get('_is_read_changed_', None))
+                        book.set('_last_read_date_', r.get('_last_read_date_', None))
                     else:
                         books_to_send.append(r['priKey'])
 
@@ -1239,6 +1249,9 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                     if '_series_sort_' in result:
                         del result['_series_sort_']
                     book = self.json_codec.raw_to_book(result, SDBook, self.PREFIX)
+                    book.set('_is_read_', result.get('_is_read_', None))
+                    book.set('_is_read_changed_', result.get('_is_read_changed_', None))
+                    book.set('_last_read_date_', r.get('_last_read_date_', None))
                     bl.add_book(book, replace_metadata=True)
                     if '_new_book_' in result:
                         book.set('_new_book_', True)
@@ -1288,7 +1301,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
         count = len(books_to_send)
         self._call_client('SEND_BOOKLISTS', {'count': count,
                      'collections': coldict,
-                     'willStreamMetadata': True},
+                     'willStreamMetadata': True,
+                     'supportsSync': True},
                      wait_for_response=False)
 
         if count:
@@ -1297,7 +1311,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                 self._set_known_metadata(book)
                 opcode, result = self._call_client(
                         'SEND_BOOK_METADATA',
-                        {'index': i, 'count': count, 'data': book},
+                        {'index': i, 'count': count, 'data': book, 'supportsSync': True},
                         print_debug_info=False,
                         wait_for_response=False)
 
@@ -1443,6 +1457,41 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
     def specialize_global_preferences(self, device_prefs):
         device_prefs.set_overrides(manage_device_metadata='on_connect')
 
+    @synchronous('sync_lock')
+    def synchronize_with_db(self, db, id_, book):
+        is_changed = book.get('_is_read_changed_', None);
+        if is_changed:
+            made_changes = False
+            # is_read_changed == 1: standard sync. Update calibre with the new value
+            # is_read_changed == 2: special first-time sync after specifying the
+            # column. Update calibre if CC's value is not null, else update CC
+            # if calibre's value is not null
+            val = book.get('_is_read_', None)
+            if is_changed == 1 or (is_changed == 2 and val):
+                self._debug('standard update book', book.get('title', 'huh?'), 'to', val)
+                if self.is_read_sync_col:
+                    db.new_api.set_field(self.is_read_sync_col, {id_: val})
+                    made_changes = True
+                if self.is_read_date_sync_col:
+                    db.new_api.set_field(self.is_read_date_sync_col,
+                                         {id_: book.get('_last_read_date_', None)})
+                    made_changes = True
+            elif self.is_read_sync_col and is_changed == 2 and not val:
+                calibre_val = db.new_api.field_for(self.is_read_sync_col,
+                                                   id_, default_value=None)
+                if calibre_val:
+                    from calibre.utils.date import UNDEFINED_DATE
+                    # This will force the metadata for the book to be sent even
+                    # if the last_mod dates matched before. Note that because
+                    # CC's last_read date is one-way sync, this could leave an
+                    # empty date in CC.
+                    self._debug('special update book', book.get('title', 'huh?'),
+                                'to', calibre_val)
+                    book.set('last_modified', UNDEFINED_DATE)
+            book.set('_is_read_changed_', None)
+            return made_changes
+        return False
+
     @synchronous('sync_lock')
     def startup(self):
         self.listen_socket = None
@@ -1466,6 +1515,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
         self.noop_counter = 0
         self.connection_attempts = {}
         self.client_wants_uuid_file_names = False
+        self.is_read_sync_col = None
+        self.is_read_date_sync_col = None
 
         message = None
         compression_quality_ok = True

src/calibre/gui2/device.py

@@ -1769,10 +1769,14 @@ class DeviceMixin(object):  # {{{
         def updateq(id_, book):
             try:
                 return (update_metadata and
-                        (db.metadata_last_modified(id_, index_is_id=True) !=
-                         getattr(book, 'last_modified', None) or
-                         (isinstance(getattr(book, 'thumbnail', None), (list, tuple))
-                          and max(book.thumbnail[0], book.thumbnail[1]) != desired_thumbnail_height
+                        (
+                         (self.device_manager.device is not None and
+                               self.device_manager.device.synchronize_with_db(db, id_, book)) or
+                         (db.metadata_last_modified(id_, index_is_id=True) !=
+                          getattr(book, 'last_modified', None) or
+                          (isinstance(getattr(book, 'thumbnail', None), (list, tuple))
+                           and max(book.thumbnail[0], book.thumbnail[1]) != desired_thumbnail_height
+                          )
                          )
                         )
                        )