Add a mechanism for device drivers to update books in the calibre database

Merge branch 'master' of https://github.com/cbhaley/calibre

src/calibre/devices/interface.py

@@ -718,6 +718,27 @@ class DevicePlugin(Plugin):
         '''
         return False
 
+    def synchronize_with_db(self, db, book_id, book_metadata):
+        '''
+        Called during book matching when a book on the device is matched with
+        a book in calibre's db. The method is responsible for syncronizing
+        data from the device to calibre's db (if needed).
+
+        The method must return a set of calibre book ids changed if calibre's
+        database was changed, None if the database was not changed. If the
+        method returns an empty set then the metadata for the book on the
+        device is updated with calibre's metadata and given back to the device,
+        but no GUI refresh of that book is done. This is useful when the calire
+        data is correct but must be sent to the device.
+
+        Extremely important: this method is called on the GUI thread. It must
+        be threadsafe with respect to the device manager's thread.
+
+        book_id: the calibre id for the book in the database.
+        book_metadata: the Metadata object for the book coming from the device.
+        '''
+        return None
+
 class BookList(list):
     '''
     A list of books. Each Book object must have the fields

src/calibre/devices/smart_device_app/driver.py

@@ -1080,6 +1080,12 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
             elif hasattr(self, 'THUMBNAIL_WIDTH'):
                     delattr(self, 'THUMBNAIL_WIDTH')
 
+            self.is_read_sync_col = result.get('isReadSyncCol', None)
+            self._debug('Device is_read sync col', self.is_read_sync_col)
+
+            self.is_read_date_sync_col = result.get('isReadDateSyncCol', False)
+            self._debug('Device is_read_date sync col', self.is_read_date_sync_col)
+
             if password:
                 returned_hash = result.get('passwordHash', None)
                 if result.get('passwordHash', None) is None:
@@ -1196,7 +1202,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
         opcode, result = self._call_client('GET_BOOK_COUNT',
                             {'canStream':True,
                              'canScan':True,
-                             'willUseCachedMetadata': self.client_can_use_metadata_cache})
+                             'willUseCachedMetadata': self.client_can_use_metadata_cache,
+                             'supportsSync': True})
         bl = CollectionsBookList(None, self.PREFIX, self.settings)
         if opcode == 'OK':
             count = result['count']
@@ -1219,6 +1226,9 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                                                        r['last_modified'])
                     if book:
                         bl.add_book(book, replace_metadata=True)
+                        book.set('_is_read_', r.get('_is_read_', None))
+                        book.set('_is_read_changed_', r.get('_is_read_changed_', None))
+                        book.set('_last_read_date_', r.get('_last_read_date_', None))
                     else:
                         books_to_send.append(r['priKey'])
 
@@ -1239,6 +1249,9 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                     if '_series_sort_' in result:
                         del result['_series_sort_']
                     book = self.json_codec.raw_to_book(result, SDBook, self.PREFIX)
+                    book.set('_is_read_', result.get('_is_read_', None))
+                    book.set('_is_read_changed_', result.get('_is_read_changed_', None))
+                    book.set('_last_read_date_', r.get('_last_read_date_', None))
                     bl.add_book(book, replace_metadata=True)
                     if '_new_book_' in result:
                         book.set('_new_book_', True)
@@ -1282,13 +1295,15 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
         # given back by "books", and one that has been plugboarded.
         books_to_send = []
         for book in booklists[0]:
-            if not self._metadata_already_on_device(book):
+            if (book.get('_force_send_metadata_', None) or
+                    not self._metadata_already_on_device(book)):
                 books_to_send.append(book)
 
         count = len(books_to_send)
         self._call_client('SEND_BOOKLISTS', {'count': count,
                      'collections': coldict,
-                     'willStreamMetadata': True},
+                     'willStreamMetadata': True,
+                     'supportsSync': True},
                      wait_for_response=False)
 
         if count:
@@ -1297,7 +1312,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                 self._set_known_metadata(book)
                 opcode, result = self._call_client(
                         'SEND_BOOK_METADATA',
-                        {'index': i, 'count': count, 'data': book},
+                        {'index': i, 'count': count, 'data': book, 'supportsSync': True},
                         print_debug_info=False,
                         wait_for_response=False)
 
@@ -1443,6 +1458,66 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
     def specialize_global_preferences(self, device_prefs):
         device_prefs.set_overrides(manage_device_metadata='on_connect')
 
+    @synchronous('sync_lock')
+    def synchronize_with_db(self, db, id_, book):
+        if not (self.is_read_sync_col or self.is_read_date_sync_col):
+            # Not syncing
+            return None
+
+        is_changed = book.get('_is_read_changed_', None);
+        is_read = book.get('_is_read_', None)
+
+        if is_changed == 2 and is_read is None:
+            # This is a special case where the user just set the sync column. In
+            # this case the device value wins if it is not None by falling
+            # through to the normal sync situation below, otherwise the calibre
+            # value wins.
+            calibre_val = db.new_api.field_for(self.is_read_sync_col,
+                                               id_, default_value=None)
+            if calibre_val is not None:
+                # This will force the metadata for the book to be sent . Note
+                # that because the devices last_read date is one-way sync, this
+                # could leave an empty date in the device.
+                book.set('_force_send_metadata_', True)
+                self._debug('special update book', book.get('title', 'huh?'),
+                            'to', calibre_val)
+                return set(id_)
+            # Both values are None. Do nothing
+            return None
+
+        orig_is_read = book.get(self.is_read_sync_col, None)
+        if is_read != orig_is_read:
+            # The value in the device's is_read checkbox is not the same as the
+            # last one that came to the device from calibre during the last
+            # connect, meaning that the user changed it. Write the one from the
+            # checkbox to calibre's db.
+            changed_books = set()
+            is_read_date = book.get('_last_read_date_', None);
+            self._debug('standard update book', book.get('title', 'huh?'), 'to',
+                        is_read, is_read_date)
+            if self.is_read_sync_col:
+                try:
+                    changed_books = db.new_api.set_field(self.is_read_sync_col,
+                                                         {id_: is_read})
+                except:
+                    self._debug('setting read sync col tossed exception',
+                                self.is_read_sync_col)
+            if self.is_read_date_sync_col:
+                try:
+                    changed_books |= db.new_api.set_field(self.is_read_date_sync_col,
+                                              {id_: is_read_date})
+                except:
+                    self._debug('setting read date sync col tossed exception',
+                                self.is_read_date_sync_col)
+            return changed_books
+
+        # The user might have changed the value in calibre. If so, that value
+        # will be sent to the device in the normal way. Note that because any
+        # updated value has already been synced and so will also be sent, the
+        # device should put the calibre value into its checkbox (or whatever it
+        # uses)
+        return None
+
     @synchronous('sync_lock')
     def startup(self):
         self.listen_socket = None
@@ -1466,6 +1541,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
         self.noop_counter = 0
         self.connection_attempts = {}
         self.client_wants_uuid_file_names = False
+        self.is_read_sync_col = None
+        self.is_read_date_sync_col = None
 
         message = None
         compression_quality_ok = True

src/calibre/gui2/device.py

@@ -1751,12 +1751,7 @@ class DeviceMixin(object):  # {{{
             self.db_book_title_cache = db_book_title_cache
             self.db_book_uuid_cache = db_book_uuid_cache
 
-        # Now iterate through all the books on the device, setting the
+        book_ids_to_refresh = set()
-        # in_library field. If the UUID matches a book in the library, then
-        # do not consider that book for other matching. In all cases set
-        # the application_id to the db_id of the matching book. This value
-        # will be used by books_on_device to indicate matches. While we are
-        # going by, update the metadata for a book if automatic management is on
 
         def update_book(id_, book) :
             if not update_metadata:
@@ -1768,17 +1763,31 @@ class DeviceMixin(object):  # {{{
 
         def updateq(id_, book):
             try:
-                return (update_metadata and
+                if not update_metadata:
-                        (db.metadata_last_modified(id_, index_is_id=True) !=
+                    return False
-                         getattr(book, 'last_modified', None) or
+
-                         (isinstance(getattr(book, 'thumbnail', None), (list, tuple))
+                if self.device_manager.device is not None:
-                          and max(book.thumbnail[0], book.thumbnail[1]) != desired_thumbnail_height
+                    set_of_ids = self.device_manager.device.synchronize_with_db(db, id_, book)
-                         )
+                    if set_of_ids is not None:
+                        book_ids_to_refresh.update(set_of_ids)
+                        return True
+
+                return (db.metadata_last_modified(id_, index_is_id=True) !=
+                        getattr(book, 'last_modified', None) or
+                        (isinstance(getattr(book, 'thumbnail', None), (list, tuple))
+                         and max(book.thumbnail[0], book.thumbnail[1]) != desired_thumbnail_height
                         )
                        )
             except:
                 return True
 
+        # Now iterate through all the books on the device, setting the
+        # in_library field. If the UUID matches a book in the library, then
+        # do not consider that book for other matching. In all cases set
+        # the application_id to the db_id of the matching book. This value
+        # will be used by books_on_device to indicate matches. While we are
+        # going by, update the metadata for a book if automatic management is on
+
         total_book_count = 0
         for booklist in booklists:
             for book in booklist:
@@ -1873,6 +1882,16 @@ class DeviceMixin(object):  # {{{
                                 FunctionDispatcher(self.metadata_synced), booklists,
                                 plugboards, add_as_step_to_job)
 
+            if book_ids_to_refresh:
+                try:
+                    prints('DeviceJob: set_books_in_library refreshing GUI for ',
+                           len(book_ids_to_refresh), 'books')
+                    self.library_view.model().refresh_ids(book_ids_to_refresh,
+                                      current_row=self.library_view.currentIndex().row())
+                except:
+                    # This shouldn't ever happen, but just in case ...
+                    traceback.print_exc()
+
         if DEBUG:
             prints('DeviceJob: set_books_in_library finished: time=',
                    time.time() - start_time)