Add ability for devices to request that book format files be sent to the device. The wireless device driver uses this to ensure (optionally) that the books on the device are up-to-date with the books in calibre.

src/calibre/devices/interface.py

@@ -732,12 +732,23 @@ class DevicePlugin(Plugin):
         a book in calibre's db. The method is responsible for syncronizing
         data from the device to calibre's db (if needed).
 
-        The method must return a set of calibre book ids changed if calibre's
+        The method must return a two-value tuple. The first value is a set of
-        database was changed, None if the database was not changed. If the
+        calibre book ids changed if calibre's database was changed or None if the
-        method returns an empty set then the metadata for the book on the
+        database was not changed. If the first value is an empty set then the
-        device is updated with calibre's metadata and given back to the device,
+        metadata for the book on the device is updated with calibre's metadata
-        but no GUI refresh of that book is done. This is useful when the calire
+        and given back to the device, but no GUI refresh of that book is done.
-        data is correct but must be sent to the device.
+        This is useful when the calire data is correct but must be sent to the
+        device.
+
+        The second value in the tuple specifies whether a book format should be
+        sent to the device. The intent is to permit verifying that the book on
+        the device is the same as the book in calibre. Return None if no book is
+        to be sent, otherwise return the base file name on the device (a string
+        like foobar.epub). Be sure to include the extension in the name. The
+        device subsystem will construct a send_books job for all books with not-
+        None returned values. Note: other than to later retrieve the extension,
+        the name is ignored in cases where the device uses a template to
+        generate the file name, which most do.
 
         Extremely important: this method is called on the GUI thread. It must
         be threadsafe with respect to the device manager's thread.

src/calibre/devices/smart_device_app/driver.py

@@ -999,7 +999,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                     'pubdateFormat': tweaks['gui_pubdate_display_format'],
                     'timestampFormat': tweaks['gui_timestamp_display_format'],
                     'lastModifiedFormat': tweaks['gui_last_modified_display_format'],
-                    'calibre_version': numeric_version})
+                    'calibre_version': numeric_version,
+                    'canSupportUpdateBooks': True})
             if opcode != 'OK':
                 # Something wrong with the return. Close the socket
                 # and continue.
@@ -1043,6 +1044,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
             self._debug('Can send OK to sendbook', self.can_send_ok_to_sendbook)
             self.can_accept_library_info = result.get('canAcceptLibraryInfo', False)
             self._debug('Can accept library info', self.can_accept_library_info)
+            self.will_ask_for_update_books = result.get('willAskForUpdateBooks', False)
+            self._debug('Will ask for update books', self.will_ask_for_update_books)
 
             if not self.settings().extra_customization[self.OPT_USE_METADATA_CACHE]:
                 self.client_can_use_metadata_cache = False
@@ -1223,7 +1226,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                              'canScan':True,
                              'willUseCachedMetadata': self.client_can_use_metadata_cache,
                              'supportsSync': (bool(self.is_read_sync_col) or
-                                              bool(self.is_read_date_sync_col))})
+                                              bool(self.is_read_date_sync_col)),
+                             'canSupportBookFormatSync': True})
         bl = CollectionsBookList(None, self.PREFIX, self.settings)
         if opcode == 'OK':
             count = result['count']
@@ -1252,6 +1256,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                         book.set('_is_read_', r.get('_is_read_', None))
                         book.set('_sync_type_', r.get('_sync_type_', None))
                         book.set('_last_read_date_', r.get('_last_read_date_', None))
+                        book.set('_format_mtime_', r.get('_format_mtime_', None))
                     else:
                         books_to_send.append(r['priKey'])
 
@@ -1529,6 +1534,30 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
     def specialize_global_preferences(self, device_prefs):
         device_prefs.set_overrides(manage_device_metadata='on_connect')
 
+    def _check_if_format_send_needed(self, db, id_, book):
+        if not self.will_ask_for_update_books:
+            return None
+
+        from calibre.utils.date import parse_date, now, isoformat
+        try:
+            if not hasattr(book, '_format_mtime_'):
+                return None
+
+            cc_mtime = parse_date(book.get('_format_mtime_'), as_utc=False)
+            ext = posixpath.splitext(book.lpath)[1][1:]
+            fmt_metadata = db.new_api.format_metadata(id_, ext)
+            if fmt_metadata:
+                calibre_mtime = fmt_metadata['mtime']
+                self._debug(book.title, 'cal_mtime', calibre_mtime, 'cc_mtime', cc_mtime)
+                if cc_mtime < calibre_mtime:
+                    book.set('_format_mtime_', isoformat(now()))
+                    return posixpath.basename(book.lpath)
+        except:
+            self._debug('exception checking if must send format', book.title)
+            traceback.print_exc()
+        return None
+
+
     @synchronous('sync_lock')
     def synchronize_with_db(self, db, id_, book):
         from calibre.utils.date import parse_date, is_date_undefined
@@ -1540,7 +1569,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
         if self.have_bad_sync_columns or not (self.is_read_sync_col or
                                               self.is_read_date_sync_col):
             # Not syncing or sync columns are invalid
-            return None
+            return (None, self._check_if_format_send_needed(db, id_, book))
 
         # Check the validity of the columns once per connection. We do it
         # here because we have access to the db to get field_metadata
@@ -1572,7 +1601,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
 
             self.have_checked_sync_columns = True
             if self.have_bad_sync_columns:
-                return None
+                return (None, self._check_if_format_send_needed(db, id_, book))
 
         sync_type = book.get('_sync_type_', None)
         # We need to check if our attributes are in the book. If they are not
@@ -1697,14 +1726,14 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
         if changed_books or force_return_changed_books:
             # One of the two values was synced, giving a (perhaps empty) list of
             # changed books. Return that.
-            return changed_books
+            return (changed_books, self._check_if_format_send_needed(db, id_, book))
 
         # Nothing was synced. The user might have changed the value in calibre.
         # If so, that value will be sent to the device in the normal way. Note
         # that because any updated value has already been synced and so will
         # also be sent, the device should put the calibre value into its
         # checkbox (or whatever it uses)
-        return None
+        return (None, self._check_if_format_send_needed(db, id_, book))
 
     @synchronous('sync_lock')
     def startup(self):

src/calibre/gui2/device.py

@@ -11,6 +11,8 @@ from PyQt5.Qt import (
     QObject, QVBoxLayout, QDialogButtonBox, QCursor, QCoreApplication,
     QApplication, QEventLoop)
 
+from calibre import isbytestring
+from calibre.constants import filesystem_encoding
 from calibre.customize.ui import (available_input_formats, available_output_formats,
     device_plugins, disabled_device_plugins)
 from calibre.devices.interface import DevicePlugin
@@ -34,6 +36,7 @@ from calibre.constants import DEBUG
 from calibre.utils.config import tweaks, device_prefs
 from calibre.utils.magick.draw import thumbnail
 from calibre.library.save_to_disk import find_plugboard
+from calibre.ptempfile import PersistentTemporaryFile
 # }}}
 
 class DeviceJob(BaseJob):  # {{{
@@ -1774,6 +1777,7 @@ class DeviceMixin(object):  # {{{
             self.db_book_uuid_cache = db_book_uuid_cache
 
         book_ids_to_refresh = set()
+        book_formats_to_send = list()
 
         def update_book(id_, book) :
             if not update_metadata:
@@ -1789,7 +1793,10 @@ class DeviceMixin(object):  # {{{
                     return False
 
                 if self.device_manager.device is not None:
-                    set_of_ids = self.device_manager.device.synchronize_with_db(db, id_, book)
+                    set_of_ids, fmt_name = \
+                            self.device_manager.device.synchronize_with_db(db, id_, book)
+                    if fmt_name is not None:
+                        book_formats_to_send.append((id_, fmt_name))
                     if set_of_ids is not None:
                         book_ids_to_refresh.update(set_of_ids)
                         return True
@@ -1913,6 +1920,38 @@ class DeviceMixin(object):  # {{{
                     # This shouldn't ever happen, but just in case ...
                     traceback.print_exc()
 
+            # Sync books if necessary
+            try:
+                files, names, metadata = [], [], []
+                for id_, fmt_name in book_formats_to_send:
+                    if DEBUG:
+                        prints('DeviceJob: Syncing book. id:', id_, 'name from device', fmt_name)
+                    ext = os.path.splitext(fmt_name)[1][1:]
+                    fmt_info = db.new_api.format_metadata(id_, ext)
+                    if fmt_info:
+                        try:
+                            pt = PersistentTemporaryFile(suffix='caltmpfmt.'+ext)
+                            db.new_api.copy_format_to(id_, ext, pt)
+                            pt.close()
+                            def to_uni(x):
+                                if isbytestring(x):
+                                    x = x.decode(filesystem_encoding)
+                                return x
+                            files.append(to_uni(os.path.abspath(pt.name)))
+                            names.append(fmt_name)
+                            metadata.append(db.new_api.get_metadata(id_, get_cover=True))
+                        except:
+                            prints('Problem creating temporary file for', fmt_name)
+                            traceback.print_exc()
+                    else:
+                        if DEBUG:
+                            prints("DeviceJob: book doesn't have that format")
+                if files:
+                    self.upload_books(files, names, metadata)
+            except:
+                # Shouldn't ever happen, but just in case
+                traceback.print_exc()
+
         if DEBUG:
             prints('DeviceJob: set_books_in_library finished: time=',
                    time.time() - start_time)