Kindle driver: Detect KFX books present on e-ink kindles. See #1496206 (Book does not show on device view in Calibre)

src/calibre/devices/kindle/driver.py

@@ -8,11 +8,12 @@ __docformat__ = 'restructuredtext en'
 Device driver for Amazon's Kindle
 '''
 
-import datetime, os, re, sys, json, hashlib
+import datetime, os, re, sys, json, hashlib, shutil
 
+from calibre.constants import DEBUG
 from calibre.devices.kindle.bookmark import Bookmark
 from calibre.devices.usbms.driver import USBMS
-from calibre import strftime, fsync
+from calibre import strftime, fsync, prints
 
 '''
 Notes on collections:
@@ -36,6 +37,9 @@ Adding a book to a collection on the Kindle does not change the book file at all
 file metadata.
 '''
 
+def get_kfx_path(path):
+    return os.path.dirname(os.path.dirname(path)).rpartition('.')[0] + '.kfx'
+
 class KINDLE(USBMS):
 
     name           = 'Kindle Device Interface'
@@ -71,8 +75,50 @@ class KINDLE(USBMS):
     WIRELESS_FILE_NAME_PATTERN = re.compile(
     r'(?P<title>[^-]+)-asin_(?P<asin>[a-zA-Z\d]{10,})-type_(?P<type>\w{4})-v_(?P<index>\d+).*')
 
+    VIRTUAL_BOOK_EXTENSIONS = frozenset({'kfx'})
+    VIRTUAL_BOOK_EXTENSION_MESSAGE = _(
+        'The following books are in KFX format. KFX is a virtual book format, and cannot'
+        ' be transferred from the device. Instead, you must go to your "Manage my'
+        ' content and devices" page on amazon.com and download the book to your computer from there.'
+        ' That will give you a regular azw3 file that you can add to calibre normally.'
+        ' Click "Show details" to see the list of books.'
+    )
+
+    def is_a_book_file(self, filename, path, prefix):
+        lpath = os.path.join(path, filename).partition(self.normalize_path(prefix))[2].replace('\\', '/')
+        return lpath.endswith('.sdr/assets/metadata.kfx')
+
+    def delete_single_book(self, path):
+        if path.replace('\\', '/').endswith('.sdr/assets/metadata.kfx'):
+            kfx_path = get_kfx_path(path)
+            if DEBUG:
+                prints('Kindle driver: Attempting to delete kfx: %r -> %r' % (path, kfx_path))
+            if os.path.exists(kfx_path):
+                os.unlink(kfx_path)
+            sdr_path = kfx_path.rpartition('.')[0] + '.sdr'
+            if os.path.exists(sdr_path):
+                shutil.rmtree(sdr_path)
+            try:
+                os.removedirs(os.path.dirname(kfx_path))
+            except Exception:
+                pass
+
+        else:
+            return USBMS.delete_single_book(self, path)
+
     @classmethod
     def metadata_from_path(cls, path):
+        if path.replace('\\', '/').endswith('.sdr/assets/metadata.kfx'):
+            from calibre.ebooks.metadata.kfx import read_metadata_kfx
+            try:
+                with lopen(path, 'rb') as f:
+                    mi = read_metadata_kfx(f)
+            except Exception:
+                import traceback
+                traceback.print_exc()
+                path = get_kfx_path(path)
+                mi = cls.metadata_from_formats([get_kfx_path(path)])
+        else:
             mi = cls.metadata_from_formats([path])
         if mi.title == _('Unknown') or ('-asin' in mi.title and '-type' in mi.title):
             match = cls.WIRELESS_FILE_NAME_PATTERN.match(os.path.basename(path))
@@ -475,7 +521,7 @@ class KINDLE2(KINDLE):
 
         apnx_path = '%s.apnx' % os.path.join(path, filename)
         apnx_builder = APNXBuilder()
-        # ## Check to see if there is an existing apnx file on Kindle we should keep.
+        # Check to see if there is an existing apnx file on Kindle we should keep.
         if opts.extra_customization[self.OPT_APNX_OVERWRITE] or not os.path.exists(apnx_path):
             try:
                 method = opts.extra_customization[self.OPT_APNX_METHOD]
@@ -527,4 +573,3 @@ class KINDLE_FIRE(KINDLE2):
 
     def upload_kindle_thumbnail(self, metadata, filepath):
         pass
-

src/calibre/devices/usbms/driver.py

@@ -174,6 +174,9 @@ class USBMS(CLI, Device):
     def formats_to_scan_for(self):
         return set(self.settings().format_map) | set(self.FORMATS)
 
+    def is_a_book_file(self, filename, path, prefix):
+        return False
+
     def books(self, oncard=None, end_session=True):
         from calibre.ebooks.metadata.meta import path_to_ext
 
@@ -216,7 +219,7 @@ class USBMS(CLI, Device):
 
         def update_booklist(filename, path, prefix):
             changed = False
-            if path_to_ext(filename) in all_formats:
+            if path_to_ext(filename) in all_formats or self.is_a_book_file(filename, path, prefix):
                 try:
                     lpath = os.path.join(path, filename).partition(self.normalize_path(prefix))[2]
                     if lpath.startswith(os.sep):
@@ -376,6 +379,9 @@ class USBMS(CLI, Device):
         self.report_progress(1.0, _('Adding books to device metadata listing...'))
         debug_print('USBMS: finished adding metadata')
 
+    def delete_single_book(self, path):
+        os.unlink(path)
+
     def delete_books(self, paths, end_session=True):
         debug_print('USBMS: deleting %d books'%(len(paths)))
         for i, path in enumerate(paths):
@@ -383,7 +389,7 @@ class USBMS(CLI, Device):
             path = self.normalize_path(path)
             if os.path.exists(path):
                 # Delete the ebook
-                os.unlink(path)
+                self.delete_single_book(path)
 
                 filepath = os.path.splitext(path)[0]
                 for ext in self.DELETE_EXTS: