Fetch My Clippings from the Kindle

src/calibre/devices/interface.py

@@ -45,7 +45,7 @@ class DevicePlugin(Plugin):
     icon = I('reader.svg')
 
     # Used by gui2.ui:annotations_fetched() and devices.kindle.driver:get_annotations()
-    UserAnnotation = namedtuple('Annotation','type, bookmark')
+    UserAnnotation = namedtuple('Annotation','type, value')
 
     @classmethod
     def get_gui_name(cls):

src/calibre/devices/kindle/driver.py

@@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en'
 '''
 Device driver for Amazon's Kindle
 '''
-import os, re, sys
+import datetime, os, re, sys
 from cStringIO import StringIO
 from struct import unpack
 
@@ -62,18 +62,11 @@ class KINDLE(USBMS):
 
     def get_annotations(self, path_map):
         MBP_FORMATS = [u'azw', u'mobi', u'prc', u'txt']
-        TAN_FORMATS = [u'tpz', u'azw1']
+        mbp_formats = set(MBP_FORMATS)
         PDR_FORMATS = [u'pdf']
-
+        pdr_formats = set(PDR_FORMATS)
-        mbp_formats = set()
+        TAN_FORMATS = [u'tpz', u'azw1']
-        for fmt in MBP_FORMATS:
+        tan_formats = set(TAN_FORMATS)
-            mbp_formats.add(fmt)
-        tan_formats = set()
-        for fmt in TAN_FORMATS:
-            tan_formats.add(fmt)
-        pdr_formats = set()
-        for fmt in PDR_FORMATS:
-            pdr_formats.add(fmt)
 
         def get_storage():
             storage = []
@@ -121,6 +114,14 @@ class KINDLE(USBMS):
                 path_map.pop(id)
             return path_map, book_ext
 
+        def get_my_clippings(storage, bookmarked_books):
+            # add an entry for 'My Clippings.txt'
+            for vol in storage:
+                mc_path = os.path.join(vol,'My Clippings.txt')
+                if os.path.exists(mc_path):
+                    return mc_path
+            return None
+
         storage = get_storage()
         path_map, book_ext = resolve_bookmark_paths(storage, path_map)
 
@@ -128,7 +129,13 @@ class KINDLE(USBMS):
         for id in path_map:
             bookmark_ext = path_map[id].rpartition('.')[2]
             myBookmark = Bookmark(path_map[id], id, book_ext[id], bookmark_ext)
-            bookmarked_books[id] = self.UserAnnotation(type='kindle', bookmark=myBookmark)
+            bookmarked_books[id] = self.UserAnnotation(type='kindle_bookmark', value=myBookmark)
+
+        mc_path = get_my_clippings(storage, bookmarked_books)
+        if mc_path:
+            timestamp = datetime.datetime.utcfromtimestamp(os.path.getmtime(mc_path))
+            bookmarked_books['clippings'] = self.UserAnnotation(type='kindle_clippings',
+                                              value=dict(path=mc_path,timestamp=timestamp))
 
         # This returns as job.result in gui2.ui.annotations_fetched(self,job)
         return bookmarked_books

src/calibre/gui2/ui.py

@@ -976,12 +976,15 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
 
     def annotations_fetched(self, job):
         from calibre.devices.usbms.device import Device
+        from calibre.ebooks.metadata import MetaInformation
         from calibre.gui2.dialogs.progress import ProgressDialog
+        from calibre.library.cli import do_add_format
 
         class Updater(QThread):
 
             update_progress = pyqtSignal(int)
             update_done     = pyqtSignal()
+            FINISHED_READING_PCT_THRESHOLD = 96
 
             def __init__(self, parent, db, annotation_map, done_callback):
                 QThread.__init__(self, parent)
@@ -1064,37 +1067,66 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
                 ka_soup.insert(0,divTag)
                 return ka_soup
 
+            def mark_book_as_read(self,id):
+                read_tag = gprefs.get('catalog_epub_mobi_read_tag')
+                self.db.set_tags(id, [read_tag], append=True)
+
             def canceled(self):
                 self.pd.hide()
 
             def run(self):
                 for (i, id) in enumerate(self.am):
                     bm = Device.UserAnnotation(self.am[id][0],self.am[id][1])
-                    user_notes_soup = self.generate_annotation_html(bm.bookmark)
+                    if bm.type == 'kindle_bookmark':
+                        user_notes_soup = self.generate_annotation_html(bm.value)
 
-                    mi = self.db.get_metadata(id, index_is_id=True)
+                        mi = self.db.get_metadata(id, index_is_id=True)
-                    if mi.comments:
-                        a_offset = mi.comments.find('<div class="user_annotations">')
-                        ad_offset = mi.comments.find('<hr class="annotations_divider" />')
-
-                        if a_offset >= 0:
-                            mi.comments = mi.comments[:a_offset]
-                        if ad_offset >= 0:
-                            mi.comments = mi.comments[:ad_offset]
                         if mi.comments:
-                            hrTag = Tag(user_notes_soup,'hr')
+                            a_offset = mi.comments.find('<div class="user_annotations">')
-                            hrTag['class'] = 'annotations_divider'
+                            ad_offset = mi.comments.find('<hr class="annotations_divider" />')
-                            user_notes_soup.insert(0,hrTag)
+
+                            if a_offset >= 0:
+                                mi.comments = mi.comments[:a_offset]
+                            if ad_offset >= 0:
+                                mi.comments = mi.comments[:ad_offset]
+                            if mi.comments:
+                                hrTag = Tag(user_notes_soup,'hr')
+                                hrTag['class'] = 'annotations_divider'
+                                user_notes_soup.insert(0,hrTag)
+
+                            mi.comments += user_notes_soup.prettify()
+                        else:
+                            mi.comments = unicode(user_notes_soup.prettify())
+                        # Update library comments
+                        self.db.set_comment(id, mi.comments)
+                        # Update 'read' tag
+                        if bm.value.percent_read >= self.FINISHED_READING_PCT_THRESHOLD:
+                            self.mark_book_as_read(id)
+                        # Add bookmark file to id
+                        self.db.add_format_with_hooks(id, bm.value.bookmark_extension,
+                                                      bm.value.path, index_is_id=True)
+                        self.update_progress.emit(i)
+                    elif bm.type == 'kindle_clippings':
+                        # Find 'My Clippings' author=Kindle in database, or add
+                        self.db.search('title:"My Clippings" author:Kindle')
+                        data = self.db.get_data_as_dict()
+                        last_update = 'Last modified %s' % strftime(u'%x %X',bm.value['timestamp'].timetuple())
+                        if data:
+                            do_add_format(self.db, data[0]['id'], 'TXT', bm.value['path'])
+                            mi = self.db.get_metadata(data[0]['id'], index_is_id=True)
+                            mi.comments = last_update
+                            self.db.set_metadata(data[0]['id'], mi)
+                        else:
+                            mi = MetaInformation('My Clippings', authors = ['Kindle'])
+                            mi.tags = ['Clippings']
+                            mi.comments = last_update
+                            self.db.add_books([bm.value['path']], ['txt'], [mi])
+
+                        # KG: This doesn't seem right, but without it the main window
+                        # shows the results of the last search for 'My Clippings' instead of
+                        # its previous contents
+                        self.db.search('')
 
-                        mi.comments += user_notes_soup.prettify()
-                    else:
-                        mi.comments = unicode(user_notes_soup.prettify())
-                    # Update library comments
-                    self.db.set_comment(id, mi.comments)
-                    # Add bookmark file to id
-                    self.db.add_format_with_hooks(id, bm.bookmark.bookmark_extension,
-                                                  bm.bookmark.path, index_is_id=True)
-                    self.update_progress.emit(i)
                 self.update_done.emit()
                 self.done_callback(self.am.keys())
 

src/calibre/library/catalog.py

@@ -1188,18 +1188,11 @@ class EPUB_MOBI(CatalogPlugin):
             from calibre.ebooks.metadata import MetaInformation
 
             MBP_FORMATS = [u'azw', u'mobi', u'prc', u'txt']
-            TAN_FORMATS = [u'tpz', u'azw1']
+            mbp_formats = set(MBP_FORMATS)
             PDR_FORMATS = [u'pdf']
-
+            pdr_formats = set(PDR_FORMATS)
-            mbp_formats = set()
+            TAN_FORMATS = [u'tpz', u'azw1']
-            for fmt in MBP_FORMATS:
+            tan_formats = set(TAN_FORMATS)
-                mbp_formats.add(fmt)
-            tan_formats = set()
-            for fmt in TAN_FORMATS:
-                tan_formats.add(fmt)
-            pdr_formats = set()
-            for fmt in PDR_FORMATS:
-                pdr_formats.add(fmt)
 
             class BookmarkDevice(Device):
                 def initialize(self, save_template):

src/calibre/ptempfile.py

@@ -51,7 +51,6 @@ class PersistentTemporaryFile(object):
         except:
             pass
 
-
 def PersistentTemporaryDirectory(suffix='', prefix='', dir=None):
     '''
     Return the path to a newly created temporary directory that will