GwR revisions for fetch annotations

src/calibre/customize/__init__.py

@@ -129,18 +129,18 @@ class Plugin(object):
                     zip_safe = False
             if zip_safe:
                 sys.path.insert(0, self.plugin_path)
-                self._sys_insertion_path = self.plugin_path
+                self.sys_insertion_path = self.plugin_path
             else:
                 from calibre.ptempfile import TemporaryDirectory
                 self._sys_insertion_tdir = TemporaryDirectory('plugin_unzip')
-                self._sys_insertion_path = self._sys_insertion_tdir.__enter__(*args)
+                self.sys_insertion_path = self._sys_insertion_tdir.__enter__(*args)
-                zf.extractall(self._sys_insertion_path)
+                zf.extractall(self.sys_insertion_path)
-                sys.path.insert(0, self._sys_insertion_path)
+                sys.path.insert(0, self.sys_insertion_path)
             zf.close()
 
 
     def __exit__(self, *args):
-        ip, it = getattr(self, '_sys_insertion_path', None), getattr(self,
+        ip, it = getattr(self, 'sys_insertion_path', None), getattr(self,
                 '_sys_insertion_tdir', None)
         if ip in sys.path:
             sys.path.remove(ip)

src/calibre/devices/kindle/driver.py

@@ -8,11 +8,14 @@ __docformat__ = 'restructuredtext en'
 Device driver for Amazon's Kindle
 '''
 
+from cStringIO import StringIO
 import os
 import re
+from struct import unpack
 import sys
 
 from calibre.devices.usbms.driver import USBMS
+from calibre.ebooks.metadata.mobi import StreamSlicer
 
 class KINDLE(USBMS):
 
@@ -44,6 +47,7 @@ class KINDLE(USBMS):
     EBOOK_DIR_CARD_A = 'documents'
     DELETE_EXTS = ['.mbp']
     SUPPORTS_SUB_DIRS = True
+    SUPPORTS_ANNOTATIONS = True
 
     WIRELESS_FILE_NAME_PATTERN = re.compile(
     r'(?P<title>[^-]+)-asin_(?P<asin>[a-zA-Z\d]{10,})-type_(?P<type>\w{4})-v_(?P<index>\d+).*')
@@ -60,6 +64,51 @@ class KINDLE(USBMS):
                                                'replace')
         return mi
 
+    def get_annotations(self, path_map):
+        def get_storage():
+            storage = []
+            if self._main_prefix:
+                storage.append(os.path.join(self._main_prefix, self.EBOOK_DIR_MAIN))
+            if self._card_a_prefix:
+                storage.append(os.path.join(self._card_a_prefix, self.EBOOK_DIR_CARD_A))
+            if self._card_b_prefix:
+                storage.append(os.path.join(self._card_b_prefix, self.EBOOK_DIR_CARD_B))
+            return storage
+
+        def resolve_mbp_paths(storage, path_map):
+            pop_list = []
+            for id in path_map:
+                for vol in storage:
+                    #print "path_map[id]: %s" % path_map[id]
+                    mbp_path = path_map[id].replace(os.path.abspath('/<storage>'),vol)
+                    #print "looking for mbp_path: %s" % mbp_path
+                    if os.path.exists(mbp_path):
+                        #print "mbp_path found"
+                        path_map[id] = mbp_path
+                        break
+                else:
+                    #print "mbp_path not found"
+                    pop_list.append(id)
+
+            # Remove non-existent mbp files
+            for id in pop_list:
+                path_map.pop(id)
+            return path_map
+
+        storage = get_storage()
+        path_map = resolve_mbp_paths(storage, path_map)
+
+        # path_map is now a mapping of valid mbp files
+        # Not yet implemented - Topaz annotations
+        bookmarked_books = {}
+        MBP_FORMATS = ['azw', 'mobi', 'prc', 'txt']
+        for id in path_map:
+            myBookmark = Bookmark(path_map[id], MBP_FORMATS, id)
+            bookmarked_books[id] = self.UserAnnotation(type='mobi', bookmark=myBookmark)
+
+        # This returns as job.result in gui2.ui.annotations_fetched(self,job)
+        return bookmarked_books
+
 
 class KINDLE2(KINDLE):
 
@@ -79,3 +128,139 @@ class KINDLE_DX(KINDLE2):
 
     PRODUCT_ID = [0x0003]
     BCD        = [0x0100]
+
+class Bookmark():
+    '''
+    A simple class fetching bookmark data
+    Kindle-specific
+    '''
+    def __init__(self, path, formats, id):
+        self.book_format = None
+        self.book_length = 0
+        self.id = id
+        self.last_read_location = 0
+        self.timestamp = 0
+        self.user_notes = None
+
+        self.get_bookmark_data(path)
+        self.get_book_length(path, formats)
+        try:
+            self.percent_read = float(100*self.last_read_location / self.book_length)
+        except:
+            self.percent_read = 0
+
+    def record(self, n):
+        if n >= self.nrecs:
+            raise ValueError('non-existent record %r' % n)
+        offoff = 78 + (8 * n)
+        start, = unpack('>I', self.data[offoff + 0:offoff + 4])
+        stop = None
+        if n < (self.nrecs - 1):
+            stop, = unpack('>I', self.data[offoff + 8:offoff + 12])
+        return StreamSlicer(self.stream, start, stop)
+
+    def get_bookmark_data(self, path, fetchUserNotes=True):
+        ''' Return the timestamp and last_read_location '''
+        with open(path,'rb') as f:
+            stream = StringIO(f.read())
+            data = StreamSlicer(stream)
+            self.timestamp, = unpack('>I', data[0x24:0x28])
+            bpar_offset, = unpack('>I', data[0x4e:0x52])
+            lrlo = bpar_offset + 0x0c
+            self.last_read_location = int(unpack('>I', data[lrlo:lrlo+4])[0])
+            entries, = unpack('>I', data[0x4a:0x4e])
+
+            # Store the annotations/locations
+            if fetchUserNotes:
+                bpl = bpar_offset + 4
+                bpar_len, = unpack('>I', data[bpl:bpl+4])
+                bpar_len += 8
+                #print "bpar_len: 0x%x" % bpar_len
+                eo = bpar_offset + bpar_len
+
+                # Walk bookmark entries
+                #print " --- %s --- " % path
+                #print "  last_read_location: %d" % self.magicKindleLocationCalculator(last_read_location)
+                current_entry = 1
+                sig = data[eo:eo+4]
+                previous_block = None
+                user_notes = {}
+
+                while sig == 'DATA':
+                    text = None
+                    entry_type = None
+                    rec_len, = unpack('>I', data[eo+4:eo+8])
+                    if rec_len == 0:
+                        current_block = "empty_data"
+                    elif  data[eo+8:eo+12] == "EBAR":
+                        current_block = "data_header"
+                        #entry_type = "data_header"
+                        location, = unpack('>I', data[eo+0x34:eo+0x38])
+                        #print "data_header location: %d" % location
+                    else:
+                        current_block = "text_block"
+                        if previous_block == 'empty_data':
+                            entry_type = 'Note'
+                        elif previous_block == 'data_header':
+                            entry_type = 'Highlight'
+                        text = data[eo+8:eo+8+rec_len].decode('utf-16-be')
+
+                    if entry_type:
+                        user_notes[location] = dict(type=entry_type, id=self.id,
+                                                    text=data[eo+8:eo+8+rec_len].decode('utf-16-be'))
+                        #print " %2d: %s %s" % (current_entry, entry_type,'at %d' % location if location else '')
+                    #if current_block == 'text_block':
+                        #self.textdump(text)
+
+                    eo += rec_len + 8
+                    current_entry += 1
+                    previous_block = current_block
+                    sig = data[eo:eo+4]
+
+                while sig == 'BKMK':
+                    # Fix start location for Highlights using BKMK data
+                    end_loc, = unpack('>I', data[eo+0x10:eo+0x14])
+                    if end_loc in user_notes and user_notes[end_loc]['type'] != 'Note':
+                        start, = unpack('>I', data[eo+8:eo+12])
+                        user_notes[start] = user_notes[end_loc]
+                        user_notes.pop(end_loc)
+                        #print "changing start location of %d to %d" % (end_loc,start)
+                    else:
+                        # If a bookmark coincides with a user annotation, the locs could
+                        # be the same - cheat by nudging -1
+                        # Skip bookmark for last_read_location
+                        if end_loc != self.last_read_location:
+                            user_notes[end_loc - 1] = dict(type='Bookmark',id=self.id,text=None)
+                    rec_len, = unpack('>I', data[eo+4:eo+8])
+                    eo += rec_len + 8
+                    sig = data[eo:eo+4]
+
+        '''
+        for location in sorted(user_notes):
+            print '  Location %d: %s\n%s' % self.magicKindleLocationCalculator(location),
+                                                     user_notes[location]['type'],
+                                    '\n'.join(self.textdump(user_notes[location]['text'])))
+        '''
+        self.user_notes = user_notes
+
+    def get_book_length(self, path, formats):
+        # This assumes only one of the possible formats exists on the Kindle
+        book_fs = None
+        for format in formats:
+            fmt = format.rpartition('.')[2]
+            book_fs = path.replace('.mbp','.%s' % fmt)
+            if os.path.exists(book_fs):
+                self.book_format = fmt
+                break
+        else:
+            #print "no files matching library formats exist on device"
+            self.book_length = 0
+            return
+
+        # Read the book len from the header
+        with open(book_fs,'rb') as f:
+            self.stream = StringIO(f.read())
+            self.data = StreamSlicer(self.stream)
+            self.nrecs, = unpack('>H', self.data[76:78])
+            record0 = self.record(0)
+            self.book_length = int(unpack('>I', record0[0x04:0x08])[0])

src/calibre/devices/usbms/device.py

@@ -17,6 +17,8 @@ import time
 import re
 import sys
 import glob
+
+from collections import namedtuple
 from itertools import repeat
 
 from calibre.devices.interface import DevicePlugin
@@ -88,6 +90,8 @@ class Device(DeviceConfig, DevicePlugin):
     EBOOK_DIR_CARD_B = ''
     DELETE_EXTS = []
 
+    # Used by gui2.ui:annotations_fetched() and devices.kindle.driver:get_annotations()
+    UserAnnotation = namedtuple('Annotation','type, bookmark')
 
     def reset(self, key='-1', log_packets=False, report_progress=None,
             detected_device=None):
@@ -793,6 +797,12 @@ class Device(DeviceConfig, DevicePlugin):
         '''
         return components
 
+    def get_annotations(self, path_map):
+        '''
+        Resolve path_map to annotation_map of files found on the device
+        '''
+        return {}
+
     def create_upload_path(self, path, mdata, fname, create_dirs=True):
         path = os.path.abspath(path)
         extra_components = []

src/calibre/devices/usbms/driver.py

@@ -123,7 +123,7 @@ class USBMS(CLI, Device):
         '''
         :path: the full path were the associated book is located.
         :filename: the name of the book file without the extension.
-        :metatdata: metadata belonging to the book. Use metadata.thumbnail
+        :metadata: metadata belonging to the book. Use metadata.thumbnail
         for cover
         '''
         pass

src/calibre/ebooks/pml/pmlconverter.py

@@ -79,7 +79,7 @@ class PML_HTMLizer(object):
         'd': ('<span style="text-decoration: line-through;">', '</span>'),
         'b': ('<span style="font-weight: bold;">', '</span>'),
         'l': ('<span style="font-size: 150%;">', '</span>'),
-        'k': ('<span style="font-size: 75%;">', '</span>'),
+        'k': ('<span style="font-size: 75%; font-variant: small-caps;">', '</span>'),
         'FN': ('<br /><br style="page-break-after: always;" /><div id="fn-%s"><p>', '</p><<small><a href="#rfn-%s">return</a></small></div>'),
         'SB': ('<br /><br style="page-break-after: always;" /><div id="sb-%s"><p>', '</p><small><a href="#rsb-%s">return</a></small></div>'),
     }
@@ -154,6 +154,11 @@ class PML_HTMLizer(object):
         self.file_name = ''
 
     def prepare_pml(self, pml):
+        # Give Chapters the form \\*='text'text\\*. This is used for generating
+        # the TOC later.
+        pml = re.sub(r'(?<=\\x)(?P<text>.*?)(?=\\x)', lambda match: '="%s"%s' % (self.strip_pml(match.group('text')), match.group('text')), pml)
+        pml = re.sub(r'(?<=\\X[0-4])(?P<text>.*?)(?=\\X[0-4])', lambda match: '="%s"%s' % (self.strip_pml(match.group('text')), match.group('text')), pml)
+
         # Remove comments
         pml = re.sub(r'(?mus)\\v(?P<text>.*?)\\v', '', pml)
 
@@ -163,7 +168,7 @@ class PML_HTMLizer(object):
         pml = re.sub(r'(?mus)(?<=.)[ ]*$', '', pml)
         pml = re.sub(r'(?mus)^[ ]*$', '', pml)
 
-        # Footnotes and Sidebars
+        # Footnotes and Sidebars.
         pml = re.sub(r'(?mus)<footnote\s+id="(?P<target>.+?)">\s*(?P<text>.*?)\s*</footnote>', lambda match: '\\FN="%s"%s\\FN' % (match.group('target'), match.group('text')) if match.group('text') else '', pml)
         pml = re.sub(r'(?mus)<sidebar\s+id="(?P<target>.+?)">\s*(?P<text>.*?)\s*</sidebar>', lambda match: '\\SB="%s"%s\\SB' % (match.group('target'), match.group('text')) if match.group('text') else '', pml)
 
@@ -171,9 +176,7 @@ class PML_HTMLizer(object):
         # &. It will display as &amp;
         pml = pml.replace('&', '&amp;')
 
-        pml = re.sub(r'(?<=\\x)(?P<text>.*?)(?=\\x)', lambda match: '="%s"%s' % (self.strip_pml(match.group('text')), match.group('text')), pml)
+        # Replace \\a and \\U with either the unicode character or the entity.
-        pml = re.sub(r'(?<=\\X[0-4])(?P<text>.*?)(?=\\X[0-4])', lambda match: '="%s"%s' % (self.strip_pml(match.group('text')), match.group('text')), pml)
-
         pml = re.sub(r'\\a(?P<num>\d{3})', lambda match: '&#%s;' % match.group('num'), pml)
         pml = re.sub(r'\\U(?P<num>[0-9a-f]{4})', lambda match: '%s' % my_unichr(int(match.group('num'), 16)), pml)
 
@@ -536,6 +539,7 @@ class PML_HTMLizer(object):
                         elif '%s%s' % (c, l) == 'Sd':
                             text = self.process_code('Sd', line, 'sb')
                     elif c in 'xXC':
+                        empty = False
                         # The PML was modified eariler so x and X put the text
                         # inside of ="" so we don't have do special processing
                         # for C.
@@ -578,9 +582,6 @@ class PML_HTMLizer(object):
                 else:
                     if c != ' ':
                         empty = False
-                    if self.state['k'][0]:
-                        text = c.upper()
-                    else:
                     text = c
                 parsed.append(text)
                 c = line.read(1)

src/calibre/gui2/device.py

@@ -8,7 +8,7 @@ from functools import partial
 from binascii import unhexlify
 
 from PyQt4.Qt import QMenu, QAction, QActionGroup, QIcon, SIGNAL, QPixmap, \
-                     Qt
+                     Qt, pyqtSignal
 
 from calibre.customize.ui import available_input_formats, available_output_formats, \
     device_plugins
@@ -218,6 +218,16 @@ class DeviceManager(Thread):
         '''Return callable that returns the list of books on device as two booklists'''
         return self.create_job(self._books, done, description=_('Get list of books on device'))
 
+    def _annotations(self, path_map):
+        return self.device.get_annotations(path_map)
+
+    def annotations(self, done, path_map):
+        '''Return mapping of ids to annotations. Each annotation is of the
+        form (type, location_info, content). path_map is a mapping of
+        ids to paths on the device.'''
+        return self.create_job(self._annotations, done, args=[path_map],
+                description=_('Get annotations from device'))
+
     def _sync_booklists(self, booklists):
         '''Sync metadata to device'''
         self.device.sync_booklists(booklists, end_session=False)
@@ -298,6 +308,8 @@ class DeviceAction(QAction):
 
 class DeviceMenu(QMenu):
 
+    fetch_annotations = pyqtSignal()
+
     def __init__(self, parent=None):
         QMenu.__init__(self, parent)
         self.group = QActionGroup(self)
@@ -389,10 +401,16 @@ class DeviceMenu(QMenu):
 
         self.connect(self.group, SIGNAL('triggered(QAction*)'),
                 self.change_default_action)
-        self.enable_device_actions(False)
         if opts.accounts:
             self.addSeparator()
             self.addMenu(self.email_to_menu)
+        self.addSeparator()
+        annot = self.addAction(_('Fetch annotations (experimental)'))
+        annot.setEnabled(False)
+        annot.triggered.connect(lambda x :
+                self.fetch_annotations.emit())
+        self.annotation_action = annot
+        self.enable_device_actions(False)
 
     def change_default_action(self, action):
         config['default_send_to_device_action'] = repr(action)
@@ -409,7 +427,8 @@ class DeviceMenu(QMenu):
                 self.action_triggered(action)
                 break
 
-    def enable_device_actions(self, enable, card_prefix=(None, None)):
+    def enable_device_actions(self, enable, card_prefix=(None, None),
+            device=None):
         for action in self.actions:
             if action.dest in ('main:', 'carda:0', 'cardb:0'):
                 if not enable:
@@ -428,6 +447,9 @@ class DeviceMenu(QMenu):
                         else:
                             action.setEnabled(False)
 
+        annot_enable = enable and getattr(device, 'SUPPORTS_ANNOTATIONS', False)
+        self.annotation_action.setEnabled(annot_enable)
+
 
 class Emailer(Thread):
 

src/calibre/gui2/dialogs/progress.py

@@ -40,7 +40,6 @@ class ProgressDialog(QDialog, Ui_Dialog):
         return property(fget=fget, fset=fset)
 
 
-
     def set_min(self, min):
         self.bar.setMinimum(min)
 

src/calibre/gui2/ui.py

@@ -9,7 +9,8 @@ __docformat__ = 'restructuredtext en'
 
 '''The main GUI'''
 
-import os, shutil, sys, textwrap, collections, time
+import collections, datetime, os, shutil, sys, textwrap, time
+from collections import namedtuple
 from xml.parsers.expat import ExpatError
 from Queue import Queue, Empty
 from threading import Thread
@@ -18,10 +19,11 @@ from PyQt4.Qt import Qt, SIGNAL, QObject, QCoreApplication, QUrl, QTimer, \
                      QModelIndex, QPixmap, QColor, QPainter, QMenu, QIcon, \
                      QToolButton, QDialog, QDesktopServices, QFileDialog, \
                      QSystemTrayIcon, QApplication, QKeySequence, QAction, \
-                     QMessageBox, QStackedLayout, QHelpEvent, QInputDialog
+                     QMessageBox, QStackedLayout, QHelpEvent, QInputDialog,\
+                     QThread
 from PyQt4.QtSvg import QSvgRenderer
 
-from calibre import  prints, patheq
+from calibre import  prints, patheq, strftime
 from calibre.constants import __version__, __appname__, isfrozen, islinux, \
                     iswindows, isosx, filesystem_encoding
 from calibre.utils.filenames import ascii_filename
@@ -54,6 +56,8 @@ from calibre.gui2.dialogs.search import SearchDialog
 from calibre.gui2.dialogs.choose_format import ChooseFormatDialog
 from calibre.gui2.dialogs.book_info import BookInfo
 from calibre.ebooks import BOOK_EXTENSIONS
+from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag, NavigableString
+from calibre.library.cli import send_message as calibre_send_message
 from calibre.library.database2 import LibraryDatabase2, CoverCache
 from calibre.gui2.dialogs.confirm_delete import confirm
 
@@ -617,6 +621,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
                 self.dispatch_sync_event)
         self.connect(self.action_sync, SIGNAL('triggered(bool)'),
                 self._sync_menu.trigger_default)
+        self._sync_menu.fetch_annotations.connect(self.fetch_annotations)
 
     def add_spare_server(self, *args):
         self.spare_servers.append(Server(limit=int(config['worker_limit']/2.0)))
@@ -855,7 +860,9 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
                 self.device_manager.device.__class__.get_gui_name()+\
                         _(' detected.'), 3000)
             self.device_connected = True
-            self._sync_menu.enable_device_actions(True, self.device_manager.device.card_prefix())
+            self._sync_menu.enable_device_actions(True,
+                    self.device_manager.device.card_prefix(),
+                    self.device_manager.device)
             self.location_view.model().device_connected(self.device_manager.device)
         else:
             self.save_device_view_settings()
@@ -918,7 +925,163 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
         self.sync_catalogs()
     ############################################################################
 
+    ######################### Fetch annotations ################################
 
+    def fetch_annotations(self, *args):
+    	# Figure out a list of ids using the same logic as the catalog generation
+    	# FUnction.  Use the currently connected device to map ids to paths
+
+        def get_ids_from_selected_rows():
+			rows = self.library_view.selectionModel().selectedRows()
+			if not rows or len(rows) < 2:
+				rows = xrange(self.library_view.model().rowCount(QModelIndex()))
+			ids = map(self.library_view.model().id, rows)
+			return ids
+
+		def generate_annotation_paths(ids, db, device):
+			# Generate a dict {1:'documents/documents/Asimov, Isaac/Foundation - Isaac Asimov.epub'}
+			# These are the not the absolute paths - individual storage mount points will need to be
+			# prepended during the search
+			path_map = {}
+			for id in ids:
+				mi = db.get_metadata(id, index_is_id=True)
+				a_path = device.create_upload_path(os.path.abspath('/<storage>'), mi, 'x.mbp', create_dirs=False)
+				path_map[id] = a_path
+			return path_map
+
+		device = self.device_manager.device
+
+        if self.current_view() is not self.library_view:
+            return error_dialog(self, _('Use library only'),
+                    _('User annotations generated from main library only'),
+                    show=True)
+        db = self.library_view.model().db
+
+		# Get the list of ids
+        ids = get_ids_from_selected_rows()
+        if not ids:
+            return error_dialog(self, _('No books selected'),
+                    _('No books selected to fetch annotations from'),
+                    show=True)
+
+		# Map ids to paths
+		path_map = generate_annotation_paths(ids, db, device)
+
+		# Dispatch to devices.kindle.driver.get_annotations()
+        self.device_manager.annotations(Dispatcher(self.annotations_fetched),
+                path_map)
+
+    def annotations_fetched(self, job):
+		from calibre.devices.usbms.device import Device
+		from calibre.gui2.dialogs.progress import ProgressDialog
+
+		class Updater(QThread):
+			def __init__(self, parent, db, annotation_map):
+				QThread.__init__(self, parent)
+				self.db = db
+				self.pd = ProgressDialog(_('Merging user annotations into database'), '',
+                			             0, len(job.result), parent=parent)
+
+				self.am = annotation_map
+				self.connect(self.pd, SIGNAL('canceled()'), self.canceled)
+				self.pd.setModal(True)
+				self.pd.show()
+
+			def generate_annotation_html(self, bookmark):
+				# Returns <div class="user_annotations"> ... </div>
+				last_read_location = bookmark.last_read_location
+				timestamp = datetime.datetime.utcfromtimestamp(bookmark.timestamp)
+				percent_read = bookmark.percent_read
+
+				ka_soup = BeautifulSoup()
+				dtc = 0
+				divTag = Tag(ka_soup,'div')
+				divTag['class'] = 'user_annotations'
+
+				# Add the last-read location
+				spanTag = Tag(ka_soup, 'span')
+				spanTag['style'] = 'font-weight:bold'
+				spanTag.insert(0,NavigableString("%s<br />Last Page Read: Location %d (%d%%)" % \
+								(strftime(u'%x', timestamp.timetuple()),
+								 last_read_location/150 + 1,
+								 percent_read)))
+
+				divTag.insert(dtc, spanTag)
+				dtc += 1
+				divTag.insert(dtc, Tag(ka_soup,'br'))
+				dtc += 1
+
+				if bookmark.user_notes:
+					user_notes = bookmark.user_notes
+					annotations = []
+
+					# Add the annotations sorted by location
+					# Italicize highlighted text
+					for location in sorted(user_notes):
+						if user_notes[location]['text']:
+							annotations.append('<b>Location %d &bull; %s</b><br />%s<br />' % \
+												(location/150 + 1,
+												 user_notes[location]['type'],
+												 user_notes[location]['text'] if \
+													user_notes[location]['type'] == 'Note' else \
+													'<i>%s</i>' % user_notes[location]['text']))
+						else:
+							annotations.append('<b>Location %d &bull; %s</b><br />' % \
+												(location/150 + 1,
+												 user_notes[location]['type']))
+
+					for annotation in annotations:
+						divTag.insert(dtc, annotation)
+						dtc += 1
+
+				ka_soup.insert(0,divTag)
+				return ka_soup
+
+			def canceled(self):
+				self.pd.hide()
+
+			def start(self):
+				QApplication.processEvents()
+				for (i, id) in enumerate(self.am):
+					bm = Device.UserAnnotation(self.am[id][0],self.am[id][1])
+					user_notes_soup = self.generate_annotation_html(bm.bookmark)
+
+					mi = self.db.get_metadata(id, index_is_id=True)
+					a_offset = mi.comments.find('<div class="user_annotations">')
+					ad_offset = mi.comments.find('<hr class="annotations_divider" />')
+
+					if a_offset >= 0:
+						mi.comments = mi.comments[:a_offset]
+					if ad_offset >= 0:
+						mi.comments = mi.comments[:ad_offset]
+					if mi.comments:
+						hrTag = Tag(user_notes_soup,'hr')
+						hrTag['class'] = 'annotations_divider'
+						user_notes_soup.insert(0,hrTag)
+
+					mi.comments += user_notes_soup.prettify()
+
+					# Update library comments
+					self.db.set_comment(id, mi.comments)
+
+					self.pd.set_value(i)
+				self.pd.hide()
+				calibre_send_message()
+
+        if not job.result: return
+
+        if self.current_view() is not self.library_view:
+            return error_dialog(self, _('Use library only'),
+                    _('User annotations generated from main library only'),
+                    show=True)
+        db = self.library_view.model().db
+
+        self.__annotation_updater = Updater(self, db, job.result)
+        self.__annotation_updater.start()
+        return
+
+
+    ############################################################################
 
     ################################# Add books ################################
 

src/calibre/library/catalog.py

@@ -925,11 +925,6 @@ class EPUB_MOBI(CatalogPlugin):
                     return False
             self.fetchBooksByAuthor()
             self.fetchBookmarks()
-
-            updateLibraryComments = True
-            if updateLibraryComments:
-                self.updateLibraryComments()
-
             self.generateHTMLDescriptions()
             self.generateHTMLByAuthor()
             if self.opts.generate_titles:
@@ -1180,10 +1175,12 @@ class EPUB_MOBI(CatalogPlugin):
             Preferences|Add/Save|Sending to device, not a customized one specified in
             the Kindle plugin
             '''
+
             from cStringIO import StringIO
             from struct import unpack
 
             from calibre.devices.usbms.device import Device
+            from calibre.devices.kindle.driver import Bookmark
             from calibre.ebooks.metadata import MetaInformation
             from calibre.ebooks.metadata.mobi import StreamSlicer
 
@@ -1194,170 +1191,12 @@ class EPUB_MOBI(CatalogPlugin):
                 def save_template(self):
                     return self._save_template
 
-            class Bookmark():
-                '''
-                A simple class storing bookmark data
-                Kindle-specific
-                '''
-                def __init__(self, path, formats, id):
-                    self.book_format = None
-                    self.book_length = 0
-                    self.id = id
-                    self.last_read_location = 0
-                    self.timestamp = 0
-                    self.user_notes = None
-
-                    self.get_bookmark_data(path)
-                    self.get_book_length(path, formats)
-
-                def record(self, n):
-                    if n >= self.nrecs:
-                        raise ValueError('non-existent record %r' % n)
-                    offoff = 78 + (8 * n)
-                    start, = unpack('>I', self.data[offoff + 0:offoff + 4])
-                    stop = None
-                    if n < (self.nrecs - 1):
-                        stop, = unpack('>I', self.data[offoff + 8:offoff + 12])
-                    return StreamSlicer(self.stream, start, stop)
-
-                def hexdump(self, src, length=16):
-                    # Diagnostic
-                    FILTER=''.join([(len(repr(chr(x)))==3) and chr(x) or '.' for x in range(256)])
-                    N=0; result=''
-                    while src:
-                       s,src = src[:length],src[length:]
-                       hexa = ' '.join(["%02X"%ord(x) for x in s])
-                       s = s.translate(FILTER)
-                       result += "%04X   %-*s   %s\n" % (N, length*3, hexa, s)
-                       N+=length
-                    print result
-
-                def textdump(self, src, width=80, indent=5):
-                    tokens = src.split(' ')
-                    result='%s' % (' ' * indent)
-                    results = []
-                    while tokens:
-                        result += tokens[0].decode('mac-roman') + ' '
-                        tokens.pop(0)
-                        if len(result) > width:
-                            results.append(result)
-                            result='%s' % (' ' * indent)
-                    if result.strip():
-                        results.append(result)
-                    return results
-
-                def get_bookmark_data(self, path, fetchUserNotes=True):
-                    ''' Return the timestamp and last_read_location '''
-                    with open(path,'rb') as f:
-                        stream = StringIO(f.read())
-                        data = StreamSlicer(stream)
-                        self.timestamp, = unpack('>I', data[0x24:0x28])
-                        bpar_offset, = unpack('>I', data[0x4e:0x52])
-                        lrlo = bpar_offset + 0x0c
-                        self.last_read_location = int(unpack('>I', data[lrlo:lrlo+4])[0])
-                        entries, = unpack('>I', data[0x4a:0x4e])
-
-                        # Store the annotations/locations
-                        if fetchUserNotes:
-                            bpl = bpar_offset + 4
-                            bpar_len, = unpack('>I', data[bpl:bpl+4])
-                            bpar_len += 8
-                            #print "bpar_len: 0x%x" % bpar_len
-                            eo = bpar_offset + bpar_len
-
-                            # Walk bookmark entries
-                            #print " --- %s --- " % path
-                            #print "  last_read_location: %d" % self.magicKindleLocationCalculator(last_read_location)
-                            current_entry = 1
-                            sig = data[eo:eo+4]
-                            previous_block = None
-                            user_notes = {}
-
-                            while sig == 'DATA':
-                                text = None
-                                entry_type = None
-                                rec_len, = unpack('>I', data[eo+4:eo+8])
-                                if rec_len == 0:
-                                    current_block = "empty_data"
-                                elif  data[eo+8:eo+12] == "EBAR":
-                                    current_block = "data_header"
-                                    #entry_type = "data_header"
-                                    location, = unpack('>I', data[eo+0x34:eo+0x38])
-                                    #print "data_header location: %d" % location
-                                else:
-                                    current_block = "text_block"
-                                    if previous_block == 'empty_data':
-                                        entry_type = 'Note'
-                                    elif previous_block == 'data_header':
-                                        entry_type = 'Highlight'
-                                    text = data[eo+8:eo+8+rec_len].decode('utf-16-be')
-
-                                if entry_type:
-                                    user_notes[location] = dict(type=entry_type, id=self.id,
-                                                                text=data[eo+8:eo+8+rec_len].decode('utf-16-be'))
-                                    #print " %2d: %s %s" % (current_entry, entry_type,'at %d' % location if location else '')
-                                #if current_block == 'text_block':
-                                    #self.textdump(text)
-
-                                eo += rec_len + 8
-                                current_entry += 1
-                                previous_block = current_block
-                                sig = data[eo:eo+4]
-
-                            while sig == 'BKMK':
-                                # Fix start location for Highlights using BKMK data
-                                end_loc, = unpack('>I', data[eo+0x10:eo+0x14])
-                                if end_loc in user_notes and user_notes[end_loc]['type'] != 'Note':
-                                    start, = unpack('>I', data[eo+8:eo+12])
-                                    user_notes[start] = user_notes[end_loc]
-                                    user_notes.pop(end_loc)
-                                    #print "changing start location of %d to %d" % (end_loc,start)
-                                else:
-                                    # If a bookmark coincides with a user annotation, the locs could
-                                    # be the same - cheat by nudging -1
-                                    # Skip bookmark for last_read_location
-                                    if end_loc != self.last_read_location:
-                                        user_notes[end_loc - 1] = dict(type='Bookmark',id=self.id,text=None)
-                                rec_len, = unpack('>I', data[eo+4:eo+8])
-                                eo += rec_len + 8
-                                sig = data[eo:eo+4]
-
-                    '''
-                    for location in sorted(user_notes):
-                        print '  Location %d: %s\n%s' % self.magicKindleLocationCalculator(location),
-                                                                 user_notes[location]['type'],
-                                                '\n'.join(self.textdump(user_notes[location]['text'])))
-                    '''
-                    self.user_notes = user_notes
-
-                def get_book_length(self, path, formats):
-                    # This assumes only one of the possible formats exists on the Kindle
-                    book_fs = None
-                    for format in formats:
-                        fmt = format.rpartition('.')[2]
-                        if fmt in ['mobi','prc','azw']:
-                            book_fs = path.replace('.mbp','.%s' % fmt)
-                            if os.path.exists(book_fs):
-                                self.book_format = fmt
-                                break
-                    else:
-                        #print "no files matching library formats exist on device"
-                        self.book_length = 0
-                        return
-
-                    # Read the book len from the header
-                    with open(book_fs,'rb') as f:
-                        self.stream = StringIO(f.read())
-                        self.data = StreamSlicer(self.stream)
-                        self.nrecs, = unpack('>H', self.data[76:78])
-                        record0 = self.record(0)
-                        self.book_length = int(unpack('>I', record0[0x04:0x08])[0])
-
             if self.generateRecentlyRead:
                 self.opts.log.info("     Collecting Kindle bookmarks matching catalog entries")
 
                 d = BookmarkDevice(None)
                 d.initialize(self.opts.connected_device['save_template'])
+
                 bookmarks = {}
                 for book in self.booksByTitle:
                     original_title = book['title'][book['title'].find(':') + 2:] if book['series'] \
@@ -1380,7 +1219,6 @@ class EPUB_MOBI(CatalogPlugin):
                                 bm_found = True
                         if bm_found:
                             break
-
                 self.bookmarked_books = bookmarks
             else:
                 self.bookmarked_books = {}
@@ -3365,98 +3203,6 @@ class EPUB_MOBI(CatalogPlugin):
 
             self.ncxSoup = ncx_soup
 
-        def updateLibraryComments(self):
-            # Append user notes to library book['comments'], catalog book['description']
-            from calibre.library.cli import send_message as calibre_send_message
-
-            if self.bookmarked_books:
-                annotations_header = '<br /><hr class="series_divider" />' + \
-                                      '<i>Kindle Annotations</i><br />'
-                for id in self.bookmarked_books:
-                    last_read_location = self.bookmarked_books[id][0].last_read_location
-                    timestamp = datetime.datetime.utcfromtimestamp(self.bookmarked_books[id][0].timestamp)
-                    reading_progress = self.bookmarked_books[id][1]['reading_progress']
-                    percent_read = self.bookmarked_books[id][1]['percent_read']
-                    ka_soup = BeautifulSoup()
-                    dtc = 0
-                    divTag = Tag(ka_soup,'div')
-                    divTag['class'] = 'kindle_annotations'
-
-                    # Add the last-read location
-                    spanTag = Tag(ka_soup, 'span')
-                    spanTag['style'] = 'font-weight:bold'
-                    spanTag.insert(0,NavigableString("%s %s<br />Last Page Read: Location %d (%d%%)" % \
-                                    (strftime(u'%x', timestamp.timetuple()),
-                                     reading_progress,
-                                     self.magicKindleLocationCalculator(last_read_location),
-                                     percent_read)))
-
-                    divTag.insert(dtc, spanTag)
-                    dtc += 1
-                    divTag.insert(dtc, Tag(ka_soup,'br'))
-                    dtc += 1
-
-                    if self.bookmarked_books[id][0].user_notes:
-                        user_notes = self.bookmarked_books[id][0].user_notes
-                        annotations = []
-
-                        if False:
-                            spanTag = Tag(ka_soup, 'span')
-                            spanTag['style'] = 'font-style:italic;font-weight:bold;text-align:right'
-                            spanTag.insert(0,NavigableString("Kindle Annotations"))
-                            divTag.insert(dtc, spanTag)
-                            dtc += 1
-                            divTag.insert(dtc, Tag(ka_soup,'br'))
-                            dtc += 1
-
-                        # Add the annotations sorted by location
-                        # Italicize highlighted text
-                        for location in sorted(user_notes):
-                            if user_notes[location]['text']:
-                                annotations.append('<b>Location %d &bull; %s</b><br />%s<br />' % \
-                                                    (self.magicKindleLocationCalculator(location),
-                                                     user_notes[location]['type'],
-                                                     user_notes[location]['text'] if \
-                                                        user_notes[location]['type'] == 'Note' else \
-                                                        '<i>%s</i>' % user_notes[location]['text']))
-                            else:
-                                annotations.append('<b>Location %d &bull; %s</b><br />' % \
-                                                    (self.magicKindleLocationCalculator(location),
-                                                     user_notes[location]['type']))
-
-                        for annotation in annotations:
-                            divTag.insert(dtc, annotation)
-                            dtc += 1
-
-                    ka_soup.insert(0,divTag)
-
-                    mi = self.db.get_metadata(id, index_is_id=True)
-                    ka_offset = mi.comments.find('<div class="kindle_annotations">')
-                    kad_offset = mi.comments.find('<hr class="annotations_divider" />')
-
-                    if ka_offset >= 0:
-                        mi.comments = mi.comments[:ka_offset]
-                    if kad_offset >= 0:
-                        mi.comments = mi.comments[:kad_offset]
-                    if mi.comments:
-                        hrTag = Tag(ka_soup,'hr')
-                        hrTag['class'] = 'annotations_divider'
-                        ka_soup.insert(0,hrTag)
-
-                    mi.comments += ka_soup.prettify()
-
-                    # Update library comments
-                    self.db.set_comment(id, mi.comments)
-                    calibre_send_message()
-
-                    # Update catalog description prior to build
-                    # This might be better to do during fetchBooksByTitle?
-                    # Try self.bookmarked_books[id][1]['description']
-                    for title in self.booksByTitle:
-                        if title['id'] == id:
-                            title['description'] = mi.comments
-                            break
-
         def writeNCX(self):
             self.updateProgressFullStep("Saving NCX")