diff --git a/src/calibre/customize/__init__.py b/src/calibre/customize/__init__.py index 2d4e89492b..0d6f041736 100644 --- a/src/calibre/customize/__init__.py +++ b/src/calibre/customize/__init__.py @@ -129,18 +129,18 @@ class Plugin(object): zip_safe = False if zip_safe: sys.path.insert(0, self.plugin_path) - self._sys_insertion_path = self.plugin_path + self.sys_insertion_path = self.plugin_path else: from calibre.ptempfile import TemporaryDirectory self._sys_insertion_tdir = TemporaryDirectory('plugin_unzip') - self._sys_insertion_path = self._sys_insertion_tdir.__enter__(*args) - zf.extractall(self._sys_insertion_path) - sys.path.insert(0, self._sys_insertion_path) + self.sys_insertion_path = self._sys_insertion_tdir.__enter__(*args) + zf.extractall(self.sys_insertion_path) + sys.path.insert(0, self.sys_insertion_path) zf.close() def __exit__(self, *args): - ip, it = getattr(self, '_sys_insertion_path', None), getattr(self, + ip, it = getattr(self, 'sys_insertion_path', None), getattr(self, '_sys_insertion_tdir', None) if ip in sys.path: sys.path.remove(ip) diff --git a/src/calibre/devices/kindle/driver.py b/src/calibre/devices/kindle/driver.py index 96aa296b5d..9bbda0f533 100644 --- a/src/calibre/devices/kindle/driver.py +++ b/src/calibre/devices/kindle/driver.py @@ -8,11 +8,14 @@ __docformat__ = 'restructuredtext en' Device driver for Amazon's Kindle ''' +from cStringIO import StringIO import os import re +from struct import unpack import sys from calibre.devices.usbms.driver import USBMS +from calibre.ebooks.metadata.mobi import StreamSlicer class KINDLE(USBMS): @@ -44,6 +47,7 @@ class KINDLE(USBMS): EBOOK_DIR_CARD_A = 'documents' DELETE_EXTS = ['.mbp'] SUPPORTS_SUB_DIRS = True + SUPPORTS_ANNOTATIONS = True WIRELESS_FILE_NAME_PATTERN = re.compile( r'(?P[^-]+)-asin_(?P<asin>[a-zA-Z\d]{10,})-type_(?P<type>\w{4})-v_(?P<index>\d+).*') @@ -60,6 +64,51 @@ class KINDLE(USBMS): 'replace') return mi + def get_annotations(self, path_map): + def get_storage(): + storage = [] + if self._main_prefix: + storage.append(os.path.join(self._main_prefix, self.EBOOK_DIR_MAIN)) + if self._card_a_prefix: + storage.append(os.path.join(self._card_a_prefix, self.EBOOK_DIR_CARD_A)) + if self._card_b_prefix: + storage.append(os.path.join(self._card_b_prefix, self.EBOOK_DIR_CARD_B)) + return storage + + def resolve_mbp_paths(storage, path_map): + pop_list = [] + for id in path_map: + for vol in storage: + #print "path_map[id]: %s" % path_map[id] + mbp_path = path_map[id].replace(os.path.abspath('/<storage>'),vol) + #print "looking for mbp_path: %s" % mbp_path + if os.path.exists(mbp_path): + #print "mbp_path found" + path_map[id] = mbp_path + break + else: + #print "mbp_path not found" + pop_list.append(id) + + # Remove non-existent mbp files + for id in pop_list: + path_map.pop(id) + return path_map + + storage = get_storage() + path_map = resolve_mbp_paths(storage, path_map) + + # path_map is now a mapping of valid mbp files + # Not yet implemented - Topaz annotations + bookmarked_books = {} + MBP_FORMATS = ['azw', 'mobi', 'prc', 'txt'] + for id in path_map: + myBookmark = Bookmark(path_map[id], MBP_FORMATS, id) + bookmarked_books[id] = self.UserAnnotation(type='mobi', bookmark=myBookmark) + + # This returns as job.result in gui2.ui.annotations_fetched(self,job) + return bookmarked_books + class KINDLE2(KINDLE): @@ -79,3 +128,139 @@ class KINDLE_DX(KINDLE2): PRODUCT_ID = [0x0003] BCD = [0x0100] + +class Bookmark(): + ''' + A simple class fetching bookmark data + Kindle-specific + ''' + def __init__(self, path, formats, id): + self.book_format = None + self.book_length = 0 + self.id = id + self.last_read_location = 0 + self.timestamp = 0 + self.user_notes = None + + self.get_bookmark_data(path) + self.get_book_length(path, formats) + try: + self.percent_read = float(100*self.last_read_location / self.book_length) + except: + self.percent_read = 0 + + def record(self, n): + if n >= self.nrecs: + raise ValueError('non-existent record %r' % n) + offoff = 78 + (8 * n) + start, = unpack('>I', self.data[offoff + 0:offoff + 4]) + stop = None + if n < (self.nrecs - 1): + stop, = unpack('>I', self.data[offoff + 8:offoff + 12]) + return StreamSlicer(self.stream, start, stop) + + def get_bookmark_data(self, path, fetchUserNotes=True): + ''' Return the timestamp and last_read_location ''' + with open(path,'rb') as f: + stream = StringIO(f.read()) + data = StreamSlicer(stream) + self.timestamp, = unpack('>I', data[0x24:0x28]) + bpar_offset, = unpack('>I', data[0x4e:0x52]) + lrlo = bpar_offset + 0x0c + self.last_read_location = int(unpack('>I', data[lrlo:lrlo+4])[0]) + entries, = unpack('>I', data[0x4a:0x4e]) + + # Store the annotations/locations + if fetchUserNotes: + bpl = bpar_offset + 4 + bpar_len, = unpack('>I', data[bpl:bpl+4]) + bpar_len += 8 + #print "bpar_len: 0x%x" % bpar_len + eo = bpar_offset + bpar_len + + # Walk bookmark entries + #print " --- %s --- " % path + #print " last_read_location: %d" % self.magicKindleLocationCalculator(last_read_location) + current_entry = 1 + sig = data[eo:eo+4] + previous_block = None + user_notes = {} + + while sig == 'DATA': + text = None + entry_type = None + rec_len, = unpack('>I', data[eo+4:eo+8]) + if rec_len == 0: + current_block = "empty_data" + elif data[eo+8:eo+12] == "EBAR": + current_block = "data_header" + #entry_type = "data_header" + location, = unpack('>I', data[eo+0x34:eo+0x38]) + #print "data_header location: %d" % location + else: + current_block = "text_block" + if previous_block == 'empty_data': + entry_type = 'Note' + elif previous_block == 'data_header': + entry_type = 'Highlight' + text = data[eo+8:eo+8+rec_len].decode('utf-16-be') + + if entry_type: + user_notes[location] = dict(type=entry_type, id=self.id, + text=data[eo+8:eo+8+rec_len].decode('utf-16-be')) + #print " %2d: %s %s" % (current_entry, entry_type,'at %d' % location if location else '') + #if current_block == 'text_block': + #self.textdump(text) + + eo += rec_len + 8 + current_entry += 1 + previous_block = current_block + sig = data[eo:eo+4] + + while sig == 'BKMK': + # Fix start location for Highlights using BKMK data + end_loc, = unpack('>I', data[eo+0x10:eo+0x14]) + if end_loc in user_notes and user_notes[end_loc]['type'] != 'Note': + start, = unpack('>I', data[eo+8:eo+12]) + user_notes[start] = user_notes[end_loc] + user_notes.pop(end_loc) + #print "changing start location of %d to %d" % (end_loc,start) + else: + # If a bookmark coincides with a user annotation, the locs could + # be the same - cheat by nudging -1 + # Skip bookmark for last_read_location + if end_loc != self.last_read_location: + user_notes[end_loc - 1] = dict(type='Bookmark',id=self.id,text=None) + rec_len, = unpack('>I', data[eo+4:eo+8]) + eo += rec_len + 8 + sig = data[eo:eo+4] + + ''' + for location in sorted(user_notes): + print ' Location %d: %s\n%s' % self.magicKindleLocationCalculator(location), + user_notes[location]['type'], + '\n'.join(self.textdump(user_notes[location]['text']))) + ''' + self.user_notes = user_notes + + def get_book_length(self, path, formats): + # This assumes only one of the possible formats exists on the Kindle + book_fs = None + for format in formats: + fmt = format.rpartition('.')[2] + book_fs = path.replace('.mbp','.%s' % fmt) + if os.path.exists(book_fs): + self.book_format = fmt + break + else: + #print "no files matching library formats exist on device" + self.book_length = 0 + return + + # Read the book len from the header + with open(book_fs,'rb') as f: + self.stream = StringIO(f.read()) + self.data = StreamSlicer(self.stream) + self.nrecs, = unpack('>H', self.data[76:78]) + record0 = self.record(0) + self.book_length = int(unpack('>I', record0[0x04:0x08])[0]) diff --git a/src/calibre/devices/usbms/device.py b/src/calibre/devices/usbms/device.py index 9c2c0d95fc..94e8785ba7 100644 --- a/src/calibre/devices/usbms/device.py +++ b/src/calibre/devices/usbms/device.py @@ -17,6 +17,8 @@ import time import re import sys import glob + +from collections import namedtuple from itertools import repeat from calibre.devices.interface import DevicePlugin @@ -88,6 +90,8 @@ class Device(DeviceConfig, DevicePlugin): EBOOK_DIR_CARD_B = '' DELETE_EXTS = [] + # Used by gui2.ui:annotations_fetched() and devices.kindle.driver:get_annotations() + UserAnnotation = namedtuple('Annotation','type, bookmark') def reset(self, key='-1', log_packets=False, report_progress=None, detected_device=None): @@ -793,6 +797,12 @@ class Device(DeviceConfig, DevicePlugin): ''' return components + def get_annotations(self, path_map): + ''' + Resolve path_map to annotation_map of files found on the device + ''' + return {} + def create_upload_path(self, path, mdata, fname, create_dirs=True): path = os.path.abspath(path) extra_components = [] diff --git a/src/calibre/devices/usbms/driver.py b/src/calibre/devices/usbms/driver.py index a28606c8cc..533bcb2357 100644 --- a/src/calibre/devices/usbms/driver.py +++ b/src/calibre/devices/usbms/driver.py @@ -123,7 +123,7 @@ class USBMS(CLI, Device): ''' :path: the full path were the associated book is located. :filename: the name of the book file without the extension. - :metatdata: metadata belonging to the book. Use metadata.thumbnail + :metadata: metadata belonging to the book. Use metadata.thumbnail for cover ''' pass diff --git a/src/calibre/ebooks/pml/pmlconverter.py b/src/calibre/ebooks/pml/pmlconverter.py index 1044dc9593..166695ff5c 100644 --- a/src/calibre/ebooks/pml/pmlconverter.py +++ b/src/calibre/ebooks/pml/pmlconverter.py @@ -79,7 +79,7 @@ class PML_HTMLizer(object): 'd': ('<span style="text-decoration: line-through;">', '</span>'), 'b': ('<span style="font-weight: bold;">', '</span>'), 'l': ('<span style="font-size: 150%;">', '</span>'), - 'k': ('<span style="font-size: 75%;">', '</span>'), + 'k': ('<span style="font-size: 75%; font-variant: small-caps;">', '</span>'), 'FN': ('<br /><br style="page-break-after: always;" /><div id="fn-%s"><p>', '</p><<small><a href="#rfn-%s">return</a></small></div>'), 'SB': ('<br /><br style="page-break-after: always;" /><div id="sb-%s"><p>', '</p><small><a href="#rsb-%s">return</a></small></div>'), } @@ -154,6 +154,11 @@ class PML_HTMLizer(object): self.file_name = '' def prepare_pml(self, pml): + # Give Chapters the form \\*='text'text\\*. This is used for generating + # the TOC later. + pml = re.sub(r'(?<=\\x)(?P<text>.*?)(?=\\x)', lambda match: '="%s"%s' % (self.strip_pml(match.group('text')), match.group('text')), pml) + pml = re.sub(r'(?<=\\X[0-4])(?P<text>.*?)(?=\\X[0-4])', lambda match: '="%s"%s' % (self.strip_pml(match.group('text')), match.group('text')), pml) + # Remove comments pml = re.sub(r'(?mus)\\v(?P<text>.*?)\\v', '', pml) @@ -163,7 +168,7 @@ class PML_HTMLizer(object): pml = re.sub(r'(?mus)(?<=.)[ ]*$', '', pml) pml = re.sub(r'(?mus)^[ ]*$', '', pml) - # Footnotes and Sidebars + # Footnotes and Sidebars. pml = re.sub(r'(?mus)<footnote\s+id="(?P<target>.+?)">\s*(?P<text>.*?)\s*</footnote>', lambda match: '\\FN="%s"%s\\FN' % (match.group('target'), match.group('text')) if match.group('text') else '', pml) pml = re.sub(r'(?mus)<sidebar\s+id="(?P<target>.+?)">\s*(?P<text>.*?)\s*</sidebar>', lambda match: '\\SB="%s"%s\\SB' % (match.group('target'), match.group('text')) if match.group('text') else '', pml) @@ -171,9 +176,7 @@ class PML_HTMLizer(object): # &. It will display as & pml = pml.replace('&', '&') - pml = re.sub(r'(?<=\\x)(?P<text>.*?)(?=\\x)', lambda match: '="%s"%s' % (self.strip_pml(match.group('text')), match.group('text')), pml) - pml = re.sub(r'(?<=\\X[0-4])(?P<text>.*?)(?=\\X[0-4])', lambda match: '="%s"%s' % (self.strip_pml(match.group('text')), match.group('text')), pml) - + # Replace \\a and \\U with either the unicode character or the entity. pml = re.sub(r'\\a(?P<num>\d{3})', lambda match: '&#%s;' % match.group('num'), pml) pml = re.sub(r'\\U(?P<num>[0-9a-f]{4})', lambda match: '%s' % my_unichr(int(match.group('num'), 16)), pml) @@ -536,6 +539,7 @@ class PML_HTMLizer(object): elif '%s%s' % (c, l) == 'Sd': text = self.process_code('Sd', line, 'sb') elif c in 'xXC': + empty = False # The PML was modified eariler so x and X put the text # inside of ="" so we don't have do special processing # for C. @@ -578,10 +582,7 @@ class PML_HTMLizer(object): else: if c != ' ': empty = False - if self.state['k'][0]: - text = c.upper() - else: - text = c + text = c parsed.append(text) c = line.read(1) diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 679e86ab48..324b0f35cf 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -8,7 +8,7 @@ from functools import partial from binascii import unhexlify from PyQt4.Qt import QMenu, QAction, QActionGroup, QIcon, SIGNAL, QPixmap, \ - Qt + Qt, pyqtSignal from calibre.customize.ui import available_input_formats, available_output_formats, \ device_plugins @@ -218,6 +218,16 @@ class DeviceManager(Thread): '''Return callable that returns the list of books on device as two booklists''' return self.create_job(self._books, done, description=_('Get list of books on device')) + def _annotations(self, path_map): + return self.device.get_annotations(path_map) + + def annotations(self, done, path_map): + '''Return mapping of ids to annotations. Each annotation is of the + form (type, location_info, content). path_map is a mapping of + ids to paths on the device.''' + return self.create_job(self._annotations, done, args=[path_map], + description=_('Get annotations from device')) + def _sync_booklists(self, booklists): '''Sync metadata to device''' self.device.sync_booklists(booklists, end_session=False) @@ -298,6 +308,8 @@ class DeviceAction(QAction): class DeviceMenu(QMenu): + fetch_annotations = pyqtSignal() + def __init__(self, parent=None): QMenu.__init__(self, parent) self.group = QActionGroup(self) @@ -389,10 +401,16 @@ class DeviceMenu(QMenu): self.connect(self.group, SIGNAL('triggered(QAction*)'), self.change_default_action) - self.enable_device_actions(False) if opts.accounts: self.addSeparator() self.addMenu(self.email_to_menu) + self.addSeparator() + annot = self.addAction(_('Fetch annotations (experimental)')) + annot.setEnabled(False) + annot.triggered.connect(lambda x : + self.fetch_annotations.emit()) + self.annotation_action = annot + self.enable_device_actions(False) def change_default_action(self, action): config['default_send_to_device_action'] = repr(action) @@ -409,7 +427,8 @@ class DeviceMenu(QMenu): self.action_triggered(action) break - def enable_device_actions(self, enable, card_prefix=(None, None)): + def enable_device_actions(self, enable, card_prefix=(None, None), + device=None): for action in self.actions: if action.dest in ('main:', 'carda:0', 'cardb:0'): if not enable: @@ -428,6 +447,9 @@ class DeviceMenu(QMenu): else: action.setEnabled(False) + annot_enable = enable and getattr(device, 'SUPPORTS_ANNOTATIONS', False) + self.annotation_action.setEnabled(annot_enable) + class Emailer(Thread): diff --git a/src/calibre/gui2/dialogs/progress.py b/src/calibre/gui2/dialogs/progress.py index 39057c5374..91f6edd252 100644 --- a/src/calibre/gui2/dialogs/progress.py +++ b/src/calibre/gui2/dialogs/progress.py @@ -40,7 +40,6 @@ class ProgressDialog(QDialog, Ui_Dialog): return property(fget=fget, fset=fset) - def set_min(self, min): self.bar.setMinimum(min) diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index abafd4e58c..e76c0ac484 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -9,7 +9,8 @@ __docformat__ = 'restructuredtext en' '''The main GUI''' -import os, shutil, sys, textwrap, collections, time +import collections, datetime, os, shutil, sys, textwrap, time +from collections import namedtuple from xml.parsers.expat import ExpatError from Queue import Queue, Empty from threading import Thread @@ -18,10 +19,11 @@ from PyQt4.Qt import Qt, SIGNAL, QObject, QCoreApplication, QUrl, QTimer, \ QModelIndex, QPixmap, QColor, QPainter, QMenu, QIcon, \ QToolButton, QDialog, QDesktopServices, QFileDialog, \ QSystemTrayIcon, QApplication, QKeySequence, QAction, \ - QMessageBox, QStackedLayout, QHelpEvent, QInputDialog + QMessageBox, QStackedLayout, QHelpEvent, QInputDialog,\ + QThread from PyQt4.QtSvg import QSvgRenderer -from calibre import prints, patheq +from calibre import prints, patheq, strftime from calibre.constants import __version__, __appname__, isfrozen, islinux, \ iswindows, isosx, filesystem_encoding from calibre.utils.filenames import ascii_filename @@ -54,6 +56,8 @@ from calibre.gui2.dialogs.search import SearchDialog from calibre.gui2.dialogs.choose_format import ChooseFormatDialog from calibre.gui2.dialogs.book_info import BookInfo from calibre.ebooks import BOOK_EXTENSIONS +from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag, NavigableString +from calibre.library.cli import send_message as calibre_send_message from calibre.library.database2 import LibraryDatabase2, CoverCache from calibre.gui2.dialogs.confirm_delete import confirm @@ -617,6 +621,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): self.dispatch_sync_event) self.connect(self.action_sync, SIGNAL('triggered(bool)'), self._sync_menu.trigger_default) + self._sync_menu.fetch_annotations.connect(self.fetch_annotations) def add_spare_server(self, *args): self.spare_servers.append(Server(limit=int(config['worker_limit']/2.0))) @@ -855,7 +860,9 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): self.device_manager.device.__class__.get_gui_name()+\ _(' detected.'), 3000) self.device_connected = True - self._sync_menu.enable_device_actions(True, self.device_manager.device.card_prefix()) + self._sync_menu.enable_device_actions(True, + self.device_manager.device.card_prefix(), + self.device_manager.device) self.location_view.model().device_connected(self.device_manager.device) else: self.save_device_view_settings() @@ -918,7 +925,163 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): self.sync_catalogs() ############################################################################ + ######################### Fetch annotations ################################ + def fetch_annotations(self, *args): + # Figure out a list of ids using the same logic as the catalog generation + # FUnction. Use the currently connected device to map ids to paths + + def get_ids_from_selected_rows(): + rows = self.library_view.selectionModel().selectedRows() + if not rows or len(rows) < 2: + rows = xrange(self.library_view.model().rowCount(QModelIndex())) + ids = map(self.library_view.model().id, rows) + return ids + + def generate_annotation_paths(ids, db, device): + # Generate a dict {1:'documents/documents/Asimov, Isaac/Foundation - Isaac Asimov.epub'} + # These are the not the absolute paths - individual storage mount points will need to be + # prepended during the search + path_map = {} + for id in ids: + mi = db.get_metadata(id, index_is_id=True) + a_path = device.create_upload_path(os.path.abspath('/<storage>'), mi, 'x.mbp', create_dirs=False) + path_map[id] = a_path + return path_map + + device = self.device_manager.device + + if self.current_view() is not self.library_view: + return error_dialog(self, _('Use library only'), + _('User annotations generated from main library only'), + show=True) + db = self.library_view.model().db + + # Get the list of ids + ids = get_ids_from_selected_rows() + if not ids: + return error_dialog(self, _('No books selected'), + _('No books selected to fetch annotations from'), + show=True) + + # Map ids to paths + path_map = generate_annotation_paths(ids, db, device) + + # Dispatch to devices.kindle.driver.get_annotations() + self.device_manager.annotations(Dispatcher(self.annotations_fetched), + path_map) + + def annotations_fetched(self, job): + from calibre.devices.usbms.device import Device + from calibre.gui2.dialogs.progress import ProgressDialog + + class Updater(QThread): + def __init__(self, parent, db, annotation_map): + QThread.__init__(self, parent) + self.db = db + self.pd = ProgressDialog(_('Merging user annotations into database'), '', + 0, len(job.result), parent=parent) + + self.am = annotation_map + self.connect(self.pd, SIGNAL('canceled()'), self.canceled) + self.pd.setModal(True) + self.pd.show() + + def generate_annotation_html(self, bookmark): + # Returns <div class="user_annotations"> ... </div> + last_read_location = bookmark.last_read_location + timestamp = datetime.datetime.utcfromtimestamp(bookmark.timestamp) + percent_read = bookmark.percent_read + + ka_soup = BeautifulSoup() + dtc = 0 + divTag = Tag(ka_soup,'div') + divTag['class'] = 'user_annotations' + + # Add the last-read location + spanTag = Tag(ka_soup, 'span') + spanTag['style'] = 'font-weight:bold' + spanTag.insert(0,NavigableString("%s<br />Last Page Read: Location %d (%d%%)" % \ + (strftime(u'%x', timestamp.timetuple()), + last_read_location/150 + 1, + percent_read))) + + divTag.insert(dtc, spanTag) + dtc += 1 + divTag.insert(dtc, Tag(ka_soup,'br')) + dtc += 1 + + if bookmark.user_notes: + user_notes = bookmark.user_notes + annotations = [] + + # Add the annotations sorted by location + # Italicize highlighted text + for location in sorted(user_notes): + if user_notes[location]['text']: + annotations.append('<b>Location %d • %s</b><br />%s<br />' % \ + (location/150 + 1, + user_notes[location]['type'], + user_notes[location]['text'] if \ + user_notes[location]['type'] == 'Note' else \ + '<i>%s</i>' % user_notes[location]['text'])) + else: + annotations.append('<b>Location %d • %s</b><br />' % \ + (location/150 + 1, + user_notes[location]['type'])) + + for annotation in annotations: + divTag.insert(dtc, annotation) + dtc += 1 + + ka_soup.insert(0,divTag) + return ka_soup + + def canceled(self): + self.pd.hide() + + def start(self): + QApplication.processEvents() + for (i, id) in enumerate(self.am): + bm = Device.UserAnnotation(self.am[id][0],self.am[id][1]) + user_notes_soup = self.generate_annotation_html(bm.bookmark) + + mi = self.db.get_metadata(id, index_is_id=True) + a_offset = mi.comments.find('<div class="user_annotations">') + ad_offset = mi.comments.find('<hr class="annotations_divider" />') + + if a_offset >= 0: + mi.comments = mi.comments[:a_offset] + if ad_offset >= 0: + mi.comments = mi.comments[:ad_offset] + if mi.comments: + hrTag = Tag(user_notes_soup,'hr') + hrTag['class'] = 'annotations_divider' + user_notes_soup.insert(0,hrTag) + + mi.comments += user_notes_soup.prettify() + + # Update library comments + self.db.set_comment(id, mi.comments) + + self.pd.set_value(i) + self.pd.hide() + calibre_send_message() + + if not job.result: return + + if self.current_view() is not self.library_view: + return error_dialog(self, _('Use library only'), + _('User annotations generated from main library only'), + show=True) + db = self.library_view.model().db + + self.__annotation_updater = Updater(self, db, job.result) + self.__annotation_updater.start() + return + + + ############################################################################ ################################# Add books ################################ diff --git a/src/calibre/library/catalog.py b/src/calibre/library/catalog.py index 412667f596..0384572891 100644 --- a/src/calibre/library/catalog.py +++ b/src/calibre/library/catalog.py @@ -925,11 +925,6 @@ class EPUB_MOBI(CatalogPlugin): return False self.fetchBooksByAuthor() self.fetchBookmarks() - - updateLibraryComments = True - if updateLibraryComments: - self.updateLibraryComments() - self.generateHTMLDescriptions() self.generateHTMLByAuthor() if self.opts.generate_titles: @@ -1180,10 +1175,12 @@ class EPUB_MOBI(CatalogPlugin): Preferences|Add/Save|Sending to device, not a customized one specified in the Kindle plugin ''' + from cStringIO import StringIO from struct import unpack from calibre.devices.usbms.device import Device + from calibre.devices.kindle.driver import Bookmark from calibre.ebooks.metadata import MetaInformation from calibre.ebooks.metadata.mobi import StreamSlicer @@ -1194,170 +1191,12 @@ class EPUB_MOBI(CatalogPlugin): def save_template(self): return self._save_template - class Bookmark(): - ''' - A simple class storing bookmark data - Kindle-specific - ''' - def __init__(self, path, formats, id): - self.book_format = None - self.book_length = 0 - self.id = id - self.last_read_location = 0 - self.timestamp = 0 - self.user_notes = None - - self.get_bookmark_data(path) - self.get_book_length(path, formats) - - def record(self, n): - if n >= self.nrecs: - raise ValueError('non-existent record %r' % n) - offoff = 78 + (8 * n) - start, = unpack('>I', self.data[offoff + 0:offoff + 4]) - stop = None - if n < (self.nrecs - 1): - stop, = unpack('>I', self.data[offoff + 8:offoff + 12]) - return StreamSlicer(self.stream, start, stop) - - def hexdump(self, src, length=16): - # Diagnostic - FILTER=''.join([(len(repr(chr(x)))==3) and chr(x) or '.' for x in range(256)]) - N=0; result='' - while src: - s,src = src[:length],src[length:] - hexa = ' '.join(["%02X"%ord(x) for x in s]) - s = s.translate(FILTER) - result += "%04X %-*s %s\n" % (N, length*3, hexa, s) - N+=length - print result - - def textdump(self, src, width=80, indent=5): - tokens = src.split(' ') - result='%s' % (' ' * indent) - results = [] - while tokens: - result += tokens[0].decode('mac-roman') + ' ' - tokens.pop(0) - if len(result) > width: - results.append(result) - result='%s' % (' ' * indent) - if result.strip(): - results.append(result) - return results - - def get_bookmark_data(self, path, fetchUserNotes=True): - ''' Return the timestamp and last_read_location ''' - with open(path,'rb') as f: - stream = StringIO(f.read()) - data = StreamSlicer(stream) - self.timestamp, = unpack('>I', data[0x24:0x28]) - bpar_offset, = unpack('>I', data[0x4e:0x52]) - lrlo = bpar_offset + 0x0c - self.last_read_location = int(unpack('>I', data[lrlo:lrlo+4])[0]) - entries, = unpack('>I', data[0x4a:0x4e]) - - # Store the annotations/locations - if fetchUserNotes: - bpl = bpar_offset + 4 - bpar_len, = unpack('>I', data[bpl:bpl+4]) - bpar_len += 8 - #print "bpar_len: 0x%x" % bpar_len - eo = bpar_offset + bpar_len - - # Walk bookmark entries - #print " --- %s --- " % path - #print " last_read_location: %d" % self.magicKindleLocationCalculator(last_read_location) - current_entry = 1 - sig = data[eo:eo+4] - previous_block = None - user_notes = {} - - while sig == 'DATA': - text = None - entry_type = None - rec_len, = unpack('>I', data[eo+4:eo+8]) - if rec_len == 0: - current_block = "empty_data" - elif data[eo+8:eo+12] == "EBAR": - current_block = "data_header" - #entry_type = "data_header" - location, = unpack('>I', data[eo+0x34:eo+0x38]) - #print "data_header location: %d" % location - else: - current_block = "text_block" - if previous_block == 'empty_data': - entry_type = 'Note' - elif previous_block == 'data_header': - entry_type = 'Highlight' - text = data[eo+8:eo+8+rec_len].decode('utf-16-be') - - if entry_type: - user_notes[location] = dict(type=entry_type, id=self.id, - text=data[eo+8:eo+8+rec_len].decode('utf-16-be')) - #print " %2d: %s %s" % (current_entry, entry_type,'at %d' % location if location else '') - #if current_block == 'text_block': - #self.textdump(text) - - eo += rec_len + 8 - current_entry += 1 - previous_block = current_block - sig = data[eo:eo+4] - - while sig == 'BKMK': - # Fix start location for Highlights using BKMK data - end_loc, = unpack('>I', data[eo+0x10:eo+0x14]) - if end_loc in user_notes and user_notes[end_loc]['type'] != 'Note': - start, = unpack('>I', data[eo+8:eo+12]) - user_notes[start] = user_notes[end_loc] - user_notes.pop(end_loc) - #print "changing start location of %d to %d" % (end_loc,start) - else: - # If a bookmark coincides with a user annotation, the locs could - # be the same - cheat by nudging -1 - # Skip bookmark for last_read_location - if end_loc != self.last_read_location: - user_notes[end_loc - 1] = dict(type='Bookmark',id=self.id,text=None) - rec_len, = unpack('>I', data[eo+4:eo+8]) - eo += rec_len + 8 - sig = data[eo:eo+4] - - ''' - for location in sorted(user_notes): - print ' Location %d: %s\n%s' % self.magicKindleLocationCalculator(location), - user_notes[location]['type'], - '\n'.join(self.textdump(user_notes[location]['text']))) - ''' - self.user_notes = user_notes - - def get_book_length(self, path, formats): - # This assumes only one of the possible formats exists on the Kindle - book_fs = None - for format in formats: - fmt = format.rpartition('.')[2] - if fmt in ['mobi','prc','azw']: - book_fs = path.replace('.mbp','.%s' % fmt) - if os.path.exists(book_fs): - self.book_format = fmt - break - else: - #print "no files matching library formats exist on device" - self.book_length = 0 - return - - # Read the book len from the header - with open(book_fs,'rb') as f: - self.stream = StringIO(f.read()) - self.data = StreamSlicer(self.stream) - self.nrecs, = unpack('>H', self.data[76:78]) - record0 = self.record(0) - self.book_length = int(unpack('>I', record0[0x04:0x08])[0]) - if self.generateRecentlyRead: self.opts.log.info(" Collecting Kindle bookmarks matching catalog entries") d = BookmarkDevice(None) d.initialize(self.opts.connected_device['save_template']) + bookmarks = {} for book in self.booksByTitle: original_title = book['title'][book['title'].find(':') + 2:] if book['series'] \ @@ -1380,7 +1219,6 @@ class EPUB_MOBI(CatalogPlugin): bm_found = True if bm_found: break - self.bookmarked_books = bookmarks else: self.bookmarked_books = {} @@ -3365,98 +3203,6 @@ class EPUB_MOBI(CatalogPlugin): self.ncxSoup = ncx_soup - def updateLibraryComments(self): - # Append user notes to library book['comments'], catalog book['description'] - from calibre.library.cli import send_message as calibre_send_message - - if self.bookmarked_books: - annotations_header = '<br /><hr class="series_divider" />' + \ - '<i>Kindle Annotations</i><br />' - for id in self.bookmarked_books: - last_read_location = self.bookmarked_books[id][0].last_read_location - timestamp = datetime.datetime.utcfromtimestamp(self.bookmarked_books[id][0].timestamp) - reading_progress = self.bookmarked_books[id][1]['reading_progress'] - percent_read = self.bookmarked_books[id][1]['percent_read'] - ka_soup = BeautifulSoup() - dtc = 0 - divTag = Tag(ka_soup,'div') - divTag['class'] = 'kindle_annotations' - - # Add the last-read location - spanTag = Tag(ka_soup, 'span') - spanTag['style'] = 'font-weight:bold' - spanTag.insert(0,NavigableString("%s %s<br />Last Page Read: Location %d (%d%%)" % \ - (strftime(u'%x', timestamp.timetuple()), - reading_progress, - self.magicKindleLocationCalculator(last_read_location), - percent_read))) - - divTag.insert(dtc, spanTag) - dtc += 1 - divTag.insert(dtc, Tag(ka_soup,'br')) - dtc += 1 - - if self.bookmarked_books[id][0].user_notes: - user_notes = self.bookmarked_books[id][0].user_notes - annotations = [] - - if False: - spanTag = Tag(ka_soup, 'span') - spanTag['style'] = 'font-style:italic;font-weight:bold;text-align:right' - spanTag.insert(0,NavigableString("Kindle Annotations")) - divTag.insert(dtc, spanTag) - dtc += 1 - divTag.insert(dtc, Tag(ka_soup,'br')) - dtc += 1 - - # Add the annotations sorted by location - # Italicize highlighted text - for location in sorted(user_notes): - if user_notes[location]['text']: - annotations.append('<b>Location %d • %s</b><br />%s<br />' % \ - (self.magicKindleLocationCalculator(location), - user_notes[location]['type'], - user_notes[location]['text'] if \ - user_notes[location]['type'] == 'Note' else \ - '<i>%s</i>' % user_notes[location]['text'])) - else: - annotations.append('<b>Location %d • %s</b><br />' % \ - (self.magicKindleLocationCalculator(location), - user_notes[location]['type'])) - - for annotation in annotations: - divTag.insert(dtc, annotation) - dtc += 1 - - ka_soup.insert(0,divTag) - - mi = self.db.get_metadata(id, index_is_id=True) - ka_offset = mi.comments.find('<div class="kindle_annotations">') - kad_offset = mi.comments.find('<hr class="annotations_divider" />') - - if ka_offset >= 0: - mi.comments = mi.comments[:ka_offset] - if kad_offset >= 0: - mi.comments = mi.comments[:kad_offset] - if mi.comments: - hrTag = Tag(ka_soup,'hr') - hrTag['class'] = 'annotations_divider' - ka_soup.insert(0,hrTag) - - mi.comments += ka_soup.prettify() - - # Update library comments - self.db.set_comment(id, mi.comments) - calibre_send_message() - - # Update catalog description prior to build - # This might be better to do during fetchBooksByTitle? - # Try self.bookmarked_books[id][1]['description'] - for title in self.booksByTitle: - if title['id'] == id: - title['description'] = mi.comments - break - def writeNCX(self): self.updateProgressFullStep("Saving NCX")