Highlights are now saved and re-applied in books

src/calibre/gui2/viewer/annotations.py

@@ -21,15 +21,15 @@ def parse_annotations(raw):
     return list(_parse_annotations(raw))
 
 
-def merge_annots_with_identical_titles(annots):
+def merge_annots_with_identical_field(annots, field='title'):
     title_groups = defaultdict(list)
     for a in annots:
-        title_groups[a['title']].append(a)
+        title_groups[a[field]].append(a)
     for tg in itervalues(title_groups):
         tg.sort(key=itemgetter('timestamp'), reverse=True)
     seen = set()
     for a in annots:
-        title = a['title']
+        title = a[field]
         if title not in seen:
             seen.add(title)
             yield title_groups[title][0]
@@ -42,13 +42,14 @@ def merge_annotations(annots, annots_map):
     lr = annots_map['last-read']
     if lr:
         lr.sort(key=itemgetter('timestamp'), reverse=True)
-    for annot_type in ('bookmark',):
+    for annot_type, field in {'bookmark': 'title', 'highlight': 'uuid'}.items():
         a = annots_map.get(annot_type)
         if a and len(a) > 1:
-            annots_map[annot_type] = list(merge_annots_with_identical_titles(a))
+            annots_map[annot_type] = list(merge_annots_with_identical_field(a, field=field))
 
 
 def serialize_annotation(annot):
+    annot = annot.copy()
     annot['timestamp'] = annot['timestamp'].isoformat()
     return annot
 
@@ -57,7 +58,7 @@ def serialize_annotations(annots_map):
     ans = []
     for atype, annots in iteritems(annots_map):
         for annot in annots:
-            annot = serialize_annotation(annot.copy())
+            annot = serialize_annotation(annot)
             annot['type'] = atype
             ans.append(annot)
     return json_dumps(ans)

src/calibre/gui2/viewer/ui.py

@@ -25,7 +25,8 @@ from calibre.gui2.dialogs.drm_error import DRMErrorMessage
 from calibre.gui2.image_popup import ImagePopup
 from calibre.gui2.main_window import MainWindow
 from calibre.gui2.viewer.annotations import (
-    merge_annotations, parse_annotations, save_annots_to_epub, serialize_annotations
+    merge_annotations, parse_annotations, save_annots_to_epub, serialize_annotation,
+    serialize_annotations
 )
 from calibre.gui2.viewer.bookmarks import BookmarkManager
 from calibre.gui2.viewer.convert_book import (
@@ -43,6 +44,7 @@ from calibre.gui2.viewer.web_view import (
 from calibre.utils.date import utcnow
 from calibre.utils.img import image_from_path
 from calibre.utils.ipc.simple_worker import WorkerError
+from calibre.utils.iso8601 import parse_iso8601
 from calibre.utils.monotonic import monotonic
 from calibre.utils.serialize import json_loads
 from polyglot.builtins import as_bytes, iteritems, itervalues
@@ -176,6 +178,7 @@ class EbookViewer(MainWindow):
         self.web_view.shortcuts_changed.connect(self.shortcuts_changed)
         self.web_view.scrollbar_context_menu.connect(self.scrollbar_context_menu)
         self.web_view.close_prep_finished.connect(self.close_prep_finished)
+        self.web_view.highlights_changed.connect(self.highlights_changed)
         self.actions_toolbar.initialize(self.web_view, self.search_dock.toggleViewAction())
         self.setCentralWidget(self.web_view)
         self.loading_overlay = LoadingOverlay(self)
@@ -486,7 +489,10 @@ class EbookViewer(MainWindow):
                 initial_position = {'type': 'ref', 'data': open_at[len('ref:'):]}
             elif is_float(open_at):
                 initial_position = {'type': 'bookpos', 'data': float(open_at)}
-        self.web_view.start_book_load(initial_position=initial_position)
+        self.web_view.start_book_load(
+            initial_position=initial_position,
+            highlights=list(map(serialize_annotation, self.current_book_data['annotations_map']['highlight']))
+        )
 
     def load_book_data(self):
         self.load_book_annotations()
@@ -556,6 +562,15 @@ class EbookViewer(MainWindow):
                 save_annots_to_epub(path, annots)
                 update_book(path, before_stat, {'calibre-book-annotations.json': annots})
 
+    def highlights_changed(self, highlights):
+        if not self.current_book_data:
+            return
+        for h in highlights:
+            h['timestamp'] = parse_iso8601(h['timestamp'], assume_utc=True)
+        amap = self.current_book_data['annotations_map']
+        amap['highlight'] = highlights
+        self.save_annotations()
+
     def save_state(self):
         with vprefs:
             vprefs['main_window_state'] = bytearray(self.saveState(self.MAIN_WINDOW_STATE_VERSION))

src/calibre/gui2/viewer/web_view.py

@@ -276,6 +276,7 @@ class ViewerBridge(Bridge):
     customize_toolbar = from_js()
     scrollbar_context_menu = from_js(object, object, object)
     close_prep_finished = from_js(object)
+    highlights_changed = from_js(object)
 
     create_view = to_js()
     start_book_load = to_js()
@@ -457,6 +458,7 @@ class WebView(RestartingWebEngineView):
     customize_toolbar = pyqtSignal()
     scrollbar_context_menu = pyqtSignal(object, object, object)
     close_prep_finished = pyqtSignal(object)
+    highlights_changed = pyqtSignal(object)
     shortcuts_changed = pyqtSignal(object)
     paged_mode_changed = pyqtSignal()
     standalone_misc_settings_changed = pyqtSignal(object)
@@ -508,6 +510,7 @@ class WebView(RestartingWebEngineView):
         self.bridge.customize_toolbar.connect(self.customize_toolbar)
         self.bridge.scrollbar_context_menu.connect(self.scrollbar_context_menu)
         self.bridge.close_prep_finished.connect(self.close_prep_finished)
+        self.bridge.highlights_changed.connect(self.highlights_changed)
         self.bridge.export_shortcut_map.connect(self.set_shortcut_map)
         self.shortcut_map = {}
         self.bridge.report_cfi.connect(self.call_callback)
@@ -597,9 +600,9 @@ class WebView(RestartingWebEngineView):
     def on_content_file_changed(self, data):
         self.current_content_file = data
 
-    def start_book_load(self, initial_position=None):
+    def start_book_load(self, initial_position=None, highlights=None):
         key = (set_book_path.path,)
-        self.execute_when_ready('start_book_load', key, initial_position, set_book_path.pathtoebook)
+        self.execute_when_ready('start_book_load', key, initial_position, set_book_path.pathtoebook, highlights or [])
 
     def execute_when_ready(self, action, *args):
         if self.bridge.ready:

src/pyj/book_list/custom_list.pyj

@@ -202,10 +202,10 @@ def render_part(part, template, book_id, metadata):
     count = rendered_count = 0
     ans = E.div()
     ans.innerHTML = part
-    walk = document.createTreeWalker(ans, NodeFilter.SHOW_TEXT, None, False)
+    iterator = document.createNodeIterator(ans, NodeFilter.SHOW_TEXT)
     replacements = v'[]'
     while True:
-        n = walk.nextNode()
+        n = iterator.nextNode()
         if not n:
             break
         rendered = E.span()

src/pyj/range_utils.pyj

@@ -4,42 +4,30 @@
 from __python__ import bound_methods, hash_literals
 
 
-def get_text_nodes(el):
+def is_non_empty_text_node(node):
-    el = el or document.body
+    return (node.nodeType is Node.TEXT_NODE or node.nodeType is Node.CDATA_SECTION_NODE) and node.nodeValue.length > 0
-    doc = el.ownerDocument or document
+
-    walker = doc.createTreeWalker(el, NodeFilter.SHOW_TEXT, None, False)
+
-    text_nodes = v'[]'
+def text_nodes_in_range(r):
+    parent = r.commonAncestorContainer
+    doc = parent.ownerDocument or document
+    iterator = doc.createNodeIterator(parent)
+    in_range = False
+    ans = v'[]'
     while True:
-        node = walker.nextNode()
+        node = iterator.nextNode()
         if not node:
             break
-        text_nodes.push(node)
+        if not in_range and node.isSameNode(r.startContainer):
-    return text_nodes
+            in_range = True
-
+        if in_range:
-
+            if is_non_empty_text_node(node):
-def create_range_from_node(node):
+                ans.push(node)
-    ans = node.ownerDocument.createRange()
+            if node.isSameNode(r.endContainer):
-    try:
+                break
-        ans.selectNode(node)
-    except:
-        ans.selectNodeContents(node)
     return ans
 
 
-def is_non_empty_text_node(node):
-    return node.textContent.length > 0
-
-
-def text_nodes_in_range(r, predicate):
-    predicate = predicate or is_non_empty_text_node
-    container = r.commonAncestorContainer
-    nodes = get_text_nodes(container.parentNode or container)
-
-    def final_predicate(node):
-        return r.intersectsNode(node) and predicate(node)
-    return nodes.filter(final_predicate)
-
-
 def remove(node):
     if node.parentNode:
         node.parentNode.removeChild(node)
@@ -65,7 +53,7 @@ def unwrap_crw(crw):
         unwrap(node)
 
 
-def create_wrapper_function(wrapper_elem, r):
+def create_wrapper_function(wrapper_elem, r, intersecting_wrappers):
     start_node = r.startContainer
     end_node = r.endContainer
     start_offset = r.startOffset
@@ -76,15 +64,17 @@ def create_wrapper_function(wrapper_elem, r):
         current_range = (node.ownerDocument or document).createRange()
         current_wrapper = wrapper_elem.cloneNode()
         current_range.selectNodeContents(node)
-        if node is start_node and start_node.nodeType is Node.TEXT_NODE:
+        if node.isSameNode(start_node):
             current_range.setStart(node, start_offset)
             start_node = current_wrapper
             start_offset = 0
-        if node is end_node and end_node.nodeType is Node.TEXT_NODE:
+        if node.isSameNode(end_node):
             current_range.setEnd(node, end_offset)
             end_node = current_wrapper
             end_offset = 1
-
+        crw = node.parentNode.dataset.calibreRangeWrapper
+        if crw:
+            intersecting_wrappers[crw] = True
         current_range.surroundContents(current_wrapper)
         return current_wrapper
 
@@ -98,49 +88,21 @@ def wrap_text_in_range(style, r):
     if not r:
         r = window.getSelection().getRangeAt(0)
     if r.isCollapsed:
-        return None
+        return None, v'[]'
 
     wrapper_elem = document.createElement('span')
     wrapper_elem.dataset.calibreRangeWrapper = v'++wrapper_counter' + ''
     if style:
         wrapper_elem.setAttribute('style', style)
 
-    wrap_node = create_wrapper_function(wrapper_elem, r)
+    intersecting_wrappers = {}
-    nodes = text_nodes_in_range(r)
+    wrap_node = create_wrapper_function(wrapper_elem, r, intersecting_wrappers)
-    nodes = nodes.map(wrap_node)
+    text_nodes_in_range(r).map(wrap_node)
-    return wrapper_elem.dataset.calibreRangeWrapper
+    crw = wrapper_elem.dataset.calibreRangeWrapper
+    v'delete intersecting_wrappers[crw]'
+    return crw, Object.keys(intersecting_wrappers)
 
 
 def reset_highlight_counter():
     nonlocal wrapper_counter
     wrapper_counter = 0
-
-
-def is_text_node(node):
-    return node.nodeType is Node.TEXT_NODE or node.nodeType is Node.CDATA_SECTION_NODE
-
-
-def range_wrappers_intersecting_selection(sel):
-    ans = {}
-    sel = sel or window.getSelection()
-    if not sel:
-        return ans
-    r = sel.getRangeAt(0)
-    if not r:
-        return ans
-    walker = document.createTreeWalker(r.commonAncestorContainer, NodeFilter.SHOW_ALL, None, False)
-    in_selection = False
-    while True:
-        node = walker.nextNode()
-        if not node:
-            break
-        if not in_selection and node is not r.startContainer:
-            continue
-        in_selection = True
-        if node.nodeType is Node.ELEMENT_NODE and node.dataset.calibreRangeWrapper:
-            ans[node.dataset.calibreRangeWrapper] = True
-        elif is_text_node(node) and node.parentNode.dataset.calibreRangeWrapper:
-            ans[node.parentNode.dataset.calibreRangeWrapper] = True
-        if node is r.endContainer:
-            break
-    return Object.keys(ans)

src/pyj/read_book/cfi.pyj

@@ -776,3 +776,14 @@ def cfi_for_selection(r):  # {{{
     }
 
 # }}}
+
+def range_from_cfi(start, end):  # {{{
+    start = decode(start)
+    end = decode(end)
+    if not start or start.error or not end or end.error:
+        return
+    r = document.createRange()
+    r.setStart(start.node, start.offset)
+    r.setEnd(end.node, end.offset)
+    return r
+# }}}

src/pyj/read_book/create_annotation.pyj

@@ -10,8 +10,64 @@ from book_list.globals import get_session_data
 from book_list.theme import cached_color_to_rgba, get_color
 from dom import clear, ensure_id, svgicon, unique_id
 from modals import error_dialog
+from read_book.globals import ui_operations
 from read_book.shortcuts import shortcut_for_key_event
 
+
+class AnnotationsManager:
+
+    def __init__(self, view):
+        self.view = view
+        self.set_highlights()
+
+    def set_highlights(self, highlights):
+        highlights = highlights or v'[]'
+        self.highlights = {h.uuid: h for h in highlights}
+
+    def remove_highlight(self, uuid):
+        h = self.highlights[uuid]
+        if h:
+            h.timestamp = Date().toISOString()
+            h.removed = True
+            v'delete h.style'
+            v'delete h.highlighted_text'
+            v'delete h.start_cfi'
+            v'delete h.end_cfi'
+            v'delete h.notes'
+            v'delete h.spine_name'
+            v'delete h.spine_index'
+
+    def add_highlight(self, msg, style, notes):
+        now = Date().toISOString()
+        for uuid in msg.removed_highlights:
+            self.remove_highlight(uuid)
+
+        annot = self.highlights[msg.uuid] = {
+            'type': 'highlight',
+            'timestamp': now,
+            'uuid': msg.uuid,
+            'highlighted_text': msg.highlighted_text,
+            'start_cfi': msg.bounds.start,
+            'end_cfi': msg.bounds.end,
+            'style': style,  # dict with color and background-color
+            'spine_name': self.view.currently_showing.name,
+            'spine_index': self.view.currently_showing.spine_index,
+        }
+        if notes:
+            annot.notes = notes
+        if ui_operations.highlights_changed:
+            ui_operations.highlights_changed(Object.values(self.highlights))
+
+    def highlights_for_currently_showing(self):
+        name = self.view.currently_showing.name
+        ans = v'[]'
+        for uuid in Object.keys(self.highlights):
+            h = self.highlights[uuid]
+            if h.spine_name is name and not h.removed and h.start_cfi:
+                ans.push(h)
+        return ans
+
+
 WAITING_FOR_CLICK = 1
 WAITING_FOR_DRAG = 2
 DRAGGING_LEFT = 3
@@ -81,6 +137,7 @@ class CreateAnnotation:
 
     def __init__(self, view):
         self.view = view
+        self.annotations_manager = self.view.annotations_manager
         self.state = WAITING_FOR_CLICK
         self.left_line_height = self.right_line_height = 8
         self.in_flow_mode = False
@@ -393,7 +450,7 @@ class CreateAnnotation:
                     _('Highlighting failed'),
                     _('Failed to apply highlighting, try adjusting extent of highlight')
                 )
-
+            self.annotations_manager.add_highlight(msg, self.current_highlight_style)
         else:
             print('Ignoring annotations message with unknown type:', msg.type)
 

src/pyj/read_book/iframe.pyj

@@ -11,11 +11,10 @@ from select import (
 
 from fs_images import fix_fullscreen_svg_images
 from iframe_comm import IframeClient
-from range_utils import (
+from range_utils import reset_highlight_counter, unwrap_crw, wrap_text_in_range
-    range_wrappers_intersecting_selection, reset_highlight_counter, unwrap_crw,
+from read_book.cfi import (
-    wrap_text_in_range
+    cfi_for_selection, range_from_cfi, scroll_to as scroll_to_cfi
 )
-from read_book.cfi import cfi_for_selection, scroll_to as scroll_to_cfi
 from read_book.extract import get_elements
 from read_book.find import reset_find_caches, select_search_result
 from read_book.flow_mode import (
@@ -244,6 +243,7 @@ class IframeBoss:
             window.URL.revokeObjectURL(self.blob_url_map[name])
         document.body.style.removeProperty('font-family')
         root_data, self.mathjax, self.blob_url_map = finalize_resources(self.book, data.name, data.resource_data)
+        self.highlights_to_apply = data.highlights
         unserialize_html(root_data, self.content_loaded, None, data.name)
 
     def on_scroll_to_frac(self, data):
@@ -345,6 +345,9 @@ class IframeBoss:
         if self.reference_mode_enabled:
             start_reference_mode()
         self.last_window_width, self.last_window_height = scroll_viewport.width(), scroll_viewport.height()
+        if self.highlights_to_apply:
+            self.apply_highlights_on_load(self.highlights_to_apply)
+            self.highlights_to_apply = None
         apply_settings()
         fix_fullscreen_svg_images()
         self.do_layout(self.is_titlepage)
@@ -654,8 +657,7 @@ class IframeBoss:
             if sel:
                 text = sel.toString()
             bounds = cfi_for_selection()
-            intersecting_wrappers = range_wrappers_intersecting_selection()
+            annot_id, intersecting_wrappers = wrap_text_in_range(data.style)
-            annot_id = wrap_text_in_range(data.style)
             removed_highlights = {}
             if annot_id is not None:
                 sel.removeAllRanges()
@@ -680,6 +682,21 @@ class IframeBoss:
         else:
             console.log('Ignoring annotations message to iframe with unknown type: ' + data.type)
 
+    def apply_highlights_on_load(self, highlights):
+        self.annot_id_uuid_map = {}
+        strcmp = v'new Intl.Collator().compare'
+        highlights.sort(def (a, b): return strcmp(a.timestamp, b.timestamp);)
+        for h in highlights:
+            r = range_from_cfi(h.start_cfi, h.end_cfi)
+            if not r:
+                continue
+            style = f'color: {h.style.color}; background-color: {h.style["background-color"]}'
+            annot_id, intersecting_wrappers = wrap_text_in_range(style, r)
+            if annot_id is not None:
+                self.annot_id_uuid_map[annot_id] = h.uuid
+                for crw in intersecting_wrappers:
+                    unwrap_crw(crw)
+                    v'delete self.annot_id_uuid_map[crw]'
 
 def main():
     main.boss = IframeBoss()

src/pyj/read_book/view.pyj

@@ -13,12 +13,12 @@ from dom import add_extra_css, build_rule, clear, set_css, svgicon, unique_id
 from iframe_comm import IframeWrapper
 from modals import error_dialog, warning_dialog
 from read_book.content_popup import ContentPopupOverlay
+from read_book.create_annotation import AnnotationsManager, CreateAnnotation
 from read_book.globals import (
     current_book, runtime, set_current_spine_item, ui_operations
 )
 from read_book.goto import get_next_section
 from read_book.open_book import add_book_to_recently_viewed
-from read_book.create_annotation import CreateAnnotation
 from read_book.overlay import Overlay
 from read_book.prefs.colors import resolve_color_scheme
 from read_book.prefs.font_size import change_font_size_by
@@ -278,6 +278,7 @@ class View:
         self.pending_load = None
         self.currently_showing = {}
         self.book_scrollbar.apply_visibility()
+        self.annotations_manager = AnnotationsManager(self)
         self.create_annotation = CreateAnnotation(self)
 
     @property
@@ -740,6 +741,7 @@ class View:
             self.content_popup_overlay.loaded_resources = {}
             self.timers.start_book(book)
         self.book = current_book.book = book
+        self.annotations_manager.set_highlights(book.highlights or v'[]')
         if runtime.is_standalone_viewer:
             add_book_to_recently_viewed(book)
         if ui_operations.update_last_read_time:
@@ -827,10 +829,13 @@ class View:
             return
         self.processing_spine_item_display = False
         initial_position = initial_position or {'replace_history':False}
-        self.currently_showing = {'name':name, 'settings':self.iframe_settings(name), 'initial_position':initial_position, 'loading':True}
-        self.show_loading()
         spine = self.book.manifest.spine
         idx = spine.indexOf(name)
+        self.currently_showing = {
+            'name':name, 'settings':self.iframe_settings(name), 'initial_position':initial_position,
+            'loading':True, 'spine_index': idx
+        }
+        self.show_loading()
         set_current_spine_item(name)
         if idx > -1:
             self.currently_showing.bookpos = 'epubcfi(/{})'.format(2 * (idx +1))
@@ -1091,6 +1096,7 @@ class View:
             initial_position=self.currently_showing.initial_position,
             settings=self.currently_showing.settings, reference_mode_enabled=self.reference_mode_enabled,
             is_titlepage=self.currently_showing.name is self.book.manifest.title_page_name,
+            highlights=self.annotations_manager.highlights_for_currently_showing(),
         )
 
     def on_content_loaded(self, data):

src/pyj/viewer-main.pyj

@@ -138,7 +138,7 @@ def show_error(title, msg, details):
     to_python.show_error(title, msg, details)
 
 
-def manifest_received(key, initial_position, pathtoebook, end_type, xhr, ev):
+def manifest_received(key, initial_position, pathtoebook, highlights, end_type, xhr, ev):
     nonlocal book
     end_type = workaround_qt_bug(xhr, end_type)
     if end_type is 'load':
@@ -147,6 +147,7 @@ def manifest_received(key, initial_position, pathtoebook, end_type, xhr, ev):
         book.manifest = data[0]
         book.metadata = book.manifest.metadata = data[1]
         book.manifest.pathtoebook = pathtoebook
+        book.highlights = highlights
         book.stored_files = {}
         book.is_complete = True
         v'delete book.manifest["metadata"]'
@@ -243,8 +244,8 @@ def show_home_page():
 
 
 @from_python
-def start_book_load(key, initial_position, pathtoebook):
+def start_book_load(key, initial_position, pathtoebook, highlights):
-    xhr = ajax('manifest', manifest_received.bind(None, key, initial_position, pathtoebook), ok_code=0)
+    xhr = ajax('manifest', manifest_received.bind(None, key, initial_position, pathtoebook, highlights), ok_code=0)
     xhr.responseType = 'json'
     xhr.send()
 
@@ -420,6 +421,8 @@ if window is window.top:
         to_python.scrollbar_context_menu(x, y, frac)
     ui_operations.close_prep_finished = def(cfi):
         to_python.close_prep_finished(cfi)
+    ui_operations.highlights_changed = def(highlights):
+        to_python.highlights_changed(highlights)
 
     document.body.appendChild(E.div(id='view'))
     window.onerror = onerror