Code to read and write annotations to the calibre db

resources/metadata_sqlite.sql

@@ -152,7 +152,7 @@ CREATE TABLE annotations ( id INTEGER PRIMARY KEY,
 	annot_type TEXT NOT NULL,
 	annot_data TEXT NOT NULL,
 	searchable_text TEXT NOT NULL,
-    UNIQUE(book, user_type, user, format, annot_id)
+    UNIQUE(book, user_type, user, format, annot_type, annot_id)
 );
 
 CREATE VIEW meta AS

src/calibre/db/backend.py

@@ -283,6 +283,41 @@ def AumSortedConcatenate():
 # }}}
 
 
+# Annotations {{{
+def annotations_for_book(cursor, book_id, fmt, user_type='local', user='viewer'):
+    for (data,) in cursor.execute(
+        'SELECT annot_data FROM annotations WHERE book=? AND format=? AND user_type=? AND user=?',
+        (book_id, fmt.upper(), user_type, user)
+    ):
+        try:
+            yield json.loads(data)
+        except Exception:
+            pass
+
+
+def save_annotations_for_book(cursor, book_id, fmt, annots_list, user_type='local', user='viewer'):
+    data = []
+    fmt = fmt.upper()
+    for annot, timestamp_in_secs in annots_list:
+        atype = annot['type']
+        if atype == 'bookmark':
+            aid = text = annot['title']
+        elif atype == 'highlight':
+            aid = annot['uuid']
+            text = annot.get('highlighed_text') or ''
+            notes = annot.get('notes') or ''
+            if notes:
+                text += '0x1f\n\n' + notes
+        else:
+            continue
+        data.append((book_id, fmt, user_type, user, timestamp_in_secs, aid, atype, json.dumps(annot), text))
+    cursor.executemany(
+        'INSERT OR REPLACE INTO annotations (book, format, user_type, user, timestamp, annot_id, annot_type, annot_data, searchable_text)'
+        ' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', data)
+    cursor.execute('INSERT OR IGNORE INTO annotations_dirtied (book) VALUES (?)', (book_id,))
+# }}}
+
+
 class Connection(apsw.Connection):  # {{{
 
     BUSY_TIMEOUT = 10000  # milliseconds
@@ -1724,6 +1759,10 @@ class DB(object):
     def get_ids_for_custom_book_data(self, name):
         return frozenset(r[0] for r in self.execute('SELECT book FROM books_plugin_data WHERE name=?', (name,)))
 
+    def annotations_for_book(self, book_id, fmt, user_type, user):
+        for x in annotations_for_book(self.conn, book_id, fmt, user_type, user):
+            yield x
+
     def conversion_options(self, book_id, fmt):
         for (data,) in self.conn.get('SELECT data FROM conversion_options WHERE book=? AND format=?', (book_id, fmt.upper())):
             if data:

src/calibre/db/cache.py

@@ -2279,6 +2279,13 @@ class Cache(object):
         if progress is not None:
             progress(_('Completed'), total, total)
 
+    @read_api
+    def annotations_map_for_book(self, book_id, fmt, user_type='local', user='viewer'):
+        ans = {}
+        for annot in self.backend.annotations_for_book(book_id, fmt, user_type, user):
+            ans.setdefault(annot['type'], []).append(annot)
+        return ans
+
 
 def import_library(library_key, importer, library_path, progress=None, abort=None):
     from calibre.db.backend import DB

src/calibre/db/schema_upgrades.py

@@ -715,7 +715,7 @@ CREATE TABLE annotations ( id INTEGER PRIMARY KEY,
     annot_type TEXT NOT NULL,
     annot_data TEXT NOT NULL,
     searchable_text TEXT NOT NULL,
-    UNIQUE(book, user_type, user, format, annot_id)
+    UNIQUE(book, user_type, user, format, annot_type, annot_id)
 );
 
 DROP INDEX IF EXISTS annot_idx;

src/calibre/gui2/actions/view.py

@@ -6,7 +6,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os, time
+import os, time, json
 from functools import partial
 
 from PyQt5.Qt import Qt, QAction, pyqtSignal
@@ -19,7 +19,7 @@ from calibre.gui2.dialogs.choose_format import ChooseFormatDialog
 from calibre.utils.config import prefs, tweaks
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.gui2.actions import InterfaceAction
-from polyglot.builtins import unicode_type
+from polyglot.builtins import unicode_type, as_bytes
 
 
 class HistoryAction(QAction):
@@ -99,13 +99,21 @@ class ViewAction(InterfaceAction):
         id_ = self.gui.library_view.model().id(row)
         self.view_format_by_id(id_, format)
 
+    def calibre_book_data(self, book_id, fmt):
+        db = self.gui.current_db.new_api
+        annotations_map = db.annotations_map_for_book(book_id, fmt)
+        return {
+            'book_id': book_id, 'uuid': db.field_for('uuid', book_id), 'fmt': fmt.upper(),
+            'annotations_map': annotations_map,
+        }
+
     def view_format_by_id(self, id_, format):
         db = self.gui.current_db
         fmt_path = db.format_abspath(id_, format,
                 index_is_id=True)
         if fmt_path:
             title = db.title(id_, index_is_id=True)
-            self._view_file(fmt_path)
+            self._view_file(fmt_path, calibre_book_data=self.calibre_book_data(id_, format))
             self.update_history([(id_, title)])
 
     def book_downloaded_for_viewing(self, job):
@@ -114,15 +122,20 @@ class ViewAction(InterfaceAction):
             return
         self._view_file(job.result)
 
-    def _launch_viewer(self, name=None, viewer='ebook-viewer', internal=True):
+    def _launch_viewer(self, name=None, viewer='ebook-viewer', internal=True, calibre_book_data=None):
         self.gui.setCursor(Qt.BusyCursor)
         try:
             if internal:
                 args = [viewer]
                 if isosx and 'ebook' in viewer:
                     args.append('--raise-window')
+
                 if name is not None:
                     args.append(name)
+                    if calibre_book_data is not None:
+                        with PersistentTemporaryFile('.json') as ptf:
+                            ptf.write(as_bytes(json.dumps(calibre_book_data)))
+                            args.append('--internal-book-data=' + ptf.name)
                 self.gui.job_manager.launch_gui_app(viewer,
                         kwargs=dict(args=args))
             else:
@@ -149,12 +162,12 @@ class ViewAction(InterfaceAction):
         finally:
             self.gui.unsetCursor()
 
-    def _view_file(self, name):
+    def _view_file(self, name, calibre_book_data=None):
         ext = os.path.splitext(name)[1].upper().replace('.',
                 '').replace('ORIGINAL_', '')
         viewer = 'lrfviewer' if ext == 'LRF' else 'ebook-viewer'
         internal = self.force_internal_viewer or ext in config['internally_viewed_formats']
-        self._launch_viewer(name, viewer, internal)
+        self._launch_viewer(name, viewer, internal, calibre_book_data=calibre_book_data)
 
     def view_specific_format(self, triggered):
         rows = list(self.gui.library_view.selectionModel().selectedRows())

src/calibre/gui2/viewer/annotations.py

@@ -3,17 +3,26 @@
 # License: GPL v3 Copyright: 2019, Kovid Goyal <kovid at kovidgoyal.net>
 
 
+import os
 from collections import defaultdict
 from io import BytesIO
 from operator import itemgetter
+from threading import Thread
 
+from calibre.gui2.viewer.convert_book import update_book
+from calibre.gui2.viewer.integration import save_annotations_list_to_library
+from calibre.gui2.viewer.web_view import viewer_config_dir
 from calibre.srv.render_book import (
     EPUB_FILE_TYPE_MAGIC, parse_annotation, parse_annotations as _parse_annotations
 )
+from calibre.utils.date import EPOCH
 from calibre.utils.serialize import json_dumps
 from calibre.utils.zipfile import safe_replace
 from polyglot.binary import as_base64_bytes
 from polyglot.builtins import iteritems, itervalues
+from polyglot.queue import Queue
+
+annotations_dir = os.path.join(viewer_config_dir, 'annots')
 
 
 def parse_annotations(raw):
@@ -53,14 +62,17 @@ def serialize_annotation(annot):
     return annot
 
 
-def serialize_annotations(annots_map):
+def annotations_as_copied_list(annots_map):
-    ans = []
     for atype, annots in iteritems(annots_map):
         for annot in annots:
+            ts = (annot['timestamp'] - EPOCH).total_seconds()
             annot = serialize_annotation(annot)
             annot['type'] = atype
-            ans.append(annot)
+            yield annot, ts
-    return json_dumps(ans)
+
+
+def annot_list_as_bytes(annots):
+    return json_dumps(tuple(annot for annot, seconds in annots))
 
 
 def split_lines(chunk, length=80):
@@ -78,3 +90,56 @@ def save_annots_to_epub(path, serialized_annots):
     with zf:
         serialized_annots = EPUB_FILE_TYPE_MAGIC + b'\n'.join(split_lines(as_base64_bytes(serialized_annots)))
         safe_replace(zf, 'META-INF/calibre_bookmarks.txt', BytesIO(serialized_annots), add_missing=True)
+
+
+def save_annotations(annotations_list, annotations_path_key, bld, pathtoebook, in_book_file):
+    annots = annot_list_as_bytes(annotations_list)
+    with open(os.path.join(annotations_dir, annotations_path_key), 'wb') as f:
+        f.write(annots)
+    if in_book_file and os.access(pathtoebook, os.W_OK):
+        before_stat = os.stat(pathtoebook)
+        save_annots_to_epub(pathtoebook, annots)
+        update_book(pathtoebook, before_stat, {'calibre-book-annotations.json': annots})
+    if bld:
+        save_annotations_list_to_library(bld, annotations_list)
+
+
+class AnnotationsSaveWorker(Thread):
+
+    def __init__(self):
+        Thread.__init__(self, name='AnnotSaveWorker')
+        self.daemon = True
+        self.queue = Queue()
+
+    def shutdown(self):
+        if self.is_alive():
+            self.queue.put(None)
+            self.join()
+
+    def run(self):
+        while True:
+            x = self.queue.get()
+            if x is None:
+                return
+            annotations_list = x['annotations_list']
+            annotations_path_key = x['annotations_path_key']
+            bld = x['book_library_details']
+            pathtoebook = x['pathtoebook']
+            in_book_file = x['in_book_file']
+            try:
+                save_annotations(annotations_list, annotations_path_key, bld, pathtoebook, in_book_file)
+            except Exception:
+                import traceback
+                traceback.print_exc()
+
+    def save_annotations(self, current_book_data, in_book_file=True):
+        alist = tuple(annotations_as_copied_list(current_book_data['annotations_map']))
+        ebp = current_book_data['pathtoebook']
+        can_save_in_book_file = ebp.lower.endswith('.epub')
+        self.queue.put({
+            'annotations_list': alist,
+            'annotations_path_key': current_book_data['annotations_path_key'],
+            'book_library_details': current_book_data['book_library_details'],
+            'pathtoebook': current_book_data['pathtoebook'],
+            'in_book_file': in_book_file and can_save_in_book_file
+        })

src/calibre/gui2/viewer/integration.py

@@ -5,10 +5,7 @@
 import os
 import re
 
-from polyglot.functools import lru_cache
 
-
-@lru_cache(maxsize=2)
 def get_book_library_details(absolute_path_to_ebook):
     absolute_path_to_ebook = os.path.abspath(os.path.expanduser(absolute_path_to_ebook))
     base = os.path.dirname(absolute_path_to_ebook)
@@ -22,3 +19,37 @@ def get_book_library_details(absolute_path_to_ebook):
     if not os.path.exists(dbpath):
         return
     return {'dbpath': dbpath, 'book_id': book_id, 'fmt': absolute_path_to_ebook.rpartition('.')[-1].upper()}
+
+
+def load_annotations_map_from_library(book_library_details):
+    import apsw
+    from calibre.db.backend import annotations_for_book, Connection
+    ans = {}
+    dbpath = book_library_details['dbpath']
+    try:
+        conn = apsw.Connection(dbpath, flags=apsw.SQLITE_OPEN_READONLY)
+    except Exception:
+        return ans
+    try:
+        conn.setbusytimeout(Connection.BUSY_TIMEOUT)
+        for annot in annotations_for_book(conn.cursor(), book_library_details['book_id'], book_library_details['fmt']):
+            ans.setdefault(annot['type'], []).append(annot)
+    finally:
+        conn.close()
+    return ans
+
+
+def save_annotations_list_to_library(book_library_details, alist):
+    import apsw
+    from calibre.db.backend import save_annotations_for_book, Connection
+    dbpath = book_library_details['dbpath']
+    try:
+        conn = apsw.Connection(dbpath, flags=apsw.SQLITE_OPEN_READWRITE)
+    except Exception:
+        return
+    try:
+        conn.setbusytimeout(Connection.BUSY_TIMEOUT)
+        with conn:
+            save_annotations_for_book(conn.cursor(), book_library_details['book_id'], book_library_details['fmt'], alist)
+    finally:
+        conn.close()

src/calibre/gui2/viewer/main.py

@@ -3,6 +3,7 @@
 # License: GPL v3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
 
 
+import json
 import os
 import sys
 from threading import Thread
@@ -195,6 +196,23 @@ def main(args=sys.argv):
     scheme.setFlags(QWebEngineUrlScheme.SecureScheme)
     QWebEngineUrlScheme.registerScheme(scheme)
     override = 'calibre-ebook-viewer' if islinux else None
+    processed_args = []
+    internal_book_data = None
+    for arg in args:
+        if arg.startswith('--internal-book-data='):
+            internal_book_data = arg.split('=', 1)[1]
+            continue
+        processed_args.append(arg)
+    if internal_book_data:
+        try:
+            with lopen(internal_book_data, 'rb') as f:
+                internal_book_data = json.load(f)
+        finally:
+            try:
+                os.remove(internal_book_data)
+            except EnvironmentError:
+                pass
+    args = processed_args
     app = Application(args, override_program_name=override, windows_app_uid=VIEWER_APP_UID)
 
     parser = option_parser()
@@ -219,7 +237,9 @@ def main(args=sys.argv):
     app.load_builtin_fonts()
     app.setWindowIcon(QIcon(I('viewer.png')))
     migrate_previous_viewer_prefs()
-    main = EbookViewer(open_at=opts.open_at, continue_reading=opts.continue_reading, force_reload=opts.force_reload)
+    main = EbookViewer(
+        open_at=opts.open_at, continue_reading=opts.continue_reading, force_reload=opts.force_reload,
+        calibre_book_data=internal_book_data)
     main.set_exception_handler()
     if len(args) > 1:
         acc.events.append(os.path.abspath(args[-1]))

src/calibre/gui2/viewer/ui.py

@@ -24,22 +24,22 @@ from calibre.gui2.dialogs.drm_error import DRMErrorMessage
 from calibre.gui2.image_popup import ImagePopup
 from calibre.gui2.main_window import MainWindow
 from calibre.gui2.viewer.annotations import (
-    merge_annotations, parse_annotations, save_annots_to_epub, serialize_annotation,
+    AnnotationsSaveWorker, annotations_dir, merge_annotations, parse_annotations,
-    serialize_annotations
+    serialize_annotation
 )
 from calibre.gui2.viewer.bookmarks import BookmarkManager
-from calibre.gui2.viewer.convert_book import (
+from calibre.gui2.viewer.convert_book import clean_running_workers, prepare_book
-    clean_running_workers, prepare_book, update_book
-)
 from calibre.gui2.viewer.highlights import HighlightsPanel
+from calibre.gui2.viewer.integration import (
+    get_book_library_details, load_annotations_map_from_library
+)
 from calibre.gui2.viewer.lookup import Lookup
 from calibre.gui2.viewer.overlay import LoadingOverlay
 from calibre.gui2.viewer.search import SearchPanel
 from calibre.gui2.viewer.toc import TOC, TOCSearch, TOCView
 from calibre.gui2.viewer.toolbars import ActionsToolBar
 from calibre.gui2.viewer.web_view import (
-    WebView, get_path_for_name, get_session_pref, set_book_path, viewer_config_dir,
+    WebView, get_path_for_name, get_session_pref, set_book_path, vprefs
-    vprefs
 )
 from calibre.utils.date import utcnow
 from calibre.utils.img import image_from_path
@@ -47,9 +47,7 @@ from calibre.utils.ipc.simple_worker import WorkerError
 from calibre.utils.iso8601 import parse_iso8601
 from calibre.utils.monotonic import monotonic
 from calibre.utils.serialize import json_loads
-from polyglot.builtins import as_bytes, iteritems, itervalues, as_unicode
+from polyglot.builtins import as_bytes, as_unicode, iteritems, itervalues
-
-annotations_dir = os.path.join(viewer_config_dir, 'annots')
 
 
 def is_float(x):
@@ -88,8 +86,10 @@ class EbookViewer(MainWindow):
     book_prepared = pyqtSignal(object, object)
     MAIN_WINDOW_STATE_VERSION = 1
 
-    def __init__(self, open_at=None, continue_reading=None, force_reload=False):
+    def __init__(self, open_at=None, continue_reading=None, force_reload=False, calibre_book_data=None):
         MainWindow.__init__(self, None)
+        self.annotations_saver = None
+        self.calibre_book_data_for_first_book = calibre_book_data
         self.shutting_down = self.close_forced = False
         self.force_reload = force_reload
         connect_lambda(self.book_preparation_started, self, lambda self: self.loading_overlay(_(
@@ -479,6 +479,8 @@ class EbookViewer(MainWindow):
         self.book_preparation_started.emit()
 
     def load_finished(self, ok, data):
+        cbd = self.calibre_book_data_for_first_book
+        self.calibre_book_data_for_first_book = None
         if self.shutting_down:
             return
         open_at, self.pending_open_at = self.pending_open_at, None
@@ -507,7 +509,7 @@ class EbookViewer(MainWindow):
         self.current_book_data = data
         self.current_book_data['annotations_map'] = defaultdict(list)
         self.current_book_data['annotations_path_key'] = path_key(data['pathtoebook']) + '.json'
-        self.load_book_data()
+        self.load_book_data(cbd)
         self.update_window_title()
         initial_cfi = self.initial_cfi_for_current_book()
         initial_position = {'type': 'cfi', 'data': initial_cfi} if initial_cfi else None
@@ -534,8 +536,13 @@ class EbookViewer(MainWindow):
             highlights=list(map(serialize_annotation, highlights))
         )
 
-    def load_book_data(self):
+    def load_book_data(self, calibre_book_data=None):
-        self.load_book_annotations()
+        self.current_book_data['book_library_details'] = get_book_library_details(self.current_book_data['pathtoebook'])
+        if calibre_book_data is not None:
+            self.current_book_data['calibre_book_id'] = calibre_book_data['book_id']
+            self.current_book_data['calibre_book_uuid'] = calibre_book_data['uuid']
+            self.current_book_data['calibre_book_fmt'] = calibre_book_data['fmt']
+        self.load_book_annotations(calibre_book_data)
         path = os.path.join(self.current_book_data['base'], 'calibre-book-manifest.json')
         with open(path, 'rb') as f:
             raw = f.read()
@@ -547,7 +554,7 @@ class EbookViewer(MainWindow):
         self.current_book_data['metadata'] = set_book_path.parsed_metadata
         self.current_book_data['manifest'] = set_book_path.parsed_manifest
 
-    def load_book_annotations(self):
+    def load_book_annotations(self, calibre_book_data=None):
         amap = self.current_book_data['annotations_map']
         path = os.path.join(self.current_book_data['base'], 'calibre-book-annotations.json')
         if os.path.exists(path):
@@ -559,6 +566,16 @@ class EbookViewer(MainWindow):
             with open(path, 'rb') as f:
                 raw = f.read()
             merge_annotations(parse_annotations(raw), amap)
+        if calibre_book_data is None:
+            bld = self.current_book_data['book_library_details']
+            if bld is not None:
+                amap = load_annotations_map_from_library(bld)
+                if amap:
+                    for annot_type, annots in iteritems(self.calibre_book_data_for_first_book['annotations_map']):
+                        merge_annotations(annots, amap)
+        else:
+            for annot_type, annots in iteritems(calibre_book_data['annotations_map']):
+                merge_annotations(annots, amap)
 
     def update_window_title(self):
         try:
@@ -590,17 +607,10 @@ class EbookViewer(MainWindow):
     def save_annotations(self, in_book_file=True):
         if not self.current_book_data:
             return
-        amap = self.current_book_data['annotations_map']
+        if self.annotations_saver is None:
-        annots = as_bytes(serialize_annotations(amap))
+            self.annotations_saver = AnnotationsSaveWorker()
-        with open(os.path.join(annotations_dir, self.current_book_data['annotations_path_key']), 'wb') as f:
+            self.annotations_saver.start()
-            f.write(annots)
+        self.annotations_saver.save_annotations(self.current_book_data, in_book_file and get_session_pref('save_annotations_in_ebook', default=True))
-        if in_book_file and self.current_book_data.get('pathtoebook', '').lower().endswith(
-                '.epub') and get_session_pref('save_annotations_in_ebook', default=True):
-            path = self.current_book_data['pathtoebook']
-            if os.access(path, os.W_OK):
-                before_stat = os.stat(path)
-                save_annots_to_epub(path, annots)
-                update_book(path, before_stat, {'calibre-book-annotations.json': annots})
 
     def highlights_changed(self, highlights):
         if not self.current_book_data:
@@ -649,11 +659,16 @@ class EbookViewer(MainWindow):
                 QTimer.singleShot(2000, self.force_close)
                 self.web_view.prepare_for_close()
             return
+        if self.shutting_down:
+            return
         self.shutting_down = True
         self.search_widget.shutdown()
         try:
-            self.save_annotations()
             self.save_state()
+            self.save_annotations()
+            if self.annotations_saver is not None:
+                self.annotations_saver.shutdown()
+                self.annotations_saver = None
         except Exception:
             import traceback
             traceback.print_exc()