E-book viewer: Use IPC to update annotations when calibre is running. Avoids possible loss of annotations due to db being locked. See #2103990 (Problems with the Annotations Browser)

src/calibre/db/annotations.py

@@ -5,6 +5,8 @@ from collections import defaultdict
 from itertools import chain
 
 from calibre.ebooks.epub.cfi.parse import cfi_sort_key
+from calibre.utils.date import EPOCH
+from calibre.utils.iso8601 import parse_iso8601
 from polyglot.builtins import itervalues
 
 no_cfi_sort_key = cfi_sort_key('/99999999')
@@ -30,6 +32,15 @@ def highlight_sort_key(hl):
     return no_cfi_sort_key
 
 
+def annotations_as_copied_list(annots_map):
+    for atype, annots in annots_map.items():
+        for annot in annots:
+            ts = (parse_iso8601(annot['timestamp'], assume_utc=True) - EPOCH).total_seconds()
+            annot = annot.copy()
+            annot['type'] = atype
+            yield annot, ts
+
+
 def sort_annot_list_by_position_in_book(annots, annot_type):
     annots.sort(key={'bookmark': bookmark_sort_key, 'highlight': highlight_sort_key}[annot_type])
 

src/calibre/db/backend.py

@@ -338,6 +338,26 @@ def save_annotations_for_book(cursor, book_id, fmt, annots_list, user_type='loca
     cursor.executemany(
         'INSERT OR REPLACE INTO annotations (book, format, user_type, user, timestamp, annot_id, annot_type, annot_data, searchable_text)'
         ' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', data)
+
+
+def save_annotations_list_to_cursor(cursor, alist, sync_annots_user, book_id, book_fmt):
+    from calibre.db.annotations import annotations_as_copied_list, merge_annotations
+    book_fmt = book_fmt.upper()
+    amap = {}
+    for annot in annotations_for_book(cursor, book_id, book_fmt):
+        amap.setdefault(annot['type'], []).append(annot)
+    merge_annotations((x[0] for x in alist), amap)
+    if sync_annots_user:
+        other_amap = {}
+        for annot in annotations_for_book(cursor, book_id, book_fmt, user_type='web', user=sync_annots_user):
+            other_amap.setdefault(annot['type'], []).append(annot)
+        merge_annotations(amap, other_amap)
+    alist = tuple(annotations_as_copied_list(amap))
+    save_annotations_for_book(cursor, book_id, book_fmt, alist)
+    if sync_annots_user:
+        alist = tuple(annotations_as_copied_list(other_amap))
+        save_annotations_for_book(cursor, book_id, book_fmt, alist, user_type='web', user=sync_annots_user)
+
 # }}}
 
 
@@ -2400,6 +2420,11 @@ class DB:
     def annotations_for_book(self, book_id, fmt, user_type, user):
         yield from annotations_for_book(self.conn, book_id, fmt, user_type, user)
 
+    def save_annotations_list(self, book_id, book_fmt, sync_annots_user, alist):
+        conn = self.conn
+        with conn:
+            save_annotations_list_to_cursor(conn.cursor(), alist, sync_annots_user, book_id, book_fmt)
+
     def search_annotations(self,
         fts_engine_query, use_stemming, highlight_start, highlight_end, snippet_size, annotation_type,
         restrict_to_book_ids, restrict_to_user, ignore_removed=False

src/calibre/db/cache.py

@@ -3307,6 +3307,10 @@ class Cache:
                 alist.append((annot, ts))
         self._set_annotations_for_book(book_id, fmt, alist, user_type=user_type, user=user)
 
+    @write_api
+    def save_annotations_list(self, book_id: int, book_fmt: str, sync_annots_user: str, alist: list[dict]) -> None:
+        self.backend.save_annotations_list(book_id, book_fmt, sync_annots_user, alist)
+
     @write_api
     def reindex_annotations(self):
         self.backend.reindex_annotations()

src/calibre/gui2/actions/view.py

@@ -136,7 +136,8 @@ class ViewAction(InterfaceAction):
                 merge_annotations(other_annotations_map, annotations_map, merge_last_read=False)
         return {
             'book_id': book_id, 'uuid': db.field_for('uuid', book_id), 'fmt': fmt.upper(),
-            'annotations_map': annotations_map, 'library_id': getattr(self.gui.current_db.new_api, 'server_library_id', None)
+            'annotations_map': annotations_map, 'library_id': getattr(self.gui.current_db.new_api, 'server_library_id', None),
+            'calibre_pid': os.getpid(),
         }
 
     def view_format_by_id(self, id_, format, open_at=None):

src/calibre/gui2/ui.py

@@ -889,6 +889,15 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin,  # {{{
             self.activateWindow()
         elif msg.startswith('shutdown:'):
             self.quit(confirm_quit=False)
+        elif msg.startswith('save-annotations:'):
+            from calibre.gui2.viewer.integration import save_annotations_in_gui
+            try:
+                if not save_annotations_in_gui(self.library_broker, msg[len('save-annotations:'):]):
+                    print('Failed to update annotations for book from viewer, book or library not found.', file=sys.stderr)
+            except Exception:
+                import traceback
+                error_dialog(self, _('Failed to update annotations'), _(
+                    'Failed to update annotations in the database for the book being currently viewed.'), det_msg=traceback.format_exc(), show=True)
         elif msg.startswith('bookedited:'):
             parts = msg.split(':')[1:]
             try:

src/calibre/gui2/viewer/annotations.py

@@ -7,32 +7,20 @@ from io import BytesIO
 from operator import itemgetter
 from threading import Thread
 
-from calibre.db.annotations import merge_annot_lists
+from calibre.db.annotations import annotations_as_copied_list, merge_annot_lists
 from calibre.gui2.viewer.convert_book import update_book
 from calibre.gui2.viewer.integration import save_annotations_list_to_library
 from calibre.gui2.viewer.web_view import viewer_config_dir
 from calibre.srv.render_book import EPUB_FILE_TYPE_MAGIC
-from calibre.utils.date import EPOCH
-from calibre.utils.iso8601 import parse_iso8601
 from calibre.utils.serialize import json_dumps, json_loads
 from calibre.utils.zipfile import safe_replace
 from polyglot.binary import as_base64_bytes
-from polyglot.builtins import iteritems
 from polyglot.queue import Queue
 
 annotations_dir = os.path.join(viewer_config_dir, 'annots')
 parse_annotations = json_loads
 
 
-def annotations_as_copied_list(annots_map):
-    for atype, annots in iteritems(annots_map):
-        for annot in annots:
-            ts = (parse_iso8601(annot['timestamp'], assume_utc=True) - EPOCH).total_seconds()
-            annot = annot.copy()
-            annot['type'] = atype
-            yield annot, ts
-
-
 def annot_list_as_bytes(annots):
     return json_dumps(tuple(annot for annot, seconds in annots))
 
@@ -54,7 +42,7 @@ def save_annots_to_epub(path, serialized_annots):
         safe_replace(zf, 'META-INF/calibre_bookmarks.txt', BytesIO(serialized_annots), add_missing=True)
 
 
-def save_annotations(annotations_list, annotations_path_key, bld, pathtoebook, in_book_file, sync_annots_user):
+def save_annotations(annotations_list, annotations_path_key, bld, pathtoebook, in_book_file, sync_annots_user, calibre_data):
     annots = annot_list_as_bytes(annotations_list)
     with open(os.path.join(annotations_dir, annotations_path_key), 'wb') as f:
         f.write(annots)
@@ -63,7 +51,7 @@ def save_annotations(annotations_list, annotations_path_key, bld, pathtoebook, i
         save_annots_to_epub(pathtoebook, annots)
         update_book(pathtoebook, before_stat, {'calibre-book-annotations.json': annots})
     if bld:
-        save_annotations_list_to_library(bld, annotations_list, sync_annots_user)
+        save_annotations_list_to_library(bld, annotations_list, sync_annots_user, calibre_data=calibre_data)
 
 
 class AnnotationsSaveWorker(Thread):
@@ -90,7 +78,7 @@ class AnnotationsSaveWorker(Thread):
             in_book_file = x['in_book_file']
             sync_annots_user = x['sync_annots_user']
             try:
-                save_annotations(annotations_list, annotations_path_key, bld, pathtoebook, in_book_file, sync_annots_user)
+                save_annotations(annotations_list, annotations_path_key, bld, pathtoebook, in_book_file, sync_annots_user, x['calibre_data'])
             except Exception:
                 import traceback
                 traceback.print_exc()
@@ -107,6 +95,11 @@ class AnnotationsSaveWorker(Thread):
             'pathtoebook': current_book_data['pathtoebook'],
             'in_book_file': in_book_file and can_save_in_book_file,
             'sync_annots_user': sync_annots_user,
+            'calibre_data': {
+                'library_id': current_book_data.get('calibre_library_id'),
+                'book_id': current_book_data.get('calibre_book_id'),
+                'book_fmt': current_book_data.get('calibre_book_fmt'),
+            },
         })
 
 

src/calibre/gui2/viewer/integration.py

@@ -52,12 +52,48 @@ def load_annotations_map_from_library(book_library_details, user_type='local', u
     return ans
 
 
-def save_annotations_list_to_library(book_library_details, alist, sync_annots_user=''):
+def send_msg_to_calibre(alist, sync_annots_user, library_id, book_id, book_fmt) -> bool:
+    import json
+
+    from calibre.gui2.listener import send_message_in_process
+
+    packet = json.dumps({
+        'alist': alist,
+        'sync_annots_user': sync_annots_user,
+        'library_id': library_id,
+        'book_id': book_id,
+        'book_fmt': book_fmt,
+    })
+    msg = 'save-annotations:' + packet
+    try:
+        send_message_in_process(msg)
+        return True
+    except Exception:
+        return False
+
+
+def save_annotations_in_gui(library_broker, msg) -> bool:
+    import json
+    data = json.loads(msg)
+    db = library_broker.get(data['library_id'])
+    if db:
+        db = db.new_api
+        with db.write_lock:
+            if db._has_format(data['book_id'], data['book_fmt']):
+                db._save_annotations_list(int(data['book_id']), data['book_fmt'].upper(), data['sync_annots_user'], data['alist'])
+                return True
+    return False
+
+
+def save_annotations_list_to_library(book_library_details, alist, sync_annots_user='', calibre_data=None):
+    calibre_data = calibre_data or {}
+    if calibre_data.get('library_id') and calibre_data.get('book_id') and send_msg_to_calibre(
+            alist, sync_annots_user, calibre_data['library_id'], calibre_data['book_id'], calibre_data['book_fmt']):
+        return
+
     import apsw
 
-    from calibre.db.annotations import merge_annotations
+    from calibre.db.backend import Connection, save_annotations_list_to_cursor
-    from calibre.db.backend import Connection, annotations_for_book, save_annotations_for_book
-    from calibre.gui2.viewer.annotations import annotations_as_copied_list
     dbpath = book_library_details['dbpath']
     try:
         conn = apsw.Connection(dbpath, flags=apsw.SQLITE_OPEN_READWRITE)
@@ -67,21 +103,7 @@ def save_annotations_list_to_library(book_library_details, alist, sync_annots_us
         conn.setbusytimeout(Connection.BUSY_TIMEOUT)
         if not database_has_annotations_support(conn.cursor()):
             return
-        amap = {}
         with conn:
-            cursor = conn.cursor()
+            save_annotations_list_to_cursor(conn.cursor(), alist, sync_annots_user, book_library_details['book_id'], book_library_details['fmt'])
-            for annot in annotations_for_book(cursor, book_library_details['book_id'], book_library_details['fmt']):
-                amap.setdefault(annot['type'], []).append(annot)
-            merge_annotations((x[0] for x in alist), amap)
-            if sync_annots_user:
-                other_amap = {}
-                for annot in annotations_for_book(cursor, book_library_details['book_id'], book_library_details['fmt'], user_type='web', user=sync_annots_user):
-                    other_amap.setdefault(annot['type'], []).append(annot)
-                merge_annotations(amap, other_amap)
-            alist = tuple(annotations_as_copied_list(amap))
-            save_annotations_for_book(cursor, book_library_details['book_id'], book_library_details['fmt'], alist)
-            if sync_annots_user:
-                alist = tuple(annotations_as_copied_list(other_amap))
-                save_annotations_for_book(cursor, book_library_details['book_id'], book_library_details['fmt'], alist, user_type='web', user=sync_annots_user)
     finally:
         conn.close()