From d6a1cbf72da3643df5998530138deafb7bb6c3d0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 27 Mar 2025 11:07:43 +0530 Subject: [PATCH] E-book viewer: Use IPC to update annotations when calibre is running. Avoids possible loss of annotations due to db being locked. See #2103990 (Problems with the Annotations Browser) --- src/calibre/db/annotations.py | 11 +++++ src/calibre/db/backend.py | 25 +++++++++++ src/calibre/db/cache.py | 4 ++ src/calibre/gui2/actions/view.py | 3 +- src/calibre/gui2/ui.py | 9 ++++ src/calibre/gui2/viewer/annotations.py | 25 ++++------- src/calibre/gui2/viewer/integration.py | 60 ++++++++++++++++++-------- 7 files changed, 101 insertions(+), 36 deletions(-) diff --git a/src/calibre/db/annotations.py b/src/calibre/db/annotations.py index 7379c014ca..ce65b3854a 100644 --- a/src/calibre/db/annotations.py +++ b/src/calibre/db/annotations.py @@ -5,6 +5,8 @@ from collections import defaultdict from itertools import chain from calibre.ebooks.epub.cfi.parse import cfi_sort_key +from calibre.utils.date import EPOCH +from calibre.utils.iso8601 import parse_iso8601 from polyglot.builtins import itervalues no_cfi_sort_key = cfi_sort_key('/99999999') @@ -30,6 +32,15 @@ def highlight_sort_key(hl): return no_cfi_sort_key +def annotations_as_copied_list(annots_map): + for atype, annots in annots_map.items(): + for annot in annots: + ts = (parse_iso8601(annot['timestamp'], assume_utc=True) - EPOCH).total_seconds() + annot = annot.copy() + annot['type'] = atype + yield annot, ts + + def sort_annot_list_by_position_in_book(annots, annot_type): annots.sort(key={'bookmark': bookmark_sort_key, 'highlight': highlight_sort_key}[annot_type]) diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index abbfbb02ec..74942f6831 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -338,6 +338,26 @@ def save_annotations_for_book(cursor, book_id, fmt, annots_list, user_type='loca cursor.executemany( 'INSERT OR REPLACE INTO annotations (book, format, user_type, user, timestamp, annot_id, annot_type, annot_data, searchable_text)' ' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', data) + + +def save_annotations_list_to_cursor(cursor, alist, sync_annots_user, book_id, book_fmt): + from calibre.db.annotations import annotations_as_copied_list, merge_annotations + book_fmt = book_fmt.upper() + amap = {} + for annot in annotations_for_book(cursor, book_id, book_fmt): + amap.setdefault(annot['type'], []).append(annot) + merge_annotations((x[0] for x in alist), amap) + if sync_annots_user: + other_amap = {} + for annot in annotations_for_book(cursor, book_id, book_fmt, user_type='web', user=sync_annots_user): + other_amap.setdefault(annot['type'], []).append(annot) + merge_annotations(amap, other_amap) + alist = tuple(annotations_as_copied_list(amap)) + save_annotations_for_book(cursor, book_id, book_fmt, alist) + if sync_annots_user: + alist = tuple(annotations_as_copied_list(other_amap)) + save_annotations_for_book(cursor, book_id, book_fmt, alist, user_type='web', user=sync_annots_user) + # }}} @@ -2400,6 +2420,11 @@ class DB: def annotations_for_book(self, book_id, fmt, user_type, user): yield from annotations_for_book(self.conn, book_id, fmt, user_type, user) + def save_annotations_list(self, book_id, book_fmt, sync_annots_user, alist): + conn = self.conn + with conn: + save_annotations_list_to_cursor(conn.cursor(), alist, sync_annots_user, book_id, book_fmt) + def search_annotations(self, fts_engine_query, use_stemming, highlight_start, highlight_end, snippet_size, annotation_type, restrict_to_book_ids, restrict_to_user, ignore_removed=False diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index c8c772a5b8..fd71d1c8fe 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -3307,6 +3307,10 @@ class Cache: alist.append((annot, ts)) self._set_annotations_for_book(book_id, fmt, alist, user_type=user_type, user=user) + @write_api + def save_annotations_list(self, book_id: int, book_fmt: str, sync_annots_user: str, alist: list[dict]) -> None: + self.backend.save_annotations_list(book_id, book_fmt, sync_annots_user, alist) + @write_api def reindex_annotations(self): self.backend.reindex_annotations() diff --git a/src/calibre/gui2/actions/view.py b/src/calibre/gui2/actions/view.py index 3d6205a4b4..f5b16ba0b8 100644 --- a/src/calibre/gui2/actions/view.py +++ b/src/calibre/gui2/actions/view.py @@ -136,7 +136,8 @@ class ViewAction(InterfaceAction): merge_annotations(other_annotations_map, annotations_map, merge_last_read=False) return { 'book_id': book_id, 'uuid': db.field_for('uuid', book_id), 'fmt': fmt.upper(), - 'annotations_map': annotations_map, 'library_id': getattr(self.gui.current_db.new_api, 'server_library_id', None) + 'annotations_map': annotations_map, 'library_id': getattr(self.gui.current_db.new_api, 'server_library_id', None), + 'calibre_pid': os.getpid(), } def view_format_by_id(self, id_, format, open_at=None): diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 7a73525d88..ba5b366302 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -889,6 +889,15 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ self.activateWindow() elif msg.startswith('shutdown:'): self.quit(confirm_quit=False) + elif msg.startswith('save-annotations:'): + from calibre.gui2.viewer.integration import save_annotations_in_gui + try: + if not save_annotations_in_gui(self.library_broker, msg[len('save-annotations:'):]): + print('Failed to update annotations for book from viewer, book or library not found.', file=sys.stderr) + except Exception: + import traceback + error_dialog(self, _('Failed to update annotations'), _( + 'Failed to update annotations in the database for the book being currently viewed.'), det_msg=traceback.format_exc(), show=True) elif msg.startswith('bookedited:'): parts = msg.split(':')[1:] try: diff --git a/src/calibre/gui2/viewer/annotations.py b/src/calibre/gui2/viewer/annotations.py index 64d40514bb..2e41ddb54a 100644 --- a/src/calibre/gui2/viewer/annotations.py +++ b/src/calibre/gui2/viewer/annotations.py @@ -7,32 +7,20 @@ from io import BytesIO from operator import itemgetter from threading import Thread -from calibre.db.annotations import merge_annot_lists +from calibre.db.annotations import annotations_as_copied_list, merge_annot_lists from calibre.gui2.viewer.convert_book import update_book from calibre.gui2.viewer.integration import save_annotations_list_to_library from calibre.gui2.viewer.web_view import viewer_config_dir from calibre.srv.render_book import EPUB_FILE_TYPE_MAGIC -from calibre.utils.date import EPOCH -from calibre.utils.iso8601 import parse_iso8601 from calibre.utils.serialize import json_dumps, json_loads from calibre.utils.zipfile import safe_replace from polyglot.binary import as_base64_bytes -from polyglot.builtins import iteritems from polyglot.queue import Queue annotations_dir = os.path.join(viewer_config_dir, 'annots') parse_annotations = json_loads -def annotations_as_copied_list(annots_map): - for atype, annots in iteritems(annots_map): - for annot in annots: - ts = (parse_iso8601(annot['timestamp'], assume_utc=True) - EPOCH).total_seconds() - annot = annot.copy() - annot['type'] = atype - yield annot, ts - - def annot_list_as_bytes(annots): return json_dumps(tuple(annot for annot, seconds in annots)) @@ -54,7 +42,7 @@ def save_annots_to_epub(path, serialized_annots): safe_replace(zf, 'META-INF/calibre_bookmarks.txt', BytesIO(serialized_annots), add_missing=True) -def save_annotations(annotations_list, annotations_path_key, bld, pathtoebook, in_book_file, sync_annots_user): +def save_annotations(annotations_list, annotations_path_key, bld, pathtoebook, in_book_file, sync_annots_user, calibre_data): annots = annot_list_as_bytes(annotations_list) with open(os.path.join(annotations_dir, annotations_path_key), 'wb') as f: f.write(annots) @@ -63,7 +51,7 @@ def save_annotations(annotations_list, annotations_path_key, bld, pathtoebook, i save_annots_to_epub(pathtoebook, annots) update_book(pathtoebook, before_stat, {'calibre-book-annotations.json': annots}) if bld: - save_annotations_list_to_library(bld, annotations_list, sync_annots_user) + save_annotations_list_to_library(bld, annotations_list, sync_annots_user, calibre_data=calibre_data) class AnnotationsSaveWorker(Thread): @@ -90,7 +78,7 @@ class AnnotationsSaveWorker(Thread): in_book_file = x['in_book_file'] sync_annots_user = x['sync_annots_user'] try: - save_annotations(annotations_list, annotations_path_key, bld, pathtoebook, in_book_file, sync_annots_user) + save_annotations(annotations_list, annotations_path_key, bld, pathtoebook, in_book_file, sync_annots_user, x['calibre_data']) except Exception: import traceback traceback.print_exc() @@ -107,6 +95,11 @@ class AnnotationsSaveWorker(Thread): 'pathtoebook': current_book_data['pathtoebook'], 'in_book_file': in_book_file and can_save_in_book_file, 'sync_annots_user': sync_annots_user, + 'calibre_data': { + 'library_id': current_book_data.get('calibre_library_id'), + 'book_id': current_book_data.get('calibre_book_id'), + 'book_fmt': current_book_data.get('calibre_book_fmt'), + }, }) diff --git a/src/calibre/gui2/viewer/integration.py b/src/calibre/gui2/viewer/integration.py index b260d90a43..33c72ae836 100644 --- a/src/calibre/gui2/viewer/integration.py +++ b/src/calibre/gui2/viewer/integration.py @@ -52,12 +52,48 @@ def load_annotations_map_from_library(book_library_details, user_type='local', u return ans -def save_annotations_list_to_library(book_library_details, alist, sync_annots_user=''): +def send_msg_to_calibre(alist, sync_annots_user, library_id, book_id, book_fmt) -> bool: + import json + + from calibre.gui2.listener import send_message_in_process + + packet = json.dumps({ + 'alist': alist, + 'sync_annots_user': sync_annots_user, + 'library_id': library_id, + 'book_id': book_id, + 'book_fmt': book_fmt, + }) + msg = 'save-annotations:' + packet + try: + send_message_in_process(msg) + return True + except Exception: + return False + + +def save_annotations_in_gui(library_broker, msg) -> bool: + import json + data = json.loads(msg) + db = library_broker.get(data['library_id']) + if db: + db = db.new_api + with db.write_lock: + if db._has_format(data['book_id'], data['book_fmt']): + db._save_annotations_list(int(data['book_id']), data['book_fmt'].upper(), data['sync_annots_user'], data['alist']) + return True + return False + + +def save_annotations_list_to_library(book_library_details, alist, sync_annots_user='', calibre_data=None): + calibre_data = calibre_data or {} + if calibre_data.get('library_id') and calibre_data.get('book_id') and send_msg_to_calibre( + alist, sync_annots_user, calibre_data['library_id'], calibre_data['book_id'], calibre_data['book_fmt']): + return + import apsw - from calibre.db.annotations import merge_annotations - from calibre.db.backend import Connection, annotations_for_book, save_annotations_for_book - from calibre.gui2.viewer.annotations import annotations_as_copied_list + from calibre.db.backend import Connection, save_annotations_list_to_cursor dbpath = book_library_details['dbpath'] try: conn = apsw.Connection(dbpath, flags=apsw.SQLITE_OPEN_READWRITE) @@ -67,21 +103,7 @@ def save_annotations_list_to_library(book_library_details, alist, sync_annots_us conn.setbusytimeout(Connection.BUSY_TIMEOUT) if not database_has_annotations_support(conn.cursor()): return - amap = {} with conn: - cursor = conn.cursor() - for annot in annotations_for_book(cursor, book_library_details['book_id'], book_library_details['fmt']): - amap.setdefault(annot['type'], []).append(annot) - merge_annotations((x[0] for x in alist), amap) - if sync_annots_user: - other_amap = {} - for annot in annotations_for_book(cursor, book_library_details['book_id'], book_library_details['fmt'], user_type='web', user=sync_annots_user): - other_amap.setdefault(annot['type'], []).append(annot) - merge_annotations(amap, other_amap) - alist = tuple(annotations_as_copied_list(amap)) - save_annotations_for_book(cursor, book_library_details['book_id'], book_library_details['fmt'], alist) - if sync_annots_user: - alist = tuple(annotations_as_copied_list(other_amap)) - save_annotations_for_book(cursor, book_library_details['book_id'], book_library_details['fmt'], alist, user_type='web', user=sync_annots_user) + save_annotations_list_to_cursor(conn.cursor(), alist, sync_annots_user, book_library_details['book_id'], book_library_details['fmt']) finally: conn.close()