diff --git a/resources/metadata_sqlite.sql b/resources/metadata_sqlite.sql index 8409bc986a..84237e58c9 100644 --- a/resources/metadata_sqlite.sql +++ b/resources/metadata_sqlite.sql @@ -152,7 +152,7 @@ CREATE TABLE annotations ( id INTEGER PRIMARY KEY, annot_type TEXT NOT NULL, annot_data TEXT NOT NULL, searchable_text TEXT NOT NULL, - UNIQUE(book, user_type, user, format, annot_id) + UNIQUE(book, user_type, user, format, annot_type, annot_id) ); CREATE VIEW meta AS diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index 13b1cd14d0..b326321e82 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -283,6 +283,41 @@ def AumSortedConcatenate(): # }}} +# Annotations {{{ +def annotations_for_book(cursor, book_id, fmt, user_type='local', user='viewer'): + for (data,) in cursor.execute( + 'SELECT annot_data FROM annotations WHERE book=? AND format=? AND user_type=? AND user=?', + (book_id, fmt.upper(), user_type, user) + ): + try: + yield json.loads(data) + except Exception: + pass + + +def save_annotations_for_book(cursor, book_id, fmt, annots_list, user_type='local', user='viewer'): + data = [] + fmt = fmt.upper() + for annot, timestamp_in_secs in annots_list: + atype = annot['type'] + if atype == 'bookmark': + aid = text = annot['title'] + elif atype == 'highlight': + aid = annot['uuid'] + text = annot.get('highlighed_text') or '' + notes = annot.get('notes') or '' + if notes: + text += '0x1f\n\n' + notes + else: + continue + data.append((book_id, fmt, user_type, user, timestamp_in_secs, aid, atype, json.dumps(annot), text)) + cursor.executemany( + 'INSERT OR REPLACE INTO annotations (book, format, user_type, user, timestamp, annot_id, annot_type, annot_data, searchable_text)' + ' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', data) + cursor.execute('INSERT OR IGNORE INTO annotations_dirtied (book) VALUES (?)', (book_id,)) +# }}} + + class Connection(apsw.Connection): # {{{ BUSY_TIMEOUT = 10000 # milliseconds @@ -1724,6 +1759,10 @@ class DB(object): def get_ids_for_custom_book_data(self, name): return frozenset(r[0] for r in self.execute('SELECT book FROM books_plugin_data WHERE name=?', (name,))) + def annotations_for_book(self, book_id, fmt, user_type, user): + for x in annotations_for_book(self.conn, book_id, fmt, user_type, user): + yield x + def conversion_options(self, book_id, fmt): for (data,) in self.conn.get('SELECT data FROM conversion_options WHERE book=? AND format=?', (book_id, fmt.upper())): if data: diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 8361f9e423..55f8fb08e7 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -2279,6 +2279,13 @@ class Cache(object): if progress is not None: progress(_('Completed'), total, total) + @read_api + def annotations_map_for_book(self, book_id, fmt, user_type='local', user='viewer'): + ans = {} + for annot in self.backend.annotations_for_book(book_id, fmt, user_type, user): + ans.setdefault(annot['type'], []).append(annot) + return ans + def import_library(library_key, importer, library_path, progress=None, abort=None): from calibre.db.backend import DB diff --git a/src/calibre/db/schema_upgrades.py b/src/calibre/db/schema_upgrades.py index f8fc5a0b50..d79796dbc2 100644 --- a/src/calibre/db/schema_upgrades.py +++ b/src/calibre/db/schema_upgrades.py @@ -715,7 +715,7 @@ CREATE TABLE annotations ( id INTEGER PRIMARY KEY, annot_type TEXT NOT NULL, annot_data TEXT NOT NULL, searchable_text TEXT NOT NULL, - UNIQUE(book, user_type, user, format, annot_id) + UNIQUE(book, user_type, user, format, annot_type, annot_id) ); DROP INDEX IF EXISTS annot_idx; diff --git a/src/calibre/gui2/actions/view.py b/src/calibre/gui2/actions/view.py index 23c880a871..779560b170 100644 --- a/src/calibre/gui2/actions/view.py +++ b/src/calibre/gui2/actions/view.py @@ -6,7 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, time +import os, time, json from functools import partial from PyQt5.Qt import Qt, QAction, pyqtSignal @@ -19,7 +19,7 @@ from calibre.gui2.dialogs.choose_format import ChooseFormatDialog from calibre.utils.config import prefs, tweaks from calibre.ptempfile import PersistentTemporaryFile from calibre.gui2.actions import InterfaceAction -from polyglot.builtins import unicode_type +from polyglot.builtins import unicode_type, as_bytes class HistoryAction(QAction): @@ -99,13 +99,21 @@ class ViewAction(InterfaceAction): id_ = self.gui.library_view.model().id(row) self.view_format_by_id(id_, format) + def calibre_book_data(self, book_id, fmt): + db = self.gui.current_db.new_api + annotations_map = db.annotations_map_for_book(book_id, fmt) + return { + 'book_id': book_id, 'uuid': db.field_for('uuid', book_id), 'fmt': fmt.upper(), + 'annotations_map': annotations_map, + } + def view_format_by_id(self, id_, format): db = self.gui.current_db fmt_path = db.format_abspath(id_, format, index_is_id=True) if fmt_path: title = db.title(id_, index_is_id=True) - self._view_file(fmt_path) + self._view_file(fmt_path, calibre_book_data=self.calibre_book_data(id_, format)) self.update_history([(id_, title)]) def book_downloaded_for_viewing(self, job): @@ -114,15 +122,20 @@ class ViewAction(InterfaceAction): return self._view_file(job.result) - def _launch_viewer(self, name=None, viewer='ebook-viewer', internal=True): + def _launch_viewer(self, name=None, viewer='ebook-viewer', internal=True, calibre_book_data=None): self.gui.setCursor(Qt.BusyCursor) try: if internal: args = [viewer] if isosx and 'ebook' in viewer: args.append('--raise-window') + if name is not None: args.append(name) + if calibre_book_data is not None: + with PersistentTemporaryFile('.json') as ptf: + ptf.write(as_bytes(json.dumps(calibre_book_data))) + args.append('--internal-book-data=' + ptf.name) self.gui.job_manager.launch_gui_app(viewer, kwargs=dict(args=args)) else: @@ -149,12 +162,12 @@ class ViewAction(InterfaceAction): finally: self.gui.unsetCursor() - def _view_file(self, name): + def _view_file(self, name, calibre_book_data=None): ext = os.path.splitext(name)[1].upper().replace('.', '').replace('ORIGINAL_', '') viewer = 'lrfviewer' if ext == 'LRF' else 'ebook-viewer' internal = self.force_internal_viewer or ext in config['internally_viewed_formats'] - self._launch_viewer(name, viewer, internal) + self._launch_viewer(name, viewer, internal, calibre_book_data=calibre_book_data) def view_specific_format(self, triggered): rows = list(self.gui.library_view.selectionModel().selectedRows()) diff --git a/src/calibre/gui2/viewer/annotations.py b/src/calibre/gui2/viewer/annotations.py index 1a6bfc306a..67986e1742 100644 --- a/src/calibre/gui2/viewer/annotations.py +++ b/src/calibre/gui2/viewer/annotations.py @@ -3,17 +3,26 @@ # License: GPL v3 Copyright: 2019, Kovid Goyal +import os from collections import defaultdict from io import BytesIO from operator import itemgetter +from threading import Thread +from calibre.gui2.viewer.convert_book import update_book +from calibre.gui2.viewer.integration import save_annotations_list_to_library +from calibre.gui2.viewer.web_view import viewer_config_dir from calibre.srv.render_book import ( EPUB_FILE_TYPE_MAGIC, parse_annotation, parse_annotations as _parse_annotations ) +from calibre.utils.date import EPOCH from calibre.utils.serialize import json_dumps from calibre.utils.zipfile import safe_replace from polyglot.binary import as_base64_bytes from polyglot.builtins import iteritems, itervalues +from polyglot.queue import Queue + +annotations_dir = os.path.join(viewer_config_dir, 'annots') def parse_annotations(raw): @@ -53,14 +62,17 @@ def serialize_annotation(annot): return annot -def serialize_annotations(annots_map): - ans = [] +def annotations_as_copied_list(annots_map): for atype, annots in iteritems(annots_map): for annot in annots: + ts = (annot['timestamp'] - EPOCH).total_seconds() annot = serialize_annotation(annot) annot['type'] = atype - ans.append(annot) - return json_dumps(ans) + yield annot, ts + + +def annot_list_as_bytes(annots): + return json_dumps(tuple(annot for annot, seconds in annots)) def split_lines(chunk, length=80): @@ -78,3 +90,56 @@ def save_annots_to_epub(path, serialized_annots): with zf: serialized_annots = EPUB_FILE_TYPE_MAGIC + b'\n'.join(split_lines(as_base64_bytes(serialized_annots))) safe_replace(zf, 'META-INF/calibre_bookmarks.txt', BytesIO(serialized_annots), add_missing=True) + + +def save_annotations(annotations_list, annotations_path_key, bld, pathtoebook, in_book_file): + annots = annot_list_as_bytes(annotations_list) + with open(os.path.join(annotations_dir, annotations_path_key), 'wb') as f: + f.write(annots) + if in_book_file and os.access(pathtoebook, os.W_OK): + before_stat = os.stat(pathtoebook) + save_annots_to_epub(pathtoebook, annots) + update_book(pathtoebook, before_stat, {'calibre-book-annotations.json': annots}) + if bld: + save_annotations_list_to_library(bld, annotations_list) + + +class AnnotationsSaveWorker(Thread): + + def __init__(self): + Thread.__init__(self, name='AnnotSaveWorker') + self.daemon = True + self.queue = Queue() + + def shutdown(self): + if self.is_alive(): + self.queue.put(None) + self.join() + + def run(self): + while True: + x = self.queue.get() + if x is None: + return + annotations_list = x['annotations_list'] + annotations_path_key = x['annotations_path_key'] + bld = x['book_library_details'] + pathtoebook = x['pathtoebook'] + in_book_file = x['in_book_file'] + try: + save_annotations(annotations_list, annotations_path_key, bld, pathtoebook, in_book_file) + except Exception: + import traceback + traceback.print_exc() + + def save_annotations(self, current_book_data, in_book_file=True): + alist = tuple(annotations_as_copied_list(current_book_data['annotations_map'])) + ebp = current_book_data['pathtoebook'] + can_save_in_book_file = ebp.lower.endswith('.epub') + self.queue.put({ + 'annotations_list': alist, + 'annotations_path_key': current_book_data['annotations_path_key'], + 'book_library_details': current_book_data['book_library_details'], + 'pathtoebook': current_book_data['pathtoebook'], + 'in_book_file': in_book_file and can_save_in_book_file + }) diff --git a/src/calibre/gui2/viewer/integration.py b/src/calibre/gui2/viewer/integration.py index 54eca1945b..95ad0a555b 100644 --- a/src/calibre/gui2/viewer/integration.py +++ b/src/calibre/gui2/viewer/integration.py @@ -5,10 +5,7 @@ import os import re -from polyglot.functools import lru_cache - -@lru_cache(maxsize=2) def get_book_library_details(absolute_path_to_ebook): absolute_path_to_ebook = os.path.abspath(os.path.expanduser(absolute_path_to_ebook)) base = os.path.dirname(absolute_path_to_ebook) @@ -22,3 +19,37 @@ def get_book_library_details(absolute_path_to_ebook): if not os.path.exists(dbpath): return return {'dbpath': dbpath, 'book_id': book_id, 'fmt': absolute_path_to_ebook.rpartition('.')[-1].upper()} + + +def load_annotations_map_from_library(book_library_details): + import apsw + from calibre.db.backend import annotations_for_book, Connection + ans = {} + dbpath = book_library_details['dbpath'] + try: + conn = apsw.Connection(dbpath, flags=apsw.SQLITE_OPEN_READONLY) + except Exception: + return ans + try: + conn.setbusytimeout(Connection.BUSY_TIMEOUT) + for annot in annotations_for_book(conn.cursor(), book_library_details['book_id'], book_library_details['fmt']): + ans.setdefault(annot['type'], []).append(annot) + finally: + conn.close() + return ans + + +def save_annotations_list_to_library(book_library_details, alist): + import apsw + from calibre.db.backend import save_annotations_for_book, Connection + dbpath = book_library_details['dbpath'] + try: + conn = apsw.Connection(dbpath, flags=apsw.SQLITE_OPEN_READWRITE) + except Exception: + return + try: + conn.setbusytimeout(Connection.BUSY_TIMEOUT) + with conn: + save_annotations_for_book(conn.cursor(), book_library_details['book_id'], book_library_details['fmt'], alist) + finally: + conn.close() diff --git a/src/calibre/gui2/viewer/main.py b/src/calibre/gui2/viewer/main.py index a37270a410..8617dac938 100644 --- a/src/calibre/gui2/viewer/main.py +++ b/src/calibre/gui2/viewer/main.py @@ -3,6 +3,7 @@ # License: GPL v3 Copyright: 2018, Kovid Goyal +import json import os import sys from threading import Thread @@ -195,6 +196,23 @@ def main(args=sys.argv): scheme.setFlags(QWebEngineUrlScheme.SecureScheme) QWebEngineUrlScheme.registerScheme(scheme) override = 'calibre-ebook-viewer' if islinux else None + processed_args = [] + internal_book_data = None + for arg in args: + if arg.startswith('--internal-book-data='): + internal_book_data = arg.split('=', 1)[1] + continue + processed_args.append(arg) + if internal_book_data: + try: + with lopen(internal_book_data, 'rb') as f: + internal_book_data = json.load(f) + finally: + try: + os.remove(internal_book_data) + except EnvironmentError: + pass + args = processed_args app = Application(args, override_program_name=override, windows_app_uid=VIEWER_APP_UID) parser = option_parser() @@ -219,7 +237,9 @@ def main(args=sys.argv): app.load_builtin_fonts() app.setWindowIcon(QIcon(I('viewer.png'))) migrate_previous_viewer_prefs() - main = EbookViewer(open_at=opts.open_at, continue_reading=opts.continue_reading, force_reload=opts.force_reload) + main = EbookViewer( + open_at=opts.open_at, continue_reading=opts.continue_reading, force_reload=opts.force_reload, + calibre_book_data=internal_book_data) main.set_exception_handler() if len(args) > 1: acc.events.append(os.path.abspath(args[-1])) diff --git a/src/calibre/gui2/viewer/ui.py b/src/calibre/gui2/viewer/ui.py index 22933f48ef..5f3f4b646d 100644 --- a/src/calibre/gui2/viewer/ui.py +++ b/src/calibre/gui2/viewer/ui.py @@ -24,22 +24,22 @@ from calibre.gui2.dialogs.drm_error import DRMErrorMessage from calibre.gui2.image_popup import ImagePopup from calibre.gui2.main_window import MainWindow from calibre.gui2.viewer.annotations import ( - merge_annotations, parse_annotations, save_annots_to_epub, serialize_annotation, - serialize_annotations + AnnotationsSaveWorker, annotations_dir, merge_annotations, parse_annotations, + serialize_annotation ) from calibre.gui2.viewer.bookmarks import BookmarkManager -from calibre.gui2.viewer.convert_book import ( - clean_running_workers, prepare_book, update_book -) +from calibre.gui2.viewer.convert_book import clean_running_workers, prepare_book from calibre.gui2.viewer.highlights import HighlightsPanel +from calibre.gui2.viewer.integration import ( + get_book_library_details, load_annotations_map_from_library +) from calibre.gui2.viewer.lookup import Lookup from calibre.gui2.viewer.overlay import LoadingOverlay from calibre.gui2.viewer.search import SearchPanel from calibre.gui2.viewer.toc import TOC, TOCSearch, TOCView from calibre.gui2.viewer.toolbars import ActionsToolBar from calibre.gui2.viewer.web_view import ( - WebView, get_path_for_name, get_session_pref, set_book_path, viewer_config_dir, - vprefs + WebView, get_path_for_name, get_session_pref, set_book_path, vprefs ) from calibre.utils.date import utcnow from calibre.utils.img import image_from_path @@ -47,9 +47,7 @@ from calibre.utils.ipc.simple_worker import WorkerError from calibre.utils.iso8601 import parse_iso8601 from calibre.utils.monotonic import monotonic from calibre.utils.serialize import json_loads -from polyglot.builtins import as_bytes, iteritems, itervalues, as_unicode - -annotations_dir = os.path.join(viewer_config_dir, 'annots') +from polyglot.builtins import as_bytes, as_unicode, iteritems, itervalues def is_float(x): @@ -88,8 +86,10 @@ class EbookViewer(MainWindow): book_prepared = pyqtSignal(object, object) MAIN_WINDOW_STATE_VERSION = 1 - def __init__(self, open_at=None, continue_reading=None, force_reload=False): + def __init__(self, open_at=None, continue_reading=None, force_reload=False, calibre_book_data=None): MainWindow.__init__(self, None) + self.annotations_saver = None + self.calibre_book_data_for_first_book = calibre_book_data self.shutting_down = self.close_forced = False self.force_reload = force_reload connect_lambda(self.book_preparation_started, self, lambda self: self.loading_overlay(_( @@ -479,6 +479,8 @@ class EbookViewer(MainWindow): self.book_preparation_started.emit() def load_finished(self, ok, data): + cbd = self.calibre_book_data_for_first_book + self.calibre_book_data_for_first_book = None if self.shutting_down: return open_at, self.pending_open_at = self.pending_open_at, None @@ -507,7 +509,7 @@ class EbookViewer(MainWindow): self.current_book_data = data self.current_book_data['annotations_map'] = defaultdict(list) self.current_book_data['annotations_path_key'] = path_key(data['pathtoebook']) + '.json' - self.load_book_data() + self.load_book_data(cbd) self.update_window_title() initial_cfi = self.initial_cfi_for_current_book() initial_position = {'type': 'cfi', 'data': initial_cfi} if initial_cfi else None @@ -534,8 +536,13 @@ class EbookViewer(MainWindow): highlights=list(map(serialize_annotation, highlights)) ) - def load_book_data(self): - self.load_book_annotations() + def load_book_data(self, calibre_book_data=None): + self.current_book_data['book_library_details'] = get_book_library_details(self.current_book_data['pathtoebook']) + if calibre_book_data is not None: + self.current_book_data['calibre_book_id'] = calibre_book_data['book_id'] + self.current_book_data['calibre_book_uuid'] = calibre_book_data['uuid'] + self.current_book_data['calibre_book_fmt'] = calibre_book_data['fmt'] + self.load_book_annotations(calibre_book_data) path = os.path.join(self.current_book_data['base'], 'calibre-book-manifest.json') with open(path, 'rb') as f: raw = f.read() @@ -547,7 +554,7 @@ class EbookViewer(MainWindow): self.current_book_data['metadata'] = set_book_path.parsed_metadata self.current_book_data['manifest'] = set_book_path.parsed_manifest - def load_book_annotations(self): + def load_book_annotations(self, calibre_book_data=None): amap = self.current_book_data['annotations_map'] path = os.path.join(self.current_book_data['base'], 'calibre-book-annotations.json') if os.path.exists(path): @@ -559,6 +566,16 @@ class EbookViewer(MainWindow): with open(path, 'rb') as f: raw = f.read() merge_annotations(parse_annotations(raw), amap) + if calibre_book_data is None: + bld = self.current_book_data['book_library_details'] + if bld is not None: + amap = load_annotations_map_from_library(bld) + if amap: + for annot_type, annots in iteritems(self.calibre_book_data_for_first_book['annotations_map']): + merge_annotations(annots, amap) + else: + for annot_type, annots in iteritems(calibre_book_data['annotations_map']): + merge_annotations(annots, amap) def update_window_title(self): try: @@ -590,17 +607,10 @@ class EbookViewer(MainWindow): def save_annotations(self, in_book_file=True): if not self.current_book_data: return - amap = self.current_book_data['annotations_map'] - annots = as_bytes(serialize_annotations(amap)) - with open(os.path.join(annotations_dir, self.current_book_data['annotations_path_key']), 'wb') as f: - f.write(annots) - if in_book_file and self.current_book_data.get('pathtoebook', '').lower().endswith( - '.epub') and get_session_pref('save_annotations_in_ebook', default=True): - path = self.current_book_data['pathtoebook'] - if os.access(path, os.W_OK): - before_stat = os.stat(path) - save_annots_to_epub(path, annots) - update_book(path, before_stat, {'calibre-book-annotations.json': annots}) + if self.annotations_saver is None: + self.annotations_saver = AnnotationsSaveWorker() + self.annotations_saver.start() + self.annotations_saver.save_annotations(self.current_book_data, in_book_file and get_session_pref('save_annotations_in_ebook', default=True)) def highlights_changed(self, highlights): if not self.current_book_data: @@ -649,11 +659,16 @@ class EbookViewer(MainWindow): QTimer.singleShot(2000, self.force_close) self.web_view.prepare_for_close() return + if self.shutting_down: + return self.shutting_down = True self.search_widget.shutdown() try: - self.save_annotations() self.save_state() + self.save_annotations() + if self.annotations_saver is not None: + self.annotations_saver.shutdown() + self.annotations_saver = None except Exception: import traceback traceback.print_exc()