Code to read and write annotations to the calibre db

This commit is contained in:
Kovid Goyal 2020-06-11 08:47:33 +05:30
parent 628ce9aa84
commit 6bda5e6aad
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
9 changed files with 233 additions and 43 deletions

View File

@ -152,7 +152,7 @@ CREATE TABLE annotations ( id INTEGER PRIMARY KEY,
annot_type TEXT NOT NULL,
annot_data TEXT NOT NULL,
searchable_text TEXT NOT NULL,
UNIQUE(book, user_type, user, format, annot_id)
UNIQUE(book, user_type, user, format, annot_type, annot_id)
);
CREATE VIEW meta AS

View File

@ -283,6 +283,41 @@ def AumSortedConcatenate():
# }}}
# Annotations {{{
def annotations_for_book(cursor, book_id, fmt, user_type='local', user='viewer'):
for (data,) in cursor.execute(
'SELECT annot_data FROM annotations WHERE book=? AND format=? AND user_type=? AND user=?',
(book_id, fmt.upper(), user_type, user)
):
try:
yield json.loads(data)
except Exception:
pass
def save_annotations_for_book(cursor, book_id, fmt, annots_list, user_type='local', user='viewer'):
data = []
fmt = fmt.upper()
for annot, timestamp_in_secs in annots_list:
atype = annot['type']
if atype == 'bookmark':
aid = text = annot['title']
elif atype == 'highlight':
aid = annot['uuid']
text = annot.get('highlighed_text') or ''
notes = annot.get('notes') or ''
if notes:
text += '0x1f\n\n' + notes
else:
continue
data.append((book_id, fmt, user_type, user, timestamp_in_secs, aid, atype, json.dumps(annot), text))
cursor.executemany(
'INSERT OR REPLACE INTO annotations (book, format, user_type, user, timestamp, annot_id, annot_type, annot_data, searchable_text)'
' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', data)
cursor.execute('INSERT OR IGNORE INTO annotations_dirtied (book) VALUES (?)', (book_id,))
# }}}
class Connection(apsw.Connection): # {{{
BUSY_TIMEOUT = 10000 # milliseconds
@ -1724,6 +1759,10 @@ class DB(object):
def get_ids_for_custom_book_data(self, name):
return frozenset(r[0] for r in self.execute('SELECT book FROM books_plugin_data WHERE name=?', (name,)))
def annotations_for_book(self, book_id, fmt, user_type, user):
for x in annotations_for_book(self.conn, book_id, fmt, user_type, user):
yield x
def conversion_options(self, book_id, fmt):
for (data,) in self.conn.get('SELECT data FROM conversion_options WHERE book=? AND format=?', (book_id, fmt.upper())):
if data:

View File

@ -2279,6 +2279,13 @@ class Cache(object):
if progress is not None:
progress(_('Completed'), total, total)
@read_api
def annotations_map_for_book(self, book_id, fmt, user_type='local', user='viewer'):
ans = {}
for annot in self.backend.annotations_for_book(book_id, fmt, user_type, user):
ans.setdefault(annot['type'], []).append(annot)
return ans
def import_library(library_key, importer, library_path, progress=None, abort=None):
from calibre.db.backend import DB

View File

@ -715,7 +715,7 @@ CREATE TABLE annotations ( id INTEGER PRIMARY KEY,
annot_type TEXT NOT NULL,
annot_data TEXT NOT NULL,
searchable_text TEXT NOT NULL,
UNIQUE(book, user_type, user, format, annot_id)
UNIQUE(book, user_type, user, format, annot_type, annot_id)
);
DROP INDEX IF EXISTS annot_idx;

View File

@ -6,7 +6,7 @@ __license__ = 'GPL v3'
__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import os, time
import os, time, json
from functools import partial
from PyQt5.Qt import Qt, QAction, pyqtSignal
@ -19,7 +19,7 @@ from calibre.gui2.dialogs.choose_format import ChooseFormatDialog
from calibre.utils.config import prefs, tweaks
from calibre.ptempfile import PersistentTemporaryFile
from calibre.gui2.actions import InterfaceAction
from polyglot.builtins import unicode_type
from polyglot.builtins import unicode_type, as_bytes
class HistoryAction(QAction):
@ -99,13 +99,21 @@ class ViewAction(InterfaceAction):
id_ = self.gui.library_view.model().id(row)
self.view_format_by_id(id_, format)
def calibre_book_data(self, book_id, fmt):
db = self.gui.current_db.new_api
annotations_map = db.annotations_map_for_book(book_id, fmt)
return {
'book_id': book_id, 'uuid': db.field_for('uuid', book_id), 'fmt': fmt.upper(),
'annotations_map': annotations_map,
}
def view_format_by_id(self, id_, format):
db = self.gui.current_db
fmt_path = db.format_abspath(id_, format,
index_is_id=True)
if fmt_path:
title = db.title(id_, index_is_id=True)
self._view_file(fmt_path)
self._view_file(fmt_path, calibre_book_data=self.calibre_book_data(id_, format))
self.update_history([(id_, title)])
def book_downloaded_for_viewing(self, job):
@ -114,15 +122,20 @@ class ViewAction(InterfaceAction):
return
self._view_file(job.result)
def _launch_viewer(self, name=None, viewer='ebook-viewer', internal=True):
def _launch_viewer(self, name=None, viewer='ebook-viewer', internal=True, calibre_book_data=None):
self.gui.setCursor(Qt.BusyCursor)
try:
if internal:
args = [viewer]
if isosx and 'ebook' in viewer:
args.append('--raise-window')
if name is not None:
args.append(name)
if calibre_book_data is not None:
with PersistentTemporaryFile('.json') as ptf:
ptf.write(as_bytes(json.dumps(calibre_book_data)))
args.append('--internal-book-data=' + ptf.name)
self.gui.job_manager.launch_gui_app(viewer,
kwargs=dict(args=args))
else:
@ -149,12 +162,12 @@ class ViewAction(InterfaceAction):
finally:
self.gui.unsetCursor()
def _view_file(self, name):
def _view_file(self, name, calibre_book_data=None):
ext = os.path.splitext(name)[1].upper().replace('.',
'').replace('ORIGINAL_', '')
viewer = 'lrfviewer' if ext == 'LRF' else 'ebook-viewer'
internal = self.force_internal_viewer or ext in config['internally_viewed_formats']
self._launch_viewer(name, viewer, internal)
self._launch_viewer(name, viewer, internal, calibre_book_data=calibre_book_data)
def view_specific_format(self, triggered):
rows = list(self.gui.library_view.selectionModel().selectedRows())

View File

@ -3,17 +3,26 @@
# License: GPL v3 Copyright: 2019, Kovid Goyal <kovid at kovidgoyal.net>
import os
from collections import defaultdict
from io import BytesIO
from operator import itemgetter
from threading import Thread
from calibre.gui2.viewer.convert_book import update_book
from calibre.gui2.viewer.integration import save_annotations_list_to_library
from calibre.gui2.viewer.web_view import viewer_config_dir
from calibre.srv.render_book import (
EPUB_FILE_TYPE_MAGIC, parse_annotation, parse_annotations as _parse_annotations
)
from calibre.utils.date import EPOCH
from calibre.utils.serialize import json_dumps
from calibre.utils.zipfile import safe_replace
from polyglot.binary import as_base64_bytes
from polyglot.builtins import iteritems, itervalues
from polyglot.queue import Queue
annotations_dir = os.path.join(viewer_config_dir, 'annots')
def parse_annotations(raw):
@ -53,14 +62,17 @@ def serialize_annotation(annot):
return annot
def serialize_annotations(annots_map):
ans = []
def annotations_as_copied_list(annots_map):
for atype, annots in iteritems(annots_map):
for annot in annots:
ts = (annot['timestamp'] - EPOCH).total_seconds()
annot = serialize_annotation(annot)
annot['type'] = atype
ans.append(annot)
return json_dumps(ans)
yield annot, ts
def annot_list_as_bytes(annots):
return json_dumps(tuple(annot for annot, seconds in annots))
def split_lines(chunk, length=80):
@ -78,3 +90,56 @@ def save_annots_to_epub(path, serialized_annots):
with zf:
serialized_annots = EPUB_FILE_TYPE_MAGIC + b'\n'.join(split_lines(as_base64_bytes(serialized_annots)))
safe_replace(zf, 'META-INF/calibre_bookmarks.txt', BytesIO(serialized_annots), add_missing=True)
def save_annotations(annotations_list, annotations_path_key, bld, pathtoebook, in_book_file):
annots = annot_list_as_bytes(annotations_list)
with open(os.path.join(annotations_dir, annotations_path_key), 'wb') as f:
f.write(annots)
if in_book_file and os.access(pathtoebook, os.W_OK):
before_stat = os.stat(pathtoebook)
save_annots_to_epub(pathtoebook, annots)
update_book(pathtoebook, before_stat, {'calibre-book-annotations.json': annots})
if bld:
save_annotations_list_to_library(bld, annotations_list)
class AnnotationsSaveWorker(Thread):
def __init__(self):
Thread.__init__(self, name='AnnotSaveWorker')
self.daemon = True
self.queue = Queue()
def shutdown(self):
if self.is_alive():
self.queue.put(None)
self.join()
def run(self):
while True:
x = self.queue.get()
if x is None:
return
annotations_list = x['annotations_list']
annotations_path_key = x['annotations_path_key']
bld = x['book_library_details']
pathtoebook = x['pathtoebook']
in_book_file = x['in_book_file']
try:
save_annotations(annotations_list, annotations_path_key, bld, pathtoebook, in_book_file)
except Exception:
import traceback
traceback.print_exc()
def save_annotations(self, current_book_data, in_book_file=True):
alist = tuple(annotations_as_copied_list(current_book_data['annotations_map']))
ebp = current_book_data['pathtoebook']
can_save_in_book_file = ebp.lower.endswith('.epub')
self.queue.put({
'annotations_list': alist,
'annotations_path_key': current_book_data['annotations_path_key'],
'book_library_details': current_book_data['book_library_details'],
'pathtoebook': current_book_data['pathtoebook'],
'in_book_file': in_book_file and can_save_in_book_file
})

View File

@ -5,10 +5,7 @@
import os
import re
from polyglot.functools import lru_cache
@lru_cache(maxsize=2)
def get_book_library_details(absolute_path_to_ebook):
absolute_path_to_ebook = os.path.abspath(os.path.expanduser(absolute_path_to_ebook))
base = os.path.dirname(absolute_path_to_ebook)
@ -22,3 +19,37 @@ def get_book_library_details(absolute_path_to_ebook):
if not os.path.exists(dbpath):
return
return {'dbpath': dbpath, 'book_id': book_id, 'fmt': absolute_path_to_ebook.rpartition('.')[-1].upper()}
def load_annotations_map_from_library(book_library_details):
import apsw
from calibre.db.backend import annotations_for_book, Connection
ans = {}
dbpath = book_library_details['dbpath']
try:
conn = apsw.Connection(dbpath, flags=apsw.SQLITE_OPEN_READONLY)
except Exception:
return ans
try:
conn.setbusytimeout(Connection.BUSY_TIMEOUT)
for annot in annotations_for_book(conn.cursor(), book_library_details['book_id'], book_library_details['fmt']):
ans.setdefault(annot['type'], []).append(annot)
finally:
conn.close()
return ans
def save_annotations_list_to_library(book_library_details, alist):
import apsw
from calibre.db.backend import save_annotations_for_book, Connection
dbpath = book_library_details['dbpath']
try:
conn = apsw.Connection(dbpath, flags=apsw.SQLITE_OPEN_READWRITE)
except Exception:
return
try:
conn.setbusytimeout(Connection.BUSY_TIMEOUT)
with conn:
save_annotations_for_book(conn.cursor(), book_library_details['book_id'], book_library_details['fmt'], alist)
finally:
conn.close()

View File

@ -3,6 +3,7 @@
# License: GPL v3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
import json
import os
import sys
from threading import Thread
@ -195,6 +196,23 @@ def main(args=sys.argv):
scheme.setFlags(QWebEngineUrlScheme.SecureScheme)
QWebEngineUrlScheme.registerScheme(scheme)
override = 'calibre-ebook-viewer' if islinux else None
processed_args = []
internal_book_data = None
for arg in args:
if arg.startswith('--internal-book-data='):
internal_book_data = arg.split('=', 1)[1]
continue
processed_args.append(arg)
if internal_book_data:
try:
with lopen(internal_book_data, 'rb') as f:
internal_book_data = json.load(f)
finally:
try:
os.remove(internal_book_data)
except EnvironmentError:
pass
args = processed_args
app = Application(args, override_program_name=override, windows_app_uid=VIEWER_APP_UID)
parser = option_parser()
@ -219,7 +237,9 @@ def main(args=sys.argv):
app.load_builtin_fonts()
app.setWindowIcon(QIcon(I('viewer.png')))
migrate_previous_viewer_prefs()
main = EbookViewer(open_at=opts.open_at, continue_reading=opts.continue_reading, force_reload=opts.force_reload)
main = EbookViewer(
open_at=opts.open_at, continue_reading=opts.continue_reading, force_reload=opts.force_reload,
calibre_book_data=internal_book_data)
main.set_exception_handler()
if len(args) > 1:
acc.events.append(os.path.abspath(args[-1]))

View File

@ -24,22 +24,22 @@ from calibre.gui2.dialogs.drm_error import DRMErrorMessage
from calibre.gui2.image_popup import ImagePopup
from calibre.gui2.main_window import MainWindow
from calibre.gui2.viewer.annotations import (
merge_annotations, parse_annotations, save_annots_to_epub, serialize_annotation,
serialize_annotations
AnnotationsSaveWorker, annotations_dir, merge_annotations, parse_annotations,
serialize_annotation
)
from calibre.gui2.viewer.bookmarks import BookmarkManager
from calibre.gui2.viewer.convert_book import (
clean_running_workers, prepare_book, update_book
)
from calibre.gui2.viewer.convert_book import clean_running_workers, prepare_book
from calibre.gui2.viewer.highlights import HighlightsPanel
from calibre.gui2.viewer.integration import (
get_book_library_details, load_annotations_map_from_library
)
from calibre.gui2.viewer.lookup import Lookup
from calibre.gui2.viewer.overlay import LoadingOverlay
from calibre.gui2.viewer.search import SearchPanel
from calibre.gui2.viewer.toc import TOC, TOCSearch, TOCView
from calibre.gui2.viewer.toolbars import ActionsToolBar
from calibre.gui2.viewer.web_view import (
WebView, get_path_for_name, get_session_pref, set_book_path, viewer_config_dir,
vprefs
WebView, get_path_for_name, get_session_pref, set_book_path, vprefs
)
from calibre.utils.date import utcnow
from calibre.utils.img import image_from_path
@ -47,9 +47,7 @@ from calibre.utils.ipc.simple_worker import WorkerError
from calibre.utils.iso8601 import parse_iso8601
from calibre.utils.monotonic import monotonic
from calibre.utils.serialize import json_loads
from polyglot.builtins import as_bytes, iteritems, itervalues, as_unicode
annotations_dir = os.path.join(viewer_config_dir, 'annots')
from polyglot.builtins import as_bytes, as_unicode, iteritems, itervalues
def is_float(x):
@ -88,8 +86,10 @@ class EbookViewer(MainWindow):
book_prepared = pyqtSignal(object, object)
MAIN_WINDOW_STATE_VERSION = 1
def __init__(self, open_at=None, continue_reading=None, force_reload=False):
def __init__(self, open_at=None, continue_reading=None, force_reload=False, calibre_book_data=None):
MainWindow.__init__(self, None)
self.annotations_saver = None
self.calibre_book_data_for_first_book = calibre_book_data
self.shutting_down = self.close_forced = False
self.force_reload = force_reload
connect_lambda(self.book_preparation_started, self, lambda self: self.loading_overlay(_(
@ -479,6 +479,8 @@ class EbookViewer(MainWindow):
self.book_preparation_started.emit()
def load_finished(self, ok, data):
cbd = self.calibre_book_data_for_first_book
self.calibre_book_data_for_first_book = None
if self.shutting_down:
return
open_at, self.pending_open_at = self.pending_open_at, None
@ -507,7 +509,7 @@ class EbookViewer(MainWindow):
self.current_book_data = data
self.current_book_data['annotations_map'] = defaultdict(list)
self.current_book_data['annotations_path_key'] = path_key(data['pathtoebook']) + '.json'
self.load_book_data()
self.load_book_data(cbd)
self.update_window_title()
initial_cfi = self.initial_cfi_for_current_book()
initial_position = {'type': 'cfi', 'data': initial_cfi} if initial_cfi else None
@ -534,8 +536,13 @@ class EbookViewer(MainWindow):
highlights=list(map(serialize_annotation, highlights))
)
def load_book_data(self):
self.load_book_annotations()
def load_book_data(self, calibre_book_data=None):
self.current_book_data['book_library_details'] = get_book_library_details(self.current_book_data['pathtoebook'])
if calibre_book_data is not None:
self.current_book_data['calibre_book_id'] = calibre_book_data['book_id']
self.current_book_data['calibre_book_uuid'] = calibre_book_data['uuid']
self.current_book_data['calibre_book_fmt'] = calibre_book_data['fmt']
self.load_book_annotations(calibre_book_data)
path = os.path.join(self.current_book_data['base'], 'calibre-book-manifest.json')
with open(path, 'rb') as f:
raw = f.read()
@ -547,7 +554,7 @@ class EbookViewer(MainWindow):
self.current_book_data['metadata'] = set_book_path.parsed_metadata
self.current_book_data['manifest'] = set_book_path.parsed_manifest
def load_book_annotations(self):
def load_book_annotations(self, calibre_book_data=None):
amap = self.current_book_data['annotations_map']
path = os.path.join(self.current_book_data['base'], 'calibre-book-annotations.json')
if os.path.exists(path):
@ -559,6 +566,16 @@ class EbookViewer(MainWindow):
with open(path, 'rb') as f:
raw = f.read()
merge_annotations(parse_annotations(raw), amap)
if calibre_book_data is None:
bld = self.current_book_data['book_library_details']
if bld is not None:
amap = load_annotations_map_from_library(bld)
if amap:
for annot_type, annots in iteritems(self.calibre_book_data_for_first_book['annotations_map']):
merge_annotations(annots, amap)
else:
for annot_type, annots in iteritems(calibre_book_data['annotations_map']):
merge_annotations(annots, amap)
def update_window_title(self):
try:
@ -590,17 +607,10 @@ class EbookViewer(MainWindow):
def save_annotations(self, in_book_file=True):
if not self.current_book_data:
return
amap = self.current_book_data['annotations_map']
annots = as_bytes(serialize_annotations(amap))
with open(os.path.join(annotations_dir, self.current_book_data['annotations_path_key']), 'wb') as f:
f.write(annots)
if in_book_file and self.current_book_data.get('pathtoebook', '').lower().endswith(
'.epub') and get_session_pref('save_annotations_in_ebook', default=True):
path = self.current_book_data['pathtoebook']
if os.access(path, os.W_OK):
before_stat = os.stat(path)
save_annots_to_epub(path, annots)
update_book(path, before_stat, {'calibre-book-annotations.json': annots})
if self.annotations_saver is None:
self.annotations_saver = AnnotationsSaveWorker()
self.annotations_saver.start()
self.annotations_saver.save_annotations(self.current_book_data, in_book_file and get_session_pref('save_annotations_in_ebook', default=True))
def highlights_changed(self, highlights):
if not self.current_book_data:
@ -649,11 +659,16 @@ class EbookViewer(MainWindow):
QTimer.singleShot(2000, self.force_close)
self.web_view.prepare_for_close()
return
if self.shutting_down:
return
self.shutting_down = True
self.search_widget.shutdown()
try:
self.save_annotations()
self.save_state()
self.save_annotations()
if self.annotations_saver is not None:
self.annotations_saver.shutdown()
self.annotations_saver = None
except Exception:
import traceback
traceback.print_exc()