From e3d6c9f192f07251c355d3d3295f0221fa955bcd Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 31 Jan 2022 20:05:06 +0530 Subject: [PATCH] More work on FTS --- resources/fts_sqlite.sql | 22 ++++++++++++++-------- resources/fts_triggers.sql | 17 +++++++++++++++++ src/calibre/db/fts/connect.py | 10 ---------- src/calibre/db/fts/schema_upgrade.py | 5 +++++ 4 files changed, 36 insertions(+), 18 deletions(-) create mode 100644 resources/fts_triggers.sql diff --git a/resources/fts_sqlite.sql b/resources/fts_sqlite.sql index 796510a5ea..79b7b2daac 100644 --- a/resources/fts_sqlite.sql +++ b/resources/fts_sqlite.sql @@ -1,3 +1,9 @@ +CREATE TABLE fts_db.dirtied_formats ( id INTEGER PRIMARY KEY, + book INTEGER NOT NULL, + format TEXT NOT NULL COLLATE NOCASE, + UNIQUE(book, format) +); + CREATE TABLE fts_db.books_text ( id INTEGER PRIMARY KEY, book INTEGER NOT NULL, format TEXT NOT NULL COLLATE NOCASE, @@ -15,22 +21,22 @@ CREATE VIRTUAL TABLE fts_db.books_fts_stemmed USING fts5(searchable_text, conten CREATE TRIGGER fts_db.books_fts_insert_trg AFTER INSERT ON fts_db.books_text BEGIN - INSERT INTO fts_db.books_fts(rowid, searchable_text) VALUES (NEW.id, NEW.searchable_text); - INSERT INTO fts_db.books_fts_stemmed(rowid, searchable_text) VALUES (NEW.id, NEW.searchable_text); + INSERT INTO books_fts(rowid, searchable_text) VALUES (NEW.id, NEW.searchable_text); + INSERT INTO books_fts_stemmed(rowid, searchable_text) VALUES (NEW.id, NEW.searchable_text); END; CREATE TRIGGER fts_db.books_fts_delete_trg AFTER DELETE ON fts_db.books_text BEGIN - INSERT INTO fts_db.books_fts(fts_db.books_fts, rowid, searchable_text) VALUES('delete', OLD.id, OLD.searchable_text); - INSERT INTO fts_db.books_fts_stemmed(fts_db.books_fts_stemmed, rowid, searchable_text) VALUES('delete', OLD.id, OLD.searchable_text); + INSERT INTO books_fts(books_fts, rowid, searchable_text) VALUES('delete', OLD.id, OLD.searchable_text); + INSERT INTO books_fts_stemmed(books_fts_stemmed, rowid, searchable_text) VALUES('delete', OLD.id, OLD.searchable_text); END; CREATE TRIGGER fts_db.books_fts_update_trg AFTER UPDATE ON fts_db.books_text BEGIN - INSERT INTO fts_db.books_fts(fts_db.books_fts, rowid, searchable_text) VALUES('delete', OLD.id, OLD.searchable_text); - INSERT INTO fts_db.books_fts(rowid, searchable_text) VALUES (NEW.id, NEW.searchable_text); - INSERT INTO fts_db.books_fts_stemmed(fts_db.books_fts_stemmed, rowid, searchable_text) VALUES('delete', OLD.id, OLD.searchable_text); - INSERT INTO fts_db.books_fts_stemmed(rowid, searchable_text) VALUES (NEW.id, NEW.searchable_text); + INSERT INTO books_fts(books_fts, rowid, searchable_text) VALUES('delete', OLD.id, OLD.searchable_text); + INSERT INTO books_fts(rowid, searchable_text) VALUES (NEW.id, NEW.searchable_text); + INSERT INTO books_fts_stemmed(books_fts_stemmed, rowid, searchable_text) VALUES('delete', OLD.id, OLD.searchable_text); + INSERT INTO books_fts_stemmed(rowid, searchable_text) VALUES (NEW.id, NEW.searchable_text); END; PRAGMA fts_db.user_version=1; diff --git a/resources/fts_triggers.sql b/resources/fts_triggers.sql new file mode 100644 index 0000000000..c1d4a41844 --- /dev/null +++ b/resources/fts_triggers.sql @@ -0,0 +1,17 @@ +CREATE TEMP TRIGGER fts_db_book_deleted_trg AFTER DELETE ON main.books BEGIN + DELETE FROM books_text WHERE book=OLD.id; + DELETE FROM dirtied_formats WHERE book=OLD.id; +END; + +CREATE TEMP TRIGGER fts_db_format_deleted_trg AFTER DELETE ON main.data BEGIN + DELETE FROM books_text WHERE book=OLD.id AND format=OLD.format; + DELETE FROM dirtied_formats WHERE book=OLD.id AND format=OLD.format; +END; + +CREATE TEMP TRIGGER fts_db_format_added_trg AFTER INSERT ON main.data BEGIN + INSERT INTO dirtied_formats(book, format) VALUES (NEW.book, NEW.format); +END; + +CREATE TEMP TRIGGER fts_db_format_updated_trg AFTER UPDATE ON main.data BEGIN + INSERT INTO dirtied_formats(book, format) VALUES (NEW.book, NEW.format); +END; diff --git a/src/calibre/db/fts/connect.py b/src/calibre/db/fts/connect.py index 1138118222..cc9b48b538 100644 --- a/src/calibre/db/fts/connect.py +++ b/src/calibre/db/fts/connect.py @@ -20,14 +20,4 @@ class FTS: main_db_path = os.path.abspath(conn.db_filename('main')) self.dbpath = os.path.join(os.path.dirname(main_db_path), 'full-text-search.db') conn.execute(f'ATTACH DATABASE "{self.dbpath}" AS fts_db') - fts_sqlite = P('fts_sqlite.sql', data=True, allow_user_override=False).decode('utf-8') - cur = self.conn.cursor() - cur.execute('BEGIN EXCLUSIVE TRANSACTION') - try: - cur.execute(fts_sqlite) - except (Exception, BaseException): - cur.execute('ROLLBACK') - raise - else: - cur.execute('COMMIT') SchemaUpgrade(conn) diff --git a/src/calibre/db/fts/schema_upgrade.py b/src/calibre/db/fts/schema_upgrade.py index 90aaee7618..3ed9fd7837 100644 --- a/src/calibre/db/fts/schema_upgrade.py +++ b/src/calibre/db/fts/schema_upgrade.py @@ -9,6 +9,9 @@ class SchemaUpgrade: self.conn = conn conn.execute('BEGIN EXCLUSIVE TRANSACTION') try: + if self.user_version == 0: + fts_sqlite = P('fts_sqlite.sql', data=True, allow_user_override=False).decode('utf-8') + conn.execute(fts_sqlite) while True: uv = self.user_version meth = getattr(self, f'upgrade_version_{uv}', None) @@ -17,6 +20,8 @@ class SchemaUpgrade: print(f'Upgrading FTS database to version {uv+1}...') meth() self.user_version = uv + 1 + fts_triggers = P('fts_triggers.sql', data=True, allow_user_override=False).decode('utf-8') + conn.execute(fts_triggers) except (Exception, BaseException): conn.execute('ROLLBACK') raise