From 0dfdbdc7d1ebc4c0621c6b63cc27074439f845f0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 31 Jul 2023 14:38:39 +0530 Subject: [PATCH] Content server: Full text search: Allow searching a restricted subset of books. Fixes #2028216 [Search the full text of matched results - Content Server FTS](https://bugs.launchpad.net/calibre/+bug/2028216) --- src/calibre/srv/code.py | 1 + src/calibre/srv/fts.py | 7 +++- src/pyj/book_list/fts.pyj | 63 ++++++++++++++++++++++-------- src/pyj/book_list/library_data.pyj | 2 +- src/pyj/book_list/views.pyj | 7 ++++ 5 files changed, 60 insertions(+), 20 deletions(-) diff --git a/src/calibre/srv/code.py b/src/calibre/srv/code.py index 41c71dbbba..9fcf8a24e7 100644 --- a/src/calibre/srv/code.py +++ b/src/calibre/srv/code.py @@ -232,6 +232,7 @@ def get_library_init_data(ctx, rd, db, num, sorts, orders, vl): ans['virtual_libraries'] = db._pref('virtual_libraries', {}) ans['bools_are_tristate'] = db._pref('bools_are_tristate', True) ans['book_display_fields'] = get_field_list(db) + ans['fts_enabled'] = db.is_fts_enabled() ans['book_details_vertical_categories'] = db._pref('book_details_vertical_categories', ()) mdata = ans['metadata'] = {} try: diff --git a/src/calibre/srv/fts.py b/src/calibre/srv/fts.py index c3959d7cab..ef572e2dbb 100644 --- a/src/calibre/srv/fts.py +++ b/src/calibre/srv/fts.py @@ -17,7 +17,7 @@ def fts_search(ctx, rd): ''' Perform the specified full text query. - Optional: ?query=&library_id=&use_stemming=&query_id=arbitrary + Optional: ?query=&library_id=&use_stemming=&query_id=arbitrary&restriction=arbitrary ''' db = get_library_data(ctx, rd)[0] @@ -34,6 +34,9 @@ def fts_search(ctx, rd): qid = rd.query.get('query_id') if qid: ans['query_id'] = qid + book_ids = None + if rd.query.get('restriction'): + book_ids = db.search('', restriction=rd.query.get('restriction')) def add_metadata(result): result.pop('id', None) @@ -47,7 +50,7 @@ def fts_search(ctx, rd): from calibre.db import FTSQueryError try: ans['results'] = tuple(db.fts_search( - query, use_stemming=use_stemming, return_text=False, process_each_result=add_metadata, + query, use_stemming=use_stemming, return_text=False, process_each_result=add_metadata, restrict_to_book_ids=book_ids, )) except FTSQueryError as e: raise HTTPUnprocessableEntity(str(e)) diff --git a/src/pyj/book_list/fts.pyj b/src/pyj/book_list/fts.pyj index 37ad234f84..b45fef4e78 100644 --- a/src/pyj/book_list/fts.pyj +++ b/src/pyj/book_list/fts.pyj @@ -7,7 +7,7 @@ from elementmaker import E from ajax import ajax, ajax_send from book_list.cover_grid import THUMBNAIL_MAX_HEIGHT, THUMBNAIL_MAX_WIDTH from book_list.globals import get_current_query, get_session_data -from book_list.library_data import current_library_id, download_url +from book_list.library_data import current_library_id, download_url, library_data from book_list.router import back, home, open_book_url, push_state from book_list.top_bar import create_top_bar from book_list.ui import query_as_href, set_panel_handler @@ -64,6 +64,7 @@ def enable_indexing(): return error_dialog(_('Permission denied'), _( 'You do not have permission to enable indexing. Only logged in users with write permission are allowed.'), xhr.error_html) return error_dialog(_('Failed to enable indexing'), _('Enabling indexing failed. Click "Show details" for more information.'), xhr.error_html) + library_data.fts_enabled = True info_dialog(_('Indexing enabled'), _('Indexing of this library has been enabled. Depending on library size it can take a long time to index all books.')) ajax_send(f'fts/indexing', True, on_response) @@ -100,11 +101,14 @@ def execute_search_interactive(): query = component('query').querySelector('input').value if not query or query.length < 1: error_dialog(_('No search query specified'), _('A search word/phrase must be specified before attempting to search')) + clear_to_help() return q = get_current_query() q.fts_panel = 'search' q.fts_query = query q.fts_use_stemming = 'y' if component('related_words').checked else 'n' + if q.restricted is 'y': + q.restriction = component('restrict').querySelector('input').value current_fts_query = {} push_state(q, replace=True) @@ -132,10 +136,15 @@ def build_search_page(): ' example, in the English language: {0} matches {1} and {2} as well').format( 'correction', 'correcting', 'corrected') ) - container.appendChild(E.div(style="padding-top: 1ex; border-bottom: solid 1px currentColor; width: 100%", related_words)) + container.appendChild(E.div(style="padding-top: 1ex;", related_words)) + + restrict_bar = create_search_bar(execute_search_interactive, 'search-books-fts-restrict', tooltip=_('Restrict the results to only books matching this query'), placeholder=_('Restrict to books matching...')) + set_css(restrict_bar, flex_grow='10', margin_right='0.5em') + restrict_bar.dataset.component = 'restrict' + container.appendChild(E.div(style="padding-top: 1ex; display: flex; width: 100%; align-items: center", svgicon('search'), E.span('\xa0'), restrict_bar)) # Search help - container.appendChild(E.div(data_component='results')) + container.appendChild(E.div(style='border-top: 1px currentColor solid; margin-top: 1ex', data_component='results')) def clear_to_waiting_for_results(msg): @@ -150,17 +159,32 @@ def clear_to_waiting_for_results(msg): )) +def toggle_restrict(): + container = component('restrict') + if not container: + return + q = get_current_query() + q.restricted = 'n' if q.restricted is 'y' else 'y' + push_state(q, replace=True) + + def clear_to_help(): container = component('results') if not container: return clear(container) container.appendChild(E.div(class_='fts-help-display')) + restrict = get_current_query().restricted is 'y' container.appendChild(E.div( style='margin-top: 1ex', E.a(_('Re-index all books in this library'), class_='blue-link', href='javascript:void(0)', onclick=reindex_all), E.span('\xa0\xa0'), E.a(_('Disable full text search'), class_='blue-link', href='javascript:void(0)', onclick=disable_fts), + E.span('\xa0\xa0'), + E.a( + _('Search all books') if restrict else _('Search a subset of books'), + class_='blue-link', href='javascript:void(0)', onclick=toggle_restrict + ), )) container = container.firstChild fts_url = 'https://www.sqlite.org/fts5.html#full_text_query_syntax' @@ -190,13 +214,16 @@ def clear_to_help(): def apply_search_panel_state(): q = get_current_query() - ftsq = {'query': q.fts_query or '', 'use_stemming': q.fts_use_stemming or 'y'} + ftsq = {'query': q.fts_query or '', 'use_stemming': q.fts_use_stemming or 'y', 'restriction': q.restriction if q.restricted is 'y' else ''} component('query').querySelector('input').value = ftsq.query component('related_words').checked = ftsq.use_stemming is 'y' + r = component('restrict') + r.parentNode.style.display = 'flex' if ftsq.restriction is not '' else 'none' + r.querySelector('input').value = q.restriction or '' if not ftsq.query: clear_to_help() return - if current_fts_query.query is not ftsq.query or current_fts_query.use_stemming is not ftsq.use_stemming: + if current_fts_query.query is not ftsq.query or current_fts_query.use_stemming is not ftsq.use_stemming or current_fts_query.restriction is not ftsq.restriction: make_new_fts_query(ftsq) clear_to_waiting_for_results(_('Searching for {}, please wait…').format(ftsq.query)) return @@ -258,6 +285,7 @@ def disable_fts_backend(): return error_dialog(_('Permission denied'), _( 'You do not have permission to disable FTS. Only logged in users with write permission are allowed to disable FTS.'), xhr.error_html) return error_dialog(_('Disabling FTS failed'), _('Disabling FTS failed. Click "Show details" for more information.'), xhr.error_html) + library_data.fts_enabled = False info_dialog(_('Full text searching disabled'), _('Full text searching for this library has been disabled. In the future the entire library will have to be re-indexed when re-enabling full text searching.'), on_close=def(): window.setTimeout(home) ) @@ -379,18 +407,19 @@ def show_snippets(snippets): container = component('results') for book_id in Object.keys(snippets): c = container.querySelector(f'[data-book-id="{book_id}"]') - v'delete c.dataset.snippetsNeeded' - s = c.querySelector('.snippets_container') - clear(s) - for x in snippets[book_id]: - f = ' '.join(x.formats) - e = E.div(E.code( - style='border: solid 1px currentColor; border-radius: 6px; padding: 0 4px; font-size: smaller', - data_formats=f, f) - ) - e.appendChild(E.span(' ')) - render_text(e, x.text) - s.appendChild(e) + if c: + v'delete c.dataset.snippetsNeeded' + s = c.querySelector('.snippets_container') + clear(s) + for x in snippets[book_id]: + f = ' '.join(x.formats) + e = E.div(E.code( + style='border: solid 1px currentColor; border-radius: 6px; padding: 0 4px; font-size: smaller', + data_formats=f, f) + ) + e.appendChild(E.span(' ')) + render_text(e, x.text) + s.appendChild(e) def fetch_snippets(): diff --git a/src/pyj/book_list/library_data.pyj b/src/pyj/book_list/library_data.pyj index 6ca87ab005..0da3d22ce4 100644 --- a/src/pyj/book_list/library_data.pyj +++ b/src/pyj/book_list/library_data.pyj @@ -83,7 +83,7 @@ def update_library_data(data): if library_data.for_library is not current_library_id(): library_data.field_names = {} library_data.for_library = current_library_id() - for key in 'search_result sortable_fields field_metadata metadata virtual_libraries book_display_fields bools_are_tristate book_details_vertical_categories'.split(' '): + for key in 'search_result sortable_fields field_metadata metadata virtual_libraries book_display_fields bools_are_tristate book_details_vertical_categories fts_enabled'.split(' '): library_data[key] = data[key] sr = library_data.search_result if sr: diff --git a/src/pyj/book_list/views.pyj b/src/pyj/book_list/views.pyj index 54f868e296..1252483e49 100644 --- a/src/pyj/book_list/views.pyj +++ b/src/pyj/book_list/views.pyj @@ -236,6 +236,11 @@ def create_more_button(more): # }}} +def fts(): + q = {'restricted': 'y', 'restriction': library_data.search_result.query} + show_panel('fts', q) + + def show_top_message(): container = component('top_message') q = loaded_books_query() @@ -254,6 +259,8 @@ def show_top_message(): if library_data.search_result.total_num: c.appendChild(E.span(_('Showing books matching:'), ' ', E.i(library_data.search_result.query), ' ', _('(total matches: {}) ').format(library_data.search_result.total_num))) c.appendChild(E.span(' ', E.a(_('Clear search'), class_='blue-link', onclick=def(): search();))) + if library_data.fts_enabled: + c.appendChild(E.span(_(' or '), E.a(_('Search the text of these books'), class_='blue-link', onclick=def(): fts();))) else: c.appendChild(E.span(_('No books matching:'), ' ', E.i(library_data.search_result.query)))