calibredb: Implement wait for indexing against remote libraries as well

This commit is contained in:
Kovid Goyal 2022-08-07 12:59:39 +05:30
parent 119df51028
commit c6efde3495
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
2 changed files with 61 additions and 66 deletions

View File

@ -2,54 +2,27 @@
# License: GPLv3 Copyright: 2017, Kovid Goyal <kovid at kovidgoyal.net> # License: GPLv3 Copyright: 2017, Kovid Goyal <kovid at kovidgoyal.net>
import sys import sys
from functools import lru_cache
from calibre.db.listeners import EventType
version = 0 # change this if you change signature of implementation() version = 0 # change this if you change signature of implementation()
@lru_cache
def indexing_progress():
from threading import Lock
from calibre.db.utils import IndexingProgress
ans = IndexingProgress()
ans.lock = Lock()
return ans
def update_indexing_progress(left, total):
ip = indexing_progress()
with ip.lock:
ip.update(left, total)
def reset_indexing_progress():
ip = indexing_progress()
with ip.lock:
ip.reset()
def indexing_progress_time_left():
ip = indexing_progress()
with ip.lock:
return ip.time_left
def implementation(db, notify_changes, action, adata=None): def implementation(db, notify_changes, action, adata=None):
if action == 'status': if action == 'status':
if db.is_fts_enabled(): if db.is_fts_enabled():
l, t = db.fts_indexing_progress() l, t, r = db.fts_indexing_progress()
return {'enabled': True, 'left': l, 'total': t} return {'enabled': True, 'left': l, 'total': t, 'rate': r}
return {'enabled': False, 'left': -1, 'total': -1} return {'enabled': False, 'left': -1, 'total': -1}
if action == 'enable': if action == 'enable':
if not db.is_fts_enabled(): if not db.is_fts_enabled():
db.enable_fts() db.enable_fts()
l, t = db.fts_indexing_progress() l, t, r = db.fts_indexing_progress()
return {'enabled': True, 'left': l, 'total': t} return {'enabled': True, 'left': l, 'total': t, 'rate': r}
if action == 'disable': if action == 'disable':
if db.is_fts_enabled(): if db.is_fts_enabled():
reset_indexing_progress()
db.enable_fts(enabled=False) db.enable_fts(enabled=False)
return return
@ -64,8 +37,20 @@ def implementation(db, notify_changes, action, adata=None):
db.reindex_fts_book(*item) db.reindex_fts_book(*item)
else: else:
db.reindex_fts() db.reindex_fts()
l, t = db.fts_indexing_progress() l, t, r = db.fts_indexing_progress()
return {'enabled': True, 'left': l, 'total': t} return {'enabled': True, 'left': l, 'total': t, 'rate': r}
if action == 'wait':
if not db.is_fts_enabled():
a = Exception(_('Full text indexing is not enabled on this library'))
a.suppress_traceback = True
raise a
if 'measure_state' in adata:
db.fts_start_measuring_rate(measure=adata['measure_state'])
if adata.get('speed'):
db.set_fts_speed(slow=adata['speed'] == 'slow')
l, t, r = db.fts_indexing_progress()
return {'left': l, 'total': t, 'rate': r}
def option_parser(get_parser, args): def option_parser(get_parser, args):
@ -97,39 +82,58 @@ Control the Full text search indexing process.
default='', default='',
choices=('fast', 'slow', ''), choices=('fast', 'slow', ''),
help=_('The speed of indexing. Use fast for fast indexing using all your computers resources' help=_('The speed of indexing. Use fast for fast indexing using all your computers resources'
' and slow for less resource intensive indexing. Note that the speed is reset to slow on every invocation.') ' and slow for less resource intensive indexing. Note that the speed is reset to slow after every invocation.')
) )
return parser return parser
def run_job(dbctx, which, **data):
try:
return dbctx.run('fts_index', which, data)
except Exception as e:
if getattr(e, 'suppress_traceback', False):
raise SystemExit(str(e))
raise
def show_progress(left, total, rate):
from calibre.db.utils import IndexingProgress
ip = IndexingProgress()
ip.update(left, total, rate)
print('\r\x1b[K' + _('{} of {} book files indexed, {}').format(total-left, total, ip.time_left), flush=True, end=' ...')
def remote_wait_for_completion(dbctx, indexing_speed):
import time
s = run_job(dbctx, 'wait', speed=indexing_speed, measure_state=True)
try:
while s['left'] > 0:
show_progress(s['left'], s['total'], s['rate'])
time.sleep(1)
s = run_job(dbctx, 'wait')
finally:
print()
run_job(dbctx, 'wait', speed='slow', measure_state=False)
def local_wait_for_completion(db, indexing_speed): def local_wait_for_completion(db, indexing_speed):
from calibre.db.listeners import EventType
from queue import Queue from queue import Queue
q = Queue() q = Queue()
def listen(event_type, library_id, event_data): def notifier(event_type, library_id, event_data):
if event_type is EventType.indexing_progress_changed: if event_type is EventType.indexing_progress_changed:
update_indexing_progress(*event_data)
q.put(event_data) q.put(event_data)
def show_progress(left, total): db.add_listener(notifier)
print('\r\x1b[K' + _('{} of {} book files indexed, {}').format(total-left, total, indexing_progress_time_left()), flush=True, end=' ...')
db.add_listener(listen)
if indexing_speed: if indexing_speed:
db.set_fts_speed(slow=indexing_speed == 'slow') db.set_fts_speed(slow=indexing_speed == 'slow')
l, t = db.fts_indexing_progress() db.fts_start_measuring_rate()
if l < 1: l, t, r = db.fts_indexing_progress()
return while l > 0:
show_progress(l, t) show_progress(l, t, r)
l, t, r = q.get()
while True: print()
l, t = q.get()
if l < 1:
print()
return
show_progress(l, t)
def main(opts, args, dbctx): def main(opts, args, dbctx):
@ -137,17 +141,6 @@ def main(opts, args, dbctx):
dbctx.option_parser.print_help() dbctx.option_parser.print_help()
raise SystemExit(_('Error: You must specify the indexing action')) raise SystemExit(_('Error: You must specify the indexing action'))
action = args[0] action = args[0]
adata = {}
def run_job(dbctx, which, **kw):
data = adata.copy()
data.update(kw)
try:
return dbctx.run('fts_index', which, data)
except Exception as e:
if getattr(e, 'suppress_traceback', False):
raise SystemExit(str(e))
raise
if action == 'status': if action == 'status':
s = run_job(dbctx, 'status') s = run_job(dbctx, 'status')
@ -213,7 +206,7 @@ def main(opts, args, dbctx):
print(_('Waiting for FTS indexing to complete, press Ctrl-C to abort...')) print(_('Waiting for FTS indexing to complete, press Ctrl-C to abort...'))
try: try:
if dbctx.is_remote: if dbctx.is_remote:
raise NotImplementedError('TODO: Implement waiting for completion via polling') remote_wait_for_completion(dbctx, opts.indexing_speed)
else: else:
local_wait_for_completion(dbctx.db.new_api, opts.indexing_speed) local_wait_for_completion(dbctx.db.new_api, opts.indexing_speed)
except KeyboardInterrupt: except KeyboardInterrupt:

View File

@ -488,6 +488,8 @@ class IndexingProgress:
return _('calculating time left') return _('calculating time left')
try: try:
seconds_left = self.left / self.indexing_rate seconds_left = self.left / self.indexing_rate
if seconds_left < 2:
return _('almost done')
return _('~{} left').format(human_readable_interval(seconds_left)) return _('~{} left').format(human_readable_interval(seconds_left))
except Exception: except Exception:
return _('calculating time left') return _('calculating time left')