From 1cb2bd10de09b7bc90fffa91466ab6158e4de019 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 24 Jul 2013 18:45:47 +0530 Subject: [PATCH] Update instead of invalidating search caches Updating is cheap when only a few books are affected. --- src/calibre/db/cache.py | 9 ++--- src/calibre/db/search.py | 64 ++++++++++++++++++++++++++++----- src/calibre/db/tests/reading.py | 6 ++++ 3 files changed, 66 insertions(+), 13 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 39bc905c50..23e4498e72 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -150,8 +150,8 @@ class Cache(object): field.clear_caches(book_ids=book_ids) @write_api - def clear_search_caches(self): - self._search_api.clear_caches() + def clear_search_caches(self, book_ids=None): + self._search_api.update_or_clear(self, book_ids) @write_api def clear_caches(self, book_ids=None, template_cache=True): @@ -165,7 +165,7 @@ class Cache(object): self.format_metadata_cache.pop(book_id, None) else: self.format_metadata_cache.clear() - self._clear_search_caches() + self._clear_search_caches(book_ids) @write_api def reload_from_db(self, clear_caches=True): @@ -828,7 +828,7 @@ class Cache(object): f.writer.set_books({book_id:now for book_id in book_ids}, self.backend) if self.composites: self._clear_composite_caches(book_ids) - self._clear_search_caches() + self._clear_search_caches(book_ids) @write_api def mark_as_dirty(self, book_ids): @@ -1407,6 +1407,7 @@ class Cache(object): @write_api def refresh_ondevice(self): self.fields['ondevice'].clear_caches() + self.clear_search_caches() @read_api def tags_older_than(self, tag, delta=None, must_have_tag=None, must_have_authors=None): diff --git a/src/calibre/db/search.py b/src/calibre/db/search.py index 332842e6ae..3b441e6b2f 100644 --- a/src/calibre/db/search.py +++ b/src/calibre/db/search.py @@ -451,7 +451,7 @@ class SavedSearchQueries(object): # {{{ return sorted(self.queries.iterkeys(), key=sort_key) # }}} -class Parser(SearchQueryParser): +class Parser(SearchQueryParser): # {{{ def __init__(self, dbcache, all_book_ids, gst, date_search, num_search, bool_search, keypair_search, limit_search_columns, limit_search_columns_to, @@ -711,8 +711,9 @@ class Parser(SearchQueryParser): if query == 'false': return candidates - matches return matches +# }}} -class LRUCache(object): +class LRUCache(object): # {{{ 'A simple Least-Recently-Used cache' @@ -748,6 +749,13 @@ class LRUCache(object): self.item_map.clear() self.age_map.clear() + def pop(self, key, default=None): + self.item_map.pop(key, default) + try: + self.age_map.remove(key) + except ValueError: + pass + def __contains__(self, key): return key in self.item_map @@ -757,8 +765,14 @@ class LRUCache(object): def __getitem__(self, key): return self.get(key) + def __iter__(self): + return self.item_map.iteritems() +# }}} + class Search(object): + MAX_CACHE_UPDATE = 50 + def __init__(self, db, opt_name, all_search_locations=()): self.all_search_locations = all_search_locations self.date_search = DateSearch() @@ -778,9 +792,47 @@ class Search(object): self.parse_cache.clear() self.all_search_locations = newlocs + def update_or_clear(self, dbcache, book_ids=None): + if book_ids and (len(book_ids) * len(self.cache)) <= self.MAX_CACHE_UPDATE: + self.update_caches(dbcache, book_ids) + else: + self.clear_caches() + def clear_caches(self): self.cache.clear() + def update_caches(self, dbcache, book_ids): + sqp = self.create_parser(dbcache) + try: + return self._update_caches(sqp, book_ids) + finally: + sqp.dbcache = sqp.lookup_saved_search = None + + def _update_caches(self, sqp, book_ids): + book_ids = sqp.all_book_ids = set(book_ids) + remove = set() + for query, result in self.cache: + try: + matches = sqp.parse(query) + except ParseException: + remove.add(query) + else: + # remove books that no longer match + result.difference_update(book_ids - matches) + # add books that now match but did not before + result.update(matches) + for query in remove: + self.cache.pop(query) + + def create_parser(self, dbcache, virtual_fields=None): + return Parser( + dbcache, set(), dbcache._pref('grouped_search_terms'), + self.date_search, self.num_search, self.bool_search, + self.keypair_search, + prefs['limit_search_columns'], + prefs['limit_search_columns_to'], self.all_search_locations, + virtual_fields, self.saved_searches.lookup, self.parse_cache) + def __call__(self, dbcache, query, search_restriction, virtual_fields=None, book_ids=None): ''' Return the set of ids of all records that match the specified @@ -788,13 +840,7 @@ class Search(object): ''' # We construct a new parser instance per search as the parse is not # thread safe. - sqp = Parser( - dbcache, set(), dbcache._pref('grouped_search_terms'), - self.date_search, self.num_search, self.bool_search, - self.keypair_search, - prefs['limit_search_columns'], - prefs['limit_search_columns_to'], self.all_search_locations, - virtual_fields, self.saved_searches.lookup, self.parse_cache) + sqp = self.create_parser(dbcache, virtual_fields) try: return self._do_search(sqp, query, search_restriction, dbcache, book_ids=book_ids) finally: diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py index dbeda9f6b0..2006348862 100644 --- a/src/calibre/db/tests/reading.py +++ b/src/calibre/db/tests/reading.py @@ -448,6 +448,7 @@ class ReadingTest(BaseTest): test(False, {3}, 'Unknown') test(True, {3}, 'Unknown') test(True, {3}, 'Unknown') + cache._search_api.MAX_CACHE_UPDATE = 0 cache.set_field('title', {3:'xxx'}) test(False, {3}, 'Unknown') # cache cleared test(True, {3}, 'Unknown') @@ -458,6 +459,11 @@ class ReadingTest(BaseTest): test(False, {3}, '', 'unknown') test(True, {3}, '', 'unknown') test(True, {3}, 'Unknown', 'unknown') + cache._search_api.MAX_CACHE_UPDATE = 100 + test(False, {2, 3}, 'title:=xxx or title:"=Title One"') + cache.set_field('publisher', {3:'ppppp', 2:'other'}) + # Test cache update worked + test(True, {2, 3}, 'title:=xxx or title:"=Title One"') # }}} def test_proxy_metadata(self): # {{{