diff --git a/src/calibre/db/__init__.py b/src/calibre/db/__init__.py index 47e44335ea..8f83a6bb81 100644 --- a/src/calibre/db/__init__.py +++ b/src/calibre/db/__init__.py @@ -116,6 +116,8 @@ Various things that require other things before they can be migrated: 4. Replace the metadatabackup thread with the new implementation when using the new backend. 5. In the new API refresh() does not re-read from disk. That might break a few things, for example content server reloading on db change as well as - dump/restore of db? + dump/restore of db and the refreshdb: action in gui2/ui.py. Probaly you'll have to create a dedicated API for + refreshing the db from disk and change the code to use it instead of the overloaded refresh (which is often used + to reread data from the db after writing to it). See reload_from_db() in cache.py 6. grep the sources for TODO ''' diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index c615f62bf7..d278e880b7 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -146,11 +146,18 @@ class Cache(object): self.formatter_template_cache = {} @write_api - def refresh(self): - self._initialize_template_cache() + def clear_caches(self, book_ids=None): + self._initialize_template_cache() # Clear the formatter template cache + for field in self.fields.itervalues(): + if hasattr(field, 'clear_caches'): + field.clear_caches(book_ids=book_ids) # Clear the composite cache and ondevice caches + self.format_metadata_cache.clear() + + @write_api + def reload_from_db(self, clear_caches=True): + if clear_caches: + self._clear_caches() for field in self.fields.itervalues(): - if hasattr(field, 'clear_cache'): - field.clear_cache() # Clear the composite cache if hasattr(field, 'table'): field.table.read(self.backend) # Reread data from metadata.db @@ -786,7 +793,7 @@ class Cache(object): if dirtied and self.composites: for name in self.composites: - self.fields[name].pop_cache(dirtied) + self.fields[name].clear_caches(book_ids=dirtied) if dirtied and update_path and do_path_update: self._update_path(dirtied, mark_as_dirtied=False) @@ -1264,6 +1271,15 @@ class Cache(object): ''' options must be a map of the form {book_id:conversion_options} ''' return self.backend.set_conversion_options(options, fmt) + @write_api + def refresh_format_cache(self): + self.fields['formats'].table.read(self.backend) + self.format_metadata_cache.clear() + + @write_api + def refresh_ondevice(self): + self.fields['ondevice'].clear_caches() + # }}} class SortKey(object): # {{{ diff --git a/src/calibre/db/fields.py b/src/calibre/db/fields.py index 20d0d75ff4..e18179e4b1 100644 --- a/src/calibre/db/fields.py +++ b/src/calibre/db/fields.py @@ -11,7 +11,7 @@ __docformat__ = 'restructuredtext en' from threading import Lock from collections import defaultdict, Counter -from calibre.db.tables import ONE_ONE, MANY_ONE, MANY_MANY +from calibre.db.tables import ONE_ONE, MANY_ONE, MANY_MANY, null from calibre.db.write import Writer from calibre.ebooks.metadata import title_sort from calibre.utils.config_base import tweaks @@ -163,14 +163,13 @@ class CompositeField(OneToOneField): self._render_cache[book_id] = ans return ans - def clear_cache(self): + def clear_caches(self, book_ids=None): with self._lock: - self._render_cache = {} - - def pop_cache(self, book_ids): - with self._lock: - for book_id in book_ids: - self._render_cache.pop(book_id, None) + if book_ids is None: + self._render_cache.clear() + else: + for book_id in book_ids: + self._render_cache.pop(book_id, None) def get_value_with_cache(self, book_id, get_metadata): with self._lock: @@ -218,11 +217,25 @@ class OnDeviceField(OneToOneField): self.name = name self.book_on_device_func = None self.is_multiple = False + self.cache = {} + self._lock = Lock() + + def clear_caches(self, book_ids=None): + with self._lock: + if book_ids is None: + self.cache.clear() + else: + for book_id in book_ids: + self.cache.pop(book_id, None) def book_on_device(self, book_id): - if callable(self.book_on_device_func): - return self.book_on_device_func(book_id) - return None + with self._lock: + ans = self.cache.get(book_id, null) + if ans is null and callable(self.book_on_device_func): + ans = self.book_on_device_func(book_id) + with self._lock: + self.cache[book_id] = ans + return None if ans is null else ans def set_book_on_device_func(self, func): self.book_on_device_func = func diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 0ec44e9670..f9e4b52896 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -69,7 +69,7 @@ class LibraryDatabase(object): self.get_property = self.data.get_property self.last_update_check = self.last_modified() - self.book_on_device_func = None + self.refresh_ids = self.data.refresh_ids self.is_case_sensitive = getattr(backend, 'is_case_sensitive', False) def close(self): @@ -108,7 +108,7 @@ class LibraryDatabase(object): def check_if_modified(self): if self.last_modified() > self.last_update_check: - self.refresh() + self.new_api.reload_from_db() self.last_update_check = utcnow() @property @@ -145,7 +145,6 @@ class LibraryDatabase(object): return [(k, v) for k, v in self.new_api.get_id_map(field).iteritems()] def refresh(self, field=None, ascending=True): - self.data.cache.refresh() self.data.refresh(field=field, ascending=ascending) def add_listener(self, listener): @@ -297,26 +296,18 @@ class LibraryDatabase(object): return [(aid, adata[aid]['name'], adata[aid]['sort'], adata[aid]['link']) for aid in authors] def book_on_device(self, book_id): - if callable(self.book_on_device_func): - return self.book_on_device_func(book_id) - return None + with self.new_api.read_lock: + return self.new_api.fields['ondevice'].book_on_device(book_id) def book_on_device_string(self, book_id): - loc = [] - count = 0 - on = self.book_on_device(book_id) - if on is not None: - m, a, b, count = on[:4] - if m is not None: - loc.append(_('Main')) - if a is not None: - loc.append(_('Card A')) - if b is not None: - loc.append(_('Card B')) - return ', '.join(loc) + ((_(' (%s books)')%count) if count > 1 else '') + return self.new_api.field_for('ondevice', book_id) def set_book_on_device_func(self, func): - self.book_on_device_func = func + self.new_api.fields['ondevice'].set_book_on_device_func(func) + + @property + def book_on_device_func(self): + return self.new_api.fields['ondevice'].book_on_device_func def books_in_series(self, series_id): with self.new_api.read_lock: @@ -474,6 +465,12 @@ class LibraryDatabase(object): book_id = index if index_is_id else self.id(index) return self.new_api.has_format(book_id, fmt) + def refresh_format_cache(self): + self.new_api.refresh_format_cache() + + def refresh_ondevice(self): + self.new_api.refresh_ondevice() + # Private interface {{{ def __iter__(self): for row in self.data.iterall(): diff --git a/src/calibre/db/view.py b/src/calibre/db/view.py index 43aed74f59..bc341698e2 100644 --- a/src/calibre/db/view.py +++ b/src/calibre/db/view.py @@ -10,6 +10,7 @@ __docformat__ = 'restructuredtext en' import weakref from functools import partial from itertools import izip, imap +from future_builtins import map from calibre.ebooks.metadata import title_sort from calibre.utils.config_base import tweaks @@ -163,6 +164,7 @@ class View(object): def id_to_index(self, book_id): return self._map.index(book_id) + row = index_to_id def _get(self, field, idx, index_is_id=True, default_value=None, fmt=lambda x:x): id_ = idx if index_is_id else self.index_to_id(idx) @@ -307,8 +309,17 @@ class View(object): def refresh(self, field=None, ascending=True): self._map = tuple(self.cache.all_book_ids()) self._map_filtered = tuple(self._map) + self.cache.clear_caches() if field is not None: self.sort(field, ascending) if self.search_restriction or self.base_restriction: self.search('', return_matches=False) + def refresh_ids(self, db, ids): + self.cache.clear_caches(book_ids=ids) + try: + return list(map(self.id_to_index, ids)) + except ValueError: + pass + return None +