Implement refresh/reload semantics

src/calibre/db/__init__.py

@@ -116,6 +116,8 @@ Various things that require other things before they can be migrated:
     4. Replace the metadatabackup thread with the new implementation when using the new backend.
     5. In the new API refresh() does not re-read from disk. That might break a
     few things, for example content server reloading on db change as well as
-    dump/restore of db?
+    dump/restore of db and the refreshdb: action in gui2/ui.py. Probaly you'll have to create a dedicated API for
+    refreshing the db from disk and change the code to use it instead of the overloaded refresh (which is often used
+    to reread data from the db after writing to it). See reload_from_db() in cache.py
     6. grep the sources for TODO
 '''

src/calibre/db/cache.py

@@ -146,11 +146,18 @@ class Cache(object):
         self.formatter_template_cache = {}
 
     @write_api
-    def refresh(self):
+    def clear_caches(self, book_ids=None):
-        self._initialize_template_cache()
+        self._initialize_template_cache()  # Clear the formatter template cache
+        for field in self.fields.itervalues():
+            if hasattr(field, 'clear_caches'):
+                field.clear_caches(book_ids=book_ids)  # Clear the composite cache and ondevice caches
+        self.format_metadata_cache.clear()
+
+    @write_api
+    def reload_from_db(self, clear_caches=True):
+        if clear_caches:
+            self._clear_caches()
         for field in self.fields.itervalues():
-            if hasattr(field, 'clear_cache'):
-                field.clear_cache()  # Clear the composite cache
             if hasattr(field, 'table'):
                 field.table.read(self.backend)  # Reread data from metadata.db
 
@@ -786,7 +793,7 @@ class Cache(object):
 
         if dirtied and self.composites:
             for name in self.composites:
-                self.fields[name].pop_cache(dirtied)
+                self.fields[name].clear_caches(book_ids=dirtied)
 
         if dirtied and update_path and do_path_update:
             self._update_path(dirtied, mark_as_dirtied=False)
@@ -1264,6 +1271,15 @@ class Cache(object):
         ''' options must be a map of the form {book_id:conversion_options} '''
         return self.backend.set_conversion_options(options, fmt)
 
+    @write_api
+    def refresh_format_cache(self):
+        self.fields['formats'].table.read(self.backend)
+        self.format_metadata_cache.clear()
+
+    @write_api
+    def refresh_ondevice(self):
+        self.fields['ondevice'].clear_caches()
+
     # }}}
 
 class SortKey(object):  # {{{

src/calibre/db/fields.py

@@ -11,7 +11,7 @@ __docformat__ = 'restructuredtext en'
 from threading import Lock
 from collections import defaultdict, Counter
 
-from calibre.db.tables import ONE_ONE, MANY_ONE, MANY_MANY
+from calibre.db.tables import ONE_ONE, MANY_ONE, MANY_MANY, null
 from calibre.db.write import Writer
 from calibre.ebooks.metadata import title_sort
 from calibre.utils.config_base import tweaks
@@ -163,12 +163,11 @@ class CompositeField(OneToOneField):
                 self._render_cache[book_id] = ans
         return ans
 
-    def clear_cache(self):
+    def clear_caches(self, book_ids=None):
-        with self._lock:
-            self._render_cache = {}
-
-    def pop_cache(self, book_ids):
         with self._lock:
+            if book_ids is None:
+                self._render_cache.clear()
+            else:
                 for book_id in book_ids:
                     self._render_cache.pop(book_id, None)
 
@@ -218,11 +217,25 @@ class OnDeviceField(OneToOneField):
         self.name = name
         self.book_on_device_func = None
         self.is_multiple = False
+        self.cache = {}
+        self._lock = Lock()
+
+    def clear_caches(self, book_ids=None):
+        with self._lock:
+            if book_ids is None:
+                self.cache.clear()
+            else:
+                for book_id in book_ids:
+                    self.cache.pop(book_id, None)
 
     def book_on_device(self, book_id):
-        if callable(self.book_on_device_func):
+        with self._lock:
-            return self.book_on_device_func(book_id)
+            ans = self.cache.get(book_id, null)
-        return None
+        if ans is null and callable(self.book_on_device_func):
+            ans = self.book_on_device_func(book_id)
+            with self._lock:
+                self.cache[book_id] = ans
+        return None if ans is null else ans
 
     def set_book_on_device_func(self, func):
         self.book_on_device_func = func

src/calibre/db/legacy.py

@@ -69,7 +69,7 @@ class LibraryDatabase(object):
         self.get_property = self.data.get_property
 
         self.last_update_check = self.last_modified()
-        self.book_on_device_func = None
+        self.refresh_ids = self.data.refresh_ids
         self.is_case_sensitive = getattr(backend, 'is_case_sensitive', False)
 
     def close(self):
@@ -108,7 +108,7 @@ class LibraryDatabase(object):
 
     def check_if_modified(self):
         if self.last_modified() > self.last_update_check:
-            self.refresh()
+            self.new_api.reload_from_db()
         self.last_update_check = utcnow()
 
     @property
@@ -145,7 +145,6 @@ class LibraryDatabase(object):
         return [(k, v) for k, v in self.new_api.get_id_map(field).iteritems()]
 
     def refresh(self, field=None, ascending=True):
-        self.data.cache.refresh()
         self.data.refresh(field=field, ascending=ascending)
 
     def add_listener(self, listener):
@@ -297,26 +296,18 @@ class LibraryDatabase(object):
             return [(aid, adata[aid]['name'], adata[aid]['sort'], adata[aid]['link']) for aid in authors]
 
     def book_on_device(self, book_id):
-        if callable(self.book_on_device_func):
+        with self.new_api.read_lock:
-            return self.book_on_device_func(book_id)
+            return self.new_api.fields['ondevice'].book_on_device(book_id)
-        return None
 
     def book_on_device_string(self, book_id):
-        loc = []
+        return self.new_api.field_for('ondevice', book_id)
-        count = 0
-        on = self.book_on_device(book_id)
-        if on is not None:
-            m, a, b, count = on[:4]
-            if m is not None:
-                loc.append(_('Main'))
-            if a is not None:
-                loc.append(_('Card A'))
-            if b is not None:
-                loc.append(_('Card B'))
-        return ', '.join(loc) + ((_(' (%s books)')%count) if count > 1 else '')
 
     def set_book_on_device_func(self, func):
-        self.book_on_device_func = func
+        self.new_api.fields['ondevice'].set_book_on_device_func(func)
+
+    @property
+    def book_on_device_func(self):
+        return self.new_api.fields['ondevice'].book_on_device_func
 
     def books_in_series(self, series_id):
         with self.new_api.read_lock:
@@ -474,6 +465,12 @@ class LibraryDatabase(object):
         book_id = index if index_is_id else self.id(index)
         return self.new_api.has_format(book_id, fmt)
 
+    def refresh_format_cache(self):
+        self.new_api.refresh_format_cache()
+
+    def refresh_ondevice(self):
+        self.new_api.refresh_ondevice()
+
     # Private interface {{{
     def __iter__(self):
         for row in self.data.iterall():

src/calibre/db/view.py

@@ -10,6 +10,7 @@ __docformat__ = 'restructuredtext en'
 import weakref
 from functools import partial
 from itertools import izip, imap
+from future_builtins import map
 
 from calibre.ebooks.metadata import title_sort
 from calibre.utils.config_base import tweaks
@@ -163,6 +164,7 @@ class View(object):
 
     def id_to_index(self, book_id):
         return self._map.index(book_id)
+    row = index_to_id
 
     def _get(self, field, idx, index_is_id=True, default_value=None, fmt=lambda x:x):
         id_ = idx if index_is_id else self.index_to_id(idx)
@@ -307,8 +309,17 @@ class View(object):
     def refresh(self, field=None, ascending=True):
         self._map = tuple(self.cache.all_book_ids())
         self._map_filtered = tuple(self._map)
+        self.cache.clear_caches()
         if field is not None:
             self.sort(field, ascending)
         if self.search_restriction or self.base_restriction:
             self.search('', return_matches=False)
 
+    def refresh_ids(self, db, ids):
+        self.cache.clear_caches(book_ids=ids)
+        try:
+            return list(map(self.id_to_index, ids))
+        except ValueError:
+            pass
+        return None
+