Merge branch 'master' of https://github.com/cbhaley/calibre

Fix #1927179 [VL set to wildcard crashes Calibre](https://bugs.launchpad.net/calibre/+bug/1927179)

src/calibre/db/cache.py

@@ -142,6 +142,7 @@ class Cache(object):
         self.formatter_template_cache = {}
         self.dirtied_cache = {}
         self.vls_for_books_cache = None
+        self.vls_for_books_cache_is_loading = False
         self.dirtied_sequence = 0
         self.cover_caches = set()
         self.clear_search_cache_count = 0
@@ -251,6 +252,7 @@ class Cache(object):
         self.clear_search_cache_count += 1
         self._search_api.update_or_clear(self, book_ids)
         self.vls_for_books_cache = None
+        self.vls_for_books_cache_is_loading = False
 
     @read_api
     def last_modified(self):
@@ -672,8 +674,26 @@ class Cache(object):
         cover_as_data is True then as mi.cover_data.
         '''
 
+        # Check if virtual_libraries_for_books rebuilt its cache. If it did then
+        # we must clear the composite caches so the new data can be taken into
+        # account. Clearing the caches requires getting a write lock, so it must
+        # be done outside of the closure of _get_metadata().
+        composite_cache_needs_to_be_cleared = False
         with self.safe_read_lock:
+            vl_cache_was_none = self.vls_for_books_cache is None
             mi = self._get_metadata(book_id, get_user_categories=get_user_categories)
+            if vl_cache_was_none and self.vls_for_books_cache is not None:
+                composite_cache_needs_to_be_cleared = True
+        if composite_cache_needs_to_be_cleared:
+            try:
+                with self.write_lock:
+                    self.clear_composite_caches()
+            except DowngradeLockError:
+                # We can't clear the composite caches because a read lock is set.
+                # As a consequence the value of a composite column that calls
+                # virtual_libraries() might be wrong. Oh well. Log and keep running.
+                print("Couldn't get write lock after vls_for_books_cache was loaded", file=sys.stderr)
+                traceback.print_exc()
 
         if get_cover:
             if cover_as_data:
@@ -2213,13 +2233,25 @@ class Cache(object):
         if self.vls_for_books_cache is None:
             # Using a list is slightly faster than a set.
             c = defaultdict(list)
+            if self.vls_for_books_cache_is_loading:
+                # We get here if resolving the books in a VL triggers another VL
+                # calculation. This can be 'real' recursion, in which case the
+                # eventual answer will be wrong. It can also be a  search using
+                # a location of 'all' that causes evaluation of a composite that
+                # references virtual libraries. If the composite isn't used in a
+                # VL then the eventual answer will be correct because get_metadata
+                # will clear the caches.
+                return c
+            self.vls_for_books_cache_is_loading = True
             libraries = self._pref('virtual_libraries', {})
             for lib, expr in libraries.items():
+                book = None
                 try:
                     for book in self._search(expr, virtual_fields=virtual_fields):
                         c[book].append(lib)
                 except Exception as e:
-                    c[book].append(_('[Error in Virtual library {0}: {1}]').format(lib, str(e)))
+                    if book:
+                        c[book].append(_('[Error in Virtual library {0}: {1}]').format(lib, str(e)))
             self.vls_for_books_cache = {b:tuple(sorted(libs, key=sort_key)) for b, libs in c.items()}
         if not book_ids:
             book_ids = self._all_book_ids()