Fix locking error for searches + composite columns

Fix a locking error when composite columns containing formats are used
and formats are added/deleted. The error would only be triggered if a search
was previously done that looked at the formats column. Also speed up
searching a little by avoiding a full get_metadata when searching
composite columns. Fixes #1233330 [Custom column Formats error](https://bugs.launchpad.net/calibre/+bug/1233330)

src/calibre/db/cache.py

@@ -18,7 +18,7 @@ from calibre.constants import iswindows, preferred_encoding
 from calibre.customize.ui import run_plugins_on_import, run_plugins_on_postimport
 from calibre.db import SPOOL_SIZE, _get_next_series_num_for_list
 from calibre.db.categories import get_categories
-from calibre.db.locking import create_locks
+from calibre.db.locking import create_locks, DowngradeLockError
 from calibre.db.errors import NoSuchFormat
 from calibre.db.fields import create_field, IDENTITY, InvalidLinkTable
 from calibre.db.search import Search
@@ -51,10 +51,19 @@ def write_api(f):
 
 def wrap_simple(lock, func):
     @wraps(func)
-    def ans(*args, **kwargs):
+    def call_func_with_lock(*args, **kwargs):
-        with lock:
+        try:
+            with lock:
+                return func(*args, **kwargs)
+        except DowngradeLockError:
+            # We already have an exclusive lock, no need to acquire a shared
+            # lock. This can happen when updating the search cache in the
+            # presence of composite columns. Updating the search cache holds an
+            # exclusive lock, but searching a composite column involves
+            # reading field values via ProxyMetadata which tries to get a
+            # shared lock.
             return func(*args, **kwargs)
-    return ans
+    return call_func_with_lock
 
 def run_import_plugins(path_or_stream, fmt):
     fmt = fmt.lower()

src/calibre/db/locking.py

@@ -17,6 +17,9 @@ class LockingError(RuntimeError):
         RuntimeError.__init__(self, msg)
         self.locking_debug_msg = extra
 
+class DowngradeLockError(LockingError):
+    pass
+
 def create_locks():
     '''
     Return a pair of locks: (read_lock, write_lock)
@@ -150,7 +153,7 @@ class SHLock(object):  # {{{
         #  to the shared queue and it will give us the lock eventually.
         if self.is_exclusive or self._exclusive_queue:
             if self._exclusive_owner is me:
-                raise LockingError("can't downgrade SHLock object")
+                raise DowngradeLockError("can't downgrade SHLock object")
             if not blocking:
                 return False
             waiter = self._take_waiter()

src/calibre/db/search.py

@@ -464,7 +464,7 @@ class Parser(SearchQueryParser):  # {{{
         return self.all_book_ids
 
     def field_iter(self, name, candidates):
-        get_metadata = partial(self.dbcache._get_metadata, get_user_categories=False)
+        get_metadata = self.dbcache._get_proxy_metadata
         try:
             field = self.dbcache.fields[name]
         except KeyError:

src/calibre/db/tests/reading.py

@@ -583,6 +583,8 @@ class ReadingTest(BaseTest):
                                    display={'composite_template': '{pubdate:format_date(d-M-yy)}', 'composite_sort':'date'})
         cache.create_custom_column('bool', 'CC6', 'composite', False, display={'composite_template': '{#yesno}', 'composite_sort':'bool'})
         cache.create_custom_column('ccm', 'CC7', 'composite', True, display={'composite_template': '{#tags}'})
+        cache.create_custom_column('ccp', 'CC8', 'composite', True, display={'composite_template': '{publisher}'})
+        cache.create_custom_column('ccf', 'CC9', 'composite', True, display={'composite_template': "{:'approximate_formats()'}"})
 
         cache = self.init_cache()
         # Test searching
@@ -607,5 +609,14 @@ class ReadingTest(BaseTest):
         # Test is_multiple sorting
         cache.set_field('#tags', {1:'b, a, c', 2:'a, b, c', 3:'a, c, b'})
         self.assertEqual([1, 2, 3], cache.multisort([('#ccm', True)]))
+
+        # Test that lock downgrading during update of search cache works
+        self.assertEqual(cache.search('#ccp:One'), {2})
+        cache.set_field('publisher', {2:'One', 1:'One'})
+        self.assertEqual(cache.search('#ccp:One'), {1, 2})
+
+        self.assertEqual(cache.search('#ccf:FMT1'), {1, 2})
+        cache.remove_formats({1:('FMT1',)})
+        self.assertEqual('FMT2', cache.field_for('#ccf', 1))
     # }}}