Merge branch 'master' of https://github.com/cbhaley/calibre

src/calibre/db/cache.py

@@ -140,9 +140,11 @@ class Cache:
     EventType = EventType
     fts_indexing_sleep_time = 4  # seconds
 
-    def __init__(self, backend):
+    def __init__(self, backend, library_database_instance=None):
         self.shutting_down = False
         self.backend = backend
+        self.library_database_instance = (None if library_database_instance is None else
+                                          weakref.ref(library_database_instance))
         self.event_dispatcher = EventDispatcher()
         self.fields = {}
         self.composites = {}

src/calibre/db/legacy.py

@@ -188,7 +188,7 @@ class LibraryDatabase:
                     read_only=read_only, restore_all_prefs=restore_all_prefs,
                     progress_callback=progress_callback,
                     load_user_formatter_functions=not is_second_db)
-        cache = self.new_api = Cache(backend)
+        cache = self.new_api = Cache(backend, library_database_instance=self)
         cache.init()
         self.data = View(cache)
         self.id = self.data.index_to_id

src/calibre/db/tests/reading.py

@@ -754,66 +754,55 @@ class ReadingTest(BaseTest):
         db = self.init_cache(self.library_path)
         db.create_custom_column('mult', 'CC1', 'composite', True, display={'composite_template': 'b,a,c'})
 
+        # need an empty metadata object to pass to the formatter
         db = self.init_legacy(self.library_path)
+        mi = db.get_metadata(1)
 
-        class GetGuiAns():
+        # test counting books matching the search
-            current_db = None
+        v = formatter.safe_format('program: book_count("series:true", 0)', {}, 'TEMPLATE ERROR', mi)
-        get_gui_ans = GetGuiAns()
+        self.assertEqual(v, '2')
-        get_gui_ans.current_db = db
-        from calibre.gui2.ui import get_gui
-        get_gui.ans = get_gui_ans
 
-        try:
+        # test counting books when none match the search
-            # need an empty metadata object to pass to the formatter
+        v = formatter.safe_format('program: book_count("series:afafaf", 0)', {}, 'TEMPLATE ERROR', mi)
-            mi = Metadata('A', 'B')
+        self.assertEqual(v, '0')
 
-            # test counting books matching the search
+        # test is_multiple values
-            v = formatter.safe_format('program: book_count("series:true", 0)', {}, 'TEMPLATE ERROR', mi)
+        v = formatter.safe_format('program: book_values("tags", "tags:true", ",", 0)', {}, 'TEMPLATE ERROR', mi)
-            self.assertEqual(v, '2')
+        self.assertEqual(set(v.split(',')), {'Tag One', 'News', 'Tag Two'})
 
-            # test counting books when none match the search
+        # test not is_multiple values
-            v = formatter.safe_format('program: book_count("series:afafaf", 0)', {}, 'TEMPLATE ERROR', mi)
+        v = formatter.safe_format('program: book_values("series", "series:true", ",", 0)', {}, 'TEMPLATE ERROR', mi)
-            self.assertEqual(v, '0')
+        self.assertEqual(v, 'A Series One')
 
-            # test is_multiple values
+        # test returning values for a column not searched for
-            v = formatter.safe_format('program: book_values("tags", "tags:true", ",", 0)', {}, 'TEMPLATE ERROR', mi)
+        v = formatter.safe_format('program: book_values("tags", "series:\\"A Series One\\"", ",", 0)', {}, 'TEMPLATE ERROR', mi)
-            self.assertEqual(set(v.split(',')), {'Tag One', 'News', 'Tag Two'})
+        self.assertEqual(set(v.split(',')), {'Tag One', 'News', 'Tag Two'})
 
-            # test not is_multiple values
+        # test getting a singleton value from books where the column is empty
-            v = formatter.safe_format('program: book_values("series", "series:true", ",", 0)', {}, 'TEMPLATE ERROR', mi)
+        v = formatter.safe_format('program: book_values("series", "series:false", ",", 0)', {}, 'TEMPLATE ERROR', mi)
-            self.assertEqual(v, 'A Series One')
+        self.assertEqual(v, '')
 
-            # test returning values for a column not searched for
+        # test getting a multiple value from books where the column is empty
-            v = formatter.safe_format('program: book_values("tags", "series:\\"A Series One\\"", ",", 0)', {}, 'TEMPLATE ERROR', mi)
+        v = formatter.safe_format('program: book_values("tags", "tags:false", ",", 0)', {}, 'TEMPLATE ERROR', mi)
-            self.assertEqual(set(v.split(',')), {'Tag One', 'News', 'Tag Two'})
+        self.assertEqual(v, '')
 
-            # test getting a singleton value from books where the column is empty
+        # test fetching an unknown column
-            v = formatter.safe_format('program: book_values("series", "series:false", ",", 0)', {}, 'TEMPLATE ERROR', mi)
+        v = formatter.safe_format('program: book_values("taaags", "tags:false", ",", 0)', {}, 'TEMPLATE ERROR', mi)
-            self.assertEqual(v, '')
+        self.assertEqual(v, "TEMPLATE ERROR The column taaags doesn't exist")
 
-            # test getting a multiple value from books where the column is empty
+        # test finding all books
-            v = formatter.safe_format('program: book_values("tags", "tags:false", ",", 0)', {}, 'TEMPLATE ERROR', mi)
+        v = formatter.safe_format('program: book_values("id", "title:true", ",", 0)', {}, 'TEMPLATE ERROR', mi)
-            self.assertEqual(v, '')
+        self.assertEqual(set(v.split(',')), {'1', '2', '3'})
 
-            # test fetching an unknown column
+        # test getting value of a composite
-            v = formatter.safe_format('program: book_values("taaags", "tags:false", ",", 0)', {}, 'TEMPLATE ERROR', mi)
+        v = formatter.safe_format('program: book_values("#mult", "id:1", ",", 0)', {}, 'TEMPLATE ERROR', mi)
-            self.assertEqual(v, "TEMPLATE ERROR The column taaags doesn't exist")
+        self.assertEqual(set(v.split(',')), {'b', 'c', 'a'})
 
-            # test finding all books
+        # test getting value of a custom float
-            v = formatter.safe_format('program: book_values("id", "title:true", ",", 0)', {}, 'TEMPLATE ERROR', mi)
+        v = formatter.safe_format('program: book_values("#float", "title:true", ",", 0)', {}, 'TEMPLATE ERROR', mi)
-            self.assertEqual(set(v.split(',')), {'1', '2', '3'})
+        self.assertEqual(set(v.split(',')), {'20.02', '10.01'})
 
-            # test getting value of a composite
+        # test getting value of an int (rating)
-            v = formatter.safe_format('program: book_values("#mult", "id:1", ",", 0)', {}, 'TEMPLATE ERROR', mi)
+        v = formatter.safe_format('program: book_values("rating", "title:true", ",", 0)', {}, 'TEMPLATE ERROR', mi)
-            self.assertEqual(set(v.split(',')), {'b', 'c', 'a'})
+        self.assertEqual(set(v.split(',')), {'4', '6'})
-
-            # test getting value of a custom float
-            v = formatter.safe_format('program: book_values("#float", "title:true", ",", 0)', {}, 'TEMPLATE ERROR', mi)
-            self.assertEqual(set(v.split(',')), {'20.02', '10.01'})
-
-            # test getting value of an int (rating)
-            v = formatter.safe_format('program: book_values("rating", "title:true", ",", 0)', {}, 'TEMPLATE ERROR', mi)
-            self.assertEqual(set(v.split(',')), {'4', '6'})
-        finally:
-            get_gui.ans = None
     # }}}

src/calibre/utils/formatter.py

@@ -15,6 +15,7 @@ from math import modf
 from calibre import prints
 from calibre.constants import DEBUG
 from calibre.ebooks.metadata.book.base import field_metadata
+from calibre.utils.config import tweaks
 from calibre.utils.formatter_functions import formatter_functions
 from calibre.utils.icu import strcmp
 from polyglot.builtins import error_message
@@ -1697,8 +1698,9 @@ class TemplateFormatter(string.Formatter):
             except StopException as e:
                 ans = error_message(e)
             except Exception as e:
-                if DEBUG:  # and getattr(e, 'is_locking_error', False):
+                if DEBUG:
-                    traceback.print_exc()
+                    if tweaks.get('show_stack_traces_in_formatter', True):
+                        traceback.print_exc()
                     if column_name:
                         prints('Error evaluating column named:', column_name)
                 ans = error_value + ' ' + error_message(e)

src/calibre/utils/formatter_functions.py

@@ -12,10 +12,10 @@ __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 import inspect, re, traceback, numbers
+from contextlib import contextmanager, suppress
 from datetime import datetime, timedelta
 from functools import partial
 from math import trunc, floor, ceil, modf
-from contextlib import suppress
 
 from calibre import human_readable, prints, prepare_string_for_xml
 from calibre.constants import DEBUG
@@ -77,6 +77,8 @@ class FormatterFunctions:
                         # Change the body of the template function to one that will
                         # return an error message. Also change the arg count to
                         # -1 (variable) to avoid template compilation errors
+                        if DEBUG:
+                            print(f'attempt to replace formatter function {f.name} with a different body')
                         replace = True
                         func = [cls.name, '', -1, self.error_function_body.format(cls.name)]
                         cls = compile_user_function(*func)
@@ -142,6 +144,27 @@ class FormatterFunction:
         if isinstance(ret, (numbers.Number, bool)):
             return str(ret)
 
+    def only_in_gui_error(self):
+        raise ValueError(_('The function {} can be used only in the GUI').format(self.name))
+
+    def get_database(self, mi):
+        proxy = mi.get('_proxy_metadata', None)
+        if proxy is None:
+            self.only_in_gui_error()
+        wr = proxy.get('_db', None)
+        if wr is None:
+            raise ValueError(_('In function {}: The database has been closed').format(self.name))
+        cache = wr()
+        if cache is None:
+            raise ValueError(_('In function {}: The database has been closed').format(self.name))
+        wr = getattr(cache, 'library_database_instance', None)
+        if wr is None:
+            self.only_in_gui_error()
+        db = wr()
+        if db is None:
+            raise ValueError(_('In function {}: The database has been closed').format(self.name))
+        return db
+
 
 class BuiltinFormatterFunction(FormatterFunction):
 
@@ -928,7 +951,7 @@ class BuiltinApproximateFormats(BuiltinFormatterFunction):
                 return ''
             data = sorted(fmt_data)
             return ','.join(v.upper() for v in data)
-        return _('This function can be used only in the GUI')
+        self.only_in_gui_error()
 
 
 class BuiltinFormatsModtimes(BuiltinFormatterFunction):
@@ -1236,7 +1259,7 @@ class BuiltinBooksize(BuiltinFormatterFunction):
             except:
                 pass
             return ''
-        return _('This function can be used only in the GUI')
+        self.only_in_gui_error()
 
 
 class BuiltinOndevice(BuiltinFormatterFunction):
@@ -1255,7 +1278,7 @@ class BuiltinOndevice(BuiltinFormatterFunction):
             if mi._proxy_metadata.ondevice_col:
                 return _('Yes')
             return ''
-        return _('This function can be used only in the GUI')
+        self.only_in_gui_error()
 
 
 class BuiltinAnnotationCount(BuiltinFormatterFunction):
@@ -1267,11 +1290,8 @@ class BuiltinAnnotationCount(BuiltinFormatterFunction):
                       'This function works only in the GUI.')
 
     def evaluate(self, formatter, kwargs, mi, locals):
-        with suppress(Exception):
+        c = self.get_database(mi).new_api.annotation_count_for_book(mi.id)
-            from calibre.gui2.ui import get_gui
+        return '' if c == 0 else str(c)
-            c = get_gui().current_db.new_api.annotation_count_for_book(mi.id)
-            return '' if c == 0 else str(c)
-        return _('This function can be used only in the GUI')
 
 
 class BuiltinIsMarked(BuiltinFormatterFunction):
@@ -1284,11 +1304,8 @@ class BuiltinIsMarked(BuiltinFormatterFunction):
                       "marks. Returns '' if the book is not marked.")
 
     def evaluate(self, formatter, kwargs, mi, locals):
-        with suppress(Exception):
+        c = self.get_database(mi).data.get_marked(mi.id)
-            from calibre.gui2.ui import get_gui
+        return c if c else ''
-            c = get_gui().current_db.data.get_marked(mi.id)
-            return c if c else ''
-        return _('This function can be used only in the GUI')
 
 
 class BuiltinSeriesSort(BuiltinFormatterFunction):
@@ -1850,14 +1867,12 @@ class BuiltinVirtualLibraries(BuiltinFormatterFunction):
                       'column\'s value in your save/send templates')
 
     def evaluate(self, formatter, kwargs, mi, locals_):
-        with suppress(Exception):
+        db = self.get_database(mi)
-            try:
+        try:
-                from calibre.gui2.ui import get_gui
+            a = db.data.get_virtual_libraries_for_books((mi.id,))
-                a = get_gui().current_db.data.get_virtual_libraries_for_books((mi.id,))
+            return ', '.join(a[mi.id])
-                return ', '.join(a[mi.id])
+        except ValueError as v:
-            except ValueError as v:
+            return str(v)
-                return str(v)
-        return _('This function can be used only in the GUI')
 
 
 class BuiltinCurrentVirtualLibraryName(BuiltinFormatterFunction):
@@ -1870,10 +1885,7 @@ class BuiltinCurrentVirtualLibraryName(BuiltinFormatterFunction):
             'Example: "program: current_virtual_library_name()".')
 
     def evaluate(self, formatter, kwargs, mi, locals):
-        with suppress(Exception):
+        return self.get_database(mi).data.get_base_restriction_name()
-            from calibre.gui2.ui import get_gui
-            return get_gui().current_db.data.get_base_restriction_name()
-        return _('This function can be used only in the GUI')
 
 
 class BuiltinUserCategories(BuiltinFormatterFunction):
@@ -1893,7 +1905,7 @@ class BuiltinUserCategories(BuiltinFormatterFunction):
             cats = {k for k, v in iteritems(mi._proxy_metadata.user_categories) if v}
             cats = sorted(cats, key=sort_key)
             return ', '.join(cats)
-        return _('This function can be used only in the GUI')
+        self.only_in_gui_error()
 
 
 class BuiltinTransliterate(BuiltinFormatterFunction):
@@ -1934,7 +1946,7 @@ class BuiltinAuthorLinks(BuiltinFormatterFunction):
                 return ''
             names = sorted(link_data.keys(), key=sort_key)
             return pair_sep.join(n + val_sep + link_data[n] for n in names)
-        return _('This function can be used only in the GUI')
+        self.only_in_gui_error()
 
 
 class BuiltinAuthorSorts(BuiltinFormatterFunction):
@@ -1970,6 +1982,8 @@ class BuiltinConnectedDeviceName(BuiltinFormatterFunction):
                       "'carda' and 'cardb'. This function works only in the GUI.")
 
     def evaluate(self, formatter, kwargs, mi, locals, storage_location):
+        # We can't use get_database() here because we need the device manager.
+        # In other words, the function really does need the GUI
         with suppress(Exception):
             # Do the import here so that we don't entangle the GUI when using
             # command line functions
@@ -1989,7 +2003,7 @@ class BuiltinConnectedDeviceName(BuiltinFormatterFunction):
             except Exception:
                 traceback.print_exc()
                 raise
-        return _('This function can be used only in the GUI')
+        self.only_in_gui_error()
 
 
 class BuiltinConnectedDeviceUUID(BuiltinFormatterFunction):
@@ -2004,6 +2018,8 @@ class BuiltinConnectedDeviceUUID(BuiltinFormatterFunction):
                       "the GUI.")
 
     def evaluate(self, formatter, kwargs, mi, locals, storage_location):
+        # We can't use get_database() here because we need the device manager.
+        # In other words, the function really does need the GUI
         with suppress(Exception):
             # Do the import here so that we don't entangle the GUI when using
             # command line functions
@@ -2023,7 +2039,7 @@ class BuiltinConnectedDeviceUUID(BuiltinFormatterFunction):
             except Exception:
                 traceback.print_exc()
                 raise
-        return _('This function can be used only in the GUI')
+        self.only_in_gui_error()
 
 
 class BuiltinCheckYesNo(BuiltinFormatterFunction):
@@ -2044,6 +2060,9 @@ class BuiltinCheckYesNo(BuiltinFormatterFunction):
                       'is usually used by the test() or is_empty() functions.')
 
     def evaluate(self, formatter, kwargs, mi, locals, field, is_undefined, is_false, is_true):
+        # 'field' is a lookup name, not a value
+        if field not in self.get_database(mi).field_metadata:
+            raise ValueError(_("The column {} doesn't exist").format(field))
         res = getattr(mi, field, None)
         if res is None:
             if is_undefined == '1':
@@ -2239,15 +2258,12 @@ class BuiltinBookCount(BuiltinFormatterFunction):
         if (not tweaks.get('allow_template_database_functions_in_composites', False) and
                 formatter.global_vars.get(rendering_composite_name, None)):
             raise ValueError(_('The book_count() function cannot be used in a composite column'))
-        with suppress(Exception):
+        db = self.get_database(mi)
-            try:
+        try:
-                from calibre.gui2.ui import get_gui
+            ids = db.search_getting_ids(query, None, use_virtual_library=use_vl != '0')
-                ids = get_gui().current_db.search_getting_ids(query, None,
+            return len(ids)
-                                                              use_virtual_library=use_vl != '0')
+        except Exception:
-                return len(ids)
+            traceback.print_exc()
-            except Exception:
-                traceback.print_exc()
-        return _('This function can be used only in the GUI')
 
 
 class BuiltinBookValues(BuiltinFormatterFunction):
@@ -2265,11 +2281,7 @@ class BuiltinBookValues(BuiltinFormatterFunction):
         if (not tweaks.get('allow_template_database_functions_in_composites', False) and
                 formatter.global_vars.get(rendering_composite_name, None)):
             raise ValueError(_('The book_values() function cannot be used in a composite column'))
-        try:
+        db = self.get_database(mi)
-            from calibre.gui2.ui import get_gui
-            db = get_gui().current_db
-        except:
-            return _('This function can be used only in the GUI')
         if column not in db.field_metadata:
             raise ValueError(_("The column {} doesn't exist").format(column))
         try: