From db2cadd070e4c682e3e0da9f4b4c2747574b7283 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Wed, 13 Apr 2011 10:54:21 +0100 Subject: [PATCH 1/3] Fixes for search: 1) raise an exception if a non-existent field is used. 2) make numeric searches correctly respect None values (python treats None as less than anything) 3) make rating-type columns treat None as zero and zero as False --- src/calibre/library/caches.py | 50 +++++++++++++----------- src/calibre/utils/search_query_parser.py | 17 ++++---- 2 files changed, 37 insertions(+), 30 deletions(-) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 01b7335bf4..af9a766174 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -391,11 +391,11 @@ class ResultCache(SearchQueryParser): # {{{ def build_numeric_relop_dict(self): self.numeric_search_relops = { '=':[1, lambda r, q: r == q], - '>':[1, lambda r, q: r > q], - '<':[1, lambda r, q: r < q], + '>':[1, lambda r, q: r is not None and r > q], + '<':[1, lambda r, q: r is not None and r < q], '!=':[2, lambda r, q: r != q], - '>=':[2, lambda r, q: r >= q], - '<=':[2, lambda r, q: r <= q] + '>=':[2, lambda r, q: r is not None and r >= q], + '<=':[2, lambda r, q: r is not None and r <= q] } def get_numeric_matches(self, location, query, candidates, val_func = None): @@ -406,17 +406,22 @@ class ResultCache(SearchQueryParser): # {{{ if val_func is None: loc = self.field_metadata[location]['rec_index'] val_func = lambda item, loc=loc: item[loc] + dt = self.field_metadata[location]['datatype'] + + q = '' + val_func = lambda item, loc=loc: item[loc] + cast = adjust = lambda x: x if query == 'false': - q = '' - relop = lambda x,y: x is None - val_func = lambda item, loc=loc: item[loc] - cast = adjust = lambda x: x + if dt == 'rating': + relop = lambda x,y: not bool(x) + else: + relop = lambda x,y: x is None elif query == 'true': - q = '' - relop = lambda x,y: x is not None - val_func = lambda item, loc=loc: item[loc] - cast = adjust = lambda x: x + if dt == 'rating': + relop = lambda x,y: bool(x) + else: + relop = lambda x,y: x is not None else: relop = None for k in self.numeric_search_relops.keys(): @@ -426,19 +431,15 @@ class ResultCache(SearchQueryParser): # {{{ if relop is None: (p, relop) = self.numeric_search_relops['='] - dt = self.field_metadata[location]['datatype'] if dt == 'int': - cast = lambda x: int (x) if x is not None else None - adjust = lambda x: x - elif dt == 'rating': cast = lambda x: int (x) + elif dt == 'rating': + cast = lambda x: 0 if x is None else int (x) adjust = lambda x: x/2 elif dt in ('float', 'composite'): - cast = lambda x : float (x) if x is not None else None - adjust = lambda x: x + cast = lambda x : float (x) else: # count operation cast = (lambda x: int (x)) - adjust = lambda x: x if len(query) > 1: mult = query[-1:].lower() @@ -450,7 +451,8 @@ class ResultCache(SearchQueryParser): # {{{ try: q = cast(query) * mult except: - return matches + raise ParseException(query, len(query), + 'Non-numeric value in query', self) for id_ in candidates: item = self._data[id_] @@ -459,11 +461,14 @@ class ResultCache(SearchQueryParser): # {{{ try: v = cast(val_func(item)) except: - v = 0 + v = None if v: v = adjust(v) if relop(v, q): matches.add(item[0]) + print v, q, 'YES' + else: + print v, q, 'NO' return matches def get_user_category_matches(self, location, query, candidates): @@ -590,8 +595,7 @@ class ResultCache(SearchQueryParser): # {{{ candidates = self.universal_set() if len(candidates) == 0: return matches - if location not in self.all_search_locations: - return matches + self.test_location_is_valid(location, query) if len(location) > 2 and location.startswith('@') and \ location[1:] in self.db_prefs['grouped_search_terms']: diff --git a/src/calibre/utils/search_query_parser.py b/src/calibre/utils/search_query_parser.py index a50ca20fc1..387ad1487e 100644 --- a/src/calibre/utils/search_query_parser.py +++ b/src/calibre/utils/search_query_parser.py @@ -20,7 +20,7 @@ import sys, string, operator from calibre.utils.pyparsing import CaselessKeyword, Group, Forward, \ CharsNotIn, Suppress, OneOrMore, MatchFirst, CaselessLiteral, \ - Optional, NoMatch, ParseException, QuotedString + Optional, NoMatch, ParseException, QuotedString, Word from calibre.constants import preferred_encoding from calibre.utils.icu import sort_key @@ -128,12 +128,8 @@ class SearchQueryParser(object): self._tests_failed = False self.optimize = optimize # Define a token - standard_locations = map(lambda x : CaselessLiteral(x)+Suppress(':'), - locations) - location = NoMatch() - for l in standard_locations: - location |= l - location = Optional(location, default='all') + self.standard_locations = locations + location = Optional(Word(string.ascii_letters+'#')+Suppress(':'), default='all') word_query = CharsNotIn(string.whitespace + '()') #quoted_query = Suppress('"')+CharsNotIn('"')+Suppress('"') quoted_query = QuotedString('"', escChar='\\') @@ -250,7 +246,14 @@ class SearchQueryParser(object): raise ParseException(query, len(query), 'undefined saved search', self) return self._get_matches(location, query, candidates) + def test_location_is_valid(self, location, query): + if location not in self.standard_locations: + raise ParseException(query, len(query), + _('No column exists with lookup name ') + location, self) + def _get_matches(self, location, query, candidates): + location = location.lower() + self.test_location_is_valid(location, query) if self.optimize: return self.get_matches(location, query, candidates=candidates) else: From ae3e40eccb5e28641275d22773e714f3a6a46dd4 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Wed, 13 Apr 2011 11:39:51 +0100 Subject: [PATCH 2/3] ... --- src/calibre/library/caches.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index af9a766174..5f4cfcba07 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -466,9 +466,6 @@ class ResultCache(SearchQueryParser): # {{{ v = adjust(v) if relop(v, q): matches.add(item[0]) - print v, q, 'YES' - else: - print v, q, 'NO' return matches def get_user_category_matches(self, location, query, candidates): From e753b0fe2a39a139ea4221a4904624dd1b063bef Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Wed, 13 Apr 2011 11:47:09 +0100 Subject: [PATCH 3/3] Add 'size' to mi produced by get_metadata. --- src/calibre/library/database2.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 50b404b4be..7b4d52dbcd 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -854,6 +854,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): mi.uuid = row[fm['uuid']] mi.title_sort = row[fm['sort']] mi.last_modified = row[fm['last_modified']] + mi.size = row[fm['size']] formats = row[fm['formats']] if not formats: formats = None