merge from trunk

src/calibre/devices/android/driver.py

@@ -29,7 +29,7 @@ class ANDROID(USBMS):
             # Motorola
             0x22b8 : { 0x41d9 : [0x216], 0x2d61 : [0x100], 0x2d67 : [0x100],
                        0x41db : [0x216], 0x4285 : [0x216], 0x42a3 : [0x216],
-                       0x4286 : [0x216] },
+                       0x4286 : [0x216], 0x42b3 : [0x216] },
 
             # Sony Ericsson
             0xfce : { 0xd12e : [0x0100]},

src/calibre/ebooks/metadata/book/base.py

@@ -324,14 +324,16 @@ class Metadata(object):
             if metadata is None:
                 traceback.print_stack()
                 return
-            metadata = copy.deepcopy(metadata)
+            m = {}
-            if '#value#' not in metadata:
+            for k in metadata:
-                if metadata['datatype'] == 'text' and metadata['is_multiple']:
+                m[k] = copy.copy(metadata[k])
-                    metadata['#value#'] = []
+            if '#value#' not in m:
+                if m['datatype'] == 'text' and m['is_multiple']:
+                    m['#value#'] = []
                 else:
-                    metadata['#value#'] = None
+                    m['#value#'] = None
             _data = object.__getattribute__(self, '_data')
-            _data['user_metadata'][field] = metadata
+            _data['user_metadata'][field] = m
 
     def template_to_attribute(self, other, ops):
         '''

src/calibre/ebooks/txt/txtml.py

@@ -8,7 +8,6 @@ __docformat__ = 'restructuredtext en'
 Transform OEB content into plain text
 '''
 
-import os
 import re
 
 from lxml import etree
@@ -33,6 +32,15 @@ BLOCK_STYLES = [
     'block',
 ]
 
+HEADING_TAGS = [
+    'h1',
+    'h2',
+    'h3',
+    'h4',
+    'h5',
+    'h6',
+]
+
 SPACE_TAGS = [
     'td',
     'br',
@@ -47,6 +55,10 @@ class TXTMLizer(object):
         self.log.info('Converting XHTML to TXT...')
         self.oeb_book = oeb_book
         self.opts = opts
+        self.toc_ids = []
+        self.last_was_heading = False
+        
+        self.create_flat_toc(self.oeb_book.toc)
 
         return self.mlize_spine()
 
@@ -58,8 +70,11 @@ class TXTMLizer(object):
             stylizer = Stylizer(item.data, item.href, self.oeb_book, self.opts, self.opts.output_profile)
             content = unicode(etree.tostring(item.data.find(XHTML('body')), encoding=unicode))
             content = self.remove_newlines(content)
-            output += self.dump_text(etree.fromstring(content), stylizer)
+            output += self.dump_text(etree.fromstring(content), stylizer, item)
-        output = self.cleanup_text(u''.join(output))
+            output += '\n\n\n\n\n\n'
+        output = u''.join(output)
+        output = u'\n'.join(l.rstrip() for l in output.splitlines())
+        output = self.cleanup_text(output)
 
         return output
 
@@ -68,6 +83,8 @@ class TXTMLizer(object):
         text = text.replace('\r\n', ' ')
         text = text.replace('\n', ' ')
         text = text.replace('\r', ' ')
+        # Condense redundant spaces created by replacing newlines with spaces.
+        text = re.sub(r'[ ]{2,}', ' ', text)
 
         return text
 
@@ -80,6 +97,14 @@ class TXTMLizer(object):
                 toc.append(u'* %s\n\n' % item.title)
         return ''.join(toc)
 
+    def create_flat_toc(self, nodes):
+        '''
+        Turns a hierarchical list of TOC href's into a flat list.
+        '''
+        for item in nodes:
+            self.toc_ids.append(item.href)
+            self.create_flat_toc(item.nodes)
+
     def cleanup_text(self, text):
         self.log.debug('\tClean up text...')
         # Replace bad characters.
@@ -92,7 +117,7 @@ class TXTMLizer(object):
         text = text.replace('\f+', ' ')
 
         # Single line paragraph.
-        text = re.sub('(?<=.)%s(?=.)' % os.linesep, ' ', text)
+        text = re.sub('(?<=.)\n(?=.)', ' ', text)
 
         # Remove multiple spaces.
         text = re.sub('[ ]{2,}', ' ', text)
@@ -101,13 +126,19 @@ class TXTMLizer(object):
         text = re.sub('\n[ ]+\n', '\n\n', text)
         if self.opts.remove_paragraph_spacing:
             text = re.sub('\n{2,}', '\n', text)
-            text = re.sub('(?imu)^(?=.)', '\t', text)
+            text = re.sub(r'(?msu)^(?P<t>[^\t\n]+?)$', lambda mo: u'%s\n\n' % mo.group('t'), text)
+            text = re.sub(r'(?msu)(?P<b>[^\n])\n+(?P<t>[^\t\n]+?)(?=\n)', lambda mo: '%s\n\n\n\n\n\n%s' % (mo.group('b'), mo.group('t')), text)
         else:
-            text = re.sub('\n{3,}', '\n\n', text)
+            text = re.sub('\n{7,}', '\n\n\n\n\n\n', text)
 
         # Replace spaces at the beginning and end of lines
+        # We don't replace tabs because those are only added
+        # when remove paragraph spacing is enabled.
         text = re.sub('(?imu)^[ ]+', '', text)
         text = re.sub('(?imu)[ ]+$', '', text)
+        
+        # Remove empty space and newlines at the beginning of the document.
+        text = re.sub(r'(?u)^[ \n]+', '', text)
 
         if self.opts.max_line_length:
             max_length = self.opts.max_line_length
@@ -145,13 +176,11 @@ class TXTMLizer(object):
 
         return text
 
-    def dump_text(self, elem, stylizer, end=''):
+    def dump_text(self, elem, stylizer, page):
         '''
         @elem: The element in the etree that we are working on.
         @stylizer: The style information attached to the element.
-        @end: The last two characters of the text from the previous element.
+        @page: OEB page used to determine absolute urls.
-              This is used to determine if a blank line is needed when starting
-              a new block element.
         '''
 
         if not isinstance(elem.tag, basestring) \
@@ -170,13 +199,22 @@ class TXTMLizer(object):
             return ['']
 
         tag = barename(elem.tag)
+        tag_id = elem.attrib.get('id', None)
         in_block = False
+        in_heading = False
+
+        # Are we in a heading?
+        # This can either be a heading tag or a TOC item.
+        if tag in HEADING_TAGS or '%s#%s' % (page.href, tag_id) in self.toc_ids:
+            in_heading = True
+            if not self.last_was_heading:
+                text.append('\n\n\n\n\n\n')
 
         # Are we in a paragraph block?
         if tag in BLOCK_TAGS or style['display'] in BLOCK_STYLES:
+            if self.opts.remove_paragraph_spacing and not in_heading:
+                text.append(u'\t')
             in_block = True
-            if not end.endswith(u'\n\n') and hasattr(elem, 'text') and elem.text:
-                text.append(u'\n\n')
 
         if tag in SPACE_TAGS:
             text.append(u' ')
@@ -185,14 +223,17 @@ class TXTMLizer(object):
         if hasattr(elem, 'text') and elem.text:
             text.append(elem.text)
 
+        # Recurse down into tags within the tag we are in.
         for item in elem:
-            en = u''
+            text += self.dump_text(item, stylizer, page)
-            if len(text) >= 2:
-                en = text[-1][-2:]
-            text += self.dump_text(item, stylizer, en)
 
         if in_block:
             text.append(u'\n\n')
+        if in_heading:
+            text.append(u'\n')
+            self.last_was_heading = True
+        else:
+            self.last_was_heading = False
 
         if hasattr(elem, 'tail') and elem.tail:
             text.append(elem.tail)

src/calibre/gui2/device.py

@@ -637,7 +637,7 @@ class DeviceMixin(object): # {{{
             self.device_manager.mount_device(kls=FOLDER_DEVICE, kind='folder', path=dir)
 
     def connect_to_bambook(self):
-        self.device_manager.mount_device(kls=BAMBOOKWifi, kind='bambook', 
+        self.device_manager.mount_device(kls=BAMBOOKWifi, kind='bambook',
                                          path=BAMBOOK.settings().extra_customization)
 
     def connect_to_itunes(self):
@@ -1266,8 +1266,8 @@ class DeviceMixin(object): # {{{
         # Force a reset if the caches are not initialized
         if reset or not hasattr(self, 'db_book_title_cache'):
             # Build a cache (map) of the library, so the search isn't On**2
-            self.db_book_title_cache = {}
+            db_book_title_cache = {}
-            self.db_book_uuid_cache = {}
+            db_book_uuid_cache = {}
             # It might be possible to get here without having initialized the
             # library view. In this case, simply give up
             try:
@@ -1278,8 +1278,8 @@ class DeviceMixin(object): # {{{
             for id in db.data.iterallids():
                 mi = db.get_metadata(id, index_is_id=True)
                 title = clean_string(mi.title)
-                if title not in self.db_book_title_cache:
+                if title not in db_book_title_cache:
-                    self.db_book_title_cache[title] = \
+                    db_book_title_cache[title] = \
                                 {'authors':{}, 'author_sort':{}, 'db_ids':{}}
                 # If there are multiple books in the library with the same title
                 # and author, then remember the last one. That is OK, because as
@@ -1287,12 +1287,14 @@ class DeviceMixin(object): # {{{
                 # as another.
                 if mi.authors:
                     authors = clean_string(authors_to_string(mi.authors))
-                    self.db_book_title_cache[title]['authors'][authors] = mi
+                    db_book_title_cache[title]['authors'][authors] = mi
                 if mi.author_sort:
                     aus = clean_string(mi.author_sort)
-                    self.db_book_title_cache[title]['author_sort'][aus] = mi
+                    db_book_title_cache[title]['author_sort'][aus] = mi
-                self.db_book_title_cache[title]['db_ids'][mi.application_id] = mi
+                db_book_title_cache[title]['db_ids'][mi.application_id] = mi
-                self.db_book_uuid_cache[mi.uuid] = mi
+                db_book_uuid_cache[mi.uuid] = mi
+            self.db_book_title_cache = db_book_title_cache
+            self.db_book_uuid_cache = db_book_uuid_cache
 
         # Now iterate through all the books on the device, setting the
         # in_library field. If the UUID matches a book in the library, then

src/calibre/library/caches.py

@@ -181,7 +181,7 @@ class ResultCache(SearchQueryParser): # {{{
         self.search_restriction = ''
         self.field_metadata = field_metadata
         self.all_search_locations = field_metadata.get_search_terms()
-        SearchQueryParser.__init__(self, self.all_search_locations)
+        SearchQueryParser.__init__(self, self.all_search_locations, optimize=True)
         self.build_date_relop_dict()
         self.build_numeric_relop_dict()
 
@@ -264,7 +264,7 @@ class ResultCache(SearchQueryParser): # {{{
                             '<=':[2, relop_le]
                         }
 
-    def get_dates_matches(self, location, query):
+    def get_dates_matches(self, location, query, candidates):
         matches = set([])
         if len(query) < 2:
             return matches
@@ -274,13 +274,15 @@ class ResultCache(SearchQueryParser): # {{{
         loc = self.field_metadata[location]['rec_index']
 
         if query == 'false':
-            for item in self._data:
+            for id_ in candidates:
+                item = self._data[id_]
                 if item is None: continue
                 if item[loc] is None or item[loc] <= UNDEFINED_DATE:
                     matches.add(item[0])
             return matches
         if query == 'true':
-            for item in self._data:
+            for id_ in candidates:
+                item = self._data[id_]
                 if item is None: continue
                 if item[loc] is not None and item[loc] > UNDEFINED_DATE:
                     matches.add(item[0])
@@ -319,7 +321,8 @@ class ResultCache(SearchQueryParser): # {{{
                 field_count = query.count('-') + 1
             else:
                 field_count = query.count('/') + 1
-        for item in self._data:
+        for id_ in candidates:
+            item = self._data[id_]
             if item is None or item[loc] is None: continue
             if relop(item[loc], qd, field_count):
                 matches.add(item[0])
@@ -335,7 +338,7 @@ class ResultCache(SearchQueryParser): # {{{
                         '<=':[2, lambda r, q: r <= q]
                     }
 
-    def get_numeric_matches(self, location, query, val_func = None):
+    def get_numeric_matches(self, location, query, candidates, val_func = None):
         matches = set([])
         if len(query) == 0:
             return matches
@@ -381,7 +384,8 @@ class ResultCache(SearchQueryParser): # {{{
         except:
             return matches
 
-        for item in self._data:
+        for id_ in candidates:
+            item = self._data[id_]
             if item is None:
                 continue
             v = val_func(item)
@@ -393,8 +397,13 @@ class ResultCache(SearchQueryParser): # {{{
                 matches.add(item[0])
         return matches
 
-    def get_matches(self, location, query, allow_recursion=True):
+    def get_matches(self, location, query, allow_recursion=True, candidates=None):
         matches = set([])
+        if candidates is None:
+            candidates = self.universal_set()
+        if len(candidates) == 0:
+            return matches
+
         if query and query.strip():
             # get metadata key associated with the search term. Eliminates
             # dealing with plurals and other aliases
@@ -476,7 +485,8 @@ class ResultCache(SearchQueryParser): # {{{
                 else:
                     q = query
 
-                for item in self._data:
+                for id_ in candidates:
+                    item = self._data[id_]
                     if item is None: continue
 
                     if col_datatype[loc] == 'bool': # complexity caused by the two-/three-value tweak

src/calibre/library/custom_columns.py

@@ -195,8 +195,8 @@ class CustomColumns(object):
             data = self.custom_column_num_map[num]
         row = self.data._data[idx] if index_is_id else self.data[idx]
         ans = row[self.FIELD_MAP[data['num']]]
-        if data['is_multiple'] and data['datatype'] == 'text':
+        if ans and data['is_multiple'] and data['datatype'] == 'text':
-            ans = ans.split('|') if ans else []
+            ans = ans.split('|')
             if data['display'].get('sort_alpha', False):
                 ans.sort(cmp=lambda x,y:cmp(x.lower(), y.lower()))
         return ans

src/calibre/library/database2.py

@@ -256,7 +256,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
              'pubdate',
              'flags',
              'uuid',
-             'has_cover'
+             'has_cover',
+            ('au_map', 'authors', 'author', 'aum_sortconcat(link.id, authors.name, authors.sort)')
             ]
         lines = []
         for col in columns:
@@ -273,9 +274,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
 
         self.FIELD_MAP = {'id':0, 'title':1, 'authors':2, 'timestamp':3,
              'size':4, 'rating':5, 'tags':6, 'comments':7, 'series':8,
-             'publisher':9, 'series_index':10,
+             'publisher':9, 'series_index':10, 'sort':11, 'author_sort':12,
-             'sort':11, 'author_sort':12, 'formats':13, 'isbn':14, 'path':15,
+             'formats':13, 'isbn':14, 'path':15, 'lccn':16, 'pubdate':17,
-             'lccn':16, 'pubdate':17, 'flags':18, 'uuid':19, 'cover':20}
+             'flags':18, 'uuid':19, 'cover':20, 'au_map':21}
 
         for k,v in self.FIELD_MAP.iteritems():
             self.field_metadata.set_field_record_index(k, v, prefer_custom=False)
@@ -687,9 +688,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         Convenience method to return metadata as a :class:`Metadata` object.
         Note that the list of formats is not verified.
         '''
+        row = self.data._data[idx] if index_is_id else self.data[idx]
+        fm = self.FIELD_MAP
+
         self.gm_count += 1
-        mi = self.data.get(idx, self.FIELD_MAP['all_metadata'],
+        mi = row[self.FIELD_MAP['all_metadata']]
-                           row_is_id = index_is_id)
         if mi is not None:
             if get_cover:
                 # Always get the cover, because the value can be wrong if the
@@ -699,49 +702,46 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
 
         self.gm_missed += 1
         mi = Metadata(None)
-        self.data.set(idx, self.FIELD_MAP['all_metadata'], mi,
+        self.data.set(idx, fm['all_metadata'], mi, row_is_id = index_is_id)
-                      row_is_id = index_is_id)
 
-        aut_list = self.authors_with_sort_strings(idx, index_is_id=index_is_id)
+        aut_list = row[fm['au_map']]
+        aut_list = [p.split(':::') for p in aut_list.split(':#:')]
         aum = []
         aus = {}
         for (author, author_sort) in aut_list:
             aum.append(author)
-            aus[author] = author_sort
+            aus[author] = author_sort.replace('|', ',')
-        mi.title       = self.title(idx, index_is_id=index_is_id)
+        mi.title       = row[fm['title']]
         mi.authors     = aum
-        mi.author_sort = self.author_sort(idx, index_is_id=index_is_id)
+        mi.author_sort = row[fm['author_sort']]
         mi.author_sort_map = aus
-        mi.comments    = self.comments(idx, index_is_id=index_is_id)
+        mi.comments    = row[fm['comments']]
-        mi.publisher   = self.publisher(idx, index_is_id=index_is_id)
+        mi.publisher   = row[fm['publisher']]
-        mi.timestamp   = self.timestamp(idx, index_is_id=index_is_id)
+        mi.timestamp   = row[fm['timestamp']]
-        mi.pubdate     = self.pubdate(idx, index_is_id=index_is_id)
+        mi.pubdate     = row[fm['pubdate']]
-        mi.uuid        = self.uuid(idx, index_is_id=index_is_id)
+        mi.uuid        = row[fm['uuid']]
-        mi.title_sort  = self.title_sort(idx, index_is_id=index_is_id)
+        mi.title_sort  = row[fm['sort']]
-        mi.formats     = self.formats(idx, index_is_id=index_is_id,
+        formats = row[fm['formats']]
-                                        verify_formats=False)
+        if not formats:
-        if hasattr(mi.formats, 'split'):
+            formats = None
-            mi.formats = mi.formats.split(',')
+        mi.formats = formats
-        else:
+        tags = row[fm['tags']]
-            mi.formats = None
-        tags = self.tags(idx, index_is_id=index_is_id)
         if tags:
             mi.tags = [i.strip() for i in tags.split(',')]
-        mi.series = self.series(idx, index_is_id=index_is_id)
+        mi.series = row[fm['series']]
         if mi.series:
-            mi.series_index = self.series_index(idx, index_is_id=index_is_id)
+            mi.series_index = row[fm['series_index']]
-        mi.rating = self.rating(idx, index_is_id=index_is_id)
+        mi.rating = row[fm['rating']]
-        mi.isbn = self.isbn(idx, index_is_id=index_is_id)
+        mi.isbn = row[fm['isbn']]
         id = idx if index_is_id else self.id(idx)
         mi.application_id = id
         mi.id = id
-        for key,meta in self.field_metadata.iteritems():
+        for key, meta in self.field_metadata.custom_iteritems():
-            if meta['is_custom']:
+            mi.set_user_metadata(key, meta)
-                mi.set_user_metadata(key, meta)
+            mi.set(key, val=self.get_custom(idx, label=meta['label'],
-                mi.set(key, val=self.get_custom(idx, label=meta['label'],
+                                            index_is_id=index_is_id),
-                                                index_is_id=index_is_id),
+                        extra=self.get_custom_extra(idx, label=meta['label'],
-                            extra=self.get_custom_extra(idx, label=meta['label'],
+                                                    index_is_id=index_is_id))
-                                                        index_is_id=index_is_id))
         if get_cover:
             mi.cover = self.cover(id, index_is_id=True, as_path=True)
         return mi
@@ -877,18 +877,17 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
 
     def formats(self, index, index_is_id=False, verify_formats=True):
         ''' Return available formats as a comma separated list or None if there are no available formats '''
-        id = index if index_is_id else self.id(index)
+        id_ = index if index_is_id else self.id(index)
-        try:
+        formats = self.data.get(id_, self.FIELD_MAP['formats'], row_is_id=True)
-            formats = self.conn.get('SELECT format FROM data WHERE book=?', (id,))
+        if not formats:
-            formats = map(lambda x:x[0], formats)
-        except:
             return None
         if not verify_formats:
-            return ','.join(formats)
+            return formats
+        formats = formats.split(',')
         ans = []
-        for format in formats:
+        for fmt in formats:
-            if self.format_abspath(id, format, index_is_id=True) is not None:
+            if self.format_abspath(id_, fmt, index_is_id=True) is not None:
-                ans.append(format)
+                ans.append(fmt)
         if not ans:
             return None
         return ','.join(ans)
@@ -1607,6 +1606,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                       ','.join([a.replace(',', '|') for a in authors]),
                       row_is_id=True)
         self.data.set(id, self.FIELD_MAP['author_sort'], ss, row_is_id=True)
+        aum = self.authors_with_sort_strings(id, index_is_id=True)
+        self.data.set(id, self.FIELD_MAP['au_map'],
+            ':#:'.join([':::'.join((au.replace(',', '|'), aus)) for (au, aus) in aum]),
+            row_is_id=True)
 
     def set_authors(self, id, authors, notify=True, commit=True):
         '''

src/calibre/library/field_metadata.py

@@ -180,6 +180,15 @@ class FieldMetadata(dict):
                            'search_terms':['author_sort'],
                            'is_custom':False,
                            'is_category':False}),
+            ('au_map',    {'table':None,
+                           'column':None,
+                           'datatype':'text',
+                           'is_multiple':',',
+                           'kind':'field',
+                           'name':None,
+                           'search_terms':[],
+                           'is_custom':False,
+                           'is_category':False}),
             ('comments',  {'table':None,
                            'column':None,
                            'datatype':'text',
@@ -400,6 +409,12 @@ class FieldMetadata(dict):
         for key in self._tb_cats:
             yield (key, self._tb_cats[key])
 
+    def custom_iteritems(self):
+        for key in self._tb_cats:
+            fm = self._tb_cats[key]
+            if fm['is_custom']:
+                yield (key, self._tb_cats[key])
+
     def items(self):
         return list(self.iteritems())
 

src/calibre/library/sqlite.py

@@ -87,6 +87,23 @@ class SortedConcatenate(object):
 class SafeSortedConcatenate(SortedConcatenate):
     sep = '|'
 
+class AumSortedConcatenate(object):
+    '''String concatenation aggregator for the author sort map'''
+    def __init__(self):
+        self.ans = {}
+
+    def step(self, ndx, author, sort):
+        if author is not None:
+            self.ans[ndx] = author + ':::' + sort
+
+    def finalize(self):
+        keys = self.ans.keys()
+        if len(keys) == 0:
+            return None
+        if len(keys) == 1:
+            return self.ans[keys[0]]
+        return ':#:'.join([self.ans[v] for v in sorted(keys)])
+
 class Connection(sqlite.Connection):
 
     def get(self, *args, **kw):
@@ -155,6 +172,7 @@ class DBThread(Thread):
         c_ext_loaded = load_c_extensions(self.conn)
         self.conn.row_factory = sqlite.Row if self.row_factory else  lambda cursor, row : list(row)
         self.conn.create_aggregate('concat', 1, Concatenate)
+        self.conn.create_aggregate('aum_sortconcat', 3, AumSortedConcatenate)
         if not c_ext_loaded:
             self.conn.create_aggregate('sortconcat', 2, SortedConcatenate)
             self.conn.create_aggregate('sort_concat', 2, SafeSortedConcatenate)

src/calibre/utils/search_query_parser.py

@@ -118,8 +118,9 @@ class SearchQueryParser(object):
                 failed.append(test[0])
         return failed
 
-    def __init__(self, locations, test=False):
+    def __init__(self, locations, test=False, optimize=False):
         self._tests_failed = False
+        self.optimize = optimize
         # Define a token
         standard_locations = map(lambda x : CaselessLiteral(x)+Suppress(':'),
                 locations)
@@ -182,38 +183,52 @@ class SearchQueryParser(object):
         # empty the list of searches used for recursion testing
         self.recurse_level = 0
         self.searches_seen = set([])
-        return self._parse(query)
+        candidates = self.universal_set()
+        return self._parse(query, candidates)
 
     # this parse is used internally because it doesn't clear the
     # recursive search test list. However, we permit seeing the
     # same search a few times because the search might appear within
     # another search.
-    def _parse(self, query):
+    def _parse(self, query, candidates=None):
         self.recurse_level += 1
         res = self._parser.parseString(query)[0]
-        t = self.evaluate(res)
+        if candidates is None:
+            candidates = self.universal_set()
+        t = self.evaluate(res, candidates)
         self.recurse_level -= 1
         return t
 
     def method(self, group_name):
         return getattr(self, 'evaluate_'+group_name)
 
-    def evaluate(self, parse_result):
+    def evaluate(self, parse_result, candidates):
-        return self.method(parse_result.getName())(parse_result)
+        return self.method(parse_result.getName())(parse_result, candidates)
 
-    def evaluate_and(self, argument):
+    def evaluate_and(self, argument, candidates):
-        return self.evaluate(argument[0]).intersection(self.evaluate(argument[1]))
+        # RHS checks only those items matched by LHS
+        # returns result of RHS check: RHmatches(LHmatches(c))
+        #  return self.evaluate(argument[0]).intersection(self.evaluate(argument[1]))
+        l = self.evaluate(argument[0], candidates)
+        return l.intersection(self.evaluate(argument[1], l))
 
-    def evaluate_or(self, argument):
+    def evaluate_or(self, argument, candidates):
-        return self.evaluate(argument[0]).union(self.evaluate(argument[1]))
+        # RHS checks only those elements not matched by LHS
+        # returns LHS union RHS: LHmatches(c) + RHmatches(c-LHmatches(c))
+        #  return self.evaluate(argument[0]).union(self.evaluate(argument[1]))
+        l = self.evaluate(argument[0], candidates)
+        return l.union(self.evaluate(argument[1], candidates.difference(l)))
 
-    def evaluate_not(self, argument):
+    def evaluate_not(self, argument, candidates):
-        return self.universal_set().difference(self.evaluate(argument[0]))
+        # unary op checks only candidates. Result: list of items matching
+        # returns: c - matches(c)
+        #  return self.universal_set().difference(self.evaluate(argument[0]))
+        return candidates.difference(self.evaluate(argument[0], candidates))
 
-    def evaluate_parenthesis(self, argument):
+    def evaluate_parenthesis(self, argument, candidates):
-        return self.evaluate(argument[0])
+        return self.evaluate(argument[0], candidates)
 
-    def evaluate_token(self, argument):
+    def evaluate_token(self, argument, candidates):
         location = argument[0]
         query = argument[1]
         if location.lower() == 'search':
@@ -224,17 +239,27 @@ class SearchQueryParser(object):
                     raise ParseException(query, len(query), 'undefined saved search', self)
                 if self.recurse_level > 5:
                     self.searches_seen.add(query)
-                return self._parse(saved_searches().lookup(query))
+                return self._parse(saved_searches().lookup(query), candidates)
             except: # convert all exceptions (e.g., missing key) to a parse error
                 raise ParseException(query, len(query), 'undefined saved search', self)
-        return self.get_matches(location, query)
+        return self._get_matches(location, query, candidates)
 
-    def get_matches(self, location, query):
+    def _get_matches(self, location, query, candidates):
+        if self.optimize:
+            return self.get_matches(location, query, candidates=candidates)
+        else:
+            return self.get_matches(location, query)
+
+    def get_matches(self, location, query, candidates=None):
         '''
         Should return the set of matches for :param:'location` and :param:`query`.
 
+        The search must be performed over all entries is :param:`candidates` is
+        None otherwise only over the items in candidates.
+
         :param:`location` is one of the items in :member:`SearchQueryParser.DEFAULT_LOCATIONS`.
         :param:`query` is a string literal.
+        :param: None or a subset of the set returned by :meth:`universal_set`.
         '''
         return set([])
 
@@ -561,7 +586,7 @@ class Tester(SearchQueryParser):
     def universal_set(self):
         return self._universal_set
 
-    def get_matches(self, location, query):
+    def get_matches(self, location, query, candidates=None):
         location = location.lower()
         if location in self.fields.keys():
             getter = operator.itemgetter(self.fields[location])
@@ -573,8 +598,13 @@ class Tester(SearchQueryParser):
         if not query:
             return set([])
         query = query.lower()
-        return set(key for key, val in self.texts.items() \
+        if candidates:
-            if query and query in getattr(getter(val), 'lower', lambda : '')())
+            return set(key for key, val in self.texts.items() \
+                if key in candidates and query and query
+                        in getattr(getter(val), 'lower', lambda : '')())
+        else:
+            return set(key for key, val in self.texts.items() \
+                if query and query in getattr(getter(val), 'lower', lambda : '')())
 
 
 
@@ -592,6 +622,7 @@ class Tester(SearchQueryParser):
 
 
 def main(args=sys.argv):
+    print 'testing unoptimized'
     tester = Tester(['authors', 'author', 'series', 'formats', 'format',
         'publisher', 'rating', 'tags', 'tag', 'comments', 'comment', 'cover',
         'isbn', 'ondevice', 'pubdate', 'size', 'date', 'title', u'#read',
@@ -601,6 +632,16 @@ def main(args=sys.argv):
         print '>>>>>>>>>>>>>> Tests Failed <<<<<<<<<<<<<<<'
         return 1
 
+    print '\n\ntesting optimized'
+    tester = Tester(['authors', 'author', 'series', 'formats', 'format',
+        'publisher', 'rating', 'tags', 'tag', 'comments', 'comment', 'cover',
+        'isbn', 'ondevice', 'pubdate', 'size', 'date', 'title', u'#read',
+        'all', 'search'], test=True, optimize=True)
+    failed = tester.run_tests()
+    if tester._tests_failed or failed:
+        print '>>>>>>>>>>>>>> Tests Failed <<<<<<<<<<<<<<<'
+        return 1
+
     return 0
 
 if __name__ == '__main__':