Sync to trunk.

src/calibre/devices/android/driver.py

@@ -29,7 +29,7 @@ class ANDROID(USBMS):
             # Motorola
             0x22b8 : { 0x41d9 : [0x216], 0x2d61 : [0x100], 0x2d67 : [0x100],
                        0x41db : [0x216], 0x4285 : [0x216], 0x42a3 : [0x216],
-                       0x4286 : [0x216] },
+                       0x4286 : [0x216], 0x42b3 : [0x216] },
 
             # Sony Ericsson
             0xfce : { 0xd12e : [0x0100]},

src/calibre/ebooks/txt/heuristicprocessor.py

@@ -7,7 +7,6 @@ __docformat__ = 'restructuredtext en'
 import re
 
 from calibre import prepare_string_for_xml
-from calibre.ebooks.unidecode.unidecoder import Unidecoder
 
 class TXTHeuristicProcessor(object):
 

src/calibre/library/caches.py

@@ -181,7 +181,7 @@ class ResultCache(SearchQueryParser): # {{{
         self.search_restriction = ''
         self.field_metadata = field_metadata
         self.all_search_locations = field_metadata.get_search_terms()
-        SearchQueryParser.__init__(self, self.all_search_locations)
+        SearchQueryParser.__init__(self, self.all_search_locations, optimize=True)
         self.build_date_relop_dict()
         self.build_numeric_relop_dict()
 
@@ -264,7 +264,7 @@ class ResultCache(SearchQueryParser): # {{{
                             '<=':[2, relop_le]
                         }
 
-    def get_dates_matches(self, location, query):
+    def get_dates_matches(self, location, query, candidates):
         matches = set([])
         if len(query) < 2:
             return matches
@@ -274,13 +274,15 @@ class ResultCache(SearchQueryParser): # {{{
         loc = self.field_metadata[location]['rec_index']
 
         if query == 'false':
-            for item in self._data:
+            for id_ in candidates:
+                item = self._data[id_]
                 if item is None: continue
                 if item[loc] is None or item[loc] <= UNDEFINED_DATE:
                     matches.add(item[0])
             return matches
         if query == 'true':
-            for item in self._data:
+            for id_ in candidates:
+                item = self._data[id_]
                 if item is None: continue
                 if item[loc] is not None and item[loc] > UNDEFINED_DATE:
                     matches.add(item[0])
@@ -319,7 +321,8 @@ class ResultCache(SearchQueryParser): # {{{
                 field_count = query.count('-') + 1
             else:
                 field_count = query.count('/') + 1
-        for item in self._data:
+        for id_ in candidates:
+            item = self._data[id_]
             if item is None or item[loc] is None: continue
             if relop(item[loc], qd, field_count):
                 matches.add(item[0])
@@ -335,7 +338,7 @@ class ResultCache(SearchQueryParser): # {{{
                         '<=':[2, lambda r, q: r <= q]
                     }
 
-    def get_numeric_matches(self, location, query, val_func = None):
+    def get_numeric_matches(self, location, query, candidates, val_func = None):
         matches = set([])
         if len(query) == 0:
             return matches
@@ -381,7 +384,8 @@ class ResultCache(SearchQueryParser): # {{{
         except:
             return matches
 
-        for item in self._data:
+        for id_ in candidates:
+            item = self._data[id_]
             if item is None:
                 continue
             v = val_func(item)
@@ -393,8 +397,13 @@ class ResultCache(SearchQueryParser): # {{{
                 matches.add(item[0])
         return matches
 
-    def get_matches(self, location, query, allow_recursion=True):
+    def get_matches(self, location, query, allow_recursion=True, candidates=None):
         matches = set([])
+        if candidates is None:
+            candidates = self.universal_set()
+        if len(candidates) == 0:
+            return matches
+
         if query and query.strip():
             # get metadata key associated with the search term. Eliminates
             # dealing with plurals and other aliases
@@ -476,7 +485,8 @@ class ResultCache(SearchQueryParser): # {{{
                 else:
                     q = query
 
-                for item in self._data:
+                for id_ in candidates:
+                    item = self._data[id]
                     if item is None: continue
 
                     if col_datatype[loc] == 'bool': # complexity caused by the two-/three-value tweak

src/calibre/translations/calibre.pot

@@ -5,8 +5,8 @@
 msgid ""
 msgstr ""
 "Project-Id-Version: calibre 0.7.38\n"
-"POT-Creation-Date: 2011-01-07 13:12+MST\n"
+"POT-Creation-Date: 2011-01-08 18:40+MST\n"
-"PO-Revision-Date: 2011-01-07 13:12+MST\n"
+"PO-Revision-Date: 2011-01-08 18:40+MST\n"
 "Last-Translator: Automatically generated\n"
 "Language-Team: LANGUAGE\n"
 "MIME-Version: 1.0\n"
@@ -2905,28 +2905,29 @@ msgstr ""
 msgid " (Preface)"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/ebooks/txt/input.py:26
+#: /home/kovid/work/calibre/src/calibre/ebooks/txt/input.py:27
 msgid ""
 "Paragraph structure.\n"
-"choices are ['auto', 'block', 'single', 'print', 'markdown']\n"
+"choices are ['auto', 'block', 'single', 'print', 'unformatted']\n"
 "* auto: Try to auto detect paragraph type.\n"
 "* block: Treat a blank line as a paragraph break.\n"
 "* single: Assume every line is a paragraph.\n"
-"* print:  Assume every line starting with 2+ spaces or a tab starts a paragraph."
+"* print:  Assume every line starting with 2+ spaces or a tab starts a paragraph.* unformatted: Most lines have hard line breaks, few/no spaces or indents."
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/ebooks/txt/input.py:35
+#: /home/kovid/work/calibre/src/calibre/ebooks/txt/input.py:37
 msgid ""
-"Formatting used within the document.* auto: Try to auto detect the document formatting.\n"
+"Formatting used within the document.* auto: Automatically decide which formatting processor to use.\n"
-"* none: Do not modify the paragraph formatting. Everything is a paragraph.\n"
+"* none: Do not process the document formatting. Everything is a paragraph and no styling is applied.\n"
-"* markdown: Run the input though the markdown pre-processor. To learn more about markdown see"
+"* heuristic: Process using heuristics to determine formatting such as chapter headings and italic text.\n"
+"* markdown: Processing using markdown formatting. To learn more about markdown see"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/ebooks/txt/input.py:41
+#: /home/kovid/work/calibre/src/calibre/ebooks/txt/input.py:46
 msgid "Normally extra spaces are condensed into a single space. With this option all spaces will be displayed."
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/ebooks/txt/input.py:44
+#: /home/kovid/work/calibre/src/calibre/ebooks/txt/input.py:49
 msgid "Do not insert a Table of Contents into the output text."
 msgstr ""
 
@@ -7225,7 +7226,7 @@ msgstr ""
 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/password_ui.py:65
 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/scheduler_ui.py:219
 #: /home/kovid/work/calibre/src/calibre/gui2/preferences/server_ui.py:130
-#: /home/kovid/work/calibre/src/calibre/gui2/wizard/send_email.py:169
+#: /home/kovid/work/calibre/src/calibre/gui2/wizard/send_email.py:172
 msgid "&Show password"
 msgstr ""
 
@@ -10621,48 +10622,56 @@ msgstr ""
 msgid "Mail successfully sent"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/wizard/send_email.py:136
+#: /home/kovid/work/calibre/src/calibre/gui2/wizard/send_email.py:95
+msgid "OK to proceed?"
+msgstr ""
+
+#: /home/kovid/work/calibre/src/calibre/gui2/wizard/send_email.py:96
+msgid "This will display your email password on the screen. Is it OK to proceed?"
+msgstr ""
+
+#: /home/kovid/work/calibre/src/calibre/gui2/wizard/send_email.py:139
 msgid "If you are setting up a new hotmail account, you must log in to it  once before you will be able to send mails."
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/wizard/send_email.py:147
+#: /home/kovid/work/calibre/src/calibre/gui2/wizard/send_email.py:150
 msgid "Setup sending email using"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/wizard/send_email.py:149
+#: /home/kovid/work/calibre/src/calibre/gui2/wizard/send_email.py:152
 msgid "If you don't have an account, you can sign up for a free {name} email account at <a href=\"http://{url}\">http://{url}</a>. {extra}"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/wizard/send_email.py:156
+#: /home/kovid/work/calibre/src/calibre/gui2/wizard/send_email.py:159
 msgid "Your %s &email address:"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/wizard/send_email.py:157
+#: /home/kovid/work/calibre/src/calibre/gui2/wizard/send_email.py:160
 msgid "Your %s &username:"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/wizard/send_email.py:158
+#: /home/kovid/work/calibre/src/calibre/gui2/wizard/send_email.py:161
 msgid "Your %s &password:"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/wizard/send_email.py:176
+#: /home/kovid/work/calibre/src/calibre/gui2/wizard/send_email.py:179
 msgid "If you plan to use email to send books to your Kindle, remember to add the your %s email address to the allowed email addresses in your Amazon.com Kindle management page."
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/wizard/send_email.py:183
+#: /home/kovid/work/calibre/src/calibre/gui2/wizard/send_email.py:186
 msgid "Setup"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/wizard/send_email.py:198
+#: /home/kovid/work/calibre/src/calibre/gui2/wizard/send_email.py:201
-#: /home/kovid/work/calibre/src/calibre/gui2/wizard/send_email.py:205
+#: /home/kovid/work/calibre/src/calibre/gui2/wizard/send_email.py:208
 msgid "Bad configuration"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/wizard/send_email.py:199
+#: /home/kovid/work/calibre/src/calibre/gui2/wizard/send_email.py:202
 msgid "You must set the From email address"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/wizard/send_email.py:206
+#: /home/kovid/work/calibre/src/calibre/gui2/wizard/send_email.py:209
 msgid "You must set the username and password for the mail server."
 msgstr ""
 

src/calibre/utils/search_query_parser.py

@@ -118,8 +118,9 @@ class SearchQueryParser(object):
                 failed.append(test[0])
         return failed
 
-    def __init__(self, locations, test=False):
+    def __init__(self, locations, test=False, optimize=False):
         self._tests_failed = False
+        self.optimize = optimize
         # Define a token
         standard_locations = map(lambda x : CaselessLiteral(x)+Suppress(':'),
                 locations)
@@ -182,38 +183,52 @@ class SearchQueryParser(object):
         # empty the list of searches used for recursion testing
         self.recurse_level = 0
         self.searches_seen = set([])
-        return self._parse(query)
+        candidates = self.universal_set()
+        return self._parse(query, candidates)
 
     # this parse is used internally because it doesn't clear the
     # recursive search test list. However, we permit seeing the
     # same search a few times because the search might appear within
     # another search.
-    def _parse(self, query):
+    def _parse(self, query, candidates=None):
         self.recurse_level += 1
         res = self._parser.parseString(query)[0]
-        t = self.evaluate(res)
+        if candidates is None:
+            candidates = self.universal_set()
+        t = self.evaluate(res, candidates)
         self.recurse_level -= 1
         return t
 
     def method(self, group_name):
         return getattr(self, 'evaluate_'+group_name)
 
-    def evaluate(self, parse_result):
+    def evaluate(self, parse_result, candidates):
-        return self.method(parse_result.getName())(parse_result)
+        return self.method(parse_result.getName())(parse_result, candidates)
 
-    def evaluate_and(self, argument):
+    def evaluate_and(self, argument, candidates):
-        return self.evaluate(argument[0]).intersection(self.evaluate(argument[1]))
+        # RHS checks only those items matched by LHS
+        # returns result of RHS check: RHmatches(LHmatches(c))
+        #  return self.evaluate(argument[0]).intersection(self.evaluate(argument[1]))
+        l = self.evaluate(argument[0], candidates)
+        return l.intersection(self.evaluate(argument[1], l))
 
-    def evaluate_or(self, argument):
+    def evaluate_or(self, argument, candidates):
-        return self.evaluate(argument[0]).union(self.evaluate(argument[1]))
+        # RHS checks only those elements not matched by LHS
+        # returns LHS union RHS: LHmatches(c) + RHmatches(c-LHmatches(c))
+        #  return self.evaluate(argument[0]).union(self.evaluate(argument[1]))
+        l = self.evaluate(argument[0], candidates)
+        return l.union(self.evaluate(argument[1], candidates.difference(l)))
 
-    def evaluate_not(self, argument):
+    def evaluate_not(self, argument, candidates):
-        return self.universal_set().difference(self.evaluate(argument[0]))
+        # unary op checks only candidates. Result: list of items matching
+        # returns: c - matches(c)
+        #  return self.universal_set().difference(self.evaluate(argument[0]))
+        return candidates.difference(self.evaluate(argument[0], candidates))
 
-    def evaluate_parenthesis(self, argument):
+    def evaluate_parenthesis(self, argument, candidates):
-        return self.evaluate(argument[0])
+        return self.evaluate(argument[0], candidates)
 
-    def evaluate_token(self, argument):
+    def evaluate_token(self, argument, candidates):
         location = argument[0]
         query = argument[1]
         if location.lower() == 'search':
@@ -224,17 +239,27 @@ class SearchQueryParser(object):
                     raise ParseException(query, len(query), 'undefined saved search', self)
                 if self.recurse_level > 5:
                     self.searches_seen.add(query)
-                return self._parse(saved_searches().lookup(query))
+                return self._parse(saved_searches().lookup(query), candidates)
             except: # convert all exceptions (e.g., missing key) to a parse error
                 raise ParseException(query, len(query), 'undefined saved search', self)
-        return self.get_matches(location, query)
+        return self._get_matches(location, query, candidates)
 
-    def get_matches(self, location, query):
+    def _get_matches(self, location, query, candidates):
+        if self.optimize:
+            return self.get_matches(location, query, candidates=candidates)
+        else:
+            return self.get_matches(location, query)
+
+    def get_matches(self, location, query, candidates=None):
         '''
         Should return the set of matches for :param:'location` and :param:`query`.
 
+        The search must be performed over all entries is :param:`candidates` is
+        None otherwise only over the items in candidates.
+
         :param:`location` is one of the items in :member:`SearchQueryParser.DEFAULT_LOCATIONS`.
         :param:`query` is a string literal.
+        :param: None or a subset of the set returned by :meth:`universal_set`.
         '''
         return set([])
 
@@ -561,7 +586,7 @@ class Tester(SearchQueryParser):
     def universal_set(self):
         return self._universal_set
 
-    def get_matches(self, location, query):
+    def get_matches(self, location, query, candidates=None):
         location = location.lower()
         if location in self.fields.keys():
             getter = operator.itemgetter(self.fields[location])
@@ -573,8 +598,13 @@ class Tester(SearchQueryParser):
         if not query:
             return set([])
         query = query.lower()
-        return set(key for key, val in self.texts.items() \
+        if candidates:
-            if query and query in getattr(getter(val), 'lower', lambda : '')())
+            return set(key for key, val in self.texts.items() \
+                if key in candidates and query and query
+                        in getattr(getter(val), 'lower', lambda : '')())
+        else:
+            return set(key for key, val in self.texts.items() \
+                if query and query in getattr(getter(val), 'lower', lambda : '')())
 
 
 
@@ -592,6 +622,7 @@ class Tester(SearchQueryParser):
 
 
 def main(args=sys.argv):
+    print 'testing unoptimized'
     tester = Tester(['authors', 'author', 'series', 'formats', 'format',
         'publisher', 'rating', 'tags', 'tag', 'comments', 'comment', 'cover',
         'isbn', 'ondevice', 'pubdate', 'size', 'date', 'title', u'#read',
@@ -601,6 +632,16 @@ def main(args=sys.argv):
         print '>>>>>>>>>>>>>> Tests Failed <<<<<<<<<<<<<<<'
         return 1
 
+    print '\n\ntesting optimized'
+    tester = Tester(['authors', 'author', 'series', 'formats', 'format',
+        'publisher', 'rating', 'tags', 'tag', 'comments', 'comment', 'cover',
+        'isbn', 'ondevice', 'pubdate', 'size', 'date', 'title', u'#read',
+        'all', 'search'], test=True, optimize=True)
+    failed = tester.run_tests()
+    if tester._tests_failed or failed:
+        print '>>>>>>>>>>>>>> Tests Failed <<<<<<<<<<<<<<<'
+        return 1
+
     return 0
 
 if __name__ == '__main__':