KG updates

resources/default_tweaks.py

@@ -245,19 +245,6 @@ sony_collection_name_template='{value}{category:| (|)}'
 sony_collection_sorting_rules = []
 
 
-#: Create search terms to apply a query across several built-in search terms.
-# Syntax: {'new term':['existing term 1', 'term 2', ...], 'new':['old'...] ...}
-# Example: create the term 'myseries' that when used as myseries:foo would
-# search all of the search categories 'series', '#myseries', and '#myseries2':
-# grouped_search_terms={'myseries':['series','#myseries', '#myseries2']}
-# Example: two search terms 'a' and 'b' both that search 'tags' and '#mytags':
-# grouped_search_terms={'a':['tags','#mytags'], 'b':['tags','#mytags']}
-# Note: You cannot create a search term that is a duplicate of an existing term.
-# Such duplicates will be silently ignored. Also note that search terms ignore
-# case. 'MySearch' and 'mysearch' are the same term.
-grouped_search_terms = {}
-
-
 #: Control how tags are applied when copying books to another library
 # Set this to True to ensure that tags in 'Tags to add when adding
 # a book' are added when copying books to another library

resources/recipes/credit_slips.recipe

@@ -0,0 +1,35 @@
+#!/usr/bin/env  python
+__license__ = 'GPL 3'
+__copyright__ = 'zotzot'
+__docformat__ = 'restructuredtext en'
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+
+class CreditSlips(BasicNewsRecipe):
+    __license__ = 'GPL v3'
+    __author__ = 'zotzot'
+    language = 'en'
+    version = 1
+    title = u'Credit Slips.org'
+    publisher = u'Bankr-L'
+    category = u'Economic blog'
+    description = u'All things about credit.'
+    cover_url = 'http://bit.ly/hyZSTr'
+    oldest_article = 50
+    max_articles_per_feed = 100
+    use_embedded_content = True
+
+    feeds = [
+(u'Credit Slips', u'http://www.creditslips.org/creditslips/atom.xml')
+]
+    conversion_options = {
+'comments': description,
+'tags': category,
+'language': 'en',
+'publisher': publisher
+}
+    extra_css = '''
+                body{font-family:verdana,arial,helvetica,geneva,sans-serif;}
+                img {float: left; margin-right: 0.5em;}
+                '''

resources/recipes/epl_talk.recipe

@@ -0,0 +1,37 @@
+#!/usr/bin/env  python
+__license__ = 'GPL 3'
+__copyright__ = 'zotzot'
+__docformat__ = 'restructuredtext en'
+'''
+http://www.epltalk.com
+'''
+from calibre.web.feeds.news import BasicNewsRecipe
+
+
+class EPLTalkRecipe(BasicNewsRecipe):
+    __license__ = 'GPL v3'
+    __author__ = u'The Gaffer'
+    language = 'en'
+    version = 1
+
+    title = u'EPL Talk'
+    publisher = u'The Gaffer'
+    publication_type = 'Blog'
+    category = u'Soccer'
+    description = u'News and Analysis from the English Premier League'
+    cover_url = 'http://bit.ly/hJxZPu'
+
+    oldest_article = 45
+    max_articles_per_feed = 150
+    use_embedded_content = True
+    remove_javascript = True
+    encoding = 'utf8'
+
+    remove_tags_after = [dict(name='div', attrs={'class':'pd-rating'})]
+
+    feeds = [(u'EPL Talk', u'http://feeds.feedburner.com/EPLTalk')]
+
+    extra_css = '''
+                body{font-family:verdana,arial,helvetica,geneva,sans-serif;}
+                img {float: left; margin-right: 0.5em;}
+                '''

resources/recipes/fan_graphs.recipe

@@ -0,0 +1,39 @@
+#!/usr/bin/env  python
+__license__  = 'GPL v3'
+__copyright__ = '2011 zotzot'
+__docformat__ = 'PEP8'
+'''
+www.fangraphs.com
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class FanGraphs(BasicNewsRecipe):
+    title = u'FanGraphs'
+    oldest_article = 21
+    max_articles_per_feed = 100
+    no_stylesheets = True
+    #delay = 1
+    use_embedded_content = False
+    encoding = 'utf8'
+    publisher = 'Fangraphs'
+    category = 'Baseball'
+    language = 'en'
+    publication_type = 'Blog'
+
+    description = 'Baseball statistical analysis, graphs, and projections.'
+    __author__ = 'David Appelman'
+    cover_url = 'http://bit.ly/g0BTdQ'
+
+    feeds = [
+    (u'Fangraphs', u'http://feeds.feedburner.com/FanGraphs?format=xml'),
+    (u'Rotographs', u'http://www.wizardrss.com/feed/feeds.feedburner.com/RotoGraphs?format=xml'),
+    (u'Community', u'http://www.wizardrss.com/feed/www.fangraphs.com/community/?feed=rss2'),
+    (u'NotGraphs', u'http://www.wizardrss.com/feed/www.fangraphs.com/not/?feed=rss2')]
+
+    extra_css = '''
+                    h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;}
+                    h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;}
+                    p{font-family:Arial,Helvetica,sans-serif;font-size:small;}
+                    body{font-family:Helvetica,Arial,sans-serif;font-size:small;}
+'''

resources/recipes/nrc.nl.recipe

@@ -38,7 +38,7 @@ class Pagina12(BasicNewsRecipe):
     keep_only_tags = [dict(attrs={'class':'uitstekendekeus'})]
     remove_tags    = [
                         dict(name=['meta','base','link','object','embed'])
-                       ,dict(attrs={'class':['reclamespace','tags-and-sharing']})
+                       ,dict(attrs={'class':['reclamespace','tags-and-sharing','sharing-is-caring']})
                      ]
     remove_attributes=['lang']
 

resources/recipes/oregonian.recipe

@@ -0,0 +1,44 @@
+from __future__ import with_statement
+__license__ = 'GPL 3'
+__copyright__ = 'zotzot'
+__docformat__ = 'restructuredtext en'
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+
+class Oregonian(BasicNewsRecipe):
+    title = u'The Oregonian'
+    oldest_article = 2
+    max_articles_per_feed = 100
+    language = 'en'
+    __author__ = 'Zotzot'
+    description = 'Portland, Oregon local newspaper'
+    publisher = 'Advance Publications'
+    category = 'news, Portland'
+    cover_url = 'http://bit.ly/gUgxGd'
+    no_stylesheets = True
+    masthead_url = 'http://bit.ly/eocL70'
+    remove_tags = [dict(name='div', attrs={'class':['footer', 'content']})]
+    use_embedded_content = False
+    remove_tags_before = dict(id='article')
+    remove_tags_after = dict(id='article')
+    feeds = [
+#(u'Timbers', u'feed://blog.oregonlive.com/timbers_impact/atom.xml'),
+(u'News', u'http://blog.oregonlive.com/news_impact/atom.xml'),
+(u'Opinion', u'http://blog.oregonlive.com/opinion_impact/atom.xml'),
+(u'Living', u'http://blog.oregonlive.com/living_impact/atom.xml'),
+(u'Sports', u'http://blog.oregonlive.com/sports_impact/atom.xml'),
+(u'Business', u'http://blog.oregonlive.com/business_impact/atom.xml')]
+
+    extra_css = '''
+                    h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;}
+                    h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;}
+                    p{font-family:Arial,Helvetica,sans-serif;font-size:small;}
+                    body{font-family:Helvetica,Arial,sans-serif;font-size:small;}
+'''
+
+
+def get_article_url(self, article):
+        url = BasicNewsRecipe.get_article_url(self, article)
+        if '/video/' not in url:
+            return url

src/calibre/customize/builtins.py

@@ -8,6 +8,7 @@ from calibre.customize import FileTypePlugin, MetadataReaderPlugin, \
     MetadataWriterPlugin, PreferencesPlugin, InterfaceActionBase
 from calibre.constants import numeric_version
 from calibre.ebooks.metadata.archive import ArchiveExtract, get_cbz_metadata
+from calibre.ebooks.metadata.opf2 import metadata_to_opf
 from calibre.ebooks.oeb.base import OEB_IMAGES
 
 # To archive plugins {{{
@@ -134,7 +135,18 @@ class TXT2TXTZ(FileTypePlugin):
             import zipfile
             of = self.temporary_file('_plugin_txt2txtz.txtz')
             txtz = zipfile.ZipFile(of.name, 'w')
+            # Add selected TXT file to archive.
             txtz.write(path_to_ebook, os.path.basename(path_to_ebook), zipfile.ZIP_DEFLATED)
+            # metadata.opf
+            if os.path.exists(os.path.join(base_dir, 'metadata.opf')):
+                txtz.write(os.path.join(base_dir, 'metadata.opf'), 'metadata.opf', zipfile.ZIP_DEFLATED)
+            else:
+                from calibre.ebooks.metadata.txt import get_metadata
+                with open(path_to_ebook, 'rb') as ebf:
+                    mi = get_metadata(ebf)
+                opf = metadata_to_opf(mi)
+                txtz.writestr('metadata.opf', opf, zipfile.ZIP_DEFLATED)
+            # images
             for image in images:
                 txtz.write(os.path.join(base_dir, image), image)
             txtz.close()
@@ -1018,3 +1030,10 @@ plugins += [LookAndFeel, Behavior, Columns, Toolbar, Search, InputOptions,
         Email, Server, Plugins, Tweaks, Misc, TemplateFunctions]
 
 #}}}
+
+# New metadata download plugins {{{
+from calibre.ebooks.metadata.sources.google import GoogleBooks
+
+plugins += [GoogleBooks]
+
+# }}}

src/calibre/customize/ui.py

@@ -20,6 +20,7 @@ from calibre.ebooks.metadata.fetch import MetadataSource
 from calibre.utils.config import make_config_dir, Config, ConfigProxy, \
                                  plugin_dir, OptionParser, prefs
 from calibre.ebooks.epub.fix import ePubFixer
+from calibre.ebooks.metadata.sources.base import Source
 
 platform = 'linux'
 if iswindows:
@@ -493,6 +494,17 @@ def epub_fixers():
                     yield plugin
 # }}}
 
+# Metadata sources2 {{{
+def metadata_plugins(capabilities):
+    capabilities = frozenset(capabilities)
+    for plugin in _initialized_plugins:
+        if isinstance(plugin, Source) and \
+                plugin.capabilities.intersection(capabilities) and \
+                not is_disabled(plugin):
+            yield plugin
+
+# }}}
+
 # Initialize plugins {{{
 
 _initialized_plugins = []

src/calibre/devices/apple/driver.py

@@ -2536,29 +2536,6 @@ class ITUNES(DriverBase):
                     if DEBUG:
                         self.log.warning("   missing <metadata> block in OPF file")
                         self.log.info("   add timestamp: %s" % metadata.timestamp)
-                '''
-                ns_map = opf_tree.nsmap.keys()
-                for item in ns_map:
-                    ns = opf_tree.nsmap[item]
-                    md_el = opf_tree.find(".//{%s}metadata" % ns)
-                    if md_el is not None:
-                        ts = md_el.find('.//{%s}meta[@name="calibre:timestamp"]')
-                        if ts:
-                            timestamp = ts.get('content')
-                            old_ts = parse_date(timestamp)
-                            metadata.timestamp = datetime.datetime(old_ts.year, old_ts.month, old_ts.day, old_ts.hour,
-                                                       old_ts.minute, old_ts.second, old_ts.microsecond+1, old_ts.tzinfo)
-                        else:
-                            metadata.timestamp = now()
-                            if DEBUG:
-                                self.log.info("   add timestamp: %s" % metadata.timestamp)
-                        break
-                else:
-                    metadata.timestamp = now()
-                    if DEBUG:
-                        self.log.warning("   missing <metadata> block in OPF file")
-                        self.log.info("   add timestamp: %s" % metadata.timestamp)
-                '''
                 # Force the language declaration for iBooks 1.1
                 #metadata.language = get_lang().replace('_', '-')
 

src/calibre/devices/kindle/apnx.py

@@ -25,7 +25,7 @@ class APNXBuilder(object):
         with open(mobi_file_path, 'rb') as mf:
             ident = PdbHeaderReader(mf).identity()
         if ident != 'BOOKMOBI':
-            raise Exception(_('Not a valid MOBI file. Reports identity of %s' % ident))
+            raise Exception(_('Not a valid MOBI file. Reports identity of %s') % ident)
 
         # Get the pages depending on the chosen parser
         pages = []

src/calibre/ebooks/conversion/preprocess.py

@@ -568,11 +568,14 @@ class HTMLPreProcessor(object):
     def smarten_punctuation(self, html):
         from calibre.utils.smartypants import smartyPants
         from calibre.ebooks.chardet import substitute_entites
+        from calibre.ebooks.conversion.utils import HeuristicProcessor
+        preprocessor = HeuristicProcessor(self.extra_opts, self.log)
         from uuid import uuid4
         start = 'calibre-smartypants-'+str(uuid4())
         stop = 'calibre-smartypants-'+str(uuid4())
         html = html.replace('<!--', start)
         html = html.replace('-->', stop)
+        html = preprocessor.fix_nbsp_indents(html)
         html = smartyPants(html)
         html = html.replace(start, '<!--')
         html = html.replace(stop, '-->')

src/calibre/ebooks/metadata/sources/base.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
 
 __license__   = 'GPL v3'
 __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
@@ -8,6 +10,12 @@ __docformat__ = 'restructuredtext en'
 import re
 
 from calibre.customize import Plugin
+from calibre.utils.logging import ThreadSafeLog, FileStream
+
+def create_log(ostream=None):
+    log = ThreadSafeLog(level=ThreadSafeLog.DEBUG)
+    log.outputs = [FileStream(ostream)]
+    return log
 
 class Source(Plugin):
 
@@ -18,14 +26,47 @@ class Source(Plugin):
 
     result_of_identify_is_complete = True
 
-    def get_author_tokens(self, authors):
+    capabilities = frozenset()
-        'Take a list of authors and return a list of tokens useful for a '
+
-        'AND search query'
+    touched_fields = frozenset()
-        # Leave ' in there for Irish names
+
-        pat = re.compile(r'[-,:;+!@#$%^&*(){}.`~"\s\[\]/]')
+    # Utility functions {{{
-        for au in authors:
+    def get_author_tokens(self, authors, only_first_author=True):
-            for tok in au.split():
+        '''
-                yield pat.sub('', tok)
+        Take a list of authors and return a list of tokens useful for an
+        AND search query. This function tries to return tokens in
+        first name middle names last name order, by assuming that if a comma is
+        in the author name, the name is in lastname, other names form.
+        '''
+
+        if authors:
+            # Leave ' in there for Irish names
+            pat = re.compile(r'[-,:;+!@#$%^&*(){}.`~"\s\[\]/]')
+            if only_first_author:
+                authors = authors[:1]
+            for au in authors:
+                parts = au.split()
+                if ',' in au:
+                    # au probably in ln, fn form
+                    parts = parts[1:] + parts[:1]
+                for tok in parts:
+                    tok = pat.sub('', tok).strip()
+                    yield tok
+
+
+    def get_title_tokens(self, title):
+        '''
+        Take a title and return a list of tokens useful for an AND search query.
+        Excludes connectives and punctuation.
+        '''
+        if title:
+            pat = re.compile(r'''[-,:;+!@#$%^&*(){}.`~"'\s\[\]/]''')
+            title = pat.sub(' ', title)
+            tokens = title.split()
+            for token in tokens:
+                token = token.strip()
+                if token and token.lower() not in ('a', 'and', 'the'):
+                    yield token
 
     def split_jobs(self, jobs, num):
         'Split a list of jobs into at most num groups, as evenly as possible'
@@ -40,6 +81,10 @@ class Source(Plugin):
                 gr.append(job)
         return [g for g in groups if g]
 
+    # }}}
+
+    # Metadata API {{{
+
     def identify(self, log, result_queue, abort, title=None, authors=None, identifiers={}):
         '''
         Identify a book by its title/author/isbn/etc.
@@ -59,3 +104,5 @@ class Source(Plugin):
         '''
         return None
 
+    # }}}
+

src/calibre/ebooks/metadata/sources/google.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
 
 __license__   = 'GPL v3'
 __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
@@ -12,8 +14,9 @@ from threading import Thread
 
 from lxml import etree
 
-from calibre.ebooks.metadata.sources import Source
+from calibre.ebooks.metadata.sources.base import Source
 from calibre.ebooks.metadata.book.base import Metadata
+from calibre.ebooks.chardet import xml_to_unicode
 from calibre.utils.date import parse_date, utcnow
 from calibre import browser, as_unicode
 
@@ -38,7 +41,18 @@ subject        = XPath('descendant::dc:subject')
 description    = XPath('descendant::dc:description')
 language       = XPath('descendant::dc:language')
 
+def get_details(browser, url):
+    try:
+        raw = browser.open_novisit(url).read()
+    except Exception as e:
+        gc = getattr(e, 'getcode', lambda : -1)
+        if gc() != 403:
+            raise
+        # Google is throttling us, wait a little
+        time.sleep(2)
+        raw = browser.open_novisit(url).read()
 
+    return raw
 
 def to_metadata(browser, log, entry_):
 
@@ -65,8 +79,8 @@ def to_metadata(browser, log, entry_):
 
     mi = Metadata(title_, authors)
     try:
-        raw = browser.open_novisit(id_url).read()
+        raw = get_details(browser, id_url)
-        feed = etree.fromstring(raw)
+        feed = etree.fromstring(xml_to_unicode(raw, strip_encoding_pats=True)[0])
         extra = entry(feed)[0]
     except:
         log.exception('Failed to get additional details for', mi.title)
@@ -142,6 +156,11 @@ class Worker(Thread):
 class GoogleBooks(Source):
 
     name = 'Google Books'
+    description = _('Downloads metadata from Google Books')
+
+    capabilities = frozenset(['identify'])
+    touched_fields = frozenset(['title', 'authors', 'isbn', 'tags', 'pubdate',
+        'comments', 'publisher', 'author_sort']) # language currently disabled
 
     def create_query(self, log, title=None, authors=None, identifiers={},
             start_index=1):
@@ -153,11 +172,14 @@ class GoogleBooks(Source):
         elif title or authors:
             def build_term(prefix, parts):
                 return ' '.join('in'+prefix + ':' + x for x in parts)
-            if title is not None:
+            title_tokens = list(self.get_title_tokens(title))
-                q += build_term('title', title.split())
+            if title_tokens:
-            if authors:
+                q += build_term('title', title_tokens)
-                q += ('+' if q else '')+build_term('author',
+            author_tokens = self.get_author_tokens(authors,
-                        self.get_author_tokens(authors))
+                    only_first_author=True)
+            if author_tokens:
+                q += ('+' if q else '') + build_term('author',
+                        author_tokens)
 
         if isinstance(q, unicode):
             q = q.encode('utf-8')
@@ -182,7 +204,8 @@ class GoogleBooks(Source):
 
         try:
             parser = etree.XMLParser(recover=True, no_network=True)
-            feed = etree.fromstring(raw, parser=parser)
+            feed = etree.fromstring(xml_to_unicode(raw,
+                strip_encoding_pats=True)[0], parser=parser)
             entries = entry(feed)
         except Exception, e:
             log.exception('Failed to parse identify results')
@@ -191,25 +214,33 @@ class GoogleBooks(Source):
 
         groups = self.split_jobs(entries, 5) # At most 5 threads
         if not groups:
-            return
+            return None
         workers = [Worker(log, entries, abort, result_queue) for entries in
                 groups]
 
         if abort.is_set():
-            return
+            return None
 
         for worker in workers: worker.start()
 
         has_alive_worker = True
         while has_alive_worker and not abort.is_set():
+            time.sleep(0.1)
             has_alive_worker = False
             for worker in workers:
                 if worker.is_alive():
                     has_alive_worker = True
-            time.sleep(0.1)
 
         return None
 
-
+if __name__ == '__main__':
-
+    # To run these test use: calibre-debug -e src/calibre/ebooks/metadata/sources/google.py
-
+    from calibre.ebooks.metadata.sources.test import (test_identify_plugin,
+            isbn_test)
+    test_identify_plugin(GoogleBooks.name,
+        [
+            (
+                {'title': 'Great Expectations', 'authors':['Charles Dickens']},
+                [isbn_test('9781607541592')]
+            ),
+    ])

src/calibre/ebooks/metadata/sources/test.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import os, tempfile
+from Queue import Queue, Empty
+from threading import Event
+
+
+from calibre.customize.ui import metadata_plugins
+from calibre import prints
+from calibre.ebooks.metadata import check_isbn
+from calibre.ebooks.metadata.sources.base import create_log
+
+def isbn_test(isbn):
+    isbn_ = check_isbn(isbn)
+
+    def test(mi):
+        misbn = check_isbn(mi.isbn)
+        return misbn and misbn == isbn_
+
+    return test
+
+def test_identify_plugin(name, tests):
+    '''
+    :param name: Plugin name
+    :param tests: List of 2-tuples. Each two tuple is of the form (args,
+                  test_funcs). args is a dict of keyword arguments to pass to
+                  the identify method. test_funcs are callables that accept a
+                  Metadata object and return True iff the object passes the
+                  test.
+    '''
+    plugin = None
+    for x in metadata_plugins(['identify']):
+        if x.name == name:
+            plugin = x
+            break
+    prints('Testing the identify function of', plugin.name)
+
+    tdir = tempfile.gettempdir()
+    lf = os.path.join(tdir, plugin.name.replace(' ', '')+'_identify_test.txt')
+    log = create_log(open(lf, 'wb'))
+    abort = Event()
+    prints('Log saved to', lf)
+
+    for kwargs, test_funcs in tests:
+        prints('Running test with:', kwargs)
+        rq = Queue()
+        args = (log, rq, abort)
+        err = plugin.identify(*args, **kwargs)
+        if err is not None:
+            prints('identify returned an error for args', args)
+            prints(err)
+            break
+
+        results = []
+        while True:
+            try:
+                results.append(rq.get_nowait())
+            except Empty:
+                break
+
+        prints('Found', len(results), 'matches:')
+
+        for mi in results:
+            prints(mi)
+            prints('\n\n')
+
+        match_found = None
+        for mi in results:
+            test_failed = False
+            for tfunc in test_funcs:
+                if not tfunc(mi):
+                    test_failed = True
+                    break
+            if not test_failed:
+                match_found = mi
+                break
+
+        if match_found is None:
+            prints('ERROR: No results that passed all tests were found')
+            prints('Log saved to', lf)
+            raise SystemExit(1)
+
+    prints('Log saved to', lf)
+

src/calibre/gui2/library/views.py

@@ -551,8 +551,10 @@ class BooksView(QTableView): # {{{
         return mods & Qt.ControlModifier or mods & Qt.ShiftModifier
 
     def mousePressEvent(self, event):
-        if event.button() == Qt.LeftButton and not self.event_has_mods():
+        ep = event.pos()
-            self.drag_start_pos = event.pos()
+        if self.indexAt(ep) in self.selectionModel().selectedIndexes() and \
+                event.button() == Qt.LeftButton and not self.event_has_mods():
+            self.drag_start_pos = ep
         return QTableView.mousePressEvent(self, event)
 
     def mouseMoveEvent(self, event):

src/calibre/gui2/preferences/search.py

@@ -10,13 +10,15 @@ from PyQt4.Qt import QApplication
 from calibre.gui2.preferences import ConfigWidgetBase, test_widget, \
         CommaSeparatedList
 from calibre.gui2.preferences.search_ui import Ui_Form
-from calibre.gui2 import config
+from calibre.gui2 import config, error_dialog
 from calibre.utils.config import prefs
 
 class ConfigWidget(ConfigWidgetBase, Ui_Form):
 
     def genesis(self, gui):
         self.gui = gui
+        db = gui.library_view.model().db
+        self.db = db
 
         r = self.register
 
@@ -24,11 +26,153 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         r('highlight_search_matches', config)
         r('limit_search_columns', prefs)
         r('limit_search_columns_to', prefs, setting=CommaSeparatedList)
-        fl = gui.library_view.model().db.field_metadata.get_search_terms()
+        fl = db.field_metadata.get_search_terms()
         self.opt_limit_search_columns_to.update_items_cache(fl)
         self.clear_history_button.clicked.connect(self.clear_histories)
 
+        self.gst_explanation.setText('<p>' + _(
+    "<b>Grouped search terms</b> are search names that permit a query to automatically "
+    "search across more than one column. For example, if you create a grouped "
+    "search term <code>allseries</code> with the value "
+    "<code>series, #myseries, #myseries2</code>, then "
+    "the query <code>allseries:adhoc</code> will find 'adhoc' in any of the "
+    "columns <code>series</code>, <code>#myseries</code>, and "
+    "<code>#myseries2</code>.<p> Enter the name of the "
+    "grouped search term in the drop-down box, enter the list of columns "
+    "to search in the value box, then push the Save button. "
+    "<p>Note: Search terms are forced to lower case; <code>MySearch</code> "
+    "and <code>mysearch</code> are the same term."
+    "<p>You can have your grouped search term show up as user categories in "
+    " the Tag Browser. Just add the grouped search term names to the Make user "
+    "categories from box. You can add multiple terms separated by commas. "
+    "The new user category will be automatically "
+    "populated with all the items in the categories included in the grouped "
+    "search term. <p>Automatic user categories permit you to see easily "
+    "all the category items that "
+    "are in the columns contained in the grouped search term. Using the above "
+    "<code>allseries</code> example, the automatically-generated user category "
+    "will contain all the series mentioned in <code>series</code>, "
+    "<code>#myseries</code>, and <code>#myseries2</code>. This "
+    "can be useful to check for duplicates, to find which column contains "
+    "a particular item, or to have hierarchical categories (categories "
+    "that contain categories)."))
+        self.gst = db.prefs.get('grouped_search_terms', {})
+        self.orig_gst_keys = self.gst.keys()
+
+        fl = []
+        for f in db.all_field_keys():
+            fm = db.metadata_for_field(f)
+            if not fm['search_terms']:
+                continue
+            if not fm['is_category']:
+                continue
+            fl.append(f)
+        self.gst_value.update_items_cache(fl)
+        self.fill_gst_box(select=None)
+
+        self.gst_delete_button.setEnabled(False)
+        self.gst_save_button.setEnabled(False)
+        self.gst_names.currentIndexChanged[int].connect(self.gst_index_changed)
+        self.gst_names.editTextChanged.connect(self.gst_text_changed)
+        self.gst_value.textChanged.connect(self.gst_text_changed)
+        self.gst_save_button.clicked.connect(self.gst_save_clicked)
+        self.gst_delete_button.clicked.connect(self.gst_delete_clicked)
+        self.gst_changed = False
+
+        if db.prefs.get('grouped_search_make_user_categories', None) is None:
+            db.prefs.set('grouped_search_make_user_categories', [])
+        r('grouped_search_make_user_categories', db.prefs, setting=CommaSeparatedList)
+        self.muc_changed = False
+        self.opt_grouped_search_make_user_categories.editingFinished.connect(
+                                                        self.muc_box_changed)
+
+    def muc_box_changed(self):
+        self.muc_changed = True
+
+    def gst_save_clicked(self):
+        idx = self.gst_names.currentIndex()
+        name = icu_lower(unicode(self.gst_names.currentText()))
+        if not name:
+            return error_dialog(self.gui, _('Grouped Search Terms'),
+                                _('The search term cannot be blank'),
+                                show=True)
+        if idx != 0:
+            orig_name = unicode(self.gst_names.itemData(idx).toString())
+        else:
+            orig_name = ''
+        if name != orig_name:
+            if name in self.db.field_metadata.get_search_terms() and \
+                    name not in self.orig_gst_keys:
+                return error_dialog(self.gui, _('Grouped Search Terms'),
+                    _('That name is already used for a column or grouped search term'),
+                    show=True)
+            if name in [icu_lower(p) for p in self.db.prefs.get('user_categories', {})]:
+                return error_dialog(self.gui, _('Grouped Search Terms'),
+                    _('That name is already used for user category'),
+                    show=True)
+
+        val = [v.strip() for v in unicode(self.gst_value.text()).split(',') if v.strip()]
+        if not val:
+            return error_dialog(self.gui, _('Grouped Search Terms'),
+                _('The value box cannot be empty'), show=True)
+
+        if orig_name and name != orig_name:
+            del self.gst[orig_name]
+        self.gst_changed = True
+        self.gst[name] = val
+        self.fill_gst_box(select=name)
+        self.changed_signal.emit()
+
+    def gst_delete_clicked(self):
+        if self.gst_names.currentIndex() == 0:
+            return error_dialog(self.gui, _('Grouped Search Terms'),
+                _('The empty grouped search term cannot be deleted'), show=True)
+        name = unicode(self.gst_names.currentText())
+        if name in self.gst:
+            del self.gst[name]
+            self.fill_gst_box(select='')
+            self.changed_signal.emit()
+            self.gst_changed = True
+
+    def fill_gst_box(self, select=None):
+        terms = sorted(self.gst.keys())
+        self.opt_grouped_search_make_user_categories.update_items_cache(terms)
+        self.gst_names.blockSignals(True)
+        self.gst_names.clear()
+        self.gst_names.addItem('', '')
+        for t in terms:
+            self.gst_names.addItem(t, t)
+        self.gst_names.blockSignals(False)
+        if select is not None:
+            if select == '':
+                self.gst_index_changed(0)
+            elif select in terms:
+                self.gst_names.setCurrentIndex(self.gst_names.findText(select))
+
+    def gst_text_changed(self):
+        self.gst_delete_button.setEnabled(False)
+        self.gst_save_button.setEnabled(True)
+
+    def gst_index_changed(self, idx):
+        self.gst_delete_button.setEnabled(idx != 0)
+        self.gst_save_button.setEnabled(False)
+        self.gst_value.blockSignals(True)
+        if idx == 0:
+            self.gst_value.setText('')
+        else:
+            name = unicode(self.gst_names.itemData(idx).toString())
+            self.gst_value.setText(','.join(self.gst[name]))
+        self.gst_value.blockSignals(False)
+
+    def commit(self):
+        if self.gst_changed:
+            self.db.prefs.set('grouped_search_terms', self.gst)
+            self.db.field_metadata.add_grouped_search_terms(self.gst)
+        return ConfigWidgetBase.commit(self)
+
     def refresh_gui(self, gui):
+        if self.muc_changed:
+            gui.tags_view.set_new_model()
         gui.search.search_as_you_type(config['search_as_you_type'])
         gui.library_view.model().set_highlight_only(config['highlight_search_matches'])
         gui.search.do_search()

src/calibre/gui2/preferences/search.ui

@@ -7,7 +7,7 @@
     <x>0</x>
     <y>0</y>
     <width>670</width>
-    <height>392</height>
+    <height>556</height>
    </rect>
   </property>
   <property name="windowTitle">
@@ -77,19 +77,6 @@
      </layout>
     </widget>
    </item>
-   <item row="4" column="0">
-    <spacer name="verticalSpacer">
-     <property name="orientation">
-      <enum>Qt::Vertical</enum>
-     </property>
-     <property name="sizeHint" stdset="0">
-      <size>
-       <width>0</width>
-       <height>0</height>
-      </size>
-     </property>
-    </spacer>
-   </item>
    <item row="3" column="0">
     <widget class="QPushButton" name="clear_history_button">
      <property name="toolTip">
@@ -100,6 +87,120 @@
      </property>
     </widget>
    </item>
+   <item row="4" column="0">
+    <widget class="QGroupBox" name="groupBox_2">
+     <property name="title">
+      <string>Grouped Search Terms</string>
+     </property>
+     <layout class="QGridLayout" name="gridLayout_3">
+      <item row="0" column="0">
+       <layout class="QHBoxLayout" name="l12">
+        <item>
+         <widget class="QLabel" name="la10">
+          <property name="text">
+           <string>&amp;Names:</string>
+          </property>
+          <property name="buddy">
+           <cstring>gst_names</cstring>
+          </property>
+         </widget>
+        </item>
+        <item>
+         <widget class="QComboBox" name="gst_names">
+          <property name="editable">
+           <bool>true</bool>
+          </property>
+          <property name="minimumContentsLength">
+           <number>10</number>
+          </property>
+          <property name="toolTip">
+           <string>Contains the names of the currently-defined group search terms.
+Create a new name by entering it into the empty box, then
+pressing Save. Rename a search term by selecting it then
+changing the name and pressing Save. Change the value of
+a search term by changing the value box then pressing Save.</string>
+          </property>
+         </widget>
+        </item>
+        <item>
+         <widget class="QToolButton" name="gst_delete_button">
+          <property name="toolTip">
+           <string>Delete the current search term</string>
+          </property>
+          <property name="text">
+           <string>...</string>
+          </property>
+          <property name="icon">
+           <iconset resource="../../../../resources/images.qrc">
+            <normaloff>:/images/trash.png</normaloff>:/images/trash.png</iconset>
+          </property>
+         </widget>
+        </item>
+        <item>
+         <widget class="MultiCompleteLineEdit" name="gst_value"/>
+        </item>
+        <item>
+         <widget class="QToolButton" name="gst_save_button">
+          <property name="toolTip">
+           <string>Save the current search term. You can rename a search term by
+changing the name then pressing Save. You can change the value
+of a search term by changing the value box then pressing Save.</string>
+          </property>
+          <property name="text">
+           <string>&amp;Save</string>
+          </property>
+         </widget>
+        </item>
+       </layout>
+      </item>
+      <item row="0" column="1" rowspan="3">
+       <widget class="QTextBrowser" name="gst_explanation">
+        <property name="sizePolicy">
+         <sizepolicy hsizetype="Expanding" vsizetype="Expanding">
+          <horstretch>0</horstretch>
+          <verstretch>100</verstretch>
+         </sizepolicy>
+        </property>
+       </widget>
+      </item>
+      <item row="1" column="0">
+       <layout class="QHBoxLayout">
+        <item>
+         <widget class="QLabel" name="l11">
+          <property name="text">
+           <string>Make &amp;user categories from:</string>
+          </property>
+          <property name="buddy">
+           <cstring>opt_grouped_search_make_user_categories</cstring>
+          </property>
+         </widget>
+        </item>
+        <item>
+         <widget class="MultiCompleteLineEdit" name="opt_grouped_search_make_user_categories">
+          <property name="toolTip">
+           <string>Enter the names of any grouped search terms you wish
+to be shown as user categories</string>
+          </property>
+         </widget>
+        </item>
+       </layout>
+      </item>
+      <item row="2" column="0">
+       <spacer name="verticalSpacer">
+        <property name="orientation">
+         <enum>Qt::Vertical</enum>
+        </property>
+        <property name="sizeHint" stdset="0">
+         <size>
+          <width>20</width>
+          <height>40</height>
+         </size>
+        </property>
+       </spacer>
+      </item>
+     </layout>
+    </widget>
+   </item>
   </layout>
  </widget>
  <customwidgets>
@@ -109,6 +210,8 @@
    <header>calibre/gui2/complete.h</header>
   </customwidget>
  </customwidgets>
- <resources/>
+ <resources>
+  <include location="../../../../resources/images.qrc"/>
+ </resources>
  <connections/>
 </ui>

src/calibre/gui2/tag_view.py

@@ -466,10 +466,7 @@ class TagTreeItem(object): # {{{
             icon_map[0] = data.icon
             self.tag, self.icon_state_map = data, list(map(QVariant, icon_map))
         if tooltip:
-            if tooltip.endswith(':'):
+            self.tooltip = tooltip + ' '
-                self.tooltip = tooltip + ' '
-            else:
-                self.tooltip = tooltip + ': '
         else:
             self.tooltip = ''
 
@@ -589,11 +586,17 @@ class TagsModel(QAbstractItemModel): # {{{
 
         # get_node_tree cannot return None here, because row_map is empty
         data = self.get_node_tree(config['sort_tags_by'])
+        gst = db.prefs.get('grouped_search_terms', {})
         self.root_item = TagTreeItem()
         for i, r in enumerate(self.row_map):
             if self.hidden_categories and self.categories[i] in self.hidden_categories:
                 continue
-            tt = _(u'The lookup/search name is "{0}"').format(r)
+            if r.startswith('@') and r[1:] in gst:
+                tt = _(u'The grouped search term name is "{0}"').format(r[1:])
+            elif r == 'news':
+                tt = ''
+            else:
+                tt = _(u'The lookup/search name is "{0}"').format(r)
             TagTreeItem(parent=self.root_item,
                     data=self.categories[i],
                     category_icon=self.category_icon_map[r],
@@ -735,6 +738,14 @@ class TagsModel(QAbstractItemModel): # {{{
         self.row_map = []
         self.categories = []
 
+        # Get the categories
+        if self.search_restriction:
+            data = self.db.get_categories(sort=sort,
+                        icon_map=self.category_icon_map,
+                        ids=self.db.search('', return_matches=True))
+        else:
+            data = self.db.get_categories(sort=sort, icon_map=self.category_icon_map)
+
         # Reconstruct the user categories, putting them into metadata
         self.db.field_metadata.remove_dynamic_categories()
         tb_cats = self.db.field_metadata
@@ -746,17 +757,16 @@ class TagsModel(QAbstractItemModel): # {{{
             except ValueError:
                 import traceback
                 traceback.print_exc()
+
+        for cat in sorted(self.db.prefs.get('grouped_search_terms', {}),
+                          key=sort_key):
+            if (u'@' + cat) in data:
+                tb_cats.add_user_category(label=u'@' + cat, name=cat)
+        self.db.data.change_search_locations(self.db.field_metadata.get_search_terms())
+
         if len(saved_searches().names()):
             tb_cats.add_search_category(label='search', name=_('Searches'))
 
-        # Now get the categories
-        if self.search_restriction:
-            data = self.db.get_categories(sort=sort,
-                        icon_map=self.category_icon_map,
-                        ids=self.db.search('', return_matches=True))
-        else:
-            data = self.db.get_categories(sort=sort, icon_map=self.category_icon_map)
-
         if self.filter_categories_by:
             for category in data.keys():
                 data[category] = [t for t in data[category]
@@ -767,6 +777,7 @@ class TagsModel(QAbstractItemModel): # {{{
             if category in data: # The search category can come and go
                 self.row_map.append(category)
                 self.categories.append(tb_categories[category]['name'])
+
         if len(old_row_map) != 0 and len(old_row_map) != len(self.row_map):
             # A category has been added or removed. We must force a rebuild of
             # the model
@@ -822,6 +833,7 @@ class TagsModel(QAbstractItemModel): # {{{
                                 not self.db.field_metadata[r]['is_custom'] and \
                                 not self.db.field_metadata[r]['kind'] == 'user' \
                             else False
+            tt = r if self.db.field_metadata[r]['kind'] == 'user' else None
             for idx,tag in enumerate(data[r]):
                 if clear_rating:
                     tag.avg_rating = None
@@ -861,10 +873,10 @@ class TagsModel(QAbstractItemModel): # {{{
                                      category_icon = category_node.icon,
                                      tooltip = None,
                                      category_key=category_node.category_key)
-                    t = TagTreeItem(parent=sub_cat, data=tag, tooltip=r,
+                    t = TagTreeItem(parent=sub_cat, data=tag, tooltip=tt,
                                         icon_map=self.icon_state_map)
                 else:
-                    t = TagTreeItem(parent=category, data=tag, tooltip=r,
+                    t = TagTreeItem(parent=category, data=tag, tooltip=tt,
                                     icon_map=self.icon_state_map)
             self.endInsertRows()
         return True

src/calibre/library/caches.py

@@ -433,6 +433,10 @@ class ResultCache(SearchQueryParser): # {{{
         if len(candidates) == 0:
             return matches
 
+        if len(location) > 2 and location.startswith('@') and \
+                    location[1:] in self.db_prefs['grouped_search_terms']:
+            location = location[1:]
+
         if query and query.strip():
             # get metadata key associated with the search term. Eliminates
             # dealing with plurals and other aliases
@@ -440,9 +444,16 @@ class ResultCache(SearchQueryParser): # {{{
             # grouped search terms
             if isinstance(location, list):
                 if allow_recursion:
+                    if query.lower() == 'false':
+                        invert = True
+                        query = 'true'
+                    else:
+                        invert = False
                     for loc in location:
                         matches |= self.get_matches(loc, query,
                                 candidates=candidates, allow_recursion=False)
+                    if invert:
+                        matches = self.universal_set() - matches
                     return matches
                 raise ParseException(query, len(query), 'Recursive query group detected', self)
 

src/calibre/library/database2.py

@@ -188,6 +188,17 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         migrate_preference('saved_searches', {})
         set_saved_searches(self, 'saved_searches')
 
+        # migrate grouped_search_terms
+        if self.prefs.get('grouped_search_terms', None) is None:
+            try:
+                ogst = tweaks.get('grouped_search_terms', {})
+                ngst = {}
+                for t in ogst:
+                    ngst[icu_lower(t)] = ogst[t]
+                self.prefs.set('grouped_search_terms', ngst)
+            except:
+                pass
+
         # Rename any user categories with names that differ only in case
         user_cats = self.prefs.get('user_categories', [])
         catmap = {}
@@ -349,12 +360,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if len(saved_searches().names()):
             tb_cats.add_search_category(label='search', name=_('Searches'))
 
-        gst = tweaks['grouped_search_terms']
+        self.field_metadata.add_grouped_search_terms(
-        for t in gst:
+                                    self.prefs.get('grouped_search_terms', {}))
-            try:
-                self.field_metadata._add_search_terms_to_map(gst[t], [t])
-            except ValueError:
-                traceback.print_exc()
 
         self.book_on_device_func = None
         self.data    = ResultCache(self.FIELD_MAP, self.field_metadata, db_prefs=self.prefs)
@@ -1293,7 +1300,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             # icon_map is not None if get_categories is to store an icon and
             # possibly a tooltip in the tag structure.
             icon = None
-            tooltip = ''
+            tooltip = '(' + category + ')'
             label = tb_cats.key_to_label(category)
             if icon_map:
                 if not tb_cats.is_custom_field(category):
@@ -1379,7 +1386,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             categories['formats'].sort(key = lambda x:x.name)
 
         #### Now do the user-defined categories. ####
-        user_categories = self.prefs['user_categories']
+        user_categories = dict.copy(self.prefs['user_categories'])
 
         # We want to use same node in the user category as in the source
         # category. To do that, we need to find the original Tag node. There is
@@ -1390,6 +1397,17 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         for c in categories.keys():
             taglist[c] = dict(map(lambda t:(t.name, t), categories[c]))
 
+        muc = self.prefs.get('grouped_search_make_user_categories', [])
+        gst = self.prefs.get('grouped_search_terms', {})
+        for c in gst:
+            if c not in muc:
+                continue
+            user_categories[c] = []
+            for sc in gst[c]:
+                if sc in categories.keys():
+                    for t in categories[sc]:
+                        user_categories[c].append([t.name, sc, 0])
+
         for user_cat in sorted(user_categories.keys(), key=sort_key):
             items = []
             for (name,label,ign) in user_categories[user_cat]:

src/calibre/library/field_metadata.py

@@ -3,7 +3,7 @@ Created on 25 May 2010
 
 @author: charles
 '''
-import copy
+import copy, traceback
 
 from calibre.utils.ordered_dict import OrderedDict
 from calibre.utils.config import tweaks
@@ -488,6 +488,20 @@ class FieldMetadata(dict):
                         del self._search_term_map[k]
                 del self._tb_cats[key]
 
+    def _remove_grouped_search_terms(self):
+        to_remove = [v for v in self._search_term_map
+                        if isinstance(self._search_term_map[v], list)]
+        for v in to_remove:
+            del self._search_term_map[v]
+
+    def add_grouped_search_terms(self, gst):
+        self._remove_grouped_search_terms()
+        for t in gst:
+            try:
+                self._add_search_terms_to_map(gst[t], [t])
+            except ValueError:
+                traceback.print_exc()
+
     def cc_series_index_column_for(self, key):
         return self._tb_cats[key]['rec_index'] + 1
 

src/calibre/utils/logging.py

@@ -10,17 +10,19 @@ INFO  = 1
 WARN  = 2
 ERROR = 3
 
-import sys, traceback
+import sys, traceback, cStringIO
 from functools import partial
-
+from threading import RLock
 
 
 
 class Stream(object):
 
-    def __init__(self, stream):
+    def __init__(self, stream=None):
         from calibre import prints
         self._prints = partial(prints, safe_encode=True)
+        if stream is None:
+            stream = cStringIO.StringIO()
         self.stream = stream
 
     def flush(self):
@@ -50,6 +52,15 @@ class ANSIStream(Stream):
     def flush(self):
         self.stream.flush()
 
+class FileStream(Stream):
+
+    def __init__(self, stream=None):
+        Stream.__init__(self, stream)
+
+    def prints(self, level, *args, **kwargs):
+        kwargs['file'] = self.stream
+        self._prints(*args, **kwargs)
+
 class HTMLStream(Stream):
 
     def __init__(self, stream=sys.stdout):
@@ -103,4 +114,14 @@ class Log(object):
     def __call__(self, *args, **kwargs):
         self.prints(INFO, *args, **kwargs)
 
+class ThreadSafeLog(Log):
+
+    def __init__(self, level=Log.INFO):
+        Log.__init__(self, level=level)
+        self._lock = RLock()
+
+    def prints(self, *args, **kwargs):
+        with self._lock:
+            Log.prints(self, *args, **kwargs)
+
 default_log = Log()