Merge

2025-07-09 03:04:10 -04:00 · 2011-02-08 23:05:01 +01:00 · 2011-02-08 23:05:01 +01:00 · 39d35efcc9
commit 39d35efcc9
parent 6a4f647668 e8033decd5
70 changed files with 1316 additions and 564 deletions
--- a/Changelog.yaml
+++ b/Changelog.yaml
@ -55,7 +55,7 @@
    - title: "Add search to the plugin preferences dialog"
  
  bug fixes:
-    - title: "Fix a bug that could cause fiels to be lost when changing metadata on east asian windows installs if the title and/or author is very long."
+    - title: "Fix a bug that could cause files to be lost when changing metadata on east asian windows installs if the title and/or author is very long."
      tickets: [8620]

    - title: "Tag browser: Fix searching with items in a user category not owrking if the main category is hidden"
@ -88,7 +88,7 @@
    - title: "Do not discard the result of a conversion if the user opens the edit metadata dialog while the conversion is running"
      tickets: [8672]

-    - title: "CHM Input: When the chm file lacks a hhc, lookf for index.html instead"
+    - title: "CHM Input: When the chm file lacks a hhc, look for index.html instead"
      tickets: [8688]

    - title: "EPUB Input: Filter some invalid media types from the spine"
--- a/resources/default_tweaks.py
+++ b/resources/default_tweaks.py
@ -39,7 +39,7 @@ completer_append_separator = False

 # The algorithm used to copy author to author_sort
 # Possible values are:
-#  invert: use "fn ln" -> "ln, fn" (the original algorithm)
+#  invert: use "fn ln" -> "ln, fn" (the default algorithm)
 #  copy  : copy author to author_sort without modification
 #  comma : use 'copy' if there is a ',' in the name, otherwise use 'invert'
 #  nocomma : "fn ln" -> "ln fn" (without the comma)
--- a/resources/images/news/kopalniawiedzy.png
+++ b/resources/images/news/kopalniawiedzy.png
--- a/resources/images/news/korespondent.png
+++ b/resources/images/news/korespondent.png
--- a/resources/jacket/stylesheet.css
+++ b/resources/jacket/stylesheet.css
@ -113,8 +113,8 @@ table.cbj_header tr.cbj_series {
 	/* display:none; */
 	}

-table.cbj_header tr.cbj_pubdate {
-	/* Uncomment the next line to remove 'Published' from banner section */
+table.cbj_header tr.cbj_pubdata {
+	/* Uncomment the next line to remove 'Published (year of publication)' from banner section */
 	/* display:none; */
 	}

--- a/resources/recipes/cinebel_be.recipe
+++ b/resources/recipes/cinebel_be.recipe
@ -1,7 +1,7 @@
 #!/usr/bin/env  python

 __license__   = 'GPL v3'
-__copyright__ = '2008, Lionel Bergeret <lbergeret at gmail.com>'
+__copyright__ = '2008-2011, Lionel Bergeret <lbergeret at gmail.com>'
 '''
 cinebel.be
 '''
@ -14,14 +14,14 @@ class Cinebel(BasicNewsRecipe):
    description           = u'Cinema news from Belgium in French'
    publisher             = u'cinebel.be'
    category              = 'news, cinema, movie, Belgium'
-    oldest_article        = 3
-    encoding              = 'utf8'
-    language              = 'fr_BE'
+    oldest_article        = 15
+    language              = 'fr'

    max_articles_per_feed = 20
    no_stylesheets        = True
    use_embedded_content  = False
    timefmt               = ' [%d %b %Y]'
+    filterDuplicates      = True

    keep_only_tags = [
 	 dict(name = 'span', attrs = {'class': 'movieMainTitle'})
@ -35,6 +35,13 @@ class Cinebel(BasicNewsRecipe):
        ,(u'Top 10'                    , u'http://www.cinebel.be/Servlets/RssServlet?languageCode=fr&rssType=2' )
    ]

+    def preprocess_html(self, soup):
+        for alink in soup.findAll('a'):
+            if alink.has_key('href'):
+               tstr = "Site officiel: " + alink['href']
+               alink.replaceWith(tstr)
+        return soup
+
    def get_cover_url(self):
        cover_url = 'http://www.cinebel.be/portal/resources/common/logo_index.gif'
        return cover_url
--- a/resources/recipes/dhnet_be.recipe
+++ b/resources/recipes/dhnet_be.recipe
@ -1,7 +1,7 @@
 #!/usr/bin/env  python

 __license__   = 'GPL v3'
-__copyright__ = '2008, Lionel Bergeret <lbergeret at gmail.com>'
+__copyright__ = '2008-2011, Lionel Bergeret <lbergeret at gmail.com>'
 '''
 dhnet.be
 '''
@ -16,7 +16,8 @@ class DHNetBe(BasicNewsRecipe):
    publisher             = u'dhnet.be'
    category              = 'news, Belgium'
    oldest_article        = 3
-    language              = 'fr_BE'
+    language              = 'fr'
+    masthead_url          = 'http://www.dhnet.be/images/homepage_logo_dh.gif'

    max_articles_per_feed = 20
    no_stylesheets        = True
@ -34,6 +35,13 @@ class DHNetBe(BasicNewsRecipe):
        ,(u'La Une Info'    , u'http://www.dhnet.be/rss/dhinfos/' )
    ]

+    def preprocess_html(self, soup):
+        for alink in soup.findAll('a'):
+            if alink.string is not None:
+               tstr = alink.string
+               alink.replaceWith(tstr)
+        return soup
+
    def get_cover_url(self):
        cover_url = strftime('http://pdf-online.dhnet.be/pdfonline/image/%Y%m%d/dh_%Y%m%d_nam_infoge_001.pdf.L.jpg')
        return cover_url
--- a/resources/recipes/europa_press.recipe
+++ b/resources/recipes/europa_press.recipe
@ -0,0 +1,55 @@
+__license__   = 'GPL v3'
+__author__    = 'Luis Hernandez'
+__copyright__ = 'Luis Hernandez<tolyluis@gmail.com>'
+__version__     = 'v1.0'
+__date__        = '30 January 2011'
+
+'''
+www.europapress.es
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1294946868(BasicNewsRecipe):
+
+    title          = u'Europa Press'
+    author            = 'Luis Hernandez'
+    description           = 'spanish news agency'
+
+    oldest_article = 2
+    max_articles_per_feed = 100
+
+    remove_javascript = True
+    no_stylesheets        = True
+    use_embedded_content  = False
+
+    language              = 'es'
+    timefmt        = '[%a, %d %b, %Y]'
+
+    remove_tags_before = dict(name='div' , attrs={'class':['nivel1 bg_3col']})
+    remove_tags_after  = dict(name='div' , attrs={'id':['ImprimirEnviarNoticia']})
+
+    remove_tags = [
+                             dict(name='ul', attrs={'id':['entidadesNoticia','MenuSecciones']})
+                            ,dict(name='div', attrs={'id':['ImprimirEnviarNoticia','PublicidadSuperior','CabeceraDerecha','Comentarios','comentarios full fbConnectAPI','ComentarEstaNoticia','ctl00_Superior_Main_MasEnChance_cajamasnoticias','gl_chn','videos_portada_derecha','galeria_portada_central','galeria_portada_central_boxes']})
+                            ,dict(name='div', attrs={'class':['infoRelacionada','col_1','buscador','caja doblecolumna strong','CHANCE_EP_Encuesta_frontal text','seccionportada col_0','seccion header','text','pie caption_over']})
+                            ,dict(name='a', attrs={'class':['buscadorLabel']})
+                            ,dict(name='span', attrs={'class':['editado']})
+                            ,dict(name='table')
+                            ,dict(name='li')
+                        ]
+
+
+    feeds = [
+                 (u'Portada'              , u'http://www.europapress.es/rss/rss.aspx')
+                ,(u'Nacional'             , u'http://www.europapress.es/rss/rss.aspx?ch=66')
+                ,(u'Internacional'       , u'http://www.europapress.es/rss/rss.aspx?ch=69')
+                ,(u'Economia'           , u'http://www.europapress.es/rss/rss.aspx?ch=136')
+                ,(u'Deportes'            , u'http://www.europapress.es/rss/rss.aspx?ch=67')
+                ,(u'Cultura'               , u'http://www.europapress.es/rss/rss.aspx?ch=126')
+                ,(u'Sociedad'            , u'http://www.europapress.es/rss/rss.aspx?ch=73')
+                ,(u'Motor'                 , u'http://www.europapress.es/rss/rss.aspx?ch=435')
+                ,(u'CHANCE'             , u'http://www.europapress.es/rss/rss.aspx?ch=549')
+                ,(u'Comunicados'      , u'http://www.europapress.es/rss/rss.aspx?ch=137')
+             ]
+
--- a/resources/recipes/irish_times.recipe
+++ b/resources/recipes/irish_times.recipe
@ -35,7 +35,7 @@ class IrishTimes(BasicNewsRecipe):
    def print_version(self, url):
         if url.count('rss.feedsportal.com'):
            u = 'http://www.irishtimes.com' + \
-                     (((url[69:].replace('0C','/')).replace('0A','0'))).replace('0Bhtml/story01.htm','_pf.html')
+                     (((url[70:].replace('0C','/')).replace('0A','0'))).replace('0Bhtml/story01.htm','_pf.html')
         else:
             u = url.replace('.html','_pf.html')
         return u
--- a/resources/recipes/kopalniawiedzy.recipe
+++ b/resources/recipes/kopalniawiedzy.recipe
@ -0,0 +1,80 @@
+# -*- coding: utf-8 -*-
+__license__   = 'GPL v3'
+__copyright__ = '2011, Attis <attis@attis.one.pl>'
+__version__ = 'v. 0.1'
+
+import re
+from calibre.web.feeds.recipes import BasicNewsRecipe
+
+class KopalniaWiedzy(BasicNewsRecipe):
+		title          = u'Kopalnia Wiedzy'
+		publisher      = u'Kopalnia Wiedzy'
+		description    = u'Ciekawostki ze świata nauki i techniki'
+		encoding       = 'utf-8'
+		__author__     = 'Attis'
+		language       = 'pl'
+		oldest_article = 7
+		max_articles_per_feed = 100
+		INDEX          = u'http://kopalniawiedzy.pl/'
+		remove_javascript     = True
+		no_stylesheets        = True
+
+		remove_tags    = [{'name':'p', 'attrs': {'class': 'keywords'} }]
+		remove_tags_after = dict(attrs={'class':'ad-square'})
+		keep_only_tags    = [dict(name="div", attrs={'id':'articleContent'})]
+		extra_css      = '.topimage {margin-top: 30px}'
+
+		preprocess_regexps = [
+				(re.compile(u'<a .* rel="lightboxText" .*><img (.*)></a>'),
+				lambda match: '<img class="topimage" ' + match.group(1) + '>' ),
+				(re.compile(u'<br  /><br  />'),
+				lambda match: '<br\/>')
+			]
+
+		feeds = [
+			(u'Biologia', u'http://kopalniawiedzy.pl/wiadomosci_biologia.rss'),
+			(u'Medycyna', u'http://kopalniawiedzy.pl/wiadomosci_medycyna.rss'),
+			(u'Psychologia', u'http://kopalniawiedzy.pl/wiadomosci_psychologia.rss'),
+			(u'Technologie', u'http://kopalniawiedzy.pl/wiadomosci_technologie.rss'),
+			(u'Ciekawostki', u'http://kopalniawiedzy.pl/wiadomosci_ciekawostki.rss'),
+			(u'Artykuły', u'http://kopalniawiedzy.pl/artykuly.rss')
+		]
+
+		def is_link_wanted(self, url, tag):
+			return tag['class'] == 'next'
+
+		def remove_beyond(self, tag, next):
+				while tag is not None and getattr(tag, 'name', None) != 'body':
+						after = getattr(tag, next)
+						while after is not None:
+								ns = getattr(tag, next)
+								after.extract()
+								after = ns
+						tag = tag.parent
+
+		def append_page(self, soup, appendtag, position):
+				pager = soup.find('a',attrs={'class':'next'})
+				if pager:
+					nexturl = self.INDEX + pager['href']
+					soup2 = self.index_to_soup(nexturl)
+					texttag = soup2.find('div', attrs={'id':'articleContent'})
+
+					tag = texttag.find(attrs={'class':'pages'})
+					self.remove_beyond(tag, 'nextSibling')
+
+					newpos = len(texttag.contents)
+					self.append_page(soup2,texttag,newpos)
+
+					appendtag.insert(position,texttag)
+
+
+		def preprocess_html(self, soup):
+				self.append_page(soup, soup.body, 3)
+
+				for item in soup.findAll('div',attrs={'class':'pages'}):
+					item.extract()
+
+				for item in soup.findAll('p', attrs={'class':'wykop'}):
+					item.extract()
+
+				return soup
--- a/resources/recipes/korespondent.recipe
+++ b/resources/recipes/korespondent.recipe
@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Attis <attis@attis.one.pl>'
+__version__ = 'v. 0.1'
+
+import re
+from calibre.web.feeds.recipes import BasicNewsRecipe
+
+class KorespondentPL(BasicNewsRecipe):
+    title          = u'Korespondent.pl'
+    publisher      = u'Korespondent.pl'
+    description    = u'Centrum wolnorynkowe - serwis ludzi wolnych'
+    encoding       = 'utf-8'
+    __author__     = 'Attis'
+    language       = 'pl'
+    oldest_article = 15
+    max_articles_per_feed = 100
+    remove_javascript     = True
+    no_stylesheets        = True
+
+    keep_only_tags = [dict(name='div', attrs={'class':'publicystyka'})]
+    remove_tags    = [{'name': 'meta'}, {'name':'div', 'attrs': {'class': 'zdjecie'} }]
+    extra_css      = '.naglowek {font-size: small}\n .tytul {font-size: x-large; padding-bottom: 10px; padding-top: 30px} \n .external {font-size: small}'
+
+    preprocess_regexps = [
+			(re.compile(u'<a href="index\.php.*>(.*)</a>'),
+			lambda match: match.group(1) ),
+			(re.compile(u'<i>'),
+			lambda match:'<i class="external">' ),
+			(re.compile(u'<p></p>Więcej'),
+			lambda match:'Więcej' ),
+			(re.compile(u'target="_blank"'),
+			lambda match:'target="_blank" class="external"' ),
+			(re.compile(u'<p align="center">\nPoczytaj inne teksty w <a href="http://www.korespondent.pl">Serwisie wolnorynkowym Korespondent.pl</a>.*</body>', re.DOTALL|re.IGNORECASE),
+			lambda match: '</div></body>'),
+			]
+
+    feeds = [(u'Serwis informacyjny', u'http://korespondent.pl/rss.xml')]
+
--- a/resources/recipes/lalibre_be.recipe
+++ b/resources/recipes/lalibre_be.recipe
@ -1,7 +1,7 @@
 #!/usr/bin/env  python

 __license__   = 'GPL v3'
-__copyright__ = '2008, Lionel Bergeret <lbergeret at gmail.com>'
+__copyright__ = '2008-2011, Lionel Bergeret <lbergeret at gmail.com>'
 '''
 lalibre.be
 '''
@ -16,18 +16,18 @@ class LaLibre(BasicNewsRecipe):
    publisher             = u'lalibre.be'
    category              = 'news, Belgium'
    oldest_article        = 3
-    language              = 'fr_BE'
+    language              = 'fr'
+    masthead_url          = 'http://www.lalibre.be/img/logoLaLibre.gif'

    max_articles_per_feed = 20
    no_stylesheets        = True
    use_embedded_content  = False
    timefmt               = ' [%d %b %Y]'

-    keep_only_tags = [
-	 dict(name = 'div', attrs = {'id': 'articleHat'})
-	,dict(name = 'p', attrs = {'id': 'publicationDate'})
-        ,dict(name = 'div', attrs = {'id': 'articleText'})
-    ]
+    remove_tags_before = dict(name = 'div', attrs = {'class': 'extraMainContent'})
+    remove_tags_after  = dict(name = 'div', attrs = {'id': 'articleText'})
+
+    remove_tags = [dict(name = 'div', attrs = {'id': 'strongArticleLinks'})]

    feeds = [
         (u'L\'actu'           , u'http://www.lalibre.be/rss/?section=10' )
@ -38,6 +38,13 @@ class LaLibre(BasicNewsRecipe):
        ,(u'Societe'           , u'http://www.lalibre.be/rss/?section=12' )
    ]

+    def preprocess_html(self, soup):
+        for alink in soup.findAll('a'):
+            if alink.string is not None:
+               tstr = alink.string
+               alink.replaceWith(tstr)
+        return soup
+
    def get_cover_url(self):
        cover_url = strftime('http://pdf-online.lalibre.be/pdfonline/image/%Y%m%d/llb_%Y%m%d_nam_libre_001.pdf.L.jpg')
        return cover_url
--- a/resources/recipes/lameuse_be.recipe
+++ b/resources/recipes/lameuse_be.recipe
@ -1,7 +1,7 @@
 #!/usr/bin/env  python

 __license__   = 'GPL v3'
-__copyright__ = '2008, Lionel Bergeret <lbergeret at gmail.com>'
+__copyright__ = '2008-2011, Lionel Bergeret <lbergeret at gmail.com>'
 '''
 lameuse.be
 '''
@ -16,8 +16,8 @@ class LaMeuse(BasicNewsRecipe):
    publisher             = u'lameuse.be'
    category              = 'news, Belgium'
    oldest_article        = 3
-    encoding              = 'utf8'
-    language              = 'fr_BE'
+    language              = 'fr'
+    masthead_url          = 'http://www.lameuse.be/images/SPV3/logo_header_LM.gif'

    max_articles_per_feed = 20
    no_stylesheets        = True
@ -32,6 +32,11 @@ class LaMeuse(BasicNewsRecipe):
         dict(name = 'div', attrs = {'class': 'sb-group'})
        ,dict(name = 'div', attrs = {'id': 'share'})
        ,dict(name = 'div', attrs = {'id': 'commentaires'})
+        ,dict(name = 'ul', attrs = {'class': 'right liensutiles'})
+        ,dict(name = 'ul', attrs = {'class': 'bas liensutiles'})
+        ,dict(name = 'p', attrs = {'class': 'ariane'})
+        ,dict(name = 'div', attrs = {'class': 'inner-bloc'})
+        ,dict(name = 'div', attrs = {'class': 'block-01'})
    ]

    feeds = [
--- a/resources/recipes/lavenir_be.recipe
+++ b/resources/recipes/lavenir_be.recipe
@ -1,7 +1,7 @@
 #!/usr/bin/env  python

 __license__   = 'GPL v3'
-__copyright__ = '2008, Lionel Bergeret <lbergeret at gmail.com>'
+__copyright__ = '2008-2011, Lionel Bergeret <lbergeret at gmail.com>'
 '''
 lavenir.net
 '''
@ -15,8 +15,7 @@ class LAvenir(BasicNewsRecipe):
    publisher             = u'lavenir.net'
    category              = 'news, Belgium'
    oldest_article        = 3
-    encoding              = 'utf8'
-    language              = 'fr_BE'
+    language              = 'fr'

    max_articles_per_feed = 20
    no_stylesheets        = True
@ -35,6 +34,13 @@ class LAvenir(BasicNewsRecipe):
        ,(u'Societe'        , u'http://www.lavenir.net/rss.aspx?foto=1&intro=1&section=info&info=12e1a2f4-7e03-4cf1-afec-016869072317' )
    ]

+    def preprocess_html(self, soup):
+        for alink in soup.findAll('a'):
+            if alink.string is not None:
+               tstr = alink.string
+               alink.replaceWith(tstr)
+        return soup
+
    def get_cover_url(self):
        cover_url = 'http://www.lavenir.net/extra/Static/journal/Pdf/1/UNE_Nationale.PDF'
        return cover_url
--- a/resources/recipes/lesoir_be.recipe
+++ b/resources/recipes/lesoir_be.recipe
@ -1,7 +1,7 @@
 #!/usr/bin/env  python

 __license__   = 'GPL v3'
-__copyright__ = '2008, Lionel Bergeret <lbergeret at gmail.com>'
+__copyright__ = '2008-2011, Lionel Bergeret <lbergeret at gmail.com>'
 '''
 lesoir.be
 '''
@ -16,7 +16,8 @@ class LeSoirBe(BasicNewsRecipe):
    publisher             = u'lesoir.be'
    category              = 'news, Belgium'
    oldest_article        = 3
-    language              = 'fr_BE'
+    language              = 'fr'
+    masthead_url          = 'http://pdf.lesoir.be/pdf/images/SOIR//logo.gif'

    max_articles_per_feed = 20
    no_stylesheets        = True
--- a/resources/recipes/new_yorker.recipe
+++ b/resources/recipes/new_yorker.recipe
@ -54,10 +54,10 @@ class NewYorker(BasicNewsRecipe):
                        ,dict(attrs={'id':['show-header','show-footer'] })
                     ]
    remove_attributes = ['lang']
-    feeds             = [(u'The New Yorker', u'http://www.newyorker.com/services/rss/feeds/everything.xml')]
+    feeds             = [(u'The New Yorker', u'http://www.newyorker.com/services/mrss/feeds/everything.xml')]

    def print_version(self, url):
-        return 'http://www.newyorker.com' + url + '?printable=true'
+        return url + '?printable=true'

    def image_url_processor(self, baseurl, url):
        return url.strip()
--- a/resources/recipes/radio_prague.recipe
+++ b/resources/recipes/radio_prague.recipe
@ -0,0 +1,43 @@
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1291540961(BasicNewsRecipe):
+
+    title          = u'Radio Praha'
+    __author__             = 'Francois Pellicaan'
+    description            = 'News and information from and about The Czech republic. '
+    oldest_article = 7
+    max_articles_per_feed = 100
+    no_stylesheets         = True
+    use_embedded_content   = False
+    remove_empty_feeds     = True
+    encoding               = 'utf8'
+    publisher              = 'Radio Prague'
+    category               = 'News'
+    language               = 'en_CZ'
+    publication_type       = 'newsportal'
+
+    extra_css = 'h1 .section { display: block; text-transform: uppercase; font-size: 10px; margin-top: 4em; } \n .title { font-size: 14px; margin-top: 4em; } \n a.photo { display: block; clear:both; }  \n .caption { font-size: 9px; display: block; clear:both; padding:0px 0px 20px 0px; } \n a { font-type: normal; }'
+
+
+    keep_only_tags = [
+                       dict(name='div', attrs={'class':['main']})
+                        ]
+    remove_tags = [
+                       dict(name='div', attrs={'class':['cleaner', 'options', 'toolsXXL']}),
+       dict(name='ul', attrs={'class':['tools']})
+                        ]
+    feeds = [
+                      (u'Current Affairs', 'http://www.radio.cz/feeds/rss/en/themes/curraffrs.xml'),
+                      (u'Society', 'http://www.radio.cz/feeds/rss/en/themes/society.xml'),
+                      (u'European Union', 'http:http://www.radio.cz/feeds/rss/en/themes/eu.xml'),
+                      (u'Foreign policy', 'http://www.radio.cz/feeds/rss/en/themes/foreignpolicy.xml'),
+                      (u'Business', 'http://www.radio.cz/feeds/rss/en/themes/business.xml'),
+                      (u'Culture', 'http://www.radio.cz/feeds/rss/en/themes/culture.xml'),
+                      (u'Czechs abroad', 'http://www.radio.cz/feeds/rss/en/themes/czechabroad.xml'),
+                      (u'History', 'http://www.radio.cz/feeds/rss/en/themes/history.xml'),
+                      (u'Nature', 'http://www.radio.cz/feeds/rss/en/themes/nature.xml'),
+                      (u'Science', 'http://www.radio.cz/feeds/rss/en/themes/science.xml'),
+                      (u'Sport', 'http://www.radio.cz/feeds/rss/en/themes/sport.xml'),
+                      (u'Travel', 'http://www.radio.cz/feeds/rss/en/themes/travel.xml'),
+                    ]
--- a/resources/recipes/radio_praha.recipe
+++ b/resources/recipes/radio_praha.recipe
@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1291540961(BasicNewsRecipe):
+
+    title          = u'Radio Praha'
+    __author__             = 'Francois Pellicaan'
+    description            = u'Česká oficiální mezinárodní vysílací stanice.'
+    oldest_article = 7
+    max_articles_per_feed = 100
+    no_stylesheets         = True
+    use_embedded_content   = False
+    remove_empty_feeds     = True
+    encoding               = 'utf8'
+    publisher              = u'Český rozhlas'
+    category               = 'News'
+    language               = 'cs'
+    publication_type       = 'newsportal'
+
+    extra_css = u'h1 .section { display: block; text-transform: uppercase; font-size: 10px; margin-top: 4em; } \n .title { font-size: 14px; margin-top: 4em; } \n a.photo { display: block; clear:both; }  \n .caption { font-size: 9px; display: block; clear:both; padding:0px 0px 20px 0px; } \n a { font-type: normal; }'
+
+
+    keep_only_tags = [
+                       dict(name='div', attrs={'class':['main']})
+                        ]
+    remove_tags = [
+                       dict(name='div', attrs={'class':['cleaner', 'options', 'toolsXXL']}),
+	   dict(name='ul', attrs={'class':['tools']})
+                        ]
+    feeds = [
+                      (u'Domácí politika', 'http://www.radio.cz/feeds/rss/cs/oblast/dompol.xml'),
+                      (u'Společnost', 'http://www.radio.cz/feeds/rss/cs/oblast/spolecnost.xml'),
+                      (u'Evropská unie', 'http://www.radio.cz/feeds/rss/cs/oblast/eu.xml'),
+                      (u'Zahraniční politika', 'http://www.radio.cz/feeds/rss/cs/oblast/zahrpol.xml'),
+                      (u'Ekonomika', 'http://www.radio.cz/feeds/rss/cs/oblast/ekonomika.xml'),
+                      (u'Kultura', 'http://www.radio.cz/feeds/rss/cs/oblast/kultura.xml'),
+                      (u'Krajané', 'http://www.radio.cz/feeds/rss/cs/oblast/krajane.xml'),
+                      (u'Historie', 'http://www.radio.cz/feeds/rss/cs/oblast/historie.xml'),
+                      (u'Příroda', 'http://www.radio.cz/feeds/rss/cs/oblast/priroda.xml'),
+                      (u'Věda', 'http://www.radio.cz/feeds/rss/cs/oblast/veda.xml'),
+                      (u'Sport', 'http://www.radio.cz/feeds/rss/cs/oblast/sport.xml'),
+                      (u'Cestování', 'http://www.radio.cz/feeds/rss/cs/oblast/cestovani.xml'),
+                    ]
--- a/src/calibre/customize/builtins.py
+++ b/src/calibre/customize/builtins.py
@ -325,6 +325,17 @@ class TXTMetadataReader(MetadataReaderPlugin):
        from calibre.ebooks.metadata.txt import get_metadata
        return get_metadata(stream)

+class TXTZMetadataReader(MetadataReaderPlugin):
+
+    name        = 'Read TXTZ metadata'
+    file_types  = set(['txtz'])
+    description = _('Read metadata from %s files') % 'TXTZ'
+    author      = 'John Schember'
+
+    def get_metadata(self, stream, ftype):
+        from calibre.ebooks.metadata.txtz import get_metadata
+        return get_metadata(stream)
+
 class ZipMetadataReader(MetadataReaderPlugin):

    name = 'Read ZIP metadata'
@ -412,6 +423,17 @@ class TOPAZMetadataWriter(MetadataWriterPlugin):
        from calibre.ebooks.metadata.topaz import set_metadata
        set_metadata(stream, mi)

+class TXTZMetadataWriter(MetadataWriterPlugin):
+
+    name        = 'Set TXTZ metadata'
+    file_types  = set(['txtz'])
+    description = _('Set metadata from %s files') % 'TXTZ'
+    author      = 'John Schember'
+
+    def set_metadata(self, stream, mi, type):
+        from calibre.ebooks.metadata.txtz import set_metadata
+        set_metadata(stream, mi)
+
 # }}}

 from calibre.ebooks.comic.input import ComicInput
@ -446,6 +468,7 @@ from calibre.ebooks.rb.output import RBOutput
 from calibre.ebooks.rtf.output import RTFOutput
 from calibre.ebooks.tcr.output import TCROutput
 from calibre.ebooks.txt.output import TXTOutput
+from calibre.ebooks.txt.output import TXTZOutput
 from calibre.ebooks.html.output import HTMLOutput
 from calibre.ebooks.snb.output import SNBOutput

@ -534,6 +557,7 @@ plugins += [
    RTFOutput,
    TCROutput,
    TXTOutput,
+    TXTZOutput,
    HTMLOutput,
    SNBOutput,
 ]
@ -770,6 +794,17 @@ class Toolbar(PreferencesPlugin):
    description = _('Customize the toolbars and context menus, changing which'
            ' actions are available in each')

+class Search(PreferencesPlugin):
+    name = 'Search'
+    icon = I('search.png')
+    gui_name = _('Customize searching')
+    category = 'Interface'
+    gui_category = _('Interface')
+    category_order = 1
+    name_order = 5
+    config_widget = 'calibre.gui2.preferences.search'
+    description = _('Customize the way searching for books works in calibre')
+
 class InputOptions(PreferencesPlugin):
    name = 'Input Options'
    icon = I('arrow-down.png')
@ -920,7 +955,7 @@ class Misc(PreferencesPlugin):
    config_widget = 'calibre.gui2.preferences.misc'
    description = _('Miscellaneous advanced configuration')

-plugins += [LookAndFeel, Behavior, Columns, Toolbar, InputOptions,
+plugins += [LookAndFeel, Behavior, Columns, Toolbar, Search, InputOptions,
        CommonOptions, OutputOptions, Adding, Saving, Sending, Plugboard,
        Email, Server, Plugins, Tweaks, Misc, TemplateFunctions]

--- a/src/calibre/devices/android/driver.py
+++ b/src/calibre/devices/android/driver.py
@ -19,10 +19,15 @@ class ANDROID(USBMS):

    VENDOR_ID   = {
            # HTC
-            0x0bb4 : { 0x0c02 : [0x100, 0x0227, 0x0226], 0x0c01 : [0x100,
-                0x0227, 0x0226], 0x0ff9
-                : [0x0100, 0x0227, 0x0226], 0x0c87: [0x0100, 0x0227, 0x0226],
-                0xc92 : [0x100], 0xc97: [0x226], 0xc99 : [0x0100]},
+            0x0bb4 : { 0x0c02 : [0x100, 0x0227, 0x0226],
+                       0x0c01 : [0x100, 0x0227, 0x0226],
+                       0x0ff9 : [0x0100, 0x0227, 0x0226],
+                       0x0c87 : [0x0100, 0x0227, 0x0226],
+                       0xc92  : [0x100],
+                       0xc97  : [0x226],
+                       0xc99  : [0x0100],
+                       0xca3  : [0x100],
+            },

            # Eken
            0x040d : { 0x8510 : [0x0001], 0x0851 : [0x1] },
@ -57,6 +62,9 @@ class ANDROID(USBMS):
            # Archos
            0x0e79 : { 0x1419: [0x0216], 0x1420 : [0x0216], 0x1422 : [0x0216]},

+            # Huawei
+            0x45e : { 0x00e1 : [0x007], },
+
            }
    EBOOK_DIR_MAIN = ['eBooks/import', 'wordplayer/calibretransfer', 'Books']
    EXTRA_CUSTOMIZATION_MESSAGE = _('Comma separated list of directories to '
@ -66,12 +74,13 @@ class ANDROID(USBMS):

    VENDOR_NAME      = ['HTC', 'MOTOROLA', 'GOOGLE_', 'ANDROID', 'ACER',
            'GT-I5700', 'SAMSUNG', 'DELL', 'LINUX', 'GOOGLE', 'ARCHOS',
-            'TELECHIP']
+            'TELECHIP', 'HUAWEI', ]
    WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE',
            '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897',
            'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID',
            'SCH-I500_CARD', 'SPH-D700_CARD', 'MB810', 'GT-P1000', 'DESIRE',
-            'SGH-T849', '_MB300', 'A70S', 'S_ANDROID', 'A101IT', 'A70H']
+            'SGH-T849', '_MB300', 'A70S', 'S_ANDROID', 'A101IT', 'A70H',
+            'IDEOS_TABLET']
    WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897',
            'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD',
            'A70S', 'A101IT']
--- a/src/calibre/devices/eb600/driver.py
+++ b/src/calibre/devices/eb600/driver.py
@ -172,10 +172,10 @@ class INVESBOOK(EB600):
    gui_name = 'Inves Book 600'

    FORMATS = ['epub', 'mobi', 'prc', 'fb2', 'html', 'pdf', 'rtf', 'txt']
+    BCD         = [0x110, 0x323]

-    VENDOR_NAME = 'INVES_E6'
-    WINDOWS_MAIN_MEM = '00INVES_E600'
-    WINDOWS_CARD_A_MEM = '00INVES_E600'
+    VENDOR_NAME = ['INVES_E6', 'INVES-WI']
+    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['00INVES_E600', 'INVES-WIBOOK']

 class BOOQ(EB600):
    name = 'Booq Device Interface'
--- a/src/calibre/ebooks/init.py
+++ b/src/calibre/ebooks/init.py
@ -25,7 +25,7 @@ class DRMError(ValueError):
 class ParserError(ValueError):
    pass

-BOOK_EXTENSIONS = ['lrf', 'rar', 'zip', 'rtf', 'lit', 'txt', 'htm', 'xhtm',
+BOOK_EXTENSIONS = ['lrf', 'rar', 'zip', 'rtf', 'lit', 'txt', 'txtz', 'htm', 'xhtm',
                   'html', 'xhtml', 'pdf', 'pdb', 'pdr', 'prc', 'mobi', 'azw', 'doc',
                   'epub', 'fb2', 'djvu', 'lrx', 'cbr', 'cbz', 'cbc', 'oebzip',
                   'rb', 'imp', 'odt', 'chm', 'tpz', 'azw1', 'pml', 'mbp', 'tan', 'snb']
--- a/src/calibre/ebooks/conversion/preprocess.py
+++ b/src/calibre/ebooks/conversion/preprocess.py
@ -245,17 +245,17 @@ class Dehyphenator(object):
        self.html = html
        self.format = format
        if format == 'html':
-            intextmatch = re.compile(u'(?<=.{%i})(?P<firstpart>[^\[\]\\\^\$\.\|\?\*\+\(\)“"\s>]+)(-|‐)\s*(?=<)(?P<wraptags>(</span>)?\s*(</[iubp]>\s*){1,2}(?P<up2threeblanks><(p|div)[^>]*>\s*(<p[^>]*>\s*</p>\s*)?</(p|div)>\s+){0,3}\s*(<[iubp][^>]*>\s*){1,2}(<span[^>]*>)?)\s*(?P<secondpart>[\w\d]+)' % length)
+            intextmatch = re.compile(u'(?<=.{%i})(?P<firstpart>[^\W\-]+)(-|‐)\s*(?=<)(?P<wraptags>(</span>)?\s*(</[iubp]>\s*){1,2}(?P<up2threeblanks><(p|div)[^>]*>\s*(<p[^>]*>\s*</p>\s*)?</(p|div)>\s+){0,3}\s*(<[iubp][^>]*>\s*){1,2}(<span[^>]*>)?)\s*(?P<secondpart>[\w\d]+)' % length)
        elif format == 'pdf':
-            intextmatch = re.compile(u'(?<=.{%i})(?P<firstpart>[^\[\]\\\^\$\.\|\?\*\+\(\)“"\s>]+)(-|‐)\s*(?P<wraptags><p>|</[iub]>\s*<p>\s*<[iub]>)\s*(?P<secondpart>[\w\d]+)'% length)
+            intextmatch = re.compile(u'(?<=.{%i})(?P<firstpart>[^\W\-]+)(-|‐)\s*(?P<wraptags><p>|</[iub]>\s*<p>\s*<[iub]>)\s*(?P<secondpart>[\w\d]+)'% length)
        elif format == 'txt':
-            intextmatch = re.compile(u'(?<=.{%i})(?P<firstpart>[^\[\]\\\^\$\.\|\?\*\+\(\)“"\s>]+)(-|‐)(\u0020|\u0009)*(?P<wraptags>(\n(\u0020|\u0009)*)+)(?P<secondpart>[\w\d]+)'% length)
+            intextmatch = re.compile(u'(?<=.{%i})(?P<firstpart>[^\W\-]+)(-|‐)(\u0020|\u0009)*(?P<wraptags>(\n(\u0020|\u0009)*)+)(?P<secondpart>[\w\d]+)'% length)
        elif format == 'individual_words':
-            intextmatch = re.compile(u'(?!<)(?P<firstpart>\w+)(-|‐)\s*(?P<secondpart>\w+)(?![^<]*?>)')
+            intextmatch = re.compile(u'(?!<)(?P<firstpart>[^\W\-]+)(-|‐)\s*(?P<secondpart>\w+)(?![^<]*?>)')
        elif format == 'html_cleanup':
-            intextmatch = re.compile(u'(?P<firstpart>[^\[\]\\\^\$\.\|\?\*\+\(\)“"\s>]+)(-|‐)\s*(?=<)(?P<wraptags></span>\s*(</[iubp]>\s*<[iubp][^>]*>\s*)?<span[^>]*>|</[iubp]>\s*<[iubp][^>]*>)?\s*(?P<secondpart>[\w\d]+)')
+            intextmatch = re.compile(u'(?P<firstpart>[^\W\-]+)(-|‐)\s*(?=<)(?P<wraptags></span>\s*(</[iubp]>\s*<[iubp][^>]*>\s*)?<span[^>]*>|</[iubp]>\s*<[iubp][^>]*>)?\s*(?P<secondpart>[\w\d]+)')
        elif format == 'txt_cleanup':
-            intextmatch = re.compile(u'(?P<firstpart>\w+)(-|‐)(?P<wraptags>\s+)(?P<secondpart>[\w\d]+)')
+            intextmatch = re.compile(u'(?P<firstpart>[^\W\-]+)(-|‐)(?P<wraptags>\s+)(?P<secondpart>[\w\d]+)')


        html = intextmatch.sub(self.dehyphenate, html)
--- a/src/calibre/ebooks/conversion/utils.py
+++ b/src/calibre/ebooks/conversion/utils.py
@ -11,6 +11,7 @@ from calibre.ebooks.conversion.preprocess import DocAnalysis, Dehyphenator
 from calibre.utils.logging import default_log
 from calibre.utils.wordcount import get_wordcount_obj

+
 class HeuristicProcessor(object):

    def __init__(self, extra_opts=None, log=None):
@ -34,10 +35,15 @@ class HeuristicProcessor(object):
        self.line_close = "(</(?P=inner3)>)?\s*(</(?P=inner2)>)?\s*(</(?P=inner1)>)?\s*</(?P=outer)>"
        self.single_blank = re.compile(r'(\s*<p[^>]*>\s*</p>)', re.IGNORECASE)
        self.scene_break_open = '<p class="scenebreak" style="text-align:center; text-indent:0%; margin-top:1em; margin-bottom:1em; page-break-before:avoid">'
+        self.common_in_text_endings = u'[\"\'—’”,\.!\?\…\)„\w]'
+        self.common_in_text_beginnings = u'[\w\'\"“‘‛]'

    def is_pdftohtml(self, src):
        return '<!-- created by calibre\'s pdftohtml -->' in src[:1000]

+    def is_abbyy(self, src):
+        return '<meta name="generator" content="ABBYY FineReader' in src[:1000]
+
    def chapter_head(self, match):
        from calibre.utils.html2text import html2text
        chap = match.group('chap')
@ -75,22 +81,23 @@ class HeuristicProcessor(object):

    def insert_indent(self, match):
        pstyle = match.group('formatting')
+        tag = match.group('tagtype')
        span = match.group('span')
        self.found_indents = self.found_indents + 1
        if pstyle:
-            if pstyle.lower().find('style'):
+            if pstyle.lower().find('style') != -1:
                pstyle = re.sub(r'"$', '; text-indent:3%"', pstyle)
            else:
                pstyle = pstyle+' style="text-indent:3%"'
            if not span:
-                return '<p '+pstyle+'>'
+                return '<'+tag+' '+pstyle+'>'
            else:
-                return '<p '+pstyle+'>'+span
+                return '<'+tag+' '+pstyle+'>'+span
        else:
            if not span:
-                return '<p style="text-indent:3%">'
+                return '<'+tag+' style="text-indent:3%">'
            else:
-                return '<p style="text-indent:3%">'+span
+                return '<'+tag+' style="text-indent:3%">'+span

    def no_markup(self, raw, percent):
        '''
@ -149,17 +156,17 @@ class HeuristicProcessor(object):
        ]

        ITALICIZE_STYLE_PATS = [
-            r'(?msu)(?<=\s)_(?P<words>\S[^_]{0,40}?\S)?_(?=[\s\.,\!\?])',
-            r'(?msu)(?<=\s)/(?P<words>\S[^/]{0,40}?\S)?/(?=[\s\.,\!\?])',
-            r'(?msu)(?<=\s)~~(?P<words>\S[^~]{0,40}?\S)?~~(?=[\s\.,\!\?])',
-            r'(?msu)(?<=\s)\*(?P<words>\S[^\*]{0,40}?\S)?\*(?=[\s\.,\!\?])',
-            r'(?msu)(?<=\s)~(?P<words>\S[^~]{0,40}?\S)?~(?=[\s\.,\!\?])',
-            r'(?msu)(?<=\s)_/(?P<words>\S[^/_]{0,40}?\S)?/_(?=[\s\.,\!\?])',
-            r'(?msu)(?<=\s)_\*(?P<words>\S[^\*_]{0,40}?\S)?\*_(?=[\s\.,\!\?])',
-            r'(?msu)(?<=\s)\*/(?P<words>\S[^/\*]{0,40}?\S)?/\*(?=[\s\.,\!\?])',
-            r'(?msu)(?<=\s)_\*/(?P<words>\S[^\*_]{0,40}?\S)?/\*_(?=[\s\.,\!\?])',
-            r'(?msu)(?<=\s)/:(?P<words>\S[^:/]{0,40}?\S)?:/(?=[\s\.,\!\?])',
-            r'(?msu)(?<=\s)\|:(?P<words>\S[^:\|]{0,40}?\S)?:\|(?=[\s\.,\!\?])',
+            r'(?msu)(?<=[\s>])_(?P<words>[^_]+)?_',
+            r'(?msu)(?<=[\s>])/(?P<words>[^/]+)?/',
+            r'(?msu)(?<=[\s>])~~(?P<words>[^~]+)?~~',
+            r'(?msu)(?<=[\s>])\*(?P<words>[^\*]+)?\*',
+            r'(?msu)(?<=[\s>])~(?P<words>[^~]+)?~',
+            r'(?msu)(?<=[\s>])_/(?P<words>[^/_]+)?/_',
+            r'(?msu)(?<=[\s>])_\*(?P<words>[^\*_]+)?\*_',
+            r'(?msu)(?<=[\s>])\*/(?P<words>[^/\*]+)?/\*',
+            r'(?msu)(?<=[\s>])_\*/(?P<words>[^\*_]+)?/\*_',
+            r'(?msu)(?<=[\s>])/:(?P<words>[^:/]+)?:/',
+            r'(?msu)(?<=[\s>])\|:(?P<words>[^:\|]+)?:\|',
        ]

        for word in ITALICIZE_WORDS:
@ -363,7 +370,7 @@ class HeuristicProcessor(object):
        return html

    def fix_nbsp_indents(self, html):
-        txtindent = re.compile(ur'<p(?P<formatting>[^>]*)>\s*(?P<span>(<span[^>]*>\s*)+)?\s*(\u00a0){2,}', re.IGNORECASE)
+        txtindent = re.compile(ur'<(?P<tagtype>p|div)(?P<formatting>[^>]*)>\s*(?P<span>(<span[^>]*>\s*)+)?\s*(\u00a0){2,}', re.IGNORECASE)
        html = txtindent.sub(self.insert_indent, html)
        if self.found_indents > 1:
            self.log.debug("replaced "+unicode(self.found_indents)+ " nbsp indents with inline styles")
@ -516,6 +523,111 @@ class HeuristicProcessor(object):

        return scene_break

+    def abbyy_processor(self, html):
+        abbyy_line = re.compile('((?P<linestart><p\sstyle="(?P<styles>[^\"]*?);?">)(?P<content>.*?)(?P<lineend></p>)|(?P<image><img[^>]*>))', re.IGNORECASE)
+        empty_paragraph = '\n<p> </p>\n'
+        self.in_blockquote = False
+        self.previous_was_paragraph = False
+        html = re.sub('</?a[^>]*>', '', html)
+
+        def check_paragraph(content):
+            content = re.sub('\s*</?span[^>]*>\s*', '', content)
+            if re.match('.*[\"\'.!?:]$', content):
+                #print "detected this as a paragraph"
+                return True
+            else:
+                return False
+
+        def convert_styles(match):
+            #print "raw styles are: "+match.group('styles')
+            content = match.group('content')
+            #print "raw content is: "+match.group('content')
+            image = match.group('image')
+
+            is_paragraph = False
+            text_align = ''
+            text_indent = ''
+            paragraph_before = ''
+            paragraph_after = ''
+            blockquote_open = '\n<blockquote>\n'
+            blockquote_close = '</blockquote>\n'
+            indented_text = 'text-indent:3%;'
+            blockquote_open_loop = ''
+            blockquote_close_loop = ''
+            debugabby = False
+
+            if image:
+                debugabby = True
+                if self.in_blockquote:
+                    self.in_blockquote = False
+                    blockquote_close_loop = blockquote_close
+                self.previous_was_paragraph = False
+                return blockquote_close_loop+'\n'+image+'\n'
+            else:
+                styles = match.group('styles').split(';')
+                is_paragraph = check_paragraph(content)
+                #print "styles for this line are: "+str(styles)
+                split_styles = []
+                for style in styles:
+                   #print "style is: "+str(style)
+                   newstyle = style.split(':')
+                   #print "newstyle is: "+str(newstyle)
+                   split_styles.append(newstyle)
+                styles = split_styles
+                for style, setting in styles:
+                    if style == 'text-align' and setting != 'left':
+                        text_align = style+':'+setting+';'
+                    if style == 'text-indent':
+                        setting = int(re.sub('\s*pt\s*', '', setting))
+                        if 9 < setting < 14:
+                            text_indent = indented_text
+                        else:
+                            text_indent = style+':'+str(setting)+'pt;'
+                    if style == 'padding':
+                        setting = re.sub('pt', '', setting).split(' ')
+                        if int(setting[1]) < 16 and int(setting[3]) < 16:
+                            if self.in_blockquote:
+                                debugabby = True
+                                if is_paragraph:
+                                    self.in_blockquote = False
+                                    blockquote_close_loop = blockquote_close
+                            if int(setting[3]) > 8 and text_indent == '':
+                                text_indent = indented_text
+                            if int(setting[0]) > 5:
+                                paragraph_before = empty_paragraph
+                            if int(setting[2]) > 5:
+                                paragraph_after = empty_paragraph
+                        elif not self.in_blockquote and self.previous_was_paragraph:
+                            debugabby = True
+                            self.in_blockquote = True
+                            blockquote_open_loop = blockquote_open
+                        if debugabby:
+                            self.log.debug('\n\n******\n')
+                            self.log.debug('padding top is: '+str(setting[0]))
+                            self.log.debug('padding right is:'
+                                    +str(setting[1]))
+                            self.log.debug('padding bottom is: ' +
+                                    str(setting[2]))
+                            self.log.debug('padding left is: '
+                                    +str(setting[3]))
+
+                #print "text-align is: "+str(text_align)
+                #print "\n***\nline is:\n     "+str(match.group(0))+'\n'
+                if debugabby:
+                    #print "this line is a paragraph = "+str(is_paragraph)+", previous line was "+str(self.previous_was_paragraph)
+                    self.log.debug("styles for this line were:", styles)
+                    self.log.debug('newline is:')
+                    self.log.debug(blockquote_open_loop+blockquote_close_loop+
+                            paragraph_before+'<p style="'+text_indent+text_align+
+                            '">'+content+'</p>'+paragraph_after+'\n\n\n\n\n')
+                #print "is_paragraph is "+str(is_paragraph)+", previous_was_paragraph is "+str(self.previous_was_paragraph)
+                self.previous_was_paragraph = is_paragraph
+                #print "previous_was_paragraph is now set to "+str(self.previous_was_paragraph)+"\n\n\n"
+                return blockquote_open_loop+blockquote_close_loop+paragraph_before+'<p style="'+text_indent+text_align+'">'+content+'</p>'+paragraph_after
+
+        html = abbyy_line.sub(convert_styles, html)
+        return html
+

    def __call__(self, html):
        self.log.debug("*********  Heuristic processing HTML  *********")
@ -530,6 +642,10 @@ class HeuristicProcessor(object):
            self.log.warn("flow is too short, not running heuristics")
            return html

+        is_abbyy = self.is_abbyy(html)
+        if is_abbyy:
+            html = self.abbyy_processor(html)
+
        # Arrange line feeds and </p> tags so the line_length and no_markup functions work correctly
        html = self.arrange_htm_line_endings(html)
        #self.dump(html, 'after_arrange_line_endings')
@ -638,7 +754,7 @@ class HeuristicProcessor(object):
            blanks_count = len(self.any_multi_blank.findall(html))
            if blanks_count >= 1:
                html = self.merge_blanks(html, blanks_count)
-            scene_break_regex = self.line_open+'(?![\w\'\"])(?P<break>((?P<break_char>((?!\s)\W))\s*(?P=break_char)?)+)\s*'+self.line_close
+            scene_break_regex = self.line_open+'(?!('+self.common_in_text_beginnings+'|.*?'+self.common_in_text_endings+'<))(?P<break>((?P<break_char>((?!\s)\W))\s*(?P=break_char)?)+)\s*'+self.line_close
            scene_break = re.compile(r'%s' % scene_break_regex, re.IGNORECASE|re.UNICODE)
            # If the user has enabled scene break replacement, then either softbreaks
            # or 'hard' scene breaks are replaced, depending on which is in use
--- a/src/calibre/ebooks/metadata/txt.py
+++ b/src/calibre/ebooks/metadata/txt.py
@ -1,16 +1,20 @@
-'''Read meta information from TXT files'''
-
-from __future__ import with_statement
+# -*- coding: utf-8 -*-

 __license__   = 'GPL v3'
 __copyright__ = '2009, John Schember <john@nachtimwald.com>'

+'''
+Read meta information from TXT files
+'''
+
 import re

 from calibre.ebooks.metadata import MetaInformation

 def get_metadata(stream, extract_cover=True):
-    """ Return metadata as a L{MetaInfo} object """
+    '''
+    Return metadata as a L{MetaInfo} object
+    '''
    mi = MetaInformation(_('Unknown'), [_('Unknown')])
    stream.seek(0)

--- a/src/calibre/ebooks/metadata/txtz.py
+++ b/src/calibre/ebooks/metadata/txtz.py
@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+
+'''
+Read meta information from TXT files
+'''
+
+import os
+
+from cStringIO import StringIO
+
+from calibre.ebooks.metadata import MetaInformation
+from calibre.ebooks.metadata.opf2 import OPF, metadata_to_opf
+from calibre.ptempfile import TemporaryDirectory
+from calibre.utils.zipfile import ZipFile, safe_replace
+
+def get_metadata(stream, extract_cover=True):
+    '''
+    Return metadata as a L{MetaInfo} object
+    '''
+    mi = MetaInformation(_('Unknown'), [_('Unknown')])
+    stream.seek(0)
+
+    with TemporaryDirectory('_untxtz_mdata') as tdir:
+        try:
+            zf = ZipFile(stream)
+            zf.extract('metadata.opf', tdir)
+            with open(os.path.join(tdir, 'metadata.opf'), 'rb') as opff:
+                mi = OPF(opff).to_book_metadata()
+        except:
+            return mi
+    return mi
+
+def set_metadata(stream, mi):
+    opf = StringIO(metadata_to_opf(mi))
+    safe_replace(stream, 'metadata.opf', opf)
--- a/src/calibre/ebooks/oeb/transforms/jacket.py
+++ b/src/calibre/ebooks/oeb/transforms/jacket.py
@ -15,6 +15,7 @@ from calibre import guess_type, strftime
 from calibre.ebooks.BeautifulSoup import BeautifulSoup
 from calibre.ebooks.oeb.base import XPath, XHTML_NS, XHTML
 from calibre.library.comments import comments_to_html
+from calibre.utils.date import is_date_undefined

 JACKET_XPATH = '//h:meta[@name="calibre-content" and @content="jacket"]'

@ -109,7 +110,7 @@ def get_rating(rating, rchar, e_rchar):

 def render_jacket(mi, output_profile,
        alt_title=_('Unknown'), alt_tags=[], alt_comments='',
-        alt_publisher=('Unknown publisher')):
+        alt_publisher=('')):
    css = P('jacket/stylesheet.css', data=True).decode('utf-8')

    try:
@ -127,9 +128,12 @@ def render_jacket(mi, output_profile,
    try:
        publisher = mi.publisher if mi.publisher else alt_publisher
    except:
-        publisher = _('Unknown publisher')
+        publisher = ''

    try:
+        if is_date_undefined(mi.pubdate):
+            pubdate = ''
+        else:
            pubdate = strftime(u'%Y', mi.pubdate.timetuple())
    except:
        pubdate = ''
@ -175,18 +179,23 @@ def render_jacket(mi, output_profile,
        soup = BeautifulSoup(generated_html)
        if not series:
            series_tag = soup.find(attrs={'class':'cbj_series'})
+            if series_tag is not None:
                series_tag.extract()
        if not rating:
            rating_tag = soup.find(attrs={'class':'cbj_rating'})
+            if rating_tag is not None:
                rating_tag.extract()
        if not tags:
            tags_tag = soup.find(attrs={'class':'cbj_tags'})
+            if tags_tag is not None:
                tags_tag.extract()
        if not pubdate:
-            pubdate_tag = soup.find(attrs={'class':'cbj_pubdate'})
+            pubdate_tag = soup.find(attrs={'class':'cbj_pubdata'})
+            if pubdate_tag is not None:
                pubdate_tag.extract()
        if output_profile.short_name != 'kindle':
            hr_tag = soup.find('hr', attrs={'class':'cbj_kindle_banner_hr'})
+            if hr_tag is not None:
                hr_tag.extract()

        return soup.renderContents(None)
--- a/src/calibre/ebooks/txt/input.py
+++ b/src/calibre/ebooks/txt/input.py
@ -4,23 +4,27 @@ __license__ = 'GPL 3'
 __copyright__ = '2009, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'

+import glob
 import os

+from calibre import _ent_pat, xml_entity_to_unicode
 from calibre.customize.conversion import InputFormatPlugin, OptionRecommendation
 from calibre.ebooks.conversion.preprocess import DocAnalysis, Dehyphenator
 from calibre.ebooks.chardet import detect
 from calibre.ebooks.txt.processor import convert_basic, convert_markdown, \
    separate_paragraphs_single_line, separate_paragraphs_print_formatted, \
    preserve_spaces, detect_paragraph_type, detect_formatting_type, \
-    normalize_line_endings, convert_textile
-from calibre import _ent_pat, xml_entity_to_unicode
+    normalize_line_endings, convert_textile, remove_indents, block_to_single_line, \
+    separate_hard_scene_breaks
+from calibre.ptempfile import TemporaryDirectory
+from calibre.utils.zipfile import ZipFile

 class TXTInput(InputFormatPlugin):

    name        = 'TXT Input'
    author      = 'John Schember'
    description = 'Convert TXT files to HTML'
-    file_types  = set(['txt'])
+    file_types  = set(['txt', 'txtz'])

    options = set([
        OptionRecommendation(name='paragraph_type', recommended_value='auto',
@ -47,6 +51,9 @@ class TXTInput(InputFormatPlugin):
        OptionRecommendation(name='preserve_spaces', recommended_value=False,
            help=_('Normally extra spaces are condensed into a single space. '
                'With this option all spaces will be displayed.')),
+        OptionRecommendation(name='txt_in_remove_indents', recommended_value=False,
+            help=_('Normally extra space at the beginning of lines is retained. '
+                   'With this option they will be removed.')),
        OptionRecommendation(name="markdown_disable_toc", recommended_value=False,
            help=_('Do not insert a Table of Contents into the output text.')),
    ])
@ -54,8 +61,22 @@ class TXTInput(InputFormatPlugin):
    def convert(self, stream, options, file_ext, log,
                accelerators):
        self.log = log
+        txt = ''
        log.debug('Reading text from file...')
+        length = 0

+        # Extract content from zip archive.
+        if file_ext == 'txtz':
+            log.debug('De-compressing content to temporary directory...')
+            with TemporaryDirectory('_untxtz') as tdir:
+                zf = ZipFile(stream)
+                zf.extractall(tdir)
+
+                txts = glob.glob(os.path.join(tdir, '*.txt'))
+                for t in txts:
+                    with open(t, 'rb') as tf:
+                        txt += tf.read()
+        else:
            txt = stream.read()

        # Get the encoding of the document.
@ -77,20 +98,6 @@ class TXTInput(InputFormatPlugin):
        # Normalize line endings
        txt = normalize_line_endings(txt)

-        # Detect formatting
-        if options.formatting_type == 'auto':
-            options.formatting_type = detect_formatting_type(txt)
-            log.debug('Auto detected formatting as %s' % options.formatting_type)
-
-        if options.formatting_type == 'heuristic':
-            setattr(options, 'enable_heuristics', True)
-            setattr(options, 'markup_chapter_headings', True)
-            setattr(options, 'italicize_common_cases', True)
-            setattr(options, 'fix_indents', True)
-            setattr(options, 'delete_blank_paragraphs', True)
-            setattr(options, 'format_scene_breaks', True)
-            setattr(options, 'dehyphenate', True)
-
        # Determine the paragraph type of the document.
        if options.paragraph_type == 'auto':
            options.paragraph_type = detect_paragraph_type(txt)
@ -100,27 +107,52 @@ class TXTInput(InputFormatPlugin):
            else:
                log.debug('Auto detected paragraph type as %s' % options.paragraph_type)

-        # Preserve spaces will replace multiple spaces to a space
-        # followed by the &nbsp; entity.
-        if options.preserve_spaces:
-            txt = preserve_spaces(txt)
+        # Detect formatting
+        if options.formatting_type == 'auto':
+            options.formatting_type = detect_formatting_type(txt)
+            log.debug('Auto detected formatting as %s' % options.formatting_type)

-        # Get length for hyphen removal and punctuation unwrap
-        docanalysis = DocAnalysis('txt', txt)
-        length = docanalysis.line_length(.5)
+        if options.formatting_type == 'heuristic':
+            setattr(options, 'enable_heuristics', True)
+            setattr(options, 'unwrap_lines', False)
+            setattr(options, 'smarten_punctuation', True)

        # Reformat paragraphs to block formatting based on the detected type.
        # We don't check for block because the processor assumes block.
        # single and print at transformed to block for processing.
-        if options.paragraph_type == 'single' or options.paragraph_type == 'unformatted':
+        if options.paragraph_type == 'single':
            txt = separate_paragraphs_single_line(txt)
        elif options.paragraph_type == 'print':
+            txt = separate_hard_scene_breaks(txt)
            txt = separate_paragraphs_print_formatted(txt)
+            txt = block_to_single_line(txt)
        elif options.paragraph_type == 'unformatted':
            from calibre.ebooks.conversion.utils import HeuristicProcessor
            # unwrap lines based on punctuation
+            docanalysis = DocAnalysis('txt', txt)
+            length = docanalysis.line_length(.5)
            preprocessor = HeuristicProcessor(options, log=getattr(self, 'log', None))
            txt = preprocessor.punctuation_unwrap(length, txt, 'txt')
+            txt = separate_paragraphs_single_line(txt)
+        else:
+            txt = separate_hard_scene_breaks(txt)
+            txt = block_to_single_line(txt)
+
+        if getattr(options, 'enable_heuristics', False) and getattr(options, 'dehyphenate', False):
+            docanalysis = DocAnalysis('txt', txt)
+            if not length:
+                length = docanalysis.line_length(.5)
+            dehyphenator = Dehyphenator(options.verbose, log=self.log)
+            txt = dehyphenator(txt,'txt', length)
+
+        # User requested transformation on the text.
+        if options.txt_in_remove_indents:
+            txt = remove_indents(txt)
+
+        # Preserve spaces will replace multiple spaces to a space
+        # followed by the &nbsp; entity.
+        if options.preserve_spaces:
+            txt = preserve_spaces(txt)

        # Process the text using the appropriate text processor.
        html = ''
@ -134,14 +166,8 @@ class TXTInput(InputFormatPlugin):
        elif options.formatting_type == 'textile':
            log.debug('Running text through textile conversion...')
            html = convert_textile(txt)
-
        else:
            log.debug('Running text through basic conversion...')
-            if options.formatting_type == 'heuristic':
-                # Dehyphenate
-                dehyphenator = Dehyphenator(options.verbose, log=self.log)
-                txt = dehyphenator(txt,'txt', length)
-
            flow_size = getattr(options, 'flow_size', 0)
            html = convert_basic(txt, epub_split_size_kb=flow_size)

@ -169,4 +195,11 @@ class TXTInput(InputFormatPlugin):
                {})
        options.debug_pipeline = odi
        os.remove(htmlfile.name)
+        
+        # Set metadata from file.
+        from calibre.customize.ui import get_file_type_metadata
+        from calibre.ebooks.oeb.transforms.metadata import meta_info_to_oeb_metadata
+        mi = get_file_type_metadata(stream, file_ext)
+        meta_info_to_oeb_metadata(mi, oeb.metadata, log)
+        
        return oeb
--- a/src/calibre/ebooks/txt/markdownml.py
+++ b/src/calibre/ebooks/txt/markdownml.py
@ -35,11 +35,9 @@ class MarkdownMLizer(object):
            html = unicode(etree.tostring(item.data, encoding=unicode))
            
            if not self.opts.keep_links:
-                html = re.sub(r'<\s*a[^>]*>', '', html)
-                html = re.sub(r'<\s*/\s*a\s*>', '', html)
+                html = re.sub(r'<\s*/*\s*a[^>]*>', '', html)
            if not self.opts.keep_image_references:
-                html = re.sub(r'<\s*img[^>]*>', '', html)
-                html = re.sub(r'<\s*img\s*>', '', html)
+                html = re.sub(r'<\s*img[^>]*>', '', html)\
            
            text = html2text(html)
        
--- a/src/calibre/ebooks/txt/output.py
+++ b/src/calibre/ebooks/txt/output.py
@ -5,11 +5,18 @@ __copyright__ = '2009, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'

 import os
+import shutil
+
+from lxml import etree

 from calibre.customize.conversion import OutputFormatPlugin, \
    OptionRecommendation
+from calibre.ebooks.oeb.base import OEB_IMAGES 
 from calibre.ebooks.txt.txtml import TXTMLizer
 from calibre.ebooks.txt.newlines import TxtNewlines, specified_newlines
+from calibre.ptempfile import TemporaryDirectory, TemporaryFile
+from calibre.utils.cleantext import clean_ascii_chars
+from calibre.utils.zipfile import ZipFile

 class TXTOutput(OutputFormatPlugin):

@ -73,6 +80,7 @@ class TXTOutput(OutputFormatPlugin):
            writer = TXTMLizer(log)

        txt = writer.extract_content(oeb_book, opts)
+        txt = clean_ascii_chars(txt)

        log.debug('\tReplacing newlines with selected type...')
        txt = specified_newlines(TxtNewlines(opts.newline).newline, txt)
@ -93,3 +101,32 @@ class TXTOutput(OutputFormatPlugin):
        if close:
            out_stream.close()

+
+class TXTZOutput(TXTOutput):
+    
+    name = 'TXTZ Output'
+    author = 'John Schember'
+    file_type = 'txtz'
+
+    def convert(self, oeb_book, output_path, input_plugin, opts, log):
+        with TemporaryDirectory('_txtz_output') as tdir:
+            # TXT
+            with TemporaryFile('index.txt') as tf:
+                TXTOutput.convert(self, oeb_book, tf, input_plugin, opts, log)
+                shutil.copy(tf, os.path.join(tdir, 'index.txt'))
+
+            # Images
+            for item in oeb_book.manifest:
+                if item.media_type in OEB_IMAGES:
+                    path = os.path.join(tdir, os.path.dirname(item.href))
+                    if not os.path.exists(path):
+                        os.makedirs(path)
+                    with open(os.path.join(tdir, item.href), 'wb') as imgf:
+                        imgf.write(item.data)
+            
+            # Metadata
+            with open(os.path.join(tdir, 'metadata.opf'), 'wb') as mdataf: 
+                mdataf.write(etree.tostring(oeb_book.metadata.to_opf1()))
+            
+            txtz = ZipFile(output_path, 'w')
+            txtz.add_dir(tdir)
--- a/src/calibre/ebooks/txt/processor.py
+++ b/src/calibre/ebooks/txt/processor.py
@ -19,20 +19,23 @@ from calibre.utils.cleantext import clean_ascii_chars
 HTML_TEMPLATE = u'<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8"/><title>%s</title></head><body>\n%s\n</body></html>'

 def clean_txt(txt):
+    '''
+    Run transformations on the text to put it into
+    consistent state.
+    '''
    if isbytestring(txt):
        txt = txt.decode('utf-8', 'replace')
    # Strip whitespace from the end of the line. Also replace
    # all line breaks with \n.
    txt = '\n'.join([line.rstrip() for line in txt.splitlines()])
    
-    # Replace whitespace at the beginning of the list with &nbsp;
-    txt = re.sub('(?m)(?P<space>[ ]+)', lambda mo: '&nbsp;' * mo.groups('space').count(' '), txt)
-    txt = re.sub('(?m)(?P<space>[\t]+)', lambda mo: '&nbsp;' * 4 * mo.groups('space').count('\t'), txt)
+    # Replace whitespace at the beginning of the line with &nbsp;
+    txt = re.sub('(?m)(?<=^)([ ]{2,}|\t+)(?=.)', '&nbsp;' * 4, txt)

    # Condense redundant spaces
    txt = re.sub('[ ]{2,}', ' ', txt)

-    # Remove blank lines from the beginning and end of the document.
+    # Remove blank space from the beginning and end of the document.
    txt = re.sub('^\s+(?=.)', '', txt)
    txt = re.sub('(?<=.)\s+$', '', txt)
    # Remove excessive line breaks.
@ -43,6 +46,15 @@ def clean_txt(txt):
    return txt

 def split_txt(txt, epub_split_size_kb=0):
+    '''
+    Ensure there are split points for converting
+    to EPUB. A misdetected paragraph type can
+    result in the entire document being one giant
+    paragraph. In this case the EPUB parser will not
+    be able to determine where to split the file
+    to accomidate the EPUB file size limitation
+    and will fail.
+    '''
    #Takes care if there is no point to split
    if epub_split_size_kb > 0:
        if isinstance(txt, unicode):
@ -60,6 +72,12 @@ def split_txt(txt, epub_split_size_kb=0):
    return txt

 def convert_basic(txt, title='', epub_split_size_kb=0):
+    '''
+    Converts plain text to html by putting all paragraphs in
+    <p> tags. It condense and retains blank lines when necessary.
+    
+    Requires paragraphs to be in single line format.
+    '''
    txt = clean_txt(txt)
    txt = split_txt(txt, epub_split_size_kb)

@ -100,14 +118,37 @@ def separate_paragraphs_single_line(txt):
    return txt

 def separate_paragraphs_print_formatted(txt):
-    txt = re.sub(u'(?miu)^(\t+|[ ]{2,})(?=.)', '\n\t', txt)
+    txt = re.sub(u'(?miu)^(?P<indent>\t+|[ ]{2,})(?=.)', lambda mo: '\n%s' % mo.group('indent'), txt)
+    return txt
+
+def separate_hard_scene_breaks(txt):
+    def sep_break(line):
+        if len(line.strip()) > 0:
+            return '\n%s\n' % line
+        else:
+            return line
+    txt = re.sub(u'(?miu)^[ \t-=~\/]+$', lambda mo: sep_break(mo.group()), txt)
+    return txt
+
+def block_to_single_line(txt):
+    txt = re.sub(r'(?<=.)\n(?=.)', ' ', txt)
    return txt

 def preserve_spaces(txt):
+    '''
+    Replaces spaces multiple spaces with &nbsp; entities.
+    '''
    txt = re.sub('(?P<space>[ ]{2,})', lambda mo: ' ' + ('&nbsp;' * (len(mo.group('space')) - 1)), txt)
    txt = txt.replace('\t', '&nbsp;&nbsp;&nbsp;&nbsp;')
    return txt

+def remove_indents(txt):
+    '''
+    Remove whitespace at the beginning of each line.
+    '''
+    txt = re.sub('(?miu)^\s+', '', txt)
+    return txt
+
 def opf_writer(path, opf_name, manifest, spine, mi):
    opf = OPFCreator(path, mi)
    opf.create_manifest(manifest)
@ -116,6 +157,9 @@ def opf_writer(path, opf_name, manifest, spine, mi):
        opf.render(opffile)

 def split_string_separator(txt, size):
+    '''
+    Splits the text by putting \n\n at the point size.
+    '''
    if len(txt) > size:
        txt = ''.join([re.sub(u'\.(?P<ends>[^.]*)$', '.\n\n\g<ends>',
            txt[i:i+size], 1) for i in
@ -124,7 +168,7 @@ def split_string_separator(txt, size) :

 def detect_paragraph_type(txt):
    '''
-    Tries to determine the formatting of the document.
+    Tries to determine the paragraph type of the document.

    block: Paragraphs are separated by a blank line.
    single: Each line is a paragraph.
@ -167,6 +211,16 @@ def detect_paragraph_type(txt):


 def detect_formatting_type(txt):
+    '''
+    Tries to determine the formatting of the document.
+    
+    markdown: Markdown formatting is used.
+    textile: Textile formatting is used.
+    heuristic: When none of the above formatting types are
+               detected heuristic is returned.
+    '''
+    # Keep a count of the number of format specific object
+    # that are found in the text.
    markdown_count = 0
    textile_count = 0

@ -176,9 +230,9 @@ def detect_formatting_type(txt):
    markdown_count += len(re.findall('(?mu)^=+$', txt))
    markdown_count += len(re.findall('(?mu)^-+$', txt))
    # Images
-    markdown_count += len(re.findall('(?u)!\[.*?\]\(.+?\)', txt))
+    markdown_count += len(re.findall('(?u)!\[.*?\](\[|\()', txt))
    # Links
-    markdown_count += len(re.findall('(?u)(^|(?P<pre>[^!]))\[.*?\]\([^)]+\)', txt))
+    markdown_count += len(re.findall('(?u)^|[^!]\[.*?\](\[|\()', txt))

    # Check for textile
    # Headings
@ -186,10 +240,12 @@ def detect_formatting_type(txt):
    # Block quote.
    textile_count += len(re.findall(r'(?mu)^bq\.', txt))
    # Images
-    textile_count += len(re.findall(r'\![^\s]+(?=.*?/)(:[^\s]+)*', txt))
+    textile_count += len(re.findall(r'(?mu)(?<=\!)\S+(?=\!)', txt))
    # Links
-    textile_count += len(re.findall(r'"(?=".*?\()(\(.+?\))*[^\(]+?(\(.+?\))*":[^\s]+', txt))
+    textile_count += len(re.findall(r'"[^"]*":\S+', txt))

+    # Decide if either markdown or textile is used in the text
+    # based on the number of unique formatting elements found.
    if markdown_count > 5 or textile_count > 5:
        if markdown_count > textile_count:
            return 'markdown'
--- a/src/calibre/ebooks/txt/textileml.py
+++ b/src/calibre/ebooks/txt/textileml.py
@ -36,11 +36,9 @@ class TextileMLizer(object):
            html = unicode(etree.tostring(item.data.find(XHTML('body')), encoding=unicode))

            if not self.opts.keep_links:
-                html = re.sub(r'<\s*a[^>]*>', '', html)
-                html = re.sub(r'<\s*/\s*a\s*>', '', html)
+                html = re.sub(r'<\s*/*\s*a[^>]*>', '', html)
            if not self.opts.keep_image_references:
                html = re.sub(r'<\s*img[^>]*>', '', html)
-                html = re.sub(r'<\s*img\s*>', '', html)

            text = html2textile(html)

--- a/src/calibre/ebooks/txt/txtml.py
+++ b/src/calibre/ebooks/txt/txtml.py
@ -55,6 +55,7 @@ class TXTMLizer(object):
        self.log.info('Converting XHTML to TXT...')
        self.oeb_book = oeb_book
        self.opts = opts
+        self.toc_titles = []
        self.toc_ids = []
        self.last_was_heading = False
        
@ -94,8 +95,8 @@ class TXTMLizer(object):
        if getattr(self.opts, 'inline_toc', None):
            self.log.debug('Generating table of contents...')
            toc.append(u'%s\n\n' % _(u'Table of Contents:'))
-            for item in self.oeb_book.toc:
-                toc.append(u'* %s\n\n' % item.title)
+            for item in self.toc_titles:
+                toc.append(u'* %s\n\n' % item)
        return ''.join(toc)

    def create_flat_toc(self, nodes):
@ -103,6 +104,7 @@ class TXTMLizer(object):
        Turns a hierarchical list of TOC href's into a flat list.
        '''
        for item in nodes:
+            self.toc_titles.append(item.title)
            self.toc_ids.append(item.href)
            self.create_flat_toc(item.nodes)

--- a/src/calibre/gui2/init.py
+++ b/src/calibre/gui2/init.py
@ -50,6 +50,7 @@ gprefs.defaults['action-layout-context-menu-device'] = (

 gprefs.defaults['show_splash_screen'] = True
 gprefs.defaults['toolbar_icon_size'] = 'medium'
+gprefs.defaults['automerge'] = 'ignore'
 gprefs.defaults['toolbar_text'] = 'auto'
 gprefs.defaults['show_child_bar'] = False
 gprefs.defaults['font'] = None
@ -105,9 +106,13 @@ def _config():
                'clicked'))
    c.add_opt('asked_library_thing_password', default=False,
            help='Asked library thing password at least once.')
-    c.add_opt('search_as_you_type', default=True,
-            help='Start searching as you type. If this is disabled then search will '
-            'only take place when the Enter or Return key is pressed.')
+    c.add_opt('search_as_you_type', default=False,
+            help=_('Start searching as you type. If this is disabled then search will '
+            'only take place when the Enter or Return key is pressed.'))
+    c.add_opt('highlight_search_matches', default=False,
+            help=_('When searching, show all books with search results '
+            'highlighted instead of showing only the matches. You can use the '
+            'N or F3 keys to go to the next match.'))
    c.add_opt('save_to_disk_template_history', default=[],
        help='Previously used Save to Disk templates')
    c.add_opt('send_to_device_template_history', default=[],
--- a/src/calibre/gui2/actions/add.py
+++ b/src/calibre/gui2/actions/add.py
@ -244,8 +244,8 @@ class AddAction(InterfaceAction):
                    x.decode(preferred_encoding, 'replace') for x in
                    self._adder.merged_books])
            info_dialog(self.gui, _('Merged some books'),
-                    _('Some duplicates were found and merged into the '
-                        'following existing books:'), det_msg=books, show=True)
+                    _('The following duplicate books were found and incoming book formats were '
+                        'processed and merged into your Calibre database according to your automerge settings:'), det_msg=books, show=True)
        if getattr(self._adder, 'critical', None):
            det_msg = []
            for name, log in self._adder.critical.items():
--- a/src/calibre/gui2/actions/next_match.py
+++ b/src/calibre/gui2/actions/next_match.py
@ -28,21 +28,12 @@ class NextMatchAction(InterfaceAction):
        self.gui.addAction(self.p_action)
        self.p_action.triggered.connect(self.move_backward)

-    def gui_layout_complete(self):
-        self.gui.search_highlight_only.setVisible(True)
-
    def location_selected(self, loc):
        self.can_move = loc == 'library'
-        try:
-            self.gui.search_highlight_only.setVisible(self.can_move)
-        except:
-            import traceback
-            traceback.print_exc()

    def move_forward(self):
        if self.can_move is None:
            self.can_move = self.gui.current_view() is self.gui.library_view
-            self.gui.search_highlight_only.setVisible(self.can_move)

        if self.can_move:
            self.gui.current_view().move_highlighted_row(forward=True)
@ -50,7 +41,6 @@ class NextMatchAction(InterfaceAction):
    def move_backward(self):
        if self.can_move is None:
            self.can_move = self.gui.current_view() is self.gui.library_view
-            self.gui.search_highlight_only.setVisible(self.can_move)

        if self.can_move:
            self.gui.current_view().move_highlighted_row(forward=False)
--- a/src/calibre/gui2/actions/preferences.py
+++ b/src/calibre/gui2/actions/preferences.py
@ -33,7 +33,8 @@ class PreferencesAction(InterfaceAction):
            x.triggered.connect(self.do_config)


-    def do_config(self, checked=False, initial_plugin=None):
+    def do_config(self, checked=False, initial_plugin=None,
+            close_after_initial=False):
        if self.gui.job_manager.has_jobs():
            d = error_dialog(self.gui, _('Cannot configure'),
                    _('Cannot configure while there are running jobs.'))
@ -44,7 +45,8 @@ class PreferencesAction(InterfaceAction):
                    _('Cannot configure before calibre is restarted.'))
            d.exec_()
            return
-        d = Preferences(self.gui, initial_plugin=initial_plugin)
+        d = Preferences(self.gui, initial_plugin=initial_plugin,
+                close_after_initial=close_after_initial)
        d.show()
        d.run_wizard_requested.connect(self.gui.run_wizard,
                type=Qt.QueuedConnection)
--- a/src/calibre/gui2/add.py
+++ b/src/calibre/gui2/add.py
@ -8,7 +8,7 @@ from functools import partial
 from PyQt4.Qt import QThread, QObject, Qt, QProgressDialog, pyqtSignal, QTimer

 from calibre.gui2.dialogs.progress import ProgressDialog
-from calibre.gui2 import question_dialog, error_dialog, info_dialog
+from calibre.gui2 import question_dialog, error_dialog, info_dialog, gprefs
 from calibre.ebooks.metadata.opf2 import OPF
 from calibre.ebooks.metadata import MetaInformation
 from calibre.constants import preferred_encoding, filesystem_encoding, DEBUG
@ -179,23 +179,47 @@ class DBAdder(QObject): # {{{
                    cover = f.read()
            orig_formats = formats
            formats = [f for f in formats if not f.lower().endswith('.opf')]
-            if prefs['add_formats_to_existing']:
+            if prefs['add_formats_to_existing']: #automerge is on
                identical_book_list = self.db.find_identical_books(mi)
-
                if identical_book_list:  # books with same author and nearly same title exist in db
                    self.merged_books.add(mi.title)
+                    seen_fmts = set([])
+
                    for identical_book in identical_book_list:
-                        self.add_formats(identical_book, formats, replace=False)
-                else:
-                    id = self.db.create_book_entry(mi, cover=cover, add_duplicates=True)
+                        ib_fmts = self.db.formats(identical_book, index_is_id=True)
+                        if ib_fmts:
+                            seen_fmts |= set(ib_fmts.split(','))
+                        replace = gprefs['automerge'] == 'overwrite'
+                        self.add_formats(identical_book, formats,
+                                replace=replace)
+                    if gprefs['automerge'] == 'new record':
+                        incoming_fmts = \
+                            set([os.path.splitext(path)[-1].replace('.',
+                                '').upper() for path in formats])
+                        if incoming_fmts.intersection(seen_fmts):
+                            # There was at least one duplicate format
+                            # so create a new record and put the
+                            # incoming formats into it
+                            # We should arguably put only the duplicate
+                            # formats, but no real harm is done by having
+                            # all formats
+                            id_ = self.db.create_book_entry(mi, cover=cover,
+                                    add_duplicates=True)
                            self.number_of_books_added += 1
-                    self.add_formats(id, formats)
+                            self.add_formats(id_, formats)
+
                else:
-                id = self.db.create_book_entry(mi, cover=cover, add_duplicates=False)
-                if id is None:
+                    # books with same author and nearly same title do not exist in db
+                    id_ = self.db.create_book_entry(mi, cover=cover, add_duplicates=True)
+                    self.number_of_books_added += 1
+                    self.add_formats(id_, formats)
+
+            else: #automerge is off
+                id_ = self.db.create_book_entry(mi, cover=cover, add_duplicates=False)
+                if id_ is None:
                    self.duplicates.append((mi, cover, orig_formats))
                else:
-                    self.add_formats(id, formats)
+                    self.add_formats(id_, formats)
                    self.number_of_books_added += 1
        else:
            self.names.append(name)
--- a/src/calibre/gui2/complete.py
+++ b/src/calibre/gui2/complete.py
@ -6,157 +6,38 @@ __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'


-from PyQt4.Qt import QLineEdit, QListView, QAbstractListModel, Qt, QTimer, \
-        QApplication, QPoint, QItemDelegate, QStyleOptionViewItem, \
-        QStyle, QEvent, pyqtSignal
+from PyQt4.Qt import QLineEdit, QAbstractListModel, Qt, \
+        QApplication, QCompleter

 from calibre.utils.config import tweaks
 from calibre.utils.icu import sort_key, lower
 from calibre.gui2 import NONE
 from calibre.gui2.widgets import EnComboBox

-class CompleterItemDelegate(QItemDelegate): # {{{
-
-    ''' Renders the current item as thought it were selected '''
-
-    def __init__(self, view):
-        self.view = view
-        QItemDelegate.__init__(self, view)
-
-    def paint(self, p, opt, idx):
-        opt = QStyleOptionViewItem(opt)
-        opt.showDecorationSelected = True
-        if self.view.currentIndex() == idx:
-            opt.state |= QStyle.State_HasFocus
-        QItemDelegate.paint(self, p, opt, idx)
-
-# }}}
-
-class CompleteWindow(QListView): # {{{
-
-    '''
-    The completion popup. For keyboard and mouse handling see
-    :meth:`eventFilter`.
-    '''
-
-    #: This signal is emitted when the user selects one of the listed
-    #: completions, by mouse or keyboard
-    completion_selected = pyqtSignal(object)
-
-    def __init__(self, widget, model):
-        self.widget = widget
-        QListView.__init__(self)
-        self.setVisible(False)
-        self.setParent(None, Qt.Popup)
-        self.setAlternatingRowColors(True)
-        self.setFocusPolicy(Qt.NoFocus)
-        self._d = CompleterItemDelegate(self)
-        self.setItemDelegate(self._d)
-        self.setModel(model)
-        self.setFocusProxy(widget)
-        self.installEventFilter(self)
-        self.clicked.connect(self.do_selected)
-        self.entered.connect(self.do_entered)
-        self.setMouseTracking(True)
-
-    def do_entered(self, idx):
-        if idx.isValid():
-            self.setCurrentIndex(idx)
-
-    def do_selected(self, idx=None):
-        idx = self.currentIndex() if idx is None else idx
-        if idx.isValid():
-            data = unicode(self.model().data(idx, Qt.DisplayRole))
-            self.completion_selected.emit(data)
-        self.hide()
-
-    def eventFilter(self, o, e):
-        if o is not self:
-            return False
-        if e.type() == e.KeyPress:
-            key = e.key()
-            if key in (Qt.Key_Escape, Qt.Key_Backtab) or \
-                    (key == Qt.Key_F4 and (e.modifiers() & Qt.AltModifier)):
-                self.hide()
-                return True
-            elif key in (Qt.Key_Enter, Qt.Key_Return, Qt.Key_Tab):
-                if key == Qt.Key_Tab and not self.currentIndex().isValid():
-                    if self.model().rowCount() > 0:
-                        self.setCurrentIndex(self.model().index(0))
-                self.do_selected()
-                return True
-            elif key in (Qt.Key_Up, Qt.Key_Down, Qt.Key_PageUp,
-                    Qt.Key_PageDown):
-                return False
-            # Send key event to associated line edit
-            self.widget.eat_focus_out = False
-            try:
-                self.widget.event(e)
-            finally:
-                self.widget.eat_focus_out = True
-            if not self.widget.hasFocus():
-                # Line edit lost focus
-                self.hide()
-            if e.isAccepted():
-                # Line edit consumed event
-                return True
-        elif e.type() == e.MouseButtonPress:
-            # Hide popup if user clicks outside it, otherwise
-            # pass event to popup
-            if not self.underMouse():
-                self.hide()
-                return True
-        elif e.type() in (e.InputMethod, e.ShortcutOverride):
-            QApplication.sendEvent(self.widget, e)
-
-        return False # Do not filter this event
-
-# }}}
-
 class CompleteModel(QAbstractListModel):

    def __init__(self, parent=None):
        QAbstractListModel.__init__(self, parent)
-        self.sep = ','
-        self.space_before_sep = False
        self.items = []
-        self.lowered_items = []
-        self.matches = []

    def set_items(self, items):
        items = [unicode(x.strip()) for x in items]
        self.items = list(sorted(items, key=lambda x: sort_key(x)))
        self.lowered_items = [lower(x) for x in self.items]
-        self.matches = []
        self.reset()

    def rowCount(self, *args):
-        return len(self.matches)
+        return len(self.items)

    def data(self, index, role):
        if role == Qt.DisplayRole:
            r = index.row()
            try:
-                return self.matches[r]
+                return self.items[r]
            except IndexError:
                pass
        return NONE

-    def get_matches(self, prefix):
-        '''
-        Return all matches that (case insensitively) start with prefix
-        '''
-        prefix = lower(prefix)
-        ans = []
-        if prefix:
-            for i, test in enumerate(self.lowered_items):
-                if test.startswith(prefix):
-                    ans.append(self.items[i])
-        return ans
-
-    def update_matches(self, matches):
-        self.matches = matches
-        self.reset()

 class MultiCompleteLineEdit(QLineEdit):
    '''
@ -170,16 +51,26 @@ class MultiCompleteLineEdit(QLineEdit):
    '''

    def __init__(self, parent=None):
-        self.eat_focus_out = True
-        self.max_visible_items = 7
-        self.current_prefix = None
        QLineEdit.__init__(self, parent)

+        self.sep = ','
+        self.space_before_sep = False
+
        self._model = CompleteModel(parent=self)
-        self.complete_window = CompleteWindow(self, self._model)
+        self._completer = c = QCompleter(self._model, self)
+        c.setWidget(self)
+        c.setCompletionMode(QCompleter.PopupCompletion)
+        c.setCaseSensitivity(Qt.CaseInsensitive)
+        c.setModelSorting(QCompleter.CaseInsensitivelySortedModel)
+        c.setCompletionRole(Qt.DisplayRole)
+        p = c.popup()
+        p.setMouseTracking(True)
+        p.entered.connect(self.item_entered)
+        c.popup().setAlternatingRowColors(True)
+
+        c.activated.connect(self.completion_selected,
+                type=Qt.QueuedConnection)
        self.textEdited.connect(self.text_edited)
-        self.complete_window.completion_selected.connect(self.completion_selected)
-        self.installEventFilter(self)

    # Interface {{{
    def update_items_cache(self, complete_items):
@ -193,33 +84,23 @@ class MultiCompleteLineEdit(QLineEdit):

    # }}}

-    def eventFilter(self, o, e):
-        if self.eat_focus_out and o is self and e.type() == QEvent.FocusOut:
-            if self.complete_window.isVisible():
-                return True # Filter this event since the cw is visible
-        return QLineEdit.eventFilter(self, o, e)
-
-    def hide_completion_window(self):
-        self.complete_window.hide()
-
+    def item_entered(self, idx):
+        self._completer.popup().setCurrentIndex(idx)

    def text_edited(self, *args):
        self.update_completions()
+        self._completer.complete()

    def update_completions(self):
        ' Update the list of completions '
-        if not self.complete_window.isVisible() and not self.hasFocus():
-            return
        cpos = self.cursorPosition()
        text = unicode(self.text())
        prefix = text[:cpos]
        self.current_prefix = prefix
        complete_prefix = prefix.lstrip()
        if self.sep:
-            complete_prefix = prefix = prefix.split(self.sep)[-1].lstrip()
-
-        matches = self._model.get_matches(complete_prefix)
-        self.update_complete_window(matches)
+            complete_prefix = prefix.split(self.sep)[-1].lstrip()
+        self._completer.setCompletionPrefix(complete_prefix)

    def get_completed_text(self, text):
        '''
@ -246,7 +127,7 @@ class MultiCompleteLineEdit(QLineEdit):


    def completion_selected(self, text):
-        prefix_len, ctext = self.get_completed_text(text)
+        prefix_len, ctext = self.get_completed_text(unicode(text))
        if self.sep is None:
            self.setText(ctext)
            self.setCursorPosition(len(ctext))
@ -255,60 +136,6 @@ class MultiCompleteLineEdit(QLineEdit):
            self.setText(ctext)
            self.setCursorPosition(cursor_pos - prefix_len + len(text))

-    def update_complete_window(self, matches):
-        self._model.update_matches(matches)
-        if matches:
-            self.show_complete_window()
-        else:
-            self.complete_window.hide()
-
-
-    def position_complete_window(self):
-        popup = self.complete_window
-        screen = QApplication.desktop().availableGeometry(self)
-        h = (popup.sizeHintForRow(0) * min(self.max_visible_items,
-            popup.model().rowCount()) + 3) + 3
-        hsb = popup.horizontalScrollBar()
-        if hsb and hsb.isVisible():
-            h += hsb.sizeHint().height()
-
-        rh = self.height()
-        pos = self.mapToGlobal(QPoint(0, self.height() - 2))
-        w = self.width()
-
-        if w > screen.width():
-            w = screen.width()
-        if (pos.x() + w) > (screen.x() + screen.width()):
-            pos.setX(screen.x() + screen.width() - w)
-        if (pos.x() < screen.x()):
-            pos.setX(screen.x())
-
-        top = pos.y() - rh - screen.top() + 2
-        bottom = screen.bottom() - pos.y()
-        h = max(h, popup.minimumHeight())
-        if h > bottom:
-            h = min(max(top, bottom), h)
-            if top > bottom:
-                pos.setY(pos.y() - h - rh + 2)
-
-        popup.setGeometry(pos.x(), pos.y(), w, h)
-
-
-    def show_complete_window(self):
-        self.position_complete_window()
-        self.complete_window.show()
-
-    def moveEvent(self, ev):
-        ret = QLineEdit.moveEvent(self, ev)
-        QTimer.singleShot(0, self.position_complete_window)
-        return ret
-
-    def resizeEvent(self, ev):
-        ret = QLineEdit.resizeEvent(self, ev)
-        QTimer.singleShot(0, self.position_complete_window)
-        return ret
-
-
    @dynamic_property
    def all_items(self):
        def fget(self):
@ -317,22 +144,6 @@ class MultiCompleteLineEdit(QLineEdit):
            self._model.set_items(items)
        return property(fget=fget, fset=fset)

-    @dynamic_property
-    def sep(self):
-        def fget(self):
-            return self._model.sep
-        def fset(self, val):
-            self._model.sep = val
-        return property(fget=fget, fset=fset)
-
-    @dynamic_property
-    def space_before_sep(self):
-        def fget(self):
-            return self._model.space_before_sep
-        def fset(self, val):
-            self._model.space_before_sep = val
-        return property(fget=fget, fset=fset)
-
 class MultiCompleteComboBox(EnComboBox):

    def __init__(self, *args):
--- a/src/calibre/gui2/convert/txt_input.py
+++ b/src/calibre/gui2/convert/txt_input.py
@ -16,7 +16,8 @@ class PluginWidget(Widget, Ui_Form):

    def __init__(self, parent, get_option, get_help, db=None, book_id=None):
        Widget.__init__(self, parent,
-            ['paragraph_type', 'formatting_type', 'markdown_disable_toc', 'preserve_spaces'])
+            ['paragraph_type', 'formatting_type', 'markdown_disable_toc',
+             'preserve_spaces', 'txt_in_remove_indents'])
        self.db, self.book_id = db, book_id
        for x in get_option('paragraph_type').option.choices:
            self.opt_paragraph_type.addItem(x)
--- a/src/calibre/gui2/convert/txt_input.ui
+++ b/src/calibre/gui2/convert/txt_input.ui
@ -7,57 +7,95 @@
    <x>0</x>
    <y>0</y>
    <width>518</width>
-    <height>300</height>
+    <height>353</height>
   </rect>
  </property>
  <property name="windowTitle">
   <string>Form</string>
  </property>
+  <layout class="QVBoxLayout" name="verticalLayout_3">
+   <item>
+    <widget class="QGroupBox" name="groupBox_3">
+     <property name="title">
+      <string>Structure</string>
+     </property>
     <layout class="QGridLayout" name="gridLayout">
      <item row="0" column="0">
       <widget class="QLabel" name="label_2">
+        <property name="sizePolicy">
+         <sizepolicy hsizetype="Minimum" vsizetype="Preferred">
+          <horstretch>0</horstretch>
+          <verstretch>0</verstretch>
+         </sizepolicy>
+        </property>
        <property name="text">
         <string>Paragraph style:</string>
        </property>
       </widget>
      </item>
      <item row="0" column="1">
-    <widget class="QComboBox" name="opt_paragraph_type"/>
+       <widget class="QComboBox" name="opt_paragraph_type">
+        <property name="sizePolicy">
+         <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
+          <horstretch>0</horstretch>
+          <verstretch>0</verstretch>
+         </sizepolicy>
+        </property>
+       </widget>
      </item>
-   <item row="5" column="0" colspan="2">
+      <item row="1" column="0">
+       <widget class="QLabel" name="label_3">
+        <property name="sizePolicy">
+         <sizepolicy hsizetype="Minimum" vsizetype="Preferred">
+          <horstretch>0</horstretch>
+          <verstretch>0</verstretch>
+         </sizepolicy>
+        </property>
+        <property name="text">
+         <string>Formatting style:</string>
+        </property>
+       </widget>
+      </item>
+      <item row="1" column="1">
+       <widget class="QComboBox" name="opt_formatting_type">
+        <property name="sizePolicy">
+         <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
+          <horstretch>0</horstretch>
+          <verstretch>0</verstretch>
+         </sizepolicy>
+        </property>
+       </widget>
+      </item>
+     </layout>
+    </widget>
+   </item>
+   <item>
+    <widget class="QGroupBox" name="groupBox_2">
+     <property name="title">
+      <string>Common</string>
+     </property>
+     <layout class="QVBoxLayout" name="verticalLayout_2">
+      <item>
       <widget class="QCheckBox" name="opt_preserve_spaces">
        <property name="text">
         <string>Preserve &amp;spaces</string>
        </property>
       </widget>
      </item>
-   <item row="6" column="0" colspan="2">
-    <spacer name="verticalSpacer">
-     <property name="orientation">
-      <enum>Qt::Vertical</enum>
-     </property>
-     <property name="sizeHint" stdset="0">
-      <size>
-       <width>20</width>
-       <height>213</height>
-      </size>
-     </property>
-    </spacer>
-   </item>
-   <item row="1" column="1">
-    <widget class="QComboBox" name="opt_formatting_type"/>
-   </item>
-   <item row="1" column="0">
-    <widget class="QLabel" name="label_3">
+      <item>
+       <widget class="QCheckBox" name="opt_txt_in_remove_indents">
        <property name="text">
-      <string>Formatting style:</string>
+         <string>Remove indents at the beginning of lines</string>
        </property>
       </widget>
      </item>
-   <item row="2" column="0" rowspan="2" colspan="2">
+     </layout>
+    </widget>
+   </item>
+   <item>
    <widget class="QGroupBox" name="groupBox">
     <property name="title">
-      <string>Markdown Options</string>
+      <string>Markdown</string>
     </property>
     <layout class="QVBoxLayout" name="verticalLayout">
      <item>
@ -83,6 +121,19 @@
     </layout>
    </widget>
   </item>
+   <item>
+    <spacer name="verticalSpacer">
+     <property name="orientation">
+      <enum>Qt::Vertical</enum>
+     </property>
+     <property name="sizeHint" stdset="0">
+      <size>
+       <width>20</width>
+       <height>213</height>
+      </size>
+     </property>
+    </spacer>
+   </item>
  </layout>
 </widget>
 <resources/>
--- a/src/calibre/gui2/dialogs/message_box.py
+++ b/src/calibre/gui2/dialogs/message_box.py
@ -89,6 +89,7 @@ class MessageBox(QDialog, Ui_Dialog):
                (__version__, unicode(self.windowTitle()),
                    unicode(self.msg.text()),
                    unicode(self.det_msg.toPlainText())))
+        if hasattr(self, 'ctc_button'):
            self.ctc_button.setText(_('Copied'))

    def showEvent(self, ev):
--- a/src/calibre/gui2/dialogs/metadata_single.py
+++ b/src/calibre/gui2/dialogs/metadata_single.py
@ -616,6 +616,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
        self.original_series_name = unicode(self.series.text()).strip()
        if len(db.custom_column_label_map) == 0:
            self.central_widget.tabBar().setVisible(False)
+            self.central_widget.setTabEnabled(1, False)
        else:
            self.create_custom_column_editors()
        self.generate_cover_button.clicked.connect(self.generate_cover)
@ -780,8 +781,8 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
                    _('You have changed the tags. In order to use the tags'
                       ' editor, you must either discard or apply these '
                       'changes. Apply changes?'), show_copy_button=False):
-                self.books_to_refresh |= self.apply_tags(commit=True, notify=True,
-                                                         allow_case_change=True)
+                self.books_to_refresh |= self.apply_tags(commit=True,
+                        notify=True)
                self.original_tags = unicode(self.tags.text())
            else:
                self.tags.setText(self.original_tags)
--- a/src/calibre/gui2/dialogs/tag_list_editor.py
+++ b/src/calibre/gui2/dialogs/tag_list_editor.py
@ -1,7 +1,7 @@
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'

-from PyQt4.QtCore import SIGNAL, Qt
+from PyQt4.QtCore import Qt, QString
 from PyQt4.QtGui import QDialog, QListWidgetItem

 from calibre.gui2.dialogs.tag_list_editor_ui import Ui_TagListEditor
@ -11,30 +11,38 @@ class ListWidgetItem(QListWidgetItem):

    def __init__(self, txt):
        QListWidgetItem.__init__(self, txt)
-        self.old_value = txt
-        self.cur_value = txt
+        self.initial_value = QString(txt)
+        self.current_value = QString(txt)
+        self.previous_value = QString(txt)

    def data(self, role):
        if role == Qt.DisplayRole:
-            if self.old_value != self.cur_value:
-                return _('%s (was %s)')%(self.cur_value, self.old_value)
+            if self.initial_value != self.current_value:
+                return _('%s (was %s)')%(self.current_value, self.initial_value)
            else:
-                return self.cur_value
+                return self.current_value
        elif role == Qt.EditRole:
-            return self.cur_value
+            return self.current_value
        else:
            return QListWidgetItem.data(self, role)

    def setData(self, role, data):
        if role == Qt.EditRole:
-            self.cur_value = data.toString()
+            self.previous_value = self.current_value
+            self.current_value = data.toString()
        QListWidgetItem.setData(self, role, data)

    def text(self):
-        return self.cur_value
+        return self.current_value
+
+    def initial_text(self):
+        return self.initial_value
+
+    def previous_text(self):
+        return self.previous_value

    def setText(self, txt):
-        self.cur_value = txt
+        self.current_value = txt
        QListWidgetItem.setText(txt)

 class TagListEditor(QDialog, Ui_TagListEditor):
@ -49,7 +57,7 @@ class TagListEditor(QDialog, Ui_TagListEditor):
        self.setWindowIcon(icon)

        self.to_rename = {}
-        self.to_delete = []
+        self.to_delete = set([])
        self.all_tags = {}

        for k,v in data:
@ -57,6 +65,7 @@ class TagListEditor(QDialog, Ui_TagListEditor):
        for tag in sorted(self.all_tags.keys(), key=key):
            item = ListWidgetItem(tag)
            item.setData(Qt.UserRole, self.all_tags[tag])
+            item.setFlags (item.flags() | Qt.ItemIsEditable)
            self.available_tags.addItem(item)

        if tag_to_match is not None:
@ -64,23 +73,20 @@ class TagListEditor(QDialog, Ui_TagListEditor):
            if len(items) == 1:
                self.available_tags.setCurrentItem(items[0])

-        self.connect(self.delete_button,  SIGNAL('clicked()'), self.delete_tags)
-        self.connect(self.rename_button,  SIGNAL('clicked()'), self.rename_tag)
-        self.connect(self.available_tags, SIGNAL('itemDoubleClicked(QListWidgetItem *)'), self._rename_tag)
-        self.connect(self.available_tags, SIGNAL('itemChanged(QListWidgetItem *)'), self.finish_editing)
+        self.delete_button.clicked.connect(self.delete_tags)
+        self.rename_button.clicked.connect(self.rename_tag)
+        self.available_tags.itemDoubleClicked.connect(self._rename_tag)
+        self.available_tags.itemChanged.connect(self.finish_editing)

    def finish_editing(self, item):
        if not item.text():
                error_dialog(self, _('Item is blank'),
                             _('An item cannot be set to nothing. Delete it instead.')).exec_()
-                item.setText(self.item_before_editing.text())
+                item.setText(item.previous_text())
                return
-        if item.text() != self.item_before_editing.text():
-            (id,ign) = self.item_before_editing.data(Qt.UserRole).toInt()
-            if item.text() not in self.to_rename:
-                self.to_rename[item.text()] = [id]
-            else:
-                self.to_rename[item.text()].append(id)
+        if item.text() != item.initial_text():
+            id_ = item.data(Qt.UserRole).toInt()[0]
+            self.to_rename[id_] = unicode(item.text())

    def rename_tag(self):
        item = self.available_tags.currentItem()
@ -91,8 +97,6 @@ class TagListEditor(QDialog, Ui_TagListEditor):
            error_dialog(self, _('No item selected'),
                         _('You must select one item from the list of Available items.')).exec_()
            return
-        self.item_before_editing = item.clone()
-        item.setFlags (item.flags() | Qt.ItemIsEditable);
        self.available_tags.editItem(item)

    def delete_tags(self, item=None):
@ -108,7 +112,7 @@ class TagListEditor(QDialog, Ui_TagListEditor):
        row = self.available_tags.row(deletes[0])
        for item in deletes:
            (id,ign) = item.data(Qt.UserRole).toInt()
-            self.to_delete.append(id)
+            self.to_delete.add(id)
            self.available_tags.takeItem(self.available_tags.row(item))

        if row >= self.available_tags.count():
--- a/src/calibre/gui2/init.py
+++ b/src/calibre/gui2/init.py
@ -64,6 +64,7 @@ class LibraryViewMixin(object): # {{{
            view.verticalHeader().sectionDoubleClicked.connect(self.iactions['View'].view_specific_book)

        self.build_context_menus()
+        self.library_view.model().set_highlight_only(config['highlight_search_matches'])

    def build_context_menus(self):
        lm = QMenu(self)
--- a/src/calibre/gui2/layout.py
+++ b/src/calibre/gui2/layout.py
@ -8,7 +8,7 @@ __docformat__ = 'restructuredtext en'
 from functools import partial

 from PyQt4.Qt import QIcon, Qt, QWidget, QToolBar, QSize, \
-    pyqtSignal, QToolButton, QMenu, QCheckBox, \
+    pyqtSignal, QToolButton, QMenu, \
    QObject, QVBoxLayout, QSizePolicy, QLabel, QHBoxLayout, QActionGroup


@ -156,7 +156,8 @@ class SearchBar(QWidget): # {{{
        x = ComboBoxWithHelp(self)
        x.setMaximumSize(QSize(150, 16777215))
        x.setObjectName("search_restriction")
-        x.setToolTip(_("Books display will be restricted to those matching the selected saved search"))
+        x.setToolTip(_('Books display will be restricted to those matching the '
+                       'selected saved search'))
        l.addWidget(x)
        parent.search_restriction = x

@ -175,7 +176,8 @@ class SearchBar(QWidget): # {{{
        x = parent.search = SearchBox2(self)
        x.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Minimum)
        x.setObjectName("search")
-        x.setToolTip(_("<p>Search the list of books by title, author, publisher, tags, comments, etc.<br><br>Words separated by spaces are ANDed"))
+        x.setToolTip(_("<p>Search the list of books by title, author, publisher, "
+                       "tags, comments, etc.<br><br>Words separated by spaces are ANDed"))
        l.addWidget(x)

        self.search_button = QToolButton()
@ -194,13 +196,11 @@ class SearchBar(QWidget): # {{{
        l.addWidget(x)
        x.setToolTip(_("Reset Quick Search"))

-        x = parent.search_highlight_only = QCheckBox()
-        x.setText(_('&Highlight'))
-        x.setToolTip('<p>'+_('When searching, highlight matched books, instead '
-            'of restricting the book list to the matches.<p> You can use the '
-            'N or F3 keys to go to the next match.'))
+        x = parent.search_options_button = QToolButton(self)
+        x.setIcon(QIcon(I('config.png')))
+        x.setObjectName("search_option_button")
        l.addWidget(x)
-        x.setVisible(False)
+        x.setToolTip(_("Change the way searching for books works"))

        x = parent.saved_search = SavedSearchBox(self)
        x.setMaximumSize(QSize(150, 16777215))
@ -227,7 +227,6 @@ class SearchBar(QWidget): # {{{
        x.setToolTip(_("Delete current saved search"))


-
 # }}}

 class Spacer(QWidget): # {{{
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@ -238,8 +238,6 @@ class BooksModel(QAbstractTableModel): # {{{

    def set_highlight_only(self, toWhat):
        self.highlight_only = toWhat
-        if self.last_search:
-            self.research()

    def get_current_highlighted_id(self):
        if len(self.ids_to_highlight) == 0 or self.current_highlighted_idx is None:
--- a/src/calibre/gui2/metadata/bulk_download.py
+++ b/src/calibre/gui2/metadata/bulk_download.py
@ -11,7 +11,7 @@ from threading import Thread
 from Queue import Queue, Empty
 from functools import partial

-from PyQt4.Qt import QObject, Qt, pyqtSignal, QTimer, QDialog, \
+from PyQt4.Qt import QObject, QTimer, QDialog, \
        QVBoxLayout, QTextBrowser, QLabel, QGroupBox, QDialogButtonBox

 from calibre.ebooks.metadata.fetch import search, get_social_metadata
@ -163,27 +163,23 @@ class DownloadMetadata(Thread):

 class DoDownload(QObject):

-    idle_process = pyqtSignal()
-
    def __init__(self, parent, title, db, ids, get_covers, set_metadata=True,
            get_social_metadata=True):
        QObject.__init__(self, parent)
        self.pd = ProgressDialog(title, min=0, max=0, parent=parent)
        self.pd.canceled_signal.connect(self.cancel)
-        self.idle_process.connect(self.do_one, type=Qt.QueuedConnection)
        self.downloader = None
        self.create = partial(DownloadMetadata, db, ids, get_covers,
                set_metadata=set_metadata,
                get_social_metadata=get_social_metadata)
-        self.timer = QTimer(self)
        self.get_covers = get_covers
-        self.timer.timeout.connect(self.do_one, type=Qt.QueuedConnection)
        self.db = db
        self.updated = set([])
        self.total = len(ids)
+        self.keep_going = True

    def exec_(self):
-        self.timer.start(50)
+        QTimer.singleShot(50, self.do_one)
        ret = self.pd.exec_()
        if getattr(self.downloader, 'exception', None) is not None and \
                ret == self.pd.Accepted:
@ -194,11 +190,14 @@ class DoDownload(QObject):
        return ret

    def cancel(self, *args):
-        self.timer.stop()
+        self.keep_going = False
        self.downloader.keep_going = False
        self.pd.reject()

    def do_one(self):
+        try:
+            if not self.keep_going:
+                return
            if self.downloader is None:
                self.downloader = self.create()
                self.downloader.start()
@ -210,7 +209,6 @@ class DoDownload(QObject):
            except Empty:
                pass
            if not self.downloader.is_alive():
-            self.timer.stop()
                while True:
                    try:
                        r = self.downloader.results.get_nowait()
@ -218,6 +216,11 @@ class DoDownload(QObject):
                    except Empty:
                        break
                self.pd.accept()
+                return
+        except:
+            self.cancel()
+            raise
+        QTimer.singleShot(50, self.do_one)

    def handle_result(self, r):
        id_, typ, ok, title = r
--- a/src/calibre/gui2/metadata/single.py
+++ b/src/calibre/gui2/metadata/single.py
@ -197,7 +197,7 @@ class MetadataSingleDialogBase(ResizableDialog):
        self.books_to_refresh = set([])
        for widget in self.basic_metadata_widgets:
            widget.initialize(self.db, id_)
-        for widget in self.custom_metadata_widgets:
+        for widget in getattr(self, 'custom_metadata_widgets', []):
            widget.initialize(id_)
        # Commented out as it doesn't play nice with Next, Prev buttons
        #self.fetch_metadata_button.setFocus(Qt.OtherFocusReason)
--- a/src/calibre/gui2/preferences/adding.py
+++ b/src/calibre/gui2/preferences/adding.py
@ -12,6 +12,7 @@ from calibre.gui2.preferences import ConfigWidgetBase, test_widget, \
 from calibre.gui2.preferences.adding_ui import Ui_Form
 from calibre.utils.config import prefs
 from calibre.gui2.widgets import FilenamePattern
+from calibre.gui2 import gprefs

 class ConfigWidget(ConfigWidgetBase, Ui_Form):

@ -23,18 +24,23 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
        r('read_file_metadata', prefs)
        r('swap_author_names', prefs)
        r('add_formats_to_existing', prefs)
+        choices = [
+                (_('Ignore duplicate incoming formats'), 'ignore'),
+                (_('Overwrite existing duplicate formats'), 'overwrite'),
+                (_('Create new record for each duplicate format'), 'new record')]
+        r('automerge', gprefs, choices=choices)
        r('new_book_tags', prefs, setting=CommaSeparatedList)

        self.filename_pattern = FilenamePattern(self)
        self.metadata_box.layout().insertWidget(0, self.filename_pattern)
        self.filename_pattern.changed_signal.connect(self.changed_signal.emit)

-
    def initialize(self):
        ConfigWidgetBase.initialize(self)
        self.filename_pattern.blockSignals(True)
        self.filename_pattern.initialize()
        self.filename_pattern.blockSignals(False)
+        self.opt_automerge.setEnabled(self.opt_add_formats_to_existing.isChecked())

    def restore_defaults(self):
        ConfigWidgetBase.restore_defaults(self)
--- a/src/calibre/gui2/preferences/adding.ui
+++ b/src/calibre/gui2/preferences/adding.ui
@ -6,7 +6,7 @@
   <rect>
    <x>0</x>
    <y>0</y>
-    <width>750</width>
+    <width>753</width>
    <height>339</height>
   </rect>
  </property>
@ -58,16 +58,33 @@
     </item>
    </layout>
   </item>
-   <item row="2" column="0" colspan="2">
+   <item row="2" column="0">
    <widget class="QCheckBox" name="opt_add_formats_to_existing">
     <property name="toolTip">
-      <string>If an existing book with a similar title and author is found that does not have the format being added, the format is added
-to the existing book, instead of creating a new entry. If the existing book already has the format, then it is silently ignored.
+      <string>Automerge: If books with similar titles and authors found, merge the incoming formats automatically into
+existing book records. The box to the right controls what happens when an existing record already has
+the incoming format. Note that this option also affects the Copy to library action.

 Title match ignores leading indefinite articles (&quot;the&quot;, &quot;a&quot;, &quot;an&quot;), punctuation, case, etc. Author match is exact.</string>
     </property>
     <property name="text">
-      <string>If books with similar titles and authors found, &amp;merge the new files automatically</string>
+      <string>&amp;Automerge added books if they already exist in the calibre library:</string>
+     </property>
+    </widget>
+   </item>
+   <item row="2" column="1">
+    <widget class="QComboBox" name="opt_automerge">
+     <property name="toolTip">
+      <string>Automerge: If books with similar titles and authors found, merge the incoming formats automatically into
+existing book records. This box controls what happens when an existing record already has
+the incoming format: 
+
+Ignore duplicate incoming files - means that existing files in your calibre library will not be replaced
+Overwrite existing duplicate files - means that existing files in your calibre library will be replaced
+Create new record for each duplicate file - means that a new book entry will be created for each duplicate file
+
+Title matching ignores leading indefinite articles (&quot;the&quot;, &quot;a&quot;, &quot;an&quot;), punctuation, case, etc.
+Author matching is exact.</string>
     </property>
    </widget>
   </item>
@ -113,5 +130,22 @@ Title match ignores leading indefinite articles (&quot;the&quot;, &quot;a&quot;,
  </layout>
 </widget>
 <resources/>
- <connections/>
+ <connections>
+  <connection>
+   <sender>opt_add_formats_to_existing</sender>
+   <signal>toggled(bool)</signal>
+   <receiver>opt_automerge</receiver>
+   <slot>setEnabled(bool)</slot>
+   <hints>
+    <hint type="sourcelabel">
+     <x>406</x>
+     <y>83</y>
+    </hint>
+    <hint type="destinationlabel">
+     <x>457</x>
+     <y>83</y>
+    </hint>
+   </hints>
+  </connection>
+ </connections>
 </ui>
--- a/src/calibre/gui2/preferences/look_feel.py
+++ b/src/calibre/gui2/preferences/look_feel.py
@ -46,7 +46,6 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
        r('disable_tray_notification', config)
        r('use_roman_numerals_for_series_number', config)
        r('separate_cover_flow', config, restart_required=True)
-        r('search_as_you_type', config)
        r('show_child_bar', gprefs)

        choices = [(_('Small'), 'small'), (_('Medium'), 'medium'),
@ -116,7 +115,6 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):


    def refresh_gui(self, gui):
-        gui.search.search_as_you_type(config['search_as_you_type'])
        self.update_font_display()
        gui.tags_view.reread_collapse_parameters()

--- a/src/calibre/gui2/preferences/look_feel.ui
+++ b/src/calibre/gui2/preferences/look_feel.ui
@ -124,23 +124,13 @@
     </property>
    </widget>
   </item>
-   <item row="6" column="0">
+   <item row="6" column="0" colspan="2">
    <widget class="QCheckBox" name="opt_separate_cover_flow">
     <property name="text">
      <string>Show cover &amp;browser in a separate window (needs restart)</string>
     </property>
    </widget>
   </item>
-   <item row="6" column="1">
-    <widget class="QCheckBox" name="opt_search_as_you_type">
-     <property name="text">
-      <string>Search as you type</string>
-     </property>
-     <property name="checked">
-      <bool>true</bool>
-     </property>
-    </widget>
-   </item>
   <item row="7" column="0" colspan="2">
    <layout class="QHBoxLayout">
     <item>
--- a/src/calibre/gui2/preferences/main.py
+++ b/src/calibre/gui2/preferences/main.py
@ -157,11 +157,12 @@ class Preferences(QMainWindow):

    run_wizard_requested = pyqtSignal()

-    def __init__(self, gui, initial_plugin=None):
+    def __init__(self, gui, initial_plugin=None, close_after_initial=False):
        QMainWindow.__init__(self, gui)
        self.gui = gui
        self.must_restart = False
        self.committed = False
+        self.close_after_initial = close_after_initial

        self.resize(900, 720)
        nh, nw = min_available_height()-25, available_width()-10
@ -306,7 +307,7 @@ class Preferences(QMainWindow):

    def esc(self, *args):
        if self.stack.currentIndex() == 1:
-            self.hide_plugin()
+            self.cancel()
        elif self.stack.currentIndex() == 0:
            self.close()

@ -331,11 +332,14 @@ class Preferences(QMainWindow):
                    show_copy_button=False)
        self.showing_widget.refresh_gui(self.gui)
        self.hide_plugin()
-        if must_restart and rc:
+        if self.close_after_initial or (must_restart and rc):
            self.close()


    def cancel(self, *args):
+        if self.close_after_initial:
+            self.close()
+        else:
            self.hide_plugin()

    def restore_defaults(self, *args):
--- a/src/calibre/gui2/preferences/plugins.py
+++ b/src/calibre/gui2/preferences/plugins.py
@ -329,7 +329,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
                    return error_dialog(self, _('Must restart'),
                            _('You must restart calibre before you can'
                                ' configure the <b>%s</b> plugin')%plugin.name, show=True)
-                if plugin.do_user_config():
+                if plugin.do_user_config(self.gui):
                    self._plugin_model.refresh_plugin(plugin)
            elif op == 'remove':
                msg = _('Plugin <b>{0}</b> successfully removed').format(plugin.name)
--- a/src/calibre/gui2/preferences/search.py
+++ b/src/calibre/gui2/preferences/search.py
@ -0,0 +1,38 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from PyQt4.Qt import QApplication
+
+from calibre.gui2.preferences import ConfigWidgetBase, test_widget, \
+        CommaSeparatedList
+from calibre.gui2.preferences.search_ui import Ui_Form
+from calibre.gui2 import config
+from calibre.utils.config import prefs
+
+class ConfigWidget(ConfigWidgetBase, Ui_Form):
+
+    def genesis(self, gui):
+        self.gui = gui
+
+        r = self.register
+
+        r('search_as_you_type', config)
+        r('highlight_search_matches', config)
+        r('limit_search_columns', prefs)
+        r('limit_search_columns_to', prefs, setting=CommaSeparatedList)
+        fl = gui.library_view.model().db.field_metadata.get_search_terms()
+        self.opt_limit_search_columns_to.update_items_cache(fl)
+
+    def refresh_gui(self, gui):
+        gui.search.search_as_you_type(config['search_as_you_type'])
+        gui.library_view.model().set_highlight_only(config['highlight_search_matches'])
+        gui.search.do_search()
+
+if __name__ == '__main__':
+    app = QApplication([])
+    test_widget('Interface', 'Search')
+
--- a/src/calibre/gui2/preferences/search.ui
+++ b/src/calibre/gui2/preferences/search.ui
@ -0,0 +1,104 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>Form</class>
+ <widget class="QWidget" name="Form">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>670</width>
+    <height>392</height>
+   </rect>
+  </property>
+  <property name="windowTitle">
+   <string>Form</string>
+  </property>
+  <layout class="QGridLayout" name="gridLayout">
+   <item row="0" column="0">
+    <widget class="QCheckBox" name="opt_search_as_you_type">
+     <property name="text">
+      <string>Search as you &amp;type</string>
+     </property>
+    </widget>
+   </item>
+   <item row="1" column="0">
+    <widget class="QCheckBox" name="opt_highlight_search_matches">
+     <property name="text">
+      <string>&amp;Highlight search results instead of restricting the book list to the results</string>
+     </property>
+    </widget>
+   </item>
+   <item row="2" column="0">
+    <widget class="QGroupBox" name="groupBox">
+     <property name="title">
+      <string>What to search by default</string>
+     </property>
+     <layout class="QGridLayout" name="gridLayout_2">
+      <item row="0" column="0" colspan="2">
+       <widget class="QLabel" name="label">
+        <property name="text">
+         <string>When you enter a search term without a prefix, by default calibre will search all metadata for matches. For example, entering, &quot;asimov&quot; will search not just authors but title/tags/series/comments/etc. Use these options if you would like to change this behavior.</string>
+        </property>
+        <property name="wordWrap">
+         <bool>true</bool>
+        </property>
+       </widget>
+      </item>
+      <item row="1" column="0" colspan="2">
+       <widget class="QCheckBox" name="opt_limit_search_columns">
+        <property name="text">
+         <string>&amp;Limit the searched metadata</string>
+        </property>
+       </widget>
+      </item>
+      <item row="2" column="0">
+       <widget class="QLabel" name="label_2">
+        <property name="text">
+         <string>&amp;Columns that non-prefixed searches are limited to:</string>
+        </property>
+        <property name="buddy">
+         <cstring>opt_limit_search_columns_to</cstring>
+        </property>
+       </widget>
+      </item>
+      <item row="2" column="1">
+       <widget class="MultiCompleteLineEdit" name="opt_limit_search_columns_to"/>
+      </item>
+      <item row="5" column="0" colspan="2">
+       <widget class="QLabel" name="label_3">
+        <property name="text">
+         <string>Note that this option affects all searches, including saved searches and restrictions. Therefore, if you use this option, it is best to ensure that you always use prefixes in your saved searches. For example, use &quot;series:Foundation&quot; rather than just &quot;Foundation&quot; in  a saved search</string>
+        </property>
+        <property name="wordWrap">
+         <bool>true</bool>
+        </property>
+       </widget>
+      </item>
+     </layout>
+    </widget>
+   </item>
+   <item row="3" column="0">
+    <spacer name="verticalSpacer">
+     <property name="orientation">
+      <enum>Qt::Vertical</enum>
+     </property>
+     <property name="sizeHint" stdset="0">
+      <size>
+       <width>0</width>
+       <height>0</height>
+      </size>
+     </property>
+    </spacer>
+   </item>
+  </layout>
+ </widget>
+ <customwidgets>
+  <customwidget>
+   <class>MultiCompleteLineEdit</class>
+   <extends>QLineEdit</extends>
+   <header>calibre/gui2.complete.h</header>
+  </customwidget>
+ </customwidgets>
+ <resources/>
+ <connections/>
+</ui>
--- a/src/calibre/gui2/search_box.py
+++ b/src/calibre/gui2/search_box.py
@ -16,7 +16,6 @@ from calibre.gui2 import config
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2.dialogs.saved_search_editor import SavedSearchEditor
 from calibre.gui2.dialogs.search import SearchDialog
-from calibre.utils.config import dynamic
 from calibre.utils.search_query_parser import saved_searches
 from calibre.utils.icu import sort_key

@ -376,9 +375,7 @@ class SearchBoxMixin(object): # {{{
            unicode(self.search.toolTip())))
        self.advanced_search_button.setStatusTip(self.advanced_search_button.toolTip())
        self.clear_button.setStatusTip(self.clear_button.toolTip())
-        self.search_highlight_only.stateChanged.connect(self.highlight_only_changed)
-        self.search_highlight_only.setChecked(
-                            dynamic.get('search_highlight_only', False))
+        self.search_options_button.clicked.connect(self.search_options_button_clicked)

    def focus_search_box(self, *args):
        self.search.setFocus(Qt.OtherFocusReason)
@ -402,14 +399,13 @@ class SearchBoxMixin(object): # {{{
        self.search.do_search()
        self.focus_to_library()

+    def search_options_button_clicked(self):
+        self.iactions['Preferences'].do_config(initial_plugin=('Interface',
+            'Search'), close_after_initial=True)
+
    def focus_to_library(self):
        self.current_view().setFocus(Qt.OtherFocusReason)

-    def highlight_only_changed(self, toWhat):
-        dynamic.set('search_highlight_only', toWhat)
-        self.current_view().model().set_highlight_only(toWhat)
-        self.focus_to_library()
-
    # }}}

 class SavedSearchBoxMixin(object): # {{{
--- a/src/calibre/gui2/tag_view.py
+++ b/src/calibre/gui2/tag_view.py
@ -1214,7 +1214,7 @@ class TagBrowserMixin(object): # {{{
            db.field_metadata.remove_user_categories()
            for k in d.categories:
                db.field_metadata.add_user_category('@' + k, k)
-            db.data.sqp_change_locations(db.field_metadata.get_search_terms())
+            db.data.change_search_locations(db.field_metadata.get_search_terms())
            self.tags_view.set_new_model()
            self.tags_view.recount()

@ -1259,9 +1259,8 @@ class TagBrowserMixin(object): # {{{
            if rename_func:
                for item in to_delete:
                    delete_func(item)
-                for text in to_rename:
-                        for old_id in to_rename[text]:
-                            rename_func(old_id, new_name=unicode(text))
+                for old_id in to_rename:
+                    rename_func(old_id, new_name=unicode(to_rename[old_id]))

            # Clean up the library view
            self.do_tag_item_renamed()
--- a/src/calibre/gui2/tools.py
+++ b/src/calibre/gui2/tools.py
@ -9,7 +9,7 @@ Logic for setting up conversion jobs

 import cPickle, os

-from PyQt4.Qt import QDialog, QProgressDialog, QString, QTimer, SIGNAL
+from PyQt4.Qt import QDialog, QProgressDialog, QString, QTimer

 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.gui2 import warning_dialog, question_dialog
@ -24,7 +24,8 @@ from calibre.ebooks.conversion.config import GuiRecommendations, \
    load_defaults, load_specifics, save_specifics
 from calibre.gui2.convert import bulk_defaults_for_input_format

-def convert_single_ebook(parent, db, book_ids, auto_conversion=False, out_format=None):
+def convert_single_ebook(parent, db, book_ids, auto_conversion=False, # {{{
+        out_format=None):
    changed = False
    jobs = []
    bad = []
@ -95,7 +96,9 @@ def convert_single_ebook(parent, db, book_ids, auto_conversion=False, out_format
            msg).exec_()

    return jobs, changed, bad
+# }}}

+# Bulk convert {{{
 def convert_bulk_ebook(parent, queue, db, book_ids, out_format=None, args=[]):
    total = len(book_ids)
    if total == 0:
@ -125,14 +128,11 @@ class QueueBulk(QProgressDialog):
        self.parent = parent
        self.use_saved_single_settings = use_saved_single_settings
        self.i, self.bad, self.jobs, self.changed = 0, [], [], False
-        self.timer = QTimer(self)
-        self.connect(self.timer, SIGNAL('timeout()'), self.do_book)
-        self.timer.start()
+        QTimer.singleShot(0, self.do_book)
        self.exec_()

    def do_book(self):
        if self.i >= len(self.book_ids):
-            self.timer.stop()
            return self.do_queue()
        book_id = self.book_ids[self.i]
        self.i += 1
@ -191,6 +191,7 @@ class QueueBulk(QProgressDialog):
            self.setValue(self.i)
        except NoSupportedInputFormats:
            self.bad.append(book_id)
+        QTimer.singleShot(0, self.do_book)

    def do_queue(self):
        self.hide()
@ -209,7 +210,9 @@ class QueueBulk(QProgressDialog):
        self.jobs.reverse()
        self.queue(self.jobs, self.changed, self.bad, *self.args)

-def fetch_scheduled_recipe(arg):
+# }}}
+
+def fetch_scheduled_recipe(arg): # {{{
    fmt = prefs['output_format'].lower()
    pt = PersistentTemporaryFile(suffix='_recipe_out.%s'%fmt.lower())
    pt.close()
@ -250,7 +253,9 @@ def fetch_scheduled_recipe(arg):

    return 'gui_convert', args, _('Fetch news from ')+arg['title'], fmt.upper(), [pt]

-def generate_catalog(parent, dbspec, ids, device_manager, db):
+# }}}
+
+def generate_catalog(parent, dbspec, ids, device_manager, db): # {{{
    from calibre.gui2.dialogs.catalog import Catalog

    # Build the Catalog dialog in gui2.dialogs.catalog
@ -308,8 +313,9 @@ def generate_catalog(parent, dbspec, ids, device_manager, db):
    # Which then calls gui2.convert.gui_conversion:gui_catalog() with the args inline
    return 'gui_catalog', args, _('Generate catalog'), out.name, d.catalog_sync, \
            d.catalog_title
+# }}}

-def convert_existing(parent, db, book_ids, output_format):
+def convert_existing(parent, db, book_ids, output_format): # {{{
    already_converted_ids = []
    already_converted_titles = []
    for book_id in book_ids:
@ -325,3 +331,5 @@ def convert_existing(parent, db, book_ids, output_format):
            book_ids = [x for x in book_ids if x not in already_converted_ids]

    return book_ids
+# }}}
+
--- a/src/calibre/gui2/ui.py
+++ b/src/calibre/gui2/ui.py
@ -483,8 +483,10 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
            action.location_selected(location)
        if location == 'library':
            self.search_restriction.setEnabled(True)
+            self.search_options_button.setEnabled(True)
        else:
            self.search_restriction.setEnabled(False)
+            self.search_options_button.setEnabled(False)
            # Reset the view in case something changed while it was invisible
            self.current_view().reset()
        self.set_number_of_books_shown()
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@ -11,7 +11,7 @@ from itertools import repeat
 from datetime import timedelta
 from threading import Thread

-from calibre.utils.config import tweaks
+from calibre.utils.config import tweaks, prefs
 from calibre.utils.date import parse_date, now, UNDEFINED_DATE
 from calibre.utils.search_query_parser import SearchQueryParser
 from calibre.utils.pyparsing import ParseException
@ -182,15 +182,16 @@ class ResultCache(SearchQueryParser): # {{{
        self.first_sort = True
        self.search_restriction = ''
        self.field_metadata = field_metadata
-        all_search_locations = field_metadata.get_search_terms()
-        SearchQueryParser.__init__(self, all_search_locations, optimize=True)
+        self.all_search_locations = field_metadata.get_search_terms()
+        SearchQueryParser.__init__(self, self.all_search_locations, optimize=True)
        self.build_date_relop_dict()
        self.build_numeric_relop_dict()

    def break_cycles(self):
        self._data = self.field_metadata = self.FIELD_MAP = \
            self.numeric_search_relops = self.date_search_relops = \
-            self.db_prefs = None
+            self.db_prefs = self.all_search_locations = None
+        self.sqp_change_locations([])


    def __getitem__(self, row):
@ -218,6 +219,10 @@ class ResultCache(SearchQueryParser): # {{{
    def universal_set(self):
        return set([i[0] for i in self._data if i is not None])

+    def change_search_locations(self, locations):
+        self.sqp_change_locations(locations)
+        self.all_search_locations = locations
+
    def build_date_relop_dict(self):
        '''
        Because the database dates have time in them, we can't use direct
@ -432,6 +437,7 @@ class ResultCache(SearchQueryParser): # {{{
            # get metadata key associated with the search term. Eliminates
            # dealing with plurals and other aliases
            location = self.field_metadata.search_term_to_field_key(icu_lower(location.strip()))
+            # grouped search terms
            if isinstance(location, list):
                if allow_recursion:
                    for loc in location:
@ -440,6 +446,20 @@ class ResultCache(SearchQueryParser): # {{{
                    return matches
                raise ParseException(query, len(query), 'Recursive query group detected', self)

+            # apply the limit if appropriate
+            if location == 'all' and prefs['limit_search_columns'] and \
+                            prefs['limit_search_columns_to']:
+                terms = set([])
+                for l in prefs['limit_search_columns_to']:
+                    l = icu_lower(l.strip())
+                    if l and l != 'all' and l in self.all_search_locations:
+                        terms.add(l)
+                if terms:
+                    for l in terms:
+                        matches |= self.get_matches(l, query,
+                            candidates=candidates, allow_recursion=allow_recursion)
+                    return matches
+
            if location in self.field_metadata:
                fm = self.field_metadata[location]
                # take care of dates special case
--- a/src/calibre/library/catalog.py
+++ b/src/calibre/library/catalog.py
@ -4442,6 +4442,7 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1])

            # Insert the link to the series or remove <a class="series">
            aTag = body.find('a', attrs={'class':'series_id'})
+            if aTag:
                if book['series']:
                    if self.opts.generate_series:
                        aTag['href'] = "%s.html#%s_series" % ('BySeries',
@ -4449,39 +4450,31 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1])
                else:
                    aTag.extract()

-            # Insert the author link (always)
+            # Insert the author link
            aTag = body.find('a', attrs={'class':'author'})
-            if self.opts.generate_authors:
+            if self.opts.generate_authors and aTag:
                aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor",
                                            self.generateAuthorAnchor(book['author']))

            if publisher == ' ':
-                try:
                publisherTag = body.find('td', attrs={'class':'publisher'})
+                if publisherTag:
                    publisherTag.contents[0].replaceWith('&nbsp;')
-                except:
-                    pass

            if not genres:
-                try:
                genresTag = body.find('p',attrs={'class':'genres'})
+                if genresTag:
                    genresTag.extract()
-                except:
-                    pass

            if not formats:
-                try:
                formatsTag = body.find('p',attrs={'class':'formats'})
+                if formatsTag:
                    formatsTag.extract()
-                except:
-                    pass

            if note_content == '':
-                try:
                tdTag = body.find('td', attrs={'class':'notes'})
+                if tdTag:
                    tdTag.contents[0].replaceWith('&nbsp;')
-                except:
-                    pass

            emptyTags = body.findAll('td', attrs={'class':'empty'})
            for mt in emptyTags:
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@ -414,7 +414,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
        row = self.data._data[index] if index_is_id else self.data[index]
        return row[self.FIELD_MAP['path']].replace('/', os.sep)

-
    def abspath(self, index, index_is_id=False, create_dirs=True):
        'Return the absolute path to the directory containing this books files as a unicode string.'
        path = os.path.join(self.library_path, self.path(index, index_is_id=index_is_id))
@ -422,7 +421,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
            os.makedirs(path)
        return path

-
    def construct_path_name(self, id):
        '''
        Construct the directory name for this book based on its metadata.
@ -432,6 +430,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
            authors = _('Unknown')
        author = ascii_filename(authors.split(',')[0])[:self.PATH_LIMIT].decode(filesystem_encoding, 'replace')
        title  = ascii_filename(self.title(id, index_is_id=True))[:self.PATH_LIMIT].decode(filesystem_encoding, 'replace')
+        while author[-1] in (' ', '.'):
+            author = author[:-1]
+        if not author:
+            author = ascii_filename(_('Unknown')).decode(filesystem_encoding, 'replace')
        path = author + '/' + title + ' (%d)'%id
        return path

--- a/src/calibre/library/server/content.py
+++ b/src/calibre/library/server/content.py
@ -124,8 +124,7 @@ class ContentServer(object):
            cherrypy.request.headers.get('Want-OPDS-Catalog', 919) != 919 or \
            ua.startswith('Stanza')

-        # A better search would be great
-        want_mobile = self.MOBILE_UA.search(ua) is not None
+        want_mobile = self.is_mobile_browser(ua)
        if self.opts.develop and not want_mobile:
            cherrypy.log('User agent: '+ua)

--- a/src/calibre/library/server/mobile.py
+++ b/src/calibre/library/server/mobile.py
@ -169,6 +169,10 @@ class MobileServer(object):

    MOBILE_UA = re.compile('(?i)(?:iPhone|Opera Mini|NetFront|webOS|Mobile|Android|imode|DoCoMo|Minimo|Blackberry|MIDP|Symbian|HD2|Kindle)')

+    def is_mobile_browser(self, ua):
+        match = self.MOBILE_UA.search(ua)
+        return match is not None and 'iPad' not in ua
+
    def add_routes(self, connect):
        connect('mobile', '/mobile', self.mobile)
        connect('mobile_css', '/mobile/style.css', self.mobile_css)
--- a/src/calibre/manual/faq.rst
+++ b/src/calibre/manual/faq.rst
@ -316,6 +316,27 @@ When you first run |app|, it will ask you for a folder in which to store your bo

 Metadata about the books is stored in the file ``metadata.db`` at the top level of the library folder This file is is a sqlite database. When backing up your library make sure you copy the entire folder and all its sub-folders.

+How does |app| manage author names and sorting?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Author names are complex, especially across cultures. |app| has a very flexible strategy for managing author names. The first thing to understand is that books and authors are separate entities in |app|. A book can have more than one author, and an author can have more than one book. You can manage the authors of a book by the edit metadata dialog. You can manage individual authors by right clicking on the author in the Tag Browser on the left of the main |app| screen and selecting :guilabel:`Manage authors`. Using this dialog you can change the name of an author and also how that name is sorted. This will automatically change the name of the author in all the books of that author. When a book has multiple authors, separate their names using the & character.
+
+Now coming to author name sorting:
+
+    * When a new author is added to |app| (this happens whenever a book by a new author is added), |app| automatically computes a sort string for both the book and the author.
+    * Authors in the Tag Browser are sorted by the sort value for the **authors**. Remember that this is different from the Author sort field for a book. 
+    * By default, this sort algorithm assumes that the author name is in ``First name Last name`` format and generates a ``Last name, First name`` sort value.
+    * You can change this algorithm by going to Preferences->Tweaks and setting the :guilabel:`author_sort_copy_method` tweak.
+    * You can force |app| to recalculate the author sort values for every author by right clicking on any author and selecting :guilabel:`Manage authors`, then pushing the `Recalculate all author sort values` button. Do this after you have set the author_sort_copy_method tweak to what you want.
+    * You can force |app| to recalculate the author sort values for all books by using the bulk metadata edit dialog (select all books and click edit metadata, check the `Automatically set author sort` checkbox, then press OK.)
+    * When recalculating the author sort values for books, |app| uses the author sort values for each individual author. Therefore, ensure that the individual author sort values are correct before recalculating the books' author sort values.
+    * You can control whether the Tag Browser display authors using their names or their sort values by setting the :guilabel:`categories_use_field_for_author_name` tweak in Preferences->Tweaks
+
+With all this flexibility, it is possible to have |app| manage your author names however you like. For example, one common request is to have |app| display author names LN, FN. To do this first set the ``author_sort_copy_method`` to ``copy``. Then change all author names to LN, FN via the Manage authors dialog. Then have |app| recalculate author sort values for both authors and books as described above.
+
+Note that you can set an individual author's sort value to whatever you want using :guilabel:`Manage authors`. This is useful when dealing with names that |app| will not get right, such as complex multi-part names like Miguel de Cervantes Saavedra or when dealing with Asian names like Sun Tzu.
+
+
 Why doesn't |app| let me store books in my own directory structure?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

--- a/src/calibre/utils/config.py
+++ b/src/calibre/utils/config.py
@ -728,6 +728,17 @@ def _prefs():
    c.add_opt('user_categories', default={}, help=_('User-created tag browser categories'))
    c.add_opt('manage_device_metadata', default='manual',
        help=_('How and when calibre updates metadata on the device.'))
+    c.add_opt('limit_search_columns', default=False,
+            help=_('When searching for text without using lookup '
+            'prefixes, as for example, Red instead of title:Red, '
+            'limit the columns searched to those named below.'))
+    c.add_opt('limit_search_columns_to',
+            default=['title', 'authors', 'tags', 'series', 'publisher'],
+            help=_('Choose columns to be searched when not using prefixes, '
+                'as for example, when searching for Redd instead of '
+                'title:Red. Enter a list of search/lookup names '
+                'separated by commas. Only takes effect if you set the option '
+                'to limit search columns above.'))

    c.add_opt('migrated', default=False, help='For Internal use. Don\'t modify.')
    return c
--- a/src/calibre/utils/html2textile.py
+++ b/src/calibre/utils/html2textile.py
@ -77,7 +77,7 @@ class EchoTarget:
            new_tag = '~'
            newline = ''
        elif tag == 'span':
-            new_tag = '%'
+            new_tag = ''
            newline = ''
        elif tag == 'a':
            self.block = True
@ -147,7 +147,7 @@ class EchoTarget:
        elif tag == 'sub':
            self.final_output.append('~')
        elif tag == 'span':
-            self.final_output.append('%')
+            self.final_output.append('')
        elif tag == 'a':
            if self.a_part['title']:
                textilized = ' "%s (%s)":%s ' % (
--- a/src/calibre/utils/localization.py
+++ b/src/calibre/utils/localization.py
@ -104,6 +104,7 @@ _extra_lang_codes = {
        'en_IN' : _('English (India)'),
        'en_TH' : _('English (Thailand)'),
        'en_CY' : _('English (Cyprus)'),
+        'en_CZ' : _('English (Czechoslovakia)'),
        'en_PK' : _('English (Pakistan)'),
        'en_HR' : _('English (Croatia)'),
        'en_IL' : _('English (Israel)'),