Merge from trunk

resources/recipes/180.recipe

@@ -12,7 +12,7 @@ class Noticias(BasicNewsRecipe):
     title                 = '180.com.uy'
     __author__            = 'Gustavo Azambuja'
     description           = 'Noticias de Uruguay'
-    language       = 'es'
+    language       = 'es_UY'
     timefmt        = '[%a, %d %b, %Y]'
     use_embedded_content  = False
     recursion             = 5

resources/recipes/7dias.recipe

@@ -20,7 +20,7 @@ class SieteDias(BasicNewsRecipe):
     no_stylesheets        = True
     use_embedded_content  = False
     encoding              = 'utf-8'
-    language = 'es'
+    language = 'es_AR'
 
     lang                  = 'es-AR'
     direction             = 'ltr'

resources/recipes/ambito.recipe

@@ -58,4 +58,4 @@ class Ambito(BasicNewsRecipe):
             del item['style']
         return soup
 
-    language = 'es'
+    language = 'es_AR'

resources/recipes/animal_politico.recipe

@@ -12,7 +12,7 @@ class AdvancedUserRecipe1290663986(BasicNewsRecipe):
     masthead_url   = 'http://www.animalpolitico.com/wp-content/themes/animal_mu/images/logo.png'
     oldest_article = 1
     max_articles_per_feed = 100
-    language       = 'es'
+    language       = 'es_MX'
 
     #feeds          = [(u'Animal Politico', u'http://www.animalpolitico.com/feed/')]
 

resources/recipes/axxon_magazine.recipe

@@ -17,7 +17,7 @@ class Axxon_news(BasicNewsRecipe):
     max_articles_per_feed = 100
     no_stylesheets        = False
     use_embedded_content  = False
-    language              = 'es'
+    language              = 'es_AR'
     encoding              = 'utf-8'
     publication_type      = 'magazine'
     INDEX                 = 'http://axxon.com.ar/rev/'

resources/recipes/axxon_news.recipe

@@ -18,7 +18,7 @@ class Axxon_news(BasicNewsRecipe):
     max_articles_per_feed = 100
     no_stylesheets        = False
     use_embedded_content  = False
-    language = 'es'
+    language = 'es_AR'
 
     lang                  = 'es-AR'
 

resources/recipes/bbc_es.recipe

@@ -0,0 +1,53 @@
+__license__   = 'GPL v3'
+__author__    = 'Luis Hernandez'
+__copyright__ = 'Luis Hernandez<tolyluis@gmail.com>'
+__version__     = 'v1.0'
+__date__        = '29 January 2011'
+
+'''
+http://www.bbc.co.uk/mundo/
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1294946868(BasicNewsRecipe):
+
+    title          = u'BBC Mundo'
+    publisher      = u'BBC'
+
+    __author__            = 'Luis Hernandez'
+    description           = 'BBC World for spanish readers'
+
+    cover_url     = 'http://1.bp.blogspot.com/_NHiOjk_uZwU/TEYy7IJAdAI/AAAAAAAABP8/coAE-pJ7_5E/s1600/bbcmundo_h.png'
+    oldest_article = 2
+    max_articles_per_feed = 100
+
+    remove_javascript = True
+    no_stylesheets        = True
+    use_embedded_content  = False
+
+    language              = 'es'
+    remove_empty_feeds    = True
+    encoding               = 'UTF-8'
+    timefmt        = '[%a, %d %b, %Y]'
+
+    remove_tags_before = dict(name='div' , attrs={'class':['g-group']})
+    remove_tags_after = dict(name='div' , attrs={'class':[' g-w8']})
+
+    remove_tags = [
+                             dict(name='ul', attrs={'class':['document-tools blq-clearfix','blq-clearfix']})
+                            ,dict(name='div', attrs={'class':['box bx-quote-bubble','socialmedia-links','list li-carousel','list li-plain rolling-news','list li-plain','box bx-livestats','li-tab content','list li-relatedlinks','list li-relatedinternetlinks']})
+                         ]
+
+    feeds = [
+                  (u'Portada'               , u'http://www.bbc.co.uk/mundo/index.xml')
+                 ,(u'Ultimas Noticias'   , u'http://www.bbc.co.uk/mundo/ultimas_noticias/index.xml')
+                 ,(u'Internacional'       , u'http://www.bbc.co.uk/mundo/temas/internacional/index.xml')
+                 ,(u'Economia'            , u'http://www.bbc.co.uk/mundo/temas/economia/index.xml')
+                 ,(u'America Latina'     , u'http://www.bbc.co.uk/mundo/temas/america_latina/index.xml')
+                 ,(u'Ciencia'                , u'http://www.bbc.co.uk/mundo/temas/ciencia/index.xml')
+                 ,(u'Salud'                  , u'http://www.bbc.co.uk/mundo/temas/salud/index.xml')
+                 ,(u'Tecnologia'           , u'http://www.bbc.co.uk/mundo/temas/tecnologia/index.xml')
+                 ,(u'Cultura'                , u'http://www.bbc.co.uk/mundo/temas/cultura/index.xml')
+             ]
+

resources/recipes/bitacora.recipe

@@ -12,7 +12,7 @@ class General(BasicNewsRecipe):
     title                 = 'bitacora.com.uy'
     __author__            = 'Gustavo Azambuja'
     description           = 'Noticias de Uruguay'
-    language       = 'es'
+    language       = 'es_UY'
     timefmt        = '[%a, %d %b, %Y]'
     use_embedded_content  = False
     recursion             = 5

resources/recipes/buenosaireseconomico.recipe

@@ -20,7 +20,7 @@ class BsAsEconomico(BasicNewsRecipe):
     no_stylesheets        = True
     use_embedded_content  = False
     encoding              = 'utf-8'
-    language = 'es'
+    language = 'es_AR'
 
     lang                  = 'es-AR'
     direction             = 'ltr'

resources/recipes/clarin.recipe

@@ -18,7 +18,7 @@ class Clarin(BasicNewsRecipe):
     use_embedded_content  = False
     no_stylesheets        = True
     encoding              = 'utf8'
-    language              = 'es'
+    language              = 'es_AR'
     publication_type      = 'newspaper'
     INDEX                 = 'http://www.clarin.com'
     masthead_url          = 'http://www.clarin.com/static/CLAClarin/images/logo-clarin-print.jpg'

resources/recipes/criticadigital.recipe

@@ -14,7 +14,7 @@ class CriticaDigital(BasicNewsRecipe):
     description           = 'Noticias de Argentina'
     oldest_article        = 2
     max_articles_per_feed = 100
-    language = 'es'
+    language = 'es_AR'
 
     no_stylesheets        = True
     use_embedded_content  = False

resources/recipes/cubadebate.recipe

@@ -11,7 +11,7 @@ class CubaDebate(BasicNewsRecipe):
     __author__            = 'Darko Miletic'
     description           = 'Contra el Terorismo Mediatico'
     oldest_article        = 15
-    language              = 'es'
+    language              = 'es_CU'
     max_articles_per_feed = 100
     no_stylesheets        = True
     use_embedded_content  = False

resources/recipes/deutsche_welle_es.recipe

@@ -16,7 +16,7 @@ class DeutscheWelle_es(BasicNewsRecipe):
     max_articles_per_feed = 100
     use_embedded_content  = False
     no_stylesheets        = True
-    language              = 'es'
+    language              = 'de_ES'
     publication_type      = 'newsportal'
     remove_empty_feeds    = True
     masthead_url          = 'http://www.dw-world.de/skins/std/channel1/pics/dw_logo1024.gif'

resources/recipes/diagonales.recipe

@@ -20,7 +20,7 @@ class Diagonales(BasicNewsRecipe):
     no_stylesheets        = True
     use_embedded_content  = False
     encoding              = 'utf-8'
-    language = 'es'
+    language = 'es_AR'
 
     lang                  = 'es-AR'
     direction             = 'ltr'

resources/recipes/el_mercurio_chile.recipe

@@ -20,7 +20,7 @@ class ElMercurio(BasicNewsRecipe):
     masthead_url          = 'http://www.emol.com/especiales/logo_emol/logo_emol.gif'
     remove_javascript     = True
     use_embedded_content  = False
-    language              = 'es'
+    language              = 'es_CL'
 
 
     conversion_options = {

resources/recipes/el_observador.recipe

@@ -13,7 +13,7 @@ class ObservaDigital(BasicNewsRecipe):
     title                 = 'Observa Digital'
     __author__            = 'yrvn'
     description           = 'Noticias de Uruguay'
-    language       = 'es'
+    language       = 'es_UY'
     timefmt        = '[%a, %d %b, %Y]'
     use_embedded_content  = False
     recursion             = 5

resources/recipes/el_pais_uy.recipe

@@ -14,7 +14,7 @@ class General(BasicNewsRecipe):
     description           = 'Noticias de Uruguay y el resto del mundo'
     publisher             = 'EL PAIS S.A.'
     category              = 'news, politics, Uruguay'
-    language       = 'es'
+    language       = 'es_UY'
     timefmt        = '[%a, %d %b, %Y]'
     use_embedded_content  = False
     recursion             = 2

resources/recipes/el_universal.recipe

@@ -20,7 +20,7 @@ class ElUniversal(BasicNewsRecipe):
     remove_javascript     = True
     remove_empty_feeds    = True
     publication_type      = 'newspaper'
-    language              = 'es'
+    language              = 'es_MX'
 
     extra_css = '''
                     body{font-family:Arial,Helvetica,sans-serif}

resources/recipes/elargentino.recipe

@@ -20,7 +20,7 @@ class ElArgentino(BasicNewsRecipe):
     use_embedded_content  = False
     encoding              = 'utf8'
     cover_url             = 'http://www.elargentino.com/TemplateWeb/MediosFooter/tapa_elargentino.png'
-    language = 'es'
+    language = 'es_AR'
 
 
     html2lrf_options = [

resources/recipes/elcomercio.recipe

@@ -18,7 +18,7 @@ class ElComercio(BasicNewsRecipe):
     no_stylesheets        = True
     encoding              = 'utf-8'
     use_embedded_content  = True
-    language              = 'es'
+    language              = 'es_EC'
     masthead_url          = 'http://ww1.elcomercio.com/nv_images/headers/EC/logo_new_08.gif'
     extra_css             = ' body{font-family: Arial,Verdana,sans-serif} img{margin-bottom: 1em} '
 

resources/recipes/elcronista.recipe

@@ -13,7 +13,7 @@ class ElCronista(BasicNewsRecipe):
     __author__            = 'Darko Miletic'
     description           = 'Noticias de Argentina'
     oldest_article        = 2
-    language = 'es'
+    language = 'es_AR'
 
     max_articles_per_feed = 100
     no_stylesheets        = True

resources/recipes/eltiempo_hn.recipe

@@ -21,7 +21,7 @@ class ElTiempoHn(BasicNewsRecipe):
     no_stylesheets        = True
     remove_javascript     = True
     encoding              = 'utf-8'
-    language = 'es'
+    language = 'es_HN'
 
     lang                  = 'es-HN'
     direction             = 'ltr'

resources/recipes/eluniversal_ve.recipe

@@ -18,7 +18,7 @@ class ElUniversal(BasicNewsRecipe):
     encoding               = 'cp1252'
     publisher              = 'El Universal'
     category               = 'news, Caracas, Venezuela, world'
-    language               = 'es'
+    language               = 'es_VE'
     cover_url              = strftime('http://static.eluniversal.com/%Y/%m/%d/portada.jpg')
 
     conversion_options = {

resources/recipes/eluniversalimpresa.recipe

@@ -3,7 +3,7 @@ from calibre.web.feeds.news import BasicNewsRecipe
 class ElUniversalImpresaRecipe(BasicNewsRecipe):
     __license__  = 'GPL v3'
     __author__ = 'kwetal'
-    language = 'es'
+    language = 'es_MX'
     version = 1
 
     title = u'El Universal (Edici\u00F3n Impresa)'

resources/recipes/eluniverso_ec.recipe

@@ -17,7 +17,7 @@ class ElUniverso_Ecuador(BasicNewsRecipe):
     no_stylesheets        = True
     encoding              = 'utf8'
     use_embedded_content  = False
-    language              = 'es'
+    language              = 'es_EC'
     remove_empty_feeds    = True
     publication_type      = 'newspaper'
     masthead_url          = 'http://servicios2.eluniverso.com/versiones/v1/img/Hd/lg_ElUniverso.gif'

resources/recipes/endgadget_ja.recipe

@@ -18,3 +18,6 @@ class EndgadgetJapan(BasicNewsRecipe):
     language = 'ja'
     encoding = 'utf-8'
     feeds          = [(u'engadget', u'http://japanese.engadget.com/rss.xml')]
+
+    remove_tags_before = dict(name="div", attrs={'id':"content_wrap"})
+    remove_tags_after = dict(name='h3', attrs={'id':'addcomments'})

resources/recipes/explosm.recipe

@@ -0,0 +1,54 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+import re
+
+class Explosm(BasicNewsRecipe):
+    title              = u'Explosm Rotated'
+    __author__        = 'Andromeda Rabbit'
+    description      = 'Explosm'
+    language            = 'en'
+    use_embedded_content = False
+    no_stylesheets    = True
+    oldest_article    = 24
+    remove_javascript   = True
+    remove_empty_feeds  = True
+    max_articles_per_feed = 10
+
+    feeds = [
+             (u'Explosm Feed', u'http://feeds.feedburner.com/Explosm')
+             ]
+
+    #match_regexps = [r'http://www.explosm.net/comics/.*']
+
+    keep_only_tags   = [dict(name='img', attrs={'alt':'Cyanide and Happiness, a daily webcomic'})]
+    remove_tags = [dict(name='div'), dict(name='span'), dict(name='table'), dict(name='br'), dict(name='nobr'), dict(name='a'), dict(name='b')]
+
+    extra_css = '''
+                    h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;}
+                    h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;}
+                    p{font-family:Arial,Helvetica,sans-serif;font-size:small;}
+                    body{font-family:Helvetica,Arial,sans-serif;font-size:small;}'''
+
+    def get_cover_url(self):
+        return 'http://cdn.shopify.com/s/files/1/0059/1872/products/cyanidetitle_large.jpg?1295846286'
+
+    def parse_feeds(self):
+        feeds = BasicNewsRecipe.parse_feeds(self)
+
+        for curfeed in feeds:
+            delList = []
+            for a,curarticle in enumerate(curfeed.articles):
+                if re.search(r'http://www.explosm.net/comics', curarticle.url) == None:
+                    delList.append(curarticle)
+            if len(delList)>0:
+                for d in delList:
+                    index = curfeed.articles.index(d)
+                    curfeed.articles[index:index+1] = []
+
+        return feeds
+
+    def skip_ad_pages(self, soup):
+        # Skip ad pages served before actual article
+        skip_tag = soup.find(name='img', attrs={'alt':'Cyanide and Happiness, a daily webcomic'})
+        if skip_tag is None:
+            return soup
+        return None

resources/recipes/freeway.recipe

@@ -12,7 +12,7 @@ class General(BasicNewsRecipe):
     title                 = 'freeway.com.uy'
     __author__            = 'Gustavo Azambuja'
     description           = 'Revista Freeway, Montevideo, Uruguay'
-    language       = 'es'
+    language       = 'es_UY'
     timefmt        = '[%a, %d %b, %Y]'
     use_embedded_content  = False
     recursion             = 1

resources/recipes/granma.recipe

@@ -20,7 +20,7 @@ class Granma(BasicNewsRecipe):
     use_embedded_content  = False
     encoding              = 'cp1252'
     cover_url             = 'http://www.granma.cubaweb.cu/imagenes/granweb229d.jpg'
-    language = 'es'
+    language = 'es_CU'
 
     remove_javascript     = True
 

resources/recipes/ieco.recipe

@@ -18,7 +18,7 @@ class iEco(BasicNewsRecipe):
     encoding              = 'utf-8'
     publisher             = 'Grupo Clarin'
     category              = 'news, economia, mercados, bolsa de valores, finanzas, empresas, negocios, empleos, emprendedores, marketinguniversidades, tecnologia, agronegocios, noticias, informacion'
-    language              = 'es'
+    language              = 'es_AR'
     cover_url             = 'http://www.ieco.clarin.com/static2/images/Tapa-PDF.gif'
     extra_css             = ' #bd{font-family: sans-serif} '
 

resources/recipes/infobae.recipe

@@ -16,7 +16,7 @@ class Infobae(BasicNewsRecipe):
     max_articles_per_feed = 100
     no_stylesheets        = True
     use_embedded_content  = False
-    language              = 'es'
+    language              = 'es_AR'
     encoding              = 'cp1252'
     masthead_url          = 'http://www.infobae.com/imgs/header/header.gif'
     remove_javascript     = True

resources/recipes/juventudrebelde.recipe

@@ -20,7 +20,7 @@ class Juventudrebelde(BasicNewsRecipe):
     no_stylesheets        = True
     use_embedded_content  = False
     encoding              = 'cp1252'
-    language = 'es'
+    language = 'es_CU'
 
     cover_url             = strftime('http://www.juventudrebelde.cu/UserFiles/File/impreso/iportada-%Y-%m-%d.jpg')
     remove_javascript     = True

resources/recipes/la_cuarta.recipe

@@ -50,4 +50,4 @@ class LaCuarta(BasicNewsRecipe):
     feeds = [(u'Noticias', u'http://lacuarta.cl/app/rss?sc=TEFDVUFSVEE=')]
 
 
-    language = 'es'
+    language = 'es_CL'

resources/recipes/la_diaria.recipe

@@ -12,7 +12,7 @@ class General(BasicNewsRecipe):
     title                 = 'La Diaria'
     __author__            = 'Gustavo Azambuja'
     description           = 'Noticias de Uruguay'
-    language       = 'es'
+    language       = 'es_UY'
     timefmt        = '[%a, %d %b, %Y]'
     use_embedded_content  = False
     recursion             = 5

resources/recipes/la_jornada.recipe

@@ -19,7 +19,7 @@ class LaJornada_mx(BasicNewsRecipe):
     no_stylesheets        = True
     encoding              = 'utf8'
     use_embedded_content  = False
-    language              = 'es'
+    language              = 'es_MX'
     remove_empty_feeds    = True
     cover_url             = strftime("http://www.jornada.unam.mx/%Y/%m/%d/portada.pdf")
     masthead_url          = 'http://www.jornada.unam.mx/v7.0/imagenes/la-jornada-trans.png'

resources/recipes/la_razon_bo.recipe

@@ -18,7 +18,7 @@ class LaRazon_Bol(BasicNewsRecipe):
     no_stylesheets        = True
     encoding              = 'cp1252'
     use_embedded_content  = False
-    language              = 'es'
+    language              = 'es_BO'
     publication_type      = 'newspaper'
     delay                 = 1
     remove_empty_feeds    = True

resources/recipes/la_segunda.recipe

@@ -19,7 +19,7 @@ class LaSegunda(BasicNewsRecipe):
     encoding              = 'cp1252'
     masthead_url          = 'http://www.lasegunda.com/imagenes/logotipo_lasegunda_Oli.gif'
     remove_empty_feeds    = True
-    language              = 'es'
+    language              = 'es_CL'
     extra_css             = ' .titulonegritastop{font-size: xx-large; font-weight: bold} '
 
     conversion_options = {

resources/recipes/lamujerdemivida.recipe

@@ -19,7 +19,7 @@ class LaMujerDeMiVida(BasicNewsRecipe):
     encoding              = 'cp1252'
     publisher             = 'La Mujer de mi Vida'
     category              = 'literatura, critica, arte, ensayos'
-    language = 'es'
+    language = 'es_AR'
 
     INDEX                 = 'http://www.lamujerdemivida.com.ar/'
     html2lrf_options = [

resources/recipes/lanacion.recipe

@@ -16,7 +16,7 @@ class Lanacion(BasicNewsRecipe):
     max_articles_per_feed = 100
     use_embedded_content  = False
     no_stylesheets        = True
-    language              = 'es'
+    language              = 'es_AR'
     publication_type      = 'newspaper'
     remove_empty_feeds    = True
     masthead_url          = 'http://www.lanacion.com.ar/imgs/layout/logos/ln341x47.gif'

resources/recipes/lanacion_chile.recipe

@@ -51,4 +51,4 @@ class LaNacionChile(BasicNewsRecipe):
             del item['style']
         return soup
 
-    language = 'es'
+    language = 'es_CL'

resources/recipes/laprensa.recipe

@@ -21,7 +21,7 @@ class LaPrensa(BasicNewsRecipe):
     encoding              = 'cp1252'
    # cover_url             = 'http://www.laprensa.com.ar/imgs/logo.gif'
     remove_javascript     = True
-    language = 'es'
+    language = 'es_AR'
     lang = 'es'
 
     html2lrf_options = [

resources/recipes/laprensa_hn.recipe

@@ -21,7 +21,7 @@ class LaPrensaHn(BasicNewsRecipe):
     no_stylesheets        = True
     remove_javascript     = True
     encoding              = 'utf-8'
-    language = 'es'
+    language = 'es_HN'
 
     lang                  = 'es-HN'
     direction             = 'ltr'

resources/recipes/laprensa_ni.recipe

@@ -22,7 +22,7 @@ class LaPrensa_ni(BasicNewsRecipe):
     use_embedded_content  = False
     encoding              = 'cp1252'
     remove_javascript     = True
-    language = 'es'
+    language = 'es_NI'
 
     months_es             = ['enero','febrero','marzo','abril','mayo','junio','julio','agosto','septiembre','octubre','noviembre','diciembre']
     current_month         = months_es[datetime.date.today().month - 1]

resources/recipes/latribuna.recipe

@@ -21,7 +21,7 @@ class LaTribuna(BasicNewsRecipe):
     no_stylesheets        = True
     remove_javascript     = True
     encoding              = 'utf-8'
-    language = 'es'
+    language = 'es_HN'
 
     lang                  = 'es-HN'
     direction             = 'ltr'

resources/recipes/ledevoir.recipe

@@ -9,6 +9,8 @@ __description__   = 'Canadian Paper '
 http://www.ledevoir.com/
 '''
 
+import re
+
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class ledevoir(BasicNewsRecipe):
@@ -32,6 +34,8 @@ class ledevoir(BasicNewsRecipe):
     remove_javascript     = True
     no_stylesheets        = True
 
+    preprocess_regexps = [(re.compile(r'(title|alt)=".*?>.*?"', re.DOTALL), lambda m: '')]
+
     keep_only_tags  = [
                         dict(name='div', attrs={'id':'article'}),
                         dict(name='ul', attrs={'id':'ariane'})

resources/recipes/los_tiempos_bo.recipe

@@ -18,7 +18,7 @@ class LosTiempos_Bol(BasicNewsRecipe):
     no_stylesheets        = True
     encoding              = 'cp1252'
     use_embedded_content  = False
-    language              = 'es'
+    language              = 'es_BO'
     publication_type      = 'newspaper'
     delay                 = 1
     remove_empty_feeds    = True

resources/recipes/milenio.recipe

@@ -12,7 +12,7 @@ import datetime
 class Milenio(BasicNewsRecipe):
     title                 = u'Milenio-diario'
     __author__            = 'Bmsleight'
-    language              = 'es'
+    language              = 'es_MX'
     description           = 'Milenio-diario'
     oldest_article        = 10
     max_articles_per_feed = 100

resources/recipes/miradasalsur.recipe

@@ -20,7 +20,7 @@ class MiradasAlSur(BasicNewsRecipe):
     no_stylesheets        = True
     use_embedded_content  = False
     encoding              = 'utf-8'
-    language = 'es'
+    language = 'es_AR'
 
     lang                  = 'es-AR'
     direction             = 'ltr'

resources/recipes/montevideo_com.recipe

@@ -12,7 +12,7 @@ class Noticias(BasicNewsRecipe):
     title                 = 'Montevideo COMM'
     __author__            = 'Gustavo Azambuja'
     description           = 'Noticias de Uruguay'
-    language       = 'es'
+    language       = 'es_UY'
     timefmt        = '[%a, %d %b, %Y]'
     use_embedded_content  = False
     recursion             = 5

resources/recipes/newsweek_argentina.recipe

@@ -20,7 +20,7 @@ class Newsweek_Argentina(BasicNewsRecipe):
     no_stylesheets        = True
     use_embedded_content  = False
     encoding              = 'utf-8'
-    language = 'es'
+    language = 'es_AR'
 
     lang                  = 'es-AR'
     direction             = 'ltr'

resources/recipes/observa_digital.recipe

@@ -12,7 +12,7 @@ class Noticias(BasicNewsRecipe):
     title                 = 'Observa Digital'
     __author__ = '2010, Gustavo Azambuja <hola at gazambuja.com>'
     description           = 'Noticias desde Uruguay'
-    language       = 'es'
+    language       = 'es_UY'
     timefmt        = '[%a, %d %b, %Y]'
     use_embedded_content  = False
     recursion             = 5

resources/recipes/pagina12.recipe

@@ -19,7 +19,7 @@ class Pagina12(BasicNewsRecipe):
     no_stylesheets        = True
     encoding              = 'cp1252'
     use_embedded_content  = False
-    language              = 'es'
+    language              = 'es_AR'
     remove_empty_feeds    = True
     publication_type      = 'newspaper'
     masthead_url          = 'http://www.pagina12.com.ar/commons/imgs/logo-home.gif'

resources/recipes/perfil.recipe

@@ -17,7 +17,7 @@ class Perfil(BasicNewsRecipe):
     no_stylesheets        = True
     encoding              = 'cp1252'
     use_embedded_content  = False
-    language              = 'es'
+    language              = 'es_AR'
     remove_empty_feeds    = True
     masthead_url          = 'http://www.perfil.com/export/sites/diarioperfil/arte/10/logo_perfilcom_mm.gif'
     extra_css             = """

resources/recipes/reptantes.recipe

@@ -13,7 +13,7 @@ class Reptantes(BasicNewsRecipe):
     description           = u"cada vez que te haces acupuntura, tu muñeco vudú sufre en algún lado"
     oldest_article        = 130
     max_articles_per_feed = 100
-    language              = 'es'
+    language              = 'es_AR'
     encoding              = 'utf-8'
     no_stylesheets        = True
     use_embedded_content  = False

resources/recipes/revista_bla.recipe

@@ -12,7 +12,7 @@ class Noticias(BasicNewsRecipe):
     title                 = 'Revista Bla'
     __author__            = 'Gustavo Azambuja'
     description           = 'Moda | Uruguay'
-    language       = 'es'
+    language       = 'es_UY'
     timefmt        = '[%a, %d %b, %Y]'
     use_embedded_content  = False
     recursion             = 5

resources/recipes/veintitres.recipe

@@ -20,7 +20,7 @@ class Veintitres(BasicNewsRecipe):
     no_stylesheets        = True
     use_embedded_content  = False
     encoding              = 'utf-8'
-    language = 'es'
+    language = 'es_AR'
 
     lang                  = 'es-AR'
     direction             = 'ltr'

resources/recipes/vijesti.recipe

@@ -1,6 +1,6 @@
 
 __license__   = 'GPL v3'
-__copyright__ = '2009-2010, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2009-2011, Darko Miletic <darko.miletic at gmail.com>'
 
 '''
 vijesti.me
@@ -18,12 +18,16 @@ class Vijesti(BasicNewsRecipe):
     oldest_article        = 2
     max_articles_per_feed = 150
     no_stylesheets        = True
-    encoding              = 'cp1250'
+    encoding              = 'utf8'
     use_embedded_content  = False
     language              = 'sr'
     publication_type      = 'newspaper'
-    masthead_url          = 'http://www.vijesti.me/img/logo.gif'    
+    extra_css             = """
-    extra_css             = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{font-family: serif1, serif} .article_description{font-family: sans1, sans-serif}'
+                                @font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} 
+                                @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} 
+                                body{font-family: Georgia,"Times New Roman",Times,serif1,serif} 
+                                .articledescription,.article,.chapter{font-family: sans1, sans-serif} 
+                            """
 
     conversion_options = {
                           'comment'          : description
@@ -34,11 +38,11 @@ class Vijesti(BasicNewsRecipe):
 
     preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
 
-    keep_only_tags = [dict(name='div', attrs={'id':'mainnews'})]
+    keep_only_tags = [dict(name='div', attrs={'id':['article_intro_text','article_text']})]
 
     remove_tags = [dict(name=['object','link','embed','form'])]
 
-    feeds = [(u'Sve vijesti', u'http://www.vijesti.me/rss.php' )]
+    feeds = [(u'Sve vijesti', u'http://www.vijesti.me/rss/' )]
 
     def preprocess_html(self, soup):
         return self.adeify_images(soup)

setup/installer/linux/freeze2.py

@@ -360,6 +360,9 @@ class LinuxFreeze(Command):
             def main():
                 try:
                     sys.argv[0] = sys.calibre_basename
+                    dfv = os.environ.get('CALIBRE_DEVELOP_FROM', None)
+                    if dfv and os.path.exists(dfv):
+                        sys.path.insert(0, os.path.abspath(dfv))
                     set_default_encoding()
                     set_helper()
                     set_qt_plugin_path()

src/calibre/customize/profiles.py

@@ -583,6 +583,7 @@ class CybookG3Output(OutputProfile):
 
     # Screen size is a best guess
     screen_size               = (600, 800)
+    comic_screen_size         = (600, 757)
     dpi                       = 168.451
     fbase                     = 16
     fsizes                    = [12, 12, 14, 16, 18, 20, 22, 24]

src/calibre/devices/eb600/driver.py

@@ -21,6 +21,7 @@ from calibre.devices.usbms.driver import USBMS
 class EB600(USBMS):
 
     name           = 'Netronix EB600 Device Interface'
+    gui_name       = 'Netronix EB600'
     description    = _('Communicate with the EB600 eBook reader.')
     author         = 'Kovid Goyal'
     supported_platforms = ['windows', 'osx', 'linux']

src/calibre/ebooks/comic/input.py

@@ -53,7 +53,7 @@ def find_pages(dir, sort_on_mtime=False, verbose=False):
         prints('\t'+'\n\t'.join([os.path.basename(p) for p in pages]))
     return pages
 
-class PageProcessor(list):
+class PageProcessor(list): # {{{
     '''
     Contains the actual image rendering logic. See :method:`render` and
     :method:`process_pages`.
@@ -111,6 +111,13 @@ class PageProcessor(list):
 
             SCRWIDTH, SCRHEIGHT = self.opts.output_profile.comic_screen_size
 
+            try:
+                if self.opts.comic_image_size:
+                    SCRWIDTH, SCRHEIGHT = map(int, [x.strip() for x in
+                        self.opts.comic_image_size.split('x')])
+            except:
+                pass # Ignore
+
             if self.opts.keep_aspect_ratio:
                 # Preserve the aspect ratio by adding border
                 aspect = float(sizex) / float(sizey)
@@ -170,6 +177,7 @@ class PageProcessor(list):
                 dest = dest[:-1]
                 os.rename(dest+'8', dest)
             self.append(dest)
+# }}}
 
 def render_pages(tasks, dest, opts, notification=lambda x, y: x):
     '''
@@ -291,7 +299,11 @@ class ComicInput(InputFormatPlugin):
         OptionRecommendation(name='no_process', recommended_value=False,
               help=_("Apply no processing to the image")),
         OptionRecommendation(name='dont_grayscale', recommended_value=False,
-            help=_('Do not convert the image to grayscale (black and white)'))
+            help=_('Do not convert the image to grayscale (black and white)')),
+        OptionRecommendation(name='comic_image_size', recommended_value=None,
+            help=_('Specify the image size as widthxheight pixels. Normally,'
+                ' an image size is automatically calculated from the output '
+                'profile, this option overrides it.')),
         ])
 
     recommendations = set([

src/calibre/ebooks/conversion/utils.py

@@ -24,10 +24,11 @@ class HeuristicProcessor(object):
         self.chapters_no_title = 0
         self.chapters_with_title = 0
         self.blanks_deleted = False
+        self.blanks_between_paragraphs = False
         self.linereg = re.compile('(?<=<p).*?(?=</p>)', re.IGNORECASE|re.DOTALL)
-        self.blankreg = re.compile(r'\s*(?P<openline><p(?!\sclass=\"softbreak\")[^>]*>)\s*(?P<closeline></p>)', re.IGNORECASE)
+        self.blankreg = re.compile(r'\s*(?P<openline><p(?!\sclass=\"(softbreak|spacer)\")[^>]*>)\s*(?P<closeline></p>)', re.IGNORECASE)
-        self.softbreak = re.compile(r'\s*(?P<openline><p(?=\sclass=\"softbreak\")[^>]*>)\s*(?P<closeline></p>)', re.IGNORECASE)
+        self.anyblank = re.compile(r'\s*(?P<openline><p[^>]*>)\s*(?P<closeline></p>)', re.IGNORECASE)
-        self.multi_blank = re.compile(r'(\s*<p[^>]*>\s*</p>){2,}', re.IGNORECASE)
+        self.multi_blank = re.compile(r'(\s*<p[^>]*>\s*</p>){2,}(?!\s*<h\d)', re.IGNORECASE)
 
     def is_pdftohtml(self, src):
         return '<!-- created by calibre\'s pdftohtml -->' in src[:1000]
@@ -42,8 +43,10 @@ class HeuristicProcessor(object):
                     " chapters. - " + unicode(chap))
             return '<h2>'+chap+'</h2>\n'
         else:
-            txt_chap = html2text(chap)
+            delete_whitespace = re.compile('^\s*(?P<c>.*?)\s*$')
-            txt_title = html2text(title)
+            delete_quotes = re.compile('\'\"')
+            txt_chap = delete_quotes.sub('', delete_whitespace.sub('\g<c>', html2text(chap)))
+            txt_title = delete_quotes.sub('', delete_whitespace.sub('\g<c>', html2text(title)))
             self.html_preprocess_sections = self.html_preprocess_sections + 1
             self.log.debug("marked " + unicode(self.html_preprocess_sections) +
                     " chapters & titles. - " + unicode(chap) + ", " + unicode(title))
@@ -155,7 +158,7 @@ class HeuristicProcessor(object):
         ]
 
         for word in ITALICIZE_WORDS:
-            html = re.sub(r'(?<=\s|>)' + word + r'(?=\s|<)', '<i>%s</i>' % word, html)
+            html = re.sub(r'(?<=\s|>)' + re.escape(word) + r'(?=\s|<)', '<i>%s</i>' % word, html)
 
         for pat in ITALICIZE_STYLE_PATS:
             html = re.sub(pat, lambda mo: '<i>%s</i>' % mo.group('words'), html)
@@ -375,9 +378,9 @@ class HeuristicProcessor(object):
         html = re.sub('<p\s?/>', '', html)
         # Get rid of empty span, bold, font, em, & italics tags
         html = re.sub(r"\s*<span[^>]*>\s*(<span[^>]*>\s*</span>){0,2}\s*</span>\s*", " ", html)
-        html = re.sub(r"\s*<(font|[ibu]|em)[^>]*>\s*(<(font|[ibu]|em)[^>]*>\s*</(font|[ibu]|em)>\s*){0,2}\s*</(font|[ibu]|em)>", " ", html)
+        html = re.sub(r"\s*<(font|[ibu]|em|strong)[^>]*>\s*(<(font|[ibu]|em|strong)[^>]*>\s*</(font|[ibu]|em|strong)>\s*){0,2}\s*</(font|[ibu]|em|strong)>", " ", html)
         html = re.sub(r"\s*<span[^>]*>\s*(<span[^>]>\s*</span>){0,2}\s*</span>\s*", " ", html)
-        html = re.sub(r"\s*<(font|[ibu]|em)[^>]*>\s*(<(font|[ibu]|em)[^>]*>\s*</(font|[ibu]|em)>\s*){0,2}\s*</(font|[ibu]|em)>", " ", html)
+        html = re.sub(r"\s*<(font|[ibu]|em|strong)[^>]*>\s*(<(font|[ibu]|em|strong)[^>]*>\s*</(font|[ibu]|em|strong)>\s*){0,2}\s*</(font|[ibu]|em|strong)>", " ", html)
         self.deleted_nbsps = True
         return html
 
@@ -416,6 +419,28 @@ class HeuristicProcessor(object):
                 return True
         return False
 
+    def detect_blank_formatting(self, html):
+        blanks_before_headings = re.compile(r'(\s*<p[^>]*>\s*</p>){1,}(?=\s*<h\d)', re.IGNORECASE)
+        blanks_after_headings = re.compile(r'(?<=</h\d>)(\s*<p[^>]*>\s*</p>){1,}', re.IGNORECASE)
+        
+        def markup_spacers(match):
+           blanks = match.group(0)
+           blanks = self.blankreg.sub('\n<p class="spacer"> </p>', blanks)
+           return blanks
+        html = blanks_before_headings.sub(markup_spacers, html)
+        html = blanks_after_headings.sub(markup_spacers, html)
+        if self.html_preprocess_sections > self.min_chapters:
+            html = re.sub('(?si)^.*?(?=<h\d)', markup_spacers, html)
+        return html
+
+    def detect_soft_breaks(self, html):
+        if not self.blanks_deleted and self.blanks_between_paragraphs:
+            html = self.multi_blank.sub('\n<p class="softbreak" style="margin-top:1.25em; margin-bottom:1.25em; page-break-before:avoid"> </p>', html)
+        else:
+            html = self.blankreg.sub('\n<p class="softbreak" style="margin-top:1.25em; margin-bottom:1.25em; page-break-before:avoid"> </p>', html)
+        return html
+
+
 
     def __call__(self, html):
         self.log.debug("*********  Heuristic processing HTML  *********")
@@ -457,23 +482,23 @@ class HeuristicProcessor(object):
         #html = re.sub('<br[^>]*>', u'<p>\u00a0</p>', html)
 
         # Determine whether the document uses interleaved blank lines
-        blanks_between_paragraphs = self.analyze_blanks(html)
+        self.blanks_between_paragraphs = self.analyze_blanks(html)
 
         #self.dump(html, 'before_chapter_markup')
         # detect chapters/sections to match xpath or splitting logic
 
         if getattr(self.extra_opts, 'markup_chapter_headings', False):
-            html = self.markup_chapters(html, self.totalwords, blanks_between_paragraphs)
+            html = self.markup_chapters(html, self.totalwords, self.blanks_between_paragraphs)
 
         if getattr(self.extra_opts, 'italicize_common_cases', False):
             html = self.markup_italicis(html)
 
         # If more than 40% of the lines are empty paragraphs and the user has enabled delete
         # blank paragraphs then delete blank lines to clean up spacing
-        if blanks_between_paragraphs and getattr(self.extra_opts, 'delete_blank_paragraphs', False):
+        if self.blanks_between_paragraphs and getattr(self.extra_opts, 'delete_blank_paragraphs', False):
             self.log.debug("deleting blank lines")
             self.blanks_deleted = True
-            html = self.multi_blank.sub('\n<p class="softbreak" style="margin-top:1.5em; margin-bottom:1.5em"> </p>', html)
+            html = self.multi_blank.sub('\n<p class="softbreak" style="margin-top:1.25em; margin-bottom:1.25em; page-break-before:avoid"> </p>', html)
             html = self.blankreg.sub('', html)
 
         # Determine line ending type
@@ -525,14 +550,13 @@ class HeuristicProcessor(object):
             html = doubleheading.sub('\g<firsthead>'+'\n<h3'+'\g<secondhead>'+'</h3>', html)
 
         if getattr(self.extra_opts, 'format_scene_breaks', False):
+            html = self.detect_blank_formatting(html)
+            html = self.detect_soft_breaks(html)
             # Center separator lines
-            html = re.sub(u'<(?P<outer>p|div)[^>]*>\s*(<(?P<inner1>font|span|[ibu])[^>]*>)?\s*(<(?P<inner2>font|span|[ibu])[^>]*>)?\s*(<(?P<inner3>font|span|[ibu])[^>]*>)?\s*(?P<break>([*#•=✦]+\s*)+)\s*(</(?P=inner3)>)?\s*(</(?P=inner2)>)?\s*(</(?P=inner1)>)?\s*</(?P=outer)>', '<p style="text-align:center; margin-top:1.25em; margin-bottom:1.25em">' + '\g<break>' + '</p>', html)
+            html = re.sub(u'<(?P<outer>p|div)[^>]*>\s*(<(?P<inner1>font|span|[ibu])[^>]*>)?\s*(<(?P<inner2>font|span|[ibu])[^>]*>)?\s*(<(?P<inner3>font|span|[ibu])[^>]*>)?\s*(?P<break>([*#•=✦]+\s*)+)\s*(</(?P=inner3)>)?\s*(</(?P=inner2)>)?\s*(</(?P=inner1)>)?\s*</(?P=outer)>', '<p style="text-align:center; margin-top:1.25em; margin-bottom:1.25em; page-break-before:avoid">' + '\g<break>' + '</p>', html)
-            if not self.blanks_deleted:
+            #html = re.sub('<p\s+class="softbreak"[^>]*>\s*</p>', '<div id="softbreak" style="margin-left: 45%; margin-right: 45%; margin-top:1.5em; margin-bottom:1.5em"><hr style="height: 3px; background:#505050" /></div>', html)
-                html = self.multi_blank.sub('\n<p class="softbreak" style="margin-top:1.5em; margin-bottom:1.5em"> </p>', html)
-            html = re.sub('<p\s+class="softbreak"[^>]*>\s*</p>', '<div id="softbreak" style="margin-left: 45%; margin-right: 45%; margin-top:1.5em; margin-bottom:1.5em"><hr style="height: 3px; background:#505050" /></div>', html)
 
         if self.deleted_nbsps:
             # put back non-breaking spaces in empty paragraphs to preserve original formatting
-            html = self.blankreg.sub('\n'+r'\g<openline>'+u'\u00a0'+r'\g<closeline>', html)
+            html = self.anyblank.sub('\n'+r'\g<openline>'+u'\u00a0'+r'\g<closeline>', html)
-            html = self.softbreak.sub('\n'+r'\g<openline>'+u'\u00a0'+r'\g<closeline>', html)
         return html

src/calibre/ebooks/lit/input.py

@@ -39,12 +39,15 @@ class LITInput(InputFormatPlugin):
                     pre = body[0]
                     from calibre.ebooks.txt.processor import convert_basic, preserve_spaces, \
                         separate_paragraphs_single_line
+                    from calibre.ebooks.chardet import xml_to_unicode
                     from lxml import etree
                     import copy
                     html = separate_paragraphs_single_line(pre.text)
                     html = preserve_spaces(html)
                     html = convert_basic(html).replace('<html>',
                             '<html xmlns="%s">'%XHTML_NS)
+                    html = xml_to_unicode(html, strip_encoding_pats=True,
+                            resolve_entities=True)[0]
                     root = etree.fromstring(html)
                     body = XPath('//h:body')(root)
                     pre.tag = XHTML('div')

src/calibre/ebooks/mobi/reader.py

@@ -488,7 +488,7 @@ class MobiReader(object):
 
 
     def remove_random_bytes(self, html):
-        return re.sub('\x14|\x15|\x19|\x1c|\x1d|\xef|\x12|\x13|\xec|\x08',
+        return re.sub('\x14|\x15|\x19|\x1c|\x1d|\xef|\x12|\x13|\xec|\x08|\x01|\x02|\x03|\x04|\x05|\x06|\x07',
                     '', html)
 
     def ensure_unit(self, raw, unit='px'):

src/calibre/ebooks/rtf/rtfml.py

@@ -24,14 +24,15 @@ from calibre.utils.magick.draw import save_cover_data_to, identify_data
 TAGS = {
     'b': '\\b',
     'del': '\\deleted',
-    'h1': '\\b \\par \\pard \\hyphpar',
+    'h1': '\\s1 \\afs32',
-    'h2': '\\b \\par \\pard \\hyphpar',
+    'h2': '\\s2 \\afs28',
-    'h3': '\\b \\par \\pard \\hyphpar',
+    'h3': '\\s3 \\afs28',
-    'h4': '\\b \\par \\pard \\hyphpar',
+    'h4': '\\s4 \\afs23',
-    'h5': '\\b \\par \\pard \\hyphpar',
+    'h5': '\\s5 \\afs23',
-    'h6': '\\b \\par \\pard \\hyphpar',
+    'h6': '\\s6 \\afs21',
-    'li': '\\par \\pard \\hyphpar \t',
+    'i': '\\i',
-    'p': '\\par \\pard \\hyphpar \t',
+    'li': '\t',
+    'p': '\t',
     'sub': '\\sub',
     'sup': '\\super',
     'u': '\\ul',
@@ -39,15 +40,9 @@ TAGS = {
 
 SINGLE_TAGS = {
     'br': '\n{\\line }\n',
-    'div': '\n{\\line }\n',
-}
-
-SINGLE_TAGS_END = {
-    'div': '\n{\\line }\n',
 }
 
 STYLES = [
-    ('display', {'block': '\\par \\pard \\hyphpar'}),
     ('font-weight', {'bold': '\\b', 'bolder': '\\b'}),
     ('font-style', {'italic': '\\i'}),
     ('text-align', {'center': '\\qc', 'left': '\\ql', 'right': '\\qr'}),
@@ -55,6 +50,7 @@ STYLES = [
 ]
 
 BLOCK_TAGS = [
+    'div',
     'p',
     'h1',
     'h2',
@@ -112,14 +108,16 @@ class RTFMLizer(object):
                 stylizer = Stylizer(item.data, item.href, self.oeb_book,
                         self.opts, self.opts.output_profile)
                 output += self.dump_text(item.data.find(XHTML('body')), stylizer)
-                output += '{\\page } '
+                output += '{\\page }'
         for item in self.oeb_book.spine:
             self.log.debug('Converting %s to RTF markup...' % item.href)
             content = unicode(etree.tostring(item.data, encoding=unicode))
             content = self.remove_newlines(content)
+            content = self.remove_tabs(content)
             content = etree.fromstring(content)
             stylizer = Stylizer(content, item.href, self.oeb_book, self.opts, self.opts.output_profile)
             output += self.dump_text(content.find(XHTML('body')), stylizer)
+            output += '{\\page }'
         output += self.footer()
         output = self.insert_images(output)
         output = self.clean_text(output)
@@ -134,8 +132,23 @@ class RTFMLizer(object):
 
         return text
 
+    def remove_tabs(self, text):
+        self.log.debug('\Replace tabs with space for processing...')
+        text = text.replace('\t', ' ')
+
+        return text
+
     def header(self):
-        return u'{\\rtf1{\\info{\\title %s}{\\author %s}}\\ansi\\ansicpg1252\\deff0\\deflang1033' % (self.oeb_book.metadata.title[0].value, authors_to_string([x.value for x in self.oeb_book.metadata.creator]))
+        header = u'{\\rtf1{\\info{\\title %s}{\\author %s}}\\ansi\\ansicpg1252\\deff0\\deflang1033\n' % (self.oeb_book.metadata.title[0].value, authors_to_string([x.value for x in self.oeb_book.metadata.creator]))
+        return header + \
+            '{\\fonttbl{\\f0\\froman\\fprq2\\fcharset128 Times New Roman;}{\\f1\\froman\\fprq2\\fcharset128 Times New Roman;}{\\f2\\fswiss\\fprq2\\fcharset128 Arial;}{\\f3\\fnil\\fprq2\\fcharset128 Arial;}{\\f4\\fnil\\fprq2\\fcharset128 MS Mincho;}{\\f5\\fnil\\fprq2\\fcharset128 Tahoma;}{\\f6\\fnil\\fprq0\\fcharset128 Tahoma;}}\n' \
+            '{\\stylesheet{\\ql \\li0\\ri0\\nowidctlpar\\wrapdefault\\faauto\\rin0\\lin0\\itap0 \\rtlch\\fcs1 \\af25\\afs24\\alang1033 \\ltrch\\fcs0 \\fs24\\lang1033\\langfe255\\cgrid\\langnp1033\\langfenp255 \\snext0 Normal;}\n' \
+            '{\\s1\\ql \\li0\\ri0\\sb240\\sa120\\keepn\\nowidctlpar\\wrapdefault\\faauto\\outlinelevel0\\rin0\\lin0\\itap0 \\rtlch\\fcs1 \\ab\\af0\\afs32\\alang1033 \\ltrch\\fcs0 \\b\\fs32\\lang1033\\langfe255\\loch\\f1\\hich\\af1\\dbch\\af26\\cgrid\\langnp1033\\langfenp255 \\sbasedon15 \\snext16 \\slink21 heading 1;}\n' \
+            '{\\s2\\ql \\li0\\ri0\\sb240\\sa120\\keepn\\nowidctlpar\\wrapdefault\\faauto\\outlinelevel1\\rin0\\lin0\\itap0 \\rtlch\\fcs1 \\ab\\ai\\af0\\afs28\\alang1033 \\ltrch\\fcs0 \\b\\i\\fs28\\lang1033\\langfe255\\loch\\f1\\hich\\af1\\dbch\\af26\\cgrid\\langnp1033\\langfenp255 \\sbasedon15 \\snext16 \\slink22 heading 2;}\n' \
+            '{\\s3\\ql \\li0\\ri0\\sb240\\sa120\\keepn\\nowidctlpar\\wrapdefault\\faauto\\outlinelevel2\\rin0\\lin0\\itap0 \\rtlch\\fcs1 \\ab\\af0\\afs28\\alang1033 \\ltrch\\fcs0 \\b\\fs28\\lang1033\\langfe255\\loch\\f1\\hich\\af1\\dbch\\af26\\cgrid\\langnp1033\\langfenp255 \\sbasedon15 \\snext16 \\slink23 heading 3;}\n' \
+            '{\\s4\\ql \\li0\\ri0\\sb240\\sa120\\keepn\\nowidctlpar\\wrapdefault\\faauto\\outlinelevel3\\rin0\\lin0\\itap0 \\rtlch\\fcs1 \\ab\\ai\\af0\\afs23\\alang1033 \\ltrch\\fcs0\\b\\i\\fs23\\lang1033\\langfe255\\loch\\f1\\hich\\af1\\dbch\\af26\\cgrid\\langnp1033\\langfenp255 \\sbasedon15 \\snext16 \\slink24 heading 4;}\n' \
+            '{\\s5\\ql \\li0\\ri0\\sb240\\sa120\\keepn\\nowidctlpar\\wrapdefault\\faauto\\outlinelevel4\\rin0\\lin0\\itap0 \\rtlch\\fcs1 \\ab\\af0\\afs23\\alang1033 \\ltrch\\fcs0 \\b\\fs23\\lang1033\\langfe255\\loch\\f1\\hich\\af1\\dbch\\af26\\cgrid\\langnp1033\\langfenp255 \\sbasedon15 \\snext16 \\slink25 heading 5;}\n' \
+            '{\\s6\\ql \\li0\\ri0\\sb240\\sa120\\keepn\\nowidctlpar\\wrapdefault\\faauto\\outlinelevel5\\rin0\\lin0\\itap0 \\rtlch\\fcs1 \\ab\\af0\\afs21\\alang1033 \\ltrch\\fcs0 \\b\\fs21\\lang1033\\langfe255\\loch\\f1\\hich\\af1\\dbch\\af26\\cgrid\\langnp1033\\langfenp255 \\sbasedon15 \\snext16 \\slink26 heading 6;}}\n'
 
     def footer(self):
         return ' }'
@@ -170,19 +183,16 @@ class RTFMLizer(object):
         return (hex_string, width, height)
 
     def clean_text(self, text):
-        # Remove excess spaces at beginning and end of lines
-        text = re.sub('(?m)^[ ]+', '', text)
-        text = re.sub('(?m)[ ]+$', '', text)
-
         # Remove excessive newlines
-        #text = re.sub('%s{1,1}' % os.linesep, '%s%s' % (os.linesep, os.linesep), text)
         text = re.sub('%s{3,}' % os.linesep, '%s%s' % (os.linesep, os.linesep), text)
 
         # Remove excessive spaces
         text = re.sub('[ ]{2,}', ' ', text)
+        text = re.sub('\t{2,}', '\t', text)
+        text = re.sub('\t ', '\t', text)
 
+        # Remove excessive line breaks
         text = re.sub(r'(\{\\line \}\s*){3,}', r'{\\line }{\\line }', text)
-        #text = re.compile(r'(\{\\line \}\s*)+(?P<brackets>}*)\s*\{\\par').sub(lambda mo: r'%s{\\par' % mo.group('brackets'), text)
 
         # Remove non-breaking spaces
         text = text.replace(u'\xa0', ' ')
@@ -222,7 +232,7 @@ class RTFMLizer(object):
             block_start = ''
             block_end = ''
             if 'block' not in tag_stack:
-                block_start = '{\\par \\pard \\hyphpar '
+                block_start = '{\\par\\pard\\hyphpar '
                 block_end = '}'
             text += '%s SPECIAL_IMAGE-%s-REPLACE_ME %s' % (block_start, src, block_end)
 
@@ -245,7 +255,7 @@ class RTFMLizer(object):
                 tag_stack.append(style_tag)
 
         # Proccess tags that contain text.
-        if hasattr(elem, 'text') and elem.text != None and elem.text.strip() != '':
+        if hasattr(elem, 'text') and elem.text:
             text += txt2rtf(elem.text)
 
         for item in elem:
@@ -254,16 +264,15 @@ class RTFMLizer(object):
         for i in range(0, tag_count):
             end_tag =  tag_stack.pop()
             if end_tag != 'block':
+                if tag in BLOCK_TAGS:
+                    text += u'\\par\\pard\\plain\\hyphpar}'
+                else:
                     text += u'}'
 
-        single_tag_end = SINGLE_TAGS_END.get(tag, None)
+        if hasattr(elem, 'tail') and elem.tail:
-        if single_tag_end:
-            text += single_tag_end
-
-        if hasattr(elem, 'tail') and elem.tail != None and elem.tail.strip() != '':
             if 'block' in tag_stack:
                 text += '%s' % txt2rtf(elem.tail)
             else:
-                text += '{\\par \\pard \\hyphpar %s}' % txt2rtf(elem.tail)
+                text += '{\\par\\pard\\hyphpar %s}' % txt2rtf(elem.tail)
 
         return text

src/calibre/ebooks/txt/output.py

@@ -8,7 +8,6 @@ import os
 
 from calibre.customize.conversion import OutputFormatPlugin, \
     OptionRecommendation
-from calibre.ebooks.txt.markdownml import MarkdownMLizer
 from calibre.ebooks.txt.txtml import TXTMLizer
 from calibre.ebooks.txt.newlines import TxtNewlines, specified_newlines
 
@@ -44,24 +43,32 @@ class TXTOutput(OutputFormatPlugin):
             recommended_value=False, level=OptionRecommendation.LOW,
             help=_('Force splitting on the max-line-length value when no space '
             'is present. Also allows max-line-length to be below the minimum')),
-        OptionRecommendation(name='markdown_format',
+        OptionRecommendation(name='txt_output_formatting',
-            recommended_value=False, level=OptionRecommendation.LOW,
+             recommended_value='plain',
-            help=_('Produce Markdown formatted text.')),
+             choices=['plain', 'markdown', 'textile'],
+             help=_('Formatting used within the document.\n'
+                    '* plain: Produce plain text.\n'
+                    '* markdown: Produce Markdown formatted text.\n'
+                    '* textile: Produce Textile formatted text.')),
         OptionRecommendation(name='keep_links',
             recommended_value=False, level=OptionRecommendation.LOW,
             help=_('Do not remove links within the document. This is only ' \
-            'useful when paired with the markdown-format option because' \
+            'useful when paired with a txt-output-formatting option that '
-            ' links are always removed with plain text output.')),
+            'is not none because links are always removed with plain text output.')),
         OptionRecommendation(name='keep_image_references',
             recommended_value=False, level=OptionRecommendation.LOW,
             help=_('Do not remove image references within the document. This is only ' \
-            'useful when paired with the markdown-format option because' \
+            'useful when paired with a txt-output-formatting option that '
-            ' image references are always removed with plain text output.')),
+            'is not none because links are always removed with plain text output.')),
      ])
 
     def convert(self, oeb_book, output_path, input_plugin, opts, log):
-        if opts.markdown_format:
+        if opts.txt_output_formatting.lower() == 'markdown':
+            from calibre.ebooks.txt.markdownml import MarkdownMLizer
             writer = MarkdownMLizer(log)
+        elif opts.txt_output_formatting.lower() == 'textile':
+            from calibre.ebooks.txt.textileml import TextileMLizer
+            writer = TextileMLizer(log)
         else:
             writer = TXTMLizer(log)
 

src/calibre/ebooks/txt/textileml.py

@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+'''
+Transform OEB content into Textile formatted plain text
+'''
+
+import re
+
+from lxml import etree
+
+from calibre.ebooks.oeb.base import XHTML
+from calibre.utils.html2textile import html2textile
+
+class TextileMLizer(object):
+
+    def __init__(self, log):
+        self.log = log
+
+    def extract_content(self, oeb_book, opts):
+        self.log.info('Converting XHTML to Textile formatted TXT...')
+        self.oeb_book = oeb_book
+        self.opts = opts
+
+        return self.mlize_spine()
+
+    def mlize_spine(self):
+        output = [u'']
+
+        for item in self.oeb_book.spine:
+            self.log.debug('Converting %s to Textile formatted TXT...' % item.href)
+
+            html = unicode(etree.tostring(item.data.find(XHTML('body')), encoding=unicode))
+
+            if not self.opts.keep_links:
+                html = re.sub(r'<\s*a[^>]*>', '', html)
+                html = re.sub(r'<\s*/\s*a\s*>', '', html)
+            if not self.opts.keep_image_references:
+                html = re.sub(r'<\s*img[^>]*>', '', html)
+                html = re.sub(r'<\s*img\s*>', '', html)
+
+            text = html2textile(html)
+
+            # Ensure the section ends with at least two new line characters.
+            # This is to prevent the last paragraph from a section being
+            # combined into the fist paragraph of the next.
+            end_chars = text[-4:]
+            # Convert all newlines to \n
+            end_chars = end_chars.replace('\r\n', '\n')
+            end_chars = end_chars.replace('\r', '\n')
+            end_chars = end_chars[-2:]
+            if not end_chars[1] == '\n':
+                text += '\n\n'
+            if end_chars[1] == '\n' and not end_chars[0] == '\n':
+                text += '\n'
+
+            output += text
+
+        output = u''.join(output)
+
+        return output

src/calibre/ebooks/txt/txtml.py

@@ -222,6 +222,8 @@ class TXTMLizer(object):
         # Scene breaks.
         if tag == 'hr':
             text.append('\n\n* * *\n\n')
+        elif style['margin-top']:
+            text.append('\n\n' + '\n' * round(style['margin-top']))
 
         # Process tags that contain text.
         if hasattr(elem, 'text') and elem.text:

src/calibre/gui2/__init__.py

@@ -120,6 +120,8 @@ def _config():
         help='Search history for the LRF viewer')
     c.add_opt('scheduler_search_history', default=[],
         help='Search history for the recipe scheduler')
+    c.add_opt('plugin_search_history', default=[],
+        help='Search history for the recipe scheduler')
     c.add_opt('worker_limit', default=6,
             help=_('Maximum number of waiting worker processes'))
     c.add_opt('get_social_metadata', default=True,
@@ -138,6 +140,7 @@ def _config():
             help=_('Show the average rating per item indication in the tag browser'))
     c.add_opt('disable_animations', default=False,
             help=_('Disable UI animations'))
+    c.add_opt
     return ConfigProxy(c)
 
 config = _config()

src/calibre/gui2/actions/choose_library.py

@@ -16,7 +16,6 @@ from calibre.utils.config import prefs
 from calibre.gui2 import gprefs, warning_dialog, Dispatcher, error_dialog, \
     question_dialog, info_dialog
 from calibre.gui2.actions import InterfaceAction
-from calibre.gui2.dialogs.check_library import CheckLibraryDialog, DBCheck
 
 class LibraryUsageStats(object): # {{{
 
@@ -139,6 +138,12 @@ class ChooseLibraryAction(InterfaceAction):
                                       None, None), attr='action_check_library')
         ac.triggered.connect(self.check_library, type=Qt.QueuedConnection)
         self.maintenance_menu.addAction(ac)
+        ac = self.create_action(spec=(_('Restore database'), 'lt.png',
+                                      None, None),
+                                      attr='action_restore_database')
+        ac.triggered.connect(self.restore_database, type=Qt.QueuedConnection)
+        self.maintenance_menu.addAction(ac)
+
         self.choose_menu.addMenu(self.maintenance_menu)
 
     def pick_random(self, *args):
@@ -267,7 +272,17 @@ class ChooseLibraryAction(InterfaceAction):
             _('Metadata will be backed up while calibre is running, at the '
               'rate of approximately 1 book every three seconds.'), show=True)
 
+    def restore_database(self):
+        from calibre.gui2.dialogs.restore_library import restore_database
+        m = self.gui.library_view.model()
+        m.stop_metadata_backup()
+        db = m.db
+        db.prefs.disable_setting = True
+        if restore_database(db, self.gui):
+            self.gui.library_moved(db.library_path, call_close=False)
+
     def check_library(self):
+        from calibre.gui2.dialogs.check_library import CheckLibraryDialog, DBCheck
         self.gui.library_view.save_state()
         m = self.gui.library_view.model()
         m.stop_metadata_backup()

src/calibre/gui2/convert/comic_input.py

@@ -22,7 +22,7 @@ class PluginWidget(Widget, Ui_Form):
                 ['colors', 'dont_normalize', 'keep_aspect_ratio', 'right2left',
                     'despeckle', 'no_sort', 'no_process', 'landscape',
                     'dont_sharpen', 'disable_trim', 'wide', 'output_format',
-                    'dont_grayscale']
+                    'dont_grayscale', 'comic_image_size']
                 )
         self.db, self.book_id = db, book_id
         for x in get_option('output_format').option.choices:

src/calibre/gui2/convert/comic_input.ui

@@ -7,7 +7,7 @@
     <x>0</x>
     <y>0</y>
     <width>599</width>
-    <height>345</height>
+    <height>398</height>
    </rect>
   </property>
   <property name="windowTitle">
@@ -37,70 +37,70 @@
      </property>
     </widget>
    </item>
-   <item row="3" column="0">
+   <item row="4" column="0">
     <widget class="QCheckBox" name="opt_dont_normalize">
      <property name="text">
       <string>Disable &amp;normalize</string>
      </property>
     </widget>
    </item>
-   <item row="4" column="0">
+   <item row="5" column="0">
     <widget class="QCheckBox" name="opt_keep_aspect_ratio">
      <property name="text">
       <string>Keep &amp;aspect ratio</string>
      </property>
     </widget>
    </item>
-   <item row="5" column="0">
+   <item row="6" column="0">
     <widget class="QCheckBox" name="opt_dont_sharpen">
      <property name="text">
       <string>Disable &amp;Sharpening</string>
      </property>
     </widget>
    </item>
-   <item row="6" column="0">
+   <item row="7" column="0">
     <widget class="QCheckBox" name="opt_disable_trim">
      <property name="text">
       <string>Disable &amp;Trimming</string>
      </property>
     </widget>
    </item>
-   <item row="7" column="0">
+   <item row="8" column="0">
     <widget class="QCheckBox" name="opt_wide">
      <property name="text">
       <string>&amp;Wide</string>
      </property>
     </widget>
    </item>
-   <item row="8" column="0">
+   <item row="9" column="0">
     <widget class="QCheckBox" name="opt_landscape">
      <property name="text">
       <string>&amp;Landscape</string>
      </property>
     </widget>
    </item>
-   <item row="9" column="0">
+   <item row="10" column="0">
     <widget class="QCheckBox" name="opt_right2left">
      <property name="text">
       <string>&amp;Right to left</string>
      </property>
     </widget>
    </item>
-   <item row="10" column="0">
+   <item row="11" column="0">
     <widget class="QCheckBox" name="opt_no_sort">
      <property name="text">
       <string>Don't so&amp;rt</string>
      </property>
     </widget>
    </item>
-   <item row="11" column="0">
+   <item row="12" column="0">
     <widget class="QCheckBox" name="opt_despeckle">
      <property name="text">
       <string>De&amp;speckle</string>
      </property>
     </widget>
    </item>
-   <item row="13" column="0">
+   <item row="14" column="0">
     <spacer name="verticalSpacer">
      <property name="orientation">
       <enum>Qt::Vertical</enum>
@@ -120,7 +120,7 @@
      </property>
     </widget>
    </item>
-   <item row="12" column="0">
+   <item row="13" column="0">
     <widget class="QLabel" name="label">
      <property name="text">
       <string>&amp;Output format:</string>
@@ -130,7 +130,7 @@
      </property>
     </widget>
    </item>
-   <item row="12" column="1">
+   <item row="13" column="1">
     <widget class="QComboBox" name="opt_output_format"/>
    </item>
    <item row="1" column="0">
@@ -140,6 +140,19 @@
      </property>
     </widget>
    </item>
+   <item row="3" column="0">
+    <widget class="QLabel" name="label_2">
+     <property name="text">
+      <string>Override image  &amp;size:</string>
+     </property>
+     <property name="buddy">
+      <cstring>opt_comic_image_size</cstring>
+     </property>
+    </widget>
+   </item>
+   <item row="3" column="1">
+    <widget class="QLineEdit" name="opt_comic_image_size"/>
+   </item>
   </layout>
  </widget>
  <resources/>

src/calibre/gui2/convert/txt_output.py

@@ -4,7 +4,6 @@ __license__ = 'GPL 3'
 __copyright__ = '2009, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
-from PyQt4.Qt import Qt
 
 from calibre.gui2.convert.txt_output_ui import Ui_Form
 from calibre.gui2.convert import Widget
@@ -21,26 +20,14 @@ class PluginWidget(Widget, Ui_Form):
     def __init__(self, parent, get_option, get_help, db=None, book_id=None):
         Widget.__init__(self, parent,
         ['newline', 'max_line_length', 'force_max_line_length',
-        'inline_toc', 'markdown_format', 'keep_links', 'keep_image_references',
+        'inline_toc', 'txt_output_formatting', 'keep_links', 'keep_image_references',
         'txt_output_encoding'])
         self.db, self.book_id = db, book_id
         for x in get_option('newline').option.choices:
             self.opt_newline.addItem(x)
+        for x in get_option('txt_output_formatting').option.choices:
+            self.opt_txt_output_formatting.addItem(x)
         self.initialize_options(get_option, get_help, db, book_id)
 
-        self.opt_markdown_format.stateChanged.connect(self.enable_markdown_format)
-        self.enable_markdown_format(self.opt_markdown_format.checkState())
-
     def break_cycles(self):
         Widget.break_cycles(self)
-
-        try:
-            self.opt_markdown_format.stateChanged.disconnect()
-        except:
-            pass
-
-    def enable_markdown_format(self, state):
-        state = state == Qt.Checked
-        self.opt_keep_links.setEnabled(state)
-        self.opt_keep_image_references.setEnabled(state)
-

src/calibre/gui2/convert/txt_output.ui

@@ -6,15 +6,38 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>477</width>
+    <width>392</width>
-    <height>300</height>
+    <height>346</height>
    </rect>
   </property>
   <property name="windowTitle">
    <string>Form</string>
   </property>
+  <layout class="QVBoxLayout" name="verticalLayout_2">
+   <item>
+    <widget class="QGroupBox" name="groupBox">
+     <property name="title">
+      <string>General</string>
+     </property>
      <layout class="QGridLayout" name="gridLayout">
       <item row="0" column="0">
+       <widget class="QLabel" name="label_3">
+        <property name="text">
+         <string>Output &amp;Encoding:</string>
+        </property>
+        <property name="buddy">
+         <cstring>opt_txt_output_encoding</cstring>
+        </property>
+       </widget>
+      </item>
+      <item row="0" column="1">
+       <widget class="EncodingComboBox" name="opt_txt_output_encoding">
+        <property name="editable">
+         <bool>true</bool>
+        </property>
+       </widget>
+      </item>
+      <item row="1" column="0">
        <widget class="QLabel" name="label">
         <property name="text">
          <string>&amp;Line ending style:</string>
@@ -24,32 +47,31 @@
         </property>
        </widget>
       </item>
-   <item row="0" column="1">
+      <item row="1" column="1">
        <widget class="QComboBox" name="opt_newline"/>
       </item>
-   <item row="8" column="0">
+      <item row="2" column="0">
-    <spacer name="verticalSpacer">
+       <widget class="QLabel" name="label_4">
-     <property name="orientation">
-      <enum>Qt::Vertical</enum>
-     </property>
-     <property name="sizeHint" stdset="0">
-      <size>
-       <width>20</width>
-       <height>246</height>
-      </size>
-     </property>
-    </spacer>
-   </item>
-   <item row="4" column="0" colspan="2">
-    <widget class="QCheckBox" name="opt_inline_toc">
         <property name="text">
-      <string>&amp;Inline TOC</string>
+         <string>&amp;Formatting:</string>
+        </property>
+        <property name="buddy">
+         <cstring>opt_txt_output_formatting</cstring>
         </property>
        </widget>
       </item>
-   <item row="1" column="1">
+      <item row="2" column="1">
-    <widget class="QSpinBox" name="opt_max_line_length"/>
+       <widget class="QComboBox" name="opt_txt_output_formatting"/>
       </item>
+     </layout>
+    </widget>
+   </item>
+   <item>
+    <widget class="QGroupBox" name="groupBox_2">
+     <property name="title">
+      <string>Plain</string>
+     </property>
+     <layout class="QGridLayout" name="gridLayout_2">
       <item row="1" column="0">
        <widget class="QLabel" name="label_2">
         <property name="text">
@@ -60,46 +82,47 @@
         </property>
        </widget>
       </item>
-   <item row="3" column="0" colspan="2">
+      <item row="1" column="1">
+       <widget class="QSpinBox" name="opt_max_line_length"/>
+      </item>
+      <item row="2" column="0" colspan="2">
        <widget class="QCheckBox" name="opt_force_max_line_length">
         <property name="text">
          <string>Force maximum line length</string>
         </property>
        </widget>
       </item>
-   <item row="5" column="0">
+      <item row="0" column="0">
-    <widget class="QCheckBox" name="opt_markdown_format">
+       <widget class="QCheckBox" name="opt_inline_toc">
         <property name="text">
-      <string>Apply Markdown formatting to text</string>
+         <string>&amp;Inline TOC</string>
         </property>
        </widget>
       </item>
-   <item row="6" column="0">
+     </layout>
+    </widget>
+   </item>
+   <item>
+    <widget class="QGroupBox" name="groupBox_3">
+     <property name="title">
+      <string>Markdown, Textile</string>
+     </property>
+     <layout class="QVBoxLayout" name="verticalLayout">
+      <item>
        <widget class="QCheckBox" name="opt_keep_links">
         <property name="text">
          <string>Do not remove links (&lt;a&gt; tags) before processing</string>
         </property>
        </widget>
       </item>
-   <item row="7" column="0">
+      <item>
        <widget class="QCheckBox" name="opt_keep_image_references">
         <property name="text">
          <string>Do not remove image references before processing</string>
         </property>
        </widget>
       </item>
-   <item row="2" column="0">
+     </layout>
-    <widget class="QLabel" name="label_3">
-     <property name="text">
-      <string>Output Encoding:</string>
-     </property>
-    </widget>
-   </item>
-   <item row="2" column="1">
-    <widget class="EncodingComboBox" name="opt_txt_output_encoding">
-     <property name="editable">
-      <bool>true</bool>
-     </property>
     </widget>
    </item>
   </layout>

src/calibre/gui2/device.py

@@ -687,7 +687,7 @@ class DeviceMixin(object): # {{{
         except:
             pass
         if not self.device_error_dialog.isVisible():
-            self.device_error_dialog.setDetailedText(job.details)
+            self.device_error_dialog.set_details(job.details)
             self.device_error_dialog.show()
 
     # Device connected {{{
@@ -838,9 +838,9 @@ class DeviceMixin(object): # {{{
                             format_count[f] = 1
             for f in self.device_manager.device.settings().format_map:
                 if f in format_count.keys():
-                    formats.append((f, _('%i of %i Books' % (format_count[f], len(rows))), True if f in aval_out_formats else False))
+                    formats.append((f, _('%i of %i Books') % (format_count[f], len(rows))), True if f in aval_out_formats else False)
                 elif f in aval_out_formats:
-                    formats.append((f, _('0 of %i Books' % len(rows)), True))
+                    formats.append((f, _('0 of %i Books') % len(rows)), True)
             d = ChooseFormatDeviceDialog(self, _('Choose format to send to device'), formats)
             if d.exec_() != QDialog.Accepted:
                 return

src/calibre/gui2/dialogs/check_library.py

@@ -7,7 +7,7 @@ import os, shutil
 
 from PyQt4.Qt import QDialog, QVBoxLayout, QHBoxLayout, QTreeWidget, QLabel, \
             QPushButton, QDialogButtonBox, QApplication, QTreeWidgetItem, \
-            QLineEdit, Qt, QProgressBar, QSize, QTimer
+            QLineEdit, Qt, QProgressBar, QSize, QTimer, QIcon, QTextEdit
 
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.library.check_library import CheckLibrary, CHECKS
@@ -16,7 +16,7 @@ from calibre import prints, as_unicode
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.library.sqlite import DBThread, OperationalError
 
-class DBCheck(QDialog):
+class DBCheck(QDialog): # {{{
 
     def __init__(self, parent, db):
         QDialog.__init__(self, parent)
@@ -74,6 +74,7 @@ class DBCheck(QDialog):
             self.reject()
 
     def start_load(self):
+        try:
             self.conn.close()
             self.pb.setMaximum(self.count)
             self.pb.setValue(0)
@@ -89,6 +90,11 @@ class DBCheck(QDialog):
             self.conn.commit()
 
             QTimer.singleShot(0, self.do_one_load)
+        except Exception, e:
+            import traceback
+            self.error = (as_unicode(e), traceback.format_exc())
+            self.reject()
+
 
     def do_one_load(self):
         if self.rejected:
@@ -128,7 +134,7 @@ class DBCheck(QDialog):
     def reject(self):
         self.rejected = True
         QDialog.reject(self)
-
+# }}}
 
 class Item(QTreeWidgetItem):
     pass
@@ -140,9 +146,70 @@ class CheckLibraryDialog(QDialog):
         self.db = db
 
         self.setWindowTitle(_('Check Library -- Problems Found'))
+        self.setWindowIcon(QIcon(I('debug.png')))
 
-        self._layout = QVBoxLayout(self)
+        self._tl = QHBoxLayout()
-        self.setLayout(self._layout)
+        self._layout = QVBoxLayout()
+        self.setLayout(self._tl)
+        self._tl.addLayout(self._layout)
+        self.helpw = QTextEdit(self)
+        self._tl.addWidget(self.helpw)
+        self.helpw.setReadOnly(True)
+        self.helpw.setText(_('''\
+        <h1>Help</h1>
+
+        <p>calibre stores the list of your books and their metadata in a
+        database. The actual book files and covers are stored as normal
+        files in the calibre library folder. The database contains a list of the files
+        and covers belonging to each book entry. This tool checks that the
+        actual files in the library folder on your computer match the
+        information in the database.</p>
+
+        <p>The result of each type of check is shown to the left. The various
+        checks are:
+        </p>
+        <ul>
+        <li><b>Invalid titles</b>: These are files and folders appearing
+        in the library where books titles should, but that do not have the
+        correct form to be a book title.</li>
+        <li><b>Extra titles</b>: These are extra files in your calibre
+        library that appear to be correctly-formed titles, but have no corresponding
+        entries in the database</li>
+        <li><b>Invalid authors</b>: These are files appearing
+        in the library where only author folders should be.</li>
+        <li><b>Extra authors</b>: These are folders in the
+        calibre library that appear to be authors but that do not have entries
+        in the database</li>
+        <li><b>Missing book formats</b>: These are book formats that are in
+        the database but have no corresponding format file in the book's folder.
+        <li><b>Extra book formats</b>: These are book format files found in
+        the book's folder but not in the database.
+        <li><b>Unknown files in books</b>: These are extra files in the
+        folder of each book that do not correspond to a known format or cover
+        file.</li>
+        <li><b>Missing cover files</b>: These represent books that are marked
+        in the database as having covers but the actual cover files are
+        missing.</li>
+        <li><b>Cover files not in database</b>: These are books that have
+        cover files but are marked as not having covers in the database.</li>
+        <li><b>Folder raising exception</b>: These represent folders in the
+        calibre library that could not be processed/understood by this
+        tool.</li>
+        </ul>
+
+        <p>There are two kinds of automatic fixes possible: <i>Delete
+        marked</i> and <i>Fix marked</i>.</p>
+        <p><i>Delete marked</i> is used to remove extra files/folders/covers that
+        have no entries in the database. Check the box next to the item you want
+        to delete. Use with caution.</p>
+        <p><i>Fix marked</i> is applicable only to covers (the two lines marked
+        'fixable'). In the case of missing cover files, checking the fixable
+        box and pushing this button will remove the cover mark from the
+        database for all the files in that category. In the case of extra
+        cover files, checking the fixable box and pushing this button will
+        add the cover mark to the database for all the files in that
+        category.</p>
+        '''))
 
         self.log = QTreeWidget(self)
         self.log.itemChanged.connect(self.item_changed)
@@ -193,7 +260,7 @@ class CheckLibraryDialog(QDialog):
         self._layout.addLayout(h)
 
         self._layout.addWidget(self.bbox)
-        self.resize(750, 500)
+        self.resize(950, 500)
         self.bbox.setEnabled(True)
 
     def do_exec(self):
@@ -341,5 +408,6 @@ class CheckLibraryDialog(QDialog):
 
 if __name__ == '__main__':
     app = QApplication([])
-    d = CheckLibraryDialog()
+    from calibre.library import db
+    d = CheckLibraryDialog(None, db())
     d.exec_()

src/calibre/gui2/dialogs/message_box.py

@@ -45,7 +45,6 @@ class MessageBox(QDialog, Ui_Dialog):
             self.ctc_button.clicked.connect(self.copy_to_clipboard)
 
 
-        if det_msg:
         self.show_det_msg = _('Show &details')
         self.hide_det_msg = _('Hide &details')
         self.det_msg_toggle = self.bb.addButton(self.show_det_msg, self.bb.ActionRole)
@@ -53,7 +52,6 @@ class MessageBox(QDialog, Ui_Dialog):
         self.det_msg_toggle.setToolTip(
                 _('Show detailed information about this error'))
 
-
         self.copy_action = QAction(self)
         self.addAction(self.copy_action)
         self.copy_action.setShortcuts(QKeySequence.Copy)
@@ -66,10 +64,14 @@ class MessageBox(QDialog, Ui_Dialog):
         else:
             self.bb.button(self.bb.Ok).setDefault(True)
 
+        if not det_msg:
+            self.det_msg_toggle.setVisible(False)
+
         self.do_resize()
 
+
     def toggle_det_msg(self, *args):
-        vis = self.det_msg.isVisible()
+        vis = unicode(self.det_msg_toggle.text()) == self.hide_det_msg
         self.det_msg_toggle.setText(self.show_det_msg if vis else
                 self.hide_det_msg)
         self.det_msg.setVisible(not vis)
@@ -100,6 +102,15 @@ class MessageBox(QDialog, Ui_Dialog):
             self.bb.button(self.bb.Ok).setFocus(Qt.OtherFocusReason)
         return ret
 
+    def set_details(self, msg):
+        if not msg:
+            msg = ''
+        self.det_msg.setPlainText(msg)
+        self.det_msg_toggle.setText(self.show_det_msg)
+        self.det_msg_toggle.setVisible(bool(msg))
+        self.det_msg.setVisible(False)
+        self.do_resize()
+
 if __name__ == '__main__':
     app = QApplication([])
     from calibre.gui2 import question_dialog

src/calibre/gui2/dialogs/restore_library.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from PyQt4.Qt import QDialog, QLabel, QVBoxLayout, QDialogButtonBox, \
+        QProgressBar, QSize, QTimer, pyqtSignal, Qt
+
+from calibre.library.restore import Restore
+from calibre.gui2 import error_dialog, question_dialog, warning_dialog, \
+    info_dialog
+
+class DBRestore(QDialog):
+
+    update_signal = pyqtSignal(object, object)
+
+    def __init__(self, parent, library_path):
+        QDialog.__init__(self, parent)
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+        self.l1 = QLabel('<b>'+_('Restoring database from backups, do not'
+            ' interrupt, this will happen in three stages')+'...')
+        self.setWindowTitle(_('Restoring database'))
+        self.l.addWidget(self.l1)
+        self.pb = QProgressBar(self)
+        self.l.addWidget(self.pb)
+        self.pb.setMaximum(0)
+        self.pb.setMinimum(0)
+        self.msg = QLabel('')
+        self.l.addWidget(self.msg)
+        self.msg.setWordWrap(True)
+        self.bb = QDialogButtonBox(QDialogButtonBox.Cancel)
+        self.l.addWidget(self.bb)
+        self.bb.rejected.connect(self.reject)
+        self.resize(self.sizeHint() + QSize(100, 50))
+        self.error = None
+        self.rejected = False
+        self.library_path = library_path
+        self.update_signal.connect(self.do_update, type=Qt.QueuedConnection)
+
+        self.restorer = Restore(library_path, self)
+        self.restorer.daemon = True
+
+        # Give the metadata backup thread time to stop
+        QTimer.singleShot(2000, self.start)
+
+
+    def start(self):
+        self.restorer.start()
+        QTimer.singleShot(10, self.update)
+
+    def reject(self):
+        self.rejected = True
+        self.restorer.progress_callback = lambda x, y: x
+        QDialog.rejecet(self)
+
+    def update(self):
+        if self.restorer.is_alive():
+            QTimer.singleShot(10, self.update)
+        else:
+            self.restorer.progress_callback = lambda x, y: x
+            self.accept()
+
+    def __call__(self, msg, step):
+        self.update_signal.emit(msg, step)
+
+    def do_update(self, msg, step):
+        if msg is None:
+            self.pb.setMaximum(step)
+        else:
+            self.msg.setText(msg)
+            self.pb.setValue(step)
+
+
+def restore_database(db, parent=None):
+    if not question_dialog(parent, _('Are you sure?'), '<p>'+
+            _('Your list of books, with all their metadata is '
+                'stored in a single file, called a database. '
+                'In addition, metadata for each individual '
+                'book is stored in that books\' folder, as '
+                'a backup.'
+                '<p>This operation will rebuild '
+                'the database from the individual book '
+                'metadata. This is useful if the '
+                'database has been corrupted and you get a '
+                'blank list of books. Note that restoring only '
+                'restores books, not any settings stored in the '
+                'database, or any custom recipes.'
+                '<p>Do you want to restore the database?')):
+        return False
+    db.conn.close()
+    d = DBRestore(parent, db.library_path)
+    d.exec_()
+    r = d.restorer
+    d.restorer = None
+    if d.rejected:
+        return True
+    if r.tb is not None:
+        error_dialog(parent, _('Failed'),
+        _('Restoring database failed, click Show details to see details'),
+        det_msg=r.tb, show=True)
+    else:
+        if r.errors_occurred:
+            warning_dialog(parent, _('Success'),
+                    _('Restoring the database succeeded with some warnings'
+                        ' click Show details to see the details.'),
+                    det_msg=r.report, show=True)
+        else:
+            info_dialog(parent, _('Success'),
+                    _('Restoring database was successful'), show=True,
+                    show_copy_button=False)
+    return True
+

src/calibre/gui2/preferences/plugins.py

@@ -17,11 +17,14 @@ from calibre.customize.ui import initialized_plugins, is_disabled, enable_plugin
                                  remove_plugin
 from calibre.gui2 import NONE, error_dialog, info_dialog, choose_files, \
         question_dialog
+from calibre.utils.search_query_parser import SearchQueryParser
+from calibre.utils.icu import lower
 
-class PluginModel(QAbstractItemModel): # {{{
+class PluginModel(QAbstractItemModel, SearchQueryParser): # {{{
 
     def __init__(self, *args):
         QAbstractItemModel.__init__(self, *args)
+        SearchQueryParser.__init__(self, ['all'])
         self.icon = QVariant(QIcon(I('plugins.png')))
         p = QIcon(self.icon).pixmap(32, 32, QIcon.Disabled, QIcon.On)
         self.disabled_icon = QVariant(QIcon(p))
@@ -40,6 +43,72 @@ class PluginModel(QAbstractItemModel): # {{{
         for plugins in self._data.values():
             plugins.sort(cmp=lambda x, y: cmp(x.name.lower(), y.name.lower()))
 
+    def universal_set(self):
+        ans = set([])
+        for c, category in enumerate(self.categories):
+            ans.add((c, -1))
+            for p, plugin in enumerate(self._data[category]):
+                ans.add((c, p))
+        return ans
+
+    def get_matches(self, location, query, candidates=None):
+        if candidates is None:
+            candidates = self.universal_set()
+        ans = set([])
+        if not query:
+            return ans
+        query = lower(query)
+        for c, p in candidates:
+            if p < 0:
+                if query in lower(self.categories[c]):
+                    ans.add((c, p))
+            else:
+                try:
+                    plugin = self._data[self.categories[c]][p]
+                except:
+                    continue
+            if query in lower(plugin.name) or query in lower(plugin.author) or \
+                    query in lower(plugin.description):
+                ans.add((c, p))
+        return ans
+
+    def find(self, query):
+        query = query.strip()
+        matches = self.parse(query)
+        if not matches:
+            return QModelIndex()
+        matches = list(sorted(matches))
+        c, p = matches[0]
+        cat_idx = self.index(c, 0, QModelIndex())
+        if p == -1:
+            return cat_idx
+        return self.index(p, 0, cat_idx)
+
+    def find_next(self, idx, query, backwards=False):
+        query = query.strip()
+        matches = self.parse(query)
+        if not matches:
+            return idx
+        if idx.parent().isValid():
+            loc = (idx.parent().row(), idx.row())
+        else:
+            loc = (idx.row(), -1)
+        if loc not in matches:
+            return self.find(query)
+        if len(matches) == 1:
+            return QModelIndex()
+        matches = list(sorted(matches))
+        i = matches.index(loc)
+        if backwards:
+            ans = i - 1 if i - 1 >= 0 else len(matches)-1
+        else:
+            ans = i + 1 if i + 1 < len(matches) else 0
+
+        ans = matches[ans]
+
+        return self.index(ans[0], 0, QModelIndex()) if ans[1] < 0 else \
+                self.index(ans[1], 0, self.index(ans[0], 0, QModelIndex()))
+
     def index(self, row, column, parent):
         if not self.hasIndex(row, column, parent):
             return QModelIndex()
@@ -127,6 +196,7 @@ class PluginModel(QAbstractItemModel): # {{{
                 return plugin
         return NONE
 
+
 # }}}
 
 class ConfigWidget(ConfigWidgetBase, Ui_Form):
@@ -144,6 +214,42 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         self.customize_plugin_button.clicked.connect(self.customize_plugin)
         self.remove_plugin_button.clicked.connect(self.remove_plugin)
         self.button_plugin_add.clicked.connect(self.add_plugin)
+        self.search.initialize('plugin_search_history',
+                help_text=_('Search for plugin'))
+        self.search.search.connect(self.find)
+        self.next_button.clicked.connect(self.find_next)
+        self.previous_button.clicked.connect(self.find_previous)
+
+    def find(self, query):
+        idx = self._plugin_model.find(query)
+        if not idx.isValid():
+            return info_dialog(self, _('No matches'),
+                    _('Could not find any matching plugins'), show=True,
+                    show_copy_button=False)
+        self.highlight_index(idx)
+
+    def highlight_index(self, idx):
+        self.plugin_view.scrollTo(idx)
+        self.plugin_view.selectionModel().select(idx,
+                self.plugin_view.selectionModel().ClearAndSelect)
+        self.plugin_view.setCurrentIndex(idx)
+
+    def find_next(self, *args):
+        idx = self.plugin_view.currentIndex()
+        if not idx.isValid():
+            idx = self._plugin_model.index(0, 0)
+        idx = self._plugin_model.find_next(idx,
+                unicode(self.search.currentText()))
+        self.highlight_index(idx)
+
+    def find_previous(self, *args):
+        idx = self.plugin_view.currentIndex()
+        if not idx.isValid():
+            idx = self._plugin_model.index(0, 0)
+        idx = self._plugin_model.find_next(idx,
+            unicode(self.search.currentText()), backwards=True)
+        self.highlight_index(idx)
+
 
     def toggle_plugin(self, *args):
         self.modify_plugin(op='toggle')
@@ -160,7 +266,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
 
     def add_plugin(self):
         path = choose_files(self, 'add a plugin dialog', _('Add plugin'),
-                filters=[(_('Plugins'), ['zip'])], all_files=False,
+                filters=[(_('Plugins') + ' (*.zip)', ['zip'])], all_files=False,
                     select_only_single_file=True)
         if not path:
             return
@@ -184,13 +290,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
                     show=True, show_copy_button=False)
             idx = self._plugin_model.plugin_to_index_by_properties(plugin)
             if idx.isValid():
-                self.plugin_view.scrollTo(idx,
+                self.highlight_index(idx)
-                        self.plugin_view.PositionAtCenter)
-                self.plugin_view.scrollTo(idx,
-                        self.plugin_view.PositionAtCenter)
-                self.plugin_view.selectionModel().select(idx,
-                        self.plugin_view.selectionModel().ClearAndSelect)
-                self.plugin_view.setCurrentIndex(idx)
         else:
             error_dialog(self, _('No valid plugin path'),
                          _('%s is not a valid plugin path')%path).exec_()

src/calibre/gui2/preferences/plugins.ui

@@ -24,6 +24,47 @@
      </property>
     </widget>
    </item>
+   <item>
+    <layout class="QHBoxLayout" name="horizontalLayout">
+     <item>
+      <widget class="SearchBox2" name="search"/>
+     </item>
+     <item>
+      <widget class="QPushButton" name="next_button">
+       <property name="sizePolicy">
+        <sizepolicy hsizetype="Maximum" vsizetype="Fixed">
+         <horstretch>0</horstretch>
+         <verstretch>0</verstretch>
+        </sizepolicy>
+       </property>
+       <property name="text">
+        <string>&amp;Next</string>
+       </property>
+       <property name="icon">
+        <iconset resource="../../../../resources/images.qrc">
+         <normaloff>:/images/arrow-down.png</normaloff>:/images/arrow-down.png</iconset>
+       </property>
+      </widget>
+     </item>
+     <item>
+      <widget class="QPushButton" name="previous_button">
+       <property name="sizePolicy">
+        <sizepolicy hsizetype="Maximum" vsizetype="Fixed">
+         <horstretch>0</horstretch>
+         <verstretch>0</verstretch>
+        </sizepolicy>
+       </property>
+       <property name="text">
+        <string>&amp;Previous</string>
+       </property>
+       <property name="icon">
+        <iconset resource="../../../../resources/images.qrc">
+         <normaloff>:/images/arrow-up.png</normaloff>:/images/arrow-up.png</iconset>
+       </property>
+      </widget>
+     </item>
+    </layout>
+   </item>
    <item>
     <widget class="QTreeView" name="plugin_view">
      <property name="alternatingRowColors">
@@ -84,6 +125,13 @@
    </item>
   </layout>
  </widget>
+ <customwidgets>
+  <customwidget>
+   <class>SearchBox2</class>
+   <extends>QComboBox</extends>
+   <header>calibre/gui2/search_box.h</header>
+  </customwidget>
+ </customwidgets>
  <resources>
   <include location="../../../../resources/images.qrc"/>
  </resources>

src/calibre/gui2/tools.py

@@ -275,7 +275,7 @@ def generate_catalog(parent, dbspec, ids, device_manager, db):
 
     if device_manager.is_device_connected:
         device = device_manager.device
-        connected_device['name'] = device.gui_name
+        connected_device['name'] = device.get_gui_name()
         try:
             storage = []
             if device._main_prefix:

src/calibre/library/check_library.py

@@ -30,8 +30,8 @@ CHECKS = [('invalid_titles',    _('Invalid titles'), True, False),
           ('missing_formats',   _('Missing book formats'), False, False),
           ('extra_formats',     _('Extra book formats'), True, False),
           ('extra_files',       _('Unknown files in books'), True, False),
-          ('missing_covers',    _('Missing covers in books'), False, True),
+          ('missing_covers',    _('Missing covers files'), False, True),
-          ('extra_covers',      _('Extra covers in books'), True, True),
+          ('extra_covers',      _('Cover files not in database'), True, True),
           ('failed_folders',    _('Folders raising exception'), False, False)
       ]
 

src/calibre/library/database2.py

@@ -1549,7 +1549,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         elif mi.cover is not None:
             if os.access(mi.cover, os.R_OK):
                 with lopen(mi.cover, 'rb') as f:
-                    doit(self.set_cover, id, f, commit=False)
+                    raw = f.read()
+                if raw:
+                    doit(self.set_cover, id, raw, commit=False)
         if mi.tags:
             doit(self.set_tags, id, mi.tags, notify=False, commit=False)
         if mi.comments:

src/calibre/library/restore.py

@@ -141,7 +141,7 @@ class Restore(Thread):
         sizes   = [os.path.getsize(os.path.join(dirpath, x)) for x in formats]
         names   = [os.path.splitext(x)[0] for x in formats]
         opf = os.path.join(dirpath, 'metadata.opf')
-        mi = OPF(opf).to_book_metadata()
+        mi = OPF(opf, basedir=dirpath).to_book_metadata()
         timestamp = os.path.getmtime(opf)
         path = os.path.relpath(dirpath, self.src_library_path).replace(os.sep,
                 '/')

src/calibre/library/server/browse.py

@@ -356,9 +356,9 @@ class BrowseServer(object):
             if category in category_icon_map:
                 icon = category_icon_map[category]
             elif meta['is_custom']:
-                icon = category_icon_map[':custom']
+                icon = category_icon_map['custom:']
             elif meta['kind'] == 'user':
-                icon = category_icon_map[':user']
+                icon = category_icon_map['user:']
             else:
                 icon = 'blank.png'
             cats.append((meta['name'], category, icon))

src/calibre/manual/faq.rst

@@ -295,7 +295,9 @@ e-ink screen :)
 Note that in the case of the Kindle, there is a way to manipulate collections via USB,
 but it requires that the Kindle be rebooted *every time* it is disconnected from the computer, for the
 changes to the collections to be recognized. As such, it is unlikely that
-any |app| developers will ever feel motivated enough to support it.
+any |app| developers will ever feel motivated enough to support it. There is however, a |app| plugin
+that allows you to create collections on your Kindle from the |app| metadata. It is available
+`here <http://www.mobileread.com/forums/showthread.php?t=118635>`_. 
 
 Library Management
 ------------------

src/calibre/utils/html2textile.py

@@ -0,0 +1,209 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (c) 2010, Webreactor - Marcin Lulek <info@webreactor.eu>
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in the
+#      documentation and/or other materials provided with the distribution.
+#    * Neither the name of the <organization> nor the
+#      names of its contributors may be used to endorse or promote products
+#      derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+from lxml import etree
+from calibre.ebooks.oeb.base import barename
+
+class EchoTarget:
+
+    def __init__(self):
+        self.final_output = []
+        self.block = False
+        self.ol_ident = 0
+        self.ul_ident = 0
+        self.list_types = []
+        self.haystack = []
+
+    def start(self, tag, attrib):
+        tag = barename(tag)
+
+        newline = '\n'
+        dot = ''
+        new_tag = ''
+
+        if tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6'):
+            new_tag = tag
+            dot = '. '
+        elif tag == 'p':
+                new_tag = ''
+                dot = ''
+        elif tag == 'blockquote':
+            new_tag = 'bq'
+            dot = '. '
+        elif tag in ('b', 'strong'):
+            new_tag = '*'
+            newline = ''
+        elif tag in ('em', 'i'):
+            new_tag = '_'
+            newline = ''
+        elif tag == 'cite':
+            new_tag = '??'
+            newline = ''
+        elif tag == 'del':
+            new_tag = '-'
+            newline = ''
+        elif tag == 'ins':
+            new_tag = '+'
+            newline = ''
+        elif tag == 'sup':
+            new_tag = '^'
+            newline = ''
+        elif tag == 'sub':
+            new_tag = '~'
+            newline = ''
+        elif tag == 'span':
+            new_tag = '%'
+            newline = ''
+        elif tag == 'a':
+            self.block = True
+            if 'title' in attrib:
+                self.a_part = {'title':attrib.get('title'),
+                               'href':attrib.get('href', '')}
+            else:
+                self.a_part = {'title':None, 'href':attrib.get('href', '')}
+            new_tag = ''
+            newline = ''
+
+        elif tag == 'img':
+            if 'alt' in attrib:
+                new_tag = ' !%s(%s)' % (attrib.get('src'), attrib.get('title'),)
+            else:
+                new_tag = ' !%s' % attrib.get('src')
+            newline = ''
+
+        elif tag in ('ul', 'ol'):
+            new_tag = ''
+            newline = ''
+            self.list_types.append(tag)
+            if tag == 'ul':
+                self.ul_ident += 1
+            else:
+                self.ol_ident += 1
+
+        elif tag == 'li':
+            indent = self.ul_ident + self.ol_ident
+            if self.list_types[-1] == 'ul':
+                new_tag = '*' * indent + ' '
+                newline = '\n'
+            else:
+                new_tag = '#' * indent + ' '
+                newline = '\n'
+
+
+        if tag not in ('ul', 'ol'):
+            textile = '%(newline)s%(tag)s%(dot)s' % \
+                                 {
+                                  'newline':newline,
+                                  'tag':new_tag,
+                                  'dot':dot
+                                  }
+            if not self.block:
+                self.final_output.append(textile)
+            else:
+                self.haystack.append(textile)
+
+    def end(self, tag):
+        tag = barename(tag)
+
+        if tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p'):
+            self.final_output.append('\n')
+        elif tag in ('b', 'strong'):
+            self.final_output.append('*')
+        elif tag in ('em', 'i'):
+            self.final_output.append('_')
+        elif tag == 'cite':
+            self.final_output.append('??')
+        elif tag == 'del':
+            self.final_output.append('-')
+        elif tag == 'ins':
+            self.final_output.append('+')
+        elif tag == 'sup':
+            self.final_output.append('^')
+        elif tag == 'sub':
+            self.final_output.append('~')
+        elif tag == 'span':
+            self.final_output.append('%')
+        elif tag == 'a':
+            if self.a_part['title']:
+                textilized = ' "%s (%s)":%s ' % (
+                                                 ''.join(self.haystack),
+                                                 self.a_part.get('title'),
+                                                 self.a_part.get('href'),
+                                                 )
+                self.haystack = []
+            else:
+                textilized = ' "%s":%s ' % (
+                                                 ''.join(self.haystack),
+                                                 self.a_part.get('href'),
+                                                 )
+                self.haystack = []
+            self.final_output.append(textilized)
+            self.block = False
+        elif tag == 'img':
+            self.final_output.append('!')
+        elif tag == 'ul':
+            self.ul_ident -= 1
+            self.list_types.pop()
+            if len(self.list_types) == 0:
+                self.final_output.append('\n')
+        elif tag == 'ol':
+            self.ol_ident -= 1
+            self.list_types.pop()
+            if len(self.list_types) == 0:
+                self.final_output.append('\n')
+
+    def data(self, data):
+        #we dont want any linebreaks inside our tags
+        node_data = data.replace('\n','')
+        if not self.block:
+            self.final_output.append(node_data)
+        else:
+            self.haystack.append(node_data)
+
+    def comment(self, text):
+        pass
+
+    def close(self):
+        return "closed!"
+
+
+def html2textile(html):
+    #1st pass
+    #clean the whitespace and convert html to xhtml
+    parser = etree.HTMLParser()
+    tree = etree.fromstring(html, parser)
+    xhtml = etree.tostring(tree, method="xml")
+    parser = etree.XMLParser(remove_blank_text=True)
+    root = etree.XML(xhtml, parser)
+    cleaned_html = etree.tostring(root)
+    #2nd pass build textile
+    target = EchoTarget()
+    parser = etree.XMLParser(target=target)
+    root = etree.fromstring(cleaned_html, parser)
+    textilized_text = ''.join(target.final_output).lstrip().rstrip()
+    return textilized_text

src/calibre/utils/localization.py

@@ -112,6 +112,16 @@ _extra_lang_codes = {
         'en_IE' : _('English (Ireland)'),
         'en_CN' : _('English (China)'),
         'es_PY' : _('Spanish (Paraguay)'),
+        'es_UY' : _('Spanish (Uruguay)'),
+        'es_AR' : _('Spanish (Argentina)'),
+        'es_MX' : _('Spanish (Mexico)'),
+        'es_CU' : _('Spanish (Cuba)'),
+        'es_CL' : _('Spanish (Chile)'),
+        'es_EC' : _('Spanish (Ecuador)'),
+        'es_HN' : _('Spanish (Honduras)'),
+        'es_VE' : _('Spanish (Venezuela)'),
+        'es_BO' : _('Spanish (Bolivia)'),
+        'es_NI' : _('Spanish (Nicaragua)'),
         'de_AT' : _('German (AT)'),
         'fr_BE' : _('French (BE)'),
         'nl'    : _('Dutch (NL)'),

src/calibre/utils/search_query_parser.py

@@ -260,12 +260,12 @@ class SearchQueryParser(object):
         '''
         Should return the set of matches for :param:'location` and :param:`query`.
 
-        The search must be performed over all entries is :param:`candidates` is
+        The search must be performed over all entries if :param:`candidates` is
         None otherwise only over the items in candidates.
 
         :param:`location` is one of the items in :member:`SearchQueryParser.DEFAULT_LOCATIONS`.
         :param:`query` is a string literal.
-        :param: None or a subset of the set returned by :meth:`universal_set`.
+        :return: None or a subset of the set returned by :meth:`universal_set`.
         '''
         return set([])