diff --git a/resources/images/document-encrypt.png b/resources/images/document-encrypt.png new file mode 100644 index 0000000000..0774342024 Binary files /dev/null and b/resources/images/document-encrypt.png differ diff --git a/resources/images/news/zerohedge.png b/resources/images/news/zerohedge.png new file mode 100644 index 0000000000..a2bc6cde14 Binary files /dev/null and b/resources/images/news/zerohedge.png differ diff --git a/resources/recipes/expansion_spanish.recipe b/resources/recipes/expansion_spanish.recipe index 31a1504eb0..f2229e90e6 100644 --- a/resources/recipes/expansion_spanish.recipe +++ b/resources/recipes/expansion_spanish.recipe @@ -1,59 +1,79 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- - __license__ = 'GPL v3' -__copyright__ = '2009, Darko Miletic ' +__author__ = 'Gerardo Diez' +__copyright__ = 'Gerardo Diez' +description = 'Main daily newspaper from Spain - v1.00 (05, Enero 2011)' +__docformat__ = 'restructuredtext en' + ''' -www.expansion.com +expansion.es ''' +from calibre.web.feeds.recipes import BasicNewsRecipe +class Publico(BasicNewsRecipe): + title =u'Expansion.com' + __author__ ='Gerardo Diez' + publisher =u'Unidad Editorial Información Económica, S.L.' + category ='finances, catalunya' + oldest_article =1 + max_articles_per_feed =100 + simultaneous_downloads =10 + cover_url =u'http://estaticos01.expansion.com/iconos/v2.x/v2.0/cabeceras/logo_expansion.png' + timefmt ='[%A, %d %B, %Y]' + encoding ='latin' + language ='es' + remove_javascript =True + no_stylesheets =True + keep_only_tags =dict(name='div', attrs={'class':['noticia primer_elemento']}) + remove_tags =[ + dict(name='div', attrs={'class':['compartir', 'metadata_desarrollo_noticia', 'relacionadas', 'mas_info','publicidad publicidad_textlink', 'ampliarfoto']}), + dict(name='ul', attrs={'class':['bolos_desarrollo_noticia']}), + dict(name='span', attrs={'class':['comentarios']}), + dict(name='p', attrs={'class':['cintillo_comentarios', 'cintillo_comentarios formulario']}), + dict(name='div', attrs={'id':['comentarios_lectores_listado']}) + ] + feeds =[ + (u'Portada', u'http://estaticos.expansion.com/rss/portada.xml'), + (u'Portada: Bolsas', u'http://estaticos.expansion.com/rss/mercados.xml'), + (u'Divisas', u'http://estaticos.expansion.com/rss/mercadosdivisas.xml'), + (u'Euribor', u'http://estaticos.expansion.com/rss/mercadoseuribor.xml'), + (u'Materias Primas', u'http://estaticos.expansion.com/rss/mercadosmateriasprimas.xml'), + (u'Renta Fija', u'http://estaticos.expansion.com/rss/mercadosrentafija.xml'), -from calibre.web.feeds.news import BasicNewsRecipe -from calibre.ebooks.BeautifulSoup import Tag + (u'Portada: Mi Dinero', u'http://estaticos.expansion.com/rss/midinero.xml'), + (u'Hipotecas', u'http://estaticos.expansion.com/rss/midinerohipotecas.xml'), + (u'Créditos', u'http://estaticos.expansion.com/rss/midinerocreditos.xml'), + (u'Pensiones', u'http://estaticos.expansion.com/rss/midineropensiones.xml'), + (u'Fondos de Inversión', u'http://estaticos.expansion.com/rss/midinerofondos.xml'), + (u'Motor', u'http://estaticos.expansion.com/rss/midineromotor.xml'), -class Expansion(BasicNewsRecipe): - title = 'Diario Expansion' - __author__ = 'Darko Miletic' - description = 'Lider de informacion de mercados, economica y politica' - publisher = 'expansion.com' - category = 'news, politics, Spain' - oldest_article = 2 - max_articles_per_feed = 100 - no_stylesheets = True - use_embedded_content = False - delay = 1 - encoding = 'iso-8859-15' - language = 'es' + (u'Portada: Empresas', u'http://estaticos.expansion.com/rss/empresas.xml'), + (u'Banca', u'http://estaticos.expansion.com/rss/empresasbanca.xml'), + (u'TMT', u'http://estaticos.expansion.com/rss/empresastmt.xml'), + (u'Energía', u'http://estaticos.expansion.com/rss/empresasenergia.xml'), + (u'Inmobiliario y Construcción', u'http://estaticos.expansion.com/rss/empresasinmobiliario.xml'), + (u'Transporte y Turismo', u'http://estaticos.expansion.com/rss/empresastransporte.xml'), + (u'Automoción e Industria', u'http://estaticos.expansion.com/rss/empresasauto-industria.xml'), + (u'Distribución', u'http://estaticos.expansion.com/rss/empresasdistribucion.xml'), + (u'Deporte y Negocio', u' http://estaticos.expansion.com/rss/empresasdeporte.xml'), + (u'Mi Negocio', u'http://estaticos.expansion.com/rss/empresasminegocio.xml'), + (u'Interiores', u'http://estaticos.expansion.com/rss/empresasinteriores.xml'), + (u'Digitech', u'http://estaticos.expansion.com/rss/empresasdigitech.xml'), - direction = 'ltr' + (u'Portada: Economía y Política', u'http://estaticos.expansion.com/rss/economiapolitica.xml'), + (u'Política', u'http://estaticos.expansion.com/rss/economia.xml'), + (u'Portada: Sociedad', u'http://estaticos.expansion.com/rss/entorno.xml'), - html2lrf_options = [ - '--comment' , description - , '--category' , category - , '--publisher', publisher - ] + (u'Portada: Opinión', u'http://estaticos.expansion.com/rss/opinion.xml'), + (u'Llaves y editoriales', u'http://estaticos.expansion.com/rss/opinioneditorialyllaves.xml'), + (u'Tribunas', u'http://estaticos.expansion.com/rss/opiniontribunas.xml'), - html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' + (u'Portada: Jurídico', u'http://estaticos.expansion.com/rss/juridico.xml'), + (u'Entrevistas', u'http://estaticos.expansion.com/rss/juridicoentrevistas.xml'), + (u'Opinión', u'http://estaticos.expansion.com/rss/juridicoopinion.xml'), + (u'Sentencias', u'http://estaticos.expansion.com/rss/juridicosentencias.xml'), - feeds = [ - (u'Ultimas noticias', u'http://rss.expansion.com/rss/descarga.htm?data2=178') - ,(u'Temas del dia' , u'http://rss.expansion.com/rss/descarga.htm?data2=178') - ] - - - keep_only_tags = [dict(name='div', attrs={'id':'principal'})] - - remove_tags = [ - dict(name=['object','link','script']) - ,dict(name='div', attrs={'class':['utilidades','tit_relacionadas']}) - ] - - remove_tags_after = [dict(name='div', attrs={'class':'tit_relacionadas'})] - - def preprocess_html(self, soup): - soup.html['dir' ] = self.direction - mcharset = Tag(soup,'meta',[("http-equiv","Content-Type"),("content","text/html; charset=utf-8")]) - soup.head.insert(0,mcharset) - for item in soup.findAll(style=True): - del item['style'] - return soup + (u'Mujer', u'http://estaticos.expansion.com/rss/mujer-empresa.xml'), + (u'Cataluña', u'http://estaticos.expansion.com/rss/catalunya.xml'), + (u'Función pública', u'http://estaticos.expansion.com/rss/funcion-publica.xml') + ] diff --git a/resources/recipes/globe_and_mail.recipe b/resources/recipes/globe_and_mail.recipe index 4cc76688c1..22cb6fa5bb 100644 --- a/resources/recipes/globe_and_mail.recipe +++ b/resources/recipes/globe_and_mail.recipe @@ -8,12 +8,13 @@ __docformat__ = 'restructuredtext en' globeandmail.com ''' +import re + from calibre.web.feeds.news import BasicNewsRecipe class AdvancedUserRecipe1287083651(BasicNewsRecipe): title = u'Globe & Mail' - __license__ = 'GPL v3' - __author__ = 'Szing' + __author__ = 'Kovid Goyal' oldest_article = 2 no_stylesheets = True max_articles_per_feed = 100 @@ -38,24 +39,19 @@ class AdvancedUserRecipe1287083651(BasicNewsRecipe): (u'Sports', u'http://www.theglobeandmail.com/auto/?service=rss') ] - keep_only_tags = [ - dict(name='h1'), - dict(name='h2', attrs={'id':'articletitle'}), - dict(name='p', attrs={'class':['leadText', 'meta', 'leadImage', 'redtext byline', 'bodyText']}), - dict(name='div', attrs={'class':['news','articlemeta','articlecopy']}), - dict(name='id', attrs={'class':'article'}), - dict(name='table', attrs={'class':'todays-market'}), - dict(name='header', attrs={'id':'leadheader'}) - ] + preprocess_regexps = [ + (re.compile(r'', re.DOTALL), lambda m: ''), + (re.compile(r'', re.DOTALL), lambda m: ''), + ] + remove_tags_before = dict(name='h1') remove_tags = [ - dict(name='div', attrs={'id':['tabInside', 'ShareArticles', 'topStories']}) - ] - - #this has to be here or the text in the article appears twice. - remove_tags_after = [dict(id='article')] + dict(name='div', attrs={'id':['ShareArticles', 'topStories']}), + dict(href=lambda x: x and 'tracking=' in x), + {'class':['articleTools', 'pagination', 'Ads', 'topad', + 'breadcrumbs', 'footerNav', 'footerUtil', 'downloadlinks']}] #Use the mobile version rather than the web version def print_version(self, url): - return url + '&service=mobile' + return url.rpartition('?')[0] + '?service=mobile' diff --git a/resources/recipes/msnbc.recipe b/resources/recipes/msnbc.recipe index 6e2fc50aaa..6e58585341 100644 --- a/resources/recipes/msnbc.recipe +++ b/resources/recipes/msnbc.recipe @@ -1,10 +1,9 @@ __license__ = 'GPL v3' -__copyright__ = '2010, Darko Miletic ' +__copyright__ = '2010-2011, Darko Miletic ' ''' msnbc.msn.com ''' -import re from calibre.web.feeds.recipes import BasicNewsRecipe class MsNBC(BasicNewsRecipe): @@ -19,7 +18,16 @@ class MsNBC(BasicNewsRecipe): publisher = 'msnbc.com' category = 'news, USA, world' language = 'en' - extra_css = ' body{ font-family: sans-serif } .head{font-family: serif; font-size: xx-large; font-weight: bold; color: #CC0000} .abstract{font-weight: bold} .source{font-size: small} .updateTime{font-size: small} ' + extra_css = """ + body{ font-family: Georgia,Times,serif } + .hide{display: none} + .caption{font-family: Arial,sans-serif; font-size: x-small} + .entry-summary{font-family: Arial,sans-serif} + .copyright{font-size: 0.95em; font-style: italic} + .source-org{font-size: small; font-family: Arial,sans-serif} + img{display: block; margin-bottom: 0.5em} + span.byline{display: none} + """ conversion_options = { 'comments' : description @@ -28,14 +36,20 @@ class MsNBC(BasicNewsRecipe): ,'publisher': publisher } - preprocess_regexps = [ - (re.compile(r'', re.DOTALL|re.IGNORECASE),lambda match: '') - ,(re.compile(r'
', re.DOTALL|re.IGNORECASE),lambda match: '
'), - ] + remove_tags_before = dict(name='h1', attrs={'id':'headline'}) + remove_tags_after = dict(name='span', attrs={'class':['copyright','Linear copyright']}) + keep_only_tags=[ + dict(attrs={'id':['headline','deck','byline','source','intelliTXT']}) + ,dict(attrs={'class':['gl_headline','articleText','drawer-content Linear','v-center3','byline','textBodyBlack']}) + ] + remove_attributes=['property','lang','rel','xmlns:fb','xmlns:v','xmlns:dc','xmlns:dcmitype','xmlns:og','xmlns:media','xmlns:vcard','typeof','itemscope','itemtype','itemprop','about','type','size','width','height','onreadystatechange','data','border','hspace','vspace'] + + remove_tags = [ + dict(name=['iframe','object','link','embed','meta','table']) + ,dict(name='span', attrs={'class':['copyright','Linear copyright']}) + ,dict(name='div', attrs={'class':'social'}) + ] - remove_tags_before = dict(name='div', attrs={'class':'head'}) - remove_tags_after = dict(name='div', attrs={'class':'copyright'}) - remove_tags = [dict(name=['iframe','object','link','script','form'])] feeds = [ (u'US News' , u'http://rss.msnbc.msn.com/id/3032524/device/rss/rss.xml' ) @@ -48,11 +62,26 @@ class MsNBC(BasicNewsRecipe): ,(u'Tech & Science', u'http://rss.msnbc.msn.com/id/3032117/device/rss/rss.xml' ) ] - def print_version(self, url): - return url + 'print/1/displaymode/1098/' - def preprocess_html(self, soup): - for item in soup.head.findAll('div'): - item.extract() + for item in soup.body.findAll('html'): + item.name='div' + for item in soup.body.findAll('div'): + if item.has_key('id') and item['id'].startswith('vine-'): + item.extract() + if item.has_key('class') and ( item['class'].startswith('ad') or item['class'].startswith('vine')): + item.extract() + for item in soup.body.findAll('img'): + if not item.has_key('alt'): + item['alt'] = 'image' + for item in soup.body.findAll('ol'): + if item.has_key('class') and item['class'].startswith('grid'): + item.extract() + for item in soup.body.findAll('span'): + if ( item.has_key('id') and item['id'].startswith('byLine') and item.string is None) or ( item.has_key('class') and item['class'].startswith('inline') ): + item.extract() + for alink in soup.findAll('a'): + if alink.string is not None: + tstr = alink.string + alink.replaceWith(tstr) return soup diff --git a/resources/recipes/nytimes.recipe b/resources/recipes/nytimes.recipe index eaa428e731..6f80f4f85f 100644 --- a/resources/recipes/nytimes.recipe +++ b/resources/recipes/nytimes.recipe @@ -685,3 +685,28 @@ class NYTimes(BasicNewsRecipe): divTag.replaceWith(tag) return soup + + def populate_article_metadata(self, article, soup, first): + shortparagraph = "" + try: + if len(article.text_summary.strip()) == 0: + articlebodies = soup.findAll('div',attrs={'class':'articleBody'}) + if articlebodies: + for articlebody in articlebodies: + if articlebody: + paras = articlebody.findAll('p') + for p in paras: + refparagraph = self.massageNCXText(self.tag_to_string(p,use_alt=False)).strip() + #account for blank paragraphs and short paragraphs by appending them to longer ones + if len(refparagraph) > 0: + if len(refparagraph) > 70: #approximately one line of text + article.summary = article.text_summary = shortparagraph + refparagraph + return + else: + shortparagraph = refparagraph + " " + if shortparagraph.strip().find(" ") == -1 and not shortparagraph.strip().endswith(":"): + shortparagraph = shortparagraph + "- " + except: + self.log("Error creating article descriptions") + return + diff --git a/resources/recipes/nytimes_sub.recipe b/resources/recipes/nytimes_sub.recipe index e56fd9cdec..8ac7c735f7 100644 --- a/resources/recipes/nytimes_sub.recipe +++ b/resources/recipes/nytimes_sub.recipe @@ -685,4 +685,27 @@ class NYTimes(BasicNewsRecipe): divTag.replaceWith(tag) return soup + def populate_article_metadata(self, article, soup, first): + shortparagraph = "" + try: + if len(article.text_summary.strip()) == 0: + articlebodies = soup.findAll('div',attrs={'class':'articleBody'}) + if articlebodies: + for articlebody in articlebodies: + if articlebody: + paras = articlebody.findAll('p') + for p in paras: + refparagraph = self.massageNCXText(self.tag_to_string(p,use_alt=False)).strip() + #account for blank paragraphs and short paragraphs by appending them to longer ones + if len(refparagraph) > 0: + if len(refparagraph) > 70: #approximately one line of text + article.summary = article.text_summary = shortparagraph + refparagraph + return + else: + shortparagraph = refparagraph + " " + if shortparagraph.strip().find(" ") == -1 and not shortparagraph.strip().endswith(":"): + shortparagraph = shortparagraph + "- " + except: + self.log("Error creating article descriptions") + return diff --git a/resources/recipes/technology_review.recipe b/resources/recipes/technology_review.recipe index cc8f13733e..e7cc6700d7 100644 --- a/resources/recipes/technology_review.recipe +++ b/resources/recipes/technology_review.recipe @@ -35,7 +35,6 @@ class TechnologyReview(BasicNewsRecipe): def get_article_url(self, article): return article.get('guid', article.get('id', None)) - def print_version(self, url): baseurl='http://www.technologyreview.com/printer_friendly_article.aspx?id=' split1 = string.split(url,"/") @@ -43,3 +42,25 @@ class TechnologyReview(BasicNewsRecipe): split2= string.split(xxx,"/") s = baseurl + split2[0] return s + + + def postprocess_html(self,soup, True): + #remove picture + headerhtml = soup.find(True, {'class':'header'}) + headerhtml.replaceWith("") + + #remove close button + closehtml = soup.find(True, {'class':'close'}) + closehtml.replaceWith("") + + #remove banner advertisement + bannerhtml = soup.find(True, {'class':'bannerad'}) + bannerhtml.replaceWith("") + + #thanks kiklop74! This code removes all links from the text + for alink in soup.findAll('a'): + if alink.string is not None: + tstr = alink.string + alink.replaceWith(tstr) + + return soup diff --git a/resources/recipes/tyzden.recipe b/resources/recipes/tyzden.recipe index c206244ff6..b8d7389fbe 100644 --- a/resources/recipes/tyzden.recipe +++ b/resources/recipes/tyzden.recipe @@ -28,7 +28,7 @@ class TyzdenRecipe(BasicNewsRecipe): if (weeknum > 1): weeknum -= 1 - title = u'.tyzden ' + str(weeknum) + '/' + str(year) + title = u'tyzden' base_url_path = 'http://www.tyzden.sk/casopis/' + str(year) + '/' + str(weeknum) base_url = base_url_path + '.html' diff --git a/resources/recipes/wired_daily.recipe b/resources/recipes/wired_daily.recipe index f06d28796e..df59c7c826 100644 --- a/resources/recipes/wired_daily.recipe +++ b/resources/recipes/wired_daily.recipe @@ -2,8 +2,10 @@ __license__ = 'GPL v3' __docformat__ = 'restructuredtext en' +import re from calibre.web.feeds.news import BasicNewsRecipe +from calibre.ebooks.chardet import xml_to_unicode class Wired_Daily(BasicNewsRecipe): @@ -15,30 +17,43 @@ class Wired_Daily(BasicNewsRecipe): no_stylesheets = True + preprocess_regexps = [(re.compile(r'', re.DOTALL), lambda m: + '')] + remove_tags_before = dict(name='div', id='content') - remove_tags = [dict(id=['social_tools', 'outerWrapper', 'sidebar', - 'footer', 'advertisement', 'blog_subscription_unit', - 'brightcove_component']), - {'class':'entryActions'}, - dict(name=['noscript', 'script'])] + remove_tags = [dict(id=['header', 'commenting_module', 'post_nav', + 'social_tools', 'sidebar', 'footer', 'social_wishlist', 'pgwidget', + 'outerWrapper', 'inf_widget']), + {'class':['entryActions', 'advertisement', 'entryTags']}, + dict(name=['noscript', 'script']), + dict(name='h4', attrs={'class':re.compile(r'rat\d+')}), + {'class':lambda x: x and x.startswith('contentjump')}, + dict(name='li', attrs={'class':['entryCategories', 'entryEdit']})] + feeds = [ ('Top News', 'http://feeds.wired.com/wired/index'), - ('Culture', 'http://feeds.wired.com/wired/culture'), - ('Software', 'http://feeds.wired.com/wired/software'), - ('Mac', 'http://feeds.feedburner.com/cultofmac/bFow'), - ('Gadgets', 'http://feeds.wired.com/wired/gadgets'), - ('Cars', 'http://feeds.wired.com/wired/cars'), - ('Entertainment', 'http://feeds.wired.com/wired/entertainment'), - ('Gaming', 'http://feeds.wired.com/wired/gaming'), - ('Science', 'http://feeds.wired.com/wired/science'), - ('Med Tech', 'http://feeds.wired.com/wired/medtech'), - ('Politics', 'http://feeds.wired.com/wired/politics'), - ('Tech Biz', 'http://feeds.wired.com/wired/techbiz'), - ('Commentary', 'http://feeds.wired.com/wired/commentary'), + ('Product Reviews', + 'http://www.wired.com/reviews/feeds/latestProductsRss'), + ('Autopia', 'http://www.wired.com/autopia/feed/'), + ('Danger Room', 'http://www.wired.com/dangerroom/feed/'), + ('Epicenter', 'http://www.wired.com/epicenter/feed/'), + ('Gadget Lab', 'http://www.wired.com/gadgetlab/feed/'), + ('Geek Dad', 'http://www.wired.com/geekdad/feed/'), + ('Playbook', 'http://www.wired.com/playbook/feed/'), + ('Rawfile', 'http://www.wired.com/rawfile/feed/'), + ('This Day in Tech', 'http://www.wired.com/thisdayintech/feed/'), + ('Threat Level', 'http://www.wired.com/threatlevel/feed/'), + ('Underwire', 'http://www.wired.com/underwire/feed/'), + ('Web Monkey', 'http://www.webmonkey.com/feed/'), + ('Science', 'http://www.wired.com/wiredscience/feed/'), ] + def populate_article_metadata(self, article, soup, first): + if article.text_summary: + article.text_summary = xml_to_unicode(article.text_summary, + resolve_entities=True)[0] + def print_version(self, url): - return url.replace('http://www.wired.com/', 'http://www.wired.com/print/') - + return url + '/all/1' diff --git a/resources/recipes/zerohedge.recipe b/resources/recipes/zerohedge.recipe new file mode 100644 index 0000000000..09f62e5b52 --- /dev/null +++ b/resources/recipes/zerohedge.recipe @@ -0,0 +1,33 @@ +__license__ = 'GPL v3' +__copyright__ = '2011, Darko Miletic ' +''' +www.zerohedge.com +''' + +from calibre.web.feeds.recipes import BasicNewsRecipe + +class ZeroHedge(BasicNewsRecipe): + title = 'Zero Hedge' + __author__ = 'Darko Miletic' + description = 'On a long enough timeline the survival rate for everyone drops to zero' + oldest_article = 10 + max_articles_per_feed = 100 + no_stylesheets = True + use_embedded_content = True + encoding = 'utf8' + publisher = 'zero hedge' + category = 'news, USA, world, economy, politics' + language = 'en' + masthead_url = 'http://www.zerohedge.com/themes/newsflash/logo.png' + publication_type = 'blog' + extra_css = 'body{ font-family: sans-serif }' + + conversion_options = { + 'comments' : description + ,'tags' : category + ,'language' : language + ,'publisher': publisher + } + + + feeds = [(u'Articles', u'http://feeds.feedburner.com/zerohedge/feed')] diff --git a/resources/templates/rtf.xsl b/resources/templates/rtf.xsl index ea1fc71172..6db1c0388d 100644 --- a/resources/templates/rtf.xsl +++ b/resources/templates/rtf.xsl @@ -287,7 +287,7 @@ ] - + @@ -297,7 +297,7 @@ - + diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index 2585b5d081..a4f7439405 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -459,6 +459,18 @@ def force_unicode(obj, enc=preferred_encoding): obj = obj.decode('utf-8') return obj +def as_unicode(obj, enc=preferred_encoding): + if not isbytestring(obj): + try: + obj = unicode(obj) + except: + try: + obj = str(obj) + except: + obj = repr(obj) + return force_unicode(obj, enc=enc) + + def human_readable(size): """ Convert a size in bytes into a human readable form """ diff --git a/src/calibre/devices/nook/driver.py b/src/calibre/devices/nook/driver.py index 987b90c748..ca05885645 100644 --- a/src/calibre/devices/nook/driver.py +++ b/src/calibre/devices/nook/driver.py @@ -91,3 +91,19 @@ class NOOK_COLOR(NOOK): EBOOK_DIR_MAIN = 'My Files/Books' + ''' + def create_upload_path(self, path, mdata, fname, create_dirs=True): + filepath = NOOK.create_upload_path(self, path, mdata, fname, + create_dirs=create_dirs) + edm = self.EBOOK_DIR_MAIN.replace('/', os.sep) + npath = os.path.join(edm, _('News')) + os.sep + if npath in filepath: + filepath = filepath.replace(npath, os.sep.join('My Files', + 'Magazines')+os.sep) + filedir = os.path.dirname(filepath) + if create_dirs and not os.path.exists(filedir): + os.makedirs(filedir) + + return filepath + ''' + diff --git a/src/calibre/ebooks/conversion/plumber.py b/src/calibre/ebooks/conversion/plumber.py index b1d760ea2d..9b22fb46ec 100644 --- a/src/calibre/ebooks/conversion/plumber.py +++ b/src/calibre/ebooks/conversion/plumber.py @@ -88,6 +88,7 @@ class Plumber(object): self.ui_reporter = report_progress self.abort_after_input_dump = abort_after_input_dump + # Pipeline options {{{ # Initialize the conversion options that are independent of input and # output formats. The input and output plugins can still disable these # options via recommendations. @@ -527,6 +528,7 @@ OptionRecommendation(name='timestamp', help=_('Set the book timestamp (used by the date column in calibre).')), ] + # }}} input_fmt = os.path.splitext(self.input)[1] if not input_fmt: diff --git a/src/calibre/ebooks/fb2/fb2ml.py b/src/calibre/ebooks/fb2/fb2ml.py index 4dd6e7c7ae..796a94533a 100644 --- a/src/calibre/ebooks/fb2/fb2ml.py +++ b/src/calibre/ebooks/fb2/fb2ml.py @@ -16,7 +16,6 @@ import uuid from lxml import etree -from calibre import guess_type from calibre import prepare_string_for_xml from calibre.constants import __appname__, __version__ from calibre.ebooks.oeb.base import XHTML, XHTML_NS, barename, namespace @@ -41,7 +40,7 @@ class FB2MLizer(object): # in different directories. FB2 images are all in a flat layout so we rename all images # into a sequential numbering system to ensure there are no collisions between image names. self.image_hrefs = {} - # Mapping of toc items and their + # Mapping of toc items and their self.toc = {} # Used to see whether a new
needs to be opened self.section_level = 0 @@ -51,7 +50,7 @@ class FB2MLizer(object): self.oeb_book = oeb_book self.opts = opts self.reset_state() - + # Used for adding
s and s to allow readers # to generate toc from the document. if self.opts.sectionize == 'toc': @@ -75,20 +74,20 @@ class FB2MLizer(object): text = re.sub(r'(?miu)<p>\s*</p>', '', text) text = re.sub(r'(?miu)\s*</p>', '</p>', text) text = re.sub(r'(?miu)</p>\s*<p>', '</p>\n\n<p>', text) - + text = re.sub(r'(?miu)<title>\s*', '', text) text = re.sub(r'(?miu)\s+', '', text) - + text = re.sub(r'(?miu)
\s*
', '', text) text = re.sub(r'(?miu)\s*
', '\n
', text) text = re.sub(r'(?miu)\s*', '\n\n', text) text = re.sub(r'(?miu)\s*
', '\n
', text) text = re.sub(r'(?miu)
\s*', '
\n', text) text = re.sub(r'(?miu)
', '
\n\n
', text) - + if self.opts.insert_blank_line: text = re.sub(r'(?miu)

', '

', text) - + return text def fb2_header(self): @@ -102,6 +101,7 @@ class FB2MLizer(object): metadata['date'] = '%i.%i.%i' % (datetime.now().day, datetime.now().month, datetime.now().year) metadata['lang'] = u''.join(self.oeb_book.metadata.lang) if self.oeb_book.metadata.lang else 'en' metadata['id'] = None + metadata['cover'] = self.get_cover() author_parts = self.oeb_book.metadata.creator[0].value.split(' ') if len(author_parts) == 1: @@ -121,10 +121,11 @@ class FB2MLizer(object): break if metadata['id'] is None: self.log.warn('No UUID identifier found') - metadata['id'] = str(uuid.uuid4()) + metadata['id'] = str(uuid.uuid4()) for key, value in metadata.items(): - metadata[key] = prepare_string_for_xml(value) + if not key == 'cover': + metadata[key] = prepare_string_for_xml(value) return u'' \ '' \ @@ -136,6 +137,7 @@ class FB2MLizer(object): '%(author_last)s' \ '' \ '%(title)s' \ + '%(cover)s' \ '%(lang)s' \ '' \ '' \ @@ -154,48 +156,64 @@ class FB2MLizer(object): def fb2_footer(self): return u'' + def get_cover(self): + cover_href = None + + # Get the raster cover if it's available. + if self.oeb_book.metadata.cover and unicode(self.oeb_book.metadata.cover[0]) in self.oeb_book.manifest.ids: + id = unicode(self.oeb_book.metadata.cover[0]) + cover_item = self.oeb_book.manifest.ids[id] + if cover_item.media_type in OEB_RASTER_IMAGES: + cover_href = cover_item.href + else: + # Figure out if we have a title page or a cover page + page_name = '' + if 'titlepage' in self.oeb_book.guide: + page_name = 'titlepage' + elif 'cover' in self.oeb_book.guide: + page_name = 'cover' + + if page_name: + cover_item = self.oeb_book.manifest.hrefs[self.oeb_book.guide[page_name].href] + # Get the first image in the page + for img in cover_item.xpath('//img'): + cover_href = cover_item.abshref(img.get('src')) + break + + if cover_href: + # Only write the image tag if it is in the manifest. + if cover_href in self.oeb_book.manifest.hrefs.keys(): + if cover_href not in self.image_hrefs.keys(): + self.image_hrefs[cover_href] = '_%s.jpg' % len(self.image_hrefs.keys()) + return u'' % self.image_hrefs[cover_href] + + return u'' + def get_text(self): text = [''] - + # Create main section if there are no others to create if self.opts.sectionize == 'nothing': text.append('
') self.section_level += 1 - - # Insert the title page / cover into the spine if it is not already referenced. - title_name = u'' - if 'titlepage' in self.oeb_book.guide: - title_name = 'titlepage' - elif 'cover' in self.oeb_book.guide: - title_name = 'cover' - if title_name: - title_item = self.oeb_book.manifest.hrefs[self.oeb_book.guide[title_name].href] - if title_item.spine_position is None and title_item.media_type == 'application/xhtml+xml': - self.oeb_book.spine.insert(0, title_item, True) - # Create xhtml page to reference cover image so it can be used. - if not title_name and self.oeb_book.metadata.cover and unicode(self.oeb_book.metadata.cover[0]) in self.oeb_book.manifest.ids: - id = unicode(self.oeb_book.metadata.cover[0]) - cover_item = self.oeb_book.manifest.ids[id] - if cover_item.media_type in OEB_RASTER_IMAGES: - self.insert_image_cover(cover_item.href) - + for item in self.oeb_book.spine: self.log.debug('Converting %s to FictionBook2 XML' % item.href) stylizer = Stylizer(item.data, item.href, self.oeb_book, self.opts, self.opts.output_profile) - + # Start a
if we must sectionize each file or if the TOC references this page page_section_open = False if self.opts.sectionize == 'files' or self.toc.get(item.href) == 'page': text.append('
') page_section_open = True self.section_level += 1 - + text += self.dump_text(item.data.find(XHTML('body')), stylizer, item) - + if page_section_open: text.append('
') self.section_level -= 1 - + # Close any open sections while self.section_level > 0: text.append('
') @@ -203,17 +221,6 @@ class FB2MLizer(object): return ''.join(text) + '' - def insert_image_cover(self, image_href): - from calibre.ebooks.oeb.base import RECOVER_PARSER - try: - root = etree.fromstring(u'' % (XHTML_NS, image_href), parser=RECOVER_PARSER) - except: - root = etree.fromstring(u'', parser=RECOVER_PARSER) - - id, href = self.oeb_book.manifest.generate('fb2_cover', 'fb2_cover.xhtml') - item = self.oeb_book.manifest.add(id, href, guess_type(href)[0], data=root) - self.oeb_book.spine.insert(0, item, True) - def fb2mlize_images(self): ''' This function uses the self.image_hrefs dictionary mapping. It is populated by the dump_text function. @@ -345,7 +352,7 @@ class FB2MLizer(object): self.toc[page.href] = None elif toc_entry and elem_tree.attrib.get('id', None): newlevel = toc_entry.get(elem_tree.attrib.get('id', None), None) - + # Start a new section if necessary if newlevel: if not (newlevel > self.section_level): diff --git a/src/calibre/ebooks/metadata/rtf.py b/src/calibre/ebooks/metadata/rtf.py index ad41125575..c20d880a2f 100644 --- a/src/calibre/ebooks/metadata/rtf.py +++ b/src/calibre/ebooks/metadata/rtf.py @@ -10,7 +10,8 @@ from calibre.ebooks.metadata import MetaInformation, string_to_authors title_pat = re.compile(r'\{\\info.*?\{\\title(.*?)(? 6: - md += '}' + md.append(r'{\subject %s}'%(comment,)) + if options.publisher: + publisher = options.publisher.encode('ascii', 'ignore') + md.append(r'{\manager %s}'%(publisher,)) + if options.tags: + tags = u', '.join(options.tags) + tags = tags.encode('ascii', 'ignore') + md.append(r'{\category %s}'%(tags,)) + if len(md) > 1: + md.append('}') stream.seek(0) src = stream.read() - ans = src[:6] + md + src[6:] + ans = src[:6] + u''.join(md) + src[6:] stream.seek(0) stream.write(ans) @@ -156,7 +169,7 @@ def set_metadata(stream, options): base_pat = r'\{\\name(.*?)(? 0: - if raw[pos] == '}': bc -= 1 - elif raw[pos] == '{': bc += 1 - pos += 1 - pict = raw[start:pos+1] - enc = re.sub(r'[^a-zA-Z0-9]', '', pict) + for enc in encs: if len(enc) % 2 == 1: enc = enc[:-1] data = enc.decode('hex') + fmt = imghdr.what(None, data) + if fmt is None: + fmt = 'wmf' count += 1 - name = (('%4d'%count).replace(' ', '0'))+'.wmf' - open(name, 'wb').write(data) + name = '%04d.%s' % (count, fmt) + with open(name, 'wb') as f: + f.write(data) imap[count] = name #open(name+'.hex', 'wb').write(enc) return self.convert_images(imap) def convert_images(self, imap): - for count, val in imap.items(): + self.default_img = None + for count, val in imap.iteritems(): try: imap[count] = self.convert_image(val) except: @@ -159,11 +169,34 @@ class RTFInput(InputFormatPlugin): return imap def convert_image(self, name): - from calibre.utils.magick import Image - img = Image() - img.open(name) + if not name.endswith('.wmf'): + return name + try: + return self.rasterize_wmf(name) + except: + self.log.exception('Failed to convert WMF image %r'%name) + return self.replace_wmf(name) + + def replace_wmf(self, name): + from calibre.ebooks import calibre_cover + if self.default_img is None: + self.default_img = calibre_cover('Conversion of WMF images is not supported', + 'Use Microsoft Word or OpenOffice to save this RTF file' + ' as HTML and convert that in calibre.', title_size=36, + author_size=20) name = name.replace('.wmf', '.jpg') - img.save(name) + with open(name, 'wb') as f: + f.write(self.default_img) + return name + + def rasterize_wmf(self, name): + from calibre.utils.wmf.parse import wmf_unwrap + with open(name, 'rb') as f: + data = f.read() + data = wmf_unwrap(data) + name = name.replace('.wmf', '.png') + with open(name, 'wb') as f: + f.write(data) return name @@ -192,27 +225,27 @@ class RTFInput(InputFormatPlugin): css += '\n'+'\n'.join(font_size_classes) css += '\n' +'\n'.join(color_classes) - for cls, val in border_styles.items(): + for cls, val in border_styles.iteritems(): css += '\n\n.%s {\n%s\n}'%(cls, val) with open('styles.css', 'ab') as f: f.write(css) - def preprocess(self, fname): - self.log('\tPreprocessing to convert unicode characters') - try: - data = open(fname, 'rb').read() - from calibre.ebooks.rtf.preprocess import RtfTokenizer, RtfTokenParser - tokenizer = RtfTokenizer(data) - tokens = RtfTokenParser(tokenizer.tokens) - data = tokens.toRTF() - fname = 'preprocessed.rtf' - with open(fname, 'wb') as f: - f.write(data) - except: - self.log.exception( - 'Failed to preprocess RTF to convert unicode sequences, ignoring...') - return fname + # def preprocess(self, fname): + # self.log('\tPreprocessing to convert unicode characters') + # try: + # data = open(fname, 'rb').read() + # from calibre.ebooks.rtf.preprocess import RtfTokenizer, RtfTokenParser + # tokenizer = RtfTokenizer(data) + # tokens = RtfTokenParser(tokenizer.tokens) + # data = tokens.toRTF() + # fname = 'preprocessed.rtf' + # with open(fname, 'wb') as f: + # f.write(data) + # except: + # self.log.exception( + # 'Failed to preprocess RTF to convert unicode sequences, ignoring...') + # return fname def convert_borders(self, doc): border_styles = [] @@ -249,17 +282,14 @@ class RTFInput(InputFormatPlugin): self.log = log self.log('Converting RTF to XML...') #Name of the preprocesssed RTF file - fname = self.preprocess(stream.name) + # fname = self.preprocess(stream.name) try: - xml = self.generate_xml(fname) + xml = self.generate_xml(stream.name) except RtfInvalidCodeException, e: + raise raise ValueError(_('This RTF file has a feature calibre does not ' 'support. Convert it to HTML first and then try it.\n%s')%e) - '''dataxml = open('dataxml.xml', 'w') - dataxml.write(xml) - dataxml.close''' - d = glob.glob(os.path.join('*_rtf_pict_dir', 'picts.rtf')) if d: imap = {} diff --git a/src/calibre/ebooks/rtf2xml/ParseRtf.py b/src/calibre/ebooks/rtf2xml/ParseRtf.py index 7b89407f79..cdd9a3d088 100755 --- a/src/calibre/ebooks/rtf2xml/ParseRtf.py +++ b/src/calibre/ebooks/rtf2xml/ParseRtf.py @@ -17,7 +17,8 @@ ######################################################################### # $Revision: 1.41 $ # $Date: 2006/03/24 23:50:07 $ -import sys,os +import sys, os + from calibre.ebooks.rtf2xml import headings_to_sections, \ line_endings, footnote, fields_small, default_encoding, \ make_lists, preamble_div, header, colors, group_borders, \ @@ -90,7 +91,6 @@ class ParseRtf: out_file = '', out_dir = None, dtd = '', - #debug = 0, #why? calibre deb_dir = None, convert_symbol = None, convert_wingdings = None, @@ -107,6 +107,7 @@ class ParseRtf: no_dtd = 0, char_data = '', ): + """ Requires: 'file' --file to parse @@ -119,12 +120,11 @@ class ParseRtf: script tries to output to directory where is script is exectued.) 'deb_dir' --debug directory. If a debug_dir is provided, the script will copy each run through as a file to examine in the debug_dir - 'perl_script'--use perl to make tokens. This runs just a bit faster. - (I will probably phase this out.) 'check_brackets' -- make sure the brackets match up after each run through a file. Only for debugging. Returns: Nothing """ + self.__file = in_file self.__out_file = out_file self.__out_dir = out_dir @@ -132,7 +132,7 @@ class ParseRtf: self.__dtd_path = dtd self.__check_file(in_file,"file_to_parse") self.__char_data = char_data - self.__debug_dir = deb_dir #self.__debug_dir = debug calibre + self.__debug_dir = deb_dir self.__check_dir(self.__temp_dir) self.__copy = self.__check_dir(self.__debug_dir) self.__convert_caps = convert_caps @@ -155,25 +155,24 @@ class ParseRtf: if hasattr(the_file, 'read'): return if the_file == None: if type == "file_to_parse": - message = "You must provide a file for the script to work" - msg = message + msg = "\nYou must provide a file for the script to work" raise RtfInvalidCodeException, msg elif os.path.exists(the_file): pass # do nothing else: - message = "The file '%s' cannot be found" % the_file - msg = message + msg = "\nThe file '%s' cannot be found" % the_file raise RtfInvalidCodeException, msg + def __check_dir(self, the_dir): """Check to see if directory exists""" if not the_dir : return dir_exists = os.path.isdir(the_dir) if not dir_exists: - message = "%s is not a directory" % the_dir - msg = message + msg = "\n%s is not a directory" % the_dir raise RtfInvalidCodeException, msg return 1 + def parse_rtf(self): """ Parse the file by calling on other classes. @@ -194,13 +193,14 @@ class ParseRtf: copy_obj.set_dir(self.__debug_dir) copy_obj.remove_files() copy_obj.copy_file(self.__temp_file, "original_file") - # new as of 2005-08-02. Do I want this? + # Function to check if bracket are well handled if self.__debug_dir or self.__run_level > 2: self.__check_brack_obj = check_brackets.CheckBrackets\ (file = self.__temp_file, bug_handler = RtfInvalidCodeException, ) - # convert Macintosh line endings to Unix line endings + #convert Macintosh and Windows line endings to Unix line endings + #why do this if you don't wb after? line_obj = line_endings.FixLineEndings( in_file = self.__temp_file, bug_handler = RtfInvalidCodeException, @@ -208,13 +208,13 @@ class ParseRtf: run_level = self.__run_level, replace_illegals = self.__replace_illegals, ) - return_value = line_obj.fix_endings() + return_value = line_obj.fix_endings() #calibre return what? self.__return_code(return_value) tokenize_obj = tokenize.Tokenize( bug_handler = RtfInvalidCodeException, in_file = self.__temp_file, copy = self.__copy, - run_level = self.__run_level,) + run_level = self.__run_level) tokenize_obj.tokenize() process_tokens_obj = process_tokens.ProcessTokens( in_file = self.__temp_file, @@ -230,12 +230,25 @@ class ParseRtf: os.remove(self.__temp_file) except OSError: pass + #Check to see if the file is correctly encoded + encode_obj = default_encoding.DefaultEncoding( + in_file = self.__temp_file, + run_level = self.__run_level, + bug_handler = RtfInvalidCodeException, + check_raw = True, + ) + platform, code_page, default_font_num = encode_obj.find_default_encoding() check_encoding_obj = check_encoding.CheckEncoding( - bug_handler = RtfInvalidCodeException, - ) - check_encoding_obj.check_encoding(self.__file) - sys.stderr.write('File "%s" does not appear to be RTF.\n' % self.__file if isinstance(self.__file, str) else self.__file.encode('utf-8')) - raise InvalidRtfException, msg + bug_handler = RtfInvalidCodeException, + ) + enc = encode_obj.get_codepage() + if enc != 'mac_roman': + enc = 'cp' + enc + if check_encoding_obj.check_encoding(self.__file, enc): + file_name = self.__file if isinstance(self.__file, str) \ + else self.__file.encode('utf-8') + msg = 'File %s does not appear to be correctly encoded.\n' % file_name + raise InvalidRtfException, msg delete_info_obj = delete_info.DeleteInfo( in_file = self.__temp_file, copy = self.__copy, @@ -508,6 +521,7 @@ class ParseRtf: indent = self.__indent, run_level = self.__run_level, no_dtd = self.__no_dtd, + encoding = encode_obj.get_codepage(), bug_handler = RtfInvalidCodeException, ) tags_obj.convert_to_tags() @@ -520,35 +534,28 @@ class ParseRtf: output_obj.output() os.remove(self.__temp_file) return self.__exit_level + def __bracket_match(self, file_name): if self.__run_level > 2: good_br, msg = self.__check_brack_obj.check_brackets() if good_br: pass - # sys.stderr.write( msg + ' in ' + file_name + "\n") + #sys.stderr.write( msg + ' in ' + file_name + "\n") else: - msg += msg + " in file '" + file_name + "'\n" + msg = '%s in file %s\n' % (msg, file_name) raise RtfInvalidCodeException, msg + def __return_code(self, num): - if num == None: - return - if int(num) > self.__exit_level: - self.__exit_level = num + if num == None: + return + if int(num) > self.__exit_level: + self.__exit_level = num + def __make_temp_file(self,file): """Make a temporary file to parse""" write_file="rtf_write_file" read_obj = file if hasattr(file, 'read') else open(file,'r') - write_obj = open(write_file, 'w') - line = "dummy" - while line: - line = read_obj.read(1000) - write_obj.write(line ) - write_obj.close() + with open(write_file, 'wb') as write_obj: + for line in read_obj: + write_obj.write(line) return write_file - """ -mi1\n -mi33\n -mi' % info) + def __empty_func(self, line): """ Print out empty tag and newlines when needed. @@ -85,10 +96,11 @@ class ConvertToTags: self.__write_new_line() if info in self.__two_new_line: self.__write_extra_new_line() + def __open_att_func(self, line): """ Process lines for open tags that have attributes. - The important infor is between [17:-1]. Take this info and split it + The important info is between [17:-1]. Take this info and split it with the delimeter '<'. The first token in this group is the element name. The rest are attributes, separated fromt their values by '>'. So read each token one at a time, and split them by '>'. @@ -119,6 +131,7 @@ class ConvertToTags: self.__write_new_line() if element_name in self.__two_new_line: self.__write_extra_new_line() + def __empty_att_func(self, line): """ Same as the __open_att_func, except a '/' is placed at the end of the tag. @@ -143,6 +156,7 @@ class ConvertToTags: self.__write_new_line() if element_name in self.__two_new_line: self.__write_extra_new_line() + def __close_func(self, line): """ Print out the closed tag and new lines, if appropriate. @@ -156,6 +170,7 @@ class ConvertToTags: self.__write_new_line() if info in self.__two_new_line: self.__write_extra_new_line() + def __text_func(self, line): """ Simply print out the information between [17:-1] @@ -163,6 +178,7 @@ class ConvertToTags: #tx') + #keep maximum compatibility with previous version + check_encoding_obj = check_encoding.CheckEncoding( + bug_handler=self.__bug_handler) + + if not check_encoding_obj.check_encoding(self.__file, verbose=False): + self.__write_obj.write('') + elif not check_encoding_obj.check_encoding(self.__file, self.__encoding): + self.__write_obj.write('' % self.__encoding) + else: + self.__write_obj.write('') + sys.stderr.write('Bad RTF encoding, revert to US-ASCII chars and' + ' hope for the best') self.__new_line = 0 self.__write_new_line() if self.__no_dtd: @@ -207,6 +237,7 @@ class ConvertToTags: ) self.__new_line = 0 self.__write_new_line() + def convert_to_tags(self): """ Read in the file one line at a time. Get the important info, between @@ -222,18 +253,14 @@ class ConvertToTags: an empty tag function. """ self.__initiate_values() - read_obj = open(self.__file, 'r') self.__write_obj = open(self.__write_to, 'w') self.__write_dec() - line_to_read = 1 - while line_to_read: - line_to_read = read_obj.readline() - line = line_to_read - self.__token_info = line[:16] - action = self.__state_dict.get(self.__token_info) - if action != None: - action(line) - read_obj.close() + with open(self.__file, 'r') as read_obj: + for line in read_obj: + self.__token_info = line[:16] + action = self.__state_dict.get(self.__token_info) + if action is not None: + action(line) self.__write_obj.close() copy_obj = copy.Copy(bug_handler = self.__bug_handler) if self.__copy: diff --git a/src/calibre/ebooks/rtf2xml/copy.py b/src/calibre/ebooks/rtf2xml/copy.py index ff029c1841..1b620b9fbf 100755 --- a/src/calibre/ebooks/rtf2xml/copy.py +++ b/src/calibre/ebooks/rtf2xml/copy.py @@ -23,6 +23,7 @@ class Copy: def __init__(self, bug_handler, file = None, deb_dir = None, ): self.__file = file self.__bug_handler = bug_handler + def set_dir(self, deb_dir): """Set the temporary directory to write files to""" if deb_dir is None: @@ -33,19 +34,11 @@ class Copy: message = "%(deb_dir)s is not a directory" % vars() raise self.__bug_handler , message Copy.__dir = deb_dir + def remove_files(self ): """Remove files from directory""" self.__remove_the_files(Copy.__dir) - """ - list_of_files = os.listdir(Copy.__dir) - list_of_files = os.listdir(the_dir) - for file in list_of_files: - rem_file = os.path.join(Copy.__dir,file) - if os.path.isdir(rem_file): - self.remove_files(rem_file) - else: - os.remove(rem_file) - """ + def __remove_the_files(self, the_dir): """Remove files from directory""" list_of_files = os.listdir(the_dir) @@ -58,6 +51,7 @@ class Copy: os.remove(rem_file) except OSError: pass + def copy_file(self, file, new_file): """ Copy the file to a new name diff --git a/src/calibre/ebooks/rtf2xml/default_encoding.py b/src/calibre/ebooks/rtf2xml/default_encoding.py index b932b465d0..53887e0d90 100755 --- a/src/calibre/ebooks/rtf2xml/default_encoding.py +++ b/src/calibre/ebooks/rtf2xml/default_encoding.py @@ -1,61 +1,142 @@ ######################################################################### # # -# # # copyright 2002 Paul Henry Tremblay # # # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # -# General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with this program; if not, write to the Free Software # -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA # -# 02111-1307 USA # -# # -# # ######################################################################### + +''' +Codepages as to RTF 1.9.1: + 437 United States IBM + 708 Arabic (ASMO 708) + 709 Arabic (ASMO 449+, BCON V4) + 710 Arabic (transparent Arabic) + 711 Arabic (Nafitha Enhanced) + 720 Arabic (transparent ASMO) + 819 Windows 3.1 (United States and Western Europe) + 850 IBM multilingual + 852 Eastern European + 860 Portuguese + 862 Hebrew + 863 French Canadian + 864 Arabic + 865 Norwegian + 866 Soviet Union + 874 Thai + 932 Japanese + 936 Simplified Chinese + 949 Korean + 950 Traditional Chinese + 1250 Eastern European + 1251 Cyrillic + 1252 Western European + 1253 Greek + 1254 Turkish + 1255 Hebrew + 1256 Arabic + 1257 Baltic + 1258 Vietnamese + 1361 Johab + 10000 MAC Roman + 10001 MAC Japan + 10004 MAC Arabic + 10005 MAC Hebrew + 10006 MAC Greek + 10007 MAC Cyrillic + 10029 MAC Latin2 + 10081 MAC Turkish + 57002 Devanagari + 57003 Bengali + 57004 Tamil + 57005 Telugu + 57006 Assamese + 57007 Oriya + 57008 Kannada + 57009 Malayalam + 57010 Gujarati + 57011 Punjabi +''' +import re + class DefaultEncoding: """ Find the default encoding for the doc """ - def __init__(self, in_file, bug_handler, run_level = 1,): - """ - Required: - 'file' - Returns: - nothing - """ + def __init__(self, in_file, bug_handler, run_level = 1, check_raw = False): self.__file = in_file self.__bug_handler = bug_handler + self.__platform = 'Windows' + self.__default_num = 'not-defined' + self.__code_page = '1252' + self.__datafetched = False + self.__fetchraw = check_raw + def find_default_encoding(self): - platform = 'Windows' - default_num = 'not-defined' - code_page = 'ansicpg1252' - read_obj = open(self.__file, 'r') - line_to_read = 1 - while line_to_read: - line_to_read = read_obj.readline() - line = line_to_read - self.__token_info = line[:16] - if self.__token_info == 'mi 3: msg = 'flag problem\n' raise self.__bug_handler, msg - return 1 + return True elif self.__token_info in self.__allowable : if self.__ob: self.__write_obj.write(self.__ob) @@ -132,85 +138,81 @@ class DeleteInfo: self.__state = 'default' else: pass - return 1 + return True elif self.__token_info == 'cw 5: - msg = 'After an asterisk, and found neither an allowable or non-allowble token\n' - msg += 'token is "%s"\n' % self.__token_info - raise self.__bug_handler + msg = ('After an asterisk, and found neither an allowable or non-allowable token\n\ + token is "%s"\n') % self.__token_info + raise self.__bug_handler, msg if not self.__ob: - self.__write_cb = 1 + self.__write_cb = True self.__ob = 0 self.__state = 'delete' self.__cb_count = 0 - return 0 + return False + def __found_list_func(self, line): """ print out control words in this group """ self.__state = 'list' + def __list_func(self, line): """ Check to see if the group has ended. - Return 1 for all control words. - Return 0 otherwise. + Return True for all control words. + Return False otherwise. """ if self.__delete_count == self.__cb_count and self.__token_info ==\ 'cb%s\n' % self.__footnote_count) self.__first_line = 0 + def __in_footnote_func(self, line): """Handle all tokens that are part of footnote""" if self.__first_line: @@ -68,6 +72,7 @@ class Footnote: 'mi ci - 'annotation' : 'annotation', + 'annotation' : 'annotation', 'blue______' : 'blue', 'bold______' : 'bold', - 'caps______' : 'caps', - 'char-style' : 'character-style', - 'dbl-strike' : 'double-strike-through', + 'caps______' : 'caps', + 'char-style' : 'character-style', + 'dbl-strike' : 'double-strike-through', 'emboss____' : 'emboss', 'engrave___' : 'engrave', 'font-color' : 'font-color', @@ -96,7 +97,7 @@ class Inline: 'font-size_' : 'font-size', 'font-style' : 'font-style', 'font-up___' : 'superscript', - 'footnot-mk' : 'footnote-marker', + 'footnot-mk' : 'footnote-marker', 'green_____' : 'green', 'hidden____' : 'hidden', 'italics___' : 'italics', @@ -107,9 +108,10 @@ class Inline: 'strike-thr' : 'strike-through', 'subscript_' : 'subscript', 'superscrip' : 'superscript', - 'underlined' : 'underlined', + 'underlined' : 'underlined', } self.__caps_list = ['false'] + def __set_list_func(self, line): """ Requires: @@ -128,6 +130,7 @@ class Inline: self.__place = 'in_list' self.__inline_list = self.__list_inline_list self.__groups_in_waiting = self.__groups_in_waiting_list + def __default_func(self, line): """ Requires: @@ -140,8 +143,8 @@ class Inline: action = self.__default_dict.get(self.__token_info) if action: action(line) - if self.__token_info != 'cw%s' % (the_key, the_dict[the_key])) self.__write_obj.write('\n') self.__groups_in_waiting[0] = 0 + def __end_para_func(self, line): """ Requires: @@ -342,6 +346,7 @@ class Inline: self.__write_obj.write('mi%s' % (the_key, the_dict[the_key])) self.__write_obj.write('\n') self.__groups_in_waiting[0] = 0 + def __found_field_func(self, line): """ Just a default function to make sure I don't prematurely exit default state """ pass + def form_tags(self): """ Requires: @@ -386,32 +393,27 @@ class Inline: the state. """ self.__initiate_values() - read_obj = open(self.__file, 'r') - self.__write_obj = open(self.__write_to, 'w') - line_to_read = 1 - while line_to_read: - line_to_read = read_obj.readline() - line = line_to_read - token = line[0:-1] - self.__token_info = '' - if token == 'tx 1: - sys.stderr.write('Removing files from old pict directory...\n') - all_files = os.listdir(self.__dir_name) - for the_file in all_files: - the_file = os.path.join(self.__dir_name, the_file) - try: - os.remove(the_file) - except OSError: - pass - if self.__run_level > 1: - sys.stderr.write('Files removed.\n') + if self.__run_level > 1: + sys.stderr.write('Removing files from old pict directory...\n') + all_files = os.listdir(self.__dir_name) + for the_file in all_files: + the_file = os.path.join(self.__dir_name, the_file) + try: + os.remove(the_file) + except OSError: + pass + if self.__run_level > 1: + sys.stderr.write('Files removed.\n') def __create_pict_file(self): """Create a file for all the pict data to be written to. """ self.__pict_file = os.path.join(self.__dir_name, 'picts.rtf') - write_pic_obj = open(self.__pict_file, 'w') - write_pic_obj.close() self.__write_pic_obj = open(self.__pict_file, 'a') def __in_pict_func(self, line): if self.__cb_count == self.__pict_br_count: - self.__in_pict = 0 + self.__in_pict = False self.__write_pic_obj.write("}\n") - return 1 + return True else: action = self.__pict_dict.get(self.__token_info) if action: - line = action(line) - self.__write_pic_obj.write(line) - return 0 + self.__write_pic_obj.write(action(line)) + return False def __default(self, line, write_obj): """Determine if each token marks the beginning of pict data. @@ -142,53 +128,50 @@ class Pict: write_obj.write('mi ml '*' : ('ml', 'asterisk__', self.default_func), ':' : ('ml', 'colon_____', self.default_func), @@ -73,7 +78,6 @@ class ProcessTokens: 'backslash' : ('nu', '\\', self.text_func), 'ob' : ('nu', '{', self.text_func), 'cb' : ('nu', '}', self.text_func), - 'line' : ('nu', 'hard-lineb', self.default_func), #calibre #'line' : ('nu', ' ', self.text_func), calibre # paragraph formatting => pf 'page' : ('pf', 'page-break', self.default_func), @@ -159,15 +163,17 @@ class ProcessTokens: 'rtf' : ('ri', 'rtf_______', self.default_func), 'deff' : ('ri', 'deflt-font', self.default_func), 'mac' : ('ri', 'macintosh_', self.default_func), + 'pc' : ('ri', 'pc________', self.default_func), + 'pca' : ('ri', 'pca_______', self.default_func), 'ansi' : ('ri', 'ansi______', self.default_func), 'ansicpg' : ('ri', 'ansi-codpg', self.default_func), # notes => nt 'footnote' : ('nt', 'footnote__', self.default_func), 'ftnalt' : ('nt', 'type______ an - 'tc' : ('an', 'toc_______', self.default_func), + 'tc' : ('an', 'toc_______', self.default_func), 'bkmkstt' : ('an', 'book-mk-st', self.default_func), - 'bkmkstart' : ('an', 'book-mk-st', self.default_func), + 'bkmkstart' : ('an', 'book-mk-st', self.default_func), 'bkmkend' : ('an', 'book-mk-en', self.default_func), 'xe' : ('an', 'index-mark', self.default_func), 'rxe' : ('an', 'place_____', self.default_func), @@ -347,7 +353,7 @@ class ProcessTokens: 10: 'Kanji numbering without the digit character', 11: 'Kanji numbering with the digit character', 1246: 'phonetic Katakana characters in aiueo order', - 1346: 'phonetic katakana characters in iroha order', + 1346: 'phonetic katakana characters in iroha order', 14: 'double byte character', 15: 'single byte character', 16: 'Kanji numbering 3', @@ -392,7 +398,7 @@ class ProcessTokens: 5121 : 'Arabic Algeria', 15361 : 'Arabic Bahrain', 3073 : 'Arabic Egypt', - 1 : 'Arabic General', + 1 : 'Arabic General', 2049 : 'Arabic Iraq', 11265 : 'Arabic Jordan', 13313 : 'Arabic Kuwait', @@ -417,7 +423,7 @@ class ProcessTokens: 1059 : 'Byelorussian', 1027 : 'Catalan', 2052 : 'Chinese China', - 4 : 'Chinese General', + 4 : 'Chinese General', 3076 : 'Chinese Hong Kong', 4100 : 'Chinese Singapore', 1028 : 'Chinese Taiwan', @@ -431,7 +437,7 @@ class ProcessTokens: 2057 : 'English British', 4105 : 'English Canada', 9225 : 'English Caribbean', - 9 : 'English General', + 9 : 'English General', 6153 : 'English Ireland', 8201 : 'English Jamaica', 5129 : 'English New Zealand', @@ -595,30 +601,37 @@ class ProcessTokens: num = num[1:] # chop off leading 0, which I added num = num.upper() # the mappings store hex in caps return 'tx 3: - msg = 'number "%s" cannot be converted to integer\n' % num + msg = 'Number "%s" cannot be converted to integer\n' % num raise self.__bug_handler, msg type = self.__number_type_dict.get(num) - if type == None: + if type is None: if self.__run_level > 3: msg = 'No type for "%s" in self.__number_type_dict\n' raise self.__bug_handler type = 'Arabic' return 'cw<%s<%snum<%s\n' % (token, num) + def divide_by_2(self, pre, token, num): num = self.divide_num(num, 2) return 'cw<%s<%s%s<%s\n' % (token, num, token) + def divide_by_20(self, pre, token, num): num = self.divide_num(num, 20) return 'cw<%s<%s%s<%s\n' % (token, num, token) + def text_func(self, pre, token, num=None): return 'tx%s<%s\n' % (third_field, token, num, token) + def bool_st_func(self, pre, token, num): if num is None or num == '' or num == '1': return 'cw<%s<%sfalse<%s\n' % (token, token) else: - msg = 'boolean should have some value module process tokens\n' - msg += 'token is ' + token + "\n" - msg += "'" + num + "'" + "\n" + msg = "boolean should have some value module process tokens\ntoken is %s\n'%s'\n" % (token, num) raise self.__bug_handler, msg + def __no_sup_sub_func(self, pre, token, num): the_string = 'cw 3: - msg = 'no number to process?\n' - msg += 'this indicates that the token ' - msg += ' \(\\li\) should have a number and does not\n' - msg += 'numerator is "%s"\n' % numerator - msg += 'denominator is "%s"\n' % denominator + msg = ('No number to process?\nthis indicates that the token \(\\li\) \ + should have a number and does not\nnumerator is \ + "%s"\ndenominator is "%s"\n') % (numerator, denominator) raise self.__bug_handler, msg if 5 > self.__return_code: self.__return_code = 5 @@ -698,9 +716,10 @@ class ProcessTokens: if string_num[-2:] == ".0": string_num = string_num[:-2] return string_num + def split_let_num(self, token): match_obj = re.search(self.__num_exp,token) - if match_obj != None: + if match_obj is not None: first = match_obj.group(1) second = match_obj.group(2) if not second: @@ -714,6 +733,7 @@ class ProcessTokens: raise self.__bug_handler return token, 0 return first, second + def convert_to_hex(self,number): """Convert a string to uppercase hexidecimal""" num = int(number) @@ -722,6 +742,7 @@ class ProcessTokens: return hex_num except: raise self.__bug_handler + def process_cw(self, token): """Change the value of the control word by determining what dictionary it belongs to""" @@ -737,89 +758,62 @@ class ProcessTokens: pre, token, action = self.dict_token.get(token, (None, None, None)) if action: return action(pre, token, num) - # unused function - def initiate_token_actions(self): - self.action_for_token={ - '{' : self.ob_func, - '}' : self.cb_func, - '\\' : self.process_cw, - } - # unused function - def evaluate_token(self,token): - """Evaluate tokens. Return a value if the token is not a - control word. Otherwise, pass token onto another method - for further evaluation.""" - token, action = self.dict_token.get(token[0:1]) - if action: - line = action(token) - return line - else : - return 'tx -1: - msg ='Invalid RTF: token "\\ " not valid. \n' - raise self.__exception_handler, msg - elif token[0:1] == "\\": - line = self.process_cw(token) - if line != None: - write_obj.write(line) - else: - fields = re.split(self.__utf_exp, token) - for field in fields: - if not field: - continue - if field[0:1] == '&': - write_obj.write('tx -1: + msg = 'Invalid RTF: token "\\ " not valid.\n' + raise self.__exception_handler, msg + elif token[:1] == "\\": + try: + token.decode('us-ascii') + except UnicodeError, msg: + msg = 'Invalid RTF: Tokens not ascii encoded.\n%s' % str(msg) + raise self.__exception_handler, msg + line = self.process_cw(token) + if line is not None: + write_obj.write(line) else: - write_obj.write('tx", ">") - line = line.replace("\\~", "\\~ ") - line = line.replace("\\_", "\\_ ") - line = line.replace("\\:", "\\: ") - line = line.replace("\\-", "\\- ") - # turn into a generic token to eliminate special - # cases and make processing easier - line = line.replace("\\{", "\\ob ") - # turn into a generic token to eliminate special - # cases and make processing easier - line = line.replace("\\}", "\\cb ") - # put a backslash in front of to eliminate special cases and - # make processing easier - line = line.replace("{", "\\{") - # put a backslash in front of to eliminate special cases and - # make processing easier - line = line.replace("}", "\\}") - line = re.sub(self.__utf_exp, self.__from_ms_to_utf8, line) - # line = re.sub( self.__neg_utf_exp, self.__neg_unicode_func, line) - line = re.sub(self.__ms_hex_exp, "\\mshex0\g<1> ", line) - ##line = line.replace("\\backslash", "\\\\") - # this is for older RTF - line = re.sub(self.__par_exp, '\\par ', line) - return line - def __compile_expressions(self): - self.__ms_hex_exp = re.compile(r"\\\'(..)") - self.__utf_exp = re.compile(r"\\u(-?\d{3,6}) {0,1}") - self.__splitexp = re.compile(r"(\\[\\{}]|{|}|\\[^\s\\{}&]+(?:\s)?)") - self.__par_exp = re.compile(r'\\$') - self.__mixed_exp = re.compile(r"(\\[a-zA-Z]+\d+)(\D+)") - ##self.num_exp = re.compile(r"(\*|:|[a-zA-Z]+)(.*)") - def __create_tokens(self): self.__compile_expressions() - read_obj = open(self.__file, 'r') - write_obj = open(self.__write_to, 'w') - line_to_read = "dummy" - while line_to_read: - line_to_read = read_obj.readline() - line = line_to_read - line = line.replace("\n", "") - line = self.__sub_line_reg(line) - tokens = re.split(self.__splitexp, line) - ##print tokens - for token in tokens: - if token != "": - write_obj.write(token + "\n") - """ - match_obj = re.search(self.__mixed_exp, token) - if match_obj != None: - first = match_obj.group(1) - second = match_obj.group(2) - write_obj.write(first + "\n") - write_obj.write(second + "\n") - else: - write_obj.write(token + "\n") - """ - read_obj.close() - write_obj.close() + #variables + self.__uc_char = 0 + self.__uc_bin = False + self.__uc_value = [1] + + def __reini_utf8_counters(self): + self.__uc_char = 0 + self.__uc_bin = False + + def __remove_uc_chars(self, startchar, token): + for i in xrange(startchar, len(token)): + if token[i] == " ": + continue + elif self.__uc_char: + self.__uc_char -= 1 + else: + return token[i:] + #if only " " and char to skip + return '' + + def __unicode_process(self, token): + #change scope in + if token == '\{': + self.__uc_value.append(self.__uc_value[-1]) + #basic error handling + self.__reini_utf8_counters() + return token + #change scope out + elif token == '\}': + self.__uc_value.pop() + self.__reini_utf8_counters() + return token + #add a uc control + elif token[:3] == '\uc': + self.__uc_value[-1] = int(token[3:]) + self.__reini_utf8_counters() + return token + #bin data to slip + elif self.__uc_bin: + self.__uc_bin = False + return '' + #uc char to remove + elif self.__uc_char: + #handle \bin tag in case of uc char to skip + if token[:4] == '\bin': + self.__uc_char -=1 + self.__uc_bin = True + return '' + elif token[:1] == "\\" : + self.__uc_char -=1 + return '' + else: + return self.__remove_uc_chars(0, token) + #go for real \u token + match_obj = self.__utf_exp.match(token) + if match_obj is not None: + self.__reini_utf8_counters() + #get value and handle negative case + uni_char = int(match_obj.group(1)) + uni_len = len(match_obj.group(1)) + 2 + if uni_char < 0: + uni_char += 65536 + uni_char = unichr(uni_char).encode('ascii', 'xmlcharrefreplace') + self.__uc_char = self.__uc_value[-1] + #there is only an unicode char + if len(token)<= uni_len: + return uni_char + #an unicode char and something else + #must be after as it is splited on \ + #necessary? maybe for \bin? + elif not self.__uc_char: + return uni_char + token[uni_len:] + #if not uc0 and chars + else: + return uni_char + self.__remove_uc_chars(uni_len, token) + #default + return token + + def __sub_reg_split(self,input_file): + input_file = self.__replace_spchar.mreplace(input_file) + input_file = self.__ms_hex_exp.sub("\\mshex0\g<1> ", input_file) + input_file = self.__utf_ud.sub("\\{\\uc0 \g<1>\\}", input_file) + #remove \n in bin data + input_file = self.__bin_exp.sub(lambda x: \ + x.group().replace('\n', '') + '\n', input_file) + #split + tokens = re.split(self.__splitexp, input_file) + #remove empty tokens and \n + return filter(lambda x: len(x) > 0 and x != '\n', tokens) + #input_file = re.sub(self.__utf_exp, self.__from_ms_to_utf8, input_file) + # line = re.sub( self.__neg_utf_exp, self.__neg_unicode_func, line) + # this is for older RTF + #line = re.sub(self.__par_exp, '\\par ', line) + #return filter(lambda x: len(x) > 0, \ + #(self.__remove_line.sub('', x) for x in tokens)) + + def __compile_expressions(self): + SIMPLE_RPL = { + "\\\\": "\\backslash ", + "\\~": "\\~ ", + "\\;": "\\; ", + "&": "&", + "<": "<", + ">": ">", + "\\~": "\\~ ", + "\\_": "\\_ ", + "\\:": "\\: ", + "\\-": "\\- ", + # turn into a generic token to eliminate special + # cases and make processing easier + "\\{": "\\ob ", + # turn into a generic token to eliminate special + # cases and make processing easier + "\\}": "\\cb ", + # put a backslash in front of to eliminate special cases and + # make processing easier + "{": "\\{", + # put a backslash in front of to eliminate special cases and + # make processing easier + "}": "\\}", + # this is for older RTF + r'\\$': '\\par ', + } + self.__replace_spchar = MReplace(SIMPLE_RPL) + #add ;? in case of char following \u + self.__ms_hex_exp = re.compile(r"\\\'([0-9a-fA-F]{2})") #r"\\\'(..)" + self.__utf_exp = re.compile(r"\\u(-?\d{3,6}) ?") + self.__bin_exp = re.compile(r"(?:\\bin(-?\d{0,10})[\n ]+)[01\n]+") + #manage upr/ud situations + self.__utf_ud = re.compile(r"\\{[\n ]?\\upr[\n ]?(?:\\{.*?\\})[\n ]?" + \ + r"\\{[\n ]?\\*[\n ]?\\ud[\n ]?(\\{.*?\\})[\n ]?\\}[\n ]?\\}") + #add \n in split for whole file reading + #why keep backslash whereas \is replaced before? + #remove \n from endline char + self.__splitexp = re.compile(r"(\\[{}]|\n|\\[^\s\\{}&]+(?:[ \t\r\f\v])?)") + #self.__bin_exp = re.compile(r"\\bin(-?\d{1,8}) {0,1}") + #self.__utf_exp = re.compile(r"^\\u(-?\d{3,6})") + #self.__splitexp = re.compile(r"(\\[\\{}]|{|}|\n|\\[^\s\\{}&]+(?:\s)?)") + #self.__par_exp = re.compile(r'\\$') + #self.__remove_line = re.compile(r'\n+') + #self.__mixed_exp = re.compile(r"(\\[a-zA-Z]+\d+)(\D+)") + ##self.num_exp = re.compile(r"(\*|:|[a-zA-Z]+)(.*)") + def tokenize(self): - """Main class for handling other methods. Reads in one line \ - at a time, usues method self.sub_line to make basic substitutions,\ - uses ? to process tokens""" - self.__create_tokens() + """Main class for handling other methods. Reads the file \ + , uses method self.sub_reg to make basic substitutions,\ + and process tokens by itself""" + #read + with open(self.__file, 'r') as read_obj: + input_file = read_obj.read() + + #process simple replacements and split giving us a correct list + #remove '' and \n in the process + tokens = self.__sub_reg_split(input_file) + #correct unicode + tokens = map(self.__unicode_process, tokens) + #remove empty items created by removing \uc + tokens = filter(lambda x: len(x) > 0, tokens) + + #write + with open(self.__write_to, 'wb') as write_obj: + write_obj.write('\n'.join(tokens)) + #Move and copy copy_obj = copy.Copy(bug_handler = self.__bug_handler) if self.__copy: copy_obj.copy_file(self.__write_to, "tokenize.data") copy_obj.rename(self.__write_to, self.__file) os.remove(self.__write_to) + + #self.__special_tokens = [ '_', '~', "'", '{', '}' ] \ No newline at end of file diff --git a/src/calibre/ebooks/textile/__init__.py b/src/calibre/ebooks/textile/__init__.py new file mode 100644 index 0000000000..74141ed884 --- /dev/null +++ b/src/calibre/ebooks/textile/__init__.py @@ -0,0 +1,6 @@ +from functions import textile, textile_restricted, Textile + +if False: + textile, textile_restricted, Textile + +__all__ = ['textile', 'textile_restricted'] diff --git a/src/calibre/ebooks/textile/functions.py b/src/calibre/ebooks/textile/functions.py new file mode 100644 index 0000000000..eca4bcecff --- /dev/null +++ b/src/calibre/ebooks/textile/functions.py @@ -0,0 +1,981 @@ +#!/usr/bin/env python +""" +PyTextile + +A Humane Web Text Generator +""" + +__version__ = '2.1.4' + +__date__ = '2009/12/04' + +__copyright__ = """ +Copyright (c) 2009, Jason Samsa, http://jsamsa.com/ +Copyright (c) 2004, Roberto A. F. De Almeida, http://dealmeida.net/ +Copyright (c) 2003, Mark Pilgrim, http://diveintomark.org/ + +Original PHP Version: +Copyright (c) 2003-2004, Dean Allen +All rights reserved. + +Thanks to Carlo Zottmann for refactoring +Textile's procedural code into a class framework + +Additions and fixes Copyright (c) 2006 Alex Shiels http://thresholdstate.com/ + +""" + +__license__ = """ +L I C E N S E +============= +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name Textile nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +""" + +import re +import uuid +from urlparse import urlparse + +def _normalize_newlines(string): + out = re.sub(r'\r\n', '\n', string) + out = re.sub(r'\n{3,}', '\n\n', out) + out = re.sub(r'\n\s*\n', '\n\n', out) + out = re.sub(r'"$', '" ', out) + return out + +def getimagesize(url): + """ + Attempts to determine an image's width and height, and returns a string + suitable for use in an tag, or None in case of failure. + Requires that PIL is installed. + + >>> getimagesize("http://www.google.com/intl/en_ALL/images/logo.gif") + ... #doctest: +ELLIPSIS, +SKIP + 'width="..." height="..."' + + """ + + try: + import ImageFile + import urllib2 + except ImportError: + return None + + try: + p = ImageFile.Parser() + f = urllib2.urlopen(url) + while True: + s = f.read(1024) + if not s: + break + p.feed(s) + if p.image: + return 'width="%i" height="%i"' % p.image.size + except (IOError, ValueError): + return None + +class Textile(object): + hlgn = r'(?:\<(?!>)|(?|\<\>|\=|[()]+(?! ))' + vlgn = r'[\-^~]' + clas = r'(?:\([^)]+\))' + lnge = r'(?:\[[^\]]+\])' + styl = r'(?:\{[^}]+\})' + cspn = r'(?:\\\d+)' + rspn = r'(?:\/\d+)' + a = r'(?:%s|%s)*' % (hlgn, vlgn) + s = r'(?:%s|%s)*' % (cspn, rspn) + c = r'(?:%s)*' % '|'.join([clas, styl, lnge, hlgn]) + + pnct = r'[-!"#$%&()*+,/:;<=>?@\'\[\\\]\.^_`{|}~]' + # urlch = r'[\w"$\-_.+!*\'(),";/?:@=&%#{}|\\^~\[\]`]' + urlch = '[\w"$\-_.+*\'(),";\/?:@=&%#{}|\\^~\[\]`]' + + url_schemes = ('http', 'https', 'ftp', 'mailto') + + btag = ('bq', 'bc', 'notextile', 'pre', 'h[1-6]', 'fn\d+', 'p') + btag_lite = ('bq', 'bc', 'p') + + glyph_defaults = ( + ('txt_quote_single_open', '‘'), + ('txt_quote_single_close', '’'), + ('txt_quote_double_open', '“'), + ('txt_quote_double_close', '”'), + ('txt_apostrophe', '’'), + ('txt_prime', '′'), + ('txt_prime_double', '″'), + ('txt_ellipsis', '…'), + ('txt_emdash', '—'), + ('txt_endash', '–'), + ('txt_dimension', '×'), + ('txt_trademark', '™'), + ('txt_registered', '®'), + ('txt_copyright', '©'), + ) + + def __init__(self, restricted=False, lite=False, noimage=False): + """docstring for __init__""" + self.restricted = restricted + self.lite = lite + self.noimage = noimage + self.get_sizes = False + self.fn = {} + self.urlrefs = {} + self.shelf = {} + self.rel = '' + self.html_type = 'xhtml' + + def textile(self, text, rel=None, head_offset=0, html_type='xhtml'): + """ + >>> import textile + >>> textile.textile('some textile') + u'\\t

some textile

' + """ + self.html_type = html_type + + # text = unicode(text) + text = _normalize_newlines(text) + + if self.restricted: + text = self.encode_html(text, quotes=False) + + if rel: + self.rel = ' rel="%s"' % rel + + text = self.getRefs(text) + + text = self.block(text, int(head_offset)) + + text = self.retrieve(text) + + return text + + def pba(self, input, element=None): + """ + Parse block attributes. + + >>> t = Textile() + >>> t.pba(r'\3') + '' + >>> t.pba(r'\\3', element='td') + ' colspan="3"' + >>> t.pba(r'/4', element='td') + ' rowspan="4"' + >>> t.pba(r'\\3/4', element='td') + ' colspan="3" rowspan="4"' + + >>> t.vAlign('^') + 'top' + + >>> t.pba('^', element='td') + ' style="vertical-align:top;"' + + >>> t.pba('{line-height:18px}') + ' style="line-height:18px;"' + + >>> t.pba('(foo-bar)') + ' class="foo-bar"' + + >>> t.pba('(#myid)') + ' id="myid"' + + >>> t.pba('(foo-bar#myid)') + ' class="foo-bar" id="myid"' + + >>> t.pba('((((') + ' style="padding-left:4em;"' + + >>> t.pba(')))') + ' style="padding-right:3em;"' + + >>> t.pba('[fr]') + ' lang="fr"' + + """ + style = [] + aclass = '' + lang = '' + colspan = '' + rowspan = '' + id = '' + + if not input: + return '' + + matched = input + if element == 'td': + m = re.search(r'\\(\d+)', matched) + if m: + colspan = m.group(1) + + m = re.search(r'/(\d+)', matched) + if m: + rowspan = m.group(1) + + if element == 'td' or element == 'tr': + m = re.search(r'(%s)' % self.vlgn, matched) + if m: + style.append("vertical-align:%s;" % self.vAlign(m.group(1))) + + m = re.search(r'\{([^}]*)\}', matched) + if m: + style.append(m.group(1).rstrip(';') + ';') + matched = matched.replace(m.group(0), '') + + m = re.search(r'\[([^\]]+)\]', matched, re.U) + if m: + lang = m.group(1) + matched = matched.replace(m.group(0), '') + + m = re.search(r'\(([^()]+)\)', matched, re.U) + if m: + aclass = m.group(1) + matched = matched.replace(m.group(0), '') + + m = re.search(r'([(]+)', matched) + if m: + style.append("padding-left:%sem;" % len(m.group(1))) + matched = matched.replace(m.group(0), '') + + m = re.search(r'([)]+)', matched) + if m: + style.append("padding-right:%sem;" % len(m.group(1))) + matched = matched.replace(m.group(0), '') + + m = re.search(r'(%s)' % self.hlgn, matched) + if m: + style.append("text-align:%s;" % self.hAlign(m.group(1))) + + m = re.search(r'^(.*)#(.*)$', aclass) + if m: + id = m.group(2) + aclass = m.group(1) + + if self.restricted: + if lang: + return ' lang="%s"' + else: + return '' + + result = [] + if style: + result.append(' style="%s"' % "".join(style)) + if aclass: + result.append(' class="%s"' % aclass) + if lang: + result.append(' lang="%s"' % lang) + if id: + result.append(' id="%s"' % id) + if colspan: + result.append(' colspan="%s"' % colspan) + if rowspan: + result.append(' rowspan="%s"' % rowspan) + return ''.join(result) + + def hasRawText(self, text): + """ + checks whether the text has text not already enclosed by a block tag + + >>> t = Textile() + >>> t.hasRawText('

foo bar biz baz

') + False + + >>> t.hasRawText(' why yes, yes it does') + True + + """ + r = re.compile(r'<(p|blockquote|div|form|table|ul|ol|pre|h\d)[^>]*?>.*', re.S).sub('', text.strip()).strip() + r = re.compile(r'<(hr|br)[^>]*?/>').sub('', r) + return '' != r + + def table(self, text): + r""" + >>> t = Textile() + >>> t.table('|one|two|three|\n|a|b|c|') + '\t\n\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\n\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\n\t
onetwothree
abc
\n\n' + """ + text = text + "\n\n" + pattern = re.compile(r'^(?:table(_?%(s)s%(a)s%(c)s)\. ?\n)?^(%(a)s%(c)s\.? ?\|.*\|)\n\n' % {'s':self.s, 'a':self.a, 'c':self.c}, re.S|re.M|re.U) + return pattern.sub(self.fTable, text) + + def fTable(self, match): + tatts = self.pba(match.group(1), 'table') + rows = [] + for row in [ x for x in match.group(2).split('\n') if x]: + rmtch = re.search(r'^(%s%s\. )(.*)' % (self.a, self.c), row.lstrip()) + if rmtch: + ratts = self.pba(rmtch.group(1), 'tr') + row = rmtch.group(2) + else: + ratts = '' + + cells = [] + for cell in row.split('|')[1:-1]: + ctyp = 'd' + if re.search(r'^_', cell): + ctyp = "h" + cmtch = re.search(r'^(_?%s%s%s\. )(.*)' % (self.s, self.a, self.c), cell) + if cmtch: + catts = self.pba(cmtch.group(1), 'td') + cell = cmtch.group(2) + else: + catts = '' + + cell = self.graf(self.span(cell)) + cells.append('\t\t\t%s' % (ctyp, catts, cell, ctyp)) + rows.append("\t\t\n%s\n\t\t" % (ratts, '\n'.join(cells))) + cells = [] + catts = None + return "\t\n%s\n\t\n\n" % (tatts, '\n'.join(rows)) + + def lists(self, text): + """ + >>> t = Textile() + >>> t.lists("* one\\n* two\\n* three") + '\\t
    \\n\\t\\t
  • one
    • \\n\\t\\t
  • two
    • \\n\\t\\t
  • three
    • \\n\\t
' + """ + pattern = re.compile(r'^([#*]+%s .*)$(?![^#*])' % self.c, re.U|re.M|re.S) + return pattern.sub(self.fList, text) + + def fList(self, match): + text = match.group(0).split("\n") + result = [] + lists = [] + for i, line in enumerate(text): + try: + nextline = text[i+1] + except IndexError: + nextline = '' + + m = re.search(r"^([#*]+)(%s%s) (.*)$" % (self.a, self.c), line, re.S) + if m: + tl, atts, content = m.groups() + nl = '' + nm = re.search(r'^([#*]+)\s.*', nextline) + if nm: + nl = nm.group(1) + if tl not in lists: + lists.append(tl) + atts = self.pba(atts) + line = "\t<%sl%s>\n\t\t
  • %s" % (self.lT(tl), atts, self.graf(content)) + else: + line = "\t\t
  • " + self.graf(content) + + if len(nl) <= len(tl): + line = line + "
    • " + for k in reversed(lists): + if len(k) > len(nl): + line = line + "\n\t" % self.lT(k) + if len(k) > 1: + line = line + "" + lists.remove(k) + + result.append(line) + return "\n".join(result) + + def lT(self, input): + if re.search(r'^#+', input): + return 'o' + else: + return 'u' + + def doPBr(self, in_): + return re.compile(r'<(p)([^>]*?)>(.*)()', re.S).sub(self.doBr, in_) + + def doBr(self, match): + if self.html_type == 'html': + content = re.sub(r'(.+)(?:(?)|(?))\n(?![#*\s|])', '\\1
    ', match.group(3)) + else: + content = re.sub(r'(.+)(?:(?)|(?))\n(?![#*\s|])', '\\1
    ', match.group(3)) + return '<%s%s>%s%s' % (match.group(1), match.group(2), content, match.group(4)) + + def block(self, text, head_offset = 0): + """ + >>> t = Textile() + >>> t.block('h1. foobar baby') + '\\t

    foobar baby

    ' + """ + if not self.lite: + tre = '|'.join(self.btag) + else: + tre = '|'.join(self.btag_lite) + text = text.split('\n\n') + + tag = 'p' + atts = cite = graf = ext = c1 = '' + + out = [] + + anon = False + for line in text: + pattern = r'^(%s)(%s%s)\.(\.?)(?::(\S+))? (.*)$' % (tre, self.a, self.c) + match = re.search(pattern, line, re.S) + if match: + if ext: + out.append(out.pop() + c1) + + tag, atts, ext, cite, graf = match.groups() + h_match = re.search(r'h([1-6])', tag) + if h_match: + head_level, = h_match.groups() + tag = 'h%i' % max(1, + min(int(head_level) + head_offset, + 6)) + o1, o2, content, c2, c1 = self.fBlock(tag, atts, ext, + cite, graf) + # leave off c1 if this block is extended, + # we'll close it at the start of the next block + + if ext: + line = "%s%s%s%s" % (o1, o2, content, c2) + else: + line = "%s%s%s%s%s" % (o1, o2, content, c2, c1) + + else: + anon = True + if ext or not re.search(r'^\s', line): + o1, o2, content, c2, c1 = self.fBlock(tag, atts, ext, + cite, line) + # skip $o1/$c1 because this is part of a continuing + # extended block + if tag == 'p' and not self.hasRawText(content): + line = content + else: + line = "%s%s%s" % (o2, content, c2) + else: + line = self.graf(line) + + line = self.doPBr(line) + if self.html_type == 'xhtml': + line = re.sub(r'
    ', '
    ', line) + + if ext and anon: + out.append(out.pop() + "\n" + line) + else: + out.append(line) + + if not ext: + tag = 'p' + atts = '' + cite = '' + graf = '' + + if ext: + out.append(out.pop() + c1) + return '\n\n'.join(out) + + def fBlock(self, tag, atts, ext, cite, content): + """ + >>> t = Textile() + >>> t.fBlock("bq", "", None, "", "Hello BlockQuote") + ('\\t
    \\n', '\\t\\t

    ', 'Hello BlockQuote', '

    ', '\\n\\t
    ') + + >>> t.fBlock("bq", "", None, "http://google.com", "Hello BlockQuote") + ('\\t
    \\n', '\\t\\t

    ', 'Hello BlockQuote', '

    ', '\\n\\t
    ') + + >>> t.fBlock("bc", "", None, "", 'printf "Hello, World";') # doctest: +ELLIPSIS + ('
    ', '', ..., '', '
    ') + + >>> t.fBlock("h1", "", None, "", "foobar") + ('', '\\t

    ', 'foobar', '

    ', '') + """ + atts = self.pba(atts) + o1 = o2 = c2 = c1 = '' + + m = re.search(r'fn(\d+)', tag) + if m: + tag = 'p' + if m.group(1) in self.fn: + fnid = self.fn[m.group(1)] + else: + fnid = m.group(1) + atts = atts + ' id="fn%s"' % fnid + if atts.find('class=') < 0: + atts = atts + ' class="footnote"' + content = ('%s' % m.group(1)) + content + + if tag == 'bq': + cite = self.checkRefs(cite) + if cite: + cite = ' cite="%s"' % cite + else: + cite = '' + o1 = "\t\n" % (cite, atts) + o2 = "\t\t" % atts + c2 = "

    " + c1 = "\n\t" + + elif tag == 'bc': + o1 = "" % atts + o2 = "" % atts + c2 = "" + c1 = "" + content = self.shelve(self.encode_html(content.rstrip("\n") + "\n")) + + elif tag == 'notextile': + content = self.shelve(content) + o1 = o2 = '' + c1 = c2 = '' + + elif tag == 'pre': + content = self.shelve(self.encode_html(content.rstrip("\n") + "\n")) + o1 = "" % atts + o2 = c2 = '' + c1 = '' + + else: + o2 = "\t<%s%s>" % (tag, atts) + c2 = "" % tag + + content = self.graf(content) + return o1, o2, content, c2, c1 + + def footnoteRef(self, text): + """ + >>> t = Textile() + >>> t.footnoteRef('foo[1] ') # doctest: +ELLIPSIS + 'foo1 ' + """ + return re.sub(r'\b\[([0-9]+)\](\s)?', self.footnoteID, text) + + def footnoteID(self, match): + id, t = match.groups() + if id not in self.fn: + self.fn[id] = str(uuid.uuid4()) + fnid = self.fn[id] + if not t: + t = '' + return '%s%s' % (fnid, id, t) + + def glyphs(self, text): + """ + >>> t = Textile() + + >>> t.glyphs("apostrophe's") + 'apostrophe’s' + + >>> t.glyphs("back in '88") + 'back in ’88' + + >>> t.glyphs('foo ...') + 'foo …' + + >>> t.glyphs('--') + '—' + + >>> t.glyphs('FooBar[tm]') + 'FooBar™' + + >>> t.glyphs("

    Cat's Cradle by Vonnegut

    ") + '

    Cat’s Cradle by Vonnegut

    ' + + """ + # fix: hackish + text = re.sub(r'"\Z', '\" ', text) + + glyph_search = ( + re.compile(r"(\w)\'(\w)"), # apostrophe's + re.compile(r'(\s)\'(\d+\w?)\b(?!\')'), # back in '88 + re.compile(r'(\S)\'(?=\s|'+self.pnct+'|<|$)'), # single closing + re.compile(r'\'/'), # single opening + re.compile(r'(\S)\"(?=\s|'+self.pnct+'|<|$)'), # double closing + re.compile(r'"'), # double opening + re.compile(r'\b([A-Z][A-Z0-9]{2,})\b(?:[(]([^)]*)[)])'), # 3+ uppercase acronym + re.compile(r'\b([A-Z][A-Z\'\-]+[A-Z])(?=[\s.,\)>])'), # 3+ uppercase + re.compile(r'\b(\s{0,1})?\.{3}'), # ellipsis + re.compile(r'(\s?)--(\s?)'), # em dash + re.compile(r'\s-(?:\s|$)'), # en dash + re.compile(r'(\d+)( ?)x( ?)(?=\d+)'), # dimension sign + re.compile(r'\b ?[([]TM[])]', re.I), # trademark + re.compile(r'\b ?[([]R[])]', re.I), # registered + re.compile(r'\b ?[([]C[])]', re.I), # copyright + ) + + glyph_replace = [x % dict(self.glyph_defaults) for x in ( + r'\1%(txt_apostrophe)s\2', # apostrophe's + r'\1%(txt_apostrophe)s\2', # back in '88 + r'\1%(txt_quote_single_close)s', # single closing + r'%(txt_quote_single_open)s', # single opening + r'\1%(txt_quote_double_close)s', # double closing + r'%(txt_quote_double_open)s', # double opening + r'\1', # 3+ uppercase acronym + r'\1', # 3+ uppercase + r'\1%(txt_ellipsis)s', # ellipsis + r'\1%(txt_emdash)s\2', # em dash + r' %(txt_endash)s ', # en dash + r'\1\2%(txt_dimension)s\3', # dimension sign + r'%(txt_trademark)s', # trademark + r'%(txt_registered)s', # registered + r'%(txt_copyright)s', # copyright + )] + + result = [] + for line in re.compile(r'(<.*?>)', re.U).split(text): + if not re.search(r'<.*>', line): + for s, r in zip(glyph_search, glyph_replace): + line = s.sub(r, line) + result.append(line) + return ''.join(result) + + def vAlign(self, input): + d = {'^':'top', '-':'middle', '~':'bottom'} + return d.get(input, '') + + def hAlign(self, input): + d = {'<':'left', '=':'center', '>':'right', '<>': 'justify'} + return d.get(input, '') + + def getRefs(self, text): + """ + what is this for? + """ + pattern = re.compile(r'(?:(?<=^)|(?<=\s))\[(.+)\]((?:http(?:s?):\/\/|\/)\S+)(?=\s|$)', re.U) + text = pattern.sub(self.refs, text) + return text + + def refs(self, match): + flag, url = match.groups() + self.urlrefs[flag] = url + return '' + + def checkRefs(self, url): + return self.urlrefs.get(url, url) + + def isRelURL(self, url): + """ + Identify relative urls. + + >>> t = Textile() + >>> t.isRelURL("http://www.google.com/") + False + >>> t.isRelURL("/foo") + True + + """ + (scheme, netloc) = urlparse(url)[0:2] + return not scheme and not netloc + + def relURL(self, url): + scheme = urlparse(url)[0] + if self.restricted and scheme and scheme not in self.url_schemes: + return '#' + return url + + def shelve(self, text): + id = str(uuid.uuid4()) + self.shelf[id] = text + return id + + def retrieve(self, text): + """ + >>> t = Textile() + >>> id = t.shelve("foobar") + >>> t.retrieve(id) + 'foobar' + """ + while True: + old = text + for k, v in self.shelf.items(): + text = text.replace(k, v) + if text == old: + break + return text + + def encode_html(self, text, quotes=True): + a = ( + ('&', '&'), + ('<', '<'), + ('>', '>') + ) + + if quotes: + a = a + ( + ("'", '''), + ('"', '"') + ) + + for k, v in a: + text = text.replace(k, v) + return text + + def graf(self, text): + if not self.lite: + text = self.noTextile(text) + text = self.code(text) + + text = self.links(text) + + if not self.noimage: + text = self.image(text) + + if not self.lite: + text = self.lists(text) + text = self.table(text) + + text = self.span(text) + text = self.footnoteRef(text) + text = self.glyphs(text) + + return text.rstrip('\n') + + def links(self, text): + """ + >>> t = Textile() + >>> t.links('fooobar "Google":http://google.com/foobar/ and hello world "flickr":http://flickr.com/photos/jsamsa/ ') # doctest: +ELLIPSIS + 'fooobar ... and hello world ...' + """ + + punct = '!"#$%&\'*+,-./:;=?@\\^_`|~' + + pattern = r''' + (?P
        [\s\[{(]|[%s]   )?
+            "                          # start
+            (?P   %s       )
+            (?P   [^"]+?   )
+            \s?
+            (?:   \(([^)]+?)\)(?=")   )?     # $title
+            ":
+            (?P    (?:ftp|https?)? (?: :// )? [-A-Za-z0-9+&@#/?=~_()|!:,.;]*[-A-Za-z0-9+&@#/=~_()|]   )
+            (?P   [^\w\/;]*?   )
+            (?=<|\s|$)
+        ''' % (re.escape(punct), self.c)
+
+        text = re.compile(pattern, re.X).sub(self.fLink, text)
+
+        return text
+
+    def fLink(self, match):
+        pre, atts, text, title, url, post = match.groups()
+
+        if pre == None:
+            pre = ''
+
+        # assume ) at the end of the url is not actually part of the url
+        # unless the url also contains a (
+        if url.endswith(')') and not url.find('(') > -1:
+            post = url[-1] + post
+            url = url[:-1]
+
+        url = self.checkRefs(url)
+
+        atts = self.pba(atts)
+        if title:
+            atts = atts +  ' title="%s"' % self.encode_html(title)
+
+        if not self.noimage:
+            text = self.image(text)
+
+        text = self.span(text)
+        text = self.glyphs(text)
+
+        url = self.relURL(url)
+        out = '%s' % (self.encode_html(url), atts, self.rel, text)
+        out = self.shelve(out)
+        return ''.join([pre, out, post])
+
+    def span(self, text):
+        """
+        >>> t = Textile()
+        >>> t.span(r"hello %(bob)span *strong* and **bold**% goodbye")
+        'hello span strong and bold goodbye'
+        """
+        qtags = (r'\*\*', r'\*', r'\?\?', r'\-', r'__', r'_', r'%', r'\+', r'~', r'\^')
+        pnct = ".,\"'?!;:"
+
+        for qtag in qtags:
+            pattern = re.compile(r"""
+                (?:^|(?<=[\s>%(pnct)s])|([\]}]))
+                (%(qtag)s)(?!%(qtag)s)
+                (%(c)s)
+                (?::(\S+))?
+                ([^\s%(qtag)s]+|\S[^%(qtag)s\n]*[^\s%(qtag)s\n])
+                ([%(pnct)s]*)
+                %(qtag)s
+                (?:$|([\]}])|(?=%(selfpnct)s{1,2}|\s))
+            """ % {'qtag':qtag, 'c':self.c, 'pnct':pnct,
+                   'selfpnct':self.pnct}, re.X)
+            text = pattern.sub(self.fSpan, text)
+        return text
+
+
+    def fSpan(self, match):
+        _, tag, atts, cite, content, end, _ = match.groups()
+
+        qtags = {
+            '*': 'strong',
+            '**': 'b',
+            '??': 'cite',
+            '_' : 'em',
+            '__': 'i',
+            '-' : 'del',
+            '%' : 'span',
+            '+' : 'ins',
+            '~' : 'sub',
+            '^' : 'sup'
+        }
+        tag = qtags[tag]
+        atts = self.pba(atts)
+        if cite:
+            atts = atts + 'cite="%s"' % cite
+
+        content = self.span(content)
+
+        out = "<%s%s>%s%s" % (tag, atts, content, end, tag)
+        return out
+
+    def image(self, text):
+        """
+        >>> t = Textile()
+        >>> t.image('!/imgs/myphoto.jpg!:http://jsamsa.com')
+        ''
+        """
+        pattern = re.compile(r"""
+            (?:[\[{])?          # pre
+            \!                 # opening !
+            (%s)               # optional style,class atts
+            (?:\. )?           # optional dot-space
+            ([^\s(!]+)         # presume this is the src
+            \s?                # optional space
+            (?:\(([^\)]+)\))?  # optional title
+            \!                 # closing
+            (?::(\S+))?        # optional href
+            (?:[\]}]|(?=\s|$)) # lookahead: space or end of string
+        """ % self.c, re.U|re.X)
+        return pattern.sub(self.fImage, text)
+
+    def fImage(self, match):
+        # (None, '', '/imgs/myphoto.jpg', None, None)
+        atts, url, title, href = match.groups()
+        atts  = self.pba(atts)
+
+        if title:
+            atts = atts + ' title="%s" alt="%s"' % (title, title)
+        else:
+            atts = atts + ' alt=""'
+
+        if not self.isRelURL(url) and self.get_sizes:
+            size = getimagesize(url)
+            if (size):
+                atts += " %s" % size
+
+        if href:
+            href = self.checkRefs(href)
+
+        url = self.checkRefs(url)
+        url = self.relURL(url)
+
+        out = []
+        if href:
+            out.append('' % href)
+        if self.html_type == 'html':
+            out.append('' % (url, atts))
+        else:
+            out.append('' % (url, atts))
+        if href:
+            out.append('')
+
+        return ''.join(out)
+
+    def code(self, text):
+        text = self.doSpecial(text, '', '', self.fCode)
+        text = self.doSpecial(text, '@', '@', self.fCode)
+        text = self.doSpecial(text, '
    ', '
    ', self.fPre)
+        return text
+
+    def fCode(self, match):
+        before, text, after = match.groups()
+        if after == None:
+            after = ''
+        # text needs to be escaped
+        if not self.restricted:
+            text = self.encode_html(text)
+        return ''.join([before, self.shelve('%s' % text), after])
+
+    def fPre(self, match):
+        before, text, after = match.groups()
+        if after == None:
+            after = ''
+        # text needs to be escapedd
+        if not self.restricted:
+            text = self.encode_html(text)
+        return ''.join([before, '
    ', self.shelve(text), '
    ', after])
+
+    def doSpecial(self, text, start, end, method=None):
+        if method == None:
+            method = self.fSpecial
+        pattern = re.compile(r'(^|\s|[\[({>])%s(.*?)%s(\s|$|[\])}])?' % (re.escape(start), re.escape(end)), re.M|re.S)
+        return pattern.sub(method, text)
+
+    def fSpecial(self, match):
+        """
+        special blocks like notextile or code
+        """
+        before, text, after = match.groups()
+        if after == None:
+            after = ''
+        return ''.join([before, self.shelve(self.encode_html(text)), after])
+
+    def noTextile(self, text):
+        text = self.doSpecial(text, '', '', self.fTextile)
+        return self.doSpecial(text, '==', '==', self.fTextile)
+
+    def fTextile(self, match):
+        before, notextile, after = match.groups()
+        if after == None:
+            after = ''
+        return ''.join([before, self.shelve(notextile), after])
+
+
+def textile(text, head_offset=0, html_type='xhtml', encoding=None, output=None):
+    """
+    this function takes additional parameters:
+    head_offset - offset to apply to heading levels (default: 0)
+    html_type - 'xhtml' or 'html' style tags (default: 'xhtml')
+    """
+    return Textile().textile(text, head_offset=head_offset,
+                             html_type=html_type)
+
+def textile_restricted(text, lite=True, noimage=True, html_type='xhtml'):
+    """
+    Restricted version of Textile designed for weblog comments and other
+    untrusted input.
+
+    Raw HTML is escaped.
+    Style attributes are disabled.
+    rel='nofollow' is added to external links.
+
+    When lite=True is set (the default):
+    Block tags are restricted to p, bq, and bc.
+    Lists and tables are disabled.
+
+    When noimage=True is set (the default):
+    Image tags are disabled.
+
+    """
+    return Textile(restricted=True, lite=lite,
+                   noimage=noimage).textile(text, rel='nofollow',
+                                            html_type=html_type)
+
diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/txt/input.py
index aaff8b55c0..0b0bd6d570 100644
--- a/src/calibre/ebooks/txt/input.py
+++ b/src/calibre/ebooks/txt/input.py
@@ -12,7 +12,7 @@ from calibre.ebooks.chardet import detect
 from calibre.ebooks.txt.processor import convert_basic, convert_markdown, \
     separate_paragraphs_single_line, separate_paragraphs_print_formatted, \
     preserve_spaces, detect_paragraph_type, detect_formatting_type, \
-    convert_heuristic, normalize_line_endings
+    convert_heuristic, normalize_line_endings, convert_textile
 from calibre import _ent_pat, xml_entity_to_unicode
 
 class TXTInput(InputFormatPlugin):
@@ -41,6 +41,7 @@ class TXTInput(InputFormatPlugin):
                    'paragraph and no styling is applied.\n'
                    '* heuristic: Process using heuristics to determine formatting such '
                    'as chapter headings and italic text.\n'
+                   '* textile: Processing using textile formatting.\n'
                    '* markdown: Processing using markdown formatting. '
                    'To learn more about markdown see')+' http://daringfireball.net/projects/markdown/'),
         OptionRecommendation(name='preserve_spaces', recommended_value=False,
@@ -91,6 +92,9 @@ class TXTInput(InputFormatPlugin):
             except RuntimeError:
                 raise ValueError('This txt file has malformed markup, it cannot be'
                     ' converted by calibre. See http://daringfireball.net/projects/markdown/syntax')
+        elif options.formatting_type == 'textile':
+            log.debug('Running text though textile conversion...')
+            html = convert_textile(txt)
         else:
             # Determine the paragraph type of the document.
             if options.paragraph_type == 'auto':
diff --git a/src/calibre/ebooks/txt/processor.py b/src/calibre/ebooks/txt/processor.py
index ef9920185f..e1979063c0 100644
--- a/src/calibre/ebooks/txt/processor.py
+++ b/src/calibre/ebooks/txt/processor.py
@@ -1,4 +1,8 @@
 # -*- coding: utf-8 -*-
+__license__   = 'GPL v3'
+__copyright__ = '2009, John Schember '
+__docformat__ = 'restructuredtext en'
+
 
 '''
 Read content from txt file.
@@ -7,14 +11,10 @@ Read content from txt file.
 import os, re
 
 from calibre import prepare_string_for_xml, isbytestring
-from calibre.ebooks.markdown import markdown
 from calibre.ebooks.metadata.opf2 import OPFCreator
 from calibre.ebooks.txt.heuristicprocessor import TXTHeuristicProcessor
 from calibre.ebooks.conversion.preprocess import DocAnalysis
-
-__license__   = 'GPL v3'
-__copyright__ = '2009, John Schember '
-__docformat__ = 'restructuredtext en'
+from calibre.utils.cleantext import clean_ascii_chars
 
 HTML_TEMPLATE = u'%s\n%s\n'
 
@@ -34,10 +34,8 @@ def clean_txt(txt):
     # Remove excessive line breaks.
     txt = re.sub('\n{3,}', '\n\n', txt)
     #remove ASCII invalid chars : 0 to 8 and 11-14 to 24
-    chars = list(range(8)) + [0x0B, 0x0E, 0x0F] + list(range(0x10, 0x19))
-    illegal_chars = re.compile(u'|'.join(map(unichr, chars)))
-    txt = illegal_chars.sub('', txt)
-    
+    txt = clean_ascii_chars(txt)
+
     return txt
 
 def split_txt(txt, epub_split_size_kb=0):
@@ -74,12 +72,18 @@ def convert_heuristic(txt, title='', epub_split_size_kb=0):
     return tp.convert(txt, title, epub_split_size_kb)
 
 def convert_markdown(txt, title='', disable_toc=False):
+    from calibre.ebooks.markdown import markdown
     md = markdown.Markdown(
           extensions=['footnotes', 'tables', 'toc'],
           extension_configs={"toc": {"disable_toc": disable_toc}},
           safe_mode=False)
     return HTML_TEMPLATE % (title, md.convert(txt))
 
+def convert_textile(txt, title=''):
+    from calibre.ebooks.textile import textile
+    html = textile(txt, encoding='utf-8')
+    return HTML_TEMPLATE % (title, html)
+
 def normalize_line_endings(txt):
     txt = txt.replace('\r\n', '\n')
     txt = txt.replace('\r', '\n')
@@ -115,32 +119,32 @@ def split_string_separator(txt, size) :
 def detect_paragraph_type(txt):
     '''
     Tries to determine the formatting of the document.
-    
+
     block: Paragraphs are separated by a blank line.
     single: Each line is a paragraph.
     print: Each paragraph starts with a 2+ spaces or a tab
            and ends when a new paragraph is reached.
     unformatted: most lines have hard line breaks, few/no blank lines or indents
-    
+
     returns block, single, print, unformatted
     '''
     txt = txt.replace('\r\n', '\n')
     txt = txt.replace('\r', '\n')
     txt_line_count = len(re.findall('(?mu)^\s*.+$', txt))
-    
+
     # Check for hard line breaks - true if 55% of the doc breaks in the same region
     docanalysis = DocAnalysis('txt', txt)
     hardbreaks = docanalysis.line_histogram(.55)
-    
+
     if hardbreaks:
         # Determine print percentage
         tab_line_count = len(re.findall('(?mu)^(\t|\s{2,}).+$', txt))
         print_percent = tab_line_count / float(txt_line_count)
-     
+
         # Determine block percentage
         empty_line_count = len(re.findall('(?mu)^\s*$', txt))
         block_percent = empty_line_count / float(txt_line_count)
-        
+
         # Compare the two types - the type with the larger number of instances wins
         # in cases where only one or the other represents the vast majority of the document neither wins
         if print_percent >= block_percent:
@@ -149,32 +153,41 @@ def detect_paragraph_type(txt):
         elif .15 <= block_percent <= .75:
             return 'block'
 
-        # Assume unformatted text with hardbreaks if nothing else matches        
+        # Assume unformatted text with hardbreaks if nothing else matches
         return 'unformatted'
-    
+
     # return single if hardbreaks is false
     return 'single'
 
 
 def detect_formatting_type(txt):
+    markdown_count = 0
+    textile_count = 0
+
     # Check for markdown
     # Headings
-    if len(re.findall('(?mu)^#+', txt)) >= 5:
-        return 'markdown'
-    if len(re.findall('(?mu)^=+$', txt)) >= 5:
-        return 'markdown'
-    if len(re.findall('(?mu)^-+$', txt)) >= 5:
-        return 'markdown'
+    markdown_count += len(re.findall('(?mu)^#+', txt))
+    markdown_count += len(re.findall('(?mu)^=+$', txt))
+    markdown_count += len(re.findall('(?mu)^-+$', txt))
     # Images
-    if len(re.findall('(?u)!\[.*?\]\(.+?\)', txt)) >= 5:
-        return 'markdown'
+    markdown_count += len(re.findall('(?u)!\[.*?\]\(.+?\)', txt))
     # Links
-    if len(re.findall('(?u)(^|(?P
    [^!]))\[.*?\]\([^)]+\)', txt)) >= 5:
-        return 'markdown'
-    # Escaped characters
-    md_escapted_characters = ['\\', '`', '*', '_', '{', '}', '[', ']', '(', ')', '#', '+', '-', '.', '!']
-    for c in md_escapted_characters:
-        if txt.count('\\'+c) > 10:
+    markdown_count += len(re.findall('(?u)(^|(?P
    [^!]))\[.*?\]\([^)]+\)', txt))
+
+    # Check for textile
+    # Headings
+    textile_count += len(re.findall(r'(?mu)^h[1-6]\.', txt))
+    # Block quote.
+    textile_count += len(re.findall(r'(?mu)^bq\.', txt))
+    # Images
+    textile_count += len(re.findall(r'\![^\s]+(:[^\s]+)*', txt))
+    # Links
+    textile_count += len(re.findall(r'"(\(.+?\))*[^\(]+?(\(.+?\))*":[^\s]+', txt))
+
+    if markdown_count > 5 or textile_count > 5:
+        if markdown_count > textile_count:
             return 'markdown'
-    
+        else:
+            return 'textile'
+
     return 'heuristic'
diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py
index df6ac45e5b..e699551150 100644
--- a/src/calibre/gui2/__init__.py
+++ b/src/calibre/gui2/__init__.py
@@ -269,10 +269,14 @@ def question_dialog(parent, title, msg, det_msg='', show_copy_button=True,
 
     return d.exec_() == yes_button
 
-def info_dialog(parent, title, msg, det_msg='', show=False):
+def info_dialog(parent, title, msg, det_msg='', show=False,
+        show_copy_button=True):
     d = MessageBox(QMessageBox.Information, title, msg, QMessageBox.Ok,
                     parent, det_msg)
     d.setIconPixmap(QPixmap(I('dialog_information.png')))
+    if not show_copy_button:
+        d.cb.setVisible(False)
+
     if show:
         return d.exec_()
     return d
diff --git a/src/calibre/gui2/catalog/catalog_bibtex.py b/src/calibre/gui2/catalog/catalog_bibtex.py
index 5030cf6ec8..7b7739bb46 100644
--- a/src/calibre/gui2/catalog/catalog_bibtex.py
+++ b/src/calibre/gui2/catalog/catalog_bibtex.py
@@ -27,14 +27,17 @@ class PluginWidget(QWidget, Ui_Form):
     def __init__(self, parent=None):
         QWidget.__init__(self, parent)
         self.setupUi(self)
-        from calibre.library.catalog import FIELDS
-        self.all_fields = []
-        for x in FIELDS :
-            if x != 'all':
-                self.all_fields.append(x)
-                QListWidgetItem(x, self.db_fields)
 
     def initialize(self, name, db): #not working properly to update
+        from calibre.library.catalog import FIELDS
+
+        self.all_fields = [x for x in FIELDS if x != 'all']
+        #add custom columns
+        self.all_fields.extend([x for x in sorted(db.custom_field_keys())])
+        #populate
+        for x in self.all_fields:
+            QListWidgetItem(x, self.db_fields)
+
         self.name = name
         fields = gprefs.get(name+'_db_fields', self.all_fields)
         # Restore the activated db_fields from last use
diff --git a/src/calibre/gui2/dialogs/drm_error.py b/src/calibre/gui2/dialogs/drm_error.py
new file mode 100644
index 0000000000..5fbba47165
--- /dev/null
+++ b/src/calibre/gui2/dialogs/drm_error.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal '
+__docformat__ = 'restructuredtext en'
+
+
+from PyQt4.Qt import QDialog
+from calibre.gui2.dialogs.drm_error_ui import Ui_Dialog
+
+class DRMErrorMessage(QDialog, Ui_Dialog):
+
+    def __init__(self, parent=None, title=None):
+        QDialog.__init__(self, parent)
+        self.setupUi(self)
+        if title is not None:
+            t = unicode(self.msg.text())
+            self.msg.setText('
    %s
    %s'%(title, t))
+        self.resize(self.sizeHint())
+
diff --git a/src/calibre/gui2/dialogs/drm_error.ui b/src/calibre/gui2/dialogs/drm_error.ui
new file mode 100644
index 0000000000..842807c9bc
--- /dev/null
+++ b/src/calibre/gui2/dialogs/drm_error.ui
@@ -0,0 +1,102 @@
+
+
+ Dialog
+ 
+  
+   
+    0
+    0
+    417
+    235
+   
+  
+  
+   This book is DRMed
+  
+  
+   
+    
+     
+      
+       0
+       0
+      
+     
+     
+      
+       132
+       16777215
+      
+     
+     
+      
+     
+     
+      :/images/document-encrypt.png
+     
+    
+   
+   
+    
+     
+      <p>This book is locked by <b>DRM</b>. To learn more about DRM and why you cannot read or convert this book in calibre, 
+<a href="http://bugs.calibre-ebook.com/wiki/DRM">click here</a>.
+     
+     
+      true
+     
+     
+      true
+     
+    
+   
+   
+    
+     
+      Qt::Horizontal
+     
+     
+      QDialogButtonBox::Close
+     
+    
+   
+  
+ 
+ 
+  
+ 
+ 
+  
+   buttonBox
+   accepted()
+   Dialog
+   accept()
+   
+    
+     248
+     254
+    
+    
+     157
+     274
+    
+   
+  
+  
+   buttonBox
+   rejected()
+   Dialog
+   reject()
+   
+    
+     316
+     260
+    
+    
+     286
+     274
+    
+   
+  
+ 
+
diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py
index e1ee4327f3..5ea8f00148 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.py
+++ b/src/calibre/gui2/dialogs/metadata_bulk.py
@@ -15,7 +15,7 @@ from calibre.ebooks.metadata import string_to_authors, authors_to_string
 from calibre.ebooks.metadata.book.base import composite_formatter
 from calibre.ebooks.metadata.meta import get_metadata
 from calibre.gui2.custom_column_widgets import populate_metadata_page
-from calibre.gui2 import error_dialog
+from calibre.gui2 import error_dialog, ResizableDialog
 from calibre.gui2.progress_indicator import ProgressIndicator
 from calibre.utils.config import dynamic
 from calibre.utils.titlecase import titlecase
@@ -49,7 +49,7 @@ def get_cover_data(path):
 
 
 
-class MyBlockingBusy(QDialog):
+class MyBlockingBusy(QDialog): # {{{
 
     do_one_signal = pyqtSignal()
 
@@ -241,8 +241,9 @@ class MyBlockingBusy(QDialog):
         self.current_index += 1
         self.do_one_signal.emit()
 
+    # }}}
 
-class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
+class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
 
     s_r_functions = {       ''              : lambda x: x,
                             _('Lower Case') : lambda x: icu_lower(x),
@@ -261,9 +262,8 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
                         ]
 
     def __init__(self, window, rows, model, tab):
-        QDialog.__init__(self, window)
+        ResizableDialog.__init__(self, window)
         Ui_MetadataBulkDialog.__init__(self)
-        self.setupUi(self)
         self.model = model
         self.db = model.db
         self.ids = [self.db.id(r) for r in rows]
diff --git a/src/calibre/gui2/dialogs/metadata_bulk.ui b/src/calibre/gui2/dialogs/metadata_bulk.ui
index 41858b099b..9240cd1af8 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.ui
+++ b/src/calibre/gui2/dialogs/metadata_bulk.ui
@@ -6,8 +6,8 @@
    
     0
     0
-    752
-    633
+    850
+    650
    
   
   
@@ -17,8 +17,8 @@
    
     :/images/edit_input.png:/images/edit_input.png
   
-  
-   
+  
+   
     
      
       
@@ -28,818 +28,836 @@
      
     
    
-   
-    
-     
-      6
+   
+    
+     
+      QFrame::NoFrame
      
-     
+     
       0
      
-     
-      
-       
+     
+      true
+     
+     
+      
+       
+        0
+        0
+        842
+        589
+       
+      
+      
+       
         0
        
-       
-        
-         &Basic metadata
-        
-        
-         
-          
-           
-            &Author(s): 
-           
-           
-            Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter
-           
-           
-            authors
-           
-          
-         
-         
-          
-           
-            A&utomatically set author sort
-           
-          
-         
-         
-          
-           
-            Author s&ort: 
-           
-           
-            Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter
-           
-           
-            author_sort
-           
-          
-         
-         
-          
-           
-            Specify how the author(s) of this book should be sorted. For example Charles Dickens should be sorted as Dickens, Charles.
-           
-          
-         
-         
-          
-           
-            &Rating:
-           
-           
-            Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter
-           
-           
-            rating
-           
-          
-         
-         
-          
-           
-            Rating of this book. 0-5 stars
-           
-           
-            Rating of this book. 0-5 stars
-           
-           
-            QAbstractSpinBox::PlusMinus
-           
-           
-            No change
-           
-           
-             stars
-           
-           
-            -1
-           
-           
-            5
-           
-           
-            -1
-           
-          
-         
-         
-          
-           
-            &Publisher: 
-           
-           
-            Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter
-           
-           
-            publisher
-           
-          
-         
-         
-          
-           
-            true
-           
-          
-         
-         
-          
-           
-            Add ta&gs: 
-           
-           
-            Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter
-           
-           
-            tags
-           
-          
-         
-         
-          
-           
-            Tags categorize the book. This is particularly useful while searching. <br><br>They can be any words or phrases, separated by commas.
-           
-          
-         
-         
-          
-           
-            Open Tag Editor
-           
-           
-            Open Tag Editor
-           
-           
-            
-             :/images/chapters.png:/images/chapters.png
-           
-          
-         
-         
-          
-           
-            &Remove tags:
-           
-           
-            remove_tags
-           
-          
-         
-         
-          
-           
-            Comma separated list of tags to remove from the books. 
-           
-          
-         
-         
-          
-           
-            Check this box to remove all tags from the books.
-           
-           
-            Remove all
-           
-          
-         
-         
-          
-           
-            &Series:
-           
-           
-            Qt::PlainText
-           
-           
-            Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter
-           
-           
-            series
-           
-          
-         
-         
-          
-           
-            
+       
+        
+         
+          0
+         
+         
+          
+           &Basic metadata
+          
+          
+           
+            
+             
+              &Author(s): 
+             
+             
+              Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter
+             
+             
+              authors
+             
+            
+           
+           
+            
+             
+              A&utomatically set author sort
+             
+            
+           
+           
+            
+             
+              Author s&ort: 
+             
+             
+              Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter
+             
+             
+              author_sort
+             
+            
+           
+           
+            
              
-              List of known series. You can add new series.
+              Specify how the author(s) of this book should be sorted. For example Charles Dickens should be sorted as Dickens, Charles.
+             
+            
+           
+           
+            
+             
+              &Rating:
+             
+             
+              Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter
+             
+             
+              rating
+             
+            
+           
+           
+            
+             
+              Rating of this book. 0-5 stars
              
              
-              List of known series. You can add new series.
+              Rating of this book. 0-5 stars
              
+             
+              QAbstractSpinBox::PlusMinus
+             
+             
+              No change
+             
+             
+               stars
+             
+             
+              -1
+             
+             
+              5
+             
+             
+              -1
+             
+            
+           
+           
+            
+             
+              &Publisher: 
+             
+             
+              Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter
+             
+             
+              publisher
+             
+            
+           
+           
+            
              
               true
              
-             
-              QComboBox::InsertAlphabetically
+            
+           
+           
+            
+             
+              Add ta&gs: 
              
-             
-              QComboBox::AdjustToContents
+             
+              Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter
+             
+             
+              tags
              
             
            
-           
-            
+           
+            
              
-              If checked, the series will be cleared
+              Tags categorize the book. This is particularly useful while searching. <br><br>They can be any words or phrases, separated by commas.
+             
+            
+           
+           
+            
+             
+              Open Tag Editor
              
              
-              Clear series
+              Open Tag Editor
+             
+             
+              
+               :/images/chapters.png:/images/chapters.png
              
             
            
-           
-            
-             
-              Qt::Horizontal
+           
+            
+             
+              &Remove tags:
              
-             
-              
-               20
-               0
-              
+             
+              remove_tags
              
-            
+            
            
-          
-         
-         
-          
-           
-            
+           
+            
              
-              If not checked, the series number for the books will be set to 1.
+              Comma separated list of tags to remove from the books. 
+             
+            
+           
+           
+            
+             
+              Check this box to remove all tags from the books.
+             
+             
+              Remove all
+             
+            
+           
+           
+            
+             
+              &Series:
+             
+             
+              Qt::PlainText
+             
+             
+              Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter
+             
+             
+              series
+             
+            
+           
+           
+            
+             
+              
+               
+                List of known series. You can add new series.
+               
+               
+                List of known series. You can add new series.
+               
+               
+                true
+               
+               
+                QComboBox::InsertAlphabetically
+               
+               
+                QComboBox::AdjustToContents
+               
+              
+             
+             
+              
+               
+                If checked, the series will be cleared
+               
+               
+                Clear series
+               
+              
+             
+             
+              
+               
+                Qt::Horizontal
+               
+               
+                
+                 20
+                 0
+                
+               
+              
+             
+            
+           
+           
+            
+             
+              
+               
+                If not checked, the series number for the books will be set to 1.
 If checked, selected books will be automatically numbered, in the order
 you selected them. So if you selected Book A and then Book B,
 Book A will have series number 1 and Book B series number 2.
-             
-             
-              Automatically number books in this series
-             
-            
-           
-           
-            
-             
-              false
-             
-             
-              Series will normally be renumbered from the highest number in the database
+               
+               
+                Automatically number books in this series
+               
+              
+             
+             
+              
+               
+                false
+               
+               
+                Series will normally be renumbered from the highest number in the database
 for that series. Checking this box will tell calibre to start numbering
 from the value in the box
+               
+               
+                Force numbers to start with 
+               
+              
+             
+             
+              
+               
+                false
+               
+               
+                1
+               
+               
+                990000
+               
+               
+                1
+               
+              
+             
+             
+              
+               
+                Qt::Horizontal
+               
+               
+                
+                 20
+                 10
+                
+               
+              
+             
+            
+           
+           
+            
+             
+              Remove &format:
+             
+             
+              remove_format
+             
+            
+           
+           
+            
+           
+           
+            
+             
+              true
+             
+            
+           
+           
+            
+             
+              &Swap title and author
+             
+            
+           
+           
+            
+             
+              Force the title to be in title case. If both this and swap authors are checked,
+title and author are swapped before the title case is set
              
              
-              Force numbers to start with 
+              Change title to title case
              
             
            
-           
-            
-             
-              false
-             
-             
-              1
-             
-             
-              990000
-             
-             
-              1
-             
-            
-           
-           
-            
-             
-              Qt::Horizontal
-             
-             
-              
-               20
-               10
-              
-             
-            
-           
-          
-         
-         
-          
-           
-            Remove &format:
-           
-           
-            remove_format
-           
-          
-         
-         
-          
-         
-         
-          
-           
-            true
-           
-          
-         
-         
-          
-           
-            &Swap title and author
-           
-          
-         
-         
-          
-           
-            Force the title to be in title case. If both this and swap authors are checked,
-title and author are swapped before the title case is set
-           
-           
-            Change title to title case
-           
-          
-         
-         
-          
-           
-            Remove stored conversion settings for the selected books.
+           
+            
+             
+              Remove stored conversion settings for the selected books.
 
 Future conversion of these books will use the default settings.
-           
-           
-            Remove &stored conversion settings for the selected books
-           
-          
-         
-         
-          
-           
-            Qt::Vertical
-           
-           
-            
-             20
-             40
-            
-           
-          
-         
-         
-          
-           
-            Change &cover
-           
-           
-            
-             
-              
-               &Generate default cover
-              
-             
-            
-            
-             
-              
-               &Remove cover
-              
-             
-            
-            
-             
-              
-               Set from &ebook file(s)
-              
-             
-            
-           
-          
-         
-        
-       
-       
-        
-         &Custom metadata
-        
-       
-       
-        
-         &Search and replace
-        
-        
-         
-          QLayout::SetMinimumSize
-         
-         
-          
-           
-            true
-           
-           
-            true
-           
-          
-         
-         
-          
-           
-            
-           
-          
-         
-         
-          
-           
-            Search &field:
-           
-           
-            search_field
-           
-          
-         
-         
-          
-           
-            The name of the field that you want to search
-           
-          
-         
-         
-          
-           
-            
+             
              
-              Search &mode:
-             
-             
-              search_mode
+              Remove &stored conversion settings for the selected books
              
             
            
-           
-            
-             
-              Choose whether to use basic text matching or advanced regular expression matching
-             
-            
-           
-           
-            
+           
+            
              
-              Qt::Horizontal
+              Qt::Vertical
              
              
               
                20
-               10
+               40
               
              
             
            
-          
-         
-         
-          
-           
-            Te&mplate:
-           
-           
-            s_r_template
-           
-          
-         
-         
-          
-           
-            
-             100
-             0
-            
-           
-           
-            Enter a template to be used as the source for the search/replace
-           
-          
-         
-         
-          
-           
-            &Search for:
-           
-           
-            search_for
-           
-          
-         
-         
-          
-           
-            
-             100
-             0
-            
-           
-           
-            Enter the what you are looking for, either plain text or a regular expression, depending on the mode
-           
-          
-         
-         
-          
-           
-            Check this box if the search string must match exactly upper and lower case. Uncheck it if case is to be ignored
-           
-           
-            Cas&e sensitive
-           
-           
-            true
-           
-          
-         
-         
-          
-           
-            &Replace with:
-           
-           
-            replace_with
-           
-          
-         
-         
-          
-           
-            The replacement text. The matched search text will be replaced with this string
-           
-          
-         
-         
-          
-           
-            
-             
-              &Apply function after replace:
-             
-             
-              replace_func
+           
+            
+             
+              Change &cover
              
+             
+              
+               
+                
+                 &Generate default cover
+                
+               
+              
+              
+               
+                
+                 &Remove cover
+                
+               
+              
+              
+               
+                
+                 Set from &ebook file(s)
+                
+               
+              
+             
             
            
-           
-            
-             
-              Specify how the text is to be processed after matching and replacement. In character mode, the entire
-field is processed. In regular expression mode, only the matched text is processed
-             
-            
-           
-           
-            
-             
-              Qt::Horizontal
-             
-             
-              
-               20
-               10
-              
-             
-            
-           
           
-         
-         
-          
-           
-            &Destination field:
+         
+         
+          
+           &Custom metadata
+          
+         
+         
+          
+           &Search and replace
+          
+          
+           
+            QLayout::SetMinimumSize
            
-           
-            destination_field
-           
-          
-         
-         
-          
-           
-            The field that the text will be put into after all replacements.
-If blank, the source field is used if the field is modifiable
-           
-          
-         
-         
-          
-           
-            
+           
+            
+             
+              true
+             
+             
+              true
+             
+            
+           
+           
+            
              
-              M&ode:
+              
+             
+            
+           
+           
+            
+             
+              Search &field:
              
              
-              replace_mode
+              search_field
              
             
            
-           
-            
+           
+            
              
-              Specify how the text should be copied into the destination.
+              The name of the field that you want to search
              
             
            
-           
-            
+           
+            
+             
+              
+               
+                Search &mode:
+               
+               
+                search_mode
+               
+              
+             
+             
+              
+               
+                Choose whether to use basic text matching or advanced regular expression matching
+               
+              
+             
+             
+              
+               
+                Qt::Horizontal
+               
+               
+                
+                 20
+                 10
+                
+               
+              
+             
+            
+           
+           
+            
+             
+              Te&mplate:
+             
+             
+              s_r_template
+             
+            
+           
+           
+            
+             
+              
+               100
+               0
+              
+             
              
-              Specifies whether result items should be split into multiple values or
-left as single values. This option has the most effect when the source field is
-not multiple and the destination field is multiple
+              Enter a template to be used as the source for the search/replace
+             
+            
+           
+           
+            
+             
+              &Search for:
+             
+             
+              search_for
+             
+            
+           
+           
+            
+             
+              
+               100
+               0
+              
+             
+             
+              Enter the what you are looking for, either plain text or a regular expression, depending on the mode
+             
+            
+           
+           
+            
+             
+              Check this box if the search string must match exactly upper and lower case. Uncheck it if case is to be ignored
              
              
-              Split &result
+              Cas&e sensitive
              
              
               true
              
             
            
-           
-            
-             
-              Qt::Horizontal
-             
-             
-              
-               20
-               10
-              
-             
-            
-           
-          
-         
-         
-          
-           
-            
-             
-              Qt::Horizontal
-             
-             
-              
-               20
-               0
-              
-             
-            
-           
-           
-            
+           
+            
              
-              For multiple-valued fields, sho&w
+              &Replace with:
              
              
-              results_count
+              replace_with
              
             
            
-           
-            
-             
-              true
-             
-             
-              1
-             
-             
-              999
-             
-             
-              999
-             
-            
-           
-           
-            
-             
-              values starting a&t
-             
-             
-              starting_from
-             
-            
-           
-           
-            
-             
-              true
-             
-             
-              1
-             
-             
-              999
-             
-             
-              1
-             
-            
-           
-           
-            
-             
-              with values separated b&y
-             
-             
-              multiple_separator
-             
-            
-           
-           
-            
+           
+            
              
-              Used when displaying test results to separate values in multiple-valued fields
+              The replacement text. The matched search text will be replaced with this string
              
             
            
-          
-         
-         
-          
-           
-            QFrame::NoFrame
-           
-           
-            true
-           
-           
-            
-             
-              0
-              0
-              726
-              334
-             
-            
-            
-             
-              
+           
+            
+             
+              
                
-                Test text
+                &Apply function after replace:
+               
+               
+                replace_func
                
               
              
-             
-              
-               
-                Test result
+             
+              
+               
+                Specify how the text is to be processed after matching and replacement. In character mode, the entire
+field is processed. In regular expression mode, only the matched text is processed
                
               
              
-             
-              
-               
-                Your test:
-               
-              
-             
-             
-              
-             
-             
-              
-             
-             
-              
+             
+              
                
-                Qt::Vertical
+                Qt::Horizontal
                
                
                 
                  20
-                 5
+                 10
                 
                
               
              
             
-           
-          
-         
-        
-       
-      
-     
-    
+           
+           
+            
+             
+              &Destination field:
+             
+             
+              destination_field
+             
+            
+           
+           
+            
+             
+              The field that the text will be put into after all replacements.
+If blank, the source field is used if the field is modifiable
+             
+            
+           
+           
+            
+             
+              
+               
+                M&ode:
+               
+               
+                replace_mode
+               
+              
+             
+             
+              
+               
+                Specify how the text should be copied into the destination.
+               
+              
+             
+             
+              
+               
+                Specifies whether result items should be split into multiple values or
+left as single values. This option has the most effect when the source field is
+not multiple and the destination field is multiple
+               
+               
+                Split &result
+               
+               
+                true
+               
+              
+             
+             
+              
+               
+                Qt::Horizontal
+               
+               
+                
+                 20
+                 10
+                
+               
+              
+             
+            
+           
+           
+            
+             
+              
+               
+                Qt::Horizontal
+               
+               
+                
+                 20
+                 0
+                
+               
+              
+             
+             
+              
+               
+                For multiple-valued fields, sho&w
+               
+               
+                results_count
+               
+              
+             
+             
+              
+               
+                true
+               
+               
+                1
+               
+               
+                999
+               
+               
+                999
+               
+              
+             
+             
+              
+               
+                values starting a&t
+               
+               
+                starting_from
+               
+              
+             
+             
+              
+               
+                true
+               
+               
+                1
+               
+               
+                999
+               
+               
+                1
+               
+              
+             
+             
+              
+               
+                with values separated b&y
+               
+               
+                multiple_separator
+               
+              
+             
+             
+              
+               
+                Used when displaying test results to separate values in multiple-valued fields
+               
+              
+             
+            
+           
+           
+            
+             
+              QFrame::NoFrame
+             
+             
+              true
+             
+             
+              
+               
+                0
+                0
+                197
+                60
+               
+              
+              
+               
+                
+                 
+                  Test text
+                 
+                
+               
+               
+                
+                 
+                  Test result
+                 
+                
+               
+               
+                
+                 
+                  Your test:
+                 
+                
+               
+               
+                
+               
+               
+                
+               
+               
+                
+                 
+                  Qt::Vertical
+                 
+                 
+                  
+                   20
+                   5
+                  
+                 
+                
+               
+              
+             
+            
+           
+          
+         
+        
+       
+      
+     
+    
    
-   
+   
     
      
       Qt::Horizontal
@@ -893,7 +911,6 @@ not multiple and the destination field is multiple
   swap_title_and_author
   change_title_to_title_case
   button_box
-  central_widget
   search_field
   search_mode
   s_r_template
diff --git a/src/calibre/gui2/dialogs/metadata_single.py b/src/calibre/gui2/dialogs/metadata_single.py
index c2588f57a8..a4e8bb6972 100644
--- a/src/calibre/gui2/dialogs/metadata_single.py
+++ b/src/calibre/gui2/dialogs/metadata_single.py
@@ -823,7 +823,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
                                 if book.series_index is not None:
                                     self.series_index.setValue(book.series_index)
                         if book.has_cover:
-                            if d.opt_auto_download_cover.isChecked() and book.has_cover:
+                            if d.opt_auto_download_cover.isChecked():
                                 self.fetch_cover()
                             else:
                                 self.fetch_cover_button.setFocus(Qt.OtherFocusReason)
diff --git a/src/calibre/gui2/dialogs/user_profiles.py b/src/calibre/gui2/dialogs/user_profiles.py
index 71c9ebcd04..04c41f0c5e 100644
--- a/src/calibre/gui2/dialogs/user_profiles.py
+++ b/src/calibre/gui2/dialogs/user_profiles.py
@@ -4,7 +4,7 @@ __copyright__ = '2008, Kovid Goyal '
 import time, os
 
 from PyQt4.Qt import SIGNAL, QUrl, QAbstractListModel, Qt, \
-        QVariant, QInputDialog
+        QVariant
 
 from calibre.web.feeds.recipes import compile_recipe
 from calibre.web.feeds.news import AutomaticNewsRecipe
@@ -256,24 +256,61 @@ class %(classname)s(%(base_class)s):
 
     def add_builtin_recipe(self):
         from calibre.web.feeds.recipes.collection import \
-            get_builtin_recipe_by_title, get_builtin_recipe_titles
-        items = sorted(get_builtin_recipe_titles(), key=sort_key)
+            get_builtin_recipe_collection, get_builtin_recipe_by_id
+        from PyQt4.Qt import QDialog, QVBoxLayout, QListWidgetItem, \
+                QListWidget, QDialogButtonBox, QSize
 
+        d = QDialog(self)
+        d.l = QVBoxLayout()
+        d.setLayout(d.l)
+        d.list = QListWidget(d)
+        d.list.doubleClicked.connect(lambda x: d.accept())
+        d.l.addWidget(d.list)
+        d.bb = QDialogButtonBox(QDialogButtonBox.Ok|QDialogButtonBox.Cancel,
+                Qt.Horizontal, d)
+        d.bb.accepted.connect(d.accept)
+        d.bb.rejected.connect(d.reject)
+        d.l.addWidget(d.bb)
+        d.setWindowTitle(_('Choose builtin recipe'))
+        items = []
+        for r in get_builtin_recipe_collection():
+            id_ = r.get('id', '')
+            title = r.get('title', '')
+            lang = r.get('language', '')
+            if id_ and title:
+                items.append((title + ' [%s]'%lang, id_))
 
-        title, ok = QInputDialog.getItem(self, _('Pick recipe'), _('Pick the recipe to customize'),
-                                     items, 0, False)
-        if ok:
-            title = unicode(title)
-            profile = get_builtin_recipe_by_title(title)
-            if self._model.has_title(title):
-                if question_dialog(self, _('Replace recipe?'),
-                    _('A custom recipe named %s already exists. Do you want to '
-                        'replace it?')%title):
-                    self._model.replace_by_title(title, profile)
-                else:
-                    return
+        items.sort(key=lambda x:sort_key(x[0]))
+        for title, id_ in items:
+            item = QListWidgetItem(title)
+            item.setData(Qt.UserRole, id_)
+            d.list.addItem(item)
+
+        d.resize(QSize(450, 400))
+        ret = d.exec_()
+        d.list.doubleClicked.disconnect()
+        if ret != d.Accepted:
+            return
+
+        items = list(d.list.selectedItems())
+        if not items:
+            return
+        item = items[-1]
+        id_ = unicode(item.data(Qt.UserRole).toString())
+        title = unicode(item.data(Qt.DisplayRole).toString()).rpartition(' [')[0]
+        profile = get_builtin_recipe_by_id(id_)
+        if profile is None:
+            raise Exception('Something weird happened')
+
+        if self._model.has_title(title):
+            if question_dialog(self, _('Replace recipe?'),
+                _('A custom recipe named %s already exists. Do you want to '
+                    'replace it?')%title):
+                self._model.replace_by_title(title, profile)
             else:
-                self.model.add(title, profile)
+                return
+        else:
+            self.model.add(title, profile)
 
         self.clear()
 
diff --git a/src/calibre/gui2/layout.py b/src/calibre/gui2/layout.py
index aaaf1b0267..2edf19d0c4 100644
--- a/src/calibre/gui2/layout.py
+++ b/src/calibre/gui2/layout.py
@@ -8,9 +8,9 @@ __docformat__ = 'restructuredtext en'
 from functools import partial
 
 from PyQt4.Qt import QIcon, Qt, QWidget, QToolBar, QSize, \
-    pyqtSignal, QToolButton, QPushButton, \
-    QObject, QVBoxLayout, QSizePolicy, QLabel, QHBoxLayout, QActionGroup, \
-    QMenu
+    pyqtSignal, QToolButton, QMenu, QCheckBox, \
+    QObject, QVBoxLayout, QSizePolicy, QLabel, QHBoxLayout, QActionGroup
+
 
 from calibre.constants import __appname__
 from calibre.gui2.search_box import SearchBox2, SavedSearchBox
@@ -178,7 +178,9 @@ class SearchBar(QWidget): # {{{
         x.setToolTip(_("
    Search the list of books by title, author, publisher, tags, comments, etc.

    Words separated by spaces are ANDed"))
         l.addWidget(x)
 
-        self.search_button = QPushButton(_('&Go!'))
+        self.search_button = QToolButton()
+        self.search_button.setToolButtonStyle(Qt.ToolButtonTextOnly)
+        self.search_button.setText(_('&Go!'))
         l.addWidget(self.search_button)
         self.search_button.setSizePolicy(QSizePolicy.Minimum,
                 QSizePolicy.Minimum)
@@ -192,6 +194,12 @@ class SearchBar(QWidget): # {{{
         l.addWidget(x)
         x.setToolTip(_("Reset Quick Search"))
 
+        x = parent.search_highlight_only = QCheckBox()
+        x.setText(_('&Highlight'))
+        x.setToolTip(_('Highlight matched books in the book list, instead '
+            'of restricting the book list to the matches.'))
+        l.addWidget(x)
+
         x = parent.saved_search = SavedSearchBox(self)
         x.setMaximumSize(QSize(150, 16777215))
         x.setMinimumContentsLength(15)
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index 49cb1ce182..eea452c238 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -10,7 +10,7 @@ from contextlib import closing
 from operator import attrgetter
 
 from PyQt4.Qt import QAbstractTableModel, Qt, pyqtSignal, QIcon, QImage, \
-        QModelIndex, QVariant, QDate
+        QModelIndex, QVariant, QDate, QColor
 
 from calibre.gui2 import NONE, config, UNDEFINED_QDATE
 from calibre.utils.pyparsing import ParseException
@@ -93,6 +93,9 @@ class BooksModel(QAbstractTableModel): # {{{
         self.bool_no_icon = QIcon(I('list_remove.png'))
         self.bool_blank_icon = QIcon(I('blank.png'))
         self.device_connected = False
+        self.rows_matching = set()
+        self.lowest_row_matching = None
+        self.highlight_only = False
         self.read_config()
 
     def change_alignment(self, colname, alignment):
@@ -229,9 +232,27 @@ class BooksModel(QAbstractTableModel): # {{{
             self.endInsertRows()
             self.count_changed()
 
+    def set_highlight_only(self, toWhat):
+        self.highlight_only = toWhat
+        if self.last_search:
+            self.research()
+
     def search(self, text, reset=True):
         try:
-            self.db.search(text)
+            if self.highlight_only:
+                self.db.search('')
+                if not text:
+                    self.rows_matching = set()
+                    self.lowest_row_matching = None
+                else:
+                    self.rows_matching = self.db.search(text, return_matches=True)
+                    if self.rows_matching:
+                        self.lowest_row_matching = self.db.row(self.rows_matching[0])
+                    self.rows_matching = set(self.rows_matching)
+            else:
+                self.rows_matching = set()
+                self.lowest_row_matching = None
+                self.db.search(text)
         except ParseException as e:
             self.searched.emit(e.msg)
             return
@@ -337,8 +358,9 @@ class BooksModel(QAbstractTableModel): # {{{
             name, val = mi.format_field(key)
             if mi.metadata_for_field(key)['datatype'] == 'comments':
                 name += ':html'
-            if val:
+            if val and name not in data:
                 data[name] = val
+
         return data
 
 
@@ -651,6 +673,9 @@ class BooksModel(QAbstractTableModel): # {{{
             return NONE
         if role in (Qt.DisplayRole, Qt.EditRole):
             return self.column_to_dc_map[col](index.row())
+        elif role == Qt.BackgroundColorRole:
+            if self.id(index) in self.rows_matching:
+                return QColor('lightgreen')
         elif role == Qt.DecorationRole:
             if self.column_to_dc_decorator_map[col] is not None:
                 return self.column_to_dc_decorator_map[index.column()](index.row())
diff --git a/src/calibre/gui2/library/views.py b/src/calibre/gui2/library/views.py
index e1e9cf4456..ea2e03fdad 100644
--- a/src/calibre/gui2/library/views.py
+++ b/src/calibre/gui2/library/views.py
@@ -680,8 +680,14 @@ class BooksView(QTableView): # {{{
     def set_editable(self, editable, supports_backloading):
         self._model.set_editable(editable)
 
+    def search_proxy(self, txt):
+        self._model.search(txt)
+        if self._model.lowest_row_matching is not None:
+            self.select_rows([self._model.lowest_row_matching], using_ids=False)
+        self.setFocus(Qt.OtherFocusReason)
+
     def connect_to_search_box(self, sb, search_done):
-        sb.search.connect(self._model.search)
+        sb.search.connect(self.search_proxy)
         self._search_done = search_done
         self._model.searched.connect(self.search_done)
 
diff --git a/src/calibre/gui2/preferences/plugins.py b/src/calibre/gui2/preferences/plugins.py
index 2fe2b3bf01..c53c634ab4 100644
--- a/src/calibre/gui2/preferences/plugins.py
+++ b/src/calibre/gui2/preferences/plugins.py
@@ -15,7 +15,8 @@ from calibre.gui2.preferences.plugins_ui import Ui_Form
 from calibre.customize.ui import initialized_plugins, is_disabled, enable_plugin, \
                                  disable_plugin, plugin_customization, add_plugin, \
                                  remove_plugin
-from calibre.gui2 import NONE, error_dialog, info_dialog, choose_files
+from calibre.gui2 import NONE, error_dialog, info_dialog, choose_files, \
+        question_dialog
 
 class PluginModel(QAbstractItemModel): # {{{
 
@@ -76,6 +77,16 @@ class PluginModel(QAbstractItemModel): # {{{
                     return self.index(j, 0, parent)
         return QModelIndex()
 
+    def plugin_to_index_by_properties(self, plugin):
+        for i, category in enumerate(self.categories):
+            parent = self.index(i, 0, QModelIndex())
+            for j, p in enumerate(self._data[category]):
+                if plugin.name == p.name and plugin.type == p.type and \
+                        plugin.author == p.author and plugin.version == p.version:
+                    return self.index(j, 0, parent)
+        return QModelIndex()
+
+
     def refresh_plugin(self, plugin, rescan=False):
         if rescan:
             self.populate()
@@ -132,7 +143,6 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         self.toggle_plugin_button.clicked.connect(self.toggle_plugin)
         self.customize_plugin_button.clicked.connect(self.customize_plugin)
         self.remove_plugin_button.clicked.connect(self.remove_plugin)
-        self.button_plugin_browse.clicked.connect(self.find_plugin)
         self.button_plugin_add.clicked.connect(self.add_plugin)
 
     def toggle_plugin(self, *args):
@@ -149,23 +159,39 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         self.modify_plugin(op='remove')
 
     def add_plugin(self):
-        path = unicode(self.plugin_path.text())
-        if path and os.access(path, os.R_OK) and path.lower().endswith('.zip'):
-            add_plugin(path)
+        path = choose_files(self, 'add a plugin dialog', _('Add plugin'),
+                filters=[(_('Plugins'), ['zip'])], all_files=False,
+                    select_only_single_file=True)
+        if not path:
+            return
+        path = path[0]
+        if path and  os.access(path, os.R_OK) and path.lower().endswith('.zip'):
+            if not question_dialog(self, _('Are you sure?'), '
    ' + \
+                    _('Installing plugins is a security risk. '
+                    'Plugins can contain a virus/malware. '
+                        'Only install it if you got it from a trusted source.'
+                        ' Are you sure you want to proceed?'),
+                    show_copy_button=False):
+                return
+            plugin = add_plugin(path)
             self._plugin_model.populate()
             self._plugin_model.reset()
             self.changed_signal.emit()
-            self.plugin_path.setText('')
+            info_dialog(self, _('Success'),
+                    _('Plugin {0} successfully installed under '
+                        ' {1} plugins. You may have to restart calibre '
+                        'for the plugin to take effect.').format(plugin.name, plugin.type),
+                    show=True, show_copy_button=False)
+            idx = self._plugin_model.plugin_to_index_by_properties(plugin)
+            if idx.isValid():
+                self.plugin_view.scrollTo(idx,
+                        self.plugin_view.PositionAtCenter)
+                self.plugin_view.scrollTo(idx,
+                        self.plugin_view.PositionAtCenter)
         else:
             error_dialog(self, _('No valid plugin path'),
                          _('%s is not a valid plugin path')%path).exec_()
 
-    def find_plugin(self):
-        path = choose_files(self, 'choose plugin dialog', _('Choose plugin'),
-                            filters=[('Plugins', ['zip'])], all_files=False,
-                            select_only_single_file=True)
-        if path:
-            self.plugin_path.setText(path[0])
 
     def modify_plugin(self, op=''):
         index = self.plugin_view.currentIndex()
@@ -191,10 +217,13 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
                 if plugin.do_user_config():
                     self._plugin_model.refresh_plugin(plugin)
             elif op == 'remove':
+                msg = _('Plugin {0} successfully removed').format(plugin.name)
                 if remove_plugin(plugin):
                     self._plugin_model.populate()
                     self._plugin_model.reset()
                     self.changed_signal.emit()
+                    info_dialog(self, _('Success'), msg, show=True,
+                            show_copy_button=False)
                 else:
                     error_dialog(self, _('Cannot remove builtin plugin'),
                          plugin.name + _(' cannot be removed. It is a '
diff --git a/src/calibre/gui2/preferences/plugins.ui b/src/calibre/gui2/preferences/plugins.ui
index 8979867bbc..83a904eb08 100644
--- a/src/calibre/gui2/preferences/plugins.ui
+++ b/src/calibre/gui2/preferences/plugins.ui
@@ -72,64 +72,14 @@
     
    
    
-    
-     
-      Add new plugin
+    
+     
+      &Add a new plugin
+     
+     
+      
+       :/images/plugins.png:/images/plugins.png
      
-     
-      
-       
-        
-         
-          
-           Plugin &file:
-          
-          
-           plugin_path
-          
-         
-        
-        
-         
-        
-        
-         
-          
-           ...
-          
-          
-           
-            :/images/document_open.png:/images/document_open.png
-          
-         
-        
-       
-      
-      
-       
-        
-         
-          
-           Qt::Horizontal
-          
-          
-           
-            40
-            20
-           
-          
-         
-        
-        
-         
-          
-           &Add
-          
-         
-        
-       
-      
-     
     
    
   
diff --git a/src/calibre/gui2/preferences/toolbar.py b/src/calibre/gui2/preferences/toolbar.py
index c13d956aea..26cdea19d3 100644
--- a/src/calibre/gui2/preferences/toolbar.py
+++ b/src/calibre/gui2/preferences/toolbar.py
@@ -37,7 +37,10 @@ class BaseModel(QAbstractListModel):
                     dont_remove_from=set(['toolbar-device']))
         if name is None:
             return FakeAction('--- '+_('Separator')+' ---', None)
-        return gui.iactions[name]
+        try:
+            return gui.iactions[name]
+        except:
+            return None
 
     def rowCount(self, parent):
         return len(self._data)
@@ -124,7 +127,8 @@ class CurrentModel(BaseModel):
         BaseModel.__init__(self)
         self.gprefs_name = 'action-layout-'+key
         current = gprefs[self.gprefs_name]
-        self._data =  [self.name_to_action(x, gui) for x in current]
+        self._data = [self.name_to_action(x, gui) for x in current]
+        self._data = [x for x in self._data if x is not None]
         self.key = key
         self.gui = gui
 
diff --git a/src/calibre/gui2/search_box.py b/src/calibre/gui2/search_box.py
index 9f74abfc86..e4073a01c9 100644
--- a/src/calibre/gui2/search_box.py
+++ b/src/calibre/gui2/search_box.py
@@ -16,6 +16,7 @@ from calibre.gui2 import config
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2.dialogs.saved_search_editor import SavedSearchEditor
 from calibre.gui2.dialogs.search import SearchDialog
+from calibre.utils.config import dynamic
 from calibre.utils.search_query_parser import saved_searches
 from calibre.utils.icu import sort_key
 
@@ -375,6 +376,9 @@ class SearchBoxMixin(object): # {{{
             unicode(self.search.toolTip())))
         self.advanced_search_button.setStatusTip(self.advanced_search_button.toolTip())
         self.clear_button.setStatusTip(self.clear_button.toolTip())
+        self.search_highlight_only.stateChanged.connect(self.highlight_only_changed)
+        self.search_highlight_only.setChecked(
+                            dynamic.get('search_highlight_only', False))
 
     def focus_search_box(self, *args):
         self.search.setFocus(Qt.OtherFocusReason)
@@ -401,6 +405,11 @@ class SearchBoxMixin(object): # {{{
     def focus_to_library(self):
         self.current_view().setFocus(Qt.OtherFocusReason)
 
+    def highlight_only_changed(self, toWhat):
+        dynamic.set('search_highlight_only', toWhat)
+        self.current_view().model().set_highlight_only(toWhat)
+        self.focus_to_library()
+
     # }}}
 
 class SavedSearchBoxMixin(object): # {{{
diff --git a/src/calibre/gui2/shortcuts.py b/src/calibre/gui2/shortcuts.py
index bdd699a69d..5e56435e10 100644
--- a/src/calibre/gui2/shortcuts.py
+++ b/src/calibre/gui2/shortcuts.py
@@ -150,7 +150,7 @@ class Delegate(QStyledItemDelegate):
         custom = []
         if editor.custom.isChecked():
             for x in ('1', '2'):
-                sc = getattr(editor, 'shortcut'+x)
+                sc = getattr(editor, 'shortcut'+x, None)
                 if sc is not None:
                     custom.append(sc)
 
@@ -266,6 +266,11 @@ class ShortcutConfig(QWidget):
         self.view.scrollTo(index)
 
 
+    @property
+    def is_editing(self):
+        return self.view.state() == self.view.EditingState
+
+
 if __name__ == '__main__':
     from calibre.gui2 import is_ok_to_use_qt
     from calibre.gui2.viewer.keys import SHORTCUTS
diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py
index 7e22839bdf..01d3180778 100644
--- a/src/calibre/gui2/ui.py
+++ b/src/calibre/gui2/ui.py
@@ -19,7 +19,7 @@ from PyQt4.Qt import Qt, SIGNAL, QTimer, \
                      QMessageBox, QHelpEvent
 
 from calibre import  prints
-from calibre.constants import __appname__, isosx, DEBUG
+from calibre.constants import __appname__, isosx
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.utils.config import prefs, dynamic
 from calibre.utils.ipc.server import Server
@@ -103,7 +103,15 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
         self.gui_debug = gui_debug
         acmap = OrderedDict()
         for action in interface_actions():
-            ac = action.load_actual_plugin(self)
+            try:
+                ac = action.load_actual_plugin(self)
+            except:
+                # Ignore errors in loading user supplied plugins
+                import traceback
+                traceback.print_exc()
+                if ac.plugin_path is None:
+                    raise
+
             ac.plugin_path = action.plugin_path
             ac.interface_action_base_plugin = action
             if ac.name in acmap:
@@ -460,12 +468,8 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
         try:
             if 'calibre.ebooks.DRMError' in job.details:
                 if not minz:
-                    d = error_dialog(self, _('Conversion Error'),
-                        _('
    Could not convert: %s
    It is a '
-                        'DRMed book. You must first remove the '
-                        'DRM using third party tools.')%\
-                            (job.description.split(':')[-1],
-                                'http://bugs.calibre-ebook.com/wiki/DRM'))
+                    from calibre.gui2.dialogs.drm_error import DRMErrorMessage
+                    d = DRMErrorMessage(self, job.description.split(':')[-1])
                     d.setModal(False)
                     d.show()
                     self._modeless_dialogs.append(d)
@@ -582,9 +586,6 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
             # Goes here, because if cf is valid, db is valid.
             db.prefs['field_metadata'] = db.field_metadata.all_metadata()
             db.commit_dirty_cache()
-            if DEBUG and db.gm_count > 0:
-                print 'get_metadata cache: {0:d} calls, {1:4.2f}% misses'.format(
-                        db.gm_count, (db.gm_missed*100.0)/db.gm_count)
         for action in self.iactions.values():
             if not action.shutting_down():
                 return
diff --git a/src/calibre/gui2/viewer/documentview.py b/src/calibre/gui2/viewer/documentview.py
index 13469f5622..4485e63373 100644
--- a/src/calibre/gui2/viewer/documentview.py
+++ b/src/calibre/gui2/viewer/documentview.py
@@ -120,6 +120,13 @@ class ConfigDialog(QDialog, Ui_Dialog):
 
 
     def accept(self, *args):
+        if self.shortcut_config.is_editing:
+            from calibre.gui2 import info_dialog
+            info_dialog(self, _('Still editing'),
+                    _('You are in the middle of editing a keyboard shortcut'
+                        ' first complete that, by clicking outside the '
+                        ' shortcut editing box.'), show=True)
+            return
         c = config()
         c.set('serif_family', unicode(self.serif_family.currentFont().family()))
         c.set('sans_family', unicode(self.sans_family.currentFont().family()))
@@ -279,7 +286,7 @@ class Document(QWebPage): # {{{
 
     @pyqtSignature("")
     def init_hyphenate(self):
-        if self.hyphenate:
+        if self.hyphenate and getattr(self, 'loaded_lang', ''):
             self.javascript('do_hyphenation("%s")'%self.loaded_lang)
 
     def after_load(self):
diff --git a/src/calibre/gui2/viewer/main.py b/src/calibre/gui2/viewer/main.py
index 25f69b1558..c5001659a0 100644
--- a/src/calibre/gui2/viewer/main.py
+++ b/src/calibre/gui2/viewer/main.py
@@ -26,6 +26,7 @@ from calibre.gui2.search_box import SearchBox2
 from calibre.ebooks.metadata import MetaInformation
 from calibre.customize.ui import available_input_formats
 from calibre.gui2.viewer.dictionary import Lookup
+from calibre import as_unicode
 
 class TOCItem(QStandardItem):
 
@@ -626,13 +627,12 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
             QApplication.processEvents()
         if worker.exception is not None:
             if isinstance(worker.exception, DRMError):
-                error_dialog(self, _('DRM Error'),
-                        _('
    This book is protected by DRM')
-                        %'http://wiki.mobileread.com/wiki/DRM').exec_()
+                from calibre.gui2.dialogs.drm_error import DRMErrorMessage
+                DRMErrorMessage(self).exec_()
             else:
                 r = getattr(worker.exception, 'reason', worker.exception)
                 error_dialog(self, _('Could not open ebook'),
-                        unicode(r), det_msg=worker.traceback, show=True)
+                        as_unicode(r), det_msg=worker.traceback, show=True)
             self.close_progress_indicator()
         else:
             self.metadata.show_opf(self.iterator.opf, os.path.splitext(pathtoebook)[1][1:])
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index 0763318912..4168360d3a 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -411,7 +411,8 @@ class ResultCache(SearchQueryParser): # {{{
             if isinstance(location, list):
                 if allow_recursion:
                     for loc in location:
-                        matches |= self.get_matches(loc, query, allow_recursion=False)
+                        matches |= self.get_matches(loc, query, candidates,
+                                                    allow_recursion=False)
                     return matches
                 raise ParseException(query, len(query), 'Recursive query group detected', self)
 
@@ -419,11 +420,11 @@ class ResultCache(SearchQueryParser): # {{{
                 fm = self.field_metadata[location]
                 # take care of dates special case
                 if fm['datatype'] == 'datetime':
-                    return self.get_dates_matches(location, query.lower())
+                    return self.get_dates_matches(location, query.lower(), candidates)
 
                 # take care of numbers special case
                 if fm['datatype'] in ('rating', 'int', 'float'):
-                    return self.get_numeric_matches(location, query.lower())
+                    return self.get_numeric_matches(location, query.lower(), candidates)
 
                 # take care of the 'count' operator for is_multiples
                 if fm['is_multiple'] and \
@@ -431,7 +432,8 @@ class ResultCache(SearchQueryParser): # {{{
                         query[1:1] in '=<>!':
                     vf = lambda item, loc=fm['rec_index'], ms=fm['is_multiple']:\
                             len(item[loc].split(ms)) if item[loc] is not None else 0
-                    return self.get_numeric_matches(location, query[1:], val_func=vf)
+                    return self.get_numeric_matches(location, query[1:],
+                                                    candidates, val_func=vf)
 
             # everything else, or 'all' matches
             matchkind = CONTAINS_MATCH
@@ -598,7 +600,6 @@ class ResultCache(SearchQueryParser): # {{{
 
     def set(self, row, col, val, row_is_id=False):
         id = row if row_is_id else self._map_filtered[row]
-        self._data[id][self.FIELD_MAP['all_metadata']] = None
         self._data[id][col] = val
 
     def get(self, row, col, row_is_id=False):
@@ -629,7 +630,6 @@ class ResultCache(SearchQueryParser): # {{{
                 self._data[id] = CacheRow(db, self.composites,
                         db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0])
                 self._data[id].append(db.book_on_device_string(id))
-                self._data[id].append(None)
             except IndexError:
                 return None
         try:
@@ -646,7 +646,6 @@ class ResultCache(SearchQueryParser): # {{{
             self._data[id] = CacheRow(db, self.composites,
                         db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0])
             self._data[id].append(db.book_on_device_string(id))
-            self._data[id].append(None)
         self._map[0:0] = ids
         self._map_filtered[0:0] = ids
 
@@ -671,7 +670,6 @@ class ResultCache(SearchQueryParser): # {{{
         for item in self._data:
             if item is not None:
                 item.append(db.book_on_device_string(item[0]))
-                item.append(None)
         self._map = [i[0] for i in self._data if i is not None]
         if field is not None:
             self.sort(field, ascending)
diff --git a/src/calibre/library/catalog.py b/src/calibre/library/catalog.py
index 349800c8ba..5cda9baa8c 100644
--- a/src/calibre/library/catalog.py
+++ b/src/calibre/library/catalog.py
@@ -1524,19 +1524,32 @@ class EPUB_MOBI(CatalogPlugin):
                     this_title['formats'] = formats
 
                 # Add user notes to be displayed in header
-                # Special case handling for datetime fields
+                # Special case handling for datetime fields and lists
                 if self.opts.header_note_source_field:
                     field_md = self.__db.metadata_for_field(self.opts.header_note_source_field)
                     notes = self.__db.get_field(record['id'],
                                         self.opts.header_note_source_field,
                                         index_is_id=True)
-                    if notes and field_md['datatype'] == 'datetime':
-                        # Reformat date fields to match UI presentation: dd MMM YYYY
-                        notes = format_date(notes,'dd MMM yyyy')
-
                     if notes:
+                        if field_md['datatype'] == 'text':
+                            if isinstance(notes,list):
+                                notes = ' · '.join(notes)
+                        elif field_md['datatype'] == 'datetime':
+                            notes = format_date(notes,'dd MMM yyyy')
+                        elif field_md['datatype'] == 'composite':
+                            m = re.match(r'\[(.+)\]$', notes)
+                            if m is not None:
+                                # Sniff for special pseudo-list string "[]"
+                                bracketed_content = m.group(1)
+                                if ',' in bracketed_content:
+                                    # Recast the comma-separated items as a list
+                                    items = bracketed_content.split(',')
+                                    items = [i.strip() for i in items]
+                                    notes = ' · '.join(items)
+                                else:
+                                    notes = bracketed_content
                         this_title['notes'] = {'source':field_md['name'],
-                                               'content':notes}
+                                                   'content':notes}
 
                 titles.append(this_title)
 
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 138560020e..5f66297322 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -298,10 +298,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                             base,
                             prefer_custom=True)
 
-        self.FIELD_MAP['ondevice'] = base+1
-        self.field_metadata.set_field_record_index('ondevice', base+1, prefer_custom=False)
-        self.FIELD_MAP['all_metadata'] = base+2
-        self.field_metadata.set_field_record_index('all_metadata', base+2, prefer_custom=False)
+        self.FIELD_MAP['ondevice'] = base = base+1
+        self.field_metadata.set_field_record_index('ondevice', base, prefer_custom=False)
 
         script = '''
         DROP VIEW IF EXISTS meta2;
@@ -343,10 +341,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         self.has_id  = self.data.has_id
         self.count   = self.data.count
 
-        # Count times get_metadata is called, and how many times in the cache
-        self.gm_count  = 0
-        self.gm_missed = 0
-
         for prop in ('author_sort', 'authors', 'comment', 'comments', 'isbn',
                      'publisher', 'rating', 'series', 'series_index', 'tags',
                      'title', 'timestamp', 'uuid', 'pubdate', 'ondevice'):
@@ -690,19 +684,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         '''
         row = self.data._data[idx] if index_is_id else self.data[idx]
         fm = self.FIELD_MAP
-
-        self.gm_count += 1
-        mi = row[self.FIELD_MAP['all_metadata']]
-        if mi is not None:
-            if get_cover:
-                # Always get the cover, because the value can be wrong if the
-                # original mi was from the OPF
-                mi.cover = self.cover(idx, index_is_id=index_is_id, as_path=True)
-            return mi
-
-        self.gm_missed += 1
         mi = Metadata(None)
-        self.data.set(idx, fm['all_metadata'], mi, row_is_id = index_is_id)
 
         aut_list = row[fm['au_map']]
         aut_list = [p.split(':::') for p in aut_list.split(':#:')]
@@ -724,6 +706,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         formats = row[fm['formats']]
         if not formats:
             formats = None
+        else:
+            formats = formats.split(',')
         mi.formats = formats
         tags = row[fm['tags']]
         if tags:
@@ -1387,7 +1371,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             if r is not None:
                 if (now - r[self.FIELD_MAP['timestamp']]) > delta:
                     tags = r[self.FIELD_MAP['tags']]
-                    if tags and tag in tags.lower():
+                    if tags and tag in [x.strip() for x in
+                            tags.lower().split(',')]:
                         yield r[self.FIELD_MAP['id']]
 
     def get_next_series_num_for(self, series):
diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py
index 676eb13d2b..2a9b7e7003 100644
--- a/src/calibre/library/field_metadata.py
+++ b/src/calibre/library/field_metadata.py
@@ -162,15 +162,6 @@ class FieldMetadata(dict):
                            'search_terms':['tags', 'tag'],
                            'is_custom':False,
                            'is_category':True}),
-            ('all_metadata',{'table':None,
-                             'column':None,
-                             'datatype':None,
-                             'is_multiple':None,
-                             'kind':'field',
-                             'name':None,
-                             'search_terms':[],
-                             'is_custom':False,
-                             'is_category':False}),
             ('author_sort',{'table':None,
                             'column':None,
                             'datatype':'text',
diff --git a/src/calibre/trac/bzr_commit_plugin.py b/src/calibre/trac/bzr_commit_plugin.py
index df6bf699d1..6c36115cae 100644
--- a/src/calibre/trac/bzr_commit_plugin.py
+++ b/src/calibre/trac/bzr_commit_plugin.py
@@ -110,6 +110,7 @@ class cmd_commit(_cmd_commit):
             suffix = 'The fix will be in the next release.'
         action = action+'ed'
         msg = '%s in branch %s. %s'%(action, nick, suffix)
+        msg = msg.replace('Fixesed', 'Fixed')
         server = xmlrpclib.ServerProxy(url)
         server.ticket.update(int(bug), msg,
                              {'status':'closed', 'resolution':'fixed'},
diff --git a/src/calibre/utils/cleantext.py b/src/calibre/utils/cleantext.py
index b4afe7576d..938960df93 100644
--- a/src/calibre/utils/cleantext.py
+++ b/src/calibre/utils/cleantext.py
@@ -3,7 +3,7 @@ __license__ = 'GPL 3'
 __copyright__ = '2010, sengian '
 __docformat__ = 'restructuredtext en'
 
-import re
+import re, htmlentitydefs
 
 _ascii_pat = None
 
@@ -21,3 +21,32 @@ def clean_ascii_chars(txt, charlist=None):
         pat = re.compile(u'|'.join(map(unichr, charlist)))
     return pat.sub('', txt)
 
+##
+# Fredrik Lundh: http://effbot.org/zone/re-sub.htm#unescape-html
+# Removes HTML or XML character references and entities from a text string.
+#
+# @param text The HTML (or XML) source text.
+# @return The plain text, as a Unicode string, if necessary.
+
+def unescape(text, rm=False, rchar=u''):
+    def fixup(m, rm=rm, rchar=rchar):
+        text = m.group(0)
+        if text[:2] == "&#":
+            # character reference
+            try:
+                if text[:3] == "&#x":
+                    return unichr(int(text[3:-1], 16))
+                else:
+                    return unichr(int(text[2:-1]))
+            except ValueError:
+                pass
+        else:
+            # named entity
+            try:
+                text = unichr(htmlentitydefs.name2codepoint[text[1:-1]])
+            except KeyError:
+                pass
+        if rm:
+            return rchar #replace by char
+        return text # leave as is
+    return re.sub("&#?\w+;", fixup, text)
diff --git a/src/calibre/utils/formatter.py b/src/calibre/utils/formatter.py
index 40760bf91b..0b5f1d1f52 100644
--- a/src/calibre/utils/formatter.py
+++ b/src/calibre/utils/formatter.py
@@ -18,6 +18,24 @@ class _Parser(object):
     LEX_NUM = 4
     LEX_EOF = 5
 
+    def _python(self, func):
+        locals = {}
+        exec func in locals
+        if 'evaluate' not in locals:
+            self.error('no evaluate function in python')
+        try:
+            result = locals['evaluate'](self.parent.kwargs)
+            if isinstance(result, (float, int)):
+                result = unicode(result)
+            elif isinstance(result, list):
+                result = ','.join(result)
+            elif isinstance(result, str):
+                result = unicode(result)
+            return result
+        except Exception as e:
+            self.error('python function threw exception: ' + e.msg)
+
+
     def _strcmp(self, x, y, lt, eq, gt):
         v = strcmp(x, y)
         if v < 0:
@@ -79,6 +97,7 @@ class _Parser(object):
             'field'    : (1, lambda s, x: s.parent.get_value(x, [], s.parent.kwargs)),
             'multiply' : (2, partial(_math, op='*')),
             'print'    : (-1, _print),
+            'python'   : (1, _python),
             'strcat'   : (-1, _concat),
             'strcmp'   : (5, _strcmp),
             'substr'   : (3, lambda s, x, y, z: x[int(y): len(x) if int(z) == 0 else int(z)]),
@@ -362,7 +381,7 @@ class TemplateFormatter(string.Formatter):
                 (r'\'.*?((?wand, data, dlen);
+
+    if (!res)
+        return magick_set_exception(self->wand);
+
+    Py_RETURN_NONE;
+}
+
+// }}}
+
 // Image.open {{{
 static PyObject *
 magick_Image_read(magick_Image *self, PyObject *args, PyObject *kwargs) {
@@ -993,6 +1013,10 @@ static PyMethodDef magick_Image_methods[] = {
     {"destroy", (PyCFunction)magick_Image_destroy, METH_VARARGS,
     "Destroy the underlying ImageMagick Wand. WARNING: After using this method, all methods on this object will raise an exception."},
 
+    {"identify", (PyCFunction)magick_Image_identify, METH_VARARGS,
+     "Identify an image from a byte buffer (string)"
+    },
+
     {"load", (PyCFunction)magick_Image_load, METH_VARARGS,
      "Load an image from a byte buffer (string)"
     },
diff --git a/src/calibre/utils/wmf/__init__.py b/src/calibre/utils/wmf/__init__.py
index 68dfb8d2b5..cb7736e06a 100644
--- a/src/calibre/utils/wmf/__init__.py
+++ b/src/calibre/utils/wmf/__init__.py
@@ -5,5 +5,52 @@ __license__   = 'GPL v3'
 __copyright__ = '2011, Kovid Goyal '
 __docformat__ = 'restructuredtext en'
 
+import glob
+
+from calibre.constants import plugins, iswindows, filesystem_encoding
+from calibre.ptempfile import TemporaryDirectory
+from calibre import CurrentDir
+from calibre.utils.magick import Image, PixelWand
+
+class Unavailable(Exception):
+    pass
+
+class NoRaster(Exception):
+    pass
+
+def extract_raster_image(wmf_data):
+    try:
+        wmf, wmf_err = plugins['wmf']
+    except KeyError:
+        raise Unavailable('libwmf not available on this platform')
+    if wmf_err:
+        raise Unavailable(wmf_err)
+
+    if iswindows:
+        import sys, os
+        appdir = sys.app_dir
+        if isinstance(appdir, unicode):
+            appdir = appdir.encode(filesystem_encoding)
+        fdir = os.path.join(appdir, 'wmffonts')
+        wmf.set_font_dir(fdir)
+
+    data = ''
+
+    with TemporaryDirectory('wmf2png') as tdir:
+        with CurrentDir(tdir):
+            wmf.render(wmf_data)
+
+            images = list(sorted(glob.glob('*.png')))
+            if not images:
+                raise NoRaster('No raster images in WMF')
+            data = open(images[0], 'rb').read()
+
+    im = Image()
+    im.load(data)
+    pw = PixelWand()
+    pw.color = '#ffffff'
+    im.rotate(pw, 180)
+
+    return im.export('png')
 
 
diff --git a/src/calibre/utils/wmf/parse.py b/src/calibre/utils/wmf/parse.py
new file mode 100644
index 0000000000..c618884e33
--- /dev/null
+++ b/src/calibre/utils/wmf/parse.py
@@ -0,0 +1,269 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal '
+__docformat__ = 'restructuredtext en'
+
+import sys, struct
+
+
+
+class WMFHeader(object):
+
+    '''
+    For header documentation, see
+    http://www.skynet.ie/~caolan/publink/libwmf/libwmf/doc/ora-wmf.html
+    '''
+
+    def __init__(self, data, log, verbose):
+        self.log, self.verbose = log, verbose
+        offset = 0
+        file_type, header_size, windows_version = struct.unpack_from(' 0:
+                params = data[offset:offset+delta]
+                offset += delta
+
+            func = self.function_map.get(func, func)
+
+            if self.verbose > 3:
+                self.log.debug('WMF Record:', size, func)
+            self.records.append((func, params))
+
+        for rec in self.records:
+            f = getattr(self, rec[0], None)
+            if callable(f):
+                f(rec[1])
+            elif self.verbose > 2:
+                self.log.debug('Ignoring record:', rec[0])
+
+        self.has_raster_image = len(self.bitmaps) > 0
+
+
+    def SetMapMode(self, params):
+        if len(params) == 2:
+            self.map_mode = struct.unpack('
 #include 
+//#include 
 
 typedef struct {
     char *data;
@@ -13,7 +14,7 @@ typedef struct {
 
 //This code is taken mostly from the Abiword wmf plugin
 
-
+// Buffer read {{{
 // returns unsigned char cast to int, or EOF
 static int wmf_WMF_read(void * context) {
     char c;
@@ -22,11 +23,11 @@ static int wmf_WMF_read(void * context) {
 	if (info->pos == info->len)
 		return EOF;
 
-	c = info->data[pos];
+	c = info->data[info->pos];
 
 	info->pos++;
 
-	return (int)c;
+	return (int)((unsigned char)c);
 }
 
 // returns (-1) on error, else 0
@@ -44,8 +45,17 @@ static long wmf_WMF_tell(void * context) {
 
 	return (long) info->pos;
 }
+// }}}
 
 
+char _png_name_buf[100];
+char *wmf_png_name(void *ctxt) {
+    int *num = (int*)ctxt;
+    *num = *num + 1;
+    snprintf(_png_name_buf, 90, "%04d.png", *num);
+    return _png_name_buf;
+}
+
 #define CLEANUP if(API) { if (stream) wmf_free(API, stream); wmf_api_destroy(API); };
 
 static PyObject *
@@ -66,9 +76,9 @@ wmf_render(PyObject *self, PyObject *args) {
 
 	unsigned int max_width  = 1600;
 	unsigned int max_height = 1200;
-	unsigned long max_flags = 0;
 
 	static const char* Default_Description = "wmf2svg";
+    int fname_counter = 0;
 
 	wmf_error_t err;
 
@@ -125,6 +135,8 @@ wmf_render(PyObject *self, PyObject *args) {
 	ddata->Description = (char *)Default_Description;
 
 	ddata->bbox = bbox;
+    ddata->image.context = (void *)&fname_counter;
+    ddata->image.name = wmf_png_name;
 
 	wmf_display_size(API, &disp_width, &disp_height, 96, 96);
 
@@ -156,9 +168,9 @@ wmf_render(PyObject *self, PyObject *args) {
 		ddata->height = (unsigned int) ceil ((double) wmf_height);
 	}
 
-	ddata->flags |= WMF_SVG_INLINE_IMAGES;
-
-	ddata->flags |= WMF_GD_OUTPUT_MEMORY | WMF_GD_OWN_BUFFER;
+    // Needs GD
+	//ddata->flags |= WMF_SVG_INLINE_IMAGES;
+	//ddata->flags |= WMF_GD_OUTPUT_MEMORY | WMF_GD_OWN_BUFFER;
 
     err = wmf_play(API, 0, &(bbox));
 
@@ -178,11 +190,32 @@ wmf_render(PyObject *self, PyObject *args) {
     return ans;
 }
 
+#ifdef _WIN32
+void set_libwmf_fontdir(const char *);
+
+static PyObject *
+wmf_setfontdir(PyObject *self, PyObject *args) {
+    char *path;
+    if (!PyArg_ParseTuple(args, "s", &path))
+        return NULL;
+    set_libwmf_fontdir(path);
+
+    Py_RETURN_NONE;
+}
+#endif
+
+
+
 
 static PyMethodDef wmf_methods[] = {
     {"render", wmf_render, METH_VARARGS,
-        "render(path) -> Render wmf as svg."
+        "render(data) -> Render wmf as svg."
     },
+#ifdef _WIN32
+    {"set_font_dir", wmf_setfontdir, METH_VARARGS,
+        "set_font_dir(path) -> Set the path to the fonts dir on windows, must be called at least once before using render()"
+    },
+#endif
 
     {NULL}  /* Sentinel */
 };
diff --git a/src/calibre/utils/zipfile.py b/src/calibre/utils/zipfile.py
index ff290abd25..c230b9dfa7 100644
--- a/src/calibre/utils/zipfile.py
+++ b/src/calibre/utils/zipfile.py
@@ -982,9 +982,12 @@ class ZipFile:
             zef_file.read(fheader[_FH_EXTRA_FIELD_LENGTH])
 
         if fname != zinfo.orig_filename:
-            raise BadZipfile, \
-                      'File name in directory "%s" and header "%s" differ.' % (
-                          zinfo.orig_filename, fname)
+            print ('WARNING: Header (%r) and directory (%r) filenames do not'
+                    ' match inside ZipFile')%(fname, zinfo.orig_filename)
+            print 'Using directory filename %r'%zinfo.orig_filename
+            #raise BadZipfile, \
+            #          'File name in directory "%r" and header "%r" differ.' % (
+            #              zinfo.orig_filename, fname)
 
         # check for encrypted flag & handle password
         is_encrypted = zinfo.flag_bits & 0x1
diff --git a/src/calibre/web/feeds/news.py b/src/calibre/web/feeds/news.py
index 7bd5301dfb..ee5b11c5f6 100644
--- a/src/calibre/web/feeds/news.py
+++ b/src/calibre/web/feeds/news.py
@@ -700,10 +700,17 @@ class BasicNewsRecipe(Recipe):
         for attr in self.remove_attributes:
             for x in soup.findAll(attrs={attr:True}):
                 del x[attr]
-        for base in list(soup.findAll(['base', 'iframe'])):
+        for base in list(soup.findAll(['base', 'iframe', 'canvas', 'embed',
+            'command', 'datalist', 'video', 'audio'])):
             base.extract()
 
         ans = self.postprocess_html(soup, first_fetch)
+
+        # Nuke HTML5 tags
+        for x in ans.findAll(['article', 'aside', 'header', 'footer', 'nav',
+            'figcaption', 'figure', 'section']):
+            x.name = 'div'
+
         if job_info:
             url, f, a, feed_len = job_info
             try:
diff --git a/src/calibre/web/feeds/recipes/collection.py b/src/calibre/web/feeds/recipes/collection.py
index a513cf3880..5dd360213b 100644
--- a/src/calibre/web/feeds/recipes/collection.py
+++ b/src/calibre/web/feeds/recipes/collection.py
@@ -108,7 +108,6 @@ def download_builtin_recipe(urn):
     br = browser()
     return br.open_novisit('http://status.calibre-ebook.com/recipe/'+urn).read()
 
-
 def get_builtin_recipe_by_title(title, log=None, download_recipe=False):
     for x in get_builtin_recipe_collection():
         if x.get('title') == title:
@@ -127,6 +126,24 @@ def get_builtin_recipe_by_title(title, log=None, download_recipe=False):
                         'Failed to download recipe, using builtin version')
             return P('recipes/%s.recipe'%urn, data=True)
 
+def get_builtin_recipe_by_id(id_, log=None, download_recipe=False):
+    for x in get_builtin_recipe_collection():
+        if x.get('id') == id_:
+            urn = x.get('id')[8:]
+            if download_recipe:
+                try:
+                    if log is not None:
+                        log('Trying to get latest version of recipe:', urn)
+                    return download_builtin_recipe(urn)
+                except:
+                    if log is None:
+                        import traceback
+                        traceback.print_exc()
+                    else:
+                        log.exception(
+                        'Failed to download recipe, using builtin version')
+            return P('recipes/%s.recipe'%urn, data=True)
+
 class SchedulerConfig(object):
 
     def __init__(self):