This commit is contained in:
Kovid Goyal 2014-03-09 10:16:21 +05:30
parent 5df9807308
commit b4b1c021f7

View File

@ -3,7 +3,15 @@ __copyright__ = '2010, Walt Anthony <workshop.northpole at gmail.com>'
'''
www.americanthinker.com
'''
import html5lib
from calibre.web.feeds.news import BasicNewsRecipe
from calibre.utils.cleantext import clean_xml_chars
from lxml import etree
def CSSSelect(expr):
from cssselect import HTMLTranslator
from lxml.etree import XPath
return XPath(HTMLTranslator().css_to_xpath(expr))
class AmericanThinker(BasicNewsRecipe):
title = u'American Thinker'
@ -18,7 +26,7 @@ class AmericanThinker(BasicNewsRecipe):
ignore_duplicate_articles = {'title', 'url'}
remove_javascript = True
no_stylesheets = True
remove_tags_before = dict(name='h1')
conversion_options = {
'comment' : description
@ -27,7 +35,14 @@ class AmericanThinker(BasicNewsRecipe):
, 'language' : language
, 'linearize_tables' : True
}
auto_claenup = True
def preprocess_raw_html(self, raw, url):
root = html5lib.parse(
clean_xml_chars(raw), treebuilder='lxml',
namespaceHTMLElements=False)
for x in CSSSelect('.article_body.bottom')(root):
x.getparent().remove(x)
return etree.tostring(root, encoding=unicode)
feeds = [(u'http://feeds.feedburner.com/americanthinker'),
(u'http://feeds.feedburner.com/AmericanThinkerBlog')