Remove junk from redesigned wall street journal

This commit is contained in:
bobbysteel 2017-06-19 01:18:17 +01:00 committed by GitHub
parent 30f3edb45a
commit f2f1f7c6b5

View File

@ -13,7 +13,6 @@ from lxml import html
from calibre.web.feeds.news import BasicNewsRecipe
def CSSSelect(expr):
expr = {
'div.whatsNews-simple': '''descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' whatsNews-simple ')]''',
@ -67,6 +66,10 @@ class WSJ(BasicNewsRecipe):
]
remove_tags = [
dict(id='right-rail'),
dict(id='narrator-nav'),
dict(name='div', id='ad_and_popular'),
dict(classes('comments-count-container')),
classes(
'insetButton insettipBox author-info media-object-video article_tools nc-exp-artmeta category'),
dict(name='span', attrs={
@ -94,7 +97,22 @@ class WSJ(BasicNewsRecipe):
if found:
self.log.debug('Found %d dynamic images in:' % found)
return soup
def get_cover_url(self):
cover = 'https://vir.wsj.net/fp/cdn/fp/assets/images/WSJ_A1.jpg'
br = BasicNewsRecipe.get_browser(self)
try:
br.open(cover)
except:
index = 'http://en.kiosko.net/us/np/wsj.html'
soup = self.index_to_soup(index)
for image in soup.findAll('img', src=True):
if image['src'].endswith('750.jpg'):
return image['src']
self.log("\nCover unavailable")
cover = None
return cover
def get_browser(self):
# To understand the signin logic read signin.js from
# https://id.wsj.com/access/pages/wsj/us/signin.html