mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-06-23 15:30:45 -04:00
210 lines
8.8 KiB
Python
210 lines
8.8 KiB
Python
#!/usr/bin/env python
|
|
__license__ = 'GPL v3'
|
|
__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
|
|
__docformat__ = 'restructuredtext en'
|
|
|
|
from calibre.web.feeds.news import BasicNewsRecipe
|
|
import copy
|
|
|
|
# http://online.wsj.com/page/us_in_todays_paper.html
|
|
|
|
def filter_classes(x):
|
|
if not x:
|
|
return False
|
|
bad_classes = {'articleInsetPoll', 'trendingNow', 'sTools', 'printSummary', 'mostPopular', 'relatedCollection'}
|
|
classes = frozenset(x.split())
|
|
return len(bad_classes.intersection(classes)) > 0
|
|
|
|
class WallStreetJournal(BasicNewsRecipe):
|
|
|
|
title = 'The Wall Street Journal'
|
|
__author__ = 'Kovid Goyal, Sujata Raman, and Joshua Oster-Morris'
|
|
description = 'News and current affairs'
|
|
needs_subscription = True
|
|
language = 'en'
|
|
|
|
max_articles_per_feed = 1000
|
|
timefmt = ' [%a, %b %d, %Y]'
|
|
no_stylesheets = True
|
|
|
|
extra_css = '''h1{color:#093D72 ; font-size:large ; font-family:Georgia,"Century Schoolbook","Times New Roman",Times,serif; }
|
|
h2{color:#474537; font-family:Georgia,"Century Schoolbook","Times New Roman",Times,serif; font-size:small; font-style:italic;}
|
|
.subhead{color:gray; font-family:Georgia,"Century Schoolbook","Times New Roman",Times,serif; font-size:small; font-style:italic;}
|
|
.insettipUnit {color:#666666; font-family:Arial,Sans-serif;font-size:xx-small }
|
|
.targetCaption{ font-size:x-small; color:#333333; font-family:Arial,Helvetica,sans-serif}
|
|
.article{font-family :Arial,Helvetica,sans-serif; font-size:x-small}
|
|
.tagline {color:#333333; font-size:xx-small}
|
|
.dateStamp {color:#666666; font-family:Arial,Helvetica,sans-serif}
|
|
h3{color:blue ;font-family:Arial,Helvetica,sans-serif; font-size:xx-small}
|
|
.byline{color:blue;font-family:Arial,Helvetica,sans-serif; font-size:xx-small}
|
|
h6{color:#333333; font-family:Georgia,"Century Schoolbook","Times New Roman",Times,serif; font-size:small;font-style:italic; }
|
|
.paperLocation{color:#666666; font-size:xx-small}'''
|
|
|
|
remove_tags_before = dict(name='h1')
|
|
remove_tags = [
|
|
dict(id=["articleTabs_tab_article",
|
|
"articleTabs_tab_comments", 'msnLinkback', 'yahooLinkback',
|
|
'articleTabs_panel_comments', 'footer', 'emailThisScrim', 'emailConfScrim', 'emailErrorScrim',
|
|
"articleTabs_tab_interactive", "articleTabs_tab_video",
|
|
"articleTabs_tab_map", "articleTabs_tab_slideshow",
|
|
"articleTabs_tab_quotes", "articleTabs_tab_document",
|
|
"printModeAd", "aFbLikeAuth", "videoModule",
|
|
"mostRecommendations", "topDiscussions"]),
|
|
{'class':['footer_columns','hidden', 'network','insetCol3wide','interactive','video','slideshow','map','insettip',
|
|
'insetClose','more_in', "insetContent", 'articleTools_bottom', 'aTools', "tooltip", "adSummary", "nav-inline"]},
|
|
dict(rel='shortcut icon'),
|
|
{'class':filter_classes},
|
|
]
|
|
remove_tags_after = [dict(id="article_story_body"), {'class':"article story"},]
|
|
|
|
use_javascript_to_login = True
|
|
|
|
def javascript_login(self, br, username, password):
|
|
br.visit('https://id.wsj.com/access/pages/wsj/us/login_standalone.html?mg=com-wsj', timeout=120)
|
|
f = br.select_form(nr=0)
|
|
f['username'] = username
|
|
f['password'] = password
|
|
br.submit(timeout=120)
|
|
|
|
def populate_article_metadata(self, article, soup, first):
|
|
if first and hasattr(self, 'add_toc_thumbnail'):
|
|
picdiv = soup.find('img')
|
|
if picdiv is not None:
|
|
self.add_toc_thumbnail(article,picdiv['src'])
|
|
|
|
def postprocess_html(self, soup, first):
|
|
for tag in soup.findAll(name=['table', 'tr', 'td']):
|
|
tag.name = 'div'
|
|
|
|
for tag in soup.findAll('div', dict(id=[
|
|
"articleThumbnail_1", "articleThumbnail_2", "articleThumbnail_3",
|
|
"articleThumbnail_4", "articleThumbnail_5", "articleThumbnail_6",
|
|
"articleThumbnail_7"])):
|
|
tag.extract()
|
|
|
|
return soup
|
|
|
|
def wsj_get_index(self):
|
|
return self.index_to_soup('http://online.wsj.com/itp')
|
|
|
|
def wsj_add_feed(self,feeds,title,url):
|
|
self.log('Found section:', title)
|
|
try:
|
|
if url.endswith('whatsnews'):
|
|
articles = self.wsj_find_wn_articles(url)
|
|
else:
|
|
articles = self.wsj_find_articles(url)
|
|
except:
|
|
articles = []
|
|
if articles:
|
|
feeds.append((title, articles))
|
|
return feeds
|
|
|
|
def abs_wsj_url(self, href):
|
|
if not href.startswith('http'):
|
|
href = 'http://online.wsj.com' + href
|
|
return href
|
|
|
|
def parse_index(self):
|
|
soup = self.wsj_get_index()
|
|
|
|
date = soup.find('span', attrs={'class':'date-date'})
|
|
if date is not None:
|
|
self.timefmt = ' [%s]'%self.tag_to_string(date)
|
|
|
|
cov = soup.find('div', attrs={'class':lambda x: x and 'itpSectionHeaderPdf' in x.split()})
|
|
if cov is not None:
|
|
a = cov.find('a', href=True)
|
|
if a is not None:
|
|
self.cover_url = a['href']
|
|
|
|
feeds = []
|
|
div = soup.find('div', attrs={'class':'itpHeader'})
|
|
div = div.find('ul', attrs={'class':'tab'})
|
|
for a in div.findAll('a', href=lambda x: x and '/itp/' in x):
|
|
pageone = a['href'].endswith('pageone')
|
|
if pageone:
|
|
title = 'Front Section'
|
|
url = self.abs_wsj_url(a['href'])
|
|
feeds = self.wsj_add_feed(feeds,title,url)
|
|
title = "What's News"
|
|
url = url.replace('pageone','whatsnews')
|
|
feeds = self.wsj_add_feed(feeds,title,url)
|
|
else:
|
|
title = self.tag_to_string(a)
|
|
url = self.abs_wsj_url(a['href'])
|
|
feeds = self.wsj_add_feed(feeds,title,url)
|
|
return feeds
|
|
|
|
def wsj_find_wn_articles(self, url):
|
|
soup = self.index_to_soup(url)
|
|
articles = []
|
|
|
|
whats_news = soup.find('div', attrs={'class':lambda x: x and 'whatsNews-simple' in x})
|
|
if whats_news is not None:
|
|
for a in whats_news.findAll('a', href=lambda x: x and '/article/' in x):
|
|
container = a.findParent(['p'])
|
|
meta = a.find(attrs={'class':'meta_sectionName'})
|
|
if meta is not None:
|
|
meta.extract()
|
|
title = self.tag_to_string(a).strip()
|
|
url = a['href']
|
|
desc = ''
|
|
if container is not None:
|
|
desc = self.tag_to_string(container)
|
|
|
|
articles.append({'title':title, 'url':url,
|
|
'description':desc, 'date':''})
|
|
|
|
self.log('\tFound WN article:', title)
|
|
self.log('\t\t', desc)
|
|
|
|
return articles
|
|
|
|
def wsj_find_articles(self, url):
|
|
soup = self.index_to_soup(url)
|
|
|
|
whats_news = soup.find('div', attrs={'class':lambda x: x and 'whatsNews-simple' in x})
|
|
if whats_news is not None:
|
|
whats_news.extract()
|
|
|
|
articles = []
|
|
|
|
flavorarea = soup.find('div', attrs={'class':lambda x: x and 'ahed' in x})
|
|
if flavorarea is not None:
|
|
flavorstory = flavorarea.find('a', href=lambda x: x and x.startswith('/article'))
|
|
if flavorstory is not None:
|
|
flavorstory['class'] = 'mjLinkItem'
|
|
metapage = soup.find('span', attrs={'class':lambda x: x and 'meta_sectionName' in x})
|
|
if metapage is not None:
|
|
flavorstory.append(copy.copy(metapage)) # metapage should always be A1 because that should be first on the page
|
|
|
|
for a in soup.findAll('a', attrs={'class':'mjLinkItem'}, href=True):
|
|
container = a.findParent(['li', 'div'])
|
|
meta = a.find(attrs={'class':'meta_sectionName'})
|
|
if meta is not None:
|
|
meta.extract()
|
|
meta = self.tag_to_string(meta).strip()
|
|
if meta:
|
|
title = self.tag_to_string(a).strip() + ' [%s]'%meta
|
|
else:
|
|
title = self.tag_to_string(a).strip()
|
|
url = self.abs_wsj_url(a['href'])
|
|
desc = ''
|
|
for p in container.findAll('p'):
|
|
desc = self.tag_to_string(p)
|
|
if not 'Subscriber Content' in desc:
|
|
break
|
|
|
|
articles.append({'title':title, 'url':url,
|
|
'description':desc, 'date':''})
|
|
|
|
self.log('\tFound article:', title)
|
|
self.log('\t\t', desc)
|
|
|
|
return articles
|
|
|
|
def cleanup(self):
|
|
self.browser.open('http://online.wsj.com/logout?url=http://online.wsj.com')
|
|
|