mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-07-09 03:04:10 -04:00
New recipe for the free parts of The Wall Street journal by Nick Redding
This commit is contained in:
Kovid Goyal
parent
4592e03552
commit
1318348d57
261
resources/recipes/wsj_free.recipe
Normal file
261
resources/recipes/wsj_free.recipe
Normal file
@ -0,0 +1,261 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
__license__ = 'GPL v3'
|
||||
|
||||
'''
|
||||
online.wsj.com.com
|
||||
'''
|
||||
import re
|
||||
from calibre.web.feeds.recipes import BasicNewsRecipe
|
||||
|
||||
class WSJ(BasicNewsRecipe):
|
||||
# formatting adapted from original recipe by Kovid Goyal and Sujata Raman
|
||||
title = u'Wall Street Journal (free)'
|
||||
__author__ = 'Nick Redding'
|
||||
language = 'en'
|
||||
description = ('All the free content from the Wall Street Journal (business'
|
||||
', financial and political news)')
|
||||
no_stylesheets = True
|
||||
timefmt = ' [%b %d]'
|
||||
extra_css = '''h1{font-size:large; font-family:Georgia,"Century Schoolbook","Times New Roman",Times,serif;}
|
||||
h2{font-family:Georgia,"Century Schoolbook","Times New Roman",Times,serif; font-size:small; font-style:italic;}
|
||||
.subhead{font-family:Georgia,"Century Schoolbook","Times New Roman",Times,serif; font-size:small; font-style:italic;}
|
||||
.insettipUnit {font-family:Arial,Sans-serif;font-size:xx-small;}
|
||||
.targetCaption{font-size:x-small; font-family:Arial,Helvetica,sans-serif;}
|
||||
.article{font-family :Arial,Helvetica,sans-serif; font-size:x-small;}
|
||||
.tagline { ont-size:xx-small;}
|
||||
.dateStamp {font-family:Arial,Helvetica,sans-serif;}
|
||||
h3{font-family:Arial,Helvetica,sans-serif; font-size:xx-small;}
|
||||
.byline {font-family:Arial,Helvetica,sans-serif; font-size:xx-small; list-style-type: none;}
|
||||
.metadataType-articleCredits {list-style-type: none;}
|
||||
h6{ font-family:Georgia,"Century Schoolbook","Times New Roman",Times,serif; font-size:small;font-style:italic;}
|
||||
.paperLocation{font-size:xx-small;}'''
|
||||
|
||||
remove_tags_before = dict(name='h1')
|
||||
remove_tags = [ dict(id=["articleTabs_tab_article", "articleTabs_tab_comments",
|
||||
"articleTabs_tab_interactive","articleTabs_tab_video",
|
||||
"articleTabs_tab_map","articleTabs_tab_slideshow"]),
|
||||
{'class':['footer_columns','network','insetCol3wide','interactive','video','slideshow','map',
|
||||
'insettip','insetClose','more_in', "insetContent", 'articleTools_bottom', 'aTools', 'tooltip',
|
||||
'adSummary', 'nav-inline','insetFullBracket']},
|
||||
dict(rel='shortcut icon'),
|
||||
]
|
||||
remove_tags_after = [dict(id="article_story_body"), {'class':"article story"}]
|
||||
|
||||
|
||||
def preprocess_html(self,soup):
|
||||
# This gets rid of the annoying superfluous bullet symbol preceding columnist bylines
|
||||
ultag = soup.find('ul',attrs={'class' : 'cMetadata metadataType-articleCredits'})
|
||||
if ultag:
|
||||
a = ultag.h3
|
||||
if a:
|
||||
ultag.replaceWith(a)
|
||||
return soup
|
||||
|
||||
def parse_index(self):
|
||||
|
||||
articles = {}
|
||||
key = None
|
||||
ans = []
|
||||
|
||||
def parse_index_page(page_name,page_title,omit_paid_content):
|
||||
|
||||
def article_title(tag):
|
||||
atag = tag.find('h2') # title is usually in an h2 tag
|
||||
if not atag: # if not, get text from the a tag
|
||||
atag = tag.find('a',href=True)
|
||||
if not atag:
|
||||
return ''
|
||||
t = self.tag_to_string(atag,False)
|
||||
if t == '':
|
||||
# sometimes the title is in the second a tag
|
||||
atag.extract()
|
||||
atag = tag.find('a',href=True)
|
||||
if not atag:
|
||||
return ''
|
||||
return self.tag_to_string(atag,False)
|
||||
return t
|
||||
return self.tag_to_string(atag,False)
|
||||
|
||||
def article_author(tag):
|
||||
atag = tag.find('strong') # author is usually in a strong tag
|
||||
if not atag:
|
||||
atag = tag.find('h4') # if not, look for an h4 tag
|
||||
if not atag:
|
||||
return ''
|
||||
return self.tag_to_string(atag,False)
|
||||
|
||||
def article_summary(tag):
|
||||
atag = tag.find('p')
|
||||
if not atag:
|
||||
return ''
|
||||
subtag = atag.strong
|
||||
if subtag:
|
||||
subtag.extract()
|
||||
return self.tag_to_string(atag,False)
|
||||
|
||||
def article_url(tag):
|
||||
atag = tag.find('a',href=True)
|
||||
if not atag:
|
||||
return ''
|
||||
url = re.sub(r'\?.*', '', atag['href'])
|
||||
return url
|
||||
|
||||
def handle_section_name(tag):
|
||||
# turns a tag into a section name with special processing
|
||||
# for Wat's News, U.S., World & U.S. and World
|
||||
s = self.tag_to_string(tag,False)
|
||||
if ("What" in s) and ("News" in s):
|
||||
s = "What's News"
|
||||
elif (s == "U.S.") or (s == "World & U.S.") or (s == "World"):
|
||||
s = s + " News"
|
||||
return s
|
||||
|
||||
|
||||
|
||||
mainurl = 'http://online.wsj.com'
|
||||
pageurl = mainurl+page_name
|
||||
#self.log("Page url %s" % pageurl)
|
||||
soup = self.index_to_soup(pageurl)
|
||||
# Find each instance of div with class including "headlineSummary"
|
||||
for divtag in soup.findAll('div',attrs={'class' : re.compile("^headlineSummary")}):
|
||||
|
||||
# divtag contains all article data as ul's and li's
|
||||
# first, check if there is an h3 tag which provides a section name
|
||||
stag = divtag.find('h3')
|
||||
if stag:
|
||||
if stag.parent['class'] == 'dynamic':
|
||||
# a carousel of articles is too complex to extract a section name
|
||||
# for each article, so we'll just call the section "Carousel"
|
||||
section_name = 'Carousel'
|
||||
else:
|
||||
section_name = handle_section_name(stag)
|
||||
else:
|
||||
section_name = "What's News"
|
||||
#self.log("div Section %s" % section_name)
|
||||
# find each top-level ul in the div
|
||||
# we don't restrict to class = newsItem because the section_name
|
||||
# sometimes changes via a ul tag inside the div
|
||||
for ultag in divtag.findAll('ul',recursive=False):
|
||||
stag = ultag.find('h3')
|
||||
if stag:
|
||||
if stag.parent.name == 'ul':
|
||||
# section name has changed
|
||||
section_name = handle_section_name(stag)
|
||||
#self.log("ul Section %s" % section_name)
|
||||
# delete the h3 tag so it doesn't get in the way
|
||||
stag.extract()
|
||||
# find each top level li in the ul
|
||||
for litag in ultag.findAll('li',recursive=False):
|
||||
stag = litag.find('h3')
|
||||
if stag:
|
||||
# section name has changed
|
||||
section_name = handle_section_name(stag)
|
||||
#self.log("li Section %s" % section_name)
|
||||
# delete the h3 tag so it doesn't get in the way
|
||||
stag.extract()
|
||||
# if there is a ul tag inside the li it is superfluous;
|
||||
# it is probably a list of related articles
|
||||
utag = litag.find('ul')
|
||||
if utag:
|
||||
utag.extract()
|
||||
# now skip paid subscriber articles if desired
|
||||
subscriber_tag = litag.find(text="Subscriber Content")
|
||||
if subscriber_tag:
|
||||
if omit_paid_content:
|
||||
continue
|
||||
# delete the tip div so it doesn't get in the way
|
||||
tiptag = litag.find("div", { "class" : "tipTargetBox" })
|
||||
if tiptag:
|
||||
tiptag.extract()
|
||||
h1tag = litag.h1
|
||||
# if there's an h1 tag, it's parent is a div which should replace
|
||||
# the li tag for the analysis
|
||||
if h1tag:
|
||||
litag = h1tag.parent
|
||||
h5tag = litag.h5
|
||||
if h5tag:
|
||||
# section mame has changed
|
||||
section_name = self.tag_to_string(h5tag,False)
|
||||
#self.log("h5 Section %s" % section_name)
|
||||
# delete the h5 tag so it doesn't get in the way
|
||||
h5tag.extract()
|
||||
url = article_url(litag)
|
||||
if url == '':
|
||||
continue
|
||||
if url.startswith("/article"):
|
||||
url = mainurl+url
|
||||
if not url.startswith("http"):
|
||||
continue
|
||||
if not url.endswith(".html"):
|
||||
continue
|
||||
if 'video' in url:
|
||||
continue
|
||||
title = article_title(litag)
|
||||
if title == '':
|
||||
continue
|
||||
#self.log("URL %s" % url)
|
||||
#self.log("Title %s" % title)
|
||||
pubdate = ''
|
||||
#self.log("Date %s" % pubdate)
|
||||
author = article_author(litag)
|
||||
if author == '':
|
||||
author = section_name
|
||||
elif author == section_name:
|
||||
author = ''
|
||||
else:
|
||||
author = section_name+': '+author
|
||||
#if not author == '':
|
||||
# self.log("Author %s" % author)
|
||||
description = article_summary(litag)
|
||||
#if not description == '':
|
||||
# self.log("Description %s" % description)
|
||||
if not articles.has_key(page_title):
|
||||
articles[page_title] = []
|
||||
articles[page_title].append(dict(title=title,url=url,date=pubdate,description=description,author=author,content=''))
|
||||
|
||||
# customization notes: delete sections you are not interested in
|
||||
# set omit_paid_content to False if you want the paid content article previews
|
||||
sectionlist = ['Front Page','Commentary','World News','US News','Business','Markets',
|
||||
'Technology','Personal Finance','Life & Style','Real Estate','Careers','Small Business']
|
||||
omit_paid_content = True
|
||||
|
||||
if 'Front Page' in sectionlist:
|
||||
parse_index_page('/home-page','Front Page',omit_paid_content)
|
||||
ans.append('Front Page')
|
||||
if 'Commentary' in sectionlist:
|
||||
parse_index_page('/public/page/news-opinion-commentary.html','Commentary',omit_paid_content)
|
||||
ans.append('Commentary')
|
||||
if 'World News' in sectionlist:
|
||||
parse_index_page('/public/page/news-global-world.html','World News',omit_paid_content)
|
||||
ans.append('World News')
|
||||
if 'US News' in sectionlist:
|
||||
parse_index_page('/public/page/news-world-business.html','US News',omit_paid_content)
|
||||
ans.append('US News')
|
||||
if 'Business' in sectionlist:
|
||||
parse_index_page('/public/page/news-business-us.html','Business',omit_paid_content)
|
||||
ans.append('Business')
|
||||
if 'Markets' in sectionlist:
|
||||
parse_index_page('/public/page/news-financial-markets-stock.html','Markets',omit_paid_content)
|
||||
ans.append('Markets')
|
||||
if 'Technology' in sectionlist:
|
||||
parse_index_page('/public/page/news-tech-technology.html','Technology',omit_paid_content)
|
||||
ans.append('Technology')
|
||||
if 'Personal Finance' in sectionlist:
|
||||
parse_index_page('/public/page/news-personal-finance.html','Personal Finance',omit_paid_content)
|
||||
ans.append('Personal Finance')
|
||||
if 'Life & Style' in sectionlist:
|
||||
parse_index_page('/public/page/news-lifestyle-arts-entertainment.html','Life & Style',omit_paid_content)
|
||||
ans.append('Life & Style')
|
||||
if 'Real Estate' in sectionlist:
|
||||
parse_index_page('/public/page/news-real-estate-homes.html','Real Estate',omit_paid_content)
|
||||
ans.append('Real Estate')
|
||||
if 'Careers' in sectionlist:
|
||||
parse_index_page('/public/page/news-career-jobs.html','Careers',omit_paid_content)
|
||||
ans.append('Careers')
|
||||
if 'Small Business' in sectionlist:
|
||||
parse_index_page('/public/page/news-small-business-marketing.html','Small Business',omit_paid_content)
|
||||
ans.append('Small Business')
|
||||
|
||||
ans = [(key, articles[key]) for key in ans if articles.has_key(key)]
|
||||
return ans
|
||||
Loading…
x
Reference in New Issue
Block a user