Update Boston Globe

This commit is contained in:
Kovid Goyal 2014-11-06 08:49:42 +05:30
parent 5f4bf540f9
commit 0afaa3e7de

View File

@ -1,53 +1,418 @@
__license__ = 'GPL v3'
__copyright__ = '2009-2010, Darko Miletic <darko.miletic at gmail.com>'
'''
www.boston.com
'''
import string, re
from calibre.web.feeds.recipes import BasicNewsRecipe
from calibre.ebooks.BeautifulSoup import Tag
from datetime import date
class BusinessStandard(BasicNewsRecipe):
title = 'The Boston Globe'
__author__ = 'Darko Miletic'
description = 'News from Boston'
oldest_article = 2
max_articles_per_feed = 100
no_stylesheets = True
delay = 1
use_embedded_content = False
auto_cleanup = True
encoding = 'utf-8'
publisher = 'Boston'
category = 'news, boston, usa, world'
language = 'en'
publication_type = 'newspaper'
masthead_url = 'http://cache.boston.com/images/globe/grslider/the_boston_globe.gif'
extra_css = ' body{font-family: Georgia, serif} div#articleBodyTop{display:block} '
class BostonGlobeSubscription(BasicNewsRecipe):
conversion_options = {
'comments' : description
,'tags' : category
,'language' : language
,'publisher' : publisher
}
# logger = logging.getLogger("mechanize")
# logger.addHandler(logging.StreamHandler(sys.stdout))
# logger.setLevel(logging.DEBUG)
title = "Boston Globe Subscription"
__author__ = 'Rob Freundlich'
description = 'Boston Globe with full articles for subscribers'
INDEX = date.today().strftime('http://www.bostonglobe.com/todayspaper/%Y/%m/%d')
todaysDate = date.today().strftime("%d/%m/%Y")
timefmt = ' [%a, %d %b, %Y]'
needs_subscription = True
remove_tags = [dict(attrs={"class":["skip-nav article-more",
"aside promo",
"article-bar bar-yellow",
"tools",
"sticky-tools",
"article-footer",
"bg-footer"]}),
dict(attrs={"id":["masthead",
"video",
"section-nav",
"meter-limit-met-popup"]})]
no_stylesheets = True
# simultaneous_downloads = 1
valid_filename_chars = "-_.%s%s" % (string.ascii_letters, string.digits)
cover_url = "http://ecx.images-amazon.com/images/I/419qC6zeKSL._SL500_AA300_.jpg"
preprocess_regexps = [
(re.compile(r'\<img src\=\"\/\/'), lambda match: '<img src="http://'),
]
comics_to_fetch = [
"ADAM@HOME",
"ARLO & JANIS",
#"ASK SHAGG",
#"CUL DE SAC",
#"CURTIS",
"DILBERT",
"DOONESBURY",
"DUSTIN",
#"THE FAMILY CIRCUS",
"F MINUS",
"FOR BETTER OR WORSE",
"FOXTROT",
#"GET FUZZY",
#"MOTHER GOOSE & GRIMM",
#"IN THE STICKS",
#"JUMPSTART",
"MONTY",
"NON SEQUITUR",
"PICKLES",
#"POOCH CAFE",
"RHYMES WITH ORANGE",
#"ROSE IS ROSE",
"STONE SOUP",
#"ZIPPY THE PINHEAD",
"ZITS"]
#keep_only_tags = [dict(attrs={'id':['INDblogEntry','blogEntry','articleHeader','articleGraphs','galleryShell']})]
#remove_tags = [
#dict(name=['object','link','script','iframe'])
#,dict(attrs={'id':['blogheadTools','bdc_emailWidget','tools','relatedContent']})
#]
def image_url_processor(self, baseurl, url):
self.log("===================\nbaseurl: ", baseurl, "\nurl: ", url)
# This is a hack because some of the URLs just have a leading
# // instead of http://
if url.startswith("//"):
url = "http:" + url
feeds = [
(u'Top Stories' , u'http://feeds.boston.com/boston/topstories' )
,(u'Patriots news', u'http://feeds.boston.com/boston/sports/football/patriots/patriots_rss')
,(u'National news', u'http://feeds.boston.com/boston/news/nation' )
,(u'World news' , u'http://feeds.boston.com/boston/news/world' )
]
url = self.get_image(url)
#def print_version(self, url):
#return url + '?page=full'
self.log("url out: ", url, "\n===================")
#def get_article_url(self, article):
#rawarticle = article.get('guid', None)
#return rawarticle.rpartition('?')[0]
return url
def get_image(self, url):
# Another hack - sometimes the URLs just have a leading /,
# in which case I stick on "http://" and the correct domain
if url.startswith("/"):
url = self.make_url(url)
# Get the image bytes
br = BasicNewsRecipe.get_browser(self)
response = br.open(url)
data = response.get_data()
# write it to a local file whose name is based on the URL
filename = ''.join(c for c in url if c in self.valid_filename_chars)
self.log("filename=%s" % filename)
f = open(filename, "wb")
f.write(data)
f.close()
return url
def is_login_form(self, form):
return form.action == "https://www.bostonglobe.com/Login"
def get_browser(self):
br = BasicNewsRecipe.get_browser(self)
if self.username is not None and self.password is not None:
# br.set_debug_http(True)
# br.set_debug_responses(True)
# br.set_debug_redirects(True)
#
# This next line is here because the connection was failing
# with a "closed by remote host". But running Fiddler seems to solve it,
# so I'm running Fiddler on port 8888 all the time now. It's a hack, but
# until I can figure out a better solution, it'll do
#
#br.set_proxies({"http":"127.0.0.1:8888", "https":"127.0.0.1:8888"})
#
# end of hack
#
br.open("https://www.bostonglobe.com/eom/SysConfig/WebPortal/BostonGlobe/Framework/regi/final-login.jsp")
br.select_form(predicate=self.is_login_form)
br["username"] = self.username
br["password"] = self.password
# pdb.set_trace()
br.submit()
return br
def make_url(self, url):
if url.startswith("//"):
return "http:" + url
return "http://www.bostonglobe.com" + url
def make_bostoncom_url(self, url):
if url.startswith("//"):
return "http:" + url
return "http://articles.boston.com" + url
def parse_index(self):
# self.logger.setLevel(logging.WARNING)
feeds = []
self.log("Getting today's paper from ", self.INDEX)
soup = self.index_to_soup(self.INDEX)
def get_top_stories():
self.log("Getting top stories")
articles = []
topStoriesDiv = soup.find("div", {"class":re.compile(".*stories-top.*")})
title = ""
url = ""
excerpt = ""
stories = topStoriesDiv.findAll("div", {"class":re.compile("story.*")})
for story in stories:
title = self.tag_to_string(story.find("h2", {"class":re.compile(".*story-title.*")}))
link = story.find("a")
if (link):
url = link["href"]
excerpt_div = story.find("div", {"class":"excerpt"})
excerpt = self.tag_to_string(excerpt_div)
articles.append({"title":title, "url":self.make_url(url), "date":self.todaysDate, "description":excerpt})
else:
self.log("Skipping ", title, " because it has no link")
if articles:
feeds.append(("Top Stories", articles))
def get_section(sectionTitle, sectionID):
self.log("Getting section", sectionTitle)
articles = []
sectionDiv = soup.find(id=sectionID)
if (sectionDiv):
sectionHeader = sectionDiv.find("h2", "hed-section")
feedTitle = self.tag_to_string(sectionHeader)
excerpts = sectionDiv.findAll("div", "sec-excerpt")
for excerpt in excerpts:
url = ""
title = ""
category = ""
author = ""
# Stories here follow similar forms to top-stories (above)
storyTitle = excerpt.find("h3", "story-title")
if (storyTitle.parent.name == "a"):
a = storyTitle.parent
url = a["href"]
title = self.tag_to_string(storyTitle)
else:
a = storyTitle.find("a")
url = a["href"]
title = self.tag_to_string(a)
hedCat = excerpt.find("p", "hed-cat")
if (hedCat):
category = self.tag_to_string(hedCat)
authorHeader = excerpt.find("h4", "author")
if (authorHeader):
author = self.tag_to_string(authorHeader)
if (category != "") & (category != " "):
title = category + ": " + title
description = ""
for para in excerpt.findAll("p"):
if (para != hedCat):
description += self.tag_to_string(para)
articles.append({"title":title, "url":self.make_url(url), "author":author, "date":self.todaysDate, "description":description})
if articles:
feeds.append((feedTitle, articles))
def getOpinionSection():
self.log("Getting section", "Editorials and Opinions")
articles = []
opinionSoup = self.index_to_soup("http://www.bostonglobe.com/opinion")
# Find and process the editorials
topStories = opinionSoup.find("div", "stories-top")
for story in topStories.findAll("div", {"class":re.compile("story.*")}):
# Story title is always in an H2 tag whose class contains "story-title"
titleTag = story.find("h2", {"class":re.compile("story-title.*")})
# Description is always in a DIV whose class is "excert"
excerptTag = story.find("div", "excerpt")
# Author is in a P whose class is "hed-cat" or in a CITE
authorTag = story.find("p", "hed-cat")
if (authorTag == None):
authorTag = story.find("cite")
# URL is in an A whose class is "story-perm". If not, it's in the only A tag
urlTag = story.find("a", "story-perm")
if (urlTag == None):
urlTag = story.find("a")
# Extract the values and build the article
title = ""
if (titleTag):
title = self.tag_to_string(titleTag)
author = ""
if (authorTag):
author = self.tag_to_string(authorTag)
description = ""
if (excerptTag):
description = self.tag_to_string(excerptTag)
url = ""
if (urlTag):
url = urlTag["href"]
articles.append({"title":title, "url":self.make_url(url), "author":author, "date":self.todaysDate, "description":description})
# Now find Letters to the Editor and process them
mainDiv = opinionSoup.find("div", {"id":"main"})
if (mainDiv == None):
print "no mainDiv found"
else:
lettersAnchor = mainDiv.find("a", {"href":re.compile(".*opinion/letters.*")})
if (lettersAnchor == None):
print "No lettersAnchor found"
else:
lettersFeatureWell = lettersAnchor.parent.parent
lettersContent = lettersFeatureWell.find("div", "content")
if (lettersContent == None):
print "No lettersContent found"
else:
mainLetterDiv = lettersContent.find("div", {"class":re.compile("main.*")})
mainLetterStories = mainLetterDiv.findAll("h4", "story-title")
if (mainLetterStories == None):
print "no mainLetterStories found"
else:
for mainLetterStory in mainLetterStories:
mainLetterAnchor = mainLetterStory.parent
if (mainLetterAnchor == None):
print "no mainLetterAnchor found"
else:
articles.append({"title":self.tag_to_string(mainLetterStory), "url":self.make_url(
mainLetterAnchor["href"]), "author":"Letter", "date":self.todaysDate, "description":""})
inSection = lettersFeatureWell.find("div", "in-section")
if (inSection == None):
print "no inSection found"
else:
lettersList = inSection.find("ul")
if (lettersList == None):
print "no lettersList found"
else:
for letter in lettersList.findAll("li"):
letterAnchor = letter.find("a")
if (letterAnchor == None):
print "no letterAnchor for ", letter
else:
articles.append({"title":self.tag_to_string(letterAnchor), "url":self.make_url(
letterAnchor["href"]), "author":"Letter", "date":self.todaysDate, "description":""})
if articles:
# for article in articles:
# print "============"
# print "TITLE", article["title"]
# print "URL", article["url"]
# print "AUTHOR", article["author"]
# print "DATE", article["date"]
# print "DESCRIPTION", article["description"]
# print "============"
feeds.append(("Editorial and Opinion", articles))
def get_comics():
articles = []
comicSoup = self.index_to_soup("http://www.bostonglobe.com/lifestyle/comics")
for personIndex in comicSoup.findAll("ol", {"class":re.compile("person-index.*")}):
for li in personIndex.findAll("li"):
title = self.tag_to_string(li.p)
if (title in self.comics_to_fetch):
url = li.a["href"]
author = self.tag_to_string(li.h2)
#comicPageSoup = self.index_to_soup(self.make_url(url))
#imageURL = comicPageSoup.findAll("a", "comic")
# if len(imageURL) > 0:
# url = imageURL[0]["href"]
# print "COMIC %s: %s" % (title, url)
articles.append({"title":title, "url":self.make_url(url), "author":author, "date":self.todaysDate, "description":""})
feeds.append(("Comics", articles))
get_top_stories()
get_section("Nation", "tp-section-thenation")
get_section("World", "tp-section-theworld")
get_section("Metro", "tp-section-metro")
getOpinionSection()
get_section("Arts & Movies", "tp-section-g:arts&movies")
get_section("Family", "tp-section-g:family")
get_section("Style", "tp-section-g:style")
get_section("Globe West", "tp-section-globewest")
get_section("Food", "tp-section-g:food")
get_section("Living", "tp-section-living")
get_section("Health", "tp-section-g:health")
get_section("Ideas", "tp-section-ideas")
get_section("Boston Globe Magazine", "tp-section-magazine")
get_comics()
#get_section("Business", "tp-section-business")
#get_section("Obituaries", "tp-section-obituaries")
#get_section("Sports", "tp-section-sports")
#get_section("Globe North", "tp-section-globenorth")
#get_section("Globe South", "tp-section-globesouth")
#get_section("Money & Careers", "tp-section-money&careers")
#get_section("Books", "tp-section-books")
#get_section("Travel", "tp-section-travel")
#get_section("Real Estate", "tp-section-realestate")
for feed in feeds:
feedTitle = feed[0]
articles = feed[1]
self.log(feedTitle)
for article in articles:
self.log(" ", article["title"], ".....", article["url"])
self.log(" ", article["description"])
return feeds
def postprocess_comics(self, soup, first):
main = soup.find("div", id="main")
sectionHead = main.find("div", "section-head")
title = sectionHead.h2
byline = sectionHead.h3
imgLink = main.find("a", "comic")
img = imgLink.img
print "title: %s" % title
print "byline: %s" % byline
print "imgLink: %s" % imgLink
print "img: %s" % img
body = Tag(soup, "body")
body.insert(0, title)
body.insert(1, byline)
body.insert(2, img)
soup.body.replaceWith(body)
return soup
def preprocess_html(self, soup):
images = soup.findAll("img")
for image in images:
if (image["src"] == ""):
if (image["data-fullsrc"]):
image["src"] = image["data-fullsrc"]
elif (image["src"].startswith("//")):
image["src"] = "http://" + image["src"]
return soup
def postprocess_html(self, soup, first):
comicsBody = soup.find("body", {"class":re.compile(".*section-comics.*")})
if (comicsBody):
return self.postprocess_comics(soup, first)
article = soup.find("div", "article")
if (article):
# Yay! We're getting the subscriber view. Easy to handle
articleHeader = article.find("div", "header")
articleByline = article.find("div", "byline")
articleBody = article.find("div", "article-body")
figureLead = article.find("div", "figure lead-figure full")
body = Tag(soup, "body")
body.insert(0, articleHeader)
body.insert(1, articleByline)
body.insert(2, articleBody)
if (figureLead):
body.insert(2, figureLead)
soup.body.replaceWith(body)
return soup