Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Update BostonGlobe

Browse Source
This commit is contained in:
Kovid Goyal 2014-11-22 10:02:20 +05:30
parent b2d36cfcf0
commit 9a11917eac
1 changed files with 44 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
recipes/boston.com.recipe
View File

@ -2,6 +2,8 @@ import string, re
from calibre.web.feeds.recipes import BasicNewsRecipe from calibre.web.feeds.recipes import BasicNewsRecipe
from calibre.ebooks.BeautifulSoup import Tag from calibre.ebooks.BeautifulSoup import Tag
from datetime import date from datetime import date
from calibre.utils.magick.draw import save_cover_data_to
from calibre.ptempfile import PersistentTemporaryFile
class BostonGlobeSubscription(BasicNewsRecipe): class BostonGlobeSubscription(BasicNewsRecipe):
@ -75,6 +77,7 @@ class BostonGlobeSubscription(BasicNewsRecipe):
return url return url
def get_image(self, url): def get_image(self, url):
# pdb.set_trace()
# Another hack - sometimes the URLs just have a leading /, # Another hack - sometimes the URLs just have a leading /,
# in which case I stick on "http://" and the correct domain # in which case I stick on "http://" and the correct domain
if url.startswith("/"): if url.startswith("/"):
@ -84,15 +87,14 @@ class BostonGlobeSubscription(BasicNewsRecipe):
br = BasicNewsRecipe.get_browser(self) br = BasicNewsRecipe.get_browser(self)
response = br.open(url) response = br.open(url)
data = response.get_data() data = response.get_data()
pt = PersistentTemporaryFile('.jpg')
# write it to a local file whose name is based on the URL pt.close()
filename = ''.join(c for c in url if c in self.valid_filename_chars) try:
self.log("filename=%s" % filename) save_cover_data_to(data, pt.name)
return 'file:///' + pt.name
f = open(filename, "wb") except:
f.write(data) self.log('Failed to load image: %s' % url)
f.close() return ''
return url
def is_login_form(self, form): def is_login_form(self, form):
return form.action == "https://www.bostonglobe.com/Login" return form.action == "https://www.bostonglobe.com/Login"
@ -225,12 +227,12 @@ class BostonGlobeSubscription(BasicNewsRecipe):
# Author is in a P whose class is "hed-cat" or in a CITE # Author is in a P whose class is "hed-cat" or in a CITE
authorTag = story.find("p", "hed-cat") authorTag = story.find("p", "hed-cat")
if (authorTag == None): if (authorTag is None):
authorTag = story.find("cite") authorTag = story.find("cite")
# URL is in an A whose class is "story-perm". If not, it's in the only A tag # URL is in an A whose class is "story-perm". If not, it's in the only A tag
urlTag = story.find("a", "story-perm") urlTag = story.find("a", "story-perm")
if (urlTag == None): if (urlTag is None):
urlTag = story.find("a") urlTag = story.find("a")
# Extract the values and build the article # Extract the values and build the article
@ -253,41 +255,41 @@ class BostonGlobeSubscription(BasicNewsRecipe):
# Now find Letters to the Editor and process them # Now find Letters to the Editor and process them
mainDiv = opinionSoup.find("div", {"id":"main"}) mainDiv = opinionSoup.find("div", {"id":"main"})
if (mainDiv == None): if (mainDiv is None):
print "no mainDiv found" print "no mainDiv found"
else: else:
lettersAnchor = mainDiv.find("a", {"href":re.compile(".*opinion/letters.*")}) lettersAnchor = mainDiv.find("a", {"href":re.compile(".*opinion/letters.*")})
if (lettersAnchor == None): if (lettersAnchor is None):
print "No lettersAnchor found" print "No lettersAnchor found"
else: else:
lettersFeatureWell = lettersAnchor.parent.parent lettersFeatureWell = lettersAnchor.parent.parent
lettersContent = lettersFeatureWell.find("div", "content") lettersContent = lettersFeatureWell.find("div", "content")
if (lettersContent == None): if (lettersContent is None):
print "No lettersContent found" print "No lettersContent found"
else: else:
mainLetterDiv = lettersContent.find("div", {"class":re.compile("main.*")}) mainLetterDiv = lettersContent.find("div", {"class":re.compile("main.*")})
mainLetterStories = mainLetterDiv.findAll("h4", "story-title") mainLetterStories = mainLetterDiv.findAll("h4", "story-title")
if (mainLetterStories == None): if (mainLetterStories is None):
print "no mainLetterStories found" print "no mainLetterStories found"
else: else:
for mainLetterStory in mainLetterStories: for mainLetterStory in mainLetterStories:
mainLetterAnchor = mainLetterStory.parent mainLetterAnchor = mainLetterStory.parent
if (mainLetterAnchor == None): if (mainLetterAnchor is None):
print "no mainLetterAnchor found" print "no mainLetterAnchor found"
else: else:
articles.append({"title":self.tag_to_string(mainLetterStory), "url":self.make_url( articles.append({"title":self.tag_to_string(mainLetterStory), "url":self.make_url(
mainLetterAnchor["href"]), "author":"Letter", "date":self.todaysDate, "description":""}) mainLetterAnchor["href"]), "author":"Letter", "date":self.todaysDate, "description":""})
inSection = lettersFeatureWell.find("div", "in-section") inSection = lettersFeatureWell.find("div", "in-section")
if (inSection == None): if (inSection is None):
print "no inSection found" print "no inSection found"
else: else:
lettersList = inSection.find("ul") lettersList = inSection.find("ul")
if (lettersList == None): if (lettersList is None):
print "no lettersList found" print "no lettersList found"
else: else:
for letter in lettersList.findAll("li"): for letter in lettersList.findAll("li"):
letterAnchor = letter.find("a") letterAnchor = letter.find("a")
if (letterAnchor == None): if (letterAnchor is None):
print "no letterAnchor for ", letter print "no letterAnchor for ", letter
else: else:
articles.append({"title":self.tag_to_string(letterAnchor), "url":self.make_url( articles.append({"title":self.tag_to_string(letterAnchor), "url":self.make_url(