Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Update NY Times some more

Browse Source
This commit is contained in:
Kovid Goyal 2014-01-12 19:29:09 +05:30
parent a60a80d125
commit 2d8dfc3a28
2 changed files with 181 additions and 190 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
recipes/nytimes.recipe
View File

@ -33,7 +33,6 @@ class NYTimes(BasicNewsRecipe):
# and 30 will get the most popular measured over 30 days. # and 30 will get the most popular measured over 30 days.
# you still only get up to 20 articles in each category # you still only get up to 20 articles in each category
# set headlinesOnly to True for the headlines-only version. If True, webEdition is ignored. # set headlinesOnly to True for the headlines-only version. If True, webEdition is ignored.
headlinesOnly = True headlinesOnly = True
@ -82,6 +81,7 @@ class NYTimes(BasicNewsRecipe):
# The maximum number of articles that will be downloaded # The maximum number of articles that will be downloaded
max_articles_per_feed = 100 max_articles_per_feed = 100
use_embedded_content = False
# Whether to omit duplicates of articles (typically arsing when articles are indexed in # Whether to omit duplicates of articles (typically arsing when articles are indexed in
# more than one section). If True, only the first occurance will be downloaded. # more than one section). If True, only the first occurance will be downloaded.
@ -122,7 +122,6 @@ class NYTimes(BasicNewsRecipe):
(u'Tech - Open', u'http://open.blogs.nytimes.com/feed/') (u'Tech - Open', u'http://open.blogs.nytimes.com/feed/')
] ]
if headlinesOnly: if headlinesOnly:
title='New York Times Headlines' title='New York Times Headlines'
description = 'Headlines from the New York Times' description = 'Headlines from the New York Times'
@ -168,8 +167,7 @@ class NYTimes(BasicNewsRecipe):
cover_margins = (18,18,'grey99') cover_margins = (18,18,'grey99')
remove_tags_before = dict(id='article') keep_only_tags = dict(id=['article', 'story', 'content'])
remove_tags_after = dict(id='article')
remove_tags = [ remove_tags = [
dict(attrs={'class':[ dict(attrs={'class':[
'articleFooter', 'articleFooter',
@ -184,6 +182,7 @@ class NYTimes(BasicNewsRecipe):
'entry-response module', 'entry-response module',
'leftNavTabs', 'leftNavTabs',
'metaFootnote', 'metaFootnote',
'inside-story',
'module box nav', 'module box nav',
'nextArticleLink', 'nextArticleLink',
'nextArticleLink clearfix', 'nextArticleLink clearfix',
@ -223,6 +222,7 @@ class NYTimes(BasicNewsRecipe):
'credit' 'credit'
]}), ]}),
dict(attrs={'class':lambda x: x and 'related-coverage-marginalia' in x.split()}), dict(attrs={'class':lambda x: x and 'related-coverage-marginalia' in x.split()}),
dict(attrs={'class':lambda x: x and 'interactive' in x.split()}),
dict(name='div', attrs={'class':re.compile('toolsList')}), # bits dict(name='div', attrs={'class':re.compile('toolsList')}), # bits
dict(name='div', attrs={'class':re.compile('postNavigation')}), # bits dict(name='div', attrs={'class':re.compile('postNavigation')}), # bits
dict(name='div', attrs={'class':'tweet'}), dict(name='div', attrs={'class':'tweet'}),
@ -271,11 +271,13 @@ class NYTimes(BasicNewsRecipe):
'related-content', # added for DealBook 'related-content', # added for DealBook
'whats-next', 'whats-next',
]), ]),
dict(name=['script', 'noscript', 'style','form','hr', 'button', 'meta'])] dict(name=['script', 'noscript', 'style','form','hr', 'button', 'meta', 'footer'])]
no_stylesheets = True no_stylesheets = True
extra_css = ''' extra_css = '''
.articleHeadline { text-align: left; margin-top:0.5em; margin-bottom:0.25em; } .articleHeadline { text-align: left; margin-top:0.5em; margin-bottom:0.25em; }
.credit { font-weight: normal; text-align: right; font-size: 50%; line-height:1em; margin-top:5px; margin-left:0; margin-right:0; margin-bottom: 0; } .credit { font-weight: normal; text-align: right; font-size:
50%; line-height:1em; margin-top:5px; margin-left:0;
margin-right:0; margin-bottom: 0; }
.byline { text-align: left; font-size: 50%; line-height:1em; margin-top:10px; margin-left:0; margin-right:0; margin-bottom: 0; } .byline { text-align: left; font-size: 50%; line-height:1em; margin-top:10px; margin-left:0; margin-right:0; margin-bottom: 0; }
.dateline { text-align: left; font-size: 50%; line-height:1em;margin-top:5px; margin-left:0; margin-right:0; margin-bottom: 0; } .dateline { text-align: left; font-size: 50%; line-height:1em;margin-top:5px; margin-left:0; margin-right:0; margin-bottom: 0; }
.kicker { font-size: 50%; line-height:1em;margin-top:5px; margin-left:0; margin-right:0; margin-bottom: 0; } .kicker { font-size: 50%; line-height:1em;margin-top:5px; margin-left:0; margin-right:0; margin-bottom: 0; }
@ -291,7 +293,6 @@ class NYTimes(BasicNewsRecipe):
.asidenote {color:blue;margin:0px 0px 0px 0px; padding: 0px 0px 0px 0px; font-size:100%;font-weight:bold;} .asidenote {color:blue;margin:0px 0px 0px 0px; padding: 0px 0px 0px 0px; font-size:100%;font-weight:bold;}
.source {text-align: left; font-size: x-small; }''' .source {text-align: left; font-size: x-small; }'''
articles = {} articles = {}
key = None key = None
ans = [] ans = []
@ -412,7 +413,6 @@ class NYTimes(BasicNewsRecipe):
def short_title(self): def short_title(self):
return self.title return self.title
def article_to_soup(self, url_or_raw, raw=False): def article_to_soup(self, url_or_raw, raw=False):
from contextlib import closing from contextlib import closing
import copy import copy
@ -446,7 +446,6 @@ class NYTimes(BasicNewsRecipe):
usrc = self.preprocess_raw_html(usrc, url_or_raw) usrc = self.preprocess_raw_html(usrc, url_or_raw)
return BeautifulSoup(usrc, markupMassage=nmassage) return BeautifulSoup(usrc, markupMassage=nmassage)
def massageNCXText(self, description): def massageNCXText(self, description):
# Kindle TOC descriptions won't render certain characters # Kindle TOC descriptions won't render certain characters
if description: if description:
@ -501,7 +500,7 @@ class NYTimes(BasicNewsRecipe):
if authorAttribution: if authorAttribution:
author = self.tag_to_string(authorAttribution, use_alt=False) author = self.tag_to_string(authorAttribution, use_alt=False)
feed = self.key if self.key is not None else 'Uncategorized' feed = self.key if self.key is not None else 'Uncategorized'
if not self.articles.has_key(feed): if feed not in self.articles:
self.ans.append(feed) self.ans.append(feed)
self.articles[feed] = [] self.articles[feed] = []
self.articles[feed].append( self.articles[feed].append(
@ -536,7 +535,6 @@ class NYTimes(BasicNewsRecipe):
desc = '' desc = ''
return(title,url,author,desc) return(title,url,author,desc)
have_emailed = False have_emailed = False
emailed_soup = self.index_to_soup('http://www.nytimes.com/most-popular-emailed?period='+self.popularPeriod) emailed_soup = self.index_to_soup('http://www.nytimes.com/most-popular-emailed?period='+self.popularPeriod)
for h3tag in emailed_soup.findAll('h3'): for h3tag in emailed_soup.findAll('h3'):
@ -565,7 +563,7 @@ class NYTimes(BasicNewsRecipe):
dict(title=title, url=url, date=strftime('%a, %d %b'), dict(title=title, url=url, date=strftime('%a, %d %b'),
description=desc, author=author, description=desc, author=author,
content='')) content=''))
viewed_ans = [(k, popular_articles[k]) for k in key_list if popular_articles.has_key(k)] viewed_ans = [(k, popular_articles[k]) for k in key_list if k in popular_articles]
for x in viewed_ans: for x in viewed_ans:
ans.append(x) ans.append(x)
return ans return ans
@ -588,10 +586,10 @@ class NYTimes(BasicNewsRecipe):
tech_articles[f.title] = [] tech_articles[f.title] = []
for a in f.articles: for a in f.articles:
tech_articles[f.title].append( tech_articles[f.title].append(
dict(title=a.title, url=a.url, date=a.date, dict(title=a.title, url=a.url.partition('?')[0], date=a.date,
description=a.summary, author=a.author, description=a.summary, author=a.author,
content=a.content)) content=a.content))
tech_ans = [(k, tech_articles[k]) for k in key_list if tech_articles.has_key(k)] tech_ans = [(k, tech_articles[k]) for k in key_list if k in tech_articles]
for x in tech_ans: for x in tech_ans:
ans.append(x) ans.append(x)
return ans return ans
@ -630,10 +628,9 @@ class NYTimes(BasicNewsRecipe):
for lidiv in div.findAll('li'): for lidiv in div.findAll('li'):
self.handle_article(lidiv) self.handle_article(lidiv)
self.ans = [(k, self.articles[k]) for k in self.ans if self.articles.has_key(k)] self.ans = [(k, self.articles[k]) for k in self.ans if k in self.articles]
return self.filter_ans(self.get_tech_feeds(self.get_popular_articles(self.ans))) return self.filter_ans(self.get_tech_feeds(self.get_popular_articles(self.ans)))
def parse_todays_index(self): def parse_todays_index(self):
soup = self.index_to_soup('http://www.nytimes.com/pages/todayspaper/index.html') soup = self.index_to_soup('http://www.nytimes.com/pages/todayspaper/index.html')
@ -663,7 +660,7 @@ class NYTimes(BasicNewsRecipe):
if not skipping: if not skipping:
self.handle_article(lidiv) self.handle_article(lidiv)
self.ans = [(k, self.articles[k]) for k in self.ans if self.articles.has_key(k)] self.ans = [(k, self.articles[k]) for k in self.ans if k in self.articles]
return self.filter_ans(self.get_tech_feeds(self.get_popular_articles(self.ans))) return self.filter_ans(self.get_tech_feeds(self.get_popular_articles(self.ans)))
def parse_headline_index(self): def parse_headline_index(self):
@ -709,13 +706,13 @@ class NYTimes(BasicNewsRecipe):
description = self.tag_to_string(desc,use_alt=False) description = self.tag_to_string(desc,use_alt=False)
else: else:
description = '' description = ''
if not self.articles.has_key(section_name): if section_name not in self.articles:
self.ans.append(section_name) self.ans.append(section_name)
self.articles[section_name] = [] self.articles[section_name] = []
print('Title '+title+' author '+author) print('Title '+title+' author '+author)
self.articles[section_name].append(dict(title=title, url=url, date=pubdate, description=description, author=author, content='')) self.articles[section_name].append(dict(title=title, url=url, date=pubdate, description=description, author=author, content=''))
self.ans = [(k, self.articles[k]) for k in self.ans if self.articles.has_key(k)] self.ans = [(k, self.articles[k]) for k in self.ans if k in self.articles]
return self.filter_ans(self.ans) return self.filter_ans(self.ans)
def parse_index(self): def parse_index(self):
@ -735,7 +732,7 @@ class NYTimes(BasicNewsRecipe):
if kill_all or (self.recursions==0): if kill_all or (self.recursions==0):
a.replaceWith(self.tag_to_string(a,False)) a.replaceWith(self.tag_to_string(a,False))
else: else:
if a.has_key('href'): if 'href' in a:
if a['href'].startswith('http://www.nytimes'): if a['href'].startswith('http://www.nytimes'):
if not a['href'].endswith('pagewanted=all'): if not a['href'].endswith('pagewanted=all'):
url = re.sub(r'\?.*', '', a['href']) url = re.sub(r'\?.*', '', a['href'])
@ -743,13 +740,13 @@ class NYTimes(BasicNewsRecipe):
a.replaceWith(self.tag_to_string(a,False)) a.replaceWith(self.tag_to_string(a,False))
else: else:
a['href'] = url+'?pagewanted=all' a['href'] = url+'?pagewanted=all'
elif not (a['href'].startswith('http://pogue') or \ elif not (a['href'].startswith('http://pogue') or
a['href'].startswith('http://bits') or \ a['href'].startswith('http://bits') or
a['href'].startswith('http://travel') or \ a['href'].startswith('http://travel') or
a['href'].startswith('http://business') or \ a['href'].startswith('http://business') or
a['href'].startswith('http://tech') or \ a['href'].startswith('http://tech') or
a['href'].startswith('http://health') or \ a['href'].startswith('http://health') or
a['href'].startswith('http://dealbook') or \ a['href'].startswith('http://dealbook') or
a['href'].startswith('http://open')): a['href'].startswith('http://open')):
a.replaceWith(self.tag_to_string(a,False)) a.replaceWith(self.tag_to_string(a,False))
return soup return soup
@ -764,7 +761,7 @@ class NYTimes(BasicNewsRecipe):
return None return None
## print("HANDLING AD FORWARD:") ## print("HANDLING AD FORWARD:")
## print(soup) # print(soup)
if self.keep_only_tags: if self.keep_only_tags:
body = Tag(soup, 'body') body = Tag(soup, 'body')
try: try:
@ -802,7 +799,6 @@ class NYTimes(BasicNewsRecipe):
return soup return soup
def preprocess_html(self, soup): def preprocess_html(self, soup):
#print(strftime("%H:%M:%S")+" -- PREPROCESS TITLE="+self.tag_to_string(soup.title)) #print(strftime("%H:%M:%S")+" -- PREPROCESS TITLE="+self.tag_to_string(soup.title))
skip_tag = soup.find(True, {'name':'skip'}) skip_tag = soup.find(True, {'name':'skip'})
@ -864,9 +860,9 @@ class NYTimes(BasicNewsRecipe):
img = atag.find('img') img = atag.find('img')
if img is not None: if img is not None:
atag.replaceWith(img) atag.replaceWith(img)
elif not atag.has_key('href'): elif 'href' not in atag:
atag.replaceWith(atag.renderContents().decode('cp1252','replace')) atag.replaceWith(atag.renderContents().decode('cp1252','replace'))
elif not (atag['href'].startswith('http://www.nytimes') or atag['href'].startswith('http://pogue') or \ elif not (atag['href'].startswith('http://www.nytimes') or atag['href'].startswith('http://pogue') or
atag['href'].startswith('http://bits') or atag['href'].startswith('http://open')): atag['href'].startswith('http://bits') or atag['href'].startswith('http://open')):
atag.replaceWith(atag.renderContents().decode('cp1252','replace')) atag.replaceWith(atag.renderContents().decode('cp1252','replace'))
hdr = soup.find('address') hdr = soup.find('address')
@ -956,8 +952,10 @@ class NYTimes(BasicNewsRecipe):
year = str(st.tm_year) year = str(st.tm_year)
month = "%.2d" % st.tm_mon month = "%.2d" % st.tm_mon
day = "%.2d" % st.tm_mday day = "%.2d" % st.tm_mday
imgstartpos = popuphtml.find('http://graphics8.nytimes.com/images/' + year + '/' + month +'/' + day +'/') + len('http://graphics8.nytimes.com/images/' + year + '/' + month +'/' + day +'/') imgstartpos = popuphtml.find('http://graphics8.nytimes.com/images/' + year + '/' + month +'/' + day +'/') + \
highResImageLink = 'http://graphics8.nytimes.com/images/' + year + '/' + month +'/' + day +'/' + popuphtml[imgstartpos:popuphtml.find('.jpg',imgstartpos)+4] len('http://graphics8.nytimes.com/images/' + year + '/' + month +'/' + day +'/')
highResImageLink = 'http://graphics8.nytimes.com/images/' + year + '/' + \
month +'/' + day +'/' + popuphtml[imgstartpos:popuphtml.find('.jpg',imgstartpos)+4]
popupSoup = BeautifulSoup(popuphtml) popupSoup = BeautifulSoup(popuphtml)
highResTag = popupSoup.find('img', {'src':highResImageLink}) highResTag = popupSoup.find('img', {'src':highResImageLink})
if highResTag: if highResTag:
@ -987,7 +985,6 @@ class NYTimes(BasicNewsRecipe):
except: except:
self.log("Error removing Enlarge this text") self.log("Error removing Enlarge this text")
return self.strip_anchors(soup,False) return self.strip_anchors(soup,False)
def postprocess_html(self,soup,first_fetch): def postprocess_html(self,soup,first_fetch):
@ -1111,7 +1108,6 @@ class NYTimes(BasicNewsRecipe):
except: except:
self.log("ERROR: fixing credit format") self.log("ERROR: fixing credit format")
try: try:
# Change <h1> to <h3> - used in editorial blogs # Change <h1> to <h3> - used in editorial blogs
masthead = soup.find("h1") masthead = soup.find("h1")
@ -1205,4 +1201,3 @@ class NYTimes(BasicNewsRecipe):
self.log("Error creating article descriptions") self.log("Error creating article descriptions")
return return

68
recipes/nytimes_sub.recipe
View File

@ -33,7 +33,6 @@ class NYTimes(BasicNewsRecipe):
# and 30 will get the most popular measured over 30 days. # and 30 will get the most popular measured over 30 days.
# you still only get up to 20 articles in each category # you still only get up to 20 articles in each category
# set headlinesOnly to True for the headlines-only version. If True, webEdition is ignored. # set headlinesOnly to True for the headlines-only version. If True, webEdition is ignored.
headlinesOnly = False headlinesOnly = False
@ -82,6 +81,7 @@ class NYTimes(BasicNewsRecipe):
# The maximum number of articles that will be downloaded # The maximum number of articles that will be downloaded
max_articles_per_feed = 100 max_articles_per_feed = 100
use_embedded_content = False
# Whether to omit duplicates of articles (typically arsing when articles are indexed in # Whether to omit duplicates of articles (typically arsing when articles are indexed in
# more than one section). If True, only the first occurance will be downloaded. # more than one section). If True, only the first occurance will be downloaded.
@ -122,7 +122,6 @@ class NYTimes(BasicNewsRecipe):
(u'Tech - Open', u'http://open.blogs.nytimes.com/feed/') (u'Tech - Open', u'http://open.blogs.nytimes.com/feed/')
] ]
if headlinesOnly: if headlinesOnly:
title='New York Times Headlines' title='New York Times Headlines'
description = 'Headlines from the New York Times' description = 'Headlines from the New York Times'
@ -168,8 +167,7 @@ class NYTimes(BasicNewsRecipe):
cover_margins = (18,18,'grey99') cover_margins = (18,18,'grey99')
remove_tags_before = dict(id='article') keep_only_tags = dict(id=['article', 'story', 'content'])
remove_tags_after = dict(id='article')
remove_tags = [ remove_tags = [
dict(attrs={'class':[ dict(attrs={'class':[
'articleFooter', 'articleFooter',
@ -184,6 +182,7 @@ class NYTimes(BasicNewsRecipe):
'entry-response module', 'entry-response module',
'leftNavTabs', 'leftNavTabs',
'metaFootnote', 'metaFootnote',
'inside-story',
'module box nav', 'module box nav',
'nextArticleLink', 'nextArticleLink',
'nextArticleLink clearfix', 'nextArticleLink clearfix',
@ -223,6 +222,7 @@ class NYTimes(BasicNewsRecipe):
'credit' 'credit'
]}), ]}),
dict(attrs={'class':lambda x: x and 'related-coverage-marginalia' in x.split()}), dict(attrs={'class':lambda x: x and 'related-coverage-marginalia' in x.split()}),
dict(attrs={'class':lambda x: x and 'interactive' in x.split()}),
dict(name='div', attrs={'class':re.compile('toolsList')}), # bits dict(name='div', attrs={'class':re.compile('toolsList')}), # bits
dict(name='div', attrs={'class':re.compile('postNavigation')}), # bits dict(name='div', attrs={'class':re.compile('postNavigation')}), # bits
dict(name='div', attrs={'class':'tweet'}), dict(name='div', attrs={'class':'tweet'}),
@ -254,6 +254,7 @@ class NYTimes(BasicNewsRecipe):
'masthead-nav', 'masthead-nav',
'memberTools', 'memberTools',
'navigation', 'navigation-ghost', 'navigation-modal', 'navigation-edge', 'navigation', 'navigation-ghost', 'navigation-modal', 'navigation-edge',
'page-footer',
'portfolioInline', 'portfolioInline',
'readerReviews', 'readerReviews',
'readerReviewsCount', 'readerReviewsCount',
@ -270,11 +271,13 @@ class NYTimes(BasicNewsRecipe):
'related-content', # added for DealBook 'related-content', # added for DealBook
'whats-next', 'whats-next',
]), ]),
dict(name=['script', 'noscript', 'style','form','hr', 'button', 'meta'])] dict(name=['script', 'noscript', 'style','form','hr', 'button', 'meta', 'footer'])]
no_stylesheets = True no_stylesheets = True
extra_css = ''' extra_css = '''
.articleHeadline { text-align: left; margin-top:0.5em; margin-bottom:0.25em; } .articleHeadline { text-align: left; margin-top:0.5em; margin-bottom:0.25em; }
.credit { font-weight: normal; text-align: right; font-size: 50%; line-height:1em; margin-top:5px; margin-left:0; margin-right:0; margin-bottom: 0; } .credit { font-weight: normal; text-align: right; font-size:
50%; line-height:1em; margin-top:5px; margin-left:0;
margin-right:0; margin-bottom: 0; }
.byline { text-align: left; font-size: 50%; line-height:1em; margin-top:10px; margin-left:0; margin-right:0; margin-bottom: 0; } .byline { text-align: left; font-size: 50%; line-height:1em; margin-top:10px; margin-left:0; margin-right:0; margin-bottom: 0; }
.dateline { text-align: left; font-size: 50%; line-height:1em;margin-top:5px; margin-left:0; margin-right:0; margin-bottom: 0; } .dateline { text-align: left; font-size: 50%; line-height:1em;margin-top:5px; margin-left:0; margin-right:0; margin-bottom: 0; }
.kicker { font-size: 50%; line-height:1em;margin-top:5px; margin-left:0; margin-right:0; margin-bottom: 0; } .kicker { font-size: 50%; line-height:1em;margin-top:5px; margin-left:0; margin-right:0; margin-bottom: 0; }
@ -290,7 +293,6 @@ class NYTimes(BasicNewsRecipe):
.asidenote {color:blue;margin:0px 0px 0px 0px; padding: 0px 0px 0px 0px; font-size:100%;font-weight:bold;} .asidenote {color:blue;margin:0px 0px 0px 0px; padding: 0px 0px 0px 0px; font-size:100%;font-weight:bold;}
.source {text-align: left; font-size: x-small; }''' .source {text-align: left; font-size: x-small; }'''
articles = {} articles = {}
key = None key = None
ans = [] ans = []
@ -419,7 +421,6 @@ class NYTimes(BasicNewsRecipe):
def short_title(self): def short_title(self):
return self.title return self.title
def article_to_soup(self, url_or_raw, raw=False): def article_to_soup(self, url_or_raw, raw=False):
from contextlib import closing from contextlib import closing
import copy import copy
@ -453,7 +454,6 @@ class NYTimes(BasicNewsRecipe):
usrc = self.preprocess_raw_html(usrc, url_or_raw) usrc = self.preprocess_raw_html(usrc, url_or_raw)
return BeautifulSoup(usrc, markupMassage=nmassage) return BeautifulSoup(usrc, markupMassage=nmassage)
def massageNCXText(self, description): def massageNCXText(self, description):
# Kindle TOC descriptions won't render certain characters # Kindle TOC descriptions won't render certain characters
if description: if description:
@ -508,7 +508,7 @@ class NYTimes(BasicNewsRecipe):
if authorAttribution: if authorAttribution:
author = self.tag_to_string(authorAttribution, use_alt=False) author = self.tag_to_string(authorAttribution, use_alt=False)
feed = self.key if self.key is not None else 'Uncategorized' feed = self.key if self.key is not None else 'Uncategorized'
if not self.articles.has_key(feed): if feed not in self.articles:
self.ans.append(feed) self.ans.append(feed)
self.articles[feed] = [] self.articles[feed] = []
self.articles[feed].append( self.articles[feed].append(
@ -543,7 +543,6 @@ class NYTimes(BasicNewsRecipe):
desc = '' desc = ''
return(title,url,author,desc) return(title,url,author,desc)
have_emailed = False have_emailed = False
emailed_soup = self.index_to_soup('http://www.nytimes.com/most-popular-emailed?period='+self.popularPeriod) emailed_soup = self.index_to_soup('http://www.nytimes.com/most-popular-emailed?period='+self.popularPeriod)
for h3tag in emailed_soup.findAll('h3'): for h3tag in emailed_soup.findAll('h3'):
@ -572,7 +571,7 @@ class NYTimes(BasicNewsRecipe):
dict(title=title, url=url, date=strftime('%a, %d %b'), dict(title=title, url=url, date=strftime('%a, %d %b'),
description=desc, author=author, description=desc, author=author,
content='')) content=''))
viewed_ans = [(k, popular_articles[k]) for k in key_list if popular_articles.has_key(k)] viewed_ans = [(k, popular_articles[k]) for k in key_list if k in popular_articles]
for x in viewed_ans: for x in viewed_ans:
ans.append(x) ans.append(x)
return ans return ans
@ -595,10 +594,10 @@ class NYTimes(BasicNewsRecipe):
tech_articles[f.title] = [] tech_articles[f.title] = []
for a in f.articles: for a in f.articles:
tech_articles[f.title].append( tech_articles[f.title].append(
dict(title=a.title, url=a.url, date=a.date, dict(title=a.title, url=a.url.partition('?')[0], date=a.date,
description=a.summary, author=a.author, description=a.summary, author=a.author,
content=a.content)) content=a.content))
tech_ans = [(k, tech_articles[k]) for k in key_list if tech_articles.has_key(k)] tech_ans = [(k, tech_articles[k]) for k in key_list if k in tech_articles]
for x in tech_ans: for x in tech_ans:
ans.append(x) ans.append(x)
return ans return ans
@ -637,10 +636,9 @@ class NYTimes(BasicNewsRecipe):
for lidiv in div.findAll('li'): for lidiv in div.findAll('li'):
self.handle_article(lidiv) self.handle_article(lidiv)
self.ans = [(k, self.articles[k]) for k in self.ans if self.articles.has_key(k)] self.ans = [(k, self.articles[k]) for k in self.ans if k in self.articles]
return self.filter_ans(self.get_tech_feeds(self.get_popular_articles(self.ans))) return self.filter_ans(self.get_tech_feeds(self.get_popular_articles(self.ans)))
def parse_todays_index(self): def parse_todays_index(self):
soup = self.index_to_soup('http://www.nytimes.com/pages/todayspaper/index.html') soup = self.index_to_soup('http://www.nytimes.com/pages/todayspaper/index.html')
@ -670,7 +668,7 @@ class NYTimes(BasicNewsRecipe):
if not skipping: if not skipping:
self.handle_article(lidiv) self.handle_article(lidiv)
self.ans = [(k, self.articles[k]) for k in self.ans if self.articles.has_key(k)] self.ans = [(k, self.articles[k]) for k in self.ans if k in self.articles]
return self.filter_ans(self.get_tech_feeds(self.get_popular_articles(self.ans))) return self.filter_ans(self.get_tech_feeds(self.get_popular_articles(self.ans)))
def parse_headline_index(self): def parse_headline_index(self):
@ -716,13 +714,13 @@ class NYTimes(BasicNewsRecipe):
description = self.tag_to_string(desc,use_alt=False) description = self.tag_to_string(desc,use_alt=False)
else: else:
description = '' description = ''
if not self.articles.has_key(section_name): if section_name not in self.articles:
self.ans.append(section_name) self.ans.append(section_name)
self.articles[section_name] = [] self.articles[section_name] = []
print('Title '+title+' author '+author) print('Title '+title+' author '+author)
self.articles[section_name].append(dict(title=title, url=url, date=pubdate, description=description, author=author, content='')) self.articles[section_name].append(dict(title=title, url=url, date=pubdate, description=description, author=author, content=''))
self.ans = [(k, self.articles[k]) for k in self.ans if self.articles.has_key(k)] self.ans = [(k, self.articles[k]) for k in self.ans if k in self.articles]
return self.filter_ans(self.ans) return self.filter_ans(self.ans)
def parse_index(self): def parse_index(self):
@ -742,7 +740,7 @@ class NYTimes(BasicNewsRecipe):
if kill_all or (self.recursions==0): if kill_all or (self.recursions==0):
a.replaceWith(self.tag_to_string(a,False)) a.replaceWith(self.tag_to_string(a,False))
else: else:
if a.has_key('href'): if 'href' in a:
if a['href'].startswith('http://www.nytimes'): if a['href'].startswith('http://www.nytimes'):
if not a['href'].endswith('pagewanted=all'): if not a['href'].endswith('pagewanted=all'):
url = re.sub(r'\?.*', '', a['href']) url = re.sub(r'\?.*', '', a['href'])
@ -750,13 +748,13 @@ class NYTimes(BasicNewsRecipe):
a.replaceWith(self.tag_to_string(a,False)) a.replaceWith(self.tag_to_string(a,False))
else: else:
a['href'] = url+'?pagewanted=all' a['href'] = url+'?pagewanted=all'
elif not (a['href'].startswith('http://pogue') or \ elif not (a['href'].startswith('http://pogue') or
a['href'].startswith('http://bits') or \ a['href'].startswith('http://bits') or
a['href'].startswith('http://travel') or \ a['href'].startswith('http://travel') or
a['href'].startswith('http://business') or \ a['href'].startswith('http://business') or
a['href'].startswith('http://tech') or \ a['href'].startswith('http://tech') or
a['href'].startswith('http://health') or \ a['href'].startswith('http://health') or
a['href'].startswith('http://dealbook') or \ a['href'].startswith('http://dealbook') or
a['href'].startswith('http://open')): a['href'].startswith('http://open')):
a.replaceWith(self.tag_to_string(a,False)) a.replaceWith(self.tag_to_string(a,False))
return soup return soup
@ -771,7 +769,7 @@ class NYTimes(BasicNewsRecipe):
return None return None
## print("HANDLING AD FORWARD:") ## print("HANDLING AD FORWARD:")
## print(soup) # print(soup)
if self.keep_only_tags: if self.keep_only_tags:
body = Tag(soup, 'body') body = Tag(soup, 'body')
try: try:
@ -809,7 +807,6 @@ class NYTimes(BasicNewsRecipe):
return soup return soup
def preprocess_html(self, soup): def preprocess_html(self, soup):
#print(strftime("%H:%M:%S")+" -- PREPROCESS TITLE="+self.tag_to_string(soup.title)) #print(strftime("%H:%M:%S")+" -- PREPROCESS TITLE="+self.tag_to_string(soup.title))
skip_tag = soup.find(True, {'name':'skip'}) skip_tag = soup.find(True, {'name':'skip'})
@ -871,9 +868,9 @@ class NYTimes(BasicNewsRecipe):
img = atag.find('img') img = atag.find('img')
if img is not None: if img is not None:
atag.replaceWith(img) atag.replaceWith(img)
elif not atag.has_key('href'): elif 'href' not in atag:
atag.replaceWith(atag.renderContents().decode('cp1252','replace')) atag.replaceWith(atag.renderContents().decode('cp1252','replace'))
elif not (atag['href'].startswith('http://www.nytimes') or atag['href'].startswith('http://pogue') or \ elif not (atag['href'].startswith('http://www.nytimes') or atag['href'].startswith('http://pogue') or
atag['href'].startswith('http://bits') or atag['href'].startswith('http://open')): atag['href'].startswith('http://bits') or atag['href'].startswith('http://open')):
atag.replaceWith(atag.renderContents().decode('cp1252','replace')) atag.replaceWith(atag.renderContents().decode('cp1252','replace'))
hdr = soup.find('address') hdr = soup.find('address')
@ -963,8 +960,10 @@ class NYTimes(BasicNewsRecipe):
year = str(st.tm_year) year = str(st.tm_year)
month = "%.2d" % st.tm_mon month = "%.2d" % st.tm_mon
day = "%.2d" % st.tm_mday day = "%.2d" % st.tm_mday
imgstartpos = popuphtml.find('http://graphics8.nytimes.com/images/' + year + '/' + month +'/' + day +'/') + len('http://graphics8.nytimes.com/images/' + year + '/' + month +'/' + day +'/') imgstartpos = popuphtml.find('http://graphics8.nytimes.com/images/' + year + '/' + month +'/' + day +'/') + \
highResImageLink = 'http://graphics8.nytimes.com/images/' + year + '/' + month +'/' + day +'/' + popuphtml[imgstartpos:popuphtml.find('.jpg',imgstartpos)+4] len('http://graphics8.nytimes.com/images/' + year + '/' + month +'/' + day +'/')
highResImageLink = 'http://graphics8.nytimes.com/images/' + year + '/' + \
month +'/' + day +'/' + popuphtml[imgstartpos:popuphtml.find('.jpg',imgstartpos)+4]
popupSoup = BeautifulSoup(popuphtml) popupSoup = BeautifulSoup(popuphtml)
highResTag = popupSoup.find('img', {'src':highResImageLink}) highResTag = popupSoup.find('img', {'src':highResImageLink})
if highResTag: if highResTag:
@ -994,7 +993,6 @@ class NYTimes(BasicNewsRecipe):
except: except:
self.log("Error removing Enlarge this text") self.log("Error removing Enlarge this text")
return self.strip_anchors(soup,False) return self.strip_anchors(soup,False)
def postprocess_html(self,soup,first_fetch): def postprocess_html(self,soup,first_fetch):
@ -1118,7 +1116,6 @@ class NYTimes(BasicNewsRecipe):
except: except:
self.log("ERROR: fixing credit format") self.log("ERROR: fixing credit format")
try: try:
# Change <h1> to <h3> - used in editorial blogs # Change <h1> to <h3> - used in editorial blogs
masthead = soup.find("h1") masthead = soup.find("h1")
@ -1212,4 +1209,3 @@ class NYTimes(BasicNewsRecipe):
self.log("Error creating article descriptions") self.log("Error creating article descriptions")
return return