Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Update New York Times Book Review

Browse Source
This commit is contained in:
Kovid Goyal 2015-12-31 15:18:30 +05:30
parent fc687e9b14
commit f624334abf
1 changed files with 31 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
recipes/nytimesbook.recipe
View File

@ -3,56 +3,48 @@ from calibre.web.feeds.news import BasicNewsRecipe
class NewYorkTimesBookReview(BasicNewsRecipe): class NewYorkTimesBookReview(BasicNewsRecipe):
title = u'New York Times Book Review' title = u'New York Times Book Review'
language = 'en' language = 'en'
description = 'The New York Times Sunday Book Review. Best downloaded on Fridays to avoid the ads that the New York Times shows of the first few days of the week.' # noqa description = 'The New York Times Sunday Book Review'
__author__ = 'Kovid Goyal' __author__ = 'Kovid Goyal'
no_stylesheets = True no_stylesheets = True
no_javascript = True no_javascript = True
auto_cleanup = True ignore_duplicate_articles = {'title', 'url'}
#keep_only_tags = [dict(id='article'), dict(id=lambda x:x and x.startswith('entry-'))] encoding = 'utf-8'
# remove_tags = [
#dict(attrs={'class':['articleBottomExtra', 'shareToolsBox', 'singleAd']}), keep_only_tags = [
#dict(attrs={'class':lambda x: x and ('shareTools' in x or 'enlargeThis' in x)}), dict(id=['story-heading', 'story-meta-footer']),
#] dict(itemprop=['associatedMedia', 'articleBody', 'reviewBody']),
]
def parse_index(self): def parse_index(self):
soup = self.index_to_soup('http://www.nytimes.com/pages/books/review/index.html') soup = self.index_to_soup('http://www.nytimes.com/pages/books/review/index.html')
# Find TOC # Find TOC
toc = soup.find('div', id='main').find( toc = soup.find('div', attrs={'class':'rank'})
'div', attrs={'class':'abColumn'}) main_articles, articles = [], []
feeds = [] feeds = [('Features', main_articles), ('Latest', articles)]
articles = [] for h2 in toc.findAll('h2', attrs={'class':'headline'}):
section_title = 'Features' a = h2.find('a', href=True)
for x in toc.findAll(['div', 'h3', 'h6'], attrs={'class':['story', 'sectionHeader', 'ledeStory']}): if a is not None:
if x['class'] == 'sectionHeader':
if articles:
feeds.append((section_title, articles))
section_title = self.tag_to_string(x)
articles = []
self.log('Found section:', section_title)
continue
if x['class'] in {'story', 'ledeStory'}:
tt = 'h3' if x['class'] == 'story' else 'h1'
try:
a = x.find(tt).find('a', href=True)
except AttributeError:
continue
title = self.tag_to_string(a) title = self.tag_to_string(a)
url = a['href'] + '&pagewanted=all' url = a['href']
self.log('\tFound article:', title, url)
desc = '' desc = ''
byline = x.find('h6', attrs={'class':'byline'}) p = h2.findNextSibling('p', attrs={'class':'summary'})
if byline is not None: if p:
desc = self.tag_to_string(byline) desc = self.tag_to_string(p)
summary = x.find('p', attrs={'class':'summary'}) main_articles.append({'title':title, 'url':url, 'description':desc})
if summary is not None: self.log('Found:', title, 'at', url)
desc += self.tag_to_string(summary)
if desc: if desc:
self.log('\t\t', desc) self.log('\t', desc)
articles.append({'title':title, 'url':url, 'date':'', for li in soup.find(id='latest-panel').find('ol').findAll('li'):
'description':desc}) a = li.find('a', attrs={'class':'story-link'}, href=True)
url = a['href']
m = a.find(attrs={'class':'story-meta'})
title = self.tag_to_string(m.find('h2'))
desc = self.tag_to_string(m.find(attrs={'class':'summary'}))
articles.append({'title':title, 'url':url, 'description':desc})
self.log('Found:', title, 'at', url)
if desc:
self.log('\t', desc)
return feeds return feeds