Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Fix The Atlantic

Browse Source
This commit is contained in:
Kovid Goyal 2011-01-06 12:43:01 -07:00
parent 3e1a531444
commit 4f3424ab72
1 changed files with 17 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
resources/recipes/atlantic.recipe
View File

@ -5,7 +5,7 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
''' '''
theatlantic.com theatlantic.com
''' '''
import string, re import re
from calibre.web.feeds.news import BasicNewsRecipe from calibre.web.feeds.news import BasicNewsRecipe
from calibre.ebooks.BeautifulSoup import Tag, NavigableString from calibre.ebooks.BeautifulSoup import Tag, NavigableString
@ -33,25 +33,27 @@ class TheAtlantic(BasicNewsRecipe):
articles = [] articles = []
soup = self.index_to_soup(self.INDEX) soup = self.index_to_soup(self.INDEX)
sectit = soup.find('h1', attrs={'class':'sectionTitle'}) ts = soup.find(id='magazineTopStories')
if sectit is not None: ds = self.tag_to_string(ts.find('h1')).split(':')[-1]
texts = self.tag_to_string(sectit).strip().split()[-2:] self.timefmt = ' [%s]'%ds
if texts:
self.timefmt = ' [%s]'%(' '.join(texts))
cover = soup.find('img', src=True, attrs={'class':'cover'}) cover = soup.find('img', src=True, attrs={'class':'cover'})
if cover is not None: if cover is not None:
self.cover_url = cover['src'] self.cover_url = cover['src']
feeds = [] feeds = []
seen_titles = set([])
for section in soup.findAll('div', attrs={'class':'magazineSection'}): for section in soup.findAll('div', attrs={'class':'magazineSection'}):
section_title = section.find(attrs={'class':'sectionHeader'}) section_title = self.tag_to_string(section.find('h2'))
section_title = string.capwords(self.tag_to_string(section_title))
self.log('Found section:', section_title) self.log('Found section:', section_title)
articles = [] articles = []
for post in section.findAll('div', attrs={'class':'post'}): for post in section.findAll('div', attrs={'class':lambda x : x and
'post' in x}):
h = post.find(['h3', 'h4']) h = post.find(['h3', 'h4'])
title = self.tag_to_string(h) title = self.tag_to_string(h)
if title in seen_titles:
continue
seen_titles.add(title)
a = post.find('a', href=True) a = post.find('a', href=True)
url = a['href'] url = a['href']
if url.startswith('/'): if url.startswith('/'):
@ -64,36 +66,23 @@ class TheAtlantic(BasicNewsRecipe):
self.log('\t\t', desc) self.log('\t\t', desc)
articles.append({'title':title, 'url':url, 'description':desc, articles.append({'title':title, 'url':url, 'description':desc,
'date':''}) 'date':''})
feeds.append((section_title, articles)) if articles:
feeds.append((section_title, articles))
poems = [] poems = []
self.log('Found section: Poems') self.log('Found section: Poems')
for poem in soup.findAll('div', attrs={'class':'poem'}): pd = soup.find('h2', text='Poetry').parent.parent
title = self.tag_to_string(poem.find('h4')) for poem in pd.findAll('h4'):
desc = self.tag_to_string(poem.find(attrs={'class':'author'})) title = self.tag_to_string(poem)
url = poem.find('a')['href'] url = poem.find('a')['href']
if url.startswith('/'): if url.startswith('/'):
url = 'http://www.theatlantic.com' + url url = 'http://www.theatlantic.com' + url
self.log('\tFound article:', title, 'at', url) self.log('\tFound article:', title, 'at', url)
self.log('\t\t', desc) poems.append({'title':title, 'url':url, 'description':'',
poems.append({'title':title, 'url':url, 'description':desc,
'date':''}) 'date':''})
if poems: if poems:
feeds.append(('Poems', poems)) feeds.append(('Poems', poems))
div = soup.find(id='advice')
if div is not None:
self.log('Found section: Advice')
title = self.tag_to_string(div.find('h4'))
url = div.find('a')['href']
if url.startswith('/'):
url = 'http://www.theatlantic.com' + url
desc = self.tag_to_string(div.find('p'))
self.log('\tFound article:', title, 'at', url)
self.log('\t\t', desc)
feeds.append(('Advice', [{'title':title, 'url':url, 'description':desc,
'date':''}]))
return feeds return feeds
def postprocess_html(self, soup, first): def postprocess_html(self, soup, first):