Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

...

Browse Source
This commit is contained in:
Kovid Goyal 2010-12-17 19:06:32 -07:00
parent fcb501c1aa
commit 7ab3ce7fe4
2 changed files with 41 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
resources/recipes/nejm.recipe
View File

@ -44,55 +44,44 @@ class NYTimes(BasicNewsRecipe):
def parse_index(self): def parse_index(self):
parse_soup = self.nejm_get_index() parse_soup = self.nejm_get_index()
div = parse_soup.find(id='centerTOC')
current_section = None
current_articles = []
feeds = [] feeds = []
for x in div.findAll(True):
if x.name == 'img' and '/toc/' in x.get('src', '') and 'uarrow.gif' not in x.get('src', ''): div = parse_soup.find(attrs={'class':'tocContent'})
# Section heading found for group in div.findAll(attrs={'class':'articleGrouping'}):
if current_articles and current_section and 'Week in the' not in current_section: feed_title = group.find(attrs={'class':'articleType'})
feeds.append((current_section, current_articles)) if feed_title is None:
current_section = x.get('alt') continue
current_articles = [] feed_title = self.tag_to_string(feed_title)
self.log('\tFound section:', current_section) articles = []
if current_section is not None and x.name == 'strong': self.log('Found section:', feed_title)
title = self.tag_to_string(x) for art in group.findAll(attrs={'class':lambda x: x and 'articleEntry'
a = x.parent.find('a', href=lambda x: x and '/full/' in x) in x}):
link = art.find(attrs={'class':lambda x:x and 'articleLink' in
x})
if link is None:
continue
a = link.find('a', href=True)
if a is None: if a is None:
continue continue
url = a.get('href', False) url = a.get('href')
if not url or not title:
continue
if url.startswith('/'): if url.startswith('/'):
url = 'http://content.nejm.org'+url url = 'http://www.nejm.org'+url
self.log('\t\tFound article:', title) title = self.tag_to_string(a)
self.log('\t\t\t', url) self.log.info('\tFound article:', title, 'at', url)
if url.startswith('/'): article = {'title':title, 'url':url, 'date':''}
url = 'http://online.wsj.com'+url au = art.find(attrs={'class':'articleAuthors'})
current_articles.append({'title': title, 'url':url, if au is not None:
'description':'', 'date':''}) article['author'] = self.tag_to_string(au)
desc = art.find(attrs={'class':'hover_text'})
if current_articles and current_section: if desc is not None:
feeds.append((current_section, current_articles)) desc = self.tag_to_string(desc)
if 'author' in article:
desc = ' by ' + article['author'] + ' ' +desc
article['description'] = desc
articles.append(article)
if articles:
feeds.append((feed_title, articles))
return feeds return feeds
def preprocess_html(self, soup):
for a in soup.findAll(text=lambda x: x and '[in this window]' in x):
a = a.findParent('a')
url = a.get('href', None)
if not url:
continue
if url.startswith('/'):
url = 'http://content.nejm.org'+url
isoup = self.index_to_soup(url)
img = isoup.find('img', src=lambda x: x and
x.startswith('/content/'))
if img is not None:
img.extract()
table = a.findParent('table')
table.replaceWith(img)
return soup

1
src/calibre/web/feeds/news.py
View File

@ -548,6 +548,7 @@ class BasicNewsRecipe(Recipe):
} }
For an example, see the recipe for downloading `The Atlantic`. For an example, see the recipe for downloading `The Atlantic`.
In addition, you can add 'author' for the author of the article.
''' '''
raise NotImplementedError raise NotImplementedError