Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Update New England Journal of Medicine

Browse Source 
Fixes #1781700 [Private bug](https://bugs.launchpad.net/calibre/+bug/1781700)
This commit is contained in:
Kovid Goyal 2018-07-29 21:54:16 +05:30
parent e9ef4a9bbf
commit f9ae34cc0f
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
1 changed files with 38 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
recipes/nejm.recipe
View File

@ -2,7 +2,13 @@
from calibre.web.feeds.recipes import BasicNewsRecipe from calibre.web.feeds.recipes import BasicNewsRecipe
class NYTimes(BasicNewsRecipe): def classes(classes):
q = frozenset(classes.split(' '))
return dict(attrs={
'class': lambda x: x and frozenset(x.split()).intersection(q)})
class NEJM(BasicNewsRecipe):
title = 'New England Journal of Medicine' title = 'New England Journal of Medicine'
__author__ = 'Kovid Goyal' __author__ = 'Kovid Goyal'
@ -12,12 +18,16 @@ class NYTimes(BasicNewsRecipe):
language = 'en' language = 'en'
no_stylesheets = True no_stylesheets = True
keep_only_tags = dict(id='content') keep_only_tags = [
dict(id='content')
]
remove_tags_after = dict(id='article_references')
remove_attributes = ['width', 'height']
# TO LOGIN # TO LOGIN
def get_browser(self): def get_browser(self):
br = BasicNewsRecipe.get_browser(self) br = BasicNewsRecipe.get_browser(self)
br.open('http://www.nejm.org/action/showLogin?uri=http://www.nejm.org/') br.open('https://www.nejm.org/action/showLogin?uri=http%3A%2F%2Fwww.nejm.org%2F')
br.select_form(name='frmLogin') br.select_form(name='frmLogin')
br['login'] = self.username br['login'] = self.username
br['password'] = self.password br['password'] = self.password
@ -29,48 +39,36 @@ class NYTimes(BasicNewsRecipe):
# TO GET ARTICLE TOC # TO GET ARTICLE TOC
def nejm_get_index(self): def nejm_get_index(self):
return self.index_to_soup('http://content.nejm.org/current.dtl') return self.index_to_soup('https://www.nejm.org/toc/nejm/medical-journal')
# To parse artice toc # To parse artice toc
def parse_index(self): def parse_index(self):
parse_soup = self.nejm_get_index() soup = self.nejm_get_index()
feeds = [] feeds = []
current_section = None
div = parse_soup.find(attrs={'class': 'tocContent'})
for group in div.findAll(attrs={'class': 'articleGrouping'}):
feed_title = group.find(attrs={'class': 'articleType'})
if feed_title is None:
continue
feed_title = self.tag_to_string(feed_title)
articles = [] articles = []
self.log('Found section:', feed_title) div = soup.find(**classes('pagefulltext'))
for art in group.findAll(attrs={'class': lambda x: x and 'articleEntry' for x in div.findAll(name=['h2', 'li']):
in x}): if x.name == 'h2':
link = art.find(attrs={'class': lambda x: x and 'articleLink' in if current_section and articles:
x}) feeds.append((current_section, articles))
if link is None: current_section = self.tag_to_string(x).strip()
continue articles = []
a = link.find('a', href=True) if current_section:
self.log(current_section)
else:
a = x.find('a')
if a is None: if a is None:
continue continue
url = a.get('href') title = self.tag_to_string(a.find('strong')).strip()
if url.startswith('/'): blurb = a.find(**classes('f-blurb'))
url = 'http://www.nejm.org' + url desc = ''
title = self.tag_to_string(a) if blurb is not None:
self.log.info('\tFound article:', title, 'at', url) desc = self.tag_to_string(blurb)
article = {'title': title, 'url': url, 'date': ''} self.log('\t', title)
au = art.find(attrs={'class': 'articleAuthors'}) articles.append({'title': title, 'url': 'https://www.nejm.org' + a['href'], 'description': desc})
if au is not None:
article['author'] = self.tag_to_string(au) if current_section and articles:
desc = art.find(attrs={'class': 'hover_text'}) feeds.append((current_section, articles))
if desc is not None:
desc = self.tag_to_string(desc)
if 'author' in article:
desc = ' by ' + article['author'] + ' ' + desc
article['description'] = desc
articles.append(article)
if articles:
feeds.append((feed_title, articles))
return feeds return feeds