Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-06-23 15:30:45 -04:00
Code Issues Packages Projects Releases Wiki Activity

Update Scientific American

Browse Source 
Needs testing with an actual subscription
This commit is contained in:
Kovid Goyal 2014-12-10 10:12:03 +05:30
parent 609c2de0e4
commit 516a4cf731
1 changed files with 74 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

145
recipes/scientific_american.recipe
View File

@ -1,95 +1,96 @@
#!/usr/bin/env python #!/usr/bin/env python
__license__ = 'GPL v3' __license__ = 'GPL v3'
import re
from calibre.web.feeds.news import BasicNewsRecipe from calibre.web.feeds.news import BasicNewsRecipe
try:
from calibre.web.feeds.jsnews import CSSSelect
except ImportError:
def CSSSelect(expr):
from cssselect import HTMLTranslator
from lxml.etree import XPath
return XPath(HTMLTranslator().css_to_xpath(expr))
def absurl(url):
if url.startswith('/'):
url = 'http://www.scientificamerican.com' + url
return url
class ScientificAmerican(BasicNewsRecipe): class ScientificAmerican(BasicNewsRecipe):
title = u'Scientific American' title = u'Scientific American'
description = u'Popular Science. Monthly magazine.' description = u'Popular Science. Monthly magazine.'
category = 'science' category = 'science'
__author__ = 'Starson17' __author__ = 'Kovid Goyal'
no_stylesheets = True no_stylesheets = True
use_embedded_content = False
language = 'en' language = 'en'
publisher = 'Nature Publishing Group' publisher = 'Nature Publishing Group'
remove_empty_feeds = True remove_empty_feeds = True
remove_javascript = True remove_javascript = True
oldest_article = 30
max_articles_per_feed = 100
conversion_options = {'linearize_tables' : True needs_subscription = 'optional'
, 'comment' : description
, 'tags' : category
, 'publisher' : publisher
, 'language' : language
}
keep_only_tags = [ keep_only_tags = [
dict(name='h2', attrs={'class':'articleTitle'}) dict(attrs={'class':['article-title', 'article-dek', 'article-author article-date', 'article-content', 'article-slatwallPayWall']}),
,dict(name='p', attrs={'id':'articleDek'})
,dict(name='p', attrs={'class':'articleInfo'})
,dict(name='div', attrs={'id':['articleContent']})
,dict(name='img', attrs={'src':re.compile(r'/media/inline/blog/Image/', re.DOTALL|re.IGNORECASE)})
] ]
remove_tags = [dict(name='a', attrs={'class':'tinyCommentCount'}) def get_browser(self, *args):
,dict(name='div', attrs={'id':'bigCoverModule'}) br = BasicNewsRecipe.get_browser(self)
,dict(name='div', attrs={'class':'addInfo'}) if self.username and self.password:
] br.open('https://www.scientificamerican.com/my-account/login/')
br.select_form(predicate=lambda f:f.attrs.get('id') == 'login')
br['emailAddress'] = self.username
br['password'] = self.password
br.submit()
return br
def parse_index(self): def parse_index(self):
soup = self.index_to_soup('http://www.scientificamerican.com/sciammag/') # Get the cover, date and issue URL
issuetag = soup.find('p',attrs={'id':'articleDek'}) root = self.index_to_soup('http://www.scientificamerican.com/sciammag/', as_tree=True)
self.timefmt = ' [%s]'%(self.tag_to_string(issuetag)) for a in CSSSelect('.archiveIssues a.cover[href]')(root):
img = soup.find('img', alt='Scientific American Magazine', src=True) self.cover_url = absurl(CSSSelect('img[src]')(a)[0].get('src'))
if img is not None: root = self.index_to_soup(absurl(a.get('href')), as_tree=True)
self.cover_url = img['src'] for a in a.xpath('following-sibling::a[@href]'):
features, feeds = [], [] self.timefmt = self.tag_to_string(a).strip()
for a in soup.find(attrs={'class':'doubleWide'}).find(attrs={'class':'primaryCol'}).findAll('a',attrs={'title':'Feature'}): break
if a is None: break
continue else:
desc = '' raise ValueError('The Scientific American website has changed, this recipe needs to be updated')
s = a.parent.parent.find(attrs={'class':'dek'})
desc = self.tag_to_string(s) # Now parse the actual issue to get the list of articles
article = { feeds = []
'url' : a['href'], for i, div in enumerate(CSSSelect('div.toc-features, div.toc-departments')(root)):
'title' : self.tag_to_string(a), if i == 0:
'date' : '', feeds.append(('Features', list(self.parse_sciam_features(div))))
'description' : desc, else:
} feeds.extend(self.parse_sciam_departments(div))
features.append(article)
feeds.append(('Features', features))
department = []
title = None
for li in soup.find(attrs={'class':'secondaryCol'}).findAll('li'):
if 'department.cfm' in li.a['href']:
if department:
feeds.append((title, department))
title = self.tag_to_string(li.a)
department = []
if 'article.cfm' in li.h3.a['href']:
article = {
'url' : li.h3.a['href'],
'title' : self.tag_to_string(li.h3.a),
'date': '',
'description': self.tag_to_string(li.p),
}
department.append(article)
if department:
feeds.append((title, department))
return feeds return feeds
def postprocess_html(self, soup, first_fetch): def parse_sciam_features(self, div):
for item in soup.findAll('a'): for h4 in CSSSelect('li a[href] h4')(div):
if 'topic.cfm' in item['href']: title = self.tag_to_string(h4)
item.replaceWith(item.string) a = h4.getparent()
return soup url = absurl(a.get('href'))
desc = ''
extra_css = ''' for span in a.xpath('following-sibling::span'):
p{font-weight: normal; font-size:small} desc = self.tag_to_string(span)
li{font-weight: normal; font-size:small} break
.headline p{font-size:x-small; font-family:Arial,Helvetica,sans-serif;} self.log('Found feature article: %s at %s' % (title, url))
h2{font-size:large; font-family:Arial,Helvetica,sans-serif;} self.log('\t' + desc)
h3{font-size:x-small;font-family:Arial,Helvetica,sans-serif;} yield {'title':title, 'url':url, 'description':desc}
'''
def parse_sciam_departments(self, div):
section_title, articles = 'Unknown', []
for x in CSSSelect('li a[href] h3, li span.deptTitle a[href]')(div):
if x.tag == 'a':
if articles:
yield section_title, list(articles)
section_title = self.tag_to_string(x)
del articles[:]
self.log('\nFound section: %s' % section_title)
else:
title = self.tag_to_string(x)
a = x.getparent()
url = absurl(a.get('href'))
articles.append({'title':title, 'url':url, 'description':''})
self.log('\tFound article: %s at %s' % (title, url))