Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Update Scientific American

Browse Source
This commit is contained in:
Kovid Goyal 2016-01-20 19:42:16 +05:30
parent 383e16cff2
commit a0b56ef2a7
1 changed files with 33 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
recipes/scientific_american.recipe
View File

@ -3,12 +3,16 @@ __license__ = 'GPL v3'
from calibre.web.feeds.news import BasicNewsRecipe from calibre.web.feeds.news import BasicNewsRecipe
from calibre.utils.date import now from calibre.utils.date import now
from css_selectors import Select
def absurl(url): def absurl(url):
if url.startswith('/'): if url.startswith('/'):
url = 'http://www.scientificamerican.com' + url url = 'http://www.scientificamerican.com' + url
return url return url
keep_classes = {'article-header', 'article-content', 'article-media', 'article-author', 'article-text'}
remove_classes = {'aside-banner', 'moreToExplore', 'article-footer'}
class ScientificAmerican(BasicNewsRecipe): class ScientificAmerican(BasicNewsRecipe):
title = u'Scientific American' title = u'Scientific American'
description = u'Popular Science. Monthly magazine. Should be downloaded around the middle of each month.' description = u'Popular Science. Monthly magazine. Should be downloaded around the middle of each month.'
@ -24,13 +28,11 @@ class ScientificAmerican(BasicNewsRecipe):
needs_subscription = 'optional' needs_subscription = 'optional'
keep_only_tags = [ keep_only_tags = [
dict(attrs={'class':['article-title', 'article-dek', 'article-author article-date', 'article-complementary', 'article-slatwallPayWall']}), dict(attrs={'class':lambda x: x and bool(set(x.split()).intersection(keep_classes))}),
dict(attrs={'class':lambda x: x and 'article-content' in x.split()}),
] ]
remove_tags = [ remove_tags = [
dict(attrs={'class':['article-footer']}), dict(attrs={'class':lambda x: x and bool(set(x.split()).intersection(remove_classes))}),
dict(id=['seeAlsoLinks']), dict(id=['seeAlsoLinks']),
dict(attrs={'class':lambda x: x and 'moreToExplore' in x.split()}),
] ]
def get_browser(self, *args): def get_browser(self, *args):
@ -46,55 +48,57 @@ class ScientificAmerican(BasicNewsRecipe):
def parse_index(self): def parse_index(self):
# Get the cover, date and issue URL # Get the cover, date and issue URL
root = self.index_to_soup('http://www.scientificamerican.com/sciammag/', as_tree=True) root = self.index_to_soup('http://www.scientificamerican.com/sciammag/', as_tree=True)
for a in root.xpath('''descendant-or-self::*[@class and contains(concat(' ', normalize-space(@class), ' '), ' archiveIssues ')]/descendant-or-self::*/a[@class and contains(concat(' ', normalize-space(@class), ' '), ' cover ') and (@href)]'''): # noqa select = Select(root)
url = a.get('href') for a in select('#sa_body .store-listing__img a[href]'):
url = absurl(a.get('href'))
month = int(filter(None, url.split('/'))[-1].partition('-')[0]) month = int(filter(None, url.split('/'))[-1].partition('-')[0])
if month > now().month: if month > now().month:
continue continue
self.cover_url = absurl(a.xpath('descendant-or-self::img[@src]')[0].get('src')) for source in a.xpath('descendant::source'):
root = self.index_to_soup(absurl(a.get('href')), as_tree=True) self.cover_url = absurl(source.get('srcset').split()[0])
for a in a.xpath('following-sibling::a[@href]'):
self.timefmt = self.tag_to_string(a).strip()
break break
break break
else: else:
raise ValueError('The Scientific American website has changed, this recipe needs to be updated') raise ValueError('The Scientific American website has changed, this recipe needs to be updated')
# Now parse the actual issue to get the list of articles # Now parse the actual issue to get the list of articles
select = Select(self.index_to_soup(url, as_tree=True))
feeds = [] feeds = []
for i, div in enumerate(root.xpath('''descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' toc-features ')] | descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' toc-departments ')]''')): # noqa for i, section in enumerate(select('#sa_body .toc-articles')):
if i == 0: if i == 0:
feeds.append(('Features', list(self.parse_sciam_features(div)))) feeds.append(('Features', list(self.parse_sciam_features(select, section))))
else: else:
feeds.extend(self.parse_sciam_departments(div)) feeds.extend(self.parse_sciam_departments(select, section))
return feeds return feeds
def parse_sciam_features(self, div): def parse_sciam_features(self, select, section):
for h4 in div.xpath('''descendant-or-self::li/descendant-or-self::*/a[@href]/descendant-or-self::*/h4'''): for article in select('article[data-article-title]', section):
title = self.tag_to_string(h4) title = article.get('data-article-title')
a = h4.getparent() for a in select('a[href]', article):
url = absurl(a.get('href')) url = absurl(a.get('href'))
break
desc = '' desc = ''
for span in a.xpath('following-sibling::span'): for p in select('p.t_body', article):
desc = self.tag_to_string(span) desc = self.tag_to_string(p)
break break
self.log('Found feature article: %s at %s' % (title, url)) self.log('Found feature article: %s at %s' % (title, url))
self.log('\t' + desc) self.log('\t' + desc)
yield {'title':title, 'url':url, 'description':desc} yield {'title':title, 'url':url, 'description':desc}
def parse_sciam_departments(self, div): def parse_sciam_departments(self, select, section):
section_title, articles = 'Unknown', [] section_title, articles = 'Unknown', []
for x in div.xpath('''descendant-or-self::li/descendant-or-self::*/a[@href]/descendant-or-self::*/h3 | descendant-or-self::li/descendant-or-self::*/span[@class and contains(concat(' ', normalize-space(@class), ' '), ' deptTitle ')]/descendant-or-self::*/a[@href]'''): # noqa for li in select('li[data-article-title]', section):
if x.tag == 'a': for span in select('span.department-title', li):
if articles: if articles:
yield section_title, list(articles) yield section_title, articles
section_title = self.tag_to_string(x) section_title, articles = self.tag_to_string(span), []
del articles[:]
self.log('\nFound section: %s' % section_title) self.log('\nFound section: %s' % section_title)
else: break
title = self.tag_to_string(x) for a in select('h2 a[href]', li):
a = x.getparent() title = self.tag_to_string(a)
url = absurl(a.get('href')) url = absurl(a.get('href'))
articles.append({'title':title, 'url':url, 'description':''}) articles.append({'title':title, 'url':url, 'description':''})
self.log('\tFound article: %s at %s' % (title, url)) self.log('\tFound article: %s at %s' % (title, url))
if articles:
yield section_title, articles