#!/usr/bin/env python # vim:fileencoding=utf-8 from __future__ import absolute_import, division, print_function, unicode_literals import json from pprint import pformat from calibre.web.feeds.news import BasicNewsRecipe from calibre import prepare_string_for_xml as escape from calibre.utils.iso8601 import parse_iso8601 def classes(classes): q = frozenset(classes.split(' ')) return dict(attrs={ 'class': lambda x: x and frozenset(x.split()).intersection(q)}) def extract_json(raw): s = raw.find("window['__natgeo__']") script = raw[s:raw.find('', s)] return json.loads( script[script.find('{'):].rstrip(';'))['page']['content']['article'] def parse_contributors(grp): for item in grp: line = '

' + escape(item['title']) + ' ' for c in item['contributors']: line += escape(c['displayName']) yield line + '

' def parse_lead_image(media): yield '

'.format( escape(media['image']['src'], True), escape(media['image']['dsc'], True)) yield '

' + escape(media['caption']) + '

' if 'credit' in media: yield '

' + escape(media['credit']) + '

' def parse_body(item): c = item['cntnt'] if item.get('type') == 'inline': if c.get('cmsType') == 'listicle': yield '

' + escape(c['title']) + "

" yield c['text'] elif c.get('cmsType') == 'image': for line in parse_lead_image(c): yield line else: yield '<{tag}>{markup}'.format( tag=item['type'], markup=c['mrkup']) def parse_article(edg): sc = edg['schma'] yield '

' + escape(edg['sctn']) + '

' yield '

' + escape(sc['sclTtl']) + '

' yield '

' + escape(sc['sclDsc']) + '

' for line in parse_contributors(edg['cntrbGrp']): yield line ts = parse_iso8601(edg['mdDt'], as_utc=False).strftime('%B %d, %Y') yield '

Published: ' + escape(ts) + '

' if 'readTime' in edg: yield '

' + escape(edg['readTime']) + '

' if edg.get('ldMda', {}).get('cmsType') == 'image': for line in parse_lead_image(edg['ldMda']): yield line for item in edg['bdy']: for line in parse_body(item): yield line def article_parse(data): yield "" for frm in data['frms']: if not frm: continue for mod in frm.get('mods', ()): for edg in mod.get('edgs', ()): if edg.get('cmsType') == 'ArticleBodyTile': for line in parse_article(edg): yield line yield "" class NatGeo(BasicNewsRecipe): title = u'National Geographic' description = 'Daily news articles from The National Geographic' language = 'en' encoding = 'utf8' publisher = 'nationalgeographic.com' category = 'science, nat geo' __author__ = 'Kovid Goyal' description = 'Inspiring people to care about the planet since 1888' timefmt = ' [%a, %d %b, %Y]' no_stylesheets = True use_embedded_content = False remove_attributes = ['style'] remove_javascript = False def parse_index(self): soup = self.index_to_soup('https://www.nationalgeographic.com/latest-stories/') ans = {} for article in soup.findAll('article'): a = article.find('a') url = a['href'] section = self.tag_to_string(article.find(**classes('SectionLabel'))) title = self.tag_to_string(article.find(**classes('PromoTile__Title--truncated'))) articles = ans.setdefault(section, []) articles.append({'title': title, 'url': url}) self.log(pformat(ans)) return list(ans.items()) def preprocess_raw_html(self, raw_html, url): data = extract_json(raw_html) return '\n'.join(article_parse(data))