#!/usr/bin/env python # vim:fileencoding=utf-8 from __future__ import absolute_import, division, print_function, unicode_literals import json from calibre import prepare_string_for_xml as escape from calibre.utils.iso8601 import parse_iso8601 from calibre.web.feeds.news import BasicNewsRecipe def classes(classes): q = frozenset(classes.split(' ')) return dict(attrs={ 'class': lambda x: x and frozenset(x.split()).intersection(q)}) def extract_json(raw): s = raw.find("window['__natgeo__']") script = raw[s:raw.find('', s)] return json.loads(script[script.find('{'):].rstrip(';'))['page']['content']['prismarticle'] def parse_contributors(grp): for item in grp: line = '
' + escape(item['title']) + ' ' for c in item['contributors']: line += escape(c['displayName']) yield line + '
' def parse_lead_image(media): if 'image' in media: yield '

' if 'dsc' in media['image']: yield '

{}
'.format( escape(media['image']['src'], True), escape(media['image']['dsc'], True) ) else: yield '
'.format(escape(media['image']['src'], True)) if 'caption' in media and 'credit' in media: yield '
' + media['caption'] + ' ' + media['credit'] + '
' elif 'caption' in media: yield '
' + media['caption'] + '
' yield '

' def parse_inline(inl): if inl.get('content', {}).get('name', '') == 'Image': props = inl['content']['props'] yield '

' if 'image' in props: yield '

'.format(props['image']['src']) if 'caption' in props: yield '
{}{}
'.format( props['caption'].get('text', ''), ' ' + props['caption'].get('credit', '') ) yield '

' if inl.get('content', {}).get('name', '') == 'ImageGroup': if 'images' in inl['content']['props']: for imgs in inl['content']['props']['images']: yield '

' if 'src' in imgs: yield '

'.format(imgs['src']) if 'caption' in imgs: yield '
{}{}
'.format( imgs['caption'].get('text', ''), ' ' + imgs['caption'].get('credit', '') ) yield '

' def parse_cont(content): for cont in content.get('content', {}): if isinstance(cont, dict): yield from parse_body(cont) if isinstance(cont, str): yield cont def parse_body(x): if isinstance(x, dict): if 'type' in x: tag = x['type'] if tag == 'inline': yield ''.join(parse_inline(x)) elif 'attrs' in x and 'href' in x.get('attrs', ''): yield '<' + tag + ' href="{}">'.format(x['attrs']['href']) for yld in parse_cont(x): yield yld yield '' else: yield '<' + tag + '>' for yld in parse_cont(x): yield yld yield '' elif isinstance(x, list): for y in x: if isinstance(y, dict): yield from parse_body(y) def parse_article(edg): sc = edg['schma'] yield '
' + escape(edg['sctn']) + '
' yield '

' + escape(sc['sclTtl']) + '

' yield '
' + escape(sc['sclDsc']) + '
' yield '

' for line in parse_contributors(edg.get('cntrbGrp', {})): yield line ts = parse_iso8601(edg['mdDt'], as_utc=False).strftime('%B %d, %Y') yield '

Published: ' + escape(ts) + '
' if 'readTime' in edg: yield '
' + escape(edg['readTime']) + '
' yield '

' if edg.get('ldMda', {}).get('cmsType') == 'image': for line in parse_lead_image(edg['ldMda']): yield line for main in edg['prismData']['mainComponents']: if main['name'] == 'Body': for item in main['props']['body']: if isinstance(item, dict): if item.get('type', '') == 'inline': yield ''.join(parse_inline(item)) elif isinstance(item, list): for line in item: yield ''.join(parse_body(line)) def article_parse(data): yield "" for frm in data['frms']: if not frm: continue for mod in frm.get('mods', ()): for edg in mod.get('edgs', ()): if edg.get('cmsType') == 'ImmersiveLeadTile': if 'image' in edg.get('cmsImage', {}): for line in parse_lead_image(edg['cmsImage']): yield line if edg.get('cmsType') == 'ArticleBodyTile': for line in parse_article(edg): yield line yield "" class NatGeo(BasicNewsRecipe): title = u'National Geographic History' description = ( 'From Caesar to Napoleon, the Pyramids to the Parthenon, the Trojan War to the Civil War—National Geographic ' 'HISTORY draws readers in with more than 5,000 years of people, places, and things to explore.' ) language = 'en' encoding = 'utf8' publisher = 'nationalgeographic.com' category = 'science, nat geo' __author__ = 'Kovid Goyal, unkn0wn' description = 'Inspiring people to care about the planet since 1888' timefmt = ' [%a, %d %b, %Y]' no_stylesheets = True use_embedded_content = False remove_attributes = ['style'] remove_javascript = False masthead_url = 'https://i.natgeofe.com/n/e76f5368-6797-4794-b7f6-8d757c79ea5c/ng-logo-2fl.png?w=600&h=600' resolve_internal_links = True recipe_specific_options = { 'res': { 'short': 'For hi-res images, select a resolution from the\nfollowing options: 800, 1000, 1200 or 1500', 'long': 'This is useful for non e-ink devices, and for a lower file size\nthan the default, use 400 or 300.', 'default': '600' } } extra_css = ''' blockquote { color:#404040; } .byline, i { font-style:italic; color:#202020; } .cap { font-size:small; } img {display:block; margin:0 auto;} .cred { font-style:italic; font-size:small; color:#404040; } .auth, .time, .sub { font-size:small; color:#5c5c5c; } ''' def get_cover_url(self): soup = self.index_to_soup('https://ngsingleissues.nationalgeographic.com/history') wrap = soup.find(attrs={'class':'product-image-wrapper'}) return wrap.img['src'] def parse_index(self): soup = self.index_to_soup('https://www.nationalgeographic.com/history/history-magazine') ans = [] for article in soup.findAll('article'): a = article.find('a') url = a['href'] if url.startswith('/'): url = 'https://www.nationalgeographic.com' + url title = self.tag_to_string(article.find(**classes('PromoTile__Title--truncated'))) ans.append({'title': title, 'url': url}) self.log(title, ' ', url) return [('Articles', ans)] def preprocess_raw_html(self, raw_html, url): data = extract_json(raw_html) return '\n'.join(article_parse(data)) def preprocess_html(self, soup): for h2 in soup.findAll('h2'): h2.name = 'h4' for img in soup.findAll('img', src=True): res = '?w=600' w = self.recipe_specific_options.get('res') if w and isinstance(w, str): res = '?w=' + w img['src'] = img['src'] + res return soup def populate_article_metadata(self, article, soup, first): summ = soup.find(attrs={'class':'byline'}) if summ: article.summary = self.tag_to_string(summ) article.text_summary = self.tag_to_string(summ)