#!/usr/bin/env python # vim:fileencoding=utf-8 # License: GPLv3 Copyright: 2020, Kovid Goyal import json from calibre import prepare_string_for_xml from calibre.web.feeds.recipes import BasicNewsRecipe # Article JSON parser {{{ def serialize_image(block): yield '

' block = block['model'] media = block['media'] alt = prepare_string_for_xml(media.get('alt') or '', True) for q in ('originalSrc', 'src'): if q in media: src = prepare_string_for_xml(media[q]) break else: raise ValueError('No src found in media block: {}'.format(media)) yield '

'.format(src, alt) caption = block.get('caption') if caption: yield '

{}

'.format(prepare_string_for_xml(caption)) yield '

' def block_tag(name, generator): yield '<' + name + '>' yield from generator yield '' def serialize_paragraph(block): block = block['model'] for x in block['blocks']: xt = x['type'] if xt == 'fragment': styles = [] model = x['model'] for attr in model['attributes']: if attr == 'bold': styles.append('font-weight: bold') elif attr in ('italic', 'italics'): styles.append('font-style: italic') if styles: prefix = ''.format('; '.join(styles)) suffix = '' else: prefix = suffix = '' yield prefix + prepare_string_for_xml(model['text']) + suffix elif xt == 'urlLink': model = x['model'] yield '{}'.format(prepare_string_for_xml(model['locator'], True), prepare_string_for_xml(model['text'])) def serialize_list(block): for x in block['model']['blocks']: if x['type'] == 'listItem': yield from block_tag('li', serialize_paragraph(x)) def serialize_text(block): block = block['model'] for x in block['blocks']: xt = x['type'] if xt == 'paragraph': yield from block_tag('p', serialize_paragraph(x)) elif xt == 'unorderedList': yield from block_tag('ul', serialize_list(x)) elif xt == 'orderedList': yield from block_tag('ol', serialize_list(x)) else: raise KeyError('Unknown block type: ' + x['type']) def serialize_contributor(contributor): if 'title' in contributor: yield '

' + prepare_string_for_xml(contributor['title']) + '

' if 'subtitle' in contributor: yield '

' + prepare_string_for_xml(contributor['subtitle']) + '

' def parse_article_json(root, abort_article): data = root['data'] has_media_experience = False for key in data: if key.startswith('article?'): article = data[key]['data'] break elif key.startswith('media-experience?'): has_media_experience = True else: if has_media_experience: abort_article('Skipping video article') return raise KeyError('No article found in data keys: {}'.format(data.keys())) lines = [] if article.get('headline'): lines.append('

{}

'.format(prepare_string_for_xml(article['headline']))) if article.get('contributor'): lines.extend(serialize_contributor(article['contributor'])) for block in article['blocks']: bt = block.get('type') if bt == 'image': lines.extend(serialize_image(block)) elif bt == 'text': lines.extend(serialize_text(block)) return '' + '\n'.join(lines) + '' # }}} class BBC(BasicNewsRecipe): title = 'BBC News (fast)' __author__ = 'Kovid Goyal' description = 'Visit BBC News for up-to-the-minute news, breaking news, video, audio and feature stories. BBC News provides trusted World and UK news as well as local and regional perspectives. Also entertainment, business, science, technology and health news.' # noqa oldest_article = 2 max_articles_per_feed = 100 no_stylesheets = True use_embedded_content = False encoding = 'utf8' publisher = 'BBC' category = 'news, UK, world' language = 'en_GB' masthead_url = 'https://news.bbcimg.co.uk/img/1_0_1/cream/hi/news/news-blocks.gif' conversion_options = { 'comments': description, 'tags': category, 'language': language, 'publisher': publisher, } feeds = [ ('Top Stories', 'https://feeds.bbci.co.uk/news/rss.xml'), ('Science/Environment', 'https://feeds.bbci.co.uk/news/science_and_environment/rss.xml'), ('Technology', 'https://feeds.bbci.co.uk/news/technology/rss.xml'), ('Entertainment/Arts', 'https://feeds.bbci.co.uk/news/entertainment_and_arts/rss.xml'), ('Magazine', 'https://feeds.bbci.co.uk/news/magazine/rss.xml'), ('Business', 'https://feeds.bbci.co.uk/news/business/rss.xml'), ('Politics', 'https://feeds.bbci.co.uk/news/politics/rss.xml'), ('Health', 'https://feeds.bbci.co.uk/news/health/rss.xml'), ('US&Canada', 'https://feeds.bbci.co.uk/news/world/us_and_canada/rss.xml'), ('Latin America', 'https://feeds.bbci.co.uk/news/world/latin_america/rss.xml'), ('Europe', 'https://feeds.bbci.co.uk/news/world/europe/rss.xml'), ('South Asia', 'https://feeds.bbci.co.uk/news/world/south_asia/rss.xml'), ('England', 'https://feeds.bbci.co.uk/news/england/rss.xml'), ('Asia-Pacific', 'https://feeds.bbci.co.uk/news/world/asia_pacific/rss.xml'), ('Africa', 'https://feeds.bbci.co.uk/news/world/africa/rss.xml') ] def preprocess_raw_html(self, raw_html, url): q = '>window.__INITIAL_DATA__={' idx = raw_html.find(q) if idx < 0: raise ValueError('Failed to find JSON') data = raw_html[idx + len(q) - 1:] idx = data.find('};') data = data[:idx+1] root = json.loads(data) return parse_article_json(root, self.abort_article)