diff --git a/recipes/economist_world_ahead.recipe b/recipes/economist_world_ahead.recipe new file mode 100644 index 0000000000..ba732b41c3 --- /dev/null +++ b/recipes/economist_world_ahead.recipe @@ -0,0 +1,345 @@ +#!/usr/bin/env python +# License: GPLv3 Copyright: 2008, Kovid Goyal + +try: + from http.cookiejar import Cookie +except ImportError: + from cookielib import Cookie + +import json +from html5_parser import parse +from lxml import etree +from collections import defaultdict + +from calibre import replace_entities +from calibre.ebooks.BeautifulSoup import NavigableString, Tag +from calibre.utils.date import parse_only_date +from calibre.web.feeds.news import BasicNewsRecipe + + +def E(parent, name, text='', **attrs): + ans = parent.makeelement(name, **attrs) + ans.text = text + parent.append(ans) + return ans + + +def process_node(node, html_parent): + ntype = node.get('type') + if ntype == 'tag': + c = html_parent.makeelement(node['name']) + c.attrib.update({k: v or '' for k, v in node.get('attribs', {}).items()}) + html_parent.append(c) + for nc in node.get('children', ()): + process_node(nc, c) + elif ntype == 'text': + text = node.get('data') + if text: + text = replace_entities(text) + if len(html_parent): + t = html_parent[-1] + t.tail = (t.tail or '') + text + else: + html_parent.text = (html_parent.text or '') + text + + +def safe_dict(data, *names): + ans = data + for x in names: + ans = ans.get(x) or {} + return ans + + +class JSONHasNoContent(ValueError): + pass + + +def load_article_from_json(raw, root): + # open('/t/raw.json', 'w').write(raw) + try: + data = json.loads(raw)['props']['pageProps']['content'] + except KeyError as e: + raise JSONHasNoContent(e) + if isinstance(data, list): + data = data[0] + body = root.xpath('//body')[0] + for child in tuple(body): + body.remove(child) + article = E(body, 'article') + E(article, 'h4', data['subheadline'], style='color: red; margin: 0') + E(article, 'h1', data['headline'], style='font-size: x-large') + E(article, 'div', data['description'], style='font-style: italic; color: #202020;') + E(article, 'div', (data['datePublishedString'] or '') + ' | ' + (data['dateline'] or ''), style='color: gray; margin: 1em') + main_image_url = safe_dict(data, 'image', 'main', 'url').get('canonical') + main_img_desc = safe_dict(data, 'image', 'main').get('description', '') + if main_image_url: + div = E(article, 'div') + try: + E(div, 'img', src=main_image_url) + except Exception: + pass + if main_img_desc != '': + div = E(article, 'figcaption', main_img_desc) + for node in data.get('text') or (): + process_node(node, article) + + +def cleanup_html_article(root): + main = root.xpath('//main')[0] + body = root.xpath('//body')[0] + for child in tuple(body): + body.remove(child) + body.append(main) + main.set('id', '') + main.tag = 'article' + for x in root.xpath('//*[@style]'): + x.set('style', '') + for x in root.xpath('//button'): + x.getparent().remove(x) + + +def classes(classes): + q = frozenset(classes.split(' ')) + return dict(attrs={ + 'class': lambda x: x and frozenset(x.split()).intersection(q)}) + + +def new_tag(soup, name, attrs=()): + impl = getattr(soup, 'new_tag', None) + if impl is not None: + return impl(name, attrs=dict(attrs)) + return Tag(soup, name, attrs=attrs or None) + + +class NoArticles(Exception): + pass + + +def process_url(url): + if url.startswith('/'): + url = 'https://www.economist.com' + url + return url + + +class Economist(BasicNewsRecipe): + + title = 'The Economist World Ahead' + language = 'en' + encoding = 'utf-8' + + __author__ = "Kovid Goyal" + description = ( + 'The World Ahead is The Economist’s future-gazing publication. It prepares audiences for what is to ' + 'come with mind-stretching insights and expert analysis—all in The Economist’s clear, elegant style.' + ) + extra_css = ''' + .headline {font-size: x-large;} + h2 { font-size: medium; } + h1 { font-size: large; } + em.Bold {font-weight:bold;font-style:normal; color:#202020;} + em.Italic {font-style:italic; color:#202020;} + p.xhead {font-weight:bold;} + .pullquote { + float: right; + font-size: larger; + font-weight: bold; + font-style: italic; + page-break-inside:avoid; + border-bottom: 3px solid black; + border-top: 3px solid black; + width: 228px; + margin: 0px 0px 10px 15px; + padding: 7px 0px 9px; + } + .flytitle-and-title__flytitle { + display: block; + font-size: smaller; + color: red; + } + img {display:block; margin:0 auto;} + ''' + oldest_article = 7.0 + resolve_internal_links = True + remove_tags = [ + dict(name=['script', 'noscript', 'title', 'iframe', 'cf_floatingcontent', 'aside', 'footer']), + dict(attrs={'aria-label': "Article Teaser"}), + dict(attrs={ + 'class': [ + 'dblClkTrk', 'ec-article-info', 'share_inline_header', + 'related-items', 'main-content-container', 'ec-topic-widget', + 'teaser', 'blog-post__bottom-panel-bottom', 'blog-post__comments-label', + 'blog-post__foot-note', 'blog-post__sharebar', 'blog-post__bottom-panel', + 'newsletter-form','share-links-header','teaser--wrapped', 'latest-updates-panel__container', + 'latest-updates-panel__article-link','blog-post__section' + ] + } + ), + dict(attrs={ + 'class': lambda x: x and 'blog-post__siblings-list-aside' in x.split()}), + classes( + 'share-links-header teaser--wrapped latest-updates-panel__container' + ' latest-updates-panel__article-link blog-post__section newsletter-form blog-post__bottom-panel' + ) + ] + keep_only_tags = [dict(name='article', id=lambda x: not x)] + no_stylesheets = True + remove_attributes = ['data-reactid', 'width', 'height'] + # economist.com has started throttling after about 60% of the total has + # downloaded with connection reset by peer (104) errors. + simultaneous_downloads = 1 + + needs_subscription = False + + def __init__(self, *args, **kwargs): + BasicNewsRecipe.__init__(self, *args, **kwargs) + if self.output_profile.short_name.startswith('kindle'): + # Reduce image sizes to get file size below amazon's email + # sending threshold + self.web2disk_options.compress_news_images = True + self.web2disk_options.compress_news_images_auto_size = 5 + self.log.warn('Kindle Output profile being used, reducing image quality to keep file size below amazon email threshold') + + + def get_browser(self): + br = BasicNewsRecipe.get_browser(self) + # Add a cookie indicating we have accepted Economist's cookie + # policy (needed when running from some European countries) + ck = Cookie( + version=0, + name='notice_preferences', + value='2:', + port=None, + port_specified=False, + domain='.economist.com', + domain_specified=False, + domain_initial_dot=True, + path='/', + path_specified=False, + secure=False, + expires=None, + discard=False, + comment=None, + comment_url=None, + rest={'HttpOnly': None}, + rfc2109=False + ) + br.cookiejar.set_cookie(ck) + br.set_handle_gzip(True) + return br + + def preprocess_raw_html(self, raw, url): + # open('/t/raw.html', 'wb').write(raw.encode('utf-8')) + root = parse(raw) + script = root.xpath('//script[@id="__NEXT_DATA__"]') + if script: + try: + load_article_from_json(script[0].text, root) + except JSONHasNoContent: + cleanup_html_article(root) + for div in root.xpath('//div[@class="lazy-image"]'): + noscript = list(div.iter('noscript')) + if noscript and noscript[0].text: + img = list(parse(noscript[0].text).iter('img')) + if img: + p = noscript[0].getparent() + idx = p.index(noscript[0]) + p.insert(idx, p.makeelement('img', src=img[0].get('src'))) + p.remove(noscript[0]) + for x in root.xpath('//*[name()="script" or name()="style" or name()="source" or name()="meta"]'): + x.getparent().remove(x) + # the economist uses for small caps with a custom font + for x in root.xpath('//small'): + if x.text and len(x) == 0: + x.text = x.text.upper() + x.tag = 'span' + x.set('style', 'font-variant: small-caps') + for x in root.xpath('//figcaption'): + x.set('style', 'text-align:center; font-size:small;') + raw = etree.tostring(root, encoding='unicode') + return raw + + def parse_index(self): + # return [('Articles', [{'title':'test', + # 'url':'https://www.economist.com/interactive/briefing/2022/06/11/huge-foundation-models-are-turbo-charging-ai-progress' + # }])] + url = 'https://www.economist.com/the-world-ahead' + # raw = open('/t/raw.html').read() + raw = self.index_to_soup(url, raw=True) + # with open('/t/raw.html', 'wb') as f: + # f.write(raw) + soup = self.index_to_soup(raw) + # nav = soup.find(attrs={'class':'navigation__wrapper'}) + # if nav is not None: + # a = nav.find('a', href=lambda x: x and '/printedition/' in x) + # if a is not None: + # self.log('Following nav link to current edition', a['href']) + # soup = self.index_to_soup(process_url(a['href'])) + ans = self.economist_parse_index(soup) + if not ans: + raise NoArticles( + 'Could not find any articles, either the ' + 'economist.com server is having trouble and you should ' + 'try later or the website format has changed and the ' + 'recipe needs to be updated.' + ) + return ans + + def economist_parse_index(self, soup): + script_tag = soup.find("script", id="__NEXT_DATA__") + if script_tag is not None: + data = json.loads(script_tag.string) + # open('/t/raw.json', 'w').write(json.dumps(data, indent=2, sort_keys=True)) + self.title = safe_dict(data, "props", "pageProps", "content", "headline") + # self.cover_url = 'https://mma.prnewswire.com/media/2275620/The_Economist_The_World_Ahead_2024.jpg?w=600' + + feeds = [] + + for coll in safe_dict(data, "props", "pageProps", "content", "collections"): + section = safe_dict(coll, "headline") or '' + self.log(section) + articles = [] + for part in safe_dict(coll, "hasPart", "parts"): + title = safe_dict(part, "headline") or '' + url = safe_dict(part, "url", "canonical") or '' + if not title or not url: + continue + desc = safe_dict(part, "description") or '' + sub = safe_dict(part, "subheadline") or '' + if sub: + desc = sub + ' :: ' + desc + if '/interactive/' in url: + self.log('\tSkipping interactive article:', title, url) + continue + self.log('\t', title, '\n\t', desc, '\n\t\t', url) + articles.append({'title': title, 'description':desc, 'url': url}) + if articles: + feeds.append((section, articles)) + return feeds + + def eco_find_image_tables(self, soup): + for x in soup.findAll('table', align=['right', 'center']): + if len(x.findAll('font')) in (1, 2) and len(x.findAll('img')) == 1: + yield x + + def postprocess_html(self, soup, first): + for img in soup.findAll('img', srcset=True): + del img['srcset'] + for table in list(self.eco_find_image_tables(soup)): + caption = table.find('font') + img = table.find('img') + div = new_tag(soup, 'div') + div['style'] = 'text-align:left;font-size:70%' + ns = NavigableString(self.tag_to_string(caption)) + div.insert(0, ns) + div.insert(1, new_tag(soup, 'br')) + del img['width'] + del img['height'] + img.extract() + div.insert(2, img) + table.replaceWith(div) + return soup + + def canonicalize_internal_url(self, url, is_link=True): + if url.endswith('/print'): + url = url.rpartition('/')[0] + return BasicNewsRecipe.canonicalize_internal_url(self, url, is_link=is_link) diff --git a/recipes/icons/economist_world_ahead.png b/recipes/icons/economist_world_ahead.png new file mode 100644 index 0000000000..7fc33d3842 Binary files /dev/null and b/recipes/icons/economist_world_ahead.png differ diff --git a/recipes/toiprint.recipe b/recipes/toiprint.recipe index 2d15ef2913..ff6a4ba7f1 100644 --- a/recipes/toiprint.recipe +++ b/recipes/toiprint.recipe @@ -83,7 +83,15 @@ class toiprint(BasicNewsRecipe): desc = 'Page No.' + url.split('_')[-3] + ' | ' + art.get('ColumnTitle', '') self.log('\t', title, '\n\t', desc.replace('\n', '')) feeds_dict[section].append({"title": title, "url": url, "description": desc}) - return [(section, articles) for section, articles in feeds_dict.items()] + def sort_key(x): + section = x[0] + try: + return ( + 'Front Page', 'Times Nation', 'Times Region', 'Times City' + ).index(section) + except Exception: + return 99999999 + return (sorted(feeds_dict.items(), key=sort_key)) def preprocess_raw_html(self, raw, *a): data = json.loads(raw)