Update economist_search.recipe

recipes/economist_search.recipe

@@ -2,20 +2,46 @@
 # License: GPLv3 Copyright: 2008, Kovid Goyal <kovid at kovidgoyal.net>
 
 import json
+import time
+from datetime import datetime, timedelta
 
 from html5_parser import parse
 from lxml import etree
 
-from calibre.ebooks.BeautifulSoup import BeautifulSoup, NavigableString, Tag
+from calibre.ebooks.BeautifulSoup import NavigableString, Tag
-from calibre.scraper.simple import read_url
 from calibre.web.feeds.news import BasicNewsRecipe
 
 
-def E(parent, name, text='', **attrs):
+def process_node(node):
-    ans = parent.makeelement(name, **attrs)
+    ntype = node.get('type', '')
-    ans.text = text
+    if ntype == 'CROSSHEAD':
-    parent.append(ans)
+        if node.get('textHtml'):
-    return ans
+            return f'<h4>{node.get("textHtml")}</h4>'
+        return f'<h4>{node.get("text", "")}</h4>'
+    elif ntype == 'PARAGRAPH':
+        if node.get('textHtml'):
+            return f'<p>{node.get("textHtml")}</p>'
+        return f'<p>{node.get("text", "")}</p>'
+    elif ntype == 'IMAGE':
+        alt = "" if node.get("altText") is None else node.get("altText")
+        cap = ""
+        if node.get('caption'):
+            if node['caption'].get('textHtml') is not None:
+                cap = node['caption']['textHtml']
+        return f'<div><img src="{node["url"]}" title="{alt}"></div><div style="text-align:center; font-size:small;">{cap}</div>'
+    elif ntype == 'PULL_QUOTE':
+        if node.get('textHtml'):
+            return f'<blockquote>{node.get("textHtml")}</blockquote>'
+        return f'<blockquote>{node.get("text", "")}</blockquote>'
+    elif ntype == 'DIVIDER':
+        return '<hr>'
+    elif ntype == 'INFOBOX':
+        for x in safe_dict(node, 'components'):
+            return f'<blockquote>{process_node(x)}</blockquote>'
+    elif ntype:
+        print('** ', ntype)
+        return ''
+
 
 def safe_dict(data, *names):
     ans = data
@@ -28,33 +54,29 @@ class JSONHasNoContent(ValueError):
     pass
 
 
-def load_article_from_json(raw, root):
+def load_article_from_json(raw):
     # open('/t/raw.json', 'w').write(raw)
+    body = ''
+    data = json.loads(raw)['props']['pageProps']['cp2Content']
+    body += f'<div style="color: red; font-size:small; font-weight:bold;">{data.get("flyTitle", "")}</div>'
+    body += f'<h1>{data["headline"]}</h1>'
+    body += f'<div style="font-style: italic; color:#202020;">{data.get("rubric", "")}</div>'
     try:
-        data = json.loads(raw)['props']['pageProps']['content']
+        date = data['dateModified']
-    except KeyError as e:
-        raise JSONHasNoContent(e)
-    if isinstance(data, list):
-        data = data[0]
-    body = root.xpath('//body')[0]
-    for child in tuple(body):
-        body.remove(child)
-    article = E(body, 'article')
-    E(article, 'div', data['subheadline'], id='subhead')
-    E(article, 'h1', data['headline'])
-    E(article, 'p', data['description'], id='desc')
-    if data['dateline'] is None:
-        E(article, 'p', (data['datePublishedString'] or ''), id='date')
-    else:
-        E(article, 'p', (data['datePublishedString'] or '') + ' | ' + (data['dateline']), id='date')
-    main_image_url = safe_dict(data, 'image', 'main', 'url').get('canonical')
-    if main_image_url:
-        div = E(article, 'div')
-        try:
-            E(div, 'img', src=main_image_url)
     except Exception:
-            pass
+        date = data['datePublished']
-    E(article, 'section', id='body')
+    dt = datetime.fromisoformat(date[:-1]) + timedelta(seconds=time.timezone)
+    dt = dt.strftime('%b %d, %Y %I:%M %p')
+    if data.get('dateline') is None:
+        body += f'<p style="color: gray; font-size: small;">{dt}</p>'
+    else:
+        body += f'<p style="color: gray; font-size: small;">{dt + " | " + (data["dateline"])}</p>'
+    main_image_url = safe_dict(data, 'leadComponent') or ''
+    if main_image_url:
+        body += process_node(data['leadComponent'])
+    for node in data.get('body'):
+        body += process_node(node)
+    return '<html><body><article>' + body + '</article></body></html>'
 
 def cleanup_html_article(root):
     main = root.xpath('//main')[0]
@@ -87,11 +109,10 @@ def process_url(url):
 
 
 class econ_search(BasicNewsRecipe):
-
     title = 'The Economist - Search'
     language = 'en'
     encoding = 'utf-8'
-    __author__ = "Kovid Goyal"
+    __author__ = "unkn0wn"
     description = (
         'Use the Advanced section of the recipe to search.'
     )
@@ -100,11 +121,11 @@ class econ_search(BasicNewsRecipe):
     no_stylesheets = True
     ignore_duplicate_articles = {'url'}
     extra_css = '''
-        em, blockquote { color:#202020; }
+        em { color:#202020; }
         img {display:block; margin:0 auto;}
         .sub { font-size:small; }
         #subhead { color: #404040; font-size:small; font-weight:bold; }'
-        #desc { font-style: italic; color:#202020; }
+        #descrip { font-style: italic; color:#202020; }
         #date { color: gray; font-size:small; }
     '''
 
@@ -137,37 +158,32 @@ class econ_search(BasicNewsRecipe):
     remove_attributes = ['data-reactid', 'width', 'height']
     # economist.com has started throttling after about 60% of the total has
     # downloaded with connection reset by peer (104) errors.
-    delay = 1
+    delay = 3
-
-    def __init__(self, *args, **kwargs):
-        BasicNewsRecipe.__init__(self, *args, **kwargs)
-        if self.output_profile.short_name.startswith('kindle'):
-            # Reduce image sizes to get file size below amazon's email
-            # sending threshold
-            self.web2disk_options.compress_news_images = True
-            self.web2disk_options.compress_news_images_auto_size = 5
-            self.log.warn('Kindle Output profile being used, reducing image quality to keep file size below amazon email threshold')
 
     def get_browser(self, *args, **kwargs):
-        # Needed to bypass cloudflare
+        kwargs['user_agent'] = (
-        kwargs['user_agent'] = 'common_words/based'
+            'Mozilla/5.0 (Linux; Android 14; 330333QCG Build/AP1A.140705.005; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/125.0.6422.165 Mobile Safari/537.36 Lamarr/3.37.0-3037003 (android)' # noqa
+        )
         br = BasicNewsRecipe.get_browser(self, *args, **kwargs)
-        br.addheaders += [('Accept-Language', 'en-GB,en-US;q=0.9,en;q=0.8')]
+        br.addheaders += [('x-requested-with', 'com.economist.lamarr')]
         return br
 
     def preprocess_raw_html(self, raw, url):
         # open('/t/raw.html', 'wb').write(raw.encode('utf-8'))
-        root = parse(raw)
+        root_ = parse(raw)
         if '/interactive/' in url:
-            return '<html><body><article><h1>' + root.xpath('//h1')[0].text + '</h1><em>' \
+            return (
-                        + 'This article is supposed to be read in a browser' \
+                '<html><body><article><h1>'
+                + root_.xpath('//h1')[0].text + '</h1><em>'
+                + 'This article is supposed to be read in a browser'
                 + '</em></article></body></html>'
-        script = root.xpath('//script[@id="__NEXT_DATA__"]')
+            )
-        if script:
+
-            try:
+        script = root_.xpath('//script[@id="__NEXT_DATA__"]')
-                load_article_from_json(script[0].text, root)
+
-            except JSONHasNoContent:
+        html = load_article_from_json(script[0].text)
-                cleanup_html_article(root)
+
+        root = parse(html)
         for div in root.xpath('//div[@class="lazy-image"]'):
             noscript = list(div.iter('noscript'))
             if noscript and noscript[0].text:
@@ -179,45 +195,32 @@ class econ_search(BasicNewsRecipe):
                     p.remove(noscript[0])
         for x in root.xpath('//*[name()="script" or name()="style" or name()="source" or name()="meta"]'):
             x.getparent().remove(x)
+        # the economist uses <small> for small caps with a custom font
+        for init in root.xpath('//span[@data-caps="initial"]'):
+            init.set('style', 'font-weight:bold;')
+        for x in root.xpath('//small'):
+            if x.text and len(x) == 0:
+                x.text = x.text.upper()
+                x.tag = 'span'
+                x.set('style', 'font-variant: small-caps')
+        for h2 in root.xpath('//h2'):
+            h2.tag = 'h4'
+        for x in root.xpath('//figcaption'):
+            x.set('style', 'text-align:center; font-size:small;')
+        for x in root.xpath('//cite'):
+            x.tag = 'blockquote'
+            x.set('style', 'color:#404040;')
         raw = etree.tostring(root, encoding='unicode')
-
-        raw_ar = read_url([], 'https://archive.is/latest/' + url)
-        archive = BeautifulSoup(str(raw_ar))
-        art = archive.find('article')
-        if art:
-            bdy = art.findAll('section')
-            if len(bdy) != 0:
-                content = bdy[-1]
-            else:
-                content = archive.find('div', attrs={'itemprop':'text'})
-            soup = BeautifulSoup(raw)
-            article = soup.find('section', attrs={'id':'body'})
-            if not article:
-                article = soup.find('div', attrs={'itemprop':'text'})
-                if not article:
-                    article = soup.find(attrs={'itemprop':'blogPost'})
-            if article and content:
-                self.log('**fetching archive content')
-                article.append(content)
-
-                div = soup.findAll(attrs={'style': lambda x: x and x.startswith(
-                        ('color:rgb(13, 13, 13);', 'color: rgb(18, 18, 18);')
-                    )})
-                for p in div:
-                     p.name = 'p'
-                return str(soup)
-            return raw
         return raw
 
     def preprocess_html(self, soup):
-        for img in soup.findAll('img', attrs={'old-src':True}):
+        width = '600'
-            img['src'] = img['old-src']
+        w = self.recipe_specific_options.get('res')
-        for a in soup.findAll('a', href=True):
+        if w and isinstance(w, str):
-            a['href'] = 'http' + a['href'].split('http')[-1]
+            width = w
-        for fig in soup.findAll('figure'):
+        for img in soup.findAll('img', src=True):
-            fig['class'] = 'sub'
+            qua = 'economist.com/cdn-cgi/image/width=' + width + ',quality=80,format=auto/'
-        for sty in soup.findAll(attrs={'style':True}):
+            img['src'] = img['src'].replace('economist.com/', qua)
-            del sty['style']
         return soup
 
     recipe_specific_options = {
@@ -234,7 +237,12 @@ class econ_search(BasicNewsRecipe):
             'short': 'number of pages',
             'long': 'number of pages of search results you want',
             'default': '2'
-        }
+        },
+        'res': {
+            'short': 'For hi-res images, select a resolution from the\nfollowing options: 834, 960, 1096, 1280, 1424',
+            'long': 'This is useful for non e-ink devices, and for a lower file size\nthan the default, use from 480, 384, 360, 256.',
+            'default': '600',
+        },
     }
 
     def parse_index(self):