From 2a058081b1f7e6624bf4ec44790ce10af64d8afe Mon Sep 17 00:00:00 2001 From: unkn0w7n <51942695+unkn0w7n@users.noreply.github.com> Date: Sat, 21 Jun 2025 07:49:09 +0530 Subject: [PATCH] Update economist_news.recipe --- recipes/economist_news.recipe | 70 +++++++++++++++++------------------ 1 file changed, 34 insertions(+), 36 deletions(-) diff --git a/recipes/economist_news.recipe b/recipes/economist_news.recipe index 28a927dbc4..e552b8e6bf 100644 --- a/recipes/economist_news.recipe +++ b/recipes/economist_news.recipe @@ -11,6 +11,7 @@ from html5_parser import parse from lxml import etree from calibre.ebooks.BeautifulSoup import NavigableString, Tag +from calibre.ptempfile import PersistentTemporaryFile from calibre.web.feeds.news import BasicNewsRecipe @@ -77,10 +78,7 @@ def process_web_node(node): def load_article_from_web_json(raw): # open('/t/raw.json', 'w').write(raw) body = '' - try: - data = json.loads(raw)['props']['pageProps']['cp2Content'] - except Exception: - data = json.loads(raw)['props']['pageProps']['content'] + data = json.loads(raw)['data']['findArticleByUrl'] body += f'
{data.get("flyTitle", "")}
' body += f'

{data["headline"]}

' if data.get('rubric') and data.get('rubric') is not None: @@ -187,14 +185,12 @@ class EconomistNews(BasicNewsRecipe): remove_attributes = ['data-reactid', 'width', 'height'] # economist.com has started throttling after about 60% of the total has # downloaded with connection reset by peer (104) errors. - delay = 3 + delay = 1 remove_empty_feeds = True ignore_duplicate_articles = {'title'} needs_subscription = False - from_web = False - recipe_specific_options = { 'days': { 'short': 'Oldest article to download from this news source. In days ', @@ -215,25 +211,19 @@ class EconomistNews(BasicNewsRecipe): self.oldest_article = float(d) def get_browser(self, *args, **kwargs): - if self.from_web: - kwargs['user_agent'] = ( - 'Mozilla/5.0 (Linux; Android 14) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.6533.103 Mobile Safari/537.36 Liskov' - ) - br = BasicNewsRecipe.get_browser(self, *args, **kwargs) - else: - kwargs['user_agent'] = 'TheEconomist-Liskov-android' - br = BasicNewsRecipe.get_browser(self, *args, **kwargs) - br.addheaders += [ - ('accept', 'multipart/mixed; deferSpec=20220824, application/json'), - ('accept-encoding', 'gzip'), - ('content-type', 'application/json'), - ('x-app-trace-id', str(uuid4())), - ('x-economist-consumer', 'TheEconomist-Liskov-android'), - ('x-teg-client-name', 'Economist-Android'), - ('x-teg-client-os', 'Android'), - ('x-teg-client-version', '4.30.0'), - ] - return br + kwargs['user_agent'] = 'TheEconomist-Liskov-android' + br = BasicNewsRecipe.get_browser(self, *args, **kwargs) + br.addheaders += [ + ('accept', 'multipart/mixed; deferSpec=20220824, application/json'), + ('accept-encoding', 'gzip'), + ('content-type', 'application/json'), + ('x-app-trace-id', str(uuid4())), + ('x-economist-consumer', 'TheEconomist-Liskov-android'), + ('x-teg-client-name', 'Economist-Android'), + ('x-teg-client-os', 'Android'), + ('x-teg-client-version', '4.40.0'), + ] + return br def economist_return_index(self, ans): if not ans: @@ -290,7 +280,6 @@ class EconomistNews(BasicNewsRecipe): self.log('\t', title, '\n\t\t', desc) if articles: feeds.append((section, articles)) - self.from_web = True return feeds def preprocess_html(self, soup): @@ -305,15 +294,7 @@ class EconomistNews(BasicNewsRecipe): def preprocess_raw_html(self, raw, url): # open('/t/raw.html', 'wb').write(raw.encode('utf-8')) - root_ = parse(raw) - if '/interactive/' in url: - return ('

' + root_.xpath('//h1')[0].text + '

' - 'This article is supposed to be read in a browser' - '
') - - script = root_.xpath('//script[@id="__NEXT_DATA__"]') - - html = load_article_from_web_json(script[0].text) + html = load_article_from_web_json(raw) root = parse(html) for div in root.xpath('//div[@class="lazy-image"]'): @@ -345,6 +326,23 @@ class EconomistNews(BasicNewsRecipe): raw = etree.tostring(root, encoding='unicode') return raw + def get_article(self, url): + query = { + 'operationName': 'ArticleDeeplinkQuery', + 'variables': '{{"ref":"{}"}}'.format(url), + 'query': 'query ArticleDeeplinkQuery($ref: String!, $includeRelatedArticles: Boolean = true ) { findArticleByUrl(url: $ref) { __typename ...ArticleDataFragment } } fragment ContentIdentityFragment on ContentIdentity { articleType forceAppWebView leadMediaType } fragment NarrationFragment on Narration { album bitrate duration filename id provider url isAiGenerated fileHash } fragment ImageTeaserFragment on ImageComponent { altText height imageType source url width } fragment PodcastAudioFragment on PodcastEpisode { id audio { url durationInSeconds } } fragment ArticleTeaserFragment on Article { id tegId url rubric headline flyTitle brand byline dateFirstPublished dateline dateModified datePublished dateRevised estimatedReadTime wordCount printHeadline contentIdentity { __typename ...ContentIdentityFragment } section { tegId name } teaserImage { __typename type ...ImageTeaserFragment } leadComponent { __typename type ...ImageTeaserFragment } narration(selectionMethod: PREFER_ACTOR_NARRATION) { __typename ...NarrationFragment } podcast { __typename ...PodcastAudioFragment } } fragment AnnotatedTextFragment on AnnotatedText { text textJson annotations { type length index attributes { name value } } } fragment ImageComponentFragment on ImageComponent { altText caption { __typename ...AnnotatedTextFragment } credit height imageType mode source url width } fragment BlockQuoteComponentFragment on BlockQuoteComponent { text textJson annotations { type length index attributes { name value } } } fragment BookInfoComponentFragment on BookInfoComponent { text textJson annotations { type length index attributes { name value } } } fragment ParagraphComponentFragment on ParagraphComponent { text textJson annotations { type length index attributes { name value } } } fragment PullQuoteComponentFragment on PullQuoteComponent { text textJson annotations { type length index attributes { name value } } } fragment CrossheadComponentFragment on CrossheadComponent { text } fragment OrderedListComponentFragment on OrderedListComponent { items { __typename ...AnnotatedTextFragment } } fragment UnorderedListComponentFragment on UnorderedListComponent { items { __typename ...AnnotatedTextFragment } } fragment VideoComponentFragment on VideoComponent { url title thumbnailImage } fragment InfoboxComponentFragment on InfoboxComponent { components { __typename type ...BlockQuoteComponentFragment ...BookInfoComponentFragment ...ParagraphComponentFragment ...PullQuoteComponentFragment ...CrossheadComponentFragment ...OrderedListComponentFragment ...UnorderedListComponentFragment ...VideoComponentFragment } } fragment InfographicComponentFragment on InfographicComponent { url title width fallback { __typename ...ImageComponentFragment } altText height width } fragment ArticleDataFragment on Article { id url brand byline rubric headline layout { headerStyle } contentIdentity { __typename ...ContentIdentityFragment } dateline dateFirstPublished dateModified datePublished dateRevised estimatedReadTime narration(selectionMethod: PREFER_ACTOR_NARRATION) { __typename ...NarrationFragment } printFlyTitle printHeadline printRubric flyTitle wordCount section { tegId name articles(pagingInfo: { pagingType: OFFSET pageSize: 6 pageNumber: 1 } ) @include(if: $includeRelatedArticles) { edges { node { __typename ...ArticleTeaserFragment } } } } teaserImage { __typename type ...ImageComponentFragment } tegId leadComponent { __typename type ...ImageComponentFragment } body { __typename type ...BlockQuoteComponentFragment ...BookInfoComponentFragment ...ParagraphComponentFragment ...PullQuoteComponentFragment ...CrossheadComponentFragment ...OrderedListComponentFragment ...UnorderedListComponentFragment ...InfoboxComponentFragment ...ImageComponentFragment ...VideoComponentFragment ...InfographicComponentFragment } footer { __typename type ...ParagraphComponentFragment } tags { name } ads { adData } podcast { __typename ...PodcastAudioFragment } }', # noqa: E501 + } + url = 'https://cp2-graphql-gateway.p.aws.economist.com/graphql?' + urlencode(query, safe='()!', quote_via=quote) + raw = self.index_to_soup(url, raw=True) + return raw + + def print_version(self, url): + art_cont = self.get_article(url) + pt = PersistentTemporaryFile('.html') + pt.write(art_cont) + pt.close() + return 'file:///' + pt.name + def eco_find_image_tables(self, soup): for x in soup.findAll('table', align=['right', 'center']): if len(x.findAll('font')) in (1, 2) and len(x.findAll('img')) == 1: