diff --git a/recipes/economist_espresso.recipe b/recipes/economist_espresso.recipe index ed94653431..18dba4922d 100644 --- a/recipes/economist_espresso.recipe +++ b/recipes/economist_espresso.recipe @@ -5,6 +5,7 @@ https://www.economist.com/the-world-in-brief import json from urllib.parse import quote, urlencode +from uuid import uuid4 from html5_parser import parse from lxml import etree @@ -154,10 +155,15 @@ class Espresso(BasicNewsRecipe): remove_attributes = ['data-reactid', 'width', 'height'] def get_browser(self, *args, **kwargs): - # Needed to bypass cloudflare - kwargs['user_agent'] = 'common_words/based' + kwargs['user_agent'] = 'TheEconomist-Lamarr-android' br = BasicNewsRecipe.get_browser(self, *args, **kwargs) - br.addheaders += [('Accept-Language', 'en-GB,en-US;q=0.9,en;q=0.8')] + br.addheaders += [ + ('accept', '*/*'), + ('content-type', 'application/json'), + ('apollographql-client-name', 'mobile-app-apollo'), + ('apollographql-client-version', '3.50.0'), + ('x-request-id', str(uuid4())), + ] return br def economist_return_index(self, ans): @@ -174,7 +180,7 @@ class Espresso(BasicNewsRecipe): query = { 'query': 'query EspressoQuery($ref:String!){espresso:canonical(ref:$ref){...EspressoFragment __typename}}fragment EspressoFragment on Content{id type hasPart(size:1 sort:"datePublished:desc"){parts{id type rubric:description hasPart(sort:"publication.context.position:asc,datePublished:desc"){parts{...ArticleFragment __typename}__typename}__typename}__typename}__typename}fragment ArticleFragment on Content{ad{grapeshot{channels{name __typename}__typename}__typename}articleSection{internal{id title:headline __typename}__typename}audio{main{id duration(format:"seconds")source:channel{id __typename}url{canonical __typename}__typename}__typename}byline dateline dateModified datePublished dateRevised flyTitle:subheadline id image{...ImageInlineFragment ...ImageMainFragment ...ImagePromoFragment __typename}print{title:headline flyTitle:subheadline rubric:description section{id title:headline __typename}__typename}publication{id tegID title:headline flyTitle:subheadline datePublished regionsAllowed url{canonical __typename}__typename}rubric:description source:channel{id __typename}tegID text(format:"json")title:headline type url{canonical __typename}topic contentIdentity{forceAppWebview mediaType articleType __typename}__typename}fragment ImageInlineFragment on Media{inline{url{canonical __typename}width height __typename}__typename}fragment ImageMainFragment on Media{main{url{canonical __typename}width height __typename}__typename}fragment ImagePromoFragment on Media{promo{url{canonical __typename}id width height __typename}__typename}', # noqa: E501 'operationName': 'EspressoQuery', - 'variables': '{"ref":"/content/jakj5ed3rml75i8j0d5i74p8adf6eem4"}', + 'variables': '{"ref":"/content/ai0db6q5mftflg1irq7hiiofp15t7nlv"}', } url = 'https://cp2-graphql-gateway.p.aws.economist.com/graphql?' + urlencode(query, safe='()!', quote_via=quote) try: diff --git a/recipes/economist_news.recipe b/recipes/economist_news.recipe index f2b50f57fa..cf2223dc85 100644 --- a/recipes/economist_news.recipe +++ b/recipes/economist_news.recipe @@ -194,7 +194,13 @@ class EconomistNews(BasicNewsRecipe): def get_browser(self, *args, **kwargs): kwargs['user_agent'] = 'TheEconomist-Lamarr-android' br = BasicNewsRecipe.get_browser(self, *args, **kwargs) - br.addheaders += [('x-request-id', str(uuid4()))] + br.addheaders += [ + ('accept', '*/*'), + ('content-type', 'application/json'), + ('apollographql-client-name', 'mobile-app-apollo'), + ('apollographql-client-version', '3.50.0'), + ('x-request-id', str(uuid4())), + ] return br def economist_return_index(self, ans): @@ -209,10 +215,9 @@ class EconomistNews(BasicNewsRecipe): def parse_index(self): query = { - 'query': 'query HomeQuery($homeId:String!$relatedId:String!$podcastsId:String!){canonical(ref:$homeId){hasPart{parts{id title:headline cta{text url __typename}type hasPart{parts{isPartOf{id context{title:headline flyTitle:subheadline rubric:description dateline image{...ImageMainFragment ...ImagePromoFragment __typename}__typename}__typename}...ArticleFragment hasPart{parts{...ArticleFragment isPartOf{id context{title:headline flyTitle:subheadline rubric:description dateline image{...ImageMainFragment ...ImagePromoFragment __typename}__typename}__typename}hasPart{parts{...ArticleFragment isPartOf{id context{title:headline flyTitle:subheadline rubric:description dateline image{...ImageMainFragment ...ImagePromoFragment __typename}__typename}__typename}__typename}__typename}__typename}__typename}isPartOf{id context{title:headline flyTitle:subheadline rubric:description dateline image{...ImageMainFragment ...ImagePromoFragment __typename}__typename}__typename}__typename}__typename}__typename}__typename}__typename}relatedTopStories:canonical(ref:$relatedId){id title:headline hasPart(size:2 sort:"datePublished:desc"){parts{...ArticleFragment isPartOf{id context{title:headline flyTitle:subheadline rubric:description dateline image{...ImageMainFragment ...ImagePromoFragment __typename}__typename}__typename}__typename}__typename}__typename}podcasts:canonical(ref:$podcastsId){id title:headline hasPart(size:6 sort:"datePublished:desc"){parts{...ArticleFragment isPartOf{id context{title:headline flyTitle:subheadline rubric:description dateline image{...ImageMainFragment ...ImagePromoFragment __typename}__typename}__typename}__typename}__typename}__typename}}fragment ArticleFragment on Content{articleSection{internal{id title:headline __typename}__typename}audio{main{id duration(format:"seconds")source:channel{id __typename}url{canonical __typename}__typename}__typename}byline dateline dateModified datePublished flyTitle:subheadline id image{...ImageInlineFragment ...ImageMainFragment ...ImagePromoFragment __typename}print{title:headline flyTitle:subheadline rubric:description section{id title:headline __typename}__typename}publication{id tegID title:headline flyTitle:subheadline datePublished regionsAllowed url{canonical __typename}__typename}rubric:description source:channel{id __typename}tegID text(format:"json")title:headline type url{canonical __typename}__typename}fragment ImageInlineFragment on Media{inline{url{canonical __typename}width height __typename}__typename}fragment ImageMainFragment on Media{main{url{canonical __typename}width height __typename}__typename}fragment ImagePromoFragment on Media{promo{url{canonical __typename}id width height __typename}__typename}', # noqa: E501 + 'query': 'query HomeQuery($homeId:String!$relatedId:String!){canonical(ref:$homeId){hasPart{parts{id title:headline cta{text url __typename}type hasPart{parts{...ArticleFragment ...VideoFragment hasPart{parts{...ArticleFragment isPartOf{id context{title:headline flyTitle:subheadline rubric:description dateline image{...ImageMainFragment ...ImagePromoFragment __typename}__typename}__typename}__typename}__typename}isPartOf{id context{title:headline flyTitle:subheadline rubric:description dateline image{...ImageMainFragment ...ImagePromoFragment __typename}__typename}__typename}__typename}__typename}__typename}__typename}__typename}relatedTopStories:canonical(ref:$relatedId){id title:headline hasPart(size:2 sort:"datePublished:desc"){parts{...ArticleFragment isPartOf{id context{title:headline flyTitle:subheadline rubric:description dateline image{...ImageMainFragment ...ImagePromoFragment __typename}__typename}__typename}__typename}__typename}__typename}}fragment ArticleFragment on Content{ad{grapeshot{channels{name __typename}__typename}__typename}articleSection{internal{id title:headline __typename}__typename}audio{main{id duration(format:"seconds")source:channel{id __typename}url{canonical __typename}__typename}__typename}byline dateline dateModified datePublished dateRevised flyTitle:subheadline id image{...ImageInlineFragment ...ImageMainFragment ...ImagePromoFragment __typename}print{title:headline flyTitle:subheadline rubric:description section{id title:headline __typename}__typename}publication{id tegID title:headline flyTitle:subheadline datePublished regionsAllowed url{canonical __typename}__typename}rubric:description source:channel{id __typename}tegID text(format:"json")title:headline type url{canonical __typename}topic contentIdentity{forceAppWebview mediaType articleType __typename}__typename}fragment ImageInlineFragment on Media{inline{url{canonical __typename}width height __typename}__typename}fragment ImageMainFragment on Media{main{url{canonical __typename}width height __typename}__typename}fragment ImagePromoFragment on Media{promo{url{canonical __typename}id width height __typename}__typename}fragment VideoFragment on Content{video{playlist{playlistId __typename}__typename}__typename}', # noqa: E501 'operationName': 'HomeQuery', - 'variables': '{"homeId":"/content/8mmm7h9v7arvfpvn4n20hakmg4ugatur",' - '"relatedId":"/content/c7kho74htgua3gif74fa4bnbjr64i1js","podcastsId":"/content/omi23dr8h15h8c33t2gkb2cju8ap758o"}', + 'variables': '{"homeId":"/content/mgo2tcc3u3002m4gndvffg3kqv7n5n3g","relatedId":"/content/bp252fp9p9dvkn6pcjog5cks9hhnrf96"}', } url = 'https://cp2-graphql-gateway.p.aws.economist.com/graphql?' + urlencode(query, safe='()!', quote_via=quote) try: diff --git a/recipes/reuters.recipe b/recipes/reuters.recipe index a7aec2e335..c9ccb365f9 100644 --- a/recipes/reuters.recipe +++ b/recipes/reuters.recipe @@ -52,6 +52,11 @@ class Reuters(BasicNewsRecipe): 'long': 'This is useful for non e-ink devices', 'default': '480', }, + 'spr': { + 'short': 'Include Sports sections?' + 'long':'Yes/No' + 'default': 'No' + } } def __init__(self, *args, **kwargs): @@ -82,6 +87,12 @@ class Reuters(BasicNewsRecipe): feeds = [] for sec, link in sections: + sp = self.recipe_specific_options.get('spr') + if sp and isinstance(sp, str): + if sp.lower().strip() != 'yes': + if sec.lower().startswith('sport'): + continue + self.log(sec) articles = []