From 2f962ad21c7841c27f965cee7b86a8aa4809b071 Mon Sep 17 00:00:00 2001 From: unkn0w7n <51942695+unkn0w7n@users.noreply.github.com> Date: Sat, 16 Mar 2024 20:17:57 +0530 Subject: [PATCH 1/2] POLITICO.EU recipe --- recipes/icons/politico_eu.png | Bin 0 -> 464 bytes recipes/poliitico_eu.recipe | 101 ++++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 recipes/icons/politico_eu.png create mode 100644 recipes/poliitico_eu.recipe diff --git a/recipes/icons/politico_eu.png b/recipes/icons/politico_eu.png new file mode 100644 index 0000000000000000000000000000000000000000..cf7d1fdc77192adc38984cbf3c0add4da8f1281b GIT binary patch literal 464 zcmV;>0WbcEP)Z}dh5(YFZt1fKH}dI$t>VP5J3L>+@ZL3B*&5Y#8IQ-l&lV0E-o zA*{4o+uiA~Q?0q$9|neJp8pFo^N4*2o3I&yxDBRFZwMd+uQnXM!n)B9TjP+e1G-v{ zTHger>O$2t7#@LD37R;DN*|0_>d*y-78V$op9j~)ZZ>INUhR#vFx^R#Ymuo?|Q0lhsFyqul!TqqERKr;Mu ziF!2J_5%Y2M1Y8HngEDUH(w7DG($qmub>0fg9ARxW!_4qt`RjaSxcP2_5@_=cmpK( z9VUa$cv6`R0uQibG+?a)G6IVZm{K+Rh5S&H*{H&yfVD3hi-fr6djEj{0000]*>([^<]*)', re.DOTALL|re.IGNORECASE), lambda match: match.group(1))] + + remove_tags_after = dict(attrs={'class': lambda x: x and 'article__more-from' in x.split()}) + + extra_css = ''' + body{font-family:Arial,Sans-serif;} + element.style{color:#FF0000;font-family:Arial,Sans-serif;} + .author{color:#808080;font-size:x-small;} + a{ color:#003399;} + .byline{color:#696969 ; font-size:x-small;} + .story{color:#000000;} + td{color:#000000;} + .figcaption__inner, .article-meta, .authors, .datetime {font-size:small; } + ''' + + def get_article_url(self, article): + url = BasicNewsRecipe.get_article_url(self, article) + if 'politico.com' not in url: + return url.split('?')[0] + + masthead_url = 'https://www.politico.eu/cdn-cgi/image/width=573,quality=80,onerror=redirect,format=auto/wp-content/uploads/2021/02/25/Politico_logo-01.png' + + def get_cover_url(self): + soup = self.index_to_soup('https://www.politico.eu/') + for cov in soup.findAll(attrs={'class':'cta__image'}): + if cov.find('a', attrs={'href':lambda x: x and 'edition.pagesuite-professional' in x}): + return cov.a.img['src'] + + feeds = [ + ('Policy', 'https://www.politico.eu/section/policy/feed'), + ('Opinion', 'https://www.politico.eu/section/opinion/feed'), + ('Newsletter', 'https://www.politico.eu/newsletter/feed'), + ('Others', 'https://www.politico.eu/feed') + ] + + def postprocess_html(self, soup, first): + for tag in soup.findAll(name=['table', 'tr', 'td']): + tag.name = 'div' + return soup From 911e15e63bb292151f05b76a4e305dd2f24fab3f Mon Sep 17 00:00:00 2001 From: unkn0w7n <51942695+unkn0w7n@users.noreply.github.com> Date: Sat, 16 Mar 2024 20:24:18 +0530 Subject: [PATCH 2/2] ... --- recipes/livemint.recipe | 6 ++++-- recipes/natgeo.recipe | 8 ++++---- recipes/natgeohis.recipe | 8 ++++---- recipes/natgeomag.recipe | 8 ++++---- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/recipes/livemint.recipe b/recipes/livemint.recipe index b59bfb6b12..2c075d8f78 100644 --- a/recipes/livemint.recipe +++ b/recipes/livemint.recipe @@ -103,7 +103,7 @@ class LiveMint(BasicNewsRecipe): dict(name=['meta', 'link', 'svg', 'button', 'iframe']), classes( 'trendingSimilarHeight moreNews mobAppDownload label msgError msgOk taboolaHeight gadgetSlider' - ' socialHolder imgbig disclamerText disqus-comment-count openinApp2 lastAdSlot bs_logo' + ' socialHolder imgbig disclamerText disqus-comment-count openinApp2 lastAdSlot bs_logo manualbacklink' ' datePublish sepStory premiumSlider moreStory Joinus moreAbout milestone benefitText' ) ] @@ -132,7 +132,9 @@ class LiveMint(BasicNewsRecipe): # remove empty p tags raw = re.sub( r'(

\s*)(<[^(\/|a|i|b|em|strong)])', '\g<2>', re.sub( - r'(

\s* \s*<\/p>)|(

\s*<\/p>)|( \s*<\/p>)', '', raw + r'(

\s* \s*<\/p>)|(

\s*<\/p>)|( \s*<\/p>)', '', re.sub( + r'(?=

\s*Also\s*Read).*?(?<=

)', '', raw + ) ) ) if '' in raw: diff --git a/recipes/natgeo.recipe b/recipes/natgeo.recipe index 4f902af9cc..40e3da9b8a 100644 --- a/recipes/natgeo.recipe +++ b/recipes/natgeo.recipe @@ -35,7 +35,8 @@ def parse_lead_image(media): yield '

' if 'dsc' in media['image']: yield '

{}
'.format( - escape(media['image']['src'], True), escape(media['image']['dsc'], True)) + escape(media['image']['src'], True), escape(media['image']['dsc'], True) + ) else: yield '
'.format(escape(media['image']['src'], True)) if 'caption' in media and 'credit' in media: @@ -84,7 +85,7 @@ def parse_body(x): if tag == 'inline': yield ''.join(parse_inline(x)) elif 'attrs' in x and 'href' in x.get('attrs', ''): - yield '<' + tag + ' href = "{}">'.format(x['attrs']['href']) + yield '<' + tag + ' href="{}">'.format(x['attrs']['href']) for yld in parse_cont(x): yield yld yield '' @@ -120,8 +121,7 @@ def parse_article(edg): for item in main['props']['body']: if isinstance(item, dict): if item.get('type', '') == 'inline': - for inl in parse_inline(item): - yield inl + yield ''.join(parse_inline(item)) elif isinstance(item, list): for line in item: yield ''.join(parse_body(line)) diff --git a/recipes/natgeohis.recipe b/recipes/natgeohis.recipe index d52a63f7c5..aee7c7caa0 100644 --- a/recipes/natgeohis.recipe +++ b/recipes/natgeohis.recipe @@ -34,7 +34,8 @@ def parse_lead_image(media): yield '

' if 'dsc' in media['image']: yield '

{}
'.format( - escape(media['image']['src'], True), escape(media['image']['dsc'], True)) + escape(media['image']['src'], True), escape(media['image']['dsc'], True) + ) else: yield '
'.format(escape(media['image']['src'], True)) if 'caption' in media and 'credit' in media: @@ -83,7 +84,7 @@ def parse_body(x): if tag == 'inline': yield ''.join(parse_inline(x)) elif 'attrs' in x and 'href' in x.get('attrs', ''): - yield '<' + tag + ' href = "{}">'.format(x['attrs']['href']) + yield '<' + tag + ' href="{}">'.format(x['attrs']['href']) for yld in parse_cont(x): yield yld yield '' @@ -119,8 +120,7 @@ def parse_article(edg): for item in main['props']['body']: if isinstance(item, dict): if item.get('type', '') == 'inline': - for inl in parse_inline(item): - yield inl + yield ''.join(parse_inline(item)) elif isinstance(item, list): for line in item: yield ''.join(parse_body(line)) diff --git a/recipes/natgeomag.recipe b/recipes/natgeomag.recipe index d4c179c88c..0ad082923e 100644 --- a/recipes/natgeomag.recipe +++ b/recipes/natgeomag.recipe @@ -39,7 +39,8 @@ def parse_lead_image(media): yield '

' if 'dsc' in media['image']: yield '

{}
'.format( - escape(media['image']['src'], True), escape(media['image']['dsc'], True)) + escape(media['image']['src'], True), escape(media['image']['dsc'], True) + ) else: yield '
'.format(escape(media['image']['src'], True)) if 'caption' in media and 'credit' in media: @@ -88,7 +89,7 @@ def parse_body(x): if tag == 'inline': yield ''.join(parse_inline(x)) elif 'attrs' in x and 'href' in x.get('attrs', ''): - yield '<' + tag + ' href = "{}">'.format(x['attrs']['href']) + yield '<' + tag + ' href="{}">'.format(x['attrs']['href']) for yld in parse_cont(x): yield yld yield '' @@ -124,8 +125,7 @@ def parse_article(edg): for item in main['props']['body']: if isinstance(item, dict): if item.get('type', '') == 'inline': - for inl in parse_inline(item): - yield inl + yield ''.join(parse_inline(item)) elif isinstance(item, list): for line in item: yield ''.join(parse_body(line))