From 6d1846abe7629709b8dd9ef96b3d8a30944b9ad7 Mon Sep 17 00:00:00 2001 From: unkn0w7n <51942695+unkn0w7n@users.noreply.github.com> Date: Mon, 9 Oct 2023 22:32:20 +0530 Subject: [PATCH] Update wash_post.recipe --- recipes/wash_post.recipe | 62 ++++++++++++++++++++++++++-------------- 1 file changed, 40 insertions(+), 22 deletions(-) diff --git a/recipes/wash_post.recipe b/recipes/wash_post.recipe index c0cf2c750f..1e275ebc1c 100644 --- a/recipes/wash_post.recipe +++ b/recipes/wash_post.recipe @@ -5,17 +5,13 @@ www.washingtonpost.com ''' from calibre.web.feeds.news import BasicNewsRecipe - - -def classes(classes): - q = frozenset(classes.split(' ')) - return dict(attrs={ - 'class': lambda x: x and frozenset(x.split()).intersection(q)}) +from html5_parser import parse +import json class TheWashingtonPost(BasicNewsRecipe): title = 'The Washington Post' - __author__ = 'Darko Miletic' + __author__ = 'Darko Miletic, unkn0wn' description = 'Leading source for news, video and opinion on politics, business, world and national news, science, travel, entertainment and more. Our local coverage includes reporting on education, crime, weather, traffic, real estate, jobs and cars for DC, Maryland and Virginia. Offering award-winning opinion writing, entertainment information and restaurant reviews.' # noqa publisher = 'The Washington Post Company' category = 'news, politics, USA' @@ -30,16 +26,11 @@ class TheWashingtonPost(BasicNewsRecipe): publication_type = 'newspaper' remove_attributes = ['style', 'width', 'height'] - keep_only_tags = [ - dict(name=['h1', 'figure']), - dict(attrs={'data-qa': 'lede-art'}), - classes('byline article-body'), - ] - remove_tags = [ - dict(name=['meta', 'link', 'svg']), - classes('inline-video author-tooltip author-image powa-wrapper'), - dict(attrs={'data-qa': ['article-body-ad', 'subscribe-promo', 'interstitial-link-wrapper']}), - ] + extra_css = ''' + .img { text-align:center; font-size:small; } + .auth { font-weight:bold; font-size:small; } + .time { font-size:small; color: #202020; } + ''' # Official feeds: https://www.washingtonpost.com/discussions/2018/10/12/washington-post-rss-feeds/ feeds = [ @@ -61,9 +52,36 @@ class TheWashingtonPost(BasicNewsRecipe): (u'Commanders', u'http://feeds.washingtonpost.com/rss/sports/redskins'), ] - def preprocess_html(self, soup, *a): - for img in soup.findAll('img', src=True): - src = img['src'] - if src.endswith('&w=32'): - img['src'] = src[:-2] + '440' + def preprocess_raw_html(self, raw, *a): + root = parse(raw) + m = root.xpath('//script[@id="__NEXT_DATA__"]') + + data = json.loads(m[0].text) + data = data['props']['pageProps']['globalContent'] + + title = '

' + data['headlines']['basic'] + '

' + subhead = '

' + data['description'].get('basic', '') + '

' + + author = '' + if 'credits' in data: + author = '
' + 'By ' + ', '.join(x['name'] for x in data['credits']['by']) \ + + ' | ' + data['publish_date'][:-14] + '
' + + body = '' + for x in data['content_elements']: + if x['type'] == 'text': + body += '

' + x['content'] + '

' + elif x['type'] == 'video': + if 'promo_image' in x: + body += '

{}

'.format( + x['promo_image']['url'], x['description'].get('basic', '') + ) + elif x['type'] == 'image': + body += '

{}

'.format(x['url'], x['credits_caption_display']) + + return '
' + title + subhead + author + body + '
' + + def preprocess_html(self, soup): + for img in soup.findAll('img', attrs={'src':True}): + img['src'] = 'https://www.washingtonpost.com/wp-apps/imrs.php?src=' + img['src'] + '&w=540' return soup