From e987f88036df2bd1acb28f8b38967d4060d1e343 Mon Sep 17 00:00:00 2001
From: unkn0w7n <51942695+unkn0w7n@users.noreply.github.com>
Date: Sat, 19 Apr 2025 08:21:55 +0530
Subject: [PATCH 1/3] Update wash_post.recipe
---
recipes/wash_post.recipe | 81 ++++++++++++++++++++++++++--------------
1 file changed, 54 insertions(+), 27 deletions(-)
diff --git a/recipes/wash_post.recipe b/recipes/wash_post.recipe
index 741d589557..d5a9ad42c7 100644
--- a/recipes/wash_post.recipe
+++ b/recipes/wash_post.recipe
@@ -16,7 +16,14 @@ from calibre.web.feeds.news import BasicNewsRecipe
class TheWashingtonPost(BasicNewsRecipe):
title = 'The Washington Post'
__author__ = 'Darko Miletic, unkn0wn'
- description = 'Leading source for news, video and opinion on politics, business, world and national news, science, travel, entertainment and more. Our local coverage includes reporting on education, crime, weather, traffic, real estate, jobs and cars for DC, Maryland and Virginia. Offering award-winning opinion writing, entertainment information and restaurant reviews.' # noqa: E501
+ description = (
+ 'Leading source for news, video and opinion on politics, business, '
+ 'world and national news, science, travel, entertainment and more. '
+ 'Our local coverage includes reporting on education, crime, weather, '
+ 'traffic, real estate, jobs and cars for DC, Maryland and Virginia. '
+ 'Offering award-winning opinion writing, entertainment information '
+ 'and restaurant reviews.'
+ )
publisher = 'The Washington Post Company'
category = 'news, politics, USA'
oldest_article = 1.2
@@ -27,7 +34,7 @@ class TheWashingtonPost(BasicNewsRecipe):
language = 'en_US'
remove_empty_feeds = True
ignore_duplicate_articles = {'url'}
- masthead_url = 'https://www.washingtonpost.com/pb/resources/img/twp-masthead-415x57.svg'
+ masthead_url = 'https://upload.wikimedia.org/wikipedia/commons/9/93/The_Logo_of_The_Washington_Post_Newspaper.svg'
publication_type = 'newspaper'
remove_attributes = ['style', 'width', 'height']
@@ -35,7 +42,7 @@ class TheWashingtonPost(BasicNewsRecipe):
'days': {
'short': 'Oldest article to download from this news source. In days ',
'long': 'For example, 0.5, gives you articles from the past 12 hours',
- 'default': str(oldest_article)
+ 'default': str(oldest_article),
}
}
@@ -51,28 +58,32 @@ class TheWashingtonPost(BasicNewsRecipe):
.time { font-size:small; color: #202020; }
'''
- def get_browser(self):
- return BasicNewsRecipe.get_browser(
- self, verify_ssl_certificates=False, user_agent='Mozilla/5.0 (Windows NT 10.0; rv:128.0) Gecko/20100101 Firefox/128.0')
+ def get_browser(self, *args, **kwargs):
+ kwargs['user_agent'] = (
+ 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
+ )
+ br = BasicNewsRecipe.get_browser(self, *args, **kwargs)
+ br.addheaders += [
+ ('Referer', 'https://www.google.com/'),
+ ('X-Forwarded-For', '66.249.66.1'),
+ ]
+ return br
# Official feeds: https://www.washingtonpost.com/discussions/2018/10/12/washington-post-rss-feeds/
feeds = [
- (u'Politics', u'http://feeds.washingtonpost.com/rss/politics'),
- (u'Opinions', u'http://feeds.washingtonpost.com/rss/opinions'),
- (u'Local', u'http://feeds.washingtonpost.com/rss/local'),
- (u'Sports', u'http://feeds.washingtonpost.com/rss/sports'),
- (u'Technology', u'http://feeds.washingtonpost.com/rss/business/technology'),
- (u'National', u'http://feeds.washingtonpost.com/rss/national'),
- (u'World', u'http://feeds.washingtonpost.com/rss/world'),
- (u'Business', u'http://feeds.washingtonpost.com/rss/business'),
- (u'Lifestyle', u'http://feeds.washingtonpost.com/rss/lifestyle'),
- (u'Entertainment',
- u'http://feeds.washingtonpost.com/rss/entertainment'),
-
+ ('Politics', 'http://feeds.washingtonpost.com/rss/politics'),
+ ('Opinions', 'http://feeds.washingtonpost.com/rss/opinions'),
+ ('Local', 'http://feeds.washingtonpost.com/rss/local'),
+ ('Sports', 'http://feeds.washingtonpost.com/rss/sports'),
+ ('Technology', 'http://feeds.washingtonpost.com/rss/business/technology'),
+ ('National', 'http://feeds.washingtonpost.com/rss/national'),
+ ('World', 'http://feeds.washingtonpost.com/rss/world'),
+ ('Business', 'http://feeds.washingtonpost.com/rss/business'),
+ ('Lifestyle', 'http://feeds.washingtonpost.com/rss/lifestyle'),
+ ('Entertainment', 'http://feeds.washingtonpost.com/rss/entertainment'),
# Undocumented feeds.
- (u'White House',
- u'http://feeds.washingtonpost.com/rss/politics/whitehouse'),
- (u'Commanders', u'http://feeds.washingtonpost.com/rss/sports/redskins'),
+ ('White House', 'http://feeds.washingtonpost.com/rss/politics/whitehouse'),
+ ('Commanders', 'http://feeds.washingtonpost.com/rss/sports/redskins'),
]
def preprocess_raw_html(self, raw, *a):
@@ -87,8 +98,14 @@ class TheWashingtonPost(BasicNewsRecipe):
author = ''
if 'credits' in data:
- author = ('' + 'By ' + ', '.join(x['name'] for x in data['credits']['by']) +
- ' | ' + data['publish_date'][:-14] + '
')
+ author = (
+ ''
+ + 'By '
+ + ', '.join(x['name'] for x in data['credits']['by'])
+ + ' | '
+ + data['publish_date'][:-14]
+ + '
'
+ )
body = ''
for x in data['content_elements']:
@@ -100,11 +117,21 @@ class TheWashingtonPost(BasicNewsRecipe):
x['promo_image']['url'], x['description'].get('basic', '')
)
elif x['type'] == 'image':
- body += '
{}
{}
' + title + subhead + author + body + '
'
+ return (
+ '' + title + subhead + author + body + '
'
+ )
def preprocess_html(self, soup):
- for img in soup.findAll('img', attrs={'src':True}):
- img['src'] = 'https://www.washingtonpost.com/wp-apps/imrs.php?src=' + img['src'] + '&w=916'
+ for img in soup.findAll('img', attrs={'src': True}):
+ img['src'] = (
+ 'https://www.washingtonpost.com/wp-apps/imrs.php?src='
+ + img['src']
+ + '&w=916'
+ )
return soup
From 7fe681d98feb6fb5da81588736ea1acd7fb2b134 Mon Sep 17 00:00:00 2001
From: unkn0w7n <51942695+unkn0w7n@users.noreply.github.com>
Date: Sat, 19 Apr 2025 08:22:16 +0530
Subject: [PATCH 2/3] Update wash_post_print.recipe
---
recipes/wash_post_print.recipe | 63 +++++++++++++++++++++++++---------
1 file changed, 46 insertions(+), 17 deletions(-)
diff --git a/recipes/wash_post_print.recipe b/recipes/wash_post_print.recipe
index 14db183c43..15e851dd38 100644
--- a/recipes/wash_post_print.recipe
+++ b/recipes/wash_post_print.recipe
@@ -15,10 +15,12 @@ class wapoprint(BasicNewsRecipe):
title = 'The Washington Post | Print Edition'
__author__ = 'unkn0wn'
description = (
- 'Leading source for news, video and opinion on politics, business, world and national news, science,'
- ' travel, entertainment and more. Our local coverage includes reporting on education, crime, weather,'
- ' traffic, real estate, jobs and cars for DC, Maryland and Virginia. Offering award-winning opinion writing,'
- ' entertainment information and restaurant reviews.'
+ 'Leading source for news, video and opinion on politics, business, '
+ 'world and national news, science, travel, entertainment and more. '
+ 'Our local coverage includes reporting on education, crime, weather, '
+ 'traffic, real estate, jobs and cars for DC, Maryland and Virginia. '
+ 'Offering award-winning opinion writing, entertainment information '
+ 'and restaurant reviews.'
)
publisher = 'The Washington Post Company'
category = 'news, politics, USA'
@@ -29,22 +31,33 @@ class wapoprint(BasicNewsRecipe):
remove_attributes = ['style', 'height', 'width']
publication_type = 'newspaper'
ignore_duplicate_articles = {'title', 'url'}
- masthead_url = 'https://www.washingtonpost.com/pb/resources/img/twp-masthead-415x57.svg'
+ masthead_url = 'https://upload.wikimedia.org/wikipedia/commons/9/93/The_Logo_of_The_Washington_Post_Newspaper.svg'
extra_css = '''
.img { text-align:center; font-size:small; }
.auth { font-weight:bold; font-size:small; }
.time { font-size:small; color: #202020; }
'''
- def get_browser(self):
- return BasicNewsRecipe.get_browser(
- self, verify_ssl_certificates=False, user_agent='Mozilla/5.0 (Windows NT 10.0; rv:128.0) Gecko/20100101 Firefox/128.0')
+ def get_browser(self, *args, **kwargs):
+ kwargs['user_agent'] = (
+ 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
+ )
+ br = BasicNewsRecipe.get_browser(self, *args, **kwargs)
+ br.addheaders += [
+ ('Referer', 'https://www.google.com/'),
+ ('X-Forwarded-For', '66.249.66.1'),
+ ]
+ return br
def parse_index(self):
soup = self.index_to_soup('https://www.washingtonpost.com/todays_paper/updates/')
- if img := soup.find('img', attrs={'src':lambda x: x and x.endswith('_FrontPage.png')}):
- self.cover_url = img['src']
- if h2 := soup.find('h2', attrs={'class':lambda x: x and 'font--subhead' in x.split()}):
+ if img := soup.find(
+ 'img', attrs={'src': lambda x: x and x.endswith('_FrontPage.png')}
+ ):
+ self.cover_url = img['src']
+ if h2 := soup.find(
+ 'h2', attrs={'class': lambda x: x and 'font--subhead' in x.split()}
+ ):
self.title = 'WaPo Print | ' + self.tag_to_string(h2)
feeds = []
@@ -77,8 +90,14 @@ class wapoprint(BasicNewsRecipe):
author = ''
if 'credits' in data:
- author = ('' + 'By ' + ', '.join(x['name'] for x in data['credits']['by']) +
- ' | ' + data['publish_date'][:-14] + '
')
+ author = (
+ ''
+ + 'By '
+ + ', '.join(x['name'] for x in data['credits']['by'])
+ + ' | '
+ + data['publish_date'][:-14]
+ + '
'
+ )
body = ''
for x in data['content_elements']:
@@ -90,13 +109,23 @@ class wapoprint(BasicNewsRecipe):
x['promo_image']['url'], x['description'].get('basic', '')
)
elif x['type'] == 'image':
- body += '{}
{}
' + title + subhead + author + body + '
'
+ return (
+ '' + title + subhead + author + body + '
'
+ )
def preprocess_html(self, soup):
- for img in soup.findAll('img', attrs={'src':True}):
- img['src'] = 'https://www.washingtonpost.com/wp-apps/imrs.php?src=' + img['src'] + '&w=916'
+ for img in soup.findAll('img', attrs={'src': True}):
+ img['src'] = (
+ 'https://www.washingtonpost.com/wp-apps/imrs.php?src='
+ + img['src']
+ + '&w=916'
+ )
return soup
def populate_article_metadata(self, article, soup, first):
From c58089b459006cafc52b8c8c1db43a01ffb40af0 Mon Sep 17 00:00:00 2001
From: unkn0w7n <51942695+unkn0w7n@users.noreply.github.com>
Date: Sat, 19 Apr 2025 08:27:17 +0530
Subject: [PATCH 3/3] Update wash_post.recipe
add cover
---
recipes/wash_post.recipe | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/recipes/wash_post.recipe b/recipes/wash_post.recipe
index d5a9ad42c7..995e6bf5c2 100644
--- a/recipes/wash_post.recipe
+++ b/recipes/wash_post.recipe
@@ -58,6 +58,13 @@ class TheWashingtonPost(BasicNewsRecipe):
.time { font-size:small; color: #202020; }
'''
+ def get_cover_url(self):
+ soup = self.index_to_soup('https://www.washingtonpost.com/todays_paper/updates/')
+ if img := soup.find(
+ 'img', attrs={'src': lambda x: x and x.endswith('_FrontPage.png')}
+ ):
+ return img['src']
+
def get_browser(self, *args, **kwargs):
kwargs['user_agent'] = (
'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'