From b22c7fddf88242a0f1af5c82e23a498608538603 Mon Sep 17 00:00:00 2001
From: unkn0w7n <51942695+unkn0w7n@users.noreply.github.com>
Date: Mon, 9 Oct 2023 22:27:17 +0530
Subject: [PATCH 1/4] update Washington Post Print
---
recipes/wash_post_print.recipe | 54 ++++++++++++++++++++++++----------
1 file changed, 39 insertions(+), 15 deletions(-)
diff --git a/recipes/wash_post_print.recipe b/recipes/wash_post_print.recipe
index bf033691b3..625bcb68f5 100644
--- a/recipes/wash_post_print.recipe
+++ b/recipes/wash_post_print.recipe
@@ -2,7 +2,9 @@
washingtonpost.com
'''
-from calibre.web.feeds.news import BasicNewsRecipe, classes
+from calibre.web.feeds.news import BasicNewsRecipe
+from html5_parser import parse
+import json
class wapoprint(BasicNewsRecipe):
title = 'The Washington Post | Print Edition'
@@ -22,18 +24,11 @@ class wapoprint(BasicNewsRecipe):
remove_attributes = ['style', 'height', 'width']
publication_type = 'newspaper'
ignore_duplicate_articles = {'title', 'url'}
-
- keep_only_tags = [
- dict(name=['h1', 'figure']),
- dict(attrs={'data-qa': 'lede-art'}),
- classes('byline article-body'),
- ]
-
- remove_tags = [
- dict(name=['meta', 'link', 'svg']),
- classes('inline-video author-tooltip author-image powa-wrapper'),
- dict(attrs={'data-qa': ['article-body-ad', 'subscribe-promo', 'interstitial-link-wrapper']}),
- ]
+ extra_css = '''
+ .img { text-align:center; font-size:small; }
+ .auth { font-weight:bold; font-size:small; }
+ .time { font-size:small; color: #202020; }
+ '''
def parse_index(self):
soup = self.index_to_soup('https://www.washingtonpost.com/todays_paper/updates/')
@@ -58,7 +53,36 @@ class wapoprint(BasicNewsRecipe):
feeds.append((secname, articles))
return feeds
+ def preprocess_raw_html(self, raw, *a):
+ root = parse(raw)
+ m = root.xpath('//script[@id="__NEXT_DATA__"]')
+
+ data = json.loads(m[0].text)
+ data = data['props']['pageProps']['globalContent']
+
+ title = '' + data['headlines']['basic'] + '
'
+ subhead = '' + data['description'].get('basic', '') + '
'
+
+ author = ''
+ if 'credits' in data:
+ author = '' + 'By ' + ', '.join(x['name'] for x in data['credits']['by']) \
+ + ' | ' + data['publish_date'][:-14] + '
'
+
+ body = ''
+ for x in data['content_elements']:
+ if x['type'] == 'text':
+ body += '' + x['content'] + '
'
+ elif x['type'] == 'video':
+ if 'promo_image' in x:
+ body += '
{}
{}
' + title + subhead + author + body + '
'
+
def preprocess_html(self, soup):
- for img in soup.findAll('img', srcset=True):
- img['src'] = img['srcset'].split()[0]
+ for img in soup.findAll('img', attrs={'src':True}):
+ img['src'] = 'https://www.washingtonpost.com/wp-apps/imrs.php?src=' + img['src'] + '&w=540'
return soup
From 6d1846abe7629709b8dd9ef96b3d8a30944b9ad7 Mon Sep 17 00:00:00 2001
From: unkn0w7n <51942695+unkn0w7n@users.noreply.github.com>
Date: Mon, 9 Oct 2023 22:32:20 +0530
Subject: [PATCH 2/4] Update wash_post.recipe
---
recipes/wash_post.recipe | 62 ++++++++++++++++++++++++++--------------
1 file changed, 40 insertions(+), 22 deletions(-)
diff --git a/recipes/wash_post.recipe b/recipes/wash_post.recipe
index c0cf2c750f..1e275ebc1c 100644
--- a/recipes/wash_post.recipe
+++ b/recipes/wash_post.recipe
@@ -5,17 +5,13 @@ www.washingtonpost.com
'''
from calibre.web.feeds.news import BasicNewsRecipe
-
-
-def classes(classes):
- q = frozenset(classes.split(' '))
- return dict(attrs={
- 'class': lambda x: x and frozenset(x.split()).intersection(q)})
+from html5_parser import parse
+import json
class TheWashingtonPost(BasicNewsRecipe):
title = 'The Washington Post'
- __author__ = 'Darko Miletic'
+ __author__ = 'Darko Miletic, unkn0wn'
description = 'Leading source for news, video and opinion on politics, business, world and national news, science, travel, entertainment and more. Our local coverage includes reporting on education, crime, weather, traffic, real estate, jobs and cars for DC, Maryland and Virginia. Offering award-winning opinion writing, entertainment information and restaurant reviews.' # noqa
publisher = 'The Washington Post Company'
category = 'news, politics, USA'
@@ -30,16 +26,11 @@ class TheWashingtonPost(BasicNewsRecipe):
publication_type = 'newspaper'
remove_attributes = ['style', 'width', 'height']
- keep_only_tags = [
- dict(name=['h1', 'figure']),
- dict(attrs={'data-qa': 'lede-art'}),
- classes('byline article-body'),
- ]
- remove_tags = [
- dict(name=['meta', 'link', 'svg']),
- classes('inline-video author-tooltip author-image powa-wrapper'),
- dict(attrs={'data-qa': ['article-body-ad', 'subscribe-promo', 'interstitial-link-wrapper']}),
- ]
+ extra_css = '''
+ .img { text-align:center; font-size:small; }
+ .auth { font-weight:bold; font-size:small; }
+ .time { font-size:small; color: #202020; }
+ '''
# Official feeds: https://www.washingtonpost.com/discussions/2018/10/12/washington-post-rss-feeds/
feeds = [
@@ -61,9 +52,36 @@ class TheWashingtonPost(BasicNewsRecipe):
(u'Commanders', u'http://feeds.washingtonpost.com/rss/sports/redskins'),
]
- def preprocess_html(self, soup, *a):
- for img in soup.findAll('img', src=True):
- src = img['src']
- if src.endswith('&w=32'):
- img['src'] = src[:-2] + '440'
+ def preprocess_raw_html(self, raw, *a):
+ root = parse(raw)
+ m = root.xpath('//script[@id="__NEXT_DATA__"]')
+
+ data = json.loads(m[0].text)
+ data = data['props']['pageProps']['globalContent']
+
+ title = '' + data['headlines']['basic'] + '
'
+ subhead = '' + data['description'].get('basic', '') + '
'
+
+ author = ''
+ if 'credits' in data:
+ author = '' + 'By ' + ', '.join(x['name'] for x in data['credits']['by']) \
+ + ' | ' + data['publish_date'][:-14] + '
'
+
+ body = ''
+ for x in data['content_elements']:
+ if x['type'] == 'text':
+ body += '' + x['content'] + '
'
+ elif x['type'] == 'video':
+ if 'promo_image' in x:
+ body += '{}
{}
' + title + subhead + author + body + '
'
+
+ def preprocess_html(self, soup):
+ for img in soup.findAll('img', attrs={'src':True}):
+ img['src'] = 'https://www.washingtonpost.com/wp-apps/imrs.php?src=' + img['src'] + '&w=540'
return soup
From 5b624b2051a997f1f9327b01d9e38dce82dae175 Mon Sep 17 00:00:00 2001
From: unkn0w7n <51942695+unkn0w7n@users.noreply.github.com>
Date: Mon, 9 Oct 2023 22:37:12 +0530
Subject: [PATCH 3/4] NatGeo
---
recipes/natgeo.recipe | 4 +++-
recipes/natgeohis.recipe | 3 ++-
recipes/natgeomag.recipe | 10 +++++++---
3 files changed, 12 insertions(+), 5 deletions(-)
diff --git a/recipes/natgeo.recipe b/recipes/natgeo.recipe
index 91201f5c33..fa512763d2 100644
--- a/recipes/natgeo.recipe
+++ b/recipes/natgeo.recipe
@@ -122,9 +122,11 @@ class NatGeo(BasicNewsRecipe):
remove_attributes = ['style']
remove_javascript = False
masthead_url = 'https://i.natgeofe.com/n/e76f5368-6797-4794-b7f6-8d757c79ea5c/ng-logo-2fl.png?w=600&h=600'
+ remove_empty_feeds = True
+ resolve_internal_links = True
extra_css = '''
- .sub { color:#404040; }
+ .sub, blockquote { color:#404040; }
.byline, i { font-style:italic; color:#202020; }
.cap {text-align:center; font-size:small; }
.cred {text-align:center; font-size:small; color:#404040; }
diff --git a/recipes/natgeohis.recipe b/recipes/natgeohis.recipe
index c4310c872c..33104376c3 100644
--- a/recipes/natgeohis.recipe
+++ b/recipes/natgeohis.recipe
@@ -124,9 +124,10 @@ class NatGeo(BasicNewsRecipe):
remove_attributes = ['style']
remove_javascript = False
masthead_url = 'https://i.natgeofe.com/n/e76f5368-6797-4794-b7f6-8d757c79ea5c/ng-logo-2fl.png?w=600&h=600'
+ resolve_internal_links = True
extra_css = '''
- .sub { color:#404040; }
+ .sub, blockquote { color:#404040; }
.byline, i { font-style:italic; color:#202020; }
.cap {text-align:center; font-size:small; }
.cred {text-align:center; font-size:small; color:#404040; }
diff --git a/recipes/natgeomag.recipe b/recipes/natgeomag.recipe
index e5fd4ec031..039360f9c4 100644
--- a/recipes/natgeomag.recipe
+++ b/recipes/natgeomag.recipe
@@ -10,6 +10,9 @@ from calibre.web.feeds.news import BasicNewsRecipe
from calibre import prepare_string_for_xml as escape
from calibre.utils.iso8601 import parse_iso8601
+edition = date.today().strftime('%B-%Y')
+
+# edition = 'March-2023'
def classes(classes):
q = frozenset(classes.split(' '))
@@ -124,9 +127,10 @@ class NatGeo(BasicNewsRecipe):
remove_javascript = False
masthead_url = 'https://i.natgeofe.com/n/e76f5368-6797-4794-b7f6-8d757c79ea5c/ng-logo-2fl.png?w=600&h=600'
remove_empty_feeds = True
+ resolve_internal_links = True
extra_css = '''
- .sub { color:#404040; }
+ .sub, blockquote { color:#404040; }
.byline, i { font-style:italic; color:#202020; }
.cap {text-align:center; font-size:small; }
.cred {text-align:center; font-size:small; color:#404040; }
@@ -134,9 +138,9 @@ class NatGeo(BasicNewsRecipe):
'''
def parse_index(self):
- url = 'https://www.nationalgeographic.com/magazine/issue/' + date.today().strftime('%B-%Y'). lower()
+ url = 'https://www.nationalgeographic.com/magazine/issue/' + edition.lower()
self.log('Downloading ', url)
- self.timefmt = ' [' + date.today().strftime('%B %Y') + ']'
+ self.timefmt = ' [' + edition + ']'
soup = self.index_to_soup(url)
png = re.findall('https://i\.natgeofe\.com\S+?national-geographic-magazine-\S+?\.jpg', soup.decode('utf-8'))
self.cover_url = png[0] + '?w=1000&h=1000'
From 45ac17216c64f4586c452a98108c0ec74ad89e88 Mon Sep 17 00:00:00 2001
From: unkn0w7n <51942695+unkn0w7n@users.noreply.github.com>
Date: Mon, 9 Oct 2023 22:56:03 +0530
Subject: [PATCH 4/4] ...
---
recipes/natgeo.recipe | 4 ++--
recipes/natgeohis.recipe | 4 ++--
recipes/natgeomag.recipe | 4 ++--
3 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/recipes/natgeo.recipe b/recipes/natgeo.recipe
index fa512763d2..75be28af39 100644
--- a/recipes/natgeo.recipe
+++ b/recipes/natgeo.recipe
@@ -33,10 +33,10 @@ def parse_contributors(grp):
def parse_lead_image(media):
if 'dsc' in media['image']:
- yield ''.format(
+ yield ''.format(
escape(media['image']['src'], True), escape(media['image']['dsc'], True))
else:
- yield ''.format(escape(media['image']['src'], True))
+ yield ''.format(escape(media['image']['src'], True))
if 'caption' in media:
yield '' + media['caption'] + '
'
if 'credit' in media:
diff --git a/recipes/natgeohis.recipe b/recipes/natgeohis.recipe
index 33104376c3..e23c17859a 100644
--- a/recipes/natgeohis.recipe
+++ b/recipes/natgeohis.recipe
@@ -32,10 +32,10 @@ def parse_contributors(grp):
def parse_lead_image(media):
if 'dsc' in media['image']:
- yield ''.format(
+ yield ''.format(
escape(media['image']['src'], True), escape(media['image']['dsc'], True))
else:
- yield ''.format(escape(media['image']['src'], True))
+ yield ''.format(escape(media['image']['src'], True))
if 'caption' in media:
yield '' + media['caption'] + '
'
if 'credit' in media:
diff --git a/recipes/natgeomag.recipe b/recipes/natgeomag.recipe
index 039360f9c4..9089ae444d 100644
--- a/recipes/natgeomag.recipe
+++ b/recipes/natgeomag.recipe
@@ -37,10 +37,10 @@ def parse_contributors(grp):
def parse_lead_image(media):
if 'dsc' in media['image']:
- yield ''.format(
+ yield ''.format(
escape(media['image']['src'], True), escape(media['image']['dsc'], True))
else:
- yield ''.format(escape(media['image']['src'], True))
+ yield ''.format(escape(media['image']['src'], True))
if 'caption' in media:
yield '' + media['caption'] + '
'
if 'credit' in media: