Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-07 18:24:30 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/unkn0w7n/calibre

Browse Source
This commit is contained in:
Kovid Goyal 2023-10-10 04:37:37 +05:30
commit 9bce31229d
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
5 changed files with 97 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
recipes/natgeo.recipe
View File

@ -33,10 +33,10 @@ def parse_contributors(grp):
def parse_lead_image(media):
if 'dsc' in media['image']:
yield '<div><img src="{}" alt="{}"></div>'.format(
yield '<p><div><img src="{}" alt="{}"></div>'.format(
escape(media['image']['src'], True), escape(media['image']['dsc'], True))
else:
yield '<div><img src="{}"></div>'.format(escape(media['image']['src'], True))
yield '<p><div><img src="{}"></div>'.format(escape(media['image']['src'], True))
if 'caption' in media:
yield '<div class="cap">' + media['caption'] + '</div>'
if 'credit' in media:
@ -122,9 +122,11 @@ class NatGeo(BasicNewsRecipe):
remove_attributes = ['style']
remove_javascript = False
masthead_url = 'https://i.natgeofe.com/n/e76f5368-6797-4794-b7f6-8d757c79ea5c/ng-logo-2fl.png?w=600&h=600'
remove_empty_feeds = True
resolve_internal_links = True
extra_css = '''
.sub { color:#404040; }
.sub, blockquote { color:#404040; }
.byline, i { font-style:italic; color:#202020; }
.cap {text-align:center; font-size:small; }
.cred {text-align:center; font-size:small; color:#404040; }

7
recipes/natgeohis.recipe
View File

@ -32,10 +32,10 @@ def parse_contributors(grp):
def parse_lead_image(media):
if 'dsc' in media['image']:
yield '<div><img src="{}" alt="{}"></div>'.format(
yield '<p><div><img src="{}" alt="{}"></div>'.format(
escape(media['image']['src'], True), escape(media['image']['dsc'], True))
else:
yield '<div><img src="{}"></div>'.format(escape(media['image']['src'], True))
yield '<p><div><img src="{}"></div>'.format(escape(media['image']['src'], True))
if 'caption' in media:
yield '<div class="cap">' + media['caption'] + '</div>'
if 'credit' in media:
@ -124,9 +124,10 @@ class NatGeo(BasicNewsRecipe):
remove_attributes = ['style']
remove_javascript = False
masthead_url = 'https://i.natgeofe.com/n/e76f5368-6797-4794-b7f6-8d757c79ea5c/ng-logo-2fl.png?w=600&h=600'
resolve_internal_links = True
extra_css = '''
.sub { color:#404040; }
.sub, blockquote { color:#404040; }
.byline, i { font-style:italic; color:#202020; }
.cap {text-align:center; font-size:small; }
.cred {text-align:center; font-size:small; color:#404040; }

14
recipes/natgeomag.recipe
View File

@ -10,6 +10,9 @@ from calibre.web.feeds.news import BasicNewsRecipe
from calibre import prepare_string_for_xml as escape
from calibre.utils.iso8601 import parse_iso8601
edition = date.today().strftime('%B-%Y')
# edition = 'March-2023'
def classes(classes):
q = frozenset(classes.split(' '))
@ -34,10 +37,10 @@ def parse_contributors(grp):
def parse_lead_image(media):
if 'dsc' in media['image']:
yield '<div><img src="{}" alt="{}"></div>'.format(
yield '<p><div><img src="{}" alt="{}"></div>'.format(
escape(media['image']['src'], True), escape(media['image']['dsc'], True))
else:
yield '<div><img src="{}"></div>'.format(escape(media['image']['src'], True))
yield '<p><div><img src="{}"></div>'.format(escape(media['image']['src'], True))
if 'caption' in media:
yield '<div class="cap">' + media['caption'] + '</div>'
if 'credit' in media:
@ -124,9 +127,10 @@ class NatGeo(BasicNewsRecipe):
remove_javascript = False
masthead_url = 'https://i.natgeofe.com/n/e76f5368-6797-4794-b7f6-8d757c79ea5c/ng-logo-2fl.png?w=600&h=600'
remove_empty_feeds = True
resolve_internal_links = True
extra_css = '''
.sub { color:#404040; }
.sub, blockquote { color:#404040; }
.byline, i { font-style:italic; color:#202020; }
.cap {text-align:center; font-size:small; }
.cred {text-align:center; font-size:small; color:#404040; }
@ -134,9 +138,9 @@ class NatGeo(BasicNewsRecipe):
'''
def parse_index(self):
url = 'https://www.nationalgeographic.com/magazine/issue/' + date.today().strftime('%B-%Y'). lower()
url = 'https://www.nationalgeographic.com/magazine/issue/' + edition.lower()
self.log('Downloading ', url)
self.timefmt = ' [' + date.today().strftime('%B %Y') + ']'
self.timefmt = ' [' + edition + ']'
soup = self.index_to_soup(url)
png = re.findall('https://i\.natgeofe\.com\S+?national-geographic-magazine-\S+?\.jpg', soup.decode('utf-8'))
self.cover_url = png[0] + '?w=1000&h=1000'

62
recipes/wash_post.recipe
View File

@ -5,17 +5,13 @@ www.washingtonpost.com
'''
from calibre.web.feeds.news import BasicNewsRecipe
def classes(classes):
q = frozenset(classes.split(' '))
return dict(attrs={
'class': lambda x: x and frozenset(x.split()).intersection(q)})
from html5_parser import parse
import json
class TheWashingtonPost(BasicNewsRecipe):
title = 'The Washington Post'
__author__ = 'Darko Miletic'
__author__ = 'Darko Miletic, unkn0wn'
description = 'Leading source for news, video and opinion on politics, business, world and national news, science, travel, entertainment and more. Our local coverage includes reporting on education, crime, weather, traffic, real estate, jobs and cars for DC, Maryland and Virginia. Offering award-winning opinion writing, entertainment information and restaurant reviews.' # noqa
publisher = 'The Washington Post Company'
category = 'news, politics, USA'
@ -30,16 +26,11 @@ class TheWashingtonPost(BasicNewsRecipe):
publication_type = 'newspaper'
remove_attributes = ['style', 'width', 'height']
keep_only_tags = [
dict(name=['h1', 'figure']),
dict(attrs={'data-qa': 'lede-art'}),
classes('byline article-body'),
]
remove_tags = [
dict(name=['meta', 'link', 'svg']),
classes('inline-video author-tooltip author-image powa-wrapper'),
dict(attrs={'data-qa': ['article-body-ad', 'subscribe-promo', 'interstitial-link-wrapper']}),
]
extra_css = '''
.img { text-align:center; font-size:small; }
.auth { font-weight:bold; font-size:small; }
.time { font-size:small; color: #202020; }
'''
# Official feeds: https://www.washingtonpost.com/discussions/2018/10/12/washington-post-rss-feeds/
feeds = [
@ -61,9 +52,36 @@ class TheWashingtonPost(BasicNewsRecipe):
(u'Commanders', u'http://feeds.washingtonpost.com/rss/sports/redskins'),
]
def preprocess_html(self, soup, *a):
for img in soup.findAll('img', src=True):
src = img['src']
if src.endswith('&w=32'):
img['src'] = src[:-2] + '440'
def preprocess_raw_html(self, raw, *a):
root = parse(raw)
m = root.xpath('//script[@id="__NEXT_DATA__"]')
data = json.loads(m[0].text)
data = data['props']['pageProps']['globalContent']
title = '<h1>' + data['headlines']['basic'] + '</h1>'
subhead = '<h3>' + data['description'].get('basic', '') + '</h3>'
author = ''
if 'credits' in data:
author = '<div><span class="auth">' + 'By ' + ', '.join(x['name'] for x in data['credits']['by']) \
+ '</span> | <span class="time">' + data['publish_date'][:-14] + '</span></div>'
body = ''
for x in data['content_elements']:
if x['type'] == 'text':
body += '<p>' + x['content'] + '</p>'
elif x['type'] == 'video':
if 'promo_image' in x:
body += '<p><div class="img"><img src="{}"><div>{}</div></div></p>'.format(
x['promo_image']['url'], x['description'].get('basic', '')
)
elif x['type'] == 'image':
body += '<p><div class="img"><img src="{}"><div>{}</div></div></p>'.format(x['url'], x['credits_caption_display'])
return '<html><body><div>' + title + subhead + author + body + '</div></body></html>'
def preprocess_html(self, soup):
for img in soup.findAll('img', attrs={'src':True}):
img['src'] = 'https://www.washingtonpost.com/wp-apps/imrs.php?src=' + img['src'] + '&w=540'
return soup

54
recipes/wash_post_print.recipe
View File

@ -2,7 +2,9 @@
washingtonpost.com
'''
from calibre.web.feeds.news import BasicNewsRecipe, classes
from calibre.web.feeds.news import BasicNewsRecipe
from html5_parser import parse
import json
class wapoprint(BasicNewsRecipe):
title = 'The Washington Post | Print Edition'
@ -22,18 +24,11 @@ class wapoprint(BasicNewsRecipe):
remove_attributes = ['style', 'height', 'width']
publication_type = 'newspaper'
ignore_duplicate_articles = {'title', 'url'}
keep_only_tags = [
dict(name=['h1', 'figure']),
dict(attrs={'data-qa': 'lede-art'}),
classes('byline article-body'),
]
remove_tags = [
dict(name=['meta', 'link', 'svg']),
classes('inline-video author-tooltip author-image powa-wrapper'),
dict(attrs={'data-qa': ['article-body-ad', 'subscribe-promo', 'interstitial-link-wrapper']}),
]
extra_css = '''
.img { text-align:center; font-size:small; }
.auth { font-weight:bold; font-size:small; }
.time { font-size:small; color: #202020; }
'''
def parse_index(self):
soup = self.index_to_soup('https://www.washingtonpost.com/todays_paper/updates/')
@ -58,7 +53,36 @@ class wapoprint(BasicNewsRecipe):
feeds.append((secname, articles))
return feeds
def preprocess_raw_html(self, raw, *a):
root = parse(raw)
m = root.xpath('//script[@id="__NEXT_DATA__"]')
data = json.loads(m[0].text)
data = data['props']['pageProps']['globalContent']
title = '<h1>' + data['headlines']['basic'] + '</h1>'
subhead = '<h3>' + data['description'].get('basic', '') + '</h3>'
author = ''
if 'credits' in data:
author = '<div><span class="auth">' + 'By ' + ', '.join(x['name'] for x in data['credits']['by']) \
+ '</span> | <span class="time">' + data['publish_date'][:-14] + '</span></div>'
body = ''
for x in data['content_elements']:
if x['type'] == 'text':
body += '<p>' + x['content'] + '</p>'
elif x['type'] == 'video':
if 'promo_image' in x:
body += '<p><div class="img"><img src="{}"><div>{}</div></div></p>'.format(
x['promo_image']['url'], x['description'].get('basic', '')
)
elif x['type'] == 'image':
body += '<p><div class="img"><img src="{}"><div>{}</div></div></p>'.format(x['url'], x['credits_caption_display'])
return '<html><body><div>' + title + subhead + author + body + '</div></body></html>'
def preprocess_html(self, soup):
for img in soup.findAll('img', srcset=True):
img['src'] = img['srcset'].split()[0]
for img in soup.findAll('img', attrs={'src':True}):
img['src'] = 'https://www.washingtonpost.com/wp-apps/imrs.php?src=' + img['src'] + '&w=540'
return soup