mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-07-07 10:14:46 -04:00
Merge branch 'master' of https://github.com/unkn0w7n/calibre
This commit is contained in:
Kovid Goyal
commit
9bce31229d
@ -33,10 +33,10 @@ def parse_contributors(grp):
|
||||
|
||||
def parse_lead_image(media):
|
||||
if 'dsc' in media['image']:
|
||||
yield '<div><img src="{}" alt="{}"></div>'.format(
|
||||
yield '<p><div><img src="{}" alt="{}"></div>'.format(
|
||||
escape(media['image']['src'], True), escape(media['image']['dsc'], True))
|
||||
else:
|
||||
yield '<div><img src="{}"></div>'.format(escape(media['image']['src'], True))
|
||||
yield '<p><div><img src="{}"></div>'.format(escape(media['image']['src'], True))
|
||||
if 'caption' in media:
|
||||
yield '<div class="cap">' + media['caption'] + '</div>'
|
||||
if 'credit' in media:
|
||||
@ -122,9 +122,11 @@ class NatGeo(BasicNewsRecipe):
|
||||
remove_attributes = ['style']
|
||||
remove_javascript = False
|
||||
masthead_url = 'https://i.natgeofe.com/n/e76f5368-6797-4794-b7f6-8d757c79ea5c/ng-logo-2fl.png?w=600&h=600'
|
||||
remove_empty_feeds = True
|
||||
resolve_internal_links = True
|
||||
|
||||
extra_css = '''
|
||||
.sub { color:#404040; }
|
||||
.sub, blockquote { color:#404040; }
|
||||
.byline, i { font-style:italic; color:#202020; }
|
||||
.cap {text-align:center; font-size:small; }
|
||||
.cred {text-align:center; font-size:small; color:#404040; }
|
||||
|
||||
@ -32,10 +32,10 @@ def parse_contributors(grp):
|
||||
|
||||
def parse_lead_image(media):
|
||||
if 'dsc' in media['image']:
|
||||
yield '<div><img src="{}" alt="{}"></div>'.format(
|
||||
yield '<p><div><img src="{}" alt="{}"></div>'.format(
|
||||
escape(media['image']['src'], True), escape(media['image']['dsc'], True))
|
||||
else:
|
||||
yield '<div><img src="{}"></div>'.format(escape(media['image']['src'], True))
|
||||
yield '<p><div><img src="{}"></div>'.format(escape(media['image']['src'], True))
|
||||
if 'caption' in media:
|
||||
yield '<div class="cap">' + media['caption'] + '</div>'
|
||||
if 'credit' in media:
|
||||
@ -124,9 +124,10 @@ class NatGeo(BasicNewsRecipe):
|
||||
remove_attributes = ['style']
|
||||
remove_javascript = False
|
||||
masthead_url = 'https://i.natgeofe.com/n/e76f5368-6797-4794-b7f6-8d757c79ea5c/ng-logo-2fl.png?w=600&h=600'
|
||||
resolve_internal_links = True
|
||||
|
||||
extra_css = '''
|
||||
.sub { color:#404040; }
|
||||
.sub, blockquote { color:#404040; }
|
||||
.byline, i { font-style:italic; color:#202020; }
|
||||
.cap {text-align:center; font-size:small; }
|
||||
.cred {text-align:center; font-size:small; color:#404040; }
|
||||
|
||||
@ -10,6 +10,9 @@ from calibre.web.feeds.news import BasicNewsRecipe
|
||||
from calibre import prepare_string_for_xml as escape
|
||||
from calibre.utils.iso8601 import parse_iso8601
|
||||
|
||||
edition = date.today().strftime('%B-%Y')
|
||||
|
||||
# edition = 'March-2023'
|
||||
|
||||
def classes(classes):
|
||||
q = frozenset(classes.split(' '))
|
||||
@ -34,10 +37,10 @@ def parse_contributors(grp):
|
||||
|
||||
def parse_lead_image(media):
|
||||
if 'dsc' in media['image']:
|
||||
yield '<div><img src="{}" alt="{}"></div>'.format(
|
||||
yield '<p><div><img src="{}" alt="{}"></div>'.format(
|
||||
escape(media['image']['src'], True), escape(media['image']['dsc'], True))
|
||||
else:
|
||||
yield '<div><img src="{}"></div>'.format(escape(media['image']['src'], True))
|
||||
yield '<p><div><img src="{}"></div>'.format(escape(media['image']['src'], True))
|
||||
if 'caption' in media:
|
||||
yield '<div class="cap">' + media['caption'] + '</div>'
|
||||
if 'credit' in media:
|
||||
@ -124,9 +127,10 @@ class NatGeo(BasicNewsRecipe):
|
||||
remove_javascript = False
|
||||
masthead_url = 'https://i.natgeofe.com/n/e76f5368-6797-4794-b7f6-8d757c79ea5c/ng-logo-2fl.png?w=600&h=600'
|
||||
remove_empty_feeds = True
|
||||
resolve_internal_links = True
|
||||
|
||||
extra_css = '''
|
||||
.sub { color:#404040; }
|
||||
.sub, blockquote { color:#404040; }
|
||||
.byline, i { font-style:italic; color:#202020; }
|
||||
.cap {text-align:center; font-size:small; }
|
||||
.cred {text-align:center; font-size:small; color:#404040; }
|
||||
@ -134,9 +138,9 @@ class NatGeo(BasicNewsRecipe):
|
||||
'''
|
||||
|
||||
def parse_index(self):
|
||||
url = 'https://www.nationalgeographic.com/magazine/issue/' + date.today().strftime('%B-%Y'). lower()
|
||||
url = 'https://www.nationalgeographic.com/magazine/issue/' + edition.lower()
|
||||
self.log('Downloading ', url)
|
||||
self.timefmt = ' [' + date.today().strftime('%B %Y') + ']'
|
||||
self.timefmt = ' [' + edition + ']'
|
||||
soup = self.index_to_soup(url)
|
||||
png = re.findall('https://i\.natgeofe\.com\S+?national-geographic-magazine-\S+?\.jpg', soup.decode('utf-8'))
|
||||
self.cover_url = png[0] + '?w=1000&h=1000'
|
||||
|
||||
@ -5,17 +5,13 @@ www.washingtonpost.com
|
||||
'''
|
||||
|
||||
from calibre.web.feeds.news import BasicNewsRecipe
|
||||
|
||||
|
||||
def classes(classes):
|
||||
q = frozenset(classes.split(' '))
|
||||
return dict(attrs={
|
||||
'class': lambda x: x and frozenset(x.split()).intersection(q)})
|
||||
from html5_parser import parse
|
||||
import json
|
||||
|
||||
|
||||
class TheWashingtonPost(BasicNewsRecipe):
|
||||
title = 'The Washington Post'
|
||||
__author__ = 'Darko Miletic'
|
||||
__author__ = 'Darko Miletic, unkn0wn'
|
||||
description = 'Leading source for news, video and opinion on politics, business, world and national news, science, travel, entertainment and more. Our local coverage includes reporting on education, crime, weather, traffic, real estate, jobs and cars for DC, Maryland and Virginia. Offering award-winning opinion writing, entertainment information and restaurant reviews.' # noqa
|
||||
publisher = 'The Washington Post Company'
|
||||
category = 'news, politics, USA'
|
||||
@ -30,16 +26,11 @@ class TheWashingtonPost(BasicNewsRecipe):
|
||||
publication_type = 'newspaper'
|
||||
remove_attributes = ['style', 'width', 'height']
|
||||
|
||||
keep_only_tags = [
|
||||
dict(name=['h1', 'figure']),
|
||||
dict(attrs={'data-qa': 'lede-art'}),
|
||||
classes('byline article-body'),
|
||||
]
|
||||
remove_tags = [
|
||||
dict(name=['meta', 'link', 'svg']),
|
||||
classes('inline-video author-tooltip author-image powa-wrapper'),
|
||||
dict(attrs={'data-qa': ['article-body-ad', 'subscribe-promo', 'interstitial-link-wrapper']}),
|
||||
]
|
||||
extra_css = '''
|
||||
.img { text-align:center; font-size:small; }
|
||||
.auth { font-weight:bold; font-size:small; }
|
||||
.time { font-size:small; color: #202020; }
|
||||
'''
|
||||
|
||||
# Official feeds: https://www.washingtonpost.com/discussions/2018/10/12/washington-post-rss-feeds/
|
||||
feeds = [
|
||||
@ -61,9 +52,36 @@ class TheWashingtonPost(BasicNewsRecipe):
|
||||
(u'Commanders', u'http://feeds.washingtonpost.com/rss/sports/redskins'),
|
||||
]
|
||||
|
||||
def preprocess_html(self, soup, *a):
|
||||
for img in soup.findAll('img', src=True):
|
||||
src = img['src']
|
||||
if src.endswith('&w=32'):
|
||||
img['src'] = src[:-2] + '440'
|
||||
def preprocess_raw_html(self, raw, *a):
|
||||
root = parse(raw)
|
||||
m = root.xpath('//script[@id="__NEXT_DATA__"]')
|
||||
|
||||
data = json.loads(m[0].text)
|
||||
data = data['props']['pageProps']['globalContent']
|
||||
|
||||
title = '<h1>' + data['headlines']['basic'] + '</h1>'
|
||||
subhead = '<h3>' + data['description'].get('basic', '') + '</h3>'
|
||||
|
||||
author = ''
|
||||
if 'credits' in data:
|
||||
author = '<div><span class="auth">' + 'By ' + ', '.join(x['name'] for x in data['credits']['by']) \
|
||||
+ '</span> | <span class="time">' + data['publish_date'][:-14] + '</span></div>'
|
||||
|
||||
body = ''
|
||||
for x in data['content_elements']:
|
||||
if x['type'] == 'text':
|
||||
body += '<p>' + x['content'] + '</p>'
|
||||
elif x['type'] == 'video':
|
||||
if 'promo_image' in x:
|
||||
body += '<p><div class="img"><img src="{}"><div>{}</div></div></p>'.format(
|
||||
x['promo_image']['url'], x['description'].get('basic', '')
|
||||
)
|
||||
elif x['type'] == 'image':
|
||||
body += '<p><div class="img"><img src="{}"><div>{}</div></div></p>'.format(x['url'], x['credits_caption_display'])
|
||||
|
||||
return '<html><body><div>' + title + subhead + author + body + '</div></body></html>'
|
||||
|
||||
def preprocess_html(self, soup):
|
||||
for img in soup.findAll('img', attrs={'src':True}):
|
||||
img['src'] = 'https://www.washingtonpost.com/wp-apps/imrs.php?src=' + img['src'] + '&w=540'
|
||||
return soup
|
||||
|
||||
@ -2,7 +2,9 @@
|
||||
washingtonpost.com
|
||||
'''
|
||||
|
||||
from calibre.web.feeds.news import BasicNewsRecipe, classes
|
||||
from calibre.web.feeds.news import BasicNewsRecipe
|
||||
from html5_parser import parse
|
||||
import json
|
||||
|
||||
class wapoprint(BasicNewsRecipe):
|
||||
title = 'The Washington Post | Print Edition'
|
||||
@ -22,18 +24,11 @@ class wapoprint(BasicNewsRecipe):
|
||||
remove_attributes = ['style', 'height', 'width']
|
||||
publication_type = 'newspaper'
|
||||
ignore_duplicate_articles = {'title', 'url'}
|
||||
|
||||
keep_only_tags = [
|
||||
dict(name=['h1', 'figure']),
|
||||
dict(attrs={'data-qa': 'lede-art'}),
|
||||
classes('byline article-body'),
|
||||
]
|
||||
|
||||
remove_tags = [
|
||||
dict(name=['meta', 'link', 'svg']),
|
||||
classes('inline-video author-tooltip author-image powa-wrapper'),
|
||||
dict(attrs={'data-qa': ['article-body-ad', 'subscribe-promo', 'interstitial-link-wrapper']}),
|
||||
]
|
||||
extra_css = '''
|
||||
.img { text-align:center; font-size:small; }
|
||||
.auth { font-weight:bold; font-size:small; }
|
||||
.time { font-size:small; color: #202020; }
|
||||
'''
|
||||
|
||||
def parse_index(self):
|
||||
soup = self.index_to_soup('https://www.washingtonpost.com/todays_paper/updates/')
|
||||
@ -58,7 +53,36 @@ class wapoprint(BasicNewsRecipe):
|
||||
feeds.append((secname, articles))
|
||||
return feeds
|
||||
|
||||
def preprocess_raw_html(self, raw, *a):
|
||||
root = parse(raw)
|
||||
m = root.xpath('//script[@id="__NEXT_DATA__"]')
|
||||
|
||||
data = json.loads(m[0].text)
|
||||
data = data['props']['pageProps']['globalContent']
|
||||
|
||||
title = '<h1>' + data['headlines']['basic'] + '</h1>'
|
||||
subhead = '<h3>' + data['description'].get('basic', '') + '</h3>'
|
||||
|
||||
author = ''
|
||||
if 'credits' in data:
|
||||
author = '<div><span class="auth">' + 'By ' + ', '.join(x['name'] for x in data['credits']['by']) \
|
||||
+ '</span> | <span class="time">' + data['publish_date'][:-14] + '</span></div>'
|
||||
|
||||
body = ''
|
||||
for x in data['content_elements']:
|
||||
if x['type'] == 'text':
|
||||
body += '<p>' + x['content'] + '</p>'
|
||||
elif x['type'] == 'video':
|
||||
if 'promo_image' in x:
|
||||
body += '<p><div class="img"><img src="{}"><div>{}</div></div></p>'.format(
|
||||
x['promo_image']['url'], x['description'].get('basic', '')
|
||||
)
|
||||
elif x['type'] == 'image':
|
||||
body += '<p><div class="img"><img src="{}"><div>{}</div></div></p>'.format(x['url'], x['credits_caption_display'])
|
||||
|
||||
return '<html><body><div>' + title + subhead + author + body + '</div></body></html>'
|
||||
|
||||
def preprocess_html(self, soup):
|
||||
for img in soup.findAll('img', srcset=True):
|
||||
img['src'] = img['srcset'].split()[0]
|
||||
for img in soup.findAll('img', attrs={'src':True}):
|
||||
img['src'] = 'https://www.washingtonpost.com/wp-apps/imrs.php?src=' + img['src'] + '&w=540'
|
||||
return soup
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user