mirror of
https://github.com/kovidgoyal/calibre.git
synced 2026-01-05 11:40:20 -05:00
306 lines
12 KiB
Python
306 lines
12 KiB
Python
#!/usr/bin/env python
|
|
import json
|
|
import re
|
|
import time
|
|
from datetime import datetime, timedelta
|
|
|
|
from calibre.web.feeds.news import BasicNewsRecipe
|
|
|
|
|
|
def extract_json(raw):
|
|
pre = re.search(r'<script>window.__preloadedData = ({.+)', raw).group(1)
|
|
js = json.JSONDecoder().raw_decode(re.sub('undefined', '[]', pre))[0]
|
|
return js['initialData']['data']['article']['sprinkledBody']['content']
|
|
|
|
def parse_image(i):
|
|
if i.get('crops'):
|
|
yield '<div><img src="{}">'.format(i['crops'][0]['renditions'][0]['url'])
|
|
elif i.get('spanImageCrops'):
|
|
yield '<div><img src="{}">'.format(i['spanImageCrops'][0]['renditions'][0]['url'])
|
|
if i.get('caption'):
|
|
yield '<div class="cap">' + ''.join(parse_types(i['caption']))
|
|
if i.get('credit'):
|
|
yield '<span class="cred"> ' + i['credit'] + '</span>'
|
|
yield '</div>'
|
|
yield '</div>'
|
|
|
|
def parse_img_grid(g):
|
|
for grd in g.get('gridMedia', {}):
|
|
yield ''.join(parse_image(grd))
|
|
if g.get('caption'):
|
|
yield '<div class="cap">{}'.format(g['caption'])
|
|
if g.get('credit'):
|
|
yield '<span class="cred"> ' + g['credit'] + '</span>'
|
|
yield '</div>'
|
|
|
|
def parse_vid(v):
|
|
if v.get('promotionalMedia'):
|
|
if v.get('headline'):
|
|
if v.get('url'):
|
|
yield '<div><b><a href="{}">Video</a>: '.format(v['url'])\
|
|
+ v['headline'].get('default', '') + '</b></div>'
|
|
elif v['headline'].get('default'):
|
|
yield '<div><b>' + v['headline']['default'] + '</b></div>'
|
|
yield ''.join(parse_types(v['promotionalMedia']))
|
|
if v.get('promotionalSummary'):
|
|
yield '<div class="cap">' + v['promotionalSummary'] + '</div>'
|
|
|
|
def parse_emb(e):
|
|
if e.get('html') and 'datawrapper.dwcdn.net' in e.get('html', ''):
|
|
dw = re.search(r'datawrapper.dwcdn.net/(.{5})', e['html']).group(1)
|
|
yield '<div><img src="{}">'.format('https://datawrapper.dwcdn.net/' + dw + '/full.png') + '</div>'
|
|
elif e.get('promotionalMedia'):
|
|
if e.get('headline'):
|
|
yield '<div><b>' + e['headline']['default'] + '</b></div>'
|
|
yield ''.join(parse_types(e['promotionalMedia']))
|
|
if e.get('note'):
|
|
yield '<div class="cap">' + e['note'] + '</div>'
|
|
|
|
def parse_byline(byl):
|
|
for b in byl.get('bylines', {}):
|
|
yield '<div>' + b['renderedRepresentation'] + '</div>'
|
|
yield '<div><b><i>'
|
|
for rl in byl.get('role', {}):
|
|
if ''.join(parse_cnt(rl)).strip():
|
|
yield ''.join(parse_cnt(rl))
|
|
yield '</i></b></div>'
|
|
|
|
def iso_date(x):
|
|
dt = datetime.fromisoformat(x[:-1]) + timedelta(seconds=time.timezone)
|
|
return dt.strftime('%b %d, %Y at %I:%M %p')
|
|
|
|
def parse_header(h):
|
|
if h.get('label'):
|
|
yield '<div class="lbl">' + ''.join(parse_types(h['label'])) + '</div>'
|
|
if h.get('headline'):
|
|
yield ''.join(parse_types(h['headline']))
|
|
if h.get('summary'):
|
|
yield '<p><i>' + ''.join(parse_types(h['summary'])) + '</i></p>'
|
|
if h.get('ledeMedia'):
|
|
yield ''.join(parse_types(h['ledeMedia']))
|
|
if h.get('byline'):
|
|
yield ''.join(parse_types(h['byline']))
|
|
if h.get('timestampBlock'):
|
|
yield ''.join(parse_types(h['timestampBlock']))
|
|
|
|
def parse_fmt_type(fm):
|
|
for f in fm.get('formats', {}):
|
|
if f.get('__typename', '') == 'BoldFormat':
|
|
yield '<strong>'
|
|
if f.get('__typename', '') == 'ItalicFormat':
|
|
yield '<em>'
|
|
if f.get('__typename', '') == 'LinkFormat':
|
|
hrf = f['url']
|
|
yield '<a href="{}">'.format(hrf)
|
|
yield fm['text']
|
|
for f in reversed(fm.get('formats', {})):
|
|
if f.get('__typename', '') == 'BoldFormat':
|
|
yield '</strong>'
|
|
if f.get('__typename', '') == 'ItalicFormat':
|
|
yield '</em>'
|
|
if f.get('__typename', '') == 'LinkFormat':
|
|
yield '</a>'
|
|
|
|
def parse_cnt(cnt):
|
|
if cnt.get('formats'):
|
|
yield ''.join(parse_fmt_type(cnt))
|
|
elif cnt.get('content'):
|
|
for cnt_ in cnt['content']:
|
|
yield from parse_types(cnt_)
|
|
elif cnt.get('text'):
|
|
yield cnt['text']
|
|
|
|
def parse_types(x):
|
|
if 'Header' in x.get('__typename', ''):
|
|
yield '\n'.join(parse_header(x))
|
|
|
|
elif x.get('__typename', '') == 'Heading1Block':
|
|
yield '<h1>' + ''.join(parse_cnt(x)) + '</h1>'
|
|
elif x.get('__typename', '') in {'Heading2Block', 'Heading3Block', 'Heading4Block'}:
|
|
yield '<h4>' + ''.join(parse_cnt(x)) + '</h4>'
|
|
|
|
elif x.get('__typename', '') == 'ParagraphBlock':
|
|
yield '<p>' + ''.join(parse_cnt(x)) + '</p>'
|
|
|
|
elif x.get('__typename', '') == 'BylineBlock':
|
|
yield '<div class="byl"><br/>' + ''.join(parse_byline(x)) + '</div>'
|
|
elif x.get('__typename', '') == 'LabelBlock':
|
|
yield '<div class="sc">' + ''.join(parse_cnt(x)) + '</div>'
|
|
elif x.get('__typename', '') == 'BlockquoteBlock':
|
|
yield '<blockquote>' + ''.join(parse_cnt(x)) + '</blockquote>'
|
|
elif x.get('__typename', '') == 'TimestampBlock':
|
|
yield '<div class="time">' + iso_date(x['timestamp']) + '</div>'
|
|
elif x.get('__typename', '') == 'LineBreakInline':
|
|
yield '<br/>'
|
|
elif x.get('__typename', '') == 'RuleBlock':
|
|
yield '<hr/>'
|
|
|
|
elif x.get('__typename', '') == 'Image':
|
|
yield ''.join(parse_image(x))
|
|
elif x.get('__typename', '') == 'ImageBlock':
|
|
yield ''.join(parse_image(x['media']))
|
|
elif x.get('__typename', '') == 'GridBlock':
|
|
yield ''.join(parse_img_grid(x))
|
|
|
|
elif x.get('__typename', '') == 'VideoBlock':
|
|
yield ''.join(parse_types(x['media']))
|
|
elif x.get('__typename', '') == 'Video':
|
|
yield ''.join(parse_vid(x))
|
|
|
|
elif x.get('__typename', '') == 'InteractiveBlock':
|
|
yield ''.join(parse_types(x['media']))
|
|
elif x.get('__typename', '') == 'EmbeddedInteractive':
|
|
yield ''.join(parse_emb(x))
|
|
|
|
elif x.get('__typename', '') == 'ListBlock':
|
|
yield '<ul>' + ''.join(parse_cnt(x)) + '</ul>'
|
|
elif x.get('__typename', '') == 'ListItemBlock':
|
|
yield '<li>' + ''.join(parse_cnt(x)) + '</li>'
|
|
|
|
elif x.get('__typename', '') == 'CapsuleBlock':
|
|
if x['capsuleContent'].get('body'):
|
|
yield ''.join(parse_cnt(x['capsuleContent']['body']))
|
|
elif x.get('__typename', '') == 'Capsule':
|
|
yield ''.join(parse_cnt(x['body']))
|
|
|
|
elif x.get('__typename', '') in {
|
|
'TextInline', 'TextOnlyDocumentBlock', 'DocumentBlock', 'SummaryBlock'
|
|
}:
|
|
yield ''.join(parse_cnt(x))
|
|
|
|
elif x.get('__typename'):
|
|
if ''.join(parse_cnt(x)).strip():
|
|
yield '<p><i>' + ''.join(parse_cnt(x)) + '</i></p>'
|
|
|
|
def article_parse(data):
|
|
yield "<html><body>"
|
|
for d in data:
|
|
yield from parse_types(d)
|
|
yield "</body></html>"
|
|
|
|
|
|
class nytFeeds(BasicNewsRecipe):
|
|
title = 'NYT News'
|
|
__author__ = 'unkn0wn'
|
|
description = (
|
|
'The New York Times is dedicated to helping people understand the world through '
|
|
'on-the-ground, expert and deeply reported independent journalism. Feeds based recipe.'
|
|
)
|
|
oldest_article = 1
|
|
encoding = 'utf-8'
|
|
use_embedded_content = False
|
|
language = 'en_US'
|
|
remove_empty_feeds = True
|
|
resolve_internal_links = True
|
|
ignore_duplicate_articles = {'title', 'url'}
|
|
masthead_url = 'https://static01.nytimes.com/newsgraphics/2015-12-23-masthead-2016/b15c3d81d3d7b59065fff9a3f3afe85aa2e2dff5/_assets/nyt-logo.png'
|
|
|
|
def get_cover_url(self):
|
|
soup = self.index_to_soup('https://www.frontpages.com/the-new-york-times/')
|
|
return 'https://www.frontpages.com' + soup.find('img', attrs={'id':'giornale-img'})['src']
|
|
|
|
recipe_specific_options = {
|
|
'days': {
|
|
'short': 'Oldest article to download from this news source. In days ',
|
|
'long': 'For example, 0.5, gives you articles from the past 12 hours',
|
|
'default': str(oldest_article)
|
|
},
|
|
'comp': {
|
|
'short': 'Compress News Images?',
|
|
'long': 'enter yes',
|
|
'default': 'no'
|
|
},
|
|
'rev': {
|
|
'short': 'Reverse the order of articles in each feed?',
|
|
'long': 'enter yes',
|
|
'default': 'no'
|
|
},
|
|
'res': {
|
|
'short': 'For hi-res images, select a resolution from the following\noptions: popup, jumbo, mobileMasterAt3x, superJumbo',
|
|
'long': 'This is useful for non e-ink devices, and for a lower file size\nthan the default, use articleInline.',
|
|
}
|
|
}
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
BasicNewsRecipe.__init__(self, *args, **kwargs)
|
|
d = self.recipe_specific_options.get('days')
|
|
if d and isinstance(d, str):
|
|
self.oldest_article = float(d)
|
|
r = self.recipe_specific_options.get('rev')
|
|
if r and isinstance(r, str):
|
|
if r.lower() == 'yes':
|
|
self.reverse_article_order = True
|
|
c = self.recipe_specific_options.get('comp')
|
|
if c and isinstance(c, str):
|
|
if c.lower() == 'yes':
|
|
self.compress_news_images = True
|
|
|
|
extra_css = '''
|
|
.byl, .time { font-size:small; color:#202020; }
|
|
.cap { font-size:small; text-align:center; }
|
|
.cred { font-style:italic; font-size:small; }
|
|
em, blockquote { color: #202020; }
|
|
.sc { font-variant: small-caps; }
|
|
.lbl { font-size:small; color:#404040; }
|
|
img { display:block; margin:0 auto; }
|
|
'''
|
|
|
|
# https://www.nytimes.com/rss
|
|
# https://developer.nytimes.com/docs/rss-api/1/overview
|
|
feeds = [
|
|
# to filter out all opinions from other sections first
|
|
'https://rss.nytimes.com/services/xml/rss/nyt/Opinion.xml',
|
|
|
|
'https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml',
|
|
'https://rss.nytimes.com/services/xml/rss/nyt/World.xml',
|
|
'https://rss.nytimes.com/services/xml/rss/nyt/US.xml',
|
|
'https://rss.nytimes.com/services/xml/rss/nyt/Business.xml',
|
|
'https://rss.nytimes.com/services/xml/rss/nyt/YourMoney.xml',
|
|
'https://rss.nytimes.com/services/xml/rss/nyt/Technology.xml',
|
|
'https://rss.nytimes.com/services/xml/rss/nyt/Science.xml',
|
|
'https://rss.nytimes.com/services/xml/rss/nyt/Climate.xml',
|
|
'https://rss.nytimes.com/services/xml/rss/nyt/Health.xml',
|
|
'https://rss.nytimes.com/services/xml/rss/nyt/Arts.xml',
|
|
'https://rss.nytimes.com/services/xml/rss/nyt/FashionandStyle.xml',
|
|
'https://rss.nytimes.com/services/xml/rss/nyt/tmagazine.xml',
|
|
'https://rss.nytimes.com/services/xml/rss/nyt/books.xml',
|
|
'https://www.nytimes.com/services/xml/rss/nyt/Travel.xml',
|
|
'http://nytimes.com/timeswire/feeds/'
|
|
]
|
|
|
|
def get_browser(self, *args, **kwargs):
|
|
kwargs['user_agent'] = 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
|
|
br = BasicNewsRecipe.get_browser(self, *args, **kwargs)
|
|
br.addheaders += [
|
|
('Referer', 'https://www.google.com/'),
|
|
('X-Forwarded-For', '66.249.66.1')
|
|
]
|
|
return br
|
|
|
|
def preprocess_raw_html(self, raw_html, url):
|
|
if '/interactive/' in url:
|
|
return '<html><body><p><em>'\
|
|
+ 'This is an interactive article, which is supposed to be read in a browser.'\
|
|
+ '</p></em></body></html>'
|
|
data = extract_json(raw_html)
|
|
return '\n'.join(article_parse(data))
|
|
|
|
def preprocess_html(self, soup):
|
|
w = self.recipe_specific_options.get('res')
|
|
if w and isinstance(w, str):
|
|
res = '-' + w
|
|
for img in soup.findAll('img', attrs={'src':True}):
|
|
if '-article' in img['src']:
|
|
ext = img['src'].split('?')[0].split('.')[-1]
|
|
img['src'] = img['src'].rsplit('-article', 1)[0] + res + '.' + ext
|
|
for c in soup.findAll('div', attrs={'class':'cap'}):
|
|
for p in c.findAll(['p', 'div']):
|
|
p.name = 'span'
|
|
return soup
|
|
|
|
def get_article_url(self, article):
|
|
url = BasicNewsRecipe.get_article_url(self, article)
|
|
# you can remove '|/espanol/' from code below to include spanish articles.
|
|
if not re.search(r'/video/|/live/|/athletic/|/espanol/', url):
|
|
return url
|