mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-07-09 03:04:10 -04:00
Merge branch 'master' of https://github.com/unkn0w7n/calibre
This commit is contained in:
Kovid Goyal
commit
fcad6379dd
BIN
recipes/icons/nytfeeds.png
Normal file
BIN
recipes/icons/nytfeeds.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 416 B |
184
recipes/nytfeeds.recipe
Normal file
184
recipes/nytfeeds.recipe
Normal file
@ -0,0 +1,184 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
import re
|
||||||
|
import json
|
||||||
|
import time
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
from calibre.web.feeds.news import BasicNewsRecipe
|
||||||
|
|
||||||
|
|
||||||
|
def extract_json(raw):
|
||||||
|
pre = re.search(r'<script>window.__preloadedData = ({.+)', raw).group(1)
|
||||||
|
js = json.JSONDecoder().raw_decode(re.sub('undefined', '[]', pre))[0]
|
||||||
|
return js['initialData']['data']['article']['sprinkledBody']['content']
|
||||||
|
|
||||||
|
def parse_image(i):
|
||||||
|
if i['__typename'] == 'Image':
|
||||||
|
yield '<div>'
|
||||||
|
yield '<img src="{}">'.format(i['crops'][0]['renditions'][0]['url'])
|
||||||
|
if 'caption' in i and i['caption']:
|
||||||
|
yield '<div class="cap">{}'.format(
|
||||||
|
i['caption'].get('text', '')
|
||||||
|
)
|
||||||
|
if 'credit' in i and i['credit']:
|
||||||
|
yield '<span class="cred"> ' + i['credit'] + '</span>'
|
||||||
|
yield '</div>'
|
||||||
|
yield '</div>'
|
||||||
|
|
||||||
|
def parse_img_grid(g):
|
||||||
|
for grd in g.get('gridMedia', {}):
|
||||||
|
yield ''.join(parse_image(grd))
|
||||||
|
if 'caption' in g and g['caption']:
|
||||||
|
yield '<div class="cap">{}'.format(g['caption'])
|
||||||
|
if 'credit' in g and g['credit']:
|
||||||
|
yield '<span class="cred"> ' + g['credit'] + '</span>'
|
||||||
|
yield '</div>'
|
||||||
|
|
||||||
|
def parse_cnt(cnt):
|
||||||
|
if cnt['__typename'] == 'TextInline':
|
||||||
|
if 'formats' in cnt and cnt['formats']:
|
||||||
|
for fmt in cnt.get('formats', {}):
|
||||||
|
if fmt['__typename'] == 'LinkFormat':
|
||||||
|
hrf = fmt['url']
|
||||||
|
yield '<a href="{}">'.format(hrf) + cnt['text'] + '</a>'
|
||||||
|
else:
|
||||||
|
yield cnt['text']
|
||||||
|
else:
|
||||||
|
yield cnt['text']
|
||||||
|
|
||||||
|
def parse_byline(byl):
|
||||||
|
for b in byl.get('bylines', {}):
|
||||||
|
yield '<div>' + b['renderedRepresentation'] + '</div>'
|
||||||
|
for rl in byl.get('role', {}):
|
||||||
|
yield '<div><i>' + ''.join(parse_cnt(rl)) + '</i></div>'
|
||||||
|
|
||||||
|
def iso_date(x):
|
||||||
|
dt = datetime.fromisoformat(x[:-1]) + timedelta(seconds=time.timezone)
|
||||||
|
return dt.strftime('%b %d, %Y at %I:%M %p')
|
||||||
|
|
||||||
|
def header_parse(h):
|
||||||
|
for ch in h['headline']['content']:
|
||||||
|
yield '<h1>' + ''.join(parse_cnt(ch)) + '</h1>'
|
||||||
|
if 'summary' in h and h['summary']:
|
||||||
|
for cs in h['summary']['content']:
|
||||||
|
yield '<p class="sub">' + ''.join(parse_cnt(cs)) + '</p>'
|
||||||
|
if 'ledeMedia' in h and h['ledeMedia']:
|
||||||
|
if h['ledeMedia'].get('__typename', '') == 'ImageBlock':
|
||||||
|
yield ''.join(parse_image(h['ledeMedia']['media']))
|
||||||
|
if 'byline' in h and h['byline']:
|
||||||
|
yield '<br><div class="byl">'
|
||||||
|
yield '\t'.join(parse_byline(h['byline']))
|
||||||
|
if 'timestampBlock' in h and h['timestampBlock']:
|
||||||
|
yield '\t<div>' + iso_date(h['timestampBlock']['timestamp']) + '</div>'
|
||||||
|
yield '</div>'
|
||||||
|
|
||||||
|
def article_parse(data):
|
||||||
|
yield "<html><body>"
|
||||||
|
for x in data:
|
||||||
|
if x.get('__typename', '') in {'HeaderBasicBlock', 'HeaderFullBleedVerticalBlock'}:
|
||||||
|
yield '\n'.join(header_parse(x))
|
||||||
|
elif x.get('__typename', '') == 'ParagraphBlock':
|
||||||
|
yield '<p>'
|
||||||
|
for para in x['content']:
|
||||||
|
yield '\t'.join(parse_cnt(para))
|
||||||
|
yield '</p>'
|
||||||
|
elif x.get('__typename', '') == 'Heading2Block':
|
||||||
|
yield '<h4>'
|
||||||
|
for para in x['content']:
|
||||||
|
yield '\t'.join(parse_cnt(para))
|
||||||
|
yield '</h4>'
|
||||||
|
elif x.get('__typename', '') == 'ImageBlock':
|
||||||
|
yield ''.join(parse_image(x['media']))
|
||||||
|
elif x.get('__typename', '') == 'GridBlock':
|
||||||
|
yield ''.join(parse_img_grid(x))
|
||||||
|
elif 'content' in x and x['content']:
|
||||||
|
yield '<p><i>'
|
||||||
|
for i in x['content']:
|
||||||
|
yield '\t'.join(parse_cnt(i))
|
||||||
|
yield '</i></p>'
|
||||||
|
yield "</body></html>"
|
||||||
|
|
||||||
|
|
||||||
|
class nytFeeds(BasicNewsRecipe):
|
||||||
|
title = 'NYT News'
|
||||||
|
__author__ = 'unkn0wn'
|
||||||
|
description = (
|
||||||
|
'The New York Times is dedicated to helping people understand the world through '
|
||||||
|
'on-the-ground, expert and deeply reported independent journalism. Feeds based recipe.'
|
||||||
|
)
|
||||||
|
oldest_article = 1
|
||||||
|
encoding = 'utf-8'
|
||||||
|
use_embedded_content = False
|
||||||
|
language = 'en_US'
|
||||||
|
remove_empty_feeds = True
|
||||||
|
resolve_internal_links = True
|
||||||
|
ignore_duplicate_articles = {'title', 'url'}
|
||||||
|
masthead_url = 'https://static01.nytimes.com/newsgraphics/2015-12-23-masthead-2016/b15c3d81d3d7b59065fff9a3f3afe85aa2e2dff5/_assets/nyt-logo.png'
|
||||||
|
|
||||||
|
def get_cover_url(self):
|
||||||
|
soup = self.index_to_soup('https://www.frontpages.com/the-new-york-times/')
|
||||||
|
return 'https://www.frontpages.com' + soup.find('img', attrs={'id':'giornale-img'})['src']
|
||||||
|
|
||||||
|
recipe_specific_options = {
|
||||||
|
'days': {
|
||||||
|
'short': 'Oldest article to download from this news source. In days ',
|
||||||
|
'long': 'For example, 0.5, gives you articles from the past 12 hours',
|
||||||
|
'default': str(oldest_article)
|
||||||
|
},
|
||||||
|
'comp': {
|
||||||
|
'short': 'Compress News Images?',
|
||||||
|
'long': 'enter yes',
|
||||||
|
'default': 'no'
|
||||||
|
},
|
||||||
|
'rev': {
|
||||||
|
'short': 'Reverse the order of articles in each feed?',
|
||||||
|
'long': 'enter yes',
|
||||||
|
'default': 'no'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def __init__(self, *args, **kwargs):
|
||||||
|
BasicNewsRecipe.__init__(self, *args, **kwargs)
|
||||||
|
d = self.recipe_specific_options.get('days')
|
||||||
|
if d and isinstance(d, str):
|
||||||
|
self.oldest_article = float(d)
|
||||||
|
r = self.recipe_specific_options.get('rev')
|
||||||
|
if r and isinstance(r, str):
|
||||||
|
if r.lower() == 'yes':
|
||||||
|
self.reverse_article_order = True
|
||||||
|
c = self.recipe_specific_options.get('comp')
|
||||||
|
if c and isinstance(c, str):
|
||||||
|
if c.lower() == 'yes':
|
||||||
|
self.compress_news_images = True
|
||||||
|
|
||||||
|
extra_css = '''
|
||||||
|
.byl { font-size:small; color:#202020; }
|
||||||
|
.cap { font-size:small; text-align:center; }
|
||||||
|
.cred { font-style:italic; font-size:small; }
|
||||||
|
.sub { font-style:italic; }
|
||||||
|
'''
|
||||||
|
|
||||||
|
feeds = [
|
||||||
|
('World', 'https://rss.nytimes.com/services/xml/rss/nyt/World.xml'),
|
||||||
|
('US', 'https://rss.nytimes.com/services/xml/rss/nyt/US.xml'),
|
||||||
|
('Business', 'https://rss.nytimes.com/services/xml/rss/nyt/Business.xml'),
|
||||||
|
('Technology', 'https://rss.nytimes.com/services/xml/rss/nyt/Technology.xml'),
|
||||||
|
('Science', 'https://rss.nytimes.com/services/xml/rss/nyt/Science.xml'),
|
||||||
|
('Arts', 'https://rss.nytimes.com/services/xml/rss/nyt/Arts.xml'),
|
||||||
|
('Fashion & Style', 'https://rss.nytimes.com/services/xml/rss/nyt/FashionandStyle.xml'),
|
||||||
|
('TMagazine', 'https://rss.nytimes.com/services/xml/rss/nyt/tmagazine.xml'),
|
||||||
|
('Travel', 'https://www.nytimes.com/services/xml/rss/nyt/Travel.xml'),
|
||||||
|
('Sunday Review', 'https://rss.nytimes.com/services/xml/rss/nyt/sunday-review.xml'),
|
||||||
|
]
|
||||||
|
|
||||||
|
def get_browser(self, *args, **kwargs):
|
||||||
|
kwargs['user_agent'] = 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
|
||||||
|
br = BasicNewsRecipe.get_browser(self, *args, **kwargs)
|
||||||
|
br.addheaders += [
|
||||||
|
('Referer', 'https://www.google.com/'),
|
||||||
|
('X-Forwarded-For', '66.249.66.1')
|
||||||
|
]
|
||||||
|
return br
|
||||||
|
|
||||||
|
def preprocess_raw_html(self, raw_html, url):
|
||||||
|
data = extract_json(raw_html)
|
||||||
|
return '\n'.join(article_parse(data))
|
||||||
Loading…
x
Reference in New Issue
Block a user