Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-07 18:24:30 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/unkn0w7n/calibre

Browse Source
This commit is contained in:
Kovid Goyal 2024-09-22 21:34:00 +05:30
commit bea3a726f2
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
4 changed files with 213 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
recipes/icons/nytimes_sub.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 301 B

After

Width:  |  Height:  |  Size: 416 B

4
recipes/nytfeeds.recipe
View File

@ -26,7 +26,7 @@ def parse_image(i):
def parse_img_grid(g): def parse_img_grid(g):
for grd in g.get('gridMedia', {}): for grd in g.get('gridMedia', {}):
yield '<br/>'.join(parse_image(grd)) yield ''.join(parse_image(grd))
if g.get('caption'): if g.get('caption'):
yield '<div class="cap">{}'.format(g['caption']) yield '<div class="cap">{}'.format(g['caption'])
if g.get('credit'): if g.get('credit'):
@ -301,5 +301,5 @@ class nytFeeds(BasicNewsRecipe):
def get_article_url(self, article): def get_article_url(self, article):
url = BasicNewsRecipe.get_article_url(self, article) url = BasicNewsRecipe.get_article_url(self, article)
# you can remove '|/espanol/' from code below to include spanish articles. # you can remove '|/espanol/' from code below to include spanish articles.
if not re.search(r'/video/|live|/athletic/|/espanol/', url): if not re.search(r'/video/|/live/|/athletic/|/espanol/', url):
return url return url

60
recipes/nytimes_sub.recipe
View File

@ -16,7 +16,7 @@ from calibre.web.feeds.news import BasicNewsRecipe
is_web_edition = False is_web_edition = False
oldest_web_edition_article = 7 # days oldest_web_edition_article = 7 # days
use_wayback_machine = True use_wayback_machine = False
# The sections to download when downloading the web edition, comment out # The sections to download when downloading the web edition, comment out
@ -89,10 +89,16 @@ class NewYorkTimes(BasicNewsRecipe):
language = 'en_US' language = 'en_US'
ignore_duplicate_articles = {'title', 'url'} ignore_duplicate_articles = {'title', 'url'}
no_stylesheets = True no_stylesheets = True
compress_news_images = True
compress_news_images_auto_size = 5 extra_css = '''
conversion_options = {'flow_size': 0} .byl, .time { font-size:small; color:#202020; }
delay = 0 if use_wayback_machine else 1 .cap { font-size:small; text-align:center; }
.cred { font-style:italic; font-size:small; }
em, blockquote { color: #202020; }
.sc { font-variant: small-caps; }
.lbl { font-size:small; color:#404040; }
img { display:block; margin:0 auto; }
'''
@property @property
def nyt_parser(self): def nyt_parser(self):
@ -106,9 +112,13 @@ class NewYorkTimes(BasicNewsRecipe):
if use_wayback_machine and not skip_wayback: if use_wayback_machine and not skip_wayback:
from calibre import browser from calibre import browser
return self.nyt_parser.download_url(url, browser()) return self.nyt_parser.download_url(url, browser())
return self.browser.open_novisit(url).read() return self.index_to_soup(url)
def preprocess_raw_html(self, raw_html, url): def preprocess_raw_html(self, raw_html, url):
if '/interactive/' in url:
return '<html><body><p><em>'\
+ 'This is an interactive article, which is supposed to be read in a browser.'\
+ '</p></em></body></html>'
html = self.nyt_parser.extract_html(self.index_to_soup(raw_html)) html = self.nyt_parser.extract_html(self.index_to_soup(raw_html))
return html return html
@ -125,9 +135,25 @@ class NewYorkTimes(BasicNewsRecipe):
'date': { 'date': {
'short': 'The date of the edition to download (YYYY/MM/DD format)', 'short': 'The date of the edition to download (YYYY/MM/DD format)',
'long': 'For example, 2024/07/16' 'long': 'For example, 2024/07/16'
},
'res': {
'short': 'For hi-res images, select a resolution from the following\noptions: popup, jumbo, mobileMasterAt3x, superJumbo',
'long': 'This is useful for non e-ink devices, and for a lower file size\nthan the default, use articleInline.',
},
'comp': {
'short': 'Compress News Images?',
'long': 'enter yes',
'default': 'no'
} }
} }
def __init__(self, *args, **kwargs):
BasicNewsRecipe.__init__(self, *args, **kwargs)
c = self.recipe_specific_options.get('comp')
if c and isinstance(c, str):
if c.lower() == 'yes':
self.compress_news_images = True
def read_todays_paper(self): def read_todays_paper(self):
INDEX = 'https://www.nytimes.com/section/todayspaper' INDEX = 'https://www.nytimes.com/section/todayspaper'
# INDEX = 'file:///t/raw.html' # INDEX = 'file:///t/raw.html'
@ -303,3 +329,25 @@ class NewYorkTimes(BasicNewsRecipe):
if is_web_edition: if is_web_edition:
return self.parse_web_sections() return self.parse_web_sections()
return self.parse_todays_page() return self.parse_todays_page()
def get_browser(self, *args, **kwargs):
kwargs['user_agent'] = 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
br = BasicNewsRecipe.get_browser(self, *args, **kwargs)
br.addheaders += [
('Referer', 'https://www.google.com/'),
('X-Forwarded-For', '66.249.66.1')
]
return br
def preprocess_html(self, soup):
w = self.recipe_specific_options.get('res')
if w and isinstance(w, str):
res = '-' + w
for img in soup.findAll('img', attrs={'src':True}):
if '-article' in img['src']:
ext = img['src'].split('?')[0].split('.')[-1]
img['src'] = img['src'].rsplit('-article', 1)[0] + res + '.' + ext
for c in soup.findAll('div', attrs={'class':'cap'}):
for p in c.findAll(['p', 'div']):
p.name = 'span'
return soup

241
src/calibre/web/site_parsers/nytimes.py
View File

@ -13,88 +13,171 @@ module_version = 5 # needed for live updates
pprint pprint
def is_heading(tn): def parse_image(i):
return tn in ('Heading1Block', 'Heading2Block', 'Heading3Block', 'Heading4Block') if i.get('crops'):
yield '<div><img src="{}">'.format(i['crops'][0]['renditions'][0]['url'])
elif i.get('spanImageCrops'):
yield '<div><img src="{}">'.format(i['spanImageCrops'][0]['renditions'][0]['url'])
if i.get('caption'):
yield '<div class="cap">' + ''.join(parse_types(i['caption']))
if i.get('credit'):
yield '<span class="cred"> ' + i['credit'] + '</span>'
yield '</div>'
yield '</div>'
def parse_img_grid(g):
for grd in g.get('gridMedia', {}):
yield ''.join(parse_image(grd))
if g.get('caption'):
yield '<div class="cap">{}'.format(g['caption'])
if g.get('credit'):
yield '<span class="cred"> ' + g['credit'] + '</span>'
yield '</div>'
def process_inline_text(lines, block): def parse_vid(v):
text = '' if v.get('promotionalMedia'):
if 'text@stripHtml' in block: if v.get('headline'):
text = escape(block['text@stripHtml']) if v.get('url'):
elif 'renderedRepresentation' in block: # happens in byline blocks yield '<div><b><a href="{}">Video</a>: '.format(v['url'])\
text = block['renderedRepresentation'] + v['headline'].get('default', '') + '</b></div>'
elif 'text' in block: elif v['headline'].get('default'):
text = block['text'] yield '<div><b>' + v['headline']['default'] + '</b></div>'
if text: yield ''.join(parse_types(v['promotionalMedia']))
for fmt in block.get('formats', ()): if v.get('promotionalSummary'):
tn = fmt['__typename'] yield '<div class="cap">' + v['promotionalSummary'] + '</div>'
if tn == 'LinkFormat':
ab = fmt
text = '<a href="{}" title="{}">{}</a>'.format(ab['url'], ab.get('title') or '', text)
elif tn == 'BoldFormat':
text = '<b>' + text + '</b>'
lines.append(text)
def parse_emb(e):
if e.get('html') and 'datawrapper.dwcdn.net' in e.get('html', ''):
dw = re.search(r'datawrapper.dwcdn.net/(.{5})', e['html']).group(1)
yield '<div><img src="{}">'.format('https://datawrapper.dwcdn.net/' + dw + '/full.png') + '</div>'
elif e.get('promotionalMedia'):
if e.get('headline'):
yield '<div><b>' + e['headline']['default'] + '</b></div>'
yield ''.join(parse_types(e['promotionalMedia']))
if e.get('note'):
yield '<div class="cap">' + e['note'] + '</div>'
def process_paragraph(lines, block, content_key='content'): def parse_byline(byl):
tn = block['__typename'] for b in byl.get('bylines', {}):
m = re.match(r'Heading([1-6])Block', tn) yield '<div>' + b['renderedRepresentation'] + '</div>'
if m is not None: yield '<div><b><i>'
tag = 'h' + m.group(1) for rl in byl.get('role', {}):
else: if ''.join(parse_cnt(rl)).strip():
tag = 'p' yield ''.join(parse_cnt(rl))
ta = block.get('textAlign') or 'LEFT' yield '</i></b></div>'
style = f'text-align: {ta.lower()}'
lines.append(f'<{tag} style="{style}">')
for item in block[content_key]:
tn = item['__typename']
if tn in ('TextInline', 'Byline'):
process_inline_text(lines, item)
lines.append('</' + tag + '>')
def iso_date(x):
dt = parse_iso8601(x, as_utc=False)
return dt.strftime('%b %d, %Y at %I:%M %p')
def process_timestamp(lines, block): def parse_header(h):
ts = block['timestamp'] if h.get('label'):
dt = parse_iso8601(ts, as_utc=False) yield '<div class="lbl">' + ''.join(parse_types(h['label'])) + '</div>'
lines.append('<p class="timestamp">' + escape(dt.strftime('%b %d, %Y')) + '</p>') if h.get('headline'):
yield ''.join(parse_types(h['headline']))
if h.get('summary'):
yield '<p><i>' + ''.join(parse_types(h['summary'])) + '</i></p>'
if h.get('ledeMedia'):
yield ''.join(parse_types(h['ledeMedia']))
if h.get('byline'):
yield ''.join(parse_types(h['byline']))
if h.get('timestampBlock'):
yield ''.join(parse_types(h['timestampBlock']))
def parse_fmt_type(fm):
for f in fm.get('formats', {}):
if f.get('__typename', '') == 'BoldFormat':
yield '<strong>'
if f.get('__typename', '') == 'ItalicFormat':
yield '<em>'
if f.get('__typename', '') == 'LinkFormat':
hrf = f['url']
yield '<a href="{}">'.format(hrf)
yield fm['text']
for f in reversed(fm.get('formats', {})):
if f.get('__typename', '') == 'BoldFormat':
yield '</strong>'
if f.get('__typename', '') == 'ItalicFormat':
yield '</em>'
if f.get('__typename', '') == 'LinkFormat':
yield '</a>'
def process_header(lines, block): def parse_cnt(cnt):
label = block.get('label') if cnt.get('formats'):
if label: yield ''.join(parse_fmt_type(cnt))
process_paragraph(lines, label) elif cnt.get('content'):
headline = block.get('headline') for cnt_ in cnt['content']:
if headline: yield from parse_types(cnt_)
process_paragraph(lines, headline) elif cnt.get('text'):
summary = block.get('summary') yield cnt['text']
if summary:
process_paragraph(lines, summary)
lm = block.get('ledeMedia')
if lm and lm.get('__typename') == 'ImageBlock':
process_image_block(lines, lm)
byline = block.get('byline')
if byline:
process_paragraph(lines, byline, content_key='bylines')
timestamp = block.get('timestampBlock')
if timestamp:
process_timestamp(lines, timestamp)
def parse_types(x):
if 'Header' in x.get('__typename', ''):
yield '\n'.join(parse_header(x))
def process_image_block(lines, block): elif x.get('__typename', '') == 'Heading1Block':
media = block['media'] yield '<h1>' + ''.join(parse_cnt(x)) + '</h1>'
caption = media.get('caption') elif x.get('__typename', '') in {'Heading2Block', 'Heading3Block', 'Heading4Block'}:
caption_lines = [] yield '<h4>' + ''.join(parse_cnt(x)) + '</h4>'
if caption:
process_inline_text(caption_lines, caption) elif x.get('__typename', '') == 'ParagraphBlock':
crops = media['crops'] yield '<p>' + ''.join(parse_cnt(x)) + '</p>'
renditions = crops[0]['renditions']
img = renditions[0]['url'] elif x.get('__typename', '') == 'BylineBlock':
if 'web.archive.org' in img: yield '<div class="byl"><br/>' + ''.join(parse_byline(x)) + '</div>'
img = img.partition('/')[-1] elif x.get('__typename', '') == 'LabelBlock':
img = img[img.find('https://'):] yield '<div class="sc">' + ''.join(parse_cnt(x)) + '</div>'
lines.append(f'<div style="text-align: center"><div style="text-align: center"><img src={quoteattr(img)}/></div><div style="font-size: smaller">') elif x.get('__typename', '') == 'BlockquoteBlock':
lines.extend(caption_lines) yield '<blockquote>' + ''.join(parse_cnt(x)) + '</blockquote>'
lines.append('</div></div>') elif x.get('__typename', '') == 'TimestampBlock':
yield '<div class="time">' + iso_date(x['timestamp']) + '</div>'
elif x.get('__typename', '') == 'LineBreakInline':
yield '<br/>'
elif x.get('__typename', '') == 'RuleBlock':
yield '<hr/>'
elif x.get('__typename', '') == 'Image':
yield ''.join(parse_image(x))
elif x.get('__typename', '') == 'ImageBlock':
yield ''.join(parse_image(x['media']))
elif x.get('__typename', '') == 'GridBlock':
yield ''.join(parse_img_grid(x))
elif x.get('__typename', '') == 'VideoBlock':
yield ''.join(parse_types(x['media']))
elif x.get('__typename', '') == 'Video':
yield ''.join(parse_vid(x))
elif x.get('__typename', '') == 'InteractiveBlock':
yield ''.join(parse_types(x['media']))
elif x.get('__typename', '') == 'EmbeddedInteractive':
yield ''.join(parse_emb(x))
elif x.get('__typename', '') == 'ListBlock':
yield '<ul>' + ''.join(parse_cnt(x)) + '</ul>'
elif x.get('__typename', '') == 'ListItemBlock':
yield '<li>' + ''.join(parse_cnt(x)) + '</li>'
elif x.get('__typename', '') == 'CapsuleBlock':
if x['capsuleContent'].get('body'):
yield ''.join(parse_cnt(x['capsuleContent']['body']))
elif x.get('__typename', '') == 'Capsule':
yield ''.join(parse_cnt(x['body']))
elif x.get('__typename', '') in {
'TextInline', 'TextOnlyDocumentBlock', 'DocumentBlock', 'SummaryBlock'
}:
yield ''.join(parse_cnt(x))
elif x.get('__typename'):
if ''.join(parse_cnt(x)).strip():
yield '<p><i>' + ''.join(parse_cnt(x)) + '</i></p>'
def article_parse(data):
yield "<html><body>"
for d in data:
yield from parse_types(d)
yield "</body></html>"
def json_to_html(raw): def json_to_html(raw):
@ -105,18 +188,8 @@ def json_to_html(raw):
except TypeError: except TypeError:
data = data['initialState'] data = data['initialState']
return live_json_to_html(data) return live_json_to_html(data)
article = next(iter(data.values())) content = data['article']['sprinkledBody']['content']
body = article['sprinkledBody']['content'] return '\n'.join(article_parse(content))
lines = []
for item in body:
tn = item['__typename']
if tn in ('HeaderBasicBlock', 'HeaderLegacyBlock', 'HeaderFullBleedVerticalBlock'):
process_header(lines, item)
elif tn in ('ParagraphBlock', 'LabelBlock', 'DetailBlock') or is_heading(tn):
process_paragraph(lines, item)
elif tn == 'ImageBlock':
process_image_block(lines, item)
return '<html><body>' + '\n'.join(lines) + '</body></html>'
def add_live_item(item, item_type, lines): def add_live_item(item, item_type, lines):