mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-09-29 15:31:08 -04:00
178 lines
7.6 KiB
Plaintext
178 lines
7.6 KiB
Plaintext
from calibre.web.feeds.news import BasicNewsRecipe
|
|
from calibre import browser
|
|
from html5_parser import parse
|
|
from calibre.ptempfile import PersistentTemporaryFile
|
|
import json
|
|
import random
|
|
import time
|
|
|
|
def get_contents(x):
|
|
otype = x.get('type', '')
|
|
if otype == 'text':
|
|
return x.get('value', '')
|
|
elif otype == 'paragraph':
|
|
return '<p>' + x.get('value', '') + ''.join(map(get_contents, x.get('content'))) + '</p>'
|
|
elif otype == 'heading':
|
|
return '<h3>' + x.get('value', '') + ''.join(map(get_contents, x.get('content'))) + '</h3>'
|
|
elif otype == 'list':
|
|
return '<ul>' + ''.join(map(get_contents, x.get('content'))) + '</ul>'
|
|
elif otype == 'listItem':
|
|
return '<li>' + x.get('value', '') + ''.join(map(get_contents, x.get('content'))) + '</li>'
|
|
elif otype == 'quote':
|
|
return '<blockquote>' + x.get('value', '') + ''.join(map(get_contents, x.get('content'))) + '</blockquote>'
|
|
elif otype == 'media':
|
|
if x['subType'] == 'photo':
|
|
return '<div><div class="img"><img src="{}"></div><div class="cap">{}</div></div>'.format(
|
|
x['data']['photo']['src'], x['data']['photo']['caption'])
|
|
elif x['subType'] == 'chart':
|
|
if x['data'] and x['data']['chart']:
|
|
return '<div class="img"><img src="{}"></div>'.format(x['data']['chart']['fallback'])
|
|
elif otype == 'link':
|
|
if x['data'] and x['content'] and x['content'][0] and x['content'][0]['value']:
|
|
if 'href' in x['data']:
|
|
return '<a href="' + x['data']['href'] + '">' + x['content'][0]['value'] + '</a>'
|
|
return '<i>' + x['content'][0]['value'] + '</i>'
|
|
elif otype == 'entity':
|
|
if x['content'] and x['content'][0] and x['content'][0]['value']:
|
|
if x['subType'] == 'story':
|
|
if x['data'] and x['data']['link'] and x['data']['link']['destination']:
|
|
if 'web' in x['data']['link']['destination']:
|
|
return '<a href="' + x['data']['link']['destination']['web'] + '">' + x['content'][0]['value'] + '</a>'
|
|
return '<i>' + x['content'][0]['value'] + '</i>'
|
|
elif x['subType'] in ('person', 'security'):
|
|
return '<i>' + x['content'][0]['value'] + '</i>'
|
|
|
|
return ''
|
|
|
|
|
|
class Bloomberg(BasicNewsRecipe):
|
|
title = u'Bloomberg'
|
|
language = 'en'
|
|
__author__ = 'unkn0wn'
|
|
no_stylesheets = True
|
|
use_embedded_content = False
|
|
remove_attributes = ['style', 'height', 'width']
|
|
ignore_duplicate_articles = {'url', 'title'}
|
|
|
|
# delay = 7 # seconds
|
|
simultaneous_downloads = 3
|
|
|
|
extra_css = '''
|
|
.auth {font-size:small; font-weight:bold;}
|
|
.time, .chart {font-size:small;}
|
|
.subhead {font-style:italic; color:#404040;}
|
|
.cat {font-size:small; color:gray;}
|
|
.news-figure-caption-text, .cap, .img {font-size:small; text-align:center;}
|
|
.news-figure-credit {font-size:small; text-align:center; color:#202020;}
|
|
'''
|
|
|
|
articles_are_obfuscated = True
|
|
|
|
def get_obfuscated_article(self, url):
|
|
br = self.get_browser()
|
|
try:
|
|
br.open(url)
|
|
except Exception as e:
|
|
url = e.hdrs.get('location')
|
|
soup = self.index_to_soup(url)
|
|
link = soup.find('a', attrs={'href':lambda x: x and x.startswith('https://www.bloomberg.com')})
|
|
skip_sections =[ # add sections you want to skip
|
|
'/video/', '/videos/', '/media/', 'podcast'
|
|
]
|
|
if any(x in link['href'] for x in skip_sections):
|
|
self.abort_article('Aborting Video article')
|
|
self.log('Found link: ', link['href'])
|
|
html = br.open(link['href']).read()
|
|
pt = PersistentTemporaryFile('.html')
|
|
pt.write(html)
|
|
pt.close()
|
|
return pt.name
|
|
|
|
|
|
def get_browser(self):
|
|
br = browser()
|
|
br.set_handle_redirect(False)
|
|
return br
|
|
|
|
feeds = [
|
|
('Features',
|
|
'https://news.google.com/rss/search?q=when:27h+allinurl:bloomberg.com%2Fnews%2Ffeatures%2F&hl=en-US&gl=US&ceid=US:en'),
|
|
('Opinion', 'https://news.google.com/rss/search?q=when:27h+allinurl:bloomberg.com%2Fopinion%2F&hl=en-US&gl=US&ceid=US:en'),
|
|
('Newsletters',
|
|
'https://news.google.com/rss/search?q=when:27h+allinurl:bloomberg.com%2Fnews%2Fnewsletters%2F&hl=en-US&gl=US&ceid=US:en'),
|
|
('News',
|
|
'https://news.google.com/rss/search?q=when:27h+allinurl:bloomberg.com%2Fnews%2Farticles%2F&hl=en-US&gl=US&ceid=US:en'),
|
|
('Others', 'https://news.google.com/rss/search?q=when:27h+allinurl:bloomberg.com&hl=en-US&gl=US&ceid=US:en')
|
|
]
|
|
|
|
def preprocess_raw_html(self, raw, *a):
|
|
root = parse(raw)
|
|
m = root.xpath('//script[@data-component-props="ArticleBody"]')
|
|
if not m:
|
|
m = root.xpath('//script[@data-component-props="FeatureBody"]')
|
|
if not m:
|
|
m2 = root.xpath('//script[@id="__NEXT_DATA__"]')
|
|
|
|
if m:
|
|
data = json.loads(m[0].text)
|
|
data = data['story']
|
|
|
|
elif m2:
|
|
data = json.loads(m2[0].text)
|
|
data = data['props']['pageProps']['story']
|
|
|
|
title = '<h1>' + data['headline'] + '</h1>'
|
|
|
|
cat = subhead = lede = auth = caption = ''
|
|
|
|
if 'primaryCategory' in data and data['primaryCategory'] is not None:
|
|
cat = '<p class="cat">' + data['primaryCategory'] + '</p>'
|
|
|
|
if len(data['abstract']) != 0 and len(data['abstract']) == 2:
|
|
subhead = '<div class="subhead"><p>' + data['abstract'][0] + '</p><p>' + data['abstract'][1] + '</p></div>'
|
|
else:
|
|
if 'summary' in data:
|
|
subhead = '<div class="subhead"><p>' + data['summary'] + '</p></div>'
|
|
|
|
if 'byline' in data and data['byline'] is not None:
|
|
auth = '<div><span class="auth">' + data['byline']\
|
|
+ '</span> | <span class="time">' + data['publishedAt'][:-14] + '</span></div>'
|
|
|
|
if 'ledeImageUrl' in data and data['ledeImageUrl'] is not None:
|
|
lede = '<p class="img"><img src="{}">'.format(data['ledeImageUrl'])
|
|
|
|
if 'ledeDescription' in data and data['ledeDescription'] is not None:
|
|
caption = '<span class="cap">' + data['ledeDescription'] + '</span>'
|
|
else:
|
|
if 'lede' in data and data['lede'] is not None:
|
|
if 'alt' in data['lede'] and data['lede']['alt'] is not None:
|
|
caption = '<span class="cap">' + data['lede']['alt'] + '</span>'
|
|
|
|
if m:
|
|
time.sleep(3)
|
|
body = data['body']
|
|
elif m2:
|
|
body = ''
|
|
body_data = data['body']['content']
|
|
for x in body_data:
|
|
pause = random.choice((0.5, 1, 1.25))
|
|
time.sleep(pause)
|
|
body += get_contents(x)
|
|
return '<html><body>' + cat + title + subhead + auth + lede + caption + '<div>' + body + '</div></body></html>'
|
|
|
|
def preprocess_html(self, soup):
|
|
for icon in soup.findAll('img', attrs={'class':'video-player__play-icon'}):
|
|
icon.decompose()
|
|
for div in soup.findAll('div', attrs={'class':'chart'}):
|
|
nos = div.find('noscript')
|
|
if nos:
|
|
nos.name = 'span'
|
|
for img in soup.findAll('img', attrs={'data-native-src':True}):
|
|
if img['data-native-src'].__contains__('videos') is False:
|
|
img['src'] = img['data-native-src']
|
|
else:
|
|
img['src'] = ''
|
|
for img in soup.findAll('img', attrs={'src':lambda x: x and x.endswith(('-1x-1.jpg', '-1x-1.png'))}):
|
|
img['src'] = img['src'].replace('-1x-1', '750x-1')
|
|
return soup
|