Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/unkn0w7n/calibre

Browse Source
This commit is contained in:
Kovid Goyal 2024-07-17 12:21:52 +05:30
commit b04bc87f66
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
2 changed files with 49 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
recipes/bloomberg-business-week.recipe
View File

@ -3,7 +3,7 @@ import time
from datetime import datetime, timedelta
from calibre.ebooks.BeautifulSoup import BeautifulSoup
from calibre.web.feeds.news import BasicNewsRecipe
from calibre.web.feeds.news import BasicNewsRecipe, classes
# https://www.bloomberg.com/magazine/businessweek/24_12
@ -61,6 +61,13 @@ class Bloomberg(BasicNewsRecipe):
)
remove_empty_feeds = True
remove_tags = [
dict(name=['button', 'svg', 'meta']),
dict(name='div', attrs={'id':['bb-that', 'bb-nav']}),
dict(attrs={'data-image-type':'audio'}),
classes('twitter-logo bb-global-footer __sticky__audio__bar__portal__ css--social-wrapper-outer')
]
extra_css = '''
.auth { font-size:small; font-weight:bold; }
.subhead, .cap span { font-style:italic; color:#202020; }
@ -99,7 +106,7 @@ class Bloomberg(BasicNewsRecipe):
feeds.append((section, articles))
return feeds
def preprocess_raw_html(self, raw, *a):
def preprocess_raw_html(self, raw, url):
data = json.loads(raw)
title = '<h1 title="{}">'.format(data['longURL']) + data['title'] + '</h1>'
@ -118,23 +125,28 @@ class Bloomberg(BasicNewsRecipe):
dt = datetime.fromtimestamp(data['updatedAt'] + time.timezone)
auth = '<p class="auth">' + 'By ' + data['byline'] + ' | Updated on ' + dt.strftime('%b %d, %Y at %I:%M %p') + '</p>'
body = ''
if data.get('type', '') == 'interactive':
body += '<p><em>' + 'This is an interactive article, which is supposed to be read in a browser.' + '</p></em>'
# body_data = data['components']
# for x in body_data:
# body += get_contents(x)
b_data = self.index_to_soup('https://cdn-mobapi.bloomberg.com/wssmobile/v1/bw/news/stories/' + url.split('/')[-1], raw=True)
body += json.loads(b_data)['html']
if 'ledeImage' in data and data['ledeImage'] is not None:
x = data['ledeImage']
if x['imageURLs']['default'].rsplit('/', 1)[0] not in body:
lede = '<br><img src="{}"><div class="img">{}</div>\n'.format(
x['imageURLs']['default'], x['caption'] + '<i> ' + x['credit'] + '</i>'
)
body = ''
if data.get('type', '') == 'interactive':
body += '<p><em>' + 'This is an interactive article, which is supposed to be read in a browser.' + '</p></em>'
body_data = data['components']
for x in body_data:
body += get_contents(x)
html = '<html><body>' + cat + title + subhead + auth + lede + caption + '<div>' + body + '</div></body></html>'
return BeautifulSoup(html).prettify()
def preprocess_html(self, soup):
for h3 in soup.findAll('h3'):
for h3 in soup.findAll(['h2', 'h3']):
h3.name = 'h4'
for icon in soup.findAll('img', attrs={'class':'video-player__play-icon'}):
icon.decompose()
@ -155,4 +167,3 @@ class Bloomberg(BasicNewsRecipe):
article.url = soup.find('h1')['title']
article.summary = self.tag_to_string(soup.find('div', attrs={'class':'subhead'}))
article.text_summary = article.summary

31
recipes/bloomberg.recipe
View File

@ -3,7 +3,7 @@ import time
from datetime import datetime, timedelta
from calibre.ebooks.BeautifulSoup import BeautifulSoup
from calibre.web.feeds.news import BasicNewsRecipe
from calibre.web.feeds.news import BasicNewsRecipe, classes
def get_contents(x):
@ -58,6 +58,14 @@ class Bloomberg(BasicNewsRecipe):
oldest_article = 1 # days
resolve_internal_links = True
remove_empty_feeds = True
cover_url = 'https://assets.bwbx.io/images/users/iqjWHBFdfxIU/ivUxvlPidC3M/v0/600x-1.jpg'
remove_tags = [
dict(name=['button', 'svg', 'meta']),
dict(name='div', attrs={'id':['bb-that', 'bb-nav']}),
dict(attrs={'data-image-type':'audio'}),
classes('twitter-logo bb-global-footer __sticky__audio__bar__portal__ css--social-wrapper-outer')
]
extra_css = '''
.auth { font-size:small; font-weight:bold; }
@ -101,7 +109,7 @@ class Bloomberg(BasicNewsRecipe):
feeds.append((section, articles))
return feeds
def preprocess_raw_html(self, raw, *a):
def preprocess_raw_html(self, raw, url):
data = json.loads(raw)
title = '<h1 title="{}">'.format(data['longURL']) + data['title'] + '</h1>'
@ -120,23 +128,28 @@ class Bloomberg(BasicNewsRecipe):
dt = datetime.fromtimestamp(data['updatedAt'] + time.timezone)
auth = '<p class="auth">' + 'By ' + data['byline'] + ' | Updated on ' + dt.strftime('%b %d, %Y at %I:%M %p') + '</p>'
body = ''
if data.get('type', '') == 'interactive':
body += '<p><em>' + 'This is an interactive article, which is supposed to be read in a browser.' + '</p></em>'
# body_data = data['components']
# for x in body_data:
# body += get_contents(x)
b_data = self.index_to_soup('https://cdn-mobapi.bloomberg.com/wssmobile/v1/bw/news/stories/' + url.split('/')[-1], raw=True)
body += json.loads(b_data)['html']
if 'ledeImage' in data and data['ledeImage'] is not None:
x = data['ledeImage']
if x['imageURLs']['default'].rsplit('/', 1)[0] not in body:
lede = '<br><img src="{}"><div class="img">{}</div>\n'.format(
x['imageURLs']['default'], x['caption'] + '<i> ' + x['credit'] + '</i>'
)
body = ''
if data.get('type', '') == 'interactive':
body += '<p><em>' + 'This is an interactive article, which is supposed to be read in a browser.' + '</p></em>'
body_data = data['components']
for x in body_data:
body += get_contents(x)
html = '<html><body>' + cat + title + subhead + auth + lede + caption + '<div>' + body + '</div></body></html>'
return BeautifulSoup(html).prettify()
def preprocess_html(self, soup):
for h3 in soup.findAll('h3'):
for h3 in soup.findAll(['h2', 'h3']):
h3.name = 'h4'
for icon in soup.findAll('img', attrs={'class':'video-player__play-icon'}):
icon.decompose()