From 93addfe147fcf6f9b8131d0f41d09505eb523753 Mon Sep 17 00:00:00 2001
From: unkn0w7n <51942695+unkn0w7n@users.noreply.github.com>
Date: Wed, 17 Jul 2024 12:16:21 +0530
Subject: [PATCH] Update Bloomberg
---
recipes/bloomberg-business-week.recipe | 37 +++++++++++++++++---------
recipes/bloomberg.recipe | 37 +++++++++++++++++---------
2 files changed, 49 insertions(+), 25 deletions(-)
diff --git a/recipes/bloomberg-business-week.recipe b/recipes/bloomberg-business-week.recipe
index e7a47d28f6..46cee92c1f 100644
--- a/recipes/bloomberg-business-week.recipe
+++ b/recipes/bloomberg-business-week.recipe
@@ -3,7 +3,7 @@ import time
from datetime import datetime, timedelta
from calibre.ebooks.BeautifulSoup import BeautifulSoup
-from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.web.feeds.news import BasicNewsRecipe, classes
# https://www.bloomberg.com/magazine/businessweek/24_12
@@ -61,6 +61,13 @@ class Bloomberg(BasicNewsRecipe):
)
remove_empty_feeds = True
+ remove_tags = [
+ dict(name=['button', 'svg', 'meta']),
+ dict(name='div', attrs={'id':['bb-that', 'bb-nav']}),
+ dict(attrs={'data-image-type':'audio'}),
+ classes('twitter-logo bb-global-footer __sticky__audio__bar__portal__ css--social-wrapper-outer')
+ ]
+
extra_css = '''
.auth { font-size:small; font-weight:bold; }
.subhead, .cap span { font-style:italic; color:#202020; }
@@ -99,7 +106,7 @@ class Bloomberg(BasicNewsRecipe):
feeds.append((section, articles))
return feeds
- def preprocess_raw_html(self, raw, *a):
+ def preprocess_raw_html(self, raw, url):
data = json.loads(raw)
title = ''.format(data['longURL']) + data['title'] + '
'
@@ -118,23 +125,28 @@ class Bloomberg(BasicNewsRecipe):
dt = datetime.fromtimestamp(data['updatedAt'] + time.timezone)
auth = '' + 'By ' + data['byline'] + ' | Updated on ' + dt.strftime('%b %d, %Y at %I:%M %p') + '
'
- if 'ledeImage' in data and data['ledeImage'] is not None:
- x = data['ledeImage']
- lede = '
{}
\n'.format(
- x['imageURLs']['default'], x['caption'] + ' ' + x['credit'] + ''
- )
-
body = ''
if data.get('type', '') == 'interactive':
body += '' + 'This is an interactive article, which is supposed to be read in a browser.' + '
'
- body_data = data['components']
- for x in body_data:
- body += get_contents(x)
+ # body_data = data['components']
+ # for x in body_data:
+ # body += get_contents(x)
+
+ b_data = self.index_to_soup('https://cdn-mobapi.bloomberg.com/wssmobile/v1/bw/news/stories/' + url.split('/')[-1], raw=True)
+ body += json.loads(b_data)['html']
+
+ if 'ledeImage' in data and data['ledeImage'] is not None:
+ x = data['ledeImage']
+ if x['imageURLs']['default'].rsplit('/', 1)[0] not in body:
+ lede = '
{}
\n'.format(
+ x['imageURLs']['default'], x['caption'] + ' ' + x['credit'] + ''
+ )
+
html = '' + cat + title + subhead + auth + lede + caption + '' + body + '
'
return BeautifulSoup(html).prettify()
def preprocess_html(self, soup):
- for h3 in soup.findAll('h3'):
+ for h3 in soup.findAll(['h2', 'h3']):
h3.name = 'h4'
for icon in soup.findAll('img', attrs={'class':'video-player__play-icon'}):
icon.decompose()
@@ -155,4 +167,3 @@ class Bloomberg(BasicNewsRecipe):
article.url = soup.find('h1')['title']
article.summary = self.tag_to_string(soup.find('div', attrs={'class':'subhead'}))
article.text_summary = article.summary
-
diff --git a/recipes/bloomberg.recipe b/recipes/bloomberg.recipe
index ec717b46b5..3a9557570b 100644
--- a/recipes/bloomberg.recipe
+++ b/recipes/bloomberg.recipe
@@ -3,7 +3,7 @@ import time
from datetime import datetime, timedelta
from calibre.ebooks.BeautifulSoup import BeautifulSoup
-from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.web.feeds.news import BasicNewsRecipe, classes
def get_contents(x):
@@ -58,6 +58,14 @@ class Bloomberg(BasicNewsRecipe):
oldest_article = 1 # days
resolve_internal_links = True
remove_empty_feeds = True
+ cover_url = 'https://assets.bwbx.io/images/users/iqjWHBFdfxIU/ivUxvlPidC3M/v0/600x-1.jpg'
+
+ remove_tags = [
+ dict(name=['button', 'svg', 'meta']),
+ dict(name='div', attrs={'id':['bb-that', 'bb-nav']}),
+ dict(attrs={'data-image-type':'audio'}),
+ classes('twitter-logo bb-global-footer __sticky__audio__bar__portal__ css--social-wrapper-outer')
+ ]
extra_css = '''
.auth { font-size:small; font-weight:bold; }
@@ -101,7 +109,7 @@ class Bloomberg(BasicNewsRecipe):
feeds.append((section, articles))
return feeds
- def preprocess_raw_html(self, raw, *a):
+ def preprocess_raw_html(self, raw, url):
data = json.loads(raw)
title = ''.format(data['longURL']) + data['title'] + '
'
@@ -120,23 +128,28 @@ class Bloomberg(BasicNewsRecipe):
dt = datetime.fromtimestamp(data['updatedAt'] + time.timezone)
auth = '' + 'By ' + data['byline'] + ' | Updated on ' + dt.strftime('%b %d, %Y at %I:%M %p') + '
'
- if 'ledeImage' in data and data['ledeImage'] is not None:
- x = data['ledeImage']
- lede = '
{}
\n'.format(
- x['imageURLs']['default'], x['caption'] + ' ' + x['credit'] + ''
- )
-
body = ''
if data.get('type', '') == 'interactive':
body += '' + 'This is an interactive article, which is supposed to be read in a browser.' + '
'
- body_data = data['components']
- for x in body_data:
- body += get_contents(x)
+ # body_data = data['components']
+ # for x in body_data:
+ # body += get_contents(x)
+
+ b_data = self.index_to_soup('https://cdn-mobapi.bloomberg.com/wssmobile/v1/bw/news/stories/' + url.split('/')[-1], raw=True)
+ body += json.loads(b_data)['html']
+
+ if 'ledeImage' in data and data['ledeImage'] is not None:
+ x = data['ledeImage']
+ if x['imageURLs']['default'].rsplit('/', 1)[0] not in body:
+ lede = '
{}
\n'.format(
+ x['imageURLs']['default'], x['caption'] + ' ' + x['credit'] + ''
+ )
+
html = '' + cat + title + subhead + auth + lede + caption + '' + body + '
'
return BeautifulSoup(html).prettify()
def preprocess_html(self, soup):
- for h3 in soup.findAll('h3'):
+ for h3 in soup.findAll(['h2', 'h3']):
h3.name = 'h4'
for icon in soup.findAll('img', attrs={'class':'video-player__play-icon'}):
icon.decompose()