diff --git a/recipes/bloomberg-business-week.recipe b/recipes/bloomberg-business-week.recipe index bb3f7c3c37..9c898a2f94 100644 --- a/recipes/bloomberg-business-week.recipe +++ b/recipes/bloomberg-business-week.recipe @@ -1,45 +1,56 @@ from calibre.web.feeds.news import BasicNewsRecipe, classes -from calibre import browser from html5_parser import parse import json import random import time def get_contents(x): + if x == '': + return '' otype = x.get('type', '') if otype == 'text': - return x.get('value', '') + if 'attributes' in x: + if 'strong' in x['attributes']: + return '' + x.get('value', '') + ''.join(map(get_contents, x.get('content', ''))) + '' + if 'emphasis' in x['attributes']: + return '' + x.get('value', '') + ''.join(map(get_contents, x.get('content', ''))) + '' + return x.get('value', '') + ''.join(map(get_contents, x.get('content', ''))) + elif otype == 'br': + return '
' elif otype == 'paragraph': - return '

' + x.get('value', '') + ''.join(map(get_contents, x.get('content'))) + '

' + return '

' + x.get('value', '') + ''.join(map(get_contents, x.get('content', ''))) + '

' elif otype == 'heading': - return '

' + x.get('value', '') + ''.join(map(get_contents, x.get('content'))) + '

' + return '

' + x.get('value', '') + ''.join(map(get_contents, x.get('content', ''))) + '

' elif otype == 'list': - return '' + return '' elif otype == 'listItem': - return '
  • ' + x.get('value', '') + ''.join(map(get_contents, x.get('content'))) + '
  • ' + return '
  • ' + x.get('value', '') + ''.join(map(get_contents, x.get('content', ''))) + '
  • ' elif otype == 'quote': - return '
    ' + x.get('value', '') + ''.join(map(get_contents, x.get('content'))) + '
    ' + return '
    ' + x.get('value', '') + ''.join(map(get_contents, x.get('content', ''))) + '
    ' elif otype == 'media': if x['subType'] == 'photo': return '
    {}
    '.format( x['data']['photo']['src'], x['data']['photo']['caption']) elif x['subType'] == 'chart': if x['data'] and x['data']['chart']: - return '
    '.format(x['data']['chart']['fallback']) + return '
    '.format(x['data']['chart']['fallback']) elif otype == 'link': - if x['data'] and x['content'] and x['content'][0] and x['content'][0]['value']: + if 'data' in x: if 'href' in x['data']: - return '' + x['content'][0]['value'] + '' - return '' + x['content'][0]['value'] + '' + return '' + x.get('value', '') + ''.join(map(get_contents, x.get('content', ''))) + '' + return '' + x.get('value', '') + ''.join(map(get_contents, x.get('content', ''))) + '' + return '' + x.get('value', '') + ''.join(map(get_contents, x.get('content', ''))) + '' elif otype == 'entity': - if x['content'] and x['content'][0] and x['content'][0]['value']: - if x['subType'] == 'story': - if x['data'] and x['data']['link'] and x['data']['link']['destination']: - if 'web' in x['data']['link']['destination']: - return '' + x['content'][0]['value'] + '' - return '' + x['content'][0]['value'] + '' - elif x['subType'] in ('person', 'security'): - return '' + x['content'][0]['value'] + '' + if x['subType'] == 'story': + if x['data'] and x['data']['link'] and x['data']['link']['destination']: + if 'web' in x['data']['link']['destination']: + return '' + x.get('value', '') + ''.join(map(get_contents, x.get('content', ''))) + '' + return '' + x.get('value', '') + ''.join(map(get_contents, x.get('content', ''))) + '' + return '' + x.get('value', '') + ''.join(map(get_contents, x.get('content', ''))) + '' + return '' + x.get('value', '') + ''.join(map(get_contents, x.get('content', ''))) + '' + elif not any(x == otype for x in ['', 'ad', 'inline-newsletter', 'tabularData']): + if any(b in x for b in ['value', 'content']): + return '' + x.get('value', '') + ''.join(map(get_contents, x.get('content', ''))) + '' return '' @@ -53,26 +64,32 @@ class Bloomberg(BasicNewsRecipe): ignore_duplicate_articles = {'url'} resolve_internal_links = True masthead_url = 'https://assets.bwbx.io/s3/javelin/public/hub/images/BW-Logo-Black-cc9035fbb3.svg' + description = ( + 'Bloomberg Businessweek helps global leaders stay ahead with insights and in-depth analysis on the people,' + ' companies, events, and trends shaping today\'s complex, global economy.' + ) - # delay = 7 # seconds - simultaneous_downloads = 3 + simultaneous_downloads = 1 extra_css = ''' .auth {font-size:small; font-weight:bold;} - .time, .chart {font-size:small;} + .time, .chart, .css--lede-byline, .css--lede-timestamp {font-size:small;} .subhead {font-style:italic; color:#404040;} + i, .col {color:#202020;} .cat {font-size:small; color:gray;} - .news-figure-caption-text, .cap, .img {font-size:small; text-align:center;} + .news-figure-caption-text, .cap, .img, .css--caption-outer-wrapper {font-size:small; text-align:center;} .news-figure-credit {font-size:small; text-align:center; color:#202020;} ''' remove_tags = [ + dict(name=['button', 'svg']), dict(name='div', attrs={'id':['bb-that', 'bb-nav']}), - classes('twitter-logo bb-global-footer') + classes('twitter-logo bb-global-footer __sticky__audio__bar__portal__ css--social-wrapper-outer') ] - def get_browser(self): - br = browser() + def get_browser(self, *a, **kw): + kw['user_agent'] = 'common_words/based' + br = BasicNewsRecipe.get_browser(self, *a, **kw) br.set_handle_redirect(False) return br @@ -163,7 +180,7 @@ class Bloomberg(BasicNewsRecipe): body = '' body_data = data['body']['content'] for x in body_data: - pause = random.choice((0.5, 1, 1.25)) + pause = random.choice((0.25, 0.5, 0.75, 1)) time.sleep(pause) body += get_contents(x) return '' + cat + title + subhead + auth + lede + caption + '
    ' + body + '
    '