Update Bloomberg Businessweek

recipes/bloomberg-business-week.recipe

@@ -1,196 +1,141 @@
 import json
-import random
 import time
-from collections import defaultdict
+from datetime import datetime, timedelta
 
-from calibre.web.feeds.news import BasicNewsRecipe, classes, prefixed_classes
+from calibre.ebooks.BeautifulSoup import BeautifulSoup
-from html5_parser import parse
+from calibre.web.feeds.news import BasicNewsRecipe
 
 
+# https://www.bloomberg.com/magazine/businessweek/24_12
+# Set past_edition to edition id, which is '24_12'.
+past_edition = None
+
 def get_contents(x):
     if x == '':
         return ''
-    otype = x.get('type', '')
+    otype = x.get('role', '')
-    if otype == 'text':
+    if otype == 'p':
-        if 'attributes' in x:
+        return '<p>' + ''.join(map(get_contents, x.get('parts', ''))) + '</p>'
-            if 'strong' in x['attributes']:
+    elif otype == 'text':
-                return '<strong>' + x.get('value', '') + ''.join(map(get_contents, x.get('content', ''))) + '</strong>'
+        if 'style' in x:
-            if 'emphasis' in x['attributes']:
+            return '<' + x['style']  + '>' + ''.join(map(get_contents, x.get('parts', ''))) + '</' + x['style']  + '>'
-                return '<em>' + x.get('value', '') + ''.join(map(get_contents, x.get('content', ''))) + '</em>'
+        return x.get('text', '') + ''.join(map(get_contents, x.get('parts', '')))
-            return '<i>' + x.get('value', '') + ''.join(map(get_contents, x.get('content', ''))) + '</i>'
-        return x.get('value', '') + ''.join(map(get_contents, x.get('content', '')))
     elif otype == 'br':
         return '<br>'
-    elif otype == 'paragraph':
+    elif otype == 'anchor':
-        return '<p>' + x.get('value', '') + ''.join(map(get_contents, x.get('content', ''))) + '</p>'
+        return '<span>' + ''.join(map(get_contents, x.get('parts', ''))) + '</span>'
-    elif otype == 'heading':
+    elif otype == 'h3':
-        return '<h3>' + x.get('value', '') + ''.join(map(get_contents, x.get('content', ''))) + '</h3>'
+        return '<h4>' + ''.join(map(get_contents, x.get('parts', ''))) + '</h4>'
-    elif otype == 'list':
+    elif otype == 'ul':
-        return '<ul>' + ''.join(map(get_contents, x.get('content', ''))) + '</ul>'
+        return '<ul>' + ''.join(map(get_contents, x.get('parts', ''))) + '</ul>'
-    elif otype == 'listItem':
+    elif otype == 'li':
-        return '<li>' + x.get('value', '') + ''.join(map(get_contents, x.get('content', ''))) + '</li>'
+        return '<li>' + ''.join(map(get_contents, x.get('parts', ''))) + '</li>'
-    elif otype == 'quote':
+    elif otype == 'webview':
-        return '<blockquote class="col">' + x.get('value', '') + ''.join(map(get_contents, x.get('content', ''))) + '</blockquote>'
+        return '<br>' + x['html'] + ''.join(map(get_contents, x.get('parts', '')))
-    elif otype == 'media':
+    elif otype == 'blockquote':
-        if x['subType'] == 'photo':
+        return '<blockquote>' + ''.join(map(get_contents, x.get('parts', ''))) + '</blockquote>'
-            return '<div><div class="img"><img src="{}"></div><div class="cap">{} <span>{}</span></div></div>'.format(
+    elif otype in {'image', 'video'}:
-                x['data']['photo']['src'], x['data']['photo']['caption'], x['data']['photo']['credit'])
+        return '<br><img src="{}"><div class="img">{}</div>\n'.format(
-        elif x['subType'] == 'chart':
+                x['imageURLs']['default'], x['caption'] + '<i> ' + x['credit'] + '</i>'
-            if x['data'] and x['data']['chart']:
+            )
-                return '<div class="img"><img src="{}"></div>'.format(x['data']['chart']['fallback'])
+    elif otype in {'correction', 'disclaimer'}:
-    elif otype == 'link':
+        return '<p class="corr">' + ''.join(map(get_contents, x.get('parts', ''))) + '</p>'
-        if 'data' in x:
-            if 'href' in x['data']:
-                return '<a href="' + x['data']['href'] + '">' + x.get('value', '') + ''.join(map(get_contents, x.get('content', ''))) + '</a>'
-            return '<i>' + x.get('value', '') + ''.join(map(get_contents, x.get('content', ''))) + '</i>'
-        return '<i>' + x.get('value', '') + ''.join(map(get_contents, x.get('content', ''))) + '</i>'
-    elif otype == 'entity':
-        if x['subType'] == 'story':
-            if x['data'] and x['data']['link'] and x['data']['link']['destination']:
-                if 'web' in x['data']['link']['destination']:
-                    return '<a href="' + x['data']['link']['destination']['web'] + '">' + x.get('value', '') + ''.join(
-                        map(get_contents, x.get('content', ''))) + '</a>'
-                return '<i>' + x.get('value', '') + ''.join(map(get_contents, x.get('content', ''))) + '</i>'
-            return '<i>' + x.get('value', '') + ''.join(map(get_contents, x.get('content', ''))) + '</i>'
-        return '<i>' + x.get('value', '') + ''.join(map(get_contents, x.get('content', ''))) + '</i>'
-    elif otype in {'div', 'callout'}:
-        return '<div>' + x.get('value', '') + ''.join(map(get_contents, x.get('content', ''))) + '</div>'
-    elif not any(x == otype for x in ['', 'ad', 'inline-newsletter', 'tabularData']):
-        if any(b in x for b in ['value', 'content']):
-            return '<i>' + x.get('value', '') + ''.join(map(get_contents, x.get('content', ''))) + '</i>'
 
+    elif not any(x == otype for x in ['', 'ad', 'inline-newsletter', 'tabularData']):
+        return '<i>' + ''.join(map(get_contents, x.get('parts', ''))) + '</i>'
     return ''
 
+
 class Bloomberg(BasicNewsRecipe):
-    title = u'Bloomberg Businessweek'
+    title = 'Bloomberg Businessweek'
     language = 'en'
     __author__ = 'unkn0wn'
     no_stylesheets = True
-    use_embedded_content = False
     remove_attributes = ['style', 'height', 'width']
+    encoding = 'utf-8'
     ignore_duplicate_articles = {'url'}
-    resolve_internal_links = True
     masthead_url = 'https://assets.bwbx.io/s3/javelin/public/hub/images/BW-Logo-Black-cc9035fbb3.svg'
     description = (
         'Bloomberg Businessweek helps global leaders stay ahead with insights and in-depth analysis on the people,'
         ' companies, events, and trends shaping today\'s complex, global economy.'
     )
-
+    remove_empty_feeds = True
-    simultaneous_downloads = 1
 
     extra_css = '''
-        .auth {font-size:small; font-weight:bold;}
+        .auth { font-size:small; font-weight:bold; }
-        .time, .chart, .css--lede-byline, .css--lede-timestamp {font-size:small;}
+        .subhead, .cap span { font-style:italic; color:#202020; }
-        .subhead, .cap span {font-style:italic; color:#404040;}
+        em, blockquote { color:#202020; }
-        em, .col {color:#202020;}
+        .cat { font-size:small; color:gray; }
-        .cat {font-size:small; color:gray;}
+        .img, .news-figure-caption-text { font-size:small; text-align:center; }
-        .news-figure-caption-text, .cap, .img, .css--caption-outer-wrapper {font-size:small; text-align:center;}
+        .corr { font-size:small; font-style:italic; color:#404040; }
+        .chart { font-size:small; }
         .news-figure-credit {font-size:small; text-align:center; color:#202020;}
     '''
 
-    remove_tags = [
-        dict(name=['button', 'svg']),
-        dict(name='div', attrs={'id':['bb-that', 'bb-nav']}),
-        classes('twitter-logo bb-global-footer __sticky__audio__bar__portal__ css--social-wrapper-outer')
-    ]
-
-    def get_browser(self, *a, **kw):
-        kw['user_agent'] = 'Mozilla/5.0 (Windows NT 10.0; rv:109.0) Gecko/20100101 Firefox/119.0'
-        br = BasicNewsRecipe.get_browser(self, *a, **kw)
-        br.set_handle_redirect(False)
-        return br
-
     def parse_index(self):
-        soup = self.index_to_soup('https://www.bloomberg.com/businessweek')
+        inx = 'https://cdn-mobapi.bloomberg.com'
-        bw = soup.find('a', href=lambda x: x and x.startswith('/magazine/businessweek/'))
+        sec = self.index_to_soup(inx + '/wssmobile/v1/bw/news/list?limit=1', raw=True)
-        edition = 'https://www.bloomberg.com' + bw['href']
+        id = json.loads(sec)['magazines'][0]['id']
-        self.log('Downloading ', edition)
+        if past_edition:
-        self.cover_url = bw.find('img')['src'].replace('25x19', '600x800')
+            id = past_edition
-        soup = self.index_to_soup(edition)
+        edit = self.index_to_soup(inx + '/wssmobile/v1/bw/news/week/' + id, raw=True)
-        if timefmt := soup.find(**prefixed_classes('styles_TableOfContentsTitle__')):
+        d = json.loads(edit)
-            self.timefmt = ' [' + (self.tag_to_string(timefmt).replace(' Issue', '')).strip() + ']'
+        self.timefmt = ' [' + d['date'] + ']'
+        self.cover_url = d['image']['thumbUrl']
 
-        feeds_dict = defaultdict(list)
+        feeds = []
 
-        sec = ''
+        for i in d['modules']:
-        toc = soup.find('section', attrs={'id':'toc-archive-businessweek'})
+            section = i['title']
-        for div in toc.findAll(**prefixed_classes('MagazinePageMagazineArchive_itemContainer__')):
+            self.log(section)
-            h3 = div.find(**prefixed_classes('MagazinePageMagazineArchive_itemSection__'))
+
-            if h3 and h3.text:
+            articles = []
-                sec = self.tag_to_string(h3)
+
-                self.log(sec)
+            for x in i['articles']:
-            a = div.find(**prefixed_classes('MagazinePageMagazineArchive_storyLink__'))
+                title = x['title']
-            url = a['href']
+                url = inx + '/wssmobile/v1/stories/' + x['id']
-            if url.startswith('http') is False:
+                self.log('\t', title)
-                url = 'https://www.bloomberg.com' + a['href']
+                articles.append({'title': title, 'url': url})
-            title = self.tag_to_string(a)
+            feeds.append((section, articles))
-            byl = div.find(**prefixed_classes('Byline_phoenix__'))
+        return feeds
-            desc = self.tag_to_string(byl)
-            self.log('\t', title, '\n\t', desc, '\n\t\t', url)
-            feeds_dict[sec].append({"title": title, "url": url, "description": desc})
-        return [(sec, articles) for sec, articles in feeds_dict.items()]
 
     def preprocess_raw_html(self, raw, *a):
-        root = parse(raw)
+        data = json.loads(raw)
-        m = root.xpath('//script[@data-component-props="ArticleBody"]')
-        if not m:
-            m = root.xpath('//script[@data-component-props="FeatureBody"]')
-            if not m:
-                m2 = root.xpath('//script[@id="__NEXT_DATA__"]')
-                if not m2:
-                    return raw
-        if m:
-            data = json.loads(m[0].text)
-            data = data['story']
 
-        else:
+        title = '<h1 title="{}">'.format(data['longURL']) + data['title'] + '</h1>'
-            data = json.loads(m2[0].text)
-            if 'story' in data['props']['pageProps']:
-                data = data['props']['pageProps']['story']
-            else:
-                return raw
-
-        title = '<h1>' + data['headline'] + '</h1>'
 
         cat = subhead = lede = auth = caption = ''
 
         if 'primaryCategory' in data and data['primaryCategory'] is not None:
-                cat = '<p class="cat">' + data['primaryCategory'] + '</p>'
+            cat = '<div class="cat">' + data['primaryCategory'] + '</div>'
 
-        if len(data['abstract']) != 0 and len(data['abstract']) == 2:
+        if 'abstract' in data and data['abstract'] and data['abstract'] is not None:
-                subhead = '<div class="subhead"><p>' + data['abstract'][0] + '</p><p>' + data['abstract'][1] + '</p></div>'
+            subhead = '<div class="subhead"><ul><li>' + '</li><li>'.join([x for x in data['abstract']]) + '</li></ul></div>'
-        else:
+        elif 'summary' in data and data['summary']:
-            if 'summary' in data:
+            subhead = '<div class="subhead"><p>' + data['summary'] + '</p></div>'
-                subhead = '<div class="subhead"><p>' + data['summary'] + '</p></div>'
 
         if 'byline' in data and data['byline'] is not None:
-                auth = '<div><span class="auth">' + data['byline']\
+            dt = datetime.fromtimestamp(data['updatedAt'] + time.timezone)
-                 + '</span> | <span class="time">' + data['publishedAt'][:-14] + '</span></div>'
+            auth = '<p class="auth">' + 'By ' + data['byline'] + ' | Updated on ' + dt.strftime('%b %d, %Y at %I:%M %p') + '</p>'
 
-        if 'ledeImageUrl' in data and data['ledeImageUrl'] is not None:
+        if 'ledeImage' in data and data['ledeImage'] is not None:
-                lede = '<p id="img"><img src="{}">'.format(data['ledeImageUrl'])
+            x = data['ledeImage']
+            lede = '<br><img src="{}"><div class="img">{}</div>\n'.format(
+                x['imageURLs']['default'], x['caption'] + '<i> ' + x['credit'] + '</i>'
+            )
 
-        if 'ledeDescription' in data and data['ledeDescription'] is not None:
+        body = ''
-                caption = '<span class="cap">' + data['ledeDescription'] + '</span>'
+        if data.get('type', '') == 'interactive':
-        else:
+            body += '<p><em>' + 'This is an interactive article, which is supposed to be read in a browser.' + '</p></em>'
-            if 'lede' in data and data['lede'] is not None:
+        body_data = data['components']
-                    if 'alt' in data['lede'] and data['lede']['alt'] is not None:
+        for x in body_data:
-                            caption = '<span class="cap">' + data['lede']['alt'] + '</span>'
+            body += get_contents(x)
-
+        html = '<html><body>' + cat + title + subhead + auth + lede + caption + '<div>' + body + '</div></body></html>'
-        if m:
+        return BeautifulSoup(html).prettify()
-            time.sleep(3)
-            body = data['body']
-        else:
-            body = ''
-            body_data = data['body']['content']
-            for x in body_data:
-                body += get_contents(x)
-            pause = random.choice((5, 6, 7, 8, 9))
-            self.log('Delay: ', pause, ' seconds')
-            time.sleep(pause)
-        return '<html><body>' + cat + title + subhead + auth + lede + caption + '<div>' + body + '</div></body></html>'
 
     def preprocess_html(self, soup):
+        for h3 in soup.findAll('h3'):
+            h3.name = 'h4'
         for icon in soup.findAll('img', attrs={'class':'video-player__play-icon'}):
             icon.decompose()
         for div in soup.findAll('div', attrs={'class':'chart'}):
@@ -205,3 +150,9 @@ class Bloomberg(BasicNewsRecipe):
         for img in soup.findAll('img', attrs={'src':lambda x: x and x.endswith(('-1x-1.jpg', '-1x-1.png'))}):
             img['src'] = img['src'].replace('-1x-1', '750x-1')
         return soup
+
+    def populate_article_metadata(self, article, soup, first):
+        article.url = soup.find('h1')['title']
+        article.summary = self.tag_to_string(soup.find('div', attrs={'class':'subhead'}))
+        article.text_summary = article.summary
+

recipes/bloomberg.recipe

@@ -19,7 +19,7 @@ def get_contents(x):
     elif otype == 'br':
         return '<br>'
     elif otype == 'anchor':
-        return '<em>' + ''.join(map(get_contents, x.get('parts', ''))) + '</em>'
+        return '<span>' + ''.join(map(get_contents, x.get('parts', ''))) + '</span>'
     elif otype == 'h3':
         return '<h4>' + ''.join(map(get_contents, x.get('parts', ''))) + '</h4>'
     elif otype == 'ul':
@@ -30,11 +30,11 @@ def get_contents(x):
         return '<br>' + x['html'] + ''.join(map(get_contents, x.get('parts', '')))
     elif otype == 'blockquote':
         return '<blockquote>' + ''.join(map(get_contents, x.get('parts', ''))) + '</blockquote>'
-    elif otype == ('image' or 'video'):
+    elif otype in {'image', 'video'}:
         return '<br><img src="{}"><div class="img">{}</div>\n'.format(
                 x['imageURLs']['default'], x['caption'] + '<i> ' + x['credit'] + '</i>'
             )
-    elif otype == ('correction' or 'disclaimer'):
+    elif otype in {'correction', 'disclaimer'}:
         return '<p class="corr">' + ''.join(map(get_contents, x.get('parts', ''))) + '</p>'
 
     elif not any(x == otype for x in ['', 'ad', 'inline-newsletter', 'tabularData']):
@@ -44,7 +44,7 @@ def get_contents(x):
 
 class Bloomberg(BasicNewsRecipe):
     title = u'Bloomberg'
-    language = 'en'
+    language = 'en_US'
     __author__ = 'unkn0wn'
     no_stylesheets = True
     remove_attributes = ['style', 'height', 'width']
@@ -89,7 +89,7 @@ class Bloomberg(BasicNewsRecipe):
                 for arts in json.loads(art_soup)['modules']:
                     if arts['stories']:
                         for x in arts['stories']:
-                            if x.get('type', '') == 'article':
+                            if x.get('type', '') in {'article', 'interactive'}:
                                 dt = datetime.fromtimestamp(x['published'] + time.timezone)
                                 if (datetime.now() - dt) > timedelta(self.oldest_article):
                                     continue
@@ -109,7 +109,7 @@ class Bloomberg(BasicNewsRecipe):
         cat = subhead = lede = auth = caption = ''
 
         if 'primaryCategory' in data and data['primaryCategory'] is not None:
-            cat = '<p class="cat">' + data['primaryCategory'] + '</p>'
+            cat = '<div class="cat">' + data['primaryCategory'] + '</div>'
 
         if 'abstract' in data and data['abstract'] and data['abstract'] is not None:
             subhead = '<div class="subhead"><ul><li>' + '</li><li>'.join([x for x in data['abstract']]) + '</li></ul></div>'
@@ -127,6 +127,8 @@ class Bloomberg(BasicNewsRecipe):
             )
 
         body = ''
+        if data.get('type', '') == 'interactive':
+            body += '<p><em>' + 'This is an interactive article, which is supposed to be read in a browser.' + '</p></em>'
         body_data = data['components']
         for x in body_data:
             body += get_contents(x)