calibre/recipes/business_standard.recipe

#!/usr/bin/env python
# vim:fileencoding=utf-8
import json
from datetime import datetime

from html5_parser import parse

from calibre.web.feeds.news import BasicNewsRecipe


class BusinessStandard(BasicNewsRecipe):
    title = 'Business Standard'
    __author__ = 'unkn0wn'
    description = "India's most respected business daily"
    language = 'en_IN'
    masthead_url = 'https://bsmedia.business-standard.com/include/_mod/site/html5/images/business-standard-logo.png'

    no_stylesheets = True
    remove_javascript = True
    remove_attributes = ['width', 'height', 'style']

    def get_browser(self):
        return BasicNewsRecipe.get_browser(self, user_agent='common_words/based')

    ignore_duplicate_articles = {'title', 'url'}
    remove_empty_feeds = True
    resolve_internal_links = True
    max_articles_per_feed = 50
    oldest_article = 1.15
    browser_type = 'webengine'

    extra_css = '''
        img {display:block; margin:0 auto;}
        .sub { font-style:italic; color:#202020; }
        .auth, .cat { font-size:small; color:#202020; }
        .cap { font-size:small; text-align:center; }
    '''

    recipe_specific_options = {
        'days': {
            'short': 'Oldest article to download from this news source. In days ',
            'long': 'For example, 0.5, gives you articles from the past 12 hours',
            'default': str(oldest_article)
        }
    }

    def get_cover_url(self):
        d = self.recipe_specific_options.get('date')
        if not (d and isinstance(d, str)):
            soup = self.index_to_soup(
                'https://www.magzter.com/IN/Business-Standard-Private-Ltd/Business-Standard/Newspaper/'
            )
            for citem in soup.findAll(
                'meta', content=lambda s: s and s.endswith('view/3.jpg')
            ):
                return citem['content']

    def __init__(self, *args, **kwargs):
        BasicNewsRecipe.__init__(self, *args, **kwargs)
        d = self.recipe_specific_options.get('days')
        if d and isinstance(d, str):
            self.oldest_article = float(d)

    # https://www.business-standard.com/rss-feeds/listing
    feeds = [
        ('Top Stories', 'https://www.business-standard.com/rss/home_page_top_stories.rss'),
        ('Todays Paper', 'https://www.business-standard.com/rss/todays-paper.rss'),
        ('Budget', 'https://www.business-standard.com/rss/budget-110.rss'),
        ('Economy', 'https://www.business-standard.com/rss/economy-102.rss'),
        ('Opinion', 'https://www.business-standard.com/rss/opinion-105.rss'),
        ('Companies', 'https://www.business-standard.com/rss/companies-101.rss'),
        ('Industries', 'https://www.business-standard.com/rss/industry-217.rss'),
        ('Market', 'https://www.business-standard.com/rss/markets-106.rss'),
        ('Politics', 'https://www.business-standard.com/rss/budget-110.rss'),
        ('World', 'https://www.business-standard.com/rss/industry-217.rss'),
        ('Technology', 'https://www.business-standard.com/rss/technology-108.rss'),
        ('Latest', 'https://www.business-standard.com/rss/latest.rss')
    ]

    def preprocess_raw_html(self, raw, *a):
        root = parse(raw)
        m = root.xpath('//script[@id="__NEXT_DATA__"]')

        data = json.loads(m[0].text)

        img_url = None
        if 'articleImageUrl' in data['props']['pageProps']['articleSchema']:
            img_url = data['props']['pageProps']['articleSchema']['articleImageUrl']

        art_url = 'https://www.business-standard.com' + data['props']['pageProps']['url']

        data = data['props']['pageProps']['data']

        title = '<h1 title="{}">'.format(art_url) + data['pageTitle'] + '</h1>'

        cat = subhead = lede = auth = caption = ''

        if 'defaultArticleCat' in data and data['defaultArticleCat'] is not None:
            if (
                'h1_tag' in data['defaultArticleCat']
                and data['defaultArticleCat']['h1_tag'] is not None
            ):
                cat = '<div class="cat">' + data['defaultArticleCat']['h1_tag'] + '</div>'

        if 'metaDescription' in data and data['metaDescription'] is not None:
            subhead = '<p class="sub">' + data['metaDescription'] + '</p>'
            self.art_desc = data['metaDescription']

        date = (datetime.fromtimestamp(int(data['publishDate']))).strftime(
            '%b %d, %Y | %I:%M %p'
        )

        authors = []
        if 'articleMappedMultipleAuthors' in data:
            for aut in data['articleMappedMultipleAuthors']:
                authors.append(data['articleMappedMultipleAuthors'][str(aut)])
        auth = (
            '<div><p class="auth">'
            + ', '.join(authors)
            + ' | '
            + data['placeName']
            + ' | '
            + date
            + '</p></div>'
        )

        if 'featuredImageObj' in data:
            if 'url' in data['featuredImageObj']:
                if img_url is not None:
                    lede = '<p class="cap"><img src="{}">'.format(img_url)
                else:
                    lede = '<p class="cap"><img src="{}">'.format(
                        data['featuredImageObj']['url']
                    )
            if 'alt_text' in data['featuredImageObj']:
                caption = '<span>' + data['featuredImageObj']['alt_text'] + '</span></p>'

        body = data['htmlContent']

        return (
            '<html><body>'
            + cat
            + title
            + subhead
            + auth
            + lede
            + caption
            + '<div><br>'
            + body
            + '</div></body></html>'
        )

    def preprocess_html(self, soup):
        for img in soup.findAll('img'):
            img.attrs = {'src': img.get('src', '')}
        for x in soup.findAll('div', 'p'):
            x.attrs = {'class': x.get('class', '')}
        for attr in self.remove_attributes:
            for x in soup.findAll(attrs={attr: True}):
                del x[attr]
        for br in soup.findAll(attrs={'class': lambda x: x and x.startswith('brtag')}):
            br.name = 'div'
        return soup