calibre/recipes/business_standard_print.recipe

#!/usr/bin/env python
# vim:fileencoding=utf-8
import json
from datetime import datetime

from html5_parser import parse

from calibre.web.feeds.news import BasicNewsRecipe


class BusinessStandardPrint(BasicNewsRecipe):
    title = 'Business Standard Print Edition'
    __author__ = 'unkn0wn'
    description = "India's most respected business daily, Articles from Today's Paper"
    language = 'en_IN'
    masthead_url = 'https://bsmedia.business-standard.com/include/_mod/site/html5/images/business-standard-logo.png'
    encoding = 'utf-8'

    no_stylesheets = True
    remove_javascript = True
    remove_attributes = ['width', 'height', 'style']

    def get_browser(self):
        return BasicNewsRecipe.get_browser(self, user_agent='common_words/based')

    ignore_duplicate_articles = {'title', 'url'}
    remove_empty_feeds = True
    resolve_internal_links = True
    browser_type = 'webengine'

    extra_css = '''
        img {display:block; margin:0 auto;}
        .sub { font-style:italic; color:#202020; }
        .auth, .cat { font-size:small; color:#202020; }
        .cap { font-size:small; text-align:center; }
    '''

    recipe_specific_options = {
        'date': {
            'short': 'The date of the print edition to download (DD-MM-YYYY format)',
            'long': 'For example, 20-09-2023',
        }
    }

    def get_cover_url(self):
        d = self.recipe_specific_options.get('date')
        if not (d and isinstance(d, str)):
            soup = self.index_to_soup(
                'https://www.magzter.com/IN/Business-Standard-Private-Ltd/Business-Standard/Newspaper/'
            )
            return soup.find('img', id=lambda s: s and 'mgd__lhd__cover' in s.split())['src']

    def parse_index(self):
        today = datetime.today().strftime('%d-%m-%Y')
        d = self.recipe_specific_options.get('date')
        if d and isinstance(d, str):
            today = d

        day, month, year = (int(x) for x in today.split('-'))
        dt = datetime(year, month, day)
        self.timefmt = ' [' + dt.strftime('%b %d, %Y') + ']'

        if dt.weekday() == 6:
            self.log.warn(
                'Business Standard Does Not Have A Print Publication On Sunday. The Reports'
                " And Columns On This Page Today Appeared In The Newspaper's Saturday Edition."
            )
        url = 'https://apibs.business-standard.com/category/today-paper?sortBy=' + today
        raw = self.index_to_soup(url, raw=True)
        data = json.loads(raw)
        data = data['data']

        feeds = []

        for section in data:
            if section == 'EpaperImage':
                continue
            self.log(section)
            articles = []
            for article in data[section]:
                title = article['heading1']
                desc = article['sub_heading']
                url = 'https://www.business-standard.com' + article['article_url']
                self.log('\t', title, '\n\t', desc, '\n\t\t', url)
                articles.append({'title': title, 'description': desc, 'url': url})
            if articles:
                feeds.append((section, articles))
        return feeds

    def preprocess_raw_html(self, raw, *a):
        root = parse(raw)
        m = root.xpath('//script[@id="__NEXT_DATA__"]')

        data = json.loads(m[0].text)

        img_url = None
        if 'articleImageUrl' in data['props']['pageProps']['articleSchema']:
            img_url = data['props']['pageProps']['articleSchema']['articleImageUrl']

        art_url = 'https://www.business-standard.com' + data['props']['pageProps']['url']

        data = data['props']['pageProps']['data']

        title = '<h1 title="{}">'.format(art_url) + data['pageTitle'] + '</h1>'

        cat = subhead = lede = auth = caption = ''

        if 'defaultArticleCat' in data and data['defaultArticleCat'] is not None:
            if (
                'h1_tag' in data['defaultArticleCat']
                and data['defaultArticleCat']['h1_tag'] is not None
            ):
                cat = '<div class="cat">' + data['defaultArticleCat']['h1_tag'] + '</div>'

        if 'metaDescription' in data and data['metaDescription'] is not None:
            subhead = '<p class="sub">' + data['metaDescription'] + '</p>'
            self.art_desc = data['metaDescription']

        date = (datetime.fromtimestamp(int(data['publishDate']))).strftime(
            '%b %d, %Y | %I:%M %p'
        )

        authors = []
        if 'articleMappedMultipleAuthors' in data:
            for aut in data['articleMappedMultipleAuthors']:
                authors.append(data['articleMappedMultipleAuthors'][str(aut)])
        auth = (
            '<div><p class="auth">'
            + ', '.join(authors)
            + ' | '
            + data['placeName']
            + ' | '
            + date
            + '</p></div>'
        )

        if 'featuredImageObj' in data:
            if 'url' in data['featuredImageObj']:
                if img_url is not None:
                    lede = '<p class="cap"><img src="{}">'.format(img_url)
                else:
                    lede = '<p class="cap"><img src="{}">'.format(
                        data['featuredImageObj']['url']
                    )
            if 'alt_text' in data['featuredImageObj']:
                caption = '<span>' + data['featuredImageObj']['alt_text'] + '</span></p>'

        body = data['htmlContent']

        return (
            '<html><body>'
            + cat
            + title
            + subhead
            + auth
            + lede
            + caption
            + '<div><br>'
            + body
            + '</div></body></html>'
        )

    def preprocess_html(self, soup):
        for img in soup.findAll('img'):
            img.attrs = {'src': img.get('src', '')}
        for x in soup.findAll('div', 'p'):
            x.attrs = {'class': x.get('class', '')}
        for attr in self.remove_attributes:
            for x in soup.findAll(attrs={attr: True}):
                del x[attr]
        for br in soup.findAll(attrs={'class': lambda x: x and x.startswith('brtag')}):
            br.name = 'div'
        return soup