calibre/recipes/nytimesbook.recipe

#!/usr/bin/env python
# vim:fileencoding=utf-8
# License: GPLv3 Copyright: 2015, Kovid Goyal <kovid at kovidgoyal.net>

from calibre.web.feeds.news import BasicNewsRecipe

use_wayback_machine = False


def absolutize(url):
    if url.startswith('/'):
        url = 'https://www.nytimes.com' + url
    return url


class NewYorkTimesBookReview(BasicNewsRecipe):
    title = u'New York Times Book Review'
    language = 'en_US'
    description = 'The New York Times Sunday Book Review'
    __author__ = 'Kovid Goyal'

    no_stylesheets = True
    no_javascript = True
    ignore_duplicate_articles = {'title', 'url'}
    encoding = 'utf-8'

    extra_css = '''
        .byl, .time { font-size:small; color:#202020; }
        .cap { font-size:small; text-align:center; }
        .cred { font-style:italic; font-size:small; }
        em, blockquote { color: #202020; }
        .sc { font-variant: small-caps; }
        .lbl { font-size:small; color:#404040; }
        img { display:block; margin:0 auto; }
    '''

    articles_are_obfuscated = use_wayback_machine

    if use_wayback_machine:
        def get_obfuscated_article(self, url):
            from calibre.ptempfile import PersistentTemporaryFile
            with PersistentTemporaryFile() as tf:
                tf.write(self.get_nyt_page(url))
            return tf.name

    @property
    def nyt_parser(self):
        ans = getattr(self, '_nyt_parser', None)
        if ans is None:
            from calibre.live import load_module
            self._nyt_parser = ans = load_module('calibre.web.site_parsers.nytimes')
        return ans

    def get_nyt_page(self, url, skip_wayback=False):
        if use_wayback_machine and not skip_wayback:
            from calibre import browser
            return self.nyt_parser.download_url(url, browser())
        return self.index_to_soup(url, raw=True)

    def preprocess_raw_html(self, raw_html, url):
        return self.nyt_parser.extract_html(self.index_to_soup(raw_html), url)

    recipe_specific_options = {
        'res': {
            'short': (
                'For hi-res images, select a resolution from the following\noptions: '
                'popup, jumbo, mobileMasterAt3x, superJumbo'
            ),
            'long': (
                'This is useful for non e-ink devices, and for a lower file size\nthan '
                'the default, use mediumThreeByTwo440, mediumThreeByTwo225, articleInline.'
            ),
        },
        'comp': {
            'short': 'Compress News Images?',
            'long': 'enter yes',
            'default': 'no'
        }
    }

    def __init__(self, *args, **kwargs):
        BasicNewsRecipe.__init__(self, *args, **kwargs)
        c = self.recipe_specific_options.get('comp')
        if c and isinstance(c, str):
            if c.lower() == 'yes':
                self.compress_news_images = True

    def parse_index(self):
        # return [('Articles', [{'url': 'https://www.nytimes.com/2022/09/08/books/review/karen-armstrong-by-the-book-interview.html', 'title':'test'}])]
        soup = self.index_to_soup(
            self.get_nyt_page('https://www.nytimes.com/pages/books/review/index.html', skip_wayback=True))

        # Find TOC
        toc = soup.find('section', id='collection-book-review').find('section').find('ol')
        main_articles, articles = [], []
        feeds = [('Features', main_articles), ('Latest', articles)]
        for li in toc.findAll('li'):
            h2 = li.find(['h2', 'h3'])
            a = h2.find('a', href=True)
            if a is not None:
                title = self.tag_to_string(a)
                url = absolutize(a['href'])
                desc = ''
                p = h2.findNextSibling('p')
                if p:
                    desc = self.tag_to_string(p)
                main_articles.append(
                    {'title': title, 'url': url, 'description': desc})
                self.log('Found:', title, 'at', url)
                if desc:
                    self.log('\t', desc)
        for li in soup.find(id='stream-panel').find('ol').findAll('li'):
            h2 = li.find(['h2', 'h3'])
            a = h2.findParent('a')
            url = absolutize(a['href'])
            p = h2.findNextSibling('p')
            title = self.tag_to_string(h2)
            desc = ''
            if p:
                desc = self.tag_to_string(p)
            articles.append({'title': title, 'url': url, 'description': desc})
            self.log('Found:', title, 'at', url)
            if desc:
                self.log('\t', desc)

        return feeds

    def get_browser(self, *args, **kwargs):
        kwargs['user_agent'] = 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
        br = BasicNewsRecipe.get_browser(self, *args, **kwargs)
        br.addheaders += [
            ('Referer', 'https://www.google.com/'),
            ('X-Forwarded-For', '66.249.66.1')
        ]
        return br

    def preprocess_html(self, soup):
        w = self.recipe_specific_options.get('res')
        if w and isinstance(w, str):
            res = '-' + w
            for img in soup.findAll('img', attrs={'src':True}):
                if '-article' in img['src']:
                    ext = img['src'].split('?')[0].split('.')[-1]
                    img['src'] = img['src'].rsplit('-article', 1)[0] + res + '.' + ext
        for c in soup.findAll('div', attrs={'class':'cap'}):
            for p in c.findAll(['p', 'div']):
                p.name = 'span'
        return soup