from calibre.web.feeds.news import BasicNewsRecipe def classes(classes): q = frozenset(classes.split(' ')) return dict(attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)}) def absolutize(url): if url.startswith('/'): url = 'https://www.nytimes.com' + url return url class NewYorkTimesBookReview(BasicNewsRecipe): title = u'New York Times Book Review' language = 'en' description = 'The New York Times Sunday Book Review' __author__ = 'Kovid Goyal' no_stylesheets = True no_javascript = True ignore_duplicate_articles = {'title', 'url'} encoding = 'utf-8' keep_only_tags = [ dict(name='h1'), dict(attrs={'data-testid':'photoviewer-wrapper'}), dict(itemprop=['author creator', 'articleBody']), ] def parse_index(self): soup = self.index_to_soup( 'https://www.nytimes.com/pages/books/review/index.html') # Find TOC toc = soup.find('section', id='collection-book-review').find('section').find('ol') main_articles, articles = [], [] feeds = [('Features', main_articles), ('Latest', articles)] for li in toc.findAll('li'): h2 = li.find('h2') a = h2.find('a', href=True) if a is not None: title = self.tag_to_string(a) url = absolutize(a['href']) desc = '' p = h2.findNextSibling('p') if p: desc = self.tag_to_string(p) main_articles.append( {'title': title, 'url': url, 'description': desc}) self.log('Found:', title, 'at', url) if desc: self.log('\t', desc) for li in soup.find(id='stream-panel').find('ol').findAll('li'): h2 = li.find('h2') a = h2.findParent('a') url = absolutize(a['href']) p = h2.findNextSibling('p') title = self.tag_to_string(h2) desc = '' if p: desc = self.tag_to_string(p) articles.append({'title': title, 'url': url, 'description': desc}) self.log('Found:', title, 'at', url) if desc: self.log('\t', desc) return feeds