diff --git a/recipes/livemint.recipe b/recipes/livemint.recipe index c43d2cafb4..53e5cb4a46 100644 --- a/recipes/livemint.recipe +++ b/recipes/livemint.recipe @@ -102,7 +102,7 @@ class LiveMint(BasicNewsRecipe): classes( 'trendingSimilarHeight moreNews mobAppDownload label msgError msgOk taboolaHeight gadgetSlider' ' socialHolder imgbig disclamerText disqus-comment-count openinApp2 lastAdSlot bs_logo author-widget' - ' datePublish sepStory premiumSlider moreStory Joinus moreAbout milestone benefitText' + ' datePublish sepStory premiumSlider moreStory Joinus moreAbout milestone benefitText checkCibilBtn' ) ] diff --git a/recipes/slate.recipe b/recipes/slate.recipe index f6783ae2f9..24ed1db877 100644 --- a/recipes/slate.recipe +++ b/recipes/slate.recipe @@ -20,66 +20,48 @@ class Slate(BasicNewsRecipe): title = 'Slate' description = 'A general-interest publication offering analysis and commentary about politics, news and culture.' __author__ = 'Kovid Goyal' - timefmt = '' no_stylesheets = True language = 'en' encoding = 'utf-8' - remove_attributes = ['style'] + remove_attributes = ['style', 'height', 'width'] + oldest_article = 2 # days INDEX = 'https://slate.com' - compress_news_images = True + resolve_internal_links = True + remove_empty_feeds = True + ignore_duplicate_articles = {'url'} + + extra_css = ''' + .article__rubric { font-size:small; color:#404040; } + .article__byline, time { font-size:small; } + .article__top-image, .image__credit, .image__meta { font-size:small; text-align:center; } + em, blockquote, .alternativeHeadline { color:#202020; } + ''' keep_only_tags = [ - classes('article__header article__content'), + classes('article__header article__top-image article__content article-hotseats__body'), ] remove_tags = [ + dict(name=['aside', 'svg', 'iframe']), classes('social-share slate-ad newsletter-signup in-article-recirc'), ] def preprocess_html(self, soup): + for h2 in soup.findAll('h2'): + h2.name = 'h4' for img in soup.findAll('img', attrs={'data-srcset': True}): - img['src'] = img['data-srcset'].split()[0] + img['src'] = img['data-src'] + '&width=600' return soup - def parse_index(self): - ans = [] - for sectitle, url in ( - ('News & Politics', '/articles/news_and_politics.html'), - ('Technology', '/articles/technology.html'), - ('Business', '/articles/business.html'), - ('Arts', '/articles/arts.html'), - ('Life', '/articles/life.html'), - ('Health & Science', '/articles/health_and_science.html'), - ('Sports', '/articles/sports.html'), - ('Double X', '/articles/double_x.html'), - ): - url = self.INDEX + url - self.log('\nFound section:', sectitle, url) - articles = self.slate_section_articles(self.index_to_soup(url)) - if articles: - ans.append((sectitle, articles)) - if self.test and len(ans) > 1: - break - return ans + feeds = [ + ('News & Politics', 'https://slate.com/feeds/news-and-politics.rss'), + ('Culture', 'https://slate.com/feeds/culture.rss'), + ('Technology', 'https://slate.com/feeds/technology.rss'), + ('Business', 'https://slate.com/feeds/business.rss'), + ('Human Interest', 'https://slate.com/feeds/human-interest.rss'), + ('Others', 'https://slate.com/feeds/all.rss') + ] - def slate_section_articles(self, soup): - ans = [] - main = soup.find('section', **classes('main')) - if main is None: - return ans - for div in main.findAll(**classes('section-feed__item')): - a = div.find('a') - url = a['href'] - if url.endswith('/'): - continue - h = a.find(['h2', 'h3', 'h4']) - title = self.tag_to_string(h) - desc = '' - for q in ('byline', 'dek'): - span = div.find(attrs={'class': lambda x: x and ('-' + q) in x}) - if span is not None: - desc += self.tag_to_string(span).strip() - self.log('\t' + title) - self.log('\t\t' + url) - ans.append({'title': title, 'description': desc.strip(), - 'date': '', 'url': url}) - return ans + def get_article_url(self, article): + url = BasicNewsRecipe.get_article_url(self, article) + if '/podcasts/' not in url: + return url.split('?')[0] diff --git a/recipes/wsj.recipe b/recipes/wsj.recipe index 5a47809939..ea8d158f91 100644 --- a/recipes/wsj.recipe +++ b/recipes/wsj.recipe @@ -196,5 +196,4 @@ class WSJ(BasicNewsRecipe): article.url = lnk['title'] art = soup.find('h1', attrs={'title':True}) if art: - self.log('found art ', art['title']) article.url = art['title']