From 4db53354715dec46e1becb56a7c56705cb997435 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 31 Jul 2021 21:19:11 +0530 Subject: [PATCH] Add the Whats news articles to WSJ --- recipes/wsj.recipe | 13 +++++++------ recipes/wsj_free.recipe | 13 +++++++------ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/recipes/wsj.recipe b/recipes/wsj.recipe index b9c6e32a94..e09fb2074b 100644 --- a/recipes/wsj.recipe +++ b/recipes/wsj.recipe @@ -169,13 +169,13 @@ class WSJ(BasicNewsRecipe): root = self.index_to_soup(url, as_tree=True) CSSSelect = Select(root) articles = [] - for container in root.xpath('descendant::div[contains(@class, "WSJTheme--list-item-")]'): + for container in root.xpath('descendant::div[contains(@class, "WSJTheme--list-item--")]'): heading = next(CSSSelect('h2, h3', container)) a = next(CSSSelect('a', heading)) title = self.tag_to_string(a) url = self.abs_wsj_url(a.get('href')) desc = '' - for p in container.xpath('descendant::p[contains(@class, "WSJTheme--description-")]'): + for p in container.xpath('descendant::p[contains(@class, "WSJTheme--description--")]'): q = self.tag_to_string(p) if 'Subscriber Content' in q: continue @@ -184,11 +184,10 @@ class WSJ(BasicNewsRecipe): articles.append({'title': title, 'url': url, 'description': desc, 'date': ''}) - if self.test and len(articles) >= self.test[1]: - break - self.log('\tFound article:', title) self.log('\t\t', desc) + if self.test and len(articles) >= self.test[1]: + break return articles @@ -250,7 +249,9 @@ class WSJ(BasicNewsRecipe): self.log('Found section:', title, 'at', url) self.wsj_add_feed(feeds, title, url) if frontpage: - self.wsj_find_wn_articles(feeds, root, CSSSelect) + articles = self.wsj_find_wn_articles(feeds, root, CSSSelect) + if articles: + feeds.append(("What's News", articles)) if self.test and len(feeds) >= self.test[0]: break return feeds diff --git a/recipes/wsj_free.recipe b/recipes/wsj_free.recipe index fc57d390e5..3cbc3e7a55 100644 --- a/recipes/wsj_free.recipe +++ b/recipes/wsj_free.recipe @@ -169,13 +169,13 @@ class WSJ(BasicNewsRecipe): root = self.index_to_soup(url, as_tree=True) CSSSelect = Select(root) articles = [] - for container in root.xpath('descendant::div[contains(@class, "WSJTheme--list-item-")]'): + for container in root.xpath('descendant::div[contains(@class, "WSJTheme--list-item--")]'): heading = next(CSSSelect('h2, h3', container)) a = next(CSSSelect('a', heading)) title = self.tag_to_string(a) url = self.abs_wsj_url(a.get('href')) desc = '' - for p in container.xpath('descendant::p[contains(@class, "WSJTheme--description-")]'): + for p in container.xpath('descendant::p[contains(@class, "WSJTheme--description--")]'): q = self.tag_to_string(p) if 'Subscriber Content' in q: continue @@ -184,11 +184,10 @@ class WSJ(BasicNewsRecipe): articles.append({'title': title, 'url': url, 'description': desc, 'date': ''}) - if self.test and len(articles) >= self.test[1]: - break - self.log('\tFound article:', title) self.log('\t\t', desc) + if self.test and len(articles) >= self.test[1]: + break return articles @@ -250,7 +249,9 @@ class WSJ(BasicNewsRecipe): self.log('Found section:', title, 'at', url) self.wsj_add_feed(feeds, title, url) if frontpage: - self.wsj_find_wn_articles(feeds, root, CSSSelect) + articles = self.wsj_find_wn_articles(feeds, root, CSSSelect) + if articles: + feeds.append(("What's News", articles)) if self.test and len(feeds) >= self.test[0]: break return feeds