#!/usr/bin/env python __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' __docformat__ = 'restructuredtext en' from calibre.web.feeds.news import BasicNewsRecipe import copy, re # http://online.wsj.com/page/us_in_todays_paper.html class WallStreetJournal(BasicNewsRecipe): title = 'The Wall Street Journal' __author__ = 'Kovid Goyal and Joshua Oster-Morris' description = 'News and current affairs' needs_subscription = True language = 'en' compress_news_images = True compress_news_images_auto_size = 5 max_articles_per_feed = 1000 timefmt = ' [%a, %b %d, %Y]' no_stylesheets = True ignore_duplicate_articles = {'url'} remove_attributes = ['style', 'data-scrim'] keep_only_tags = [ dict(name='h1'), dict(name='h2', attrs={'class':['subhead', 'subHed deck']}), dict(name='span', itemprop='author', rel='author'), dict(name='article', id=['article-contents', 'articleBody']), dict(name='div', id='article_story_body'), dict(name='div', attrs={'class':'snippet-ad-login'}), ] remove_tags = [ dict(attrs={'class':['insetButton', 'insettipBox', 'author-info', 'media-object-video']}), dict(attrs={'class':lambda x: x and 'article_tools' in x.split()}), dict(name='span', attrs={'data-country-code':True, 'data-ticker-code':True}), ] preprocess_regexps = [ (re.compile(r'', re.DOTALL), lambda m: ''), (re.compile(r'.+?', re.DOTALL), lambda m:''), ] use_javascript_to_login = True def javascript_login(self, br, username, password): br.visit('https://id.wsj.com/access/pages/wsj/us/login_standalone.html?mg=com-wsj', timeout=120) f = br.select_form(nr=0) f['username'] = username f['password'] = password br.submit(timeout=120) def populate_article_metadata(self, article, soup, first): if first and hasattr(self, 'add_toc_thumbnail'): picdiv = soup.find('img', src=True) if picdiv is not None: self.add_toc_thumbnail(article,picdiv['src']) def preprocess_html(self, soup): # Remove thumbnail for zoomable images for div in soup.findAll('div', attrs={'class':lambda x: x and 'insetZoomTargetBox' in x.split()}): img = div.find('img') if img is not None: img.extract() # Use large images for img in soup.findAll('img', attrs={'data-enlarge':True}): img['src'] = img['data-enlarge'] return soup def wsj_get_index(self): return self.index_to_soup('http://online.wsj.com/itp') def wsj_add_feed(self,feeds,title,url): self.log('Found section:', title) try: if url.endswith('whatsnews'): articles = self.wsj_find_wn_articles(url) else: articles = self.wsj_find_articles(url) except: articles = [] if articles: feeds.append((title, articles)) return feeds def abs_wsj_url(self, href): if not href.startswith('http'): href = 'http://online.wsj.com' + href return href def parse_index(self): soup = self.wsj_get_index() date = soup.find('span', attrs={'class':'date-date'}) if date is not None: self.timefmt = ' [%s]'%self.tag_to_string(date) cov = soup.find('div', attrs={'class':lambda x: x and 'itpSectionHeaderPdf' in x.split()}) if cov is not None: a = cov.find('a', href=True) if a is not None: self.cover_url = a['href'] feeds = [] div = soup.find('div', attrs={'class':'itpHeader'}) div = div.find('ul', attrs={'class':'tab'}) for a in div.findAll('a', href=lambda x: x and '/itp/' in x): pageone = a['href'].endswith('pageone') if pageone: title = 'Front Section' url = self.abs_wsj_url(a['href']) feeds = self.wsj_add_feed(feeds,title,url) title = "What's News" url = url.replace('pageone','whatsnews') feeds = self.wsj_add_feed(feeds,title,url) else: title = self.tag_to_string(a) url = self.abs_wsj_url(a['href']) feeds = self.wsj_add_feed(feeds,title,url) for li in soup.findAll('li', attrs={'class':'ahed_listitem'}): h2 = li.find('h2') if h2 is None: continue a = h2.find('a', href=True) if a is None: continue url = a['href'] title = self.tag_to_string(a) p = h2.findNextSibling('p') if p is not None: desc = self.tag_to_string(p) else: desc = '' if feeds: feeds[0][1].append({'title':title, 'url':url, 'description':desc, 'date':''}) return feeds def wsj_find_wn_articles(self, url): soup = self.index_to_soup(url) articles = [] whats_news = soup.find('div', attrs={'class':lambda x: x and 'whatsNews-simple' in x}) if whats_news is not None: for a in whats_news.findAll('a', href=lambda x: x and '/article/' in x): container = a.findParent(['p']) meta = a.find(attrs={'class':'meta_sectionName'}) if meta is not None: meta.extract() title = self.tag_to_string(a).strip() url = a['href'] desc = '' if container is not None: desc = self.tag_to_string(container) articles.append({'title':title, 'url':url, 'description':desc, 'date':''}) self.log('\tFound WN article:', title) self.log('\t\t', desc) return articles def wsj_find_articles(self, url): soup = self.index_to_soup(url) whats_news = soup.find('div', attrs={'class':lambda x: x and 'whatsNews-simple' in x}) if whats_news is not None: whats_news.extract() articles = [] flavorarea = soup.find('div', attrs={'class':lambda x: x and 'ahed' in x}) if flavorarea is not None: flavorstory = flavorarea.find('a', href=lambda x: x and x.startswith('/article')) if flavorstory is not None: flavorstory['class'] = 'mjLinkItem' metapage = soup.find('span', attrs={'class':lambda x: x and 'meta_sectionName' in x}) if metapage is not None: flavorstory.append(copy.copy(metapage)) # metapage should always be A1 because that should be first on the page for a in soup.findAll('a', attrs={'class':'mjLinkItem'}, href=True): container = a.findParent(['li', 'div']) meta = a.find(attrs={'class':'meta_sectionName'}) if meta is not None: meta.extract() meta = self.tag_to_string(meta).strip() if meta: title = self.tag_to_string(a).strip() + ' [%s]'%meta else: title = self.tag_to_string(a).strip() url = self.abs_wsj_url(a['href']) desc = '' for p in container.findAll('p'): desc = self.tag_to_string(p) if 'Subscriber Content' not in desc: break articles.append({'title':title, 'url':url, 'description':desc, 'date':''}) self.log('\tFound article:', title) self.log('\t\t', desc) return articles def cleanup(self): self.browser.open('http://online.wsj.com/logout?url=http://online.wsj.com')