Update Barrons

Gets rid of the use of javascript_login since that no longer works in
modern calibre
This commit is contained in:
Kovid Goyal 2016-05-17 09:41:17 +05:30
parent 3534ef228d
commit 81f8814f75
2 changed files with 56 additions and 22 deletions

View File

@ -1,5 +1,18 @@
#!/usr/bin/env python2
# vim:fileencoding=utf-8
# License: GPLv3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
from __future__ import (unicode_literals, division, absolute_import,
print_function)
import json
from mechanize import Request
from urllib import quote
from calibre.web.feeds.news import BasicNewsRecipe
USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64; rv:45.0) Gecko/20100101 Firefox/45.0'
class Barrons(BasicNewsRecipe):
title = 'Barron\'s'
@ -14,11 +27,9 @@ class Barrons(BasicNewsRecipe):
no_stylesheets = True
match_regexps = ['http://online.barrons.com/.*?html\?mod=.*?|file:.*']
conversion_options = {'linearize_tables': True}
##delay = 1
# Don't grab articles more than 7 days old
oldest_article = 7
use_javascript_to_login = True
requires_version = (0, 9, 16)
keep_only_tags = [dict(attrs={'class':lambda x: x and (x.startswith('sector one column') or x.startswith('sector two column'))})]
@ -29,12 +40,46 @@ class Barrons(BasicNewsRecipe):
dict(name='span', attrs={'data-country-code':True, 'data-ticker-code':True}),
]
def javascript_login(self, br, username, password):
br.visit('http://commerce.barrons.com/auth/login')
f = br.select_form(nr=0)
f['username'] = username
f['password'] = password
br.submit(timeout=120)
def get_browser(self):
# To understand the signin logic read signin.js from
# https://id.barrons.com/access/pages/barrons/us/login_standalone.html?mg=com-barrons
# This is the same login servie as used by WSJ
br = BasicNewsRecipe.get_browser(self, user_agent=USER_AGENT)
url = 'https://id.barrons.com/access/pages/barrons/us/login_standalone.html?mg=com-barrons'
# br.set_debug_http(True)
br.open(url).read()
rurl = 'https://id.barrons.com/auth/submitlogin.json'
rq = Request(rurl, headers={
'Accept': 'application/json, text/javascript, */*; q=0.01',
'Accept-Language': 'en-US,en;q=0.8',
'Content-Type': 'application/json',
'Referer': url,
'X-HTTP-Method-Override': 'POST',
'X-Requested-With': 'XMLHttpRequest',
}, data=json.dumps({
'username': self.username,
'password': self.password,
'realm': 'default',
'savelogin': 'true',
'template': 'default',
'url': quote('http://online.barrons.com'),
}))
r = br.open(rq)
if r.code != 200:
raise ValueError('Failed to login, check username and password')
data = json.loads(r.read())
# from pprint import pprint
# pprint(data)
if data.get('result') != 'success':
raise ValueError(
'Failed to login (XHR failed), check username and password')
br.set_cookie('m', data['username'], '.barrons.com')
raw = br.open(data['url']).read()
# open('/t/raw.html', 'wb').write(raw)
if b'>Logout<' not in raw:
raise ValueError(
'Failed to login (auth URL failed), check username and password')
return br
# Use the print version of a page when available.
def print_version(self, url):
@ -53,26 +98,14 @@ class Barrons(BasicNewsRecipe):
# Comment out the feeds you don't want retrieved.
# Because these feeds are sorted alphabetically when converted to LRF, you may want to number them to put them in the order you desire
def get_feeds(self):
return [
feeds = [
('This Week\'s Magazine', 'http://online.barrons.com/xml/rss/3_7510.xml'),
('Online Exclusives', 'http://online.barrons.com/xml/rss/3_7515.xml'),
('Companies', 'http://online.barrons.com/xml/rss/3_7516.xml'),
('Markets', 'http://online.barrons.com/xml/rss/3_7517.xml'),
('Technology', 'http://online.barrons.com/xml/rss/3_7518.xml'),
('Funds/Q&A', 'http://online.barrons.com/xml/rss/3_7519.xml'),
]
]
def get_article_url(self, article):
return article.get('link', None)
def get_cover_url(self):
cover_url = None
index = 'http://online.barrons.com/home-page'
soup = self.index_to_soup(index)
link_item = soup.find('ul',attrs={'class':'newsItem barronsMag'})
if link_item:
cover_url = link_item.img['src']
return cover_url

View File

@ -93,6 +93,7 @@ class WSJ(BasicNewsRecipe):
def get_browser(self):
# To understand the signin logic read signin.js from
# https://id.wsj.com/access/pages/wsj/us/signin.html
# This is the same login servie as used by Barrons
br = BasicNewsRecipe.get_browser(self, user_agent=USER_AGENT)
# self.wsj_itp_page = open('/t/raw.html').read()
# return br