Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Fixed recipe to follow recent Nikkei site renewal.

Browse Source
This commit is contained in:
kentan 2017-12-24 00:52:44 -07:00
parent d2efdc648a
commit aebcda11a5
1 changed files with 13 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
recipes/nikkei_news.recipe
View File

@ -39,13 +39,6 @@ class NikkeiNet_paper_subscription(BasicNewsRecipe):
def get_browser(self): def get_browser(self):
br = BasicNewsRecipe.get_browser(self) br = BasicNewsRecipe.get_browser(self)
# pp.pprint(self.parse_index())
# exit(1)
# br.set_debug_http(True)
# br.set_debug_redirects(True)
# br.set_debug_responses(True)
if self.username is not None and self.password is not None: if self.username is not None and self.password is not None:
print "-------------------------open top page-------------------------------------" print "-------------------------open top page-------------------------------------"
br.open('http://www.nikkei.com/') br.open('http://www.nikkei.com/')
@ -53,11 +46,12 @@ class NikkeiNet_paper_subscription(BasicNewsRecipe):
try: try:
url = list(br.links( url = list(br.links(
url_regex="www.nikkei.com/etc/accounts/login"))[0].url url_regex="www.nikkei.com/etc/accounts/login"))[0].url
except IndexError:
print "Found IndexError"
url = 'http://www.nikkei.com/etc/accounts/login?dps=3&pageflag=top&url=http%3A%2F%2Fwww.nikkei.com%2F'
except StopIteration: except StopIteration:
url = 'http://www.nikkei.com/etc/accounts/login?dps=3&pageflag=top&url=http%3A%2F%2Fwww.nikkei.com%2F' url = 'http://www.nikkei.com/etc/accounts/login?dps=3&pageflag=top&url=http%3A%2F%2Fwww.nikkei.com%2F'
br.open(url) # br.follow_link(link) br.open(url)
# response = br.response()
# print response.get_data()
print "-------------------------JS redirect(send autoPostForm)--------------------" print "-------------------------JS redirect(send autoPostForm)--------------------"
br.select_form(name='autoPostForm') br.select_form(name='autoPostForm')
br.submit() br.submit()
@ -67,14 +61,10 @@ class NikkeiNet_paper_subscription(BasicNewsRecipe):
br['LA7010Form01:LA7010Email'] = self.username br['LA7010Form01:LA7010Email'] = self.username
br['LA7010Form01:LA7010Password'] = self.password br['LA7010Form01:LA7010Password'] = self.password
br.submit(id='LA7010Form01:submitBtn') br.submit(id='LA7010Form01:submitBtn')
# response = br.response()
print "-------------------------JS redirect---------------------------------------" print "-------------------------JS redirect---------------------------------------"
br.select_form(nr=0) br.select_form(nr=0)
br.submit() br.submit()
# br.set_debug_http(False)
# br.set_debug_redirects(False)
# br.set_debug_responses(False)
return br return br
def cleanup(self): def cleanup(self):
@ -85,25 +75,17 @@ class NikkeiNet_paper_subscription(BasicNewsRecipe):
print "-------------------------get index of paper--------------------------------" print "-------------------------get index of paper--------------------------------"
result = [] result = []
soup = self.index_to_soup('http://www.nikkei.com/paper/') soup = self.index_to_soup('http://www.nikkei.com/paper/')
# soup = self.index_to_soup(self.test_data()) sections = soup.findAll(attrs={'class': re.compile(".*cmn-article_title.*")})
sections = soup.findAll(
'div', 'cmn-section kn-special JSID_baseSection')
if len(sections) == 0:
sections = soup.findAll('div', 'cmn-section kn-special')
for sect in sections: for sect in sections:
sect_title = sect.find('h3', 'cmnc-title').string sect_title = sect.find(attrs={'class' : re.compile(".*cmnc-((large)|(middle)|(small)).*")})
if sect_title is None: continue
sect_title = sect_title.contents[0]
sect_result = [] sect_result = []
for elem in sect.findAll(attrs={'class': ['cmn-article_title']}): url = sect.a['href']
if elem.span.a is None or elem.span.a['href'].startswith('javascript'):
continue
url = 'http://www.nikkei.com' + elem.span.a['href']
# print version.
url = re.sub("/article/", "/print-article/", url) url = re.sub("/article/", "/print-article/", url)
span = elem.span.a.span url = 'http://www.nikkei.com' + url
if ((span is not None) and (len(span.contents) > 1)): sect_result.append(dict(title=sect_title, url=url, date='',description='', content=''))
title = span.contents[1].string
sect_result.append(dict(title=title, url=url, date='',
description='', content=''))
result.append([sect_title, sect_result]) result.append([sect_title, sect_result])
return result return result