Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-08 18:54:09 -04:00
Code Issues Packages Projects Releases Wiki Activity

Update Foreign Affairs

Browse Source
This commit is contained in:
Kovid Goyal 2013-01-02 08:25:27 +05:30
parent 90f19d6e85
commit b3b37a2029
1 changed files with 55 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
recipes/foreignaffairs.recipe
View File

@ -11,21 +11,21 @@ class ForeignAffairsRecipe(BasicNewsRecipe):
by Chen Wei weichen302@gmx.com, 2012-02-05''' by Chen Wei weichen302@gmx.com, 2012-02-05'''
__license__ = 'GPL v3' __license__ = 'GPL v3'
__author__ = 'kwetal' __author__ = 'Rick Shang, kwetal'
language = 'en' language = 'en'
version = 1.01 version = 1.01
title = u'Foreign Affairs (Subcription or (free) Registration)' title = u'Foreign Affairs (Subcription)'
publisher = u'Council on Foreign Relations' publisher = u'Council on Foreign Relations'
category = u'USA, Foreign Affairs' category = u'USA, Foreign Affairs'
description = u'The leading forum for serious discussion of American foreign policy and international affairs.' description = u'The leading forum for serious discussion of American foreign policy and international affairs.'
no_stylesheets = True no_stylesheets = True
remove_javascript = True remove_javascript = True
needs_subscription = True
INDEX = 'http://www.foreignaffairs.com' INDEX = 'http://www.foreignaffairs.com'
FRONTPAGE = 'http://www.foreignaffairs.com/magazine' FRONTPAGE = 'http://www.foreignaffairs.com/magazine'
INCLUDE_PREMIUM = False
remove_tags = [] remove_tags = []
@ -68,43 +68,57 @@ class ForeignAffairsRecipe(BasicNewsRecipe):
def parse_index(self): def parse_index(self):
answer = [] answer = []
soup = self.index_to_soup(self.FRONTPAGE) soup = self.index_to_soup(self.FRONTPAGE)
sec_start = soup.findAll('div', attrs={'class':'panel-separator'}) #get dates
date = re.split('\s\|\s',self.tag_to_string(soup.head.title.string))[0]
self.timefmt = u' [%s]'%date
sec_start = soup.findAll('div', attrs= {'class':'panel-pane'})
for sec in sec_start: for sec in sec_start:
content = sec.nextSibling articles = []
if content: section = self.tag_to_string(sec.find('h2'))
section = self.tag_to_string(content.find('h2')) if 'Books' in section:
articles = [] reviewsection=sec.find('div', attrs = {'class': 'item-list'})
for subsection in reviewsection.findAll('div'):
tags = [] subsectiontitle=self.tag_to_string(subsection.span.a)
for div in content.findAll('div', attrs = {'class': re.compile(r'view-row\s+views-row-[0-9]+\s+views-row-[odd|even].*')}): subsectionurl=self.INDEX + subsection.span.a['href']
tags.append(div) soup1 = self.index_to_soup(subsectionurl)
for li in content.findAll('li'): for div in soup1.findAll('div', attrs = {'class': 'views-field-title'}):
tags.append(li) if div.find('a') is not None:
originalauthor=self.tag_to_string(div.findNext('div', attrs = {'class':'views-field-field-article-book-nid'}).div.a)
for div in tags: title=subsectiontitle+': '+self.tag_to_string(div.span.a)+' by '+originalauthor
title = url = description = author = None url=self.INDEX+div.span.a['href']
atr=div.findNext('div', attrs = {'class': 'views-field-field-article-display-authors-value'})
if self.INCLUDE_PREMIUM: if atr is not None:
found_premium = False author=self.tag_to_string(atr.span.a)
else: else:
found_premium = div.findAll('span', attrs={'class': author=''
'premium-icon'}) desc=div.findNext('span', attrs = {'class': 'views-field-field-article-summary-value'})
if not found_premium: if desc is not None:
tag = div.find('div', attrs={'class': 'views-field-title'}) description=self.tag_to_string(desc.div.p)
else:
if tag: description=''
a = tag.find('a') articles.append({'title':title, 'date':None, 'url':url, 'description':description, 'author':author})
if a: subsectiontitle=''
title = self.tag_to_string(a) else:
url = self.INDEX + a['href'] for div in sec.findAll('div', attrs = {'class': 'views-field-title'}):
author = self.tag_to_string(div.find('div', attrs = {'class': 'views-field-field-article-display-authors-value'})) if div.find('a') is not None:
tag_summary = div.find('span', attrs = {'class': 'views-field-field-article-summary-value'}) title=self.tag_to_string(div.span.a)
description = self.tag_to_string(tag_summary) url=self.INDEX+div.span.a['href']
articles.append({'title':title, 'date':None, 'url':url, atr=div.findNext('div', attrs = {'class': 'views-field-field-article-display-authors-value'})
'description':description, 'author':author}) if atr is not None:
if articles: author=self.tag_to_string(atr.span.a)
else:
author=''
desc=div.findNext('span', attrs = {'class': 'views-field-field-article-summary-value'})
if desc is not None:
description=self.tag_to_string(desc.div.p)
else:
description=''
articles.append({'title':title, 'date':None, 'url':url, 'description':description, 'author':author})
if articles:
answer.append((section, articles)) answer.append((section, articles))
return answer return answer
@ -115,15 +129,17 @@ class ForeignAffairsRecipe(BasicNewsRecipe):
return soup return soup
needs_subscription = True
def get_browser(self): def get_browser(self):
br = BasicNewsRecipe.get_browser() br = BasicNewsRecipe.get_browser()
if self.username is not None and self.password is not None: if self.username is not None and self.password is not None:
br.open('https://www.foreignaffairs.com/user?destination=home') br.open('https://www.foreignaffairs.com/user?destination=user%3Fop%3Dlo')
br.select_form(nr = 1) br.select_form(nr = 1)
br['name'] = self.username br['name'] = self.username
br['pass'] = self.password br['pass'] = self.password
br.submit() br.submit()
return br return br
def cleanup(self):
self.browser.open('http://www.foreignaffairs.com/logout?destination=user%3Fop=lo')