Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-08-07 09:01:38 -04:00
Code Issues Packages Projects Releases Wiki Activity

Fix Chronicle of Higher Education

Browse Source
This commit is contained in:
Kovid Goyal 2012-09-02 09:35:50 +05:30
parent a907f919da
commit 9360c5833a
1 changed files with 17 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
recipes/chronicle_higher_ed.recipe
View File

@ -13,13 +13,13 @@ class Chronicle(BasicNewsRecipe):
keep_only_tags = [ keep_only_tags = [
dict(name='div', attrs={'class':'article'}), dict(name='div', attrs={'class':'article'}),
] ]
remove_tags = [dict(name='div',attrs={'class':'related module1'})] remove_tags = [dict(name='div',attrs={'class':['related module1','maintitle']}),
dict(name='div', attrs={'id':['section-nav','icon-row']})]
no_javascript = True no_javascript = True
no_stylesheets = True no_stylesheets = True
needs_subscription = True needs_subscription = True
def get_browser(self): def get_browser(self):
br = BasicNewsRecipe.get_browser() br = BasicNewsRecipe.get_browser()
if self.username is not None and self.password is not None: if self.username is not None and self.password is not None:
@ -27,7 +27,7 @@ class Chronicle(BasicNewsRecipe):
br.select_form(nr=1) br.select_form(nr=1)
br['username'] = self.username br['username'] = self.username
br['password'] = self.password br['password'] = self.password
br.submit() br.submit()
return br return br
def parse_index(self): def parse_index(self):
@ -47,33 +47,35 @@ class Chronicle(BasicNewsRecipe):
#Go to the main body #Go to the main body
soup = self.index_to_soup(issueurl) soup = self.index_to_soup(issueurl)
div0 = soup.find ('div', attrs={'id':'article-body'}) div = soup.find ('div', attrs={'id':'article-body'})
feeds = OrderedDict() feeds = OrderedDict()
for div in div0.findAll('div',attrs={'class':'module1'}): section_title = ''
section_title = self.tag_to_string(div.find('h3')) for post in div.findAll('li'):
for post in div.findAll('li',attrs={'class':'sub-promo'}): articles = []
articles = [] a=post.find('a', href=True)
a=post.find('a', href=True) if a is not None:
title=self.tag_to_string(a) title=self.tag_to_string(a)
url="http://chronicle.com"+a['href'].strip() url="http://chronicle.com"+a['href'].strip()
sectiontitle=post.findPrevious('h3')
if sectiontitle is None:
sectiontitle=post.findPrevious('h4')
section_title=self.tag_to_string(sectiontitle)
desc=self.tag_to_string(post.find('p')) desc=self.tag_to_string(post.find('p'))
articles.append({'title':title, 'url':url, 'description':desc, 'date':''}) articles.append({'title':title, 'url':url, 'description':desc, 'date':''})
if articles: if articles:
if section_title not in feeds: if section_title not in feeds:
feeds[section_title] = [] feeds[section_title] = []
feeds[section_title] += articles feeds[section_title] += articles
ans = [(key, val) for key, val in feeds.iteritems()] ans = [(key, val) for key, val in feeds.iteritems()]
return ans return ans
def preprocess_html(self,soup): def preprocess_html(self,soup):
#process all the images #process all the images
for div in soup.findAll('div', attrs={'class':'tableauPlaceholder'}): for div in soup.findAll('div', attrs={'class':'tableauPlaceholder'}):
noscripts=div.find('noscript').a noscripts=div.find('noscript').a
div.replaceWith(noscripts) div.replaceWith(noscripts)
for div0 in soup.findAll('div',text='Powered by Tableau'): for div0 in soup.findAll('div',text='Powered by Tableau'):
div0.extract() div0.extract()
return soup return soup