Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-08 10:44:09 -04:00
Code Issues Packages Projects Releases Wiki Activity

Update Harvard Business Review

Browse Source
This commit is contained in:
Kovid Goyal 2015-04-03 15:04:46 +05:30
parent 0cb35c0495
commit 1cd38f1fb2
1 changed files with 43 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

118
recipes/hbr.recipe
View File

@ -1,4 +1,5 @@
from calibre.web.feeds.news import BasicNewsRecipe from calibre.web.feeds.news import BasicNewsRecipe
from css_selectors import Select
class HBR(BasicNewsRecipe): class HBR(BasicNewsRecipe):
@ -13,88 +14,55 @@ class HBR(BasicNewsRecipe):
LOGIN_URL = 'https://hbr.org/login?request_url=/' LOGIN_URL = 'https://hbr.org/login?request_url=/'
LOGOUT_URL = 'https://hbr.org/logout?request_url=/' LOGOUT_URL = 'https://hbr.org/logout?request_url=/'
keep_only_tags = [dict(name='div', id='pageContainer')] keep_only_tags = [
remove_tags = [dict(id=['mastheadContainer', 'magazineHeadline', dict(attrs={'class':['article-hed', 'byline']}),
'articleToolbarTopRD', 'pageRightSubColumn', 'pageRightColumn', dict(attrs={'class':lambda x: x and 'article' in x.split()}),
'todayOnHBRListWidget', 'mostWidget', 'keepUpWithHBR', ]
'mailingListTout', 'partnerCenter', 'pageFooter', remove_tags = [
'superNavHeadContainer', 'hbrDisqus', 'article-toolbox', dict(name='personalization-placement'),
'articleToolbarTop', 'articleToolbarBottom', 'articleToolbarRD']), ]
dict(name='iframe')]
extra_css = '''
a {font-family:Georgia,"Times New Roman",Times,serif; font-style:italic; color:#000000; }
.article{font-family:Georgia,"Times New Roman",Times,serif; font-size: xx-small;}
h2{font-family:Georgia,"Times New Roman",Times,serif; font-weight:bold; font-size:large; }
h4{font-family:Georgia,"Times New Roman",Times,serif; font-weight:bold; font-size:small; }
#articleAuthors{font-family:Georgia,"Times New Roman",Times,serif; font-style:italic; color:#000000;font-size:x-small;}
#summaryText{font-family:Georgia,"Times New Roman",Times,serif; font-weight:bold; font-size:x-small;}
'''
use_javascript_to_login = True use_javascript_to_login = True
def javascript_login(self, br, username, password): def javascript_login(self, br, username, password):
from calibre.web.jsbrowser.browser import Timeout br.visit('https://hbr.org/sign-in')
try: br.run_for_a_time(15)
br.visit('https://hbr.org/login?request_url=/', timeout=20) f = br.select_form('sign-in form')
except Timeout: f['login-email'] = username
pass f['login-password'] = password
br.click('#form-wrapper h3[tabindex="0"]', wait_for_load=False) br.submit('[js-target="submit-sign-in"]', wait_for_load=False)
f = br.select_form('#login-form') br.run_for_a_time(15)
f['username'] = username
f['password'] = password
br.submit(wait_for_load=False)
br.run_for_a_time(30)
def map_url(self, url): def hbr_parse_toc(self, url):
if url.endswith('/ar/1'): root = self.index_to_soup(url, as_tree=True)
return url[:-1]+'pr' select = Select(root)
section = 'Unknown'
def hbr_parse_toc(self, soup):
feeds = []
current_section = None
articles = [] articles = []
for x in soup.find(id='issueFeaturesContent').findAll(['li', 'h4']): feeds = []
if x.name == 'h4': toc = next(select('stream-content[data-stream-name="table-of-contents"] ul'))
if x.get('class', None) == 'basic': for x in toc.xpath('descendant::*[local-name()="h4" or local-name()="stream-item"]'):
continue if 'h4' in x.tag:
if current_section is not None and articles: if articles:
feeds.append((current_section, articles)) feeds.append((section, articles))
current_section = self.tag_to_string(x).capitalize() section = self.tag_to_string(x)
articles = [] articles = []
self.log('\tFound section:', current_section) self.log('Found section:', section)
else: else:
a = x.find('a', href=True) title, url = x.get('data-title'), x.get('data-url')
if a is None: desc = ''.join(c.tail or '' for c in x).strip()
continue authors = x.get('data-authors')
title = self.tag_to_string(a) if authors:
url = a['href'] desc = 'by ' + authors + ': ' + desc
if '/ar/' not in url: self.log('\tFound article:', title, url, desc)
continue articles.append({'title':title, 'url':'https://hbr.org' + url, 'description':desc})
if url.startswith('/'): if articles:
url = 'http://hbr.org' + url feeds.append((section, articles))
url = self.map_url(url)
p = x.find('p', attrs={'class':'author'})
desc = ''
if p is not None:
desc = self.tag_to_string(p)
self.log('\t\tFound article:', title)
self.log('\t\t\t', url)
self.log('\t\t\t', desc)
articles.append({'title':title, 'url':url, 'description':desc,
'date':''})
if current_section is not None and articles:
feeds.append((current_section, articles))
return feeds return feeds
def parse_index(self): def parse_index(self):
soup0 = self.index_to_soup('http://hbr.org/magazine') soup = self.index_to_soup('http://hbr.org/magazine')
datencover = soup0.find('ul', attrs={'id':'magazineArchiveCarousel'}).findAll('li')[-1] fig = soup.find('figure', attrs={'class': lambda x: x and 'magazine-cover' in x.split()})
# find date & cover url = 'https://hbr.org' + fig.findParent('a', href=True)['href']
self.cover_url=datencover.img['src'] img = fig.find('img')
dates=self.tag_to_string(datencover.img['alt']) self.cover_url = 'https://hbr.org' + img['src']
self.timefmt = u' [%s]'%dates self.timefmt = img['alt']
soup = self.index_to_soup(soup0.find('div', attrs={'class':'magazine_page'}).a['href']) return self.hbr_parse_toc(url)
feeds = self.hbr_parse_toc(soup)
return feeds