Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Update Forbes

Browse Source 
No longer depends on Qt WebKit
This commit is contained in:
Kovid Goyal 2016-04-25 13:57:24 +05:30
parent 4c66829e6a
commit 83c4a67765
1 changed files with 51 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
recipes/forbes.recipe
View File

@ -1,8 +1,13 @@
from __future__ import (unicode_literals, division, absolute_import, from __future__ import (unicode_literals, division, absolute_import,
print_function) print_function)
from calibre.web.feeds.jsnews import JavascriptRecipe import html5lib, json, re
from calibre.web.feeds.news import BasicNewsRecipe
class Forbes(JavascriptRecipe): def classes(classes):
q = frozenset(classes.split(' '))
return dict(attrs={'class':lambda x:x and frozenset(x.split()).intersection(q)})
class Forbes(BasicNewsRecipe):
title = u'Forbes' title = u'Forbes'
description = 'Business and Financial News' description = 'Business and Financial News'
__author__ = 'Kovid Goyal' __author__ = 'Kovid Goyal'
@ -10,16 +15,20 @@ class Forbes(JavascriptRecipe):
max_articles_per_feed = 20 max_articles_per_feed = 20
language = 'en' language = 'en'
encoding = 'utf-8' encoding = 'utf-8'
recursions = 9
links_from_selectors = ('a.article-pagination-next',)
keep_only_tags = ('h1.article-headline', 'div.article-body-content',)
remove_tags = ('div.vestpocket', 'div.article-print-bar', 'div.article-comment', 'p.previous-page')
no_stylesheets = True no_stylesheets = True
ignore_duplicate_articles = {'title', 'url'}
remove_empty_feeds = True
cover_url = u'http://www.forbes.com/media/current_covers/forbes_120_160.gif' extra_css = '''
div.fb-captioned-img {
font-size: smaller;
margin-top: 1em; margin-bottom: 1em;
}
div.fb-captioned-img img {
display:block;
margin-left: auto; margin-right: auto;
}
'''
feeds = [ feeds = [
(u'Latest', u'http://www.forbes.com/news/index.xml'), (u'Latest', u'http://www.forbes.com/news/index.xml'),
(u'Most Popular', u'http://www.forbes.com/feeds/popstories.xml'), (u'Most Popular', u'http://www.forbes.com/feeds/popstories.xml'),
@ -29,29 +38,37 @@ class Forbes(JavascriptRecipe):
(u'Leadership', u'http://www.forbes.com/leadership/index.xml'), (u'Leadership', u'http://www.forbes.com/leadership/index.xml'),
] ]
def load_complete(self, browser, url, recursion_level): def get_browser(self):
browser.wait_for_element('h1.article-headline') br = BasicNewsRecipe.get_browser(self)
# browser.wait_for_element('div.article-injected-body') br.set_cookie('dailyWelcomeCookie', 'true', '.forbes.com')
return True br.set_cookie('welcomeAd', 'true', '.forbes.com')
return br
def get_publication_data(self, browser): def preprocess_raw_html(self, raw, url):
# return {'index':[('Test', [{'title':'Test Article', 'url':'http://www.forbes.com/sites/stevekeen/2015/08/26/why-china-had-to-crash-part-1/'}])]} # noqa root = html5lib.parse(raw, namespaceHTMLElements=False, treebuilder='lxml')
index = [] for script in root.xpath('//script'):
for feed in self.parse_feeds(): if script.text and script.text.startswith('try {'):
articles = [] idx = script.text.find('fbs_settings.content = {')
for article in feed.articles: if idx > -1:
articles.append({'title':article.title, 'url':article.url, 'description':article.text_summary}) text = script.text.partition('=')[2].lstrip()
if articles: ridx = text.rfind('} catch(err)')
index.append((feed.title, articles)) text = text[:ridx].rstrip().rstrip(';')
return {'index':index} data = json.loads(text)
# from pprint import pformat
# print(pformat(data), file=open('/t/data.py', 'w'))
break
else:
raise ValueError('Failed to find serialized JSON content')
title = data['brandVoiceTitle']
body = data['body']
def cap(m):
val = m.group()
if val.startswith('[/'):
return '</div>'
return '<div class="fb-captioned-img">'
body = re.sub(r'\[/?caption[^\]]*\]', cap, body)
return '''<html><head><title>{0}</title></head><body><h1>{0}</h1><div>{1}</div></body></html>'''.format(title, body)
def preprocess_stage2(self, article, browser, url, recursion_level): # def parse_index(self):
mf = browser.page.mainFrame() # return [('Articles', [{'title':'Test', 'url':
if recursion_level > 0: # 'http://www.forbes.com/sites/hamdiraini/2016/04/25/bazin-seeks-startups-to-accelerate-accorhotels-transformation/'}])]
for sel in ('div.contrib-group', 'h1.article-headline'):
for elem in mf.findAllElements(sel):
if not elem.isNull():
elem.removeFromDocument()
for elem in mf.findAllElements('div.article-pagination'):
if not elem.isNull():
elem.removeFromDocument()