mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-07-09 03:04:10 -04:00
Updated recipes: Psychology Today, WSJ
This commit is contained in:
Kovid Goyal
parent
caa451213e
commit
dc97a0c2e1
@ -1,39 +1,44 @@
|
||||
from calibre.web.feeds.news import BasicNewsRecipe
|
||||
from calibre.ebooks.BeautifulSoup import BeautifulSoup
|
||||
|
||||
class PsychologyToday(BasicNewsRecipe):
|
||||
from calibre.ptempfile import PersistentTemporaryFile
|
||||
from calibre.webfeeds.news import BasicNewsRecipe
|
||||
|
||||
class AdvancedUserRecipe1275708473(BasicNewsRecipe):
|
||||
title = u'Psychology Today'
|
||||
_author__ = 'rty'
|
||||
publisher = u'www.psychologytoday.com'
|
||||
category = u'Psychology'
|
||||
max_articles_per_feed = 100
|
||||
remove_javascript = True
|
||||
use_embedded_content = False
|
||||
no_stylesheets = True
|
||||
language = 'en'
|
||||
__author__ = 'Krittika Goyal'
|
||||
oldest_article = 1 #days
|
||||
max_articles_per_feed = 25
|
||||
#encoding = 'latin1'
|
||||
|
||||
remove_stylesheets = True
|
||||
#remove_tags_before = dict(name='h1', attrs={'class':'heading'})
|
||||
#remove_tags_after = dict(name='td', attrs={'class':'newptool1'})
|
||||
temp_files = []
|
||||
articles_are_obfuscated = True
|
||||
remove_tags = [
|
||||
dict(name='iframe'),
|
||||
dict(name='div', attrs={'class':['pt-box-title', 'pt-box-content', 'blog-entry-footer', 'item-list', 'article-sub-meta']}),
|
||||
dict(name='div', attrs={'id':['block-td_search_160', 'block-cam_search_160']}),
|
||||
#dict(name='ul', attrs={'class':'article-tools'}),
|
||||
#dict(name='ul', attrs={'class':'articleTools'}),
|
||||
dict(name='div', attrs={'class':['print-source_url','field-items','print-footer']}),
|
||||
dict(name='span', attrs={'class':'print-footnote'}),
|
||||
]
|
||||
remove_tags_before = dict(name='h1', attrs={'class':'print-title'})
|
||||
remove_tags_after = dict(name='div', attrs={'class':['field-items','print-footer']})
|
||||
|
||||
feeds = [
|
||||
('PSY TODAY',
|
||||
'http://www.psychologytoday.com/articles/index.rss'),
|
||||
]
|
||||
feeds = [(u'Contents', u'http://www.psychologytoday.com/articles/index.rss')]
|
||||
|
||||
def preprocess_html(self, soup):
|
||||
story = soup.find(name='div', attrs={'id':'contentColumn'})
|
||||
#td = heading.findParent(name='td')
|
||||
#td.extract()
|
||||
soup = BeautifulSoup('<html><head><title>t</title></head><body></body></html>')
|
||||
body = soup.find(name='body')
|
||||
body.insert(0, story)
|
||||
for x in soup.findAll(name='p', text=lambda x:x and '-->' in x):
|
||||
p = x.findParent('p')
|
||||
if p is not None:
|
||||
p.extract()
|
||||
return soup
|
||||
def get_article_url(self, article):
|
||||
return article.get('link', None)
|
||||
|
||||
def get_obfuscated_article(self, url):
|
||||
br = self.get_browser()
|
||||
br.open(url)
|
||||
response = br.follow_link(url_regex = r'/print/[0-9]+', nr = 0)
|
||||
html = response.read()
|
||||
self.temp_files.append(PersistentTemporaryFile('_fa.html'))
|
||||
self.temp_files[-1].write(html)
|
||||
self.temp_files[-1].close()
|
||||
return self.temp_files[-1].name
|
||||
|
||||
def get_cover_url(self):
|
||||
index = 'http://www.psychologytoday.com/magazine/'
|
||||
soup = self.index_to_soup(index)
|
||||
for image in soup.findAll('img',{ "class" : "imagefield imagefield-field_magazine_cover" }):
|
||||
return image['src'] + '.jpg'
|
||||
return None
|
||||
|
||||
@ -4,13 +4,14 @@ __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
|
||||
__docformat__ = 'restructuredtext en'
|
||||
|
||||
from calibre.web.feeds.news import BasicNewsRecipe
|
||||
import copy
|
||||
|
||||
# http://online.wsj.com/page/us_in_todays_paper.html
|
||||
|
||||
class WallStreetJournal(BasicNewsRecipe):
|
||||
|
||||
title = 'The Wall Street Journal (US)'
|
||||
__author__ = 'Kovid Goyal and Sujata Raman'
|
||||
title = 'The Wall Street Journal'
|
||||
__author__ = 'Kovid Goyal, Sujata Raman, and Joshua Oster-Morris'
|
||||
description = 'News and current affairs'
|
||||
needs_subscription = True
|
||||
language = 'en'
|
||||
@ -67,6 +68,16 @@ class WallStreetJournal(BasicNewsRecipe):
|
||||
def wsj_get_index(self):
|
||||
return self.index_to_soup('http://online.wsj.com/itp')
|
||||
|
||||
def wsj_add_feed(self,feeds,title,url):
|
||||
self.log('Found section:', title)
|
||||
if url.endswith('whatsnews'):
|
||||
articles = self.wsj_find_wn_articles(url)
|
||||
else:
|
||||
articles = self.wsj_find_articles(url)
|
||||
if articles:
|
||||
feeds.append((title, articles))
|
||||
return feeds
|
||||
|
||||
def parse_index(self):
|
||||
soup = self.wsj_get_index()
|
||||
|
||||
@ -82,25 +93,62 @@ class WallStreetJournal(BasicNewsRecipe):
|
||||
div = soup.find('div', attrs={'class':'itpHeader'})
|
||||
div = div.find('ul', attrs={'class':'tab'})
|
||||
for a in div.findAll('a', href=lambda x: x and '/itp/' in x):
|
||||
pageone = a['href'].endswith('pageone')
|
||||
if pageone:
|
||||
title = 'Front Section'
|
||||
url = 'http://online.wsj.com' + a['href']
|
||||
feeds = self.wsj_add_feed(feeds,title,url)
|
||||
title = 'What''s News'
|
||||
url = url.replace('pageone','whatsnews')
|
||||
feeds = self.wsj_add_feed(feeds,title,url)
|
||||
else:
|
||||
title = self.tag_to_string(a)
|
||||
url = 'http://online.wsj.com' + a['href']
|
||||
self.log('Found section:', title)
|
||||
articles = self.wsj_find_articles(url)
|
||||
if articles:
|
||||
feeds.append((title, articles))
|
||||
|
||||
feeds = self.wsj_add_feed(feeds,title,url)
|
||||
return feeds
|
||||
|
||||
def wsj_find_wn_articles(self, url):
|
||||
soup = self.index_to_soup(url)
|
||||
articles = []
|
||||
|
||||
whats_news = soup.find('div', attrs={'class':lambda x: x and 'whatsNews-simple' in x})
|
||||
if whats_news is not None:
|
||||
for a in whats_news.findAll('a', href=lambda x: x and '/article/' in x):
|
||||
container = a.findParent(['p'])
|
||||
meta = a.find(attrs={'class':'meta_sectionName'})
|
||||
if meta is not None:
|
||||
meta.extract()
|
||||
title = self.tag_to_string(a).strip()
|
||||
url = a['href']
|
||||
desc = ''
|
||||
if container is not None:
|
||||
desc = self.tag_to_string(container)
|
||||
|
||||
articles.append({'title':title, 'url':url,
|
||||
'description':desc, 'date':''})
|
||||
|
||||
self.log('\tFound WN article:', title)
|
||||
|
||||
return articles
|
||||
|
||||
def wsj_find_articles(self, url):
|
||||
soup = self.index_to_soup(url)
|
||||
|
||||
whats_news = soup.find('div', attrs={'class':lambda x: x and
|
||||
'whatsNews-simple' in x})
|
||||
whats_news = soup.find('div', attrs={'class':lambda x: x and 'whatsNews-simple' in x})
|
||||
if whats_news is not None:
|
||||
whats_news.extract()
|
||||
|
||||
articles = []
|
||||
|
||||
flavorarea = soup.find('div', attrs={'class':lambda x: x and 'ahed' in x})
|
||||
if flavorarea is not None:
|
||||
flavorstory = flavorarea.find('a', href=lambda x: x and x.startswith('/article'))
|
||||
if flavorstory is not None:
|
||||
flavorstory['class'] = 'mjLinkItem'
|
||||
metapage = soup.find('span', attrs={'class':lambda x: x and 'meta_sectionName' in x})
|
||||
if metapage is not None:
|
||||
flavorstory.append( copy.copy(metapage) ) #metapage should always be A1 because that should be first on the page
|
||||
|
||||
for a in soup.findAll('a', attrs={'class':'mjLinkItem'}, href=True):
|
||||
container = a.findParent(['li', 'div'])
|
||||
meta = a.find(attrs={'class':'meta_sectionName'})
|
||||
@ -118,26 +166,9 @@ class WallStreetJournal(BasicNewsRecipe):
|
||||
|
||||
self.log('\tFound article:', title)
|
||||
|
||||
'''
|
||||
# Find related articles
|
||||
a.extract()
|
||||
for a in container.findAll('a', href=lambda x: x and '/article/'
|
||||
in x and 'articleTabs' not in x):
|
||||
url = a['href']
|
||||
if not url.startswith('http:'):
|
||||
url = 'http://online.wsj.com'+url
|
||||
title = self.tag_to_string(a).strip()
|
||||
if not title or title.startswith('['): continue
|
||||
if title:
|
||||
articles.append({'title':self.tag_to_string(a),
|
||||
'url':url, 'description':'', 'date':''})
|
||||
self.log('\t\tFound related:', title)
|
||||
'''
|
||||
|
||||
return articles
|
||||
|
||||
|
||||
def cleanup(self):
|
||||
self.browser.open('http://online.wsj.com/logout?url=http://online.wsj.com')
|
||||
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user