Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Improved recipe for The Christian Science Monitor

Browse Source
This commit is contained in:
Kovid Goyal 2009-10-08 09:59:30 -06:00
parent 4fd2585773
commit 26d992843d
1 changed files with 61 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
src/calibre/web/feeds/recipes/recipe_chr_mon.py
View File

@ -1,5 +1,6 @@
import re import re
from calibre import strftime
from calibre.web.feeds.news import BasicNewsRecipe from calibre.web.feeds.news import BasicNewsRecipe
class ChristianScienceMonitor(BasicNewsRecipe): class ChristianScienceMonitor(BasicNewsRecipe):
@ -7,43 +8,77 @@ class ChristianScienceMonitor(BasicNewsRecipe):
title = 'Christian Science Monitor' title = 'Christian Science Monitor'
description = 'Providing context and clarity on national and international news, peoples and cultures' description = 'Providing context and clarity on national and international news, peoples and cultures'
max_articles_per_feed = 20 max_articles_per_feed = 20
__author__ = 'Kovid Goyal' __author__ = 'Kovid Goyal and Sujata Raman'
language = 'en' language = 'en'
encoding = 'utf-8'
no_stylesheets = True no_stylesheets = True
use_embedded_content = False use_embedded_content = False
preprocess_regexps = [ (re.compile(i[0], re.IGNORECASE | re.DOTALL), i[1]) for i in
preprocess_regexps = [ (re.compile(i[0], re.IGNORECASE | re.DOTALL), i[1]) for i in
[ [
(r'<body.*?<div id="story"', lambda match : '<body><div id="story"'), (r'<body.*?<div id="story"', lambda match : '<body><div id="story"'),
(r'<div class="pubdate">.*?</div>', lambda m: ''), (r'<div class="pubdate">.*?</div>', lambda m: ''),
(r'Full HTML version of this story which may include photos, graphics, and related links.*</body>', (r'Full HTML version of this story which may include photos, graphics, and related links.*</body>',
lambda match : '</body>'), lambda match : '</body>'),
]] ]]
def parse_index(self): extra_css = '''
soup = self.index_to_soup('http://www.csmonitor.com/textedition') h1{ color:#000000;font-family: Georgia,Times,"Times New Roman",serif; font-size: large}
feeds = [] .sub{ color:#000000;font-family: Georgia,Times,"Times New Roman",serif; font-size: small;}
for tag in soup.findAll(['h2', 'p']): .byline{ font-family:Arial,Helvetica,sans-serif ; color:#999999; font-size: x-small;}
if tag.name == 'h2': .postdate{color:#999999 ; font-family:Arial,Helvetica,sans-serif ; font-size: x-small; }
title = self.tag_to_string(tag) h3{color:#999999 ; font-family:Arial,Helvetica,sans-serif ; font-size: x-small; }
feeds.append((title, [])) .photoCutline{ color:#333333 ; font-family:Arial,Helvetica,sans-serif ; font-size: x-small; }
elif tag.has_key('class') and tag['class'] == 'story' and feeds: .photoCredit{ color:#999999 ; font-family:Arial,Helvetica,sans-serif ; font-size: x-small; }
a = tag.find('a') #story{font-family:Arial,Tahoma,Verdana,Helvetica,sans-serif ; font-size: small; }
if a is not None and a.has_key('href'): #main{font-family:Arial,Tahoma,Verdana,Helvetica,sans-serif ; font-size: small; }
art = { #photo-details{ font-family:Arial,Helvetica,sans-serif ; color:#999999; font-size: x-small;}
'title': self.tag_to_string(a), span.name{color:#205B87;font-family: Georgia,Times,"Times New Roman",serif; font-size: x-small}
'url' : 'http://www.csmonitor.com'+a['href'], p#dateline{color:#444444 ; font-family:Arial,Helvetica,sans-serif ; font-style:italic;}
'date' : '', '''
} feeds = [
a.extract() (u'Top Stories' , u'http://rss.csmonitor.com/feeds/top'),
art['description'] = self.tag_to_string(tag).strip() (u'World' , u'http://rss.csmonitor.com/feeds/world'),
feeds[-1][1].append(art) (u'USA' , u'http://rss.csmonitor.com/feeds/usa'),
return feeds (u'Commentary' , u'http://rss.csmonitor.com/feeds/commentary'),
(u'Money' , u'http://rss.csmonitor.com/feeds/wam'),
(u'Learning' , u'http://rss.csmonitor.com/feeds/learning'),
(u'Living', u'http://rss.csmonitor.com/feeds/living'),
(u'Innovation', u'http://rss.csmonitor.com/feeds/scitech'),
(u'Gardening', u'http://rss.csmonitor.com/feeds/gardening'),
(u'Environment',u'http://rss.csmonitor.com/feeds/environment'),
(u'Arts', u'http://rss.csmonitor.com/feeds/arts'),
(u'Books', u'http://rss.csmonitor.com/feeds/books'),
(u'Home Forum' , u'http://rss.csmonitor.com/feeds/homeforum')
]
keep_only_tags = [
dict(name='div', attrs={'id':['story','main']}),
]
remove_tags = [
dict(name='div', attrs={'id':['story-tools','videoPlayer','storyRelatedBottom','enlarge-photo','photo-paginate']}),
dict(name='div', attrs={'class':[ 'spacer3','divvy spacer7','comment','storyIncludeBottom']}),
dict(name='ul', attrs={'class':[ 'centerliststories']}) ,
dict(name='form', attrs={'id':[ 'commentform']}) ,
]
def find_articles(self, section):
ans = []
for x in section.findAll('head4'):
title = ' '.join(x.findAll(text=True)).strip()
a = x.find('a')
if not a: continue
href = a['href']
ans.append({'title':title, 'url':href, 'description':'', 'date': strftime('%a, %d %b')})
#for x in ans:
# x['url'] += '/output/print'
return ans
def postprocess_html(self, soup, first_fetch): def postprocess_html(self, soup, first_fetch):
html = soup.find('html') html = soup.find('html')
if html is None: if html is None: