Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

...

Browse Source
This commit is contained in:
Kovid Goyal 2010-05-08 11:00:24 -06:00
parent 8d4a502620
commit 062f943b4c
1 changed files with 24 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
resources/recipes/discover_magazine.recipe
View File

@ -9,13 +9,12 @@ discovermagazine.com
import re import re
from calibre.web.feeds.news import BasicNewsRecipe from calibre.web.feeds.news import BasicNewsRecipe
from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag
class DiscoverMagazine(BasicNewsRecipe): class DiscoverMagazine(BasicNewsRecipe):
title = u'Discover Magazine' title = u'Discover Magazine'
description = u'Science, Technology and the Future' description = u'Science, Technology and the Future'
__author__ = 'Starson17' __author__ = 'Starson17'
language = 'en' language = 'en'
oldest_article = 33 oldest_article = 33
@ -25,7 +24,7 @@ class DiscoverMagazine(BasicNewsRecipe):
use_embedded_content = False use_embedded_content = False
encoding = 'utf-8' encoding = 'utf-8'
extra_css = '.headline {font-size: x-large;} \n .fact {padding-top: 10pt}' extra_css = '.headline {font-size: x-large;} \n .fact {padding-top: 10pt}'
remove_tags = [ remove_tags = [
dict(name='div', attrs={'id':['searchModule', 'mainMenu', 'tool-box']}), dict(name='div', attrs={'id':['searchModule', 'mainMenu', 'tool-box']}),
dict(name='div', attrs={'id':['footer','teaser','already-subscriber','teaser-suite','related-articles']}), dict(name='div', attrs={'id':['footer','teaser','already-subscriber','teaser-suite','related-articles']}),
@ -33,29 +32,27 @@ class DiscoverMagazine(BasicNewsRecipe):
dict(name='img', attrs={'src':'http://discovermagazine.com/onebyone.gif'})] dict(name='img', attrs={'src':'http://discovermagazine.com/onebyone.gif'})]
remove_tags_after = [dict(name='div', attrs={'class':'listingBar'})] remove_tags_after = [dict(name='div', attrs={'class':'listingBar'})]
def append_page(self, soup, appendtag, position): def append_page(self, soup, appendtag, position):
pager = soup.find('span',attrs={'class':'next'}) pager = soup.find('span',attrs={'class':'next'})
if pager: if pager:
nexturl = pager.a['href'] nexturl = pager.a['href']
print 'nexturl is: ', nexturl
soup2 = self.index_to_soup(nexturl) soup2 = self.index_to_soup(nexturl)
texttag = soup2.find('div', attrs={'class':'articlebody'}) texttag = soup2.find('div', attrs={'class':'articlebody'})
newpos = len(texttag.contents) newpos = len(texttag.contents)
self.append_page(soup2,texttag,newpos) self.append_page(soup2,texttag,newpos)
texttag.extract() texttag.extract()
appendtag.insert(position,texttag) appendtag.insert(position,texttag)
def preprocess_html(self, soup): def preprocess_html(self, soup):
mtag = '<meta http-equiv="Content-Language" content="en-US"/>\n<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>' mtag = '<meta http-equiv="Content-Language" content="en-US"/>\n<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>'
print 'soup1 is: ', soup soup.head.insert(0,mtag)
soup.head.insert(0,mtag)
self.append_page(soup, soup.body, 3) self.append_page(soup, soup.body, 3)
pager = soup.find('div',attrs={'class':'listingBar'}) pager = soup.find('div',attrs={'class':'listingBar'})
if pager: if pager:
pager.extract() pager.extract()
return soup return soup
def postprocess_html(self, soup, first_fetch): def postprocess_html(self, soup, first_fetch):
for tag in soup.findAll(text=re.compile('^This article is a sample')): for tag in soup.findAll(text=re.compile('^This article is a sample')):
tag.parent.extract() tag.parent.extract()
@ -67,24 +64,23 @@ class DiscoverMagazine(BasicNewsRecipe):
tag.extract() tag.extract()
for tag in soup.findAll('br'): for tag in soup.findAll('br'):
tag.extract() tag.extract()
print 'soup2 is: ', soup return soup
return soup
feeds = [ feeds = [
(u'Technology', u'http://discovermagazine.com/topics/technology/rss.xml'), (u'Technology', u'http://discovermagazine.com/topics/technology/rss.xml'),
(u'Health - Medicine', u'http://discovermagazine.com/topics/health-medicine/rss.xml'), (u'Health - Medicine', u'http://discovermagazine.com/topics/health-medicine/rss.xml'),
(u'Mind Brain', u'http://discovermagazine.com/topics/mind-brain/rss.xml'), (u'Mind Brain', u'http://discovermagazine.com/topics/mind-brain/rss.xml'),
(u'Space', u'http://discovermagazine.com/topics/space/rss.xml'), (u'Space', u'http://discovermagazine.com/topics/space/rss.xml'),
(u'Human Origins', u'http://discovermagazine.com/topics/human-origins/rss.xml'), (u'Human Origins', u'http://discovermagazine.com/topics/human-origins/rss.xml'),
(u'Living World', u'http://discovermagazine.com/topics/living-world/rss.xml'), (u'Living World', u'http://discovermagazine.com/topics/living-world/rss.xml'),
(u'Environment', u'http://discovermagazine.com/topics/environment/rss.xml'), (u'Environment', u'http://discovermagazine.com/topics/environment/rss.xml'),
(u'Physics & Math', u'http://discovermagazine.com/topics/physics-math/rss.xml'), (u'Physics & Math', u'http://discovermagazine.com/topics/physics-math/rss.xml'),
(u"20 Things you didn't know about...", u'http://discovermagazine.com/columns/20-things-you-didnt-know/rss.xml'), (u"20 Things you didn't know about...", u'http://discovermagazine.com/columns/20-things-you-didnt-know/rss.xml'),
(u'Fuzzy Math', u'http://discovermagazine.com/columns/fuzzy-math/rss.xml'), (u'Fuzzy Math', u'http://discovermagazine.com/columns/fuzzy-math/rss.xml'),
(u'The Brain', u'http://discovermagazine.com/columns/the-brain/rss.xml'), (u'The Brain', u'http://discovermagazine.com/columns/the-brain/rss.xml'),
(u'What is This', u'http://discovermagazine.com/columns/what-is-this/rss.xml'), (u'What is This', u'http://discovermagazine.com/columns/what-is-this/rss.xml'),
(u'Vital Signs', u'http://discovermagazine.com/columns/vital-signs/rss.xml'), (u'Vital Signs', u'http://discovermagazine.com/columns/vital-signs/rss.xml'),
(u'Think Tech', u'http://discovermagazine.com/columns/think-tech/rss.xml'), (u'Think Tech', u'http://discovermagazine.com/columns/think-tech/rss.xml'),
(u'Future Tech', u'http://discovermagazine.com/columns/future-tech/rss.xml'), (u'Future Tech', u'http://discovermagazine.com/columns/future-tech/rss.xml'),
(u'Discover Interview', u'http://discovermagazine.com/columns/discover-interview/rss.xml'), (u'Discover Interview', u'http://discovermagazine.com/columns/discover-interview/rss.xml'),
] ]