Improved recipe for Discover Magazine

resources/recipes/discover_magazine.recipe

@@ -7,13 +7,15 @@ __docformat__ = 'restructuredtext en'
 discovermagazine.com
 '''
 
+import re
 from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag
 
 class DiscoverMagazine(BasicNewsRecipe):
 
     title = u'Discover Magazine'
     description = u'Science, Technology and the Future' 
-    __author__ = 'Mike Diaz' 
+    __author__ = 'Starson17' 
     language = 'en'
 
     oldest_article = 33
@@ -24,10 +26,49 @@ class DiscoverMagazine(BasicNewsRecipe):
     encoding = 'utf-8'
     extra_css = '.headline {font-size: x-large;} \n .fact {padding-top: 10pt}'
     
-    remove_tags = [dict(name='div', attrs={'id':['searchModule', 'mainMenu', 'tool-box']}),
+    remove_tags = [
+                   dict(name='div', attrs={'id':['searchModule', 'mainMenu', 'tool-box']}),
+                   dict(name='div', attrs={'id':['footer','teaser','already-subscriber','teaser-suite','related-articles']}),
+                   dict(name='div', attrs={'class':['column']}),
                    dict(name='img', attrs={'src':'http://discovermagazine.com/onebyone.gif'})]
 
-    remove_tags_after = [dict(name='div', attrs={'class':'articlebody'})]
+    remove_tags_after = [dict(name='div', attrs={'class':'listingBar'})]
+   
+    def append_page(self, soup, appendtag, position):
+        pager = soup.find('span',attrs={'class':'next'})
+        if pager:
+           nexturl = pager.a['href']
+           print 'nexturl is: ', nexturl
+           soup2 = self.index_to_soup(nexturl)
+           texttag = soup2.find('div', attrs={'class':'articlebody'})
+           newpos = len(texttag.contents)          
+           self.append_page(soup2,texttag,newpos)
+           texttag.extract()
+           appendtag.insert(position,texttag)
+    
+    def preprocess_html(self, soup):
+        mtag = '<meta http-equiv="Content-Language" content="en-US"/>\n<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>'
+        print 'soup1 is: ', soup
+        soup.head.insert(0,mtag)    
+        self.append_page(soup, soup.body, 3)
+        pager = soup.find('div',attrs={'class':'listingBar'})
+        if pager:
+           pager.extract()        
+        return soup
+        
+    def postprocess_html(self, soup, first_fetch):
+        for tag in soup.findAll(text=re.compile('^This article is a sample')):
+            tag.parent.extract()
+        for tag in soup.findAll(['table', 'tr', 'td']):
+            tag.name = 'div'
+        for tag in soup.findAll('div', attrs={'class':'discreet advert'}):
+            tag.extract()
+        for tag in soup.findAll('hr', attrs={'size':'1'}):
+            tag.extract()
+        for tag in soup.findAll('br'):
+            tag.extract()
+        print 'soup2 is: ', soup    
+        return soup        
  
     feeds = [
              (u'Technology', u'http://discovermagazine.com/topics/technology/rss.xml'), 
@@ -38,10 +79,12 @@ class DiscoverMagazine(BasicNewsRecipe):
              (u'Living World', u'http://discovermagazine.com/topics/living-world/rss.xml'), 
              (u'Environment', u'http://discovermagazine.com/topics/environment/rss.xml'), 
              (u'Physics & Math', u'http://discovermagazine.com/topics/physics-math/rss.xml'), 
-             (u'Vital Signs', u'http://discovermagazine.com/columns/vital-signs/rss.xml'), 
              (u"20 Things you didn't know about...", u'http://discovermagazine.com/columns/20-things-you-didnt-know/rss.xml'), 
              (u'Fuzzy Math', u'http://discovermagazine.com/columns/fuzzy-math/rss.xml'), 
              (u'The Brain', u'http://discovermagazine.com/columns/the-brain/rss.xml'), 
-             (u'Stupid Science Word of the Month', u'http://discovermagazine.com/columns/stupid-science-word-of-the-month/rss.xml'), 
+             (u'What is This', u'http://discovermagazine.com/columns/what-is-this/rss.xml'),
-             (u'Science Not Fiction', u'http://blogs.discovermagazine.com/sciencenotfiction/wp-rss.php')
+             (u'Vital Signs', u'http://discovermagazine.com/columns/vital-signs/rss.xml'), 
+             (u'Think Tech', u'http://discovermagazine.com/columns/think-tech/rss.xml'),
+             (u'Future Tech', u'http://discovermagazine.com/columns/future-tech/rss.xml'),
+             (u'Discover Interview', u'http://discovermagazine.com/columns/discover-interview/rss.xml'),
             ]