Improved Independent

recipes/independent.recipe

@@ -1,8 +1,9 @@
 # adapted from old recipe by Darko Miletic <darko.miletic at gmail.com>
 
-import re
+import string, re
+from calibre import strftime
 from calibre.web.feeds.recipes import BasicNewsRecipe
-from calibre.ebooks.BeautifulSoup import Tag, NavigableString
+from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag, NavigableString
 
 
 class TheIndependentNew(BasicNewsRecipe):
@@ -82,15 +83,16 @@ class TheIndependentNew(BasicNewsRecipe):
     
                 img['src'] = item['href']
     
-                #insert heading
+                #insert caption if available
-                tag = Tag(soup,'h3')
+                if img['title'] is not None and (len(img['title']) > 1):
-                text = NavigableString('Caption: ' + img['title'])
+                    tag = Tag(soup,'h3')
-                tag.insert(0,text)
+                    text = NavigableString(img['title'])
+                    tag.insert(0,text)
                 
-                #picture before text
+                    #picture before text
-                img.extract()
+                    img.extract()
-                item.insert(0,img)
+                    item.insert(0,img)
-                item.insert(1,tag)
+                    item.insert(1,tag)
     
                 # remove link
                 item.name = "div"
@@ -99,16 +101,41 @@ class TheIndependentNew(BasicNewsRecipe):
                 
                 
         #remove empty subtitles
+        """
+        currently the subtitle is located in first paragraph after
+        sibling <h3 class="subtitle"> tag. This may be 'fixed' at
+        some point. 
+        """       
         subtitle = soup.find('h3',attrs={'class' : 'subtitle'})
-        subtitleText = subtitle.findNext('p')
+        if subtitle is not None:
-        if subtitleText is not None:
+	        subtitleText = subtitle.findNext('p')
-            if len(subtitleText.contents[0]) <= 1 :
+	        if subtitleText is not None:
-                subtitleText.extract()
+	            if len(subtitleText.contents[0]) <= 1 : 
-                subtitle.extract()
+	                subtitleText.extract()
+	                subtitle.extract()
                 
         
         return soup
         
+        
+        
+    def postprocess_html(self,soup, first_fetch):
+        #find broken images and remove captions
+        for item in soup.findAll('div', attrs={'class' : 'byline'}):
+            img = item.findNext('img')
+            if img is not None and img['src'] is not None:
+                # broken images still point to remote url       
+                pattern = re.compile('http://www.independent.co.uk.*')    
+                if pattern.match(img["src"]) is not None:
+                    caption = img.findNextSibling('h3')
+                    if caption is not None:
+                        caption.extract()
+                    img.extract()
+        return soup
+                        
+                    
+        
+
     feeds = [
         (u'News - UK',
          u'http://www.independent.co.uk/news/uk/?service=rss'),
@@ -125,7 +152,7 @@ class TheIndependentNew(BasicNewsRecipe):
         (u'News - Education',
          u'http://www.independent.co.uk/news/education/?service=rss'),
         (u'News - Obituaries',
-         u'http://rss.feedsportal.com/c/266/f/3531/index.rss'),
+         u'http://www.independent.co.uk/news/obituaries/?service=rss'),
         (u'News - Corrections',
          u'http://www.independent.co.uk/news/corrections/?service=rss'
          ),
@@ -146,11 +173,11 @@ class TheIndependentNew(BasicNewsRecipe):
          u'http://www.independent.co.uk/sport/motor-racing/?service=rss'
          ),
         (u'Sport - Olympics',
-         u'http://rss.feedsportal.com/c/266/f/3800/index.rss'),
+         u'http://www.independent.co.uk/sport/olympics/?service=rss'),
         (u'Sport - Racing',
          u'http://www.independent.co.uk/sport/racing/?service=rss'),
         (u'Sport - Rugby League',
-         u'http://rss.feedsportal.com/c/266/f/3795/index.rss'),
+         u'http://www.independent.co.uk/sport/general/rugby-league/?service=rss'),
         (u'Sport - Rugby Union',
          u'http://www.independent.co.uk/sport/rugby/rugby-union/?service=rss'
          ),
@@ -216,3 +243,6 @@ class TheIndependentNew(BasicNewsRecipe):
          u'http://www.independent.co.uk/extras/indybest/?service=rss'),
         ]
         
+        
+        
+