Adds cover image, CSS for body and subtitles, preprocess line breaks, added description, tags, etc.

recipes/mit_technology_review.recipe

@@ -2,19 +2,25 @@
 from __future__ import unicode_literals
 __license__   = 'GPL v3'
 __copyright__ = '2015 Michael Marotta <mikefm at gmail.net>'
+## Written April 2015
+## Last edited 4/18/15
 '''
 technologyreview.com
 '''
+import re
 from calibre.web.feeds.news import BasicNewsRecipe
+import urllib2, httplib
 
 class MitTechnologyReview(BasicNewsRecipe):
 
     title      = 'MIT Technology Review Magazine'
     __author__ = 'Michael Marotta'
-    description = 'MIT Technology Review (The Magazine)'
+    description = 'Bi-monthly magazine version of MIT Technology Review. This is different than the recipe named simply "Technology Review" which downloads the rss feed with daily articles fromt he website.'
     INDEX = 'http://www.technologyreview.com/magazine/'
     language = 'en'
     encoding = 'utf-8'
+    simultaneous_downloads = 20
+    tags = 'news, technology, science'
 
     keep_only_tags = [
         {'attrs':{'class':['body', 'intro', 'article-magazine', 'byline', 'view-byline', 'sticky-wrap', 'body hack']}},
@@ -22,10 +28,24 @@ class MitTechnologyReview(BasicNewsRecipe):
     remove_tags        = [
         {'name': ['meta', 'link', 'noscript', 'clearfix', 'flag']},
     ]
-    no_stylesheets = True
+    no_stylesheets = True   
+    preprocess_regexps     = [(re.compile(r'<br[ ]*/>', re.IGNORECASE), lambda m: ''),
+                              (re.compile(r'<br[ ]*clear.*/>', re.IGNORECASE), lambda m: '')]
+
+    extra_css = 'body { font-family: helvetica, sans-serif; } \
+                 h2 { text-align: left; font-size: 1em; font-weight: bold; }}'
 
     def parse_index(self):
         soup = self.index_to_soup(self.INDEX)
+        #find cover
+        dUrl = urllib2.urlopen(self.INDEX) #gets /magazin/year/month part of url to find class with cover image
+        dateUrl = dUrl.geturl()[-18:]
+        cover = soup.find('a', attrs={'href':dateUrl})
+        if cover is not None:
+            img = cover.find('img', src=True)
+            if img is not None:
+                self.cover_url = img['src']
+        #parse articles
         col = soup.find(attrs={'class':'view-content'})
         current_section, current_articles = None, []
         feeds = []
@@ -37,20 +57,22 @@ class MitTechnologyReview(BasicNewsRecipe):
                 current_articles = []
                 self.log('Found section:', current_section)
             elif current_section:
-                a = tag  # since tag itself is a <a> tag use it directly instead of using find
+                a=tag       #since tag itself is a <a> tag use it directly instead of using find
-                if not self.tag_to_string(a.h2):
+                if a is not None:
-                    title = self.tag_to_string(a.h1)
+                    if self.tag_to_string(a.h2) == "":
-                else:
+                        title = self.tag_to_string(a.h1)
-                    title = self.tag_to_string(a.h2) + ": " + self.tag_to_string(a.h1)
+                    else:
-                url = a['href']
+                        title = self.tag_to_string(a.h2) + ": " + self.tag_to_string(a.h1)
-                if url.startswith('/'):
+                    if "http://www.technologyreview.com" in a['href']:
-                    url = "http://www.technologyreview.com" + url
+                        url = a['href']
-                if title and url:
+                    else:
-                    p = tag.find('p', attrs={'class':'columns-off'})
+                        url = "http://www.technologyreview.com" + a['href']
-                    desc = self.tag_to_string(p) if p is not None else ''
+                    if title and url:
-                    current_articles.append({'title':title, 'url':url, 'description':desc})
+                        p = tag.find('p', attrs={'class':'columns-off'})
-                    self.log('\tArticle:', title, '[%s]' % url)
+                        desc = self.tag_to_string(p) if p is not None else ''
-                    self.log('\t\t', desc)
+                        current_articles.append({'title':title, 'url':url, 'description':desc})
+                        self.log('\tArticle:', title, '[%s]' % url)
+                        self.log('\t\t', desc)   
         if current_section and current_articles:
             feeds.append((current_section, current_articles))
-        return feeds
+        return feeds