Adds cover image, CSS for body and subtitles, preprocess line breaks, added description, tags, etc.

recipes/mit_technology_review.recipe

@@ -2,19 +2,25 @@
 from __future__ import unicode_literals
 __license__   = 'GPL v3'
 __copyright__ = '2015 Michael Marotta <mikefm at gmail.net>'
+## Written April 2015
+## Last edited 4/18/15
 '''
 technologyreview.com
 '''
+import re
 from calibre.web.feeds.news import BasicNewsRecipe
+import urllib2, httplib
 
 class MitTechnologyReview(BasicNewsRecipe):
 
     title      = 'MIT Technology Review Magazine'
     __author__ = 'Michael Marotta'
-    description = 'MIT Technology Review (The Magazine)'
+    description = 'Bi-monthly magazine version of MIT Technology Review. This is different than the recipe named simply "Technology Review" which downloads the rss feed with daily articles fromt he website.'
     INDEX = 'http://www.technologyreview.com/magazine/'
     language = 'en'
     encoding = 'utf-8'
+    simultaneous_downloads = 20
+    tags = 'news, technology, science'
 
     keep_only_tags = [
         {'attrs':{'class':['body', 'intro', 'article-magazine', 'byline', 'view-byline', 'sticky-wrap', 'body hack']}},
@@ -23,9 +29,23 @@ class MitTechnologyReview(BasicNewsRecipe):
         {'name': ['meta', 'link', 'noscript', 'clearfix', 'flag']},
     ]
     no_stylesheets = True   
+    preprocess_regexps     = [(re.compile(r'<br[ ]*/>', re.IGNORECASE), lambda m: ''),
+                              (re.compile(r'<br[ ]*clear.*/>', re.IGNORECASE), lambda m: '')]
+
+    extra_css = 'body { font-family: helvetica, sans-serif; } \
+                 h2 { text-align: left; font-size: 1em; font-weight: bold; }}'
 
     def parse_index(self):
         soup = self.index_to_soup(self.INDEX)
+        #find cover
+        dUrl = urllib2.urlopen(self.INDEX) #gets /magazin/year/month part of url to find class with cover image
+        dateUrl = dUrl.geturl()[-18:]
+        cover = soup.find('a', attrs={'href':dateUrl})
+        if cover is not None:
+            img = cover.find('img', src=True)
+            if img is not None:
+                self.cover_url = img['src']
+        #parse articles
         col = soup.find(attrs={'class':'view-content'})
         current_section, current_articles = None, []
         feeds = []
@@ -37,14 +57,16 @@ class MitTechnologyReview(BasicNewsRecipe):
                 current_articles = []
                 self.log('Found section:', current_section)
             elif current_section:
-                a = tag  # since tag itself is a <a> tag use it directly instead of using find
-                if not self.tag_to_string(a.h2):
+                a=tag       #since tag itself is a <a> tag use it directly instead of using find
+                if a is not None:
+                    if self.tag_to_string(a.h2) == "":
                         title = self.tag_to_string(a.h1)
                     else:
                         title = self.tag_to_string(a.h2) + ": " + self.tag_to_string(a.h1)
+                    if "http://www.technologyreview.com" in a['href']:
                         url = a['href']
-                if url.startswith('/'):
-                    url = "http://www.technologyreview.com" + url
+                    else:
+                        url = "http://www.technologyreview.com" + a['href']
                     if title and url:
                         p = tag.find('p', attrs={'class':'columns-off'})
                         desc = self.tag_to_string(p) if p is not None else ''