Discover Magazine Monthly by Michael Marotta

Merge branch 'master' of https://github.com/truth1ness/calibre

recipes/discover_magazine_monthly.recipe

@@ -0,0 +1,92 @@
+#!/usr/bin/env  python2
+from __future__ import unicode_literals
+__license__   = 'GPL v3'
+__copyright__ = '2015 Michael Marotta <mikefm at gmail.net>'
+# Written April 2015
+# Last edited 4/17/15
+'''
+discovermagazine.com
+'''
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class DiscoverMagazine(BasicNewsRecipe):
+
+    title      = 'Discover Magazine Monthly'
+    __author__ = 'Michael Marotta'
+    description = 'Monthly magazine version of Discover Magazine (not rss feed).'
+    language = 'en'
+    encoding = 'utf-8'
+    simultaneous_downloads = 20
+    tags = 'news, technology, science'
+    INDEX = 'http://www.discovermagazine.com'
+
+    keep_only_tags = [
+        {'attrs':{'class':['headline', 'deck', 'belowDeck', 'mediaContainer', 'segment', 'cover']}},
+    ]
+    remove_tags = [dict(name='div', attrs={'class': ['ladder', 'mobile', 'popular', 'open', 'scistarter']})]
+
+    # Login stuff
+    needs_subscription = True
+    use_javascript_to_login = True
+    requires_version = (0, 9, 20)
+
+    def javascript_login(self, br, username, password):
+        br.visit('http://discovermagazine.com', timeout=120)
+        f = br.select_form('div.login.section div.form')
+        f['username'] = username
+        f['password'] = password
+        br.submit('input[id="signInButton"]', timeout=120)
+        br.run_for_a_time(20)
+        # br.show_browser()
+    # End login stuff
+
+    no_stylesheets = True
+    preprocess_regexps     = [(re.compile(r'<br[ ]*/>', re.IGNORECASE), lambda m: ''),
+                              (re.compile(r'<br[ ]*clear.*/>', re.IGNORECASE), lambda m: '')]
+
+    extra_css = 'body { font-family: helvetica, sans-serif; } \
+                 .belowdeck {font-style: italic; padding=bottom: 10px; max-width: none} \
+                 .caption {font-style: italic; padding=bottom: 10px; max-width: none} \
+                 .caption1 {font-style: italic; padding=bottom: 10px; max-width: none} \
+                 h2 { text-align: left; font-size: 1em; font-weight: bold; }}'
+
+    def parse_index(self):
+        # gets current month from homepage and append to index
+        soup = self.index_to_soup(self.INDEX)
+        c = soup.find(name=['a'], attrs={'title':['See inside the current issue of Discover Magazine']})
+        currentMonth = self.tag_to_string(c['href'])
+        self.INDEX = self.INDEX + currentMonth
+        # continue parsing
+        soup = self.index_to_soup(self.INDEX)
+        col = soup.find(attrs={'class':'issue'})
+        current_section, current_articles = None, []
+        feeds = []
+        # find cover
+        cover = soup.find('div', attrs={'class':'cover'})
+        if cover is not None:
+            img = cover.find('img', src=True)
+            if img is not None:
+                self.cover_url = 'http://www.discovermagazine.com' + img['src'].replace(' ', '%20')  # [:-7]
+        # parse articles
+        for tag in col.findAll(name=['h3', 'div'], attrs={'class':['bottomBorder', 'headline']}):
+            if tag.name == 'h3':
+                if current_section and current_articles:
+                    feeds.append((current_section, current_articles))
+                current_section = self.tag_to_string(tag).capitalize()
+                current_articles = []
+                self.log('Found section:', current_section)
+            elif current_section:
+                a = tag.find('a', href=True)
+                if a is not None:
+                    title = self.tag_to_string(a)
+                    url = 'http://www.discovermagazine.com' + a['href']
+                    if title and url:
+                        p = tag.find('div', attrs={'class':'snippet'})
+                        desc = self.tag_to_string(p) if p is not None else ''
+                        current_articles.append({'title':title, 'url':url, 'description':desc})
+                        self.log('\tArticle:', title, '[%s]' % url)
+                        self.log('\t\t', desc)
+        if current_section and current_articles:
+            feeds.append((current_section, current_articles))
+        return feeds

recipes/mit_technology_review.recipe

@@ -2,19 +2,26 @@
 from __future__ import unicode_literals
 __license__   = 'GPL v3'
 __copyright__ = '2015 Michael Marotta <mikefm at gmail.net>'
+# Written April 2015
+# Last edited 4/18/15
 '''
 technologyreview.com
 '''
+import re
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class MitTechnologyReview(BasicNewsRecipe):
 
     title      = 'MIT Technology Review Magazine'
     __author__ = 'Michael Marotta'
-    description = 'MIT Technology Review (The Magazine)'
+    description = ('Bi-monthly magazine version of MIT Technology Review.'
+                   ' This is different than the recipe named simply "Technology Review"'
+                   ' which downloads the rss feed with daily articles from the website.')
     INDEX = 'http://www.technologyreview.com/magazine/'
     language = 'en'
     encoding = 'utf-8'
+    simultaneous_downloads = 20
+    tags = 'news, technology, science'
 
     keep_only_tags = [
         {'attrs':{'class':['body', 'intro', 'article-magazine', 'byline', 'view-byline', 'sticky-wrap', 'body hack']}},
@@ -23,9 +30,17 @@ class MitTechnologyReview(BasicNewsRecipe):
         {'name': ['meta', 'link', 'noscript', 'clearfix', 'flag']},
     ]
     no_stylesheets = True
+    preprocess_regexps     = [(re.compile(r'<br[ ]*/>', re.IGNORECASE), lambda m: ''),
+                              (re.compile(r'<br[ ]*clear.*/>', re.IGNORECASE), lambda m: '')]
+
+    extra_css = 'body { font-family: helvetica, sans-serif; } \
+                 h2 { text-align: left; font-size: 1em; font-weight: bold; }}'
 
     def parse_index(self):
         soup = self.index_to_soup(self.INDEX)
+        # find cover
+        self.cover_url = soup.find('li', attrs={'class':'cover'}).find('img', src=True)['src']
+        # parse articles
         col = soup.find(attrs={'class':'view-content'})
         current_section, current_articles = None, []
         feeds = []
@@ -38,13 +53,15 @@ class MitTechnologyReview(BasicNewsRecipe):
                 self.log('Found section:', current_section)
             elif current_section:
                 a=tag  # since tag itself is a <a> tag use it directly instead of using find
-                if not self.tag_to_string(a.h2):
+                if a is not None:
+                    if self.tag_to_string(a.h2) == "":
                         title = self.tag_to_string(a.h1)
                     else:
                         title = self.tag_to_string(a.h2) + ": " + self.tag_to_string(a.h1)
+                    if "http://www.technologyreview.com" in a['href']:
                         url = a['href']
-                if url.startswith('/'):
+                    else:
-                    url = "http://www.technologyreview.com" + url
+                        url = "http://www.technologyreview.com" + a['href']
                     if title and url:
                         p = tag.find('p', attrs={'class':'columns-off'})
                         desc = self.tag_to_string(p) if p is not None else ''