diff --git a/recipes/discover_magazine_monthly.recipe b/recipes/discover_magazine_monthly.recipe new file mode 100644 index 0000000000..203729d67e --- /dev/null +++ b/recipes/discover_magazine_monthly.recipe @@ -0,0 +1,92 @@ +#!/usr/bin/env python2 +from __future__ import unicode_literals +__license__ = 'GPL v3' +__copyright__ = '2015 Michael Marotta ' +# Written April 2015 +# Last edited 4/17/15 +''' +discovermagazine.com +''' +import re +from calibre.web.feeds.news import BasicNewsRecipe + +class DiscoverMagazine(BasicNewsRecipe): + + title = 'Discover Magazine Monthly' + __author__ = 'Michael Marotta' + description = 'Monthly magazine version of Discover Magazine (not rss feed).' + language = 'en' + encoding = 'utf-8' + simultaneous_downloads = 20 + tags = 'news, technology, science' + INDEX = 'http://www.discovermagazine.com' + + keep_only_tags = [ + {'attrs':{'class':['headline', 'deck', 'belowDeck', 'mediaContainer', 'segment', 'cover']}}, + ] + remove_tags = [dict(name='div', attrs={'class': ['ladder', 'mobile', 'popular', 'open', 'scistarter']})] + + # Login stuff + needs_subscription = True + use_javascript_to_login = True + requires_version = (0, 9, 20) + + def javascript_login(self, br, username, password): + br.visit('http://discovermagazine.com', timeout=120) + f = br.select_form('div.login.section div.form') + f['username'] = username + f['password'] = password + br.submit('input[id="signInButton"]', timeout=120) + br.run_for_a_time(20) + # br.show_browser() + # End login stuff + + no_stylesheets = True + preprocess_regexps = [(re.compile(r'', re.IGNORECASE), lambda m: ''), + (re.compile(r'', re.IGNORECASE), lambda m: '')] + + extra_css = 'body { font-family: helvetica, sans-serif; } \ + .belowdeck {font-style: italic; padding=bottom: 10px; max-width: none} \ + .caption {font-style: italic; padding=bottom: 10px; max-width: none} \ + .caption1 {font-style: italic; padding=bottom: 10px; max-width: none} \ + h2 { text-align: left; font-size: 1em; font-weight: bold; }}' + + def parse_index(self): + # gets current month from homepage and append to index + soup = self.index_to_soup(self.INDEX) + c = soup.find(name=['a'], attrs={'title':['See inside the current issue of Discover Magazine']}) + currentMonth = self.tag_to_string(c['href']) + self.INDEX = self.INDEX + currentMonth + # continue parsing + soup = self.index_to_soup(self.INDEX) + col = soup.find(attrs={'class':'issue'}) + current_section, current_articles = None, [] + feeds = [] + # find cover + cover = soup.find('div', attrs={'class':'cover'}) + if cover is not None: + img = cover.find('img', src=True) + if img is not None: + self.cover_url = 'http://www.discovermagazine.com' + img['src'].replace(' ', '%20') # [:-7] + # parse articles + for tag in col.findAll(name=['h3', 'div'], attrs={'class':['bottomBorder', 'headline']}): + if tag.name == 'h3': + if current_section and current_articles: + feeds.append((current_section, current_articles)) + current_section = self.tag_to_string(tag).capitalize() + current_articles = [] + self.log('Found section:', current_section) + elif current_section: + a = tag.find('a', href=True) + if a is not None: + title = self.tag_to_string(a) + url = 'http://www.discovermagazine.com' + a['href'] + if title and url: + p = tag.find('div', attrs={'class':'snippet'}) + desc = self.tag_to_string(p) if p is not None else '' + current_articles.append({'title':title, 'url':url, 'description':desc}) + self.log('\tArticle:', title, '[%s]' % url) + self.log('\t\t', desc) + if current_section and current_articles: + feeds.append((current_section, current_articles)) + return feeds diff --git a/recipes/mit_technology_review.recipe b/recipes/mit_technology_review.recipe index 80b8d77dd4..ac81cb8ae4 100644 --- a/recipes/mit_technology_review.recipe +++ b/recipes/mit_technology_review.recipe @@ -2,19 +2,26 @@ from __future__ import unicode_literals __license__ = 'GPL v3' __copyright__ = '2015 Michael Marotta ' +# Written April 2015 +# Last edited 4/18/15 ''' technologyreview.com ''' +import re from calibre.web.feeds.news import BasicNewsRecipe class MitTechnologyReview(BasicNewsRecipe): title = 'MIT Technology Review Magazine' __author__ = 'Michael Marotta' - description = 'MIT Technology Review (The Magazine)' + description = ('Bi-monthly magazine version of MIT Technology Review.' + ' This is different than the recipe named simply "Technology Review"' + ' which downloads the rss feed with daily articles from the website.') INDEX = 'http://www.technologyreview.com/magazine/' language = 'en' encoding = 'utf-8' + simultaneous_downloads = 20 + tags = 'news, technology, science' keep_only_tags = [ {'attrs':{'class':['body', 'intro', 'article-magazine', 'byline', 'view-byline', 'sticky-wrap', 'body hack']}}, @@ -23,9 +30,17 @@ class MitTechnologyReview(BasicNewsRecipe): {'name': ['meta', 'link', 'noscript', 'clearfix', 'flag']}, ] no_stylesheets = True + preprocess_regexps = [(re.compile(r'', re.IGNORECASE), lambda m: ''), + (re.compile(r'', re.IGNORECASE), lambda m: '')] + + extra_css = 'body { font-family: helvetica, sans-serif; } \ + h2 { text-align: left; font-size: 1em; font-weight: bold; }}' def parse_index(self): soup = self.index_to_soup(self.INDEX) + # find cover + self.cover_url = soup.find('li', attrs={'class':'cover'}).find('img', src=True)['src'] + # parse articles col = soup.find(attrs={'class':'view-content'}) current_section, current_articles = None, [] feeds = [] @@ -37,20 +52,22 @@ class MitTechnologyReview(BasicNewsRecipe): current_articles = [] self.log('Found section:', current_section) elif current_section: - a = tag # since tag itself is a tag use it directly instead of using find - if not self.tag_to_string(a.h2): - title = self.tag_to_string(a.h1) - else: - title = self.tag_to_string(a.h2) + ": " + self.tag_to_string(a.h1) - url = a['href'] - if url.startswith('/'): - url = "http://www.technologyreview.com" + url - if title and url: - p = tag.find('p', attrs={'class':'columns-off'}) - desc = self.tag_to_string(p) if p is not None else '' - current_articles.append({'title':title, 'url':url, 'description':desc}) - self.log('\tArticle:', title, '[%s]' % url) - self.log('\t\t', desc) + a=tag # since tag itself is a tag use it directly instead of using find + if a is not None: + if self.tag_to_string(a.h2) == "": + title = self.tag_to_string(a.h1) + else: + title = self.tag_to_string(a.h2) + ": " + self.tag_to_string(a.h1) + if "http://www.technologyreview.com" in a['href']: + url = a['href'] + else: + url = "http://www.technologyreview.com" + a['href'] + if title and url: + p = tag.find('p', attrs={'class':'columns-off'}) + desc = self.tag_to_string(p) if p is not None else '' + current_articles.append({'title':title, 'url':url, 'description':desc}) + self.log('\tArticle:', title, '[%s]' % url) + self.log('\t\t', desc) if current_section and current_articles: feeds.append((current_section, current_articles)) return feeds