From 78c958f2f55bf9331dbbc388cdbfc036ffae5277 Mon Sep 17 00:00:00 2001 From: truth1ness Date: Sat, 18 Apr 2015 23:22:46 -0400 Subject: [PATCH 1/2] Adds cover image, CSS for body and subtitles, preprocess line breaks, added description, tags, etc. --- recipes/mit_technology_review.recipe | 56 +++++++++++++++++++--------- 1 file changed, 39 insertions(+), 17 deletions(-) diff --git a/recipes/mit_technology_review.recipe b/recipes/mit_technology_review.recipe index 80b8d77dd4..2a4fa217be 100644 --- a/recipes/mit_technology_review.recipe +++ b/recipes/mit_technology_review.recipe @@ -2,19 +2,25 @@ from __future__ import unicode_literals __license__ = 'GPL v3' __copyright__ = '2015 Michael Marotta ' +## Written April 2015 +## Last edited 4/18/15 ''' technologyreview.com ''' +import re from calibre.web.feeds.news import BasicNewsRecipe +import urllib2, httplib class MitTechnologyReview(BasicNewsRecipe): title = 'MIT Technology Review Magazine' __author__ = 'Michael Marotta' - description = 'MIT Technology Review (The Magazine)' + description = 'Bi-monthly magazine version of MIT Technology Review. This is different than the recipe named simply "Technology Review" which downloads the rss feed with daily articles fromt he website.' INDEX = 'http://www.technologyreview.com/magazine/' language = 'en' encoding = 'utf-8' + simultaneous_downloads = 20 + tags = 'news, technology, science' keep_only_tags = [ {'attrs':{'class':['body', 'intro', 'article-magazine', 'byline', 'view-byline', 'sticky-wrap', 'body hack']}}, @@ -22,10 +28,24 @@ class MitTechnologyReview(BasicNewsRecipe): remove_tags = [ {'name': ['meta', 'link', 'noscript', 'clearfix', 'flag']}, ] - no_stylesheets = True + no_stylesheets = True + preprocess_regexps = [(re.compile(r'', re.IGNORECASE), lambda m: ''), + (re.compile(r'', re.IGNORECASE), lambda m: '')] + + extra_css = 'body { font-family: helvetica, sans-serif; } \ + h2 { text-align: left; font-size: 1em; font-weight: bold; }}' def parse_index(self): soup = self.index_to_soup(self.INDEX) + #find cover + dUrl = urllib2.urlopen(self.INDEX) #gets /magazin/year/month part of url to find class with cover image + dateUrl = dUrl.geturl()[-18:] + cover = soup.find('a', attrs={'href':dateUrl}) + if cover is not None: + img = cover.find('img', src=True) + if img is not None: + self.cover_url = img['src'] + #parse articles col = soup.find(attrs={'class':'view-content'}) current_section, current_articles = None, [] feeds = [] @@ -37,20 +57,22 @@ class MitTechnologyReview(BasicNewsRecipe): current_articles = [] self.log('Found section:', current_section) elif current_section: - a = tag # since tag itself is a tag use it directly instead of using find - if not self.tag_to_string(a.h2): - title = self.tag_to_string(a.h1) - else: - title = self.tag_to_string(a.h2) + ": " + self.tag_to_string(a.h1) - url = a['href'] - if url.startswith('/'): - url = "http://www.technologyreview.com" + url - if title and url: - p = tag.find('p', attrs={'class':'columns-off'}) - desc = self.tag_to_string(p) if p is not None else '' - current_articles.append({'title':title, 'url':url, 'description':desc}) - self.log('\tArticle:', title, '[%s]' % url) - self.log('\t\t', desc) + a=tag #since tag itself is a tag use it directly instead of using find + if a is not None: + if self.tag_to_string(a.h2) == "": + title = self.tag_to_string(a.h1) + else: + title = self.tag_to_string(a.h2) + ": " + self.tag_to_string(a.h1) + if "http://www.technologyreview.com" in a['href']: + url = a['href'] + else: + url = "http://www.technologyreview.com" + a['href'] + if title and url: + p = tag.find('p', attrs={'class':'columns-off'}) + desc = self.tag_to_string(p) if p is not None else '' + current_articles.append({'title':title, 'url':url, 'description':desc}) + self.log('\tArticle:', title, '[%s]' % url) + self.log('\t\t', desc) if current_section and current_articles: feeds.append((current_section, current_articles)) - return feeds + return feeds \ No newline at end of file From 7a3672fa0164205aefcf9c96733b42d5b61a213d Mon Sep 17 00:00:00 2001 From: truth1ness Date: Sat, 18 Apr 2015 23:29:36 -0400 Subject: [PATCH 2/2] New Discover Magazine Monthly/Print edition, different than current rss based discover_magazine.recipe --- recipes/discover_magazine_monthly.recipe | 93 ++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 recipes/discover_magazine_monthly.recipe diff --git a/recipes/discover_magazine_monthly.recipe b/recipes/discover_magazine_monthly.recipe new file mode 100644 index 0000000000..9cf4a70942 --- /dev/null +++ b/recipes/discover_magazine_monthly.recipe @@ -0,0 +1,93 @@ +#!/usr/bin/env python2 +from __future__ import unicode_literals +__license__ = 'GPL v3' +__copyright__ = '2015 Michael Marotta ' +## Written April 2015 +## Last edited 4/17/15 +''' +discovermagazine.com +''' +import re +from calibre.web.feeds.news import BasicNewsRecipe + +class DiscoverMagazine(BasicNewsRecipe): + + title = 'Discover Magazine Monthly' + __author__ = 'Michael Marotta' + description = 'Monthly magazine version of Discover Magazine (not rss feed).' + language = 'en' + encoding = 'utf-8' + simultaneous_downloads = 20 + tags = 'news, technology, science' + INDEX = 'http://www.discovermagazine.com' + + keep_only_tags = [ + {'attrs':{'class':['headline', 'deck', 'belowDeck', 'mediaContainer', 'segment', 'cover']}}, + ] + remove_tags = [dict(name='div', attrs={'class': ['ladder', 'mobile', 'popular', 'open', 'scistarter']})] + + + # Login stuff + needs_subscription = True + use_javascript_to_login = True + requires_version = (0, 9, 20) + + def javascript_login(self, br, username, password): + br.visit('http://discovermagazine.com', timeout=120) + f = br.select_form('div.login.section div.form') + f['username'] = username + f['password'] = password + br.submit('input[id="signInButton"]', timeout=120) + br.run_for_a_time(20) + #br.show_browser() + # End login stuff + + no_stylesheets = True + preprocess_regexps = [(re.compile(r'', re.IGNORECASE), lambda m: ''), + (re.compile(r'', re.IGNORECASE), lambda m: '')] + + extra_css = 'body { font-family: helvetica, sans-serif; } \ + .belowdeck {font-style: italic; padding=bottom: 10px; max-width: none} \ + .caption {font-style: italic; padding=bottom: 10px; max-width: none} \ + .caption1 {font-style: italic; padding=bottom: 10px; max-width: none} \ + h2 { text-align: left; font-size: 1em; font-weight: bold; }}' + + def parse_index(self): + #gets current month from homepage and append to index + soup = self.index_to_soup(self.INDEX) + c = soup.find(name=['a'], attrs={'title':['See inside the current issue of Discover Magazine']}) + currentMonth = self.tag_to_string(c['href']) + self.INDEX = self.INDEX + currentMonth + #continue parsing + soup = self.index_to_soup(self.INDEX) + col = soup.find(attrs={'class':'issue'}) + current_section, current_articles = None, [] + feeds = [] + #find cover + cover = soup.find('div', attrs={'class':'cover'}) + if cover is not None: + img = cover.find('img', src=True) + if img is not None: + self.cover_url = 'http://www.discovermagazine.com' + img['src'].replace(' ', '%20')#[:-7] + #parse articles + for tag in col.findAll(name=['h3', 'div'], attrs={'class':['bottomBorder', 'headline']}): + if tag.name == 'h3': + if current_section and current_articles: + feeds.append((current_section, current_articles)) + current_section = self.tag_to_string(tag).capitalize() + current_articles = [] + self.log('Found section:', current_section) + elif current_section: + a = tag.find('a', href=True) + if a is not None: + title = self.tag_to_string(a) + url = 'http://www.discovermagazine.com' + a['href'] + if title and url: + p = tag.find('div', attrs={'class':'snippet'}) + desc = self.tag_to_string(p) if p is not None else '' + current_articles.append({'title':title, 'url':url, 'description':desc}) + self.log('\tArticle:', title, '[%s]' % url) + self.log('\t\t', desc) + if current_section and current_articles: + feeds.append((current_section, current_articles)) + return feeds \ No newline at end of file