diff --git a/resources/images/news/foreignaffairs.png b/resources/images/news/foreignaffairs.png new file mode 100644 index 0000000000..91e6440b2e Binary files /dev/null and b/resources/images/news/foreignaffairs.png differ diff --git a/resources/recipes/foreignaffairs.recipe b/resources/recipes/foreignaffairs.recipe new file mode 100644 index 0000000000..69511cbd09 --- /dev/null +++ b/resources/recipes/foreignaffairs.recipe @@ -0,0 +1,124 @@ +from calibre.web.feeds.news import BasicNewsRecipe +import re +from calibre.ptempfile import PersistentTemporaryFile + +class ForeignAffairsRecipe(BasicNewsRecipe): + __license__ = 'GPL v3' + __author__ = 'kwetal' + language = 'en' + version = 1 + + title = u'Foreign Affairs (Subcription or (free) Registration)' + publisher = u'Council on Foreign Relations' + category = u'USA, Foreign Affairs' + description = u'The leading forum for serious discussion of American foreign policy and international affairs.' + + no_stylesheets = True + remove_javascript = True + + INDEX = 'http://www.foreignaffairs.com' + + remove_tags = [] + remove_tags.append(dict(name = 'base')) + #remove_tags.append(dict(name = '', attrs = {'': ''})) + + remove_tags_before = dict(name = 'h1', attrs = {'class': 'print-title'}) + + remove_tags_after = dict(name = 'div', attrs = {'class': 'print-footer'}) + + extra_css = ''' + body{font-family:verdana,arial,helvetica,geneva,sans-serif;} + div.print-footer {font-size: x-small; color: #696969;} + ''' + + conversion_options = {'comments': description, 'tags': category, 'language': 'en', + 'publisher': publisher} + + temp_files = [] + articles_are_obfuscated = True + + def get_obfuscated_article(self, url): + br = self.get_browser() + br.open(url) + + response = br.follow_link(url_regex = r'/print/[0-9]+', nr = 0) + html = response.read() + + self.temp_files.append(PersistentTemporaryFile('_fa.html')) + self.temp_files[-1].write(html) + self.temp_files[-1].close() + + return self.temp_files[-1].name + + def parse_index(self): + soup = self.index_to_soup('http://www.foreignaffairs.com/magazine') + articles = [] + answer = [] + content = soup.find('div', attrs = {'class': 'center-wrapper'}) + if content: + for div in content.findAll('div', attrs = {'class': re.compile(r'view-row\s+views-row-[0-9]+\s+views-row-[odd|even].*')}): + tag = div.find('div', attrs = {'class': 'views-field-title'}) + if tag: + a = tag.find('a') + if a: + title = self.tag_to_string(a) + url = self.INDEX + a['href'] + + author = self.tag_to_string(div.find('div', attrs = {'class': 'views-field-field-article-display-authors-value'})) + tag = div.find('span', attrs = {'class': 'views-field-field-article-summary-value'}) + # If they ever fix their markup, this will break :-( + summary = self.tag_to_string(tag.findNextSibling('p')) + description = author + '
' + summary + + articles.append({'title': title, 'date': None, 'url': url, 'description': description}) + else: + continue + else: + continue + + answer.append(('Magazine', articles)) + + ul = content.find('ul') + if ul: + articles = [] + for li in ul.findAll('li'): + tag = li.find('div', attrs = {'class': 'views-field-title'}) + if tag: + a = tag.find('a') + if a: + title = self.tag_to_string(a) + url = self.INDEX + a['href'] + description = '' + tag = li.find('div', attrs = {'class': 'views-field-field-article-display-authors-value'}) + if tag: + description = self.tag_to_string(tag) + + articles.append({'title': title, 'date': None, 'url': url, 'description': description}) + else: + continue + else: + continue + + answer.append(('Letters to the Editor', articles)) + + return answer + + def preprocess_html(self, soup): + for img in soup.findAll('img', attrs = {'src': True}): + if not img['src'].startswith('http://'): + img['src'] = self.INDEX + img['src'] + + return soup + + needs_subscription = True + + def get_browser(self): + br = BasicNewsRecipe.get_browser() + if self.username is not None and self.password is not None: + br.open('https://www.foreignaffairs.com/user?destination=home') + br.select_form(nr = 1) + br['name'] = self.username + br['pass'] = self.password + br.submit() + return br +