diff --git a/recipes/foreignaffairs.recipe b/recipes/foreignaffairs.recipe index 2af669cb5a..aa9aa398f3 100644 --- a/recipes/foreignaffairs.recipe +++ b/recipes/foreignaffairs.recipe @@ -1,5 +1,8 @@ +#!/usr/bin/env python2 from calibre.web.feeds.news import BasicNewsRecipe import re +import html5lib +from lxml import html def select_form(form): @@ -35,44 +38,20 @@ class ForeignAffairsRecipe(BasicNewsRecipe): needs_subscription = True INDEX = 'http://www.foreignaffairs.com' - FRONTPAGE = 'http://www.foreignaffairs.com/magazine' + FRONTPAGE = INDEX + '/magazine' - remove_tags = [dict(name='svg')] - remove_tags_before = dict(name='div', attrs={'class': 'print-content'}) - remove_tags_after = dict(name='div', attrs={'class': 'print-footer'}) - - extra_css = ''' - body{font-family:verdana,arial,helvetica,geneva,sans-serif;} - div.print-footer {font-size: x-small; color: #696969;} - ''' + keep_only_tags = [ + dict(attrs={'class':lambda x: x and set(x.split()).intersection(set('article-header l-article-column'.split()))}), + ] conversion_options = {'comments': description, 'tags': category, 'language': 'en', 'publisher': publisher} - temp_files = [] - - def get_cover_url(self): - soup = self.index_to_soup(self.FRONTPAGE) - div = soup.find('div', attrs={'class':'magazine-hero__image image_auto_width'}) - img_url = div.find('img')['src'] - return img_url # The url includes the https:// as necessary - - def get_print_url(self, url): - article_soup = self.index_to_soup(url.strip()) - - if article_soup is not None: - shortlink = article_soup.find('a', attrs={'class':re.compile(r'\bicon-print\b')}) - if shortlink: - return shortlink['href'] - else: - return url - else: - return url - def parse_index(self): - answer = [] soup = self.index_to_soup(self.FRONTPAGE) + div = soup.find('div', attrs={'class':'magazine-hero__image image_auto_width'}) + self.cover_url = div.find('img')['src'] # get dates date = re.split('\s\|\s',self.tag_to_string(soup.head.title.string))[0] self.title = "Foreign Affairs ({})".format(date) @@ -85,8 +64,7 @@ class ForeignAffairsRecipe(BasicNewsRecipe): for article_block in sec.findAll('article'): if article_block.find('a') is not None: title=self.tag_to_string(article_block.div.a.h2) - article_url = article_block.div.a['href'] - url = self.get_print_url(article_url) + url = article_block.div.a['href'] atr=article_block.findNext('p', attrs={'class': 'author'}) if atr is not None: author=self.tag_to_string(atr) @@ -102,6 +80,14 @@ class ForeignAffairsRecipe(BasicNewsRecipe): answer.append((section, articles)) return answer + def preprocess_raw_html(self, raw_html, url): + root = html5lib.parse(raw_html, treebuilder='lxml', namespaceHTMLElements=False).getroot() + for svg in tuple(root.iter('{*}svg')): + svg.getparent().remove(svg) + for meta in tuple(root.iter('{*}meta')): + meta.getparent().remove(meta) + return html.tostring(root) + def preprocess_html(self, soup): for img in soup.findAll('img', attrs={'src': True}): if not img['src'].startswith('http'): @@ -110,8 +96,6 @@ class ForeignAffairsRecipe(BasicNewsRecipe): return soup def get_browser(self): - import html5lib - from lxml import html br = BasicNewsRecipe.get_browser(self) if self.username is not None and self.password is not None: # mechanize fails to parse the html correctly, so use html5lib to @@ -125,6 +109,3 @@ class ForeignAffairsRecipe(BasicNewsRecipe): br.form['pass'] = self.password br.submit() return br - - def cleanup(self): - self.browser.open('https://www.foreignaffairs.com/user/logout')