diff --git a/recipes/cnn.recipe b/recipes/cnn.recipe index 3cb66d2967..b1ca0aaf47 100644 --- a/recipes/cnn.recipe +++ b/recipes/cnn.recipe @@ -18,8 +18,9 @@ class CNN(BasicNewsRecipe): no_stylesheets = True use_embedded_content = False oldest_article = 15 - #recursions = 1 - #match_regexps = [r'http://sportsillustrated.cnn.com/.*/[1-9].html'] + ignore_duplicate_articles = {'url'} + # recursions = 1 + # match_regexps = [r'http://sportsillustrated.cnn.com/.*/[1-9].html'] max_articles_per_feed = 25 extra_css = ''' @@ -36,25 +37,18 @@ class CNN(BasicNewsRecipe): (re.compile(r'', re.DOTALL), lambda m: ''), ] - keep_only_tags = [dict(id=['cnnContentContainer', 'storycontent'])] - remove_tags = [ - {'class':['cnn_strybtntools', 'cnn_strylftcntnt', - 'cnn_strybtntools', 'cnn_strybtntoolsbttm', 'cnn_strybtmcntnt', - 'cnn_strycntntrgt', 'hed_side', 'foot', 'cnn_strylftcntnt cnn_strylftcexpbx']}, - {'class':['cnn_html_media_title_new', 'cnn_html_media_title_new cnn_html_media_title_none', - 'cnnArticleGalleryCaptionControlText', 'articleGalleryNavContainer']}, - {'id':['articleGalleryNav00JumpPrev', 'articleGalleryNav00Prev', - 'articleGalleryNav00Next', 'articleGalleryNav00JumpNext']}, - {'style':['display:none']}, - dict(id=['ie_column']), + keep_only_tags = [ + dict(id=['body-text', 'storycontent']), + dict(attrs={'class':['pg-headline', 'metadata']}), + ] + remove_tags = [ ] - feeds = [ ('Top News', 'http://rss.cnn.com/rss/cnn_topstories.rss'), ('World', 'http://rss.cnn.com/rss/cnn_world.rss'), ('U.S.', 'http://rss.cnn.com/rss/cnn_us.rss'), - #('Sports', 'http://rss.cnn.com/rss/si_topstories.rss'), + # ('Sports', 'http://rss.cnn.com/rss/si_topstories.rss'), ('Business', 'http://rss.cnn.com/rss/money_latest.rss'), ('Politics', 'http://rss.cnn.com/rss/cnn_allpolitics.rss'), ('Law', 'http://rss.cnn.com/rss/cnn_law.rss'), @@ -67,9 +61,21 @@ class CNN(BasicNewsRecipe): ('Most Popular', 'http://rss.cnn.com/rss/cnn_mostpopular.rss') ] + def preprocess_html(self, soup): + body = soup.find('body') + for h2 in soup.findAll(attrs={'class':'pg-headline'}): + h2.extract() + body.insert(0, h2) + for img in soup.findAll('img', attrs={'data-src-medium':True}): + img['src'] = img['data-src-medium'] + return soup + def get_article_url(self, article): ans = BasicNewsRecipe.get_article_url(self, article) - return ans.partition('?')[0] + ans = ans.partition('?')[0] + if '.com/videos/' in ans: + ans = None + return ans def get_masthead_url(self): masthead = 'http://i.cdn.turner.com/cnn/.element/img/3.0/global/header/intl/hdr-globe-central.gif'