From 36e90c83485d9969d8916cce1eff02b0978281ab Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 5 Nov 2009 08:37:16 -0700 Subject: [PATCH] Fix #3902 (CNN articles are trailers only) --- resources/recipes/cnn.recipe | 67 ++++++++++++++++++++++++------------ 1 file changed, 45 insertions(+), 22 deletions(-) diff --git a/resources/recipes/cnn.recipe b/resources/recipes/cnn.recipe index 868f3f3641..21c4139d6e 100644 --- a/resources/recipes/cnn.recipe +++ b/resources/recipes/cnn.recipe @@ -16,44 +16,60 @@ class CNN(BasicNewsRecipe): no_stylesheets = True use_embedded_content = False oldest_article = 15 - + recursions = 1 + match_regexps = [r'http://sportsillustrated.cnn.com/.*/[1-9].html'] + extra_css = ''' - h1{font-family :Arial,Helvetica,sans-serif; font-size:large} + .cnn_strycntntlft{font-family :Arial,Helvetica,sans-serif;} h2{font-family :Arial,Helvetica,sans-serif; font-size:x-small} .cnnTxtCmpnt{font-family :Arial,Helvetica,sans-serif; font-size:x-small} - .cnnTMcontent{font-family :Arial,Helvetica,sans-serif; font-size:xx-small;color:#575757} - .storytext{font-family :Arial,Helvetica,sans-serif; font-size:x-small} - .storybyline{font-family :Arial,Helvetica,sans-serif; font-size:xx-small; color:#575757} + .cnnTMcontent{font-family :Arial,Helvetica,sans-serif; font-size:x-small;color:#575757} + .storytext{font-family :Arial,Helvetica,sans-serif; font-size:small} + .storybyline{font-family :Arial,Helvetica,sans-serif; font-size:x-small; color:#575757} .credit{font-family :Arial,Helvetica,sans-serif; font-size:xx-small; color:#575757} - .storyBrandingBanner{font-family :Arial,Helvetica,sans-serif; font-size:xx-small; color:#575757} - .storytimestamp{font-family :Arial,Helvetica,sans-serif; font-size:xx-small; color:#575757} - .timestamp{font-family :Arial,Helvetica,sans-serif; font-size:xx-small; color:#575757} - .subhead p{font-family :Arial,Helvetica,sans-serif; font-size:xx-small;} - .cnnStoryContent{font-family :Arial,Helvetica,sans-serif; font-size:xx-small} - .cnnContentContainer{font-family :Arial,Helvetica,sans-serif; font-size:xx-small} + .storyBrandingBanner{font-family :Arial,Helvetica,sans-serif; font-size:x-small; color:#575757} + .storytimestamp{font-family :Arial,Helvetica,sans-serif; font-size:x-small; color:#575757} + .timestamp{font-family :Arial,Helvetica,sans-serif; font-size:x-small; color:#575757} + .cnn_strytmstmp{font-family :Arial,Helvetica,sans-serif; font-size:x-small; color:#666666;} + .cnn_stryimg640caption{font-family :Arial,Helvetica,sans-serif; font-size:x-small; color:#666666;} + .cnn_strylccimg300cntr{font-family :Arial,Helvetica,sans-serif; font-size:x-small; color:#666666;} + .cnn_stryichgfcpt{font-family :Arial,Helvetica,sans-serif; font-size:x-small; color:#666666;} + .cnnByline{font-family :Arial,Helvetica,sans-serif; font-size:x-small; color:#666666;} + .cnn_bulletbin cnnStryHghLght{ font-size:xx-small;} + .subhead p{font-family :Arial,Helvetica,sans-serif; font-size:x-small;} + .cnnStoryContent{font-family :Arial,Helvetica,sans-serif; font-size:x-small} + .cnnContentContainer{font-family :Arial,Helvetica,sans-serif; font-size:x-small} .col1{font-family :Arial,Helvetica,sans-serif; font-size:x-small; color:#666666;} .col3{color:#333333; font-family :Arial,Helvetica,sans-serif; font-size:x-small;font-weight:bold;} - .cnnInlineT1Caption{font-family :Arial,Helvetica,sans-serif; font-size:xx-small;font-weight:bold;} - .cnnInlineT1Credit{font-family :Arial,Helvetica,sans-serif; font-size:xx-small;color:#333333;} - .col10{color:#5A637E} - .cnnTimeStamp{font-family :Arial,Helvetica,sans-serif; font-size:xx-small;color:#333333;} - .galleryhedDek{font-family :Arial,Helvetica,sans-serif; font-size:xx-small;color:#575757;} - .galleryWidgetHeader{font-family :Arial,Helvetica,sans-serif; font-size:xx-small;color:#004276;} - .article-content{font-family :Arial,Helvetica,sans-serif; font-size:xx-small} - .cnnRecapStory{font-family :Arial,Helvetica,sans-serif; font-size:xx-small} + .cnnInlineT1Caption{font-family :Arial,Helvetica,sans-serif; font-size:x-small;font-weight:bold;} + .cnnInlineT1Credit{font-family :Arial,Helvetica,sans-serif; font-size:x-small;color:#333333;} + .col10{color:#5A637E;} + .cnnInlineRailBulletList{color:black;} + .cnnLine0{font-family :Arial,Helvetica,sans-serif; color:#666666;font-weight:bold;} + .cnnTimeStamp{font-family :Arial,Helvetica,sans-serif; font-size:x-small;color:#333333;} + .galleryhedDek{font-family :Arial,Helvetica,sans-serif; font-size:x-small;color:#575757;} + .galleryWidgetHeader{font-family :Arial,Helvetica,sans-serif; font-size:x-small;color:#004276;} + .article-content{font-family :Arial,Helvetica,sans-serif; font-size:x-small} + .cnnRecapStory{font-family :Arial,Helvetica,sans-serif; font-size:x-small} + h1{font-family :Arial,Helvetica,sans-serif; font-size:x-large} + .captionname{font-family :Arial,Helvetica,sans-serif; font-size:x-small;color:#575757;} + inStoryIE{{font-family :Arial,Helvetica,sans-serif; font-size:x-small;} ''' keep_only_tags = [ - dict(name='div', attrs={'class':["cnnWCBoxContent","cnnContent","cnnMainBodySecs"]}), + dict(name='div', attrs={'class':["cnnWCBoxContent","cnnContent","cnnMainBodySecs","cnn_storyarea","cnn_strycntntlft","cnn_strytmstmp"]}), dict(name='div', attrs={'id':["contentBody","content"]}), - dict(name='td', attrs={'id':["cnnRecapStory"]}),] + dict(name='td', attrs={'id':["cnnRecapStory"]}), + dict(name='td', attrs={'class':["cnnBigSideLeft"]}), + ] remove_tags = [ - dict(name='div', attrs={'class':["storyLink","article-tools clearfix","widget video related-video vList","cnnFooterBox","scrollArrows","boxHeading","cnnInlineMailbag","mainCol_lastBlock","cnn_bookmarks","cnnFooterBox","cnnEndOfStory","cnnInlineSL","cnnStoryHighlights","cnnFooterClick","cnnSnapShotHeader","cnnStoryToolsFooter","cnnWsnr","cnnUGCBox","cnnTopNewsModule","cnnStoryElementBox","cnnStoryPhotoBoxNavigation"]}), + dict(name='div', attrs={'class':["cnnInlineRailSelectList","cnn_strycntntrgt","cnn_strybtntools","cnn_strylctcntr cnn_strylctcqrelt","cnn_strylceclbtn","cnn_stryftsbttm","cnn_strybtmmorebx","storyLink","article-tools clearfix","widget video related-video vList","cnnFooterBox","scrollArrows","boxHeading","cnnInlineMailbag","mainCol_lastBlock","cnn_bookmarks","cnnFooterBox","cnnEndOfStory","cnnInlineSL","cnnStoryHighlights","cnnFooterClick","cnnSnapShotHeader","cnnStoryToolsFooter","cnnWsnr","cnnUGCBox","cnnTopNewsModule","cnnStoryElementBox","cnnStoryPhotoBoxNavigation"]}), dict(name='span', attrs={'class':["cnnEmbeddedMosLnk"]}), dict(name='div', attrs={'id':["cnnIncldHlder","articleCommentsContainer","featuredContent","superstarsWidget","shareMenuContainer","rssMenuContainer","storyBrandingBanner","cnnRightCol","siteFeatures","quigo628","rightColumn","clickIncludeBox","cnnHeaderRightCol","cnnSCFontLabel","cnnSnapShotBottomRight","cnnSCFontButtons","rightColumn"]}), dict(name='p', attrs={'class':["cnnTopics"]}), dict(name='td', attrs={'class':["cnnRightRail"]}), dict(name='table', attrs={'class':["cnnTMbox"]}), dict(name='ul', attrs={'id':["cnnTopNav","cnnBotNav","cnnSBNav"]}), + dict(name='div', attrs={'id':["cnn_ftrcntnt"]}) ] # def print_version(self, url): @@ -75,3 +91,10 @@ class CNN(BasicNewsRecipe): ('Offbeat', 'http://rss.cnn.com/rss/cnn_offbeat.rss'), ('Most Popular', 'http://rss.cnn.com/rss/cnn_mostpopular.rss') ] + + def preprocess_html(self, soup): + + for tag in soup.findAll(name=['ul','li']): + tag.name = 'div' + + return soup