Update Chicago Tribune

recipes/chicago_tribune.recipe

@@ -6,92 +6,48 @@ __docformat__ = 'restructuredtext en'
 from calibre.web.feeds.news import BasicNewsRecipe
 
 
+def classes(classes):
+    q = frozenset(classes.split(' '))
+    return dict(attrs={
+        'class': lambda x: x and frozenset(x.split()).intersection(q)})
+
+
 class ChicagoTribune(BasicNewsRecipe):
 
     title = 'Chicago Tribune'
-    __author__ = 'Kovid Goyal, Sujata Raman and a.peter'
+    __author__ = 'Kovid Goyal'
     description = 'Politics, local and business news from Chicago'
     language = 'en'
-    version = 2
-
     use_embedded_content = False
     no_stylesheets = True
     remove_javascript = True
-    recursions = 1
 
-    keep_only_tags = [dict(name='div', attrs={'class': ["story", "entry-asset asset hentry"]}),
+    keep_only_tags = [
-                      dict(name='div', attrs={
+        dict(name='h1'),
-                           'id': ["pagebody", "story", "maincontentcontainer"]}),
+        dict(attrs={'data-content-size': 'leadart'}),
+        dict(itemprop='articleBody'),
     ]
-    remove_tags_after = [{'class': ['photo_article', ]}]
-    remove_empty_feeds = True
-
-    match_regexps = [r'page=[0-9]+']
 
     remove_tags = [
-        {'id': ["moduleArticleTools", "content-bottom", "rail", "articleRelates module", "toolSet",
+        classes('trb_ar_cont trb_ar_main_ad trb_em_r_cc'),
-                "relatedrailcontent", "div-wrapper", "beta", "atp-comments", "footer", 'gallery-subcontent', 'subFooter']},
-        {'class': ["clearfix", "relatedTitle", "articleRelates module",
-                   "asset-footer", "tools", "comments", "featurePromo", "featurePromo fp-topjobs brownBackground", 'ndn_embed',
-                   "clearfix fullSpan brownBackground", "curvedContent", 'nextgen-share-tools', 'outbrainTools', 'google-ad-story-bottom']},
-        dict(name='font', attrs={'id': ["cr-other-headlines"]})]
-    extra_css = '''
-                    h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;}
-                    h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;}
-                    .byline {font-family:Arial,Helvetica,sans-serif; font-size:xx-small;}
-                    .date {font-family:Arial,Helvetica,sans-serif; font-size:xx-small;}
-                    p{font-family:Arial,Helvetica,sans-serif;font-size:small;}
-                    .copyright {font-family:Arial,Helvetica,sans-serif;font-size:xx-small;text-align:center}
-                    .story{font-family:Arial,Helvetica,sans-serif;font-size:small;}
-                    .entry-asset asset hentry{font-family:Arial,Helvetica,sans-serif;font-size:small;}
-                    .pagebody{font-family:Arial,Helvetica,sans-serif;font-size:small;}
-                    .maincontentcontainer{font-family:Arial,Helvetica,sans-serif;font-size:small;}
-                    .story-body{font-family:Arial,Helvetica,sans-serif;font-size:small;}
-                    body{font-family:Helvetica,Arial,sans-serif;font-size:small;}
-                '''
-    feeds = [
-        ('Latest news', 'http://feeds.chicagotribune.com/chicagotribune/news/'),
-        ('Local news', 'http://feeds.chicagotribune.com/chicagotribune/news/local/'),
-        ('Nation/world', 'http://feeds.chicagotribune.com/chicagotribune/news/nationworld/'),
-        ('Hot topics', 'http://feeds.chicagotribune.com/chicagotribune/hottopics/'),
-        ('Most E-mailed stories', 'http://feeds.chicagotribune.com/chicagotribune/email/'),
-        ('Opinion', 'http://feeds.chicagotribune.com/chicagotribune/opinion/'),
-        ('Off Topic', 'http://feeds.chicagotribune.com/chicagotribune/offtopic/'),
-        ('Politics', 'http://feeds.feedburner.com/chicagotribune/cloutstreet/'),
-        ('Business news', 'http://feeds.chicagotribune.com/chicagotribune/business/'),
-        ('Jobs and Careers', 'http://feeds.chicagotribune.com/chicagotribune/career/'),
-        ('Local scene', 'http://feeds.chicagotribune.com/chicagohomes/localscene/'),
-        ('Phil Rosenthal', 'http://feeds.chicagotribune.com/chicagotribune/rosenthal/'),
-        ('Your Money', 'http://feeds.chicagotribune.com/chicagotribune/yourmoney/'),
-        ('Jon Hilkevitch - Getting around',
-         'http://feeds.chicagotribune.com/chicagotribune/gettingaround/'),
-        ('Jon Yates - What\'s your problem?',
-         'http://feeds.chicagotribune.com/chicagotribune/problem/'),
-        ('Garisson Keillor', 'http://feeds.chicagotribune.com/chicagotribune/keillor/'),
-        ('Marks Jarvis - On Money',
-         'http://feeds.chicagotribune.com/chicagotribune/marksjarvisonmoney/'),
-        ('Sports', 'http://feeds.chicagotribune.com/chicagotribune/sports/'),
-        ('Arts and Architecture',
-         'http://feeds.chicagotribune.com/chicagotribune/arts/'),
-        ('Books', 'http://feeds.chicagotribune.com/chicagotribune/books/'),
-        ('Movies', 'http://feeds.chicagotribune.com/chicagotribune/movies/'),
-        ('Music', 'http://feeds.chicagotribune.com/chicagotribune/music/'),
-        ('TV', 'http://feeds.chicagotribune.com/chicagotribune/tv/'),
-        ('Hypertext', 'http://feeds.chicagotribune.com/chicagotribune/hypertext/'),
-        ('iPhone Blog', 'http://feeds.feedburner.com/redeye/iphoneblog'),
-        ('Julie\'s Health Club',
-         'http://feeds.chicagotribune.com/chicagotribune_julieshealthclub/'),
     ]
 
-    def get_article_url(self, article):
+    feeds = [
-        url = BasicNewsRecipe.get_article_url(self, article)
+        ('Breaking news', 'https://www.chicagotribune.com/news/local/breaking/rss2.0.xml'),
-        url = url.split('/')[-2]
+        ('Trending news', 'https://www.chicagotribune.com/news/trending/rss2.0.xml'),
-        encoding = {'0B': '.', '0C': '/', '0A': '0', '0F': '=', '0G': '&',
+        ('Opinion', 'https://www.chicagotribune.com/news/opinion/rss2.0.xml'),
-                    '0D': '?', '0E': '-', '0N': '.com', '0L': 'http://', '0S':
+        ('Business news', 'https://www.chicagotribune.com/business/rss2.0.xml'),
-                    'www.', '0I': '_', '0H': ','}
+        ('Sports', 'https://www.chicagotribune.com/sports/rss2.0.xml'),
-        for k, v in encoding.iteritems():
+        ('Arts and Entertainment',
-            url = url.replace(k, v)
+        'https://www.chicagotribune.com/entertainment/rss2.0.xml'),
-        return url.partition('?')[0]
+        ('Life & Style',
+        'https://www.chicagotribune.com/lifestyles/rss2.0.xml'),
+    ]
+
+    def preprocess_html(self, soup):
+        for img in soup.findAll('img', attrs={'data-baseurl': True}):
+            img['src'] = img['data-baseurl']
+        return soup
 
     def skip_ad_pages(self, soup):
         text = soup.find(text='click here to continue to article')
@@ -100,19 +56,3 @@ class ChicagoTribune(BasicNewsRecipe):
             url = a.get('href')
             if url:
                 return self.index_to_soup(url, raw=True)
-
-    def postprocess_html(self, soup, first_fetch):
-        # Remove the navigation bar. It was kept until now to be able to follow
-        # the links to further pages. But now we don't need them anymore.
-        for nav in soup.findAll(attrs={'class': ['toppaginate', 'article-nav clearfix']}):
-            nav.extract()
-
-        for t in soup.findAll(['table', 'tr', 'td']):
-            t.name = 'div'
-
-        for tag in soup.findAll('form', dict(attrs={'name': ["comments_form"]})):
-            tag.extract()
-        for tag in soup.findAll('font', dict(attrs={'id': ["cr-other-headlines"]})):
-            tag.extract()
-
-        return soup