Update Associated Press

recipes/ap.recipe

@@ -1,6 +1,21 @@
+#!/usr/bin/env python2
+# vim:fileencoding=utf-8
+# License: GPLv3 Copyright: 2017, Kovid Goyal <kovid at kovidgoyal.net>
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import json
+
 from calibre.web.feeds.news import BasicNewsRecipe
 
 
+def classes(classes):
+    q = frozenset(classes.split(' '))
+    return dict(
+        attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)}
+    )
+
+
 class AssociatedPress(BasicNewsRecipe):
 
     title = u'Associated Press'
@@ -8,45 +23,49 @@ class AssociatedPress(BasicNewsRecipe):
     __author__ = 'Krittika Goyal'
     use_embedded_content = False
     language = 'en'
+    encoding = 'utf-8'
     no_stylesheets = True
-    conversion_options = {
+    ignore_duplicate_articles = {'title', 'url'}
-        'linearize_tables': True
+    remove_empty_feeds = False
-    }
+    keep_only_tags = [
-    keep_only_tags = {'name': 'table', 'attrs': {
+        classes('ap_headTitle'),
-        'class': lambda x: x and 'ap-story-table' in x.split()}}
+        dict(id="byLine"),
-    remove_tags = [
+        dict(id=lambda x: x and x.startswith('storyBodyDiv')),
-        {'class': ['ap-mediabox-table']},
-        {'name': 'img', 'src': lambda x: x and '//analytics.' in x},
     ]
 
     def parse_index(self):
         feeds = []
-        fronts = ('HOME', 'US', 'WORLD', 'BUSINESS', 'TECHNOLOGY',
+        limit = self.test[0] if self.test else 100
-                  'SPORTS', 'ENTERTAINMENT', 'HEALTH', 'SCIENCE', 'POLITICS')
+        for front in (
-        for front in fronts:
+            'topnews sports politics entertainment usnews oddities'
-            feeds.append([front.capitalize(), self.parse_section(front)])
+            ' Travel technology lifestyle business Health science intlnews'.split()
-        feeds[0][0] = 'Top Stories'
+        ):
+            name = {
+                'topnews': 'Top News',
+                'intlnews': 'International',
+                'usnews': 'U.S. News'
+            }.get(front, front).capitalize()
+            feeds.append([name, self.parse_section(front)])
+            if len(feeds) >= limit:
+                break
         return feeds
 
     def parse_section(self, front):
-        self.log('Processing section:', front)
+        url = 'https://afs-prod.appspot.com/api/v2/feed/tag?tags=apf-' + front
-        soup = self.index_to_soup(
+        self.log('Processing section:', front, 'at', url)
-            'http://hosted.ap.org/dynamic/fronts/%s?SITE=AP' % front)
+        data = self.index_to_soup(url, raw=True)
-
+        data = json.loads(data)
+        cards = data.get('cards', ())
         articles = []
-        for x in soup.findAll('p', attrs={'class': ['ap-newsbriefitem-p', 'ap-topheadlineitem-p']}):
+
-            if not x.contents:
+        for card in cards:
-                x = x.parent
+            for article in card['contents']:
-            a = x.find('a', href=True)
+                url = article['localLinkUrl']
-            title = self.tag_to_string(a)
+                title = article.get('flattenedFirstWords')
-            url = "http://hosted.ap.org" + a['href']
+                if not title:
-            p = x.find(attrs={'class': 'topheadlinebody'})
+                    continue
-            desc = ''
+                title = title.split('\u2014')[-1]
-            if p is not None:
+                self.log('\tFound article:', title, 'at', url)
-                desc = self.tag_to_string(p)
-            self.log('\tFound article:', title, '\n\t\t', desc)
                 articles.append({'title': title, 'url': url})
-
+        self.log('')
-        self.log('\n\n')
-
         return articles