Update New Yorker

recipes/new_yorker.recipe

@@ -2,17 +2,22 @@
 # -*- coding: utf-8 -*-
 __license__ = 'GPL v3'
 
+from collections import defaultdict
 from calibre.web.feeds.news import BasicNewsRecipe
 from calibre import browser
 
 
+def absurl(x):
+    if x.startswith('/') and not x.startswith('//'):
+        x = 'https://www.newyorker.com' + x
+    return x
+
+
 class NewYorker(BasicNewsRecipe):
 
     title = u'New Yorker Magazine'
     description = u'Content from the New Yorker website'
 
-    masthead_url = 'https://www.newyorker.com/images/elements/print/newyorker_printlogo.gif'
-
     url_list = []
     language = 'en'
     __author__ = 'Kovid Goyal'
@@ -26,8 +31,13 @@ class NewYorker(BasicNewsRecipe):
                 '''
     needs_subscription = 'optional'
     keep_only_tags = [
-        dict(itemprop=['headline', 'alternativeHeadline', 'author', 'articleBody']),
+        dict(attrs={'class':lambda x: x and 'ArticleHeader__hed___' in x}),
-        dict(id=['featured-item', 'article-content']),
+        dict(attrs={'class':lambda x: x and 'ArticleHeader__dek___' in x}),
+        dict(attrs={'class':lambda x: x and 'Byline__articleHeader___' in x}),
+        dict(attrs={'class':lambda x: x and 'ArticleLedeImage__container___' in x}),
+        dict(id='articleBody'),
+        dict(attrs={'class':lambda x: x and 'ArticleDisclaimer__articleDisclaimer___' in x}),
+        dict(attrs={'class':lambda x: x and 'ArticleContributors__bio___' in x}),
     ]
     remove_tags = [
         dict(attrs={'class': lambda x: x and set(x.split()).intersection(
@@ -39,69 +49,47 @@ class NewYorker(BasicNewsRecipe):
     def parse_index(self):
         soup = self.index_to_soup(
             'https://www.newyorker.com/magazine?intcid=magazine')
-        ph = soup.find(
+        # soup = self.index_to_soup('file:///t/raw.html')
-            'div', attrs={'class': lambda x: x and 'cover-info' in x.split()})
+        cover_img =  soup.find(attrs={'class': lambda x: x and 'MagazineCover__cover___' in x})
-        if ph is not None:
+        if cover_img is not None:
-            img = ph.find('img')
+            cover_img = cover_img.find('img')
-            if img is not None:
+            if cover_img is not None:
-                try:
+                self.cover_url = cover_img.get('src', cover_img.get('data-src', cover_img.get('srcset').split()[0]))
-                    self.cover_url = img['data-src']
+                self.log('Found cover:', self.cover_url)
-                except KeyError:
+        stories = defaultdict(list)
-                    self.cover_url = img['src']
+        last_section = 'Unknown'
-        articles = []
+        for story in soup.findAll(attrs={'class': lambda x: x and 'River__riverItemContent___' in x}):
-        current_section = 'Current Issue'
+            try:
-        feeds = []
+                section = self.tag_to_string(story.find('a')['title']) or last_section
-        for story in soup.findAll(['h5', 'article']):
+            except KeyError:
-            if story.name == 'h5':
+                section = last_section
-                if articles:
+            last_section = section
-                    feeds.append((current_section, articles))
+            a = story.find('h4').find('a')
-                current_section, articles = self.tag_to_string(story), []
+            title = a.contents[1]
-                self.log('\nFound section: ' + current_section)
+            url = absurl(a['href'])
-                continue
-            if story['itemtype'] != 'http://schema.org/Article':
-                continue
-            h2 = story.find('h2')
-            url = h2.find('a', href=True)['href']
-            a = h2.find('a')
-            title = self.tag_to_string(a)
-            h3 = h2.findNextSibling('h3')
             desc = ''
-            if h3 is not None:
+            body = story.find(attrs={'class': 'River__dek___CayIg'})
-                desc += self.tag_to_string(h3)
+            if body is not None:
-            p = h2.findNextSibling('p')
+                desc = body.contents[0]
-            if p is not None:
+            self.log('Found article:', title)
-                desc += '. \n' + self.tag_to_string(p)
+            self.log('\t' + url)
+            self.log('\t' + desc)
+            self.log('')
+            stories[section].append({'title':title, 'url':url, 'description':desc})
 
-            self.log('  ', title)
+        return [(k, stories[k]) for k in sorted(stories)]
-            self.log('    ', url)
-            if desc:
-                self.log('    ', desc)
-            articles.append({'title': title, 'url': url, 'date': '',
-                             'description': desc})
-        if articles:
-            feeds.append((current_section, articles))
-
-        return feeds
 
     def preprocess_html(self, soup):
         for img in soup.findAll('img'):
             try:
-                ds = img['data-src']
+                ds = img['srcset'].split()[0]
+                del img['srcset']
             except KeyError:
                 continue
             if ds:
                 img['src'] = ds
         return soup
 
-    def postprocess_html(self, soup, *a):
-        ab = soup.find(id='articleBody')
-        if ab is not None:
-            fi = soup.find(id='featured-item')
-            if fi is not None:
-                p = fi.parent
-                p.insert(len(p) - 2, fi)
-        return soup
-
     # The New Yorker changes the content it delivers based on cookies, so the
     # following ensures that we send no cookies
     def get_browser(self, *args, **kwargs):