Update ABC News

recipes/abc_au.recipe

@@ -1,61 +1,94 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+from __future__ import unicode_literals, division, absolute_import, print_function
+
 __license__ = 'GPL v3'
-__copyright__ = '2011, Pat Stapleton <pat.stapleton at gmail.com>'
+__copyright__ = '2020, Pat Stapleton <pat.stapleton at gmail.com>'
 '''
-abc.net.au/news
+Recipe for ABC News Australia (online)
 '''
-import re
+from calibre.web.feeds.news import BasicNewsRecipe
-from calibre.web.feeds.recipes import BasicNewsRecipe
 
 
 class ABCNews(BasicNewsRecipe):
     title          = 'ABC News'
-    __author__ = 'Pat Stapleton, Dean Cording, James Cridland'
+    language       = 'en_AU'
+    __author__     = 'Pat Stapleton'
     description = 'From the Australian Broadcasting Corporation. The ABC is owned and funded by the Australian Government, but is editorially independent.'
     masthead_url = 'https://www.abc.net.au/cm/lb/8212706/data/news-logo-2017---desktop-print-data.png'
     cover_url = 'https://www.abc.net.au/news/linkableblob/8413676/data/abc-news-og-data.jpg'
     cover_margins = (0,20,'#000000')
-    oldest_article = 2
-    handle_gzip = True
-    no_stylesheets = True
-    use_embedded_content = False
     scale_news_images_to_device = True
-    encoding = 'utf8'
+    oldest_article = 7  # days
-    publisher = 'ABC News'
+    max_articles_per_feed = 100
-    category = 'Australia,News'
-    language = 'en_AU'
     publication_type = 'newspaper'
-    extra_css = '.byline{font-size:smaller;margin-bottom:10px;}.inline-caption{display:block;font-size:smaller;text-decoration: none;}'
+
-    preprocess_regexps = [(re.compile(
+#    auto_cleanup   = True # enable this as a backup option if recipe stops working
-        r'<a class="inline-caption" href="http://maps\.google\.com.*?/a>', re.DOTALL), lambda m: '')]  # Remove map links
+
-    conversion_options = {
+#    use_embedded_content = False # if set to true will assume that all the article content is within the feed (i.e. won't try to fetch more data)
-        'comments': description, 'tags': category, 'language': language, 'publisher': publisher, 'linearize_tables': False
+
+    no_stylesheets = True
+    remove_javascript = True
+
+    keep_only_tags = [dict(id='content')]  # the article content is contained in <main id="content" /> tag
+
+    # ************************************
+    # Regular expressions for remove_tags:
+    # ************************************
+    # remove aside tag - used for overlapping boxes within article
+    # aside_reg_exp = '^.*aside.*$'
+
+    # ************************************
+    # Clear out all the unwanted html tags:
+    # ************************************
+    remove_tags = [
+#        dict(name='aside', attrs={'name': re.compile(aside_reg_exp, re.IGNORECASE)})
+        {
+            'name': ['meta', 'link', 'noscript', 'aside']
+        },
+        {
+            'attrs': {
+                'data-component': ['Ticker', 'PublishedDate', 'Timestamp', 'Link', 'ShareLink', 'ShareUtility',
+                'RelatedStories', 'ArticleTopStories', 'ArticleTopStoriesCard', 'ArticleJustInStories',
+                'RelatedTopics', 'Player', 'ArticleSidebar', 'TopStoriesSidebar', 'UtilityBar']
             }
+        }
+    ]
 
-    keep_only_tags = [dict(attrs={'class': ['article section']})]
+    # ************************************
+    # Tidy up the output to look neat for reading
+    # ************************************
+    remove_attributes = ['width', 'height', 'style']
+    extra_css = '.byline{font-size:smaller;margin-bottom:10px;}.inline-caption{display:block;font-size:smaller;text-decoration: none;}'
 
-    remove_tags = [dict(attrs={'class': ['inner', 'attached-content', 'authorpromo', 'related', 'tags', 'tools', 'attached-content ready',
+    # ************************************
-                                         'inline-content story left', 'inline-content map left contracted',
+    # Fix images (dynamically generated by ABC news)
-                                         'inline-content full embedYouTube embedded', 'published',
+    # ************************************
-                                         'story-map', 'statepromo', 'topics', ]})]
+    def preprocess_html(self, soup):
-    # inner = key points
+        for img in soup.findAll('img', attrs={'data-src': True}):
-    # attached-content = related stories
+            for x in img['data-src'].split():
-    # authorpromo = "Contact NameOfJournalist"
+                if '/' in x:
-
+                    img['src'] = x
-    remove_attributes = ['width', 'height']
+        return soup
+    compress_news_images = True
 
     feeds          = [
         ('Top Stories', 'https://www.abc.net.au/news/feed/45910/rss.xml'),
-        ('Politics', 'https://www.abc.net.au/news/feed/51120/rss.xml'),
+        ('Politics', 'https://www.abc.net.au/news/feed/1534/rss.xml'),
-        ('World', 'https://www.abc.net.au/news/feed/6497190/rss.xml'),
+        ('World', 'https://www.abc.net.au/news/feed/4405318/rss.xml'),
         ('Business', 'https://www.abc.net.au/news/feed/51892/rss.xml'),
-        ('Analysis', 'https://www.abc.net.au/news/feed/7571224/rss.xml'),
+        ('Analysis', 'https://www.abc.net.au/news/feed/7571268/rss.xml'),
         ('Sport', 'https://www.abc.net.au/news/feed/2942460/rss.xml'),
+        ('Science', 'https://www.abc.net.au/news/feed/8132426/rss.xml'),
+        ('Health', 'https://www.abc.net.au/news/feed/9167762/rss.xml'),
+        ('Arts and Entertainment', 'https://www.abc.net.au/news/feed/472/rss.xml'),
+        ('Fact Check', 'https://www.abc.net.au/news/feed/5306468/rss.xml'),
         ('Adelaide', 'https://www.abc.net.au/news/feed/8057540/rss.xml'),
-        ('Brisbane', 'https://www.abc.net.au/news/feed/12858/rss.xml'),
+        ('Brisbane', 'https://www.abc.net.au/news/feed/8053540/rss.xml'),
-        ('Canberra', 'https://www.abc.net.au/news/feed/6910/rss.xml'),
+        ('Canberra', 'https://www.abc.net.au/news/feed/8057234/rss.xml'),
         ('Darwin', 'https://www.abc.net.au/news/feed/8057648/rss.xml'),
         ('Hobart', 'https://www.abc.net.au/news/feed/8054562/rss.xml'),
-        ('Melbourne', 'https://www.abc.net.au/news/feed/21708/rss.xml'),
+        ('Melbourne', 'https://www.abc.net.au/news/feed/8057136/rss.xml'),
-        ('Sydney', 'https://www.abc.net.au/news/feed/10232/rss.xml'),
+        ('Perth', 'https://www.abc.net.au/news/feed/8057096/rss.xml'),
-        ('Perth', 'https://www.abc.net.au/news/feed/24886/rss.xml'),
+        ('Sydney', 'https://www.abc.net.au/news/feed/8055316/rss.xml'),
     ]