Australian Financial Review by unkn0wn

recipes/afr.recipe

@@ -0,0 +1,89 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.ptempfile import PersistentTemporaryFile
+
+
+class afr(BasicNewsRecipe):
+    title = 'Australian Financial Review'
+    __author__ = 'unkn0wn'
+    description = (
+        'For more than 65 years The Australian Financial Review has been the authority on business,'
+        ' finance and investment news in Australia. It has a reputation for independent, award-winning '
+        'journalism and is essential reading for Australia\'s business and investor community.'
+    )
+    masthead_url = 'https://www.nineforbrands.com.au/wp-content/uploads/2020/08/AFR-DHOSP-Logo-black-RGB.png'
+    encoding = 'utf-8'
+    language = 'en_AU'
+
+    use_embedded_content = False
+    timefmt = ' [%d %b %Y]'
+    max_articles_per_feed = 25
+    no_stylesheets = True
+    remove_empty_feeds = True
+    remove_attributes = ['style', 'height', 'width']
+
+    keep_only_tags = [
+        dict(name=['article', 'main'], attrs={'id':'content'})
+    ]
+
+    remove_tags = [
+        dict(attrs={'data-testid': [
+                'ArticleTools', 'ArticleBreadcrumb-Links', 'ad-wrapper', 'ArticleFooter', 'ArticleTags',
+                'beyondwords-player-wrapper'
+        ]}),
+        dict(name=['button', 'aside', 'svg']),
+    ]
+
+    remove_tags_after= [ dict(name='aside', attrs={'id':'stickyContainer'})]
+
+    extra_css = '''
+        #img-cap {font-size:small; text-align:center;}
+        [data-testid="AuthorNames"], [data-testid="ArticleTimestamp"] {font-size:small;}
+    '''
+
+    ignore_duplicate_articles = {'title'}
+    resolve_internal_links  = True
+    remove_empty_feeds = True
+
+    articles_are_obfuscated = True
+
+    def get_obfuscated_article(self, url):
+        br = self.get_browser()
+        try:
+            br.open(url)
+        except Exception as e:
+            url = e.hdrs.get('location')
+        soup = self.index_to_soup(url)
+        link = soup.find('a', href=True)
+        skip_sections =[ # add sections you want to skip
+            '/video/', '/videos/', '/media/', 'podcast-'
+        ]
+        if any(x in link['href'] for x in skip_sections):
+            self.log('Aborting Article ', link['href'])
+            self.abort_article('skipping video links')
+
+        self.log('Downloading ', link['href'])
+        html = br.open(link['href']).read()
+        pt = PersistentTemporaryFile('.html')
+        pt.write(html)
+        pt.close()
+        return pt.name
+
+    def preprocess_html(self, soup):
+        for img in soup.findAll('img', attrs={'data-src':True}):
+            img['src'] = img['data-src']
+        for fig in soup.findAll('figcaption'):
+            fig['id'] = 'img-cap'
+        return soup
+
+    feeds = []
+
+    sections = [
+        'companies', 'market', 'politics', 'policy', 'world', 'wealth', 'street-talk',
+        'chaticleer', 'rear-window', 'life-and-luxury', 'technology', 'property',
+        'work-and-careers',
+    ]
+
+    for sec in sections:
+        a = 'https://news.google.com/rss/search?q=when:27h+allinurl:https%3A%2F%2Fwww.afr.com{}&hl=en-AU&gl=AU&ceid=AU:en'
+        feeds.append((sec.capitalize(), a.format('%2F' + sec + '%2F')))
+    feeds.append(('Others', a.format('')))