Create hackernews_with_comments.recipe

2025-07-09 03:04:10 -04:00 · 2024-09-04 12:55:57 +05:30 · 2024-09-04 12:55:57 +05:30 · aa2d5b8c5f
commit aa2d5b8c5f
parent d0556746f2
1 changed files with 149 additions and 0 deletions
--- a/recipes/hackernews_with_comments.recipe
+++ b/recipes/hackernews_with_comments.recipe
@ -0,0 +1,149 @@
+#!/usr/bin/env python
+
+__license__ = 'GPL v3'
+'''
+Hacker News (with comments)
+'''
+from calibre.ptempfile import PersistentTemporaryFile
+from calibre.web.feeds.news import BasicNewsRecipe
+
+try:
+    from urllib.parse import urlparse
+except ImportError:
+    from urlparse import urlparse
+import re
+
+
+class HNWithComments(BasicNewsRecipe):
+    title = 'HN With actual comments'
+    __author__ = 'Tom Scholl & David Kerschner'
+    description = u'Hacker News, run by Y Combinator. Anything that good hackers would find interesting, with a focus on programming and startups.'
+    publisher = 'Y Combinator'
+    category = 'news, programming, it, technology'
+    delay = 1
+    max_articles_per_feed = 20
+    oldest_article = 3
+    use_embedded_content = False
+    no_stylesheets = True
+    encoding = 'utf-8'
+    language = 'en'
+    requires_version = (0, 8, 16)
+
+    feeds = [
+        (u'Hacker News Frontpage', 'https://hnrss.org/frontpage'),
+        (u'Ask Hacker News', 'https://hnrss.org/ask')
+    ]
+
+    temp_files = []
+    articles_are_obfuscated = True
+
+    def get_readable_content(self, url):
+        self.log('get_readable_content(' + url + ')')
+        br = self.get_browser()
+        f = br.open(url)
+        html = f.read()
+        f.close()
+
+        return self.extract_readable_article(html, url)
+
+    def get_hn_content(self, url):
+        self.log('get_hn_content(' + url + ')')
+        soup = self.index_to_soup(url)
+        main = soup.find('tr').findNextSiblings('tr', limit=2)[1].td
+
+        title_element = main.select('td.title .titleline a')[0]
+        self.log('title_element=' + repr(title_element))
+        title = self.tag_to_string(title_element)
+        self.log('title=' + title)
+        link = title_element['href']
+        # link = main.find('td', 'title').find('a')['href']
+        if link.startswith('item?'):
+            link = 'https://news.ycombinator.com/' + link
+        readable_link = link.rpartition('http://')[2].rpartition('https://')[2]
+        subtext = self.tag_to_string(main.find('td', 'subtext'))
+
+        title_content_td = main.find('td', 'title').findParent(
+            'tr').findNextSiblings('tr', limit=3)[2].findAll('td', limit=2)[1]
+        title_content = u''
+        if not title_content_td.find('form'):
+            title_content_td.name = 'div'
+            title_content = title_content_td.prettify()
+
+        comments = u''
+        for td in main.findAll('td', 'default'):
+            comhead = td.find('span', 'comhead')
+            if comhead:
+                com_title = u'<h4>' + \
+                    self.tag_to_string(comhead).replace(
+                        ' | link', '') + u'</h4>'
+                comhead.parent.extract()
+                br = td.find('br')
+                if br:
+                    br.extract()
+                reply = td.find('a', attrs={'href': re.compile('^reply?')})
+                if reply:
+                    reply.parent.extract()
+                td.name = 'div'
+                indent_width = (int(td.parent.find('td').img['width']) * 2) / 3
+                td['style'] = 'padding-left: ' + str(indent_width) + 'px'
+                comments = comments + com_title + td.prettify()
+
+        body = u'<h3>' + title + u'</h3><p><a href="' + link + u'">' + readable_link + \
+            u'</a><br/><strong>' + subtext + u'</strong></p>' + title_content + u'<br/>'
+        body = body + comments
+        return u'<html><title>' + title + u'</title><body>' + body + '</body></html>'
+
+    def parse_feeds(self):
+        a = super(HNWithCommentsLinkAlt, self).parse_feeds()
+        self.hn_articles = a[0].articles
+        return a
+
+    def get_obfuscated_article(self, url):
+        self.log('get_obfuscated_article with url=' + url)
+        if url.startswith('https://news.ycombinator.com'):
+            content = self.get_hn_content(url)
+        else:
+            # TODO: use content-type header instead of url
+            is_image = False
+            for ext in ['.jpg', '.png', '.svg', '.gif', '.jpeg', '.tiff', '.bmp', ]:
+                if url.endswith(ext):
+                    is_image = True
+                    break
+
+            if is_image:
+                self.log('using image_content (' + url + ')')
+                content = u'<html><body><img src="' + url + u'"></body></html>'
+            else:
+                content = self.get_readable_content(url)
+
+            article = 0
+            for a in self.hn_articles:
+                if a.url == url:
+                    article = a
+
+        # content = re.sub(r'</body>\s*</html>\s*$', '', content) + \
+        #    article.summary + '</body></html>'
+
+        if not isinstance(content, bytes):
+            content = content.encode('utf-8')
+        self.temp_files.append(PersistentTemporaryFile('_fa.html'))
+        self.temp_files[-1].write(content)
+        self.temp_files[-1].close()
+        return self.temp_files[-1].name
+
+    def is_link_wanted(self, url, tag):
+        if url.endswith('.pdf'):
+            return False
+        return True
+
+    def prettyify_url(self, url):
+        return urlparse(url).hostname
+
+    def populate_article_metadata(self, article, soup, first):
+        article.text_summary = self.prettyify_url(article.url)
+        article.summary = article.text_summary
+
+#    def parse_index(self):
+#        feeds = []
+#        feeds.append((u'Hacker News',[{'title': 'Testing', 'url': 'https://news.ycombinator.com/item?id=2935944'}]))
+#        return feeds