Merge branch 'master' of https://github.com/unkn0w7n/calibre

recipes/newsminute.recipe

@@ -0,0 +1,70 @@
+from calibre.web.feeds.news import BasicNewsRecipe, classes
+from calibre.ptempfile import PersistentTemporaryFile
+
+
+class newsminute(BasicNewsRecipe):
+    title = 'The News Minute'
+    __author__ = 'unkn0wn'
+    description = (
+        'The News Minute is a digital news platform reporting and writing on issues in India, with a '
+        'specific focus on the 5 southern states. Our content includes news, ground reportage, news '
+        'analysis, opinion and blogs. Our core strengths include our deep access in the southern states, '
+        'incisive editorial acumen and insightful news analysis and opinions.'
+    )
+    language = 'en_IN'
+
+    no_stylesheets = True
+    remove_javascript = True
+    masthead_url = 'https://pkcindia.com/wp-content/uploads/2021/09/TMN-Logo-1.png'
+    ignore_duplicate_articles = {'title', 'url'}
+    resolve_internal_links  = True
+    remove_empty_feeds = True
+    remove_attributes = ['style', 'height', 'width']
+    articles_are_obfuscated = True
+
+    def get_obfuscated_article(self, url):
+        br = self.get_browser()
+        try:
+            br.open(url)
+        except Exception as e:
+            url = e.hdrs.get('location')
+        soup = self.index_to_soup(url)
+        link = soup.find('a', href=True)
+        skip_sections =[ # add sections you want to skip
+            '/video/', '/videos/', '/media/', 'podcast-'
+        ]
+        if any(x in link['href'] for x in skip_sections):
+            self.log('Aborting Article ', link['href'])
+            self.abort_article('skipping video links')
+
+        self.log('Downloading ', link['href'])
+        html = br.open(link['href']).read()
+        pt = PersistentTemporaryFile('.html')
+        pt.write(html)
+        pt.close()
+        return pt.name
+
+    keep_only_tags = [
+        classes(
+            'arr--section-name arr--story--headline-h1 arr--sub-headline arr--hero-image author-card-wrapper arr--story-page-card-wrapper'
+        ),
+    ]
+
+    feeds = []
+
+    sections = [
+        'tamil-nadu', 'telangana', 'andhra-pradesh', 'karnataka', 'kerala'
+    ]
+
+    for sec in sections:
+        a = 'https://news.google.com/rss/search?q=when:27h+allinurl:https%3A%2F%2Fwww.thenewsminute.com{}&hl=en-IN&gl=IN&ceid=IN:en'
+        feeds.append((sec.capitalize(), a.format('%2F' + sec + '%2F')))
+    feeds.append(('Others', a.format('')))
+
+    def populate_article_metadata(self, article, soup, first):
+        article.title = article.title.replace(' - The News Minute', '')
+
+    def preprocess_html(self, soup):
+        for img in soup.findAll('img', attrs={'data-src':True}):
+            img['src'] = img['data-src']
+        return soup

recipes/project_syndicate.recipe

@@ -0,0 +1,71 @@
+from calibre.web.feeds.news import BasicNewsRecipe, classes
+from calibre.scraper.simple import read_url
+from calibre.ptempfile import PersistentTemporaryFile
+
+class projectsynd(BasicNewsRecipe):
+    title = 'Project Syndicate'
+    __author__ = 'unkn0wn'
+    description = (
+        'Project Syndicate produces and delivers original, high-quality commentaries to a global audience. '
+        'Featuring exclusive contributions by prominent political leaders, policymakers, scholars, business '
+        'leaders, and civic activists from around the world, we provide news media and their readers with cutting-edge '
+        'analysis and insight, regardless of ability to pay. Our membership includes over 500 media outlets – more '
+        'than half of which receive our commentaries for free or at subsidized rates – in 156 countries.'
+    )
+    language = 'en'
+    no_stylesheets = True
+    remove_javascript = True
+    masthead_url = 'https://www.project-syndicate.org/images/PS_Logotype.svg'
+    ignore_duplicate_articles = {'title', 'url'}
+    resolve_internal_links  = True
+    remove_empty_feeds = True
+    remove_attributes = ['style', 'height', 'width']
+    oldest_article = 7 # days
+    storage = []
+
+    articles_are_obfuscated = True
+    def get_obfuscated_article(self, url):
+        raw = read_url(self.storage, 'https://archive.is/latest/' + url)
+        pt = PersistentTemporaryFile('.html')
+        pt.write(raw.encode('utf-8'))
+        pt.close()
+        return pt.name
+
+    extra_css = '''
+        [itemprop^="associatedMedia"]{ font-size:small; text-align:center; }
+        [itemprop="author"], time { font-size:small; }
+        .sub, em { font-style:italic; color:#202020; }
+    '''
+
+    keep_only_tags = [
+        dict(attrs={'itemprop':lambda x: x and 'associatedMedia' in x.split()}),
+        dict(attrs={'itemprop':['headline', 'datePublished', 'author', 'abstract', 'articleBody']}),
+    ]
+    
+    remove_tags = [
+        dict(name=['button', 'svg']),
+        dict(attrs={'data-message-area':True}),
+        dict(attrs={'id':['editorspicks', 'movie_player']}),
+    ]
+
+    feeds = [
+        ('Economics & Finance', 'https://www.project-syndicate.org/rss/section/economics'),
+        ('Politics & World Affairs', 'https://www.project-syndicate.org/rss/section/politics-world-affairs'),
+        ('On-Point', 'https://www.project-syndicate.org/rss/onpoint'),
+        ('Sustainability Now', 'https://www.project-syndicate.org/rss/section/environment-sustainability'),
+        ('Smart Development', 'https://www.project-syndicate.org/rss/section/global-health-development'),
+        ('Culture & Society', 'https://www.project-syndicate.org/rss/section/culture-society'),
+        ('Innovation & Technology', 'https://www.project-syndicate.org/rss/section/innovation-technology'),
+    ]
+
+    def preprocess_html(self, soup):
+        for img in soup.findAll('img', attrs={'old-src':True}):
+            img['src'] = img['old-src'].replace('medium', 'xlarge')
+        if abst := soup.find(attrs={'itemprop':'abstract'}).find('div'):
+            abst.name = 'p'
+            abst['class'] = 'sub'
+        for div in soup.findAll('div', attrs={'data-line-id':True}):
+            div.name = 'p'
+        for a in soup.findAll('a', href=True):
+            a['href'] = 'http' + a['href'].split('http')[-1]
+        return soup