From 634871213f3d6503673c385ce0a40e380de12398 Mon Sep 17 00:00:00 2001 From: unkn0w7n <51942695+unkn0w7n@users.noreply.github.com> Date: Thu, 5 Sep 2024 18:16:10 +0530 Subject: [PATCH] update ORF replace google feeds --- ...ver_reach_foundation.png => orfonline.png} | Bin recipes/observer_reach_foundation.recipe | 86 -------------- recipes/orfonline.recipe | 110 ++++++++++++++++++ 3 files changed, 110 insertions(+), 86 deletions(-) rename recipes/icons/{observer_reach_foundation.png => orfonline.png} (100%) delete mode 100644 recipes/observer_reach_foundation.recipe create mode 100644 recipes/orfonline.recipe diff --git a/recipes/icons/observer_reach_foundation.png b/recipes/icons/orfonline.png similarity index 100% rename from recipes/icons/observer_reach_foundation.png rename to recipes/icons/orfonline.png diff --git a/recipes/observer_reach_foundation.recipe b/recipes/observer_reach_foundation.recipe deleted file mode 100644 index 2aa3833c4e..0000000000 --- a/recipes/observer_reach_foundation.recipe +++ /dev/null @@ -1,86 +0,0 @@ -from urllib.parse import quote - -from calibre.web.feeds.news import BasicNewsRecipe, classes - - -class ORF(BasicNewsRecipe): - title = 'Observer Research Foundation' - description = ( - 'Set up in 1990, ORF seeks to lead and aid policy thinking towards building a strong and prosperous India' - ' in a fair and equitable world. It helps discover and inform India’s choices, and carries Indian voices ' - 'and ideas to forums shaping global debates. ' - ) - language = 'en_IN' - __author__ = 'unkn0wn' - encoding = 'utf-8' - no_stylesheets = True - remove_javascript = True - masthead_url = 'https://www.orfonline.org/wp-content/uploads/2015/09/Logo_ORF_JPEG.jpg' - remove_attributes = ['style', 'height', 'width'] - ignore_duplicate_articles = {'url', 'title'} - reverse_article_order = True - remove_empty_feeds = True - simultaneous_downloads = 1 - art_url = '' - - extra_css = ''' - img {display:block; margin:0 auto;} - .report-slider, .author_panel {font-size:small; color:#404040;} - .report {font-size:small; font-weight:bold;} - .excert-italic, .recent-block-people {font-style:italic; color:#202020;} - blockquote, em {color:#202020;} - .espert_speak_panel {font-size:small;} - ''' - - articles_are_obfuscated = True - - def get_obfuscated_article(self, url): - br = self.get_browser() - soup = self.index_to_soup(url) - link = soup.a['href'] - skip_sections =[ # add sections you want to skip - '/video/', '/videos/', '/multimedia/', 'marathi', 'hindi', 'bangla' - ] - if any(x in link for x in skip_sections): - self.abort_article('skipping video links ', link) - self.log('Found ', link) - self.art_url = link - html = br.open(link).read() - return ({ 'data': html, 'url': link }) - - - def get_browser(self): - return BasicNewsRecipe.get_browser(self, user_agent='common_words/based') - - resolve_internal_links = True - remove_empty_feeds = True - - keep_only_tags = [ - dict(name='h1'), - classes('author_panel espert_speak_panel expert_panel_content') - ] - remove_tags = [ - classes( - 'social socialshare comment-area-section telegramhtml post-tag ' - 'research-prev research-next col-md-4 button_group sharethis-p tags' - ) - ] - - feeds = [] - - when = '170' # hours > 7 days - index = 'https://www.orfonline.org' - - sections = [ - 'expert-speak', 'books', 'event-reports', 'events', 'forums', 'gp-orf-series', 'issue-brief', 'monitors', - 'occasional-paper', 'primer', 'series', 'surveys-polls', 'young-voices', 'research' - ] - a = 'https://news.google.com/rss/search?q=when:{}h+allinurl:{}&hl=en-IN&gl=IN&ceid=IN:en' - for sec in sections: - sec_index = index + '/' + sec + '/' - feeds.append((sec.capitalize(), a.format(when, quote(sec_index, safe='')))) - feeds.append(('Others', a.format(when, quote(index, safe='')))) - - def populate_article_metadata(self, article, soup, first): - article.url = self.art_url - article.title = article.title.replace(' - Observer Research Foundation', '') diff --git a/recipes/orfonline.recipe b/recipes/orfonline.recipe new file mode 100644 index 0000000000..dfa5662c7f --- /dev/null +++ b/recipes/orfonline.recipe @@ -0,0 +1,110 @@ +#!/usr/bin/env python +from datetime import datetime, timedelta +from calibre.utils.date import parse_date +from calibre.web.feeds.news import BasicNewsRecipe, classes + + +class ORF(BasicNewsRecipe): + title = 'Observer Research Foundation' + description = ( + 'Set up in 1990, ORF seeks to lead and aid policy thinking towards building a strong and prosperous India' + ' in a fair and equitable world. It helps discover and inform India’s choices, and carries Indian voices ' + 'and ideas to forums shaping global debates.' + ) + language = 'en_IN' + __author__ = 'unkn0wn' + encoding = 'utf-8' + no_stylesheets = True + remove_javascript = True + masthead_url = 'https://www.orfonline.org/wp-content/uploads/2015/09/Logo_ORF_JPEG.jpg' + remove_attributes = ['style', 'height', 'width'] + ignore_duplicate_articles = {'url', 'title'} + remove_empty_feeds = True + oldest_article = 7 + simultaneous_downloads = 1 + + recipe_specific_options = { + 'days': { + 'short': 'Oldest article to download from this news source. In days ', + 'long': 'For example, 0.5, gives you articles from the past 12 hours', + 'default': str(oldest_article) + } + } + + def __init__(self, *args, **kwargs): + BasicNewsRecipe.__init__(self, *args, **kwargs) + d = self.recipe_specific_options.get('days') + if d and isinstance(d, str): + self.oldest_article = float(d) + + extra_css = ''' + img {display:block; margin:0 auto;} + .report-slider, .author_panel {font-size:small; color:#404040;} + .report {font-size:small; font-weight:bold;} + .excert-italic, .recent-block-people {font-style:italic; color:#202020;} + blockquote, em {color:#202020;} + .espert_speak_panel {font-size:small;} + ''' + + def get_browser(self): + return BasicNewsRecipe.get_browser(self, user_agent='common_words/based') + + resolve_internal_links = True + remove_empty_feeds = True + + keep_only_tags = [ + dict(name='h1'), + classes('author_panel espert_speak_panel expert_panel_content') + ] + remove_tags = [ + classes( + 'social socialshare comment-area-section telegramhtml post-tag ' + 'research-prev research-next col-md-4 button_group sharethis-p tags' + ) + ] + + def parse_index(self): + index = 'https://www.orfonline.org' + secs = [ + ('Expert Speak', '/expert-speak'), + ('Young Voices', '/expert-speak-category/young-voices'), + ('Commentary', '/content-type/commentary'), + ('Issue Briefs', '/content-type/issue-briefs'), + ('Books', '/content-type/books'), + ('Special Reports', '/content-type/special-reports') + ] + + feeds = [] + + for x, y in secs: + section = x + self.log(section) + + articles = [] + + soup = self.index_to_soup(index + y) + for a in soup.findAll('a', attrs={'href':True}): + if not a.find('h3'): + continue + div = a.find_previous_sibling(attrs={'class':'topic_story'}) + if div: + if div.find(**classes('show_date')): + date = parse_date(self.tag_to_string( + div.find(**classes('show_date')) + ) + ).replace(tzinfo=None) + if (datetime.now() - date) > timedelta(self.oldest_article): + self.log(' Skipping ', self.tag_to_string(a)) + continue + url = a['href'] + title = self.tag_to_string(a) + desc = '' + if a.find_next_sibling('p'): + desc = self.tag_to_string(a.find_next_sibling('p')) + if a.find_next_sibling(attrs={'class':'author_name'}): + desc = self.tag_to_string(a.find_next_sibling(attrs={'class':'author_name'})) + ' | ' + desc + self.log('\t', title, '\n\t', desc, '\n\t\t', url) + articles.append({'title': title, 'url': url, 'description': desc}) + if articles: + feeds.append((section, articles)) + return feeds