diff --git a/recipes/icons/ancient_egypt.png b/recipes/icons/ancient_egypt.png index 8eb9f7f334..d27808ace4 100644 Binary files a/recipes/icons/ancient_egypt.png and b/recipes/icons/ancient_egypt.png differ diff --git a/recipes/icons/business_standard.png b/recipes/icons/business_standard.png index 41800ff4f1..47711f71c8 100644 Binary files a/recipes/icons/business_standard.png and b/recipes/icons/business_standard.png differ diff --git a/recipes/icons/business_standard_print.png b/recipes/icons/business_standard_print.png index 41800ff4f1..47711f71c8 100644 Binary files a/recipes/icons/business_standard_print.png and b/recipes/icons/business_standard_print.png differ diff --git a/recipes/icons/harpers.png b/recipes/icons/harpers.png index 2bcd486ad1..422b94014d 100644 Binary files a/recipes/icons/harpers.png and b/recipes/icons/harpers.png differ diff --git a/recipes/icons/horizons.png b/recipes/icons/horizons.png index 28f98997f0..7207f0b1a7 100644 Binary files a/recipes/icons/horizons.png and b/recipes/icons/horizons.png differ diff --git a/recipes/icons/lex_fridman_podcast.png b/recipes/icons/lex_fridman_podcast.png index 18427862c2..6b2a6ab3da 100644 Binary files a/recipes/icons/lex_fridman_podcast.png and b/recipes/icons/lex_fridman_podcast.png differ diff --git a/recipes/icons/military_history.png b/recipes/icons/military_history.png index c991353513..52cf036779 100644 Binary files a/recipes/icons/military_history.png and b/recipes/icons/military_history.png differ diff --git a/recipes/icons/minerva_magazine.png b/recipes/icons/minerva_magazine.png index 1e3585c000..53e1124af9 100644 Binary files a/recipes/icons/minerva_magazine.png and b/recipes/icons/minerva_magazine.png differ diff --git a/recipes/icons/mit_technology_review.png b/recipes/icons/mit_technology_review.png index 9a731e5127..8fb14fe9d0 100644 Binary files a/recipes/icons/mit_technology_review.png and b/recipes/icons/mit_technology_review.png differ diff --git a/recipes/icons/natgeo.png b/recipes/icons/natgeo.png index 8a7f1b583d..69834ab831 100644 Binary files a/recipes/icons/natgeo.png and b/recipes/icons/natgeo.png differ diff --git a/recipes/icons/natgeohis.png b/recipes/icons/natgeohis.png index 8a7f1b583d..69834ab831 100644 Binary files a/recipes/icons/natgeohis.png and b/recipes/icons/natgeohis.png differ diff --git a/recipes/icons/natgeomag.png b/recipes/icons/natgeomag.png index 8a7f1b583d..69834ab831 100644 Binary files a/recipes/icons/natgeomag.png and b/recipes/icons/natgeomag.png differ diff --git a/recipes/icons/nautilus.png b/recipes/icons/nautilus.png index eeeb38d263..89c00fc814 100644 Binary files a/recipes/icons/nautilus.png and b/recipes/icons/nautilus.png differ diff --git a/recipes/icons/new_scientist_mag.png b/recipes/icons/new_scientist_mag.png index 844c1e951c..cc04bfb404 100644 Binary files a/recipes/icons/new_scientist_mag.png and b/recipes/icons/new_scientist_mag.png differ diff --git a/recipes/icons/nikkeiasia.png b/recipes/icons/nikkeiasia.png index 92f1ee523f..65172b9f88 100644 Binary files a/recipes/icons/nikkeiasia.png and b/recipes/icons/nikkeiasia.png differ diff --git a/recipes/icons/observer_reach_foundation.png b/recipes/icons/orfonline.png similarity index 100% rename from recipes/icons/observer_reach_foundation.png rename to recipes/icons/orfonline.png diff --git a/recipes/icons/the_week_magazine_free.png b/recipes/icons/the_week_magazine_free.png index 28afccf9e3..073161d5b6 100644 Binary files a/recipes/icons/the_week_magazine_free.png and b/recipes/icons/the_week_magazine_free.png differ diff --git a/recipes/icons/the_week_uk.png b/recipes/icons/the_week_uk.png index 28afccf9e3..073161d5b6 100644 Binary files a/recipes/icons/the_week_uk.png and b/recipes/icons/the_week_uk.png differ diff --git a/recipes/icons/times_online.png b/recipes/icons/times_online.png index 61447d8965..94bf20cda2 100644 Binary files a/recipes/icons/times_online.png and b/recipes/icons/times_online.png differ diff --git a/recipes/icons/tls_mag.png b/recipes/icons/tls_mag.png index 21391b1fec..2ff2938c1f 100644 Binary files a/recipes/icons/tls_mag.png and b/recipes/icons/tls_mag.png differ diff --git a/recipes/icons/toi.png b/recipes/icons/toi.png index 1c6911c2b7..fbf1778e7c 100644 Binary files a/recipes/icons/toi.png and b/recipes/icons/toi.png differ diff --git a/recipes/icons/toiprint.png b/recipes/icons/toiprint.png index 1c6911c2b7..fbf1778e7c 100644 Binary files a/recipes/icons/toiprint.png and b/recipes/icons/toiprint.png differ diff --git a/recipes/icons/world_archeology.png b/recipes/icons/world_archeology.png index 363717470a..c03c4e6ff8 100644 Binary files a/recipes/icons/world_archeology.png and b/recipes/icons/world_archeology.png differ diff --git a/recipes/icons/wsj.png b/recipes/icons/wsj.png index 5fb5496c5a..33eb9f371f 100644 Binary files a/recipes/icons/wsj.png and b/recipes/icons/wsj.png differ diff --git a/recipes/icons/wsj_mag.png b/recipes/icons/wsj_mag.png index 5fb5496c5a..33eb9f371f 100644 Binary files a/recipes/icons/wsj_mag.png and b/recipes/icons/wsj_mag.png differ diff --git a/recipes/icons/wsj_news.png b/recipes/icons/wsj_news.png index 5fb5496c5a..33eb9f371f 100644 Binary files a/recipes/icons/wsj_news.png and b/recipes/icons/wsj_news.png differ diff --git a/recipes/icons/znetwork.png b/recipes/icons/znetwork.png new file mode 100644 index 0000000000..3e72c63843 Binary files /dev/null and b/recipes/icons/znetwork.png differ diff --git a/recipes/observer_reach_foundation.recipe b/recipes/observer_reach_foundation.recipe deleted file mode 100644 index 2aa3833c4e..0000000000 --- a/recipes/observer_reach_foundation.recipe +++ /dev/null @@ -1,86 +0,0 @@ -from urllib.parse import quote - -from calibre.web.feeds.news import BasicNewsRecipe, classes - - -class ORF(BasicNewsRecipe): - title = 'Observer Research Foundation' - description = ( - 'Set up in 1990, ORF seeks to lead and aid policy thinking towards building a strong and prosperous India' - ' in a fair and equitable world. It helps discover and inform India’s choices, and carries Indian voices ' - 'and ideas to forums shaping global debates. ' - ) - language = 'en_IN' - __author__ = 'unkn0wn' - encoding = 'utf-8' - no_stylesheets = True - remove_javascript = True - masthead_url = 'https://www.orfonline.org/wp-content/uploads/2015/09/Logo_ORF_JPEG.jpg' - remove_attributes = ['style', 'height', 'width'] - ignore_duplicate_articles = {'url', 'title'} - reverse_article_order = True - remove_empty_feeds = True - simultaneous_downloads = 1 - art_url = '' - - extra_css = ''' - img {display:block; margin:0 auto;} - .report-slider, .author_panel {font-size:small; color:#404040;} - .report {font-size:small; font-weight:bold;} - .excert-italic, .recent-block-people {font-style:italic; color:#202020;} - blockquote, em {color:#202020;} - .espert_speak_panel {font-size:small;} - ''' - - articles_are_obfuscated = True - - def get_obfuscated_article(self, url): - br = self.get_browser() - soup = self.index_to_soup(url) - link = soup.a['href'] - skip_sections =[ # add sections you want to skip - '/video/', '/videos/', '/multimedia/', 'marathi', 'hindi', 'bangla' - ] - if any(x in link for x in skip_sections): - self.abort_article('skipping video links ', link) - self.log('Found ', link) - self.art_url = link - html = br.open(link).read() - return ({ 'data': html, 'url': link }) - - - def get_browser(self): - return BasicNewsRecipe.get_browser(self, user_agent='common_words/based') - - resolve_internal_links = True - remove_empty_feeds = True - - keep_only_tags = [ - dict(name='h1'), - classes('author_panel espert_speak_panel expert_panel_content') - ] - remove_tags = [ - classes( - 'social socialshare comment-area-section telegramhtml post-tag ' - 'research-prev research-next col-md-4 button_group sharethis-p tags' - ) - ] - - feeds = [] - - when = '170' # hours > 7 days - index = 'https://www.orfonline.org' - - sections = [ - 'expert-speak', 'books', 'event-reports', 'events', 'forums', 'gp-orf-series', 'issue-brief', 'monitors', - 'occasional-paper', 'primer', 'series', 'surveys-polls', 'young-voices', 'research' - ] - a = 'https://news.google.com/rss/search?q=when:{}h+allinurl:{}&hl=en-IN&gl=IN&ceid=IN:en' - for sec in sections: - sec_index = index + '/' + sec + '/' - feeds.append((sec.capitalize(), a.format(when, quote(sec_index, safe='')))) - feeds.append(('Others', a.format(when, quote(index, safe='')))) - - def populate_article_metadata(self, article, soup, first): - article.url = self.art_url - article.title = article.title.replace(' - Observer Research Foundation', '') diff --git a/recipes/orfonline.recipe b/recipes/orfonline.recipe new file mode 100644 index 0000000000..dfa5662c7f --- /dev/null +++ b/recipes/orfonline.recipe @@ -0,0 +1,110 @@ +#!/usr/bin/env python +from datetime import datetime, timedelta +from calibre.utils.date import parse_date +from calibre.web.feeds.news import BasicNewsRecipe, classes + + +class ORF(BasicNewsRecipe): + title = 'Observer Research Foundation' + description = ( + 'Set up in 1990, ORF seeks to lead and aid policy thinking towards building a strong and prosperous India' + ' in a fair and equitable world. It helps discover and inform India’s choices, and carries Indian voices ' + 'and ideas to forums shaping global debates.' + ) + language = 'en_IN' + __author__ = 'unkn0wn' + encoding = 'utf-8' + no_stylesheets = True + remove_javascript = True + masthead_url = 'https://www.orfonline.org/wp-content/uploads/2015/09/Logo_ORF_JPEG.jpg' + remove_attributes = ['style', 'height', 'width'] + ignore_duplicate_articles = {'url', 'title'} + remove_empty_feeds = True + oldest_article = 7 + simultaneous_downloads = 1 + + recipe_specific_options = { + 'days': { + 'short': 'Oldest article to download from this news source. In days ', + 'long': 'For example, 0.5, gives you articles from the past 12 hours', + 'default': str(oldest_article) + } + } + + def __init__(self, *args, **kwargs): + BasicNewsRecipe.__init__(self, *args, **kwargs) + d = self.recipe_specific_options.get('days') + if d and isinstance(d, str): + self.oldest_article = float(d) + + extra_css = ''' + img {display:block; margin:0 auto;} + .report-slider, .author_panel {font-size:small; color:#404040;} + .report {font-size:small; font-weight:bold;} + .excert-italic, .recent-block-people {font-style:italic; color:#202020;} + blockquote, em {color:#202020;} + .espert_speak_panel {font-size:small;} + ''' + + def get_browser(self): + return BasicNewsRecipe.get_browser(self, user_agent='common_words/based') + + resolve_internal_links = True + remove_empty_feeds = True + + keep_only_tags = [ + dict(name='h1'), + classes('author_panel espert_speak_panel expert_panel_content') + ] + remove_tags = [ + classes( + 'social socialshare comment-area-section telegramhtml post-tag ' + 'research-prev research-next col-md-4 button_group sharethis-p tags' + ) + ] + + def parse_index(self): + index = 'https://www.orfonline.org' + secs = [ + ('Expert Speak', '/expert-speak'), + ('Young Voices', '/expert-speak-category/young-voices'), + ('Commentary', '/content-type/commentary'), + ('Issue Briefs', '/content-type/issue-briefs'), + ('Books', '/content-type/books'), + ('Special Reports', '/content-type/special-reports') + ] + + feeds = [] + + for x, y in secs: + section = x + self.log(section) + + articles = [] + + soup = self.index_to_soup(index + y) + for a in soup.findAll('a', attrs={'href':True}): + if not a.find('h3'): + continue + div = a.find_previous_sibling(attrs={'class':'topic_story'}) + if div: + if div.find(**classes('show_date')): + date = parse_date(self.tag_to_string( + div.find(**classes('show_date')) + ) + ).replace(tzinfo=None) + if (datetime.now() - date) > timedelta(self.oldest_article): + self.log(' Skipping ', self.tag_to_string(a)) + continue + url = a['href'] + title = self.tag_to_string(a) + desc = '' + if a.find_next_sibling('p'): + desc = self.tag_to_string(a.find_next_sibling('p')) + if a.find_next_sibling(attrs={'class':'author_name'}): + desc = self.tag_to_string(a.find_next_sibling(attrs={'class':'author_name'})) + ' | ' + desc + self.log('\t', title, '\n\t', desc, '\n\t\t', url) + articles.append({'title': title, 'url': url, 'description': desc}) + if articles: + feeds.append((section, articles)) + return feeds diff --git a/recipes/tls_mag.recipe b/recipes/tls_mag.recipe index 9a277afe7c..f845787688 100644 --- a/recipes/tls_mag.recipe +++ b/recipes/tls_mag.recipe @@ -12,6 +12,7 @@ def re_html(y): if y: soup = BeautifulSoup(y.rstrip()) return soup.text + return '' def get_id(url): rq = browser().open(url) diff --git a/recipes/znetwork.recipe b/recipes/znetwork.recipe new file mode 100644 index 0000000000..bde5a325b6 --- /dev/null +++ b/recipes/znetwork.recipe @@ -0,0 +1,69 @@ +#!/usr/bin/env python +from calibre.web.feeds.news import BasicNewsRecipe, classes + + +class znetwork(BasicNewsRecipe): + title = 'ZNetwork' + __author__ = 'unkn0wn' + language = 'en' + description = 'ZNetwork: Left News, Analysis, Vision & Strategy' + oldest_article = 2 + no_stylesheets = True + encoding = 'utf-8' + remove_javascript = True + use_embedded_content = False + remove_attributes = ['style', 'height', 'width'] + masthead_url = 'https://znetwork.org/wp-content/uploads/2022/08/Z_logo_64.png' + ignore_duplicate_articles = {'url'} + browser_type = 'qt' + + recipe_specific_options = { + 'days': { + 'short': 'Oldest article to download from this news source. In days ', + 'long': 'For example, 0.5, gives you articles from the past 12 hours', + 'default': str(oldest_article) + } + } + + def __init__(self, *args, **kwargs): + BasicNewsRecipe.__init__(self, *args, **kwargs) + d = self.recipe_specific_options.get('days') + if d and isinstance(d, str): + self.oldest_article = float(d) + + extra_css = ''' + .cat-labels { font-size:small; color:#404040; } + .post-author, .date, .author-info, .z-post-source, .read-time { font-size:small; } + .sub-title { font-style:italic; color:#202020; } + em, blockquote { color:#202020; } + .wp-caption-text, .wp-element-caption { font-size:small; text-align:center; } + ''' + + keep_only_tags = [ + classes( + 'cat-labels post-title z-post-source sub-title post-author date' + ' read-time featured entry-content author-info' + ) + ] + + remove_tags = [ + dict(name=['aside', 'svg', 'source', 'iframe']), + classes('support-z-text yarpp-related yarpp-related-website') + ] + + remove_tags_after = [classes('author-info')] + + feeds = [('Articles', 'https://znetwork.org/feed/')] + + def preprocess_html(self, soup): + for spn in soup.findAll(**classes('post-author date')): + spn.string = spn.text + ' | ' + div = soup.findAll(**classes('featured')) + if div: + div.name = 'p' + p = soup.findAll(**classes('sub-title')) + if p: + p.name = 'p' + for h in soup.findAll(['h2', 'h3']): + h.name = 'h4' + return soup