Merge branch 'master' of https://github.com/unkn0w7n/calibre

recipes/observer_reach_foundation.recipe

@@ -1,86 +0,0 @@
-from urllib.parse import quote
-
-from calibre.web.feeds.news import BasicNewsRecipe, classes
-
-
-class ORF(BasicNewsRecipe):
-    title = 'Observer Research Foundation'
-    description = (
-        'Set up in 1990, ORF seeks to lead and aid policy thinking towards building a strong and prosperous India'
-        ' in a fair and equitable world. It helps discover and inform India’s choices, and carries Indian voices '
-        'and ideas to forums shaping global debates. '
-    )
-    language = 'en_IN'
-    __author__ = 'unkn0wn'
-    encoding = 'utf-8'
-    no_stylesheets = True
-    remove_javascript = True
-    masthead_url = 'https://www.orfonline.org/wp-content/uploads/2015/09/Logo_ORF_JPEG.jpg'
-    remove_attributes = ['style', 'height', 'width']
-    ignore_duplicate_articles = {'url', 'title'}
-    reverse_article_order = True
-    remove_empty_feeds = True
-    simultaneous_downloads = 1
-    art_url = ''
-
-    extra_css = '''
-        img {display:block; margin:0 auto;}
-        .report-slider, .author_panel {font-size:small; color:#404040;}
-        .report {font-size:small; font-weight:bold;}
-        .excert-italic, .recent-block-people {font-style:italic; color:#202020;}
-        blockquote, em {color:#202020;}
-        .espert_speak_panel {font-size:small;}
-    '''
-
-    articles_are_obfuscated = True
-
-    def get_obfuscated_article(self, url):
-        br = self.get_browser()
-        soup = self.index_to_soup(url)
-        link = soup.a['href']
-        skip_sections =[ # add sections you want to skip
-            '/video/', '/videos/', '/multimedia/', 'marathi', 'hindi', 'bangla'
-        ]
-        if any(x in link for x in skip_sections):
-            self.abort_article('skipping video links ', link)
-        self.log('Found ', link)
-        self.art_url = link
-        html = br.open(link).read()
-        return ({ 'data': html, 'url': link })
-
-
-    def get_browser(self):
-        return BasicNewsRecipe.get_browser(self, user_agent='common_words/based')
-
-    resolve_internal_links = True
-    remove_empty_feeds = True
-
-    keep_only_tags = [
-        dict(name='h1'),
-        classes('author_panel espert_speak_panel expert_panel_content')
-    ]
-    remove_tags = [
-        classes(
-            'social socialshare comment-area-section telegramhtml post-tag '
-            'research-prev research-next col-md-4 button_group sharethis-p tags'
-        )
-    ]
-
-    feeds = []
-
-    when = '170' # hours > 7 days
-    index = 'https://www.orfonline.org'
-
-    sections = [
-        'expert-speak', 'books', 'event-reports', 'events', 'forums', 'gp-orf-series', 'issue-brief', 'monitors',
-        'occasional-paper', 'primer', 'series', 'surveys-polls', 'young-voices', 'research'
-    ]
-    a = 'https://news.google.com/rss/search?q=when:{}h+allinurl:{}&hl=en-IN&gl=IN&ceid=IN:en'
-    for sec in sections:
-        sec_index = index + '/' + sec + '/'
-        feeds.append((sec.capitalize(), a.format(when, quote(sec_index, safe=''))))
-    feeds.append(('Others', a.format(when, quote(index, safe=''))))
-
-    def populate_article_metadata(self, article, soup, first):
-        article.url = self.art_url
-        article.title = article.title.replace(' - Observer Research Foundation', '')

recipes/orfonline.recipe

@@ -0,0 +1,110 @@
+#!/usr/bin/env python
+from datetime import datetime, timedelta
+from calibre.utils.date import parse_date
+from calibre.web.feeds.news import BasicNewsRecipe, classes
+
+
+class ORF(BasicNewsRecipe):
+    title = 'Observer Research Foundation'
+    description = (
+        'Set up in 1990, ORF seeks to lead and aid policy thinking towards building a strong and prosperous India'
+        ' in a fair and equitable world. It helps discover and inform India’s choices, and carries Indian voices '
+        'and ideas to forums shaping global debates.'
+    )
+    language = 'en_IN'
+    __author__ = 'unkn0wn'
+    encoding = 'utf-8'
+    no_stylesheets = True
+    remove_javascript = True
+    masthead_url = 'https://www.orfonline.org/wp-content/uploads/2015/09/Logo_ORF_JPEG.jpg'
+    remove_attributes = ['style', 'height', 'width']
+    ignore_duplicate_articles = {'url', 'title'}
+    remove_empty_feeds = True
+    oldest_article = 7
+    simultaneous_downloads = 1
+
+    recipe_specific_options = {
+        'days': {
+            'short': 'Oldest article to download from this news source. In days ',
+            'long': 'For example, 0.5, gives you articles from the past 12 hours',
+            'default': str(oldest_article)
+        }
+    }
+
+    def __init__(self, *args, **kwargs):
+        BasicNewsRecipe.__init__(self, *args, **kwargs)
+        d = self.recipe_specific_options.get('days')
+        if d and isinstance(d, str):
+            self.oldest_article = float(d)
+
+    extra_css = '''
+        img {display:block; margin:0 auto;}
+        .report-slider, .author_panel {font-size:small; color:#404040;}
+        .report {font-size:small; font-weight:bold;}
+        .excert-italic, .recent-block-people {font-style:italic; color:#202020;}
+        blockquote, em {color:#202020;}
+        .espert_speak_panel {font-size:small;}
+    '''
+
+    def get_browser(self):
+        return BasicNewsRecipe.get_browser(self, user_agent='common_words/based')
+
+    resolve_internal_links = True
+    remove_empty_feeds = True
+
+    keep_only_tags = [
+        dict(name='h1'),
+        classes('author_panel espert_speak_panel expert_panel_content')
+    ]
+    remove_tags = [
+        classes(
+            'social socialshare comment-area-section telegramhtml post-tag '
+            'research-prev research-next col-md-4 button_group sharethis-p tags'
+        )
+    ]
+
+    def parse_index(self):
+        index = 'https://www.orfonline.org'
+        secs = [
+            ('Expert Speak', '/expert-speak'),
+            ('Young Voices', '/expert-speak-category/young-voices'),
+            ('Commentary', '/content-type/commentary'),
+            ('Issue Briefs', '/content-type/issue-briefs'),
+            ('Books', '/content-type/books'),
+            ('Special Reports', '/content-type/special-reports')
+        ]
+
+        feeds = []
+
+        for x, y in secs:
+            section = x
+            self.log(section)
+
+            articles = []
+
+            soup = self.index_to_soup(index + y)
+            for a in soup.findAll('a', attrs={'href':True}):
+                if not a.find('h3'):
+                    continue
+                div = a.find_previous_sibling(attrs={'class':'topic_story'})
+                if div:
+                    if div.find(**classes('show_date')):
+                        date = parse_date(self.tag_to_string(
+                                    div.find(**classes('show_date'))
+                                )
+                            ).replace(tzinfo=None)
+                        if (datetime.now() - date) > timedelta(self.oldest_article):
+                            self.log('      Skipping ', self.tag_to_string(a))
+                            continue
+                url = a['href']
+                title = self.tag_to_string(a)
+                desc = ''
+                if a.find_next_sibling('p'):
+                    desc = self.tag_to_string(a.find_next_sibling('p'))
+                if a.find_next_sibling(attrs={'class':'author_name'}):
+                    desc = self.tag_to_string(a.find_next_sibling(attrs={'class':'author_name'})) + ' | ' + desc
+                self.log('\t', title, '\n\t', desc, '\n\t\t', url)
+                articles.append({'title': title, 'url': url, 'description': desc})
+            if articles:
+                feeds.append((section, articles))
+        return feeds

recipes/tls_mag.recipe

@@ -12,6 +12,7 @@ def re_html(y):
     if y:
         soup = BeautifulSoup(y.rstrip())
         return soup.text
+    return ''
 
 def get_id(url):
     rq = browser().open(url)

recipes/znetwork.recipe

@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+from calibre.web.feeds.news import BasicNewsRecipe, classes
+
+
+class znetwork(BasicNewsRecipe):
+    title = 'ZNetwork'
+    __author__ = 'unkn0wn'
+    language = 'en'
+    description = 'ZNetwork: Left News, Analysis, Vision & Strategy'
+    oldest_article = 2
+    no_stylesheets = True
+    encoding = 'utf-8'
+    remove_javascript = True
+    use_embedded_content = False
+    remove_attributes = ['style', 'height', 'width']
+    masthead_url = 'https://znetwork.org/wp-content/uploads/2022/08/Z_logo_64.png'
+    ignore_duplicate_articles = {'url'}
+    browser_type = 'qt'
+
+    recipe_specific_options = {
+        'days': {
+            'short': 'Oldest article to download from this news source. In days ',
+            'long': 'For example, 0.5, gives you articles from the past 12 hours',
+            'default': str(oldest_article)
+        }
+    }
+
+    def __init__(self, *args, **kwargs):
+        BasicNewsRecipe.__init__(self, *args, **kwargs)
+        d = self.recipe_specific_options.get('days')
+        if d and isinstance(d, str):
+            self.oldest_article = float(d)
+
+    extra_css = '''
+        .cat-labels { font-size:small; color:#404040; }
+        .post-author, .date, .author-info, .z-post-source, .read-time { font-size:small; }
+        .sub-title { font-style:italic; color:#202020; }
+        em, blockquote { color:#202020; }
+        .wp-caption-text, .wp-element-caption { font-size:small; text-align:center; }
+    '''
+
+    keep_only_tags = [
+        classes(
+            'cat-labels post-title z-post-source sub-title post-author date'
+            ' read-time featured entry-content author-info'
+        )
+    ]
+
+    remove_tags = [
+        dict(name=['aside', 'svg', 'source', 'iframe']),
+        classes('support-z-text yarpp-related yarpp-related-website')
+    ]
+
+    remove_tags_after = [classes('author-info')]
+
+    feeds = [('Articles', 'https://znetwork.org/feed/')]
+
+    def preprocess_html(self, soup):
+        for spn in soup.findAll(**classes('post-author date')):
+            spn.string = spn.text + ' | '
+        div = soup.findAll(**classes('featured'))
+        if div:
+            div.name = 'p'
+        p = soup.findAll(**classes('sub-title'))
+        if p:
+            p.name = 'p'
+        for h in soup.findAll(['h2', 'h3']):
+            h.name = 'h4'
+        return soup