Merge branch 'master' of https://github.com/unkn0w7n/calibre

recipes/natgeo_kids.recipe

@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+
+class NatGeo(BasicNewsRecipe):
+    title = 'National Geographic Kids'
+    description = 'The National Geographic, an American monthly magazine'
+    language = 'en'
+    encoding = 'utf8'
+    publisher = 'kids.nationalgeographic.com'
+    category = 'science, nat geo'
+    __author__ = 'unkn0wn'
+    description = 'Inspiring people to care about the planet since 1888'
+    timefmt = ' [%a, %d %b, %Y]'
+    use_embedded_content = False
+    remove_javascript = True
+    masthead_url = 'https://i.natgeofe.com/n/e76f5368-6797-4794-b7f6-8d757c79ea5c/ng-logo-2fl.png?w=600&h=600'
+    remove_empty_feeds = True
+    resolve_internal_links = True
+    ignore_duplicate_articles = {'title', 'url'}
+
+    recipe_specific_options = {
+        'res': {
+            'short': 'For hi-res images, select a resolution from the\nfollowing options: 800, 1000, 1200 or 1500',
+            'long': 'This is useful for non e-ink devices, and for a lower file size\nthan the default, use 400 or 300.',
+            'default': '600',
+        },
+    }
+
+    @property
+    def natgeo_parser(self):
+        ans = getattr(self, '_natgeo_parser', None)
+        if ans is None:
+            from calibre.live import load_module
+
+            self._natgeo_parser = ans = load_module('calibre.web.site_parsers.natgeo')
+        return ans
+
+    def preprocess_raw_html(self, raw_html, url):
+        return self.natgeo_parser.extract_html(raw_html)
+
+    extra_css = """
+        blockquote { color:#404040; }
+        .byline, i { font-style:italic; color:#202020; }
+        .cap { font-size:small; }
+        img {display:block; margin:0 auto;}
+        .cred { font-style:italic; font-size:small; color:#404040; }
+        .auth, .time, .sub { font-size:small; color:#5c5c5c; }
+    """
+
+    def parse_index(self):
+        index = 'https://kids.nationalgeographic.com/'
+        sections = [
+            'Front Page', 'animals', 'history', 'science',
+            'space', 'homework-help', 'crafts',
+        ]
+        feeds = []
+        for sec in sections:
+            section = sec.capitalize()
+            self.log(section)
+            url = index + sec
+            if sec.startswith('Front'):
+                url = index
+            self.log('Fetching articles from ', url)
+            soup = self.index_to_soup(url)
+            articles = []
+            for a in soup.findAll('a', attrs={'href': lambda x: x and '/article/' in x}):
+                if a.find('img') and '/games/' in a['href']:
+                    continue
+                url = a['href']
+                title = self.tag_to_string(a)
+                self.log('\t', title, '\n\t\t', url)
+                articles.append({'title': title, 'url': url})
+            if articles:
+                feeds.append((section, articles))
+        return feeds
+
+    def preprocess_html(self, soup):
+        for h2 in soup.findAll('h2'):
+            h2.name = 'h4'
+        for img in soup.findAll('img', src=True):
+            res = '?w=600'
+            w = self.recipe_specific_options.get('res')
+            if w and isinstance(w, str):
+                res = '?w=' + w
+            img['src'] = img['src'] + res
+        return soup
+
+    def populate_article_metadata(self, article, soup, first):
+        summ = soup.find(attrs={'class': 'byline'})
+        if summ:
+            article.summary = self.tag_to_string(summ)
+            article.text_summary = self.tag_to_string(summ)

recipes/natgeo_traveller.recipe

@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+from pprint import pformat
+
+from calibre.web.feeds.news import BasicNewsRecipe, classes
+
+
+class NatGeo(BasicNewsRecipe):
+    title = 'National Geographic Traveller'
+    description = 'News articles from The National Geographic Traveller, Download Monthly.'
+    language = 'en'
+    encoding = 'utf8'
+    publisher = 'nationalgeographic.com'
+    category = 'science, nat geo'
+    __author__ = 'unkn0wn'
+    description = 'Inspiring people to care about the planet since 1888'
+    timefmt = ' [%a, %d %b, %Y]'
+    no_stylesheets = True
+    use_embedded_content = False
+    remove_attributes = ['style']
+    remove_javascript = False
+    masthead_url = 'https://i.natgeofe.com/n/e76f5368-6797-4794-b7f6-8d757c79ea5c/ng-logo-2fl.png?w=600&h=600'
+    remove_empty_feeds = True
+    resolve_internal_links = True
+    ignore_duplicate_articles = {'url'}
+
+    recipe_specific_options = {
+        'res': {
+            'short': 'For hi-res images, select a resolution from the\nfollowing options: 800, 1000, 1200 or 1500',
+            'long': 'This is useful for non e-ink devices, and for a lower file size\nthan the default, use 400 or 300.',
+            'default': '600',
+        }
+    }
+
+    @property
+    def natgeo_parser(self):
+        ans = getattr(self, '_natgeo_parser', None)
+        if ans is None:
+            from calibre.live import load_module
+
+            self._natgeo_parser = ans = load_module('calibre.web.site_parsers.natgeo')
+        return ans
+
+    def preprocess_raw_html(self, raw_html, url):
+        return self.natgeo_parser.extract_html(raw_html)
+
+    extra_css = """
+        blockquote { color:#404040; }
+        .byline, i { font-style:italic; color:#202020; }
+        .cap { font-size:small; }
+        img {display:block; margin:0 auto;}
+        .cred { font-style:italic; font-size:small; color:#404040; }
+        .auth, .time, .sub { font-size:small; color:#5c5c5c; }
+    """
+
+    def parse_index(self):
+        pages = [
+            'https://www.nationalgeographic.com/travel/topic/national-geographic-traveller-uk'
+        ]
+
+        feeds = []
+
+        for sec in pages:
+            soup = self.index_to_soup(sec)
+            parsed = self.articles_from_soup(soup)
+            if parsed:
+                feeds += parsed
+        return feeds
+
+    def articles_from_soup(self, soup):
+        ans = {}
+        for article in soup.findAll('article'):
+            a = article.find('a')
+            url = a['href']
+            if url.startswith('/'):
+                url = 'https://www.nationalgeographic.com' + url
+            section = self.tag_to_string(article.find(**classes('SectionLabel')))
+            if section.startswith('Paid Content'):
+                continue
+            title = self.tag_to_string(
+                article.find(**classes('PromoTile__Title--truncated'))
+            )
+            articles = ans.setdefault(section, [])
+            articles.append({'title': title, 'url': url})
+        self.log(pformat(ans))
+        return list(ans.items())
+
+    def preprocess_html(self, soup):
+        for h2 in soup.findAll('h2'):
+            h2.name = 'h4'
+        for img in soup.findAll('img', src=True):
+            res = '?w=600'
+            w = self.recipe_specific_options.get('res')
+            if w and isinstance(w, str):
+                res = '?w=' + w
+            img['src'] = img['src'] + res
+        return soup
+
+    def populate_article_metadata(self, article, soup, first):
+        summ = soup.find(attrs={'class': 'byline'})
+        if summ:
+            article.summary = self.tag_to_string(summ)
+            article.text_summary = self.tag_to_string(summ)

recipes/nauka_w_polsce.recipe

@@ -1,59 +0,0 @@
-import re
-
-from calibre.web.feeds.news import BasicNewsRecipe
-
-
-class NaukawPolsce(BasicNewsRecipe):
-    title = u'PAP Nauka w Polsce'
-    __author__ = 'fenuks'
-    description = u'Serwis Nauka w Polsce ma za zadanie popularyzację polskiej nauki. Można na nim znaleźć wiadomości takie jak: osiągnięcia polskich naukowców, wydarzenia na polskich uczelniach, osiągnięcia studentów, konkursy dla badaczy, staże i stypendia naukowe, wydarzenia w polskiej nauce, kalendarium wydarzeń w nauce, materiały wideo o nauce.'  # noqa
-    category = 'science'
-    language = 'pl'
-    cover_url = 'http://www.naukawpolsce.pap.pl/Themes/Pap/images/logo-pl.gif'
-    oldest_article = 7
-    max_articles_per_feed = 100
-    no_stylesheets = True
-    remove_empty_feeds = True
-    extra_css = '.miniaturka {float: left; margin-right: 5px; max-width: 350px;} .miniaturka-dol-strony {display: inline-block; margin: 0 15px; width: 120px;}'
-    ignore_duplicate_articles = {'title', 'url'}
-    index = 'http://www.naukawpolsce.pl'
-    keep_only_tags = [dict(name='div', attrs={'class': 'margines wiadomosc'})]
-    remove_tags = [dict(name='div', attrs={'class': 'tagi'})]
-
-    def find_articles(self, url):
-        articles = []
-        soup = self.index_to_soup(url)
-        for i in soup.findAll(name='div', attrs={'class': 'aktualnosci-margines lista-depesz information-content'}):
-            title = i.h1.a.string
-            url = self.index + i.h1.a['href']
-            date = ''  # i.span.string
-            articles.append({'title': title,
-                             'url': url,
-                             'date': date,
-                             'description': ''
-                             })
-        return articles
-
-    def parse_index(self):
-        feeds = []
-        feeds.append((u"Historia i kultura", self.find_articles(
-            'http://www.naukawpolsce.pl/historia-i-kultura/')))
-        feeds.append((u"Kosmos", self.find_articles(
-            'http://www.naukawpolsce.pl/kosmos/')))
-        feeds.append((u"Przyroda", self.find_articles(
-            'http://www.naukawpolsce.pl/przyroda/')))
-        feeds.append((u"Społeczeństwo", self.find_articles(
-            'http://www.naukawpolsce.pl/spoleczenstwo/')))
-        feeds.append((u"Technologie", self.find_articles(
-            'http://www.naukawpolsce.pl/technologie/')))
-        feeds.append((u"Uczelnie", self.find_articles(
-            'http://www.naukawpolsce.pl/uczelnie/')))
-        feeds.append((u"Nauki medyczne", self.find_articles(
-            'http://www.naukawpolsce.pl/zdrowie/')))
-
-        return feeds
-
-    def preprocess_html(self, soup):
-        for p in soup.findAll(name='p', text=re.compile(' ')):
-            p.extract()
-        return soup

recipes/nbonline.recipe

@@ -1,33 +0,0 @@
-from calibre.web.feeds.news import BasicNewsRecipe
-
-
-class NBOnline(BasicNewsRecipe):
-    title = u'Nikkei Business Online'
-    language = 'ja'
-    description = u'Nikkei Business Online.\u6CE8\uFF1A\u30E6\u30FC\u30B6\u30FC\u540D\u306Bemail\u30A2\u30C9\u30EC\u30B9\u3068\u30E6\u30FC\u30B6\u30FC\u540D\u3092\u30BB\u30DF\u30B3\u30ED\u30F3\u3067\u533A\u5207\u3063\u3066\u5165\u308C\u3066\u304F\u3060\u3055\u3044\u3002\u4F8B\uFF1Aemail@address.jp;username . PLEASE NOTE: You need to put your email address and username into username field separated by ; (semi-colon).'  # noqa
-    __author__ = 'Ado Nishimura'
-    needs_subscription = True
-    oldest_article = 7
-    max_articles_per_feed = 100
-    remove_tags_before = dict(id='kanban')
-    remove_tags = [dict(name='div', id='footer')]
-
-    feeds = [('Nikkei Business Online',
-              'http://business.nikkeibp.co.jp/rss/all_nbo.rdf')]
-
-    def get_cover_url(self):
-        return 'http://business.nikkeibp.co.jp/images/nbo/200804/parts/logo.gif'
-
-    def get_browser(self):
-        br = BasicNewsRecipe.get_browser(self)
-        if self.username is not None and self.password is not None:
-            br.open('https://signon.nikkeibp.co.jp/front/login/?ct=p&ts=nbo')
-            br.select_form(name='loginActionForm')
-            br['email'] = self.username.split(';')[0]
-            br['userId'] = self.username.split(';')[1]
-            br['password'] = self.password
-            br.submit()
-        return br
-
-    def print_version(self, url):
-        return url + '?ST=print'

recipes/neviditelny_pes.recipe

@@ -1,31 +0,0 @@
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
-from __future__ import unicode_literals
-
-from calibre.web.feeds.recipes import BasicNewsRecipe
-
-
-class pesRecipe(BasicNewsRecipe):
-    __author__ = 'bubak'
-    title = u'Neviditelný pes'
-    publisher = u''
-    description = u'Neviditelný pes'
-    oldest_article = 1
-    max_articles_per_feed = 20
-    use_embedded_content = False
-    remove_empty_feeds = True
-
-    feeds = [
-        (u'Neviditelný pes',
-         u'http://neviditelnypes.lidovky.cz/export/rss.asp?c=pes_neviditelny')
-    ]
-
-    language = 'cs'
-    cover_url = 'http://g.zpravy.cz/o/pes/logo_pes.jpg'
-    remove_javascript = True
-    no_stylesheets = True
-    extra_css             = """
-                            """
-
-    remove_tags = []
-    remove_tags_before = dict(name='div', attrs={'id': 'art-full'})
-    remove_tags_after = dict(name='div', attrs={'id': 'authors'})

recipes/new_london_day.recipe

@@ -1,73 +0,0 @@
-__license__ = 'GPL 3'
-__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
-__docformat__ = 'restructuredtext en'
-
-from calibre.web.feeds.news import BasicNewsRecipe
-
-
-class AdvancedUserRecipe1294342201(BasicNewsRecipe):
-    title = u'New London Day'
-    __author__ = 'Being'
-    description = 'State, local and business news from New London, CT'
-    language = 'en'
-    oldest_article = 1
-    max_articles_per_feed = 200
-
-    use_embedded_content = False
-    no_stylesheets = True
-    remove_javascript = True
-    remove_tags_before = dict(id='article')
-    remove_tags_after = dict(id='article')
-    remove_tags = [dict(attrs={'class': ['articleTools', 'post-tools', 'side_tool', 'nextArticleLink clearfix']}),
-                   dict(id=['footer', 'toolsRight', 'articleInline', 'navigation',
-                            'archive', 'side_search', 'blog_sidebar', 'side_tool', 'side_index']),
-                   dict(name=['script', 'noscript', 'style'])]
-    remove_tags_after = [{'class': ['photo_article', ]}]
-    remove_tags = [{'id': ["moduleArticleTools", "content-bottom", "rail", "articleRelates module", "toolSet", "relatedrailcontent", "div-wrapper", "beta", "atp-comments", "footer"]}, {'class': ["clearfix", "relatedTitle", "articleRelates module", "asset-footer", "tools", "comments", "featurePromo", "featurePromo fp-topjobs brownBackground", "clearfix fullSpan brownBackground", "curvedContent"]},  # noqa
-                   dict(name='font', attrs={'id': ["cr-other-headlines"]})]
-    extra_css = '''
-                    h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;}
-                    h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;}
-                    .byline {font-family:Arial,Helvetica,sans-serif; font-size:xx-small;}
-                    .date {font-family:Arial,Helvetica,sans-serif; font-size:xx-small;}
-                    p{font-family:Arial,Helvetica,sans-serif;font-size:small;}
-                    .copyright {font-family:Arial,Helvetica,sans-serif;font-size:xx-small;text-align:center}
-                    .story{font-family:Arial,Helvetica,sans-serif;font-size:small;}
-                    .entry-asset asset hentry{font-family:Arial,Helvetica,sans-serif;font-size:small;}
-                    .pagebody{font-family:Arial,Helvetica,sans-serif;font-size:small;}
-                    .maincontentcontainer{font-family:Arial,Helvetica,sans-serif;font-size:small;}
-                    .story-body{font-family:Arial,Helvetica,sans-serif;font-size:small;}
-                    body{font-family:Helvetica,Arial,sans-serif;font-size:small;}
-    '''
-
-    feeds = [
-        (u'All News', u'http://www.theday.com/section/rss'),
-        (u'Breaking News', u'http://www.theday.com/section/rss01'),
-        (u'Police and Courts', u'http://www.theday.com/section/rss02'),
-        (u'State News', u'http://www.theday.com/section/rss03'),
-        (u'Local Business', u'http://www.theday.com/section/rss04'),
-        (u'Entertainment', u'http://www.theday.com/section/rss05'),
-        (u'Opinion', u'http://www.theday.com/section/rss06'),
-        (u'Casinos', u'http://www.theday.com/section/rss12'),
-        (u'Defense and Military', u'http://www.theday.com/section/rss14'),
-        (u'Ann Baldelli Ruminations', u'http://www.theday.com/section/rss20'),
-        (u'Paul Choiniere Ruminations', u'http://www.theday.com/section/rss21'),
-        (u'Michael Costanza Omnivore', u'http://www.theday.com/section/rss23'),
-        (u'Rebecca Dangelo Reel Life', u'http://www.theday.com/section/rss25'), ]
-
-    def print_version(self, url):
-        return url.replace('/index.html', '/print.html')
-
-    def get_article_url(self, article):
-        return article.get('feedburner_origlink', article.get('guid', article.get('link')))
-
-    def postprocess_html(self, soup, first_fetch):
-        for t in soup.findAll(['table', 'tr', 'td']):
-            t.name = 'div'
-
-        for tag in soup.findAll('form', dict(attrs={'name': ["comments_form"]})):
-            tag.extract()
-        for tag in soup.findAll('font', dict(attrs={'id': ["cr-other-headlines"]})):
-            tag.extract()
-
-        return soup