Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Update New Scientist feeds

Browse Source
This commit is contained in:
Kovid Goyal 2021-04-15 20:12:03 +05:30
parent 82df992abe
commit 98c83d2dec
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
1 changed files with 25 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
recipes/new_scientist.recipe
View File

@ -29,6 +29,12 @@ import re
from calibre.web.feeds.news import BasicNewsRecipe from calibre.web.feeds.news import BasicNewsRecipe
def classes(classes):
q = frozenset(classes.split(' '))
return dict(attrs={
'class': lambda x: x and frozenset(x.split()).intersection(q)})
class NewScientist(BasicNewsRecipe): class NewScientist(BasicNewsRecipe):
title = 'New Scientist - Online News w. subscription' title = 'New Scientist - Online News w. subscription'
description = 'Science news and science articles from New Scientist.' description = 'Science news and science articles from New Scientist.'
@ -39,7 +45,6 @@ class NewScientist(BasicNewsRecipe):
max_articles_per_feed = 100 max_articles_per_feed = 100
no_stylesheets = True no_stylesheets = True
use_embedded_content = False use_embedded_content = False
masthead_url = 'http://www.newscientist.com/img/misc/ns_logo.jpg'
encoding = 'utf-8' encoding = 'utf-8'
needs_subscription = 'optional' needs_subscription = 'optional'
remove_empty_feeds = True remove_empty_feeds = True
@ -58,26 +63,25 @@ class NewScientist(BasicNewsRecipe):
.wp-caption-text{font-family: "Lato Bold", sans-serif; font-size:x-small;} .wp-caption-text{font-family: "Lato Bold", sans-serif; font-size:x-small;}
""" """
conversion_options = {
'comment': description, 'tags': category, 'publisher': publisher, 'language': language
}
preprocess_regexps = [(re.compile(
r'</title>.*?</head>', re.DOTALL | re.IGNORECASE), lambda match: '</title></head>')]
keep_only_tags = [ keep_only_tags = [
dict(attrs={'class': ['article-header', 'article-content']})] classes('article-header article-conten')
remove_tags_after = dict(name='p', attrs={'class': 'print-headline'}) ]
# Whether to convert images to grayscale for eInk readers. remove_tags = [
Convert_Grayscale = False classes('social__button-container')
]
def get_article_url(self, article):
ans = BasicNewsRecipe.get_article_url(self, article)
return ans.partition('?')[0]
def is_login_form(self, form): def is_login_form(self, form):
return "action" in form.attrs and form.attrs['action'] == "/ns-login.php" return "action" in form.attrs and form.attrs['action'] == "/ns-login.php"
def get_browser(self): def get_browser(self):
br = BasicNewsRecipe.get_browser(self) br = BasicNewsRecipe.get_browser(self)
br.open('http://www.newscientist.com/')
if self.username is not None and self.password is not None: if self.username is not None and self.password is not None:
br.open('http://www.newscientist.com/')
try: try:
br.open('https://www.newscientist.com/login/') br.open('https://www.newscientist.com/login/')
br.select_form(predicate=self.is_login_form) br.select_form(predicate=self.is_login_form)
@ -89,33 +93,18 @@ class NewScientist(BasicNewsRecipe):
'Unable to locate login form! Switching to free mode.') 'Unable to locate login form! Switching to free mode.')
return br return br
remove_tags = [
dict(name=['link', 'base', 'meta', 'iframe', 'object', 'embed']), dict(attrs={
'class': ['ad-leaderboard', 'article-topics']}), dict(attrs={'id': 'mpu-mid-article'})
]
feeds = [ feeds = [
('News', 'https://www.newscientist.com/section/news/feed/'),
(u'Latest Headlines', u'http://feeds.newscientist.com/science-news'), ('Features', 'https://www.newscientist.com/section/features/feed/'),
(u'Magazine', u'http://feeds.newscientist.com/magazine'), ('Physics', 'https://www.newscientist.com/subject/physics/feed/'),
(u'Health', u'http://feeds.newscientist.com/health'), ('Technology', 'https://www.newscientist.com/subject/technology/feed/'),
(u'Life', u'http://feeds.newscientist.com/life'), ('Space', 'https://www.newscientist.com/subject/space/feed/'),
(u'Space', u'http://feeds.newscientist.com/space'), ('Life', 'https://www.newscientist.com/subject/life/feed/'),
(u'Physics and Mathematics', u'http://feeds.newscientist.com/physics-math'), ('Earth', 'https://www.newscientist.com/subject/earth/feed/'),
(u'Environment', u'http://feeds.newscientist.com/environment'), ('Health', 'https://www.newscientist.com/subject/health/feed/'),
(u'Science in Society', u'http://feeds.newscientist.com/science-in-society'), ('Humans', 'https://www.newscientist.com/subject/humans/feed/'),
(u'Tech', u'http://feeds.newscientist.com/tech')
] ]
def get_article_url(self, article):
articleurl = BasicNewsRecipe.get_article_url(self, article)
# resolve redirect.
urlverified = self.browser.open_novisit(articleurl).geturl()
if '?' in urlverified:
pleft, ppart, pright = urlverified.rpartition('?')
urlverified = pleft
return urlverified
def get_cover_url(self): def get_cover_url(self):
cover_url = None cover_url = None
soup = self.index_to_soup( soup = self.index_to_soup(