mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-06-23 15:30:45 -04:00
131 lines
4.8 KiB
Python
131 lines
4.8 KiB
Python
##
|
|
# Title: New Scientist RSS recipe
|
|
# Contact: AprilHare, Darko Miletic <darko.miletic at gmail.com>
|
|
##
|
|
# License: GNU General Public License v3 - http://www.gnu.org/copyleft/gpl.html
|
|
# Copyright: 2008-2016, AprilHare, Darko Miletic <darko.miletic at gmail.com>
|
|
##
|
|
# Written: 2008
|
|
# Last Edited: Jan 2016
|
|
##
|
|
|
|
'''
|
|
01-19-2012: Added GrayScale Image conversion and Duplicant article removals
|
|
12-31-2015: Major rewrite due to massive changes in site structure
|
|
01-27-2016: Added support for series index and minor cleanup
|
|
'''
|
|
|
|
__license__ = 'GNU General Public License v3 - http://www.gnu.org/copyleft/gpl.html'
|
|
__copyright__ = '2008-2016, AprilHare, Darko Miletic <darko.miletic at gmail.com>'
|
|
__version__ = 'v0.6.1'
|
|
__date__ = '2016-01-27'
|
|
__author__ = 'Darko Miletic'
|
|
|
|
'''
|
|
newscientist.com
|
|
'''
|
|
|
|
from calibre.web.feeds.news import BasicNewsRecipe
|
|
|
|
|
|
def classes(classes):
|
|
q = frozenset(classes.split(' '))
|
|
return dict(attrs={
|
|
'class': lambda x: x and frozenset(x.split()).intersection(q)})
|
|
|
|
|
|
class NewScientist(BasicNewsRecipe):
|
|
title = 'New Scientist - Online News w. subscription'
|
|
description = 'Science news and science articles from New Scientist, based on feeds.'
|
|
language = 'en'
|
|
publisher = 'Reed Business Information Ltd.'
|
|
category = 'science news, science articles, science jobs, drugs, cancer, depression, computer software'
|
|
oldest_article = 15
|
|
max_articles_per_feed = 100
|
|
no_stylesheets = True
|
|
use_embedded_content = False
|
|
encoding = 'utf-8'
|
|
needs_subscription = 'optional'
|
|
remove_empty_feeds = True
|
|
ignore_duplicate_articles = {'url'}
|
|
resolve_internal_links = True
|
|
remove_attributes = ['style', 'height', 'width']
|
|
masthead_url = 'https://cdn.shopify.com/s/files/1/0266/6843/3505/files/logo.svg?v=1629189295'
|
|
|
|
conversion_options = {
|
|
'comment': description,
|
|
'tags': category,
|
|
'publisher': publisher,
|
|
'language': language
|
|
}
|
|
|
|
extra_css = '''
|
|
img {display:block; margin:0 auto;}
|
|
.ArticleHeader__Category { font-size:small; color:#404040; }
|
|
.ArticleHeader__Author, .ArticleHeader__DateTimeWrapper { font-size:small; }
|
|
.ArticleHeader__Copy { font-style:italic; color:#202020; }
|
|
.ArticleImage { font-size:small; text-align:center; }
|
|
.ArticleImageCaption__Credit { font-size:smaller; }
|
|
'''
|
|
|
|
keep_only_tags = [
|
|
classes('ArticleHeader ArticleContent')
|
|
]
|
|
|
|
remove_tags = [
|
|
dict(name=['svg', 'button']),
|
|
classes('ArticleHeader__SocialWrapper AdvertWrapper ReadMoreWithImage ArticleTopics')
|
|
]
|
|
|
|
def preprocess_html(self, soup):
|
|
time = soup.find(**classes('ArticleHeader__DateTimeWrapper'))
|
|
if time:
|
|
time.name = 'div'
|
|
for img in soup.findAll('img', attrs={'data-src':True}):
|
|
img['src'] = img['data-src'].split('?')[0] + '?width=700'
|
|
for figc in soup.findAll('figcaption'):
|
|
for p in figc.findAll('p'):
|
|
p.name = 'div'
|
|
return soup
|
|
|
|
def get_article_url(self, article):
|
|
ans = BasicNewsRecipe.get_article_url(self, article)
|
|
return ans.partition('?')[0]
|
|
|
|
def print_version(self, url):
|
|
if '/video/' in url:
|
|
return None
|
|
return url
|
|
|
|
def get_browser(self):
|
|
br = BasicNewsRecipe.get_browser(self)
|
|
if self.username is not None and self.password is not None:
|
|
def is_login_form(form):
|
|
return 'action' in form.attrs and form.attrs['action'] == '/login/'
|
|
|
|
br.open('https://www.newscientist.com/login/')
|
|
br.select_form(predicate=is_login_form)
|
|
br['email'] = self.username
|
|
br['password'] = self.password
|
|
res = br.submit().read()
|
|
if b'>Your account<' not in res:
|
|
raise ValueError('Failed to log in to New Scientist, check your username and password')
|
|
return br
|
|
|
|
feeds = [
|
|
('News', 'https://www.newscientist.com/section/news/feed/'),
|
|
('Features', 'https://www.newscientist.com/section/features/feed/'),
|
|
('Physics', 'https://www.newscientist.com/subject/physics/feed/'),
|
|
('Technology', 'https://www.newscientist.com/subject/technology/feed/'),
|
|
('Space', 'https://www.newscientist.com/subject/space/feed/'),
|
|
('Life', 'https://www.newscientist.com/subject/life/feed/'),
|
|
('Earth', 'https://www.newscientist.com/subject/earth/feed/'),
|
|
('Health', 'https://www.newscientist.com/subject/health/feed/'),
|
|
('Humans', 'https://www.newscientist.com/subject/humans/feed/'),
|
|
]
|
|
|
|
def get_cover_url(self):
|
|
soup = self.index_to_soup('https://www.newscientist.com/issues/current/')
|
|
div = soup.find('div', attrs={'class':'ThisWeeksMagazineHero__CoverInfo'})
|
|
return div.find(**classes('ThisWeeksMagazineHero__ImageLink')).img['src']
|