mirror of
https://github.com/kovidgoyal/calibre.git
synced 2026-03-05 16:33:43 -05:00
100 lines
2.7 KiB
Python
100 lines
2.7 KiB
Python
#!/usr/bin/env python
|
|
|
|
__license__ = 'GPL v3'
|
|
'''
|
|
sciencenews.org
|
|
'''
|
|
|
|
from calibre.web.feeds.news import BasicNewsRecipe
|
|
import datetime
|
|
import re
|
|
|
|
|
|
class ScienceNewsIssue(BasicNewsRecipe):
|
|
title = u'Science News'
|
|
description = ("Science News is an award-winning bi-weekly newsmagazine covering the most important research"
|
|
" in all fields of science. This recipe downloads all the articles from the latest issue.")
|
|
category = u'Science, Technology, News'
|
|
publisher = u'Society for Science & the Public'
|
|
oldest_article = 14
|
|
language = 'en'
|
|
max_articles_per_feed = 50
|
|
no_stylesheets = True
|
|
use_embedded_content = False
|
|
timefmt = ' [%A, %d %B, %Y]'
|
|
auto_cleanup = False
|
|
|
|
keep_only_tags = [
|
|
dict(
|
|
attrs={
|
|
'class':
|
|
lambda x: x and (
|
|
'single__content___' in x or 'header-default__title___' in x or
|
|
'header-default__deck___' in x or 'header-default__figure___' in
|
|
x
|
|
)
|
|
}
|
|
)
|
|
]
|
|
remove_tags = [
|
|
dict(
|
|
attrs={'class': lambda x: x and ('newsletter-signup__wrapper___' in x)}
|
|
)
|
|
]
|
|
|
|
def parse_index(self):
|
|
|
|
# Get URL of latest mag page
|
|
ld = self._get_mag_date()
|
|
url = f"https://www.sciencenews.org/sn-magazine/{ld:%B}-{ld.day}-{ld.year}"
|
|
url = url.lower()
|
|
|
|
# Get articles
|
|
soup = self.index_to_soup(url)
|
|
re_article = re.compile("https://www.sciencenews.org/article/")
|
|
stories = []
|
|
past_urls = set()
|
|
for sec in soup.find_all(href=re_article):
|
|
|
|
article_url = sec["href"]
|
|
article_title = sec.text.strip()
|
|
|
|
# Ignore image URLs which do not have text title
|
|
if article_title == "":
|
|
continue
|
|
|
|
# Ignore if link is a duplicate
|
|
if article_url in past_urls:
|
|
continue
|
|
|
|
past_urls.add(article_url)
|
|
article_info = {
|
|
"url": article_url,
|
|
"title": article_title,
|
|
}
|
|
stories.append(article_info)
|
|
|
|
index = [
|
|
("Articles", stories),
|
|
]
|
|
return index
|
|
|
|
def _get_mag_date(self):
|
|
"""Return date of latest magazine issue.
|
|
It is published every 2 weeks."""
|
|
|
|
d = datetime.date(2022, 6, 18)
|
|
t = datetime.date.today()
|
|
ld = None
|
|
while d <= t:
|
|
ld = d
|
|
d += datetime.timedelta(days=14)
|
|
return ld
|
|
|
|
def get_cover_url(self):
|
|
ld = self._get_mag_date()
|
|
url = ld.strftime(
|
|
"https://www.sciencenews.org/wp-content/uploads/%Y/%m/%m%d%y_cover.jpg"
|
|
)
|
|
return url
|