Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Update Psychology Today

Browse Source
This commit is contained in:
Kovid Goyal 2020-10-26 07:40:16 +05:30
parent 3e9c653b8a
commit 13d12199d5
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
1 changed files with 26 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
recipes/psych.recipe
View File

@ -4,8 +4,18 @@
from calibre.web.feeds.recipes import BasicNewsRecipe from calibre.web.feeds.recipes import BasicNewsRecipe
def check_words(words): def absurl(url):
return lambda x: x and frozenset(words.split()).intersection(x.split()) if url.startswith("//"):
return "https:" + url
if url.startswith("/"):
return "https://www.psychologytoday.com" + url
return url
def classes(classes):
q = frozenset(classes.split(' '))
return dict(attrs={
'class': lambda x: x and frozenset(x.split()).intersection(q)})
class PsychologyToday(BasicNewsRecipe): class PsychologyToday(BasicNewsRecipe):
@ -19,37 +29,28 @@ class PsychologyToday(BasicNewsRecipe):
' connection, health, family, the workplace and culture.') ' connection, health, family, the workplace and culture.')
language = 'en' language = 'en'
encoding = 'UTF-8' encoding = 'UTF-8'
no_javascript = True
no_stylesheets = True no_stylesheets = True
publication_type = 'magazine'
keep_only_tags = [ keep_only_tags = [dict(attrs={'id': 'block-pt-content'})]
dict(name='div', attrs={'data-type': 'article'}) remove_tags = [classes('pt-social-media')]
]
remove_tags = [
dict(attrs={'id': 'pt-social-media'}),
]
def parse_index(self): def parse_index(self):
soup = self.index_to_soup('http://www.psychologytoday.com/magazine') soup = self.index_to_soup('https://www.psychologytoday.com/us/magazine/archive')
div = soup.find(id='block-views-magazine-issues-block') a = soup.find(**classes('magazine-thumbnail')).a
a = div.find('h4').find('a') self.timefmt = ' [%s]' % a['title']
self.timefmt = ' [%s]' % self.tag_to_string(a).capitalize() self.cover_url = absurl(a.img['src'])
soup = self.index_to_soup('http://www.psychologytoday.com' + a['href']) soup = self.index_to_soup(absurl(a['href']))
div = soup.find(role='main')
self.cover_url = div.find(
'img', src=lambda x: x and '/field_magazine_cover/' in x)['src'].partition('?')[0]
articles = [] articles = []
for x in div.findAll('div', {'class': check_words('collection__item')}): for article in soup.find('div', role='article').findAll('article'):
h = x.find(['h2', 'h3', 'h4', 'h5'], { title = self.tag_to_string(article.find(['h2','h3'])).strip()
'class': check_words('blog__title blog_entry__title')}) url = absurl(article.find(['h2','h3']).a['href'])
title = self.tag_to_string(h)
url = 'http://www.psychologytoday.com' + h.find('a')['href']
self.log('\n', title, 'at', url) self.log('\n', title, 'at', url)
desc = x.find(['div', 'p'], {'class': check_words( desc = self.tag_to_string(article.find('p',**classes('description'))).strip()
'collection__subtitle blog_entry__teaser')}).find(text=True) author = self.tag_to_string(article.find('p',**classes('byline')).a).strip()
if desc: if desc:
self.log(desc) self.log(desc)
else: else:
desc = '' desc = ''
articles.append({'title': title, 'url': url, 'description': desc}) articles.append({'title': title, 'url': url, 'description': desc, 'author': author})
return [('Current Issue', articles)] return [('Current Issue', articles)]