diff --git a/recipes/psych.recipe b/recipes/psych.recipe index f5f00ec0d2..12eeacd4a5 100644 --- a/recipes/psych.recipe +++ b/recipes/psych.recipe @@ -4,8 +4,18 @@ from calibre.web.feeds.recipes import BasicNewsRecipe -def check_words(words): - return lambda x: x and frozenset(words.split()).intersection(x.split()) +def absurl(url): + if url.startswith("//"): + return "https:" + url + if url.startswith("/"): + return "https://www.psychologytoday.com" + url + return url + + +def classes(classes): + q = frozenset(classes.split(' ')) + return dict(attrs={ + 'class': lambda x: x and frozenset(x.split()).intersection(q)}) class PsychologyToday(BasicNewsRecipe): @@ -19,37 +29,28 @@ class PsychologyToday(BasicNewsRecipe): ' connection, health, family, the workplace and culture.') language = 'en' encoding = 'UTF-8' - no_javascript = True no_stylesheets = True + publication_type = 'magazine' - keep_only_tags = [ - dict(name='div', attrs={'data-type': 'article'}) - ] - remove_tags = [ - dict(attrs={'id': 'pt-social-media'}), - ] + keep_only_tags = [dict(attrs={'id': 'block-pt-content'})] + remove_tags = [classes('pt-social-media')] def parse_index(self): - soup = self.index_to_soup('http://www.psychologytoday.com/magazine') - div = soup.find(id='block-views-magazine-issues-block') - a = div.find('h4').find('a') - self.timefmt = ' [%s]' % self.tag_to_string(a).capitalize() - soup = self.index_to_soup('http://www.psychologytoday.com' + a['href']) - div = soup.find(role='main') - self.cover_url = div.find( - 'img', src=lambda x: x and '/field_magazine_cover/' in x)['src'].partition('?')[0] + soup = self.index_to_soup('https://www.psychologytoday.com/us/magazine/archive') + a = soup.find(**classes('magazine-thumbnail')).a + self.timefmt = ' [%s]' % a['title'] + self.cover_url = absurl(a.img['src']) + soup = self.index_to_soup(absurl(a['href'])) articles = [] - for x in div.findAll('div', {'class': check_words('collection__item')}): - h = x.find(['h2', 'h3', 'h4', 'h5'], { - 'class': check_words('blog__title blog_entry__title')}) - title = self.tag_to_string(h) - url = 'http://www.psychologytoday.com' + h.find('a')['href'] + for article in soup.find('div', role='article').findAll('article'): + title = self.tag_to_string(article.find(['h2','h3'])).strip() + url = absurl(article.find(['h2','h3']).a['href']) self.log('\n', title, 'at', url) - desc = x.find(['div', 'p'], {'class': check_words( - 'collection__subtitle blog_entry__teaser')}).find(text=True) + desc = self.tag_to_string(article.find('p',**classes('description'))).strip() + author = self.tag_to_string(article.find('p',**classes('byline')).a).strip() if desc: self.log(desc) else: desc = '' - articles.append({'title': title, 'url': url, 'description': desc}) + articles.append({'title': title, 'url': url, 'description': desc, 'author': author}) return [('Current Issue', articles)]