Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Update Popular Science

Browse Source
This commit is contained in:
Kovid Goyal 2020-07-01 21:38:36 +05:30
parent 9fed1d5092
commit ea059d5e4c
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
1 changed files with 32 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
recipes/popscience.recipe
View File

@ -1,5 +1,14 @@
#!/usr/bin/env python2
# vim:fileencoding=utf-8
# License: GPLv3 Copyright: 2020, Kovid Goyal <kovid at kovidgoyal.net>
from __future__ import absolute_import, division, print_function, unicode_literals
import os
from collections import defaultdict
from calibre.ptempfile import PersistentTemporaryFile
from calibre.web.feeds.news import BasicNewsRecipe from calibre.web.feeds.news import BasicNewsRecipe
import re
class AdvancedUserRecipe1282101454(BasicNewsRecipe): class AdvancedUserRecipe1282101454(BasicNewsRecipe):
@ -8,36 +17,33 @@ class AdvancedUserRecipe1282101454(BasicNewsRecipe):
__author__ = 'Kovid Goyal' __author__ = 'Kovid Goyal'
description = 'Popular Science' description = 'Popular Science'
publisher = 'Popular Science' publisher = 'Popular Science'
oldest_article = 7 # change this if you want more current articles. I like to go a week in
max_articles_per_feed = 100 max_articles_per_feed = 100
no_stylesheets = True no_stylesheets = True
remove_javascript = True
use_embedded_content = False
remove_empty_feeds = True remove_empty_feeds = True
ignore_duplicate_articles = {'url'} ignore_duplicate_articles = {'url'}
feeds = [ def parse_index(self):
('Gadgets', 'http://www.popsci.com/full-feed/gadgets'), from lxml import etree
('Cars', 'http://www.popsci.com/full-feed/cars'), raw = self.index_to_soup('https://www.popsci.com/arcio/rss/', raw=True)
('Science', 'http://www.popsci.com/full-feed/science'), root = etree.fromstring(raw)
('Technology', 'http://www.popsci.com/full-feed/technology'), ans = defaultdict(list)
('DIY', 'http://www.popsci.com/full-feed/diy'), for item in root.iterdescendants('item'):
('Animals', 'https://www.popsci.com/rss-animals.xml'), title = item.find('title').text
('Space', 'https://www.popsci.com/rss-space.xml'), desc = item.find('description').text
('Environment', 'https://www.popsci.com/rss-environment.xml'), author = item.find('dc:creator', {'dc': 'http://purl.org/dc/elements/1.1/'}).text
('Eastern Arsenal', 'https://www.popsci.com/rss-eastern-arsenal.xml'), section = item.find('category').text
] content = item.find('content:encoded', {'content': 'http://purl.org/rss/1.0/modules/content/'}).text
content = '<h1>{title}</h1><h3>{desc}</h3><p>{author}</p><div>{content}</div>'.format(
pane_node_body = re.compile('pane-node-(?:\\w+-){0,9}body') title=title, author=author, desc=desc, content=content)
desc += ' by ' + author
keep_only_tags = [ with PersistentTemporaryFile('popsci.html') as pt:
dict(attrs={'class': lambda x: x and frozenset('pane-node-header'.split()).issubset(frozenset(x.split()))}), pt.write(content.encode('utf-8'))
dict(attrs={'class': pane_node_body}), ans[section].append({
] 'title': title,
'description': desc,
remove_tags = [ 'url': 'file:///' + pt.name.replace(os.sep, '/'),
dict(attrs={'class': lambda x: x and frozenset('ads seperator'.split()).issubset(frozenset(x.split()))}), })
] return [(sec, ans[sec]) for sec in sorted(ans)]
def preprocess_html(self, soup): def preprocess_html(self, soup):
for img in soup.findAll('img', attrs={'data-medsrc': True}): for img in soup.findAll('img', attrs={'data-medsrc': True}):