Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Update National Geographic

Browse Source
This commit is contained in:
Kovid Goyal 2018-04-13 07:51:00 +05:30
parent 8665b6c870
commit 0a94c5420c
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
1 changed files with 29 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
recipes/natgeo.recipe
View File

@ -1,13 +1,16 @@
#!/usr/bin/env python2 #!/usr/bin/env python2
# vim:fileencoding=utf-8 # vim:fileencoding=utf-8
from __future__ import (unicode_literals, division, absolute_import, from __future__ import absolute_import, division, print_function, unicode_literals
print_function)
import re import json
from calibre.web.feeds.news import BasicNewsRecipe
from calibre.ebooks.BeautifulSoup import Tag from calibre.ebooks.BeautifulSoup import Tag
import html5lib from calibre.web.feeds.news import BasicNewsRecipe
from lxml.html import tostring
from css_selectors import Select
def classes(classes):
q = frozenset(classes.split(' '))
return dict(attrs={
'class': lambda x: x and frozenset(x.split()).intersection(q)})
class NatGeo(BasicNewsRecipe): class NatGeo(BasicNewsRecipe):
@ -26,6 +29,16 @@ class NatGeo(BasicNewsRecipe):
no_stylesheets = True no_stylesheets = True
use_embedded_content = False use_embedded_content = False
remove_attributes = ['style'] remove_attributes = ['style']
remove_javascript = False
keep_only_tags = [
classes('mainArt byline'),
dict(id='article__body'),
]
remove_tags = [
classes('hide-from-mobile ad-holder enlarge-button'),
dict(name='svg meta'.split()),
]
feeds = [ feeds = [
(u'Daily News', u'http://feeds.nationalgeographic.com/ng/News/News_Main') (u'Daily News', u'http://feeds.nationalgeographic.com/ng/News/News_Main')
@ -39,30 +52,13 @@ class NatGeo(BasicNewsRecipe):
feed.articles.remove(article) feed.articles.remove(article)
return feeds return feeds
def preprocess_raw_html(self, raw_html, url):
# BeautifulSoup does not parse the natgeo html correctly, so we use a
# custom cleanup routine
# NatGeo embeds huge inline svg images as interactive graphics, this
# breaks conversion because split points cannot be found
raw_html = re.sub(r'<svg.+?</svg>', '', raw_html, flags=re.DOTALL)
root = html5lib.parse(
raw_html, namespaceHTMLElements=False, treebuilder='lxml')
select = Select(root)
keep = tuple(select('.mainArt')) + tuple(select('.byline')
) + tuple(select('#article__body'))
body = root.xpath('//body')[0]
for elem in keep:
body.append(elem)
for child in tuple(body.iterchildren('*')):
if child not in keep:
body.remove(child)
for head in root.xpath('//head'):
for child in tuple(head.iterchildren('*')):
head.remove(child)
return tostring(root, encoding=unicode)
def preprocess_html(self, soup): def preprocess_html(self, soup):
for div in soup.findAll(attrs={'data-pestle-module': 'PictureFill'}):
script = div.find('script')
src = json.loads(self.tag_to_string(script))['src']
div.name = 'img'
div['src'] = src
for div in soup.findAll(attrs={'data-src': True, 'class': 'delayed-image-load'}): for div in soup.findAll(attrs={'data-src': True, 'class': 'delayed-image-load'}):
url = div['data-src'] url = div['data-src']
idx = url.find('.jpg/{width') idx = url.find('.jpg/{width')
@ -71,4 +67,7 @@ class NatGeo(BasicNewsRecipe):
img = Tag(soup, "img") img = Tag(soup, "img")
img['src'] = url img['src'] = url
div.append(img) div.append(img)
for script in soup.findAll('script'):
script.extract()
return soup return soup