Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Update Ars Technica

Browse Source
This commit is contained in:
Kovid Goyal 2016-09-03 09:00:47 +05:30
parent efcecb8e67
commit 943f6eef2e
1 changed files with 28 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
recipes/ars_technica.recipe
View File

@ -4,6 +4,7 @@ __copyright__ = '2008-2012, Darko Miletic <darko.miletic at gmail.com>'
arstechnica.com arstechnica.com
''' '''
import re
from calibre.web.feeds.news import BasicNewsRecipe from calibre.web.feeds.news import BasicNewsRecipe
from calibre.ebooks.BeautifulSoup import BeautifulSoup from calibre.ebooks.BeautifulSoup import BeautifulSoup
@ -14,36 +15,34 @@ class ArsTechnica(BasicNewsRecipe):
__author__ = 'Darko Miletic, Sujata Raman, Alexis Rohou, Tom Sparks' __author__ = 'Darko Miletic, Sujata Raman, Alexis Rohou, Tom Sparks'
description = 'Ars Technica: Serving the technologist for 1.2 decades' description = 'Ars Technica: Serving the technologist for 1.2 decades'
publisher = 'Conde Nast Publications' publisher = 'Conde Nast Publications'
category = 'news, IT, technology'
oldest_article = 5 oldest_article = 5
max_articles_per_feed = 100 max_articles_per_feed = 100
no_stylesheets = True no_stylesheets = True
encoding = 'utf-8' encoding = 'utf-8'
use_embedded_content = False use_embedded_content = False
remove_empty_feeds = True remove_empty_feeds = True
publication_type = 'newsportal'
extra_css = ''' extra_css = '''
body {font-family: Arial,sans-serif} body {font-family: Arial,sans-serif}
.heading{font-family: "Times New Roman",serif} .heading{font-family: "Times New Roman",serif}
.byline{font-weight: bold; line-height: 1em; font-size: 0.625em; text-decoration: none} .byline{font-weight: bold; line-height: 1em; font-size: 0.625em; text-decoration: none}
img{display: block} img{display: block}
.caption-text{font-size:small; font-style:italic} .caption-text{font-size:small; font-style:italic}
.caption-byline{font-size:small; font-style:italic; font-weight:bold} .caption-byline{font-size:small; font-style:italic; font-weight:bold}
.video, .page-numbers, .story-sidebar { display: none }
.image { display: block }
''' '''
conversion_options = {
'comments': description, 'tags': category, 'language': language, 'publisher': publisher
}
keep_only_tags = [ keep_only_tags = [
dict(attrs={'class': 'standalone'}), dict(attrs={'id': 'article-guts'}) dict(itemprop=['headline', 'description']), dict(attrs={'class': ['post-meta', 'article-guts', 'standalone']})
] ]
remove_tags = [ remove_tags = [
dict(name=['object', 'link', 'embed', 'iframe', 'meta']), dict(attrs={'class': 'corner-info'}), dict(attrs={ dict(name=['object', 'link', 'embed', 'iframe', 'meta']),
'id': 'article-footer-wrap'}), dict(attrs={'class': 'article-expander'}), dict(name='nav', attrs={'class': 'subheading'}) dict(attrs={'class': ['video', 'corner-info', 'article-expander']}),
dict(id=['social-left', 'article-footer-wrap']),
dict(name='nav', attrs={'class': 'subheading'}),
] ]
remove_attributes = ['lang'] remove_attributes = ['lang', 'style']
feeds = [ feeds = [
(u'Ars Features (All our long-form feature articles)', u'http://feeds.arstechnica.com/arstechnica/features'), (u'Ars Features (All our long-form feature articles)', u'http://feeds.arstechnica.com/arstechnica/features'),
@ -68,12 +67,13 @@ class ArsTechnica(BasicNewsRecipe):
nurl = nexttag.parent['href'] nurl = nexttag.parent['href']
rawc = self.index_to_soup(nurl, True) rawc = self.index_to_soup(nurl, True)
soup2 = BeautifulSoup(rawc, fromEncoding=self.encoding) soup2 = BeautifulSoup(rawc, fromEncoding=self.encoding)
texttag = soup2.find(attrs={'id': 'article-guts'}) texttag = soup2.find(attrs={'class': 'article-guts'})
newpos = len(texttag.contents) if texttag is not None:
self.append_page(soup2, texttag, newpos) newpos = len(texttag.contents)
texttag.extract() self.append_page(soup2, texttag, newpos)
pager.extract() texttag.extract()
appendtag.insert(position, texttag) pager.extract()
appendtag.insert(position, texttag)
def preprocess_html(self, soup): def preprocess_html(self, soup):
self.append_page(soup, soup.body, 3) self.append_page(soup, soup.body, 3)
@ -89,9 +89,12 @@ class ArsTechnica(BasicNewsRecipe):
else: else:
str = self.tag_to_string(item) str = self.tag_to_string(item)
item.replaceWith(str) item.replaceWith(str)
for item in soup.findAll('img'): for div in soup.findAll('div', attrs={'class':'image', 'style':lambda x: x and 'background-image' in x}):
if 'alt' not in item: url = re.search(r'''url\(['"]?([^'")]+)''', div['style'])
item['alt'] = 'image' if url is not None:
div.name = 'img'
div['src'] = url.group(1)
div['style'] = ''
return soup return soup
def preprocess_raw_html(self, raw, url): def preprocess_raw_html(self, raw, url):