Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Fix #4635 (ARS Technica feed returns TOC but no content)

Browse Source
This commit is contained in:
Kovid Goyal 2010-01-22 09:27:49 -07:00
parent 48b98f4a04
commit 427061d5b8
2 changed files with 31 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Changelog.yaml
View File

@ -127,6 +127,7 @@
- FTD
- The National Post
- Blic
- Ars Technica
- version: 0.6.34

51
resources/recipes/ars_technica.recipe
View File

@ -1,12 +1,12 @@
#!/usr/bin/env python
__license__ = 'GPL v3'
__copyright__ = '2008-2009, Darko Miletic <darko.miletic at gmail.com>'
__copyright__ = '2008-2010, Darko Miletic <darko.miletic at gmail.com>'
'''
arstechnica.com
'''
from calibre.web.feeds.news import BasicNewsRecipe
from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag
class ArsTechnica2(BasicNewsRecipe):
title = u'Ars Technica'
@ -18,24 +18,24 @@ class ArsTechnica2(BasicNewsRecipe):
oldest_article = 2
max_articles_per_feed = 100
no_stylesheets = True
encoding = 'utf8'
remove_javascript = True
encoding = 'utf-8'
use_embedded_content = False
extra_css = ' body {font-family: sans-serif} .byline{font-weight: bold; line-height: 1em; font-size: 0.625em; text-decoration: none} '
extra_css = '''
.news-item-title{font-size: medium ;font-family:Arial,Helvetica,sans-serif; font-weight:bold;}
.news-item-teaser{font-size: small ;font-family:Arial,Helvetica,sans-serif; font-weight:bold;}
.news-item-byline{font-size:xx-small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;}
.news-item-text{font-size:x-small;font-family:Arial,Helvetica,sans-serif;}
.news-item-figure-caption-text{font-size:xx-small; font-family:Arial,Helvetica,sans-serif;font-weight:bold;}
.news-item-figure-caption-byline{font-size:xx-small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;}
'''
conversion_options = {
'comments' : description
,'tags' : category
,'language' : language
,'publisher' : publisher
}
keep_only_tags = [dict(name='div', attrs={'id':['news-item-info','news-item']})]
keep_only_tags = [dict(name='div', attrs={'id':['story','etc-story']})]
remove_tags = [
dict(name=['object','link','embed'])
,dict(name='div', attrs={'class':'related-stories'})
,dict(name='div', attrs={'class':'read-more-link'})
]
@ -52,14 +52,19 @@ class ArsTechnica2(BasicNewsRecipe):
]
def append_page(self, soup, appendtag, position):
pager = soup.find('div',attrs={'id':'pager'})
pager = soup.find('div',attrs={'class':'pager'})
if pager:
for atag in pager.findAll('a',href=True):
str = self.tag_to_string(atag)
if str.startswith('Next'):
soup2 = self.index_to_soup(atag['href'])
texttag = soup2.find('div', attrs={'class':'news-item-text'})
nurl = 'http://arstechnica.com' + atag['href']
rawc = self.index_to_soup(nurl,True)
soup2 = BeautifulSoup(rawc, fromEncoding=self.encoding)
readmoretag = soup2.find('div', attrs={'class':'read-more-link'})
if readmoretag:
readmoretag.extract()
texttag = soup2.find('div', attrs={'class':'body'})
for it in texttag.findAll(style=True):
del it['style']
@ -71,10 +76,12 @@ class ArsTechnica2(BasicNewsRecipe):
def preprocess_html(self, soup):
ftag = soup.find('div', attrs={'class':'news-item-byline'})
ftag = soup.find('div', attrs={'class':'byline'})
if ftag:
ftag.insert(4,'<br /><br />')
brtag = Tag(soup,'br')
brtag2 = Tag(soup,'br')
ftag.insert(4,brtag)
ftag.insert(5,brtag2)
for item in soup.findAll(style=True):
del item['style']
@ -83,5 +90,7 @@ class ArsTechnica2(BasicNewsRecipe):
return soup
def get_article_url(self, article):
return article.get('feedburner_origlink', None).rpartition('?')[0]