diff --git a/Changelog.yaml b/Changelog.yaml index eb221f0213..8399532bb1 100644 --- a/Changelog.yaml +++ b/Changelog.yaml @@ -127,6 +127,7 @@ - FTD - The National Post - Blic + - Ars Technica - version: 0.6.34 diff --git a/resources/recipes/ars_technica.recipe b/resources/recipes/ars_technica.recipe index e5b54edc03..717a47dd0c 100644 --- a/resources/recipes/ars_technica.recipe +++ b/resources/recipes/ars_technica.recipe @@ -1,12 +1,12 @@ -#!/usr/bin/env python __license__ = 'GPL v3' -__copyright__ = '2008-2009, Darko Miletic ' +__copyright__ = '2008-2010, Darko Miletic ' ''' arstechnica.com ''' from calibre.web.feeds.news import BasicNewsRecipe +from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag class ArsTechnica2(BasicNewsRecipe): title = u'Ars Technica' @@ -18,24 +18,24 @@ class ArsTechnica2(BasicNewsRecipe): oldest_article = 2 max_articles_per_feed = 100 no_stylesheets = True - encoding = 'utf8' - remove_javascript = True + encoding = 'utf-8' use_embedded_content = False + extra_css = ' body {font-family: sans-serif} .byline{font-weight: bold; line-height: 1em; font-size: 0.625em; text-decoration: none} ' - extra_css = ''' - .news-item-title{font-size: medium ;font-family:Arial,Helvetica,sans-serif; font-weight:bold;} - .news-item-teaser{font-size: small ;font-family:Arial,Helvetica,sans-serif; font-weight:bold;} - .news-item-byline{font-size:xx-small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;} - .news-item-text{font-size:x-small;font-family:Arial,Helvetica,sans-serif;} - .news-item-figure-caption-text{font-size:xx-small; font-family:Arial,Helvetica,sans-serif;font-weight:bold;} - .news-item-figure-caption-byline{font-size:xx-small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;} - ''' + conversion_options = { + 'comments' : description + ,'tags' : category + ,'language' : language + ,'publisher' : publisher + } - keep_only_tags = [dict(name='div', attrs={'id':['news-item-info','news-item']})] + + + keep_only_tags = [dict(name='div', attrs={'id':['story','etc-story']})] remove_tags = [ dict(name=['object','link','embed']) - ,dict(name='div', attrs={'class':'related-stories'}) + ,dict(name='div', attrs={'class':'read-more-link'}) ] @@ -52,14 +52,19 @@ class ArsTechnica2(BasicNewsRecipe): ] def append_page(self, soup, appendtag, position): - pager = soup.find('div',attrs={'id':'pager'}) + pager = soup.find('div',attrs={'class':'pager'}) if pager: for atag in pager.findAll('a',href=True): str = self.tag_to_string(atag) if str.startswith('Next'): - soup2 = self.index_to_soup(atag['href']) - - texttag = soup2.find('div', attrs={'class':'news-item-text'}) + nurl = 'http://arstechnica.com' + atag['href'] + rawc = self.index_to_soup(nurl,True) + soup2 = BeautifulSoup(rawc, fromEncoding=self.encoding) + + readmoretag = soup2.find('div', attrs={'class':'read-more-link'}) + if readmoretag: + readmoretag.extract() + texttag = soup2.find('div', attrs={'class':'body'}) for it in texttag.findAll(style=True): del it['style'] @@ -71,10 +76,12 @@ class ArsTechnica2(BasicNewsRecipe): def preprocess_html(self, soup): - - ftag = soup.find('div', attrs={'class':'news-item-byline'}) + ftag = soup.find('div', attrs={'class':'byline'}) if ftag: - ftag.insert(4,'

') + brtag = Tag(soup,'br') + brtag2 = Tag(soup,'br') + ftag.insert(4,brtag) + ftag.insert(5,brtag2) for item in soup.findAll(style=True): del item['style'] @@ -83,5 +90,7 @@ class ArsTechnica2(BasicNewsRecipe): return soup + def get_article_url(self, article): + return article.get('feedburner_origlink', None).rpartition('?')[0]