Fix #4635 (ARS Technica feed returns TOC but no content)

Changelog.yaml

@@ -127,6 +127,7 @@
     - FTD
     - The National Post
     - Blic
+    - Ars Technica
 
 
 - version: 0.6.34

resources/recipes/ars_technica.recipe

@@ -1,12 +1,12 @@
-#!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008-2009, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2010, Darko Miletic <darko.miletic at gmail.com>'
 '''
 arstechnica.com
 '''
 
 from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag
 
 class ArsTechnica2(BasicNewsRecipe):
     title                 = u'Ars Technica'
@@ -18,24 +18,24 @@ class ArsTechnica2(BasicNewsRecipe):
     oldest_article        = 2
     max_articles_per_feed = 100
     no_stylesheets        = True
-    encoding              = 'utf8'
+    encoding              = 'utf-8'
-    remove_javascript     = True
     use_embedded_content  = False
+    extra_css             = ' body {font-family: sans-serif} .byline{font-weight: bold; line-height: 1em; font-size: 0.625em; text-decoration: none} '
 
-    extra_css = '''
+    conversion_options = {
-                    .news-item-title{font-size: medium ;font-family:Arial,Helvetica,sans-serif; font-weight:bold;}
+                             'comments'  : description
-                    .news-item-teaser{font-size: small ;font-family:Arial,Helvetica,sans-serif; font-weight:bold;}
+                            ,'tags'      : category
-                    .news-item-byline{font-size:xx-small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;}
+                            ,'language'  : language
-                    .news-item-text{font-size:x-small;font-family:Arial,Helvetica,sans-serif;}
+                            ,'publisher' : publisher
-                    .news-item-figure-caption-text{font-size:xx-small; font-family:Arial,Helvetica,sans-serif;font-weight:bold;}
+                         }
-                    .news-item-figure-caption-byline{font-size:xx-small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;}
-                '''
 
-    keep_only_tags = [dict(name='div', attrs={'id':['news-item-info','news-item']})]
+
+
+    keep_only_tags = [dict(name='div', attrs={'id':['story','etc-story']})]
 
     remove_tags = [
                      dict(name=['object','link','embed'])
-                    ,dict(name='div', attrs={'class':'related-stories'})
+                    ,dict(name='div', attrs={'class':'read-more-link'})
                   ]
 
 
@@ -52,14 +52,19 @@ class ArsTechnica2(BasicNewsRecipe):
             ]
 
     def append_page(self, soup, appendtag, position):
-        pager = soup.find('div',attrs={'id':'pager'})
+        pager = soup.find('div',attrs={'class':'pager'})
         if pager:
            for atag in pager.findAll('a',href=True):
                str = self.tag_to_string(atag)
                if str.startswith('Next'):
-                  soup2 = self.index_to_soup(atag['href'])
+                  nurl = 'http://arstechnica.com' + atag['href']
-
+                  rawc = self.index_to_soup(nurl,True)
-                  texttag = soup2.find('div', attrs={'class':'news-item-text'})
+                  soup2 = BeautifulSoup(rawc, fromEncoding=self.encoding)
+ 
+                  readmoretag = soup2.find('div', attrs={'class':'read-more-link'})
+                  if readmoretag:
+                     readmoretag.extract()
+                  texttag = soup2.find('div', attrs={'class':'body'})
                   for it in texttag.findAll(style=True):
                       del it['style']
 
@@ -71,10 +76,12 @@ class ArsTechnica2(BasicNewsRecipe):
 
 
     def preprocess_html(self, soup):
-
+        ftag = soup.find('div', attrs={'class':'byline'})
-        ftag = soup.find('div', attrs={'class':'news-item-byline'})
         if ftag:
-           ftag.insert(4,'<br /><br />')
+           brtag = Tag(soup,'br')
+           brtag2 = Tag(soup,'br')
+           ftag.insert(4,brtag)
+           ftag.insert(5,brtag2)
 
         for item in soup.findAll(style=True):
            del item['style']
@@ -83,5 +90,7 @@ class ArsTechnica2(BasicNewsRecipe):
 
         return soup
 
+    def get_article_url(self, article):
+        return article.get('feedburner_origlink',  None).rpartition('?')[0]