New recipe for Physicsworld and updated Ars Technica recipe for multi-page support

src/calibre/web/feeds/recipes/__init__.py

@@ -29,7 +29,7 @@ recipe_modules = ['recipe_' + r for r in (
            'jb_online', 'estadao', 'o_globo', 'vijesti', 'elmundo', 'the_oz',
            'honoluluadvertiser', 'starbulletin', 'exiled', 'indy_star', 'dna',
            'pobjeda', 'chicago_breaking_news', 'glasgow_herald', 'linuxdevices',
-           'hindu', 'cincinnati_enquirer',
+           'hindu', 'cincinnati_enquirer', 'physics_world',
           )]
 
 import re, imp, inspect, time, os

src/calibre/web/feeds/recipes/recipe_ars_technica.py

@@ -15,7 +15,6 @@ class ArsTechnica2(BasicNewsRecipe):
     description           = 'The art of technology'    
     publisher             = 'Ars Technica'
     category              = 'news, IT, technology'    
-    language              = _('English')
     oldest_article        = 2
     max_articles_per_feed = 100
     no_stylesheets        = True
@@ -50,11 +49,29 @@ class ArsTechnica2(BasicNewsRecipe):
              ,(u'Nobel Intent (Science content)'       , u'http://feeds.arstechnica.com/arstechnica/science/'    )
              ,(u'Law & Disorder (Tech policy content)' , u'http://feeds.arstechnica.com/arstechnica/tech-policy/')
             ]
-
+    
+    def append_page(self, soup, appendtag, position):
+        pager = soup.find('div',attrs={'id':'pager'})
+        if pager:
+           for atag in pager.findAll('a',href=True):
+               str = self.tag_to_string(atag)
+               if str.startswith('Next'):
+                  soup2 = self.index_to_soup(atag['href'])
+                  texttag = soup2.find('div', attrs={'class':'news-item-text'})
+                  for it in texttag.findAll(style=True):
+                      del it['style']
+                  newpos = len(texttag.contents)
+                  self.append_page(soup2,texttag,newpos)
+                  texttag.extract()
+                  pager.extract()
+                  appendtag.insert(position,texttag)
+        
+    
     def preprocess_html(self, soup):
         ftag = soup.find('div', attrs={'class':'news-item-byline'})
         if ftag:
            ftag.insert(4,'<br /><br />')
         for item in soup.findAll(style=True):
             del item['style']
+        self.append_page(soup, soup.body, 3)
         return soup

src/calibre/web/feeds/recipes/recipe_physics_world.py

@@ -0,0 +1,35 @@
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class PhysicsWorld(BasicNewsRecipe):
+    title          = u'Physicsworld'
+    description    = 'News from the world of physics'
+    __author__     = 'Hypernova'
+    language       = _('English')
+    oldest_article = 7
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    remove_javascript     = True
+    needs_subscription = True
+    remove_tags_before = dict(name='h1')
+    remove_tags_after = [dict(name='div', attrs={'id':'shareThis'})]
+    preprocess_regexps = [
+   (re.compile(r'<div id="shareThis">.*</body>', re.DOTALL|re.IGNORECASE),
+    lambda match: '</body>'),
+]    
+    feeds          = [
+                          (u'Headlines News', u'http://feeds.feedburner.com/PhysicsWorldNews')
+                      ]
+    
+    def get_browser(self):
+        br = BasicNewsRecipe.get_browser(self)
+        if self.username is not None and self.password is not None:
+            br.open('http://physicsworld.com/cws/sign-in')
+            br.select_form(nr=1)
+            br['username'] = self.username
+            br['password'] = self.password
+            br.submit()
+        return br
+        
+