from calibre.web.feeds.recipes import BasicNewsRecipe def class_sel(cls): def f(x): return x and cls in x.split() return f class Spectator(BasicNewsRecipe): title = 'Spectator Magazine' __author__ = 'Kovid Goyal' description = 'Magazine' language = 'en' no_stylesheets = True keep_only_tags = dict(name='div', attrs={'class':['article-head', 'article-image', 'article-body']}) remove_tags = [ dict(name='div', attrs={'id':['disqus_thread']}), dict(attrs={'class':['middle-promo']}), ] def parse_spec_section(self, div): h2 = div.find('h2') sectitle = self.tag_to_string(h2) self.log('Section:', sectitle) articles = [] for div in div.findAll('div', id=lambda x: x and x.startswith('post-')): h2 = div.find('h2', attrs={'class':class_sel('post-title')}) title = self.tag_to_string(h2) a = h2.find('a') url = a['href'] desc = '' self.log('\tArticle:', title) p = div.find('p') if p is not None: desc = self.tag_to_string(p) articles.append({'title':title, 'url':url, 'description':desc}) return sectitle, articles def parse_index(self): soup = self.index_to_soup('http://www.spectator.co.uk/magazine/') a = soup.find('a', attrs={'class':'magazine-issue-wrap'}) self.timefmt = a['title'] self.cover_url = a['href'] feeds = [] div = soup.find(id='magazine-full') for x in div.findAll(attrs={'class':class_sel('magazine-section-holder')}): title, articles = self.parse_spec_section(x) if articles: feeds.append((title, articles)) return feeds