diff --git a/recipes/guardian.recipe b/recipes/guardian.recipe index c420e7d6af..b9b20790bd 100644 --- a/recipes/guardian.recipe +++ b/recipes/guardian.recipe @@ -6,10 +6,8 @@ __docformat__ = 'restructuredtext en' ''' www.guardian.co.uk ''' -from calibre import strftime from calibre.web.feeds.news import BasicNewsRecipe from datetime import date -from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag class Guardian(BasicNewsRecipe): @@ -23,15 +21,16 @@ class Guardian(BasicNewsRecipe): cover_pic = 'Guardian digital edition' masthead_url = 'http://static.guim.co.uk/static/f76b43f9dcfd761f0ecf7099a127b603b2922118/common/images/logos/the-guardian/titlepiece.gif' - __author__ = 'Seabound and Sujata Raman' + __author__ = 'Kovid Goyal' language = 'en_GB' oldest_article = 7 max_articles_per_feed = 100 remove_javascript = True encoding = 'utf-8' - compress_news_images = True - compress_news_images_auto_size = 8 + remove_empty_feeds = True + no_stylesheets = True + remove_attributes = ['style'] # List of section titles to ignore # For example: ['Sport'] @@ -40,129 +39,37 @@ class Guardian(BasicNewsRecipe): timefmt = ' [%a, %d %b %Y]' keep_only_tags = [ - dict(name='div', attrs={'id':["content","article_header","main-article-info",]}), - dict(attrs={'class':lambda x: x and set(x.split()).intersection({'content__head', 'content__main'})}), + dict(attrs={'class': lambda x: x and 'content__main-column' in x.split()}), ] remove_tags = [ - dict(name='div', attrs={'class':[ - "video-content","videos-third-column", 'meta__extras', 'submeta-container submeta-container--break-at-leftcol ']}), - dict(name='div', attrs={'id':["article-toolbox","subscribe-feeds",]}), - dict(name='div', attrs={'class':["guardian-tickets promo-component",]}), - dict(name='ul', attrs={'class':["pagination"]}), - dict(name='ul', attrs={'id':["content-actions"]}), - # article history link - dict(name='a', attrs={'class':["rollover history-link"]}), - # "a version of this article ..." speil - dict(name='div' , attrs={'class' : ['section']}), - # "about this article" js dialog - dict(name='div', attrs={'class':["share-top",]}), - # author picture - dict(name='img', attrs={'class':["contributor-pic-small"]}), - # embedded videos/captions - dict(name='span',attrs={'class' : ['inline embed embed-media']}), + dict(attrs={'class': lambda x:x and '--twitter' in x}), + dict(attrs={'data-component': ['share', 'social']}), + dict(attrs={'data-link-name': 'block share'}), + dict(attrs={'class': lambda x:x and 'inline-expand-image' in x}), + dict(attrs={'class': lambda x:x and 'modern-visible' in x.split()}), + ] + remove_tags_after = [ + dict(attrs={'class': lambda x: x and 'content__article-body' in x.split()}), ] - use_embedded_content = False - - no_stylesheets = True - extra_css = ''' - .article-attributes{font-size: x-small; font-family:Arial,Helvetica,sans-serif;} - .h1{font-size: large ;font-family:georgia,serif; font-weight:bold;} - .stand-first-alone{color:#666666; font-size:small; font-family:Arial,Helvetica,sans-serif;} - .caption{color:#666666; font-size:x-small; font-family:Arial,Helvetica,sans-serif;} - #article-wrapper{font-size:small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;} - .main-article-info{font-family:Arial,Helvetica,sans-serif;} - #full-contents{font-size:small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;} - #match-stats-summary{font-size:small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;} - ''' - - def get_article_url(self, article): - url = article.get('guid', None) - if '/video/' in url or '/flyer/' in url or '/quiz/' in url or \ - '/gallery/' in url or 'ivebeenthere' in url or \ - 'pickthescore' in url or 'audioslideshow' in url : - url = None - return url - - def populate_article_metadata(self, article, soup, first): - if first and hasattr(self, 'add_toc_thumbnail'): - picdiv = soup.find('img') - if picdiv is not None: - self.add_toc_thumbnail(article,picdiv['src']) def preprocess_html(self, soup): - - # multiple html sections in soup, useful stuff in the first - html = soup.find('html') - soup2 = BeautifulSoup() - soup2.insert(0,html) - - soup = soup2 - - for item in soup.findAll(style=True): - del item['style'] - - for item in soup.findAll(face=True): - del item['face'] - for tag in soup.findAll(name=['ul','li']): - tag.name = 'div' - - # removes number next to rating stars - items_to_remove = [] - rating_container = soup.find('div', attrs={'class': ['rating-container']}) - if rating_container: - for item in rating_container: - if isinstance(item, Tag) and str(item.name) == 'span': - items_to_remove.append(item) - - for item in items_to_remove: - item.extract() - + for img in soup.findAll('img', srcset=True): + img['src'] = img['srcset'].partition(' ')[0] + img['srcset'] = '' return soup - def find_sections(self): - # soup = self.index_to_soup("http://www.guardian.co.uk/theobserver") - soup = self.index_to_soup(self.base_url) - # find cover pic - img = soup.find('img',attrs={'alt':self.cover_pic}) - if img is not None: - self.cover_url = img['src'] - # end find cover pic - - idx = soup.find('div', id='book-index') - for s in idx.findAll('strong', attrs={'class':'book'}): - a = s.find('a', href=True) - section_title = self.tag_to_string(a) - if section_title not in self.ignore_sections: - prefix = '' - if section_title != 'Main section': - prefix = section_title + ': ' - for subsection in s.parent.findAll('a', attrs={'class':'book-section'}): - yield (prefix + self.tag_to_string(subsection), subsection['href']) - - def find_articles(self, url): - soup = self.index_to_soup(url) - div = soup.find('div', attrs={'class':'book-index'}) - for ul in div.findAll('ul', attrs={'class':'trailblock'}): - for li in ul.findAll('li'): - a = li.find(href=True) - if not a: - continue - title = self.tag_to_string(a) - url = a['href'] - if not title or not url: - continue - tt = li.find('div', attrs={'class':'trailtext'}) - if tt is not None: - for da in tt.findAll('a'): - da.extract() - desc = self.tag_to_string(tt).strip() - yield { - 'title': title, 'url':url, 'description':desc, - 'date' : strftime('%a, %d %b'), - } - def parse_index(self): feeds = [] - for title, href in self.find_sections(): - feeds.append((title, list(self.find_articles(href)))) + soup = self.index_to_soup(self.base_url) + for section in soup.findAll('section'): + title = self.tag_to_string(section.find(attrs={'class':'fc-container__header__title'})).strip().capitalize() + self.log('\nFound section:', title) + feeds.append((title, [])) + for li in section.findAll('li'): + for a in li.findAll('a', attrs={'data-link-name':'article'}, href=True): + title = self.tag_to_string(a).strip() + url = a['href'] + self.log(' ', title, url) + feeds[-1][1].append({'title':title, 'url':url}) + break return feeds