#!/usr/bin/env python __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' __docformat__ = 'restructuredtext en' ''' www.guardian.co.uk ''' from calibre import random_user_agent from calibre.web.feeds.news import BasicNewsRecipe from datetime import date from lxml import etree def classes(classes): q = frozenset(classes.split(' ')) return dict(attrs={ 'class': lambda x: x and frozenset(x.split()).intersection(q)}) class Guardian(BasicNewsRecipe): title = u'The Guardian and The Observer' if date.today().weekday() == 6: base_url = "https://www.theguardian.com/theobserver" cover_pic = 'Observer digital edition' masthead_url = 'http://static.guim.co.uk/sys-images/Guardian/Pix/site_furniture/2010/10/19/1287478087992/The-Observer-001.gif' else: base_url = "https://www.theguardian.com/uk" cover_pic = 'Guardian digital edition' masthead_url = 'http://static.guim.co.uk/static/f76b43f9dcfd761f0ecf7099a127b603b2922118/common/images/logos/the-guardian/titlepiece.gif' __author__ = 'Kovid Goyal' language = 'en_GB' oldest_article = 7 max_articles_per_feed = 100 remove_javascript = True encoding = 'utf-8' remove_empty_feeds = True no_stylesheets = True remove_attributes = ['style', 'width', 'height'] ignore_duplicate_articles = {'title', 'url'} timefmt = ' [%a, %d %b %Y]' remove_tags = [ dict(attrs={'class': lambda x: x and '--twitter' in x}), dict(attrs={'class': lambda x: x and 'submeta' in x.split()}), dict(attrs={'data-component': ['share', 'social']}), dict(attrs={'data-link-name': 'block share'}), dict(attrs={'class': lambda x: x and 'inline-expand-image' in x}), dict(name='a', attrs={'aria-label': lambda x: x and 'Share On' in x}), dict(name='a', attrs={'class': lambda x: x and 'social__action js-social__action--top' in x}), dict(name='div', attrs={'id': 'share-count-root'}), dict(attrs={'class': lambda x: x and 'modern-visible' in x.split()}), classes('badge-slot reveal-caption__checkbox mobile-only element-rich-link'), dict(name=['link', 'meta', 'style', 'svg', 'input', 'source']), ] remove_tags_after = [ dict(attrs={'class': lambda x: x and 'content__article-body' in x.split()}), dict(attrs={'class': lambda x: x and 'article-body-commercial-selector' in x.split()}), ] extra_css = """ img { max-width: 100% !important; max-height: 100% !important; } a span { color: #E05E02; } figcaption span { font-size: 0.5em; color: #6B6B6B; } """ def get_browser(self, *a, **kw): # This site returns images in JPEG-XR format if the user agent is IE if not hasattr(self, 'non_ie_ua'): try: self.non_ie_ua = random_user_agent(allow_ie=False) except TypeError: self.non_ie_ua = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.111 Safari/537.36' kw['user_agent'] = self.non_ie_ua br = BasicNewsRecipe.get_browser(self, *a, **kw) return br def parse_section(self, url, title_prefix=''): feeds = [] soup = self.index_to_soup(url) for section in soup.findAll('section'): title = title_prefix + self.tag_to_string(section.find( attrs={'class': 'fc-container__header__title'})).strip().capitalize() self.log('\nFound section:', title) if 'Video' in title: self.log('=======> Skip section:', title) continue feeds.append((title, [])) for li in section.findAll('li'): for a in li.findAll('a', attrs={'data-link-name': 'article'}, href=True): title = self.tag_to_string(a).strip() url = a['href'] self.log(' ', title, url) feeds[-1][1].append({'title': title, 'url': url}) break return feeds def parse_index(self): feeds = self.parse_section(self.base_url) feeds += self.parse_section( 'https://www.theguardian.com/uk/sport', 'Sport - ') if date.today().weekday() in (5, 6): feeds += self.parse_section('https://www.theguardian.com/theguardian/weekend', 'Weekend - ') return feeds def preprocess_html(self, soup): old_body = soup.find('body') if old_body is not None: main_column = soup.find(**classes('content__main-column')) if main_column is None: for section in soup.findAll('section'): if section.find('h1') is not None: main_column = section break if main_column is not None: body = soup.new_tag('body') body.append(main_column) old_body.replaceWith(body) for img in soup.findAll('img', srcset=True): img['src'] = img['srcset'].partition(' ')[0] img['srcset'] = '' return soup def preprocess_raw_html(self, raw, url): try: root = self.index_to_soup(raw, as_tree=True) # find out Share on facebook elements share_platforms = root.xpath('//div//div//div/ul/li/a[starts-with(@aria-label,"Share on")]') if share_platforms and len(share_platforms) > 0: parent_div = share_platforms[0].getparent().getparent().getparent() ul_elem = share_platforms[0].getparent().getparent() # remove ul element parent_div.remove(ul_elem) share_counts = root.xpath('//div//div//div[contains(@id,"-count-root")]') if share_counts and len(share_counts) > 0: share_parent_div = share_counts[0].getparent().getparent() # remove share count element share_parent_div.remove(share_counts[0].getparent()) pub_date_div = root.xpath('//div//div//div//input[@id="dateToggle"]') if pub_date_div: input_elem = pub_date_div[0] date_parent = input_elem.getparent() # remove the input element date_parent.remove(input_elem) # try to remove last modified p last_modified_p = date_parent.xpath('./p') if last_modified_p and len(last_modified_p) > 0: date_parent.remove(last_modified_p[0]) # rebuild the html raw string raw = etree.tostring(root, encoding='unicode') except Exception as e: self.log('preprocess_raw_html error -> {}'.format(e)) return raw