mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-09-29 15:31:08 -04:00
78 lines
2.9 KiB
Python
78 lines
2.9 KiB
Python
#!/usr/bin/env python2
|
|
__license__ = 'GPL v3'
|
|
__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
|
|
__docformat__ = 'restructuredtext en'
|
|
|
|
'''
|
|
www.guardian.co.uk
|
|
'''
|
|
from calibre.web.feeds.news import BasicNewsRecipe
|
|
from datetime import date
|
|
|
|
class Guardian(BasicNewsRecipe):
|
|
|
|
title = u'The Guardian and The Observer'
|
|
if date.today().weekday() == 6:
|
|
base_url = "http://www.guardian.co.uk/theobserver"
|
|
cover_pic = 'Observer digital edition'
|
|
masthead_url = 'http://static.guim.co.uk/sys-images/Guardian/Pix/site_furniture/2010/10/19/1287478087992/The-Observer-001.gif'
|
|
else:
|
|
base_url = "http://www.guardian.co.uk/theguardian"
|
|
cover_pic = 'Guardian digital edition'
|
|
masthead_url = 'http://static.guim.co.uk/static/f76b43f9dcfd761f0ecf7099a127b603b2922118/common/images/logos/the-guardian/titlepiece.gif'
|
|
|
|
__author__ = 'Kovid Goyal'
|
|
language = 'en_GB'
|
|
|
|
oldest_article = 7
|
|
max_articles_per_feed = 100
|
|
remove_javascript = True
|
|
encoding = 'utf-8'
|
|
remove_empty_feeds = True
|
|
no_stylesheets = True
|
|
remove_attributes = ['style']
|
|
ignore_duplicate_articles = {'title', 'url'}
|
|
|
|
timefmt = ' [%a, %d %b %Y]'
|
|
|
|
keep_only_tags = [
|
|
dict(attrs={'class': lambda x: x and 'content__main-column' in x.split()}),
|
|
]
|
|
remove_tags = [
|
|
dict(attrs={'class': lambda x:x and '--twitter' in x}),
|
|
dict(attrs={'data-component': ['share', 'social']}),
|
|
dict(attrs={'data-link-name': 'block share'}),
|
|
dict(attrs={'class': lambda x:x and 'inline-expand-image' in x}),
|
|
dict(attrs={'class': lambda x:x and 'modern-visible' in x.split()}),
|
|
]
|
|
remove_tags_after = [
|
|
dict(attrs={'class': lambda x: x and 'content__article-body' in x.split()}),
|
|
]
|
|
|
|
def preprocess_html(self, soup):
|
|
for img in soup.findAll('img', srcset=True):
|
|
img['src'] = img['srcset'].partition(' ')[0]
|
|
img['srcset'] = ''
|
|
return soup
|
|
|
|
def parse_section(self, url, title_prefix=''):
|
|
feeds = []
|
|
soup = self.index_to_soup(url)
|
|
for section in soup.findAll('section'):
|
|
title = title_prefix + self.tag_to_string(section.find(attrs={'class':'fc-container__header__title'})).strip().capitalize()
|
|
self.log('\nFound section:', title)
|
|
feeds.append((title, []))
|
|
for li in section.findAll('li'):
|
|
for a in li.findAll('a', attrs={'data-link-name':'article'}, href=True):
|
|
title = self.tag_to_string(a).strip()
|
|
url = a['href']
|
|
self.log(' ', title, url)
|
|
feeds[-1][1].append({'title':title, 'url':url})
|
|
break
|
|
return feeds
|
|
|
|
def parse_index(self):
|
|
feeds = self.parse_section(self.base_url)
|
|
feeds += self.parse_section('http://www.theguardian.com/uk/sport', 'Sport - ')
|
|
return feeds
|