mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-07-09 03:04:10 -04:00
Update grantland.com
This commit is contained in:
Kovid Goyal
parent
a34407cc60
commit
f998c8bf4b
@ -2,105 +2,75 @@ import re
|
||||
from calibre.web.feeds.news import BasicNewsRecipe
|
||||
|
||||
class GrantLand(BasicNewsRecipe):
|
||||
title = u"Grantland"
|
||||
description = 'Writings on Sports & Pop Culture'
|
||||
language = 'en'
|
||||
__author__ = 'barty on mobileread.com forum'
|
||||
max_articles_per_feed = 100
|
||||
no_stylesheets = False
|
||||
# auto_cleanup is too aggressive sometimes and we end up with blank articles
|
||||
auto_cleanup = False
|
||||
timefmt = ' [%a, %d %b %Y]'
|
||||
oldest_article = 365
|
||||
title = u"Grantland"
|
||||
description = 'Writings on Sports & Pop Culture'
|
||||
language = 'en'
|
||||
__author__ = 'barty on mobileread.com forum'
|
||||
max_articles_per_feed = 100
|
||||
no_stylesheets = True
|
||||
# auto_cleanup is too aggressive sometimes and we end up with blank articles
|
||||
auto_cleanup = False
|
||||
timefmt = ' [%a, %d %b %Y]'
|
||||
oldest_article = 90
|
||||
|
||||
cover_url = 'http://cdn0.sbnation.com/imported_assets/740965/blog_grantland_grid_3.jpg'
|
||||
masthead_url = 'http://a1.espncdn.com/prod/assets/grantland/grantland-logo.jpg'
|
||||
cover_url = 'http://cdn0.sbnation.com/imported_assets/740965/blog_grantland_grid_3.jpg'
|
||||
masthead_url = 'http://a1.espncdn.com/prod/assets/grantland/grantland-logo.jpg'
|
||||
|
||||
INDEX = 'http://www.grantland.com'
|
||||
CATEGORIES = [
|
||||
# comment out categories you don't want
|
||||
# (user friendly name, url suffix, max number of articles to load)
|
||||
('Today in Grantland','',20),
|
||||
('In Case You Missed It','incaseyoumissedit',35),
|
||||
]
|
||||
INDEX = 'http://www.grantland.com'
|
||||
CATEGORIES = [
|
||||
# comment out second line if you don't want older articles
|
||||
# (user friendly name, url suffix, max number of articles to load)
|
||||
('Today in Grantland','',20),
|
||||
('In Case You Missed It','incaseyoumissedit',35),
|
||||
]
|
||||
|
||||
remove_tags = [
|
||||
{'name':['head','style','script']},
|
||||
{'id':['header']},
|
||||
{'class':re.compile(r'\bside|\bad\b|floatright|tags')}
|
||||
]
|
||||
remove_tags_before = {'class':'wrapper'}
|
||||
remove_tags_after = [{'id':'content'}]
|
||||
remove_tags = [
|
||||
{'name':['style','aside','nav','footer','script']},
|
||||
{'name':'h1','text':'Grantland'},
|
||||
{'id':['header','col-right']},
|
||||
{'class':['connect_widget']},
|
||||
{'name':'section','class':re.compile(r'\b(ad|module)\b')},
|
||||
]
|
||||
|
||||
preprocess_regexps = [
|
||||
# <header> tags with an img inside are just blog banners, don't need them
|
||||
# note: there are other useful <header> tags so we don't want to just strip all of them
|
||||
(re.compile(r'<header class.+?<img .+?>.+?</header>', re.DOTALL|re.IGNORECASE),lambda m: ''),
|
||||
# delete everything between the *last* <hr class="small" /> and </article>
|
||||
(re.compile(r'<hr class="small"(?:(?!<hr class="small").)+</article>', re.DOTALL|re.IGNORECASE),lambda m: '<hr class="small" /></article>'),
|
||||
]
|
||||
extra_css = """cite, time { font-size: 0.8em !important; margin-right: 1em !important; }
|
||||
img + cite { display:block; text-align:right}"""
|
||||
preprocess_regexps = [
|
||||
# remove blog banners
|
||||
(re.compile(r'<a href="/blog/(?:(?!</a>).)+</a>', re.DOTALL|re.IGNORECASE), lambda m: ''),
|
||||
]
|
||||
|
||||
def parse_index(self):
|
||||
feeds = []
|
||||
seen_urls = set([])
|
||||
def parse_index(self):
|
||||
feeds = []
|
||||
seen_urls = set([])
|
||||
|
||||
for category in self.CATEGORIES:
|
||||
for category in self.CATEGORIES:
|
||||
|
||||
(cat_name, tag, max_articles) = category
|
||||
self.log('Reading category:', cat_name)
|
||||
articles = []
|
||||
(cat_name, tag, max_articles) = category
|
||||
self.log('Reading category:', cat_name)
|
||||
articles = []
|
||||
|
||||
page = "%s/%s" % (self.INDEX, tag)
|
||||
soup = self.index_to_soup(page)
|
||||
headers = soup.findAll('h2' if tag=='' else 'h3')
|
||||
page = "%s/%s" % (self.INDEX, tag)
|
||||
soup = self.index_to_soup(page)
|
||||
|
||||
for header in headers:
|
||||
tag = header.find('a',href=True)
|
||||
if tag is None:
|
||||
continue
|
||||
url = tag['href']
|
||||
if url in seen_urls:
|
||||
continue
|
||||
title = self.tag_to_string(tag)
|
||||
if 'Podcast:' in title or 'In Case You Missed It' in title:
|
||||
continue
|
||||
desc = dt = ''
|
||||
# get at the div that contains description and other info
|
||||
div = header.parent.find('div')
|
||||
if div is not None:
|
||||
desc = self.tag_to_string(div)
|
||||
dt = div.find('time')
|
||||
if dt is not None:
|
||||
dt = self.tag_to_string( dt)
|
||||
main = soup.find('div',id='col-main')
|
||||
if main is None:
|
||||
main = soup
|
||||
|
||||
# if div contains the same url that is in h2/h3
|
||||
# that means this is a series split into multiple articles
|
||||
if div.find('a',href=url):
|
||||
self.log('\tFound series:', title)
|
||||
# grab all articles in series
|
||||
for tag in div.findAll('a',href=True):
|
||||
url = tag['href']
|
||||
if url in seen_urls:
|
||||
continue
|
||||
self.log('\t', url)
|
||||
seen_urls.add(url)
|
||||
articles.append({'title':title+' - '+self.tag_to_string( tag),
|
||||
'url':url,'description':desc,'date':dt})
|
||||
else:
|
||||
self.log('\tFound article:', title)
|
||||
self.log('\t', url)
|
||||
seen_urls.add(url)
|
||||
articles.append({'title':title,'url':url,'description':desc,'date':dt})
|
||||
for tag in main.findAll('a', href=re.compile(r'(story|post)/_/id/\d+')):
|
||||
url = tag['href']
|
||||
if url in seen_urls:
|
||||
continue
|
||||
title = tag.string
|
||||
# blank title probably means <a href=".."><img /></a>. skip
|
||||
if not title:
|
||||
continue
|
||||
self.log('\tFound article:', title)
|
||||
self.log('\t', url)
|
||||
articles.append({'title':title,'url':url})
|
||||
seen_urls.add(url)
|
||||
|
||||
if len(articles) >= max_articles:
|
||||
break
|
||||
if len(articles) >= max_articles:
|
||||
break
|
||||
|
||||
if articles:
|
||||
feeds.append((cat_name, articles))
|
||||
if articles:
|
||||
feeds.append((cat_name, articles))
|
||||
|
||||
return feeds
|
||||
|
||||
def print_version(self, url):
|
||||
return url+'?view=print'
|
||||
return feeds
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user