Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-31 14:33:54 -04:00
Code Issues Packages Projects Releases Wiki Activity

Update grantland.com

Browse Source
This commit is contained in:
Kovid Goyal 2012-01-22 10:08:26 +05:30
parent a34407cc60
commit f998c8bf4b
1 changed files with 59 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

148
recipes/grantland.recipe
View File

@ -2,105 +2,75 @@ import re
from calibre.web.feeds.news import BasicNewsRecipe from calibre.web.feeds.news import BasicNewsRecipe
class GrantLand(BasicNewsRecipe): class GrantLand(BasicNewsRecipe):
title = u"Grantland" title = u"Grantland"
description = 'Writings on Sports & Pop Culture' description = 'Writings on Sports & Pop Culture'
language = 'en' language = 'en'
__author__ = 'barty on mobileread.com forum' __author__ = 'barty on mobileread.com forum'
max_articles_per_feed = 100 max_articles_per_feed = 100
no_stylesheets = False no_stylesheets = True
# auto_cleanup is too aggressive sometimes and we end up with blank articles # auto_cleanup is too aggressive sometimes and we end up with blank articles
auto_cleanup = False auto_cleanup = False
timefmt = ' [%a, %d %b %Y]' timefmt = ' [%a, %d %b %Y]'
oldest_article = 365 oldest_article = 90
cover_url = 'http://cdn0.sbnation.com/imported_assets/740965/blog_grantland_grid_3.jpg' cover_url = 'http://cdn0.sbnation.com/imported_assets/740965/blog_grantland_grid_3.jpg'
masthead_url = 'http://a1.espncdn.com/prod/assets/grantland/grantland-logo.jpg' masthead_url = 'http://a1.espncdn.com/prod/assets/grantland/grantland-logo.jpg'
INDEX = 'http://www.grantland.com' INDEX = 'http://www.grantland.com'
CATEGORIES = [ CATEGORIES = [
# comment out categories you don't want # comment out second line if you don't want older articles
# (user friendly name, url suffix, max number of articles to load) # (user friendly name, url suffix, max number of articles to load)
('Today in Grantland','',20), ('Today in Grantland','',20),
('In Case You Missed It','incaseyoumissedit',35), ('In Case You Missed It','incaseyoumissedit',35),
] ]
remove_tags = [ remove_tags = [
{'name':['head','style','script']}, {'name':['style','aside','nav','footer','script']},
{'id':['header']}, {'name':'h1','text':'Grantland'},
{'class':re.compile(r'\bside|\bad\b|floatright|tags')} {'id':['header','col-right']},
] {'class':['connect_widget']},
remove_tags_before = {'class':'wrapper'} {'name':'section','class':re.compile(r'\b(ad|module)\b')},
remove_tags_after = [{'id':'content'}] ]
preprocess_regexps = [ preprocess_regexps = [
# <header> tags with an img inside are just blog banners, don't need them # remove blog banners
# note: there are other useful <header> tags so we don't want to just strip all of them (re.compile(r'<a href="/blog/(?:(?!</a>).)+</a>', re.DOTALL|re.IGNORECASE), lambda m: ''),
(re.compile(r'<header class.+?<img .+?>.+?</header>', re.DOTALL|re.IGNORECASE),lambda m: ''), ]
# delete everything between the *last* <hr class="small" /> and </article>
(re.compile(r'<hr class="small"(?:(?!<hr class="small").)+</article>', re.DOTALL|re.IGNORECASE),lambda m: '<hr class="small" /></article>'),
]
extra_css = """cite, time { font-size: 0.8em !important; margin-right: 1em !important; }
img + cite { display:block; text-align:right}"""
def parse_index(self): def parse_index(self):
feeds = [] feeds = []
seen_urls = set([]) seen_urls = set([])
for category in self.CATEGORIES: for category in self.CATEGORIES:
(cat_name, tag, max_articles) = category (cat_name, tag, max_articles) = category
self.log('Reading category:', cat_name) self.log('Reading category:', cat_name)
articles = [] articles = []
page = "%s/%s" % (self.INDEX, tag) page = "%s/%s" % (self.INDEX, tag)
soup = self.index_to_soup(page) soup = self.index_to_soup(page)
headers = soup.findAll('h2' if tag=='' else 'h3')
for header in headers: main = soup.find('div',id='col-main')
tag = header.find('a',href=True) if main is None:
if tag is None: main = soup
continue
url = tag['href']
if url in seen_urls:
continue
title = self.tag_to_string(tag)
if 'Podcast:' in title or 'In Case You Missed It' in title:
continue
desc = dt = ''
# get at the div that contains description and other info
div = header.parent.find('div')
if div is not None:
desc = self.tag_to_string(div)
dt = div.find('time')
if dt is not None:
dt = self.tag_to_string( dt)
# if div contains the same url that is in h2/h3 for tag in main.findAll('a', href=re.compile(r'(story|post)/_/id/\d+')):
# that means this is a series split into multiple articles url = tag['href']
if div.find('a',href=url): if url in seen_urls:
self.log('\tFound series:', title) continue
# grab all articles in series title = tag.string
for tag in div.findAll('a',href=True): # blank title probably means <a href=".."><img /></a>. skip
url = tag['href'] if not title:
if url in seen_urls: continue
continue self.log('\tFound article:', title)
self.log('\t', url) self.log('\t', url)
seen_urls.add(url) articles.append({'title':title,'url':url})
articles.append({'title':title+' - '+self.tag_to_string( tag), seen_urls.add(url)
'url':url,'description':desc,'date':dt})
else:
self.log('\tFound article:', title)
self.log('\t', url)
seen_urls.add(url)
articles.append({'title':title,'url':url,'description':desc,'date':dt})
if len(articles) >= max_articles: if len(articles) >= max_articles:
break break
if articles: if articles:
feeds.append((cat_name, articles)) feeds.append((cat_name, articles))
return feeds return feeds
def print_version(self, url):
return url+'?view=print'