mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-06-23 15:30:45 -04:00
514 lines
20 KiB
Plaintext
514 lines
20 KiB
Plaintext
# adapted from old recipe by Darko Miletic <darko.miletic at gmail.com>
|
|
|
|
import re
|
|
from calibre.web.feeds.recipes import BasicNewsRecipe
|
|
from calibre.ebooks.BeautifulSoup import Tag, NavigableString
|
|
|
|
|
|
class TheIndependentNew(BasicNewsRecipe):
|
|
|
|
# flag to enable/disable article graphics on business pages/some others
|
|
# eg http://www.independent.co.uk/news/world/europe/berlusconi-departure-fails-to-calm-the-markets-6259682.html
|
|
# -max dimensions can be altered using the .pictureContainer img selector in the css
|
|
_FETCH_ARTICLE_GRAPHICS = True
|
|
|
|
#Flag to enable/disable image fetching (not business)
|
|
_FETCH_IMAGES = True
|
|
|
|
|
|
#used for converting rating to stars
|
|
_STAR_URL = 'http://www.independent.co.uk/skins/ind/images/rating_star.png'
|
|
_NO_STAR_URL = 'http://www.independent.co.uk/skins/ind/images/rating_star_grey.png'
|
|
|
|
|
|
title = u'The Independent'
|
|
__author__ = 'Will'
|
|
description = 'The latest in UK News and World News from The \
|
|
Independent. Wide range of international and local news, sports \
|
|
news, commentary and opinion pieces.Independent News - Breaking news \
|
|
that matters. Your daily comprehensive news source - The \
|
|
Independent Newspaper'
|
|
publisher = 'The Independent'
|
|
category = 'news, UK'
|
|
no_stylesheets = True
|
|
use_embedded_content = False
|
|
remove_empty_feeds = True
|
|
language = 'en_GB'
|
|
publication_type = 'newspaper'
|
|
masthead_url = 'http://www.independent.co.uk/independent.co.uk/editorial/logo/independent_Masthead.png'
|
|
encoding = 'utf-8'
|
|
remove_tags =[
|
|
dict(attrs={'id' : ['RelatedArtTag','renderBiography']}),
|
|
dict(attrs={'class' : ['autoplay','openBiogPopup']}),
|
|
dict(name='img',attrs={'alt' : ['Get Adobe Flash player']}),
|
|
dict(attrs={'style' : re.compile('.*')}),
|
|
]
|
|
|
|
keep_only_tags =[dict(attrs={'id':'main'})]
|
|
recursions = 0
|
|
|
|
# fixes non compliant html nesting and 'marks' article graphics links
|
|
preprocess_regexps = [
|
|
(re.compile('<span class="storyTop ">(?P<nested>.*?)</span>', re.DOTALL),
|
|
lambda match: '<div class="storyTop">' + match.group('nested') + '</div>'),
|
|
(re.compile('(<strong>.*?[Cc]lick.*?<a.*?((HERE)|([Hh]ere)).*?</strong>)', re.DOTALL),
|
|
lambda match: '<div class="article-graphic">' + match.group(0) + '</div>'),
|
|
]
|
|
|
|
|
|
conversion_options = {
|
|
'comment' : description
|
|
, 'tags' : category
|
|
, 'publisher' : publisher
|
|
, 'language' : language
|
|
}
|
|
|
|
extra_css = """
|
|
h1{font-family: Georgia,serif }
|
|
body{font-family: Verdana,Arial,Helvetica,sans-serif}
|
|
img{margin-bottom: 0.4em; display:block}
|
|
.starRating img {float: left}
|
|
.starRating {margin-top:0.4em; display: block}
|
|
.image {clear:left; font-size: x-small; color:#888888;}
|
|
.articleByTimeLocation {font-size: x-small; color:#888888;
|
|
margin-bottom:0.2em ; margin-top:0.2em ; display:block}
|
|
.subtitle {clear:left}
|
|
.column-1 h1 { color: #191919}
|
|
.column-1 h2 { color: #333333}
|
|
.column-1 h3 { color: #444444}
|
|
.column-1 p { color: #777777}
|
|
.column-1 p,a,h1,h2,h3 { margin: 0; }
|
|
.column-1 div{color:#888888; margin: 0;}
|
|
.articleContent {display: block; clear:left;}
|
|
.storyTop{}
|
|
.pictureContainer img { max-width: 400px; max-height: 400px;}
|
|
"""
|
|
|
|
oldest_article = 1
|
|
max_articles_per_feed = 100
|
|
|
|
_processed_urls = []
|
|
|
|
|
|
def get_article_url(self, article):
|
|
url = super(self.__class__,self).get_article_url(article)
|
|
|
|
title = article.get('title', None)
|
|
if title and re.search("^Video:",title):
|
|
return None
|
|
|
|
#remove duplicates
|
|
if not (url in self._processed_urls):
|
|
self._processed_urls.append(url)
|
|
else:
|
|
url = None
|
|
return url
|
|
|
|
def populate_article_metadata(self, article, soup, first):
|
|
if first and hasattr(self, 'add_toc_thumbnail'):
|
|
picdiv = soup.find('img')
|
|
if picdiv is not None:
|
|
self.add_toc_thumbnail(article,picdiv['src'])
|
|
|
|
def preprocess_html(self, soup):
|
|
|
|
#remove 'advertorial articles'
|
|
strapline = soup.find('div',attrs={'class' : re.compile('.*strapLine.*')})
|
|
if strapline:
|
|
for para in strapline.findAll('p'):
|
|
if len(para.contents) and isinstance(para.contents[0],NavigableString) \
|
|
and para.contents[0] == 'ADVERTORIAL FEATURE':
|
|
return None
|
|
|
|
items_to_extract = []
|
|
slideshow_elements = []
|
|
|
|
for item in soup.findAll(attrs={'class' : re.compile("widget.*")}):
|
|
remove = True
|
|
pattern = re.compile('((articleContent)|(title))$')
|
|
if (pattern.search(item['class'])) is not None:
|
|
remove = False
|
|
|
|
# corrections
|
|
# story content always good
|
|
pattern = re.compile('storyContent')
|
|
if (pattern.search(item['class'])) is not None:
|
|
remove = False
|
|
|
|
#images
|
|
pattern = re.compile('slideshow')
|
|
if (pattern.search(item['class'])) is not None:
|
|
if self._FETCH_IMAGES:
|
|
remove = False
|
|
slideshow_elements.append(item)
|
|
else:
|
|
remove = True
|
|
|
|
#social widgets always bad
|
|
pattern = re.compile('socialwidget')
|
|
if (pattern.search(item['class'])) is not None:
|
|
remove = True
|
|
|
|
if remove:
|
|
items_to_extract.append(item)
|
|
|
|
for item in items_to_extract:
|
|
item.extract()
|
|
|
|
items_to_extract = []
|
|
|
|
if self._FETCH_IMAGES:
|
|
for element in slideshow_elements:
|
|
for item in element.findAll('a',attrs={'href' : re.compile('.*')}):
|
|
if item.img is not None:
|
|
#use full size image
|
|
img = item.findNext('img')
|
|
|
|
img['src'] = item['href']
|
|
|
|
#insert caption if available
|
|
if img.get('title') and (len(img['title']) > 1):
|
|
tag = Tag(soup,'h3')
|
|
text = NavigableString(img['title'])
|
|
tag.insert(0,text)
|
|
|
|
#picture before text
|
|
img.extract()
|
|
item.insert(0,img)
|
|
item.insert(1,tag)
|
|
|
|
# remove link
|
|
item.name = "div"
|
|
item["class"]='image'
|
|
del item["href"]
|
|
|
|
|
|
#remove empty subtitles
|
|
"""
|
|
currently the subtitle is located in first paragraph after
|
|
sibling <h3 class="subtitle"> tag. This may be 'fixed' at
|
|
some point.
|
|
"""
|
|
subtitle = soup.find('h3',attrs={'class' : 'subtitle'})
|
|
if subtitle is not None:
|
|
subtitleText = subtitle.findNext('p')
|
|
if subtitleText is not None:
|
|
if len(subtitleText.contents[0]) <= 1 :
|
|
subtitleText.extract()
|
|
subtitle.extract()
|
|
|
|
|
|
#replace rating numbers with stars
|
|
for item in soup.findAll('div',attrs={ 'class' : 'starRating'}):
|
|
if item is not None:
|
|
soup2 = self._insertRatingStars(soup,item)
|
|
if soup2 is not None:
|
|
soup = soup2
|
|
|
|
|
|
#remove empty paragraph tags in storyTop which can leave a space
|
|
#between first paragraph and rest of story
|
|
nested_content = False
|
|
storyTop = soup.find('div',attrs={ 'class' : ['storyTop']})
|
|
for item in storyTop.findAll('p'):
|
|
for nested in item:
|
|
if isinstance(nested, Tag):
|
|
nested_content = True
|
|
break
|
|
if not nested_content and item.contents is not None and len(item.contents[0]) <= 1 :
|
|
items_to_extract.append(item)
|
|
|
|
for item in items_to_extract:
|
|
item.extract()
|
|
|
|
items_to_extract = []
|
|
|
|
|
|
#remove line breaks immediately next to tags with default margins
|
|
#to prevent double line spacing and narrow columns of text
|
|
storyTop = soup.find('div',attrs={ 'class' : ['storyTop']})
|
|
self._remove_undesired_line_breaks_from_tag(storyTop,soup)
|
|
|
|
|
|
#replace article graphics link with the graphics themselves
|
|
if self._FETCH_ARTICLE_GRAPHICS:
|
|
items_to_insert = []
|
|
for item in soup.findAll('div', attrs={'class' : ['article-graphic']}):
|
|
strong = item.find('strong')
|
|
if not strong:
|
|
continue
|
|
for child in strong:
|
|
if isinstance(child,Tag):
|
|
if str(child.name) == 'a':
|
|
items_to_insert.extend(self._get_article_graphic(strong,child['href'],soup))
|
|
|
|
for item in items_to_insert:
|
|
item[0].replaceWith(item[1])
|
|
|
|
for item in items_to_extract:
|
|
item.extract()
|
|
|
|
return soup
|
|
|
|
|
|
def _get_article_graphic(self,old_item,url,soup):
|
|
|
|
items_to_insert = []
|
|
|
|
if re.search('\.jpg$',str(url)):
|
|
div = Tag(soup,'div')
|
|
div['class'] = 'pictureContainer'
|
|
img = Tag(soup,'img')
|
|
img['src'] = url
|
|
img['alt'] = 'article graphic'
|
|
div.insert(0,img)
|
|
items_to_insert.append((old_item,div,))
|
|
return items_to_insert
|
|
|
|
soup2 = self.index_to_soup(url)
|
|
for item in soup2.findAll('div',attrs={'class' : re.compile("widget picture article.*")}):
|
|
items_to_insert.append((old_item,item),)
|
|
return items_to_insert
|
|
|
|
|
|
def _insertRatingStars(self,soup,item):
|
|
if item.contents is None or len(item.contents) < 1:
|
|
return
|
|
rating = item.contents[0]
|
|
|
|
try:
|
|
rating = float(item.contents[0])
|
|
except:
|
|
print 'Could not convert decimal rating to star: malformatted float.'
|
|
return
|
|
for i in range(1,6):
|
|
star = Tag(soup,'img')
|
|
if i <= rating:
|
|
star['src'] = self._STAR_URL
|
|
else:
|
|
star['src'] = self._NO_STAR_URL
|
|
star['alt'] = 'star number ' + str(i)
|
|
item.insert(i,star)
|
|
#item.contents[0] = NavigableString('(' + str(rating) + ')')
|
|
item.contents[0] = ''
|
|
|
|
def postprocess_html(self,soup, first_fetch):
|
|
#find broken images and remove captions
|
|
items_to_extract = []
|
|
for item in soup.findAll('div', attrs={'class' : 'image'}):
|
|
img = item.findNext('img')
|
|
if img and img.get('src'):
|
|
# broken images still point to remote url
|
|
pattern = re.compile('http://www.independent.co.uk.*')
|
|
if pattern.match(img["src"]) is not None:
|
|
caption = img.findNextSibling('h3')
|
|
if caption is not None:
|
|
items_to_extract.append(caption)
|
|
items_to_extract.append(img)
|
|
|
|
for item in items_to_extract:
|
|
item.extract()
|
|
return soup
|
|
|
|
def _recurisvely_linearise_tag_tree(
|
|
self,
|
|
item,
|
|
linearised= None,
|
|
count=0,
|
|
limit = 100
|
|
):
|
|
linearised = linearised or []
|
|
count = count + 1
|
|
if count > limit:
|
|
return linearised
|
|
if not (isinstance(item,Tag)):
|
|
return linearised
|
|
for nested in item:
|
|
linearised.append(nested)
|
|
linearised = self._recurisvely_linearise_tag_tree(nested,linearised, count)
|
|
return linearised
|
|
|
|
|
|
def _get_previous_tag(self,current_index, tag_tree):
|
|
if current_index == 0:
|
|
return None
|
|
else:
|
|
return tag_tree[current_index - 1]
|
|
|
|
|
|
def _get_next_tag(self,current_index, tag_tree):
|
|
if current_index < len(tag_tree) - 1:
|
|
return tag_tree[current_index + 1]
|
|
else:
|
|
return None
|
|
|
|
|
|
def _list_match(self,test_str, list_regex):
|
|
for regex in list_regex:
|
|
match = re.match(regex, test_str)
|
|
if match is not None:
|
|
return True
|
|
return False
|
|
|
|
def _remove_undesired_line_breaks_from_tag(self,parent,soup):
|
|
|
|
if parent is None:
|
|
return
|
|
|
|
|
|
tag_tree = self._recurisvely_linearise_tag_tree(parent)
|
|
items_to_remove = []
|
|
|
|
|
|
for item in tag_tree:
|
|
if item == u'\n':
|
|
items_to_remove.append(item)
|
|
continue;
|
|
|
|
for item in items_to_remove:
|
|
tag_tree.remove(item)
|
|
|
|
|
|
spaced_tags = [r'p', r'h\d', r'blockquote']
|
|
tags_to_extract = []
|
|
tags_to_replace = []
|
|
for (i, tag) in enumerate(tag_tree):
|
|
if isinstance(tag, Tag):
|
|
if str(tag) == '<br />':
|
|
previous_tag = self._get_previous_tag(i, tag_tree)
|
|
|
|
if isinstance(previous_tag, Tag):
|
|
previous_tag_is_spaced = previous_tag is not None\
|
|
and self._list_match(str(previous_tag.name),
|
|
spaced_tags)
|
|
else:
|
|
previous_tag_is_spaced = False
|
|
|
|
next_tag = self._get_next_tag(i, tag_tree)
|
|
|
|
if isinstance(next_tag, Tag):
|
|
next_tag_is_spaced = next_tag is not None\
|
|
and self._list_match(str(next_tag.name), spaced_tags)
|
|
else:
|
|
next_tag_is_spaced = False
|
|
|
|
if previous_tag_is_spaced or next_tag_is_spaced or i == 0\
|
|
or i == len(tag_tree) - 1:
|
|
tags_to_extract.append(tag)
|
|
else:
|
|
tags_to_replace.append((tag,NavigableString(' '),))
|
|
|
|
|
|
for pair in tags_to_replace:
|
|
pair[0].replaceWith(pair[1])
|
|
for tag in tags_to_extract:
|
|
tag.extract()
|
|
|
|
feeds = [
|
|
(u'News - UK',
|
|
u'http://www.independent.co.uk/news/uk/?service=rss'),
|
|
(u'News - World',
|
|
u'http://www.independent.co.uk/news/world/?service=rss'),
|
|
(u'News - Business',
|
|
u'http://www.independent.co.uk/news/business/?service=rss'),
|
|
(u'News - People',
|
|
u'http://www.independent.co.uk/news/people/?service=rss'),
|
|
(u'News - Science',
|
|
u'http://www.independent.co.uk/news/science/?service=rss'),
|
|
(u'News - Media',
|
|
u'http://www.independent.co.uk/news/media/?service=rss'),
|
|
(u'News - Education',
|
|
u'http://www.independent.co.uk/news/education/?service=rss'),
|
|
(u'News - Obituaries',
|
|
u'http://www.independent.co.uk/news/obituaries/?service=rss'),
|
|
(u'News - Corrections',
|
|
u'http://www.independent.co.uk/news/corrections/?service=rss'
|
|
),
|
|
(u'Opinion',
|
|
u'http://www.independent.co.uk/opinion/?service=rss'),
|
|
(u'Environment',
|
|
u'http://www.independent.co.uk/environment/?service=rss'),
|
|
(u'Sport - Athletics',
|
|
u'http://www.independent.co.uk/sport/general/athletics/?service=rss'
|
|
),
|
|
(u'Sport - Cricket',
|
|
u'http://www.independent.co.uk/sport/cricket/?service=rss'),
|
|
(u'Sport - Football',
|
|
u'http://www.independent.co.uk/sport/football/?service=rss'),
|
|
(u'Sport - Golf',
|
|
u'http://www.independent.co.uk/sport/golf/?service=rss'),
|
|
(u'Sport - Motor racing',
|
|
u'http://www.independent.co.uk/sport/motor-racing/?service=rss'
|
|
),
|
|
(u'Sport - Olympics',
|
|
u'http://www.independent.co.uk/sport/olympics/?service=rss'),
|
|
(u'Sport - Racing',
|
|
u'http://www.independent.co.uk/sport/racing/?service=rss'),
|
|
(u'Sport - Rugby League',
|
|
u'http://www.independent.co.uk/sport/general/rugby-league/?service=rss'),
|
|
(u'Sport - Rugby Union',
|
|
u'http://www.independent.co.uk/sport/rugby/rugby-union/?service=rss'
|
|
),
|
|
(u'Sport - Sailing',
|
|
u'http://www.independent.co.uk/sport/general/sailing/?service=rss'
|
|
),
|
|
(u'Sport - Tennis',
|
|
u'http://www.independent.co.uk/sport/tennis/?service=rss'),
|
|
(u'Sport - Others',
|
|
u'http://www.independent.co.uk/sport/general/others/?service=rss'
|
|
),
|
|
(u'Life & Style - Fashion',
|
|
u'http://www.independent.co.uk/life-style/fashion/?service=rss'
|
|
),
|
|
(u'Life & Style -Food & Drink',
|
|
u'http://www.independent.co.uk/life-style/food-and-drink/?service=rss'
|
|
),
|
|
(u'Life & Style - Health and Families',
|
|
u'http://www.independent.co.uk/life-style/health-and-families/?service=rss'
|
|
),
|
|
(u'Life & Style - House & Home',
|
|
u'http://www.independent.co.uk/life-style/house-and-home/'),
|
|
(u'Life & Style - History',
|
|
u'http://www.independent.co.uk/life-style/history/?service=rss'
|
|
),
|
|
(u'Life & Style - Gadgets & Tech',
|
|
u'http://www.independent.co.uk/life-style/gadgets-and-tech/?service=rss'
|
|
),
|
|
(u'Life & Style - Motoring',
|
|
u'http://www.independent.co.uk/life-style/motoring/?service=rss'
|
|
),
|
|
(u'Arts & Ents - Art',
|
|
u'http://www.independent.co.uk/arts-entertainment/art/?service=rss'
|
|
),
|
|
(u'Arts & Ents - Architecture',
|
|
u'http://www.independent.co.uk/arts-entertainment/architecture/?service=rss'
|
|
),
|
|
(u'Arts & Ents - Music',
|
|
u'http://www.independent.co.uk/arts-entertainment/music/?service=rss'
|
|
),
|
|
(u'Arts & Ents - Classical',
|
|
u'http://www.independent.co.uk/arts-entertainment/classical/?service=rss'
|
|
),
|
|
(u'Arts & Ents - Films',
|
|
u'http://www.independent.co.uk/arts-entertainment/films/?service=rss'
|
|
),
|
|
(u'Arts & Ents - TV',
|
|
u'http://www.independent.co.uk/arts-entertainment/tv/?service=rss'
|
|
),
|
|
(u'Arts & Ents - Theatre and Dance',
|
|
u'http://www.independent.co.uk/arts-entertainment/theatre-dance/?service=rss'
|
|
),
|
|
(u'Arts & Ents - Comedy',
|
|
u'http://www.independent.co.uk/arts-entertainment/comedy/?service=rss'
|
|
),
|
|
(u'Arts & Ents - Books',
|
|
u'http://www.independent.co.uk/arts-entertainment/books/?service=rss'
|
|
),
|
|
(u'Travel', u'http://www.independent.co.uk/travel/?service=rss'
|
|
),
|
|
(u'Money', u'http://www.independent.co.uk/money/?service=rss'),
|
|
(u'IndyBest',
|
|
u'http://www.independent.co.uk/extras/indybest/?service=rss'),
|
|
]
|
|
|