mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-07-08 18:54:09 -04:00
New recipe for India Today and improved recipe for The Independent
This commit is contained in:
Kovid Goyal
parent
7333cc7319
commit
4b66da2fe6
@ -1,26 +1,55 @@
|
|||||||
from calibre.web.feeds.news import BasicNewsRecipe
|
from calibre.web.feeds.news import BasicNewsRecipe
|
||||||
|
from calibre.ebooks.BeautifulSoup import BeautifulSoup
|
||||||
|
|
||||||
class Independent(BasicNewsRecipe):
|
class TheIndependent(BasicNewsRecipe):
|
||||||
title = u'The Independent'
|
title = u'The Independent'
|
||||||
oldest_article = 1
|
language = 'en_UK'
|
||||||
language = 'en_GB'
|
__author__ = 'Krittika Goyal'
|
||||||
__author__ = 'Jimmy Patrick'
|
oldest_article = 1 #days
|
||||||
max_articles_per_feed = 100
|
max_articles_per_feed = 25
|
||||||
|
encoding = 'latin1'
|
||||||
|
|
||||||
feeds = [(u'UK', u'http://www.independent.co.uk/news/uk/rss'),
|
remove_stylesheets = True
|
||||||
(u'World', u'http://www.independent.co.uk/news/world/rss'),
|
#remove_tags_before = dict(name='h1', attrs={'class':'heading'})
|
||||||
(u'Sport', u'http://www.independent.co.uk/sport/rss'),
|
#remove_tags_after = dict(name='td', attrs={'class':'newptool1'})
|
||||||
(u'Arts & Entertainment', u'http://www.independent.co.uk/arts-entertainment/rss'),
|
remove_tags = [
|
||||||
(u'Life & Style',u'http://www.independent.co.uk/life-style/fashion/news/rss'),
|
dict(name='iframe'),
|
||||||
(u'Business',u'http://www.independent.co.uk/news/business/rss'),
|
dict(name='div', attrs={'class':'related-articles'}),
|
||||||
(u'Science',u'http://www.independent.co.uk/news/science/rss'),
|
dict(name='div', attrs={'id':['qrformdiv', 'inSection', 'alpha-inner']}),
|
||||||
(u'Media',u'http://www.independent.co.uk/news/media/rss')
|
dict(name='ul', attrs={'class':'article-tools'}),
|
||||||
]
|
dict(name='ul', attrs={'class':'articleTools'}),
|
||||||
|
]
|
||||||
|
|
||||||
keep_only_tags = [dict(id=['article'])]
|
feeds = [
|
||||||
remove_tags = [dict(name='div', attrs={'class':'share-links'}),
|
('UK',
|
||||||
dict(name='ul', attrs={'class':'article-tools'}),
|
'http://www.independent.co.uk/news/uk/rss'),
|
||||||
dict(name='div', attrs={'class':'related-articles'})
|
('World',
|
||||||
]
|
'http://www.independent.co.uk/news/world/rss'),
|
||||||
|
('Sport',
|
||||||
|
'http://www.independent.co.uk/sport/rss'),
|
||||||
|
('Arts and Entertainment',
|
||||||
|
'http://www.independent.co.uk/arts-entertainment/rss'),
|
||||||
|
('Business',
|
||||||
|
'http://www.independent.co.uk/news/business/rss'),
|
||||||
|
('Life and Style',
|
||||||
|
'http://www.independent.co.uk/life-style/gadgets-and-tech/news/rss'),
|
||||||
|
('Science',
|
||||||
|
'http://www.independent.co.uk/news/science/rss'),
|
||||||
|
('People',
|
||||||
|
'http://www.independent.co.uk/news/people/rss'),
|
||||||
|
('Media',
|
||||||
|
'http://www.independent.co.uk/news/media/rss'),
|
||||||
|
('Health and Families',
|
||||||
|
'http://www.independent.co.uk/life-style/health-and-families/rss'),
|
||||||
|
('Obituaries',
|
||||||
|
'http://www.independent.co.uk/news/obituaries/rss'),
|
||||||
|
]
|
||||||
|
|
||||||
extra_css = "body{color:black;}"
|
def preprocess_html(self, soup):
|
||||||
|
story = soup.find(name='div', attrs={'id':'mainColumn'})
|
||||||
|
#td = heading.findParent(name='td')
|
||||||
|
#td.extract()
|
||||||
|
soup = BeautifulSoup('<html><head><title>t</title></head><body></body></html>')
|
||||||
|
body = soup.find(name='body')
|
||||||
|
body.insert(0, story)
|
||||||
|
return soup
|
||||||
|
|||||||
76
resources/recipes/india_today.recipe
Normal file
76
resources/recipes/india_today.recipe
Normal file
@ -0,0 +1,76 @@
|
|||||||
|
from calibre.web.feeds.news import BasicNewsRecipe
|
||||||
|
|
||||||
|
class IndiaToday(BasicNewsRecipe):
|
||||||
|
|
||||||
|
title = 'India Today'
|
||||||
|
__author__ = 'Kovid Goyal'
|
||||||
|
language = 'en_IN'
|
||||||
|
timefmt = ' [%d %m, %Y]'
|
||||||
|
|
||||||
|
oldest_article = 700
|
||||||
|
max_articles_per_feed = 10
|
||||||
|
|
||||||
|
no_stylesheets = True
|
||||||
|
|
||||||
|
remove_tags_before = dict(id='content_story_title')
|
||||||
|
remove_tags_after = dict(id='rightblockdiv')
|
||||||
|
remove_tags = [dict(id=['rightblockdiv', 'share_links'])]
|
||||||
|
|
||||||
|
extra_css = '#content_story_title { font-size: 170%; font-weight: bold;}'
|
||||||
|
conversion_options = { 'linearize_tables': True }
|
||||||
|
|
||||||
|
def it_get_index(self):
|
||||||
|
soup = self.index_to_soup('http://indiatoday.intoday.in/site/archive')
|
||||||
|
a = soup.find('a', href=lambda x: x and 'issueId=' in x)
|
||||||
|
url = 'http://indiatoday.intoday.in/site/'+a.get('href')
|
||||||
|
img = a.find('img')
|
||||||
|
self.cover_url = img.get('src')
|
||||||
|
return self.index_to_soup(url)
|
||||||
|
|
||||||
|
def parse_index(self):
|
||||||
|
soup = self.it_get_index()
|
||||||
|
feeds, current_section, current_articles = [], None, []
|
||||||
|
for x in soup.findAll(name=['h1', 'a']):
|
||||||
|
if x.name == 'h1':
|
||||||
|
if current_section and current_articles:
|
||||||
|
feeds.append((current_section, current_articles))
|
||||||
|
current_section = self.tag_to_string(x)
|
||||||
|
current_articles = []
|
||||||
|
self.log('\tFound section:', current_section)
|
||||||
|
elif x.name == 'a' and 'Story' in x.get('href', ''):
|
||||||
|
title = self.tag_to_string(x)
|
||||||
|
url = x.get('href')
|
||||||
|
url = url.replace(' ', '%20')
|
||||||
|
if not url.startswith('/'):
|
||||||
|
url = 'http://indiatoday.intoday.in/site/' + url
|
||||||
|
if title and url:
|
||||||
|
url += '?complete=1'
|
||||||
|
self.log('\tFound article:', title)
|
||||||
|
self.log('\t\t', url)
|
||||||
|
desc = ''
|
||||||
|
h3 = x.parent.findNextSibling('h3')
|
||||||
|
if h3 is not None:
|
||||||
|
desc = 'By ' + self.tag_to_string(h3)
|
||||||
|
h4 = h3.findNextSibling('h4')
|
||||||
|
if h4 is not None:
|
||||||
|
desc = self.tag_to_string(h4) + ' ' + desc
|
||||||
|
if desc:
|
||||||
|
self.log('\t\t', desc)
|
||||||
|
current_articles.append({'title':title, 'description':desc,
|
||||||
|
'url':url, 'date':''})
|
||||||
|
|
||||||
|
if current_section and current_articles:
|
||||||
|
feeds.append((current_section, current_articles))
|
||||||
|
|
||||||
|
return feeds
|
||||||
|
|
||||||
|
def postprocess_html(self, soup, first):
|
||||||
|
a = soup.find(text='Print')
|
||||||
|
if a is not None:
|
||||||
|
tr = a.findParent('tr')
|
||||||
|
if tr is not None:
|
||||||
|
tr.extract()
|
||||||
|
return soup
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@ -819,10 +819,14 @@ class BasicNewsRecipe(Recipe):
|
|||||||
if '?' in ext:
|
if '?' in ext:
|
||||||
ext = ''
|
ext = ''
|
||||||
ext = ext.lower() if ext else 'jpg'
|
ext = ext.lower() if ext else 'jpg'
|
||||||
self.report_progress(1, _('Downloading cover from %s')%cu)
|
|
||||||
cpath = os.path.join(self.output_dir, 'cover.'+ext)
|
cpath = os.path.join(self.output_dir, 'cover.'+ext)
|
||||||
with nested(open(cpath, 'wb'), closing(self.browser.open(cu))) as (cfile, r):
|
if os.access(cu, os.R_OK):
|
||||||
cfile.write(r.read())
|
with open(cpath, 'wb') as cfile:
|
||||||
|
cfile.write(open(cu, 'rb').read())
|
||||||
|
else:
|
||||||
|
self.report_progress(1, _('Downloading cover from %s')%cu)
|
||||||
|
with nested(open(cpath, 'wb'), closing(self.browser.open(cu))) as (cfile, r):
|
||||||
|
cfile.write(r.read())
|
||||||
if ext.lower() == 'pdf':
|
if ext.lower() == 'pdf':
|
||||||
from calibre.ebook.metadata.pdf import get_metadata
|
from calibre.ebook.metadata.pdf import get_metadata
|
||||||
stream = open(cpath, 'rb')
|
stream = open(cpath, 'rb')
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user