mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-07-09 03:04:10 -04:00
Merge branch 'master' of https://github.com/unkn0w7n/calibre
This commit is contained in:
Kovid Goyal
commit
086add99d0
@ -134,27 +134,27 @@ class Bloomberg(BasicNewsRecipe):
|
||||
cat = subhead = lede = auth = caption = ''
|
||||
|
||||
if 'primaryCategory' in data and data['primaryCategory'] is not None:
|
||||
cat = '<p id="cat">' + data['primaryCategory'] + '</p>'
|
||||
cat = '<p class="cat">' + data['primaryCategory'] + '</p>'
|
||||
|
||||
if len(data['abstract']) != 0 and len(data['abstract']) == 2:
|
||||
subhead = '<div id="subhead"><p>' + data['abstract'][0] + '</p><p>' + data['abstract'][1] + '</p></div>'
|
||||
subhead = '<div class="subhead"><p>' + data['abstract'][0] + '</p><p>' + data['abstract'][1] + '</p></div>'
|
||||
else:
|
||||
if 'summary' in data:
|
||||
subhead = '<div id="subhead"><p>' + data['summary'] + '</p></div>'
|
||||
subhead = '<div class="subhead"><p>' + data['summary'] + '</p></div>'
|
||||
|
||||
if 'byline' in data and data['byline'] is not None:
|
||||
auth = '<div><span id="auth">' + data['byline']\
|
||||
+ '</span> | <span id="time">' + data['publishedAt'][:-14] + '</span></div>'
|
||||
auth = '<div><span class="auth">' + data['byline']\
|
||||
+ '</span> | <span class="time">' + data['publishedAt'][:-14] + '</span></div>'
|
||||
|
||||
if 'ledeImageUrl' in data and data['ledeImageUrl'] is not None:
|
||||
lede = '<p id="img"><img src="{}">'.format(data['ledeImageUrl'])
|
||||
|
||||
if 'ledeDescription' in data and data['ledeDescription'] is not None:
|
||||
caption = '<span id="cap">' + data['ledeDescription'] + '</span>'
|
||||
caption = '<span class="cap">' + data['ledeDescription'] + '</span>'
|
||||
else:
|
||||
if 'lede' in data and data['lede'] is not None:
|
||||
if 'alt' in data['lede'] and data['lede']['alt'] is not None:
|
||||
caption = '<span id="cap">' + data['lede']['alt'] + '</span>'
|
||||
caption = '<span class="cap">' + data['lede']['alt'] + '</span>'
|
||||
|
||||
if m:
|
||||
time.sleep(3)
|
||||
|
||||
@ -1,58 +1,107 @@
|
||||
__license__ = 'GPL v3'
|
||||
__copyright__ = '2009-2012, Darko Miletic <darko.miletic at gmail.com>'
|
||||
'''
|
||||
www.business-standard.com
|
||||
'''
|
||||
|
||||
from calibre.web.feeds.recipes import BasicNewsRecipe
|
||||
|
||||
|
||||
def classes(classes):
|
||||
q = frozenset(classes.split(' '))
|
||||
return dict(attrs={
|
||||
'class': lambda x: x and frozenset(x.split()).intersection(q)})
|
||||
|
||||
from calibre.web.feeds.news import BasicNewsRecipe
|
||||
from calibre.ptempfile import PersistentTemporaryFile
|
||||
from html5_parser import parse
|
||||
from datetime import datetime
|
||||
import json
|
||||
|
||||
class BusinessStandard(BasicNewsRecipe):
|
||||
title = 'Business Standard'
|
||||
__author__ = 'Darko Miletic'
|
||||
__author__ = 'unkn0wn'
|
||||
description = "India's most respected business daily"
|
||||
oldest_article = 1
|
||||
max_articles_per_feed = 20
|
||||
no_stylesheets = True
|
||||
use_embedded_content = False
|
||||
encoding = 'utf-8'
|
||||
publisher = 'Business Standard Limited'
|
||||
category = 'news, business, money, india, world'
|
||||
language = 'en_IN'
|
||||
|
||||
masthead_url = 'https://bsmedia.business-standard.com/include/_mod/site/html5/images/business-standard-logo.png'
|
||||
|
||||
no_stylesheets = True
|
||||
remove_javascript = True
|
||||
remove_attributes = ['width', 'height', 'style']
|
||||
|
||||
def get_cover_url(self):
|
||||
soup = self.index_to_soup('https://www.magzter.com/IN/Business-Standard-Private-Ltd/Business-Standard/Newspaper/')
|
||||
for citem in soup.findAll('meta', content=lambda s: s and s.endswith('view/3.jpg')):
|
||||
return citem['content']
|
||||
|
||||
remove_attributes = ['width', 'height', 'style']
|
||||
def get_browser(self):
|
||||
return BasicNewsRecipe.get_browser(self, user_agent='common_words/based')
|
||||
|
||||
keep_only_tags = [
|
||||
classes('headline alternativeHeadline full-img article-content__img pubDate'),
|
||||
dict(name='span', attrs={'class':'p-content'}),
|
||||
]
|
||||
remove_tags = [
|
||||
classes('also-read-panel')
|
||||
ignore_duplicate_articles = {'title', 'url'}
|
||||
remove_empty_feeds = True
|
||||
resolve_internal_links = True
|
||||
simultaneous_downloads = 1
|
||||
|
||||
extra_css = '''
|
||||
.auth, .cat { font-size:small; color:#202020; }
|
||||
.cap { font-size:small; text-align:center; }
|
||||
'''
|
||||
|
||||
art_url = ''
|
||||
art_desc = ''
|
||||
|
||||
articles_are_obfuscated = True
|
||||
|
||||
def get_obfuscated_article(self, url):
|
||||
br = self.get_browser()
|
||||
soup = self.index_to_soup(url)
|
||||
link = soup.find('a', attrs={'href':lambda x: x and x.startswith('https://www.business-standard.com')})
|
||||
skip_sections =[ # add sections you want to skip
|
||||
'/video/', '/videos/', '/multimedia/',
|
||||
]
|
||||
if any(x in link['href'] for x in skip_sections):
|
||||
self.abort_article('skipping video links ', link['href'])
|
||||
self.art_url = link['href']
|
||||
self.log('Found ', link['href'])
|
||||
html = br.open(link['href']).read()
|
||||
pt = PersistentTemporaryFile('.html')
|
||||
pt.write(html)
|
||||
pt.close()
|
||||
return pt.name
|
||||
|
||||
feeds = []
|
||||
|
||||
sections = [
|
||||
'india-news', 'economy', 'opinion', 'markets', 'companies', 'industry', 'finance', 'world-news',
|
||||
# 'politics', 'cricket', 'sports', 'technology', 'book', 'education', 'specials'
|
||||
]
|
||||
|
||||
feeds = [
|
||||
(u'Companies', u'https://www.business-standard.com/rss/companies-101.rss'),
|
||||
(u'Economy and Policy', u'https://www.business-standard.com/rss/economy-policy-102.rss'),
|
||||
(u'Finance', u'https://www.business-standard.com/rss/finance-103.rss'),
|
||||
(u'Beyond Business', u'https://www.business-standard.com/rss/beyond-business-104.rss'),
|
||||
(u'Opinion', 'https://www.business-standard.com/rss/opinion-105.rss'),
|
||||
(u'Markets', u'https://www.business-standard.com/rss/markets-106.rss'),
|
||||
(u'Technology', u'https://www.business-standard.com/rss/technology-108.rss'),
|
||||
(u'Personal Finance', u'https://www.business-standard.com/rss/pf-114.rss'),
|
||||
(u'International', u'https://www.business-standard.com/rss/international-116.rss'),
|
||||
# (u'Today\'s Paper', u'https://www.business-standard.com/rss/todays-paper.rss'),
|
||||
# for todays paper - subscrition required
|
||||
]
|
||||
for sec in sections:
|
||||
a = 'https://news.google.com/rss/search?q=when:27h+allinurl:business-standard.com{}&hl=en-IN&gl=IN&ceid=IN:en'
|
||||
feeds.append((sec.capitalize(), a.format('%2F' + sec + '%2F')))
|
||||
# feeds.append(('Others', a.format('')))
|
||||
|
||||
def preprocess_raw_html(self, raw, *a):
|
||||
root = parse(raw)
|
||||
m = root.xpath('//script[@id="__NEXT_DATA__"]')
|
||||
|
||||
data = json.loads(m[0].text)
|
||||
data = data['props']['pageProps']['data']
|
||||
|
||||
title = '<h1>' + data['pageTitle'] + '</h1>'
|
||||
|
||||
cat = subhead = lede = auth = caption = ''
|
||||
|
||||
if 'defaultArticleCat' in data and data['defaultArticleCat'] is not None:
|
||||
if 'h1_tag' in data['defaultArticleCat'] and data['defaultArticleCat']['h1_tag'] is not None:
|
||||
cat = '<div><p class="cat">' + data['defaultArticleCat']['h1_tag'] + '</p></div>'
|
||||
|
||||
if 'metaDescription' in data and data['metaDescription'] is not None:
|
||||
subhead = '<h3>' + data['metaDescription'] + '</h3>'
|
||||
self.art_desc = data['metaDescription']
|
||||
|
||||
date = (datetime.fromtimestamp(int(data['publishDate']))).strftime('%b %d, %Y | %I:%M %p')
|
||||
|
||||
if 'multiple_authors_name' in data:
|
||||
auth = '<div><p class="auth">' + data['multiple_authors_name'] + ' | ' + data['placeName'] + ' | ' + date + '</p></div>'
|
||||
|
||||
if data['featuredImageObj'] and 'url' in data['featuredImageObj']:
|
||||
lede = '<p class="cap"><img src="{}">'.format(data['featuredImageObj']['url'])
|
||||
if 'alt_text' in data['featuredImageObj']:
|
||||
caption = '<span>' + data['featuredImageObj']['alt_text'] + '</span></p>'
|
||||
|
||||
body = data['htmlContent']
|
||||
|
||||
return '<html><body>' + cat + title + subhead + auth + lede + caption + '<div><p></p>' + body + '</div></body></html>'
|
||||
|
||||
def populate_article_metadata(self, article, soup, first):
|
||||
article.url = self.art_url
|
||||
article.summary = self.art_desc
|
||||
article.text_summary = self.art_desc
|
||||
article.title = article.title.replace(' - Business Standard', '')
|
||||
|
||||
@ -26,8 +26,22 @@ class TagesspiegelRss(BasicNewsRecipe):
|
||||
ignore_duplicate_articles = {'title', 'url'}
|
||||
remove_empty_feeds = True
|
||||
|
||||
def get_browser(self):
|
||||
return BasicNewsRecipe.get_browser(self, verify_ssl_certificates=False)
|
||||
|
||||
def get_cover_url(self):
|
||||
from datetime import date
|
||||
cover = 'https://img.kiosko.net/' + date.today().strftime('%Y/%m/%d') + '/de/tagesspiegel.750.jpg'
|
||||
return cover
|
||||
|
||||
keep_only_tags = [
|
||||
classes('ts-lead ts-article-body ts-intro ts-title ts-authors')
|
||||
dict(name = 'header', attrs={'class':'Bo'}),
|
||||
dict(name = 'div', attrs={'id':'story-elements'})
|
||||
]
|
||||
|
||||
remove_tags = [
|
||||
dict(name = 'aside'),
|
||||
classes('iqd_mainAd Bs')
|
||||
]
|
||||
|
||||
feeds = [
|
||||
@ -42,5 +56,3 @@ class TagesspiegelRss(BasicNewsRecipe):
|
||||
(u'Wissen', u'http://www.tagesspiegel.de/contentexport/feed/wissen')
|
||||
]
|
||||
|
||||
def get_masthead_url(self):
|
||||
return 'http://www.tagesspiegel.de/images/tsp_logo/3114/6.png'
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user