Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Update Bloomberg and Deccan Herald

Browse Source
This commit is contained in:
Kovid Goyal 2023-02-21 17:23:35 +05:30
parent 1ef455148a
commit 109a845751
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
2 changed files with 34 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
recipes/bloomberg.recipe
View File

@ -4,7 +4,6 @@ from calibre.ptempfile import PersistentTemporaryFile
import json
import re
class Bloomberg(BasicNewsRecipe):
title = u'Bloomberg'
language = 'en'
@ -12,7 +11,7 @@ class Bloomberg(BasicNewsRecipe):
no_stylesheets = True
use_embedded_content = False
remove_attributes = ['style', 'height', 'width']
ignore_duplicate_articles = {'url'}
ignore_duplicate_articles = {'url', 'title'}
resolve_internal_links = True
oldest_article = 2 # days
delay = 1.5
@ -36,6 +35,9 @@ class Bloomberg(BasicNewsRecipe):
url = e.hdrs.get('location')
soup = self.index_to_soup(url)
link = soup.find('a', attrs={'href':lambda x: x and x.startswith('https://www.bloomberg.com')})
if '/videos/' in link['href']:
self.abort_article('Aborting Video article')
self.log('Found link: ', link['href'])
html = br.open(link['href']).read()
pt = PersistentTemporaryFile('.html')
pt.write(html)
@ -49,7 +51,14 @@ class Bloomberg(BasicNewsRecipe):
return br
feeds = [
('Articles', 'https://news.google.com/rss/search?q=when:24h+allinurl:bloomberg.com&hl=en-US&gl=US&ceid=US:en'),
('Features',
'https://news.google.com/rss/search?q=when:27h+allinurl:bloomberg.com%2Fnews%2Ffeatures%2F&hl=en-US&gl=US&ceid=US:en'),
('News',
'https://news.google.com/rss/search?q=when:27h+allinurl:bloomberg.com%2Fnews%2Farticles%2F&hl=en-US&gl=US&ceid=US:en'),
('Opinion', 'https://news.google.com/rss/search?q=when:27h+allinurl:bloomberg.com%2Fopinion%2F&hl=en-US&gl=US&ceid=US:en'),
('Newsletters',
'https://news.google.com/rss/search?q=when:27h+allinurl:bloomberg.com%2Fnews%2Fnewsletters%2F&hl=en-US&gl=US&ceid=US:en'),
('Others', 'https://news.google.com/rss/search?q=when:27h+allinurl:bloomberg.com&hl=en-US&gl=US&ceid=US:en')
]
def preprocess_raw_html(self, raw, *a):
@ -84,12 +93,12 @@ class Bloomberg(BasicNewsRecipe):
if 'ledeImageUrl' in data:
if data['ledeImageUrl'] is not None:
lede = '<p><img src="{}">'.format(data['ledeImageUrl'].replace('\\', ''))
lede = '<p><img src="{}">'.format(data['ledeImageUrl'])
if data['ledeDescription'] is not None:
caption = '<span id="cap">' + data['ledeDescription'] + '</span>'
body = data['body'].replace('\\', '')
body = data['body']
html = '<html><body>' + cat + title + subhead + auth + lede + caption + '<div>' + body
return html

16
recipes/deccan_herald.recipe
View File

@ -8,7 +8,7 @@ class herald(BasicNewsRecipe):
language = 'en_IN'
no_stylesheets = True
remove_attributes = ['height', 'width', 'style']
ignore_duplicate_articles = {'url'}
ignore_duplicate_articles = {'url', 'title'}
encoding = 'utf-8'
articles_are_obfuscated = True
@ -22,7 +22,7 @@ class herald(BasicNewsRecipe):
soup = self.index_to_soup(url)
link = soup.find('a', href=True)
skip_sections =[ # add sections you want to skip
'/sports/', '/video/', '/bengaluru-crime/', '/metrolife/',
'/video/', '/bengaluru-crime/', '/metrolife/',
'/karnataka-districts/', '/brandspot/', '/entertainment/',
]
if any(x in link['href'] for x in skip_sections):
@ -51,5 +51,15 @@ class herald(BasicNewsRecipe):
]
feeds = [
('DH', 'https://news.google.com/rss/search?q=when:27h+allinurl:deccanherald.com&hl=en-IN&gl=IN&ceid=IN:en')
('Nation', 'https://news.google.com/rss/search?q=when:27h+allinurl:deccanherald.com%2Fnational%2F&hl=en-IN&gl=IN&ceid=IN:en'),
('Karnataka', 'https://news.google.com/rss/search?q=when:27h+allinurl:deccanherald.com%2Fstate%2F&hl=en-IN&gl=IN&ceid=IN:en'),
('Opinion', 'https://news.google.com/rss/search?q=when:27h+allinurl:deccanherald.com%2Fopinion%2F&hl=en-IN&gl=IN&ceid=IN:en'),
('City',
'https://news.google.com/rss/search?q=when:27h+allinurl:deccanherald.com%2Fcity%2F&hl=en-IN&gl=IN&ceid=IN:en'),
('Business', 'https://news.google.com/rss/search?q=when:27h+allinurl:deccanherald.com%2Fbusiness%2F&hl=en-IN&gl=IN&ceid=IN:en'),
('World',
'https://news.google.com/rss/search?q=when:27h+allinurl:deccanherald.com%2Finternational%2F&hl=en-IN&gl=IN&ceid=IN:en'),
('Sports',
'https://news.google.com/rss/search?q=when:27h+allinurl:deccanherald.com%2Fsports%2F&hl=en-IN&gl=IN&ceid=IN:en'),
('Others', 'https://news.google.com/rss/search?q=when:27h+allinurl:deccanherald.com&hl=en-IN&gl=IN&ceid=IN:en'),
]