mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-07-09 03:04:10 -04:00
Updating for paywall added 2 years ago
Cleanup and login now required.
This commit is contained in:
parent
8c382e7e4f
commit
062fe39393
@ -3,28 +3,34 @@ __copyright__ = '2008-2010, Darko Miletic <darko.miletic at gmail.com>'
|
|||||||
'''
|
'''
|
||||||
telegraph.co.uk
|
telegraph.co.uk
|
||||||
'''
|
'''
|
||||||
|
from mechanize import Request
|
||||||
import json
|
|
||||||
from calibre import random_user_agent
|
from calibre import random_user_agent
|
||||||
from calibre.web.feeds.news import BasicNewsRecipe
|
from calibre.web.feeds.news import BasicNewsRecipe
|
||||||
|
from mechanize import Request
|
||||||
|
from css_selectors import Select
|
||||||
|
|
||||||
|
try:
|
||||||
|
import urllib.parse as urlparse
|
||||||
|
except ImportError:
|
||||||
|
import urlparse
|
||||||
|
try:
|
||||||
|
from urllib.parse import quote
|
||||||
|
except ImportError:
|
||||||
|
from urllib import quote
|
||||||
|
|
||||||
def classes(classes):
|
def classes(classes):
|
||||||
q = frozenset(classes.split(' '))
|
q = frozenset(classes.split(' '))
|
||||||
return dict(
|
return dict(
|
||||||
attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)}
|
attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)})
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def absolutize(url):
|
def absolutize(url):
|
||||||
if url.startswith('/'):
|
if url.startswith('/'):
|
||||||
url = 'http://www.telegraph.co.uk' + url
|
url = 'http://www.telegraph.co.uk' + url
|
||||||
return url
|
return url
|
||||||
|
|
||||||
|
|
||||||
class TelegraphUK(BasicNewsRecipe):
|
class TelegraphUK(BasicNewsRecipe):
|
||||||
title = 'Telegraph.co.uk'
|
title = 'The Telegraph (UK)'
|
||||||
__author__ = 'Darko Miletic and Sujata Raman'
|
__author__ = 'A10KiloHam, based on work by Darko Miletic and Sujata Raman'
|
||||||
description = 'News from United Kingdom'
|
description = 'News from United Kingdom'
|
||||||
oldest_article = 2
|
oldest_article = 2
|
||||||
category = 'news, politics, UK'
|
category = 'news, politics, UK'
|
||||||
@ -33,39 +39,70 @@ class TelegraphUK(BasicNewsRecipe):
|
|||||||
no_stylesheets = True
|
no_stylesheets = True
|
||||||
language = 'en_GB'
|
language = 'en_GB'
|
||||||
encoding = 'utf-8'
|
encoding = 'utf-8'
|
||||||
|
needs_subscription = True
|
||||||
ignore_duplicate_articles = {'title', 'url'}
|
ignore_duplicate_articles = {'title', 'url'}
|
||||||
remove_empty_feeds = True
|
remove_empty_feeds = True
|
||||||
use_embedded_content = False
|
use_embedded_content = False
|
||||||
|
INDEX = 'https://www.telegraph.co.uk/'
|
||||||
|
LOGIN = 'https://secure.telegraph.co.uk/customer/secure/login/?redirectTo=https%3A%2F%2Fwww.telegraph.co.uk%2F'
|
||||||
|
PREFIX = u'https://www.telegraph.co.uk'
|
||||||
|
|
||||||
feeds = [
|
feeds = [
|
||||||
(u'UK News', u'http://www.telegraph.co.uk/news/uknews/rss'),
|
(u'News', u'http://www.telegraph.co.uk/news/rss.xml'),
|
||||||
(u'World News', u'http://www.telegraph.co.uk/news/worldnews/rss'),
|
(u'Politics', u'https://www.telegraph.co.uk/politics/rss.xml'),
|
||||||
(u'Politics', u'http://www.telegraph.co.uk/news/newstopics/politics/rss'),
|
(u'Business', u'http://www.telegraph.co.uk/business/rss.xml'),
|
||||||
(u'Finance', u'http://www.telegraph.co.uk/finance/rss'),
|
(u'Money', u'http://www.telegraph.co.uk/money/rss.xml'),
|
||||||
(u'Technology News', u'http://www.telegraph.co.uk/scienceandtechnology/technology/technologynews/rss'),
|
(u'Technology', u'http://www.telegraph.co.uk/technology/rss.xml'),
|
||||||
(u'UK News', u'http://www.telegraph.co.uk/scienceandtechnology/technology/technologyreviews/rss'),
|
(u'Science', u'http://www.telegraph.co.uk/science/rss.xml'),
|
||||||
(u'Science News', u'http://www.telegraph.co.uk/scienceandtechnology/science/sciencenews/rss'),
|
(u'Opinion', u'http://www.telegraph.co.uk/opinion/rss.xml'),
|
||||||
(u'Sport', u'http://www.telegraph.co.uk/sport/rss'),
|
(u'Travel', u'http://www.telegraph.co.uk/travel/rss.xml'),
|
||||||
(u'Earth News', u'http://www.telegraph.co.uk/earth/earthnews/rss'),
|
(u'Culture', u'http://www.telegraph.co.uk/culture/rss.xml'),
|
||||||
(u'Comment', u'http://www.telegraph.co.uk/comment/rss'),
|
(u'Lifestyle', u'http://www.telegraph.co.uk/lifestyle/rss.xml'),
|
||||||
(u'Travel', u'http://www.telegraph.co.uk/travel/rss'),
|
(u'Money', u'http://www.telegraph.co.uk/opinion/rss.xml'),
|
||||||
(u'How about that?', u'http://www.telegraph.co.uk/news/newstopics/howaboutthat/rss')
|
(u'Opinion', u'http://www.telegraph.co.uk/money/rss.xml'),
|
||||||
|
(u'Fashion', u'http://www.telegraph.co.uk/fashion/rss.xml')
|
||||||
]
|
]
|
||||||
|
|
||||||
keep_only_tags = [
|
keep_only_tags = [
|
||||||
classes('lead-asset-image-container headline__heading footer-author'),
|
classes('lead-asset-image-container headline__heading footer-author article-author__meta'),
|
||||||
dict(itemprop='articleBody'),
|
dict(itemprop='articleBody'),
|
||||||
]
|
]
|
||||||
|
|
||||||
remove_tags = [
|
remove_tags = [
|
||||||
dict(name=['link', 'meta', 'style']),
|
dict(name=['link', 'meta', 'style']),
|
||||||
classes('videoPlayer'),
|
classes('videoPlayer'),
|
||||||
]
|
]
|
||||||
remove_attributes = 'width height'.split()
|
remove_attributes = 'width height'.split()
|
||||||
|
|
||||||
def get_browser(self):
|
def get_cover_url(self):
|
||||||
return BasicNewsRecipe.get_browser(
|
from datetime import date
|
||||||
self, user_agent=random_user_agent(allow_ie=False)
|
cover = 'http://img.kiosko.net/' + str(date.today().year) + '/' + date.today().strftime('%m') + '/' + date.today().strftime('%d') + '/uk/daily_telegraph.750.jpg'
|
||||||
)
|
br = BasicNewsRecipe.get_browser(self)
|
||||||
|
try:
|
||||||
|
br.open(cover)
|
||||||
|
except:
|
||||||
|
index = 'http://en.kiosko.net/uk/np/daily_telegraph.html'
|
||||||
|
soup = self.index_to_soup(index)
|
||||||
|
for image in soup.findAll('img', src=True):
|
||||||
|
if image['src'].endswith('750.jpg'):
|
||||||
|
return image['src']
|
||||||
|
self.log("\nCover unavailable")
|
||||||
|
cover = None
|
||||||
|
return cover
|
||||||
|
|
||||||
|
def get_browser(self, *a, **kw):
|
||||||
|
USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0'
|
||||||
|
br = BasicNewsRecipe.get_browser(self, user_agent=USER_AGENT)
|
||||||
|
self.log('Forming login request...')
|
||||||
|
if self.username is not None and self.password is not None:
|
||||||
|
self.log('Starting login process...')
|
||||||
|
br.open(self.LOGIN)
|
||||||
|
br.select_form(nr=0)
|
||||||
|
br['email'] = self.username
|
||||||
|
br['password'] = self.password
|
||||||
|
self.log('Sending login request...')
|
||||||
|
br.submit()
|
||||||
|
return br
|
||||||
|
|
||||||
def get_article_url(self, article):
|
def get_article_url(self, article):
|
||||||
url = article.get('link', None)
|
url = article.get('link', None)
|
||||||
@ -85,6 +122,5 @@ class TelegraphUK(BasicNewsRecipe):
|
|||||||
for img in soup.findAll('div', attrs={'data-js': 'LazyImage'}):
|
for img in soup.findAll('div', attrs={'data-js': 'LazyImage'}):
|
||||||
img['style'] = ''
|
img['style'] = ''
|
||||||
img.name = 'img'
|
img.name = 'img'
|
||||||
srcs = [x.split(' ')[0].strip() for x in img['data-srcset'].split(',')]
|
img['src'] = img['data-srcset'].split()[0]
|
||||||
img['src'] = srcs[1 if len(srcs) > 0 else 0]
|
|
||||||
return soup
|
return soup
|
||||||
|
Loading…
x
Reference in New Issue
Block a user