mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-07-09 03:04:10 -04:00
Merge branch 'master' of https://github.com/unkn0w7n/calibre
This commit is contained in:
Kovid Goyal
commit
bf4732a4ff
@ -10,8 +10,7 @@ from base64 import standard_b64encode
|
||||
from datetime import date, timedelta
|
||||
from mechanize import Request
|
||||
|
||||
from calibre.ptempfile import PersistentTemporaryFile
|
||||
from calibre.web.feeds.news import BasicNewsRecipe
|
||||
from calibre.web.feeds.news import BasicNewsRecipe, classes
|
||||
from css_selectors import Select
|
||||
|
||||
try:
|
||||
@ -24,7 +23,7 @@ except ImportError:
|
||||
from urllib import quote
|
||||
|
||||
|
||||
needs_subscription = 'optional'
|
||||
needs_subscription = True
|
||||
|
||||
|
||||
def substring_classes(classes):
|
||||
@ -60,68 +59,33 @@ class WSJ(BasicNewsRecipe):
|
||||
needs_subscription = needs_subscription
|
||||
WSJ_ITP = 'https://www.wsj.com/print-edition/today'
|
||||
|
||||
storage = []
|
||||
|
||||
extra_css = '''
|
||||
#big-top-caption { font-size:small; text-align:center; }
|
||||
[data-type:"tagline"], em { font-style:italic; color:#202020; }
|
||||
.auth { font-size:small; }
|
||||
.imageCaption{font-size:small; text-align:center;}
|
||||
.sub-head{font-style:italic; color:#404040;}
|
||||
.bylineWrap{font-size:small; text-align:left;}
|
||||
'''
|
||||
|
||||
keep_only_tags = [
|
||||
dict(name=['h1', 'h2']),
|
||||
dict(attrs={'aria-describedby':'big-top-caption'}),
|
||||
dict(attrs={'id':'big-top-caption'}),
|
||||
dict(name='article')
|
||||
dict(attrs={'class': lambda x: x and 'HeadlineContainer' in ''.join(x)}),
|
||||
dict(name='main'),
|
||||
]
|
||||
|
||||
remove_tags = [
|
||||
dict(name=['button', 'svg', 'ufc-follow-author-widget']),
|
||||
dict(attrs={'aria-label':['Sponsored Offers', 'Listen To Article', 'What to Read Next']}),
|
||||
dict(attrs={'data-type':'inset'}),
|
||||
dict(attrs={'id':lambda x: x and x.startswith(('wrapper-INLINE', 'audio-tag-inner-audio-'))})
|
||||
classes(
|
||||
'wsj-ad newsletter-inset media-object-video media-object-podcast print-header article-body-tools'
|
||||
' podcast--iframe dynamic-inset-overflow-button snippet-logo'),
|
||||
dict(role=["toolbar", "complementary"]),
|
||||
dict(attrs={"aria-label": ["Sponsored Offers", "What to Read Next", "breadcrumbs", "Listen To Article"]}),
|
||||
dict(name='amp-iframe'), # interactive graphics
|
||||
]
|
||||
|
||||
articles_are_obfuscated = True
|
||||
def get_obfuscated_article(self, url):
|
||||
from calibre.scraper.simple import read_url
|
||||
br = self.get_browser()
|
||||
br.set_handle_redirect(False)
|
||||
try:
|
||||
br.open(url)
|
||||
except Exception as e:
|
||||
url = e.hdrs.get('location')
|
||||
raw = read_url(self.storage, 'https://archive.is/latest/' + url)
|
||||
pt = PersistentTemporaryFile('.html')
|
||||
pt.write(raw.encode('utf-8'))
|
||||
pt.close()
|
||||
return pt.name
|
||||
|
||||
def preprocess_html(self, soup):
|
||||
for img in soup.findAll('img', attrs={'old-src':True}):
|
||||
img['src'] = img['old-src']
|
||||
for p in soup.findAll('div', attrs={'data-type':['paragraph', 'image']}):
|
||||
p.name = 'p'
|
||||
for a in soup.findAll('a', href=True):
|
||||
a['href'] = 'http' + a['href'].split('http')[-1]
|
||||
for fig in soup.findAll('figure'):
|
||||
if fig.find('video'):
|
||||
fig.extract()
|
||||
for figc in soup.findAll('figcaption'):
|
||||
figc['id'] = 'big-top-caption'
|
||||
if name:= soup.find('h2', attrs={'itemprop':'name'}):
|
||||
name.extract()
|
||||
for h2 in soup.findAll('h2'):
|
||||
if self.tag_to_string(h2).startswith('What to Read Next'):
|
||||
h2.extract()
|
||||
for ph in soup.findAll('a', attrs={'data-type':['phrase', 'link']}):
|
||||
if div := ph.findParent('div'):
|
||||
div.name = 'span'
|
||||
for auth in soup.findAll('a', attrs={'aria-label': lambda x: x and x.startswith('Author page')}):
|
||||
if div := auth.find_previous_sibling('div'):
|
||||
div.name = 'span'
|
||||
if parent := auth.findParent('div'):
|
||||
parent['class'] = 'auth'
|
||||
for by in soup.findAll(**classes('bylineWrap')):
|
||||
for p in by.findAll('p'):
|
||||
p.name = 'span'
|
||||
h2 = soup.find('h2', attrs={'class':'sub-head'})
|
||||
if h2:
|
||||
h2.name = 'p'
|
||||
return soup
|
||||
|
||||
# login {{{
|
||||
@ -133,7 +97,7 @@ class WSJ(BasicNewsRecipe):
|
||||
br.set_cookie('ccpaApplies', 'false', '.wsj.com')
|
||||
return br
|
||||
|
||||
if False and needs_subscription: # disabled as we currently use archive.is
|
||||
if needs_subscription:
|
||||
def get_browser(self, *a, **kw):
|
||||
from pprint import pprint
|
||||
pprint
|
||||
@ -304,6 +268,6 @@ class WSJ(BasicNewsRecipe):
|
||||
return [
|
||||
('Testing', [
|
||||
{'title': 'Subscriber Article',
|
||||
'url': self.abs_wsj_url('https://www.wsj.com/articles/remington-gun-call-of-duty-video-game-93059a66')},
|
||||
'url': self.abs_wsj_url('https://www.wsj.com/articles/egg-prices-jump-as-bird-flu-hits-poultry-flocks-11648900800')},
|
||||
]),
|
||||
]
|
||||
|
||||
@ -10,8 +10,7 @@ from base64 import standard_b64encode
|
||||
from datetime import date, timedelta
|
||||
from mechanize import Request
|
||||
|
||||
from calibre.ptempfile import PersistentTemporaryFile
|
||||
from calibre.web.feeds.news import BasicNewsRecipe
|
||||
from calibre.web.feeds.news import BasicNewsRecipe, classes
|
||||
from css_selectors import Select
|
||||
|
||||
try:
|
||||
@ -60,69 +59,33 @@ class WSJ(BasicNewsRecipe):
|
||||
needs_subscription = needs_subscription
|
||||
WSJ_ITP = 'https://www.wsj.com/print-edition/today'
|
||||
|
||||
storage = []
|
||||
|
||||
extra_css = '''
|
||||
#big-top-caption { font-size:small; text-align:center; }
|
||||
[data-type:"tagline"] { font-style:italic; color:#202020; }
|
||||
[data-type:"tagline"], em { font-style:italic; color:#202020; }
|
||||
.auth { font-size:small; }
|
||||
.imageCaption{font-size:small; text-align:center;}
|
||||
.sub-head{font-style:italic; color:#404040;}
|
||||
.bylineWrap{font-size:small; text-align:left;}
|
||||
'''
|
||||
|
||||
keep_only_tags = [
|
||||
dict(name=['h1', 'h2']),
|
||||
dict(attrs={'aria-describedby':'big-top-caption'}),
|
||||
dict(attrs={'id':'big-top-caption'}),
|
||||
dict(name='article')
|
||||
dict(attrs={'class': lambda x: x and 'HeadlineContainer' in ''.join(x)}),
|
||||
dict(name='main'),
|
||||
]
|
||||
|
||||
remove_tags = [
|
||||
dict(name=['button', 'svg', 'ufc-follow-author-widget']),
|
||||
dict(attrs={'aria-label':['Sponsored Offers', 'Listen To Article', 'What to Read Next']}),
|
||||
dict(attrs={'data-type':'inset'}),
|
||||
dict(attrs={'id':lambda x: x and x.startswith(('wrapper-INLINE', 'audio-tag-inner-audio-'))})
|
||||
classes(
|
||||
'wsj-ad newsletter-inset media-object-video media-object-podcast print-header article-body-tools'
|
||||
' podcast--iframe dynamic-inset-overflow-button snippet-logo'),
|
||||
dict(role=["toolbar", "complementary"]),
|
||||
dict(attrs={"aria-label": ["Sponsored Offers", "What to Read Next", "breadcrumbs", "Listen To Article"]}),
|
||||
dict(name='amp-iframe'), # interactive graphics
|
||||
]
|
||||
|
||||
articles_are_obfuscated = True
|
||||
def get_obfuscated_article(self, url):
|
||||
from calibre.scraper.simple import read_url
|
||||
br = self.get_browser()
|
||||
br.set_handle_redirect(False)
|
||||
try:
|
||||
br.open(url)
|
||||
except Exception as e:
|
||||
url = e.hdrs.get('location')
|
||||
raw = read_url(self.storage, 'https://archive.is/latest/' + url)
|
||||
pt = PersistentTemporaryFile('.html')
|
||||
pt.write(raw.encode('utf-8'))
|
||||
pt.close()
|
||||
return pt.name
|
||||
|
||||
def preprocess_html(self, soup):
|
||||
for img in soup.findAll('img', attrs={'old-src':True}):
|
||||
img['src'] = img['old-src']
|
||||
for p in soup.findAll('div', attrs={'data-type':['paragraph', 'image']}):
|
||||
p.name = 'p'
|
||||
for a in soup.findAll('a', href=True):
|
||||
a['href'] = 'http' + a['href'].split('http')[-1]
|
||||
for fig in soup.findAll('figure'):
|
||||
if fig.find('video'):
|
||||
fig.extract()
|
||||
for figc in soup.findAll('figcaption'):
|
||||
figc['id'] = 'big-top-caption'
|
||||
if name:= soup.find('h2', attrs={'itemprop':'name'}):
|
||||
name.extract()
|
||||
for h2 in soup.findAll('h2'):
|
||||
if self.tag_to_string(h2).startswith('What to Read Next'):
|
||||
h2.extract()
|
||||
for ph in soup.findAll('a', attrs={'data-type':['phrase', 'link']}):
|
||||
if div := ph.findParent('div'):
|
||||
div.name = 'span'
|
||||
for auth in soup.findAll('a', attrs={'aria-label': lambda x: x and x.startswith('Author page')}):
|
||||
if div := auth.find_previous_sibling('div'):
|
||||
div.name = 'span'
|
||||
if parent := auth.findParent('div'):
|
||||
parent['class'] = 'auth'
|
||||
for by in soup.findAll(**classes('bylineWrap')):
|
||||
for p in by.findAll('p'):
|
||||
p.name = 'span'
|
||||
h2 = soup.find('h2', attrs={'class':'sub-head'})
|
||||
if h2:
|
||||
h2.name = 'p'
|
||||
return soup
|
||||
|
||||
# login {{{
|
||||
@ -134,7 +97,7 @@ class WSJ(BasicNewsRecipe):
|
||||
br.set_cookie('ccpaApplies', 'false', '.wsj.com')
|
||||
return br
|
||||
|
||||
if False and needs_subscription: # disabled as we currently use archive.is
|
||||
if needs_subscription:
|
||||
def get_browser(self, *a, **kw):
|
||||
from pprint import pprint
|
||||
pprint
|
||||
@ -305,6 +268,6 @@ class WSJ(BasicNewsRecipe):
|
||||
return [
|
||||
('Testing', [
|
||||
{'title': 'Subscriber Article',
|
||||
'url': self.abs_wsj_url('https://www.wsj.com/articles/remington-gun-call-of-duty-video-game-93059a66')},
|
||||
'url': self.abs_wsj_url('https://www.wsj.com/articles/egg-prices-jump-as-bird-flu-hits-poultry-flocks-11648900800')},
|
||||
]),
|
||||
]
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user