mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-06-23 15:30:45 -04:00
Merge branch 'master' of https://github.com/unkn0w7n/calibre
This commit is contained in:
Kovid Goyal
commit
2016d80094
BIN
recipes/icons/nyt_tmag.png
Normal file
BIN
recipes/icons/nyt_tmag.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 416 B |
Binary file not shown.
|
Before Width: | Height: | Size: 9.5 KiB After Width: | Height: | Size: 416 B |
127
recipes/nyt_tmag.recipe
Normal file
127
recipes/nyt_tmag.recipe
Normal file
@ -0,0 +1,127 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
from calibre.web.feeds.news import BasicNewsRecipe
|
||||
|
||||
|
||||
use_wayback_machine = False
|
||||
|
||||
|
||||
class NytMag(BasicNewsRecipe):
|
||||
title = 'NYT T Magazine'
|
||||
__author__ = 'unkn0wn'
|
||||
description = 'The latest from The New York Times Style Magazine.'
|
||||
oldest_article = 30 # days
|
||||
encoding = 'utf-8'
|
||||
use_embedded_content = False
|
||||
language = 'en_US'
|
||||
remove_empty_feeds = True
|
||||
resolve_internal_links = True
|
||||
ignore_duplicate_articles = {'title', 'url'}
|
||||
masthead_url = 'https://static01.nytimes.com/newsgraphics/2015-12-23-masthead-2016/b15c3d81d3d7b59065fff9a3f3afe85aa2e2dff5/_assets/nyt-logo.png'
|
||||
|
||||
feeds = [
|
||||
'https://rss.nytimes.com/services/xml/rss/nyt/tmagazine.xml',
|
||||
'https://rss.nytimes.com/services/xml/rss/nyt/FashionandStyle.xml',
|
||||
]
|
||||
|
||||
recipe_specific_options = {
|
||||
'days': {
|
||||
'short': 'Oldest article to download from this news source. In days ',
|
||||
'long': 'For example, 0.5, gives you articles from the past 12 hours',
|
||||
'default': str(oldest_article),
|
||||
},
|
||||
'comp': {'short': 'Compress News Images?', 'long': 'enter yes', 'default': 'no'},
|
||||
'rev': {
|
||||
'short': 'Reverse the order of articles in each feed?',
|
||||
'long': 'enter yes',
|
||||
'default': 'no',
|
||||
},
|
||||
'res': {
|
||||
'short': (
|
||||
'For hi-res images, select a resolution from the following\noptions: '
|
||||
'popup, jumbo, mobileMasterAt3x, superJumbo'
|
||||
),
|
||||
'long': (
|
||||
'This is useful for non e-ink devices, and for a lower file size\nthan '
|
||||
'the default, use mediumThreeByTwo440, mediumThreeByTwo225, articleInline.'
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
BasicNewsRecipe.__init__(self, *args, **kwargs)
|
||||
d = self.recipe_specific_options.get('days')
|
||||
if d and isinstance(d, str):
|
||||
self.oldest_article = float(d)
|
||||
r = self.recipe_specific_options.get('rev')
|
||||
if r and isinstance(r, str):
|
||||
if r.lower() == 'yes':
|
||||
self.reverse_article_order = True
|
||||
c = self.recipe_specific_options.get('comp')
|
||||
if c and isinstance(c, str):
|
||||
if c.lower() == 'yes':
|
||||
self.compress_news_images = True
|
||||
|
||||
extra_css = """
|
||||
.byl, .time { font-size:small; color:#202020; }
|
||||
.cap { font-size:small; text-align:center; }
|
||||
.cred { font-style:italic; font-size:small; }
|
||||
em, blockquote { color: #202020; }
|
||||
.sc { font-variant: small-caps; }
|
||||
.lbl { font-size:small; color:#404040; }
|
||||
img { display:block; margin:0 auto; }
|
||||
"""
|
||||
|
||||
@property
|
||||
def nyt_parser(self):
|
||||
ans = getattr(self, '_nyt_parser', None)
|
||||
if ans is None:
|
||||
from calibre.live import load_module
|
||||
|
||||
self._nyt_parser = ans = load_module('calibre.web.site_parsers.nytimes')
|
||||
return ans
|
||||
|
||||
def get_nyt_page(self, url, skip_wayback=False):
|
||||
if use_wayback_machine and not skip_wayback:
|
||||
from calibre import browser
|
||||
|
||||
return self.nyt_parser.download_url(url, browser())
|
||||
return self.index_to_soup(url, raw=True)
|
||||
|
||||
articles_are_obfuscated = use_wayback_machine
|
||||
|
||||
if use_wayback_machine:
|
||||
|
||||
def get_obfuscated_article(self, url):
|
||||
from calibre.ptempfile import PersistentTemporaryFile
|
||||
|
||||
with PersistentTemporaryFile() as tf:
|
||||
tf.write(self.get_nyt_page(url))
|
||||
return tf.name
|
||||
|
||||
def preprocess_raw_html(self, raw_html, url):
|
||||
return self.nyt_parser.extract_html(self.index_to_soup(raw_html), url)
|
||||
|
||||
def get_browser(self, *args, **kwargs):
|
||||
kwargs['user_agent'] = (
|
||||
'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
|
||||
)
|
||||
br = BasicNewsRecipe.get_browser(self, *args, **kwargs)
|
||||
br.addheaders += [
|
||||
('Referer', 'https://www.google.com/'),
|
||||
('X-Forwarded-For', '66.249.66.1'),
|
||||
]
|
||||
return br
|
||||
|
||||
def preprocess_html(self, soup):
|
||||
w = self.recipe_specific_options.get('res')
|
||||
if w and isinstance(w, str):
|
||||
res = '-' + w
|
||||
for img in soup.findAll('img', attrs={'src': True}):
|
||||
if '-article' in img['src']:
|
||||
ext = img['src'].split('?')[0].split('.')[-1]
|
||||
img['src'] = img['src'].rsplit('-article', 1)[0] + res + '.' + ext
|
||||
for c in soup.findAll('div', attrs={'class': 'cap'}):
|
||||
for p in c.findAll(['p', 'div']):
|
||||
p.name = 'span'
|
||||
return soup
|
||||
@ -1,179 +1,9 @@
|
||||
#!/usr/bin/env python
|
||||
import json
|
||||
import re
|
||||
|
||||
from calibre.utils.iso8601 import parse_iso8601
|
||||
from calibre.web.feeds.news import BasicNewsRecipe
|
||||
|
||||
|
||||
def extract_json(raw):
|
||||
pre = re.search(r'<script>window.__preloadedData = ({.+)', raw).group(1)
|
||||
js = json.JSONDecoder().raw_decode(re.sub('undefined', '[]', pre))[0]
|
||||
return js['initialData']['data']['article']['sprinkledBody']['content']
|
||||
|
||||
def parse_image(i):
|
||||
crop = i.get('crops') or i.get('spanImageCrops')
|
||||
if crop:
|
||||
yield f'<div><img src="{crop[0]["renditions"][0]["url"]}" title="{i.get("altText", "")}">'
|
||||
if i.get('caption'):
|
||||
yield f'<div class="cap">{"".join(parse_types(i["caption"]))}'
|
||||
if i.get('credit'):
|
||||
yield f'<span class="cred"> {i["credit"]}</span>'
|
||||
yield '</div>'
|
||||
elif i.get('legacyHtmlCaption'):
|
||||
if i['legacyHtmlCaption'].strip():
|
||||
yield f'<div class="cap">{i["legacyHtmlCaption"]}</div>'
|
||||
yield '</div>'
|
||||
|
||||
def parse_img_grid(g):
|
||||
for grd in g.get('gridMedia', {}):
|
||||
yield ''.join(parse_image(grd))
|
||||
if g.get('caption'):
|
||||
yield f'<div class="cap">{g["caption"]}'
|
||||
if g.get('credit'):
|
||||
yield f'<span class="cred"> {g["credit"]}</span>'
|
||||
yield '</div>'
|
||||
|
||||
def parse_vid(v):
|
||||
if v.get('promotionalMedia'):
|
||||
headline = v.get("headline", {}).get("default", "")
|
||||
rendition = v.get('renditions')
|
||||
yield (f'<div><b><a href="{rendition[0]["url"]}">Video</a>: {headline}</b></div>'
|
||||
if rendition else f'<div><b>{headline}</b></div>')
|
||||
yield ''.join(parse_types(v["promotionalMedia"]))
|
||||
if v.get('promotionalSummary'):
|
||||
yield f'<div class="cap">{v["promotionalSummary"]}</div>'
|
||||
|
||||
def parse_emb(e):
|
||||
if e.get('html') and 'datawrapper.dwcdn.net' in e.get('html', ''):
|
||||
dw = re.search(r'datawrapper.dwcdn.net/(.{5})', e['html']).group(1)
|
||||
yield f'<div><img src="https://datawrapper.dwcdn.net/{dw}/full.png"></div>'
|
||||
elif e.get('promotionalMedia'):
|
||||
if e.get('headline'):
|
||||
yield f'<div><b>{e["headline"]["default"]}</b></div>'
|
||||
yield ''.join(parse_types(e["promotionalMedia"]))
|
||||
if e.get('note'):
|
||||
yield f'<div class="cap">{e["note"]}</div>'
|
||||
|
||||
def parse_byline(byl):
|
||||
for b in byl.get('bylines', {}):
|
||||
yield f'<div>{b["renderedRepresentation"]}</div>'
|
||||
yield '<div><b><i>'
|
||||
for rl in byl.get('role', {}):
|
||||
if ''.join(parse_cnt(rl)).strip():
|
||||
yield ''.join(parse_cnt(rl))
|
||||
yield '</i></b></div>'
|
||||
|
||||
def iso_date(x):
|
||||
dt = parse_iso8601(x, as_utc=False)
|
||||
return dt.strftime('%b %d, %Y at %I:%M %p')
|
||||
|
||||
def parse_header(h):
|
||||
if h.get('label'):
|
||||
yield f'<div class="lbl">{"".join(parse_types(h["label"]))}</div>'
|
||||
if h.get('headline'):
|
||||
yield ''.join(parse_types(h["headline"]))
|
||||
if h.get('summary'):
|
||||
yield f'<p><i>{"".join(parse_types(h["summary"]))}</i></p>'
|
||||
if h.get('ledeMedia'):
|
||||
yield ''.join(parse_types(h["ledeMedia"]))
|
||||
if h.get('byline'):
|
||||
yield ''.join(parse_types(h["byline"]))
|
||||
if h.get('timestampBlock'):
|
||||
yield ''.join(parse_types(h["timestampBlock"]))
|
||||
|
||||
def parse_fmt_type(fm):
|
||||
for f in fm.get('formats', {}):
|
||||
ftype = f.get("__typename", "")
|
||||
if ftype == "BoldFormat":
|
||||
yield '<strong>'
|
||||
if ftype == "ItalicFormat":
|
||||
yield '<em>'
|
||||
if ftype == "LinkFormat":
|
||||
hrf = f["url"]
|
||||
yield f'<a href="{hrf}">'
|
||||
yield fm.get("text", "")
|
||||
for f in reversed(fm.get('formats', {})):
|
||||
ftype = f.get("__typename", "")
|
||||
if ftype == "BoldFormat":
|
||||
yield '</strong>'
|
||||
if ftype == "ItalicFormat":
|
||||
yield '</em>'
|
||||
if ftype == "LinkFormat":
|
||||
yield '</a>'
|
||||
|
||||
def parse_cnt(cnt):
|
||||
for k in cnt:
|
||||
if isinstance(cnt[k], list):
|
||||
if k == 'formats':
|
||||
yield ''.join(parse_fmt_type(cnt))
|
||||
else:
|
||||
for cnt_ in cnt[k]:
|
||||
yield ''.join(parse_types(cnt_))
|
||||
if isinstance(cnt[k], dict):
|
||||
yield ''.join(parse_types(cnt[k]))
|
||||
if cnt.get('text') and 'formats' not in cnt and 'content' not in cnt:
|
||||
if isinstance(cnt['text'], str):
|
||||
yield cnt['text']
|
||||
|
||||
def parse_types(x):
|
||||
typename = x.get('__typename', '')
|
||||
|
||||
if 'Header' in typename:
|
||||
yield '\n'.join(parse_header(x))
|
||||
|
||||
elif typename.startswith('Heading'):
|
||||
htag = 'h' + re.match(r'Heading([1-6])Block', typename).group(1)
|
||||
yield f'<{htag}>{"".join(parse_cnt(x))}</{htag}>'
|
||||
|
||||
elif typename == 'ParagraphBlock':
|
||||
yield f'<p>{"".join(parse_cnt(x))}</p>'
|
||||
|
||||
elif typename == 'BylineBlock':
|
||||
yield f'<div class="byl"><br/>{"".join(parse_byline(x))}</div>'
|
||||
elif typename == 'LabelBlock':
|
||||
yield f'<div class="sc">{"".join(parse_cnt(x))}</div>'
|
||||
elif typename == 'BlockquoteBlock':
|
||||
yield f'<blockquote>{"".join(parse_cnt(x))}</blockquote>'
|
||||
elif typename == 'TimestampBlock':
|
||||
yield f'<div class="time">{iso_date(x["timestamp"])}</div>'
|
||||
elif typename == 'LineBreakInline':
|
||||
yield '<br/>'
|
||||
elif typename == 'RuleBlock':
|
||||
yield '<hr/>'
|
||||
|
||||
elif typename == 'Image':
|
||||
yield "".join(parse_image(x))
|
||||
|
||||
elif typename == 'GridBlock':
|
||||
yield "".join(parse_img_grid(x))
|
||||
|
||||
elif typename == 'Video':
|
||||
yield "".join(parse_vid(x))
|
||||
|
||||
elif typename == 'EmbeddedInteractive':
|
||||
yield "".join(parse_emb(x))
|
||||
|
||||
elif typename == 'ListBlock':
|
||||
yield f'<ul>{"".join(parse_cnt(x))}</ul>'
|
||||
elif typename == 'ListItemBlock':
|
||||
yield f'<li>{"".join(parse_cnt(x))}</li>'
|
||||
|
||||
elif typename == 'TextInline':
|
||||
yield "".join(parse_cnt(x))
|
||||
|
||||
elif typename in {'DetailBlock', 'TextRunKV'}:
|
||||
yield f'<p><i>{"".join(parse_cnt(x))}</i></p>'
|
||||
|
||||
elif typename and typename not in {'RelatedLinksBlock', 'Dropzone'}:
|
||||
if "".join(parse_cnt(x)).strip():
|
||||
yield "".join(parse_cnt(x))
|
||||
|
||||
def article_parse(data):
|
||||
yield "<html><body>"
|
||||
for d in data:
|
||||
yield from parse_types(d)
|
||||
yield "</body></html>"
|
||||
use_wayback_machine = False
|
||||
|
||||
|
||||
class NytFeeds(BasicNewsRecipe):
|
||||
@ -192,30 +22,59 @@ class NytFeeds(BasicNewsRecipe):
|
||||
ignore_duplicate_articles = {'title', 'url'}
|
||||
masthead_url = 'https://static01.nytimes.com/newsgraphics/2015-12-23-masthead-2016/b15c3d81d3d7b59065fff9a3f3afe85aa2e2dff5/_assets/nyt-logo.png'
|
||||
|
||||
# https://www.nytimes.com/rss
|
||||
# https://developer.nytimes.com/docs/rss-api/1/overview
|
||||
feeds = [
|
||||
# to filter out all opinions from other sections first
|
||||
'https://rss.nytimes.com/services/xml/rss/nyt/Opinion.xml',
|
||||
'https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml',
|
||||
'https://rss.nytimes.com/services/xml/rss/nyt/World.xml',
|
||||
'https://rss.nytimes.com/services/xml/rss/nyt/US.xml',
|
||||
'https://rss.nytimes.com/services/xml/rss/nyt/Business.xml',
|
||||
'https://rss.nytimes.com/services/xml/rss/nyt/YourMoney.xml',
|
||||
# 'https://rss.nytimes.com/services/xml/rss/nyt/Technology.xml',
|
||||
'https://rss.nytimes.com/services/xml/rss/nyt/Science.xml',
|
||||
'https://rss.nytimes.com/services/xml/rss/nyt/Climate.xml',
|
||||
'https://rss.nytimes.com/services/xml/rss/nyt/Health.xml',
|
||||
'https://rss.nytimes.com/services/xml/rss/nyt/Arts.xml',
|
||||
# 'https://rss.nytimes.com/services/xml/rss/nyt/FashionandStyle.xml',
|
||||
# 'https://rss.nytimes.com/services/xml/rss/nyt/tmagazine.xml',
|
||||
# 'https://rss.nytimes.com/services/xml/rss/nyt/books.xml',
|
||||
'https://www.nytimes.com/services/xml/rss/nyt/Travel.xml',
|
||||
'https://rss.nytimes.com/services/xml/rss/nyt/well.xml',
|
||||
'https://rss.nytimes.com/services/xml/rss/nyt/Sports.xml',
|
||||
# 'http://nytimes.com/timeswire/feeds/',
|
||||
]
|
||||
|
||||
def get_cover_url(self):
|
||||
soup = self.index_to_soup('https://www.frontpages.com/the-new-york-times/')
|
||||
return 'https://www.frontpages.com' + soup.find('img', attrs={'id':'giornale-img'})['src']
|
||||
return (
|
||||
'https://www.frontpages.com'
|
||||
+ soup.find('img', attrs={'id': 'giornale-img'})['src']
|
||||
)
|
||||
|
||||
recipe_specific_options = {
|
||||
'days': {
|
||||
'short': 'Oldest article to download from this news source. In days ',
|
||||
'long': 'For example, 0.5, gives you articles from the past 12 hours',
|
||||
'default': str(oldest_article)
|
||||
},
|
||||
'comp': {
|
||||
'short': 'Compress News Images?',
|
||||
'long': 'enter yes',
|
||||
'default': 'no'
|
||||
'default': str(oldest_article),
|
||||
},
|
||||
'comp': {'short': 'Compress News Images?', 'long': 'enter yes', 'default': 'no'},
|
||||
'rev': {
|
||||
'short': 'Reverse the order of articles in each feed?',
|
||||
'long': 'enter yes',
|
||||
'default': 'no'
|
||||
'default': 'no',
|
||||
},
|
||||
'res': {
|
||||
'short': 'For hi-res images, select a resolution from the following\noptions: popup, jumbo, mobileMasterAt3x, superJumbo',
|
||||
'long': 'This is useful for non e-ink devices, and for a lower file size\nthan the default, use articleInline.',
|
||||
}
|
||||
'short': (
|
||||
'For hi-res images, select a resolution from the following\noptions: '
|
||||
'popup, jumbo, mobileMasterAt3x, superJumbo'
|
||||
),
|
||||
'long': (
|
||||
'This is useful for non e-ink devices, and for a lower file size\nthan '
|
||||
'the default, use mediumThreeByTwo440, mediumThreeByTwo225, articleInline.'
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
@ -232,67 +91,66 @@ class NytFeeds(BasicNewsRecipe):
|
||||
if c.lower() == 'yes':
|
||||
self.compress_news_images = True
|
||||
|
||||
extra_css = '''
|
||||
extra_css = """
|
||||
.byl, .time { font-size:small; color:#202020; }
|
||||
.cap { font-size:small; }
|
||||
.cap { font-size:small; text-align:center; }
|
||||
.cred { font-style:italic; font-size:small; }
|
||||
em, blockquote { color: #202020; }
|
||||
.sc { font-variant: small-caps; }
|
||||
.lbl { font-size:small; color:#404040; }
|
||||
img { display:block; margin:0 auto; }
|
||||
'''
|
||||
"""
|
||||
|
||||
# https://www.nytimes.com/rss
|
||||
# https://developer.nytimes.com/docs/rss-api/1/overview
|
||||
feeds = [
|
||||
# to filter out all opinions from other sections first
|
||||
'https://rss.nytimes.com/services/xml/rss/nyt/Opinion.xml',
|
||||
@property
|
||||
def nyt_parser(self):
|
||||
ans = getattr(self, '_nyt_parser', None)
|
||||
if ans is None:
|
||||
from calibre.live import load_module
|
||||
|
||||
'https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml',
|
||||
'https://rss.nytimes.com/services/xml/rss/nyt/World.xml',
|
||||
'https://rss.nytimes.com/services/xml/rss/nyt/US.xml',
|
||||
'https://rss.nytimes.com/services/xml/rss/nyt/Business.xml',
|
||||
'https://rss.nytimes.com/services/xml/rss/nyt/YourMoney.xml',
|
||||
'https://rss.nytimes.com/services/xml/rss/nyt/Technology.xml',
|
||||
'https://rss.nytimes.com/services/xml/rss/nyt/Science.xml',
|
||||
'https://rss.nytimes.com/services/xml/rss/nyt/Climate.xml',
|
||||
'https://rss.nytimes.com/services/xml/rss/nyt/Health.xml',
|
||||
'https://rss.nytimes.com/services/xml/rss/nyt/Arts.xml',
|
||||
'https://rss.nytimes.com/services/xml/rss/nyt/FashionandStyle.xml',
|
||||
'https://rss.nytimes.com/services/xml/rss/nyt/tmagazine.xml',
|
||||
'https://rss.nytimes.com/services/xml/rss/nyt/books.xml',
|
||||
'https://www.nytimes.com/services/xml/rss/nyt/Travel.xml',
|
||||
'https://rss.nytimes.com/services/xml/rss/nyt/well.xml',
|
||||
'https://rss.nytimes.com/services/xml/rss/nyt/Sports.xml',
|
||||
'http://nytimes.com/timeswire/feeds/'
|
||||
]
|
||||
self._nyt_parser = ans = load_module('calibre.web.site_parsers.nytimes')
|
||||
return ans
|
||||
|
||||
def get_nyt_page(self, url, skip_wayback=False):
|
||||
if use_wayback_machine and not skip_wayback:
|
||||
from calibre import browser
|
||||
|
||||
return self.nyt_parser.download_url(url, browser())
|
||||
return self.index_to_soup(url, raw=True)
|
||||
|
||||
articles_are_obfuscated = use_wayback_machine
|
||||
|
||||
if use_wayback_machine:
|
||||
|
||||
def get_obfuscated_article(self, url):
|
||||
from calibre.ptempfile import PersistentTemporaryFile
|
||||
|
||||
with PersistentTemporaryFile() as tf:
|
||||
tf.write(self.get_nyt_page(url))
|
||||
return tf.name
|
||||
|
||||
def preprocess_raw_html(self, raw_html, url):
|
||||
return self.nyt_parser.extract_html(self.index_to_soup(raw_html), url)
|
||||
|
||||
def get_browser(self, *args, **kwargs):
|
||||
kwargs['user_agent'] = 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
|
||||
kwargs['user_agent'] = (
|
||||
'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
|
||||
)
|
||||
br = BasicNewsRecipe.get_browser(self, *args, **kwargs)
|
||||
br.addheaders += [
|
||||
('Referer', 'https://www.google.com/'),
|
||||
('X-Forwarded-For', '66.249.66.1')
|
||||
('X-Forwarded-For', '66.249.66.1'),
|
||||
]
|
||||
return br
|
||||
|
||||
def preprocess_raw_html(self, raw_html, url):
|
||||
if '/interactive/' in url:
|
||||
return '<html><body><p><em>'\
|
||||
+ 'This is an interactive article, which is supposed to be read in a browser.'\
|
||||
+ '</p></em></body></html>'
|
||||
data = extract_json(raw_html)
|
||||
return '\n'.join(article_parse(data))
|
||||
|
||||
def preprocess_html(self, soup):
|
||||
w = self.recipe_specific_options.get('res')
|
||||
if w and isinstance(w, str):
|
||||
res = '-' + w
|
||||
for img in soup.findAll('img', attrs={'src':True}):
|
||||
for img in soup.findAll('img', attrs={'src': True}):
|
||||
if '-article' in img['src']:
|
||||
ext = img['src'].split('?')[0].split('.')[-1]
|
||||
img['src'] = img['src'].rsplit('-article', 1)[0] + res + '.' + ext
|
||||
for c in soup.findAll('div', attrs={'class':'cap'}):
|
||||
for c in soup.findAll('div', attrs={'class': 'cap'}):
|
||||
for p in c.findAll(['p', 'div']):
|
||||
p.name = 'span'
|
||||
return soup
|
||||
|
||||
@ -88,7 +88,7 @@ class NewYorkTimes(BasicNewsRecipe):
|
||||
is_web_edition = True
|
||||
oldest_web_edition_article = 7 # days
|
||||
|
||||
extra_css = '''
|
||||
extra_css = """
|
||||
.byl, .time { font-size:small; color:#202020; }
|
||||
.cap { font-size:small; text-align:center; }
|
||||
.cred { font-style:italic; font-size:small; }
|
||||
@ -96,7 +96,7 @@ class NewYorkTimes(BasicNewsRecipe):
|
||||
.sc { font-variant: small-caps; }
|
||||
.lbl { font-size:small; color:#404040; }
|
||||
img { display:block; margin:0 auto; }
|
||||
'''
|
||||
"""
|
||||
|
||||
@property
|
||||
def nyt_parser(self):
|
||||
@ -113,12 +113,7 @@ class NewYorkTimes(BasicNewsRecipe):
|
||||
return self.index_to_soup(url, raw=True)
|
||||
|
||||
def preprocess_raw_html(self, raw_html, url):
|
||||
if '/interactive/' in url:
|
||||
return '<html><body><p><em>'\
|
||||
+ 'This is an interactive article, which is supposed to be read in a browser.'\
|
||||
+ '</p></em></body></html>'
|
||||
html = self.nyt_parser.extract_html(self.index_to_soup(raw_html))
|
||||
return html
|
||||
return self.nyt_parser.extract_html(self.index_to_soup(raw_html), url)
|
||||
|
||||
articles_are_obfuscated = use_wayback_machine
|
||||
|
||||
@ -144,8 +139,14 @@ class NewYorkTimes(BasicNewsRecipe):
|
||||
'long': 'For example, 2024/07/16'
|
||||
},
|
||||
'res': {
|
||||
'short': 'For hi-res images, select a resolution from the following\noptions: popup, jumbo, mobileMasterAt3x, superJumbo',
|
||||
'long': 'This is useful for non e-ink devices, and for a lower file size\nthan the default, use articleInline.',
|
||||
'short': (
|
||||
'For hi-res images, select a resolution from the following\noptions: '
|
||||
'popup, jumbo, mobileMasterAt3x, superJumbo'
|
||||
),
|
||||
'long': (
|
||||
'This is useful for non e-ink devices, and for a lower file size\nthan '
|
||||
'the default, use mediumThreeByTwo440, mediumThreeByTwo225, articleInline.'
|
||||
),
|
||||
},
|
||||
'comp': {
|
||||
'short': 'Compress News Images?',
|
||||
|
||||
@ -88,7 +88,7 @@ class NewYorkTimes(BasicNewsRecipe):
|
||||
is_web_edition = False
|
||||
oldest_web_edition_article = 7 # days
|
||||
|
||||
extra_css = '''
|
||||
extra_css = """
|
||||
.byl, .time { font-size:small; color:#202020; }
|
||||
.cap { font-size:small; text-align:center; }
|
||||
.cred { font-style:italic; font-size:small; }
|
||||
@ -96,7 +96,7 @@ class NewYorkTimes(BasicNewsRecipe):
|
||||
.sc { font-variant: small-caps; }
|
||||
.lbl { font-size:small; color:#404040; }
|
||||
img { display:block; margin:0 auto; }
|
||||
'''
|
||||
"""
|
||||
|
||||
@property
|
||||
def nyt_parser(self):
|
||||
@ -113,12 +113,7 @@ class NewYorkTimes(BasicNewsRecipe):
|
||||
return self.index_to_soup(url, raw=True)
|
||||
|
||||
def preprocess_raw_html(self, raw_html, url):
|
||||
if '/interactive/' in url:
|
||||
return '<html><body><p><em>'\
|
||||
+ 'This is an interactive article, which is supposed to be read in a browser.'\
|
||||
+ '</p></em></body></html>'
|
||||
html = self.nyt_parser.extract_html(self.index_to_soup(raw_html))
|
||||
return html
|
||||
return self.nyt_parser.extract_html(self.index_to_soup(raw_html), url)
|
||||
|
||||
articles_are_obfuscated = use_wayback_machine
|
||||
|
||||
@ -144,8 +139,14 @@ class NewYorkTimes(BasicNewsRecipe):
|
||||
'long': 'For example, 2024/07/16'
|
||||
},
|
||||
'res': {
|
||||
'short': 'For hi-res images, select a resolution from the following\noptions: popup, jumbo, mobileMasterAt3x, superJumbo',
|
||||
'long': 'This is useful for non e-ink devices, and for a lower file size\nthan the default, use articleInline.',
|
||||
'short': (
|
||||
'For hi-res images, select a resolution from the following\noptions: '
|
||||
'popup, jumbo, mobileMasterAt3x, superJumbo'
|
||||
),
|
||||
'long': (
|
||||
'This is useful for non e-ink devices, and for a lower file size\nthan '
|
||||
'the default, use mediumThreeByTwo440, mediumThreeByTwo225, articleInline.'
|
||||
),
|
||||
},
|
||||
'comp': {
|
||||
'short': 'Compress News Images?',
|
||||
|
||||
@ -1,23 +1,67 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
|
||||
from calibre.web.feeds.news import BasicNewsRecipe
|
||||
|
||||
|
||||
use_wayback_machine = False
|
||||
|
||||
|
||||
class NYTimesTechnology(BasicNewsRecipe):
|
||||
class NytTech(BasicNewsRecipe):
|
||||
title = 'New York Times Technology Beat'
|
||||
language = 'en_US'
|
||||
description = 'The latest in technology - Gadgetwise'
|
||||
__author__ = 'unkn0wn'
|
||||
oldest_article = 14 # days
|
||||
no_stylesheets = True
|
||||
no_javascript = True
|
||||
ignore_duplicate_articles = {'title', 'url'}
|
||||
description = 'The latest in technology - Gadgetwise'
|
||||
oldest_article = 14 # days
|
||||
encoding = 'utf-8'
|
||||
use_embedded_content = False
|
||||
language = 'en_US'
|
||||
remove_empty_feeds = True
|
||||
resolve_internal_links = True
|
||||
ignore_duplicate_articles = {'title', 'url'}
|
||||
masthead_url = 'https://static01.nytimes.com/newsgraphics/2015-12-23-masthead-2016/b15c3d81d3d7b59065fff9a3f3afe85aa2e2dff5/_assets/nyt-logo.png'
|
||||
|
||||
extra_css = '''
|
||||
feeds = [
|
||||
'https://rss.nytimes.com/services/xml/rss/nyt/Technology.xml',
|
||||
]
|
||||
|
||||
recipe_specific_options = {
|
||||
'days': {
|
||||
'short': 'Oldest article to download from this news source. In days ',
|
||||
'long': 'For example, 0.5, gives you articles from the past 12 hours',
|
||||
'default': str(oldest_article),
|
||||
},
|
||||
'comp': {'short': 'Compress News Images?', 'long': 'enter yes', 'default': 'no'},
|
||||
'rev': {
|
||||
'short': 'Reverse the order of articles in each feed?',
|
||||
'long': 'enter yes',
|
||||
'default': 'no',
|
||||
},
|
||||
'res': {
|
||||
'short': (
|
||||
'For hi-res images, select a resolution from the following\noptions: '
|
||||
'popup, jumbo, mobileMasterAt3x, superJumbo'
|
||||
),
|
||||
'long': (
|
||||
'This is useful for non e-ink devices, and for a lower file size\nthan '
|
||||
'the default, use mediumThreeByTwo440, mediumThreeByTwo225, articleInline.'
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
BasicNewsRecipe.__init__(self, *args, **kwargs)
|
||||
d = self.recipe_specific_options.get('days')
|
||||
if d and isinstance(d, str):
|
||||
self.oldest_article = float(d)
|
||||
r = self.recipe_specific_options.get('rev')
|
||||
if r and isinstance(r, str):
|
||||
if r.lower() == 'yes':
|
||||
self.reverse_article_order = True
|
||||
c = self.recipe_specific_options.get('comp')
|
||||
if c and isinstance(c, str):
|
||||
if c.lower() == 'yes':
|
||||
self.compress_news_images = True
|
||||
|
||||
extra_css = """
|
||||
.byl, .time { font-size:small; color:#202020; }
|
||||
.cap { font-size:small; text-align:center; }
|
||||
.cred { font-style:italic; font-size:small; }
|
||||
@ -25,76 +69,46 @@ class NYTimesTechnology(BasicNewsRecipe):
|
||||
.sc { font-variant: small-caps; }
|
||||
.lbl { font-size:small; color:#404040; }
|
||||
img { display:block; margin:0 auto; }
|
||||
'''
|
||||
|
||||
articles_are_obfuscated = use_wayback_machine
|
||||
|
||||
if use_wayback_machine:
|
||||
def get_obfuscated_article(self, url):
|
||||
from calibre.ptempfile import PersistentTemporaryFile
|
||||
with PersistentTemporaryFile() as tf:
|
||||
tf.write(self.get_nyt_page(url))
|
||||
return tf.name
|
||||
"""
|
||||
|
||||
@property
|
||||
def nyt_parser(self):
|
||||
ans = getattr(self, '_nyt_parser', None)
|
||||
if ans is None:
|
||||
from calibre.live import load_module
|
||||
|
||||
self._nyt_parser = ans = load_module('calibre.web.site_parsers.nytimes')
|
||||
return ans
|
||||
|
||||
def get_nyt_page(self, url, skip_wayback=False):
|
||||
if use_wayback_machine and not skip_wayback:
|
||||
from calibre import browser
|
||||
|
||||
return self.nyt_parser.download_url(url, browser())
|
||||
return self.index_to_soup(url, raw=True)
|
||||
|
||||
articles_are_obfuscated = use_wayback_machine
|
||||
|
||||
if use_wayback_machine:
|
||||
|
||||
def get_obfuscated_article(self, url):
|
||||
from calibre.ptempfile import PersistentTemporaryFile
|
||||
|
||||
with PersistentTemporaryFile() as tf:
|
||||
tf.write(self.get_nyt_page(url))
|
||||
return tf.name
|
||||
|
||||
def preprocess_raw_html(self, raw_html, url):
|
||||
if '/interactive/' in url:
|
||||
return '<html><body><p><em>'\
|
||||
+ 'This is an interactive article, which is supposed to be read in a browser.'\
|
||||
+ '</p></em></body></html>'
|
||||
html = self.nyt_parser.extract_html(self.index_to_soup(raw_html))
|
||||
return html
|
||||
|
||||
recipe_specific_options = {
|
||||
'days': {
|
||||
'short': 'Oldest article to download from this news source. In days ',
|
||||
'long': 'For example, 0.5, gives you articles from the past 12 hours',
|
||||
'default': str(oldest_article)
|
||||
},
|
||||
'res': {
|
||||
'short': 'For hi-res images, select a resolution from the following\noptions: popup, jumbo, mobileMasterAt3x, superJumbo',
|
||||
'long': 'This is useful for non e-ink devices, and for a lower file size\nthan the default, use articleInline.',
|
||||
},
|
||||
'comp': {
|
||||
'short': 'Compress News Images?',
|
||||
'long': 'enter yes',
|
||||
'default': 'no'
|
||||
}
|
||||
}
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
BasicNewsRecipe.__init__(self, *args, **kwargs)
|
||||
c = self.recipe_specific_options.get('comp')
|
||||
if c and isinstance(c, str):
|
||||
if c.lower() == 'yes':
|
||||
self.compress_news_images = True
|
||||
d = self.recipe_specific_options.get('days')
|
||||
if d and isinstance(d, str):
|
||||
self.oldest_article = float(d)
|
||||
|
||||
feeds = [
|
||||
(u'Gadgetwise', u'http://gadgetwise.blogs.nytimes.com/feed/'),
|
||||
]
|
||||
return self.nyt_parser.extract_html(self.index_to_soup(raw_html), url)
|
||||
|
||||
def get_browser(self, *args, **kwargs):
|
||||
kwargs['user_agent'] = 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
|
||||
kwargs['user_agent'] = (
|
||||
'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
|
||||
)
|
||||
br = BasicNewsRecipe.get_browser(self, *args, **kwargs)
|
||||
br.addheaders += [
|
||||
('Referer', 'https://www.google.com/'),
|
||||
('X-Forwarded-For', '66.249.66.1')
|
||||
('X-Forwarded-For', '66.249.66.1'),
|
||||
]
|
||||
return br
|
||||
|
||||
@ -102,11 +116,11 @@ class NYTimesTechnology(BasicNewsRecipe):
|
||||
w = self.recipe_specific_options.get('res')
|
||||
if w and isinstance(w, str):
|
||||
res = '-' + w
|
||||
for img in soup.findAll('img', attrs={'src':True}):
|
||||
for img in soup.findAll('img', attrs={'src': True}):
|
||||
if '-article' in img['src']:
|
||||
ext = img['src'].split('?')[0].split('.')[-1]
|
||||
img['src'] = img['src'].rsplit('-article', 1)[0] + res + '.' + ext
|
||||
for c in soup.findAll('div', attrs={'class':'cap'}):
|
||||
for c in soup.findAll('div', attrs={'class': 'cap'}):
|
||||
for p in c.findAll(['p', 'div']):
|
||||
p.name = 'span'
|
||||
return soup
|
||||
|
||||
@ -24,7 +24,7 @@ class NewYorkTimesBookReview(BasicNewsRecipe):
|
||||
ignore_duplicate_articles = {'title', 'url'}
|
||||
encoding = 'utf-8'
|
||||
|
||||
extra_css = '''
|
||||
extra_css = """
|
||||
.byl, .time { font-size:small; color:#202020; }
|
||||
.cap { font-size:small; text-align:center; }
|
||||
.cred { font-style:italic; font-size:small; }
|
||||
@ -32,7 +32,7 @@ class NewYorkTimesBookReview(BasicNewsRecipe):
|
||||
.sc { font-variant: small-caps; }
|
||||
.lbl { font-size:small; color:#404040; }
|
||||
img { display:block; margin:0 auto; }
|
||||
'''
|
||||
"""
|
||||
|
||||
articles_are_obfuscated = use_wayback_machine
|
||||
|
||||
@ -58,17 +58,18 @@ class NewYorkTimesBookReview(BasicNewsRecipe):
|
||||
return self.index_to_soup(url, raw=True)
|
||||
|
||||
def preprocess_raw_html(self, raw_html, url):
|
||||
if '/interactive/' in url:
|
||||
return '<html><body><p><em>'\
|
||||
+ 'This is an interactive article, which is supposed to be read in a browser.'\
|
||||
+ '</p></em></body></html>'
|
||||
html = self.nyt_parser.extract_html(self.index_to_soup(raw_html))
|
||||
return html
|
||||
return self.nyt_parser.extract_html(self.index_to_soup(raw_html), url)
|
||||
|
||||
recipe_specific_options = {
|
||||
'res': {
|
||||
'short': 'For hi-res images, select a resolution from the following\noptions: popup, jumbo, mobileMasterAt3x, superJumbo',
|
||||
'long': 'This is useful for non e-ink devices, and for a lower file size\nthan the default, use articleInline.',
|
||||
'short': (
|
||||
'For hi-res images, select a resolution from the following\noptions: '
|
||||
'popup, jumbo, mobileMasterAt3x, superJumbo'
|
||||
),
|
||||
'long': (
|
||||
'This is useful for non e-ink devices, and for a lower file size\nthan '
|
||||
'the default, use mediumThreeByTwo440, mediumThreeByTwo225, articleInline.'
|
||||
),
|
||||
},
|
||||
'comp': {
|
||||
'short': 'Compress News Images?',
|
||||
|
||||
@ -9,7 +9,7 @@ from xml.sax.saxutils import escape, quoteattr
|
||||
|
||||
from calibre.utils.iso8601 import parse_iso8601
|
||||
|
||||
module_version = 8 # needed for live updates
|
||||
module_version = 9 # needed for live updates
|
||||
pprint
|
||||
|
||||
|
||||
@ -27,6 +27,7 @@ def parse_image(i):
|
||||
yield f'<div class="cap">{i["legacyHtmlCaption"]}</div>'
|
||||
yield '</div>'
|
||||
|
||||
|
||||
def parse_img_grid(g):
|
||||
for grd in g.get('gridMedia', {}):
|
||||
yield ''.join(parse_image(grd))
|
||||
@ -36,16 +37,21 @@ def parse_img_grid(g):
|
||||
yield f'<span class="cred"> {g["credit"]}</span>'
|
||||
yield '</div>'
|
||||
|
||||
|
||||
def parse_vid(v):
|
||||
if v.get('promotionalMedia'):
|
||||
headline = v.get("headline", {}).get("default", "")
|
||||
headline = v.get('headline', {}).get('default', '')
|
||||
rendition = v.get('renditions')
|
||||
yield (f'<div><b><a href="{rendition[0]["url"]}">Video</a>: {headline}</b></div>'
|
||||
if rendition else f'<div><b>{headline}</b></div>')
|
||||
yield ''.join(parse_types(v["promotionalMedia"]))
|
||||
yield (
|
||||
f'<div><b><a href="{rendition[0]["url"]}">Video</a>: {headline}</b></div>'
|
||||
if rendition
|
||||
else f'<div><b>{headline}</b></div>'
|
||||
)
|
||||
yield ''.join(parse_types(v['promotionalMedia']))
|
||||
if v.get('promotionalSummary'):
|
||||
yield f'<div class="cap">{v["promotionalSummary"]}</div>'
|
||||
|
||||
|
||||
def parse_emb(e):
|
||||
if e.get('html') and 'datawrapper.dwcdn.net' in e.get('html', ''):
|
||||
dw = re.search(r'datawrapper.dwcdn.net/(.{5})', e['html']).group(1)
|
||||
@ -53,57 +59,62 @@ def parse_emb(e):
|
||||
elif e.get('promotionalMedia'):
|
||||
if e.get('headline'):
|
||||
yield f'<div><b>{e["headline"]["default"]}</b></div>'
|
||||
yield ''.join(parse_types(e["promotionalMedia"]))
|
||||
yield ''.join(parse_types(e['promotionalMedia']))
|
||||
if e.get('note'):
|
||||
yield f'<div class="cap">{e["note"]}</div>'
|
||||
|
||||
|
||||
def parse_byline(byl):
|
||||
for b in byl.get('bylines', {}):
|
||||
yield f'<div>{b["renderedRepresentation"]}</div>'
|
||||
yield '<div><b><i>'
|
||||
yield f'<div><b>{b["renderedRepresentation"]}</b></div>'
|
||||
yield '<div><i>'
|
||||
for rl in byl.get('role', {}):
|
||||
if ''.join(parse_cnt(rl)).strip():
|
||||
yield ''.join(parse_cnt(rl))
|
||||
yield '</i></b></div>'
|
||||
yield '</i></div>'
|
||||
|
||||
|
||||
def iso_date(x):
|
||||
dt = parse_iso8601(x, as_utc=False)
|
||||
return dt.strftime('%b %d, %Y at %I:%M %p')
|
||||
|
||||
|
||||
def parse_header(h):
|
||||
if h.get('label'):
|
||||
yield f'<div class="lbl">{"".join(parse_types(h["label"]))}</div>'
|
||||
if h.get('headline'):
|
||||
yield ''.join(parse_types(h["headline"]))
|
||||
yield ''.join(parse_types(h['headline']))
|
||||
if h.get('summary'):
|
||||
yield f'<p><i>{"".join(parse_types(h["summary"]))}</i></p>'
|
||||
if h.get('ledeMedia'):
|
||||
yield ''.join(parse_types(h["ledeMedia"]))
|
||||
yield ''.join(parse_types(h['ledeMedia']))
|
||||
if h.get('byline'):
|
||||
yield ''.join(parse_types(h["byline"]))
|
||||
yield ''.join(parse_types(h['byline']))
|
||||
if h.get('timestampBlock'):
|
||||
yield ''.join(parse_types(h["timestampBlock"]))
|
||||
yield ''.join(parse_types(h['timestampBlock']))
|
||||
|
||||
|
||||
def parse_fmt_type(fm):
|
||||
for f in fm.get('formats', {}):
|
||||
ftype = f.get("__typename", "")
|
||||
if ftype == "BoldFormat":
|
||||
ftype = f.get('__typename', '')
|
||||
if ftype == 'BoldFormat':
|
||||
yield '<strong>'
|
||||
if ftype == "ItalicFormat":
|
||||
if ftype == 'ItalicFormat':
|
||||
yield '<em>'
|
||||
if ftype == "LinkFormat":
|
||||
hrf = f["url"]
|
||||
if ftype == 'LinkFormat':
|
||||
hrf = f['url']
|
||||
yield f'<a href="{hrf}">'
|
||||
yield fm.get("text", "")
|
||||
yield fm.get('text', '')
|
||||
for f in reversed(fm.get('formats', {})):
|
||||
ftype = f.get("__typename", "")
|
||||
if ftype == "BoldFormat":
|
||||
ftype = f.get('__typename', '')
|
||||
if ftype == 'BoldFormat':
|
||||
yield '</strong>'
|
||||
if ftype == "ItalicFormat":
|
||||
if ftype == 'ItalicFormat':
|
||||
yield '</em>'
|
||||
if ftype == "LinkFormat":
|
||||
if ftype == 'LinkFormat':
|
||||
yield '</a>'
|
||||
|
||||
|
||||
def parse_cnt(cnt):
|
||||
for k in cnt:
|
||||
if isinstance(cnt[k], list):
|
||||
@ -118,17 +129,22 @@ def parse_cnt(cnt):
|
||||
if isinstance(cnt['text'], str):
|
||||
yield cnt['text']
|
||||
|
||||
|
||||
def parse_types(x):
|
||||
typename = x.get('__typename', '')
|
||||
|
||||
align = ''
|
||||
if x.get('textAlign'):
|
||||
align = f' style="text-align: {x["textAlign"].lower()};"'
|
||||
|
||||
if 'Header' in typename:
|
||||
yield '\n'.join(parse_header(x))
|
||||
|
||||
elif typename.startswith('Heading'):
|
||||
htag = 'h' + re.match(r'Heading([1-6])Block', typename).group(1)
|
||||
yield f'<{htag}>{"".join(parse_cnt(x))}</{htag}>'
|
||||
yield f'<{htag}{align}>{"".join(parse_cnt(x))}</{htag}>'
|
||||
|
||||
elif typename == 'ParagraphBlock':
|
||||
elif typename in {'ParagraphBlock', 'DetailBlock', 'TextRunKV'}:
|
||||
yield f'<p>{"".join(parse_cnt(x))}</p>'
|
||||
|
||||
elif typename == 'BylineBlock':
|
||||
@ -145,16 +161,16 @@ def parse_types(x):
|
||||
yield '<hr/>'
|
||||
|
||||
elif typename == 'Image':
|
||||
yield "".join(parse_image(x))
|
||||
yield ''.join(parse_image(x))
|
||||
|
||||
elif typename == 'GridBlock':
|
||||
yield "".join(parse_img_grid(x))
|
||||
yield ''.join(parse_img_grid(x))
|
||||
|
||||
elif typename == 'Video':
|
||||
yield "".join(parse_vid(x))
|
||||
yield ''.join(parse_vid(x))
|
||||
|
||||
elif typename == 'EmbeddedInteractive':
|
||||
yield "".join(parse_emb(x))
|
||||
yield ''.join(parse_emb(x))
|
||||
|
||||
elif typename == 'ListBlock':
|
||||
yield f'<ul>{"".join(parse_cnt(x))}</ul>'
|
||||
@ -162,20 +178,22 @@ def parse_types(x):
|
||||
yield f'\n<li>{"".join(parse_cnt(x))}</li>'
|
||||
|
||||
elif typename == 'TextInline':
|
||||
yield "".join(parse_cnt(x))
|
||||
yield ''.join(parse_cnt(x))
|
||||
|
||||
elif typename in {'DetailBlock', 'TextRunKV'}:
|
||||
yield f'<p><i>{"".join(parse_cnt(x))}</i></p>'
|
||||
elif typename and typename not in {
|
||||
'RelatedLinksBlock',
|
||||
'EmailSignupBlock',
|
||||
'Dropzone',
|
||||
}:
|
||||
if ''.join(parse_cnt(x)).strip():
|
||||
yield ''.join(parse_cnt(x))
|
||||
|
||||
elif typename and typename not in {'RelatedLinksBlock', 'Dropzone'}:
|
||||
if "".join(parse_cnt(x)).strip():
|
||||
yield "".join(parse_cnt(x))
|
||||
|
||||
def article_parse(data):
|
||||
yield "<html><body>"
|
||||
yield '<html><body>'
|
||||
for d in data:
|
||||
yield from parse_types(d)
|
||||
yield "</body></html>"
|
||||
yield '</body></html>'
|
||||
|
||||
|
||||
def json_to_html(raw):
|
||||
@ -232,7 +250,7 @@ def add_live_item(item, item_type, lines):
|
||||
|
||||
|
||||
def live_json_to_html(data):
|
||||
for k, v in data["ROOT_QUERY"].items():
|
||||
for k, v in data['ROOT_QUERY'].items():
|
||||
if isinstance(v, dict) and 'id' in v:
|
||||
root = data[v['id']]
|
||||
s = data[root['storylines'][0]['id']]
|
||||
@ -249,24 +267,32 @@ def live_json_to_html(data):
|
||||
return '<html><body>' + '\n'.join(lines) + '</body></html>'
|
||||
|
||||
|
||||
def extract_html(soup):
|
||||
def extract_html(soup, url):
|
||||
if '/interactive/' in url:
|
||||
return (
|
||||
'<html><body><p><em>'
|
||||
+ 'This is an interactive article, which is supposed to be read in a browser.'
|
||||
+ '</p></em></body></html>'
|
||||
)
|
||||
script = soup.findAll('script', text=lambda x: x and 'window.__preloadedData' in x)[0]
|
||||
script = str(script)
|
||||
raw = script[script.find('{'):script.rfind(';')].strip().rstrip(';')
|
||||
raw = script[script.find('{') : script.rfind(';')].strip().rstrip(';')
|
||||
return json_to_html(raw)
|
||||
|
||||
|
||||
def download_url_from_wayback(category, url, br=None):
|
||||
from mechanize import Request
|
||||
|
||||
host = 'http://localhost:8090'
|
||||
host = 'https://wayback1.calibre-ebook.com'
|
||||
rq = Request(
|
||||
host + '/' + category,
|
||||
data=json.dumps({"url": url}),
|
||||
headers={'User-Agent': 'calibre', 'Content-Type': 'application/json'}
|
||||
data=json.dumps({'url': url}),
|
||||
headers={'User-Agent': 'calibre', 'Content-Type': 'application/json'},
|
||||
)
|
||||
if br is None:
|
||||
from calibre import browser
|
||||
|
||||
br = browser()
|
||||
br.set_handle_gzip(True)
|
||||
return br.open_novisit(rq, timeout=3 * 60).read()
|
||||
@ -284,6 +310,7 @@ if __name__ == '__main__':
|
||||
raw = open(f).read()
|
||||
if f.endswith('.html'):
|
||||
from calibre.ebooks.BeautifulSoup import BeautifulSoup
|
||||
|
||||
soup = BeautifulSoup(raw)
|
||||
print(extract_html(soup))
|
||||
else:
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user