Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/unkn0w7n/calibre

Browse Source
This commit is contained in:
Kovid Goyal 2024-05-15 11:20:48 +05:30
commit f2abea9be5
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
1 changed files with 49 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
recipes/wsj.recipe
View File

@ -8,6 +8,11 @@ from calibre.web.feeds.news import BasicNewsRecipe, classes
from html5_parser import parse
# Past 6 editions are available for download.
# For available past editions see log and set date to, for example, '20240513'.
past_edition = None
class WSJ(BasicNewsRecipe):
title = 'The Wall Street Journal'
__author__ = 'unkn0wn'
@ -21,6 +26,7 @@ class WSJ(BasicNewsRecipe):
no_javascript = True
no_stylesheets = True
remove_attributes = ['style', 'height', 'width']
resolve_internal_links = True
extra_css = '''
#subhed, em { font-style:italic; color:#202020; }
@ -81,21 +87,22 @@ class WSJ(BasicNewsRecipe):
div.extract()
return soup
def _download_cover(self):
import os
from calibre import browser
from contextlib import closing
from calibre.utils.img import add_borders_to_image, image_to_data, save_cover_data_to
br = browser()
raw = br.open('https://www.frontpages.com/the-wall-street-journal/')
soup = BeautifulSoup(raw.read())
cu = 'https://www.frontpages.com' + soup.find('img', attrs={'id':'giornale-img'})['src']
self.report_progress(1, _('Downloading cover from %s')%cu)
with closing(br.open(cu, timeout=self.timeout)) as r:
cdata = r.read()
cpath = os.path.join(self.output_dir, 'cover.jpg')
save_cover_data_to(cdata, cpath)
self.cover_path = cpath
if not past_edition:
def _download_cover(self):
import os
from calibre import browser
from contextlib import closing
from calibre.utils.img import save_cover_data_to
br = browser()
raw = br.open('https://www.frontpages.com/the-wall-street-journal/')
soup = BeautifulSoup(raw.read())
cu = 'https://www.frontpages.com' + soup.find('img', attrs={'id':'giornale-img'})['src']
self.report_progress(1, _('Downloading cover from %s')%cu)
with closing(br.open(cu, timeout=self.timeout)) as r:
cdata = r.read()
cpath = os.path.join(self.output_dir, 'cover.jpg')
save_cover_data_to(cdata, cpath)
self.cover_path = cpath
def get_browser(self, *args, **kw):
kw['user_agent'] = 'okhttp/4.10.0'
@ -110,13 +117,27 @@ class WSJ(BasicNewsRecipe):
def parse_index(self):
index = 'https://bartender.mobile.dowjones.io'
catalog = json.loads(self.index_to_soup(index + '/catalogs/v1/wsj/us/catalog.json', raw=True))
edit = []
for itm in catalog['items']:
if itm['type'] == 'ITP':
edit.append(itm['key'][3:])
self.log('**Past Editions available :', ', '.join(edit))
for itm in catalog['items']:
if past_edition:
if itm['key'] == 'ITP' + past_edition:
key = itm['key']
manifest = itm['manifest']
dt = datetime.fromisoformat(itm['date'][:-1]) + timedelta(seconds=time.timezone)
dt = dt.strftime('%b %d, %Y')
self.log('Downloading Past Edition ', dt)
self.timefmt = ' [' + dt + ']'
break
elif itm['type'] == 'ITP':
key = itm['key']
manifest = itm['manifest']
dt = datetime.fromisoformat(itm['date'][:-1]) + timedelta(seconds=time.timezone)
dt = dt.strftime('%b %d, %Y')
self.log(dt)
self.log('Downloading ', dt)
self.timefmt = ' [' + dt + ']'
break
@ -151,9 +172,20 @@ class WSJ(BasicNewsRecipe):
for x in root.xpath('//image'):
x.tag = 'img'
return BeautifulSoup(raw).prettify()
return raw
else:
soup = BeautifulSoup(raw)
url = soup.find('meta', attrs={'property':'og:url'})
if url:
h1 = soup.find('h1')
if h1:
h1['title'] = url['content']
return soup.prettify()
def populate_article_metadata(self, article, soup, first):
lnk = soup.find('p', attrs={'id':'share-link'})
if lnk:
article.url = lnk['title']
art = soup.find('h1', attrs={'title':True})
if art:
self.log('found art ', art['title'])
article.url = art['title']