From c7d258252d05518ca33ea4bf4dcf15a3c63850ba Mon Sep 17 00:00:00 2001 From: unkn0w7n <51942695+unkn0w7n@users.noreply.github.com> Date: Sat, 4 Oct 2025 08:59:03 +0530 Subject: [PATCH 1/2] Update wsj_free.recipe --- recipes/wsj_free.recipe | 423 ++++++++++++++++------------------------ 1 file changed, 170 insertions(+), 253 deletions(-) diff --git a/recipes/wsj_free.recipe b/recipes/wsj_free.recipe index db05c4b22a..80844c639f 100644 --- a/recipes/wsj_free.recipe +++ b/recipes/wsj_free.recipe @@ -1,57 +1,39 @@ #!/usr/bin/env python # vim:fileencoding=utf-8 -# License: GPLv3 Copyright: 2016, Kovid Goyal - -from __future__ import absolute_import, division, print_function, unicode_literals - import json -import random import time -from base64 import standard_b64encode -from datetime import date, timedelta +from urllib.parse import quote, urlencode -from css_selectors import Select -from mechanize import Request - -from calibre.ptempfile import PersistentTemporaryFile from calibre.web.feeds.news import BasicNewsRecipe -try: - import urllib.parse as urlparse -except ImportError: - import urlparse -try: - from urllib.parse import quote -except ImportError: - from urllib import quote - - -needs_subscription = False - class WSJ(BasicNewsRecipe): - - if needs_subscription: - title = 'The Wall Street Journal' - else: - title = 'The Wall Street Journal (free)' - __author__ = 'Kovid Goyal' - description = 'Old code. Use WSJ recipes by unkn0wn' + title = 'The Wall Street Journal' + __author__ = 'unkn0wn' + description = ( + 'The Print Edition of WSJ. The Wall Street Journal is your source ' + 'for breaking news, analysis and insights from the U.S. and ' + "around the world, the world's leading business and finance publication." + ) language = 'en_US' masthead_url = 'https://s.wsj.net/media/wsj_amp_masthead_lg.png' - - compress_news_images = True - compress_news_images_auto_size = 7 - - timefmt = ' [%a, %b %d, %Y]' + encoding = 'utf-8' + no_javascript = True no_stylesheets = True - ignore_duplicate_articles = {'url'} - remove_attributes = ['style','height','width'] - needs_subscription = needs_subscription - WSJ_ITP = 'https://www.wsj.com/print-edition/today' - delay = 1 + remove_attributes = ['style', 'height', 'width'] + resolve_internal_links = True - storage = [] + recipe_specific_options = { + 'date': { + 'short': 'The date of the edition to download (YYYY-MM-DD format)\nOnly the past 6 editions will be available ', + 'long': 'For example, 2024-05-13', + }, + 'res': { + 'short': 'For hi-res images, select a resolution from the\nfollowing options: 800, 1000, 1200 or 1500', + 'long': 'This is useful for non e-ink devices, and for a lower file size\nthan the default, use 400 or 300.', + 'default': '600', + }, + } extra_css = ''' #big-top-caption { font-size:small; text-align:center; } @@ -60,70 +42,89 @@ class WSJ(BasicNewsRecipe): .sub, em, i { color: #202020; } ''' - def get_cover_url(self): - soup = self.index_to_soup('https://www.frontpages.com/the-wall-street-journal/') - return 'https://www.frontpages.com' + soup.find('img', attrs={'id':'giornale-img'})['src'] - keep_only_tags = [ dict(name=['h1', 'h2']), - dict(attrs={'aria-describedby':'big-top-caption'}), - dict(attrs={'id':'big-top-caption'}), - dict(name='article', attrs={'style':lambda x: x and 'article-body' in x}) + dict(attrs={'aria-describedby': 'big-top-caption'}), + dict(attrs={'id': 'big-top-caption'}), + dict(name='article', attrs={'style': lambda x: x and 'article-body' in x}), ] remove_tags = [ - dict(attrs={'data-type':['inset', 'video']}), - dict(attrs={'data-testid':'ad-container'}), - dict(attrs={'data-spotim-app':'conversation'}), + dict(attrs={'data-type': ['inset', 'video']}), + dict(attrs={'data-testid': 'ad-container'}), + dict(attrs={'data-spotim-app': 'conversation'}), dict(name=['button', 'svg', 'old-script', 'video']), - dict(attrs={'aria-label':[ - 'Sponsored Offers', 'Listen To Article', 'What to Read Next', 'Utility Bar', - 'Conversation', 'List of Comments', 'Comment', 'JR More Articles' - ]}), - dict(attrs={'data-spot-im-class':['message-text', 'messages-list', 'message-view', 'conversation-root']}), - dict(attrs={'id':lambda x: x and x.startswith( - ('comments_sector', 'wrapper-INLINE', 'audio-tag-inner-audio-', 'article-comments-tool') - )}), - dict(name='div', attrs={'data-message-depth':True}) + dict( + attrs={ + 'aria-label': [ + 'Sponsored Offers', + 'Listen To Article', + 'What to Read Next', + 'Utility Bar', + 'Conversation', + 'List of Comments', + 'Comment', + 'JR More Articles', + ] + } + ), + dict( + attrs={ + 'data-spot-im-class': [ + 'message-text', + 'messages-list', + 'message-view', + 'conversation-root', + ] + } + ), + dict( + attrs={ + 'id': lambda x: x + and x.startswith(( + 'comments_sector', + 'wrapper-INLINE', + 'audio-tag-inner-audio-', + 'article-comments-tool', + )) + } + ), + dict(name='div', attrs={'data-message-depth': True}), ] articles_are_obfuscated = True + def get_obfuscated_article(self, url): from calibre.scraper.simple import read_url - br = self.get_browser() - br.set_handle_redirect(False) - try: - br.open(url) - except Exception as e: - hdrs_location = e.hdrs.get('location') - if hdrs_location: - url = e.hdrs.get('location') - raw = read_url(self.storage, 'https://archive.is/latest/' + url) - pt = PersistentTemporaryFile('.html') - pt.write(raw.encode('utf-8')) - pt.close() - return pt.name + raw = read_url([], 'https://archive.is/latest/' + url) + return {'data': raw, 'url': url} def preprocess_html(self, soup): - for img in soup.findAll('img', attrs={'old-src':True}): - img['src'] = img['old-src'] - for p in soup.findAll('div', attrs={'data-type':['paragraph', 'image']}): + res = '?width=600' + w = self.recipe_specific_options.get('res') + if w and isinstance(w, str): + res = '?width=' + w + for img in soup.findAll('img', attrs={'currentsourceurl': True}): + img['src'] = img['currentsourceurl'].split('?')[0] + res + for p in soup.findAll('div', attrs={'data-type': ['paragraph', 'image']}): p.name = 'p' for a in soup.findAll('a', href=True): a['href'] = 'http' + a['href'].split('http')[-1] for figc in soup.findAll('figcaption'): figc['id'] = 'big-top-caption' - if name:= soup.find('h2', attrs={'itemprop':'name'}): + if name := soup.find('h2', attrs={'itemprop': 'name'}): name.extract() for h2 in soup.findAll('h2'): if self.tag_to_string(h2).startswith(('What to Read Next', 'Conversation')): h2.extract() h2.name = 'h3' h2['class'] = 'sub' - for ph in soup.findAll('a', attrs={'data-type':['phrase', 'link']}): + for ph in soup.findAll('a', attrs={'data-type': ['phrase', 'link']}): if div := ph.findParent('div'): div.name = 'span' - for auth in soup.findAll('a', attrs={'aria-label': lambda x: x and x.startswith('Author page')}): + for auth in soup.findAll( + 'a', attrs={'aria-label': lambda x: x and x.startswith('Author page')} + ): if div := auth.find_previous_sibling('div'): div.name = 'span' if parent := auth.findParent('div'): @@ -133,186 +134,102 @@ class WSJ(BasicNewsRecipe): y.extract() return soup - # login {{{ - - def get_browser_for_wsj(self, *a, **kw): - br = BasicNewsRecipe.get_browser(self, *a, **kw) - br.set_cookie('wsjregion', 'na,us', '.wsj.com') - br.set_cookie('gdprApplies', 'false', '.wsj.com') - br.set_cookie('ccpaApplies', 'false', '.wsj.com') + def get_browser(self, *args, **kw): + br = BasicNewsRecipe.get_browser(self, *args, **kw) + br.addheaders += [ + ('apollographql-client-name', 'wsj-mobile-android-release'), + ] return br - if False and needs_subscription: # disabled as we currently use archive.is - def get_browser(self, *a, **kw): - from pprint import pprint - pprint - # To understand the login logic read app-min.js from - # https://sso.accounts.dowjones.com/login - itp = quote(self.WSJ_ITP, safe='') - start_url = 'https://accounts.wsj.com/login?target=' + itp - self.log('Starting login process at', start_url) - br = self.get_browser_for_wsj(*a, **kw) - # br.set_debug_http(True) - res = br.open(start_url) - sso_url = res.geturl() - query = urlparse.parse_qs(urlparse.urlparse(sso_url).query) - query = {k:v[0] for k, v in query.items()} - # pprint(query) - request_query = { - 'username': self.username, - 'password': self.password, - 'client_id': query['client'], - 'tenant': 'sso', - '_intstate': 'deprecated', - 'connection': 'DJldap', - 'headers': { - 'X-REMOTE-USER': self.username, - 'x-_dj-_client__id': query['client'], - }, - } - for cookie in br.cookiejar: - if cookie.name in ('_csrf', 'csrf'): - request_query['_csrf'] = cookie.value - for k in 'scope connection nonce state ui_locales ns mars protocol redirect_uri'.split(): - if k in query: - request_query[k] = query[k] - # pprint(request_query) - login_url = 'https://sso.accounts.dowjones.com/usernamepassword/login' - # you can get the version below from lib-min.js - # search for: "\d+\.\d+\.\d+" - # This might need to be updated in the future - auth0_client = json.dumps({'name': 'auth0.js-ulp', 'version': '9.11.3'}) - if not isinstance(auth0_client, bytes): - auth0_client = auth0_client.encode('utf-8') - auth0_client = standard_b64encode(auth0_client) - if isinstance(auth0_client, bytes): - auth0_client = auth0_client.decode('ascii') - rq = Request(login_url, headers={ - 'Accept': 'text/html', - 'Accept-Language': 'en-US,en;q=0.8', - 'Origin': 'https://sso.accounts.dowjones.com', - 'Auth0-Client': auth0_client.rstrip('='), - 'X-HTTP-Method-Override': 'POST', - 'X-Requested-With': 'XMLHttpRequest', - 'X-Remote-User': self.username, - 'x-dj-client_id': request_query['client_id'], - }, data=request_query) - self.log('Sending login request...') - try: - res = br.open(rq) - except Exception as err: - if hasattr(err, 'read'): - raise Exception('Login request failed with error: {} and body: {}'.format(err, err.read().decode('utf-8', 'replace'))) - raise - if res.code != 200: - raise ValueError('Failed to login, check your username and password') - br.select_form(nr=0) - self.log('Performing login callback...') - res = br.submit() - self.log('Print edition resolved url:', res.geturl()) - self.wsj_itp_page = raw = res.read() - if b'/logout' not in raw: - raise ValueError( - 'Failed to login (callback URL failed), check username and password') - return br - else: - def get_browser(self, *a, **kw): - br = self.get_browser_for_wsj(*a, **kw) - res = br.open(self.WSJ_ITP) - url = res.geturl() - if '/20210913/' in url: - today = date.today() - q = today.isoformat().replace('-', '') - try: - res = br.open(url.replace('/20210913/', '/' + q + '/')) - except Exception: - today -= timedelta(days=1) - q = today.isoformat().replace('-', '') - res = br.open(url.replace('/20210913/', '/' + q + '/')) - self.log('Print edition resolved url:', res.geturl()) - self.wsj_itp_page = res.read() - return br - # }}} + def _download_cover(self): + import os + from contextlib import closing - def abs_wsj_url(self, href, modify_query=True): - if not href.startswith('http'): - href = 'https://www.wsj.com' + href - if modify_query: - href = href - return href.split('?')[0] + from calibre import browser + from calibre.utils.img import save_cover_data_to - def wsj_find_articles(self, url, ahed=False): - root = self.index_to_soup(url, as_tree=True) - CSSSelect = Select(root) - articles = [] - for container in root.xpath('descendant::div[contains(@class, "WSJTheme--list-item--")]'): - heading = next(CSSSelect('h2, h3', container)) - a = next(CSSSelect('a', heading)) - title = self.tag_to_string(a) - url = self.abs_wsj_url(a.get('href')) - desc = '' - for p in container.xpath('descendant::p[contains(@class, "WSJTheme--description--")]'): - q = self.tag_to_string(p) - if 'Subscriber Content' in q: - continue - desc += q - break - - articles.append({'title': title, 'url': url, - 'description': desc, 'date': ''}) - self.log('\tFound article:', title) - self.log('\t\t', desc + ' ' + url) - if self.test and len(articles) >= self.test[1]: - break - - return articles - - def wsj_add_feed(self, feeds, title, url): - try: - for i in range(5): - articles = self.wsj_find_articles(url) - if articles: - break - else: - pause = random.choice((1, 1.5, 2, 2.5)) - self.log.warn('No articles found in', url, 'retrying after', pause, 'seconds') - time.sleep(pause) - except Exception: - self.log.exception('Failed to parse section:', title) - articles = [] - if articles: - feeds.append((title, articles)) - else: - self.log.warn('No articles found in', url) + br = browser() + raw = br.open( + 'https://frontpages.freedomforum.org/newspapers/wsj-The_Wall_Street_Journal' + ) + soup = BeautifulSoup(raw.read()) + cu = soup.find( + 'img', + attrs={ + 'alt': 'Front Page Image', + 'src': lambda x: x and x.endswith('front-page-large.jpg'), + }, + )['src'].replace('-large', '-medium') + self.report_progress(1, _('Downloading cover from %s') % cu) + with closing(br.open(cu, timeout=self.timeout)) as r: + cdata = r.read() + cpath = os.path.join(self.output_dir, 'cover.jpg') + save_cover_data_to(cdata, cpath) + self.cover_path = cpath def parse_index(self): - # return self.test_wsj_index() - root = self.index_to_soup(self.wsj_itp_page, as_tree=True) - CSSSelect = Select(root) - # from calibre.utils.ipython import ipython - # ipython({'root': root, 'CSSSelect': CSSSelect, 'raw': self.wsj_itp_page}) - for inp in CSSSelect('.DayPickerInput > input'): - if inp.get('placeholder'): - self.timefmt = inp.get('placeholder') - break + query = { + 'operationName': 'IssueQuery', + 'variables': '{"publication":"WSJ","region":"US","masthead":"ITPNEXTGEN"}', + 'extensions': '{"persistedQuery":{"version":1,"sha256Hash":"d938226e7d1c1fff050e7d084c72179e2713dcf4736d3a442c618c55b896f847"}}', + } + url = 'https://shared-data.dowjones.io/gateway/graphql?' + urlencode( + query, safe='()!', quote_via=quote + ) + raw = self.index_to_soup(url, raw=True) + + cat_data = json.loads(raw)['data']['mobileIssuesByMasthead'] + edit = [x['datedLabel'] for x in cat_data][1:] + self.log('**Past Editions available : ' + ' | '.join(edit)) + + past_edition = self.recipe_specific_options.get('date') + + for itm in cat_data: + if past_edition and isinstance(past_edition, str): + if past_edition in itm['publishedDateUtc']: + self.timefmt = ' [' + itm['datedLabel'] + sections_ = itm['sections'] + break + self.timefmt = f' [{itm["datedLabel"]}]' + sections_ = itm['sections'] + break + + self.log('Downloading ', self.timefmt) feeds = [] - for container in root.xpath('descendant::*[contains(@class, "WSJTheme--top-menu-item--")]'): - for a in container.xpath('descendant::a[contains(@class, "WSJTheme--section-link--")]'): - title = self.tag_to_string(a).capitalize().strip().replace('U.s.', 'U.S.') - if not title: - continue - url = self.abs_wsj_url(a.get('href'), modify_query=False) - self.log('Found section:', title, 'at', url) - self.wsj_add_feed(feeds, title, url) - if self.test and len(feeds) >= self.test[0]: - break - return feeds - def test_wsj_index(self): - return [ - ('Testing', [ - {'title': 'Subscriber Article', - 'url': self.abs_wsj_url('https://www.wsj.com/articles/remington-gun-call-of-duty-video-game-93059a66')}, - ]), - ] + for sec in sections_[:-1]: + time.sleep(3) + section = sec['label'] + self.log(section) + cont_id = sec['key'] + + query = { + 'operationName': 'SectionQuery', + 'variables': '{{"id":"{}"}}'.format(cont_id), + 'extensions': '{"persistedQuery":{"version":1,"sha256Hash":"207fe93376f379bf223ed2734cf9313a28291293366a803db923666fa6b45026"}}', + } + sec_url = 'https://shared-data.dowjones.io/gateway/graphql?' + urlencode( + query, safe='()!', quote_via=quote + ) + sec_raw = self.index_to_soup(sec_url, raw=True) + + sec_data = json.loads(sec_raw)['data']['summaryCollectionContent'][ + 'collectionItems' + ] + + articles = [] + + for art in sec_data: + for arts in art['collectionItems']: + mobi = arts['content']['mobileSummary'] + title = mobi['headline']['text'] + try: + desc = mobi['description']['content']['text'] + except TypeError: + desc = '' + url = arts['content']['sourceUrl'] + self.log(' ', title, '\n\t', desc, '\n\t', url) + articles.append({'title': title, 'description': desc, 'url': url}) + feeds.append((section, articles)) + return feeds From c7cbe5a230ce962592472c456cde0e2ba6355dd9 Mon Sep 17 00:00:00 2001 From: unkn0w7n <51942695+unkn0w7n@users.noreply.github.com> Date: Sat, 4 Oct 2025 09:03:56 +0530 Subject: [PATCH 2/2] Update WSJ redirect to archive --- recipes/wsj.recipe | 229 ++++++++++++++++++--------------------------- 1 file changed, 90 insertions(+), 139 deletions(-) diff --git a/recipes/wsj.recipe b/recipes/wsj.recipe index 498175ca97..829bcbe9ab 100644 --- a/recipes/wsj.recipe +++ b/recipes/wsj.recipe @@ -2,31 +2,9 @@ # vim:fileencoding=utf-8 import json import time -from itertools import zip_longest from urllib.parse import quote, urlencode -from calibre import browser -from calibre.ebooks.BeautifulSoup import BeautifulSoup -from calibre.ptempfile import PersistentTemporaryFile -from calibre.web.feeds.news import BasicNewsRecipe, classes - - -def get_article(article_id): - from mechanize import Request - mat_url = 'https://mats.mobile.dowjones.io/translate/' + article_id + '/jpml' - headers = { - 'User-Agent': 'okhttp/4.10.0', - 'Accept-Encoding': 'gzip', - 'Cache-Control': 'no-cache', - 'x-api-key': ('e''0''5''9''9''5''f''f''4''4''2''1''4''3''2''5''5''e''b''8''3''8''1''f''7''2''d''4''9''1''3''b''f''7''5''0''3''d''6''c'), # noqa: ISC001 - } - br = browser() - req = Request( - mat_url, - headers=headers, - ) - res = br.open(req) - return res.read() +from calibre.web.feeds.news import BasicNewsRecipe class WSJ(BasicNewsRecipe): @@ -44,7 +22,6 @@ class WSJ(BasicNewsRecipe): no_stylesheets = True remove_attributes = ['style', 'height', 'width'] resolve_internal_links = True - simultaneous_downloads = 20 recipe_specific_options = { 'date': { @@ -59,114 +36,109 @@ class WSJ(BasicNewsRecipe): } extra_css = ''' - #subhed, em { font-style:italic; color:#202020; } - #byline, #time-to-read, #orig-pubdate-string, .article-byline, time, #flashline { font-size:small; } - .figc { font-size:small; text-align:center; } - img {display:block; margin:0 auto;} + #big-top-caption { font-size:small; text-align:center; } + [data-type:"tagline"] { font-style:italic; color:#202020; } + .auth, time { font-size:small; } + .sub, em, i { color: #202020; } ''' - remove_tags = [ - dict(name='panel', attrs={'id': 'summary-image'}), - dict(name='panel', attrs={'layout': 'inline'}), - dict(name='panel', attrs={'embed': 'inner-article-ad'}), - dict(name='span', attrs={'embed': 'ticker'}), - classes('lamrelated-articles-inset-panel'), - dict( - name='p', - attrs={ - 'id': [ - 'keywords', - 'orig-pubdate-number', - 'type', - 'is-custom-flashline', - 'grouphed', - 'author-ids', - 'article-manifest', - 'body-extract', - 'category', - 'sub-category', - 'socialhed', - 'summary', - 'deckline', - 'article-flashline', - ] - }, - ), + keep_only_tags = [ + dict(name=['h1', 'h2']), + dict(attrs={'aria-describedby': 'big-top-caption'}), + dict(attrs={'id': 'big-top-caption'}), + dict(name='article', attrs={'style': lambda x: x and 'article-body' in x}), ] - remove_tags_before = [dict(name='p', attrs={'id': 'orig-pubdate-string'})] + remove_tags = [ + dict(attrs={'data-type': ['inset', 'video']}), + dict(attrs={'data-testid': 'ad-container'}), + dict(attrs={'data-spotim-app': 'conversation'}), + dict(name=['button', 'svg', 'old-script', 'video']), + dict( + attrs={ + 'aria-label': [ + 'Sponsored Offers', + 'Listen To Article', + 'What to Read Next', + 'Utility Bar', + 'Conversation', + 'List of Comments', + 'Comment', + 'JR More Articles', + ] + } + ), + dict( + attrs={ + 'data-spot-im-class': [ + 'message-text', + 'messages-list', + 'message-view', + 'conversation-root', + ] + } + ), + dict( + attrs={ + 'id': lambda x: x + and x.startswith(( + 'comments_sector', + 'wrapper-INLINE', + 'audio-tag-inner-audio-', + 'article-comments-tool', + )) + } + ), + dict(name='div', attrs={'data-message-depth': True}), + ] - def media_bucket(self, x): + articles_are_obfuscated = True + + def get_obfuscated_article(self, url): + from calibre.scraper.simple import read_url + raw = read_url([], 'https://archive.is/latest/' + url) + return {'data': raw, 'url': url} + + def preprocess_html(self, soup): res = '?width=600' w = self.recipe_specific_options.get('res') if w and isinstance(w, str): res = '?width=' + w - if x.get('type', '') == 'image': - if ( - x.get('subtype', '') == 'graphic' - or 'images.wsj.net' not in x['manifest-url'] - ): - return '
{}
\n'.format( - x['manifest-url'], x['caption'] + ' ' + x['credit'] + '' - ) - return '
{}
\n'.format( - x['manifest-url'].split('?')[0] + res, - x['caption'] + ' ' + x['credit'] + '', - ) - if x.get('type', '') == 'video': - return ( - '
{}
\n'.format( - x['share_link'], - x['thumbnail_url'].split('?')[0] + res, - x['caption'] + ' ' + x['credit'] + '', - ) - ) - return - - def preprocess_html(self, soup): - jpml = soup.find('jpml') - if jpml: - jpml.name = 'article' - h1 = soup.find('p', attrs={'id': 'headline'}) - if h1: - h1.name = 'h1' + for img in soup.findAll('img', attrs={'currentsourceurl': True}): + img['src'] = img['currentsourceurl'].split('?')[0] + res + for p in soup.findAll('div', attrs={'data-type': ['paragraph', 'image']}): + p.name = 'p' + for a in soup.findAll('a', href=True): + a['href'] = 'http' + a['href'].split('http')[-1] + for figc in soup.findAll('figcaption'): + figc['id'] = 'big-top-caption' + if name := soup.find('h2', attrs={'itemprop': 'name'}): + name.extract() for h2 in soup.findAll('h2'): - h2.name = 'h4' - dt = soup.find('p', attrs={'id': 'orig-pubdate-string'}) - read = soup.find('p', attrs={'id': 'time-to-read'}) - byl = soup.find('p', attrs={'id': 'byline'}) - fl = soup.find('p', attrs={'id': 'flashline'}) - if dt and byl and read and fl: - dt.name = read.name = byl.name = fl.name = 'div' - byl.insert(0, dt) - byl.insert(0, read) - url = soup.find('p', attrs={'id': 'share-link'}) - if url: - url.name = 'div' - url['title'] = self.tag_to_string(url).strip() - url.string = '' - panel = soup.find('panel', attrs={'id': 'metadata'}) - if panel: - buck = panel.find('p', attrs={'id': 'media-bucket'}) - if buck: - data = json.loads(buck.string) - buck.extract() - i_lst = [self.media_bucket(x) for x in data['items']] - m_itm = soup.findAll('panel', attrs={'class': 'media-item'}) - if i_lst and m_itm: - for x, y in list(zip_longest(m_itm, i_lst)): - x.insert_after(BeautifulSoup(y, 'html.parser')) - return soup - - def postprocess_html(self, soup, first_fetch): - for pan in soup.findAll('panel'): - pan.name = 'div' + if self.tag_to_string(h2).startswith(('What to Read Next', 'Conversation')): + h2.extract() + h2.name = 'h3' + h2['class'] = 'sub' + for ph in soup.findAll('a', attrs={'data-type': ['phrase', 'link']}): + if div := ph.findParent('div'): + div.name = 'span' + for auth in soup.findAll( + 'a', attrs={'aria-label': lambda x: x and x.startswith('Author page')} + ): + if div := auth.find_previous_sibling('div'): + div.name = 'span' + if parent := auth.findParent('div'): + parent['class'] = 'auth' + for x in soup.findAll('ufc-follow-author-widget'): + if y := x.findParent('div'): + y.extract() return soup def _download_cover(self): import os from contextlib import closing + from calibre import browser from calibre.utils.img import save_cover_data_to br = browser() @@ -259,29 +231,8 @@ class WSJ(BasicNewsRecipe): desc = mobi['description']['content']['text'] except TypeError: desc = '' - url = arts['id'] - self.log(' ', title, '\n\t', desc) + url = arts['content']['sourceUrl'] + self.log(' ', title, '\n\t', desc, '\n\t', url) articles.append({'title': title, 'description': desc, 'url': url}) feeds.append((section, articles)) return feeds - - def populate_article_metadata(self, article, soup, first): - lnk = soup.find('div', attrs={'id': 'share-link'}) - if lnk: - article.url = lnk['title'] - - def print_version(self, url): - # query = { - # 'operationName': 'ArticleContent', - # 'variables': '{{"id":"{}","idType":"originid"}}'.format(url), - # 'extensions': '{"persistedQuery":{"version":1,"sha256Hash":"9b0e21df0753a90766c892f57603a7dc1b4c6cb87a2612bdf825c76e4e41a2fb"}}', - # } - # art_url = 'https://shared-data.dowjones.io/gateway/graphql?' + urlencode( - # query, safe='()!', quote_via=quote - # ) - # art_raw = self.index_to_soup(art_url, raw=True) - art_cont = get_article(url) - pt = PersistentTemporaryFile('.html') - pt.write(art_cont) - pt.close() - return 'file:///' + pt.name