From 9e628896060439e590482bfc5f971ecb93d98630 Mon Sep 17 00:00:00 2001 From: unkn0w7n <51942695+unkn0w7n@users.noreply.github.com> Date: Fri, 20 Oct 2023 09:27:18 +0530 Subject: [PATCH] revert WSJ archive.is version no longer works --- recipes/wsj.recipe | 78 +++++++++++------------------------------ recipes/wsj_free.recipe | 77 +++++++++++----------------------------- 2 files changed, 41 insertions(+), 114 deletions(-) diff --git a/recipes/wsj.recipe b/recipes/wsj.recipe index a8bb4872cb..5c76fcc47c 100644 --- a/recipes/wsj.recipe +++ b/recipes/wsj.recipe @@ -10,8 +10,7 @@ from base64 import standard_b64encode from datetime import date, timedelta from mechanize import Request -from calibre.ptempfile import PersistentTemporaryFile -from calibre.web.feeds.news import BasicNewsRecipe +from calibre.web.feeds.news import BasicNewsRecipe, classes from css_selectors import Select try: @@ -24,7 +23,7 @@ except ImportError: from urllib import quote -needs_subscription = 'optional' +needs_subscription = True def substring_classes(classes): @@ -60,68 +59,33 @@ class WSJ(BasicNewsRecipe): needs_subscription = needs_subscription WSJ_ITP = 'https://www.wsj.com/print-edition/today' - storage = [] - extra_css = ''' - #big-top-caption { font-size:small; text-align:center; } - [data-type:"tagline"], em { font-style:italic; color:#202020; } - .auth { font-size:small; } + .imageCaption{font-size:small; text-align:center;} + .sub-head{font-style:italic; color:#404040;} + .bylineWrap{font-size:small; text-align:left;} ''' keep_only_tags = [ - dict(name=['h1', 'h2']), - dict(attrs={'aria-describedby':'big-top-caption'}), - dict(attrs={'id':'big-top-caption'}), - dict(name='article') + dict(attrs={'class': lambda x: x and 'HeadlineContainer' in ''.join(x)}), + dict(name='main'), ] remove_tags = [ - dict(name=['button', 'svg', 'ufc-follow-author-widget']), - dict(attrs={'aria-label':['Sponsored Offers', 'Listen To Article', 'What to Read Next']}), - dict(attrs={'data-type':'inset'}), - dict(attrs={'id':lambda x: x and x.startswith(('wrapper-INLINE', 'audio-tag-inner-audio-'))}) + classes( + 'wsj-ad newsletter-inset media-object-video media-object-podcast print-header article-body-tools' + ' podcast--iframe dynamic-inset-overflow-button snippet-logo'), + dict(role=["toolbar", "complementary"]), + dict(attrs={"aria-label": ["Sponsored Offers", "What to Read Next", "breadcrumbs", "Listen To Article"]}), + dict(name='amp-iframe'), # interactive graphics ] - articles_are_obfuscated = True - def get_obfuscated_article(self, url): - from calibre.scraper.simple import read_url - br = self.get_browser() - br.set_handle_redirect(False) - try: - br.open(url) - except Exception as e: - url = e.hdrs.get('location') - raw = read_url(self.storage, 'https://archive.is/latest/' + url) - pt = PersistentTemporaryFile('.html') - pt.write(raw.encode('utf-8')) - pt.close() - return pt.name - def preprocess_html(self, soup): - for img in soup.findAll('img', attrs={'old-src':True}): - img['src'] = img['old-src'] - for p in soup.findAll('div', attrs={'data-type':['paragraph', 'image']}): - p.name = 'p' - for a in soup.findAll('a', href=True): - a['href'] = 'http' + a['href'].split('http')[-1] - for fig in soup.findAll('figure'): - if fig.find('video'): - fig.extract() - for figc in soup.findAll('figcaption'): - figc['id'] = 'big-top-caption' - if name:= soup.find('h2', attrs={'itemprop':'name'}): - name.extract() - for h2 in soup.findAll('h2'): - if self.tag_to_string(h2).startswith('What to Read Next'): - h2.extract() - for ph in soup.findAll('a', attrs={'data-type':['phrase', 'link']}): - if div := ph.findParent('div'): - div.name = 'span' - for auth in soup.findAll('a', attrs={'aria-label': lambda x: x and x.startswith('Author page')}): - if div := auth.find_previous_sibling('div'): - div.name = 'span' - if parent := auth.findParent('div'): - parent['class'] = 'auth' + for by in soup.findAll(**classes('bylineWrap')): + for p in by.findAll('p'): + p.name = 'span' + h2 = soup.find('h2', attrs={'class':'sub-head'}) + if h2: + h2.name = 'p' return soup # login {{{ @@ -133,7 +97,7 @@ class WSJ(BasicNewsRecipe): br.set_cookie('ccpaApplies', 'false', '.wsj.com') return br - if False and needs_subscription: # disabled as we currently use archive.is + if needs_subscription: def get_browser(self, *a, **kw): from pprint import pprint pprint @@ -304,6 +268,6 @@ class WSJ(BasicNewsRecipe): return [ ('Testing', [ {'title': 'Subscriber Article', - 'url': self.abs_wsj_url('https://www.wsj.com/articles/remington-gun-call-of-duty-video-game-93059a66')}, + 'url': self.abs_wsj_url('https://www.wsj.com/articles/egg-prices-jump-as-bird-flu-hits-poultry-flocks-11648900800')}, ]), ] diff --git a/recipes/wsj_free.recipe b/recipes/wsj_free.recipe index 66f6eda334..cba4720ded 100644 --- a/recipes/wsj_free.recipe +++ b/recipes/wsj_free.recipe @@ -10,8 +10,7 @@ from base64 import standard_b64encode from datetime import date, timedelta from mechanize import Request -from calibre.ptempfile import PersistentTemporaryFile -from calibre.web.feeds.news import BasicNewsRecipe +from calibre.web.feeds.news import BasicNewsRecipe, classes from css_selectors import Select try: @@ -60,69 +59,33 @@ class WSJ(BasicNewsRecipe): needs_subscription = needs_subscription WSJ_ITP = 'https://www.wsj.com/print-edition/today' - storage = [] - extra_css = ''' - #big-top-caption { font-size:small; text-align:center; } - [data-type:"tagline"] { font-style:italic; color:#202020; } - [data-type:"tagline"], em { font-style:italic; color:#202020; } - .auth { font-size:small; } + .imageCaption{font-size:small; text-align:center;} + .sub-head{font-style:italic; color:#404040;} + .bylineWrap{font-size:small; text-align:left;} ''' keep_only_tags = [ - dict(name=['h1', 'h2']), - dict(attrs={'aria-describedby':'big-top-caption'}), - dict(attrs={'id':'big-top-caption'}), - dict(name='article') + dict(attrs={'class': lambda x: x and 'HeadlineContainer' in ''.join(x)}), + dict(name='main'), ] remove_tags = [ - dict(name=['button', 'svg', 'ufc-follow-author-widget']), - dict(attrs={'aria-label':['Sponsored Offers', 'Listen To Article', 'What to Read Next']}), - dict(attrs={'data-type':'inset'}), - dict(attrs={'id':lambda x: x and x.startswith(('wrapper-INLINE', 'audio-tag-inner-audio-'))}) + classes( + 'wsj-ad newsletter-inset media-object-video media-object-podcast print-header article-body-tools' + ' podcast--iframe dynamic-inset-overflow-button snippet-logo'), + dict(role=["toolbar", "complementary"]), + dict(attrs={"aria-label": ["Sponsored Offers", "What to Read Next", "breadcrumbs", "Listen To Article"]}), + dict(name='amp-iframe'), # interactive graphics ] - articles_are_obfuscated = True - def get_obfuscated_article(self, url): - from calibre.scraper.simple import read_url - br = self.get_browser() - br.set_handle_redirect(False) - try: - br.open(url) - except Exception as e: - url = e.hdrs.get('location') - raw = read_url(self.storage, 'https://archive.is/latest/' + url) - pt = PersistentTemporaryFile('.html') - pt.write(raw.encode('utf-8')) - pt.close() - return pt.name - def preprocess_html(self, soup): - for img in soup.findAll('img', attrs={'old-src':True}): - img['src'] = img['old-src'] - for p in soup.findAll('div', attrs={'data-type':['paragraph', 'image']}): - p.name = 'p' - for a in soup.findAll('a', href=True): - a['href'] = 'http' + a['href'].split('http')[-1] - for fig in soup.findAll('figure'): - if fig.find('video'): - fig.extract() - for figc in soup.findAll('figcaption'): - figc['id'] = 'big-top-caption' - if name:= soup.find('h2', attrs={'itemprop':'name'}): - name.extract() - for h2 in soup.findAll('h2'): - if self.tag_to_string(h2).startswith('What to Read Next'): - h2.extract() - for ph in soup.findAll('a', attrs={'data-type':['phrase', 'link']}): - if div := ph.findParent('div'): - div.name = 'span' - for auth in soup.findAll('a', attrs={'aria-label': lambda x: x and x.startswith('Author page')}): - if div := auth.find_previous_sibling('div'): - div.name = 'span' - if parent := auth.findParent('div'): - parent['class'] = 'auth' + for by in soup.findAll(**classes('bylineWrap')): + for p in by.findAll('p'): + p.name = 'span' + h2 = soup.find('h2', attrs={'class':'sub-head'}) + if h2: + h2.name = 'p' return soup # login {{{ @@ -134,7 +97,7 @@ class WSJ(BasicNewsRecipe): br.set_cookie('ccpaApplies', 'false', '.wsj.com') return br - if False and needs_subscription: # disabled as we currently use archive.is + if needs_subscription: def get_browser(self, *a, **kw): from pprint import pprint pprint @@ -305,6 +268,6 @@ class WSJ(BasicNewsRecipe): return [ ('Testing', [ {'title': 'Subscriber Article', - 'url': self.abs_wsj_url('https://www.wsj.com/articles/remington-gun-call-of-duty-video-game-93059a66')}, + 'url': self.abs_wsj_url('https://www.wsj.com/articles/egg-prices-jump-as-bird-flu-hits-poultry-flocks-11648900800')}, ]), ]