From aae0ca3b2c070be51dadb8a296316479a6e26af0 Mon Sep 17 00:00:00 2001 From: unkn0w7n <51942695+unkn0w7n@users.noreply.github.com> Date: Wed, 14 Aug 2024 18:26:14 +0530 Subject: [PATCH] Update WSJ add options for hi-res images --- recipes/wsj.recipe | 46 +++++++++++++++++++++++--------------- recipes/wsj_mag.recipe | 49 ++++++++++++++++++++++++++--------------- recipes/wsj_news.recipe | 46 +++++++++++++++++++++++--------------- 3 files changed, 87 insertions(+), 54 deletions(-) diff --git a/recipes/wsj.recipe b/recipes/wsj.recipe index 1476e15a29..6e14e4e353 100644 --- a/recipes/wsj.recipe +++ b/recipes/wsj.recipe @@ -9,21 +9,6 @@ from calibre.ebooks.BeautifulSoup import BeautifulSoup from calibre.web.feeds.news import BasicNewsRecipe, classes -def media_bucket(x): - if x.get('type', '') == 'image': - if x.get('subtype', '') == 'graphic' or 'images.wsj.net' not in x['manifest-url']: - return '
{}
\n'.format( - x['manifest-url'], x['caption'] + ' ' + x['credit'] + '' - ) - return '
{}
\n'.format( - x['manifest-url'].split('?')[0] + '?width=600', x['caption'] + ' ' + x['credit'] + '' - ) - if x.get('type', '') == 'video': - return '
{}
\n'.format( - x['share_link'], x['thumbnail_url'].split('?')[0] + '?width=600', x['caption'] + ' ' + x['credit'] + '' - ) - return - class WSJ(BasicNewsRecipe): title = 'The Wall Street Journal' __author__ = 'unkn0wn' @@ -43,6 +28,11 @@ class WSJ(BasicNewsRecipe): 'date': { 'short': 'The date of the edition to download (YYYYMMDD format)\nOnly the past 6 editions will be available ', 'long': 'For example, 20240513' + }, + 'res': { + 'short': 'For hi-res images, select a resolution from the\nfollowing options: 800, 1000, 1200 or 1500', + 'long': 'This is useful for non e-ink devices, and for a lower file size\nthan the default, use 400 or 300.', + 'default': '600' } } @@ -69,6 +59,25 @@ class WSJ(BasicNewsRecipe): dict(name='p', attrs={'id':'orig-pubdate-string'}) ] + def media_bucket(self, x): + res = '?width=600' + w = self.recipe_specific_options.get('res') + if w and isinstance(w, str): + res = '?width=' + w + if x.get('type', '') == 'image': + if x.get('subtype', '') == 'graphic' or 'images.wsj.net' not in x['manifest-url']: + return '
{}
\n'.format( + x['manifest-url'], x['caption'] + ' ' + x['credit'] + '' + ) + return '
{}
\n'.format( + x['manifest-url'].split('?')[0] + res, x['caption'] + ' ' + x['credit'] + '' + ) + if x.get('type', '') == 'video': + return '
{}
\n'.format( + x['share_link'], x['thumbnail_url'].split('?')[0] + res, x['caption'] + ' ' + x['credit'] + '' + ) + return + def preprocess_html(self, soup): jpml = soup.find('jpml') if jpml: @@ -81,8 +90,9 @@ class WSJ(BasicNewsRecipe): dt = soup.find('p', attrs={'id':'orig-pubdate-string'}) read = soup.find('p', attrs={'id':'time-to-read'}) byl = soup.find('p', attrs={'id':'byline'}) - if dt and byl and read: - dt.name = read.name = byl.name = 'div' + fl = soup.find('p', attrs={'id':'flashline'}) + if dt and byl and read and fl: + dt.name = read.name = byl.name = fl.name = 'div' byl.insert(0, dt) byl.insert(0, read) url = soup.find('p', attrs={'id':'share-link'}) @@ -96,7 +106,7 @@ class WSJ(BasicNewsRecipe): if buck: data = json.loads(buck.string) buck.extract() - i_lst = [media_bucket(x) for x in data['items']] + i_lst = [self.media_bucket(x) for x in data['items']] m_itm = soup.findAll('panel', attrs={'class':'media-item'}) if i_lst and m_itm: for x, y in list(zip_longest(m_itm, i_lst)): diff --git a/recipes/wsj_mag.recipe b/recipes/wsj_mag.recipe index 890f04a2d8..cae5fc0b68 100644 --- a/recipes/wsj_mag.recipe +++ b/recipes/wsj_mag.recipe @@ -9,21 +9,6 @@ from calibre.ebooks.BeautifulSoup import BeautifulSoup from calibre.web.feeds.news import BasicNewsRecipe, classes -def media_bucket(x): - if x.get('type', '') == 'image': - if x.get('subtype', '') == 'graphic': - return '
{}
\n'.format( - x['manifest-url'], x['caption'] + ' ' + x['credit'] + '' - ) - return '
{}
\n'.format( - x['manifest-url'].split('?')[0] + '?width=600', x['caption'] + ' ' + x['credit'] + '' - ) - if x.get('type', '') == 'video': - return '
{}
\n'.format( - x['share_link'], x['thumbnail_url'].split('?')[0] + '?width=600', x['caption'] + ' ' + x['credit'] + '' - ) - return - class WSJ(BasicNewsRecipe): title = 'WSJ. Magazine' __author__ = 'unkn0wn' @@ -39,6 +24,14 @@ class WSJ(BasicNewsRecipe): remove_attributes = ['style', 'height', 'width'] resolve_internal_links = True + recipe_specific_options = { + 'res': { + 'short': 'For hi-res images, select a resolution from the\nfollowing options: 800, 1000, 1200 or 1500', + 'long': 'This is useful for non e-ink devices, and for a lower file size\nthan the default, use 400 or 300.', + 'default': '600' + } + } + extra_css = ''' #subhed, em { font-style:italic; color:#202020; } #byline, #time-to-read, #orig-pubdate-string, .article-byline, time, #flashline { font-size:small; } @@ -62,6 +55,25 @@ class WSJ(BasicNewsRecipe): dict(name='p', attrs={'id':'orig-pubdate-string'}) ] + def media_bucket(self, x): + res = '?width=600' + w = self.recipe_specific_options.get('res') + if w and isinstance(w, str): + res = '?width=' + w + if x.get('type', '') == 'image': + if x.get('subtype', '') == 'graphic' or 'images.wsj.net' not in x['manifest-url']: + return '
{}
\n'.format( + x['manifest-url'], x['caption'] + ' ' + x['credit'] + '' + ) + return '
{}
\n'.format( + x['manifest-url'].split('?')[0] + res, x['caption'] + ' ' + x['credit'] + '' + ) + if x.get('type', '') == 'video': + return '
{}
\n'.format( + x['share_link'], x['thumbnail_url'].split('?')[0] + res, x['caption'] + ' ' + x['credit'] + '' + ) + return + def preprocess_html(self, soup): jpml = soup.find('jpml') if jpml: @@ -74,8 +86,9 @@ class WSJ(BasicNewsRecipe): dt = soup.find('p', attrs={'id':'orig-pubdate-string'}) read = soup.find('p', attrs={'id':'time-to-read'}) byl = soup.find('p', attrs={'id':'byline'}) - if dt and byl and read: - dt.name = read.name = byl.name = 'div' + fl = soup.find('p', attrs={'id':'flashline'}) + if dt and byl and read and fl: + dt.name = read.name = byl.name = fl.name = 'div' byl.insert(0, dt) byl.insert(0, read) url = soup.find('p', attrs={'id':'share-link'}) @@ -88,7 +101,7 @@ class WSJ(BasicNewsRecipe): if buck: data = json.loads(buck.string) buck.extract() - i_lst = [media_bucket(x) for x in data['items']] + i_lst = [self.media_bucket(x) for x in data['items']] m_itm = soup.findAll('panel', attrs={'class':'media-item'}) if i_lst and m_itm: for x, y in list(zip_longest(m_itm, i_lst)): diff --git a/recipes/wsj_news.recipe b/recipes/wsj_news.recipe index d79f405b13..4d0c53a9fb 100644 --- a/recipes/wsj_news.recipe +++ b/recipes/wsj_news.recipe @@ -9,21 +9,6 @@ from calibre.ebooks.BeautifulSoup import BeautifulSoup from calibre.web.feeds.news import BasicNewsRecipe, classes -def media_bucket(x): - if x.get('type', '') == 'image': - if x.get('subtype', '') == 'graphic' or 'images.wsj.net' not in x['manifest-url']: - return '
{}
\n'.format( - x['manifest-url'], x['caption'] + ' ' + x['credit'] + '' - ) - return '
{}
\n'.format( - x['manifest-url'].split('?')[0] + '?width=600', x['caption'] + ' ' + x['credit'] + '' - ) - if x.get('type', '') == 'video': - return '
{}
\n'.format( - x['share_link'], x['thumbnail_url'].split('?')[0] + '?width=600', x['caption'] + ' ' + x['credit'] + '' - ) - return - class WSJ(BasicNewsRecipe): title = 'WSJ News' __author__ = 'unkn0wn' @@ -47,6 +32,11 @@ class WSJ(BasicNewsRecipe): 'short': 'Oldest article to download from this news source. In days ', 'long': 'For example, 0.5, gives you articles from the past 12 hours', 'default': str(oldest_article) + }, + 'res': { + 'short': 'For hi-res images, select a resolution from the\nfollowing options: 800, 1000, 1200 or 1500', + 'long': 'This is useful for non e-ink devices, and for a lower file size\nthan the default, use 400 or 300.', + 'default': '600' } } @@ -79,6 +69,25 @@ class WSJ(BasicNewsRecipe): dict(name='p', attrs={'id':'orig-pubdate-string'}) ] + def media_bucket(self, x): + res = '?width=600' + w = self.recipe_specific_options.get('res') + if w and isinstance(w, str): + res = '?width=' + w + if x.get('type', '') == 'image': + if x.get('subtype', '') == 'graphic' or 'images.wsj.net' not in x['manifest-url']: + return '
{}
\n'.format( + x['manifest-url'], x['caption'] + ' ' + x['credit'] + '' + ) + return '
{}
\n'.format( + x['manifest-url'].split('?')[0] + res, x['caption'] + ' ' + x['credit'] + '' + ) + if x.get('type', '') == 'video': + return '
{}
\n'.format( + x['share_link'], x['thumbnail_url'].split('?')[0] + res, x['caption'] + ' ' + x['credit'] + '' + ) + return + def preprocess_html(self, soup): jpml = soup.find('jpml') if jpml: @@ -91,8 +100,9 @@ class WSJ(BasicNewsRecipe): dt = soup.find('p', attrs={'id':'orig-pubdate-string'}) read = soup.find('p', attrs={'id':'time-to-read'}) byl = soup.find('p', attrs={'id':'byline'}) - if dt and byl and read: - dt.name = read.name = byl.name = 'div' + fl = soup.find('p', attrs={'id':'flashline'}) + if dt and byl and read and fl: + dt.name = read.name = byl.name = fl.name = 'div' byl.insert(0, dt) byl.insert(0, read) url = soup.find('p', attrs={'id':'share-link'}) @@ -106,7 +116,7 @@ class WSJ(BasicNewsRecipe): if buck: data = json.loads(buck.string) buck.extract() - i_lst = [media_bucket(x) for x in data['items']] + i_lst = [self.media_bucket(x) for x in data['items']] m_itm = soup.findAll('panel', attrs={'class':'media-item'}) if i_lst and m_itm: for x, y in list(zip_longest(m_itm, i_lst)):