mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-07-09 03:04:10 -04:00
Removethumbnails instead of larger images in WSJ recipe
This commit is contained in:
Kovid Goyal
parent
5faeaba6c2
commit
f104f1e07d
@ -7,19 +7,19 @@ from calibre.web.feeds.news import BasicNewsRecipe
|
|||||||
|
|
||||||
# http://online.wsj.com/page/us_in_todays_paper.html
|
# http://online.wsj.com/page/us_in_todays_paper.html
|
||||||
|
|
||||||
class WallStreetJournal(BasicNewsRecipe):
|
class WallStreetJournal(BasicNewsRecipe):
|
||||||
|
|
||||||
title = 'The Wall Street Journal'
|
title = 'The Wall Street Journal'
|
||||||
__author__ = 'Kovid Goyal and Sujata Raman'
|
__author__ = 'Kovid Goyal'
|
||||||
description = 'News and current affairs.'
|
description = 'News and current affairs.'
|
||||||
needs_subscription = True
|
needs_subscription = True
|
||||||
language = _('English')
|
language = _('English')
|
||||||
max_articles_per_feed = 10
|
max_articles_per_feed = 10
|
||||||
timefmt = ' [%a, %b %d, %Y]'
|
timefmt = ' [%a, %b %d, %Y]'
|
||||||
no_stylesheets = True
|
no_stylesheets = True
|
||||||
|
|
||||||
extra_css = '''h1{color:#093D72 ; font-size:large ; font-family:Georgia,"Century Schoolbook","Times New Roman",Times,serif; }
|
extra_css = '''h1{color:#093D72 ; font-size:large ; font-family:Georgia,"Century Schoolbook","Times New Roman",Times,serif; }
|
||||||
h2{color:gray; font-family:Georgia,"Century Schoolbook","Times New Roman",Times,serif; font-size:small; font-style:italic;}
|
h2{color:#474537; font-family:Georgia,"Century Schoolbook","Times New Roman",Times,serif; font-size:small; font-style:italic;}
|
||||||
.subhead{color:gray; font-family:Georgia,"Century Schoolbook","Times New Roman",Times,serif; font-size:small; font-style:italic;}
|
.subhead{color:gray; font-family:Georgia,"Century Schoolbook","Times New Roman",Times,serif; font-size:small; font-style:italic;}
|
||||||
.insettipUnit {color:#666666; font-family:Arial,Sans-serif;font-size:xx-small }
|
.insettipUnit {color:#666666; font-family:Arial,Sans-serif;font-size:xx-small }
|
||||||
.targetCaption{ font-size:x-small; color:#333333; font-family:Arial,Helvetica,sans-serif}
|
.targetCaption{ font-size:x-small; color:#333333; font-family:Arial,Helvetica,sans-serif}
|
||||||
@ -30,89 +30,89 @@ class WallStreetJournal(BasicNewsRecipe):
|
|||||||
.byline{color:blue;font-family:Arial,Helvetica,sans-serif; font-size:xx-small}
|
.byline{color:blue;font-family:Arial,Helvetica,sans-serif; font-size:xx-small}
|
||||||
h6{color:#333333; font-family:Georgia,"Century Schoolbook","Times New Roman",Times,serif; font-size:small;font-style:italic; }
|
h6{color:#333333; font-family:Georgia,"Century Schoolbook","Times New Roman",Times,serif; font-size:small;font-style:italic; }
|
||||||
.paperLocation{color:#666666; font-size:xx-small}'''
|
.paperLocation{color:#666666; font-size:xx-small}'''
|
||||||
|
|
||||||
remove_tags_before = dict(name='h1')
|
remove_tags_before = dict(name='h1')
|
||||||
remove_tags = [
|
remove_tags = [
|
||||||
dict(id=["articleTabs_tab_article", "articleTabs_tab_comments", "articleTabs_tab_interactive","articleTabs_tab_video","articleTabs_tab_map","articleTabs_tab_slideshow"]),
|
dict(id=["articleTabs_tab_article", "articleTabs_tab_comments", "articleTabs_tab_interactive","articleTabs_tab_video","articleTabs_tab_map","articleTabs_tab_slideshow"]),
|
||||||
{'class':['footer_columns','network','insetCol3wide','interactive','video','slideshow','map','insettip','more_in', "insetContent", 'articleTools_bottom', 'aTools', "tooltip", "adSummary", "nav-inline"]},
|
{'class':['footer_columns','network','insetCol3wide','interactive','video','slideshow','map','insettip','insetClose','more_in', "insetContent", 'articleTools_bottom', 'aTools', "tooltip", "adSummary", "nav-inline"]},
|
||||||
dict(rel='shortcut icon'),
|
dict(rel='shortcut icon'),
|
||||||
]
|
]
|
||||||
remove_tags_after = [dict(id="article_story_body"), {'class':"article story"},]
|
remove_tags_after = [dict(id="article_story_body"), {'class':"article story"},]
|
||||||
|
|
||||||
|
|
||||||
def get_browser(self):
|
def get_browser(self):
|
||||||
br = BasicNewsRecipe.get_browser()
|
br = BasicNewsRecipe.get_browser()
|
||||||
if self.username is not None and self.password is not None:
|
if self.username is not None and self.password is not None:
|
||||||
br.open('http://commerce.wsj.com/auth/login')
|
br.open('http://commerce.wsj.com/auth/login')
|
||||||
br.select_form(nr=0)
|
br.select_form(nr=0)
|
||||||
br['user'] = self.username
|
br['user'] = self.username
|
||||||
br['password'] = self.password
|
br['password'] = self.password
|
||||||
br.submit()
|
br.submit()
|
||||||
return br
|
return br
|
||||||
|
|
||||||
def postprocess_html(self, soup, first):
|
def postprocess_html(self, soup, first):
|
||||||
for tag in soup.findAll(name=['table', 'tr', 'td']):
|
for tag in soup.findAll(name=['table', 'tr', 'td']):
|
||||||
tag.name = 'div'
|
tag.name = 'div'
|
||||||
|
|
||||||
for tag in soup.findAll('div', dict(id=["articleImage_1", "articleImage_2", "articleImage_3", "articleImage_4", "articleImage_5", "articleImage_6", "articleImage_7"])):
|
for tag in soup.findAll('div', dict(id=["articleThumbnail_1", "articleThumbnail_2", "articleThumbnail_3", "articleThumbnail_4", "articleThumbnail_5", "articleThumbnail_6", "articleThumbnail_7"])):
|
||||||
tag.extract()
|
tag.extract()
|
||||||
|
|
||||||
return soup
|
return soup
|
||||||
|
|
||||||
def get_article_url(self, article):
|
def get_article_url(self, article):
|
||||||
try:
|
try:
|
||||||
return article.feedburner_origlink.split('?')[0]
|
return article.feedburner_origlink.split('?')[0]
|
||||||
except AttributeError:
|
except AttributeError:
|
||||||
return article.link.split('?')[0]
|
return article.link.split('?')[0]
|
||||||
|
|
||||||
def cleanup(self):
|
def cleanup(self):
|
||||||
self.browser.open('http://online.wsj.com/logout?url=http://online.wsj.com')
|
self.browser.open('http://online.wsj.com/logout?url=http://online.wsj.com')
|
||||||
|
|
||||||
feeds = [
|
feeds = [
|
||||||
#('Most Emailed - Day', 'http://online.wsj.com/xml/rss/3_7030.xml'),
|
#('Most Emailed - Day', 'http://online.wsj.com/xml/rss/3_7030.xml'),
|
||||||
#('Most Emailed - Week', 'http://online.wsj.com/xml/rss/3_7253.xml'),
|
#('Most Emailed - Week', 'http://online.wsj.com/xml/rss/3_7253.xml'),
|
||||||
#('Most Emailed - Month', 'http://online.wsj.com/xml/rss/3_7254.xml'),
|
#('Most Emailed - Month', 'http://online.wsj.com/xml/rss/3_7254.xml'),
|
||||||
(' Most Viewed - Day', 'http://online.wsj.com/xml/rss/3_7198.xml'),
|
(' Most Viewed - Day', 'http://online.wsj.com/xml/rss/3_7198.xml'),
|
||||||
(' Most Viewed - Week', 'http://online.wsj.com/xml/rss/3_7251.xml'),
|
(' Most Viewed - Week', 'http://online.wsj.com/xml/rss/3_7251.xml'),
|
||||||
#('Most Viewed - Month', 'http://online.wsj.com/xml/rss/3_7252.xml'),
|
#('Most Viewed - Month', 'http://online.wsj.com/xml/rss/3_7252.xml'),
|
||||||
('Today\'s Newspaper - Page One', 'http://online.wsj.com/xml/rss/3_7205.xml'),
|
('Today\'s Newspaper - Page One', 'http://online.wsj.com/xml/rss/3_7205.xml'),
|
||||||
('Today\'s Newspaper - Marketplace', 'http://online.wsj.com/xml/rss/3_7206.xml'),
|
('Today\'s Newspaper - Marketplace', 'http://online.wsj.com/xml/rss/3_7206.xml'),
|
||||||
('Today\'s Newspaper - Money & Investing', 'http://online.wsj.com/xml/rss/3_7207.xml'),
|
('Today\'s Newspaper - Money & Investing', 'http://online.wsj.com/xml/rss/3_7207.xml'),
|
||||||
('Today\'s Newspaper - Personal Journal', 'http://online.wsj.com/xml/rss/3_7208.xml'),
|
('Today\'s Newspaper - Personal Journal', 'http://online.wsj.com/xml/rss/3_7208.xml'),
|
||||||
('Today\'s Newspaper - Weekend Journal', 'http://online.wsj.com/xml/rss/3_7209.xml'),
|
('Today\'s Newspaper - Weekend Journal', 'http://online.wsj.com/xml/rss/3_7209.xml'),
|
||||||
('Opinion', 'http://online.wsj.com/xml/rss/3_7041.xml'),
|
('Opinion', 'http://online.wsj.com/xml/rss/3_7041.xml'),
|
||||||
('News - U.S.: What\'s News', 'http://online.wsj.com/xml/rss/3_7011.xml'),
|
('News - U.S.: What\'s News', 'http://online.wsj.com/xml/rss/3_7011.xml'),
|
||||||
('News - U.S. Business', 'http://online.wsj.com/xml/rss/3_7014.xml'),
|
('News - U.S. Business', 'http://online.wsj.com/xml/rss/3_7014.xml'),
|
||||||
('News - Europe: What\'s News', 'http://online.wsj.com/xml/rss/3_7012.xml'),
|
('News - Europe: What\'s News', 'http://online.wsj.com/xml/rss/3_7012.xml'),
|
||||||
('News - Asia: What\'s News', 'http://online.wsj.com/xml/rss/3_7013.xml'),
|
('News - Asia: What\'s News', 'http://online.wsj.com/xml/rss/3_7013.xml'),
|
||||||
('News - World News', 'http://online.wsj.com/xml/rss/3_7085.xml'),
|
('News - World News', 'http://online.wsj.com/xml/rss/3_7085.xml'),
|
||||||
('News - Economy', 'http://online.wsj.com/xml/rss/3_7086.xml'),
|
('News - Economy', 'http://online.wsj.com/xml/rss/3_7086.xml'),
|
||||||
('News - Earnings', 'http://online.wsj.com/xml/rss/3_7088.xml'),
|
('News - Earnings', 'http://online.wsj.com/xml/rss/3_7088.xml'),
|
||||||
('News - Health', 'http://online.wsj.com/xml/rss/3_7089.xml'),
|
('News - Health', 'http://online.wsj.com/xml/rss/3_7089.xml'),
|
||||||
('News - Law', 'http://online.wsj.com/xml/rss/3_7091.xml'),
|
('News - Law', 'http://online.wsj.com/xml/rss/3_7091.xml'),
|
||||||
('News - Media & Marketing', 'http://online.wsj.com/xml/rss/3_7020.xml'),
|
('News - Media & Marketing', 'http://online.wsj.com/xml/rss/3_7020.xml'),
|
||||||
('Technology - What\'s News', 'http://online.wsj.com/xml/rss/3_7015.xml'),
|
('Technology - What\'s News', 'http://online.wsj.com/xml/rss/3_7015.xml'),
|
||||||
('Technology - Gadgets', 'http://online.wsj.com/xml/rss/3_7094.xml'),
|
('Technology - Gadgets', 'http://online.wsj.com/xml/rss/3_7094.xml'),
|
||||||
('Technology - Telecommunications', 'http://online.wsj.com/xml/rss/3_7095.xml'),
|
('Technology - Telecommunications', 'http://online.wsj.com/xml/rss/3_7095.xml'),
|
||||||
('Technology - E-commerce/Media', 'http://online.wsj.com/xml/rss/3_7096.xml'),
|
('Technology - E-commerce/Media', 'http://online.wsj.com/xml/rss/3_7096.xml'),
|
||||||
('Technology - Asia', 'http://online.wsj.com/xml/rss/3_7097.xml'),
|
('Technology - Asia', 'http://online.wsj.com/xml/rss/3_7097.xml'),
|
||||||
('Technology - Europe', 'http://online.wsj.com/xml/rss/3_7098.xml'),
|
('Technology - Europe', 'http://online.wsj.com/xml/rss/3_7098.xml'),
|
||||||
('Markets - News', 'http://online.wsj.com/xml/rss/3_7031.xml'),
|
('Markets - News', 'http://online.wsj.com/xml/rss/3_7031.xml'),
|
||||||
('Markets - Europe News', 'http://online.wsj.com/xml/rss/3_7101.xml'),
|
('Markets - Europe News', 'http://online.wsj.com/xml/rss/3_7101.xml'),
|
||||||
('Markets - Asia News', 'http://online.wsj.com/xml/rss/3_7102.xml'),
|
('Markets - Asia News', 'http://online.wsj.com/xml/rss/3_7102.xml'),
|
||||||
('Markets - Deals & Deal Makers', 'http://online.wsj.com/xml/rss/3_7099.xml'),
|
('Markets - Deals & Deal Makers', 'http://online.wsj.com/xml/rss/3_7099.xml'),
|
||||||
('Markets - Hedge Funds', 'http://online.wsj.com/xml/rss/3_7199.xml'),
|
('Markets - Hedge Funds', 'http://online.wsj.com/xml/rss/3_7199.xml'),
|
||||||
('Personal Journal', 'http://online.wsj.com/xml/rss/3_7200.xml'),
|
('Personal Journal', 'http://online.wsj.com/xml/rss/3_7200.xml'),
|
||||||
('Personal Journal - Money', 'http://online.wsj.com/xml/rss/3_7104.xml'),
|
('Personal Journal - Money', 'http://online.wsj.com/xml/rss/3_7104.xml'),
|
||||||
('Personal Journal - Health', 'http://online.wsj.com/xml/rss/3_7089.xml'),
|
('Personal Journal - Health', 'http://online.wsj.com/xml/rss/3_7089.xml'),
|
||||||
('Personal Journal - Autos', 'http://online.wsj.com/xml/rss/3_7092.xml'),
|
('Personal Journal - Autos', 'http://online.wsj.com/xml/rss/3_7092.xml'),
|
||||||
('Personal Journal - Homes', 'http://online.wsj.com/xml/rss/3_7105.xml'),
|
('Personal Journal - Homes', 'http://online.wsj.com/xml/rss/3_7105.xml'),
|
||||||
('Personal Journal - Travel', 'http://online.wsj.com/xml/rss/3_7106.xml'),
|
('Personal Journal - Travel', 'http://online.wsj.com/xml/rss/3_7106.xml'),
|
||||||
('Personal Journal - Careers', 'http://online.wsj.com/xml/rss/3_7107.xml'),
|
('Personal Journal - Careers', 'http://online.wsj.com/xml/rss/3_7107.xml'),
|
||||||
('Weekend & Leisure', 'http://online.wsj.com/xml/rss/3_7201.xml'),
|
('Weekend & Leisure', 'http://online.wsj.com/xml/rss/3_7201.xml'),
|
||||||
('Weekend & Leisure - Weekend Journal', 'http://online.wsj.com/xml/rss/3_7202.xml'),
|
('Weekend & Leisure - Weekend Journal', 'http://online.wsj.com/xml/rss/3_7202.xml'),
|
||||||
('Weekend & Leisure - Arts & Entertainment', 'http://online.wsj.com/xml/rss/3_7177.xml'),
|
('Weekend & Leisure - Arts & Entertainment', 'http://online.wsj.com/xml/rss/3_7177.xml'),
|
||||||
('Weekend & Leisure - Books', 'http://online.wsj.com/xml/rss/3_7203.xml'),
|
('Weekend & Leisure - Books', 'http://online.wsj.com/xml/rss/3_7203.xml'),
|
||||||
('Weekend & Leisure - Sports', 'http://online.wsj.com/xml/rss/3_7204.xml'),
|
('Weekend & Leisure - Sports', 'http://online.wsj.com/xml/rss/3_7204.xml'),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user