Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Update The Australian

Browse Source
This commit is contained in:
Kovid Goyal 2014-12-14 09:55:20 +05:30
parent 8aaf03ef28
commit 2157b1dcf4
1 changed files with 39 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
recipes/the_oz.recipe
View File

@ -7,11 +7,13 @@ __docformat__ = 'restructuredtext en'
http://www.theaustralian.news.com.au/ http://www.theaustralian.news.com.au/
''' '''
from calibre.web.feeds.news import BasicNewsRecipe from calibre import browser
from calibre.web.feeds.jsnews import JavascriptRecipe
from calibre.web.feeds import feed_from_xml
class DailyTelegraph(BasicNewsRecipe): class DailyTelegraph(JavascriptRecipe):
title = u'The Australian' title = u'The Australian'
__author__ = u'Matthew Briggs and Sujata Raman' __author__ = u'Kovid Goyal'
description = (u'National broadsheet newspaper from down under - colloquially known as The Oz' description = (u'National broadsheet newspaper from down under - colloquially known as The Oz'
'. You will need to have a subscription to ' '. You will need to have a subscription to '
'http://www.theaustralian.com.au to get full articles.') 'http://www.theaustralian.com.au to get full articles.')
@ -23,27 +25,16 @@ class DailyTelegraph(BasicNewsRecipe):
remove_javascript = True remove_javascript = True
no_stylesheets = True no_stylesheets = True
encoding = 'utf8' encoding = 'utf8'
remove_empty_feeds = True
ignore_duplicate_articles = {'url'}
keep_only_tags = [dict(name='div', attrs={'id': 'story'})] keep_only_tags = ['div#story']
remove_tags = [
# remove_tags = [dict(name=['object','link'])] '.story-info', '.story-header-tools', '.module-controls', '.story-sidebar',
remove_tags = [dict(name='div', attrs={'class': 'story-info'}), '.story-footer', '#comments', '.story-extras', '.story-related', '.vms-nav',
dict(name='div', attrs={'class': 'story-header-tools'}), '.vms-endcard', '.vms-discover', '.share-tools', '.story-comments-link',
dict(name='div', attrs={'class': 'story-sidebar'}), '.vms-controls', '.ooyala-player', '.vms-countdown', '.vms-header', '.comments',
dict(name='div', attrs={'class': 'story-footer'}), ]
dict(name='div', attrs={'id': 'comments'}),
dict(name='div', attrs={'class': 'story-extras story-extras-2'}),
dict(name='div', attrs={'class': 'group item-count-1 story-related'})
]
extra_css = '''
h1{font-family :Georgia,"Times New Roman",Times,serif; font-size:large; }
#article{font-family :Georgia,"Times New Roman",Times,serif; font-size: x-small;}
.module-subheader{font-family :Tahoma,Geneva,Arial,Helvetica,sans-serif; color:#666666; font-size: xx-small;}
.intro{ font-family:Trebuchet MS,Trebuchet,Helvetica,sans-serif;font-size: x-small; }
.article-source{font-family:Trebuchet MS,Trebuchet,Helvetica,sans-serif; color:#666666; font-size: xx-small;}
.caption{font-family:Trebuchet MS,Trebuchet,Helvetica,sans-serif; font-size: xx-small;}
'''
feeds = [ feeds = [
(u'News', u'http://feeds.news.com.au/public/rss/2.0/aus_news_807.xml'), (u'News', u'http://feeds.news.com.au/public/rss/2.0/aus_news_807.xml'),
@ -63,29 +54,34 @@ class DailyTelegraph(BasicNewsRecipe):
(u'Business', u'http://feeds.news.com.au/public/rss/2.0/aus_business_811.xml'), (u'Business', u'http://feeds.news.com.au/public/rss/2.0/aus_business_811.xml'),
(u'Aviation', u'http://feeds.news.com.au/public/rss/2.0/aus_business_aviation_706.xml'), (u'Aviation', u'http://feeds.news.com.au/public/rss/2.0/aus_business_aviation_706.xml'),
(u'Commercial Property', u'http://feeds.news.com.au/public/rss/2.0/aus_business_commercial_property_708.xml'), (u'Commercial Property', u'http://feeds.news.com.au/public/rss/2.0/aus_business_commercial_property_708.xml'),
(u'Mining', u'http://feeds.news.com.au/public/rss/2.0/aus_business_mining_704.xml')] (u'Mining', u'http://feeds.news.com.au/public/rss/2.0/aus_business_mining_704.xml')
]
def get_browser(self): def get_publication_data(self, br):
br = BasicNewsRecipe.get_browser(self) br = browser()
if self.username and self.password: ans = {}
br.open('http://www.theaustralian.com.au') feeds = ans['index'] = []
br.select_form(nr=1) for title, url in self.feeds:
br['username'] = self.username raw = br.open_novisit(url).read()
br['password'] = self.password self.log('Fetching feed: %s' % title)
raw = br.submit().read() feed = feed_from_xml(raw, title=title, log=self.log,
if '>log out' not in raw.lower(): oldest_article=self.oldest_article, max_articles_per_feed=self.max_articles_per_feed, get_article_url=self.get_article_url)
raise ValueError('Failed to log in to www.theaustralian.com.au' if len(feed) > 0:
' are your username and password correct?') feeds.append((title, [
return br {'title':a.title, 'url':a.url, 'description':a.text_summary} for a in feed.articles]))
return ans
def do_login(self, browser, username, password):
if username and password:
browser.visit('http://www.theaustralian.com.au/login')
form = browser.select_form('form[action="https://idp.news.com.au/idp/Authn/rest"]')
form['username'] = username
form['password'] = password
browser.submit(submit_control_selector='button[type="submit"]', timeout=60)
if '>Log Out' not in browser.html:
raise ValueError('Failed to log in, check your username and password')
def get_article_url(self, article): def get_article_url(self, article):
return article.id return article.id
# br = self.get_browser()
# br.open(article.link).read()
# print br.geturl()
# return br.geturl()